From 475bcbf71940f4d7f91cc309552d9ad76bff8aa7 Mon Sep 17 00:00:00 2001 From: Piotr Adamczyk Date: Wed, 20 Jan 2021 14:49:10 +0100 Subject: [PATCH 01/21] style: Add Kotlin Lint to tests --- build.gradle.kts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 1dc09f2324..268f8b338a 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -21,9 +21,7 @@ plugins { tasks { "lintKotlinMain"(LintTask::class) { exclude( - "**/*Generated.kt", - "**/*Test.kt", - "**/Test*.kt" // we can expand this list + "**/*Generated.kt" // we can expand this list ) } } From de08709a153d249396309194cd46797ddcfcb9b9 Mon Sep 17 00:00:00 2001 From: Adam Date: Tue, 12 Jan 2021 10:21:30 +0100 Subject: [PATCH 02/21] Added support for detecting changes in configs, added mixpanel --- buildSrc/src/main/kotlin/Dependencies.kt | 1 + buildSrc/src/main/kotlin/Versions.kt | 3 + test_runner/build.gradle.kts | 1 + .../src/main/kotlin/ftl/analytics/Mixpanel.kt | 71 +++++++++++++++++++ .../test/android/AndroidRunCommand.kt | 2 + .../cli/firebase/test/ios/IosRunCommand.kt | 2 + 6 files changed, 80 insertions(+) create mode 100644 test_runner/src/main/kotlin/ftl/analytics/Mixpanel.kt diff --git a/buildSrc/src/main/kotlin/Dependencies.kt b/buildSrc/src/main/kotlin/Dependencies.kt index dda33ce66e..590a7bcc42 100644 --- a/buildSrc/src/main/kotlin/Dependencies.kt +++ b/buildSrc/src/main/kotlin/Dependencies.kt @@ -1,5 +1,6 @@ object Dependencies { const val SENTRY = "io.sentry:sentry:${Versions.SENTRY}" + const val MIXPANEL = "com.mixpanel:mixpanel-java:${Versions.MIXPANEL}" const val DD_PLIST = "com.googlecode.plist:dd-plist:${Versions.DD_PLIST}" const val DEX_TEST_PARSER = "com.linkedin.dextestparser:parser:${Versions.DEX_TEST_PARSER}" diff --git a/buildSrc/src/main/kotlin/Versions.kt b/buildSrc/src/main/kotlin/Versions.kt index 94eb63e905..e832776ce4 100644 --- a/buildSrc/src/main/kotlin/Versions.kt +++ b/buildSrc/src/main/kotlin/Versions.kt @@ -2,6 +2,9 @@ object Versions { // https://github.com/getsentry/sentry-java/releases const val SENTRY = "3.2.0" + //https://github.com/mixpanel/mixpanel-java/releases + const val MIXPANEL = "1.5.0" + // https://github.com/3breadt/dd-plist/releases const val DD_PLIST = "1.23" diff --git a/test_runner/build.gradle.kts b/test_runner/build.gradle.kts index 23b15cb663..46661265d7 100644 --- a/test_runner/build.gradle.kts +++ b/test_runner/build.gradle.kts @@ -186,6 +186,7 @@ tasks.withType { dependencies { implementation(project(":common")) implementation(Dependencies.SENTRY) + implementation(Dependencies.MIXPANEL) implementation(Dependencies.DD_PLIST) implementation(Dependencies.DEX_TEST_PARSER) diff --git a/test_runner/src/main/kotlin/ftl/analytics/Mixpanel.kt b/test_runner/src/main/kotlin/ftl/analytics/Mixpanel.kt new file mode 100644 index 0000000000..70dd49ab16 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/analytics/Mixpanel.kt @@ -0,0 +1,71 @@ +package ftl.analytics + +import com.fasterxml.jackson.core.type.TypeReference +import com.fasterxml.jackson.module.kotlin.jsonMapper +import com.fasterxml.jackson.module.kotlin.kotlinModule +import com.mixpanel.mixpanelapi.MessageBuilder +import com.mixpanel.mixpanelapi.MixpanelAPI +import ftl.args.AndroidArgs +import ftl.args.IosArgs +import org.json.JSONObject + +private const val MIXPANEL_API_TOKEN = "f7490466a203188ecf591b9e7b0ae19d" +private const val CONFIGURATION_KEY = "configuration" + +private val messageBuilder by lazy { + MessageBuilder(MIXPANEL_API_TOKEN) +} + +private val apiClient by lazy { + MixpanelAPI() +} + +private val objectMapper by lazy { + jsonMapper { + addModule(kotlinModule()) + } +} + +fun registerUser(projectId: String) { + messageBuilder.set( + projectId, + JSONObject("PROJECT_ID" to projectId) + ).sendMessage() +} + +private fun JSONObject.sendMessage() = apiClient.sendMessage(this) + +fun AndroidArgs.sendConfiguration() { + val defaultArgs = AndroidArgs.default() + val defaultArgsMap = defaultArgs.objectToMap() + val defaultCommonArgs = defaultArgs.commonArgs.objectToMap() + + objectToMap().filter { it.key != "commonArgs" }.getNonDefaultArgs(defaultArgsMap) + .plus(commonArgs.objectToMap().getNonDefaultArgs(defaultCommonArgs)) + .let { + messageBuilder.event(project, CONFIGURATION_KEY, JSONObject(it)) + } +} + +fun IosArgs.sendConfiguration() { + val defaultArgs = IosArgs.default() + val defaultArgsMap = defaultArgs.objectToMap() + val defaultCommonArgs = defaultArgs.commonArgs.objectToMap() + + objectToMap().filter { it.key != "commonArgs" }.getNonDefaultArgs(defaultArgsMap) + .plus(commonArgs.objectToMap().getNonDefaultArgs(defaultCommonArgs)) + .let { + messageBuilder.event(project, CONFIGURATION_KEY, JSONObject(it)) + } +} + +private fun Map.getNonDefaultArgs(defaultArgs: Map) = + keys.fold(mapOf()) { acc, key -> + acc.compareValues(key, this, defaultArgs[key]) + } + +private fun Map.compareValues(key: String, source: Map, defaultValue: Any?) = + if (source[key] != defaultValue) this + (key to source[key]) + else this + +private fun Any.objectToMap() = objectMapper.convertValue(this, object : TypeReference>() {}) diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt index f5d893ba56..b8bd77c173 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/android/AndroidRunCommand.kt @@ -1,6 +1,7 @@ package ftl.cli.firebase.test.android import flank.common.logLn +import ftl.analytics.sendConfiguration import ftl.args.AndroidArgs import ftl.args.setupLogLevel import ftl.args.validate @@ -65,6 +66,7 @@ class AndroidRunCommand : CommonRunCommand(), Runnable { DEVICE_SYSTEM to "android", TEST_TYPE to type?.name.orEmpty() ) + sendConfiguration() }.validate().run { runBlocking { if (dumpShards) dumpShards() diff --git a/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt b/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt index 0237de6084..d31139f1e1 100644 --- a/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt +++ b/test_runner/src/main/kotlin/ftl/cli/firebase/test/ios/IosRunCommand.kt @@ -1,6 +1,7 @@ package ftl.cli.firebase.test.ios import flank.common.logLn +import ftl.analytics.sendConfiguration import ftl.args.IosArgs import ftl.args.setupLogLevel import ftl.args.validate @@ -65,6 +66,7 @@ class IosRunCommand : CommonRunCommand(), Runnable { DEVICE_SYSTEM to "ios", TEST_TYPE to type?.name.orEmpty() ) + sendConfiguration() }.validate().run { if (dumpShards) dumpShards() else runBlocking { newTestRun() } From 9b76a593ffaab572e094ba4751475858e7c0c5dc Mon Sep 17 00:00:00 2001 From: Adam Date: Wed, 13 Jan 2021 23:23:07 +0100 Subject: [PATCH 03/21] Update Mixpanel.kt --- .../src/main/kotlin/ftl/analytics/Mixpanel.kt | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/test_runner/src/main/kotlin/ftl/analytics/Mixpanel.kt b/test_runner/src/main/kotlin/ftl/analytics/Mixpanel.kt index 70dd49ab16..7ecde4fa7c 100644 --- a/test_runner/src/main/kotlin/ftl/analytics/Mixpanel.kt +++ b/test_runner/src/main/kotlin/ftl/analytics/Mixpanel.kt @@ -6,11 +6,14 @@ import com.fasterxml.jackson.module.kotlin.kotlinModule import com.mixpanel.mixpanelapi.MessageBuilder import com.mixpanel.mixpanelapi.MixpanelAPI import ftl.args.AndroidArgs +import ftl.args.IArgs import ftl.args.IosArgs import org.json.JSONObject private const val MIXPANEL_API_TOKEN = "f7490466a203188ecf591b9e7b0ae19d" private const val CONFIGURATION_KEY = "configuration" +private const val PROJECT_ID = "project_id" +private const val COMMON_ARGS = "commonArgs" private val messageBuilder by lazy { MessageBuilder(MIXPANEL_API_TOKEN) @@ -26,33 +29,44 @@ private val objectMapper by lazy { } } -fun registerUser(projectId: String) { +private fun IArgs.registerUser() { messageBuilder.set( - projectId, - JSONObject("PROJECT_ID" to projectId) + project, + JSONObject( + mapOf( + PROJECT_ID to project, + "name" to project + ) + ) ).sendMessage() } private fun JSONObject.sendMessage() = apiClient.sendMessage(this) fun AndroidArgs.sendConfiguration() { + registerUser() val defaultArgs = AndroidArgs.default() val defaultArgsMap = defaultArgs.objectToMap() val defaultCommonArgs = defaultArgs.commonArgs.objectToMap() - objectToMap().filter { it.key != "commonArgs" }.getNonDefaultArgs(defaultArgsMap) + objectToMap().filter { it.key != COMMON_ARGS }.getNonDefaultArgs(defaultArgsMap) .plus(commonArgs.objectToMap().getNonDefaultArgs(defaultCommonArgs)) .let { - messageBuilder.event(project, CONFIGURATION_KEY, JSONObject(it)) + messageBuilder.event( + project, + CONFIGURATION_KEY, + JSONObject(it) + ).sendMessage() } } fun IosArgs.sendConfiguration() { + registerUser() val defaultArgs = IosArgs.default() val defaultArgsMap = defaultArgs.objectToMap() val defaultCommonArgs = defaultArgs.commonArgs.objectToMap() - objectToMap().filter { it.key != "commonArgs" }.getNonDefaultArgs(defaultArgsMap) + objectToMap().filter { it.key != COMMON_ARGS }.getNonDefaultArgs(defaultArgsMap) .plus(commonArgs.objectToMap().getNonDefaultArgs(defaultCommonArgs)) .let { messageBuilder.event(project, CONFIGURATION_KEY, JSONObject(it)) From dd80e71f221d2b98b8365fd7192ba9f9038a1f1b Mon Sep 17 00:00:00 2001 From: Adam Date: Wed, 13 Jan 2021 23:23:41 +0100 Subject: [PATCH 04/21] Update Mixpanel.kt --- test_runner/src/main/kotlin/ftl/analytics/Mixpanel.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test_runner/src/main/kotlin/ftl/analytics/Mixpanel.kt b/test_runner/src/main/kotlin/ftl/analytics/Mixpanel.kt index 7ecde4fa7c..c79dbe1afe 100644 --- a/test_runner/src/main/kotlin/ftl/analytics/Mixpanel.kt +++ b/test_runner/src/main/kotlin/ftl/analytics/Mixpanel.kt @@ -14,6 +14,7 @@ private const val MIXPANEL_API_TOKEN = "f7490466a203188ecf591b9e7b0ae19d" private const val CONFIGURATION_KEY = "configuration" private const val PROJECT_ID = "project_id" private const val COMMON_ARGS = "commonArgs" +private const val NAME_KEY = "name" private val messageBuilder by lazy { MessageBuilder(MIXPANEL_API_TOKEN) @@ -35,7 +36,7 @@ private fun IArgs.registerUser() { JSONObject( mapOf( PROJECT_ID to project, - "name" to project + NAME_KEY to project ) ) ).sendMessage() From f1d64749655e634f7494bbd3e244348cd571df0e Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 14 Jan 2021 00:12:58 +0100 Subject: [PATCH 05/21] Update Mixpanel.kt --- .../src/main/kotlin/ftl/analytics/Mixpanel.kt | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/test_runner/src/main/kotlin/ftl/analytics/Mixpanel.kt b/test_runner/src/main/kotlin/ftl/analytics/Mixpanel.kt index c79dbe1afe..d539fc343a 100644 --- a/test_runner/src/main/kotlin/ftl/analytics/Mixpanel.kt +++ b/test_runner/src/main/kotlin/ftl/analytics/Mixpanel.kt @@ -53,11 +53,10 @@ fun AndroidArgs.sendConfiguration() { objectToMap().filter { it.key != COMMON_ARGS }.getNonDefaultArgs(defaultArgsMap) .plus(commonArgs.objectToMap().getNonDefaultArgs(defaultCommonArgs)) .let { - messageBuilder.event( - project, - CONFIGURATION_KEY, - JSONObject(it) - ).sendMessage() + it.forEach { + messageBuilder.event(project, + it.key,JSONObject(mapOf(it.key to it.value))).sendMessage() + } } } @@ -70,7 +69,10 @@ fun IosArgs.sendConfiguration() { objectToMap().filter { it.key != COMMON_ARGS }.getNonDefaultArgs(defaultArgsMap) .plus(commonArgs.objectToMap().getNonDefaultArgs(defaultCommonArgs)) .let { - messageBuilder.event(project, CONFIGURATION_KEY, JSONObject(it)) + it.forEach { + messageBuilder.event(project, + it.key,JSONObject(mapOf(it.key to it.value))).sendMessage() + } } } From c850a651f04e324eb7b3fc42b6ba264fb68e7ce0 Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 14 Jan 2021 00:13:25 +0100 Subject: [PATCH 06/21] Update Mixpanel.kt --- .../src/main/kotlin/ftl/analytics/Mixpanel.kt | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/test_runner/src/main/kotlin/ftl/analytics/Mixpanel.kt b/test_runner/src/main/kotlin/ftl/analytics/Mixpanel.kt index d539fc343a..4760f223d6 100644 --- a/test_runner/src/main/kotlin/ftl/analytics/Mixpanel.kt +++ b/test_runner/src/main/kotlin/ftl/analytics/Mixpanel.kt @@ -54,8 +54,10 @@ fun AndroidArgs.sendConfiguration() { .plus(commonArgs.objectToMap().getNonDefaultArgs(defaultCommonArgs)) .let { it.forEach { - messageBuilder.event(project, - it.key,JSONObject(mapOf(it.key to it.value))).sendMessage() + messageBuilder.event( + project, + it.key, JSONObject(mapOf(it.key to it.value)) + ).sendMessage() } } } @@ -70,8 +72,10 @@ fun IosArgs.sendConfiguration() { .plus(commonArgs.objectToMap().getNonDefaultArgs(defaultCommonArgs)) .let { it.forEach { - messageBuilder.event(project, - it.key,JSONObject(mapOf(it.key to it.value))).sendMessage() + messageBuilder.event( + project, + it.key, JSONObject(mapOf(it.key to it.value)) + ).sendMessage() } } } From 62a74ed817a2eea5d6030be7c1c6f04018c06d05 Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 14 Jan 2021 22:53:07 +0100 Subject: [PATCH 07/21] Fix crash on ios, set api token --- .../src/main/kotlin/ftl/analytics/Mixpanel.kt | 24 +++++++++---------- .../src/main/kotlin/ftl/args/IosArgs.kt | 3 +++ 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/test_runner/src/main/kotlin/ftl/analytics/Mixpanel.kt b/test_runner/src/main/kotlin/ftl/analytics/Mixpanel.kt index 4760f223d6..68dd7ada9c 100644 --- a/test_runner/src/main/kotlin/ftl/analytics/Mixpanel.kt +++ b/test_runner/src/main/kotlin/ftl/analytics/Mixpanel.kt @@ -10,7 +10,7 @@ import ftl.args.IArgs import ftl.args.IosArgs import org.json.JSONObject -private const val MIXPANEL_API_TOKEN = "f7490466a203188ecf591b9e7b0ae19d" +private const val MIXPANEL_API_TOKEN = "d9728b2c8e6ca9fd6de1fcd32dd8cdc2" private const val CONFIGURATION_KEY = "configuration" private const val PROJECT_ID = "project_id" private const val COMMON_ARGS = "commonArgs" @@ -53,12 +53,11 @@ fun AndroidArgs.sendConfiguration() { objectToMap().filter { it.key != COMMON_ARGS }.getNonDefaultArgs(defaultArgsMap) .plus(commonArgs.objectToMap().getNonDefaultArgs(defaultCommonArgs)) .let { - it.forEach { - messageBuilder.event( - project, - it.key, JSONObject(mapOf(it.key to it.value)) - ).sendMessage() - } + messageBuilder.event( + project, + CONFIGURATION_KEY, + JSONObject(it) + ).sendMessage() } } @@ -71,12 +70,11 @@ fun IosArgs.sendConfiguration() { objectToMap().filter { it.key != COMMON_ARGS }.getNonDefaultArgs(defaultArgsMap) .plus(commonArgs.objectToMap().getNonDefaultArgs(defaultCommonArgs)) .let { - it.forEach { - messageBuilder.event( - project, - it.key, JSONObject(mapOf(it.key to it.value)) - ).sendMessage() - } + messageBuilder.event( + project, + CONFIGURATION_KEY, + JSONObject(it) + ).sendMessage() } } diff --git a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt index 78f5864043..b43ec7f07f 100644 --- a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt @@ -1,5 +1,6 @@ package ftl.args +import com.fasterxml.jackson.annotation.JsonIgnore import com.google.common.annotations.VisibleForTesting import ftl.args.yml.Type import ftl.ios.xctest.XcTestRunData @@ -20,6 +21,8 @@ data class IosArgs( ) : IArgs by commonArgs { override val useLegacyJUnitResult = true + + @get:JsonIgnore val xcTestRunData: XcTestRunData by lazy { calculateXcTestRunData() } companion object : IosArgsCompanion() From 6037185eca3b463755a97df25a325ef073ddfb91 Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 14 Jan 2021 22:58:23 +0100 Subject: [PATCH 08/21] Update Mixpanel.kt --- .../src/main/kotlin/ftl/analytics/Mixpanel.kt | 21 +++++++------------ 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/test_runner/src/main/kotlin/ftl/analytics/Mixpanel.kt b/test_runner/src/main/kotlin/ftl/analytics/Mixpanel.kt index 68dd7ada9c..d3b7f89309 100644 --- a/test_runner/src/main/kotlin/ftl/analytics/Mixpanel.kt +++ b/test_runner/src/main/kotlin/ftl/analytics/Mixpanel.kt @@ -52,13 +52,8 @@ fun AndroidArgs.sendConfiguration() { objectToMap().filter { it.key != COMMON_ARGS }.getNonDefaultArgs(defaultArgsMap) .plus(commonArgs.objectToMap().getNonDefaultArgs(defaultCommonArgs)) - .let { - messageBuilder.event( - project, - CONFIGURATION_KEY, - JSONObject(it) - ).sendMessage() - } + .createEvent(project) + .sendMessage() } fun IosArgs.sendConfiguration() { @@ -69,15 +64,13 @@ fun IosArgs.sendConfiguration() { objectToMap().filter { it.key != COMMON_ARGS }.getNonDefaultArgs(defaultArgsMap) .plus(commonArgs.objectToMap().getNonDefaultArgs(defaultCommonArgs)) - .let { - messageBuilder.event( - project, - CONFIGURATION_KEY, - JSONObject(it) - ).sendMessage() - } + .createEvent(project) + .sendMessage() } +private fun Map.createEvent(projectId: String) = + messageBuilder.event(projectId, CONFIGURATION_KEY, JSONObject(this)) + private fun Map.getNonDefaultArgs(defaultArgs: Map) = keys.fold(mapOf()) { acc, key -> acc.compareValues(key, this, defaultArgs[key]) From eec73bf9a616f1e95f5bd1f01212bcb4bacb8af5 Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 14 Jan 2021 23:28:07 +0100 Subject: [PATCH 09/21] add disable usage statistics flag --- .../ftl/analytics/{Mixpanel.kt => UsageStatistics.kt} | 4 ++-- test_runner/src/main/kotlin/ftl/args/CommonArgs.kt | 3 ++- test_runner/src/main/kotlin/ftl/args/CreateCommonArgs.kt | 3 ++- test_runner/src/main/kotlin/ftl/args/IArgs.kt | 2 ++ .../src/main/kotlin/ftl/config/common/CommonFlankConfig.kt | 7 +++++++ 5 files changed, 15 insertions(+), 4 deletions(-) rename test_runner/src/main/kotlin/ftl/analytics/{Mixpanel.kt => UsageStatistics.kt} (94%) diff --git a/test_runner/src/main/kotlin/ftl/analytics/Mixpanel.kt b/test_runner/src/main/kotlin/ftl/analytics/UsageStatistics.kt similarity index 94% rename from test_runner/src/main/kotlin/ftl/analytics/Mixpanel.kt rename to test_runner/src/main/kotlin/ftl/analytics/UsageStatistics.kt index d3b7f89309..ba415704d6 100644 --- a/test_runner/src/main/kotlin/ftl/analytics/Mixpanel.kt +++ b/test_runner/src/main/kotlin/ftl/analytics/UsageStatistics.kt @@ -44,7 +44,7 @@ private fun IArgs.registerUser() { private fun JSONObject.sendMessage() = apiClient.sendMessage(this) -fun AndroidArgs.sendConfiguration() { +fun AndroidArgs.sendConfiguration() = takeUnless { disableUsageStatistics }?.let { registerUser() val defaultArgs = AndroidArgs.default() val defaultArgsMap = defaultArgs.objectToMap() @@ -56,7 +56,7 @@ fun AndroidArgs.sendConfiguration() { .sendMessage() } -fun IosArgs.sendConfiguration() { +fun IosArgs.sendConfiguration() = takeUnless { disableUsageStatistics }?.let { registerUser() val defaultArgs = IosArgs.default() val defaultArgsMap = defaultArgs.objectToMap() diff --git a/test_runner/src/main/kotlin/ftl/args/CommonArgs.kt b/test_runner/src/main/kotlin/ftl/args/CommonArgs.kt index 6e6082e597..42702a307c 100644 --- a/test_runner/src/main/kotlin/ftl/args/CommonArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/CommonArgs.kt @@ -43,5 +43,6 @@ data class CommonArgs( override val disableResultsUpload: Boolean, override val defaultTestTime: Double, override val defaultClassTestTime: Double, - override val useAverageTestTimeForNewTests: Boolean + override val useAverageTestTimeForNewTests: Boolean, + override val disableUsageStatistics: Boolean ) : IArgs diff --git a/test_runner/src/main/kotlin/ftl/args/CreateCommonArgs.kt b/test_runner/src/main/kotlin/ftl/args/CreateCommonArgs.kt index 53ab3f4fa0..caadbb318a 100644 --- a/test_runner/src/main/kotlin/ftl/args/CreateCommonArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/CreateCommonArgs.kt @@ -51,7 +51,8 @@ fun CommonConfig.createCommonArgs( disableResultsUpload = flank::disableResultsUpload.require(), defaultTestTime = flank::defaultTestTime.require(), defaultClassTestTime = flank::defaultClassTestTime.require(), - useAverageTestTimeForNewTests = flank::useAverageTestTimeForNewTests.require() + useAverageTestTimeForNewTests = flank::useAverageTestTimeForNewTests.require(), + disableUsageStatistics = flank.disableUsageStatistics ?: false ).apply { ArgsHelper.createJunitBucket(project, smartFlankGcsPath) } diff --git a/test_runner/src/main/kotlin/ftl/args/IArgs.kt b/test_runner/src/main/kotlin/ftl/args/IArgs.kt index 901fcb1b52..93770f402f 100644 --- a/test_runner/src/main/kotlin/ftl/args/IArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IArgs.kt @@ -71,6 +71,8 @@ interface IArgs { val defaultClassTestTime: Double val useAverageTestTimeForNewTests: Boolean + val disableUsageStatistics: Boolean + fun useLocalResultDir() = localResultDir != defaultLocalResultsDir companion object { diff --git a/test_runner/src/main/kotlin/ftl/config/common/CommonFlankConfig.kt b/test_runner/src/main/kotlin/ftl/config/common/CommonFlankConfig.kt index 359d462ee8..efe16ca583 100644 --- a/test_runner/src/main/kotlin/ftl/config/common/CommonFlankConfig.kt +++ b/test_runner/src/main/kotlin/ftl/config/common/CommonFlankConfig.kt @@ -173,6 +173,13 @@ data class CommonFlankConfig @JsonIgnore constructor( @set:JsonProperty("use-average-test-time-for-new-tests") var useAverageTestTimeForNewTests: Boolean? by data + @set:CommandLine.Option( + names = ["--disable-usage-statistics"], + description = ["If set to true flank not send usage statistics."] + ) + @set:JsonProperty("disable-usage-statistics") + var disableUsageStatistics: Boolean? by data + constructor() : this(mutableMapOf().withDefault { null }) companion object : IYmlKeys { From b6f5a3746f91cbe5be44ed59f610c089e8c9e096 Mon Sep 17 00:00:00 2001 From: Adam Date: Fri, 15 Jan 2021 18:03:04 +0100 Subject: [PATCH 10/21] Usage statistics disabled in tests --- test_runner/src/main/kotlin/ftl/analytics/UsageStatistics.kt | 5 +++-- test_runner/src/main/kotlin/ftl/args/IArgs.kt | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/test_runner/src/main/kotlin/ftl/analytics/UsageStatistics.kt b/test_runner/src/main/kotlin/ftl/analytics/UsageStatistics.kt index ba415704d6..45f50646dc 100644 --- a/test_runner/src/main/kotlin/ftl/analytics/UsageStatistics.kt +++ b/test_runner/src/main/kotlin/ftl/analytics/UsageStatistics.kt @@ -8,6 +8,7 @@ import com.mixpanel.mixpanelapi.MixpanelAPI import ftl.args.AndroidArgs import ftl.args.IArgs import ftl.args.IosArgs +import ftl.args.blockSendingUsageStatistics import org.json.JSONObject private const val MIXPANEL_API_TOKEN = "d9728b2c8e6ca9fd6de1fcd32dd8cdc2" @@ -44,7 +45,7 @@ private fun IArgs.registerUser() { private fun JSONObject.sendMessage() = apiClient.sendMessage(this) -fun AndroidArgs.sendConfiguration() = takeUnless { disableUsageStatistics }?.let { +fun AndroidArgs.sendConfiguration() = takeUnless { blockSendingUsageStatistics }?.let { registerUser() val defaultArgs = AndroidArgs.default() val defaultArgsMap = defaultArgs.objectToMap() @@ -56,7 +57,7 @@ fun AndroidArgs.sendConfiguration() = takeUnless { disableUsageStatistics }?.let .sendMessage() } -fun IosArgs.sendConfiguration() = takeUnless { disableUsageStatistics }?.let { +fun IosArgs.sendConfiguration() = takeUnless { blockSendingUsageStatistics }?.let { registerUser() val defaultArgs = IosArgs.default() val defaultArgsMap = defaultArgs.objectToMap() diff --git a/test_runner/src/main/kotlin/ftl/args/IArgs.kt b/test_runner/src/main/kotlin/ftl/args/IArgs.kt index 93770f402f..dac3e70887 100644 --- a/test_runner/src/main/kotlin/ftl/args/IArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IArgs.kt @@ -6,6 +6,7 @@ import ftl.args.yml.Type import ftl.config.Device import ftl.config.common.CommonFlankConfig.Companion.defaultLocalResultsDir import ftl.run.status.OutputStyle +import ftl.util.readVersion import ftl.util.timeoutToMils // Properties common to both Android and iOS @@ -93,3 +94,6 @@ val IArgs.logLevel fun IArgs.setupLogLevel() { setLogLevel(logLevel) } + +val IArgs.blockSendingUsageStatistics + get() = disableUsageStatistics || readVersion() == "local_snapshot" From feff626612616d13a0a5f14a05b7a5e032820fc7 Mon Sep 17 00:00:00 2001 From: Adam Date: Sun, 17 Jan 2021 17:41:07 +0100 Subject: [PATCH 11/21] Update UsageStatistics.kt --- .../kotlin/ftl/analytics/UsageStatistics.kt | 68 +++++++++---------- 1 file changed, 33 insertions(+), 35 deletions(-) diff --git a/test_runner/src/main/kotlin/ftl/analytics/UsageStatistics.kt b/test_runner/src/main/kotlin/ftl/analytics/UsageStatistics.kt index 45f50646dc..d047f9dff1 100644 --- a/test_runner/src/main/kotlin/ftl/analytics/UsageStatistics.kt +++ b/test_runner/src/main/kotlin/ftl/analytics/UsageStatistics.kt @@ -17,21 +17,29 @@ private const val PROJECT_ID = "project_id" private const val COMMON_ARGS = "commonArgs" private const val NAME_KEY = "name" -private val messageBuilder by lazy { - MessageBuilder(MIXPANEL_API_TOKEN) -} - -private val apiClient by lazy { - MixpanelAPI() +fun AndroidArgs.sendConfiguration() = takeUnless { blockSendingUsageStatistics }?.let { + registerUser() + AndroidArgs.default().let { defaultArgs -> + objectToMap().filterNonCommonArgs().getNonDefaultArgs(defaultArgs.objectToMap()) + .plus(commonArgs.objectToMap().getNonDefaultArgs(defaultArgs.commonArgs.objectToMap())) + .createEvent(project) + .sendMessage() + } } -private val objectMapper by lazy { - jsonMapper { - addModule(kotlinModule()) +fun IosArgs.sendConfiguration() = takeUnless { blockSendingUsageStatistics }?.let { + registerUser() + IosArgs.default().let { defaultArgs -> + objectToMap().filterNonCommonArgs().getNonDefaultArgs(defaultArgs.objectToMap()) + .plus(commonArgs.objectToMap().getNonDefaultArgs(defaultArgs.commonArgs.objectToMap())) + .createEvent(project) + .sendMessage() } } -private fun IArgs.registerUser() { +private fun Map.filterNonCommonArgs() = filter { it.key != COMMON_ARGS } + +private fun IArgs.registerUser(): IArgs = apply { messageBuilder.set( project, JSONObject( @@ -45,33 +53,11 @@ private fun IArgs.registerUser() { private fun JSONObject.sendMessage() = apiClient.sendMessage(this) -fun AndroidArgs.sendConfiguration() = takeUnless { blockSendingUsageStatistics }?.let { - registerUser() - val defaultArgs = AndroidArgs.default() - val defaultArgsMap = defaultArgs.objectToMap() - val defaultCommonArgs = defaultArgs.commonArgs.objectToMap() - - objectToMap().filter { it.key != COMMON_ARGS }.getNonDefaultArgs(defaultArgsMap) - .plus(commonArgs.objectToMap().getNonDefaultArgs(defaultCommonArgs)) - .createEvent(project) - .sendMessage() -} - -fun IosArgs.sendConfiguration() = takeUnless { blockSendingUsageStatistics }?.let { - registerUser() - val defaultArgs = IosArgs.default() - val defaultArgsMap = defaultArgs.objectToMap() - val defaultCommonArgs = defaultArgs.commonArgs.objectToMap() - - objectToMap().filter { it.key != COMMON_ARGS }.getNonDefaultArgs(defaultArgsMap) - .plus(commonArgs.objectToMap().getNonDefaultArgs(defaultCommonArgs)) - .createEvent(project) - .sendMessage() -} - private fun Map.createEvent(projectId: String) = messageBuilder.event(projectId, CONFIGURATION_KEY, JSONObject(this)) +private fun Any.objectToMap() = objectMapper.convertValue(this, object : TypeReference>() {}) + private fun Map.getNonDefaultArgs(defaultArgs: Map) = keys.fold(mapOf()) { acc, key -> acc.compareValues(key, this, defaultArgs[key]) @@ -81,4 +67,16 @@ private fun Map.compareValues(key: String, source: Map>() {}) +private val messageBuilder by lazy { + MessageBuilder(MIXPANEL_API_TOKEN) +} + +private val apiClient by lazy { + MixpanelAPI() +} + +private val objectMapper by lazy { + jsonMapper { + addModule(kotlinModule()) + } +} From 32c6fd51a6fc0ce4e49cbd7d90d59ccadd451b56 Mon Sep 17 00:00:00 2001 From: Adam Date: Sun, 17 Jan 2021 21:27:58 +0100 Subject: [PATCH 12/21] Get ignored anonymized members by reflection, some changes in the implementation --- .../ftl/analytics/AndroidUsageStatistics.kt | 14 +++ .../ftl/analytics/IosUsageStatistics.kt | 14 +++ .../ftl/analytics/StatisticsAnnotations.kt | 5 + .../kotlin/ftl/analytics/UsageStatistics.kt | 93 +++++++++---------- .../ftl/analytics/UsageStatisticsClient.kt | 33 +++++++ .../src/main/kotlin/ftl/args/AndroidArgs.kt | 28 ++++++ test_runner/src/main/kotlin/ftl/args/IArgs.kt | 20 ++++ .../src/main/kotlin/ftl/args/IosArgs.kt | 11 +++ 8 files changed, 168 insertions(+), 50 deletions(-) create mode 100644 test_runner/src/main/kotlin/ftl/analytics/AndroidUsageStatistics.kt create mode 100644 test_runner/src/main/kotlin/ftl/analytics/IosUsageStatistics.kt create mode 100644 test_runner/src/main/kotlin/ftl/analytics/StatisticsAnnotations.kt create mode 100644 test_runner/src/main/kotlin/ftl/analytics/UsageStatisticsClient.kt diff --git a/test_runner/src/main/kotlin/ftl/analytics/AndroidUsageStatistics.kt b/test_runner/src/main/kotlin/ftl/analytics/AndroidUsageStatistics.kt new file mode 100644 index 0000000000..7f74a0c846 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/analytics/AndroidUsageStatistics.kt @@ -0,0 +1,14 @@ +package ftl.analytics + +import ftl.args.AndroidArgs +import ftl.args.blockSendingUsageStatistics + +fun AndroidArgs.sendConfiguration() = takeUnless { blockSendingUsageStatistics }?.let { + registerUser() + AndroidArgs.default().let { defaultArgs -> + objectToMap().filterNonCommonArgs().getNonDefaultArgs(defaultArgs.objectToMap()) + .plus(commonArgs.objectToMap().getNonDefaultArgs(defaultArgs.commonArgs.objectToMap())) + .createEvent(project) + .sendMessage() + } +} diff --git a/test_runner/src/main/kotlin/ftl/analytics/IosUsageStatistics.kt b/test_runner/src/main/kotlin/ftl/analytics/IosUsageStatistics.kt new file mode 100644 index 0000000000..76368722b7 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/analytics/IosUsageStatistics.kt @@ -0,0 +1,14 @@ +package ftl.analytics + +import ftl.args.IosArgs +import ftl.args.blockSendingUsageStatistics + +fun IosArgs.sendConfiguration() = takeUnless { blockSendingUsageStatistics }?.let { + registerUser() + IosArgs.default().let { defaultArgs -> + objectToMap().filterNonCommonArgs().getNonDefaultArgs(defaultArgs.objectToMap()) + .plus(commonArgs.objectToMap().getNonDefaultArgs(defaultArgs.commonArgs.objectToMap())) + .createEvent(project) + .sendMessage() + } +} diff --git a/test_runner/src/main/kotlin/ftl/analytics/StatisticsAnnotations.kt b/test_runner/src/main/kotlin/ftl/analytics/StatisticsAnnotations.kt new file mode 100644 index 0000000000..f7cca1f585 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/analytics/StatisticsAnnotations.kt @@ -0,0 +1,5 @@ +package ftl.analytics + +annotation class IgnoreInStatistics + +annotation class AnonymizeInStatistics diff --git a/test_runner/src/main/kotlin/ftl/analytics/UsageStatistics.kt b/test_runner/src/main/kotlin/ftl/analytics/UsageStatistics.kt index d047f9dff1..1c210c2002 100644 --- a/test_runner/src/main/kotlin/ftl/analytics/UsageStatistics.kt +++ b/test_runner/src/main/kotlin/ftl/analytics/UsageStatistics.kt @@ -1,82 +1,75 @@ package ftl.analytics import com.fasterxml.jackson.core.type.TypeReference -import com.fasterxml.jackson.module.kotlin.jsonMapper -import com.fasterxml.jackson.module.kotlin.kotlinModule -import com.mixpanel.mixpanelapi.MessageBuilder -import com.mixpanel.mixpanelapi.MixpanelAPI import ftl.args.AndroidArgs import ftl.args.IArgs import ftl.args.IosArgs -import ftl.args.blockSendingUsageStatistics import org.json.JSONObject +import kotlin.reflect.KClass -private const val MIXPANEL_API_TOKEN = "d9728b2c8e6ca9fd6de1fcd32dd8cdc2" -private const val CONFIGURATION_KEY = "configuration" private const val PROJECT_ID = "project_id" private const val COMMON_ARGS = "commonArgs" private const val NAME_KEY = "name" +private const val COVERAGE_FILE_PATH = "coverageFilePath" +private const val ENVIRONMENT_VARIABLES = "environmentVariables" +private const val ANONYMIZE_VALUE = "..." -fun AndroidArgs.sendConfiguration() = takeUnless { blockSendingUsageStatistics }?.let { - registerUser() - AndroidArgs.default().let { defaultArgs -> - objectToMap().filterNonCommonArgs().getNonDefaultArgs(defaultArgs.objectToMap()) - .plus(commonArgs.objectToMap().getNonDefaultArgs(defaultArgs.commonArgs.objectToMap())) - .createEvent(project) - .sendMessage() - } +private val keysToRemove by lazy { + IArgs::class.findMembersWithAnnotation(IgnoreInStatistics::class) + + AndroidArgs::class.findMembersWithAnnotation(IgnoreInStatistics::class) + + IosArgs::class.findMembersWithAnnotation(IgnoreInStatistics::class) } -fun IosArgs.sendConfiguration() = takeUnless { blockSendingUsageStatistics }?.let { - registerUser() - IosArgs.default().let { defaultArgs -> - objectToMap().filterNonCommonArgs().getNonDefaultArgs(defaultArgs.objectToMap()) - .plus(commonArgs.objectToMap().getNonDefaultArgs(defaultArgs.commonArgs.objectToMap())) - .createEvent(project) - .sendMessage() - } +private val keysToAnonymize by lazy { + IArgs::class.findMembersWithAnnotation(AnonymizeInStatistics::class) + + AndroidArgs::class.findMembersWithAnnotation(AnonymizeInStatistics::class) + + IosArgs::class.findMembersWithAnnotation(AnonymizeInStatistics::class) } -private fun Map.filterNonCommonArgs() = filter { it.key != COMMON_ARGS } +private fun KClass<*>.findMembersWithAnnotation(annotationType: KClass<*>) = this.members.filter { + it.annotations.any { annotation -> annotation.annotationClass == annotationType } +}.map { + it.name +} -private fun IArgs.registerUser(): IArgs = apply { +internal fun IArgs.registerUser(): IArgs = apply { messageBuilder.set( - project, - JSONObject( - mapOf( - PROJECT_ID to project, - NAME_KEY to project - ) - ) + project, mapOf(PROJECT_ID to project, NAME_KEY to project).toJSONObject() ).sendMessage() } -private fun JSONObject.sendMessage() = apiClient.sendMessage(this) +internal fun Any.objectToMap() = objectMapper.convertValue(this, object : TypeReference>() {}) -private fun Map.createEvent(projectId: String) = - messageBuilder.event(projectId, CONFIGURATION_KEY, JSONObject(this)) +internal fun Map.filterNonCommonArgs() = filter { it.key != COMMON_ARGS } -private fun Any.objectToMap() = objectMapper.convertValue(this, object : TypeReference>() {}) - -private fun Map.getNonDefaultArgs(defaultArgs: Map) = +internal fun Map.getNonDefaultArgs(defaultArgs: Map) = keys.fold(mapOf()) { acc, key -> acc.compareValues(key, this, defaultArgs[key]) } +@Suppress("UNCHECKED_CAST") private fun Map.compareValues(key: String, source: Map, defaultValue: Any?) = - if (source[key] != defaultValue) this + (key to source[key]) - else this + keysToAnonymize.let { keysToAnonymize -> + when { + (key == ENVIRONMENT_VARIABLES) -> this + (source[key] as Map).filterSensitiveDataInEnv() + source[key] == defaultValue || key in keysToRemove -> this + keysToAnonymize.contains(key) -> this + (key to source[key]?.toAnonymous()) + else -> this + (key to source[key]) + } + } -private val messageBuilder by lazy { - MessageBuilder(MIXPANEL_API_TOKEN) -} +private fun Map.filterSensitiveDataInEnv() = keys.fold( + mapOf(), + { acc, key -> + acc + + if (key == COVERAGE_FILE_PATH) (key to ANONYMIZE_VALUE) + else (key to (this[key].orEmpty())) + } +) -private val apiClient by lazy { - MixpanelAPI() +private fun Any.toAnonymous() = when (this) { + is List<*> -> "Count: $size" + else -> ANONYMIZE_VALUE } -private val objectMapper by lazy { - jsonMapper { - addModule(kotlinModule()) - } -} +private fun Map<*, *>.toJSONObject() = JSONObject(this) diff --git a/test_runner/src/main/kotlin/ftl/analytics/UsageStatisticsClient.kt b/test_runner/src/main/kotlin/ftl/analytics/UsageStatisticsClient.kt new file mode 100644 index 0000000000..0e1644524d --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/analytics/UsageStatisticsClient.kt @@ -0,0 +1,33 @@ +package ftl.analytics + +import com.fasterxml.jackson.module.kotlin.jsonMapper +import com.fasterxml.jackson.module.kotlin.kotlinModule +import com.mixpanel.mixpanelapi.MessageBuilder +import com.mixpanel.mixpanelapi.MixpanelAPI +import ftl.util.SESSION_ID +import ftl.util.sessionId +import org.json.JSONObject + +private const val MIXPANEL_API_TOKEN = "d9728b2c8e6ca9fd6de1fcd32dd8cdc2" +private const val CONFIGURATION_KEY = "configuration" + +internal val messageBuilder by lazy { + MessageBuilder(MIXPANEL_API_TOKEN) +} + +internal val apiClient by lazy { + MixpanelAPI() +} + +internal val objectMapper by lazy { + jsonMapper { + addModule(kotlinModule()) + } +} + +internal fun JSONObject.sendMessage() = apiClient.sendMessage(this) + +internal fun Map.createEvent(projectId: String) = + (this + Pair(SESSION_ID, sessionId)).run { + messageBuilder.event(projectId, CONFIGURATION_KEY, JSONObject(this)) + } diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt index 5aee437742..e68b5c8d3e 100644 --- a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt @@ -1,29 +1,57 @@ package ftl.args +import ftl.analytics.AnonymizeInStatistics import ftl.args.yml.AppTestPair import ftl.args.yml.Type data class AndroidArgs( val commonArgs: CommonArgs, + + @property:AnonymizeInStatistics val appApk: String?, + + @property:AnonymizeInStatistics val testApk: String?, + + @property:AnonymizeInStatistics val additionalApks: List, + val autoGoogleLogin: Boolean, val useOrchestrator: Boolean, + + @property:AnonymizeInStatistics val roboDirectives: List, + + @property:AnonymizeInStatistics val roboScript: String?, + + @property:AnonymizeInStatistics val environmentVariables: Map, // should not be printed, because could contain sensitive information val grantPermissions: String?, + + @property:AnonymizeInStatistics val scenarioLabels: List, + + @property:AnonymizeInStatistics val obbFiles: List, + + @property:AnonymizeInStatistics val obbNames: List, val performanceMetrics: Boolean, val numUniformShards: Int?, + + @property:AnonymizeInStatistics val testRunnerClass: String?, + + @property:AnonymizeInStatistics val testTargets: List, + + @property:AnonymizeInStatistics val additionalAppTestApks: List, override val useLegacyJUnitResult: Boolean, val obfuscateDumpShards: Boolean, + + @property:AnonymizeInStatistics val testTargetsForShard: ShardChunks ) : IArgs by commonArgs { companion object : AndroidArgsCompanion() diff --git a/test_runner/src/main/kotlin/ftl/args/IArgs.kt b/test_runner/src/main/kotlin/ftl/args/IArgs.kt index dac3e70887..feace6ae7d 100644 --- a/test_runner/src/main/kotlin/ftl/args/IArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IArgs.kt @@ -2,6 +2,8 @@ package ftl.args import flank.common.OutputLogLevel import flank.common.setLogLevel +import ftl.analytics.AnonymizeInStatistics +import ftl.analytics.IgnoreInStatistics import ftl.args.yml.Type import ftl.config.Device import ftl.config.common.CommonFlankConfig.Companion.defaultLocalResultsDir @@ -12,23 +14,35 @@ import ftl.util.timeoutToMils // Properties common to both Android and iOS interface IArgs { // original YAML data + @IgnoreInStatistics val data: String // GcloudYml val devices: List + + @AnonymizeInStatistics val resultsBucket: String + + @AnonymizeInStatistics val resultsDir: String + val recordVideo: Boolean val testTimeout: String val async: Boolean + + @AnonymizeInStatistics val clientDetails: Map? val networkProfile: String? val project: String val resultsHistoryName: String? val flakyTestAttempts: Int + + @AnonymizeInStatistics val otherFiles: Map val scenarioNumbers: List val type: Type? get() = null + + @AnonymizeInStatistics val directoriesToPull: List val failFast: Boolean @@ -36,11 +50,17 @@ interface IArgs { val maxTestShards: Int val shardTime: Int val repeatTests: Int + + @AnonymizeInStatistics val smartFlankGcsPath: String val smartFlankDisableUpload: Boolean val testTargetsAlwaysRun: List + + @AnonymizeInStatistics val filesToDownload: List val disableSharding: Boolean + + @AnonymizeInStatistics val localResultDir: String val runTimeout: String val parsedTimeout: Long diff --git a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt index b43ec7f07f..99065ef6a7 100644 --- a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt @@ -2,6 +2,7 @@ package ftl.args import com.fasterxml.jackson.annotation.JsonIgnore import com.google.common.annotations.VisibleForTesting +import ftl.analytics.AnonymizeInStatistics import ftl.args.yml.Type import ftl.ios.xctest.XcTestRunData import ftl.ios.xctest.calculateXcTestRunData @@ -10,12 +11,22 @@ import ftl.run.exception.FlankConfigurationError data class IosArgs( val commonArgs: CommonArgs, + + @property:AnonymizeInStatistics val xctestrunZip: String, + + @property:AnonymizeInStatistics val xctestrunFile: String, val xcodeVersion: String?, + + @property:AnonymizeInStatistics val testTargets: List, val obfuscateDumpShards: Boolean, + + @property:AnonymizeInStatistics val additionalIpas: List, + + @property:AnonymizeInStatistics val app: String, val testSpecialEntitlements: Boolean? ) : IArgs by commonArgs { From b6a1ff0762ef9ee529d110de25e00645e52a5d62 Mon Sep 17 00:00:00 2001 From: Adam Date: Mon, 18 Jan 2021 18:48:04 +0100 Subject: [PATCH 13/21] Implementation changes, added and updated tests --- .../ftl/analytics/AndroidUsageStatistics.kt | 14 --- .../ftl/analytics/IosUsageStatistics.kt | 14 --- .../ftl/analytics/PrepareConfiguration.kt | 27 +++++ .../ftl/analytics/SendUsageStatistics.kt | 17 +++ .../ftl/analytics/StatisticDataFilters.kt | 47 ++++++++ .../ftl/analytics/StatisticsAnnotations.kt | 5 - .../kotlin/ftl/analytics/UsageStatistics.kt | 64 +---------- .../ftl/analytics/UsageStatisticsClient.kt | 6 +- .../src/main/kotlin/ftl/args/AndroidArgs.kt | 1 + .../src/main/kotlin/ftl/args/IosArgs.kt | 1 + .../ftl/config/common/CommonFlankConfig.kt | 1 + .../ftl/analytics/UsageStatisticsTest.kt | 107 ++++++++++++++++++ .../test/kotlin/ftl/args/AndroidArgsTest.kt | 2 + .../src/test/kotlin/ftl/args/IosArgsTest.kt | 2 + 14 files changed, 213 insertions(+), 95 deletions(-) delete mode 100644 test_runner/src/main/kotlin/ftl/analytics/AndroidUsageStatistics.kt delete mode 100644 test_runner/src/main/kotlin/ftl/analytics/IosUsageStatistics.kt create mode 100644 test_runner/src/main/kotlin/ftl/analytics/PrepareConfiguration.kt create mode 100644 test_runner/src/main/kotlin/ftl/analytics/SendUsageStatistics.kt create mode 100644 test_runner/src/main/kotlin/ftl/analytics/StatisticDataFilters.kt delete mode 100644 test_runner/src/main/kotlin/ftl/analytics/StatisticsAnnotations.kt create mode 100644 test_runner/src/test/kotlin/ftl/analytics/UsageStatisticsTest.kt diff --git a/test_runner/src/main/kotlin/ftl/analytics/AndroidUsageStatistics.kt b/test_runner/src/main/kotlin/ftl/analytics/AndroidUsageStatistics.kt deleted file mode 100644 index 7f74a0c846..0000000000 --- a/test_runner/src/main/kotlin/ftl/analytics/AndroidUsageStatistics.kt +++ /dev/null @@ -1,14 +0,0 @@ -package ftl.analytics - -import ftl.args.AndroidArgs -import ftl.args.blockSendingUsageStatistics - -fun AndroidArgs.sendConfiguration() = takeUnless { blockSendingUsageStatistics }?.let { - registerUser() - AndroidArgs.default().let { defaultArgs -> - objectToMap().filterNonCommonArgs().getNonDefaultArgs(defaultArgs.objectToMap()) - .plus(commonArgs.objectToMap().getNonDefaultArgs(defaultArgs.commonArgs.objectToMap())) - .createEvent(project) - .sendMessage() - } -} diff --git a/test_runner/src/main/kotlin/ftl/analytics/IosUsageStatistics.kt b/test_runner/src/main/kotlin/ftl/analytics/IosUsageStatistics.kt deleted file mode 100644 index 76368722b7..0000000000 --- a/test_runner/src/main/kotlin/ftl/analytics/IosUsageStatistics.kt +++ /dev/null @@ -1,14 +0,0 @@ -package ftl.analytics - -import ftl.args.IosArgs -import ftl.args.blockSendingUsageStatistics - -fun IosArgs.sendConfiguration() = takeUnless { blockSendingUsageStatistics }?.let { - registerUser() - IosArgs.default().let { defaultArgs -> - objectToMap().filterNonCommonArgs().getNonDefaultArgs(defaultArgs.objectToMap()) - .plus(commonArgs.objectToMap().getNonDefaultArgs(defaultArgs.commonArgs.objectToMap())) - .createEvent(project) - .sendMessage() - } -} diff --git a/test_runner/src/main/kotlin/ftl/analytics/PrepareConfiguration.kt b/test_runner/src/main/kotlin/ftl/analytics/PrepareConfiguration.kt new file mode 100644 index 0000000000..b5d9fdae5f --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/analytics/PrepareConfiguration.kt @@ -0,0 +1,27 @@ +package ftl.analytics + +import com.fasterxml.jackson.core.type.TypeReference +import com.google.common.annotations.VisibleForTesting +import ftl.args.AndroidArgs +import ftl.args.IArgs +import ftl.args.IosArgs + +private const val COMMON_ARGS = "commonArgs" + +internal fun AndroidArgs.createEventMap( + defaultArgs: AndroidArgs = AndroidArgs.default(), +) = toArgsMap(defaultArgs).plus(commonArgs.toArgsMap(defaultArgs.commonArgs)) + +internal fun IosArgs.createEventMap( + defaultArgs: IosArgs = IosArgs.default(), +) = toArgsMap(defaultArgs).plus(commonArgs.toArgsMap(defaultArgs.commonArgs)) + +private fun IArgs.toArgsMap( + defaultArgs: IArgs +) = objectToMap().filterNonCommonArgs().getNonDefaultArgs(defaultArgs.objectToMap()) + +@VisibleForTesting +internal fun Any.objectToMap() = objectMapper.convertValue(this, object : TypeReference>() {}) + +@VisibleForTesting +internal fun Map.filterNonCommonArgs() = filter { it.key != COMMON_ARGS } diff --git a/test_runner/src/main/kotlin/ftl/analytics/SendUsageStatistics.kt b/test_runner/src/main/kotlin/ftl/analytics/SendUsageStatistics.kt new file mode 100644 index 0000000000..ba2413109d --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/analytics/SendUsageStatistics.kt @@ -0,0 +1,17 @@ +package ftl.analytics + +import ftl.args.AndroidArgs +import ftl.args.IArgs +import ftl.args.IosArgs +import ftl.args.blockSendingUsageStatistics + +fun AndroidArgs.sendConfiguration() = sendConfiguration(events = createEventMap()) + +fun IosArgs.sendConfiguration() = sendConfiguration(events = createEventMap()) + +fun IArgs.sendConfiguration(events: Map) = takeUnless { blockSendingUsageStatistics }?.run { + registerUser() + events + .toEvent(project) + .send() +} diff --git a/test_runner/src/main/kotlin/ftl/analytics/StatisticDataFilters.kt b/test_runner/src/main/kotlin/ftl/analytics/StatisticDataFilters.kt new file mode 100644 index 0000000000..de01d82f62 --- /dev/null +++ b/test_runner/src/main/kotlin/ftl/analytics/StatisticDataFilters.kt @@ -0,0 +1,47 @@ +package ftl.analytics + +import ftl.args.AndroidArgs +import ftl.args.IArgs +import ftl.args.IosArgs +import kotlin.reflect.KClass + +private const val ANONYMIZE_VALUE = "..." + +annotation class IgnoreInStatistics +annotation class AnonymizeInStatistics + +private val classesForStatistics = listOf(IArgs::class, AndroidArgs::class, IosArgs::class) + +private fun KClass<*>.ignoredMembersForStatistics() = findMembersWithAnnotation(IgnoreInStatistics::class) +private fun KClass<*>.anonymousMembersForStatistics() = findMembersWithAnnotation(AnonymizeInStatistics::class) + +private fun KClass<*>.findMembersWithAnnotation(annotationType: KClass<*>) = members.filter { member -> + member.annotations.any { annotation -> annotation.annotationClass == annotationType } +}.map { + it.name +} + +internal val keysToRemove by lazy { + classesForStatistics.map { it.ignoredMembersForStatistics() }.flatten() +} + +internal val keysToAnonymize by lazy { + classesForStatistics.map { it.anonymousMembersForStatistics() }.flatten() +} + +internal fun Map.removeNotNeededKeys(defaultArgs: Map) = + filterNot { (key, value) -> + value == defaultArgs[key] || key in keysToRemove + } + +internal fun Map.filterSensitiveValues() = mapValues { it.anonymousSensitiveValues() } + +private fun Map.Entry.anonymousSensitiveValues() = + if (keysToAnonymize.contains(key)) value.toAnonymous() + else value + +private fun Any.toAnonymous(): Any = when (this) { + is Map<*, *> -> mapValues { ANONYMIZE_VALUE } + is List<*> -> "Count: $size" + else -> ANONYMIZE_VALUE +} diff --git a/test_runner/src/main/kotlin/ftl/analytics/StatisticsAnnotations.kt b/test_runner/src/main/kotlin/ftl/analytics/StatisticsAnnotations.kt deleted file mode 100644 index f7cca1f585..0000000000 --- a/test_runner/src/main/kotlin/ftl/analytics/StatisticsAnnotations.kt +++ /dev/null @@ -1,5 +0,0 @@ -package ftl.analytics - -annotation class IgnoreInStatistics - -annotation class AnonymizeInStatistics diff --git a/test_runner/src/main/kotlin/ftl/analytics/UsageStatistics.kt b/test_runner/src/main/kotlin/ftl/analytics/UsageStatistics.kt index 1c210c2002..bdde643de6 100644 --- a/test_runner/src/main/kotlin/ftl/analytics/UsageStatistics.kt +++ b/test_runner/src/main/kotlin/ftl/analytics/UsageStatistics.kt @@ -1,75 +1,19 @@ package ftl.analytics -import com.fasterxml.jackson.core.type.TypeReference -import ftl.args.AndroidArgs import ftl.args.IArgs -import ftl.args.IosArgs import org.json.JSONObject -import kotlin.reflect.KClass private const val PROJECT_ID = "project_id" -private const val COMMON_ARGS = "commonArgs" private const val NAME_KEY = "name" -private const val COVERAGE_FILE_PATH = "coverageFilePath" -private const val ENVIRONMENT_VARIABLES = "environmentVariables" -private const val ANONYMIZE_VALUE = "..." - -private val keysToRemove by lazy { - IArgs::class.findMembersWithAnnotation(IgnoreInStatistics::class) + - AndroidArgs::class.findMembersWithAnnotation(IgnoreInStatistics::class) + - IosArgs::class.findMembersWithAnnotation(IgnoreInStatistics::class) -} - -private val keysToAnonymize by lazy { - IArgs::class.findMembersWithAnnotation(AnonymizeInStatistics::class) + - AndroidArgs::class.findMembersWithAnnotation(AnonymizeInStatistics::class) + - IosArgs::class.findMembersWithAnnotation(AnonymizeInStatistics::class) -} - -private fun KClass<*>.findMembersWithAnnotation(annotationType: KClass<*>) = this.members.filter { - it.annotations.any { annotation -> annotation.annotationClass == annotationType } -}.map { - it.name -} internal fun IArgs.registerUser(): IArgs = apply { messageBuilder.set( project, mapOf(PROJECT_ID to project, NAME_KEY to project).toJSONObject() - ).sendMessage() + ).send() } -internal fun Any.objectToMap() = objectMapper.convertValue(this, object : TypeReference>() {}) - -internal fun Map.filterNonCommonArgs() = filter { it.key != COMMON_ARGS } - -internal fun Map.getNonDefaultArgs(defaultArgs: Map) = - keys.fold(mapOf()) { acc, key -> - acc.compareValues(key, this, defaultArgs[key]) - } - -@Suppress("UNCHECKED_CAST") -private fun Map.compareValues(key: String, source: Map, defaultValue: Any?) = - keysToAnonymize.let { keysToAnonymize -> - when { - (key == ENVIRONMENT_VARIABLES) -> this + (source[key] as Map).filterSensitiveDataInEnv() - source[key] == defaultValue || key in keysToRemove -> this - keysToAnonymize.contains(key) -> this + (key to source[key]?.toAnonymous()) - else -> this + (key to source[key]) - } - } - -private fun Map.filterSensitiveDataInEnv() = keys.fold( - mapOf(), - { acc, key -> - acc + - if (key == COVERAGE_FILE_PATH) (key to ANONYMIZE_VALUE) - else (key to (this[key].orEmpty())) - } -) - -private fun Any.toAnonymous() = when (this) { - is List<*> -> "Count: $size" - else -> ANONYMIZE_VALUE -} +internal fun Map.getNonDefaultArgs( + defaultArgs: Map +) = removeNotNeededKeys(defaultArgs).filterSensitiveValues() private fun Map<*, *>.toJSONObject() = JSONObject(this) diff --git a/test_runner/src/main/kotlin/ftl/analytics/UsageStatisticsClient.kt b/test_runner/src/main/kotlin/ftl/analytics/UsageStatisticsClient.kt index 0e1644524d..880ff1e5c8 100644 --- a/test_runner/src/main/kotlin/ftl/analytics/UsageStatisticsClient.kt +++ b/test_runner/src/main/kotlin/ftl/analytics/UsageStatisticsClient.kt @@ -2,6 +2,7 @@ package ftl.analytics import com.fasterxml.jackson.module.kotlin.jsonMapper import com.fasterxml.jackson.module.kotlin.kotlinModule +import com.google.common.annotations.VisibleForTesting import com.mixpanel.mixpanelapi.MessageBuilder import com.mixpanel.mixpanelapi.MixpanelAPI import ftl.util.SESSION_ID @@ -25,9 +26,10 @@ internal val objectMapper by lazy { } } -internal fun JSONObject.sendMessage() = apiClient.sendMessage(this) +@VisibleForTesting +internal fun JSONObject.send() = apiClient.sendMessage(this) -internal fun Map.createEvent(projectId: String) = +internal fun Map.toEvent(projectId: String) = (this + Pair(SESSION_ID, sessionId)).run { messageBuilder.event(projectId, CONFIGURATION_KEY, JSONObject(this)) } diff --git a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt index e68b5c8d3e..7ed5dc1127 100644 --- a/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt @@ -116,6 +116,7 @@ AndroidArgs output-style: ${outputStyle.name.toLowerCase()} disable-results-upload: $disableResultsUpload default-class-test-time: $defaultClassTestTime + disable-usage-statistics: $disableUsageStatistics """.trimIndent() } } diff --git a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt index 99065ef6a7..252949e3d9 100644 --- a/test_runner/src/main/kotlin/ftl/args/IosArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IosArgs.kt @@ -87,6 +87,7 @@ IosArgs output-style: ${outputStyle.name.toLowerCase()} disable-results-upload: $disableResultsUpload default-class-test-time: $defaultClassTestTime + disable-usage-statistics: $disableUsageStatistics """.trimIndent() } } diff --git a/test_runner/src/main/kotlin/ftl/config/common/CommonFlankConfig.kt b/test_runner/src/main/kotlin/ftl/config/common/CommonFlankConfig.kt index efe16ca583..df54723e27 100644 --- a/test_runner/src/main/kotlin/ftl/config/common/CommonFlankConfig.kt +++ b/test_runner/src/main/kotlin/ftl/config/common/CommonFlankConfig.kt @@ -212,6 +212,7 @@ data class CommonFlankConfig @JsonIgnore constructor( defaultTestTime = DEFAULT_TEST_TIME_SEC defaultClassTestTime = DEFAULT_CLASS_TEST_TIME_SEC useAverageTestTimeForNewTests = false + disableUsageStatistics = false } } } diff --git a/test_runner/src/test/kotlin/ftl/analytics/UsageStatisticsTest.kt b/test_runner/src/test/kotlin/ftl/analytics/UsageStatisticsTest.kt new file mode 100644 index 0000000000..360aa6cca8 --- /dev/null +++ b/test_runner/src/test/kotlin/ftl/analytics/UsageStatisticsTest.kt @@ -0,0 +1,107 @@ +package ftl.analytics + +import com.google.common.truth.Truth +import ftl.args.AndroidArgs +import ftl.args.IosArgs +import ftl.util.readVersion +import io.mockk.every +import io.mockk.mockkStatic +import io.mockk.unmockkAll +import io.mockk.verify +import org.json.JSONObject +import org.junit.After +import org.junit.Test + +class UsageStatisticsTest { + + @After + fun afterTest() { + unmockkAll() + } + + @Test + fun `should filter default args for android`() { + val default = AndroidArgs.default() + val args = default.copy(appApk = "test").objectToMap() + + val nonDefaultArgs = args.filterNonCommonArgs().getNonDefaultArgs(default.objectToMap()) + + Truth.assertThat(nonDefaultArgs).containsKey("appApk") + Truth.assertThat(nonDefaultArgs.count()).isEqualTo(1) + } + + @Test + fun `should filter default args for ios`() { + val default = IosArgs.default() + val args = default.copy(xctestrunFile = "test").objectToMap() + + val nonDefaultArgs = args.filterNonCommonArgs().getNonDefaultArgs(default.objectToMap()) + + Truth.assertThat(nonDefaultArgs).containsKey("xctestrunFile") + Truth.assertThat(nonDefaultArgs.count()).isEqualTo(1) + } + + @Test + fun `should filter default args for ios with commonArgs`() { + val default = IosArgs.default() + val args = + default.copy(xctestrunFile = "test", commonArgs = default.commonArgs.copy(resultsBucket = "test_bucket")) + + val nonDefaultArgs = args.createEventMap(default) + + Truth.assertThat(nonDefaultArgs).containsKey("xctestrunFile") + Truth.assertThat(nonDefaultArgs).containsKey("resultsBucket") + Truth.assertThat(nonDefaultArgs.count()).isEqualTo(2) + } + + @Test + fun `should filter default args for android with commonArgs`() { + val default = AndroidArgs.default() + val args = default.copy(testApk = "test", commonArgs = default.commonArgs.copy(resultsBucket = "test_bucket")) + + val nonDefaultArgs = args.createEventMap(default) + + Truth.assertThat(nonDefaultArgs).containsKey("testApk") + Truth.assertThat(nonDefaultArgs).containsKey("resultsBucket") + Truth.assertThat(nonDefaultArgs.count()).isEqualTo(2) + } + + @Test + fun `should anonymize maps to (key,anonymizedValue)`() { + val default = AndroidArgs.default() + val args = default.copy(environmentVariables = mapOf("testKey" to "testValue", "testKey2" to "testValue2")) + + val nonDefaultArgs = args.createEventMap(default) + (nonDefaultArgs["environmentVariables"] as? Map<*, *>)?.let { environmentVariables -> + Truth.assertThat(environmentVariables.count()).isEqualTo(2) + Truth.assertThat(environmentVariables.values.all { it == "..." }).isTrue() + } + + Truth.assertThat(nonDefaultArgs.count()).isEqualTo(1) + Truth.assertThat(nonDefaultArgs.keys).containsExactly("environmentVariables") + } + + @Test + fun `should not run send configuration if unit tests`() { + mockkStatic(JSONObject::send) + + AndroidArgs.default().sendConfiguration() + + verify(inverse = true) { any().send() } + } + + @Test + fun `should not run send configuration if disable statistic param set`() { + mockkStatic(JSONObject::send) + mockkStatic(::readVersion) + + every { readVersion() } returns "test" + + AndroidArgs.default().run { + copy(commonArgs = commonArgs.copy(disableUsageStatistics = true)) + .sendConfiguration() + } + + verify(inverse = true) { any().send() } + } +} diff --git a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt index c14560558a..87a4f49408 100644 --- a/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt @@ -377,6 +377,7 @@ AndroidArgs output-style: single disable-results-upload: true default-class-test-time: 30.0 + disable-usage-statistics: false """.trimIndent() ) } @@ -448,6 +449,7 @@ AndroidArgs output-style: verbose disable-results-upload: false default-class-test-time: 240.0 + disable-usage-statistics: false """.trimIndent(), args.toString() ) diff --git a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt index 223dae2a32..44cd6b407c 100644 --- a/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt +++ b/test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt @@ -277,6 +277,7 @@ IosArgs output-style: single disable-results-upload: true default-class-test-time: 30.0 + disable-usage-statistics: false """.trimIndent() ) } @@ -337,6 +338,7 @@ IosArgs output-style: verbose disable-results-upload: false default-class-test-time: 240.0 + disable-usage-statistics: false """.trimIndent(), args.toString() ) From c556a4cb1054ae0f6d6ec7e44a7adc3f4ce54ee9 Mon Sep 17 00:00:00 2001 From: Adam Date: Tue, 19 Jan 2021 16:38:55 +0100 Subject: [PATCH 14/21] Updated integration tests --- .../test/resources/compare/AllTestFilteredIT-android-compare | 1 + .../src/test/resources/compare/AllTestFilteredIT-ios-compare | 1 + .../src/test/resources/compare/GameloopIT-android-compare | 1 + .../src/test/resources/compare/GameloopIT-ios-compare | 1 + .../src/test/resources/compare/IgnoreFailedIT-compare | 2 +- .../src/test/resources/compare/MultipleApksIT-compare | 2 +- .../test/resources/compare/MultipleDevicesIT-android-compare | 2 +- .../src/test/resources/compare/RunTimeoutIT-compare | 1 + .../src/test/resources/compare/SanityRoboIT-compare | 1 + .../src/test/resources/compare/TestFilteringIT-compare | 1 + 10 files changed, 10 insertions(+), 3 deletions(-) diff --git a/integration_tests/src/test/resources/compare/AllTestFilteredIT-android-compare b/integration_tests/src/test/resources/compare/AllTestFilteredIT-android-compare index f533cd7110..ce5d216d95 100644 --- a/integration_tests/src/test/resources/compare/AllTestFilteredIT-android-compare +++ b/integration_tests/src/test/resources/compare/AllTestFilteredIT-android-compare @@ -61,6 +61,7 @@ AndroidArgs output-style: verbose disable-results-upload: false default-class-test-time: 240.0 + disable-usage-statistics: false RunTests No tests for app-single-success-debug-androidTest.apk diff --git a/integration_tests/src/test/resources/compare/AllTestFilteredIT-ios-compare b/integration_tests/src/test/resources/compare/AllTestFilteredIT-ios-compare index b9fb87811e..dd499703ff 100644 --- a/integration_tests/src/test/resources/compare/AllTestFilteredIT-ios-compare +++ b/integration_tests/src/test/resources/compare/AllTestFilteredIT-ios-compare @@ -50,6 +50,7 @@ IosArgs output-style: verbose disable-results-upload: false default-class-test-time: 240.0 + disable-usage-statistics: false Found xctest: [0-9a-zA-Z\/_.-]*\/test_runner\/src\/test\/kotlin\/ftl\/fixtures\/tmp\/ios\/EarlGreyExample\/Debug-iphoneos\/EarlGreyExampleSwift.app\/PlugIns\/EarlGreyExampleSwiftTests.xctest isMacOS = (true \(mac os x\)|false \(linux\)) (PATH=~\/.flank\s)?nm -U "[0-9a-zA-Z\/_.-]*\/test_runner\/src\/test\/kotlin\/ftl\/fixtures\/tmp\/ios\/EarlGreyExample\/Debug-iphoneos\/EarlGreyExampleSwift.app\/PlugIns\/EarlGreyExampleSwiftTests.xctest\/EarlGreyExampleSwiftTests" diff --git a/integration_tests/src/test/resources/compare/GameloopIT-android-compare b/integration_tests/src/test/resources/compare/GameloopIT-android-compare index 4e7e2356b1..241fd45289 100644 --- a/integration_tests/src/test/resources/compare/GameloopIT-android-compare +++ b/integration_tests/src/test/resources/compare/GameloopIT-android-compare @@ -62,6 +62,7 @@ AndroidArgs output-style: verbose disable-results-upload: false default-class-test-time: 240.0 + disable-usage-statistics: false \s* RunTests Uploading \[test.obb\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* diff --git a/integration_tests/src/test/resources/compare/GameloopIT-ios-compare b/integration_tests/src/test/resources/compare/GameloopIT-ios-compare index 57f8a56821..77ba1df0db 100644 --- a/integration_tests/src/test/resources/compare/GameloopIT-ios-compare +++ b/integration_tests/src/test/resources/compare/GameloopIT-ios-compare @@ -50,6 +50,7 @@ IosArgs output-style: verbose disable-results-upload: false default-class-test-time: 240.0 + disable-usage-statistics: false RunTests \s* diff --git a/integration_tests/src/test/resources/compare/IgnoreFailedIT-compare b/integration_tests/src/test/resources/compare/IgnoreFailedIT-compare index f2eacb2c3d..63c5e572a3 100644 --- a/integration_tests/src/test/resources/compare/IgnoreFailedIT-compare +++ b/integration_tests/src/test/resources/compare/IgnoreFailedIT-compare @@ -1 +1 @@ -AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-single-error-debug-androidTest.apk additional-apks: auto-google-login: false use-orchestrator: true directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: null device: - model: NexusLowRes version: 28 locale: en orientation: portrait num-flaky-test-attempts: 0 test-targets-for-shard: fail-fast: false flank: max-test-shards: 1 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: run-timeout: -1 legacy-junit-result: false ignore-failed-tests: true output-style: verbose disable-results-upload: true default-class-test-time: 240.0 RunTests Smart Flank cache hit: 0\% \(0 \/ 1\) Shard times: 120s Saved 1 shards to android_shards.json Uploading \[app-debug.apk\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[app-single-error-debug-androidTest.apk\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* 1 test \/ 1 shard 1 matrix ids created in \d{1,2}m \d{1,2}s Raw results will be stored in your GCS bucket at \[https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\] Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* CostReport Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m MatrixResultsReport 0 \/ 1 \(0.00\%\) 1 matrices failed [\s\S]* More details are available at: https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? FetchArtifacts Updating matrix file \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \ No newline at end of file +AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-single-error-debug-androidTest.apk additional-apks: auto-google-login: false use-orchestrator: true directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: null device: - model: NexusLowRes version: 28 locale: en orientation: portrait num-flaky-test-attempts: 0 test-targets-for-shard: fail-fast: false flank: max-test-shards: 1 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: run-timeout: -1 legacy-junit-result: false ignore-failed-tests: true output-style: verbose disable-results-upload: true default-class-test-time: 240.0 disable-usage-statistics: false RunTests Smart Flank cache hit: 0\% \(0 \/ 1\) Shard times: 120s Saved 1 shards to android_shards.json Uploading \[app-debug.apk\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[app-single-error-debug-androidTest.apk\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* 1 test \/ 1 shard 1 matrix ids created in \d{1,2}m \d{1,2}s Raw results will be stored in your GCS bucket at \[https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\] Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* CostReport Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m MatrixResultsReport 0 \/ 1 \(0.00\%\) 1 matrices failed [\s\S]* More details are available at: https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? FetchArtifacts Updating matrix file \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \ No newline at end of file diff --git a/integration_tests/src/test/resources/compare/MultipleApksIT-compare b/integration_tests/src/test/resources/compare/MultipleApksIT-compare index 260c4e5ae4..f1198bd420 100644 --- a/integration_tests/src/test/resources/compare/MultipleApksIT-compare +++ b/integration_tests/src/test/resources/compare/MultipleApksIT-compare @@ -1 +1 @@ -AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk test: null additional-apks: auto-google-login: false use-orchestrator: false directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]MainActivity_robo_script.json device: - model: NexusLowRes version: 28 locale: en orientation: portrait num-flaky-test-attempts: 0 test-targets-for-shard: fail-fast: false \s* flank: max-test-shards: 50 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: - app: null test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-success-debug-androidTest.apk - app: null test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-error-debug-androidTest.apk - app: null test: gs:\/\/flank-open-source.appspot.com\/integration\/app-single-success-debug-androidTest.apk run-timeout: -1 legacy-junit-result: false ignore-failed-tests: false output-style: single disable-results-upload: false default-class-test-time: 240.0 \s* RunTests \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s \s* Smart Flank cache hit: 0\% \(0 \/ 1\) Shard times: 120s \s* Saved 3 shards to android_shards.json Uploading \[android_shards.json\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[app-debug.apk\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* (?=.*Uploading \[MainActivity_robo_script.json\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.*\s*)(?=.*Uploading \[app-multiple-success-debug-androidTest.apk\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.*\s*)(?=.*Uploading \[app-multiple-error-debug-androidTest.apk\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.*\s*).* [\s\S]* 11 tests \+ 8 parameterized classes \/ 19 shards \s* Uploading \[session_id.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* 4 matrix ids created in \d{1,2}m \d{1,2}s Raw results will be stored in your GCS bucket at \[https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\] \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* CostReport Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m [\s\S]* Uploading \[CostReport.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* MatrixResultsReport 3 \/ 4 \(75\.00\%\) 1 matrices failed [\s\S]* More details are available at: https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \s* Uploading \[MatrixResultsReport.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[HtmlErrorReport.html\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[JUnitReport.xml\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[matrix_ids.json\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* FetchArtifacts Updating matrix file \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \ No newline at end of file +AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk test: null additional-apks: auto-google-login: false use-orchestrator: false directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]MainActivity_robo_script.json device: - model: NexusLowRes version: 28 locale: en orientation: portrait num-flaky-test-attempts: 0 test-targets-for-shard: fail-fast: false \s* flank: max-test-shards: 50 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: - app: null test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-success-debug-androidTest.apk - app: null test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-error-debug-androidTest.apk - app: null test: gs:\/\/flank-open-source.appspot.com\/integration\/app-single-success-debug-androidTest.apk run-timeout: -1 legacy-junit-result: false ignore-failed-tests: false output-style: single disable-results-upload: false default-class-test-time: 240.0 disable-usage-statistics: false \s* RunTests \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 120s, 120s, 120s, 120s, 120s, 240s, 240s, 240s, 240s \s* Smart Flank cache hit: 0\% \(0 \/ 1\) Shard times: 120s \s* Saved 3 shards to android_shards.json Uploading \[android_shards.json\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[app-debug.apk\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* (?=.*Uploading \[MainActivity_robo_script.json\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.*\s*)(?=.*Uploading \[app-multiple-success-debug-androidTest.apk\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.*\s*)(?=.*Uploading \[app-multiple-error-debug-androidTest.apk\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.*\s*).* [\s\S]* 11 tests \+ 8 parameterized classes \/ 19 shards \s* Uploading \[session_id.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* 4 matrix ids created in \d{1,2}m \d{1,2}s Raw results will be stored in your GCS bucket at \[https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\] \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* CostReport Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m [\s\S]* Uploading \[CostReport.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* MatrixResultsReport 3 \/ 4 \(75\.00\%\) 1 matrices failed [\s\S]* More details are available at: https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \s* Uploading \[MatrixResultsReport.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[HtmlErrorReport.html\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[JUnitReport.xml\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[matrix_ids.json\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* FetchArtifacts Updating matrix file \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \ No newline at end of file diff --git a/integration_tests/src/test/resources/compare/MultipleDevicesIT-android-compare b/integration_tests/src/test/resources/compare/MultipleDevicesIT-android-compare index af53f43d76..a5f7fd7024 100644 --- a/integration_tests/src/test/resources/compare/MultipleDevicesIT-android-compare +++ b/integration_tests/src/test/resources/compare/MultipleDevicesIT-android-compare @@ -1 +1 @@ -AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk test: null additional-apks: auto-google-login: false use-orchestrator: false directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: null device: - model: NexusLowRes version: 28 locale: en orientation: portrait - model: Pixel2 version: 28 locale: en orientation: portrait - model: HUR version: 28 locale: en orientation: portrait num-flaky-test-attempts: 2 test-targets-for-shard: fail-fast: false \s* flank: max-test-shards: 5 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: - app: null test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-success-debug-androidTest.apk - app: null test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-error-debug-androidTest.apk - app: null test: gs:\/\/flank-open-source.appspot.com\/integration\/app-single-success-debug-androidTest.apk run-timeout: -1 legacy-junit-result: false ignore-failed-tests: false output-style: single disable-results-upload: false default-class-test-time: 240.0 \s* RunTests \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 240s, 240s, 360s, 360s, 360s \s* \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 240s, 240s, 360s, 360s, 360s \s* \s* Smart Flank cache hit: 0\% \(0 \/ 1\) Shard times: 120s \s* Saved 3 shards to android_shards.json Uploading \[android_shards.json\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[app-debug.apk\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* [\s\S]* 11 tests \+ 8 parameterized classes \/ 11 shards \s* Uploading \[session_id.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* 3 matrix ids created in \d{1,2}m \d{1,2}s Raw results will be stored in your GCS bucket at \[https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9-]*\/[.a-zA-Z0-9_-]*\] \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* CostReport Physical devices \$\d{1,2}.\d{1,2} for \d{1,2}m \s* Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m \s* Total \$\d{1,2}.\d{1,2} for \d{1,2}m \s* Uploading \[CostReport.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* MatrixResultsReport 2 \/ 3 \(66\.67\%\) 1 matrices failed [\s\S]* More details are available at: https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \s* Uploading \[MatrixResultsReport.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[HtmlErrorReport.html\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[JUnitReport.xml\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* (Uploading \[performanceMetrics.json\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/matrix_[0-9]\/\.*\s*){3} Uploading \[matrix_ids.json\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* FetchArtifacts Updating matrix file \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \ No newline at end of file +AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk test: null additional-apks: auto-google-login: false use-orchestrator: false directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: null device: - model: NexusLowRes version: 28 locale: en orientation: portrait - model: Pixel2 version: 28 locale: en orientation: portrait - model: HUR version: 28 locale: en orientation: portrait num-flaky-test-attempts: 2 test-targets-for-shard: fail-fast: false \s* flank: max-test-shards: 5 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: - app: null test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-success-debug-androidTest.apk - app: null test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-error-debug-androidTest.apk - app: null test: gs:\/\/flank-open-source.appspot.com\/integration\/app-single-success-debug-androidTest.apk run-timeout: -1 legacy-junit-result: false ignore-failed-tests: false output-style: single disable-results-upload: false default-class-test-time: 240.0 disable-usage-statistics: false \s* RunTests \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 240s, 240s, 360s, 360s, 360s \s* \s* Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 240s, 240s, 360s, 360s, 360s \s* \s* Smart Flank cache hit: 0\% \(0 \/ 1\) Shard times: 120s \s* Saved 3 shards to android_shards.json Uploading \[android_shards.json\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[app-debug.apk\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* [\s\S]* 11 tests \+ 8 parameterized classes \/ 11 shards \s* Uploading \[session_id.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* 3 matrix ids created in \d{1,2}m \d{1,2}s Raw results will be stored in your GCS bucket at \[https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9-]*\/[.a-zA-Z0-9_-]*\] \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? [\s\S]* CostReport Physical devices \$\d{1,2}.\d{1,2} for \d{1,2}m \s* Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m \s* Total \$\d{1,2}.\d{1,2} for \d{1,2}m \s* Uploading \[CostReport.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* MatrixResultsReport 2 \/ 3 \(66\.67\%\) 1 matrices failed [\s\S]* More details are available at: https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \s* Uploading \[MatrixResultsReport.txt\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[HtmlErrorReport.html\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* Uploading \[JUnitReport.xml\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* (Uploading \[performanceMetrics.json\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/matrix_[0-9]\/\.*\s*){3} Uploading \[matrix_ids.json\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* FetchArtifacts Updating matrix file \s* Matrices webLink matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)? \ No newline at end of file diff --git a/integration_tests/src/test/resources/compare/RunTimeoutIT-compare b/integration_tests/src/test/resources/compare/RunTimeoutIT-compare index ce7a3bfc4f..c159d3d21d 100644 --- a/integration_tests/src/test/resources/compare/RunTimeoutIT-compare +++ b/integration_tests/src/test/resources/compare/RunTimeoutIT-compare @@ -60,6 +60,7 @@ AndroidArgs output-style: verbose disable-results-upload: true default-class-test-time: 240.0 + disable-usage-statistics: false RunTests Saved 1 shards to android_shards.json diff --git a/integration_tests/src/test/resources/compare/SanityRoboIT-compare b/integration_tests/src/test/resources/compare/SanityRoboIT-compare index 9186f6f6d3..e1d3d3a22c 100644 --- a/integration_tests/src/test/resources/compare/SanityRoboIT-compare +++ b/integration_tests/src/test/resources/compare/SanityRoboIT-compare @@ -60,6 +60,7 @@ AndroidArgs output-style: verbose disable-results-upload: false default-class-test-time: 240.0 + disable-usage-statistics: false RunTests Uploading \[app-debug.apk\] to https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9_-]*\/[.a-zA-Z0-9_-]*\/\.* diff --git a/integration_tests/src/test/resources/compare/TestFilteringIT-compare b/integration_tests/src/test/resources/compare/TestFilteringIT-compare index 3aafeafa3c..436656ea7a 100644 --- a/integration_tests/src/test/resources/compare/TestFilteringIT-compare +++ b/integration_tests/src/test/resources/compare/TestFilteringIT-compare @@ -63,6 +63,7 @@ AndroidArgs output-style: verbose disable-results-upload: false default-class-test-time: 240.0 + disable-usage-statistics: false \s* RunTests No tests for app-single-success-debug-androidTest.apk From 5f8bbaaf60e0a31e6fadb6c0433c18a768f4ef45 Mon Sep 17 00:00:00 2001 From: Adam Date: Tue, 19 Jan 2021 17:00:11 +0100 Subject: [PATCH 15/21] respects the same analytics opt out as gcloud CLI --- .../kotlin/ftl/analytics/SendUsageStatistics.kt | 14 ++++++++------ test_runner/src/main/kotlin/ftl/args/IArgs.kt | 4 ++-- .../src/main/kotlin/ftl/util/CrashReporter.kt | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/test_runner/src/main/kotlin/ftl/analytics/SendUsageStatistics.kt b/test_runner/src/main/kotlin/ftl/analytics/SendUsageStatistics.kt index ba2413109d..460f8031ab 100644 --- a/test_runner/src/main/kotlin/ftl/analytics/SendUsageStatistics.kt +++ b/test_runner/src/main/kotlin/ftl/analytics/SendUsageStatistics.kt @@ -4,14 +4,16 @@ import ftl.args.AndroidArgs import ftl.args.IArgs import ftl.args.IosArgs import ftl.args.blockSendingUsageStatistics +import ftl.util.isGoogleAnalyticsDisabled fun AndroidArgs.sendConfiguration() = sendConfiguration(events = createEventMap()) fun IosArgs.sendConfiguration() = sendConfiguration(events = createEventMap()) -fun IArgs.sendConfiguration(events: Map) = takeUnless { blockSendingUsageStatistics }?.run { - registerUser() - events - .toEvent(project) - .send() -} +fun IArgs.sendConfiguration(events: Map, rootPath: String = System.getProperty("user.home")) = + takeUnless { blockSendingUsageStatistics || isGoogleAnalyticsDisabled(rootPath) }?.run { + registerUser() + events + .toEvent(project) + .send() + } diff --git a/test_runner/src/main/kotlin/ftl/args/IArgs.kt b/test_runner/src/main/kotlin/ftl/args/IArgs.kt index feace6ae7d..36757967fa 100644 --- a/test_runner/src/main/kotlin/ftl/args/IArgs.kt +++ b/test_runner/src/main/kotlin/ftl/args/IArgs.kt @@ -1,6 +1,7 @@ package ftl.args import flank.common.OutputLogLevel +import flank.common.config.isTest import flank.common.setLogLevel import ftl.analytics.AnonymizeInStatistics import ftl.analytics.IgnoreInStatistics @@ -8,7 +9,6 @@ import ftl.args.yml.Type import ftl.config.Device import ftl.config.common.CommonFlankConfig.Companion.defaultLocalResultsDir import ftl.run.status.OutputStyle -import ftl.util.readVersion import ftl.util.timeoutToMils // Properties common to both Android and iOS @@ -116,4 +116,4 @@ fun IArgs.setupLogLevel() { } val IArgs.blockSendingUsageStatistics - get() = disableUsageStatistics || readVersion() == "local_snapshot" + get() = disableUsageStatistics || isTest() diff --git a/test_runner/src/main/kotlin/ftl/util/CrashReporter.kt b/test_runner/src/main/kotlin/ftl/util/CrashReporter.kt index c6fc30e2a8..6029e451db 100644 --- a/test_runner/src/main/kotlin/ftl/util/CrashReporter.kt +++ b/test_runner/src/main/kotlin/ftl/util/CrashReporter.kt @@ -43,7 +43,7 @@ internal fun initCrashReporter( else -> initializeCrashReportWrapper() } -private fun isGoogleAnalyticsDisabled(rootPath: String) = +internal fun isGoogleAnalyticsDisabled(rootPath: String) = File(rootPath, "$GSUTIL_FOLDER/$ANALYTICS_FILE").run { exists() && readText().trim() == DISABLED } private fun initializeCrashReportWrapper() { From 1d476643845617f91b449e9f0ea656b6dc17da4b Mon Sep 17 00:00:00 2001 From: adamfilipow92 <64852261+adamfilipow92@users.noreply.github.com> Date: Wed, 20 Jan 2021 10:30:52 +0100 Subject: [PATCH 16/21] Update test_runner/src/main/kotlin/ftl/analytics/UsageStatisticsClient.kt Co-authored-by: piotradamczyk5 <65554637+piotradamczyk5@users.noreply.github.com> --- .../src/main/kotlin/ftl/analytics/UsageStatisticsClient.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_runner/src/main/kotlin/ftl/analytics/UsageStatisticsClient.kt b/test_runner/src/main/kotlin/ftl/analytics/UsageStatisticsClient.kt index 880ff1e5c8..f6c480a1e9 100644 --- a/test_runner/src/main/kotlin/ftl/analytics/UsageStatisticsClient.kt +++ b/test_runner/src/main/kotlin/ftl/analytics/UsageStatisticsClient.kt @@ -31,5 +31,5 @@ internal fun JSONObject.send() = apiClient.sendMessage(this) internal fun Map.toEvent(projectId: String) = (this + Pair(SESSION_ID, sessionId)).run { - messageBuilder.event(projectId, CONFIGURATION_KEY, JSONObject(this)) + messageBuilder.event(projectId, CONFIGURATION_KEY, toJSONObject()) } From 8b4ef6732853d41984e833b336b677d82c1d638a Mon Sep 17 00:00:00 2001 From: adamfilipow92 <64852261+adamfilipow92@users.noreply.github.com> Date: Wed, 20 Jan 2021 10:31:05 +0100 Subject: [PATCH 17/21] Update test_runner/src/main/kotlin/ftl/analytics/SendUsageStatistics.kt Co-authored-by: piotradamczyk5 <65554637+piotradamczyk5@users.noreply.github.com> --- .../src/main/kotlin/ftl/analytics/SendUsageStatistics.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_runner/src/main/kotlin/ftl/analytics/SendUsageStatistics.kt b/test_runner/src/main/kotlin/ftl/analytics/SendUsageStatistics.kt index 460f8031ab..ac6474c67a 100644 --- a/test_runner/src/main/kotlin/ftl/analytics/SendUsageStatistics.kt +++ b/test_runner/src/main/kotlin/ftl/analytics/SendUsageStatistics.kt @@ -10,7 +10,7 @@ fun AndroidArgs.sendConfiguration() = sendConfiguration(events = createEventMap( fun IosArgs.sendConfiguration() = sendConfiguration(events = createEventMap()) -fun IArgs.sendConfiguration(events: Map, rootPath: String = System.getProperty("user.home")) = +fun IArgs.sendConfiguration(events: Map, rootPath: String = userHome) = takeUnless { blockSendingUsageStatistics || isGoogleAnalyticsDisabled(rootPath) }?.run { registerUser() events From 9960a37aad1366d0cb4ae44e52cc8774e92f704a Mon Sep 17 00:00:00 2001 From: Adam Date: Wed, 20 Jan 2021 10:37:32 +0100 Subject: [PATCH 18/21] Implementation improvements --- .../ftl/analytics/StatisticDataFilters.kt | 23 ++++++++++--------- .../kotlin/ftl/analytics/UsageStatistics.kt | 2 +- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/test_runner/src/main/kotlin/ftl/analytics/StatisticDataFilters.kt b/test_runner/src/main/kotlin/ftl/analytics/StatisticDataFilters.kt index de01d82f62..6a9dcf3997 100644 --- a/test_runner/src/main/kotlin/ftl/analytics/StatisticDataFilters.kt +++ b/test_runner/src/main/kotlin/ftl/analytics/StatisticDataFilters.kt @@ -5,30 +5,31 @@ import ftl.args.IArgs import ftl.args.IosArgs import kotlin.reflect.KClass -private const val ANONYMIZE_VALUE = "..." - annotation class IgnoreInStatistics annotation class AnonymizeInStatistics -private val classesForStatistics = listOf(IArgs::class, AndroidArgs::class, IosArgs::class) +internal val keysToRemove by lazy { + classesForStatistics.map { it.ignoredMembersForStatistics() }.flatten() +} private fun KClass<*>.ignoredMembersForStatistics() = findMembersWithAnnotation(IgnoreInStatistics::class) + +internal val keysToAnonymize by lazy { + classesForStatistics.map { it.anonymousMembersForStatistics() }.flatten() +} + private fun KClass<*>.anonymousMembersForStatistics() = findMembersWithAnnotation(AnonymizeInStatistics::class) +private val classesForStatistics = listOf(IArgs::class, AndroidArgs::class, IosArgs::class) + +private const val ANONYMIZE_VALUE = "..." + private fun KClass<*>.findMembersWithAnnotation(annotationType: KClass<*>) = members.filter { member -> member.annotations.any { annotation -> annotation.annotationClass == annotationType } }.map { it.name } -internal val keysToRemove by lazy { - classesForStatistics.map { it.ignoredMembersForStatistics() }.flatten() -} - -internal val keysToAnonymize by lazy { - classesForStatistics.map { it.anonymousMembersForStatistics() }.flatten() -} - internal fun Map.removeNotNeededKeys(defaultArgs: Map) = filterNot { (key, value) -> value == defaultArgs[key] || key in keysToRemove diff --git a/test_runner/src/main/kotlin/ftl/analytics/UsageStatistics.kt b/test_runner/src/main/kotlin/ftl/analytics/UsageStatistics.kt index bdde643de6..bf7fb33d8b 100644 --- a/test_runner/src/main/kotlin/ftl/analytics/UsageStatistics.kt +++ b/test_runner/src/main/kotlin/ftl/analytics/UsageStatistics.kt @@ -16,4 +16,4 @@ internal fun Map.getNonDefaultArgs( defaultArgs: Map ) = removeNotNeededKeys(defaultArgs).filterSensitiveValues() -private fun Map<*, *>.toJSONObject() = JSONObject(this) +internal fun Map<*, *>.toJSONObject() = JSONObject(this) From 67e3c232c6ae1a004ce3ea603f32b9a74dded277 Mon Sep 17 00:00:00 2001 From: Adam Date: Wed, 20 Jan 2021 10:45:11 +0100 Subject: [PATCH 19/21] Update docs --- docs/index.md | 10 ++++++++++ test_runner/flank.ios.yml | 5 +++++ test_runner/flank.yml | 5 +++++ 3 files changed, 20 insertions(+) diff --git a/docs/index.md b/docs/index.md index 4e3e5a323f..e604b542a6 100644 --- a/docs/index.md +++ b/docs/index.md @@ -320,6 +320,11 @@ flank: ## Disables flank results upload on gcloud storage. ## Default: false # disable-results-upload: false + + ### Disable usage statistics flag + ## Disable sending usage statistics (without sensitive data) to the analytic tool. + ## Default: false + # disable-usage-statistics: false ``` ### Android example @@ -666,6 +671,11 @@ flank: ## Disables flank results upload on gcloud storage. ## Default: false # disable-results-upload: false + + ### Disable usage statistics flag + ## Disable sending usage statistics (without sensitive data) to the analytic tool. + ## Default: false + # disable-usage-statistics: false ``` ## Android code coverage diff --git a/test_runner/flank.ios.yml b/test_runner/flank.ios.yml index 67f66f8362..4f3c74c51f 100644 --- a/test_runner/flank.ios.yml +++ b/test_runner/flank.ios.yml @@ -251,3 +251,8 @@ flank: ## Disables flank results upload on gcloud storage. ## Default: false # disable-results-upload: false + + ### Disable usage statistics flag + ## Disable sending usage statistics (without sensitive data) to the analytic tool. + ## Default: false + # disable-usage-statistics: false diff --git a/test_runner/flank.yml b/test_runner/flank.yml index de262c2a7b..fd0f5ad610 100644 --- a/test_runner/flank.yml +++ b/test_runner/flank.yml @@ -342,3 +342,8 @@ flank: # test-target-for-shard: # - package com.package1.for.shard1 # - class com.package2.for.shard2.Class + + ### Disable usage statistics flag + ## Disable sending usage statistics (without sensitive data) to the analytic tool. + ## Default: false + # disable-usage-statistics: false From 2e5f40b28a478a698fda9ddf9ded009b37f38729 Mon Sep 17 00:00:00 2001 From: Adam Date: Wed, 20 Jan 2021 10:51:29 +0100 Subject: [PATCH 20/21] Update SendUsageStatistics.kt --- test_runner/src/main/kotlin/ftl/analytics/SendUsageStatistics.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/test_runner/src/main/kotlin/ftl/analytics/SendUsageStatistics.kt b/test_runner/src/main/kotlin/ftl/analytics/SendUsageStatistics.kt index ac6474c67a..6af70e452c 100644 --- a/test_runner/src/main/kotlin/ftl/analytics/SendUsageStatistics.kt +++ b/test_runner/src/main/kotlin/ftl/analytics/SendUsageStatistics.kt @@ -1,5 +1,6 @@ package ftl.analytics +import flank.common.userHome import ftl.args.AndroidArgs import ftl.args.IArgs import ftl.args.IosArgs From c91227f727223124482d4b07c902f363f8d6d524 Mon Sep 17 00:00:00 2001 From: Piotr Adamczyk Date: Wed, 20 Jan 2021 15:01:36 +0100 Subject: [PATCH 21/21] fixed MR comments --- build.gradle.kts | 4 +++- .../ftl/analytics/StatisticDataFilters.kt | 24 +++++++++---------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 268f8b338a..1dc09f2324 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -21,7 +21,9 @@ plugins { tasks { "lintKotlinMain"(LintTask::class) { exclude( - "**/*Generated.kt" // we can expand this list + "**/*Generated.kt", + "**/*Test.kt", + "**/Test*.kt" // we can expand this list ) } } diff --git a/test_runner/src/main/kotlin/ftl/analytics/StatisticDataFilters.kt b/test_runner/src/main/kotlin/ftl/analytics/StatisticDataFilters.kt index 6a9dcf3997..1bf12fc827 100644 --- a/test_runner/src/main/kotlin/ftl/analytics/StatisticDataFilters.kt +++ b/test_runner/src/main/kotlin/ftl/analytics/StatisticDataFilters.kt @@ -9,25 +9,23 @@ annotation class IgnoreInStatistics annotation class AnonymizeInStatistics internal val keysToRemove by lazy { - classesForStatistics.map { it.ignoredMembersForStatistics() }.flatten() + classesForStatistics.map(findMembersWithAnnotation(IgnoreInStatistics::class)).flatten() } -private fun KClass<*>.ignoredMembersForStatistics() = findMembersWithAnnotation(IgnoreInStatistics::class) - internal val keysToAnonymize by lazy { - classesForStatistics.map { it.anonymousMembersForStatistics() }.flatten() + classesForStatistics.map(findMembersWithAnnotation(AnonymizeInStatistics::class)).flatten() } -private fun KClass<*>.anonymousMembersForStatistics() = findMembersWithAnnotation(AnonymizeInStatistics::class) - private val classesForStatistics = listOf(IArgs::class, AndroidArgs::class, IosArgs::class) -private const val ANONYMIZE_VALUE = "..." - -private fun KClass<*>.findMembersWithAnnotation(annotationType: KClass<*>) = members.filter { member -> - member.annotations.any { annotation -> annotation.annotationClass == annotationType } -}.map { - it.name +private fun findMembersWithAnnotation( + annotationType: KClass<*> +): KClass<*>.() -> List = { + members.filter { member -> + member.annotations.any { annotation -> annotation.annotationClass == annotationType } + }.map { + it.name + } } internal fun Map.removeNotNeededKeys(defaultArgs: Map) = @@ -46,3 +44,5 @@ private fun Any.toAnonymous(): Any = when (this) { is List<*> -> "Count: $size" else -> ANONYMIZE_VALUE } + +private const val ANONYMIZE_VALUE = "..."