Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added disableSharding flag #473

Merged
merged 5 commits into from
Jan 24, 2019
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Empty file added test_app/apks/invalid.apk
Empty file.
4 changes: 4 additions & 0 deletions test_runner/src/main/kotlin/ftl/args/AndroidArgs.kt
Original file line number Diff line number Diff line change
@@ -70,9 +70,12 @@ class AndroidArgs(
override val smartFlankGcsPath = flank.smartFlankGcsPath
override val testTargetsAlwaysRun = cli?.testTargetsAlwaysRun ?: flank.testTargetsAlwaysRun
override val filesToDownload = cli?.filesToDownload ?: flank.filesToDownload
override val disableSharding = cli?.disableSharding ?: flank.disableSharding

// computed properties not specified in yaml
override val testShardChunks: List<List<String>> by lazy {
if (disableSharding) return@lazy listOf(emptyList<String>())

// Download test APK if necessary so it can be used to validate test methods
var testLocalApk = testApk
if (testApk.startsWith(FtlConstants.GCS_PREFIX)) {
@@ -168,6 +171,7 @@ ${devicesToString(devices)}
${listToString(filesToDownload)}
test-targets-always-run:
${listToString(testTargetsAlwaysRun)}
disableSharding: $disableSharding
""".trimIndent()
}

1 change: 1 addition & 0 deletions test_runner/src/main/kotlin/ftl/args/IArgs.kt
Original file line number Diff line number Diff line change
@@ -21,6 +21,7 @@ interface IArgs {
val smartFlankGcsPath: String
val testTargetsAlwaysRun: List<String>
val filesToDownload: List<String>
val disableSharding: Boolean

// computed property
val testShardChunks: List<List<String>>
2 changes: 2 additions & 0 deletions test_runner/src/main/kotlin/ftl/args/IosArgs.kt
Original file line number Diff line number Diff line change
@@ -55,6 +55,7 @@ class IosArgs(
override val smartFlankGcsPath = flank.smartFlankGcsPath
override val testTargetsAlwaysRun = cli?.testTargetsAlwaysRun ?: flank.testTargetsAlwaysRun
override val filesToDownload = cli?.filesToDownload ?: flank.filesToDownload
override val disableSharding = cli?.disableSharding ?: flank.disableSharding

private val iosFlank = iosFlankYml.flank
val testTargets = cli?.testTargets ?: iosFlank.testTargets
@@ -133,6 +134,7 @@ ${listToString(filesToDownload)}
# iOS flank
test-targets:
${listToString(testTargets)}
disableSharding: $disableSharding
""".trimIndent()
}

3 changes: 2 additions & 1 deletion test_runner/src/main/kotlin/ftl/args/yml/FlankYml.kt
Original file line number Diff line number Diff line change
@@ -12,6 +12,7 @@ class FlankYmlParams(
val shardTime: Int = -1,
val repeatTests: Int = 1,
val smartFlankGcsPath: String = "",
val disableSharding: Boolean = false,

@field:JsonProperty("test-targets-always-run")
val testTargetsAlwaysRun: List<String> = emptyList(),
@@ -21,7 +22,7 @@ class FlankYmlParams(
) {
companion object : IYmlKeys {
override val keys = listOf(
"testShards", "shardTime", "repeatTests", "smartFlankGcsPath", "test-targets-always-run", "files-to-download"
"testShards", "shardTime", "repeatTests", "smartFlankGcsPath", "disableSharding", "test-targets-always-run", "files-to-download"
)
}

3 changes: 2 additions & 1 deletion test_runner/src/main/kotlin/ftl/args/yml/GcloudYml.kt
Original file line number Diff line number Diff line change
@@ -38,7 +38,8 @@ class GcloudYmlParams(
) {
companion object : IYmlKeys {
override val keys =
listOf("results-bucket", "results-dir", "record-video", "timeout", "async", "project", "results-history-name", "flaky-test-attempts")
listOf("results-bucket", "results-dir", "record-video", "timeout", "async", "project",
"results-history-name", "flaky-test-attempts")
}

init {
Original file line number Diff line number Diff line change
@@ -251,6 +251,12 @@ class AndroidRunCommand : Runnable {
)
var filesToDownload: List<String>? = null

@Option(
names = ["--disable-sharding"],
description = ["Disable sharding."]
)
var disableSharding: Boolean? = null

@Option(
names = ["--results-dir"],
description = [
Original file line number Diff line number Diff line change
@@ -142,6 +142,12 @@ class IosRunCommand : Runnable {
)
var filesToDownload: List<String>? = null

@Option(
names = ["--disable-sharding"],
description = ["Disable sharding."]
)
var disableSharding: Boolean? = null

@Option(
names = ["--test"],
description = ["The path to the test package (a zip file containing the iOS app " +
4 changes: 2 additions & 2 deletions test_runner/src/main/kotlin/ftl/gc/GcIosTestMatrix.kt
Original file line number Diff line number Diff line change
@@ -43,8 +43,8 @@ object GcIosTestMatrix {
val methods = args.testShardChunks.elementAt(testShardsIndex)

// Parameterized tests on iOS don't shard correctly.
// Avoid changing Xctestrun file when test shards is 1.
val generatedXctestrun = if (args.testShards == 1) {
// Avoid changing Xctestrun file when disableSharding is on.
val generatedXctestrun = if (args.disableSharding) {
xcTestParsed.toByteArray()
} else {
Xctestrun.rewrite(xcTestParsed, methods)
46 changes: 45 additions & 1 deletion test_runner/src/test/kotlin/ftl/args/AndroidArgsTest.kt
Original file line number Diff line number Diff line change
@@ -18,6 +18,7 @@ import picocli.CommandLine
class AndroidArgsTest {
private val empty = emptyList<String>()
private val appApk = "../test_app/apks/app-debug.apk"
private val invalidApk = "../test_app/apks/invalid.apk"
private val testApk = "../test_app/apks/app-debug-androidTest.apk"
private val testErrorApk = "../test_app/apks/error-androidTest.apk"
private val appApkAbsolutePath = appApk.absolutePath()
@@ -67,6 +68,7 @@ class AndroidArgsTest {
test-targets-always-run:
- class example.Test#grantPermission
- class example.Test#grantPermission2
disableSharding: true
"""

@Rule
@@ -168,6 +170,7 @@ class AndroidArgsTest {
"class example.Test#grantPermission2"
)
)
assert(disableSharding, true)
}
}

@@ -222,6 +225,7 @@ AndroidArgs
test-targets-always-run:
- class example.Test#grantPermission
- class example.Test#grantPermission2
disableSharding: true
""".trimIndent()
)
}
@@ -261,6 +265,7 @@ AndroidArgs
assert(repeatTests, 1)
assert(filesToDownload, empty)
assert(testTargetsAlwaysRun, empty)
assert(disableSharding, false)
}
}

@@ -300,6 +305,28 @@ AndroidArgs
assertThat(androidArgs.testApk).isEqualTo(testApkAbsolutePath)
}

@Test
fun `disableSharding allows using invalid apk`() {
val yaml = """
gcloud:
app: $invalidApk
test: $invalidApk
flank:
disableSharding: true
"""
AndroidArgs.load(yaml).testShardChunks
}

@Test(expected = RuntimeException::class)
fun `Invalid apk throws`() {
val yaml = """
gcloud:
app: $invalidApk
test: $invalidApk
"""
AndroidArgs.load(yaml).testShardChunks
}

@Test
fun cli_app() {
val cli = AndroidRunCommand()
@@ -673,7 +700,7 @@ AndroidArgs
}

@Test
fun `cli shardTime`() {
fun cli_shardTime() {
val cli = AndroidRunCommand()
CommandLine(cli).parse("--shard-time=3")

@@ -689,6 +716,23 @@ AndroidArgs
assertThat(AndroidArgs.load(yaml, cli).shardTime).isEqualTo(3)
}

@Test
fun cli_disableSharding() {
val cli = AndroidRunCommand()
CommandLine(cli).parse("--disable-sharding")

val yaml = """
gcloud:
app: $appApk
test: $testApk

flank:
disableSharding: false
"""
assertThat(AndroidArgs.load(yaml).disableSharding).isEqualTo(false)
assertThat(AndroidArgs.load(yaml, cli).disableSharding).isEqualTo(true)
}

@Test
fun cli_repeatTests() {
val cli = AndroidRunCommand()
23 changes: 22 additions & 1 deletion test_runner/src/test/kotlin/ftl/args/IosArgsTest.kt
Original file line number Diff line number Diff line change
@@ -63,6 +63,7 @@ class IosArgsTest {
test-targets:
- b/testBasicSelection
- b/testBasicSelection2
disableSharding: true
"""

@Rule
@@ -128,6 +129,7 @@ class IosArgsTest {
assert(testTargets, listOf("b/testBasicSelection", "b/testBasicSelection2"))

assert(flakyTestAttempts, 4)
assert(disableSharding, true)
}
}

@@ -174,6 +176,7 @@ IosArgs
test-targets:
- b/testBasicSelection
- b/testBasicSelection2
disableSharding: true
""".trimIndent()
)
}
@@ -208,6 +211,7 @@ IosArgs
assert(repeatTests, 1)
assert(testTargetsAlwaysRun, emptyList<String>())
assert(filesToDownload, emptyList<String>())
assert(disableSharding, false)

// IosFlankYml
assert(testTargets, empty)
@@ -377,7 +381,7 @@ IosArgs
}

@Test
fun `cli shardTime`() {
fun cli_shardTime() {
val cli = IosRunCommand()
CommandLine(cli).parse("--shard-time=3")

@@ -393,6 +397,23 @@ IosArgs
assertThat(IosArgs.load(yaml, cli).shardTime).isEqualTo(3)
}

@Test
fun cli_disableSharding() {
val cli = IosRunCommand()
CommandLine(cli).parse("--disable-sharding")

val yaml = """
gcloud:
test: $testPath
xctestrun-file: $xctestrunFile

flank:
disableSharding: false
"""
assertThat(IosArgs.load(yaml).disableSharding).isEqualTo(false)
assertThat(IosArgs.load(yaml, cli).disableSharding).isEqualTo(true)
}

@Test
fun cli_repeatTests() {
val cli = IosRunCommand()
Original file line number Diff line number Diff line change
@@ -84,6 +84,7 @@ class AndroidRunCommandTest {
assertThat(cmd.filesToDownload).isNull()
assertThat(cmd.resultsDir).isNull()
assertThat(cmd.flakyTestAttempts).isNull()
assertThat(cmd.disableSharding).isNull()
}

@Test
@@ -300,4 +301,12 @@ class AndroidRunCommandTest {

assertThat(cmd.shardTime).isEqualTo(99)
}

@Test
fun `disableSharding parse`() {
val cmd = AndroidRunCommand()
CommandLine(cmd).parse("--disable-sharding")

assertThat(cmd.disableSharding).isEqualTo(true)
}
}
Original file line number Diff line number Diff line change
@@ -71,6 +71,7 @@ class IosRunCommandTest {
assertThat(cmd.testTargetsAlwaysRun).isNull()
assertThat(cmd.testTargets).isNull()
assertThat(cmd.filesToDownload).isNull()
assertThat(cmd.disableSharding).isNull()
assertThat(cmd.test).isNull()
assertThat(cmd.xctestrunFile).isNull()
assertThat(cmd.xcodeVersion).isNull()
@@ -234,4 +235,12 @@ class IosRunCommandTest {

assertThat(cmd.shardTime).isEqualTo(99)
}

@Test
fun `disableShard parse`() {
val cmd = IosRunCommand()
CommandLine(cmd).parse("--disable-sharding")

assertThat(cmd.disableSharding).isEqualTo(true)
}
}