Skip to content

Commit

Permalink
feat(ios): add xctest parser (#848)
Browse files Browse the repository at this point in the history
Co-authored-by: Konstantin Aksenov <aksenov.kostya@gmail.com>
  • Loading branch information
Malinskiy and Vacxe authored Oct 21, 2023
1 parent 3f3959e commit d140e93
Show file tree
Hide file tree
Showing 21 changed files with 578 additions and 28 deletions.
4 changes: 3 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,8 @@ See [contributing docs][contributing]

## License

See [LICENSE][LICENSE]
Marathon codebase is GPL 2.0 [LICENSE][LICENSE] with following optional components under specific licenses:
* [libxctest-parser][libxctest-parser-license]

<!--
Repo References
Expand All @@ -161,5 +162,6 @@ Link References
[contributing]:https://docs.marathonlabs.io/intro/contribute
[prs]:http://makeapullrequest.com "Make a Pull Request (external link) ➶"
[LICENSE]:https://github.com/MarathonLabs/marathon/blob/-/LICENSE
[libxctest-parser-license]: https://github.com/MarathonLabs/marathon/blob/-/vendor/vendor-ios/src/main/resources/EULA.md

[marathon-cloud]:https://marathonlabs.io
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,7 @@ sealed class VendorConfiguration {
@JsonProperty("compactOutput") val compactOutput: Boolean = false,
@JsonProperty("rsync") val rsync: RsyncConfiguration = RsyncConfiguration(),
@JsonProperty("xcodebuildTestArgs") val xcodebuildTestArgs: Map<String, String> = emptyMap(),
@JsonProperty("testParserConfiguration") val testParserConfiguration: com.malinskiy.marathon.config.vendor.ios.TestParserConfiguration = com.malinskiy.marathon.config.vendor.ios.TestParserConfiguration.NmTestParserConfiguration,

@JsonProperty("signing") val signing: SigningConfiguration = SigningConfiguration(),
) : VendorConfiguration() {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
package com.malinskiy.marathon.config.vendor.ios

import com.fasterxml.jackson.annotation.JsonSubTypes
import com.fasterxml.jackson.annotation.JsonTypeInfo

@JsonTypeInfo(
use = JsonTypeInfo.Id.NAME,
include = JsonTypeInfo.As.PROPERTY,
property = "type"
)
@JsonSubTypes(
JsonSubTypes.Type(value = TestParserConfiguration.NmTestParserConfiguration::class, name = "nm"),
JsonSubTypes.Type(value = TestParserConfiguration.XCTestParserConfiguration::class, name = "xctest"),
)
sealed class TestParserConfiguration {
object NmTestParserConfiguration : TestParserConfiguration()
object XCTestParserConfiguration : TestParserConfiguration()
}
42 changes: 41 additions & 1 deletion docs/docs/ios/configure.md
Original file line number Diff line number Diff line change
Expand Up @@ -357,7 +357,7 @@ timeoutConfiguration:
screenshot: PT10S
video: PT300S
erase: PT30S
shutdown: PT30S
shutdown: PT30S
delete: PT30S
create: PT30S
boot: PT30S
Expand Down Expand Up @@ -454,6 +454,46 @@ rsync:
remotePath: "/usr/bin/rsync-custom"
```

### Test parser

:::tip

If you need to parallelize the execution of tests generated at runtime
(i.e. flutter) - xctest parser is your choice.

:::

Test parsing (collecting a list of tests expected to execute) can be done using either binary inspection using nm,
or injecting marathon's proprietary blob and allows marathon to collect a list of tests expected to run without actually running them.

:::note

We don't provide source code for the libxctest-parser module. By using libxctest-parser you're automatically accepting it's [EULA][libxctest-parser-license]

:::

| YAML type | Pros | Const |
|-----------|-------------------------------------------------------------------------------------------------------------------:|-----------------------------------------------------------------------------------------------------------:|
| "nm" | Doesn't require installation of apps onto the device | Doesn't support runtime-generated tests, e.g. flutter |
| "xctest" | Supports precise test parsing and any runtime-generated tests hence allows marathon to parallelize their execution | Requires a booted iOS device for parsing and a fake test run including installation of test app under test |

Default test parser is nm.

<Tabs>
<TabItem value="YAML" label="Marathonfile">

```yaml
vendorConfiguration:
type: "iOS"
testParserConfiguration:
type: "xctest"
```

</TabItem>
</Tabs>


[1]: workers.md
[2]: /configuration/dynamic-configuration.md
[3]: https://en.wikipedia.org/wiki/ISO_8601
[libxctest-parser-license]: https://github.com/MarathonLabs/marathon/blob/-/vendor/vendor-ios/src/main/resources/EULA.md
1 change: 1 addition & 0 deletions vendor/vendor-ios/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ dependencies {
implementation(Libraries.kotlinProcess)
implementation(project(":core"))
testImplementation(TestLibraries.kluent)
testImplementation(TestLibraries.assertk)
testImplementation(TestLibraries.mockitoKotlin)
testImplementation(TestLibraries.testContainers)
testImplementation(TestLibraries.testContainersJupiter)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ class AppleApplicationInstaller(
) {
private val logger = MarathonLogging.logger {}

suspend fun prepareInstallation(device: AppleSimulatorDevice) {
suspend fun prepareInstallation(device: AppleSimulatorDevice, useXctestParser: Boolean = false) {
val bundle = vendorConfiguration.bundle ?: throw ConfigurationException("no xctest found for configuration")

val xctest = bundle.xctest
Expand All @@ -44,7 +44,7 @@ class AppleApplicationInstaller(
}
val remoteTestBinary = device.remoteFileManager.joinPath(remoteXctest, testBinary.name)
val testType = getTestTypeFor(device, device.sdk, remoteTestBinary)
TestRootFactory(device, vendorConfiguration).generate(testType, bundle)
TestRootFactory(device, vendorConfiguration).generate(testType, bundle, useXctestParser)
grantPermissions(device)

bundle.extraApplications?.forEach {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@ import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator
import com.fasterxml.jackson.module.kotlin.KotlinFeature
import com.fasterxml.jackson.module.kotlin.KotlinModule
import com.google.gson.GsonBuilder
import com.malinskiy.marathon.config.Configuration
import com.malinskiy.marathon.config.vendor.VendorConfiguration
import com.malinskiy.marathon.config.vendor.ios.TestParserConfiguration
import com.malinskiy.marathon.device.DeviceProvider
import com.malinskiy.marathon.execution.TestParser
import com.malinskiy.marathon.execution.bundle.TestBundleIdentifier
Expand All @@ -32,7 +35,20 @@ val AppleVendor = module {
)
AppleDeviceProvider(get(), get(), get(), gson, objectMapper, get(), get())
}
single<TestParser?> { AppleTestParser(get(), get(), get()) }
single<TestParser?> {
val configuration = get<Configuration>()
val iosConfiguration = configuration.vendorConfiguration as? VendorConfiguration.IOSConfiguration
val testParserConfiguration = iosConfiguration?.testParserConfiguration
when {
testParserConfiguration != null && testParserConfiguration is TestParserConfiguration.XCTestParserConfiguration -> XCTestParser(
get(),
get(),
get()
)

else -> NmTestParser(get(), get(), get())
}
}
single<MarathonLogConfigurator> { AppleLogConfigurator(get()) }

val appleTestBundleIdentifier = AppleTestBundleIdentifier()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import com.malinskiy.marathon.config.Configuration
import com.malinskiy.marathon.config.exceptions.ConfigurationException
import com.malinskiy.marathon.config.vendor.VendorConfiguration
import com.malinskiy.marathon.device.Device
import com.malinskiy.marathon.device.DeviceProvider
import com.malinskiy.marathon.exceptions.TestParsingException
import com.malinskiy.marathon.execution.RemoteTestParser
import com.malinskiy.marathon.execution.withRetry
Expand All @@ -14,12 +13,12 @@ import com.malinskiy.marathon.test.Test
import kotlinx.coroutines.CancellationException
import java.io.File

class AppleTestParser(
class NmTestParser(
private val configuration: Configuration,
private val vendorConfiguration: VendorConfiguration.IOSConfiguration,
private val testBundleIdentifier: AppleTestBundleIdentifier
) : RemoteTestParser<AppleDeviceProvider> {
private val logger = MarathonLogging.logger(AppleTestParser::class.java.simpleName)
private val logger = MarathonLogging.logger(NmTestParser::class.java.simpleName)

override suspend fun extract(device: Device): List<Test> {
val app = vendorConfiguration.bundle?.app ?: throw IllegalArgumentException("No application bundle provided")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ class RemoteFileManager(private val device: AppleDevice) {
fun remoteXctestrunFile(): String = remoteFile(xctestrunFileName())

fun remoteXctestFile(): String = remoteFile(xctestFileName())
fun remoteXctestParserFile(): String = remoteFile(`libXctestParserFileName`())
fun remoteApplication(): String = remoteFile(appUnderTestFileName())
fun remoteExtraApplication(name: String) = remoteFile(name)

Expand All @@ -52,6 +53,8 @@ class RemoteFileManager(private val device: AppleDevice) {
fun xctestrunFileName(): String = "marathon.xctestrun"

private fun xctestFileName(): String = "marathon.xctest"
private fun libXctestParserFileName(): String = "libxctest-parser.dylib"

fun appUnderTestFileName(): String = "appUnderTest.app"

private fun xcresultFileName(batch: TestBatch): String =
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
package com.malinskiy.marathon.ios

import com.malinskiy.marathon.config.Configuration
import com.malinskiy.marathon.config.exceptions.ConfigurationException
import com.malinskiy.marathon.config.vendor.VendorConfiguration
import com.malinskiy.marathon.device.Device
import com.malinskiy.marathon.exceptions.TestParsingException
import com.malinskiy.marathon.execution.RemoteTestParser
import com.malinskiy.marathon.execution.withRetry
import com.malinskiy.marathon.ios.model.AppleTestBundle
import com.malinskiy.marathon.ios.test.TestEvent
import com.malinskiy.marathon.ios.test.TestRequest
import com.malinskiy.marathon.ios.test.TestStarted
import com.malinskiy.marathon.log.MarathonLogging
import com.malinskiy.marathon.test.Test
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.channels.ReceiveChannel
import kotlin.io.path.outputStream

class XCTestParser(
private val configuration: Configuration,
private val vendorConfiguration: VendorConfiguration.IOSConfiguration,
private val testBundleIdentifier: AppleTestBundleIdentifier
) : RemoteTestParser<AppleDeviceProvider> {
private val logger = MarathonLogging.logger(XCTestParser::class.java.simpleName)

override suspend fun extract(device: Device): List<Test> {
return withRetry(3, 0) {
try {
val device =
device as? AppleSimulatorDevice ?: throw ConfigurationException("Unexpected device type for remote test parsing")
return@withRetry parseTests(device, configuration, vendorConfiguration)
} catch (e: CancellationException) {
throw e
} catch (e: Exception) {
logger.debug(e) { "Remote parsing failed. Retrying" }
throw e
}
}
}

private suspend fun parseTests(
device: AppleSimulatorDevice,
configuration: Configuration,
vendorConfiguration: VendorConfiguration.IOSConfiguration
): List<Test> {
val appleApplicationInstaller = AppleApplicationInstaller(configuration, vendorConfiguration, testBundleIdentifier)
appleApplicationInstaller.prepareInstallation(device, useXctestParser = true)

val dylib = javaClass.getResourceAsStream("/libxctest-parser.dylib")
val tempFile = kotlin.io.path.createTempFile().apply {
outputStream().use {
dylib.copyTo(it)
}
}.toFile()
val remoteLibParseTests = device.remoteFileManager.remoteXctestParserFile()
if (!device.pushFile(tempFile, remoteLibParseTests)) {
throw TestParsingException("failed to push libparse-tests.dylib for test parsing")
}

val remoteXctestrunFile = device.remoteFileManager.remoteXctestrunFile()
val remoteDir = device.remoteFileManager.parentOf(remoteXctestrunFile)

logger.debug("Remote xctestrun = $remoteXctestrunFile")

val runnerRequest = TestRequest(
workdir = remoteDir,
remoteXctestrun = remoteXctestrunFile,
coverage = false,
)
var channel: ReceiveChannel<List<TestEvent>>? = null
var tests = mutableSetOf<Test>()
try {
val localChannel = device.executeTestRequest(runnerRequest)
channel = localChannel
for (events in localChannel) {
for (event in events) {
when (event) {
is TestStarted -> {
tests.add(event.id)
}
else -> Unit
}
}
}

logger.debug { "Execution finished" }
} catch (e: CancellationException) {
val errorMessage = "Test parsing got stuck. " +
"You can increase the timeout in settings if it's too strict"
logger.error(e) { errorMessage }
} finally {
channel?.cancel()
}

val xctest = vendorConfiguration.bundle?.xctest ?: throw IllegalArgumentException("No test bundle provided")
val possibleTestBinaries = xctest.listFiles()?.filter { it.isFile && it.extension == "" }
?: throw ConfigurationException("missing test binaries in xctest folder at $xctest")
val testBinary = when (possibleTestBinaries.size) {
0 -> throw ConfigurationException("missing test binaries in xctest folder at $xctest")
1 -> possibleTestBinaries[0]
else -> {
logger.warn { "Multiple test binaries present in xctest folder" }
possibleTestBinaries.find { it.name == xctest.nameWithoutExtension } ?: possibleTestBinaries.first()
}
}

val testBundle = AppleTestBundle(vendorConfiguration.bundle?.application, xctest, testBinary)
val result = tests.toList()
result.forEach { testBundleIdentifier.put(it, testBundle) }

return result
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ class Xcodebuild(
val args = mutableMapOf<String, String>().apply {
putAll(vendorConfiguration.xcodebuildTestArgs)
put("-enableCodeCoverage", codeCoverageFlag(request))
put("-resultBundlePath", request.xcresult)
request.xcresult?.let { put("-resultBundlePath", it) }
put("-destination-timeout", timeoutConfiguration.testDestination.seconds.toString())
put("-destination", "\'platform=iOS simulator,id=$udid\'")
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,9 @@ class TestRunProgressParser(

val logger = MarathonLogging.logger(TestRunProgressParser::class.java.simpleName)

val TEST_CASE_STARTED = """Test Case '-\[([a-zA-Z0-9_.]+)\.([a-zA-Z0-9_]+) ([a-zA-Z0-9_]+)]' started\.""".toRegex()
val TEST_CASE_STARTED = """Test Case '-\[([a-zA-Z0-9_.]+) ([a-zA-Z0-9_]+)]' started\.""".toRegex()
val TEST_CASE_FINISHED =
"""Test Case '-\[([a-zA-Z0-9_.]+)\.([a-zA-Z0-9_]+) ([a-zA-Z0-9_]+)]' (passed|failed|skipped) \(([\d\.]+) seconds\)\.""".toRegex()
"""Test Case '-\[([a-zA-Z0-9_.]+) ([a-zA-Z0-9_]+)]' (passed|failed|skipped) \(([\d\.]+) seconds\)\.""".toRegex()

/**
* $1 = file
Expand Down Expand Up @@ -61,11 +61,17 @@ class TestRunProgressParser(

private fun parseTestFinished(line: String): TestEvent? {
val matchResult = TEST_CASE_FINISHED.find(line)
val pkg = packageNameFormatter.format(matchResult?.groups?.get(1)?.value)
val clazz = matchResult?.groups?.get(2)?.value
val method = matchResult?.groups?.get(3)?.value
val result = matchResult?.groups?.get(4)?.value
val duration = matchResult?.groups?.get(5)?.value?.toFloat()
val pkgWithClass = matchResult?.groups?.get(1)?.value
var pkg: String? = null
var clazz: String? = null
if (pkgWithClass != null) {
pkg = packageNameFormatter.format(pkgWithClass.substringBeforeLast('.', missingDelimiterValue = ""))
clazz = pkgWithClass.substringAfter('.', missingDelimiterValue = pkgWithClass)
}

val method = matchResult?.groups?.get(2)?.value
val result = matchResult?.groups?.get(3)?.value
val duration = matchResult?.groups?.get(4)?.value?.toFloat()

logger.debug { "Test $pkg.$clazz.$method finished with result <$result> after $duration seconds" }

Expand Down Expand Up @@ -101,9 +107,14 @@ class TestRunProgressParser(
private fun parseTestStarted(line: String): TestStarted? {
failingTestLine = null
val matchResult = TEST_CASE_STARTED.find(line)
val pkg = packageNameFormatter.format(matchResult?.groups?.get(1)?.value)
val clazz = matchResult?.groups?.get(2)?.value
val method = matchResult?.groups?.get(3)?.value
val pkgWithClass = matchResult?.groups?.get(1)?.value
var pkg: String? = null
var clazz: String? = null
if (pkgWithClass != null) {
pkg = packageNameFormatter.format(pkgWithClass.substringBeforeLast('.', missingDelimiterValue = ""))
clazz = pkgWithClass.substringAfter('.', missingDelimiterValue = pkgWithClass)
}
val method = matchResult?.groups?.get(2)?.value

return if (pkg != null && clazz != null && method != null) {
val test = Test(pkg, clazz, method, emptyList())
Expand Down
Original file line number Diff line number Diff line change
@@ -1,16 +1,17 @@
package com.malinskiy.marathon.ios.test

import com.malinskiy.marathon.test.Test
import com.malinskiy.marathon.test.toTestName

data class TestRequest(
val workdir: String,
val remoteXctestrun: String,
val tests: List<Test>,
val xcresult: String,
val coverage: Boolean,
val tests: List<Test>? = null,
val xcresult: String? = null,
) {
fun toXcodebuildTestFilter(): Array<String> {
return tests.map { "'-only-testing:${it.pkg}/${it.clazz}/${it.method}'" }.toTypedArray()
return tests?.map { "'-only-testing:${it.toTestName(packageSeparator = '/', methodSeparator = '/')}'" }?.toTypedArray() ?: emptyArray()
}
}

Loading

0 comments on commit d140e93

Please sign in to comment.