Merge pull request #945 from vfadc/feat/pulling-xcresult-policy

feat(apple): pulling xcresult policy

configuration/src/main/kotlin/com/malinskiy/marathon/config/serialization/ConfigurationFactory.kt

@@ -19,6 +19,7 @@ import com.malinskiy.marathon.config.serialization.yaml.SerializeModule
 import com.malinskiy.marathon.config.vendor.VendorConfiguration
 import com.malinskiy.marathon.config.vendor.apple.AppleTestBundleConfiguration
 import com.malinskiy.marathon.config.vendor.apple.SshAuthentication
+import com.malinskiy.marathon.config.vendor.apple.ios.PullingPolicy
 import org.apache.commons.text.StringSubstitutor
 import org.apache.commons.text.lookup.StringLookupFactory
 import java.io.File
@@ -105,13 +106,29 @@ class ConfigurationFactory(
                         knownHostsPath = optionalknownHostsPath,
                     )
 
+                    val xcresultConfiguration = when (iosConfiguration.xcresult.pull) {
+                        true -> iosConfiguration.xcresult.copy(
+                            pullingPolicy = PullingPolicy.ALWAYS,
+                            remoteClean = iosConfiguration.xcresult.remoteClean
+                        )
+
+                        false -> iosConfiguration.xcresult.copy(
+                            pullingPolicy = PullingPolicy.NEVER,
+                            remoteClean = iosConfiguration.xcresult.remoteClean
+                        )
+
+                        null -> iosConfiguration.xcresult
+                    }
+
                     configuration.vendorConfiguration.copy(
                         bundle = resolvedBundle,
                         mediaFiles = resolvedMediaFiles,
                         devicesFile = optionalDevices,
                         ssh = optionalSshConfiguration,
+                        xcresult = xcresultConfiguration,
                     )
                 }
+
                 is VendorConfiguration.MacosConfiguration -> {
                     // Any relative path specified in Marathonfile should be resolved against the directory Marathonfile is in
                     val macosConfiguration: VendorConfiguration.MacosConfiguration = configuration.vendorConfiguration
@@ -147,10 +164,25 @@ class ConfigurationFactory(
                         knownHostsPath = optionalknownHostsPath,
                     )
 
+                    val xcresultConfiguration = when (macosConfiguration.xcresult.pull) {
+                        true -> macosConfiguration.xcresult.copy(
+                            pullingPolicy = PullingPolicy.ALWAYS,
+                            remoteClean = macosConfiguration.xcresult.remoteClean
+                        )
+
+                        false -> macosConfiguration.xcresult.copy(
+                            pullingPolicy = PullingPolicy.NEVER,
+                            remoteClean = macosConfiguration.xcresult.remoteClean
+                        )
+
+                        null -> macosConfiguration.xcresult
+                    }
+
                     configuration.vendorConfiguration.copy(
                         bundle = resolvedBundle,
                         devicesFile = optionalDevices,
                         ssh = optionalSshConfiguration,
+                        xcresult = xcresultConfiguration,
                     )
                 }
 

configuration/src/main/kotlin/com/malinskiy/marathon/config/vendor/apple/ios/XcresultConfiguration.kt

@@ -3,7 +3,8 @@ package com.malinskiy.marathon.config.vendor.apple.ios
 import com.fasterxml.jackson.annotation.JsonProperty
 
 data class XcresultConfiguration(
-    @JsonProperty("pull") val pull: Boolean = true,
+    @JsonProperty("pull") val pull: Boolean? = null,
+    @JsonProperty("pullingPolicy") val pullingPolicy: PullingPolicy = PullingPolicy.ALWAYS,
     @JsonProperty("remoteClean") val remoteClean: Boolean = true,
     @JsonProperty("attachments") val attachments: AttachmentsConfiguration = AttachmentsConfiguration(),
 )
@@ -29,3 +30,9 @@ enum class Lifetime(val value: String) {
      */
     @JsonProperty("KEEP_NEVER") KEEP_NEVER("keepNever");
 }
+
+enum class PullingPolicy {
+    @JsonProperty("NEVER") NEVER,
+    @JsonProperty("ALWAYS") ALWAYS,
+    @JsonProperty("ON_FAILURE") ON_FAILURE,
+}

configuration/src/test/kotlin/com/malinskiy/marathon/config/serialization/IosConfigurationFactoryTest.kt

@@ -21,6 +21,9 @@ import com.fasterxml.jackson.module.kotlin.KotlinModule
 import com.malinskiy.marathon.config.serialization.yaml.SerializeModule
 import com.malinskiy.marathon.config.vendor.apple.TimeoutConfiguration
 import com.malinskiy.marathon.config.vendor.apple.TestType
+import com.malinskiy.marathon.config.vendor.apple.ios.PullingPolicy
+import com.malinskiy.marathon.config.vendor.apple.ios.XcresultConfiguration
+import org.amshove.kluent.`should be`
 import org.amshove.kluent.`should be equal to`
 import org.amshove.kluent.shouldContain
 import org.amshove.kluent.shouldHaveSize
@@ -79,6 +82,10 @@ class IosConfigurationFactoryTest {
             rsync = RsyncConfiguration(
                 remotePath = "/usr/local/bin/rsync",
             ),
+            xcresult = XcresultConfiguration(
+                pullingPolicy = PullingPolicy.ON_FAILURE,
+                remoteClean = false
+            ),
             hideRunnerOutput = true,
             compactOutput = true,
             devicesFile = file.parentFile.resolve("Testdevices").canonicalFile,
@@ -138,4 +145,25 @@ class IosConfigurationFactoryTest {
             testDestination = Duration.ofSeconds(34),
         )
     }
+
+    @Test
+    fun `should set pulling policy as always when pull is true`() {
+        val file = File(ConfigurationFactoryTest::class.java.getResource("/fixture/config/ios/sample_6.yaml").file)
+        val configuration = parser.parse(file)
+
+        val xcresultConfiguration = (configuration.vendorConfiguration as VendorConfiguration.IOSConfiguration).xcresult
+        xcresultConfiguration.pullingPolicy `should be equal to` PullingPolicy.ALWAYS
+        xcresultConfiguration.remoteClean `should be` false
+    }
+
+
+    @Test
+    fun `should set pulling policy as never when pull is false`() {
+        val file = File(ConfigurationFactoryTest::class.java.getResource("/fixture/config/ios/sample_7.yaml").file)
+        val configuration = parser.parse(file)
+
+        val xcresultConfiguration = (configuration.vendorConfiguration as VendorConfiguration.IOSConfiguration).xcresult
+        xcresultConfiguration.pullingPolicy `should be equal to` PullingPolicy.NEVER
+        xcresultConfiguration.remoteClean `should be` false
+    }
 }

configuration/src/test/resources/fixture/config/ios/sample_2.yaml

@@ -12,6 +12,9 @@ vendorConfiguration:
     knownHostsPath: "known_hosts"
     keepAliveInterval: PT300S
     debug: true
+  xcresult:
+    pullingPolicy: ON_FAILURE
+    remoteClean: false
   lifecycle:
     onPrepare: []
   rsync:

configuration/src/test/resources/fixture/config/ios/sample_6.yaml

@@ -0,0 +1,10 @@
+name: "sample-app tests"
+outputDir: "./marathon"
+vendorConfiguration:
+  type: "iOS"
+  bundle:
+    derivedDataDir: "derivedDataDir"
+  xcresult:
+    pull: true
+    pullingPolicy: ON_FAILURE
+    remoteClean: false

configuration/src/test/resources/fixture/config/ios/sample_7.yaml

@@ -0,0 +1,10 @@
+name: "sample-app tests"
+outputDir: "./marathon"
+vendorConfiguration:
+  type: "iOS"
+  bundle:
+    derivedDataDir: "derivedDataDir"
+  xcresult:
+    pull: false
+    pullingPolicy: ON_FAILURE
+    remoteClean: false

docs/runner/apple/configure/ios.md

@@ -228,10 +228,12 @@ As of the time of writing marathon doesn't support merging the xcresult and trea
 
 ```yaml
   xcresult:
-    pull: true
+    pullingPolicy: ALWAYS
     remoteClean: true
 ```
 
+Possible values for the `pullingPolicy` are `ALWAYS` (by default), `NEVER` and `ON_FAILURE` - which means pulling the xcresult only for failed batches (if at least one test in the batch is failed), including retried failures.
+
 #### Attachment lifetime
 
 Marathon generates the xctestrun file for each batch and can specify custom lifecycle attachments. By default, system attachments will be

docs/runner/apple/configure/macos.md

@@ -167,10 +167,12 @@ As of the time of writing marathon doesn't support merging the xcresult and trea
 
 ```yaml
   xcresult:
-    pull: true
+    pullingPolicy: ALWAYS
     remoteClean: true
 ```
 
+Possible values for the `pullingPolicy` are `ALWAYS` (by default), `NEVER` and `ON_FAILURE` - which means pulling the xcresult only for failed batches (if at least one test in the batch is failed), including retried failures.
+
 #### Attachment lifetime
 Marathon generates the xctestrun file for each batch and can specify custom lifecycle attachments. By default, system attachments will be 
 deleted on success and user attachments will always be kept in the xcresult, but you can override this:  

sample/ios-app/Marathonfile-uiTests

@@ -27,7 +27,7 @@ vendorConfiguration:
       key: ${HOME}/.ssh/marathon
     knownHostsPath: ${HOME}/.ssh/known_hosts
   xcresult:
-    pull: true
+    pullingStrategy: ALWAYS
     remoteClean: true
   lifecycle:
     onPrepare: []

sample/ios-app/Marathonfile-unitTests

@@ -29,7 +29,7 @@ vendorConfiguration:
   testParserConfiguration:
     type: nm
   xcresult:
-    pull: true
+    pullingStrategy: ALWAYS
     remoteClean: true
   screenRecordConfiguration:
     videoConfiguration:

vendor/vendor-apple/base/src/main/kotlin/com/malinskiy/marathon/apple/listener/ResultBundleRunListener.kt

@@ -2,12 +2,15 @@ package com.malinskiy.marathon.apple.listener
 
 import com.malinskiy.marathon.apple.AppleDevice
 import com.malinskiy.marathon.apple.RemoteFileManager
+import com.malinskiy.marathon.apple.logparser.parser.DeviceFailureReason
+import com.malinskiy.marathon.config.vendor.apple.ios.PullingPolicy
 import com.malinskiy.marathon.config.vendor.apple.ios.XcresultConfiguration
 import com.malinskiy.marathon.device.DevicePoolId
 import com.malinskiy.marathon.device.toDeviceInfo
 import com.malinskiy.marathon.io.FileManager
 import com.malinskiy.marathon.io.FolderType
 import com.malinskiy.marathon.log.MarathonLogging
+import com.malinskiy.marathon.test.Test
 import com.malinskiy.marathon.test.TestBatch
 import java.io.File
 
@@ -20,10 +23,23 @@ class ResultBundleRunListener(
 ) : AppleTestRunListener {
 
     private val logger = MarathonLogging.logger {}
+
+    private var isBatchFailed = false
+
+    override suspend fun testRunFailed(errorMessage: String, reason: DeviceFailureReason) {
+        super.testRunFailed(errorMessage, reason)
+        isBatchFailed = true
+    }
+
+    override suspend fun testFailed(test: Test, startTime: Long, endTime: Long, trace: String?) {
+        super.testFailed(test, startTime, endTime, trace)
+        isBatchFailed = true
+    }
+
     override suspend fun afterTestRun() {
         super.afterTestRun()
         val remotePath = device.remoteFileManager.remoteXcresultFile(batch)
-        if (xcresultConfiguration.pull) {
+        if (isXcresultPullNeeded()) {
             val localPath = File(fileManager.createFolder(FolderType.DEVICE_FILES, poolId, device = device.toDeviceInfo()), "xcresult").apply { mkdirs() }
             if (!device.pullFolder(remotePath, localPath)) {
                 logger.warn { "failed to pull result bundle" }
@@ -36,4 +52,12 @@ class ResultBundleRunListener(
             device.remoteFileManager.removeRemotePath(remotePath)
         }
     }
+
+    private fun isXcresultPullNeeded() : Boolean {
+        return when (xcresultConfiguration.pullingPolicy) {
+            PullingPolicy.ALWAYS -> true
+            PullingPolicy.NEVER -> false
+            PullingPolicy.ON_FAILURE -> isBatchFailed
+        }
+    }
 }