Allow to omit some values in test arg file

avito-tech · Aug 8, 2020 · 8a2a22a · 8a2a22a
1 parent 9308138
commit 8a2a22a
Show file tree

Hide file tree

Showing 21 changed files with 839 additions and 239 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,46 @@
 
 All notable changes to this project will be documented in this file.
 
+## 2020-08-08
+
+- Most of test arg file values are now optional. This is to make it more user friendly for OSS community. Example of valid yet runnable test arg file is:
+
+```json
+{
+    "jobId": "jobId",
+    "entries": [
+        {
+            "testsToRun": ["all"],
+            "testDestination": {"deviceType": "iPhone X", "runtime": "11.3"},
+            "testType": "uiTest",
+            "buildArtifacts": {
+                "appBundle": "http://example.com/App.zip#MyApp/MyApp.app",
+                "runner": "http://example.com/App.zip#Tests/UITests-Runner.app",
+                "xcTestBundle": "http://example.com/App.zip#Tests/UITests-Runner.app/PlugIns/UITests.xctest"
+            }
+        }
+    ]
+}
+```
+
+- Emcee now vaidates test arg file for common errors on launch before submitting job to a shared queue ("fail fast" techique):
+
+  - If xcodebuild is selected with private simulators, Emcee will fail immediately.
+  - If any build artifact is missing, Emcee will fail immediately.
+
+Examples of such failures:
+
+```shell
+Test arg file has the following errors:
+Test arg file entry at index 0 has configuration error: xcodebuild is not compatible with provided simulator location (insideEmceeTempFolder). Use insideUserLibrary instead.
+Test arg file entry at index 1 has configuration error: Test type appTest requires appBundle to be provided
+```
+
+- New test arg file syntax for enumerating tests to be run:
+
+  - `"testsToRun": ["all"]` is shorter eqivalent of `"testsToRun": [{"predicateType": "allDiscoveredTests"}]`
+  - `"testsToRun": ["Class/test"]` is shorter eqivalent of `"testsToRun": [{"predicateType": "singleTestName", "testName": "Class/test"}]`
+
 ## 2020-08-07
 
 - `QueueServerRunConfiguration` and everything related to it has been renamed to `QueueServerConfiguration`, including CLI argument `--queue-server-run-configuration` which was renamed to `--queue-server-configuration`.
@@ -13,7 +53,6 @@ All notable changes to this project will be documented in this file.
 New `kickstart` command allows to (re-)start a given worker in case if it went south. Syntax:
 
 ```shell
-
 $ Emcee kickstart --queue-server <queue address:port> --worker-id <worker id> --worker-id <another worker id>
 ```
 
@@ -39,8 +78,8 @@ Emcee will kill SpringBoard and cfprefsd if any plist changes in order to apply
 ## 2020-05-15
 
 - `Package.swift` file is now generated. All `import` statements are parsed to do that. On CI, the check has been added to verify that `Package.swift` is commited correctly.
-New `make gen` command will generate both `Package.swift` and Xcode project.
-Test helper targets are detected by `TestHelper` suffix in their names. These targets are kept as normal ones (not `.testTarget()`).
+  New `make gen` command will generate both `Package.swift` and Xcode project.
+  Test helper targets are detected by `TestHelper` suffix in their names. These targets are kept as normal ones (not `.testTarget()`).
 
 - New command `disableWorker --queue-server host:1234 --worker-id some.worker.id` allows to disable worker from load. Queue will not provide any buckets for execution to disabled worker. Useful for performing some maintenance and etc. Enabling worker feature is TBD. 
 
@@ -79,7 +118,6 @@ Runtime dump feature has been renamed to test discovery. `RuntimeDump` module is
 `xctestBundle` object in build artifacts now has `testDiscoveryMode` field instead of `runtimeDumpMode`, and the supported values are `runtimeLogicTest` and `runtimeAppTest`.
 `testsToRun` value for running all available tests has been renamed from `allProvidedByRuntimeDump` to `allDiscoveredTests`.
 
-
 ## 2020-03-30
 
 Support for grouping jobs. 
@@ -217,12 +255,13 @@ Confguration above defines the following behaviour:
 ### Changed
 
 - `environment` and `testType` fiels are required to be present in test arg file. 
-```json
+
+  ```json
     ...
     "environment": {"ENV1": "VAL1", ...},
     "testType": "uiTest",  # supported values are "appTest", "logicTest", "uiTest"
     ...
-```
+  ```
 
 - Test arg file JSON entries is now expected to have `toolResources` field. This field describes the tools used to perform testing. This is an object with `testRunnerTool` and `simulatorControlTool`. Example:
 

diff --git a/Package.swift b/Package.swift
@@ -1838,6 +1838,7 @@ let package = Package(
                 "BuildArtifacts",
                 "BuildArtifactsTestHelpers",
                 "PluginSupport",
+                "ResourceLocation",
                 "RunnerModels",
                 "RunnerTestHelpers",
                 "SimulatorPoolModels",

diff --git a/Sources/BuildArtifacts/XcTestBundle.swift b/Sources/BuildArtifacts/XcTestBundle.swift
@@ -19,7 +19,7 @@ public struct XcTestBundle: Codable, Hashable, CustomStringConvertible {
         // Try fallback value first
         if let fallbackLocation = try? decoder.singleValueContainer().decode(TestBundleLocation.self) {
             self.location = fallbackLocation
-            self.testDiscoveryMode = .runtimeLogicTest
+            self.testDiscoveryMode = .parseFunctionSymbols
         } else {
             let container = try decoder.container(keyedBy: CodingKeys.self)
             self.location = try container.decode(TestBundleLocation.self, forKey: .location)

diff --git a/Sources/EmceeLib/Arguments/ArgumentDescriptions.swift b/Sources/EmceeLib/Arguments/ArgumentDescriptions.swift
@@ -3,9 +3,6 @@ import Foundation
 
 final class ArgumentDescriptions {
     static let emceeVersion = doubleDashedDescription(dashlessName: "emcee-version", overview: "Explicit version of Emcee binary")
-    static let jobGroupId = doubleDashedDescription(dashlessName: "job-group-id", overview: "Unique job group id that groups various jobs into a single group")
-    static let jobGroupPriority = doubleDashedDescription(dashlessName: "job-group-priority", overview: "Priority of the job group")
-    static let jobId = doubleDashedDescription(dashlessName: "job-id", overview: "Unique job id, usually a random string, e.g. UUID")
     static let junit = doubleDashedDescription(dashlessName: "junit", overview: "Path where the combined (for all test destinations) Junit report file should be created")
     static let output = doubleDashedDescription(dashlessName: "output", overview: "Path to file where to store the output")
     static let plugin = doubleDashedDescription(dashlessName: "plugin", overview: "URL to ZIP file with .emceeplugin bundle. Plugin bundle should contain an executable: MyPlugin.emceeplugin/Plugin", multiple: true)

diff --git a/Sources/EmceeLib/Commands/RunTestsOnRemoteQueueCommand.swift b/Sources/EmceeLib/Commands/RunTestsOnRemoteQueueCommand.swift
@@ -34,9 +34,6 @@ public final class RunTestsOnRemoteQueueCommand: Command {
     public let description = "Starts queue server on remote machine if needed and runs tests on the remote queue. Waits for resuls to come back."
     public let arguments: Arguments = [
         ArgumentDescriptions.emceeVersion.asOptional,
-        ArgumentDescriptions.jobGroupId.asOptional,
-        ArgumentDescriptions.jobGroupPriority.asOptional,
-        ArgumentDescriptions.jobId.asRequired,
         ArgumentDescriptions.junit.asOptional,
         ArgumentDescriptions.queueServerConfigurationLocation.asRequired,
         ArgumentDescriptions.remoteCacheConfig.asOptional,
@@ -52,8 +49,9 @@ public final class RunTestsOnRemoteQueueCommand: Command {
     private let processControllerProvider: ProcessControllerProvider
     private let requestSenderProvider: RequestSenderProvider
     private let resourceLocationResolver: ResourceLocationResolver
-    private let uniqueIdentifierGenerator: UniqueIdentifierGenerator
     private let runtimeDumpRemoteCacheProvider: RuntimeDumpRemoteCacheProvider
+    private let testArgFileValidator = TestArgFileValidator()
+    private let uniqueIdentifierGenerator: UniqueIdentifierGenerator
 
     public init(
         dateProvider: DateProvider,
@@ -89,13 +87,10 @@ public final class RunTestsOnRemoteQueueCommand: Command {
             resourceLocationResolver: resourceLocationResolver
         )
 
-        let jobId: JobId = try payload.expectedSingleTypedValue(argumentName: ArgumentDescriptions.jobId.name)
-        let jobGroupId: JobGroupId = try payload.optionalSingleTypedValue(argumentName: ArgumentDescriptions.jobGroupId.name) ?? JobGroupId(value: jobId.value)
         let emceeVersion: Version = try payload.optionalSingleTypedValue(argumentName: ArgumentDescriptions.emceeVersion.name) ?? EmceeVersion.version
-
         let tempFolder = try TemporaryFolder(containerPath: try payload.expectedSingleTypedValue(argumentName: ArgumentDescriptions.tempFolder.name))
         let testArgFile = try ArgumentsReader.testArgFile(try payload.expectedSingleTypedValue(argumentName: ArgumentDescriptions.testArgFile.name))
-        let jobGroupPriority: Priority = try payload.optionalSingleTypedValue(argumentName: ArgumentDescriptions.jobGroupPriority.name) ?? testArgFile.priority
+        try testArgFileValidator.validate(testArgFile: testArgFile)
 
         let remoteCacheConfig = try ArgumentsReader.remoteCacheConfig(
             try payload.optionalSingleTypedValue(argumentName: ArgumentDescriptions.remoteCacheConfig.name)
@@ -105,13 +100,10 @@ public final class RunTestsOnRemoteQueueCommand: Command {
             emceeVersion: emceeVersion,
             queueServerDeploymentDestination: queueServerConfiguration.queueServerDeploymentDestination,
             queueServerConfigurationLocation: queueServerConfigurationLocation,
-            jobId: jobId,
+            jobId: testArgFile.jobId,
             tempFolder: tempFolder
         )
         let jobResults = try runTestsOnRemotelyRunningQueue(
-            jobGroupId: jobGroupId,
-            jobGroupPriority: jobGroupPriority,
-            jobId: jobId,
             queueServerAddress: runningQueueServerAddress,
             remoteCacheConfig: remoteCacheConfig,
             tempFolder: tempFolder,
@@ -186,9 +178,6 @@ public final class RunTestsOnRemoteQueueCommand: Command {
     }
 
     private func runTestsOnRemotelyRunningQueue(
-        jobGroupId: JobGroupId,
-        jobGroupPriority: Priority,
-        jobId: JobId,
         queueServerAddress: SocketAddress,
         remoteCacheConfig: RuntimeDumpRemoteCacheConfig?,
         tempFolder: TemporaryFolder,
@@ -229,11 +218,11 @@ public final class RunTestsOnRemoteQueueCommand: Command {
         let queueClient = SynchronousQueueClient(queueServerAddress: queueServerAddress)
 
         defer {
-            Logger.info("Will delete job \(jobId)")
+            Logger.info("Will delete job \(testArgFile.jobId)")
             do {
-                _ = try queueClient.delete(jobId: jobId)
+                _ = try queueClient.delete(jobId: testArgFile.jobId)
             } catch {
-                Logger.error("Failed to delete job \(jobId): \(error)")
+                Logger.error("Failed to delete job \(testArgFile.jobId): \(error)")
             }
         }
 
@@ -253,10 +242,10 @@ public final class RunTestsOnRemoteQueueCommand: Command {
             do {
                 _ = try queueClient.scheduleTests(
                     prioritizedJob: PrioritizedJob(
-                        jobGroupId: jobGroupId,
-                        jobGroupPriority: jobGroupPriority,
-                        jobId: jobId,
-                        jobPriority: testArgFile.priority
+                        jobGroupId: testArgFile.jobGroupId,
+                        jobGroupPriority: testArgFile.jobGroupPriority,
+                        jobId: testArgFile.jobId,
+                        jobPriority: testArgFile.jobPriority
                     ),
                     scheduleStrategy: testArgFileEntry.scheduleStrategy,
                     testEntryConfigurations: testEntryConfigurations,
@@ -271,15 +260,15 @@ public final class RunTestsOnRemoteQueueCommand: Command {
         var caughtSignal = false
         SignalHandling.addSignalHandler(signals: [.int, .term]) { signal in
             Logger.info("Caught \(signal) signal")
-            Logger.info("Will delete job \(jobId)")
-            _ = try? queueClient.delete(jobId: jobId)
+            Logger.info("Will delete job \(testArgFile.jobId)")
+            _ = try? queueClient.delete(jobId: testArgFile.jobId)
             caughtSignal = true
         }
 
         Logger.info("Will now wait for job queue to deplete")
         try SynchronousWaiter().waitWhile(pollPeriod: 30.0, description: "Wait for job queue to deplete") {
             if caughtSignal { return false }
-            let jobState = try queueClient.jobState(jobId: jobId)
+            let jobState = try queueClient.jobState(jobId: testArgFile.jobId)
             switch jobState.queueState {
             case .deleted:
                 return false
@@ -289,7 +278,7 @@ public final class RunTestsOnRemoteQueueCommand: Command {
             }
         }
         Logger.info("Will now fetch job results")
-        return try queueClient.jobResults(jobId: jobId)
+        return try queueClient.jobResults(jobId: testArgFile.jobId)
     }
 
     private func selectPort(ports: Set<SocketModels.Port>) throws -> SocketModels.Port {

diff --git a/Sources/EmceeLib/Utils/TestDiscoveryModeDeterminer.swift b/Sources/EmceeLib/Utils/TestDiscoveryModeDeterminer.swift
@@ -18,7 +18,7 @@ public enum TestDicoveryModeInputValidationError: Error, CustomStringConvertible
 }
 
 public final class TestDiscoveryModeDeterminer {
-    public static func testDiscoveryMode(testArgFileEntry: TestArgFile.Entry) throws -> TestDiscoveryMode {
+    public static func testDiscoveryMode(testArgFileEntry: TestArgFileEntry) throws -> TestDiscoveryMode {
         switch testArgFileEntry.buildArtifacts.xcTestBundle.testDiscoveryMode {
         case .parseFunctionSymbols:
             return .parseFunctionSymbols

diff --git a/Sources/EmceeLib/Utils/TestEntriesValidator/TestEntriesValidator.swift b/Sources/EmceeLib/Utils/TestEntriesValidator/TestEntriesValidator.swift
@@ -8,20 +8,20 @@ import TestArgFile
 import TestDiscovery
 
 public final class TestEntriesValidator {
-    private let testArgFileEntries: [TestArgFile.Entry]
+    private let testArgFileEntries: [TestArgFileEntry]
     private let testDiscoveryQuerier: TestDiscoveryQuerier
     private let transformer = TestToRunIntoTestEntryTransformer()
 
     public init(
-        testArgFileEntries: [TestArgFile.Entry],
+        testArgFileEntries: [TestArgFileEntry],
         testDiscoveryQuerier: TestDiscoveryQuerier
     ) {
         self.testArgFileEntries = testArgFileEntries
         self.testDiscoveryQuerier = testDiscoveryQuerier
     }
 
     public func validatedTestEntries(
-        intermediateResult: (TestArgFile.Entry, [ValidatedTestEntry]) throws -> ()
+        intermediateResult: (TestArgFileEntry, [ValidatedTestEntry]) throws -> ()
     ) throws -> [ValidatedTestEntry] {
         var result = [ValidatedTestEntry]()
 
@@ -35,7 +35,7 @@ public final class TestEntriesValidator {
     }
 
     private func validatedTestEntries(
-        testArgFileEntry: TestArgFile.Entry
+        testArgFileEntry: TestArgFileEntry
     ) throws -> [ValidatedTestEntry] {
         let configuration = TestDiscoveryConfiguration(
             developerDir: testArgFileEntry.developerDir,

diff --git a/Sources/EmceeLib/Utils/TestEntryConfigurationGenerator.swift b/Sources/EmceeLib/Utils/TestEntryConfigurationGenerator.swift
@@ -8,11 +8,11 @@ import TestDiscovery
 
 public final class TestEntryConfigurationGenerator {
     private let validatedEntries: [ValidatedTestEntry]
-    private let testArgFileEntry: TestArgFile.Entry
+    private let testArgFileEntry: TestArgFileEntry
 
     public init(
         validatedEntries: [ValidatedTestEntry],
-        testArgFileEntry: TestArgFile.Entry
+        testArgFileEntry: TestArgFileEntry
     ) {
         self.validatedEntries = validatedEntries
         self.testArgFileEntry = testArgFileEntry

diff --git a/Sources/SimulatorPoolModels/SimulatorLocalizationSettings.swift b/Sources/SimulatorPoolModels/SimulatorLocalizationSettings.swift
@@ -30,7 +30,7 @@ public struct SimulatorLocalizationSettings: Codable, CustomStringConvertible, H
         self.didShowContinuousPathIntroduction = didShowContinuousPathIntroduction
     }
 
-        public var description: String {
-            return "<\(type(of: self)) \(localeIdentifier), keyboards: \(keyboards), passcodeKeyboards: \(passcodeKeyboards), languages: \(languages), addingEmojiKeybordHandled \(addingEmojiKeybordHandled), enableKeyboardExpansion \(enableKeyboardExpansion), didShowInternationalInfoAlert \(didShowInternationalInfoAlert), didShowContinuousPathIntroduction \(didShowContinuousPathIntroduction)>"
+    public var description: String {
+        return "<\(type(of: self)) \(localeIdentifier), keyboards: \(keyboards), passcodeKeyboards: \(passcodeKeyboards), languages: \(languages), addingEmojiKeybordHandled \(addingEmojiKeybordHandled), enableKeyboardExpansion \(enableKeyboardExpansion), didShowInternationalInfoAlert \(didShowInternationalInfoAlert), didShowContinuousPathIntroduction \(didShowContinuousPathIntroduction)>"
     }
 }
diff --git a/Sources/TestArgFile/ReportOutput.swift b/Sources/TestArgFile/ReportOutput.swift
@@ -1,6 +1,6 @@
 import Foundation
 
-public struct ReportOutput: Codable {
+public struct ReportOutput: Codable, Equatable {
     /// Absolute path where Junit report should be created. If nil, report won't be created.
     public let junit: String?