diff --git a/CHANGELOG.md b/CHANGELOG.md index b9ece5f0..b1dcc1f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,46 @@ All notable changes to this project will be documented in this file. +## 2020-08-08 + +- Most of test arg file values are now optional. This is to make it more user friendly for OSS community. Example of valid yet runnable test arg file is: + +```json +{ + "jobId": "jobId", + "entries": [ + { + "testsToRun": ["all"], + "testDestination": {"deviceType": "iPhone X", "runtime": "11.3"}, + "testType": "uiTest", + "buildArtifacts": { + "appBundle": "http://example.com/App.zip#MyApp/MyApp.app", + "runner": "http://example.com/App.zip#Tests/UITests-Runner.app", + "xcTestBundle": "http://example.com/App.zip#Tests/UITests-Runner.app/PlugIns/UITests.xctest" + } + } + ] +} +``` + +- Emcee now vaidates test arg file for common errors on launch before submitting job to a shared queue ("fail fast" techique): + + - If xcodebuild is selected with private simulators, Emcee will fail immediately. + - If any build artifact is missing, Emcee will fail immediately. + +Examples of such failures: + +```shell +Test arg file has the following errors: +Test arg file entry at index 0 has configuration error: xcodebuild is not compatible with provided simulator location (insideEmceeTempFolder). Use insideUserLibrary instead. +Test arg file entry at index 1 has configuration error: Test type appTest requires appBundle to be provided +``` + +- New test arg file syntax for enumerating tests to be run: + + - `"testsToRun": ["all"]` is shorter eqivalent of `"testsToRun": [{"predicateType": "allDiscoveredTests"}]` + - `"testsToRun": ["Class/test"]` is shorter eqivalent of `"testsToRun": [{"predicateType": "singleTestName", "testName": "Class/test"}]` + ## 2020-08-07 - `QueueServerRunConfiguration` and everything related to it has been renamed to `QueueServerConfiguration`, including CLI argument `--queue-server-run-configuration` which was renamed to `--queue-server-configuration`. @@ -13,7 +53,6 @@ All notable changes to this project will be documented in this file. New `kickstart` command allows to (re-)start a given worker in case if it went south. Syntax: ```shell - $ Emcee kickstart --queue-server --worker-id --worker-id ``` @@ -39,8 +78,8 @@ Emcee will kill SpringBoard and cfprefsd if any plist changes in order to apply ## 2020-05-15 - `Package.swift` file is now generated. All `import` statements are parsed to do that. On CI, the check has been added to verify that `Package.swift` is commited correctly. -New `make gen` command will generate both `Package.swift` and Xcode project. -Test helper targets are detected by `TestHelper` suffix in their names. These targets are kept as normal ones (not `.testTarget()`). + New `make gen` command will generate both `Package.swift` and Xcode project. + Test helper targets are detected by `TestHelper` suffix in their names. These targets are kept as normal ones (not `.testTarget()`). - New command `disableWorker --queue-server host:1234 --worker-id some.worker.id` allows to disable worker from load. Queue will not provide any buckets for execution to disabled worker. Useful for performing some maintenance and etc. Enabling worker feature is TBD. @@ -79,7 +118,6 @@ Runtime dump feature has been renamed to test discovery. `RuntimeDump` module is `xctestBundle` object in build artifacts now has `testDiscoveryMode` field instead of `runtimeDumpMode`, and the supported values are `runtimeLogicTest` and `runtimeAppTest`. `testsToRun` value for running all available tests has been renamed from `allProvidedByRuntimeDump` to `allDiscoveredTests`. - ## 2020-03-30 Support for grouping jobs. @@ -217,12 +255,13 @@ Confguration above defines the following behaviour: ### Changed - `environment` and `testType` fiels are required to be present in test arg file. -```json + + ```json ... "environment": {"ENV1": "VAL1", ...}, "testType": "uiTest", # supported values are "appTest", "logicTest", "uiTest" ... -``` + ``` - Test arg file JSON entries is now expected to have `toolResources` field. This field describes the tools used to perform testing. This is an object with `testRunnerTool` and `simulatorControlTool`. Example: diff --git a/Package.swift b/Package.swift index c68a4b11..aa43682d 100644 --- a/Package.swift +++ b/Package.swift @@ -1838,6 +1838,7 @@ let package = Package( "BuildArtifacts", "BuildArtifactsTestHelpers", "PluginSupport", + "ResourceLocation", "RunnerModels", "RunnerTestHelpers", "SimulatorPoolModels", diff --git a/Sources/BuildArtifacts/XcTestBundle.swift b/Sources/BuildArtifacts/XcTestBundle.swift index e391355d..c0fb9ce2 100644 --- a/Sources/BuildArtifacts/XcTestBundle.swift +++ b/Sources/BuildArtifacts/XcTestBundle.swift @@ -19,7 +19,7 @@ public struct XcTestBundle: Codable, Hashable, CustomStringConvertible { // Try fallback value first if let fallbackLocation = try? decoder.singleValueContainer().decode(TestBundleLocation.self) { self.location = fallbackLocation - self.testDiscoveryMode = .runtimeLogicTest + self.testDiscoveryMode = .parseFunctionSymbols } else { let container = try decoder.container(keyedBy: CodingKeys.self) self.location = try container.decode(TestBundleLocation.self, forKey: .location) diff --git a/Sources/EmceeLib/Arguments/ArgumentDescriptions.swift b/Sources/EmceeLib/Arguments/ArgumentDescriptions.swift index 70a8d653..b65393c3 100644 --- a/Sources/EmceeLib/Arguments/ArgumentDescriptions.swift +++ b/Sources/EmceeLib/Arguments/ArgumentDescriptions.swift @@ -3,9 +3,6 @@ import Foundation final class ArgumentDescriptions { static let emceeVersion = doubleDashedDescription(dashlessName: "emcee-version", overview: "Explicit version of Emcee binary") - static let jobGroupId = doubleDashedDescription(dashlessName: "job-group-id", overview: "Unique job group id that groups various jobs into a single group") - static let jobGroupPriority = doubleDashedDescription(dashlessName: "job-group-priority", overview: "Priority of the job group") - static let jobId = doubleDashedDescription(dashlessName: "job-id", overview: "Unique job id, usually a random string, e.g. UUID") static let junit = doubleDashedDescription(dashlessName: "junit", overview: "Path where the combined (for all test destinations) Junit report file should be created") static let output = doubleDashedDescription(dashlessName: "output", overview: "Path to file where to store the output") static let plugin = doubleDashedDescription(dashlessName: "plugin", overview: "URL to ZIP file with .emceeplugin bundle. Plugin bundle should contain an executable: MyPlugin.emceeplugin/Plugin", multiple: true) diff --git a/Sources/EmceeLib/Commands/RunTestsOnRemoteQueueCommand.swift b/Sources/EmceeLib/Commands/RunTestsOnRemoteQueueCommand.swift index e8edf766..cf1e9317 100644 --- a/Sources/EmceeLib/Commands/RunTestsOnRemoteQueueCommand.swift +++ b/Sources/EmceeLib/Commands/RunTestsOnRemoteQueueCommand.swift @@ -34,9 +34,6 @@ public final class RunTestsOnRemoteQueueCommand: Command { public let description = "Starts queue server on remote machine if needed and runs tests on the remote queue. Waits for resuls to come back." public let arguments: Arguments = [ ArgumentDescriptions.emceeVersion.asOptional, - ArgumentDescriptions.jobGroupId.asOptional, - ArgumentDescriptions.jobGroupPriority.asOptional, - ArgumentDescriptions.jobId.asRequired, ArgumentDescriptions.junit.asOptional, ArgumentDescriptions.queueServerConfigurationLocation.asRequired, ArgumentDescriptions.remoteCacheConfig.asOptional, @@ -52,8 +49,9 @@ public final class RunTestsOnRemoteQueueCommand: Command { private let processControllerProvider: ProcessControllerProvider private let requestSenderProvider: RequestSenderProvider private let resourceLocationResolver: ResourceLocationResolver - private let uniqueIdentifierGenerator: UniqueIdentifierGenerator private let runtimeDumpRemoteCacheProvider: RuntimeDumpRemoteCacheProvider + private let testArgFileValidator = TestArgFileValidator() + private let uniqueIdentifierGenerator: UniqueIdentifierGenerator public init( dateProvider: DateProvider, @@ -89,13 +87,10 @@ public final class RunTestsOnRemoteQueueCommand: Command { resourceLocationResolver: resourceLocationResolver ) - let jobId: JobId = try payload.expectedSingleTypedValue(argumentName: ArgumentDescriptions.jobId.name) - let jobGroupId: JobGroupId = try payload.optionalSingleTypedValue(argumentName: ArgumentDescriptions.jobGroupId.name) ?? JobGroupId(value: jobId.value) let emceeVersion: Version = try payload.optionalSingleTypedValue(argumentName: ArgumentDescriptions.emceeVersion.name) ?? EmceeVersion.version - let tempFolder = try TemporaryFolder(containerPath: try payload.expectedSingleTypedValue(argumentName: ArgumentDescriptions.tempFolder.name)) let testArgFile = try ArgumentsReader.testArgFile(try payload.expectedSingleTypedValue(argumentName: ArgumentDescriptions.testArgFile.name)) - let jobGroupPriority: Priority = try payload.optionalSingleTypedValue(argumentName: ArgumentDescriptions.jobGroupPriority.name) ?? testArgFile.priority + try testArgFileValidator.validate(testArgFile: testArgFile) let remoteCacheConfig = try ArgumentsReader.remoteCacheConfig( try payload.optionalSingleTypedValue(argumentName: ArgumentDescriptions.remoteCacheConfig.name) @@ -105,13 +100,10 @@ public final class RunTestsOnRemoteQueueCommand: Command { emceeVersion: emceeVersion, queueServerDeploymentDestination: queueServerConfiguration.queueServerDeploymentDestination, queueServerConfigurationLocation: queueServerConfigurationLocation, - jobId: jobId, + jobId: testArgFile.jobId, tempFolder: tempFolder ) let jobResults = try runTestsOnRemotelyRunningQueue( - jobGroupId: jobGroupId, - jobGroupPriority: jobGroupPriority, - jobId: jobId, queueServerAddress: runningQueueServerAddress, remoteCacheConfig: remoteCacheConfig, tempFolder: tempFolder, @@ -186,9 +178,6 @@ public final class RunTestsOnRemoteQueueCommand: Command { } private func runTestsOnRemotelyRunningQueue( - jobGroupId: JobGroupId, - jobGroupPriority: Priority, - jobId: JobId, queueServerAddress: SocketAddress, remoteCacheConfig: RuntimeDumpRemoteCacheConfig?, tempFolder: TemporaryFolder, @@ -229,11 +218,11 @@ public final class RunTestsOnRemoteQueueCommand: Command { let queueClient = SynchronousQueueClient(queueServerAddress: queueServerAddress) defer { - Logger.info("Will delete job \(jobId)") + Logger.info("Will delete job \(testArgFile.jobId)") do { - _ = try queueClient.delete(jobId: jobId) + _ = try queueClient.delete(jobId: testArgFile.jobId) } catch { - Logger.error("Failed to delete job \(jobId): \(error)") + Logger.error("Failed to delete job \(testArgFile.jobId): \(error)") } } @@ -253,10 +242,10 @@ public final class RunTestsOnRemoteQueueCommand: Command { do { _ = try queueClient.scheduleTests( prioritizedJob: PrioritizedJob( - jobGroupId: jobGroupId, - jobGroupPriority: jobGroupPriority, - jobId: jobId, - jobPriority: testArgFile.priority + jobGroupId: testArgFile.jobGroupId, + jobGroupPriority: testArgFile.jobGroupPriority, + jobId: testArgFile.jobId, + jobPriority: testArgFile.jobPriority ), scheduleStrategy: testArgFileEntry.scheduleStrategy, testEntryConfigurations: testEntryConfigurations, @@ -271,15 +260,15 @@ public final class RunTestsOnRemoteQueueCommand: Command { var caughtSignal = false SignalHandling.addSignalHandler(signals: [.int, .term]) { signal in Logger.info("Caught \(signal) signal") - Logger.info("Will delete job \(jobId)") - _ = try? queueClient.delete(jobId: jobId) + Logger.info("Will delete job \(testArgFile.jobId)") + _ = try? queueClient.delete(jobId: testArgFile.jobId) caughtSignal = true } Logger.info("Will now wait for job queue to deplete") try SynchronousWaiter().waitWhile(pollPeriod: 30.0, description: "Wait for job queue to deplete") { if caughtSignal { return false } - let jobState = try queueClient.jobState(jobId: jobId) + let jobState = try queueClient.jobState(jobId: testArgFile.jobId) switch jobState.queueState { case .deleted: return false @@ -289,7 +278,7 @@ public final class RunTestsOnRemoteQueueCommand: Command { } } Logger.info("Will now fetch job results") - return try queueClient.jobResults(jobId: jobId) + return try queueClient.jobResults(jobId: testArgFile.jobId) } private func selectPort(ports: Set) throws -> SocketModels.Port { diff --git a/Sources/EmceeLib/Utils/TestDiscoveryModeDeterminer.swift b/Sources/EmceeLib/Utils/TestDiscoveryModeDeterminer.swift index b631bdaf..f98089b3 100644 --- a/Sources/EmceeLib/Utils/TestDiscoveryModeDeterminer.swift +++ b/Sources/EmceeLib/Utils/TestDiscoveryModeDeterminer.swift @@ -18,7 +18,7 @@ public enum TestDicoveryModeInputValidationError: Error, CustomStringConvertible } public final class TestDiscoveryModeDeterminer { - public static func testDiscoveryMode(testArgFileEntry: TestArgFile.Entry) throws -> TestDiscoveryMode { + public static func testDiscoveryMode(testArgFileEntry: TestArgFileEntry) throws -> TestDiscoveryMode { switch testArgFileEntry.buildArtifacts.xcTestBundle.testDiscoveryMode { case .parseFunctionSymbols: return .parseFunctionSymbols diff --git a/Sources/EmceeLib/Utils/TestEntriesValidator/TestEntriesValidator.swift b/Sources/EmceeLib/Utils/TestEntriesValidator/TestEntriesValidator.swift index 9d9d57c4..035ec51f 100644 --- a/Sources/EmceeLib/Utils/TestEntriesValidator/TestEntriesValidator.swift +++ b/Sources/EmceeLib/Utils/TestEntriesValidator/TestEntriesValidator.swift @@ -8,12 +8,12 @@ import TestArgFile import TestDiscovery public final class TestEntriesValidator { - private let testArgFileEntries: [TestArgFile.Entry] + private let testArgFileEntries: [TestArgFileEntry] private let testDiscoveryQuerier: TestDiscoveryQuerier private let transformer = TestToRunIntoTestEntryTransformer() public init( - testArgFileEntries: [TestArgFile.Entry], + testArgFileEntries: [TestArgFileEntry], testDiscoveryQuerier: TestDiscoveryQuerier ) { self.testArgFileEntries = testArgFileEntries @@ -21,7 +21,7 @@ public final class TestEntriesValidator { } public func validatedTestEntries( - intermediateResult: (TestArgFile.Entry, [ValidatedTestEntry]) throws -> () + intermediateResult: (TestArgFileEntry, [ValidatedTestEntry]) throws -> () ) throws -> [ValidatedTestEntry] { var result = [ValidatedTestEntry]() @@ -35,7 +35,7 @@ public final class TestEntriesValidator { } private func validatedTestEntries( - testArgFileEntry: TestArgFile.Entry + testArgFileEntry: TestArgFileEntry ) throws -> [ValidatedTestEntry] { let configuration = TestDiscoveryConfiguration( developerDir: testArgFileEntry.developerDir, diff --git a/Sources/EmceeLib/Utils/TestEntryConfigurationGenerator.swift b/Sources/EmceeLib/Utils/TestEntryConfigurationGenerator.swift index 06d9c398..ef37f007 100644 --- a/Sources/EmceeLib/Utils/TestEntryConfigurationGenerator.swift +++ b/Sources/EmceeLib/Utils/TestEntryConfigurationGenerator.swift @@ -8,11 +8,11 @@ import TestDiscovery public final class TestEntryConfigurationGenerator { private let validatedEntries: [ValidatedTestEntry] - private let testArgFileEntry: TestArgFile.Entry + private let testArgFileEntry: TestArgFileEntry public init( validatedEntries: [ValidatedTestEntry], - testArgFileEntry: TestArgFile.Entry + testArgFileEntry: TestArgFileEntry ) { self.validatedEntries = validatedEntries self.testArgFileEntry = testArgFileEntry diff --git a/Sources/SimulatorPoolModels/SimulatorLocalizationSettings.swift b/Sources/SimulatorPoolModels/SimulatorLocalizationSettings.swift index 0bf5c27c..214c11c4 100644 --- a/Sources/SimulatorPoolModels/SimulatorLocalizationSettings.swift +++ b/Sources/SimulatorPoolModels/SimulatorLocalizationSettings.swift @@ -30,7 +30,7 @@ public struct SimulatorLocalizationSettings: Codable, CustomStringConvertible, H self.didShowContinuousPathIntroduction = didShowContinuousPathIntroduction } - public var description: String { - return "<\(type(of: self)) \(localeIdentifier), keyboards: \(keyboards), passcodeKeyboards: \(passcodeKeyboards), languages: \(languages), addingEmojiKeybordHandled \(addingEmojiKeybordHandled), enableKeyboardExpansion \(enableKeyboardExpansion), didShowInternationalInfoAlert \(didShowInternationalInfoAlert), didShowContinuousPathIntroduction \(didShowContinuousPathIntroduction)>" + public var description: String { + return "<\(type(of: self)) \(localeIdentifier), keyboards: \(keyboards), passcodeKeyboards: \(passcodeKeyboards), languages: \(languages), addingEmojiKeybordHandled \(addingEmojiKeybordHandled), enableKeyboardExpansion \(enableKeyboardExpansion), didShowInternationalInfoAlert \(didShowInternationalInfoAlert), didShowContinuousPathIntroduction \(didShowContinuousPathIntroduction)>" } } diff --git a/Sources/TestArgFile/ReportOutput.swift b/Sources/TestArgFile/ReportOutput.swift index 5258bbd4..8c300a5b 100644 --- a/Sources/TestArgFile/ReportOutput.swift +++ b/Sources/TestArgFile/ReportOutput.swift @@ -1,6 +1,6 @@ import Foundation -public struct ReportOutput: Codable { +public struct ReportOutput: Codable, Equatable { /// Absolute path where Junit report should be created. If nil, report won't be created. public let junit: String? diff --git a/Sources/TestArgFile/TestArgFile.swift b/Sources/TestArgFile/TestArgFile.swift index 01950092..49bcee64 100644 --- a/Sources/TestArgFile/TestArgFile.swift +++ b/Sources/TestArgFile/TestArgFile.swift @@ -1,78 +1,53 @@ -import BuildArtifacts -import DeveloperDirModels import Foundation -import PluginSupport import QueueModels -import RunnerModels -import ScheduleStrategy -import SimulatorPoolModels -import WorkerCapabilitiesModels /// Represents --test-arg-file file contents which describes test plan. -public struct TestArgFile: Codable { - public struct Entry: Codable, Equatable { - public let buildArtifacts: BuildArtifacts - public let developerDir: DeveloperDir - public let environment: [String: String] - public let numberOfRetries: UInt - public let pluginLocations: Set - public let scheduleStrategy: ScheduleStrategyType - public let simulatorControlTool: SimulatorControlTool - public let simulatorOperationTimeouts: SimulatorOperationTimeouts - public let simulatorSettings: SimulatorSettings - public let testDestination: TestDestination - public let testRunnerTool: TestRunnerTool - public let testTimeoutConfiguration: TestTimeoutConfiguration - public let testType: TestType - public let testsToRun: [TestToRun] - public let workerCapabilityRequirements: Set - - public init( - buildArtifacts: BuildArtifacts, - developerDir: DeveloperDir, - environment: [String: String], - numberOfRetries: UInt, - pluginLocations: Set, - scheduleStrategy: ScheduleStrategyType, - simulatorControlTool: SimulatorControlTool, - simulatorOperationTimeouts: SimulatorOperationTimeouts, - simulatorSettings: SimulatorSettings, - testDestination: TestDestination, - testRunnerTool: TestRunnerTool, - testTimeoutConfiguration: TestTimeoutConfiguration, - testType: TestType, - testsToRun: [TestToRun], - workerCapabilityRequirements: Set - ) { - self.buildArtifacts = buildArtifacts - self.developerDir = developerDir - self.environment = environment - self.numberOfRetries = numberOfRetries - self.pluginLocations = pluginLocations - self.scheduleStrategy = scheduleStrategy - self.simulatorControlTool = simulatorControlTool - self.simulatorOperationTimeouts = simulatorOperationTimeouts - self.simulatorSettings = simulatorSettings - self.testDestination = testDestination - self.testRunnerTool = testRunnerTool - self.testTimeoutConfiguration = testTimeoutConfiguration - self.testType = testType - self.testsToRun = testsToRun - self.workerCapabilityRequirements = workerCapabilityRequirements - } - } - - public let entries: [Entry] - public let priority: Priority +public struct TestArgFile: Codable, Equatable { + public let entries: [TestArgFileEntry] + public let jobGroupId: JobGroupId + public let jobGroupPriority: Priority + public let jobId: JobId + public let jobPriority: Priority public let testDestinationConfigurations: [TestDestinationConfiguration] public init( - entries: [Entry], - priority: Priority, + entries: [TestArgFileEntry], + jobGroupId: JobGroupId, + jobGroupPriority: Priority, + jobId: JobId, + jobPriority: Priority, testDestinationConfigurations: [TestDestinationConfiguration] ) { self.entries = entries - self.priority = priority + self.jobGroupId = jobGroupId + self.jobGroupPriority = jobGroupPriority + self.jobId = jobId + self.jobPriority = jobPriority self.testDestinationConfigurations = testDestinationConfigurations } + + public init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + + let entries = try container.decode([TestArgFileEntry].self, forKey: .entries) + let jobId = try container.decode(JobId.self, forKey: .jobId) + + let jobPriority = try container.decodeIfPresent(Priority.self, forKey: .jobPriority) ?? + TestArgFileDefaultValues.priority + let jobGroupId = try container.decodeIfPresent(JobGroupId.self, forKey: .jobGroupId) ?? + JobGroupId(jobId.value) + let jobGroupPriority = try container.decodeIfPresent(Priority.self, forKey: .jobGroupPriority) ?? + jobPriority + let testDestinationConfigurations = try container.decodeIfPresent([TestDestinationConfiguration].self, forKey: .testDestinationConfigurations) ?? + [] + + self.init( + entries: entries, + jobGroupId: jobGroupId, + jobGroupPriority: jobGroupPriority, + jobId: jobId, + jobPriority: jobPriority, + testDestinationConfigurations: testDestinationConfigurations + ) + } } diff --git a/Sources/TestArgFile/TestArgFileDefaultValues.swift b/Sources/TestArgFile/TestArgFileDefaultValues.swift new file mode 100644 index 00000000..4a072e0a --- /dev/null +++ b/Sources/TestArgFile/TestArgFileDefaultValues.swift @@ -0,0 +1,49 @@ +import BuildArtifacts +import DeveloperDirModels +import Foundation +import PluginSupport +import QueueModels +import RunnerModels +import ScheduleStrategy +import SimulatorPoolModels +import WorkerCapabilitiesModels + +public enum TestArgFileDefaultValues { + public static let developerDir = DeveloperDir.current + public static let environment: [String: String] = [:] + public static let numberOfRetries: UInt = 1 + public static let pluginLocations: Set = [] + public static let priority = Priority.medium + public static let scheduleStrategy: ScheduleStrategyType = .progressive + public static let simulatorControlTool = SimulatorControlTool( + location: .insideUserLibrary, + tool: .simctl + ) + public static let simulatorOperationTimeouts = SimulatorOperationTimeouts( + create: 60, + boot: 180, + delete: 30, + shutdown: 30, + automaticSimulatorShutdown: 300, + automaticSimulatorDelete: 300 + ) + public static let simulatorSettings = SimulatorSettings( + simulatorLocalizationSettings: SimulatorLocalizationSettings( + localeIdentifier: "ru_US", + keyboards: ["ru_RU@sw=Russian;hw=Automatic", "en_US@sw=QWERTY;hw=Automatic"], + passcodeKeyboards: ["ru_RU@sw=Russian;hw=Automatic", "en_US@sw=QWERTY;hw=Automatic"], + languages: ["ru-US", "en", "ru-RU"], + addingEmojiKeybordHandled: true, + enableKeyboardExpansion: true, + didShowInternationalInfoAlert: true, + didShowContinuousPathIntroduction: true + ), + watchdogSettings: WatchdogSettings(bundleIds: [], timeout: 20) + ) + public static let testRunnerTool: TestRunnerTool = .xcodebuild(nil) + public static let testTimeoutConfiguration = TestTimeoutConfiguration( + singleTestMaximumDuration: 180, + testRunnerMaximumSilenceDuration: 60 + ) + public static let workerCapabilityRequirements: Set = [] +} diff --git a/Sources/TestArgFile/TestArgFileEntry.swift b/Sources/TestArgFile/TestArgFileEntry.swift new file mode 100644 index 00000000..912ac491 --- /dev/null +++ b/Sources/TestArgFile/TestArgFileEntry.swift @@ -0,0 +1,112 @@ +import BuildArtifacts +import DeveloperDirModels +import Foundation +import PluginSupport +import QueueModels +import RunnerModels +import ScheduleStrategy +import SimulatorPoolModels +import WorkerCapabilitiesModels + +public struct TestArgFileEntry: Codable, Equatable { + public let buildArtifacts: BuildArtifacts + public let developerDir: DeveloperDir + public let environment: [String: String] + public let numberOfRetries: UInt + public let pluginLocations: Set + public let scheduleStrategy: ScheduleStrategyType + public let simulatorControlTool: SimulatorControlTool + public let simulatorOperationTimeouts: SimulatorOperationTimeouts + public let simulatorSettings: SimulatorSettings + public let testDestination: TestDestination + public let testRunnerTool: TestRunnerTool + public let testTimeoutConfiguration: TestTimeoutConfiguration + public let testType: TestType + public let testsToRun: [TestToRun] + public let workerCapabilityRequirements: Set + + public init( + buildArtifacts: BuildArtifacts, + developerDir: DeveloperDir, + environment: [String: String], + numberOfRetries: UInt, + pluginLocations: Set, + scheduleStrategy: ScheduleStrategyType, + simulatorControlTool: SimulatorControlTool, + simulatorOperationTimeouts: SimulatorOperationTimeouts, + simulatorSettings: SimulatorSettings, + testDestination: TestDestination, + testRunnerTool: TestRunnerTool, + testTimeoutConfiguration: TestTimeoutConfiguration, + testType: TestType, + testsToRun: [TestToRun], + workerCapabilityRequirements: Set + ) { + self.buildArtifacts = buildArtifacts + self.developerDir = developerDir + self.environment = environment + self.numberOfRetries = numberOfRetries + self.pluginLocations = pluginLocations + self.scheduleStrategy = scheduleStrategy + self.simulatorControlTool = simulatorControlTool + self.simulatorOperationTimeouts = simulatorOperationTimeouts + self.simulatorSettings = simulatorSettings + self.testDestination = testDestination + self.testRunnerTool = testRunnerTool + self.testTimeoutConfiguration = testTimeoutConfiguration + self.testType = testType + self.testsToRun = testsToRun + self.workerCapabilityRequirements = workerCapabilityRequirements + } + + public init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + + let buildArtifacts = try container.decode(BuildArtifacts.self, forKey: .buildArtifacts) + let testDestination = try container.decode(TestDestination.self, forKey: .testDestination) + let testType = try container.decode(TestType.self, forKey: .testType) + let testsToRun = try container.decode([TestToRun].self, forKey: .testsToRun) + + let developerDir = try container.decodeIfPresent(DeveloperDir.self, forKey: .developerDir) ?? + TestArgFileDefaultValues.developerDir + let environment = try container.decodeIfPresent([String: String].self, forKey: .environment) ?? + TestArgFileDefaultValues.environment + let numberOfRetries = try container.decodeIfPresent(UInt.self, forKey: .numberOfRetries) ?? + TestArgFileDefaultValues.numberOfRetries + let pluginLocations = try container.decodeIfPresent(Set.self, forKey: .pluginLocations) ?? + TestArgFileDefaultValues.pluginLocations + let scheduleStrategy = try container.decodeIfPresent(ScheduleStrategyType.self, forKey: .scheduleStrategy) ?? + TestArgFileDefaultValues.scheduleStrategy + let simulatorControlTool = try container.decodeIfPresent(SimulatorControlTool.self, forKey: .simulatorControlTool) ?? + TestArgFileDefaultValues.simulatorControlTool + let simulatorOperationTimeouts = try container.decodeIfPresent(SimulatorOperationTimeouts.self, forKey: .simulatorOperationTimeouts) ?? + TestArgFileDefaultValues.simulatorOperationTimeouts + let simulatorSettings = try container.decodeIfPresent(SimulatorSettings.self, forKey: .simulatorSettings) ?? + TestArgFileDefaultValues.simulatorSettings + + let testRunnerTool = try container.decodeIfPresent(TestRunnerTool.self, forKey: .testRunnerTool) ?? + TestArgFileDefaultValues.testRunnerTool + let testTimeoutConfiguration = try container.decodeIfPresent(TestTimeoutConfiguration.self, forKey: .testTimeoutConfiguration) ?? + TestArgFileDefaultValues.testTimeoutConfiguration + let workerCapabilityRequirements = try container.decodeIfPresent(Set.self, forKey: .workerCapabilityRequirements) ?? + TestArgFileDefaultValues.workerCapabilityRequirements + + self.init( + buildArtifacts: buildArtifacts, + developerDir: developerDir, + environment: environment, + numberOfRetries: numberOfRetries, + pluginLocations: pluginLocations, + scheduleStrategy: scheduleStrategy, + simulatorControlTool: simulatorControlTool, + simulatorOperationTimeouts: simulatorOperationTimeouts, + simulatorSettings: simulatorSettings, + testDestination: testDestination, + testRunnerTool: testRunnerTool, + testTimeoutConfiguration: testTimeoutConfiguration, + testType: testType, + testsToRun: testsToRun, + workerCapabilityRequirements: workerCapabilityRequirements + ) + } +} diff --git a/Sources/TestArgFile/TestArgFileValidator.swift b/Sources/TestArgFile/TestArgFileValidator.swift new file mode 100644 index 00000000..d88333d1 --- /dev/null +++ b/Sources/TestArgFile/TestArgFileValidator.swift @@ -0,0 +1,94 @@ +import BuildArtifacts +import Foundation +import RunnerModels +import SimulatorPoolModels + +public final class TestArgFileValidator { + public init() {} + + public struct TestArgFileValidationError: Error, CustomStringConvertible { + public let errors: [Error] + + public var description: String { + "Test arg file has the following errors:\n" + errors.map { "\($0)" }.joined(separator: "\n") + } + } + + private struct EntryValidationError: Error, CustomStringConvertible { + let entryIndex: Int + let error: Error + + var description: String { + "Test arg file entry at index \(entryIndex) has configuration error: \(error)" + } + } + + public func validate(testArgFile: TestArgFile) throws { + var errors = [EntryValidationError]() + + for (index, entry) in testArgFile.entries.enumerated() { + do { + try validate(entry: entry, index: index) + } catch { + errors.append(EntryValidationError(entryIndex: index, error: error)) + } + } + + if !errors.isEmpty { + throw TestArgFileValidationError(errors: errors) + } + } + + private func validate(entry: TestArgFileEntry, index: Int) throws { + try validate(buildArtifacts: entry.buildArtifacts, testType: entry.testType) + try validate(simulatorControlTool: entry.simulatorControlTool, testRunnerTool: entry.testRunnerTool) + } + + private func validate(simulatorControlTool: SimulatorControlTool, testRunnerTool: TestRunnerTool) throws { + enum SimulatorToolAndTestRunnerToolMisconfiguration: Error, CustomStringConvertible { + case xcodebuildAndSimulatorLocationIncompatibility(simulatorLocation: SimulatorLocation) + + var description: String { + switch self { + case .xcodebuildAndSimulatorLocationIncompatibility(let simulatorLocation): + return "xcodebuild is not compatible with provided simulator location (\(simulatorLocation.rawValue)). Use \(SimulatorLocation.insideUserLibrary.rawValue) instead." + } + } + } + + switch (simulatorControlTool.location, testRunnerTool) { + case (.insideEmceeTempFolder, .xcodebuild): + throw SimulatorToolAndTestRunnerToolMisconfiguration.xcodebuildAndSimulatorLocationIncompatibility(simulatorLocation: simulatorControlTool.location) + default: + return + } + } + + private func validate(buildArtifacts: BuildArtifacts, testType: TestType) throws { + enum BuildArtifactsValidationError: Error, CustomStringConvertible { + case missingBuildArtifact(TestType, kind: String) + var description: String { + switch self { + case .missingBuildArtifact(let testType, let kind): + return "Test type \(testType.rawValue) requires \(kind) to be provided" + } + } + } + + switch testType { + case .logicTest: + break + case .appTest: + if buildArtifacts.appBundle == nil { + throw BuildArtifactsValidationError.missingBuildArtifact(testType, kind: "appBundle") + } + case .uiTest: + if buildArtifacts.appBundle == nil { + throw BuildArtifactsValidationError.missingBuildArtifact(testType, kind: "appBundle") + } + if buildArtifacts.runner == nil { + throw BuildArtifactsValidationError.missingBuildArtifact(testType, kind: "runner (XCTRunner.app)") + } + } + } +} diff --git a/Sources/TestArgFile/TestDestinationConfiguration.swift b/Sources/TestArgFile/TestDestinationConfiguration.swift index 74302009..e30afdee 100644 --- a/Sources/TestArgFile/TestDestinationConfiguration.swift +++ b/Sources/TestArgFile/TestDestinationConfiguration.swift @@ -2,7 +2,7 @@ import Foundation import RunnerModels import SimulatorPoolModels -public struct TestDestinationConfiguration: Codable { +public struct TestDestinationConfiguration: Codable, Equatable { public let testDestination: TestDestination public let reportOutput: ReportOutput diff --git a/Sources/TestArgFile/TestToRun.swift b/Sources/TestArgFile/TestToRun.swift index bac1da3d..3ec3dea1 100644 --- a/Sources/TestArgFile/TestToRun.swift +++ b/Sources/TestArgFile/TestToRun.swift @@ -30,14 +30,24 @@ public enum TestToRun: Codable, CustomStringConvertible, Hashable { } public init(from decoder: Decoder) throws { - let container = try decoder.container(keyedBy: CodingKeys.self) - let predicateType = try container.decode(PredicateType.self, forKey: .predicateType) - - switch predicateType { - case .allDiscoveredTests: - self = .allDiscoveredTests - case .singleTestName: - self = .testName(try container.decode(TestName.self, forKey: .testName)) + do { + let container = try decoder.singleValueContainer() + let value = try container.decode(String.self) + if value == "all" { + self = .allDiscoveredTests + } else { + self = .testName(try TestName(from: decoder)) + } + } catch { + let container = try decoder.container(keyedBy: CodingKeys.self) + let predicateType = try container.decode(PredicateType.self, forKey: .predicateType) + + switch predicateType { + case .allDiscoveredTests: + self = .allDiscoveredTests + case .singleTestName: + self = .testName(try container.decode(TestName.self, forKey: .testName)) + } } } diff --git a/Tests/EmceeLibTests/TestEntriesValidatorTests.swift b/Tests/EmceeLibTests/TestEntriesValidatorTests.swift index 3bd98f25..b92ab6e6 100644 --- a/Tests/EmceeLibTests/TestEntriesValidatorTests.swift +++ b/Tests/EmceeLibTests/TestEntriesValidatorTests.swift @@ -88,7 +88,7 @@ final class TestEntriesValidatorTests: XCTestCase { } private func createValidator( - testArgFileEntries: [TestArgFile.Entry] + testArgFileEntries: [TestArgFileEntry] ) -> TestEntriesValidator { return TestEntriesValidator( testArgFileEntries: testArgFileEntries, @@ -99,8 +99,8 @@ final class TestEntriesValidatorTests: XCTestCase { private func createTestEntry( testType: TestType, buildArtifacts: BuildArtifacts = BuildArtifactsFixtures.fakeEmptyBuildArtifacts() - ) throws -> TestArgFile.Entry { - return TestArgFile.Entry( + ) throws -> TestArgFileEntry { + return TestArgFileEntry( buildArtifacts: buildArtifacts, developerDir: .current, environment: [:], diff --git a/Tests/EmceeLibTests/TestEntryConfigurationGeneratorTests.swift b/Tests/EmceeLibTests/TestEntryConfigurationGeneratorTests.swift index e96a27a9..6975880e 100644 --- a/Tests/EmceeLibTests/TestEntryConfigurationGeneratorTests.swift +++ b/Tests/EmceeLibTests/TestEntryConfigurationGeneratorTests.swift @@ -45,7 +45,7 @@ final class TestEntryConfigurationGeneratorTests: XCTestCase { func test() { let generator = TestEntryConfigurationGenerator( validatedEntries: validatedEntries, - testArgFileEntry: TestArgFile.Entry( + testArgFileEntry: TestArgFileEntry( buildArtifacts: buildArtifacts, developerDir: .current, environment: [:], @@ -82,7 +82,7 @@ final class TestEntryConfigurationGeneratorTests: XCTestCase { func test_repeated_items() { let generator = TestEntryConfigurationGenerator( validatedEntries: validatedEntries, - testArgFileEntry: TestArgFile.Entry( + testArgFileEntry: TestArgFileEntry( buildArtifacts: buildArtifacts, developerDir: .current, environment: [:], @@ -121,7 +121,7 @@ final class TestEntryConfigurationGeneratorTests: XCTestCase { func test__all_available_tests() { let generator = TestEntryConfigurationGenerator( validatedEntries: validatedEntries, - testArgFileEntry: TestArgFile.Entry( + testArgFileEntry: TestArgFileEntry( buildArtifacts: buildArtifacts, developerDir: .current, environment: [:], diff --git a/Tests/TestArgFileTests/TestArgFileEntryTests.swift b/Tests/TestArgFileTests/TestArgFileEntryTests.swift new file mode 100644 index 00000000..cfa2ebcc --- /dev/null +++ b/Tests/TestArgFileTests/TestArgFileEntryTests.swift @@ -0,0 +1,189 @@ +import BuildArtifacts +import BuildArtifactsTestHelpers +import Foundation +import PluginSupport +import RunnerModels +import RunnerTestHelpers +import SimulatorPoolModels +import SimulatorPoolTestHelpers +import TestArgFile +import TestHelpers +import XCTest + +final class TestArgFileEntryTests: XCTestCase { + func test___decoding_full_json() throws { + let json = """ + { + "testsToRun": [ + {"predicateType": "singleTestName", "testName": "ClassName/testMethod"} + ], + "environment": {"value": "key"}, + "numberOfRetries": 42, + "testDestination": {"deviceType": "iPhone SE", "runtime": "11.3"}, + "testType": "logicTest", + "buildArtifacts": { + "appBundle": "/appBundle", + "runner": "/runner", + "xcTestBundle": { + "location": "/xcTestBundle", + "testDiscoveryMode": "runtimeAppTest" + }, + "additionalApplicationBundles": ["/additionalApp1", "/additionalApp2"] + }, + "testRunnerTool": {"toolType": "fbxctest", "fbxctestLocation": "http://example.com/fbxctest.zip"}, + "simulatorControlTool": { + "location": "insideUserLibrary", + "tool": { + "toolType": "fbsimctl", + "location": "http://example.com/fbsimctl.zip" + } + }, + "developerDir": {"kind": "current"}, + "pluginLocations": [ + "http://example.com/plugin.zip#sample.emceeplugin" + ], + "scheduleStrategy": "unsplit", + "simulatorOperationTimeouts": { + "create": 50, + "boot": 51, + "delete": 52, + "shutdown": 53, + "automaticSimulatorShutdown": 54, + "automaticSimulatorDelete": 55 + }, + "simulatorSettings": { + "simulatorLocalizationSettings": { + "localeIdentifier": "ru_US", + "keyboards": ["ru_RU@sw=Russian;hw=Automatic", "en_US@sw=QWERTY;hw=Automatic"], + "passcodeKeyboards": ["ru_RU@sw=Russian;hw=Automatic", "en_US@sw=QWERTY;hw=Automatic"], + "languages": ["ru-US", "en", "ru-RU"], + "addingEmojiKeybordHandled": true, + "enableKeyboardExpansion": true, + "didShowInternationalInfoAlert": true, + "didShowContinuousPathIntroduction": true + }, + "watchdogSettings": { + "bundleIds": ["sample.app"], + "timeout": 42 + }, + }, + "testTimeoutConfiguration": { + "singleTestMaximumDuration": 42, + "testRunnerMaximumSilenceDuration": 24 + }, + "workerCapabilityRequirements": [] + } + """.data(using: .utf8)! + + let entry = assertDoesNotThrow { + try JSONDecoder().decode(TestArgFileEntry.self, from: json) + } + + XCTAssertEqual( + entry, + TestArgFileEntry( + buildArtifacts: buildArtifacts(), + developerDir: .current, + environment: ["value": "key"], + numberOfRetries: 42, + pluginLocations: [ + PluginLocation(.remoteUrl(URL(string: "http://example.com/plugin.zip#sample.emceeplugin")!)) + ], + scheduleStrategy: .unsplit, + simulatorControlTool: SimulatorControlTool( + location: .insideUserLibrary, + tool: .fbsimctl(FbsimctlLocation(.remoteUrl(URL(string: "http://example.com/fbsimctl.zip")!))) + ), + simulatorOperationTimeouts: SimulatorOperationTimeouts( + create: 50, + boot: 51, + delete: 52, + shutdown: 53, + automaticSimulatorShutdown: 54, + automaticSimulatorDelete: 55 + ), + simulatorSettings: SimulatorSettings( + simulatorLocalizationSettings: SimulatorLocalizationSettingsFixture().simulatorLocalizationSettings(), + watchdogSettings: WatchdogSettings(bundleIds: ["sample.app"], timeout: 42) + ), + testDestination: try TestDestination(deviceType: "iPhone SE", runtime: "11.3"), + testRunnerTool: .fbxctest(FbxctestLocation(.remoteUrl(URL(string: "http://example.com/fbxctest.zip")!))), + testTimeoutConfiguration: TestTimeoutConfiguration( + singleTestMaximumDuration: 42, + testRunnerMaximumSilenceDuration: 24 + ), + testType: .logicTest, + testsToRun: [.testName(TestName(className: "ClassName", methodName: "testMethod"))], + workerCapabilityRequirements: [] + ) + ) + } + + func test___decoding_short_json() throws { + let json = """ + { + "testsToRun": [ + "all", + "ClassName/testMethod", + {"predicateType": "singleTestName", "testName": "ClassName/testMethod"} + ], + "testDestination": {"deviceType": "iPhone SE", "runtime": "11.3"}, + "testType": "logicTest", + "buildArtifacts": { + "appBundle": "/appBundle", + "runner": "/runner", + "xcTestBundle": { + "location": "/xcTestBundle", + "testDiscoveryMode": "runtimeAppTest" + }, + "additionalApplicationBundles": ["/additionalApp1", "/additionalApp2"] + } + } + """.data(using: .utf8)! + + let entry = assertDoesNotThrow { + try JSONDecoder().decode(TestArgFileEntry.self, from: json) + } + + XCTAssertEqual( + entry, + TestArgFileEntry( + buildArtifacts: buildArtifacts(), + developerDir: TestArgFileDefaultValues.developerDir, + environment: TestArgFileDefaultValues.environment, + numberOfRetries: TestArgFileDefaultValues.numberOfRetries, + pluginLocations: TestArgFileDefaultValues.pluginLocations, + scheduleStrategy: TestArgFileDefaultValues.scheduleStrategy, + simulatorControlTool: TestArgFileDefaultValues.simulatorControlTool, + simulatorOperationTimeouts: TestArgFileDefaultValues.simulatorOperationTimeouts, + simulatorSettings: TestArgFileDefaultValues.simulatorSettings, + testDestination: try TestDestination(deviceType: "iPhone SE", runtime: "11.3"), + testRunnerTool: TestArgFileDefaultValues.testRunnerTool, + testTimeoutConfiguration: TestArgFileDefaultValues.testTimeoutConfiguration, + testType: .logicTest, + testsToRun: [ + .allDiscoveredTests, + .testName(TestName(className: "ClassName", methodName: "testMethod")), + .testName(TestName(className: "ClassName", methodName: "testMethod")), + ], + workerCapabilityRequirements: TestArgFileDefaultValues.workerCapabilityRequirements + ) + ) + } + + private func buildArtifacts( + appBundle: String? = "/appBundle", + runner: String? = "/runner", + additionalApplicationBundles: [String] = ["/additionalApp1", "/additionalApp2"], + testDiscoveryMode: XcTestBundleTestDiscoveryMode = .runtimeAppTest + ) -> BuildArtifacts { + return BuildArtifactsFixtures.withLocalPaths( + appBundle: appBundle, + runner: runner, + xcTestBundle: "/xcTestBundle", + additionalApplicationBundles: additionalApplicationBundles, + testDiscoveryMode: testDiscoveryMode + ) + } +} + diff --git a/Tests/TestArgFileTests/TestArgFileTests.swift b/Tests/TestArgFileTests/TestArgFileTests.swift index e61279ae..59e39d2f 100644 --- a/Tests/TestArgFileTests/TestArgFileTests.swift +++ b/Tests/TestArgFileTests/TestArgFileTests.swift @@ -1,136 +1,104 @@ import BuildArtifacts -import BuildArtifactsTestHelpers import Foundation -import PluginSupport -import RunnerModels -import RunnerTestHelpers +import ResourceLocation import SimulatorPoolModels -import SimulatorPoolTestHelpers import TestArgFile -import TestHelpers import XCTest final class TestArgFileTests: XCTestCase { func test___decoding_full_json() throws { let json = """ { - "testsToRun": [ - {"predicateType": "singleTestName", "testName": "ClassName/testMethod"} - ], - "environment": {"value": "key"}, - "numberOfRetries": 42, - "testDestination": {"deviceType": "iPhone SE", "runtime": "11.3"}, - "testType": "logicTest", - "buildArtifacts": { - "appBundle": "/appBundle", - "runner": "/runner", - "xcTestBundle": { - "location": "/xcTestBundle", - "testDiscoveryMode": "runtimeAppTest" - }, - "additionalApplicationBundles": ["/additionalApp1", "/additionalApp2"] - }, - "testRunnerTool": {"toolType": "fbxctest", "fbxctestLocation": "http://example.com/fbxctest.zip"}, - "simulatorControlTool": { - "location": "insideUserLibrary", - "tool": { - "toolType": "fbsimctl", - "location": "http://example.com/fbsimctl.zip" - } - }, - "developerDir": {"kind": "current"}, - "pluginLocations": [ - "http://example.com/plugin.zip#sample.emceeplugin" - ], - "scheduleStrategy": "unsplit", - "simulatorOperationTimeouts": { - "create": 50, - "boot": 51, - "delete": 52, - "shutdown": 53, - "automaticSimulatorShutdown": 54, - "automaticSimulatorDelete": 55 - }, - "simulatorSettings": { - "simulatorLocalizationSettings": { - "localeIdentifier": "ru_US", - "keyboards": ["ru_RU@sw=Russian;hw=Automatic", "en_US@sw=QWERTY;hw=Automatic"], - "passcodeKeyboards": ["ru_RU@sw=Russian;hw=Automatic", "en_US@sw=QWERTY;hw=Automatic"], - "languages": ["ru-US", "en", "ru-RU"], - "addingEmojiKeybordHandled": true, - "enableKeyboardExpansion": true, - "didShowInternationalInfoAlert": true, - "didShowContinuousPathIntroduction": true - }, - "watchdogSettings": { - "bundleIds": ["sample.app"], - "timeout": 42 - }, - }, - "testTimeoutConfiguration": { - "singleTestMaximumDuration": 42, - "testRunnerMaximumSilenceDuration": 24 - }, - "workerCapabilityRequirements": [] + "entries": [], + "jobGroupId": "jobGroupId", + "jobGroupPriority": 100, + "jobId": "jobId", + "jobPriority": 500, + "testDestinationConfigurations": [] } """.data(using: .utf8)! - let entry = assertDoesNotThrow { - try JSONDecoder().decode(TestArgFile.Entry.self, from: json) + let testArgFile = assertDoesNotThrow { + try JSONDecoder().decode(TestArgFile.self, from: json) } XCTAssertEqual( - entry, - TestArgFile.Entry( - buildArtifacts: buildArtifacts(), - developerDir: .current, - environment: ["value": "key"], - numberOfRetries: 42, - pluginLocations: [ - PluginLocation(.remoteUrl(URL(string: "http://example.com/plugin.zip#sample.emceeplugin")!)) - ], - scheduleStrategy: .unsplit, - simulatorControlTool: SimulatorControlTool( - location: .insideUserLibrary, - tool: .fbsimctl(FbsimctlLocation(.remoteUrl(URL(string: "http://example.com/fbsimctl.zip")!))) - ), - simulatorOperationTimeouts: SimulatorOperationTimeouts( - create: 50, - boot: 51, - delete: 52, - shutdown: 53, - automaticSimulatorShutdown: 54, - automaticSimulatorDelete: 55 - ), - simulatorSettings: SimulatorSettings( - simulatorLocalizationSettings: SimulatorLocalizationSettingsFixture().simulatorLocalizationSettings(), - watchdogSettings: WatchdogSettings(bundleIds: ["sample.app"], timeout: 42) - ), - testDestination: try TestDestination(deviceType: "iPhone SE", runtime: "11.3"), - testRunnerTool: .fbxctest(FbxctestLocation(.remoteUrl(URL(string: "http://example.com/fbxctest.zip")!))), - testTimeoutConfiguration: TestTimeoutConfiguration( - singleTestMaximumDuration: 42, - testRunnerMaximumSilenceDuration: 24 - ), - testType: .logicTest, - testsToRun: [.testName(TestName(className: "ClassName", methodName: "testMethod"))], - workerCapabilityRequirements: [] + testArgFile, + TestArgFile( + entries: [], + jobGroupId: "jobGroupId", + jobGroupPriority: 100, + jobId: "jobId", + jobPriority: 500, + testDestinationConfigurations: [] + ) + ) + } + + func test___decoding_short_json() throws { + let json = """ + { + "entries": [], + "jobId": "jobId", + } + """.data(using: .utf8)! + + let testArgFile = assertDoesNotThrow { + try JSONDecoder().decode(TestArgFile.self, from: json) + } + + XCTAssertEqual( + testArgFile, + TestArgFile( + entries: [], + jobGroupId: "jobId", + jobGroupPriority: TestArgFileDefaultValues.priority, + jobId: "jobId", + jobPriority: TestArgFileDefaultValues.priority, + testDestinationConfigurations: [] ) ) } - private func buildArtifacts( - appBundle: String? = "/appBundle", - runner: String? = "/runner", - additionalApplicationBundles: [String] = ["/additionalApp1", "/additionalApp2"], - testDiscoveryMode: XcTestBundleTestDiscoveryMode = .runtimeAppTest - ) -> BuildArtifacts { - return BuildArtifactsFixtures.withLocalPaths( - appBundle: appBundle, - runner: runner, - xcTestBundle: "/xcTestBundle", - additionalApplicationBundles: additionalApplicationBundles, - testDiscoveryMode: testDiscoveryMode + func test___complete_short_example() throws { + let json = """ + { + "jobId": "jobId", + "entries": [ + { + "testsToRun": ["all"], + "testDestination": {"deviceType": "iPhone X", "runtime": "11.3"}, + "testType": "uiTest", + "buildArtifacts": { + "appBundle": "http://example.com/App.zip#MyApp/MyApp.app", + "runner": "http://example.com/App.zip#Tests/UITests-Runner.app", + "xcTestBundle": "http://example.com/App.zip#Tests/UITests-Runner.app/PlugIns/UITests.xctest" + } + } + ] + } + """.data(using: .utf8)! + + let testArgFile = assertDoesNotThrow { + try JSONDecoder().decode(TestArgFile.self, from: json) + } + + XCTAssertEqual(testArgFile.jobId, "jobId") + XCTAssertEqual(testArgFile.entries.count, 1) + XCTAssertEqual(testArgFile.entries[0].testsToRun, [.allDiscoveredTests]) + XCTAssertEqual(testArgFile.entries[0].testDestination, try TestDestination(deviceType: "iPhone X", runtime: "11.3")) + XCTAssertEqual(testArgFile.entries[0].testType, .uiTest) + XCTAssertEqual( + testArgFile.entries[0].buildArtifacts, + BuildArtifacts( + appBundle: AppBundleLocation(try .from("http://example.com/App.zip#MyApp/MyApp.app")), + runner: RunnerAppLocation(try .from("http://example.com/App.zip#Tests/UITests-Runner.app")), + xcTestBundle: XcTestBundle( + location: TestBundleLocation(try .from("http://example.com/App.zip#Tests/UITests-Runner.app/PlugIns/UITests.xctest")), + testDiscoveryMode: .parseFunctionSymbols + ), + additionalApplicationBundles: [] + ) ) } } diff --git a/Tests/TestArgFileTests/TestArgFileValidatorTests.swift b/Tests/TestArgFileTests/TestArgFileValidatorTests.swift new file mode 100644 index 00000000..34908ab2 --- /dev/null +++ b/Tests/TestArgFileTests/TestArgFileValidatorTests.swift @@ -0,0 +1,177 @@ +import BuildArtifacts +import BuildArtifactsTestHelpers +import Foundation +import RunnerModels +import SimulatorPoolModels +import SimulatorPoolTestHelpers +import TestArgFile +import TestHelpers +import XCTest + +final class TestArgFileValidatorTests: XCTestCase { + func test___successful() { + let testArgFile = TestArgFile( + entries: [ + TestArgFileEntry( + buildArtifacts: BuildArtifactsFixtures.fakeEmptyBuildArtifacts(), + developerDir: .current, + environment: [:], + numberOfRetries: 0, + pluginLocations: [], + scheduleStrategy: .unsplit, + simulatorControlTool: SimulatorControlTool(location: .insideUserLibrary, tool: .simctl), + simulatorOperationTimeouts: SimulatorOperationTimeoutsFixture().simulatorOperationTimeouts(), + simulatorSettings: SimulatorSettingsFixtures().simulatorSettings(), + testDestination: TestDestinationFixtures.testDestination, + testRunnerTool: .xcodebuild(nil), + testTimeoutConfiguration: TestTimeoutConfiguration(singleTestMaximumDuration: 0, testRunnerMaximumSilenceDuration: 0), + testType: .appTest, + testsToRun: [], + workerCapabilityRequirements: [] + ) + ], + jobGroupId: "", + jobGroupPriority: 0, + jobId: "", + jobPriority: 0, + testDestinationConfigurations: [] + ) + + assertDoesNotThrow { + try TestArgFileValidator().validate(testArgFile: testArgFile) + } + } + + func test___insideEmceeTempFolder_and_xcodebuild___incompatible() { + let testArgFile = TestArgFile( + entries: [ + TestArgFileEntry( + buildArtifacts: BuildArtifactsFixtures.fakeEmptyBuildArtifacts(), + developerDir: .current, + environment: [:], + numberOfRetries: 0, + pluginLocations: [], + scheduleStrategy: .unsplit, + simulatorControlTool: SimulatorControlTool(location: .insideEmceeTempFolder, tool: .simctl), + simulatorOperationTimeouts: SimulatorOperationTimeoutsFixture().simulatorOperationTimeouts(), + simulatorSettings: SimulatorSettingsFixtures().simulatorSettings(), + testDestination: TestDestinationFixtures.testDestination, + testRunnerTool: .xcodebuild(nil), + testTimeoutConfiguration: TestTimeoutConfiguration(singleTestMaximumDuration: 0, testRunnerMaximumSilenceDuration: 0), + testType: .appTest, + testsToRun: [], + workerCapabilityRequirements: [] + ) + ], + jobGroupId: "", + jobGroupPriority: 0, + jobId: "", + jobPriority: 0, + testDestinationConfigurations: [] + ) + + assertThrows { + try TestArgFileValidator().validate(testArgFile: testArgFile) + } + } + + func test___appTest_should_require_appBundle_presense() { + let testArgFile = TestArgFile( + entries: [ + TestArgFileEntry( + buildArtifacts: BuildArtifactsFixtures.withLocalPaths(appBundle: nil, runner: nil, xcTestBundle: "", additionalApplicationBundles: []), + developerDir: .current, + environment: [:], + numberOfRetries: 0, + pluginLocations: [], + scheduleStrategy: .unsplit, + simulatorControlTool: SimulatorControlTool(location: .insideUserLibrary, tool: .simctl), + simulatorOperationTimeouts: SimulatorOperationTimeoutsFixture().simulatorOperationTimeouts(), + simulatorSettings: SimulatorSettingsFixtures().simulatorSettings(), + testDestination: TestDestinationFixtures.testDestination, + testRunnerTool: .xcodebuild(nil), + testTimeoutConfiguration: TestTimeoutConfiguration(singleTestMaximumDuration: 0, testRunnerMaximumSilenceDuration: 0), + testType: .appTest, + testsToRun: [], + workerCapabilityRequirements: [] + ) + ], + jobGroupId: "", + jobGroupPriority: 0, + jobId: "", + jobPriority: 0, + testDestinationConfigurations: [] + ) + + assertThrows { + try TestArgFileValidator().validate(testArgFile: testArgFile) + } + } + + func test___uiTest_should_require_appBundle_presense() { + let testArgFile = TestArgFile( + entries: [ + TestArgFileEntry( + buildArtifacts: BuildArtifactsFixtures.withLocalPaths(appBundle: nil, runner: nil, xcTestBundle: "", additionalApplicationBundles: []), + developerDir: .current, + environment: [:], + numberOfRetries: 0, + pluginLocations: [], + scheduleStrategy: .unsplit, + simulatorControlTool: SimulatorControlTool(location: .insideUserLibrary, tool: .simctl), + simulatorOperationTimeouts: SimulatorOperationTimeoutsFixture().simulatorOperationTimeouts(), + simulatorSettings: SimulatorSettingsFixtures().simulatorSettings(), + testDestination: TestDestinationFixtures.testDestination, + testRunnerTool: .xcodebuild(nil), + testTimeoutConfiguration: TestTimeoutConfiguration(singleTestMaximumDuration: 0, testRunnerMaximumSilenceDuration: 0), + testType: .uiTest, + testsToRun: [], + workerCapabilityRequirements: [] + ) + ], + jobGroupId: "", + jobGroupPriority: 0, + jobId: "", + jobPriority: 0, + testDestinationConfigurations: [] + ) + + assertThrows { + try TestArgFileValidator().validate(testArgFile: testArgFile) + } + } + + func test___uiTest_should_require_runner_presense() { + let testArgFile = TestArgFile( + entries: [ + TestArgFileEntry( + buildArtifacts: BuildArtifactsFixtures.withLocalPaths(appBundle: "", runner: nil, xcTestBundle: "", additionalApplicationBundles: []), + developerDir: .current, + environment: [:], + numberOfRetries: 0, + pluginLocations: [], + scheduleStrategy: .unsplit, + simulatorControlTool: SimulatorControlTool(location: .insideUserLibrary, tool: .simctl), + simulatorOperationTimeouts: SimulatorOperationTimeoutsFixture().simulatorOperationTimeouts(), + simulatorSettings: SimulatorSettingsFixtures().simulatorSettings(), + testDestination: TestDestinationFixtures.testDestination, + testRunnerTool: .xcodebuild(nil), + testTimeoutConfiguration: TestTimeoutConfiguration(singleTestMaximumDuration: 0, testRunnerMaximumSilenceDuration: 0), + testType: .uiTest, + testsToRun: [], + workerCapabilityRequirements: [] + ) + ], + jobGroupId: "", + jobGroupPriority: 0, + jobId: "", + jobPriority: 0, + testDestinationConfigurations: [] + ) + + assertThrows { + try TestArgFileValidator().validate(testArgFile: testArgFile) + } + } +} +