From afe2505539181edbf0a26fdcd9a9a600ccfd457e Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Tue, 11 Feb 2025 18:04:34 -0600 Subject: [PATCH] Revert "Introduce a severity level for issues, and a 'warning' severity (#931)" This reverts commit 6a49142b223f214eff5da6c1224935a80e62e512. --- .../ABI/EntryPoints/ABIEntryPoint.swift | 6 +- .../Testing/ABI/EntryPoints/EntryPoint.swift | 37 +----- .../ABI/v0/Encoded/ABIv0.EncodedIssue.swift | 19 --- .../ABI/v0/Encoded/ABIv0.EncodedMessage.swift | 3 - Sources/Testing/Events/Event.swift | 5 +- .../Event.ConsoleOutputRecorder.swift | 2 - .../Event.HumanReadableOutputRecorder.swift | 61 ++++----- .../Events/Recorder/Event.Symbol.swift | 16 +-- Sources/Testing/Issues/Issue.swift | 113 ++++------------- .../Running/Configuration+EventHandling.swift | 20 --- Sources/Testing/Running/Configuration.swift | 45 ++----- .../Testing/Running/Runner.RuntimeState.swift | 13 +- Tests/TestingTests/ConfigurationTests.swift | 25 ---- Tests/TestingTests/EntryPointTests.swift | 81 ------------ Tests/TestingTests/EventRecorderTests.swift | 117 +++++------------- Tests/TestingTests/IssueTests.swift | 8 +- .../Runner.RuntimeStateTests.swift | 2 +- Tests/TestingTests/RunnerTests.swift | 4 +- 18 files changed, 109 insertions(+), 468 deletions(-) delete mode 100644 Tests/TestingTests/ConfigurationTests.swift delete mode 100644 Tests/TestingTests/EntryPointTests.swift diff --git a/Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift b/Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift index 143f7b549..cc150740e 100644 --- a/Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift @@ -47,7 +47,7 @@ extension ABIv0 { /// callback. public static var entryPoint: EntryPoint { return { configurationJSON, recordHandler in - try await _entryPoint( + try await Testing.entryPoint( configurationJSON: configurationJSON, recordHandler: recordHandler ) == EXIT_SUCCESS @@ -87,7 +87,7 @@ typealias ABIEntryPoint_v0 = @Sendable ( @usableFromInline func copyABIEntryPoint_v0() -> UnsafeMutableRawPointer { let result = UnsafeMutablePointer.allocate(capacity: 1) result.initialize { configurationJSON, recordHandler in - try await _entryPoint( + try await entryPoint( configurationJSON: configurationJSON, eventStreamVersionIfNil: -1, recordHandler: recordHandler @@ -104,7 +104,7 @@ typealias ABIEntryPoint_v0 = @Sendable ( /// /// This function will be removed (with its logic incorporated into /// ``ABIv0/entryPoint-swift.type.property``) in a future update. -private func _entryPoint( +private func entryPoint( configurationJSON: UnsafeRawBufferPointer?, eventStreamVersionIfNil: Int? = nil, recordHandler: @escaping @Sendable (_ recordJSON: UnsafeRawBufferPointer) -> Void diff --git a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift index 89094c88f..a0a5df2a0 100644 --- a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift @@ -20,8 +20,6 @@ private import _TestingInternals /// writes events to the standard error stream in addition to passing them /// to this function. /// -/// - Returns: An exit code representing the result of running tests. -/// /// External callers cannot call this function directly. The can use /// ``ABIv0/entryPoint-swift.type.property`` to get a reference to an ABI-stable /// version of this function. @@ -42,7 +40,7 @@ func entryPoint(passing args: __CommandLineArguments_v0?, eventHandler: Event.Ha // Set up the event handler. configuration.eventHandler = { [oldEventHandler = configuration.eventHandler] event, context in - if case let .issueRecorded(issue) = event.kind, !issue.isKnown, issue.severity >= .error { + if case let .issueRecorded(issue) = event.kind, !issue.isKnown { exitCode.withLock { exitCode in exitCode = EXIT_FAILURE } @@ -272,13 +270,6 @@ public struct __CommandLineArguments_v0: Sendable { /// The value(s) of the `--skip` argument. public var skip: [String]? - /// Whether or not to include tests with the `.hidden` trait when constructing - /// a test filter based on these arguments. - /// - /// This property is intended for use in testing the testing library itself. - /// It is not parsed as a command-line argument. - var includeHiddenTests: Bool? - /// The value of the `--repetitions` argument. public var repetitions: Int? @@ -287,13 +278,6 @@ public struct __CommandLineArguments_v0: Sendable { /// The value of the `--experimental-attachments-path` argument. public var experimentalAttachmentsPath: String? - - /// Whether or not the experimental warning issue severity feature should be - /// enabled. - /// - /// This property is intended for use in testing the testing library itself. - /// It is not parsed as a command-line argument. - var isWarningIssueRecordedEventEnabled: Bool? } extension __CommandLineArguments_v0: Codable { @@ -533,9 +517,6 @@ public func configurationForEntryPoint(from args: __CommandLineArguments_v0) thr filters.append(try testFilter(forRegularExpressions: args.skip, label: "--skip", membership: .excluding)) configuration.testFilter = filters.reduce(.unfiltered) { $0.combining(with: $1) } - if args.includeHiddenTests == true { - configuration.testFilter.includeHiddenTests = true - } // Set up the iteration policy for the test run. var repetitionPolicy: Configuration.RepetitionPolicy = .once @@ -566,22 +547,6 @@ public func configurationForEntryPoint(from args: __CommandLineArguments_v0) thr configuration.exitTestHandler = ExitTest.handlerForEntryPoint() #endif - // Warning issues (experimental). - if args.isWarningIssueRecordedEventEnabled == true { - configuration.eventHandlingOptions.isWarningIssueRecordedEventEnabled = true - } else { - switch args.eventStreamVersion { - case .some(...0): - // If the event stream version was explicitly specified to a value < 1, - // disable the warning issue event to maintain legacy behavior. - configuration.eventHandlingOptions.isWarningIssueRecordedEventEnabled = false - default: - // Otherwise the requested event stream version is ≥ 1, so don't change - // the warning issue event setting. - break - } - } - return configuration } diff --git a/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedIssue.swift b/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedIssue.swift index 97c051d28..2bf1c8462 100644 --- a/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedIssue.swift +++ b/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedIssue.swift @@ -16,19 +16,6 @@ extension ABIv0 { /// assists in converting values to JSON; clients that consume this JSON are /// expected to write their own decoders. struct EncodedIssue: Sendable { - /// An enumeration representing the level of severity of a recorded issue. - /// - /// For descriptions of individual cases, see ``Issue/Severity-swift.enum``. - enum Severity: String, Sendable { - case warning - case error - } - - /// The severity of this issue. - /// - /// - Warning: Severity is not yet part of the JSON schema. - var _severity: Severity - /// Whether or not this issue is known to occur. var isKnown: Bool @@ -46,11 +33,6 @@ extension ABIv0 { var _error: EncodedError? init(encoding issue: borrowing Issue, in eventContext: borrowing Event.Context) { - _severity = switch issue.severity { - case .warning: .warning - case .error: .error - } - isKnown = issue.isKnown sourceLocation = issue.sourceLocation if let backtrace = issue.sourceContext.backtrace { @@ -66,4 +48,3 @@ extension ABIv0 { // MARK: - Codable extension ABIv0.EncodedIssue: Codable {} -extension ABIv0.EncodedIssue.Severity: Codable {} diff --git a/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedMessage.swift b/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedMessage.swift index cf44f0af0..5cfbf647c 100644 --- a/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedMessage.swift +++ b/Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedMessage.swift @@ -25,7 +25,6 @@ extension ABIv0 { case `default` case skip case pass - case passWithWarnings = "_passWithWarnings" case passWithKnownIssue case fail case difference @@ -45,8 +44,6 @@ extension ABIv0 { } else { .pass } - case .passWithWarnings: - .passWithWarnings case .fail: .fail case .difference: diff --git a/Sources/Testing/Events/Event.swift b/Sources/Testing/Events/Event.swift index b81f1c2c7..60e564d5a 100644 --- a/Sources/Testing/Events/Event.swift +++ b/Sources/Testing/Events/Event.swift @@ -290,7 +290,10 @@ extension Event { if let configuration = configuration ?? Configuration.current { // The caller specified a configuration, or the current task has an // associated configuration. Post to either configuration's event handler. - if configuration.eventHandlingOptions.shouldHandleEvent(self) { + switch kind { + case .expectationChecked where !configuration.deliverExpectationCheckedEvents: + break + default: configuration.handleEvent(self, in: context) } } else { diff --git a/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift index b375b2da1..cce3a732c 100644 --- a/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift @@ -162,8 +162,6 @@ extension Event.Symbol { return "\(_ansiEscapeCodePrefix)90m\(symbolCharacter)\(_resetANSIEscapeCode)" } return "\(_ansiEscapeCodePrefix)92m\(symbolCharacter)\(_resetANSIEscapeCode)" - case .passWithWarnings: - return "\(_ansiEscapeCodePrefix)93m\(symbolCharacter)\(_resetANSIEscapeCode)" case .fail: return "\(_ansiEscapeCodePrefix)91m\(symbolCharacter)\(_resetANSIEscapeCode)" case .warning: diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index 2e40b2789..98303f11c 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -56,9 +56,8 @@ extension Event { /// The instant at which the test started. var startInstant: Test.Clock.Instant - /// The number of issues recorded for the test, grouped by their - /// level of severity. - var issueCount: [Issue.Severity: Int] = [:] + /// The number of issues recorded for the test. + var issueCount = 0 /// The number of known issues recorded for the test. var knownIssueCount = 0 @@ -115,36 +114,27 @@ extension Event.HumanReadableOutputRecorder { /// - graph: The graph to walk while counting issues. /// /// - Returns: A tuple containing the number of issues recorded in `graph`. - private func _issueCounts(in graph: Graph?) -> (errorIssueCount: Int, warningIssueCount: Int, knownIssueCount: Int, totalIssueCount: Int, description: String) { + private func _issueCounts(in graph: Graph?) -> (issueCount: Int, knownIssueCount: Int, totalIssueCount: Int, description: String) { guard let graph else { - return (0, 0, 0, 0, "") + return (0, 0, 0, "") } - let errorIssueCount = graph.compactMap(\.value?.issueCount[.error]).reduce(into: 0, +=) - let warningIssueCount = graph.compactMap(\.value?.issueCount[.warning]).reduce(into: 0, +=) + let issueCount = graph.compactMap(\.value?.issueCount).reduce(into: 0, +=) let knownIssueCount = graph.compactMap(\.value?.knownIssueCount).reduce(into: 0, +=) - let totalIssueCount = errorIssueCount + warningIssueCount + knownIssueCount + let totalIssueCount = issueCount + knownIssueCount // Construct a string describing the issue counts. - let description = switch (errorIssueCount > 0, warningIssueCount > 0, knownIssueCount > 0) { - case (true, true, true): - " with \(totalIssueCount.counting("issue")) (including \(warningIssueCount.counting("warning")) and \(knownIssueCount.counting("known issue")))" - case (true, false, true): + let description = switch (issueCount > 0, knownIssueCount > 0) { + case (true, true): " with \(totalIssueCount.counting("issue")) (including \(knownIssueCount.counting("known issue")))" - case (false, true, true): - " with \(warningIssueCount.counting("warning")) and \(knownIssueCount.counting("known issue"))" - case (false, false, true): + case (false, true): " with \(knownIssueCount.counting("known issue"))" - case (true, true, false): - " with \(totalIssueCount.counting("issue")) (including \(warningIssueCount.counting("warning")))" - case (true, false, false): + case (true, false): " with \(totalIssueCount.counting("issue"))" - case(false, true, false): - " with \(warningIssueCount.counting("warning"))" - case(false, false, false): + case(false, false): "" } - return (errorIssueCount, warningIssueCount, knownIssueCount, totalIssueCount, description) + return (issueCount, knownIssueCount, totalIssueCount, description) } } @@ -277,8 +267,7 @@ extension Event.HumanReadableOutputRecorder { if issue.isKnown { testData.knownIssueCount += 1 } else { - let issueCount = testData.issueCount[issue.severity] ?? 0 - testData.issueCount[issue.severity] = issueCount + 1 + testData.issueCount += 1 } context.testData[id] = testData @@ -366,7 +355,7 @@ extension Event.HumanReadableOutputRecorder { let testData = testDataGraph?.value ?? .init(startInstant: instant) let issues = _issueCounts(in: testDataGraph) let duration = testData.startInstant.descriptionOfDuration(to: instant) - return if issues.errorIssueCount > 0 { + return if issues.issueCount > 0 { CollectionOfOne( Message( symbol: .fail, @@ -374,7 +363,7 @@ extension Event.HumanReadableOutputRecorder { ) ) + _formattedComments(for: test) } else { - [ + [ Message( symbol: .pass(knownIssueCount: issues.knownIssueCount), stringValue: "\(_capitalizedTitle(for: test)) \(testName) passed after \(duration)\(issues.description)." @@ -411,19 +400,13 @@ extension Event.HumanReadableOutputRecorder { "" } let symbol: Event.Symbol - let subject: String + let known: String if issue.isKnown { symbol = .pass(knownIssueCount: 1) - subject = "a known issue" + known = " known" } else { - switch issue.severity { - case .warning: - symbol = .passWithWarnings - subject = "a warning" - case .error: - symbol = .fail - subject = "an issue" - } + symbol = .fail + known = "n" } var additionalMessages = [Message]() @@ -452,13 +435,13 @@ extension Event.HumanReadableOutputRecorder { let primaryMessage: Message = if parameterCount == 0 { Message( symbol: symbol, - stringValue: "\(_capitalizedTitle(for: test)) \(testName) recorded \(subject)\(atSourceLocation): \(issue.kind)", + stringValue: "\(_capitalizedTitle(for: test)) \(testName) recorded a\(known) issue\(atSourceLocation): \(issue.kind)", conciseStringValue: String(describing: issue.kind) ) } else { Message( symbol: symbol, - stringValue: "\(_capitalizedTitle(for: test)) \(testName) recorded \(subject) with \(parameterCount.counting("argument")) \(labeledArguments)\(atSourceLocation): \(issue.kind)", + stringValue: "\(_capitalizedTitle(for: test)) \(testName) recorded a\(known) issue with \(parameterCount.counting("argument")) \(labeledArguments)\(atSourceLocation): \(issue.kind)", conciseStringValue: String(describing: issue.kind) ) } @@ -515,7 +498,7 @@ extension Event.HumanReadableOutputRecorder { let runStartInstant = context.runStartInstant ?? instant let duration = runStartInstant.descriptionOfDuration(to: instant) - return if issues.errorIssueCount > 0 { + return if issues.issueCount > 0 { [ Message( symbol: .fail, diff --git a/Sources/Testing/Events/Recorder/Event.Symbol.swift b/Sources/Testing/Events/Recorder/Event.Symbol.swift index 3a3f6df8e..0f50ed95c 100644 --- a/Sources/Testing/Events/Recorder/Event.Symbol.swift +++ b/Sources/Testing/Events/Recorder/Event.Symbol.swift @@ -22,14 +22,10 @@ extension Event { /// The symbol to use when a test passes. /// /// - Parameters: - /// - knownIssueCount: The number of known issues recorded for the test. - /// The default value is `0`. + /// - knownIssueCount: The number of known issues encountered by the end + /// of the test. case pass(knownIssueCount: Int = 0) - /// The symbol to use when a test passes with one or more warnings. - @_spi(Experimental) - case passWithWarnings - /// The symbol to use when a test fails. case fail @@ -66,8 +62,6 @@ extension Event.Symbol { } else { ("\u{10105B}", "checkmark.diamond.fill") } - case .passWithWarnings: - ("\u{100123}", "questionmark.diamond.fill") case .fail: ("\u{100884}", "xmark.diamond.fill") case .difference: @@ -128,9 +122,6 @@ extension Event.Symbol { // Unicode: HEAVY CHECK MARK return "\u{2714}" } - case .passWithWarnings: - // Unicode: QUESTION MARK - return "\u{003F}" case .fail: // Unicode: HEAVY BALLOT X return "\u{2718}" @@ -166,9 +157,6 @@ extension Event.Symbol { // Unicode: SQUARE ROOT return "\u{221A}" } - case .passWithWarnings: - // Unicode: QUESTION MARK - return "\u{003F}" case .fail: // Unicode: MULTIPLICATION SIGN return "\u{00D7}" diff --git a/Sources/Testing/Issues/Issue.swift b/Sources/Testing/Issues/Issue.swift index 5d7449b7b..dae58400a 100644 --- a/Sources/Testing/Issues/Issue.swift +++ b/Sources/Testing/Issues/Issue.swift @@ -79,32 +79,6 @@ public struct Issue: Sendable { /// The kind of issue this value represents. public var kind: Kind - /// An enumeration representing the level of severity of a recorded issue. - /// - /// The supported levels, in increasing order of severity, are: - /// - /// - ``warning`` - /// - ``error`` - @_spi(Experimental) - public enum Severity: Sendable { - /// The severity level for an issue which should be noted but is not - /// necessarily an error. - /// - /// An issue with warning severity does not cause the test it's associated - /// with to be marked as a failure, but is noted in the results. - case warning - - /// The severity level for an issue which represents an error in a test. - /// - /// An issue with error severity causes the test it's associated with to be - /// marked as a failure. - case error - } - - /// The severity of this issue. - @_spi(Experimental) - public var severity: Severity - /// Any comments provided by the developer and associated with this issue. /// /// If no comment was supplied when the issue occurred, the value of this @@ -123,20 +97,16 @@ public struct Issue: Sendable { /// /// - Parameters: /// - kind: The kind of issue this value represents. - /// - severity: The severity of this issue. The default value is - /// ``Severity-swift.enum/error``. /// - comments: An array of comments describing the issue. This array may be /// empty. /// - sourceContext: A ``SourceContext`` indicating where and how this issue /// occurred. init( kind: Kind, - severity: Severity = .error, comments: [Comment], sourceContext: SourceContext ) { self.kind = kind - self.severity = severity self.comments = comments self.sourceContext = sourceContext } @@ -184,31 +154,27 @@ public struct Issue: Sendable { } } -extension Issue.Severity: Comparable {} - // MARK: - CustomStringConvertible, CustomDebugStringConvertible extension Issue: CustomStringConvertible, CustomDebugStringConvertible { public var description: String { - let joinedComments = if comments.isEmpty { - "" - } else { - ": " + comments.lazy - .map(\.rawValue) - .joined(separator: "\n") + if comments.isEmpty { + return String(describing: kind) } - return "\(kind) (\(severity))\(joinedComments)" + let joinedComments = comments.lazy + .map(\.rawValue) + .joined(separator: "\n") + return "\(kind): \(joinedComments)" } public var debugDescription: String { - let joinedComments = if comments.isEmpty { - "" - } else { - ": " + comments.lazy - .map(\.rawValue) - .joined(separator: "\n") + if comments.isEmpty { + return "\(kind)\(sourceLocation.map { " at \($0)" } ?? "")" } - return "\(kind)\(sourceLocation.map { " at \($0)" } ?? "") (\(severity))\(joinedComments)" + let joinedComments: String = comments.lazy + .map(\.rawValue) + .joined(separator: "\n") + return "\(kind)\(sourceLocation.map { " at \($0)" } ?? ""): \(joinedComments)" } } @@ -268,17 +234,6 @@ extension Issue.Kind: CustomStringConvertible { } } -extension Issue.Severity: CustomStringConvertible { - public var description: String { - switch self { - case .warning: - "warning" - case .error: - "error" - } - } -} - #if !SWT_NO_SNAPSHOT_TYPES // MARK: - Snapshotting @@ -289,10 +244,6 @@ extension Issue { /// The kind of issue this value represents. public var kind: Kind.Snapshot - /// The severity of this issue. - @_spi(Experimental) - public var severity: Severity - /// Any comments provided by the developer and associated with this issue. /// /// If no comment was supplied when the issue occurred, the value of this @@ -317,22 +268,10 @@ extension Issue { self.kind = Issue.Kind.Snapshot(snapshotting: issue.kind) self.comments = issue.comments } - self.severity = issue.severity self.sourceContext = issue.sourceContext self.isKnown = issue.isKnown } - public init(from decoder: any Decoder) throws { - let container = try decoder.container(keyedBy: CodingKeys.self) - self.kind = try container.decode(Issue.Kind.Snapshot.self, forKey: .kind) - self.comments = try container.decode([Comment].self, forKey: .comments) - self.sourceContext = try container.decode(SourceContext.self, forKey: .sourceContext) - self.isKnown = try container.decode(Bool.self, forKey: .isKnown) - - // Severity is a new field, so fall back to .error if it's not present. - self.severity = try container.decodeIfPresent(Issue.Severity.self, forKey: .severity) ?? .error - } - /// The error which was associated with this issue, if any. /// /// The value of this property is non-`nil` when ``kind-swift.property`` is @@ -356,8 +295,6 @@ extension Issue { } } -extension Issue.Severity: Codable {} - extension Issue.Kind { /// Serializable kinds of issues which may be recorded. @_spi(ForToolsIntegrationOnly) @@ -541,25 +478,23 @@ extension Issue.Kind { extension Issue.Snapshot: CustomStringConvertible, CustomDebugStringConvertible { public var description: String { - let joinedComments = if comments.isEmpty { - "" - } else { - ": " + comments.lazy - .map(\.rawValue) - .joined(separator: "\n") + if comments.isEmpty { + return String(describing: kind) } - return "\(kind) (\(severity))\(joinedComments)" + let joinedComments = comments.lazy + .map(\.rawValue) + .joined(separator: "\n") + return "\(kind): \(joinedComments)" } public var debugDescription: String { - let joinedComments = if comments.isEmpty { - "" - } else { - ": " + comments.lazy - .map(\.rawValue) - .joined(separator: "\n") + if comments.isEmpty { + return "\(kind)\(sourceLocation.map { " at \($0)" } ?? "")" } - return "\(kind)\(sourceLocation.map { " at \($0)" } ?? "") (\(severity))\(joinedComments)" + let joinedComments: String = comments.lazy + .map(\.rawValue) + .joined(separator: "\n") + return "\(kind)\(sourceLocation.map { " at \($0)" } ?? ""): \(joinedComments)" } } diff --git a/Sources/Testing/Running/Configuration+EventHandling.swift b/Sources/Testing/Running/Configuration+EventHandling.swift index e3c189f8b..025f07d2c 100644 --- a/Sources/Testing/Running/Configuration+EventHandling.swift +++ b/Sources/Testing/Running/Configuration+EventHandling.swift @@ -38,23 +38,3 @@ extension Configuration { return eventHandler(event, contextCopy) } } - -extension Configuration.EventHandlingOptions { - /// Determine whether the specified event should be handled according to the - /// options in this instance. - /// - /// - Parameters: - /// - event: The event to consider handling. - /// - /// - Returns: Whether or not the event should be handled or suppressed. - func shouldHandleEvent(_ event: borrowing Event) -> Bool { - switch event.kind { - case let .issueRecorded(issue): - issue.severity > .warning || isWarningIssueRecordedEventEnabled - case .expectationChecked: - isExpectationCheckedEventEnabled - default: - true - } - } -} diff --git a/Sources/Testing/Running/Configuration.swift b/Sources/Testing/Running/Configuration.swift index a917c2f5b..f4ae59813 100644 --- a/Sources/Testing/Running/Configuration.swift +++ b/Sources/Testing/Running/Configuration.swift @@ -178,33 +178,14 @@ public struct Configuration: Sendable { // MARK: - Event handling - /// A type describing options to use when delivering events to this - /// configuration's event handler - public struct EventHandlingOptions: Sendable { - /// Whether or not events of the kind ``Event/Kind-swift.enum/issueRecorded(_:)`` - /// containing issues with warning (or lower) severity should be delivered - /// to the event handler of the configuration these options are applied to. - /// - /// By default, events matching this criteria are not delivered to event - /// handlers since this is an experimental feature. - /// - /// - Warning: Warning issues are not yet an approved feature. - @_spi(Experimental) - public var isWarningIssueRecordedEventEnabled: Bool = false - - /// Whether or not events of the kind - /// ``Event/Kind-swift.enum/expectationChecked(_:)`` should be delivered to - /// the event handler of the configuration these options are applied to. - /// - /// By default, events of this kind are not delivered to event handlers - /// because they occur frequently in a typical test run and can generate - /// significant back-pressure on the event handler. - public var isExpectationCheckedEventEnabled: Bool = false - } - - /// The options to use when delivering events to this configuration's event - /// handler. - public var eventHandlingOptions: EventHandlingOptions = .init() + /// Whether or not events of the kind + /// ``Event/Kind-swift.enum/expectationChecked(_:)`` should be delivered to + /// this configuration's ``eventHandler`` closure. + /// + /// By default, events of this kind are not delivered to event handlers + /// because they occur frequently in a typical test run and can generate + /// significant backpressure on the event handler. + public var deliverExpectationCheckedEvents: Bool = false /// The event handler to which events should be passed when they occur. public var eventHandler: Event.Handler = { _, _ in } @@ -344,14 +325,4 @@ extension Configuration { } } #endif - - @available(*, deprecated, message: "Set eventHandlingOptions.isExpectationCheckedEventEnabled instead.") - public var deliverExpectationCheckedEvents: Bool { - get { - eventHandlingOptions.isExpectationCheckedEventEnabled - } - set { - eventHandlingOptions.isExpectationCheckedEventEnabled = newValue - } - } } diff --git a/Sources/Testing/Running/Runner.RuntimeState.swift b/Sources/Testing/Running/Runner.RuntimeState.swift index 9ae299412..f69e13cd6 100644 --- a/Sources/Testing/Running/Runner.RuntimeState.swift +++ b/Sources/Testing/Running/Runner.RuntimeState.swift @@ -132,7 +132,7 @@ extension Configuration { /// - Returns: A unique number identifying `self` that can be /// passed to `_removeFromAll(identifiedBy:)`` to unregister it. private func _addToAll() -> UInt64 { - if eventHandlingOptions.isExpectationCheckedEventEnabled { + if deliverExpectationCheckedEvents { Self._deliverExpectationCheckedEventsCount.increment() } return Self._all.withLock { all in @@ -152,14 +152,16 @@ extension Configuration { let configuration = Self._all.withLock { all in all.instances.removeValue(forKey: id) } - if let configuration, configuration.eventHandlingOptions.isExpectationCheckedEventEnabled { + if let configuration, configuration.deliverExpectationCheckedEvents { Self._deliverExpectationCheckedEventsCount.decrement() } } /// An atomic counter that tracks the number of "current" configurations that - /// have set ``EventHandlingOptions/isExpectationCheckedEventEnabled`` to - /// `true`. + /// have set ``deliverExpectationCheckedEvents`` to `true`. + /// + /// On older Apple platforms, this property is not available and ``all`` is + /// directly consulted instead (which is less efficient.) private static let _deliverExpectationCheckedEventsCount = Locked(rawValue: 0) /// Whether or not events of the kind @@ -169,8 +171,7 @@ extension Configuration { /// /// To determine if an individual instance of ``Configuration`` is listening /// for these events, consult the per-instance - /// ``Configuration/EventHandlingOptions/isExpectationCheckedEventEnabled`` - /// property. + /// ``Configuration/deliverExpectationCheckedEvents`` property. static var deliverExpectationCheckedEvents: Bool { _deliverExpectationCheckedEventsCount.rawValue > 0 } diff --git a/Tests/TestingTests/ConfigurationTests.swift b/Tests/TestingTests/ConfigurationTests.swift deleted file mode 100644 index a735d8ac5..000000000 --- a/Tests/TestingTests/ConfigurationTests.swift +++ /dev/null @@ -1,25 +0,0 @@ -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2025 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for Swift project authors -// - -@_spi(ForToolsIntegrationOnly) import Testing - -@Suite("Configuration Tests") -struct ConfigurationTests { - @Test - @available(*, deprecated, message: "Testing a deprecated SPI.") - func deliverExpectationCheckedEventsProperty() throws { - var configuration = Configuration() - #expect(!configuration.deliverExpectationCheckedEvents) - #expect(!configuration.eventHandlingOptions.isExpectationCheckedEventEnabled) - - configuration.deliverExpectationCheckedEvents = true - #expect(configuration.eventHandlingOptions.isExpectationCheckedEventEnabled) - } -} diff --git a/Tests/TestingTests/EntryPointTests.swift b/Tests/TestingTests/EntryPointTests.swift deleted file mode 100644 index eae7d4b7e..000000000 --- a/Tests/TestingTests/EntryPointTests.swift +++ /dev/null @@ -1,81 +0,0 @@ -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2025 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for Swift project authors -// - -@testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing -private import _TestingInternals - -@Suite("Entry point tests") -struct EntryPointTests { - @Test("Entry point filter with filtering of hidden tests enabled") - func hiddenTests() async throws { - var arguments = __CommandLineArguments_v0() - arguments.filter = ["_someHiddenTest"] - arguments.includeHiddenTests = true - arguments.eventStreamVersion = 0 - arguments.verbosity = .min - - await confirmation("Test event started", expectedCount: 1) { testMatched in - _ = await entryPoint(passing: arguments) { event, context in - if case .testStarted = event.kind { - testMatched() - } - } - } - } - - @Test("Entry point with WarningIssues feature enabled exits with success if all issues have severity < .error") - func warningIssues() async throws { - var arguments = __CommandLineArguments_v0() - arguments.filter = ["_recordWarningIssue"] - arguments.includeHiddenTests = true - arguments.eventStreamVersion = 0 - arguments.verbosity = .min - - let exitCode = await confirmation("Test matched", expectedCount: 1) { testMatched in - await entryPoint(passing: arguments) { event, context in - if case .testStarted = event.kind { - testMatched() - } else if case let .issueRecorded(issue) = event.kind { - Issue.record("Unexpected issue \(issue) was recorded.") - } - } - } - #expect(exitCode == EXIT_SUCCESS) - } - - @Test("Entry point with WarningIssues feature enabled propagates warning issues and exits with success if all issues have severity < .error") - func warningIssuesEnabled() async throws { - var arguments = __CommandLineArguments_v0() - arguments.filter = ["_recordWarningIssue"] - arguments.includeHiddenTests = true - arguments.eventStreamVersion = 0 - arguments.isWarningIssueRecordedEventEnabled = true - arguments.verbosity = .min - - let exitCode = await confirmation("Warning issue recorded", expectedCount: 1) { issueRecorded in - await entryPoint(passing: arguments) { event, context in - if case let .issueRecorded(issue) = event.kind { - #expect(issue.severity == .warning) - issueRecorded() - } - } - } - #expect(exitCode == EXIT_SUCCESS) - } -} - -// MARK: - Fixtures - -@Test(.hidden) private func _someHiddenTest() {} - -@Test(.hidden) private func _recordWarningIssue() { - // Intentionally _only_ record issues with warning (or lower) severity. - Issue(kind: .unconditional, severity: .warning, comments: [], sourceContext: .init()).record() -} diff --git a/Tests/TestingTests/EventRecorderTests.swift b/Tests/TestingTests/EventRecorderTests.swift index 6b5b9bd81..97619b755 100644 --- a/Tests/TestingTests/EventRecorderTests.swift +++ b/Tests/TestingTests/EventRecorderTests.swift @@ -59,7 +59,7 @@ struct EventRecorderTests { } var configuration = Configuration() - configuration.eventHandlingOptions.isExpectationCheckedEventEnabled = true + configuration.deliverExpectationCheckedEvents = true let eventRecorder = Event.ConsoleOutputRecorder(options: options, writingUsing: stream.write) configuration.eventHandler = { event, context in eventRecorder.record(event, in: context) @@ -98,7 +98,7 @@ struct EventRecorderTests { let stream = Stream() var configuration = Configuration() - configuration.eventHandlingOptions.isExpectationCheckedEventEnabled = true + configuration.deliverExpectationCheckedEvents = true let eventRecorder = Event.ConsoleOutputRecorder(writingUsing: stream.write) configuration.eventHandler = { event, context in eventRecorder.record(event, in: context) @@ -123,7 +123,7 @@ struct EventRecorderTests { let stream = Stream() var configuration = Configuration() - configuration.eventHandlingOptions.isExpectationCheckedEventEnabled = true + configuration.deliverExpectationCheckedEvents = true let eventRecorder = Event.ConsoleOutputRecorder(writingUsing: stream.write) configuration.eventHandler = { event, context in eventRecorder.record(event, in: context) @@ -183,20 +183,15 @@ struct EventRecorderTests { @Test( "Issue counts are summed correctly on test end", arguments: [ - ("f()", #".* Test f\(\) failed after .+ seconds with 5 issues \(including 3 known issues\)\."#), - ("g()", #".* Test g\(\) failed after .+ seconds with 2 issues \(including 1 known issue\)\."#), - ("h()", #".* Test h\(\) passed after .+ seconds with 1 warning\."#), - ("i()", #".* Test i\(\) failed after .+ seconds with 2 issues \(including 1 warning\)\."#), - ("j()", #".* Test j\(\) passed after .+ seconds with 1 warning and 1 known issue\."#), - ("k()", #".* Test k\(\) passed after .+ seconds with 1 known issue\."#), - ("PredictablyFailingTests", #".* Suite PredictablyFailingTests failed after .+ seconds with 13 issues \(including 3 warnings and 6 known issues\)\."#), + ("f()", false, (total: 5, expected: 3)), + ("g()", false, (total: 2, expected: 1)), + ("PredictablyFailingTests", true, (total: 7, expected: 4)), ] ) - func issueCountSummingAtTestEnd(testName: String, expectedPattern: String) async throws { + func issueCountSummingAtTestEnd(testName: String, isSuite: Bool, issueCount: (total: Int, expected: Int)) async throws { let stream = Stream() var configuration = Configuration() - configuration.eventHandlingOptions.isWarningIssueRecordedEventEnabled = true let eventRecorder = Event.ConsoleOutputRecorder(writingUsing: stream.write) configuration.eventHandler = { event, context in eventRecorder.record(event, in: context) @@ -209,13 +204,28 @@ struct EventRecorderTests { print(buffer, terminator: "") } - let expectedSuffixRegex = try Regex(expectedPattern) - #expect(try buffer - .split(whereSeparator: \.isNewline) - .compactMap(expectedSuffixRegex.wholeMatch(in:)) - .first != nil, - "buffer: \(buffer)" + let testFailureRegex = Regex { + One(.anyGraphemeCluster) + " \(isSuite ? "Suite" : "Test") \(testName) failed " + ZeroOrMore(.any) + " with " + Capture { OneOrMore(.digit) } transform: { Int($0) } + " issue" + Optionally("s") + " (including " + Capture { OneOrMore(.digit) } transform: { Int($0) } + " known issue" + Optionally("s") + ")." + } + let match = try #require( + buffer + .split(whereSeparator: \.isNewline) + .compactMap(testFailureRegex.wholeMatch(in:)) + .first ) + #expect(issueCount.total == match.output.1) + #expect(issueCount.expected == match.output.2) } #endif @@ -284,51 +294,8 @@ struct EventRecorderTests { .compactMap(runFailureRegex.wholeMatch(in:)) .first ) - #expect(match.output.1 == 9) - #expect(match.output.2 == 5) - } - - @Test("Issue counts are summed correctly on run end for a test with only warning issues") - @available(_regexAPI, *) - func warningIssueCountSummingAtRunEnd() async throws { - let stream = Stream() - - var configuration = Configuration() - configuration.eventHandlingOptions.isWarningIssueRecordedEventEnabled = true - let eventRecorder = Event.ConsoleOutputRecorder(writingUsing: stream.write) - configuration.eventHandler = { event, context in - eventRecorder.record(event, in: context) - } - - await runTestFunction(named: "h()", in: PredictablyFailingTests.self, configuration: configuration) - - let buffer = stream.buffer.rawValue - if testsWithSignificantIOAreEnabled { - print(buffer, terminator: "") - } - - let runFailureRegex = Regex { - One(.anyGraphemeCluster) - " Test run with " - OneOrMore(.digit) - " test" - Optionally("s") - " passed " - ZeroOrMore(.any) - " with " - Capture { OneOrMore(.digit) } transform: { Int($0) } - " warning" - Optionally("s") - "." - } - let match = try #require( - buffer - .split(whereSeparator: \.isNewline) - .compactMap(runFailureRegex.wholeMatch(in:)) - .first, - "buffer: \(buffer)" - ) - #expect(match.output.1 == 1) + #expect(match.output.1 == 7) + #expect(match.output.2 == 4) } #endif @@ -341,7 +308,7 @@ struct EventRecorderTests { let stream = Stream() var configuration = Configuration() - configuration.eventHandlingOptions.isExpectationCheckedEventEnabled = true + configuration.deliverExpectationCheckedEvents = true let eventRecorder = Event.JUnitXMLRecorder(writingUsing: stream.write) configuration.eventHandler = { event, context in eventRecorder.record(event, in: context) @@ -543,26 +510,4 @@ struct EventRecorderTests { #expect(Bool(false)) } } - - @Test(.hidden) func h() { - Issue(kind: .unconditional, severity: .warning, comments: [], sourceContext: .init()).record() - } - - @Test(.hidden) func i() { - Issue(kind: .unconditional, severity: .warning, comments: [], sourceContext: .init()).record() - #expect(Bool(false)) - } - - @Test(.hidden) func j() { - Issue(kind: .unconditional, severity: .warning, comments: [], sourceContext: .init()).record() - withKnownIssue { - #expect(Bool(false)) - } - } - - @Test(.hidden) func k() { - withKnownIssue { - Issue(kind: .unconditional, severity: .warning, comments: [], sourceContext: .init()).record() - } - } } diff --git a/Tests/TestingTests/IssueTests.swift b/Tests/TestingTests/IssueTests.swift index f73676d42..8b7d86358 100644 --- a/Tests/TestingTests/IssueTests.swift +++ b/Tests/TestingTests/IssueTests.swift @@ -301,7 +301,7 @@ final class IssueTests: XCTestCase { let expectationChecked = expectation(description: "expectation checked") var configuration = Configuration() - configuration.eventHandlingOptions.isExpectationCheckedEventEnabled = true + configuration.deliverExpectationCheckedEvents = true configuration.eventHandler = { event, _ in guard case let .expectationChecked(expectation) = event.kind else { return @@ -1124,12 +1124,12 @@ final class IssueTests: XCTestCase { do { let sourceLocation = SourceLocation.init(fileID: "FakeModule/FakeFile.swift", filePath: "", line: 9999, column: 1) let issue = Issue(kind: .system, comments: ["Some issue"], sourceContext: SourceContext(sourceLocation: sourceLocation)) - XCTAssertEqual(issue.description, "A system failure occurred (error): Some issue") - XCTAssertEqual(issue.debugDescription, "A system failure occurred at FakeFile.swift:9999:1 (error): Some issue") + XCTAssertEqual(issue.description, "A system failure occurred: Some issue") + XCTAssertEqual(issue.debugDescription, "A system failure occurred at FakeFile.swift:9999:1: Some issue") } do { let issue = Issue(kind: .system, comments: ["Some issue"], sourceContext: SourceContext(sourceLocation: nil)) - XCTAssertEqual(issue.debugDescription, "A system failure occurred (error): Some issue") + XCTAssertEqual(issue.debugDescription, "A system failure occurred: Some issue") } } diff --git a/Tests/TestingTests/Runner.RuntimeStateTests.swift b/Tests/TestingTests/Runner.RuntimeStateTests.swift index 1576c49e7..e4ee33079 100644 --- a/Tests/TestingTests/Runner.RuntimeStateTests.swift +++ b/Tests/TestingTests/Runner.RuntimeStateTests.swift @@ -34,7 +34,7 @@ struct Runner_RuntimeStateTests { // an event to be posted during the test below without causing any real // issues to be recorded or otherwise confuse the testing harness. var configuration = Configuration.current ?? .init() - configuration.eventHandlingOptions.isExpectationCheckedEventEnabled = true + configuration.deliverExpectationCheckedEvents = true await Configuration.withCurrent(configuration) { await withTaskGroup(of: Void.self) { group in diff --git a/Tests/TestingTests/RunnerTests.swift b/Tests/TestingTests/RunnerTests.swift index 335f8be37..857cfdd81 100644 --- a/Tests/TestingTests/RunnerTests.swift +++ b/Tests/TestingTests/RunnerTests.swift @@ -426,7 +426,7 @@ final class RunnerTests: XCTestCase { func testExpectationCheckedEventHandlingWhenDisabled() async { var configuration = Configuration() - configuration.eventHandlingOptions.isExpectationCheckedEventEnabled = false + configuration.deliverExpectationCheckedEvents = false configuration.eventHandler = { event, _ in if case .expectationChecked = event.kind { XCTFail("Expectation checked event was posted unexpectedly") @@ -459,7 +459,7 @@ final class RunnerTests: XCTestCase { #endif var configuration = Configuration() - configuration.eventHandlingOptions.isExpectationCheckedEventEnabled = true + configuration.deliverExpectationCheckedEvents = true configuration.eventHandler = { event, _ in guard case let .expectationChecked(expectation) = event.kind else { return