[CIVIS-9212] send tests skipped by ITR to the backend (#93)

* added sending of tests skipped by ITR
* better span encoding

Sources/DatadogSDKTesting/DDTags.swift

@@ -58,6 +58,7 @@ internal enum DDTestTags {
     static let testIsUITest = "test.is_ui_test"
     static let testIsRUMActive = "test.is_rum_active"
     static let testCommand = "test.command"
+    static let testSkippedByITR = "test.skipped_by_itr"
 }
 
 internal enum DDOSTags {

Sources/DatadogSDKTesting/DDTest.swift

@@ -11,7 +11,6 @@ import Foundation
 
 public class DDTest: NSObject {
     static let testNameRegex = try! NSRegularExpression(pattern: "([\\w]+) ([\\w]+)", options: .caseInsensitive)
-    static let supportsSkipping = NSClassFromString("XCTSkippedTestContext") != nil
     var currentTestExecutionOrder: Int
     var initialProcessId = Int(ProcessInfo.processInfo.processIdentifier)
 
@@ -149,44 +148,7 @@ public class DDTest: NSObject {
     ///   - status: the status reported for this test
     ///   - endTime: Optional, the time where the test ended
     @objc public func end(status: DDTestStatus, endTime: Date? = nil) {
-        let testEndTime = endTime ?? DDTestMonitor.clock.now
-        let testStatus: String
-        switch status {
-            case .pass:
-                testStatus = DDTagValues.statusPass
-                span.status = .ok
-            case .fail:
-                testStatus = DDTagValues.statusFail
-                suite.status = .fail
-                module.status = .fail
-                span.status = .error(description: "Test failed")
-                setErrorInformation()
-            case .skip:
-                testStatus = DDTagValues.statusSkip
-                span.status = .ok
-        }
-
-        span.setAttribute(key: DDTestTags.testStatus, value: testStatus)
-
-        if let coverageHelper = DDTestMonitor.instance?.coverageHelper {
-            coverageHelper.writeProfile()
-            let testSessionId = module.sessionId.rawValue
-            let testSuiteId = suite.id.rawValue
-            let spanId = span.context.spanId.rawValue
-            let coverageFileURL = coverageHelper.getURLForTest(name: name, testSessionId: testSessionId, testSuiteId: testSuiteId, spanId: spanId)
-            coverageHelper.coverageWorkQueue.addOperation {
-                guard FileManager.default.fileExists(atPath: coverageFileURL.path) else {
-                    return
-                }
-                DDTestMonitor.tracer.eventsExporter?.export(coverage: coverageFileURL, testSessionId: testSessionId, testSuiteId: testSuiteId, spanId: spanId, workspacePath: DDTestMonitor.env.workspacePath, binaryImagePaths: BinaryImages.binaryImagesPath)
-            }
-        }
-
-        StderrCapture.syncData()
-        span.end(time: testEndTime)
-        DDTestMonitor.instance?.networkInstrumentation?.endAndCleanAliveSpans()
-        DDCrashes.setCustomData(customData: Data())
-        DDTestMonitor.instance?.currentTest = nil
+        self.end(status: status.itr, endTime: endTime)
     }
 
     @objc public func end(status: DDTestStatus) {
@@ -248,6 +210,64 @@ public class DDTest: NSObject {
     }
 }
 
+extension DDTest {
+    func end(status: DDTestStatus.ITR, endTime: Date? = nil) {
+        let testEndTime = endTime ?? DDTestMonitor.clock.now
+
+        switch status {
+        case .pass:
+            span.setAttribute(key: DDTestTags.testStatus, value: DDTagValues.statusPass)
+            span.status = .ok
+        case .fail:
+            span.setAttribute(key: DDTestTags.testStatus, value: DDTagValues.statusFail)
+            suite.status = .fail
+            module.status = .fail
+            span.status = .error(description: "Test failed")
+            setErrorInformation()
+        case .skip(itr: let itr):
+            span.setAttribute(key: DDTestTags.testStatus, value: DDTagValues.statusSkip)
+            if itr { span.setAttribute(key: DDTestTags.testSkippedByITR, value: true) }
+            span.status = .ok
+        }
+
+        if let coverageHelper = DDTestMonitor.instance?.coverageHelper {
+            coverageHelper.writeProfile()
+            let testSessionId = module.sessionId.rawValue
+            let testSuiteId = suite.id.rawValue
+            let spanId = span.context.spanId.rawValue
+            let coverageFileURL = coverageHelper.getURLForTest(name: name, testSessionId: testSessionId, testSuiteId: testSuiteId, spanId: spanId)
+            coverageHelper.coverageWorkQueue.addOperation {
+                guard FileManager.default.fileExists(atPath: coverageFileURL.path) else {
+                    return
+                }
+                DDTestMonitor.tracer.eventsExporter?.export(coverage: coverageFileURL, testSessionId: testSessionId, testSuiteId: testSuiteId, spanId: spanId, workspacePath: DDTestMonitor.env.workspacePath, binaryImagePaths: BinaryImages.binaryImagesPath)
+            }
+        }
+
+        StderrCapture.syncData()
+        span.end(time: testEndTime)
+        DDTestMonitor.instance?.networkInstrumentation?.endAndCleanAliveSpans()
+        DDCrashes.setCustomData(customData: Data())
+        DDTestMonitor.instance?.currentTest = nil
+    }
+}
+
+extension DDTestStatus {
+    enum ITR: Equatable {
+        case pass
+        case fail
+        case skip(itr: Bool)
+    }
+
+    var itr: ITR {
+        switch self {
+        case .pass: return .pass
+        case .fail: return .fail
+        case .skip: return .skip(itr: false)
+        }
+    }
+}
+
 private struct ErrorInfo {
     var type: String
     var message: String

Sources/DatadogSDKTesting/DDTestObserver.swift

@@ -9,120 +9,164 @@ import Foundation
 
 class DDTestObserver: NSObject, XCTestObservation {
     static let testNameRegex = try! NSRegularExpression(pattern: "([\\w]+) ([\\w]+)", options: .caseInsensitive)
-    static let supportsSkipping = NSClassFromString("XCTSkippedTestContext") != nil
     static let tracerVersion = (Bundle(for: DDTestObserver.self).infoDictionary?["CFBundleShortVersionString"] as? String) ?? "unknown"
+
+    enum State {
+        case none
+        case module(DDTestModule)
+        case suite(DDTestSuite)
+        case test(DDTest)
+    }
 
-    var module: DDTestModule?
-    var suite: DDTestSuite?
-    var test: DDTest?
+    private(set) var state: State
 
     override init() {
         XCUIApplication.swizzleMethods
+        state = .none
         super.init()
     }
 
     func startObserving() {
         XCTestObservationCenter.shared.addTestObserver(self)
     }
+
+    func stopObserving() {
+        XCTestObservationCenter.shared.removeTestObserver(self)
+    }
 
     func testBundleWillStart(_ testBundle: Bundle) {
-        let bundleName = testBundle.bundleURL.deletingPathExtension().lastPathComponent
+        guard case .none = state else {
+            Log.print("testBundleWillStart: Bad observer state: \(state), expected: .none")
+            return
+        }
+        let bundleName = testBundle.name
         Log.debug("testBundleWillStart: \(bundleName)")
-        module = DDTestModule.start(bundleName: bundleName)
-        module?.testFramework = "XCTest"
+        let module = DDTestModule.start(bundleName: bundleName)
+        module.testFramework = "XCTest"
+        state = .module(module)
     }
 
     func testBundleDidFinish(_ testBundle: Bundle) {
+        guard case .module(let module) = state else {
+            Log.print("testBundleDidFinish: Bad observer state: \(state), expected: .module")
+            return
+        }
+        guard module.bundleName == testBundle.name else {
+            Log.print("Bad module: \(testBundle.name), expected: \(module.bundleName)")
+            state = .none
+            return
+        }
         /// We need to wait for all the traces to be written to the backend before exiting
-        module?.end()
-        Log.debug("testBundleDidFinish: \(testBundle.bundleURL.deletingPathExtension().lastPathComponent)")
+        module.end()
+        state = .none
+        Log.debug("testBundleDidFinish: \(module.bundleName)")
     }
 
     func testSuiteWillStart(_ testSuite: XCTestSuite) {
-        if module?.configError ?? false {
-            Log.debug("testSuiteWillStart: Failed, module config error")
+        guard case .module(let module) = state else {
+            Log.print("testSuiteWillStart: Bad observer state: \(state), expected: .module")
+            return
+        }
+
+        if module.configError {
+            Log.print("testSuiteWillStart: Failed, module config error")
             testSuite.testRun?.stop()
             exit(1)
         }
 
         guard let tests = testSuite.value(forKey: "_mutableTests") as? NSArray,
-              tests.firstObject is XCTestCase,
-              let module = module
+              (tests.count == 0 || tests.firstObject is XCTestCase)
         else {
             return
         }
 
         Log.measure(name: "waiting itrWorkQueue") {
             DDTestMonitor.instance?.itrWorkQueue.waitUntilAllOperationsAreFinished()
         }
+
+        Log.debug("testSuiteWillStart: \(testSuite.name)")
+        state = .suite(module.suiteStart(name: testSuite.name))
 
         if let itr = DDTestMonitor.instance?.itr {
             let skippableTests = itr.skippableTests.filter { $0.suite == testSuite.name }.map { "-[\(testSuite.name) \($0.name)]" }
-            let finalTests = tests.filter { !skippableTests.contains(($0 as AnyObject).name) }
+
+            let skippedTests = tests.filter { skippableTests.contains(($0 as! XCTest).name) }
+
+            let finalTests = tests.filter { !skippableTests.contains(($0 as! XCTest).name) }
             testSuite.setValue(finalTests, forKey: "_mutableTests")
-            if !finalTests.isEmpty {
-                suite = module.suiteStart(name: testSuite.name)
-                Log.debug("testSuiteWillStart: \(testSuite.name)")
+
+            skippedTests.forEach { test in
+                self.testCaseWillStart(test as! XCTestCase)
+                guard case .test(let test) = self.state else { return }
+                test.end(status: .skip(itr: true))
+                self.state = .suite(test.suite)
             }
-            let testsToSkip = tests.count - finalTests.count
-            Log.print("ITR skipped \(testsToSkip) tests")
-            if testsToSkip > 0 {
+
+            if !skippedTests.isEmpty {
+                Log.print("ITR skipped \(skippedTests.count) tests")
                 module.itrSkipped = true
             }
-        } else {
-            suite = module.suiteStart(name: testSuite.name)
-            Log.debug("testSuiteWillStart: \(testSuite.name)")
         }
     }
 
     func testSuiteDidFinish(_ testSuite: XCTestSuite) {
-        if let tests = testSuite.value(forKey: "_mutableTests") as? NSArray,
-           tests.firstObject is XCTestCase
-        {
-            suite?.end()
-            Log.debug("testSuiteDidFinish: \(testSuite.name)")
+        guard case .suite(let suite) = state else {
+            Log.print("testSuiteDidFinish: Bad observer state: \(state), expected: .suite")
+            return
+        }
+        guard suite.name == testSuite.name else {
+            Log.print("Bad suite: \(testSuite.name), expected: \(suite.name)")
+            return
         }
+        suite.end()
+        state = .module(suite.module)
+        Log.debug("testSuiteDidFinish: \(suite.name)")
     }
 
     func testCaseWillStart(_ testCase: XCTestCase) {
-        guard let suite = suite,
-              let namematch = DDTestObserver.testNameRegex.firstMatch(in: testCase.name, range: NSRange(location: 0, length: testCase.name.count)),
-              let nameRange = Range(namematch.range(at: 2), in: testCase.name)
-        else {
+        guard case .suite(let suite) = state else {
+            Log.print("testCaseWillStart: Bad observer state: \(state), expected: .suite")
             return
         }
-        let testName = String(testCase.name[nameRange])
-        test = suite.testStart(name: testName)
+        let testName: String
+        if let match = DDTestObserver.testNameRegex.firstMatch(in: testCase.name, range: NSRange(location: 0, length: testCase.name.count)),
+           let range = Range(match.range(at: 2), in: testCase.name)
+        {
+            testName = String(testCase.name[range])
+        } else {
+            testName = testCase.name
+        }
+        Log.debug("testCaseWillStart: \(testName)")
+        state = .test(suite.testStart(name: testName))
     }
 
     func testCaseDidFinish(_ testCase: XCTestCase) {
-        guard let test = test
-        else {
+        guard case .test(let test) = state else {
+            Log.print("testCaseDidFinish: Bad observer state: \(state), expected: .test")
             return
         }
-        addBenchmarkTagsIfNeeded(testCase: testCase, test: test)
-
-        if DDTestObserver.supportsSkipping, testCase.testRun?.hasBeenSkipped == true {
-            test.end(status: .skip)
-        } else if testCase.testRun?.hasSucceeded ?? false {
-            test.end(status: .pass)
-        } else {
-            test.end(status: .fail)
+        guard testCase.name.contains(test.name) else {
+            Log.print("Bad test: \(testCase), expected: \(test.name)")
+            return
         }
+        addBenchmarkTagsIfNeeded(testCase: testCase, test: test)
+        test.end(status: testCase.testRun?.status ?? .fail)
+        state = .suite(test.suite)
+        Log.debug("testCaseDidFinish: \(test.name)")
     }
 
     #if swift(>=5.3)
     func testCase(_ testCase: XCTestCase, didRecord issue: XCTIssue) {
-        guard let test = test
-        else {
+        guard case .test(let test) = state else {
+            Log.print("testCase:didRecord: Bad observer state: \(state), expected: .test")
             return
         }
         test.setErrorInfo(type: issue.compactDescription, message: issue.description, callstack: nil)
     }
     #else
     func testCase(_ testCase: XCTestCase, didFailWithDescription description: String, inFile filePath: String?, atLine lineNumber: Int) {
-        guard let test = test
-        else {
+        guard case .test(let test) = state else {
+            Log.print("testCase:didFailWithDescription: Bad observer state: \(state), expected: .test")
             return
         }
         test.setErrorInfo(type: description, message: "test_failure: \(filePath ?? ""):\(lineNumber)", callstack: nil)
@@ -233,3 +277,14 @@ class DDTestObserver: NSObject, XCTestObservation {
         }
     }
 }
+
+extension XCTestRun {
+    var status: DDTestStatus.ITR {
+        if XCTestRun.supportsSkipping && hasBeenSkipped {
+            return .skip(itr: false)
+        }
+        return hasSucceeded ? .pass : .fail
+    }
+
+    static let supportsSkipping = NSClassFromString("XCTSkippedTestContext") != nil
+}

Sources/DatadogSDKTesting/Environment/Environment.swift

@@ -158,7 +158,7 @@ internal final class Environment {
         return .init(
             repositoryURL: env["DD_GIT_REPOSITORY_URL"],
             branch: isTag ? nil : branch,
-            tag: isTag ? branch : env["DD_GIT_TAG"],
+            tag: Git.normalize(branchOrTag: env["DD_GIT_TAG"]).0 ?? (isTag ? branch : nil),
             commitSHA: env["DD_GIT_COMMIT_SHA"],
             commitMessage: env["DD_GIT_COMMIT_MESSAGE"],
             authorName: env["DD_GIT_COMMIT_AUTHOR_NAME"],

Sources/DatadogSDKTesting/Utils/SwiftExtensions.swift

@@ -224,3 +224,9 @@ extension Dictionary where Key == String, Value == String {
         return hashedData
     }
 }
+
+extension Bundle {
+    var name: String {
+        bundleURL.deletingPathExtension().lastPathComponent
+    }
+}