Update xctest test result parsing for Xcode 16 (#431)

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
     hooks:
       - id: end-of-file-fixer
       - id: trailing-whitespace
@@ -9,16 +9,16 @@ repos:
     hooks:
       - id: add-trailing-comma
   - repo: https://github.com/psf/black
-    rev: 24.4.2
+    rev: 24.10.0
     hooks:
       - id: black
         language_version: python3.11
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: v0.5.0
+    rev: v0.6.9
     hooks:
       - id: ruff
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.10.1
+    rev: v1.11.2
     hooks:
       - id: mypy
         additional_dependencies: [types-requests]

CHANGELOG.md

@@ -1,3 +1,24 @@
+Version 0.54.0
+-------------
+
+This release contains changes from [PR #431](https://github.com/codemagic-ci-cd/cli-tools/pull/431)
+
+**Features**
+- Use new `xcresulttool` APIs for XcResult parsing when Xcode 16+ is selected. Applies to `xcode-project` actions `run-tests`, `test-summary` and `junit-test-results`.
+
+**Bugfixes**
+- Omit `failures` and `errors` attributes from JUnit `<testsuites>` in case none of the child `<testsuite>` elements specify those values instead of setting them to `0`.
+
+**Development**
+- Add new XcResult to JUnit test results converter implementation `Xcode16XcResultConverter`.
+- Rename `XcResultConverter` to `LegacyXcResultConverter`.
+- Add abstract `XcResultConverter` which automatically chooses correct implementation.
+- Refactor module `codemagic.models.xctest.xcresult` to a package and move its contents to submodule `legacy_xcresult`. All public definitions remain accessible using the old namespace.
+- Add submodule `xcode_16_xcresult` to package `codemagic.models.xctest.xcresult`.
+- Remove method `get_tool_version` from `XcResultTool`.
+- Add methods `is_legacy`, `get_test_report_summary` and `get_test_report_tests` to `XcResultTool`.
+- Prohibit using `XcResultTool` methods `get_object` and `get_bundle` when Xcode 16 or newer is selected.
+
 Version 0.53.9
 -------------
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "codemagic-cli-tools"
-version = "0.53.9"
+version = "0.54.0"
 description = "CLI tools used in Codemagic builds"
 readme = "README.md"
 authors = [

src/codemagic/__version__.py

@@ -1,5 +1,5 @@
 __title__ = "codemagic-cli-tools"
 __description__ = "CLI tools used in Codemagic builds"
-__version__ = "0.53.9.dev"
+__version__ = "0.54.0.dev"
 __url__ = "https://github.com/codemagic-ci-cd/cli-tools"
 __licence__ = "GNU General Public License v3.0"

src/codemagic/models/junit/definitions.py

@@ -30,13 +30,17 @@ def disabled(self) -> int:
         return sum(suite.disabled for suite in self.test_suites if suite.disabled)
 
     @property
-    def errors(self) -> int:
+    def errors(self) -> Optional[int]:
         """Total number of tests with error result from all testsuites."""
+        if all(suite.errors is None for suite in self.test_suites):
+            return None
         return sum(suite.errors for suite in self.test_suites if suite.errors)
 
     @property
-    def failures(self) -> int:
+    def failures(self) -> Optional[int]:
         """Total number of failed tests from all testsuites."""
+        if all(suite.failures is None for suite in self.test_suites):
+            return None
         return sum(suite.failures for suite in self.test_suites if suite.failures)
 
     @property

src/codemagic/models/junit/printer.py

@@ -1,5 +1,7 @@
 from typing import Callable
+from typing import Iterable
 from typing import List
+from typing import Union
 
 from codemagic.cli import Colors
 from codemagic.models.table import Header
@@ -18,17 +20,11 @@ def __init__(self, print_function: Callable[[str], None]):
         self.print = print_function
 
     def _print_test_suites_summary(self, test_suites: TestSuites):
-        tests_color = Colors.GREEN if test_suites.tests else Colors.RED
-        fails_color = Colors.RED if test_suites.failures else Colors.GREEN
-        errors_color = Colors.RED if test_suites.errors else Colors.GREEN
         table = Table(
             [
                 Header("Test run summary"),
                 Line("Test suites", len(test_suites.test_suites)),
-                Line("Total tests ran", test_suites.tests, value_color=tests_color),
-                Line("Total tests failed", test_suites.failures, value_color=fails_color),
-                Line("Total tests errored", test_suites.errors, value_color=errors_color),
-                Line("Total tests skipped", test_suites.skipped),
+                *self._iter_total_test_lines(test_suites),
             ],
             align_values_left=False,
         )
@@ -56,20 +52,13 @@ def to_line(tc):
         return [Header("Skipped tests"), *map(to_line, skipped_tests)] if skipped_tests else []
 
     def _print_test_suite(self, test_suite: TestSuite):
-        tests_color = Colors.GREEN if test_suite.tests else Colors.RED
-        fails_color = Colors.RED if test_suite.failures else Colors.GREEN
-        errors_color = Colors.RED if test_suite.errors else Colors.GREEN
-
         table = Table(
             [
                 Header("Testsuite summary"),
                 Line("Name", test_suite.name),
                 *(Line(p.name.replace("_", " ").title(), p.value) for p in test_suite.properties),
                 Spacer(),
-                Line("Total tests ran", test_suite.tests or 0, value_color=tests_color),
-                Line("Total tests failed", test_suite.failures or 0, value_color=fails_color),
-                Line("Total tests errored", test_suite.errors or 0, value_color=errors_color),
-                Line("Total tests skipped", test_suite.skipped or 0),
+                *self._iter_total_test_lines(test_suite),
                 *self._get_test_suite_errored_lines(test_suite),
                 *self._get_test_suite_failed_lines(test_suite),
                 *self._get_test_suite_skipped_lines(test_suite),
@@ -78,6 +67,21 @@ def _print_test_suite(self, test_suite: TestSuite):
 
         self.print(table.construct())
 
+    @classmethod
+    def _iter_total_test_lines(cls, suite: Union[TestSuite, TestSuites]) -> Iterable[Line]:
+        tests_color = Colors.GREEN if suite.tests else Colors.RED
+        yield Line("Total tests ran", suite.tests or 0, value_color=tests_color)
+
+        if suite.failures is not None:
+            fails_color = Colors.RED if suite.failures else Colors.GREEN
+            yield Line("Total tests failed", suite.failures or 0, value_color=fails_color)
+
+        if suite.errors is not None:
+            errors_color = Colors.RED if suite.errors else Colors.GREEN
+            yield Line("Total tests errored", suite.errors or 0, value_color=errors_color)
+
+        yield Line("Total tests skipped", suite.skipped or 0)
+
     @classmethod
     def _truncate(cls, s: str, max_width: int = 80) -> str:
         t = s[:max_width]

src/codemagic/models/xctests/converter.py

@@ -2,10 +2,14 @@
 
 import pathlib
 import re
+from abc import ABC
+from abc import abstractmethod
 from datetime import datetime
 from typing import Iterator
 from typing import List
 from typing import Optional
+from typing import Union
+from typing import cast
 
 from codemagic.models.junit import Error
 from codemagic.models.junit import Failure
@@ -20,13 +24,43 @@
 from .xcresult import ActionsInvocationRecord
 from .xcresult import ActionTestableSummary
 from .xcresult import ActionTestMetadata
+from .xcresult import XcDevice
+from .xcresult import XcSummary
+from .xcresult import XcTestNode
+from .xcresult import XcTestNodeType
+from .xcresult import XcTestResult
+from .xcresult import XcTests
+from .xcresulttool import XcResultTool
 
 
-class XcResultConverter:
+class XcResultConverter(ABC):
+    def __new__(cls, *args, **kwargs):
+        if cls is XcResultConverter:
+            if XcResultTool.is_legacy():
+                cls = LegacyXcResultConverter
+            else:
+                cls = Xcode16XcResultConverter
+        return object.__new__(cls)
+
+    def __init__(self, xcresult: pathlib.Path):
+        self.xcresult = xcresult
+
     @classmethod
-    def _timestamp(cls, date: datetime) -> str:
+    def _timestamp(cls, date: Union[datetime, float, int]) -> str:
+        if isinstance(date, (float, int)):
+            date = datetime.fromtimestamp(date)
         return date.strftime("%Y-%m-%dT%H:%M:%S")
 
+    @classmethod
+    def xcresult_to_junit(cls, xcresult: pathlib.Path) -> TestSuites:
+        return cls(xcresult).convert()
+
+    @abstractmethod
+    def convert(self) -> TestSuites:
+        raise NotImplementedError()
+
+
+class LegacyXcResultConverter(XcResultConverter):
     @classmethod
     def _get_test_case_error(cls, test: ActionTestMetadata) -> Optional[Error]:
         if not test.is_error():
@@ -140,7 +174,169 @@ def actions_invocation_record_to_junit(cls, actions_invocation_record: ActionsIn
             test_suites.extend(cls._get_action_test_suites(action))
         return TestSuites(name="", test_suites=test_suites)
 
+    def convert(self) -> TestSuites:
+        actions_invocation_record = ActionsInvocationRecord.from_xcresult(self.xcresult)
+        return self.actions_invocation_record_to_junit(actions_invocation_record)
+
+
+class Xcode16XcResultConverter(XcResultConverter):
     @classmethod
-    def xcresult_to_junit(cls, xcresult: pathlib.Path) -> TestSuites:
-        actions_invocation_record = ActionsInvocationRecord.from_xcresult(xcresult)
-        return cls.actions_invocation_record_to_junit(actions_invocation_record)
+    def _iter_nodes(cls, root_node: XcTestNode, node_type: XcTestNodeType) -> Iterator[XcTestNode]:
+        if root_node.node_type is node_type:
+            yield root_node
+        else:
+            for child in root_node.children:
+                yield from cls._iter_nodes(child, node_type)
+
+    @classmethod
+    def _get_run_destination(cls, root_node: XcTestNode) -> Optional[XcDevice]:
+        # TODO: support multiple run destinations
+        #  As a first iteration only one test destination is supported as in legacy mode
+
+        parent: Union[XcTests, XcTestNode] = root_node
+        while isinstance(parent, XcTestNode):
+            parent = parent.parent
+
+        tests = cast(XcTests, parent)
+        if not tests.devices:
+            return None
+        return tests.devices[0]
+
+    @classmethod
+    def _get_test_suite_name(cls, xc_test_suite: XcTestNode) -> str:
+        name = xc_test_suite.name or ""
+        device_info = ""
+
+        device = cls._get_run_destination(xc_test_suite)
+        if device:
+            platform = re.sub("simulator", "", device.platform, flags=re.IGNORECASE).strip()
+            device_info = f"{platform} {device.os_version} {device.model_name}"
+
+        if name and device_info:
+            return f"{name} [{device_info}]"
+        return name or device_info
+
+    @classmethod
+    def _get_test_case_error(cls, xc_test_case: XcTestNode) -> Optional[Error]:
+        if xc_test_case.result is not XcTestResult.FAILED:
+            return None
+
+        failure_messages_nodes = cls._iter_nodes(xc_test_case, XcTestNodeType.FAILURE_MESSAGE)
+        failure_messages = [node.name for node in failure_messages_nodes if node.name]
+        return Error(
+            message=failure_messages[0] if failure_messages else "",
+            type="Error" if any("caught error" in m for m in failure_messages) else "Failure",
+            error_description="\n".join(failure_messages) if len(failure_messages) > 1 else None,
+        )
+
+    @classmethod
+    def _get_test_case_skipped(cls, xc_test_case: XcTestNode) -> Optional[Skipped]:
+        if xc_test_case.result is not XcTestResult.SKIPPED:
+            return None
+
+        failure_messages_nodes = cls._iter_nodes(xc_test_case, XcTestNodeType.FAILURE_MESSAGE)
+        skipped_message_nodes = (node for node in failure_messages_nodes if node.result is XcTestResult.SKIPPED)
+        skipped_messages = [node.name for node in skipped_message_nodes if node.name]
+
+        return Skipped(message="\n".join(skipped_messages))
+
+    @classmethod
+    def _get_test_node_duration(cls, xc_test_case: XcTestNode) -> float:
+        if not xc_test_case.duration:
+            return 0.0
+
+        duration = xc_test_case.duration.replace(",", ".")
+        if duration.endswith("s"):
+            duration = duration[:-1]
+        return float(duration)
+
+    @classmethod
+    def _get_test_case(cls, xc_test_case: XcTestNode, xc_test_suite: XcTestNode) -> TestCase:
+        if xc_test_case.name:
+            method_name = xc_test_case.name
+        elif xc_test_case.node_identifier:
+            method_name = xc_test_case.node_identifier.split("/")[-1]
+        else:
+            method_name = ""
+
+        if xc_test_case.node_identifier:
+            classname = xc_test_case.node_identifier.split("/", maxsplit=1)[0]
+        elif xc_test_suite.name:
+            classname = xc_test_suite.name
+        else:
+            classname = ""
+
+        return TestCase(
+            name=method_name,
+            classname=classname,
+            error=cls._get_test_case_error(xc_test_case),
+            time=cls._get_test_node_duration(xc_test_case),
+            status=xc_test_case.result.value if xc_test_case.result else None,
+            skipped=cls._get_test_case_skipped(xc_test_case),
+        )
+
+    @classmethod
+    def _get_test_suite_properties(
+        cls,
+        xc_test_suite: XcTestNode,
+        xc_test_result_summary: XcSummary,
+    ) -> List[Property]:
+        device = cls._get_run_destination(xc_test_suite)
+
+        properties: List[Property] = [Property(name="title", value=xc_test_suite.name)]
+        if xc_test_result_summary.start_time:
+            properties.append(Property(name="started_time", value=cls._timestamp(xc_test_result_summary.start_time)))
+        if xc_test_result_summary.finish_time:
+            properties.append(Property(name="ended_time", value=cls._timestamp(xc_test_result_summary.finish_time)))
+        if device and device.model_name:
+            properties.append(Property(name="device_name", value=device.model_name))
+        if device and device.architecture:
+            properties.append(Property(name="device_architecture", value=device.architecture))
+        if device and device.device_id:
+            properties.append(Property(name="device_identifier", value=device.device_id))
+        if device and device.os_version:
+            properties.append(Property(name="device_operating_system", value=device.os_version))
+        if device and device.platform:
+            properties.append(Property(name="device_platform", value=device.platform))
+
+        return sorted(properties, key=lambda p: p.name)
+
+    @classmethod
+    def _get_test_suite(cls, xc_test_suite: XcTestNode, xc_test_result_summary: XcSummary) -> TestSuite:
+        xc_test_cases = list(cls._iter_nodes(xc_test_suite, XcTestNodeType.TEST_CASE))
+
+        timestamp = None
+        if xc_test_result_summary.finish_time:
+            timestamp = cls._timestamp(xc_test_result_summary.finish_time)
+
+        return TestSuite(
+            name=cls._get_test_suite_name(xc_test_suite),
+            tests=len(xc_test_cases),
+            disabled=0,  # Disabled tests are completely excluded from reports
+            errors=sum(1 for xc_test_case in xc_test_cases if xc_test_case.result is XcTestResult.FAILED),
+            failures=None,  # Xcode doesn't differentiate errors from failures, consider everything as error
+            package=xc_test_suite.name,
+            skipped=sum(1 for xc_test_case in xc_test_cases if xc_test_case.result is XcTestResult.SKIPPED),
+            time=sum(cls._get_test_node_duration(xc_test_case) for xc_test_case in xc_test_cases),
+            timestamp=timestamp,
+            testcases=[cls._get_test_case(xc_test_case, xc_test_suite) for xc_test_case in xc_test_cases],
+            properties=cls._get_test_suite_properties(xc_test_suite, xc_test_result_summary),
+        )
+
+    def convert(self) -> TestSuites:
+        tests_output = XcResultTool.get_test_report_tests(self.xcresult)
+        summary_output = XcResultTool.get_test_report_summary(self.xcresult)
+
+        xc_tests = XcTests.from_dict(tests_output)
+        xc_summary = XcSummary.from_dict(summary_output)
+
+        test_suites = [
+            self._get_test_suite(xc_test_suite_node, xc_summary)
+            for xc_test_node in xc_tests.test_nodes
+            for xc_test_suite_node in self._iter_nodes(xc_test_node, XcTestNodeType.TEST_SUITE)
+        ]
+
+        return TestSuites(
+            name=xc_summary.title,
+            test_suites=test_suites,
+        )