[wptrunner] Add option to skip subtest checks for incomplete tests

tools/wptrunner/wptrunner/browsers/chrome.py

@@ -7,11 +7,11 @@
 from .base import cmd_arg
 from ..executors import executor_kwargs as base_executor_kwargs
 from ..executors.executorwebdriver import WebDriverCrashtestExecutor  # noqa: F401
-from ..executors.base import WdspecExecutor  # noqa: F401
 from ..executors.executorchrome import (  # noqa: F401
     ChromeDriverPrintRefTestExecutor,
     ChromeDriverRefTestExecutor,
     ChromeDriverTestharnessExecutor,
+    ChromeDriverWdspecExecutor,
 )
 
 
@@ -21,7 +21,7 @@
                  "executor": {"testharness": "ChromeDriverTestharnessExecutor",
                               "reftest": "ChromeDriverRefTestExecutor",
                               "print-reftest": "ChromeDriverPrintRefTestExecutor",
-                              "wdspec": "WdspecExecutor",
+                              "wdspec": "ChromeDriverWdspecExecutor",
                               "crashtest": "WebDriverCrashtestExecutor"},
                  "browser_kwargs": "browser_kwargs",
                  "executor_kwargs": "executor_kwargs",
@@ -63,6 +63,7 @@ def executor_kwargs(logger, test_type, test_environment, run_info_data,
     executor_kwargs["close_after_done"] = True
     executor_kwargs["sanitizer_enabled"] = sanitizer_enabled
     executor_kwargs["reuse_window"] = kwargs.get("reuse_window", False)
+    executor_kwargs["check_incomplete_subtests"] = kwargs["check_incomplete_subtests"]
 
     capabilities = {
         "goog:chromeOptions": {

tools/wptrunner/wptrunner/executors/executorchrome.py

@@ -11,6 +11,7 @@
 from .base import (
     CrashtestExecutor,
     TestharnessExecutor,
+    WdspecExecutor,
     get_pages,
 )
 from .executorwebdriver import (
@@ -218,12 +219,34 @@ class ChromeDriverRefTestExecutor(WebDriverRefTestExecutor, _SanitizerMixin):  #
     protocol_cls = ChromeDriverProtocol
 
 
+_incomplete_statuses = {"INTERNAL-ERROR", "TIMEOUT", "EXTERNAL-TIMEOUT", "CRASH"}
+
+
 class ChromeDriverTestharnessExecutor(WebDriverTestharnessExecutor, _SanitizerMixin):  # type: ignore
     protocol_cls = ChromeDriverProtocol
 
-    def __init__(self, *args, reuse_window=False, **kwargs):
+    def __init__(self, *args, reuse_window=False, check_incomplete_subtests=False, **kwargs):
         super().__init__(*args, **kwargs)
         self.protocol.reuse_window = reuse_window
+        self._check_incomplete_subtests = check_incomplete_subtests
+
+    def convert_result(self, test, results, extra=None):
+        harness_result, subtest_results = super().convert_result(test, results, extra)
+        if not self._check_incomplete_subtests and harness_result.status in _incomplete_statuses:
+            subtest_results = []
+        return harness_result, subtest_results
+
+
+class ChromeDriverWdspecExecutor(WdspecExecutor):
+    def __init__(self, *args, check_incomplete_subtests=False, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._check_incomplete_subtests = check_incomplete_subtests
+
+    def convert_result(self, test, results):
+        harness_result, subtest_results = super().convert_result(test, results)
+        if not self._check_incomplete_subtests and harness_result.status in _incomplete_statuses:
+            subtest_results = []
+        return harness_result, subtest_results
 
 
 class ChromeDriverPrintRefTestExecutor(ChromeDriverRefTestExecutor):

tools/wptrunner/wptrunner/wptcommandline.py

@@ -382,6 +382,11 @@ def create_parser(product_choices=None):
         action="store_true",
         dest="sanitizer_enabled",
         help="Only alert on sanitizer-related errors and crashes.")
+    chrome_group.add_argument(
+        "--no-check-incomplete-subtests",
+        action="store_false",
+        dest="check_incomplete_subtests",
+        help="Do not check or report subtest results for tests that time out or crash.")
     chrome_group.add_argument(
         "--reuse-window",
         action="store_true",