diff --git a/tools/ci/ci_wptrunner_infrastructure.sh b/tools/ci/ci_wptrunner_infrastructure.sh index d6d6803974fb05..c32c9438703662 100755 --- a/tools/ci/ci_wptrunner_infrastructure.sh +++ b/tools/ci/ci_wptrunner_infrastructure.sh @@ -18,14 +18,14 @@ test_infrastructure() { } main() { - PRODUCTS=( "firefox" "chrome" ) + PRODUCTS=( "firefox" "chrome" "chrome_webdriver" ) for PRODUCT in "${PRODUCTS[@]}"; do if [ "$PRODUCT" != "firefox" ]; then # Firefox is expected to work using pref settings for DNS # Don't adjust the hostnames in that case to ensure this keeps working hosts_fixup fi - if [ "$PRODUCT" == "chrome" ]; then + if [[ "$PRODUCT" == "chrome"* ]]; then install_chrome unstable test_infrastructure "--binary=$(which google-chrome-unstable)" else diff --git a/tools/webdriver/webdriver/error.py b/tools/webdriver/webdriver/error.py index e148e8fe800700..23ffc40b31ffc9 100644 --- a/tools/webdriver/webdriver/error.py +++ b/tools/webdriver/webdriver/error.py @@ -6,8 +6,10 @@ class WebDriverException(Exception): http_status = None status_code = None - def __init__(self, message=None, stacktrace=None): + def __init__(self, http_status=None, status_code=None, message=None, stacktrace=None): super(WebDriverException, self) + self.http_status = http_status + self.status_code = status_code self.message = message self.stacktrace = stacktrace @@ -171,6 +173,8 @@ def from_response(response): """ if response.status == 200: raise UnknownErrorException( + response.status, + None, "Response is not an error:\n" "%s" % json.dumps(response.body)) @@ -178,6 +182,8 @@ def from_response(response): value = response.body["value"] else: raise UnknownErrorException( + response.status, + None, "Expected 'value' key in response body:\n" "%s" % json.dumps(response.body)) @@ -187,7 +193,7 @@ def from_response(response): stack = value["stacktrace"] or None cls = get(code) - return cls(message, stacktrace=stack) + return cls(response.status, code, message, stacktrace=stack) def get(error_code): diff --git a/tools/wpt/browser.py b/tools/wpt/browser.py index 65752a25f12864..7b943c2e73ccbb 100644 --- a/tools/wpt/browser.py +++ b/tools/wpt/browser.py @@ -486,6 +486,12 @@ def install_webdriver(self, dest=None, channel=None): def version(self, binary): return None +class ChromeWebDriver(Chrome): + """Chrome-specific interface for chrome without using selenium. + + Includes webdriver installation. + """ + product = "chrome_webdriver" class Opera(Browser): """Opera-specific interface. @@ -582,6 +588,10 @@ def version(self, binary): return None +class EdgeWebDriver(Edge): + product = "edge_webdriver" + + class InternetExplorer(Browser): """Internet Explorer-specific interface.""" @@ -629,6 +639,10 @@ def version(self, binary): return None +class SafariWebDriver(Safari): + product = "safari_webdriver" + + class Servo(Browser): """Servo-specific interface.""" diff --git a/tools/wpt/run.py b/tools/wpt/run.py index 036c38f25869bc..a1e5637bf1dede 100644 --- a/tools/wpt/run.py +++ b/tools/wpt/run.py @@ -273,6 +273,10 @@ def setup_kwargs(self, kwargs): else: raise WptrunError("Unable to locate or install chromedriver binary") +class ChromeWebDriver(Chrome): + name = "chrome_webdriver" + browser_cls = browser.ChromeWebDriver + class Opera(BrowserSetup): name = "opera" @@ -318,6 +322,11 @@ def setup_kwargs(self, kwargs): kwargs["webdriver_binary"] = webdriver_binary +class EdgeWebDriver(Edge): + name = "edge_webdriver" + browser_cls = browser.EdgeWebDriver + + class InternetExplorer(BrowserSetup): name = "ie" browser_cls = browser.InternetExplorer @@ -356,6 +365,11 @@ def setup_kwargs(self, kwargs): kwargs["webdriver_binary"] = webdriver_binary +class SafariWebDriver(Safari): + name = "safari_webdriver" + browser_cls = browser.SafariWebDriver + + class Sauce(BrowserSetup): name = "sauce" browser_cls = browser.Sauce @@ -402,9 +416,12 @@ def setup_kwargs(self, kwargs): "firefox": Firefox, "chrome": Chrome, "chrome_android": ChromeAndroid, + "chrome_webdriver": ChromeWebDriver, "edge": Edge, + "edge_webdriver": EdgeWebDriver, "ie": InternetExplorer, "safari": Safari, + "safari_webdriver": SafariWebDriver, "servo": Servo, "sauce": Sauce, "opera": Opera, diff --git a/tools/wptrunner/wptrunner/browsers/__init__.py b/tools/wptrunner/wptrunner/browsers/__init__.py index d8682e16a551e5..08949f794834fd 100644 --- a/tools/wptrunner/wptrunner/browsers/__init__.py +++ b/tools/wptrunner/wptrunner/browsers/__init__.py @@ -24,11 +24,14 @@ product_list = ["chrome", "chrome_android", + "chrome_webdriver", "edge", + "edge_webdriver", "fennec", "firefox", "ie", "safari", + "safari_webdriver", "sauce", "servo", "servodriver", diff --git a/tools/wptrunner/wptrunner/browsers/base.py b/tools/wptrunner/wptrunner/browsers/base.py index dc03ef711b6014..70324bec31f23d 100644 --- a/tools/wptrunner/wptrunner/browsers/base.py +++ b/tools/wptrunner/wptrunner/browsers/base.py @@ -2,12 +2,32 @@ import platform import socket from abc import ABCMeta, abstractmethod +from copy import deepcopy from ..wptcommandline import require_arg # noqa: F401 here = os.path.split(__file__)[0] +def inherit(super_module, child_globals, product_name): + super_wptrunner = super_module.__wptrunner__ + child_globals["__wptrunner__"] = child_wptrunner = deepcopy(super_wptrunner) + + child_wptrunner["product"] = product_name + + for k in ("check_args", "browser", "browser_kwargs", "executor_kwargs", + "env_extras", "env_options"): + attr = super_wptrunner[k] + child_globals[attr] = getattr(super_module, attr) + + for v in super_module.__wptrunner__["executor"].values(): + child_globals[v] = getattr(super_module, v) + + if "run_info_extras" in super_wptrunner: + attr = super_wptrunner["run_info_extras"] + child_globals[attr] = getattr(super_module, attr) + + def cmd_arg(name, value=None): prefix = "-" if platform.system() == "Windows" else "--" rv = prefix + name diff --git a/tools/wptrunner/wptrunner/browsers/chrome_webdriver.py b/tools/wptrunner/wptrunner/browsers/chrome_webdriver.py new file mode 100644 index 00000000000000..a63460f4544af6 --- /dev/null +++ b/tools/wptrunner/wptrunner/browsers/chrome_webdriver.py @@ -0,0 +1,50 @@ +from .base import inherit +from . import chrome + +from ..executors import executor_kwargs as base_executor_kwargs +from ..executors.executorwebdriver import (WebDriverTestharnessExecutor, # noqa: F401 + WebDriverRefTestExecutor) # noqa: F401 + + +inherit(chrome, globals(), "chrome_webdriver") + +# __wptrunner__ magically appears from inherit, F821 is undefined name +__wptrunner__["executor_kwargs"] = "executor_kwargs" # noqa: F821 +__wptrunner__["executor"]["testharness"] = "WebDriverTestharnessExecutor" # noqa: F821 +__wptrunner__["executor"]["reftest"] = "WebDriverRefTestExecutor" # noqa: F821 + + +def executor_kwargs(test_type, server_config, cache_manager, run_info_data, + **kwargs): + executor_kwargs = base_executor_kwargs(test_type, server_config, + cache_manager, run_info_data, + **kwargs) + executor_kwargs["close_after_done"] = True + + capabilities = { + "browserName": "chrome", + "platform": "ANY", + "version": "", + "goog:chromeOptions": { + "prefs": { + "profile": { + "default_content_setting_values": { + "popups": 1 + } + } + }, + "w3c": True + } + } + + for (kwarg, capability) in [("binary", "binary"), ("binary_args", "args")]: + if kwargs[kwarg] is not None: + capabilities["goog:chromeOptions"][capability] = kwargs[kwarg] + + if test_type == "testharness": + capabilities["goog:chromeOptions"]["useAutomationExtension"] = False + capabilities["goog:chromeOptions"]["excludeSwitches"] = ["enable-automation"] + + executor_kwargs["capabilities"] = capabilities + + return executor_kwargs diff --git a/tools/wptrunner/wptrunner/browsers/edge_webdriver.py b/tools/wptrunner/wptrunner/browsers/edge_webdriver.py new file mode 100644 index 00000000000000..c2545de46f0b5d --- /dev/null +++ b/tools/wptrunner/wptrunner/browsers/edge_webdriver.py @@ -0,0 +1,12 @@ +from .base import inherit +from . import edge + +from ..executors.executorwebdriver import (WebDriverTestharnessExecutor, # noqa: F401 + WebDriverRefTestExecutor) # noqa: F401 + + +inherit(edge, globals(), "edge_webdriver") + +# __wptrunner__ magically appears from inherit, F821 is undefined name +__wptrunner__["executor"]["testharness"] = "WebDriverTestharnessExecutor" # noqa: F821 +__wptrunner__["executor"]["reftest"] = "WebDriverRefTestExecutor" # noqa: F821 diff --git a/tools/wptrunner/wptrunner/browsers/safari_webdriver.py b/tools/wptrunner/wptrunner/browsers/safari_webdriver.py new file mode 100644 index 00000000000000..12735c141b3ad8 --- /dev/null +++ b/tools/wptrunner/wptrunner/browsers/safari_webdriver.py @@ -0,0 +1,12 @@ +from .base import inherit +from . import safari + +from ..executors.executorwebdriver import (WebDriverTestharnessExecutor, # noqa: F401 + WebDriverRefTestExecutor) # noqa: F401 + + +inherit(safari, globals(), "safari_webdriver") + +# __wptrunner__ magically appears from inherit, F821 is undefined name +__wptrunner__["executor"]["testharness"] = "WebDriverTestharnessExecutor" # noqa: F821 +__wptrunner__["executor"]["reftest"] = "WebDriverRefTestExecutor" # noqa: F821 diff --git a/tools/wptrunner/wptrunner/executors/executorselenium.py b/tools/wptrunner/wptrunner/executors/executorselenium.py index d9b67968ddf66e..0675461d5d8758 100644 --- a/tools/wptrunner/wptrunner/executors/executorselenium.py +++ b/tools/wptrunner/wptrunner/executors/executorselenium.py @@ -95,14 +95,14 @@ def close_old_windows(self): def get_test_window(self, window_id, parent): test_window = None - if window_id: - try: - # Try this, it's in Level 1 but nothing supports it yet - win_s = self.webdriver.execute_script("return window['%s'];" % self.window_id) - win_obj = json.loads(win_s) - test_window = win_obj["window-fcc6-11e5-b4f8-330a88ab9d7f"] - except Exception: - pass + try: + # Try using the JSON serialization of the WindowProxy object, + # it's in Level 1 but nothing supports it yet + win_s = self.webdriver.execute_script("return window['%s'];" % window_id) + win_obj = json.loads(win_s) + test_window = win_obj["window-fcc6-11e5-b4f8-330a88ab9d7f"] + except Exception: + pass if test_window is None: after = self.webdriver.window_handles @@ -296,7 +296,7 @@ def do_testharness(self, protocol, url, timeout): parent_window = protocol.testharness.close_old_windows() # Now start the test harness protocol.base.execute_script(self.script % format_map) - test_window = protocol.testharness.get_test_window(webdriver, parent_window) + test_window = protocol.testharness.get_test_window(self.window_id, parent_window) handler = CallbackHandler(self.logger, protocol, test_window) while True: diff --git a/tools/wptrunner/wptrunner/executors/executorwebdriver.py b/tools/wptrunner/wptrunner/executors/executorwebdriver.py new file mode 100644 index 00000000000000..127c909e810a26 --- /dev/null +++ b/tools/wptrunner/wptrunner/executors/executorwebdriver.py @@ -0,0 +1,373 @@ +import json +import os +import socket +import threading +import traceback +import urlparse +import uuid + +from .base import (CallbackHandler, + RefTestExecutor, + RefTestImplementation, + TestharnessExecutor, + extra_timeout, + strip_server) +from .protocol import (BaseProtocolPart, + TestharnessProtocolPart, + Protocol, + SelectorProtocolPart, + ClickProtocolPart, + SendKeysProtocolPart, + TestDriverProtocolPart) +from ..testrunner import Stop + +import webdriver as client + +here = os.path.join(os.path.split(__file__)[0]) + +class WebDriverBaseProtocolPart(BaseProtocolPart): + def setup(self): + self.webdriver = self.parent.webdriver + + def execute_script(self, script, async=False): + method = self.webdriver.execute_async_script if async else self.webdriver.execute_script + return method(script) + + def set_timeout(self, timeout): + try: + self.webdriver.timeouts.script = timeout + except client.WebDriverException: + # workaround https://bugs.chromium.org/p/chromedriver/issues/detail?id=2057 + body = {"type": "script", "ms": timeout * 1000} + self.webdriver.send_session_command("POST", "timeouts", body) + + @property + def current_window(self): + return self.webdriver.window_handle + + def set_window(self, handle): + self.webdriver.window_handle = handle + + def wait(self): + while True: + try: + self.webdriver.execute_async_script("") + except client.TimeoutException: + pass + except (socket.timeout, client.NoSuchWindowException, + client.UnknownErrorException, IOError): + break + except Exception as e: + self.logger.error(traceback.format_exc(e)) + break + + +class WebDriverTestharnessProtocolPart(TestharnessProtocolPart): + def setup(self): + self.webdriver = self.parent.webdriver + + def load_runner(self, url_protocol): + url = urlparse.urljoin(self.parent.executor.server_url(url_protocol), + "/testharness_runner.html") + self.logger.debug("Loading %s" % url) + + self.webdriver.url = url + self.webdriver.execute_script("document.title = '%s'" % + threading.current_thread().name.replace("'", '"')) + + def close_old_windows(self): + exclude = self.webdriver.window_handle + handles = [item for item in self.webdriver.handles if item != exclude] + for handle in handles: + try: + self.webdriver.window_handle = handle + self.webdriver.close() + except client.NoSuchWindowException: + pass + self.webdriver.window_handle = exclude + return exclude + + def get_test_window(self, window_id, parent): + test_window = None + try: + # Try using the JSON serialization of the WindowProxy object, + # it's in Level 1 but nothing supports it yet + win_s = self.webdriver.execute_script("return window['%s'];" % window_id) + win_obj = json.loads(win_s) + test_window = win_obj["window-fcc6-11e5-b4f8-330a88ab9d7f"] + except Exception: + pass + + if test_window is None: + after = self.webdriver.handles + if len(after) == 2: + test_window = next(iter(set(after) - set([parent]))) + elif after[0] == parent and len(after) > 2: + # Hope the first one here is the test window + test_window = after[1] + else: + raise Exception("unable to find test window") + + assert test_window != parent + return test_window + + +class WebDriverSelectorProtocolPart(SelectorProtocolPart): + def setup(self): + self.webdriver = self.parent.webdriver + + def elements_by_selector(self, selector): + return self.webdriver.find.css(selector) + + +class WebDriverClickProtocolPart(ClickProtocolPart): + def setup(self): + self.webdriver = self.parent.webdriver + + def element(self, element): + return element.click() + + +class WebDriverSendKeysProtocolPart(SendKeysProtocolPart): + def setup(self): + self.webdriver = self.parent.webdriver + + def send_keys(self, element, keys): + try: + return element.send_keys(keys) + except client.UnknownErrorException as e: + # workaround https://bugs.chromium.org/p/chromedriver/issues/detail?id=1999 + if (e.http_status != 500 or + e.status_code != "unknown error"): + raise + return element.send_element_command("POST", "value", {"value": list(keys)}) + + +class WebDriverTestDriverProtocolPart(TestDriverProtocolPart): + def setup(self): + self.webdriver = self.parent.webdriver + + def send_message(self, message_type, status, message=None): + obj = { + "type": "testdriver-%s" % str(message_type), + "status": str(status) + } + if message: + obj["message"] = str(message) + self.webdriver.execute_script("window.postMessage(%s, '*')" % json.dumps(obj)) + + +class WebDriverProtocol(Protocol): + implements = [WebDriverBaseProtocolPart, + WebDriverTestharnessProtocolPart, + WebDriverSelectorProtocolPart, + WebDriverClickProtocolPart, + WebDriverSendKeysProtocolPart, + WebDriverTestDriverProtocolPart] + + def __init__(self, executor, browser, capabilities, **kwargs): + super(WebDriverProtocol, self).__init__(executor, browser) + self.capabilities = capabilities + self.url = browser.webdriver_url + self.webdriver = None + + def connect(self): + """Connect to browser via WebDriver.""" + self.logger.debug("Connecting to WebDriver on URL: %s" % self.url) + + host, port = self.url.split(":")[1].strip("/"), self.url.split(':')[-1].strip("/") + + capabilities = {"alwaysMatch": self.capabilities} + self.webdriver = client.Session(host, port, capabilities=capabilities) + self.webdriver.start() + + + def after_conect(self): + pass + + def teardown(self): + self.logger.debug("Hanging up on WebDriver session") + try: + self.webdriver.quit() + except Exception: + pass + del self.webdriver + + def is_alive(self): + try: + # Get a simple property over the connection + self.webdriver.window_handle + except (socket.timeout, client.UnknownErrorException): + return False + return True + + def after_connect(self): + self.testharness.load_runner(self.executor.last_environment["protocol"]) + + +class WebDriverRun(object): + def __init__(self, func, protocol, url, timeout): + self.func = func + self.result = None + self.protocol = protocol + self.url = url + self.timeout = timeout + self.result_flag = threading.Event() + + def run(self): + timeout = self.timeout + + try: + self.protocol.base.set_timeout((timeout + extra_timeout)) + except client.UnknownErrorException: + self.logger.error("Lost WebDriver connection") + return Stop + + executor = threading.Thread(target=self._run) + executor.start() + + flag = self.result_flag.wait(timeout + 2 * extra_timeout) + if self.result is None: + assert not flag + self.result = False, ("EXTERNAL-TIMEOUT", None) + + return self.result + + def _run(self): + try: + self.result = True, self.func(self.protocol, self.url, self.timeout) + except client.TimeoutException: + self.result = False, ("EXTERNAL-TIMEOUT", None) + except (socket.timeout, client.UnknownErrorException): + self.result = False, ("CRASH", None) + except Exception as e: + if (isinstance(e, client.WebDriverException) and + e.http_status == 408 and + e.status_code == "asynchronous script timeout"): + # workaround for https://bugs.chromium.org/p/chromedriver/issues/detail?id=2001 + self.result = False, ("EXTERNAL-TIMEOUT", None) + else: + message = getattr(e, "message", "") + if message: + message += "\n" + message += traceback.format_exc(e) + self.result = False, ("ERROR", message) + finally: + self.result_flag.set() + + +class WebDriverTestharnessExecutor(TestharnessExecutor): + supports_testdriver = True + + def __init__(self, browser, server_config, timeout_multiplier=1, + close_after_done=True, capabilities=None, debug_info=None, + **kwargs): + """WebDriver-based executor for testharness.js tests""" + TestharnessExecutor.__init__(self, browser, server_config, + timeout_multiplier=timeout_multiplier, + debug_info=debug_info) + self.protocol = WebDriverProtocol(self, browser, capabilities) + with open(os.path.join(here, "testharness_webdriver.js")) as f: + self.script = f.read() + with open(os.path.join(here, "testharness_webdriver_resume.js")) as f: + self.script_resume = f.read() + self.close_after_done = close_after_done + self.window_id = str(uuid.uuid4()) + + def is_alive(self): + return self.protocol.is_alive() + + def on_environment_change(self, new_environment): + if new_environment["protocol"] != self.last_environment["protocol"]: + self.protocol.testharness.load_runner(new_environment["protocol"]) + + def do_test(self, test): + url = self.test_url(test) + + success, data = WebDriverRun(self.do_testharness, + self.protocol, + url, + test.timeout * self.timeout_multiplier).run() + + if success: + return self.convert_result(test, data) + + return (test.result_cls(*data), []) + + def do_testharness(self, protocol, url, timeout): + format_map = {"abs_url": url, + "url": strip_server(url), + "window_id": self.window_id, + "timeout_multiplier": self.timeout_multiplier, + "timeout": timeout * 1000} + + parent_window = protocol.testharness.close_old_windows() + # Now start the test harness + protocol.base.execute_script(self.script % format_map) + test_window = protocol.testharness.get_test_window(self.window_id, parent_window) + + handler = CallbackHandler(self.logger, protocol, test_window) + while True: + result = protocol.base.execute_script( + self.script_resume % format_map, async=True) + done, rv = handler(result) + if done: + break + return rv + + +class WebDriverRefTestExecutor(RefTestExecutor): + def __init__(self, browser, server_config, timeout_multiplier=1, + screenshot_cache=None, close_after_done=True, + debug_info=None, capabilities=None, **kwargs): + """WebDriver-based executor for reftests""" + RefTestExecutor.__init__(self, + browser, + server_config, + screenshot_cache=screenshot_cache, + timeout_multiplier=timeout_multiplier, + debug_info=debug_info) + self.protocol = WebDriverProtocol(self, browser, + capabilities=capabilities) + self.implementation = RefTestImplementation(self) + self.close_after_done = close_after_done + self.has_window = False + + with open(os.path.join(here, "reftest.js")) as f: + self.script = f.read() + with open(os.path.join(here, "reftest-wait_webdriver.js")) as f: + self.wait_script = f.read() + + def is_alive(self): + return self.protocol.is_alive() + + def do_test(self, test): + self.protocol.webdriver.window.size = (600, 600) + + result = self.implementation.run_test(test) + + return self.convert_result(test, result) + + def screenshot(self, test, viewport_size, dpi): + # https://github.com/w3c/wptrunner/issues/166 + assert viewport_size is None + assert dpi is None + + return WebDriverRun(self._screenshot, + self.protocol, + self.test_url(test), + test.timeout).run() + + def _screenshot(self, protocol, url, timeout): + webdriver = protocol.webdriver + webdriver.url = url + + webdriver.execute_async_script(self.wait_script) + + screenshot = webdriver.screenshot() + + # strip off the data:img/png, part of the url + if screenshot.startswith("data:image/png;base64,"): + screenshot = screenshot.split(",", 1)[1] + + return screenshot diff --git a/tools/wptrunner/wptrunner/tests/base.py b/tools/wptrunner/wptrunner/tests/base.py index b5173f3b513ea4..84dc4f2e7f82ee 100644 --- a/tools/wptrunner/wptrunner/tests/base.py +++ b/tools/wptrunner/wptrunner/tests/base.py @@ -17,7 +17,9 @@ current_tox_env_split = os.environ["CURRENT_TOX_ENV"].split("-") tox_env_extra_browsers = { - "chrome": {"chrome_android"}, + "chrome": {"chrome_android", "chrome_webdriver"}, + "edge": {"edge_webdriver"}, + "safari": {"safari_webdriver"}, "servo": {"servodriver"}, }