[wptrunner] Decouple testdriver infrastructure from testharness

tools/wptrunner/wptrunner/environment.py

@@ -23,8 +23,6 @@
 sys.path.insert(0, repo_root)
 from tools import localpaths  # noqa: F401
 
-from wptserve.handlers import StringHandler
-
 serve = None
 
 
@@ -225,29 +223,47 @@ def get_routes(self):
             self.config.aliases,
             self.config)
 
+        testharnessreport_format_args = {
+            "output": self.pause_after_test,
+            "timeout_multiplier": self.testharness_timeout_multipler,
+            "explicit_timeout": "true" if self.debug_info is not None else "false",
+            "debug": "true" if self.debug_test else "false",
+        }
         for path, format_args, content_type, route in [
                 ("testharness_runner.html", {}, "text/html", "/testharness_runner.html"),
                 ("print_pdf_runner.html", {}, "text/html", "/print_pdf_runner.html"),
-                (os.path.join(here, "..", "..", "third_party", "pdf_js", "pdf.js"), None,
+                (os.path.join(here, "..", "..", "third_party", "pdf_js", "pdf.js"), {},
                  "text/javascript", "/_pdf_js/pdf.js"),
-                (os.path.join(here, "..", "..", "third_party", "pdf_js", "pdf.worker.js"), None,
+                (os.path.join(here, "..", "..", "third_party", "pdf_js", "pdf.worker.js"), {},
                  "text/javascript", "/_pdf_js/pdf.worker.js"),
-                (self.options.get("testharnessreport", "testharnessreport.js"),
-                 {"output": self.pause_after_test,
-                  "timeout_multiplier": self.testharness_timeout_multipler,
-                  "explicit_timeout": "true" if self.debug_info is not None else "false",
-                  "debug": "true" if self.debug_test else "false"},
-                 "text/javascript;charset=utf8",
-                 "/resources/testharnessreport.js")]:
-            path = os.path.normpath(os.path.join(here, path))
+                (
+                    self.options.get("testharnessreport", [
+                        # All testharness tests, even those that don't use testdriver, require
+                        # `message-queue.js` to signal completion.
+                        os.path.join("executors", "message-queue.js"),
+                        "testharnessreport.js"]),
+                    testharnessreport_format_args,
+                    "text/javascript;charset=utf8",
+                    "/resources/testharnessreport.js",
+                ),
+                (
+                    [os.path.join(repo_root, "resources", "testdriver.js"),
+                     # Include `message-queue.js` to support testdriver in non-testharness tests.
+                     os.path.join("executors", "message-queue.js"),
+                     "testdriver-extra.js"],
+                    {},
+                    "text/javascript",
+                    "/resources/testdriver.js",
+                ),
+        ]:
+            paths = [path] if isinstance(path, str) else path
+            abs_paths = [os.path.normpath(os.path.join(here, path)) for path in paths]
             # Note that .headers. files don't apply to static routes, so we need to
             # readd any static headers here.
             headers = {"Cache-Control": "max-age=3600"}
-            route_builder.add_static(path, format_args, content_type, route,
+            route_builder.add_static(abs_paths, format_args, content_type, route,
                                      headers=headers)
 
-        route_builder.add_handler("GET", "/resources/testdriver.js", TestdriverLoader())
-
         for url_base, test_root in self.test_paths.items():
             if url_base == "/":
                 continue
@@ -315,27 +331,6 @@ def test_servers(self):
         return failed, pending
 
 
-class TestdriverLoader:
-    """A special static handler for serving `/resources/testdriver.js`.
-
-    This handler lazily reads `testdriver{,-extra}.js` so that wptrunner doesn't
-    need to pass the entire file contents to child `wptserve` processes, which
-    can slow `wptserve` startup by several seconds (crbug.com/1479850).
-    """
-    def __init__(self):
-        self._handler = None
-
-    def __call__(self, request, response):
-        if not self._handler:
-            data = b""
-            with open(os.path.join(repo_root, "resources", "testdriver.js"), "rb") as fp:
-                data += fp.read()
-            with open(os.path.join(here, "testdriver-extra.js"), "rb") as fp:
-                data += fp.read()
-            self._handler = StringHandler(data, "text/javascript")
-        return self._handler(request, response)
-
-
 def wait_for_service(logger: StructuredLogger,
                      host: str,
                      port: int,

tools/wptrunner/wptrunner/executors/executorwebdriver.py

@@ -746,72 +746,19 @@ def run_func(self):
             self.result_flag.set()
 
 
-class WebDriverTestharnessExecutor(TestharnessExecutor):
-    supports_testdriver = True
-    protocol_cls = WebDriverProtocol
-    _get_next_message = None
-
-    def __init__(self, logger, browser, server_config, timeout_multiplier=1,
-                 close_after_done=True, capabilities=None, debug_info=None,
-                 cleanup_after_test=True, **kwargs):
-        """WebDriver-based executor for testharness.js tests"""
-        TestharnessExecutor.__init__(self, logger, browser, server_config,
-                                     timeout_multiplier=timeout_multiplier,
-                                     debug_info=debug_info)
-        self.protocol = self.protocol_cls(self, browser, capabilities)
-        with open(os.path.join(here, "testharness_webdriver_resume.js")) as f:
-            self.script_resume = f.read()
-        with open(os.path.join(here, "window-loaded.js")) as f:
-            self.window_loaded_script = f.read()
-
-        if hasattr(self.protocol, 'bidi_script'):
-            # If `bidi_script` is available, the messages can be handled via BiDi.
-            self._get_next_message = self._get_next_message_bidi
-        else:
-            self._get_next_message = self._get_next_message_classic
-
-        self.close_after_done = close_after_done
-        self.cleanup_after_test = cleanup_after_test
-
-    def is_alive(self):
-        return self.protocol.is_alive()
-
-    def on_environment_change(self, new_environment):
-        if new_environment["protocol"] != self.last_environment["protocol"]:
-            self.protocol.testharness.load_runner(new_environment["protocol"])
-
-    def do_test(self, test):
-        url = self.test_url(test)
-
-        success, data = WebDriverRun(self.logger,
-                                     self.do_testharness,
-                                     self.protocol,
-                                     url,
-                                     test.timeout * self.timeout_multiplier,
-                                     self.extra_timeout).run()
-
-        if success:
-            data, extra = data
-            return self.convert_result(test, data, extra=extra)
-
-        return (test.make_result(*data), [])
-
-    def do_testharness(self, protocol, url, timeout):
-        # The previous test may not have closed its old windows (if something
-        # went wrong or if cleanup_after_test was False), so clean up here.
-        protocol.testharness.close_old_windows()
+# TODO(web-platform-tests/wpt#13183): Add testdriver support to the other
+# executors.
+class TestDriverExecutorMixin:
+    def __init__(self, script_resume: str):
+        self.script_resume = script_resume
 
+    def run_testdriver(self, protocol, url, timeout):
         # If protocol implements `bidi_events`, remove all the existing subscriptions.
         if hasattr(protocol, 'bidi_events'):
             # Use protocol loop to run the async cleanup.
             protocol.loop.run_until_complete(protocol.bidi_events.unsubscribe_all())
 
-        # Now start the test harness
         test_window = self.get_or_create_test_window(protocol)
-        self.protocol.base.set_window(test_window)
-        # Wait until about:blank has been loaded
-        protocol.base.execute_script(self.window_loaded_script, asynchronous=True)
-
         # Exceptions occurred outside the main loop.
         unexpected_exceptions = []
 
@@ -896,36 +843,14 @@ async def process_bidi_event(method, params):
             # Use protocol loop to run the async cleanup.
             protocol.loop.run_until_complete(protocol.bidi_events.unsubscribe_all())
 
-        extra = {}
-        if leak_part := getattr(protocol, "leak", None):
-            testharness_window = protocol.base.current_window
-            extra_windows = set(protocol.base.window_handles())
-            extra_windows -= {protocol.testharness.runner_handle, testharness_window}
-            protocol.testharness.close_windows(extra_windows)
-            try:
-                protocol.base.set_window(testharness_window)
-                if counters := leak_part.check():
-                    extra["leak_counters"] = counters
-            except webdriver_error.NoSuchWindowException:
-                pass
-            finally:
-                protocol.base.set_window(protocol.testharness.runner_handle)
-
-        # Attempt to clean up any leftover windows, if allowed. This is
-        # preferable as it will blame the correct test if something goes wrong
-        # closing windows, but if the user wants to see the test results we
-        # have to leave the window(s) open.
-        if self.cleanup_after_test:
-            protocol.testharness.close_old_windows()
-
         if len(unexpected_exceptions) > 0:
             # TODO: what to do if there are more then 1 unexpected exceptions?
             raise unexpected_exceptions[0]
 
-        return rv, extra
+        return rv
 
     def get_or_create_test_window(self, protocol):
-        return protocol.base.create_window()
+        return protocol.base.current_window
 
     def _get_next_message_classic(self, protocol, url, _):
         """
@@ -969,6 +894,99 @@ def _get_next_message_bidi(self, protocol, url, test_window):
         return deserialized_message
 
 
+class WebDriverTestharnessExecutor(TestharnessExecutor, TestDriverExecutorMixin):
+    supports_testdriver = True
+    protocol_cls = WebDriverProtocol
+    _get_next_message = None
+
+    def __init__(self, logger, browser, server_config, timeout_multiplier=1,
+                 close_after_done=True, capabilities=None, debug_info=None,
+                 cleanup_after_test=True, **kwargs):
+        """WebDriver-based executor for testharness.js tests"""
+        TestharnessExecutor.__init__(self, logger, browser, server_config,
+                                     timeout_multiplier=timeout_multiplier,
+                                     debug_info=debug_info)
+        self.protocol = self.protocol_cls(self, browser, capabilities)
+        with open(os.path.join(here, "testharness_webdriver_resume.js")) as f:
+            script_resume = f.read()
+        TestDriverExecutorMixin.__init__(self, script_resume)
+        with open(os.path.join(here, "window-loaded.js")) as f:
+            self.window_loaded_script = f.read()
+
+        if hasattr(self.protocol, 'bidi_script'):
+            # If `bidi_script` is available, the messages can be handled via BiDi.
+            self._get_next_message = self._get_next_message_bidi
+        else:
+            self._get_next_message = self._get_next_message_classic
+
+        self.close_after_done = close_after_done
+        self.cleanup_after_test = cleanup_after_test
+
+    def is_alive(self):
+        return self.protocol.is_alive()
+
+    def on_environment_change(self, new_environment):
+        if new_environment["protocol"] != self.last_environment["protocol"]:
+            self.protocol.testharness.load_runner(new_environment["protocol"])
+
+    def do_test(self, test):
+        url = self.test_url(test)
+
+        success, data = WebDriverRun(self.logger,
+                                     self.do_testharness,
+                                     self.protocol,
+                                     url,
+                                     test.timeout * self.timeout_multiplier,
+                                     self.extra_timeout).run()
+
+        if success:
+            data, extra = data
+            return self.convert_result(test, data, extra=extra)
+
+        return (test.make_result(*data), [])
+
+    def do_testharness(self, protocol, url, timeout):
+        try:
+            # The previous test may not have closed its old windows (if something
+            # went wrong or if cleanup_after_test was False), so clean up here.
+            protocol.testharness.close_old_windows()
+            raw_results = self.run_testdriver(protocol, url, timeout)
+            extra = {}
+            if counters := self._check_for_leaks(protocol):
+                extra["leak_counters"] = counters
+            return raw_results, extra
+        finally:
+            # Attempt to clean up any leftover windows, if allowed. This is
+            # preferable as it will blame the correct test if something goes
+            # wrong closing windows, but if the user wants to see the test
+            # results we have to leave the window(s) open.
+            if self.cleanup_after_test:
+                protocol.testharness.close_old_windows()
+
+    def _check_for_leaks(self, protocol):
+        leak_part = getattr(protocol, "leak", None)
+        if not leak_part:
+            return None
+        testharness_window = protocol.base.current_window
+        extra_windows = set(protocol.base.window_handles())
+        extra_windows -= {protocol.testharness.runner_handle, testharness_window}
+        protocol.testharness.close_windows(extra_windows)
+        try:
+            protocol.base.set_window(testharness_window)
+            return leak_part.check()
+        except webdriver_error.NoSuchWindowException:
+            pass
+        finally:
+            protocol.base.set_window(protocol.testharness.runner_handle)
+
+    def get_or_create_test_window(self, protocol):
+        test_window = protocol.base.create_window()
+        protocol.base.set_window(test_window)
+        # Wait until about:blank has been loaded
+        protocol.base.execute_script(self.window_loaded_script, asynchronous=True)
+        return test_window
+
+
 class WebDriverRefTestExecutor(RefTestExecutor):
     protocol_cls = WebDriverProtocol
 

tools/wptrunner/wptrunner/executors/message-queue.js

@@ -0,0 +1,77 @@
+(function() {
+  if (window.__wptrunner_message_queue && window.__wptrunner_process_next_event) {
+    // Another script already set up the testdriver infrastructure.
+    return;
+  }
+
+  class MessageQueue {
+    constructor() {
+      this.item_id = 0;
+      this._queue = [];
+    }
+
+    push(item) {
+      let cmd_id = this.item_id++;
+      item.id = cmd_id;
+      this._queue.push(item);
+      __wptrunner_process_next_event();
+      return cmd_id;
+    }
+
+    shift() {
+      return this._queue.shift();
+    }
+  }
+
+  window.__wptrunner_testdriver_callback = null;
+  window.__wptrunner_message_queue = new MessageQueue();
+  window.__wptrunner_url = null;
+
+  window.__wptrunner_process_next_event = function() {
+    /* This function handles the next testdriver event. The presence of
+       window.testdriver_callback is used as a switch; when that function
+       is present we are able to handle the next event and when is is not
+       present we must wait. Therefore to drive the event processing, this
+       function must be called in two circumstances:
+         * Every time there is a new event that we may be able to handle
+         * Every time we set the callback function
+       This function unsets the callback, so no further testdriver actions
+       will be run until it is reset, which wptrunner does after it has
+       completed handling the current action.
+     */
+
+    if (!window.__wptrunner_testdriver_callback) {
+      return;
+    }
+    var data = window.__wptrunner_message_queue.shift();
+    if (!data) {
+      return;
+    }
+
+    var payload = undefined;
+
+    switch(data.type) {
+    case "complete":
+      var tests = data.tests;
+      var status = data.status;
+
+      var subtest_results = tests.map(function(x) {
+        return [x.name, x.status, x.message, x.stack];
+      });
+      payload = [status.status,
+                 status.message,
+                 status.stack,
+                 subtest_results];
+      clearTimeout(window.__wptrunner_timer);
+      break;
+    case "action":
+      payload = data;
+      break;
+    default:
+      return;
+    }
+    var callback = window.__wptrunner_testdriver_callback;
+    window.__wptrunner_testdriver_callback = null;
+    callback([__wptrunner_url, data.type, payload]);
+  };
+})();