Start running Windows CI again

.appveyor.yml

@@ -0,0 +1,44 @@
+skip_branch_with_pr: true
+
+clone_depth: 50
+
+environment:
+  matrix:
+    - PYTHON: "C:\\Python27"
+      JOB: "tools_unittest"
+      SCRIPT: "tools\\ci\\ci_tools_unittest.ps1"
+      TOX_DIR: "tools"
+      TOXENV: "py27"
+    - PYTHON: "C:\\Python27"
+      JOB: "tools_unittest"
+      SCRIPT: "tools\\ci\\ci_tools_unittest.ps1"
+      TOX_DIR: "tools\\wptrunner"
+      TOXENV: "py27"
+    - PYTHON: "C:\\Python27"
+      JOB: "wpt_integration"
+      SCRIPT: "tools\\ci\\ci_tools_unittest.ps1"
+      TOX_DIR: "tools\\wpt"
+      TOXENV: "py27"
+    - PYTHON: "C:\\Python27"
+      JOB: "wptrunner_infrastructure"
+      SCRIPT: "tools\\ci\\ci_wptrunner_infrastructure.ps1"
+    - PYTHON: "C:\\Python36"
+      JOB: "tools_unittest"
+      SCRIPT: "tools\\ci\\ci_tools_unittest.ps1"
+      TOX_DIR: "tools"
+      TOXENV: "py36"
+
+only_commits:
+  files:
+    - "tools\\**\\*"
+
+install:
+  - ps: "tools\\ci\\install.ps1"
+
+build: off
+
+test_script:
+  - ps: ". $env:SCRIPT"
+
+cache:
+  - MANIFEST.json

tools/ci/ci_tools_unittest.ps1

@@ -0,0 +1,11 @@
+Set-PSDebug -Trace 1
+
+. "$PSScriptRoot\lib.ps1"
+
+$WPT_ROOT = "$PSScriptRoot\..\.."
+cd $WPT_ROOT
+
+cd $env:TOX_DIR
+pip install -U tox
+run_applicable_tox
+

tools/ci/ci_wptrunner_infrastructure.ps1

@@ -0,0 +1,19 @@
+Set-PSDebug -Trace 1
+
+. "$PSScriptRoot\lib.ps1"
+
+$WPT_ROOT = "$PSScriptRoot\..\.."
+cd $WPT_ROOT
+
+$products = @("firefox", "chrome")
+foreach ($product in $products) {
+    $args = ""
+    if (!$product.Equals("firefox")) {
+        # Firefox is expected to work using pref settings for DNS
+        # Don't adjust the hostnames in that case to ensure this keeps working
+        hosts_fixup
+    } else {
+        $args += " --install-browser"
+    }
+    python ./wpt run --yes --metadata infrastructure/metadata/ --install-fonts $args $product infrastructure/
+}

tools/ci/install.ps1

@@ -0,0 +1,21 @@
+Set-PSDebug -Trace 1
+
+git fetch -q --depth=50 origin master
+
+$env:PATH = "$env:PYTHON;$env:PATH"
+
+$RELEVANT_JOBS = $(python ./wpt test-jobs).split(
+    [string[]]$null,
+    [System.StringSplitOptions]::RemoveEmptyEntries
+)
+
+if ($env:RUN_JOB -or $RELEVANT_JOBS.contains($env:JOB)) {
+    $env:RUN_JOB = $true
+} else {
+    $env:RUN_JOB = $false
+}
+
+if ($env:RUN_JOB) {
+    pip install -U setuptools
+    pip install -U requests
+}

tools/ci/lib.ps1

@@ -0,0 +1,25 @@
+function hosts_fixup {
+    $path="$env:SystemRoot\System32\drivers\etc\hosts"
+    Get-Content $path
+    python wpt make-hosts-file | Out-File $path -Encoding ascii -Append
+    Get-Content $path
+}
+
+function run_applicable_tox {
+    # instead of just running TOXENV (e.g., py27)
+    # run all environments that start with TOXENV
+    # (e.g., py27-firefox as well as py27)
+    $OLD_TOXENV=$env:TOXENV
+    Remove-Item env:TOXENV
+    $RUN_ENVS = $(tox -l).split(
+        [string[]]$null,
+        [System.StringSplitOptions]::RemoveEmptyEntries
+    ).Where({$_.StartsWith("$OLD_TOXENV-") -or $_.Equals($OLD_TOXENV)}) -join ","
+    if ($RUN_ENVS) {
+        tox -e "$RUN_ENVS"
+    }
+    if ($LastExitCode -ne 0) {
+        throw
+    }
+    $env:TOXENV = $OLD_TOXENV
+}

tools/wpt/testfiles.py

@@ -57,6 +57,10 @@ def branch_point():
         not_heads = [item for item in git("rev-parse", "--not", "--branches", "--remotes").split("\n")
                      if item != "^%s" % head]
 
+        # if we don't have any other branches, just return the HEAD commit
+        if not not_heads:
+            return head
+
         # get all commits on HEAD but not reachable from anything in not_heads
         commits = git("rev-list", "--topo-order", "--parents", "HEAD", *not_heads)
         commit_parents = OrderedDict()

tools/wpt/tests/test_testfiles.py

@@ -0,0 +1,33 @@
+import mock
+import pytest
+
+from tools.wpt import testfiles
+
+def get_get_git_cmd(rv_dict):
+    def get_git_cmd(root):
+        def git(*args):
+            return rv_dict[args]
+        return git
+    return get_git_cmd
+
+
+@pytest.mark.parametrize("environ,git_rv_dict,expected_rv", [
+    ({"TRAVIS_PULL_REQUEST": "false", "TRAVIS_BRANCH": "master"},
+     {("rev-parse", "HEAD"): "1a" * 20},
+     "1a" * 20),
+    ({"TRAVIS_PULL_REQUEST": "true", "TRAVIS_BRANCH": "foobar"},
+     {("merge-base", "HEAD", "foobar"): "1a" * 20},
+     "1a" * 20),
+    ({},
+     {("rev-parse", "HEAD"): "78978db3956ad3137784600bba20c8dcee6b638b",
+      ("rev-parse", "--not", "--branches", "--remotes"): "^78978db3956ad3137784600bba20c8dcee6b638b",
+      ("rev-list", "--topo-order", "--parents", "HEAD"):
+      "78978db3956ad3137784600bba20c8dcee6b638b "
+      "a4d5a787c59ca4176e70d665b5e335380077a29c\na4d5a787c59ca4176e70d665b5e335380077a29c"},
+     "78978db3956ad3137784600bba20c8dcee6b638b"),
+])
+def test_branch_point(environ, git_rv_dict, expected_rv):
+    with mock.patch.object(testfiles, "get_git_cmd", get_get_git_cmd(git_rv_dict)):
+        with mock.patch.object(testfiles.os, "environ", environ):
+            rv = testfiles.branch_point()
+    assert expected_rv == rv

tools/wptserve/tests/functional/test_input_file.py

@@ -3,6 +3,8 @@
 
 import pytest
 
+from six import PY2
+
 from wptserve.request import InputFile
 
 bstr = b'This is a test document\nWith new lines\nSeveral in fact...'
@@ -120,7 +122,7 @@ def test_readlines():
     assert input_file.readlines() == test_file.readlines()
 
 
-@pytest.mark.xfail(sys.platform == "win32",
+@pytest.mark.xfail(PY2 and sys.platform == "win32",
                    reason="https://github.com/web-platform-tests/wpt/issues/12949")
 def test_readlines_file_bigger_than_buffer():
     old_max_buf = InputFile.max_buffer_size
@@ -139,7 +141,7 @@ def test_iter():
         assert a == b
 
 
-@pytest.mark.xfail(sys.platform == "win32",
+@pytest.mark.xfail(PY2 and sys.platform == "win32",
                    reason="https://github.com/web-platform-tests/wpt/issues/12949")
 def test_iter_file_bigger_than_buffer():
     old_max_buf = InputFile.max_buffer_size

tools/wptserve/tests/functional/test_pipes.py

@@ -62,8 +62,6 @@ def test_sub_config(self):
         expected = b"localhost localhost %i" % self.server.port
         self.assertEqual(resp.read().rstrip(), expected)
 
-    @pytest.mark.xfail(sys.platform == "win32",
-                       reason="https://github.com/web-platform-tests/wpt/issues/12949")
     def test_sub_file_hash(self):
         resp = self.request("/sub_file_hash.sub.txt")
         expected = b"""
@@ -84,8 +82,6 @@ def test_sub_headers(self):
         expected = b"PASS"
         self.assertEqual(resp.read().rstrip(), expected)
 
-    @pytest.mark.xfail(sys.platform == "win32",
-                       reason="https://github.com/web-platform-tests/wpt/issues/12949")
     def test_sub_location(self):
         resp = self.request("/sub_location.sub.txt?query_string")
         expected = """

tools/wptserve/wptserve/sslutils/openssl.py

@@ -6,14 +6,25 @@
 import tempfile
 from datetime import datetime, timedelta
 
-from six import iteritems
+from six import iteritems, PY2
 
 # Amount of time beyond the present to consider certificates "expired." This
 # allows certificates to be proactively re-generated in the "buffer" period
 # prior to their exact expiration time.
 CERT_EXPIRY_BUFFER = dict(hours=6)
 
 
+def _ensure_str(s, encoding):
+    """makes sure s is an instance of str, converting with encoding if needed"""
+    if isinstance(s, str):
+        return s
+
+    if PY2:
+        return s.encode(encoding)
+    else:
+        return s.decode(encoding)
+
+
 class OpenSSL(object):
     def __init__(self, logger, binary, base_path, conf_path, hosts, duration,
                  base_conf_path=None):
@@ -65,18 +76,14 @@ def __call__(self, cmd, *args, **kwargs):
             self.cmd += ["-config", self.conf_path]
         self.cmd += list(args)
 
-        # Copy the environment, converting to plain strings. Windows
-        # StartProcess is picky about all the keys/values being plain strings,
-        # but at least in MSYS shells, the os.environ dictionary can be mixed.
+        # Copy the environment, converting to plain strings. Win32 StartProcess
+        # is picky about all the keys/values being str (on both Py2/3).
         env = {}
         for k, v in iteritems(os.environ):
-            try:
-                env[k.encode("utf8")] = v.encode("utf8")
-            except UnicodeDecodeError:
-                pass
+            env[_ensure_str(k, "utf8")] = _ensure_str(v, "utf8")
 
         if self.base_conf_path is not None:
-            env["OPENSSL_CONF"] = self.base_conf_path.encode("utf8")
+            env["OPENSSL_CONF"] = _ensure_str(self.base_conf_path, "utf-8")
 
         self.proc = subprocess.Popen(self.cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
                                      env=env)