diff --git a/.appveyor.yml b/.appveyor.yml new file mode 100644 index 00000000000000..e6b499f7324b28 --- /dev/null +++ b/.appveyor.yml @@ -0,0 +1,44 @@ +skip_branch_with_pr: true + +clone_depth: 50 + +environment: + matrix: + - PYTHON: "C:\\Python27" + JOB: "tools_unittest" + SCRIPT: "tools\\ci\\ci_tools_unittest.ps1" + TOX_DIR: "tools" + TOXENV: "py27" + - PYTHON: "C:\\Python27" + JOB: "tools_unittest" + SCRIPT: "tools\\ci\\ci_tools_unittest.ps1" + TOX_DIR: "tools\\wptrunner" + TOXENV: "py27" + - PYTHON: "C:\\Python27" + JOB: "wpt_integration" + SCRIPT: "tools\\ci\\ci_tools_unittest.ps1" + TOX_DIR: "tools\\wpt" + TOXENV: "py27" + - PYTHON: "C:\\Python27" + JOB: "wptrunner_infrastructure" + SCRIPT: "tools\\ci\\ci_wptrunner_infrastructure.ps1" + - PYTHON: "C:\\Python36" + JOB: "tools_unittest" + SCRIPT: "tools\\ci\\ci_tools_unittest.ps1" + TOX_DIR: "tools" + TOXENV: "py36" + +only_commits: + files: + - "tools\\**\\*" + +install: + - ps: "tools\\ci\\install.ps1" + +build: off + +test_script: + - ps: ". $env:SCRIPT" + +cache: + - MANIFEST.json diff --git a/tools/ci/ci_tools_unittest.ps1 b/tools/ci/ci_tools_unittest.ps1 new file mode 100644 index 00000000000000..320cbd71c126b5 --- /dev/null +++ b/tools/ci/ci_tools_unittest.ps1 @@ -0,0 +1,11 @@ +Set-PSDebug -Trace 1 + +. "$PSScriptRoot\lib.ps1" + +$WPT_ROOT = "$PSScriptRoot\..\.." +cd $WPT_ROOT + +cd $env:TOX_DIR +pip install -U tox +run_applicable_tox + diff --git a/tools/ci/ci_wptrunner_infrastructure.ps1 b/tools/ci/ci_wptrunner_infrastructure.ps1 new file mode 100644 index 00000000000000..a053a9325f8116 --- /dev/null +++ b/tools/ci/ci_wptrunner_infrastructure.ps1 @@ -0,0 +1,19 @@ +Set-PSDebug -Trace 1 + +. "$PSScriptRoot\lib.ps1" + +$WPT_ROOT = "$PSScriptRoot\..\.." +cd $WPT_ROOT + +$products = @("firefox", "chrome") +foreach ($product in $products) { + $args = "" + if (!$product.Equals("firefox")) { + # Firefox is expected to work using pref settings for DNS + # Don't adjust the hostnames in that case to ensure this keeps working + hosts_fixup + } else { + $args += " --install-browser" + } + python ./wpt run --yes --metadata infrastructure/metadata/ --install-fonts $args $product infrastructure/ +} diff --git a/tools/ci/install.ps1 b/tools/ci/install.ps1 new file mode 100644 index 00000000000000..e9352e2f7feeab --- /dev/null +++ b/tools/ci/install.ps1 @@ -0,0 +1,21 @@ +Set-PSDebug -Trace 1 + +git fetch -q --depth=50 origin master + +$env:PATH = "$env:PYTHON;$env:PATH" + +$RELEVANT_JOBS = $(python ./wpt test-jobs).split( + [string[]]$null, + [System.StringSplitOptions]::RemoveEmptyEntries +) + +if ($env:RUN_JOB -or $RELEVANT_JOBS.contains($env:JOB)) { + $env:RUN_JOB = $true +} else { + $env:RUN_JOB = $false +} + +if ($env:RUN_JOB) { + pip install -U setuptools + pip install -U requests +} diff --git a/tools/ci/lib.ps1 b/tools/ci/lib.ps1 new file mode 100644 index 00000000000000..b0a89f42cb82c2 --- /dev/null +++ b/tools/ci/lib.ps1 @@ -0,0 +1,25 @@ +function hosts_fixup { + $path="$env:SystemRoot\System32\drivers\etc\hosts" + Get-Content $path + python wpt make-hosts-file | Out-File $path -Encoding ascii -Append + Get-Content $path +} + +function run_applicable_tox { + # instead of just running TOXENV (e.g., py27) + # run all environments that start with TOXENV + # (e.g., py27-firefox as well as py27) + $OLD_TOXENV=$env:TOXENV + Remove-Item env:TOXENV + $RUN_ENVS = $(tox -l).split( + [string[]]$null, + [System.StringSplitOptions]::RemoveEmptyEntries + ).Where({$_.StartsWith("$OLD_TOXENV-") -or $_.Equals($OLD_TOXENV)}) -join "," + if ($RUN_ENVS) { + tox -e "$RUN_ENVS" + } + if ($LastExitCode -ne 0) { + throw + } + $env:TOXENV = $OLD_TOXENV +} diff --git a/tools/wpt/testfiles.py b/tools/wpt/testfiles.py index 35a4b97d7ffaa2..f9e2f425aac3ce 100644 --- a/tools/wpt/testfiles.py +++ b/tools/wpt/testfiles.py @@ -57,6 +57,10 @@ def branch_point(): not_heads = [item for item in git("rev-parse", "--not", "--branches", "--remotes").split("\n") if item != "^%s" % head] + # if we don't have any other branches, just return the HEAD commit + if not not_heads: + return head + # get all commits on HEAD but not reachable from anything in not_heads commits = git("rev-list", "--topo-order", "--parents", "HEAD", *not_heads) commit_parents = OrderedDict() diff --git a/tools/wpt/tests/test_testfiles.py b/tools/wpt/tests/test_testfiles.py new file mode 100644 index 00000000000000..d8abbf6b1864c1 --- /dev/null +++ b/tools/wpt/tests/test_testfiles.py @@ -0,0 +1,33 @@ +import mock +import pytest + +from tools.wpt import testfiles + +def get_get_git_cmd(rv_dict): + def get_git_cmd(root): + def git(*args): + return rv_dict[args] + return git + return get_git_cmd + + +@pytest.mark.parametrize("environ,git_rv_dict,expected_rv", [ + ({"TRAVIS_PULL_REQUEST": "false", "TRAVIS_BRANCH": "master"}, + {("rev-parse", "HEAD"): "1a" * 20}, + "1a" * 20), + ({"TRAVIS_PULL_REQUEST": "true", "TRAVIS_BRANCH": "foobar"}, + {("merge-base", "HEAD", "foobar"): "1a" * 20}, + "1a" * 20), + ({}, + {("rev-parse", "HEAD"): "78978db3956ad3137784600bba20c8dcee6b638b", + ("rev-parse", "--not", "--branches", "--remotes"): "^78978db3956ad3137784600bba20c8dcee6b638b", + ("rev-list", "--topo-order", "--parents", "HEAD"): + "78978db3956ad3137784600bba20c8dcee6b638b " + "a4d5a787c59ca4176e70d665b5e335380077a29c\na4d5a787c59ca4176e70d665b5e335380077a29c"}, + "78978db3956ad3137784600bba20c8dcee6b638b"), +]) +def test_branch_point(environ, git_rv_dict, expected_rv): + with mock.patch.object(testfiles, "get_git_cmd", get_get_git_cmd(git_rv_dict)): + with mock.patch.object(testfiles.os, "environ", environ): + rv = testfiles.branch_point() + assert expected_rv == rv diff --git a/tools/wptserve/tests/functional/test_input_file.py b/tools/wptserve/tests/functional/test_input_file.py index b3f0fba48abad1..215ac8e910ec84 100644 --- a/tools/wptserve/tests/functional/test_input_file.py +++ b/tools/wptserve/tests/functional/test_input_file.py @@ -3,6 +3,8 @@ import pytest +from six import PY2 + from wptserve.request import InputFile bstr = b'This is a test document\nWith new lines\nSeveral in fact...' @@ -120,7 +122,7 @@ def test_readlines(): assert input_file.readlines() == test_file.readlines() -@pytest.mark.xfail(sys.platform == "win32", +@pytest.mark.xfail(PY2 and sys.platform == "win32", reason="https://github.com/web-platform-tests/wpt/issues/12949") def test_readlines_file_bigger_than_buffer(): old_max_buf = InputFile.max_buffer_size @@ -139,7 +141,7 @@ def test_iter(): assert a == b -@pytest.mark.xfail(sys.platform == "win32", +@pytest.mark.xfail(PY2 and sys.platform == "win32", reason="https://github.com/web-platform-tests/wpt/issues/12949") def test_iter_file_bigger_than_buffer(): old_max_buf = InputFile.max_buffer_size diff --git a/tools/wptserve/tests/functional/test_pipes.py b/tools/wptserve/tests/functional/test_pipes.py index fdac4537d64fb5..937a5866f79f70 100644 --- a/tools/wptserve/tests/functional/test_pipes.py +++ b/tools/wptserve/tests/functional/test_pipes.py @@ -62,8 +62,6 @@ def test_sub_config(self): expected = b"localhost localhost %i" % self.server.port self.assertEqual(resp.read().rstrip(), expected) - @pytest.mark.xfail(sys.platform == "win32", - reason="https://github.com/web-platform-tests/wpt/issues/12949") def test_sub_file_hash(self): resp = self.request("/sub_file_hash.sub.txt") expected = b""" @@ -84,8 +82,6 @@ def test_sub_headers(self): expected = b"PASS" self.assertEqual(resp.read().rstrip(), expected) - @pytest.mark.xfail(sys.platform == "win32", - reason="https://github.com/web-platform-tests/wpt/issues/12949") def test_sub_location(self): resp = self.request("/sub_location.sub.txt?query_string") expected = """ diff --git a/tools/wptserve/wptserve/sslutils/openssl.py b/tools/wptserve/wptserve/sslutils/openssl.py index ebbcf68d716d1d..bdabc4fffb4c64 100644 --- a/tools/wptserve/wptserve/sslutils/openssl.py +++ b/tools/wptserve/wptserve/sslutils/openssl.py @@ -6,7 +6,7 @@ import tempfile from datetime import datetime, timedelta -from six import iteritems +from six import iteritems, PY2 # Amount of time beyond the present to consider certificates "expired." This # allows certificates to be proactively re-generated in the "buffer" period @@ -14,6 +14,17 @@ CERT_EXPIRY_BUFFER = dict(hours=6) +def _ensure_str(s, encoding): + """makes sure s is an instance of str, converting with encoding if needed""" + if isinstance(s, str): + return s + + if PY2: + return s.encode(encoding) + else: + return s.decode(encoding) + + class OpenSSL(object): def __init__(self, logger, binary, base_path, conf_path, hosts, duration, base_conf_path=None): @@ -65,18 +76,14 @@ def __call__(self, cmd, *args, **kwargs): self.cmd += ["-config", self.conf_path] self.cmd += list(args) - # Copy the environment, converting to plain strings. Windows - # StartProcess is picky about all the keys/values being plain strings, - # but at least in MSYS shells, the os.environ dictionary can be mixed. + # Copy the environment, converting to plain strings. Win32 StartProcess + # is picky about all the keys/values being str (on both Py2/3). env = {} for k, v in iteritems(os.environ): - try: - env[k.encode("utf8")] = v.encode("utf8") - except UnicodeDecodeError: - pass + env[_ensure_str(k, "utf8")] = _ensure_str(v, "utf8") if self.base_conf_path is not None: - env["OPENSSL_CONF"] = self.base_conf_path.encode("utf8") + env["OPENSSL_CONF"] = _ensure_str(self.base_conf_path, "utf-8") self.proc = subprocess.Popen(self.cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env)