Validate pull requests in TaskCluster

.taskcluster.yml

@@ -2,81 +2,155 @@ version: 1
 policy:
   pullRequests: collaborators
 tasks:
-  $if: tasks_for == "github-push"
-  then:
-    $flattenDeep:
-      $map:
-        $flatten:
-          $match: {
-            event.ref == "refs/heads/master": [{name: firefox, channel: nightly}, {name: chrome, channel: dev}],
-            event.ref == "refs/heads/epochs/daily": [{name: firefox, channel: beta}, {name: chrome, channel: beta}],
-            event.ref == "refs/heads/epochs/weekly": [{name: firefox, channel: stable}, {name: chrome, channel: stable}]
-            }
-      each(browser):
+  $flattenDeep:
+    - $if: tasks_for == "github-push"
+      then:
         $map:
-          - [testharness, 1, 15]
-          - [testharness, 2, 15]
-          - [testharness, 3, 15]
-          - [testharness, 4, 15]
-          - [testharness, 5, 15]
-          - [testharness, 6, 15]
-          - [testharness, 7, 15]
-          - [testharness, 8, 15]
-          - [testharness, 9, 15]
-          - [testharness, 10, 15]
-          - [testharness, 11, 15]
-          - [testharness, 12, 15]
-          - [testharness, 13, 15]
-          - [testharness, 14, 15]
-          - [testharness, 15, 15]
-          - [reftest, 1, 10]
-          - [reftest, 2, 10]
-          - [reftest, 3, 10]
-          - [reftest, 4, 10]
-          - [reftest, 5, 10]
-          - [reftest, 6, 10]
-          - [reftest, 7, 10]
-          - [reftest, 8, 10]
-          - [reftest, 9, 10]
-          - [reftest, 10, 10]
-          - [wdspec, 1, 1]
-        each(chunk):
-          taskId: {$eval: 'as_slugid(browser.name + browser.channel + chunk[0] + str(chunk[1]))'}
-          taskGroupId: {$eval: 'as_slugid("task group")'}
-          created: {$fromNow: ''}
-          deadline: {$fromNow: '24 hours'}
-          provisionerId: aws-provisioner-v1
-          workerType:
-            $if: event.repository.full_name == 'web-platform-tests/wpt'
-            then:
-              wpt-docker-worker
-            else:
-              github-worker
-          metadata:
-            name: wpt-${browser.name}-${browser.channel}-${chunk[0]}-${chunk[1]}
-            description: >-
-              A subset of WPT's "${chunk[0]}" tests (chunk number ${chunk[1]}
-              of ${chunk[2]}), run in the ${browser.channel} release of
-              ${browser.name}.
-            owner: ${event.pusher.email}
-            source: ${event.repository.url}
-          payload:
-            image: harjgam/web-platform-tests:0.14
-            maxRunTime: 7200
-            artifacts:
-              public/results:
-                path: /home/test/artifacts
-                type: directory
-            command:
-              - /bin/bash
-              - --login
-              - -c
-              - "~/start.sh ${event.repository.url} ${event.ref[len('refs/heads/'):]} ${event.after} ${browser.name} ${browser.channel} &&
-              cd ~/web-platform-tests &&
-              ./tools/ci/ci_taskcluster.sh ${browser.name} ${browser.channel} ${chunk[0]} ${chunk[1]} ${chunk[2]}"
-  # > NOTE: A well-designed template should produce `tasks: []` for any
-  # > unrecognized `task_for` values; this allows later expansion of this
-  # > service to handle more events.
-  #
-  # https://docs.taskcluster.net/docs/reference/integrations/taskcluster-github/docs/taskcluster-yml-v1
-  else: []
+          $flatten:
+            $match: {
+              event.ref == "refs/heads/master": [{name: firefox, channel: nightly}, {name: chrome, channel: dev}],
+              event.ref == "refs/heads/epochs/daily": [{name: firefox, channel: beta}, {name: chrome, channel: beta}],
+              event.ref == "refs/heads/epochs/weekly": [{name: firefox, channel: stable}, {name: chrome, channel: stable}]
+              }
+        each(browser):
+          $map:
+            - [testharness, 1, 15]
+            - [testharness, 2, 15]
+            - [testharness, 3, 15]
+            - [testharness, 4, 15]
+            - [testharness, 5, 15]
+            - [testharness, 6, 15]
+            - [testharness, 7, 15]
+            - [testharness, 8, 15]
+            - [testharness, 9, 15]
+            - [testharness, 10, 15]
+            - [testharness, 11, 15]
+            - [testharness, 12, 15]
+            - [testharness, 13, 15]
+            - [testharness, 14, 15]
+            - [testharness, 15, 15]
+            - [reftest, 1, 10]
+            - [reftest, 2, 10]
+            - [reftest, 3, 10]
+            - [reftest, 4, 10]
+            - [reftest, 5, 10]
+            - [reftest, 6, 10]
+            - [reftest, 7, 10]
+            - [reftest, 8, 10]
+            - [reftest, 9, 10]
+            - [reftest, 10, 10]
+            - [wdspec, 1, 1]
+          each(chunk):
+            taskId: {$eval: 'as_slugid(browser.name + browser.channel + chunk[0] + str(chunk[1]))'}
+            taskGroupId: {$eval: 'as_slugid("task group")'}
+            created: {$fromNow: ''}
+            deadline: {$fromNow: '24 hours'}
+            provisionerId: aws-provisioner-v1
+            workerType:
+              $if: event.repository.full_name == 'web-platform-tests/wpt'
+              then:
+                wpt-docker-worker
+              else:
+                github-worker
+            metadata:
+              name: wpt-${browser.name}-${browser.channel}-${chunk[0]}-${chunk[1]}
+              description: >-
+                A subset of WPT's "${chunk[0]}" tests (chunk number ${chunk[1]}
+                of ${chunk[2]}), run in the ${browser.channel} release of
+                ${browser.name}.
+              owner: ${event.pusher.email}
+              source: ${event.repository.url}
+            payload:
+              image: jugglinmike/web-platform-tests:0.21
+              maxRunTime: 7200
+              artifacts:
+                public/results:
+                  path: /home/test/artifacts
+                  type: directory
+              command:
+                - /bin/bash
+                - --login
+                - -c
+                - set -ex;
+                  ~/start.sh
+                    ${event.repository.url}
+                    ${event.ref}
+                    ${event.after}
+                    ${browser.name}
+                    ${browser.channel};
+                  cd ~/web-platform-tests;
+                  ./tools/ci/taskcluster-run.py
+                    ${browser.name}
+                    --
+                    --channel=${browser.channel}
+                    --log-wptreport=../artifacts/wpt_report.json
+                    --no-fail-on-unexpected
+                    --test-type=${chunk[0]}
+                    --this-chunk=${chunk[1]}
+                    --total-chunks=${chunk[2]};
+    - $if: tasks_for == "github-pull-request"
+      then:
+        $map: [{name: firefox, channel: nightly}, {name: chrome, channel: dev}]
+        each(browser):
+          $map:
+            - name: wpt-${browser.name}-${browser.channel}-stability
+              description: >-
+                Verify that all tests affected by a pull request are stable
+                when executed in ${browser.name}.
+              extra_args: '--verify'
+            - name: wpt-${browser.name}-${browser.channel}-results
+              description: >-
+                Collect results for all tests affected by a pull request in
+                ${browser.name}.
+              extra_args: '--no-fail-on-unexpected --log-wptreport=../artifacts/wpt_report.json'
+          each(operation):
+            taskId: {$eval: 'as_slugid(operation.name)'}
+            taskGroupId: {$eval: 'as_slugid("task group")'}
+            created: {$fromNow: ''}
+            deadline: {$fromNow: '24 hours'}
+            provisionerId: aws-provisioner-v1
+            workerType:
+              $if: event.repository.full_name == 'web-platform-tests/wpt'
+              then:
+                wpt-docker-worker
+              else:
+                github-worker
+            metadata:
+              name: ${operation.name}
+              description: ${operation.description}
+              owner: ${event.pull_request.user.login}@users.noreply.github.com
+              source: ${event.repository.url}
+            payload:
+              image: jugglinmike/web-platform-tests:0.21
+              maxRunTime: 7200
+              artifacts:
+                public/results:
+                  path: /home/test/artifacts
+                  type: directory
+              # Fetch the GitHub-provided merge commit (rather than the pull
+              # request branch) so that the tasks simulate the behavior of the
+              # submitted patch after it is merged. Using the merge commit also
+              # simplifies detection of modified files because the first parent
+              # of the merge commit can consistently be used to summarize the
+              # changes.
+              command:
+                - /bin/bash
+                - --login
+                - -c
+                - set -ex;
+                  ~/start.sh
+                    ${event.repository.clone_url}
+                    refs/pull/${event.number}/merge
+                    FETCH_HEAD
+                    ${browser.name}
+                    ${browser.channel};
+                  cd ~/web-platform-tests;
+                  result=0;
+                  ./tools/ci/taskcluster-run.py
+                    --commit-range HEAD^
+                    ${browser.name}
+                    --
+                    --channel=${browser.channel}
+                    ${operation.extra_args} || result=$?;
+                  echo $result > ../artifacts/run-return-code.txt;
+                  echo "Command exited with code $result (failures are allowed while this task is being vetted)."

dom/events/CustomEvent.html

@@ -4,6 +4,10 @@
 <script src="/resources/testharnessreport.js"></script>
 <div id="log"></div>
 <script>
+test(function() {
+  assert_true(Math.random() > 0.5);
+}, 'intentionally unstable test');
+
 test(function() {
   var type = "foo";
 

tools/ci/taskcluster-run.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python
+
+import argparse
+import gzip
+import logging
+import os
+import shutil
+import subprocess
+
+browser_specific_args = {
+    "firefox": ["--install-browser"]
+}
+
+def tests_affected(commit_range):
+    output = subprocess.check_output([
+        "python", "./wpt", "tests-affected", "--null", commit_range
+    ], stderr=open(os.devnull, "w"))
+
+    tests = output.split("\0")
+
+    # Account for trailing null byte
+    if tests and not tests[-1]:
+        tests.pop()
+
+    return tests
+
+
+def find_wptreport(args):
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--log-wptreport', action='store')
+    return parser.parse_known_args(args)[0].log_wptreport
+
+
+def gzip_file(filename):
+    with open(filename, 'rb') as f_in:
+        with gzip.open('%s.gz' % filename, 'wb') as f_out:
+            shutil.copyfileobj(f_in, f_out)
+
+
+def main(product, commit_range, wpt_args):
+    """Invoke the `wpt run` command according to the needs of the TaskCluster
+    continuous integration service."""
+
+    logger = logging.getLogger("tc-run")
+    logger.setLevel(logging.INFO)
+    handler = logging.StreamHandler()
+    handler.setFormatter(
+        logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+    )
+    logger.addHandler(handler)
+
+    child = subprocess.Popen(['python', './wpt', 'manifest-download'])
+    child.wait()
+
+    if commit_range:
+        logger.info(
+            "Identifying tests affected in range '%s'..." % commit_range
+        )
+        tests = tests_affected(commit_range)
+        logger.info("Identified %s affected tests" % len(tests))
+
+        if not tests:
+            logger.info("Quitting because no tests were affected.")
+            return
+    else:
+        tests = []
+        logger.info("Running all tests")
+
+    wpt_args += [
+        "--log-tbpl=../artifacts/log_tbpl.log",
+        "--log-tbpl-level=info",
+        "--log-mach=-",
+        "-y",
+        "--no-pause",
+        "--no-restart-on-unexpected",
+        "--install-fonts"
+    ]
+    wpt_args += browser_specific_args.get(product, [])
+
+    command = ["python", "./wpt", "run"] + wpt_args + [product] + tests
+
+    logger.info("Executing command: %s" % " ".join(command))
+
+    subprocess.check_call(command)
+
+    wptreport = find_wptreport(wpt_args)
+    if wptreport:
+        gzip_file(wptreport)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=main.__doc__)
+    parser.add_argument("--commit-range", action="store",
+                        help="""Git commit range. If specified, this will be
+                             supplied to the `wpt tests-affected` command to
+                             determine the list of test to execute""")
+    parser.add_argument("product", action="store",
+                        help="Browser to run tests in")
+    parser.add_argument("wpt_args", nargs="*",
+                        help="Arguments to forward to `wpt run` command")
+    main(**vars(parser.parse_args()))

tools/docker/start.sh

@@ -12,24 +12,29 @@
 set -ex
 
 REMOTE=${1:-https://github.com/web-platform-tests/wpt}
-BRANCH=${2:-master}
-REV=${3:-FETCH_HEAD}
+REF=${2:-master}
+REVISION=${3:-FETCH_HEAD}
 BROWSER=${4:-all}
 CHANNEL=${5:-nightly}
 
 cd ~
 
-# Initially we just fetch 50 commits in order to save several minutes of fetching
-git clone ${REMOTE} --single-branch --branch ${BRANCH} --no-checkout -q --depth=50 web-platform-tests
+mkdir web-platform-tests
 cd web-platform-tests
-if [[ ! `git rev-parse --verify -q ${REV}` ]];
+git init
+git remote add origin ${REMOTE}
+
+# Initially we just fetch 50 commits in order to save several minutes of fetching
+git fetch --quiet --depth=50 origin ${REF}
+
+if [[ ! `git rev-parse --verify -q ${REVISION}` ]];
 then
     # But if for some reason the commit under test isn't in that range, we give in and
     # fetch everything
     git fetch -q --unshallow ${REMOTE}
-    git rev-parse --verify ${REV}
+    git rev-parse --verify ${REVISION}
 fi
-git checkout -b build ${REV}
+git checkout -b build ${REVISION}
 
 sudo sh -c './wpt make-hosts-file >> /etc/hosts'