From 541acf33e94a7fee8dc48cbcca4f067ed866a8d7 Mon Sep 17 00:00:00 2001
From: Mike Pennisi <mike@mikepennisi.com>
Date: Tue, 7 Aug 2018 17:33:19 -0400
Subject: [PATCH 01/18] [infra] Validate pull requests in TaskCluster

---
 .taskcluster.yml           | 187 ++++++++++++++++++++++++++-----------
 tools/ci/ci_taskcluster.sh |  44 ++++++++-
 tools/docker/Dockerfile    |   6 ++
 3 files changed, 179 insertions(+), 58 deletions(-)

diff --git a/.taskcluster.yml b/.taskcluster.yml
index 70bde33c354105..71267779ca411b 100644
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -2,42 +2,106 @@ version: 1
 policy:
   pullRequests: collaborators
 tasks:
-  $if: tasks_for == "github-push"
-  then:
-    $if: event.ref == "refs/heads/master"
-    then:
-      $flattenDeep:
+  $flattenDeep:
+    - $if: tasks_for == "github-push"
+      then:
+        $if: event.ref == "refs/heads/master"
+        then:
+          $map: [{name: firefox, channel: nightly}, {name: chrome, channel: dev}]
+          each(browser):
+            $map:
+              - [testharness, 1, 15]
+              - [testharness, 2, 15]
+              - [testharness, 3, 15]
+              - [testharness, 4, 15]
+              - [testharness, 5, 15]
+              - [testharness, 6, 15]
+              - [testharness, 7, 15]
+              - [testharness, 8, 15]
+              - [testharness, 9, 15]
+              - [testharness, 10, 15]
+              - [testharness, 11, 15]
+              - [testharness, 12, 15]
+              - [testharness, 13, 15]
+              - [testharness, 14, 15]
+              - [testharness, 15, 15]
+              - [reftest, 1, 10]
+              - [reftest, 2, 10]
+              - [reftest, 3, 10]
+              - [reftest, 4, 10]
+              - [reftest, 5, 10]
+              - [reftest, 6, 10]
+              - [reftest, 7, 10]
+              - [reftest, 8, 10]
+              - [reftest, 9, 10]
+              - [reftest, 10, 10]
+              - [wdspec, 1, 1]
+            each(chunk):
+              taskId: {$eval: 'as_slugid(browser.name + browser.channel + chunk[0] + str(chunk[1]))'}
+              taskGroupId: {$eval: 'as_slugid("task group")'}
+              created: {$fromNow: ''}
+              deadline: {$fromNow: '24 hours'}
+              provisionerId: aws-provisioner-v1
+              # Contributors interested in configurating TaskCluster to run
+              # against their fork of WPT should change the `workerType` to
+              # "github-worker".
+              workerType: wpt-docker-worker
+              metadata:
+                name: wpt-${browser.name}-${browser.channel}-${chunk[0]}-${chunk[1]}
+                description: >-
+                  A subset of WPT's "${chunk[0]}" tests (chunk number ${chunk[1]}
+                  of ${chunk[2]}), run in the ${browser.channel} release of
+                  ${browser.name}.
+                owner: ${event.pusher.email}
+                source: ${event.repository.url}
+              payload:
+                image: jugglinmike/web-platform-tests:0.18
+                maxRunTime: 7200
+                artifacts:
+                  public/results:
+                    path: /home/test/artifacts
+                    type: directory
+                command:
+                  - /bin/bash
+                  - --login
+                  - -c
+                  - set -ex;
+                    ~/start.sh
+                      ${event.repository.url}
+                      ${event.ref[len('refs/heads/'):]}
+                      ${event.after}
+                      ${browser.name}-${browser.channel};
+                    cd ~/web-platform-tests;
+                    ./tools/ci/ci_taskcluster.sh
+                      --test-type=${chunk[0]}
+                      --this-chunk=${chunk[1]}
+                      --total-chunks=${chunk[2]}
+                      ${browser.name};
+    - $if: tasks_for == "github-pull-request"
+      then:
         $map: [{name: firefox, channel: nightly}, {name: chrome, channel: dev}]
         each(browser):
           $map:
-            - [testharness, 1, 15]
-            - [testharness, 2, 15]
-            - [testharness, 3, 15]
-            - [testharness, 4, 15]
-            - [testharness, 5, 15]
-            - [testharness, 6, 15]
-            - [testharness, 7, 15]
-            - [testharness, 8, 15]
-            - [testharness, 9, 15]
-            - [testharness, 10, 15]
-            - [testharness, 11, 15]
-            - [testharness, 12, 15]
-            - [testharness, 13, 15]
-            - [testharness, 14, 15]
-            - [testharness, 15, 15]
-            - [reftest, 1, 10]
-            - [reftest, 2, 10]
-            - [reftest, 3, 10]
-            - [reftest, 4, 10]
-            - [reftest, 5, 10]
-            - [reftest, 6, 10]
-            - [reftest, 7, 10]
-            - [reftest, 8, 10]
-            - [reftest, 9, 10]
-            - [reftest, 10, 10]
-            - [wdspec, 1, 1]
-          each(chunk):
-            taskId: {$eval: 'as_slugid(browser.name + browser.channel + chunk[0] + str(chunk[1]))'}
+            - name: wpt-${browser.name}-${browser.channel}-stability
+              description: >-
+                Verify that all tests affected by a pull request are stable
+                when executed in ${browser.name}.
+              command:
+                echo $affected |
+                  base64 -d |
+                  xargs -0 ./tools/ci/ci_taskcluster.sh --verify ${browser.name}
+              creates_artifacts: false
+            - name: wpt-${browser.name}-${browser.channel}-results
+              description: >-
+                Collect results for all tests affected by a pull request in
+                ${browser.name}.
+              command:
+                echo $affected |
+                  base64 -d |
+                  xargs -0 ./tools/ci/ci_taskcluster.sh ${browser.name}
+              creates_artifacts: true
+          each(operation):
+            taskId: {$eval: 'as_slugid(operation.name)'}
             taskGroupId: {$eval: 'as_slugid("task group")'}
             created: {$fromNow: ''}
             deadline: {$fromNow: '24 hours'}
@@ -47,31 +111,48 @@ tasks:
             # "github-worker".
             workerType: wpt-docker-worker
             metadata:
-              name: wpt-${browser.name}-${browser.channel}-${chunk[0]}-${chunk[1]}
-              description: >-
-                A subset of WPT's "${chunk[0]}" tests (chunk number ${chunk[1]}
-                of ${chunk[2]}), run in the ${browser.channel} release of
-                ${browser.name}.
-              owner: ${event.pusher.email}
+              name: ${operation.name}
+              description: ${operation.description}
+              owner: ${event.pull_request.user.login}@users.noreply.github.com
               source: ${event.repository.url}
             payload:
-              image: gsnedders/web-platform-tests:0.13
+              image: jugglinmike/web-platform-tests:0.18
               maxRunTime: 7200
               artifacts:
-                public/results:
-                  path: /home/test/artifacts
-                  type: directory
+                $if: operation.creates_artifacts
+                then:
+                  public/results:
+                    path: /home/test/artifacts
+                    type: directory
+              # Bash removes null bytes from string values when set as
+              # environment variables. This invalidates the output of `wpt
+              # affected-tests` because it uses the null byte as a separator
+              # between test names. The list of effected tests is
+              # Base64-encoded in order to preserve the null bytes.
               command:
                 - /bin/bash
                 - --login
                 - -c
-                - "~/start.sh ${event.repository.url} ${event.ref[len('refs/heads/'):]} ${event.after} ${browser.name}-${browser.channel} &&
-                  cd ~/web-platform-tests &&
-                  ./tools/ci/ci_taskcluster.sh ${browser.name} ${chunk[0]} ${chunk[1]} ${chunk[2]}"
-    else: []
-  # > NOTE: A well-designed template should produce `tasks: []` for any
-  # > unrecognized `task_for` values; this allows later expansion of this
-  # > service to handle more events.
-  #
-  # https://docs.taskcluster.net/docs/reference/integrations/taskcluster-github/docs/taskcluster-yml-v1
-  else: []
+                - set -ex;
+                  ~/start.sh
+                    ${event.repository.clone_url}
+                    ${event.pull_request.base.ref}
+                    ${event.pull_request.base.sha}
+                    ${browser.name}-${browser.channel};
+                  cd ~/web-platform-tests;
+                  ./wpt manifest-download || true;
+                  git fetch
+                    ${event.repository.clone_url}
+                    refs/pull/${event.number}/head;
+                  git checkout FETCH_HEAD;
+                  affected=$(./wpt tests-affected
+                    --null
+                    ${event.pull_request.base.sha}...
+                    | base64 --wrap 0);
+                  if [ -z $(echo $affected | base64 -d) ]; then
+                    echo Zero tests affected.;
+                    exit 0;
+                  fi;
+                  echo Affected tests:;
+                  echo $affected | base64 -d | sed 's/\x0/\n/g';
+                  ${operation.command}
diff --git a/tools/ci/ci_taskcluster.sh b/tools/ci/ci_taskcluster.sh
index 118243881a938b..0db8b9db31741f 100755
--- a/tools/ci/ci_taskcluster.sh
+++ b/tools/ci/ci_taskcluster.sh
@@ -2,9 +2,43 @@
 
 ./wpt manifest-download
 
-if [ $1 == "firefox" ]; then
-    ./wpt run firefox --log-tbpl=../artifacts/log_tbpl.log --log-tbpl-level=info --log-wptreport=../artifacts/wpt_report.json --log-mach=- --this-chunk=$3 --total-chunks=$4 --test-type=$2 -y --install-browser --no-pause --no-restart-on-unexpected --reftest-internal --install-fonts --no-fail-on-unexpected
-elif [ $1 == "chrome" ]; then
-    ./wpt run chrome --log-tbpl=../artifacts/log_tbpl.log --log-tbpl-level=info --log-wptreport=../artifacts/wpt_report.json --log-mach=- --this-chunk=$3 --total-chunks=$4 --test-type=$2 -y  --no-pause --no-restart-on-unexpected --install-fonts --no-fail-on-unexpected
+# The first script argument that is not prefixed with a dash (`-`) is assumed
+# to be the name of the browser under test. This restricts the syntax available
+# to consumers: value-accepting options must be specified using the equals sign
+# (`=`).
+for argument in $@; do
+  if [ ${argument:0:1} = '-' ]; then
+    continue
+  fi
+
+  browser_name=$argument
+
+  break
+done
+
+browser_specific_args=''
+
+if [ $browser_name = 'firefox' ]; then
+  browser_specific_args='--install-browser --reftest-internal'
 fi
-gzip ../artifacts/wpt_report.json
+
+./wpt run \
+  --log-tbpl=../artifacts/log_tbpl.log \
+  --log-tbpl-level=info \
+  --log-wptreport=../artifacts/wpt_report.json \
+  --log-mach=- \
+  -y \
+  --no-pause \
+  --no-restart-on-unexpected \
+  --install-fonts \
+  --no-fail-on-unexpected \
+  $browser_specific_args \
+  $@
+
+run_status=$?
+
+if [ -f ../artifacts/wpt_report.json ]; then
+  gzip ../artifacts/wpt_report.json
+fi
+
+exit $run_status
diff --git a/tools/docker/Dockerfile b/tools/docker/Dockerfile
index b0541dc6c98337..2362ef39623600 100644
--- a/tools/docker/Dockerfile
+++ b/tools/docker/Dockerfile
@@ -69,4 +69,10 @@ WORKDIR /home/test/
 
 COPY .bashrc /home/test/.bashrc
 
+# Ubuntu inserts a logout script which exits with a non-zero status. This makes
+# it impossible to use the `exit` built-in from a login shell where the `-e`
+# option has been set. Since the script is intended to clear the console for
+# interactive sessions, it may safely be removed.
+RUN rm /home/test/.bash_logout
+
 COPY start.sh /home/test/start.sh

From c3f109bdfa187c17042ec2d0074f4ea0b5bf13b4 Mon Sep 17 00:00:00 2001
From: Mike Pennisi <mike@mikepennisi.com>
Date: Thu, 23 Aug 2018 12:49:08 -0400
Subject: [PATCH 02/18] [infra] Temporarily ignore stability status in CI

---
 .taskcluster.yml | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/.taskcluster.yml b/.taskcluster.yml
index 71267779ca411b..2d7ac4e0a27e6d 100644
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -85,12 +85,16 @@ tasks:
             - name: wpt-${browser.name}-${browser.channel}-stability
               description: >-
                 Verify that all tests affected by a pull request are stable
-                when executed in ${browser.name}.
+                when executed in ${browser.name}. As of 2018-08-23, this task
+                is being vetted for correctness, so its outcome will not
+                influence the status of the pull request.
               command:
+                status=PASS;
                 echo $affected |
                   base64 -d |
-                  xargs -0 ./tools/ci/ci_taskcluster.sh --verify ${browser.name}
-              creates_artifacts: false
+                  xargs -0 ./tools/ci/ci_taskcluster.sh --verify ${browser.name} || status=FAIL;
+                echo $status > ../artifacts/stability-status.txt
+              creates_artifacts: true
             - name: wpt-${browser.name}-${browser.channel}-results
               description: >-
                 Collect results for all tests affected by a pull request in

From a1f01d0ddcbb61a7efc84ab00dbb038fbb6736f0 Mon Sep 17 00:00:00 2001
From: Mike Pennisi <mike@mikepennisi.com>
Date: Thu, 23 Aug 2018 20:39:35 -0400
Subject: [PATCH 03/18] Demonstrate task with zero affected tests


From ee8d57d48a28d3f1d9c738964e640a4dcbc1e92d Mon Sep 17 00:00:00 2001
From: Mike Pennisi <mike@mikepennisi.com>
Date: Thu, 23 Aug 2018 18:13:15 -0400
Subject: [PATCH 04/18] DO NOT MERGE Introduce instability

---
 dom/events/CustomEvent.html | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/dom/events/CustomEvent.html b/dom/events/CustomEvent.html
index abc6c370fd2e87..45b5cbefbd2f13 100644
--- a/dom/events/CustomEvent.html
+++ b/dom/events/CustomEvent.html
@@ -4,6 +4,10 @@
 <script src="/resources/testharnessreport.js"></script>
 <div id="log"></div>
 <script>
+test(function() {
+  assert_true(Math.random() > 0.5);
+}, 'intentionally unstable test');
+
 test(function() {
   var type = "foo";
 

From 7808fb12ab5068b5fb5846d93b7170b27d3db31c Mon Sep 17 00:00:00 2001
From: Mike Pennisi <mike@mikepennisi.com>
Date: Tue, 28 Aug 2018 20:08:53 -0400
Subject: [PATCH 05/18] Re-implement script in Python

---
 .taskcluster.yml            | 80 ++++++++++++++++++-------------------
 tools/ci/ci_taskcluster.sh  | 44 --------------------
 tools/ci/taskcluster-run.py | 69 ++++++++++++++++++++++++++++++++
 tools/docker/Dockerfile     |  6 ---
 4 files changed, 108 insertions(+), 91 deletions(-)
 delete mode 100755 tools/ci/ci_taskcluster.sh
 create mode 100755 tools/ci/taskcluster-run.py

diff --git a/.taskcluster.yml b/.taskcluster.yml
index 2d7ac4e0a27e6d..c6495181131c18 100644
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -55,7 +55,7 @@ tasks:
                 owner: ${event.pusher.email}
                 source: ${event.repository.url}
               payload:
-                image: jugglinmike/web-platform-tests:0.18
+                image: gsnedders/web-platform-tests:0.13
                 maxRunTime: 7200
                 artifacts:
                   public/results:
@@ -72,11 +72,23 @@ tasks:
                       ${event.after}
                       ${browser.name}-${browser.channel};
                     cd ~/web-platform-tests;
-                    ./tools/ci/ci_taskcluster.sh
+                    ./wpt manifest-download || true;
+                    ./tools/ci/taskcluster-run.py
+                      --browser ${browser.name}
+                      --
+                      --log-tbpl=../artifacts/log_tbpl.log
+                      --log-tbpl-level=info
+                      --log-wptreport=../artifacts/wpt_report.json
+                      --log-mach=-
+                      -y
+                      --no-pause
+                      --no-restart-on-unexpected
+                      --install-fonts
+                      --no-fail-on-unexpected
                       --test-type=${chunk[0]}
                       --this-chunk=${chunk[1]}
-                      --total-chunks=${chunk[2]}
-                      ${browser.name};
+                      --total-chunks=${chunk[2]};
+                    gzip ../artifacts/wpt_report.json;
     - $if: tasks_for == "github-pull-request"
       then:
         $map: [{name: firefox, channel: nightly}, {name: chrome, channel: dev}]
@@ -85,25 +97,13 @@ tasks:
             - name: wpt-${browser.name}-${browser.channel}-stability
               description: >-
                 Verify that all tests affected by a pull request are stable
-                when executed in ${browser.name}. As of 2018-08-23, this task
-                is being vetted for correctness, so its outcome will not
-                influence the status of the pull request.
-              command:
-                status=PASS;
-                echo $affected |
-                  base64 -d |
-                  xargs -0 ./tools/ci/ci_taskcluster.sh --verify ${browser.name} || status=FAIL;
-                echo $status > ../artifacts/stability-status.txt
-              creates_artifacts: true
+                when executed in ${browser.name}.
+              extra_args: '--verify'
             - name: wpt-${browser.name}-${browser.channel}-results
               description: >-
                 Collect results for all tests affected by a pull request in
                 ${browser.name}.
-              command:
-                echo $affected |
-                  base64 -d |
-                  xargs -0 ./tools/ci/ci_taskcluster.sh ${browser.name}
-              creates_artifacts: true
+              extra_args: '--no-fail-on-unexpected --log-wptreport=../artifacts/wpt_report.json'
           each(operation):
             taskId: {$eval: 'as_slugid(operation.name)'}
             taskGroupId: {$eval: 'as_slugid("task group")'}
@@ -120,19 +120,12 @@ tasks:
               owner: ${event.pull_request.user.login}@users.noreply.github.com
               source: ${event.repository.url}
             payload:
-              image: jugglinmike/web-platform-tests:0.18
+              image: gsnedders/web-platform-tests:0.13
               maxRunTime: 7200
               artifacts:
-                $if: operation.creates_artifacts
-                then:
-                  public/results:
-                    path: /home/test/artifacts
-                    type: directory
-              # Bash removes null bytes from string values when set as
-              # environment variables. This invalidates the output of `wpt
-              # affected-tests` because it uses the null byte as a separator
-              # between test names. The list of effected tests is
-              # Base64-encoded in order to preserve the null bytes.
+                public/results:
+                  path: /home/test/artifacts
+                  type: directory
               command:
                 - /bin/bash
                 - --login
@@ -149,14 +142,19 @@ tasks:
                     ${event.repository.clone_url}
                     refs/pull/${event.number}/head;
                   git checkout FETCH_HEAD;
-                  affected=$(./wpt tests-affected
-                    --null
-                    ${event.pull_request.base.sha}...
-                    | base64 --wrap 0);
-                  if [ -z $(echo $affected | base64 -d) ]; then
-                    echo Zero tests affected.;
-                    exit 0;
-                  fi;
-                  echo Affected tests:;
-                  echo $affected | base64 -d | sed 's/\x0/\n/g';
-                  ${operation.command}
+                  result=0;
+                  ./tools/ci/taskcluster-run.py
+                    --browser ${browser.name}
+                    --commit-range ${event.pull_request.base.sha}...
+                    --
+                    ${operation.extra_args}
+                    --log-tbpl=../artifacts/log_tbpl.log
+                    --log-tbpl-level=info
+                    --log-mach=-
+                    -y
+                    --no-pause
+                    --no-restart-on-unexpected
+                     --install-fonts || result=$?;
+                  if [ -f ../artifacts/wpt_report.json ]; then gzip ../artifacts/wpt_report.json; fi;
+                  echo $result > ../artifacts/run-return-code.txt;
+                  echo "Command exited with code $result (failures are allowed while this task is being vetted)."
diff --git a/tools/ci/ci_taskcluster.sh b/tools/ci/ci_taskcluster.sh
deleted file mode 100755
index 0db8b9db31741f..00000000000000
--- a/tools/ci/ci_taskcluster.sh
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/bin/bash
-
-./wpt manifest-download
-
-# The first script argument that is not prefixed with a dash (`-`) is assumed
-# to be the name of the browser under test. This restricts the syntax available
-# to consumers: value-accepting options must be specified using the equals sign
-# (`=`).
-for argument in $@; do
-  if [ ${argument:0:1} = '-' ]; then
-    continue
-  fi
-
-  browser_name=$argument
-
-  break
-done
-
-browser_specific_args=''
-
-if [ $browser_name = 'firefox' ]; then
-  browser_specific_args='--install-browser --reftest-internal'
-fi
-
-./wpt run \
-  --log-tbpl=../artifacts/log_tbpl.log \
-  --log-tbpl-level=info \
-  --log-wptreport=../artifacts/wpt_report.json \
-  --log-mach=- \
-  -y \
-  --no-pause \
-  --no-restart-on-unexpected \
-  --install-fonts \
-  --no-fail-on-unexpected \
-  $browser_specific_args \
-  $@
-
-run_status=$?
-
-if [ -f ../artifacts/wpt_report.json ]; then
-  gzip ../artifacts/wpt_report.json
-fi
-
-exit $run_status
diff --git a/tools/ci/taskcluster-run.py b/tools/ci/taskcluster-run.py
new file mode 100755
index 00000000000000..0bf5236c7d8a4b
--- /dev/null
+++ b/tools/ci/taskcluster-run.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+
+import argparse
+import logging
+import os
+import subprocess
+
+browser_specific_args = {
+    "firefox": ["--install-browser", "--reftest-internal"]
+}
+
+def tests_affected(commit_range):
+    output = subprocess.check_output([
+        "python", "./wpt", "tests-affected", "--null", commit_range
+    ], stderr=open(os.devnull, "w"))
+
+    tests = output.split("\0")
+
+    # Account for trailing null byte
+    if not tests[-1]:
+        tests.pop()
+
+    return tests
+
+
+def main(browser, commit_range, wpt_args):
+    """Invoke the `wpt run` command according to the needs of the TaskCluster
+    continuous integration service."""
+
+    logger = logging.getLogger("tc-run")
+    logger.setLevel(logging.INFO)
+    handler = logging.StreamHandler()
+    handler.setFormatter(
+        logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+    )
+    logger.addHandler(handler)
+
+    if commit_range:
+        logger.info(
+            "Identifying tests affected in range '%s'..." % commit_range
+        )
+        tests = tests_affected(commit_range)
+        logger.info("Identified %s affected tests" % len(tests))
+
+        if not tests:
+            logger.info("Quitting because no tests were affected.")
+            return
+    else:
+        tests = []
+        logger.info("Running all tests")
+
+    wpt_args += browser_specific_args.get(browser, [])
+
+    command = ["python", "./wpt", "run"] + wpt_args + [browser] + tests
+
+    logger.info("Executing command: %s" % " ".join(command))
+
+    subprocess.check_call(command)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description=main.__doc__)
+    parser.add_argument("--browser", action="store", required=True)
+    parser.add_argument("--commit-range", action="store",
+                        help="""Git commit range. If specified, this will be
+                             supplied to the `wpt tests-affected` command to
+                             determine the list of tests executed""")
+    parser.add_argument("wpt_args", nargs="*")
+    main(**vars(parser.parse_args()))
diff --git a/tools/docker/Dockerfile b/tools/docker/Dockerfile
index 2362ef39623600..b0541dc6c98337 100644
--- a/tools/docker/Dockerfile
+++ b/tools/docker/Dockerfile
@@ -69,10 +69,4 @@ WORKDIR /home/test/
 
 COPY .bashrc /home/test/.bashrc
 
-# Ubuntu inserts a logout script which exits with a non-zero status. This makes
-# it impossible to use the `exit` built-in from a login shell where the `-e`
-# option has been set. Since the script is intended to clear the console for
-# interactive sessions, it may safely be removed.
-RUN rm /home/test/.bash_logout
-
 COPY start.sh /home/test/start.sh

From 0d6c2a25b90ded1cee711c91fff66edb9034fdc7 Mon Sep 17 00:00:00 2001
From: Mike Pennisi <mike@mikepennisi.com>
Date: Tue, 28 Aug 2018 21:26:56 -0400
Subject: [PATCH 06/18] Reduce repetition

---
 .taskcluster.yml            | 24 +++-----------------
 tools/ci/taskcluster-run.py | 44 ++++++++++++++++++++++++++++++++-----
 2 files changed, 41 insertions(+), 27 deletions(-)

diff --git a/.taskcluster.yml b/.taskcluster.yml
index c6495181131c18..4f3c644b887e89 100644
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -72,23 +72,14 @@ tasks:
                       ${event.after}
                       ${browser.name}-${browser.channel};
                     cd ~/web-platform-tests;
-                    ./wpt manifest-download || true;
                     ./tools/ci/taskcluster-run.py
-                      --browser ${browser.name}
+                      ${browser.name}
                       --
-                      --log-tbpl=../artifacts/log_tbpl.log
-                      --log-tbpl-level=info
                       --log-wptreport=../artifacts/wpt_report.json
-                      --log-mach=-
-                      -y
-                      --no-pause
-                      --no-restart-on-unexpected
-                      --install-fonts
                       --no-fail-on-unexpected
                       --test-type=${chunk[0]}
                       --this-chunk=${chunk[1]}
                       --total-chunks=${chunk[2]};
-                    gzip ../artifacts/wpt_report.json;
     - $if: tasks_for == "github-pull-request"
       then:
         $map: [{name: firefox, channel: nightly}, {name: chrome, channel: dev}]
@@ -137,24 +128,15 @@ tasks:
                     ${event.pull_request.base.sha}
                     ${browser.name}-${browser.channel};
                   cd ~/web-platform-tests;
-                  ./wpt manifest-download || true;
                   git fetch
                     ${event.repository.clone_url}
                     refs/pull/${event.number}/head;
                   git checkout FETCH_HEAD;
                   result=0;
                   ./tools/ci/taskcluster-run.py
-                    --browser ${browser.name}
                     --commit-range ${event.pull_request.base.sha}...
+                    ${browser.name}
                     --
-                    ${operation.extra_args}
-                    --log-tbpl=../artifacts/log_tbpl.log
-                    --log-tbpl-level=info
-                    --log-mach=-
-                    -y
-                    --no-pause
-                    --no-restart-on-unexpected
-                     --install-fonts || result=$?;
-                  if [ -f ../artifacts/wpt_report.json ]; then gzip ../artifacts/wpt_report.json; fi;
+                    ${operation.extra_args} || result=$?;
                   echo $result > ../artifacts/run-return-code.txt;
                   echo "Command exited with code $result (failures are allowed while this task is being vetted)."
diff --git a/tools/ci/taskcluster-run.py b/tools/ci/taskcluster-run.py
index 0bf5236c7d8a4b..22909cc13db2dc 100755
--- a/tools/ci/taskcluster-run.py
+++ b/tools/ci/taskcluster-run.py
@@ -1,8 +1,10 @@
 #!/usr/bin/env python
 
 import argparse
+import gzip
 import logging
 import os
+import shutil
 import subprocess
 
 browser_specific_args = {
@@ -23,7 +25,19 @@ def tests_affected(commit_range):
     return tests
 
 
-def main(browser, commit_range, wpt_args):
+def find_wptreport(args):
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--log-wptreport', action='store')
+    return parser.parse_known_args(args)[0].log_wptreport
+
+
+def gzip_file(filename):
+    with open(filename, 'rb') as f_in:
+        with gzip.open('%s.gz' % filename, 'wb') as f_out:
+            shutil.copyfileobj(f_in, f_out)
+
+
+def main(product, commit_range, wpt_args):
     """Invoke the `wpt run` command according to the needs of the TaskCluster
     continuous integration service."""
 
@@ -35,6 +49,9 @@ def main(browser, commit_range, wpt_args):
     )
     logger.addHandler(handler)
 
+    child = subprocess.Popen(['python', './wpt', 'manifest-download'])
+    child.wait()
+
     if commit_range:
         logger.info(
             "Identifying tests affected in range '%s'..." % commit_range
@@ -49,21 +66,36 @@ def main(browser, commit_range, wpt_args):
         tests = []
         logger.info("Running all tests")
 
-    wpt_args += browser_specific_args.get(browser, [])
+    wpt_args += [
+        "--log-tbpl=../artifacts/log_tbpl.log",
+        "--log-tbpl-level=info",
+        "--log-mach=-",
+        "-y",
+        "--no-pause",
+        "--no-restart-on-unexpected",
+        "--install-fonts"
+    ]
+    wpt_args += browser_specific_args.get(product, [])
 
-    command = ["python", "./wpt", "run"] + wpt_args + [browser] + tests
+    command = ["python", "./wpt", "run"] + wpt_args + [product] + tests
 
     logger.info("Executing command: %s" % " ".join(command))
 
     subprocess.check_call(command)
 
+    wptreport = find_wptreport(wpt_args)
+    if wptreport:
+        gzip_file(wptreport)
+
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description=main.__doc__)
-    parser.add_argument("--browser", action="store", required=True)
     parser.add_argument("--commit-range", action="store",
                         help="""Git commit range. If specified, this will be
                              supplied to the `wpt tests-affected` command to
-                             determine the list of tests executed""")
-    parser.add_argument("wpt_args", nargs="*")
+                             determine the list of test to execute""")
+    parser.add_argument("product", action="store",
+                        help="Browser to run tests in")
+    parser.add_argument("wpt_args", nargs="*",
+                        help="Arguments to forward to `wpt run` command")
     main(**vars(parser.parse_args()))

From c1792ee9291fb802903c1651ddb56bd95545e7e3 Mon Sep 17 00:00:00 2001
From: Mike Pennisi <mike@mikepennisi.com>
Date: Tue, 28 Aug 2018 22:04:08 -0400
Subject: [PATCH 07/18] Trigger TaskCluster


From f422774e24389d3c13c2416fe5a9774273cbcf34 Mon Sep 17 00:00:00 2001
From: Mike Pennisi <mike@mikepennisi.com>
Date: Wed, 29 Aug 2018 15:11:03 -0400
Subject: [PATCH 08/18] Trigger CI


From 150058ee2fe6a2e8124bdbe2bf12851fd4db2ac0 Mon Sep 17 00:00:00 2001
From: Mike Pennisi <mike@mikepennisi.com>
Date: Wed, 29 Aug 2018 18:31:01 -0400
Subject: [PATCH 09/18] Correct commit range

---
 .taskcluster.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.taskcluster.yml b/.taskcluster.yml
index 4f3c644b887e89..2066f5e24d2703 100644
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -134,7 +134,7 @@ tasks:
                   git checkout FETCH_HEAD;
                   result=0;
                   ./tools/ci/taskcluster-run.py
-                    --commit-range ${event.pull_request.base.sha}...
+                    --commit-range ${event.pull_request.base.sha}..
                     ${browser.name}
                     --
                     ${operation.extra_args} || result=$?;

From 0b623f3091286b1e0196623591bad81d0b303673 Mon Sep 17 00:00:00 2001
From: Mike Pennisi <mike@mikepennisi.com>
Date: Wed, 29 Aug 2018 18:35:52 -0400
Subject: [PATCH 10/18] Trigger CI


From 30b4ae41b8ab550fd708f53e28373c0c0be16892 Mon Sep 17 00:00:00 2001
From: Mike Pennisi <mike@mikepennisi.com>
Date: Thu, 30 Aug 2018 21:33:43 -0400
Subject: [PATCH 11/18] Incorporate review feedback

---
 .taskcluster.yml      | 16 +++++++---------
 tools/docker/start.sh | 23 +++++++++++++++--------
 2 files changed, 22 insertions(+), 17 deletions(-)

diff --git a/.taskcluster.yml b/.taskcluster.yml
index 2066f5e24d2703..344d42401d3980 100644
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -55,7 +55,7 @@ tasks:
                 owner: ${event.pusher.email}
                 source: ${event.repository.url}
               payload:
-                image: gsnedders/web-platform-tests:0.13
+                image: jugglinmike/web-platform-tests:0.19
                 maxRunTime: 7200
                 artifacts:
                   public/results:
@@ -70,6 +70,7 @@ tasks:
                       ${event.repository.url}
                       ${event.ref[len('refs/heads/'):]}
                       ${event.after}
+                      50
                       ${browser.name}-${browser.channel};
                     cd ~/web-platform-tests;
                     ./tools/ci/taskcluster-run.py
@@ -111,7 +112,7 @@ tasks:
               owner: ${event.pull_request.user.login}@users.noreply.github.com
               source: ${event.repository.url}
             payload:
-              image: gsnedders/web-platform-tests:0.13
+              image: jugglinmike/web-platform-tests:0.19
               maxRunTime: 7200
               artifacts:
                 public/results:
@@ -124,17 +125,14 @@ tasks:
                 - set -ex;
                   ~/start.sh
                     ${event.repository.clone_url}
-                    ${event.pull_request.base.ref}
-                    ${event.pull_request.base.sha}
+                    refs/pull/${event.number}/head
+                    ${event.pull_request.head.sha}
+                    ${event.pull_request.commits + 1}
                     ${browser.name}-${browser.channel};
                   cd ~/web-platform-tests;
-                  git fetch
-                    ${event.repository.clone_url}
-                    refs/pull/${event.number}/head;
-                  git checkout FETCH_HEAD;
                   result=0;
                   ./tools/ci/taskcluster-run.py
-                    --commit-range ${event.pull_request.base.sha}..
+                    --commit-range HEAD~${event.pull_request.commits}
                     ${browser.name}
                     --
                     ${operation.extra_args} || result=$?;
diff --git a/tools/docker/start.sh b/tools/docker/start.sh
index 3325bb24f587df..9c034a44f5bd4d 100755
--- a/tools/docker/start.sh
+++ b/tools/docker/start.sh
@@ -12,23 +12,30 @@
 set -ex
 
 REMOTE=${1:-https://github.com/web-platform-tests/wpt}
-BRANCH=${2:-master}
-REV=${3:-FETCH_HEAD}
-BROWSER=${4:-all}
+REF=${2:-master}
+REVISION=${3:-FETCH_HEAD}
+DEPTH=${4:-50}
+BROWSER=${5:-all}
 
 cd ~
 
-# Initially we just fetch 50 commits in order to save several minutes of fetching
-git clone ${REMOTE} --single-branch --branch ${BRANCH} --no-checkout -q --depth=50 web-platform-tests
+mkdir web-platform-tests
 cd web-platform-tests
-if [[ ! `git rev-parse --verify -q ${REV}` ]];
+git init
+git remote add origin ${REMOTE}
+
+# Initially we fetch a limited number of commits in order to save several
+# minutes of fetching
+git fetch --quiet --depth=${DEPTH} origin ${REF}
+
+if [[ ! `git rev-parse --verify -q ${REVISION}` ]];
 then
     # But if for some reason the commit under test isn't in that range, we give in and
     # fetch everything
     git fetch -q --unshallow ${REMOTE}
-    git rev-parse --verify ${REV}
+    git rev-parse --verify ${REVISION}
 fi
-git checkout -b build ${REV}
+git checkout -b build ${REVISION}
 
 sudo sh -c './wpt make-hosts-file >> /etc/hosts'
 

From 3371a4c7d3f474306427b6df7aae4a51c4bf7c79 Mon Sep 17 00:00:00 2001
From: Mike Pennisi <mike@mikepennisi.com>
Date: Fri, 31 Aug 2018 15:25:08 -0400
Subject: [PATCH 12/18] Test at GitHub-provided merge commit

This improves the authenticity of the reported results because it
simulates how the patch will behave after it is merged. This also mimics
the behavior of the TravisCI continuous integration platform.
---
 .taskcluster.yml      | 18 +++++++++++-------
 tools/docker/start.sh |  8 +++-----
 2 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/.taskcluster.yml b/.taskcluster.yml
index 344d42401d3980..a14f4631eed98a 100644
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -55,7 +55,7 @@ tasks:
                 owner: ${event.pusher.email}
                 source: ${event.repository.url}
               payload:
-                image: jugglinmike/web-platform-tests:0.19
+                image: jugglinmike/web-platform-tests:0.20
                 maxRunTime: 7200
                 artifacts:
                   public/results:
@@ -70,7 +70,6 @@ tasks:
                       ${event.repository.url}
                       ${event.ref[len('refs/heads/'):]}
                       ${event.after}
-                      50
                       ${browser.name}-${browser.channel};
                     cd ~/web-platform-tests;
                     ./tools/ci/taskcluster-run.py
@@ -112,12 +111,18 @@ tasks:
               owner: ${event.pull_request.user.login}@users.noreply.github.com
               source: ${event.repository.url}
             payload:
-              image: jugglinmike/web-platform-tests:0.19
+              image: jugglinmike/web-platform-tests:0.20
               maxRunTime: 7200
               artifacts:
                 public/results:
                   path: /home/test/artifacts
                   type: directory
+              # Fetch the GitHub-provided merge commit (rather than the pull
+              # request branch) so that the tasks simulate the behavior of the
+              # submitted patch after it is merged. Using the merge commit also
+              # simplifies detection of modified files because the first parent
+              # of the merge commit can consistently be used to summarize the
+              # changes.
               command:
                 - /bin/bash
                 - --login
@@ -125,14 +130,13 @@ tasks:
                 - set -ex;
                   ~/start.sh
                     ${event.repository.clone_url}
-                    refs/pull/${event.number}/head
-                    ${event.pull_request.head.sha}
-                    ${event.pull_request.commits + 1}
+                    refs/pull/${event.number}/merge
+                    FETCH_HEAD
                     ${browser.name}-${browser.channel};
                   cd ~/web-platform-tests;
                   result=0;
                   ./tools/ci/taskcluster-run.py
-                    --commit-range HEAD~${event.pull_request.commits}
+                    --commit-range HEAD^
                     ${browser.name}
                     --
                     ${operation.extra_args} || result=$?;
diff --git a/tools/docker/start.sh b/tools/docker/start.sh
index 9c034a44f5bd4d..46e4f43b69a515 100755
--- a/tools/docker/start.sh
+++ b/tools/docker/start.sh
@@ -14,8 +14,7 @@ set -ex
 REMOTE=${1:-https://github.com/web-platform-tests/wpt}
 REF=${2:-master}
 REVISION=${3:-FETCH_HEAD}
-DEPTH=${4:-50}
-BROWSER=${5:-all}
+BROWSER=${4:-all}
 
 cd ~
 
@@ -24,9 +23,8 @@ cd web-platform-tests
 git init
 git remote add origin ${REMOTE}
 
-# Initially we fetch a limited number of commits in order to save several
-# minutes of fetching
-git fetch --quiet --depth=${DEPTH} origin ${REF}
+# Initially we just fetch 50 commits in order to save several minutes of fetching
+git fetch --quiet --depth=50 origin ${REF}
 
 if [[ ! `git rev-parse --verify -q ${REVISION}` ]];
 then

From cbc779c0a445b4dc1f091d759adf1cc223f3bbdf Mon Sep 17 00:00:00 2001
From: Mike Pennisi <mike@mikepennisi.com>
Date: Fri, 31 Aug 2018 16:04:54 -0400
Subject: [PATCH 13/18] Simplify reference

The `start.sh` script now supports all git references, so this
computation is no longer necessary.
---
 .taskcluster.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.taskcluster.yml b/.taskcluster.yml
index a14f4631eed98a..3c99913eb486f0 100644
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -68,7 +68,7 @@ tasks:
                   - set -ex;
                     ~/start.sh
                       ${event.repository.url}
-                      ${event.ref[len('refs/heads/'):]}
+                      ${event.ref}
                       ${event.after}
                       ${browser.name}-${browser.channel};
                     cd ~/web-platform-tests;

From 887512c7c962ca5aa6e690666af03ab2f39a317a Mon Sep 17 00:00:00 2001
From: Mike Pennisi <mike@mikepennisi.com>
Date: Fri, 31 Aug 2018 16:11:25 -0400
Subject: [PATCH 14/18] Remove superflous command

---
 tools/ci/taskcluster-run.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tools/ci/taskcluster-run.py b/tools/ci/taskcluster-run.py
index 22909cc13db2dc..d7d66fa4e628b1 100755
--- a/tools/ci/taskcluster-run.py
+++ b/tools/ci/taskcluster-run.py
@@ -49,9 +49,6 @@ def main(product, commit_range, wpt_args):
     )
     logger.addHandler(handler)
 
-    child = subprocess.Popen(['python', './wpt', 'manifest-download'])
-    child.wait()
-
     if commit_range:
         logger.info(
             "Identifying tests affected in range '%s'..." % commit_range

From d9a12c3b3484278b586704097db646cecabc5a98 Mon Sep 17 00:00:00 2001
From: Mike Pennisi <mike@mikepennisi.com>
Date: Fri, 31 Aug 2018 16:40:45 -0400
Subject: [PATCH 15/18] Revert "Remove superflous command"

This reverts commit 887512c7c962ca5aa6e690666af03ab2f39a317a.
---
 tools/ci/taskcluster-run.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tools/ci/taskcluster-run.py b/tools/ci/taskcluster-run.py
index d7d66fa4e628b1..22909cc13db2dc 100755
--- a/tools/ci/taskcluster-run.py
+++ b/tools/ci/taskcluster-run.py
@@ -49,6 +49,9 @@ def main(product, commit_range, wpt_args):
     )
     logger.addHandler(handler)
 
+    child = subprocess.Popen(['python', './wpt', 'manifest-download'])
+    child.wait()
+
     if commit_range:
         logger.info(
             "Identifying tests affected in range '%s'..." % commit_range

From 80714c95fad0d59343f65ba4766ff1b34e0110a4 Mon Sep 17 00:00:00 2001
From: Mike Pennisi <mike@mikepennisi.com>
Date: Wed, 5 Sep 2018 20:24:19 -0400
Subject: [PATCH 16/18] Remove superfluous argument

`--reftest-internal` is enabled by default in GNU/Linux environments.
---
 tools/ci/taskcluster-run.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/ci/taskcluster-run.py b/tools/ci/taskcluster-run.py
index 22909cc13db2dc..379fc20f53fe10 100755
--- a/tools/ci/taskcluster-run.py
+++ b/tools/ci/taskcluster-run.py
@@ -8,7 +8,7 @@
 import subprocess
 
 browser_specific_args = {
-    "firefox": ["--install-browser", "--reftest-internal"]
+    "firefox": ["--install-browser"]
 }
 
 def tests_affected(commit_range):

From c4ea324a03398ea2ef34a6f0abf4c49519c913d7 Mon Sep 17 00:00:00 2001
From: Mike Pennisi <mike@mikepennisi.com>
Date: Wed, 5 Sep 2018 20:28:49 -0400
Subject: [PATCH 17/18] Accommodate jobs with zero affected tests

---
 tools/ci/taskcluster-run.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/ci/taskcluster-run.py b/tools/ci/taskcluster-run.py
index 379fc20f53fe10..45416501e03934 100755
--- a/tools/ci/taskcluster-run.py
+++ b/tools/ci/taskcluster-run.py
@@ -19,7 +19,7 @@ def tests_affected(commit_range):
     tests = output.split("\0")
 
     # Account for trailing null byte
-    if not tests[-1]:
+    if tests and not tests[-1]:
         tests.pop()
 
     return tests

From e79a708f567688d95d92d54c4a66dee4ec8df441 Mon Sep 17 00:00:00 2001
From: Mike Pennisi <mike@mikepennisi.com>
Date: Wed, 12 Sep 2018 11:55:58 -0400
Subject: [PATCH 18/18] Revert "DO NOT MERGE Introduce instability"

This reverts commit ee8d57d48a28d3f1d9c738964e640a4dcbc1e92d.
---
 dom/events/CustomEvent.html | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/dom/events/CustomEvent.html b/dom/events/CustomEvent.html
index 45b5cbefbd2f13..abc6c370fd2e87 100644
--- a/dom/events/CustomEvent.html
+++ b/dom/events/CustomEvent.html
@@ -4,10 +4,6 @@
 <script src="/resources/testharnessreport.js"></script>
 <div id="log"></div>
 <script>
-test(function() {
-  assert_true(Math.random() > 0.5);
-}, 'intentionally unstable test');
-
 test(function() {
   var type = "foo";