PARTIAL: Begin pull request task

.taskcluster.yml

@@ -2,76 +2,143 @@ version: 1
 policy:
   pullRequests: collaborators
 tasks:
-  $if: tasks_for == "github-push"
-  then:
-    $if: event.ref == "refs/heads/master"
-    then:
-      $flattenDeep:
+  $flattenDeep:
+    - $if: tasks_for == "skip for now"
+      then:
+        $if: event.ref == "refs/heads/master"
+        then:
+          $map: [{name: firefox, channel: nightly}, {name: chrome, channel: dev}]
+          each(browser):
+            $map:
+              - [testharness, 1, 15]
+              - [testharness, 2, 15]
+              - [testharness, 3, 15]
+              - [testharness, 4, 15]
+              - [testharness, 5, 15]
+              - [testharness, 6, 15]
+              - [testharness, 7, 15]
+              - [testharness, 8, 15]
+              - [testharness, 9, 15]
+              - [testharness, 10, 15]
+              - [testharness, 11, 15]
+              - [testharness, 12, 15]
+              - [testharness, 13, 15]
+              - [testharness, 14, 15]
+              - [testharness, 15, 15]
+              - [reftest, 1, 10]
+              - [reftest, 2, 10]
+              - [reftest, 3, 10]
+              - [reftest, 4, 10]
+              - [reftest, 5, 10]
+              - [reftest, 6, 10]
+              - [reftest, 7, 10]
+              - [reftest, 8, 10]
+              - [reftest, 9, 10]
+              - [reftest, 10, 10]
+              - [wdspec, 1, 1]
+            each(chunk):
+              taskId: {$eval: 'as_slugid(browser.name + browser.channel + chunk[0] + str(chunk[1]))'}
+              taskGroupId: {$eval: 'as_slugid("task group")'}
+              created: {$fromNow: ''}
+              deadline: {$fromNow: '24 hours'}
+              provisionerId: aws-provisioner-v1
+              # Contributors interested in configurating TaskCluster to run
+              # against their fork of WPT should change the `workerType` to
+              # "github-worker".
+              workerType: github-worker
+              metadata:
+                name: wpt-${browser.name}-${browser.channel}-${chunk[0]}-${chunk[1]}
+                description: >-
+                  A subset of WPT's "${chunk[0]}" tests (chunk number ${chunk[1]}
+                  of ${chunk[2]}), run in the ${browser.channel} release of
+                  ${browser.name}.
+                owner: ${event.pusher.email}
+                source: ${event.repository.url}
+              payload:
+                image: gsnedders/web-platform-tests:0.13
+                maxRunTime: 7200
+                artifacts:
+                  public/results:
+                    path: /home/test/artifacts
+                    type: directory
+                command:
+                  - /bin/bash
+                  - --login
+                  - -c
+                  - set -ex;
+                    ~/start.sh
+                      ${event.repository.url}
+                      ${event.ref}
+                      ${event.after}
+                      ${browser.name}-${browser.channel};
+                    cd ~/web-platform-tests;
+                    ./tools/ci/ci_taskcluster.sh
+                      --test-type=${chunk[0]}
+                      --this-chunk=${chunk[1]}
+                      --total-chunks=${chunk[2]}
+                      ${browser.name};
+    - $if: tasks_for == "github-pull-request"
+      then:
         $map: [{name: firefox, channel: nightly}, {name: chrome, channel: dev}]
         each(browser):
           $map:
-            - [testharness, 1, 15]
-            - [testharness, 2, 15]
-            - [testharness, 3, 15]
-            - [testharness, 4, 15]
-            - [testharness, 5, 15]
-            - [testharness, 6, 15]
-            - [testharness, 7, 15]
-            - [testharness, 8, 15]
-            - [testharness, 9, 15]
-            - [testharness, 10, 15]
-            - [testharness, 11, 15]
-            - [testharness, 12, 15]
-            - [testharness, 13, 15]
-            - [testharness, 14, 15]
-            - [testharness, 15, 15]
-            - [reftest, 1, 10]
-            - [reftest, 2, 10]
-            - [reftest, 3, 10]
-            - [reftest, 4, 10]
-            - [reftest, 5, 10]
-            - [reftest, 6, 10]
-            - [reftest, 7, 10]
-            - [reftest, 8, 10]
-            - [reftest, 9, 10]
-            - [reftest, 10, 10]
-            - [wdspec, 1, 1]
-          each(chunk):
-            taskId: {$eval: 'as_slugid(browser.name + browser.channel + chunk[0] + str(chunk[1]))'}
+            - name: wpt-${browser.name}-${browser.channel}-stability
+              description: >-
+                Verify that tests modified in a pull request are stable when
+                executed in ${browser.name}.
+              command:
+                echo $affected | base64 -d | xargs -0 ./tools/ci/ci_taskcluster.sh --verify ${browser.name}
+              creates_artifacts: false
+            - name: wpt-${browser.name}-${browser.channel}-results
+              description: >-
+                Collect results for modified tests in ${browser.name}.
+              command:
+                echo $affected | base64 -d | xargs -0 ./tools/ci/ci_taskcluster.sh ${browser.name}
+              creates_artifacts: true
+          each(operation):
+            taskId: {$eval: 'as_slugid(operation.name)'}
             taskGroupId: {$eval: 'as_slugid("task group")'}
             created: {$fromNow: ''}
             deadline: {$fromNow: '24 hours'}
             provisionerId: aws-provisioner-v1
             # Contributors interested in configurating TaskCluster to run
             # against their fork of WPT should change the `workerType` to
             # "github-worker".
-            workerType: wpt-docker-worker
+            workerType: github-worker
             metadata:
-              name: wpt-${browser.name}-${browser.channel}-${chunk[0]}-${chunk[1]}
-              description: >-
-                A subset of WPT's "${chunk[0]}" tests (chunk number ${chunk[1]}
-                of ${chunk[2]}), run in the ${browser.channel} release of
-                ${browser.name}.
-              owner: ${event.pusher.email}
+              name: ${operation.name}
+              description: ${operation.description}
+              owner: ${event.pull_request.user.login}@users.noreply.github.com
               source: ${event.repository.url}
             payload:
               image: gsnedders/web-platform-tests:0.13
               maxRunTime: 7200
               artifacts:
-                public/results:
-                  path: /home/test/artifacts
-                  type: directory
+                $if: operation.creates_artifacts
+                then:
+                  public/results:
+                    path: /home/test/artifacts
+                    type: directory
               command:
                 - /bin/bash
                 - --login
                 - -c
-                - "~/start.sh ${event.repository.url} ${event.ref[len('refs/heads/'):]} ${event.after} ${browser.name}-${browser.channel} &&
-                  cd ~/web-platform-tests &&
-                  ./tools/ci/ci_taskcluster.sh ${browser.name} ${chunk[0]} ${chunk[1]} ${chunk[2]}"
-    else: []
-  # > NOTE: A well-designed template should produce `tasks: []` for any
-  # > unrecognized `task_for` values; this allows later expansion of this
-  # > service to handle more events.
-  #
-  # https://docs.taskcluster.net/docs/reference/integrations/taskcluster-github/docs/taskcluster-yml-v1
-  else: []
+                - set -ex;
+                  ~/start.sh
+                    ${event.repository.clone_url}
+                    ${event.pull_request.base.ref}
+                    ${event.pull_request.base.sha}
+                    ${browser.name}-${browser.channel};
+                  cd ~/web-platform-tests;
+                  ./wpt manifest-download || true;
+                  git fetch
+                    ${event.repository.clone_url}
+                    refs/pull/${event.number}/head;
+                  git checkout FETCH_HEAD;
+                  affected=$(./wpt tests-affected
+                    --null
+                    ${event.pull_request.base.sha}
+                    | base64);
+                  echo Affected tests:;
+                  echo $affected | base64 -d | sed 's/\x0/\\n/g';
+                  ${operation.command}

dom/events/CustomEvent.html

@@ -4,6 +4,10 @@
 <script src="/resources/testharnessreport.js"></script>
 <div id="log"></div>
 <script>
+test(function() {
+  assert_true(Math.random() > 0.5);
+}, "Intentionally unstable test");
+
 test(function() {
   var type = "foo";
 

tools/ci/ci_taskcluster.sh

@@ -2,9 +2,39 @@
 
 ./wpt manifest-download
 
-if [ $1 == "firefox" ]; then
-    ./wpt run firefox --log-tbpl=../artifacts/log_tbpl.log --log-tbpl-level=info --log-wptreport=../artifacts/wpt_report.json --log-mach=- --this-chunk=$3 --total-chunks=$4 --test-type=$2 -y --install-browser --no-pause --no-restart-on-unexpected --reftest-internal --install-fonts --no-fail-on-unexpected
-elif [ $1 == "chrome" ]; then
-    ./wpt run chrome --log-tbpl=../artifacts/log_tbpl.log --log-tbpl-level=info --log-wptreport=../artifacts/wpt_report.json --log-mach=- --this-chunk=$3 --total-chunks=$4 --test-type=$2 -y  --no-pause --no-restart-on-unexpected --install-fonts --no-fail-on-unexpected
+# The first script argument that is not prefixed with a dash (`-`) is assumed
+# to be the name of the browser under test. This restricts the syntax available
+# to consumers: value-accepting options must be specified using the equals sign
+# (`=`).
+for argument in $@; do
+  if [ ${argument:0:1} == '-' ]; then
+    continue
+  fi
+
+  browser_name=$arg
+
+  break
+done
+
+browser_specific_args=''
+
+if [ $browser_name == 'firefox' ]; then
+  browser_specific_args='--install-browser --reftest-internal'
+fi
+
+./wpt run \
+  --log-tbpl=../artifacts/log_tbpl.log \
+  --log-tbpl-level=info \
+  --log-wptreport=../artifacts/wpt_report.json \
+  --log-mach=- \
+  -y \
+  --no-pause \
+  --no-restart-on-unexpected \
+  --install-fonts \
+  --no-fail-on-unexpected \
+  $browser_specific_args \
+  $@
+
+if [ -f ../artifacts/wpt_report.json ]; then
+  gzip ../artifacts/wpt_report.json
 fi
-gzip ../artifacts/wpt_report.json

tools/docker/Dockerfile

@@ -69,4 +69,4 @@ WORKDIR /home/test/
 
 COPY .bashrc /home/test/.bashrc
 
-COPY start.sh /home/test/start.sh
+COPY provision.sh /home/test/provision.sh