wpt: fail runner if expected failures don't fail (#1736)

nodejs · Oct 27, 2022 · 00a752f · 00a752f
1 parent 6883ba5
commit 00a752f
Show file tree

Hide file tree

Showing 3 changed files with 25 additions and 5 deletions.
diff --git a/test/wpt/runner/runner/runner.mjs b/test/wpt/runner/runner/runner.mjs
@@ -1,3 +1,4 @@
+import { deepStrictEqual } from 'node:assert'
 import { EventEmitter } from 'node:events'
 import { readdirSync, readFileSync, statSync } from 'node:fs'
 import { basename, isAbsolute, join, resolve } from 'node:path'
@@ -28,6 +29,9 @@ export class WPTRunner extends EventEmitter {
   /** @type {import('../../status/fetch.status.json')} */
   #status
 
+  /** Tests that have expectedly failed mapped by file name */
+  #statusOutput = {}
+
   #stats = {
     completed: 0,
     failed: 0,
@@ -138,6 +142,10 @@ export class WPTRunner extends EventEmitter {
         if (flaky?.includes(name)) {
           this.#stats.expectedFailures += 1
         } else if (allowUnexpectedFailures || fail?.includes(name)) {
+          if (!allowUnexpectedFailures) {
+            this.#statusOutput[fileName] ??= []
+            this.#statusOutput[fileName].push(name)
+          }
           this.#stats.expectedFailures += 1
         } else {
           process.exitCode = 1
@@ -161,6 +169,18 @@ export class WPTRunner extends EventEmitter {
    * Called after every test has completed.
    */
   handleRunnerCompletion () {
+    const expectedFailuresObject = Object.keys(this.#status).reduce((a, b) => {
+      if (Array.isArray(this.#status[b].fail)) {
+        a[b] = [...this.#status[b].fail]
+      }
+      return a
+    }, {})
+
+    deepStrictEqual(
+      this.#statusOutput,
+      expectedFailuresObject
+    )
+
     this.emit('completion')
     const { completed, failed, success, expectedFailures } = this.#stats
     console.log(

diff --git a/test/wpt/status/FileAPI.status.json b/test/wpt/status/FileAPI.status.json
@@ -6,8 +6,6 @@
 	},
 	"idlharness.any.js": {
 		"fail": [
-			"URL interface: operation revokeObjectURL(DOMString)",
-			"URL interface: operation createObjectURL((Blob or MediaSource))",
 			"Blob interface: attribute size",
 			"Blob interface: attribute type",
 			"Blob interface: operation slice(optional long long, optional long long, optional DOMString)",
@@ -21,7 +19,9 @@
 			"FileList interface: existence and properties of interface prototype object's \"constructor\" property",
 			"FileList interface: existence and properties of interface prototype object's @@unscopables property",
 			"FileList interface: operation item(unsigned long)",
-			"FileList interface: attribute length"
+			"FileList interface: attribute length",
+			"URL interface: operation createObjectURL((Blob or MediaSource))",
+			"URL interface: operation revokeObjectURL(DOMString)"
 		]
 	},
 	"filereader_events.any.js": {

diff --git a/test/wpt/status/fetch.status.json b/test/wpt/status/fetch.status.json
@@ -1,9 +1,9 @@
 {
 	"general.any.js": {
 		"fail": [
-			"Stream errors once aborted. Underlying connection closed.",
+			"Already aborted signal rejects immediately",
 			"Underlying connection is closed when aborting after receiving response - no-cors",
-			"Already aborted signal rejects immediately"
+			"Stream errors once aborted. Underlying connection closed."
 		]
 	},
 	"request-disturbed.any.js": {