Send ReSpec specs to spec-generator before looking for status (#1000)

Fixes #944. Note generation of ReSpec specs with JSDOM is still not possible in
practice, due to missing APIs in JSDOM required by ReSpec.

To make sure that the result would work for all WICG specs (in case
browser-specs becomes the source for Specref for WICG specs), I made a built
without retrieving info from Specref for WICG specs. Result looks good, and
would actually fix a couple of outdated titles and incorrect spec statuses.

specs.json

@@ -237,18 +237,8 @@
       ]
     }
   },
-  {
-    "url": "https://screen-share.github.io/captured-mouse-events/",
-    "nightly": {
-      "status": "Draft Community Group Report"
-    }
-  },
-  {
-    "url": "https://screen-share.github.io/element-capture/",
-    "nightly": {
-      "status": "Draft Community Group Report"
-    }
-  },
+  "https://screen-share.github.io/captured-mouse-events/",
+  "https://screen-share.github.io/element-capture/",
   {
     "url": "https://sourcemaps.info/spec.html",
     "shortname": "sourcemap",
@@ -401,12 +391,7 @@
   "https://wicg.github.io/datacue/",
   "https://wicg.github.io/deprecation-reporting/",
   "https://wicg.github.io/digital-goods/",
-  {
-    "url": "https://wicg.github.io/direct-sockets/",
-    "nightly": {
-      "status": "Unofficial Proposal Draft"
-    }
-  },
+  "https://wicg.github.io/direct-sockets/",
   "https://wicg.github.io/document-picture-in-picture/",
   "https://wicg.github.io/document-policy/",
   "https://wicg.github.io/element-timing/",

src/fetch-info.js

@@ -273,6 +273,12 @@ async function fetchInfoFromSpecs(specs, options) {
       }
     }
 
+    if (dom.window.document.head.querySelector("script[src*='respec']")) {
+      // Non-generated ReSpec spec, let's get the generated version from
+      // spec-generator
+      dom = await JSDOMFromURL(`https://labs.w3.org/spec-generator/?type=respec&url=${encodeURIComponent(url)}`);
+    }
+
     // Extract first heading when set
     let title = dom.window.document.querySelector("h1");
     if (!title) {
@@ -301,10 +307,15 @@ async function fetchInfoFromSpecs(specs, options) {
         ".head #subtitle",          // WHATWG specs
         ".head #living-standard"    // HTML spec
       ].join(","));
-      const match = subtitle?.textContent.match(/^\s*(.*?)(,| — Last Updated)? \d{1,2} \w+ \d{4}\s*$/);
-      const status = (match ? match[1] : "Editor's Draft")
+      const match = subtitle?.textContent.match(/^\s*(.+?)(,| — Last Updated)?\s+\d{1,2} \w+ \d{4}\s*$/);
+      let status = (match ? match[1] : "Editor's Draft")
         .replace(/’/g, "'")     // Bikeshed generates curly quotes
         .replace(/^W3C /, "");  // Once every full moon, a "W3C " prefix gets added
+      // And once every other full moon, spec has a weird status
+      // (e.g., https://privacycg.github.io/gpc-spec/)
+      if (status === "Proposal" || status === "Unofficial Draft") {
+        status = "Unofficial Proposal Draft";
+      }
       return {
         nightly: { url, status },
         title

test/fetch-info.js

@@ -85,15 +85,15 @@ describe("fetch-info module (without W3C API key)", function () {
 
     it("extracts spec info from a Respec spec when needed", async () => {
       const spec = {
-        url: "https://w3c.github.io/respec/examples/tpac_2019.html",
+        url: "https://screen-share.github.io/element-capture/",
         shortname: "respec"
       };
       const info = await fetchInfo([spec]);
       assert.ok(info[spec.shortname]);
       assert.equal(info[spec.shortname].source, "spec");
       assert.equal(info[spec.shortname].nightly.url, spec.url);
-      assert.equal(info[spec.shortname].nightly.status, "Editor's Draft");
-      assert.equal(info[spec.shortname].title, "TPAC 2019 - New Features");
+      assert.equal(info[spec.shortname].nightly.status, "Draft Community Group Report");
+      assert.equal(info[spec.shortname].title, "Element Capture");
     });
 
     it("extracts right title from an ECMAScript proposal spec", async () => {