From 7a127f71c4df1f1cc8bc2869055b182243c0ff9d Mon Sep 17 00:00:00 2001
From: Sidhartha Chatterjee <me@sidharthachatterjee.com>
Date: Wed, 2 Sep 2020 20:05:52 +0530
Subject: [PATCH 001/123] Prototype SSR wip

---
 .../src/index.js                              |   4 +
 .../gatsby/cache-dir/develop-static-entry.js  | 179 +++++++++++++++++-
 packages/gatsby/src/utils/start-server.ts     |  59 ++----
 3 files changed, 201 insertions(+), 41 deletions(-)

diff --git a/packages/babel-plugin-remove-graphql-queries/src/index.js b/packages/babel-plugin-remove-graphql-queries/src/index.js
index 68f9934fb1f9e..5c81bcf8528a2 100644
--- a/packages/babel-plugin-remove-graphql-queries/src/index.js
+++ b/packages/babel-plugin-remove-graphql-queries/src/index.js
@@ -189,6 +189,8 @@ export default function ({ types: t }) {
           JSXIdentifier(path2) {
             if (
               (process.env.NODE_ENV === `test` ||
+                state.opts.stage === `develop` ||
+                state.opts.stage === `develop-html` ||
                 state.opts.stage === `build-html`) &&
               path2.isJSXIdentifier({ name: `StaticQuery` }) &&
               path2.referencesImport(`gatsby`) &&
@@ -230,6 +232,8 @@ export default function ({ types: t }) {
           CallExpression(path2) {
             if (
               (process.env.NODE_ENV === `test` ||
+                state.opts.stage === `develop` ||
+                state.opts.stage === `develop-html` ||
                 state.opts.stage === `build-html`) &&
               isUseStaticQuery(path2)
             ) {
diff --git a/packages/gatsby/cache-dir/develop-static-entry.js b/packages/gatsby/cache-dir/develop-static-entry.js
index 9e245316e40f2..ba90ebd040a41 100644
--- a/packages/gatsby/cache-dir/develop-static-entry.js
+++ b/packages/gatsby/cache-dir/develop-static-entry.js
@@ -1,7 +1,14 @@
 import React from "react"
-import { renderToStaticMarkup } from "react-dom/server"
-import { merge } from "lodash"
+const fs = require(`fs`)
+import { renderToString, renderToStaticMarkup } from "react-dom/server"
+const { merge } = require(`lodash`)
+const { join } = require(`path`)
 import apiRunner from "./api-runner-ssr"
+const { grabMatchParams } = require(`./find-path`)
+const syncRequires = require(`$virtual/sync-requires`)
+
+const { RouteAnnouncerProps } = require(`./route-announcer-props`)
+const { ServerLocation, Router, isRedirect } = require(`@reach/router`)
 // import testRequireError from "./test-require-error"
 // For some extremely mysterious reason, webpack adds the above module *after*
 // this module so that when this code runs, testRequireError is undefined.
@@ -27,6 +34,7 @@ try {
 Html = Html && Html.__esModule ? Html.default : Html
 
 export default (pagePath, callback) => {
+  let bodyHtml = ``
   let headComponents = [
     <meta key="environment" name="note" content="environment=development" />,
   ]
@@ -67,6 +75,10 @@ export default (pagePath, callback) => {
     headComponents = components
   }
 
+  const replaceBodyHTMLString = body => {
+    bodyHtml = body
+  }
+
   const getPreBodyComponents = () => preBodyComponents
 
   const replacePreBodyComponents = components => {
@@ -79,6 +91,167 @@ export default (pagePath, callback) => {
     postBodyComponents = components
   }
 
+  const getPageDataPath = path => {
+    const fixedPagePath = path === `/` ? `index` : path
+    return join(`page-data`, fixedPagePath, `page-data.json`)
+  }
+
+  // const getPageDataUrl = pagePath => {
+  //   const pageDataPath = getPageDataPath(pagePath)
+  //   return `${__PATH_PREFIX__}/${pageDataPath}`
+  // }
+
+  const getPageData = pagePath => {
+    const pageDataPath = getPageDataPath(pagePath)
+    const absolutePageDataPath = join(process.cwd(), `public`, pageDataPath)
+    const pageDataRaw = fs.readFileSync(absolutePageDataPath)
+
+    try {
+      return JSON.parse(pageDataRaw.toString())
+    } catch (err) {
+      return null
+    }
+  }
+
+  // const getStaticQueryPath = hash => `page-data/sq/d/${hash}.json`
+
+  // const getStaticQueryResults = staticQueryHashes =>
+  //   staticQueryHashes.map(hash => {
+  //     const absoluteStaticQueryDataPath = join(
+  //       process.cwd(),
+  //       `public`,
+  //       getStaticQueryPath(hash)
+  //     )
+  //     try {
+  //       return JSON.parse(
+  //         fs.readFileSync(absoluteStaticQueryDataPath).toString()
+  //       )
+  //     } catch (error) {
+  //       console.log(error)
+  //     }
+  //   })
+
+  // const appDataPath = join(`page-data`, `app-data.json`)
+
+  // const getAppDataUrl = memoize(() => {
+  //   let appData
+
+  //   try {
+  //     const absoluteAppDataPath = join(process.cwd(), `public`, appDataPath)
+  //     const appDataRaw = fs.readFileSync(absoluteAppDataPath)
+  //     appData = JSON.parse(appDataRaw.toString())
+
+  //     if (!appData) {
+  //       return null
+  //     }
+  //   } catch (err) {
+  //     return null
+  //   }
+
+  //   return `${__PATH_PREFIX__}/${appDataPath}`
+  // })
+
+  const pageData = getPageData(pagePath)
+  // const pageDataUrl = getPageDataUrl(pagePath)
+
+  // const appDataUrl = getAppDataUrl()
+
+  const {
+    componentChunkName,
+    //  staticQueryHashes = []
+  } = pageData
+
+  // const staticQueryData = getStaticQueryResults(staticQueryHashes)
+
+  // console.log({
+  //   staticQueryData,
+  //   pageData,
+  // })
+
+  // const pageDataResult = {
+  //   ...pageData,
+  //   staticQueryResults: staticQueryData,
+  // }
+
+  const createElement = React.createElement
+
+  class RouteHandler extends React.Component {
+    render() {
+      const props = {
+        ...this.props,
+        ...pageData.result,
+        params: {
+          ...grabMatchParams(this.props.location.pathname),
+          ...(pageData.result?.pageContext?.__params || {}),
+        },
+        // pathContext was deprecated in v2. Renamed to pageContext
+        pathContext: pageData.result ? pageData.result.pageContext : undefined,
+      }
+
+      const pageElement = createElement(
+        syncRequires.components[componentChunkName],
+        props
+      )
+
+      // console.log({
+      //   element: syncRequires.components[componentChunkName].toString(),
+      // })
+
+      const wrappedPage = apiRunner(
+        `wrapPageElement`,
+        { element: pageElement, props },
+        pageElement,
+        ({ result }) => {
+          return { element: result, props }
+        }
+      ).pop()
+
+      return wrappedPage
+    }
+  }
+
+  const routerElement = (
+    <ServerLocation url={`${__BASE_PATH__}${pagePath}`}>
+      <Router id="gatsby-focus-wrapper" baseuri={__BASE_PATH__}>
+        <RouteHandler path="/*" />
+      </Router>
+      <div {...RouteAnnouncerProps} />
+    </ServerLocation>
+  )
+
+  const bodyComponent = apiRunner(
+    `wrapRootElement`,
+    { element: routerElement, pathname: pagePath },
+    routerElement,
+    ({ result }) => {
+      return { element: result, pathname: pagePath }
+    }
+  ).pop()
+
+  // Let the site or plugin render the page component.
+  apiRunner(`replaceRenderer`, {
+    bodyComponent,
+    replaceBodyHTMLString,
+    setHeadComponents,
+    setHtmlAttributes,
+    setBodyAttributes,
+    setPreBodyComponents,
+    setPostBodyComponents,
+    setBodyProps,
+    pathname: pagePath,
+    pathPrefix: __PATH_PREFIX__,
+  })
+
+  // If no one stepped up, we'll handle it.
+  if (!bodyHtml) {
+    try {
+      bodyHtml = renderToString(bodyComponent)
+    } catch (e) {
+      // ignore @reach/router redirect errors
+      if (!isRedirect(e)) throw e
+    }
+  }
+
   apiRunner(`onRenderBody`, {
     setHeadComponents,
     setHtmlAttributes,
@@ -101,7 +274,7 @@ export default (pagePath, callback) => {
 
   const htmlElement = React.createElement(Html, {
     ...bodyProps,
-    body: ``,
+    body: bodyHtml,
     headComponents: headComponents.concat([
       <script key={`io`} src="/socket.io/socket.io.js" />,
     ]),
diff --git a/packages/gatsby/src/utils/start-server.ts b/packages/gatsby/src/utils/start-server.ts
index 3d4dd6919028b..a7e311aad4ea1 100644
--- a/packages/gatsby/src/utils/start-server.ts
+++ b/packages/gatsby/src/utils/start-server.ts
@@ -64,40 +64,8 @@ export async function startServer(
   app: Express,
   workerPool: JestWorker = WorkerPool.create()
 ): Promise<IServer> {
-  const indexHTMLActivity = report.phantomActivity(`building index.html`, {})
-  indexHTMLActivity.start()
   const directory = program.directory
   const directoryPath = withBasePath(directory)
-  const createIndexHtml = async (activity: ActivityTracker): Promise<void> => {
-    try {
-      await buildHTML({
-        program,
-        stage: Stage.DevelopHTML,
-        pagePaths: [`/`],
-        workerPool,
-        activity,
-      })
-    } catch (err) {
-      if (err.name !== `WebpackError`) {
-        report.panic(err)
-        return
-      }
-      report.panic(
-        report.stripIndent`
-          There was an error compiling the html.js component for the development server.
-
-          See our docs page on debugging HTML builds for help https://gatsby.dev/debug-html
-        `,
-        err
-      )
-    }
-  }
-
-  await createIndexHtml(indexHTMLActivity)
-
-  indexHTMLActivity.end()
-
-  // report.stateUpdate(`webpack`, `IN_PROGRESS`)
 
   const webpackActivity = report.activityTimer(`Building development bundle`, {
     id: `webpack-develop`,
@@ -281,12 +249,27 @@ export async function startServer(
   })
 
   // Render an HTML page and serve it.
-  app.use((_, res) => {
-    res.sendFile(directoryPath(`public/index.html`), err => {
-      if (err) {
-        res.status(500).end()
-      }
+  app.use(async (req, res) => {
+    // res.sendFile(directoryPath(`public/index.html`), err => {
+    //   if (err) {
+    //     res.status(500).end()
+    //   }
+    // })
+
+    const htmlActivity = report.phantomActivity(`building index.html`, {})
+    htmlActivity.start()
+
+    const responseHtml = await buildHTML({
+      program,
+      stage: Stage.DevelopHTML,
+      pagePaths: [req.path],
+      workerPool,
+      activity: htmlActivity,
     })
+
+    res.status(200).send(responseHtml)
+
+    htmlActivity.end()
   })
 
   /**
@@ -306,7 +289,7 @@ export async function startServer(
   )
 
   chokidar.watch(watchGlobs).on(`change`, async () => {
-    await createIndexHtml(indexHTMLActivity)
+    // await createIndexHtml(indexHTMLActivity)
     // eslint-disable-next-line no-unused-expressions
     socket?.to(`clients`).emit(`reload`)
   })

From 3f920e3482c1c34bc4775e20b9c166a3caa2e442 Mon Sep 17 00:00:00 2001
From: Sidhartha Chatterjee <me@sidharthachatterjee.com>
Date: Thu, 3 Sep 2020 14:46:10 +0530
Subject: [PATCH 002/123] Make it work

---
 packages/gatsby/src/commands/build-html.ts    |  2 +-
 packages/gatsby/src/utils/start-server.ts     | 64 +++++++++++++------
 .../gatsby/src/utils/worker/render-html.ts    | 16 ++---
 3 files changed, 52 insertions(+), 30 deletions(-)

diff --git a/packages/gatsby/src/commands/build-html.ts b/packages/gatsby/src/commands/build-html.ts
index ea245dd303b7a..62596232ff9e2 100644
--- a/packages/gatsby/src/commands/build-html.ts
+++ b/packages/gatsby/src/commands/build-html.ts
@@ -40,7 +40,7 @@ const doBuildRenderer = async (
   return `${directory}/public/render-page.js`
 }
 
-const buildRenderer = async (
+export const buildRenderer = async (
   program: IProgram,
   stage: Stage,
   parentSpan: IActivity
diff --git a/packages/gatsby/src/utils/start-server.ts b/packages/gatsby/src/utils/start-server.ts
index a7e311aad4ea1..e23f4c5e70199 100644
--- a/packages/gatsby/src/utils/start-server.ts
+++ b/packages/gatsby/src/utils/start-server.ts
@@ -14,13 +14,14 @@ import { formatError } from "graphql"
 
 import webpackConfig from "../utils/webpack.config"
 import { store, emitter } from "../redux"
-import { buildHTML } from "../commands/build-html"
+import { buildHTML, buildRenderer } from "../commands/build-html"
 import { withBasePath } from "../utils/path"
 import report from "gatsby-cli/lib/reporter"
 import launchEditor from "react-dev-utils/launchEditor"
 import cors from "cors"
 import telemetry from "gatsby-telemetry"
 import * as WorkerPool from "../utils/worker/pool"
+import { renderHTML } from "../utils/worker/render-html"
 import http from "http"
 import https from "https"
 
@@ -181,12 +182,6 @@ export async function startServer(
     res.end()
   })
 
-  // Disable directory indexing i.e. serving index.html from a directory.
-  // This can lead to serving stale html files during development.
-  //
-  // We serve by default an empty index.html that sets up the dev environment.
-  app.use(developStatic(`public`, { index: false }))
-
   const webpackDevMiddlewareInstance = (webpackDevMiddleware(compiler, {
     logLevel: `silent`,
     publicPath: devConfig.output.publicPath,
@@ -248,30 +243,55 @@ export async function startServer(
     res.status(404).end()
   })
 
+  const buildRendererActivity = report.activityTimer(
+    `Building renderer bundle`,
+    {
+      id: `webpack-renderer`,
+    }
+  )
+
+  await buildRenderer(
+    program,
+    Stage.DevelopHTML,
+    buildRendererActivity.span
+  ).then(console.log)
+
   // Render an HTML page and serve it.
-  app.use(async (req, res) => {
-    // res.sendFile(directoryPath(`public/index.html`), err => {
-    //   if (err) {
-    //     res.status(500).end()
-    //   }
-    // })
+  app.use(async (req, res, next) => {
+    const { pages } = store.getState()
+
+    if (!pages.has(req.path)) {
+      return next()
+    }
 
     const htmlActivity = report.phantomActivity(`building index.html`, {})
     htmlActivity.start()
 
-    const responseHtml = await buildHTML({
-      program,
-      stage: Stage.DevelopHTML,
-      pagePaths: [req.path],
-      workerPool,
-      activity: htmlActivity,
+    const [response] = await renderHTML({
+      htmlComponentRendererPath: `${program.directory}/public/render-page.js`,
+      paths: [req.path],
+      envVars: [
+        [`NODE_ENV`, process.env.NODE_ENV || ``],
+        [
+          `gatsby_executing_command`,
+          process.env.gatsby_executing_command || ``,
+        ],
+        [`gatsby_log_level`, process.env.gatsby_log_level || ``],
+      ],
     })
 
-    res.status(200).send(responseHtml)
+    res.status(200).send(response)
 
     htmlActivity.end()
+    return null
   })
 
+  // Disable directory indexing i.e. serving index.html from a directory.
+  // This can lead to serving stale html files during development.
+  //
+  // We serve by default an empty index.html that sets up the dev environment.
+  app.use(developStatic(`public`, { index: false }))
+
   /**
    * Set up the HTTP server and socket.io.
    **/
@@ -289,7 +309,9 @@ export async function startServer(
   )
 
   chokidar.watch(watchGlobs).on(`change`, async () => {
-    // await createIndexHtml(indexHTMLActivity)
+    // console.log(`Time to build a renderer`)
+    // await buildRenderer(program, Stage.DevelopHTML, webpackActivity)
+    // console.log(`We built a renderer`)
     // eslint-disable-next-line no-unused-expressions
     socket?.to(`clients`).emit(`reload`)
   })
diff --git a/packages/gatsby/src/utils/worker/render-html.ts b/packages/gatsby/src/utils/worker/render-html.ts
index a63ddf150ac3d..94e54430f3b6f 100644
--- a/packages/gatsby/src/utils/worker/render-html.ts
+++ b/packages/gatsby/src/utils/worker/render-html.ts
@@ -9,8 +9,8 @@ export const renderHTML = ({
   envVars,
 }: {
   htmlComponentRendererPath: string
-  paths: string
-  envVars: [string, string][]
+  paths: string[]
+  envVars: string[][]
 }): Promise<unknown[]> => {
   // This is being executed in child process, so we need to set some vars
   // for modules that aren't bundled by webpack.
@@ -23,12 +23,12 @@ export const renderHTML = ({
         const htmlComponentRenderer = require(htmlComponentRendererPath)
         try {
           htmlComponentRenderer.default(path, (_throwAway, htmlString) => {
-            resolve(
-              fs.outputFile(
-                getPageHtmlFilePath(join(process.cwd(), `public`), path),
-                htmlString
-              )
-            )
+            // resolve(
+            //   fs.outputFile(
+            //     getPageHtmlFilePath(join(process.cwd(), `public`), path),
+            resolve(htmlString)
+            // )
+            // )
           })
         } catch (e) {
           // add some context to error so we can display more helpful message

From 0d614dde4a104ca7ec1c14d3b32c7117f1e84105 Mon Sep 17 00:00:00 2001
From: Sidhartha Chatterjee <me@sidharthachatterjee.com>
Date: Thu, 3 Sep 2020 16:59:34 +0530
Subject: [PATCH 003/123] Fix issue with running two instances of webpack

---
 packages/gatsby/src/commands/build-html.ts    |  2 +-
 .../src/services/start-webpack-server.ts      | 11 ++++-
 packages/gatsby/src/utils/start-server.ts     | 49 +++++++++----------
 3 files changed, 33 insertions(+), 29 deletions(-)

diff --git a/packages/gatsby/src/commands/build-html.ts b/packages/gatsby/src/commands/build-html.ts
index 62596232ff9e2..2b20b56dabd0b 100644
--- a/packages/gatsby/src/commands/build-html.ts
+++ b/packages/gatsby/src/commands/build-html.ts
@@ -43,7 +43,7 @@ const doBuildRenderer = async (
 export const buildRenderer = async (
   program: IProgram,
   stage: Stage,
-  parentSpan: IActivity
+  parentSpan?: IActivity
 ): Promise<string> => {
   const { directory } = program
   const config = await webpackConfig(program, directory, stage, null, {
diff --git a/packages/gatsby/src/services/start-webpack-server.ts b/packages/gatsby/src/services/start-webpack-server.ts
index 60362ab084420..ec5a2cacc2c64 100644
--- a/packages/gatsby/src/services/start-webpack-server.ts
+++ b/packages/gatsby/src/services/start-webpack-server.ts
@@ -46,6 +46,7 @@ export async function startWebpackServer({
   webpackWatching.suspend()
 
   compiler.hooks.invalid.tap(`log compiling`, function () {
+    console.log(`log compiling`)
     if (!webpackActivity) {
       // mark webpack as pending if we are not in the middle of compilation already
       // when input is invalidated during compilation, webpack will automatically
@@ -56,6 +57,7 @@ export async function startWebpackServer({
   })
 
   compiler.hooks.watchRun.tapAsync(`log compiling`, function (_, done) {
+    console.log(`watchRun`)
     if (!webpackActivity) {
       // there can be multiple `watchRun` events before receiving single `done` event
       // webpack will not emit assets or `done` event until all pending invalidated
@@ -71,11 +73,17 @@ export async function startWebpackServer({
 
   let isFirstCompile = true
 
-  return new Promise(resolve => {
+  console.log(`return`)
+
+  return new Promise((resolve, reject) => {
+    console.log({
+      compiler,
+    })
     compiler.hooks.done.tapAsync(`print gatsby instructions`, async function (
       stats,
       done
     ) {
+      console.log(`done`)
       // "done" event fires when Webpack has finished recompiling the bundle.
       // Whether or not you have warnings or errors, you will get this event.
 
@@ -163,6 +171,7 @@ export async function startWebpackServer({
       markWebpackStatusAsDone()
       done()
       emitter.emit(`COMPILATION_DONE`, stats)
+      console.log(`about to resolve`)
       resolve({ compiler, websocketManager, webpackWatching })
     })
   })
diff --git a/packages/gatsby/src/utils/start-server.ts b/packages/gatsby/src/utils/start-server.ts
index e23f4c5e70199..3069035724827 100644
--- a/packages/gatsby/src/utils/start-server.ts
+++ b/packages/gatsby/src/utils/start-server.ts
@@ -81,6 +81,8 @@ export async function startServer(
     { parentSpan: webpackActivity.span }
   )
 
+  await buildRenderer(program, Stage.DevelopHTML)
+
   const compiler = webpack(devConfig)
 
   /**
@@ -243,18 +245,12 @@ export async function startServer(
     res.status(404).end()
   })
 
-  const buildRendererActivity = report.activityTimer(
-    `Building renderer bundle`,
-    {
-      id: `webpack-renderer`,
-    }
-  )
-
-  await buildRenderer(
-    program,
-    Stage.DevelopHTML,
-    buildRendererActivity.span
-  ).then(console.log)
+  // const buildRendererActivity = report.activityTimer(
+  //   `Building renderer bundle`,
+  //   {
+  //     id: `webpack-renderer`,
+  //   }
+  // )
 
   // Render an HTML page and serve it.
   app.use(async (req, res, next) => {
@@ -264,8 +260,8 @@ export async function startServer(
       return next()
     }
 
-    const htmlActivity = report.phantomActivity(`building index.html`, {})
-    htmlActivity.start()
+    // const htmlActivity = report.phantomActivity(`building index.html`, {})
+    // htmlActivity.start()
 
     const [response] = await renderHTML({
       htmlComponentRendererPath: `${program.directory}/public/render-page.js`,
@@ -282,8 +278,7 @@ export async function startServer(
 
     res.status(200).send(response)
 
-    htmlActivity.end()
-    return null
+    // htmlActivity.end()
   })
 
   // Disable directory indexing i.e. serving index.html from a directory.
@@ -304,17 +299,17 @@ export async function startServer(
   const listener = server.listen(program.port, `localhost`)
 
   // Register watcher that rebuilds index.html every time html.js changes.
-  const watchGlobs = [`src/html.js`, `plugins/**/gatsby-ssr.js`].map(path =>
-    slash(directoryPath(path))
-  )
-
-  chokidar.watch(watchGlobs).on(`change`, async () => {
-    // console.log(`Time to build a renderer`)
-    // await buildRenderer(program, Stage.DevelopHTML, webpackActivity)
-    // console.log(`We built a renderer`)
-    // eslint-disable-next-line no-unused-expressions
-    socket?.to(`clients`).emit(`reload`)
-  })
+  // const watchGlobs = [`src/html.js`, `plugins/**/gatsby-ssr.js`].map(path =>
+  //   slash(directoryPath(path))
+  // )
+
+  // chokidar.watch(watchGlobs).on(`change`, async () => {
+  //   // console.log(`Time to build a renderer`)
+  //   // await buildRenderer(program, Stage.DevelopHTML, webpackActivity)
+  //   // console.log(`We built a renderer`)
+  //   // eslint-disable-next-line no-unused-expressions
+  //   socket?.to(`clients`).emit(`reload`)
+  // })
 
   return {
     compiler,

From 4d073f5793df103a46daf5497cff3ea03bf69748 Mon Sep 17 00:00:00 2001
From: Sidhartha Chatterjee <me@sidharthachatterjee.com>
Date: Tue, 8 Sep 2020 15:45:56 +0530
Subject: [PATCH 004/123] Add a test suite for SSR

---
 integration-tests/ssr/LICENSE                 | 21 ++++++++++++
 integration-tests/ssr/README.md               |  1 +
 .../ssr/__tests__/__snapshots__/ssr.js.snap   |  3 ++
 integration-tests/ssr/__tests__/ssr.js        |  9 +++++
 integration-tests/ssr/gatsby-config.js        | 10 ++++++
 integration-tests/ssr/jest.config.js          |  3 ++
 integration-tests/ssr/package.json            | 34 +++++++++++++++++++
 integration-tests/ssr/src/pages/index.js      | 15 ++++++++
 packages/gatsby/cache-dir/app.js              |  2 +-
 packages/gatsby/package.json                  |  2 +-
 .../src/services/start-webpack-server.ts      | 11 +-----
 packages/gatsby/src/utils/page-data.ts        |  4 +--
 packages/gatsby/src/utils/start-server.ts     |  4 +--
 13 files changed, 102 insertions(+), 17 deletions(-)
 create mode 100644 integration-tests/ssr/LICENSE
 create mode 100644 integration-tests/ssr/README.md
 create mode 100644 integration-tests/ssr/__tests__/__snapshots__/ssr.js.snap
 create mode 100644 integration-tests/ssr/__tests__/ssr.js
 create mode 100644 integration-tests/ssr/gatsby-config.js
 create mode 100644 integration-tests/ssr/jest.config.js
 create mode 100644 integration-tests/ssr/package.json
 create mode 100644 integration-tests/ssr/src/pages/index.js

diff --git a/integration-tests/ssr/LICENSE b/integration-tests/ssr/LICENSE
new file mode 100644
index 0000000000000..20f91f2b3c52b
--- /dev/null
+++ b/integration-tests/ssr/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 gatsbyjs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/integration-tests/ssr/README.md b/integration-tests/ssr/README.md
new file mode 100644
index 0000000000000..7aca9ce8f432c
--- /dev/null
+++ b/integration-tests/ssr/README.md
@@ -0,0 +1 @@
+## SSR test suite
diff --git a/integration-tests/ssr/__tests__/__snapshots__/ssr.js.snap b/integration-tests/ssr/__tests__/__snapshots__/ssr.js.snap
new file mode 100644
index 0000000000000..050d55e99b7d6
--- /dev/null
+++ b/integration-tests/ssr/__tests__/__snapshots__/ssr.js.snap
@@ -0,0 +1,3 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`SSR is run for a page when it is requested 1`] = `"<!DOCTYPE html><html><head><meta charSet=\\"utf-8\\"/><meta http-equiv=\\"x-ua-compatible\\" content=\\"ie=edge\\"/><meta name=\\"viewport\\" content=\\"width=device-width, initial-scale=1, shrink-to-fit=no\\"/><meta name=\\"note\\" content=\\"environment=development\\"/><script src=\\"/socket.io/socket.io.js\\"></script></head><body><div id=\\"___gatsby\\"><div style=\\"outline:none\\" tabindex=\\"-1\\" id=\\"gatsby-focus-wrapper\\"><div>Hello world</div></div><div id=\\"gatsby-announcer\\" style=\\"position:absolute;top:0;width:1px;height:1px;padding:0;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;border:0\\" aria-live=\\"assertive\\" aria-atomic=\\"true\\"></div></div><script src=\\"/polyfill.js\\" nomodule=\\"\\"></script><script src=\\"/commons.js\\"></script></body></html>"`;
diff --git a/integration-tests/ssr/__tests__/ssr.js b/integration-tests/ssr/__tests__/ssr.js
new file mode 100644
index 0000000000000..66c7d855d87b4
--- /dev/null
+++ b/integration-tests/ssr/__tests__/ssr.js
@@ -0,0 +1,9 @@
+const fetch = require(`node-fetch`)
+
+describe(`SSR`, () => {
+  test(`is run for a page when it is requested`, async () => {
+    const html = await fetch(`http://localhost:8000/`).then(res => res.text())
+
+    expect(html).toMatchSnapshot()
+  })
+})
diff --git a/integration-tests/ssr/gatsby-config.js b/integration-tests/ssr/gatsby-config.js
new file mode 100644
index 0000000000000..cc785a9507538
--- /dev/null
+++ b/integration-tests/ssr/gatsby-config.js
@@ -0,0 +1,10 @@
+module.exports = {
+  siteMetadata: {
+    title: `Hello world`,
+    author: `Sid Chatterjee`,
+    twitter: `chatsidhartha`,
+    github: `sidharthachatterjee`,
+    moreInfo: `Sid is amazing`,
+  },
+  plugins: [],
+}
diff --git a/integration-tests/ssr/jest.config.js b/integration-tests/ssr/jest.config.js
new file mode 100644
index 0000000000000..4e5a78b25d7bf
--- /dev/null
+++ b/integration-tests/ssr/jest.config.js
@@ -0,0 +1,3 @@
+module.exports = {
+  testPathIgnorePatterns: [`/node_modules/`, `__tests__/fixtures`, `.cache`],
+}
diff --git a/integration-tests/ssr/package.json b/integration-tests/ssr/package.json
new file mode 100644
index 0000000000000..6dc86f8c0ddd4
--- /dev/null
+++ b/integration-tests/ssr/package.json
@@ -0,0 +1,34 @@
+{
+  "name": "ssr",
+  "private": true,
+  "author": "Sid Chatterjee",
+  "description": "A simplified bare-bones starter for Gatsby",
+  "version": "0.1.0",
+  "license": "MIT",
+  "scripts": {
+    "build": "gatsby build",
+    "develop": "gatsby develop",
+    "serve": "gatsby serve",
+    "clean": "gatsby clean",
+    "test:jest": "jest --config=jest.config.js --runInBand",
+    "test": "start-server-and-test develop http://localhost:8000 test:jest"
+  },
+  "dependencies": {
+    "gatsby": "2.24.49-dev-1599559013196",
+    "react": "^16.12.0",
+    "react-dom": "^16.12.0"
+  },
+  "devDependencies": {
+    "fs-extra": "^9.0.0",
+    "jest": "^24.0.0",
+    "jest-cli": "^24.0.0",
+    "start-server-and-test": "^1.11.3"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/gatsbyjs/gatsby-starter-hello-world"
+  },
+  "bugs": {
+    "url": "https://github.com/gatsbyjs/gatsby/issues"
+  }
+}
diff --git a/integration-tests/ssr/src/pages/index.js b/integration-tests/ssr/src/pages/index.js
new file mode 100644
index 0000000000000..850e99d4638bb
--- /dev/null
+++ b/integration-tests/ssr/src/pages/index.js
@@ -0,0 +1,15 @@
+import React from "react"
+import { useStaticQuery, graphql } from "gatsby"
+
+export default function Inline() {
+  const { site } = useStaticQuery(graphql`
+    {
+      site {
+        siteMetadata {
+          title
+        }
+      }
+    }
+  `)
+  return <div>{site.siteMetadata.title}</div>
+}
diff --git a/packages/gatsby/cache-dir/app.js b/packages/gatsby/cache-dir/app.js
index e0f8dd419487f..d2279d87a15fd 100644
--- a/packages/gatsby/cache-dir/app.js
+++ b/packages/gatsby/cache-dir/app.js
@@ -89,7 +89,7 @@ apiRunnerAsync(`onClientEntry`).then(() => {
   const renderer = apiRunner(
     `replaceHydrateFunction`,
     undefined,
-    ReactDOM.render
+    ReactDOM.hydrate
   )[0]
 
   Promise.all([
diff --git a/packages/gatsby/package.json b/packages/gatsby/package.json
index 2e29c64661ac6..b23cd3542bf47 100644
--- a/packages/gatsby/package.json
+++ b/packages/gatsby/package.json
@@ -228,7 +228,7 @@
     "remark-mdxjs": "^2.0.0-next.3"
   },
   "scripts": {
-    "build": "npm run build:types && npm run build:src && npm run build:internal-plugins && npm run build:rawfiles && npm run build:cjs",
+    "build": "npm run build:src && npm run build:internal-plugins && npm run build:rawfiles && npm run build:cjs",
     "postbuild": "node scripts/output-api-file.js && yarn workspace gatsby-admin build",
     "build:internal-plugins": "copyfiles -u 1 src/internal-plugins/**/package.json dist",
     "build:rawfiles": "copyfiles -u 1 src/internal-plugins/**/raw_* dist",
diff --git a/packages/gatsby/src/services/start-webpack-server.ts b/packages/gatsby/src/services/start-webpack-server.ts
index ec5a2cacc2c64..60362ab084420 100644
--- a/packages/gatsby/src/services/start-webpack-server.ts
+++ b/packages/gatsby/src/services/start-webpack-server.ts
@@ -46,7 +46,6 @@ export async function startWebpackServer({
   webpackWatching.suspend()
 
   compiler.hooks.invalid.tap(`log compiling`, function () {
-    console.log(`log compiling`)
     if (!webpackActivity) {
       // mark webpack as pending if we are not in the middle of compilation already
       // when input is invalidated during compilation, webpack will automatically
@@ -57,7 +56,6 @@ export async function startWebpackServer({
   })
 
   compiler.hooks.watchRun.tapAsync(`log compiling`, function (_, done) {
-    console.log(`watchRun`)
     if (!webpackActivity) {
       // there can be multiple `watchRun` events before receiving single `done` event
       // webpack will not emit assets or `done` event until all pending invalidated
@@ -73,17 +71,11 @@ export async function startWebpackServer({
 
   let isFirstCompile = true
 
-  console.log(`return`)
-
-  return new Promise((resolve, reject) => {
-    console.log({
-      compiler,
-    })
+  return new Promise(resolve => {
     compiler.hooks.done.tapAsync(`print gatsby instructions`, async function (
       stats,
       done
     ) {
-      console.log(`done`)
       // "done" event fires when Webpack has finished recompiling the bundle.
       // Whether or not you have warnings or errors, you will get this event.
 
@@ -171,7 +163,6 @@ export async function startWebpackServer({
       markWebpackStatusAsDone()
       done()
       emitter.emit(`COMPILATION_DONE`, stats)
-      console.log(`about to resolve`)
       resolve({ compiler, websocketManager, webpackWatching })
     })
   })
diff --git a/packages/gatsby/src/utils/page-data.ts b/packages/gatsby/src/utils/page-data.ts
index 23f7a6fa269cd..5facd539968fa 100644
--- a/packages/gatsby/src/utils/page-data.ts
+++ b/packages/gatsby/src/utils/page-data.ts
@@ -171,10 +171,10 @@ export async function flush(): Promise<void> {
   return
 }
 
-export function enqueueFlush(): void {
+export async function enqueueFlush(): Promise<void> {
   if (isWebpackStatusPending()) {
     isFlushPending = true
   } else {
-    flush()
+    await flush()
   }
 }
diff --git a/packages/gatsby/src/utils/start-server.ts b/packages/gatsby/src/utils/start-server.ts
index 3069035724827..acedab19cbf11 100644
--- a/packages/gatsby/src/utils/start-server.ts
+++ b/packages/gatsby/src/utils/start-server.ts
@@ -1,5 +1,3 @@
-import chokidar from "chokidar"
-
 import webpackHotMiddleware from "webpack-hot-middleware"
 import webpackDevMiddleware, {
   WebpackDevMiddleware,
@@ -276,7 +274,7 @@ export async function startServer(
       ],
     })
 
-    res.status(200).send(response)
+    return res.status(200).send(response)
 
     // htmlActivity.end()
   })

From 57a95ed526e4ebc19c9f003e4d2b84429c18d269 Mon Sep 17 00:00:00 2001
From: Sidhartha Chatterjee <me@sidharthachatterjee.com>
Date: Wed, 9 Sep 2020 15:20:18 +0530
Subject: [PATCH 005/123] Linting

---
 packages/gatsby/src/utils/start-server.ts       | 6 ++----
 packages/gatsby/src/utils/worker/render-html.ts | 6 +++---
 2 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/packages/gatsby/src/utils/start-server.ts b/packages/gatsby/src/utils/start-server.ts
index acedab19cbf11..94609affa9c19 100644
--- a/packages/gatsby/src/utils/start-server.ts
+++ b/packages/gatsby/src/utils/start-server.ts
@@ -12,7 +12,7 @@ import { formatError } from "graphql"
 
 import webpackConfig from "../utils/webpack.config"
 import { store, emitter } from "../redux"
-import { buildHTML, buildRenderer } from "../commands/build-html"
+import { buildRenderer } from "../commands/build-html"
 import { withBasePath } from "../utils/path"
 import report from "gatsby-cli/lib/reporter"
 import launchEditor from "react-dev-utils/launchEditor"
@@ -26,7 +26,6 @@ import https from "https"
 import { developStatic } from "../commands/develop-static"
 import withResolverContext from "../schema/context"
 import { websocketManager, WebsocketManager } from "../utils/websocket-manager"
-import { slash } from "gatsby-core-utils"
 import apiRunnerNode from "../utils/api-runner-node"
 import { Express } from "express"
 
@@ -64,7 +63,6 @@ export async function startServer(
   workerPool: JestWorker = WorkerPool.create()
 ): Promise<IServer> {
   const directory = program.directory
-  const directoryPath = withBasePath(directory)
 
   const webpackActivity = report.activityTimer(`Building development bundle`, {
     id: `webpack-develop`,
@@ -290,7 +288,7 @@ export async function startServer(
    **/
   const server = new http.Server(app)
 
-  const socket = websocketManager.init({ server, directory: program.directory })
+  // const socket = websocketManager.init({ server, directory: program.directory })
 
   // hardcoded `localhost`, because host should match `target` we set
   // in http proxy in `develop-proxy`
diff --git a/packages/gatsby/src/utils/worker/render-html.ts b/packages/gatsby/src/utils/worker/render-html.ts
index f353b222f3843..f286157a24104 100644
--- a/packages/gatsby/src/utils/worker/render-html.ts
+++ b/packages/gatsby/src/utils/worker/render-html.ts
@@ -1,7 +1,7 @@
-import fs from "fs-extra"
+// import fs from "fs-extra"
 import Promise from "bluebird"
-import { join } from "path"
-import { getPageHtmlFilePath } from "../../utils/page-html"
+// import { join } from "path"
+// import { getPageHtmlFilePath } from "../../utils/page-html"
 
 export const renderHTML = ({
   htmlComponentRendererPath,

From 19cf373f52b5c06873dc96e1af7246664a4c84d0 Mon Sep 17 00:00:00 2001
From: Sidhartha Chatterjee <me@sidharthachatterjee.com>
Date: Wed, 9 Sep 2020 15:30:26 +0530
Subject: [PATCH 006/123] Run tests in CI

---
 .circleci/config.yml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 7c2f91806e8a0..37879b3322c75 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -282,6 +282,12 @@ jobs:
       - e2e-test:
           test_path: integration-tests/artifacts
 
+  integration_tests_ssr:
+    executor: node
+    steps:
+      - e2e-test:
+          test_path: integration-tests/ssr
+
   e2e_tests_path-prefix:
     <<: *e2e-executor
     environment:
@@ -580,6 +586,8 @@ workflows:
           <<: *e2e-test-workflow
       - integration_tests_artifacts:
           <<: *e2e-test-workflow
+      - integration_tests_ssr:
+          <<: *e2e-test-workflow
       - integration_tests_gatsby_cli:
           requires:
             - bootstrap

From 655b6858215245de6ed895d956c2a8bbe3872172 Mon Sep 17 00:00:00 2001
From: Sidhartha Chatterjee <me@sidharthachatterjee.com>
Date: Wed, 9 Sep 2020 15:56:10 +0530
Subject: [PATCH 007/123] Lint

---
 packages/gatsby/src/utils/start-server.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/gatsby/src/utils/start-server.ts b/packages/gatsby/src/utils/start-server.ts
index 94609affa9c19..62360f868107e 100644
--- a/packages/gatsby/src/utils/start-server.ts
+++ b/packages/gatsby/src/utils/start-server.ts
@@ -13,7 +13,7 @@ import { formatError } from "graphql"
 import webpackConfig from "../utils/webpack.config"
 import { store, emitter } from "../redux"
 import { buildRenderer } from "../commands/build-html"
-import { withBasePath } from "../utils/path"
+// import { withBasePath } from "../utils/path"
 import report from "gatsby-cli/lib/reporter"
 import launchEditor from "react-dev-utils/launchEditor"
 import cors from "cors"

From 89b671c484cfbb408322707565f46f1eef1f1aab Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Tue, 13 Oct 2020 14:53:24 -0600
Subject: [PATCH 008/123] Show activity for HTML rendering + renable socket.io
 so server doesn't crash

---
 packages/gatsby/src/utils/start-server.ts | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/packages/gatsby/src/utils/start-server.ts b/packages/gatsby/src/utils/start-server.ts
index 62360f868107e..5c0cc2049b9df 100644
--- a/packages/gatsby/src/utils/start-server.ts
+++ b/packages/gatsby/src/utils/start-server.ts
@@ -256,8 +256,11 @@ export async function startServer(
       return next()
     }
 
-    // const htmlActivity = report.phantomActivity(`building index.html`, {})
-    // htmlActivity.start()
+    const htmlActivity = report.activityTimer(
+      `building HTML for path "${req.path}"`,
+      {}
+    )
+    htmlActivity.start()
 
     const [response] = await renderHTML({
       htmlComponentRendererPath: `${program.directory}/public/render-page.js`,
@@ -272,9 +275,9 @@ export async function startServer(
       ],
     })
 
-    return res.status(200).send(response)
-
-    // htmlActivity.end()
+    // TODO add support for 404 and general rendering errors
+    htmlActivity.end()
+    res.status(200).send(response)
   })
 
   // Disable directory indexing i.e. serving index.html from a directory.
@@ -288,7 +291,7 @@ export async function startServer(
    **/
   const server = new http.Server(app)
 
-  // const socket = websocketManager.init({ server, directory: program.directory })
+  const socket = websocketManager.init({ server, directory: program.directory })
 
   // hardcoded `localhost`, because host should match `target` we set
   // in http proxy in `develop-proxy`

From a90ea0c0ea329067c9859b9855abfef15762e406 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Tue, 13 Oct 2020 16:31:11 -0600
Subject: [PATCH 009/123] Add error page when templates don't render correctly

---
 packages/gatsby/src/utils/start-server.ts | 125 +++++++++++++++++++---
 1 file changed, 113 insertions(+), 12 deletions(-)

diff --git a/packages/gatsby/src/utils/start-server.ts b/packages/gatsby/src/utils/start-server.ts
index 5c0cc2049b9df..7ac5c5bafd233 100644
--- a/packages/gatsby/src/utils/start-server.ts
+++ b/packages/gatsby/src/utils/start-server.ts
@@ -9,6 +9,10 @@ import graphqlHTTP from "express-graphql"
 import graphqlPlayground from "graphql-playground-middleware-express"
 import graphiqlExplorer from "gatsby-graphiql-explorer"
 import { formatError } from "graphql"
+import path from "path"
+import fs from "fs"
+import { codeFrameColumns } from "@babel/code-frame"
+import ansiHTML from "ansi-html"
 
 import webpackConfig from "../utils/webpack.config"
 import { store, emitter } from "../redux"
@@ -247,6 +251,87 @@ export async function startServer(
   //     id: `webpack-renderer`,
   //   }
   // )
+  const getPosition = function (stackObject) {
+    var filename, line, row
+    // Because the JavaScript error stack has not yet been standardized,
+    // wrap the stack parsing in a try/catch for a soft fail if an
+    // unexpected stack is encountered.
+    try {
+      var filteredStack = stackObject.filter(function (s) {
+        return /\(.+?\)$/.test(s)
+      })
+      var splitLine
+      // For current Node & Chromium Error stacks
+      if (filteredStack.length > 0) {
+        splitLine = filteredStack[0].match(/(?:\()(.+?)(?:\))$/)[1].split(":")
+        // For older, future, or otherwise unexpected stacks
+      } else {
+        splitLine = stackObject[0].split(":")
+      }
+      var splitLength = splitLine.length
+      filename = splitLine[splitLength - 3]
+      line = Number(splitLine[splitLength - 2])
+      row = Number(splitLine[splitLength - 1])
+    } catch (err) {
+      filename = ""
+      line = 0
+      row = 0
+    }
+    return {
+      filename: filename,
+      line: line,
+      row: row,
+    }
+  }
+  const parseError = function (err) {
+    var stack = err.stack ? err.stack : ""
+    var stackObject = stack.split("\n")
+    var position = getPosition(stackObject)
+    // Remove the `/lib/` added by webpack
+    var filename = path.join(
+      directory,
+      ...position.filename.split(path.sep).slice(2)
+    )
+    var code = require(`fs`).readFileSync(filename, `utf-8`)
+    var line = position.line
+    var row = position.row
+    ansiHTML.setColors({
+      reset: ["555", "fff"], // FOREGROUND-COLOR or [FOREGROUND-COLOR] or [, BACKGROUND-COLOR] or [FOREGROUND-COLOR, BACKGROUND-COLOR]
+      black: "aaa", // String
+      red: "bbb",
+      green: "ccc",
+      yellow: "ddd",
+      blue: "eee",
+      magenta: "fff",
+      cyan: "999",
+      lightgrey: "888",
+      darkgrey: "777",
+    })
+    var codeFrame = ansiHTML(
+      codeFrameColumns(
+        code,
+        {
+          start: { line: row, column: line },
+        },
+        { forceColor: true }
+      )
+    )
+    var splitMessage = err.message ? err.message.split("\n") : [""]
+    var message = splitMessage[splitMessage.length - 1]
+    var type = err.type ? err.type : err.name
+    var data = {
+      filename: filename,
+      code,
+      codeFrame,
+      line: line,
+      row: row,
+      message: message,
+      type: type,
+      stack: stack,
+      arguments: err.arguments,
+    }
+    return data
+  }
 
   // Render an HTML page and serve it.
   app.use(async (req, res, next) => {
@@ -262,22 +347,38 @@ export async function startServer(
     )
     htmlActivity.start()
 
-    const [response] = await renderHTML({
-      htmlComponentRendererPath: `${program.directory}/public/render-page.js`,
-      paths: [req.path],
-      envVars: [
-        [`NODE_ENV`, process.env.NODE_ENV || ``],
-        [
-          `gatsby_executing_command`,
-          process.env.gatsby_executing_command || ``,
+    let response = `error`
+    try {
+      let renderResponse = await renderHTML({
+        htmlComponentRendererPath: `${program.directory}/public/render-page.js`,
+        paths: [req.path],
+        envVars: [
+          [`NODE_ENV`, process.env.NODE_ENV || ``],
+          [
+            `gatsby_executing_command`,
+            process.env.gatsby_executing_command || ``,
+          ],
+          [`gatsby_log_level`, process.env.gatsby_log_level || ``],
         ],
-        [`gatsby_log_level`, process.env.gatsby_log_level || ``],
-      ],
-    })
+      })
+      response = renderResponse[0]
+      res.status(200).send(response)
+    } catch (e) {
+      let error = parseError(e)
+      console.log(error)
+      res.status(500).send(`<h1>Error<h1>
+        <h2>The page didn't SSR correctly</h2>
+        <ul>
+          <li><strong>URL path:</strong> ${req.path}</li>
+          <li><strong>File path:</strong> ${error.filename}</li>
+        </ul>
+        <h3>error message</h3>
+        <p><code>${error.message}</code></p>
+        <pre>${error.codeFrame}</pre>`)
+    }
 
     // TODO add support for 404 and general rendering errors
     htmlActivity.end()
-    res.status(200).send(response)
   })
 
   // Disable directory indexing i.e. serving index.html from a directory.

From d93d36d2f58330bd966c87c7ed03804f0940e3da Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Wed, 14 Oct 2020 11:17:16 -0600
Subject: [PATCH 010/123] Rebuild dev ssr bundle when source files change

---
 packages/gatsby/src/commands/build-html.ts    |  37 ++++---
 packages/gatsby/src/utils/start-server.ts     | 100 +++++++++---------
 .../gatsby/src/utils/worker/render-html.ts    |  20 ++--
 3 files changed, 88 insertions(+), 69 deletions(-)

diff --git a/packages/gatsby/src/commands/build-html.ts b/packages/gatsby/src/commands/build-html.ts
index 0c2fbbae96e60..79cd20dfa4f88 100644
--- a/packages/gatsby/src/commands/build-html.ts
+++ b/packages/gatsby/src/commands/build-html.ts
@@ -14,26 +14,35 @@ import { IProgram, Stage } from "./types"
 type IActivity = any // TODO
 type IWorkerPool = any // TODO
 
-const runWebpack = (compilerConfig): Bluebird<webpack.Stats> =>
+const runWebpack = (compilerConfig, stage: Stage): Bluebird<webpack.Stats> =>
   new Bluebird((resolve, reject) => {
-    webpack(compilerConfig).run((err, stats) => {
-      if (err) {
-        reject(err)
-      } else {
-        resolve(stats)
-      }
-    })
+    if (stage === `build-html`) {
+      webpack(compilerConfig).run((err, stats) => {
+        if (err) {
+          reject(err)
+        } else {
+          resolve(stats)
+        }
+      })
+    } else if (stage === `develop-html`) {
+      webpack(compilerConfig).watch({}, (err, stats) => {
+        if (err) {
+          reject(err)
+        } else {
+          resolve(stats)
+        }
+      })
+    }
   })
 
 const doBuildRenderer = async (
   { directory }: IProgram,
-  webpackConfig: webpack.Configuration
+  webpackConfig: webpack.Configuration,
+  stage: Stage
 ): Promise<string> => {
-  const stats = await runWebpack(webpackConfig)
+  const stats = await runWebpack(webpackConfig, stage)
   if (stats.hasErrors()) {
-    reporter.panic(
-      structureWebpackErrors(`build-html`, stats.compilation.errors)
-    )
+    reporter.panic(structureWebpackErrors(stage, stats.compilation.errors))
   }
 
   // render-page.js is hard coded in webpack.config
@@ -50,7 +59,7 @@ export const buildRenderer = async (
     parentSpan,
   })
 
-  return doBuildRenderer(program, config)
+  return doBuildRenderer(program, config, stage)
 }
 
 const deleteRenderer = async (rendererPath: string): Promise<void> => {
diff --git a/packages/gatsby/src/utils/start-server.ts b/packages/gatsby/src/utils/start-server.ts
index 7ac5c5bafd233..d90d0b9475636 100644
--- a/packages/gatsby/src/utils/start-server.ts
+++ b/packages/gatsby/src/utils/start-server.ts
@@ -3,6 +3,7 @@ import webpackDevMiddleware, {
   WebpackDevMiddleware,
 } from "webpack-dev-middleware"
 import got from "got"
+import chokidar from "chokidar"
 import webpack from "webpack"
 import express from "express"
 import graphqlHTTP from "express-graphql"
@@ -13,11 +14,12 @@ import path from "path"
 import fs from "fs"
 import { codeFrameColumns } from "@babel/code-frame"
 import ansiHTML from "ansi-html"
+import { slash } from "gatsby-core-utils"
 
 import webpackConfig from "../utils/webpack.config"
 import { store, emitter } from "../redux"
 import { buildRenderer } from "../commands/build-html"
-// import { withBasePath } from "../utils/path"
+import { withBasePath } from "../utils/path"
 import report from "gatsby-cli/lib/reporter"
 import launchEditor from "react-dev-utils/launchEditor"
 import cors from "cors"
@@ -67,6 +69,7 @@ export async function startServer(
   workerPool: JestWorker = WorkerPool.create()
 ): Promise<IServer> {
   const directory = program.directory
+  const directoryPath = withBasePath(directory)
 
   const webpackActivity = report.activityTimer(`Building development bundle`, {
     id: `webpack-develop`,
@@ -252,28 +255,28 @@ export async function startServer(
   //   }
   // )
   const getPosition = function (stackObject) {
-    var filename, line, row
+    let filename, line, row
     // Because the JavaScript error stack has not yet been standardized,
     // wrap the stack parsing in a try/catch for a soft fail if an
     // unexpected stack is encountered.
     try {
-      var filteredStack = stackObject.filter(function (s) {
+      const filteredStack = stackObject.filter(function (s) {
         return /\(.+?\)$/.test(s)
       })
-      var splitLine
+      let splitLine
       // For current Node & Chromium Error stacks
       if (filteredStack.length > 0) {
-        splitLine = filteredStack[0].match(/(?:\()(.+?)(?:\))$/)[1].split(":")
+        splitLine = filteredStack[0].match(/(?:\()(.+?)(?:\))$/)[1].split(`:`)
         // For older, future, or otherwise unexpected stacks
       } else {
-        splitLine = stackObject[0].split(":")
+        splitLine = stackObject[0].split(`:`)
       }
-      var splitLength = splitLine.length
+      const splitLength = splitLine.length
       filename = splitLine[splitLength - 3]
       line = Number(splitLine[splitLength - 2])
       row = Number(splitLine[splitLength - 1])
     } catch (err) {
-      filename = ""
+      filename = ``
       line = 0
       row = 0
     }
@@ -284,30 +287,32 @@ export async function startServer(
     }
   }
   const parseError = function (err) {
-    var stack = err.stack ? err.stack : ""
-    var stackObject = stack.split("\n")
-    var position = getPosition(stackObject)
+    console.log(`raw err`, err)
+    const stack = err.stack ? err.stack : ``
+    const stackObject = stack.split(`\n`)
+    const position = getPosition(stackObject)
     // Remove the `/lib/` added by webpack
-    var filename = path.join(
+    const filename = path.join(
       directory,
       ...position.filename.split(path.sep).slice(2)
     )
-    var code = require(`fs`).readFileSync(filename, `utf-8`)
-    var line = position.line
-    var row = position.row
+    console.log(filename, position.filename)
+    const code = require(`fs`).readFileSync(filename, `utf-8`)
+    const line = position.line
+    const row = position.row
     ansiHTML.setColors({
-      reset: ["555", "fff"], // FOREGROUND-COLOR or [FOREGROUND-COLOR] or [, BACKGROUND-COLOR] or [FOREGROUND-COLOR, BACKGROUND-COLOR]
-      black: "aaa", // String
-      red: "bbb",
-      green: "ccc",
-      yellow: "ddd",
-      blue: "eee",
-      magenta: "fff",
-      cyan: "999",
-      lightgrey: "888",
-      darkgrey: "777",
+      reset: [`542C85`, `fff`], // FOREGROUND-COLOR or [FOREGROUND-COLOR] or [, BACKGROUND-COLOR] or [FOREGROUND-COLOR, BACKGROUND-COLOR]
+      black: `aaa`, // String
+      red: `bbb`,
+      green: `ccc`,
+      yellow: `DB3A00`,
+      blue: `eee`,
+      magenta: `fff`,
+      cyan: `008577`,
+      lightgrey: `888`,
+      darkgrey: `777`,
     })
-    var codeFrame = ansiHTML(
+    const codeFrame = ansiHTML(
       codeFrameColumns(
         code,
         {
@@ -316,10 +321,10 @@ export async function startServer(
         { forceColor: true }
       )
     )
-    var splitMessage = err.message ? err.message.split("\n") : [""]
-    var message = splitMessage[splitMessage.length - 1]
-    var type = err.type ? err.type : err.name
-    var data = {
+    const splitMessage = err.message ? err.message.split(`\n`) : [``]
+    const message = splitMessage[splitMessage.length - 1]
+    const type = err.type ? err.type : err.name
+    const data = {
       filename: filename,
       code,
       codeFrame,
@@ -341,17 +346,15 @@ export async function startServer(
       return next()
     }
 
-    const htmlActivity = report.activityTimer(
-      `building HTML for path "${req.path}"`,
-      {}
-    )
+    const htmlActivity = report.activityTimer(`building HTML`, {})
     htmlActivity.start()
 
     let response = `error`
     try {
-      let renderResponse = await renderHTML({
+      const renderResponse = await renderHTML({
         htmlComponentRendererPath: `${program.directory}/public/render-page.js`,
         paths: [req.path],
+        stage: Stage.DevelopHTML,
         envVars: [
           [`NODE_ENV`, process.env.NODE_ENV || ``],
           [
@@ -364,13 +367,12 @@ export async function startServer(
       response = renderResponse[0]
       res.status(200).send(response)
     } catch (e) {
-      let error = parseError(e)
-      console.log(error)
+      const error = parseError(e)
       res.status(500).send(`<h1>Error<h1>
         <h2>The page didn't SSR correctly</h2>
         <ul>
-          <li><strong>URL path:</strong> ${req.path}</li>
-          <li><strong>File path:</strong> ${error.filename}</li>
+          <li><strong>URL path:</strong> <code>${req.path}</code></li>
+          <li><strong>File path:</strong> <code>${error.filename}</code></li>
         </ul>
         <h3>error message</h3>
         <p><code>${error.message}</code></p>
@@ -399,17 +401,17 @@ export async function startServer(
   const listener = server.listen(program.port, `localhost`)
 
   // Register watcher that rebuilds index.html every time html.js changes.
-  // const watchGlobs = [`src/html.js`, `plugins/**/gatsby-ssr.js`].map(path =>
-  //   slash(directoryPath(path))
-  // )
+  const watchGlobs = [`src/html.js`, `plugins/**/gatsby-ssr.js`].map(path =>
+    slash(directoryPath(path))
+  )
 
-  // chokidar.watch(watchGlobs).on(`change`, async () => {
-  //   // console.log(`Time to build a renderer`)
-  //   // await buildRenderer(program, Stage.DevelopHTML, webpackActivity)
-  //   // console.log(`We built a renderer`)
-  //   // eslint-disable-next-line no-unused-expressions
-  //   socket?.to(`clients`).emit(`reload`)
-  // })
+  chokidar.watch(watchGlobs).on(`change`, async () => {
+    console.log(`Time to build a renderer`)
+    await buildRenderer(program, Stage.DevelopHTML, webpackActivity)
+    console.log(`We built a renderer`)
+    // eslint-disable-next-line no-unused-expressions
+    socket?.to(`clients`).emit(`reload`)
+  })
 
   return {
     compiler,
diff --git a/packages/gatsby/src/utils/worker/render-html.ts b/packages/gatsby/src/utils/worker/render-html.ts
index f286157a24104..7996fbe0e9cb6 100644
--- a/packages/gatsby/src/utils/worker/render-html.ts
+++ b/packages/gatsby/src/utils/worker/render-html.ts
@@ -6,6 +6,7 @@ import Promise from "bluebird"
 export const renderHTML = ({
   htmlComponentRendererPath,
   paths,
+  stage,
   envVars,
 }: {
   htmlComponentRendererPath: string
@@ -20,15 +21,22 @@ export const renderHTML = ({
     paths,
     path =>
       new Promise((resolve, reject) => {
+        // Make sure we get the latest version during development
+        if (stage === `develop-html`) {
+          delete require.cache[require.resolve(htmlComponentRendererPath)]
+        }
         const htmlComponentRenderer = require(htmlComponentRendererPath)
         try {
           htmlComponentRenderer.default(path, (_throwAway, htmlString) => {
-            // resolve(
-            //   fs.outputFile(
-            //     getPageHtmlFilePath(join(process.cwd(), `public`), path),
-            resolve(htmlString)
-            // )
-            // )
+            if (stage === `develop-html`) {
+              resolve(htmlString)
+            } else {
+              resolve(
+                fs.outputFile(
+                  getPageHtmlFilePath(join(process.cwd(), `public`), path)
+                )
+              )
+            }
           })
         } catch (e) {
           // add some context to error so we can display more helpful message

From cfb1a3bc1a1e3be11bb99c0091d760fd8bb2f7d0 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Wed, 14 Oct 2020 11:24:57 -0600
Subject: [PATCH 011/123] Fix some lint errors

---
 packages/gatsby/src/utils/start-server.ts       | 11 ++++++++---
 packages/gatsby/src/utils/worker/render-html.ts |  6 +++---
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/packages/gatsby/src/utils/start-server.ts b/packages/gatsby/src/utils/start-server.ts
index d90d0b9475636..9adf732105859 100644
--- a/packages/gatsby/src/utils/start-server.ts
+++ b/packages/gatsby/src/utils/start-server.ts
@@ -11,7 +11,7 @@ import graphqlPlayground from "graphql-playground-middleware-express"
 import graphiqlExplorer from "gatsby-graphiql-explorer"
 import { formatError } from "graphql"
 import path from "path"
-import fs from "fs"
+import fs from "fs-extra"
 import { codeFrameColumns } from "@babel/code-frame"
 import ansiHTML from "ansi-html"
 import { slash } from "gatsby-core-utils"
@@ -255,7 +255,9 @@ export async function startServer(
   //   }
   // )
   const getPosition = function (stackObject) {
-    let filename, line, row
+    let filename
+    let line
+    let row
     // Because the JavaScript error stack has not yet been standardized,
     // wrap the stack parsing in a try/catch for a soft fail if an
     // unexpected stack is encountered.
@@ -297,7 +299,7 @@ export async function startServer(
       ...position.filename.split(path.sep).slice(2)
     )
     console.log(filename, position.filename)
-    const code = require(`fs`).readFileSync(filename, `utf-8`)
+    const code = fs.readFileSync(filename, `utf-8`)
     const line = position.line
     const row = position.row
     ansiHTML.setColors({
@@ -381,6 +383,9 @@ export async function startServer(
 
     // TODO add support for 404 and general rendering errors
     htmlActivity.end()
+
+    // Make eslint happy
+    return null
   })
 
   // Disable directory indexing i.e. serving index.html from a directory.
diff --git a/packages/gatsby/src/utils/worker/render-html.ts b/packages/gatsby/src/utils/worker/render-html.ts
index 7996fbe0e9cb6..c359d9cc1eab0 100644
--- a/packages/gatsby/src/utils/worker/render-html.ts
+++ b/packages/gatsby/src/utils/worker/render-html.ts
@@ -1,7 +1,7 @@
-// import fs from "fs-extra"
+import fs from "fs-extra"
 import Promise from "bluebird"
-// import { join } from "path"
-// import { getPageHtmlFilePath } from "../../utils/page-html"
+import { join } from "path"
+import { getPageHtmlFilePath } from "../../utils/page-html"
 
 export const renderHTML = ({
   htmlComponentRendererPath,

From e4ab9c0870732d5b533e6c5d1ea866b937dced97 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Wed, 14 Oct 2020 14:28:21 -0600
Subject: [PATCH 012/123] Fix building html

---
 packages/gatsby/src/commands/build-html.ts    |  1 -
 .../gatsby/src/utils/worker/render-html.ts    | 22 +++++++++++--------
 2 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/packages/gatsby/src/commands/build-html.ts b/packages/gatsby/src/commands/build-html.ts
index 79cd20dfa4f88..79dd777cadb54 100644
--- a/packages/gatsby/src/commands/build-html.ts
+++ b/packages/gatsby/src/commands/build-html.ts
@@ -85,7 +85,6 @@ const renderHTMLQueue = async (
     [`gatsby_log_level`, process.env.gatsby_log_level],
   ]
 
-  // const start = process.hrtime()
   const segments = chunk(pages, 50)
 
   await Bluebird.map(segments, async pageSegment => {
diff --git a/packages/gatsby/src/utils/worker/render-html.ts b/packages/gatsby/src/utils/worker/render-html.ts
index c359d9cc1eab0..c15c85d54dc24 100644
--- a/packages/gatsby/src/utils/worker/render-html.ts
+++ b/packages/gatsby/src/utils/worker/render-html.ts
@@ -27,17 +27,21 @@ export const renderHTML = ({
         }
         const htmlComponentRenderer = require(htmlComponentRendererPath)
         try {
-          htmlComponentRenderer.default(path, (_throwAway, htmlString) => {
-            if (stage === `develop-html`) {
-              resolve(htmlString)
-            } else {
-              resolve(
-                fs.outputFile(
-                  getPageHtmlFilePath(join(process.cwd(), `public`), path)
+          htmlComponentRenderer.default(
+            path,
+            async (_throwAway, htmlString) => {
+              if (stage === `develop-html`) {
+                resolve(htmlString)
+              } else {
+                resolve(
+                  fs.outputFile(
+                    getPageHtmlFilePath(join(process.cwd(), `public`), path),
+                    htmlString
+                  )
                 )
-              )
+              }
             }
-          })
+          )
         } catch (e) {
           // add some context to error so we can display more helpful message
           e.context = {

From ce714f3de8a8be795b04eaeacee7526a67e28590 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Wed, 14 Oct 2020 14:31:15 -0600
Subject: [PATCH 013/123] use gatsby colors for syntax highlighting

---
 packages/gatsby/src/utils/start-server.ts | 31 +++++++++++++++--------
 1 file changed, 20 insertions(+), 11 deletions(-)

diff --git a/packages/gatsby/src/utils/start-server.ts b/packages/gatsby/src/utils/start-server.ts
index 9adf732105859..fe7754fa0551a 100644
--- a/packages/gatsby/src/utils/start-server.ts
+++ b/packages/gatsby/src/utils/start-server.ts
@@ -288,8 +288,18 @@ export async function startServer(
       row: row,
     }
   }
+  // Colors taken from Gatsby's design tokens
+  // https://github.com/gatsbyjs/gatsby/blob/d8acab3a135fa8250a0eb3a47c67300dde6eae32/packages/gatsby-design-tokens/src/colors.js#L185-L205
+  const colors = {
+    background: `fdfaf6`,
+    text: `452475`,
+    green: `137886`,
+    darkGreen: `006500`,
+    comment: `527713`,
+    keyword: `096fb3`,
+    yellow: `DB3A00`,
+  }
   const parseError = function (err) {
-    console.log(`raw err`, err)
     const stack = err.stack ? err.stack : ``
     const stackObject = stack.split(`\n`)
     const position = getPosition(stackObject)
@@ -298,27 +308,26 @@ export async function startServer(
       directory,
       ...position.filename.split(path.sep).slice(2)
     )
-    console.log(filename, position.filename)
     const code = fs.readFileSync(filename, `utf-8`)
     const line = position.line
     const row = position.row
     ansiHTML.setColors({
-      reset: [`542C85`, `fff`], // FOREGROUND-COLOR or [FOREGROUND-COLOR] or [, BACKGROUND-COLOR] or [FOREGROUND-COLOR, BACKGROUND-COLOR]
+      reset: [colors.text, colors.background], // FOREGROUND-COLOR or [FOREGROUND-COLOR] or [, BACKGROUND-COLOR] or [FOREGROUND-COLOR, BACKGROUND-COLOR]
       black: `aaa`, // String
-      red: `bbb`,
-      green: `ccc`,
-      yellow: `DB3A00`,
+      red: colors.keyword,
+      green: colors.green,
+      yellow: colors.yellow,
       blue: `eee`,
       magenta: `fff`,
-      cyan: `008577`,
+      cyan: colors.darkGreen,
       lightgrey: `888`,
-      darkgrey: `777`,
+      darkgrey: colors.comment,
     })
     const codeFrame = ansiHTML(
       codeFrameColumns(
         code,
         {
-          start: { line: row, column: line },
+          start: { line: line, column: row },
         },
         { forceColor: true }
       )
@@ -370,7 +379,7 @@ export async function startServer(
       res.status(200).send(response)
     } catch (e) {
       const error = parseError(e)
-      res.status(500).send(`<h1>Error<h1>
+      res.status(500).send(`<title>Develop SSR Error</title><h1>Error<h1>
         <h2>The page didn't SSR correctly</h2>
         <ul>
           <li><strong>URL path:</strong> <code>${req.path}</code></li>
@@ -378,7 +387,7 @@ export async function startServer(
         </ul>
         <h3>error message</h3>
         <p><code>${error.message}</code></p>
-        <pre>${error.codeFrame}</pre>`)
+        <pre style="background:#${colors.background};padding:8px;">${error.codeFrame}</pre>`)
     }
 
     // TODO add support for 404 and general rendering errors

From be7b2ae66cc00a3a1ef09811fbf6bde2eafcd8ba Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Wed, 14 Oct 2020 16:43:05 -0600
Subject: [PATCH 014/123] Add test script to compare dev & prod output

---
 integration-tests/ssr/__tests__/ssr.js        | 13 +++
 integration-tests/ssr/package.json            |  8 +-
 integration-tests/ssr/test-output.js          | 96 +++++++++++++++++++
 .../src/gatsby-ssr.js                         | 22 ++---
 packages/gatsby/cache-dir/static-entry.js     |  1 +
 packages/gatsby/src/utils/start-server.ts     |  2 +-
 6 files changed, 127 insertions(+), 15 deletions(-)
 create mode 100644 integration-tests/ssr/test-output.js

diff --git a/integration-tests/ssr/__tests__/ssr.js b/integration-tests/ssr/__tests__/ssr.js
index 66c7d855d87b4..fc99eccf93534 100644
--- a/integration-tests/ssr/__tests__/ssr.js
+++ b/integration-tests/ssr/__tests__/ssr.js
@@ -1,4 +1,5 @@
 const fetch = require(`node-fetch`)
+const { execFile } = require("child_process")
 
 describe(`SSR`, () => {
   test(`is run for a page when it is requested`, async () => {
@@ -6,4 +7,16 @@ describe(`SSR`, () => {
 
     expect(html).toMatchSnapshot()
   })
+  test(`dev & build outputs match`, async () => {
+    const childProcess = execFile(`yarn`, [`test-output`])
+    let exitCode
+    await new Promise(resolve => {
+      childProcess.on(`exit`, code => {
+        exitCode = { exitCode: code }
+        resolve()
+      })
+    })
+
+    expect(exitCode).toEqual({ exitCode: 0 })
+  })
 })
diff --git a/integration-tests/ssr/package.json b/integration-tests/ssr/package.json
index 6dc86f8c0ddd4..df2df97d94216 100644
--- a/integration-tests/ssr/package.json
+++ b/integration-tests/ssr/package.json
@@ -10,18 +10,22 @@
     "develop": "gatsby develop",
     "serve": "gatsby serve",
     "clean": "gatsby clean",
+    "test-output": "node test-output.js",
     "test:jest": "jest --config=jest.config.js --runInBand",
-    "test": "start-server-and-test develop http://localhost:8000 test:jest"
+    "start-dev-server": "start-server-and-test develop http://localhost:8000 test:jest",
+    "test": "npm-run-all -s build start-dev-server"
   },
   "dependencies": {
-    "gatsby": "2.24.49-dev-1599559013196",
+    "gatsby": "^2.24.49",
     "react": "^16.12.0",
     "react-dom": "^16.12.0"
   },
   "devDependencies": {
     "fs-extra": "^9.0.0",
     "jest": "^24.0.0",
+    "jest-diff": "^24.0.0",
     "jest-cli": "^24.0.0",
+    "npm-run-all": "4.1.5",
     "start-server-and-test": "^1.11.3"
   },
   "repository": {
diff --git a/integration-tests/ssr/test-output.js b/integration-tests/ssr/test-output.js
new file mode 100644
index 0000000000000..cf2d1731c387a
--- /dev/null
+++ b/integration-tests/ssr/test-output.js
@@ -0,0 +1,96 @@
+// To run the test script manually on a site (e.g. to test a plugin):
+// - build the site first
+// - start the develop server
+// - run this script
+;(async function () {
+  const { getPageHtmlFilePath } = require(`gatsby/dist/utils/page-html`)
+  const { join } = require(`path`)
+  const fs = require(`fs-extra`)
+  const fetch = require(`node-fetch`)
+  const diff = require(`jest-diff`)
+  const prettier = require(`prettier`)
+  const cheerio = require(`cheerio`)
+  const stripAnsi = require(`strip-ansi`)
+
+  const devSiteBasePath = `http://localhost:8000`
+
+  const comparePath = async path => {
+    const format = htmlStr => prettier.format(htmlStr, { parser: `html` })
+
+    const filterHtml = htmlStr => {
+      const $ = cheerio.load(htmlStr)
+      // There are many script tag differences
+      $(`script`).remove()
+      // Only added in production. Dev uses css-loader
+      $(`#gatsby-global-css`).remove()
+      // Only in prod
+      $(`link[rel="preload"]`).remove()
+      // Only in prod
+      $(`meta[name="generator"]`).remove()
+      // Only in dev
+      $(`meta[name="note"]`).remove()
+
+      return $.html()
+    }
+
+    const builtHtml = format(
+      filterHtml(
+        fs.readFileSync(
+          getPageHtmlFilePath(join(process.cwd(), `public`), path),
+          `utf-8`
+        )
+      )
+    )
+
+    const rawDevHtml = await fetch(`${devSiteBasePath}/${path}`).then(res =>
+      res.text()
+    )
+
+    const devHtml = format(filterHtml(rawDevHtml))
+    const diffResult = diff(devHtml, builtHtml, {
+      contextLines: 3,
+      expand: false,
+    })
+    if (
+      stripAnsi(diffResult) === `Compared values have no visual difference.`
+    ) {
+      return true
+    } else {
+      console.log(`path "${path}" has differences between dev & prod`)
+      console.log(diffResult)
+      return false
+    }
+  }
+
+  const response = await fetch(`${devSiteBasePath}/__graphql`, {
+    method: `POST`,
+    headers: { "Content-Type": `application/json` },
+    body: JSON.stringify({
+      query: `query MyQuery {
+  allSitePage {
+    nodes {
+      path
+    }
+  }
+}
+`,
+    }),
+  }).then(res => res.json()) // expecting a json response
+
+  const paths = response.data.allSitePage.nodes
+    .map(n => n.path)
+    .filter(p => p !== `/dev-404-page/`)
+
+  console.log(
+    `testing these paths for differences between dev & prod outputs`,
+    paths
+  )
+
+  const results = await Promise.all(paths.map(p => comparePath(p)))
+  // Test all true
+  if (results.every(r => r)) {
+    process.exit(0)
+  } else {
+    process.exit(1)
+  }
+})()
diff --git a/packages/gatsby-plugin-typography/src/gatsby-ssr.js b/packages/gatsby-plugin-typography/src/gatsby-ssr.js
index 3ec41f70ce26c..7a3bdbafe9f70 100644
--- a/packages/gatsby-plugin-typography/src/gatsby-ssr.js
+++ b/packages/gatsby-plugin-typography/src/gatsby-ssr.js
@@ -3,19 +3,17 @@ import { TypographyStyle, GoogleFont } from "react-typography"
 import typography from "typography-plugin-cache-endpoint"
 
 exports.onRenderBody = ({ setHeadComponents }, pluginOptions) => {
-  if (process.env.BUILD_STAGE === `build-html`) {
-    const googleFont = [].concat(
-      pluginOptions.omitGoogleFont ? (
-        []
-      ) : (
-        <GoogleFont key={`GoogleFont`} typography={typography} />
-      )
+  const googleFont = [].concat(
+    pluginOptions.omitGoogleFont ? (
+      []
+    ) : (
+      <GoogleFont key={`GoogleFont`} typography={typography} />
     )
-    setHeadComponents([
-      <TypographyStyle key={`TypographyStyle`} typography={typography} />,
-      ...googleFont,
-    ])
-  }
+  )
+  setHeadComponents([
+    <TypographyStyle key={`TypographyStyle`} typography={typography} />,
+    ...googleFont,
+  ])
 }
 
 // Move Typography.js styles to the top of the head section so they're loaded first
diff --git a/packages/gatsby/cache-dir/static-entry.js b/packages/gatsby/cache-dir/static-entry.js
index c0f6b3577c783..61ff359538ac4 100644
--- a/packages/gatsby/cache-dir/static-entry.js
+++ b/packages/gatsby/cache-dir/static-entry.js
@@ -415,6 +415,7 @@ export default (pagePath, callback) => {
         headComponents.unshift(
           <style
             data-href={`${__PATH_PREFIX__}/${style.name}`}
+            id={`gatsby-global-css`}
             dangerouslySetInnerHTML={{
               __html: fs.readFileSync(
                 join(process.cwd(), `public`, style.name),
diff --git a/packages/gatsby/src/utils/start-server.ts b/packages/gatsby/src/utils/start-server.ts
index fe7754fa0551a..2a5515edf6656 100644
--- a/packages/gatsby/src/utils/start-server.ts
+++ b/packages/gatsby/src/utils/start-server.ts
@@ -357,7 +357,7 @@ export async function startServer(
       return next()
     }
 
-    const htmlActivity = report.activityTimer(`building HTML`, {})
+    const htmlActivity = report.phantomActivity(`building HTML for path`, {})
     htmlActivity.start()
 
     let response = `error`

From 96c0f335666abf04e25f2c13bbd8739f00d4bb7c Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Wed, 14 Oct 2020 16:48:55 -0600
Subject: [PATCH 015/123] Add return types

---
 packages/gatsby/src/utils/start-server.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/gatsby/src/utils/start-server.ts b/packages/gatsby/src/utils/start-server.ts
index 2a5515edf6656..0962a94a94e5c 100644
--- a/packages/gatsby/src/utils/start-server.ts
+++ b/packages/gatsby/src/utils/start-server.ts
@@ -254,7 +254,7 @@ export async function startServer(
   //     id: `webpack-renderer`,
   //   }
   // )
-  const getPosition = function (stackObject) {
+  const getPosition = function (stackObject): object {
     let filename
     let line
     let row
@@ -299,7 +299,7 @@ export async function startServer(
     keyword: `096fb3`,
     yellow: `DB3A00`,
   }
-  const parseError = function (err) {
+  const parseError = function (err): object {
     const stack = err.stack ? err.stack : ``
     const stackObject = stack.split(`\n`)
     const position = getPosition(stackObject)

From 9d8bde26fdccfd231642ba7c34153d52f6fb9839 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Wed, 14 Oct 2020 17:14:45 -0600
Subject: [PATCH 016/123] Don't respond to dev server page loads until any
 sourcing/transforming is done

---
 packages/gatsby/src/commands/develop-process.ts |  2 ++
 packages/gatsby/src/utils/start-server.ts       | 14 ++++++++++++++
 2 files changed, 16 insertions(+)

diff --git a/packages/gatsby/src/commands/develop-process.ts b/packages/gatsby/src/commands/develop-process.ts
index bc94860d97bc8..78595cbf46080 100644
--- a/packages/gatsby/src/commands/develop-process.ts
+++ b/packages/gatsby/src/commands/develop-process.ts
@@ -137,4 +137,6 @@ module.exports = async (program: IDevelopArgs): Promise<void> => {
   }
 
   service.start()
+
+  program.developMachineService = service
 }
diff --git a/packages/gatsby/src/utils/start-server.ts b/packages/gatsby/src/utils/start-server.ts
index 0962a94a94e5c..cfb2def50537e 100644
--- a/packages/gatsby/src/utils/start-server.ts
+++ b/packages/gatsby/src/utils/start-server.ts
@@ -357,6 +357,20 @@ export async function startServer(
       return next()
     }
 
+    // Sleep until any work the server is doing has finished.
+    await new Promise(resolve => {
+      if (program.developMachineService._state.value == `waiting`) {
+        resolve()
+      } else {
+        const intervalId = setInterval(() => {
+          if (program.developMachineService._state.value == `waiting`) {
+            clearInterval(intervalId)
+            resolve()
+          }
+        }, 50)
+      }
+    })
+
     const htmlActivity = report.phantomActivity(`building HTML for path`, {})
     htmlActivity.start()
 

From ac9642e4bed873561b9c1dd41ca1b6afe5145174 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Wed, 14 Oct 2020 17:33:08 -0600
Subject: [PATCH 017/123] typings

---
 packages/gatsby/src/commands/types.ts         |  1 +
 packages/gatsby/src/utils/start-server.ts     | 28 +++++++++++++------
 .../gatsby/src/utils/worker/render-html.ts    |  2 ++
 3 files changed, 22 insertions(+), 9 deletions(-)

diff --git a/packages/gatsby/src/commands/types.ts b/packages/gatsby/src/commands/types.ts
index 1cc9c692aef7c..cb70e4a69bca6 100644
--- a/packages/gatsby/src/commands/types.ts
+++ b/packages/gatsby/src/commands/types.ts
@@ -26,6 +26,7 @@ export interface IProgram {
   [`key-file`]?: string
   directory: string
   https?: boolean
+  developMachineService: any
   sitePackageJson: PackageJson
   ssl?: ICert
   inspect?: number
diff --git a/packages/gatsby/src/utils/start-server.ts b/packages/gatsby/src/utils/start-server.ts
index cfb2def50537e..7d2f4de0cbe86 100644
--- a/packages/gatsby/src/utils/start-server.ts
+++ b/packages/gatsby/src/utils/start-server.ts
@@ -248,13 +248,12 @@ export async function startServer(
     res.status(404).end()
   })
 
-  // const buildRendererActivity = report.activityTimer(
-  //   `Building renderer bundle`,
-  //   {
-  //     id: `webpack-renderer`,
-  //   }
-  // )
-  const getPosition = function (stackObject): object {
+  interface IErrorPosition {
+    filename: string
+    line: number
+    row: number
+  }
+  const getPosition = function (stackObject): IErrorPosition {
     let filename
     let line
     let row
@@ -299,7 +298,19 @@ export async function startServer(
     keyword: `096fb3`,
     yellow: `DB3A00`,
   }
-  const parseError = function (err): object {
+
+  interface IParsedError {
+    filename: string
+    code: string
+    codeFrame: string
+    line: number
+    row: number
+    message: string
+    type: string
+    stack: [string]
+  }
+
+  const parseError = function (err): IParsedError {
     const stack = err.stack ? err.stack : ``
     const stackObject = stack.split(`\n`)
     const position = getPosition(stackObject)
@@ -344,7 +355,6 @@ export async function startServer(
       message: message,
       type: type,
       stack: stack,
-      arguments: err.arguments,
     }
     return data
   }
diff --git a/packages/gatsby/src/utils/worker/render-html.ts b/packages/gatsby/src/utils/worker/render-html.ts
index c15c85d54dc24..c2c2344a0ffb8 100644
--- a/packages/gatsby/src/utils/worker/render-html.ts
+++ b/packages/gatsby/src/utils/worker/render-html.ts
@@ -2,6 +2,7 @@ import fs from "fs-extra"
 import Promise from "bluebird"
 import { join } from "path"
 import { getPageHtmlFilePath } from "../../utils/page-html"
+import { Stage } from "../../commands/types"
 
 export const renderHTML = ({
   htmlComponentRendererPath,
@@ -11,6 +12,7 @@ export const renderHTML = ({
 }: {
   htmlComponentRendererPath: string
   paths: Array<string>
+  stage: Stage
   envVars: Array<Array<string>>
 }): Promise<Array<unknown>> => {
   // This is being executed in child process, so we need to set some vars

From f8f00a31aada8478c1b808fd76dc190c67c1b5d5 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Wed, 14 Oct 2020 19:41:18 -0600
Subject: [PATCH 018/123] fix types... maybe

---
 packages/gatsby/src/commands/types.ts            | 2 +-
 packages/gatsby/src/utils/start-server.ts        | 4 +---
 packages/gatsby/src/utils/webpack-error-utils.ts | 4 +++-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/packages/gatsby/src/commands/types.ts b/packages/gatsby/src/commands/types.ts
index cb70e4a69bca6..2e9299e2670c7 100644
--- a/packages/gatsby/src/commands/types.ts
+++ b/packages/gatsby/src/commands/types.ts
@@ -26,7 +26,7 @@ export interface IProgram {
   [`key-file`]?: string
   directory: string
   https?: boolean
-  developMachineService: any
+  developMachineService?: any
   sitePackageJson: PackageJson
   ssl?: ICert
   inspect?: number
diff --git a/packages/gatsby/src/utils/start-server.ts b/packages/gatsby/src/utils/start-server.ts
index 7d2f4de0cbe86..bf08d43c789b8 100644
--- a/packages/gatsby/src/utils/start-server.ts
+++ b/packages/gatsby/src/utils/start-server.ts
@@ -384,7 +384,6 @@ export async function startServer(
     const htmlActivity = report.phantomActivity(`building HTML for path`, {})
     htmlActivity.start()
 
-    let response = `error`
     try {
       const renderResponse = await renderHTML({
         htmlComponentRendererPath: `${program.directory}/public/render-page.js`,
@@ -399,8 +398,7 @@ export async function startServer(
           [`gatsby_log_level`, process.env.gatsby_log_level || ``],
         ],
       })
-      response = renderResponse[0]
-      res.status(200).send(response)
+      res.status(200).send(renderResponse[0])
     } catch (e) {
       const error = parseError(e)
       res.status(500).send(`<title>Develop SSR Error</title><h1>Error<h1>
diff --git a/packages/gatsby/src/utils/webpack-error-utils.ts b/packages/gatsby/src/utils/webpack-error-utils.ts
index eff94f8e4f53a..cb669b27960e1 100644
--- a/packages/gatsby/src/utils/webpack-error-utils.ts
+++ b/packages/gatsby/src/utils/webpack-error-utils.ts
@@ -1,14 +1,16 @@
 import reporter from "gatsby-cli/lib/reporter"
 import { Stats } from "webpack"
 import { IMatch } from "../types"
+import { Stage } from "../commands/types"
 
 const stageCodeToReadableLabel = {
   "build-javascript": `Generating JavaScript bundles`,
   "build-html": `Generating SSR bundle`,
+  "develop-html": `Generating development SSR bundle`,
   develop: `Generating development JavaScript bundle`,
 } as const
 
-type Stage = keyof typeof stageCodeToReadableLabel
+// type Stage = keyof typeof stageCodeToReadableLabel
 type StageLabel = typeof stageCodeToReadableLabel[Stage]
 
 interface ITransformedWebpackError {

From cbf16e07c2fcfa018e839aff76a91285f8094eba Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Thu, 15 Oct 2020 14:46:48 -0600
Subject: [PATCH 019/123] maybe typescript happy

---
 packages/gatsby/src/services/start-webpack-server.ts |  3 ++-
 packages/gatsby/src/utils/webpack-error-utils.ts     | 12 ++++++------
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/packages/gatsby/src/services/start-webpack-server.ts b/packages/gatsby/src/services/start-webpack-server.ts
index 60362ab084420..c9b7ce4a3b880 100644
--- a/packages/gatsby/src/services/start-webpack-server.ts
+++ b/packages/gatsby/src/services/start-webpack-server.ts
@@ -4,6 +4,7 @@ import formatWebpackMessages from "react-dev-utils/formatWebpackMessages"
 import chalk from "chalk"
 import { Compiler } from "webpack"
 import { isEqual } from "lodash"
+import { Stage } from "../commands/types"
 
 import {
   reportWebpackWarnings,
@@ -116,7 +117,7 @@ export async function startWebpackServer({
 
         if (!isSuccessful) {
           const errors = structureWebpackErrors(
-            `develop`,
+            Stage.Develop,
             stats.compilation.errors
           )
           webpackActivity.panicOnBuild(errors)
diff --git a/packages/gatsby/src/utils/webpack-error-utils.ts b/packages/gatsby/src/utils/webpack-error-utils.ts
index cb669b27960e1..be27f0c57f66b 100644
--- a/packages/gatsby/src/utils/webpack-error-utils.ts
+++ b/packages/gatsby/src/utils/webpack-error-utils.ts
@@ -1,16 +1,16 @@
 import reporter from "gatsby-cli/lib/reporter"
 import { Stats } from "webpack"
 import { IMatch } from "../types"
-import { Stage } from "../commands/types"
+import { Stage as StageEnum } from "../commands/types"
 
 const stageCodeToReadableLabel = {
-  "build-javascript": `Generating JavaScript bundles`,
-  "build-html": `Generating SSR bundle`,
-  "develop-html": `Generating development SSR bundle`,
-  develop: `Generating development JavaScript bundle`,
+  BuildJavascript: `Generating JavaScript bundles`,
+  BuildHTML: `Generating SSR bundle`,
+  DevelopHTML: `Generating development SSR bundle`,
+  Develop: `Generating development JavaScript bundle`,
 } as const
 
-// type Stage = keyof typeof stageCodeToReadableLabel
+type Stage = keyof typeof StageEnum
 type StageLabel = typeof stageCodeToReadableLabel[Stage]
 
 interface ITransformedWebpackError {

From 1311c34efa13ed0042ce739939efe7e7472a71b1 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Thu, 15 Oct 2020 14:57:08 -0600
Subject: [PATCH 020/123] Add missing globals to tests & update snapshots

---
 .../cache-dir/__tests__/__snapshots__/static-entry.js.snap  | 6 +++---
 packages/gatsby/cache-dir/__tests__/static-entry.js         | 6 ++++++
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/packages/gatsby/cache-dir/__tests__/__snapshots__/static-entry.js.snap b/packages/gatsby/cache-dir/__tests__/__snapshots__/static-entry.js.snap
index a32218f3e0838..2a5a48623247a 100644
--- a/packages/gatsby/cache-dir/__tests__/__snapshots__/static-entry.js.snap
+++ b/packages/gatsby/cache-dir/__tests__/__snapshots__/static-entry.js.snap
@@ -1,10 +1,10 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
-exports[`develop-static-entry onPreRenderHTML can be used to replace headComponents 1`] = `"<!DOCTYPE html><html><head><meta charSet=\\"utf-8\\"/><meta http-equiv=\\"x-ua-compatible\\" content=\\"ie=edge\\"/><meta name=\\"viewport\\" content=\\"width=device-width, initial-scale=1, shrink-to-fit=no\\"/><style> .style3  </style><style> .style2  </style><style> .style1  </style><meta name=\\"note\\" content=\\"environment=development\\"/><script src=\\"/socket.io/socket.io.js\\"></script></head><body><div id=\\"___gatsby\\"></div><script src=\\"/polyfill.js\\" nomodule=\\"\\"></script><script src=\\"/commons.js\\"></script></body></html>"`;
+exports[`develop-static-entry onPreRenderHTML can be used to replace headComponents 1`] = `"<!DOCTYPE html><html><head><meta charSet=\\"utf-8\\"/><meta http-equiv=\\"x-ua-compatible\\" content=\\"ie=edge\\"/><meta name=\\"viewport\\" content=\\"width=device-width, initial-scale=1, shrink-to-fit=no\\"/><style> .style3  </style><style> .style2  </style><style> .style1  </style><meta name=\\"note\\" content=\\"environment=development\\"/><script src=\\"/socket.io/socket.io.js\\"></script></head><body><div id=\\"___gatsby\\"><div style=\\"outline:none\\" tabindex=\\"-1\\" id=\\"gatsby-focus-wrapper\\"></div><div id=\\"gatsby-announcer\\" style=\\"position:absolute;top:0;width:1px;height:1px;padding:0;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;border:0\\" aria-live=\\"assertive\\" aria-atomic=\\"true\\"></div></div><script src=\\"/polyfill.js\\" nomodule=\\"\\"></script><script src=\\"/commons.js\\"></script></body></html>"`;
 
-exports[`develop-static-entry onPreRenderHTML can be used to replace postBodyComponents 1`] = `"<!DOCTYPE html><html><head><meta charSet=\\"utf-8\\"/><meta http-equiv=\\"x-ua-compatible\\" content=\\"ie=edge\\"/><meta name=\\"viewport\\" content=\\"width=device-width, initial-scale=1, shrink-to-fit=no\\"/><meta name=\\"note\\" content=\\"environment=development\\"/><script src=\\"/socket.io/socket.io.js\\"></script></head><body><div id=\\"___gatsby\\"></div><div> div3 </div><div> div2 </div><div> div1 </div><script src=\\"/polyfill.js\\" nomodule=\\"\\"></script><script src=\\"/commons.js\\"></script></body></html>"`;
+exports[`develop-static-entry onPreRenderHTML can be used to replace postBodyComponents 1`] = `"<!DOCTYPE html><html><head><meta charSet=\\"utf-8\\"/><meta http-equiv=\\"x-ua-compatible\\" content=\\"ie=edge\\"/><meta name=\\"viewport\\" content=\\"width=device-width, initial-scale=1, shrink-to-fit=no\\"/><meta name=\\"note\\" content=\\"environment=development\\"/><script src=\\"/socket.io/socket.io.js\\"></script></head><body><div id=\\"___gatsby\\"><div style=\\"outline:none\\" tabindex=\\"-1\\" id=\\"gatsby-focus-wrapper\\"></div><div id=\\"gatsby-announcer\\" style=\\"position:absolute;top:0;width:1px;height:1px;padding:0;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;border:0\\" aria-live=\\"assertive\\" aria-atomic=\\"true\\"></div></div><div> div3 </div><div> div2 </div><div> div1 </div><script src=\\"/polyfill.js\\" nomodule=\\"\\"></script><script src=\\"/commons.js\\"></script></body></html>"`;
 
-exports[`develop-static-entry onPreRenderHTML can be used to replace preBodyComponents 1`] = `"<!DOCTYPE html><html><head><meta charSet=\\"utf-8\\"/><meta http-equiv=\\"x-ua-compatible\\" content=\\"ie=edge\\"/><meta name=\\"viewport\\" content=\\"width=device-width, initial-scale=1, shrink-to-fit=no\\"/><meta name=\\"note\\" content=\\"environment=development\\"/><script src=\\"/socket.io/socket.io.js\\"></script></head><body><div> div3 </div><div> div2 </div><div> div1 </div><div id=\\"___gatsby\\"></div><script src=\\"/polyfill.js\\" nomodule=\\"\\"></script><script src=\\"/commons.js\\"></script></body></html>"`;
+exports[`develop-static-entry onPreRenderHTML can be used to replace preBodyComponents 1`] = `"<!DOCTYPE html><html><head><meta charSet=\\"utf-8\\"/><meta http-equiv=\\"x-ua-compatible\\" content=\\"ie=edge\\"/><meta name=\\"viewport\\" content=\\"width=device-width, initial-scale=1, shrink-to-fit=no\\"/><meta name=\\"note\\" content=\\"environment=development\\"/><script src=\\"/socket.io/socket.io.js\\"></script></head><body><div> div3 </div><div> div2 </div><div> div1 </div><div id=\\"___gatsby\\"><div style=\\"outline:none\\" tabindex=\\"-1\\" id=\\"gatsby-focus-wrapper\\"></div><div id=\\"gatsby-announcer\\" style=\\"position:absolute;top:0;width:1px;height:1px;padding:0;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;border:0\\" aria-live=\\"assertive\\" aria-atomic=\\"true\\"></div></div><script src=\\"/polyfill.js\\" nomodule=\\"\\"></script><script src=\\"/commons.js\\"></script></body></html>"`;
 
 exports[`static-entry onPreRenderHTML can be used to replace headComponents 1`] = `"<!DOCTYPE html><html><head><meta charSet=\\"utf-8\\"/><meta http-equiv=\\"x-ua-compatible\\" content=\\"ie=edge\\"/><meta name=\\"viewport\\" content=\\"width=device-width, initial-scale=1, shrink-to-fit=no\\"/><link as=\\"fetch\\" rel=\\"preload\\" href=\\"/page-data/app-data.json\\" crossorigin=\\"anonymous\\"/><link as=\\"fetch\\" rel=\\"preload\\" href=\\"/page-data/about/page-data.json\\" crossorigin=\\"anonymous\\"/><style> .style3  </style><style> .style2  </style><style> .style1  </style><meta name=\\"generator\\" content=\\"Gatsby 2.0.0\\"/></head><body><div id=\\"___gatsby\\"><div style=\\"outline:none\\" tabindex=\\"-1\\" id=\\"gatsby-focus-wrapper\\"></div><div id=\\"gatsby-announcer\\" style=\\"position:absolute;top:0;width:1px;height:1px;padding:0;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;border:0\\" aria-live=\\"assertive\\" aria-atomic=\\"true\\"></div></div><script id=\\"gatsby-script-loader\\">/*<![CDATA[*/window.pagePath=\\"/about/\\";/*]]>*/</script><script id=\\"gatsby-chunk-mapping\\">/*<![CDATA[*/window.___chunkMapping={};/*]]>*/</script></body></html>"`;
 
diff --git a/packages/gatsby/cache-dir/__tests__/static-entry.js b/packages/gatsby/cache-dir/__tests__/static-entry.js
index 7311c4ab1db4c..0fc3270b64ce8 100644
--- a/packages/gatsby/cache-dir/__tests__/static-entry.js
+++ b/packages/gatsby/cache-dir/__tests__/static-entry.js
@@ -138,6 +138,12 @@ const fakeComponentsPluginFactory = type => {
 }
 
 describe(`develop-static-entry`, () => {
+  beforeEach(() => {
+    global.__PATH_PREFIX__ = ``
+    global.__BASE_PATH__ = ``
+    global.__ASSET_PREFIX__ = ``
+  })
+
   test(`onPreRenderHTML can be used to replace headComponents`, done => {
     global.plugins = [fakeStylesPlugin, reverseHeadersPlugin]
 

From 4c86edb023372e6b677ec3d4b28aac83f1098cdf Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Thu, 15 Oct 2020 15:07:01 -0600
Subject: [PATCH 021/123] moer merry type work

---
 packages/gatsby/src/utils/webpack-error-utils.ts | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/gatsby/src/utils/webpack-error-utils.ts b/packages/gatsby/src/utils/webpack-error-utils.ts
index be27f0c57f66b..37e731bfc79f5 100644
--- a/packages/gatsby/src/utils/webpack-error-utils.ts
+++ b/packages/gatsby/src/utils/webpack-error-utils.ts
@@ -20,7 +20,7 @@ interface ITransformedWebpackError {
     start: string
   }
   context: {
-    stage: Stage
+    stage: StageEnum
     stageLabel: StageLabel
     sourceMessage?: string
     [key: string]: unknown
@@ -28,7 +28,7 @@ interface ITransformedWebpackError {
 }
 
 const transformWebpackError = (
-  stage: keyof typeof stageCodeToReadableLabel,
+  stage: StageEnum,
   webpackError: any
 ): ITransformedWebpackError => {
   const handlers = [
@@ -102,7 +102,7 @@ const transformWebpackError = (
 }
 
 export const structureWebpackErrors = (
-  stage: Stage,
+  stage: StageEnum,
   webpackError: any
 ): Array<ITransformedWebpackError> | ITransformedWebpackError => {
   if (Array.isArray(webpackError)) {

From f32c6823dc0e8a189ab87c8ba23679947b6f6f21 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Thu, 15 Oct 2020 15:30:38 -0600
Subject: [PATCH 022/123] Remove outdated typography.js test

---
 .../gatsby-plugin-typography/src/__tests__/gatsby-ssr.js    | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/packages/gatsby-plugin-typography/src/__tests__/gatsby-ssr.js b/packages/gatsby-plugin-typography/src/__tests__/gatsby-ssr.js
index dfbf50dc2adb9..ed708e9d75181 100644
--- a/packages/gatsby-plugin-typography/src/__tests__/gatsby-ssr.js
+++ b/packages/gatsby-plugin-typography/src/__tests__/gatsby-ssr.js
@@ -33,12 +33,6 @@ describe(`onRenderBody`, () => {
     ])
   })
 
-  it(`only invokes setHeadComponents if BUILD_STAGE is build-html`, () => {
-    const api = setup({}, `develop`)
-
-    expect(api.setHeadComponents).not.toHaveBeenCalled()
-  })
-
   it(`does not add google font if omitGoogleFont is passed`, () => {
     const api = setup({
       omitGoogleFont: true,

From 0e0cefd1c2ce428f854b7dd818e6ef863568f620 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Thu, 15 Oct 2020 17:24:36 -0600
Subject: [PATCH 023/123] Start migrating route handler to spawned service

---
 .../src/services/develop-html-route/index.ts  | 200 ++++++++++++++++++
 .../src/state-machines/develop/actions.ts     |  29 +++
 .../src/state-machines/develop/index.ts       |  27 ++-
 packages/gatsby/src/utils/start-server.ts     | 176 ---------------
 4 files changed, 254 insertions(+), 178 deletions(-)
 create mode 100644 packages/gatsby/src/services/develop-html-route/index.ts

diff --git a/packages/gatsby/src/services/develop-html-route/index.ts b/packages/gatsby/src/services/develop-html-route/index.ts
new file mode 100644
index 0000000000000..2642509e89af4
--- /dev/null
+++ b/packages/gatsby/src/services/develop-html-route/index.ts
@@ -0,0 +1,200 @@
+import { InvokeCallback } from "xstate"
+import report from "gatsby-cli/lib/reporter"
+import path from "path"
+import fs from "fs-extra"
+import { codeFrameColumns } from "@babel/code-frame"
+import ansiHTML from "ansi-html"
+
+import { renderHTML } from "../../utils/worker/render-html"
+import { Stage } from "../../commands/types"
+
+export const createDevelopHTMLRoute = ({
+  app,
+  program,
+  store,
+}): InvokeCallback => (callback, onReceive): void => {
+  interface IErrorPosition {
+    filename: string
+    line: number
+    row: number
+  }
+
+  const getPosition = function (stackObject): IErrorPosition {
+    let filename
+    let line
+    let row
+    // Because the JavaScript error stack has not yet been standardized,
+    // wrap the stack parsing in a try/catch for a soft fail if an
+    // unexpected stack is encountered.
+    try {
+      const filteredStack = stackObject.filter(function (s) {
+        return /\(.+?\)$/.test(s)
+      })
+      let splitLine
+      // For current Node & Chromium Error stacks
+      if (filteredStack.length > 0) {
+        splitLine = filteredStack[0].match(/(?:\()(.+?)(?:\))$/)[1].split(`:`)
+        // For older, future, or otherwise unexpected stacks
+      } else {
+        splitLine = stackObject[0].split(`:`)
+      }
+      const splitLength = splitLine.length
+      filename = splitLine[splitLength - 3]
+      line = Number(splitLine[splitLength - 2])
+      row = Number(splitLine[splitLength - 1])
+    } catch (err) {
+      filename = ``
+      line = 0
+      row = 0
+    }
+    return {
+      filename: filename,
+      line: line,
+      row: row,
+    }
+  }
+  // Colors taken from Gatsby's design tokens
+  // https://github.com/gatsbyjs/gatsby/blob/d8acab3a135fa8250a0eb3a47c67300dde6eae32/packages/gatsby-design-tokens/src/colors.js#L185-L205
+  const colors = {
+    background: `fdfaf6`,
+    text: `452475`,
+    green: `137886`,
+    darkGreen: `006500`,
+    comment: `527713`,
+    keyword: `096fb3`,
+    yellow: `DB3A00`,
+  }
+
+  interface IParsedError {
+    filename: string
+    code: string
+    codeFrame: string
+    line: number
+    row: number
+    message: string
+    type: string
+    stack: [string]
+  }
+
+  const parseError = function (err): IParsedError {
+    const stack = err.stack ? err.stack : ``
+    const stackObject = stack.split(`\n`)
+    const position = getPosition(stackObject)
+    // Remove the `/lib/` added by webpack
+    const filename = path.join(
+      program.directory,
+      ...position.filename.split(path.sep).slice(2)
+    )
+    const code = fs.readFileSync(filename, `utf-8`)
+    const line = position.line
+    const row = position.row
+    ansiHTML.setColors({
+      reset: [colors.text, colors.background], // FOREGROUND-COLOR or [FOREGROUND-COLOR] or [, BACKGROUND-COLOR] or [FOREGROUND-COLOR, BACKGROUND-COLOR]
+      black: `aaa`, // String
+      red: colors.keyword,
+      green: colors.green,
+      yellow: colors.yellow,
+      blue: `eee`,
+      magenta: `fff`,
+      cyan: colors.darkGreen,
+      lightgrey: `888`,
+      darkgrey: colors.comment,
+    })
+    const codeFrame = ansiHTML(
+      codeFrameColumns(
+        code,
+        {
+          start: { line: line, column: row },
+        },
+        { forceColor: true }
+      )
+    )
+    const splitMessage = err.message ? err.message.split(`\n`) : [``]
+    const message = splitMessage[splitMessage.length - 1]
+    const type = err.type ? err.type : err.name
+    const data = {
+      filename: filename,
+      code,
+      codeFrame,
+      line: line,
+      row: row,
+      message: message,
+      type: type,
+      stack: stack,
+    }
+    return data
+  }
+
+  let outsideResolve
+  onReceive(event => {
+    console.log({ event })
+    if (event.type === `SEND_DEVELOP_HTML_RESPONSES`) {
+      outsideResolve()
+    }
+  })
+  // Render an HTML page and serve it.
+  app.get(`*`, async (req, res, next) => {
+    const { pages } = store.getState()
+
+    if (!pages.has(req.path)) {
+      return next()
+    }
+
+    // Sleep until any work the server is doing has finished.
+    await new Promise(resolve => {
+      outsideResolve = resolve
+      callback("DEVELOP_HTML_REQUEST_RECEIVED")
+      console.log(`waiting for response`)
+    })
+
+    await new Promise(resolve => {
+      if (program.developMachineService._state.value == `waiting`) {
+        resolve()
+      } else {
+        const intervalId = setInterval(() => {
+          if (program.developMachineService._state.value == `waiting`) {
+            clearInterval(intervalId)
+            resolve()
+          }
+        }, 50)
+      }
+    })
+
+    const htmlActivity = report.phantomActivity(`building HTML for path`, {})
+    htmlActivity.start()
+
+    try {
+      const renderResponse = await renderHTML({
+        htmlComponentRendererPath: `${program.directory}/public/render-page.js`,
+        paths: [req.path],
+        stage: Stage.DevelopHTML,
+        envVars: [
+          [`NODE_ENV`, process.env.NODE_ENV || ``],
+          [
+            `gatsby_executing_command`,
+            process.env.gatsby_executing_command || ``,
+          ],
+          [`gatsby_log_level`, process.env.gatsby_log_level || ``],
+        ],
+      })
+      res.status(200).send(renderResponse[0])
+    } catch (e) {
+      const error = parseError(e)
+      res.status(500).send(`<title>Develop SSR Error</title><h1>Error<h1>
+        <h2>The page didn't SSR correctly</h2>
+        <ul>
+          <li><strong>URL path:</strong> <code>${req.path}</code></li>
+          <li><strong>File path:</strong> <code>${error.filename}</code></li>
+        </ul>
+        <h3>error message</h3>
+        <p><code>${error.message}</code></p>
+        <pre style="background:#${colors.background};padding:8px;">${error.codeFrame}</pre>`)
+    }
+
+    // TODO add support for 404 and general rendering errors
+    htmlActivity.end()
+
+    // Make eslint happy
+    return null
+  })
+}
diff --git a/packages/gatsby/src/state-machines/develop/actions.ts b/packages/gatsby/src/state-machines/develop/actions.ts
index 3fcf59f1052b8..57909e5ec86eb 100644
--- a/packages/gatsby/src/state-machines/develop/actions.ts
+++ b/packages/gatsby/src/state-machines/develop/actions.ts
@@ -1,10 +1,12 @@
 import {
+  actions
   assign,
   AnyEventObject,
   ActionFunction,
   spawn,
   ActionFunctionMap,
   DoneEventObject,
+  send,
 } from "xstate"
 import { IBuildContext } from "../../services"
 import { boundActionCreators } from "../../redux/actions"
@@ -14,7 +16,11 @@ import { saveState } from "../../db"
 import reporter from "gatsby-cli/lib/reporter"
 import { ProgramStatus } from "../../redux/types"
 import { createWebpackWatcher } from "../../services/listen-to-webpack"
+import { createDevelopHTMLRoute } from "../../services/develop-html-route"
 import { callRealApi } from "../../utils/call-deferred-api"
+
+const { pure } = actions;
+
 /**
  * Handler for when we're inside handlers that should be able to mutate nodes
  * Instead of queueing, we call it right away
@@ -118,6 +124,27 @@ export const spawnWebpackListener = assign<IBuildContext, AnyEventObject>({
   },
 })
 
+export const spawnDevelopHtmlRouteHandler = assign<
+  IBuildContext,
+  AnyEventObject
+>({
+  developHTMLRoute: ({ app, program, store }) => {
+    return spawn(createDevelopHTMLRoute({ app, program, store }))
+  },
+})
+
+export const sendDevHtmlResponses = pure(() => {
+  console.log(`hi?? sendDevHtmlResponses`)
+  const sendAction = send("SEND_DEVELOP_HTML_RESPONSES", {
+    to: context => {
+      console.log(Object.keys(context))
+      return context.developHTMLRoute
+    },
+  })
+  console.log(sendAction)
+  return sendAction
+})
+
 export const assignWebhookBody = assign<IBuildContext, AnyEventObject>({
   webhookBody: (_context, { payload }) => payload?.webhookBody,
 })
@@ -170,6 +197,8 @@ export const buildActions: ActionFunctionMap<IBuildContext, AnyEventObject> = {
   clearWebhookBody,
   finishParentSpan,
   spawnWebpackListener,
+  spawnDevelopHtmlRouteHandler,
+  sendDevHtmlResponses,
   markSourceFilesDirty,
   markSourceFilesClean,
   markNodesClean,
diff --git a/packages/gatsby/src/state-machines/develop/index.ts b/packages/gatsby/src/state-machines/develop/index.ts
index d88848b345c99..cb4957a5b5e09 100644
--- a/packages/gatsby/src/state-machines/develop/index.ts
+++ b/packages/gatsby/src/state-machines/develop/index.ts
@@ -1,4 +1,4 @@
-import { MachineConfig, AnyEventObject, forwardTo, Machine } from "xstate"
+import { MachineConfig, AnyEventObject, forwardTo, Machine, send } from "xstate"
 import { IDataLayerContext } from "../data-layer/types"
 import { IQueryRunningContext } from "../query-running/types"
 import { IWaitingContext } from "../waiting/types"
@@ -31,6 +31,19 @@ const developConfig: MachineConfig<IBuildContext, any, AnyEventObject> = {
       target: `reloadingData`,
       actions: `assignWebhookBody`,
     },
+    // Global handler for event — make sure we never wait more than 1 second
+    // to respond to requests to the develop server. We'd prefer to wait
+    // for all sourcing/transforming/query running to finish but if this is taking
+    // awhile, we'll show a stale version of the page and trust hot reloading
+    // to push the work in progress once it finishes.
+    DEVELOP_HTML_REQUEST_RECEIVED: {
+      actions: [
+        send("SEND_DEVELOP_HTML_RESPONSES", {
+          to: context => context.developHTMLRoute,
+          delay: 1000,
+        }),
+      ],
+    },
   },
   states: {
     // Here we handle the initial bootstrap
@@ -194,6 +207,7 @@ const developConfig: MachineConfig<IBuildContext, any, AnyEventObject> = {
           actions: [
             `assignServers`,
             `spawnWebpackListener`,
+            `spawnDevelopHtmlRouteHandler`,
             `markSourceFilesClean`,
           ],
         },
@@ -205,7 +219,7 @@ const developConfig: MachineConfig<IBuildContext, any, AnyEventObject> = {
     },
     // Idle, waiting for events that make us rebuild
     waiting: {
-      entry: [`saveDbState`, `resetRecompileCount`],
+      entry: [`saveDbState`, `resetRecompileCount`, `sendDevHtmlResponses`],
       on: {
         // Forward these events to the child machine, so it can handle batching
         ADD_NODE_MUTATION: {
@@ -218,6 +232,15 @@ const developConfig: MachineConfig<IBuildContext, any, AnyEventObject> = {
         EXTRACT_QUERIES_NOW: {
           target: `runningQueries`,
         },
+        DEVELOP_HTML_REQUEST_RECEIVED: {
+          actions: [
+            // I don't get why this isn't working.
+            `sendDevHtmlResponses`,
+            send("SEND_DEVELOP_HTML_RESPONSES", {
+              to: context => context.developHTMLRoute,
+            }),
+          ],
+        },
       },
       invoke: {
         id: `waiting`,
diff --git a/packages/gatsby/src/utils/start-server.ts b/packages/gatsby/src/utils/start-server.ts
index bf08d43c789b8..9b4e4e33bbdd3 100644
--- a/packages/gatsby/src/utils/start-server.ts
+++ b/packages/gatsby/src/utils/start-server.ts
@@ -10,10 +10,6 @@ import graphqlHTTP from "express-graphql"
 import graphqlPlayground from "graphql-playground-middleware-express"
 import graphiqlExplorer from "gatsby-graphiql-explorer"
 import { formatError } from "graphql"
-import path from "path"
-import fs from "fs-extra"
-import { codeFrameColumns } from "@babel/code-frame"
-import ansiHTML from "ansi-html"
 import { slash } from "gatsby-core-utils"
 
 import webpackConfig from "../utils/webpack.config"
@@ -25,7 +21,6 @@ import launchEditor from "react-dev-utils/launchEditor"
 import cors from "cors"
 import telemetry from "gatsby-telemetry"
 import * as WorkerPool from "../utils/worker/pool"
-import { renderHTML } from "../utils/worker/render-html"
 import http from "http"
 import https from "https"
 
@@ -248,177 +243,6 @@ export async function startServer(
     res.status(404).end()
   })
 
-  interface IErrorPosition {
-    filename: string
-    line: number
-    row: number
-  }
-  const getPosition = function (stackObject): IErrorPosition {
-    let filename
-    let line
-    let row
-    // Because the JavaScript error stack has not yet been standardized,
-    // wrap the stack parsing in a try/catch for a soft fail if an
-    // unexpected stack is encountered.
-    try {
-      const filteredStack = stackObject.filter(function (s) {
-        return /\(.+?\)$/.test(s)
-      })
-      let splitLine
-      // For current Node & Chromium Error stacks
-      if (filteredStack.length > 0) {
-        splitLine = filteredStack[0].match(/(?:\()(.+?)(?:\))$/)[1].split(`:`)
-        // For older, future, or otherwise unexpected stacks
-      } else {
-        splitLine = stackObject[0].split(`:`)
-      }
-      const splitLength = splitLine.length
-      filename = splitLine[splitLength - 3]
-      line = Number(splitLine[splitLength - 2])
-      row = Number(splitLine[splitLength - 1])
-    } catch (err) {
-      filename = ``
-      line = 0
-      row = 0
-    }
-    return {
-      filename: filename,
-      line: line,
-      row: row,
-    }
-  }
-  // Colors taken from Gatsby's design tokens
-  // https://github.com/gatsbyjs/gatsby/blob/d8acab3a135fa8250a0eb3a47c67300dde6eae32/packages/gatsby-design-tokens/src/colors.js#L185-L205
-  const colors = {
-    background: `fdfaf6`,
-    text: `452475`,
-    green: `137886`,
-    darkGreen: `006500`,
-    comment: `527713`,
-    keyword: `096fb3`,
-    yellow: `DB3A00`,
-  }
-
-  interface IParsedError {
-    filename: string
-    code: string
-    codeFrame: string
-    line: number
-    row: number
-    message: string
-    type: string
-    stack: [string]
-  }
-
-  const parseError = function (err): IParsedError {
-    const stack = err.stack ? err.stack : ``
-    const stackObject = stack.split(`\n`)
-    const position = getPosition(stackObject)
-    // Remove the `/lib/` added by webpack
-    const filename = path.join(
-      directory,
-      ...position.filename.split(path.sep).slice(2)
-    )
-    const code = fs.readFileSync(filename, `utf-8`)
-    const line = position.line
-    const row = position.row
-    ansiHTML.setColors({
-      reset: [colors.text, colors.background], // FOREGROUND-COLOR or [FOREGROUND-COLOR] or [, BACKGROUND-COLOR] or [FOREGROUND-COLOR, BACKGROUND-COLOR]
-      black: `aaa`, // String
-      red: colors.keyword,
-      green: colors.green,
-      yellow: colors.yellow,
-      blue: `eee`,
-      magenta: `fff`,
-      cyan: colors.darkGreen,
-      lightgrey: `888`,
-      darkgrey: colors.comment,
-    })
-    const codeFrame = ansiHTML(
-      codeFrameColumns(
-        code,
-        {
-          start: { line: line, column: row },
-        },
-        { forceColor: true }
-      )
-    )
-    const splitMessage = err.message ? err.message.split(`\n`) : [``]
-    const message = splitMessage[splitMessage.length - 1]
-    const type = err.type ? err.type : err.name
-    const data = {
-      filename: filename,
-      code,
-      codeFrame,
-      line: line,
-      row: row,
-      message: message,
-      type: type,
-      stack: stack,
-    }
-    return data
-  }
-
-  // Render an HTML page and serve it.
-  app.use(async (req, res, next) => {
-    const { pages } = store.getState()
-
-    if (!pages.has(req.path)) {
-      return next()
-    }
-
-    // Sleep until any work the server is doing has finished.
-    await new Promise(resolve => {
-      if (program.developMachineService._state.value == `waiting`) {
-        resolve()
-      } else {
-        const intervalId = setInterval(() => {
-          if (program.developMachineService._state.value == `waiting`) {
-            clearInterval(intervalId)
-            resolve()
-          }
-        }, 50)
-      }
-    })
-
-    const htmlActivity = report.phantomActivity(`building HTML for path`, {})
-    htmlActivity.start()
-
-    try {
-      const renderResponse = await renderHTML({
-        htmlComponentRendererPath: `${program.directory}/public/render-page.js`,
-        paths: [req.path],
-        stage: Stage.DevelopHTML,
-        envVars: [
-          [`NODE_ENV`, process.env.NODE_ENV || ``],
-          [
-            `gatsby_executing_command`,
-            process.env.gatsby_executing_command || ``,
-          ],
-          [`gatsby_log_level`, process.env.gatsby_log_level || ``],
-        ],
-      })
-      res.status(200).send(renderResponse[0])
-    } catch (e) {
-      const error = parseError(e)
-      res.status(500).send(`<title>Develop SSR Error</title><h1>Error<h1>
-        <h2>The page didn't SSR correctly</h2>
-        <ul>
-          <li><strong>URL path:</strong> <code>${req.path}</code></li>
-          <li><strong>File path:</strong> <code>${error.filename}</code></li>
-        </ul>
-        <h3>error message</h3>
-        <p><code>${error.message}</code></p>
-        <pre style="background:#${colors.background};padding:8px;">${error.codeFrame}</pre>`)
-    }
-
-    // TODO add support for 404 and general rendering errors
-    htmlActivity.end()
-
-    // Make eslint happy
-    return null
-  })
-
   // Disable directory indexing i.e. serving index.html from a directory.
   // This can lead to serving stale html files during development.
   //

From 494415f5c0f99cebdf8e7a5cad227ab7e17291ee Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Fri, 16 Oct 2020 10:43:27 -0600
Subject: [PATCH 024/123] back out moving dev html route into state machine

---
 .../src/services/develop-html-route/index.ts  | 200 ------------------
 .../src/state-machines/develop/actions.ts     |  11 -
 .../src/state-machines/develop/index.ts       |  25 +--
 .../gatsby/src/utils/develop-html-route.ts    | 169 +++++++++++++++
 packages/gatsby/src/utils/start-server.ts     |  10 +-
 5 files changed, 177 insertions(+), 238 deletions(-)
 delete mode 100644 packages/gatsby/src/services/develop-html-route/index.ts
 create mode 100644 packages/gatsby/src/utils/develop-html-route.ts

diff --git a/packages/gatsby/src/services/develop-html-route/index.ts b/packages/gatsby/src/services/develop-html-route/index.ts
deleted file mode 100644
index 2642509e89af4..0000000000000
--- a/packages/gatsby/src/services/develop-html-route/index.ts
+++ /dev/null
@@ -1,200 +0,0 @@
-import { InvokeCallback } from "xstate"
-import report from "gatsby-cli/lib/reporter"
-import path from "path"
-import fs from "fs-extra"
-import { codeFrameColumns } from "@babel/code-frame"
-import ansiHTML from "ansi-html"
-
-import { renderHTML } from "../../utils/worker/render-html"
-import { Stage } from "../../commands/types"
-
-export const createDevelopHTMLRoute = ({
-  app,
-  program,
-  store,
-}): InvokeCallback => (callback, onReceive): void => {
-  interface IErrorPosition {
-    filename: string
-    line: number
-    row: number
-  }
-
-  const getPosition = function (stackObject): IErrorPosition {
-    let filename
-    let line
-    let row
-    // Because the JavaScript error stack has not yet been standardized,
-    // wrap the stack parsing in a try/catch for a soft fail if an
-    // unexpected stack is encountered.
-    try {
-      const filteredStack = stackObject.filter(function (s) {
-        return /\(.+?\)$/.test(s)
-      })
-      let splitLine
-      // For current Node & Chromium Error stacks
-      if (filteredStack.length > 0) {
-        splitLine = filteredStack[0].match(/(?:\()(.+?)(?:\))$/)[1].split(`:`)
-        // For older, future, or otherwise unexpected stacks
-      } else {
-        splitLine = stackObject[0].split(`:`)
-      }
-      const splitLength = splitLine.length
-      filename = splitLine[splitLength - 3]
-      line = Number(splitLine[splitLength - 2])
-      row = Number(splitLine[splitLength - 1])
-    } catch (err) {
-      filename = ``
-      line = 0
-      row = 0
-    }
-    return {
-      filename: filename,
-      line: line,
-      row: row,
-    }
-  }
-  // Colors taken from Gatsby's design tokens
-  // https://github.com/gatsbyjs/gatsby/blob/d8acab3a135fa8250a0eb3a47c67300dde6eae32/packages/gatsby-design-tokens/src/colors.js#L185-L205
-  const colors = {
-    background: `fdfaf6`,
-    text: `452475`,
-    green: `137886`,
-    darkGreen: `006500`,
-    comment: `527713`,
-    keyword: `096fb3`,
-    yellow: `DB3A00`,
-  }
-
-  interface IParsedError {
-    filename: string
-    code: string
-    codeFrame: string
-    line: number
-    row: number
-    message: string
-    type: string
-    stack: [string]
-  }
-
-  const parseError = function (err): IParsedError {
-    const stack = err.stack ? err.stack : ``
-    const stackObject = stack.split(`\n`)
-    const position = getPosition(stackObject)
-    // Remove the `/lib/` added by webpack
-    const filename = path.join(
-      program.directory,
-      ...position.filename.split(path.sep).slice(2)
-    )
-    const code = fs.readFileSync(filename, `utf-8`)
-    const line = position.line
-    const row = position.row
-    ansiHTML.setColors({
-      reset: [colors.text, colors.background], // FOREGROUND-COLOR or [FOREGROUND-COLOR] or [, BACKGROUND-COLOR] or [FOREGROUND-COLOR, BACKGROUND-COLOR]
-      black: `aaa`, // String
-      red: colors.keyword,
-      green: colors.green,
-      yellow: colors.yellow,
-      blue: `eee`,
-      magenta: `fff`,
-      cyan: colors.darkGreen,
-      lightgrey: `888`,
-      darkgrey: colors.comment,
-    })
-    const codeFrame = ansiHTML(
-      codeFrameColumns(
-        code,
-        {
-          start: { line: line, column: row },
-        },
-        { forceColor: true }
-      )
-    )
-    const splitMessage = err.message ? err.message.split(`\n`) : [``]
-    const message = splitMessage[splitMessage.length - 1]
-    const type = err.type ? err.type : err.name
-    const data = {
-      filename: filename,
-      code,
-      codeFrame,
-      line: line,
-      row: row,
-      message: message,
-      type: type,
-      stack: stack,
-    }
-    return data
-  }
-
-  let outsideResolve
-  onReceive(event => {
-    console.log({ event })
-    if (event.type === `SEND_DEVELOP_HTML_RESPONSES`) {
-      outsideResolve()
-    }
-  })
-  // Render an HTML page and serve it.
-  app.get(`*`, async (req, res, next) => {
-    const { pages } = store.getState()
-
-    if (!pages.has(req.path)) {
-      return next()
-    }
-
-    // Sleep until any work the server is doing has finished.
-    await new Promise(resolve => {
-      outsideResolve = resolve
-      callback("DEVELOP_HTML_REQUEST_RECEIVED")
-      console.log(`waiting for response`)
-    })
-
-    await new Promise(resolve => {
-      if (program.developMachineService._state.value == `waiting`) {
-        resolve()
-      } else {
-        const intervalId = setInterval(() => {
-          if (program.developMachineService._state.value == `waiting`) {
-            clearInterval(intervalId)
-            resolve()
-          }
-        }, 50)
-      }
-    })
-
-    const htmlActivity = report.phantomActivity(`building HTML for path`, {})
-    htmlActivity.start()
-
-    try {
-      const renderResponse = await renderHTML({
-        htmlComponentRendererPath: `${program.directory}/public/render-page.js`,
-        paths: [req.path],
-        stage: Stage.DevelopHTML,
-        envVars: [
-          [`NODE_ENV`, process.env.NODE_ENV || ``],
-          [
-            `gatsby_executing_command`,
-            process.env.gatsby_executing_command || ``,
-          ],
-          [`gatsby_log_level`, process.env.gatsby_log_level || ``],
-        ],
-      })
-      res.status(200).send(renderResponse[0])
-    } catch (e) {
-      const error = parseError(e)
-      res.status(500).send(`<title>Develop SSR Error</title><h1>Error<h1>
-        <h2>The page didn't SSR correctly</h2>
-        <ul>
-          <li><strong>URL path:</strong> <code>${req.path}</code></li>
-          <li><strong>File path:</strong> <code>${error.filename}</code></li>
-        </ul>
-        <h3>error message</h3>
-        <p><code>${error.message}</code></p>
-        <pre style="background:#${colors.background};padding:8px;">${error.codeFrame}</pre>`)
-    }
-
-    // TODO add support for 404 and general rendering errors
-    htmlActivity.end()
-
-    // Make eslint happy
-    return null
-  })
-}
diff --git a/packages/gatsby/src/state-machines/develop/actions.ts b/packages/gatsby/src/state-machines/develop/actions.ts
index 57909e5ec86eb..bfe90eb4a6c23 100644
--- a/packages/gatsby/src/state-machines/develop/actions.ts
+++ b/packages/gatsby/src/state-machines/develop/actions.ts
@@ -16,7 +16,6 @@ import { saveState } from "../../db"
 import reporter from "gatsby-cli/lib/reporter"
 import { ProgramStatus } from "../../redux/types"
 import { createWebpackWatcher } from "../../services/listen-to-webpack"
-import { createDevelopHTMLRoute } from "../../services/develop-html-route"
 import { callRealApi } from "../../utils/call-deferred-api"
 
 const { pure } = actions;
@@ -124,15 +123,6 @@ export const spawnWebpackListener = assign<IBuildContext, AnyEventObject>({
   },
 })
 
-export const spawnDevelopHtmlRouteHandler = assign<
-  IBuildContext,
-  AnyEventObject
->({
-  developHTMLRoute: ({ app, program, store }) => {
-    return spawn(createDevelopHTMLRoute({ app, program, store }))
-  },
-})
-
 export const sendDevHtmlResponses = pure(() => {
   console.log(`hi?? sendDevHtmlResponses`)
   const sendAction = send("SEND_DEVELOP_HTML_RESPONSES", {
@@ -197,7 +187,6 @@ export const buildActions: ActionFunctionMap<IBuildContext, AnyEventObject> = {
   clearWebhookBody,
   finishParentSpan,
   spawnWebpackListener,
-  spawnDevelopHtmlRouteHandler,
   sendDevHtmlResponses,
   markSourceFilesDirty,
   markSourceFilesClean,
diff --git a/packages/gatsby/src/state-machines/develop/index.ts b/packages/gatsby/src/state-machines/develop/index.ts
index cb4957a5b5e09..b995ffbcf913a 100644
--- a/packages/gatsby/src/state-machines/develop/index.ts
+++ b/packages/gatsby/src/state-machines/develop/index.ts
@@ -31,19 +31,6 @@ const developConfig: MachineConfig<IBuildContext, any, AnyEventObject> = {
       target: `reloadingData`,
       actions: `assignWebhookBody`,
     },
-    // Global handler for event — make sure we never wait more than 1 second
-    // to respond to requests to the develop server. We'd prefer to wait
-    // for all sourcing/transforming/query running to finish but if this is taking
-    // awhile, we'll show a stale version of the page and trust hot reloading
-    // to push the work in progress once it finishes.
-    DEVELOP_HTML_REQUEST_RECEIVED: {
-      actions: [
-        send("SEND_DEVELOP_HTML_RESPONSES", {
-          to: context => context.developHTMLRoute,
-          delay: 1000,
-        }),
-      ],
-    },
   },
   states: {
     // Here we handle the initial bootstrap
@@ -207,7 +194,6 @@ const developConfig: MachineConfig<IBuildContext, any, AnyEventObject> = {
           actions: [
             `assignServers`,
             `spawnWebpackListener`,
-            `spawnDevelopHtmlRouteHandler`,
             `markSourceFilesClean`,
           ],
         },
@@ -219,7 +205,7 @@ const developConfig: MachineConfig<IBuildContext, any, AnyEventObject> = {
     },
     // Idle, waiting for events that make us rebuild
     waiting: {
-      entry: [`saveDbState`, `resetRecompileCount`, `sendDevHtmlResponses`],
+      entry: [`saveDbState`, `resetRecompileCount`],
       on: {
         // Forward these events to the child machine, so it can handle batching
         ADD_NODE_MUTATION: {
@@ -232,15 +218,6 @@ const developConfig: MachineConfig<IBuildContext, any, AnyEventObject> = {
         EXTRACT_QUERIES_NOW: {
           target: `runningQueries`,
         },
-        DEVELOP_HTML_REQUEST_RECEIVED: {
-          actions: [
-            // I don't get why this isn't working.
-            `sendDevHtmlResponses`,
-            send("SEND_DEVELOP_HTML_RESPONSES", {
-              to: context => context.developHTMLRoute,
-            }),
-          ],
-        },
       },
       invoke: {
         id: `waiting`,
diff --git a/packages/gatsby/src/utils/develop-html-route.ts b/packages/gatsby/src/utils/develop-html-route.ts
new file mode 100644
index 0000000000000..0fa57cde77ef8
--- /dev/null
+++ b/packages/gatsby/src/utils/develop-html-route.ts
@@ -0,0 +1,169 @@
+import { InvokeCallback } from "xstate"
+import report from "gatsby-cli/lib/reporter"
+import path from "path"
+import fs from "fs-extra"
+import { codeFrameColumns } from "@babel/code-frame"
+import ansiHTML from "ansi-html"
+
+import { renderHTML } from "./worker/render-html"
+import { Stage } from "../commands/types"
+
+interface IErrorPosition {
+  filename: string
+  line: number
+  row: number
+}
+
+const getPosition = function (stackObject): IErrorPosition {
+  let filename
+  let line
+  let row
+  // Because the JavaScript error stack has not yet been standardized,
+  // wrap the stack parsing in a try/catch for a soft fail if an
+  // unexpected stack is encountered.
+  try {
+    const filteredStack = stackObject.filter(function (s) {
+      return /\(.+?\)$/.test(s)
+    })
+    let splitLine
+    // For current Node & Chromium Error stacks
+    if (filteredStack.length > 0) {
+      splitLine = filteredStack[0].match(/(?:\()(.+?)(?:\))$/)[1].split(`:`)
+      // For older, future, or otherwise unexpected stacks
+    } else {
+      splitLine = stackObject[0].split(`:`)
+    }
+    const splitLength = splitLine.length
+    filename = splitLine[splitLength - 3]
+    line = Number(splitLine[splitLength - 2])
+    row = Number(splitLine[splitLength - 1])
+  } catch (err) {
+    filename = ``
+    line = 0
+    row = 0
+  }
+  return {
+    filename: filename,
+    line: line,
+    row: row,
+  }
+}
+// Colors taken from Gatsby's design tokens
+// https://github.com/gatsbyjs/gatsby/blob/d8acab3a135fa8250a0eb3a47c67300dde6eae32/packages/gatsby-design-tokens/src/colors.js#L185-L205
+const colors = {
+  background: `fdfaf6`,
+  text: `452475`,
+  green: `137886`,
+  darkGreen: `006500`,
+  comment: `527713`,
+  keyword: `096fb3`,
+  yellow: `DB3A00`,
+}
+
+interface IParsedError {
+  filename: string
+  code: string
+  codeFrame: string
+  line: number
+  row: number
+  message: string
+  type: string
+  stack: [string]
+}
+
+const parseError = function (err, directory): IParsedError {
+  const stack = err.stack ? err.stack : ``
+  const stackObject = stack.split(`\n`)
+  const position = getPosition(stackObject)
+  // Remove the `/lib/` added by webpack
+  const filename = path.join(
+    directory,
+    ...position.filename.split(path.sep).slice(2)
+  )
+  const code = fs.readFileSync(filename, `utf-8`)
+  const line = position.line
+  const row = position.row
+  ansiHTML.setColors({
+    reset: [colors.text, colors.background], // FOREGROUND-COLOR or [FOREGROUND-COLOR] or [, BACKGROUND-COLOR] or [FOREGROUND-COLOR, BACKGROUND-COLOR]
+    black: `aaa`, // String
+    red: colors.keyword,
+    green: colors.green,
+    yellow: colors.yellow,
+    blue: `eee`,
+    magenta: `fff`,
+    cyan: colors.darkGreen,
+    lightgrey: `888`,
+    darkgrey: colors.comment,
+  })
+  const codeFrame = ansiHTML(
+    codeFrameColumns(
+      code,
+      {
+        start: { line: line, column: row },
+      },
+      { forceColor: true }
+    )
+  )
+  const splitMessage = err.message ? err.message.split(`\n`) : [``]
+  const message = splitMessage[splitMessage.length - 1]
+  const type = err.type ? err.type : err.name
+  const data = {
+    filename: filename,
+    code,
+    codeFrame,
+    line: line,
+    row: row,
+    message: message,
+    type: type,
+    stack: stack,
+  }
+  return data
+}
+
+export const route = ({ app, program, directory, store }) => {
+  // Render an HTML page and serve it.
+  app.get(`*`, async (req, res, next) => {
+    const { pages } = store.getState()
+
+    if (!pages.has(req.path)) {
+      return next()
+    }
+
+    const htmlActivity = report.phantomActivity(`building HTML for path`, {})
+    htmlActivity.start()
+
+    try {
+      const renderResponse = await renderHTML({
+        htmlComponentRendererPath: `${program.directory}/public/render-page.js`,
+        paths: [req.path],
+        stage: Stage.DevelopHTML,
+        envVars: [
+          [`NODE_ENV`, process.env.NODE_ENV || ``],
+          [
+            `gatsby_executing_command`,
+            process.env.gatsby_executing_command || ``,
+          ],
+          [`gatsby_log_level`, process.env.gatsby_log_level || ``],
+        ],
+      })
+      res.status(200).send(renderResponse[0])
+    } catch (e) {
+      const error = parseError(e, program.directory)
+      res.status(500).send(`<title>Develop SSR Error</title><h1>Error<h1>
+        <h2>The page didn't SSR correctly</h2>
+        <ul>
+          <li><strong>URL path:</strong> <code>${req.path}</code></li>
+          <li><strong>File path:</strong> <code>${error.filename}</code></li>
+        </ul>
+        <h3>error message</h3>
+        <p><code>${error.message}</code></p>
+        <pre style="background:#${colors.background};padding:8px;">${error.codeFrame}</pre>`)
+    }
+
+    // TODO add support for 404 and general rendering errors
+    htmlActivity.end()
+
+    // Make eslint happy
+    return null
+  })
+}
diff --git a/packages/gatsby/src/utils/start-server.ts b/packages/gatsby/src/utils/start-server.ts
index 9b4e4e33bbdd3..5b88e7f610dd8 100644
--- a/packages/gatsby/src/utils/start-server.ts
+++ b/packages/gatsby/src/utils/start-server.ts
@@ -11,6 +11,9 @@ import graphqlPlayground from "graphql-playground-middleware-express"
 import graphiqlExplorer from "gatsby-graphiql-explorer"
 import { formatError } from "graphql"
 import { slash } from "gatsby-core-utils"
+import telemetry from "gatsby-telemetry"
+import http from "http"
+import https from "https"
 
 import webpackConfig from "../utils/webpack.config"
 import { store, emitter } from "../redux"
@@ -19,10 +22,8 @@ import { withBasePath } from "../utils/path"
 import report from "gatsby-cli/lib/reporter"
 import launchEditor from "react-dev-utils/launchEditor"
 import cors from "cors"
-import telemetry from "gatsby-telemetry"
 import * as WorkerPool from "../utils/worker/pool"
-import http from "http"
-import https from "https"
+import { route as developHtmlRoute } from "./develop-html-route"
 
 import { developStatic } from "../commands/develop-static"
 import withResolverContext from "../schema/context"
@@ -177,6 +178,9 @@ export async function startServer(
     res.end()
   })
 
+  // Setup HTML route.
+  developHtmlRoute({ app, program, directory, store })
+
   app.get(`/__open-stack-frame-in-editor`, (req, res) => {
     launchEditor(req.query.fileName, req.query.lineNumber)
     res.end()

From ff59c2d68407a4ce12b61b63b6bad8e225d9d57a Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Fri, 16 Oct 2020 10:46:15 -0600
Subject: [PATCH 025/123] cleanups

---
 packages/gatsby/src/state-machines/develop/actions.ts | 3 ---
 packages/gatsby/src/state-machines/develop/index.ts   | 2 +-
 packages/gatsby/src/utils/develop-html-route.ts       | 5 ++---
 3 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/packages/gatsby/src/state-machines/develop/actions.ts b/packages/gatsby/src/state-machines/develop/actions.ts
index bfe90eb4a6c23..98b3f2243614a 100644
--- a/packages/gatsby/src/state-machines/develop/actions.ts
+++ b/packages/gatsby/src/state-machines/develop/actions.ts
@@ -1,5 +1,4 @@
 import {
-  actions
   assign,
   AnyEventObject,
   ActionFunction,
@@ -18,8 +17,6 @@ import { ProgramStatus } from "../../redux/types"
 import { createWebpackWatcher } from "../../services/listen-to-webpack"
 import { callRealApi } from "../../utils/call-deferred-api"
 
-const { pure } = actions;
-
 /**
  * Handler for when we're inside handlers that should be able to mutate nodes
  * Instead of queueing, we call it right away
diff --git a/packages/gatsby/src/state-machines/develop/index.ts b/packages/gatsby/src/state-machines/develop/index.ts
index b995ffbcf913a..d88848b345c99 100644
--- a/packages/gatsby/src/state-machines/develop/index.ts
+++ b/packages/gatsby/src/state-machines/develop/index.ts
@@ -1,4 +1,4 @@
-import { MachineConfig, AnyEventObject, forwardTo, Machine, send } from "xstate"
+import { MachineConfig, AnyEventObject, forwardTo, Machine } from "xstate"
 import { IDataLayerContext } from "../data-layer/types"
 import { IQueryRunningContext } from "../query-running/types"
 import { IWaitingContext } from "../waiting/types"
diff --git a/packages/gatsby/src/utils/develop-html-route.ts b/packages/gatsby/src/utils/develop-html-route.ts
index 0fa57cde77ef8..6c273e8504b83 100644
--- a/packages/gatsby/src/utils/develop-html-route.ts
+++ b/packages/gatsby/src/utils/develop-html-route.ts
@@ -1,4 +1,3 @@
-import { InvokeCallback } from "xstate"
 import report from "gatsby-cli/lib/reporter"
 import path from "path"
 import fs from "fs-extra"
@@ -120,9 +119,9 @@ const parseError = function (err, directory): IParsedError {
   return data
 }
 
-export const route = ({ app, program, directory, store }) => {
+export const route = ({ app, program, store }) => {
   // Render an HTML page and serve it.
-  app.get(`*`, async (req, res, next) => {
+  return app.get(`*`, async (req, res, next) => {
     const { pages } = store.getState()
 
     if (!pages.has(req.path)) {

From 93e8fa1057490fc3bbbf2c2b36d9f78bc29450e5 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Fri, 16 Oct 2020 10:46:53 -0600
Subject: [PATCH 026/123] more cleanups

---
 .../gatsby/src/state-machines/develop/actions.ts     | 12 ------------
 packages/gatsby/src/utils/develop-html-route.ts      |  5 ++---
 2 files changed, 2 insertions(+), 15 deletions(-)

diff --git a/packages/gatsby/src/state-machines/develop/actions.ts b/packages/gatsby/src/state-machines/develop/actions.ts
index 98b3f2243614a..34c09d331f7f0 100644
--- a/packages/gatsby/src/state-machines/develop/actions.ts
+++ b/packages/gatsby/src/state-machines/develop/actions.ts
@@ -120,18 +120,6 @@ export const spawnWebpackListener = assign<IBuildContext, AnyEventObject>({
   },
 })
 
-export const sendDevHtmlResponses = pure(() => {
-  console.log(`hi?? sendDevHtmlResponses`)
-  const sendAction = send("SEND_DEVELOP_HTML_RESPONSES", {
-    to: context => {
-      console.log(Object.keys(context))
-      return context.developHTMLRoute
-    },
-  })
-  console.log(sendAction)
-  return sendAction
-})
-
 export const assignWebhookBody = assign<IBuildContext, AnyEventObject>({
   webhookBody: (_context, { payload }) => payload?.webhookBody,
 })
diff --git a/packages/gatsby/src/utils/develop-html-route.ts b/packages/gatsby/src/utils/develop-html-route.ts
index 6c273e8504b83..7431d24d112d9 100644
--- a/packages/gatsby/src/utils/develop-html-route.ts
+++ b/packages/gatsby/src/utils/develop-html-route.ts
@@ -119,9 +119,9 @@ const parseError = function (err, directory): IParsedError {
   return data
 }
 
-export const route = ({ app, program, store }) => {
+export const route = ({ app, program, store }) =>
   // Render an HTML page and serve it.
-  return app.get(`*`, async (req, res, next) => {
+  app.get(`*`, async (req, res, next) => {
     const { pages } = store.getState()
 
     if (!pages.has(req.path)) {
@@ -165,4 +165,3 @@ export const route = ({ app, program, store }) => {
     // Make eslint happy
     return null
   })
-}

From 29a291e8da4d407bad580a6e5bb7edad3a18794c Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Fri, 16 Oct 2020 10:47:18 -0600
Subject: [PATCH 027/123] yet moer cleanups

---
 packages/gatsby/src/state-machines/develop/actions.ts | 2 --
 1 file changed, 2 deletions(-)

diff --git a/packages/gatsby/src/state-machines/develop/actions.ts b/packages/gatsby/src/state-machines/develop/actions.ts
index 34c09d331f7f0..3186e7382b17f 100644
--- a/packages/gatsby/src/state-machines/develop/actions.ts
+++ b/packages/gatsby/src/state-machines/develop/actions.ts
@@ -5,7 +5,6 @@ import {
   spawn,
   ActionFunctionMap,
   DoneEventObject,
-  send,
 } from "xstate"
 import { IBuildContext } from "../../services"
 import { boundActionCreators } from "../../redux/actions"
@@ -172,7 +171,6 @@ export const buildActions: ActionFunctionMap<IBuildContext, AnyEventObject> = {
   clearWebhookBody,
   finishParentSpan,
   spawnWebpackListener,
-  sendDevHtmlResponses,
   markSourceFilesDirty,
   markSourceFilesClean,
   markNodesClean,

From 8ab60c9403cc21704e89ca072923f421770cf97c Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Fri, 16 Oct 2020 11:06:45 -0600
Subject: [PATCH 028/123] Add test for error parsing & codeframe creation

---
 .../__snapshots__/develop-html-route.ts.snap  | 169 ++++++++++++++++++
 .../src/utils/__tests__/develop-html-route.ts |   8 +
 .../src/utils/__tests__/fixtures/blog-post.js | 121 +++++++++++++
 .../utils/__tests__/fixtures/error-object.js  |  33 ++++
 .../gatsby/src/utils/develop-html-route.ts    |   5 +-
 5 files changed, 335 insertions(+), 1 deletion(-)
 create mode 100644 packages/gatsby/src/utils/__tests__/__snapshots__/develop-html-route.ts.snap
 create mode 100644 packages/gatsby/src/utils/__tests__/develop-html-route.ts
 create mode 100644 packages/gatsby/src/utils/__tests__/fixtures/blog-post.js
 create mode 100644 packages/gatsby/src/utils/__tests__/fixtures/error-object.js

diff --git a/packages/gatsby/src/utils/__tests__/__snapshots__/develop-html-route.ts.snap b/packages/gatsby/src/utils/__tests__/__snapshots__/develop-html-route.ts.snap
new file mode 100644
index 0000000000000..84675989a68f5
--- /dev/null
+++ b/packages/gatsby/src/utils/__tests__/__snapshots__/develop-html-route.ts.snap
@@ -0,0 +1,169 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`error parsing returns an object w/ the parsed error & codeframe 1`] = `
+Object {
+  "code": "import React from \\"react\\"
+import Helmet from \\"react-helmet\\"
+import { Link, graphql } from \\"gatsby\\"
+import typography from \\"../utils/typography\\"
+import ReadNext from \\"../components/ReadNext\\"
+import Layout from \\"../layouts/index.js\\"
+import profilePic from \\"../images/kyle-round-small-pantheon.jpg\\"
+
+const { rhythm, scale } = typography
+
+class BlogPostRoute extends React.Component {
+  render() {
+    const post = this.props.data.markdownRemark
+    //console.log(post)
+
+    let pizza = window.width2
+    let tags
+    let tagsSection
+    if (this.props.data.markdownRemark.fields.tagSlugs) {
+      const tagsArray = this.props.data.markdownRemark.fields.tagSlugs
+      tags = tagsArray.map((tag, i) => {
+        const divider = i < tagsArray.length - 1 && <span>{\\" | \\"}</span>
+        return (
+          <span key={tag}>
+            <Link to={tag}>
+              {this.props.data.markdownRemark.frontmatter.tags[i]}
+            </Link>
+            {divider}
+          </span>
+        )
+      })
+      tagsSection = (
+        <em
+          style={{
+            ...scale(-1 / 5),
+            display: \\"block\\",
+            marginBottom: rhythm(1),
+          }}
+        >
+          Tagged with {tags}
+        </em>
+      )
+    }
+
+    return (
+      <Layout location={this.props.location}>
+        <Helmet
+          title={\`\${post.frontmatter.title}\`}
+          meta={[{ name: \\"description\\", content: post.excerpt }]}
+        />
+        <h1>{post.frontmatter.title}</h1>
+        <div dangerouslySetInnerHTML={{ __html: post.html }} />
+        {tagsSection}
+        <p
+          style={{
+            ...scale(-1 / 5),
+            display: \\"block\\",
+            marginBottom: rhythm(1),
+          }}
+        >
+          Posted {post.frontmatter.date}
+        </p>
+        <hr
+          style={{
+            marginBottom: rhythm(1),
+          }}
+        />
+        <ReadNext nextPost={post.frontmatter.readNext} />
+        <p
+          style={{
+            marginBottom: 0,
+          }}
+        >
+          <img
+            src={profilePic}
+            alt=\\"Kyle's profile pic\\"
+            style={{
+              borderRadius: \`100%\`,
+              float: \\"left\\",
+              marginRight: rhythm(1 / 4),
+              marginBottom: 0,
+              width: rhythm(2),
+              height: rhythm(2),
+            }}
+          />
+          <strong>{this.props.data.site.siteMetadata.author}</strong> lives and
+          works in {this.props.data.site.siteMetadata.homeCity} building useful
+          things.{\\" \\"}
+          <a href=\\"https://twitter.com/kylemathews\\">
+            You should follow him on Twitter
+          </a>
+        </p>
+      </Layout>
+    )
+  }
+}
+
+export default BlogPostRoute
+
+export const pageQuery = graphql\`
+  query BlogPostBySlug($slug: String!) {
+    site {
+      siteMetadata {
+        author
+        homeCity
+      }
+    }
+    markdownRemark(fields: { slug: { eq: $slug } }) {
+      html
+      excerpt
+      fields {
+        tagSlugs
+      }
+      frontmatter {
+        title
+        tags
+        date(formatString: \\"MMMM DD, YYYY\\")
+      }
+    }
+  }
+\`
+",
+  "codeFrame": "<span style=\\"font-weight:normal;opacity:1;color:#452475;background:#fdfaf6;\\"> <span style=\\"color:#527713;\\"> 14 | </span>    <span style=\\"color:#527713;\\">//console.log(post)</span></span>
+<span style=\\"font-weight:normal;opacity:1;color:#452475;background:#fdfaf6;\\"> <span style=\\"color:#527713;\\"> 15 | </span></span>
+<span style=\\"font-weight:normal;opacity:1;color:#452475;background:#fdfaf6;\\"><span style=\\"color:#096fb3;\\"><span style=\\"font-weight:bold;\\">></span></span><span style=\\"color:#527713;\\"> 16 | </span>    let pizza <span style=\\"color:#DB3A00;\\">=</span> window<span style=\\"color:#DB3A00;\\">.</span>width2</span>
+<span style=\\"font-weight:normal;opacity:1;color:#452475;background:#fdfaf6;\\"> <span style=\\"color:#527713;\\">    | </span>                <span style=\\"color:#096fb3;\\"><span style=\\"font-weight:bold;\\">^</span></span></span>
+<span style=\\"font-weight:normal;opacity:1;color:#452475;background:#fdfaf6;\\"> <span style=\\"color:#527713;\\"> 17 | </span>    let tags</span>
+<span style=\\"font-weight:normal;opacity:1;color:#452475;background:#fdfaf6;\\"> <span style=\\"color:#527713;\\"> 18 | </span>    let tagsSection</span>
+<span style=\\"font-weight:normal;opacity:1;color:#452475;background:#fdfaf6;\\"> <span style=\\"color:#527713;\\"> 19 | </span>    <span style=\\"color:#006500;\\">if</span> (<span style=\\"color:#006500;\\">this</span><span style=\\"color:#DB3A00;\\">.</span>props<span style=\\"color:#DB3A00;\\">.</span>data<span style=\\"color:#DB3A00;\\">.</span>markdownRemark<span style=\\"color:#DB3A00;\\">.</span>fields<span style=\\"color:#DB3A00;\\">.</span>tagSlugs) {</span>",
+  "filename": "<PROJECT_ROOT>/packages/gatsby/src/utils/__tests__/fixtures/blog-post.js",
+  "line": 16,
+  "message": "window is not defined",
+  "row": 17,
+  "stack": "ReferenceError: window is not defined
+    at BlogPostRoute.render (<HOME_DIR>/programs/blog/public/webpack:/lib/fixtures/blog-post.js:16:17)
+    at processChild (<HOME_DIR>/programs/blog/node_modules/react-dom/cjs/react-dom-server.node.development.js:3134:18)
+    at resolve (<HOME_DIR>/programs/blog/node_modules/react-dom/cjs/react-dom-server.node.development.js:2960:5)
+    at ReactDOMServerRenderer.render (<HOME_DIR>/programs/blog/node_modules/react-dom/cjs/react-dom-server.node.development.js:3435:22)
+    at ReactDOMServerRenderer.read (<HOME_DIR>/programs/blog/node_modules/react-dom/cjs/react-dom-server.node.development.js:3373:29)
+    at renderToString (<HOME_DIR>/programs/blog/node_modules/react-dom/cjs/react-dom-server.node.development.js:3988:27)
+    at Module.default (<HOME_DIR>/programs/blog/public/webpack:/lib/.cache/develop-static-entry.js:248:32)
+    at <HOME_DIR>/programs/blog/node_modules/gatsby/src/utils/worker/render-html.ts:32:11
+    at <HOME_DIR>/programs/blog/node_modules/gatsby/src/utils/worker/render-html.ts:25:7
+From previous event:
+    at renderHTML (<HOME_DIR>/programs/blog/node_modules/gatsby/src/utils/worker/render-html.ts:22:18)
+    at <HOME_DIR>/programs/blog/node_modules/gatsby/src/utils/develop-html-route.ts:140:36
+    at Layer.handle [as handle_request] (<HOME_DIR>/programs/blog/node_modules/express/lib/router/layer.js:95:5)
+    at next (<HOME_DIR>/programs/blog/node_modules/express/lib/router/route.js:137:13)
+    at Route.dispatch (<HOME_DIR>/programs/blog/node_modules/express/lib/router/route.js:112:3)
+    at Layer.handle [as handle_request] (<HOME_DIR>/programs/blog/node_modules/express/lib/router/layer.js:95:5)
+    at <HOME_DIR>/programs/blog/node_modules/express/lib/router/index.js:281:22
+    at param (<HOME_DIR>/programs/blog/node_modules/express/lib/router/index.js:354:14)
+    at param (<HOME_DIR>/programs/blog/node_modules/express/lib/router/index.js:365:14)
+    at Function.process_params (<HOME_DIR>/programs/blog/node_modules/express/lib/router/index.js:410:3)
+    at next (<HOME_DIR>/programs/blog/node_modules/express/lib/router/index.js:275:10)
+    at cors (<HOME_DIR>/programs/blog/node_modules/cors/lib/index.js:188:7)
+    at <HOME_DIR>/programs/blog/node_modules/cors/lib/index.js:224:17
+    at originCallback (<HOME_DIR>/programs/blog/node_modules/cors/lib/index.js:214:15)
+    at <HOME_DIR>/programs/blog/node_modules/cors/lib/index.js:219:13
+    at optionsCallback (<HOME_DIR>/programs/blog/node_modules/cors/lib/index.js:199:9)
+    at corsMiddleware (<HOME_DIR>/programs/blog/node_modules/cors/lib/index.js:204:7)
+    at Layer.handle [as handle_request] (<HOME_DIR>/programs/blog/node_modules/express/lib/router/layer.js:95:5)",
+  "type": "ReferenceError",
+}
+`;
diff --git a/packages/gatsby/src/utils/__tests__/develop-html-route.ts b/packages/gatsby/src/utils/__tests__/develop-html-route.ts
new file mode 100644
index 0000000000000..e3bea3a02fec2
--- /dev/null
+++ b/packages/gatsby/src/utils/__tests__/develop-html-route.ts
@@ -0,0 +1,8 @@
+import { parseError } from "../develop-html-route"
+import error from "./fixtures/error-object"
+
+describe(`error parsing`, () => {
+  it(`returns an object w/ the parsed error & codeframe`, () => {
+    expect(parseError(error, __dirname)).toMatchSnapshot()
+  })
+})
diff --git a/packages/gatsby/src/utils/__tests__/fixtures/blog-post.js b/packages/gatsby/src/utils/__tests__/fixtures/blog-post.js
new file mode 100644
index 0000000000000..511387d201093
--- /dev/null
+++ b/packages/gatsby/src/utils/__tests__/fixtures/blog-post.js
@@ -0,0 +1,121 @@
+import React from "react"
+import Helmet from "react-helmet"
+import { Link, graphql } from "gatsby"
+import typography from "../utils/typography"
+import ReadNext from "../components/ReadNext"
+import Layout from "../layouts/index.js"
+import profilePic from "../images/kyle-round-small-pantheon.jpg"
+
+const { rhythm, scale } = typography
+
+class BlogPostRoute extends React.Component {
+  render() {
+    const post = this.props.data.markdownRemark
+    //console.log(post)
+
+    let pizza = window.width2
+    let tags
+    let tagsSection
+    if (this.props.data.markdownRemark.fields.tagSlugs) {
+      const tagsArray = this.props.data.markdownRemark.fields.tagSlugs
+      tags = tagsArray.map((tag, i) => {
+        const divider = i < tagsArray.length - 1 && <span>{" | "}</span>
+        return (
+          <span key={tag}>
+            <Link to={tag}>
+              {this.props.data.markdownRemark.frontmatter.tags[i]}
+            </Link>
+            {divider}
+          </span>
+        )
+      })
+      tagsSection = (
+        <em
+          style={{
+            ...scale(-1 / 5),
+            display: "block",
+            marginBottom: rhythm(1),
+          }}
+        >
+          Tagged with {tags}
+        </em>
+      )
+    }
+
+    return (
+      <Layout location={this.props.location}>
+        <Helmet
+          title={`${post.frontmatter.title}`}
+          meta={[{ name: "description", content: post.excerpt }]}
+        />
+        <h1>{post.frontmatter.title}</h1>
+        <div dangerouslySetInnerHTML={{ __html: post.html }} />
+        {tagsSection}
+        <p
+          style={{
+            ...scale(-1 / 5),
+            display: "block",
+            marginBottom: rhythm(1),
+          }}
+        >
+          Posted {post.frontmatter.date}
+        </p>
+        <hr
+          style={{
+            marginBottom: rhythm(1),
+          }}
+        />
+        <ReadNext nextPost={post.frontmatter.readNext} />
+        <p
+          style={{
+            marginBottom: 0,
+          }}
+        >
+          <img
+            src={profilePic}
+            alt="Kyle's profile pic"
+            style={{
+              borderRadius: `100%`,
+              float: "left",
+              marginRight: rhythm(1 / 4),
+              marginBottom: 0,
+              width: rhythm(2),
+              height: rhythm(2),
+            }}
+          />
+          <strong>{this.props.data.site.siteMetadata.author}</strong> lives and
+          works in {this.props.data.site.siteMetadata.homeCity} building useful
+          things.{" "}
+          <a href="https://twitter.com/kylemathews">
+            You should follow him on Twitter
+          </a>
+        </p>
+      </Layout>
+    )
+  }
+}
+
+export default BlogPostRoute
+
+export const pageQuery = graphql`
+  query BlogPostBySlug($slug: String!) {
+    site {
+      siteMetadata {
+        author
+        homeCity
+      }
+    }
+    markdownRemark(fields: { slug: { eq: $slug } }) {
+      html
+      excerpt
+      fields {
+        tagSlugs
+      }
+      frontmatter {
+        title
+        tags
+        date(formatString: "MMMM DD, YYYY")
+      }
+    }
+  }
+`
diff --git a/packages/gatsby/src/utils/__tests__/fixtures/error-object.js b/packages/gatsby/src/utils/__tests__/fixtures/error-object.js
new file mode 100644
index 0000000000000..3e0ee527a72ed
--- /dev/null
+++ b/packages/gatsby/src/utils/__tests__/fixtures/error-object.js
@@ -0,0 +1,33 @@
+module.exports = {
+  stack: `ReferenceError: window is not defined
+    at BlogPostRoute.render (/Users/kylemathews/programs/blog/public/webpack:/lib/fixtures/blog-post.js:16:17)
+    at processChild (/Users/kylemathews/programs/blog/node_modules/react-dom/cjs/react-dom-server.node.development.js:3134:18)
+    at resolve (/Users/kylemathews/programs/blog/node_modules/react-dom/cjs/react-dom-server.node.development.js:2960:5)
+    at ReactDOMServerRenderer.render (/Users/kylemathews/programs/blog/node_modules/react-dom/cjs/react-dom-server.node.development.js:3435:22)
+    at ReactDOMServerRenderer.read (/Users/kylemathews/programs/blog/node_modules/react-dom/cjs/react-dom-server.node.development.js:3373:29)
+    at renderToString (/Users/kylemathews/programs/blog/node_modules/react-dom/cjs/react-dom-server.node.development.js:3988:27)
+    at Module.default (/Users/kylemathews/programs/blog/public/webpack:/lib/.cache/develop-static-entry.js:248:32)
+    at /Users/kylemathews/programs/blog/node_modules/gatsby/src/utils/worker/render-html.ts:32:11
+    at /Users/kylemathews/programs/blog/node_modules/gatsby/src/utils/worker/render-html.ts:25:7
+From previous event:
+    at renderHTML (/Users/kylemathews/programs/blog/node_modules/gatsby/src/utils/worker/render-html.ts:22:18)
+    at /Users/kylemathews/programs/blog/node_modules/gatsby/src/utils/develop-html-route.ts:140:36
+    at Layer.handle [as handle_request] (/Users/kylemathews/programs/blog/node_modules/express/lib/router/layer.js:95:5)
+    at next (/Users/kylemathews/programs/blog/node_modules/express/lib/router/route.js:137:13)
+    at Route.dispatch (/Users/kylemathews/programs/blog/node_modules/express/lib/router/route.js:112:3)
+    at Layer.handle [as handle_request] (/Users/kylemathews/programs/blog/node_modules/express/lib/router/layer.js:95:5)
+    at /Users/kylemathews/programs/blog/node_modules/express/lib/router/index.js:281:22
+    at param (/Users/kylemathews/programs/blog/node_modules/express/lib/router/index.js:354:14)
+    at param (/Users/kylemathews/programs/blog/node_modules/express/lib/router/index.js:365:14)
+    at Function.process_params (/Users/kylemathews/programs/blog/node_modules/express/lib/router/index.js:410:3)
+    at next (/Users/kylemathews/programs/blog/node_modules/express/lib/router/index.js:275:10)
+    at cors (/Users/kylemathews/programs/blog/node_modules/cors/lib/index.js:188:7)
+    at /Users/kylemathews/programs/blog/node_modules/cors/lib/index.js:224:17
+    at originCallback (/Users/kylemathews/programs/blog/node_modules/cors/lib/index.js:214:15)
+    at /Users/kylemathews/programs/blog/node_modules/cors/lib/index.js:219:13
+    at optionsCallback (/Users/kylemathews/programs/blog/node_modules/cors/lib/index.js:199:9)
+    at corsMiddleware (/Users/kylemathews/programs/blog/node_modules/cors/lib/index.js:204:7)
+    at Layer.handle [as handle_request] (/Users/kylemathews/programs/blog/node_modules/express/lib/router/layer.js:95:5)`,
+  message: `window is not defined`,
+  type: `ReferenceError`,
+}
diff --git a/packages/gatsby/src/utils/develop-html-route.ts b/packages/gatsby/src/utils/develop-html-route.ts
index 7431d24d112d9..ba66069a829ca 100644
--- a/packages/gatsby/src/utils/develop-html-route.ts
+++ b/packages/gatsby/src/utils/develop-html-route.ts
@@ -70,15 +70,18 @@ interface IParsedError {
   stack: [string]
 }
 
-const parseError = function (err, directory): IParsedError {
+// Code borrowed and modified from https://github.com/watilde/parse-error
+export const parseError = function (err, directory): IParsedError {
   const stack = err.stack ? err.stack : ``
   const stackObject = stack.split(`\n`)
   const position = getPosition(stackObject)
+
   // Remove the `/lib/` added by webpack
   const filename = path.join(
     directory,
     ...position.filename.split(path.sep).slice(2)
   )
+
   const code = fs.readFileSync(filename, `utf-8`)
   const line = position.line
   const row = position.row

From 68d27102bbde0d07f1fdee1b079293183d5791c2 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Fri, 16 Oct 2020 11:07:55 -0600
Subject: [PATCH 029/123] add return type

---
 packages/gatsby/src/utils/develop-html-route.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/gatsby/src/utils/develop-html-route.ts b/packages/gatsby/src/utils/develop-html-route.ts
index ba66069a829ca..9c647dec78fee 100644
--- a/packages/gatsby/src/utils/develop-html-route.ts
+++ b/packages/gatsby/src/utils/develop-html-route.ts
@@ -122,7 +122,7 @@ export const parseError = function (err, directory): IParsedError {
   return data
 }
 
-export const route = ({ app, program, store }) =>
+export const route = ({ app, program, store }): any =>
   // Render an HTML page and serve it.
   app.get(`*`, async (req, res, next) => {
     const { pages } = store.getState()

From 2e05cb5ac70bd0a9c32137aacff89cd93df61b67 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Fri, 16 Oct 2020 11:38:36 -0600
Subject: [PATCH 030/123] Feature flag functionality behind env variable

---
 .../gatsby/cache-dir/develop-static-entry.js  | 34 ++++---
 .../gatsby/src/utils/develop-html-route.ts    |  1 +
 packages/gatsby/src/utils/start-server.ts     | 92 ++++++++++++++-----
 3 files changed, 88 insertions(+), 39 deletions(-)

diff --git a/packages/gatsby/cache-dir/develop-static-entry.js b/packages/gatsby/cache-dir/develop-static-entry.js
index ba90ebd040a41..af2cf0464add8 100644
--- a/packages/gatsby/cache-dir/develop-static-entry.js
+++ b/packages/gatsby/cache-dir/develop-static-entry.js
@@ -228,24 +228,28 @@ export default (pagePath, callback) => {
     }
   ).pop()
 
-  // Let the site or plugin render the page component.
-  apiRunner(`replaceRenderer`, {
-    bodyComponent,
-    replaceBodyHTMLString,
-    setHeadComponents,
-    setHtmlAttributes,
-    setBodyAttributes,
-    setPreBodyComponents,
-    setPostBodyComponents,
-    setBodyProps,
-    pathname: pagePath,
-    pathPrefix: __PATH_PREFIX__,
-  })
+  if (process.env.GATSBY_EXPERIMENTAL_DEV_SSR) {
+    // Let the site or plugin render the page component.
+    apiRunner(`replaceRenderer`, {
+      bodyComponent,
+      replaceBodyHTMLString,
+      setHeadComponents,
+      setHtmlAttributes,
+      setBodyAttributes,
+      setPreBodyComponents,
+      setPostBodyComponents,
+      setBodyProps,
+      pathname: pagePath,
+      pathPrefix: __PATH_PREFIX__,
+    })
+  }
 
   // If no one stepped up, we'll handle it.
   if (!bodyHtml) {
     try {
-      bodyHtml = renderToString(bodyComponent)
+      if (process.env.GATSBY_EXPERIMENTAL_DEV_SSR) {
+        bodyHtml = renderToString(bodyComponent)
+      }
     } catch (e) {
       // ignore @reach/router redirect errors
       if (!isRedirect(e)) throw e
@@ -274,7 +278,7 @@ export default (pagePath, callback) => {
 
   const htmlElement = React.createElement(Html, {
     ...bodyProps,
-    body: bodyHtml,
+    body: process.env.GATSBY_EXPERIMENTAL_DEV_SSR ? bodyHtml : ``,
     headComponents: headComponents.concat([
       <script key={`io`} src="/socket.io/socket.io.js" />,
     ]),
diff --git a/packages/gatsby/src/utils/develop-html-route.ts b/packages/gatsby/src/utils/develop-html-route.ts
index 9c647dec78fee..3d1e76c1fd1fe 100644
--- a/packages/gatsby/src/utils/develop-html-route.ts
+++ b/packages/gatsby/src/utils/develop-html-route.ts
@@ -72,6 +72,7 @@ interface IParsedError {
 
 // Code borrowed and modified from https://github.com/watilde/parse-error
 export const parseError = function (err, directory): IParsedError {
+  console.log(err)
   const stack = err.stack ? err.stack : ``
   const stackObject = stack.split(`\n`)
   const position = getPosition(stackObject)
diff --git a/packages/gatsby/src/utils/start-server.ts b/packages/gatsby/src/utils/start-server.ts
index 5b88e7f610dd8..d9f2abda60d51 100644
--- a/packages/gatsby/src/utils/start-server.ts
+++ b/packages/gatsby/src/utils/start-server.ts
@@ -1,33 +1,34 @@
+import chokidar from "chokidar"
+
 import webpackHotMiddleware from "webpack-hot-middleware"
 import webpackDevMiddleware, {
   WebpackDevMiddleware,
 } from "webpack-dev-middleware"
 import got from "got"
-import chokidar from "chokidar"
 import webpack from "webpack"
 import express from "express"
 import graphqlHTTP from "express-graphql"
 import graphqlPlayground from "graphql-playground-middleware-express"
 import graphiqlExplorer from "gatsby-graphiql-explorer"
 import { formatError } from "graphql"
-import { slash } from "gatsby-core-utils"
-import telemetry from "gatsby-telemetry"
 import http from "http"
 import https from "https"
+import cors from "cors"
+import telemetry from "gatsby-telemetry"
 
 import webpackConfig from "../utils/webpack.config"
 import { store, emitter } from "../redux"
-import { buildRenderer } from "../commands/build-html"
+import { buildRenderer, buildHTML } from "../commands/build-html"
 import { withBasePath } from "../utils/path"
 import report from "gatsby-cli/lib/reporter"
 import launchEditor from "react-dev-utils/launchEditor"
-import cors from "cors"
 import * as WorkerPool from "../utils/worker/pool"
 import { route as developHtmlRoute } from "./develop-html-route"
 
 import { developStatic } from "../commands/develop-static"
 import withResolverContext from "../schema/context"
 import { websocketManager, WebsocketManager } from "../utils/websocket-manager"
+import { slash } from "gatsby-core-utils"
 import apiRunnerNode from "../utils/api-runner-node"
 import { Express } from "express"
 
@@ -64,8 +65,40 @@ export async function startServer(
   app: Express,
   workerPool: JestWorker = WorkerPool.create()
 ): Promise<IServer> {
+  const indexHTMLActivity = report.phantomActivity(`building index.html`, {})
+  indexHTMLActivity.start()
   const directory = program.directory
   const directoryPath = withBasePath(directory)
+  const createIndexHtml = async (activity: ActivityTracker): Promise<void> => {
+    try {
+      await buildHTML({
+        program,
+        stage: Stage.DevelopHTML,
+        pagePaths: [`/`],
+        workerPool,
+        activity,
+      })
+    } catch (err) {
+      if (err.name !== `WebpackError`) {
+        report.panic(err)
+        return
+      }
+      report.panic(
+        report.stripIndent`
+          There was an error compiling the html.js component for the development server.
+
+          See our docs page on debugging HTML builds for help https://gatsby.dev/debug-html
+        `,
+        err
+      )
+    }
+  }
+
+  await createIndexHtml(indexHTMLActivity)
+
+  indexHTMLActivity.end()
+
+  // report.stateUpdate(`webpack`, `IN_PROGRESS`)
 
   const webpackActivity = report.activityTimer(`Building development bundle`, {
     id: `webpack-develop`,
@@ -178,14 +211,17 @@ export async function startServer(
     res.end()
   })
 
-  // Setup HTML route.
-  developHtmlRoute({ app, program, directory, store })
-
   app.get(`/__open-stack-frame-in-editor`, (req, res) => {
     launchEditor(req.query.fileName, req.query.lineNumber)
     res.end()
   })
 
+  // Disable directory indexing i.e. serving index.html from a directory.
+  // This can lead to serving stale html files during development.
+  //
+  // We serve by default an empty index.html that sets up the dev environment.
+  app.use(developStatic(`public`, { index: false }))
+
   const webpackDevMiddlewareInstance = (webpackDevMiddleware(compiler, {
     logLevel: `silent`,
     publicPath: devConfig.output.publicPath,
@@ -247,11 +283,19 @@ export async function startServer(
     res.status(404).end()
   })
 
-  // Disable directory indexing i.e. serving index.html from a directory.
-  // This can lead to serving stale html files during development.
-  //
-  // We serve by default an empty index.html that sets up the dev environment.
-  app.use(developStatic(`public`, { index: false }))
+  // Render an HTML page and serve it.
+  if (process.env.GATSBY_EXPERIMENTAL_DEV_SSR) {
+    // Setup HTML route.
+    developHtmlRoute({ app, program, directory, store })
+  } else {
+    app.use((_, res) => {
+      res.sendFile(directoryPath(`public/index.html`), err => {
+        if (err) {
+          res.status(500).end()
+        }
+      })
+    })
+  }
 
   /**
    * Set up the HTTP server and socket.io.
@@ -264,18 +308,18 @@ export async function startServer(
   // in http proxy in `develop-proxy`
   const listener = server.listen(program.port, `localhost`)
 
-  // Register watcher that rebuilds index.html every time html.js changes.
-  const watchGlobs = [`src/html.js`, `plugins/**/gatsby-ssr.js`].map(path =>
-    slash(directoryPath(path))
-  )
+  if (!process.env.GATSBY_EXPERIMENTAL_DEV_SSR) {
+    // Register watcher that rebuilds index.html every time html.js changes.
+    const watchGlobs = [`src/html.js`, `plugins/**/gatsby-ssr.js`].map(path =>
+      slash(directoryPath(path))
+    )
 
-  chokidar.watch(watchGlobs).on(`change`, async () => {
-    console.log(`Time to build a renderer`)
-    await buildRenderer(program, Stage.DevelopHTML, webpackActivity)
-    console.log(`We built a renderer`)
-    // eslint-disable-next-line no-unused-expressions
-    socket?.to(`clients`).emit(`reload`)
-  })
+    chokidar.watch(watchGlobs).on(`change`, async () => {
+      await createIndexHtml(indexHTMLActivity)
+      // eslint-disable-next-line no-unused-expressions
+      socket?.to(`clients`).emit(`reload`)
+    })
+  }
 
   return {
     compiler,

From 616d8561500cadf7840ceaec439d9c21a898682c Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Fri, 16 Oct 2020 11:44:09 -0600
Subject: [PATCH 031/123] Lighten how much the dev ssr html webpack instance is
 watching

---
 packages/gatsby/src/commands/build-html.ts      | 17 +++++++++++------
 packages/gatsby/src/utils/develop-html-route.ts |  1 -
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/packages/gatsby/src/commands/build-html.ts b/packages/gatsby/src/commands/build-html.ts
index 79dd777cadb54..b825edc78ea8d 100644
--- a/packages/gatsby/src/commands/build-html.ts
+++ b/packages/gatsby/src/commands/build-html.ts
@@ -25,13 +25,18 @@ const runWebpack = (compilerConfig, stage: Stage): Bluebird<webpack.Stats> =>
         }
       })
     } else if (stage === `develop-html`) {
-      webpack(compilerConfig).watch({}, (err, stats) => {
-        if (err) {
-          reject(err)
-        } else {
-          resolve(stats)
+      webpack(compilerConfig).watch(
+        {
+          ignored: /node_modules/,
+        },
+        (err, stats) => {
+          if (err) {
+            reject(err)
+          } else {
+            resolve(stats)
+          }
         }
-      })
+      )
     }
   })
 
diff --git a/packages/gatsby/src/utils/develop-html-route.ts b/packages/gatsby/src/utils/develop-html-route.ts
index 3d1e76c1fd1fe..9c647dec78fee 100644
--- a/packages/gatsby/src/utils/develop-html-route.ts
+++ b/packages/gatsby/src/utils/develop-html-route.ts
@@ -72,7 +72,6 @@ interface IParsedError {
 
 // Code borrowed and modified from https://github.com/watilde/parse-error
 export const parseError = function (err, directory): IParsedError {
-  console.log(err)
   const stack = err.stack ? err.stack : ``
   const stackObject = stack.split(`\n`)
   const position = getPosition(stackObject)

From 3442a006e920c703006089e41c625eec6323b009 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Fri, 16 Oct 2020 11:52:49 -0600
Subject: [PATCH 032/123] cleanup

---
 .../gatsby/cache-dir/develop-static-entry.js  | 62 -------------------
 1 file changed, 62 deletions(-)

diff --git a/packages/gatsby/cache-dir/develop-static-entry.js b/packages/gatsby/cache-dir/develop-static-entry.js
index af2cf0464add8..47e509777eef9 100644
--- a/packages/gatsby/cache-dir/develop-static-entry.js
+++ b/packages/gatsby/cache-dir/develop-static-entry.js
@@ -96,11 +96,6 @@ export default (pagePath, callback) => {
     return join(`page-data`, fixedPagePath, `page-data.json`)
   }
 
-  // const getPageDataUrl = pagePath => {
-  //   const pageDataPath = getPageDataPath(pagePath)
-  //   return `${__PATH_PREFIX__}/${pageDataPath}`
-  // }
-
   const getPageData = pagePath => {
     const pageDataPath = getPageDataPath(pagePath)
     const absolutePageDataPath = join(process.cwd(), `public`, pageDataPath)
@@ -113,66 +108,13 @@ export default (pagePath, callback) => {
     }
   }
 
-  // const getStaticQueryPath = hash => `page-data/sq/d/${hash}.json`
-
-  // const getStaticQueryResults = staticQueryHashes =>
-  //   staticQueryHashes.map(hash => {
-  //     const absoluteStaticQueryDataPath = join(
-  //       process.cwd(),
-  //       `public`,
-  //       getStaticQueryPath(hash)
-  //     )
-  //     try {
-  //       return JSON.parse(
-  //         fs.readFileSync(absoluteStaticQueryDataPath).toString()
-  //       )
-  //     } catch (error) {
-  //       console.log(error)
-  //     }
-  //   })
-
-  // const appDataPath = join(`page-data`, `app-data.json`)
-
-  // const getAppDataUrl = memoize(() => {
-  //   let appData
-
-  //   try {
-  //     const absoluteAppDataPath = join(process.cwd(), `public`, appDataPath)
-  //     const appDataRaw = fs.readFileSync(absoluteAppDataPath)
-  //     appData = JSON.parse(appDataRaw.toString())
-
-  //     if (!appData) {
-  //       return null
-  //     }
-  //   } catch (err) {
-  //     return null
-  //   }
-
-  //   return `${__PATH_PREFIX__}/${appDataPath}`
-  // })
-
   const pageData = getPageData(pagePath)
-  // const pageDataUrl = getPageDataUrl(pagePath)
-
-  // const appDataUrl = getAppDataUrl()
 
   const {
     componentChunkName,
     //  staticQueryHashes = []
   } = pageData
 
-  // const staticQueryData = getStaticQueryResults(staticQueryHashes)
-
-  // console.log({
-  //   staticQueryData,
-  //   pageData,
-  // })
-
-  // const pageDataResult = {
-  //   ...pageData,
-  //   staticQueryResults: staticQueryData,
-  // }
-
   const createElement = React.createElement
 
   class RouteHandler extends React.Component {
@@ -193,10 +135,6 @@ export default (pagePath, callback) => {
         props
       )
 
-      // console.log({
-      //   element: syncRequires.components[componentChunkName].toString(),
-      // })
-
       const wrappedPage = apiRunner(
         `wrapPageElement`,
         { element: pageElement, props },

From 424baaa9a06bcfb1358f4312803ba54c95c072ef Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Fri, 16 Oct 2020 11:55:51 -0600
Subject: [PATCH 033/123] track usage while we're in experimental stage

---
 packages/gatsby/src/utils/develop-html-route.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/packages/gatsby/src/utils/develop-html-route.ts b/packages/gatsby/src/utils/develop-html-route.ts
index 9c647dec78fee..2564c35001090 100644
--- a/packages/gatsby/src/utils/develop-html-route.ts
+++ b/packages/gatsby/src/utils/develop-html-route.ts
@@ -3,6 +3,7 @@ import path from "path"
 import fs from "fs-extra"
 import { codeFrameColumns } from "@babel/code-frame"
 import ansiHTML from "ansi-html"
+const { trackCli } = require(`gatsby-telemetry`)
 
 import { renderHTML } from "./worker/render-html"
 import { Stage } from "../commands/types"
@@ -125,6 +126,7 @@ export const parseError = function (err, directory): IParsedError {
 export const route = ({ app, program, store }): any =>
   // Render an HTML page and serve it.
   app.get(`*`, async (req, res, next) => {
+    trackCli(`GATSBY_EXPERIMENTAL_DEV_SSR`)
     const { pages } = store.getState()
 
     if (!pages.has(req.path)) {

From d4e4b2e802a1f87cff3de007c8f8cd7ff277e368 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Fri, 16 Oct 2020 12:00:09 -0600
Subject: [PATCH 034/123] update snapshots

---
 .../cache-dir/__tests__/__snapshots__/static-entry.js.snap  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/gatsby/cache-dir/__tests__/__snapshots__/static-entry.js.snap b/packages/gatsby/cache-dir/__tests__/__snapshots__/static-entry.js.snap
index 2a5a48623247a..a32218f3e0838 100644
--- a/packages/gatsby/cache-dir/__tests__/__snapshots__/static-entry.js.snap
+++ b/packages/gatsby/cache-dir/__tests__/__snapshots__/static-entry.js.snap
@@ -1,10 +1,10 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
-exports[`develop-static-entry onPreRenderHTML can be used to replace headComponents 1`] = `"<!DOCTYPE html><html><head><meta charSet=\\"utf-8\\"/><meta http-equiv=\\"x-ua-compatible\\" content=\\"ie=edge\\"/><meta name=\\"viewport\\" content=\\"width=device-width, initial-scale=1, shrink-to-fit=no\\"/><style> .style3  </style><style> .style2  </style><style> .style1  </style><meta name=\\"note\\" content=\\"environment=development\\"/><script src=\\"/socket.io/socket.io.js\\"></script></head><body><div id=\\"___gatsby\\"><div style=\\"outline:none\\" tabindex=\\"-1\\" id=\\"gatsby-focus-wrapper\\"></div><div id=\\"gatsby-announcer\\" style=\\"position:absolute;top:0;width:1px;height:1px;padding:0;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;border:0\\" aria-live=\\"assertive\\" aria-atomic=\\"true\\"></div></div><script src=\\"/polyfill.js\\" nomodule=\\"\\"></script><script src=\\"/commons.js\\"></script></body></html>"`;
+exports[`develop-static-entry onPreRenderHTML can be used to replace headComponents 1`] = `"<!DOCTYPE html><html><head><meta charSet=\\"utf-8\\"/><meta http-equiv=\\"x-ua-compatible\\" content=\\"ie=edge\\"/><meta name=\\"viewport\\" content=\\"width=device-width, initial-scale=1, shrink-to-fit=no\\"/><style> .style3  </style><style> .style2  </style><style> .style1  </style><meta name=\\"note\\" content=\\"environment=development\\"/><script src=\\"/socket.io/socket.io.js\\"></script></head><body><div id=\\"___gatsby\\"></div><script src=\\"/polyfill.js\\" nomodule=\\"\\"></script><script src=\\"/commons.js\\"></script></body></html>"`;
 
-exports[`develop-static-entry onPreRenderHTML can be used to replace postBodyComponents 1`] = `"<!DOCTYPE html><html><head><meta charSet=\\"utf-8\\"/><meta http-equiv=\\"x-ua-compatible\\" content=\\"ie=edge\\"/><meta name=\\"viewport\\" content=\\"width=device-width, initial-scale=1, shrink-to-fit=no\\"/><meta name=\\"note\\" content=\\"environment=development\\"/><script src=\\"/socket.io/socket.io.js\\"></script></head><body><div id=\\"___gatsby\\"><div style=\\"outline:none\\" tabindex=\\"-1\\" id=\\"gatsby-focus-wrapper\\"></div><div id=\\"gatsby-announcer\\" style=\\"position:absolute;top:0;width:1px;height:1px;padding:0;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;border:0\\" aria-live=\\"assertive\\" aria-atomic=\\"true\\"></div></div><div> div3 </div><div> div2 </div><div> div1 </div><script src=\\"/polyfill.js\\" nomodule=\\"\\"></script><script src=\\"/commons.js\\"></script></body></html>"`;
+exports[`develop-static-entry onPreRenderHTML can be used to replace postBodyComponents 1`] = `"<!DOCTYPE html><html><head><meta charSet=\\"utf-8\\"/><meta http-equiv=\\"x-ua-compatible\\" content=\\"ie=edge\\"/><meta name=\\"viewport\\" content=\\"width=device-width, initial-scale=1, shrink-to-fit=no\\"/><meta name=\\"note\\" content=\\"environment=development\\"/><script src=\\"/socket.io/socket.io.js\\"></script></head><body><div id=\\"___gatsby\\"></div><div> div3 </div><div> div2 </div><div> div1 </div><script src=\\"/polyfill.js\\" nomodule=\\"\\"></script><script src=\\"/commons.js\\"></script></body></html>"`;
 
-exports[`develop-static-entry onPreRenderHTML can be used to replace preBodyComponents 1`] = `"<!DOCTYPE html><html><head><meta charSet=\\"utf-8\\"/><meta http-equiv=\\"x-ua-compatible\\" content=\\"ie=edge\\"/><meta name=\\"viewport\\" content=\\"width=device-width, initial-scale=1, shrink-to-fit=no\\"/><meta name=\\"note\\" content=\\"environment=development\\"/><script src=\\"/socket.io/socket.io.js\\"></script></head><body><div> div3 </div><div> div2 </div><div> div1 </div><div id=\\"___gatsby\\"><div style=\\"outline:none\\" tabindex=\\"-1\\" id=\\"gatsby-focus-wrapper\\"></div><div id=\\"gatsby-announcer\\" style=\\"position:absolute;top:0;width:1px;height:1px;padding:0;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;border:0\\" aria-live=\\"assertive\\" aria-atomic=\\"true\\"></div></div><script src=\\"/polyfill.js\\" nomodule=\\"\\"></script><script src=\\"/commons.js\\"></script></body></html>"`;
+exports[`develop-static-entry onPreRenderHTML can be used to replace preBodyComponents 1`] = `"<!DOCTYPE html><html><head><meta charSet=\\"utf-8\\"/><meta http-equiv=\\"x-ua-compatible\\" content=\\"ie=edge\\"/><meta name=\\"viewport\\" content=\\"width=device-width, initial-scale=1, shrink-to-fit=no\\"/><meta name=\\"note\\" content=\\"environment=development\\"/><script src=\\"/socket.io/socket.io.js\\"></script></head><body><div> div3 </div><div> div2 </div><div> div1 </div><div id=\\"___gatsby\\"></div><script src=\\"/polyfill.js\\" nomodule=\\"\\"></script><script src=\\"/commons.js\\"></script></body></html>"`;
 
 exports[`static-entry onPreRenderHTML can be used to replace headComponents 1`] = `"<!DOCTYPE html><html><head><meta charSet=\\"utf-8\\"/><meta http-equiv=\\"x-ua-compatible\\" content=\\"ie=edge\\"/><meta name=\\"viewport\\" content=\\"width=device-width, initial-scale=1, shrink-to-fit=no\\"/><link as=\\"fetch\\" rel=\\"preload\\" href=\\"/page-data/app-data.json\\" crossorigin=\\"anonymous\\"/><link as=\\"fetch\\" rel=\\"preload\\" href=\\"/page-data/about/page-data.json\\" crossorigin=\\"anonymous\\"/><style> .style3  </style><style> .style2  </style><style> .style1  </style><meta name=\\"generator\\" content=\\"Gatsby 2.0.0\\"/></head><body><div id=\\"___gatsby\\"><div style=\\"outline:none\\" tabindex=\\"-1\\" id=\\"gatsby-focus-wrapper\\"></div><div id=\\"gatsby-announcer\\" style=\\"position:absolute;top:0;width:1px;height:1px;padding:0;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;border:0\\" aria-live=\\"assertive\\" aria-atomic=\\"true\\"></div></div><script id=\\"gatsby-script-loader\\">/*<![CDATA[*/window.pagePath=\\"/about/\\";/*]]>*/</script><script id=\\"gatsby-chunk-mapping\\">/*<![CDATA[*/window.___chunkMapping={};/*]]>*/</script></body></html>"`;
 

From cdea9a04a7962de5820fc36ef3543ba6a89f5c03 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Fri, 16 Oct 2020 12:07:45 -0600
Subject: [PATCH 035/123] cleanup

---
 packages/gatsby/src/utils/start-server.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/gatsby/src/utils/start-server.ts b/packages/gatsby/src/utils/start-server.ts
index d9f2abda60d51..6d8521adc34b0 100644
--- a/packages/gatsby/src/utils/start-server.ts
+++ b/packages/gatsby/src/utils/start-server.ts
@@ -286,7 +286,7 @@ export async function startServer(
   // Render an HTML page and serve it.
   if (process.env.GATSBY_EXPERIMENTAL_DEV_SSR) {
     // Setup HTML route.
-    developHtmlRoute({ app, program, directory, store })
+    developHtmlRoute({ app, program, store })
   } else {
     app.use((_, res) => {
       res.sendFile(directoryPath(`public/index.html`), err => {

From a10f0709758ab3cf73189af47c2838528753e7ad Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Fri, 16 Oct 2020 12:20:28 -0600
Subject: [PATCH 036/123] Restore support for dev 404 page

---
 .../gatsby/cache-dir/develop-static-entry.js  |  7 +++++-
 packages/gatsby/src/utils/start-server.ts     | 25 ++++++++++++-------
 2 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/packages/gatsby/cache-dir/develop-static-entry.js b/packages/gatsby/cache-dir/develop-static-entry.js
index 47e509777eef9..6dc2b893573a5 100644
--- a/packages/gatsby/cache-dir/develop-static-entry.js
+++ b/packages/gatsby/cache-dir/develop-static-entry.js
@@ -214,9 +214,14 @@ export default (pagePath, callback) => {
     pathname: pagePath,
   })
 
+  let bodyStr = ``
+  if (process.env.GATSBY_EXPERIMENTAL_DEV_SSR && pagePath !== `/dev-404/`) {
+    bodyStr = bodyHtml
+  }
+
   const htmlElement = React.createElement(Html, {
     ...bodyProps,
-    body: process.env.GATSBY_EXPERIMENTAL_DEV_SSR ? bodyHtml : ``,
+    body: bodyStr,
     headComponents: headComponents.concat([
       <script key={`io`} src="/socket.io/socket.io.js" />,
     ]),
diff --git a/packages/gatsby/src/utils/start-server.ts b/packages/gatsby/src/utils/start-server.ts
index 6d8521adc34b0..b73590750bfec 100644
--- a/packages/gatsby/src/utils/start-server.ts
+++ b/packages/gatsby/src/utils/start-server.ts
@@ -70,11 +70,14 @@ export async function startServer(
   const directory = program.directory
   const directoryPath = withBasePath(directory)
   const createIndexHtml = async (activity: ActivityTracker): Promise<void> => {
+    const genericHtmlPath = process.env.GATSBY_EXPERIMENTAL_DEV_SSR
+      ? `/dev-404-page/`
+      : `/`
     try {
       await buildHTML({
         program,
         stage: Stage.DevelopHTML,
-        pagePaths: [`/`],
+        pagePaths: [genericHtmlPath],
         workerPool,
         activity,
       })
@@ -287,16 +290,20 @@ export async function startServer(
   if (process.env.GATSBY_EXPERIMENTAL_DEV_SSR) {
     // Setup HTML route.
     developHtmlRoute({ app, program, store })
-  } else {
-    app.use((_, res) => {
-      res.sendFile(directoryPath(`public/index.html`), err => {
-        if (err) {
-          res.status(500).end()
-        }
-      })
-    })
   }
 
+  // We still need this even w/ ssr for the dev 404 page.
+  const genericHtmlPath = process.env.GATSBY_EXPERIMENTAL_DEV_SSR
+    ? `/public/dev-404-page/index.html`
+    : `/public/index.html`
+  app.use((_, res) => {
+    res.sendFile(directoryPath(genericHtmlPath), err => {
+      if (err) {
+        res.status(500).end()
+      }
+    })
+  })
+
   /**
    * Set up the HTTP server and socket.io.
    **/

From 21cab9db40b204bbee843e63ae4f5b5a4ac75f41 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Fri, 16 Oct 2020 13:59:34 -0600
Subject: [PATCH 037/123] fix test

---
 .../utils/__tests__/fixtures/error-object.js  | 54 +++++++++----------
 1 file changed, 27 insertions(+), 27 deletions(-)

diff --git a/packages/gatsby/src/utils/__tests__/fixtures/error-object.js b/packages/gatsby/src/utils/__tests__/fixtures/error-object.js
index 3e0ee527a72ed..2bd7b3325ca29 100644
--- a/packages/gatsby/src/utils/__tests__/fixtures/error-object.js
+++ b/packages/gatsby/src/utils/__tests__/fixtures/error-object.js
@@ -1,33 +1,33 @@
 module.exports = {
   stack: `ReferenceError: window is not defined
-    at BlogPostRoute.render (/Users/kylemathews/programs/blog/public/webpack:/lib/fixtures/blog-post.js:16:17)
-    at processChild (/Users/kylemathews/programs/blog/node_modules/react-dom/cjs/react-dom-server.node.development.js:3134:18)
-    at resolve (/Users/kylemathews/programs/blog/node_modules/react-dom/cjs/react-dom-server.node.development.js:2960:5)
-    at ReactDOMServerRenderer.render (/Users/kylemathews/programs/blog/node_modules/react-dom/cjs/react-dom-server.node.development.js:3435:22)
-    at ReactDOMServerRenderer.read (/Users/kylemathews/programs/blog/node_modules/react-dom/cjs/react-dom-server.node.development.js:3373:29)
-    at renderToString (/Users/kylemathews/programs/blog/node_modules/react-dom/cjs/react-dom-server.node.development.js:3988:27)
-    at Module.default (/Users/kylemathews/programs/blog/public/webpack:/lib/.cache/develop-static-entry.js:248:32)
-    at /Users/kylemathews/programs/blog/node_modules/gatsby/src/utils/worker/render-html.ts:32:11
-    at /Users/kylemathews/programs/blog/node_modules/gatsby/src/utils/worker/render-html.ts:25:7
+    at BlogPostRoute.render (<HOME_DIR>/programs/blog/public/webpack:/lib/fixtures/blog-post.js:16:17)
+    at processChild (<HOME_DIR>/programs/blog/node_modules/react-dom/cjs/react-dom-server.node.development.js:3134:18)
+    at resolve (<HOME_DIR>/programs/blog/node_modules/react-dom/cjs/react-dom-server.node.development.js:2960:5)
+    at ReactDOMServerRenderer.render (<HOME_DIR>/programs/blog/node_modules/react-dom/cjs/react-dom-server.node.development.js:3435:22)
+    at ReactDOMServerRenderer.read (<HOME_DIR>/programs/blog/node_modules/react-dom/cjs/react-dom-server.node.development.js:3373:29)
+    at renderToString (<HOME_DIR>/programs/blog/node_modules/react-dom/cjs/react-dom-server.node.development.js:3988:27)
+    at Module.default (<HOME_DIR>/programs/blog/public/webpack:/lib/.cache/develop-static-entry.js:248:32)
+    at <HOME_DIR>/programs/blog/node_modules/gatsby/src/utils/worker/render-html.ts:32:11
+    at <HOME_DIR>/programs/blog/node_modules/gatsby/src/utils/worker/render-html.ts:25:7
 From previous event:
-    at renderHTML (/Users/kylemathews/programs/blog/node_modules/gatsby/src/utils/worker/render-html.ts:22:18)
-    at /Users/kylemathews/programs/blog/node_modules/gatsby/src/utils/develop-html-route.ts:140:36
-    at Layer.handle [as handle_request] (/Users/kylemathews/programs/blog/node_modules/express/lib/router/layer.js:95:5)
-    at next (/Users/kylemathews/programs/blog/node_modules/express/lib/router/route.js:137:13)
-    at Route.dispatch (/Users/kylemathews/programs/blog/node_modules/express/lib/router/route.js:112:3)
-    at Layer.handle [as handle_request] (/Users/kylemathews/programs/blog/node_modules/express/lib/router/layer.js:95:5)
-    at /Users/kylemathews/programs/blog/node_modules/express/lib/router/index.js:281:22
-    at param (/Users/kylemathews/programs/blog/node_modules/express/lib/router/index.js:354:14)
-    at param (/Users/kylemathews/programs/blog/node_modules/express/lib/router/index.js:365:14)
-    at Function.process_params (/Users/kylemathews/programs/blog/node_modules/express/lib/router/index.js:410:3)
-    at next (/Users/kylemathews/programs/blog/node_modules/express/lib/router/index.js:275:10)
-    at cors (/Users/kylemathews/programs/blog/node_modules/cors/lib/index.js:188:7)
-    at /Users/kylemathews/programs/blog/node_modules/cors/lib/index.js:224:17
-    at originCallback (/Users/kylemathews/programs/blog/node_modules/cors/lib/index.js:214:15)
-    at /Users/kylemathews/programs/blog/node_modules/cors/lib/index.js:219:13
-    at optionsCallback (/Users/kylemathews/programs/blog/node_modules/cors/lib/index.js:199:9)
-    at corsMiddleware (/Users/kylemathews/programs/blog/node_modules/cors/lib/index.js:204:7)
-    at Layer.handle [as handle_request] (/Users/kylemathews/programs/blog/node_modules/express/lib/router/layer.js:95:5)`,
+    at renderHTML (<HOME_DIR>/programs/blog/node_modules/gatsby/src/utils/worker/render-html.ts:22:18)
+    at <HOME_DIR>/programs/blog/node_modules/gatsby/src/utils/develop-html-route.ts:140:36
+    at Layer.handle [as handle_request] (<HOME_DIR>/programs/blog/node_modules/express/lib/router/layer.js:95:5)
+    at next (<HOME_DIR>/programs/blog/node_modules/express/lib/router/route.js:137:13)
+    at Route.dispatch (<HOME_DIR>/programs/blog/node_modules/express/lib/router/route.js:112:3)
+    at Layer.handle [as handle_request] (<HOME_DIR>/programs/blog/node_modules/express/lib/router/layer.js:95:5)
+    at <HOME_DIR>/programs/blog/node_modules/express/lib/router/index.js:281:22
+    at param (<HOME_DIR>/programs/blog/node_modules/express/lib/router/index.js:354:14)
+    at param (<HOME_DIR>/programs/blog/node_modules/express/lib/router/index.js:365:14)
+    at Function.process_params (<HOME_DIR>/programs/blog/node_modules/express/lib/router/index.js:410:3)
+    at next (<HOME_DIR>/programs/blog/node_modules/express/lib/router/index.js:275:10)
+    at cors (<HOME_DIR>/programs/blog/node_modules/cors/lib/index.js:188:7)
+    at <HOME_DIR>/programs/blog/node_modules/cors/lib/index.js:224:17
+    at originCallback (<HOME_DIR>/programs/blog/node_modules/cors/lib/index.js:214:15)
+    at <HOME_DIR>/programs/blog/node_modules/cors/lib/index.js:219:13
+    at optionsCallback (<HOME_DIR>/programs/blog/node_modules/cors/lib/index.js:199:9)
+    at corsMiddleware (<HOME_DIR>/programs/blog/node_modules/cors/lib/index.js:204:7)
+    at Layer.handle [as handle_request] (<HOME_DIR>/programs/blog/node_modules/express/lib/router/layer.js:95:5)`,
   message: `window is not defined`,
   type: `ReferenceError`,
 }

From c50327f8d3fd5198e8cde3b42ff434f734061038 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Mon, 19 Oct 2020 09:53:15 -0600
Subject: [PATCH 038/123] Catch reading errors

---
 packages/gatsby/src/utils/develop-html-route.ts | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/packages/gatsby/src/utils/develop-html-route.ts b/packages/gatsby/src/utils/develop-html-route.ts
index 2564c35001090..fff8348b9eba4 100644
--- a/packages/gatsby/src/utils/develop-html-route.ts
+++ b/packages/gatsby/src/utils/develop-html-route.ts
@@ -83,7 +83,12 @@ export const parseError = function (err, directory): IParsedError {
     ...position.filename.split(path.sep).slice(2)
   )
 
-  const code = fs.readFileSync(filename, `utf-8`)
+  let code
+  try {
+    code = fs.readFileSync(filename, `utf-8`)
+  } catch (e) {
+    report.error(`Couldn't read the file ${filename}`, e, err)
+  }
   const line = position.line
   const row = position.row
   ansiHTML.setColors({

From 6be8fe630dd3d61c6f8d7507cef6499d333d9241 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Mon, 19 Oct 2020 11:00:21 -0600
Subject: [PATCH 039/123] Make static queries work in dev-ssr

---
 packages/babel-plugin-remove-graphql-queries/src/index.ts | 2 --
 1 file changed, 2 deletions(-)

diff --git a/packages/babel-plugin-remove-graphql-queries/src/index.ts b/packages/babel-plugin-remove-graphql-queries/src/index.ts
index 58ac53a28aaaf..49c5ac6e3087b 100644
--- a/packages/babel-plugin-remove-graphql-queries/src/index.ts
+++ b/packages/babel-plugin-remove-graphql-queries/src/index.ts
@@ -269,7 +269,6 @@ export default function ({ types: t }): PluginObj {
           JSXIdentifier(path2: NodePath<JSXIdentifier>): void {
             if (
               (process.env.NODE_ENV === `test` ||
-                state.opts.stage === `develop` ||
                 state.opts.stage === `develop-html` ||
                 state.opts.stage === `build-html`) &&
               path2.isJSXIdentifier({ name: `StaticQuery` }) &&
@@ -313,7 +312,6 @@ export default function ({ types: t }): PluginObj {
           CallExpression(path2: NodePath<CallExpression>): void {
             if (
               (process.env.NODE_ENV === `test` ||
-                state.opts.stage === `develop` ||
                 state.opts.stage === `develop-html` ||
                 state.opts.stage === `build-html`) &&
               isUseStaticQuery(path2)

From f156049977e98744c3393ab96d8c91d83af76866 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Mon, 19 Oct 2020 11:02:37 -0600
Subject: [PATCH 040/123] Keep the renderer around

---
 packages/gatsby/src/commands/build-html.ts | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/gatsby/src/commands/build-html.ts b/packages/gatsby/src/commands/build-html.ts
index b825edc78ea8d..f299137607078 100644
--- a/packages/gatsby/src/commands/build-html.ts
+++ b/packages/gatsby/src/commands/build-html.ts
@@ -160,5 +160,7 @@ export const buildHTML = async ({
 }): Promise<void> => {
   const rendererPath = await buildRenderer(program, stage, activity.span)
   await doBuildPages(rendererPath, pagePaths, activity, workerPool)
-  await deleteRenderer(rendererPath)
+  if (stage !== `develop-html`) {
+    await deleteRenderer(rendererPath)
+  }
 }

From b1eaa4abe1a5eff04befcf83a9a7263745f8e8fb Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Mon, 19 Oct 2020 11:04:17 -0600
Subject: [PATCH 041/123] Send 'loading' page if webpack is busy

---
 packages/gatsby/src/utils/develop-html-route.ts | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/packages/gatsby/src/utils/develop-html-route.ts b/packages/gatsby/src/utils/develop-html-route.ts
index fff8348b9eba4..9f38ee5481b85 100644
--- a/packages/gatsby/src/utils/develop-html-route.ts
+++ b/packages/gatsby/src/utils/develop-html-route.ts
@@ -7,6 +7,7 @@ const { trackCli } = require(`gatsby-telemetry`)
 
 import { renderHTML } from "./worker/render-html"
 import { Stage } from "../commands/types"
+import { isWebpackStatusPending } from "./webpack-status"
 
 interface IErrorPosition {
   filename: string
@@ -131,6 +132,14 @@ export const parseError = function (err, directory): IParsedError {
 export const route = ({ app, program, store }): any =>
   // Render an HTML page and serve it.
   app.get(`*`, async (req, res, next) => {
+    if (isWebpackStatusPending()) {
+      res
+        .status(202)
+        .send(
+          `webpack isn't yet finished compiling code. Try refreshing once it's done.`
+        )
+    }
+
     trackCli(`GATSBY_EXPERIMENTAL_DEV_SSR`)
     const { pages } = store.getState()
 

From e11cc0a23ce29372c8a97bc48cafbe5ce71813e4 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Mon, 19 Oct 2020 11:05:06 -0600
Subject: [PATCH 042/123] Show more error so can debug CI

---
 packages/gatsby/src/utils/develop-html-route.ts | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/gatsby/src/utils/develop-html-route.ts b/packages/gatsby/src/utils/develop-html-route.ts
index 9f38ee5481b85..672e87e84620c 100644
--- a/packages/gatsby/src/utils/develop-html-route.ts
+++ b/packages/gatsby/src/utils/develop-html-route.ts
@@ -88,7 +88,8 @@ export const parseError = function (err, directory): IParsedError {
   try {
     code = fs.readFileSync(filename, `utf-8`)
   } catch (e) {
-    report.error(`Couldn't read the file ${filename}`, e, err)
+    console.log(`original error`, err)
+    report.error(`Couldn't read the file ${filename}`, e)
   }
   const line = position.line
   const row = position.row

From 00037e14acfd6e0630375ab99d177e471b11f874 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Mon, 19 Oct 2020 11:05:34 -0600
Subject: [PATCH 043/123] lazily build dev 404 page so it happens after
 webpack/queries are run

---
 packages/gatsby/src/utils/start-server.ts | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/packages/gatsby/src/utils/start-server.ts b/packages/gatsby/src/utils/start-server.ts
index b73590750bfec..457491f16636d 100644
--- a/packages/gatsby/src/utils/start-server.ts
+++ b/packages/gatsby/src/utils/start-server.ts
@@ -97,8 +97,6 @@ export async function startServer(
     }
   }
 
-  await createIndexHtml(indexHTMLActivity)
-
   indexHTMLActivity.end()
 
   // report.stateUpdate(`webpack`, `IN_PROGRESS`)
@@ -292,11 +290,15 @@ export async function startServer(
     developHtmlRoute({ app, program, store })
   }
 
+  const genericHtmlPageCreated = false
   // We still need this even w/ ssr for the dev 404 page.
   const genericHtmlPath = process.env.GATSBY_EXPERIMENTAL_DEV_SSR
     ? `/public/dev-404-page/index.html`
     : `/public/index.html`
-  app.use((_, res) => {
+  app.use(async (_, res) => {
+    if (!genericHtmlPageCreated) {
+      await createIndexHtml(indexHTMLActivity)
+    }
     res.sendFile(directoryPath(genericHtmlPath), err => {
       if (err) {
         res.status(500).end()

From 5b8089acf4692f9f7148c935b2c585304fff8f89 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Mon, 19 Oct 2020 11:28:45 -0600
Subject: [PATCH 044/123] just use /

---
 packages/gatsby/src/utils/develop-html-route.ts | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/gatsby/src/utils/develop-html-route.ts b/packages/gatsby/src/utils/develop-html-route.ts
index 672e87e84620c..965a447c70817 100644
--- a/packages/gatsby/src/utils/develop-html-route.ts
+++ b/packages/gatsby/src/utils/develop-html-route.ts
@@ -81,7 +81,9 @@ export const parseError = function (err, directory): IParsedError {
   // Remove the `/lib/` added by webpack
   const filename = path.join(
     directory,
-    ...position.filename.split(path.sep).slice(2)
+    // Don't need to use path.sep as webpack always uses a single forward slash
+    // as a path seperator.
+    ...position.filename.split(`/`).slice(2)
   )
 
   let code

From 803172037658a6eedb6e37d9b8bd1013b3aedc59 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Mon, 19 Oct 2020 12:28:10 -0600
Subject: [PATCH 045/123] Add ansi-html as depdnency

---
 packages/gatsby/package.json | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/gatsby/package.json b/packages/gatsby/package.json
index 5b4847f7c2bc0..1d84f6ea1720b 100644
--- a/packages/gatsby/package.json
+++ b/packages/gatsby/package.json
@@ -26,6 +26,7 @@
     "@typescript-eslint/eslint-plugin": "^2.24.0",
     "@typescript-eslint/parser": "^2.24.0",
     "address": "1.1.2",
+    "ansi-html": "^0.0.7",
     "autoprefixer": "^9.8.4",
     "axios": "^0.20.0",
     "babel-core": "7.0.0-bridge.0",

From f44f2852f2df6c8bc9f4e5999e8ffc3499f42454 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Mon, 19 Oct 2020 12:35:31 -0600
Subject: [PATCH 046/123] Run test w/ experimental flag

---
 integration-tests/ssr/package.json | 33 +++++++++++++++---------------
 1 file changed, 17 insertions(+), 16 deletions(-)

diff --git a/integration-tests/ssr/package.json b/integration-tests/ssr/package.json
index df2df97d94216..2dc9bd54e909c 100644
--- a/integration-tests/ssr/package.json
+++ b/integration-tests/ssr/package.json
@@ -1,38 +1,39 @@
 {
   "name": "ssr",
-  "private": true,
-  "author": "Sid Chatterjee",
   "description": "A simplified bare-bones starter for Gatsby",
   "version": "0.1.0",
-  "license": "MIT",
-  "scripts": {
-    "build": "gatsby build",
-    "develop": "gatsby develop",
-    "serve": "gatsby serve",
-    "clean": "gatsby clean",
-    "test-output": "node test-output.js",
-    "test:jest": "jest --config=jest.config.js --runInBand",
-    "start-dev-server": "start-server-and-test develop http://localhost:8000 test:jest",
-    "test": "npm-run-all -s build start-dev-server"
+  "author": "Sid Chatterjee",
+  "bugs": {
+    "url": "https://github.com/gatsbyjs/gatsby/issues"
   },
   "dependencies": {
-    "gatsby": "^2.24.49",
+    "gatsby": "2.24.82-dev-1603131999086",
     "react": "^16.12.0",
     "react-dom": "^16.12.0"
   },
   "devDependencies": {
+    "cross-env": "^7.0.2",
     "fs-extra": "^9.0.0",
     "jest": "^24.0.0",
-    "jest-diff": "^24.0.0",
     "jest-cli": "^24.0.0",
+    "jest-diff": "^24.0.0",
     "npm-run-all": "4.1.5",
     "start-server-and-test": "^1.11.3"
   },
+  "license": "MIT",
+  "private": true,
   "repository": {
     "type": "git",
     "url": "https://github.com/gatsbyjs/gatsby-starter-hello-world"
   },
-  "bugs": {
-    "url": "https://github.com/gatsbyjs/gatsby/issues"
+  "scripts": {
+    "build": "gatsby build",
+    "clean": "gatsby clean",
+    "develop": "gatsby develop",
+    "serve": "gatsby serve",
+    "start-dev-server": "start-server-and-test develop http://localhost:8000 test:jest",
+    "test": "cross-env GATSBY_EXPERIMENTAL_DEV_SSR=true npm-run-all -s build start-dev-server",
+    "test-output": "node test-output.js",
+    "test:jest": "jest --config=jest.config.js --runInBand"
   }
 }

From cabc58a524121360775279ef51341d8dac08a619 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Mon, 19 Oct 2020 16:06:16 -0600
Subject: [PATCH 047/123] meaningless change to run tests again

---
 integration-tests/ssr/package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/integration-tests/ssr/package.json b/integration-tests/ssr/package.json
index 2dc9bd54e909c..c67b44f5c1c4c 100644
--- a/integration-tests/ssr/package.json
+++ b/integration-tests/ssr/package.json
@@ -1,6 +1,6 @@
 {
   "name": "ssr",
-  "description": "A simplified bare-bones starter for Gatsby",
+  "description": "A simplified bare-bones starter for Gatsby.",
   "version": "0.1.0",
   "author": "Sid Chatterjee",
   "bugs": {

From 37bdaa024fbe2e860bc7096e240f3d0d4596990d Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Mon, 19 Oct 2020 20:31:36 -0600
Subject: [PATCH 048/123] use older version compatable with CI

---
 integration-tests/ssr/package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/integration-tests/ssr/package.json b/integration-tests/ssr/package.json
index c67b44f5c1c4c..f6517f4147da6 100644
--- a/integration-tests/ssr/package.json
+++ b/integration-tests/ssr/package.json
@@ -12,7 +12,7 @@
     "react-dom": "^16.12.0"
   },
   "devDependencies": {
-    "cross-env": "^7.0.2",
+    "cross-env": "^5.0.2",
     "fs-extra": "^9.0.0",
     "jest": "^24.0.0",
     "jest-cli": "^24.0.0",

From feab2f906cbc33b2f1ba0eafbe205b03178ce698 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Tue, 20 Oct 2020 09:12:58 -0600
Subject: [PATCH 049/123] Update
 packages/gatsby/cache-dir/develop-static-entry.js

Co-authored-by: Peter van der Zee <209817+pvdz@users.noreply.github.com>
---
 packages/gatsby/cache-dir/develop-static-entry.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/gatsby/cache-dir/develop-static-entry.js b/packages/gatsby/cache-dir/develop-static-entry.js
index 6dc2b893573a5..beade6ff95758 100644
--- a/packages/gatsby/cache-dir/develop-static-entry.js
+++ b/packages/gatsby/cache-dir/develop-static-entry.js
@@ -99,10 +99,10 @@ export default (pagePath, callback) => {
   const getPageData = pagePath => {
     const pageDataPath = getPageDataPath(pagePath)
     const absolutePageDataPath = join(process.cwd(), `public`, pageDataPath)
-    const pageDataRaw = fs.readFileSync(absolutePageDataPath)
+    const pageDataJson = fs.readFileSync(absolutePageDataPath, 'utf8')
 
     try {
-      return JSON.parse(pageDataRaw.toString())
+      return JSON.parse(pageDataJson)
     } catch (err) {
       return null
     }

From 8608b3854abab386ee04351ca74b0d216603220c Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Tue, 20 Oct 2020 09:21:04 -0600
Subject: [PATCH 050/123] remove unnecessary changes

---
 packages/gatsby/src/commands/develop-process.ts       | 2 --
 packages/gatsby/src/commands/types.ts                 | 1 -
 packages/gatsby/src/state-machines/develop/actions.ts | 1 -
 3 files changed, 4 deletions(-)

diff --git a/packages/gatsby/src/commands/develop-process.ts b/packages/gatsby/src/commands/develop-process.ts
index 78595cbf46080..bc94860d97bc8 100644
--- a/packages/gatsby/src/commands/develop-process.ts
+++ b/packages/gatsby/src/commands/develop-process.ts
@@ -137,6 +137,4 @@ module.exports = async (program: IDevelopArgs): Promise<void> => {
   }
 
   service.start()
-
-  program.developMachineService = service
 }
diff --git a/packages/gatsby/src/commands/types.ts b/packages/gatsby/src/commands/types.ts
index 2e9299e2670c7..1cc9c692aef7c 100644
--- a/packages/gatsby/src/commands/types.ts
+++ b/packages/gatsby/src/commands/types.ts
@@ -26,7 +26,6 @@ export interface IProgram {
   [`key-file`]?: string
   directory: string
   https?: boolean
-  developMachineService?: any
   sitePackageJson: PackageJson
   ssl?: ICert
   inspect?: number
diff --git a/packages/gatsby/src/state-machines/develop/actions.ts b/packages/gatsby/src/state-machines/develop/actions.ts
index 3186e7382b17f..3fcf59f1052b8 100644
--- a/packages/gatsby/src/state-machines/develop/actions.ts
+++ b/packages/gatsby/src/state-machines/develop/actions.ts
@@ -15,7 +15,6 @@ import reporter from "gatsby-cli/lib/reporter"
 import { ProgramStatus } from "../../redux/types"
 import { createWebpackWatcher } from "../../services/listen-to-webpack"
 import { callRealApi } from "../../utils/call-deferred-api"
-
 /**
  * Handler for when we're inside handlers that should be able to mutate nodes
  * Instead of queueing, we call it right away

From 9f16346efe232ca5aeb106860c94f7f4e8419e7a Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Tue, 20 Oct 2020 09:30:35 -0600
Subject: [PATCH 051/123] Consistently use imports

---
 packages/gatsby/cache-dir/develop-static-entry.js | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/packages/gatsby/cache-dir/develop-static-entry.js b/packages/gatsby/cache-dir/develop-static-entry.js
index beade6ff95758..d1daa4bc155eb 100644
--- a/packages/gatsby/cache-dir/develop-static-entry.js
+++ b/packages/gatsby/cache-dir/develop-static-entry.js
@@ -1,14 +1,14 @@
 import React from "react"
-const fs = require(`fs`)
+import fs from "fs"
 import { renderToString, renderToStaticMarkup } from "react-dom/server"
-const { merge } = require(`lodash`)
-const { join } = require(`path`)
+import { merge } from "lodash"
+import { join } from "path"
 import apiRunner from "./api-runner-ssr"
-const { grabMatchParams } = require(`./find-path`)
+import { grabMatchParams } from "./find-path"
 const syncRequires = require(`$virtual/sync-requires`)
 
-const { RouteAnnouncerProps } = require(`./route-announcer-props`)
-const { ServerLocation, Router, isRedirect } = require(`@reach/router`)
+import { RouteAnnouncerProps } from "./route-announcer-props"
+import { ServerLocation, Router, isRedirect } from "@reach/router"
 // import testRequireError from "./test-require-error"
 // For some extremely mysterious reason, webpack adds the above module *after*
 // this module so that when this code runs, testRequireError is undefined.
@@ -99,7 +99,7 @@ export default (pagePath, callback) => {
   const getPageData = pagePath => {
     const pageDataPath = getPageDataPath(pagePath)
     const absolutePageDataPath = join(process.cwd(), `public`, pageDataPath)
-    const pageDataJson = fs.readFileSync(absolutePageDataPath, 'utf8')
+    const pageDataJson = fs.readFileSync(absolutePageDataPath, `utf8`)
 
     try {
       return JSON.parse(pageDataJson)

From 2b3452bb10d5661661dfd33842ba00264d25ed15 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Tue, 20 Oct 2020 09:33:07 -0600
Subject: [PATCH 052/123] fix

---
 packages/gatsby/cache-dir/develop-static-entry.js | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/packages/gatsby/cache-dir/develop-static-entry.js b/packages/gatsby/cache-dir/develop-static-entry.js
index d1daa4bc155eb..03b2e3cd7697f 100644
--- a/packages/gatsby/cache-dir/develop-static-entry.js
+++ b/packages/gatsby/cache-dir/develop-static-entry.js
@@ -110,10 +110,7 @@ export default (pagePath, callback) => {
 
   const pageData = getPageData(pagePath)
 
-  const {
-    componentChunkName,
-    //  staticQueryHashes = []
-  } = pageData
+  const componentChunkName = pageData?.componentChunkName
 
   const createElement = React.createElement
 

From e8da16ce94d0e1f8792b97e68eb843cad7f77bf2 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Tue, 20 Oct 2020 09:46:37 -0600
Subject: [PATCH 053/123] Conditionally generate body str

---
 .../gatsby/cache-dir/develop-static-entry.js  | 273 +++++++++---------
 1 file changed, 139 insertions(+), 134 deletions(-)

diff --git a/packages/gatsby/cache-dir/develop-static-entry.js b/packages/gatsby/cache-dir/develop-static-entry.js
index 03b2e3cd7697f..c9c985ef19979 100644
--- a/packages/gatsby/cache-dir/develop-static-entry.js
+++ b/packages/gatsby/cache-dir/develop-static-entry.js
@@ -43,131 +43,157 @@ export default (pagePath, callback) => {
   let preBodyComponents = []
   let postBodyComponents = []
   let bodyProps = {}
-  let htmlStr
 
-  const setHeadComponents = components => {
-    headComponents = headComponents.concat(components)
-  }
-
-  const setHtmlAttributes = attributes => {
-    htmlAttributes = merge(htmlAttributes, attributes)
-  }
-
-  const setBodyAttributes = attributes => {
-    bodyAttributes = merge(bodyAttributes, attributes)
-  }
-
-  const setPreBodyComponents = components => {
-    preBodyComponents = preBodyComponents.concat(components)
-  }
+  const generateBodyHTML = () => {
+    const setHeadComponents = components => {
+      headComponents = headComponents.concat(components)
+    }
 
-  const setPostBodyComponents = components => {
-    postBodyComponents = postBodyComponents.concat(components)
-  }
+    const setHtmlAttributes = attributes => {
+      htmlAttributes = merge(htmlAttributes, attributes)
+    }
 
-  const setBodyProps = props => {
-    bodyProps = merge({}, bodyProps, props)
-  }
+    const setBodyAttributes = attributes => {
+      bodyAttributes = merge(bodyAttributes, attributes)
+    }
 
-  const getHeadComponents = () => headComponents
+    const setPreBodyComponents = components => {
+      preBodyComponents = preBodyComponents.concat(components)
+    }
 
-  const replaceHeadComponents = components => {
-    headComponents = components
-  }
+    const setPostBodyComponents = components => {
+      postBodyComponents = postBodyComponents.concat(components)
+    }
 
-  const replaceBodyHTMLString = body => {
-    bodyHtml = body
-  }
+    const setBodyProps = props => {
+      bodyProps = merge({}, bodyProps, props)
+    }
 
-  const getPreBodyComponents = () => preBodyComponents
+    const getHeadComponents = () => headComponents
 
-  const replacePreBodyComponents = components => {
-    preBodyComponents = components
-  }
+    const replaceHeadComponents = components => {
+      headComponents = components
+    }
 
-  const getPostBodyComponents = () => postBodyComponents
+    const replaceBodyHTMLString = body => {
+      bodyHtml = body
+    }
 
-  const replacePostBodyComponents = components => {
-    postBodyComponents = components
-  }
+    const getPreBodyComponents = () => preBodyComponents
 
-  const getPageDataPath = path => {
-    const fixedPagePath = path === `/` ? `index` : path
-    return join(`page-data`, fixedPagePath, `page-data.json`)
-  }
+    const replacePreBodyComponents = components => {
+      preBodyComponents = components
+    }
 
-  const getPageData = pagePath => {
-    const pageDataPath = getPageDataPath(pagePath)
-    const absolutePageDataPath = join(process.cwd(), `public`, pageDataPath)
-    const pageDataJson = fs.readFileSync(absolutePageDataPath, `utf8`)
+    const getPostBodyComponents = () => postBodyComponents
 
-    try {
-      return JSON.parse(pageDataJson)
-    } catch (err) {
-      return null
+    const replacePostBodyComponents = components => {
+      postBodyComponents = components
     }
-  }
-
-  const pageData = getPageData(pagePath)
 
-  const componentChunkName = pageData?.componentChunkName
+    const getPageDataPath = path => {
+      const fixedPagePath = path === `/` ? `index` : path
+      return join(`page-data`, fixedPagePath, `page-data.json`)
+    }
 
-  const createElement = React.createElement
+    const getPageData = pagePath => {
+      const pageDataPath = getPageDataPath(pagePath)
+      const absolutePageDataPath = join(process.cwd(), `public`, pageDataPath)
+      const pageDataJson = fs.readFileSync(absolutePageDataPath, `utf8`)
 
-  class RouteHandler extends React.Component {
-    render() {
-      const props = {
-        ...this.props,
-        ...pageData.result,
-        params: {
-          ...grabMatchParams(this.props.location.pathname),
-          ...(pageData.result?.pageContext?.__params || {}),
-        },
-        // pathContext was deprecated in v2. Renamed to pageContext
-        pathContext: pageData.result ? pageData.result.pageContext : undefined,
+      try {
+        return JSON.parse(pageDataJson)
+      } catch (err) {
+        return null
       }
+    }
 
-      const pageElement = createElement(
-        syncRequires.components[componentChunkName],
-        props
-      )
-
-      const wrappedPage = apiRunner(
-        `wrapPageElement`,
-        { element: pageElement, props },
-        pageElement,
-        ({ result }) => {
-          return { element: result, props }
+    const pageData = getPageData(pagePath)
+
+    const componentChunkName = pageData?.componentChunkName
+
+    const createElement = React.createElement
+
+    class RouteHandler extends React.Component {
+      render() {
+        const props = {
+          ...this.props,
+          ...pageData.result,
+          params: {
+            ...grabMatchParams(this.props.location.pathname),
+            ...(pageData.result?.pageContext?.__params || {}),
+          },
+          // pathContext was deprecated in v2. Renamed to pageContext
+          pathContext: pageData.result
+            ? pageData.result.pageContext
+            : undefined,
         }
-      ).pop()
 
-      return wrappedPage
+        const pageElement = createElement(
+          syncRequires.components[componentChunkName],
+          props
+        )
+
+        const wrappedPage = apiRunner(
+          `wrapPageElement`,
+          { element: pageElement, props },
+          pageElement,
+          ({ result }) => {
+            return { element: result, props }
+          }
+        ).pop()
+
+        return wrappedPage
+      }
     }
-  }
 
-  const routerElement = (
-    <ServerLocation url={`${__BASE_PATH__}${pagePath}`}>
-      <Router id="gatsby-focus-wrapper" baseuri={__BASE_PATH__}>
-        <RouteHandler path="/*" />
-      </Router>
-      <div {...RouteAnnouncerProps} />
-    </ServerLocation>
-  )
-
-  const bodyComponent = apiRunner(
-    `wrapRootElement`,
-    { element: routerElement, pathname: pagePath },
-    routerElement,
-    ({ result }) => {
-      return { element: result, pathname: pagePath }
+    const routerElement = (
+      <ServerLocation url={`${__BASE_PATH__}${pagePath}`}>
+        <Router id="gatsby-focus-wrapper" baseuri={__BASE_PATH__}>
+          <RouteHandler path="/*" />
+        </Router>
+        <div {...RouteAnnouncerProps} />
+      </ServerLocation>
+    )
+
+    const bodyComponent = apiRunner(
+      `wrapRootElement`,
+      { element: routerElement, pathname: pagePath },
+      routerElement,
+      ({ result }) => {
+        return { element: result, pathname: pagePath }
+      }
+    ).pop()
+
+    if (process.env.GATSBY_EXPERIMENTAL_DEV_SSR) {
+      // Let the site or plugin render the page component.
+      apiRunner(`replaceRenderer`, {
+        bodyComponent,
+        replaceBodyHTMLString,
+        setHeadComponents,
+        setHtmlAttributes,
+        setBodyAttributes,
+        setPreBodyComponents,
+        setPostBodyComponents,
+        setBodyProps,
+        pathname: pagePath,
+        pathPrefix: __PATH_PREFIX__,
+      })
+    }
+
+    // If no one stepped up, we'll handle it.
+    if (!bodyHtml) {
+      try {
+        if (process.env.GATSBY_EXPERIMENTAL_DEV_SSR) {
+          bodyHtml = renderToString(bodyComponent)
+        }
+      } catch (e) {
+        // ignore @reach/router redirect errors
+        if (!isRedirect(e)) throw e
+      }
     }
-  ).pop()
 
-  if (process.env.GATSBY_EXPERIMENTAL_DEV_SSR) {
-    // Let the site or plugin render the page component.
-    apiRunner(`replaceRenderer`, {
-      bodyComponent,
-      replaceBodyHTMLString,
+    apiRunner(`onRenderBody`, {
       setHeadComponents,
       setHtmlAttributes,
       setBodyAttributes,
@@ -175,45 +201,24 @@ export default (pagePath, callback) => {
       setPostBodyComponents,
       setBodyProps,
       pathname: pagePath,
-      pathPrefix: __PATH_PREFIX__,
     })
-  }
-
-  // If no one stepped up, we'll handle it.
-  if (!bodyHtml) {
-    try {
-      if (process.env.GATSBY_EXPERIMENTAL_DEV_SSR) {
-        bodyHtml = renderToString(bodyComponent)
-      }
-    } catch (e) {
-      // ignore @reach/router redirect errors
-      if (!isRedirect(e)) throw e
-    }
-  }
 
-  apiRunner(`onRenderBody`, {
-    setHeadComponents,
-    setHtmlAttributes,
-    setBodyAttributes,
-    setPreBodyComponents,
-    setPostBodyComponents,
-    setBodyProps,
-    pathname: pagePath,
-  })
+    apiRunner(`onPreRenderHTML`, {
+      getHeadComponents,
+      replaceHeadComponents,
+      getPreBodyComponents,
+      replacePreBodyComponents,
+      getPostBodyComponents,
+      replacePostBodyComponents,
+      pathname: pagePath,
+    })
 
-  apiRunner(`onPreRenderHTML`, {
-    getHeadComponents,
-    replaceHeadComponents,
-    getPreBodyComponents,
-    replacePreBodyComponents,
-    getPostBodyComponents,
-    replacePostBodyComponents,
-    pathname: pagePath,
-  })
+    return bodyHtml
+  }
 
   let bodyStr = ``
   if (process.env.GATSBY_EXPERIMENTAL_DEV_SSR && pagePath !== `/dev-404/`) {
-    bodyStr = bodyHtml
+    bodyStr = generateBodyHTML()
   }
 
   const htmlElement = React.createElement(Html, {
@@ -230,7 +235,7 @@ export default (pagePath, callback) => {
       <script key={`commons`} src="/commons.js" />,
     ]),
   })
-  htmlStr = renderToStaticMarkup(htmlElement)
+  let htmlStr = renderToStaticMarkup(htmlElement)
   htmlStr = `<!DOCTYPE html>${htmlStr}`
 
   callback(null, htmlStr)

From cc25d74292484282254b37744c22f810ade64f4c Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Tue, 20 Oct 2020 09:50:24 -0600
Subject: [PATCH 054/123] remove unneeded change

---
 .../gatsby/src/utils/worker/render-html.ts    | 23 ++++++++-----------
 1 file changed, 10 insertions(+), 13 deletions(-)

diff --git a/packages/gatsby/src/utils/worker/render-html.ts b/packages/gatsby/src/utils/worker/render-html.ts
index c2c2344a0ffb8..858d154842c0a 100644
--- a/packages/gatsby/src/utils/worker/render-html.ts
+++ b/packages/gatsby/src/utils/worker/render-html.ts
@@ -29,21 +29,18 @@ export const renderHTML = ({
         }
         const htmlComponentRenderer = require(htmlComponentRendererPath)
         try {
-          htmlComponentRenderer.default(
-            path,
-            async (_throwAway, htmlString) => {
-              if (stage === `develop-html`) {
-                resolve(htmlString)
-              } else {
-                resolve(
-                  fs.outputFile(
-                    getPageHtmlFilePath(join(process.cwd(), `public`), path),
-                    htmlString
-                  )
+          htmlComponentRenderer.default(path, (_throwAway, htmlString) => {
+            if (stage === `develop-html`) {
+              resolve(htmlString)
+            } else {
+              resolve(
+                fs.outputFile(
+                  getPageHtmlFilePath(join(process.cwd(), `public`), path),
+                  htmlString
                 )
-              }
+              )
             }
-          )
+          })
         } catch (e) {
           // add some context to error so we can display more helpful message
           e.context = {

From 27c18ddaec6a2e44cec51565a998119c2ea40315 Mon Sep 17 00:00:00 2001
From: Michal Piechowiak <misiek.piechowiak@gmail.com>
Date: Tue, 20 Oct 2020 10:00:47 -0600
Subject: [PATCH 055/123] Add find-page-by-path util from @pieh

---
 .../src/utils/__tests__/find-page-by-path.ts  | 172 ++++++++++++++++++
 .../gatsby/src/utils/find-page-by-path.ts     |  88 +++++++++
 2 files changed, 260 insertions(+)
 create mode 100644 packages/gatsby/src/utils/__tests__/find-page-by-path.ts
 create mode 100644 packages/gatsby/src/utils/find-page-by-path.ts

diff --git a/packages/gatsby/src/utils/__tests__/find-page-by-path.ts b/packages/gatsby/src/utils/__tests__/find-page-by-path.ts
new file mode 100644
index 0000000000000..0154f96df6289
--- /dev/null
+++ b/packages/gatsby/src/utils/__tests__/find-page-by-path.ts
@@ -0,0 +1,172 @@
+import { findPageByPath } from "../find-page-by-path"
+import { IGatsbyPage, IGatsbyState } from "../../redux/types"
+
+function generatePagesState(
+  pages: Array<string | Partial<IGatsbyPage>>
+): IGatsbyState {
+  const state: IGatsbyState = {
+    pages: new Map<string, IGatsbyPage>(
+      pages.map(pathOrObject => {
+        let pageObject
+
+        if (typeof pathOrObject !== `object`) {
+          pageObject = {
+            path: pathOrObject,
+          }
+        } else {
+          pageObject = pathOrObject
+        }
+
+        if (!pageObject.path) {
+          throw new Error(
+            `Missing path\n\n${JSON.stringify(pageObject, null, 2)}`
+          )
+        }
+
+        return [pageObject.path, pageObject]
+      })
+    ),
+  } as IGatsbyState
+
+  return state
+}
+
+const commonPages = [
+  `/`,
+  `no-slashes`,
+  `/leading-slash`,
+  `trailing-slash/`,
+  `/leading-and-trailing-slash/`,
+  {
+    path: `/app/`,
+    matchPath: `/app/*`,
+  },
+  `/app/static/`,
+]
+
+const state = generatePagesState([...commonPages])
+
+describe(`findPageByPath`, () => {
+  function assertGetterWithSlashMatrix(
+    pathWithoutTrailingOrLeadingSlashes: string,
+    exactPath: string
+  ) {
+    const withTrailingSlash = pathWithoutTrailingOrLeadingSlashes + `/`
+    const withLeadingSlash = `/` + pathWithoutTrailingOrLeadingSlashes
+    const withLeadingAndTrailingSlash =
+      `/` + pathWithoutTrailingOrLeadingSlashes + `/`
+
+    it(`Finds a page when request doesn't have trailing and leading slashes ("${pathWithoutTrailingOrLeadingSlashes}")`, () => {
+      const page = findPageByPath(state, pathWithoutTrailingOrLeadingSlashes)
+      expect(page).toBeDefined()
+      expect(page?.path).toEqual(exactPath)
+    })
+
+    it(`Finds a page when request have trailing slash ("${withTrailingSlash}")`, () => {
+      const page = findPageByPath(state, withTrailingSlash)
+      expect(page).toBeDefined()
+      expect(page?.path).toEqual(exactPath)
+    })
+
+    it(`Finds a page when request have leading slash ("${withLeadingSlash}")`, () => {
+      const page = findPageByPath(state, withLeadingSlash)
+      expect(page).toBeDefined()
+      expect(page?.path).toEqual(exactPath)
+    })
+
+    it(`Finds a page when request have both leading and trailing slash ("${withLeadingAndTrailingSlash}")`, () => {
+      const page = findPageByPath(state, withLeadingAndTrailingSlash)
+      expect(page).toBeDefined()
+      expect(page?.path).toEqual(exactPath)
+    })
+  }
+
+  describe(`Index`, () => {
+    it(`Exact match`, () => {
+      const page = findPageByPath(state, `/`)
+      expect(page).toBeDefined()
+      expect(page?.path).toEqual(`/`)
+    })
+
+    it(`Empty pathname -> "/"`, () => {
+      const page = findPageByPath(state, ``)
+      expect(page).toBeDefined()
+      expect(page?.path).toEqual(`/`)
+    })
+  })
+
+  describe(`Page with no slashes ("no-slashes")`, () => {
+    assertGetterWithSlashMatrix(`no-slashes`, `no-slashes`)
+  })
+
+  describe(`Page with leading slash ("/leading-slash")`, () => {
+    assertGetterWithSlashMatrix(`leading-slash`, `/leading-slash`)
+  })
+
+  describe(`Page with trailing slash ("trailing-slash/")`, () => {
+    assertGetterWithSlashMatrix(`trailing-slash`, `trailing-slash/`)
+  })
+
+  describe(`Page with leading and trailing slash ("/leading-and-trailing-slash/")`, () => {
+    assertGetterWithSlashMatrix(
+      `leading-and-trailing-slash`,
+      `/leading-and-trailing-slash/`
+    )
+  })
+
+  describe(`Client-only-paths`, () => {
+    it(`Can match client-only path by matchPath`, () => {
+      const page = findPageByPath(state, `/app/test`)
+      expect(page).toBeDefined()
+      expect(page?.path).toEqual(`/app/`)
+    })
+
+    it(`Can match client-only path by static`, () => {
+      const page = findPageByPath(state, `/app`)
+      expect(page).toBeDefined()
+      expect(page?.path).toEqual(`/app/`)
+    })
+
+    it(`Will prefer static page over client-only in case both match`, () => {
+      const page = findPageByPath(state, `/app/static`)
+      expect(page).toBeDefined()
+      expect(page?.path).toEqual(`/app/static/`)
+    })
+  })
+
+  describe(`No page found`, () => {
+    const stateWithBoth404Pages = generatePagesState([
+      ...commonPages,
+      // for fallbacks
+      `/dev-404-page/`,
+      `/404.html`,
+    ])
+
+    it(`Will return 404 pages if requested not found (will prefer /dev-404-page/)`, () => {
+      const page = findPageByPath(stateWithBoth404Pages, `/does-not-exist/`)
+      expect(page).toBeDefined()
+      expect(page?.path).toEqual(`/dev-404-page/`)
+    })
+
+    it(`Will return 404 page if requested not found (will fallback to /404.html)`, () => {
+      const stateWith404HtmlPage = generatePagesState([
+        ...commonPages,
+        // for fallbacks (no /dev-404-page/)
+        `/404.html`,
+      ])
+
+      const page = findPageByPath(stateWith404HtmlPage, `/does-not-exist/`)
+      expect(page).toBeDefined()
+      expect(page?.path).toEqual(`/404.html`)
+    })
+
+    it(`Returns undefined if not found any page and fallback turned off`, () => {
+      const page = findPageByPath(
+        stateWithBoth404Pages,
+        `/does-not-exist/`,
+        false
+      )
+      expect(page).toBeUndefined()
+    })
+  })
+})
diff --git a/packages/gatsby/src/utils/find-page-by-path.ts b/packages/gatsby/src/utils/find-page-by-path.ts
new file mode 100644
index 0000000000000..267d780076a57
--- /dev/null
+++ b/packages/gatsby/src/utils/find-page-by-path.ts
@@ -0,0 +1,88 @@
+import { IGatsbyPage, IGatsbyState } from "../redux/types"
+import { match } from "@reach/router/lib/utils"
+
+export function findPageByPath(
+  state: IGatsbyState,
+  path: string,
+  fallbackTo404: boolean = false
+): IGatsbyPage | undefined {
+  const { pages } = state
+
+  // first check by exact path
+  let page = pages.get(path)
+  if (page) {
+    return page
+  }
+
+  if (path === ``) {
+    // from my tests I never was able to make request with
+    // completely empty pathname, but just for the sake
+    // of completeness - try available alternative
+    page = pages.get(`/`)
+    if (page) {
+      return page
+    }
+  }
+  // Gatsby doesn't allow for page path to be empty string,
+  // so skipping trying to get page for "" path if we can't
+  // find page for `/`
+  else if (page !== `/`) {
+    // check various trailing/leading slashes combinations
+    const hasLeadingSlash = path.startsWith(`/`)
+    const hasTrailingSlash = path.endsWith(`/`)
+    for (const leadingSlash of [true, false]) {
+      for (const trailingSlash of [true, false]) {
+        if (
+          leadingSlash === hasLeadingSlash &&
+          trailingSlash === hasTrailingSlash
+        ) {
+          // we checked this already
+          continue
+        }
+
+        let newPath = path
+
+        if (leadingSlash !== hasLeadingSlash) {
+          if (leadingSlash) {
+            newPath = `/` + newPath
+          } else {
+            newPath = newPath.substring(1)
+          }
+        }
+
+        if (trailingSlash !== hasTrailingSlash) {
+          if (trailingSlash) {
+            newPath = newPath + `/`
+          } else {
+            newPath = newPath.substring(0, newPath.length - 1)
+          }
+        }
+
+        page = pages.get(newPath)
+        if (page) {
+          return page
+        }
+      }
+    }
+  }
+
+  // we didn't find exact static page, time to check matchPaths
+  for (const [, page] of pages) {
+    if (page.matchPath && match(page.matchPath, path)) {
+      return page
+    }
+  }
+
+  if (fallbackTo404) {
+    page = findPageByPath(state, `/dev-404-page/`, false)
+    if (page) {
+      return page
+    }
+
+    page = findPageByPath(state, `/404.html`, false)
+    if (page) {
+      return page
+    }
+  }
+  return undefined
+}

From 3059e56ebca23b534ae677d1d59d2d93cfd11289 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Tue, 20 Oct 2020 10:02:37 -0600
Subject: [PATCH 056/123] make typescript happy

---
 packages/gatsby/src/utils/__tests__/find-page-by-path.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/gatsby/src/utils/__tests__/find-page-by-path.ts b/packages/gatsby/src/utils/__tests__/find-page-by-path.ts
index 0154f96df6289..05aa9930f125d 100644
--- a/packages/gatsby/src/utils/__tests__/find-page-by-path.ts
+++ b/packages/gatsby/src/utils/__tests__/find-page-by-path.ts
@@ -50,7 +50,7 @@ describe(`findPageByPath`, () => {
   function assertGetterWithSlashMatrix(
     pathWithoutTrailingOrLeadingSlashes: string,
     exactPath: string
-  ) {
+  ): any {
     const withTrailingSlash = pathWithoutTrailingOrLeadingSlashes + `/`
     const withLeadingSlash = `/` + pathWithoutTrailingOrLeadingSlashes
     const withLeadingAndTrailingSlash =

From 78be60680222bf0cd3559d77d9fb77e812bb27da Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Tue, 20 Oct 2020 10:14:29 -0600
Subject: [PATCH 057/123] Switch to use @pieh's page finder util

---
 packages/gatsby/src/utils/develop-html-route.ts | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/packages/gatsby/src/utils/develop-html-route.ts b/packages/gatsby/src/utils/develop-html-route.ts
index 965a447c70817..dbc2bbe3f4fbd 100644
--- a/packages/gatsby/src/utils/develop-html-route.ts
+++ b/packages/gatsby/src/utils/develop-html-route.ts
@@ -3,8 +3,9 @@ import path from "path"
 import fs from "fs-extra"
 import { codeFrameColumns } from "@babel/code-frame"
 import ansiHTML from "ansi-html"
-const { trackCli } = require(`gatsby-telemetry`)
+import { trackCli } from "gatsby-telemetry"
 
+import { findPageByPath } from "./find-page-by-path"
 import { renderHTML } from "./worker/render-html"
 import { Stage } from "../commands/types"
 import { isWebpackStatusPending } from "./webpack-status"
@@ -144,9 +145,10 @@ export const route = ({ app, program, store }): any =>
     }
 
     trackCli(`GATSBY_EXPERIMENTAL_DEV_SSR`)
-    const { pages } = store.getState()
 
-    if (!pages.has(req.path)) {
+    const pathObj = findPageByPath(store.getState(), req.path)
+
+    if (!pathObj) {
       return next()
     }
 
@@ -156,7 +158,7 @@ export const route = ({ app, program, store }): any =>
     try {
       const renderResponse = await renderHTML({
         htmlComponentRendererPath: `${program.directory}/public/render-page.js`,
-        paths: [req.path],
+        paths: [pathObj.path],
         stage: Stage.DevelopHTML,
         envVars: [
           [`NODE_ENV`, process.env.NODE_ENV || ``],

From d37c1c441e8510e9f9fffffe08b80e0249f2c7c2 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Tue, 20 Oct 2020 21:08:38 -0600
Subject: [PATCH 058/123] fix find-page-by-path tests

---
 .../gatsby/src/utils/__tests__/find-page-by-path.ts  | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/packages/gatsby/src/utils/__tests__/find-page-by-path.ts b/packages/gatsby/src/utils/__tests__/find-page-by-path.ts
index 05aa9930f125d..15b9ccb9141e3 100644
--- a/packages/gatsby/src/utils/__tests__/find-page-by-path.ts
+++ b/packages/gatsby/src/utils/__tests__/find-page-by-path.ts
@@ -143,7 +143,11 @@ describe(`findPageByPath`, () => {
     ])
 
     it(`Will return 404 pages if requested not found (will prefer /dev-404-page/)`, () => {
-      const page = findPageByPath(stateWithBoth404Pages, `/does-not-exist/`)
+      const page = findPageByPath(
+        stateWithBoth404Pages,
+        `/does-not-exist/`,
+        true
+      )
       expect(page).toBeDefined()
       expect(page?.path).toEqual(`/dev-404-page/`)
     })
@@ -155,7 +159,11 @@ describe(`findPageByPath`, () => {
         `/404.html`,
       ])
 
-      const page = findPageByPath(stateWith404HtmlPage, `/does-not-exist/`)
+      const page = findPageByPath(
+        stateWith404HtmlPage,
+        `/does-not-exist/`,
+        true
+      )
       expect(page).toBeDefined()
       expect(page?.path).toEqual(`/404.html`)
     })

From a5b9b459ce2f7282e02784223a5488e6f60849ab Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Tue, 20 Oct 2020 21:14:56 -0600
Subject: [PATCH 059/123] Enable dev ssr for tests

---
 .../__snapshots__/static-entry.js.snap        |  6 +--
 .../gatsby/cache-dir/develop-static-entry.js  | 37 +++++++++----------
 2 files changed, 21 insertions(+), 22 deletions(-)

diff --git a/packages/gatsby/cache-dir/__tests__/__snapshots__/static-entry.js.snap b/packages/gatsby/cache-dir/__tests__/__snapshots__/static-entry.js.snap
index a32218f3e0838..2a5a48623247a 100644
--- a/packages/gatsby/cache-dir/__tests__/__snapshots__/static-entry.js.snap
+++ b/packages/gatsby/cache-dir/__tests__/__snapshots__/static-entry.js.snap
@@ -1,10 +1,10 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
-exports[`develop-static-entry onPreRenderHTML can be used to replace headComponents 1`] = `"<!DOCTYPE html><html><head><meta charSet=\\"utf-8\\"/><meta http-equiv=\\"x-ua-compatible\\" content=\\"ie=edge\\"/><meta name=\\"viewport\\" content=\\"width=device-width, initial-scale=1, shrink-to-fit=no\\"/><style> .style3  </style><style> .style2  </style><style> .style1  </style><meta name=\\"note\\" content=\\"environment=development\\"/><script src=\\"/socket.io/socket.io.js\\"></script></head><body><div id=\\"___gatsby\\"></div><script src=\\"/polyfill.js\\" nomodule=\\"\\"></script><script src=\\"/commons.js\\"></script></body></html>"`;
+exports[`develop-static-entry onPreRenderHTML can be used to replace headComponents 1`] = `"<!DOCTYPE html><html><head><meta charSet=\\"utf-8\\"/><meta http-equiv=\\"x-ua-compatible\\" content=\\"ie=edge\\"/><meta name=\\"viewport\\" content=\\"width=device-width, initial-scale=1, shrink-to-fit=no\\"/><style> .style3  </style><style> .style2  </style><style> .style1  </style><meta name=\\"note\\" content=\\"environment=development\\"/><script src=\\"/socket.io/socket.io.js\\"></script></head><body><div id=\\"___gatsby\\"><div style=\\"outline:none\\" tabindex=\\"-1\\" id=\\"gatsby-focus-wrapper\\"></div><div id=\\"gatsby-announcer\\" style=\\"position:absolute;top:0;width:1px;height:1px;padding:0;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;border:0\\" aria-live=\\"assertive\\" aria-atomic=\\"true\\"></div></div><script src=\\"/polyfill.js\\" nomodule=\\"\\"></script><script src=\\"/commons.js\\"></script></body></html>"`;
 
-exports[`develop-static-entry onPreRenderHTML can be used to replace postBodyComponents 1`] = `"<!DOCTYPE html><html><head><meta charSet=\\"utf-8\\"/><meta http-equiv=\\"x-ua-compatible\\" content=\\"ie=edge\\"/><meta name=\\"viewport\\" content=\\"width=device-width, initial-scale=1, shrink-to-fit=no\\"/><meta name=\\"note\\" content=\\"environment=development\\"/><script src=\\"/socket.io/socket.io.js\\"></script></head><body><div id=\\"___gatsby\\"></div><div> div3 </div><div> div2 </div><div> div1 </div><script src=\\"/polyfill.js\\" nomodule=\\"\\"></script><script src=\\"/commons.js\\"></script></body></html>"`;
+exports[`develop-static-entry onPreRenderHTML can be used to replace postBodyComponents 1`] = `"<!DOCTYPE html><html><head><meta charSet=\\"utf-8\\"/><meta http-equiv=\\"x-ua-compatible\\" content=\\"ie=edge\\"/><meta name=\\"viewport\\" content=\\"width=device-width, initial-scale=1, shrink-to-fit=no\\"/><meta name=\\"note\\" content=\\"environment=development\\"/><script src=\\"/socket.io/socket.io.js\\"></script></head><body><div id=\\"___gatsby\\"><div style=\\"outline:none\\" tabindex=\\"-1\\" id=\\"gatsby-focus-wrapper\\"></div><div id=\\"gatsby-announcer\\" style=\\"position:absolute;top:0;width:1px;height:1px;padding:0;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;border:0\\" aria-live=\\"assertive\\" aria-atomic=\\"true\\"></div></div><div> div3 </div><div> div2 </div><div> div1 </div><script src=\\"/polyfill.js\\" nomodule=\\"\\"></script><script src=\\"/commons.js\\"></script></body></html>"`;
 
-exports[`develop-static-entry onPreRenderHTML can be used to replace preBodyComponents 1`] = `"<!DOCTYPE html><html><head><meta charSet=\\"utf-8\\"/><meta http-equiv=\\"x-ua-compatible\\" content=\\"ie=edge\\"/><meta name=\\"viewport\\" content=\\"width=device-width, initial-scale=1, shrink-to-fit=no\\"/><meta name=\\"note\\" content=\\"environment=development\\"/><script src=\\"/socket.io/socket.io.js\\"></script></head><body><div> div3 </div><div> div2 </div><div> div1 </div><div id=\\"___gatsby\\"></div><script src=\\"/polyfill.js\\" nomodule=\\"\\"></script><script src=\\"/commons.js\\"></script></body></html>"`;
+exports[`develop-static-entry onPreRenderHTML can be used to replace preBodyComponents 1`] = `"<!DOCTYPE html><html><head><meta charSet=\\"utf-8\\"/><meta http-equiv=\\"x-ua-compatible\\" content=\\"ie=edge\\"/><meta name=\\"viewport\\" content=\\"width=device-width, initial-scale=1, shrink-to-fit=no\\"/><meta name=\\"note\\" content=\\"environment=development\\"/><script src=\\"/socket.io/socket.io.js\\"></script></head><body><div> div3 </div><div> div2 </div><div> div1 </div><div id=\\"___gatsby\\"><div style=\\"outline:none\\" tabindex=\\"-1\\" id=\\"gatsby-focus-wrapper\\"></div><div id=\\"gatsby-announcer\\" style=\\"position:absolute;top:0;width:1px;height:1px;padding:0;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;border:0\\" aria-live=\\"assertive\\" aria-atomic=\\"true\\"></div></div><script src=\\"/polyfill.js\\" nomodule=\\"\\"></script><script src=\\"/commons.js\\"></script></body></html>"`;
 
 exports[`static-entry onPreRenderHTML can be used to replace headComponents 1`] = `"<!DOCTYPE html><html><head><meta charSet=\\"utf-8\\"/><meta http-equiv=\\"x-ua-compatible\\" content=\\"ie=edge\\"/><meta name=\\"viewport\\" content=\\"width=device-width, initial-scale=1, shrink-to-fit=no\\"/><link as=\\"fetch\\" rel=\\"preload\\" href=\\"/page-data/app-data.json\\" crossorigin=\\"anonymous\\"/><link as=\\"fetch\\" rel=\\"preload\\" href=\\"/page-data/about/page-data.json\\" crossorigin=\\"anonymous\\"/><style> .style3  </style><style> .style2  </style><style> .style1  </style><meta name=\\"generator\\" content=\\"Gatsby 2.0.0\\"/></head><body><div id=\\"___gatsby\\"><div style=\\"outline:none\\" tabindex=\\"-1\\" id=\\"gatsby-focus-wrapper\\"></div><div id=\\"gatsby-announcer\\" style=\\"position:absolute;top:0;width:1px;height:1px;padding:0;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;border:0\\" aria-live=\\"assertive\\" aria-atomic=\\"true\\"></div></div><script id=\\"gatsby-script-loader\\">/*<![CDATA[*/window.pagePath=\\"/about/\\";/*]]>*/</script><script id=\\"gatsby-chunk-mapping\\">/*<![CDATA[*/window.___chunkMapping={};/*]]>*/</script></body></html>"`;
 
diff --git a/packages/gatsby/cache-dir/develop-static-entry.js b/packages/gatsby/cache-dir/develop-static-entry.js
index c9c985ef19979..4191584a91acf 100644
--- a/packages/gatsby/cache-dir/develop-static-entry.js
+++ b/packages/gatsby/cache-dir/develop-static-entry.js
@@ -165,28 +165,24 @@ export default (pagePath, callback) => {
       }
     ).pop()
 
-    if (process.env.GATSBY_EXPERIMENTAL_DEV_SSR) {
-      // Let the site or plugin render the page component.
-      apiRunner(`replaceRenderer`, {
-        bodyComponent,
-        replaceBodyHTMLString,
-        setHeadComponents,
-        setHtmlAttributes,
-        setBodyAttributes,
-        setPreBodyComponents,
-        setPostBodyComponents,
-        setBodyProps,
-        pathname: pagePath,
-        pathPrefix: __PATH_PREFIX__,
-      })
-    }
+    // Let the site or plugin render the page component.
+    apiRunner(`replaceRenderer`, {
+      bodyComponent,
+      replaceBodyHTMLString,
+      setHeadComponents,
+      setHtmlAttributes,
+      setBodyAttributes,
+      setPreBodyComponents,
+      setPostBodyComponents,
+      setBodyProps,
+      pathname: pagePath,
+      pathPrefix: __PATH_PREFIX__,
+    })
 
     // If no one stepped up, we'll handle it.
     if (!bodyHtml) {
       try {
-        if (process.env.GATSBY_EXPERIMENTAL_DEV_SSR) {
-          bodyHtml = renderToString(bodyComponent)
-        }
+        bodyHtml = renderToString(bodyComponent)
       } catch (e) {
         // ignore @reach/router redirect errors
         if (!isRedirect(e)) throw e
@@ -217,7 +213,10 @@ export default (pagePath, callback) => {
   }
 
   let bodyStr = ``
-  if (process.env.GATSBY_EXPERIMENTAL_DEV_SSR && pagePath !== `/dev-404/`) {
+  if (
+    process.env.NODE_ENV == `test` ||
+    (process.env.GATSBY_EXPERIMENTAL_DEV_SSR && pagePath !== `/dev-404/`)
+  ) {
     bodyStr = generateBodyHTML()
   }
 

From 9e2346632b1698f950e249701052fb3830fe3b6d Mon Sep 17 00:00:00 2001
From: Sidhartha Chatterjee <me@sidharthachatterjee.com>
Date: Wed, 21 Oct 2020 18:20:12 +0530
Subject: [PATCH 060/123] Add build:types again

---
 packages/gatsby/package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/gatsby/package.json b/packages/gatsby/package.json
index 1d84f6ea1720b..d0aeb01d0de7c 100644
--- a/packages/gatsby/package.json
+++ b/packages/gatsby/package.json
@@ -236,7 +236,7 @@
     "remark-mdxjs": "^2.0.0-next.3"
   },
   "scripts": {
-    "build": "npm run build:src && npm run build:internal-plugins && npm run build:rawfiles && npm run build:cjs",
+    "build": "npm run build:types && npm run build:src && npm run build:internal-plugins && npm run build:rawfiles && npm run build:cjs",
     "postbuild": "node scripts/output-api-file.js && yarn workspace gatsby-admin build",
     "build:internal-plugins": "copyfiles -u 1 src/internal-plugins/**/package.json dist",
     "build:rawfiles": "copyfiles -u 1 src/internal-plugins/**/raw_* dist",

From 8a4f585f373d6328474bbc7c039d80eaee887f3e Mon Sep 17 00:00:00 2001
From: Sidhartha Chatterjee <me@sidharthachatterjee.com>
Date: Wed, 21 Oct 2020 18:49:42 +0530
Subject: [PATCH 061/123] Do not await a flush

---
 packages/gatsby/src/utils/page-data.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/gatsby/src/utils/page-data.ts b/packages/gatsby/src/utils/page-data.ts
index c1f7e8a92f5cc..75582fc36cd1d 100644
--- a/packages/gatsby/src/utils/page-data.ts
+++ b/packages/gatsby/src/utils/page-data.ts
@@ -173,11 +173,11 @@ export async function flush(): Promise<void> {
   return
 }
 
-export async function enqueueFlush(): Promise<void> {
+export function enqueueFlush(): void {
   if (isWebpackStatusPending()) {
     isFlushPending = true
   } else {
-    await flush()
+    flush()
   }
 }
 

From bfccb5061e4cca398d43f9014971c2aee7f8d9e8 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Wed, 21 Oct 2020 15:57:10 -0600
Subject: [PATCH 062/123] Only delete the render-page.js module cache when it
 changes

---
 packages/gatsby/src/commands/build-html.ts      | 12 ++++++++++--
 packages/gatsby/src/utils/worker/render-html.ts |  4 ----
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/packages/gatsby/src/commands/build-html.ts b/packages/gatsby/src/commands/build-html.ts
index f299137607078..e52a3f0433501 100644
--- a/packages/gatsby/src/commands/build-html.ts
+++ b/packages/gatsby/src/commands/build-html.ts
@@ -14,7 +14,11 @@ import { IProgram, Stage } from "./types"
 type IActivity = any // TODO
 type IWorkerPool = any // TODO
 
-const runWebpack = (compilerConfig, stage: Stage): Bluebird<webpack.Stats> =>
+const runWebpack = (
+  compilerConfig,
+  stage: Stage,
+  directory
+): Bluebird<webpack.Stats> =>
   new Bluebird((resolve, reject) => {
     if (stage === `build-html`) {
       webpack(compilerConfig).run((err, stats) => {
@@ -33,6 +37,10 @@ const runWebpack = (compilerConfig, stage: Stage): Bluebird<webpack.Stats> =>
           if (err) {
             reject(err)
           } else {
+            // Make sure we get the latest version during development
+            delete require.cache[
+              require.resolve(`${directory}/public/render-page.js`)
+            ]
             resolve(stats)
           }
         }
@@ -45,7 +53,7 @@ const doBuildRenderer = async (
   webpackConfig: webpack.Configuration,
   stage: Stage
 ): Promise<string> => {
-  const stats = await runWebpack(webpackConfig, stage)
+  const stats = await runWebpack(webpackConfig, stage, directory)
   if (stats.hasErrors()) {
     reporter.panic(structureWebpackErrors(stage, stats.compilation.errors))
   }
diff --git a/packages/gatsby/src/utils/worker/render-html.ts b/packages/gatsby/src/utils/worker/render-html.ts
index 858d154842c0a..b6f59d8f298af 100644
--- a/packages/gatsby/src/utils/worker/render-html.ts
+++ b/packages/gatsby/src/utils/worker/render-html.ts
@@ -23,10 +23,6 @@ export const renderHTML = ({
     paths,
     path =>
       new Promise((resolve, reject) => {
-        // Make sure we get the latest version during development
-        if (stage === `develop-html`) {
-          delete require.cache[require.resolve(htmlComponentRendererPath)]
-        }
         const htmlComponentRenderer = require(htmlComponentRendererPath)
         try {
           htmlComponentRenderer.default(path, (_throwAway, htmlString) => {

From 93d4a649c37832a2904834a7f1480962bb7e27b5 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Wed, 21 Oct 2020 18:06:56 -0600
Subject: [PATCH 063/123] Try to reduce memory retention

---
 packages/gatsby/src/commands/build-html.ts    | 42 +++++++++++++++----
 packages/gatsby/src/utils/webpack.config.js   |  3 +-
 .../gatsby/src/utils/worker/render-html.ts    |  7 ++--
 3 files changed, 41 insertions(+), 11 deletions(-)

diff --git a/packages/gatsby/src/commands/build-html.ts b/packages/gatsby/src/commands/build-html.ts
index e52a3f0433501..4edf39e63acf9 100644
--- a/packages/gatsby/src/commands/build-html.ts
+++ b/packages/gatsby/src/commands/build-html.ts
@@ -5,6 +5,7 @@ import { createErrorFromString } from "gatsby-cli/lib/reporter/errors"
 import telemetry from "gatsby-telemetry"
 import { chunk } from "lodash"
 import webpack from "webpack"
+import crypto from "crypto"
 
 import webpackConfig from "../utils/webpack.config"
 import { structureWebpackErrors } from "../utils/webpack-error-utils"
@@ -14,6 +15,26 @@ import { IProgram, Stage } from "./types"
 type IActivity = any // TODO
 type IWorkerPool = any // TODO
 
+let oldHash = ``
+function getChecksum(path) {
+  return new Promise(function (resolve, reject) {
+    // crypto.createHash('sha1');
+    // crypto.createHash('sha256');
+    const hash = crypto.createHash("md5")
+    const input = fs.createReadStream(path)
+
+    input.on("error", reject)
+
+    input.on("data", function (chunk) {
+      hash.update(chunk)
+    })
+
+    input.on("close", function () {
+      resolve(hash.digest("hex"))
+    })
+  })
+}
+
 const runWebpack = (
   compilerConfig,
   stage: Stage,
@@ -23,9 +44,9 @@ const runWebpack = (
     if (stage === `build-html`) {
       webpack(compilerConfig).run((err, stats) => {
         if (err) {
-          reject(err)
+          return reject(err)
         } else {
-          resolve(stats)
+          return resolve(stats)
         }
       })
     } else if (stage === `develop-html`) {
@@ -37,11 +58,18 @@ const runWebpack = (
           if (err) {
             reject(err)
           } else {
-            // Make sure we get the latest version during development
-            delete require.cache[
-              require.resolve(`${directory}/public/render-page.js`)
-            ]
-            resolve(stats)
+            const pathToRenderPage = `${directory}/public/render-page.js`
+
+            getChecksum(pathToRenderPage).then(newHash => {
+              if (oldHash !== `` && newHash !== oldHash) {
+                // Make sure we get the latest version during development
+                delete require.cache[require.resolve(pathToRenderPage)]
+              }
+
+              oldHash = newHash
+
+              return resolve(stats)
+            })
           }
         }
       )
diff --git a/packages/gatsby/src/utils/webpack.config.js b/packages/gatsby/src/utils/webpack.config.js
index 56971525534f2..fa5e29a2140e1 100644
--- a/packages/gatsby/src/utils/webpack.config.js
+++ b/packages/gatsby/src/utils/webpack.config.js
@@ -236,9 +236,10 @@ module.exports = async (
     switch (stage) {
       case `develop`:
         return `cheap-module-source-map`
+      case: `develop-html`:
+          return `inline-cheap-source-map`
       // use a normal `source-map` for the html phases since
       // it gives better line and column numbers
-      case `develop-html`:
       case `build-html`:
       case `build-javascript`:
         return `source-map`
diff --git a/packages/gatsby/src/utils/worker/render-html.ts b/packages/gatsby/src/utils/worker/render-html.ts
index b6f59d8f298af..d1dae270352ee 100644
--- a/packages/gatsby/src/utils/worker/render-html.ts
+++ b/packages/gatsby/src/utils/worker/render-html.ts
@@ -19,17 +19,18 @@ export const renderHTML = ({
   // for modules that aren't bundled by webpack.
   envVars.forEach(([key, value]) => (process.env[key] = value))
 
+  const htmlComponentRenderer = require(htmlComponentRendererPath)
+
   return Promise.map(
     paths,
     path =>
       new Promise((resolve, reject) => {
-        const htmlComponentRenderer = require(htmlComponentRendererPath)
         try {
           htmlComponentRenderer.default(path, (_throwAway, htmlString) => {
             if (stage === `develop-html`) {
-              resolve(htmlString)
+              return resolve(htmlString)
             } else {
-              resolve(
+              return resolve(
                 fs.outputFile(
                   getPageHtmlFilePath(join(process.cwd(), `public`), path),
                   htmlString

From f1532e3be5d7f5a18f9fa90cf786b0a4a08c2ff6 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Wed, 21 Oct 2020 21:53:34 -0600
Subject: [PATCH 064/123] Fix recreating dev 404 page on every request + cache
 requires

---
 packages/gatsby/cache-dir/develop-static-entry.js |  2 +-
 packages/gatsby/src/commands/build-html.ts        | 10 +++++-----
 packages/gatsby/src/utils/start-server.ts         |  3 ++-
 packages/gatsby/src/utils/webpack.config.js       |  5 ++---
 packages/gatsby/src/utils/worker/render-html.ts   | 10 +++++++++-
 5 files changed, 19 insertions(+), 11 deletions(-)

diff --git a/packages/gatsby/cache-dir/develop-static-entry.js b/packages/gatsby/cache-dir/develop-static-entry.js
index 4191584a91acf..0c9e3475c81a6 100644
--- a/packages/gatsby/cache-dir/develop-static-entry.js
+++ b/packages/gatsby/cache-dir/develop-static-entry.js
@@ -215,7 +215,7 @@ export default (pagePath, callback) => {
   let bodyStr = ``
   if (
     process.env.NODE_ENV == `test` ||
-    (process.env.GATSBY_EXPERIMENTAL_DEV_SSR && pagePath !== `/dev-404/`)
+    (process.env.GATSBY_EXPERIMENTAL_DEV_SSR && pagePath !== `/dev-404-page/`)
   ) {
     bodyStr = generateBodyHTML()
   }
diff --git a/packages/gatsby/src/commands/build-html.ts b/packages/gatsby/src/commands/build-html.ts
index 4edf39e63acf9..9476ed2f448ce 100644
--- a/packages/gatsby/src/commands/build-html.ts
+++ b/packages/gatsby/src/commands/build-html.ts
@@ -20,17 +20,17 @@ function getChecksum(path) {
   return new Promise(function (resolve, reject) {
     // crypto.createHash('sha1');
     // crypto.createHash('sha256');
-    const hash = crypto.createHash("md5")
+    const hash = crypto.createHash(`md5`)
     const input = fs.createReadStream(path)
 
-    input.on("error", reject)
+    input.on(`error`, reject)
 
-    input.on("data", function (chunk) {
+    input.on(`data`, function (chunk) {
       hash.update(chunk)
     })
 
-    input.on("close", function () {
-      resolve(hash.digest("hex"))
+    input.on(`close`, function () {
+      resolve(hash.digest(`hex`))
     })
   })
 }
diff --git a/packages/gatsby/src/utils/start-server.ts b/packages/gatsby/src/utils/start-server.ts
index 457491f16636d..10098b88343aa 100644
--- a/packages/gatsby/src/utils/start-server.ts
+++ b/packages/gatsby/src/utils/start-server.ts
@@ -290,7 +290,7 @@ export async function startServer(
     developHtmlRoute({ app, program, store })
   }
 
-  const genericHtmlPageCreated = false
+  let genericHtmlPageCreated = false
   // We still need this even w/ ssr for the dev 404 page.
   const genericHtmlPath = process.env.GATSBY_EXPERIMENTAL_DEV_SSR
     ? `/public/dev-404-page/index.html`
@@ -298,6 +298,7 @@ export async function startServer(
   app.use(async (_, res) => {
     if (!genericHtmlPageCreated) {
       await createIndexHtml(indexHTMLActivity)
+      genericHtmlPageCreated = true
     }
     res.sendFile(directoryPath(genericHtmlPath), err => {
       if (err) {
diff --git a/packages/gatsby/src/utils/webpack.config.js b/packages/gatsby/src/utils/webpack.config.js
index fa5e29a2140e1..d2c2cf96dc37b 100644
--- a/packages/gatsby/src/utils/webpack.config.js
+++ b/packages/gatsby/src/utils/webpack.config.js
@@ -135,7 +135,7 @@ module.exports = async (
         }
       case `build-html`:
       case `develop-html`:
-        // A temp file required by static-site-generator-plugin. See plugins() below.
+        // Generate the file needed to SSR pages.
         // Deleted by build-html.js, since it's not needed for production.
         return {
           path: directoryPath(`public`),
@@ -236,10 +236,9 @@ module.exports = async (
     switch (stage) {
       case `develop`:
         return `cheap-module-source-map`
-      case: `develop-html`:
-          return `inline-cheap-source-map`
       // use a normal `source-map` for the html phases since
       // it gives better line and column numbers
+      case `develop-html`:
       case `build-html`:
       case `build-javascript`:
         return `source-map`
diff --git a/packages/gatsby/src/utils/worker/render-html.ts b/packages/gatsby/src/utils/worker/render-html.ts
index d1dae270352ee..8822698365d68 100644
--- a/packages/gatsby/src/utils/worker/render-html.ts
+++ b/packages/gatsby/src/utils/worker/render-html.ts
@@ -4,6 +4,8 @@ import { join } from "path"
 import { getPageHtmlFilePath } from "../../utils/page-html"
 import { Stage } from "../../commands/types"
 
+const renderers = {}
+
 export const renderHTML = ({
   htmlComponentRendererPath,
   paths,
@@ -19,7 +21,13 @@ export const renderHTML = ({
   // for modules that aren't bundled by webpack.
   envVars.forEach(([key, value]) => (process.env[key] = value))
 
-  const htmlComponentRenderer = require(htmlComponentRendererPath)
+  let htmlComponentRenderer
+  if (require.cache[htmlComponentRendererPath]) {
+    htmlComponentRenderer = renderers[htmlComponentRendererPath]
+  } else {
+    renderers[htmlComponentRendererPath] = require(htmlComponentRendererPath)
+    htmlComponentRenderer = renderers[htmlComponentRendererPath]
+  }
 
   return Promise.map(
     paths,

From 734f5b7d0e3936ae886cfe449d05209a17a70f13 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Wed, 21 Oct 2020 21:56:57 -0600
Subject: [PATCH 065/123] Add return

---
 packages/gatsby/src/commands/build-html.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/gatsby/src/commands/build-html.ts b/packages/gatsby/src/commands/build-html.ts
index 9476ed2f448ce..12c173437e82f 100644
--- a/packages/gatsby/src/commands/build-html.ts
+++ b/packages/gatsby/src/commands/build-html.ts
@@ -16,7 +16,7 @@ type IActivity = any // TODO
 type IWorkerPool = any // TODO
 
 let oldHash = ``
-function getChecksum(path) {
+function getChecksum(path): string {
   return new Promise(function (resolve, reject) {
     // crypto.createHash('sha1');
     // crypto.createHash('sha256');

From 36c33d740739badc165ce1d2fd8ef0984b2c30d4 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Wed, 21 Oct 2020 21:57:10 -0600
Subject: [PATCH 066/123] this wasn't necessary

---
 packages/gatsby/src/utils/worker/render-html.ts | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/packages/gatsby/src/utils/worker/render-html.ts b/packages/gatsby/src/utils/worker/render-html.ts
index 8822698365d68..bfacf364235c0 100644
--- a/packages/gatsby/src/utils/worker/render-html.ts
+++ b/packages/gatsby/src/utils/worker/render-html.ts
@@ -21,13 +21,7 @@ export const renderHTML = ({
   // for modules that aren't bundled by webpack.
   envVars.forEach(([key, value]) => (process.env[key] = value))
 
-  let htmlComponentRenderer
-  if (require.cache[htmlComponentRendererPath]) {
-    htmlComponentRenderer = renderers[htmlComponentRendererPath]
-  } else {
-    renderers[htmlComponentRendererPath] = require(htmlComponentRendererPath)
-    htmlComponentRenderer = renderers[htmlComponentRendererPath]
-  }
+  const htmlComponentRenderer = require(htmlComponentRendererPath)
 
   return Promise.map(
     paths,

From b57cf79279814ea2022b790b19161a85f98bda94 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Wed, 21 Oct 2020 21:58:52 -0600
Subject: [PATCH 067/123] Remove unused var

---
 packages/gatsby/src/utils/worker/render-html.ts | 2 --
 1 file changed, 2 deletions(-)

diff --git a/packages/gatsby/src/utils/worker/render-html.ts b/packages/gatsby/src/utils/worker/render-html.ts
index bfacf364235c0..d1dae270352ee 100644
--- a/packages/gatsby/src/utils/worker/render-html.ts
+++ b/packages/gatsby/src/utils/worker/render-html.ts
@@ -4,8 +4,6 @@ import { join } from "path"
 import { getPageHtmlFilePath } from "../../utils/page-html"
 import { Stage } from "../../commands/types"
 
-const renderers = {}
-
 export const renderHTML = ({
   htmlComponentRendererPath,
   paths,

From bad3cda093b9dd1648492e6a0b3e809cd64abcf9 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Wed, 21 Oct 2020 22:45:19 -0600
Subject: [PATCH 068/123] fix return type

---
 packages/gatsby/src/commands/build-html.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/gatsby/src/commands/build-html.ts b/packages/gatsby/src/commands/build-html.ts
index 12c173437e82f..ded69ad2c7e1b 100644
--- a/packages/gatsby/src/commands/build-html.ts
+++ b/packages/gatsby/src/commands/build-html.ts
@@ -16,7 +16,7 @@ type IActivity = any // TODO
 type IWorkerPool = any // TODO
 
 let oldHash = ``
-function getChecksum(path): string {
+function getChecksum(path): Promise<string> {
   return new Promise(function (resolve, reject) {
     // crypto.createHash('sha1');
     // crypto.createHash('sha256');

From 257ec8643dae51cb6fc2e2bb1d8ce1e1bc04ab5c Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Fri, 23 Oct 2020 12:06:25 -0700
Subject: [PATCH 069/123] Share cache across develop/develop-html instances of
 webpack

---
 packages/gatsby/src/utils/webpack.config.js | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/packages/gatsby/src/utils/webpack.config.js b/packages/gatsby/src/utils/webpack.config.js
index d2c2cf96dc37b..fda97e07819bb 100644
--- a/packages/gatsby/src/utils/webpack.config.js
+++ b/packages/gatsby/src/utils/webpack.config.js
@@ -677,6 +677,13 @@ module.exports = async (
     }
   }
 
+  // Share the in-memory cache across develop & develop-html to reduce
+  // memory usage in dev.
+  const sharedDevelopCache = {}
+  if (stage === `develop` || stage === `develop-html`) {
+    config.cache = sharedDevelopCache
+  }
+
   store.dispatch(actions.replaceWebpackConfig(config))
   const getConfig = () => store.getState().webpack
 

From e33e84365b57232cb6a90f545a6fec7241404ef5 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Fri, 23 Oct 2020 17:42:33 -0700
Subject: [PATCH 070/123] This caused a lot of runtime tests to fail

---
 packages/gatsby/cache-dir/app.js | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/gatsby/cache-dir/app.js b/packages/gatsby/cache-dir/app.js
index 6174389fe1132..629a3aeefbac2 100644
--- a/packages/gatsby/cache-dir/app.js
+++ b/packages/gatsby/cache-dir/app.js
@@ -101,7 +101,9 @@ apiRunnerAsync(`onClientEntry`).then(() => {
   const renderer = apiRunner(
     `replaceHydrateFunction`,
     undefined,
-    ReactDOM.hydrate
+    // TODO replace with hydrate once dev SSR is ready
+    // but only for SSRed pages.
+    ReactDOM.render
   )[0]
 
   Promise.all([

From 6482f0361262ae915489a49726eec7afcf282157 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Fri, 23 Oct 2020 17:43:26 -0700
Subject: [PATCH 071/123] Use the webpack hash

---
 packages/gatsby/src/commands/build-html.ts | 37 ++++++----------------
 1 file changed, 9 insertions(+), 28 deletions(-)

diff --git a/packages/gatsby/src/commands/build-html.ts b/packages/gatsby/src/commands/build-html.ts
index ded69ad2c7e1b..45ad3e0130a05 100644
--- a/packages/gatsby/src/commands/build-html.ts
+++ b/packages/gatsby/src/commands/build-html.ts
@@ -5,7 +5,6 @@ import { createErrorFromString } from "gatsby-cli/lib/reporter/errors"
 import telemetry from "gatsby-telemetry"
 import { chunk } from "lodash"
 import webpack from "webpack"
-import crypto from "crypto"
 
 import webpackConfig from "../utils/webpack.config"
 import { structureWebpackErrors } from "../utils/webpack-error-utils"
@@ -16,25 +15,7 @@ type IActivity = any // TODO
 type IWorkerPool = any // TODO
 
 let oldHash = ``
-function getChecksum(path): Promise<string> {
-  return new Promise(function (resolve, reject) {
-    // crypto.createHash('sha1');
-    // crypto.createHash('sha256');
-    const hash = crypto.createHash(`md5`)
-    const input = fs.createReadStream(path)
-
-    input.on(`error`, reject)
-
-    input.on(`data`, function (chunk) {
-      hash.update(chunk)
-    })
-
-    input.on(`close`, function () {
-      resolve(hash.digest(`hex`))
-    })
-  })
-}
-
+let newHash = ``
 const runWebpack = (
   compilerConfig,
   stage: Stage,
@@ -58,18 +39,18 @@ const runWebpack = (
           if (err) {
             reject(err)
           } else {
+            resolve(stats)
             const pathToRenderPage = `${directory}/public/render-page.js`
+            newHash = stats.hash || ``
 
-            getChecksum(pathToRenderPage).then(newHash => {
-              if (oldHash !== `` && newHash !== oldHash) {
-                // Make sure we get the latest version during development
-                delete require.cache[require.resolve(pathToRenderPage)]
-              }
+            if (oldHash !== `` && newHash !== oldHash) {
+              // Make sure we get the latest version during development
+              delete require.cache[require.resolve(pathToRenderPage)]
+            }
 
-              oldHash = newHash
+            oldHash = newHash
 
-              return resolve(stats)
-            })
+            return resolve(stats)
           }
         }
       )

From 26916f070b5f2d1cb9e4736a85f48fd60bde94a5 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Fri, 23 Oct 2020 17:43:47 -0700
Subject: [PATCH 072/123] This didn't work

---
 packages/gatsby/src/utils/webpack.config.js | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/packages/gatsby/src/utils/webpack.config.js b/packages/gatsby/src/utils/webpack.config.js
index fda97e07819bb..d2c2cf96dc37b 100644
--- a/packages/gatsby/src/utils/webpack.config.js
+++ b/packages/gatsby/src/utils/webpack.config.js
@@ -677,13 +677,6 @@ module.exports = async (
     }
   }
 
-  // Share the in-memory cache across develop & develop-html to reduce
-  // memory usage in dev.
-  const sharedDevelopCache = {}
-  if (stage === `develop` || stage === `develop-html`) {
-    config.cache = sharedDevelopCache
-  }
-
   store.dispatch(actions.replaceWebpackConfig(config))
   const getConfig = () => store.getState().webpack
 

From b7fcd007c744dc2eafa2c7e241e4b1a8814f2132 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Fri, 23 Oct 2020 17:56:37 -0700
Subject: [PATCH 073/123] fix lint error

---
 packages/gatsby/src/commands/build-html.ts | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/packages/gatsby/src/commands/build-html.ts b/packages/gatsby/src/commands/build-html.ts
index 45ad3e0130a05..1af1fd14d92b6 100644
--- a/packages/gatsby/src/commands/build-html.ts
+++ b/packages/gatsby/src/commands/build-html.ts
@@ -37,9 +37,8 @@ const runWebpack = (
         },
         (err, stats) => {
           if (err) {
-            reject(err)
+            return reject(err)
           } else {
-            resolve(stats)
             const pathToRenderPage = `${directory}/public/render-page.js`
             newHash = stats.hash || ``
 

From 37cf64c2123bdbbaf7b8bf56f7c8468f73c44788 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Sun, 25 Oct 2020 19:15:06 -0700
Subject: [PATCH 074/123] Meaningless change to try tests again

---
 packages/gatsby/src/commands/build-html.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/gatsby/src/commands/build-html.ts b/packages/gatsby/src/commands/build-html.ts
index 1af1fd14d92b6..704489c931477 100644
--- a/packages/gatsby/src/commands/build-html.ts
+++ b/packages/gatsby/src/commands/build-html.ts
@@ -42,8 +42,8 @@ const runWebpack = (
             const pathToRenderPage = `${directory}/public/render-page.js`
             newHash = stats.hash || ``
 
+            // Make sure we get the latest version during development
             if (oldHash !== `` && newHash !== oldHash) {
-              // Make sure we get the latest version during development
               delete require.cache[require.resolve(pathToRenderPage)]
             }
 

From 850231e9cc2b968dc5b3ca9b2183a99495df923b Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Mon, 26 Oct 2020 15:33:38 -0700
Subject: [PATCH 075/123] SSR pages in jest-worker so memory doesn't accumulate
 in main process

---
 packages/gatsby/src/commands/build-html.ts    |   5 +-
 .../gatsby/src/utils/develop-html-route.ts    | 151 +----------------
 .../gatsby/src/utils/render-dev-html-child.ts | 155 ++++++++++++++++++
 packages/gatsby/src/utils/render-dev-html.ts  |  36 ++++
 packages/gatsby/src/utils/start-server.ts     |  89 +++-------
 5 files changed, 226 insertions(+), 210 deletions(-)
 create mode 100644 packages/gatsby/src/utils/render-dev-html-child.ts
 create mode 100644 packages/gatsby/src/utils/render-dev-html.ts

diff --git a/packages/gatsby/src/commands/build-html.ts b/packages/gatsby/src/commands/build-html.ts
index 704489c931477..4ce77f18d0dd5 100644
--- a/packages/gatsby/src/commands/build-html.ts
+++ b/packages/gatsby/src/commands/build-html.ts
@@ -7,6 +7,7 @@ import { chunk } from "lodash"
 import webpack from "webpack"
 
 import webpackConfig from "../utils/webpack.config"
+import { restartWorker } from "../utils/render-dev-html"
 import { structureWebpackErrors } from "../utils/webpack-error-utils"
 
 import { IProgram, Stage } from "./types"
@@ -42,9 +43,9 @@ const runWebpack = (
             const pathToRenderPage = `${directory}/public/render-page.js`
             newHash = stats.hash || ``
 
-            // Make sure we get the latest version during development
+            // Make sure we use the latest version during development
             if (oldHash !== `` && newHash !== oldHash) {
-              delete require.cache[require.resolve(pathToRenderPage)]
+              restartWorker()
             }
 
             oldHash = newHash
diff --git a/packages/gatsby/src/utils/develop-html-route.ts b/packages/gatsby/src/utils/develop-html-route.ts
index dbc2bbe3f4fbd..c06d5c6f25a37 100644
--- a/packages/gatsby/src/utils/develop-html-route.ts
+++ b/packages/gatsby/src/utils/develop-html-route.ts
@@ -1,138 +1,10 @@
 import report from "gatsby-cli/lib/reporter"
-import path from "path"
-import fs from "fs-extra"
-import { codeFrameColumns } from "@babel/code-frame"
-import ansiHTML from "ansi-html"
 import { trackCli } from "gatsby-telemetry"
 
 import { findPageByPath } from "./find-page-by-path"
-import { renderHTML } from "./worker/render-html"
-import { Stage } from "../commands/types"
+import { renderDevHTML } from "./render-dev-html"
 import { isWebpackStatusPending } from "./webpack-status"
 
-interface IErrorPosition {
-  filename: string
-  line: number
-  row: number
-}
-
-const getPosition = function (stackObject): IErrorPosition {
-  let filename
-  let line
-  let row
-  // Because the JavaScript error stack has not yet been standardized,
-  // wrap the stack parsing in a try/catch for a soft fail if an
-  // unexpected stack is encountered.
-  try {
-    const filteredStack = stackObject.filter(function (s) {
-      return /\(.+?\)$/.test(s)
-    })
-    let splitLine
-    // For current Node & Chromium Error stacks
-    if (filteredStack.length > 0) {
-      splitLine = filteredStack[0].match(/(?:\()(.+?)(?:\))$/)[1].split(`:`)
-      // For older, future, or otherwise unexpected stacks
-    } else {
-      splitLine = stackObject[0].split(`:`)
-    }
-    const splitLength = splitLine.length
-    filename = splitLine[splitLength - 3]
-    line = Number(splitLine[splitLength - 2])
-    row = Number(splitLine[splitLength - 1])
-  } catch (err) {
-    filename = ``
-    line = 0
-    row = 0
-  }
-  return {
-    filename: filename,
-    line: line,
-    row: row,
-  }
-}
-// Colors taken from Gatsby's design tokens
-// https://github.com/gatsbyjs/gatsby/blob/d8acab3a135fa8250a0eb3a47c67300dde6eae32/packages/gatsby-design-tokens/src/colors.js#L185-L205
-const colors = {
-  background: `fdfaf6`,
-  text: `452475`,
-  green: `137886`,
-  darkGreen: `006500`,
-  comment: `527713`,
-  keyword: `096fb3`,
-  yellow: `DB3A00`,
-}
-
-interface IParsedError {
-  filename: string
-  code: string
-  codeFrame: string
-  line: number
-  row: number
-  message: string
-  type: string
-  stack: [string]
-}
-
-// Code borrowed and modified from https://github.com/watilde/parse-error
-export const parseError = function (err, directory): IParsedError {
-  const stack = err.stack ? err.stack : ``
-  const stackObject = stack.split(`\n`)
-  const position = getPosition(stackObject)
-
-  // Remove the `/lib/` added by webpack
-  const filename = path.join(
-    directory,
-    // Don't need to use path.sep as webpack always uses a single forward slash
-    // as a path seperator.
-    ...position.filename.split(`/`).slice(2)
-  )
-
-  let code
-  try {
-    code = fs.readFileSync(filename, `utf-8`)
-  } catch (e) {
-    console.log(`original error`, err)
-    report.error(`Couldn't read the file ${filename}`, e)
-  }
-  const line = position.line
-  const row = position.row
-  ansiHTML.setColors({
-    reset: [colors.text, colors.background], // FOREGROUND-COLOR or [FOREGROUND-COLOR] or [, BACKGROUND-COLOR] or [FOREGROUND-COLOR, BACKGROUND-COLOR]
-    black: `aaa`, // String
-    red: colors.keyword,
-    green: colors.green,
-    yellow: colors.yellow,
-    blue: `eee`,
-    magenta: `fff`,
-    cyan: colors.darkGreen,
-    lightgrey: `888`,
-    darkgrey: colors.comment,
-  })
-  const codeFrame = ansiHTML(
-    codeFrameColumns(
-      code,
-      {
-        start: { line: line, column: row },
-      },
-      { forceColor: true }
-    )
-  )
-  const splitMessage = err.message ? err.message.split(`\n`) : [``]
-  const message = splitMessage[splitMessage.length - 1]
-  const type = err.type ? err.type : err.name
-  const data = {
-    filename: filename,
-    code,
-    codeFrame,
-    line: line,
-    row: row,
-    message: message,
-    type: type,
-    stack: stack,
-  }
-  return data
-}
-
 export const route = ({ app, program, store }): any =>
   // Render an HTML page and serve it.
   app.get(`*`, async (req, res, next) => {
@@ -156,22 +28,13 @@ export const route = ({ app, program, store }): any =>
     htmlActivity.start()
 
     try {
-      const renderResponse = await renderHTML({
+      const renderResponse = await renderDevHTML({
+        path: pathObj.path,
         htmlComponentRendererPath: `${program.directory}/public/render-page.js`,
-        paths: [pathObj.path],
-        stage: Stage.DevelopHTML,
-        envVars: [
-          [`NODE_ENV`, process.env.NODE_ENV || ``],
-          [
-            `gatsby_executing_command`,
-            process.env.gatsby_executing_command || ``,
-          ],
-          [`gatsby_log_level`, process.env.gatsby_log_level || ``],
-        ],
+        directory: program.directory,
       })
-      res.status(200).send(renderResponse[0])
-    } catch (e) {
-      const error = parseError(e, program.directory)
+      res.status(200).send(renderResponse)
+    } catch (error) {
       res.status(500).send(`<title>Develop SSR Error</title><h1>Error<h1>
         <h2>The page didn't SSR correctly</h2>
         <ul>
@@ -180,7 +43,7 @@ export const route = ({ app, program, store }): any =>
         </ul>
         <h3>error message</h3>
         <p><code>${error.message}</code></p>
-        <pre style="background:#${colors.background};padding:8px;">${error.codeFrame}</pre>`)
+        <pre style="background:#fdfaf6;padding:8px;">${error.codeFrame}</pre>`)
     }
 
     // TODO add support for 404 and general rendering errors
diff --git a/packages/gatsby/src/utils/render-dev-html-child.ts b/packages/gatsby/src/utils/render-dev-html-child.ts
new file mode 100644
index 0000000000000..d909d85f71058
--- /dev/null
+++ b/packages/gatsby/src/utils/render-dev-html-child.ts
@@ -0,0 +1,155 @@
+import { codeFrameColumns } from "@babel/code-frame"
+import ansiHTML from "ansi-html"
+import fs from "fs-extra"
+import sysPath from "path"
+import report from "gatsby-cli/lib/reporter"
+
+interface IErrorPosition {
+  filename: string
+  line: number
+  row: number
+}
+
+const getPosition = function (stackObject): IErrorPosition {
+  let filename
+  let line
+  let row
+  // Because the JavaScript error stack has not yet been standardized,
+  // wrap the stack parsing in a try/catch for a soft fail if an
+  // unexpected stack is encountered.
+  try {
+    const filteredStack = stackObject.filter(function (s) {
+      return /\(.+?\)$/.test(s)
+    })
+    let splitLine
+    // For current Node & Chromium Error stacks
+    if (filteredStack.length > 0) {
+      splitLine = filteredStack[0].match(/(?:\()(.+?)(?:\))$/)[1].split(`:`)
+      // For older, future, or otherwise unexpected stacks
+    } else {
+      splitLine = stackObject[0].split(`:`)
+    }
+    const splitLength = splitLine.length
+    filename = splitLine[splitLength - 3]
+    line = Number(splitLine[splitLength - 2])
+    row = Number(splitLine[splitLength - 1])
+  } catch (err) {
+    filename = ``
+    line = 0
+    row = 0
+  }
+  return {
+    filename: filename,
+    line: line,
+    row: row,
+  }
+}
+// Colors taken from Gatsby's design tokens
+// https://github.com/gatsbyjs/gatsby/blob/d8acab3a135fa8250a0eb3a47c67300dde6eae32/packages/gatsby-design-tokens/src/colors.js#L185-L205
+const colors = {
+  background: `fdfaf6`,
+  text: `452475`,
+  green: `137886`,
+  darkGreen: `006500`,
+  comment: `527713`,
+  keyword: `096fb3`,
+  yellow: `DB3A00`,
+}
+
+interface IParsedError {
+  filename: string
+  code: string
+  codeFrame: string
+  line: number
+  row: number
+  message: string
+  type: string
+  stack: [string]
+}
+
+// Code borrowed and modified from https://github.com/watilde/parse-error
+export const parseError = function (err, directory): IParsedError {
+  const stack = err.stack ? err.stack : ``
+  const stackObject = stack.split(`\n`)
+  const position = getPosition(stackObject)
+
+  // Remove the `/lib/` added by webpack
+  const filename = sysPath.join(
+    directory,
+    // Don't need to use path.sep as webpack always uses a single forward slash
+    // as a path seperator.
+    ...position.filename.split(`/`).slice(2)
+  )
+
+  let code
+  try {
+    code = fs.readFileSync(filename, `utf-8`)
+  } catch (e) {
+    console.log(err)
+    report.error(`Couldn't read the file ${filename}`, e)
+  }
+  const line = position.line
+  const row = position.row
+  ansiHTML.setColors({
+    reset: [colors.text, colors.background], // FOREGROUND-COLOR or [FOREGROUND-COLOR] or [, BACKGROUND-COLOR] or [FOREGROUND-COLOR, BACKGROUND-COLOR]
+    black: `aaa`, // String
+    red: colors.keyword,
+    green: colors.green,
+    yellow: colors.yellow,
+    blue: `eee`,
+    magenta: `fff`,
+    cyan: colors.darkGreen,
+    lightgrey: `888`,
+    darkgrey: colors.comment,
+  })
+  const codeFrame = ansiHTML(
+    codeFrameColumns(
+      code,
+      {
+        start: { line: line, column: row },
+      },
+      { forceColor: true }
+    )
+  )
+  const splitMessage = err.message ? err.message.split(`\n`) : [``]
+  const message = splitMessage[splitMessage.length - 1]
+  const type = err.type ? err.type : err.name
+  const data = {
+    filename: filename,
+    code,
+    codeFrame,
+    line: line,
+    row: row,
+    message: message,
+    type: type,
+    stack: stack,
+  }
+  return data
+}
+
+export function renderHTML({ path, htmlComponentRendererPath, directory }) {
+  return new Promise((resolve, reject) => {
+    console.log(`inside worker`, { path, htmlComponentRendererPath, directory })
+    const htmlComponentRenderer = require(htmlComponentRendererPath)
+    try {
+      console.time(`SSR`)
+      htmlComponentRenderer.default(path, (_throwAway, htmlString) => {
+        resolve(htmlString)
+        console.timeEnd(`SSR`)
+      })
+    } catch (err) {
+      const stack = err.stack ? err.stack : ``
+      // Only generate error pages for webpack errors. If it's not a webpack
+      // error, it's not a user error so probably a system error so we'll just
+      // panic and quit.
+      const regex = /webpack:\/lib\//gm
+      if (!stack.match(regex)) {
+        report.panic(err)
+        return
+      }
+      const error = parseError(err, directory)
+      reject(error)
+      // return { error }
+    }
+  })
+}
diff --git a/packages/gatsby/src/utils/render-dev-html.ts b/packages/gatsby/src/utils/render-dev-html.ts
new file mode 100644
index 0000000000000..a55c0b8d56009
--- /dev/null
+++ b/packages/gatsby/src/utils/render-dev-html.ts
@@ -0,0 +1,36 @@
+import sysPath from "path"
+import JestWorker from "jest-worker"
+
+const startWorker = (): any =>
+  new JestWorker(sysPath.join(__dirname, `./render-dev-html-child.js`), {
+    exposedMethods: [`renderHTML`],
+    numWorkers: 1,
+    enableWorkerThreads: true,
+  })
+
+let worker = startWorker()
+
+export const restartWorker = (): null => {
+  const oldWorker = worker
+  const newWorker = startWorker()
+  worker = newWorker
+  oldWorker.end()
+}
+
+export const renderDevHTML = ({
+  path,
+  htmlComponentRendererPath,
+  directory,
+}): Promise<string> =>
+  new Promise(async (resolve, reject) => {
+    try {
+      const response = await worker.renderHTML({
+        path,
+        htmlComponentRendererPath,
+        directory,
+      })
+      resolve(response)
+    } catch (error) {
+      reject(error)
+    }
+  })
diff --git a/packages/gatsby/src/utils/start-server.ts b/packages/gatsby/src/utils/start-server.ts
index 10098b88343aa..9bf7ec06f5b1b 100644
--- a/packages/gatsby/src/utils/start-server.ts
+++ b/packages/gatsby/src/utils/start-server.ts
@@ -1,5 +1,3 @@
-import chokidar from "chokidar"
-
 import webpackHotMiddleware from "webpack-hot-middleware"
 import webpackDevMiddleware, {
   WebpackDevMiddleware,
@@ -18,7 +16,8 @@ import telemetry from "gatsby-telemetry"
 
 import webpackConfig from "../utils/webpack.config"
 import { store, emitter } from "../redux"
-import { buildRenderer, buildHTML } from "../commands/build-html"
+import { buildRenderer } from "../commands/build-html"
+import { renderDevHTML } from "./render-dev-html"
 import { withBasePath } from "../utils/path"
 import report from "gatsby-cli/lib/reporter"
 import launchEditor from "react-dev-utils/launchEditor"
@@ -28,7 +27,6 @@ import { route as developHtmlRoute } from "./develop-html-route"
 import { developStatic } from "../commands/develop-static"
 import withResolverContext from "../schema/context"
 import { websocketManager, WebsocketManager } from "../utils/websocket-manager"
-import { slash } from "gatsby-core-utils"
 import apiRunnerNode from "../utils/api-runner-node"
 import { Express } from "express"
 
@@ -65,41 +63,7 @@ export async function startServer(
   app: Express,
   workerPool: JestWorker = WorkerPool.create()
 ): Promise<IServer> {
-  const indexHTMLActivity = report.phantomActivity(`building index.html`, {})
-  indexHTMLActivity.start()
   const directory = program.directory
-  const directoryPath = withBasePath(directory)
-  const createIndexHtml = async (activity: ActivityTracker): Promise<void> => {
-    const genericHtmlPath = process.env.GATSBY_EXPERIMENTAL_DEV_SSR
-      ? `/dev-404-page/`
-      : `/`
-    try {
-      await buildHTML({
-        program,
-        stage: Stage.DevelopHTML,
-        pagePaths: [genericHtmlPath],
-        workerPool,
-        activity,
-      })
-    } catch (err) {
-      if (err.name !== `WebpackError`) {
-        report.panic(err)
-        return
-      }
-      report.panic(
-        report.stripIndent`
-          There was an error compiling the html.js component for the development server.
-
-          See our docs page on debugging HTML builds for help https://gatsby.dev/debug-html
-        `,
-        err
-      )
-    }
-  }
-
-  indexHTMLActivity.end()
-
-  // report.stateUpdate(`webpack`, `IN_PROGRESS`)
 
   const webpackActivity = report.activityTimer(`Building development bundle`, {
     id: `webpack-develop`,
@@ -290,21 +254,31 @@ export async function startServer(
     developHtmlRoute({ app, program, store })
   }
 
-  let genericHtmlPageCreated = false
   // We still need this even w/ ssr for the dev 404 page.
   const genericHtmlPath = process.env.GATSBY_EXPERIMENTAL_DEV_SSR
-    ? `/public/dev-404-page/index.html`
-    : `/public/index.html`
-  app.use(async (_, res) => {
-    if (!genericHtmlPageCreated) {
-      await createIndexHtml(indexHTMLActivity)
-      genericHtmlPageCreated = true
-    }
-    res.sendFile(directoryPath(genericHtmlPath), err => {
-      if (err) {
-        res.status(500).end()
+    ? `/dev-404-page/`
+    : `/`
+  app.use(async (req, res) => {
+    const fullUrl = req.protocol + `://` + req.get(`host`) + req.originalUrl
+    // This isn't used in development.
+    if (fullUrl.endsWith(`app-data.json`)) {
+      res.json({ webpackCompilationHash: `123` })
+      // If this gets here, it's a non-existant file so just send back 404.
+    } else if (fullUrl.endsWith(`.json`)) {
+      res.json({}).status(404)
+    } else {
+      try {
+        const renderResponse = await renderDevHTML({
+          path: genericHtmlPath,
+          htmlComponentRendererPath: `${program.directory}/public/render-page.js`,
+          directory: program.directory,
+        })
+        res.status(200).send(renderResponse)
+      } catch (e) {
+        console.log(e)
+        res.send(e).status(500)
       }
-    })
+    }
   })
 
   /**
@@ -312,25 +286,12 @@ export async function startServer(
    **/
   const server = new http.Server(app)
 
-  const socket = websocketManager.init({ server, directory: program.directory })
+  websocketManager.init({ server, directory: program.directory })
 
   // hardcoded `localhost`, because host should match `target` we set
   // in http proxy in `develop-proxy`
   const listener = server.listen(program.port, `localhost`)
 
-  if (!process.env.GATSBY_EXPERIMENTAL_DEV_SSR) {
-    // Register watcher that rebuilds index.html every time html.js changes.
-    const watchGlobs = [`src/html.js`, `plugins/**/gatsby-ssr.js`].map(path =>
-      slash(directoryPath(path))
-    )
-
-    chokidar.watch(watchGlobs).on(`change`, async () => {
-      await createIndexHtml(indexHTMLActivity)
-      // eslint-disable-next-line no-unused-expressions
-      socket?.to(`clients`).emit(`reload`)
-    })
-  }
-
   return {
     compiler,
     listener,

From 2581da478271d46602ab93f0a1ecfa8fcef47906 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Mon, 26 Oct 2020 15:53:35 -0700
Subject: [PATCH 076/123] fix lint

---
 packages/gatsby/src/utils/render-dev-html-child.ts | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/packages/gatsby/src/utils/render-dev-html-child.ts b/packages/gatsby/src/utils/render-dev-html-child.ts
index d909d85f71058..3c569e3b8052f 100644
--- a/packages/gatsby/src/utils/render-dev-html-child.ts
+++ b/packages/gatsby/src/utils/render-dev-html-child.ts
@@ -127,7 +127,11 @@ export const parseError = function (err, directory): IParsedError {
   return data
 }
 
-export function renderHTML({ path, htmlComponentRendererPath, directory }) {
+export function renderHTML({
+  path,
+  htmlComponentRendererPath,
+  directory,
+}): Promise<string | object> {
   return new Promise((resolve, reject) => {
     console.log(`inside worker`, { path, htmlComponentRendererPath, directory })
     const htmlComponentRenderer = require(htmlComponentRendererPath)

From a4ffe3f5d3a9d8052e7b659d1666c68e9d1abab2 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Mon, 26 Oct 2020 16:04:45 -0700
Subject: [PATCH 077/123] make typescript happy too

---
 packages/gatsby/src/commands/build-html.ts   | 9 ++-------
 packages/gatsby/src/utils/render-dev-html.ts | 2 +-
 packages/gatsby/src/utils/start-server.ts    | 1 -
 3 files changed, 3 insertions(+), 9 deletions(-)

diff --git a/packages/gatsby/src/commands/build-html.ts b/packages/gatsby/src/commands/build-html.ts
index 4ce77f18d0dd5..69fe712f2a865 100644
--- a/packages/gatsby/src/commands/build-html.ts
+++ b/packages/gatsby/src/commands/build-html.ts
@@ -17,11 +17,7 @@ type IWorkerPool = any // TODO
 
 let oldHash = ``
 let newHash = ``
-const runWebpack = (
-  compilerConfig,
-  stage: Stage,
-  directory
-): Bluebird<webpack.Stats> =>
+const runWebpack = (compilerConfig, stage: Stage): Bluebird<webpack.Stats> =>
   new Bluebird((resolve, reject) => {
     if (stage === `build-html`) {
       webpack(compilerConfig).run((err, stats) => {
@@ -40,7 +36,6 @@ const runWebpack = (
           if (err) {
             return reject(err)
           } else {
-            const pathToRenderPage = `${directory}/public/render-page.js`
             newHash = stats.hash || ``
 
             // Make sure we use the latest version during development
@@ -62,7 +57,7 @@ const doBuildRenderer = async (
   webpackConfig: webpack.Configuration,
   stage: Stage
 ): Promise<string> => {
-  const stats = await runWebpack(webpackConfig, stage, directory)
+  const stats = await runWebpack(webpackConfig, stage)
   if (stats.hasErrors()) {
     reporter.panic(structureWebpackErrors(stage, stats.compilation.errors))
   }
diff --git a/packages/gatsby/src/utils/render-dev-html.ts b/packages/gatsby/src/utils/render-dev-html.ts
index a55c0b8d56009..5f3c326d27c94 100644
--- a/packages/gatsby/src/utils/render-dev-html.ts
+++ b/packages/gatsby/src/utils/render-dev-html.ts
@@ -10,7 +10,7 @@ const startWorker = (): any =>
 
 let worker = startWorker()
 
-export const restartWorker = (): null => {
+export const restartWorker = (): void => {
   const oldWorker = worker
   const newWorker = startWorker()
   worker = newWorker
diff --git a/packages/gatsby/src/utils/start-server.ts b/packages/gatsby/src/utils/start-server.ts
index 9bf7ec06f5b1b..7dc2c44683de6 100644
--- a/packages/gatsby/src/utils/start-server.ts
+++ b/packages/gatsby/src/utils/start-server.ts
@@ -18,7 +18,6 @@ import webpackConfig from "../utils/webpack.config"
 import { store, emitter } from "../redux"
 import { buildRenderer } from "../commands/build-html"
 import { renderDevHTML } from "./render-dev-html"
-import { withBasePath } from "../utils/path"
 import report from "gatsby-cli/lib/reporter"
 import launchEditor from "react-dev-utils/launchEditor"
 import * as WorkerPool from "../utils/worker/pool"

From 98fbde150e0050827333ab7d20697b12c6aefe71 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Mon, 26 Oct 2020 16:41:44 -0700
Subject: [PATCH 078/123] fix test import

---
 packages/gatsby/src/utils/__tests__/develop-html-route.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/gatsby/src/utils/__tests__/develop-html-route.ts b/packages/gatsby/src/utils/__tests__/develop-html-route.ts
index e3bea3a02fec2..1b1395c64973c 100644
--- a/packages/gatsby/src/utils/__tests__/develop-html-route.ts
+++ b/packages/gatsby/src/utils/__tests__/develop-html-route.ts
@@ -1,4 +1,4 @@
-import { parseError } from "../develop-html-route"
+import { parseError } from "../render-dev-html-child"
 import error from "./fixtures/error-object"
 
 describe(`error parsing`, () => {

From a2528700c520451b1940768afcc9bba12776b16f Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Mon, 26 Oct 2020 18:19:11 -0700
Subject: [PATCH 079/123] Automatically fork the dev ssr renderer so it's ready
 to go when the user requests a page

---
 .../gatsby/src/utils/render-dev-html-child.ts |  2 ++
 packages/gatsby/src/utils/render-dev-html.ts  | 23 +++++++++++++++----
 2 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/packages/gatsby/src/utils/render-dev-html-child.ts b/packages/gatsby/src/utils/render-dev-html-child.ts
index 3c569e3b8052f..084f10f89361f 100644
--- a/packages/gatsby/src/utils/render-dev-html-child.ts
+++ b/packages/gatsby/src/utils/render-dev-html-child.ts
@@ -131,7 +131,9 @@ export function renderHTML({
   path,
   htmlComponentRendererPath,
   directory,
+  warming = false,
 }): Promise<string | object> {
+  if (warming) return null
   return new Promise((resolve, reject) => {
     console.log(`inside worker`, { path, htmlComponentRendererPath, directory })
     const htmlComponentRenderer = require(htmlComponentRendererPath)
diff --git a/packages/gatsby/src/utils/render-dev-html.ts b/packages/gatsby/src/utils/render-dev-html.ts
index 5f3c326d27c94..dc435526df2b1 100644
--- a/packages/gatsby/src/utils/render-dev-html.ts
+++ b/packages/gatsby/src/utils/render-dev-html.ts
@@ -1,13 +1,26 @@
 import sysPath from "path"
 import JestWorker from "jest-worker"
 
-const startWorker = (): any =>
-  new JestWorker(sysPath.join(__dirname, `./render-dev-html-child.js`), {
-    exposedMethods: [`renderHTML`],
-    numWorkers: 1,
-    enableWorkerThreads: true,
+const startWorker = (): any => {
+  const newWorker = new JestWorker(
+    sysPath.join(__dirname, `./render-dev-html-child.js`),
+    {
+      exposedMethods: [`renderHTML`],
+      numWorkers: 1,
+      enableWorkerThreads: true,
+    }
+  )
+
+  // jest-worker is lazy with forking but we want to fork immediately so the user
+  // doesn't have to wait.
+  // @ts-ignore
+  newWorker.renderHTML({
+    warming: true,
   })
 
+  return newWorker
+}
+
 let worker = startWorker()
 
 export const restartWorker = (): void => {

From 63765ed2305f5ed06d8e2b175e00ae800862cae2 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Mon, 26 Oct 2020 19:05:56 -0700
Subject: [PATCH 080/123] Add structured logging on dev ssr failure

---
 .../gatsby-cli/src/structured-errors/error-map.ts  |  8 ++++++++
 .../npm/fixtures/package-json/package.json         |  4 ++--
 packages/gatsby/src/utils/develop-html-route.ts    | 14 ++++++++++++++
 3 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/packages/gatsby-cli/src/structured-errors/error-map.ts b/packages/gatsby-cli/src/structured-errors/error-map.ts
index 2632acb24b882..05eb83d11e546 100644
--- a/packages/gatsby-cli/src/structured-errors/error-map.ts
+++ b/packages/gatsby-cli/src/structured-errors/error-map.ts
@@ -536,6 +536,14 @@ const errors = {
     level: Level.ERROR,
     docsUrl: `https://www.gatsbyjs.org/docs/gatsby-cli/#new`,
   },
+  "11614": {
+    text: ({ path, filePath }): string =>
+      `The path "${path}" errored during SSR. Edit its component ${filePath} to
+    resolve the error. See this docs page for more information
+    https://gatsby.dev/debug-html`,
+    level: Level.WARNING,
+    docsUrl: `https://gatsby.dev/debug-html`,
+  },
   // Watchdog
   "11701": {
     text: (context): string =>
diff --git a/packages/gatsby-recipes/src/providers/npm/fixtures/package-json/package.json b/packages/gatsby-recipes/src/providers/npm/fixtures/package-json/package.json
index 3e53932c9b0a5..7e530d8e6b0bb 100644
--- a/packages/gatsby-recipes/src/providers/npm/fixtures/package-json/package.json
+++ b/packages/gatsby-recipes/src/providers/npm/fixtures/package-json/package.json
@@ -1,4 +1,4 @@
 {
-  "name": "test",
+  "name": "test-npm-provider",
   "scripts": {}
-}
\ No newline at end of file
+}
diff --git a/packages/gatsby/src/utils/develop-html-route.ts b/packages/gatsby/src/utils/develop-html-route.ts
index c06d5c6f25a37..c58a94381ac76 100644
--- a/packages/gatsby/src/utils/develop-html-route.ts
+++ b/packages/gatsby/src/utils/develop-html-route.ts
@@ -35,6 +35,20 @@ export const route = ({ app, program, store }): any =>
       })
       res.status(200).send(renderResponse)
     } catch (error) {
+      report.error({
+        id: `11614`,
+        filePath: error.filename,
+        location: {
+          start: {
+            line: error.line,
+            column: error.row,
+          },
+        },
+        context: {
+          path: pathObj.path,
+          filePath: error.filename,
+        },
+      })
       res.status(500).send(`<title>Develop SSR Error</title><h1>Error<h1>
         <h2>The page didn't SSR correctly</h2>
         <ul>

From 3e9a3aa30c93d290990b83ff7e17f315d28b078a Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Mon, 26 Oct 2020 19:12:11 -0700
Subject: [PATCH 081/123] Need require.resolve I think

---
 packages/gatsby/src/utils/render-dev-html.ts | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/packages/gatsby/src/utils/render-dev-html.ts b/packages/gatsby/src/utils/render-dev-html.ts
index dc435526df2b1..7d077decd201a 100644
--- a/packages/gatsby/src/utils/render-dev-html.ts
+++ b/packages/gatsby/src/utils/render-dev-html.ts
@@ -1,9 +1,8 @@
-import sysPath from "path"
 import JestWorker from "jest-worker"
 
 const startWorker = (): any => {
   const newWorker = new JestWorker(
-    sysPath.join(__dirname, `./render-dev-html-child.js`),
+    require.resolve(`./render-dev-html-child.js`),
     {
       exposedMethods: [`renderHTML`],
       numWorkers: 1,

From b284388787e8be07961551f50000dd7d6a204ca4 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Mon, 26 Oct 2020 19:21:06 -0700
Subject: [PATCH 082/123] Add filepath + line/column to terminal error

---
 .../gatsby-cli/src/structured-errors/error-map.ts  | 14 ++++++++++----
 packages/gatsby/src/utils/develop-html-route.ts    |  2 ++
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/packages/gatsby-cli/src/structured-errors/error-map.ts b/packages/gatsby-cli/src/structured-errors/error-map.ts
index 05eb83d11e546..2abca4348ad20 100644
--- a/packages/gatsby-cli/src/structured-errors/error-map.ts
+++ b/packages/gatsby-cli/src/structured-errors/error-map.ts
@@ -537,10 +537,16 @@ const errors = {
     docsUrl: `https://www.gatsbyjs.org/docs/gatsby-cli/#new`,
   },
   "11614": {
-    text: ({ path, filePath }): string =>
-      `The path "${path}" errored during SSR. Edit its component ${filePath} to
-    resolve the error. See this docs page for more information
-    https://gatsby.dev/debug-html`,
+    text: ({
+      path,
+      filePath,
+      line,
+      column,
+    }): string => `The path "${path}" errored during SSR.
+
+    Edit its component ${filePath}:${line}:${column} to resolve the error.
+
+    See this docs page for more information https://gatsby.dev/debug-html`,
     level: Level.WARNING,
     docsUrl: `https://gatsby.dev/debug-html`,
   },
diff --git a/packages/gatsby/src/utils/develop-html-route.ts b/packages/gatsby/src/utils/develop-html-route.ts
index c58a94381ac76..380ea0180d21d 100644
--- a/packages/gatsby/src/utils/develop-html-route.ts
+++ b/packages/gatsby/src/utils/develop-html-route.ts
@@ -47,6 +47,8 @@ export const route = ({ app, program, store }): any =>
         context: {
           path: pathObj.path,
           filePath: error.filename,
+          line: error.line,
+          column: error.row,
         },
       })
       res.status(500).send(`<title>Develop SSR Error</title><h1>Error<h1>

From ebfa18d80090d0b677b96334808d7e6024b11237 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Mon, 26 Oct 2020 19:40:03 -0700
Subject: [PATCH 083/123] try try again

---
 packages/gatsby/src/utils/render-dev-html.ts | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/packages/gatsby/src/utils/render-dev-html.ts b/packages/gatsby/src/utils/render-dev-html.ts
index 7d077decd201a..33bf47c8044e1 100644
--- a/packages/gatsby/src/utils/render-dev-html.ts
+++ b/packages/gatsby/src/utils/render-dev-html.ts
@@ -1,14 +1,11 @@
 import JestWorker from "jest-worker"
 
 const startWorker = (): any => {
-  const newWorker = new JestWorker(
-    require.resolve(`./render-dev-html-child.js`),
-    {
-      exposedMethods: [`renderHTML`],
-      numWorkers: 1,
-      enableWorkerThreads: true,
-    }
-  )
+  const newWorker = new JestWorker(require.resolve(`./render-dev-html-child`), {
+    exposedMethods: [`renderHTML`],
+    numWorkers: 1,
+    enableWorkerThreads: true,
+  })
 
   // jest-worker is lazy with forking but we want to fork immediately so the user
   // doesn't have to wait.

From 0df848387dad4e0d8ee83921431625043262cc65 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Mon, 26 Oct 2020 19:59:29 -0700
Subject: [PATCH 084/123] Fixes hopefully

---
 .../npm/__snapshots__/package-json.test.js.snap      | 12 ++++++------
 .../providers/npm/fixtures/package-json/package.json |  2 +-
 packages/gatsby/src/utils/render-dev-html-child.ts   | 10 +++++-----
 3 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/packages/gatsby-recipes/src/providers/npm/__snapshots__/package-json.test.js.snap b/packages/gatsby-recipes/src/providers/npm/__snapshots__/package-json.test.js.snap
index 30e1b37d8de15..9f9bc6d5ac66a 100644
--- a/packages/gatsby-recipes/src/providers/npm/__snapshots__/package-json.test.js.snap
+++ b/packages/gatsby-recipes/src/providers/npm/__snapshots__/package-json.test.js.snap
@@ -12,7 +12,7 @@ Object {
 exports[`packageJson resource e2e package resource test: PackageJson create plan 1`] = `
 Object {
   "currentState": "{
-  \\"name\\": \\"test\\",
+  \\"name\\": \\"test-npm-provider\\",
   \\"scripts\\": {}
 }",
   "describe": "Add husky to package.json",
@@ -23,13 +23,13 @@ Object {
 +   \\"husky\\": \\"{
 +   /\\"hooks/\\": {}
 + }\\",
-    \\"name\\": \\"test\\",
+    \\"name\\": \\"test-npm-provider\\",
     \\"scripts\\": Object {},
   }",
   "id": "husky",
   "name": "husky",
   "newState": "{
-  \\"name\\": \\"test\\",
+  \\"name\\": \\"test-npm-provider\\",
   \\"scripts\\": {},
   \\"husky\\": \\"{/n  /\\"hooks/\\": {}/n}\\"
 }",
@@ -50,7 +50,7 @@ Object {
 exports[`packageJson resource e2e package resource test: PackageJson update plan 1`] = `
 Object {
   "currentState": "{
-  \\"name\\": \\"test\\",
+  \\"name\\": \\"test-npm-provider\\",
   \\"scripts\\": {},
   \\"husky\\": \\"{/n  /\\"hooks/\\": {}/n}\\"
 }",
@@ -66,12 +66,12 @@ Object {
 +     /\\"pre-commit/\\": /\\"lint-staged/\\"
 +   }
   }\\",
-    \\"name\\": \\"test\\",
+    \\"name\\": \\"test-npm-provider\\",
     \\"scripts\\": Object {},",
   "id": "husky",
   "name": "husky",
   "newState": "{
-  \\"name\\": \\"test\\",
+  \\"name\\": \\"test-npm-provider\\",
   \\"scripts\\": {},
   \\"husky\\": \\"{/n  /\\"hooks/\\": {/n    /\\"pre-commit/\\": /\\"lint-staged/\\"/n  }/n}\\"
 }",
diff --git a/packages/gatsby-recipes/src/providers/npm/fixtures/package-json/package.json b/packages/gatsby-recipes/src/providers/npm/fixtures/package-json/package.json
index 7e530d8e6b0bb..7e184820f5c19 100644
--- a/packages/gatsby-recipes/src/providers/npm/fixtures/package-json/package.json
+++ b/packages/gatsby-recipes/src/providers/npm/fixtures/package-json/package.json
@@ -1,4 +1,4 @@
 {
   "name": "test-npm-provider",
   "scripts": {}
-}
+}
\ No newline at end of file
diff --git a/packages/gatsby/src/utils/render-dev-html-child.ts b/packages/gatsby/src/utils/render-dev-html-child.ts
index 084f10f89361f..3ebf865ff15d8 100644
--- a/packages/gatsby/src/utils/render-dev-html-child.ts
+++ b/packages/gatsby/src/utils/render-dev-html-child.ts
@@ -1,8 +1,8 @@
-import { codeFrameColumns } from "@babel/code-frame"
-import ansiHTML from "ansi-html"
-import fs from "fs-extra"
-import sysPath from "path"
-import report from "gatsby-cli/lib/reporter"
+const { codeFrameColumns } = require(`@babel/code-frame`)
+const ansiHTML = require(`ansi-html`)
+const fs = require(`fs-extra`)
+const sysPath = require(`path`)
+const report = require(`gatsby-cli/lib/reporter`)
 
 interface IErrorPosition {
   filename: string

From 1231fde9f117788b62f148fb46470c255b3e4e74 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Mon, 26 Oct 2020 20:41:41 -0700
Subject: [PATCH 085/123] =?UTF-8?q?typescript=20=F0=9F=98=B1?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 packages/gatsby/src/commands/build-html.ts    |  2 +-
 .../utils/{ => dev-ssr}/develop-html-route.ts |  4 +--
 .../render-dev-html-child.js}                 | 35 +++++--------------
 .../utils/{ => dev-ssr}/render-dev-html.ts    |  0
 packages/gatsby/src/utils/start-server.ts     |  4 +--
 5 files changed, 14 insertions(+), 31 deletions(-)
 rename packages/gatsby/src/utils/{ => dev-ssr}/develop-html-route.ts (94%)
 rename packages/gatsby/src/utils/{render-dev-html-child.ts => dev-ssr/render-dev-html-child.js} (85%)
 rename packages/gatsby/src/utils/{ => dev-ssr}/render-dev-html.ts (100%)

diff --git a/packages/gatsby/src/commands/build-html.ts b/packages/gatsby/src/commands/build-html.ts
index 69fe712f2a865..cb25bcc0e34c5 100644
--- a/packages/gatsby/src/commands/build-html.ts
+++ b/packages/gatsby/src/commands/build-html.ts
@@ -7,7 +7,7 @@ import { chunk } from "lodash"
 import webpack from "webpack"
 
 import webpackConfig from "../utils/webpack.config"
-import { restartWorker } from "../utils/render-dev-html"
+import { restartWorker } from "../utils/dev-ssr/render-dev-html"
 import { structureWebpackErrors } from "../utils/webpack-error-utils"
 
 import { IProgram, Stage } from "./types"
diff --git a/packages/gatsby/src/utils/develop-html-route.ts b/packages/gatsby/src/utils/dev-ssr/develop-html-route.ts
similarity index 94%
rename from packages/gatsby/src/utils/develop-html-route.ts
rename to packages/gatsby/src/utils/dev-ssr/develop-html-route.ts
index 380ea0180d21d..b720d6aea4eda 100644
--- a/packages/gatsby/src/utils/develop-html-route.ts
+++ b/packages/gatsby/src/utils/dev-ssr/develop-html-route.ts
@@ -1,9 +1,9 @@
 import report from "gatsby-cli/lib/reporter"
 import { trackCli } from "gatsby-telemetry"
 
-import { findPageByPath } from "./find-page-by-path"
+import { findPageByPath } from "../find-page-by-path"
 import { renderDevHTML } from "./render-dev-html"
-import { isWebpackStatusPending } from "./webpack-status"
+import { isWebpackStatusPending } from "../webpack-status"
 
 export const route = ({ app, program, store }): any =>
   // Render an HTML page and serve it.
diff --git a/packages/gatsby/src/utils/render-dev-html-child.ts b/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js
similarity index 85%
rename from packages/gatsby/src/utils/render-dev-html-child.ts
rename to packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js
index 3ebf865ff15d8..66d959885a039 100644
--- a/packages/gatsby/src/utils/render-dev-html-child.ts
+++ b/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js
@@ -1,16 +1,10 @@
-const { codeFrameColumns } = require(`@babel/code-frame`)
-const ansiHTML = require(`ansi-html`)
-const fs = require(`fs-extra`)
-const sysPath = require(`path`)
-const report = require(`gatsby-cli/lib/reporter`)
+const { codeFrameColumns } = require("@babel/code-frame")
+const ansiHTML = require("ansi-html")
+const fs = require("fs-extra")
+const sysPath = require("path")
+const report = require("gatsby-cli/lib/reporter")
 
-interface IErrorPosition {
-  filename: string
-  line: number
-  row: number
-}
-
-const getPosition = function (stackObject): IErrorPosition {
+const getPosition = function (stackObject) {
   let filename
   let line
   let row
@@ -56,19 +50,8 @@ const colors = {
   yellow: `DB3A00`,
 }
 
-interface IParsedError {
-  filename: string
-  code: string
-  codeFrame: string
-  line: number
-  row: number
-  message: string
-  type: string
-  stack: [string]
-}
-
 // Code borrowed and modified from https://github.com/watilde/parse-error
-export const parseError = function (err, directory): IParsedError {
+exports.parseError = function (err, directory) {
   const stack = err.stack ? err.stack : ``
   const stackObject = stack.split(`\n`)
   const position = getPosition(stackObject)
@@ -127,12 +110,12 @@ export const parseError = function (err, directory): IParsedError {
   return data
 }
 
-export function renderHTML({
+exports.renderHTML = ({
   path,
   htmlComponentRendererPath,
   directory,
   warming = false,
-}): Promise<string | object> {
+}) => {
   if (warming) return null
   return new Promise((resolve, reject) => {
     console.log(`inside worker`, { path, htmlComponentRendererPath, directory })
diff --git a/packages/gatsby/src/utils/render-dev-html.ts b/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
similarity index 100%
rename from packages/gatsby/src/utils/render-dev-html.ts
rename to packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
diff --git a/packages/gatsby/src/utils/start-server.ts b/packages/gatsby/src/utils/start-server.ts
index 7dc2c44683de6..87ef9b489c030 100644
--- a/packages/gatsby/src/utils/start-server.ts
+++ b/packages/gatsby/src/utils/start-server.ts
@@ -17,11 +17,11 @@ import telemetry from "gatsby-telemetry"
 import webpackConfig from "../utils/webpack.config"
 import { store, emitter } from "../redux"
 import { buildRenderer } from "../commands/build-html"
-import { renderDevHTML } from "./render-dev-html"
+import { renderDevHTML } from "./dev-ssr/render-dev-html"
 import report from "gatsby-cli/lib/reporter"
 import launchEditor from "react-dev-utils/launchEditor"
 import * as WorkerPool from "../utils/worker/pool"
-import { route as developHtmlRoute } from "./develop-html-route"
+import { route as developHtmlRoute } from "./dev-ssr/develop-html-route"
 
 import { developStatic } from "../commands/develop-static"
 import withResolverContext from "../schema/context"

From 4994599dba89a14341e8d2f4f76e87eb700937f8 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Mon, 26 Oct 2020 20:46:01 -0700
Subject: [PATCH 086/123] lint

---
 .../src/utils/__tests__/develop-html-route.ts      |  2 +-
 .../src/utils/dev-ssr/render-dev-html-child.js     | 14 ++++++++------
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/packages/gatsby/src/utils/__tests__/develop-html-route.ts b/packages/gatsby/src/utils/__tests__/develop-html-route.ts
index 1b1395c64973c..04146c8cc31f3 100644
--- a/packages/gatsby/src/utils/__tests__/develop-html-route.ts
+++ b/packages/gatsby/src/utils/__tests__/develop-html-route.ts
@@ -1,4 +1,4 @@
-import { parseError } from "../render-dev-html-child"
+import { parseError } from "../dev-ssr/render-dev-html-child"
 import error from "./fixtures/error-object"
 
 describe(`error parsing`, () => {
diff --git a/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js b/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js
index 66d959885a039..83b7ba9c4cca5 100644
--- a/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js
+++ b/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js
@@ -1,8 +1,8 @@
-const { codeFrameColumns } = require("@babel/code-frame")
-const ansiHTML = require("ansi-html")
-const fs = require("fs-extra")
-const sysPath = require("path")
-const report = require("gatsby-cli/lib/reporter")
+const { codeFrameColumns } = require(`@babel/code-frame`)
+const ansiHTML = require(`ansi-html`)
+const fs = require(`fs-extra`)
+const sysPath = require(`path`)
+const report = require(`gatsby-cli/lib/reporter`)
 
 const getPosition = function (stackObject) {
   let filename
@@ -51,7 +51,7 @@ const colors = {
 }
 
 // Code borrowed and modified from https://github.com/watilde/parse-error
-exports.parseError = function (err, directory) {
+const parseError = function (err, directory) {
   const stack = err.stack ? err.stack : ``
   const stackObject = stack.split(`\n`)
   const position = getPosition(stackObject)
@@ -110,6 +110,8 @@ exports.parseError = function (err, directory) {
   return data
 }
 
+exports.parseError = parseError
+
 exports.renderHTML = ({
   path,
   htmlComponentRendererPath,

From a0cb22cd959800a56e12f60952ff42f14026d354 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Mon, 26 Oct 2020 22:33:30 -0700
Subject: [PATCH 087/123] Try tweaking jest settings

---
 .circleci/config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 5cc237a72ac22..2704cfcb8a2e1 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -67,7 +67,7 @@ aliases:
       - <<: *attach_to_bootstrap
       - run: yarn list react
       - run:
-          command: node --max-old-space-size=2048 ./node_modules/.bin/jest -w 1 --ci
+          command: node --max-old-space-size=2048 ./node_modules/.bin/jest --runInBand --colors --ci
           environment:
             GENERATE_JEST_REPORT: true
             JEST_JUNIT_OUTPUT_DIR: ./test-results/jest-node/

From 822c4829312985189f3d64833e89c9732ce1cd4f Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Mon, 26 Oct 2020 22:54:18 -0700
Subject: [PATCH 088/123] Debuggin

---
 .circleci/config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 2704cfcb8a2e1..2b20c8e91220e 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -67,7 +67,7 @@ aliases:
       - <<: *attach_to_bootstrap
       - run: yarn list react
       - run:
-          command: node --max-old-space-size=2048 ./node_modules/.bin/jest --runInBand --colors --ci
+          command: node --max-old-space-size=2048 ./node_modules/.bin/jest --runInBand --detectOpenHandles --colors --ci
           environment:
             GENERATE_JEST_REPORT: true
             JEST_JUNIT_OUTPUT_DIR: ./test-results/jest-node/

From 325cd4a7f055daed7986b21aa1ea983ff5a45848 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Mon, 26 Oct 2020 23:15:39 -0700
Subject: [PATCH 089/123] use default reporter

---
 .circleci/config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 2b20c8e91220e..00aac18b47502 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -67,7 +67,7 @@ aliases:
       - <<: *attach_to_bootstrap
       - run: yarn list react
       - run:
-          command: node --max-old-space-size=2048 ./node_modules/.bin/jest --runInBand --detectOpenHandles --colors --ci
+          command: node --max-old-space-size=2048 ./node_modules/.bin/jest --runInBand --detectOpenHandles --colors --ci --reporters="default"
           environment:
             GENERATE_JEST_REPORT: true
             JEST_JUNIT_OUTPUT_DIR: ./test-results/jest-node/

From 421e99345e1f5d733646949bdf53ac2cf12a4042 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Mon, 26 Oct 2020 23:54:48 -0700
Subject: [PATCH 090/123] explicitly init dev html worker pool so it doesn't
 start during tests

---
 packages/gatsby/src/utils/dev-ssr/render-dev-html.ts | 5 ++++-
 packages/gatsby/src/utils/start-server.ts            | 4 +++-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts b/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
index 33bf47c8044e1..7ad079d6bd21f 100644
--- a/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
+++ b/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
@@ -17,7 +17,10 @@ const startWorker = (): any => {
   return newWorker
 }
 
-let worker = startWorker()
+let worker
+export const initDevWorkerPool = (): void => {
+  worker = startWorker()
+}
 
 export const restartWorker = (): void => {
   const oldWorker = worker
diff --git a/packages/gatsby/src/utils/start-server.ts b/packages/gatsby/src/utils/start-server.ts
index 87ef9b489c030..6bb870a549e7e 100644
--- a/packages/gatsby/src/utils/start-server.ts
+++ b/packages/gatsby/src/utils/start-server.ts
@@ -17,7 +17,7 @@ import telemetry from "gatsby-telemetry"
 import webpackConfig from "../utils/webpack.config"
 import { store, emitter } from "../redux"
 import { buildRenderer } from "../commands/build-html"
-import { renderDevHTML } from "./dev-ssr/render-dev-html"
+import { renderDevHTML, initDevWorkerPool } from "./dev-ssr/render-dev-html"
 import report from "gatsby-cli/lib/reporter"
 import launchEditor from "react-dev-utils/launchEditor"
 import * as WorkerPool from "../utils/worker/pool"
@@ -69,6 +69,8 @@ export async function startServer(
   })
   webpackActivity.start()
 
+  initDevWorkerPool()
+
   const devConfig = await webpackConfig(
     program,
     directory,

From 85db9ec08ab4a5cb0feb82fc57f137b76c5b9371 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Tue, 27 Oct 2020 00:14:39 -0700
Subject: [PATCH 091/123] restore original ci settings

---
 .circleci/config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 00aac18b47502..5cc237a72ac22 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -67,7 +67,7 @@ aliases:
       - <<: *attach_to_bootstrap
       - run: yarn list react
       - run:
-          command: node --max-old-space-size=2048 ./node_modules/.bin/jest --runInBand --detectOpenHandles --colors --ci --reporters="default"
+          command: node --max-old-space-size=2048 ./node_modules/.bin/jest -w 1 --ci
           environment:
             GENERATE_JEST_REPORT: true
             JEST_JUNIT_OUTPUT_DIR: ./test-results/jest-node/

From 95a85b9e76b44f8606facce6fda2534590f793fa Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Tue, 27 Oct 2020 00:15:43 -0700
Subject: [PATCH 092/123] sup

---
 packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js b/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js
index 83b7ba9c4cca5..eec5a3c06b3c0 100644
--- a/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js
+++ b/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js
@@ -118,7 +118,9 @@ exports.renderHTML = ({
   directory,
   warming = false,
 }) => {
-  if (warming) return null
+  if (warming) {
+    return `warmed up`
+  }
   return new Promise((resolve, reject) => {
     console.log(`inside worker`, { path, htmlComponentRendererPath, directory })
     const htmlComponentRenderer = require(htmlComponentRendererPath)

From 4c2b1708ba39ae3a68d99aec9bd70bed04623336 Mon Sep 17 00:00:00 2001
From: Sidhartha Chatterjee <me@sidharthachatterjee.com>
Date: Tue, 27 Oct 2020 15:16:09 +0530
Subject: [PATCH 093/123] Update
 packages/gatsby-cli/src/structured-errors/error-map.ts

Co-authored-by: Lennart <lekoarts@gmail.com>
---
 packages/gatsby-cli/src/structured-errors/error-map.ts | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/packages/gatsby-cli/src/structured-errors/error-map.ts b/packages/gatsby-cli/src/structured-errors/error-map.ts
index 2abca4348ad20..b8b6474a5aa83 100644
--- a/packages/gatsby-cli/src/structured-errors/error-map.ts
+++ b/packages/gatsby-cli/src/structured-errors/error-map.ts
@@ -544,9 +544,7 @@ const errors = {
       column,
     }): string => `The path "${path}" errored during SSR.
 
-    Edit its component ${filePath}:${line}:${column} to resolve the error.
-
-    See this docs page for more information https://gatsby.dev/debug-html`,
+    Edit its component ${filePath}:${line}:${column} to resolve the error.`,
     level: Level.WARNING,
     docsUrl: `https://gatsby.dev/debug-html`,
   },

From 45568e4b4a66ce5a04f7159802f99d84806662e6 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Tue, 27 Oct 2020 10:46:00 -0700
Subject: [PATCH 094/123] console.logs seem to break jest-worker on CI

---
 packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js b/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js
index eec5a3c06b3c0..0c31e31fd6a67 100644
--- a/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js
+++ b/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js
@@ -122,13 +122,10 @@ exports.renderHTML = ({
     return `warmed up`
   }
   return new Promise((resolve, reject) => {
-    console.log(`inside worker`, { path, htmlComponentRendererPath, directory })
     const htmlComponentRenderer = require(htmlComponentRendererPath)
     try {
-      console.time(`SSR`)
       htmlComponentRenderer.default(path, (_throwAway, htmlString) => {
         resolve(htmlString)
-        console.timeEnd(`SSR`)
       })
     } catch (err) {
       const stack = err.stack ? err.stack : ``

From f2a862c7fc5603a26682d4cfddf3dfeb488012de Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Tue, 27 Oct 2020 11:47:05 -0700
Subject: [PATCH 095/123] Increase pageLoadTimeout

---
 e2e-tests/development-runtime/cypress.json | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/e2e-tests/development-runtime/cypress.json b/e2e-tests/development-runtime/cypress.json
index 48fdc722d5837..e5f461955b27a 100644
--- a/e2e-tests/development-runtime/cypress.json
+++ b/e2e-tests/development-runtime/cypress.json
@@ -1,5 +1,6 @@
 {
   "baseUrl": "http://localhost:8000",
   "failOnStatusCode": false,
-  "chromeWebSecurity": false
+  "chromeWebSecurity": false,
+  "pageLoadTimeout": 10000
 }

From 89da3acb5ddf0c4cdb111789407fcd2380497696 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Tue, 27 Oct 2020 12:26:03 -0700
Subject: [PATCH 096/123] try taskTimeout

---
 e2e-tests/development-runtime/cypress.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/e2e-tests/development-runtime/cypress.json b/e2e-tests/development-runtime/cypress.json
index e5f461955b27a..2b5f349e2eb29 100644
--- a/e2e-tests/development-runtime/cypress.json
+++ b/e2e-tests/development-runtime/cypress.json
@@ -2,5 +2,5 @@
   "baseUrl": "http://localhost:8000",
   "failOnStatusCode": false,
   "chromeWebSecurity": false,
-  "pageLoadTimeout": 10000
+  "taskTimeout": 10000
 }

From e780c5d929c7132a15ebc9d4c36c80dde9992eda Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Tue, 27 Oct 2020 12:31:11 -0700
Subject: [PATCH 097/123] This might be confusing cypress

---
 .../gatsby/src/utils/dev-ssr/develop-html-route.ts | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/packages/gatsby/src/utils/dev-ssr/develop-html-route.ts b/packages/gatsby/src/utils/dev-ssr/develop-html-route.ts
index b720d6aea4eda..22d34de4c4b59 100644
--- a/packages/gatsby/src/utils/dev-ssr/develop-html-route.ts
+++ b/packages/gatsby/src/utils/dev-ssr/develop-html-route.ts
@@ -8,13 +8,13 @@ import { isWebpackStatusPending } from "../webpack-status"
 export const route = ({ app, program, store }): any =>
   // Render an HTML page and serve it.
   app.get(`*`, async (req, res, next) => {
-    if (isWebpackStatusPending()) {
-      res
-        .status(202)
-        .send(
-          `webpack isn't yet finished compiling code. Try refreshing once it's done.`
-        )
-    }
+    // if (isWebpackStatusPending()) {
+    // res
+    // .status(202)
+    // .send(
+    // `webpack isn't yet finished compiling code. Try refreshing once it's done.`
+    // )
+    // }
 
     trackCli(`GATSBY_EXPERIMENTAL_DEV_SSR`)
 

From c7918b8043e5178f17ca3bc7a80ab907485efca3 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Tue, 27 Oct 2020 14:54:08 -0700
Subject: [PATCH 098/123] Don't re-spawn the worker process on every change as
 that's very expensive. Just delete the module cache for 25 edits before
 re-spawning

---
 packages/gatsby/src/commands/build-html.ts    | 10 +++++---
 .../src/utils/dev-ssr/develop-html-route.ts   | 14 +++++------
 .../utils/dev-ssr/render-dev-html-child.js    |  4 ++++
 .../src/utils/dev-ssr/render-dev-html.ts      | 23 ++++++++++++++-----
 4 files changed, 35 insertions(+), 16 deletions(-)

diff --git a/packages/gatsby/src/commands/build-html.ts b/packages/gatsby/src/commands/build-html.ts
index cb25bcc0e34c5..c35c859712187 100644
--- a/packages/gatsby/src/commands/build-html.ts
+++ b/packages/gatsby/src/commands/build-html.ts
@@ -17,7 +17,11 @@ type IWorkerPool = any // TODO
 
 let oldHash = ``
 let newHash = ``
-const runWebpack = (compilerConfig, stage: Stage): Bluebird<webpack.Stats> =>
+const runWebpack = (
+  compilerConfig,
+  stage: Stage,
+  directory
+): Bluebird<webpack.Stats> =>
   new Bluebird((resolve, reject) => {
     if (stage === `build-html`) {
       webpack(compilerConfig).run((err, stats) => {
@@ -40,7 +44,7 @@ const runWebpack = (compilerConfig, stage: Stage): Bluebird<webpack.Stats> =>
 
             // Make sure we use the latest version during development
             if (oldHash !== `` && newHash !== oldHash) {
-              restartWorker()
+              restartWorker(`${directory}/public/render-page.js`)
             }
 
             oldHash = newHash
@@ -57,7 +61,7 @@ const doBuildRenderer = async (
   webpackConfig: webpack.Configuration,
   stage: Stage
 ): Promise<string> => {
-  const stats = await runWebpack(webpackConfig, stage)
+  const stats = await runWebpack(webpackConfig, stage, directory)
   if (stats.hasErrors()) {
     reporter.panic(structureWebpackErrors(stage, stats.compilation.errors))
   }
diff --git a/packages/gatsby/src/utils/dev-ssr/develop-html-route.ts b/packages/gatsby/src/utils/dev-ssr/develop-html-route.ts
index 22d34de4c4b59..b720d6aea4eda 100644
--- a/packages/gatsby/src/utils/dev-ssr/develop-html-route.ts
+++ b/packages/gatsby/src/utils/dev-ssr/develop-html-route.ts
@@ -8,13 +8,13 @@ import { isWebpackStatusPending } from "../webpack-status"
 export const route = ({ app, program, store }): any =>
   // Render an HTML page and serve it.
   app.get(`*`, async (req, res, next) => {
-    // if (isWebpackStatusPending()) {
-    // res
-    // .status(202)
-    // .send(
-    // `webpack isn't yet finished compiling code. Try refreshing once it's done.`
-    // )
-    // }
+    if (isWebpackStatusPending()) {
+      res
+        .status(202)
+        .send(
+          `webpack isn't yet finished compiling code. Try refreshing once it's done.`
+        )
+    }
 
     trackCli(`GATSBY_EXPERIMENTAL_DEV_SSR`)
 
diff --git a/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js b/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js
index 0c31e31fd6a67..72220eae26a11 100644
--- a/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js
+++ b/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js
@@ -143,3 +143,7 @@ exports.renderHTML = ({
     }
   })
 }
+
+exports.deleteModuleCache = htmlComponentRendererPath => {
+  delete require.cache[require.resolve(htmlComponentRendererPath)]
+}
diff --git a/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts b/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
index 7ad079d6bd21f..da20d9d03946d 100644
--- a/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
+++ b/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
@@ -2,7 +2,7 @@ import JestWorker from "jest-worker"
 
 const startWorker = (): any => {
   const newWorker = new JestWorker(require.resolve(`./render-dev-html-child`), {
-    exposedMethods: [`renderHTML`],
+    exposedMethods: [`renderHTML`, `deleteModuleCache`],
     numWorkers: 1,
     enableWorkerThreads: true,
   })
@@ -22,11 +22,22 @@ export const initDevWorkerPool = (): void => {
   worker = startWorker()
 }
 
-export const restartWorker = (): void => {
-  const oldWorker = worker
-  const newWorker = startWorker()
-  worker = newWorker
-  oldWorker.end()
+let changeCount = 0
+export const restartWorker = (htmlComponentRendererPath): void => {
+  changeCount += 1
+  // Forking is expensive — each time we re-require the outputted webpack
+  // file, memory grows ~10 mb — 25 regenerations means ~250mb which seems
+  // like an accepatable amount of memory to grow before we reclaim it
+  // by rebooting the worker process.
+  if (changeCount > 25) {
+    const oldWorker = worker
+    const newWorker = startWorker()
+    worker = newWorker
+    oldWorker.end()
+    changeCount = 0
+  } else {
+    worker.deleteModuleCache(htmlComponentRendererPath)
+  }
 }
 
 export const renderDevHTML = ({

From fd51701b6a2077c71b91f7823646ff310b99fc39 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Tue, 27 Oct 2020 15:31:09 -0700
Subject: [PATCH 099/123] cleanups

---
 e2e-tests/development-runtime/cypress.json      |  3 +--
 .../src/utils/dev-ssr/render-dev-html-child.js  | 17 +++++------------
 .../gatsby/src/utils/dev-ssr/render-dev-html.ts |  6 ++----
 3 files changed, 8 insertions(+), 18 deletions(-)

diff --git a/e2e-tests/development-runtime/cypress.json b/e2e-tests/development-runtime/cypress.json
index 2b5f349e2eb29..48fdc722d5837 100644
--- a/e2e-tests/development-runtime/cypress.json
+++ b/e2e-tests/development-runtime/cypress.json
@@ -1,6 +1,5 @@
 {
   "baseUrl": "http://localhost:8000",
   "failOnStatusCode": false,
-  "chromeWebSecurity": false,
-  "taskTimeout": 10000
+  "chromeWebSecurity": false
 }
diff --git a/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js b/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js
index 72220eae26a11..cba064cd5df95 100644
--- a/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js
+++ b/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js
@@ -74,7 +74,7 @@ const parseError = function (err, directory) {
   const line = position.line
   const row = position.row
   ansiHTML.setColors({
-    reset: [colors.text, colors.background], // FOREGROUND-COLOR or [FOREGROUND-COLOR] or [, BACKGROUND-COLOR] or [FOREGROUND-COLOR, BACKGROUND-COLOR]
+    reset: [colors.text, colors.background], // [FOREGROUND-COLOR, BACKGROUND-COLOR]
     black: `aaa`, // String
     red: colors.keyword,
     green: colors.green,
@@ -112,16 +112,8 @@ const parseError = function (err, directory) {
 
 exports.parseError = parseError
 
-exports.renderHTML = ({
-  path,
-  htmlComponentRendererPath,
-  directory,
-  warming = false,
-}) => {
-  if (warming) {
-    return `warmed up`
-  }
-  return new Promise((resolve, reject) => {
+exports.renderHTML = ({ path, htmlComponentRendererPath, directory }) =>
+  new Promise((resolve, reject) => {
     const htmlComponentRenderer = require(htmlComponentRendererPath)
     try {
       htmlComponentRenderer.default(path, (_throwAway, htmlString) => {
@@ -142,8 +134,9 @@ exports.renderHTML = ({
       // return { error }
     }
   })
-}
 
 exports.deleteModuleCache = htmlComponentRendererPath => {
   delete require.cache[require.resolve(htmlComponentRendererPath)]
 }
+
+exports.warmup = () => `warmed`
diff --git a/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts b/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
index da20d9d03946d..a638197a93de2 100644
--- a/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
+++ b/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
@@ -2,7 +2,7 @@ import JestWorker from "jest-worker"
 
 const startWorker = (): any => {
   const newWorker = new JestWorker(require.resolve(`./render-dev-html-child`), {
-    exposedMethods: [`renderHTML`, `deleteModuleCache`],
+    exposedMethods: [`renderHTML`, `deleteModuleCache`, `warmup`],
     numWorkers: 1,
     enableWorkerThreads: true,
   })
@@ -10,9 +10,7 @@ const startWorker = (): any => {
   // jest-worker is lazy with forking but we want to fork immediately so the user
   // doesn't have to wait.
   // @ts-ignore
-  newWorker.renderHTML({
-    warming: true,
-  })
+  newWorker.warmup()
 
   return newWorker
 }

From 2693163b572a08f9cf191fc45f2c807157ab85f2 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Thu, 29 Oct 2020 08:02:35 -0700
Subject: [PATCH 100/123] Update
 packages/gatsby/cache-dir/develop-static-entry.js

Co-authored-by: Ward Peeters <ward@coding-tech.com>
---
 packages/gatsby/cache-dir/develop-static-entry.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/gatsby/cache-dir/develop-static-entry.js b/packages/gatsby/cache-dir/develop-static-entry.js
index 0c9e3475c81a6..c983467dee6dd 100644
--- a/packages/gatsby/cache-dir/develop-static-entry.js
+++ b/packages/gatsby/cache-dir/develop-static-entry.js
@@ -5,7 +5,7 @@ import { merge } from "lodash"
 import { join } from "path"
 import apiRunner from "./api-runner-ssr"
 import { grabMatchParams } from "./find-path"
-const syncRequires = require(`$virtual/sync-requires`)
+import syncRequires from "$virtual/sync-requires"
 
 import { RouteAnnouncerProps } from "./route-announcer-props"
 import { ServerLocation, Router, isRedirect } from "@reach/router"

From 89b51b8eb950413742391d3974db3953a2f6f125 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Thu, 29 Oct 2020 08:19:23 -0700
Subject: [PATCH 101/123] Cleanups suggested by @wardpeet

---
 packages/gatsby/src/utils/dev-ssr/develop-html-route.ts | 8 --------
 packages/gatsby/src/utils/start-server.ts               | 2 +-
 2 files changed, 1 insertion(+), 9 deletions(-)

diff --git a/packages/gatsby/src/utils/dev-ssr/develop-html-route.ts b/packages/gatsby/src/utils/dev-ssr/develop-html-route.ts
index b720d6aea4eda..92d4c241e72b0 100644
--- a/packages/gatsby/src/utils/dev-ssr/develop-html-route.ts
+++ b/packages/gatsby/src/utils/dev-ssr/develop-html-route.ts
@@ -8,14 +8,6 @@ import { isWebpackStatusPending } from "../webpack-status"
 export const route = ({ app, program, store }): any =>
   // Render an HTML page and serve it.
   app.get(`*`, async (req, res, next) => {
-    if (isWebpackStatusPending()) {
-      res
-        .status(202)
-        .send(
-          `webpack isn't yet finished compiling code. Try refreshing once it's done.`
-        )
-    }
-
     trackCli(`GATSBY_EXPERIMENTAL_DEV_SSR`)
 
     const pathObj = findPageByPath(store.getState(), req.path)
diff --git a/packages/gatsby/src/utils/start-server.ts b/packages/gatsby/src/utils/start-server.ts
index 6bb870a549e7e..a4ae54b8f608c 100644
--- a/packages/gatsby/src/utils/start-server.ts
+++ b/packages/gatsby/src/utils/start-server.ts
@@ -276,7 +276,7 @@ export async function startServer(
         })
         res.status(200).send(renderResponse)
       } catch (e) {
-        console.log(e)
+        report.panic(e)
         res.send(e).status(500)
       }
     }

From 55e44dcdc7664ba03b66a66698efe40d62c05813 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Thu, 29 Oct 2020 08:49:12 -0700
Subject: [PATCH 102/123] fix lint

---
 packages/gatsby/src/utils/dev-ssr/develop-html-route.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/gatsby/src/utils/dev-ssr/develop-html-route.ts b/packages/gatsby/src/utils/dev-ssr/develop-html-route.ts
index 92d4c241e72b0..50fec4ddadb63 100644
--- a/packages/gatsby/src/utils/dev-ssr/develop-html-route.ts
+++ b/packages/gatsby/src/utils/dev-ssr/develop-html-route.ts
@@ -3,7 +3,6 @@ import { trackCli } from "gatsby-telemetry"
 
 import { findPageByPath } from "../find-page-by-path"
 import { renderDevHTML } from "./render-dev-html"
-import { isWebpackStatusPending } from "../webpack-status"
 
 export const route = ({ app, program, store }): any =>
   // Render an HTML page and serve it.

From 850fe070fe2e563d77d087e23831b7ce2f0d6345 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Thu, 29 Oct 2020 09:41:23 -0700
Subject: [PATCH 103/123] Lazily compile page components

This makes the initial creation of the dev ssr bundle ~85% faster.
---
 .../gatsby/cache-dir/develop-static-entry.js  |  4 +-
 .../src/utils/dev-ssr/render-dev-html.ts      | 67 +++++++++++++++++++
 2 files changed, 69 insertions(+), 2 deletions(-)

diff --git a/packages/gatsby/cache-dir/develop-static-entry.js b/packages/gatsby/cache-dir/develop-static-entry.js
index c983467dee6dd..2763218d55c20 100644
--- a/packages/gatsby/cache-dir/develop-static-entry.js
+++ b/packages/gatsby/cache-dir/develop-static-entry.js
@@ -5,7 +5,7 @@ import { merge } from "lodash"
 import { join } from "path"
 import apiRunner from "./api-runner-ssr"
 import { grabMatchParams } from "./find-path"
-import syncRequires from "$virtual/sync-requires"
+import lazySyncRequires from "$virtual/lazy-sync-requires"
 
 import { RouteAnnouncerProps } from "./route-announcer-props"
 import { ServerLocation, Router, isRedirect } from "@reach/router"
@@ -130,7 +130,7 @@ export default (pagePath, callback) => {
         }
 
         const pageElement = createElement(
-          syncRequires.components[componentChunkName],
+          lazySyncRequires.components[componentChunkName],
           props
         )
 
diff --git a/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts b/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
index a638197a93de2..7cfa00f0a0347 100644
--- a/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
+++ b/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
@@ -1,4 +1,9 @@
 import JestWorker from "jest-worker"
+import fs from "fs-extra"
+import { joinPath } from "gatsby-core-utils"
+
+import { store } from "../../redux"
+import { writeModule } from "../gatsby-webpack-virtual-modules"
 
 const startWorker = (): any => {
   const newWorker = new JestWorker(require.resolve(`./render-dev-html-child`), {
@@ -38,6 +43,65 @@ export const restartWorker = (htmlComponentRendererPath): void => {
   }
 }
 
+interface IGatsbyPageComponent {
+  component: string
+  componentChunkName: string
+}
+
+// TODO: Remove all "hot" references in this `syncRequires` variable when fast-refresh is the default
+const hotImport =
+  process.env.GATSBY_HOT_LOADER !== `fast-refresh`
+    ? `const { hot } = require("react-hot-loader/root")`
+    : ``
+const hotMethod = process.env.GATSBY_HOT_LOADER !== `fast-refresh` ? `hot` : ``
+
+const writeLazyRequires = (pageComponents): void => {
+  // Create file with sync requires of components/json files.
+  let lazySyncRequires = `${hotImport}
+
+// prefer default export if available
+const preferDefault = m => (m && m.default) || m
+\n\n`
+  lazySyncRequires += `exports.components = {\n${[...pageComponents.values()]
+    .map(
+      (c: IGatsbyPageComponent): string =>
+        `  "${
+          c.componentChunkName
+        }": ${hotMethod}(preferDefault(require("${joinPath(c.component)}")))`
+    )
+    .join(`,\n`)}
+}\n\n`
+
+  writeModule(`$virtual/lazy-sync-requires`, lazySyncRequires)
+}
+
+const pageComponents = new Map()
+const ensurePathComponentInSSRBundle = async (path, directory): void => {
+  const pages = [...store.getState().pages.values()]
+  const { component, componentChunkName } = pages.find(p => p.path === path)
+  if (!pageComponents.has(component)) {
+    pageComponents.set(component, { component, componentChunkName })
+    await writeLazyRequires(pageComponents)
+    const htmlComponentRendererPath = joinPath(
+      directory,
+      `public/render-page.js`
+    )
+    await new Promise(async resolve => {
+      const watcher = fs.watch(htmlComponentRendererPath, () => {
+        // It's changed, clean up the watcher.
+        watcher.close()
+        // Make sure the worker is ready.
+        await worker.deleteModuleCache(htmlComponentRendererPath)
+        resolve()
+      })
+    })
+  }
+  // else nothing to do so we return.
+}
+
+// Initialize the virtual module.
+writeLazyRequires(pageComponents)
+
 export const renderDevHTML = ({
   path,
   htmlComponentRendererPath,
@@ -45,6 +109,9 @@ export const renderDevHTML = ({
 }): Promise<string> =>
   new Promise(async (resolve, reject) => {
     try {
+      // Write component to file & wait for public/render-page.js to update
+      await ensurePathComponentInSSRBundle(path, directory)
+
       const response = await worker.renderHTML({
         path,
         htmlComponentRendererPath,

From c5e740d6e1e523e4cfe92c279043e7516604d608 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Thu, 29 Oct 2020 10:00:21 -0700
Subject: [PATCH 104/123] fix typescript

---
 .../src/utils/dev-ssr/render-dev-html.ts      | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts b/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
index 7cfa00f0a0347..d93124dbbe880 100644
--- a/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
+++ b/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
@@ -76,12 +76,18 @@ const preferDefault = m => (m && m.default) || m
 }
 
 const pageComponents = new Map()
-const ensurePathComponentInSSRBundle = async (path, directory): void => {
+const ensurePathComponentInSSRBundle = async (
+  path,
+  directory
+): Promise<void> => {
   const pages = [...store.getState().pages.values()]
-  const { component, componentChunkName } = pages.find(p => p.path === path)
-  if (!pageComponents.has(component)) {
-    pageComponents.set(component, { component, componentChunkName })
-    await writeLazyRequires(pageComponents)
+  const page = pages.find(p => p.path === path)
+  if (page && !pageComponents.has(page.component)) {
+    pageComponents.set(page.component, {
+      component: page.component,
+      componentChunkName: page.componentChunkName,
+    })
+    writeLazyRequires(pageComponents)
     const htmlComponentRendererPath = joinPath(
       directory,
       `public/render-page.js`
@@ -91,8 +97,7 @@ const ensurePathComponentInSSRBundle = async (path, directory): void => {
         // It's changed, clean up the watcher.
         watcher.close()
         // Make sure the worker is ready.
-        await worker.deleteModuleCache(htmlComponentRendererPath)
-        resolve()
+        worker.deleteModuleCache(htmlComponentRendererPath).then(resolve)
       })
     })
   }

From c2a719139e40ce8ef8d31979d8205de1ca885597 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Thu, 29 Oct 2020 10:24:07 -0700
Subject: [PATCH 105/123] mock /lazy-sync-requires

---
 packages/gatsby/cache-dir/__tests__/static-entry.js | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/packages/gatsby/cache-dir/__tests__/static-entry.js b/packages/gatsby/cache-dir/__tests__/static-entry.js
index 0fc3270b64ce8..a547bf71405a0 100644
--- a/packages/gatsby/cache-dir/__tests__/static-entry.js
+++ b/packages/gatsby/cache-dir/__tests__/static-entry.js
@@ -16,6 +16,19 @@ jest.mock(`gatsby/package.json`, () => {
     version: `2.0.0`,
   }
 })
+jest.mock(
+  `$virtual/lazy-sync-requires`,
+  () => {
+    return {
+      components: {
+        "page-component---src-pages-test-js": () => null,
+      },
+    }
+  },
+  {
+    virtual: true,
+  }
+)
 
 jest.mock(
   `$virtual/sync-requires`,

From dfeabe6720e571569210565ca89ee376edd73b94 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Thu, 29 Oct 2020 14:23:25 -0700
Subject: [PATCH 106/123] =?UTF-8?q?The=20lazy=20bundling=20created=20a=20r?=
 =?UTF-8?q?ace=20condition=20where=20two=20pages=20could=20be=20simultaneo?=
 =?UTF-8?q?usly=20requested=20but=20both=20would=20think=20they're=20done?=
 =?UTF-8?q?=20as=20soon=20as=20the=20first=20to=20arrive=20finishes=20?=
 =?UTF-8?q?=E2=80=94=20'suspend'=20rendering=20until=20the=20pageComponent?=
 =?UTF-8?q?=20is=20found=20to=20avoid=20this?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../gatsby/cache-dir/develop-static-entry.js  |  4 ++
 .../utils/dev-ssr/render-dev-html-child.js    | 51 +++++++++++++------
 .../src/utils/dev-ssr/render-dev-html.ts      |  8 ++-
 3 files changed, 46 insertions(+), 17 deletions(-)

diff --git a/packages/gatsby/cache-dir/develop-static-entry.js b/packages/gatsby/cache-dir/develop-static-entry.js
index 2763218d55c20..3a8449d532212 100644
--- a/packages/gatsby/cache-dir/develop-static-entry.js
+++ b/packages/gatsby/cache-dir/develop-static-entry.js
@@ -129,6 +129,10 @@ export default (pagePath, callback) => {
             : undefined,
         }
 
+        if (!lazySyncRequires.components[componentChunkName]) {
+          throw new Error(`try again`)
+        }
+
         const pageElement = createElement(
           lazySyncRequires.components[componentChunkName],
           props
diff --git a/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js b/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js
index cba064cd5df95..57e075d2d1881 100644
--- a/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js
+++ b/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js
@@ -112,26 +112,47 @@ const parseError = function (err, directory) {
 
 exports.parseError = parseError
 
+const callRenderFunction = ({ path, htmlComponentRendererPath }) => {
+  const htmlComponentRenderer = require(htmlComponentRendererPath)
+  return new Promise(resolve => {
+    htmlComponentRenderer.default(path, (_throwAway, htmlString) => {
+      // console.log(`rendered correctly`, { htmlString, attempts })
+      resolve(htmlString)
+    })
+  })
+}
+
 exports.renderHTML = ({ path, htmlComponentRendererPath, directory }) =>
-  new Promise((resolve, reject) => {
-    const htmlComponentRenderer = require(htmlComponentRendererPath)
+  new Promise(async (resolve, reject) => {
     try {
-      htmlComponentRenderer.default(path, (_throwAway, htmlString) => {
-        resolve(htmlString)
+      const htmlString = await callRenderFunction({
+        path,
+        htmlComponentRendererPath,
       })
+      resolve(htmlString)
     } catch (err) {
-      const stack = err.stack ? err.stack : ``
-      // Only generate error pages for webpack errors. If it's not a webpack
-      // error, it's not a user error so probably a system error so we'll just
-      // panic and quit.
-      const regex = /webpack:\/lib\//gm
-      if (!stack.match(regex)) {
-        report.panic(err)
-        return
+      // The bundle might not yet have the page component we need yet
+      // so develop-static-entry.js will throw (kinda like React suspense)
+      // and we keep trying until it's there.
+      if (err.message === `try again`) {
+        setTimeout(() => {
+          exports.deleteModuleCache(htmlComponentRendererPath)
+          exports.renderHTML({ path, htmlComponentRendererPath, directory })
+        }, 100)
+      } else {
+        const stack = err.stack ? err.stack : ``
+        // Only generate error pages for webpack errors. If it's not a webpack
+        // error, it's not a user error so probably a system error so we'll just
+        // panic and quit.
+        const regex = /webpack:\/lib\//gm
+        if (!stack.match(regex)) {
+          console.log(err)
+          return
+        }
+        const error = parseError(err, directory)
+        reject(error)
+        // return { error }
       }
-      const error = parseError(err, directory)
-      reject(error)
-      // return { error }
     }
   })
 
diff --git a/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts b/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
index d93124dbbe880..54962829ce525 100644
--- a/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
+++ b/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
@@ -10,8 +10,12 @@ const startWorker = (): any => {
     exposedMethods: [`renderHTML`, `deleteModuleCache`, `warmup`],
     numWorkers: 1,
     enableWorkerThreads: true,
+    forkOptions: { silent: false },
   })
 
+  newWorker.getStdout().pipe(process.stdout)
+  newWorker.getStderr().pipe(process.stderr)
+
   // jest-worker is lazy with forking but we want to fork immediately so the user
   // doesn't have to wait.
   // @ts-ignore
@@ -117,12 +121,12 @@ export const renderDevHTML = ({
       // Write component to file & wait for public/render-page.js to update
       await ensurePathComponentInSSRBundle(path, directory)
 
-      const response = await worker.renderHTML({
+      const htmlString = await worker.renderHTML({
         path,
         htmlComponentRendererPath,
         directory,
       })
-      resolve(response)
+      resolve(htmlString)
     } catch (error) {
       reject(error)
     }

From 9d9af276ab4ed77342021e4386d44774610fbac0 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Thu, 29 Oct 2020 14:42:56 -0700
Subject: [PATCH 107/123] Add more pages to make sure we're going to hit the
 race condition

---
 integration-tests/ssr/src/pages/hi.js  | 15 +++++++++++++++
 integration-tests/ssr/src/pages/hi2.js | 15 +++++++++++++++
 integration-tests/ssr/src/pages/hi3.js | 15 +++++++++++++++
 3 files changed, 45 insertions(+)
 create mode 100644 integration-tests/ssr/src/pages/hi.js
 create mode 100644 integration-tests/ssr/src/pages/hi2.js
 create mode 100644 integration-tests/ssr/src/pages/hi3.js

diff --git a/integration-tests/ssr/src/pages/hi.js b/integration-tests/ssr/src/pages/hi.js
new file mode 100644
index 0000000000000..5cd1e99c3b450
--- /dev/null
+++ b/integration-tests/ssr/src/pages/hi.js
@@ -0,0 +1,15 @@
+import React from "react"
+import { useStaticQuery, graphql } from "gatsby"
+
+export default function Inline() {
+  const { site } = useStaticQuery(graphql`
+    {
+      site {
+        siteMetadata {
+          title
+        }
+      }
+    }
+  `)
+  return <div>hi2 {site.siteMetadata.title}</div>
+}
diff --git a/integration-tests/ssr/src/pages/hi2.js b/integration-tests/ssr/src/pages/hi2.js
new file mode 100644
index 0000000000000..5cd1e99c3b450
--- /dev/null
+++ b/integration-tests/ssr/src/pages/hi2.js
@@ -0,0 +1,15 @@
+import React from "react"
+import { useStaticQuery, graphql } from "gatsby"
+
+export default function Inline() {
+  const { site } = useStaticQuery(graphql`
+    {
+      site {
+        siteMetadata {
+          title
+        }
+      }
+    }
+  `)
+  return <div>hi2 {site.siteMetadata.title}</div>
+}
diff --git a/integration-tests/ssr/src/pages/hi3.js b/integration-tests/ssr/src/pages/hi3.js
new file mode 100644
index 0000000000000..d8729091dc969
--- /dev/null
+++ b/integration-tests/ssr/src/pages/hi3.js
@@ -0,0 +1,15 @@
+import React from "react"
+import { useStaticQuery, graphql } from "gatsby"
+
+export default function Inline() {
+  const { site } = useStaticQuery(graphql`
+    {
+      site {
+        siteMetadata {
+          title
+        }
+      }
+    }
+  `)
+  return <div> hi3{site.siteMetadata.title}</div>
+}

From e55886bf0744c927a18d0839749dfe6bcd26baa3 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Thu, 29 Oct 2020 16:47:50 -0700
Subject: [PATCH 108/123] Check file directly that the page component has been
 added

This is a lot simpler & more reliable
---
 integration-tests/ssr/src/pages/hi.js         |   2 +-
 .../gatsby/cache-dir/develop-static-entry.js  |   6 +-
 .../utils/dev-ssr/render-dev-html-child.js    |  51 +++------
 .../src/utils/dev-ssr/render-dev-html.ts      | 101 ++++++++++++++----
 4 files changed, 97 insertions(+), 63 deletions(-)

diff --git a/integration-tests/ssr/src/pages/hi.js b/integration-tests/ssr/src/pages/hi.js
index 5cd1e99c3b450..20466045525dd 100644
--- a/integration-tests/ssr/src/pages/hi.js
+++ b/integration-tests/ssr/src/pages/hi.js
@@ -11,5 +11,5 @@ export default function Inline() {
       }
     }
   `)
-  return <div>hi2 {site.siteMetadata.title}</div>
+  return <div>hi1 {site.siteMetadata.title}</div>
 }
diff --git a/packages/gatsby/cache-dir/develop-static-entry.js b/packages/gatsby/cache-dir/develop-static-entry.js
index 3a8449d532212..16f97a21c851f 100644
--- a/packages/gatsby/cache-dir/develop-static-entry.js
+++ b/packages/gatsby/cache-dir/develop-static-entry.js
@@ -129,12 +129,8 @@ export default (pagePath, callback) => {
             : undefined,
         }
 
-        if (!lazySyncRequires.components[componentChunkName]) {
-          throw new Error(`try again`)
-        }
-
         const pageElement = createElement(
-          lazySyncRequires.components[componentChunkName],
+          lazySyncRequires.lazyComponents[componentChunkName],
           props
         )
 
diff --git a/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js b/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js
index 57e075d2d1881..9a93c6534e5de 100644
--- a/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js
+++ b/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js
@@ -112,47 +112,26 @@ const parseError = function (err, directory) {
 
 exports.parseError = parseError
 
-const callRenderFunction = ({ path, htmlComponentRendererPath }) => {
-  const htmlComponentRenderer = require(htmlComponentRendererPath)
-  return new Promise(resolve => {
-    htmlComponentRenderer.default(path, (_throwAway, htmlString) => {
-      // console.log(`rendered correctly`, { htmlString, attempts })
-      resolve(htmlString)
-    })
-  })
-}
-
 exports.renderHTML = ({ path, htmlComponentRendererPath, directory }) =>
-  new Promise(async (resolve, reject) => {
+  new Promise((resolve, reject) => {
     try {
-      const htmlString = await callRenderFunction({
-        path,
-        htmlComponentRendererPath,
+      const htmlComponentRenderer = require(htmlComponentRendererPath)
+      htmlComponentRenderer.default(path, (_throwAway, htmlString) => {
+        // console.log(`rendered correctly`, { htmlString, attempts })
+        resolve(htmlString)
       })
-      resolve(htmlString)
     } catch (err) {
-      // The bundle might not yet have the page component we need yet
-      // so develop-static-entry.js will throw (kinda like React suspense)
-      // and we keep trying until it's there.
-      if (err.message === `try again`) {
-        setTimeout(() => {
-          exports.deleteModuleCache(htmlComponentRendererPath)
-          exports.renderHTML({ path, htmlComponentRendererPath, directory })
-        }, 100)
-      } else {
-        const stack = err.stack ? err.stack : ``
-        // Only generate error pages for webpack errors. If it's not a webpack
-        // error, it's not a user error so probably a system error so we'll just
-        // panic and quit.
-        const regex = /webpack:\/lib\//gm
-        if (!stack.match(regex)) {
-          console.log(err)
-          return
-        }
-        const error = parseError(err, directory)
-        reject(error)
-        // return { error }
+      const stack = err.stack ? err.stack : ``
+      // Only generate error pages for webpack errors. If it's not a webpack
+      // error, it's not a user error so probably a system error so we'll just
+      // panic and quit.
+      const regex = /webpack:\/lib\//gm
+      if (!stack.match(regex)) {
+        console.log(err)
+        return
       }
+      const error = parseError(err, directory)
+      reject(error)
     }
   })
 
diff --git a/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts b/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
index 54962829ce525..6331fab5b26e9 100644
--- a/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
+++ b/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
@@ -1,6 +1,7 @@
 import JestWorker from "jest-worker"
 import fs from "fs-extra"
 import { joinPath } from "gatsby-core-utils"
+import report from "gatsby-cli/lib/reporter"
 
 import { store } from "../../redux"
 import { writeModule } from "../gatsby-webpack-virtual-modules"
@@ -9,7 +10,6 @@ const startWorker = (): any => {
   const newWorker = new JestWorker(require.resolve(`./render-dev-html-child`), {
     exposedMethods: [`renderHTML`, `deleteModuleCache`, `warmup`],
     numWorkers: 1,
-    enableWorkerThreads: true,
     forkOptions: { silent: false },
   })
 
@@ -66,7 +66,9 @@ const writeLazyRequires = (pageComponents): void => {
 // prefer default export if available
 const preferDefault = m => (m && m.default) || m
 \n\n`
-  lazySyncRequires += `exports.components = {\n${[...pageComponents.values()]
+  lazySyncRequires += `exports.lazyComponents = {\n${[
+    ...pageComponents.values(),
+  ]
     .map(
       (c: IGatsbyPageComponent): string =>
         `  "${
@@ -80,32 +82,89 @@ const preferDefault = m => (m && m.default) || m
 }
 
 const pageComponents = new Map()
+const pageComponentsWritten = new Set()
+const inFlightPromises = new Map()
+
+const searchFileForString = (substring, filePath): Promise<boolean> =>
+  new Promise(resolve => {
+    const stream = fs.createReadStream(filePath)
+    let found = false
+    stream.on(`data`, function (d) {
+      if (d.includes(substring)) {
+        found = true
+        stream.close()
+        resolve(found)
+      }
+    })
+    stream.on(`error`, function () {
+      resolve(found)
+    })
+    stream.on(`close`, function () {
+      resolve(found)
+    })
+  })
+
 const ensurePathComponentInSSRBundle = async (
   path,
   directory
-): Promise<void> => {
+): Promise<any> => {
   const pages = [...store.getState().pages.values()]
   const page = pages.find(p => p.path === path)
-  if (page && !pageComponents.has(page.component)) {
-    pageComponents.set(page.component, {
-      component: page.component,
-      componentChunkName: page.componentChunkName,
-    })
-    writeLazyRequires(pageComponents)
-    const htmlComponentRendererPath = joinPath(
-      directory,
-      `public/render-page.js`
-    )
-    await new Promise(async resolve => {
-      const watcher = fs.watch(htmlComponentRendererPath, () => {
-        // It's changed, clean up the watcher.
-        watcher.close()
-        // Make sure the worker is ready.
-        worker.deleteModuleCache(htmlComponentRendererPath).then(resolve)
-      })
+
+  // This shouldn't happen.
+  if (!page) {
+    report.panic(`page not found`)
+  }
+
+  // If we know it's written, return.
+  if (pageComponentsWritten.has(page.componentChunkName)) {
+    return true
+  }
+
+  let promiseResolve
+  // If we're already handling this path, return its promise.
+  if (inFlightPromises.has(page.componentChunkName)) {
+    return inFlightPromises.get(page.componentChunkName)
+  } else {
+    const promise = new Promise(function (resolve) {
+      promiseResolve = resolve
     })
+
+    inFlightPromises.set(page.componentChunkName, promise)
   }
-  // else nothing to do so we return.
+
+  // Write out the component information to lazy-sync-requires
+  pageComponents.set(page.component, {
+    component: page.component,
+    componentChunkName: page.componentChunkName,
+  })
+  writeLazyRequires(pageComponents)
+
+  // Now wait for it to be written to public/render-page.js
+  const htmlComponentRendererPath = joinPath(directory, `public/render-page.js`)
+  const watcher = fs.watch(htmlComponentRendererPath, async () => {
+    // This search takes 1-10ms
+    const found = await searchFileForString(
+      page.componentChunkName,
+      htmlComponentRendererPath
+    )
+    if (found) {
+      // It's changed, clean up the watcher.
+      watcher.close()
+      // Make sure the worker is ready and then resolve.
+      worker.deleteModuleCache(htmlComponentRendererPath).then(promiseResolve)
+    }
+  })
+
+  // We're done, delete the promise from the in-flight promises
+  // and add it to the set of done paths.
+  inFlightPromises.get(page.componentChunkName).then(() => {
+    inFlightPromises.delete(page.componentChunkName)
+    pageComponentsWritten.add(page.componentChunkName)
+  })
+
+  // Return the promise for the first request.
+  return inFlightPromises.get(page.componentChunkName)
 }
 
 // Initialize the virtual module.

From e63dd380f4c3d1aa28fb89eb97ba659965e318cc Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Thu, 29 Oct 2020 17:04:07 -0700
Subject: [PATCH 109/123] for some reason this lets log warnings from React not
 break jest-worker

---
 packages/gatsby/src/utils/dev-ssr/render-dev-html.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts b/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
index 6331fab5b26e9..abd0abf31545a 100644
--- a/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
+++ b/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
@@ -9,6 +9,7 @@ import { writeModule } from "../gatsby-webpack-virtual-modules"
 const startWorker = (): any => {
   const newWorker = new JestWorker(require.resolve(`./render-dev-html-child`), {
     exposedMethods: [`renderHTML`, `deleteModuleCache`, `warmup`],
+    enableWorkerThreads: true,
     numWorkers: 1,
     forkOptions: { silent: false },
   })

From b13521f8b094f93f23c8d45691b25b10e9d56604 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Thu, 29 Oct 2020 17:29:27 -0700
Subject: [PATCH 110/123] fix test & comment

---
 packages/gatsby/cache-dir/__tests__/static-entry.js  | 2 +-
 packages/gatsby/src/utils/dev-ssr/render-dev-html.ts | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/packages/gatsby/cache-dir/__tests__/static-entry.js b/packages/gatsby/cache-dir/__tests__/static-entry.js
index a547bf71405a0..327a5e9a7e6c8 100644
--- a/packages/gatsby/cache-dir/__tests__/static-entry.js
+++ b/packages/gatsby/cache-dir/__tests__/static-entry.js
@@ -20,7 +20,7 @@ jest.mock(
   `$virtual/lazy-sync-requires`,
   () => {
     return {
-      components: {
+      lazyComponents: {
         "page-component---src-pages-test-js": () => null,
       },
     }
diff --git a/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts b/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
index abd0abf31545a..db8afc6c06ec9 100644
--- a/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
+++ b/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
@@ -145,6 +145,10 @@ const ensurePathComponentInSSRBundle = async (
   const htmlComponentRendererPath = joinPath(directory, `public/render-page.js`)
   const watcher = fs.watch(htmlComponentRendererPath, async () => {
     // This search takes 1-10ms
+    // We do it as there can be a race conditions where two pages
+    // are requested at the same time which means that both are told render-page.js
+    // has changed when the first page is complete meaning the second
+    // page's component won't be in the render meaning its SSR will fail.
     const found = await searchFileForString(
       page.componentChunkName,
       htmlComponentRendererPath

From d10ec645e142a24a9b2257388aca26b837c2b071 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Fri, 30 Oct 2020 15:39:07 -0700
Subject: [PATCH 111/123] We can't use the gatsby reporter inside a child as it
 uses process.send for console.* which breaks jest-worker

---
 packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js | 3 +--
 packages/gatsby/src/utils/dev-ssr/render-dev-html.ts       | 3 ---
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js b/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js
index 9a93c6534e5de..bccf4d84ab390 100644
--- a/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js
+++ b/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js
@@ -2,7 +2,6 @@ const { codeFrameColumns } = require(`@babel/code-frame`)
 const ansiHTML = require(`ansi-html`)
 const fs = require(`fs-extra`)
 const sysPath = require(`path`)
-const report = require(`gatsby-cli/lib/reporter`)
 
 const getPosition = function (stackObject) {
   let filename
@@ -69,7 +68,7 @@ const parseError = function (err, directory) {
     code = fs.readFileSync(filename, `utf-8`)
   } catch (e) {
     console.log(err)
-    report.error(`Couldn't read the file ${filename}`, e)
+    console.log(`Couldn't read the file ${filename}`, e)
   }
   const line = position.line
   const row = position.row
diff --git a/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts b/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
index db8afc6c06ec9..51182ed595262 100644
--- a/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
+++ b/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
@@ -14,9 +14,6 @@ const startWorker = (): any => {
     forkOptions: { silent: false },
   })
 
-  newWorker.getStdout().pipe(process.stdout)
-  newWorker.getStderr().pipe(process.stderr)
-
   // jest-worker is lazy with forking but we want to fork immediately so the user
   // doesn't have to wait.
   // @ts-ignore

From 4f159a87af0b265d5c310796e887d6dae8d12ca7 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Sat, 31 Oct 2020 12:00:44 -0700
Subject: [PATCH 112/123] Move writing lazyComponents to requires-writer &
 still use old develop-static-entry if no flag

---
 .../ssr/__tests__../src/pages/bad-page.js     |   9 +
 .../ssr/__tests__/__snapshots__/ssr.js.snap   |  18 ++
 .../ssr/__tests__/fixtures/bad-page.js        |   9 +
 integration-tests/ssr/__tests__/ssr.js        |  37 +++
 .../cache-dir/__tests__/static-entry.js       |   2 +-
 .../gatsby/cache-dir/develop-static-entry.js  | 245 +++++-------------
 .../cache-dir/new-develop-static-entry.js     | 235 +++++++++++++++++
 .../gatsby/src/bootstrap/requires-writer.ts   |  89 ++++---
 packages/gatsby/src/redux/actions/public.js   |  13 +
 packages/gatsby/src/redux/reducers/index.ts   |   2 +
 .../src/redux/reducers/ssr-visited-page.ts    |  24 ++
 packages/gatsby/src/redux/types.ts            |   7 +
 .../src/utils/dev-ssr/develop-html-route.ts   |  10 +-
 .../utils/dev-ssr/render-dev-html-child.js    |  23 +-
 .../src/utils/dev-ssr/render-dev-html.ts      | 173 +++++--------
 packages/gatsby/src/utils/start-server.ts     |   8 +-
 packages/gatsby/src/utils/webpack.config.js   |   4 +-
 17 files changed, 562 insertions(+), 346 deletions(-)
 create mode 100644 integration-tests/ssr/__tests__../src/pages/bad-page.js
 create mode 100644 integration-tests/ssr/__tests__/fixtures/bad-page.js
 create mode 100644 packages/gatsby/cache-dir/new-develop-static-entry.js
 create mode 100644 packages/gatsby/src/redux/reducers/ssr-visited-page.ts

diff --git a/integration-tests/ssr/__tests__../src/pages/bad-page.js b/integration-tests/ssr/__tests__../src/pages/bad-page.js
new file mode 100644
index 0000000000000..429c52813c049
--- /dev/null
+++ b/integration-tests/ssr/__tests__../src/pages/bad-page.js
@@ -0,0 +1,9 @@
+import React from "react"
+
+const Component = () => {
+  const a = window.width
+
+  return <div>hi</div>
+}
+
+export default Component
diff --git a/integration-tests/ssr/__tests__/__snapshots__/ssr.js.snap b/integration-tests/ssr/__tests__/__snapshots__/ssr.js.snap
index 050d55e99b7d6..658927d3380ca 100644
--- a/integration-tests/ssr/__tests__/__snapshots__/ssr.js.snap
+++ b/integration-tests/ssr/__tests__/__snapshots__/ssr.js.snap
@@ -1,3 +1,21 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
 exports[`SSR is run for a page when it is requested 1`] = `"<!DOCTYPE html><html><head><meta charSet=\\"utf-8\\"/><meta http-equiv=\\"x-ua-compatible\\" content=\\"ie=edge\\"/><meta name=\\"viewport\\" content=\\"width=device-width, initial-scale=1, shrink-to-fit=no\\"/><meta name=\\"note\\" content=\\"environment=development\\"/><script src=\\"/socket.io/socket.io.js\\"></script></head><body><div id=\\"___gatsby\\"><div style=\\"outline:none\\" tabindex=\\"-1\\" id=\\"gatsby-focus-wrapper\\"><div>Hello world</div></div><div id=\\"gatsby-announcer\\" style=\\"position:absolute;top:0;width:1px;height:1px;padding:0;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;border:0\\" aria-live=\\"assertive\\" aria-atomic=\\"true\\"></div></div><script src=\\"/polyfill.js\\" nomodule=\\"\\"></script><script src=\\"/commons.js\\"></script></body></html>"`;
+
+exports[`SSR it generates an error page correctly 1`] = `
+"<title>Develop SSR Error</title><h1>Error<h1>
+        <h2>The page didn't SSR correctly</h2>
+        <ul>
+          <li><strong>URL path:</strong> <code>/bad-page/</code></li>
+          <li><strong>File path:</strong> <code>src/pages/bad-page.js</code></li>
+        </ul>
+        <h3>error message</h3>
+        <p><code>window is not defined</code></p>
+        <pre style=\\"background:#fdfaf6;padding:8px;\\"><span style=\\"font-weight:normal;opacity:1;color:#452475;background:#fdfaf6;\\"> <span style=\\"color:#527713;\\"> 2 | </span></span>
+<span style=\\"font-weight:normal;opacity:1;color:#452475;background:#fdfaf6;\\"> <span style=\\"color:#527713;\\"> 3 | </span><span style=\\"color:#006500;\\">const</span> <span style=\\"color:#DB3A00;\\">Component</span> <span style=\\"color:#DB3A00;\\">=</span> () <span style=\\"color:#DB3A00;\\">=></span> {</span>
+<span style=\\"font-weight:normal;opacity:1;color:#452475;background:#fdfaf6;\\"><span style=\\"color:#096fb3;\\"><span style=\\"font-weight:bold;\\">></span></span><span style=\\"color:#527713;\\"> 4 | </span>  <span style=\\"color:#006500;\\">const</span> a <span style=\\"color:#DB3A00;\\">=</span> window<span style=\\"color:#DB3A00;\\">.</span>width</span>
+<span style=\\"font-weight:normal;opacity:1;color:#452475;background:#fdfaf6;\\"> <span style=\\"color:#527713;\\">   | </span>            <span style=\\"color:#096fb3;\\"><span style=\\"font-weight:bold;\\">^</span></span></span>
+<span style=\\"font-weight:normal;opacity:1;color:#452475;background:#fdfaf6;\\"> <span style=\\"color:#527713;\\"> 5 | </span></span>
+<span style=\\"font-weight:normal;opacity:1;color:#452475;background:#fdfaf6;\\"> <span style=\\"color:#527713;\\"> 6 | </span>  <span style=\\"color:#006500;\\">return</span> <span style=\\"color:#DB3A00;\\"><</span><span style=\\"color:#DB3A00;\\">div</span><span style=\\"color:#DB3A00;\\">></span>hi<span style=\\"color:#DB3A00;\\"><</span><span style=\\"color:#DB3A00;\\">/</span><span style=\\"color:#DB3A00;\\">div</span><span style=\\"color:#DB3A00;\\">></span></span>
+<span style=\\"font-weight:normal;opacity:1;color:#452475;background:#fdfaf6;\\"> <span style=\\"color:#527713;\\"> 7 | </span>}</span></pre>"
+`;
diff --git a/integration-tests/ssr/__tests__/fixtures/bad-page.js b/integration-tests/ssr/__tests__/fixtures/bad-page.js
new file mode 100644
index 0000000000000..429c52813c049
--- /dev/null
+++ b/integration-tests/ssr/__tests__/fixtures/bad-page.js
@@ -0,0 +1,9 @@
+import React from "react"
+
+const Component = () => {
+  const a = window.width
+
+  return <div>hi</div>
+}
+
+export default Component
diff --git a/integration-tests/ssr/__tests__/ssr.js b/integration-tests/ssr/__tests__/ssr.js
index fc99eccf93534..f9a2a80e8811b 100644
--- a/integration-tests/ssr/__tests__/ssr.js
+++ b/integration-tests/ssr/__tests__/ssr.js
@@ -1,5 +1,7 @@
 const fetch = require(`node-fetch`)
 const { execFile } = require("child_process")
+const fs = require(`fs-extra`)
+const path = require(`path`)
 
 describe(`SSR`, () => {
   test(`is run for a page when it is requested`, async () => {
@@ -19,4 +21,39 @@ describe(`SSR`, () => {
 
     expect(exitCode).toEqual({ exitCode: 0 })
   })
+  test(`it generates an error page correctly`, async () => {
+    const src = path.join(__dirname, `/fixtures/bad-page.js`)
+    const dest = path.join(__dirname, `../src/pages/bad-page.js`)
+    const result = fs.copySync(src, dest)
+
+    const pageUrl = `http://localhost:8000/bad-page/`
+    await new Promise(resolve => {
+      const testInterval = setInterval(() => {
+        fetch(pageUrl).then(res => {
+          if (res.status !== 404) {
+            clearInterval(testInterval)
+            resolve()
+          }
+        })
+      }, 1000)
+    })
+
+    const rawDevHtml = await fetch(
+      `http://localhost:8000/bad-page/`
+    ).then(res => res.text())
+    expect(rawDevHtml).toMatchSnapshot()
+    fs.remove(dest)
+
+    // After the page is gone, it'll 404.
+    await new Promise(resolve => {
+      const testInterval = setInterval(() => {
+        fetch(pageUrl).then(res => {
+          if (res.status === 404) {
+            clearInterval(testInterval)
+            resolve()
+          }
+        })
+      }, 400)
+    })
+  })
 })
diff --git a/packages/gatsby/cache-dir/__tests__/static-entry.js b/packages/gatsby/cache-dir/__tests__/static-entry.js
index 327a5e9a7e6c8..04367853d9202 100644
--- a/packages/gatsby/cache-dir/__tests__/static-entry.js
+++ b/packages/gatsby/cache-dir/__tests__/static-entry.js
@@ -2,7 +2,7 @@ import React from "react"
 import fs from "fs"
 const { join } = require(`path`)
 
-import DevelopStaticEntry from "../develop-static-entry"
+import DevelopStaticEntry from "../new-develop-static-entry"
 
 jest.mock(`fs`, () => {
   const fs = jest.requireActual(`fs`)
diff --git a/packages/gatsby/cache-dir/develop-static-entry.js b/packages/gatsby/cache-dir/develop-static-entry.js
index 16f97a21c851f..9e245316e40f2 100644
--- a/packages/gatsby/cache-dir/develop-static-entry.js
+++ b/packages/gatsby/cache-dir/develop-static-entry.js
@@ -1,14 +1,7 @@
 import React from "react"
-import fs from "fs"
-import { renderToString, renderToStaticMarkup } from "react-dom/server"
+import { renderToStaticMarkup } from "react-dom/server"
 import { merge } from "lodash"
-import { join } from "path"
 import apiRunner from "./api-runner-ssr"
-import { grabMatchParams } from "./find-path"
-import lazySyncRequires from "$virtual/lazy-sync-requires"
-
-import { RouteAnnouncerProps } from "./route-announcer-props"
-import { ServerLocation, Router, isRedirect } from "@reach/router"
 // import testRequireError from "./test-require-error"
 // For some extremely mysterious reason, webpack adds the above module *after*
 // this module so that when this code runs, testRequireError is undefined.
@@ -34,7 +27,6 @@ try {
 Html = Html && Html.__esModule ? Html.default : Html
 
 export default (pagePath, callback) => {
-  let bodyHtml = ``
   let headComponents = [
     <meta key="environment" name="note" content="environment=development" />,
   ]
@@ -43,186 +35,73 @@ export default (pagePath, callback) => {
   let preBodyComponents = []
   let postBodyComponents = []
   let bodyProps = {}
+  let htmlStr
+
+  const setHeadComponents = components => {
+    headComponents = headComponents.concat(components)
+  }
+
+  const setHtmlAttributes = attributes => {
+    htmlAttributes = merge(htmlAttributes, attributes)
+  }
+
+  const setBodyAttributes = attributes => {
+    bodyAttributes = merge(bodyAttributes, attributes)
+  }
+
+  const setPreBodyComponents = components => {
+    preBodyComponents = preBodyComponents.concat(components)
+  }
+
+  const setPostBodyComponents = components => {
+    postBodyComponents = postBodyComponents.concat(components)
+  }
 
-  const generateBodyHTML = () => {
-    const setHeadComponents = components => {
-      headComponents = headComponents.concat(components)
-    }
-
-    const setHtmlAttributes = attributes => {
-      htmlAttributes = merge(htmlAttributes, attributes)
-    }
-
-    const setBodyAttributes = attributes => {
-      bodyAttributes = merge(bodyAttributes, attributes)
-    }
-
-    const setPreBodyComponents = components => {
-      preBodyComponents = preBodyComponents.concat(components)
-    }
-
-    const setPostBodyComponents = components => {
-      postBodyComponents = postBodyComponents.concat(components)
-    }
-
-    const setBodyProps = props => {
-      bodyProps = merge({}, bodyProps, props)
-    }
-
-    const getHeadComponents = () => headComponents
-
-    const replaceHeadComponents = components => {
-      headComponents = components
-    }
-
-    const replaceBodyHTMLString = body => {
-      bodyHtml = body
-    }
-
-    const getPreBodyComponents = () => preBodyComponents
-
-    const replacePreBodyComponents = components => {
-      preBodyComponents = components
-    }
-
-    const getPostBodyComponents = () => postBodyComponents
-
-    const replacePostBodyComponents = components => {
-      postBodyComponents = components
-    }
-
-    const getPageDataPath = path => {
-      const fixedPagePath = path === `/` ? `index` : path
-      return join(`page-data`, fixedPagePath, `page-data.json`)
-    }
-
-    const getPageData = pagePath => {
-      const pageDataPath = getPageDataPath(pagePath)
-      const absolutePageDataPath = join(process.cwd(), `public`, pageDataPath)
-      const pageDataJson = fs.readFileSync(absolutePageDataPath, `utf8`)
-
-      try {
-        return JSON.parse(pageDataJson)
-      } catch (err) {
-        return null
-      }
-    }
-
-    const pageData = getPageData(pagePath)
-
-    const componentChunkName = pageData?.componentChunkName
-
-    const createElement = React.createElement
-
-    class RouteHandler extends React.Component {
-      render() {
-        const props = {
-          ...this.props,
-          ...pageData.result,
-          params: {
-            ...grabMatchParams(this.props.location.pathname),
-            ...(pageData.result?.pageContext?.__params || {}),
-          },
-          // pathContext was deprecated in v2. Renamed to pageContext
-          pathContext: pageData.result
-            ? pageData.result.pageContext
-            : undefined,
-        }
-
-        const pageElement = createElement(
-          lazySyncRequires.lazyComponents[componentChunkName],
-          props
-        )
-
-        const wrappedPage = apiRunner(
-          `wrapPageElement`,
-          { element: pageElement, props },
-          pageElement,
-          ({ result }) => {
-            return { element: result, props }
-          }
-        ).pop()
-
-        return wrappedPage
-      }
-    }
-
-    const routerElement = (
-      <ServerLocation url={`${__BASE_PATH__}${pagePath}`}>
-        <Router id="gatsby-focus-wrapper" baseuri={__BASE_PATH__}>
-          <RouteHandler path="/*" />
-        </Router>
-        <div {...RouteAnnouncerProps} />
-      </ServerLocation>
-    )
-
-    const bodyComponent = apiRunner(
-      `wrapRootElement`,
-      { element: routerElement, pathname: pagePath },
-      routerElement,
-      ({ result }) => {
-        return { element: result, pathname: pagePath }
-      }
-    ).pop()
-
-    // Let the site or plugin render the page component.
-    apiRunner(`replaceRenderer`, {
-      bodyComponent,
-      replaceBodyHTMLString,
-      setHeadComponents,
-      setHtmlAttributes,
-      setBodyAttributes,
-      setPreBodyComponents,
-      setPostBodyComponents,
-      setBodyProps,
-      pathname: pagePath,
-      pathPrefix: __PATH_PREFIX__,
-    })
-
-    // If no one stepped up, we'll handle it.
-    if (!bodyHtml) {
-      try {
-        bodyHtml = renderToString(bodyComponent)
-      } catch (e) {
-        // ignore @reach/router redirect errors
-        if (!isRedirect(e)) throw e
-      }
-    }
-
-    apiRunner(`onRenderBody`, {
-      setHeadComponents,
-      setHtmlAttributes,
-      setBodyAttributes,
-      setPreBodyComponents,
-      setPostBodyComponents,
-      setBodyProps,
-      pathname: pagePath,
-    })
-
-    apiRunner(`onPreRenderHTML`, {
-      getHeadComponents,
-      replaceHeadComponents,
-      getPreBodyComponents,
-      replacePreBodyComponents,
-      getPostBodyComponents,
-      replacePostBodyComponents,
-      pathname: pagePath,
-    })
-
-    return bodyHtml
+  const setBodyProps = props => {
+    bodyProps = merge({}, bodyProps, props)
   }
 
-  let bodyStr = ``
-  if (
-    process.env.NODE_ENV == `test` ||
-    (process.env.GATSBY_EXPERIMENTAL_DEV_SSR && pagePath !== `/dev-404-page/`)
-  ) {
-    bodyStr = generateBodyHTML()
+  const getHeadComponents = () => headComponents
+
+  const replaceHeadComponents = components => {
+    headComponents = components
+  }
+
+  const getPreBodyComponents = () => preBodyComponents
+
+  const replacePreBodyComponents = components => {
+    preBodyComponents = components
   }
 
+  const getPostBodyComponents = () => postBodyComponents
+
+  const replacePostBodyComponents = components => {
+    postBodyComponents = components
+  }
+
+  apiRunner(`onRenderBody`, {
+    setHeadComponents,
+    setHtmlAttributes,
+    setBodyAttributes,
+    setPreBodyComponents,
+    setPostBodyComponents,
+    setBodyProps,
+    pathname: pagePath,
+  })
+
+  apiRunner(`onPreRenderHTML`, {
+    getHeadComponents,
+    replaceHeadComponents,
+    getPreBodyComponents,
+    replacePreBodyComponents,
+    getPostBodyComponents,
+    replacePostBodyComponents,
+    pathname: pagePath,
+  })
+
   const htmlElement = React.createElement(Html, {
     ...bodyProps,
-    body: bodyStr,
+    body: ``,
     headComponents: headComponents.concat([
       <script key={`io`} src="/socket.io/socket.io.js" />,
     ]),
@@ -234,7 +113,7 @@ export default (pagePath, callback) => {
       <script key={`commons`} src="/commons.js" />,
     ]),
   })
-  let htmlStr = renderToStaticMarkup(htmlElement)
+  htmlStr = renderToStaticMarkup(htmlElement)
   htmlStr = `<!DOCTYPE html>${htmlStr}`
 
   callback(null, htmlStr)
diff --git a/packages/gatsby/cache-dir/new-develop-static-entry.js b/packages/gatsby/cache-dir/new-develop-static-entry.js
new file mode 100644
index 0000000000000..a061ac77e5bc7
--- /dev/null
+++ b/packages/gatsby/cache-dir/new-develop-static-entry.js
@@ -0,0 +1,235 @@
+import React from "react"
+import fs from "fs"
+import { renderToString, renderToStaticMarkup } from "react-dom/server"
+import { merge } from "lodash"
+import { join } from "path"
+import apiRunner from "./api-runner-ssr"
+import { grabMatchParams } from "./find-path"
+import lazySyncRequires from "$virtual/lazy-sync-requires"
+
+import { RouteAnnouncerProps } from "./route-announcer-props"
+import { ServerLocation, Router, isRedirect } from "@reach/router"
+// import testRequireError from "./test-require-error"
+// For some extremely mysterious reason, webpack adds the above module *after*
+// this module so that when this code runs, testRequireError is undefined.
+// So in the meantime, we'll just inline it.
+const testRequireError = (moduleName, err) => {
+  const regex = new RegExp(`Error: Cannot find module\\s.${moduleName}`)
+  const firstLine = err.toString().split(`\n`)[0]
+  return regex.test(firstLine)
+}
+
+let Html
+try {
+  Html = require(`../src/html`)
+} catch (err) {
+  if (testRequireError(`../src/html`, err)) {
+    Html = require(`./default-html`)
+  } else {
+    console.log(`There was an error requiring "src/html.js"\n\n`, err, `\n\n`)
+    process.exit()
+  }
+}
+
+Html = Html && Html.__esModule ? Html.default : Html
+
+export default (pagePath, callback) => {
+  let bodyHtml = ``
+  let headComponents = [
+    <meta key="environment" name="note" content="environment=development" />,
+  ]
+  let htmlAttributes = {}
+  let bodyAttributes = {}
+  let preBodyComponents = []
+  let postBodyComponents = []
+  let bodyProps = {}
+
+  const generateBodyHTML = () => {
+    const setHeadComponents = components => {
+      headComponents = headComponents.concat(components)
+    }
+
+    const setHtmlAttributes = attributes => {
+      htmlAttributes = merge(htmlAttributes, attributes)
+    }
+
+    const setBodyAttributes = attributes => {
+      bodyAttributes = merge(bodyAttributes, attributes)
+    }
+
+    const setPreBodyComponents = components => {
+      preBodyComponents = preBodyComponents.concat(components)
+    }
+
+    const setPostBodyComponents = components => {
+      postBodyComponents = postBodyComponents.concat(components)
+    }
+
+    const setBodyProps = props => {
+      bodyProps = merge({}, bodyProps, props)
+    }
+
+    const getHeadComponents = () => headComponents
+
+    const replaceHeadComponents = components => {
+      headComponents = components
+    }
+
+    const replaceBodyHTMLString = body => {
+      bodyHtml = body
+    }
+
+    const getPreBodyComponents = () => preBodyComponents
+
+    const replacePreBodyComponents = components => {
+      preBodyComponents = components
+    }
+
+    const getPostBodyComponents = () => postBodyComponents
+
+    const replacePostBodyComponents = components => {
+      postBodyComponents = components
+    }
+
+    const getPageDataPath = path => {
+      const fixedPagePath = path === `/` ? `index` : path
+      return join(`page-data`, fixedPagePath, `page-data.json`)
+    }
+
+    const getPageData = pagePath => {
+      const pageDataPath = getPageDataPath(pagePath)
+      const absolutePageDataPath = join(process.cwd(), `public`, pageDataPath)
+      const pageDataJson = fs.readFileSync(absolutePageDataPath, `utf8`)
+
+      try {
+        return JSON.parse(pageDataJson)
+      } catch (err) {
+        return null
+      }
+    }
+
+    const pageData = getPageData(pagePath)
+
+    const componentChunkName = pageData?.componentChunkName
+
+    const createElement = React.createElement
+
+    class RouteHandler extends React.Component {
+      render() {
+        const props = {
+          ...this.props,
+          ...pageData.result,
+          params: {
+            ...grabMatchParams(this.props.location.pathname),
+            ...(pageData.result?.pageContext?.__params || {}),
+          },
+          // pathContext was deprecated in v2. Renamed to pageContext
+          pathContext: pageData.result
+            ? pageData.result.pageContext
+            : undefined,
+        }
+
+        const pageElement = createElement(
+          lazySyncRequires.lazyComponents[componentChunkName],
+          props
+        )
+
+        const wrappedPage = apiRunner(
+          `wrapPageElement`,
+          { element: pageElement, props },
+          pageElement,
+          ({ result }) => {
+            return { element: result, props }
+          }
+        ).pop()
+
+        return wrappedPage
+      }
+    }
+
+    const routerElement = (
+      <ServerLocation url={`${__BASE_PATH__}${pagePath}`}>
+        <Router id="gatsby-focus-wrapper" baseuri={__BASE_PATH__}>
+          <RouteHandler path="/*" />
+        </Router>
+        <div {...RouteAnnouncerProps} />
+      </ServerLocation>
+    )
+
+    const bodyComponent = apiRunner(
+      `wrapRootElement`,
+      { element: routerElement, pathname: pagePath },
+      routerElement,
+      ({ result }) => {
+        return { element: result, pathname: pagePath }
+      }
+    ).pop()
+
+    // Let the site or plugin render the page component.
+    apiRunner(`replaceRenderer`, {
+      bodyComponent,
+      replaceBodyHTMLString,
+      setHeadComponents,
+      setHtmlAttributes,
+      setBodyAttributes,
+      setPreBodyComponents,
+      setPostBodyComponents,
+      setBodyProps,
+      pathname: pagePath,
+      pathPrefix: __PATH_PREFIX__,
+    })
+
+    // If no one stepped up, we'll handle it.
+    if (!bodyHtml) {
+      try {
+        bodyHtml = renderToString(bodyComponent)
+      } catch (e) {
+        // ignore @reach/router redirect errors
+        if (!isRedirect(e)) throw e
+      }
+    }
+
+    apiRunner(`onRenderBody`, {
+      setHeadComponents,
+      setHtmlAttributes,
+      setBodyAttributes,
+      setPreBodyComponents,
+      setPostBodyComponents,
+      setBodyProps,
+      pathname: pagePath,
+    })
+
+    apiRunner(`onPreRenderHTML`, {
+      getHeadComponents,
+      replaceHeadComponents,
+      getPreBodyComponents,
+      replacePreBodyComponents,
+      getPostBodyComponents,
+      replacePostBodyComponents,
+      pathname: pagePath,
+    })
+
+    return bodyHtml
+  }
+
+  const bodyStr = generateBodyHTML()
+
+  const htmlElement = React.createElement(Html, {
+    ...bodyProps,
+    body: bodyStr,
+    headComponents: headComponents.concat([
+      <script key={`io`} src="/socket.io/socket.io.js" />,
+    ]),
+    htmlAttributes,
+    bodyAttributes,
+    preBodyComponents,
+    postBodyComponents: postBodyComponents.concat([
+      <script key={`polyfill`} src="/polyfill.js" noModule={true} />,
+      <script key={`commons`} src="/commons.js" />,
+    ]),
+  })
+  let htmlStr = renderToStaticMarkup(htmlElement)
+  htmlStr = `<!DOCTYPE html>${htmlStr}`
+
+  callback(null, htmlStr)
+}
diff --git a/packages/gatsby/src/bootstrap/requires-writer.ts b/packages/gatsby/src/bootstrap/requires-writer.ts
index 6373d86ed77cb..ae2efc640967c 100644
--- a/packages/gatsby/src/bootstrap/requires-writer.ts
+++ b/packages/gatsby/src/bootstrap/requires-writer.ts
@@ -162,26 +162,41 @@ const getMatchPaths = (
 
 const createHash = (
   matchPaths: Array<IGatsbyPageMatchPath>,
-  components: Array<IGatsbyPageComponent>
+  components: Array<IGatsbyPageComponent>,
+  cleanedSSRVisitedPageComponents: Array<IGatsbyPageComponent>
 ): string =>
   crypto
     .createHash(`md5`)
-    .update(JSON.stringify({ matchPaths, components }))
+    .update(
+      JSON.stringify({
+        matchPaths,
+        components,
+        cleanedSSRVisitedPageComponents,
+      })
+    )
     .digest(`hex`)
 
 // Write out pages information.
 export const writeAll = async (state: IGatsbyState): Promise<boolean> => {
-  // console.log(`on requiresWriter progress`)
   const { program } = state
   const pages = [...state.pages.values()]
   const matchPaths = getMatchPaths(pages)
   const components = getComponents(pages)
+  const ssrVisitedPageComponents = [...state.ssrVisitedPages.values()]
+
+  // Remove any page components that no longer exist.
+  const cleanedSSRVisitedPageComponents = components.filter(c =>
+    ssrVisitedPageComponents.some(s => s === c.componentChunkName)
+  )
 
-  const newHash = createHash(matchPaths, components)
+  const newHash = createHash(
+    matchPaths,
+    components,
+    cleanedSSRVisitedPageComponents
+  )
 
   if (newHash === lastHash) {
     // Nothing changed. No need to rewrite files
-    // console.log(`on requiresWriter END1`)
     return false
   }
 
@@ -230,6 +245,24 @@ const preferDefault = m => (m && m.default) || m
     .join(`,\n`)}
 }\n\n`
 
+  // Create file with sync requires of visited page components files.
+  let lazySyncRequires = `${hotImport}
+
+  // prefer default export if available
+  const preferDefault = m => (m && m.default) || m
+  \n\n`
+  lazySyncRequires += `exports.lazyComponents = {\n${cleanedSSRVisitedPageComponents
+    .map(
+      (c: IGatsbyPageComponent): string =>
+        `  "${
+          c.componentChunkName
+        }": ${hotMethod}(preferDefault(require("${joinPath(c.component)}")))`
+    )
+    .join(`,\n`)}
+  }\n\n`
+
+  writeModule(`$virtual/lazy-sync-requires`, lazySyncRequires)
+
   const writeAndMove = (
     virtualFilePath: string,
     file: string,
@@ -264,45 +297,17 @@ const preferDefault = m => (m && m.default) || m
   return true
 }
 
-const debouncedWriteAll = _.debounce(
-  async (): Promise<void> => {
-    const activity = reporter.activityTimer(`write out requires`, {
-      id: `requires-writer`,
-    })
-    activity.start()
-    await writeAll(store.getState())
-    activity.end()
-  },
-  500,
-  {
-    // using "leading" can cause double `writeAll` call - particularly
-    // when refreshing data using `/__refresh` hook.
-    leading: false,
-  }
-)
-
 /**
- * Start listening to CREATE/DELETE_PAGE events so we can rewrite
+ * Start listening to CREATE_SSR_VISITED_PAGE events so we can rewrite
  * files as required
  */
+let listenerCreated = false
 export const startListener = (): void => {
-  emitter.on(`CREATE_PAGE`, (): void => {
-    reporter.pendingActivity({ id: `requires-writer` })
-    debouncedWriteAll()
-  })
-
-  emitter.on(`CREATE_PAGE_END`, (): void => {
-    reporter.pendingActivity({ id: `requires-writer` })
-    debouncedWriteAll()
-  })
-
-  emitter.on(`DELETE_PAGE`, (): void => {
-    reporter.pendingActivity({ id: `requires-writer` })
-    debouncedWriteAll()
-  })
-
-  emitter.on(`DELETE_PAGE_BY_PATH`, (): void => {
-    reporter.pendingActivity({ id: `requires-writer` })
-    debouncedWriteAll()
-  })
+  if (!listenerCreated) {
+    emitter.on(`CREATE_SSR_VISITED_PAGE`, (): void => {
+      reporter.pendingActivity({ id: `requires-writer` })
+      writeAll(store.getState())
+    })
+    listenerCreated = true
+  }
 }
diff --git a/packages/gatsby/src/redux/actions/public.js b/packages/gatsby/src/redux/actions/public.js
index 6b4e9d0bfd279..53e05425ef7f9 100644
--- a/packages/gatsby/src/redux/actions/public.js
+++ b/packages/gatsby/src/redux/actions/public.js
@@ -1391,4 +1391,17 @@ actions.removePageData = (id: PageDataRemove) => {
   }
 }
 
+/**
+ * Record that a page was SSRed.
+ *
+ * @param {Object} $0
+ * @param {string} $0.id the path to the page.
+ */
+actions.createSSRVisitedPage = (page: Page) => {
+  return {
+    type: `CREATE_SSR_VISITED_PAGE`,
+    payload: page,
+  }
+}
+
 module.exports = { actions }
diff --git a/packages/gatsby/src/redux/reducers/index.ts b/packages/gatsby/src/redux/reducers/index.ts
index b8fdcc55279d8..7b5816cfd06d0 100644
--- a/packages/gatsby/src/redux/reducers/index.ts
+++ b/packages/gatsby/src/redux/reducers/index.ts
@@ -1,6 +1,7 @@
 import { nodesReducer } from "./nodes"
 import { reducer as logReducer } from "gatsby-cli/lib/reporter/redux/reducer"
 import { pagesReducer } from "./pages"
+import { ssrVisitedPageReducer } from "./ssr-visited-page"
 import { redirectsReducer } from "./redirects"
 import { schemaReducer } from "./schema"
 import { definitionsReducer } from "./definitions"
@@ -43,6 +44,7 @@ export {
   configReducer as config,
   schemaReducer as schema,
   pagesReducer as pages,
+  ssrVisitedPageReducer as ssrVisitedPages,
   statusReducer as status,
   componentDataDependenciesReducer as componentDataDependencies,
   componentsReducer as components,
diff --git a/packages/gatsby/src/redux/reducers/ssr-visited-page.ts b/packages/gatsby/src/redux/reducers/ssr-visited-page.ts
new file mode 100644
index 0000000000000..16ab31074226d
--- /dev/null
+++ b/packages/gatsby/src/redux/reducers/ssr-visited-page.ts
@@ -0,0 +1,24 @@
+import {
+  IGatsbyState,
+  IDeleteCacheAction,
+  ICreateSSRVisitedPage,
+} from "../types"
+
+export const ssrVisitedPageReducer = (
+  state: IGatsbyState["ssrVisitedPages"] = new Set<string>(),
+  action: IDeleteCacheAction | ICreateSSRVisitedPage
+): IGatsbyState["ssrVisitedPages"] => {
+  switch (action.type) {
+    case `DELETE_CACHE`:
+      return new Set()
+
+    case `CREATE_SSR_VISITED_PAGE`: {
+      state.add(action.payload.componentChunkName)
+
+      return state
+    }
+
+    default:
+      return state
+  }
+}
diff --git a/packages/gatsby/src/redux/types.ts b/packages/gatsby/src/redux/types.ts
index fadd072fa69e8..0de09a86e2e0a 100644
--- a/packages/gatsby/src/redux/types.ts
+++ b/packages/gatsby/src/redux/types.ts
@@ -259,6 +259,7 @@ export interface IGatsbyState {
   }
   pageDataStats: Map<SystemPath, number>
   pageData: Map<Identifier, string>
+  ssrVisitedPages: Set<string>
 }
 
 export interface ICachedReduxState {
@@ -555,6 +556,12 @@ interface ISetSchemaComposerAction {
   payload: SchemaComposer<any>
 }
 
+export interface ICreateSSRVisitedPage {
+  type: `CREATE_SSR_VISITED_PAGE`
+  payload: IGatsbyPage
+  plugin?: IGatsbyPlugin
+}
+
 export interface ICreatePageAction {
   type: `CREATE_PAGE`
   payload: IGatsbyPage
diff --git a/packages/gatsby/src/utils/dev-ssr/develop-html-route.ts b/packages/gatsby/src/utils/dev-ssr/develop-html-route.ts
index 50fec4ddadb63..3fbf4d1b04401 100644
--- a/packages/gatsby/src/utils/dev-ssr/develop-html-route.ts
+++ b/packages/gatsby/src/utils/dev-ssr/develop-html-route.ts
@@ -21,11 +21,20 @@ export const route = ({ app, program, store }): any =>
     try {
       const renderResponse = await renderDevHTML({
         path: pathObj.path,
+        page: pathObj,
+        store,
         htmlComponentRendererPath: `${program.directory}/public/render-page.js`,
         directory: program.directory,
       })
       res.status(200).send(renderResponse)
     } catch (error) {
+      // THe page errored but couldn't read the page component.
+      // This is a race condition when a page is deleted but its requested
+      // immediately after before anything can recompile.
+      if (error === `404 page`) {
+        return next()
+      }
+
       report.error({
         id: `11614`,
         filePath: error.filename,
@@ -53,7 +62,6 @@ export const route = ({ app, program, store }): any =>
         <pre style="background:#fdfaf6;padding:8px;">${error.codeFrame}</pre>`)
     }
 
-    // TODO add support for 404 and general rendering errors
     htmlActivity.end()
 
     // Make eslint happy
diff --git a/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js b/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js
index bccf4d84ab390..efae993b5cbde 100644
--- a/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js
+++ b/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js
@@ -67,8 +67,9 @@ const parseError = function (err, directory) {
   try {
     code = fs.readFileSync(filename, `utf-8`)
   } catch (e) {
-    console.log(err)
-    console.log(`Couldn't read the file ${filename}`, e)
+    console.log(`Couldn't read the file ${filename}`)
+    console.log(`original error`, err)
+    return `404 page`
   }
   const line = position.line
   const row = position.row
@@ -97,7 +98,7 @@ const parseError = function (err, directory) {
   const message = splitMessage[splitMessage.length - 1]
   const type = err.type ? err.type : err.name
   const data = {
-    filename: filename,
+    filename: sysPath.relative(directory, filename),
     code,
     codeFrame,
     line: line,
@@ -113,24 +114,30 @@ exports.parseError = parseError
 
 exports.renderHTML = ({ path, htmlComponentRendererPath, directory }) =>
   new Promise((resolve, reject) => {
+    require(`source-map-support`).install()
     try {
       const htmlComponentRenderer = require(htmlComponentRendererPath)
       htmlComponentRenderer.default(path, (_throwAway, htmlString) => {
-        // console.log(`rendered correctly`, { htmlString, attempts })
         resolve(htmlString)
       })
     } catch (err) {
       const stack = err.stack ? err.stack : ``
+
       // Only generate error pages for webpack errors. If it's not a webpack
       // error, it's not a user error so probably a system error so we'll just
       // panic and quit.
       const regex = /webpack:\/lib\//gm
-      if (!stack.match(regex)) {
+      const moduleBuildFailed = /Module.build.failed/gm
+      if (stack.match(moduleBuildFailed)) {
+        reject(`500 page`)
+      } else if (!stack.match(regex)) {
+        console.log(`unexpected error while SSRing the path: ${path}`)
         console.log(err)
-        return
+        reject(err)
+      } else {
+        const error = parseError(err, directory)
+        reject(error)
       }
-      const error = parseError(err, directory)
-      reject(error)
     }
   })
 
diff --git a/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts b/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
index 51182ed595262..7ea133a5eb203 100644
--- a/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
+++ b/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
@@ -2,14 +2,16 @@ import JestWorker from "jest-worker"
 import fs from "fs-extra"
 import { joinPath } from "gatsby-core-utils"
 import report from "gatsby-cli/lib/reporter"
+import _ from "lodash"
 
-import { store } from "../../redux"
-import { writeModule } from "../gatsby-webpack-virtual-modules"
+import { startListener } from "../../bootstrap/requires-writer"
+import { findPageByPath } from "../find-page-by-path"
+import { boundActionCreators } from "../../redux/actions"
+const { createSSRVisitedPage } = boundActionCreators
 
 const startWorker = (): any => {
   const newWorker = new JestWorker(require.resolve(`./render-dev-html-child`), {
     exposedMethods: [`renderHTML`, `deleteModuleCache`, `warmup`],
-    enableWorkerThreads: true,
     numWorkers: 1,
     forkOptions: { silent: false },
   })
@@ -45,44 +47,6 @@ export const restartWorker = (htmlComponentRendererPath): void => {
   }
 }
 
-interface IGatsbyPageComponent {
-  component: string
-  componentChunkName: string
-}
-
-// TODO: Remove all "hot" references in this `syncRequires` variable when fast-refresh is the default
-const hotImport =
-  process.env.GATSBY_HOT_LOADER !== `fast-refresh`
-    ? `const { hot } = require("react-hot-loader/root")`
-    : ``
-const hotMethod = process.env.GATSBY_HOT_LOADER !== `fast-refresh` ? `hot` : ``
-
-const writeLazyRequires = (pageComponents): void => {
-  // Create file with sync requires of components/json files.
-  let lazySyncRequires = `${hotImport}
-
-// prefer default export if available
-const preferDefault = m => (m && m.default) || m
-\n\n`
-  lazySyncRequires += `exports.lazyComponents = {\n${[
-    ...pageComponents.values(),
-  ]
-    .map(
-      (c: IGatsbyPageComponent): string =>
-        `  "${
-          c.componentChunkName
-        }": ${hotMethod}(preferDefault(require("${joinPath(c.component)}")))`
-    )
-    .join(`,\n`)}
-}\n\n`
-
-  writeModule(`$virtual/lazy-sync-requires`, lazySyncRequires)
-}
-
-const pageComponents = new Map()
-const pageComponentsWritten = new Set()
-const inFlightPromises = new Map()
-
 const searchFileForString = (substring, filePath): Promise<boolean> =>
   new Promise(resolve => {
     const stream = fs.createReadStream(filePath)
@@ -103,92 +67,85 @@ const searchFileForString = (substring, filePath): Promise<boolean> =>
   })
 
 const ensurePathComponentInSSRBundle = async (
-  path,
+  page,
   directory
 ): Promise<any> => {
-  const pages = [...store.getState().pages.values()]
-  const page = pages.find(p => p.path === path)
-
   // This shouldn't happen.
   if (!page) {
     report.panic(`page not found`)
   }
 
-  // If we know it's written, return.
-  if (pageComponentsWritten.has(page.componentChunkName)) {
-    return true
-  }
-
-  let promiseResolve
-  // If we're already handling this path, return its promise.
-  if (inFlightPromises.has(page.componentChunkName)) {
-    return inFlightPromises.get(page.componentChunkName)
-  } else {
-    const promise = new Promise(function (resolve) {
-      promiseResolve = resolve
+  // Now check if it's written to public/render-page.js
+  const htmlComponentRendererPath = joinPath(directory, `public/render-page.js`)
+  // This search takes 1-10ms
+  // We do it as there can be a race conditions where two pages
+  // are requested at the same time which means that both are told render-page.js
+  // has changed when the first page is complete meaning the second
+  // page's component won't be in the render meaning its SSR will fail.
+  let found = await searchFileForString(
+    page.componentChunkName,
+    htmlComponentRendererPath
+  )
+
+  if (!found) {
+    await new Promise(resolve => {
+      let readAttempts = 0
+      const searchForStringInterval = setInterval(async () => {
+        readAttempts += 1
+        found = await searchFileForString(
+          page.componentChunkName,
+          htmlComponentRendererPath
+        )
+        if (found || readAttempts === 5) {
+          clearInterval(searchForStringInterval)
+          resolve()
+        }
+      }, 300)
     })
-
-    inFlightPromises.set(page.componentChunkName, promise)
   }
 
-  // Write out the component information to lazy-sync-requires
-  pageComponents.set(page.component, {
-    component: page.component,
-    componentChunkName: page.componentChunkName,
-  })
-  writeLazyRequires(pageComponents)
-
-  // Now wait for it to be written to public/render-page.js
-  const htmlComponentRendererPath = joinPath(directory, `public/render-page.js`)
-  const watcher = fs.watch(htmlComponentRendererPath, async () => {
-    // This search takes 1-10ms
-    // We do it as there can be a race conditions where two pages
-    // are requested at the same time which means that both are told render-page.js
-    // has changed when the first page is complete meaning the second
-    // page's component won't be in the render meaning its SSR will fail.
-    const found = await searchFileForString(
-      page.componentChunkName,
-      htmlComponentRendererPath
-    )
-    if (found) {
-      // It's changed, clean up the watcher.
-      watcher.close()
-      // Make sure the worker is ready and then resolve.
-      worker.deleteModuleCache(htmlComponentRendererPath).then(promiseResolve)
-    }
-  })
-
-  // We're done, delete the promise from the in-flight promises
-  // and add it to the set of done paths.
-  inFlightPromises.get(page.componentChunkName).then(() => {
-    inFlightPromises.delete(page.componentChunkName)
-    pageComponentsWritten.add(page.componentChunkName)
-  })
-
-  // Return the promise for the first request.
-  return inFlightPromises.get(page.componentChunkName)
+  return found
 }
 
-// Initialize the virtual module.
-writeLazyRequires(pageComponents)
-
 export const renderDevHTML = ({
   path,
+  page,
+  store,
   htmlComponentRendererPath,
   directory,
 }): Promise<string> =>
   new Promise(async (resolve, reject) => {
-    try {
+    startListener()
+    let pageObj
+    if (!page) {
+      pageObj = findPageByPath(store.getState(), path)
+    } else {
+      pageObj = page
+    }
+
+    // Record this page was requested. This will kick off adding its page
+    // component to the ssr bundle (if that's not already happened)
+    createSSRVisitedPage(pageObj)
+
+    let pageFound = true
+    if (process.env.GATSBY_EXPERIMENTAL_DEV_SSR) {
       // Write component to file & wait for public/render-page.js to update
-      await ensurePathComponentInSSRBundle(path, directory)
+      pageFound = await ensurePathComponentInSSRBundle(pageObj, directory)
+      if (!pageFound) {
+        reject(`404 page`)
+      }
+    }
 
-      const htmlString = await worker.renderHTML({
-        path,
-        htmlComponentRendererPath,
-        directory,
-      })
-      resolve(htmlString)
-    } catch (error) {
-      reject(error)
+    if (pageFound) {
+      try {
+        const htmlString = await worker.renderHTML({
+          path,
+          htmlComponentRendererPath,
+          directory,
+        })
+        resolve(htmlString)
+      } catch (error) {
+        reject(error)
+      }
     }
   })
diff --git a/packages/gatsby/src/utils/start-server.ts b/packages/gatsby/src/utils/start-server.ts
index a4ae54b8f608c..8b1f63dfb0e59 100644
--- a/packages/gatsby/src/utils/start-server.ts
+++ b/packages/gatsby/src/utils/start-server.ts
@@ -271,12 +271,16 @@ export async function startServer(
       try {
         const renderResponse = await renderDevHTML({
           path: genericHtmlPath,
+          // Let renderDevHTML figure it out.
+          page: undefined,
+          store,
           htmlComponentRendererPath: `${program.directory}/public/render-page.js`,
           directory: program.directory,
         })
-        res.status(200).send(renderResponse)
+        const status = process.env.GATSBY_EXPERIMENTAL_DEV_SSR ? 404 : 200
+        res.status(status).send(renderResponse)
       } catch (e) {
-        report.panic(e)
+        report.error(e)
         res.send(e).status(500)
       }
     }
diff --git a/packages/gatsby/src/utils/webpack.config.js b/packages/gatsby/src/utils/webpack.config.js
index d2c2cf96dc37b..8aee86277f10d 100644
--- a/packages/gatsby/src/utils/webpack.config.js
+++ b/packages/gatsby/src/utils/webpack.config.js
@@ -172,7 +172,9 @@ module.exports = async (
         }
       case `develop-html`:
         return {
-          main: directoryPath(`.cache/develop-static-entry`),
+          main: process.env.GATSBY_EXPERIMENTAL_DEV_SSR
+            ? directoryPath(`.cache/new-develop-static-entry`)
+            : directoryPath(`.cache/develop-static-entry`),
         }
       case `build-html`:
         return {

From 2ffb15d0c0908059d9a6f0c7b62465016f1efeb1 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Sat, 31 Oct 2020 13:29:08 -0700
Subject: [PATCH 113/123] update tests

---
 .../gatsby/src/bootstrap/__tests__/requires-writer.js  | 10 ++++++++++
 .../__tests__/__snapshots__/develop-html-route.ts.snap |  2 +-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/packages/gatsby/src/bootstrap/__tests__/requires-writer.js b/packages/gatsby/src/bootstrap/__tests__/requires-writer.js
index b82fa698dc0e6..047f089ede4b9 100644
--- a/packages/gatsby/src/bootstrap/__tests__/requires-writer.js
+++ b/packages/gatsby/src/bootstrap/__tests__/requires-writer.js
@@ -30,6 +30,8 @@ describe(`requires-writer`, () => {
   const program = {
     directory: `/dir`,
   }
+  const ssrVisitedPages = new Set(`hi`)
+
   let originalDateNow = global.Date.now
 
   beforeEach(() => {
@@ -61,6 +63,7 @@ describe(`requires-writer`, () => {
       await requiresWriter.writeAll({
         pages,
         program,
+        ssrVisitedPages,
       })
 
       expect(spy).toBeCalledWith(
@@ -108,6 +111,7 @@ describe(`requires-writer`, () => {
       await requiresWriter.writeAll({
         pages,
         program,
+        ssrVisitedPages,
       })
 
       expect(matchPaths[0].path).toBe(pages.get(`/app/login/`).path)
@@ -141,6 +145,7 @@ describe(`requires-writer`, () => {
       await requiresWriter.writeAll({
         pages,
         program,
+        ssrVisitedPages,
       })
 
       expect(matchPaths[0].path).toBe(pages.get(`/app/clients/static`).path)
@@ -165,6 +170,7 @@ describe(`requires-writer`, () => {
       await requiresWriter.writeAll({
         pages,
         program,
+        ssrVisitedPages,
       })
 
       expect(matchPaths[0].path).toBe(pages.get(`/`).path)
@@ -219,6 +225,7 @@ describe(`requires-writer`, () => {
       await requiresWriter.writeAll({
         pages,
         program,
+        ssrVisitedPages,
       })
 
       expect(matchPaths.map(p => p.matchPath)).toMatchInlineSnapshot(`
@@ -245,6 +252,7 @@ describe(`requires-writer`, () => {
       await requiresWriter.writeAll({
         pages,
         program,
+        ssrVisitedPages,
       })
 
       const matchPathsForInvertedInput = matchPaths
@@ -252,6 +260,7 @@ describe(`requires-writer`, () => {
       await requiresWriter.writeAll({
         pages,
         program,
+        ssrVisitedPages,
       })
 
       expect(matchPathsForInvertedInput).toEqual(matchPaths)
@@ -263,6 +272,7 @@ describe(`requires-writer`, () => {
         await requiresWriter.writeAll({
           pages,
           program,
+          ssrVisitedPages,
         })
 
         const allMatchingPages = matchPaths
diff --git a/packages/gatsby/src/utils/__tests__/__snapshots__/develop-html-route.ts.snap b/packages/gatsby/src/utils/__tests__/__snapshots__/develop-html-route.ts.snap
index 84675989a68f5..0877e20af053a 100644
--- a/packages/gatsby/src/utils/__tests__/__snapshots__/develop-html-route.ts.snap
+++ b/packages/gatsby/src/utils/__tests__/__snapshots__/develop-html-route.ts.snap
@@ -131,7 +131,7 @@ export const pageQuery = graphql\`
 <span style=\\"font-weight:normal;opacity:1;color:#452475;background:#fdfaf6;\\"> <span style=\\"color:#527713;\\"> 17 | </span>    let tags</span>
 <span style=\\"font-weight:normal;opacity:1;color:#452475;background:#fdfaf6;\\"> <span style=\\"color:#527713;\\"> 18 | </span>    let tagsSection</span>
 <span style=\\"font-weight:normal;opacity:1;color:#452475;background:#fdfaf6;\\"> <span style=\\"color:#527713;\\"> 19 | </span>    <span style=\\"color:#006500;\\">if</span> (<span style=\\"color:#006500;\\">this</span><span style=\\"color:#DB3A00;\\">.</span>props<span style=\\"color:#DB3A00;\\">.</span>data<span style=\\"color:#DB3A00;\\">.</span>markdownRemark<span style=\\"color:#DB3A00;\\">.</span>fields<span style=\\"color:#DB3A00;\\">.</span>tagSlugs) {</span>",
-  "filename": "<PROJECT_ROOT>/packages/gatsby/src/utils/__tests__/fixtures/blog-post.js",
+  "filename": "fixtures/blog-post.js",
   "line": 16,
   "message": "window is not defined",
   "row": 17,

From d58b8865e9e628c4ef0cb5bd00d70861976a3fb9 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Sat, 31 Oct 2020 13:57:26 -0700
Subject: [PATCH 114/123] Don't render body of pages w/ matchPath

---
 .../cache-dir/new-develop-static-entry.js     |  4 ++--
 .../utils/dev-ssr/render-dev-html-child.js    | 23 +++++++++++++++----
 .../src/utils/dev-ssr/render-dev-html.ts      |  6 +++++
 3 files changed, 27 insertions(+), 6 deletions(-)

diff --git a/packages/gatsby/cache-dir/new-develop-static-entry.js b/packages/gatsby/cache-dir/new-develop-static-entry.js
index a061ac77e5bc7..6f2295f235a9a 100644
--- a/packages/gatsby/cache-dir/new-develop-static-entry.js
+++ b/packages/gatsby/cache-dir/new-develop-static-entry.js
@@ -33,7 +33,7 @@ try {
 
 Html = Html && Html.__esModule ? Html.default : Html
 
-export default (pagePath, callback) => {
+export default (pagePath, isClientOnlyPage, callback) => {
   let bodyHtml = ``
   let headComponents = [
     <meta key="environment" name="note" content="environment=development" />,
@@ -212,7 +212,7 @@ export default (pagePath, callback) => {
     return bodyHtml
   }
 
-  const bodyStr = generateBodyHTML()
+  const bodyStr = isClientOnlyPage ? `` : generateBodyHTML()
 
   const htmlElement = React.createElement(Html, {
     ...bodyProps,
diff --git a/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js b/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js
index efae993b5cbde..0c44cd1039bf3 100644
--- a/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js
+++ b/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js
@@ -112,14 +112,29 @@ const parseError = function (err, directory) {
 
 exports.parseError = parseError
 
-exports.renderHTML = ({ path, htmlComponentRendererPath, directory }) =>
+exports.renderHTML = ({
+  path,
+  htmlComponentRendererPath,
+  isClientOnlyPage = false,
+  directory,
+}) =>
   new Promise((resolve, reject) => {
     require(`source-map-support`).install()
     try {
       const htmlComponentRenderer = require(htmlComponentRendererPath)
-      htmlComponentRenderer.default(path, (_throwAway, htmlString) => {
-        resolve(htmlString)
-      })
+      if (process.env.GATSBY_EXPERIMENTAL_DEV_SSR) {
+        htmlComponentRenderer.default(
+          path,
+          isClientOnlyPage,
+          (_throwAway, htmlString) => {
+            resolve(htmlString)
+          }
+        )
+      } else {
+        htmlComponentRenderer.default(path, (_throwAway, htmlString) => {
+          resolve(htmlString)
+        })
+      }
     } catch (err) {
       const stack = err.stack ? err.stack : ``
 
diff --git a/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts b/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
index 7ea133a5eb203..9611c0745d698 100644
--- a/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
+++ b/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
@@ -123,6 +123,11 @@ export const renderDevHTML = ({
       pageObj = page
     }
 
+    let isClientOnlyPage = false
+    if (pageObj.matchPath) {
+      isClientOnlyPage = true
+    }
+
     // Record this page was requested. This will kick off adding its page
     // component to the ssr bundle (if that's not already happened)
     createSSRVisitedPage(pageObj)
@@ -142,6 +147,7 @@ export const renderDevHTML = ({
           path,
           htmlComponentRendererPath,
           directory,
+          isClientOnlyPage,
         })
         resolve(htmlString)
       } catch (error) {

From 414430b157c65a696103381debba65d885ae49d7 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Sat, 31 Oct 2020 13:59:15 -0700
Subject: [PATCH 115/123] use core util joinPath so works on windows

---
 integration-tests/ssr/__tests__/ssr.js              |  6 +++---
 packages/gatsby/cache-dir/__tests__/static-entry.js | 10 +++++-----
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/integration-tests/ssr/__tests__/ssr.js b/integration-tests/ssr/__tests__/ssr.js
index f9a2a80e8811b..9408dfe444f61 100644
--- a/integration-tests/ssr/__tests__/ssr.js
+++ b/integration-tests/ssr/__tests__/ssr.js
@@ -1,7 +1,7 @@
 const fetch = require(`node-fetch`)
 const { execFile } = require("child_process")
 const fs = require(`fs-extra`)
-const path = require(`path`)
+import { joinPath } from "gatsby-core-utils"
 
 describe(`SSR`, () => {
   test(`is run for a page when it is requested`, async () => {
@@ -22,8 +22,8 @@ describe(`SSR`, () => {
     expect(exitCode).toEqual({ exitCode: 0 })
   })
   test(`it generates an error page correctly`, async () => {
-    const src = path.join(__dirname, `/fixtures/bad-page.js`)
-    const dest = path.join(__dirname, `../src/pages/bad-page.js`)
+    const src = joinPath(__dirname, `/fixtures/bad-page.js`)
+    const dest = joinPath(__dirname, `../src/pages/bad-page.js`)
     const result = fs.copySync(src, dest)
 
     const pageUrl = `http://localhost:8000/bad-page/`
diff --git a/packages/gatsby/cache-dir/__tests__/static-entry.js b/packages/gatsby/cache-dir/__tests__/static-entry.js
index 04367853d9202..5b0e89fd4d813 100644
--- a/packages/gatsby/cache-dir/__tests__/static-entry.js
+++ b/packages/gatsby/cache-dir/__tests__/static-entry.js
@@ -160,7 +160,7 @@ describe(`develop-static-entry`, () => {
   test(`onPreRenderHTML can be used to replace headComponents`, done => {
     global.plugins = [fakeStylesPlugin, reverseHeadersPlugin]
 
-    DevelopStaticEntry(`/about/`, (_, html) => {
+    DevelopStaticEntry(`/about/`, false, (_, html) => {
       expect(html).toMatchSnapshot()
       done()
     })
@@ -172,7 +172,7 @@ describe(`develop-static-entry`, () => {
       reverseBodyComponentsPluginFactory(`Post`),
     ]
 
-    DevelopStaticEntry(`/about/`, (_, html) => {
+    DevelopStaticEntry(`/about/`, false, (_, html) => {
       expect(html).toMatchSnapshot()
       done()
     })
@@ -184,14 +184,14 @@ describe(`develop-static-entry`, () => {
       reverseBodyComponentsPluginFactory(`Pre`),
     ]
 
-    DevelopStaticEntry(`/about/`, (_, html) => {
+    DevelopStaticEntry(`/about/`, false, (_, html) => {
       expect(html).toMatchSnapshot()
       done()
     })
   })
 
   test(`onPreRenderHTML adds metatag note for development environment`, done => {
-    DevelopStaticEntry(`/about/`, (_, html) => {
+    DevelopStaticEntry(`/about/`, false, (_, html) => {
       expect(html).toContain(
         `<meta name="note" content="environment=development"/>`
       )
@@ -202,7 +202,7 @@ describe(`develop-static-entry`, () => {
   test(`onPreRenderHTML adds metatag note for development environment after replaceHeadComponents`, done => {
     global.plugins = [reverseHeadersPlugin]
 
-    DevelopStaticEntry(`/about/`, (_, html) => {
+    DevelopStaticEntry(`/about/`, false, (_, html) => {
       expect(html).toContain(
         `<meta name="note" content="environment=development"/>`
       )

From 67c79b64b537e56fa89d82ba1b78fc893ab378d5 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Sat, 31 Oct 2020 14:32:36 -0700
Subject: [PATCH 116/123] try again

---
 integration-tests/ssr/__tests__/ssr.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/integration-tests/ssr/__tests__/ssr.js b/integration-tests/ssr/__tests__/ssr.js
index 9408dfe444f61..c1bdb83fc1842 100644
--- a/integration-tests/ssr/__tests__/ssr.js
+++ b/integration-tests/ssr/__tests__/ssr.js
@@ -1,5 +1,5 @@
 const fetch = require(`node-fetch`)
-const { execFile } = require("child_process")
+const { execFile } = require(`child_process`)
 const fs = require(`fs-extra`)
 import { joinPath } from "gatsby-core-utils"
 
@@ -23,7 +23,7 @@ describe(`SSR`, () => {
   })
   test(`it generates an error page correctly`, async () => {
     const src = joinPath(__dirname, `/fixtures/bad-page.js`)
-    const dest = joinPath(__dirname, `../src/pages/bad-page.js`)
+    const dest = joinPath(__dirname, `/../src/pages/bad-page.js`)
     const result = fs.copySync(src, dest)
 
     const pageUrl = `http://localhost:8000/bad-page/`

From f70c177c093ff57bc92e5e31de7b2acfe96ca811 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Sat, 31 Oct 2020 14:48:23 -0700
Subject: [PATCH 117/123] try try again

---
 integration-tests/ssr/__tests__/ssr.js | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/integration-tests/ssr/__tests__/ssr.js b/integration-tests/ssr/__tests__/ssr.js
index c1bdb83fc1842..fe21392860eaa 100644
--- a/integration-tests/ssr/__tests__/ssr.js
+++ b/integration-tests/ssr/__tests__/ssr.js
@@ -1,7 +1,7 @@
 const fetch = require(`node-fetch`)
-const { execFile } = require(`child_process`)
+const { execFile } = require("child_process")
 const fs = require(`fs-extra`)
-import { joinPath } from "gatsby-core-utils"
+const path = require(`path`)
 
 describe(`SSR`, () => {
   test(`is run for a page when it is requested`, async () => {
@@ -22,9 +22,9 @@ describe(`SSR`, () => {
     expect(exitCode).toEqual({ exitCode: 0 })
   })
   test(`it generates an error page correctly`, async () => {
-    const src = joinPath(__dirname, `/fixtures/bad-page.js`)
-    const dest = joinPath(__dirname, `/../src/pages/bad-page.js`)
-    const result = fs.copySync(src, dest)
+    const src = path.resolve(`./__tests__/fixtures/bad-page.js`)
+    const dest = path.resolve(`./src/pages/bad-page.js`)
+    fs.copySync(src, dest)
 
     const pageUrl = `http://localhost:8000/bad-page/`
     await new Promise(resolve => {

From 867bc42c16c8d17ea88d05cd22bfd8ea4d6afe58 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Sat, 31 Oct 2020 15:02:13 -0700
Subject: [PATCH 118/123] remove mistakenly added file

---
 integration-tests/ssr/__tests__../src/pages/bad-page.js | 9 ---------
 integration-tests/ssr/__tests__/ssr.js                  | 4 ++--
 2 files changed, 2 insertions(+), 11 deletions(-)
 delete mode 100644 integration-tests/ssr/__tests__../src/pages/bad-page.js

diff --git a/integration-tests/ssr/__tests__../src/pages/bad-page.js b/integration-tests/ssr/__tests__../src/pages/bad-page.js
deleted file mode 100644
index 429c52813c049..0000000000000
--- a/integration-tests/ssr/__tests__../src/pages/bad-page.js
+++ /dev/null
@@ -1,9 +0,0 @@
-import React from "react"
-
-const Component = () => {
-  const a = window.width
-
-  return <div>hi</div>
-}
-
-export default Component
diff --git a/integration-tests/ssr/__tests__/ssr.js b/integration-tests/ssr/__tests__/ssr.js
index fe21392860eaa..6315c3c1e2fc3 100644
--- a/integration-tests/ssr/__tests__/ssr.js
+++ b/integration-tests/ssr/__tests__/ssr.js
@@ -22,8 +22,8 @@ describe(`SSR`, () => {
     expect(exitCode).toEqual({ exitCode: 0 })
   })
   test(`it generates an error page correctly`, async () => {
-    const src = path.resolve(`./__tests__/fixtures/bad-page.js`)
-    const dest = path.resolve(`./src/pages/bad-page.js`)
+    const src = path.join(__dirname, `/fixtures/bad-page.js`)
+    const dest = path.join(__dirname, `../src/pages/bad-page.js`)
     fs.copySync(src, dest)
 
     const pageUrl = `http://localhost:8000/bad-page/`

From 0462eb829488fd84547d914b52a7bb5d6f5dc1bf Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Sat, 31 Oct 2020 15:39:49 -0700
Subject: [PATCH 119/123] fix pnp test

---
 integration-tests/ssr/__tests__/ssr.js           | 16 +++++++++-------
 packages/gatsby/package.json                     |  1 +
 .../src/utils/dev-ssr/render-dev-html-child.js   |  2 +-
 3 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/integration-tests/ssr/__tests__/ssr.js b/integration-tests/ssr/__tests__/ssr.js
index 6315c3c1e2fc3..9c5202a71f498 100644
--- a/integration-tests/ssr/__tests__/ssr.js
+++ b/integration-tests/ssr/__tests__/ssr.js
@@ -46,13 +46,15 @@ describe(`SSR`, () => {
 
     // After the page is gone, it'll 404.
     await new Promise(resolve => {
-      const testInterval = setInterval(() => {
-        fetch(pageUrl).then(res => {
-          if (res.status === 404) {
-            clearInterval(testInterval)
-            resolve()
-          }
-        })
+      setTimeout(() => {
+        const testInterval = setInterval(() => {
+          fetch(pageUrl).then(res => {
+            if (res.status === 404) {
+              clearInterval(testInterval)
+              resolve()
+            }
+          })
+        }, 400)
       }, 400)
     })
   })
diff --git a/packages/gatsby/package.json b/packages/gatsby/package.json
index 74e51647ffbef..d893f79b67bf8 100644
--- a/packages/gatsby/package.json
+++ b/packages/gatsby/package.json
@@ -138,6 +138,7 @@
     "slugify": "^1.4.4",
     "socket.io": "^2.3.0",
     "socket.io-client": "2.3.0",
+    "source-map-support": "^0.5.19",
     "st": "^2.0.0",
     "stack-trace": "^0.0.10",
     "string-similarity": "^1.2.2",
diff --git a/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js b/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js
index 0c44cd1039bf3..7c38ccb8934d0 100644
--- a/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js
+++ b/packages/gatsby/src/utils/dev-ssr/render-dev-html-child.js
@@ -1,3 +1,4 @@
+require(`source-map-support`).install()
 const { codeFrameColumns } = require(`@babel/code-frame`)
 const ansiHTML = require(`ansi-html`)
 const fs = require(`fs-extra`)
@@ -119,7 +120,6 @@ exports.renderHTML = ({
   directory,
 }) =>
   new Promise((resolve, reject) => {
-    require(`source-map-support`).install()
     try {
       const htmlComponentRenderer = require(htmlComponentRendererPath)
       if (process.env.GATSBY_EXPERIMENTAL_DEV_SSR) {

From d7ed32d7566a25018b9934cc293c7816ec736c0e Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Mon, 9 Nov 2020 10:48:48 -0800
Subject: [PATCH 120/123] Move lazy ssr bundling code to own branch

---
 .../bootstrap/__tests__/requires-writer.js    | 14 ++-
 .../gatsby/src/bootstrap/requires-writer.ts   | 89 ++++++++++---------
 packages/gatsby/src/redux/actions/public.js   | 13 +++
 packages/gatsby/src/redux/reducers/index.ts   |  2 +
 .../src/redux/reducers/ssr-visited-page.ts    | 24 +++++
 packages/gatsby/src/redux/types.ts            | 10 ++-
 6 files changed, 106 insertions(+), 46 deletions(-)
 create mode 100644 packages/gatsby/src/redux/reducers/ssr-visited-page.ts

diff --git a/packages/gatsby/src/bootstrap/__tests__/requires-writer.js b/packages/gatsby/src/bootstrap/__tests__/requires-writer.js
index b82fa698dc0e6..cbb446634a47c 100644
--- a/packages/gatsby/src/bootstrap/__tests__/requires-writer.js
+++ b/packages/gatsby/src/bootstrap/__tests__/requires-writer.js
@@ -5,7 +5,7 @@ const { match } = require(`@reach/router/lib/utils`)
 const now = Date.now()
 
 const generatePagesState = pages => {
-  let state = new Map()
+  const state = new Map()
   pages.forEach(page => {
     state.set(page.path, {
       component: ``,
@@ -30,7 +30,9 @@ describe(`requires-writer`, () => {
   const program = {
     directory: `/dir`,
   }
-  let originalDateNow = global.Date.now
+  const ssrVisitedPages = new Set(`hi`)
+
+  const originalDateNow = global.Date.now
 
   beforeEach(() => {
     global.Date.now = () => now
@@ -61,6 +63,7 @@ describe(`requires-writer`, () => {
       await requiresWriter.writeAll({
         pages,
         program,
+        ssrVisitedPages,
       })
 
       expect(spy).toBeCalledWith(
@@ -108,6 +111,7 @@ describe(`requires-writer`, () => {
       await requiresWriter.writeAll({
         pages,
         program,
+        ssrVisitedPages,
       })
 
       expect(matchPaths[0].path).toBe(pages.get(`/app/login/`).path)
@@ -141,6 +145,7 @@ describe(`requires-writer`, () => {
       await requiresWriter.writeAll({
         pages,
         program,
+        ssrVisitedPages,
       })
 
       expect(matchPaths[0].path).toBe(pages.get(`/app/clients/static`).path)
@@ -165,6 +170,7 @@ describe(`requires-writer`, () => {
       await requiresWriter.writeAll({
         pages,
         program,
+        ssrVisitedPages,
       })
 
       expect(matchPaths[0].path).toBe(pages.get(`/`).path)
@@ -219,6 +225,7 @@ describe(`requires-writer`, () => {
       await requiresWriter.writeAll({
         pages,
         program,
+        ssrVisitedPages,
       })
 
       expect(matchPaths.map(p => p.matchPath)).toMatchInlineSnapshot(`
@@ -245,6 +252,7 @@ describe(`requires-writer`, () => {
       await requiresWriter.writeAll({
         pages,
         program,
+        ssrVisitedPages,
       })
 
       const matchPathsForInvertedInput = matchPaths
@@ -252,6 +260,7 @@ describe(`requires-writer`, () => {
       await requiresWriter.writeAll({
         pages,
         program,
+        ssrVisitedPages,
       })
 
       expect(matchPathsForInvertedInput).toEqual(matchPaths)
@@ -263,6 +272,7 @@ describe(`requires-writer`, () => {
         await requiresWriter.writeAll({
           pages,
           program,
+          ssrVisitedPages,
         })
 
         const allMatchingPages = matchPaths
diff --git a/packages/gatsby/src/bootstrap/requires-writer.ts b/packages/gatsby/src/bootstrap/requires-writer.ts
index 6373d86ed77cb..ae2efc640967c 100644
--- a/packages/gatsby/src/bootstrap/requires-writer.ts
+++ b/packages/gatsby/src/bootstrap/requires-writer.ts
@@ -162,26 +162,41 @@ const getMatchPaths = (
 
 const createHash = (
   matchPaths: Array<IGatsbyPageMatchPath>,
-  components: Array<IGatsbyPageComponent>
+  components: Array<IGatsbyPageComponent>,
+  cleanedSSRVisitedPageComponents: Array<IGatsbyPageComponent>
 ): string =>
   crypto
     .createHash(`md5`)
-    .update(JSON.stringify({ matchPaths, components }))
+    .update(
+      JSON.stringify({
+        matchPaths,
+        components,
+        cleanedSSRVisitedPageComponents,
+      })
+    )
     .digest(`hex`)
 
 // Write out pages information.
 export const writeAll = async (state: IGatsbyState): Promise<boolean> => {
-  // console.log(`on requiresWriter progress`)
   const { program } = state
   const pages = [...state.pages.values()]
   const matchPaths = getMatchPaths(pages)
   const components = getComponents(pages)
+  const ssrVisitedPageComponents = [...state.ssrVisitedPages.values()]
+
+  // Remove any page components that no longer exist.
+  const cleanedSSRVisitedPageComponents = components.filter(c =>
+    ssrVisitedPageComponents.some(s => s === c.componentChunkName)
+  )
 
-  const newHash = createHash(matchPaths, components)
+  const newHash = createHash(
+    matchPaths,
+    components,
+    cleanedSSRVisitedPageComponents
+  )
 
   if (newHash === lastHash) {
     // Nothing changed. No need to rewrite files
-    // console.log(`on requiresWriter END1`)
     return false
   }
 
@@ -230,6 +245,24 @@ const preferDefault = m => (m && m.default) || m
     .join(`,\n`)}
 }\n\n`
 
+  // Create file with sync requires of visited page components files.
+  let lazySyncRequires = `${hotImport}
+
+  // prefer default export if available
+  const preferDefault = m => (m && m.default) || m
+  \n\n`
+  lazySyncRequires += `exports.lazyComponents = {\n${cleanedSSRVisitedPageComponents
+    .map(
+      (c: IGatsbyPageComponent): string =>
+        `  "${
+          c.componentChunkName
+        }": ${hotMethod}(preferDefault(require("${joinPath(c.component)}")))`
+    )
+    .join(`,\n`)}
+  }\n\n`
+
+  writeModule(`$virtual/lazy-sync-requires`, lazySyncRequires)
+
   const writeAndMove = (
     virtualFilePath: string,
     file: string,
@@ -264,45 +297,17 @@ const preferDefault = m => (m && m.default) || m
   return true
 }
 
-const debouncedWriteAll = _.debounce(
-  async (): Promise<void> => {
-    const activity = reporter.activityTimer(`write out requires`, {
-      id: `requires-writer`,
-    })
-    activity.start()
-    await writeAll(store.getState())
-    activity.end()
-  },
-  500,
-  {
-    // using "leading" can cause double `writeAll` call - particularly
-    // when refreshing data using `/__refresh` hook.
-    leading: false,
-  }
-)
-
 /**
- * Start listening to CREATE/DELETE_PAGE events so we can rewrite
+ * Start listening to CREATE_SSR_VISITED_PAGE events so we can rewrite
  * files as required
  */
+let listenerCreated = false
 export const startListener = (): void => {
-  emitter.on(`CREATE_PAGE`, (): void => {
-    reporter.pendingActivity({ id: `requires-writer` })
-    debouncedWriteAll()
-  })
-
-  emitter.on(`CREATE_PAGE_END`, (): void => {
-    reporter.pendingActivity({ id: `requires-writer` })
-    debouncedWriteAll()
-  })
-
-  emitter.on(`DELETE_PAGE`, (): void => {
-    reporter.pendingActivity({ id: `requires-writer` })
-    debouncedWriteAll()
-  })
-
-  emitter.on(`DELETE_PAGE_BY_PATH`, (): void => {
-    reporter.pendingActivity({ id: `requires-writer` })
-    debouncedWriteAll()
-  })
+  if (!listenerCreated) {
+    emitter.on(`CREATE_SSR_VISITED_PAGE`, (): void => {
+      reporter.pendingActivity({ id: `requires-writer` })
+      writeAll(store.getState())
+    })
+    listenerCreated = true
+  }
 }
diff --git a/packages/gatsby/src/redux/actions/public.js b/packages/gatsby/src/redux/actions/public.js
index 6b4e9d0bfd279..53e05425ef7f9 100644
--- a/packages/gatsby/src/redux/actions/public.js
+++ b/packages/gatsby/src/redux/actions/public.js
@@ -1391,4 +1391,17 @@ actions.removePageData = (id: PageDataRemove) => {
   }
 }
 
+/**
+ * Record that a page was SSRed.
+ *
+ * @param {Object} $0
+ * @param {string} $0.id the path to the page.
+ */
+actions.createSSRVisitedPage = (page: Page) => {
+  return {
+    type: `CREATE_SSR_VISITED_PAGE`,
+    payload: page,
+  }
+}
+
 module.exports = { actions }
diff --git a/packages/gatsby/src/redux/reducers/index.ts b/packages/gatsby/src/redux/reducers/index.ts
index 218898bd67c60..089c517787344 100644
--- a/packages/gatsby/src/redux/reducers/index.ts
+++ b/packages/gatsby/src/redux/reducers/index.ts
@@ -1,6 +1,7 @@
 import { nodesReducer } from "./nodes"
 import { reducer as logReducer } from "gatsby-cli/lib/reporter/redux/reducer"
 import { pagesReducer } from "./pages"
+import { ssrVisitedPageReducer } from "./ssr-visited-page"
 import { redirectsReducer } from "./redirects"
 import { schemaReducer } from "./schema"
 import { definitionsReducer } from "./definitions"
@@ -43,6 +44,7 @@ export {
   configReducer as config,
   schemaReducer as schema,
   pagesReducer as pages,
+  ssrVisitedPageReducer as ssrVisitedPages,
   statusReducer as status,
   componentsReducer as components,
   staticQueryComponentsReducer as staticQueryComponents,
diff --git a/packages/gatsby/src/redux/reducers/ssr-visited-page.ts b/packages/gatsby/src/redux/reducers/ssr-visited-page.ts
new file mode 100644
index 0000000000000..16ab31074226d
--- /dev/null
+++ b/packages/gatsby/src/redux/reducers/ssr-visited-page.ts
@@ -0,0 +1,24 @@
+import {
+  IGatsbyState,
+  IDeleteCacheAction,
+  ICreateSSRVisitedPage,
+} from "../types"
+
+export const ssrVisitedPageReducer = (
+  state: IGatsbyState["ssrVisitedPages"] = new Set<string>(),
+  action: IDeleteCacheAction | ICreateSSRVisitedPage
+): IGatsbyState["ssrVisitedPages"] => {
+  switch (action.type) {
+    case `DELETE_CACHE`:
+      return new Set()
+
+    case `CREATE_SSR_VISITED_PAGE`: {
+      state.add(action.payload.componentChunkName)
+
+      return state
+    }
+
+    default:
+      return state
+  }
+}
diff --git a/packages/gatsby/src/redux/types.ts b/packages/gatsby/src/redux/types.ts
index 4eed602548beb..04d28ce8841d9 100644
--- a/packages/gatsby/src/redux/types.ts
+++ b/packages/gatsby/src/redux/types.ts
@@ -274,6 +274,7 @@ export interface IGatsbyState {
   }
   pageDataStats: Map<SystemPath, number>
   pageData: Map<Identifier, string>
+  ssrVisitedPages: Set<string>
 }
 
 export interface ICachedReduxState {
@@ -589,6 +590,12 @@ interface ISetSchemaComposerAction {
   payload: SchemaComposer<any>
 }
 
+export interface ICreateSSRVisitedPage {
+  type: `CREATE_SSR_VISITED_PAGE`
+  payload: IGatsbyPage
+  plugin?: IGatsbyPlugin
+}
+
 export interface ICreatePageAction {
   type: `CREATE_PAGE`
   payload: IGatsbyPage
@@ -737,8 +744,7 @@ export interface IAddChildNodeToParentNodeAction {
 
 export interface IDeleteNodeAction {
   type: `DELETE_NODE`
-  // FIXME: figure out why payload can be undefined here
-  payload: IGatsbyNode | void
+  payload: IGatsbyNode
 }
 
 export interface IDeleteNodesAction {

From cb4c6f9577901cc8e8c933f77e406f53723e4f36 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Mon, 9 Nov 2020 11:13:26 -0800
Subject: [PATCH 121/123] Move lazy bundling changes to
 https://github.com/gatsbyjs/gatsby/pull/27932

---
 .../bootstrap/__tests__/requires-writer.js    | 10 --
 .../gatsby/src/bootstrap/requires-writer.ts   | 89 ++++++++---------
 packages/gatsby/src/redux/actions/public.js   | 13 ---
 packages/gatsby/src/redux/reducers/index.ts   |  2 -
 .../src/redux/reducers/ssr-visited-page.ts    | 24 -----
 packages/gatsby/src/redux/types.ts            | 10 +-
 .../src/utils/dev-ssr/render-dev-html.ts      | 97 ++-----------------
 packages/gatsby/src/utils/start-server.ts     |  2 +-
 8 files changed, 55 insertions(+), 192 deletions(-)
 delete mode 100644 packages/gatsby/src/redux/reducers/ssr-visited-page.ts

diff --git a/packages/gatsby/src/bootstrap/__tests__/requires-writer.js b/packages/gatsby/src/bootstrap/__tests__/requires-writer.js
index 047f089ede4b9..b82fa698dc0e6 100644
--- a/packages/gatsby/src/bootstrap/__tests__/requires-writer.js
+++ b/packages/gatsby/src/bootstrap/__tests__/requires-writer.js
@@ -30,8 +30,6 @@ describe(`requires-writer`, () => {
   const program = {
     directory: `/dir`,
   }
-  const ssrVisitedPages = new Set(`hi`)
-
   let originalDateNow = global.Date.now
 
   beforeEach(() => {
@@ -63,7 +61,6 @@ describe(`requires-writer`, () => {
       await requiresWriter.writeAll({
         pages,
         program,
-        ssrVisitedPages,
       })
 
       expect(spy).toBeCalledWith(
@@ -111,7 +108,6 @@ describe(`requires-writer`, () => {
       await requiresWriter.writeAll({
         pages,
         program,
-        ssrVisitedPages,
       })
 
       expect(matchPaths[0].path).toBe(pages.get(`/app/login/`).path)
@@ -145,7 +141,6 @@ describe(`requires-writer`, () => {
       await requiresWriter.writeAll({
         pages,
         program,
-        ssrVisitedPages,
       })
 
       expect(matchPaths[0].path).toBe(pages.get(`/app/clients/static`).path)
@@ -170,7 +165,6 @@ describe(`requires-writer`, () => {
       await requiresWriter.writeAll({
         pages,
         program,
-        ssrVisitedPages,
       })
 
       expect(matchPaths[0].path).toBe(pages.get(`/`).path)
@@ -225,7 +219,6 @@ describe(`requires-writer`, () => {
       await requiresWriter.writeAll({
         pages,
         program,
-        ssrVisitedPages,
       })
 
       expect(matchPaths.map(p => p.matchPath)).toMatchInlineSnapshot(`
@@ -252,7 +245,6 @@ describe(`requires-writer`, () => {
       await requiresWriter.writeAll({
         pages,
         program,
-        ssrVisitedPages,
       })
 
       const matchPathsForInvertedInput = matchPaths
@@ -260,7 +252,6 @@ describe(`requires-writer`, () => {
       await requiresWriter.writeAll({
         pages,
         program,
-        ssrVisitedPages,
       })
 
       expect(matchPathsForInvertedInput).toEqual(matchPaths)
@@ -272,7 +263,6 @@ describe(`requires-writer`, () => {
         await requiresWriter.writeAll({
           pages,
           program,
-          ssrVisitedPages,
         })
 
         const allMatchingPages = matchPaths
diff --git a/packages/gatsby/src/bootstrap/requires-writer.ts b/packages/gatsby/src/bootstrap/requires-writer.ts
index ae2efc640967c..6373d86ed77cb 100644
--- a/packages/gatsby/src/bootstrap/requires-writer.ts
+++ b/packages/gatsby/src/bootstrap/requires-writer.ts
@@ -162,41 +162,26 @@ const getMatchPaths = (
 
 const createHash = (
   matchPaths: Array<IGatsbyPageMatchPath>,
-  components: Array<IGatsbyPageComponent>,
-  cleanedSSRVisitedPageComponents: Array<IGatsbyPageComponent>
+  components: Array<IGatsbyPageComponent>
 ): string =>
   crypto
     .createHash(`md5`)
-    .update(
-      JSON.stringify({
-        matchPaths,
-        components,
-        cleanedSSRVisitedPageComponents,
-      })
-    )
+    .update(JSON.stringify({ matchPaths, components }))
     .digest(`hex`)
 
 // Write out pages information.
 export const writeAll = async (state: IGatsbyState): Promise<boolean> => {
+  // console.log(`on requiresWriter progress`)
   const { program } = state
   const pages = [...state.pages.values()]
   const matchPaths = getMatchPaths(pages)
   const components = getComponents(pages)
-  const ssrVisitedPageComponents = [...state.ssrVisitedPages.values()]
-
-  // Remove any page components that no longer exist.
-  const cleanedSSRVisitedPageComponents = components.filter(c =>
-    ssrVisitedPageComponents.some(s => s === c.componentChunkName)
-  )
 
-  const newHash = createHash(
-    matchPaths,
-    components,
-    cleanedSSRVisitedPageComponents
-  )
+  const newHash = createHash(matchPaths, components)
 
   if (newHash === lastHash) {
     // Nothing changed. No need to rewrite files
+    // console.log(`on requiresWriter END1`)
     return false
   }
 
@@ -245,24 +230,6 @@ const preferDefault = m => (m && m.default) || m
     .join(`,\n`)}
 }\n\n`
 
-  // Create file with sync requires of visited page components files.
-  let lazySyncRequires = `${hotImport}
-
-  // prefer default export if available
-  const preferDefault = m => (m && m.default) || m
-  \n\n`
-  lazySyncRequires += `exports.lazyComponents = {\n${cleanedSSRVisitedPageComponents
-    .map(
-      (c: IGatsbyPageComponent): string =>
-        `  "${
-          c.componentChunkName
-        }": ${hotMethod}(preferDefault(require("${joinPath(c.component)}")))`
-    )
-    .join(`,\n`)}
-  }\n\n`
-
-  writeModule(`$virtual/lazy-sync-requires`, lazySyncRequires)
-
   const writeAndMove = (
     virtualFilePath: string,
     file: string,
@@ -297,17 +264,45 @@ const preferDefault = m => (m && m.default) || m
   return true
 }
 
+const debouncedWriteAll = _.debounce(
+  async (): Promise<void> => {
+    const activity = reporter.activityTimer(`write out requires`, {
+      id: `requires-writer`,
+    })
+    activity.start()
+    await writeAll(store.getState())
+    activity.end()
+  },
+  500,
+  {
+    // using "leading" can cause double `writeAll` call - particularly
+    // when refreshing data using `/__refresh` hook.
+    leading: false,
+  }
+)
+
 /**
- * Start listening to CREATE_SSR_VISITED_PAGE events so we can rewrite
+ * Start listening to CREATE/DELETE_PAGE events so we can rewrite
  * files as required
  */
-let listenerCreated = false
 export const startListener = (): void => {
-  if (!listenerCreated) {
-    emitter.on(`CREATE_SSR_VISITED_PAGE`, (): void => {
-      reporter.pendingActivity({ id: `requires-writer` })
-      writeAll(store.getState())
-    })
-    listenerCreated = true
-  }
+  emitter.on(`CREATE_PAGE`, (): void => {
+    reporter.pendingActivity({ id: `requires-writer` })
+    debouncedWriteAll()
+  })
+
+  emitter.on(`CREATE_PAGE_END`, (): void => {
+    reporter.pendingActivity({ id: `requires-writer` })
+    debouncedWriteAll()
+  })
+
+  emitter.on(`DELETE_PAGE`, (): void => {
+    reporter.pendingActivity({ id: `requires-writer` })
+    debouncedWriteAll()
+  })
+
+  emitter.on(`DELETE_PAGE_BY_PATH`, (): void => {
+    reporter.pendingActivity({ id: `requires-writer` })
+    debouncedWriteAll()
+  })
 }
diff --git a/packages/gatsby/src/redux/actions/public.js b/packages/gatsby/src/redux/actions/public.js
index 53e05425ef7f9..6b4e9d0bfd279 100644
--- a/packages/gatsby/src/redux/actions/public.js
+++ b/packages/gatsby/src/redux/actions/public.js
@@ -1391,17 +1391,4 @@ actions.removePageData = (id: PageDataRemove) => {
   }
 }
 
-/**
- * Record that a page was SSRed.
- *
- * @param {Object} $0
- * @param {string} $0.id the path to the page.
- */
-actions.createSSRVisitedPage = (page: Page) => {
-  return {
-    type: `CREATE_SSR_VISITED_PAGE`,
-    payload: page,
-  }
-}
-
 module.exports = { actions }
diff --git a/packages/gatsby/src/redux/reducers/index.ts b/packages/gatsby/src/redux/reducers/index.ts
index 089c517787344..218898bd67c60 100644
--- a/packages/gatsby/src/redux/reducers/index.ts
+++ b/packages/gatsby/src/redux/reducers/index.ts
@@ -1,7 +1,6 @@
 import { nodesReducer } from "./nodes"
 import { reducer as logReducer } from "gatsby-cli/lib/reporter/redux/reducer"
 import { pagesReducer } from "./pages"
-import { ssrVisitedPageReducer } from "./ssr-visited-page"
 import { redirectsReducer } from "./redirects"
 import { schemaReducer } from "./schema"
 import { definitionsReducer } from "./definitions"
@@ -44,7 +43,6 @@ export {
   configReducer as config,
   schemaReducer as schema,
   pagesReducer as pages,
-  ssrVisitedPageReducer as ssrVisitedPages,
   statusReducer as status,
   componentsReducer as components,
   staticQueryComponentsReducer as staticQueryComponents,
diff --git a/packages/gatsby/src/redux/reducers/ssr-visited-page.ts b/packages/gatsby/src/redux/reducers/ssr-visited-page.ts
deleted file mode 100644
index 16ab31074226d..0000000000000
--- a/packages/gatsby/src/redux/reducers/ssr-visited-page.ts
+++ /dev/null
@@ -1,24 +0,0 @@
-import {
-  IGatsbyState,
-  IDeleteCacheAction,
-  ICreateSSRVisitedPage,
-} from "../types"
-
-export const ssrVisitedPageReducer = (
-  state: IGatsbyState["ssrVisitedPages"] = new Set<string>(),
-  action: IDeleteCacheAction | ICreateSSRVisitedPage
-): IGatsbyState["ssrVisitedPages"] => {
-  switch (action.type) {
-    case `DELETE_CACHE`:
-      return new Set()
-
-    case `CREATE_SSR_VISITED_PAGE`: {
-      state.add(action.payload.componentChunkName)
-
-      return state
-    }
-
-    default:
-      return state
-  }
-}
diff --git a/packages/gatsby/src/redux/types.ts b/packages/gatsby/src/redux/types.ts
index 04d28ce8841d9..4eed602548beb 100644
--- a/packages/gatsby/src/redux/types.ts
+++ b/packages/gatsby/src/redux/types.ts
@@ -274,7 +274,6 @@ export interface IGatsbyState {
   }
   pageDataStats: Map<SystemPath, number>
   pageData: Map<Identifier, string>
-  ssrVisitedPages: Set<string>
 }
 
 export interface ICachedReduxState {
@@ -590,12 +589,6 @@ interface ISetSchemaComposerAction {
   payload: SchemaComposer<any>
 }
 
-export interface ICreateSSRVisitedPage {
-  type: `CREATE_SSR_VISITED_PAGE`
-  payload: IGatsbyPage
-  plugin?: IGatsbyPlugin
-}
-
 export interface ICreatePageAction {
   type: `CREATE_PAGE`
   payload: IGatsbyPage
@@ -744,7 +737,8 @@ export interface IAddChildNodeToParentNodeAction {
 
 export interface IDeleteNodeAction {
   type: `DELETE_NODE`
-  payload: IGatsbyNode
+  // FIXME: figure out why payload can be undefined here
+  payload: IGatsbyNode | void
 }
 
 export interface IDeleteNodesAction {
diff --git a/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts b/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
index 9611c0745d698..989419186ac6f 100644
--- a/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
+++ b/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
@@ -6,8 +6,6 @@ import _ from "lodash"
 
 import { startListener } from "../../bootstrap/requires-writer"
 import { findPageByPath } from "../find-page-by-path"
-import { boundActionCreators } from "../../redux/actions"
-const { createSSRVisitedPage } = boundActionCreators
 
 const startWorker = (): any => {
   const newWorker = new JestWorker(require.resolve(`./render-dev-html-child`), {
@@ -47,66 +45,6 @@ export const restartWorker = (htmlComponentRendererPath): void => {
   }
 }
 
-const searchFileForString = (substring, filePath): Promise<boolean> =>
-  new Promise(resolve => {
-    const stream = fs.createReadStream(filePath)
-    let found = false
-    stream.on(`data`, function (d) {
-      if (d.includes(substring)) {
-        found = true
-        stream.close()
-        resolve(found)
-      }
-    })
-    stream.on(`error`, function () {
-      resolve(found)
-    })
-    stream.on(`close`, function () {
-      resolve(found)
-    })
-  })
-
-const ensurePathComponentInSSRBundle = async (
-  page,
-  directory
-): Promise<any> => {
-  // This shouldn't happen.
-  if (!page) {
-    report.panic(`page not found`)
-  }
-
-  // Now check if it's written to public/render-page.js
-  const htmlComponentRendererPath = joinPath(directory, `public/render-page.js`)
-  // This search takes 1-10ms
-  // We do it as there can be a race conditions where two pages
-  // are requested at the same time which means that both are told render-page.js
-  // has changed when the first page is complete meaning the second
-  // page's component won't be in the render meaning its SSR will fail.
-  let found = await searchFileForString(
-    page.componentChunkName,
-    htmlComponentRendererPath
-  )
-
-  if (!found) {
-    await new Promise(resolve => {
-      let readAttempts = 0
-      const searchForStringInterval = setInterval(async () => {
-        readAttempts += 1
-        found = await searchFileForString(
-          page.componentChunkName,
-          htmlComponentRendererPath
-        )
-        if (found || readAttempts === 5) {
-          clearInterval(searchForStringInterval)
-          resolve()
-        }
-      }, 300)
-    })
-  }
-
-  return found
-}
-
 export const renderDevHTML = ({
   path,
   page,
@@ -128,30 +66,15 @@ export const renderDevHTML = ({
       isClientOnlyPage = true
     }
 
-    // Record this page was requested. This will kick off adding its page
-    // component to the ssr bundle (if that's not already happened)
-    createSSRVisitedPage(pageObj)
-
-    let pageFound = true
-    if (process.env.GATSBY_EXPERIMENTAL_DEV_SSR) {
-      // Write component to file & wait for public/render-page.js to update
-      pageFound = await ensurePathComponentInSSRBundle(pageObj, directory)
-      if (!pageFound) {
-        reject(`404 page`)
-      }
-    }
-
-    if (pageFound) {
-      try {
-        const htmlString = await worker.renderHTML({
-          path,
-          htmlComponentRendererPath,
-          directory,
-          isClientOnlyPage,
-        })
-        resolve(htmlString)
-      } catch (error) {
-        reject(error)
-      }
+    try {
+      const htmlString = await worker.renderHTML({
+        path,
+        htmlComponentRendererPath,
+        directory,
+        isClientOnlyPage,
+      })
+      resolve(htmlString)
+    } catch (error) {
+      reject(error)
     }
   })
diff --git a/packages/gatsby/src/utils/start-server.ts b/packages/gatsby/src/utils/start-server.ts
index 8b1f63dfb0e59..7f7aa4af65d3d 100644
--- a/packages/gatsby/src/utils/start-server.ts
+++ b/packages/gatsby/src/utils/start-server.ts
@@ -291,7 +291,7 @@ export async function startServer(
    **/
   const server = new http.Server(app)
 
-  websocketManager.init({ server, directory: program.directory })
+  websocketManager.init({ server })
 
   // hardcoded `localhost`, because host should match `target` we set
   // in http proxy in `develop-proxy`

From 3bad8bcc22d11a8c8187bf38f8eb9ba098343cf7 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Mon, 9 Nov 2020 11:18:22 -0800
Subject: [PATCH 122/123] Copy over another change

---
 .../src/utils/dev-ssr/render-dev-html.ts      | 151 ++++++++++++++++++
 1 file changed, 151 insertions(+)
 create mode 100644 packages/gatsby/src/utils/dev-ssr/render-dev-html.ts

diff --git a/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts b/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
new file mode 100644
index 0000000000000..d44b1dd3c76fd
--- /dev/null
+++ b/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
@@ -0,0 +1,151 @@
+import JestWorker from "jest-worker"
+import fs from "fs-extra"
+import { joinPath } from "gatsby-core-utils"
+import report from "gatsby-cli/lib/reporter"
+import _ from "lodash"
+
+import { startListener } from "../../bootstrap/requires-writer"
+import { findPageByPath } from "../find-page-by-path"
+
+const startWorker = (): any => {
+  const newWorker = new JestWorker(require.resolve(`./render-dev-html-child`), {
+    exposedMethods: [`renderHTML`, `deleteModuleCache`, `warmup`],
+    numWorkers: 1,
+    forkOptions: { silent: false },
+  })
+
+  // jest-worker is lazy with forking but we want to fork immediately so the user
+  // doesn't have to wait.
+  // @ts-ignore
+  newWorker.warmup()
+
+  return newWorker
+}
+
+let worker
+export const initDevWorkerPool = (): void => {
+  worker = startWorker()
+}
+
+let changeCount = 0
+export const restartWorker = (htmlComponentRendererPath): void => {
+  changeCount += 1
+  // Forking is expensive — each time we re-require the outputted webpack
+  // file, memory grows ~10 mb — 25 regenerations means ~250mb which seems
+  // like an accepatable amount of memory to grow before we reclaim it
+  // by rebooting the worker process.
+  if (changeCount > 25) {
+    const oldWorker = worker
+    const newWorker = startWorker()
+    worker = newWorker
+    oldWorker.end()
+    changeCount = 0
+  } else {
+    worker.deleteModuleCache(htmlComponentRendererPath)
+  }
+}
+
+const searchFileForString = (substring, filePath): Promise<boolean> =>
+  new Promise(resolve => {
+    const stream = fs.createReadStream(filePath)
+    let found = false
+    stream.on(`data`, function (d) {
+      if (d.includes(substring)) {
+        found = true
+        stream.close()
+        resolve(found)
+      }
+    })
+    stream.on(`error`, function () {
+      resolve(found)
+    })
+    stream.on(`close`, function () {
+      resolve(found)
+    })
+  })
+
+const ensurePathComponentInSSRBundle = async (
+  page,
+  directory
+): Promise<any> => {
+  // This shouldn't happen.
+  if (!page) {
+    report.panic(`page not found`)
+  }
+
+  // Now check if it's written to public/render-page.js
+  const htmlComponentRendererPath = joinPath(directory, `public/render-page.js`)
+  // This search takes 1-10ms
+  // We do it as there can be a race conditions where two pages
+  // are requested at the same time which means that both are told render-page.js
+  // has changed when the first page is complete meaning the second
+  // page's component won't be in the render meaning its SSR will fail.
+  let found = await searchFileForString(
+    page.componentChunkName,
+    htmlComponentRendererPath
+  )
+
+  if (!found) {
+    await new Promise(resolve => {
+      let readAttempts = 0
+      const searchForStringInterval = setInterval(async () => {
+        readAttempts += 1
+        found = await searchFileForString(
+          page.componentChunkName,
+          htmlComponentRendererPath
+        )
+        if (found || readAttempts === 5) {
+          clearInterval(searchForStringInterval)
+          resolve()
+        }
+      }, 300)
+    })
+  }
+
+  return found
+}
+
+export const renderDevHTML = ({
+  path,
+  page,
+  store,
+  htmlComponentRendererPath,
+  directory,
+}): Promise<string> =>
+  new Promise(async (resolve, reject) => {
+    startListener()
+    let pageObj
+    if (!page) {
+      pageObj = findPageByPath(store.getState(), path)
+    } else {
+      pageObj = page
+    }
+
+    let isClientOnlyPage = false
+    if (pageObj.matchPath) {
+      isClientOnlyPage = true
+    }
+
+    let pageFound = true
+    if (process.env.GATSBY_EXPERIMENTAL_DEV_SSR) {
+      // Write component to file & wait for public/render-page.js to update
+      pageFound = await ensurePathComponentInSSRBundle(pageObj, directory)
+      if (!pageFound) {
+        reject(`404 page`)
+      }
+    }
+
+    if (pageFound) {
+      try {
+        const htmlString = await worker.renderHTML({
+          path,
+          htmlComponentRendererPath,
+          directory,
+          isClientOnlyPage,
+        })
+        resolve(htmlString)
+      } catch (error) {
+        reject(error)
+      }
+    }
+  })

From d0800ea917ae28688fe1b6c727597e1aad40b089 Mon Sep 17 00:00:00 2001
From: Kyle Mathews <mathews.kyle@gmail.com>
Date: Mon, 9 Nov 2020 11:20:47 -0800
Subject: [PATCH 123/123] More cleanup

---
 packages/gatsby/src/utils/dev-ssr/render-dev-html.ts | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts b/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
index 989419186ac6f..e2ccf33281c6e 100644
--- a/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
+++ b/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts
@@ -1,7 +1,4 @@
 import JestWorker from "jest-worker"
-import fs from "fs-extra"
-import { joinPath } from "gatsby-core-utils"
-import report from "gatsby-cli/lib/reporter"
 import _ from "lodash"
 
 import { startListener } from "../../bootstrap/requires-writer"