feat(gatsby): split page-renderer per route

docs/docs/html-generation.md

@@ -95,7 +95,7 @@ Finally, we call [react-dom](https://reactjs.org/docs/react-dom.html) and render
 
 ## build-html.js
 
-So, we've built the means to generate HTML for a page. This webpack bundle is saved to `public/render-page.js`. Next, we need to use it to generate HTML for all the site's pages.
+So, we've built the means to generate HTML for a page. These webpack bundles are saved to `.cache/page-ssr/routes`. Next, we need to use it to generate HTML for all the site's pages.
 
 Page HTML does not depend on other pages. So we can perform this step in parallel. We use the [jest-worker](https://github.com/facebook/jest/tree/master/packages/jest-worker) library to make this easier. By default, the [render-html.ts](https://github.com/gatsbyjs/gatsby/blob/master/packages/gatsby/src/utils/worker/render-html.ts) creates a pool of workers equal to the number of physical cores on your machine. You can configure the number of pools by passing an optional environment variable, [`GATSBY_CPU_COUNT`](/docs/multi-core-builds). It then partitions the pages into groups and sends them to the workers, which run [worker](https://github.com/gatsbyjs/gatsby/blob/master/packages/gatsby/src/utils/worker).
 

packages/gatsby/cache-dir/__tests__/static-entry.js

@@ -34,11 +34,14 @@ jest.mock(
 )
 
 jest.mock(
-  `$virtual/sync-requires`,
+  `$virtual/async-requires`,
   () => {
     return {
       components: {
-        "page-component---src-pages-test-js": () => null,
+        "page-component---src-pages-test-js": () =>
+          Promise.resolve({
+            default: () => null,
+          }),
       },
     }
   },

packages/gatsby/cache-dir/develop-static-entry.js

@@ -3,6 +3,8 @@ import React from "react"
 import { renderToStaticMarkup } from "react-dom/server"
 import { merge } from "lodash"
 import { apiRunner } from "./api-runner-ssr"
+import asyncRequires from "$virtual/async-requires"
+
 // import testRequireError from "./test-require-error"
 // For some extremely mysterious reason, webpack adds the above module *after*
 // this module so that when this code runs, testRequireError is undefined.
@@ -125,3 +127,7 @@ export default ({ pagePath }) => {
 
   return htmlStr
 }
+
+export function getPageChunk({ componentChunkName }) {
+  return asyncRequires.components[componentChunkName]()
+}

packages/gatsby/cache-dir/ssr-develop-static-entry.js

@@ -220,9 +220,6 @@ export default async function staticPage(
           />
         )
       })
-
-    const createElement = React.createElement
-
     class RouteHandler extends React.Component {
       render() {
         const props = {
@@ -239,7 +236,7 @@ export default async function staticPage(
           syncRequires.ssrComponents[componentChunkName] &&
           !isClientOnlyPage
         ) {
-          pageElement = createElement(
+          pageElement = React.createElement(
             syncRequires.ssrComponents[componentChunkName],
             props
           )

packages/gatsby/cache-dir/static-entry.js

@@ -13,7 +13,7 @@ const { WritableAsPromise } = require(`./server-utils/writable-as-promise`)
 
 const { RouteAnnouncerProps } = require(`./route-announcer-props`)
 const { apiRunner, apiRunnerAsync } = require(`./api-runner-ssr`)
-const syncRequires = require(`$virtual/sync-requires`)
+const asyncRequires = require(`$virtual/async-requires`)
 const { version: gatsbyVersion } = require(`gatsby/package.json`)
 const { grabMatchParams } = require(`./find-path`)
 
@@ -209,6 +209,7 @@ export default async function staticPage({
     const pageDataUrl = getPageDataUrl(pagePath)
 
     const { componentChunkName, staticQueryHashes = [] } = pageData
+    const pageComponent = await asyncRequires.components[componentChunkName]()
 
     const staticQueryUrls = staticQueryHashes.map(getStaticQueryUrl)
 
@@ -223,10 +224,7 @@ export default async function staticPage({
           },
         }
 
-        const pageElement = createElement(
-          syncRequires.components[componentChunkName],
-          props
-        )
+        const pageElement = createElement(pageComponent.default, props)
 
         const wrappedPage = apiRunner(
           `wrapPageElement`,
@@ -479,3 +477,7 @@ export default async function staticPage({
     throw e
   }
 }
+
+export function getPageChunk({ componentChunkName }) {
+  return asyncRequires.components[componentChunkName]()
+}

packages/gatsby/src/bootstrap/requires-writer.ts

@@ -249,10 +249,7 @@ const preferDefault = m => (m && m.default) || m
 }\n\n`
 
   // Create file with async requires of components/json files.
-  let asyncRequires = `// prefer default export if available
-const preferDefault = m => (m && m.default) || m
-\n`
-  asyncRequires += `exports.components = {\n${components
+  const asyncRequires = `exports.components = {\n${components
     .map((c: IGatsbyPageComponent): string => {
       // we need a relative import path to keep contenthash the same if directory changes
       const relativeComponentPath = path.relative(

packages/gatsby/src/commands/build-html.ts

@@ -15,6 +15,7 @@ import { getPageData } from "../utils/get-page-data"
 
 import { Span } from "opentracing"
 import { IProgram, Stage } from "./types"
+import { ROUTES_DIRECTORY } from "../constants"
 import { PackageJson } from "../.."
 import type { GatsbyWorkerPool } from "../utils/worker/pool"
 import { IPageDataWithQueryResult } from "../utils/page-data"
@@ -32,6 +33,7 @@ export interface IBuildArgs extends IProgram {
   profile: boolean
   graphqlTracing: boolean
   openTracingConfigFile: string
+  // TODO remove in v4
   keepPageRenderer: boolean
 }
 
@@ -124,7 +126,7 @@ const runWebpack = (
             } = require(`../utils/dev-ssr/render-dev-html`)
             // Make sure we use the latest version during development
             if (oldHash !== `` && newHash !== oldHash) {
-              restartWorker(`${directory}/public/render-page.js`)
+              restartWorker(`${directory}/${ROUTES_DIRECTORY}render-page.js`)
             }
 
             oldHash = newHash
@@ -167,7 +169,7 @@ const doBuildRenderer = async (
 
   // render-page.js is hard coded in webpack.config
   return {
-    rendererPath: `${directory}/public/render-page.js`,
+    rendererPath: `${directory}/${ROUTES_DIRECTORY}render-page.js`,
     waitForCompilerClose,
   }
 }
@@ -185,6 +187,7 @@ export const buildRenderer = async (
   return doBuildRenderer(program, config, stage, parentSpan)
 }
 
+// TODO remove after v4 release and update cloud internals
 export const deleteRenderer = async (rendererPath: string): Promise<void> => {
   try {
     await fs.remove(rendererPath)
@@ -193,7 +196,6 @@ export const deleteRenderer = async (rendererPath: string): Promise<void> => {
     // This function will fail on Windows with no further consequences.
   }
 }
-
 export interface IRenderHtmlResult {
   unsafeBuiltinsUsageByPagePath: Record<string, Array<string>>
 }
@@ -370,23 +372,21 @@ export const buildHTML = async ({
 }): Promise<void> => {
   const { rendererPath } = await buildRenderer(program, stage, activity.span)
   await doBuildPages(rendererPath, pagePaths, activity, workerPool, stage)
-  await deleteRenderer(rendererPath)
 }
 
 export async function buildHTMLPagesAndDeleteStaleArtifacts({
-  pageRenderer,
   workerPool,
   buildSpan,
   program,
 }: {
-  pageRenderer: string
   workerPool: GatsbyWorkerPool
   buildSpan?: Span
   program: IBuildArgs
 }): Promise<{
   toRegenerate: Array<string>
   toDelete: Array<string>
 }> {
+  const pageRenderer = `${program.directory}/${ROUTES_DIRECTORY}render-page.js`
   buildUtils.markHtmlDirtyIfResultOfUsedStaticQueryChanged()
 
   const { toRegenerate, toDelete, toCleanupFromTrackedState } =
@@ -441,8 +441,7 @@ export async function buildHTMLPagesAndDeleteStaleArtifacts({
     reporter.info(`There are no new or changed html files to build.`)
   }
 
-  // TODO move to per page builds in _routes directory
-  if (!program.keepPageRenderer && _CFLAGS_.GATSBY_MAJOR !== `4`) {
+  if (_CFLAGS_.GATSBY_MAJOR !== `4` && !program.keepPageRenderer) {
     try {
       await deleteRenderer(pageRenderer)
     } catch (err) {

packages/gatsby/src/commands/build.ts

@@ -47,8 +47,6 @@ import {
   mergeWorkerState,
   runQueriesInWorkersQueue,
 } from "../utils/worker/pool"
-import webpackConfig from "../utils/webpack.config.js"
-import { webpack } from "webpack"
 import { createGraphqlEngineBundle } from "../schema/graphql-engine/bundle-webpack"
 import { createPageSSRBundle } from "../utils/page-ssr-module/bundle-webpack"
 import { shouldGenerateEngines } from "../utils/engines-helpers"
@@ -237,60 +235,15 @@ module.exports = async function build(program: IBuildArgs): Promise<void> {
     { parentSpan: buildSpan }
   )
   buildSSRBundleActivityProgress.start()
-  let pageRenderer = ``
   let waitForCompilerCloseBuildHtml
   try {
     const result = await buildRenderer(
       program,
       Stage.BuildHTML,
       buildSSRBundleActivityProgress.span
     )
-    pageRenderer = result.rendererPath
-    if (_CFLAGS_.GATSBY_MAJOR === `4` && shouldGenerateEngines()) {
-      // for now copy page-render to `.cache` so page-ssr module can require it as a sibling module
-      const outputDir = path.join(program.directory, `.cache`, `page-ssr`)
-      engineBundlingPromises.push(
-        fs
-          .ensureDir(outputDir)
-          .then(() =>
-            fs.copyFile(
-              result.rendererPath,
-              path.join(outputDir, `render-page.js`)
-            )
-          )
-      )
-    }
     waitForCompilerCloseBuildHtml = result.waitForCompilerClose
 
-    // TODO Move to page-renderer
-    if (_CFLAGS_.GATSBY_MAJOR === `4`) {
-      const routesWebpackConfig = await webpackConfig(
-        program,
-        program.directory,
-        `build-ssr`,
-        null,
-        { parentSpan: buildSSRBundleActivityProgress.span }
-      )
-
-      await new Promise((resolve, reject) => {
-        const compiler = webpack(routesWebpackConfig)
-        compiler.run(err => {
-          if (err) {
-            return void reject(err)
-          }
-
-          compiler.close(error => {
-            if (error) {
-              return void reject(error)
-            }
-            return void resolve(undefined)
-          })
-
-          return undefined
-        })
-      })
-    }
-
     if (_CFLAGS_.GATSBY_MAJOR === `4` && shouldGenerateEngines()) {
       Promise.all(engineBundlingPromises).then(() => {
         if (process.send) {
@@ -319,7 +272,6 @@ module.exports = async function build(program: IBuildArgs): Promise<void> {
   const { toRegenerate, toDelete } =
     await buildHTMLPagesAndDeleteStaleArtifacts({
       program,
-      pageRenderer,
       workerPool,
       buildSpan,
     })

packages/gatsby/src/commands/types.ts

@@ -50,6 +50,4 @@ export enum Stage {
   DevelopHTML = `develop-html`,
   BuildJavascript = `build-javascript`,
   BuildHTML = `build-html`,
-  // TODO move to BuildHTML when queryengine pieces are merged
-  SSR = `build-ssr`,
 }

packages/gatsby/src/constants.ts

@@ -1 +1,2 @@
-export const ROUTES_DIRECTORY = `.cache/page-ssr/routes`
+export const ROUTES_DIRECTORY =
+  _CFLAGS_.GATSBY_MAJOR === `4` ? `.cache/page-ssr/routes/` : `public/`

packages/gatsby/src/utils/babel-loader-helpers.js

@@ -74,7 +74,10 @@ const prepareOptions = (babel, options = {}, resolve = require.resolve) => {
   const requiredPresets = []
 
   // Stage specific plugins to add
-  if (stage === `build-html` || stage === `develop-html`) {
+  if (
+    _CFLAGS_.GATSBY_MAJOR !== `4` &&
+    (stage === `build-html` || stage === `develop-html`)
+  ) {
     requiredPlugins.push(
       babel.createConfigItem([resolve(`babel-plugin-dynamic-import-node`)], {
         type: `plugin`,

packages/gatsby/src/utils/dev-ssr/render-dev-html.ts

@@ -4,6 +4,7 @@ import nodePath from "path"
 import report from "gatsby-cli/lib/reporter"
 import { isCI } from "gatsby-core-utils"
 import { Stats } from "webpack"
+import { ROUTES_DIRECTORY } from "../../constants"
 import { startListener } from "../../bootstrap/requires-writer"
 import { findPageByPath } from "../find-page-by-path"
 import { getPageData as getPageDataExperimental } from "../get-page-data"
@@ -112,10 +113,11 @@ const ensurePathComponentInSSRBundle = async (
     report.panic(`page not found`, page)
   }
 
-  // Now check if it's written to public/render-page.js
+  // Now check if it's written to the correct path
   const htmlComponentRendererPath = nodePath.join(
     directory,
-    `public/render-page.js`
+    ROUTES_DIRECTORY,
+    `render-page.js`
   )
 
   // This search takes 1-10ms
@@ -232,7 +234,7 @@ export const renderDevHTML = ({
       })
     }
 
-    // Wait for public/render-page.js to update w/ the page component.
+    // Wait for html-renderer to update w/ the page component.
     const found = await ensurePathComponentInSSRBundle(pageObj, directory)
 
     // If we can't find the page, just force set isClientOnlyPage

packages/gatsby/src/utils/flags.ts

@@ -102,6 +102,11 @@ const activeFlags: Array<IFlag> = [
     description: `Server Side Render (SSR) pages on full reloads during develop. Helps you detect SSR bugs and fix them without needing to do full builds. See umbrella issue for how to update custom webpack config.`,
     umbrellaIssue: `https://gatsby.dev/dev-ssr-feedback`,
     testFitness: (): fitnessEnum => {
+      // TODO Re-enable after gatsybcamp
+      if (_CFLAGS_.GATSBY_MAJOR === `4`) {
+        return false
+      }
+
       if (sampleSiteForExperiment(`DEV_SSR`, 20)) {
         return `OPT_IN`
       } else {

packages/gatsby/src/utils/page-ssr-module/bundle-webpack.ts

@@ -54,7 +54,7 @@ export async function createPageSSRBundle(): Promise<any> {
     },
     // those are required in some runtime paths, but we don't need them
     externals: [
-      `./render-page`,
+      /^\.\/routes/,
       `electron`, // :shrug: `got` seems to have electron specific code path
       mod.builtinModules.reduce((acc, builtinModule) => {
         if (builtinModule === `fs`) {

packages/gatsby/src/utils/page-ssr-module/entry.ts

@@ -15,7 +15,7 @@ import {
   getPagePathFromPageDataPath,
 } from "../page-data-helpers"
 // @ts-ignore render-page import will become valid later on (it's marked as external)
-import htmlComponentRenderer from "./render-page"
+import htmlComponentRenderer, { getPageChunk } from "./routes/render-page"
 import { getServerData, IServerData } from "../get-server-data"
 
 export interface ITemplateDetails {
@@ -37,9 +37,6 @@ const pageTemplateDetailsMap: Record<
   // @ts-ignore INLINED_TEMPLATE_TO_DETAILS is being "inlined" by bundler
 > = INLINED_TEMPLATE_TO_DETAILS
 
-// eslint-disable-next-line @typescript-eslint/naming-convention
-declare const __non_webpack_require__: typeof require
-
 export async function getData({
   pathName,
   graphqlEngine,
@@ -88,13 +85,12 @@ export async function getData({
 
   // 4. (if SSR) run getServerData
   if (page.mode === `SSR`) {
-    const mod = __non_webpack_require__(`./routes/${page.componentChunkName}`)
     executionPromises.push(
-      getServerData(req, page, potentialPagePath, mod).then(
-        serverDataResults => {
+      getPageChunk(page)
+        .then(mod => getServerData(req, page, potentialPagePath, mod))
+        .then(serverDataResults => {
           serverData = serverDataResults
-        }
-      )
+        })
     )
   }