feat(gatsby): make dev ssr bundling lazy (#28149)

* feat(gatsby): make dev ssr bundling lazy

* fix tests

* fix race condition

* typescript 😅

* await getPageData to ensure it's been created before SSRing

* instance isn't set in SSR

* Fix a few more instances of this env var

* Avoid setting anything for experimental features

Co-authored-by: Michal Piechowiak <misiek.piechowiak@gmail.com>

packages/gatsby/cache-dir/__tests__/static-entry.js

@@ -21,7 +21,7 @@ jest.mock(
   `$virtual/ssr-sync-requires`,
   () => {
     return {
-      components: {
+      ssrComponents: {
         "page-component---src-pages-test-js": () => null,
       },
     }

packages/gatsby/cache-dir/ssr-develop-static-entry.js

@@ -131,7 +131,7 @@ export default (pagePath, isClientOnlyPage, callback) => {
         }
 
         const pageElement = createElement(
-          syncRequires.components[componentChunkName],
+          syncRequires.ssrComponents[componentChunkName],
           props
         )
 

packages/gatsby/src/bootstrap/requires-writer.ts

@@ -164,7 +164,8 @@ const createHash = (
   matchPaths: Array<IGatsbyPageMatchPath>,
   components: Array<IGatsbyPageComponent>,
   cleanedClientVisitedPageComponents: Array<IGatsbyPageComponent>,
-  notVisitedPageComponents: Array<IGatsbyPageComponent>
+  notVisitedPageComponents: Array<IGatsbyPageComponent>,
+  cleanedSSRVisitedPageComponents: Array<IGatsbyPageComponent>
 ): string =>
   crypto
     .createHash(`md5`)
@@ -174,6 +175,7 @@ const createHash = (
         components,
         cleanedClientVisitedPageComponents,
         notVisitedPageComponents,
+        cleanedSSRVisitedPageComponents,
       })
     )
     .digest(`hex`)
@@ -184,11 +186,27 @@ export const writeAll = async (state: IGatsbyState): Promise<boolean> => {
   const pages = [...state.pages.values()]
   const matchPaths = getMatchPaths(pages)
   const components = getComponents(pages)
+  let cleanedSSRVisitedPageComponents: Array<IGatsbyPageComponent> = []
+
+  if (process.env.GATSBY_EXPERIMENTAL_DEV_SSR) {
+    const ssrVisitedPageComponents = [
+      ...(state.visitedPages.get(`server`)?.values() || []),
+    ]
+
+    // Remove any page components that no longer exist.
+    cleanedSSRVisitedPageComponents = components.filter(c =>
+      ssrVisitedPageComponents.some(s => s === c.componentChunkName)
+    )
+  }
+
+  let cleanedClientVisitedPageComponents: Array<IGatsbyPageComponent> = []
+  let notVisitedPageComponents: Array<IGatsbyPageComponent> = []
 
-  let cleanedClientVisitedPageComponents: Array<IGatsbyPageComponent> = components
-  let notVisitedPageComponents: Array<IGatsbyPageComponent> = components
   if (process.env.GATSBY_EXPERIMENTAL_LAZY_DEVJS) {
-    const clientVisitedPageComponents = [...state.clientVisitedPages.values()]
+    const clientVisitedPageComponents = [
+      ...(state.visitedPages.get(`client`)?.values() || []),
+    ]
+
     // Remove any page components that no longer exist.
     cleanedClientVisitedPageComponents = components.filter(component =>
       clientVisitedPageComponents.some(
@@ -211,7 +229,8 @@ export const writeAll = async (state: IGatsbyState): Promise<boolean> => {
     matchPaths,
     components,
     cleanedClientVisitedPageComponents,
-    notVisitedPageComponents
+    notVisitedPageComponents,
+    cleanedSSRVisitedPageComponents
   )
 
   if (newHash === lastHash) {
@@ -229,6 +248,25 @@ export const writeAll = async (state: IGatsbyState): Promise<boolean> => {
   const hotMethod =
     process.env.GATSBY_HOT_LOADER !== `fast-refresh` ? `hot` : ``
 
+  if (process.env.GATSBY_EXPERIMENTAL_DEV_SSR) {
+    // Create file with sync requires of visited page components files.
+    let lazySyncRequires = `${hotImport}
+  // prefer default export if available
+  const preferDefault = m => (m && m.default) || m
+  \n\n`
+    lazySyncRequires += `exports.ssrComponents = {\n${cleanedSSRVisitedPageComponents
+      .map(
+        (c: IGatsbyPageComponent): string =>
+          `  "${
+            c.componentChunkName
+          }": ${hotMethod}(preferDefault(require("${joinPath(c.component)}")))`
+      )
+      .join(`,\n`)}
+  }\n\n`
+
+    writeModule(`$virtual/ssr-sync-requires`, lazySyncRequires)
+  }
+
   // Create file with sync requires of components/json files.
   let syncRequires = `${hotImport}
 
@@ -245,10 +283,6 @@ const preferDefault = m => (m && m.default) || m
     .join(`,\n`)}
 }\n\n`
 
-  // webpack only seems to trigger re-renders once per virtual
-  // file so we need a seperate one for each webpack instance.
-  writeModule(`$virtual/ssr-sync-requires`, syncRequires)
-
   if (process.env.GATSBY_EXPERIMENTAL_LAZY_DEVJS) {
     // Create file with sync requires of visited page components files.
     let lazyClientSyncRequires = `${hotImport}
@@ -331,17 +365,6 @@ const preferDefault = m => (m && m.default) || m
   return true
 }
 
-if (process.env.GATSBY_EXPERIMENTAL_LAZY_DEVJS) {
-  /**
-   * Start listening to CREATE_CLIENT_VISITED_PAGE events so we can rewrite
-   * files as required
-   */
-  emitter.on(`CREATE_CLIENT_VISITED_PAGE`, (): void => {
-    reporter.pendingActivity({ id: `requires-writer` })
-    writeAll(store.getState())
-  })
-}
-
 const debouncedWriteAll = _.debounce(
   async (): Promise<void> => {
     const activity = reporter.activityTimer(`write out requires`, {
@@ -359,6 +382,28 @@ const debouncedWriteAll = _.debounce(
   }
 )
 
+if (process.env.GATSBY_EXPERIMENTAL_LAZY_DEVJS) {
+  /**
+   * Start listening to CREATE_CLIENT_VISITED_PAGE events so we can rewrite
+   * files as required
+   */
+  emitter.on(`CREATE_CLIENT_VISITED_PAGE`, (): void => {
+    reporter.pendingActivity({ id: `requires-writer` })
+    debouncedWriteAll()
+  })
+}
+
+if (process.env.GATSBY_EXPERIMENTAL_DEV_SSR) {
+  /**
+   * Start listening to CREATE_SERVER_VISITED_PAGE events so we can rewrite
+   * files as required
+   */
+  emitter.on(`CREATE_SERVER_VISITED_PAGE`, (): void => {
+    reporter.pendingActivity({ id: `requires-writer` })
+    debouncedWriteAll()
+  })
+}
+
 /**
  * Start listening to CREATE/DELETE_PAGE events so we can rewrite
  * files as required

packages/gatsby/src/redux/actions/public.js

@@ -1404,4 +1404,17 @@ actions.createClientVisitedPage = (chunkName: string) => {
   }
 }
 
+/**
+ * Record that a page was visited on the server..
+ *
+ * @param {Object} $0
+ * @param {string} $0.id the chunkName for the page component.
+ */
+actions.createServerVisitedPage = (chunkName: string) => {
+  return {
+    type: `CREATE_SERVER_VISITED_PAGE`,
+    payload: { componentChunkName: chunkName },
+  }
+}
+
 module.exports = { actions }

packages/gatsby/src/redux/reducers/index.ts

@@ -27,7 +27,7 @@ import { schemaCustomizationReducer } from "./schema-customization"
 import { inferenceMetadataReducer } from "./inference-metadata"
 import { staticQueriesByTemplateReducer } from "./static-queries-by-template"
 import { queriesReducer } from "./queries"
-import { clientVisitedPageReducer } from "./client-visited-page"
+import { visitedPagesReducer } from "./visited-page"
 
 /**
  * @property exports.nodesTouched Set<string>
@@ -44,7 +44,7 @@ export {
   configReducer as config,
   schemaReducer as schema,
   pagesReducer as pages,
-  clientVisitedPageReducer as clientVisitedPages,
+  visitedPagesReducer as visitedPages,
   statusReducer as status,
   componentsReducer as components,
   staticQueryComponentsReducer as staticQueryComponents,

packages/gatsby/src/redux/reducers/visited-page.ts

@@ -0,0 +1,56 @@
+import {
+  IGatsbyState,
+  IDeleteCacheAction,
+  ICreateClientVisitedPage,
+  ICreateServerVisitedPage,
+} from "../types"
+
+type StateMap = Map<"client" | "server", Set<string>>
+
+// The develop server always wants these page components.
+const createDefault = (): StateMap => {
+  const defaults = new Set<string>()
+  defaults.add(`component---cache-dev-404-page-js`)
+  defaults.add(`component---src-pages-404-js`)
+
+  const state: StateMap = new Map([
+    [`client`, new Set(defaults)],
+    [`server`, new Set(defaults)],
+  ])
+
+  return state
+}
+
+export const visitedPagesReducer = (
+  state: IGatsbyState["visitedPages"] = createDefault(),
+  action:
+    | IDeleteCacheAction
+    | ICreateClientVisitedPage
+    | ICreateServerVisitedPage
+): IGatsbyState["visitedPages"] => {
+  switch (action.type) {
+    case `DELETE_CACHE`:
+      return createDefault()
+
+    case `CREATE_CLIENT_VISITED_PAGE`: {
+      const client = state.get(`client`)
+      if (client) {
+        client.add(action.payload.componentChunkName)
+      }
+
+      return state
+    }
+
+    case `CREATE_SERVER_VISITED_PAGE`: {
+      const server = state.get(`server`)
+      if (server) {
+        server.add(action.payload.componentChunkName)
+      }
+
+      return state
+    }
+
+    default:
+      return state
+  }
+}

packages/gatsby/src/redux/types.ts

@@ -276,7 +276,7 @@ export interface IGatsbyState {
   }
   pageDataStats: Map<SystemPath, number>
   pageData: Map<Identifier, string>
-  clientVisitedPages: Set<string>
+  visitedPages: Map<string, Set<string>>
 }
 
 export interface ICachedReduxState {
@@ -603,6 +603,12 @@ export interface ICreateClientVisitedPage {
   plugin?: IGatsbyPlugin
 }
 
+export interface ICreateServerVisitedPage {
+  type: `CREATE_SERVER_VISITED_PAGE`
+  payload: IGatsbyPage
+  plugin?: IGatsbyPlugin
+}
+
 export interface ICreatePageAction {
   type: `CREATE_PAGE`
   payload: IGatsbyPage