From 70b81a6e825c583387728c02d83a70e0d4e16072 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 23 Nov 2020 08:57:39 -0800 Subject: [PATCH] feat(gatsby): make dev ssr bundling lazy (#28149) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(gatsby): make dev ssr bundling lazy * fix tests * fix race condition * typescript 😅 * await getPageData to ensure it's been created before SSRing * instance isn't set in SSR * Fix a few more instances of this env var * Avoid setting anything for experimental features Co-authored-by: Michal Piechowiak --- .../cache-dir/__tests__/static-entry.js | 2 +- .../cache-dir/ssr-develop-static-entry.js | 2 +- .../gatsby/src/bootstrap/requires-writer.ts | 85 ++++++++++++++----- packages/gatsby/src/redux/actions/public.js | 13 +++ .../src/redux/reducers/client-visited-page.ts | 29 ------- packages/gatsby/src/redux/reducers/index.ts | 4 +- .../gatsby/src/redux/reducers/visited-page.ts | 56 ++++++++++++ packages/gatsby/src/redux/types.ts | 8 +- .../src/utils/dev-ssr/render-dev-html.ts | 85 ++++++++++++++++++- 9 files changed, 229 insertions(+), 55 deletions(-) delete mode 100644 packages/gatsby/src/redux/reducers/client-visited-page.ts create mode 100644 packages/gatsby/src/redux/reducers/visited-page.ts diff --git a/packages/gatsby/cache-dir/__tests__/static-entry.js b/packages/gatsby/cache-dir/__tests__/static-entry.js index 011e7b65ad3bb..4f26a9c6bb6dd 100644 --- a/packages/gatsby/cache-dir/__tests__/static-entry.js +++ b/packages/gatsby/cache-dir/__tests__/static-entry.js @@ -21,7 +21,7 @@ jest.mock( `$virtual/ssr-sync-requires`, () => { return { - components: { + ssrComponents: { "page-component---src-pages-test-js": () => null, }, } diff --git a/packages/gatsby/cache-dir/ssr-develop-static-entry.js b/packages/gatsby/cache-dir/ssr-develop-static-entry.js index efd2083cd401d..f35cb8a3db0c8 100644 --- a/packages/gatsby/cache-dir/ssr-develop-static-entry.js +++ b/packages/gatsby/cache-dir/ssr-develop-static-entry.js @@ -131,7 +131,7 @@ export default (pagePath, isClientOnlyPage, callback) => { } const pageElement = createElement( - syncRequires.components[componentChunkName], + syncRequires.ssrComponents[componentChunkName], props ) diff --git a/packages/gatsby/src/bootstrap/requires-writer.ts b/packages/gatsby/src/bootstrap/requires-writer.ts index d607937f861df..cb4bc1e1644d2 100644 --- a/packages/gatsby/src/bootstrap/requires-writer.ts +++ b/packages/gatsby/src/bootstrap/requires-writer.ts @@ -164,7 +164,8 @@ const createHash = ( matchPaths: Array, components: Array, cleanedClientVisitedPageComponents: Array, - notVisitedPageComponents: Array + notVisitedPageComponents: Array, + cleanedSSRVisitedPageComponents: Array ): string => crypto .createHash(`md5`) @@ -174,6 +175,7 @@ const createHash = ( components, cleanedClientVisitedPageComponents, notVisitedPageComponents, + cleanedSSRVisitedPageComponents, }) ) .digest(`hex`) @@ -184,11 +186,27 @@ export const writeAll = async (state: IGatsbyState): Promise => { const pages = [...state.pages.values()] const matchPaths = getMatchPaths(pages) const components = getComponents(pages) + let cleanedSSRVisitedPageComponents: Array = [] + + if (process.env.GATSBY_EXPERIMENTAL_DEV_SSR) { + const ssrVisitedPageComponents = [ + ...(state.visitedPages.get(`server`)?.values() || []), + ] + + // Remove any page components that no longer exist. + cleanedSSRVisitedPageComponents = components.filter(c => + ssrVisitedPageComponents.some(s => s === c.componentChunkName) + ) + } + + let cleanedClientVisitedPageComponents: Array = [] + let notVisitedPageComponents: Array = [] - let cleanedClientVisitedPageComponents: Array = components - let notVisitedPageComponents: Array = components if (process.env.GATSBY_EXPERIMENTAL_LAZY_DEVJS) { - const clientVisitedPageComponents = [...state.clientVisitedPages.values()] + const clientVisitedPageComponents = [ + ...(state.visitedPages.get(`client`)?.values() || []), + ] + // Remove any page components that no longer exist. cleanedClientVisitedPageComponents = components.filter(component => clientVisitedPageComponents.some( @@ -211,7 +229,8 @@ export const writeAll = async (state: IGatsbyState): Promise => { matchPaths, components, cleanedClientVisitedPageComponents, - notVisitedPageComponents + notVisitedPageComponents, + cleanedSSRVisitedPageComponents ) if (newHash === lastHash) { @@ -229,6 +248,25 @@ export const writeAll = async (state: IGatsbyState): Promise => { const hotMethod = process.env.GATSBY_HOT_LOADER !== `fast-refresh` ? `hot` : `` + if (process.env.GATSBY_EXPERIMENTAL_DEV_SSR) { + // Create file with sync requires of visited page components files. + let lazySyncRequires = `${hotImport} + // prefer default export if available + const preferDefault = m => (m && m.default) || m + \n\n` + lazySyncRequires += `exports.ssrComponents = {\n${cleanedSSRVisitedPageComponents + .map( + (c: IGatsbyPageComponent): string => + ` "${ + c.componentChunkName + }": ${hotMethod}(preferDefault(require("${joinPath(c.component)}")))` + ) + .join(`,\n`)} + }\n\n` + + writeModule(`$virtual/ssr-sync-requires`, lazySyncRequires) + } + // Create file with sync requires of components/json files. let syncRequires = `${hotImport} @@ -245,10 +283,6 @@ const preferDefault = m => (m && m.default) || m .join(`,\n`)} }\n\n` - // webpack only seems to trigger re-renders once per virtual - // file so we need a seperate one for each webpack instance. - writeModule(`$virtual/ssr-sync-requires`, syncRequires) - if (process.env.GATSBY_EXPERIMENTAL_LAZY_DEVJS) { // Create file with sync requires of visited page components files. let lazyClientSyncRequires = `${hotImport} @@ -331,17 +365,6 @@ const preferDefault = m => (m && m.default) || m return true } -if (process.env.GATSBY_EXPERIMENTAL_LAZY_DEVJS) { - /** - * Start listening to CREATE_CLIENT_VISITED_PAGE events so we can rewrite - * files as required - */ - emitter.on(`CREATE_CLIENT_VISITED_PAGE`, (): void => { - reporter.pendingActivity({ id: `requires-writer` }) - writeAll(store.getState()) - }) -} - const debouncedWriteAll = _.debounce( async (): Promise => { const activity = reporter.activityTimer(`write out requires`, { @@ -359,6 +382,28 @@ const debouncedWriteAll = _.debounce( } ) +if (process.env.GATSBY_EXPERIMENTAL_LAZY_DEVJS) { + /** + * Start listening to CREATE_CLIENT_VISITED_PAGE events so we can rewrite + * files as required + */ + emitter.on(`CREATE_CLIENT_VISITED_PAGE`, (): void => { + reporter.pendingActivity({ id: `requires-writer` }) + debouncedWriteAll() + }) +} + +if (process.env.GATSBY_EXPERIMENTAL_DEV_SSR) { + /** + * Start listening to CREATE_SERVER_VISITED_PAGE events so we can rewrite + * files as required + */ + emitter.on(`CREATE_SERVER_VISITED_PAGE`, (): void => { + reporter.pendingActivity({ id: `requires-writer` }) + debouncedWriteAll() + }) +} + /** * Start listening to CREATE/DELETE_PAGE events so we can rewrite * files as required diff --git a/packages/gatsby/src/redux/actions/public.js b/packages/gatsby/src/redux/actions/public.js index 9b965f13d57cd..09cadf7045bf2 100644 --- a/packages/gatsby/src/redux/actions/public.js +++ b/packages/gatsby/src/redux/actions/public.js @@ -1404,4 +1404,17 @@ actions.createClientVisitedPage = (chunkName: string) => { } } +/** + * Record that a page was visited on the server.. + * + * @param {Object} $0 + * @param {string} $0.id the chunkName for the page component. + */ +actions.createServerVisitedPage = (chunkName: string) => { + return { + type: `CREATE_SERVER_VISITED_PAGE`, + payload: { componentChunkName: chunkName }, + } +} + module.exports = { actions } diff --git a/packages/gatsby/src/redux/reducers/client-visited-page.ts b/packages/gatsby/src/redux/reducers/client-visited-page.ts deleted file mode 100644 index f7108cb777919..0000000000000 --- a/packages/gatsby/src/redux/reducers/client-visited-page.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { - IGatsbyState, - IDeleteCacheAction, - ICreateClientVisitedPage, -} from "../types" - -// The develop server always wants these page components. -const defaults = new Set() -defaults.add(`component---cache-dev-404-page-js`) -defaults.add(`component---src-pages-404-js`) - -export const clientVisitedPageReducer = ( - state: IGatsbyState["clientVisitedPages"] = new Set(defaults), - action: IDeleteCacheAction | ICreateClientVisitedPage -): IGatsbyState["clientVisitedPages"] => { - switch (action.type) { - case `DELETE_CACHE`: - return new Set(defaults) - - case `CREATE_CLIENT_VISITED_PAGE`: { - state.add(action.payload.componentChunkName) - - return state - } - - default: - return state - } -} diff --git a/packages/gatsby/src/redux/reducers/index.ts b/packages/gatsby/src/redux/reducers/index.ts index 8d22ed280416f..8d5a424860191 100644 --- a/packages/gatsby/src/redux/reducers/index.ts +++ b/packages/gatsby/src/redux/reducers/index.ts @@ -27,7 +27,7 @@ import { schemaCustomizationReducer } from "./schema-customization" import { inferenceMetadataReducer } from "./inference-metadata" import { staticQueriesByTemplateReducer } from "./static-queries-by-template" import { queriesReducer } from "./queries" -import { clientVisitedPageReducer } from "./client-visited-page" +import { visitedPagesReducer } from "./visited-page" /** * @property exports.nodesTouched Set @@ -44,7 +44,7 @@ export { configReducer as config, schemaReducer as schema, pagesReducer as pages, - clientVisitedPageReducer as clientVisitedPages, + visitedPagesReducer as visitedPages, statusReducer as status, componentsReducer as components, staticQueryComponentsReducer as staticQueryComponents, diff --git a/packages/gatsby/src/redux/reducers/visited-page.ts b/packages/gatsby/src/redux/reducers/visited-page.ts new file mode 100644 index 0000000000000..e15545e91e194 --- /dev/null +++ b/packages/gatsby/src/redux/reducers/visited-page.ts @@ -0,0 +1,56 @@ +import { + IGatsbyState, + IDeleteCacheAction, + ICreateClientVisitedPage, + ICreateServerVisitedPage, +} from "../types" + +type StateMap = Map<"client" | "server", Set> + +// The develop server always wants these page components. +const createDefault = (): StateMap => { + const defaults = new Set() + defaults.add(`component---cache-dev-404-page-js`) + defaults.add(`component---src-pages-404-js`) + + const state: StateMap = new Map([ + [`client`, new Set(defaults)], + [`server`, new Set(defaults)], + ]) + + return state +} + +export const visitedPagesReducer = ( + state: IGatsbyState["visitedPages"] = createDefault(), + action: + | IDeleteCacheAction + | ICreateClientVisitedPage + | ICreateServerVisitedPage +): IGatsbyState["visitedPages"] => { + switch (action.type) { + case `DELETE_CACHE`: + return createDefault() + + case `CREATE_CLIENT_VISITED_PAGE`: { + const client = state.get(`client`) + if (client) { + client.add(action.payload.componentChunkName) + } + + return state + } + + case `CREATE_SERVER_VISITED_PAGE`: { + const server = state.get(`server`) + if (server) { + server.add(action.payload.componentChunkName) + } + + return state + } + + default: + return state + } +} diff --git a/packages/gatsby/src/redux/types.ts b/packages/gatsby/src/redux/types.ts index 1b88ff5cef2d6..fc2b5734ec7eb 100644 --- a/packages/gatsby/src/redux/types.ts +++ b/packages/gatsby/src/redux/types.ts @@ -276,7 +276,7 @@ export interface IGatsbyState { } pageDataStats: Map pageData: Map - clientVisitedPages: Set + visitedPages: Map> } export interface ICachedReduxState { @@ -603,6 +603,12 @@ export interface ICreateClientVisitedPage { plugin?: IGatsbyPlugin } +export interface ICreateServerVisitedPage { + type: `CREATE_SERVER_VISITED_PAGE` + payload: IGatsbyPage + plugin?: IGatsbyPlugin +} + export interface ICreatePageAction { type: `CREATE_PAGE` payload: IGatsbyPage diff --git a/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts b/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts index a27dcd33c9a69..70ed3f5a30746 100644 --- a/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts +++ b/packages/gatsby/src/utils/dev-ssr/render-dev-html.ts @@ -1,5 +1,7 @@ import JestWorker from "jest-worker" -import _ from "lodash" +import fs from "fs-extra" +import { joinPath } from "gatsby-core-utils" +import report from "gatsby-cli/lib/reporter" import { startListener } from "../../bootstrap/requires-writer" import { findPageByPath } from "../find-page-by-path" @@ -43,6 +45,68 @@ export const restartWorker = (htmlComponentRendererPath): void => { } } +const searchFileForString = (substring, filePath): Promise => + new Promise(resolve => { + // See if the chunk is in the newComponents array (not the notVisited). + const chunkRegex = RegExp(`exports.ssrComponents.*${substring}.*}`, `gs`) + const stream = fs.createReadStream(filePath) + let found = false + stream.on(`data`, function (d) { + if (chunkRegex.test(d.toString())) { + found = true + stream.close() + resolve(found) + } + }) + stream.on(`error`, function () { + resolve(found) + }) + stream.on(`close`, function () { + resolve(found) + }) + }) + +const ensurePathComponentInSSRBundle = async ( + page, + directory +): Promise => { + // This shouldn't happen. + if (!page) { + report.panic(`page not found`, page) + } + + // Now check if it's written to public/render-page.js + const htmlComponentRendererPath = joinPath(directory, `public/render-page.js`) + // This search takes 1-10ms + // We do it as there can be a race conditions where two pages + // are requested at the same time which means that both are told render-page.js + // has changed when the first page is complete meaning the second + // page's component won't be in the render meaning its SSR will fail. + let found = await searchFileForString( + page.componentChunkName, + htmlComponentRendererPath + ) + + if (!found) { + await new Promise(resolve => { + let readAttempts = 0 + const searchForStringInterval = setInterval(async () => { + readAttempts += 1 + found = await searchFileForString( + page.componentChunkName, + htmlComponentRendererPath + ) + if (found || readAttempts === 5) { + clearInterval(searchForStringInterval) + resolve() + } + }, 300) + }) + } + + return found +} + export const renderDevHTML = ({ path, page, @@ -64,6 +128,25 @@ export const renderDevHTML = ({ isClientOnlyPage = true } + const { boundActionCreators } = require(`../../redux/actions`) + const { + createServerVisitedPage, + createClientVisitedPage, + } = boundActionCreators + // Record this page was requested. This will kick off adding its page + // component to the ssr bundle (if that's not already happened) + createServerVisitedPage(pageObj.componentChunkName) + + // We'll also get a head start on compiling the client code (this + // call has no effect if the page component is already in the client bundle). + createClientVisitedPage(pageObj.componentChunkName) + + // Ensure the query has been run and written out. + await getPageDataExperimental(pageObj.path) + + // Wait for public/render-page.js to update w/ the page component. + await ensurePathComponentInSSRBundle(pageObj, directory) + // Ensure the query has been run and written out. await getPageDataExperimental(pageObj.path)