From 1193d2e559d97836c3c43cba7f7981e3d88db957 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Thu, 25 Jun 2020 17:32:48 +0100 Subject: [PATCH 01/44] Move bootstrap into machine --- .../gatsby/src/commands/develop-process.ts | 132 ++++++++++++++---- packages/gatsby/src/services/build-schema.ts | 5 +- .../src/services/create-pages-statefully.ts | 5 +- packages/gatsby/src/services/create-pages.ts | 16 ++- .../gatsby/src/services/customize-schema.ts | 4 +- packages/gatsby/src/services/index.ts | 74 ++++++++-- .../gatsby/src/services/post-bootstrap.ts | 9 +- packages/gatsby/src/services/source-nodes.ts | 15 +- .../src/services/write-out-redirects.ts | 4 +- .../src/state-machines/data-layer/actions.ts | 47 +++++++ .../src/state-machines/data-layer/index.ts | 95 +++++++++++++ .../src/state-machines/data-layer/services.ts | 20 +++ .../src/state-machines/data-layer/types.ts | 29 ++++ .../shared-transition-configs.ts | 35 +++++ 14 files changed, 418 insertions(+), 72 deletions(-) create mode 100644 packages/gatsby/src/state-machines/data-layer/actions.ts create mode 100644 packages/gatsby/src/state-machines/data-layer/index.ts create mode 100644 packages/gatsby/src/state-machines/data-layer/services.ts create mode 100644 packages/gatsby/src/state-machines/data-layer/types.ts create mode 100644 packages/gatsby/src/state-machines/shared-transition-configs.ts diff --git a/packages/gatsby/src/commands/develop-process.ts b/packages/gatsby/src/commands/develop-process.ts index dfd56780c9a17..1617e36849744 100644 --- a/packages/gatsby/src/commands/develop-process.ts +++ b/packages/gatsby/src/commands/develop-process.ts @@ -1,5 +1,3 @@ -import { bootstrap } from "../bootstrap" -import { store } from "../redux" import { syncStaticDir } from "../utils/get-static-dir" import report from "gatsby-cli/lib/reporter" import chalk from "chalk" @@ -29,9 +27,21 @@ import { runPageQueries, startWebpackServer, writeOutRequires, + IBuildContext, + initialize, } from "../services" import { boundActionCreators } from "../redux/actions" import { ProgramStatus } from "../redux/types" +import { + MachineConfig, + AnyEventObject, + assign, + Machine, + DoneEventObject, + interpret, +} from "xstate" +import { DataLayerResult, dataLayerMachine } from "../state-machines/data-layer" +import { IDataLayerContext } from "../state-machines/data-layer/types" // const isInteractive = process.stdout.isTTY @@ -108,34 +118,98 @@ module.exports = async (program: IProgram): Promise => { throw e } - // Start bootstrap process. - const { gatsbyNodeGraphQLFunction, workerPool } = await bootstrap({ program }) - - // Start the createPages hot reloader. - bootstrapPageHotReloader(gatsbyNodeGraphQLFunction) - - // Start the schema hot reloader. - bootstrapSchemaHotReloader() - - const { queryIds } = await calculateDirtyQueries({ store }) + const developConfig: MachineConfig = { + id: `build`, + initial: `initializing`, + states: { + initializing: { + invoke: { + src: `initialize`, + onDone: { + target: `initializingDataLayer`, + actions: `assignStoreAndWorkerPool`, + }, + }, + }, + initializingDataLayer: { + invoke: { + src: `initializeDataLayer`, + data: ({ parentSpan, store }: IBuildContext): IDataLayerContext => { + return { parentSpan, store, firstRun: true } + }, + onDone: { + actions: `assignDataLayer`, + target: `doingEverythingElse`, + }, + }, + }, + doingEverythingElse: { + invoke: { + src: async ({ + gatsbyNodeGraphQLFunction, + workerPool, + store, + }): Promise => { + // All the stuff that's not in the state machine yet + // Start the createPages hot reloader. + bootstrapPageHotReloader(gatsbyNodeGraphQLFunction) + + // Start the schema hot reloader. + bootstrapSchemaHotReloader() + + const { queryIds } = await calculateDirtyQueries({ store }) + + await runStaticQueries({ queryIds, store, program }) + await runPageQueries({ queryIds, store, program }) + await writeOutRequires({ store }) + boundActionCreators.setProgramStatus( + ProgramStatus.BOOTSTRAP_QUERY_RUNNING_FINISHED + ) + + await db.saveState() + + await waitUntilAllJobsComplete() + requiresWriter.startListener() + db.startAutosave() + queryUtil.startListeningToDevelopQueue({ + graphqlTracing: program.graphqlTracing, + }) + queryWatcher.startWatchDeletePage() + const app = express() + + await startWebpackServer({ program, app, workerPool }) + }, + }, + }, + }, + } - await runStaticQueries({ queryIds, store, program }) - await runPageQueries({ queryIds, store, program }) - await writeOutRequires({ store }) - boundActionCreators.setProgramStatus( - ProgramStatus.BOOTSTRAP_QUERY_RUNNING_FINISHED + const service = interpret( + // eslint-disable-next-line new-cap + Machine(developConfig, { + services: { + initializeDataLayer: dataLayerMachine, + initialize, + }, + actions: { + assignStoreAndWorkerPool: assign( + (_context, event) => { + console.log({ event }) + const { store, workerPool } = event.data + return { + store, + workerPool, + } + } + ), + assignDataLayer: assign( + (_, { data }): DataLayerResult => data + ), + }, + }).withContext({ program }) ) - - await db.saveState() - - await waitUntilAllJobsComplete() - requiresWriter.startListener() - db.startAutosave() - queryUtil.startListeningToDevelopQueue({ - graphqlTracing: program.graphqlTracing, + service.onTransition(state => { + console.log(`transition to`, state.value) }) - queryWatcher.startWatchDeletePage() - const app = express() - - await startWebpackServer({ program, app, workerPool }) + service.start() } diff --git a/packages/gatsby/src/services/build-schema.ts b/packages/gatsby/src/services/build-schema.ts index b74d8b186fe2a..71565adec1f26 100644 --- a/packages/gatsby/src/services/build-schema.ts +++ b/packages/gatsby/src/services/build-schema.ts @@ -1,12 +1,11 @@ -import { IBuildContext } from "./" - import { build } from "../schema" import reporter from "gatsby-cli/lib/reporter" +import { IDataLayerContext } from "../state-machines/data-layer/types" export async function buildSchema({ store, parentSpan, -}: Partial): Promise { +}: Partial): Promise { if (!store) { reporter.panic(`Cannot build schema before store initialization`) } diff --git a/packages/gatsby/src/services/create-pages-statefully.ts b/packages/gatsby/src/services/create-pages-statefully.ts index 88fab343afee6..a1c4798077521 100644 --- a/packages/gatsby/src/services/create-pages-statefully.ts +++ b/packages/gatsby/src/services/create-pages-statefully.ts @@ -1,12 +1,11 @@ -import { IBuildContext } from "./" - import reporter from "gatsby-cli/lib/reporter" import apiRunnerNode from "../utils/api-runner-node" +import { IDataLayerContext } from "../state-machines/data-layer/types" export async function createPagesStatefully({ parentSpan, gatsbyNodeGraphQLFunction, -}: Partial): Promise { +}: Partial): Promise { // A variant on createPages for plugins that want to // have full control over adding/removing pages. The normal // "createPages" API is called every time (during development) diff --git a/packages/gatsby/src/services/create-pages.ts b/packages/gatsby/src/services/create-pages.ts index 1ba27d8f9806d..f5c0b1eee60e1 100644 --- a/packages/gatsby/src/services/create-pages.ts +++ b/packages/gatsby/src/services/create-pages.ts @@ -1,13 +1,15 @@ -import { IBuildContext } from "./" - import reporter from "gatsby-cli/lib/reporter" import apiRunnerNode from "../utils/api-runner-node" +import { IDataLayerContext } from "../state-machines/data-layer/types" export async function createPages({ parentSpan, gatsbyNodeGraphQLFunction, store, -}: Partial): Promise { +}: Partial): Promise<{ + deletedPages: string[] + changedPages: string[] +}> { if (!store) { reporter.panic(`store not initialized`) } @@ -60,8 +62,8 @@ export async function createPages({ // ) // tim.end() - // return { - // changedPages, - // deletedPages, - // } + return { + changedPages: [], + deletedPages: [], + } } diff --git a/packages/gatsby/src/services/customize-schema.ts b/packages/gatsby/src/services/customize-schema.ts index d261f3c2da5ef..e9dcf4328439b 100644 --- a/packages/gatsby/src/services/customize-schema.ts +++ b/packages/gatsby/src/services/customize-schema.ts @@ -1,11 +1,11 @@ import reporter from "gatsby-cli/lib/reporter" import { createSchemaCustomization } from "../utils/create-schema-customization" -import { IBuildContext } from "./" +import { IDataLayerContext } from "../state-machines/data-layer/types" export async function customizeSchema({ parentSpan, refresh, // webhookBody,//coming soon -}: Partial): Promise { +}: Partial): Promise { const activity = reporter.activityTimer(`createSchemaCustomization`, { parentSpan, }) diff --git a/packages/gatsby/src/services/index.ts b/packages/gatsby/src/services/index.ts index 2c498710ee80f..3025520f9074b 100644 --- a/packages/gatsby/src/services/index.ts +++ b/packages/gatsby/src/services/index.ts @@ -1,16 +1,60 @@ -export { startWebpackServer } from "./start-webpack-server" -export { rebuildSchemaWithSitePage } from "./rebuild-schema-with-site-pages" -export { extractQueries } from "./extract-queries" -export { writeOutRedirects } from "./write-out-redirects" -export { postBootstrap } from "./post-bootstrap" -export { buildSchema } from "./build-schema" -export { createPages } from "./create-pages" -export { createPagesStatefully } from "./create-pages-statefully" -export { customizeSchema } from "./customize-schema" -export { initialize } from "./initialize" -export { sourceNodes } from "./source-nodes" -export { writeOutRequires } from "./write-out-requires" -export { calculateDirtyQueries } from "./calculate-dirty-queries" -export { runStaticQueries } from "./run-static-queries" -export { runPageQueries } from "./run-page-queries" +import { ServiceConfig } from "xstate" +import { IBuildContext } from "./" + +import { startWebpackServer } from "./start-webpack-server" +import { rebuildSchemaWithSitePage } from "./rebuild-schema-with-site-pages" +import { extractQueries } from "./extract-queries" +import { writeOutRedirects } from "./write-out-redirects" +import { postBootstrap } from "./post-bootstrap" +import { buildSchema } from "./build-schema" +import { createPages } from "./create-pages" +import { createPagesStatefully } from "./create-pages-statefully" +import { customizeSchema } from "./customize-schema" +import { initialize } from "./initialize" +import { sourceNodes } from "./source-nodes" +import { writeOutRequires } from "./write-out-requires" +import { calculateDirtyQueries } from "./calculate-dirty-queries" +import { runStaticQueries } from "./run-static-queries" +import { runPageQueries } from "./run-page-queries" + +import { waitUntilAllJobsComplete } from "../utils/wait-until-jobs-complete" + export * from "./types" + +export { + customizeSchema, + sourceNodes, + createPages, + buildSchema, + createPagesStatefully, + extractQueries, + writeOutRequires, + calculateDirtyQueries, + runStaticQueries, + runPageQueries, + initialize, + waitUntilAllJobsComplete, + postBootstrap, + writeOutRedirects, + startWebpackServer, + rebuildSchemaWithSitePage, +} + +export const buildServices: Record> = { + customizeSchema, + sourceNodes, + createPages, + buildSchema, + createPagesStatefully, + extractQueries, + writeOutRequires, + calculateDirtyQueries, + runStaticQueries, + runPageQueries, + initialize, + waitUntilAllJobsComplete, + postBootstrap, + writeOutRedirects, + startWebpackServer, + rebuildSchemaWithSitePage, +} diff --git a/packages/gatsby/src/services/post-bootstrap.ts b/packages/gatsby/src/services/post-bootstrap.ts index 48061182744ba..58fd98b4b82dd 100644 --- a/packages/gatsby/src/services/post-bootstrap.ts +++ b/packages/gatsby/src/services/post-bootstrap.ts @@ -1,11 +1,12 @@ import reporter from "gatsby-cli/lib/reporter" -import { IBuildContext } from "./" import { emitter } from "../redux" import apiRunnerNode from "../utils/api-runner-node" +import { IDataLayerContext } from "../state-machines/data-layer/types" +import { boundActionCreators } from "../redux/actions" export async function postBootstrap({ parentSpan, -}: Partial): Promise { +}: Partial): Promise { const activity = reporter.activityTimer(`onPostBootstrap`, { parentSpan, }) @@ -17,7 +18,5 @@ export async function postBootstrap({ bootstrap finished - ${process.uptime().toFixed(3)}s `) emitter.emit(`BOOTSTRAP_FINISHED`, {}) - require(`../redux/actions`).boundActionCreators.setProgramStatus( - `BOOTSTRAP_FINISHED` - ) + boundActionCreators.setProgramStatus(`BOOTSTRAP_FINISHED`) } diff --git a/packages/gatsby/src/services/source-nodes.ts b/packages/gatsby/src/services/source-nodes.ts index 3ac9b83029da5..48fa9dd5656f4 100644 --- a/packages/gatsby/src/services/source-nodes.ts +++ b/packages/gatsby/src/services/source-nodes.ts @@ -1,6 +1,6 @@ -import { IBuildContext } from "./" import sourceNodesAndRemoveStaleNodes from "../utils/source-nodes" import reporter from "gatsby-cli/lib/reporter" +import { IDataLayerContext } from "../state-machines/data-layer/types" // import { findChangedPages } from "../utils/check-for-changed-pages" // import { IGatsbyPage } from "../redux/types" @@ -8,7 +8,10 @@ export async function sourceNodes({ parentSpan, webhookBody, store, -}: Partial): Promise { +}: Partial): Promise<{ + deletedPages: string[] + changedPages: string[] +}> { if (!store) { reporter.panic(`No redux store`) } @@ -54,8 +57,8 @@ export async function sourceNodes({ // tim.end() activity.end() - // return { - // deletedPages, - // changedPages, - // } + return { + deletedPages: [], + changedPages: [], + } } diff --git a/packages/gatsby/src/services/write-out-redirects.ts b/packages/gatsby/src/services/write-out-redirects.ts index a3bf974151fac..d7c30f690e390 100644 --- a/packages/gatsby/src/services/write-out-redirects.ts +++ b/packages/gatsby/src/services/write-out-redirects.ts @@ -1,10 +1,10 @@ -import { IBuildContext } from "./" import reporter from "gatsby-cli/lib/reporter" import { writeRedirects } from "../bootstrap/redirects-writer" +import { IDataLayerContext } from "../state-machines/data-layer/types" export async function writeOutRedirects({ parentSpan, -}: Partial): Promise { +}: Partial): Promise { // Write out redirects. const activity = reporter.activityTimer(`write out redirect data`, { parentSpan, diff --git a/packages/gatsby/src/state-machines/data-layer/actions.ts b/packages/gatsby/src/state-machines/data-layer/actions.ts new file mode 100644 index 0000000000000..54af7989edaa8 --- /dev/null +++ b/packages/gatsby/src/state-machines/data-layer/actions.ts @@ -0,0 +1,47 @@ +import { + assign, + AnyEventObject, + ActionFunction, + AssignAction, + DoneInvokeEvent, + ActionFunctionMap, +} from "xstate" +import { createGraphQLRunner } from "../../bootstrap/create-graphql-runner" +import reporter from "gatsby-cli/lib/reporter" +import { IDataLayerContext } from "./types" + +const concatUnique = (array1?: T[], array2?: T[]): T[] => + Array.from(new Set((array1 || []).concat(array2 || []))) + +type BuildMachineAction = + | ActionFunction + | AssignAction + +export const assignChangedPages: BuildMachineAction = assign< + IDataLayerContext, + DoneInvokeEvent<{ + changedPages: string[] + deletedPages: string[] + }> +>((context, event) => { + console.log({ event }) + return { + pagesToBuild: concatUnique(context.pagesToBuild, event.data.changedPages), + pagesToDelete: concatUnique(context.pagesToDelete, event.data.deletedPages), + } +}) + +export const assignGatsbyNodeGraphQL: BuildMachineAction = assign< + IDataLayerContext +>({ + gatsbyNodeGraphQLFunction: ({ store }: IDataLayerContext) => + store ? createGraphQLRunner(store, reporter) : undefined, +}) + +export const dataLayerActions: ActionFunctionMap< + IDataLayerContext, + AnyEventObject +> = { + assignChangedPages, + assignGatsbyNodeGraphQL, +} diff --git a/packages/gatsby/src/state-machines/data-layer/index.ts b/packages/gatsby/src/state-machines/data-layer/index.ts new file mode 100644 index 0000000000000..ab745a5c3620b --- /dev/null +++ b/packages/gatsby/src/state-machines/data-layer/index.ts @@ -0,0 +1,95 @@ +import { MachineConfig, Machine } from "xstate" +import { dataLayerActions } from "./actions" +import { IDataLayerContext } from "./types" +import { dataLayerServices } from "./services" + +export type DataLayerResult = Pick< + IDataLayerContext, + | "gatsbyNodeGraphQLFunction" + | "graphqlRunner" + | "pagesToBuild" + | "pagesToDelete" +> + +export const dataLayerStates: MachineConfig = { + initial: `customizingSchema`, + states: { + customizingSchema: { + invoke: { + src: `customizeSchema`, + id: `customizing-schema`, + onDone: { + target: `sourcingNodes`, + }, + }, + }, + sourcingNodes: { + invoke: { + src: `sourceNodes`, + id: `sourcing-nodes`, + onDone: { + target: `buildingSchema`, + actions: `assignChangedPages`, + }, + }, + }, + buildingSchema: { + invoke: { + id: `building-schema`, + src: `buildSchema`, + onDone: { + target: `creatingPages`, + actions: `assignGatsbyNodeGraphQL`, + }, + }, + }, + creatingPages: { + invoke: { + id: `creating-pages`, + src: `createPages`, + onDone: [ + { + target: `creatingPagesStatefully`, + actions: `assignChangedPages`, + cond: (context): boolean => !!context.firstRun, + }, + { + target: `done`, + actions: `assignChangedPages`, + }, + ], + }, + }, + creatingPagesStatefully: { + invoke: { + src: `createPagesStatefully`, + id: `creating-pages-statefully`, + onDone: { + target: `done`, + }, + }, + }, + done: { + type: `final`, + data: ({ + gatsbyNodeGraphQLFunction, + graphqlRunner, + pagesToBuild, + pagesToDelete, + }): DataLayerResult => { + return { + gatsbyNodeGraphQLFunction, + graphqlRunner, + pagesToBuild, + pagesToDelete, + } + }, + }, + }, +} + +// eslint-disable-next-line new-cap +export const dataLayerMachine = Machine(dataLayerStates, { + actions: dataLayerActions, + services: dataLayerServices, +}) diff --git a/packages/gatsby/src/state-machines/data-layer/services.ts b/packages/gatsby/src/state-machines/data-layer/services.ts new file mode 100644 index 0000000000000..fd04e3c19df98 --- /dev/null +++ b/packages/gatsby/src/state-machines/data-layer/services.ts @@ -0,0 +1,20 @@ +import { ServiceConfig } from "xstate" +import { + customizeSchema, + createPages, + createPagesStatefully, + buildSchema, + sourceNodes, +} from "../../services" +import { IDataLayerContext } from "./types" + +export const dataLayerServices: Record< + string, + ServiceConfig +> = { + customizeSchema, + sourceNodes, + createPages, + buildSchema, + createPagesStatefully, +} diff --git a/packages/gatsby/src/state-machines/data-layer/types.ts b/packages/gatsby/src/state-machines/data-layer/types.ts new file mode 100644 index 0000000000000..155c83021a059 --- /dev/null +++ b/packages/gatsby/src/state-machines/data-layer/types.ts @@ -0,0 +1,29 @@ +import { Span } from "opentracing" +import { IProgram } from "../../commands/types" +import { Runner } from "../../bootstrap/create-graphql-runner" +import { GraphQLRunner } from "../../query/graphql-runner" +import { Store, AnyAction } from "redux" +import { IGatsbyState } from "../../redux/types" +import JestWorker from "jest-worker" +export interface IGroupedQueryIds { + pageQueryIds: string[] + staticQueryIds: string[] +} + +export interface IMutationAction { + type: string + payload: unknown[] +} +export interface IDataLayerContext { + firstRun?: boolean + program?: IProgram + store?: Store + parentSpan?: Span + gatsbyNodeGraphQLFunction?: Runner + graphqlRunner?: GraphQLRunner + webhookBody?: Record + refresh?: boolean + workerPool?: JestWorker + pagesToBuild?: string[] + pagesToDelete?: string[] +} diff --git a/packages/gatsby/src/state-machines/shared-transition-configs.ts b/packages/gatsby/src/state-machines/shared-transition-configs.ts new file mode 100644 index 0000000000000..38ef502c9804a --- /dev/null +++ b/packages/gatsby/src/state-machines/shared-transition-configs.ts @@ -0,0 +1,35 @@ +import { TransitionConfig, AnyEventObject, DoneInvokeEvent } from "xstate" +import { IBuildContext } from "../services" + +type BuildTransition = TransitionConfig + +/** + * Event handler used in all states where we're not ready to process node + * mutations. Instead we add it to a batch to process when we're next idle + */ +export const ADD_NODE_MUTATION: BuildTransition = { + actions: `addNodeMutation`, +} + +/** + * Event handler used in all states where we're not ready to process a file change + * Instead we add it to a batch to process when we're next idle + */ +export const SOURCE_FILE_CHANGED: BuildTransition = { + actions: `markFilesDirty`, +} + +/** + * When running queries we might add nodes (e.g from resolvers). If so we'll + * want to re-run queries and schema inference + */ +export const runMutationAndMarkDirty: BuildTransition = { + actions: [`markNodesDirty`, `callApi`], +} + +export const onError: TransitionConfig> = { + target: `#build.waiting`, + actions: (_context, event): void => { + console.error(`Error`, event) + }, +} From 696d5fa45d289c03f3cab0a98926ea9548982625 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Fri, 26 Jun 2020 10:29:17 +0100 Subject: [PATCH 02/44] Add parent span and query extraction --- .../gatsby/src/commands/develop-process.ts | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/packages/gatsby/src/commands/develop-process.ts b/packages/gatsby/src/commands/develop-process.ts index 1617e36849744..d6333e442b91b 100644 --- a/packages/gatsby/src/commands/develop-process.ts +++ b/packages/gatsby/src/commands/develop-process.ts @@ -29,6 +29,8 @@ import { writeOutRequires, IBuildContext, initialize, + postBootstrap, + extractQueries, } from "../services" import { boundActionCreators } from "../redux/actions" import { ProgramStatus } from "../redux/types" @@ -42,6 +44,9 @@ import { } from "xstate" import { DataLayerResult, dataLayerMachine } from "../state-machines/data-layer" import { IDataLayerContext } from "../state-machines/data-layer/types" +import { globalTracer } from "opentracing" + +const tracer = globalTracer() // const isInteractive = process.stdout.isTTY @@ -76,6 +81,8 @@ process.on(`message`, msg => { } }) +const bootstrapSpan = tracer.startSpan(`bootstrap`) + module.exports = async (program: IProgram): Promise => { // We want to prompt the feedback request when users quit develop // assuming they pass the heuristic check to know they are a user @@ -151,6 +158,15 @@ module.exports = async (program: IProgram): Promise => { store, }): Promise => { // All the stuff that's not in the state machine yet + + // These were previously in `bootstrap()` but are now + // in part of the state machine that hasn't been added yet + await postBootstrap({ parentSpan: bootstrapSpan }) + await extractQueries({ parentSpan: bootstrapSpan }) + bootstrapSpan.finish() + + // These are the parts that weren't in bootstrap + // Start the createPages hot reloader. bootstrapPageHotReloader(gatsbyNodeGraphQLFunction) @@ -206,7 +222,7 @@ module.exports = async (program: IProgram): Promise => { (_, { data }): DataLayerResult => data ), }, - }).withContext({ program }) + }).withContext({ program, parentSpan: bootstrapSpan }) ) service.onTransition(state => { console.log(`transition to`, state.value) From 77b7f3b3783d994b4820cafeeabab48c5aa502d5 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Fri, 26 Jun 2020 10:42:44 +0100 Subject: [PATCH 03/44] Add rebuildSchemaWithSitePage --- packages/gatsby/src/commands/develop-process.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/gatsby/src/commands/develop-process.ts b/packages/gatsby/src/commands/develop-process.ts index d6333e442b91b..52678eb63438e 100644 --- a/packages/gatsby/src/commands/develop-process.ts +++ b/packages/gatsby/src/commands/develop-process.ts @@ -31,6 +31,7 @@ import { initialize, postBootstrap, extractQueries, + rebuildSchemaWithSitePage, } from "../services" import { boundActionCreators } from "../redux/actions" import { ProgramStatus } from "../redux/types" @@ -162,6 +163,8 @@ module.exports = async (program: IProgram): Promise => { // These were previously in `bootstrap()` but are now // in part of the state machine that hasn't been added yet await postBootstrap({ parentSpan: bootstrapSpan }) + await rebuildSchemaWithSitePage({ parentSpan: bootstrapSpan }) + await extractQueries({ parentSpan: bootstrapSpan }) bootstrapSpan.finish() From 6f97db67f2371ddb6712b451ede60fb27ba1e0c3 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Fri, 26 Jun 2020 10:48:01 +0100 Subject: [PATCH 04/44] Use values from context --- packages/gatsby/src/commands/develop-process.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/gatsby/src/commands/develop-process.ts b/packages/gatsby/src/commands/develop-process.ts index 52678eb63438e..619550220c71f 100644 --- a/packages/gatsby/src/commands/develop-process.ts +++ b/packages/gatsby/src/commands/develop-process.ts @@ -126,6 +126,8 @@ module.exports = async (program: IProgram): Promise => { throw e } + const app = express() + const developConfig: MachineConfig = { id: `build`, initial: `initializing`, @@ -155,8 +157,10 @@ module.exports = async (program: IProgram): Promise => { invoke: { src: async ({ gatsbyNodeGraphQLFunction, + graphqlRunner, workerPool, store, + app, }): Promise => { // All the stuff that's not in the state machine yet @@ -178,8 +182,8 @@ module.exports = async (program: IProgram): Promise => { const { queryIds } = await calculateDirtyQueries({ store }) - await runStaticQueries({ queryIds, store, program }) - await runPageQueries({ queryIds, store, program }) + await runStaticQueries({ queryIds, store, program, graphqlRunner }) + await runPageQueries({ queryIds, store, program, graphqlRunner }) await writeOutRequires({ store }) boundActionCreators.setProgramStatus( ProgramStatus.BOOTSTRAP_QUERY_RUNNING_FINISHED @@ -194,7 +198,6 @@ module.exports = async (program: IProgram): Promise => { graphqlTracing: program.graphqlTracing, }) queryWatcher.startWatchDeletePage() - const app = express() await startWebpackServer({ program, app, workerPool }) }, @@ -225,7 +228,7 @@ module.exports = async (program: IProgram): Promise => { (_, { data }): DataLayerResult => data ), }, - }).withContext({ program, parentSpan: bootstrapSpan }) + }).withContext({ program, parentSpan: bootstrapSpan, app }) ) service.onTransition(state => { console.log(`transition to`, state.value) From 36e8ef45cb85a598fbadfcb2814b9b63f18b728f Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Fri, 26 Jun 2020 10:51:40 +0100 Subject: [PATCH 05/44] Remove logs --- packages/gatsby/src/commands/develop-process.ts | 4 ++-- packages/gatsby/src/state-machines/data-layer/actions.ts | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/gatsby/src/commands/develop-process.ts b/packages/gatsby/src/commands/develop-process.ts index 619550220c71f..2b2785ffbe0b0 100644 --- a/packages/gatsby/src/commands/develop-process.ts +++ b/packages/gatsby/src/commands/develop-process.ts @@ -46,6 +46,7 @@ import { import { DataLayerResult, dataLayerMachine } from "../state-machines/data-layer" import { IDataLayerContext } from "../state-machines/data-layer/types" import { globalTracer } from "opentracing" +import reporter from "gatsby-cli/lib/reporter" const tracer = globalTracer() @@ -216,7 +217,6 @@ module.exports = async (program: IProgram): Promise => { actions: { assignStoreAndWorkerPool: assign( (_context, event) => { - console.log({ event }) const { store, workerPool } = event.data return { store, @@ -231,7 +231,7 @@ module.exports = async (program: IProgram): Promise => { }).withContext({ program, parentSpan: bootstrapSpan, app }) ) service.onTransition(state => { - console.log(`transition to`, state.value) + reporter.verbose(`transition to ${JSON.stringify(state.value)}`) }) service.start() } diff --git a/packages/gatsby/src/state-machines/data-layer/actions.ts b/packages/gatsby/src/state-machines/data-layer/actions.ts index 54af7989edaa8..0e7fbc38c785a 100644 --- a/packages/gatsby/src/state-machines/data-layer/actions.ts +++ b/packages/gatsby/src/state-machines/data-layer/actions.ts @@ -24,7 +24,6 @@ export const assignChangedPages: BuildMachineAction = assign< deletedPages: string[] }> >((context, event) => { - console.log({ event }) return { pagesToBuild: concatUnique(context.pagesToBuild, event.data.changedPages), pagesToDelete: concatUnique(context.pagesToDelete, event.data.deletedPages), From 07d074196994f0d9b30f7af0abb202297f98ef04 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Fri, 26 Jun 2020 11:42:39 +0100 Subject: [PATCH 06/44] Add redirectListener --- packages/gatsby/src/commands/develop-process.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/gatsby/src/commands/develop-process.ts b/packages/gatsby/src/commands/develop-process.ts index 2b2785ffbe0b0..c5a58fa5fcf83 100644 --- a/packages/gatsby/src/commands/develop-process.ts +++ b/packages/gatsby/src/commands/develop-process.ts @@ -17,7 +17,7 @@ import { userPassesFeedbackRequestHeuristic, showFeedbackRequest, } from "../utils/feedback" - +import { startRedirectListener } from "../bootstrap/redirects-writer" import { markWebpackStatusAsPending } from "../utils/webpack-status" import { IProgram } from "./types" @@ -32,6 +32,7 @@ import { postBootstrap, extractQueries, rebuildSchemaWithSitePage, + writeOutRedirects, } from "../services" import { boundActionCreators } from "../redux/actions" import { ProgramStatus } from "../redux/types" @@ -171,6 +172,9 @@ module.exports = async (program: IProgram): Promise => { await rebuildSchemaWithSitePage({ parentSpan: bootstrapSpan }) await extractQueries({ parentSpan: bootstrapSpan }) + await writeOutRedirects({ parentSpan: bootstrapSpan }) + + startRedirectListener() bootstrapSpan.finish() // These are the parts that weren't in bootstrap From cd3a90944a3f61731b05a030f82c1828ca545f51 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Fri, 26 Jun 2020 14:13:27 +0100 Subject: [PATCH 07/44] Changes from review --- .../src/state-machines/data-layer/actions.ts | 4 +-- .../shared-transition-configs.ts | 35 ------------------- 2 files changed, 2 insertions(+), 37 deletions(-) delete mode 100644 packages/gatsby/src/state-machines/shared-transition-configs.ts diff --git a/packages/gatsby/src/state-machines/data-layer/actions.ts b/packages/gatsby/src/state-machines/data-layer/actions.ts index 0e7fbc38c785a..ef8d48ff21210 100644 --- a/packages/gatsby/src/state-machines/data-layer/actions.ts +++ b/packages/gatsby/src/state-machines/data-layer/actions.ts @@ -10,8 +10,8 @@ import { createGraphQLRunner } from "../../bootstrap/create-graphql-runner" import reporter from "gatsby-cli/lib/reporter" import { IDataLayerContext } from "./types" -const concatUnique = (array1?: T[], array2?: T[]): T[] => - Array.from(new Set((array1 || []).concat(array2 || []))) +const concatUnique = (array1: T[] = [], array2: T[] = []): T[] => + Array.from(new Set(array1.concat(array2))) type BuildMachineAction = | ActionFunction diff --git a/packages/gatsby/src/state-machines/shared-transition-configs.ts b/packages/gatsby/src/state-machines/shared-transition-configs.ts deleted file mode 100644 index 38ef502c9804a..0000000000000 --- a/packages/gatsby/src/state-machines/shared-transition-configs.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { TransitionConfig, AnyEventObject, DoneInvokeEvent } from "xstate" -import { IBuildContext } from "../services" - -type BuildTransition = TransitionConfig - -/** - * Event handler used in all states where we're not ready to process node - * mutations. Instead we add it to a batch to process when we're next idle - */ -export const ADD_NODE_MUTATION: BuildTransition = { - actions: `addNodeMutation`, -} - -/** - * Event handler used in all states where we're not ready to process a file change - * Instead we add it to a batch to process when we're next idle - */ -export const SOURCE_FILE_CHANGED: BuildTransition = { - actions: `markFilesDirty`, -} - -/** - * When running queries we might add nodes (e.g from resolvers). If so we'll - * want to re-run queries and schema inference - */ -export const runMutationAndMarkDirty: BuildTransition = { - actions: [`markNodesDirty`, `callApi`], -} - -export const onError: TransitionConfig> = { - target: `#build.waiting`, - actions: (_context, event): void => { - console.error(`Error`, event) - }, -} From d2b615bdc52d1bf1dc21fe72bb37061529bba512 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Fri, 26 Jun 2020 16:57:52 +0100 Subject: [PATCH 08/44] Log child state transitions --- .../gatsby/src/commands/develop-process.ts | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/packages/gatsby/src/commands/develop-process.ts b/packages/gatsby/src/commands/develop-process.ts index c5a58fa5fcf83..b88bb878d7a81 100644 --- a/packages/gatsby/src/commands/develop-process.ts +++ b/packages/gatsby/src/commands/develop-process.ts @@ -43,6 +43,8 @@ import { Machine, DoneEventObject, interpret, + Actor, + Interpreter, } from "xstate" import { DataLayerResult, dataLayerMachine } from "../state-machines/data-layer" import { IDataLayerContext } from "../state-machines/data-layer/types" @@ -234,8 +236,27 @@ module.exports = async (program: IProgram): Promise => { }, }).withContext({ program, parentSpan: bootstrapSpan, app }) ) + + const isInterpreter = ( + actor: Actor | Interpreter + ): actor is Interpreter => `machine` in actor + + const listeners = new WeakSet() service.onTransition(state => { - reporter.verbose(`transition to ${JSON.stringify(state.value)}`) + console.log(`Transition to ${JSON.stringify(state.value)}`) + // eslint-disable-next-line no-unused-expressions + service.children?.forEach(child => { + // We want to ensure we don't attach a listener to the same + // actor. We don't need to worry about detaching the listener + // because xstate handles that for us when the actor is stopped. + + if (isInterpreter(child) && !listeners.has(child)) { + child.onTransition(substate => { + console.log(`Transition to`, state.value, `>`, substate.value) + }) + listeners.add(child) + } + }) }) service.start() } From 01280a9aa01ad2ee0a7b36b859e3f36d7aff4a99 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Mon, 29 Jun 2020 10:38:14 +0100 Subject: [PATCH 09/44] Add state machine for query running --- .../gatsby/src/commands/develop-process.ts | 59 ++++++--- packages/gatsby/src/query/query-watcher.js | 5 +- .../src/services/calculate-dirty-queries.ts | 5 +- .../gatsby/src/services/extract-queries.ts | 5 +- .../rebuild-schema-with-site-pages.ts | 4 +- .../gatsby/src/services/run-page-queries.ts | 4 +- .../gatsby/src/services/run-static-queries.ts | 4 +- .../src/services/write-out-redirects.ts | 4 +- .../gatsby/src/services/write-out-requires.ts | 4 +- .../state-machines/query-running/actions.ts | 50 ++++++++ .../src/state-machines/query-running/index.ts | 115 ++++++++++++++++++ .../state-machines/query-running/services.ts | 26 ++++ .../src/state-machines/query-running/types.ts | 22 ++++ .../shared-transition-configs.ts | 28 +++++ 14 files changed, 300 insertions(+), 35 deletions(-) create mode 100644 packages/gatsby/src/state-machines/query-running/actions.ts create mode 100644 packages/gatsby/src/state-machines/query-running/index.ts create mode 100644 packages/gatsby/src/state-machines/query-running/services.ts create mode 100644 packages/gatsby/src/state-machines/query-running/types.ts create mode 100644 packages/gatsby/src/state-machines/shared-transition-configs.ts diff --git a/packages/gatsby/src/commands/develop-process.ts b/packages/gatsby/src/commands/develop-process.ts index b88bb878d7a81..64caad0da33ab 100644 --- a/packages/gatsby/src/commands/develop-process.ts +++ b/packages/gatsby/src/commands/develop-process.ts @@ -22,15 +22,11 @@ import { markWebpackStatusAsPending } from "../utils/webpack-status" import { IProgram } from "./types" import { - calculateDirtyQueries, - runStaticQueries, - runPageQueries, startWebpackServer, writeOutRequires, IBuildContext, initialize, postBootstrap, - extractQueries, rebuildSchemaWithSitePage, writeOutRedirects, } from "../services" @@ -49,7 +45,8 @@ import { import { DataLayerResult, dataLayerMachine } from "../state-machines/data-layer" import { IDataLayerContext } from "../state-machines/data-layer/types" import { globalTracer } from "opentracing" -import reporter from "gatsby-cli/lib/reporter" +import { IQueryRunningContext } from "../state-machines/query-running/types" +import { queryRunningMachine } from "../state-machines/query-running" const tracer = globalTracer() @@ -153,27 +150,20 @@ module.exports = async (program: IProgram): Promise => { }, onDone: { actions: `assignDataLayer`, - target: `doingEverythingElse`, + target: `finishingBootstrap`, }, }, }, - doingEverythingElse: { + finishingBootstrap: { invoke: { src: async ({ gatsbyNodeGraphQLFunction, - graphqlRunner, - workerPool, - store, - app, - }): Promise => { - // All the stuff that's not in the state machine yet - + }: IBuildContext): Promise => { // These were previously in `bootstrap()` but are now // in part of the state machine that hasn't been added yet await postBootstrap({ parentSpan: bootstrapSpan }) await rebuildSchemaWithSitePage({ parentSpan: bootstrapSpan }) - await extractQueries({ parentSpan: bootstrapSpan }) await writeOutRedirects({ parentSpan: bootstrapSpan }) startRedirectListener() @@ -186,11 +176,41 @@ module.exports = async (program: IProgram): Promise => { // Start the schema hot reloader. bootstrapSchemaHotReloader() + }, + onDone: { + target: `runningQueries`, + }, + }, + }, + runningQueries: { + invoke: { + src: `runQueries`, + data: ({ + program, + store, + parentSpan, + gatsbyNodeGraphQLFunction, + graphqlRunner, + }: IBuildContext): IQueryRunningContext => { + return { + firstRun: true, + program, + store, + parentSpan, + gatsbyNodeGraphQLFunction, + graphqlRunner, + } + }, + onDone: { + target: `doingEverythingElse`, + }, + }, + }, + doingEverythingElse: { + invoke: { + src: async ({ workerPool, store, app }): Promise => { + // All the stuff that's not in the state machine yet - const { queryIds } = await calculateDirtyQueries({ store }) - - await runStaticQueries({ queryIds, store, program, graphqlRunner }) - await runPageQueries({ queryIds, store, program, graphqlRunner }) await writeOutRequires({ store }) boundActionCreators.setProgramStatus( ProgramStatus.BOOTSTRAP_QUERY_RUNNING_FINISHED @@ -219,6 +239,7 @@ module.exports = async (program: IProgram): Promise => { services: { initializeDataLayer: dataLayerMachine, initialize, + runQueries: queryRunningMachine, }, actions: { assignStoreAndWorkerPool: assign( diff --git a/packages/gatsby/src/query/query-watcher.js b/packages/gatsby/src/query/query-watcher.js index cc16b7c1d4f2f..d20676a3feea3 100644 --- a/packages/gatsby/src/query/query-watcher.js +++ b/packages/gatsby/src/query/query-watcher.js @@ -195,6 +195,9 @@ exports.extractQueries = ({ parentSpan } = {}) => { return updateStateAndRunQueries(true, { parentSpan }).then(() => { // During development start watching files to recompile & run // queries on the fly. + + // TODO: move this into a spawned service, and emit events rather than + // directly triggering the compilation if (process.env.NODE_ENV !== `production`) { watch(store.getState().program.directory) } @@ -252,7 +255,7 @@ exports.startWatchDeletePage = () => { const componentPath = slash(action.payload.component) const { pages } = store.getState() let otherPageWithTemplateExists = false - for (let page of pages.values()) { + for (const page of pages.values()) { if (slash(page.component) === componentPath) { otherPageWithTemplateExists = true break diff --git a/packages/gatsby/src/services/calculate-dirty-queries.ts b/packages/gatsby/src/services/calculate-dirty-queries.ts index 80b2af00cd5b5..e3b637a19e1c2 100644 --- a/packages/gatsby/src/services/calculate-dirty-queries.ts +++ b/packages/gatsby/src/services/calculate-dirty-queries.ts @@ -1,10 +1,11 @@ import { calcInitialDirtyQueryIds, groupQueryIds } from "../query" -import { IBuildContext, IGroupedQueryIds } from "./" +import { IGroupedQueryIds } from "./" import reporter from "gatsby-cli/lib/reporter" +import { IQueryRunningContext } from "../state-machines/query-running/types" export async function calculateDirtyQueries({ store, -}: Partial): Promise<{ +}: Partial): Promise<{ queryIds: IGroupedQueryIds }> { if (!store) { diff --git a/packages/gatsby/src/services/extract-queries.ts b/packages/gatsby/src/services/extract-queries.ts index 30441f904a084..425cd506e402c 100644 --- a/packages/gatsby/src/services/extract-queries.ts +++ b/packages/gatsby/src/services/extract-queries.ts @@ -1,12 +1,11 @@ -import { IBuildContext } from "./" - import reporter from "gatsby-cli/lib/reporter" import { extractQueries as extractQueriesAndWatch } from "../query/query-watcher" import apiRunnerNode from "../utils/api-runner-node" +import { IQueryRunningContext } from "../state-machines/query-running/types" export async function extractQueries({ parentSpan, -}: Partial): Promise { +}: Partial): Promise { const activity = reporter.activityTimer(`onPreExtractQueries`, { parentSpan, }) diff --git a/packages/gatsby/src/services/rebuild-schema-with-site-pages.ts b/packages/gatsby/src/services/rebuild-schema-with-site-pages.ts index a58c5fdb76410..c5b41c4859886 100644 --- a/packages/gatsby/src/services/rebuild-schema-with-site-pages.ts +++ b/packages/gatsby/src/services/rebuild-schema-with-site-pages.ts @@ -1,10 +1,10 @@ -import { IBuildContext } from "./" import { rebuildWithSitePage } from "../schema" import reporter from "gatsby-cli/lib/reporter" +import { IQueryRunningContext } from "../state-machines/query-running/types" export async function rebuildSchemaWithSitePage({ parentSpan, -}: Partial): Promise { +}: Partial): Promise { const activity = reporter.activityTimer(`updating schema`, { parentSpan, }) diff --git a/packages/gatsby/src/services/run-page-queries.ts b/packages/gatsby/src/services/run-page-queries.ts index 3e810e4314459..2770e7bf78e5d 100644 --- a/packages/gatsby/src/services/run-page-queries.ts +++ b/packages/gatsby/src/services/run-page-queries.ts @@ -1,6 +1,6 @@ import { processPageQueries } from "../query" -import { IBuildContext } from "./" import reporter from "gatsby-cli/lib/reporter" +import { IQueryRunningContext } from "../state-machines/query-running/types" export async function runPageQueries({ parentSpan, @@ -8,7 +8,7 @@ export async function runPageQueries({ store, program, graphqlRunner, -}: Partial): Promise { +}: Partial): Promise { if (!store) { reporter.panic(`Cannot run service without a redux store`) } diff --git a/packages/gatsby/src/services/run-static-queries.ts b/packages/gatsby/src/services/run-static-queries.ts index 822563bc35211..94d27b22de07f 100644 --- a/packages/gatsby/src/services/run-static-queries.ts +++ b/packages/gatsby/src/services/run-static-queries.ts @@ -1,6 +1,6 @@ import { processStaticQueries } from "../query" -import { IBuildContext } from "./" import reporter from "gatsby-cli/lib/reporter" +import { IQueryRunningContext } from "../state-machines/query-running/types" export async function runStaticQueries({ parentSpan, @@ -8,7 +8,7 @@ export async function runStaticQueries({ store, program, graphqlRunner, -}: Partial): Promise { +}: Partial): Promise { if (!store) { reporter.panic(`Cannot run service without a redux store`) } diff --git a/packages/gatsby/src/services/write-out-redirects.ts b/packages/gatsby/src/services/write-out-redirects.ts index d7c30f690e390..5745db6dc6fdf 100644 --- a/packages/gatsby/src/services/write-out-redirects.ts +++ b/packages/gatsby/src/services/write-out-redirects.ts @@ -1,10 +1,10 @@ import reporter from "gatsby-cli/lib/reporter" import { writeRedirects } from "../bootstrap/redirects-writer" -import { IDataLayerContext } from "../state-machines/data-layer/types" +import { IQueryRunningContext } from "../state-machines/query-running/types" export async function writeOutRedirects({ parentSpan, -}: Partial): Promise { +}: Partial): Promise { // Write out redirects. const activity = reporter.activityTimer(`write out redirect data`, { parentSpan, diff --git a/packages/gatsby/src/services/write-out-requires.ts b/packages/gatsby/src/services/write-out-requires.ts index 88f6c9d772102..0cb141b2b7d73 100644 --- a/packages/gatsby/src/services/write-out-requires.ts +++ b/packages/gatsby/src/services/write-out-requires.ts @@ -1,11 +1,11 @@ -import { IBuildContext } from "./" import reporter from "gatsby-cli/lib/reporter" import { writeAll } from "../bootstrap/requires-writer" +import { IQueryRunningContext } from "../state-machines/query-running/types" export async function writeOutRequires({ store, parentSpan, -}: Partial): Promise { +}: Partial): Promise { if (!store) { reporter.panic(`No redux store`) } diff --git a/packages/gatsby/src/state-machines/query-running/actions.ts b/packages/gatsby/src/state-machines/query-running/actions.ts new file mode 100644 index 0000000000000..5ba796244aad7 --- /dev/null +++ b/packages/gatsby/src/state-machines/query-running/actions.ts @@ -0,0 +1,50 @@ +import { IQueryRunningContext } from "./types" +import { DoneInvokeEvent, assign, ActionFunctionMap } from "xstate" + +export const emitStaticQueryDataToWebsocket = ( + { websocketManager }: IQueryRunningContext, + { data: { results } }: DoneInvokeEvent +): void => { + if (websocketManager && results) { + results.forEach((result, id) => { + websocketManager.emitStaticQueryData({ + result, + id, + }) + }) + } +} + +export const emitPageDataToWebsocket = ( + { websocketManager }: IQueryRunningContext, + { data: { results } }: DoneInvokeEvent +): void => { + if (websocketManager && results) { + results.forEach((result, id) => { + websocketManager.emitPageData({ + result, + id, + }) + }) + } +} + +export const assignDirtyQueries = assign< + IQueryRunningContext, + DoneInvokeEvent +>((_context, { data }) => { + const { queryIds } = data + return { + filesDirty: false, + queryIds, + } +}) + +export const queryActions: ActionFunctionMap< + IQueryRunningContext, + DoneInvokeEvent +> = { + assignDirtyQueries, + emitPageDataToWebsocket, + emitStaticQueryDataToWebsocket, +} diff --git a/packages/gatsby/src/state-machines/query-running/index.ts b/packages/gatsby/src/state-machines/query-running/index.ts new file mode 100644 index 0000000000000..11c1c76068f2d --- /dev/null +++ b/packages/gatsby/src/state-machines/query-running/index.ts @@ -0,0 +1,115 @@ +import { MachineConfig, Machine } from "xstate" +import { IQueryRunningContext } from "./types" +import { queryRunningServices } from "./services" +import { queryActions } from "./actions" + +const extractQueriesIfDirty = { + cond: (ctx: IQueryRunningContext): boolean => !!ctx.filesDirty, + target: `extractingQueries`, +} + +export const queryStates: MachineConfig = { + initial: `extractingQueries`, + states: { + extractingQueries: { + id: `extracting-queries`, + invoke: { + id: `extracting-queries`, + src: `extractQueries`, + onDone: [ + { + actions: `resetGraphQlRunner`, + target: `writingRequires`, + }, + ], + }, + }, + writingRequires: { + invoke: { + src: `writeOutRequires`, + id: `writing-requires`, + onDone: { + target: `calculatingDirtyQueries`, + }, + }, + }, + calculatingDirtyQueries: { + // Disabled because we're not watching for node mutations + // at the moment + // on: { + // "": extractQueriesIfDirty, + // SOURCE_FILE_CHANGED, + // }, + invoke: { + id: `calculating-dirty-queries`, + src: `calculateDirtyQueries`, + onDone: { + target: `runningStaticQueries`, + actions: `assignDirtyQueries`, + }, + }, + }, + runningStaticQueries: { + on: { + "": extractQueriesIfDirty, + }, + invoke: { + src: `runStaticQueries`, + id: `running-static-queries`, + onDone: [ + { + target: `runningPageQueries`, + actions: `emitStaticQueryDataToWebsocket`, + // Only emit if there's a websocket manager + // This won't be the case on first run, and the query data + // will be emitted when the client first connects + cond: (context): boolean => !!context.websocketManager, + }, + { + target: `runningPageQueries`, + }, + ], + }, + }, + runningPageQueries: { + // on: { + // "": extractQueriesIfDirty, + // }, + invoke: { + src: `runPageQueries`, + id: `running-page-queries`, + onDone: [ + { + target: `waitingForJobs`, + actions: `emitPageDataToWebsocket`, + // Only emit if there's a websocket manager + // This won't be the case on first run, and the page data + // will be emitted when the client first connects + cond: (context): boolean => !!context.websocketManager, + }, + { + target: `waitingForJobs`, + }, + ], + }, + }, + + waitingForJobs: { + invoke: { + src: `waitUntilAllJobsComplete`, + id: `waiting-for-jobs`, + onDone: { + target: `done`, + }, + }, + }, + done: { + type: `final`, + }, + }, +} +// eslint-disable-next-line new-cap +export const queryRunningMachine = Machine(queryStates, { + actions: queryActions, + services: queryRunningServices, +}) diff --git a/packages/gatsby/src/state-machines/query-running/services.ts b/packages/gatsby/src/state-machines/query-running/services.ts new file mode 100644 index 0000000000000..e5b7aaa1b2794 --- /dev/null +++ b/packages/gatsby/src/state-machines/query-running/services.ts @@ -0,0 +1,26 @@ +import { ServiceConfig } from "xstate" +import { + extractQueries, + writeOutRequires, + calculateDirtyQueries, + runStaticQueries, + runPageQueries, + waitUntilAllJobsComplete, + writeOutRedirects, + rebuildSchemaWithSitePage, +} from "../../services" +import { IQueryRunningContext } from "./types" + +export const queryRunningServices: Record< + string, + ServiceConfig +> = { + extractQueries, + writeOutRequires, + calculateDirtyQueries, + runStaticQueries, + runPageQueries, + waitUntilAllJobsComplete, + writeOutRedirects, + rebuildSchemaWithSitePage, +} diff --git a/packages/gatsby/src/state-machines/query-running/types.ts b/packages/gatsby/src/state-machines/query-running/types.ts new file mode 100644 index 0000000000000..d977fd967111b --- /dev/null +++ b/packages/gatsby/src/state-machines/query-running/types.ts @@ -0,0 +1,22 @@ +import { Span } from "opentracing" +import { IProgram } from "../../commands/types" +import { Runner } from "../../bootstrap/create-graphql-runner" +import { GraphQLRunner } from "../../query/graphql-runner" +import { Store, AnyAction } from "redux" +import { IGatsbyState } from "../../redux/types" +import { IGroupedQueryIds } from "../data-layer/types" +import { WebsocketManager } from "../../utils/websocket-manager" + +export interface IQueryRunningContext { + firstRun?: boolean + program?: IProgram + store?: Store + parentSpan?: Span + gatsbyNodeGraphQLFunction?: Runner + graphqlRunner?: GraphQLRunner + pagesToBuild?: string[] + pagesToDelete?: string[] + queryIds?: IGroupedQueryIds + websocketManager?: WebsocketManager + filesDirty?: boolean +} diff --git a/packages/gatsby/src/state-machines/shared-transition-configs.ts b/packages/gatsby/src/state-machines/shared-transition-configs.ts new file mode 100644 index 0000000000000..e0a12a715db0f --- /dev/null +++ b/packages/gatsby/src/state-machines/shared-transition-configs.ts @@ -0,0 +1,28 @@ +import { TransitionConfig, AnyEventObject } from "xstate" +import { IBuildContext } from "../services" + +type BuildTransition = TransitionConfig + +/** + * Event handler used in all states where we're not ready to process node + * mutations. Instead we add it to a batch to process when we're next idle + */ +export const ADD_NODE_MUTATION: BuildTransition = { + actions: `addNodeMutation`, +} + +/** + * Event handler used in all states where we're not ready to process a file change + * Instead we add it to a batch to process when we're next idle + */ +export const SOURCE_FILE_CHANGED: BuildTransition = { + actions: `markFilesDirty`, +} + +/** + * When running queries we might add nodes (e.g from resolvers). If so we'll + * want to re-run queries and schema inference + */ +export const runMutationAndMarkDirty: BuildTransition = { + actions: [`markNodesDirty`, `callApi`], +} From d049022d88e9727c6b35ec9dc0a2e3357b254da6 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Wed, 1 Jul 2020 12:29:19 +0100 Subject: [PATCH 10/44] Changes from review --- packages/gatsby/src/commands/develop-process.ts | 4 ++-- packages/gatsby/src/state-machines/data-layer/actions.ts | 8 ++++++-- packages/gatsby/src/state-machines/data-layer/index.ts | 3 +-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/packages/gatsby/src/commands/develop-process.ts b/packages/gatsby/src/commands/develop-process.ts index c5a58fa5fcf83..25432e9b2b120 100644 --- a/packages/gatsby/src/commands/develop-process.ts +++ b/packages/gatsby/src/commands/develop-process.ts @@ -84,9 +84,9 @@ process.on(`message`, msg => { } }) -const bootstrapSpan = tracer.startSpan(`bootstrap`) - module.exports = async (program: IProgram): Promise => { + const bootstrapSpan = tracer.startSpan(`bootstrap`) + // We want to prompt the feedback request when users quit develop // assuming they pass the heuristic check to know they are a user // we want to request feedback from, and we're not annoying them. diff --git a/packages/gatsby/src/state-machines/data-layer/actions.ts b/packages/gatsby/src/state-machines/data-layer/actions.ts index ef8d48ff21210..bb48dfd1479c3 100644 --- a/packages/gatsby/src/state-machines/data-layer/actions.ts +++ b/packages/gatsby/src/state-machines/data-layer/actions.ts @@ -33,8 +33,12 @@ export const assignChangedPages: BuildMachineAction = assign< export const assignGatsbyNodeGraphQL: BuildMachineAction = assign< IDataLayerContext >({ - gatsbyNodeGraphQLFunction: ({ store }: IDataLayerContext) => - store ? createGraphQLRunner(store, reporter) : undefined, + gatsbyNodeGraphQLFunction: ({ store }: IDataLayerContext) => { + if (!store) { + reporter.panic(`Missing redux store`) + } + return createGraphQLRunner(store, reporter) + }, }) export const dataLayerActions: ActionFunctionMap< diff --git a/packages/gatsby/src/state-machines/data-layer/index.ts b/packages/gatsby/src/state-machines/data-layer/index.ts index ab745a5c3620b..b814d2dece6b3 100644 --- a/packages/gatsby/src/state-machines/data-layer/index.ts +++ b/packages/gatsby/src/state-machines/data-layer/index.ts @@ -11,7 +11,7 @@ export type DataLayerResult = Pick< | "pagesToDelete" > -export const dataLayerStates: MachineConfig = { +const dataLayerStates: MachineConfig = { initial: `customizingSchema`, states: { customizingSchema: { @@ -88,7 +88,6 @@ export const dataLayerStates: MachineConfig = { }, } -// eslint-disable-next-line new-cap export const dataLayerMachine = Machine(dataLayerStates, { actions: dataLayerActions, services: dataLayerServices, From 2dad1292462af01f5a0e832367f866ca69863011 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Wed, 1 Jul 2020 12:37:14 +0100 Subject: [PATCH 11/44] Changes from review --- packages/gatsby/src/commands/develop-process.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gatsby/src/commands/develop-process.ts b/packages/gatsby/src/commands/develop-process.ts index 25432e9b2b120..7305660a51994 100644 --- a/packages/gatsby/src/commands/develop-process.ts +++ b/packages/gatsby/src/commands/develop-process.ts @@ -168,7 +168,6 @@ module.exports = async (program: IProgram): Promise => { // These were previously in `bootstrap()` but are now // in part of the state machine that hasn't been added yet - await postBootstrap({ parentSpan: bootstrapSpan }) await rebuildSchemaWithSitePage({ parentSpan: bootstrapSpan }) await extractQueries({ parentSpan: bootstrapSpan }) @@ -176,6 +175,7 @@ module.exports = async (program: IProgram): Promise => { startRedirectListener() bootstrapSpan.finish() + await postBootstrap({ parentSpan: bootstrapSpan }) // These are the parts that weren't in bootstrap From 26420c4c49782307be3cf7194c7dafe6ce9d9e96 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Wed, 1 Jul 2020 12:44:08 +0100 Subject: [PATCH 12/44] Switch to reporter --- packages/gatsby/src/commands/develop-process.ts | 9 ++++++--- .../gatsby/src/state-machines/query-running/index.ts | 1 - 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/gatsby/src/commands/develop-process.ts b/packages/gatsby/src/commands/develop-process.ts index c73e13b715cb0..e50231be574ff 100644 --- a/packages/gatsby/src/commands/develop-process.ts +++ b/packages/gatsby/src/commands/develop-process.ts @@ -234,7 +234,6 @@ module.exports = async (program: IProgram): Promise => { } const service = interpret( - // eslint-disable-next-line new-cap Machine(developConfig, { services: { initializeDataLayer: dataLayerMachine, @@ -264,7 +263,7 @@ module.exports = async (program: IProgram): Promise => { const listeners = new WeakSet() service.onTransition(state => { - console.log(`Transition to ${JSON.stringify(state.value)}`) + report.verbose(`Transition to ${JSON.stringify(state.value)}`) // eslint-disable-next-line no-unused-expressions service.children?.forEach(child => { // We want to ensure we don't attach a listener to the same @@ -273,7 +272,11 @@ module.exports = async (program: IProgram): Promise => { if (isInterpreter(child) && !listeners.has(child)) { child.onTransition(substate => { - console.log(`Transition to`, state.value, `>`, substate.value) + report.verbose( + `Transition to ${JSON.stringify(state.value)} > ${JSON.stringify( + substate.value + )}` + ) }) listeners.add(child) } diff --git a/packages/gatsby/src/state-machines/query-running/index.ts b/packages/gatsby/src/state-machines/query-running/index.ts index 11c1c76068f2d..53265ffee4eca 100644 --- a/packages/gatsby/src/state-machines/query-running/index.ts +++ b/packages/gatsby/src/state-machines/query-running/index.ts @@ -108,7 +108,6 @@ export const queryStates: MachineConfig = { }, }, } -// eslint-disable-next-line new-cap export const queryRunningMachine = Machine(queryStates, { actions: queryActions, services: queryRunningServices, From 7bc5056f78ef7277b6b683f05c4609bf8eda0bda Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Wed, 1 Jul 2020 15:08:34 +0100 Subject: [PATCH 13/44] Use assertStore --- packages/gatsby/src/state-machines/data-layer/actions.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/gatsby/src/state-machines/data-layer/actions.ts b/packages/gatsby/src/state-machines/data-layer/actions.ts index bb48dfd1479c3..72f89e5ca8ee5 100644 --- a/packages/gatsby/src/state-machines/data-layer/actions.ts +++ b/packages/gatsby/src/state-machines/data-layer/actions.ts @@ -9,6 +9,7 @@ import { import { createGraphQLRunner } from "../../bootstrap/create-graphql-runner" import reporter from "gatsby-cli/lib/reporter" import { IDataLayerContext } from "./types" +import { assertStore } from "../../utils/assert-store" const concatUnique = (array1: T[] = [], array2: T[] = []): T[] => Array.from(new Set(array1.concat(array2))) @@ -34,9 +35,7 @@ export const assignGatsbyNodeGraphQL: BuildMachineAction = assign< IDataLayerContext >({ gatsbyNodeGraphQLFunction: ({ store }: IDataLayerContext) => { - if (!store) { - reporter.panic(`Missing redux store`) - } + assertStore(store) return createGraphQLRunner(store, reporter) }, }) From 2ce7b774d174d6107143c03273c4ebb99a18e325 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Wed, 1 Jul 2020 15:59:45 +0100 Subject: [PATCH 14/44] Remove unused action --- .../gatsby/src/state-machines/query-running/index.ts | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/packages/gatsby/src/state-machines/query-running/index.ts b/packages/gatsby/src/state-machines/query-running/index.ts index 53265ffee4eca..3881910778a8f 100644 --- a/packages/gatsby/src/state-machines/query-running/index.ts +++ b/packages/gatsby/src/state-machines/query-running/index.ts @@ -3,11 +3,6 @@ import { IQueryRunningContext } from "./types" import { queryRunningServices } from "./services" import { queryActions } from "./actions" -const extractQueriesIfDirty = { - cond: (ctx: IQueryRunningContext): boolean => !!ctx.filesDirty, - target: `extractingQueries`, -} - export const queryStates: MachineConfig = { initial: `extractingQueries`, states: { @@ -50,9 +45,9 @@ export const queryStates: MachineConfig = { }, }, runningStaticQueries: { - on: { - "": extractQueriesIfDirty, - }, + // on: { + // "": extractQueriesIfDirty, + // }, invoke: { src: `runStaticQueries`, id: `running-static-queries`, From 133099dd15cd3aa24063b6ef35ea84992145db05 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Wed, 1 Jul 2020 16:02:01 +0100 Subject: [PATCH 15/44] Remove unusued config --- .../shared-transition-configs.ts | 28 ------------------- 1 file changed, 28 deletions(-) delete mode 100644 packages/gatsby/src/state-machines/shared-transition-configs.ts diff --git a/packages/gatsby/src/state-machines/shared-transition-configs.ts b/packages/gatsby/src/state-machines/shared-transition-configs.ts deleted file mode 100644 index e0a12a715db0f..0000000000000 --- a/packages/gatsby/src/state-machines/shared-transition-configs.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { TransitionConfig, AnyEventObject } from "xstate" -import { IBuildContext } from "../services" - -type BuildTransition = TransitionConfig - -/** - * Event handler used in all states where we're not ready to process node - * mutations. Instead we add it to a batch to process when we're next idle - */ -export const ADD_NODE_MUTATION: BuildTransition = { - actions: `addNodeMutation`, -} - -/** - * Event handler used in all states where we're not ready to process a file change - * Instead we add it to a batch to process when we're next idle - */ -export const SOURCE_FILE_CHANGED: BuildTransition = { - actions: `markFilesDirty`, -} - -/** - * When running queries we might add nodes (e.g from resolvers). If so we'll - * want to re-run queries and schema inference - */ -export const runMutationAndMarkDirty: BuildTransition = { - actions: [`markNodesDirty`, `callApi`], -} From e7f426e9c5dc28c15834ace8588bd0625f7f13e9 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Wed, 1 Jul 2020 16:02:01 +0100 Subject: [PATCH 16/44] Remove unusued config --- .../src/state-machines/query-running/index.ts | 12 -------- .../shared-transition-configs.ts | 28 ------------------- 2 files changed, 40 deletions(-) delete mode 100644 packages/gatsby/src/state-machines/shared-transition-configs.ts diff --git a/packages/gatsby/src/state-machines/query-running/index.ts b/packages/gatsby/src/state-machines/query-running/index.ts index 3881910778a8f..b1136ce5d0aa3 100644 --- a/packages/gatsby/src/state-machines/query-running/index.ts +++ b/packages/gatsby/src/state-machines/query-running/index.ts @@ -29,12 +29,6 @@ export const queryStates: MachineConfig = { }, }, calculatingDirtyQueries: { - // Disabled because we're not watching for node mutations - // at the moment - // on: { - // "": extractQueriesIfDirty, - // SOURCE_FILE_CHANGED, - // }, invoke: { id: `calculating-dirty-queries`, src: `calculateDirtyQueries`, @@ -45,9 +39,6 @@ export const queryStates: MachineConfig = { }, }, runningStaticQueries: { - // on: { - // "": extractQueriesIfDirty, - // }, invoke: { src: `runStaticQueries`, id: `running-static-queries`, @@ -67,9 +58,6 @@ export const queryStates: MachineConfig = { }, }, runningPageQueries: { - // on: { - // "": extractQueriesIfDirty, - // }, invoke: { src: `runPageQueries`, id: `running-page-queries`, diff --git a/packages/gatsby/src/state-machines/shared-transition-configs.ts b/packages/gatsby/src/state-machines/shared-transition-configs.ts deleted file mode 100644 index e0a12a715db0f..0000000000000 --- a/packages/gatsby/src/state-machines/shared-transition-configs.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { TransitionConfig, AnyEventObject } from "xstate" -import { IBuildContext } from "../services" - -type BuildTransition = TransitionConfig - -/** - * Event handler used in all states where we're not ready to process node - * mutations. Instead we add it to a batch to process when we're next idle - */ -export const ADD_NODE_MUTATION: BuildTransition = { - actions: `addNodeMutation`, -} - -/** - * Event handler used in all states where we're not ready to process a file change - * Instead we add it to a batch to process when we're next idle - */ -export const SOURCE_FILE_CHANGED: BuildTransition = { - actions: `markFilesDirty`, -} - -/** - * When running queries we might add nodes (e.g from resolvers). If so we'll - * want to re-run queries and schema inference - */ -export const runMutationAndMarkDirty: BuildTransition = { - actions: [`markNodesDirty`, `callApi`], -} From 7da48a64761e52337b1b5e721f9a4ba9e85eba91 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Wed, 1 Jul 2020 16:13:22 +0100 Subject: [PATCH 17/44] Add gql runner reset --- .../src/state-machines/query-running/actions.ts | 16 ++++++++++++++++ .../src/state-machines/query-running/index.ts | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/packages/gatsby/src/state-machines/query-running/actions.ts b/packages/gatsby/src/state-machines/query-running/actions.ts index 5ba796244aad7..57155ee35c9e4 100644 --- a/packages/gatsby/src/state-machines/query-running/actions.ts +++ b/packages/gatsby/src/state-machines/query-running/actions.ts @@ -1,5 +1,7 @@ import { IQueryRunningContext } from "./types" import { DoneInvokeEvent, assign, ActionFunctionMap } from "xstate" +import { GraphQLRunner } from "../../query/graphql-runner" +import { assertStore } from "../../utils/assert-store" export const emitStaticQueryDataToWebsocket = ( { websocketManager }: IQueryRunningContext, @@ -40,10 +42,24 @@ export const assignDirtyQueries = assign< } }) +export const resetGraphQLRunner = assign< + IQueryRunningContext, + DoneInvokeEvent +>({ + graphqlRunner: ({ store, program }) => { + assertStore(store) + return new GraphQLRunner(store, { + collectStats: true, + graphqlTracing: program?.graphqlTracing, + }) + }, +}) + export const queryActions: ActionFunctionMap< IQueryRunningContext, DoneInvokeEvent > = { + resetGraphQLRunner, assignDirtyQueries, emitPageDataToWebsocket, emitStaticQueryDataToWebsocket, diff --git a/packages/gatsby/src/state-machines/query-running/index.ts b/packages/gatsby/src/state-machines/query-running/index.ts index b1136ce5d0aa3..345f329754a5e 100644 --- a/packages/gatsby/src/state-machines/query-running/index.ts +++ b/packages/gatsby/src/state-machines/query-running/index.ts @@ -13,7 +13,7 @@ export const queryStates: MachineConfig = { src: `extractQueries`, onDone: [ { - actions: `resetGraphQlRunner`, + actions: `resetGraphQLRunner`, target: `writingRequires`, }, ], From a4ac2dc45ecd887b4ee62892463f15ce53ec35ad Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Thu, 2 Jul 2020 16:32:34 +0100 Subject: [PATCH 18/44] Handle node mutation queuing and batching in state machine --- .../gatsby/src/commands/develop-process.ts | 162 ++++++++++++------ packages/gatsby/src/commands/types.ts | 1 + .../src/services/create-pages-statefully.ts | 2 +- packages/gatsby/src/services/create-pages.ts | 47 ++--- packages/gatsby/src/services/index.ts | 2 + .../src/services/listen-for-mutations.ts | 22 +++ .../gatsby/src/services/run-mutation-batch.ts | 26 +++ packages/gatsby/src/services/source-nodes.ts | 44 ++--- .../src/services/start-webpack-server.ts | 16 +- packages/gatsby/src/services/types.ts | 10 ++ packages/gatsby/src/state-machines/actions.ts | 95 ++++++++++ .../src/state-machines/data-layer/actions.ts | 27 +-- .../src/state-machines/data-layer/index.ts | 17 +- .../src/state-machines/data-layer/types.ts | 2 + .../state-machines/query-running/actions.ts | 33 +--- .../src/state-machines/query-running/index.ts | 33 +--- .../shared-transition-configs.ts | 26 +++ .../src/state-machines/waiting/actions.ts | 21 +++ .../src/state-machines/waiting/index.ts | 94 ++++++++++ .../src/state-machines/waiting/services.ts | 3 + .../src/state-machines/waiting/types.ts | 14 ++ packages/gatsby/src/utils/api-runner-node.js | 54 +++++- packages/gatsby/src/utils/changed-pages.ts | 54 ++++++ packages/gatsby/src/utils/source-nodes.ts | 3 + packages/gatsby/src/utils/start-server.ts | 4 +- 25 files changed, 623 insertions(+), 189 deletions(-) create mode 100644 packages/gatsby/src/services/listen-for-mutations.ts create mode 100644 packages/gatsby/src/services/run-mutation-batch.ts create mode 100644 packages/gatsby/src/state-machines/actions.ts create mode 100644 packages/gatsby/src/state-machines/shared-transition-configs.ts create mode 100644 packages/gatsby/src/state-machines/waiting/actions.ts create mode 100644 packages/gatsby/src/state-machines/waiting/index.ts create mode 100644 packages/gatsby/src/state-machines/waiting/services.ts create mode 100644 packages/gatsby/src/state-machines/waiting/types.ts create mode 100644 packages/gatsby/src/utils/changed-pages.ts diff --git a/packages/gatsby/src/commands/develop-process.ts b/packages/gatsby/src/commands/develop-process.ts index e50231be574ff..0105ee9554c23 100644 --- a/packages/gatsby/src/commands/develop-process.ts +++ b/packages/gatsby/src/commands/develop-process.ts @@ -3,16 +3,10 @@ import report from "gatsby-cli/lib/reporter" import chalk from "chalk" import telemetry from "gatsby-telemetry" import express from "express" -import { bootstrapSchemaHotReloader } from "../bootstrap/schema-hot-reloader" -import bootstrapPageHotReloader from "../bootstrap/page-hot-reloader" import { initTracer } from "../utils/tracer" import db from "../db" import { detectPortInUseAndPrompt } from "../utils/detect-port-in-use-and-prompt" import onExit from "signal-exit" -import queryUtil from "../query" -import queryWatcher from "../query/query-watcher" -import * as requiresWriter from "../bootstrap/requires-writer" -import { waitUntilAllJobsComplete } from "../utils/wait-until-jobs-complete" import { userPassesFeedbackRequestHeuristic, showFeedbackRequest, @@ -22,31 +16,36 @@ import { markWebpackStatusAsPending } from "../utils/webpack-status" import { IProgram } from "./types" import { - startWebpackServer, - writeOutRequires, IBuildContext, initialize, - postBootstrap, rebuildSchemaWithSitePage, writeOutRedirects, + startWebpackServer, } from "../services" import { boundActionCreators } from "../redux/actions" import { ProgramStatus } from "../redux/types" import { MachineConfig, AnyEventObject, - assign, Machine, - DoneEventObject, interpret, Actor, Interpreter, + forwardTo, } from "xstate" -import { DataLayerResult, dataLayerMachine } from "../state-machines/data-layer" +import { dataLayerMachine } from "../state-machines/data-layer" import { IDataLayerContext } from "../state-machines/data-layer/types" import { globalTracer } from "opentracing" import { IQueryRunningContext } from "../state-machines/query-running/types" import { queryRunningMachine } from "../state-machines/query-running" +import { IWaitingContext } from "../state-machines/waiting/types" +import { + ADD_NODE_MUTATION, + runMutationAndMarkDirty, +} from "../state-machines/shared-transition-configs" +import { buildActions } from "../state-machines/actions" +import { waitingMachine } from "../state-machines/waiting" +import reporter from "gatsby-cli/lib/reporter" const tracer = globalTracer() @@ -84,6 +83,7 @@ process.on(`message`, msg => { }) module.exports = async (program: IProgram): Promise => { + reporter.setVerbose(program.verbose) const bootstrapSpan = tracer.startSpan(`bootstrap`) // We want to prompt the feedback request when users quit develop @@ -138,27 +138,31 @@ module.exports = async (program: IProgram): Promise => { src: `initialize`, onDone: { target: `initializingDataLayer`, - actions: `assignStoreAndWorkerPool`, + actions: [`assignStoreAndWorkerPool`, `spawnMutationListener`], }, }, }, initializingDataLayer: { + on: { + ADD_NODE_MUTATION: runMutationAndMarkDirty, + }, invoke: { src: `initializeDataLayer`, data: ({ parentSpan, store }: IBuildContext): IDataLayerContext => { return { parentSpan, store, firstRun: true } }, onDone: { - actions: `assignDataLayer`, + actions: `assignServiceResult`, target: `finishingBootstrap`, }, }, }, finishingBootstrap: { + on: { + ADD_NODE_MUTATION: runMutationAndMarkDirty, + }, invoke: { - src: async ({ - gatsbyNodeGraphQLFunction, - }: IBuildContext): Promise => { + src: async (): Promise => { // These were previously in `bootstrap()` but are now // in part of the state machine that hasn't been added yet await rebuildSchemaWithSitePage({ parentSpan: bootstrapSpan }) @@ -167,15 +171,6 @@ module.exports = async (program: IProgram): Promise => { startRedirectListener() bootstrapSpan.finish() - await postBootstrap({ parentSpan: bootstrapSpan }) - - // These are the parts that weren't in bootstrap - - // Start the createPages hot reloader. - bootstrapPageHotReloader(gatsbyNodeGraphQLFunction) - - // Start the schema hot reloader. - bootstrapSchemaHotReloader() }, onDone: { target: `runningQueries`, @@ -183,6 +178,9 @@ module.exports = async (program: IProgram): Promise => { }, }, runningQueries: { + on: { + ADD_NODE_MUTATION, + }, invoke: { src: `runQueries`, data: ({ @@ -191,6 +189,7 @@ module.exports = async (program: IProgram): Promise => { parentSpan, gatsbyNodeGraphQLFunction, graphqlRunner, + websocketManager, }: IBuildContext): IQueryRunningContext => { return { firstRun: true, @@ -199,6 +198,7 @@ module.exports = async (program: IProgram): Promise => { parentSpan, gatsbyNodeGraphQLFunction, graphqlRunner, + websocketManager, } }, onDone: { @@ -207,26 +207,79 @@ module.exports = async (program: IProgram): Promise => { }, }, doingEverythingElse: { + on: { + ADD_NODE_MUTATION, + }, invoke: { - src: async ({ workerPool, store, app }): Promise => { + src: async (): Promise => { // All the stuff that's not in the state machine yet - await writeOutRequires({ store }) boundActionCreators.setProgramStatus( ProgramStatus.BOOTSTRAP_QUERY_RUNNING_FINISHED ) await db.saveState() - await waitUntilAllJobsComplete() - requiresWriter.startListener() db.startAutosave() - queryUtil.startListeningToDevelopQueue({ - graphqlTracing: program.graphqlTracing, - }) - queryWatcher.startWatchDeletePage() - - await startWebpackServer({ program, app, workerPool }) + }, + onDone: [ + { + target: `startingDevServers`, + cond: ({ compiler }: IBuildContext): boolean => !compiler, + }, + { + target: `waiting`, + }, + ], + }, + }, + startingDevServers: { + on: { + ADD_NODE_MUTATION, + }, + invoke: { + src: `startWebpackServer`, + onDone: { + target: `waiting`, + actions: `assignServers`, + }, + }, + }, + waiting: { + on: { + ADD_NODE_MUTATION: { + actions: forwardTo(`waiting`), + }, + }, + invoke: { + id: `waiting`, + src: `waitForMutations`, + data: ({ + store, + nodeMutationBatch = [], + }: IBuildContext): IWaitingContext => { + return { store, nodeMutationBatch } + }, + onDone: { + // These are mutations added while we were running the last + // batch. We'll hold on to them til we've finished this build. + actions: `assignServiceResult`, + target: `rebuildingPages`, + }, + }, + }, + rebuildingPages: { + on: { + ADD_NODE_MUTATION, + }, + invoke: { + src: `initializeDataLayer`, + data: ({ parentSpan, store }: IBuildContext): IDataLayerContext => { + return { parentSpan, store, firstRun: false, skipSourcing: true } + }, + onDone: { + actions: `assignServiceResult`, + target: `runningQueries`, }, }, }, @@ -239,21 +292,10 @@ module.exports = async (program: IProgram): Promise => { initializeDataLayer: dataLayerMachine, initialize, runQueries: queryRunningMachine, + waitForMutations: waitingMachine, + startWebpackServer: startWebpackServer, }, - actions: { - assignStoreAndWorkerPool: assign( - (_context, event) => { - const { store, workerPool } = event.data - return { - store, - workerPool, - } - } - ), - assignDataLayer: assign( - (_, { data }): DataLayerResult => data - ), - }, + actions: buildActions, }).withContext({ program, parentSpan: bootstrapSpan, app }) ) @@ -262,8 +304,17 @@ module.exports = async (program: IProgram): Promise => { ): actor is Interpreter => `machine` in actor const listeners = new WeakSet() + + let last = service.state + service.onTransition(state => { - report.verbose(`Transition to ${JSON.stringify(state.value)}`) + if (!last) { + last = state + } else if (!state.changed || last.matches(state)) { + return + } + last = state + reporter.verbose(`Transition to ${JSON.stringify(state.value)}`) // eslint-disable-next-line no-unused-expressions service.children?.forEach(child => { // We want to ensure we don't attach a listener to the same @@ -271,8 +322,15 @@ module.exports = async (program: IProgram): Promise => { // because xstate handles that for us when the actor is stopped. if (isInterpreter(child) && !listeners.has(child)) { + let sublast = child.state child.onTransition(substate => { - report.verbose( + if (!sublast) { + sublast = substate + } else if (!substate.changed || sublast.matches(substate)) { + return + } + sublast = substate + reporter.verbose( `Transition to ${JSON.stringify(state.value)} > ${JSON.stringify( substate.value )}` diff --git a/packages/gatsby/src/commands/types.ts b/packages/gatsby/src/commands/types.ts index 4155abffde1fb..f843e0b0ac275 100644 --- a/packages/gatsby/src/commands/types.ts +++ b/packages/gatsby/src/commands/types.ts @@ -24,6 +24,7 @@ export interface IProgram { sitePackageJson: PackageJson ssl?: ICert graphqlTracing?: boolean + verbose?: boolean setStore?: (store: Store) => void } diff --git a/packages/gatsby/src/services/create-pages-statefully.ts b/packages/gatsby/src/services/create-pages-statefully.ts index a1c4798077521..fe935c4bc19dc 100644 --- a/packages/gatsby/src/services/create-pages-statefully.ts +++ b/packages/gatsby/src/services/create-pages-statefully.ts @@ -21,7 +21,7 @@ export async function createPagesStatefully({ traceId: `initial-createPagesStatefully`, waitForCascadingActions: true, parentSpan: activity.span, - // deferNodeMutation: true, //later + deferNodeMutation: true, }, { activity, diff --git a/packages/gatsby/src/services/create-pages.ts b/packages/gatsby/src/services/create-pages.ts index 0f456bf26914d..7b5f9a10a42b0 100644 --- a/packages/gatsby/src/services/create-pages.ts +++ b/packages/gatsby/src/services/create-pages.ts @@ -2,11 +2,14 @@ import reporter from "gatsby-cli/lib/reporter" import apiRunnerNode from "../utils/api-runner-node" import { IDataLayerContext } from "../state-machines/data-layer/types" import { assertStore } from "../utils/assert-store" +import { IGatsbyPage } from "../redux/types" +import { deleteUntouchedPages, findChangedPages } from "../utils/changed-pages" export async function createPages({ parentSpan, gatsbyNodeGraphQLFunction, store, + firstRun, }: Partial): Promise<{ deletedPages: string[] changedPages: string[] @@ -16,14 +19,14 @@ export async function createPages({ parentSpan, }) activity.start() - // const timestamp = Date.now() - // const currentPages = new Map(store.getState().pages) + const timestamp = Date.now() + const currentPages = new Map(store.getState().pages) await apiRunnerNode( `createPages`, { graphql: gatsbyNodeGraphQLFunction, - traceId: `initial-createPages`, + traceId: firstRun ? `initial-createPages` : `createPages`, waitForCascadingActions: true, parentSpan: activity.span, }, @@ -38,31 +41,31 @@ export async function createPages({ ) activity.end() - // reporter.info(`Checking for deleted pages`) + reporter.info(`Checking for deleted pages`) - // const deletedPages = deleteUntouchedPages(store.getState().pages, timestamp) + const deletedPages = deleteUntouchedPages(store.getState().pages, timestamp) - // reporter.info( - // `Deleted ${deletedPages.length} page${deletedPages.length === 1 ? `` : `s`}` - // ) + reporter.info( + `Deleted ${deletedPages.length} page${deletedPages.length === 1 ? `` : `s`}` + ) - // const tim = reporter.activityTimer(`Checking for changed pages`) - // tim.start() + const tim = reporter.activityTimer(`Checking for changed pages`) + tim.start() - // const { changedPages } = findChangedPages( - // currentPages, - // store.getState().pages - // ) + const { changedPages } = findChangedPages( + currentPages, + store.getState().pages + ) - // reporter.info( - // `Found ${changedPages.length} changed page${ - // changedPages.length === 1 ? `` : `s` - // }` - // ) - // tim.end() + reporter.info( + `Found ${changedPages.length} changed page${ + changedPages.length === 1 ? `` : `s` + }` + ) + tim.end() return { - changedPages: [], - deletedPages: [], + changedPages, + deletedPages, } } diff --git a/packages/gatsby/src/services/index.ts b/packages/gatsby/src/services/index.ts index 3025520f9074b..7fc3a0c99f47c 100644 --- a/packages/gatsby/src/services/index.ts +++ b/packages/gatsby/src/services/index.ts @@ -18,6 +18,7 @@ import { runStaticQueries } from "./run-static-queries" import { runPageQueries } from "./run-page-queries" import { waitUntilAllJobsComplete } from "../utils/wait-until-jobs-complete" +import { runMutationBatch } from "./run-mutation-batch" export * from "./types" @@ -38,6 +39,7 @@ export { writeOutRedirects, startWebpackServer, rebuildSchemaWithSitePage, + runMutationBatch, } export const buildServices: Record> = { diff --git a/packages/gatsby/src/services/listen-for-mutations.ts b/packages/gatsby/src/services/listen-for-mutations.ts new file mode 100644 index 0000000000000..3dac1d7e08613 --- /dev/null +++ b/packages/gatsby/src/services/listen-for-mutations.ts @@ -0,0 +1,22 @@ +import { emitter } from "../redux" +import { InvokeCallback, Sender } from "xstate" + +export const listenForMutations: InvokeCallback = (callback: Sender) => { + const emitMutation = (event: unknown): void => { + callback({ type: `ADD_NODE_MUTATION`, payload: event }) + } + + const emitFileChange = (event: unknown): void => { + console.log({ event }) + callback({ type: `SOURCE_FILE_CHANGED`, payload: event }) + } + + emitter.on(`ENQUEUE_NODE_MUTATION`, emitMutation) + + emitter.on(`SOURCE_FILE_CHANGED`, emitFileChange) + + return (): void => { + emitter.off(`ENQUEUE_NODE_MUTATION`, emitMutation) + emitter.off(`SOURCE_FILE_CHANGED`, emitFileChange) + } +} diff --git a/packages/gatsby/src/services/run-mutation-batch.ts b/packages/gatsby/src/services/run-mutation-batch.ts new file mode 100644 index 0000000000000..b0f93a805e053 --- /dev/null +++ b/packages/gatsby/src/services/run-mutation-batch.ts @@ -0,0 +1,26 @@ +import { IMutationAction } from "../state-machines/data-layer/types" +import { Store, AnyAction } from "redux" +import { IGatsbyState } from "../redux/types" +import { IWaitingContext } from "../state-machines/waiting/types" +import { assertStore } from "../utils/assert-store" +import { actions } from "../redux/actions" + +const callRealApi = async ( + event: IMutationAction, + store?: Store +): Promise => { + assertStore(store) + const { type, payload } = event + if (type in actions) { + store.dispatch(actions[type](...payload)) + } +} + +export const runMutationBatch = ({ + runningBatch = [], + store, +}: Partial): Promise => + // Consume the entire batch and run actions + Promise.all(runningBatch.map(payload => callRealApi(payload, store))).then( + () => void 0 + ) diff --git a/packages/gatsby/src/services/source-nodes.ts b/packages/gatsby/src/services/source-nodes.ts index caf426815c64e..da87f54b3024a 100644 --- a/packages/gatsby/src/services/source-nodes.ts +++ b/packages/gatsby/src/services/source-nodes.ts @@ -2,8 +2,8 @@ import sourceNodesAndRemoveStaleNodes from "../utils/source-nodes" import reporter from "gatsby-cli/lib/reporter" import { IDataLayerContext } from "../state-machines/data-layer/types" import { assertStore } from "../utils/assert-store" -// import { findChangedPages } from "../utils/check-for-changed-pages" -// import { IGatsbyPage } from "../redux/types" +import { IGatsbyPage } from "../redux/types" +import { findChangedPages } from "../utils/changed-pages" export async function sourceNodes({ parentSpan, @@ -19,10 +19,10 @@ export async function sourceNodes({ parentSpan, }) activity.start() - // const currentPages = new Map(store.getState().pages) + const currentPages = new Map(store.getState().pages) await sourceNodesAndRemoveStaleNodes({ parentSpan: activity.span, - // deferNodeMutation: !!(webhookBody && Object.keys(webhookBody).length), // Coming soon + deferNodeMutation: true, webhookBody, }) @@ -34,31 +34,31 @@ export async function sourceNodes({ .join(`, `)}]` ) - // reporter.info(`Checking for deleted pages`) + reporter.info(`Checking for deleted pages`) // Add this back when we enable page creation outside of onCreatePages - // const tim = reporter.activityTimer(`Checking for changed pages`) - // tim.start() + const tim = reporter.activityTimer(`Checking for changed pages`) + tim.start() - // const { changedPages, deletedPages } = findChangedPages( - // currentPages, - // store.getState().pages - // ) + const { changedPages, deletedPages } = findChangedPages( + currentPages, + store.getState().pages + ) - // reporter.info( - // `Deleted ${deletedPages.length} page${deletedPages.length === 1 ? `` : `s`}` - // ) + reporter.info( + `Deleted ${deletedPages.length} page${deletedPages.length === 1 ? `` : `s`}` + ) - // reporter.info( - // `Found ${changedPages.length} changed page${ - // changedPages.length === 1 ? `` : `s` - // }` - // ) - // tim.end() + reporter.info( + `Found ${changedPages.length} changed page${ + changedPages.length === 1 ? `` : `s` + }` + ) + tim.end() activity.end() return { - deletedPages: [], - changedPages: [], + deletedPages, + changedPages, } } diff --git a/packages/gatsby/src/services/start-webpack-server.ts b/packages/gatsby/src/services/start-webpack-server.ts index 095ab6fed280e..eefbe9971db5f 100644 --- a/packages/gatsby/src/services/start-webpack-server.ts +++ b/packages/gatsby/src/services/start-webpack-server.ts @@ -12,7 +12,7 @@ import { import { printDeprecationWarnings } from "../utils/print-deprecation-warnings" import { printInstructions } from "../utils/print-instructions" import { prepareUrls } from "../utils/prepare-urls" -import { startServer } from "../utils/start-server" +import { startServer, IWebpackWatchingPauseResume } from "../utils/start-server" import { WebsocketManager } from "../utils/websocket-manager" import { IBuildContext } from "./" import { @@ -28,15 +28,17 @@ export async function startWebpackServer({ }: Partial): Promise<{ compiler: Compiler websocketManager: WebsocketManager + webpackWatching: IWebpackWatchingPauseResume }> { if (!program || !app) { throw new Error(`Missing required params`) } - let { compiler, webpackActivity, websocketManager } = await startServer( - program, - app, - workerPool - ) + let { + compiler, + webpackActivity, + websocketManager, + webpackWatching, + } = await startServer(program, app, workerPool) compiler.hooks.invalid.tap(`log compiling`, function () { markWebpackStatusAsPending() @@ -112,7 +114,7 @@ export async function startWebpackServer({ enqueueFlush() markWebpackStatusAsDone() done() - resolve({ compiler, websocketManager }) + resolve({ compiler, websocketManager, webpackWatching }) }) }) } diff --git a/packages/gatsby/src/services/types.ts b/packages/gatsby/src/services/types.ts index 3ec2a3766f28a..a390ab685923f 100644 --- a/packages/gatsby/src/services/types.ts +++ b/packages/gatsby/src/services/types.ts @@ -6,6 +6,10 @@ import { Store, AnyAction } from "redux" import { IGatsbyState } from "../redux/types" import { Express } from "express" import JestWorker from "jest-worker" +import { Actor, AnyEventObject } from "xstate" +import { Compiler } from "webpack" +import { WebsocketManager } from "../utils/websocket-manager" +import { IWebpackWatchingPauseResume } from "../utils/start-server" export interface IGroupedQueryIds { pageQueryIds: string[] staticQueryIds: string[] @@ -26,4 +30,10 @@ export interface IBuildContext { refresh?: boolean workerPool?: JestWorker app?: Express + nodesMutatedDuringQueryRun?: boolean + mutationListener?: Actor + nodeMutationBatch?: IMutationAction[] + compiler?: Compiler + websocketManager?: WebsocketManager + webpackWatching?: IWebpackWatchingPauseResume } diff --git a/packages/gatsby/src/state-machines/actions.ts b/packages/gatsby/src/state-machines/actions.ts new file mode 100644 index 0000000000000..57cf17721a4a9 --- /dev/null +++ b/packages/gatsby/src/state-machines/actions.ts @@ -0,0 +1,95 @@ +import { + assign, + AnyEventObject, + ActionFunction, + spawn, + ActionFunctionMap, + DoneEventObject, +} from "xstate" +import { Store } from "redux" +import { IBuildContext, IMutationAction } from "../services" +import { actions } from "../redux/actions" +import { listenForMutations } from "../services/listen-for-mutations" +import { DataLayerResult } from "./data-layer" +import { assertStore } from "../utils/assert-store" + +export const callRealApi = async ( + event: IMutationAction, + store?: Store +): Promise => { + assertStore(store) + const { type, payload } = event + if (type in actions) { + store.dispatch(actions[type](...payload)) + } + return null +} + +/** + * Handler for when we're inside handlers that should be able to mutate nodes + */ +export const callApi: ActionFunction = async ( + { store }, + event +): Promise => callRealApi(event.payload, store) + +/** + * Event handler used in all states where we're not ready to process node + * mutations. Instead we add it to a batch to process when we're next idle + */ +export const addNodeMutation = assign({ + nodeMutationBatch: ({ nodeMutationBatch = [] }, { payload }) => { + // It's not pretty, but it's much quicker than concat + nodeMutationBatch.push(payload) + return nodeMutationBatch + }, +}) + +export const assignStoreAndWorkerPool = assign( + (_context, event) => { + const { store, workerPool } = event.data + return { + store, + workerPool, + } + } +) + +export const assignServiceResult = assign( + (_context, { data }): DataLayerResult => data +) + +export const spawnMutationListener = assign({ + mutationListener: () => spawn(listenForMutations, `listen-for-mutations`), +}) + +export const assignServers = assign( + (_context, { data }) => { + return { + ...data, + firstRun: false, + } + } +) + +/** + * Event handler used in all states where we're not ready to process a file change + * Instead we add it to a batch to process when we're next idle + */ +// export const markFilesDirty: BuildMachineAction = assign({ +// filesDirty: true, +// }) + +export const markNodesDirty = assign({ + nodesMutatedDuringQueryRun: true, +}) + +export const buildActions: ActionFunctionMap = { + callApi, + markNodesDirty, + addNodeMutation, + spawnMutationListener, + assignStoreAndWorkerPool, + assignServiceResult, + assignServers, +} diff --git a/packages/gatsby/src/state-machines/data-layer/actions.ts b/packages/gatsby/src/state-machines/data-layer/actions.ts index 72f89e5ca8ee5..810b5503ad9b4 100644 --- a/packages/gatsby/src/state-machines/data-layer/actions.ts +++ b/packages/gatsby/src/state-machines/data-layer/actions.ts @@ -1,24 +1,14 @@ -import { - assign, - AnyEventObject, - ActionFunction, - AssignAction, - DoneInvokeEvent, - ActionFunctionMap, -} from "xstate" +import { assign, DoneInvokeEvent, ActionFunctionMap } from "xstate" import { createGraphQLRunner } from "../../bootstrap/create-graphql-runner" import reporter from "gatsby-cli/lib/reporter" import { IDataLayerContext } from "./types" +import { callApi, markNodesDirty } from "../actions" import { assertStore } from "../../utils/assert-store" const concatUnique = (array1: T[] = [], array2: T[] = []): T[] => Array.from(new Set(array1.concat(array2))) -type BuildMachineAction = - | ActionFunction - | AssignAction - -export const assignChangedPages: BuildMachineAction = assign< +export const assignChangedPages = assign< IDataLayerContext, DoneInvokeEvent<{ changedPages: string[] @@ -31,19 +21,16 @@ export const assignChangedPages: BuildMachineAction = assign< } }) -export const assignGatsbyNodeGraphQL: BuildMachineAction = assign< - IDataLayerContext ->({ +export const assignGatsbyNodeGraphQL = assign({ gatsbyNodeGraphQLFunction: ({ store }: IDataLayerContext) => { assertStore(store) return createGraphQLRunner(store, reporter) }, }) -export const dataLayerActions: ActionFunctionMap< - IDataLayerContext, - AnyEventObject -> = { +export const dataLayerActions: ActionFunctionMap = { assignChangedPages, assignGatsbyNodeGraphQL, + callApi, + markNodesDirty, } diff --git a/packages/gatsby/src/state-machines/data-layer/index.ts b/packages/gatsby/src/state-machines/data-layer/index.ts index b814d2dece6b3..a730fe01ca31e 100644 --- a/packages/gatsby/src/state-machines/data-layer/index.ts +++ b/packages/gatsby/src/state-machines/data-layer/index.ts @@ -12,8 +12,22 @@ export type DataLayerResult = Pick< > const dataLayerStates: MachineConfig = { - initial: `customizingSchema`, + initial: `start`, states: { + start: { + on: { + "": [ + { + target: `buildingSchema`, + cond: ({ skipSourcing }: IDataLayerContext): boolean => + !!skipSourcing, + }, + { + target: `customizingSchema`, + }, + ], + }, + }, customizingSchema: { invoke: { src: `customizeSchema`, @@ -44,6 +58,7 @@ const dataLayerStates: MachineConfig = { }, }, creatingPages: { + on: { ADD_NODE_MUTATION: { actions: [`markNodesDirty`, `callApi`] } }, invoke: { id: `creating-pages`, src: `createPages`, diff --git a/packages/gatsby/src/state-machines/data-layer/types.ts b/packages/gatsby/src/state-machines/data-layer/types.ts index 155c83021a059..6aac97ab71cb0 100644 --- a/packages/gatsby/src/state-machines/data-layer/types.ts +++ b/packages/gatsby/src/state-machines/data-layer/types.ts @@ -15,6 +15,8 @@ export interface IMutationAction { payload: unknown[] } export interface IDataLayerContext { + skipSourcing?: boolean + nodesMutatedDuringQueryRun?: boolean firstRun?: boolean program?: IProgram store?: Store diff --git a/packages/gatsby/src/state-machines/query-running/actions.ts b/packages/gatsby/src/state-machines/query-running/actions.ts index 57155ee35c9e4..11ebe97ad31a4 100644 --- a/packages/gatsby/src/state-machines/query-running/actions.ts +++ b/packages/gatsby/src/state-machines/query-running/actions.ts @@ -2,33 +2,10 @@ import { IQueryRunningContext } from "./types" import { DoneInvokeEvent, assign, ActionFunctionMap } from "xstate" import { GraphQLRunner } from "../../query/graphql-runner" import { assertStore } from "../../utils/assert-store" +import { enqueueFlush } from "../../utils/page-data" -export const emitStaticQueryDataToWebsocket = ( - { websocketManager }: IQueryRunningContext, - { data: { results } }: DoneInvokeEvent -): void => { - if (websocketManager && results) { - results.forEach((result, id) => { - websocketManager.emitStaticQueryData({ - result, - id, - }) - }) - } -} - -export const emitPageDataToWebsocket = ( - { websocketManager }: IQueryRunningContext, - { data: { results } }: DoneInvokeEvent -): void => { - if (websocketManager && results) { - results.forEach((result, id) => { - websocketManager.emitPageData({ - result, - id, - }) - }) - } +export const flushPageData = (): void => { + enqueueFlush() } export const assignDirtyQueries = assign< @@ -36,6 +13,7 @@ export const assignDirtyQueries = assign< DoneInvokeEvent >((_context, { data }) => { const { queryIds } = data + console.log({ queryIds }) return { filesDirty: false, queryIds, @@ -61,6 +39,5 @@ export const queryActions: ActionFunctionMap< > = { resetGraphQLRunner, assignDirtyQueries, - emitPageDataToWebsocket, - emitStaticQueryDataToWebsocket, + flushPageData, } diff --git a/packages/gatsby/src/state-machines/query-running/index.ts b/packages/gatsby/src/state-machines/query-running/index.ts index 345f329754a5e..39161dac73304 100644 --- a/packages/gatsby/src/state-machines/query-running/index.ts +++ b/packages/gatsby/src/state-machines/query-running/index.ts @@ -42,38 +42,19 @@ export const queryStates: MachineConfig = { invoke: { src: `runStaticQueries`, id: `running-static-queries`, - onDone: [ - { - target: `runningPageQueries`, - actions: `emitStaticQueryDataToWebsocket`, - // Only emit if there's a websocket manager - // This won't be the case on first run, and the query data - // will be emitted when the client first connects - cond: (context): boolean => !!context.websocketManager, - }, - { - target: `runningPageQueries`, - }, - ], + onDone: { + target: `runningPageQueries`, + }, }, }, runningPageQueries: { invoke: { src: `runPageQueries`, id: `running-page-queries`, - onDone: [ - { - target: `waitingForJobs`, - actions: `emitPageDataToWebsocket`, - // Only emit if there's a websocket manager - // This won't be the case on first run, and the page data - // will be emitted when the client first connects - cond: (context): boolean => !!context.websocketManager, - }, - { - target: `waitingForJobs`, - }, - ], + onDone: { + target: `waitingForJobs`, + actions: `flushPageData`, + }, }, }, diff --git a/packages/gatsby/src/state-machines/shared-transition-configs.ts b/packages/gatsby/src/state-machines/shared-transition-configs.ts new file mode 100644 index 0000000000000..6daccc840793e --- /dev/null +++ b/packages/gatsby/src/state-machines/shared-transition-configs.ts @@ -0,0 +1,26 @@ +/** + * Event handler used in all states where we're not ready to process node + * mutations. Instead we add it to a batch to process when we're next idle + */ +export const ADD_NODE_MUTATION = { + actions: `addNodeMutation`, +} + +/** + * Event handler used in all states where we're not ready to process a file change + * Instead we add it to a batch to process when we're next idle + */ +// export const SOURCE_FILE_CHANGED: TransitionConfig< +// Pick, +// AnyEventObject +// > = { +// actions: `markFilesDirty`, +// } + +/** + * When running queries we might add nodes (e.g from resolvers). If so we'll + * want to re-run queries and schema inference + */ +export const runMutationAndMarkDirty = { + actions: [`markNodesDirty`, `callApi`], +} diff --git a/packages/gatsby/src/state-machines/waiting/actions.ts b/packages/gatsby/src/state-machines/waiting/actions.ts new file mode 100644 index 0000000000000..b1e01d67c68ab --- /dev/null +++ b/packages/gatsby/src/state-machines/waiting/actions.ts @@ -0,0 +1,21 @@ +import { AssignAction, assign, ActionFunctionMap } from "xstate" +import { IWaitingContext } from "./types" +import { AnyAction } from "redux" + +/** + * Event handler used when we're not ready to process node mutations. + * Instead we add it to a batch to process when we're next idle + */ +export const addNodeMutation: AssignAction = assign( + { + nodeMutationBatch: ({ nodeMutationBatch = [] }, { payload }) => { + // It's not pretty, but it's much quicker than concat + nodeMutationBatch.push(payload) + return nodeMutationBatch + }, + } +) + +export const waitingActions: ActionFunctionMap = { + addNodeMutation, +} diff --git a/packages/gatsby/src/state-machines/waiting/index.ts b/packages/gatsby/src/state-machines/waiting/index.ts new file mode 100644 index 0000000000000..06b717cffa81c --- /dev/null +++ b/packages/gatsby/src/state-machines/waiting/index.ts @@ -0,0 +1,94 @@ +import { MachineConfig, assign, Machine } from "xstate" +import { IWaitingContext } from "./types" +import { waitingActions } from "./actions" +import { waitingServices } from "./services" + +const NODE_MUTATION_BATCH_SIZE = 100 +const NODE_MUTATION_BATCH_TIMEOUT = 1000 + +export type WaitingResult = Pick + +export const waitingStates: MachineConfig = { + initial: `idle`, + states: { + idle: { + on: { + "": + // If we already have queued node mutations, move + // immediately to batching + { + cond: (ctx): boolean => !!ctx.nodeMutationBatch?.length, + target: `batchingNodeMutations`, + }, + ADD_NODE_MUTATION: { + actions: `addNodeMutation`, + target: `batchingNodeMutations`, + }, + }, + }, + + batchingNodeMutations: { + on: { + // Check if the batch is already full on entry + "": { + cond: (ctx): boolean => + ctx.nodeMutationBatch?.length >= NODE_MUTATION_BATCH_SIZE, + target: `committingBatch`, + }, + // More mutations added to batch + ADD_NODE_MUTATION: [ + // If this fills the batch then commit it + { + actions: `addNodeMutation`, + cond: (ctx): boolean => + ctx.nodeMutationBatch?.length >= NODE_MUTATION_BATCH_SIZE, + target: `committingBatch`, + }, + // otherwise just add it to the batch + { + actions: `addNodeMutation`, + }, + ], + }, + // Time's up + after: { + [NODE_MUTATION_BATCH_TIMEOUT]: `committingBatch`, + }, + }, + committingBatch: { + entry: assign(({ nodeMutationBatch }) => { + return { + nodeMutationBatch: [], + runningBatch: nodeMutationBatch, + } + }), + on: { + // While we're running the batch we need to batch any incoming mutations too + ADD_NODE_MUTATION: { + actions: `addNodeMutation`, + }, + }, + invoke: { + src: `runMutationBatch`, + onDone: { + actions: assign({ + runningBatch: [], + }), + target: `rebuild`, + }, + }, + }, + rebuild: { + type: `final`, + data: ({ nodeMutationBatch }): WaitingResult => { + return { nodeMutationBatch } + }, + }, + }, +} + +// eslint-disable-next-line new-cap +export const waitingMachine = Machine(waitingStates, { + actions: waitingActions, + services: waitingServices, +}) diff --git a/packages/gatsby/src/state-machines/waiting/services.ts b/packages/gatsby/src/state-machines/waiting/services.ts new file mode 100644 index 0000000000000..f1362e07feaaa --- /dev/null +++ b/packages/gatsby/src/state-machines/waiting/services.ts @@ -0,0 +1,3 @@ +import { runMutationBatch } from "../../services" + +export const waitingServices = { runMutationBatch } diff --git a/packages/gatsby/src/state-machines/waiting/types.ts b/packages/gatsby/src/state-machines/waiting/types.ts new file mode 100644 index 0000000000000..bf915d3c0cae1 --- /dev/null +++ b/packages/gatsby/src/state-machines/waiting/types.ts @@ -0,0 +1,14 @@ +import { Store, AnyAction } from "redux" +import { IGatsbyState } from "../../redux/types" + +export interface IMutationAction { + type: string + payload: unknown[] +} +export interface IWaitingContext { + nodeMutationBatch: IMutationAction[] + store?: Store + runningBatch?: IMutationAction[] + filesDirty?: boolean + webhookBody?: Record +} diff --git a/packages/gatsby/src/utils/api-runner-node.js b/packages/gatsby/src/utils/api-runner-node.js index 94d7cc4f1418c..aced089d54e1b 100644 --- a/packages/gatsby/src/utils/api-runner-node.js +++ b/packages/gatsby/src/utils/api-runner-node.js @@ -37,9 +37,11 @@ const { loadNodeContent } = require(`../db/nodes`) // metadata to actions they create. const boundPluginActionCreators = {} const doubleBind = (boundActionCreators, api, plugin, actionOptions) => { - const { traceId } = actionOptions - if (boundPluginActionCreators[plugin.name + api + traceId]) { - return boundPluginActionCreators[plugin.name + api + traceId] + const { traceId, deferNodeMutation } = actionOptions + const defer = deferNodeMutation ? `defer-node-mutation` : `` + const actionKey = plugin.name + api + traceId + defer + if (boundPluginActionCreators[actionKey]) { + return boundPluginActionCreators[actionKey] } else { const keys = Object.keys(boundActionCreators) const doubleBoundActionCreators = {} @@ -59,9 +61,7 @@ const doubleBind = (boundActionCreators, api, plugin, actionOptions) => { } } } - boundPluginActionCreators[ - plugin.name + api + traceId - ] = doubleBoundActionCreators + boundPluginActionCreators[actionKey] = doubleBoundActionCreators return doubleBoundActionCreators } } @@ -80,13 +80,46 @@ const initAPICallTracing = parentSpan => { } } +const deferredAction = type => (...args) => { + emitter.emit(`ENQUEUE_NODE_MUTATION`, { + type, + payload: args, + }) +} + +const NODE_MUTATION_ACTIONS = [ + `createNode`, + `deleteNode`, + `deleteNodes`, + `touchNode`, + `createParentChildLink`, + `createNodeField`, +] + +const deferActions = actions => { + // console.log(`actual actions`, actions) + const returnValue = NODE_MUTATION_ACTIONS.reduce((prev, next) => { + prev[next] = deferredAction(next) + return prev + }, actions) + // console.log(`returnValue`, returnValue) + return returnValue +} + const getLocalReporter = (activity, reporter) => activity ? { ...reporter, panicOnBuild: activity.panicOnBuild.bind(activity) } : reporter +const pluginNodeCache = new Map() + const runAPI = (plugin, api, args, activity) => { - const gatsbyNode = require(`${plugin.resolve}/gatsby-node`) + let gatsbyNode = pluginNodeCache.get(plugin.name) + if (!gatsbyNode) { + gatsbyNode = require(`${plugin.resolve}/gatsby-node`) + pluginNodeCache.set(plugin.name, gatsbyNode) + } + if (gatsbyNode[api]) { const parentSpan = args && args.parentSpan const spanOptions = parentSpan ? { childOf: parentSpan } : {} @@ -103,10 +136,15 @@ const runAPI = (plugin, api, args, activity) => { ...publicActions, ...(restrictedActionsAvailableInAPI[api] || {}), } - const boundActionCreators = bindActionCreators( + let boundActionCreators = bindActionCreators( availableActions, store.dispatch ) + + if (args.deferNodeMutation) { + boundActionCreators = deferActions(boundActionCreators) + } + const doubleBoundActionCreators = doubleBind( boundActionCreators, api, diff --git a/packages/gatsby/src/utils/changed-pages.ts b/packages/gatsby/src/utils/changed-pages.ts new file mode 100644 index 0000000000000..643190f4379c8 --- /dev/null +++ b/packages/gatsby/src/utils/changed-pages.ts @@ -0,0 +1,54 @@ +import { boundActionCreators } from "../redux/actions" +const { deletePage, deleteComponentsDependencies } = boundActionCreators + +import { isEqualWith, IsEqualCustomizer } from "lodash" +import { IGatsbyPage } from "../redux/types" + +export function deleteUntouchedPages( + currentPages: Map, + timestamp: number +): string[] { + const deletedPages: string[] = [] + + // Delete pages that weren't updated when running createPages. + currentPages.forEach(page => { + if ( + !page.isCreatedByStatefulCreatePages && + page.updatedAt < timestamp && + page.path !== `/404.html` + ) { + deleteComponentsDependencies([page.path]) + deletePage(page) + deletedPages.push(page.path, `/page-data${page.path}`) + } + }) + return deletedPages +} + +export function findChangedPages( + oldPages: Map, + currentPages: Map +): { + changedPages: string[] + deletedPages: string[] +} { + const changedPages: string[] = [] + + const compareWithoutUpdated: IsEqualCustomizer = (_left, _right, key) => + key === `updatedAt` || undefined + + currentPages.forEach((newPage, key) => { + const oldPage = oldPages.get(key) + if (!oldPage || !isEqualWith(newPage, oldPage, compareWithoutUpdated)) { + changedPages.push(key) + } + }) + const deletedPages: string[] = [] + oldPages.forEach((_page, key) => { + if (!currentPages.has(key)) { + deletedPages.push(key) + } + }) + + return { changedPages, deletedPages } +} diff --git a/packages/gatsby/src/utils/source-nodes.ts b/packages/gatsby/src/utils/source-nodes.ts index 68329e2f7e092..78a9ae3b5c3ae 100644 --- a/packages/gatsby/src/utils/source-nodes.ts +++ b/packages/gatsby/src/utils/source-nodes.ts @@ -85,13 +85,16 @@ function deleteStaleNodes(state: IGatsbyState, nodes: Node[]): void { export default async ({ webhookBody, parentSpan, + deferNodeMutation = false, }: { webhookBody?: unknown parentSpan?: Span + deferNodeMutation?: boolean }): Promise => { await apiRunner(`sourceNodes`, { traceId: `initial-sourceNodes`, waitForCascadingActions: true, + deferNodeMutation, parentSpan, webhookBody: webhookBody || {}, }) diff --git a/packages/gatsby/src/utils/start-server.ts b/packages/gatsby/src/utils/start-server.ts index 4708b8278219f..6c71e2f947d4e 100644 --- a/packages/gatsby/src/utils/start-server.ts +++ b/packages/gatsby/src/utils/start-server.ts @@ -53,7 +53,7 @@ interface IServer { webpackWatching: IWebpackWatchingPauseResume } -interface IWebpackWatchingPauseResume { +export interface IWebpackWatchingPauseResume extends webpack.Watching { suspend: () => void resume: () => void } @@ -63,7 +63,7 @@ interface IWebpackWatchingPauseResume { type PatchedWebpackDevMiddleware = WebpackDevMiddleware & express.RequestHandler & { context: { - watching: webpack.Watching & IWebpackWatchingPauseResume + watching: IWebpackWatchingPauseResume } } From 6ca668f387a236f268b1403951bc1819c82d6c8c Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Fri, 3 Jul 2020 09:29:33 +0100 Subject: [PATCH 19/44] Use new pagedata utils --- .../src/state-machines/query-running/actions.ts | 17 ++++------------- .../src/state-machines/query-running/index.ts | 17 ++++------------- 2 files changed, 8 insertions(+), 26 deletions(-) diff --git a/packages/gatsby/src/state-machines/query-running/actions.ts b/packages/gatsby/src/state-machines/query-running/actions.ts index 57155ee35c9e4..e4ff6d7585a24 100644 --- a/packages/gatsby/src/state-machines/query-running/actions.ts +++ b/packages/gatsby/src/state-machines/query-running/actions.ts @@ -2,6 +2,7 @@ import { IQueryRunningContext } from "./types" import { DoneInvokeEvent, assign, ActionFunctionMap } from "xstate" import { GraphQLRunner } from "../../query/graphql-runner" import { assertStore } from "../../utils/assert-store" +import { enqueueFlush } from "../../utils/page-data" export const emitStaticQueryDataToWebsocket = ( { websocketManager }: IQueryRunningContext, @@ -17,18 +18,8 @@ export const emitStaticQueryDataToWebsocket = ( } } -export const emitPageDataToWebsocket = ( - { websocketManager }: IQueryRunningContext, - { data: { results } }: DoneInvokeEvent -): void => { - if (websocketManager && results) { - results.forEach((result, id) => { - websocketManager.emitPageData({ - result, - id, - }) - }) - } +export const flushPageData = (): void => { + enqueueFlush() } export const assignDirtyQueries = assign< @@ -61,6 +52,6 @@ export const queryActions: ActionFunctionMap< > = { resetGraphQLRunner, assignDirtyQueries, - emitPageDataToWebsocket, emitStaticQueryDataToWebsocket, + flushPageData, } diff --git a/packages/gatsby/src/state-machines/query-running/index.ts b/packages/gatsby/src/state-machines/query-running/index.ts index 345f329754a5e..e7da5b777d37b 100644 --- a/packages/gatsby/src/state-machines/query-running/index.ts +++ b/packages/gatsby/src/state-machines/query-running/index.ts @@ -61,19 +61,10 @@ export const queryStates: MachineConfig = { invoke: { src: `runPageQueries`, id: `running-page-queries`, - onDone: [ - { - target: `waitingForJobs`, - actions: `emitPageDataToWebsocket`, - // Only emit if there's a websocket manager - // This won't be the case on first run, and the page data - // will be emitted when the client first connects - cond: (context): boolean => !!context.websocketManager, - }, - { - target: `waitingForJobs`, - }, - ], + onDone: { + target: `waitingForJobs`, + actions: `flushPageData`, + }, }, }, From d3e7a59c388c69a8c60d0273a0aafe28142e9001 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Fri, 3 Jul 2020 09:50:35 +0100 Subject: [PATCH 20/44] Use develop queue --- packages/gatsby/src/query/index.js | 4 +++- packages/gatsby/src/query/queue.js | 11 +++++++++++ .../src/state-machines/query-running/actions.ts | 15 --------------- .../src/state-machines/query-running/index.ts | 16 +++------------- 4 files changed, 17 insertions(+), 29 deletions(-) diff --git a/packages/gatsby/src/query/index.js b/packages/gatsby/src/query/index.js index 88de7ab91b880..a8c9d7a96773b 100644 --- a/packages/gatsby/src/query/index.js +++ b/packages/gatsby/src/query/index.js @@ -166,7 +166,9 @@ const processQueries = async ( queryJobs, { activity, graphqlRunner, graphqlTracing } ) => { - const queue = queryQueue.createBuildQueue(graphqlRunner, { graphqlTracing }) + const queue = queryQueue.createAppropriateQueue(graphqlRunner, { + graphqlTracing, + }) return queryQueue.processBatch(queue, queryJobs, activity) } diff --git a/packages/gatsby/src/query/queue.js b/packages/gatsby/src/query/queue.js index ed80fb7d6cbf6..104a19e05c764 100644 --- a/packages/gatsby/src/query/queue.js +++ b/packages/gatsby/src/query/queue.js @@ -59,6 +59,16 @@ const createDevelopQueue = getRunner => { return new Queue(handler, queueOptions) } +const createAppropriateQueue = (graphqlRunner, runnerOptions = {}) => { + if (process.env.NODE_ENV === `production`) { + return createBuildQueue(graphqlRunner, runnerOptions) + } + if (!graphqlRunner) { + graphqlRunner = new GraphQLRunner(store, runnerOptions) + } + return createDevelopQueue(() => graphqlRunner) +} + /** * Returns a promise that pushes jobs onto queue and resolves onces * they're all finished processing (or rejects if one or more jobs @@ -116,4 +126,5 @@ module.exports = { createBuildQueue, createDevelopQueue, processBatch, + createAppropriateQueue, } diff --git a/packages/gatsby/src/state-machines/query-running/actions.ts b/packages/gatsby/src/state-machines/query-running/actions.ts index e4ff6d7585a24..c329e95ea8356 100644 --- a/packages/gatsby/src/state-machines/query-running/actions.ts +++ b/packages/gatsby/src/state-machines/query-running/actions.ts @@ -4,20 +4,6 @@ import { GraphQLRunner } from "../../query/graphql-runner" import { assertStore } from "../../utils/assert-store" import { enqueueFlush } from "../../utils/page-data" -export const emitStaticQueryDataToWebsocket = ( - { websocketManager }: IQueryRunningContext, - { data: { results } }: DoneInvokeEvent -): void => { - if (websocketManager && results) { - results.forEach((result, id) => { - websocketManager.emitStaticQueryData({ - result, - id, - }) - }) - } -} - export const flushPageData = (): void => { enqueueFlush() } @@ -52,6 +38,5 @@ export const queryActions: ActionFunctionMap< > = { resetGraphQLRunner, assignDirtyQueries, - emitStaticQueryDataToWebsocket, flushPageData, } diff --git a/packages/gatsby/src/state-machines/query-running/index.ts b/packages/gatsby/src/state-machines/query-running/index.ts index e7da5b777d37b..39161dac73304 100644 --- a/packages/gatsby/src/state-machines/query-running/index.ts +++ b/packages/gatsby/src/state-machines/query-running/index.ts @@ -42,19 +42,9 @@ export const queryStates: MachineConfig = { invoke: { src: `runStaticQueries`, id: `running-static-queries`, - onDone: [ - { - target: `runningPageQueries`, - actions: `emitStaticQueryDataToWebsocket`, - // Only emit if there's a websocket manager - // This won't be the case on first run, and the query data - // will be emitted when the client first connects - cond: (context): boolean => !!context.websocketManager, - }, - { - target: `runningPageQueries`, - }, - ], + onDone: { + target: `runningPageQueries`, + }, }, }, runningPageQueries: { From 186eb0cb16d6dda2ac150766abdc19f421c9c558 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Fri, 3 Jul 2020 10:08:37 +0100 Subject: [PATCH 21/44] New xstate syntax --- .../src/state-machines/data-layer/index.ts | 22 +++++++--------- .../state-machines/query-running/actions.ts | 1 - .../src/state-machines/waiting/index.ts | 26 +++++++++---------- 3 files changed, 23 insertions(+), 26 deletions(-) diff --git a/packages/gatsby/src/state-machines/data-layer/index.ts b/packages/gatsby/src/state-machines/data-layer/index.ts index a730fe01ca31e..61d69c1a2ffa3 100644 --- a/packages/gatsby/src/state-machines/data-layer/index.ts +++ b/packages/gatsby/src/state-machines/data-layer/index.ts @@ -15,18 +15,16 @@ const dataLayerStates: MachineConfig = { initial: `start`, states: { start: { - on: { - "": [ - { - target: `buildingSchema`, - cond: ({ skipSourcing }: IDataLayerContext): boolean => - !!skipSourcing, - }, - { - target: `customizingSchema`, - }, - ], - }, + always: [ + { + target: `buildingSchema`, + cond: ({ skipSourcing }: IDataLayerContext): boolean => + !!skipSourcing, + }, + { + target: `customizingSchema`, + }, + ], }, customizingSchema: { invoke: { diff --git a/packages/gatsby/src/state-machines/query-running/actions.ts b/packages/gatsby/src/state-machines/query-running/actions.ts index 11ebe97ad31a4..c329e95ea8356 100644 --- a/packages/gatsby/src/state-machines/query-running/actions.ts +++ b/packages/gatsby/src/state-machines/query-running/actions.ts @@ -13,7 +13,6 @@ export const assignDirtyQueries = assign< DoneInvokeEvent >((_context, { data }) => { const { queryIds } = data - console.log({ queryIds }) return { filesDirty: false, queryIds, diff --git a/packages/gatsby/src/state-machines/waiting/index.ts b/packages/gatsby/src/state-machines/waiting/index.ts index 06b717cffa81c..39d8b6094b795 100644 --- a/packages/gatsby/src/state-machines/waiting/index.ts +++ b/packages/gatsby/src/state-machines/waiting/index.ts @@ -12,14 +12,14 @@ export const waitingStates: MachineConfig = { initial: `idle`, states: { idle: { + always: + // If we already have queued node mutations, move + // immediately to batching + { + cond: (ctx): boolean => !!ctx.nodeMutationBatch?.length, + target: `batchingNodeMutations`, + }, on: { - "": - // If we already have queued node mutations, move - // immediately to batching - { - cond: (ctx): boolean => !!ctx.nodeMutationBatch?.length, - target: `batchingNodeMutations`, - }, ADD_NODE_MUTATION: { actions: `addNodeMutation`, target: `batchingNodeMutations`, @@ -28,13 +28,13 @@ export const waitingStates: MachineConfig = { }, batchingNodeMutations: { + // Check if the batch is already full on entry + always: { + cond: (ctx): boolean => + ctx.nodeMutationBatch?.length >= NODE_MUTATION_BATCH_SIZE, + target: `committingBatch`, + }, on: { - // Check if the batch is already full on entry - "": { - cond: (ctx): boolean => - ctx.nodeMutationBatch?.length >= NODE_MUTATION_BATCH_SIZE, - target: `committingBatch`, - }, // More mutations added to batch ADD_NODE_MUTATION: [ // If this fills the batch then commit it From c0acfe1828d01676e4c687b504052e1021d26281 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Fri, 3 Jul 2020 12:07:34 +0100 Subject: [PATCH 22/44] Work-around xstate bug --- packages/gatsby/src/commands/develop-process.ts | 3 ++- packages/gatsby/src/state-machines/data-layer/index.ts | 1 + packages/gatsby/src/state-machines/waiting/index.ts | 5 +++-- packages/gatsby/src/state-machines/waiting/types.ts | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/gatsby/src/commands/develop-process.ts b/packages/gatsby/src/commands/develop-process.ts index 0105ee9554c23..551a1a239932a 100644 --- a/packages/gatsby/src/commands/develop-process.ts +++ b/packages/gatsby/src/commands/develop-process.ts @@ -32,6 +32,7 @@ import { Actor, Interpreter, forwardTo, + State, } from "xstate" import { dataLayerMachine } from "../state-machines/data-layer" import { IDataLayerContext } from "../state-machines/data-layer/types" @@ -305,7 +306,7 @@ module.exports = async (program: IProgram): Promise => { const listeners = new WeakSet() - let last = service.state + let last: State service.onTransition(state => { if (!last) { diff --git a/packages/gatsby/src/state-machines/data-layer/index.ts b/packages/gatsby/src/state-machines/data-layer/index.ts index 61d69c1a2ffa3..adfb28a704bcf 100644 --- a/packages/gatsby/src/state-machines/data-layer/index.ts +++ b/packages/gatsby/src/state-machines/data-layer/index.ts @@ -13,6 +13,7 @@ export type DataLayerResult = Pick< const dataLayerStates: MachineConfig = { initial: `start`, + context: {}, states: { start: { always: [ diff --git a/packages/gatsby/src/state-machines/waiting/index.ts b/packages/gatsby/src/state-machines/waiting/index.ts index 39d8b6094b795..2ae41b4d7e92c 100644 --- a/packages/gatsby/src/state-machines/waiting/index.ts +++ b/packages/gatsby/src/state-machines/waiting/index.ts @@ -10,6 +10,7 @@ export type WaitingResult = Pick export const waitingStates: MachineConfig = { initial: `idle`, + context: {}, states: { idle: { always: @@ -31,7 +32,7 @@ export const waitingStates: MachineConfig = { // Check if the batch is already full on entry always: { cond: (ctx): boolean => - ctx.nodeMutationBatch?.length >= NODE_MUTATION_BATCH_SIZE, + (ctx.nodeMutationBatch?.length || 0) >= NODE_MUTATION_BATCH_SIZE, target: `committingBatch`, }, on: { @@ -41,7 +42,7 @@ export const waitingStates: MachineConfig = { { actions: `addNodeMutation`, cond: (ctx): boolean => - ctx.nodeMutationBatch?.length >= NODE_MUTATION_BATCH_SIZE, + (ctx.nodeMutationBatch?.length || 0) >= NODE_MUTATION_BATCH_SIZE, target: `committingBatch`, }, // otherwise just add it to the batch diff --git a/packages/gatsby/src/state-machines/waiting/types.ts b/packages/gatsby/src/state-machines/waiting/types.ts index bf915d3c0cae1..63c1d14e491dc 100644 --- a/packages/gatsby/src/state-machines/waiting/types.ts +++ b/packages/gatsby/src/state-machines/waiting/types.ts @@ -6,7 +6,7 @@ export interface IMutationAction { payload: unknown[] } export interface IWaitingContext { - nodeMutationBatch: IMutationAction[] + nodeMutationBatch?: IMutationAction[] store?: Store runningBatch?: IMutationAction[] filesDirty?: boolean From 12d5f2000b72c2e4c8c25a92cbdf8e50ea70cd18 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Fri, 3 Jul 2020 12:29:34 +0100 Subject: [PATCH 23/44] Track first run --- packages/gatsby/src/commands/develop-process.ts | 8 ++++++-- packages/gatsby/src/services/types.ts | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/gatsby/src/commands/develop-process.ts b/packages/gatsby/src/commands/develop-process.ts index e50231be574ff..90c05eba28c6a 100644 --- a/packages/gatsby/src/commands/develop-process.ts +++ b/packages/gatsby/src/commands/develop-process.ts @@ -191,9 +191,10 @@ module.exports = async (program: IProgram): Promise => { parentSpan, gatsbyNodeGraphQLFunction, graphqlRunner, + firstRun, }: IBuildContext): IQueryRunningContext => { return { - firstRun: true, + firstRun, program, store, parentSpan, @@ -228,6 +229,9 @@ module.exports = async (program: IProgram): Promise => { await startWebpackServer({ program, app, workerPool }) }, + onDone: { + actions: assign({ firstRun: false }), + }, }, }, }, @@ -254,7 +258,7 @@ module.exports = async (program: IProgram): Promise => { (_, { data }): DataLayerResult => data ), }, - }).withContext({ program, parentSpan: bootstrapSpan, app }) + }).withContext({ program, parentSpan: bootstrapSpan, app, firstRun: true }) ) const isInterpreter = ( diff --git a/packages/gatsby/src/services/types.ts b/packages/gatsby/src/services/types.ts index 3ec2a3766f28a..de7a956452861 100644 --- a/packages/gatsby/src/services/types.ts +++ b/packages/gatsby/src/services/types.ts @@ -16,6 +16,7 @@ export interface IMutationAction { payload: unknown[] } export interface IBuildContext { + firstRun?: boolean program?: IProgram store?: Store parentSpan?: Span From 8987299c937e5dc2ff749c9fdcaf6adf60a3ec72 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Fri, 3 Jul 2020 12:29:34 +0100 Subject: [PATCH 24/44] Track first run --- packages/gatsby/src/commands/develop-process.ts | 8 ++++++-- packages/gatsby/src/services/types.ts | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/gatsby/src/commands/develop-process.ts b/packages/gatsby/src/commands/develop-process.ts index e50231be574ff..484ff75e19d19 100644 --- a/packages/gatsby/src/commands/develop-process.ts +++ b/packages/gatsby/src/commands/develop-process.ts @@ -191,9 +191,10 @@ module.exports = async (program: IProgram): Promise => { parentSpan, gatsbyNodeGraphQLFunction, graphqlRunner, + firstRun, }: IBuildContext): IQueryRunningContext => { return { - firstRun: true, + firstRun, program, store, parentSpan, @@ -228,6 +229,9 @@ module.exports = async (program: IProgram): Promise => { await startWebpackServer({ program, app, workerPool }) }, + onDone: { + actions: assign({ firstRun: false }), + }, }, }, }, @@ -254,7 +258,7 @@ module.exports = async (program: IProgram): Promise => { (_, { data }): DataLayerResult => data ), }, - }).withContext({ program, parentSpan: bootstrapSpan, app }) + }).withContext({ program, parentSpan: bootstrapSpan, app, firstRun: true }) ) const isInterpreter = ( diff --git a/packages/gatsby/src/services/types.ts b/packages/gatsby/src/services/types.ts index 3ec2a3766f28a..de7a956452861 100644 --- a/packages/gatsby/src/services/types.ts +++ b/packages/gatsby/src/services/types.ts @@ -16,6 +16,7 @@ export interface IMutationAction { payload: unknown[] } export interface IBuildContext { + firstRun?: boolean program?: IProgram store?: Store parentSpan?: Span From fa05e0df95e07fc5d8a72e7a93c41f93d73ac24c Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Fri, 3 Jul 2020 13:28:52 +0100 Subject: [PATCH 25/44] Disable --quiet in e2e --- scripts/e2e-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/e2e-test.sh b/scripts/e2e-test.sh index debf03b636a25..24a34154d04d2 100755 --- a/scripts/e2e-test.sh +++ b/scripts/e2e-test.sh @@ -9,7 +9,7 @@ command -v gatsby-dev || command -v sudo && sudo npm install -g gatsby-dev-cli | # setting up child integration test link to gatsby packages cd "$SRC_PATH" && gatsby-dev --set-path-to-repo "$GATSBY_PATH" && -gatsby-dev --scan-once --quiet && # copies _all_ files in gatsby/packages +gatsby-dev --scan-once && # copies _all_ files in gatsby/packages chmod +x ./node_modules/.bin/gatsby && # this is sometimes necessary to ensure executable sh -c "$CUSTOM_COMMAND" && echo "e2e test run succeeded" From 4e8609fdb22c6e17dfc8570bf92727a305df4c69 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Fri, 3 Jul 2020 14:58:51 +0100 Subject: [PATCH 26/44] Don't defer node mutation if we're outside the state machine --- packages/gatsby/src/bootstrap/index.ts | 2 +- packages/gatsby/src/commands/develop-process.ts | 8 ++++++-- packages/gatsby/src/services/create-pages-statefully.ts | 3 ++- packages/gatsby/src/services/source-nodes.ts | 3 ++- packages/gatsby/src/state-machines/data-layer/types.ts | 1 + 5 files changed, 12 insertions(+), 5 deletions(-) diff --git a/packages/gatsby/src/bootstrap/index.ts b/packages/gatsby/src/bootstrap/index.ts index 7cb9632a23c21..c59fb0df1db7b 100644 --- a/packages/gatsby/src/bootstrap/index.ts +++ b/packages/gatsby/src/bootstrap/index.ts @@ -29,7 +29,7 @@ export async function bootstrap( : {} initialContext.parentSpan = tracer.startSpan(`bootstrap`, spanArgs) - + initialContext.firstRun = true const context = { ...initialContext, ...(await initialize(initialContext)), diff --git a/packages/gatsby/src/commands/develop-process.ts b/packages/gatsby/src/commands/develop-process.ts index 0d472c21d36c2..428e9d46648d3 100644 --- a/packages/gatsby/src/commands/develop-process.ts +++ b/packages/gatsby/src/commands/develop-process.ts @@ -149,8 +149,12 @@ module.exports = async (program: IProgram): Promise => { }, invoke: { src: `initializeDataLayer`, - data: ({ parentSpan, store }: IBuildContext): IDataLayerContext => { - return { parentSpan, store, firstRun: true } + data: ({ + parentSpan, + store, + firstRun, + }: IBuildContext): IDataLayerContext => { + return { parentSpan, store, firstRun, deferNodeMutation: true } }, onDone: { actions: `assignServiceResult`, diff --git a/packages/gatsby/src/services/create-pages-statefully.ts b/packages/gatsby/src/services/create-pages-statefully.ts index fe935c4bc19dc..163263006c9cb 100644 --- a/packages/gatsby/src/services/create-pages-statefully.ts +++ b/packages/gatsby/src/services/create-pages-statefully.ts @@ -5,6 +5,7 @@ import { IDataLayerContext } from "../state-machines/data-layer/types" export async function createPagesStatefully({ parentSpan, gatsbyNodeGraphQLFunction, + deferNodeMutation, }: Partial): Promise { // A variant on createPages for plugins that want to // have full control over adding/removing pages. The normal @@ -21,7 +22,7 @@ export async function createPagesStatefully({ traceId: `initial-createPagesStatefully`, waitForCascadingActions: true, parentSpan: activity.span, - deferNodeMutation: true, + deferNodeMutation, }, { activity, diff --git a/packages/gatsby/src/services/source-nodes.ts b/packages/gatsby/src/services/source-nodes.ts index da87f54b3024a..997be37ed864f 100644 --- a/packages/gatsby/src/services/source-nodes.ts +++ b/packages/gatsby/src/services/source-nodes.ts @@ -9,6 +9,7 @@ export async function sourceNodes({ parentSpan, webhookBody, store, + deferNodeMutation, }: Partial): Promise<{ deletedPages: string[] changedPages: string[] @@ -22,7 +23,7 @@ export async function sourceNodes({ const currentPages = new Map(store.getState().pages) await sourceNodesAndRemoveStaleNodes({ parentSpan: activity.span, - deferNodeMutation: true, + deferNodeMutation, webhookBody, }) diff --git a/packages/gatsby/src/state-machines/data-layer/types.ts b/packages/gatsby/src/state-machines/data-layer/types.ts index 6aac97ab71cb0..ecc3940614248 100644 --- a/packages/gatsby/src/state-machines/data-layer/types.ts +++ b/packages/gatsby/src/state-machines/data-layer/types.ts @@ -15,6 +15,7 @@ export interface IMutationAction { payload: unknown[] } export interface IDataLayerContext { + deferNodeMutation?: boolean skipSourcing?: boolean nodesMutatedDuringQueryRun?: boolean firstRun?: boolean From 425eafc2965d8efdaafe4ad7278d2d31c3ce0d0d Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Fri, 3 Jul 2020 14:59:17 +0100 Subject: [PATCH 27/44] Re-quieten e2e --- scripts/e2e-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/e2e-test.sh b/scripts/e2e-test.sh index 24a34154d04d2..debf03b636a25 100755 --- a/scripts/e2e-test.sh +++ b/scripts/e2e-test.sh @@ -9,7 +9,7 @@ command -v gatsby-dev || command -v sudo && sudo npm install -g gatsby-dev-cli | # setting up child integration test link to gatsby packages cd "$SRC_PATH" && gatsby-dev --set-path-to-repo "$GATSBY_PATH" && -gatsby-dev --scan-once && # copies _all_ files in gatsby/packages +gatsby-dev --scan-once --quiet && # copies _all_ files in gatsby/packages chmod +x ./node_modules/.bin/gatsby && # this is sometimes necessary to ensure executable sh -c "$CUSTOM_COMMAND" && echo "e2e test run succeeded" From aadc60be9d2ad9cd3209d326ab1da105e83da6a3 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Mon, 6 Jul 2020 15:03:48 +0100 Subject: [PATCH 28/44] Listen for query file changes --- .../gatsby/src/commands/develop-process.ts | 15 ++++++++++-- packages/gatsby/src/query/index.js | 1 + packages/gatsby/src/query/query-watcher.js | 24 +++++++++---------- .../src/services/calculate-dirty-queries.ts | 12 +++++++--- .../src/services/listen-for-mutations.ts | 6 +++++ packages/gatsby/src/services/types.ts | 1 + packages/gatsby/src/state-machines/actions.ts | 5 ++++ .../state-machines/query-running/actions.ts | 7 +++++- .../src/state-machines/query-running/index.ts | 11 +++++++++ .../shared-transition-configs.ts | 4 ++++ .../src/state-machines/waiting/actions.ts | 15 +++++++++++- .../src/state-machines/waiting/index.ts | 5 ++++ packages/gatsby/src/utils/start-server.ts | 2 -- 13 files changed, 87 insertions(+), 21 deletions(-) diff --git a/packages/gatsby/src/commands/develop-process.ts b/packages/gatsby/src/commands/develop-process.ts index 428e9d46648d3..fe6478f4ddfd3 100644 --- a/packages/gatsby/src/commands/develop-process.ts +++ b/packages/gatsby/src/commands/develop-process.ts @@ -43,6 +43,7 @@ import { IWaitingContext } from "../state-machines/waiting/types" import { ADD_NODE_MUTATION, runMutationAndMarkDirty, + QUERY_FILE_CHANGED, } from "../state-machines/shared-transition-configs" import { buildActions } from "../state-machines/actions" import { waitingMachine } from "../state-machines/waiting" @@ -185,8 +186,12 @@ module.exports = async (program: IProgram): Promise => { runningQueries: { on: { ADD_NODE_MUTATION, + QUERY_FILE_CHANGED: { + actions: forwardTo(`run-queries`), + }, }, invoke: { + id: `run-queries`, src: `runQueries`, data: ({ program, @@ -214,6 +219,7 @@ module.exports = async (program: IProgram): Promise => { }, doingEverythingElse: { on: { + QUERY_FILE_CHANGED, ADD_NODE_MUTATION, }, invoke: { @@ -241,6 +247,7 @@ module.exports = async (program: IProgram): Promise => { }, startingDevServers: { on: { + QUERY_FILE_CHANGED, ADD_NODE_MUTATION, }, invoke: { @@ -256,6 +263,12 @@ module.exports = async (program: IProgram): Promise => { ADD_NODE_MUTATION: { actions: forwardTo(`waiting`), }, + QUERY_FILE_CHANGED: { + actions: forwardTo(`waiting`), + }, + EXTRACT_QUERIES_NOW: { + target: `runningQueries`, + }, }, invoke: { id: `waiting`, @@ -267,8 +280,6 @@ module.exports = async (program: IProgram): Promise => { return { store, nodeMutationBatch } }, onDone: { - // These are mutations added while we were running the last - // batch. We'll hold on to them til we've finished this build. actions: `assignServiceResult`, target: `rebuildingPages`, }, diff --git a/packages/gatsby/src/query/index.js b/packages/gatsby/src/query/index.js index a8c9d7a96773b..2a699896d29c7 100644 --- a/packages/gatsby/src/query/index.js +++ b/packages/gatsby/src/query/index.js @@ -366,6 +366,7 @@ const enqueueExtractedPageComponent = componentPath => { module.exports = { calcInitialDirtyQueryIds, + calcDirtyQueryIds, processPageQueries, processStaticQueries, groupQueryIds, diff --git a/packages/gatsby/src/query/query-watcher.js b/packages/gatsby/src/query/query-watcher.js index d20676a3feea3..bb59fad06f1e5 100644 --- a/packages/gatsby/src/query/query-watcher.js +++ b/packages/gatsby/src/query/query-watcher.js @@ -196,8 +196,7 @@ exports.extractQueries = ({ parentSpan } = {}) => { // During development start watching files to recompile & run // queries on the fly. - // TODO: move this into a spawned service, and emit events rather than - // directly triggering the compilation + // TODO: move this into a spawned service if (process.env.NODE_ENV !== `production`) { watch(store.getState().program.directory) } @@ -222,10 +221,6 @@ const watchComponent = componentPath => { } } -const debounceCompile = _.debounce(() => { - updateStateAndRunQueries() -}, 100) - const watch = async rootDir => { if (watcher) return @@ -238,13 +233,18 @@ const watch = async rootDir => { }) watcher = chokidar - .watch([ - slash(path.join(rootDir, `/src/**/*.{js,jsx,ts,tsx}`)), - ...packagePaths, - ]) + .watch( + [slash(path.join(rootDir, `/src/**/*.{js,jsx,ts,tsx}`)), ...packagePaths], + { ignoreInitial: true } + ) .on(`change`, path => { - report.pendingActivity({ id: `query-extraction` }) - debounceCompile() + emitter.emit(`QUERY_FILE_CHANGED`, path) + }) + .on(`add`, path => { + emitter.emit(`QUERY_FILE_CHANGED`, path) + }) + .on(`unlink`, path => { + emitter.emit(`QUERY_FILE_CHANGED`, path) }) filesToWatch.forEach(filePath => watcher.add(filePath)) diff --git a/packages/gatsby/src/services/calculate-dirty-queries.ts b/packages/gatsby/src/services/calculate-dirty-queries.ts index adf3bd7501bb5..4309074c0fab4 100644 --- a/packages/gatsby/src/services/calculate-dirty-queries.ts +++ b/packages/gatsby/src/services/calculate-dirty-queries.ts @@ -1,18 +1,24 @@ -import { calcInitialDirtyQueryIds, groupQueryIds } from "../query" +import { + calcInitialDirtyQueryIds, + calcDirtyQueryIds, + groupQueryIds, +} from "../query" import { IGroupedQueryIds } from "./" import { IQueryRunningContext } from "../state-machines/query-running/types" import { assertStore } from "../utils/assert-store" export async function calculateDirtyQueries({ store, + firstRun, }: Partial): Promise<{ queryIds: IGroupedQueryIds }> { assertStore(store) const state = store.getState() - // TODO: Check filesDirty from context - const queryIds = calcInitialDirtyQueryIds(state) + const queryIds = firstRun + ? calcInitialDirtyQueryIds(state) + : calcDirtyQueryIds(state) return { queryIds: groupQueryIds(queryIds) } } diff --git a/packages/gatsby/src/services/listen-for-mutations.ts b/packages/gatsby/src/services/listen-for-mutations.ts index 3dac1d7e08613..f5a54b084a72a 100644 --- a/packages/gatsby/src/services/listen-for-mutations.ts +++ b/packages/gatsby/src/services/listen-for-mutations.ts @@ -11,12 +11,18 @@ export const listenForMutations: InvokeCallback = (callback: Sender) => { callback({ type: `SOURCE_FILE_CHANGED`, payload: event }) } + const emitQueryChange = (event: unknown): void => { + callback({ type: `QUERY_FILE_CHANGED`, payload: event }) + } + emitter.on(`ENQUEUE_NODE_MUTATION`, emitMutation) emitter.on(`SOURCE_FILE_CHANGED`, emitFileChange) + emitter.on(`QUERY_FILE_CHANGED`, emitQueryChange) return (): void => { emitter.off(`ENQUEUE_NODE_MUTATION`, emitMutation) emitter.off(`SOURCE_FILE_CHANGED`, emitFileChange) + emitter.off(`QUERY_FILE_CHANGED`, emitQueryChange) } } diff --git a/packages/gatsby/src/services/types.ts b/packages/gatsby/src/services/types.ts index f373bba5959b7..5d6bd9a39cc0f 100644 --- a/packages/gatsby/src/services/types.ts +++ b/packages/gatsby/src/services/types.ts @@ -37,4 +37,5 @@ export interface IBuildContext { compiler?: Compiler websocketManager?: WebsocketManager webpackWatching?: IWebpackWatchingPauseResume + queryFilesDirty?: boolean } diff --git a/packages/gatsby/src/state-machines/actions.ts b/packages/gatsby/src/state-machines/actions.ts index 57cf17721a4a9..66da5a8ed1730 100644 --- a/packages/gatsby/src/state-machines/actions.ts +++ b/packages/gatsby/src/state-machines/actions.ts @@ -55,6 +55,10 @@ export const assignStoreAndWorkerPool = assign( } ) +export const markQueryFilesDirty = assign({ + queryFilesDirty: true, +}) + export const assignServiceResult = assign( (_context, { data }): DataLayerResult => data ) @@ -92,4 +96,5 @@ export const buildActions: ActionFunctionMap = { assignStoreAndWorkerPool, assignServiceResult, assignServers, + markQueryFilesDirty, } diff --git a/packages/gatsby/src/state-machines/query-running/actions.ts b/packages/gatsby/src/state-machines/query-running/actions.ts index c329e95ea8356..93d3ebfae03f7 100644 --- a/packages/gatsby/src/state-machines/query-running/actions.ts +++ b/packages/gatsby/src/state-machines/query-running/actions.ts @@ -8,13 +8,17 @@ export const flushPageData = (): void => { enqueueFlush() } +export const markFilesClean = assign< + IQueryRunningContext, + DoneInvokeEvent +>({ filesDirty: false }) + export const assignDirtyQueries = assign< IQueryRunningContext, DoneInvokeEvent >((_context, { data }) => { const { queryIds } = data return { - filesDirty: false, queryIds, } }) @@ -39,4 +43,5 @@ export const queryActions: ActionFunctionMap< resetGraphQLRunner, assignDirtyQueries, flushPageData, + markFilesClean, } diff --git a/packages/gatsby/src/state-machines/query-running/index.ts b/packages/gatsby/src/state-machines/query-running/index.ts index 39161dac73304..7df9e6ddb0f0b 100644 --- a/packages/gatsby/src/state-machines/query-running/index.ts +++ b/packages/gatsby/src/state-machines/query-running/index.ts @@ -3,11 +3,18 @@ import { IQueryRunningContext } from "./types" import { queryRunningServices } from "./services" import { queryActions } from "./actions" +const extractQueriesIfFilesAreDirty = { + cond: (ctx): boolean => !!ctx.filesDirty, + target: `extractingQueries`, +} + export const queryStates: MachineConfig = { initial: `extractingQueries`, + context: {}, states: { extractingQueries: { id: `extracting-queries`, + onEntry: `markFilesClean`, invoke: { id: `extracting-queries`, src: `extractQueries`, @@ -20,6 +27,7 @@ export const queryStates: MachineConfig = { }, }, writingRequires: { + always: extractQueriesIfFilesAreDirty, invoke: { src: `writeOutRequires`, id: `writing-requires`, @@ -29,6 +37,7 @@ export const queryStates: MachineConfig = { }, }, calculatingDirtyQueries: { + always: extractQueriesIfFilesAreDirty, invoke: { id: `calculating-dirty-queries`, src: `calculateDirtyQueries`, @@ -39,6 +48,7 @@ export const queryStates: MachineConfig = { }, }, runningStaticQueries: { + always: extractQueriesIfFilesAreDirty, invoke: { src: `runStaticQueries`, id: `running-static-queries`, @@ -48,6 +58,7 @@ export const queryStates: MachineConfig = { }, }, runningPageQueries: { + always: extractQueriesIfFilesAreDirty, invoke: { src: `runPageQueries`, id: `running-page-queries`, diff --git a/packages/gatsby/src/state-machines/shared-transition-configs.ts b/packages/gatsby/src/state-machines/shared-transition-configs.ts index 6daccc840793e..26f5dfd4d3386 100644 --- a/packages/gatsby/src/state-machines/shared-transition-configs.ts +++ b/packages/gatsby/src/state-machines/shared-transition-configs.ts @@ -6,6 +6,10 @@ export const ADD_NODE_MUTATION = { actions: `addNodeMutation`, } +export const QUERY_FILE_CHANGED = { + actions: `markQueryFilesDirty`, +} + /** * Event handler used in all states where we're not ready to process a file change * Instead we add it to a batch to process when we're next idle diff --git a/packages/gatsby/src/state-machines/waiting/actions.ts b/packages/gatsby/src/state-machines/waiting/actions.ts index b1e01d67c68ab..25161051a12d8 100644 --- a/packages/gatsby/src/state-machines/waiting/actions.ts +++ b/packages/gatsby/src/state-machines/waiting/actions.ts @@ -1,4 +1,12 @@ -import { AssignAction, assign, ActionFunctionMap } from "xstate" +import { + AssignAction, + assign, + ActionFunctionMap, + ActionFunction, + sendParent, + InvokeCreator, + AnyEventObject, +} from "xstate" import { IWaitingContext } from "./types" import { AnyAction } from "redux" @@ -16,6 +24,11 @@ export const addNodeMutation: AssignAction = assign( } ) +export const extractQueries = sendParent( + `EXTRACT_QUERIES_NOW` +) + export const waitingActions: ActionFunctionMap = { addNodeMutation, + extractQueries, } diff --git a/packages/gatsby/src/state-machines/waiting/index.ts b/packages/gatsby/src/state-machines/waiting/index.ts index 2ae41b4d7e92c..4b6dd2b4b34dd 100644 --- a/packages/gatsby/src/state-machines/waiting/index.ts +++ b/packages/gatsby/src/state-machines/waiting/index.ts @@ -25,6 +25,11 @@ export const waitingStates: MachineConfig = { actions: `addNodeMutation`, target: `batchingNodeMutations`, }, + // We only listen for this when idling because if we receive it at any + // other point we're already going to create pages etc + QUERY_FILE_CHANGED: { + actions: `extractQueries`, + }, }, }, diff --git a/packages/gatsby/src/utils/start-server.ts b/packages/gatsby/src/utils/start-server.ts index 6c71e2f947d4e..25d69819fd42d 100644 --- a/packages/gatsby/src/utils/start-server.ts +++ b/packages/gatsby/src/utils/start-server.ts @@ -194,7 +194,6 @@ export async function startServer( **/ const REFRESH_ENDPOINT = `/__refresh` const refresh = async (req: express.Request): Promise => { - stopSchemaHotReloader() let activity = report.activityTimer(`createSchemaCustomization`, {}) activity.start() await createSchemaCustomization({ @@ -211,7 +210,6 @@ export async function startServer( activity.start() await rebuildSchema({ parentSpan: activity }) activity.end() - startSchemaHotReloader() } app.use(REFRESH_ENDPOINT, express.json()) app.post(REFRESH_ENDPOINT, (req, res) => { From 33dc4fa2478a78a80bfa549edb3f1ce77004094b Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Mon, 6 Jul 2020 15:17:17 +0100 Subject: [PATCH 29/44] Lint --- packages/gatsby/src/query/query-watcher.js | 1 - packages/gatsby/src/state-machines/waiting/actions.ts | 2 -- packages/gatsby/src/utils/start-server.ts | 5 ----- 3 files changed, 8 deletions(-) diff --git a/packages/gatsby/src/query/query-watcher.js b/packages/gatsby/src/query/query-watcher.js index bb59fad06f1e5..a3c33eca6b0d4 100644 --- a/packages/gatsby/src/query/query-watcher.js +++ b/packages/gatsby/src/query/query-watcher.js @@ -8,7 +8,6 @@ * - Whenever a query changes, re-run all pages that rely on this query. ***/ -const _ = require(`lodash`) const chokidar = require(`chokidar`) const path = require(`path`) diff --git a/packages/gatsby/src/state-machines/waiting/actions.ts b/packages/gatsby/src/state-machines/waiting/actions.ts index 25161051a12d8..c5e4039705788 100644 --- a/packages/gatsby/src/state-machines/waiting/actions.ts +++ b/packages/gatsby/src/state-machines/waiting/actions.ts @@ -2,9 +2,7 @@ import { AssignAction, assign, ActionFunctionMap, - ActionFunction, sendParent, - InvokeCreator, AnyEventObject, } from "xstate" import { IWaitingContext } from "./types" diff --git a/packages/gatsby/src/utils/start-server.ts b/packages/gatsby/src/utils/start-server.ts index 25d69819fd42d..633875af3ae21 100644 --- a/packages/gatsby/src/utils/start-server.ts +++ b/packages/gatsby/src/utils/start-server.ts @@ -34,11 +34,6 @@ import { Express } from "express" import { Stage, IProgram } from "../commands/types" import JestWorker from "jest-worker" -import { - startSchemaHotReloader, - stopSchemaHotReloader, -} from "../bootstrap/schema-hot-reloader" - import sourceNodes from "../utils/source-nodes" import { createSchemaCustomization } from "../utils/create-schema-customization" import { rebuild as rebuildSchema } from "../schema" From 50faf938c1287cc31f9ff92dd96677df1bc3e846 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Mon, 6 Jul 2020 16:45:46 +0100 Subject: [PATCH 30/44] Handle webhook --- .../gatsby/src/commands/develop-process.ts | 15 +++++++++++-- .../src/services/listen-for-mutations.ts | 8 +++++-- packages/gatsby/src/state-machines/actions.ts | 13 ++++++++++++ .../shared-transition-configs.ts | 5 +++++ packages/gatsby/src/utils/start-server.ts | 21 ++----------------- 5 files changed, 39 insertions(+), 23 deletions(-) diff --git a/packages/gatsby/src/commands/develop-process.ts b/packages/gatsby/src/commands/develop-process.ts index fe6478f4ddfd3..2c83e9df8235b 100644 --- a/packages/gatsby/src/commands/develop-process.ts +++ b/packages/gatsby/src/commands/develop-process.ts @@ -44,6 +44,7 @@ import { ADD_NODE_MUTATION, runMutationAndMarkDirty, QUERY_FILE_CHANGED, + WEBHOOK_RECEIVED, } from "../state-machines/shared-transition-configs" import { buildActions } from "../state-machines/actions" import { waitingMachine } from "../state-machines/waiting" @@ -154,11 +155,18 @@ module.exports = async (program: IProgram): Promise => { parentSpan, store, firstRun, + webhookBody, }: IBuildContext): IDataLayerContext => { - return { parentSpan, store, firstRun, deferNodeMutation: true } + return { + parentSpan, + store, + firstRun, + deferNodeMutation: true, + webhookBody, + } }, onDone: { - actions: `assignServiceResult`, + actions: [`assignServiceResult`, `clearWebhookBody`], target: `finishingBootstrap`, }, }, @@ -166,6 +174,7 @@ module.exports = async (program: IProgram): Promise => { finishingBootstrap: { on: { ADD_NODE_MUTATION: runMutationAndMarkDirty, + WEBHOOK_RECEIVED, }, invoke: { src: async (): Promise => { @@ -221,6 +230,7 @@ module.exports = async (program: IProgram): Promise => { on: { QUERY_FILE_CHANGED, ADD_NODE_MUTATION, + WEBHOOK_RECEIVED, }, invoke: { src: async (): Promise => { @@ -260,6 +270,7 @@ module.exports = async (program: IProgram): Promise => { }, waiting: { on: { + WEBHOOK_RECEIVED, ADD_NODE_MUTATION: { actions: forwardTo(`waiting`), }, diff --git a/packages/gatsby/src/services/listen-for-mutations.ts b/packages/gatsby/src/services/listen-for-mutations.ts index f5a54b084a72a..a833a276d94b9 100644 --- a/packages/gatsby/src/services/listen-for-mutations.ts +++ b/packages/gatsby/src/services/listen-for-mutations.ts @@ -7,7 +7,6 @@ export const listenForMutations: InvokeCallback = (callback: Sender) => { } const emitFileChange = (event: unknown): void => { - console.log({ event }) callback({ type: `SOURCE_FILE_CHANGED`, payload: event }) } @@ -15,14 +14,19 @@ export const listenForMutations: InvokeCallback = (callback: Sender) => { callback({ type: `QUERY_FILE_CHANGED`, payload: event }) } - emitter.on(`ENQUEUE_NODE_MUTATION`, emitMutation) + const emitWebhook = (event: unknown): void => { + callback({ type: `WEBHOOK_RECEIVED`, payload: event }) + } + emitter.on(`ENQUEUE_NODE_MUTATION`, emitMutation) + emitter.on(`WEBHOOK_RECEIVED`, emitWebhook) emitter.on(`SOURCE_FILE_CHANGED`, emitFileChange) emitter.on(`QUERY_FILE_CHANGED`, emitQueryChange) return (): void => { emitter.off(`ENQUEUE_NODE_MUTATION`, emitMutation) emitter.off(`SOURCE_FILE_CHANGED`, emitFileChange) + emitter.off(`WEBHOOK_RECEIVED`, emitWebhook) emitter.off(`QUERY_FILE_CHANGED`, emitQueryChange) } } diff --git a/packages/gatsby/src/state-machines/actions.ts b/packages/gatsby/src/state-machines/actions.ts index 66da5a8ed1730..116d3c0cb3ccc 100644 --- a/packages/gatsby/src/state-machines/actions.ts +++ b/packages/gatsby/src/state-machines/actions.ts @@ -76,6 +76,17 @@ export const assignServers = assign( } ) +export const assignWebhookBody = assign({ + webhookBody: (_context, { payload }) => { + console.log(`body`, payload?.webhookBody) + return payload?.webhookBody + }, +}) + +export const clearWebhookBody = assign({ + webhookBody: undefined, +}) + /** * Event handler used in all states where we're not ready to process a file change * Instead we add it to a batch to process when we're next idle @@ -97,4 +108,6 @@ export const buildActions: ActionFunctionMap = { assignServiceResult, assignServers, markQueryFilesDirty, + assignWebhookBody, + clearWebhookBody, } diff --git a/packages/gatsby/src/state-machines/shared-transition-configs.ts b/packages/gatsby/src/state-machines/shared-transition-configs.ts index 26f5dfd4d3386..4106307b0b366 100644 --- a/packages/gatsby/src/state-machines/shared-transition-configs.ts +++ b/packages/gatsby/src/state-machines/shared-transition-configs.ts @@ -10,6 +10,11 @@ export const QUERY_FILE_CHANGED = { actions: `markQueryFilesDirty`, } +export const WEBHOOK_RECEIVED = { + target: `initializingDataLayer`, + actions: `assignWebhookBody`, +} + /** * Event handler used in all states where we're not ready to process a file change * Instead we add it to a batch to process when we're next idle diff --git a/packages/gatsby/src/utils/start-server.ts b/packages/gatsby/src/utils/start-server.ts index 633875af3ae21..d1d2be626d2de 100644 --- a/packages/gatsby/src/utils/start-server.ts +++ b/packages/gatsby/src/utils/start-server.ts @@ -13,7 +13,7 @@ import graphiqlExplorer from "gatsby-graphiql-explorer" import { formatError } from "graphql" import webpackConfig from "../utils/webpack.config" -import { store } from "../redux" +import { store, emitter } from "../redux" import { buildHTML } from "../commands/build-html" import { withBasePath } from "../utils/path" import report from "gatsby-cli/lib/reporter" @@ -34,9 +34,6 @@ import { Express } from "express" import { Stage, IProgram } from "../commands/types" import JestWorker from "jest-worker" -import sourceNodes from "../utils/source-nodes" -import { createSchemaCustomization } from "../utils/create-schema-customization" -import { rebuild as rebuildSchema } from "../schema" type ActivityTracker = any // TODO: Replace this with proper type once reporter is typed interface IServer { @@ -182,29 +179,15 @@ export async function startServer( ) /** - * This will be removed in state machine * Refresh external data sources. * This behavior is disabled by default, but the ENABLE_GATSBY_REFRESH_ENDPOINT env var enables it * If no GATSBY_REFRESH_TOKEN env var is available, then no Authorization header is required **/ const REFRESH_ENDPOINT = `/__refresh` const refresh = async (req: express.Request): Promise => { - let activity = report.activityTimer(`createSchemaCustomization`, {}) - activity.start() - await createSchemaCustomization({ - refresh: true, - }) - activity.end() - activity = report.activityTimer(`Refreshing source data`, {}) - activity.start() - await sourceNodes({ + emitter.emit(`WEBHOOK_RECEIVED`, { webhookBody: req.body, }) - activity.end() - activity = report.activityTimer(`rebuild schema`) - activity.start() - await rebuildSchema({ parentSpan: activity }) - activity.end() } app.use(REFRESH_ENDPOINT, express.json()) app.post(REFRESH_ENDPOINT, (req, res) => { From 059798b744f7fc48db3a4cb66c4bc1cc096bb9b6 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Tue, 7 Jul 2020 09:33:11 +0100 Subject: [PATCH 31/44] Changes from review --- packages/gatsby/src/bootstrap/index.ts | 16 ++++++++---- .../gatsby/src/commands/develop-process.ts | 26 +++++++++++++++---- packages/gatsby/src/services/types.ts | 2 +- 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/packages/gatsby/src/bootstrap/index.ts b/packages/gatsby/src/bootstrap/index.ts index 7cb9632a23c21..0bb37c8b7ab9c 100644 --- a/packages/gatsby/src/bootstrap/index.ts +++ b/packages/gatsby/src/bootstrap/index.ts @@ -19,7 +19,7 @@ import JestWorker from "jest-worker" const tracer = globalTracer() export async function bootstrap( - initialContext: IBuildContext + initialContext: Partial ): Promise<{ gatsbyNodeGraphQLFunction: Runner workerPool: JestWorker @@ -28,11 +28,17 @@ export async function bootstrap( ? { childOf: initialContext.parentSpan } : {} - initialContext.parentSpan = tracer.startSpan(`bootstrap`, spanArgs) + const parentSpan = tracer.startSpan(`bootstrap`, spanArgs) - const context = { + const bootstrapContext: IBuildContext = { ...initialContext, - ...(await initialize(initialContext)), + parentSpan, + firstRun: true, + } + + const context = { + ...bootstrapContext, + ...(await initialize(bootstrapContext)), } await customizeSchema(context) @@ -59,7 +65,7 @@ export async function bootstrap( await postBootstrap(context) - initialContext.parentSpan.finish() + parentSpan.finish() return { gatsbyNodeGraphQLFunction: context.gatsbyNodeGraphQLFunction, diff --git a/packages/gatsby/src/commands/develop-process.ts b/packages/gatsby/src/commands/develop-process.ts index 484ff75e19d19..d7e6212c23bc6 100644 --- a/packages/gatsby/src/commands/develop-process.ts +++ b/packages/gatsby/src/commands/develop-process.ts @@ -1,5 +1,5 @@ import { syncStaticDir } from "../utils/get-static-dir" -import report from "gatsby-cli/lib/reporter" +import reporter from "gatsby-cli/lib/reporter" import chalk from "chalk" import telemetry from "gatsby-telemetry" import express from "express" @@ -41,6 +41,7 @@ import { interpret, Actor, Interpreter, + State, } from "xstate" import { DataLayerResult, dataLayerMachine } from "../state-machines/data-layer" import { IDataLayerContext } from "../state-machines/data-layer/types" @@ -100,7 +101,7 @@ module.exports = async (program: IProgram): Promise => { ) if (process.env.GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES) { - report.panic( + reporter.panic( `The flag ${chalk.yellow( `GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES` )} is not available with ${chalk.cyan( @@ -110,7 +111,7 @@ module.exports = async (program: IProgram): Promise => { } initTracer(program.openTracingConfigFile) markWebpackStatusAsPending() - report.pendingActivity({ id: `webpack-develop` }) + reporter.pendingActivity({ id: `webpack-develop` }) telemetry.trackCli(`DEVELOP_START`) telemetry.startBackgroundUpdate() @@ -266,8 +267,16 @@ module.exports = async (program: IProgram): Promise => { ): actor is Interpreter => `machine` in actor const listeners = new WeakSet() + let last: State + service.onTransition(state => { - report.verbose(`Transition to ${JSON.stringify(state.value)}`) + if (!last) { + last = state + } else if (!state.changed || last.matches(state)) { + return + } + last = state + reporter.verbose(`Transition to ${JSON.stringify(state.value)}`) // eslint-disable-next-line no-unused-expressions service.children?.forEach(child => { // We want to ensure we don't attach a listener to the same @@ -275,8 +284,15 @@ module.exports = async (program: IProgram): Promise => { // because xstate handles that for us when the actor is stopped. if (isInterpreter(child) && !listeners.has(child)) { + let sublast = child.state child.onTransition(substate => { - report.verbose( + if (!sublast) { + sublast = substate + } else if (!substate.changed || sublast.matches(substate)) { + return + } + sublast = substate + reporter.verbose( `Transition to ${JSON.stringify(state.value)} > ${JSON.stringify( substate.value )}` diff --git a/packages/gatsby/src/services/types.ts b/packages/gatsby/src/services/types.ts index de7a956452861..1b961b5479370 100644 --- a/packages/gatsby/src/services/types.ts +++ b/packages/gatsby/src/services/types.ts @@ -16,7 +16,7 @@ export interface IMutationAction { payload: unknown[] } export interface IBuildContext { - firstRun?: boolean + firstRun: boolean program?: IProgram store?: Store parentSpan?: Span From e9bf25996f4b61e77f1efe8c1a27a0334150b25b Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Tue, 7 Jul 2020 10:31:24 +0100 Subject: [PATCH 32/44] Fix typings --- packages/gatsby/src/services/types.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gatsby/src/services/types.ts b/packages/gatsby/src/services/types.ts index b5013de60ba63..5d6bd9a39cc0f 100644 --- a/packages/gatsby/src/services/types.ts +++ b/packages/gatsby/src/services/types.ts @@ -20,7 +20,7 @@ export interface IMutationAction { payload: unknown[] } export interface IBuildContext { - firstRun: boolean + firstRun?: boolean program?: IProgram store?: Store parentSpan?: Span From e514a76127940bcfffa2a70a2f4922e4598fd381 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Tue, 7 Jul 2020 15:37:23 +0100 Subject: [PATCH 33/44] Changes from review --- .../gatsby/src/commands/develop-process.ts | 42 +++++++++++-------- packages/gatsby/src/services/create-pages.ts | 6 +-- .../src/services/listen-for-mutations.ts | 2 +- .../gatsby/src/services/run-mutation-batch.ts | 14 +++---- packages/gatsby/src/services/source-nodes.ts | 9 ++-- packages/gatsby/src/state-machines/actions.ts | 18 ++++---- .../src/state-machines/data-layer/actions.ts | 19 ++++++--- .../src/state-machines/data-layer/index.ts | 2 +- .../src/state-machines/query-running/index.ts | 9 ++-- .../src/state-machines/waiting/index.ts | 25 +++++------ .../src/state-machines/waiting/types.ts | 4 +- packages/gatsby/src/utils/api-runner-node.js | 12 +++--- packages/gatsby/src/utils/changed-pages.ts | 10 ++--- packages/gatsby/src/utils/source-nodes.ts | 6 +-- 14 files changed, 90 insertions(+), 88 deletions(-) diff --git a/packages/gatsby/src/commands/develop-process.ts b/packages/gatsby/src/commands/develop-process.ts index 11c98d695c333..2f552994aff0a 100644 --- a/packages/gatsby/src/commands/develop-process.ts +++ b/packages/gatsby/src/commands/develop-process.ts @@ -43,8 +43,6 @@ import { IWaitingContext } from "../state-machines/waiting/types" import { ADD_NODE_MUTATION, runMutationAndMarkDirty, - QUERY_FILE_CHANGED, - WEBHOOK_RECEIVED, } from "../state-machines/shared-transition-configs" import { buildActions } from "../state-machines/actions" import { waitingMachine } from "../state-machines/waiting" @@ -136,6 +134,12 @@ module.exports = async (program: IProgram): Promise => { initial: `initializing`, states: { initializing: { + on: { + // Ignore mutation events because we'll be running everything anyway + ADD_NODE_MUTATION: undefined, + QUERY_FILE_CHANGED: undefined, + WEBHOOK_RECEIVED: undefined, + }, invoke: { src: `initialize`, onDone: { @@ -147,6 +151,8 @@ module.exports = async (program: IProgram): Promise => { initializingDataLayer: { on: { ADD_NODE_MUTATION: runMutationAndMarkDirty, + // Ignore, because we're about to extract them anyway + QUERY_FILE_CHANGED: undefined, }, invoke: { src: `initializeDataLayer`, @@ -173,7 +179,8 @@ module.exports = async (program: IProgram): Promise => { finishingBootstrap: { on: { ADD_NODE_MUTATION: runMutationAndMarkDirty, - WEBHOOK_RECEIVED, + // Ignore, because we're about to extract them anyway + QUERY_FILE_CHANGED: undefined, }, invoke: { src: async (): Promise => { @@ -193,7 +200,6 @@ module.exports = async (program: IProgram): Promise => { }, runningQueries: { on: { - ADD_NODE_MUTATION, QUERY_FILE_CHANGED: { actions: forwardTo(`run-queries`), }, @@ -226,11 +232,6 @@ module.exports = async (program: IProgram): Promise => { }, }, doingEverythingElse: { - on: { - QUERY_FILE_CHANGED, - ADD_NODE_MUTATION, - WEBHOOK_RECEIVED, - }, invoke: { src: async (): Promise => { // All the stuff that's not in the state machine yet @@ -255,10 +256,6 @@ module.exports = async (program: IProgram): Promise => { }, }, startingDevServers: { - on: { - QUERY_FILE_CHANGED, - ADD_NODE_MUTATION, - }, invoke: { src: `startWebpackServer`, onDone: { @@ -269,7 +266,6 @@ module.exports = async (program: IProgram): Promise => { }, waiting: { on: { - WEBHOOK_RECEIVED, ADD_NODE_MUTATION: { actions: forwardTo(`waiting`), }, @@ -287,7 +283,7 @@ module.exports = async (program: IProgram): Promise => { store, nodeMutationBatch = [], }: IBuildContext): IWaitingContext => { - return { store, nodeMutationBatch } + return { store, nodeMutationBatch, runningBatch: [] } }, onDone: { actions: `assignServiceResult`, @@ -296,9 +292,6 @@ module.exports = async (program: IProgram): Promise => { }, }, rebuildingPages: { - on: { - ADD_NODE_MUTATION, - }, invoke: { src: `initializeDataLayer`, data: ({ parentSpan, store }: IBuildContext): IDataLayerContext => { @@ -311,6 +304,19 @@ module.exports = async (program: IProgram): Promise => { }, }, }, + // Transitions shared by all states, except where overridden + on: { + ADD_NODE_MUTATION: { + actions: `addNodeMutation`, + }, + QUERY_FILE_CHANGED: { + actions: `markQueryFilesDirty`, + }, + WEBHOOK_RECEIVED: { + target: `initializingDataLayer`, + actions: `assignWebhookBody`, + }, + }, } const service = interpret( diff --git a/packages/gatsby/src/services/create-pages.ts b/packages/gatsby/src/services/create-pages.ts index 7b5f9a10a42b0..343b734cdd120 100644 --- a/packages/gatsby/src/services/create-pages.ts +++ b/packages/gatsby/src/services/create-pages.ts @@ -41,11 +41,11 @@ export async function createPages({ ) activity.end() - reporter.info(`Checking for deleted pages`) + reporter.verbose(`Checking for deleted pages`) const deletedPages = deleteUntouchedPages(store.getState().pages, timestamp) - reporter.info( + reporter.verbose( `Deleted ${deletedPages.length} page${deletedPages.length === 1 ? `` : `s`}` ) @@ -57,7 +57,7 @@ export async function createPages({ store.getState().pages ) - reporter.info( + reporter.verbose( `Found ${changedPages.length} changed page${ changedPages.length === 1 ? `` : `s` }` diff --git a/packages/gatsby/src/services/listen-for-mutations.ts b/packages/gatsby/src/services/listen-for-mutations.ts index a833a276d94b9..dc21cf75edb95 100644 --- a/packages/gatsby/src/services/listen-for-mutations.ts +++ b/packages/gatsby/src/services/listen-for-mutations.ts @@ -23,7 +23,7 @@ export const listenForMutations: InvokeCallback = (callback: Sender) => { emitter.on(`SOURCE_FILE_CHANGED`, emitFileChange) emitter.on(`QUERY_FILE_CHANGED`, emitQueryChange) - return (): void => { + return function unsubscribeFromMutationListening(): void { emitter.off(`ENQUEUE_NODE_MUTATION`, emitMutation) emitter.off(`SOURCE_FILE_CHANGED`, emitFileChange) emitter.off(`WEBHOOK_RECEIVED`, emitWebhook) diff --git a/packages/gatsby/src/services/run-mutation-batch.ts b/packages/gatsby/src/services/run-mutation-batch.ts index b0f93a805e053..51b36f481008a 100644 --- a/packages/gatsby/src/services/run-mutation-batch.ts +++ b/packages/gatsby/src/services/run-mutation-batch.ts @@ -5,10 +5,10 @@ import { IWaitingContext } from "../state-machines/waiting/types" import { assertStore } from "../utils/assert-store" import { actions } from "../redux/actions" -const callRealApi = async ( +const callRealApi = ( event: IMutationAction, store?: Store -): Promise => { +): void => { assertStore(store) const { type, payload } = event if (type in actions) { @@ -16,11 +16,9 @@ const callRealApi = async ( } } -export const runMutationBatch = ({ +// Consume the entire batch and run actions +export const runMutationBatch = async ({ runningBatch = [], store, -}: Partial): Promise => - // Consume the entire batch and run actions - Promise.all(runningBatch.map(payload => callRealApi(payload, store))).then( - () => void 0 - ) +}: Partial): Promise => + Promise.all(runningBatch.map(payload => callRealApi(payload, store))) diff --git a/packages/gatsby/src/services/source-nodes.ts b/packages/gatsby/src/services/source-nodes.ts index 997be37ed864f..e14498d33537e 100644 --- a/packages/gatsby/src/services/source-nodes.ts +++ b/packages/gatsby/src/services/source-nodes.ts @@ -9,7 +9,7 @@ export async function sourceNodes({ parentSpan, webhookBody, store, - deferNodeMutation, + deferNodeMutation = false, }: Partial): Promise<{ deletedPages: string[] changedPages: string[] @@ -35,9 +35,8 @@ export async function sourceNodes({ .join(`, `)}]` ) - reporter.info(`Checking for deleted pages`) + reporter.verbose(`Checking for deleted pages`) - // Add this back when we enable page creation outside of onCreatePages const tim = reporter.activityTimer(`Checking for changed pages`) tim.start() @@ -46,11 +45,11 @@ export async function sourceNodes({ store.getState().pages ) - reporter.info( + reporter.verbose( `Deleted ${deletedPages.length} page${deletedPages.length === 1 ? `` : `s`}` ) - reporter.info( + reporter.verbose( `Found ${changedPages.length} changed page${ changedPages.length === 1 ? `` : `s` }` diff --git a/packages/gatsby/src/state-machines/actions.ts b/packages/gatsby/src/state-machines/actions.ts index 116d3c0cb3ccc..4accf988745f8 100644 --- a/packages/gatsby/src/state-machines/actions.ts +++ b/packages/gatsby/src/state-machines/actions.ts @@ -12,26 +12,25 @@ import { actions } from "../redux/actions" import { listenForMutations } from "../services/listen-for-mutations" import { DataLayerResult } from "./data-layer" import { assertStore } from "../utils/assert-store" +import reporter from "gatsby-cli/lib/reporter" -export const callRealApi = async ( - event: IMutationAction, - store?: Store -): Promise => { +export const callRealApi = (event: IMutationAction, store?: Store): void => { assertStore(store) const { type, payload } = event if (type in actions) { store.dispatch(actions[type](...payload)) + } else { + reporter.log(`Could not dispatch unknown action "${type}`) } - return null } /** * Handler for when we're inside handlers that should be able to mutate nodes */ -export const callApi: ActionFunction = async ( +export const callApi: ActionFunction = ( { store }, event -): Promise => callRealApi(event.payload, store) +) => callRealApi(event.payload, store) /** * Event handler used in all states where we're not ready to process node @@ -77,10 +76,7 @@ export const assignServers = assign( ) export const assignWebhookBody = assign({ - webhookBody: (_context, { payload }) => { - console.log(`body`, payload?.webhookBody) - return payload?.webhookBody - }, + webhookBody: (_context, { payload }) => payload?.webhookBody, }) export const clearWebhookBody = assign({ diff --git a/packages/gatsby/src/state-machines/data-layer/actions.ts b/packages/gatsby/src/state-machines/data-layer/actions.ts index 810b5503ad9b4..6ff1e2491da3e 100644 --- a/packages/gatsby/src/state-machines/data-layer/actions.ts +++ b/packages/gatsby/src/state-machines/data-layer/actions.ts @@ -4,6 +4,7 @@ import reporter from "gatsby-cli/lib/reporter" import { IDataLayerContext } from "./types" import { callApi, markNodesDirty } from "../actions" import { assertStore } from "../../utils/assert-store" +import { GraphQLRunner } from "../../query/graphql-runner" const concatUnique = (array1: T[] = [], array2: T[] = []): T[] => Array.from(new Set(array1.concat(array2))) @@ -21,16 +22,22 @@ export const assignChangedPages = assign< } }) -export const assignGatsbyNodeGraphQL = assign({ - gatsbyNodeGraphQLFunction: ({ store }: IDataLayerContext) => { +export const assignGraphQLRunners = assign( + ({ store, program }) => { assertStore(store) - return createGraphQLRunner(store, reporter) - }, -}) + return { + gatsbyNodeGraphQLFunction: createGraphQLRunner(store, reporter), + graphqlRunner: new GraphQLRunner(store, { + collectStats: true, + graphqlTracing: program?.graphqlTracing, + }), + } + } +) export const dataLayerActions: ActionFunctionMap = { assignChangedPages, - assignGatsbyNodeGraphQL, + assignGraphQLRunners, callApi, markNodesDirty, } diff --git a/packages/gatsby/src/state-machines/data-layer/index.ts b/packages/gatsby/src/state-machines/data-layer/index.ts index adfb28a704bcf..56f5de475c4f6 100644 --- a/packages/gatsby/src/state-machines/data-layer/index.ts +++ b/packages/gatsby/src/state-machines/data-layer/index.ts @@ -52,7 +52,7 @@ const dataLayerStates: MachineConfig = { src: `buildSchema`, onDone: { target: `creatingPages`, - actions: `assignGatsbyNodeGraphQL`, + actions: `assignGraphQLRunners`, }, }, }, diff --git a/packages/gatsby/src/state-machines/query-running/index.ts b/packages/gatsby/src/state-machines/query-running/index.ts index 7df9e6ddb0f0b..a4aea1ea1f6a2 100644 --- a/packages/gatsby/src/state-machines/query-running/index.ts +++ b/packages/gatsby/src/state-machines/query-running/index.ts @@ -18,12 +18,9 @@ export const queryStates: MachineConfig = { invoke: { id: `extracting-queries`, src: `extractQueries`, - onDone: [ - { - actions: `resetGraphQLRunner`, - target: `writingRequires`, - }, - ], + onDone: { + target: `writingRequires`, + }, }, }, writingRequires: { diff --git a/packages/gatsby/src/state-machines/waiting/index.ts b/packages/gatsby/src/state-machines/waiting/index.ts index 4b6dd2b4b34dd..7cab16825cef5 100644 --- a/packages/gatsby/src/state-machines/waiting/index.ts +++ b/packages/gatsby/src/state-machines/waiting/index.ts @@ -10,16 +10,18 @@ export type WaitingResult = Pick export const waitingStates: MachineConfig = { initial: `idle`, - context: {}, + context: { + nodeMutationBatch: [], + runningBatch: [], + }, states: { idle: { - always: + always: { // If we already have queued node mutations, move // immediately to batching - { - cond: (ctx): boolean => !!ctx.nodeMutationBatch?.length, - target: `batchingNodeMutations`, - }, + cond: (ctx): boolean => !!ctx.nodeMutationBatch.length, + target: `batchingNodeMutations`, + }, on: { ADD_NODE_MUTATION: { actions: `addNodeMutation`, @@ -37,27 +39,27 @@ export const waitingStates: MachineConfig = { // Check if the batch is already full on entry always: { cond: (ctx): boolean => - (ctx.nodeMutationBatch?.length || 0) >= NODE_MUTATION_BATCH_SIZE, + ctx.nodeMutationBatch.length >= NODE_MUTATION_BATCH_SIZE, target: `committingBatch`, }, on: { // More mutations added to batch ADD_NODE_MUTATION: [ - // If this fills the batch then commit it { + // If this fills the batch then commit it actions: `addNodeMutation`, cond: (ctx): boolean => - (ctx.nodeMutationBatch?.length || 0) >= NODE_MUTATION_BATCH_SIZE, + ctx.nodeMutationBatch.length >= NODE_MUTATION_BATCH_SIZE, target: `committingBatch`, }, - // otherwise just add it to the batch { + // otherwise just add it to the batch actions: `addNodeMutation`, }, ], }, - // Time's up after: { + // Time's up [NODE_MUTATION_BATCH_TIMEOUT]: `committingBatch`, }, }, @@ -93,7 +95,6 @@ export const waitingStates: MachineConfig = { }, } -// eslint-disable-next-line new-cap export const waitingMachine = Machine(waitingStates, { actions: waitingActions, services: waitingServices, diff --git a/packages/gatsby/src/state-machines/waiting/types.ts b/packages/gatsby/src/state-machines/waiting/types.ts index 63c1d14e491dc..0fbbc2a7240c0 100644 --- a/packages/gatsby/src/state-machines/waiting/types.ts +++ b/packages/gatsby/src/state-machines/waiting/types.ts @@ -6,9 +6,9 @@ export interface IMutationAction { payload: unknown[] } export interface IWaitingContext { - nodeMutationBatch?: IMutationAction[] + nodeMutationBatch: IMutationAction[] store?: Store - runningBatch?: IMutationAction[] + runningBatch: IMutationAction[] filesDirty?: boolean webhookBody?: Record } diff --git a/packages/gatsby/src/utils/api-runner-node.js b/packages/gatsby/src/utils/api-runner-node.js index aced089d54e1b..e35811d35a9b9 100644 --- a/packages/gatsby/src/utils/api-runner-node.js +++ b/packages/gatsby/src/utils/api-runner-node.js @@ -97,13 +97,11 @@ const NODE_MUTATION_ACTIONS = [ ] const deferActions = actions => { - // console.log(`actual actions`, actions) - const returnValue = NODE_MUTATION_ACTIONS.reduce((prev, next) => { - prev[next] = deferredAction(next) - return prev - }, actions) - // console.log(`returnValue`, returnValue) - return returnValue + const deferred = { ...actions } + NODE_MUTATION_ACTIONS.forEach(action => { + deferred[action] = deferredAction(action) + }) + return deferred } const getLocalReporter = (activity, reporter) => diff --git a/packages/gatsby/src/utils/changed-pages.ts b/packages/gatsby/src/utils/changed-pages.ts index 643190f4379c8..8ca0836fa68bc 100644 --- a/packages/gatsby/src/utils/changed-pages.ts +++ b/packages/gatsby/src/utils/changed-pages.ts @@ -6,7 +6,7 @@ import { IGatsbyPage } from "../redux/types" export function deleteUntouchedPages( currentPages: Map, - timestamp: number + timeBeforeApisRan: number ): string[] { const deletedPages: string[] = [] @@ -14,7 +14,7 @@ export function deleteUntouchedPages( currentPages.forEach(page => { if ( !page.isCreatedByStatefulCreatePages && - page.updatedAt < timestamp && + page.updatedAt < timeBeforeApisRan && page.path !== `/404.html` ) { deleteComponentsDependencies([page.path]) @@ -37,10 +37,10 @@ export function findChangedPages( const compareWithoutUpdated: IsEqualCustomizer = (_left, _right, key) => key === `updatedAt` || undefined - currentPages.forEach((newPage, key) => { - const oldPage = oldPages.get(key) + currentPages.forEach((newPage, path) => { + const oldPage = oldPages.get(path) if (!oldPage || !isEqualWith(newPage, oldPage, compareWithoutUpdated)) { - changedPages.push(key) + changedPages.push(path) } }) const deletedPages: string[] = [] diff --git a/packages/gatsby/src/utils/source-nodes.ts b/packages/gatsby/src/utils/source-nodes.ts index 78a9ae3b5c3ae..fb9b982a27388 100644 --- a/packages/gatsby/src/utils/source-nodes.ts +++ b/packages/gatsby/src/utils/source-nodes.ts @@ -87,9 +87,9 @@ export default async ({ parentSpan, deferNodeMutation = false, }: { - webhookBody?: unknown - parentSpan?: Span - deferNodeMutation?: boolean + webhookBody: unknown + parentSpan: Span + deferNodeMutation: boolean }): Promise => { await apiRunner(`sourceNodes`, { traceId: `initial-sourceNodes`, From 9edd938a2aa55d38c9464462db0190678425fe01 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Tue, 7 Jul 2020 16:31:42 +0100 Subject: [PATCH 34/44] Typefix --- .../gatsby/src/commands/develop-process.ts | 29 +++++++++---------- .../shared-transition-configs.ts | 17 ----------- 2 files changed, 13 insertions(+), 33 deletions(-) diff --git a/packages/gatsby/src/commands/develop-process.ts b/packages/gatsby/src/commands/develop-process.ts index 2f552994aff0a..a0d008ac8caf6 100644 --- a/packages/gatsby/src/commands/develop-process.ts +++ b/packages/gatsby/src/commands/develop-process.ts @@ -40,10 +40,7 @@ import { globalTracer } from "opentracing" import { IQueryRunningContext } from "../state-machines/query-running/types" import { queryRunningMachine } from "../state-machines/query-running" import { IWaitingContext } from "../state-machines/waiting/types" -import { - ADD_NODE_MUTATION, - runMutationAndMarkDirty, -} from "../state-machines/shared-transition-configs" +import { runMutationAndMarkDirty } from "../state-machines/shared-transition-configs" import { buildActions } from "../state-machines/actions" import { waitingMachine } from "../state-machines/waiting" @@ -319,18 +316,18 @@ module.exports = async (program: IProgram): Promise => { }, } - const service = interpret( - Machine(developConfig, { - services: { - initializeDataLayer: dataLayerMachine, - initialize, - runQueries: queryRunningMachine, - waitForMutations: waitingMachine, - startWebpackServer: startWebpackServer, - }, - actions: buildActions, - }).withContext({ program, parentSpan: bootstrapSpan, app, firstRun: true }) - ) + const machine = Machine(developConfig, { + services: { + initializeDataLayer: dataLayerMachine, + initialize, + runQueries: queryRunningMachine, + waitForMutations: waitingMachine, + startWebpackServer: startWebpackServer, + }, + actions: buildActions, + }).withContext({ program, parentSpan: bootstrapSpan, app, firstRun: true }) + + const service = interpret(machine) const isInterpreter = ( actor: Actor | Interpreter diff --git a/packages/gatsby/src/state-machines/shared-transition-configs.ts b/packages/gatsby/src/state-machines/shared-transition-configs.ts index 4106307b0b366..9507e31369319 100644 --- a/packages/gatsby/src/state-machines/shared-transition-configs.ts +++ b/packages/gatsby/src/state-machines/shared-transition-configs.ts @@ -1,20 +1,3 @@ -/** - * Event handler used in all states where we're not ready to process node - * mutations. Instead we add it to a batch to process when we're next idle - */ -export const ADD_NODE_MUTATION = { - actions: `addNodeMutation`, -} - -export const QUERY_FILE_CHANGED = { - actions: `markQueryFilesDirty`, -} - -export const WEBHOOK_RECEIVED = { - target: `initializingDataLayer`, - actions: `assignWebhookBody`, -} - /** * Event handler used in all states where we're not ready to process a file change * Instead we add it to a batch to process when we're next idle From 94960466c15d50158f031c7d8bfe126275776980 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Tue, 14 Jul 2020 13:43:11 +0100 Subject: [PATCH 35/44] feat(gatsby): Move final parts into develop state machine (#25716) * Move remaining parts into state machine * Move top level state machine into state machines dir * Add machine ids * Add missing imports --- .../gatsby/src/commands/develop-process.ts | 240 +----------------- .../src/services/write-out-redirects.ts | 6 +- .../src/state-machines/data-layer/actions.ts | 2 +- .../src/state-machines/data-layer/index.ts | 35 ++- .../src/state-machines/data-layer/services.ts | 4 + .../state-machines/{ => develop}/actions.ts | 14 +- .../src/state-machines/develop/index.ts | 171 +++++++++++++ .../src/state-machines/develop/services.ts | 13 + .../state-machines/query-running/actions.ts | 13 + .../src/state-machines/query-running/index.ts | 15 ++ .../shared-transition-configs.ts | 18 -- .../src/state-machines/waiting/index.ts | 1 + 12 files changed, 273 insertions(+), 259 deletions(-) rename packages/gatsby/src/state-machines/{ => develop}/actions.ts (87%) create mode 100644 packages/gatsby/src/state-machines/develop/index.ts create mode 100644 packages/gatsby/src/state-machines/develop/services.ts delete mode 100644 packages/gatsby/src/state-machines/shared-transition-configs.ts diff --git a/packages/gatsby/src/commands/develop-process.ts b/packages/gatsby/src/commands/develop-process.ts index a0d008ac8caf6..379620f392d91 100644 --- a/packages/gatsby/src/commands/develop-process.ts +++ b/packages/gatsby/src/commands/develop-process.ts @@ -4,45 +4,19 @@ import chalk from "chalk" import telemetry from "gatsby-telemetry" import express from "express" import { initTracer } from "../utils/tracer" -import db from "../db" import { detectPortInUseAndPrompt } from "../utils/detect-port-in-use-and-prompt" import onExit from "signal-exit" import { userPassesFeedbackRequestHeuristic, showFeedbackRequest, } from "../utils/feedback" -import { startRedirectListener } from "../bootstrap/redirects-writer" import { markWebpackStatusAsPending } from "../utils/webpack-status" import { IProgram } from "./types" -import { - IBuildContext, - initialize, - rebuildSchemaWithSitePage, - writeOutRedirects, - startWebpackServer, -} from "../services" -import { boundActionCreators } from "../redux/actions" -import { ProgramStatus } from "../redux/types" -import { - MachineConfig, - AnyEventObject, - Machine, - interpret, - Actor, - Interpreter, - forwardTo, - State, -} from "xstate" -import { dataLayerMachine } from "../state-machines/data-layer" -import { IDataLayerContext } from "../state-machines/data-layer/types" +import { IBuildContext } from "../services" +import { AnyEventObject, interpret, Actor, Interpreter, State } from "xstate" import { globalTracer } from "opentracing" -import { IQueryRunningContext } from "../state-machines/query-running/types" -import { queryRunningMachine } from "../state-machines/query-running" -import { IWaitingContext } from "../state-machines/waiting/types" -import { runMutationAndMarkDirty } from "../state-machines/shared-transition-configs" -import { buildActions } from "../state-machines/actions" -import { waitingMachine } from "../state-machines/waiting" +import { developMachine } from "../state-machines/develop" const tracer = globalTracer() @@ -81,7 +55,6 @@ process.on(`message`, msg => { module.exports = async (program: IProgram): Promise => { reporter.setVerbose(program.verbose) - const bootstrapSpan = tracer.startSpan(`bootstrap`) // We want to prompt the feedback request when users quit develop // assuming they pass the heuristic check to know they are a user @@ -125,207 +98,14 @@ module.exports = async (program: IProgram): Promise => { } const app = express() + const parentSpan = tracer.startSpan(`bootstrap`) - const developConfig: MachineConfig = { - id: `build`, - initial: `initializing`, - states: { - initializing: { - on: { - // Ignore mutation events because we'll be running everything anyway - ADD_NODE_MUTATION: undefined, - QUERY_FILE_CHANGED: undefined, - WEBHOOK_RECEIVED: undefined, - }, - invoke: { - src: `initialize`, - onDone: { - target: `initializingDataLayer`, - actions: [`assignStoreAndWorkerPool`, `spawnMutationListener`], - }, - }, - }, - initializingDataLayer: { - on: { - ADD_NODE_MUTATION: runMutationAndMarkDirty, - // Ignore, because we're about to extract them anyway - QUERY_FILE_CHANGED: undefined, - }, - invoke: { - src: `initializeDataLayer`, - data: ({ - parentSpan, - store, - firstRun, - webhookBody, - }: IBuildContext): IDataLayerContext => { - return { - parentSpan, - store, - firstRun, - deferNodeMutation: true, - webhookBody, - } - }, - onDone: { - actions: [`assignServiceResult`, `clearWebhookBody`], - target: `finishingBootstrap`, - }, - }, - }, - finishingBootstrap: { - on: { - ADD_NODE_MUTATION: runMutationAndMarkDirty, - // Ignore, because we're about to extract them anyway - QUERY_FILE_CHANGED: undefined, - }, - invoke: { - src: async (): Promise => { - // These were previously in `bootstrap()` but are now - // in part of the state machine that hasn't been added yet - await rebuildSchemaWithSitePage({ parentSpan: bootstrapSpan }) - - await writeOutRedirects({ parentSpan: bootstrapSpan }) - - startRedirectListener() - bootstrapSpan.finish() - }, - onDone: { - target: `runningQueries`, - }, - }, - }, - runningQueries: { - on: { - QUERY_FILE_CHANGED: { - actions: forwardTo(`run-queries`), - }, - }, - invoke: { - id: `run-queries`, - src: `runQueries`, - data: ({ - program, - store, - parentSpan, - gatsbyNodeGraphQLFunction, - graphqlRunner, - websocketManager, - firstRun, - }: IBuildContext): IQueryRunningContext => { - return { - firstRun, - program, - store, - parentSpan, - gatsbyNodeGraphQLFunction, - graphqlRunner, - websocketManager, - } - }, - onDone: { - target: `doingEverythingElse`, - }, - }, - }, - doingEverythingElse: { - invoke: { - src: async (): Promise => { - // All the stuff that's not in the state machine yet - - boundActionCreators.setProgramStatus( - ProgramStatus.BOOTSTRAP_QUERY_RUNNING_FINISHED - ) - - await db.saveState() - - db.startAutosave() - }, - onDone: [ - { - target: `startingDevServers`, - cond: ({ compiler }: IBuildContext): boolean => !compiler, - }, - { - target: `waiting`, - }, - ], - }, - }, - startingDevServers: { - invoke: { - src: `startWebpackServer`, - onDone: { - target: `waiting`, - actions: `assignServers`, - }, - }, - }, - waiting: { - on: { - ADD_NODE_MUTATION: { - actions: forwardTo(`waiting`), - }, - QUERY_FILE_CHANGED: { - actions: forwardTo(`waiting`), - }, - EXTRACT_QUERIES_NOW: { - target: `runningQueries`, - }, - }, - invoke: { - id: `waiting`, - src: `waitForMutations`, - data: ({ - store, - nodeMutationBatch = [], - }: IBuildContext): IWaitingContext => { - return { store, nodeMutationBatch, runningBatch: [] } - }, - onDone: { - actions: `assignServiceResult`, - target: `rebuildingPages`, - }, - }, - }, - rebuildingPages: { - invoke: { - src: `initializeDataLayer`, - data: ({ parentSpan, store }: IBuildContext): IDataLayerContext => { - return { parentSpan, store, firstRun: false, skipSourcing: true } - }, - onDone: { - actions: `assignServiceResult`, - target: `runningQueries`, - }, - }, - }, - }, - // Transitions shared by all states, except where overridden - on: { - ADD_NODE_MUTATION: { - actions: `addNodeMutation`, - }, - QUERY_FILE_CHANGED: { - actions: `markQueryFilesDirty`, - }, - WEBHOOK_RECEIVED: { - target: `initializingDataLayer`, - actions: `assignWebhookBody`, - }, - }, - } - - const machine = Machine(developConfig, { - services: { - initializeDataLayer: dataLayerMachine, - initialize, - runQueries: queryRunningMachine, - waitForMutations: waitingMachine, - startWebpackServer: startWebpackServer, - }, - actions: buildActions, - }).withContext({ program, parentSpan: bootstrapSpan, app, firstRun: true }) + const machine = developMachine.withContext({ + program, + parentSpan, + app, + firstRun: true, + }) const service = interpret(machine) diff --git a/packages/gatsby/src/services/write-out-redirects.ts b/packages/gatsby/src/services/write-out-redirects.ts index 5745db6dc6fdf..18822eb781ff3 100644 --- a/packages/gatsby/src/services/write-out-redirects.ts +++ b/packages/gatsby/src/services/write-out-redirects.ts @@ -1,5 +1,8 @@ import reporter from "gatsby-cli/lib/reporter" -import { writeRedirects } from "../bootstrap/redirects-writer" +import { + writeRedirects, + startRedirectListener, +} from "../bootstrap/redirects-writer" import { IQueryRunningContext } from "../state-machines/query-running/types" export async function writeOutRedirects({ @@ -11,5 +14,6 @@ export async function writeOutRedirects({ }) activity.start() await writeRedirects() + startRedirectListener() activity.end() } diff --git a/packages/gatsby/src/state-machines/data-layer/actions.ts b/packages/gatsby/src/state-machines/data-layer/actions.ts index 6ff1e2491da3e..0ae2747a536d4 100644 --- a/packages/gatsby/src/state-machines/data-layer/actions.ts +++ b/packages/gatsby/src/state-machines/data-layer/actions.ts @@ -2,7 +2,7 @@ import { assign, DoneInvokeEvent, ActionFunctionMap } from "xstate" import { createGraphQLRunner } from "../../bootstrap/create-graphql-runner" import reporter from "gatsby-cli/lib/reporter" import { IDataLayerContext } from "./types" -import { callApi, markNodesDirty } from "../actions" +import { callApi, markNodesDirty } from "../develop/actions" import { assertStore } from "../../utils/assert-store" import { GraphQLRunner } from "../../query/graphql-runner" diff --git a/packages/gatsby/src/state-machines/data-layer/index.ts b/packages/gatsby/src/state-machines/data-layer/index.ts index 56f5de475c4f6..1e70213e1fa5f 100644 --- a/packages/gatsby/src/state-machines/data-layer/index.ts +++ b/packages/gatsby/src/state-machines/data-layer/index.ts @@ -13,14 +13,14 @@ export type DataLayerResult = Pick< const dataLayerStates: MachineConfig = { initial: `start`, + id: `dataLayerMachine`, context: {}, states: { start: { always: [ { target: `buildingSchema`, - cond: ({ skipSourcing }: IDataLayerContext): boolean => - !!skipSourcing, + cond: `shouldSkipSourcing`, }, { target: `customizingSchema`, @@ -65,10 +65,10 @@ const dataLayerStates: MachineConfig = { { target: `creatingPagesStatefully`, actions: `assignChangedPages`, - cond: (context): boolean => !!context.firstRun, + cond: `firstRun`, }, { - target: `done`, + target: `rebuildingSchemaWithSitePage`, actions: `assignChangedPages`, }, ], @@ -78,6 +78,28 @@ const dataLayerStates: MachineConfig = { invoke: { src: `createPagesStatefully`, id: `creating-pages-statefully`, + onDone: { + target: `rebuildingSchemaWithSitePage`, + }, + }, + }, + rebuildingSchemaWithSitePage: { + invoke: { + src: `rebuildSchemaWithSitePage`, + onDone: [ + { + target: `writingOutRedirects`, + cond: `firstRun`, + }, + { + target: `done`, + }, + ], + }, + }, + writingOutRedirects: { + invoke: { + src: `writeOutRedirects`, onDone: { target: `done`, }, @@ -105,4 +127,9 @@ const dataLayerStates: MachineConfig = { export const dataLayerMachine = Machine(dataLayerStates, { actions: dataLayerActions, services: dataLayerServices, + guards: { + firstRun: ({ firstRun }: IDataLayerContext): boolean => !!firstRun, + shouldSkipSourcing: ({ skipSourcing }: IDataLayerContext): boolean => + !!skipSourcing, + }, }) diff --git a/packages/gatsby/src/state-machines/data-layer/services.ts b/packages/gatsby/src/state-machines/data-layer/services.ts index fd04e3c19df98..4871e90d2d0a1 100644 --- a/packages/gatsby/src/state-machines/data-layer/services.ts +++ b/packages/gatsby/src/state-machines/data-layer/services.ts @@ -5,6 +5,8 @@ import { createPagesStatefully, buildSchema, sourceNodes, + rebuildSchemaWithSitePage, + writeOutRedirects, } from "../../services" import { IDataLayerContext } from "./types" @@ -17,4 +19,6 @@ export const dataLayerServices: Record< createPages, buildSchema, createPagesStatefully, + rebuildSchemaWithSitePage, + writeOutRedirects, } diff --git a/packages/gatsby/src/state-machines/actions.ts b/packages/gatsby/src/state-machines/develop/actions.ts similarity index 87% rename from packages/gatsby/src/state-machines/actions.ts rename to packages/gatsby/src/state-machines/develop/actions.ts index 4accf988745f8..c5d22becb03ad 100644 --- a/packages/gatsby/src/state-machines/actions.ts +++ b/packages/gatsby/src/state-machines/develop/actions.ts @@ -7,11 +7,11 @@ import { DoneEventObject, } from "xstate" import { Store } from "redux" -import { IBuildContext, IMutationAction } from "../services" -import { actions } from "../redux/actions" -import { listenForMutations } from "../services/listen-for-mutations" -import { DataLayerResult } from "./data-layer" -import { assertStore } from "../utils/assert-store" +import { IBuildContext, IMutationAction } from "../../services" +import { actions } from "../../redux/actions" +import { listenForMutations } from "../../services/listen-for-mutations" +import { DataLayerResult } from "../data-layer" +import { assertStore } from "../../utils/assert-store" import reporter from "gatsby-cli/lib/reporter" export const callRealApi = (event: IMutationAction, store?: Store): void => { @@ -83,6 +83,9 @@ export const clearWebhookBody = assign({ webhookBody: undefined, }) +export const finishParentSpan = ({ parentSpan }: IBuildContext): void => + parentSpan?.finish() + /** * Event handler used in all states where we're not ready to process a file change * Instead we add it to a batch to process when we're next idle @@ -106,4 +109,5 @@ export const buildActions: ActionFunctionMap = { markQueryFilesDirty, assignWebhookBody, clearWebhookBody, + finishParentSpan, } diff --git a/packages/gatsby/src/state-machines/develop/index.ts b/packages/gatsby/src/state-machines/develop/index.ts new file mode 100644 index 0000000000000..7b37579b01ccb --- /dev/null +++ b/packages/gatsby/src/state-machines/develop/index.ts @@ -0,0 +1,171 @@ +import { MachineConfig, AnyEventObject, forwardTo, Machine } from "xstate" +import { IDataLayerContext } from "../data-layer/types" +import { IQueryRunningContext } from "../query-running/types" +import { IWaitingContext } from "../waiting/types" +import { buildActions } from "./actions" +import { developServices } from "./services" +import { IBuildContext } from "../../services" + +/** + * This is the top-level state machine for the `gatsby develop` command + */ +const developConfig: MachineConfig = { + id: `build`, + initial: `initializing`, + states: { + initializing: { + on: { + // Ignore mutation events because we'll be running everything anyway + ADD_NODE_MUTATION: undefined, + QUERY_FILE_CHANGED: undefined, + WEBHOOK_RECEIVED: undefined, + }, + invoke: { + src: `initialize`, + onDone: { + target: `initializingDataLayer`, + actions: [`assignStoreAndWorkerPool`, `spawnMutationListener`], + }, + }, + }, + initializingDataLayer: { + on: { + ADD_NODE_MUTATION: { + actions: [`markNodesDirty`, `callApi`], + }, + // Ignore, because we're about to extract them anyway + QUERY_FILE_CHANGED: undefined, + }, + invoke: { + src: `initializeDataLayer`, + data: ({ + parentSpan, + store, + firstRun, + webhookBody, + }: IBuildContext): IDataLayerContext => { + return { + parentSpan, + store, + firstRun, + deferNodeMutation: true, + webhookBody, + } + }, + onDone: { + actions: [ + `assignServiceResult`, + `clearWebhookBody`, + `finishParentSpan`, + ], + target: `runningQueries`, + }, + }, + }, + runningQueries: { + on: { + QUERY_FILE_CHANGED: { + actions: forwardTo(`run-queries`), + }, + }, + invoke: { + id: `run-queries`, + src: `runQueries`, + data: ({ + program, + store, + parentSpan, + gatsbyNodeGraphQLFunction, + graphqlRunner, + websocketManager, + firstRun, + }: IBuildContext): IQueryRunningContext => { + return { + firstRun, + program, + store, + parentSpan, + gatsbyNodeGraphQLFunction, + graphqlRunner, + websocketManager, + } + }, + onDone: [ + { + target: `startingDevServers`, + cond: ({ compiler }: IBuildContext): boolean => !compiler, + }, + { + target: `waiting`, + }, + ], + }, + }, + startingDevServers: { + invoke: { + src: `startWebpackServer`, + onDone: { + target: `waiting`, + actions: `assignServers`, + }, + }, + }, + waiting: { + on: { + ADD_NODE_MUTATION: { + actions: forwardTo(`waiting`), + }, + QUERY_FILE_CHANGED: { + actions: forwardTo(`waiting`), + }, + EXTRACT_QUERIES_NOW: { + target: `runningQueries`, + }, + }, + invoke: { + id: `waiting`, + src: `waitForMutations`, + data: ({ + store, + nodeMutationBatch = [], + }: IBuildContext): IWaitingContext => { + return { store, nodeMutationBatch, runningBatch: [] } + }, + onDone: { + actions: `assignServiceResult`, + target: `rebuildingPages`, + }, + }, + }, + rebuildingPages: { + invoke: { + src: `initializeDataLayer`, + data: ({ parentSpan, store }: IBuildContext): IDataLayerContext => { + return { parentSpan, store, firstRun: false, skipSourcing: true } + }, + onDone: { + actions: `assignServiceResult`, + target: `runningQueries`, + }, + }, + }, + }, + // Transitions shared by all states, except where overridden + on: { + ADD_NODE_MUTATION: { + actions: `addNodeMutation`, + }, + QUERY_FILE_CHANGED: { + actions: `markQueryFilesDirty`, + }, + WEBHOOK_RECEIVED: { + target: `initializingDataLayer`, + actions: `assignWebhookBody`, + }, + }, +} + +export const developMachine = Machine(developConfig, { + services: developServices, + actions: buildActions, +}) diff --git a/packages/gatsby/src/state-machines/develop/services.ts b/packages/gatsby/src/state-machines/develop/services.ts new file mode 100644 index 0000000000000..a233e8eadacf3 --- /dev/null +++ b/packages/gatsby/src/state-machines/develop/services.ts @@ -0,0 +1,13 @@ +import { IBuildContext, startWebpackServer, initialize } from "../../services" +import { dataLayerMachine } from "../data-layer" +import { queryRunningMachine } from "../query-running" +import { waitingMachine } from "../waiting" +import { ServiceConfig } from "xstate" + +export const developServices: Record> = { + initializeDataLayer: dataLayerMachine, + initialize: initialize, + runQueries: queryRunningMachine, + waitForMutations: waitingMachine, + startWebpackServer: startWebpackServer, +} diff --git a/packages/gatsby/src/state-machines/query-running/actions.ts b/packages/gatsby/src/state-machines/query-running/actions.ts index 93d3ebfae03f7..f717305b169ea 100644 --- a/packages/gatsby/src/state-machines/query-running/actions.ts +++ b/packages/gatsby/src/state-machines/query-running/actions.ts @@ -3,6 +3,9 @@ import { DoneInvokeEvent, assign, ActionFunctionMap } from "xstate" import { GraphQLRunner } from "../../query/graphql-runner" import { assertStore } from "../../utils/assert-store" import { enqueueFlush } from "../../utils/page-data" +import { boundActionCreators } from "../../redux/actions" +import { ProgramStatus } from "../../redux/types" +import db from "../../db" export const flushPageData = (): void => { enqueueFlush() @@ -36,6 +39,15 @@ export const resetGraphQLRunner = assign< }, }) +const finishUpQueries = async (): Promise => { + boundActionCreators.setProgramStatus( + ProgramStatus.BOOTSTRAP_QUERY_RUNNING_FINISHED + ) + await db.saveState() + + db.startAutosave() +} + export const queryActions: ActionFunctionMap< IQueryRunningContext, DoneInvokeEvent @@ -44,4 +56,5 @@ export const queryActions: ActionFunctionMap< assignDirtyQueries, flushPageData, markFilesClean, + finishUpQueries, } diff --git a/packages/gatsby/src/state-machines/query-running/index.ts b/packages/gatsby/src/state-machines/query-running/index.ts index a4aea1ea1f6a2..b20daa6d01258 100644 --- a/packages/gatsby/src/state-machines/query-running/index.ts +++ b/packages/gatsby/src/state-machines/query-running/index.ts @@ -10,6 +10,7 @@ const extractQueriesIfFilesAreDirty = { export const queryStates: MachineConfig = { initial: `extractingQueries`, + id: `queryRunningMachine`, context: {}, states: { extractingQueries: { @@ -75,6 +76,20 @@ export const queryStates: MachineConfig = { }, }, }, + finishingUp: { + on: { + always: [ + { + target: `done`, + actions: `finishUpQueries`, + cond: ({ firstRun }): boolean => !!firstRun, + }, + { + target: `done`, + }, + ], + }, + }, done: { type: `final`, }, diff --git a/packages/gatsby/src/state-machines/shared-transition-configs.ts b/packages/gatsby/src/state-machines/shared-transition-configs.ts deleted file mode 100644 index 9507e31369319..0000000000000 --- a/packages/gatsby/src/state-machines/shared-transition-configs.ts +++ /dev/null @@ -1,18 +0,0 @@ -/** - * Event handler used in all states where we're not ready to process a file change - * Instead we add it to a batch to process when we're next idle - */ -// export const SOURCE_FILE_CHANGED: TransitionConfig< -// Pick, -// AnyEventObject -// > = { -// actions: `markFilesDirty`, -// } - -/** - * When running queries we might add nodes (e.g from resolvers). If so we'll - * want to re-run queries and schema inference - */ -export const runMutationAndMarkDirty = { - actions: [`markNodesDirty`, `callApi`], -} diff --git a/packages/gatsby/src/state-machines/waiting/index.ts b/packages/gatsby/src/state-machines/waiting/index.ts index 7cab16825cef5..6446c194afeeb 100644 --- a/packages/gatsby/src/state-machines/waiting/index.ts +++ b/packages/gatsby/src/state-machines/waiting/index.ts @@ -9,6 +9,7 @@ const NODE_MUTATION_BATCH_TIMEOUT = 1000 export type WaitingResult = Pick export const waitingStates: MachineConfig = { + id: `waitingMachine`, initial: `idle`, context: { nodeMutationBatch: [], From 1cab0bea4c9c4c70fff48145d768e5d7b5a9b590 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Wed, 15 Jul 2020 16:33:30 +0100 Subject: [PATCH 36/44] Resolve api promises --- packages/gatsby/src/services/types.ts | 1 + .../gatsby/src/state-machines/develop/actions.ts | 5 +++-- packages/gatsby/src/utils/api-runner-node.js | 12 +++++++----- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/packages/gatsby/src/services/types.ts b/packages/gatsby/src/services/types.ts index 5d6bd9a39cc0f..ccba9330af9a4 100644 --- a/packages/gatsby/src/services/types.ts +++ b/packages/gatsby/src/services/types.ts @@ -18,6 +18,7 @@ export interface IGroupedQueryIds { export interface IMutationAction { type: string payload: unknown[] + resolve: (result: unknown) => void } export interface IBuildContext { firstRun?: boolean diff --git a/packages/gatsby/src/state-machines/develop/actions.ts b/packages/gatsby/src/state-machines/develop/actions.ts index c5d22becb03ad..92aca0fc9d277 100644 --- a/packages/gatsby/src/state-machines/develop/actions.ts +++ b/packages/gatsby/src/state-machines/develop/actions.ts @@ -16,9 +16,10 @@ import reporter from "gatsby-cli/lib/reporter" export const callRealApi = (event: IMutationAction, store?: Store): void => { assertStore(store) - const { type, payload } = event + const { type, payload, resolve } = event if (type in actions) { - store.dispatch(actions[type](...payload)) + const action = actions[type](...payload) + resolve(store.dispatch(action)) } else { reporter.log(`Could not dispatch unknown action "${type}`) } diff --git a/packages/gatsby/src/utils/api-runner-node.js b/packages/gatsby/src/utils/api-runner-node.js index ea4891bb9f6fb..a180d8c321c6a 100644 --- a/packages/gatsby/src/utils/api-runner-node.js +++ b/packages/gatsby/src/utils/api-runner-node.js @@ -80,12 +80,14 @@ const initAPICallTracing = parentSpan => { } } -const deferredAction = type => (...args) => { - emitter.emit(`ENQUEUE_NODE_MUTATION`, { - type, - payload: args, +const deferredAction = type => (...args) => + new Promise(resolve => { + emitter.emit(`ENQUEUE_NODE_MUTATION`, { + type, + payload: args, + resolve, + }) }) -} const NODE_MUTATION_ACTIONS = [ `createNode`, From 00a975d03e304fd114af3c2cb08e995bf9e351e9 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Thu, 16 Jul 2020 14:17:05 +0100 Subject: [PATCH 37/44] Remove unused action --- .../gatsby/src/state-machines/query-running/actions.ts | 6 ------ .../gatsby/src/state-machines/query-running/index.ts | 10 ---------- 2 files changed, 16 deletions(-) diff --git a/packages/gatsby/src/state-machines/query-running/actions.ts b/packages/gatsby/src/state-machines/query-running/actions.ts index f717305b169ea..3edfad13d6b87 100644 --- a/packages/gatsby/src/state-machines/query-running/actions.ts +++ b/packages/gatsby/src/state-machines/query-running/actions.ts @@ -11,11 +11,6 @@ export const flushPageData = (): void => { enqueueFlush() } -export const markFilesClean = assign< - IQueryRunningContext, - DoneInvokeEvent ->({ filesDirty: false }) - export const assignDirtyQueries = assign< IQueryRunningContext, DoneInvokeEvent @@ -55,6 +50,5 @@ export const queryActions: ActionFunctionMap< resetGraphQLRunner, assignDirtyQueries, flushPageData, - markFilesClean, finishUpQueries, } diff --git a/packages/gatsby/src/state-machines/query-running/index.ts b/packages/gatsby/src/state-machines/query-running/index.ts index b20daa6d01258..ea3b9da8c2526 100644 --- a/packages/gatsby/src/state-machines/query-running/index.ts +++ b/packages/gatsby/src/state-machines/query-running/index.ts @@ -3,11 +3,6 @@ import { IQueryRunningContext } from "./types" import { queryRunningServices } from "./services" import { queryActions } from "./actions" -const extractQueriesIfFilesAreDirty = { - cond: (ctx): boolean => !!ctx.filesDirty, - target: `extractingQueries`, -} - export const queryStates: MachineConfig = { initial: `extractingQueries`, id: `queryRunningMachine`, @@ -15,7 +10,6 @@ export const queryStates: MachineConfig = { states: { extractingQueries: { id: `extracting-queries`, - onEntry: `markFilesClean`, invoke: { id: `extracting-queries`, src: `extractQueries`, @@ -25,7 +19,6 @@ export const queryStates: MachineConfig = { }, }, writingRequires: { - always: extractQueriesIfFilesAreDirty, invoke: { src: `writeOutRequires`, id: `writing-requires`, @@ -35,7 +28,6 @@ export const queryStates: MachineConfig = { }, }, calculatingDirtyQueries: { - always: extractQueriesIfFilesAreDirty, invoke: { id: `calculating-dirty-queries`, src: `calculateDirtyQueries`, @@ -46,7 +38,6 @@ export const queryStates: MachineConfig = { }, }, runningStaticQueries: { - always: extractQueriesIfFilesAreDirty, invoke: { src: `runStaticQueries`, id: `running-static-queries`, @@ -56,7 +47,6 @@ export const queryStates: MachineConfig = { }, }, runningPageQueries: { - always: extractQueriesIfFilesAreDirty, invoke: { src: `runPageQueries`, id: `running-page-queries`, From c8079689e2e0087c02c975f0147b3759209a9350 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Thu, 16 Jul 2020 14:33:25 +0100 Subject: [PATCH 38/44] Move logging into helper --- .../gatsby/src/commands/develop-process.ts | 50 ++++------------- .../gatsby/src/utils/state-machine-logging.ts | 53 +++++++++++++++++++ 2 files changed, 62 insertions(+), 41 deletions(-) create mode 100644 packages/gatsby/src/utils/state-machine-logging.ts diff --git a/packages/gatsby/src/commands/develop-process.ts b/packages/gatsby/src/commands/develop-process.ts index 1266a41fc563b..c4d52f51176ac 100644 --- a/packages/gatsby/src/commands/develop-process.ts +++ b/packages/gatsby/src/commands/develop-process.ts @@ -14,10 +14,10 @@ import { import { markWebpackStatusAsPending } from "../utils/webpack-status" import { IProgram, IDebugInfo } from "./types" -import { IBuildContext } from "../services" -import { AnyEventObject, interpret, Actor, Interpreter, State } from "xstate" +import { interpret } from "xstate" import { globalTracer } from "opentracing" import { developMachine } from "../state-machines/develop" +import { logTransitions } from "../utils/state-machine-logging" const tracer = globalTracer() @@ -69,6 +69,10 @@ const openDebuggerPort = (debugInfo: IDebugInfo): void => { } module.exports = async (program: IDevelopArgs): Promise => { + if (program.verbose) { + reporter.setVerbose(true) + } + if (program.debugInfo) { openDebuggerPort(program.debugInfo) } @@ -126,45 +130,9 @@ module.exports = async (program: IDevelopArgs): Promise => { const service = interpret(machine) - const isInterpreter = ( - actor: Actor | Interpreter - ): actor is Interpreter => `machine` in actor - - const listeners = new WeakSet() - let last: State + if (program.verbose) { + logTransitions(service) + } - service.onTransition(state => { - if (!last) { - last = state - } else if (!state.changed || last.matches(state)) { - return - } - last = state - reporter.verbose(`Transition to ${JSON.stringify(state.value)}`) - // eslint-disable-next-line no-unused-expressions - service.children?.forEach(child => { - // We want to ensure we don't attach a listener to the same - // actor. We don't need to worry about detaching the listener - // because xstate handles that for us when the actor is stopped. - - if (isInterpreter(child) && !listeners.has(child)) { - let sublast = child.state - child.onTransition(substate => { - if (!sublast) { - sublast = substate - } else if (!substate.changed || sublast.matches(substate)) { - return - } - sublast = substate - reporter.verbose( - `Transition to ${JSON.stringify(state.value)} > ${JSON.stringify( - substate.value - )}` - ) - }) - listeners.add(child) - } - }) - }) service.start() } diff --git a/packages/gatsby/src/utils/state-machine-logging.ts b/packages/gatsby/src/utils/state-machine-logging.ts new file mode 100644 index 0000000000000..9e4778c08e375 --- /dev/null +++ b/packages/gatsby/src/utils/state-machine-logging.ts @@ -0,0 +1,53 @@ +import { + DefaultContext, + Interpreter, + Actor, + State, + AnyEventObject, +} from "xstate" +import reporter from "gatsby-cli/lib/reporter" + +const isInterpreter = ( + actor: Actor | Interpreter +): actor is Interpreter => `machine` in actor + +export function logTransitions( + service: Interpreter +): void { + const listeners = new WeakSet() + let last: State + + service.onTransition(state => { + if (!last) { + last = state + } else if (!state.changed || last.matches(state)) { + return + } + last = state + reporter.verbose(`Transition to ${JSON.stringify(state.value)}`) + // eslint-disable-next-line no-unused-expressions + service.children?.forEach(child => { + // We want to ensure we don't attach a listener to the same + // actor. We don't need to worry about detaching the listener + // because xstate handles that for us when the actor is stopped. + + if (isInterpreter(child) && !listeners.has(child)) { + let sublast = child.state + child.onTransition(substate => { + if (!sublast) { + sublast = substate + } else if (!substate.changed || sublast.matches(substate)) { + return + } + sublast = substate + reporter.verbose( + `Transition to ${JSON.stringify(state.value)} > ${JSON.stringify( + substate.value + )}` + ) + }) + listeners.add(child) + } + }) + }) +} From 69f9a7a5b67f3f7aad77b3591da023b22186e256 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Thu, 16 Jul 2020 16:07:00 +0100 Subject: [PATCH 39/44] Changes from review --- packages/gatsby/src/services/types.ts | 2 +- .../src/state-machines/develop/actions.ts | 5 ++++- .../src/state-machines/develop/index.ts | 2 +- packages/gatsby/src/utils/api-runner-node.js | 21 +++++++++++++------ 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/packages/gatsby/src/services/types.ts b/packages/gatsby/src/services/types.ts index ccba9330af9a4..25432115141f1 100644 --- a/packages/gatsby/src/services/types.ts +++ b/packages/gatsby/src/services/types.ts @@ -18,7 +18,7 @@ export interface IGroupedQueryIds { export interface IMutationAction { type: string payload: unknown[] - resolve: (result: unknown) => void + resolve?: (result: unknown) => void } export interface IBuildContext { firstRun?: boolean diff --git a/packages/gatsby/src/state-machines/develop/actions.ts b/packages/gatsby/src/state-machines/develop/actions.ts index 92aca0fc9d277..2577cca796a34 100644 --- a/packages/gatsby/src/state-machines/develop/actions.ts +++ b/packages/gatsby/src/state-machines/develop/actions.ts @@ -19,7 +19,10 @@ export const callRealApi = (event: IMutationAction, store?: Store): void => { const { type, payload, resolve } = event if (type in actions) { const action = actions[type](...payload) - resolve(store.dispatch(action)) + const result = store.dispatch(action) + if (resolve) { + resolve(result) + } } else { reporter.log(`Could not dispatch unknown action "${type}`) } diff --git a/packages/gatsby/src/state-machines/develop/index.ts b/packages/gatsby/src/state-machines/develop/index.ts index 7b37579b01ccb..b226fc8768dde 100644 --- a/packages/gatsby/src/state-machines/develop/index.ts +++ b/packages/gatsby/src/state-machines/develop/index.ts @@ -93,7 +93,7 @@ const developConfig: MachineConfig = { onDone: [ { target: `startingDevServers`, - cond: ({ compiler }: IBuildContext): boolean => !compiler, + cond: ({ firstRun }: IBuildContext): boolean => !!firstRun, }, { target: `waiting`, diff --git a/packages/gatsby/src/utils/api-runner-node.js b/packages/gatsby/src/utils/api-runner-node.js index a180d8c321c6a..aa252c4951c83 100644 --- a/packages/gatsby/src/utils/api-runner-node.js +++ b/packages/gatsby/src/utils/api-runner-node.js @@ -80,14 +80,23 @@ const initAPICallTracing = parentSpan => { } } -const deferredAction = type => (...args) => - new Promise(resolve => { - emitter.emit(`ENQUEUE_NODE_MUTATION`, { - type, - payload: args, - resolve, +const deferredAction = type => (...args) => { + // Regular createNode returns a Promise, but when deferred we need + // to wrap it in another which we resolve when it's actually called + if (type === `createNode`) { + return new Promise(resolve => { + emitter.emit(`ENQUEUE_NODE_MUTATION`, { + type, + payload: args, + resolve, + }) }) + } + return emitter.emit(`ENQUEUE_NODE_MUTATION`, { + type, + payload: args, }) +} const NODE_MUTATION_ACTIONS = [ `createNode`, From 523478ac6211839dd2e80d001989c5eac2ecfc83 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Fri, 17 Jul 2020 09:40:47 +0100 Subject: [PATCH 40/44] Manually save db --- packages/gatsby/src/state-machines/develop/actions.ts | 4 ++++ packages/gatsby/src/state-machines/develop/index.ts | 2 ++ packages/gatsby/src/state-machines/query-running/actions.ts | 4 ---- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/gatsby/src/state-machines/develop/actions.ts b/packages/gatsby/src/state-machines/develop/actions.ts index 2577cca796a34..18c6cd5317829 100644 --- a/packages/gatsby/src/state-machines/develop/actions.ts +++ b/packages/gatsby/src/state-machines/develop/actions.ts @@ -12,6 +12,7 @@ import { actions } from "../../redux/actions" import { listenForMutations } from "../../services/listen-for-mutations" import { DataLayerResult } from "../data-layer" import { assertStore } from "../../utils/assert-store" +import { saveState } from "../../db" import reporter from "gatsby-cli/lib/reporter" export const callRealApi = (event: IMutationAction, store?: Store): void => { @@ -90,6 +91,8 @@ export const clearWebhookBody = assign({ export const finishParentSpan = ({ parentSpan }: IBuildContext): void => parentSpan?.finish() +export const saveDbState = (): Promise => saveState() + /** * Event handler used in all states where we're not ready to process a file change * Instead we add it to a batch to process when we're next idle @@ -114,4 +117,5 @@ export const buildActions: ActionFunctionMap = { assignWebhookBody, clearWebhookBody, finishParentSpan, + saveDbState, } diff --git a/packages/gatsby/src/state-machines/develop/index.ts b/packages/gatsby/src/state-machines/develop/index.ts index b226fc8768dde..a8fc92dbfa0e4 100644 --- a/packages/gatsby/src/state-machines/develop/index.ts +++ b/packages/gatsby/src/state-machines/develop/index.ts @@ -111,6 +111,8 @@ const developConfig: MachineConfig = { }, }, waiting: { + // We may want to save this is more places, but this should do for now + entry: `saveDbState`, on: { ADD_NODE_MUTATION: { actions: forwardTo(`waiting`), diff --git a/packages/gatsby/src/state-machines/query-running/actions.ts b/packages/gatsby/src/state-machines/query-running/actions.ts index 3edfad13d6b87..57e69bbbb4204 100644 --- a/packages/gatsby/src/state-machines/query-running/actions.ts +++ b/packages/gatsby/src/state-machines/query-running/actions.ts @@ -5,7 +5,6 @@ import { assertStore } from "../../utils/assert-store" import { enqueueFlush } from "../../utils/page-data" import { boundActionCreators } from "../../redux/actions" import { ProgramStatus } from "../../redux/types" -import db from "../../db" export const flushPageData = (): void => { enqueueFlush() @@ -38,9 +37,6 @@ const finishUpQueries = async (): Promise => { boundActionCreators.setProgramStatus( ProgramStatus.BOOTSTRAP_QUERY_RUNNING_FINISHED ) - await db.saveState() - - db.startAutosave() } export const queryActions: ActionFunctionMap< From b8142155d603ed254184fd2005e0d0a92c48b819 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Fri, 17 Jul 2020 10:09:15 +0100 Subject: [PATCH 41/44] Add comments --- .../src/state-machines/develop/actions.ts | 12 +++++ .../src/state-machines/develop/index.ts | 47 ++++++++++++++----- .../state-machines/query-running/actions.ts | 16 ------- .../src/state-machines/query-running/index.ts | 8 +++- .../src/state-machines/waiting/index.ts | 10 +++- 5 files changed, 62 insertions(+), 31 deletions(-) diff --git a/packages/gatsby/src/state-machines/develop/actions.ts b/packages/gatsby/src/state-machines/develop/actions.ts index 18c6cd5317829..8f8f8f94f92d3 100644 --- a/packages/gatsby/src/state-machines/develop/actions.ts +++ b/packages/gatsby/src/state-machines/develop/actions.ts @@ -15,12 +15,20 @@ import { assertStore } from "../../utils/assert-store" import { saveState } from "../../db" import reporter from "gatsby-cli/lib/reporter" +/** + * These are the deferred redux actions sent from api-runner-node + * They may include a `resolve` prop (if they are createNode actions). + * If so, we resolve the promise when we're done + */ export const callRealApi = (event: IMutationAction, store?: Store): void => { assertStore(store) const { type, payload, resolve } = event if (type in actions) { + // If this is a createNode action then this will be a thunk. + // No worries, we just dispatch it like any other const action = actions[type](...payload) const result = store.dispatch(action) + // Somebody may be waiting for this if (resolve) { resolve(result) } @@ -31,6 +39,7 @@ export const callRealApi = (event: IMutationAction, store?: Store): void => { /** * Handler for when we're inside handlers that should be able to mutate nodes + * Instead of queueing, we call it right away */ export const callApi: ActionFunction = ( { store }, @@ -67,6 +76,9 @@ export const assignServiceResult = assign( (_context, { data }): DataLayerResult => data ) +/** + * This spawns the service that listens to the `emitter` for various mutation events + */ export const spawnMutationListener = assign({ mutationListener: () => spawn(listenForMutations, `listen-for-mutations`), }) diff --git a/packages/gatsby/src/state-machines/develop/index.ts b/packages/gatsby/src/state-machines/develop/index.ts index a8fc92dbfa0e4..61884b46fb6d9 100644 --- a/packages/gatsby/src/state-machines/develop/index.ts +++ b/packages/gatsby/src/state-machines/develop/index.ts @@ -12,7 +12,26 @@ import { IBuildContext } from "../../services" const developConfig: MachineConfig = { id: `build`, initial: `initializing`, + // These are mutation events, sent to this machine by the mutation listener + // in `services/listen-for-mutations.ts` + on: { + // These are deferred node mutations, mainly `createNode` + ADD_NODE_MUTATION: { + actions: `addNodeMutation`, + }, + // Sent by query watcher, these are chokidar file events. They mean we + // need to extract queries + QUERY_FILE_CHANGED: { + actions: `markQueryFilesDirty`, + }, + // These are calls to the refresh endpoint. Also used by Gatsby Preview + WEBHOOK_RECEIVED: { + target: `initializingDataLayer`, + actions: `assignWebhookBody`, + }, + }, states: { + // Here we handle the initial bootstrap initializing: { on: { // Ignore mutation events because we'll be running everything anyway @@ -28,8 +47,10 @@ const developConfig: MachineConfig = { }, }, }, + // Sourcing nodes, customising and inferring schema, then running createPages initializingDataLayer: { on: { + // We need to run mutations immediately when in this state ADD_NODE_MUTATION: { actions: [`markNodesDirty`, `callApi`], }, @@ -62,6 +83,7 @@ const developConfig: MachineConfig = { }, }, }, + // Running page and static queries and generating the SSRed HTML and page data runningQueries: { on: { QUERY_FILE_CHANGED: { @@ -71,6 +93,7 @@ const developConfig: MachineConfig = { invoke: { id: `run-queries`, src: `runQueries`, + // This is all the data that we're sending to the child machine data: ({ program, store, @@ -92,15 +115,19 @@ const developConfig: MachineConfig = { }, onDone: [ { + // If this is first run, start webpack and websocket servers target: `startingDevServers`, cond: ({ firstRun }: IBuildContext): boolean => !!firstRun, }, { + // ...otherwise just wait. xstate performs just the first transition + // that matches. target: `waiting`, }, ], }, }, + // Spin up webpack and socket.io startingDevServers: { invoke: { src: `startWebpackServer`, @@ -110,16 +137,19 @@ const developConfig: MachineConfig = { }, }, }, + // Idle, waiting for events that make us rebuild waiting: { // We may want to save this is more places, but this should do for now entry: `saveDbState`, on: { + // Forward these events to the child machine, so it can handle batching ADD_NODE_MUTATION: { actions: forwardTo(`waiting`), }, QUERY_FILE_CHANGED: { actions: forwardTo(`waiting`), }, + // This event is sent from the child EXTRACT_QUERIES_NOW: { target: `runningQueries`, }, @@ -127,18 +157,22 @@ const developConfig: MachineConfig = { invoke: { id: `waiting`, src: `waitForMutations`, + // Send existing queued mutations to the child machine, which will execute them data: ({ store, nodeMutationBatch = [], }: IBuildContext): IWaitingContext => { return { store, nodeMutationBatch, runningBatch: [] } }, + // "done" means we need to rebuild onDone: { actions: `assignServiceResult`, target: `rebuildingPages`, }, }, }, + // This is the same machine as initializingDataLayer, but sets the context to skip + // sourcing and jump straight to rebuilding pages rebuildingPages: { invoke: { src: `initializeDataLayer`, @@ -152,19 +186,6 @@ const developConfig: MachineConfig = { }, }, }, - // Transitions shared by all states, except where overridden - on: { - ADD_NODE_MUTATION: { - actions: `addNodeMutation`, - }, - QUERY_FILE_CHANGED: { - actions: `markQueryFilesDirty`, - }, - WEBHOOK_RECEIVED: { - target: `initializingDataLayer`, - actions: `assignWebhookBody`, - }, - }, } export const developMachine = Machine(developConfig, { diff --git a/packages/gatsby/src/state-machines/query-running/actions.ts b/packages/gatsby/src/state-machines/query-running/actions.ts index 57e69bbbb4204..d50f95fa7e883 100644 --- a/packages/gatsby/src/state-machines/query-running/actions.ts +++ b/packages/gatsby/src/state-machines/query-running/actions.ts @@ -1,7 +1,5 @@ import { IQueryRunningContext } from "./types" import { DoneInvokeEvent, assign, ActionFunctionMap } from "xstate" -import { GraphQLRunner } from "../../query/graphql-runner" -import { assertStore } from "../../utils/assert-store" import { enqueueFlush } from "../../utils/page-data" import { boundActionCreators } from "../../redux/actions" import { ProgramStatus } from "../../redux/types" @@ -20,19 +18,6 @@ export const assignDirtyQueries = assign< } }) -export const resetGraphQLRunner = assign< - IQueryRunningContext, - DoneInvokeEvent ->({ - graphqlRunner: ({ store, program }) => { - assertStore(store) - return new GraphQLRunner(store, { - collectStats: true, - graphqlTracing: program?.graphqlTracing, - }) - }, -}) - const finishUpQueries = async (): Promise => { boundActionCreators.setProgramStatus( ProgramStatus.BOOTSTRAP_QUERY_RUNNING_FINISHED @@ -43,7 +28,6 @@ export const queryActions: ActionFunctionMap< IQueryRunningContext, DoneInvokeEvent > = { - resetGraphQLRunner, assignDirtyQueries, flushPageData, finishUpQueries, diff --git a/packages/gatsby/src/state-machines/query-running/index.ts b/packages/gatsby/src/state-machines/query-running/index.ts index ea3b9da8c2526..288d61cb0c924 100644 --- a/packages/gatsby/src/state-machines/query-running/index.ts +++ b/packages/gatsby/src/state-machines/query-running/index.ts @@ -3,6 +3,10 @@ import { IQueryRunningContext } from "./types" import { queryRunningServices } from "./services" import { queryActions } from "./actions" +/** + * This is a child state machine, spawned to perform the query running + */ + export const queryStates: MachineConfig = { initial: `extractingQueries`, id: `queryRunningMachine`, @@ -56,7 +60,7 @@ export const queryStates: MachineConfig = { }, }, }, - + // This waits for the jobs API to finish waitingForJobs: { invoke: { src: `waitUntilAllJobsComplete`, @@ -69,6 +73,8 @@ export const queryStates: MachineConfig = { finishingUp: { on: { always: [ + // If there's an array of transitions xstate only + // performs the first one that matches a guard condition { target: `done`, actions: `finishUpQueries`, diff --git a/packages/gatsby/src/state-machines/waiting/index.ts b/packages/gatsby/src/state-machines/waiting/index.ts index 6446c194afeeb..09d120a1df398 100644 --- a/packages/gatsby/src/state-machines/waiting/index.ts +++ b/packages/gatsby/src/state-machines/waiting/index.ts @@ -8,6 +8,10 @@ const NODE_MUTATION_BATCH_TIMEOUT = 1000 export type WaitingResult = Pick +/** + * This idle state also handles batching of node mutations and running of + * mutations when we first start it + */ export const waitingStates: MachineConfig = { id: `waitingMachine`, initial: `idle`, @@ -46,6 +50,7 @@ export const waitingStates: MachineConfig = { on: { // More mutations added to batch ADD_NODE_MUTATION: [ + // You know the score: only run the first matching transition { // If this fills the batch then commit it actions: `addNodeMutation`, @@ -54,7 +59,7 @@ export const waitingStates: MachineConfig = { target: `committingBatch`, }, { - // otherwise just add it to the batch + // ...otherwise just add it to the batch actions: `addNodeMutation`, }, ], @@ -79,6 +84,7 @@ export const waitingStates: MachineConfig = { }, invoke: { src: `runMutationBatch`, + // When we're done, clear the running batch ready for next time onDone: { actions: assign({ runningBatch: [], @@ -89,6 +95,8 @@ export const waitingStates: MachineConfig = { }, rebuild: { type: `final`, + // This is returned to the parent. The batch includes + // any mutations that arrived while we were running the other batch data: ({ nodeMutationBatch }): WaitingResult => { return { nodeMutationBatch } }, From 38c05f5da9aa12e0fb44d5c4c234a4b97bfc41ef Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Fri, 17 Jul 2020 11:34:19 +0100 Subject: [PATCH 42/44] Remove first run from query running --- .../gatsby/src/state-machines/develop/actions.ts | 10 +++++++++- .../gatsby/src/state-machines/develop/index.ts | 3 +-- .../src/state-machines/query-running/actions.ts | 9 --------- .../src/state-machines/query-running/index.ts | 16 ---------------- 4 files changed, 10 insertions(+), 28 deletions(-) diff --git a/packages/gatsby/src/state-machines/develop/actions.ts b/packages/gatsby/src/state-machines/develop/actions.ts index 8f8f8f94f92d3..6773cd5f3460a 100644 --- a/packages/gatsby/src/state-machines/develop/actions.ts +++ b/packages/gatsby/src/state-machines/develop/actions.ts @@ -8,12 +8,13 @@ import { } from "xstate" import { Store } from "redux" import { IBuildContext, IMutationAction } from "../../services" -import { actions } from "../../redux/actions" +import { actions, boundActionCreators } from "../../redux/actions" import { listenForMutations } from "../../services/listen-for-mutations" import { DataLayerResult } from "../data-layer" import { assertStore } from "../../utils/assert-store" import { saveState } from "../../db" import reporter from "gatsby-cli/lib/reporter" +import { ProgramStatus } from "../../redux/types" /** * These are the deferred redux actions sent from api-runner-node @@ -68,6 +69,12 @@ export const assignStoreAndWorkerPool = assign( } ) +const setQueryRunningFinished = async (): Promise => { + boundActionCreators.setProgramStatus( + ProgramStatus.BOOTSTRAP_QUERY_RUNNING_FINISHED + ) +} + export const markQueryFilesDirty = assign({ queryFilesDirty: true, }) @@ -130,4 +137,5 @@ export const buildActions: ActionFunctionMap = { clearWebhookBody, finishParentSpan, saveDbState, + setQueryRunningFinished, } diff --git a/packages/gatsby/src/state-machines/develop/index.ts b/packages/gatsby/src/state-machines/develop/index.ts index 61884b46fb6d9..d4e4899f68977 100644 --- a/packages/gatsby/src/state-machines/develop/index.ts +++ b/packages/gatsby/src/state-machines/develop/index.ts @@ -101,10 +101,8 @@ const developConfig: MachineConfig = { gatsbyNodeGraphQLFunction, graphqlRunner, websocketManager, - firstRun, }: IBuildContext): IQueryRunningContext => { return { - firstRun, program, store, parentSpan, @@ -117,6 +115,7 @@ const developConfig: MachineConfig = { { // If this is first run, start webpack and websocket servers target: `startingDevServers`, + actions: `setQueryRunningFinished`, cond: ({ firstRun }: IBuildContext): boolean => !!firstRun, }, { diff --git a/packages/gatsby/src/state-machines/query-running/actions.ts b/packages/gatsby/src/state-machines/query-running/actions.ts index d50f95fa7e883..fe9013a4ee104 100644 --- a/packages/gatsby/src/state-machines/query-running/actions.ts +++ b/packages/gatsby/src/state-machines/query-running/actions.ts @@ -1,8 +1,6 @@ import { IQueryRunningContext } from "./types" import { DoneInvokeEvent, assign, ActionFunctionMap } from "xstate" import { enqueueFlush } from "../../utils/page-data" -import { boundActionCreators } from "../../redux/actions" -import { ProgramStatus } from "../../redux/types" export const flushPageData = (): void => { enqueueFlush() @@ -18,17 +16,10 @@ export const assignDirtyQueries = assign< } }) -const finishUpQueries = async (): Promise => { - boundActionCreators.setProgramStatus( - ProgramStatus.BOOTSTRAP_QUERY_RUNNING_FINISHED - ) -} - export const queryActions: ActionFunctionMap< IQueryRunningContext, DoneInvokeEvent > = { assignDirtyQueries, flushPageData, - finishUpQueries, } diff --git a/packages/gatsby/src/state-machines/query-running/index.ts b/packages/gatsby/src/state-machines/query-running/index.ts index 288d61cb0c924..7cc23417aae59 100644 --- a/packages/gatsby/src/state-machines/query-running/index.ts +++ b/packages/gatsby/src/state-machines/query-running/index.ts @@ -70,22 +70,6 @@ export const queryStates: MachineConfig = { }, }, }, - finishingUp: { - on: { - always: [ - // If there's an array of transitions xstate only - // performs the first one that matches a guard condition - { - target: `done`, - actions: `finishUpQueries`, - cond: ({ firstRun }): boolean => !!firstRun, - }, - { - target: `done`, - }, - ], - }, - }, done: { type: `final`, }, From 98936a8f9392e0593abb1ecac2d7dd9d7e44eb3b Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Fri, 17 Jul 2020 12:46:58 +0100 Subject: [PATCH 43/44] Refactor into separate data layer machines --- packages/gatsby/src/bootstrap/index.ts | 1 - .../gatsby/src/commands/develop-process.ts | 1 - packages/gatsby/src/services/create-pages.ts | 3 +- packages/gatsby/src/services/types.ts | 1 - .../src/state-machines/data-layer/index.ts | 251 +++++++++++------- .../src/state-machines/data-layer/services.ts | 4 +- .../src/state-machines/data-layer/types.ts | 2 - .../src/state-machines/develop/actions.ts | 1 - .../src/state-machines/develop/index.ts | 68 +++-- .../src/state-machines/develop/services.ts | 10 +- 10 files changed, 215 insertions(+), 127 deletions(-) diff --git a/packages/gatsby/src/bootstrap/index.ts b/packages/gatsby/src/bootstrap/index.ts index 0bb37c8b7ab9c..5e8ec23d599d7 100644 --- a/packages/gatsby/src/bootstrap/index.ts +++ b/packages/gatsby/src/bootstrap/index.ts @@ -33,7 +33,6 @@ export async function bootstrap( const bootstrapContext: IBuildContext = { ...initialContext, parentSpan, - firstRun: true, } const context = { diff --git a/packages/gatsby/src/commands/develop-process.ts b/packages/gatsby/src/commands/develop-process.ts index c4d52f51176ac..80e2e0c83b9b5 100644 --- a/packages/gatsby/src/commands/develop-process.ts +++ b/packages/gatsby/src/commands/develop-process.ts @@ -125,7 +125,6 @@ module.exports = async (program: IDevelopArgs): Promise => { program, parentSpan, app, - firstRun: true, }) const service = interpret(machine) diff --git a/packages/gatsby/src/services/create-pages.ts b/packages/gatsby/src/services/create-pages.ts index 343b734cdd120..797da30ecce66 100644 --- a/packages/gatsby/src/services/create-pages.ts +++ b/packages/gatsby/src/services/create-pages.ts @@ -9,7 +9,6 @@ export async function createPages({ parentSpan, gatsbyNodeGraphQLFunction, store, - firstRun, }: Partial): Promise<{ deletedPages: string[] changedPages: string[] @@ -26,7 +25,7 @@ export async function createPages({ `createPages`, { graphql: gatsbyNodeGraphQLFunction, - traceId: firstRun ? `initial-createPages` : `createPages`, + traceId: `initial-createPages`, waitForCascadingActions: true, parentSpan: activity.span, }, diff --git a/packages/gatsby/src/services/types.ts b/packages/gatsby/src/services/types.ts index 25432115141f1..6a023df96242f 100644 --- a/packages/gatsby/src/services/types.ts +++ b/packages/gatsby/src/services/types.ts @@ -21,7 +21,6 @@ export interface IMutationAction { resolve?: (result: unknown) => void } export interface IBuildContext { - firstRun?: boolean program?: IProgram store?: Store parentSpan?: Span diff --git a/packages/gatsby/src/state-machines/data-layer/index.ts b/packages/gatsby/src/state-machines/data-layer/index.ts index 1e70213e1fa5f..f1a02f30d0235 100644 --- a/packages/gatsby/src/state-machines/data-layer/index.ts +++ b/packages/gatsby/src/state-machines/data-layer/index.ts @@ -1,4 +1,4 @@ -import { MachineConfig, Machine } from "xstate" +import { Machine, StatesConfig, MachineOptions } from "xstate" import { dataLayerActions } from "./actions" import { IDataLayerContext } from "./types" import { dataLayerServices } from "./services" @@ -11,125 +11,182 @@ export type DataLayerResult = Pick< | "pagesToDelete" > -const dataLayerStates: MachineConfig = { - initial: `start`, - id: `dataLayerMachine`, - context: {}, - states: { - start: { - always: [ - { - target: `buildingSchema`, - cond: `shouldSkipSourcing`, - }, - { - target: `customizingSchema`, - }, - ], +const loadDataStates: StatesConfig = { + customizingSchema: { + invoke: { + src: `customizeSchema`, + id: `customizing-schema`, + onDone: { + target: `sourcingNodes`, + }, }, - customizingSchema: { - invoke: { - src: `customizeSchema`, - id: `customizing-schema`, - onDone: { - target: `sourcingNodes`, - }, + }, + sourcingNodes: { + invoke: { + src: `sourceNodes`, + id: `sourcing-nodes`, + onDone: { + target: `buildingSchema`, + actions: `assignChangedPages`, }, }, - sourcingNodes: { - invoke: { - src: `sourceNodes`, - id: `sourcing-nodes`, - onDone: { - target: `buildingSchema`, - actions: `assignChangedPages`, - }, + }, +} + +const initialCreatePagesStates: StatesConfig = { + buildingSchema: { + invoke: { + id: `building-schema`, + src: `buildSchema`, + onDone: { + target: `creatingPages`, + actions: `assignGraphQLRunners`, }, }, - buildingSchema: { - invoke: { - id: `building-schema`, - src: `buildSchema`, - onDone: { - target: `creatingPages`, - actions: `assignGraphQLRunners`, - }, + }, + creatingPages: { + on: { ADD_NODE_MUTATION: { actions: [`markNodesDirty`, `callApi`] } }, + invoke: { + id: `creating-pages`, + src: `createPages`, + onDone: { + target: `creatingPagesStatefully`, + actions: `assignChangedPages`, }, }, - creatingPages: { - on: { ADD_NODE_MUTATION: { actions: [`markNodesDirty`, `callApi`] } }, - invoke: { - id: `creating-pages`, - src: `createPages`, - onDone: [ - { - target: `creatingPagesStatefully`, - actions: `assignChangedPages`, - cond: `firstRun`, - }, - { - target: `rebuildingSchemaWithSitePage`, - actions: `assignChangedPages`, - }, - ], + }, + creatingPagesStatefully: { + invoke: { + src: `createPagesStatefully`, + id: `creating-pages-statefully`, + onDone: { + target: `rebuildingSchemaWithSitePage`, }, }, - creatingPagesStatefully: { - invoke: { - src: `createPagesStatefully`, - id: `creating-pages-statefully`, - onDone: { - target: `rebuildingSchemaWithSitePage`, - }, + }, + rebuildingSchemaWithSitePage: { + invoke: { + src: `rebuildSchemaWithSitePage`, + onDone: { + target: `writingOutRedirects`, }, }, - rebuildingSchemaWithSitePage: { - invoke: { - src: `rebuildSchemaWithSitePage`, - onDone: [ - { - target: `writingOutRedirects`, - cond: `firstRun`, - }, - { - target: `done`, - }, - ], + }, + writingOutRedirects: { + invoke: { + src: `writeOutRedirectsAndWatch`, + onDone: { + target: `done`, }, }, - writingOutRedirects: { - invoke: { - src: `writeOutRedirects`, - onDone: { - target: `done`, - }, + }, +} + +const recreatePagesStates: StatesConfig = { + buildingSchema: { + invoke: { + id: `building-schema`, + src: `buildSchema`, + onDone: { + target: `creatingPages`, + actions: `assignGraphQLRunners`, + }, + }, + }, + creatingPages: { + on: { ADD_NODE_MUTATION: { actions: [`markNodesDirty`, `callApi`] } }, + invoke: { + id: `creating-pages`, + src: `createPages`, + onDone: { + target: `rebuildingSchemaWithSitePage`, + actions: `assignChangedPages`, }, }, - done: { - type: `final`, - data: ({ + }, + rebuildingSchemaWithSitePage: { + invoke: { + src: `rebuildSchemaWithSitePage`, + onDone: { + target: `done`, + }, + }, + }, +} + +const doneState: StatesConfig = { + done: { + type: `final`, + data: ({ + gatsbyNodeGraphQLFunction, + graphqlRunner, + pagesToBuild, + pagesToDelete, + }): DataLayerResult => { + return { gatsbyNodeGraphQLFunction, graphqlRunner, pagesToBuild, pagesToDelete, - }): DataLayerResult => { - return { - gatsbyNodeGraphQLFunction, - graphqlRunner, - pagesToBuild, - pagesToDelete, - } - }, + } }, }, } -export const dataLayerMachine = Machine(dataLayerStates, { +const options: Partial> = { actions: dataLayerActions, services: dataLayerServices, - guards: { - firstRun: ({ firstRun }: IDataLayerContext): boolean => !!firstRun, - shouldSkipSourcing: ({ skipSourcing }: IDataLayerContext): boolean => - !!skipSourcing, +} + +/** + * Machine used during first run + */ + +export const initializeDataMachine = Machine( + { + id: `initializeDataMachine`, + context: {}, + initial: `customizingSchema`, + states: { + ...loadDataStates, + ...initialCreatePagesStates, + ...doneState, + }, + }, + options +) + +/** + * Machine used when we need to source nodes again + */ + +export const reloadDataMachine = Machine( + { + id: `reloadDataMachine`, + context: {}, + initial: `customizingSchema`, + states: { + ...loadDataStates, + ...recreatePagesStates, + ...doneState, + }, + }, + options +) + +/** + * Machine used when we need to re-create pages after a + * node mutation outside of sourceNodes + */ +export const recreatePagesMachine = Machine( + { + id: `recreatePagesMachine`, + context: {}, + initial: `buildingSchema`, + states: { + ...recreatePagesStates, + ...doneState, + }, }, -}) + options +) diff --git a/packages/gatsby/src/state-machines/data-layer/services.ts b/packages/gatsby/src/state-machines/data-layer/services.ts index 4871e90d2d0a1..b8271a67b1107 100644 --- a/packages/gatsby/src/state-machines/data-layer/services.ts +++ b/packages/gatsby/src/state-machines/data-layer/services.ts @@ -6,7 +6,7 @@ import { buildSchema, sourceNodes, rebuildSchemaWithSitePage, - writeOutRedirects, + writeOutRedirects as writeOutRedirectsAndWatch, } from "../../services" import { IDataLayerContext } from "./types" @@ -20,5 +20,5 @@ export const dataLayerServices: Record< buildSchema, createPagesStatefully, rebuildSchemaWithSitePage, - writeOutRedirects, + writeOutRedirectsAndWatch, } diff --git a/packages/gatsby/src/state-machines/data-layer/types.ts b/packages/gatsby/src/state-machines/data-layer/types.ts index ecc3940614248..3df3f4628e775 100644 --- a/packages/gatsby/src/state-machines/data-layer/types.ts +++ b/packages/gatsby/src/state-machines/data-layer/types.ts @@ -16,9 +16,7 @@ export interface IMutationAction { } export interface IDataLayerContext { deferNodeMutation?: boolean - skipSourcing?: boolean nodesMutatedDuringQueryRun?: boolean - firstRun?: boolean program?: IProgram store?: Store parentSpan?: Span diff --git a/packages/gatsby/src/state-machines/develop/actions.ts b/packages/gatsby/src/state-machines/develop/actions.ts index 6773cd5f3460a..f2619fc6e76c0 100644 --- a/packages/gatsby/src/state-machines/develop/actions.ts +++ b/packages/gatsby/src/state-machines/develop/actions.ts @@ -94,7 +94,6 @@ export const assignServers = assign( (_context, { data }) => { return { ...data, - firstRun: false, } } ) diff --git a/packages/gatsby/src/state-machines/develop/index.ts b/packages/gatsby/src/state-machines/develop/index.ts index d4e4899f68977..93362427908de 100644 --- a/packages/gatsby/src/state-machines/develop/index.ts +++ b/packages/gatsby/src/state-machines/develop/index.ts @@ -24,9 +24,10 @@ const developConfig: MachineConfig = { QUERY_FILE_CHANGED: { actions: `markQueryFilesDirty`, }, - // These are calls to the refresh endpoint. Also used by Gatsby Preview + // These are calls to the refresh endpoint. Also used by Gatsby Preview. + // Saves the webhook body from the event into context, then reloads data WEBHOOK_RECEIVED: { - target: `initializingDataLayer`, + target: `reloadingData`, actions: `assignWebhookBody`, }, }, @@ -42,13 +43,13 @@ const developConfig: MachineConfig = { invoke: { src: `initialize`, onDone: { - target: `initializingDataLayer`, + target: `initializingData`, actions: [`assignStoreAndWorkerPool`, `spawnMutationListener`], }, }, }, // Sourcing nodes, customising and inferring schema, then running createPages - initializingDataLayer: { + initializingData: { on: { // We need to run mutations immediately when in this state ADD_NODE_MUTATION: { @@ -58,19 +59,17 @@ const developConfig: MachineConfig = { QUERY_FILE_CHANGED: undefined, }, invoke: { - src: `initializeDataLayer`, + src: `initializeData`, data: ({ parentSpan, store, - firstRun, webhookBody, }: IBuildContext): IDataLayerContext => { return { parentSpan, store, - firstRun, - deferNodeMutation: true, webhookBody, + deferNodeMutation: true, } }, onDone: { @@ -113,14 +112,14 @@ const developConfig: MachineConfig = { }, onDone: [ { - // If this is first run, start webpack and websocket servers + // If we have no compiler (i.e. it's first run), then spin up the + // webpack and socket.io servers target: `startingDevServers`, actions: `setQueryRunningFinished`, - cond: ({ firstRun }: IBuildContext): boolean => !!firstRun, + cond: ({ compiler }: IBuildContext): boolean => !!compiler, }, { - // ...otherwise just wait. xstate performs just the first transition - // that matches. + // ...otherwise just wait. target: `waiting`, }, ], @@ -166,17 +165,50 @@ const developConfig: MachineConfig = { // "done" means we need to rebuild onDone: { actions: `assignServiceResult`, - target: `rebuildingPages`, + target: `recreatingPages`, + }, + }, + }, + // Almost the same as initializing data, but skips various first-run stuff + reloadingData: { + on: { + // We need to run mutations immediately when in this state + ADD_NODE_MUTATION: { + actions: [`markNodesDirty`, `callApi`], + }, + // Ignore, because we're about to extract them anyway + QUERY_FILE_CHANGED: undefined, + }, + invoke: { + src: `reloadData`, + data: ({ + parentSpan, + store, + webhookBody, + }: IBuildContext): IDataLayerContext => { + return { + parentSpan, + store, + webhookBody, + deferNodeMutation: true, + } + }, + onDone: { + actions: [ + `assignServiceResult`, + `clearWebhookBody`, + `finishParentSpan`, + ], + target: `runningQueries`, }, }, }, - // This is the same machine as initializingDataLayer, but sets the context to skip - // sourcing and jump straight to rebuilding pages - rebuildingPages: { + // Rebuild pages if a node has been mutated outside of sourceNodes + recreatingPages: { invoke: { - src: `initializeDataLayer`, + src: `recreatePages`, data: ({ parentSpan, store }: IBuildContext): IDataLayerContext => { - return { parentSpan, store, firstRun: false, skipSourcing: true } + return { parentSpan, store, deferNodeMutation: true } }, onDone: { actions: `assignServiceResult`, diff --git a/packages/gatsby/src/state-machines/develop/services.ts b/packages/gatsby/src/state-machines/develop/services.ts index a233e8eadacf3..6f94baaebd00c 100644 --- a/packages/gatsby/src/state-machines/develop/services.ts +++ b/packages/gatsby/src/state-machines/develop/services.ts @@ -1,11 +1,17 @@ import { IBuildContext, startWebpackServer, initialize } from "../../services" -import { dataLayerMachine } from "../data-layer" +import { + initializeDataMachine, + reloadDataMachine, + recreatePagesMachine, +} from "../data-layer" import { queryRunningMachine } from "../query-running" import { waitingMachine } from "../waiting" import { ServiceConfig } from "xstate" export const developServices: Record> = { - initializeDataLayer: dataLayerMachine, + initializeData: initializeDataMachine, + reloadData: reloadDataMachine, + recreatePages: recreatePagesMachine, initialize: initialize, runQueries: queryRunningMachine, waitForMutations: waitingMachine, From 3f758f534f5b0e4a274884f1267ff0b6ed8a29b0 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Fri, 17 Jul 2020 15:51:41 +0100 Subject: [PATCH 44/44] Fix condition --- packages/gatsby/src/state-machines/develop/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gatsby/src/state-machines/develop/index.ts b/packages/gatsby/src/state-machines/develop/index.ts index 93362427908de..cd79e4d9831f4 100644 --- a/packages/gatsby/src/state-machines/develop/index.ts +++ b/packages/gatsby/src/state-machines/develop/index.ts @@ -116,7 +116,7 @@ const developConfig: MachineConfig = { // webpack and socket.io servers target: `startingDevServers`, actions: `setQueryRunningFinished`, - cond: ({ compiler }: IBuildContext): boolean => !!compiler, + cond: ({ compiler }: IBuildContext): boolean => !compiler, }, { // ...otherwise just wait.