feat(gatsby): Defer node mutation during querying (#25479)

* Move bootstrap into machine

* Add parent span and query extraction

* Add rebuildSchemaWithSitePage

* Use values from context

* Remove logs

* Add redirectListener

* Changes from review

* Log child state transitions

* Add state machine for query running

* Changes from review

* Changes from review

* Switch to reporter

* Use assertStore

* Remove unused action

* Remove unusued config

* Remove unusued config

* Add gql runner reset

* Handle node mutation queuing and batching in state machine

* Use new pagedata utils

* Use develop queue

* New xstate syntax

* Work-around xstate bug

* Track first run

* Track first run

* Disable --quiet in e2e

* Don't defer node mutation if we're outside the state machine

* Re-quieten e2e

* Listen for query file changes

* Lint

* Handle webhook

* Changes from review

* Fix typings

* Changes from review

* Typefix

* feat(gatsby): Move final parts into develop state machine (#25716)

* Move remaining parts into state machine

* Move top level state machine into state machines dir

* Add machine ids

* Add missing imports

* Resolve api promises

* Remove unused action

* Move logging into helper

* Changes from review

* Manually save db

* Add comments

* Remove first run from query running

* Refactor into separate data layer machines

* Fix condition

Co-authored-by: gatsbybot <mathews.kyle+gatsbybot@gmail.com>

packages/gatsby/src/bootstrap/index.ts

@@ -33,7 +33,6 @@ export async function bootstrap(
   const bootstrapContext: IBuildContext = {
     ...initialContext,
     parentSpan,
-    firstRun: true,
   }
 
   const context = {

packages/gatsby/src/commands/develop-process.ts

@@ -4,51 +4,20 @@ import chalk from "chalk"
 import telemetry from "gatsby-telemetry"
 import express from "express"
 import inspector from "inspector"
-import { bootstrapSchemaHotReloader } from "../bootstrap/schema-hot-reloader"
-import bootstrapPageHotReloader from "../bootstrap/page-hot-reloader"
 import { initTracer } from "../utils/tracer"
-import db from "../db"
 import { detectPortInUseAndPrompt } from "../utils/detect-port-in-use-and-prompt"
 import onExit from "signal-exit"
-import queryUtil from "../query"
-import queryWatcher from "../query/query-watcher"
-import * as requiresWriter from "../bootstrap/requires-writer"
-import { waitUntilAllJobsComplete } from "../utils/wait-until-jobs-complete"
 import {
   userPassesFeedbackRequestHeuristic,
   showFeedbackRequest,
 } from "../utils/feedback"
-import { startRedirectListener } from "../bootstrap/redirects-writer"
 import { markWebpackStatusAsPending } from "../utils/webpack-status"
 
 import { IProgram, IDebugInfo } from "./types"
-import {
-  startWebpackServer,
-  writeOutRequires,
-  IBuildContext,
-  initialize,
-  postBootstrap,
-  rebuildSchemaWithSitePage,
-  writeOutRedirects,
-} from "../services"
-import { boundActionCreators } from "../redux/actions"
-import { ProgramStatus } from "../redux/types"
-import {
-  MachineConfig,
-  AnyEventObject,
-  assign,
-  Machine,
-  DoneEventObject,
-  interpret,
-  Actor,
-  Interpreter,
-  State,
-} from "xstate"
-import { DataLayerResult, dataLayerMachine } from "../state-machines/data-layer"
-import { IDataLayerContext } from "../state-machines/data-layer/types"
+import { interpret } from "xstate"
 import { globalTracer } from "opentracing"
-import { IQueryRunningContext } from "../state-machines/query-running/types"
-import { queryRunningMachine } from "../state-machines/query-running"
+import { developMachine } from "../state-machines/develop"
+import { logTransitions } from "../utils/state-machine-logging"
 
 const tracer = globalTracer()
 
@@ -100,12 +69,14 @@ const openDebuggerPort = (debugInfo: IDebugInfo): void => {
 }
 
 module.exports = async (program: IDevelopArgs): Promise<void> => {
+  if (program.verbose) {
+    reporter.setVerbose(true)
+  }
+
   if (program.debugInfo) {
     openDebuggerPort(program.debugInfo)
   }
 
-  const bootstrapSpan = tracer.startSpan(`bootstrap`)
-
   // We want to prompt the feedback request when users quit develop
   // assuming they pass the heuristic check to know they are a user
   // we want to request feedback from, and we're not annoying them.
@@ -148,178 +119,19 @@ module.exports = async (program: IDevelopArgs): Promise<void> => {
   }
 
   const app = express()
+  const parentSpan = tracer.startSpan(`bootstrap`)
 
-  const developConfig: MachineConfig<IBuildContext, any, AnyEventObject> = {
-    id: `build`,
-    initial: `initializing`,
-    states: {
-      initializing: {
-        invoke: {
-          src: `initialize`,
-          onDone: {
-            target: `initializingDataLayer`,
-            actions: `assignStoreAndWorkerPool`,
-          },
-        },
-      },
-      initializingDataLayer: {
-        invoke: {
-          src: `initializeDataLayer`,
-          data: ({ parentSpan, store }: IBuildContext): IDataLayerContext => {
-            return { parentSpan, store, firstRun: true }
-          },
-          onDone: {
-            actions: `assignDataLayer`,
-            target: `finishingBootstrap`,
-          },
-        },
-      },
-      finishingBootstrap: {
-        invoke: {
-          src: async ({
-            gatsbyNodeGraphQLFunction,
-          }: IBuildContext): Promise<void> => {
-            // These were previously in `bootstrap()` but are now
-            // in part of the state machine that hasn't been added yet
-            await rebuildSchemaWithSitePage({ parentSpan: bootstrapSpan })
-
-            await writeOutRedirects({ parentSpan: bootstrapSpan })
-
-            startRedirectListener()
-            bootstrapSpan.finish()
-            await postBootstrap({ parentSpan: bootstrapSpan })
-
-            // These are the parts that weren't in bootstrap
-
-            // Start the createPages hot reloader.
-            bootstrapPageHotReloader(gatsbyNodeGraphQLFunction)
-
-            // Start the schema hot reloader.
-            bootstrapSchemaHotReloader()
-          },
-          onDone: {
-            target: `runningQueries`,
-          },
-        },
-      },
-      runningQueries: {
-        invoke: {
-          src: `runQueries`,
-          data: ({
-            program,
-            store,
-            parentSpan,
-            gatsbyNodeGraphQLFunction,
-            graphqlRunner,
-            firstRun,
-          }: IBuildContext): IQueryRunningContext => {
-            return {
-              firstRun,
-              program,
-              store,
-              parentSpan,
-              gatsbyNodeGraphQLFunction,
-              graphqlRunner,
-            }
-          },
-          onDone: {
-            target: `doingEverythingElse`,
-          },
-        },
-      },
-      doingEverythingElse: {
-        invoke: {
-          src: async ({ workerPool, store, app }): Promise<void> => {
-            // All the stuff that's not in the state machine yet
-
-            await writeOutRequires({ store })
-            boundActionCreators.setProgramStatus(
-              ProgramStatus.BOOTSTRAP_QUERY_RUNNING_FINISHED
-            )
-
-            await db.saveState()
+  const machine = developMachine.withContext({
+    program,
+    parentSpan,
+    app,
+  })
 
-            await waitUntilAllJobsComplete()
-            requiresWriter.startListener()
-            db.startAutosave()
-            queryUtil.startListeningToDevelopQueue({
-              graphqlTracing: program.graphqlTracing,
-            })
-            queryWatcher.startWatchDeletePage()
+  const service = interpret(machine)
 
-            await startWebpackServer({ program, app, workerPool, store })
-          },
-          onDone: {
-            actions: assign<IBuildContext, any>({ firstRun: false }),
-          },
-        },
-      },
-    },
+  if (program.verbose) {
+    logTransitions(service)
   }
 
-  const service = interpret(
-    Machine(developConfig, {
-      services: {
-        initializeDataLayer: dataLayerMachine,
-        initialize,
-        runQueries: queryRunningMachine,
-      },
-      actions: {
-        assignStoreAndWorkerPool: assign<IBuildContext, DoneEventObject>(
-          (_context, event) => {
-            const { store, workerPool } = event.data
-            return {
-              store,
-              workerPool,
-            }
-          }
-        ),
-        assignDataLayer: assign<IBuildContext, DoneEventObject>(
-          (_, { data }): DataLayerResult => data
-        ),
-      },
-    }).withContext({ program, parentSpan: bootstrapSpan, app, firstRun: true })
-  )
-
-  const isInterpreter = <T>(
-    actor: Actor<T> | Interpreter<T>
-  ): actor is Interpreter<T> => `machine` in actor
-
-  const listeners = new WeakSet()
-  let last: State<IBuildContext, AnyEventObject, any, any>
-
-  service.onTransition(state => {
-    if (!last) {
-      last = state
-    } else if (!state.changed || last.matches(state)) {
-      return
-    }
-    last = state
-    reporter.verbose(`Transition to ${JSON.stringify(state.value)}`)
-    // eslint-disable-next-line no-unused-expressions
-    service.children?.forEach(child => {
-      // We want to ensure we don't attach a listener to the same
-      // actor. We don't need to worry about detaching the listener
-      // because xstate handles that for us when the actor is stopped.
-
-      if (isInterpreter(child) && !listeners.has(child)) {
-        let sublast = child.state
-        child.onTransition(substate => {
-          if (!sublast) {
-            sublast = substate
-          } else if (!substate.changed || sublast.matches(substate)) {
-            return
-          }
-          sublast = substate
-          reporter.verbose(
-            `Transition to ${JSON.stringify(state.value)} > ${JSON.stringify(
-              substate.value
-            )}`
-          )
-        })
-        listeners.add(child)
-      }
-    })
-  })
   service.start()
 }

packages/gatsby/src/commands/types.ts

@@ -31,6 +31,7 @@ export interface IProgram {
   inspect?: number
   inspectBrk?: number
   graphqlTracing?: boolean
+  verbose?: boolean
   setStore?: (store: Store<IGatsbyState, AnyAction>) => void
 }
 

packages/gatsby/src/query/index.js

@@ -366,6 +366,7 @@ const enqueueExtractedPageComponent = componentPath => {
 
 module.exports = {
   calcInitialDirtyQueryIds,
+  calcDirtyQueryIds,
   processPageQueries,
   processStaticQueries,
   groupQueryIds,

packages/gatsby/src/query/query-watcher.js

@@ -8,7 +8,6 @@
  * - Whenever a query changes, re-run all pages that rely on this query.
  ***/
 
-const _ = require(`lodash`)
 const chokidar = require(`chokidar`)
 
 const path = require(`path`)
@@ -196,8 +195,7 @@ exports.extractQueries = ({ parentSpan } = {}) => {
     // During development start watching files to recompile & run
     // queries on the fly.
 
-    // TODO: move this into a spawned service, and emit events rather than
-    // directly triggering the compilation
+    // TODO: move this into a spawned service
     if (process.env.NODE_ENV !== `production`) {
       watch(store.getState().program.directory)
     }
@@ -222,10 +220,6 @@ const watchComponent = componentPath => {
   }
 }
 
-const debounceCompile = _.debounce(() => {
-  updateStateAndRunQueries()
-}, 100)
-
 const watch = async rootDir => {
   if (watcher) return
 
@@ -238,13 +232,18 @@ const watch = async rootDir => {
   })
 
   watcher = chokidar
-    .watch([
-      slash(path.join(rootDir, `/src/**/*.{js,jsx,ts,tsx}`)),
-      ...packagePaths,
-    ])
+    .watch(
+      [slash(path.join(rootDir, `/src/**/*.{js,jsx,ts,tsx}`)), ...packagePaths],
+      { ignoreInitial: true }
+    )
     .on(`change`, path => {
-      report.pendingActivity({ id: `query-extraction` })
-      debounceCompile()
+      emitter.emit(`QUERY_FILE_CHANGED`, path)
+    })
+    .on(`add`, path => {
+      emitter.emit(`QUERY_FILE_CHANGED`, path)
+    })
+    .on(`unlink`, path => {
+      emitter.emit(`QUERY_FILE_CHANGED`, path)
     })
 
   filesToWatch.forEach(filePath => watcher.add(filePath))

packages/gatsby/src/services/calculate-dirty-queries.ts

@@ -1,18 +1,24 @@
-import { calcInitialDirtyQueryIds, groupQueryIds } from "../query"
+import {
+  calcInitialDirtyQueryIds,
+  calcDirtyQueryIds,
+  groupQueryIds,
+} from "../query"
 import { IGroupedQueryIds } from "./"
 import { IQueryRunningContext } from "../state-machines/query-running/types"
 import { assertStore } from "../utils/assert-store"
 
 export async function calculateDirtyQueries({
   store,
+  firstRun,
 }: Partial<IQueryRunningContext>): Promise<{
   queryIds: IGroupedQueryIds
 }> {
   assertStore(store)
 
   const state = store.getState()
-  // TODO: Check filesDirty from context
 
-  const queryIds = calcInitialDirtyQueryIds(state)
+  const queryIds = firstRun
+    ? calcInitialDirtyQueryIds(state)
+    : calcDirtyQueryIds(state)
   return { queryIds: groupQueryIds(queryIds) }
 }

packages/gatsby/src/services/create-pages-statefully.ts

@@ -5,6 +5,7 @@ import { IDataLayerContext } from "../state-machines/data-layer/types"
 export async function createPagesStatefully({
   parentSpan,
   gatsbyNodeGraphQLFunction,
+  deferNodeMutation,
 }: Partial<IDataLayerContext>): Promise<void> {
   // A variant on createPages for plugins that want to
   // have full control over adding/removing pages. The normal
@@ -21,7 +22,7 @@ export async function createPagesStatefully({
       traceId: `initial-createPagesStatefully`,
       waitForCascadingActions: true,
       parentSpan: activity.span,
-      // deferNodeMutation: true, //later
+      deferNodeMutation,
     },
     {
       activity,