[v2] improve how we load GraphQL query results in development/production (aka Ludicrous Mode)

packages/gatsby-source-filesystem/package.json

@@ -16,7 +16,8 @@
  "mime": "^2.2.0",
  "pretty-bytes": "^4.0.2",
  "slash": "^1.0.0",
- "valid-url": "^1.0.9"
+ "valid-url": "^1.0.9",
+ "xstate": "^3.1.0"
  },
  "devDependencies": {
  "@babel/cli": "^7.0.0-beta.42",

packages/gatsby-source-filesystem/src/create-file-node.js

@@ -7,13 +7,6 @@ const prettyBytes = require(`pretty-bytes`)
 const md5File = require(`bluebird`).promisify(require(`md5-file`))
 const crypto = require(`crypto`)
 
-const createId = path => {
- const slashed = slash(path)
- return `${slashed} absPath of file`
-}
-
-exports.createId = createId
-
 exports.createFileNode = async (
  pathToFile,
  createNodeId,

packages/gatsby-source-filesystem/src/gatsby-node.js

@@ -1,10 +1,77 @@
 const chokidar = require(`chokidar`)
 const fs = require(`fs`)
+const { Machine } = require(`xstate`)
 
 const { createId, createFileNode } = require(`./create-file-node`)
 
+/**
+ * Create a state machine to manage Chokidar's not-ready/ready states and for
+ * emitting file system events into Gatsby.
+ *
+ * On the latter, this solves the problem where if you call createNode for the
+ * same File node in quick succession, this can leave Gatsby's internal state
+ * in disarray causing queries to fail. The latter state machine tracks when
+ * Gatsby is "processing" a node update or when it's "idle". If updates come in
+ * while Gatsby is processing, we queue them until the system returns to an
+ * "idle" state.
+ */
+const fsMachine = Machine({
+ key: "emitFSEvents",
+ parallel: true,
+ strict: true,
+ states: {
+ CHOKIDAR: {
+ initial: `CHOKIDAR_NOT_READY`,
+ states: {
+ CHOKIDAR_NOT_READY: {
+ on: {
+ CHOKIDAR_READY: "CHOKIDAR_WATCHING",
+ BOOTSTRAP_FINISHED: "CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED",
+ },
+ },
+ CHOKIDAR_WATCHING: {
+ on: {
+ BOOTSTRAP_FINISHED: "CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED",
+ CHOKIDAR_READY: "CHOKIDAR_WATCHING",
+ },
+ },
+ CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED: {
+ on: {
+ CHOKIDAR_READY: "CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED",
+ },
+ },
+ },
+ },
+ PROCESSING: {
+ initial: `BOOTSTRAPPING`,
+ states: {
+ BOOTSTRAPPING: {
+ on: {
+ BOOTSTRAP_FINISHED: "IDLE",
+ },
+ },
+ IDLE: {
+ on: {
+ EMIT_FS_EVENT: `PROCESSING`,
+ },
+ },
+ PROCESSING: {
+ on: {
+ QUERY_QUEUE_DRAINED: `IDLE`,
+ TOUCH_NODE: `IDLE`,
+ },
+ },
+ },
+ },
+ },
+})
+
+let currentState = fsMachine.initialState
+
+const fileQueue = new Map()
+
 exports.sourceNodes = (
- { actions, getNode, createNodeId, hasNodeChanged, reporter },
+ { actions, getNode, createNodeId, hasNodeChanged, reporter, emitter },
  pluginOptions
 ) => {
  const { createNode, deleteNode } = actions
@@ -21,8 +88,36 @@ Please pick a path to an existing directory.
 See docs here - https://www.gatsbyjs.org/packages/gatsby-source-filesystem/
  `)
  }
+ let fileNodeQueue = new Map()
+
+ // Once bootstrap is finished, we only let one File node update go through
+ // the system at a time.
+ emitter.on(`BOOTSTRAP_FINISHED`, () => {
+ currentState = fsMachine.transition(
+ currentState.value,
+ `BOOTSTRAP_FINISHED`
+ )
+ })
+ emitter.on(`TOUCH_NODE`, () => {
+ // If we create a node which is the same as the previous version, createNode
+ // returns TOUCH_NODE and then nothing else happens so we listen to that
+ // to return the state back to IDLE.
+ currentState = fsMachine.transition(currentState.value, `TOUCH_NODE`)
+ })
 
- let ready = false
+ emitter.on(`QUERY_QUEUE_DRAINED`, () => {
+ currentState = fsMachine.transition(
+ currentState.value,
+ `QUERY_QUEUE_DRAINED`
+ )
+ // If we have any updates queued, run one of them now.
+ if (fileNodeQueue.size > 0) {
+ const toProcess = fileNodeQueue.get(Array.from(fileNodeQueue.keys())[0])
+ fileNodeQueue.delete(toProcess.id)
+ currentState = fsMachine.transition(currentState.value, `EMIT_FS_EVENT`)
+ createNode(toProcess)
+ }
+ })
 
  const watcher = chokidar.watch(pluginOptions.path, {
  ignored: [
@@ -36,8 +131,21 @@ See docs here - https://www.gatsbyjs.org/packages/gatsby-source-filesystem/
  ],
  })
 
- const createAndProcessNode = path =>
- createFileNode(path, createNodeId, pluginOptions).then(createNode)
+ const createAndProcessNode = path => {
+ const fileNodePromise = createFileNode(
+ path,
+ createNodeId,
+ pluginOptions
+ ).then(fileNode => {
+ if (currentState.value.PROCESSING === `PROCESSING`) {
+ fileNodeQueue.set(fileNode.id, fileNode)
+ } else {
+ currentState = fsMachine.transition(currentState.value, `EMIT_FS_EVENT`)
+ createNode(fileNode)
+ }
+ })
+ return fileNodePromise
+ }
 
  // For every path that is reported before the 'ready' event, we throw them
  // into a queue and then flush the queue when 'ready' event arrives.
@@ -50,49 +158,68 @@ See docs here - https://www.gatsbyjs.org/packages/gatsby-source-filesystem/
  }
 
  watcher.on(`add`, path => {
- if (ready) {
- reporter.info(`added file at ${path}`)
+ if (currentState.value.CHOKIDAR !== `CHOKIDAR_NOT_READY`) {
+ if (
+ currentState.value.CHOKIDAR === `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED`
+ ) {
+ reporter.info(`added file at ${path}`)
+ }
  createAndProcessNode(path).catch(err => reporter.error(err))
  } else {
  pathQueue.push(path)
  }
  })
 
  watcher.on(`change`, path => {
- reporter.info(`changed file at ${path}`)
+ if (
+ currentState.value.CHOKIDAR === `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED`
+ ) {
+ reporter.info(`changed file at ${path}`)
+ }
  createAndProcessNode(path).catch(err => reporter.error(err))
  })
 
  watcher.on(`unlink`, path => {
- reporter.info(`file deleted at ${path}`)
- const node = getNode(createId(path))
+ if (
+ currentState.value.CHOKIDAR === `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED`
+ ) {
+ reporter.info(`file deleted at ${path}`)
+ }
+ const node = getNode(createNodeId(path))
  // It's possible the file node was never created as sometimes tools will
  // write and then immediately delete temporary files to the file system.
  if (node) {
+ currentState = fsMachine.transition(currentState.value, `EMIT_FS_EVENT`)
  deleteNode(node.id, node)
  }
  })
 
  watcher.on(`addDir`, path => {
- if (ready) {
- reporter.info(`added directory at ${path}`)
+ if (currentState.value.CHOKIDAR !== `CHOKIDAR_NOT_READY`) {
+ if (
+ currentState.value.CHOKIDAR === `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED`
+ ) {
+ reporter.info(`added directory at ${path}`)
+ }
  createAndProcessNode(path).catch(err => reporter.error(err))
  } else {
  pathQueue.push(path)
  }
  })
 
  watcher.on(`unlinkDir`, path => {
- reporter.info(`directory deleted at ${path}`)
- const node = getNode(createId(path))
+ if (
+ currentState.value.CHOKIDAR === `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED`
+ ) {
+ reporter.info(`directory deleted at ${path}`)
+ }
+ const node = getNode(createNodeId(path))
  deleteNode(node.id, node)
  })
 
  return new Promise((resolve, reject) => {
  watcher.on(`ready`, () => {
- if (ready) return
-
- ready = true
+ currentState = fsMachine.transition(currentState.value, `CHOKIDAR_READY`)
  flushPathQueue().then(resolve, reject)
  })
  })

packages/gatsby/cache-dir/app.js

@@ -11,7 +11,12 @@ window.___emitter = emitter
 // Let the site/plugins run code very early.
 apiRunnerAsync(`onClientEntry`).then(() => {
  // Hook up the client to socket.io on server
- socketIo()
+ const socket = socketIo()
+ if (socket) {
+ socket.on(`reload`, () => {
+ window.location.reload()
+ })
+ }
 
  /**
  * Service Workers are persistent by nature. They stick around,

packages/gatsby/cache-dir/component-renderer.js

@@ -126,7 +126,7 @@ class ComponentRenderer extends React.Component {
  })
  const replacementComponent = pluginResponses[0]
  // If page.
- if (this.props.page) {
+ if (this.props.isPage) {
  if (this.state.pageResources) {
  return (
  replacementComponent ||
@@ -163,7 +163,7 @@ class ComponentRenderer extends React.Component {
 }
 
 ComponentRenderer.propTypes = {
- page: PropTypes.bool,
+ isPage: PropTypes.bool,
  layout: PropTypes.bool,
  location: PropTypes.object,
 }

packages/gatsby/cache-dir/json-store.js

@@ -0,0 +1,98 @@
+import React, { createElement } from "react"
+import { Route } from "react-router-dom"
+import ComponentRenderer from "./component-renderer"
+import syncRequires from "./sync-requires"
+import socketIo from "./socketIo"
+import omit from "lodash/omit"
+import get from "lodash/get"
+
+const getPathFromProps = props => {
+ if (props.isPage) {
+ return get(props.pageResources, `page.path`)
+ } else {
+ return `/dev-404-page/`
+ }
+}
+
+class JSONStore extends React.Component {
+ constructor(props) {
+ super(props)
+ this.state = {
+ data: {},
+ path: null,
+ }
+
+ this.setPageData = this.setPageData.bind(this)
+
+ this.socket = socketIo()
+ this.socket.on(`queryResult`, this.setPageData)
+ }
+
+ componentWillMount() {
+ this.registerPath(getPathFromProps(this.props))
+ }
+
+ componentWillReceiveProps(nextProps) {
+ const { path } = this.state
+ const newPath = getPathFromProps(nextProps)
+
+ if (path !== newPath) {
+ this.unregisterPath(path)
+ this.registerPath(newPath)
+ }
+ }
+
+ registerPath(path) {
+ this.setState({ path })
+ this.socket.emit(`registerPath`, path)
+ }
+
+ unregisterPath(path) {
+ this.setState({ path: null })
+ this.socket.emit(`unregisterPath`, path)
+ }
+
+ componentWillUnmount() {
+ this.unregisterPath(this.state.path)
+ }
+
+ setPageData({ path, result }) {
+ this.setState({
+ data: {
+ ...this.state.data,
+ [path]: result,
+ },
+ })
+ }
+
+ render() {
+ const { isPage, pages, pageResources } = this.props
+ const data = this.state.data[this.state.path]
+ const propsWithoutPages = omit(this.props, `pages`)
+
+ if (!data) {
+ return null
+ } else if (isPage) {
+ return createElement(ComponentRenderer, {
+ key: `normal-page`,
+ ...propsWithoutPages,
+ ...data,
+ })
+ } else {
+ const dev404Page = pages.find(p => /^\/dev-404-page/.test(p.path))
+ return createElement(Route, {
+ key: `404-page`,
+ component: props =>
+ createElement(
+ syncRequires.components[dev404Page.componentChunkName],
+ {
+ ...propsWithoutPages,
+ ...data,
+ }
+ ),
+ })
+ }
+ }
+}
+
+export default JSONStore