Add fileFinder for build sourcing (no file watchers)

packages/gatsby-source-filesystem/package.json

@@ -16,6 +16,7 @@
  "md5-file": "^3.1.1",
  "mime": "^2.2.0",
  "pretty-bytes": "^4.0.2",
+ "readdirp-walk": "^1.6.0",
  "slash": "^1.0.0",
  "valid-url": "^1.0.9",
  "xstate": "^3.1.0"

packages/gatsby-source-filesystem/src/__tests__/fileFinder.js

@@ -0,0 +1,27 @@
+const fileFinder = require(`../file-finder`)
+
+describe(`file-finder`, () => {
+ it(`resolves an array of file paths`, async done => {
+ expect.assertions(3)
+ const queue = await fileFinder(`${__dirname}/fixtures/test-fs`)
+ const expectedQueue = [
+ `${__dirname}/fixtures/test-fs/index.md`,
+ `${__dirname}/fixtures/test-fs/dirA/index.md`,
+ `${__dirname}/fixtures/test-fs/dirB/index.md`,
+ `${__dirname}/fixtures/test-fs/dirA/a/index.md`,
+ `${__dirname}/fixtures/test-fs/dirA/a/A/index.md`,
+ `${__dirname}/fixtures/test-fs/dirA/b/index.md`,
+ `${__dirname}/fixtures/test-fs/dirA/b/A/index.md`,
+ `${__dirname}/fixtures/test-fs/dirA/c/index.md`,
+ `${__dirname}/fixtures/test-fs/dirA/c/A/index.md`,
+ ]
+
+ expect(queue.length).toBeGreaterThan(0)
+ expect(queue.length).toEqual(expectedQueue.length)
+ const hasAllFiles = expectedQueue.every(expected =>
+ queue.some(item => item === expected)
+ )
+ expect(hasAllFiles).toBe(true)
+ done()
+ })
+})

packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/.toBeIgnored.md

@@ -0,0 +1 @@
+should never be queued

packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/a/A/index.md

@@ -0,0 +1 @@
+"DirA a A" content

packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/a/index.md

@@ -0,0 +1 @@
+"DirA a" content

packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/b/A/index.md

@@ -0,0 +1 @@
+"DirA a A" content

packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/b/index.md

@@ -0,0 +1 @@
+"DirA b" content

packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/c/A/index.md

@@ -0,0 +1 @@
+"DirA a A" content

packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/c/index.md

@@ -0,0 +1 @@
+"DirA c" content

packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirA/index.md

@@ -0,0 +1 @@
+"DirA" content

packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/dirB/index.md

@@ -0,0 +1 @@
+"DirB" content

packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/empty/.placeholder.md

@@ -0,0 +1 @@
+This file is only so we can commit an empty folder. It should be ignored during testing.

packages/gatsby-source-filesystem/src/__tests__/fixtures/test-fs/index.md

@@ -0,0 +1 @@
+root dir content

packages/gatsby-source-filesystem/src/file-finder.js

@@ -0,0 +1,34 @@
+const readdirp = require(`readdirp-walk`)
+
+const ignoredRE = /^\./
+const ignored = [`yarn.lock`, `package-lock.json`, `node_modules`, `dist`]
+
+function fileFinder(dir) {
+ return new Promise(resolve => {
+ let fileList = []
+ const stream = readdirp({ root: dir })
+
+ stream.on(`data`, data => {
+ const { name, fullPath, stat } = data
+
+ if (
+ stat.isDirectory() ||
+ ignoredRE.test(name) ||
+ ignored.includes(name)
+ ) {
+ return
+ }
+ if (fullPath.includes(`node_modules`) || fullPath.includes(`build`)) {
+ return
+ }
+ fileList.push(fullPath)
+ return
+ })
+
+ stream.on(`end`, () => {
+ resolve(fileList)
+ })
+ })
+}
+
+module.exports = fileFinder

packages/gatsby-source-filesystem/src/file-watcher.js

@@ -0,0 +1,212 @@
+const path = require(`path`)
+const chokidar = require(`chokidar`)
+const { Machine } = require(`xstate`)
+
+const { createFileNode } = require(`./create-file-node`)
+/**
+ * Create a state machine to manage Chokidar's not-ready/ready states and for
+ * emitting file system events into Gatsby.
+ *
+ * On the latter, this solves the problem where if you call createNode for the
+ * same File node in quick succession, this can leave Gatsby's internal state
+ * in disarray causing queries to fail. The latter state machine tracks when
+ * Gatsby is "processing" a node update or when it's "idle". If updates come in
+ * while Gatsby is processing, we queue them until the system returns to an
+ * "idle" state.
+ */
+const createFSMachine = () =>
+ Machine({
+ key: `emitFSEvents`,
+ parallel: true,
+ strict: true,
+ states: {
+ CHOKIDAR: {
+ initial: `CHOKIDAR_NOT_READY`,
+ states: {
+ CHOKIDAR_NOT_READY: {
+ on: {
+ CHOKIDAR_READY: `CHOKIDAR_WATCHING`,
+ BOOTSTRAP_FINISHED: `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED`,
+ },
+ },
+ CHOKIDAR_WATCHING: {
+ on: {
+ BOOTSTRAP_FINISHED: `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED`,
+ CHOKIDAR_READY: `CHOKIDAR_WATCHING`,
+ },
+ },
+ CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED: {
+ on: {
+ CHOKIDAR_READY: `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED`,
+ },
+ },
+ },
+ },
+ PROCESSING: {
+ initial: `BOOTSTRAPPING`,
+ states: {
+ BOOTSTRAPPING: {
+ on: {
+ BOOTSTRAP_FINISHED: `IDLE`,
+ },
+ },
+ IDLE: {
+ on: {
+ EMIT_FS_EVENT: `PROCESSING`,
+ },
+ },
+ PROCESSING: {
+ on: {
+ QUERY_QUEUE_DRAINED: `IDLE`,
+ TOUCH_NODE: `IDLE`,
+ },
+ },
+ },
+ },
+ },
+ })
+
+function fileWatcher(
+ { actions, getNode, createNodeId, reporter, emitter },
+ pluginOptions
+) {
+ const { createNode, deleteNode } = actions
+ // Validate that the path is absolute.
+ // Absolute paths are required to resolve images correctly.
+ if (!path.isAbsolute(pluginOptions.path)) {
+ pluginOptions.path = path.resolve(process.cwd(), pluginOptions.path)
+ }
+
+ const fsMachine = createFSMachine()
+ let currentState = fsMachine.initialState
+ let pathQueue = []
+ let fileNodeQueue = new Map()
+ const createAndProcessNode = path => {
+ const fileNodePromise = createFileNode(
+ path,
+ createNodeId,
+ pluginOptions
+ ).then(fileNode => {
+ if (currentState.value.PROCESSING === `PROCESSING`) {
+ fileNodeQueue.set(fileNode.id, fileNode)
+ } else {
+ currentState = fsMachine.transition(currentState.value, `EMIT_FS_EVENT`)
+ createNode(fileNode)
+ }
+ })
+ return fileNodePromise
+ }
+
+ const watcher = chokidar.watch(pluginOptions.path, {
+ ignored: [
+ `**/*.un~`,
+ `**/.gitignore`,
+ `**/.npmignore`,
+ `**/.babelrc`,
+ `**/yarn.lock`,
+ `**/node_modules`,
+ `../**/dist/**`,
+ ],
+ })
+
+ emitter.on(`BOOTSTRAP_FINISHED`, () => {
+ currentState = fsMachine.transition(
+ currentState.value,
+ `BOOTSTRAP_FINISHED`
+ )
+ })
+ emitter.on(`TOUCH_NODE`, () => {
+ // If we create a node which is the same as the previous version, createNode
+ // returns TOUCH_NODE and then nothing else happens so we listen to that
+ // to return the state back to IDLE.
+ currentState = fsMachine.transition(currentState.value, `TOUCH_NODE`)
+ })
+
+ emitter.on(`QUERY_QUEUE_DRAINED`, () => {
+ currentState = fsMachine.transition(
+ currentState.value,
+ `QUERY_QUEUE_DRAINED`
+ )
+ // If we have any updates queued, run one of them now.
+ if (fileNodeQueue.size > 0) {
+ const toProcess = fileNodeQueue.get(Array.from(fileNodeQueue.keys())[0])
+ fileNodeQueue.delete(toProcess.id)
+ currentState = fsMachine.transition(currentState.value, `EMIT_FS_EVENT`)
+ createNode(toProcess)
+ }
+ })
+ watcher.on(`add`, path => {
+ if (currentState.value.CHOKIDAR !== `CHOKIDAR_NOT_READY`) {
+ if (
+ currentState.value.CHOKIDAR === `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED`
+ ) {
+ reporter.info(`added file at ${path}`)
+ }
+ createAndProcessNode(path).catch(err => reporter.error(err))
+ } else {
+ pathQueue.push(path)
+ }
+ })
+
+ watcher.on(`change`, path => {
+ if (
+ currentState.value.CHOKIDAR === `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED`
+ ) {
+ reporter.info(`changed file at ${path}`)
+ }
+ createAndProcessNode(path).catch(err => reporter.error(err))
+ })
+
+ watcher.on(`unlink`, path => {
+ if (
+ currentState.value.CHOKIDAR === `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED`
+ ) {
+ reporter.info(`file deleted at ${path}`)
+ }
+ const node = getNode(createNodeId(path))
+ // It's possible the file node was never created as sometimes tools will
+ // write and then immediately delete temporary files to the file system.
+ if (node) {
+ currentState = fsMachine.transition(currentState.value, `EMIT_FS_EVENT`)
+ deleteNode({ node })
+ }
+ })
+
+ watcher.on(`addDir`, path => {
+ if (currentState.value.CHOKIDAR !== `CHOKIDAR_NOT_READY`) {
+ if (
+ currentState.value.CHOKIDAR === `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED`
+ ) {
+ reporter.info(`added directory at ${path}`)
+ }
+ createAndProcessNode(path).catch(err => reporter.error(err))
+ } else {
+ pathQueue.push(path)
+ }
+ })
+
+ watcher.on(`unlinkDir`, path => {
+ if (
+ currentState.value.CHOKIDAR === `CHOKIDAR_WATCHING_BOOTSTRAP_FINISHED`
+ ) {
+ reporter.info(`directory deleted at ${path}`)
+ }
+ const node = getNode(createNodeId(path))
+ deleteNode({ node })
+ })
+
+ const flushPathQueue = () => {
+ let queue = pathQueue.slice()
+ pathQueue = []
+ return Promise.all(queue.map(createAndProcessNode))
+ }
+
+ return new Promise((resolve, reject) => {
+ watcher.on(`ready`, () => {
+ currentState = fsMachine.transition(currentState.value, `CHOKIDAR_READY`)
+ flushPathQueue().then(resolve, reject)
+ })
+ })
+}
+
+module.exports = fileWatcher