feat(gatsby): Optimize creating many child nodes from one parent

packages/gatsby-transformer-csv/src/gatsby-node.js

@@ -1,6 +1,5 @@
 const csv = require(`csvtojson`)
 const _ = require(`lodash`)
-
 const { typeNameFromFile } = require(`./index`)
 
 const convertToJson = (data, options) =>
@@ -15,6 +14,8 @@ function shouldOnCreateNode({ node }, pluginOptions = {}) {
   return extensions ? extensions.includes(extension) : extension === `csv`
 }
 
+const typeCache = new Map()
+
 async function onCreateNode(
   { node, actions, loadNodeContent, createNodeId, createContentDigest },
   pluginOptions
@@ -35,14 +36,26 @@ async function onCreateNode(
     if (pluginOptions && _.isFunction(typeName)) {
       return pluginOptions.typeName({ node, object })
     } else if (pluginOptions && _.isString(typeName)) {
-      return _.upperFirst(_.camelCase(typeName))
+      if (typeCache.has(node.internal.type)) {
+        return typeCache.get(node.internal.type)
+      } else {
+        const type = _.upperFirst(_.camelCase(typeName))
+        typeCache.set(node.internal.type, type)
+        return type
+      }
     } else {
-      return typeNameFromFile({ node })
+      if (typeCache.has(node.internal.type)) {
+        return typeCache.get(node.internal.type)
+      } else {
+        const type = typeNameFromFile({ node })
+        typeCache.set(node.internal.type, type)
+        return type
+      }
     }
   }
 
   // Generate the new node
-  async function transformObject(obj, i) {
+  function transformObject(obj, i) {
     const csvNode = {
       ...obj,
       id:
@@ -59,21 +72,35 @@ async function onCreateNode(
       },
     }
 
-    await createNode(csvNode)
+    createNode(csvNode)
     createParentChildLink({ parent: node, child: csvNode })
   }
 
+  async function transformArrayChunk({ chunk, startCount }) {
+    for (let i = 0, l = chunk.length; i < l; i++) {
+      const obj = chunk[i]
+      transformObject(obj, i + startCount)
+      await new Promise(resolve =>
+        setImmediate(() => {
+          resolve()
+        })
+      )
+    }
+  }
+
   if (_.isArray(parsedContent)) {
     if (pluginOptions && nodePerFile) {
       if (pluginOptions && _.isString(nodePerFile)) {
-        await transformObject({ [nodePerFile]: parsedContent }, 0)
+        transformObject({ [nodePerFile]: parsedContent }, 0)
       } else {
-        await transformObject({ items: parsedContent }, 0)
+        transformObject({ items: parsedContent }, 0)
       }
     } else {
-      for (let i = 0, l = parsedContent.length; i < l; i++) {
-        const obj = parsedContent[i]
-        await transformObject(obj, i)
+      const chunks = _.chunk(parsedContent, 100)
+      let count = 0
+      for (const chunk of chunks) {
+        await transformArrayChunk({ chunk, startCount: count })
+        count += chunk.length
       }
     }
   }

packages/gatsby-transformer-json/src/gatsby-node.js

@@ -6,6 +6,8 @@ function shouldOnCreateNode({ node }) {
   return node.internal.mediaType === `application/json`
 }
 
+const typeCache = new Map()
+
 async function onCreateNode(
   { node, actions, loadNodeContent, createNodeId, createContentDigest },
   pluginOptions
@@ -16,15 +18,35 @@ async function onCreateNode(
     } else if (pluginOptions && _.isString(pluginOptions.typeName)) {
       return pluginOptions.typeName
     } else if (node.internal.type !== `File`) {
-      return _.upperFirst(_.camelCase(`${node.internal.type} Json`))
+      if (typeCache.has(node.internal.type)) {
+        return typeCache.get(node.internal.type)
+      } else {
+        const type = _.upperFirst(_.camelCase(`${node.internal.type} Json`))
+        typeCache.set(node.internal.type, type)
+        return type
+      }
     } else if (isArray) {
-      return _.upperFirst(_.camelCase(`${node.name} Json`))
+      if (typeCache.has(node.name)) {
+        return typeCache.get(node.name)
+      } else {
+        const type = _.upperFirst(_.camelCase(`${node.name} Json`))
+        typeCache.set(node.name, type)
+        return type
+      }
     } else {
-      return _.upperFirst(_.camelCase(`${path.basename(node.dir)} Json`))
+      if (typeCache.has(node.dir)) {
+        return typeCache.get(node.dir)
+      } else {
+        const type = _.upperFirst(
+          _.camelCase(`${path.basename(node.dir)} Json`)
+        )
+        typeCache.set(node.dir, type)
+        return type
+      }
     }
   }
 
-  async function transformObject(obj, id, type) {
+  function transformObject(obj, id, type) {
     const jsonNode = {
       ...obj,
       id,
@@ -38,7 +60,7 @@ async function onCreateNode(
     if (obj.id) {
       jsonNode[`jsonId`] = obj.id
     }
-    await createNode(jsonNode)
+    createNode(jsonNode)
     createParentChildLink({ parent: node, child: jsonNode })
   }
 
@@ -55,18 +77,35 @@ async function onCreateNode(
     throw new Error(`Unable to parse JSON: ${hint}`)
   }
 
-  if (_.isArray(parsedContent)) {
-    for (let i = 0, l = parsedContent.length; i < l; i++) {
-      const obj = parsedContent[i]
-
-      await transformObject(
+  async function transformArrayChunk({ chunk, startCount }) {
+    for (let i = 0, l = chunk.length; i < l; i++) {
+      const obj = chunk[i]
+      transformObject(
         obj,
-        createNodeId(`${node.id} [${i}] >>> JSON`),
-        getType({ node, object: obj, isArray: true })
+        createNodeId(`${node.id} [${i + startCount}] >>> JSON`),
+        getType({
+          node,
+          object: obj,
+          isArray: true,
+        })
+      )
+      await new Promise(resolve =>
+        setImmediate(() => {
+          resolve()
+        })
       )
     }
+  }
+
+  if (_.isArray(parsedContent)) {
+    const chunks = _.chunk(parsedContent, 100)
+    let count = 0
+    for (const chunk of chunks) {
+      await transformArrayChunk({ chunk, startCount: count })
+      count += chunk.length
+    }
   } else if (_.isPlainObject(parsedContent)) {
-    await transformObject(
+    transformObject(
       parsedContent,
       createNodeId(`${node.id} >>> JSON`),
       getType({ node, object: parsedContent, isArray: false })

packages/gatsby/src/redux/actions/create-parent-child-link.ts

@@ -0,0 +1,49 @@
+import type { IGatsbyNode, IAddChildNodeToParentNodeAction } from "../types"
+import Batcher from "../../utils/batcher"
+import { store } from "../"
+import { getNode } from "../../datastore"
+
+const isTestEnv = process.env.NODE_ENV === `test`
+const batchCount = isTestEnv ? 1 : 1000
+
+type CreateParentChildLinkFn = (
+  payload: {
+    parent: IGatsbyNode
+    child: IGatsbyNode
+  },
+  plugin?: string
+) => IAddChildNodeToParentNodeAction
+
+export const createParentChildLinkBatcher =
+  new Batcher<CreateParentChildLinkFn>(batchCount)
+
+createParentChildLinkBatcher.bulkCall(createCalls => {
+  const nodesMap = new Map()
+
+  // Add children to parent node(s) and dispatch.
+  createCalls.forEach(call => {
+    const { parent, child } = call[0]
+    const parentId = parent.id
+
+    let parentNode
+    if (nodesMap.has(parentId)) {
+      parentNode = nodesMap.get(parentId)
+    } else {
+      parentNode = getNode(parentId)
+
+      if (!parentNode.children.includes(child.id)) {
+        parentNode.children.push(child.id)
+      }
+
+      nodesMap.set(parentId, parentNode)
+    }
+  })
+
+  nodesMap.forEach(parentNode => {
+    const payload: IAddChildNodeToParentNodeAction = {
+      type: `ADD_CHILD_NODE_TO_PARENT_NODE`,
+      payload: parentNode,
+    }
+    store.dispatch(payload)
+  })
+})

packages/gatsby/src/redux/actions/public.js

@@ -35,6 +35,7 @@ import { createJobV2FromInternalJob } from "./internal"
 import { maybeSendJobToMainProcess } from "../../utils/jobs/worker-messaging"
 import { reportOnce } from "../../utils/report-once"
 import { wrapNode } from "../../utils/detect-node-mutations"
+import { createParentChildLinkBatcher } from "./create-parent-child-link"
 
 const isNotTestEnv = process.env.NODE_ENV !== `test`
 const isTestEnv = process.env.NODE_ENV === `test`
@@ -1040,15 +1041,9 @@ actions.createParentChildLink = (
   { parent, child }: { parent: any, child: any },
   plugin?: Plugin
 ) => {
-  if (!parent.children.includes(child.id)) {
-    parent.children.push(child.id)
-  }
+  createParentChildLinkBatcher.add({ parent, child })
 
-  return {
-    type: `ADD_CHILD_NODE_TO_PARENT_NODE`,
-    plugin,
-    payload: parent,
-  }
+  return []
 }
 
 /**

packages/gatsby/src/utils/source-nodes.ts

@@ -5,6 +5,7 @@ import { store } from "../redux"
 import { getDataStore, getNode } from "../datastore"
 import { actions } from "../redux/actions"
 import { IGatsbyState, IGatsbyNode } from "../redux/types"
+import { createParentChildLinkBatcher } from "../redux/actions/create-parent-child-link"
 import type { GatsbyIterable } from "../datastore/common/iterable"
 
 const { deleteNode } = actions
@@ -114,6 +115,9 @@ export default async ({
     pluginName,
   })
 
+  // Flush createParentChildLinkBatcher to ensure parent nodes are written with their children dependencies.
+  createParentChildLinkBatcher.flush()
+
   await getDataStore().ready()
 
   // We only warn for plugins w/o nodes and delete stale nodes on the first sourcing.