From c08048d488c5bd3369bf766e84c4f6a11eac5848 Mon Sep 17 00:00:00 2001
From: Tyler Barnes <tylerdbarnes@gmail.com>
Date: Mon, 27 Mar 2023 09:55:24 -0700
Subject: [PATCH] fix(gatsby): don't block event loop during inference (#37780)

Don't block the event loop during inference
---
 .../src/redux/reducers/inference-metadata.ts  |  9 +++-
 packages/gatsby/src/redux/types.ts            |  1 +
 packages/gatsby/src/schema/index.js           | 50 +++++++++++++++----
 3 files changed, 48 insertions(+), 12 deletions(-)

diff --git a/packages/gatsby/src/redux/reducers/inference-metadata.ts b/packages/gatsby/src/redux/reducers/inference-metadata.ts
index 1db7018170934..014acb6ba95eb 100644
--- a/packages/gatsby/src/redux/reducers/inference-metadata.ts
+++ b/packages/gatsby/src/redux/reducers/inference-metadata.ts
@@ -37,9 +37,14 @@ const incrementalReducer = (
 
     case `BUILD_TYPE_METADATA`: {
       // Overwrites existing metadata
-      const { nodes, typeName } = action.payload
+      const { nodes, typeName, clearExistingMetadata } = action.payload
       if (!state[typeName]?.ignored) {
-        state[typeName] = addNodes(initialTypeMetadata(), nodes)
+        const initialMetadata =
+          clearExistingMetadata || !state[typeName]
+            ? initialTypeMetadata()
+            : state[typeName]
+
+        state[typeName] = addNodes(initialMetadata, nodes)
       }
       return state
     }
diff --git a/packages/gatsby/src/redux/types.ts b/packages/gatsby/src/redux/types.ts
index f539bddf3175c..87867a5079886 100644
--- a/packages/gatsby/src/redux/types.ts
+++ b/packages/gatsby/src/redux/types.ts
@@ -1056,6 +1056,7 @@ interface IBuildTypeMetadataAction {
   type: `BUILD_TYPE_METADATA`
   payload: {
     nodes: Array<IGatsbyNode>
+    clearExistingMetadata: boolean
     typeName: string
   }
 }
diff --git a/packages/gatsby/src/schema/index.js b/packages/gatsby/src/schema/index.js
index 5ea5810fe2ef0..34bb0817d8695 100644
--- a/packages/gatsby/src/schema/index.js
+++ b/packages/gatsby/src/schema/index.js
@@ -60,18 +60,48 @@ const buildInferenceMetadata = ({ types }) =>
     // TODO: use async iterators when we switch to node>=10
     //  or better investigate if we can offload metadata building to worker/Jobs API
     //  and then feed the result into redux?
-    const processNextType = () => {
+    const processNextType = async () => {
       const typeName = typeNames.pop()
-      store.dispatch({
-        type: `BUILD_TYPE_METADATA`,
-        payload: {
-          typeName,
-          nodes: getDataStore().iterateNodesByType(typeName),
-        },
-      })
+
+      let processingNodes = []
+      let dispatchCount = 0
+      function dispatchNodes() {
+        return new Promise(res => {
+          store.dispatch({
+            type: `BUILD_TYPE_METADATA`,
+            payload: {
+              typeName,
+              // only clear metadata on the first chunk for this type
+              clearExistingMetadata: dispatchCount++ === 0,
+              nodes: processingNodes,
+            },
+          })
+          setImmediate(() => {
+            // clear this array after BUILD_TYPE_METADATA reducer has synchronously run
+            processingNodes = []
+            // dont block the event loop. node may decide to free previous processingNodes array from memory if it needs to.
+            setImmediate(() => {
+              res(null)
+            })
+          })
+        })
+      }
+
+      for (const node of getDataStore().iterateNodesByType(typeName)) {
+        processingNodes.push(node)
+
+        if (processingNodes.length > 1000) {
+          await dispatchNodes()
+        }
+      }
+
+      if (processingNodes.length > 0) {
+        await dispatchNodes()
+      }
+
       if (typeNames.length > 0) {
-        // Give event-loop a break
-        setTimeout(processNextType, 0)
+        // dont block the event loop
+        setImmediate(() => processNextType())
       } else {
         resolve()
       }