Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Don't block the event loop during inference
Browse files Browse the repository at this point in the history
TylerBarnes committed Mar 24, 2023

Unverified

No user is associated with the committer email.
1 parent 2f3f372 commit 71e7e31
Showing 3 changed files with 48 additions and 12 deletions.
9 changes: 7 additions & 2 deletions packages/gatsby/src/redux/reducers/inference-metadata.ts
Original file line number Diff line number Diff line change
@@ -37,9 +37,14 @@ const incrementalReducer = (

case `BUILD_TYPE_METADATA`: {
// Overwrites existing metadata
const { nodes, typeName } = action.payload
const { nodes, typeName, clearExistingMetadata } = action.payload
if (!state[typeName]?.ignored) {
state[typeName] = addNodes(initialTypeMetadata(), nodes)
const initialMetadata =
clearExistingMetadata || !state[typeName]
? initialTypeMetadata()
: state[typeName]

state[typeName] = addNodes(initialMetadata, nodes)
}
return state
}
1 change: 1 addition & 0 deletions packages/gatsby/src/redux/types.ts
Original file line number Diff line number Diff line change
@@ -1056,6 +1056,7 @@ interface IBuildTypeMetadataAction {
type: `BUILD_TYPE_METADATA`
payload: {
nodes: Array<IGatsbyNode>
clearExistingMetadata: boolean
typeName: string
}
}
50 changes: 40 additions & 10 deletions packages/gatsby/src/schema/index.js
Original file line number Diff line number Diff line change
@@ -60,18 +60,48 @@ const buildInferenceMetadata = ({ types }) =>
// TODO: use async iterators when we switch to node>=10
// or better investigate if we can offload metadata building to worker/Jobs API
// and then feed the result into redux?
const processNextType = () => {
const processNextType = async () => {
const typeName = typeNames.pop()
store.dispatch({
type: `BUILD_TYPE_METADATA`,
payload: {
typeName,
nodes: getDataStore().iterateNodesByType(typeName),
},
})

let processingNodes = []
let dispatchCount = 0
function dispatchNodes() {
return new Promise(res => {
store.dispatch({
type: `BUILD_TYPE_METADATA`,
payload: {
typeName,
// only clear metadata on the first chunk for this type
clearExistingMetadata: dispatchCount++ === 0,
nodes: processingNodes,
},
})
setImmediate(() => {
// clear this array after BUILD_TYPE_METADATA reducer has synchronously run
processingNodes = []
// dont block the event loop. node may decide to free previous processingNodes array from memory if it needs to.
setImmediate(() => {
res(null)
})
})
})
}

for (const node of getDataStore().iterateNodesByType(typeName)) {
processingNodes.push(node)

if (processingNodes.length > 1000) {
await dispatchNodes()
}
}

if (processingNodes.length > 0) {
await dispatchNodes()
}

if (typeNames.length > 0) {
// Give event-loop a break
setTimeout(processNextType, 0)
// dont block the event loop
setImmediate(() => processNextType())
} else {
resolve()
}

0 comments on commit 71e7e31

Please sign in to comment.