feat(gatsby, gatsby-source-contentful): add public action to disable stale node checks

packages/gatsby-source-contentful/src/normalize.js

@@ -5,7 +5,8 @@ import { getGatsbyVersion } from "gatsby-core-utils"
 import { lt, prerelease } from "semver"
 
 const typePrefix = `Contentful`
-const makeTypeName = type => _.upperFirst(_.camelCase(`${typePrefix} ${type}`))
+export const makeTypeName = type =>
+  _.upperFirst(_.camelCase(`${typePrefix} ${type}`))
 
 const GATSBY_VERSION_MANIFEST_V2 = `4.3.0`
 const gatsbyVersion =

packages/gatsby-source-contentful/src/source-nodes.js

@@ -1,4 +1,5 @@
 // @ts-check
+import { hasFeature } from "gatsby-plugin-utils/has-feature"
 import isOnline from "is-online"
 import _ from "lodash"
 
@@ -11,6 +12,7 @@ import {
   createAssetNodes,
   createNodesForContentType,
   makeId,
+  makeTypeName,
 } from "./normalize"
 import { createPluginConfig } from "./plugin-options"
 import { CODES } from "./report"
@@ -40,7 +42,7 @@ const CONTENT_DIGEST_COUNTER_SEPARATOR = `_COUNT_`
  * or the fallback field or the default field.
  */
 
-let isFirstSource = true
+let isFirstSourceNodesCallOfCurrentNodeProcess = true
 export async function sourceNodes(
   {
     actions,
@@ -55,14 +57,26 @@ export async function sourceNodes(
   },
   pluginOptions
 ) {
-  const { createNode, touchNode, deleteNode, unstable_createNodeManifest } =
-    actions
+  const hasStatefulSourceNodes = hasFeature(`stateful-source-nodes`)
+  const needToTouchNodes = !hasStatefulSourceNodes
+
+  const {
+    createNode,
+    touchNode,
+    deleteNode,
+    unstable_createNodeManifest,
+    enableStatefulSourceNodes,
+  } = actions
+
   const online = await isOnline()
 
+  if (hasStatefulSourceNodes) {
+    enableStatefulSourceNodes()
+  }
   // Gatsby only checks if a node has been touched on the first sourcing.
   // As iterating and touching nodes can grow quite expensive on larger sites with
   // 1000s of nodes, we'll skip doing this on subsequent sources.
-  if (isFirstSource) {
+  else if (isFirstSourceNodesCallOfCurrentNodeProcess && needToTouchNodes) {
     getNodes().forEach(node => {
       if (node.internal.owner !== `gatsby-source-contentful`) {
         return
@@ -73,9 +87,10 @@ export async function sourceNodes(
         touchNode(getNode(node.fields.localFile))
       }
     })
-    isFirstSource = false
   }
 
+  isFirstSourceNodesCallOfCurrentNodeProcess = false
+
   if (
     !online &&
     process.env.GATSBY_CONTENTFUL_OFFLINE === `true` &&

packages/gatsby/index.d.ts

@@ -21,6 +21,7 @@ export type AvailableFeatures =
   | "image-cdn"
   | "graphql-typegen"
   | "content-file-path"
+  | "stateful-source-nodes"
 
 export {
   Link,
@@ -433,6 +434,11 @@ export interface GatsbyNode<
     calllback: PluginCallback<void>
   ): void | Promise<void>
 
+  /**
+   * Marks the source plugin that called this function as stateful. Gatsby will not check for stale nodes for any plugin that calls this.
+   */
+  enableStatefulSourceNodes?(this: void, plugin?: ActionPlugin)
+
   /**
    * Called when a new node is created. Plugins wishing to extend or
    * transform nodes created by other plugins should implement this API.

packages/gatsby/scripts/output-api-file.js

@@ -41,7 +41,7 @@ async function outputFile() {
   }, {})
 
   /** @type {Array<import("../index").AvailableFeatures>} */
-  output.features = ["image-cdn", "graphql-typegen", "content-file-path", "slices"];
+  output.features = ["image-cdn", "graphql-typegen", "content-file-path", "slices", "stateful-source-nodes"];
 
   return fs.writeFile(
     path.resolve(OUTPUT_FILE_NAME),

packages/gatsby/src/query/__tests__/data-tracking.js

@@ -70,6 +70,13 @@ jest.mock(`gatsby-cli/lib/reporter`, () => {
         end: jest.fn(),
       }
     },
+    createProgress: () => {
+      return {
+        start: jest.fn(),
+        tick: jest.fn(),
+        end: jest.fn(),
+      }
+    },
     phantomActivity: () => {
       return {
         start: jest.fn(),

packages/gatsby/src/redux/actions/public.js

@@ -539,8 +539,6 @@ ${reservedFields.map(f => `  * "${f}"`).join(`\n`)}
   return actions
 }
 
-const deleteNodeDeprecationWarningDisplayedMessages = new Set()
-
 /**
  * Delete a node
  * @param {object} node A node object. See the "createNode" action for more information about the node object details.
@@ -879,10 +877,18 @@ const createNode = (
     }
   }
 
+  const setTypeOwnerAction = {
+    type: `SET_TYPE_OWNER`,
+    plugin,
+    payload: {
+      typeName: node.internal.type,
+    },
+  }
+
   if (deleteActions && deleteActions.length) {
-    return [...deleteActions, updateNodeAction]
+    return [...deleteActions, updateNodeAction, setTypeOwnerAction]
   } else {
-    return updateNodeAction
+    return [updateNodeAction, setTypeOwnerAction]
   }
 }
 
@@ -921,8 +927,6 @@ actions.createNode =
     }
   }
 
-const touchNodeDeprecationWarningDisplayedMessages = new Set()
-
 /**
  * "Touch" a node. Tells Gatsby a node still exists and shouldn't
  * be garbage collected. Primarily useful for source plugins fetching
@@ -1513,6 +1517,18 @@ actions.unstable_createNodeManifest = (
   }
 }
 
+/**
+ * Marks a source plugin as "stateful" which disables automatically deleting untouched nodes. Stateful source plugins manage deleting their own nodes without stale node checks in Gatsby.
+ *
+ * @param {void} $0
+ */
+actions.enableStatefulSourceNodes = (plugin: Plugin) => {
+  return {
+    type: `ENABLE_STATEFUL_SOURCE_PLUGIN`,
+    plugin,
+  }
+}
+
 /**
  * Stores request headers for a given domain to be later used when making requests for Image CDN (and potentially other features).
  *

packages/gatsby/src/redux/index.ts

@@ -110,6 +110,8 @@ export const saveState = (): void => {
 
   return writeToCache({
     nodes: state.nodes,
+    pluginNamesToOwnedNodeTypes: state.pluginNamesToOwnedNodeTypes,
+    statefulSourcePlugins: state.statefulSourcePlugins,
     status: state.status,
     components: state.components,
     jobsV2: state.jobsV2,

packages/gatsby/src/redux/reducers/index.ts

@@ -1,3 +1,4 @@
+import { pluginNamesToOwnedNodeTypesReducer } from "./type-owners"
 import { nodesReducer } from "./nodes"
 import { reducer as logReducer } from "gatsby-cli/lib/reporter/redux/reducers/logs"
 import { pagesReducer } from "./pages"
@@ -32,6 +33,7 @@ import { telemetryReducer } from "./telemetry"
 import { nodeManifestReducer } from "./node-manifest"
 import { reducer as pageTreeReducer } from "gatsby-cli/lib/reporter/redux/reducers/page-tree"
 import { setRequestHeadersReducer } from "./set-request-headers"
+import { statefulSourcePluginsReducer } from "./stateful-source-plugins"
 import { slicesReducer } from "./slices"
 import { componentsUsingSlicesReducer } from "./components-using-slices"
 import { slicesByTemplateReducer } from "./slices-by-template"
@@ -42,6 +44,7 @@ import { slicesByTemplateReducer } from "./slices-by-template"
 export {
   definitionsReducer as definitions,
   programReducer as program,
+  pluginNamesToOwnedNodeTypesReducer as pluginNamesToOwnedNodeTypes,
   nodesReducer as nodes,
   nodesByTypeReducer as nodesByType,
   resolvedNodesCacheReducer as resolvedNodesCache,
@@ -73,6 +76,7 @@ export {
   nodeManifestReducer as nodeManifests,
   pageTreeReducer as pageTree,
   setRequestHeadersReducer as requestHeaders,
+  statefulSourcePluginsReducer as statefulSourcePlugins,
   slicesReducer as slices,
   componentsUsingSlicesReducer as componentsUsingSlices,
   slicesByTemplateReducer as slicesByTemplate,

packages/gatsby/src/redux/reducers/stateful-source-plugins.ts

@@ -0,0 +1,19 @@
+import { IGatsbyState, IEnableStatefulSourcePluginAction } from "../types"
+
+/**
+ * Flags a source plugin as being "stateful" which means it manages its own data updates and Gatsby doesn't look for "stale" nodes after each `sourceNodes` run.
+ */
+export const statefulSourcePluginsReducer = (
+  statefulSourcePlugins: IGatsbyState["statefulSourcePlugins"] = new Set(),
+  action: IEnableStatefulSourcePluginAction
+): IGatsbyState["statefulSourcePlugins"] => {
+  switch (action.type) {
+    case `ENABLE_STATEFUL_SOURCE_PLUGIN`: {
+      statefulSourcePlugins.add(action.plugin.name)
+
+      return statefulSourcePlugins
+    }
+    default:
+      return statefulSourcePlugins
+  }
+}

packages/gatsby/src/redux/reducers/type-owners.ts

@@ -0,0 +1,26 @@
+import { ActionsUnion, IGatsbyState } from "../types"
+
+export const pluginNamesToOwnedNodeTypesReducer = (
+  pluginNamesToOwnedNodeTypes: IGatsbyState["pluginNamesToOwnedNodeTypes"] = new Map(),
+  action: ActionsUnion
+): IGatsbyState["pluginNamesToOwnedNodeTypes"] => {
+  switch (action.type) {
+    case `SET_TYPE_OWNER`: {
+      const { typeName } = action.payload
+      const owner = action.plugin.name
+
+      const existingOwner = pluginNamesToOwnedNodeTypes.get(owner)
+
+      if (!existingOwner) {
+        pluginNamesToOwnedNodeTypes.set(owner, new Set([typeName]))
+      } else {
+        existingOwner.add(typeName)
+      }
+
+      return pluginNamesToOwnedNodeTypes
+    }
+
+    default:
+      return pluginNamesToOwnedNodeTypes
+  }
+}

packages/gatsby/src/redux/types.ts

@@ -262,8 +262,13 @@ export interface IGatsbyState {
   nodesByType: Map<string, GatsbyNodes>
   resolvedNodesCache: Map<string, any> // TODO
   nodesTouched: Set<string>
+  pluginNamesToOwnedNodeTypes: Map<
+    IGatsbyPlugin[`name`],
+    Set<IGatsbyNode[`internal`][`type`]>
+  >
   nodeManifests: Array<INodeManifest>
   requestHeaders: Map<string, { [header: string]: string }>
+  statefulSourcePlugins: Set<string>
   telemetry: ITelemetry
   lastAction: ActionsUnion
   flattenedPlugins: Array<{
@@ -397,6 +402,8 @@ export type GatsbyStateKeys = keyof IGatsbyState
 
 export interface ICachedReduxState {
   nodes?: IGatsbyState["nodes"]
+  pluginNamesToOwnedNodeTypes?: IGatsbyState["pluginNamesToOwnedNodeTypes"]
+  statefulSourcePlugins?: IGatsbyState["statefulSourcePlugins"]
   status: IGatsbyState["status"]
   components: IGatsbyState["components"]
   jobsV2: IGatsbyState["jobsV2"]
@@ -418,6 +425,7 @@ export type ActionsUnion =
   | IAddThirdPartySchema
   | IApiFinishedAction
   | ICreateFieldExtension
+  | ISetTypeOwnerAction
   | ICreateNodeAction
   | ICreatePageAction
   | ICreatePageDependencyAction
@@ -487,6 +495,7 @@ export type ActionsUnion =
   | ISetJobV2Context
   | IClearJobV2Context
   | ISetDomainRequestHeaders
+  | IEnableStatefulSourcePluginAction
   | ICreateSliceAction
   | IDeleteSliceAction
   | ISetSSRTemplateWebpackCompilationHashAction
@@ -984,6 +993,14 @@ export interface ISetSiteFunctions {
   payload: IGatsbyState["functions"]
 }
 
+export interface ISetTypeOwnerAction {
+  type: `SET_TYPE_OWNER`
+  plugin: IGatsbyPlugin
+  payload: {
+    typeName: IGatsbyNode["internal"]["type"]
+  }
+}
+
 export interface ICreateNodeAction {
   type: `CREATE_NODE`
   payload: IGatsbyNode
@@ -1135,6 +1152,11 @@ export interface ISetDomainRequestHeaders {
   }
 }
 
+export interface IEnableStatefulSourcePluginAction {
+  type: `ENABLE_STATEFUL_SOURCE_PLUGIN`
+  plugin: IGatsbyPlugin
+}
+
 export interface IProcessGatsbyImageSourceUrlAction {
   type: `PROCESS_GATSBY_IMAGE_SOURCE_URL`
   payload: {

packages/gatsby/src/utils/api-runner-node.js

@@ -98,14 +98,20 @@ const doubleBind = (boundActionCreators, api, plugin, actionOptions) => {
       const boundActionCreator = boundActionCreators[key]
       if (typeof boundActionCreator === `function`) {
         doubleBoundActionCreators[key] = (...args) => {
+          if (args.length === 0) {
+            return boundActionCreator(plugin, actionOptions)
+          }
           // Let action callers override who the plugin is. Shouldn't be
           // used that often.
-          if (args.length === 1) {
+          else if (args.length === 1) {
             return boundActionCreator(args[0], plugin, actionOptions)
           } else if (args.length === 2) {
             return boundActionCreator(args[0], args[1], actionOptions)
           }
-          return undefined
+
+          throw new Error(
+            `Unhandled redux action: ${key}, in plugin: ${plugin.name}`
+          )
         }
       }
     }