From 0e9feebf11d071fb912ad25649b068eb39b78d8b Mon Sep 17 00:00:00 2001 From: Tyler Barnes Date: Thu, 13 Apr 2023 00:24:25 -0700 Subject: [PATCH] feat(gatsby-source-wordpress): opt out of stale node checks (#37920) * opt out of stale node checks * update error message --------- Co-authored-by: Michal Piechowiak --- .../src/steps/persist-cached-images.ts | 9 ++++-- .../fetch-and-create-non-node-root-fields.js | 31 ++++++++++--------- .../source-nodes/fetch-nodes/fetch-nodes.js | 9 ++++-- .../src/steps/source-nodes/index.ts | 7 ++++- .../update-nodes/fetch-node-updates.js | 5 +++ .../update-nodes/wp-actions/delete.js | 23 +++++++------- .../update-nodes/wp-actions/update.js | 23 ++++++++------ .../src/utils/gatsby-features.ts | 4 +++ 8 files changed, 69 insertions(+), 42 deletions(-) create mode 100644 packages/gatsby-source-wordpress/src/utils/gatsby-features.ts diff --git a/packages/gatsby-source-wordpress/src/steps/persist-cached-images.ts b/packages/gatsby-source-wordpress/src/steps/persist-cached-images.ts index 611a3d0a65db4..7bbce4fa00d93 100644 --- a/packages/gatsby-source-wordpress/src/steps/persist-cached-images.ts +++ b/packages/gatsby-source-wordpress/src/steps/persist-cached-images.ts @@ -2,6 +2,7 @@ import { Step } from "./../utils/run-steps" import store from "~/store" import { getGatsbyApi } from "~/utils/get-gatsby-api" import { getPersistentCache } from "~/utils/cache" +import { needToTouchNodes } from "~/utils/gatsby-features" const persistPreviouslyCachedImages: Step = async (): Promise => { const { helpers, pluginOptions } = getGatsbyApi() @@ -11,9 +12,11 @@ const persistPreviouslyCachedImages: Step = async (): Promise => { `${pluginOptions.schema.typePrefix}MediaItem` ) - // and touch them so they aren't garbage collected. - // we will remove them as needed when receiving DELETE events from WP - mediaItemNodes.forEach(node => helpers.actions.touchNode(node)) + if (needToTouchNodes) { + // and if needed touch them so they aren't garbage collected. + // we will remove them as needed when receiving DELETE events from WP + mediaItemNodes.forEach(node => helpers.actions.touchNode(node)) + } const imageNodeMetaByUrl = await getPersistentCache({ key: `image-node-meta-by-url`, diff --git a/packages/gatsby-source-wordpress/src/steps/source-nodes/create-nodes/fetch-and-create-non-node-root-fields.js b/packages/gatsby-source-wordpress/src/steps/source-nodes/create-nodes/fetch-and-create-non-node-root-fields.js index a623e36bd5d53..47fba4c4e4f50 100644 --- a/packages/gatsby-source-wordpress/src/steps/source-nodes/create-nodes/fetch-and-create-non-node-root-fields.js +++ b/packages/gatsby-source-wordpress/src/steps/source-nodes/create-nodes/fetch-and-create-non-node-root-fields.js @@ -5,6 +5,7 @@ import { createNodeWithSideEffects } from "./create-nodes" import fetchReferencedMediaItemsAndCreateNodes from "../fetch-nodes/fetch-referenced-media-items" import { CREATED_NODE_IDS } from "~/constants" import { getPersistentCache, setPersistentCache } from "~/utils/cache" +import { needToTouchNodes } from "../../../utils/gatsby-features" const fetchAndCreateNonNodeRootFields = async () => { const state = store.getState() @@ -69,20 +70,22 @@ const fetchAndCreateNonNodeRootFields = async () => { referencedMediaItemNodeIds: newMediaItemIds, }) - const previouslyCachedNodeIds = await getPersistentCache({ - key: CREATED_NODE_IDS, - }) - - const createdNodeIds = [ - ...new Set([ - ...(previouslyCachedNodeIds || []), - ...referencedMediaItemNodeIdsArray, - ]), - ] - - // save the node id's so we can touch them on the next build - // so that we don't have to refetch all nodes - await setPersistentCache({ key: CREATED_NODE_IDS, value: createdNodeIds }) + if (needToTouchNodes) { + const previouslyCachedNodeIds = await getPersistentCache({ + key: CREATED_NODE_IDS, + }) + + const createdNodeIds = [ + ...new Set([ + ...(previouslyCachedNodeIds || []), + ...referencedMediaItemNodeIdsArray, + ]), + ] + + // save the node id's so we can touch them on the next build + // so that we don't have to refetch all nodes + await setPersistentCache({ key: CREATED_NODE_IDS, value: createdNodeIds }) + } store.dispatch.logger.stopActivityTimer({ typeName: `MediaItems`, diff --git a/packages/gatsby-source-wordpress/src/steps/source-nodes/fetch-nodes/fetch-nodes.js b/packages/gatsby-source-wordpress/src/steps/source-nodes/fetch-nodes/fetch-nodes.js index b2ae702860479..7d40a1905981a 100644 --- a/packages/gatsby-source-wordpress/src/steps/source-nodes/fetch-nodes/fetch-nodes.js +++ b/packages/gatsby-source-wordpress/src/steps/source-nodes/fetch-nodes/fetch-nodes.js @@ -15,6 +15,7 @@ import { setPersistentCache, } from "~/utils/cache" import { buildTypeName } from "../../create-schema-customization/helpers" +import { needToTouchNodes } from "../../../utils/gatsby-features" /** * fetchWPGQLContentNodes @@ -227,7 +228,9 @@ export const fetchAndCreateAllNodes = async () => { }) } - // save the node id's so we can touch them on the next build - // so that we don't have to refetch all nodes - await setPersistentCache({ key: CREATED_NODE_IDS, value: createdNodeIds }) + if (needToTouchNodes) { + // save the node id's so we can touch them on the next build + // so that we don't have to refetch all nodes + await setPersistentCache({ key: CREATED_NODE_IDS, value: createdNodeIds }) + } } diff --git a/packages/gatsby-source-wordpress/src/steps/source-nodes/index.ts b/packages/gatsby-source-wordpress/src/steps/source-nodes/index.ts index bfa331f4ac110..3c0dd6cb115cb 100644 --- a/packages/gatsby-source-wordpress/src/steps/source-nodes/index.ts +++ b/packages/gatsby-source-wordpress/src/steps/source-nodes/index.ts @@ -8,9 +8,14 @@ import store from "~/store" import fetchAndCreateNonNodeRootFields from "./create-nodes/fetch-and-create-non-node-root-fields" import { allowFileDownloaderProgressBarToClear } from "./create-nodes/create-remote-file-node/progress-bar-promise" import { sourcePreviews } from "~/steps/preview" +import { hasStatefulSourceNodes } from "~/utils/gatsby-features" const sourceNodes: Step = async helpers => { - const { cache, webhookBody, refetchAll } = helpers + const { cache, webhookBody, refetchAll, actions } = helpers + + if (hasStatefulSourceNodes) { + actions.enableStatefulSourceNodes() + } // fetch non-node root fields such as settings. // For now, we're refetching them on every build diff --git a/packages/gatsby-source-wordpress/src/steps/source-nodes/update-nodes/fetch-node-updates.js b/packages/gatsby-source-wordpress/src/steps/source-nodes/update-nodes/fetch-node-updates.js index 709a554b3e35d..280c883c53119 100644 --- a/packages/gatsby-source-wordpress/src/steps/source-nodes/update-nodes/fetch-node-updates.js +++ b/packages/gatsby-source-wordpress/src/steps/source-nodes/update-nodes/fetch-node-updates.js @@ -3,8 +3,13 @@ import { fetchAndRunWpActions } from "./wp-actions" import { formatLogMessage } from "~/utils/format-log-message" import { getGatsbyApi } from "~/utils/get-gatsby-api" import { getPersistentCache } from "~/utils/cache" +import { needToTouchNodes } from "../../../utils/gatsby-features" export const touchValidNodes = async () => { + if (!needToTouchNodes) { + return + } + const { helpers } = getGatsbyApi() const { actions } = helpers diff --git a/packages/gatsby-source-wordpress/src/steps/source-nodes/update-nodes/wp-actions/delete.js b/packages/gatsby-source-wordpress/src/steps/source-nodes/update-nodes/wp-actions/delete.js index 5c9677dc55c07..335ba29beacbe 100644 --- a/packages/gatsby-source-wordpress/src/steps/source-nodes/update-nodes/wp-actions/delete.js +++ b/packages/gatsby-source-wordpress/src/steps/source-nodes/update-nodes/wp-actions/delete.js @@ -10,16 +10,15 @@ import { fetchGraphql } from "~/utils/fetch-graphql" import { getQueryInfoBySingleFieldName } from "../../helpers" import { CREATED_NODE_IDS } from "~/constants" import { setPersistentCache, getPersistentCache } from "~/utils/cache" +import { needToTouchNodes } from "../../../../utils/gatsby-features" -const wpActionDELETE = async ({ - helpers, - // cachedNodeIds, - wpAction, -}) => { +const wpActionDELETE = async ({ helpers, wpAction }) => { const { reporter, actions, getNode } = helpers try { - const cachedNodeIds = await getPersistentCache({ key: CREATED_NODE_IDS }) + const cachedNodeIds = needToTouchNodes + ? await getPersistentCache({ key: CREATED_NODE_IDS }) + : null // get the node ID from the WPGQL id const nodeId = wpAction.referencedNodeGlobalRelayID @@ -60,7 +59,7 @@ const wpActionDELETE = async ({ wpStore: store, })) || {} - if (additionalNodeIds && additionalNodeIds.length) { + if (needToTouchNodes && additionalNodeIds && additionalNodeIds.length) { additionalNodeIds.forEach(id => cachedNodeIds.push(id)) } } @@ -81,12 +80,12 @@ const wpActionDELETE = async ({ reporter.log(``) } - // Remove this from cached node id's so we don't try to touch it - const validNodeIds = cachedNodeIds.filter(cachedId => cachedId !== nodeId) + if (needToTouchNodes) { + // Remove this from cached node id's so we don't try to touch it + const validNodeIds = cachedNodeIds.filter(cachedId => cachedId !== nodeId) - await setPersistentCache({ key: CREATED_NODE_IDS, value: validNodeIds }) - - // return validNodeIds + await setPersistentCache({ key: CREATED_NODE_IDS, value: validNodeIds }) + } } catch (e) { Object.entries(wpAction).forEach(([key, value]) => { reporter.warn(`${key} ${JSON.stringify(value)}`) diff --git a/packages/gatsby-source-wordpress/src/steps/source-nodes/update-nodes/wp-actions/update.js b/packages/gatsby-source-wordpress/src/steps/source-nodes/update-nodes/wp-actions/update.js index 4f53436feb684..4f6a2a2feb595 100644 --- a/packages/gatsby-source-wordpress/src/steps/source-nodes/update-nodes/wp-actions/update.js +++ b/packages/gatsby-source-wordpress/src/steps/source-nodes/update-nodes/wp-actions/update.js @@ -19,6 +19,7 @@ import { } from "~/steps/create-schema-customization/helpers" import { processNode } from "~/steps/source-nodes/create-nodes/process-node" import { getPersistentCache, setPersistentCache } from "~/utils/cache" +import { needToTouchNodes } from "../../../../utils/gatsby-features" export const fetchAndCreateSingleNode = async ({ singleName, @@ -170,7 +171,7 @@ export const createSingleNode = async ({ const { typeInfo } = getQueryInfoBySingleFieldName(singleName) - if (!cachedNodeIds) { + if (!cachedNodeIds && needToTouchNodes) { cachedNodeIds = await getPersistentCache({ key: CREATED_NODE_IDS }) } @@ -259,12 +260,11 @@ export const createSingleNode = async ({ if (remoteNode) { actions.createNode(remoteNode) + } + if (remoteNode && needToTouchNodes) { cachedNodeIds.push(remoteNode.id) - - if (additionalNodeIds && additionalNodeIds.length) { - additionalNodeIds.forEach(id => cachedNodeIds.push(id)) - } + additionalNodeIds?.forEach(id => cachedNodeIds.push(id)) await setPersistentCache({ key: CREATED_NODE_IDS, value: cachedNodeIds }) } @@ -304,13 +304,18 @@ const wpActionUPDATE = async ({ helpers, wpAction }) => { const existingNode = await getNode(nodeId) if (wpAction.referencedNodeStatus !== `publish`) { - // if the post status isn't publish anymore, we need to remove the node - // by removing it from cached nodes so it's garbage collected by Gatsby - const validNodeIds = cachedNodeIds.filter(cachedId => cachedId !== nodeId) + if (needToTouchNodes) { + // if the post status isn't publish anymore, we need to remove the node + // by removing it from cached nodes so it's garbage collected by Gatsby + const validNodeIds = cachedNodeIds.filter(cachedId => cachedId !== nodeId) - await setPersistentCache({ key: CREATED_NODE_IDS, value: validNodeIds }) + await setPersistentCache({ key: CREATED_NODE_IDS, value: validNodeIds }) + } if (existingNode) { + // calling touchNode here ensures owners is populated before we delete. + // In some cases calling delete node without touching the node first throws an error. this is a bug in Gatsby that has been fixed but this code remains to be backwards compatible with earlier versions that have the bug still. + // TODO remove in the next major version await actions.touchNode(existingNode) await actions.deleteNode(existingNode) reportUpdate({ setAction: `DELETE` }) diff --git a/packages/gatsby-source-wordpress/src/utils/gatsby-features.ts b/packages/gatsby-source-wordpress/src/utils/gatsby-features.ts new file mode 100644 index 0000000000000..20367892198de --- /dev/null +++ b/packages/gatsby-source-wordpress/src/utils/gatsby-features.ts @@ -0,0 +1,4 @@ +import { hasFeature } from "gatsby-plugin-utils" + +export const hasStatefulSourceNodes = hasFeature(`stateful-source-nodes`) +export const needToTouchNodes = !hasStatefulSourceNodes