Skip to content

Commit

Permalink
fix(gatsby-source-drupal): cache backlink records (#33444) (#33701)
Browse files Browse the repository at this point in the history 
* fix(gatsby-source-drupal): check relationships type exists on node before filtering (#33181) (#33228)

* fix(gatsby-source-drupal): check relationships type exists on node before filtering

* Update packages/gatsby-source-drupal/src/utils.js

Co-authored-by: Dustin Schau <DSchau@users.noreply.github.com>

* format

Co-authored-by: Dustin Schau <DSchau@users.noreply.github.com>
(cherry picked from commit d4f8355)

Co-authored-by: Kyle Mathews <mathews.kyle@gmail.com>

* chore(release): Publish

 - babel-plugin-remove-graphql-queries@3.14.0
 - babel-preset-gatsby-package@1.14.0
 - babel-preset-gatsby@1.14.0
 - create-gatsby@1.14.0
 - gatsby-admin@0.24.0
 - gatsby-cli@3.14.0
 - gatsby-codemods@2.14.0
 - gatsby-core-utils@2.14.0
 - gatsby-cypress@1.14.0
 - gatsby-design-tokens@3.14.0
 - gatsby-dev-cli@3.14.0
 - gatsby-graphiql-explorer@1.14.0
 - gatsby-legacy-polyfills@1.14.0
 - gatsby-link@3.14.0
 - gatsby-page-utils@1.14.0
 - gatsby-plugin-benchmark-reporting@1.14.0
 - gatsby-plugin-canonical-urls@3.14.0
 - gatsby-plugin-catch-links@3.14.0
 - gatsby-plugin-coffeescript@3.14.0
 - gatsby-plugin-create-client-paths@3.14.0
 - gatsby-plugin-cxs@3.14.0
 - gatsby-plugin-emotion@6.14.0
 - gatsby-plugin-facebook-analytics@3.14.0
 - gatsby-plugin-feed@3.14.0
 - gatsby-plugin-flow@2.14.0
 - gatsby-plugin-fullstory@3.14.0
 - gatsby-plugin-gatsby-cloud@3.2.0
 - gatsby-plugin-google-analytics@3.14.0
 - gatsby-plugin-google-gtag@3.14.0
 - gatsby-plugin-google-tagmanager@3.14.0
 - gatsby-plugin-graphql-config@0.23.0
 - gatsby-plugin-image@1.14.0
 - gatsby-plugin-jss@3.14.0
 - gatsby-plugin-layout@2.14.0
 - gatsby-plugin-less@5.14.0
 - gatsby-plugin-lodash@4.14.0
 - gatsby-plugin-manifest@3.14.0
 - gatsby-plugin-mdx@2.14.0
 - gatsby-plugin-netlify-cms@5.14.0
 - gatsby-plugin-no-sourcemaps@3.14.0
 - gatsby-plugin-nprogress@3.14.0
 - gatsby-plugin-offline@4.14.0
 - gatsby-plugin-page-creator@3.14.0
 - gatsby-plugin-postcss@4.14.0
 - gatsby-plugin-preact@5.14.0
 - gatsby-plugin-preload-fonts@2.14.0
 - gatsby-plugin-react-css-modules@3.14.0
 - gatsby-plugin-react-helmet@4.14.0
 - gatsby-plugin-remove-trailing-slashes@3.14.0
 - gatsby-plugin-sass@4.14.0
 - gatsby-plugin-schema-snapshot@2.14.0
 - gatsby-plugin-sharp@3.14.0
 - gatsby-plugin-sitemap@4.10.0
 - gatsby-plugin-styled-components@4.14.0
 - gatsby-plugin-styled-jsx@4.14.0
 - gatsby-plugin-styletron@6.14.0
 - gatsby-plugin-stylus@3.14.0
 - gatsby-plugin-subfont@3.14.0
 - gatsby-plugin-twitter@3.14.0
 - gatsby-plugin-typescript@3.14.0
 - gatsby-plugin-typography@3.14.0
 - gatsby-plugin-utils@1.14.0
 - gatsby-react-router-scroll@4.14.0
 - gatsby-recipes@0.25.0
 - gatsby-remark-autolink-headers@4.11.0
 - gatsby-remark-code-repls@5.11.0
 - gatsby-remark-copy-linked-files@4.11.0
 - gatsby-remark-custom-blocks@3.14.0
 - gatsby-remark-embed-snippet@6.11.0
 - gatsby-remark-graphviz@3.11.0
 - gatsby-remark-images-contentful@4.11.0
 - gatsby-remark-images@5.11.0
 - gatsby-remark-katex@5.11.0
 - gatsby-remark-prismjs@5.11.0
 - gatsby-remark-responsive-iframe@4.11.0
 - gatsby-remark-smartypants@4.11.0
 - gatsby-source-contentful@5.14.0
 - gatsby-source-drupal@4.14.0
 - gatsby-source-faker@3.14.0
 - gatsby-source-filesystem@3.14.0
 - gatsby-source-graphql@3.14.0
 - gatsby-source-hacker-news@3.14.0
 - gatsby-source-lever@3.14.0
 - gatsby-source-medium@3.14.0
 - gatsby-source-mongodb@3.14.0
 - gatsby-source-npm-package-search@3.14.0
 - gatsby-source-shopify@5.6.0
 - gatsby-source-wikipedia@3.14.0
 - gatsby-source-wordpress@5.14.0
 - gatsby-telemetry@2.14.0
 - gatsby-transformer-asciidoc@2.14.0
 - gatsby-transformer-csv@3.14.0
 - gatsby-transformer-documentationjs@5.14.0
 - gatsby-transformer-excel@3.14.0
 - gatsby-transformer-hjson@3.14.0
 - gatsby-transformer-javascript-frontmatter@3.14.0
 - gatsby-transformer-javascript-static-exports@3.14.0
 - gatsby-transformer-json@3.14.0
 - gatsby-transformer-pdf@2.14.0
 - gatsby-transformer-react-docgen@6.14.0
 - gatsby-transformer-remark@4.11.0
 - gatsby-transformer-screenshot@3.14.0
 - gatsby-transformer-sharp@3.14.0
 - gatsby-transformer-sqip@3.14.0
 - gatsby-transformer-toml@3.14.0
 - gatsby-transformer-xml@3.14.0
 - gatsby-transformer-yaml@3.14.0
 - gatsby-worker@0.5.0
 - gatsby@3.14.0

* Preserve backlink/reference lookup tables in cache

* test

* fix bad merge

* try try again

* Improve log

* Get working with tests

Co-authored-by: GatsbyJS Bot <mathews.kyle+gatsbybot@gmail.com>
Co-authored-by: Vladimir Razuvaev <vladimir.razuvaev@gmail.com>
Co-authored-by: Krzysztof Szot <kszot@reference.be>
(cherry picked from commit 0def3ac)

Co-authored-by: Kyle Mathews <mathews.kyle@gmail.com>
  • Loading branch information
@KyleAMathews
GatsbyJS Bot and KyleAMathews authored Oct 27, 2021
1 parent b80c53a commit 26c51c0
Show file tree
Hide file tree
Showing 3 changed files with 54 additions and 3 deletions.
10 changes: 10 additions & 0 deletions packages/gatsby-source-drupal/src/__tests__/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,15 @@ jest.mock(`gatsby-source-filesystem`, () => {
}
})

function makeCache() {
const store = new Map()
return {
get: async id => store.get(id),
set: async (key, value) => store.set(key, value),
store,
}
}

const normalize = require(`../normalize`)
const downloadFileSpy = jest.spyOn(normalize, `downloadFile`)

Expand Down Expand Up @@ -75,6 +84,7 @@ describe(`gatsby-source-drupal`, () => {
store,
getNode: id => nodes[id],
getNodes,
cache: makeCache(),
}

beforeAll(async () => {
Expand Down
9 changes: 9 additions & 0 deletions packages/gatsby-source-drupal/src/gatsby-node.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ const { setOptions, getOptions } = require(`./plugin-options`)

const { nodeFromData, downloadFile, isFileNode } = require(`./normalize`)
const {
initRefsLookups,
storeRefsLookups,
handleReferences,
handleWebhookUpdate,
handleDeletedNode,
Expand Down Expand Up @@ -150,6 +152,8 @@ exports.sourceNodes = async (
} = pluginOptions
const { createNode, setPluginStatus, touchNode } = actions

await initRefsLookups({ cache, getNode })

// Update the concurrency limit from the plugin options
requestQueue.concurrency = concurrentAPIRequests

Expand Down Expand Up @@ -202,6 +206,7 @@ ${JSON.stringify(webhookBody, null, 4)}`
}

changesActivity.end()
await storeRefsLookups({ cache })
return
}

Expand Down Expand Up @@ -232,6 +237,7 @@ ${JSON.stringify(webhookBody, null, 4)}`
return
}
changesActivity.end()
await storeRefsLookups({ cache })
return
}

Expand Down Expand Up @@ -362,6 +368,7 @@ ${JSON.stringify(webhookBody, null, 4)}`

drupalFetchIncrementalActivity.end()
fastBuildsSpan.finish()
await storeRefsLookups({ cache })
return
}

Expand All @@ -372,6 +379,7 @@ ${JSON.stringify(webhookBody, null, 4)}`
initialSourcing = false

if (!requireFullRebuild) {
await storeRefsLookups({ cache })
return
}
}
Expand Down Expand Up @@ -635,6 +643,7 @@ ${JSON.stringify(webhookBody, null, 4)}`
initialSourcing = false

createNodesSpan.finish()
await storeRefsLookups({ cache, getNodes })
return
}

Expand Down
38 changes: 35 additions & 3 deletions packages/gatsby-source-drupal/src/utils.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,38 @@ const {

const { getOptions } = require(`./plugin-options`)

const backRefsNamesLookup = new Map()
const referencedNodesLookup = new Map()
let backRefsNamesLookup = new Map()
let referencedNodesLookup = new Map()

const initRefsLookups = async ({ cache }) => {
const backRefsNamesLookupStr = await cache.get(`backRefsNamesLookup`)
const referencedNodesLookupStr = await cache.get(`referencedNodesLookup`)

if (backRefsNamesLookupStr) {
backRefsNamesLookup = new Map(JSON.parse(backRefsNamesLookupStr))
}

if (referencedNodesLookupStr) {
referencedNodesLookup = new Map(JSON.parse(referencedNodesLookupStr))
}
}

exports.initRefsLookups = initRefsLookups

const storeRefsLookups = async ({ cache }) => {
await Promise.all([
cache.set(
`backRefsNamesLookup`,
JSON.stringify(Array.from(backRefsNamesLookup.entries()))
),
cache.set(
`referencedNodesLookup`,
JSON.stringify(Array.from(referencedNodesLookup.entries()))
),
])
}

exports.storeRefsLookups = storeRefsLookups

const handleReferences = (
node,
Expand Down Expand Up @@ -333,7 +363,9 @@ ${JSON.stringify(nodeToUpdate, null, 4)}
}
node.internal.contentDigest = createContentDigest(node)
createNode(node)
reporter.log(`Updated Gatsby node: ${node.id}`)
reporter.log(
`Updated Gatsby node: id: ${node.id} — type: ${node.internal.type}`
)
}
}

Expand Down

0 comments on commit 26c51c0

Please sign in to comment.