fix(gatsby-source-contentful): maintain back reference map between runs

packages/gatsby-source-contentful/src/__tests__/gatsby-node.js

@@ -534,6 +534,9 @@ describe(`gatsby-node`, () => {
  Array [
  "contentful-content-types-testSpaceId-master",
  ],
+ Array [
+ "contentful-foreign-reference-map-state-testSpaceId-master",
+ ],
  ]
  `)
 
@@ -546,6 +549,7 @@ describe(`gatsby-node`, () => {
  expect(cache.set.mock.calls.map(v => v[0])).toMatchInlineSnapshot(`
  Array [
  "contentful-content-types-testSpaceId-master",
+ "contentful-foreign-reference-map-state-testSpaceId-master",
  ]
  `)
  expect(actions.createNode).toHaveBeenCalledTimes(32)

packages/gatsby-source-contentful/src/__tests__/normalize.js

@@ -101,15 +101,17 @@ describe(`generic`, () => {
  entryList,
  })
 
- const foreignReferenceMap = buildForeignReferenceMap({
+ const foreignReferenceMapState = buildForeignReferenceMap({
  contentTypeItems,
  entryList,
  resolvable,
  defaultLocale,
  space,
  useNameForId: true,
+ previousForeignReferenceMapState: undefined,
+ deletedEntries: [],
  })
- const referenceKeys = Object.keys(foreignReferenceMap)
+ const referenceKeys = Object.keys(foreignReferenceMapState.backLinks)
  const expectedReferenceKeys = [
  `2Y8LhXLnYAYqKCGEWG4EKI___Asset`,
  `3wtvPBbBjiMKqKKga8I2Cu___Asset`,
@@ -129,7 +131,7 @@ describe(`generic`, () => {
  expect(referenceKeys).toHaveLength(expectedReferenceKeys.length)
  expect(referenceKeys).toEqual(expect.arrayContaining(expectedReferenceKeys))
 
- Object.keys(foreignReferenceMap).forEach(referenceId => {
+ Object.keys(foreignReferenceMapState.backLinks).forEach(referenceId => {
  expect(resolvable).toContain(referenceId)
 
  let expectedLength = 1
@@ -139,7 +141,9 @@ describe(`generic`, () => {
  if (referenceId === `7LAnCobuuWYSqks6wAwY2a___Entry`) {
  expectedLength = 3
  }
- expect(foreignReferenceMap[referenceId]).toHaveLength(expectedLength)
+ expect(foreignReferenceMapState.backLinks[referenceId]).toHaveLength(
+ expectedLength
+ )
  })
  })
 })
@@ -156,22 +160,26 @@ describe(`Process contentful data (by name)`, () => {
  entryList,
  })
 
- const foreignReferenceMap = buildForeignReferenceMap({
+ const foreignReferenceMapState = buildForeignReferenceMap({
  contentTypeItems,
  entryList,
  resolvable,
  defaultLocale,
  space,
  useNameForId: true,
+ previousForeignReferenceMapState: undefined,
+ deletedEntries: [],
  })
 
- expect(foreignReferenceMap[`24DPGBDeGEaYy8ms4Y8QMQ___Entry`][0].name).toBe(
- `product___NODE`
- )
+ expect(
+ foreignReferenceMapState.backLinks[`24DPGBDeGEaYy8ms4Y8QMQ___Entry`][0]
+ .name
+ ).toBe(`product___NODE`)
 
- expect(foreignReferenceMap[`2Y8LhXLnYAYqKCGEWG4EKI___Asset`][0].name).toBe(
- `brand___NODE`
- )
+ expect(
+ foreignReferenceMapState.backLinks[`2Y8LhXLnYAYqKCGEWG4EKI___Asset`][0]
+ .name
+ ).toBe(`brand___NODE`)
  })
 
  it(`creates nodes for each entry`, () => {
@@ -192,6 +200,8 @@ describe(`Process contentful data (by name)`, () => {
  defaultLocale,
  space,
  useNameForId: true,
+ previousForeignReferenceMapState: undefined,
+ deletedEntries: [],
  })
 
  const createNode = jest.fn()
@@ -291,6 +301,8 @@ describe(`Process existing mutated nodes in warm build`, () => {
  defaultLocale,
  space,
  useNameForId: true,
+ previousForeignReferenceMapState: undefined,
+ deletedEntries: [],
  })
 
  const createNode = jest.fn()
@@ -377,22 +389,26 @@ describe(`Process contentful data (by id)`, () => {
  assets: currentSyncData.assets,
  entryList,
  })
- const foreignReferenceMap = buildForeignReferenceMap({
+ const foreignReferenceMapState = buildForeignReferenceMap({
  contentTypeItems,
  entryList,
  resolvable,
  defaultLocale,
  space,
  useNameForId: false,
+ previousForeignReferenceMapState: undefined,
+ deletedEntries: [],
  })
 
- expect(foreignReferenceMap[`24DPGBDeGEaYy8ms4Y8QMQ___Entry`][0].name).toBe(
- `2pqfxujwe8qsykum0u6w8m___NODE`
- )
+ expect(
+ foreignReferenceMapState.backLinks[`24DPGBDeGEaYy8ms4Y8QMQ___Entry`][0]
+ .name
+ ).toBe(`2pqfxujwe8qsykum0u6w8m___NODE`)
 
- expect(foreignReferenceMap[`2Y8LhXLnYAYqKCGEWG4EKI___Asset`][0].name).toBe(
- `sfztzbsum8coewygeuyes___NODE`
- )
+ expect(
+ foreignReferenceMapState.backLinks[`2Y8LhXLnYAYqKCGEWG4EKI___Asset`][0]
+ .name
+ ).toBe(`sfztzbsum8coewygeuyes___NODE`)
  })
 
  it(`creates nodes for each entry`, () => {
@@ -411,6 +427,8 @@ describe(`Process contentful data (by id)`, () => {
  defaultLocale,
  space,
  useNameForId: false,
+ previousForeignReferenceMapState: undefined,
+ deletedEntries: [],
  })
 
  const createNode = jest.fn()

packages/gatsby-source-contentful/src/normalize.js

@@ -101,15 +101,45 @@ export const buildResolvableSet = ({
  return resolvable
 }
 
+function cleanupReferencesFromEntry(foreignReferenceMapState, entry) {
+ const { links, backLinks } = foreignReferenceMapState
+ const entryId = entry.sys.id
+
+ const entryLinks = links[entryId]
+ if (entryLinks) {
+ entryLinks.forEach(link => {
+ const backLinksForLink = backLinks[link]
+ if (backLinksForLink) {
+ backLinks[link] = backLinksForLink.filter(({ id }) => id !== entryId)
+ }
+ })
+ }
+
+ delete links[entryId]
+}
+
 export const buildForeignReferenceMap = ({
  contentTypeItems,
  entryList,
  resolvable,
  defaultLocale,
  space,
  useNameForId,
+ previousForeignReferenceMapState,
+ deletedEntries,
 }) => {
- const foreignReferenceMap = {}
+ const foreignReferenceMapState = previousForeignReferenceMapState || {
+ links: {},
+ backLinks: {},
+ }
+
+ const { links, backLinks } = foreignReferenceMapState
+
+ for (const deletedEntry of deletedEntries) {
+ // remove stored entries from entry that is being deleted
+ cleanupReferencesFromEntry(foreignReferenceMapState, deletedEntry)
+ }
+
  contentTypeItems.forEach((contentTypeItem, i) => {
  // Establish identifier for content type
  // Use `name` if specified, otherwise, use internal id (usually a natural-language constant,
@@ -122,6 +152,9 @@ export const buildForeignReferenceMap = ({
  }
 
  entryList[i].forEach(entryItem => {
+ // clear links added in previous runs for given entry, as we will recreate them anyway
+ cleanupReferencesFromEntry(foreignReferenceMapState, entryItem)
+
  const entryItemFields = entryItem.fields
  Object.keys(entryItemFields).forEach(entryItemFieldKey => {
  if (entryItemFields[entryItemFieldKey]) {
@@ -143,15 +176,21 @@ export const buildForeignReferenceMap = ({
  return
  }
 
- if (!foreignReferenceMap[key]) {
- foreignReferenceMap[key] = []
+ if (!backLinks[key]) {
+ backLinks[key] = []
  }
- foreignReferenceMap[key].push({
+ backLinks[key].push({
  name: `${contentTypeItemId}___NODE`,
  id: entryItem.sys.id,
  spaceId: space.sys.id,
  type: entryItem.sys.type,
  })
+
+ if (!links[entryItem.sys.id]) {
+ links[entryItem.sys.id] = []
+ }
+
+ links[entryItem.sys.id].push(key)
  })
  }
  } else if (
@@ -166,22 +205,28 @@ export const buildForeignReferenceMap = ({
  return
  }
 
- if (!foreignReferenceMap[key]) {
- foreignReferenceMap[key] = []
+ if (!backLinks[key]) {
+ backLinks[key] = []
  }
- foreignReferenceMap[key].push({
+ backLinks[key].push({
  name: `${contentTypeItemId}___NODE`,
  id: entryItem.sys.id,
  spaceId: space.sys.id,
  type: entryItem.sys.type,
  })
+
+ if (!links[entryItem.sys.id]) {
+ links[entryItem.sys.id] = []
+ }
+
+ links[entryItem.sys.id].push(key)
  }
  }
  })
  })
  })
 
- return foreignReferenceMap
+ return foreignReferenceMapState
 }
 
 function prepareTextNode(id, node, key, text) {

packages/gatsby-source-contentful/src/source-nodes.js

@@ -117,6 +117,7 @@ export async function sourceNodes(
 
  const CACHE_SYNC_TOKEN = `contentful-sync-token-${sourceId}`
  const CACHE_CONTENT_TYPES = `contentful-content-types-${sourceId}`
+ const CACHE_FOREIGN_REFERENCE_MAP_STATE = `contentful-foreign-reference-map-state-${sourceId}`
 
  /*
  * Subsequent calls of Contentfuls sync API return only changed data.
@@ -240,16 +241,24 @@ export async function sourceNodes(
  assets,
  })
 
+ const previousForeignReferenceMapState = await cache.get(
+ CACHE_FOREIGN_REFERENCE_MAP_STATE
+ )
  // Build foreign reference map before starting to insert any nodes
- const foreignReferenceMap = buildForeignReferenceMap({
+ const foreignReferenceMapState = buildForeignReferenceMap({
  contentTypeItems,
  entryList,
  resolvable,
  defaultLocale,
  space,
  useNameForId: pluginConfig.get(`useNameForId`),
+ previousForeignReferenceMapState,
+ deletedEntries: currentSyncData?.deletedEntries,
  })
 
+ await cache.set(CACHE_FOREIGN_REFERENCE_MAP_STATE, foreignReferenceMapState)
+ const foreignReferenceMap = foreignReferenceMapState.backLinks
+
  reporter.verbose(`Resolving Contentful references`)
 
  const newOrUpdatedEntries = new Set()