diff --git a/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js b/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js index 381cb5db9dcb5..c334573c8fc18 100644 --- a/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js +++ b/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js @@ -534,6 +534,9 @@ describe(`gatsby-node`, () => { Array [ "contentful-content-types-testSpaceId-master", ], + Array [ + "contentful-foreign-reference-map-state-testSpaceId-master", + ], ] `) @@ -546,6 +549,7 @@ describe(`gatsby-node`, () => { expect(cache.set.mock.calls.map(v => v[0])).toMatchInlineSnapshot(` Array [ "contentful-content-types-testSpaceId-master", + "contentful-foreign-reference-map-state-testSpaceId-master", ] `) expect(actions.createNode).toHaveBeenCalledTimes(32) diff --git a/packages/gatsby-source-contentful/src/__tests__/normalize.js b/packages/gatsby-source-contentful/src/__tests__/normalize.js index 0f6ca5884f8ae..2e1f2be440e17 100644 --- a/packages/gatsby-source-contentful/src/__tests__/normalize.js +++ b/packages/gatsby-source-contentful/src/__tests__/normalize.js @@ -101,15 +101,17 @@ describe(`generic`, () => { entryList, }) - const foreignReferenceMap = buildForeignReferenceMap({ + const foreignReferenceMapState = buildForeignReferenceMap({ contentTypeItems, entryList, resolvable, defaultLocale, space, useNameForId: true, + previousForeignReferenceMapState: undefined, + deletedEntries: [], }) - const referenceKeys = Object.keys(foreignReferenceMap) + const referenceKeys = Object.keys(foreignReferenceMapState.backLinks) const expectedReferenceKeys = [ `2Y8LhXLnYAYqKCGEWG4EKI___Asset`, `3wtvPBbBjiMKqKKga8I2Cu___Asset`, @@ -129,7 +131,7 @@ describe(`generic`, () => { expect(referenceKeys).toHaveLength(expectedReferenceKeys.length) expect(referenceKeys).toEqual(expect.arrayContaining(expectedReferenceKeys)) - Object.keys(foreignReferenceMap).forEach(referenceId => { + Object.keys(foreignReferenceMapState.backLinks).forEach(referenceId => { expect(resolvable).toContain(referenceId) let expectedLength = 1 @@ -139,7 +141,9 @@ describe(`generic`, () => { if (referenceId === `7LAnCobuuWYSqks6wAwY2a___Entry`) { expectedLength = 3 } - expect(foreignReferenceMap[referenceId]).toHaveLength(expectedLength) + expect(foreignReferenceMapState.backLinks[referenceId]).toHaveLength( + expectedLength + ) }) }) }) @@ -156,22 +160,26 @@ describe(`Process contentful data (by name)`, () => { entryList, }) - const foreignReferenceMap = buildForeignReferenceMap({ + const foreignReferenceMapState = buildForeignReferenceMap({ contentTypeItems, entryList, resolvable, defaultLocale, space, useNameForId: true, + previousForeignReferenceMapState: undefined, + deletedEntries: [], }) - expect(foreignReferenceMap[`24DPGBDeGEaYy8ms4Y8QMQ___Entry`][0].name).toBe( - `product___NODE` - ) + expect( + foreignReferenceMapState.backLinks[`24DPGBDeGEaYy8ms4Y8QMQ___Entry`][0] + .name + ).toBe(`product___NODE`) - expect(foreignReferenceMap[`2Y8LhXLnYAYqKCGEWG4EKI___Asset`][0].name).toBe( - `brand___NODE` - ) + expect( + foreignReferenceMapState.backLinks[`2Y8LhXLnYAYqKCGEWG4EKI___Asset`][0] + .name + ).toBe(`brand___NODE`) }) it(`creates nodes for each entry`, () => { @@ -192,6 +200,8 @@ describe(`Process contentful data (by name)`, () => { defaultLocale, space, useNameForId: true, + previousForeignReferenceMapState: undefined, + deletedEntries: [], }) const createNode = jest.fn() @@ -291,6 +301,8 @@ describe(`Process existing mutated nodes in warm build`, () => { defaultLocale, space, useNameForId: true, + previousForeignReferenceMapState: undefined, + deletedEntries: [], }) const createNode = jest.fn() @@ -377,22 +389,26 @@ describe(`Process contentful data (by id)`, () => { assets: currentSyncData.assets, entryList, }) - const foreignReferenceMap = buildForeignReferenceMap({ + const foreignReferenceMapState = buildForeignReferenceMap({ contentTypeItems, entryList, resolvable, defaultLocale, space, useNameForId: false, + previousForeignReferenceMapState: undefined, + deletedEntries: [], }) - expect(foreignReferenceMap[`24DPGBDeGEaYy8ms4Y8QMQ___Entry`][0].name).toBe( - `2pqfxujwe8qsykum0u6w8m___NODE` - ) + expect( + foreignReferenceMapState.backLinks[`24DPGBDeGEaYy8ms4Y8QMQ___Entry`][0] + .name + ).toBe(`2pqfxujwe8qsykum0u6w8m___NODE`) - expect(foreignReferenceMap[`2Y8LhXLnYAYqKCGEWG4EKI___Asset`][0].name).toBe( - `sfztzbsum8coewygeuyes___NODE` - ) + expect( + foreignReferenceMapState.backLinks[`2Y8LhXLnYAYqKCGEWG4EKI___Asset`][0] + .name + ).toBe(`sfztzbsum8coewygeuyes___NODE`) }) it(`creates nodes for each entry`, () => { @@ -411,6 +427,8 @@ describe(`Process contentful data (by id)`, () => { defaultLocale, space, useNameForId: false, + previousForeignReferenceMapState: undefined, + deletedEntries: [], }) const createNode = jest.fn() diff --git a/packages/gatsby-source-contentful/src/normalize.js b/packages/gatsby-source-contentful/src/normalize.js index 23e655b04e27d..ed443e0dfc6d1 100644 --- a/packages/gatsby-source-contentful/src/normalize.js +++ b/packages/gatsby-source-contentful/src/normalize.js @@ -101,6 +101,23 @@ export const buildResolvableSet = ({ return resolvable } +function cleanupReferencesFromEntry(foreignReferenceMapState, entry) { + const { links, backLinks } = foreignReferenceMapState + const entryId = entry.sys.id + + const entryLinks = links[entryId] + if (entryLinks) { + entryLinks.forEach(link => { + const backLinksForLink = backLinks[link] + if (backLinksForLink) { + backLinks[link] = backLinksForLink.filter(({ id }) => id !== entryId) + } + }) + } + + delete links[entryId] +} + export const buildForeignReferenceMap = ({ contentTypeItems, entryList, @@ -108,8 +125,21 @@ export const buildForeignReferenceMap = ({ defaultLocale, space, useNameForId, + previousForeignReferenceMapState, + deletedEntries, }) => { - const foreignReferenceMap = {} + const foreignReferenceMapState = previousForeignReferenceMapState || { + links: {}, + backLinks: {}, + } + + const { links, backLinks } = foreignReferenceMapState + + for (const deletedEntry of deletedEntries) { + // remove stored entries from entry that is being deleted + cleanupReferencesFromEntry(foreignReferenceMapState, deletedEntry) + } + contentTypeItems.forEach((contentTypeItem, i) => { // Establish identifier for content type // Use `name` if specified, otherwise, use internal id (usually a natural-language constant, @@ -122,6 +152,9 @@ export const buildForeignReferenceMap = ({ } entryList[i].forEach(entryItem => { + // clear links added in previous runs for given entry, as we will recreate them anyway + cleanupReferencesFromEntry(foreignReferenceMapState, entryItem) + const entryItemFields = entryItem.fields Object.keys(entryItemFields).forEach(entryItemFieldKey => { if (entryItemFields[entryItemFieldKey]) { @@ -143,15 +176,21 @@ export const buildForeignReferenceMap = ({ return } - if (!foreignReferenceMap[key]) { - foreignReferenceMap[key] = [] + if (!backLinks[key]) { + backLinks[key] = [] } - foreignReferenceMap[key].push({ + backLinks[key].push({ name: `${contentTypeItemId}___NODE`, id: entryItem.sys.id, spaceId: space.sys.id, type: entryItem.sys.type, }) + + if (!links[entryItem.sys.id]) { + links[entryItem.sys.id] = [] + } + + links[entryItem.sys.id].push(key) }) } } else if ( @@ -166,22 +205,28 @@ export const buildForeignReferenceMap = ({ return } - if (!foreignReferenceMap[key]) { - foreignReferenceMap[key] = [] + if (!backLinks[key]) { + backLinks[key] = [] } - foreignReferenceMap[key].push({ + backLinks[key].push({ name: `${contentTypeItemId}___NODE`, id: entryItem.sys.id, spaceId: space.sys.id, type: entryItem.sys.type, }) + + if (!links[entryItem.sys.id]) { + links[entryItem.sys.id] = [] + } + + links[entryItem.sys.id].push(key) } } }) }) }) - return foreignReferenceMap + return foreignReferenceMapState } function prepareTextNode(id, node, key, text) { diff --git a/packages/gatsby-source-contentful/src/source-nodes.js b/packages/gatsby-source-contentful/src/source-nodes.js index 5b9927de5cf82..87ca7a8bd982a 100644 --- a/packages/gatsby-source-contentful/src/source-nodes.js +++ b/packages/gatsby-source-contentful/src/source-nodes.js @@ -117,6 +117,7 @@ export async function sourceNodes( const CACHE_SYNC_TOKEN = `contentful-sync-token-${sourceId}` const CACHE_CONTENT_TYPES = `contentful-content-types-${sourceId}` + const CACHE_FOREIGN_REFERENCE_MAP_STATE = `contentful-foreign-reference-map-state-${sourceId}` /* * Subsequent calls of Contentfuls sync API return only changed data. @@ -240,16 +241,24 @@ export async function sourceNodes( assets, }) + const previousForeignReferenceMapState = await cache.get( + CACHE_FOREIGN_REFERENCE_MAP_STATE + ) // Build foreign reference map before starting to insert any nodes - const foreignReferenceMap = buildForeignReferenceMap({ + const foreignReferenceMapState = buildForeignReferenceMap({ contentTypeItems, entryList, resolvable, defaultLocale, space, useNameForId: pluginConfig.get(`useNameForId`), + previousForeignReferenceMapState, + deletedEntries: currentSyncData?.deletedEntries, }) + await cache.set(CACHE_FOREIGN_REFERENCE_MAP_STATE, foreignReferenceMapState) + const foreignReferenceMap = foreignReferenceMapState.backLinks + reporter.verbose(`Resolving Contentful references`) const newOrUpdatedEntries = new Set()