From 2b79805f66c9f89a57200c6d5616aa6a4731671a Mon Sep 17 00:00:00 2001 From: Peter van der Zee Date: Fri, 21 Feb 2020 17:44:24 +0100 Subject: [PATCH] Fix mock for glob, fix infinite loop --- .../__tests__/__snapshots__/index.js.snap | 63 ++++++++++- packages/gatsby/src/redux/__tests__/index.js | 105 +++++++++++++++--- packages/gatsby/src/redux/persist.ts | 12 +- 3 files changed, 158 insertions(+), 22 deletions(-) diff --git a/packages/gatsby/src/redux/__tests__/__snapshots__/index.js.snap b/packages/gatsby/src/redux/__tests__/__snapshots__/index.js.snap index 9b332e53a51f8..0da13e1af0fa5 100644 --- a/packages/gatsby/src/redux/__tests__/__snapshots__/index.js.snap +++ b/packages/gatsby/src/redux/__tests__/__snapshots__/index.js.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`redux db should write cache to disk 1`] = ` +exports[`redux db should write loki cache to disk 1`] = ` Object { "componentDataDependencies": Object { "connections": Map {}, @@ -28,3 +28,64 @@ Object { "webpackCompilationHash": "", } `; + +exports[`redux db should write redux cache to disk 1`] = ` +Object { + "componentDataDependencies": Object { + "connections": Map {}, + "nodes": Map {}, + }, + "components": Map { + "/Users/username/dev/site/src/templates/my-sweet-new-page.js" => Object { + "componentPath": "/Users/username/dev/site/src/templates/my-sweet-new-page.js", + "isInBootstrap": true, + "pages": Set { + "/my-sweet-new-page/", + }, + "query": "", + }, + }, + "jobsV2": Object { + "complete": Map {}, + "incomplete": Map {}, + }, + "nodes": Map { + "pageA" => Object { + "id": "pageA", + "internal": Object { + "type": "Ding", + }, + }, + "pageB" => Object { + "id": "pageB", + "internal": Object { + "type": "Dong", + }, + }, + }, + "nodesByType": Map { + "Ding" => Map { + "pageA" => Object { + "id": "pageA", + "internal": Object { + "type": "Ding", + }, + }, + }, + "Dong" => Map { + "pageB" => Object { + "id": "pageB", + "internal": Object { + "type": "Dong", + }, + }, + }, + }, + "pageDataStats": Map {}, + "staticQueryComponents": Map {}, + "status": Object { + "plugins": Object {}, + }, + "webpackCompilationHash": "", +} +`; diff --git a/packages/gatsby/src/redux/__tests__/index.js b/packages/gatsby/src/redux/__tests__/index.js index c31e9d11b56b7..e90eb1742765f 100644 --- a/packages/gatsby/src/redux/__tests__/index.js +++ b/packages/gatsby/src/redux/__tests__/index.js @@ -45,6 +45,47 @@ jest.mock(`fs-extra`, () => { removeSync: jest.fn(file => mockWrittenContent.delete(file)), } }) +jest.mock(`glob`, () => { + return { + sync: jest.fn(pattern => { + // Tricky. + // Expecting a path prefix, ending with star. Else this won't work :/ + if (pattern.slice(-1) !== `*`) { + throw new Error(`Expected pattern ending with star`) + } + let globPrefix = pattern.slice(0, -1) + if (globPrefix.includes(`*`)) { + throw new Error(`Expected pattern to be a prefix`) + } + const files = [] + mockWrittenContent.forEach((value, key) => { + if (key.startsWith(globPrefix)) { + files.push(key) + } + }) + return files + }), + } +}) + +function getFakeNodes() { + // Set nodes to something or the cache will fail because it asserts this + // Actual nodes content should match TS type; these are verified + let map /*: Map*/ = new Map() + map.set(`pageA`, { + id: `pageA`, + internal: { + type: `Ding`, + }, + }) + map.set(`pageB`, { + id: `pageB`, + internal: { + type: `Dong`, + }, + }) + return map +} describe(`redux db`, () => { const initialComponentsState = _.cloneDeep(store.getState().components) @@ -70,29 +111,61 @@ describe(`redux db`, () => { mockWrittenContent.clear() }) - it(`should write cache to disk`, async () => { - expect(initialComponentsState).toEqual(new Map()) + // yuck - loki and redux will have different shape of redux state (nodes and nodesByType) + // Note: branched skips will keep snapshots with and without loki env var + if (process.env.GATSBY_DB_NODES === `loki`) { + it.skip(`should write redux cache to disk`, async () => {}) + it(`should write loki cache to disk`, async () => { + expect(initialComponentsState).toEqual(new Map()) - await saveState() + store.getState().nodes = getFakeNodes() - expect(writeToCache).toBeCalled() + await saveState() - // reset state in memory - store.dispatch({ - type: `DELETE_CACHE`, + expect(writeToCache).toBeCalled() + + // reset state in memory + store.dispatch({ + type: `DELETE_CACHE`, + }) + // make sure store in memory is empty + expect(store.getState().components).toEqual(initialComponentsState) + + // read data that was previously cached + const data = readState() + + // make sure data was read and is not the same as our clean redux state + expect(data.components).not.toEqual(initialComponentsState) + + expect(_.omit(data, [`nodes`, `nodesByType`])).toMatchSnapshot() }) - // make sure store in memory is empty - expect(store.getState().components).toEqual(initialComponentsState) + } else { + it.skip(`should write loki cache to disk`, async () => {}) + it(`should write redux cache to disk`, async () => { + expect(initialComponentsState).toEqual(new Map()) - // read data that was previously cached - const data = readState() + store.getState().nodes = getFakeNodes() - // make sure data was read and is not the same as our clean redux state - expect(data.components).not.toEqual(initialComponentsState) + await saveState() - // yuck - loki and redux will have different shape of redux state (nodes and nodesByType) - expect(_.omit(data, [`nodes`, `nodesByType`])).toMatchSnapshot() - }) + expect(writeToCache).toBeCalled() + + // reset state in memory + store.dispatch({ + type: `DELETE_CACHE`, + }) + // make sure store in memory is empty + expect(store.getState().components).toEqual(initialComponentsState) + + // read data that was previously cached + const data = readState() + + // make sure data was read and is not the same as our clean redux state + expect(data.components).not.toEqual(initialComponentsState) + + expect(data).toMatchSnapshot() + }) + } it(`should drop legacy file if exists`, async () => { expect(initialComponentsState).toEqual(new Map()) diff --git a/packages/gatsby/src/redux/persist.ts b/packages/gatsby/src/redux/persist.ts index e5d38c98e75c4..48fa9393f68ad 100644 --- a/packages/gatsby/src/redux/persist.ts +++ b/packages/gatsby/src/redux/persist.ts @@ -56,9 +56,7 @@ export function readFromCache(): ICachedReduxState { const nodes: [string, IReduxNode][] = [].concat(...chunks) - if (chunks.length) { - obj.nodes = new Map(nodes) - } else { + if (!chunks.length) { report.info( `Cache exists but contains no nodes. There should be at least some nodes available so it seems the cache was corrupted. Disregarding the cache and proceeding as if there was none.` ) @@ -66,6 +64,8 @@ export function readFromCache(): ICachedReduxState { return {} as ICachedReduxState } + obj.nodes = new Map(nodes) + return obj } @@ -78,7 +78,7 @@ function guessSafeChunkSize(values: [string, IReduxNode][]): number { const nodesToTest = 11 // Very arbitrary number const valueCount = values.length - const step = Math.floor(valueCount / nodesToTest) + const step = Math.max(1, Math.ceil(valueCount / nodesToTest)) let maxSize = 0 for (let i = 0; i < valueCount; i += step) { const size = v8.serialize(values[i]).length @@ -99,6 +99,7 @@ function prepareCacheFolder( // This prevents an OOM when the page nodes collectively contain to much data const map = contents.nodes contents.nodes = undefined + writeFileSync(reduxRestFile(targetDir), v8.serialize(contents)) // Now restore them on the redux store contents.nodes = map @@ -124,6 +125,7 @@ function safelyRenameToBak(reduxCacheFolder: string): string { const tmpSuffix = `.bak` let suffixCounter = 0 let bakName = reduxCacheFolder + tmpSuffix // Start without number + while (existsSync(bakName)) { ++suffixCounter bakName = reduxCacheFolder + tmpSuffix + suffixCounter @@ -166,7 +168,7 @@ export function writeToCache(contents: ICachedReduxState): void { removeSync(bakName) } } catch (e) { - console.warn( + report.warn( `Non-fatal: Deleting the old cache folder failed, left behind in \`${bakName}\`. Rimraf reported this error: ${e}` ) }