Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(gatsby): Add nodes db module #9416

Merged
merged 6 commits into from
Oct 29, 2018
Merged

feat(gatsby): Add nodes db module #9416

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
790aadd
Add nodes namespace to support loki feature flag
Moocar Oct 26, 2018
4b92859
naughty console log
Moocar Oct 26, 2018
9e67724
switch for db nodes backend
Moocar Oct 26, 2018
873ae2c
Merge branch 'master' into integrate-loki2
Moocar Oct 28, 2018
49e4f0e
revert snapshot for gatsby-remark-embed
Moocar Oct 28, 2018
ca71179
bump peerDeps to point to (yet to be release) gatsby version that add…
pieh Oct 29, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 15 additions & 15 deletions packages/gatsby-remark-embed-snippet/src/__tests__/__snapshots__/index.js.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -442,16 +442,16 @@ Object {
},
"type": "html",
"value": "<div class=\\"gatsby-highlight\\">
<pre class=\\"language-jsx\\"><code><span class=\\"token tag\\"><span class=\\"token tag\\"><span class=\\"token punctuation\\">&lt;</span>div</span><span class=\\"token punctuation\\">></span></span><span class=\\"token plain-text\\"></span>
<span class=\\"gatsby-highlight-code-line\\"><span class=\\"token plain-text\\"></span><span class=\\"token tag\\"><span class=\\"token tag\\"><span class=\\"token punctuation\\">&lt;</span>button</span><span class=\\"token punctuation\\">></span></span><span class=\\"token plain-text\\">Add Item</span><span class=\\"token tag\\"><span class=\\"token tag\\"><span class=\\"token punctuation\\">&lt;/</span>button</span><span class=\\"token punctuation\\">></span></span><span class=\\"token plain-text\\"></span>
</span><span class=\\"token plain-text\\"></span>
<span class=\\"token plain-text\\"></span><span class=\\"token tag\\"><span class=\\"token tag\\"><span class=\\"token punctuation\\">&lt;</span>ReactCSSTransitionGroup</span>
<pre class=\\"language-jsx\\"><code><span class=\\"token tag\\"><span class=\\"token tag\\"><span class=\\"token punctuation\\">&lt;</span>div</span><span class=\\"token punctuation\\">></span></span><span class=\\"token plain-text\\">
<span class=\\"gatsby-highlight-code-line\\"></span><span class=\\"token tag\\"><span class=\\"token tag\\"><span class=\\"token punctuation\\">&lt;</span>button</span><span class=\\"token punctuation\\">></span></span><span class=\\"token plain-text\\">Add Item</span><span class=\\"token tag\\"><span class=\\"token tag\\"><span class=\\"token punctuation\\">&lt;/</span>button</span><span class=\\"token punctuation\\">></span></span><span class=\\"token plain-text\\">
</span>
</span><span class=\\"token tag\\"><span class=\\"token tag\\"><span class=\\"token punctuation\\">&lt;</span>ReactCSSTransitionGroup</span>
<span class=\\"token attr-name\\">transitionName</span><span class=\\"token attr-value\\"><span class=\\"token punctuation\\">=</span><span class=\\"token punctuation\\">\\"</span>example<span class=\\"token punctuation\\">\\"</span></span>
<span class=\\"token attr-name\\">transitionEnterTimeout</span><span class=\\"token script language-javascript\\"><span class=\\"token script-punctuation punctuation\\">=</span><span class=\\"token punctuation\\">{</span><span class=\\"token number\\">500</span><span class=\\"token punctuation\\">}</span></span>
<span class=\\"token attr-name\\">transitionLeaveTimeout</span><span class=\\"token script language-javascript\\"><span class=\\"token script-punctuation punctuation\\">=</span><span class=\\"token punctuation\\">{</span><span class=\\"token number\\">300</span><span class=\\"token punctuation\\">}</span></span><span class=\\"token punctuation\\">></span></span><span class=\\"token plain-text\\"></span>
<span class=\\"gatsby-highlight-code-line\\"><span class=\\"token plain-text\\"></span><span class=\\"token punctuation\\">{</span>items<span class=\\"token punctuation\\">}</span><span class=\\"token plain-text\\"></span>
</span><span class=\\"token plain-text\\"></span><span class=\\"token tag\\"><span class=\\"token tag\\"><span class=\\"token punctuation\\">&lt;/</span>ReactCSSTransitionGroup</span><span class=\\"token punctuation\\">></span></span><span class=\\"token plain-text\\"></span>
<span class=\\"token plain-text\\"></span><span class=\\"token tag\\"><span class=\\"token tag\\"><span class=\\"token punctuation\\">&lt;/</span>div</span><span class=\\"token punctuation\\">></span></span></code></pre>
<span class=\\"token attr-name\\">transitionLeaveTimeout</span><span class=\\"token script language-javascript\\"><span class=\\"token script-punctuation punctuation\\">=</span><span class=\\"token punctuation\\">{</span><span class=\\"token number\\">300</span><span class=\\"token punctuation\\">}</span></span><span class=\\"token punctuation\\">></span></span><span class=\\"token plain-text\\">
<span class=\\"gatsby-highlight-code-line\\"></span><span class=\\"token punctuation\\">{</span>items<span class=\\"token punctuation\\">}</span><span class=\\"token plain-text\\">
</span></span><span class=\\"token tag\\"><span class=\\"token tag\\"><span class=\\"token punctuation\\">&lt;/</span>ReactCSSTransitionGroup</span><span class=\\"token punctuation\\">></span></span><span class=\\"token plain-text\\">
</span><span class=\\"token tag\\"><span class=\\"token tag\\"><span class=\\"token punctuation\\">&lt;/</span>div</span><span class=\\"token punctuation\\">></span></span></code></pre>
</div>",
},
],
Expand Down Expand Up @@ -632,13 +632,13 @@ Object {
},
"type": "html",
"value": "<div class=\\"gatsby-highlight\\">
<pre class=\\"language-jsx\\"><code><span class=\\"token tag\\"><span class=\\"token tag\\"><span class=\\"token punctuation\\">&lt;</span>ul</span><span class=\\"token punctuation\\">></span></span><span class=\\"token plain-text\\"></span>
<span class=\\"token plain-text\\"></span><span class=\\"token tag\\"><span class=\\"token tag\\"><span class=\\"token punctuation\\">&lt;</span>li</span><span class=\\"token punctuation\\">></span></span><span class=\\"token plain-text\\">Not highlighted</span><span class=\\"token tag\\"><span class=\\"token tag\\"><span class=\\"token punctuation\\">&lt;/</span>li</span><span class=\\"token punctuation\\">></span></span><span class=\\"token plain-text\\"></span>
<span class=\\"gatsby-highlight-code-line\\"><span class=\\"token plain-text\\"></span><span class=\\"token tag\\"><span class=\\"token tag\\"><span class=\\"token punctuation\\">&lt;</span>li</span><span class=\\"token punctuation\\">></span></span><span class=\\"token plain-text\\">Highlighted</span><span class=\\"token tag\\"><span class=\\"token tag\\"><span class=\\"token punctuation\\">&lt;/</span>li</span><span class=\\"token punctuation\\">></span></span><span class=\\"token plain-text\\"></span>
</span><span class=\\"gatsby-highlight-code-line\\"><span class=\\"token plain-text\\"></span><span class=\\"token tag\\"><span class=\\"token tag\\"><span class=\\"token punctuation\\">&lt;</span>li</span><span class=\\"token punctuation\\">></span></span><span class=\\"token plain-text\\">Highlighted</span><span class=\\"token tag\\"><span class=\\"token tag\\"><span class=\\"token punctuation\\">&lt;/</span>li</span><span class=\\"token punctuation\\">></span></span><span class=\\"token plain-text\\"></span>
</span><span class=\\"gatsby-highlight-code-line\\"><span class=\\"token plain-text\\"></span><span class=\\"token tag\\"><span class=\\"token tag\\"><span class=\\"token punctuation\\">&lt;</span>li</span><span class=\\"token punctuation\\">></span></span><span class=\\"token plain-text\\">Highlighted</span><span class=\\"token tag\\"><span class=\\"token tag\\"><span class=\\"token punctuation\\">&lt;/</span>li</span><span class=\\"token punctuation\\">></span></span><span class=\\"token plain-text\\"></span>
</span><span class=\\"token plain-text\\"></span><span class=\\"token tag\\"><span class=\\"token tag\\"><span class=\\"token punctuation\\">&lt;</span>li</span><span class=\\"token punctuation\\">></span></span><span class=\\"token plain-text\\">Not highlighted</span><span class=\\"token tag\\"><span class=\\"token tag\\"><span class=\\"token punctuation\\">&lt;/</span>li</span><span class=\\"token punctuation\\">></span></span><span class=\\"token plain-text\\"></span>
<span class=\\"token plain-text\\"></span><span class=\\"token tag\\"><span class=\\"token tag\\"><span class=\\"token punctuation\\">&lt;/</span>div</span><span class=\\"token punctuation\\">></span></span></code></pre>
<pre class=\\"language-jsx\\"><code><span class=\\"token tag\\"><span class=\\"token tag\\"><span class=\\"token punctuation\\">&lt;</span>ul</span><span class=\\"token punctuation\\">></span></span><span class=\\"token plain-text\\">
</span><span class=\\"token tag\\"><span class=\\"token tag\\"><span class=\\"token punctuation\\">&lt;</span>li</span><span class=\\"token punctuation\\">></span></span><span class=\\"token plain-text\\">Not highlighted</span><span class=\\"token tag\\"><span class=\\"token tag\\"><span class=\\"token punctuation\\">&lt;/</span>li</span><span class=\\"token punctuation\\">></span></span><span class=\\"token plain-text\\">
<span class=\\"gatsby-highlight-code-line\\"></span><span class=\\"token tag\\"><span class=\\"token tag\\"><span class=\\"token punctuation\\">&lt;</span>li</span><span class=\\"token punctuation\\">></span></span><span class=\\"token plain-text\\">Highlighted</span><span class=\\"token tag\\"><span class=\\"token tag\\"><span class=\\"token punctuation\\">&lt;/</span>li</span><span class=\\"token punctuation\\">></span></span><span class=\\"token plain-text\\">
</span><span class=\\"gatsby-highlight-code-line\\"></span><span class=\\"token tag\\"><span class=\\"token tag\\"><span class=\\"token punctuation\\">&lt;</span>li</span><span class=\\"token punctuation\\">></span></span><span class=\\"token plain-text\\">Highlighted</span><span class=\\"token tag\\"><span class=\\"token tag\\"><span class=\\"token punctuation\\">&lt;/</span>li</span><span class=\\"token punctuation\\">></span></span><span class=\\"token plain-text\\">
</span><span class=\\"gatsby-highlight-code-line\\"></span><span class=\\"token tag\\"><span class=\\"token tag\\"><span class=\\"token punctuation\\">&lt;</span>li</span><span class=\\"token punctuation\\">></span></span><span class=\\"token plain-text\\">Highlighted</span><span class=\\"token tag\\"><span class=\\"token tag\\"><span class=\\"token punctuation\\">&lt;/</span>li</span><span class=\\"token punctuation\\">></span></span><span class=\\"token plain-text\\">
</span></span><span class=\\"token tag\\"><span class=\\"token tag\\"><span class=\\"token punctuation\\">&lt;</span>li</span><span class=\\"token punctuation\\">></span></span><span class=\\"token plain-text\\">Not highlighted</span><span class=\\"token tag\\"><span class=\\"token tag\\"><span class=\\"token punctuation\\">&lt;/</span>li</span><span class=\\"token punctuation\\">></span></span><span class=\\"token plain-text\\">
</span><span class=\\"token tag\\"><span class=\\"token tag\\"><span class=\\"token punctuation\\">&lt;/</span>div</span><span class=\\"token punctuation\\">></span></span></code></pre>
</div>",
},
],
Expand Down
6 changes: 2 additions & 4 deletions packages/gatsby-source-contentful/src/gatsby-node.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -216,17 +216,15 @@ exports.sourceNodes = async (
// Check if there are any ContentfulAsset nodes and if gatsby-image is installed. If so,
// add fragments for ContentfulAsset and gatsby-image. The fragment will cause an error
// if there's not ContentfulAsset nodes and without gatsby-image, the fragment is useless.
exports.onPreExtractQueries = async ({ store, getNodes }) => {
exports.onPreExtractQueries = async ({ store, getNodesByType }) => {
const program = store.getState().program

const CACHE_DIR = path.resolve(
`${program.directory}/.cache/contentful/assets/`
)
await fs.ensureDir(CACHE_DIR)

const nodes = getNodes()

if (!nodes.some(n => n.internal.type === `ContentfulAsset`)) {
if (getNodesByType(`ContentfulAsset`).length == 0) {
return
}

Expand Down
93 changes: 52 additions & 41 deletions packages/gatsby-transformer-remark/src/__tests__/extend-node.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,12 @@ const {
const extendNodeType = require(`../extend-node-type`)

// given a set of nodes and a query, return the result of the query
async function queryResult(nodes, fragment, { types = [] } = {}, additionalParameters) {
async function queryResult(
nodes,
fragment,
{ types = [] } = {},
additionalParameters
) {
const inferredFields = inferObjectStructureFromNodes({
nodes,
types: [...types],
Expand All @@ -23,7 +28,7 @@ async function queryResult(nodes, fragment, { types = [] } = {}, additionalParam
get: () => null,
set: () => null,
},
getNodes: () => [],
getNodesByType: type => [],
...additionalParameters,
},
{
Expand Down Expand Up @@ -70,7 +75,13 @@ async function queryResult(nodes, fragment, { types = [] } = {}, additionalParam
return result
}

const bootstrapTest = (label, content, query, test, additionalParameters = {}) => {
const bootstrapTest = (
label,
content,
query,
test,
additionalParameters = {}
) => {
const node = {
id: `whatever`,
children: [],
Expand All @@ -82,7 +93,7 @@ const bootstrapTest = (label, content, query, test, additionalParameters = {}) =
// Make some fake functions its expecting.
const loadNodeContent = node => Promise.resolve(node.content)

it(label, async (done) => {
it(label, async done => {
node.content = content
const createNode = markdownNode => {
queryResult(
Expand All @@ -96,8 +107,7 @@ const bootstrapTest = (label, content, query, test, additionalParameters = {}) =
try {
test(result.data.listNode[0])
done()
}
catch(err) {
} catch (err) {
done.fail(err)
}
})
Expand All @@ -106,19 +116,19 @@ const bootstrapTest = (label, content, query, test, additionalParameters = {}) =
const actions = { createNode, createParentChildLink }
const createNodeId = jest.fn()
createNodeId.mockReturnValue(`uuid-from-gatsby`)
await onCreateNode({
node,
loadNodeContent,
actions,
createNodeId,
},
{ ...additionalParameters }
await onCreateNode(
{
node,
loadNodeContent,
actions,
createNodeId,
},
{ ...additionalParameters }
)
})
})
}

describe(`Excerpt is generated correctly from schema`, () => {

bootstrapTest(
`correctly loads an excerpt`,
`---
Expand All @@ -131,7 +141,7 @@ Where oh where is my little pony?`,
title
}
`,
(node) => {
node => {
expect(node).toMatchSnapshot()
expect(node.excerpt).toMatch(`Where oh where is my little pony?`)
}
Expand All @@ -148,7 +158,7 @@ date: "2017-09-18T23:19:51.246Z"
title
}
`,
(node) => {
node => {
expect(node).toMatchSnapshot()
expect(node.excerpt).toMatch(``)
}
Expand All @@ -171,7 +181,7 @@ In quis lectus sed eros efficitur luctus. Morbi tempor, nisl eget feugiat tincid
title
}
`,
(node) => {
node => {
expect(node).toMatchSnapshot()
expect(node.excerpt).toMatch(`Where oh where is my little pony?`)
},
Expand All @@ -194,7 +204,7 @@ In quis lectus sed eros efficitur luctus. Morbi tempor, nisl eget feugiat tincid
title
}
`,
(node) => {
node => {
expect(node).toMatchSnapshot()
expect(node.excerpt.length).toBe(139)
}
Expand All @@ -208,7 +218,7 @@ In quis lectus sed eros efficitur luctus. Morbi tempor, nisl eget feugiat tincid
title
}
`,
(node) => {
node => {
expect(node).toMatchSnapshot()
expect(node.excerpt.length).toBe(46)
}
Expand All @@ -222,7 +232,7 @@ In quis lectus sed eros efficitur luctus. Morbi tempor, nisl eget feugiat tincid
title
}
`,
(node) => {
node => {
expect(node).toMatchSnapshot()
expect(node.excerpt.length).toBe(50)
}
Expand All @@ -248,16 +258,15 @@ In quis lectus sed eros efficitur luctus. Morbi tempor, nisl eget feugiat tincid
frontmatter {
title
}`,
(node) => {
node => {
expect(node).toMatchSnapshot()
expect(node.wordCount).toEqual(
{
expect(node.wordCount).toEqual({
paragraphs: 2,
sentences: 19,
words: 150,
}
)
})
})
}
)

const content = `---
title: "my little pony"
Expand All @@ -276,16 +285,15 @@ date: "2017-09-18T23:19:51.246Z"
frontmatter {
title
}`,
(node) => {
node => {
expect(node).toMatchSnapshot()
expect(node.wordCount).toEqual(
{
expect(node.wordCount).toEqual({
paragraphs: null,
sentences: null,
words: null,
}
)
})
})
}
)

bootstrapTest(
`correctly uses a default value for timeToRead`,
Expand All @@ -294,10 +302,11 @@ date: "2017-09-18T23:19:51.246Z"
frontmatter {
title
}`,
(node) => {
node => {
expect(node).toMatchSnapshot()
expect(node.timeToRead).toBe(1)
})
}
)
})

describe(`Table of contents is generated correctly from schema`, () => {
Expand All @@ -322,11 +331,12 @@ some other text
frontmatter {
title
}`,
(node) => {
node => {
expect(node).toMatchSnapshot()
expect(console.warn).toBeCalled()
expect(node.tableOfContents).toBe(null)
})
}
)

bootstrapTest(
`correctly generates table of contents`,
Expand All @@ -350,9 +360,10 @@ final text
frontmatter {
title
}`,
(node) => {
expect(node).toMatchSnapshot()
})
node => {
expect(node).toMatchSnapshot()
}
)
})

describe(`Links are correctly prefixed`, () => {
Expand All @@ -366,7 +377,7 @@ This is [a reference]
[a reference]: /path/to/page2
`,
`html`,
(node) => {
node => {
expect(node).toMatchSnapshot()
expect(node.html).toMatch(`<a href="/prefix/path/to/page1">`)
expect(node.html).toMatch(`<a href="/prefix/path/to/page2">`)
Expand Down
24 changes: 18 additions & 6 deletions packages/gatsby-transformer-remark/src/extend-node-type.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ const withPathPrefix = (url, pathPrefix) =>
const ASTPromiseMap = new Map()

module.exports = (
{ type, store, pathPrefix, getNode, getNodes, cache, reporter },
{ type, store, pathPrefix, getNode, getNodesByType, cache, reporter },
pluginOptions
) => {
if (type.name !== `MarkdownRemark`) {
Expand All @@ -74,7 +74,13 @@ module.exports = (

return new Promise((resolve, reject) => {
// Setup Remark.
const { commonmark = true, footnotes = true, pedantic = true, gfm = true, blocks } = pluginOptions
const {
commonmark = true,
footnotes = true,
pedantic = true,
gfm = true,
blocks,
} = pluginOptions
const remarkOptions = {
gfm,
commonmark,
Expand Down Expand Up @@ -113,7 +119,7 @@ module.exports = (
} else {
const ASTGenerationPromise = new Promise(async resolve => {
if (process.env.NODE_ENV !== `production` || !fileNodes) {
fileNodes = getNodes().filter(n => n.internal.type === `File`)
fileNodes = getNodesByType(`File`)
}
const ast = await new Promise((resolve, reject) => {
// Use Bluebird's Promise function "each" to run remark plugins serially.
Expand Down Expand Up @@ -180,7 +186,7 @@ module.exports = (
// typegen plugins just modify the auto-generated types to add derived fields
// as well as computationally expensive fields.
if (process.env.NODE_ENV !== `production` || !fileNodes) {
fileNodes = getNodes().filter(n => n.internal.type === `File`)
fileNodes = getNodesByType(`File`)
}
// Use Bluebird's Promise function "each" to run remark plugins serially.
Promise.each(pluginOptions.plugins, plugin => {
Expand Down Expand Up @@ -249,10 +255,16 @@ module.exports = (
const addSlugToUrl = function(node) {
if (node.url) {
if (_.get(markdownNode, pathToSlugField) === undefined) {
console.warn(`Skipping TableOfContents. Field '${pathToSlugField}' missing from markdown node`)
console.warn(
`Skipping TableOfContents. Field '${pathToSlugField}' missing from markdown node`
)
return null
}
node.url = [pathPrefix, _.get(markdownNode, pathToSlugField), node.url]
node.url = [
pathPrefix,
_.get(markdownNode, pathToSlugField),
node.url,
]
.join(`/`)
.replace(/\/\//g, `/`)
}
Expand Down
Loading