For Contentful, filter out unresolvable entries and create markdown t…

…ext nodes (#1202)

docs/docs/plugins.md

@@ -94,6 +94,7 @@ you can place the files in a `src` subfolder and build them to the plugin folder
 * [gatsby-remark-responsive-image](/docs/packages/gatsby-remark-responsive-image/)
 * [gatsby-remark-smartypants](/docs/packages/gatsby-remark-smartypants/)
 * [gatsby-sharp](/docs/packages/gatsby-sharp/)
+* [gatsby-source-contentful](/docs/packages/gatsby-source-contentful/)
 * [gatsby-source-drupal](/docs/packages/gatsby-source-drupal/)
 * [gatsby-source-filesystem](/docs/packages/gatsby-source-filesystem/)
 * [gatsby-source-hacker-news](/docs/packages/gatsby-source-hacker-news/)

packages/gatsby-plugin-offline/src/app-shell.js

@@ -2,6 +2,8 @@ import React from "react"
 
 class AppShell extends React.Component {
  componentDidMount() {
+ // TODO check if page exists and if not,
+ // force a hard reload.
  window.___navigateTo(this.props.location.pathname)
  }
 

packages/gatsby-source-contentful/package.json

@@ -17,6 +17,7 @@
  "axios": "^0.16.1",
  "bluebird": "^3.4.6",
  "contentful": "^4.3.0",
+ "json-stringify-safe": "^5.0.1",
  "lodash": "^4.17.2"
  }
 }

packages/gatsby-source-contentful/src/gatsby-node.js

@@ -1,5 +1,8 @@
 const contentful = require(`contentful`)
 const crypto = require(`crypto`)
+const stringify = require("json-stringify-safe")
+
+const digest = str => crypto.createHash(`md5`).update(str).digest(`hex`)
 
 const typePrefix = `contentful__`
 const conflictFieldPrefix = `contentful`
@@ -58,6 +61,19 @@ exports.sourceNodes = async (
  console.log(`assets fetched`, assets.items.length)
  console.timeEnd(`fetch Contentful data`)
 
+ // Create map of not resolvable ids so we can filter them out while creating
+ // links.
+ const notResolvable = new Map()
+ entryList.forEach(ents => {
+ if (ents.errors) {
+ ents.errors.forEach(error => {
+ if (error.sys.id === `notResolvable`) {
+ notResolvable.set(error.details.id, error.details)
+ }
+ })
+ }
+ })
+
  const contentTypeItems = contentTypes.items
 
  // Build foreign reference map before starting to insert any nodes
@@ -75,6 +91,11 @@ exports.sourceNodes = async (
  entryItemFieldValue[0].sys.id
  ) {
  entryItemFieldValue.forEach(v => {
+ // Don't create link to an unresolvable field.
+ if (notResolvable.has(v.sys.id)) {
+ return
+ }
+
  if (!foreignReferenceMap[v.sys.id]) {
  foreignReferenceMap[v.sys.id] = []
  }
@@ -87,7 +108,8 @@ exports.sourceNodes = async (
  } else if (
  entryItemFieldValue.sys &&
  entryItemFieldValue.sys.type &&
- entryItemFieldValue.sys.id
+ entryItemFieldValue.sys.id &&
+ !notResolvable.has(entryItemFieldValue.sys.id)
  ) {
  if (!foreignReferenceMap[entryItemFieldValue.sys.id]) {
  foreignReferenceMap[entryItemFieldValue.sys.id] = []
@@ -101,6 +123,26 @@ exports.sourceNodes = async (
  })
  })
 
+ function createTextNode(node, text, createNode) {
+ const textNode = {
+ id: `${node.id}TextNode`,
+ parent: node.id,
+ children: [],
+ text,
+ internal: {
+ type: `ComponentDescription`,
+ mediaType: `text/x-markdown`,
+ content: text,
+ contentDigest: digest(text),
+ },
+ }
+
+ node.children = node.children.concat([textNode.id])
+ createNode(textNode)
+
+ return textNode.id
+ }
+
  contentTypeItems.forEach((contentTypeItem, i) => {
  const contentTypeItemId = contentTypeItem.sys.id
 
@@ -138,13 +180,17 @@ exports.sourceNodes = async (
  ) {
  entryItemFields[
  `${entryItemFieldKey}___NODE`
- ] = entryItemFieldValue.map(v => v.sys.id)
+ ] = entryItemFieldValue
+ .filter(v => !notResolvable.has(v.sys.id))
+ .map(v => v.sys.id)
+
  delete entryItemFields[entryItemFieldKey]
  }
  } else if (
  entryItemFieldValue.sys &&
  entryItemFieldValue.sys.type &&
- entryItemFieldValue.sys.id
+ entryItemFieldValue.sys.id &&
+ !notResolvable.has(entryItemFieldValue.sys.id)
  ) {
  entryItemFields[`${entryItemFieldKey}___NODE`] =
  entryItemFieldValue.sys.id
@@ -167,30 +213,42 @@ exports.sourceNodes = async (
  })
  }
 
- const entryNode = {
+ let entryNode = {
  id: entryItem.sys.id,
  parent: contentTypeItemId,
  children: [],
- ...entryItemFields,
  internal: {
  type: `${makeTypeName(contentTypeItemId)}`,
- content: JSON.stringify(entryItem),
  mediaType: `application/json`,
  },
  }
 
+ // Replace text fields with text nodes so we can process their markdown
+ // into HTML.
+ Object.keys(entryItemFields).forEach(entryItemFieldKey => {
+ if (entryItemFieldKey === `text`) {
+ entryItemFields[`${entryItemFieldKey}___NODE`] = createTextNode(
+ entryNode,
+ entryItemFields[entryItemFieldKey],
+ createNode
+ )
+
+ delete entryItemFields[entryItemFieldKey]
+ }
+ })
+
+ entryNode = { ...entryItemFields, ...entryNode }
+
  // Get content digest of node.
- const contentDigest = crypto
- .createHash(`md5`)
- .update(JSON.stringify(entryNode))
- .digest(`hex`)
+ const contentDigest = digest(stringify(entryNode))
 
  entryNode.internal.contentDigest = contentDigest
 
  return entryNode
  })
+
  // Create a node for each content type
- const contentTypeItemStr = JSON.stringify(contentTypeItem)
+ const contentTypeItemStr = stringify(contentTypeItem)
 
  const contentTypeNode = {
  id: contentTypeItemId,
@@ -201,16 +259,12 @@ exports.sourceNodes = async (
  description: contentTypeItem.description,
  internal: {
  type: `${makeTypeName(`ContentType`)}`,
- content: contentTypeItemStr,
- mediaType: `application/json`,
+ mediaType: `text/x-contentful`,
  },
  }
 
  // Get content digest of node.
- const contentDigest = crypto
- .createHash(`md5`)
- .update(JSON.stringify(contentTypeNode))
- .digest(`hex`)
+ const contentDigest = digest(stringify(contentTypeNode))
 
  contentTypeNode.internal.contentDigest = contentDigest
 
@@ -222,7 +276,7 @@ exports.sourceNodes = async (
 
  assets.items.forEach(assetItem => {
  // Create a node for each asset. They may be referenced by Entries
- const assetItemStr = JSON.stringify(assetItem)
+ const assetItemStr = stringify(assetItem)
 
  const assetNode = {
  id: assetItem.sys.id,
@@ -231,16 +285,12 @@ exports.sourceNodes = async (
  ...assetItem.fields,
  internal: {
  type: `${makeTypeName(`Asset`)}`,
- content: assetItemStr,
- mediaType: `application/json`,
+ mediaType: `text/x-contentful`,
  },
  }
 
  // Get content digest of node.
- const contentDigest = crypto
- .createHash(`md5`)
- .update(JSON.stringify(assetNode))
- .digest(`hex`)
+ const contentDigest = digest(stringify(assetNode))
 
  assetNode.internal.contentDigest = contentDigest
 

packages/gatsby/package.json

@@ -55,6 +55,7 @@
  "is-relative-url": "^2.0.0",
  "joi": "^9.1.1",
  "json-loader": "^0.5.2",
+ "json-stringify-safe": "^5.0.1",
  "json5": "^0.5.0",
  "loader-utils": "^0.2.16",
  "lodash": "^4.17.4",

packages/gatsby/src/internal-plugins/dev-404-page/raw_dev-404-page.js

@@ -35,7 +35,7 @@ class Dev404Page extends React.Component {
  <h2>Pages ({this.props.data.allSitePage.totalCount})</h2>
  <ul>
  {this.props.data.allSitePage.edges.map(({ node }) =>
- <li><Link to={node.path}>{node.path}</Link></li>
+ <li key={node.path}><Link to={node.path}>{node.path}</Link></li>
  )}
  </ul>
  </div>}

packages/gatsby/src/redux/index.js

@@ -4,6 +4,7 @@ const _ = require(`lodash`)
 const { composeWithDevTools } = require(`remote-redux-devtools`)
 const fs = require(`fs`)
 const EventEmitter = require(`eventemitter2`)
+const stringify = require(`json-stringify-safe`)
 
 // Create event emitter for actions
 const emitter = new EventEmitter()
@@ -47,7 +48,7 @@ const saveState = _.debounce(state => {
  const pickedState = _.pick(state, [`nodes`, `status`, `pageDataDependencies`])
  fs.writeFile(
  `${process.cwd()}/.cache/redux-state.json`,
- JSON.stringify(pickedState, null, 2),
+ stringify(pickedState, null, 2),
  () => {}
  )
 }, 1000)

packages/gatsby/src/schema/infer-graphql-type.js

@@ -253,6 +253,7 @@ function inferFromFieldName(value, selector, types): GraphQLFieldConfig<*, *> {
  const [, , linkedField] = key.split(`___`)
 
  const linkedNode = findLinkedNode(value, linkedField)
+
  invariant(
  linkedNode,
  oneLine`
@@ -464,9 +465,10 @@ export function inferObjectStructureFromNodes({
  shouldInferFile(nodes, nextSelector, value)
  ) {
  inferredField = inferFromUri(key, types)
+ }
 
-  // Finally our automatic inference of field value type.
- } else {
+ // Finally our automatic inference of field value type.
+ if (!inferredField) {
  inferredField = inferGraphQLType({
  nodes,
  types,