Adds support for arbitrary compile-to-JS languages (#732)

- removed coffee-script, coffee-loader and node-cjsx modules from package.json - removed .coffee and .cjsx from default Webpack config - added two new API hooks: - resolvableExtensions() enables the program to resolve additional extensions as source files. By default, this array consists of [ '.js', '.jsx'] - preprocessSource() enables the program to compile pages and layouts to JS before attempting to generate query information. - when attempting to parse the source of a transpiled-to-JS file, the first parseable source returned by a plugin will be used to collect page queries (i.e., first plugin wins) - an ExportNamedDeclaration visitor is now used to acquire the pageQuery export, allowing for page queries expressed either as TemplateLiterals (i.e., ES6/TypeScript) or StringLiterals (i.e., CoffeeScript). Note that an ExportNamedDeclaration (i.e., ES6 export) must be used for this pageQuery export. Both TypeScript and CoffeeScript (2.x) support this syntax. A more traditional, CommonJS-style module.exports.pageQuery will not work.
gatsbyjs · Mar 21, 2017 · 41c5ce6 · 41c5ce6
1 parent e4db6cb
commit 41c5ce6
Show file tree

Hide file tree

Showing 5 changed files with 119 additions and 163 deletions.
diff --git a/packages/gatsby/lib/bootstrap/index.js b/packages/gatsby/lib/bootstrap/index.js
@@ -1,26 +1,23 @@
 /* @flow weak */
-import { graphql } from "graphql"
 import Promise from "bluebird"
 import queryRunner from "../utils/query-runner"
-import { pagesDB, siteDB, programDB } from "../utils/globals"
 import path from "path"
-import glob from "glob"
+import globCB from "glob"
 import _ from "lodash"
 import createPath from "./create-path"
-import mkdirp from "mkdirp"
 import fs from "fs-extra"
 import Joi from "joi"
 import chalk from "chalk"
-const { layoutComponentChunkName } = require("../utils/js-chunk-names")
+import apiRunnerNode from "../utils/api-runner-node"
+import { graphql } from "graphql"
+import { pagesDB, siteDB, programDB } from "../utils/globals"
+import { gatsbyConfigSchema, pageSchema } from "../joi-schemas/joi"
+import { layoutComponentChunkName } from "../utils/js-chunk-names"
 
 const mkdirs = Promise.promisify(fs.mkdirs)
 const copy = Promise.promisify(fs.copy)
 const removeDir = Promise.promisify(fs.remove)
-
-// Joi schemas
-import { gatsbyConfigSchema, pageSchema } from "../joi-schemas/joi"
-
-import apiRunnerNode from "../utils/api-runner-node"
+const glob = Promise.promisify(globCB)
 
 Promise.onPossiblyUnhandledRejection(error => {
   throw error
@@ -34,55 +31,52 @@ process.on(`unhandledRejection`, error => {
 // algorithm is glob /pages directory for js/jsx/cjsx files *not*
 // underscored. Then create url w/ our path algorithm *unless* user
 // takes control of that page component in gatsby-node.
-const autoPathCreator = program =>
-  new Promise(resolve => {
-    const pagesDirectory = path.join(program.directory, `pages`)
-    let autoPages = []
-    glob(`${pagesDirectory}/**/?(*.js|*.jsx|*.cjsx)`, (err, files) => {
-      // Create initial page objects.
-      autoPages = files.map(filePath => ({
-        component: filePath,
-        componentChunkName: layoutComponentChunkName(
-          program.directory,
-          filePath
-        ),
-        path: filePath,
-      }))
-
-      // Convert path to one relative to the pages directory.
-      autoPages = autoPages.map(page => ({
-        ...page,
-        path: path.relative(pagesDirectory, page.path),
-      }))
-
-      // Remove pages starting with an underscore.
-      autoPages = _.filter(autoPages, page => page.path.slice(0, 1) !== `_`)
-
-      // Remove page templates.
-      autoPages = _.filter(
-        autoPages,
-        page => page.path.slice(0, 9) !== `template-`
-      )
+const autoPathCreator = async program => {
+  const pagesDirectory = path.join(program.directory, `pages`)
+  const exts = program.extensions.map(e => `*${ e }`).join('|')
+  const files = await glob(`${pagesDirectory}/**/?(${ exts })`)
+  // Create initial page objects.
+  let autoPages = files.map(filePath => ({
+    component: filePath,
+    componentChunkName: layoutComponentChunkName(
+      program.directory,
+      filePath
+    ),
+    path: filePath,
+  }))
+
+  // Convert path to one relative to the pages directory.
+  autoPages = autoPages.map(page => ({
+    ...page,
+    path: path.relative(pagesDirectory, page.path),
+  }))
+
+  // Remove pages starting with an underscore.
+  autoPages = _.filter(autoPages, page => page.path.slice(0, 1) !== `_`)
+
+  // Remove page templates.
+  autoPages = _.filter(
+    autoPages,
+    page => page.path.slice(0, 9) !== `template-`
+  )
 
-      // Convert to our path format.
-      autoPages = autoPages.map(page => ({
-        ...page,
-        path: createPath(pagesDirectory, page.component),
-      }))
+  // Convert to our path format.
+  autoPages = autoPages.map(page => ({
+    ...page,
+    path: createPath(pagesDirectory, page.component),
+  }))
 
-      // Validate pages.
-      autoPages.forEach(page => {
-        const { error } = Joi.validate(page, pageSchema)
-        if (error) {
-          console.log(chalk.blue.bgYellow(`A page object failed validation`))
-          console.log(page)
-          console.log(chalk.bold.red(error))
-        }
-      })
-
-      resolve(autoPages)
-    })
-  })
+  // Validate pages.
+  autoPages.forEach(page => {
+    const { error } = Joi.validate(page, pageSchema)
+    if (error) {
+      console.log(chalk.blue.bgYellow(`A page object failed validation`))
+      console.log(page)
+      console.log(chalk.bold.red(error))
+    }
+  });
+  return autoPages;
+}
 
 module.exports = async program => {
   console.log(`lib/bootstrap/index.js time since started:`, process.uptime())
@@ -201,7 +195,7 @@ module.exports = async program => {
   const srcDir = `${__dirname}/../intermediate-representation-dir`
   const siteDir = `${program.directory}/.intermediate-representation`
   try {
-    //await removeDir(siteDir)
+    // await removeDir(siteDir)
     await copy(srcDir, siteDir, { clobber: true })
     await mkdirs(`${program.directory}/.intermediate-representation/json`)
   } catch (e) {
@@ -268,7 +262,7 @@ module.exports = async program => {
   // Create Schema.
   console.time(`create schema`)
   const schema = await require(`../schema/new`)()
-  //const schema = await require(`../schema`)()
+  // const schema = await require(`../schema`)()
   const graphqlRunner = (query, context) =>
     graphql(schema, query, context, context, context)
   console.timeEnd(`create schema`)
@@ -332,6 +326,11 @@ module.exports = async program => {
   pagesDB(pagesMap)
   console.log(`added pages to in-memory db`)
 
+  // Collect resolvable extensions and attach to program.
+  const extensions = [ `.js`, `.jsx` ]
+  const apiResults = await apiRunnerNode('resolvableExtensions')
+  program.extensions = apiResults.reduce((a, b) => a.concat(b), extensions)
+
   // TODO move this to own source plugin per component type
   // (js/cjsx/typescript, etc.)
   const autoPages = await autoPathCreator(program, pages)
@@ -360,8 +359,8 @@ module.exports = async program => {
   })
   console.log(`validated modified pages`)
 
-  //console.log(`bootstrap finished, time since started:`, process.uptime())
-  //cb(null, schema)
+  // console.log(`bootstrap finished, time since started:`, process.uptime())
+  // cb(null, schema)
 
   await queryRunner(program, graphqlRunner)
   await apiRunnerNode(`generateSideEffects`)

diff --git a/packages/gatsby/lib/utils/build-html.js b/packages/gatsby/lib/utils/build-html.js
@@ -6,7 +6,6 @@ import webpackConfig from "./webpack.config"
 import { pagesDB } from "./globals"
 
 const debug = require("debug")("gatsby:html")
-require(`node-cjsx`).transform()
 
 module.exports = async program => {
   const { directory } = program

diff --git a/packages/gatsby/lib/utils/query-runner.js b/packages/gatsby/lib/utils/query-runner.js
@@ -6,8 +6,8 @@ import traverse from "babel-traverse"
 import path from "path"
 import parseFilepath from "parse-filepath"
 import glob from "glob"
-const Promise = require("bluebird")
-
+import apiRunnerNode from "./api-runner-node";
+import Promise from "bluebird";
 import { pagesDB, siteDB, programDB } from "./globals"
 import { layoutComponentChunkName, pathChunkName } from "./js-chunk-names"
 
@@ -17,7 +17,8 @@ const babylon = require("babylon")
 const pascalCase = _.flow(_.camelCase, _.upperFirst)
 
 const hashStr = function(str) {
-  let hash = 5381, i = str.length
+  let hash = 5381
+  let i = str.length
 
   while (i) {
     hash = hash * 33 ^ str.charCodeAt(--i)
@@ -221,96 +222,69 @@ const writeChildRoutes = () => {
 }
 const debouncedWriteChildRoutes = _.debounce(writeChildRoutes, 250)
 
-const babelPlugin = function({ types: t }) {
-  return {
-    visitor: {
-      TemplateLiteral(path, state) {
-        if (
-          path.parentPath.parentPath.parentPath.type !==
-          `ExportNamedDeclaration`
-        ) {
-          return
-        }
-        const exportPath = path.parentPath.parentPath.parentPath
-        const name = _.get(
-          exportPath,
-          `node.declaration.declarations[0].id.name`
-        )
-        if (name === `pageQuery`) {
-          const quasis = _.get(path, `node.quasis`, [])
-          const expressions = path.get(`expressions`)
-          const chunks = []
-          quasis.forEach(quasi => {
-            chunks.push(quasi.value.cooked)
-            const expr = expressions.shift()
-            if (expr) {
-              chunks.push(
-                expr.scope.bindings[expr.node.name].path.get(
-                  `value`
-                ).parentPath.node.init.quasis[0].value.cooked
-              )
-            }
-          })
-          const query = chunks.join(``)
-          console.time(`graphql query time`)
-          const graphql = state.opts.graphql
-          //path.parentPath.replaceWithSourceString(`require('fixme.json')`);
-        }
-      },
-    },
-  }
-}
-
 // Queue for processing files
 const q = queue(
-  ({ file, graphql, directory }, callback) => {
-    const fileStr = fs.readFileSync(file, `utf-8`)
+  async ({ file, graphql, directory }, callback) => {
+    let fileStr = fs.readFileSync(file, `utf-8`)
     let ast
-    try {
-      ast = babylon.parse(fileStr, {
-        sourceType: `module`,
-        sourceFilename: true,
-        plugins: [`*`],
-      })
-    } catch (e) {
-      console.log(`Failed to parse ${file}`)
-      console.log(e)
+    // Preprocess and attempt to parse source; return an AST if we can, log an error if we can't.
+    // I'm unconvinced that this is an especially good implementation...
+    const transpiled = await apiRunnerNode(`preprocessSource`, { filename: file, contents: fileStr })
+    if (transpiled.length) {
+      for (const item of transpiled) {
+        try {
+          const tmp = babylon.parse(item, {
+            sourceType: `module`,
+            plugins: [`*`]
+          })
+          ast = tmp;
+          break
+        } catch (e) {
+          console.info(e);
+          continue
+        }
+      }
+      if (ast === undefined) {
+        console.error(`Failed to parse preprocessed file ${ file }`)
+      }
+    } else {
+      try {
+        ast = babylon.parse(fileStr, {
+          sourceType: `module`,
+          sourceFilename: true,
+          plugins: [`*`],
+        })
+      } catch (e) {
+        console.log(`Failed to parse ${file}`)
+        console.log(e)
+      }
     }
 
     // Get query for this file.
     let query
     traverse(ast, {
-      TemplateLiteral(path, state) {
-        if (
-          path.parentPath.parentPath.parentPath.type !==
-          `ExportNamedDeclaration`
-        ) {
-          return
-        }
-        const exportPath = path.parentPath.parentPath.parentPath
-        const name = _.get(
-          exportPath,
-          `node.declaration.declarations[0].id.name`
-        )
+      ExportNamedDeclaration(path, state) {
+        // cache declaration node
+        const declaration = path.node.declaration.declarations[0]
+        // we're looking for a ES6 named export called "pageQuery"
+        const name = declaration.id.name
         if (name === `pageQuery`) {
-          const quasis = _.get(path, `node.quasis`, [])
-          const expressions = path.get(`expressions`)
-          const chunks = []
-          quasis.forEach(quasi => {
-            chunks.push(quasi.value.cooked)
-            const expr = expressions.shift()
-            if (expr) {
-              chunks.push(
-                expr.scope.bindings[expr.node.name].path.get(
-                  `value`
-                ).parentPath.node.init.quasis[0].value.cooked
-              )
+          const type = declaration.init.type;
+          if (type === `TemplateLiteral`) {
+            // most pageQueries will be template strings
+            const chunks = []
+            for (const quasi of declaration.init.quasis) {
+              chunks.push(quasi.value.cooked)
             }
-          })
-          query = chunks.join(``)
-        }
-      },
-    })
+            query = chunks.join(``)
+          } else if (type === `StringLiteral`) {
+            // fun fact: CoffeeScript can only generate StringLiterals
+            query = declaration.init.extra.rawValue
+          }
+          console.time(`graphql query time`)
+        } else return
+      }
+    });
     const absFile = path.resolve(file)
     // Get paths for this file.
     const paths = []
@@ -328,12 +302,6 @@ const q = queue(
     }
 
     const handleResult = (pathInfo, result = {}) => {
-      //if (result.errors) {
-      //console.log(
-      //`graphql errors from file: ${absFile}`,
-      //result.errors,
-      //)
-      //}
       // Combine the result with the path context.
       result.pathContext = pathInfo.context
       const clonedResult = { ...result }

diff --git a/packages/gatsby/lib/utils/webpack.config.js b/packages/gatsby/lib/utils/webpack.config.js
@@ -250,7 +250,8 @@ module.exports = async (
 
   function resolve() {
     return {
-      extensions: [``, `.js`, `.jsx`, `.cjsx`, `.coffee`],
+      // use the program's extension list (generated via the 'resolvableExtensions' API hook)
+      extensions: [ ``, ...program.extensions ],
       // Hierarchy of directories for Webpack to look for module.
       // First is the site directory.
       // Then in the special directory of isomorphic modules Gatsby ships with.
@@ -274,20 +275,12 @@ module.exports = async (
 
   function module(config) {
     // Common config for every env.
-    config.loader(`cjsx`, {
-      test: /\.cjsx$/,
-      loaders: [`coffee`, `cjsx`],
-    })
     config.loader(`js`, {
       test: /\.jsx?$/, // Accept either .js or .jsx files.
       exclude: /(node_modules|bower_components)/,
       loader: `babel`,
       query: babelConfig,
     })
-    config.loader(`coffee`, {
-      test: /\.coffee$/,
-      loader: `coffee`,
-    })
     config.loader(`json`, {
       test: /\.json$/,
       loaders: [`json`],