From 41c5ce67b80484bb6417788db043ce578d0d9cd3 Mon Sep 17 00:00:00 2001 From: Noah Lange Date: Tue, 21 Mar 2017 12:58:09 -0500 Subject: [PATCH] Adds support for arbitrary compile-to-JS languages (#732) - removed coffee-script, coffee-loader and node-cjsx modules from package.json - removed .coffee and .cjsx from default Webpack config - added two new API hooks: - resolvableExtensions() enables the program to resolve additional extensions as source files. By default, this array consists of [ '.js', '.jsx'] - preprocessSource() enables the program to compile pages and layouts to JS before attempting to generate query information. - when attempting to parse the source of a transpiled-to-JS file, the first parseable source returned by a plugin will be used to collect page queries (i.e., first plugin wins) - an ExportNamedDeclaration visitor is now used to acquire the pageQuery export, allowing for page queries expressed either as TemplateLiterals (i.e., ES6/TypeScript) or StringLiterals (i.e., CoffeeScript). Note that an ExportNamedDeclaration (i.e., ES6 export) must be used for this pageQuery export. Both TypeScript and CoffeeScript (2.x) support this syntax. A more traditional, CommonJS-style module.exports.pageQuery will not work. --- packages/gatsby/lib/bootstrap/index.js | 121 ++++++++-------- packages/gatsby/lib/utils/build-html.js | 1 - packages/gatsby/lib/utils/query-runner.js | 146 ++++++++------------ packages/gatsby/lib/utils/webpack.config.js | 11 +- packages/gatsby/package.json | 3 - 5 files changed, 119 insertions(+), 163 deletions(-) diff --git a/packages/gatsby/lib/bootstrap/index.js b/packages/gatsby/lib/bootstrap/index.js index a0081707e58ba..2daf86a959068 100644 --- a/packages/gatsby/lib/bootstrap/index.js +++ b/packages/gatsby/lib/bootstrap/index.js @@ -1,26 +1,23 @@ /* @flow weak */ -import { graphql } from "graphql" import Promise from "bluebird" import queryRunner from "../utils/query-runner" -import { pagesDB, siteDB, programDB } from "../utils/globals" import path from "path" -import glob from "glob" +import globCB from "glob" import _ from "lodash" import createPath from "./create-path" -import mkdirp from "mkdirp" import fs from "fs-extra" import Joi from "joi" import chalk from "chalk" -const { layoutComponentChunkName } = require("../utils/js-chunk-names") +import apiRunnerNode from "../utils/api-runner-node" +import { graphql } from "graphql" +import { pagesDB, siteDB, programDB } from "../utils/globals" +import { gatsbyConfigSchema, pageSchema } from "../joi-schemas/joi" +import { layoutComponentChunkName } from "../utils/js-chunk-names" const mkdirs = Promise.promisify(fs.mkdirs) const copy = Promise.promisify(fs.copy) const removeDir = Promise.promisify(fs.remove) - -// Joi schemas -import { gatsbyConfigSchema, pageSchema } from "../joi-schemas/joi" - -import apiRunnerNode from "../utils/api-runner-node" +const glob = Promise.promisify(globCB) Promise.onPossiblyUnhandledRejection(error => { throw error @@ -34,55 +31,52 @@ process.on(`unhandledRejection`, error => { // algorithm is glob /pages directory for js/jsx/cjsx files *not* // underscored. Then create url w/ our path algorithm *unless* user // takes control of that page component in gatsby-node. -const autoPathCreator = program => - new Promise(resolve => { - const pagesDirectory = path.join(program.directory, `pages`) - let autoPages = [] - glob(`${pagesDirectory}/**/?(*.js|*.jsx|*.cjsx)`, (err, files) => { - // Create initial page objects. - autoPages = files.map(filePath => ({ - component: filePath, - componentChunkName: layoutComponentChunkName( - program.directory, - filePath - ), - path: filePath, - })) - - // Convert path to one relative to the pages directory. - autoPages = autoPages.map(page => ({ - ...page, - path: path.relative(pagesDirectory, page.path), - })) - - // Remove pages starting with an underscore. - autoPages = _.filter(autoPages, page => page.path.slice(0, 1) !== `_`) - - // Remove page templates. - autoPages = _.filter( - autoPages, - page => page.path.slice(0, 9) !== `template-` - ) +const autoPathCreator = async program => { + const pagesDirectory = path.join(program.directory, `pages`) + const exts = program.extensions.map(e => `*${ e }`).join('|') + const files = await glob(`${pagesDirectory}/**/?(${ exts })`) + // Create initial page objects. + let autoPages = files.map(filePath => ({ + component: filePath, + componentChunkName: layoutComponentChunkName( + program.directory, + filePath + ), + path: filePath, + })) + + // Convert path to one relative to the pages directory. + autoPages = autoPages.map(page => ({ + ...page, + path: path.relative(pagesDirectory, page.path), + })) + + // Remove pages starting with an underscore. + autoPages = _.filter(autoPages, page => page.path.slice(0, 1) !== `_`) + + // Remove page templates. + autoPages = _.filter( + autoPages, + page => page.path.slice(0, 9) !== `template-` + ) - // Convert to our path format. - autoPages = autoPages.map(page => ({ - ...page, - path: createPath(pagesDirectory, page.component), - })) + // Convert to our path format. + autoPages = autoPages.map(page => ({ + ...page, + path: createPath(pagesDirectory, page.component), + })) - // Validate pages. - autoPages.forEach(page => { - const { error } = Joi.validate(page, pageSchema) - if (error) { - console.log(chalk.blue.bgYellow(`A page object failed validation`)) - console.log(page) - console.log(chalk.bold.red(error)) - } - }) - - resolve(autoPages) - }) - }) + // Validate pages. + autoPages.forEach(page => { + const { error } = Joi.validate(page, pageSchema) + if (error) { + console.log(chalk.blue.bgYellow(`A page object failed validation`)) + console.log(page) + console.log(chalk.bold.red(error)) + } + }); + return autoPages; +} module.exports = async program => { console.log(`lib/bootstrap/index.js time since started:`, process.uptime()) @@ -201,7 +195,7 @@ module.exports = async program => { const srcDir = `${__dirname}/../intermediate-representation-dir` const siteDir = `${program.directory}/.intermediate-representation` try { - //await removeDir(siteDir) + // await removeDir(siteDir) await copy(srcDir, siteDir, { clobber: true }) await mkdirs(`${program.directory}/.intermediate-representation/json`) } catch (e) { @@ -268,7 +262,7 @@ module.exports = async program => { // Create Schema. console.time(`create schema`) const schema = await require(`../schema/new`)() - //const schema = await require(`../schema`)() + // const schema = await require(`../schema`)() const graphqlRunner = (query, context) => graphql(schema, query, context, context, context) console.timeEnd(`create schema`) @@ -332,6 +326,11 @@ module.exports = async program => { pagesDB(pagesMap) console.log(`added pages to in-memory db`) + // Collect resolvable extensions and attach to program. + const extensions = [ `.js`, `.jsx` ] + const apiResults = await apiRunnerNode('resolvableExtensions') + program.extensions = apiResults.reduce((a, b) => a.concat(b), extensions) + // TODO move this to own source plugin per component type // (js/cjsx/typescript, etc.) const autoPages = await autoPathCreator(program, pages) @@ -360,8 +359,8 @@ module.exports = async program => { }) console.log(`validated modified pages`) - //console.log(`bootstrap finished, time since started:`, process.uptime()) - //cb(null, schema) + // console.log(`bootstrap finished, time since started:`, process.uptime()) + // cb(null, schema) await queryRunner(program, graphqlRunner) await apiRunnerNode(`generateSideEffects`) diff --git a/packages/gatsby/lib/utils/build-html.js b/packages/gatsby/lib/utils/build-html.js index f237fe86b62e4..1ea8ff0d9d99d 100644 --- a/packages/gatsby/lib/utils/build-html.js +++ b/packages/gatsby/lib/utils/build-html.js @@ -6,7 +6,6 @@ import webpackConfig from "./webpack.config" import { pagesDB } from "./globals" const debug = require("debug")("gatsby:html") -require(`node-cjsx`).transform() module.exports = async program => { const { directory } = program diff --git a/packages/gatsby/lib/utils/query-runner.js b/packages/gatsby/lib/utils/query-runner.js index 80be626111464..d57eba83178eb 100644 --- a/packages/gatsby/lib/utils/query-runner.js +++ b/packages/gatsby/lib/utils/query-runner.js @@ -6,8 +6,8 @@ import traverse from "babel-traverse" import path from "path" import parseFilepath from "parse-filepath" import glob from "glob" -const Promise = require("bluebird") - +import apiRunnerNode from "./api-runner-node"; +import Promise from "bluebird"; import { pagesDB, siteDB, programDB } from "./globals" import { layoutComponentChunkName, pathChunkName } from "./js-chunk-names" @@ -17,7 +17,8 @@ const babylon = require("babylon") const pascalCase = _.flow(_.camelCase, _.upperFirst) const hashStr = function(str) { - let hash = 5381, i = str.length + let hash = 5381 + let i = str.length while (i) { hash = hash * 33 ^ str.charCodeAt(--i) @@ -221,96 +222,69 @@ const writeChildRoutes = () => { } const debouncedWriteChildRoutes = _.debounce(writeChildRoutes, 250) -const babelPlugin = function({ types: t }) { - return { - visitor: { - TemplateLiteral(path, state) { - if ( - path.parentPath.parentPath.parentPath.type !== - `ExportNamedDeclaration` - ) { - return - } - const exportPath = path.parentPath.parentPath.parentPath - const name = _.get( - exportPath, - `node.declaration.declarations[0].id.name` - ) - if (name === `pageQuery`) { - const quasis = _.get(path, `node.quasis`, []) - const expressions = path.get(`expressions`) - const chunks = [] - quasis.forEach(quasi => { - chunks.push(quasi.value.cooked) - const expr = expressions.shift() - if (expr) { - chunks.push( - expr.scope.bindings[expr.node.name].path.get( - `value` - ).parentPath.node.init.quasis[0].value.cooked - ) - } - }) - const query = chunks.join(``) - console.time(`graphql query time`) - const graphql = state.opts.graphql - //path.parentPath.replaceWithSourceString(`require('fixme.json')`); - } - }, - }, - } -} - // Queue for processing files const q = queue( - ({ file, graphql, directory }, callback) => { - const fileStr = fs.readFileSync(file, `utf-8`) + async ({ file, graphql, directory }, callback) => { + let fileStr = fs.readFileSync(file, `utf-8`) let ast - try { - ast = babylon.parse(fileStr, { - sourceType: `module`, - sourceFilename: true, - plugins: [`*`], - }) - } catch (e) { - console.log(`Failed to parse ${file}`) - console.log(e) + // Preprocess and attempt to parse source; return an AST if we can, log an error if we can't. + // I'm unconvinced that this is an especially good implementation... + const transpiled = await apiRunnerNode(`preprocessSource`, { filename: file, contents: fileStr }) + if (transpiled.length) { + for (const item of transpiled) { + try { + const tmp = babylon.parse(item, { + sourceType: `module`, + plugins: [`*`] + }) + ast = tmp; + break + } catch (e) { + console.info(e); + continue + } + } + if (ast === undefined) { + console.error(`Failed to parse preprocessed file ${ file }`) + } + } else { + try { + ast = babylon.parse(fileStr, { + sourceType: `module`, + sourceFilename: true, + plugins: [`*`], + }) + } catch (e) { + console.log(`Failed to parse ${file}`) + console.log(e) + } } // Get query for this file. let query traverse(ast, { - TemplateLiteral(path, state) { - if ( - path.parentPath.parentPath.parentPath.type !== - `ExportNamedDeclaration` - ) { - return - } - const exportPath = path.parentPath.parentPath.parentPath - const name = _.get( - exportPath, - `node.declaration.declarations[0].id.name` - ) + ExportNamedDeclaration(path, state) { + // cache declaration node + const declaration = path.node.declaration.declarations[0] + // we're looking for a ES6 named export called "pageQuery" + const name = declaration.id.name if (name === `pageQuery`) { - const quasis = _.get(path, `node.quasis`, []) - const expressions = path.get(`expressions`) - const chunks = [] - quasis.forEach(quasi => { - chunks.push(quasi.value.cooked) - const expr = expressions.shift() - if (expr) { - chunks.push( - expr.scope.bindings[expr.node.name].path.get( - `value` - ).parentPath.node.init.quasis[0].value.cooked - ) + const type = declaration.init.type; + if (type === `TemplateLiteral`) { + // most pageQueries will be template strings + const chunks = [] + for (const quasi of declaration.init.quasis) { + chunks.push(quasi.value.cooked) } - }) - query = chunks.join(``) - } - }, - }) + query = chunks.join(``) + } else if (type === `StringLiteral`) { + // fun fact: CoffeeScript can only generate StringLiterals + query = declaration.init.extra.rawValue + } + console.time(`graphql query time`) + } else return + } + }); const absFile = path.resolve(file) // Get paths for this file. const paths = [] @@ -328,12 +302,6 @@ const q = queue( } const handleResult = (pathInfo, result = {}) => { - //if (result.errors) { - //console.log( - //`graphql errors from file: ${absFile}`, - //result.errors, - //) - //} // Combine the result with the path context. result.pathContext = pathInfo.context const clonedResult = { ...result } diff --git a/packages/gatsby/lib/utils/webpack.config.js b/packages/gatsby/lib/utils/webpack.config.js index df84999352511..9fb189dc755aa 100644 --- a/packages/gatsby/lib/utils/webpack.config.js +++ b/packages/gatsby/lib/utils/webpack.config.js @@ -250,7 +250,8 @@ module.exports = async ( function resolve() { return { - extensions: [``, `.js`, `.jsx`, `.cjsx`, `.coffee`], + // use the program's extension list (generated via the 'resolvableExtensions' API hook) + extensions: [ ``, ...program.extensions ], // Hierarchy of directories for Webpack to look for module. // First is the site directory. // Then in the special directory of isomorphic modules Gatsby ships with. @@ -274,20 +275,12 @@ module.exports = async ( function module(config) { // Common config for every env. - config.loader(`cjsx`, { - test: /\.cjsx$/, - loaders: [`coffee`, `cjsx`], - }) config.loader(`js`, { test: /\.jsx?$/, // Accept either .js or .jsx files. exclude: /(node_modules|bower_components)/, loader: `babel`, query: babelConfig, }) - config.loader(`coffee`, { - test: /\.coffee$/, - loader: `coffee`, - }) config.loader(`json`, { test: /\.json$/, loaders: [`json`], diff --git a/packages/gatsby/package.json b/packages/gatsby/package.json index dbde8157923a5..835b28d4e5e3a 100644 --- a/packages/gatsby/package.json +++ b/packages/gatsby/package.json @@ -36,9 +36,6 @@ "chalk": "^1.1.3", "chokidar": "^1.6.1", "chunk-manifest-webpack-plugin": "0.1.0", - "cjsx-loader": "^3.0.0", - "coffee-loader": "^0.7.2", - "coffee-script": "^1.9.3", "commander": "^2.9.0", "css-loader": "^0.26.1", "debug": "^2.6.0",