From 6af048e1a99996fd46bad040bb619b94223d4c06 Mon Sep 17 00:00:00 2001 From: Ty Hopp Date: Mon, 14 Feb 2022 13:29:59 +0800 Subject: [PATCH 1/6] Refactor load plugins modules --- .../src/bootstrap/load-plugins/index.ts | 95 +---- .../load-plugins/load-internal-plugins.ts | 158 ++++++++ .../gatsby/src/bootstrap/load-plugins/load.ts | 382 ------------------ .../bootstrap/load-plugins/process-plugin.ts | 93 +++++ .../bootstrap/load-plugins/resolve-plugin.ts | 100 +++++ .../load-plugins/utils/create-hash.ts | 17 + .../bootstrap/load-plugins/utils/create-id.ts | 19 + .../load-plugins/utils/flatten-plugins.ts | 42 ++ .../bootstrap/load-plugins/utils/get-api.ts | 10 + .../load-plugins/utils/handle-gatsby-cloud.ts | 39 ++ .../bootstrap/load-plugins/utils/normalize.ts | 35 ++ .../src/bootstrap/load-plugins/validate.ts | 8 +- .../gatsby/src/bootstrap/load-themes/index.js | 4 +- 13 files changed, 525 insertions(+), 477 deletions(-) create mode 100644 packages/gatsby/src/bootstrap/load-plugins/load-internal-plugins.ts delete mode 100644 packages/gatsby/src/bootstrap/load-plugins/load.ts create mode 100644 packages/gatsby/src/bootstrap/load-plugins/process-plugin.ts create mode 100644 packages/gatsby/src/bootstrap/load-plugins/resolve-plugin.ts create mode 100644 packages/gatsby/src/bootstrap/load-plugins/utils/create-hash.ts create mode 100644 packages/gatsby/src/bootstrap/load-plugins/utils/create-id.ts create mode 100644 packages/gatsby/src/bootstrap/load-plugins/utils/flatten-plugins.ts create mode 100644 packages/gatsby/src/bootstrap/load-plugins/utils/get-api.ts create mode 100644 packages/gatsby/src/bootstrap/load-plugins/utils/handle-gatsby-cloud.ts create mode 100644 packages/gatsby/src/bootstrap/load-plugins/utils/normalize.ts diff --git a/packages/gatsby/src/bootstrap/load-plugins/index.ts b/packages/gatsby/src/bootstrap/load-plugins/index.ts index f2d8eb2f769ad..c587cc74a5540 100644 --- a/packages/gatsby/src/bootstrap/load-plugins/index.ts +++ b/packages/gatsby/src/bootstrap/load-plugins/index.ts @@ -1,102 +1,19 @@ -import _ from "lodash" - import { store } from "../../redux" import { IGatsbyState } from "../../redux/types" import * as nodeAPIs from "../../utils/api-node-docs" import * as browserAPIs from "../../utils/api-browser-docs" import ssrAPIs from "../../../cache-dir/api-ssr-docs" -import { loadPlugins as loadPluginsInternal } from "./load" +import { loadInternalPlugins } from "./load-internal-plugins" import { collatePluginAPIs, handleBadExports, handleMultipleReplaceRenderers, - ExportType, - ICurrentAPIs, validateConfigPluginsOptions, } from "./validate" -import { - IPluginInfo, - IFlattenedPlugin, - ISiteConfig, - IRawSiteConfig, -} from "./types" -import { IPluginRefObject, PluginRef } from "gatsby-plugin-utils/dist/types" - -const getAPI = (api: { - [exportType in ExportType]: { [api: string]: boolean } -}): ICurrentAPIs => - _.keys(api).reduce>((merged, key) => { - merged[key] = _.keys(api[key]) - return merged - }, {}) as ICurrentAPIs - -// Create a "flattened" array of plugins with all subplugins -// brought to the top-level. This simplifies running gatsby-* files -// for subplugins. -const flattenPlugins = (plugins: Array): Array => { - const flattened: Array = [] - const extractPlugins = (plugin: IPluginInfo): void => { - if (plugin.subPluginPaths) { - for (const subPluginPath of plugin.subPluginPaths) { - // @pieh: - // subPluginPath can look like someOption.randomFieldThatIsMarkedAsSubplugins - // Reason for doing stringified path with . separator was that it was just easier to prevent duplicates - // in subPluginPaths array (as each subplugin in the gatsby-config would add subplugin path). - const segments = subPluginPath.split(`.`) - let roots: Array = [plugin.pluginOptions] - for (const segment of segments) { - if (segment === `[]`) { - roots = roots.flat() - } else { - roots = roots.map(root => root[segment]) - } - } - roots = roots.flat() - - roots.forEach(subPlugin => { - flattened.push(subPlugin) - extractPlugins(subPlugin) - }) - } - } - } - - plugins.forEach(plugin => { - flattened.push(plugin) - extractPlugins(plugin) - }) - - return flattened -} - -function normalizePlugin(plugin): IPluginRefObject { - if (typeof plugin === `string`) { - return { - resolve: plugin, - options: {}, - } - } - - if (plugin.options?.plugins) { - plugin.options = { - ...plugin.options, - plugins: normalizePlugins(plugin.options.plugins), - } - } - - return plugin -} - -function normalizePlugins(plugins?: Array): Array { - return (plugins || []).map(normalizePlugin) -} - -const normalizeConfig = (config: IRawSiteConfig = {}): ISiteConfig => { - return { - ...config, - plugins: (config.plugins || []).map(normalizePlugin), - } -} +import { IFlattenedPlugin, IRawSiteConfig } from "./types" +import { normalizeConfig } from "./utils/normalize" +import { getAPI } from "./utils/get-api" +import { flattenPlugins } from "./utils/flatten-plugins" export async function loadPlugins( rawConfig: IRawSiteConfig = {}, @@ -115,7 +32,7 @@ export async function loadPlugins( }) // Collate internal plugins, site config plugins, site default plugins - const pluginInfos = loadPluginsInternal(config, rootDir) + const pluginInfos = await loadInternalPlugins(config, rootDir) // Create a flattened array of the plugins const pluginArray = flattenPlugins(pluginInfos) diff --git a/packages/gatsby/src/bootstrap/load-plugins/load-internal-plugins.ts b/packages/gatsby/src/bootstrap/load-plugins/load-internal-plugins.ts new file mode 100644 index 0000000000000..cc7058b08867f --- /dev/null +++ b/packages/gatsby/src/bootstrap/load-plugins/load-internal-plugins.ts @@ -0,0 +1,158 @@ +import { slash } from "gatsby-core-utils" +import path from "path" +import reporter from "gatsby-cli/lib/reporter" +import { store } from "../../redux" +import { + IPluginInfo, + IPluginRefObject, + IPluginRefOptions, + ISiteConfig, +} from "./types" +import { processPlugin } from "./process-plugin" +import { createPluginId } from "./utils/create-id" +import { createFileContentHash } from "./utils/create-hash" +import { + addGatsbyPluginCloudPluginWhenInstalled, + incompatibleGatsbyCloudPlugin, + GATSBY_CLOUD_PLUGIN_NAME, +} from "./utils/handle-gatsby-cloud" + +const TYPESCRIPT_PLUGIN_NAME = `gatsby-plugin-typescript` + +export async function loadInternalPlugins( + config: ISiteConfig = {}, + rootDir: string +): Promise> { + // Instantiate plugins. + const plugins: Array = [] + const configuredPluginNames = new Set() + + // Add internal plugins + const internalPluginPaths = [ + `../../internal-plugins/dev-404-page`, + `../../internal-plugins/load-babel-config`, + `../../internal-plugins/internal-data-bridge`, + `../../internal-plugins/prod-404-500`, + `../../internal-plugins/webpack-theme-component-shadowing`, + `../../internal-plugins/bundle-optimisations`, + `../../internal-plugins/functions`, + ].filter(Boolean) as Array + + for (const internalPluginPath of internalPluginPaths) { + const internalPluginAbsolutePath = path.join(__dirname, internalPluginPath) + const processedPlugin = await processPlugin( + internalPluginAbsolutePath, + rootDir + ) + plugins.push(processedPlugin) + } + + // Add plugins from the site config. + if (config.plugins) { + for (const plugin of config.plugins) { + const processedPlugin = await processPlugin(plugin, rootDir) + plugins.push(processedPlugin) + configuredPluginNames.add(processedPlugin.name) + } + } + + // the order of all of these page-creators matters. The "last plugin wins", + // so the user's site comes last, and each page-creator instance has to + // match the plugin definition order before that. This works fine for themes + // because themes have already been added in the proper order to the plugins + // array + for (const plugin of [...plugins]) { + const processedPlugin = await processPlugin( + { + resolve: require.resolve(`gatsby-plugin-page-creator`), + options: { + path: slash(path.join(plugin.resolve, `src/pages`)), + pathCheck: false, + }, + }, + rootDir + ) + + plugins.push(processedPlugin) + } + + if ( + _CFLAGS_.GATSBY_MAJOR === `4` && + configuredPluginNames.has(GATSBY_CLOUD_PLUGIN_NAME) && + incompatibleGatsbyCloudPlugin(plugins) + ) { + reporter.panic( + `Plugin gatsby-plugin-gatsby-cloud is not compatible with your gatsby version. Please upgrade to gatsby-plugin-gatsby-cloud@next` + ) + } + + if ( + !configuredPluginNames.has(GATSBY_CLOUD_PLUGIN_NAME) && + (process.env.GATSBY_CLOUD === `true` || process.env.GATSBY_CLOUD === `1`) + ) { + await addGatsbyPluginCloudPluginWhenInstalled(plugins, rootDir) + } + + // Support Typescript by default but allow users to override it + if (!configuredPluginNames.has(TYPESCRIPT_PLUGIN_NAME)) { + const processedTypeScriptPlugin = await processPlugin( + { + resolve: require.resolve(TYPESCRIPT_PLUGIN_NAME), + options: { + // TODO(@mxstbr): Do not hard-code these defaults but infer them from the + // pluginOptionsSchema of gatsby-plugin-typescript + allExtensions: false, + isTSX: false, + jsxPragma: `React`, + }, + }, + rootDir + ) + plugins.push(processedTypeScriptPlugin) + } + + // Add the site's default "plugin" i.e. gatsby-x files in root of site. + plugins.push({ + resolve: slash(process.cwd()), + id: createPluginId(`default-site-plugin`), + name: `default-site-plugin`, + version: createFileContentHash(process.cwd(), `gatsby-*`), + pluginOptions: { + plugins: [], + }, + }) + + const program = store.getState().program + + // default options for gatsby-plugin-page-creator + let pageCreatorOptions: IPluginRefOptions | undefined = { + path: slash(path.join(program.directory, `src/pages`)), + pathCheck: false, + } + + if (config.plugins) { + const pageCreatorPlugin = config.plugins.find( + (plugin): plugin is IPluginRefObject => + typeof plugin !== `string` && + plugin.resolve === `gatsby-plugin-page-creator` && + slash((plugin.options && plugin.options.path) || ``) === + slash(path.join(program.directory, `src/pages`)) + ) + if (pageCreatorPlugin) { + // override the options if there are any user specified options + pageCreatorOptions = pageCreatorPlugin.options + } + } + + const processedPageCreatorPlugin = await processPlugin( + { + resolve: require.resolve(`gatsby-plugin-page-creator`), + options: pageCreatorOptions, + }, + rootDir + ) + + plugins.push(processedPageCreatorPlugin) + + return plugins +} diff --git a/packages/gatsby/src/bootstrap/load-plugins/load.ts b/packages/gatsby/src/bootstrap/load-plugins/load.ts deleted file mode 100644 index fd6623f601d51..0000000000000 --- a/packages/gatsby/src/bootstrap/load-plugins/load.ts +++ /dev/null @@ -1,382 +0,0 @@ -import _ from "lodash" -import { slash, createRequireFromPath } from "gatsby-core-utils" -import fs from "fs" -import path from "path" -import crypto from "crypto" -import glob from "glob" -import { sync as existsSync } from "fs-exists-cached" -import reporter from "gatsby-cli/lib/reporter" -import { silent as resolveFromSilent } from "resolve-from" -import * as semver from "semver" -import { warnOnIncompatiblePeerDependency } from "./validate" -import { store } from "../../redux" -import { createNodeId } from "../../utils/create-node-id" -import { - IPluginInfo, - PluginRef, - IPluginRefObject, - IPluginRefOptions, - ISiteConfig, -} from "./types" -import { PackageJson } from "../../.." - -const GATSBY_CLOUD_PLUGIN_NAME = `gatsby-plugin-gatsby-cloud` -const TYPESCRIPT_PLUGIN_NAME = `gatsby-plugin-typescript` - -function createFileContentHash(root: string, globPattern: string): string { - const hash = crypto.createHash(`md5`) - const files = glob.sync(`${root}/${globPattern}`, { nodir: true }) - - files.forEach(filepath => { - hash.update(fs.readFileSync(filepath)) - }) - - return hash.digest(`hex`) -} - -/** - * Make sure key is unique to plugin options. E.g. there could - * be multiple source-filesystem plugins, with different names - * (docs, blogs). - * - * @param name Name of the plugin - * @param pluginObject Object of the plugin - */ -const createPluginId = ( - name: string, - pluginObject: IPluginRefObject | null = null -): string => - createNodeId( - name + (pluginObject ? JSON.stringify(pluginObject.options) : ``), - `Plugin` - ) - -/** - * @param plugin - * This should be a plugin spec object where possible but can also be the - * name of a plugin. - * - * When it is a name, it can be a name of a local plugin, the name of a plugin - * located in node_modules, or a Gatsby internal plugin. In the last case the - * plugin will be an absolute path. - * @param rootDir - * This is the project location, from which are found the plugins - */ -export function resolvePlugin( - plugin: PluginRef, - rootDir: string | null -): IPluginInfo { - const pluginName = _.isString(plugin) ? plugin : plugin.resolve - - // Respect the directory that the plugin was sourced from initially - rootDir = (!_.isString(plugin) && plugin.parentDir) || rootDir - - // Only find plugins when we're not given an absolute path - if (!existsSync(pluginName) && rootDir) { - // Find the plugin in the local plugins folder - const resolvedPath = slash(path.join(rootDir, `plugins/${pluginName}`)) - - if (existsSync(resolvedPath)) { - if (existsSync(`${resolvedPath}/package.json`)) { - const packageJSON = JSON.parse( - fs.readFileSync(`${resolvedPath}/package.json`, `utf-8`) - ) as PackageJson - const name = packageJSON.name || pluginName - warnOnIncompatiblePeerDependency(name, packageJSON) - - return { - resolve: resolvedPath, - name, - id: createPluginId(name), - version: - packageJSON.version || createFileContentHash(resolvedPath, `**`), - } - } else { - // Make package.json a requirement for local plugins too - throw new Error(`Plugin ${pluginName} requires a package.json file`) - } - } - } - - /** - * Here we have an absolute path to an internal plugin, or a name of a module - * which should be located in node_modules. - */ - try { - const requireSource = - rootDir !== null - ? createRequireFromPath(`${rootDir}/:internal:`) - : require - - // If the path is absolute, resolve the directory of the internal plugin, - // otherwise resolve the directory containing the package.json - const resolvedPath = slash( - path.dirname( - requireSource.resolve( - path.isAbsolute(pluginName) - ? pluginName - : `${pluginName}/package.json` - ) - ) - ) - - const packageJSON = JSON.parse( - fs.readFileSync(`${resolvedPath}/package.json`, `utf-8`) - ) - warnOnIncompatiblePeerDependency(packageJSON.name, packageJSON) - - return { - resolve: resolvedPath, - id: createPluginId(packageJSON.name), - name: packageJSON.name, - version: packageJSON.version, - } - } catch (err) { - if (process.env.gatsby_log_level === `verbose`) { - reporter.panicOnBuild( - `plugin "${pluginName} threw the following error:\n`, - err - ) - } else { - reporter.panicOnBuild( - `There was a problem loading plugin "${pluginName}". Perhaps you need to install its package?\nUse --verbose to see actual error.` - ) - } - throw new Error(`unreachable`) - } -} - -function addGatsbyPluginCloudPluginWhenInstalled( - plugins: Array, - processPlugin: (plugin: PluginRef) => IPluginInfo, - rootDir: string -): void { - const cloudPluginLocation = resolveFromSilent( - rootDir, - GATSBY_CLOUD_PLUGIN_NAME - ) - - if (cloudPluginLocation) { - plugins.push( - processPlugin({ - resolve: cloudPluginLocation, - options: {}, - }) - ) - } -} - -function incompatibleGatsbyCloudPlugin(plugins: Array): boolean { - const plugin = plugins.find( - plugin => plugin.name === GATSBY_CLOUD_PLUGIN_NAME - ) - - return !semver.satisfies(plugin!.version, `>=4.0.0-alpha`, { - includePrerelease: true, - }) -} - -export function loadPlugins( - config: ISiteConfig = {}, - rootDir: string -): Array { - // Instantiate plugins. - const plugins: Array = [] - const configuredPluginNames = new Set() - - // Create fake little site with a plugin for testing this - // w/ snapshots. Move plugin processing to its own module. - // Also test adding to redux store. - function processPlugin(plugin: PluginRef): IPluginInfo { - if (_.isString(plugin)) { - const info = resolvePlugin(plugin, rootDir) - - return { - ...info, - pluginOptions: { - plugins: [], - }, - } - } else { - plugin.options = plugin.options || {} - - // Throw an error if there is an "option" key. - if ( - _.isEmpty(plugin.options) && - !_.isEmpty((plugin as { option?: unknown }).option) - ) { - throw new Error( - `Plugin "${plugin.resolve}" has an "option" key in the configuration. Did you mean "options"?` - ) - } - - // Plugins can have plugins. - if (plugin.subPluginPaths) { - for (const subPluginPath of plugin.subPluginPaths) { - const segments = subPluginPath.split(`.`) - let roots: Array = [plugin.options] - - let pathToSwap = segments - - for (const segment of segments) { - if (segment === `[]`) { - pathToSwap = pathToSwap.slice(0, pathToSwap.length - 1) - roots = roots.flat() - } else { - roots = roots.map(root => root[segment]) - } - } - roots = roots.flat() - - const processed = roots.map(processPlugin) - _.set(plugin.options, pathToSwap, processed) - } - } - - // Add some default values for tests as we don't actually - // want to try to load anything during tests. - if (plugin.resolve === `___TEST___`) { - const name = `TEST` - - return { - id: createPluginId(name, plugin), - name, - version: `0.0.0-test`, - pluginOptions: { - plugins: [], - }, - resolve: `__TEST__`, - } - } - - const info = resolvePlugin(plugin, rootDir) - - return { - ...info, - modulePath: plugin.modulePath, - module: plugin.module, - subPluginPaths: plugin.subPluginPaths - ? Array.from(plugin.subPluginPaths) - : undefined, - id: createPluginId(info.name, plugin), - pluginOptions: _.merge({ plugins: [] }, plugin.options), - } - } - } - - // Add internal plugins - const internalPlugins = [ - `../../internal-plugins/dev-404-page`, - `../../internal-plugins/load-babel-config`, - `../../internal-plugins/internal-data-bridge`, - `../../internal-plugins/prod-404-500`, - `../../internal-plugins/webpack-theme-component-shadowing`, - `../../internal-plugins/bundle-optimisations`, - `../../internal-plugins/functions`, - ].filter(Boolean) as Array - internalPlugins.forEach(relPath => { - const absPath = path.join(__dirname, relPath) - plugins.push(processPlugin(absPath)) - }) - - // Add plugins from the site config. - if (config.plugins) { - config.plugins.forEach(plugin => { - const processedPlugin = processPlugin(plugin) - plugins.push(processedPlugin) - configuredPluginNames.add(processedPlugin.name) - }) - } - - // the order of all of these page-creators matters. The "last plugin wins", - // so the user's site comes last, and each page-creator instance has to - // match the plugin definition order before that. This works fine for themes - // because themes have already been added in the proper order to the plugins - // array - plugins.forEach(plugin => { - plugins.push( - processPlugin({ - resolve: require.resolve(`gatsby-plugin-page-creator`), - options: { - path: slash(path.join(plugin.resolve, `src/pages`)), - pathCheck: false, - }, - }) - ) - }) - - if ( - _CFLAGS_.GATSBY_MAJOR === `4` && - configuredPluginNames.has(GATSBY_CLOUD_PLUGIN_NAME) && - incompatibleGatsbyCloudPlugin(plugins) - ) { - reporter.panic( - `Plugin gatsby-plugin-gatsby-cloud is not compatible with your gatsby version. Please upgrade to gatsby-plugin-gatsby-cloud@next` - ) - } - - if ( - !configuredPluginNames.has(GATSBY_CLOUD_PLUGIN_NAME) && - (process.env.GATSBY_CLOUD === `true` || process.env.GATSBY_CLOUD === `1`) - ) { - addGatsbyPluginCloudPluginWhenInstalled(plugins, processPlugin, rootDir) - } - - // Suppor Typescript by default but allow users to override it - if (!configuredPluginNames.has(TYPESCRIPT_PLUGIN_NAME)) { - plugins.push( - processPlugin({ - resolve: require.resolve(TYPESCRIPT_PLUGIN_NAME), - options: { - // TODO(@mxstbr): Do not hard-code these defaults but infer them from the - // pluginOptionsSchema of gatsby-plugin-typescript - allExtensions: false, - isTSX: false, - jsxPragma: `React`, - }, - }) - ) - } - - // Add the site's default "plugin" i.e. gatsby-x files in root of site. - plugins.push({ - resolve: slash(process.cwd()), - id: createPluginId(`default-site-plugin`), - name: `default-site-plugin`, - version: createFileContentHash(process.cwd(), `gatsby-*`), - pluginOptions: { - plugins: [], - }, - }) - - const program = store.getState().program - - // default options for gatsby-plugin-page-creator - let pageCreatorOptions: IPluginRefOptions | undefined = { - path: slash(path.join(program.directory, `src/pages`)), - pathCheck: false, - } - - if (config.plugins) { - const pageCreatorPlugin = config.plugins.find( - (plugin): plugin is IPluginRefObject => - typeof plugin !== `string` && - plugin.resolve === `gatsby-plugin-page-creator` && - slash((plugin.options && plugin.options.path) || ``) === - slash(path.join(program.directory, `src/pages`)) - ) - if (pageCreatorPlugin) { - // override the options if there are any user specified options - pageCreatorOptions = pageCreatorPlugin.options - } - } - - plugins.push( - processPlugin({ - resolve: require.resolve(`gatsby-plugin-page-creator`), - options: pageCreatorOptions, - }) - ) - - return plugins -} diff --git a/packages/gatsby/src/bootstrap/load-plugins/process-plugin.ts b/packages/gatsby/src/bootstrap/load-plugins/process-plugin.ts new file mode 100644 index 0000000000000..0d31ecbbfcbe6 --- /dev/null +++ b/packages/gatsby/src/bootstrap/load-plugins/process-plugin.ts @@ -0,0 +1,93 @@ +import { IPluginInfo, PluginRef } from "./types" +import { createPluginId } from "./utils/create-id" +import { resolvePlugin } from "./resolve-plugin" +import { isString, isEmpty, set, merge } from "lodash" + +export async function processPlugin( + plugin: PluginRef, + rootDir: string +): Promise { + // Respect the directory that the plugin was sourced from initially + rootDir = (!isString(plugin) && plugin.parentDir) || rootDir + + if (isString(plugin)) { + const info = await resolvePlugin(plugin, rootDir) + + return { + ...info, + pluginOptions: { + plugins: [], + }, + } + } + + plugin.options = plugin.options || {} + + // Throw an error if there is an "option" key. + if ( + isEmpty(plugin.options) && + !isEmpty((plugin as { option?: unknown }).option) + ) { + throw new Error( + `Plugin "${plugin.resolve}" has an "option" key in the configuration. Did you mean "options"?` + ) + } + + // Plugins can have plugins. + if (plugin.subPluginPaths) { + for (const subPluginPath of plugin.subPluginPaths) { + const segments = subPluginPath.split(`.`) + let roots: Array = [plugin.options] + + let pathToSwap = segments + + for (const segment of segments) { + if (segment === `[]`) { + pathToSwap = pathToSwap.slice(0, pathToSwap.length - 1) + roots = roots.flat() + } else { + roots = roots.map(root => root[segment]) + } + } + roots = roots.flat() + + const processed: Array = [] + + for (const root of roots) { + const result = await processPlugin(root, rootDir) + processed.push(result) + } + + set(plugin.options, pathToSwap, processed) + } + } + + // Add some default values for tests as we don't actually + // want to try to load anything during tests. + if (plugin.resolve === `___TEST___`) { + const name = `TEST` + + return { + id: createPluginId(name, plugin), + name, + version: `0.0.0-test`, + pluginOptions: { + plugins: [], + }, + resolve: `__TEST__`, + } + } + + const info = await resolvePlugin(plugin, rootDir) + + return { + ...info, + modulePath: plugin.modulePath, + module: plugin.module, + subPluginPaths: plugin.subPluginPaths + ? Array.from(plugin.subPluginPaths) + : undefined, + id: createPluginId(info.name, plugin), + pluginOptions: merge({ plugins: [] }, plugin.options), + } +} diff --git a/packages/gatsby/src/bootstrap/load-plugins/resolve-plugin.ts b/packages/gatsby/src/bootstrap/load-plugins/resolve-plugin.ts new file mode 100644 index 0000000000000..9cd103b95eca6 --- /dev/null +++ b/packages/gatsby/src/bootstrap/load-plugins/resolve-plugin.ts @@ -0,0 +1,100 @@ +import path from "path" +import fs from "fs" +import { slash, createRequireFromPath } from "gatsby-core-utils" +import { sync as existsSync } from "fs-exists-cached" +import { warnOnIncompatiblePeerDependency } from "./validate" +import { PackageJson } from "../../.." +import { IPluginInfo, PluginRef } from "./types" +import { createPluginId } from "./utils/create-id" +import { createFileContentHash } from "./utils/create-hash" +import reporter from "gatsby-cli/lib/reporter" +import { isString } from "lodash" + +/** + * @param plugin + * This should be a plugin spec object where possible but can also be the + * name of a plugin. + * + * When it is a name, it can be a name of a local plugin, the name of a plugin + * located in node_modules, or a Gatsby internal plugin. In the last case the + * plugin will be an absolute path. + * @param rootDir + * This is the project location, from which are found the plugins + */ +export function resolvePlugin(plugin: PluginRef, rootDir: string): IPluginInfo { + const pluginName = isString(plugin) ? plugin : plugin.resolve + + // Only find plugins when we're not given an absolute path + if (!existsSync(pluginName) && rootDir) { + // Find the plugin in the local plugins folder + const resolvedPath = slash(path.join(rootDir, `plugins/${pluginName}`)) + + if (existsSync(resolvedPath)) { + if (existsSync(`${resolvedPath}/package.json`)) { + const packageJSON = JSON.parse( + fs.readFileSync(`${resolvedPath}/package.json`, `utf-8`) + ) as PackageJson + const name = packageJSON.name || pluginName + warnOnIncompatiblePeerDependency(name, packageJSON) + + return { + resolve: resolvedPath, + name, + id: createPluginId(name), + version: + packageJSON.version || createFileContentHash(resolvedPath, `**`), + } + } else { + // Make package.json a requirement for local plugins too + throw new Error(`Plugin ${pluginName} requires a package.json file`) + } + } + } + + /** + * Here we have an absolute path to an internal plugin, or a name of a module + * which should be located in node_modules. + */ + try { + const requireSource = + rootDir !== null + ? createRequireFromPath(`${rootDir}/:internal:`) + : require + + // If the path is absolute, resolve the directory of the internal plugin, + // otherwise resolve the directory containing the package.json + const resolvedPath = slash( + path.dirname( + requireSource.resolve( + path.isAbsolute(pluginName) + ? pluginName + : `${pluginName}/package.json` + ) + ) + ) + + const packageJSON = JSON.parse( + fs.readFileSync(`${resolvedPath}/package.json`, `utf-8`) + ) + warnOnIncompatiblePeerDependency(packageJSON.name, packageJSON) + + return { + resolve: resolvedPath, + id: createPluginId(packageJSON.name), + name: packageJSON.name, + version: packageJSON.version, + } + } catch (err) { + if (process.env.gatsby_log_level === `verbose`) { + reporter.panicOnBuild( + `plugin "${pluginName} threw the following error:\n`, + err + ) + } else { + reporter.panicOnBuild( + `There was a problem loading plugin "${pluginName}". Perhaps you need to install its package?\nUse --verbose to see actual error.` + ) + } + throw new Error(`unreachable`) + } +} diff --git a/packages/gatsby/src/bootstrap/load-plugins/utils/create-hash.ts b/packages/gatsby/src/bootstrap/load-plugins/utils/create-hash.ts new file mode 100644 index 0000000000000..cc8130ccf00e1 --- /dev/null +++ b/packages/gatsby/src/bootstrap/load-plugins/utils/create-hash.ts @@ -0,0 +1,17 @@ +import fs from "fs" +import crypto from "crypto" +import glob from "glob" + +export function createFileContentHash( + root: string, + globPattern: string +): string { + const hash = crypto.createHash(`md5`) + const files = glob.sync(`${root}/${globPattern}`, { nodir: true }) + + files.forEach(filepath => { + hash.update(fs.readFileSync(filepath)) + }) + + return hash.digest(`hex`) +} diff --git a/packages/gatsby/src/bootstrap/load-plugins/utils/create-id.ts b/packages/gatsby/src/bootstrap/load-plugins/utils/create-id.ts new file mode 100644 index 0000000000000..d33d0bbca6326 --- /dev/null +++ b/packages/gatsby/src/bootstrap/load-plugins/utils/create-id.ts @@ -0,0 +1,19 @@ +import { createNodeId } from "../../../utils/create-node-id" +import { IPluginRefObject } from "../types" + +/** + * Make sure key is unique to plugin options. E.g. there could + * be multiple source-filesystem plugins, with different names + * (docs, blogs). + * + * @param name Name of the plugin + * @param pluginObject Object of the plugin + */ +export const createPluginId = ( + name: string, + pluginObject: IPluginRefObject | null = null +): string => + createNodeId( + name + (pluginObject ? JSON.stringify(pluginObject.options) : ``), + `Plugin` + ) diff --git a/packages/gatsby/src/bootstrap/load-plugins/utils/flatten-plugins.ts b/packages/gatsby/src/bootstrap/load-plugins/utils/flatten-plugins.ts new file mode 100644 index 0000000000000..291ecd9f628b6 --- /dev/null +++ b/packages/gatsby/src/bootstrap/load-plugins/utils/flatten-plugins.ts @@ -0,0 +1,42 @@ +import { IPluginInfo } from "../types" + +// Create a "flattened" array of plugins with all subplugins +// brought to the top-level. This simplifies running gatsby-* files +// for subplugins. +export const flattenPlugins = ( + plugins: Array +): Array => { + const flattened: Array = [] + const extractPlugins = (plugin: IPluginInfo): void => { + if (plugin.subPluginPaths) { + for (const subPluginPath of plugin.subPluginPaths) { + // @pieh: + // subPluginPath can look like someOption.randomFieldThatIsMarkedAsSubplugins + // Reason for doing stringified path with . separator was that it was just easier to prevent duplicates + // in subPluginPaths array (as each subplugin in the gatsby-config would add subplugin path). + const segments = subPluginPath.split(`.`) + let roots: Array = [plugin.pluginOptions] + for (const segment of segments) { + if (segment === `[]`) { + roots = roots.flat() + } else { + roots = roots.map(root => root[segment]) + } + } + roots = roots.flat() + + roots.forEach(subPlugin => { + flattened.push(subPlugin) + extractPlugins(subPlugin) + }) + } + } + } + + plugins.forEach(plugin => { + flattened.push(plugin) + extractPlugins(plugin) + }) + + return flattened +} diff --git a/packages/gatsby/src/bootstrap/load-plugins/utils/get-api.ts b/packages/gatsby/src/bootstrap/load-plugins/utils/get-api.ts new file mode 100644 index 0000000000000..3185b3cf19f79 --- /dev/null +++ b/packages/gatsby/src/bootstrap/load-plugins/utils/get-api.ts @@ -0,0 +1,10 @@ +import { ExportType, ICurrentAPIs } from "../validate" +import { keys } from "lodash" + +export const getAPI = (api: { + [exportType in ExportType]: { [api: string]: boolean } +}): ICurrentAPIs => + keys(api).reduce>((merged, key) => { + merged[key] = keys(api[key]) + return merged + }, {}) as ICurrentAPIs diff --git a/packages/gatsby/src/bootstrap/load-plugins/utils/handle-gatsby-cloud.ts b/packages/gatsby/src/bootstrap/load-plugins/utils/handle-gatsby-cloud.ts new file mode 100644 index 0000000000000..0f4509e32e406 --- /dev/null +++ b/packages/gatsby/src/bootstrap/load-plugins/utils/handle-gatsby-cloud.ts @@ -0,0 +1,39 @@ +import { silent as resolveFromSilent } from "resolve-from" +import * as semver from "semver" +import { IPluginInfo } from "../types" +import { processPlugin } from "../process-plugin" + +export const GATSBY_CLOUD_PLUGIN_NAME = `gatsby-plugin-gatsby-cloud` + +export async function addGatsbyPluginCloudPluginWhenInstalled( + plugins: Array, + rootDir: string +): Promise { + const cloudPluginLocation = resolveFromSilent( + rootDir, + GATSBY_CLOUD_PLUGIN_NAME + ) + + if (cloudPluginLocation) { + const processedGatsbyCloudPlugin = await processPlugin( + { + resolve: cloudPluginLocation, + options: {}, + }, + rootDir + ) + plugins.push(processedGatsbyCloudPlugin) + } +} + +export function incompatibleGatsbyCloudPlugin( + plugins: Array +): boolean { + const plugin = plugins.find( + plugin => plugin.name === GATSBY_CLOUD_PLUGIN_NAME + ) + + return !semver.satisfies(plugin!.version, `>=4.0.0-alpha`, { + includePrerelease: true, + }) +} diff --git a/packages/gatsby/src/bootstrap/load-plugins/utils/normalize.ts b/packages/gatsby/src/bootstrap/load-plugins/utils/normalize.ts new file mode 100644 index 0000000000000..444ea58c3459c --- /dev/null +++ b/packages/gatsby/src/bootstrap/load-plugins/utils/normalize.ts @@ -0,0 +1,35 @@ +import { ISiteConfig, IRawSiteConfig } from "../types" +import { IPluginRefObject, PluginRef } from "gatsby-plugin-utils/dist/types" + +export function normalizePlugin( + plugin: IPluginRefObject | string +): IPluginRefObject { + if (typeof plugin === `string`) { + return { + resolve: plugin, + options: {}, + } + } + + if (plugin.options?.plugins) { + plugin.options = { + ...plugin.options, + plugins: normalizePlugins(plugin.options.plugins), + } + } + + return plugin +} + +export function normalizePlugins( + plugins?: Array +): Array { + return (plugins || []).map(normalizePlugin) +} + +export const normalizeConfig = (config: IRawSiteConfig = {}): ISiteConfig => { + return { + ...config, + plugins: (config.plugins || []).map(normalizePlugin), + } +} diff --git a/packages/gatsby/src/bootstrap/load-plugins/validate.ts b/packages/gatsby/src/bootstrap/load-plugins/validate.ts index a66b44559759f..57f72b49bbefd 100644 --- a/packages/gatsby/src/bootstrap/load-plugins/validate.ts +++ b/packages/gatsby/src/bootstrap/load-plugins/validate.ts @@ -18,7 +18,7 @@ import { IPluginInfoOptions, ISiteConfig, } from "./types" -import { resolvePlugin } from "./load" +import { resolvePlugin } from "./resolve-plugin" interface IApi { version?: string @@ -180,7 +180,7 @@ export async function handleBadExports({ async function validatePluginsOptions( plugins: Array, - rootDir: string | null + rootDir: string ): Promise<{ errors: number plugins: Array @@ -190,7 +190,7 @@ async function validatePluginsOptions( plugins.map(async plugin => { let gatsbyNode try { - const resolvedPlugin = resolvePlugin(plugin, rootDir) + const resolvedPlugin = await resolvePlugin(plugin, rootDir) gatsbyNode = require(`${resolvedPlugin.resolve}/gatsby-node`) } catch (err) { gatsbyNode = {} @@ -384,7 +384,7 @@ async function validatePluginsOptions( export async function validateConfigPluginsOptions( config: ISiteConfig = {}, - rootDir: string | null + rootDir: string ): Promise { if (!config.plugins) return diff --git a/packages/gatsby/src/bootstrap/load-themes/index.js b/packages/gatsby/src/bootstrap/load-themes/index.js index 70a81894c943c..f7c8f13be92dc 100644 --- a/packages/gatsby/src/bootstrap/load-themes/index.js +++ b/packages/gatsby/src/bootstrap/load-themes/index.js @@ -6,7 +6,7 @@ const _ = require(`lodash`) const debug = require(`debug`)(`gatsby:load-themes`) import { preferDefault } from "../prefer-default" import { getConfigFile } from "../get-config-file" -const { resolvePlugin } = require(`../load-plugins/load`) +import { resolvePlugin } from "../load-plugins/resolve-plugin" const reporter = require(`gatsby-cli/lib/reporter`) // get the gatsby-config file for a theme @@ -34,7 +34,7 @@ const resolveTheme = async ( pathToLocalTheme = path.join(rootDir, `plugins`, themeName) // is a local plugin OR it doesn't exist try { - const { resolve } = resolvePlugin(themeName, rootDir) + const { resolve } = await resolvePlugin(themeName, rootDir) themeDir = resolve } catch (localErr) { reporter.panic(`Failed to resolve ${themeName}`, localErr) From 46c94931840db29f8b267fdef0c41890293b7761 Mon Sep 17 00:00:00 2001 From: Ty Hopp Date: Tue, 15 Feb 2022 11:21:12 +0800 Subject: [PATCH 2/6] Break loadConfigAndPlugins into two functions --- .../gatsby-cli/__tests__/build.js | 5 ++- .../gatsby-cli/__tests__/develop.js | 5 ++- .../gatsby-cli/__tests__/repl.js | 5 ++- .../index.ts} | 27 ++++++---------- packages/gatsby/src/services/initialize.ts | 31 +++++++++++-------- .../src/utils/worker/__tests__/queries.ts | 8 +++-- .../src/utils/worker/__tests__/schema.ts | 8 +++-- .../worker/child/load-config-and-plugins.ts | 8 +++-- 8 files changed, 51 insertions(+), 46 deletions(-) rename packages/gatsby/src/bootstrap/{load-config-and-plugins.ts => load-config/index.ts} (73%) diff --git a/integration-tests/gatsby-cli/__tests__/build.js b/integration-tests/gatsby-cli/__tests__/build.js index e31a54dee070a..f1a0a648df841 100644 --- a/integration-tests/gatsby-cli/__tests__/build.js +++ b/integration-tests/gatsby-cli/__tests__/build.js @@ -12,9 +12,8 @@ describe(`gatsby build`, () => { it(`creates a built gatsby site`, () => { const [code, logs] = GatsbyCLI.from(cwd).invoke(`build`) - logs.should.contain( - `success open and validate gatsby-configs, load plugins` - ) + logs.should.contain(`success load gatsby config`) + logs.should.contain(`success load plugins`) logs.should.contain(`success onPreInit`) logs.should.contain(`success initialize cache`) logs.should.contain(`success copy gatsby files`) diff --git a/integration-tests/gatsby-cli/__tests__/develop.js b/integration-tests/gatsby-cli/__tests__/develop.js index db41335abe2f7..591b83aabb577 100644 --- a/integration-tests/gatsby-cli/__tests__/develop.js +++ b/integration-tests/gatsby-cli/__tests__/develop.js @@ -26,9 +26,8 @@ describe(`gatsby develop`, () => { // 3. Make sure logs for the user contain expected results const logs = getLogs() - logs.should.contain( - `success open and validate gatsby-configs, load plugins` - ) + logs.should.contain(`success load gatsby config`) + logs.should.contain(`success load plugins`) logs.should.contain(`success onPreInit`) logs.should.contain(`success initialize cache`) logs.should.contain(`success copy gatsby files`) diff --git a/integration-tests/gatsby-cli/__tests__/repl.js b/integration-tests/gatsby-cli/__tests__/repl.js index 9eec28120ea7f..55ff0a8dfc0ad 100644 --- a/integration-tests/gatsby-cli/__tests__/repl.js +++ b/integration-tests/gatsby-cli/__tests__/repl.js @@ -21,9 +21,8 @@ describe(`gatsby repl`, () => { // 3. Make assertions const logs = getLogs() - logs.should.contain( - `success open and validate gatsby-configs, load plugins` - ) + logs.should.contain(`success load gatsby config`) + logs.should.contain(`success load plugins`) logs.should.contain(`success onPreInit`) logs.should.contain(`success initialize cache`) logs.should.contain(`success copy gatsby files`) diff --git a/packages/gatsby/src/bootstrap/load-config-and-plugins.ts b/packages/gatsby/src/bootstrap/load-config/index.ts similarity index 73% rename from packages/gatsby/src/bootstrap/load-config-and-plugins.ts rename to packages/gatsby/src/bootstrap/load-config/index.ts index 51dc308bb1992..baf6eef4b61fc 100644 --- a/packages/gatsby/src/bootstrap/load-config-and-plugins.ts +++ b/packages/gatsby/src/bootstrap/load-config/index.ts @@ -1,19 +1,15 @@ import reporter from "gatsby-cli/lib/reporter" import telemetry from "gatsby-telemetry" +import { preferDefault } from "../prefer-default" +import { getConfigFile } from "../get-config-file" +import { internalActions } from "../../redux/actions" +import loadThemes from "../load-themes" +import { store } from "../../redux" +import handleFlags from "../../utils/handle-flags" +import availableFlags from "../../utils/flags" +import { IProgram } from "../../commands/types" -import { IFlattenedPlugin } from "./load-plugins/types" - -import { preferDefault } from "../bootstrap/prefer-default" -import { getConfigFile } from "../bootstrap/get-config-file" -import { loadPlugins } from "../bootstrap/load-plugins" -import { internalActions } from "../redux/actions" -import loadThemes from "../bootstrap/load-themes" -import { store } from "../redux" -import handleFlags from "../utils/handle-flags" -import availableFlags from "../utils/flags" -import { IProgram } from "../commands/types" - -export async function loadConfigAndPlugins({ +export async function loadConfig({ siteDirectory, processFlags = false, }: { @@ -22,7 +18,6 @@ export async function loadConfigAndPlugins({ program?: IProgram }): Promise<{ config: any - flattenedPlugins: Array }> { // Try opening the site's gatsby-config.js file. const { configModule, configFilePath } = await getConfigFile( @@ -94,7 +89,5 @@ export async function loadConfigAndPlugins({ store.dispatch(internalActions.setSiteConfig(config)) - const flattenedPlugins = await loadPlugins(config, siteDirectory) - - return { config, flattenedPlugins } + return { config } } diff --git a/packages/gatsby/src/services/initialize.ts b/packages/gatsby/src/services/initialize.ts index d0e3da2c6de7a..e3201ad9397e3 100644 --- a/packages/gatsby/src/services/initialize.ts +++ b/packages/gatsby/src/services/initialize.ts @@ -19,7 +19,8 @@ import { IPluginInfoOptions } from "../bootstrap/load-plugins/types" import { IGatsbyState, IStateProgram } from "../redux/types" import { IBuildContext } from "./types" import { detectLmdbStore } from "../datastore" -import { loadConfigAndPlugins } from "../bootstrap/load-config-and-plugins" +import { loadConfig } from "../bootstrap/load-config" +import { loadPlugins } from "../bootstrap/load-plugins" import type { InternalJob } from "../utils/jobs/types" import { enableNodeMutationsDetection } from "../utils/detect-node-mutations" import { resolveModule } from "../utils/module-resolver" @@ -160,19 +161,25 @@ export async function initialize({ emitter.on(`END_JOB`, onEndJob) - // Try opening the site's gatsby-config.js file. - let activity = reporter.activityTimer( - `open and validate gatsby-configs, load plugins`, - { - parentSpan, - } - ) + // Load gatsby config + let activity = reporter.activityTimer(`load gatsby config`, { + parentSpan, + }) activity.start() - - const { config, flattenedPlugins } = await loadConfigAndPlugins({ - siteDirectory: program.directory, + const siteDirectory = program.directory + const { config } = await loadConfig({ + siteDirectory, processFlags: true, }) + activity.end() + + // Load plugins + activity = reporter.activityTimer(`load plugins`, { + parentSpan, + }) + activity.start() + const flattenedPlugins = await loadPlugins(config, siteDirectory) + activity.end() // TODO: figure out proper way of disabling loading indicator // for now GATSBY_QUERY_ON_DEMAND_LOADING_INDICATOR=false gatsby develop @@ -197,8 +204,6 @@ export async function initialize({ ) } - activity.end() - if (process.env.GATSBY_EXPERIMENTAL_QUERY_ON_DEMAND) { if (process.env.gatsby_executing_command !== `develop`) { // we don't want to ever have this flag enabled for anything than develop diff --git a/packages/gatsby/src/utils/worker/__tests__/queries.ts b/packages/gatsby/src/utils/worker/__tests__/queries.ts index 961fb82c945cb..a9d92ad9cb5b3 100644 --- a/packages/gatsby/src/utils/worker/__tests__/queries.ts +++ b/packages/gatsby/src/utils/worker/__tests__/queries.ts @@ -10,7 +10,8 @@ import { emitter, loadPartialStateFromDisk, } from "../../../redux" -import { loadConfigAndPlugins } from "../../../bootstrap/load-config-and-plugins" +import { loadConfig } from "../../../bootstrap/load-config" +import { loadPlugins } from "../../../bootstrap/load-plugins" import { createTestWorker, describeWhenLMDB, @@ -124,7 +125,10 @@ describeWhenLMDB(`worker (queries)`, () => { worker = createTestWorker() const siteDirectory = path.join(__dirname, `fixtures`, `sample-site`) - await loadConfigAndPlugins({ siteDirectory }) + const { config } = await loadConfig({ + siteDirectory, + }) + await loadPlugins(config, siteDirectory) await Promise.all(worker.all.loadConfigAndPlugins({ siteDirectory })) await sourceNodesAndRemoveStaleNodes({ webhookBody: {} }) await getDataStore().ready() diff --git a/packages/gatsby/src/utils/worker/__tests__/schema.ts b/packages/gatsby/src/utils/worker/__tests__/schema.ts index 33e42dc742ae5..2025f08d36cec 100644 --- a/packages/gatsby/src/utils/worker/__tests__/schema.ts +++ b/packages/gatsby/src/utils/worker/__tests__/schema.ts @@ -6,7 +6,8 @@ import { CombinedState } from "redux" import { build } from "../../../schema" import sourceNodesAndRemoveStaleNodes from "../../source-nodes" import { savePartialStateToDisk, store } from "../../../redux" -import { loadConfigAndPlugins } from "../../../bootstrap/load-config-and-plugins" +import { loadConfig } from "../../../bootstrap/load-config" +import { loadPlugins } from "../../../bootstrap/load-plugins" import { createTestWorker, describeWhenLMDB, @@ -56,7 +57,10 @@ describeWhenLMDB(`worker (schema)`, () => { worker = createTestWorker() const siteDirectory = path.join(__dirname, `fixtures`, `sample-site`) - await loadConfigAndPlugins({ siteDirectory }) + const { config } = await loadConfig({ + siteDirectory, + }) + await loadPlugins(config, siteDirectory) await Promise.all(worker.all.loadConfigAndPlugins({ siteDirectory })) await sourceNodesAndRemoveStaleNodes({ webhookBody: {} }) await getDataStore().ready() diff --git a/packages/gatsby/src/utils/worker/child/load-config-and-plugins.ts b/packages/gatsby/src/utils/worker/child/load-config-and-plugins.ts index 7af88996f8fed..afbd24a41c3aa 100644 --- a/packages/gatsby/src/utils/worker/child/load-config-and-plugins.ts +++ b/packages/gatsby/src/utils/worker/child/load-config-and-plugins.ts @@ -1,9 +1,10 @@ -import { loadConfigAndPlugins as internalLoadConfigAndPlugins } from "../../../bootstrap/load-config-and-plugins" +import { loadConfig } from "../../../bootstrap/load-config" +import { loadPlugins } from "../../../bootstrap/load-plugins" import { store } from "../../../redux" import apiRunnerNode from "../../api-runner-node" export async function loadConfigAndPlugins( - ...args: Parameters + ...args: Parameters ): Promise { const [{ siteDirectory, program }] = args @@ -14,7 +15,8 @@ export async function loadConfigAndPlugins( directory: siteDirectory, }, }) - await internalLoadConfigAndPlugins(...args) + const { config } = await loadConfig(...args) + await loadPlugins(config, siteDirectory) // Cache is already initialized if (_CFLAGS_.GATSBY_MAJOR === `4`) { From a6f5d3832f455406df50004ab40793d8429f8a6e Mon Sep 17 00:00:00 2001 From: Ty Hopp Date: Tue, 15 Feb 2022 13:55:05 +0800 Subject: [PATCH 3/6] Extract out local plugin check --- .../bootstrap/load-plugins/resolve-plugin.ts | 41 ++++++++----------- .../load-plugins/utils/check-local-plugin.ts | 41 +++++++++++++++++++ 2 files changed, 59 insertions(+), 23 deletions(-) create mode 100644 packages/gatsby/src/bootstrap/load-plugins/utils/check-local-plugin.ts diff --git a/packages/gatsby/src/bootstrap/load-plugins/resolve-plugin.ts b/packages/gatsby/src/bootstrap/load-plugins/resolve-plugin.ts index 9cd103b95eca6..4244c51594cd0 100644 --- a/packages/gatsby/src/bootstrap/load-plugins/resolve-plugin.ts +++ b/packages/gatsby/src/bootstrap/load-plugins/resolve-plugin.ts @@ -1,7 +1,6 @@ import path from "path" import fs from "fs" import { slash, createRequireFromPath } from "gatsby-core-utils" -import { sync as existsSync } from "fs-exists-cached" import { warnOnIncompatiblePeerDependency } from "./validate" import { PackageJson } from "../../.." import { IPluginInfo, PluginRef } from "./types" @@ -9,6 +8,7 @@ import { createPluginId } from "./utils/create-id" import { createFileContentHash } from "./utils/create-hash" import reporter from "gatsby-cli/lib/reporter" import { isString } from "lodash" +import { checkLocalPlugin } from "./utils/check-local-plugin" /** * @param plugin @@ -24,30 +24,25 @@ import { isString } from "lodash" export function resolvePlugin(plugin: PluginRef, rootDir: string): IPluginInfo { const pluginName = isString(plugin) ? plugin : plugin.resolve - // Only find plugins when we're not given an absolute path - if (!existsSync(pluginName) && rootDir) { - // Find the plugin in the local plugins folder - const resolvedPath = slash(path.join(rootDir, `plugins/${pluginName}`)) + // Handle local plugins + const { validLocalPlugin, localPluginPath = `` } = checkLocalPlugin( + plugin, + rootDir + ) - if (existsSync(resolvedPath)) { - if (existsSync(`${resolvedPath}/package.json`)) { - const packageJSON = JSON.parse( - fs.readFileSync(`${resolvedPath}/package.json`, `utf-8`) - ) as PackageJson - const name = packageJSON.name || pluginName - warnOnIncompatiblePeerDependency(name, packageJSON) + if (validLocalPlugin && localPluginPath) { + const packageJSON = JSON.parse( + fs.readFileSync(`${localPluginPath}/package.json`, `utf-8`) + ) as PackageJson + const name = packageJSON.name || pluginName + warnOnIncompatiblePeerDependency(name, packageJSON) - return { - resolve: resolvedPath, - name, - id: createPluginId(name), - version: - packageJSON.version || createFileContentHash(resolvedPath, `**`), - } - } else { - // Make package.json a requirement for local plugins too - throw new Error(`Plugin ${pluginName} requires a package.json file`) - } + return { + resolve: localPluginPath, + name, + id: createPluginId(name), + version: + packageJSON.version || createFileContentHash(localPluginPath, `**`), } } diff --git a/packages/gatsby/src/bootstrap/load-plugins/utils/check-local-plugin.ts b/packages/gatsby/src/bootstrap/load-plugins/utils/check-local-plugin.ts new file mode 100644 index 0000000000000..974926b56c656 --- /dev/null +++ b/packages/gatsby/src/bootstrap/load-plugins/utils/check-local-plugin.ts @@ -0,0 +1,41 @@ +import { PluginRef } from "../types" +import { sync as existsSync } from "fs-exists-cached" +import { slash } from "gatsby-core-utils" +import path from "path" + +/** + * Checks if a plugin is a valid local plugin and returns the resolved path if it is. + */ +export function checkLocalPlugin( + plugin: PluginRef, + rootDir: string +): { validLocalPlugin: boolean; localPluginPath?: string } { + const pluginName = typeof plugin === `string` ? plugin : plugin.resolve + + // Make sure the plugin exists relatively + if (existsSync(pluginName) || !rootDir) { + return { + validLocalPlugin: false, + } + } + + const resolvedPath = slash(path.join(rootDir, `plugins/${pluginName}`)) + + if (!existsSync(resolvedPath)) { + return { + validLocalPlugin: false, + } + } + + const resolvedPackageJson = existsSync(`${resolvedPath}/package.json`) + + // package.json is a requirement for local plugins + if (!resolvedPackageJson) { + throw new Error(`Local plugin ${pluginName} requires a package.json file`) + } + + return { + validLocalPlugin: true, + localPluginPath: resolvedPath, + } +} From 927a40d9974a629cb5249301aa4e062e288ad068 Mon Sep 17 00:00:00 2001 From: Ty Hopp Date: Tue, 15 Feb 2022 18:23:39 +0800 Subject: [PATCH 4/6] Revert async back to original sync funcs --- .../gatsby/src/bootstrap/load-config/index.ts | 5 +- .../src/bootstrap/load-plugins/index.ts | 2 +- .../load-plugins/load-internal-plugins.ts | 54 +++++++++---------- .../bootstrap/load-plugins/process-plugin.ts | 11 ++-- .../load-plugins/utils/handle-gatsby-cloud.ts | 6 +-- 5 files changed, 36 insertions(+), 42 deletions(-) diff --git a/packages/gatsby/src/bootstrap/load-config/index.ts b/packages/gatsby/src/bootstrap/load-config/index.ts index baf6eef4b61fc..7fe38181458c3 100644 --- a/packages/gatsby/src/bootstrap/load-config/index.ts +++ b/packages/gatsby/src/bootstrap/load-config/index.ts @@ -8,6 +8,7 @@ import { store } from "../../redux" import handleFlags from "../../utils/handle-flags" import availableFlags from "../../utils/flags" import { IProgram } from "../../commands/types" +import { ISiteConfig } from "../load-plugins/types" export async function loadConfig({ siteDirectory, @@ -17,7 +18,7 @@ export async function loadConfig({ processFlags?: boolean program?: IProgram }): Promise<{ - config: any + config: ISiteConfig }> { // Try opening the site's gatsby-config.js file. const { configModule, configFilePath } = await getConfigFile( @@ -89,5 +90,5 @@ export async function loadConfig({ store.dispatch(internalActions.setSiteConfig(config)) - return { config } + return config } diff --git a/packages/gatsby/src/bootstrap/load-plugins/index.ts b/packages/gatsby/src/bootstrap/load-plugins/index.ts index c587cc74a5540..dee24495d4eff 100644 --- a/packages/gatsby/src/bootstrap/load-plugins/index.ts +++ b/packages/gatsby/src/bootstrap/load-plugins/index.ts @@ -32,7 +32,7 @@ export async function loadPlugins( }) // Collate internal plugins, site config plugins, site default plugins - const pluginInfos = await loadInternalPlugins(config, rootDir) + const pluginInfos = loadInternalPlugins(config, rootDir) // Create a flattened array of the plugins const pluginArray = flattenPlugins(pluginInfos) diff --git a/packages/gatsby/src/bootstrap/load-plugins/load-internal-plugins.ts b/packages/gatsby/src/bootstrap/load-plugins/load-internal-plugins.ts index cc7058b08867f..fb5f149118dfd 100644 --- a/packages/gatsby/src/bootstrap/load-plugins/load-internal-plugins.ts +++ b/packages/gatsby/src/bootstrap/load-plugins/load-internal-plugins.ts @@ -19,16 +19,16 @@ import { const TYPESCRIPT_PLUGIN_NAME = `gatsby-plugin-typescript` -export async function loadInternalPlugins( +export function loadInternalPlugins( config: ISiteConfig = {}, rootDir: string -): Promise> { +): Array { // Instantiate plugins. const plugins: Array = [] const configuredPluginNames = new Set() // Add internal plugins - const internalPluginPaths = [ + const internalPlugins = [ `../../internal-plugins/dev-404-page`, `../../internal-plugins/load-babel-config`, `../../internal-plugins/internal-data-bridge`, @@ -38,22 +38,18 @@ export async function loadInternalPlugins( `../../internal-plugins/functions`, ].filter(Boolean) as Array - for (const internalPluginPath of internalPluginPaths) { - const internalPluginAbsolutePath = path.join(__dirname, internalPluginPath) - const processedPlugin = await processPlugin( - internalPluginAbsolutePath, - rootDir - ) - plugins.push(processedPlugin) - } + internalPlugins.forEach(relPath => { + const absPath = path.join(__dirname, relPath) + plugins.push(processPlugin(absPath, rootDir)) + }) // Add plugins from the site config. if (config.plugins) { - for (const plugin of config.plugins) { - const processedPlugin = await processPlugin(plugin, rootDir) + config.plugins.forEach(plugin => { + const processedPlugin = processPlugin(plugin, rootDir) plugins.push(processedPlugin) configuredPluginNames.add(processedPlugin.name) - } + }) } // the order of all of these page-creators matters. The "last plugin wins", @@ -61,20 +57,20 @@ export async function loadInternalPlugins( // match the plugin definition order before that. This works fine for themes // because themes have already been added in the proper order to the plugins // array - for (const plugin of [...plugins]) { - const processedPlugin = await processPlugin( - { - resolve: require.resolve(`gatsby-plugin-page-creator`), - options: { - path: slash(path.join(plugin.resolve, `src/pages`)), - pathCheck: false, + plugins.forEach(plugin => { + plugins.push( + processPlugin( + { + resolve: require.resolve(`gatsby-plugin-page-creator`), + options: { + path: slash(path.join(plugin.resolve, `src/pages`)), + pathCheck: false, + }, }, - }, - rootDir + rootDir + ) ) - - plugins.push(processedPlugin) - } + }) if ( _CFLAGS_.GATSBY_MAJOR === `4` && @@ -90,12 +86,12 @@ export async function loadInternalPlugins( !configuredPluginNames.has(GATSBY_CLOUD_PLUGIN_NAME) && (process.env.GATSBY_CLOUD === `true` || process.env.GATSBY_CLOUD === `1`) ) { - await addGatsbyPluginCloudPluginWhenInstalled(plugins, rootDir) + addGatsbyPluginCloudPluginWhenInstalled(plugins, rootDir) } // Support Typescript by default but allow users to override it if (!configuredPluginNames.has(TYPESCRIPT_PLUGIN_NAME)) { - const processedTypeScriptPlugin = await processPlugin( + const processedTypeScriptPlugin = processPlugin( { resolve: require.resolve(TYPESCRIPT_PLUGIN_NAME), options: { @@ -144,7 +140,7 @@ export async function loadInternalPlugins( } } - const processedPageCreatorPlugin = await processPlugin( + const processedPageCreatorPlugin = processPlugin( { resolve: require.resolve(`gatsby-plugin-page-creator`), options: pageCreatorOptions, diff --git a/packages/gatsby/src/bootstrap/load-plugins/process-plugin.ts b/packages/gatsby/src/bootstrap/load-plugins/process-plugin.ts index 0d31ecbbfcbe6..c9707e4c8069f 100644 --- a/packages/gatsby/src/bootstrap/load-plugins/process-plugin.ts +++ b/packages/gatsby/src/bootstrap/load-plugins/process-plugin.ts @@ -3,15 +3,12 @@ import { createPluginId } from "./utils/create-id" import { resolvePlugin } from "./resolve-plugin" import { isString, isEmpty, set, merge } from "lodash" -export async function processPlugin( - plugin: PluginRef, - rootDir: string -): Promise { +export function processPlugin(plugin: PluginRef, rootDir: string): IPluginInfo { // Respect the directory that the plugin was sourced from initially rootDir = (!isString(plugin) && plugin.parentDir) || rootDir if (isString(plugin)) { - const info = await resolvePlugin(plugin, rootDir) + const info = resolvePlugin(plugin, rootDir) return { ...info, @@ -54,7 +51,7 @@ export async function processPlugin( const processed: Array = [] for (const root of roots) { - const result = await processPlugin(root, rootDir) + const result = processPlugin(root, rootDir) processed.push(result) } @@ -78,7 +75,7 @@ export async function processPlugin( } } - const info = await resolvePlugin(plugin, rootDir) + const info = resolvePlugin(plugin, rootDir) return { ...info, diff --git a/packages/gatsby/src/bootstrap/load-plugins/utils/handle-gatsby-cloud.ts b/packages/gatsby/src/bootstrap/load-plugins/utils/handle-gatsby-cloud.ts index 0f4509e32e406..78ff139fb0033 100644 --- a/packages/gatsby/src/bootstrap/load-plugins/utils/handle-gatsby-cloud.ts +++ b/packages/gatsby/src/bootstrap/load-plugins/utils/handle-gatsby-cloud.ts @@ -5,17 +5,17 @@ import { processPlugin } from "../process-plugin" export const GATSBY_CLOUD_PLUGIN_NAME = `gatsby-plugin-gatsby-cloud` -export async function addGatsbyPluginCloudPluginWhenInstalled( +export function addGatsbyPluginCloudPluginWhenInstalled( plugins: Array, rootDir: string -): Promise { +): void { const cloudPluginLocation = resolveFromSilent( rootDir, GATSBY_CLOUD_PLUGIN_NAME ) if (cloudPluginLocation) { - const processedGatsbyCloudPlugin = await processPlugin( + const processedGatsbyCloudPlugin = processPlugin( { resolve: cloudPluginLocation, options: {}, From 535724a3054b5338618bc244de4363a71a4cc078 Mon Sep 17 00:00:00 2001 From: Ty Hopp Date: Tue, 15 Feb 2022 18:34:56 +0800 Subject: [PATCH 5/6] Adjust load config interface --- packages/gatsby/src/bootstrap/load-config/index.ts | 6 ++---- packages/gatsby/src/bootstrap/load-plugins/index.ts | 5 +++-- packages/gatsby/src/services/initialize.ts | 2 +- packages/gatsby/src/utils/worker/__tests__/queries.ts | 2 +- packages/gatsby/src/utils/worker/__tests__/schema.ts | 2 +- .../src/utils/worker/child/load-config-and-plugins.ts | 2 +- 6 files changed, 9 insertions(+), 10 deletions(-) diff --git a/packages/gatsby/src/bootstrap/load-config/index.ts b/packages/gatsby/src/bootstrap/load-config/index.ts index 7fe38181458c3..bce4397305c9e 100644 --- a/packages/gatsby/src/bootstrap/load-config/index.ts +++ b/packages/gatsby/src/bootstrap/load-config/index.ts @@ -8,7 +8,7 @@ import { store } from "../../redux" import handleFlags from "../../utils/handle-flags" import availableFlags from "../../utils/flags" import { IProgram } from "../../commands/types" -import { ISiteConfig } from "../load-plugins/types" +import { IGatsbyConfig } from "../../internal" export async function loadConfig({ siteDirectory, @@ -17,9 +17,7 @@ export async function loadConfig({ siteDirectory: string processFlags?: boolean program?: IProgram -}): Promise<{ - config: ISiteConfig -}> { +}): Promise { // Try opening the site's gatsby-config.js file. const { configModule, configFilePath } = await getConfigFile( siteDirectory, diff --git a/packages/gatsby/src/bootstrap/load-plugins/index.ts b/packages/gatsby/src/bootstrap/load-plugins/index.ts index dee24495d4eff..7d743760e5cc4 100644 --- a/packages/gatsby/src/bootstrap/load-plugins/index.ts +++ b/packages/gatsby/src/bootstrap/load-plugins/index.ts @@ -10,13 +10,14 @@ import { handleMultipleReplaceRenderers, validateConfigPluginsOptions, } from "./validate" -import { IFlattenedPlugin, IRawSiteConfig } from "./types" +import { IFlattenedPlugin } from "./types" import { normalizeConfig } from "./utils/normalize" import { getAPI } from "./utils/get-api" import { flattenPlugins } from "./utils/flatten-plugins" +import { IGatsbyConfig } from "../../internal" export async function loadPlugins( - rawConfig: IRawSiteConfig = {}, + rawConfig: IGatsbyConfig, rootDir: string ): Promise> { // Turn all strings in plugins: [`...`] into the { resolve: ``, options: {} } form diff --git a/packages/gatsby/src/services/initialize.ts b/packages/gatsby/src/services/initialize.ts index e3201ad9397e3..daeffcfac3148 100644 --- a/packages/gatsby/src/services/initialize.ts +++ b/packages/gatsby/src/services/initialize.ts @@ -167,7 +167,7 @@ export async function initialize({ }) activity.start() const siteDirectory = program.directory - const { config } = await loadConfig({ + const config = await loadConfig({ siteDirectory, processFlags: true, }) diff --git a/packages/gatsby/src/utils/worker/__tests__/queries.ts b/packages/gatsby/src/utils/worker/__tests__/queries.ts index a9d92ad9cb5b3..90c5270a06ad5 100644 --- a/packages/gatsby/src/utils/worker/__tests__/queries.ts +++ b/packages/gatsby/src/utils/worker/__tests__/queries.ts @@ -125,7 +125,7 @@ describeWhenLMDB(`worker (queries)`, () => { worker = createTestWorker() const siteDirectory = path.join(__dirname, `fixtures`, `sample-site`) - const { config } = await loadConfig({ + const config = await loadConfig({ siteDirectory, }) await loadPlugins(config, siteDirectory) diff --git a/packages/gatsby/src/utils/worker/__tests__/schema.ts b/packages/gatsby/src/utils/worker/__tests__/schema.ts index 2025f08d36cec..27826cc50f267 100644 --- a/packages/gatsby/src/utils/worker/__tests__/schema.ts +++ b/packages/gatsby/src/utils/worker/__tests__/schema.ts @@ -57,7 +57,7 @@ describeWhenLMDB(`worker (schema)`, () => { worker = createTestWorker() const siteDirectory = path.join(__dirname, `fixtures`, `sample-site`) - const { config } = await loadConfig({ + const config = await loadConfig({ siteDirectory, }) await loadPlugins(config, siteDirectory) diff --git a/packages/gatsby/src/utils/worker/child/load-config-and-plugins.ts b/packages/gatsby/src/utils/worker/child/load-config-and-plugins.ts index afbd24a41c3aa..2203b48f39818 100644 --- a/packages/gatsby/src/utils/worker/child/load-config-and-plugins.ts +++ b/packages/gatsby/src/utils/worker/child/load-config-and-plugins.ts @@ -15,7 +15,7 @@ export async function loadConfigAndPlugins( directory: siteDirectory, }, }) - const { config } = await loadConfig(...args) + const config = await loadConfig(...args) await loadPlugins(config, siteDirectory) // Cache is already initialized From beefd52369b83cbdb8e254f8e9c82d824a0fedfe Mon Sep 17 00:00:00 2001 From: Ty Hopp Date: Tue, 15 Feb 2022 18:48:43 +0800 Subject: [PATCH 6/6] Missed awaits --- packages/gatsby/src/bootstrap/load-plugins/validate.ts | 2 +- packages/gatsby/src/bootstrap/load-themes/index.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/gatsby/src/bootstrap/load-plugins/validate.ts b/packages/gatsby/src/bootstrap/load-plugins/validate.ts index 57f72b49bbefd..a227c1a3619a6 100644 --- a/packages/gatsby/src/bootstrap/load-plugins/validate.ts +++ b/packages/gatsby/src/bootstrap/load-plugins/validate.ts @@ -190,7 +190,7 @@ async function validatePluginsOptions( plugins.map(async plugin => { let gatsbyNode try { - const resolvedPlugin = await resolvePlugin(plugin, rootDir) + const resolvedPlugin = resolvePlugin(plugin, rootDir) gatsbyNode = require(`${resolvedPlugin.resolve}/gatsby-node`) } catch (err) { gatsbyNode = {} diff --git a/packages/gatsby/src/bootstrap/load-themes/index.js b/packages/gatsby/src/bootstrap/load-themes/index.js index f7c8f13be92dc..d38dfef3c1895 100644 --- a/packages/gatsby/src/bootstrap/load-themes/index.js +++ b/packages/gatsby/src/bootstrap/load-themes/index.js @@ -34,7 +34,7 @@ const resolveTheme = async ( pathToLocalTheme = path.join(rootDir, `plugins`, themeName) // is a local plugin OR it doesn't exist try { - const { resolve } = await resolvePlugin(themeName, rootDir) + const { resolve } = resolvePlugin(themeName, rootDir) themeDir = resolve } catch (localErr) { reporter.panic(`Failed to resolve ${themeName}`, localErr)