From b1210dac69b27075cb13c0590853fcda82d0b902 Mon Sep 17 00:00:00 2001 From: "Fred K. Schott" Date: Sat, 20 Jun 2020 11:47:02 -0700 Subject: [PATCH] run optimized dependency install after build --- src/commands/build.ts | 170 +++++++++++++++++++++++------------------- src/config.ts | 12 ++- src/util.ts | 1 - 3 files changed, 97 insertions(+), 86 deletions(-) diff --git a/src/commands/build.ts b/src/commands/build.ts index c62d8d004a..dd48385b0f 100644 --- a/src/commands/build.ts +++ b/src/commands/build.ts @@ -10,7 +10,8 @@ import path from 'path'; import rimraf from 'rimraf'; import {BuildScript} from '../config'; import {transformEsmImports} from '../rewrite-imports'; -import {BUILD_DEPENDENCIES_DIR, CommandOptions, ImportMap} from '../util'; +import {printStats} from '../stats-formatter'; +import {CommandOptions} from '../util'; import { generateEnvModule, getFileBuilderForWorker, @@ -20,42 +21,44 @@ import { } from './build-util'; import {stopEsbuild} from './esbuildPlugin'; import {createImportResolver} from './import-resolver'; -import {command as installCommand} from './install'; +import {getInstallTargets, run as installRunner} from './install'; import {paint} from './paint'; import srcFileExtensionMapping from './src-file-extension-mapping'; -export async function command(commandOptions: CommandOptions) { - const {cwd, config} = commandOptions; - - // Start with a fresh install of your dependencies, for production - const installCommandOptions = merge(commandOptions, { - config: { - installOptions: { - dest: BUILD_DEPENDENCIES_DIR, - env: {NODE_ENV: process.env.NODE_ENV || 'production'}, - treeshake: config.installOptions.treeshake ?? true, - }, +async function installOptimizedDependencies( + allBuiltJsFiles: [string, string][], + installDest: string, + commandOptions: CommandOptions, +) { + console.log('optimizing dependencies...'); + const installConfig = merge(commandOptions.config, { + installOptions: { + dest: installDest, + env: {NODE_ENV: process.env.NODE_ENV || 'production'}, + treeshake: commandOptions.config.installOptions.treeshake ?? true, }, }); - const dependencyImportMapLoc = path.join(config.installOptions.dest, 'import-map.json'); - - // Start with a fresh install of your dependencies, always. - console.log(chalk.yellow('! rebuilding dependencies...')); - await installCommand(installCommandOptions); + // 1. Scan imports from your final built JS files + const installTargets = await getInstallTargets(installConfig, allBuiltJsFiles); + // 2. Install dependencies, based on the scan of your final build + const installResult = await installRunner( + {...commandOptions, config: installConfig}, + installTargets, + ); + return installResult; +} +export async function command(commandOptions: CommandOptions) { + const {cwd, config} = commandOptions; const messageBus = new EventEmitter(); const relevantWorkers: BuildScript[] = []; const allBuildExtensions: string[] = []; - let dependencyImportMap: ImportMap = {imports: {}}; - try { - dependencyImportMap = require(dependencyImportMapLoc); - } catch (err) { - // no import-map found, safe to ignore - } - for (const workerConfig of config.scripts) { - const {type, match} = workerConfig; + const {id, type, match} = workerConfig; + if (id === 'mount:web_modules') { + continue; + } if (type === 'build' || type === 'run' || type === 'mount' || type === 'bundle') { relevantWorkers.push(workerConfig); } @@ -65,6 +68,7 @@ export async function command(commandOptions: CommandOptions) { } let bundleWorker = config.scripts.find((s) => s.type === 'bundle'); + let installWorker = config.scripts.find((s) => s.id === 'mount:web_modules')!; const isBundledHardcoded = config.devOptions.bundle !== undefined; const isBundled = isBundledHardcoded ? !!config.devOptions.bundle : !!bundleWorker; if (!bundleWorker) { @@ -170,6 +174,9 @@ export async function command(commandOptions: CommandOptions) { const mountDirDetails: any[] = relevantWorkers .map((scriptConfig) => { const {id, type, args} = scriptConfig; + if (id === 'mount:web_modules') { + return false; + } if (type !== 'mount') { return false; } @@ -180,13 +187,12 @@ export async function command(commandOptions: CommandOptions) { .filter(Boolean); const includeFileSets: [string, string, string[]][] = []; - const allProxiedFiles = new Set(); for (const [id, dirDisk, dirDest] of mountDirDetails) { messageBus.emit('WORKER_UPDATE', {id, state: ['RUNNING', 'yellow']}); let allFiles; try { allFiles = glob.sync(`**/*`, { - ignore: id === 'mount:web_modules' ? [] : config.exclude, + ignore: config.exclude, cwd: dirDisk, absolute: true, nodir: true, @@ -197,12 +203,11 @@ export async function command(commandOptions: CommandOptions) { allFiles.map(async (f) => { f = path.resolve(f); // this is necessary since glob.sync() returns paths with / on windows. path.resolve() will switch them to the native path separator. if ( - !f.startsWith(commandOptions.config.installOptions.dest) && - (allBuildExtensions.includes(path.extname(f).substr(1)) || - path.extname(f) === '.jsx' || - path.extname(f) === '.tsx' || - path.extname(f) === '.ts' || - path.extname(f) === '.js') + allBuildExtensions.includes(path.extname(f).substr(1)) || + path.extname(f) === '.jsx' || + path.extname(f) === '.tsx' || + path.extname(f) === '.ts' || + path.extname(f) === '.js' ) { allBuildNeededFiles.push(f); return; @@ -227,9 +232,8 @@ export async function command(commandOptions: CommandOptions) { } } - const webModulesScript = config.scripts.find((script) => script.id === 'mount:web_modules')!; - const webModulesPath = webModulesScript.args.toUrl; const allBuiltFromFiles = new Set(); + const allBuiltJsFiles: [string, string, string][] = []; for (const workerConfig of relevantWorkers) { const {id, match, type} = workerConfig; if (type !== 'build' || match.length === 0) { @@ -276,6 +280,7 @@ export async function command(commandOptions: CommandOptions) { continue; } + allBuiltFromFiles.add(fileLoc); if (path.extname(outPath) === '.js') { if (resources?.css) { const cssOutPath = outPath.replace(/.js$/, '.css'); @@ -283,56 +288,61 @@ export async function command(commandOptions: CommandOptions) { await fs.writeFile(cssOutPath, resources.css); code = `import './${path.basename(cssOutPath)}';\n` + code; } - const resolveImportSpecifier = createImportResolver({ - fileLoc, - webModulesPath, - dependencyImportMap, - isDev: false, - isBundled, - config, - }); - code = await transformEsmImports(code, (spec) => { - // Try to resolve the specifier to a known URL in the project - const resolvedImportUrl = resolveImportSpecifier(spec); - if (resolvedImportUrl) { - // We treat ".proxy.js" files special: we need to make sure that they exist on disk - // in the final build, so we mark them to be written to disk at the next step. - if (resolvedImportUrl.endsWith('.proxy.js')) { - allProxiedFiles.add( - resolvedImportUrl.startsWith('/') - ? path.resolve(cwd, spec) - : path.resolve(path.dirname(outPath), spec), - ); - } - return resolvedImportUrl; - } - // If that fails, return a placeholder import and attempt to resolve. - let [missingPackageName, ...deepPackagePathParts] = spec.split('/'); - if (missingPackageName.startsWith('@')) { - missingPackageName += '/' + deepPackagePathParts.shift(); - } - messageBus.emit('MISSING_WEB_MODULE', { - id: fileLoc, - data: { - spec: spec, - pkgName: missingPackageName, - }, - }); - // Sort of lazy, but we expect "MISSING_WEB_MODULE" to exit the build with an error. - // So, just return the original import here since it will never be seen. - return spec; - }); code = wrapImportMeta({code, env: true, hmr: false, config}); + allBuiltJsFiles.push([outPath, code, fileLoc]); + } else { + await fs.mkdir(path.dirname(outPath), {recursive: true}); + await fs.writeFile(outPath, code); } - await fs.mkdir(path.dirname(outPath), {recursive: true}); - await fs.writeFile(outPath, code); - allBuiltFromFiles.add(fileLoc); } } messageBus.emit('WORKER_COMPLETE', {id, error: null}); } stopEsbuild(); + + const webModulesPath = installWorker.args.toUrl; + const installDest = path.join(buildDirectoryLoc, webModulesPath); + const installResult = await installOptimizedDependencies( + (allBuiltJsFiles as any) as [string, string][], + installDest, + commandOptions, + ); + if (!installResult.success || installResult.hasError) { + process.exit(); + } + + const allProxiedFiles = new Set(); + for (const [outLoc, code, fileLoc] of allBuiltJsFiles) { + const resolveImportSpecifier = createImportResolver({ + fileLoc, + webModulesPath, + dependencyImportMap: installResult.importMap, + isDev: false, + isBundled, + config, + }); + const resolvedCode = await transformEsmImports(code, (spec) => { + // Try to resolve the specifier to a known URL in the project + const resolvedImportUrl = resolveImportSpecifier(spec); + if (resolvedImportUrl) { + // We treat ".proxy.js" files special: we need to make sure that they exist on disk + // in the final build, so we mark them to be written to disk at the next step. + if (resolvedImportUrl.endsWith('.proxy.js')) { + allProxiedFiles.add( + resolvedImportUrl.startsWith('/') + ? path.resolve(cwd, spec) + : path.resolve(path.dirname(outLoc), spec), + ); + } + return resolvedImportUrl; + } + return spec; + }); + await fs.mkdir(path.dirname(outLoc), {recursive: true}); + await fs.writeFile(outLoc, resolvedCode); + } + for (const proxiedFileLoc of allProxiedFiles) { const proxiedCode = await fs.readFile(proxiedFileLoc, {encoding: 'utf8'}); const proxiedExt = path.extname(proxiedFileLoc); @@ -354,6 +364,10 @@ export async function command(commandOptions: CommandOptions) { await fs.writeFile(proxyFileLoc, proxyCode, {encoding: 'utf8'}); } + if (installResult.stats) { + console.log(printStats(installResult.stats)); + } + if (!isBundled) { messageBus.emit('WORKER_COMPLETE', {id: bundleWorker.id, error: null}); messageBus.emit('WORKER_UPDATE', { diff --git a/src/config.ts b/src/config.ts index e7bff2f7ed..f0a08ed874 100644 --- a/src/config.ts +++ b/src/config.ts @@ -8,7 +8,7 @@ import path from 'path'; import {Plugin as RollupPlugin} from 'rollup'; import yargs from 'yargs-parser'; import {esbuildPlugin} from './commands/esbuildPlugin'; -import {BUILD_DEPENDENCIES_DIR, DEV_DEPENDENCIES_DIR} from './util'; +import {DEV_DEPENDENCIES_DIR} from './util'; const CONFIG_NAME = 'snowpack'; const ALWAYS_EXCLUDE = ['**/node_modules/**/*', '**/.types/**/*']; @@ -315,9 +315,7 @@ function handleLegacyProxyScripts(config: any) { } type RawScripts = Record; -function normalizeScripts(cwd: string, scripts: RawScripts): BuildScript[] { - const dependenciesLoc = - process.env.NODE_ENV === 'production' ? BUILD_DEPENDENCIES_DIR : DEV_DEPENDENCIES_DIR; +export function normalizeScripts(cwd: string, scripts: RawScripts): BuildScript[] { const processedScripts: BuildScript[] = []; if (Object.keys(scripts).filter((k) => k.startsWith('bundle:')).length > 1) { handleConfigError(`scripts can only contain 1 script of type "bundle:".`); @@ -360,9 +358,9 @@ function normalizeScripts(cwd: string, scripts: RawScripts): BuildScript[] { const dirUrl = to || `/${cmdArr[0]}`; // mount:web_modules is a special case script where the fromDisk - // arg is harcoded to match the internal dependency dir + // arg is hard-coded to match the internal dependency directory. if (scriptId === 'mount:web_modules') { - dirDisk = dependenciesLoc; + dirDisk = DEV_DEPENDENCIES_DIR; } newScriptConfig.args = { @@ -394,7 +392,7 @@ function normalizeScripts(cwd: string, scripts: RawScripts): BuildScript[] { match: ['web_modules'], cmd: `mount $WEB_MODULES --to /web_modules`, args: { - fromDisk: dependenciesLoc, + fromDisk: DEV_DEPENDENCIES_DIR, toUrl: '/web_modules', }, }); diff --git a/src/util.ts b/src/util.ts index f359c73cf4..a80de189aa 100644 --- a/src/util.ts +++ b/src/util.ts @@ -19,7 +19,6 @@ export const BUILD_CACHE = path.join(GLOBAL_CACHE_DIR, 'build-cache-1.4'); export const PROJECT_CACHE_DIR = projectCacheDir({name: 'snowpack'}); export const DEV_DEPENDENCIES_DIR = path.join(PROJECT_CACHE_DIR, 'dev'); -export const BUILD_DEPENDENCIES_DIR = path.join(PROJECT_CACHE_DIR, 'build'); const LOCKFILE_HASH_FILE = '.hash'; export const HAS_CDN_HASH_REGEX = /\-[a-zA-Z0-9]{16,}/;