diff --git a/src/cache.ts b/src/cache.ts index 034387b7..7b9fe403 100644 --- a/src/cache.ts +++ b/src/cache.ts @@ -1,7 +1,13 @@ -import * as crypto from 'crypto'; -import * as fs from 'fs-extra'; -import * as path from 'path'; -import * as zlib from 'zlib'; +import crypto from 'crypto'; +import debug from 'debug'; +import fs from 'fs-extra'; +import path from 'path'; +import zlib from 'zlib'; + +const d = debug('electron-rebuild'); + +// Update this number if you change the caching logic to ensure no bad cache hits +const ELECTRON_REBUILD_CACHE_ID = 1; class Snap { constructor(public hash: string, public data: Buffer) {} @@ -11,6 +17,17 @@ interface Snapshot { [key: string]: Snap | Snapshot; } +type HashTree = { [path: string]: string | HashTree }; + +type CacheOptions = { + ABI: string; + arch: string; + debug: boolean; + electronVersion: string; + headerURL: string; + modulePath: string; +}; + const takeSnapshot = async (dir: string, relativeTo = dir): Promise => { const snap: Snapshot = {}; await Promise.all((await fs.readdir(dir)).map(async (child) => { @@ -99,3 +116,54 @@ export const lookupModuleState = async (cachePath: string, key: string): Promise } return false; }; + +function dHashTree(tree: HashTree, hash: crypto.Hash): void { + for (const key of Object.keys(tree).sort()) { + hash.update(key); + if (typeof tree[key] === 'string') { + hash.update(tree[key] as string); + } else { + dHashTree(tree[key] as HashTree, hash); + } + } +} + +async function hashDirectory(dir: string, relativeTo?: string): Promise { + relativeTo ??= dir; + d('hashing dir', dir); + const dirTree: HashTree = {}; + await Promise.all((await fs.readdir(dir)).map(async (child) => { + d('found child', child, 'in dir', dir); + // Ignore output directories + if (dir === relativeTo && (child === 'build' || child === 'bin')) return; + // Don't hash nested node_modules + if (child === 'node_modules') return; + + const childPath = path.resolve(dir, child); + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const relative = path.relative(relativeTo!, childPath); + if ((await fs.stat(childPath)).isDirectory()) { + dirTree[relative] = await hashDirectory(childPath, relativeTo); + } else { + dirTree[relative] = crypto.createHash('SHA256').update(await fs.readFile(childPath)).digest('hex'); + } + })); + + return dirTree; +} + +export async function generateCacheKey(opts: CacheOptions): Promise { + const tree = await hashDirectory(opts.modulePath); + const hasher = crypto.createHash('SHA256') + .update(`${ELECTRON_REBUILD_CACHE_ID}`) + .update(path.basename(opts.modulePath)) + .update(opts.ABI) + .update(opts.arch) + .update(opts.debug ? 'debug' : 'not debug') + .update(opts.headerURL) + .update(opts.electronVersion); + dHashTree(tree, hasher); + const hash = hasher.digest('hex'); + d('calculated hash of', opts.modulePath, 'to be', hash); + return hash; +} diff --git a/src/cli.ts b/src/cli.ts index fc87c2c5..473ae612 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -6,9 +6,10 @@ import * as path from 'path'; import ora = require('ora'); import yargs from 'yargs/yargs'; -import { rebuild, ModuleType } from './rebuild'; import { getProjectRootPath } from './search-module'; import { locateElectronModule } from './electron-locator'; +import { ModuleType } from './module-walker'; +import { rebuild } from './rebuild'; const argv = yargs(process.argv.slice(2)).version(false).options({ version: { alias: 'v', type: 'string', description: 'The version of Electron to build against' }, diff --git a/src/module-rebuilder.ts b/src/module-rebuilder.ts index 66298cbc..b525110a 100644 --- a/src/module-rebuilder.ts +++ b/src/module-rebuilder.ts @@ -6,18 +6,18 @@ import { cacheModuleState } from './cache'; import { NodeGyp } from './module-type/node-gyp'; import { Prebuildify } from './module-type/prebuildify'; import { PrebuildInstall } from './module-type/prebuild-install'; -import { Rebuilder } from './rebuild'; +import { IRebuilder } from './types'; const d = debug('electron-rebuild'); export class ModuleRebuilder { private modulePath: string; private nodeGyp: NodeGyp; - private rebuilder: Rebuilder; + private rebuilder: IRebuilder; private prebuildify: Prebuildify; private prebuildInstall: PrebuildInstall; - constructor(rebuilder: Rebuilder, modulePath: string) { + constructor(rebuilder: IRebuilder, modulePath: string) { this.modulePath = modulePath; this.rebuilder = rebuilder; @@ -89,12 +89,13 @@ export class ModuleRebuilder { return false; } - async rebuildNodeGypModule(cacheKey: string): Promise { + async rebuildNodeGypModule(cacheKey: string): Promise { await this.nodeGyp.rebuildModule(); d('built via node-gyp:', this.nodeGyp.moduleName); await this.writeMetadata(); await this.replaceExistingNativeModule(); await this.cacheModuleState(cacheKey); + return true; } async replaceExistingNativeModule(): Promise { @@ -121,4 +122,10 @@ export class ModuleRebuilder { async writeMetadata(): Promise { await fs.outputFile(this.metaPath, this.metaData); } + + async rebuild(cacheKey: string): Promise { + return (await this.findPrebuildifyModule(cacheKey)) || + (await this.findPrebuildInstallModule(cacheKey)) || + (await this.rebuildNodeGypModule(cacheKey)); + } } diff --git a/src/module-type/index.ts b/src/module-type/index.ts index 420ae17b..bdd83c48 100644 --- a/src/module-type/index.ts +++ b/src/module-type/index.ts @@ -3,18 +3,18 @@ import path from 'path'; import { NodeAPI } from '../node-api'; import { readPackageJson } from '../read-package-json'; -import { Rebuilder } from '../rebuild'; +import { IRebuilder } from '../types'; type PackageJSONValue = string | Record; export class NativeModule { - protected rebuilder: Rebuilder; + protected rebuilder: IRebuilder; private _moduleName: string | undefined; protected modulePath: string public nodeAPI: NodeAPI; private packageJSON: Record; - constructor(rebuilder: Rebuilder, modulePath: string) { + constructor(rebuilder: IRebuilder, modulePath: string) { this.rebuilder = rebuilder; this.modulePath = modulePath; this.nodeAPI = new NodeAPI(this.moduleName, this.rebuilder.electronVersion); diff --git a/src/module-walker.ts b/src/module-walker.ts new file mode 100644 index 00000000..d03485c8 --- /dev/null +++ b/src/module-walker.ts @@ -0,0 +1,149 @@ +import debug from 'debug'; +import fs from 'fs-extra'; +import path from 'path'; + +import { readPackageJson } from './read-package-json'; +import { searchForModule, searchForNodeModules } from './search-module'; + +const d = debug('electron-rebuild'); + +export type ModuleType = 'prod' | 'dev' | 'optional'; + +export class ModuleWalker { + buildPath: string; + modulesToRebuild: string[]; + onlyModules: string[] | null; + prodDeps: Set; + projectRootPath?: string; + realModulePaths: Set; + realNodeModulesPaths: Set; + types: ModuleType[]; + + constructor(buildPath: string, projectRootPath: string | undefined, types: ModuleType[], prodDeps: Set, onlyModules: string[] | null) { + this.buildPath = buildPath; + this.modulesToRebuild = []; + this.projectRootPath = projectRootPath; + this.types = types; + this.prodDeps = prodDeps; + this.onlyModules = onlyModules; + this.realModulePaths = new Set(); + this.realNodeModulesPaths = new Set(); + } + + get nodeModulesPaths(): Promise { + return searchForNodeModules( + this.buildPath, + this.projectRootPath + ); + } + + async walkModules(): Promise { + const rootPackageJson = await readPackageJson(this.buildPath); + const markWaiters: Promise[] = []; + const depKeys = []; + + if (this.types.includes('prod') || this.onlyModules) { + depKeys.push(...Object.keys(rootPackageJson.dependencies || {})); + } + if (this.types.includes('optional') || this.onlyModules) { + depKeys.push(...Object.keys(rootPackageJson.optionalDependencies || {})); + } + if (this.types.includes('dev') || this.onlyModules) { + depKeys.push(...Object.keys(rootPackageJson.devDependencies || {})); + } + + for (const key of depKeys) { + this.prodDeps[key] = true; + const modulePaths: string[] = await searchForModule( + this.buildPath, + key, + this.projectRootPath + ); + for (const modulePath of modulePaths) { + markWaiters.push(this.markChildrenAsProdDeps(modulePath)); + } + } + + await Promise.all(markWaiters); + + d('identified prod deps:', this.prodDeps); + } + + async findModule(moduleName: string, fromDir: string, foundFn: ((p: string) => Promise)): Promise { + + const testPaths = await searchForModule( + fromDir, + moduleName, + this.projectRootPath + ); + const foundFns = testPaths.map(testPath => foundFn(testPath)); + + return Promise.all(foundFns); + } + + async markChildrenAsProdDeps(modulePath: string): Promise { + if (!await fs.pathExists(modulePath)) { + return; + } + + d('exploring', modulePath); + let childPackageJson; + try { + childPackageJson = await readPackageJson(modulePath, true); + } catch (err) { + return; + } + const moduleWait: Promise[] = []; + + const callback = this.markChildrenAsProdDeps.bind(this); + for (const key of Object.keys(childPackageJson.dependencies || {}).concat(Object.keys(childPackageJson.optionalDependencies || {}))) { + if (this.prodDeps[key]) { + continue; + } + + this.prodDeps[key] = true; + + moduleWait.push(this.findModule(key, modulePath, callback)); + } + + await Promise.all(moduleWait); + } + + async findAllModulesIn(nodeModulesPath: string, prefix = ''): Promise { + // Some package managers use symbolic links when installing node modules + // we need to be sure we've never tested the a package before by resolving + // all symlinks in the path and testing against a set + const realNodeModulesPath = await fs.realpath(nodeModulesPath); + if (this.realNodeModulesPaths.has(realNodeModulesPath)) { + return; + } + this.realNodeModulesPaths.add(realNodeModulesPath); + + d('scanning:', realNodeModulesPath); + + for (const modulePath of await fs.readdir(realNodeModulesPath)) { + // Ignore the magical .bin directory + if (modulePath === '.bin') continue; + // Ensure that we don't mark modules as needing to be rebuilt more than once + // by ignoring / resolving symlinks + const realPath = await fs.realpath(path.resolve(nodeModulesPath, modulePath)); + + if (this.realModulePaths.has(realPath)) { + continue; + } + this.realModulePaths.add(realPath); + + if (this.prodDeps[`${prefix}${modulePath}`] && (!this.onlyModules || this.onlyModules.includes(modulePath))) { + this.modulesToRebuild.push(realPath); + } + + if (modulePath.startsWith('@')) { + await this.findAllModulesIn(realPath, `${modulePath}/`); + } + + if (await fs.pathExists(path.resolve(nodeModulesPath, modulePath, 'node_modules'))) { + await this.findAllModulesIn(path.resolve(realPath, 'node_modules')); + } + } + } +} diff --git a/src/rebuild.ts b/src/rebuild.ts index 56cfc37e..58a54d14 100644 --- a/src/rebuild.ts +++ b/src/rebuild.ts @@ -1,4 +1,3 @@ -import * as crypto from 'crypto'; import debug from 'debug'; import { EventEmitter } from 'events'; import * as fs from 'fs-extra'; @@ -6,12 +5,10 @@ import * as nodeAbi from 'node-abi'; import * as os from 'os'; import * as path from 'path'; -import { readPackageJson } from './read-package-json'; -import { lookupModuleState } from './cache'; -import { searchForModule, searchForNodeModules } from './search-module'; - -export type ModuleType = 'prod' | 'dev' | 'optional'; -export type RebuildMode = 'sequential' | 'parallel'; +import { generateCacheKey, lookupModuleState } from './cache'; +import { BuildType, IRebuilder, RebuildMode } from './types'; +import { ModuleRebuilder } from './module-rebuilder'; +import { ModuleType, ModuleWalker } from './module-walker'; export interface RebuildOptions { buildPath: string; @@ -33,48 +30,33 @@ export interface RebuildOptions { disablePreGypCopy?: boolean; } -export type HashTree = { [path: string]: string | HashTree }; - export interface RebuilderOptions extends RebuildOptions { lifecycle: EventEmitter; } -export enum BuildType { - Debug = 'Debug', - Release = 'Release', -} - const d = debug('electron-rebuild'); const defaultMode: RebuildMode = 'sequential'; const defaultTypes: ModuleType[] = ['prod', 'optional']; -// Update this number if you change the caching logic to ensure no bad cache hits -const ELECTRON_REBUILD_CACHE_ID = 1; -export class Rebuilder { +export class Rebuilder implements IRebuilder { private ABIVersion: string | undefined; + private moduleWalker: ModuleWalker; nodeGypPath: string; - prodDeps: Set; rebuilds: (() => Promise)[]; - realModulePaths: Set; - realNodeModulesPaths: Set; public lifecycle: EventEmitter; public buildPath: string; public electronVersion: string; public platform: string = process.platform; public arch: string; - public extraModules: string[]; - public onlyModules: string[] | null; public force: boolean; public headerURL: string; - public types: ModuleType[]; public mode: RebuildMode; public debug: boolean; public useCache: boolean; public cachePath: string; public prebuildTagPrefix: string; - public projectRootPath?: string; public msvsVersion?: string; public useElectronClang: boolean; public disablePreGypCopy: boolean; @@ -84,11 +66,8 @@ export class Rebuilder { this.buildPath = options.buildPath; this.electronVersion = options.electronVersion; this.arch = options.arch || process.arch; - this.extraModules = options.extraModules || []; - this.onlyModules = options.onlyModules || null; this.force = options.force || false; this.headerURL = options.headerURL || 'https://www.electronjs.org/headers'; - this.types = options.types || defaultTypes; this.mode = options.mode || defaultMode; this.debug = options.debug || false; this.useCache = options.useCache || false; @@ -102,7 +81,6 @@ export class Rebuilder { console.warn('[WARNING]: Electron Rebuild has force enabled and cache enabled, force take precedence and the cache will not be used.'); this.useCache = false; } - this.projectRootPath = options.projectRootPath; if (typeof this.electronVersion === 'number') { if (`${this.electronVersion}`.split('.').length === 1) { @@ -116,10 +94,29 @@ export class Rebuilder { } this.ABIVersion = options.forceABI?.toString(); - this.prodDeps = this.extraModules.reduce((acc: Set, x: string) => acc.add(x), new Set()); + const onlyModules = options.onlyModules || null; + const extraModules = (options.extraModules || []).reduce((acc: Set, x: string) => acc.add(x), new Set()); + const types = options.types || defaultTypes; + this.moduleWalker = new ModuleWalker( + this.buildPath, + options.projectRootPath, + types, + extraModules, + onlyModules, + ); this.rebuilds = []; - this.realModulePaths = new Set(); - this.realNodeModulesPaths = new Set(); + + d( + 'rebuilding with args:', + this.buildPath, + this.electronVersion, + this.arch, + extraModules, + this.force, + this.headerURL, + types, + this.debug + ); } get ABI(): string { @@ -139,56 +136,17 @@ export class Rebuilder { if (!path.isAbsolute(this.buildPath)) { throw new Error('Expected buildPath to be an absolute path'); } - d( - 'rebuilding with args:', - this.buildPath, - this.electronVersion, - this.arch, - this.extraModules, - this.force, - this.headerURL, - this.types, - this.debug - ); this.lifecycle.emit('start'); - const rootPackageJson = await readPackageJson(this.buildPath); - const markWaiters: Promise[] = []; - const depKeys = []; + await this.moduleWalker.walkModules(); - if (this.types.indexOf('prod') !== -1 || this.onlyModules) { - depKeys.push(...Object.keys(rootPackageJson.dependencies || {})); - } - if (this.types.indexOf('optional') !== -1 || this.onlyModules) { - depKeys.push(...Object.keys(rootPackageJson.optionalDependencies || {})); - } - if (this.types.indexOf('dev') !== -1 || this.onlyModules) { - depKeys.push(...Object.keys(rootPackageJson.devDependencies || {})); + for (const nodeModulesPath of await this.moduleWalker.nodeModulesPaths) { + await this.moduleWalker.findAllModulesIn(nodeModulesPath); } - for (const key of depKeys) { - this.prodDeps[key] = true; - const modulePaths: string[] = await searchForModule( - this.buildPath, - key, - this.projectRootPath - ); - for (const modulePath of modulePaths) { - markWaiters.push(this.markChildrenAsProdDeps(modulePath)); - } - } - - await Promise.all(markWaiters); - - d('identified prod deps:', this.prodDeps); - - const nodeModulesPaths = await searchForNodeModules( - this.buildPath, - this.projectRootPath - ); - for (const nodeModulesPath of nodeModulesPaths) { - await this.rebuildAllModulesIn(nodeModulesPath); + for (const modulePath of this.moduleWalker.modulesToRebuild) { + this.rebuilds.push(() => this.rebuildModuleAt(modulePath)); } this.rebuilds.push(() => this.rebuildModuleAt(this.buildPath)); @@ -202,61 +160,11 @@ export class Rebuilder { } } - private hashDirectory = async (dir: string, relativeTo = dir): Promise => { - d('hashing dir', dir); - const dirTree: HashTree = {}; - await Promise.all((await fs.readdir(dir)).map(async (child) => { - d('found child', child, 'in dir', dir); - // Ignore output directories - if (dir === relativeTo && (child === 'build' || child === 'bin')) return; - // Don't hash nested node_modules - if (child === 'node_modules') return; - - const childPath = path.resolve(dir, child); - const relative = path.relative(relativeTo, childPath); - if ((await fs.stat(childPath)).isDirectory()) { - dirTree[relative] = await this.hashDirectory(childPath, relativeTo); - } else { - dirTree[relative] = crypto.createHash('SHA256').update(await fs.readFile(childPath)).digest('hex'); - } - })); - return dirTree; - } - - private dHashTree = (tree: HashTree, hash: crypto.Hash): void => { - for (const key of Object.keys(tree).sort()) { - hash.update(key); - if (typeof tree[key] === 'string') { - hash.update(tree[key] as string); - } else { - this.dHashTree(tree[key] as HashTree, hash); - } - } - } - - private generateCacheKey = async (opts: { modulePath: string }): Promise => { - const tree = await this.hashDirectory(opts.modulePath); - const hasher = crypto.createHash('SHA256') - .update(`${ELECTRON_REBUILD_CACHE_ID}`) - .update(path.basename(opts.modulePath)) - .update(this.ABI) - .update(this.arch) - .update(this.debug ? 'debug' : 'not debug') - .update(this.headerURL) - .update(this.electronVersion); - this.dHashTree(tree, hasher); - const hash = hasher.digest('hex'); - d('calculated hash of', opts.modulePath, 'to be', hash); - return hash; - } - async rebuildModuleAt(modulePath: string): Promise { if (!(await fs.pathExists(path.resolve(modulePath, 'binding.gyp')))) { return; } - // eslint-disable-next-line @typescript-eslint/no-var-requires - const { ModuleRebuilder } = require('./module-rebuilder'); const moduleRebuilder = new ModuleRebuilder(this, modulePath); this.lifecycle.emit('module-found', path.basename(modulePath)); @@ -268,14 +176,19 @@ export class Rebuilder { return; } - if (await moduleRebuilder.prebuildInstallNativeModuleExists(modulePath)) { + if (await moduleRebuilder.prebuildInstallNativeModuleExists()) { d(`skipping: ${path.basename(modulePath)} as it was prebuilt`); return; } let cacheKey!: string; if (this.useCache) { - cacheKey = await this.generateCacheKey({ + cacheKey = await generateCacheKey({ + ABI: this.ABI, + arch: this.arch, + debug: this.debug, + electronVersion: this.electronVersion, + headerURL: this.headerURL, modulePath, }); @@ -287,95 +200,9 @@ export class Rebuilder { } } - if (await moduleRebuilder.findPrebuildifyModule(cacheKey)) { + if (await moduleRebuilder.rebuild(cacheKey)) { this.lifecycle.emit('module-done'); - return; } - - if (await moduleRebuilder.findPrebuildInstallModule(cacheKey)) { - this.lifecycle.emit('module-done'); - return; - } - await moduleRebuilder.rebuildNodeGypModule(cacheKey); - this.lifecycle.emit('module-done'); - } - - async rebuildAllModulesIn(nodeModulesPath: string, prefix = ''): Promise { - // Some package managers use symbolic links when installing node modules - // we need to be sure we've never tested the a package before by resolving - // all symlinks in the path and testing against a set - const realNodeModulesPath = await fs.realpath(nodeModulesPath); - if (this.realNodeModulesPaths.has(realNodeModulesPath)) { - return; - } - this.realNodeModulesPaths.add(realNodeModulesPath); - - d('scanning:', realNodeModulesPath); - - for (const modulePath of await fs.readdir(realNodeModulesPath)) { - // Ignore the magical .bin directory - if (modulePath === '.bin') continue; - // Ensure that we don't mark modules as needing to be rebuilt more than once - // by ignoring / resolving symlinks - const realPath = await fs.realpath(path.resolve(nodeModulesPath, modulePath)); - - if (this.realModulePaths.has(realPath)) { - continue; - } - this.realModulePaths.add(realPath); - - if (this.prodDeps[`${prefix}${modulePath}`] && (!this.onlyModules || this.onlyModules.includes(modulePath))) { - this.rebuilds.push(() => this.rebuildModuleAt(realPath)); - } - - if (modulePath.startsWith('@')) { - await this.rebuildAllModulesIn(realPath, `${modulePath}/`); - } - - if (await fs.pathExists(path.resolve(nodeModulesPath, modulePath, 'node_modules'))) { - await this.rebuildAllModulesIn(path.resolve(realPath, 'node_modules')); - } - } - } - - async findModule(moduleName: string, fromDir: string, foundFn: ((p: string) => Promise)): Promise { - - const testPaths = await searchForModule( - fromDir, - moduleName, - this.projectRootPath - ); - const foundFns = testPaths.map(testPath => foundFn(testPath)); - - return Promise.all(foundFns); - } - - async markChildrenAsProdDeps(modulePath: string): Promise { - if (!await fs.pathExists(modulePath)) { - return; - } - - d('exploring', modulePath); - let childPackageJson; - try { - childPackageJson = await readPackageJson(modulePath, true); - } catch (err) { - return; - } - const moduleWait: Promise[] = []; - - const callback = this.markChildrenAsProdDeps.bind(this); - for (const key of Object.keys(childPackageJson.dependencies || {}).concat(Object.keys(childPackageJson.optionalDependencies || {}))) { - if (this.prodDeps[key]) { - continue; - } - - this.prodDeps[key] = true; - - moduleWait.push(this.findModule(key, modulePath, callback)); - } - - await Promise.all(moduleWait); } } diff --git a/src/types.ts b/src/types.ts new file mode 100644 index 00000000..59fa5c92 --- /dev/null +++ b/src/types.ts @@ -0,0 +1,28 @@ +import { EventEmitter } from 'events'; + +export enum BuildType { + Debug = 'Debug', + Release = 'Release', +} + +export type RebuildMode = 'sequential' | 'parallel'; + +export interface IRebuilder { + ABI: string; + arch: string; + buildPath: string; + buildType: BuildType; + cachePath: string; + debug: boolean; + disablePreGypCopy: boolean; + electronVersion: string; + force: boolean; + headerURL: string; + lifecycle: EventEmitter; + mode: RebuildMode; + msvsVersion?: string; + platform: string; + prebuildTagPrefix: string; + useCache: boolean; + useElectronClang: boolean; +}