refactor: extract module walker & cache key logic to separate files

src/cache.ts

@@ -1,7 +1,13 @@
-import * as crypto from 'crypto';
-import * as fs from 'fs-extra';
-import * as path from 'path';
-import * as zlib from 'zlib';
+import crypto from 'crypto';
+import debug from 'debug';
+import fs from 'fs-extra';
+import path from 'path';
+import zlib from 'zlib';
+
+const d = debug('electron-rebuild');
+
+// Update this number if you change the caching logic to ensure no bad cache hits
+const ELECTRON_REBUILD_CACHE_ID = 1;
 
 class Snap {
   constructor(public hash: string, public data: Buffer) {}
@@ -11,6 +17,17 @@ interface Snapshot {
   [key: string]: Snap | Snapshot;
 }
 
+type HashTree = { [path: string]: string | HashTree };
+
+type CacheOptions = {
+  ABI: string;
+  arch: string;
+  debug: boolean;
+  electronVersion: string;
+  headerURL: string;
+  modulePath: string;
+};
+
 const takeSnapshot = async (dir: string, relativeTo = dir): Promise<Snapshot> => {
   const snap: Snapshot = {};
   await Promise.all((await fs.readdir(dir)).map(async (child) => {
@@ -99,3 +116,54 @@ export const lookupModuleState = async (cachePath: string, key: string): Promise
   }
   return false;
 };
+
+function dHashTree(tree: HashTree, hash: crypto.Hash): void {
+  for (const key of Object.keys(tree).sort()) {
+    hash.update(key);
+    if (typeof tree[key] === 'string') {
+      hash.update(tree[key] as string);
+    } else {
+      dHashTree(tree[key] as HashTree, hash);
+    }
+  }
+}
+
+async function hashDirectory(dir: string, relativeTo?: string): Promise<HashTree> {
+  relativeTo ??= dir;
+  d('hashing dir', dir);
+  const dirTree: HashTree = {};
+  await Promise.all((await fs.readdir(dir)).map(async (child) => {
+    d('found child', child, 'in dir', dir);
+    // Ignore output directories
+    if (dir === relativeTo && (child === 'build' || child === 'bin')) return;
+    // Don't hash nested node_modules
+    if (child === 'node_modules') return;
+
+    const childPath = path.resolve(dir, child);
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    const relative = path.relative(relativeTo!, childPath);
+    if ((await fs.stat(childPath)).isDirectory()) {
+      dirTree[relative] = await hashDirectory(childPath, relativeTo);
+    } else {
+      dirTree[relative] = crypto.createHash('SHA256').update(await fs.readFile(childPath)).digest('hex');
+    }
+  }));
+
+  return dirTree;
+}
+
+export async function generateCacheKey(opts: CacheOptions): Promise<string> {
+  const tree = await hashDirectory(opts.modulePath);
+  const hasher = crypto.createHash('SHA256')
+    .update(`${ELECTRON_REBUILD_CACHE_ID}`)
+    .update(path.basename(opts.modulePath))
+    .update(opts.ABI)
+    .update(opts.arch)
+    .update(opts.debug ? 'debug' : 'not debug')
+    .update(opts.headerURL)
+    .update(opts.electronVersion);
+  dHashTree(tree, hasher);
+  const hash = hasher.digest('hex');
+  d('calculated hash of', opts.modulePath, 'to be', hash);
+  return hash;
+}

src/cli.ts

@@ -6,9 +6,10 @@ import * as path from 'path';
 import ora = require('ora');
 import yargs from 'yargs/yargs';
 
-import { rebuild, ModuleType } from './rebuild';
 import { getProjectRootPath } from './search-module';
 import { locateElectronModule } from './electron-locator';
+import { ModuleType } from './module-walker';
+import { rebuild } from './rebuild';
 
 const argv = yargs(process.argv.slice(2)).version(false).options({
   version: { alias: 'v', type: 'string', description: 'The version of Electron to build against' },

src/module-rebuilder.ts

@@ -6,18 +6,18 @@ import { cacheModuleState } from './cache';
 import { NodeGyp } from './module-type/node-gyp';
 import { Prebuildify } from './module-type/prebuildify';
 import { PrebuildInstall } from './module-type/prebuild-install';
-import { Rebuilder } from './rebuild';
+import { IRebuilder } from './types';
 
 const d = debug('electron-rebuild');
 
 export class ModuleRebuilder {
   private modulePath: string;
   private nodeGyp: NodeGyp;
-  private rebuilder: Rebuilder;
+  private rebuilder: IRebuilder;
   private prebuildify: Prebuildify;
   private prebuildInstall: PrebuildInstall;
 
-  constructor(rebuilder: Rebuilder, modulePath: string) {
+  constructor(rebuilder: IRebuilder, modulePath: string) {
     this.modulePath = modulePath;
     this.rebuilder = rebuilder;
 
@@ -89,12 +89,13 @@ export class ModuleRebuilder {
     return false;
   }
 
-  async rebuildNodeGypModule(cacheKey: string): Promise<void> {
+  async rebuildNodeGypModule(cacheKey: string): Promise<boolean> {
     await this.nodeGyp.rebuildModule();
     d('built via node-gyp:', this.nodeGyp.moduleName);
     await this.writeMetadata();
     await this.replaceExistingNativeModule();
     await this.cacheModuleState(cacheKey);
+    return true;
   }
 
   async replaceExistingNativeModule(): Promise<void> {
@@ -121,4 +122,10 @@ export class ModuleRebuilder {
   async writeMetadata(): Promise<void> {
     await fs.outputFile(this.metaPath, this.metaData);
   }
+
+  async rebuild(cacheKey: string): Promise<boolean> {
+    return (await this.findPrebuildifyModule(cacheKey)) ||
+      (await this.findPrebuildInstallModule(cacheKey)) ||
+      (await this.rebuildNodeGypModule(cacheKey));
+  }
 }

src/module-type/index.ts

@@ -3,18 +3,18 @@ import path from 'path';
 
 import { NodeAPI } from '../node-api';
 import { readPackageJson } from '../read-package-json';
-import { Rebuilder } from '../rebuild';
+import { IRebuilder } from '../types';
 
 type PackageJSONValue = string | Record<string, unknown>;
 
 export class NativeModule {
-  protected rebuilder: Rebuilder;
+  protected rebuilder: IRebuilder;
   private _moduleName: string | undefined;
   protected modulePath: string
   public nodeAPI: NodeAPI;
   private packageJSON: Record<string, PackageJSONValue | undefined>;
 
-  constructor(rebuilder: Rebuilder, modulePath: string) {
+  constructor(rebuilder: IRebuilder, modulePath: string) {
     this.rebuilder = rebuilder;
     this.modulePath = modulePath;
     this.nodeAPI = new NodeAPI(this.moduleName, this.rebuilder.electronVersion);

src/module-walker.ts

@@ -0,0 +1,149 @@
+import debug from 'debug';
+import fs from 'fs-extra';
+import path from 'path';
+
+import { readPackageJson } from './read-package-json';
+import { searchForModule, searchForNodeModules } from './search-module';
+
+const d = debug('electron-rebuild');
+
+export type ModuleType = 'prod' | 'dev' | 'optional';
+
+export class ModuleWalker {
+  buildPath: string;
+  modulesToRebuild: string[];
+  onlyModules: string[] | null;
+  prodDeps: Set<string>;
+  projectRootPath?: string;
+  realModulePaths: Set<string>;
+  realNodeModulesPaths: Set<string>;
+  types: ModuleType[];
+
+  constructor(buildPath: string, projectRootPath: string | undefined, types: ModuleType[], prodDeps: Set<string>, onlyModules: string[] | null) {
+    this.buildPath = buildPath;
+    this.modulesToRebuild = [];
+    this.projectRootPath = projectRootPath;
+    this.types = types;
+    this.prodDeps = prodDeps;
+    this.onlyModules = onlyModules;
+    this.realModulePaths = new Set();
+    this.realNodeModulesPaths = new Set();
+  }
+
+  get nodeModulesPaths(): Promise<string[]> {
+    return searchForNodeModules(
+      this.buildPath,
+      this.projectRootPath
+    );
+  }
+
+  async walkModules(): Promise<void> {
+    const rootPackageJson = await readPackageJson(this.buildPath);
+    const markWaiters: Promise<void>[] = [];
+    const depKeys = [];
+
+    if (this.types.includes('prod') || this.onlyModules) {
+      depKeys.push(...Object.keys(rootPackageJson.dependencies || {}));
+    }
+    if (this.types.includes('optional') || this.onlyModules) {
+      depKeys.push(...Object.keys(rootPackageJson.optionalDependencies || {}));
+    }
+    if (this.types.includes('dev') || this.onlyModules) {
+      depKeys.push(...Object.keys(rootPackageJson.devDependencies || {}));
+    }
+
+    for (const key of depKeys) {
+      this.prodDeps[key] = true;
+      const modulePaths: string[] = await searchForModule(
+        this.buildPath,
+        key,
+        this.projectRootPath
+      );
+      for (const modulePath of modulePaths) {
+        markWaiters.push(this.markChildrenAsProdDeps(modulePath));
+      }
+    }
+
+    await Promise.all(markWaiters);
+
+    d('identified prod deps:', this.prodDeps);
+  }
+
+  async findModule(moduleName: string, fromDir: string, foundFn: ((p: string) => Promise<void>)): Promise<void[]> {
+
+    const testPaths = await searchForModule(
+      fromDir,
+      moduleName,
+      this.projectRootPath
+    );
+    const foundFns = testPaths.map(testPath => foundFn(testPath));
+
+    return Promise.all(foundFns);
+  }
+
+  async markChildrenAsProdDeps(modulePath: string): Promise<void> {
+    if (!await fs.pathExists(modulePath)) {
+      return;
+    }
+
+    d('exploring', modulePath);
+    let childPackageJson;
+    try {
+      childPackageJson = await readPackageJson(modulePath, true);
+    } catch (err) {
+      return;
+    }
+    const moduleWait: Promise<void[]>[] = [];
+
+    const callback = this.markChildrenAsProdDeps.bind(this);
+    for (const key of Object.keys(childPackageJson.dependencies || {}).concat(Object.keys(childPackageJson.optionalDependencies || {}))) {
+      if (this.prodDeps[key]) {
+        continue;
+      }
+
+      this.prodDeps[key] = true;
+
+      moduleWait.push(this.findModule(key, modulePath, callback));
+    }
+
+    await Promise.all(moduleWait);
+  }
+
+  async findAllModulesIn(nodeModulesPath: string, prefix = ''): Promise<void> {
+    // Some package managers use symbolic links when installing node modules
+    // we need to be sure we've never tested the a package before by resolving
+    // all symlinks in the path and testing against a set
+    const realNodeModulesPath = await fs.realpath(nodeModulesPath);
+    if (this.realNodeModulesPaths.has(realNodeModulesPath)) {
+      return;
+    }
+    this.realNodeModulesPaths.add(realNodeModulesPath);
+
+    d('scanning:', realNodeModulesPath);
+
+    for (const modulePath of await fs.readdir(realNodeModulesPath)) {
+      // Ignore the magical .bin directory
+      if (modulePath === '.bin') continue;
+      // Ensure that we don't mark modules as needing to be rebuilt more than once
+      // by ignoring / resolving symlinks
+      const realPath = await fs.realpath(path.resolve(nodeModulesPath, modulePath));
+
+      if (this.realModulePaths.has(realPath)) {
+        continue;
+      }
+      this.realModulePaths.add(realPath);
+
+      if (this.prodDeps[`${prefix}${modulePath}`] && (!this.onlyModules || this.onlyModules.includes(modulePath))) {
+        this.modulesToRebuild.push(realPath);
+      }
+
+      if (modulePath.startsWith('@')) {
+        await this.findAllModulesIn(realPath, `${modulePath}/`);
+      }
+
+      if (await fs.pathExists(path.resolve(nodeModulesPath, modulePath, 'node_modules'))) {
+        await this.findAllModulesIn(path.resolve(realPath, 'node_modules'));
+      }
+    }
+  }
+}