feat: add ParallelModulePlugin (#415)

* feat: add parallel module building

* feat: test module parallelization

* feat(parallel): move parallelization to build from create

* chore: organize TransformNormalModulePlugin for webpack 4

* chore: organize worker start and stop in parallel plugin

* chore: iterate forking cli

* fixup! feat: test module parallelization

* chore: add parallel log messages

* chore: clean up parallelization for real use

* chore: extract support: use factoryMeta in place of build meta

* fixup! chore: organize TransformNormalModulePlugin for webpack 4

* chore: export getter for ParallelModulePlugin

* chore: standardize test timeouts at 30s

* chore: lint

* chore: add ParallelModulePlugin to README

README.md

@@ -62,7 +62,11 @@ Some further configuration is possible through provided plugins.
 ```js
   plugins: [
     new HardSourceWebpackPlugin(),
+```
+
+### ExcludeModulePlugin
 
+```js
     // You can optionally exclude items that may not be working with HardSource
     // or items with custom loaders while you are actively developing the
     // loader.
@@ -80,6 +84,30 @@ Some further configuration is possible through provided plugins.
         include: path.join(__dirname, 'vendor'),
       },
     ]),
+```
+
+### ParallelModulePlugin
+
+```js
+    // HardSource includes an experimental plugin for parallelizing webpack
+    // across multiple processes. It requires that the extra processes have the
+    // same configuration. `mode` must be set in the config. Making standard
+    // use with webpack-dev-server or webpack-serve is difficult. Using it with
+    // webpack-dev-server or webpack-serve means disabling any automatic
+    // configuration and configuring hot module replacement support manually.
+    new HardSourceWebpackPlugin.ParallelModulePlugin({
+      // How to launch the extra processes. Default:
+      fork: (fork, compiler, webpackBin) => fork(
+        webpackBin(),
+        ['--config', __filename], {
+          silent: true,
+        }
+      ),
+      // Number of workers to spawn. Default:
+      numWorkers: () => require('os').cpus().length,
+      // Number of modules built before launching parallel building. Default:
+      minModules: 10,
+    }),
   ]
 ```
 

index.js

@@ -577,3 +577,9 @@ HardSourceWebpackPlugin.SerializerAppend2Plugin = SerializerAppend2Plugin;
 HardSourceWebpackPlugin.SerializerAppendPlugin = SerializerAppendPlugin;
 HardSourceWebpackPlugin.SerializerCacachePlugin = SerializerCacachePlugin;
 HardSourceWebpackPlugin.SerializerJsonPlugin = SerializerJsonPlugin;
+
+Object.defineProperty(HardSourceWebpackPlugin, 'ParallelModulePlugin', {
+  get() {
+    return require('./lib/ParallelModulePlugin');
+  },
+});

lib/ParallelModulePlugin.js

@@ -0,0 +1,306 @@
+const { fork: cpFork } = require('child_process');
+const { cpus } = require('os');
+const { resolve } = require('path');
+
+const logMessages = require('./util/log-messages');
+const pluginCompat = require('./util/plugin-compat');
+
+const webpackBin = () => {
+  try {
+    return require.resolve('webpack-cli');
+  } catch (e) {}
+  try {
+    return require.resolve('webpack-command');
+  } catch (e) {}
+  throw new Error('webpack cli tool not installed or discoverable');
+};
+
+const configPath = compiler => {
+  try {
+    return require.resolve(
+      resolve(compiler.options.context || process.cwd(), 'webpack.config'),
+    );
+  } catch (e) {}
+  try {
+    return require.resolve(resolve(process.cwd(), 'webpack.config'));
+  } catch (e) {}
+  throw new Error('config not in obvious location');
+};
+
+class ParallelModulePlugin {
+  constructor(options) {
+    this.options = options;
+  }
+
+  apply(compiler) {
+    try {
+      require('webpack/lib/JavascriptGenerator');
+    } catch (e) {
+      logMessages.parallelRequireWebpack4(compiler);
+      return;
+    }
+
+    const options = this.options || {};
+    const fork =
+      options.fork ||
+      ((fork, compiler, webpackBin) =>
+        fork(webpackBin(compiler), ['--config', configPath(compiler)], {
+          silent: true,
+        }));
+    const numWorkers = options.numWorkers
+      ? typeof options.numWorkers === 'function'
+        ? options.numWorkers
+        : () => options.numWorkers
+      : () => cpus().length;
+    const minModules =
+      typeof options.minModules === 'number' ? options.minModules : 10;
+
+    const compilerHooks = pluginCompat.hooks(compiler);
+
+    let freeze, thaw;
+
+    compilerHooks._hardSourceMethods.tap('ParallelModulePlugin', methods => {
+      freeze = methods.freeze;
+      thaw = methods.thaw;
+    });
+
+    compilerHooks.thisCompilation.tap(
+      'ParallelModulePlugin',
+      (compilation, params) => {
+        const compilationHooks = pluginCompat.hooks(compilation);
+        const nmfHooks = pluginCompat.hooks(params.normalModuleFactory);
+
+        const doMaster = () => {
+          const jobs = {};
+          const readyJobs = {};
+          const workers = [];
+
+          let nextWorkerIndex = 0;
+
+          let start = 0;
+          let started = false;
+          let configMismatch = false;
+
+          let modules = 0;
+
+          const startWorkers = () => {
+            const _numWorkers = numWorkers();
+            logMessages.parallelStartWorkers(compiler, {
+              numWorkers: _numWorkers,
+            });
+
+            for (let i = 0; i < _numWorkers; i++) {
+              const worker = fork(cpFork, compiler, webpackBin);
+              workers.push(worker);
+              worker.on('message', _result => {
+                if (configMismatch) {
+                  return;
+                }
+
+                if (_result.startsWith('ready:')) {
+                  const configHash = _result.split(':')[1];
+                  if (configHash !== compiler.__hardSource_configHash) {
+                    logMessages.parallelConfigMismatch(compiler, {
+                      outHash: compiler.__hardSource_configHash,
+                      theirHash: configHash,
+                    });
+
+                    configMismatch = true;
+                    killWorkers();
+                    for (const id in jobs) {
+                      jobs[id].cb({ error: true });
+                      delete readyJobs[id];
+                      delete jobs[id];
+                    }
+                    return;
+                  }
+                }
+
+                if (Object.values(readyJobs).length) {
+                  const id = Object.keys(readyJobs)[0];
+                  worker.send(
+                    JSON.stringify({
+                      id,
+                      data: readyJobs[id].data,
+                    }),
+                  );
+                  delete readyJobs[id];
+                } else {
+                  worker.ready = true;
+                }
+
+                if (_result.startsWith('ready:')) {
+                  start = Date.now();
+                  return;
+                }
+
+                const result = JSON.parse(_result);
+                jobs[result.id].cb(result);
+                delete [result.id];
+              });
+            }
+          };
+
+          const killWorkers = () => {
+            Object.values(workers).forEach(worker => worker.kill());
+          };
+
+          const doJob = (module, cb) => {
+            if (configMismatch) {
+              cb({ error: new Error('config mismatch') });
+              return;
+            }
+
+            const id = 'xxxxxxxx-xxxxxxxx'.replace(/x/g, () =>
+              Math.random()
+                .toString(16)
+                .substring(2, 3),
+            );
+            jobs[id] = {
+              id,
+              data: freeze('Module', null, module, {
+                id: module.identifier(),
+                compilation,
+              }),
+              cb,
+            };
+
+            const worker = Object.values(workers).find(worker => worker.ready);
+            if (worker) {
+              worker.ready = false;
+              worker.send(
+                JSON.stringify({
+                  id,
+                  data: jobs[id].data,
+                }),
+              );
+            } else {
+              readyJobs[id] = jobs[id];
+            }
+
+            if (!started) {
+              started = true;
+              startWorkers();
+            }
+          };
+
+          const _create = params.normalModuleFactory.create;
+          params.normalModuleFactory.create = (data, cb) => {
+            _create.call(params.normalModuleFactory, data, (err, module) => {
+              if (err) {
+                return cb(err);
+              }
+              if (module.constructor.name === 'NormalModule') {
+                const build = module.build;
+                module.build = (
+                  options,
+                  compilation,
+                  resolver,
+                  fs,
+                  callback,
+                ) => {
+                  if (modules < minModules) {
+                    build.call(
+                      module,
+                      options,
+                      compilation,
+                      resolver,
+                      fs,
+                      callback,
+                    );
+                    modules += 1;
+                    return;
+                  }
+
+                  try {
+                    doJob(module, result => {
+                      if (result.error) {
+                        build.call(
+                          module,
+                          options,
+                          compilation,
+                          resolver,
+                          fs,
+                          callback,
+                        );
+                      } else {
+                        thaw('Module', module, result.module, {
+                          compilation,
+                          normalModuleFactory: params.normalModuleFactory,
+                          contextModuleFactory: params.contextModuleFactory,
+                        });
+                        callback();
+                      }
+                    });
+                  } catch (e) {
+                    logMessages.parallelErrorSendingJob(compiler, e);
+                    build.call(
+                      module,
+                      options,
+                      compilation,
+                      resolver,
+                      fs,
+                      callback,
+                    );
+                  }
+                };
+                cb(null, module);
+              } else {
+                cb(err, module);
+              }
+            });
+          };
+
+          compilationHooks.seal.tap('ParallelModulePlugin', () => {
+            killWorkers();
+          });
+        };
+
+        const doChild = () => {
+          const _create = params.normalModuleFactory.create;
+          params.normalModuleFactory.create = (data, cb) => {};
+
+          process.send('ready:' + compiler.__hardSource_configHash);
+
+          process.on('message', _job => {
+            const job = JSON.parse(_job);
+            const module = thaw('Module', null, job.data, {
+              compilation,
+              normalModuleFactory: params.normalModuleFactory,
+              contextModuleFactory: params.contextModuleFactory,
+            });
+
+            module.build(
+              compilation.options,
+              compilation,
+              compilation.resolverFactory.get('normal', module.resolveOptions),
+              compilation.inputFileSystem,
+              error => {
+                process.send(
+                  JSON.stringify({
+                    id: job.id,
+                    error: error,
+                    module:
+                      module &&
+                      freeze('Module', null, module, {
+                        id: module.identifier(),
+                        compilation,
+                      }),
+                  }),
+                );
+              },
+            );
+          });
+        };
+
+        if (!process.send) {
+          doMaster();
+        } else {
+          doChild();
+        }
+      },
+    );
+  }
+}
+
+module.exports = ParallelModulePlugin;

lib/SupportExtractTextPlugin.js

@@ -20,7 +20,7 @@ class SupportExtractTextPlugin {
         // that assets get built.
         if (
           module[extractTextNS] ||
-          (!module.buildMeta && module.meta && module.meta[extractTextNS])
+          (!module.factoryMeta && module.meta && module.meta[extractTextNS])
         ) {
           return null;
         }