Add node worker-thread support to jest-worker

.gitignore

@@ -1,3 +1,4 @@
+.idea
 .DS_STORE
 .eslintcache
 *.swp

CHANGELOG.md

@@ -33,6 +33,7 @@
 - `[jest-haste-map]` Accept a `getCacheKey` method in `hasteImplModulePath` modules to reset the cache when the logic changes ([#7350](https://github.com/facebook/jest/pull/7350))
 - `[jest-config]` Add `haste.computeSha1` option to compute the sha-1 of the files in the haste map ([#7345](https://github.com/facebook/jest/pull/7345))
 - `[expect]` `expect(Infinity).toBeCloseTo(Infinity)` Treats `Infinity` as equal in toBeCloseTo matcher ([#7405](https://github.com/facebook/jest/pull/7405))
+- `[jest-worker]` Add node worker-thread support to jest-worker ([#7408](https://github.com/facebook/jest/pull/7408))
 
 ### Fixes
 

packages/jest-resolve/src/isBuiltinModule.js

@@ -15,11 +15,13 @@ declare var process: {
   binding(type: string): {},
 };
 
+const EXPERIMENTAL_MODULES = ['worker_threads'];
+
 const BUILTIN_MODULES =
-  builtinModules ||
-  Object.keys(process.binding('natives')).filter(
-    (module: string) => !/^internal\//.test(module),
-  );
+  builtinModules.concat(EXPERIMENTAL_MODULES) ||
+  Object.keys(process.binding('natives'))
+    .filter((module: string) => !/^internal\//.test(module))
+    .concat(EXPERIMENTAL_MODULES);
 
 export default function isBuiltinModule(module: string): boolean {
   return BUILTIN_MODULES.indexOf(module) !== -1;

packages/jest-worker/README.md

@@ -39,6 +39,12 @@ export function hello(param) {
 }
 ```
 
+## Experimental worker
+
+Node 10 shipped with [worker-threads](https://nodejs.org/api/worker_threads.html), a "threading API" that uses SharedArrayBuffers to communicate between the main process and its child threads. This experimental Node feature can significantly improve the communication time between parent and child processes in `jest-worker`.
+
+We will use worker threads where available. To enable in Node 10+, run the Node process with the `--experimental-worker` flag.
+
 ## API
 
 The only exposed method is a constructor (`Worker`) that is initialized by passing the worker path, plus an options object.
@@ -77,6 +83,12 @@ By default, no process is bound to any worker.
 
 The arguments that will be passed to the `setup` method during initialization.
 
+#### `workerPool: (workerPath: string, options?: WorkerPoolOptions) => WorkerPoolInterface` (optional)
+
+Provide a custom worker pool to be used for spawning child processes. By default, Jest will use a node thread pool if available and fall back to child process threads.
+
+The arguments that will be passed to the `setup` method during initialization.
+
 ## Worker
 
 The returned `Worker` instance has all the exposed methods, plus some additional ones to interact with the workers itself:

packages/jest-worker/src/Farm.js

@@ -0,0 +1,160 @@
+/**
+ * Copyright (c) 2017-present, Facebook, Inc. All rights reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @flow
+ */
+
+'use strict';
+
+import type {
+  ChildMessage,
+  QueueChildMessage,
+  WorkerInterface,
+  OnStart,
+  OnEnd,
+} from './types';
+import {CHILD_MESSAGE_CALL} from './types';
+
+export default class Farm {
+  _computeWorkerKey: (string, ...Array<any>) => ?string;
+  _cacheKeys: {[string]: WorkerInterface, __proto__: null};
+  _callback: Function;
+  _last: Array<QueueChildMessage>;
+  _locks: Array<boolean>;
+  _numOfWorkers: number;
+  _offset: number;
+  _queue: Array<?QueueChildMessage>;
+
+  constructor(
+    numOfWorkers: number,
+    callback: Function,
+    computeWorkerKey?: (string, ...Array<any>) => ?string,
+  ) {
+    this._callback = callback;
+    this._numOfWorkers = numOfWorkers;
+    this._cacheKeys = Object.create(null);
+    this._queue = [];
+    this._last = [];
+    this._locks = [];
+    this._offset = 0;
+    if (computeWorkerKey) {
+      this._computeWorkerKey = computeWorkerKey;
+    }
+  }
+
+  doWork(method: string, ...args: Array<any>): Promise<mixed> {
+    return new Promise((resolve, reject) => {
+      const computeWorkerKey = this._computeWorkerKey;
+      const request: ChildMessage = [CHILD_MESSAGE_CALL, false, method, args];
+
+      let worker: ?WorkerInterface = null;
+      let hash: ?string = null;
+
+      if (computeWorkerKey) {
+        hash = computeWorkerKey.apply(this, [method].concat(args));
+        worker = hash == null ? null : this._cacheKeys[hash];
+      }
+
+      const onStart: OnStart = (worker: WorkerInterface) => {
+        if (hash != null) {
+          this._cacheKeys[hash] = worker;
+        }
+      };
+
+      const onEnd: OnEnd = (error: ?Error, result: ?mixed) => {
+        if (error) {
+          reject(error);
+        } else {
+          resolve(result);
+        }
+      };
+
+      const task = {onEnd, onStart, request};
+      if (worker) {
+        this._enqueue(task, worker.getWorkerId());
+      } else {
+        this._push(task);
+      }
+    });
+  }
+
+  _getNextJob(workerId: number): ?QueueChildMessage {
+    let queueHead = this._queue[workerId];
+
+    while (queueHead && queueHead.request[1]) {
+      queueHead = queueHead.next;
+    }
+
+    this._queue[workerId] = queueHead;
+
+    return queueHead;
+  }
+
+  _process(workerId: number): Farm {
+    if (this.isLocked(workerId)) {
+      return this;
+    }
+
+    const job = this._getNextJob(workerId);
+
+    if (!job) {
+      return this;
+    }
+
+    const onEnd = (error: ?Error, result: mixed) => {
+      job.onEnd(error, result);
+      this.unlock(workerId);
+      this._process(workerId);
+    };
+
+    this.lock(workerId);
+
+    this._callback(workerId, job.request, job.onStart, onEnd);
+
+    job.request[1] = true;
+
+    return this;
+  }
+
+  _enqueue(task: QueueChildMessage, workerId: number): Farm {
+    if (task.request[1]) {
+      return this;
+    }
+
+    if (this._queue[workerId]) {
+      this._last[workerId].next = task;
+    } else {
+      this._queue[workerId] = task;
+    }
+
+    this._last[workerId] = task;
+    this._process(workerId);
+
+    return this;
+  }
+
+  _push(task: QueueChildMessage): Farm {
+    for (let i = 0; i < this._numOfWorkers; i++) {
+      const workerIdx = (this._offset + i) % this._numOfWorkers;
+      this._enqueue(task, workerIdx);
+    }
+    this._offset++;
+
+    return this;
+  }
+
+  lock(workerId: number): void {
+    this._locks[workerId] = true;
+  }
+
+  unlock(workerId: number): void {
+    this._locks[workerId] = false;
+  }
+
+  isLocked(workerId: number): boolean {
+    return this._locks[workerId];
+  }
+}

packages/jest-worker/src/WorkerPool.js

@@ -0,0 +1,55 @@
+/**
+ * Copyright (c) 2017-present, Facebook, Inc. All rights reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @flow
+ */
+
+'use strict';
+
+import BaseWorkerPool from './base/BaseWorkerPool';
+
+import type {
+  ChildMessage,
+  WorkerOptions,
+  OnStart,
+  OnEnd,
+  WorkerPoolInterface,
+  WorkerInterface,
+} from './types';
+
+const canUseWorkerThreads = () => {
+  try {
+    // $FlowFixMe: Flow doesn't know about experimental APIs
+    require('worker_threads');
+    return true;
+  } catch (_) {
+    return false;
+  }
+};
+
+class WorkerPool extends BaseWorkerPool implements WorkerPoolInterface {
+  send(
+    workerId: number,
+    request: ChildMessage,
+    onStart: OnStart,
+    onEnd: OnEnd,
+  ): void {
+    this.getWorkerById(workerId).send(request, onStart, onEnd);
+  }
+
+  createWorker(workerOptions: WorkerOptions): WorkerInterface {
+    let Worker;
+    if (canUseWorkerThreads()) {
+      Worker = require('./workers/NodeThreadsWorker').default;
+    } else {
+      Worker = require('./workers/ChildProcessWorker').default;
+    }
+
+    return new Worker(workerOptions);
+  }
+}
+
+export default WorkerPool;

packages/jest-worker/src/__performance_tests__/test.js

@@ -1,4 +1,9 @@
-// Copyright (c) 2014-present, Facebook, Inc. All rights reserved.
+/**
+ * Copyright (c) 2017-present, Facebook, Inc. All rights reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
 
 'use strict';
 

packages/jest-worker/src/__performance_tests__/workers/jest_worker.js

@@ -1,4 +1,9 @@
-// Copyright (c) 2014-present, Facebook, Inc. All rights reserved.
+/**
+ * Copyright (c) 2017-present, Facebook, Inc. All rights reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
 
 'use strict';
 

packages/jest-worker/src/__performance_tests__/workers/pi.js

@@ -1,4 +1,9 @@
-// Copyright (c) 2014-present, Facebook, Inc. All rights reserved.
+/**
+ * Copyright (c) 2017-present, Facebook, Inc. All rights reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
 
 'use strict';
 

packages/jest-worker/src/__performance_tests__/workers/worker_farm.js

@@ -1,4 +1,9 @@
-// Copyright (c) 2014-present, Facebook, Inc. All rights reserved.
+/**
+ * Copyright (c) 2017-present, Facebook, Inc. All rights reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
 
 'use strict';