From 418c6f9ae6874e99b87d40b5fe14182088416179 Mon Sep 17 00:00:00 2001 From: Alexey Kureev Date: Mon, 9 Jul 2018 14:27:49 +0100 Subject: [PATCH 01/35] Restructure workers (create a base class and inherit from it) Clean up types, create missing interfaes and make Flow and Jest happy --- .../src/{worker.js => ChildProcessWorker.js} | 117 +--------- packages/jest-worker/src/NodeThreadsWorker.js | 54 +++++ packages/jest-worker/src/WorkerPool.js | 33 +++ ...ker.test.js => ChildProcessWorker.test.js} | 2 +- .../jest-worker/src/__tests__/index.test.js | 4 +- packages/jest-worker/src/base/BaseWorker.js | 123 +++++++++++ .../jest-worker/src/base/BaseWorkerPool.js | 107 +++++++++ packages/jest-worker/src/child.js | 17 +- packages/jest-worker/src/index.js | 204 +++++++----------- packages/jest-worker/src/types.js | 34 ++- 10 files changed, 450 insertions(+), 245 deletions(-) rename packages/jest-worker/src/{worker.js => ChildProcessWorker.js} (54%) create mode 100644 packages/jest-worker/src/NodeThreadsWorker.js create mode 100644 packages/jest-worker/src/WorkerPool.js rename packages/jest-worker/src/__tests__/{worker.test.js => ChildProcessWorker.test.js} (99%) create mode 100644 packages/jest-worker/src/base/BaseWorker.js create mode 100644 packages/jest-worker/src/base/BaseWorkerPool.js diff --git a/packages/jest-worker/src/worker.js b/packages/jest-worker/src/ChildProcessWorker.js similarity index 54% rename from packages/jest-worker/src/worker.js rename to packages/jest-worker/src/ChildProcessWorker.js index 5eee64af241e..cf7e413b2670 100644 --- a/packages/jest-worker/src/worker.js +++ b/packages/jest-worker/src/ChildProcessWorker.js @@ -11,22 +11,14 @@ import childProcess from 'child_process'; -import { - CHILD_MESSAGE_INITIALIZE, - PARENT_MESSAGE_ERROR, - PARENT_MESSAGE_OK, -} from './types'; +import BaseWorker from './base/BaseWorker'; + +import {PARENT_MESSAGE_ERROR, CHILD_MESSAGE_INITIALIZE} from './types'; import type {ChildProcess} from 'child_process'; import type {Readable} from 'stream'; -import type { - ChildMessage, - QueueChildMessage, - OnProcessEnd, - OnProcessStart, - WorkerOptions, -} from './types'; +import type {ChildMessage, OnEnd, OnStart, WorkerOptions} from './types'; /** * This class wraps the child process and provides a nice interface to @@ -46,19 +38,15 @@ import type { * field is changed to "true", so that other workers which might encounter the * same call skip it. */ -export default class { - _busy: boolean; +export default class ChildProcessWorker extends BaseWorker { _child: ChildProcess; - _last: ?QueueChildMessage; - _options: WorkerOptions; - _queue: ?QueueChildMessage; - _retries: number; constructor(options: WorkerOptions) { + super(); this._options = options; this._queue = null; - this._initialize(); + this.initialize(); } getStdout(): Readable { @@ -69,11 +57,7 @@ export default class { return this._child.stderr; } - send( - request: ChildMessage, - onProcessStart: OnProcessStart, - onProcessEnd: OnProcessEnd, - ) { + send(request: ChildMessage, onProcessStart: OnStart, onProcessEnd: OnEnd) { const item = {next: null, onProcessEnd, onProcessStart, request}; if (this._last) { @@ -86,7 +70,7 @@ export default class { this._process(); } - _initialize() { + initialize() { const child = childProcess.fork( require.resolve('./child'), // $FlowFixMe: Flow does not work well with Object.assign. @@ -129,87 +113,4 @@ export default class { ]); } } - - _process() { - if (this._busy) { - return; - } - - let item = this._queue; - - // Calls in the queue might have already been processed by another worker, - // so we have to skip them. - while (item && item.request[1]) { - item = item.next; - } - - this._queue = item; - - if (item) { - // Flag the call as processed, so that other workers know that they don't - // have to process it as well. - item.request[1] = true; - - // Tell the parent that this item is starting to be processed. - item.onProcessStart(this); - - this._retries = 0; - this._busy = true; - - // $FlowFixMe: wrong "ChildProcess.send" signature. - this._child.send(item.request); - } else { - this._last = item; - } - } - - _receive(response: any /* Should be ParentMessage */) { - const item = this._queue; - - if (!item) { - throw new TypeError('Unexpected response with an empty queue'); - } - - const onProcessEnd = item.onProcessEnd; - - this._busy = false; - this._process(); - - switch (response[0]) { - case PARENT_MESSAGE_OK: - onProcessEnd(null, response[1]); - break; - - case PARENT_MESSAGE_ERROR: - let error = response[4]; - - if (error != null && typeof error === 'object') { - const extra = error; - const NativeCtor = global[response[1]]; - const Ctor = typeof NativeCtor === 'function' ? NativeCtor : Error; - - error = new Ctor(response[2]); - // $FlowFixMe: adding custom properties to errors. - error.type = response[1]; - error.stack = response[3]; - - for (const key in extra) { - // $FlowFixMe: adding custom properties to errors. - error[key] = extra[key]; - } - } - - onProcessEnd(error, null); - break; - - default: - throw new TypeError('Unexpected response from worker: ' + response[0]); - } - } - - _exit(exitCode: number) { - if (exitCode !== 0) { - this._initialize(); - } - } } diff --git a/packages/jest-worker/src/NodeThreadsWorker.js b/packages/jest-worker/src/NodeThreadsWorker.js new file mode 100644 index 000000000000..eed83ac9ae4b --- /dev/null +++ b/packages/jest-worker/src/NodeThreadsWorker.js @@ -0,0 +1,54 @@ +/** + * Copyright (c) 2017-present, Facebook, Inc. All rights reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @flow + */ + +'use strict'; + +import BaseWorker from './base/BaseWorker'; + +import type {WorkerOptions} from './types'; + +export default class ExpirementalWorker extends BaseWorker { + _options: WorkerOptions; + + constructor(options: WorkerOptions) { + super(); + this._options = options; + this._queue = null; + + this.initialize(); + } + + initialize() { + // $FlowFixMe: Flow doesn't know about experimental features of Node + const {Worker} = require('worker_threads'); + + const worker = new Worker('./child', { + eval: false, + stderr: true, + stdout: true, + + // $FlowFixMe: Flow does not work well with Object.assign. + workerData: Object.assign( + { + cwd: process.cwd(), + env: Object.assign({}, process.env, { + JEST_WORKER_ID: this._options.workerId, + }), + // Suppress --debug / --inspect flags while preserving others (like --harmony). + execArgv: process.execArgv.filter(v => !/^--(debug|inspect)/.test(v)), + silent: true, + }, + this._options.forkOptions, + ), + }); + + worker.on('message', this._receive.bind(this)); + worker.on('exit', this._exit.bind(this)); + } +} diff --git a/packages/jest-worker/src/WorkerPool.js b/packages/jest-worker/src/WorkerPool.js new file mode 100644 index 000000000000..61a148b4ffe2 --- /dev/null +++ b/packages/jest-worker/src/WorkerPool.js @@ -0,0 +1,33 @@ +/** + * Copyright (c) 2017-present, Facebook, Inc. All rights reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @flow + */ + +'use strict'; + +import BaseWorkerPool from './base/BaseWorkerPool'; + +import type { + ChildMessage, + OnStart, + OnEnd, + WorkerInterface, + WorkerPool as WorkerPoolInterface, +} from './types'; + +class WorkerPool extends BaseWorkerPool implements WorkerPoolInterface { + send( + worker: WorkerInterface, + request: ChildMessage, + onStart: OnStart, + onEnd: OnEnd, + ): void { + worker.send(request, onStart, onEnd); + } +} + +export default WorkerPool; diff --git a/packages/jest-worker/src/__tests__/worker.test.js b/packages/jest-worker/src/__tests__/ChildProcessWorker.test.js similarity index 99% rename from packages/jest-worker/src/__tests__/worker.test.js rename to packages/jest-worker/src/__tests__/ChildProcessWorker.test.js index dd4ad947d86e..f21e22e2eee6 100644 --- a/packages/jest-worker/src/__tests__/worker.test.js +++ b/packages/jest-worker/src/__tests__/ChildProcessWorker.test.js @@ -38,7 +38,7 @@ beforeEach(() => { return forkInterface; }); - Worker = require('../worker').default; + Worker = require('../ChildProcessWorker').default; }); afterEach(() => { diff --git a/packages/jest-worker/src/__tests__/index.test.js b/packages/jest-worker/src/__tests__/index.test.js index 74e84c9f0f28..13a389320331 100644 --- a/packages/jest-worker/src/__tests__/index.test.js +++ b/packages/jest-worker/src/__tests__/index.test.js @@ -30,7 +30,7 @@ beforeEach(() => { // The worker mock returns a worker with custom methods, plus it stores them // in a global list, so that they can be accessed later. This list is reset in // every test. - jest.mock('../worker', () => { + jest.mock('../ChildProcessWorker', () => { const fakeClass = jest.fn(() => { const fakeWorker = { getStderr: () => ({once() {}, pipe() {}}), @@ -63,7 +63,7 @@ beforeEach(() => { virtual: true, }); - Worker = require('../worker').default; + Worker = require('../ChildProcessWorker').default; Farm = require('../index').default; }); diff --git a/packages/jest-worker/src/base/BaseWorker.js b/packages/jest-worker/src/base/BaseWorker.js new file mode 100644 index 000000000000..86c707c669c2 --- /dev/null +++ b/packages/jest-worker/src/base/BaseWorker.js @@ -0,0 +1,123 @@ +/** + * Copyright (c) 2017-present, Facebook, Inc. All rights reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @flow + */ + +'use strict'; + +import {PARENT_MESSAGE_ERROR, PARENT_MESSAGE_OK} from '../types'; + +import type {Readable} from 'stream'; + +import type {ChildMessage, QueueChildMessage, WorkerOptions} from '../types'; + +export default class BaseWorker { + _busy: boolean; + _child: any; + _last: ?QueueChildMessage; + _options: WorkerOptions; + _queue: ?QueueChildMessage; + _retries: number; + + getStdout(): Readable { + return this._child.stdout; + } + + getStderr(): Readable { + return this._child.stderr; + } + + initialize() { + throw Error('Initializer is required for custom Worker implementations'); + } + + send(message: ChildMessage, onStart: Function, onEnd: Function): void { + throw Error('"send" method is required for custom Worker implementations'); + } + + _process() { + if (this._busy) { + return; + } + + let item = this._queue; + + // Calls in the queue might have already been processed by another worker, + // so we have to skip them. + while (item && item.request[1]) { + item = item.next; + } + + this._queue = item; + + if (item) { + // Flag the call as processed, so that other workers know that they don't + // have to process it as well. + item.request[1] = true; + + // Tell the parent that this item is starting to be processed. + item.onProcessStart(this); + + this._retries = 0; + this._busy = true; + + this._child.send(item.request); + } else { + this._last = item; + } + } + + _receive(response: any /* Should be ParentMessage */) { + const item = this._queue; + + if (!item) { + throw new TypeError('Unexpected response with an empty queue'); + } + + const onProcessEnd = item.onProcessEnd; + + this._busy = false; + this._process(); + + switch (response[0]) { + case PARENT_MESSAGE_OK: + onProcessEnd(null, response[1]); + break; + + case PARENT_MESSAGE_ERROR: + let error = response[4]; + + if (error != null && typeof error === 'object') { + const extra = error; + const NativeCtor = global[response[1]]; + const Ctor = typeof NativeCtor === 'function' ? NativeCtor : Error; + + error = new Ctor(response[2]); + // $FlowFixMe: adding custom properties to errors. + error.type = response[1]; + error.stack = response[3]; + + for (const key in extra) { + // $FlowFixMe: adding custom properties to errors. + error[key] = extra[key]; + } + } + + onProcessEnd(error, null); + break; + + default: + throw new TypeError('Unexpected response from worker: ' + response[0]); + } + } + + _exit(exitCode: number) { + if (exitCode !== 0) { + this.initialize(); + } + } +} diff --git a/packages/jest-worker/src/base/BaseWorkerPool.js b/packages/jest-worker/src/base/BaseWorkerPool.js new file mode 100644 index 000000000000..ab3e67931808 --- /dev/null +++ b/packages/jest-worker/src/base/BaseWorkerPool.js @@ -0,0 +1,107 @@ +/** + * Copyright (c) 2017-present, Facebook, Inc. All rights reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @flow + */ + +'use strict'; + +import mergeStream from 'merge-stream'; +import os from 'os'; +import path from 'path'; + +import Worker from '../ChildProcessWorker'; +import {CHILD_MESSAGE_END} from '../types'; + +import type {Readable} from 'stream'; +import type {FarmOptions, WorkerOptions, WorkerInterface} from '../types'; + +/* istanbul ignore next */ +const emptyMethod = () => {}; + +export default class BaseWorkerPool { + _stderr: Readable; + _stdout: Readable; + _options: FarmOptions; + _workers: Array; + + constructor(workerPath: string, options: FarmOptions) { + this._options = options; + + const numWorkers = options.numWorkers || os.cpus().length - 1; + this._workers = new Array(numWorkers); + + if (!path.isAbsolute(workerPath)) { + workerPath = require.resolve(workerPath); + } + + const stdout = mergeStream(); + const stderr = mergeStream(); + + for (let i = 0; i < numWorkers; i++) { + const workerOptions: WorkerOptions = { + forkOptions: options.forkOptions || {}, + maxRetries: options.maxRetries || 3, + workerId: i + 1, + workerPath, + }; + + const worker = this.createWorker(workerOptions); + const workerStdout = worker.getStdout(); + const workerStderr = worker.getStderr(); + + if (workerStdout) { + stdout.add(workerStdout); + } + + if (workerStderr) { + stderr.add(workerStderr); + } + + this._workers[i] = worker; + + // $FlowFixMe + this.getStderr = this.getStderr.bind(this); + // $FlowFixMe + this.getStdout = this.getStdout.bind(this); + // $FlowFixMe + this.getWorkers = this.getWorkers.bind(this); + // $FlowFixMe + this.end = this.end.bind(this); + } + + this._stdout = stdout; + this._stderr = stderr; + } + + getStderr(): Readable { + return this._stderr; + } + + getStdout(): Readable { + return this._stdout; + } + + getWorkers(): Array { + return this._workers; + } + + createWorker(workerOptions: WorkerOptions): Worker { + return new Worker(workerOptions); + } + + end(): void { + // We do not cache the request object here. If so, it would only be only + // processed by one of the workers, and we want them all to close. + for (let i = 0; i < this._workers.length; i++) { + this._workers[i].send( + [CHILD_MESSAGE_END, false], + emptyMethod, + emptyMethod, + ); + } + } +} diff --git a/packages/jest-worker/src/child.js b/packages/jest-worker/src/child.js index dc372bf9a088..a7a543ec7330 100644 --- a/packages/jest-worker/src/child.js +++ b/packages/jest-worker/src/child.js @@ -54,11 +54,20 @@ process.on('message', (request: any /* Should be ChildMessage */) => { }); function reportSuccess(result: any) { - if (!process || !process.send) { - throw new Error('Child can only be used on a forked process'); - } + try { + // $FlowFixMe: Flow doesn't know about experimental features of Node + const {isMainThread, parentPort} = require('worker_threads'); + if (!isMainThread) { + throw Error("Worker can't be used in the main thread"); + } + parentPort.postMessage([PARENT_MESSAGE_OK, result]); + } catch (_) { + if (!process || !process.send) { + throw new Error('Child can only be used on a forked process'); + } - process.send([PARENT_MESSAGE_OK, result]); + process.send([PARENT_MESSAGE_OK, result]); + } } function reportError(error: Error) { diff --git a/packages/jest-worker/src/index.js b/packages/jest-worker/src/index.js index f661e65aa667..9bf99896fa65 100644 --- a/packages/jest-worker/src/index.js +++ b/packages/jest-worker/src/index.js @@ -9,18 +9,38 @@ 'use strict'; -import mergeStream from 'merge-stream'; -import os from 'os'; -import path from 'path'; - -import type {FarmOptions} from './types'; +import type { + WorkerPool as WorkerPoolInterface, + FarmOptions, + ChildMessage, +} from './types'; import type {Readable} from 'stream'; -import {CHILD_MESSAGE_CALL, CHILD_MESSAGE_END} from './types'; -import Worker from './worker'; +import {CHILD_MESSAGE_CALL, WorkerInterface} from './types'; +import WorkerPool from './WorkerPool'; + +function getExposedMethods( + workerPath: string, + options?: FarmOptions = {}, +): $ReadOnlyArray { + let exposedMethods = options.exposedMethods; + + // If no methods list is given, try getting it by auto-requiring the module. + if (!exposedMethods) { + // $FlowFixMe: This has to be a dynamic require. + const module: Function | Object = require(workerPath); + + exposedMethods = Object.keys(module).filter( + name => typeof module[name] === 'function', + ); -/* istanbul ignore next */ -const emptyMethod = () => {}; + if (typeof module === 'function') { + exposedMethods.push('default'); + } + } + + return exposedMethods; +} /** * The Jest farm (publicly called "Worker") is a class that allows you to queue @@ -29,7 +49,7 @@ const emptyMethod = () => {}; * of the child processes, and bridged to the main process. * * Bridged methods are specified by using the "exposedMethods" property of the - * options "object". This is an array of strings, where each of them corresponds + * "options" object. This is an array of strings, where each of them corresponds * to the exported name in the loaded module. * * You can also control the amount of workers by using the "numWorkers" property @@ -47,67 +67,26 @@ const emptyMethod = () => {}; * processed by the same worker. This is specially useful if your workers are * caching results. */ -export default class { - _stdout: Readable; - _stderr: Readable; +export default class JestWorker { + _cacheKeys: {[string]: WorkerInterface, __proto__: null}; _ending: boolean; - _cacheKeys: {[string]: Worker, __proto__: null}; - _options: FarmOptions; - _workers: Array; _offset: number; + _options: FarmOptions; + _threadPool: WorkerPoolInterface; constructor(workerPath: string, options?: FarmOptions = {}) { - const numWorkers = options.numWorkers || os.cpus().length - 1; - const workers = new Array(numWorkers); - const stdout = mergeStream(); - const stderr = mergeStream(); - - if (!path.isAbsolute(workerPath)) { - workerPath = require.resolve(workerPath); - } - - const sharedWorkerOptions = { - forkOptions: options.forkOptions || {}, - maxRetries: options.maxRetries || 3, - workerPath, - }; - - for (let i = 0; i < numWorkers; i++) { - const workerOptions = Object.assign({}, sharedWorkerOptions, { - workerId: i + 1, - }); - const worker = new Worker(workerOptions); - const workerStdout = worker.getStdout(); - const workerStderr = worker.getStderr(); - - if (workerStdout) { - stdout.add(workerStdout); - } - - if (workerStderr) { - stderr.add(workerStderr); - } - - workers[i] = worker; - } - - let exposedMethods = options.exposedMethods; - - // If no methods list is given, try getting it by auto-requiring the module. - if (!exposedMethods) { - // $FlowFixMe: This has to be a dynamic require. - const child = require(workerPath); - - exposedMethods = Object.keys(child).filter( - name => typeof child[name] === 'function', - ); + this._cacheKeys = Object.create(null); + this._offset = 0; + this._options = options; + this._threadPool = options.WorkerPool + ? new options.WorkerPool(workerPath, options) + : new WorkerPool(workerPath, options); - if (typeof child === 'function') { - exposedMethods.push('default'); - } - } + this._bindExposedWorkerMethods(workerPath, options); + } - exposedMethods.forEach(name => { + _bindExposedWorkerMethods(workerPath: string, options?: FarmOptions): void { + getExposedMethods(workerPath, options).forEach(name => { if (name.startsWith('_')) { return; } @@ -117,72 +96,35 @@ export default class { } // $FlowFixMe: dynamic extension of the class instance is expected. - this[name] = this._makeCall.bind(this, name); + this[name] = this._callFunctionWithArgs.bind(this, name); }); - - this._stdout = stdout; - this._stderr = stderr; - this._ending = false; - this._cacheKeys = Object.create(null); - this._options = options; - this._workers = workers; - this._offset = 0; - } - - getStdout(): Readable { - return this._stdout; - } - - getStderr(): Readable { - return this._stderr; - } - - end() { - if (this._ending) { - throw new Error('Farm is ended, no more calls can be done to it'); - } - - const workers = this._workers; - - // We do not cache the request object here. If so, it would only be only - // processed by one of the workers, and we want them all to close. - for (let i = 0; i < workers.length; i++) { - workers[i].send([CHILD_MESSAGE_END, false], emptyMethod, emptyMethod); - } - - this._ending = true; } // eslint-disable-next-line no-unclear-flowtypes - _makeCall(method: string, ...args: Array): Promise { + _callFunctionWithArgs(method: string, ...args: Array): Promise { if (this._ending) { throw new Error('Farm is ended, no more calls can be done to it'); } return new Promise((resolve, reject) => { const {computeWorkerKey} = this._options; - const workers = this._workers; - const length = workers.length; - const cacheKeys = this._cacheKeys; - const request = [CHILD_MESSAGE_CALL, false, method, args]; + const request: ChildMessage = [CHILD_MESSAGE_CALL, false, method, args]; - let worker = null; - let hash = null; + let worker: ?WorkerInterface = null; + let hash: ?string = null; if (computeWorkerKey) { hash = computeWorkerKey.apply(this, [method].concat(args)); - worker = hash == null ? null : cacheKeys[hash]; + worker = hash == null ? null : this._cacheKeys[hash]; } - // Do not use a fat arrow since we need the "this" value, which points to - // the worker that executed the call. - const onProcessStart = worker => { + const onStart: onStart = (worker: WorkerInterface) => { if (hash != null) { - cacheKeys[hash] = worker; + this._cacheKeys[hash] = worker; } }; - const onProcessEnd = (error, result) => { + const onEnd: onEnd = (error: Error, result: mixed) => { if (error) { reject(error); } else { @@ -190,22 +132,36 @@ export default class { } }; - // If a worker is pre-selected, use it... if (worker) { - worker.send(request, onProcessStart, onProcessEnd); - return; + this._threadPool.send(worker, request, onStart, onEnd); + } else { + const workers = this._threadPool.getWorkers(); + const length = workers.length; + + for (let i = 0; i < length; i++) { + const worker = workers[(i + this._offset) % length]; + this._threadPool.send(worker, request, onStart, onEnd); + } + this._offset++; } + }); + } - // ... otherwise use all workers, so the first one available will pick it. - for (let i = 0; i < length; i++) { - workers[(i + this._offset) % length].send( - request, - onProcessStart, - onProcessEnd, - ); - } + getStderr(): Readable { + return this._threadPool.getStderr(); + } - this._offset++; - }); + getStdout(): Readable { + return this._threadPool.getStdout(); + } + + end(): void { + if (this._ending) { + throw new Error('Farm is ended, no more calls can be done to it'); + } + + this._threadPool.end(); + + this._ending = true; } } diff --git a/packages/jest-worker/src/types.js b/packages/jest-worker/src/types.js index f2ca164cff54..b0acf5efa8ab 100644 --- a/packages/jest-worker/src/types.js +++ b/packages/jest-worker/src/types.js @@ -24,7 +24,7 @@ export const PARENT_MESSAGE_ERROR: 1 = 1; // Option objects. -import type Worker from './worker'; +import type {Readable} from 'stream'; export type ForkOptions = { cwd?: string, @@ -37,12 +37,34 @@ export type ForkOptions = { gid?: number, }; +export interface WorkerPool { + _options: FarmOptions; + _stderr: Readable; + _stdout: Readable; + _workers: Array; + getStderr(): Readable; + getStdout(): Readable; + getWorkers(): Array; + createWorker(WorkerOptions): WorkerInterface; + send(WorkerInterface, ChildMessage, Function, Function): void; + end(): void; +} + +export interface WorkerInterface { + send(ChildMessage, Function, Function): void; + getStderr(): Readable; + getStdout(): Readable; + _exit(number): void; +} + export type FarmOptions = { computeWorkerKey?: (string, ...Array) => ?string, exposedMethods?: $ReadOnlyArray, forkOptions?: ForkOptions, maxRetries?: number, numWorkers?: number, + WorkerPool?: (workerPath: string, options?: FarmOptions) => WorkerPool, + useNodeWorkersIfPossible?: boolean, }; export type WorkerOptions = {| @@ -50,6 +72,7 @@ export type WorkerOptions = {| maxRetries: number, workerId: number, workerPath: string, + useNodeWorkersIfPossible?: boolean, |}; // Messages passed from the parent to the children. @@ -95,13 +118,12 @@ export type ParentMessageError = [ export type ParentMessage = ParentMessageOk | ParentMessageError; // Queue types. - -export type OnProcessStart = Worker => void; -export type OnProcessEnd = (?Error, ?any) => void; +export type OnStart = WorkerInterface => void; +export type OnEnd = (?Error, ?any) => void; export type QueueChildMessage = {| request: ChildMessage, - onProcessStart: OnProcessStart, - onProcessEnd: OnProcessEnd, + onProcessStart: OnStart, + onProcessEnd: OnEnd, next: ?QueueChildMessage, |}; From 100c49f54477029cbe9721c84b48626a9bdabd33 Mon Sep 17 00:00:00 2001 From: Alexey Kureev Date: Mon, 9 Jul 2018 14:42:09 +0100 Subject: [PATCH 02/35] Move child.js to workers folder --- jest-worker | 0 .../src/__tests__/ChildProcessWorker.test.js | 4 ++-- .../jest-worker/src/__tests__/child.test.js | 8 ++++---- .../jest-worker/src/__tests__/index.test.js | 6 +++--- .../jest-worker/src/base/BaseWorkerPool.js | 2 +- .../src/{ => workers}/ChildProcessWorker.js | 6 +++--- .../src/{ => workers}/NodeThreadsWorker.js | 11 +++++++++-- .../jest-worker/src/{ => workers}/child.js | 19 +++++-------------- 8 files changed, 27 insertions(+), 29 deletions(-) create mode 100644 jest-worker rename packages/jest-worker/src/{ => workers}/ChildProcessWorker.js (96%) rename packages/jest-worker/src/{ => workers}/NodeThreadsWorker.js (77%) rename packages/jest-worker/src/{ => workers}/child.js (83%) diff --git a/jest-worker b/jest-worker new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/packages/jest-worker/src/__tests__/ChildProcessWorker.test.js b/packages/jest-worker/src/__tests__/ChildProcessWorker.test.js index f21e22e2eee6..7f11a20ebb32 100644 --- a/packages/jest-worker/src/__tests__/ChildProcessWorker.test.js +++ b/packages/jest-worker/src/__tests__/ChildProcessWorker.test.js @@ -38,7 +38,7 @@ beforeEach(() => { return forkInterface; }); - Worker = require('../ChildProcessWorker').default; + Worker = require('../workers/ChildProcessWorker').default; }); afterEach(() => { @@ -47,7 +47,7 @@ afterEach(() => { }); it('passes fork options down to child_process.fork, adding the defaults', () => { - const child = require.resolve('../child'); + const child = require.resolve('../workers/child'); process.execArgv = ['--inspect', '-p']; diff --git a/packages/jest-worker/src/__tests__/child.test.js b/packages/jest-worker/src/__tests__/child.test.js index 6bd8e6e54af1..c6f5a1540436 100644 --- a/packages/jest-worker/src/__tests__/child.test.js +++ b/packages/jest-worker/src/__tests__/child.test.js @@ -27,7 +27,7 @@ beforeEach(() => { mockCount = 0; jest.mock( - '../my-fancy-worker', + '../workers/my-fancy-worker', () => { mockCount++; @@ -74,7 +74,7 @@ beforeEach(() => { ); jest.mock( - '../my-fancy-standalone-worker', + '../workers/my-fancy-standalone-worker', () => jest.fn().mockImplementation(() => 12345), {virtual: true}, ); @@ -82,7 +82,7 @@ beforeEach(() => { // This mock emulates a transpiled Babel module that carries a default export // that corresponds to a method. jest.mock( - '../my-fancy-babel-worker', + '../workers/my-fancy-babel-worker', () => ({ __esModule: true, default: jest.fn().mockImplementation(() => 67890), @@ -94,7 +94,7 @@ beforeEach(() => { process.send = jest.fn(); // Require the child! - require('../child'); + require('../workers/child'); }); afterEach(() => { diff --git a/packages/jest-worker/src/__tests__/index.test.js b/packages/jest-worker/src/__tests__/index.test.js index 13a389320331..18bf16743dc9 100644 --- a/packages/jest-worker/src/__tests__/index.test.js +++ b/packages/jest-worker/src/__tests__/index.test.js @@ -30,7 +30,7 @@ beforeEach(() => { // The worker mock returns a worker with custom methods, plus it stores them // in a global list, so that they can be accessed later. This list is reset in // every test. - jest.mock('../ChildProcessWorker', () => { + jest.mock('../workers/ChildProcessWorker', () => { const fakeClass = jest.fn(() => { const fakeWorker = { getStderr: () => ({once() {}, pipe() {}}), @@ -63,8 +63,8 @@ beforeEach(() => { virtual: true, }); - Worker = require('../ChildProcessWorker').default; - Farm = require('../index').default; + Worker = require('../workers/ChildProcessWorker').default; + Farm = require('..').default; }); afterEach(() => { diff --git a/packages/jest-worker/src/base/BaseWorkerPool.js b/packages/jest-worker/src/base/BaseWorkerPool.js index ab3e67931808..6155daa9b788 100644 --- a/packages/jest-worker/src/base/BaseWorkerPool.js +++ b/packages/jest-worker/src/base/BaseWorkerPool.js @@ -13,7 +13,7 @@ import mergeStream from 'merge-stream'; import os from 'os'; import path from 'path'; -import Worker from '../ChildProcessWorker'; +import Worker from '../workers/ChildProcessWorker'; import {CHILD_MESSAGE_END} from '../types'; import type {Readable} from 'stream'; diff --git a/packages/jest-worker/src/ChildProcessWorker.js b/packages/jest-worker/src/workers/ChildProcessWorker.js similarity index 96% rename from packages/jest-worker/src/ChildProcessWorker.js rename to packages/jest-worker/src/workers/ChildProcessWorker.js index cf7e413b2670..2f70cb94ab13 100644 --- a/packages/jest-worker/src/ChildProcessWorker.js +++ b/packages/jest-worker/src/workers/ChildProcessWorker.js @@ -11,14 +11,14 @@ import childProcess from 'child_process'; -import BaseWorker from './base/BaseWorker'; +import BaseWorker from '../base/BaseWorker'; -import {PARENT_MESSAGE_ERROR, CHILD_MESSAGE_INITIALIZE} from './types'; +import {PARENT_MESSAGE_ERROR, CHILD_MESSAGE_INITIALIZE} from '../types'; import type {ChildProcess} from 'child_process'; import type {Readable} from 'stream'; -import type {ChildMessage, OnEnd, OnStart, WorkerOptions} from './types'; +import type {ChildMessage, OnEnd, OnStart, WorkerOptions} from '../types'; /** * This class wraps the child process and provides a nice interface to diff --git a/packages/jest-worker/src/NodeThreadsWorker.js b/packages/jest-worker/src/workers/NodeThreadsWorker.js similarity index 77% rename from packages/jest-worker/src/NodeThreadsWorker.js rename to packages/jest-worker/src/workers/NodeThreadsWorker.js index eed83ac9ae4b..8789f514641a 100644 --- a/packages/jest-worker/src/NodeThreadsWorker.js +++ b/packages/jest-worker/src/workers/NodeThreadsWorker.js @@ -9,9 +9,9 @@ 'use strict'; -import BaseWorker from './base/BaseWorker'; +import BaseWorker from '../base/BaseWorker'; -import type {WorkerOptions} from './types'; +import type {WorkerOptions} from '../types'; export default class ExpirementalWorker extends BaseWorker { _options: WorkerOptions; @@ -24,6 +24,13 @@ export default class ExpirementalWorker extends BaseWorker { this.initialize(); } + // $FlowFixMe: Flow doesn't know about experimental features of Node + // const {isMainThread, parentPort} = require('worker_threads'); + // if (!isMainThread) { + // throw Error("Worker can't be used in the main thread"); + // } + // parentPort.postMessage([PARENT_MESSAGE_OK, result]); + initialize() { // $FlowFixMe: Flow doesn't know about experimental features of Node const {Worker} = require('worker_threads'); diff --git a/packages/jest-worker/src/child.js b/packages/jest-worker/src/workers/child.js similarity index 83% rename from packages/jest-worker/src/child.js rename to packages/jest-worker/src/workers/child.js index a7a543ec7330..9eb66f0293b4 100644 --- a/packages/jest-worker/src/child.js +++ b/packages/jest-worker/src/workers/child.js @@ -15,7 +15,7 @@ import { CHILD_MESSAGE_INITIALIZE, PARENT_MESSAGE_ERROR, PARENT_MESSAGE_OK, -} from './types'; +} from '../types'; let file = null; @@ -54,20 +54,11 @@ process.on('message', (request: any /* Should be ChildMessage */) => { }); function reportSuccess(result: any) { - try { - // $FlowFixMe: Flow doesn't know about experimental features of Node - const {isMainThread, parentPort} = require('worker_threads'); - if (!isMainThread) { - throw Error("Worker can't be used in the main thread"); - } - parentPort.postMessage([PARENT_MESSAGE_OK, result]); - } catch (_) { - if (!process || !process.send) { - throw new Error('Child can only be used on a forked process'); - } - - process.send([PARENT_MESSAGE_OK, result]); + if (!process || !process.send) { + throw new Error('Child can only be used on a forked process'); } + + process.send([PARENT_MESSAGE_OK, result]); } function reportError(error: Error) { From f786cb6142212c5dea11a9376199d5114011465b Mon Sep 17 00:00:00 2001 From: Alexey Kureev Date: Mon, 9 Jul 2018 18:25:32 +0100 Subject: [PATCH 03/35] Restructure base classes and make a working POC --- packages/jest-worker/src/base/BaseWorker.js | 87 ----------- packages/jest-worker/src/types.js | 12 ++ .../src/workers/ChildProcessWorker.js | 110 ++++++++++++-- .../src/workers/NodeThreadsWorker.js | 137 ++++++++++++++++-- packages/jest-worker/src/workers/child.js | 14 +- .../jest-worker/src/workers/threadChild.js | 115 +++++++++++++++ 6 files changed, 358 insertions(+), 117 deletions(-) create mode 100644 packages/jest-worker/src/workers/threadChild.js diff --git a/packages/jest-worker/src/base/BaseWorker.js b/packages/jest-worker/src/base/BaseWorker.js index 86c707c669c2..80818becfb2c 100644 --- a/packages/jest-worker/src/base/BaseWorker.js +++ b/packages/jest-worker/src/base/BaseWorker.js @@ -9,28 +9,17 @@ 'use strict'; -import {PARENT_MESSAGE_ERROR, PARENT_MESSAGE_OK} from '../types'; - import type {Readable} from 'stream'; import type {ChildMessage, QueueChildMessage, WorkerOptions} from '../types'; export default class BaseWorker { _busy: boolean; - _child: any; _last: ?QueueChildMessage; _options: WorkerOptions; _queue: ?QueueChildMessage; _retries: number; - getStdout(): Readable { - return this._child.stdout; - } - - getStderr(): Readable { - return this._child.stderr; - } - initialize() { throw Error('Initializer is required for custom Worker implementations'); } @@ -39,82 +28,6 @@ export default class BaseWorker { throw Error('"send" method is required for custom Worker implementations'); } - _process() { - if (this._busy) { - return; - } - - let item = this._queue; - - // Calls in the queue might have already been processed by another worker, - // so we have to skip them. - while (item && item.request[1]) { - item = item.next; - } - - this._queue = item; - - if (item) { - // Flag the call as processed, so that other workers know that they don't - // have to process it as well. - item.request[1] = true; - - // Tell the parent that this item is starting to be processed. - item.onProcessStart(this); - - this._retries = 0; - this._busy = true; - - this._child.send(item.request); - } else { - this._last = item; - } - } - - _receive(response: any /* Should be ParentMessage */) { - const item = this._queue; - - if (!item) { - throw new TypeError('Unexpected response with an empty queue'); - } - - const onProcessEnd = item.onProcessEnd; - - this._busy = false; - this._process(); - - switch (response[0]) { - case PARENT_MESSAGE_OK: - onProcessEnd(null, response[1]); - break; - - case PARENT_MESSAGE_ERROR: - let error = response[4]; - - if (error != null && typeof error === 'object') { - const extra = error; - const NativeCtor = global[response[1]]; - const Ctor = typeof NativeCtor === 'function' ? NativeCtor : Error; - - error = new Ctor(response[2]); - // $FlowFixMe: adding custom properties to errors. - error.type = response[1]; - error.stack = response[3]; - - for (const key in extra) { - // $FlowFixMe: adding custom properties to errors. - error[key] = extra[key]; - } - } - - onProcessEnd(error, null); - break; - - default: - throw new TypeError('Unexpected response from worker: ' + response[0]); - } - } - _exit(exitCode: number) { if (exitCode !== 0) { this.initialize(); diff --git a/packages/jest-worker/src/types.js b/packages/jest-worker/src/types.js index b0acf5efa8ab..2d4d60e365fc 100644 --- a/packages/jest-worker/src/types.js +++ b/packages/jest-worker/src/types.js @@ -25,6 +25,7 @@ export const PARENT_MESSAGE_ERROR: 1 = 1; // Option objects. import type {Readable} from 'stream'; +const EventEmitter = require('events'); export type ForkOptions = { cwd?: string, @@ -77,10 +78,21 @@ export type WorkerOptions = {| // Messages passed from the parent to the children. +export type MessagePort = { + ...typeof EventEmitter, + postMessage(any): void, +}; + +export type MessageChannel = { + port1: MessagePort, + port2: MessagePort, +}; + export type ChildMessageInitialize = [ typeof CHILD_MESSAGE_INITIALIZE, // type boolean, // processed string, // file + ?MessagePort, // MessagePort ]; export type ChildMessageCall = [ diff --git a/packages/jest-worker/src/workers/ChildProcessWorker.js b/packages/jest-worker/src/workers/ChildProcessWorker.js index 2f70cb94ab13..b8a400461d2e 100644 --- a/packages/jest-worker/src/workers/ChildProcessWorker.js +++ b/packages/jest-worker/src/workers/ChildProcessWorker.js @@ -13,7 +13,12 @@ import childProcess from 'child_process'; import BaseWorker from '../base/BaseWorker'; -import {PARENT_MESSAGE_ERROR, CHILD_MESSAGE_INITIALIZE} from '../types'; +import { + PARENT_MESSAGE_ERROR, + CHILD_MESSAGE_INITIALIZE, + PARENT_MESSAGE_OK, + WorkerInterface, +} from '../types'; import type {ChildProcess} from 'child_process'; import type {Readable} from 'stream'; @@ -38,7 +43,7 @@ import type {ChildMessage, OnEnd, OnStart, WorkerOptions} from '../types'; * field is changed to "true", so that other workers which might encounter the * same call skip it. */ -export default class ChildProcessWorker extends BaseWorker { +export default class ChildProcessWorker extends BaseWorker implements WorkerInterface { _child: ChildProcess; constructor(options: WorkerOptions) { @@ -49,25 +54,81 @@ export default class ChildProcessWorker extends BaseWorker { this.initialize(); } - getStdout(): Readable { - return this._child.stdout; - } + _process() { + if (this._busy) { + return; + } - getStderr(): Readable { - return this._child.stderr; - } + let item = this._queue; - send(request: ChildMessage, onProcessStart: OnStart, onProcessEnd: OnEnd) { - const item = {next: null, onProcessEnd, onProcessStart, request}; + // Calls in the queue might have already been processed by another worker, + // so we have to skip them. + while (item && item.request[1]) { + item = item.next; + } - if (this._last) { - this._last.next = item; + this._queue = item; + + if (item) { + // Flag the call as processed, so that other workers know that they don't + // have to process it as well. + item.request[1] = true; + + // Tell the parent that this item is starting to be processed. + item.onProcessStart(this); + + this._retries = 0; + this._busy = true; + + // $FlowFixMe: wrong "ChildProcess.send" signature. + this._child.send(item.request); } else { - this._queue = item; + this._last = item; } + } - this._last = item; + _receive(response: any /* Should be ParentMessage */) { + const item = this._queue; + + if (!item) { + throw new TypeError('Unexpected response with an empty queue'); + } + + const onProcessEnd = item.onProcessEnd; + + this._busy = false; this._process(); + + switch (response[0]) { + case PARENT_MESSAGE_OK: + onProcessEnd(null, response[1]); + break; + + case PARENT_MESSAGE_ERROR: + let error = response[4]; + + if (error != null && typeof error === 'object') { + const extra = error; + const NativeCtor = global[response[1]]; + const Ctor = typeof NativeCtor === 'function' ? NativeCtor : Error; + + error = new Ctor(response[2]); + // $FlowFixMe: adding custom properties to errors. + error.type = response[1]; + error.stack = response[3]; + + for (const key in extra) { + // $FlowFixMe: adding custom properties to errors. + error[key] = extra[key]; + } + } + + onProcessEnd(error, null); + break; + + default: + throw new TypeError('Unexpected response from worker: ' + response[0]); + } } initialize() { @@ -113,4 +174,25 @@ export default class ChildProcessWorker extends BaseWorker { ]); } } + + send(request: ChildMessage, onProcessStart: OnStart, onProcessEnd: OnEnd) { + const item = {next: null, onProcessEnd, onProcessStart, request}; + + if (this._last) { + this._last.next = item; + } else { + this._queue = item; + } + + this._last = item; + this._process(); + } + + getStdout(): Readable { + return this._child.stdout; + } + + getStderr(): Readable { + return this._child.stderr; + } } diff --git a/packages/jest-worker/src/workers/NodeThreadsWorker.js b/packages/jest-worker/src/workers/NodeThreadsWorker.js index 8789f514641a..21634143d11f 100644 --- a/packages/jest-worker/src/workers/NodeThreadsWorker.js +++ b/packages/jest-worker/src/workers/NodeThreadsWorker.js @@ -11,10 +11,21 @@ import BaseWorker from '../base/BaseWorker'; -import type {WorkerOptions} from '../types'; +import { + CHILD_MESSAGE_INITIALIZE, + PARENT_MESSAGE_OK, + PARENT_MESSAGE_ERROR, +} from '../types'; + +import type {Readable} from 'stream'; +import type {ChildMessage, OnEnd, OnStart, WorkerOptions} from '../types'; + +// $FlowFixMe: Flow doesn't know about experimental features of Node +const {Worker, MessageChannel} = require('worker_threads'); export default class ExpirementalWorker extends BaseWorker { _options: WorkerOptions; + _worker: Worker; constructor(options: WorkerOptions) { super(); @@ -24,18 +35,90 @@ export default class ExpirementalWorker extends BaseWorker { this.initialize(); } - // $FlowFixMe: Flow doesn't know about experimental features of Node - // const {isMainThread, parentPort} = require('worker_threads'); - // if (!isMainThread) { - // throw Error("Worker can't be used in the main thread"); - // } - // parentPort.postMessage([PARENT_MESSAGE_OK, result]); + _process() { + if (this._busy) { + return; + } - initialize() { - // $FlowFixMe: Flow doesn't know about experimental features of Node - const {Worker} = require('worker_threads'); + let item = this._queue; + + // Calls in the queue might have already been processed by another worker, + // so we have to skip them. + while (item && item.request[1]) { + item = item.next; + } + + this._queue = item; + + if (item) { + // Flag the call as processed, so that other workers know that they don't + // have to process it as well. + item.request[1] = true; + + // Tell the parent that this item is starting to be processed. + item.onProcessStart(this); + + this._retries = 0; + this._busy = true; + + if (!this._worker) { + throw Error( + "Can't process the request without having an active worker", + ); + } + + this._worker.postMessage(item.request); + } else { + this._last = item; + } + } + + _onMessage(response: any /* Should be ParentMessage */) { + const item = this._queue; + + if (!item) { + throw new TypeError('Unexpected response with an empty queue'); + } + + const onProcessEnd = item.onProcessEnd; + + this._busy = false; + this._process(); + + switch (response[0]) { + case PARENT_MESSAGE_OK: + onProcessEnd(null, response[1]); + break; + + case PARENT_MESSAGE_ERROR: + let error = response[4]; - const worker = new Worker('./child', { + if (error != null && typeof error === 'object') { + const extra = error; + const NativeCtor = global[response[1]]; + const Ctor = typeof NativeCtor === 'function' ? NativeCtor : Error; + + error = new Ctor(response[2]); + // $FlowFixMe: adding custom properties to errors. + error.type = response[1]; + error.stack = response[3]; + + for (const key in extra) { + // $FlowFixMe: adding custom properties to errors. + error[key] = extra[key]; + } + } + + onProcessEnd(error, null); + break; + + default: + throw new TypeError('Unexpected response from worker: ' + response[0]); + } + } + + initialize() { + this._worker = new Worker(__dirname + '/threadChild.js', { eval: false, stderr: true, stdout: true, @@ -55,7 +138,35 @@ export default class ExpirementalWorker extends BaseWorker { ), }); - worker.on('message', this._receive.bind(this)); - worker.on('exit', this._exit.bind(this)); + this._worker.on('message', this._onMessage.bind(this)); + this._worker.on('exit', this._exit.bind(this)); + + const {port1} = new MessageChannel(); + + this._worker.postMessage( + [CHILD_MESSAGE_INITIALIZE, false, this._options.workerPath, port1], + [port1], + ); + } + + send(request: ChildMessage, onProcessStart: OnStart, onProcessEnd: OnEnd) { + const item = {next: null, onProcessEnd, onProcessStart, request}; + + if (this._last) { + this._last.next = item; + } else { + this._queue = item; + } + + this._last = item; + this._process(); + } + + getStdout(): Readable { + return this._worker.stdout; + } + + getStderr(): Readable { + return this._worker.stderr; } } diff --git a/packages/jest-worker/src/workers/child.js b/packages/jest-worker/src/workers/child.js index 9eb66f0293b4..3ae927f792dc 100644 --- a/packages/jest-worker/src/workers/child.js +++ b/packages/jest-worker/src/workers/child.js @@ -17,6 +17,12 @@ import { PARENT_MESSAGE_OK, } from '../types'; +import type { + ChildMessage, + ChildMessageInitialize, + ChildMessageCall, +} from '../types'; + let file = null; /** @@ -32,14 +38,16 @@ let file = null; * If an invalid message is detected, the child will exit (by throwing) with a * non-zero exit code. */ -process.on('message', (request: any /* Should be ChildMessage */) => { +process.on('message', (request: any) => { switch (request[0]) { case CHILD_MESSAGE_INITIALIZE: - file = request[2]; + const init: ChildMessageInitialize = request; + file = init[2]; break; case CHILD_MESSAGE_CALL: - execMethod(request[2], request[3]); + const call: ChildMessageCall = request; + execMethod(call[2], call[3]); break; case CHILD_MESSAGE_END: diff --git a/packages/jest-worker/src/workers/threadChild.js b/packages/jest-worker/src/workers/threadChild.js new file mode 100644 index 000000000000..5450bd54d31b --- /dev/null +++ b/packages/jest-worker/src/workers/threadChild.js @@ -0,0 +1,115 @@ +/** + * Copyright (c) 2017-present, Facebook, Inc. All rights reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @flow + */ + +'use strict'; + +import { + CHILD_MESSAGE_CALL, + CHILD_MESSAGE_END, + CHILD_MESSAGE_INITIALIZE, + PARENT_MESSAGE_ERROR, + PARENT_MESSAGE_OK, +} from '../types'; + +import type { + ChildMessage, + ChildMessageInitialize, + ChildMessageCall, +} from '../types'; + +let file = null; + +// $FlowFixMe +import {parentPort, isMainThread} from 'worker_threads'; + +/** + * This file is a small bootstrapper for workers. It sets up the communication + * between the worker and the parent process, interpreting parent messages and + * sending results back. + * + * The file loaded will be lazily initialized the first time any of the workers + * is called. This is done for optimal performance: if the farm is initialized, + * but no call is made to it, child Node processes will be consuming the least + * possible amount of memory. + * + * If an invalid message is detected, the child will exit (by throwing) with a + * non-zero exit code. + */ +parentPort.on('message', (request: any /* Should be ChildMessage */): void => { + switch (request[0]) { + case CHILD_MESSAGE_INITIALIZE: + const init: ChildMessageInitialize = request; + file = init[2]; + break; + + case CHILD_MESSAGE_CALL: + const call: ChildMessageCall = request; + execMethod(call[2], call[3]); + break; + + case CHILD_MESSAGE_END: + process.exit(0); + break; + + default: + throw new TypeError( + 'Unexpected request from parent process: ' + request[0], + ); + } +}); + +function reportSuccess(result: any) { + if (isMainThread) { + throw new Error('Child can only be used on a forked process'); + } + + parentPort.postMessage([PARENT_MESSAGE_OK, result]); +} + +function reportError(error: Error) { + if (isMainThread) { + throw new Error('Child can only be used on a forked process'); + } + + if (error == null) { + error = new Error('"null" or "undefined" thrown'); + } + + parentPort.postMessage([ + PARENT_MESSAGE_ERROR, + error.constructor && error.constructor.name, + error.message, + error.stack, + // $FlowFixMe: this is safe to just inherit from Object. + typeof error === 'object' ? Object.assign({}, error) : error, + ]); +} + +function execMethod(method: string, args: $ReadOnlyArray): void { + // $FlowFixMe: This has to be a dynamic require. + const main = require(file); + let result; + + try { + if (method === 'default') { + result = (main.__esModule ? main['default'] : main).apply(global, args); + } else { + result = main[method].apply(main, args); + } + } catch (err) { + reportError(err); + return; + } + + if (result && typeof result.then === 'function') { + result.then(reportSuccess, reportError); + } else { + reportSuccess(result); + } +} From 2a1ac0e098404937aa3ad4447823ebb5b4b2521b Mon Sep 17 00:00:00 2001 From: Alexey Kureev Date: Mon, 9 Jul 2018 19:14:48 +0100 Subject: [PATCH 04/35] Remove BaseWorker --- packages/jest-worker/src/base/BaseWorker.js | 36 ------ packages/jest-worker/src/types.js | 3 +- .../src/workers/ChildProcessWorker.js | 107 ++++++++++-------- .../src/workers/NodeThreadsWorker.js | 86 ++++++++------ .../jest-worker/src/workers/threadChild.js | 2 +- 5 files changed, 115 insertions(+), 119 deletions(-) delete mode 100644 packages/jest-worker/src/base/BaseWorker.js diff --git a/packages/jest-worker/src/base/BaseWorker.js b/packages/jest-worker/src/base/BaseWorker.js deleted file mode 100644 index 80818becfb2c..000000000000 --- a/packages/jest-worker/src/base/BaseWorker.js +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright (c) 2017-present, Facebook, Inc. All rights reserved. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - * - * @flow - */ - -'use strict'; - -import type {Readable} from 'stream'; - -import type {ChildMessage, QueueChildMessage, WorkerOptions} from '../types'; - -export default class BaseWorker { - _busy: boolean; - _last: ?QueueChildMessage; - _options: WorkerOptions; - _queue: ?QueueChildMessage; - _retries: number; - - initialize() { - throw Error('Initializer is required for custom Worker implementations'); - } - - send(message: ChildMessage, onStart: Function, onEnd: Function): void { - throw Error('"send" method is required for custom Worker implementations'); - } - - _exit(exitCode: number) { - if (exitCode !== 0) { - this.initialize(); - } - } -} diff --git a/packages/jest-worker/src/types.js b/packages/jest-worker/src/types.js index 2d4d60e365fc..6f456f78917b 100644 --- a/packages/jest-worker/src/types.js +++ b/packages/jest-worker/src/types.js @@ -55,7 +55,8 @@ export interface WorkerInterface { send(ChildMessage, Function, Function): void; getStderr(): Readable; getStdout(): Readable; - _exit(number): void; + onExit(number): void; + onMessage(any): void; } export type FarmOptions = { diff --git a/packages/jest-worker/src/workers/ChildProcessWorker.js b/packages/jest-worker/src/workers/ChildProcessWorker.js index b8a400461d2e..71408c994caa 100644 --- a/packages/jest-worker/src/workers/ChildProcessWorker.js +++ b/packages/jest-worker/src/workers/ChildProcessWorker.js @@ -11,8 +11,6 @@ import childProcess from 'child_process'; -import BaseWorker from '../base/BaseWorker'; - import { PARENT_MESSAGE_ERROR, CHILD_MESSAGE_INITIALIZE, @@ -23,7 +21,13 @@ import { import type {ChildProcess} from 'child_process'; import type {Readable} from 'stream'; -import type {ChildMessage, OnEnd, OnStart, WorkerOptions} from '../types'; +import type { + ChildMessage, + OnEnd, + OnStart, + WorkerOptions, + QueueChildMessage, +} from '../types'; /** * This class wraps the child process and provides a nice interface to @@ -43,8 +47,13 @@ import type {ChildMessage, OnEnd, OnStart, WorkerOptions} from '../types'; * field is changed to "true", so that other workers which might encounter the * same call skip it. */ -export default class ChildProcessWorker extends BaseWorker implements WorkerInterface { +export default class ChildProcessWorker implements WorkerInterface { _child: ChildProcess; + _busy: boolean; + _last: ?QueueChildMessage; + _options: WorkerOptions; + _queue: ?QueueChildMessage; + _retries: number; constructor(options: WorkerOptions) { super(); @@ -87,7 +96,51 @@ export default class ChildProcessWorker extends BaseWorker implements WorkerInte } } - _receive(response: any /* Should be ParentMessage */) { + initialize() { + const child = childProcess.fork( + require.resolve('./child'), + // $FlowFixMe: Flow does not work well with Object.assign. + Object.assign( + { + cwd: process.cwd(), + env: Object.assign({}, process.env, { + JEST_WORKER_ID: this._options.workerId, + }), + // Suppress --debug / --inspect flags while preserving others (like --harmony). + execArgv: process.execArgv.filter(v => !/^--(debug|inspect)/.test(v)), + silent: true, + }, + this._options.forkOptions, + ), + ); + + child.on('message', this.onMessage.bind(this)); + child.on('exit', this.onExit.bind(this)); + + // $FlowFixMe: wrong "ChildProcess.send" signature. + child.send([CHILD_MESSAGE_INITIALIZE, false, this._options.workerPath]); + + this._retries++; + this._child = child; + this._busy = false; + + // If we exceeded the amount of retries, we will emulate an error reply + // coming from the child. This avoids code duplication related with cleaning + // the queue, and scheduling the next call. + if (this._retries > this._options.maxRetries) { + const error = new Error('Call retries were exceeded'); + + this.onMessage([ + PARENT_MESSAGE_ERROR, + error.name, + error.message, + error.stack, + {type: 'WorkerError'}, + ]); + } + } + + onMessage(response: any /* Should be ParentMessage */) { const item = this._queue; if (!item) { @@ -131,47 +184,9 @@ export default class ChildProcessWorker extends BaseWorker implements WorkerInte } } - initialize() { - const child = childProcess.fork( - require.resolve('./child'), - // $FlowFixMe: Flow does not work well with Object.assign. - Object.assign( - { - cwd: process.cwd(), - env: Object.assign({}, process.env, { - JEST_WORKER_ID: this._options.workerId, - }), - // Suppress --debug / --inspect flags while preserving others (like --harmony). - execArgv: process.execArgv.filter(v => !/^--(debug|inspect)/.test(v)), - silent: true, - }, - this._options.forkOptions, - ), - ); - - child.on('message', this._receive.bind(this)); - child.on('exit', this._exit.bind(this)); - - // $FlowFixMe: wrong "ChildProcess.send" signature. - child.send([CHILD_MESSAGE_INITIALIZE, false, this._options.workerPath]); - - this._retries++; - this._child = child; - this._busy = false; - - // If we exceeded the amount of retries, we will emulate an error reply - // coming from the child. This avoids code duplication related with cleaning - // the queue, and scheduling the next call. - if (this._retries > this._options.maxRetries) { - const error = new Error('Call retries were exceeded'); - - this._receive([ - PARENT_MESSAGE_ERROR, - error.name, - error.message, - error.stack, - {type: 'WorkerError'}, - ]); + onExit(exitCode: number) { + if (exitCode !== 0) { + this.initialize(); } } diff --git a/packages/jest-worker/src/workers/NodeThreadsWorker.js b/packages/jest-worker/src/workers/NodeThreadsWorker.js index 21634143d11f..ae5f29add9cd 100644 --- a/packages/jest-worker/src/workers/NodeThreadsWorker.js +++ b/packages/jest-worker/src/workers/NodeThreadsWorker.js @@ -9,8 +9,6 @@ 'use strict'; -import BaseWorker from '../base/BaseWorker'; - import { CHILD_MESSAGE_INITIALIZE, PARENT_MESSAGE_OK, @@ -18,14 +16,26 @@ import { } from '../types'; import type {Readable} from 'stream'; -import type {ChildMessage, OnEnd, OnStart, WorkerOptions} from '../types'; +import type { + ChildMessage, + OnEnd, + OnStart, + WorkerOptions, + WorkerInterface, + QueueChildMessage, +} from '../types'; // $FlowFixMe: Flow doesn't know about experimental features of Node const {Worker, MessageChannel} = require('worker_threads'); -export default class ExpirementalWorker extends BaseWorker { +export default class ExpirementalWorker implements WorkerInterface { _options: WorkerOptions; _worker: Worker; + _busy: boolean; + _last: ?QueueChildMessage; + _options: WorkerOptions; + _queue: ?QueueChildMessage; + _retries: number; constructor(options: WorkerOptions) { super(); @@ -73,7 +83,39 @@ export default class ExpirementalWorker extends BaseWorker { } } - _onMessage(response: any /* Should be ParentMessage */) { + initialize() { + this._worker = new Worker(__dirname + '/threadChild.js', { + eval: false, + stderr: true, + stdout: true, + + // $FlowFixMe: Flow does not work well with Object.assign. + workerData: Object.assign( + { + cwd: process.cwd(), + env: Object.assign({}, process.env, { + JEST_WORKER_ID: this._options.workerId, + }), + // Suppress --debug / --inspect flags while preserving others (like --harmony). + execArgv: process.execArgv.filter(v => !/^--(debug|inspect)/.test(v)), + silent: true, + }, + this._options.forkOptions, + ), + }); + + this._worker.on('message', this.onMessage.bind(this)); + this._worker.on('exit', this.onExit.bind(this)); + + const {port1} = new MessageChannel(); + + this._worker.postMessage( + [CHILD_MESSAGE_INITIALIZE, false, this._options.workerPath, port1], + [port1], + ); + } + + onMessage(response: any /* Should be ParentMessage */) { const item = this._queue; if (!item) { @@ -117,36 +159,10 @@ export default class ExpirementalWorker extends BaseWorker { } } - initialize() { - this._worker = new Worker(__dirname + '/threadChild.js', { - eval: false, - stderr: true, - stdout: true, - - // $FlowFixMe: Flow does not work well with Object.assign. - workerData: Object.assign( - { - cwd: process.cwd(), - env: Object.assign({}, process.env, { - JEST_WORKER_ID: this._options.workerId, - }), - // Suppress --debug / --inspect flags while preserving others (like --harmony). - execArgv: process.execArgv.filter(v => !/^--(debug|inspect)/.test(v)), - silent: true, - }, - this._options.forkOptions, - ), - }); - - this._worker.on('message', this._onMessage.bind(this)); - this._worker.on('exit', this._exit.bind(this)); - - const {port1} = new MessageChannel(); - - this._worker.postMessage( - [CHILD_MESSAGE_INITIALIZE, false, this._options.workerPath, port1], - [port1], - ); + onExit(exitCode: number) { + if (exitCode !== 0) { + this.initialize(); + } } send(request: ChildMessage, onProcessStart: OnStart, onProcessEnd: OnEnd) { diff --git a/packages/jest-worker/src/workers/threadChild.js b/packages/jest-worker/src/workers/threadChild.js index 5450bd54d31b..6360962e9dea 100644 --- a/packages/jest-worker/src/workers/threadChild.js +++ b/packages/jest-worker/src/workers/threadChild.js @@ -94,7 +94,7 @@ function reportError(error: Error) { function execMethod(method: string, args: $ReadOnlyArray): void { // $FlowFixMe: This has to be a dynamic require. const main = require(file); - let result; + let result: mixed; try { if (method === 'default') { From 59ebd73bbb8f212b9475cf99a5b6f89ad2058a02 Mon Sep 17 00:00:00 2001 From: Alexey Kureev Date: Mon, 9 Jul 2018 20:33:23 +0100 Subject: [PATCH 05/35] Remove MessageChannel and cleanup super() calls --- .../jest-worker/src/workers/ChildProcessWorker.js | 1 - .../jest-worker/src/workers/NodeThreadsWorker.js | 12 +++++------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/packages/jest-worker/src/workers/ChildProcessWorker.js b/packages/jest-worker/src/workers/ChildProcessWorker.js index 71408c994caa..b322b72099d7 100644 --- a/packages/jest-worker/src/workers/ChildProcessWorker.js +++ b/packages/jest-worker/src/workers/ChildProcessWorker.js @@ -56,7 +56,6 @@ export default class ChildProcessWorker implements WorkerInterface { _retries: number; constructor(options: WorkerOptions) { - super(); this._options = options; this._queue = null; diff --git a/packages/jest-worker/src/workers/NodeThreadsWorker.js b/packages/jest-worker/src/workers/NodeThreadsWorker.js index ae5f29add9cd..25f77bb2fbef 100644 --- a/packages/jest-worker/src/workers/NodeThreadsWorker.js +++ b/packages/jest-worker/src/workers/NodeThreadsWorker.js @@ -38,7 +38,6 @@ export default class ExpirementalWorker implements WorkerInterface { _retries: number; constructor(options: WorkerOptions) { - super(); this._options = options; this._queue = null; @@ -107,12 +106,11 @@ export default class ExpirementalWorker implements WorkerInterface { this._worker.on('message', this.onMessage.bind(this)); this._worker.on('exit', this.onExit.bind(this)); - const {port1} = new MessageChannel(); - - this._worker.postMessage( - [CHILD_MESSAGE_INITIALIZE, false, this._options.workerPath, port1], - [port1], - ); + this._worker.postMessage([ + CHILD_MESSAGE_INITIALIZE, + false, + this._options.workerPath, + ]); } onMessage(response: any /* Should be ParentMessage */) { From 9b1f19b4378f35fdb8f566c6b62ff171022b6950 Mon Sep 17 00:00:00 2001 From: Alexey Kureev Date: Mon, 9 Jul 2018 20:33:59 +0100 Subject: [PATCH 06/35] Use worker threads implementation if possible --- .../jest-worker/src/base/BaseWorkerPool.js | 10 +++++--- packages/jest-worker/src/index.js | 25 +++++++++++++++---- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/packages/jest-worker/src/base/BaseWorkerPool.js b/packages/jest-worker/src/base/BaseWorkerPool.js index 6155daa9b788..c29f09135af5 100644 --- a/packages/jest-worker/src/base/BaseWorkerPool.js +++ b/packages/jest-worker/src/base/BaseWorkerPool.js @@ -13,7 +13,8 @@ import mergeStream from 'merge-stream'; import os from 'os'; import path from 'path'; -import Worker from '../workers/ChildProcessWorker'; +import ChildProcessWorker from '../workers/ChildProcessWorker'; +import NodeThreadsWorker from '../workers/NodeThreadsWorker'; import {CHILD_MESSAGE_END} from '../types'; import type {Readable} from 'stream'; @@ -45,6 +46,7 @@ export default class BaseWorkerPool { const workerOptions: WorkerOptions = { forkOptions: options.forkOptions || {}, maxRetries: options.maxRetries || 3, + useNodeWorkersIfPossible: options.useNodeWorkersIfPossible, workerId: i + 1, workerPath, }; @@ -89,8 +91,10 @@ export default class BaseWorkerPool { return this._workers; } - createWorker(workerOptions: WorkerOptions): Worker { - return new Worker(workerOptions); + createWorker(workerOptions: WorkerOptions): WorkerInterface { + return workerOptions.useNodeWorkersIfPossible + ? new NodeThreadsWorker(workerOptions) + : new ChildProcessWorker(workerOptions); } end(): void { diff --git a/packages/jest-worker/src/index.js b/packages/jest-worker/src/index.js index 9bf99896fa65..0ff93001448f 100644 --- a/packages/jest-worker/src/index.js +++ b/packages/jest-worker/src/index.js @@ -19,6 +19,10 @@ import type {Readable} from 'stream'; import {CHILD_MESSAGE_CALL, WorkerInterface} from './types'; import WorkerPool from './WorkerPool'; +const defaultFarmOptions = { + useNodeWorkersIfPossible: canUseWorkerThreads(), +}; + function getExposedMethods( workerPath: string, options?: FarmOptions = {}, @@ -42,6 +46,16 @@ function getExposedMethods( return exposedMethods; } +function canUseWorkerThreads(): boolean { + let workerThreadsAreSupported = false; + try { + require.resolve('worker_threads'); + workerThreadsAreSupported = true; + } catch (_) {} + + return workerThreadsAreSupported; +} + /** * The Jest farm (publicly called "Worker") is a class that allows you to queue * methods across multiple child processes, in order to parallelize work. This @@ -77,12 +91,13 @@ export default class JestWorker { constructor(workerPath: string, options?: FarmOptions = {}) { this._cacheKeys = Object.create(null); this._offset = 0; - this._options = options; - this._threadPool = options.WorkerPool - ? new options.WorkerPool(workerPath, options) - : new WorkerPool(workerPath, options); + this._options = Object.assign({}, defaultFarmOptions, options); + + this._threadPool = this._options.WorkerPool + ? new this._options.WorkerPool(workerPath, this._options) + : new WorkerPool(workerPath, this._options); - this._bindExposedWorkerMethods(workerPath, options); + this._bindExposedWorkerMethods(workerPath, this._options); } _bindExposedWorkerMethods(workerPath: string, options?: FarmOptions): void { From bea945f9713dacd6fd7ed32378d7fe89e8c7e8a7 Mon Sep 17 00:00:00 2001 From: Alexey Kureev Date: Tue, 10 Jul 2018 19:54:16 +0100 Subject: [PATCH 07/35] Restructure queues --- packages/jest-worker/src/WorkerPool.js | 2 +- .../jest-worker/src/WorkerQueueManager.js | 119 ++++++++++++++++++ .../jest-worker/src/base/BaseWorkerPool.js | 2 +- packages/jest-worker/src/index.js | 31 ++--- packages/jest-worker/src/types.js | 17 ++- .../src/workers/ChildProcessWorker.js | 96 ++------------ .../src/workers/NodeThreadsWorker.js | 80 ++---------- packages/jest-worker/src/workers/child.js | 6 +- .../jest-worker/src/workers/threadChild.js | 57 ++++----- 9 files changed, 195 insertions(+), 215 deletions(-) create mode 100644 packages/jest-worker/src/WorkerQueueManager.js diff --git a/packages/jest-worker/src/WorkerPool.js b/packages/jest-worker/src/WorkerPool.js index 61a148b4ffe2..117dda8f8fcf 100644 --- a/packages/jest-worker/src/WorkerPool.js +++ b/packages/jest-worker/src/WorkerPool.js @@ -16,7 +16,7 @@ import type { OnStart, OnEnd, WorkerInterface, - WorkerPool as WorkerPoolInterface, + WorkerPoolInterface, } from './types'; class WorkerPool extends BaseWorkerPool implements WorkerPoolInterface { diff --git a/packages/jest-worker/src/WorkerQueueManager.js b/packages/jest-worker/src/WorkerQueueManager.js new file mode 100644 index 000000000000..67826b8357b3 --- /dev/null +++ b/packages/jest-worker/src/WorkerQueueManager.js @@ -0,0 +1,119 @@ +/** + * Copyright (c) 2017-present, Facebook, Inc. All rights reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @flow + */ + +'use strict'; + +import type { + QueueChildMessage, + WorkerPoolInterface, + WorkerInterface, +} from './types'; + +export default class WorkerQueueManager { + _workerPool: WorkerPoolInterface; + _queue: Array; + _last: Array; + _locks: Array; + _offset: number; + + constructor(workerPool: WorkerPoolInterface) { + this._workerPool = workerPool; + this._queue = []; + this._last = []; + this._locks = []; + this._offset = 0; + + // If we exceeded the amount of retries, we will emulate an error reply + // coming from the child. This avoids code duplication related with cleaning + // the queue, and scheduling the next call. + + // if (this._retries > this._options.maxRetries) { + // const error = new Error('Call retries were exceeded'); + + // this.onMessage([ + // PARENT_MESSAGE_ERROR, + // error.name, + // error.message, + // error.stack, + // {type: 'WorkerError'}, + // ]); + // } + } + + enqueue(task: QueueChildMessage, workerId?: number): WorkerQueueManager { + if (workerId != null) { + if (this._queue[workerId]) { + this._last[workerId].next = task; + this._last[workerId] = task; + } else { + this._queue[workerId] = task; + this._last[workerId] = task; + } + + this.run(workerId); + } else { + const numOfWorkers = this._workerPool.getWorkers().length; + for (let i = 0; i < numOfWorkers; i++) { + const workerIdx = (this._offset + i) % numOfWorkers; + this.enqueue(task, workerIdx); + } + this._offset++; + } + + return this; + } + + run(workerId: number): WorkerQueueManager { + if (this.isLocked(workerId)) { + return this; + } + + const job = this._queue[workerId]; + const worker = this._workerPool.getWorkers()[workerId]; + + if (!job) { + return this; + } + + if (job.owner) { + this._queue[workerId] = job.next ? job.next : null; + return this.run(workerId); + } + + if (!worker) { + throw Error(`Worker with ID "${workerId}" is not found`); + } + + const onEnd = (error: ?Error, result: mixed, worker: WorkerInterface) => { + this.unlock(workerId); + job.onEnd(error, result, worker); + this.run(workerId); + }; + + this.lock(workerId); + + this._workerPool.send(worker, job.request, job.onStart, onEnd); + + job.owner = worker; + + return this; + } + + lock(workerId: number): void { + this._locks[workerId] = true; + } + + unlock(workerId: number): void { + this._locks[workerId] = false; + } + + isLocked(workerId: number): boolean { + return this._locks[workerId]; + } +} diff --git a/packages/jest-worker/src/base/BaseWorkerPool.js b/packages/jest-worker/src/base/BaseWorkerPool.js index c29f09135af5..7d4e396a006e 100644 --- a/packages/jest-worker/src/base/BaseWorkerPool.js +++ b/packages/jest-worker/src/base/BaseWorkerPool.js @@ -47,7 +47,7 @@ export default class BaseWorkerPool { forkOptions: options.forkOptions || {}, maxRetries: options.maxRetries || 3, useNodeWorkersIfPossible: options.useNodeWorkersIfPossible, - workerId: i + 1, + workerId: i, workerPath, }; diff --git a/packages/jest-worker/src/index.js b/packages/jest-worker/src/index.js index 0ff93001448f..7ca28886d7b2 100644 --- a/packages/jest-worker/src/index.js +++ b/packages/jest-worker/src/index.js @@ -9,15 +9,12 @@ 'use strict'; -import type { - WorkerPool as WorkerPoolInterface, - FarmOptions, - ChildMessage, -} from './types'; +import type {WorkerPoolInterface, FarmOptions, ChildMessage} from './types'; import type {Readable} from 'stream'; import {CHILD_MESSAGE_CALL, WorkerInterface} from './types'; import WorkerPool from './WorkerPool'; +import WorkerQueueManager from './WorkerQueueManager'; const defaultFarmOptions = { useNodeWorkersIfPossible: canUseWorkerThreads(), @@ -84,18 +81,18 @@ function canUseWorkerThreads(): boolean { export default class JestWorker { _cacheKeys: {[string]: WorkerInterface, __proto__: null}; _ending: boolean; - _offset: number; _options: FarmOptions; + _queueManager: WorkerQueueManager; _threadPool: WorkerPoolInterface; constructor(workerPath: string, options?: FarmOptions = {}) { this._cacheKeys = Object.create(null); - this._offset = 0; this._options = Object.assign({}, defaultFarmOptions, options); this._threadPool = this._options.WorkerPool ? new this._options.WorkerPool(workerPath, this._options) : new WorkerPool(workerPath, this._options); + this._queueManager = new WorkerQueueManager(this._threadPool); this._bindExposedWorkerMethods(workerPath, this._options); } @@ -139,7 +136,11 @@ export default class JestWorker { } }; - const onEnd: onEnd = (error: Error, result: mixed) => { + const onEnd: onEnd = ( + error: Error, + result: mixed, + worker: WorkerInterface, + ) => { if (error) { reject(error); } else { @@ -147,18 +148,10 @@ export default class JestWorker { } }; - if (worker) { - this._threadPool.send(worker, request, onStart, onEnd); - } else { - const workers = this._threadPool.getWorkers(); - const length = workers.length; + const task = {onEnd, onStart, request}; + const workerId = worker ? worker.getWorkerId() : undefined; - for (let i = 0; i < length; i++) { - const worker = workers[(i + this._offset) % length]; - this._threadPool.send(worker, request, onStart, onEnd); - } - this._offset++; - } + this._queueManager.enqueue(task, workerId); }); } diff --git a/packages/jest-worker/src/types.js b/packages/jest-worker/src/types.js index 6f456f78917b..014851edc2a2 100644 --- a/packages/jest-worker/src/types.js +++ b/packages/jest-worker/src/types.js @@ -38,7 +38,7 @@ export type ForkOptions = { gid?: number, }; -export interface WorkerPool { +export interface WorkerPoolInterface { _options: FarmOptions; _stderr: Readable; _stdout: Readable; @@ -53,6 +53,7 @@ export interface WorkerPool { export interface WorkerInterface { send(ChildMessage, Function, Function): void; + getWorkerId(): number; getStderr(): Readable; getStdout(): Readable; onExit(number): void; @@ -65,7 +66,10 @@ export type FarmOptions = { forkOptions?: ForkOptions, maxRetries?: number, numWorkers?: number, - WorkerPool?: (workerPath: string, options?: FarmOptions) => WorkerPool, + WorkerPool?: ( + workerPath: string, + options?: FarmOptions, + ) => WorkerPoolInterface, useNodeWorkersIfPossible?: boolean, }; @@ -132,11 +136,12 @@ export type ParentMessage = ParentMessageOk | ParentMessageError; // Queue types. export type OnStart = WorkerInterface => void; -export type OnEnd = (?Error, ?any) => void; +export type OnEnd = (?Error, ?any, WorkerInterface) => void; export type QueueChildMessage = {| request: ChildMessage, - onProcessStart: OnStart, - onProcessEnd: OnEnd, - next: ?QueueChildMessage, + onStart: OnStart, + onEnd: OnEnd, + owner?: WorkerInterface, + next?: QueueChildMessage, |}; diff --git a/packages/jest-worker/src/workers/ChildProcessWorker.js b/packages/jest-worker/src/workers/ChildProcessWorker.js index b322b72099d7..7ce62ab0ed8c 100644 --- a/packages/jest-worker/src/workers/ChildProcessWorker.js +++ b/packages/jest-worker/src/workers/ChildProcessWorker.js @@ -21,13 +21,7 @@ import { import type {ChildProcess} from 'child_process'; import type {Readable} from 'stream'; -import type { - ChildMessage, - OnEnd, - OnStart, - WorkerOptions, - QueueChildMessage, -} from '../types'; +import type {ChildMessage, OnEnd, OnStart, WorkerOptions} from '../types'; /** * This class wraps the child process and provides a nice interface to @@ -49,52 +43,14 @@ import type { */ export default class ChildProcessWorker implements WorkerInterface { _child: ChildProcess; - _busy: boolean; - _last: ?QueueChildMessage; _options: WorkerOptions; - _queue: ?QueueChildMessage; - _retries: number; + _onProcessEnd: OnEnd; constructor(options: WorkerOptions) { this._options = options; - this._queue = null; - this.initialize(); } - _process() { - if (this._busy) { - return; - } - - let item = this._queue; - - // Calls in the queue might have already been processed by another worker, - // so we have to skip them. - while (item && item.request[1]) { - item = item.next; - } - - this._queue = item; - - if (item) { - // Flag the call as processed, so that other workers know that they don't - // have to process it as well. - item.request[1] = true; - - // Tell the parent that this item is starting to be processed. - item.onProcessStart(this); - - this._retries = 0; - this._busy = true; - - // $FlowFixMe: wrong "ChildProcess.send" signature. - this._child.send(item.request); - } else { - this._last = item; - } - } - initialize() { const child = childProcess.fork( require.resolve('./child'), @@ -119,41 +75,13 @@ export default class ChildProcessWorker implements WorkerInterface { // $FlowFixMe: wrong "ChildProcess.send" signature. child.send([CHILD_MESSAGE_INITIALIZE, false, this._options.workerPath]); - this._retries++; this._child = child; - this._busy = false; - - // If we exceeded the amount of retries, we will emulate an error reply - // coming from the child. This avoids code duplication related with cleaning - // the queue, and scheduling the next call. - if (this._retries > this._options.maxRetries) { - const error = new Error('Call retries were exceeded'); - - this.onMessage([ - PARENT_MESSAGE_ERROR, - error.name, - error.message, - error.stack, - {type: 'WorkerError'}, - ]); - } } onMessage(response: any /* Should be ParentMessage */) { - const item = this._queue; - - if (!item) { - throw new TypeError('Unexpected response with an empty queue'); - } - - const onProcessEnd = item.onProcessEnd; - - this._busy = false; - this._process(); - switch (response[0]) { case PARENT_MESSAGE_OK: - onProcessEnd(null, response[1]); + this._onProcessEnd(null, response[1], this); break; case PARENT_MESSAGE_ERROR: @@ -175,7 +103,7 @@ export default class ChildProcessWorker implements WorkerInterface { } } - onProcessEnd(error, null); + this._onProcessEnd(error, null, this); break; default: @@ -190,16 +118,14 @@ export default class ChildProcessWorker implements WorkerInterface { } send(request: ChildMessage, onProcessStart: OnStart, onProcessEnd: OnEnd) { - const item = {next: null, onProcessEnd, onProcessStart, request}; - - if (this._last) { - this._last.next = item; - } else { - this._queue = item; - } + onProcessStart(this); + this._onProcessEnd = onProcessEnd; + // $FlowFixMe + this._child.send(request); + } - this._last = item; - this._process(); + getWorkerId(): number { + return this._options.workerId; } getStdout(): Readable { diff --git a/packages/jest-worker/src/workers/NodeThreadsWorker.js b/packages/jest-worker/src/workers/NodeThreadsWorker.js index 25f77bb2fbef..ee00229377fb 100644 --- a/packages/jest-worker/src/workers/NodeThreadsWorker.js +++ b/packages/jest-worker/src/workers/NodeThreadsWorker.js @@ -22,72 +22,26 @@ import type { OnStart, WorkerOptions, WorkerInterface, - QueueChildMessage, } from '../types'; // $FlowFixMe: Flow doesn't know about experimental features of Node -const {Worker, MessageChannel} = require('worker_threads'); +const {Worker} = require('worker_threads'); export default class ExpirementalWorker implements WorkerInterface { - _options: WorkerOptions; _worker: Worker; - _busy: boolean; - _last: ?QueueChildMessage; _options: WorkerOptions; - _queue: ?QueueChildMessage; - _retries: number; + _onProcessEnd: OnEnd; constructor(options: WorkerOptions) { this._options = options; - this._queue = null; - this.initialize(); } - _process() { - if (this._busy) { - return; - } - - let item = this._queue; - - // Calls in the queue might have already been processed by another worker, - // so we have to skip them. - while (item && item.request[1]) { - item = item.next; - } - - this._queue = item; - - if (item) { - // Flag the call as processed, so that other workers know that they don't - // have to process it as well. - item.request[1] = true; - - // Tell the parent that this item is starting to be processed. - item.onProcessStart(this); - - this._retries = 0; - this._busy = true; - - if (!this._worker) { - throw Error( - "Can't process the request without having an active worker", - ); - } - - this._worker.postMessage(item.request); - } else { - this._last = item; - } - } - initialize() { this._worker = new Worker(__dirname + '/threadChild.js', { eval: false, stderr: true, stdout: true, - // $FlowFixMe: Flow does not work well with Object.assign. workerData: Object.assign( { @@ -114,20 +68,9 @@ export default class ExpirementalWorker implements WorkerInterface { } onMessage(response: any /* Should be ParentMessage */) { - const item = this._queue; - - if (!item) { - throw new TypeError('Unexpected response with an empty queue'); - } - - const onProcessEnd = item.onProcessEnd; - - this._busy = false; - this._process(); - switch (response[0]) { case PARENT_MESSAGE_OK: - onProcessEnd(null, response[1]); + this._onProcessEnd(null, response[1], this); break; case PARENT_MESSAGE_ERROR: @@ -149,7 +92,7 @@ export default class ExpirementalWorker implements WorkerInterface { } } - onProcessEnd(error, null); + this._onProcessEnd(error, null, this); break; default: @@ -164,16 +107,13 @@ export default class ExpirementalWorker implements WorkerInterface { } send(request: ChildMessage, onProcessStart: OnStart, onProcessEnd: OnEnd) { - const item = {next: null, onProcessEnd, onProcessStart, request}; - - if (this._last) { - this._last.next = item; - } else { - this._queue = item; - } + onProcessStart(this); + this._onProcessEnd = onProcessEnd; + this._worker.postMessage(request); + } - this._last = item; - this._process(); + getWorkerId(): number { + return this._options.workerId; } getStdout(): Readable { diff --git a/packages/jest-worker/src/workers/child.js b/packages/jest-worker/src/workers/child.js index 3ae927f792dc..f918a5c4d5a8 100644 --- a/packages/jest-worker/src/workers/child.js +++ b/packages/jest-worker/src/workers/child.js @@ -17,11 +17,7 @@ import { PARENT_MESSAGE_OK, } from '../types'; -import type { - ChildMessage, - ChildMessageInitialize, - ChildMessageCall, -} from '../types'; +import type {ChildMessageInitialize, ChildMessageCall} from '../types'; let file = null; diff --git a/packages/jest-worker/src/workers/threadChild.js b/packages/jest-worker/src/workers/threadChild.js index 6360962e9dea..50f00a8582da 100644 --- a/packages/jest-worker/src/workers/threadChild.js +++ b/packages/jest-worker/src/workers/threadChild.js @@ -17,16 +17,14 @@ import { PARENT_MESSAGE_OK, } from '../types'; -import type { - ChildMessage, - ChildMessageInitialize, - ChildMessageCall, -} from '../types'; +import type {ChildMessageInitialize, ChildMessageCall} from '../types'; let file = null; -// $FlowFixMe +/* eslint-disable import/no-unresolved */ +// $FlowFixMe: Flow doesn't support experimental node modules import {parentPort, isMainThread} from 'worker_threads'; +/* eslint-enable import/no-unresolved */ /** * This file is a small bootstrapper for workers. It sets up the communication @@ -41,28 +39,31 @@ import {parentPort, isMainThread} from 'worker_threads'; * If an invalid message is detected, the child will exit (by throwing) with a * non-zero exit code. */ -parentPort.on('message', (request: any /* Should be ChildMessage */): void => { - switch (request[0]) { - case CHILD_MESSAGE_INITIALIZE: - const init: ChildMessageInitialize = request; - file = init[2]; - break; - - case CHILD_MESSAGE_CALL: - const call: ChildMessageCall = request; - execMethod(call[2], call[3]); - break; - - case CHILD_MESSAGE_END: - process.exit(0); - break; - - default: - throw new TypeError( - 'Unexpected request from parent process: ' + request[0], - ); - } -}); +parentPort.on( + 'message', + (request: any /* Should be ChildMessage */): void => { + switch (request[0]) { + case CHILD_MESSAGE_INITIALIZE: + const init: ChildMessageInitialize = request; + file = init[2]; + break; + + case CHILD_MESSAGE_CALL: + const call: ChildMessageCall = request; + execMethod(call[2], call[3]); + break; + + case CHILD_MESSAGE_END: + process.exit(0); + break; + + default: + throw new TypeError( + 'Unexpected request from parent process: ' + request[0], + ); + } + }, +); function reportSuccess(result: any) { if (isMainThread) { From 3e9ce0235e48a6c555cda6f57dcbf592db19658a Mon Sep 17 00:00:00 2001 From: Alexey Kureev Date: Wed, 11 Jul 2018 12:33:51 +0100 Subject: [PATCH 08/35] Support experimental modules in jest-resolver --- packages/jest-resolve/src/is_builtin_module.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/jest-resolve/src/is_builtin_module.js b/packages/jest-resolve/src/is_builtin_module.js index 67762edc1ee8..38b83d1be6c0 100644 --- a/packages/jest-resolve/src/is_builtin_module.js +++ b/packages/jest-resolve/src/is_builtin_module.js @@ -15,11 +15,13 @@ declare var process: { binding(type: string): {}, }; +const EXPERIMENTAL_MODULES = ['worker_threads']; + const BUILTIN_MODULES = - builtinModules || - Object.keys(process.binding('natives')).filter( - (module: string) => !/^internal\//.test(module), - ); + builtinModules.concat(EXPERIMENTAL_MODULES) || + Object.keys(process.binding('natives')) + .filter((module: string) => !/^internal\//.test(module)) + .concat([EXPERIMENTAL_MODULES]); export default function isBuiltinModule(module: string): boolean { return BUILTIN_MODULES.indexOf(module) !== -1; From 02dedef69d4f97a5553037a7f580df36fed5c7e1 Mon Sep 17 00:00:00 2001 From: Alexey Kureev Date: Wed, 11 Jul 2018 13:48:24 +0100 Subject: [PATCH 09/35] Rename child.js to processChild.js --- packages/jest-worker/src/__tests__/ChildProcessWorker.test.js | 2 +- packages/jest-worker/src/__tests__/child.test.js | 2 +- packages/jest-worker/src/workers/ChildProcessWorker.js | 2 +- packages/jest-worker/src/workers/{child.js => processChild.js} | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename packages/jest-worker/src/workers/{child.js => processChild.js} (100%) diff --git a/packages/jest-worker/src/__tests__/ChildProcessWorker.test.js b/packages/jest-worker/src/__tests__/ChildProcessWorker.test.js index 7f11a20ebb32..a4bdbf993c0c 100644 --- a/packages/jest-worker/src/__tests__/ChildProcessWorker.test.js +++ b/packages/jest-worker/src/__tests__/ChildProcessWorker.test.js @@ -47,7 +47,7 @@ afterEach(() => { }); it('passes fork options down to child_process.fork, adding the defaults', () => { - const child = require.resolve('../workers/child'); + const child = require.resolve('../workers/processChild'); process.execArgv = ['--inspect', '-p']; diff --git a/packages/jest-worker/src/__tests__/child.test.js b/packages/jest-worker/src/__tests__/child.test.js index c6f5a1540436..27841447df5d 100644 --- a/packages/jest-worker/src/__tests__/child.test.js +++ b/packages/jest-worker/src/__tests__/child.test.js @@ -94,7 +94,7 @@ beforeEach(() => { process.send = jest.fn(); // Require the child! - require('../workers/child'); + require('../workers/processChild'); }); afterEach(() => { diff --git a/packages/jest-worker/src/workers/ChildProcessWorker.js b/packages/jest-worker/src/workers/ChildProcessWorker.js index 7ce62ab0ed8c..2db81508768c 100644 --- a/packages/jest-worker/src/workers/ChildProcessWorker.js +++ b/packages/jest-worker/src/workers/ChildProcessWorker.js @@ -53,7 +53,7 @@ export default class ChildProcessWorker implements WorkerInterface { initialize() { const child = childProcess.fork( - require.resolve('./child'), + require.resolve('./processChild'), // $FlowFixMe: Flow does not work well with Object.assign. Object.assign( { diff --git a/packages/jest-worker/src/workers/child.js b/packages/jest-worker/src/workers/processChild.js similarity index 100% rename from packages/jest-worker/src/workers/child.js rename to packages/jest-worker/src/workers/processChild.js From 5e8347b8ac201a0199e6c043cad19b72cd665fd9 Mon Sep 17 00:00:00 2001 From: Alexey Kureev Date: Wed, 11 Jul 2018 14:01:43 +0100 Subject: [PATCH 10/35] Remove private properties from WorkerPoolInterface --- packages/jest-worker/src/types.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/jest-worker/src/types.js b/packages/jest-worker/src/types.js index 014851edc2a2..15c1b5cbef1b 100644 --- a/packages/jest-worker/src/types.js +++ b/packages/jest-worker/src/types.js @@ -39,10 +39,6 @@ export type ForkOptions = { }; export interface WorkerPoolInterface { - _options: FarmOptions; - _stderr: Readable; - _stdout: Readable; - _workers: Array; getStderr(): Readable; getStdout(): Readable; getWorkers(): Array; From e60bc1f2493ae6eef80f68d51cb2abfba78d4474 Mon Sep 17 00:00:00 2001 From: Alexey Kureev Date: Wed, 11 Jul 2018 14:22:41 +0100 Subject: [PATCH 11/35] Move common line outside of if-else --- packages/jest-worker/src/WorkerQueueManager.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/jest-worker/src/WorkerQueueManager.js b/packages/jest-worker/src/WorkerQueueManager.js index 67826b8357b3..cbee3126ce25 100644 --- a/packages/jest-worker/src/WorkerQueueManager.js +++ b/packages/jest-worker/src/WorkerQueueManager.js @@ -50,12 +50,12 @@ export default class WorkerQueueManager { if (workerId != null) { if (this._queue[workerId]) { this._last[workerId].next = task; - this._last[workerId] = task; } else { this._queue[workerId] = task; - this._last[workerId] = task; } + this._last[workerId] = task; + this.run(workerId); } else { const numOfWorkers = this._workerPool.getWorkers().length; From b7470d62af7c94d10c030c7a0e2663f958de8c2b Mon Sep 17 00:00:00 2001 From: Alexey Kureev Date: Wed, 11 Jul 2018 16:00:20 +0100 Subject: [PATCH 12/35] Unify interface (use workerId) and remove recursion --- packages/jest-worker/src/WorkerPool.js | 12 +-- .../jest-worker/src/WorkerQueueManager.js | 86 +++++++++++-------- .../jest-worker/src/base/BaseWorkerPool.js | 4 + packages/jest-worker/src/types.js | 3 +- 4 files changed, 57 insertions(+), 48 deletions(-) diff --git a/packages/jest-worker/src/WorkerPool.js b/packages/jest-worker/src/WorkerPool.js index 117dda8f8fcf..3c4055356738 100644 --- a/packages/jest-worker/src/WorkerPool.js +++ b/packages/jest-worker/src/WorkerPool.js @@ -11,22 +11,16 @@ import BaseWorkerPool from './base/BaseWorkerPool'; -import type { - ChildMessage, - OnStart, - OnEnd, - WorkerInterface, - WorkerPoolInterface, -} from './types'; +import type {ChildMessage, OnStart, OnEnd, WorkerPoolInterface} from './types'; class WorkerPool extends BaseWorkerPool implements WorkerPoolInterface { send( - worker: WorkerInterface, + workerId: number, request: ChildMessage, onStart: OnStart, onEnd: OnEnd, ): void { - worker.send(request, onStart, onEnd); + this.getWorkerById(workerId).send(request, onStart, onEnd); } } diff --git a/packages/jest-worker/src/WorkerQueueManager.js b/packages/jest-worker/src/WorkerQueueManager.js index cbee3126ce25..cc535d9fe642 100644 --- a/packages/jest-worker/src/WorkerQueueManager.js +++ b/packages/jest-worker/src/WorkerQueueManager.js @@ -46,8 +46,56 @@ export default class WorkerQueueManager { // } } + _process(workerId: number): WorkerQueueManager { + if (this.isLocked(workerId)) { + return this; + } + + const job = this.getNextJob(workerId); + + if (!job) { + return this; + } + + const onEnd = (error: ?Error, result: mixed, worker: WorkerInterface) => { + this.unlock(workerId); + job.onEnd(error, result, worker); + this._process(workerId); + }; + + this.lock(workerId); + + this._workerPool.send(workerId, job.request, job.onStart, onEnd); + + job.request[1] = true; + + return this; + } + + getNextJob(workerId: number): ?QueueChildMessage { + if (!this._queue[workerId]) { + return null; + } + + let job; + + while (this._queue[workerId]) { + if (!this._queue[workerId].request[1]) { + job = this._queue[workerId]; + break; + } + this._queue[workerId] = this._queue[workerId].next; + } + + return job; + } + enqueue(task: QueueChildMessage, workerId?: number): WorkerQueueManager { if (workerId != null) { + if (task.request[1]) { + return this; + } + if (this._queue[workerId]) { this._last[workerId].next = task; } else { @@ -56,7 +104,7 @@ export default class WorkerQueueManager { this._last[workerId] = task; - this.run(workerId); + this._process(workerId); } else { const numOfWorkers = this._workerPool.getWorkers().length; for (let i = 0; i < numOfWorkers; i++) { @@ -69,42 +117,6 @@ export default class WorkerQueueManager { return this; } - run(workerId: number): WorkerQueueManager { - if (this.isLocked(workerId)) { - return this; - } - - const job = this._queue[workerId]; - const worker = this._workerPool.getWorkers()[workerId]; - - if (!job) { - return this; - } - - if (job.owner) { - this._queue[workerId] = job.next ? job.next : null; - return this.run(workerId); - } - - if (!worker) { - throw Error(`Worker with ID "${workerId}" is not found`); - } - - const onEnd = (error: ?Error, result: mixed, worker: WorkerInterface) => { - this.unlock(workerId); - job.onEnd(error, result, worker); - this.run(workerId); - }; - - this.lock(workerId); - - this._workerPool.send(worker, job.request, job.onStart, onEnd); - - job.owner = worker; - - return this; - } - lock(workerId: number): void { this._locks[workerId] = true; } diff --git a/packages/jest-worker/src/base/BaseWorkerPool.js b/packages/jest-worker/src/base/BaseWorkerPool.js index 7d4e396a006e..8d2a0563fdbf 100644 --- a/packages/jest-worker/src/base/BaseWorkerPool.js +++ b/packages/jest-worker/src/base/BaseWorkerPool.js @@ -91,6 +91,10 @@ export default class BaseWorkerPool { return this._workers; } + getWorkerById(workerId: number): WorkerInterface { + return this._workers[workerId]; + } + createWorker(workerOptions: WorkerOptions): WorkerInterface { return workerOptions.useNodeWorkersIfPossible ? new NodeThreadsWorker(workerOptions) diff --git a/packages/jest-worker/src/types.js b/packages/jest-worker/src/types.js index 15c1b5cbef1b..2fd34c789833 100644 --- a/packages/jest-worker/src/types.js +++ b/packages/jest-worker/src/types.js @@ -43,7 +43,7 @@ export interface WorkerPoolInterface { getStdout(): Readable; getWorkers(): Array; createWorker(WorkerOptions): WorkerInterface; - send(WorkerInterface, ChildMessage, Function, Function): void; + send(number, ChildMessage, Function, Function): void; end(): void; } @@ -138,6 +138,5 @@ export type QueueChildMessage = {| request: ChildMessage, onStart: OnStart, onEnd: OnEnd, - owner?: WorkerInterface, next?: QueueChildMessage, |}; From 092d70069ac5df2a5f827d018f72334f465f9ad7 Mon Sep 17 00:00:00 2001 From: Alexey Kureev Date: Wed, 11 Jul 2018 16:01:19 +0100 Subject: [PATCH 13/35] Remove opt-out option for worker_threads in node 10.5+ --- packages/jest-worker/src/index.js | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/jest-worker/src/index.js b/packages/jest-worker/src/index.js index 7ca28886d7b2..eeaa5f240531 100644 --- a/packages/jest-worker/src/index.js +++ b/packages/jest-worker/src/index.js @@ -16,10 +16,6 @@ import {CHILD_MESSAGE_CALL, WorkerInterface} from './types'; import WorkerPool from './WorkerPool'; import WorkerQueueManager from './WorkerQueueManager'; -const defaultFarmOptions = { - useNodeWorkersIfPossible: canUseWorkerThreads(), -}; - function getExposedMethods( workerPath: string, options?: FarmOptions = {}, @@ -46,7 +42,8 @@ function getExposedMethods( function canUseWorkerThreads(): boolean { let workerThreadsAreSupported = false; try { - require.resolve('worker_threads'); + // $FlowFixMe: Flow doesn't know about experimental APIs + require('worker_threads'); workerThreadsAreSupported = true; } catch (_) {} @@ -87,7 +84,9 @@ export default class JestWorker { constructor(workerPath: string, options?: FarmOptions = {}) { this._cacheKeys = Object.create(null); - this._options = Object.assign({}, defaultFarmOptions, options); + this._options = Object.assign({}, options, { + useNodeWorkersIfPossible: canUseWorkerThreads(), + }); this._threadPool = this._options.WorkerPool ? new this._options.WorkerPool(workerPath, this._options) From 3947fcbaeca23935b175d5d87089129663d4df92 Mon Sep 17 00:00:00 2001 From: Alexey Kureev Date: Wed, 11 Jul 2018 16:21:43 +0100 Subject: [PATCH 14/35] Alphabetical import sorting --- packages/jest-worker/src/workers/ChildProcessWorker.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/jest-worker/src/workers/ChildProcessWorker.js b/packages/jest-worker/src/workers/ChildProcessWorker.js index 2db81508768c..ab2a44af9083 100644 --- a/packages/jest-worker/src/workers/ChildProcessWorker.js +++ b/packages/jest-worker/src/workers/ChildProcessWorker.js @@ -12,8 +12,8 @@ import childProcess from 'child_process'; import { - PARENT_MESSAGE_ERROR, CHILD_MESSAGE_INITIALIZE, + PARENT_MESSAGE_ERROR, PARENT_MESSAGE_OK, WorkerInterface, } from '../types'; From c715f007c887aa08f42eacc1dc8bde55a6750769 Mon Sep 17 00:00:00 2001 From: Alexey Kureev Date: Wed, 11 Jul 2018 17:11:02 +0100 Subject: [PATCH 15/35] Unlock worker after onEnd --- packages/jest-worker/src/WorkerQueueManager.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/jest-worker/src/WorkerQueueManager.js b/packages/jest-worker/src/WorkerQueueManager.js index cc535d9fe642..74029942c35e 100644 --- a/packages/jest-worker/src/WorkerQueueManager.js +++ b/packages/jest-worker/src/WorkerQueueManager.js @@ -58,8 +58,8 @@ export default class WorkerQueueManager { } const onEnd = (error: ?Error, result: mixed, worker: WorkerInterface) => { - this.unlock(workerId); job.onEnd(error, result, worker); + this.unlock(workerId); this._process(workerId); }; From 4614e54666dff390213194d742052409ae43862b Mon Sep 17 00:00:00 2001 From: Alexey Kureev Date: Wed, 11 Jul 2018 17:22:04 +0100 Subject: [PATCH 16/35] Cache queue head in the getNextJob loop --- packages/jest-worker/src/WorkerQueueManager.js | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/jest-worker/src/WorkerQueueManager.js b/packages/jest-worker/src/WorkerQueueManager.js index 74029942c35e..647b5aeca4fc 100644 --- a/packages/jest-worker/src/WorkerQueueManager.js +++ b/packages/jest-worker/src/WorkerQueueManager.js @@ -73,18 +73,21 @@ export default class WorkerQueueManager { } getNextJob(workerId: number): ?QueueChildMessage { - if (!this._queue[workerId]) { + let queueHead = this._queue[workerId]; + + if (!queueHead) { return null; } let job; - while (this._queue[workerId]) { - if (!this._queue[workerId].request[1]) { - job = this._queue[workerId]; + while (queueHead) { + this._queue[workerId] = queueHead; + if (!queueHead.request[1]) { + job = queueHead; break; } - this._queue[workerId] = this._queue[workerId].next; + queueHead = queueHead.next; } return job; From e0bfb83b7bda3fd59c52e24caa6fed19b4de61d6 Mon Sep 17 00:00:00 2001 From: Alexey Kureev Date: Wed, 11 Jul 2018 17:28:41 +0100 Subject: [PATCH 17/35] Elegant while loop --- packages/jest-worker/src/WorkerQueueManager.js | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/packages/jest-worker/src/WorkerQueueManager.js b/packages/jest-worker/src/WorkerQueueManager.js index 647b5aeca4fc..83d393856cec 100644 --- a/packages/jest-worker/src/WorkerQueueManager.js +++ b/packages/jest-worker/src/WorkerQueueManager.js @@ -79,18 +79,13 @@ export default class WorkerQueueManager { return null; } - let job; - - while (queueHead) { - this._queue[workerId] = queueHead; - if (!queueHead.request[1]) { - job = queueHead; - break; - } + while (queueHead && queueHead.request[1]) { queueHead = queueHead.next; } - return job; + this._queue[workerId] = queueHead; + + return queueHead; } enqueue(task: QueueChildMessage, workerId?: number): WorkerQueueManager { From 995858729f0fd34be719ea4bf8eb3a9b9ef9f6a0 Mon Sep 17 00:00:00 2001 From: Alexey Kureev Date: Wed, 11 Jul 2018 17:42:53 +0100 Subject: [PATCH 18/35] Remove redundand .binds --- packages/jest-worker/src/base/BaseWorkerPool.js | 9 --------- 1 file changed, 9 deletions(-) diff --git a/packages/jest-worker/src/base/BaseWorkerPool.js b/packages/jest-worker/src/base/BaseWorkerPool.js index 8d2a0563fdbf..d16566f5bc97 100644 --- a/packages/jest-worker/src/base/BaseWorkerPool.js +++ b/packages/jest-worker/src/base/BaseWorkerPool.js @@ -64,15 +64,6 @@ export default class BaseWorkerPool { } this._workers[i] = worker; - - // $FlowFixMe - this.getStderr = this.getStderr.bind(this); - // $FlowFixMe - this.getStdout = this.getStdout.bind(this); - // $FlowFixMe - this.getWorkers = this.getWorkers.bind(this); - // $FlowFixMe - this.end = this.end.bind(this); } this._stdout = stdout; From 29e04d25694cb02525e0c20bad584053b3e0a7dc Mon Sep 17 00:00:00 2001 From: Alexey Kureev Date: Thu, 12 Jul 2018 11:44:27 +0100 Subject: [PATCH 19/35] Clean up interfaces and responsibilites --- ...{WorkerQueueManager.js => QueueManager.js} | 66 +++++++++---------- packages/jest-worker/src/WorkerPool.js | 17 ++++- .../jest-worker/src/base/BaseWorkerPool.js | 26 +++----- packages/jest-worker/src/index.js | 57 +++++++++------- packages/jest-worker/src/types.js | 13 ++-- .../src/workers/ChildProcessWorker.js | 4 +- .../src/workers/NodeThreadsWorker.js | 4 +- 7 files changed, 106 insertions(+), 81 deletions(-) rename packages/jest-worker/src/{WorkerQueueManager.js => QueueManager.js} (62%) diff --git a/packages/jest-worker/src/WorkerQueueManager.js b/packages/jest-worker/src/QueueManager.js similarity index 62% rename from packages/jest-worker/src/WorkerQueueManager.js rename to packages/jest-worker/src/QueueManager.js index 83d393856cec..1bea7016237b 100644 --- a/packages/jest-worker/src/WorkerQueueManager.js +++ b/packages/jest-worker/src/QueueManager.js @@ -9,21 +9,19 @@ 'use strict'; -import type { - QueueChildMessage, - WorkerPoolInterface, - WorkerInterface, -} from './types'; - -export default class WorkerQueueManager { - _workerPool: WorkerPoolInterface; - _queue: Array; +import type {QueueChildMessage} from './types'; + +export default class QueueManager { + _callback: Function; _last: Array; _locks: Array; + _numOfWorkers: number; _offset: number; + _queue: Array; - constructor(workerPool: WorkerPoolInterface) { - this._workerPool = workerPool; + constructor(numOfWorkers: number, callback: Function) { + this._callback = callback; + this._numOfWorkers = numOfWorkers; this._queue = []; this._last = []; this._locks = []; @@ -46,7 +44,7 @@ export default class WorkerQueueManager { // } } - _process(workerId: number): WorkerQueueManager { + _process(workerId: number): QueueManager { if (this.isLocked(workerId)) { return this; } @@ -57,15 +55,15 @@ export default class WorkerQueueManager { return this; } - const onEnd = (error: ?Error, result: mixed, worker: WorkerInterface) => { - job.onEnd(error, result, worker); + const onEnd = (error: ?Error, result: mixed) => { + job.onEnd(error, result); this.unlock(workerId); this._process(workerId); }; this.lock(workerId); - this._workerPool.send(workerId, job.request, job.onStart, onEnd); + this._callback(workerId, job.request, job.onStart, onEnd); job.request[1] = true; @@ -88,29 +86,29 @@ export default class WorkerQueueManager { return queueHead; } - enqueue(task: QueueChildMessage, workerId?: number): WorkerQueueManager { - if (workerId != null) { - if (task.request[1]) { - return this; - } + enqueue(task: QueueChildMessage, workerId: number): QueueManager { + if (task.request[1]) { + return this; + } + + if (this._queue[workerId]) { + this._last[workerId].next = task; + } else { + this._queue[workerId] = task; + } - if (this._queue[workerId]) { - this._last[workerId].next = task; - } else { - this._queue[workerId] = task; - } + this._last[workerId] = task; + this._process(workerId); - this._last[workerId] = task; + return this; + } - this._process(workerId); - } else { - const numOfWorkers = this._workerPool.getWorkers().length; - for (let i = 0; i < numOfWorkers; i++) { - const workerIdx = (this._offset + i) % numOfWorkers; - this.enqueue(task, workerIdx); - } - this._offset++; + push(task: QueueChildMessage): QueueManager { + for (let i = 0; i < this._numOfWorkers; i++) { + const workerIdx = (this._offset + i) % this._numOfWorkers; + this.enqueue(task, workerIdx); } + this._offset++; return this; } diff --git a/packages/jest-worker/src/WorkerPool.js b/packages/jest-worker/src/WorkerPool.js index 3c4055356738..9ec6e4537ca1 100644 --- a/packages/jest-worker/src/WorkerPool.js +++ b/packages/jest-worker/src/WorkerPool.js @@ -10,8 +10,17 @@ 'use strict'; import BaseWorkerPool from './base/BaseWorkerPool'; +import ChildProcessWorker from './workers/ChildProcessWorker'; +import NodeThreadsWorker from './workers/NodeThreadsWorker'; -import type {ChildMessage, OnStart, OnEnd, WorkerPoolInterface} from './types'; +import type { + ChildMessage, + WorkerOptions, + OnStart, + OnEnd, + WorkerPoolInterface, + WorkerInterface, +} from './types'; class WorkerPool extends BaseWorkerPool implements WorkerPoolInterface { send( @@ -22,6 +31,12 @@ class WorkerPool extends BaseWorkerPool implements WorkerPoolInterface { ): void { this.getWorkerById(workerId).send(request, onStart, onEnd); } + + createWorker(workerOptions: WorkerOptions): WorkerInterface { + return this._options.useWorkers + ? new NodeThreadsWorker(workerOptions) + : new ChildProcessWorker(workerOptions); + } } export default WorkerPool; diff --git a/packages/jest-worker/src/base/BaseWorkerPool.js b/packages/jest-worker/src/base/BaseWorkerPool.js index d16566f5bc97..391c27d2f391 100644 --- a/packages/jest-worker/src/base/BaseWorkerPool.js +++ b/packages/jest-worker/src/base/BaseWorkerPool.js @@ -10,15 +10,12 @@ 'use strict'; import mergeStream from 'merge-stream'; -import os from 'os'; import path from 'path'; -import ChildProcessWorker from '../workers/ChildProcessWorker'; -import NodeThreadsWorker from '../workers/NodeThreadsWorker'; import {CHILD_MESSAGE_END} from '../types'; import type {Readable} from 'stream'; -import type {FarmOptions, WorkerOptions, WorkerInterface} from '../types'; +import type {WorkerPoolOptions, WorkerOptions, WorkerInterface} from '../types'; /* istanbul ignore next */ const emptyMethod = () => {}; @@ -26,14 +23,12 @@ const emptyMethod = () => {}; export default class BaseWorkerPool { _stderr: Readable; _stdout: Readable; - _options: FarmOptions; + _options: WorkerPoolOptions; _workers: Array; - constructor(workerPath: string, options: FarmOptions) { + constructor(workerPath: string, options: WorkerPoolOptions) { this._options = options; - - const numWorkers = options.numWorkers || os.cpus().length - 1; - this._workers = new Array(numWorkers); + this._workers = new Array(options.numWorkers); if (!path.isAbsolute(workerPath)) { workerPath = require.resolve(workerPath); @@ -42,11 +37,12 @@ export default class BaseWorkerPool { const stdout = mergeStream(); const stderr = mergeStream(); - for (let i = 0; i < numWorkers; i++) { + const {forkOptions, maxRetries} = options; + + for (let i = 0; i < options.numWorkers; i++) { const workerOptions: WorkerOptions = { - forkOptions: options.forkOptions || {}, - maxRetries: options.maxRetries || 3, - useNodeWorkersIfPossible: options.useNodeWorkersIfPossible, + forkOptions, + maxRetries, workerId: i, workerPath, }; @@ -87,9 +83,7 @@ export default class BaseWorkerPool { } createWorker(workerOptions: WorkerOptions): WorkerInterface { - return workerOptions.useNodeWorkersIfPossible - ? new NodeThreadsWorker(workerOptions) - : new ChildProcessWorker(workerOptions); + throw Error('Missing method createWorker in WorkerPool'); } end(): void { diff --git a/packages/jest-worker/src/index.js b/packages/jest-worker/src/index.js index eeaa5f240531..26c303fd3a6e 100644 --- a/packages/jest-worker/src/index.js +++ b/packages/jest-worker/src/index.js @@ -9,16 +9,22 @@ 'use strict'; -import type {WorkerPoolInterface, FarmOptions, ChildMessage} from './types'; -import type {Readable} from 'stream'; - +import os from 'os'; import {CHILD_MESSAGE_CALL, WorkerInterface} from './types'; import WorkerPool from './WorkerPool'; -import WorkerQueueManager from './WorkerQueueManager'; +import QueueManager from './QueueManager'; + +import type { + WorkerPoolInterface, + WorkerPoolOptions, + FarmOptions, + ChildMessage, +} from './types'; +import type {Readable} from 'stream'; function getExposedMethods( workerPath: string, - options?: FarmOptions = {}, + options: FarmOptions, ): $ReadOnlyArray { let exposedMethods = options.exposedMethods; @@ -79,24 +85,33 @@ export default class JestWorker { _cacheKeys: {[string]: WorkerInterface, __proto__: null}; _ending: boolean; _options: FarmOptions; - _queueManager: WorkerQueueManager; + _queueManager: QueueManager; _threadPool: WorkerPoolInterface; - constructor(workerPath: string, options?: FarmOptions = {}) { + constructor(workerPath: string, options?: FarmOptions) { this._cacheKeys = Object.create(null); - this._options = Object.assign({}, options, { - useNodeWorkersIfPossible: canUseWorkerThreads(), - }); + this._options = Object.assign({}, options); + + const workerPoolOptions: WorkerPoolOptions = { + forkOptions: this._options.forkOptions || {}, + maxRetries: this._options.maxRetries || 3, + numWorkers: os.cpus().length - 1, + useWorkers: canUseWorkerThreads(), + }; this._threadPool = this._options.WorkerPool - ? new this._options.WorkerPool(workerPath, this._options) - : new WorkerPool(workerPath, this._options); - this._queueManager = new WorkerQueueManager(this._threadPool); + ? new this._options.WorkerPool(workerPath, workerPoolOptions) + : new WorkerPool(workerPath, workerPoolOptions); + + this._queueManager = new QueueManager( + workerPoolOptions.numWorkers, + this._threadPool.send.bind(this._threadPool), + ); this._bindExposedWorkerMethods(workerPath, this._options); } - _bindExposedWorkerMethods(workerPath: string, options?: FarmOptions): void { + _bindExposedWorkerMethods(workerPath: string, options: FarmOptions): void { getExposedMethods(workerPath, options).forEach(name => { if (name.startsWith('_')) { return; @@ -135,11 +150,7 @@ export default class JestWorker { } }; - const onEnd: onEnd = ( - error: Error, - result: mixed, - worker: WorkerInterface, - ) => { + const onEnd: onEnd = (error: Error, result: mixed) => { if (error) { reject(error); } else { @@ -148,9 +159,11 @@ export default class JestWorker { }; const task = {onEnd, onStart, request}; - const workerId = worker ? worker.getWorkerId() : undefined; - - this._queueManager.enqueue(task, workerId); + if (worker) { + this._queueManager.enqueue(task, worker.getWorkerId()); + } else { + this._queueManager.push(task); + } }); } diff --git a/packages/jest-worker/src/types.js b/packages/jest-worker/src/types.js index 2fd34c789833..e0ab87c06b38 100644 --- a/packages/jest-worker/src/types.js +++ b/packages/jest-worker/src/types.js @@ -64,17 +64,22 @@ export type FarmOptions = { numWorkers?: number, WorkerPool?: ( workerPath: string, - options?: FarmOptions, + options?: WorkerPoolOptions, ) => WorkerPoolInterface, - useNodeWorkersIfPossible?: boolean, }; +export type WorkerPoolOptions = {| + forkOptions: ForkOptions, + maxRetries: number, + numWorkers: number, + useWorkers: boolean, +|}; + export type WorkerOptions = {| forkOptions: ForkOptions, maxRetries: number, workerId: number, workerPath: string, - useNodeWorkersIfPossible?: boolean, |}; // Messages passed from the parent to the children. @@ -132,7 +137,7 @@ export type ParentMessage = ParentMessageOk | ParentMessageError; // Queue types. export type OnStart = WorkerInterface => void; -export type OnEnd = (?Error, ?any, WorkerInterface) => void; +export type OnEnd = (?Error, ?any) => void; export type QueueChildMessage = {| request: ChildMessage, diff --git a/packages/jest-worker/src/workers/ChildProcessWorker.js b/packages/jest-worker/src/workers/ChildProcessWorker.js index ab2a44af9083..0c8fd4941d03 100644 --- a/packages/jest-worker/src/workers/ChildProcessWorker.js +++ b/packages/jest-worker/src/workers/ChildProcessWorker.js @@ -81,7 +81,7 @@ export default class ChildProcessWorker implements WorkerInterface { onMessage(response: any /* Should be ParentMessage */) { switch (response[0]) { case PARENT_MESSAGE_OK: - this._onProcessEnd(null, response[1], this); + this._onProcessEnd(null, response[1]); break; case PARENT_MESSAGE_ERROR: @@ -103,7 +103,7 @@ export default class ChildProcessWorker implements WorkerInterface { } } - this._onProcessEnd(error, null, this); + this._onProcessEnd(error, null); break; default: diff --git a/packages/jest-worker/src/workers/NodeThreadsWorker.js b/packages/jest-worker/src/workers/NodeThreadsWorker.js index ee00229377fb..2401719a5bbd 100644 --- a/packages/jest-worker/src/workers/NodeThreadsWorker.js +++ b/packages/jest-worker/src/workers/NodeThreadsWorker.js @@ -70,7 +70,7 @@ export default class ExpirementalWorker implements WorkerInterface { onMessage(response: any /* Should be ParentMessage */) { switch (response[0]) { case PARENT_MESSAGE_OK: - this._onProcessEnd(null, response[1], this); + this._onProcessEnd(null, response[1]); break; case PARENT_MESSAGE_ERROR: @@ -92,7 +92,7 @@ export default class ExpirementalWorker implements WorkerInterface { } } - this._onProcessEnd(error, null, this); + this._onProcessEnd(error, null); break; default: From d0041a9fa4ebc99f7c8d8cef1fe1e427d233c6ef Mon Sep 17 00:00:00 2001 From: Rick Hanlon Date: Sun, 25 Nov 2018 21:06:17 +0000 Subject: [PATCH 20/35] Update jest-worker --- .gitignore | 1 + .idea/codeStyles/Project.xml | 36 - .idea/codeStyles/codeStyleConfig.xml | 5 - .idea/inspectionProfiles/Project_Default.xml | 6 - .idea/jest.iml | 12 - .idea/misc.xml | 6 - .idea/modules.xml | 8 - .idea/vcs.xml | 6 - .idea/workspace.xml | 1718 ----------------- e2e/setup-files-after-env-config/package.json | 7 - .../__tests__/basic-support.test.js | 1 - .../__snapshots__/basic-support.test.js.snap | 7 - .../__tests__/basic-support.test.js | 1 - .../src/{QueueManager.js => Farm.js} | 112 +- packages/jest-worker/src/Worker.js | 233 --- packages/jest-worker/src/WorkerPool.js | 24 +- .../src/__performance_tests__/test.js | 11 +- .../workers/jest_worker.js | 7 +- .../src/__performance_tests__/workers/pi.js | 7 +- .../workers/worker_farm.js | 7 +- .../jest-worker/src/__tests__/Farm.test.js | 274 +++ .../src/__tests__/WorkerPool.test.js | 104 + .../src/__tests__/index-integration.test.js | 151 -- .../jest-worker/src/__tests__/index.test.js | 381 +--- .../src/__tests__/process-integration.test.js | 157 ++ .../src/__tests__/thread-integration.test.js | 158 ++ .../jest-worker/src/base/BaseWorkerPool.js | 3 +- .../src/base/__tests__/BaseWorkerPool.test.js | 224 +++ packages/jest-worker/src/index.js | 74 +- packages/jest-worker/src/types.js | 2 +- packages/jest-worker/src/worker.js | 233 --- .../src/workers/ChildProcessWorker.js | 19 + .../src/workers/NodeThreadsWorker.js | 28 +- .../__tests__/ChildProcessWorker.test.js | 116 +- .../__tests__/NodeThreadsWorker.test.js | 282 +++ .../__tests__/processChild.test.js} | 10 +- .../src/workers/__tests__/threadChild.test.js | 407 ++++ .../jest-worker/src/workers/processChild.js | 9 +- .../jest-worker/src/workers/threadChild.js | 134 +- 39 files changed, 1967 insertions(+), 3014 deletions(-) delete mode 100644 .idea/codeStyles/Project.xml delete mode 100644 .idea/codeStyles/codeStyleConfig.xml delete mode 100644 .idea/inspectionProfiles/Project_Default.xml delete mode 100644 .idea/jest.iml delete mode 100644 .idea/misc.xml delete mode 100644 .idea/modules.xml delete mode 100644 .idea/vcs.xml delete mode 100644 .idea/workspace.xml delete mode 100644 e2e/setup-files-after-env-config/package.json delete mode 100644 e2e/toMatchInlineSnapshot/__tests__/basic-support.test.js delete mode 100644 e2e/toMatchSnapshot/__tests__/__snapshots__/basic-support.test.js.snap delete mode 100644 e2e/toMatchSnapshot/__tests__/basic-support.test.js rename packages/jest-worker/src/{QueueManager.js => Farm.js} (50%) delete mode 100644 packages/jest-worker/src/Worker.js create mode 100644 packages/jest-worker/src/__tests__/Farm.test.js create mode 100644 packages/jest-worker/src/__tests__/WorkerPool.test.js delete mode 100644 packages/jest-worker/src/__tests__/index-integration.test.js create mode 100644 packages/jest-worker/src/__tests__/process-integration.test.js create mode 100644 packages/jest-worker/src/__tests__/thread-integration.test.js create mode 100644 packages/jest-worker/src/base/__tests__/BaseWorkerPool.test.js delete mode 100644 packages/jest-worker/src/worker.js rename packages/jest-worker/src/{ => workers}/__tests__/ChildProcessWorker.test.js (69%) create mode 100644 packages/jest-worker/src/workers/__tests__/NodeThreadsWorker.test.js rename packages/jest-worker/src/{__tests__/child.test.js => workers/__tests__/processChild.test.js} (97%) create mode 100644 packages/jest-worker/src/workers/__tests__/threadChild.test.js diff --git a/.gitignore b/.gitignore index b04ea25d91b8..60ba0f22d861 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.idea .DS_STORE .eslintcache *.swp diff --git a/.idea/codeStyles/Project.xml b/.idea/codeStyles/Project.xml deleted file mode 100644 index ff4a08b69e63..000000000000 --- a/.idea/codeStyles/Project.xml +++ /dev/null @@ -1,36 +0,0 @@ - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml deleted file mode 100644 index 79ee123c2b23..000000000000 --- a/.idea/codeStyles/codeStyleConfig.xml +++ /dev/null @@ -1,5 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml deleted file mode 100644 index c6cc8c8196a2..000000000000 --- a/.idea/inspectionProfiles/Project_Default.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/jest.iml b/.idea/jest.iml deleted file mode 100644 index 24643cc37449..000000000000 --- a/.idea/jest.iml +++ /dev/null @@ -1,12 +0,0 @@ - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index 3668dc8caa1b..000000000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index cd6cbe53a60e..000000000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 94a25f7f4cb4..000000000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml deleted file mode 100644 index a12b438eccbb..000000000000 --- a/.idea/workspace.xml +++ /dev/null @@ -1,1718 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - runInBand - 1 - runIn - ._rec - - - - - - - - - - - - - - true - - true - true - - - true - DEFINITION_ORDER - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -