feat: implement --shard option (jestjs#12546)

.circleci/config.yml

@@ -21,26 +21,28 @@ jobs:
         type: string
     working_directory: ~/jest
     executor: node/default
+    parallelism: 4
     steps:
       - checkout
       - node/install:
           node-version: << parameters.node-version >>
       - node/install-packages: *install
       - run:
-          command: yarn test-ci-partial
+          command: yarn test-ci-partial --shard=$(expr $CIRCLE_NODE_INDEX + 1)/$CIRCLE_NODE_TOTAL
       - store_test_results:
           path: reports/junit
 
   test-jest-jasmine:
     working_directory: ~/jest
     executor: node/default
+    parallelism: 4
     steps:
       - checkout
       - node/install:
           node-version: lts/*
       - node/install-packages: *install
       - run:
-          command: JEST_JASMINE=1 yarn test-ci-partial && JEST_JASMINE=1 yarn test-leak
+          command: JEST_JASMINE=1 yarn test-ci-partial --shard=$(expr $CIRCLE_NODE_INDEX + 1)/$CIRCLE_NODE_TOTAL && JEST_JASMINE=1 yarn test-leak
       - store_test_results:
           path: reports/junit
 

.github/workflows/nodejs.yml

@@ -65,13 +65,15 @@ jobs:
         run: yarn lint:prettier:ci
       - name: check copyright headers
         run: yarn check-copyright-headers
+
   test:
-    name: Node v${{ matrix.node-version }} on ${{ matrix.os }}
+    name: Node v${{ matrix.node-version }} on ${{ matrix.os }} (${{ matrix.shard }})
     strategy:
       fail-fast: false
       matrix:
         node-version: [12.x, 14.x, 16.x, 17.x]
         os: [ubuntu-latest, macOS-latest, windows-latest]
+        shard: ['1/4', '2/4', '3/4', '4/4']
     runs-on: ${{ matrix.os }}
     needs: prepare-yarn-cache
 
@@ -96,14 +98,15 @@ jobs:
         id: cpu-cores
         uses: SimenB/github-actions-cpu-cores@v1
       - name: run tests
-        run: yarn test-ci-partial:parallel --max-workers ${{ steps.cpu-cores.outputs.count }}
+        run: yarn test-ci-partial:parallel --max-workers ${{ steps.cpu-cores.outputs.count }} --shard=${{ matrix.shard }}
 
   test-jasmine:
-    name: Node LTS on ${{ matrix.os }} using jest-jasmine2
+    name: Node LTS on ${{ matrix.os }} using jest-jasmine2 (${{ matrix.shard }})
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macOS-latest, windows-latest]
+        shard: ['1/4', '2/4', '3/4', '4/4']
     runs-on: ${{ matrix.os }}
     needs: prepare-yarn-cache
 
@@ -128,10 +131,14 @@ jobs:
         id: cpu-cores
         uses: SimenB/github-actions-cpu-cores@v1
       - name: run tests using jest-jasmine
-        run: yarn jest-jasmine-ci --max-workers ${{ steps.cpu-cores.outputs.count }}
+        run: yarn jest-jasmine-ci --max-workers ${{ steps.cpu-cores.outputs.count }} --shard=${{ matrix.shard }}
 
   test-coverage:
-    name: Node LTS on Ubuntu with coverage
+    name: Node LTS on Ubuntu with coverage (${{ matrix.shard }})
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: ['1/4', '2/4', '3/4', '4/4']
     runs-on: ubuntu-latest
     needs: prepare-yarn-cache
 
@@ -151,7 +158,7 @@ jobs:
         uses: SimenB/github-actions-cpu-cores@v1
       - name: run tests with coverage
         run: |
-          yarn jest-coverage --color --config jest.config.ci.js --max-workers ${{ steps.cpu-cores.outputs.count }}
+          yarn jest-coverage --color --config jest.config.ci.js --max-workers ${{ steps.cpu-cores.outputs.count }} --shard=${{ matrix.shard }}
           yarn test-leak
       - name: map coverage
         run: node ./scripts/mapCoverage.js

CHANGELOG.md

@@ -6,6 +6,7 @@
 - `[expect]` Expose `AsymmetricMatchers`, `MatcherFunction` and `MatcherFunctionWithState` interfaces ([#12363](https://github.com/facebook/jest/pull/12363), [#12376](https://github.com/facebook/jest/pull/12376))
 - `[jest-circus, jest-jasmine2]` Allowed classes and functions as `describe` and `it`/`test` names ([#12484](https://github.com/facebook/jest/pull/12484))
 - `[jest-cli, jest-config]` [**BREAKING**] Remove `testURL` config, use `testEnvironmentOptions.url` instead ([#10797](https://github.com/facebook/jest/pull/10797))
+- `[jest-cli, jest-core]` Add `--shard` parameter for distributed parallel test execution ([#12546](https://github.com/facebook/jest/pull/12546))
 - `[jest-config]` [**BREAKING**] Stop shipping `jest-environment-jsdom` by default ([#12354](https://github.com/facebook/jest/pull/12354))
 - `[jest-config]` [**BREAKING**] Stop shipping `jest-jasmine2` by default ([#12355](https://github.com/facebook/jest/pull/12355))
 - `[jest-config, @jest/types]` Add `ci` to `GlobalConfig` ([#12378](https://github.com/facebook/jest/pull/12378))

docs/CLI.md

@@ -338,6 +338,24 @@ Run only the tests of the specified projects. Jest uses the attribute `displayNa
 
 A list of paths to modules that run some code to configure or to set up the testing framework before each test. Beware that files imported by the setup scripts will not be mocked during testing.
 
+### `--shard`
+
+The test suite shard to execute in a format of `(?<shardIndex>\d+)/(?<shardCount>\d+)`.
+
+`shardIndex` describes which shard to select while `shardCount` controls the number of shards the suite should be split into.
+
+`shardIndex` and `shardCount` have to be 1-based, positive numbers, and `shardIndex` has to be lower than or equal to `shardCount`.
+
+When `shard` is specified the configured [`testSquencer`](Configuration.md#testsequencer-string) has to implement a `shard` method.
+
+For example, to split the suite into three shards, each running one third of the tests:
+
+```
+jest --shard=1/3
+jest --shard=2/3
+jest --shard=3/3
+```
+
 ### `--showConfig`
 
 Print your Jest config and then exits.
@@ -389,7 +407,7 @@ Lets you specify a custom test runner.
 
 ### `--testSequencer=<path>`
 
-Lets you specify a custom test sequencer. Please refer to the documentation of the corresponding configuration property for details.
+Lets you specify a custom test sequencer. Please refer to the [`testSequencer` configuration](Configuration.md#testsequencer-string) for details.
 
 ### `--testTimeout=<number>`
 

docs/Configuration.md

@@ -1327,7 +1327,13 @@ An example of such function can be found in our default [jasmine2 test runner pa
 
 Default: `@jest/test-sequencer`
 
-This option allows you to use a custom sequencer instead of Jest's default. `sort` may optionally return a Promise.
+This option allows you to use a custom sequencer instead of Jest's default.
+
+:::tip
+
+Both `sort` and `shard` may optionally return a `Promise`.
+
+:::
 
 Example:
 
@@ -1337,6 +1343,24 @@ Sort test path alphabetically.
 const Sequencer = require('@jest/test-sequencer').default;
 
 class CustomSequencer extends Sequencer {
+  /**
+   * Select tests for shard requested via --shard=shardIndex/shardCount
+   * Sharding is applied before sorting
+   */
+  shard(tests, {shardIndex, shardCount}) {
+    const shardSize = Math.ceil(tests.length / options.shardCount);
+    const shardStart = shardSize * (options.shardIndex - 1);
+    const shardEnd = shardSize * options.shardIndex;
+
+    return [...tests]
+      .sort((a, b) => (a.path > b.path ? 1 : -1))
+      .slice(shardStart, shardEnd);
+  }
+
+  /**
+   * Sort test to determine order of execution
+   * Sorting is applied after sharding
+   */
   sort(tests) {
     // Test structure information
     // https://github.com/facebook/jest/blob/6b8b1404a1d9254e7d5d90a8934087a9c9899dab/packages/jest-runner/src/types.ts#L17-L21

docs/Troubleshooting.md

@@ -182,6 +182,18 @@ jest --maxWorkers=4
 yarn test --maxWorkers=4
 ```
 
+If you use GitHub Actions, you can use [`github-actions-cpu-cores`](https://github.com/SimenB/github-actions-cpu-cores) to detect number of CPUs, and pass that to Jest.
+
+```yaml
+- name: Get number of CPU cores
+  id: cpu-cores
+  uses: SimenB/github-actions-cpu-cores@v1
+- name: run tests
+  run: yarn jest --max-workers ${{ steps.cpu-cores.outputs.count }}
+```
+
+Another thing you can do is use the [`shard`](CLI.md#--shard) flag to parallelize the test run across multiple machines.
+
 ## `coveragePathIgnorePatterns` seems to not have any effect.
 
 Make sure you are not using the `babel-plugin-istanbul` plugin. Jest wraps Istanbul, and therefore also tells Istanbul what files to instrument with coverage collection. When using `babel-plugin-istanbul`, every file that is processed by Babel will have coverage collection code, hence it is not being ignored by `coveragePathIgnorePatterns`.

e2e/__tests__/shard.test.ts

@@ -0,0 +1,86 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+import * as path from 'path';
+import runJest from '../runJest';
+
+test('--shard=1/1', () => {
+  const result = runJest('shard', ['--shard=1/1', '--listTests']);
+
+  const paths = result.stdout
+    .split('\n')
+    .filter(Boolean)
+    .map(file => path.basename(file))
+    .sort();
+
+  expect(paths).toEqual(['1.test.js', '2.test.js', '3.test.js']);
+});
+
+test('--shard=1/2', () => {
+  const result = runJest('shard', ['--shard=1/2', '--listTests']);
+
+  const paths = result.stdout
+    .split('\n')
+    .filter(Boolean)
+    .map(file => path.basename(file))
+    .sort();
+
+  expect(paths).toEqual(['1.test.js', '3.test.js']);
+});
+
+test('--shard=2/2', () => {
+  const result = runJest('shard', ['--shard=2/2', '--listTests']);
+
+  const paths = result.stdout
+    .split('\n')
+    .filter(Boolean)
+    .map(file => path.basename(file));
+
+  expect(paths).toEqual(['2.test.js']);
+});
+
+test('--shard=4/4', () => {
+  const result = runJest('shard', ['--shard=4/4', '--listTests']);
+
+  const paths = result.stdout
+    .split('\n')
+    .filter(Boolean)
+    .map(file => path.basename(file));
+
+  // project only has 3 files
+  // shards > 3 are empty
+  expect(paths).toEqual([]);
+});
+
+test('--shard=1/2 custom non-sharding test sequencer', () => {
+  const result = runJest('shard', [
+    '--shard=1/2',
+    '--listTests',
+    '--testSequencer=./no-sharding-test-sequencer.js',
+  ]);
+
+  expect(result).toMatchObject({
+    failed: true,
+    stderr: expect.stringMatching(
+      /Shard (.*) requested, but test sequencer (.*) in (.*) has no shard method./,
+    ),
+  });
+});
+
+test('--shard=1/2 custom sharding test sequencer', () => {
+  const result = runJest('shard', [
+    '--shard=1/2',
+    '--listTests',
+    '--testSequencer=./sharding-test-sequencer.js',
+  ]);
+
+  const paths = result.stdout
+    .split('\n')
+    .filter(Boolean)
+    .map(file => path.basename(file));
+
+  expect(paths).toEqual(['3.test.js']);
+});

e2e/shard/__tests__/1.test.js

@@ -0,0 +1,13 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+'use strict';
+
+test('1-1', () => expect(2).toBe(2));
+
+test('1-2', () => expect(2).toBe(2));
+
+test('1-3', () => expect(2).toBe(2));

e2e/shard/__tests__/2.test.js

@@ -0,0 +1,13 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+'use strict';
+
+test('2-1', () => expect(2).toBe(2));
+
+test('2-2', () => expect(2).toBe(2));
+
+test('2-3', () => expect(2).toBe(2));

e2e/shard/__tests__/3.test.js

@@ -0,0 +1,13 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+'use strict';
+
+test('3-1', () => expect(2).toBe(2));
+
+test('3-2', () => expect(2).toBe(2));
+
+test('3-3', () => expect(2).toBe(2));

e2e/shard/no-sharding-test-sequencer.js

@@ -0,0 +1,11 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+module.exports = class NoShardingSequencer {
+  sort(tests) {
+    return tests;
+  }
+};

e2e/shard/package.json

@@ -0,0 +1,5 @@
+{
+  "jest": {
+    "testEnvironment": "node"
+  }
+}

e2e/shard/sharding-test-sequencer.js

@@ -0,0 +1,18 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+module.exports = class NoShardingSequencer {
+  shard(tests) {
+    return [
+      Array.from(tests).sort((a, b) =>
+        a.path < b.path ? -1 : a.path > b.path ? 1 : 0,
+      )[2],
+    ];
+  }
+  sort(tests) {
+    return tests;
+  }
+};