Fix retryTimes and add e2e regression test (#6762)

* Fix retryTimes and add e2e regression test

* Lint fixes

* Use dispatch for resetting test errors on retry

* Update CHANGELOG

CHANGELOG.md

@@ -8,6 +8,7 @@
 
 ### Fixes
 
+- `[jest-circus]` Fix retryTimes so errors are reset before re-running ([#6762](https://github.com/facebook/jest/pull/6762))
 - `[docs]` Update `expect.objectContaining()` description ([#6754](https://github.com/facebook/jest/pull/6754))
 - `[babel-jest]` Make `getCacheKey()` take into account `createTransformer` options ([#6699](https://github.com/facebook/jest/pull/6699))
 - `[docs]` Fix contributors link ([#6711](https://github.com/facebook/jest/pull/6711))

e2e/__tests__/test_retries.test.js

@@ -28,7 +28,14 @@ describe('Test Retries', () => {
     fs.unlinkSync(outputFilePath);
   });
 
-  it('retries failed tests if configured', () => {
+  it('retries failed tests', () => {
+    const result = runJest('test-retries', ['e2e.test.js']);
+
+    expect(result.code).toEqual(0);
+    expect(result.failed).toBe(false);
+  });
+
+  it('reporter shows more than 1 invocation if test is retried', () => {
     let jsonResult;
 
     const reporterConfig = {
@@ -59,7 +66,7 @@ describe('Test Retries', () => {
     expect(jsonResult.testResults[0].testResults[0].invocations).toBe(4);
   });
 
-  it('does not retry by default', () => {
+  it('reporter shows 1 invocation if tests are not retried', () => {
     let jsonResult;
 
     const reporterConfig = {

e2e/test-retries/__tests__/e2e.test.js

@@ -0,0 +1,29 @@
+/**
+ * Copyright (c) 2014-present, Facebook, Inc. All rights reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+'use strict';
+
+const path = require('path');
+const fs = require('fs');
+
+const countPath = path.join(__dirname, '.tries');
+
+beforeAll(() => {
+  fs.writeFileSync(countPath, '0', 'utf8');
+});
+
+jest.retryTimes(3);
+
+it('retries', () => {
+  const tries = parseInt(fs.readFileSync(countPath, 'utf8'), 10);
+  fs.writeFileSync(countPath, `${tries + 1}`, 'utf8');
+  expect(tries).toEqual(3);
+});
+
+afterAll(() => {
+  // cleanup
+  fs.unlinkSync(countPath);
+});

packages/jest-circus/src/__mocks__/test_event_handler.js

@@ -25,6 +25,7 @@ const testEventHandler: EventHandler = (event, state) => {
       break;
     }
     case 'test_start':
+    case 'test_retry':
     case 'test_done': {
       console.log(event.name + ':', event.test.name);
       break;

packages/jest-circus/src/event_handler.js

@@ -125,6 +125,10 @@ const handler: EventHandler = (event, state): void => {
       event.test.errors.push([error, asyncError]);
       break;
     }
+    case 'test_retry': {
+      event.test.errors = [];
+      break;
+    }
     case 'run_start': {
       global[TEST_TIMEOUT_SYMBOL] &&
         (state.testTimeout = global[TEST_TIMEOUT_SYMBOL]);

packages/jest-circus/src/run.js

@@ -64,6 +64,9 @@ const _runTestsForDescribeBlock = async (describeBlock: DescribeBlock) => {
     let numRetriesAvailable = retryTimes;
 
     while (numRetriesAvailable > 0 && test.errors.length > 0) {
+      // Clear errors so retries occur
+      dispatch({name: 'test_retry', test});
+
       await _runTest(test);
       numRetriesAvailable--;
     }

types/Circus.js

@@ -91,6 +91,10 @@ export type Event =
       error: Exception,
       test: TestEntry,
     |}
+  | {|
+      name: 'test_retry',
+      test: TestEntry,
+    |}
   | {|
       // the `test` in this case is all hooks + it/test function, not just the
       // function passed to `it/test`