feat: introduce global configuration for retry setting (fix #3598) (#3603)

Co-authored-by: Vladimir <sleuths.slews0s@icloud.com>

Author: HenryZhang-ZHY

docs/config/index.md

@@ -1452,3 +1452,12 @@ This config option affects truncating values in `test.each` titles and inside th
 Stop test execution when given number of tests have failed.
 
 By default Vitest will run all of your test cases even if some of them fail. This may not be desired for CI builds where you are only interested in 100% successful builds and would like to stop test execution as early as possible when test failures occur. The `bail` option can be used to speed up CI runs by preventing it from running more tests when failures have occured.
+
+### retry
+
+- **Type:** `number`
+- **Default:** `0`
+- **CLI:** `--retry=<value>`
+- **Version:** Since Vitest 0.32.3
+
+Retry the test specific number of times if it fails.

docs/guide/cli.md

@@ -93,6 +93,7 @@ Run only [benchmark](https://vitest.dev/guide/features.html#benchmarking-experim
 | `--inspect` | Enables Node.js inspector |
 | `--inspect-brk` | Enables Node.js inspector with break |
 | `--bail <number>` | Stop test execution when given number of tests have failed |
+| `--retry <times>` | Retry the test specific number of times if it fails |
 | `-h, --help` | Display available CLI options |
 
 ::: tip

packages/runner/src/suite.ts

@@ -78,7 +78,7 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m
       mode,
       suite: undefined!,
       fails: this.fails,
-      retry: options?.retry,
+      retry: options?.retry ?? runner.config.retry,
       repeats: options?.repeats,
       meta: Object.create(null),
     } as Omit<Test, 'context'> as Test

packages/runner/src/types/runner.ts

@@ -20,6 +20,7 @@ export interface VitestRunnerConfig {
   maxConcurrency: number
   testTimeout: number
   hookTimeout: number
+  retry: number
 }
 
 export type VitestRunnerImportSource = 'collect' | 'setup'

packages/vitest/src/node/cli.ts

@@ -48,6 +48,7 @@ cli
   .option('--inspect-brk', 'Enable Node.js inspector with break')
   .option('--test-timeout <time>', 'Default timeout of a test in milliseconds (default: 5000)')
   .option('--bail <number>', 'Stop test execution when given number of tests have failed', { default: 0 })
+  .option('--retry <times>', 'Retry the test specific number of times if it fails', { default: 0 })
   .help()
 
 cli

packages/vitest/src/types/config.ts

@@ -597,6 +597,13 @@ export interface InlineConfig {
    * Stop test execution when given number of tests have failed.
    */
   bail?: number
+
+  /**
+   * Retry the test specific number of times if it fails.
+   *
+   * @default 0
+  */
+  retry?: number
 }
 
 export interface TypecheckConfig {

test/config/fixtures/retry/retry.test.ts

@@ -0,0 +1,10 @@
+import { expect, test } from 'vitest'
+
+let number = 0
+test('should passed', () => {
+  expect(number++).toBe(3)
+})
+
+test('retry but still failed', () => {
+  expect(number++).toBe(4)
+})

test/config/fixtures/retry/vitest.config.ts

@@ -0,0 +1,8 @@
+import { defineConfig } from 'vitest/config'
+
+export default defineConfig({
+  test: {
+    include: ['*.test.ts'],
+    retry: 3,
+  },
+})

test/config/test/retry.test.ts

@@ -0,0 +1,26 @@
+import path from 'node:path'
+import { describe, expect, test } from 'vitest'
+import { runVitest } from '../../test-utils'
+
+const root = path.resolve('./fixtures/retry')
+function run(testNamePattern?: string) {
+  return runVitest({
+    root,
+    testNamePattern,
+  })
+}
+
+describe('retry', () => {
+  test('should passed', async () => {
+    const { stdout } = await run('should passed')
+    expect(stdout).toContain('1 passed')
+  })
+
+  test('retry but still failed', async () => {
+    const { stdout } = await run('retry but still failed')
+    expect(stdout).toContain('expected 1 to be 4')
+    expect(stdout).toContain('expected 2 to be 4')
+    expect(stdout).toContain('expected 3 to be 4')
+    expect(stdout).toContain('1 failed')
+  })
+})