feat!(runner): support concurrent suites (#5491)

docs/api/index.md

@@ -684,15 +684,18 @@ In order to do that run `vitest` with specific file containing the tests in ques
 
 - **Alias:** `suite.concurrent`
 
-`describe.concurrent` in a suite marks every tests as concurrent
+`describe.concurrent` runs all inner suites and tests in parallel
 
 ```ts twoslash
 import { describe, test } from 'vitest'
 // ---cut---
-// All tests within this suite will be run in parallel
+// All suites and tests within this suite will be run in parallel
 describe.concurrent('suite', () => {
   test('concurrent test 1', async () => { /* ... */ })
-  test('concurrent test 2', async () => { /* ... */ })
+  describe('concurrent suite 2', async () => {
+    test('concurrent test inner 1', async () => { /* ... */ })
+    test('concurrent test inner 2', async () => { /* ... */ })
+  })
   test.concurrent('concurrent test 3', async () => { /* ... */ })
 })
 ```

packages/runner/src/run.ts

@@ -332,8 +332,7 @@ export async function runSuite(suite: Suite, runner: VitestRunner) {
       else {
         for (let tasksGroup of partitionSuiteChildren(suite)) {
           if (tasksGroup[0].concurrent === true) {
-            const mutex = limit(runner.config.maxConcurrency)
-            await Promise.all(tasksGroup.map(c => mutex(() => runSuiteChild(c, runner))))
+            await Promise.all(tasksGroup.map(c => runSuiteChild(c, runner)))
           }
           else {
             const { sequence } = runner.config
@@ -386,15 +385,19 @@ export async function runSuite(suite: Suite, runner: VitestRunner) {
   }
 }
 
+let limitMaxConcurrency: ReturnType<typeof limit>
+
 async function runSuiteChild(c: Task, runner: VitestRunner) {
   if (c.type === 'test' || c.type === 'custom')
-    return runTest(c, runner)
+    return limitMaxConcurrency(() => runTest(c, runner))
 
   else if (c.type === 'suite')
     return runSuite(c, runner)
 }
 
 export async function runFiles(files: File[], runner: VitestRunner) {
+  limitMaxConcurrency ??= limit(runner.config.maxConcurrency)
+
   for (const file of files) {
     if (!file.tasks.length && !runner.config.passWithNoTests) {
       if (!file.result?.errors?.length) {

packages/runner/src/suite.ts

@@ -222,6 +222,7 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m
       shuffle,
       tasks: [],
       meta: Object.create(null),
+      concurrent: suiteOptions?.concurrent,
     }
 
     if (runner && includeLocation && runner.config.includeTaskLocation) {

packages/runner/src/types/tasks.ts

@@ -180,7 +180,7 @@ export interface TestOptions {
    */
   repeats?: number
   /**
-   * Whether tests run concurrently.
+   * Whether suites and tests run concurrently.
    * Tests inherit `concurrent` from `describe()` and nested `describe()` will inherit from parent's `concurrent`.
    */
   concurrent?: boolean

test/cli/fixtures/fails/concurrent-suite-deadlock.test.ts

@@ -0,0 +1,41 @@
+import { createDefer } from '@vitest/utils'
+import { describe, test, vi } from 'vitest'
+
+// 3 tests depend on each other,
+// so they will deadlock when maxConcurrency < 3
+//
+//  [a]  [b]  [c]
+//   * ->
+//        * ->
+//          <- *
+//     <------
+
+vi.setConfig({ maxConcurrency: 2 })
+
+describe('wrapper', { concurrent: true, timeout: 500 }, () => {
+  const defers = [
+    createDefer<void>(),
+    createDefer<void>(),
+    createDefer<void>(),
+  ]
+
+  describe('1st suite', () => {
+    test('a', async () => {
+      defers[0].resolve()
+      await defers[2]
+    })
+
+    test('b', async () => {
+      await defers[0]
+      defers[1].resolve()
+      await defers[2]
+    })
+  })
+
+  describe('2nd suite', () => {
+    test('c', async () => {
+      await defers[1]
+      defers[2].resolve()
+    })
+  })
+})

test/cli/fixtures/fails/concurrent-test-deadlock.test.ts

@@ -0,0 +1,37 @@
+import { describe, test, vi } from 'vitest'
+import { createDefer } from '@vitest/utils'
+
+// 3 tests depend on each other,
+// so they will deadlock when maxConcurrency < 3
+//
+//  [a]  [b]  [c]
+//   * ->
+//        * ->
+//          <- *
+//     <------
+
+vi.setConfig({ maxConcurrency: 2 })
+
+describe('wrapper', { concurrent: true, timeout: 500 }, () => {
+  const defers = [
+    createDefer<void>(),
+    createDefer<void>(),
+    createDefer<void>(),
+  ]
+
+  test('a', async () => {
+    defers[0].resolve()
+    await defers[2]
+  })
+
+  test('b', async () => {
+    await defers[0]
+    defers[1].resolve()
+    await defers[2]
+  })
+
+  test('c', async () => {
+    await defers[1]
+    defers[2].resolve()
+  })
+})

test/cli/package.json

@@ -10,6 +10,7 @@
     "@types/ws": "^8.5.9",
     "@vitejs/plugin-basic-ssl": "^1.0.2",
     "@vitest/runner": "workspace:^",
+    "@vitest/utils": "workspace:*",
     "debug": "^4.3.4",
     "execa": "^8.0.1",
     "unplugin-swc": "^1.4.4",

test/cli/test/__snapshots__/fails.test.ts.snap

@@ -2,6 +2,10 @@
 
 exports[`should fail .dot-folder/dot-test.test.ts > .dot-folder/dot-test.test.ts 1`] = `"AssertionError: expected true to be false // Object.is equality"`;
 
+exports[`should fail concurrent-suite-deadlock.test.ts > concurrent-suite-deadlock.test.ts 1`] = `"Error: Test timed out in 500ms."`;
+
+exports[`should fail concurrent-test-deadlock.test.ts > concurrent-test-deadlock.test.ts 1`] = `"Error: Test timed out in 500ms."`;
+
 exports[`should fail each-timeout.test.ts > each-timeout.test.ts 1`] = `"Error: Test timed out in 10ms."`;
 
 exports[`should fail empty.test.ts > empty.test.ts 1`] = `"Error: No test suite found in file <rootDir>/empty.test.ts"`;

test/core/test/concurrent-suite.test.ts

@@ -0,0 +1,184 @@
+import { createDefer } from '@vitest/utils'
+import { afterAll, describe, expect, test } from 'vitest'
+
+describe('basic', () => {
+  const defers = [
+    createDefer<void>(),
+    createDefer<void>(),
+    createDefer<void>(),
+    createDefer<void>(),
+  ]
+
+  afterAll(async () => {
+    await defers[3]
+  })
+
+  describe('1st suite', { concurrent: true }, () => {
+    test('0', async () => {
+      defers[0].resolve()
+    })
+
+    test('1', async () => {
+      await defers[2] // this would deadlock if sequential
+      defers[1].resolve()
+    })
+  })
+
+  describe('2nd suite', { concurrent: true }, () => {
+    test('2', async () => {
+      await defers[0]
+      defers[2].resolve()
+    })
+    test('3', async () => {
+      await defers[1]
+      defers[3].resolve()
+    })
+  })
+})
+
+describe('inherits option', { concurrent: true }, () => {
+  const defers = [
+    createDefer<void>(),
+    createDefer<void>(),
+    createDefer<void>(),
+    createDefer<void>(),
+  ]
+
+  afterAll(async () => {
+    await defers[3]
+  })
+
+  describe('1st suite', () => {
+    test('0', async () => {
+      defers[0].resolve()
+    })
+
+    test('1', async () => {
+      await defers[2] // this would deadlock if sequential
+      defers[1].resolve()
+    })
+  })
+
+  describe('2nd suite', () => {
+    test('2', async () => {
+      await defers[0]
+      defers[2].resolve()
+    })
+    test('3', async () => {
+      await defers[1]
+      defers[3].resolve()
+    })
+  })
+})
+
+describe('works with describe.each', () => {
+  const defers = [
+    createDefer<void>(),
+    createDefer<void>(),
+    createDefer<void>(),
+    createDefer<void>(),
+  ]
+
+  afterAll(async () => {
+    await defers[3]
+  })
+
+  describe.each(['1st suite', '2nd suite'])('%s', { concurrent: true }, (s) => {
+    if (s === '1st suite') {
+      test('0', async () => {
+        defers[0].resolve()
+      })
+
+      test('1', async () => {
+        await defers[2] // this would deadlock if sequential
+        defers[1].resolve()
+      })
+    }
+
+    if (s === '2nd suite') {
+      test('2', async () => {
+        await defers[0]
+        defers[2].resolve()
+      })
+      test('3', async () => {
+        await defers[1]
+        defers[3].resolve()
+      })
+    }
+  })
+})
+
+describe('override concurrent', { concurrent: true }, () => {
+  checkParallelSuites()
+
+  describe('s-x', { concurrent: false }, () => {
+    checkSequentialTests()
+  })
+
+  describe.sequential('s-x-1', () => {
+    checkSequentialTests()
+  })
+
+  // TODO: not working?
+  // describe('s-x-2', { sequential: true, }, () => {
+  //   checkSequentialTests()
+  // })
+
+  describe('s-y', () => {
+    checkParallelTests()
+  })
+})
+
+const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms))
+
+function checkSequentialTests() {
+  let x = 0
+
+  test('t1', async () => {
+    await sleep(200)
+    expect(x).toBe(0)
+    x++
+  })
+
+  test('t2', async () => {
+    expect(x).toBe(1)
+  })
+}
+
+function checkParallelTests() {
+  const defers = [
+    createDefer<void>(),
+    createDefer<void>(),
+  ]
+
+  test('t1', async () => {
+    defers[0].resolve()
+    await defers[1]
+  })
+
+  test('t2', async () => {
+    await defers[0]
+    defers[1].resolve()
+  })
+}
+
+function checkParallelSuites() {
+  const defers = [
+    createDefer<void>(),
+    createDefer<void>(),
+  ]
+
+  describe('s1', () => {
+    test('t1', async () => {
+      defers[0].resolve()
+      await defers[1]
+    })
+  })
+
+  describe('s2', () => {
+    test('t1', async () => {
+      await defers[0]
+      defers[1].resolve()
+    })
+  })
+}