test: reduce usage of execa, add runVitest test utils (#3436)

test/bail/package.json

@@ -6,7 +6,6 @@
   },
   "devDependencies": {
     "@vitest/browser": "workspace:*",
-    "execa": "^7.0.0",
     "vite": "latest",
     "vitest": "workspace:*",
     "webdriverio": "latest"

test/bail/test/bail.test.ts

@@ -1,34 +1,26 @@
-import { expect, test } from 'vitest'
-import { execa } from 'execa'
+import { type UserConfig, expect, test } from 'vitest'
 
-const configs: string[][] = []
-const pools = [['--threads', 'true'], ['--threads', 'false'], ['--single-thread']]
+import { runVitest } from '../../test-utils'
+
+const configs: UserConfig[] = []
+const pools: UserConfig[] = [{ threads: true }, { threads: false }, { singleThread: true }]
 
 if (process.platform !== 'win32')
-  pools.push(['--browser'])
-
-for (const isolate of ['true', 'false']) {
-  for (const pool of pools) {
-    configs.push([
-      '--bail',
-      '1',
-      '--isolate',
-      isolate,
-      ...pool,
-    ])
-  }
+  pools.push({ browser: { enabled: true, name: 'chrome' } })
+
+for (const isolate of [true, false]) {
+  for (const pool of pools)
+    configs.push({ isolate, ...pool })
 }
 
 for (const config of configs) {
-  test(`should bail with "${config.join(' ')}"`, async () => {
-    const { exitCode, stdout } = await execa('vitest', [
-      '--no-color',
-      '--root',
-      'fixtures',
+  test(`should bail with "${JSON.stringify(config)}"`, async () => {
+    process.env.THREADS = config?.threads ? 'true' : 'false'
+
+    const { exitCode, stdout } = await runVitest({
+      root: './fixtures',
+      bail: 1,
       ...config,
-    ], {
-      reject: false,
-      env: { THREADS: config.join(' ').includes('--threads true') ? 'true' : 'false' },
     })
 
     expect(exitCode).toBe(1)

test/benchmark/package.json

@@ -4,10 +4,9 @@
   "scripts": {
     "test": "node test.mjs",
     "bench:json": "vitest bench --reporter=json",
-    "bench": "vitest bench",
-    "coverage": "vitest run --coverage"
+    "bench": "vitest bench"
   },
   "devDependencies": {
-    "execa": "^6.1.0"
+    "vitest": "workspace:*"
   }
 }

test/benchmark/test.mjs

@@ -1,19 +1,14 @@
+import { existsSync, rmSync } from 'node:fs'
 import { readFile } from 'node:fs/promises'
-import { execa } from 'execa'
+import { startVitest } from 'vitest/node'
 
-let error
-await execa('npx', ['vitest', 'bench', 'base.bench', 'mode.bench', 'only.bench'], {
-  env: {
-    ...process.env,
-    CI: 'true',
-    NO_COLOR: 'true',
-  },
-})
-  .catch((e) => {
-    error = e
-  })
+if (existsSync('./bench.json'))
+  rmSync('./bench.json')
 
-if (error) {
+try {
+  await startVitest('benchmark', ['base.bench', 'mode.bench', 'only.bench'])
+}
+catch (error) {
   console.error(error)
   process.exit(1)
 }

test/benchmark/vitest.config.ts

@@ -19,7 +19,7 @@ export default defineConfig({
         onWatcherRerun: noop,
         onServerRestart: noop,
         onUserConsoleLog: noop,
-      }, 'default'],
+      }],
     },
   },
 })

test/config/package.json

@@ -5,8 +5,6 @@
     "test": "vitest run"
   },
   "devDependencies": {
-    "execa": "^7.0.0",
-    "strip-ansi": "^7.0.1",
     "vite": "latest",
     "vitest": "workspace:*"
   }

test/config/test/failures.test.ts

@@ -1,73 +1,82 @@
 import { expect, test } from 'vitest'
+import type { UserConfig } from 'vitest/config'
 import { version } from 'vitest/package.json'
 
-import { runVitest } from './utils'
+import * as testUtils from '../../test-utils'
+
+function runVitest(config: NonNullable<UserConfig['test']> & { shard?: any }) {
+  return testUtils.runVitest(config, ['fixtures/test/'])
+}
+
+function runVitestCli(...cliArgs: string[]) {
+  return testUtils.runVitestCli('run', 'fixtures/test/', ...cliArgs)
+}
 
 test('shard cannot be used with watch mode', async () => {
-  const { error } = await runVitest('watch', ['--shard', '1/2'])
+  const { stderr } = await runVitest({ watch: true, shard: '1/2' })
 
-  expect(error).toMatch('Error: You cannot use --shard option with enabled watch')
+  expect(stderr).toMatch('Error: You cannot use --shard option with enabled watch')
 })
 
 test('shard must be positive number', async () => {
-  const { error } = await runVitest('run', ['--shard', '"-1"'])
+  const { stderr } = await runVitest({ shard: '-1' })
 
-  expect(error).toMatch('Error: --shard <count> must be a positive number')
+  expect(stderr).toMatch('Error: --shard <count> must be a positive number')
 })
 
 test('shard index must be smaller than count', async () => {
-  const { error } = await runVitest('run', ['--shard', '2/1'])
+  const { stderr } = await runVitest({ shard: '2/1' })
 
-  expect(error).toMatch('Error: --shard <index> must be a positive number less then <count>')
+  expect(stderr).toMatch('Error: --shard <index> must be a positive number less then <count>')
 })
 
 test('inspect requires changing threads or singleThread', async () => {
-  const { error } = await runVitest('run', ['--inspect'])
+  const { stderr } = await runVitest({ inspect: true })
 
-  expect(error).toMatch('Error: You cannot use --inspect without "threads: false" or "singleThread: true"')
+  expect(stderr).toMatch('Error: You cannot use --inspect without "threads: false" or "singleThread: true"')
 })
 
 test('inspect cannot be used with threads', async () => {
-  const { error } = await runVitest('run', ['--inspect', '--threads', 'true'])
+  const { stderr } = await runVitest({ inspect: true, threads: true })
 
-  expect(error).toMatch('Error: You cannot use --inspect without "threads: false" or "singleThread: true"')
+  expect(stderr).toMatch('Error: You cannot use --inspect without "threads: false" or "singleThread: true"')
 })
 
 test('inspect-brk cannot be used with threads', async () => {
-  const { error } = await runVitest('run', ['--inspect-brk', '--threads', 'true'])
+  const { stderr } = await runVitest({ inspectBrk: true, threads: true })
 
-  expect(error).toMatch('Error: You cannot use --inspect-brk without "threads: false" or "singleThread: true"')
+  expect(stderr).toMatch('Error: You cannot use --inspect-brk without "threads: false" or "singleThread: true"')
 })
 
 test('c8 coverage provider cannot be used with browser', async () => {
-  const { error } = await runVitest('run', ['--coverage.enabled', '--browser'])
+  const { stderr } = await runVitest({ coverage: { enabled: true }, browser: { enabled: true, name: 'chrome' } })
+
+  expect(stderr).toMatch('Error: @vitest/coverage-c8 does not work with --browser. Use @vitest/coverage-istanbul instead')
+})
 
-  expect(error).toMatch('Error: @vitest/coverage-c8 does not work with --browser. Use @vitest/coverage-istanbul instead')
+test('version number is printed when coverage provider fails to load', async () => {
+  const { stderr, stdout } = await runVitest({
+    coverage: {
+      enabled: true,
+      provider: 'custom',
+      customProviderModule: './non-existing-module.ts',
+    },
+  })
+
+  expect(stdout).toMatch(`RUN  v${version}`)
+  expect(stderr).toMatch('Error: Failed to load custom CoverageProviderModule from ./non-existing-module.ts')
 })
 
 test('boolean coverage flag without dot notation, with more dot notation options', async () => {
-  const { error } = await runVitest('run', ['--coverage', '--coverage.reporter', 'text'])
+  const { stderr } = await runVitestCli('--coverage', '--coverage.reporter', 'text')
 
-  expect(error).toMatch('Error: A boolean argument "--coverage" was used with dot notation arguments "--coverage.reporter".')
-  expect(error).toMatch('Please specify the "--coverage" argument with dot notation as well: "--coverage.enabled"')
+  expect(stderr).toMatch('Error: A boolean argument "--coverage" was used with dot notation arguments "--coverage.reporter".')
+  expect(stderr).toMatch('Please specify the "--coverage" argument with dot notation as well: "--coverage.enabled"')
 })
 
 test('boolean browser flag without dot notation, with more dot notation options', async () => {
-  const { error } = await runVitest('run', ['--browser', '--browser.name', 'chrome'])
-
-  expect(error).toMatch('Error: A boolean argument "--browser" was used with dot notation arguments "--browser.name".')
-  expect(error).toMatch('Please specify the "--browser" argument with dot notation as well: "--browser.enabled"')
-})
+  const { stderr } = await runVitestCli('run', '--browser', '--browser.name', 'chrome')
 
-test('version number is printed when coverage provider fails to load', async () => {
-  const { error, output } = await runVitest('run', [
-    '--coverage.enabled',
-    '--coverage.provider',
-    'custom',
-    '--coverage.customProviderModule',
-    './non-existing-module.ts',
-  ])
-
-  expect(output).toMatch(`RUN  v${version}`)
-  expect(error).toMatch('Error: Failed to load custom CoverageProviderModule from ./non-existing-module.ts')
+  expect(stderr).toMatch('Error: A boolean argument "--browser" was used with dot notation arguments "--browser.name".')
+  expect(stderr).toMatch('Please specify the "--browser" argument with dot notation as well: "--browser.enabled"')
 })