feat: add blob reporter (#5663)

.gitignore

@@ -22,3 +22,4 @@ docs/public/sponsors
 .eslintcache
 docs/.vitepress/cache/
 !test/cli/fixtures/dotted-files/**/.cache
+.vitest-reports

README.md

@@ -32,7 +32,7 @@ Next generation testing framework powered by Vite.
 
 - [Vite](https://vitejs.dev/)'s config, transformers, resolvers, and plugins. Use the same setup from your app!
 - [Jest Snapshot](https://jestjs.io/docs/snapshot-testing)
-- [Chai](https://www.chaijs.com/) built-in for assertions, with [Jest expect](https://jestjs.io/docs/expect) compatible APIs.
+- [Chai](https://www.chaijs.com/) built-in for assertions, with [Jest expect](https://jestjs.io/docs/expect) compatible APIs
 - [Smart & instant watch mode](https://vitest.dev/guide/features.html#watch-mode), like HMR for tests!
 - [Native code coverage](https://vitest.dev/guide/features.html#coverage) via [`v8`](https://v8.dev/blog/javascript-code-coverage) or [`istanbul`](https://istanbul.js.org/).
 - [Tinyspy](https://github.com/tinylibs/tinyspy) built-in for mocking, stubbing, and spies.
@@ -45,6 +45,7 @@ Next generation testing framework powered by Vite.
 - ESM first, top level await
 - Out-of-box TypeScript / JSX support
 - Filtering, timeouts, concurrent for suite and tests
+- Sharding support
 
 > Vitest 1.0 requires Vite >=v5.0.0 and Node >=v18.0.0
 

docs/.vitepress/components.d.ts

@@ -1,10 +1,10 @@
 /* eslint-disable */
-/* prettier-ignore */
 // @ts-nocheck
 // Generated by unplugin-vue-components
 // Read more: https://github.com/vuejs/core/pull/3399
 export {}
 
+/* prettier-ignore */
 declare module 'vue' {
   export interface GlobalComponents {
     Contributors: typeof import('./components/Contributors.vue')['default']

docs/.vitepress/components/FeaturesList.vue

@@ -4,7 +4,7 @@
     dir="auto"
     flex="~ col gap2 md:gap-3"
   >
-    <ListItem><a target="_blank" href="https://vitejs.dev" rel="noopener noreferrer">Vite</a>'s config, transformers, resolvers, and plugins.</ListItem>
+    <ListItem><a target="_blank" href="https://vitejs.dev" rel="noopener noreferrer">Vite</a>'s config, transformers, resolvers, and plugins</ListItem>
     <ListItem>Use the same setup from your app to run the tests!</ListItem>
     <ListItem><a target="_blank" href="https://twitter.com/antfu7/status/1468233216939245579" rel="noopener noreferrer">Smart & instant watch mode, like HMR for tests!</a></ListItem>
     <ListItem>Component testing for Vue, React, Svelte, Lit, Marko and more</ListItem>
@@ -25,6 +25,7 @@
     <ListItem>Code coverage via <a target="_blank" href="https://v8.dev/blog/javascript-code-coverage" rel="noopener noreferrer">v8</a> or <a target="_blank" href="https://istanbul.js.org/" rel="noopener noreferrer">istanbul</a></ListItem>
     <ListItem>Rust-like <a href="/guide/in-source">in-source testing</a></ListItem>
     <ListItem>Type Testing via <a target="_blank" href="https://github.com/mmkal/expect-type" rel="noopener noreferrer">expect-type</a></ListItem>
+    <ListItem>Sharding support</ListItem>
   </ul>
 </template>
 

docs/guide/cli-table.md

@@ -120,3 +120,4 @@
 | `--no-color` | Removes colors from the console output |
 | `--clearScreen` | Clear terminal screen when re-running tests during watch mode (default: `true`) |
 | `--standalone` | Start Vitest without running tests. File filters will be ignored, tests will be running only on change (default: `false`) |
+| `--mergeReports [path]` | Paths to blob reports directory. If this options is used, Vitest won't run any tests, it will only report previously recorded tests |

docs/guide/cli.md

@@ -113,4 +113,18 @@ vitest --api=false
 You cannot use this option with `--watch` enabled (enabled in dev by default).
 :::
 
+::: tip
+If `--reporter=blob` is used without an output file, the default path will include the current shard config to avoid collisions with other Vitest processes.
+:::
+
+### merge-reports
+
+- **Type:** `boolean | string`
+
+Merges every blob report located in the specified folder (`.vitest-reports` by default). You can use any reporters with this command (except [`blob`](/guide/reporters#blob-reporter)):
+
+```sh
+vitest --merge-reports --reporter=junit
+```
+
 [cac's dot notation]: https://github.com/cacjs/cac#dot-nested-options

docs/guide/features.md

@@ -229,3 +229,14 @@ test('my types work properly', () => {
   assertType(mount({ name: 42 }))
 })
 ```
+
+## Sharding
+
+Run tests on different machines using [`--shard`](/guide/cli#shard) and [`--reporter=blob`](/guide/reporters#blob-reporter) flags.
+All test results can be merged at the end of your CI pipeline using `--merge-reports` command:
+
+```bash
+vitest --shard=1/2 --reporter=blob
+vitest --shard=2/2 --reporter=blob
+vitest --merge-reports --reporter=junit
+```

docs/guide/reporters.md

@@ -462,6 +462,26 @@ export default defineConfig({
 <img alt="Github Actions" img-dark src="https://github.com/vitest-dev/vitest/assets/4232207/336cddc2-df6b-4b8a-8e72-4d00010e37f5">
 <img alt="Github Actions" img-light src="https://github.com/vitest-dev/vitest/assets/4232207/ce8447c1-0eab-4fe1-abef-d0d322290dca">
 
+### Blob Reporter
+
+Stores test results on the machine so they can be later merged using [`--merge-reports`](/guide/cli#merge-reports) command.
+By default, stores all results in `.vitest-reports` folder, but can be overriden with `--outputFile` or `--outputFile.blob` flags.
+
+```bash
+npx vitest --reporter=blob --outputFile=reports/blob-1.json
+```
+
+We recommend using this reporter if you are running Vitest on different machines with the [`--shard`](/guide/cli#shard) flag.
+All blob reports can be merged into any report by using `--merge-reports` command at the end of your CI pipeline:
+
+```bash
+npx vitest --merge-reports=reports --reporter=json --reporter=default
+```
+
+::: tip
+Both `--reporter=blob` and `--merge-reports` do not work in watch mode.
+:::
+
 ## Custom Reporters
 
 You can use third-party custom reporters installed from NPM by specifying their package name in the reporters' option:

packages/runner/src/collect.ts

@@ -1,8 +1,7 @@
-import { relative } from 'pathe'
 import { processError } from '@vitest/utils/error'
 import type { File, SuiteHooks } from './types'
 import type { VitestRunner } from './types/runner'
-import { calculateSuiteHash, generateHash, interpretTaskModes, someTasksAreOnly } from './utils/collect'
+import { calculateSuiteHash, createFileTask, interpretTaskModes, someTasksAreOnly } from './utils/collect'
 import { clearCollectorContext, createSuiteHooks, getDefaultSuite } from './suite'
 import { getHooks, setHooks } from './map'
 import { collectorContext } from './context'
@@ -16,19 +15,9 @@ export async function collectTests(paths: string[], runner: VitestRunner): Promi
   const config = runner.config
 
   for (const filepath of paths) {
-    const path = relative(config.root, filepath)
-    const file: File = {
-      id: generateHash(`${path}${config.name || ''}`),
-      name: path,
-      type: 'suite',
-      mode: 'run',
-      filepath,
-      tasks: [],
-      meta: Object.create(null),
-      projectName: config.name,
-      file: undefined!,
-    }
-    file.file = file
+    const file = createFileTask(filepath, config.root, config.name)
+
+    runner.onCollectStart?.(file)
 
     clearCollectorContext(filepath, runner)
 

packages/runner/src/types/runner.ts

@@ -53,6 +53,10 @@ export interface VitestRunner {
    * First thing that's getting called before actually collecting and running tests.
    */
   onBeforeCollect?: (paths: string[]) => unknown
+  /**
+   * Called after the file task was created but not collected yet.
+   */
+  onCollectStart?: (file: File) => unknown
   /**
    * Called after collecting tests and before "onBeforeRun".
    */

packages/runner/src/utils/collect.ts

@@ -1,5 +1,6 @@
 import { processError } from '@vitest/utils/error'
-import type { Suite, TaskBase } from '../types'
+import { relative } from 'pathe'
+import type { File, Suite, TaskBase } from '../types'
 
 /**
  * If any tasks been marked as `only`, mark all other tasks as `skip`.
@@ -92,3 +93,20 @@ export function calculateSuiteHash(parent: Suite) {
       calculateSuiteHash(t)
   })
 }
+
+export function createFileTask(filepath: string, root: string, projectName: string) {
+  const path = relative(root, filepath)
+  const file: File = {
+    id: generateHash(`${path}${projectName || ''}`),
+    name: path,
+    type: 'suite',
+    mode: 'run',
+    filepath,
+    tasks: [],
+    meta: Object.create(null),
+    projectName,
+    file: undefined!,
+  }
+  file.file = file
+  return file
+}

packages/ui/client/components/views/ViewConsoleOutput.vue

@@ -15,6 +15,8 @@ const formattedLogs = computed(() => {
 
 function getTaskName(id?: string) {
   const task = id && client.state.idMap.get(id)
+  if (task && 'filepath' in task)
+    return task.name
   return (task ? getNames(task).slice(1).join(' > ') : '-') || '-'
 }
 </script>

packages/ui/client/composables/client/index.ts

@@ -3,8 +3,7 @@ import type { WebSocketStatus } from '@vueuse/core'
 import type { ErrorWithDiff, File, ResolvedConfig } from 'vitest'
 import type { Ref } from 'vue'
 import { reactive } from 'vue'
-import { relative } from 'pathe'
-import { generateHash } from '@vitest/runner/utils'
+import { createFileTask } from '@vitest/runner/utils'
 import type { RunState } from '../../../types'
 import { ENTRY_URL, isReport } from '../../constants'
 import { parseError } from '../error'
@@ -92,21 +91,8 @@ watch(
       ])
       if (_config.standalone) {
         const filenames = await client.rpc.getTestFiles()
-        const files = filenames.map<File>(([name, filepath]) => {
-          const path = relative(_config.root, filepath)
-          return {
-            filepath,
-            name: path,
-            id: /* #__PURE__ */ generateHash(`${path}${name || ''}`),
-            mode: 'skip',
-            type: 'suite',
-            result: {
-              state: 'skip',
-            },
-            meta: {},
-            tasks: [],
-            projectName: name,
-          }
+        const files = filenames.map<File>(([{ name, root }, filepath]) => {
+          return /* #__PURE__ */ createFileTask(filepath, root, name)
         })
         client.state.collectFiles(files)
       }

packages/vite-node/src/utils.ts

@@ -45,11 +45,9 @@ export function normalizeRequestId(id: string, base?: string): string {
     .replace(/\?+$/, '') // remove end query mark
 }
 
-export const queryRE = /\?.*$/s
-export const hashRE = /#.*$/s
-
+const postfixRE = /[?#].*$/
 export function cleanUrl(url: string): string {
-  return url.replace(hashRE, '').replace(queryRE, '')
+  return url.replace(postfixRE, '')
 }
 
 const internalRequests = [

packages/vitest/src/api/setup.ts

@@ -11,7 +11,7 @@ import type { ViteDevServer } from 'vite'
 import type { StackTraceParserOptions } from '@vitest/utils/source-map'
 import { API_PATH } from '../constants'
 import type { Vitest } from '../node'
-import type { File, ModuleGraphData, Reporter, TaskResultPack, UserConsoleLog } from '../types'
+import type { Awaitable, File, ModuleGraphData, Reporter, SerializableSpec, TaskResultPack, UserConsoleLog } from '../types'
 import { getModuleGraph, isPrimitive, noop, stringifyReplace } from '../utils'
 import type { WorkspaceProject } from '../node/workspace'
 import { parseErrorStacktrace } from '../utils/source-map'
@@ -166,7 +166,10 @@ export function setup(vitestOrWorkspace: Vitest | WorkspaceProject, _server?: Vi
         },
         async getTestFiles() {
           const spec = await ctx.globTestFiles()
-          return spec.map(([project, file]) => [project.getName(), file]) as [string, string][]
+          return spec.map(([project, file]) => [{
+            name: project.getName(),
+            root: project.config.root,
+          }, file])
         },
       },
       {
@@ -208,6 +211,14 @@ export class WebSocketReporter implements Reporter {
     })
   }
 
+  onSpecsCollected(specs?: SerializableSpec[] | undefined): Awaitable<void> {
+    if (this.clients.size === 0)
+      return
+    this.clients.forEach((client) => {
+      client.onSpecsCollected?.(specs)?.catch?.(noop)
+    })
+  }
+
   async onTaskUpdate(packs: TaskResultPack[]) {
     if (this.clients.size === 0)
       return

packages/vitest/src/api/types.ts

@@ -15,7 +15,7 @@ export interface WebSocketHandlers {
   getCountOfFailedTests: () => number
   sendLog: (log: UserConsoleLog) => void
   getFiles: () => File[]
-  getTestFiles: () => Promise<[name: string, file: string][]>
+  getTestFiles: () => Promise<[{ name: string; root: string }, file: string][]>
   getPaths: () => string[]
   getConfig: () => ResolvedConfig
   resolveSnapshotPath: (testPath: string) => string
@@ -39,7 +39,7 @@ export interface WebSocketHandlers {
   debug: (...args: string[]) => void
 }
 
-export interface WebSocketEvents extends Pick<Reporter, 'onCollected' | 'onFinished' | 'onTaskUpdate' | 'onUserConsoleLog' | 'onPathsCollected'> {
+export interface WebSocketEvents extends Pick<Reporter, 'onCollected' | 'onFinished' | 'onTaskUpdate' | 'onUserConsoleLog' | 'onPathsCollected' | 'onSpecsCollected'> {
   onCancel: (reason: CancelReason) => void
   onFinishedReportCoverage: () => void
 }

packages/vitest/src/node/cli/cli-api.ts

@@ -95,7 +95,9 @@ export async function startVitest(
   })
 
   try {
-    if (ctx.config.standalone)
+    if (ctx.config.mergeReports)
+      await ctx.mergeReports()
+    else if (ctx.config.standalone)
       await ctx.init()
     else
       await ctx.start(cliFilters)

packages/vitest/src/node/cli/cli-config.ts

@@ -606,6 +606,15 @@ export const cliOptionsConfig: VitestCLIOptions = {
   standalone: {
     description: 'Start Vitest without running tests. File filters will be ignored, tests will be running only on change (default: `false`)',
   },
+  mergeReports: {
+    description: 'Paths to blob reports directory. If this options is used, Vitest won\'t run any tests, it will only report previously recorded tests',
+    argument: '[path]',
+    transform(value) {
+      if (!value || typeof value === 'boolean')
+        return '.vitest-reports'
+      return value
+    },
+  },
 
   // disable CLI options
   cliExclude: null,

packages/vitest/src/node/config.ts

@@ -139,6 +139,9 @@ export function resolveConfig(
   if (resolved.standalone && !resolved.watch)
     throw new Error(`Vitest standalone mode requires --watch`)
 
+  if (resolved.mergeReports && resolved.watch)
+    throw new Error(`Cannot merge reports with --watch enabled`)
+
   if (resolved.maxWorkers)
     resolved.maxWorkers = Number(resolved.maxWorkers)