fix(typecheck): run both runtime and typecheck tests if typecheck.include overlaps with include (#6256)

Author: sheremet-va

docs/guide/testing-types.md

@@ -14,7 +14,13 @@ Vitest allows you to write tests for your types, using `expectTypeOf` or `assert
 
 Under the hood Vitest calls `tsc` or `vue-tsc`, depending on your config, and parses results. Vitest will also print out type errors in your source code, if it finds any. You can disable it with [`typecheck.ignoreSourceErrors`](/config/#typecheck-ignoresourceerrors) config option.
 
-Keep in mind that Vitest doesn't run or compile these files, they are only statically analyzed by the compiler, and because of that you cannot use any dynamic statements. Meaning, you cannot use dynamic test names, and `test.each`, `test.runIf`, `test.skipIf`, `test.concurrent` APIs. But you can use other APIs, like `test`, `describe`, `.only`, `.skip` and `.todo`.
+Keep in mind that Vitest doesn't run these files, they are only statically analyzed by the compiler. Meaning, that if you use a dynamic name or `test.each` or `test.for`, the test name will not be evaluated - it will be displayed as is.
+
+::: warning
+Before Vitest 2.1, your `typecheck.include` overrode the `include` pattern, so your runtime tests did not actually run; they were only type-checked.
+
+Since Vitest 2.1, if your `include` and `typecheck.include` overlap, Vitest will report type tests and runtime tests as separate entries.
+:::
 
 Using CLI flags, like `--allowOnly` and `-t` are also supported for type checking.
 

packages/browser/src/node/plugin.ts

@@ -157,7 +157,7 @@ export default (browserServer: BrowserServer, base = '/'): Plugin[] => {
       name: 'vitest:browser:tests',
       enforce: 'pre',
       async config() {
-        const allTestFiles = await project.globTestFiles()
+        const { testFiles: allTestFiles } = await project.globTestFiles()
         const browserTestFiles = allTestFiles.filter(
           file => getFilePoolName(project, file) === 'browser',
         )

packages/browser/src/node/pool.ts

@@ -1,7 +1,7 @@
 import * as nodeos from 'node:os'
 import crypto from 'node:crypto'
 import { relative } from 'pathe'
-import type { BrowserProvider, ProcessPool, Vitest, WorkspaceProject } from 'vitest/node'
+import type { BrowserProvider, ProcessPool, Vitest, WorkspaceProject, WorkspaceSpec } from 'vitest/node'
 import { createDebugger } from 'vitest/node'
 
 const debug = createDebugger('vitest:browser:pool')
@@ -92,7 +92,7 @@ export function createBrowserPool(ctx: Vitest): ProcessPool {
     await Promise.all(promises)
   }
 
-  const runWorkspaceTests = async (method: 'run' | 'collect', specs: [WorkspaceProject, string][]) => {
+  const runWorkspaceTests = async (method: 'run' | 'collect', specs: WorkspaceSpec[]) => {
     const groupedFiles = new Map<WorkspaceProject, string[]>()
     for (const [project, file] of specs) {
       const files = groupedFiles.get(project) || []

packages/browser/src/node/serverTester.ts

@@ -22,7 +22,7 @@ export async function resolveTester(
   const { contextId, testFile } = server.resolveTesterUrl(url.pathname)
   const project = server.project
   const state = server.state
-  const testFiles = await project.globTestFiles()
+  const { testFiles } = await project.globTestFiles()
   // if decoded test file is "__vitest_all__" or not in the list of known files, run all tests
   const tests
     = testFile === '__vitest_all__'

packages/ui/client/components/FileDetails.vue

@@ -35,6 +35,10 @@ const failedSnapshot = computed(() => {
   return current.value && hasFailedSnapshot(current.value)
 })
 
+const isTypecheck = computed(() => {
+  return !!current.value?.meta?.typecheck
+})
+
 function open() {
   const filePath = current.value?.filepath
   if (filePath) {
@@ -122,6 +126,7 @@ debouncedWatch(
     <div>
       <div p="2" h-10 flex="~ gap-2" items-center bg-header border="b base">
         <StatusIcon :state="current.result?.state" :mode="current.mode" :failed-snapshot="failedSnapshot" />
+        <div v-if="isTypecheck" v-tooltip.bottom="'This is a typecheck test. It won\'t report results of the runtime tests'" class="i-logos:typescript-icon" flex-shrink-0 />
         <div
           v-if="current?.file.projectName"
           font-light

packages/ui/client/components/explorer/ExplorerItem.vue

@@ -173,8 +173,8 @@ const projectNameTextColor = computed(() => {
       <div :class="opened ? 'i-carbon:chevron-down' : 'i-carbon:chevron-right op20'" op20 />
     </div>
     <StatusIcon :state="state" :mode="task.mode" :failed-snapshot="failedSnapshot" w-4 />
-    <div v-if="type === 'suite' && typecheck" class="i-logos:typescript-icon" flex-shrink-0 mr-2 />
     <div flex items-end gap-2 overflow-hidden>
+      <div v-if="type === 'file' && typecheck" v-tooltip.bottom="'This is a typecheck test. It won\'t report results of the runtime tests'" class="i-logos:typescript-icon" flex-shrink-0 />
       <span text-sm truncate font-light>
         <span v-if="type === 'file' && projectName" class="rounded-full p-1 mr-1 text-xs" :style="{ backgroundColor: projectNameColor, color: projectNameTextColor }">
           {{ projectName }}

packages/ui/client/composables/explorer/types.ts

@@ -55,12 +55,12 @@ export interface ParentTreeNode extends UITaskTreeNode {
 export interface SuiteTreeNode extends ParentTreeNode {
   fileId: string
   type: 'suite'
-  typecheck?: boolean
 }
 
 export interface FileTreeNode extends ParentTreeNode {
   type: 'file'
   filepath: string
+  typecheck: boolean | undefined
   projectName?: string
   projectNameColor: string
   collectDuration?: number

packages/ui/client/composables/explorer/utils.ts

@@ -46,6 +46,7 @@ export function createOrUpdateFileNode(
   let fileNode = explorerTree.nodes.get(file.id) as FileTreeNode | undefined
 
   if (fileNode) {
+    fileNode.typecheck = !!file.meta && 'typecheck' in file.meta
     fileNode.state = file.result?.state
     fileNode.mode = file.mode
     fileNode.duration = file.result?.duration
@@ -66,6 +67,7 @@ export function createOrUpdateFileNode(
       type: 'file',
       children: new Set(),
       tasks: [],
+      typecheck: !!file.meta && 'typecheck' in file.meta,
       indent: 0,
       duration: file.result?.duration,
       filepath: file.filepath,
@@ -141,9 +143,6 @@ export function createOrUpdateNode(
       taskNode.mode = task.mode
       taskNode.duration = task.result?.duration
       taskNode.state = task.result?.state
-      if (isSuiteNode(taskNode)) {
-        taskNode.typecheck = !!task.meta && 'typecheck' in task.meta
-      }
     }
     else {
       if (isAtomTest(task)) {
@@ -168,7 +167,6 @@ export function createOrUpdateNode(
           parentId,
           name: task.name,
           mode: task.mode,
-          typecheck: !!task.meta && 'typecheck' in task.meta,
           type: 'suite',
           expandable: true,
           // When the current run finish, we will expand all nodes when required, here we expand only the opened nodes

packages/vitest/src/api/setup.ts

@@ -103,12 +103,13 @@ export function setup(ctx: Vitest, _server?: ViteDevServer) {
         },
         async getTestFiles() {
           const spec = await ctx.globTestFiles()
-          return spec.map(([project, file]) => [
+          return spec.map(([project, file, options]) => [
             {
               name: project.config.name,
               root: project.config.root,
             },
             file,
+            options,
           ])
         },
       },

packages/vitest/src/node/core.ts

@@ -17,7 +17,7 @@ import { WebSocketReporter } from '../api/setup'
 import type { SerializedCoverageConfig } from '../runtime/config'
 import type { SerializedSpec } from '../runtime/types/utils'
 import type { ArgumentsType, OnServerRestartHandler, ProvidedContext, UserConsoleLog } from '../types/general'
-import { createPool } from './pool'
+import { createPool, getFilePoolName } from './pool'
 import type { ProcessPool, WorkspaceSpec } from './pool'
 import { createBenchmarkReporters, createReporters } from './reporters/utils'
 import { StateManager } from './state'
@@ -77,10 +77,14 @@ export class Vitest {
 
   private resolvedProjects: WorkspaceProject[] = []
   public projects: WorkspaceProject[] = []
-  private projectsTestFiles = new Map<string, Set<WorkspaceProject>>()
 
   public distPath!: string
 
+  private _cachedSpecs = new Map<string, WorkspaceSpec[]>()
+
+  /** @deprecated use `_cachedSpecs` */
+  projectTestFiles = this._cachedSpecs
+
   constructor(
     public readonly mode: VitestRunMode,
     options: VitestOptions = {},
@@ -103,7 +107,7 @@ export class Vitest {
     this.coverageProvider = undefined
     this.runningPromise = undefined
     this.distPath = undefined!
-    this.projectsTestFiles.clear()
+    this._cachedSpecs.clear()
 
     const resolved = resolveConfig(this.mode, options, server.config, this.logger)
 
@@ -190,6 +194,13 @@ export class Vitest {
     this.getCoreWorkspaceProject().provide(key, value)
   }
 
+  /**
+   * @deprecated internal, use `_createCoreProject` instead
+   */
+  createCoreProject() {
+    return this._createCoreProject()
+  }
+
   /**
    * @internal
    */
@@ -202,6 +213,9 @@ export class Vitest {
     return this.coreWorkspaceProject
   }
 
+  /**
+   * @deprecated use Reported Task API instead
+   */
   public getProjectByTaskId(taskId: string): WorkspaceProject {
     const task = this.state.idMap.get(taskId)
     const projectName = (task as File).projectName || task?.file?.projectName || ''
@@ -216,7 +230,7 @@ export class Vitest {
       || this.projects[0]
   }
 
-  private async getWorkspaceConfigPath() {
+  private async getWorkspaceConfigPath(): Promise<string | null> {
     if (this.config.workspace) {
       return this.config.workspace
     }
@@ -423,8 +437,8 @@ export class Vitest {
     }
   }
 
-  private async getTestDependencies(filepath: WorkspaceSpec, deps = new Set<string>()) {
-    const addImports = async ([project, filepath]: WorkspaceSpec) => {
+  private async getTestDependencies([project, filepath]: WorkspaceSpec, deps = new Set<string>()) {
+    const addImports = async (project: WorkspaceProject, filepath: string) => {
       if (deps.has(filepath)) {
         return
       }
@@ -440,13 +454,13 @@ export class Vitest {
         const path = await project.server.pluginContainer.resolveId(dep, filepath, { ssr: true })
         const fsPath = path && !path.external && path.id.split('?')[0]
         if (fsPath && !fsPath.includes('node_modules') && !deps.has(fsPath) && existsSync(fsPath)) {
-          await addImports([project, fsPath])
+          await addImports(project, fsPath)
         }
       }))
     }
 
-    await addImports(filepath)
-    deps.delete(filepath[1])
+    await addImports(project, filepath)
+    deps.delete(filepath)
 
     return deps
   }
@@ -500,12 +514,31 @@ export class Vitest {
     return runningTests
   }
 
+  /**
+   * @deprecated remove when vscode extension supports "getFileWorkspaceSpecs"
+   */
   getProjectsByTestFile(file: string) {
-    const projects = this.projectsTestFiles.get(file)
-    if (!projects) {
-      return []
+    return this.getFileWorkspaceSpecs(file)
+  }
+
+  getFileWorkspaceSpecs(file: string) {
+    const _cached = this._cachedSpecs.get(file)
+    if (_cached) {
+      return _cached
+    }
+
+    const specs: WorkspaceSpec[] = []
+    for (const project of this.projects) {
+      if (project.isTestFile(file)) {
+        const pool = getFilePoolName(project, file)
+        specs.push([project, file, { pool }])
+      }
+      if (project.isTypecheckFile(file)) {
+        specs.push([project, file, { pool: 'typescript' }])
+      }
     }
-    return Array.from(projects).map(project => [project, file] as WorkspaceSpec)
+    specs.forEach(spec => this.ensureSpecCached(spec))
+    return specs
   }
 
   async initializeGlobalSetup(paths: WorkspaceSpec[]) {
@@ -538,8 +571,11 @@ export class Vitest {
 
     await this.report('onPathsCollected', filepaths)
     await this.report('onSpecsCollected', specs.map(
-      ([project, file]) =>
-        [{ name: project.config.name, root: project.config.root }, file] as SerializedSpec,
+      ([project, file, options]) =>
+        [{
+          name: project.config.name,
+          root: project.config.root,
+        }, file, options] satisfies SerializedSpec,
     ))
 
     // previous run
@@ -856,7 +892,6 @@ export class Vitest {
       }))
 
       if (matchingProjects.length > 0) {
-        this.projectsTestFiles.set(id, new Set(matchingProjects))
         this.changedTests.add(id)
         this.scheduleRerun([id])
       }
@@ -1054,17 +1089,32 @@ export class Vitest {
   public async globTestFiles(filters: string[] = []) {
     const files: WorkspaceSpec[] = []
     await Promise.all(this.projects.map(async (project) => {
-      const specs = await project.globTestFiles(filters)
-      specs.forEach((file) => {
-        files.push([project, file])
-        const projects = this.projectsTestFiles.get(file) || new Set()
-        projects.add(project)
-        this.projectsTestFiles.set(file, projects)
+      const { testFiles, typecheckTestFiles } = await project.globTestFiles(filters)
+      testFiles.forEach((file) => {
+        const pool = getFilePoolName(project, file)
+        const spec: WorkspaceSpec = [project, file, { pool }]
+        this.ensureSpecCached(spec)
+        files.push(spec)
+      })
+      typecheckTestFiles.forEach((file) => {
+        const spec: WorkspaceSpec = [project, file, { pool: 'typescript' }]
+        this.ensureSpecCached(spec)
+        files.push(spec)
       })
     }))
     return files
   }
 
+  private ensureSpecCached(spec: WorkspaceSpec) {
+    const file = spec[1]
+    const specs = this._cachedSpecs.get(file) || []
+    const included = specs.some(_s => _s[0] === spec[0] && _s[2].pool === spec[2].pool)
+    if (!included) {
+      specs.push(spec)
+      this._cachedSpecs.set(file, specs)
+    }
+  }
+
   // The server needs to be running for communication
   shouldKeepServer() {
     return !!this.config?.watch

packages/vitest/src/node/pool.ts

@@ -10,7 +10,7 @@ import type { WorkspaceProject } from './workspace'
 import { createTypecheckPool } from './pools/typecheck'
 import { createVmForksPool } from './pools/vmForks'
 
-export type WorkspaceSpec = [project: WorkspaceProject, testFile: string]
+export type WorkspaceSpec = [project: WorkspaceProject, testFile: string, options: { pool: Pool }]
 export type RunWithFiles = (
   files: WorkspaceSpec[],
   invalidates?: string[]
@@ -39,14 +39,7 @@ export const builtinPools: BuiltinPool[] = [
   'typescript',
 ]
 
-function getDefaultPoolName(project: WorkspaceProject, file: string): Pool {
-  if (project.config.typecheck.enabled) {
-    for (const glob of project.config.typecheck.include) {
-      if (mm.isMatch(file, glob, { cwd: project.config.root })) {
-        return 'typescript'
-      }
-    }
-  }
+function getDefaultPoolName(project: WorkspaceProject): Pool {
   if (project.config.browser.enabled) {
     return 'browser'
   }
@@ -64,7 +57,7 @@ export function getFilePoolName(project: WorkspaceProject, file: string) {
       return pool as Pool
     }
   }
-  return getDefaultPoolName(project, file)
+  return getDefaultPoolName(project)
 }
 
 export function createPool(ctx: Vitest): ProcessPool {
@@ -172,7 +165,7 @@ export function createPool(ctx: Vitest): ProcessPool {
     }
 
     for (const spec of files) {
-      const pool = getFilePoolName(spec[0], spec[1])
+      const { pool } = spec[2]
       filesByPool[pool] ??= []
       filesByPool[pool].push(spec)
     }