fix(snapshot)!: fail test with obsolete snapshot on CI (#7963)

Author: hi-ogawa

packages/runner/src/run.ts

@@ -576,9 +576,9 @@ export async function runSuite(suite: Suite, runner: VitestRunner): Promise<void
 
     suite.result.duration = now() - start
 
-    updateTask('suite-finished', suite, runner)
-
     await runner.onAfterRunSuite?.(suite)
+
+    updateTask('suite-finished', suite, runner)
   }
 }
 

packages/vitest/src/runtime/runners/test.ts

@@ -14,6 +14,7 @@ import type { SerializedConfig } from '../config'
 // import type { VitestExecutor } from '../execute'
 import { getState, GLOBAL_EXPECT, setState } from '@vitest/expect'
 import { getNames, getTestName, getTests } from '@vitest/runner/utils'
+import { processError } from '@vitest/utils/error'
 import { normalize } from 'pathe'
 import { createExpect } from '../../integrations/chai/index'
 import { inject } from '../../integrations/inject'
@@ -76,6 +77,18 @@ export class VitestTestRunner implements VitestRunner {
       }
 
       const result = await this.snapshotClient.finish(suite.file.filepath)
+      if (
+        this.workerState.config.snapshotOptions.updateSnapshot === 'none'
+        && result.unchecked
+      ) {
+        let message = `Obsolete snapshots found when no snapshot update is expected.\n`
+        for (const key of result.uncheckedKeys) {
+          message += `· ${key}\n`
+        }
+        suite.result!.errors ??= []
+        suite.result!.errors.push(processError(new Error(message)))
+        suite.result!.state = 'fail'
+      }
       await rpc().snapshotSaved(result)
     }
 

test/snapshots/test/fixtures/obsolete/src/__snapshots__/test1.test.ts.snap

@@ -0,0 +1,7 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`bar 1`] = `"bar"`;
+
+exports[`foo 1`] = `"foo"`;
+
+exports[`fuu 1`] = `"fuu"`;

test/snapshots/test/fixtures/obsolete/src/__snapshots__/test2.test.ts.snap

@@ -0,0 +1,5 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`bar 1`] = `"bar"`;
+
+exports[`foo 1`] = `"foo"`;

test/snapshots/test/fixtures/obsolete/src/test1.test.ts

@@ -0,0 +1,15 @@
+import { expect, it } from 'vitest'
+
+it('foo', () => {
+  if (process.env.TEST_OBSOLETE) return
+  expect("foo").toMatchSnapshot();
+})
+
+it('fuu', () => {
+  if (process.env.TEST_OBSOLETE) return
+  expect("fuu").toMatchSnapshot();
+})
+
+it('bar', () => {
+  expect("bar").toMatchSnapshot();
+})

test/snapshots/test/fixtures/obsolete/src/test2.test.ts

@@ -0,0 +1,9 @@
+import { expect, it } from 'vitest'
+
+it('foo', () => {
+  expect("foo").toMatchSnapshot();
+})
+
+it('bar', () => {
+  expect("bar").toMatchSnapshot();
+})

test/snapshots/test/fixtures/obsolete/vitest.config.ts

@@ -0,0 +1,3 @@
+import { defineConfig } from 'vitest/config'
+
+export default defineConfig({})

test/snapshots/test/obsolete.test.ts

@@ -0,0 +1,39 @@
+import fs from 'node:fs'
+import path from 'node:path'
+import { expect, test } from 'vitest'
+import { runVitestCli } from '../../test-utils'
+
+test('obsolete snapshot fails CI', async () => {
+  // cleanup snapshot
+  const root = path.join(import.meta.dirname, 'fixtures/obsolete')
+  fs.rmSync(path.join(root, 'src/__snapshots__'), { recursive: true, force: true })
+
+  // initial run to write snapshot
+  let vitest = await runVitestCli('--root', root, '--update')
+  expect(vitest.stdout).toContain('Snapshots  5 written')
+  expect(vitest.stderr).toBe('')
+
+  // test fails with obsolete snapshots
+  // (use cli to test `updateSnapshot: 'none'`)
+  vitest = await runVitestCli(
+    {
+      nodeOptions: {
+        env: {
+          CI: 'true',
+          TEST_OBSOLETE: 'true',
+        },
+      },
+    },
+    '--root',
+    root,
+  )
+  expect(vitest.stdout).toContain('2 obsolete')
+  expect(vitest.stdout).toContain('Test Files  1 failed | 1 passed')
+  expect(vitest.stdout).toContain('Tests  5 passed')
+  expect(vitest.stderr).toContain(`
+Error: Obsolete snapshots found when no snapshot update is expected.
+· foo 1
+· fuu 1
+`)
+  expect(vitest.exitCode).toBe(1)
+})