diff --git a/docs/guide/migration.md b/docs/guide/migration.md
index 76bdff63967f57..f141f742b892a3 100644
--- a/docs/guide/migration.md
+++ b/docs/guide/migration.md
@@ -58,6 +58,8 @@ Also there are other breaking changes which only affect few users.
   - Top level `this` was rewritten to `globalThis` by default when building. This behavior is now removed.
 - [[#14231] feat!: add extension to internal virtual modules](https://github.com/vitejs/vite/pull/14231)
   - Internal virtual modules' id now has an extension (`.js`).
+- [[#5657] fix: return 404 for resources requests outside the base path](https://github.com/vitejs/vite/pull/5657)
+  - In the past, Vite responded to requests outside the base path without `Accept: text/html`, as if they were requested with the base path. Vite no longer does that and responds with 404 instead.
 
 ## Migration from v3
 
diff --git a/packages/vite/src/node/server/middlewares/base.ts b/packages/vite/src/node/server/middlewares/base.ts
index c6af2302aef3d9..4d6da95a411f34 100644
--- a/packages/vite/src/node/server/middlewares/base.ts
+++ b/packages/vite/src/node/server/middlewares/base.ts
@@ -33,10 +33,12 @@ export function baseMiddleware({
       })
       res.end()
       return
-    } else if (req.headers.accept?.includes('text/html')) {
-      // non-based page visit
-      const redirectPath =
-        withTrailingSlash(url) !== base ? joinUrlSegments(base, url) : base
+    }
+
+    // non-based page visit
+    const redirectPath =
+      withTrailingSlash(url) !== base ? joinUrlSegments(base, url) : base
+    if (req.headers.accept?.includes('text/html')) {
       res.writeHead(404, {
         'Content-Type': 'text/html',
       })
@@ -45,8 +47,16 @@ export function baseMiddleware({
           `did you mean to visit <a href="${redirectPath}">${redirectPath}</a> instead?`,
       )
       return
+    } else {
+      // not found for resources
+      res.writeHead(404, {
+        'Content-Type': 'text/plain',
+      })
+      res.end(
+        `The server is configured with a public base URL of ${base} - ` +
+          `did you mean to visit ${redirectPath} instead?`,
+      )
+      return
     }
-
-    next()
   }
 }
diff --git a/playground/assets/__tests__/assets.spec.ts b/playground/assets/__tests__/assets.spec.ts
index 70d0fcfe4e8e54..ab482bcdef3d2b 100644
--- a/playground/assets/__tests__/assets.spec.ts
+++ b/playground/assets/__tests__/assets.spec.ts
@@ -8,6 +8,7 @@ import {
   getBg,
   getColor,
   isBuild,
+  isServe,
   listAssets,
   notifyRebuildComplete,
   page,
@@ -60,6 +61,22 @@ test('should fallback to index.html when accessing non-existant html file', asyn
   expect((await fetchPath('doesnt-exist.html')).status).toBe(200)
 })
 
+describe.runIf(isServe)('outside base', () => {
+  test('should get a 404 with html', async () => {
+    const res = await fetch(new URL('/baz', viteTestUrl), {
+      headers: { Accept: 'text/html,*/*' },
+    })
+    expect(res.status).toBe(404)
+    expect(res.headers.get('Content-Type')).toBe('text/html')
+  })
+
+  test('should get a 404 with text', async () => {
+    const res = await fetch(new URL('/baz', viteTestUrl))
+    expect(res.status).toBe(404)
+    expect(res.headers.get('Content-Type')).toBe('text/plain')
+  })
+})
+
 describe('injected scripts', () => {
   test('@vite/client', async () => {
     const hasClient = await page.$(