Fix standalone rendering for unmatched _next routes (#51611)

Request data flow in the server

```
request ---> router worker (1) ---> ipc ---> render worker app (2)
                                                              |-----> render worker pages (3)
```

When it's hitting `_next/*` unmatched routes in standalone server, it will render 404, but when you hit `_next/*` it will render app not-found as the app router is always enabled, but router worker isn't set up with require-hook for proper built-in react version, then the app-render will fail with `./server.edge` exports not found error.

We detect if it's in the render worker, then do the app paths rendering instead of directly looking up for app paths in route worker. This could avoid unexpected accesses to the app-render

Fixes #51482 
Fixes #50232
Closes #51506
Reverts changes in #51172
fix NEXT-1260

packages/next/src/server/app-render/use-flight-response.tsx

@@ -3,7 +3,8 @@ import type { FlightResponseRef } from './flight-response-ref'
 import { readableStreamTee } from '../stream-utils/node-web-streams-helper'
 import { encodeText, decodeText } from '../stream-utils/encode-decode'
 import { htmlEscapeJsonString } from '../htmlescape'
-import { isEdgeRuntime } from './app-render'
+
+const isEdgeRuntime = process.env.NEXT_RUNTIME === 'edge'
 
 /**
  * Render Flight stream.

packages/next/src/server/base-server.ts

@@ -2431,15 +2431,16 @@ export default abstract class Server<ServerOptions extends Options = Options> {
       }
     }
     const { res, query } = ctx
+
     try {
       let result: null | FindComponentsResult = null
 
       const is404 = res.statusCode === 404
       let using404Page = false
 
-      // use static 404 page if available and is 404 response
       if (is404) {
-        if (this.hasAppDir) {
+        // Rendering app routes only in render worker to make sure the require-hook is setup
+        if (this.hasAppDir && this.isRenderWorker) {
           // Use the not-found entry in app directory
           result = await this.findPageComponents({
             pathname: this.renderOpts.dev ? '/not-found' : '/_not-found',

packages/next/src/server/lib/render-server-standalone.ts

@@ -5,8 +5,6 @@ import httpProxy from 'next/dist/compiled/http-proxy'
 import { Worker } from 'next/dist/compiled/jest-worker'
 import { normalizeRepeatedSlashes } from '../../shared/lib/utils'
 
-const renderServerPath = require.resolve('./render-server')
-
 export const createServerHandler = async ({
   port,
   hostname,
@@ -20,7 +18,7 @@ export const createServerHandler = async ({
   dev?: boolean
   minimalMode: boolean
 }) => {
-  const routerWorker = new Worker(renderServerPath, {
+  const routerWorker = new Worker(require.resolve('./render-server'), {
     numWorkers: 1,
     maxRetries: 10,
     forkOptions: {

packages/next/src/server/next.ts

@@ -141,10 +141,10 @@ export class NextServer {
   }
 
   async prepare() {
-    const server = await this.getServer()
-
     if (this.standaloneMode) return
 
+    const server = await this.getServer()
+
     // We shouldn't prepare the server in production,
     // because this code won't be executed when deployed
     if (this.options.dev) {

test/production/standalone-mode/basic/app/not-found.js

@@ -0,0 +1,3 @@
+export default function NotFound() {
+  return 'app-not-found'
+}

test/production/standalone-mode/basic/index.test.ts

@@ -0,0 +1,44 @@
+import { createNextDescribe } from 'e2e-utils'
+
+createNextDescribe(
+  'standalone mode - metadata routes',
+  {
+    files: __dirname,
+    dependencies: {
+      swr: 'latest',
+    },
+  },
+  ({ next }) => {
+    beforeAll(async () => {
+      // Hide source files to make sure route.js can read files from source
+      // in order to hit the prerender cache
+      await next.renameFolder('app', 'app_hidden')
+    })
+
+    it('should handle metadata icons correctly', async () => {
+      const faviconRes = await next.fetch('/favicon.ico')
+      const iconRes = await next.fetch('/icon.svg')
+      expect(faviconRes.status).toBe(200)
+      expect(iconRes.status).toBe(200)
+    })
+
+    it('should handle correctly not-found.js', async () => {
+      const res = await next.fetch('/not-found/does-not-exist')
+      expect(res.status).toBe(404)
+      const html = await res.text()
+      expect(html).toContain('app-not-found')
+    })
+
+    it('should handle private _next unmatched route correctly', async () => {
+      const res = await next.fetch('/_next/does-not-exist')
+      expect(res.status).toBe(404)
+      const html = await res.text()
+      expect(html).toContain('app-not-found')
+    })
+
+    it('should handle pages rendering correctly', async () => {
+      const browser = await next.browser('/hello')
+      expect(await browser.elementByCss('#content').text()).toBe('hello-bar')
+    })
+  }
+)

test/production/standalone-mode/basic/pages/hello.js

@@ -0,0 +1,16 @@
+import useSWR, { useSWRConfig } from 'swr'
+
+export default function Page({ foo }) {
+  const { data } = useSWR('hello', (v) => v, { fallbackData: 'hello' })
+  useSWRConfig() // call SWR context
+
+  return <div id="content">{`${data}-${foo}`}</div>
+}
+
+export function getServerSideProps() {
+  return {
+    props: {
+      foo: 'bar',
+    },
+  }
+}