Fix: Prefetching lazily generated param

packages/next/src/export/routes/app-page.ts

@@ -146,7 +146,7 @@ export async function exportAppPage(
       postponed,
       fetchTags,
       fetchMetrics,
-      segmentFlightData,
+      segmentData,
     } = metadata
 
     // Ensure we don't postpone without having PPR enabled.
@@ -200,7 +200,7 @@ export async function exportAppPage(
           flightData
         )
 
-        if (segmentFlightData) {
+        if (segmentData) {
           // Emit the per-segment prefetch data. We emit them as separate files
           // so that the cache handler has the option to treat each as a
           // separate entry.
@@ -210,7 +210,7 @@ export async function exportAppPage(
             RSC_SEGMENTS_DIR_SUFFIX
           )
           const tasks = []
-          for (const [segmentPath, buffer] of segmentFlightData.entries()) {
+          for (const [segmentPath, buffer] of segmentData) {
             segmentPaths.push(segmentPath)
             const segmentDataFilePath =
               segmentPath === '/'

packages/next/src/server/app-render/app-render.tsx

@@ -2844,7 +2844,7 @@ async function prerenderToStream(
 
         const flightData = await streamToBuffer(reactServerResult.asStream())
         metadata.flightData = flightData
-        metadata.segmentFlightData = await collectSegmentData(
+        metadata.segmentData = await collectSegmentData(
           flightData,
           finalRenderPrerenderStore,
           ComponentMod,
@@ -3323,7 +3323,7 @@ async function prerenderToStream(
           serverPrerenderStreamResult.asStream()
         )
         metadata.flightData = flightData
-        metadata.segmentFlightData = await collectSegmentData(
+        metadata.segmentData = await collectSegmentData(
           flightData,
           finalClientPrerenderStore,
           ComponentMod,
@@ -3455,7 +3455,7 @@ async function prerenderToStream(
 
       if (shouldGenerateStaticFlightData(workStore)) {
         metadata.flightData = flightData
-        metadata.segmentFlightData = await collectSegmentData(
+        metadata.segmentData = await collectSegmentData(
           flightData,
           ssrPrerenderStore,
           ComponentMod,
@@ -3647,7 +3647,7 @@ async function prerenderToStream(
       if (shouldGenerateStaticFlightData(workStore)) {
         const flightData = await streamToBuffer(reactServerResult.asStream())
         metadata.flightData = flightData
-        metadata.segmentFlightData = await collectSegmentData(
+        metadata.segmentData = await collectSegmentData(
           flightData,
           prerenderLegacyStore,
           ComponentMod,
@@ -3801,7 +3801,7 @@ async function prerenderToStream(
           reactServerPrerenderResult.asStream()
         )
         metadata.flightData = flightData
-        metadata.segmentFlightData = await collectSegmentData(
+        metadata.segmentData = await collectSegmentData(
           flightData,
           prerenderLegacyStore,
           ComponentMod,

packages/next/src/server/base-server.ts

@@ -2749,7 +2749,7 @@ export default abstract class Server<
             rscData: metadata.flightData,
             postponed: metadata.postponed,
             status: res.statusCode,
-            segmentData: undefined,
+            segmentData: metadata.segmentData,
           } satisfies CachedAppPageValue,
           revalidate: metadata.revalidate,
           isFallback: !!fallbackRouteParams,
@@ -3057,8 +3057,9 @@ export default abstract class Server<
         // it's a 404 — either the segment is fully dynamic, or an invalid segment
         // path was requested.
         if (cacheEntry.value.segmentData) {
-          const matchedSegment =
-            cacheEntry.value.segmentData[segmentPrefetchHeader]
+          const matchedSegment = cacheEntry.value.segmentData.get(
+            segmentPrefetchHeader
+          )
           if (matchedSegment !== undefined) {
             return {
               type: 'rsc',

packages/next/src/server/lib/incremental-cache/file-system-cache.ts

@@ -194,14 +194,14 @@ export default class FileSystemCache implements CacheHandler {
             )
           } catch {}
 
-          let maybeSegmentData: { [segmentPath: string]: string } | undefined
+          let maybeSegmentData: Map<string, Buffer> | undefined
           if (meta?.segmentPaths) {
             // Collect all the segment data for this page.
             // TODO: To optimize file system reads, we should consider creating
             // separate cache entries for each segment, rather than storing them
             // all on the page's entry. Though the behavior is
             // identical regardless.
-            const segmentData: { [segmentPath: string]: string } = {}
+            const segmentData: Map<string, Buffer> = new Map()
             maybeSegmentData = segmentData
             const segmentsDir = key + RSC_SEGMENTS_DIR_SUFFIX
             await Promise.all(
@@ -213,9 +213,9 @@ export default class FileSystemCache implements CacheHandler {
                   IncrementalCacheKind.APP_PAGE
                 )
                 try {
-                  segmentData[segmentPath] = await this.fs.readFile(
-                    segmentDataFilePath,
-                    'utf8'
+                  segmentData.set(
+                    segmentPath,
+                    await this.fs.readFile(segmentDataFilePath)
                   )
                 } catch {
                   // This shouldn't happen, but if for some reason we fail to

packages/next/src/server/render-result.ts

@@ -34,7 +34,7 @@ export type AppPageRenderResultMetadata = {
   fetchTags?: string
   fetchMetrics?: FetchMetrics
 
-  segmentFlightData?: Map<string, Buffer>
+  segmentData?: Map<string, Buffer>
 
   /**
    * In development, the cache is warmed up before the render. This is attached

packages/next/src/server/response-cache/types.ts

@@ -77,7 +77,7 @@ export interface CachedAppPageValue {
   status: number | undefined
   postponed: string | undefined
   headers: OutgoingHttpHeaders | undefined
-  segmentData: { [segmentPath: string]: string } | undefined
+  segmentData: Map<string, Buffer> | undefined
 }
 
 export interface CachedPageValue {
@@ -118,7 +118,7 @@ export interface IncrementalCachedAppPageValue {
   headers: OutgoingHttpHeaders | undefined
   postponed: string | undefined
   status: number | undefined
-  segmentData: { [segmentPath: string]: string } | undefined
+  segmentData: Map<string, Buffer> | undefined
 }
 
 export interface IncrementalCachedPageValue {

test/e2e/app-dir/segment-cache/basic/app/lazily-generated-params/[param]/page.tsx

@@ -0,0 +1,14 @@
+import { Suspense } from 'react'
+
+async function Content({ params }) {
+  const { param } = await params
+  return <div id="target-page-with-lazily-generated-param">Param: {param}</div>
+}
+
+export default async function Target({ params }) {
+  return (
+    <Suspense fallback="Loading...">
+      <Content params={params} />
+    </Suspense>
+  )
+}

test/e2e/app-dir/segment-cache/basic/app/lazily-generated-params/page.tsx

@@ -0,0 +1,22 @@
+import Link from 'next/link'
+
+// TODO: Once the appropriate API exists/is implemented, configure the param to
+// be statically generated on demand but not at build time (`dynamicParams =
+// true` isn't supported when `dynamicIO` is enabled.) For now this test case
+// seems to work without extra configuration but it might not in the future.
+
+export default function LazilyGeneratedParamsStartPage() {
+  return (
+    <>
+      <p>
+        Demonstrates that we can prefetch param that is not generated at build
+        time but is lazily generated on demand
+      </p>
+      <ul>
+        <li>
+          <Link href="/lazily-generated-params/some-param-value">Target</Link>
+        </li>
+      </ul>
+    </>
+  )
+}

test/e2e/app-dir/segment-cache/basic/segment-cache-basic.test.ts

@@ -1,5 +1,5 @@
 import { nextTestSetup } from 'e2e-utils'
-import type { Page, Route } from 'playwright'
+import type * as Playwright from 'playwright'
 
 describe('segment cache (basic tests)', () => {
   const { next, isNextDev, skipped } = nextTestSetup({
@@ -19,9 +19,9 @@ describe('segment cache (basic tests)', () => {
     const prefetchLock = interceptor.lockPrefetches()
 
     const browser = await next.browser('/', {
-      beforePageLoad(page: Page) {
-        page.route('**/*', async (route: Route) => {
-          await interceptor.interceptRoute(route)
+      beforePageLoad(page: Playwright.Page) {
+        page.route('**/*', async (route: Playwright.Route) => {
+          await interceptor.interceptRoute(page, route)
         })
       },
     })
@@ -43,9 +43,9 @@ describe('segment cache (basic tests)', () => {
   it('navigate with prefetched data', async () => {
     const interceptor = createRequestInterceptor()
     const browser = await next.browser('/', {
-      beforePageLoad(page: Page) {
-        page.route('**/*', async (route: Route) => {
-          await interceptor.interceptRoute(route)
+      beforePageLoad(page: Playwright.Page) {
+        page.route('**/*', async (route: Playwright.Route) => {
+          await interceptor.interceptRoute(page, route)
         })
       },
     })
@@ -77,12 +77,48 @@ describe('segment cache (basic tests)', () => {
     )
   })
 
+  it('navigate to page with lazily-generated (not at build time) static param', async () => {
+    const interceptor = createRequestInterceptor()
+    const browser = await interceptor.waitForPrefetches(async () => {
+      const b = await next.browser('/lazily-generated-params', {
+        beforePageLoad(page: Playwright.Page) {
+          page.route('**/*', async (route: Playwright.Route) => {
+            await interceptor.interceptRoute(page, route)
+          })
+        },
+      })
+      await b.elementByCss('a')
+      return b
+    })
+
+    const navigationsLock = interceptor.lockNavigations()
+
+    // Navigate to the test page
+    const link = await browser.elementByCss('a')
+    await link.click()
+
+    // We should be able to render the page with the dynamic param, because
+    // it is lazily generated
+    const target = await browser.elementById(
+      'target-page-with-lazily-generated-param'
+    )
+    expect(await target.innerHTML()).toMatchInlineSnapshot(
+      `"Param: some-param-value"`
+    )
+
+    await navigationsLock.release()
+
+    // TODO: Once #73540 lands we can also test that the dynamic nav was skipped
+    // const navigations = await navigationsLock.release()
+    // expect(navigations.size).toBe(0)
+  })
+
   it('prefetch interception route', async () => {
     const interceptor = createRequestInterceptor()
     const browser = await next.browser('/interception/feed', {
-      beforePageLoad(page: Page) {
-        page.route('**/*', async (route: Route) => {
-          await interceptor.interceptRoute(route)
+      beforePageLoad(page: Playwright.Page) {
+        page.route('**/*', async (route: Playwright.Route) => {
+          await interceptor.interceptRoute(page, route)
         })
       },
     })
@@ -112,8 +148,11 @@ function createRequestInterceptor() {
   // implementation details as much as possible, so the only thing this does
   // for now is let you block and release requests from happening based on
   // their type (prefetch requests, navigation requests).
-  let pendingPrefetches: Set<Route> | null = null
-  let pendingNavigations: Set<Route> | null = null
+  let pendingPrefetches: Set<Playwright.Route> | null = null
+  let pendingNavigations: Set<Playwright.Route> | null = null
+
+  let prefetchesPromise: PromiseWithResolvers<void> = null
+  let lastPrefetchRequest: Playwright.Request | null = null
 
   return {
     lockNavigations() {
@@ -131,6 +170,7 @@ function createRequestInterceptor() {
           for (const route of routes) {
             route.continue()
           }
+          return routes
         },
       }
     },
@@ -150,17 +190,75 @@ function createRequestInterceptor() {
           for (const route of routes) {
             route.continue()
           }
+          return routes
         },
       }
     },
 
-    async interceptRoute(route: Route) {
-      const requestHeaders = await route.request().allHeaders()
+    /**
+     * Waits for the next for the next prefetch request, then keeps waiting
+     * until the prefetch queue is empty (to account for network throttling).
+     *
+     * If no prefetches are initiated, this will timeout.
+     */
+    async waitForPrefetches<T>(
+      scope: () => Promise<T> | T = (): undefined => {}
+    ): Promise<T> {
+      if (prefetchesPromise === null) {
+        let resolve
+        let reject
+        const promise: Promise<void> = new Promise((res, rej) => {
+          resolve = res
+          reject = rej
+        })
+        prefetchesPromise = {
+          resolve,
+          reject,
+          promise,
+        }
+      }
+      const result = await scope()
+      if (prefetchesPromise !== null) {
+        await prefetchesPromise.promise
+      }
+      return result
+    },
+
+    async interceptRoute(page: Playwright.Page, route: Playwright.Route) {
+      const request = route.request()
+      const requestHeaders = await request.allHeaders()
 
       if (requestHeaders['RSC'.toLowerCase()]) {
         // This is an RSC request. Check if it's a prefetch or a navigation.
         if (requestHeaders['Next-Router-Prefetch'.toLowerCase()]) {
           // This is a prefetch request.
+          if (prefetchesPromise !== null) {
+            // Wait for the prefetch response to finish, then wait an additional
+            // async task for additional prefetches to be initiated.
+            lastPrefetchRequest = request
+            const waitForMorePrefetches = async () => {
+              const response = await request.response()
+              await response.finished()
+              await page.evaluate(
+                () =>
+                  // If the prefetch queue is network throttled, the next
+                  // request should be issued within a microtask of the previous
+                  // one finishing.
+                  new Promise<void>((res) => requestIdleCallback(() => res()))
+              )
+              if (request === lastPrefetchRequest) {
+                // No further prefetches were initiated. Assume the prefetch
+                // queue is now empty.
+                prefetchesPromise.resolve()
+                prefetchesPromise = null
+                lastPrefetchRequest = null
+              }
+            }
+            waitForMorePrefetches().then(
+              () => {},
+              () => {}
+            )
+          }
           if (pendingPrefetches !== null) {
             pendingPrefetches.add(route)
             return