Remove param values from static route tree (vercel#82376)

Removes the param values from the router tree sent by the server during
a client-side prefetch.
    
This increases the cacheability of prefetch responses because if the
response does not include a certain param in the component data, it can
be reused for all possible values of that param. Previously this was not
possible because even if the components did not reference a param, it
was included in the router tree regardless.
    
This does not affect the router tree sent during dynamic navigations, or
during initial render. We should remove the param values from those
responses, too, both for consistency and to increase the cacheability of
HTML fallback responses. That work will land in future PRs.

Author: acdlite

packages/next/src/client/components/router-reducer/fetch-server-response.ts

@@ -31,7 +31,10 @@ import {
 } from '../../flight-data-helpers'
 import { getAppBuildId } from '../../app-build-id'
 import { setCacheBustingSearchParam } from './set-cache-busting-search-param'
-import { getRenderedPathname } from '../../route-params'
+import {
+  getRenderedPathname,
+  urlToUrlWithoutFlightMarker,
+} from '../../route-params'
 
 const createFromReadableStream =
   createFromReadableStreamBrowser as (typeof import('react-server-dom-webpack/client.browser'))['createFromReadableStream']
@@ -64,26 +67,11 @@ export type RequestHeaders = {
   'Next-Test-Fetch-Priority'?: RequestInit['priority']
 }
 
-export function urlToUrlWithoutFlightMarker(url: string): URL {
-  const urlWithoutFlightParameters = new URL(url, location.origin)
-  urlWithoutFlightParameters.searchParams.delete(NEXT_RSC_UNION_QUERY)
-  if (process.env.NODE_ENV === 'production') {
-    if (
-      process.env.__NEXT_CONFIG_OUTPUT === 'export' &&
-      urlWithoutFlightParameters.pathname.endsWith('.txt')
-    ) {
-      const { pathname } = urlWithoutFlightParameters
-      const length = pathname.endsWith('/index.txt') ? 10 : 4
-      // Slice off `/index.txt` or `.txt` from the end of the pathname
-      urlWithoutFlightParameters.pathname = pathname.slice(0, -length)
-    }
-  }
-  return urlWithoutFlightParameters
-}
-
 function doMpaNavigation(url: string): FetchServerResponseResult {
   return {
-    flightData: urlToUrlWithoutFlightMarker(url).toString(),
+    flightData: urlToUrlWithoutFlightMarker(
+      new URL(url, location.origin)
+    ).toString(),
     canonicalUrl: undefined,
     couldBeIntercepted: false,
     prerendered: false,
@@ -180,7 +168,7 @@ export async function fetchServerResponse(
       abortController.signal
     )
 
-    const responseUrl = urlToUrlWithoutFlightMarker(res.url)
+    const responseUrl = urlToUrlWithoutFlightMarker(new URL(res.url))
     const canonicalUrl = res.redirected ? responseUrl : undefined
 
     const contentType = res.headers.get('content-type') || ''

packages/next/src/client/components/segment-cache-impl/cache.ts

@@ -45,6 +45,7 @@ import type {
 } from './cache-key'
 import {
   doesStaticSegmentAppearInURL,
+  getCacheKeyForDynamicParam,
   getRenderedPathname,
   getRenderedSearch,
   parseDynamicParamFromURLPart,
@@ -900,8 +901,11 @@ function convertRootTreePrefetchToRouteTree(
   // Remove trailing and leading slashes
   const pathnameParts = renderedPathname.split('/').filter((p) => p !== '')
   const index = 0
+  const rootSegment = ROOT_SEGMENT_CACHE_KEY
   return convertTreePrefetchToRouteTree(
     rootTree.tree,
+    rootSegment,
+    null,
     ROOT_SEGMENT_REQUEST_KEY,
     ROOT_SEGMENT_CACHE_KEY,
     pathnameParts,
@@ -911,6 +915,8 @@ function convertRootTreePrefetchToRouteTree(
 
 function convertTreePrefetchToRouteTree(
   prefetch: TreePrefetch,
+  segment: FlightRouterStateSegment,
+  param: RouteParam | null,
   requestKey: SegmentRequestKey,
   cacheKey: SegmentCacheKey,
   pathnameParts: Array<string>,
@@ -922,52 +928,71 @@ function convertTreePrefetchToRouteTree(
   // it once instead of on every access. This same cache key is also used to
   // request the segment from the server.
 
-  let segment = prefetch.segment
-
-  let doesAppearInURL: boolean
-  let param: RouteParam | null = null
-  if (Array.isArray(segment)) {
-    // This segment is parameterized. Get the param from the pathname.
-    const paramType = segment[2] as DynamicParamTypesShort
-    const paramValue = parseDynamicParamFromURLPart(
-      paramType,
-      pathnameParts,
-      pathnamePartsIndex
-    )
-    param = {
-      name: segment[0],
-      value: paramValue,
-      type: paramType,
-    }
-
-    // Assign a cache key to the segment, based on the param value. In the
-    // pre-Segment Cache implementation, the server computes this and sends it
-    // in the body of the response. In the Segment Cache implementation, the
-    // server sends an empty string and we fill it in here.
-    // TODO: This will land in a follow up PR.
-    // segment[1] = getCacheKeyForDynamicParam(paramValue)
-
-    doesAppearInURL = true
-  } else {
-    doesAppearInURL = doesStaticSegmentAppearInURL(segment)
-  }
-
-  // Only increment the index if the segment appears in the URL. If it's a
-  // "virtual" segment, like a route group, it remains the same.
-  const childPathnamePartsIndex = doesAppearInURL
-    ? pathnamePartsIndex + 1
-    : pathnamePartsIndex
-
   let slots: { [parallelRouteKey: string]: RouteTree } | null = null
   const prefetchSlots = prefetch.slots
   if (prefetchSlots !== null) {
     slots = {}
     for (let parallelRouteKey in prefetchSlots) {
       const childPrefetch = prefetchSlots[parallelRouteKey]
-      const childSegment = childPrefetch.segment
-      // TODO: Eventually, the param values will not be included in the response
-      // from the server. We'll instead fill them in on the client by parsing
-      // the URL. This is where we'll do that.
+      const childParamName = childPrefetch.name
+      const childParamType = childPrefetch.paramType
+      const childServerSentParamKey = childPrefetch.paramKey
+
+      let childDoesAppearInURL: boolean
+      let childParam: RouteParam | null = null
+      let childSegment: FlightRouterStateSegment
+      if (childParamType !== null) {
+        // This segment is parameterized. Get the param from the pathname.
+        const childParamValue = parseDynamicParamFromURLPart(
+          childParamType,
+          pathnameParts,
+          pathnamePartsIndex
+        )
+
+        // Assign a cache key to the segment, based on the param value. In the
+        // pre-Segment Cache implementation, the server computes this and sends
+        // it in the body of the response. In the Segment Cache implementation,
+        // the server sends an empty string and we fill it in here.
+
+        // TODO: We're intentionally not adding the search param to page
+        // segments here; it's tracked separately and added back during a read.
+        // This would clearer if we waited to construct the segment until it's
+        // read from the cache, since that's effectively what we're
+        // doing anyway.
+        const renderedSearch = '' as NormalizedSearch
+        const childParamKey =
+          // The server omits this field from the prefetch response when
+          // clientParamParsing is enabled. The flag only exists while we're
+          // testing the feature, in case there's a bug and we need to revert.
+          // TODO: Remove once clientParamParsing is enabled everywhere.
+          childServerSentParamKey !== null
+            ? childServerSentParamKey
+            : // If no param key was sent, use the value parsed on the client.
+              getCacheKeyForDynamicParam(childParamValue, renderedSearch)
+
+        childParam = {
+          name: childParamName,
+          value: childParamValue,
+          type: childParamType,
+        }
+        childSegment = [
+          childParamName,
+          childParamKey,
+          childParamType,
+          childParamValue,
+        ]
+        childDoesAppearInURL = true
+      } else {
+        childSegment = childParamName
+        childDoesAppearInURL = doesStaticSegmentAppearInURL(childParamName)
+      }
+
+      // Only increment the index if the segment appears in the URL. If it's a
+      // "virtual" segment, like a route group, it remains the same.
+      const childPathnamePartsIndex = childDoesAppearInURL
+        ? pathnamePartsIndex + 1
+        : pathnamePartsIndex
+
       const childRequestKeyPart = createSegmentRequestKeyPart(childSegment)
       const childRequestKey = appendSegmentRequestKeyPart(
         requestKey,
@@ -981,6 +1006,8 @@ function convertTreePrefetchToRouteTree(
       )
       slots[parallelRouteKey] = convertTreePrefetchToRouteTree(
         childPrefetch,
+        childSegment,
+        childParam,
         childRequestKey,
         childCacheKey,
         pathnameParts,

packages/next/src/client/route-params.ts

@@ -1,14 +1,19 @@
 import type { DynamicParamTypesShort } from '../server/app-render/types'
-import { PAGE_SEGMENT_KEY } from '../shared/lib/segment'
+import {
+  addSearchParamsIfPageSegment,
+  DEFAULT_SEGMENT_KEY,
+  PAGE_SEGMENT_KEY,
+} from '../shared/lib/segment'
 import { ROOT_SEGMENT_REQUEST_KEY } from '../shared/lib/segment-cache/segment-value-encoding'
 import {
   NEXT_REWRITTEN_PATH_HEADER,
   NEXT_REWRITTEN_QUERY_HEADER,
+  NEXT_RSC_UNION_QUERY,
 } from './components/app-router-headers'
-import type { RSCResponse } from './components/router-reducer/fetch-server-response'
 import type { NormalizedSearch } from './components/segment-cache'
+import type { RSCResponse } from './components/router-reducer/fetch-server-response'
 
-type RouteParamValue = string | Array<string> | null
+export type RouteParamValue = string | Array<string> | null
 
 export type RouteParam = {
   name: string
@@ -28,15 +33,18 @@ export function getRenderedSearch(response: RSCResponse): NormalizedSearch {
   }
   // If the header is not present, there was no rewrite, so we use the search
   // query of the response URL.
-  return new URL(response.url).search as NormalizedSearch
+  return urlToUrlWithoutFlightMarker(new URL(response.url))
+    .search as NormalizedSearch
 }
 
 export function getRenderedPathname(response: RSCResponse): string {
   // If the server performed a rewrite, the pathname used to render the
   // page will be different from the pathname in the request URL. In this case,
   // the response will include a header that gives the rewritten pathname.
   const rewrittenPath = response.headers.get(NEXT_REWRITTEN_PATH_HEADER)
-  return rewrittenPath ?? new URL(response.url).pathname
+  return (
+    rewrittenPath ?? urlToUrlWithoutFlightMarker(new URL(response.url)).pathname
+  )
 }
 
 export function parseDynamicParamFromURLPart(
@@ -101,7 +109,9 @@ export function doesStaticSegmentAppearInURL(segment: string): boolean {
     // TODO: Investigate why the loader produces these fake page segments.
     segment.startsWith(PAGE_SEGMENT_KEY) ||
     // Route groups.
-    (segment[0] === '(' && segment.endsWith(')'))
+    (segment[0] === '(' && segment.endsWith(')')) ||
+    segment === DEFAULT_SEGMENT_KEY ||
+    segment === '/_not-found'
   ) {
     return false
   } else {
@@ -111,14 +121,41 @@ export function doesStaticSegmentAppearInURL(segment: string): boolean {
 }
 
 export function getCacheKeyForDynamicParam(
-  paramValue: RouteParamValue
+  paramValue: RouteParamValue,
+  renderedSearch: NormalizedSearch
 ): string {
   // This needs to match the logic in get-dynamic-param.ts, until we're able to
   // unify the various implementations so that these are always computed on
   // the client.
-  return typeof paramValue === 'string'
-    ? paramValue
-    : paramValue === null
-      ? ''
-      : paramValue.join('/')
+  if (typeof paramValue === 'string') {
+    // TODO: Refactor or remove this helper function to accept a string rather
+    // than the whole segment type. Also we can probably just append the
+    // search string instead of turning it into JSON.
+    const pageSegmentWithSearchParams = addSearchParamsIfPageSegment(
+      paramValue,
+      Object.fromEntries(new URLSearchParams(renderedSearch))
+    ) as string
+    return pageSegmentWithSearchParams
+  } else if (paramValue === null) {
+    return ''
+  } else {
+    return paramValue.join('/')
+  }
+}
+
+export function urlToUrlWithoutFlightMarker(url: URL): URL {
+  const urlWithoutFlightParameters = new URL(url)
+  urlWithoutFlightParameters.searchParams.delete(NEXT_RSC_UNION_QUERY)
+  if (process.env.NODE_ENV === 'production') {
+    if (
+      process.env.__NEXT_CONFIG_OUTPUT === 'export' &&
+      urlWithoutFlightParameters.pathname.endsWith('.txt')
+    ) {
+      const { pathname } = urlWithoutFlightParameters
+      const length = pathname.endsWith('/index.txt') ? 10 : 4
+      // Slice off `/index.txt` or `.txt` from the end of the pathname
+      urlWithoutFlightParameters.pathname = pathname.slice(0, -length)
+    }
+  }
+  return urlWithoutFlightParameters
 }

packages/next/src/server/app-render/app-render.tsx

@@ -4679,6 +4679,7 @@ async function collectSegmentData(
 
   const staleTime = prerenderStore.stale
   return await ComponentMod.collectSegmentData(
+    renderOpts.experimental.clientParamParsing,
     fullPageDataBuffer,
     staleTime,
     clientReferenceManifest.clientModules as ManifestNode,