Add initial changes for i18n support (#17370)

This adds the initial changes outlined in the [i18n routing RFC](#17078). This currently treats the locale prefix on routes similar to how the basePath is treated in that the config doesn't require any changes to your pages directory and is automatically stripped/added based on the detected locale that should be used. 

Currently redirecting occurs on the `/` route if a locale is detected regardless of if an optional catch-all route would match the `/` route or not we may want to investigate whether we want to disable this redirection automatically if an `/index.js` file isn't present at root of the pages directory. 

TODO: 

- [x] ensure locale detection/populating works in serverless mode correctly
- [x] add tests for locale handling in different modes, fallback/getStaticProps/getServerSideProps

To be continued in fall-up PRs

- [ ] add tests for revalidate, auto-export, basePath + i18n
- [ ] add mapping of domains with locales
- [ ] investigate detecting locale against non-index routes and populating the locale in a cookie

x-ref: #17110

Author: ijjk

packages/next/build/entries.ts

@@ -97,6 +97,9 @@ export function createEntrypoints(
     loadedEnvFiles: Buffer.from(JSON.stringify(loadedEnvFiles)).toString(
       'base64'
     ),
+    i18n: config.experimental.i18n
+      ? JSON.stringify(config.experimental.i18n)
+      : '',
   }
 
   Object.keys(pages).forEach((page) => {

packages/next/build/index.ts

@@ -563,7 +563,9 @@ export default async function build(
           let workerResult = await staticCheckWorkers.isPageStatic(
             page,
             serverBundle,
-            runtimeEnvConfig
+            runtimeEnvConfig,
+            config.experimental.i18n?.locales,
+            config.experimental.i18n?.defaultLocale
           )
 
           if (workerResult.isHybridAmp) {

packages/next/build/utils.ts

@@ -27,6 +27,7 @@ import { denormalizePagePath } from '../next-server/server/normalize-page-path'
 import { BuildManifest } from '../next-server/server/get-page-files'
 import { removePathTrailingSlash } from '../client/normalize-trailing-slash'
 import type { UnwrapPromise } from '../lib/coalesced-function'
+import { normalizeLocalePath } from '../next-server/lib/i18n/normalize-locale-path'
 
 const fileGzipStats: { [k: string]: Promise<number> } = {}
 const fsStatGzip = (file: string) => {
@@ -530,7 +531,9 @@ export async function getJsPageSizeInKb(
 
 export async function buildStaticPaths(
   page: string,
-  getStaticPaths: GetStaticPaths
+  getStaticPaths: GetStaticPaths,
+  locales?: string[],
+  defaultLocale?: string
 ): Promise<
   Omit<UnwrapPromise<ReturnType<GetStaticPaths>>, 'paths'> & { paths: string[] }
 > {
@@ -595,7 +598,17 @@ export async function buildStaticPaths(
     // route.
     if (typeof entry === 'string') {
       entry = removePathTrailingSlash(entry)
-      const result = _routeMatcher(entry)
+
+      const localePathResult = normalizeLocalePath(entry, locales)
+      let cleanedEntry = entry
+
+      if (localePathResult.detectedLocale) {
+        cleanedEntry = entry.substr(localePathResult.detectedLocale.length + 1)
+      } else if (defaultLocale) {
+        entry = `/${defaultLocale}${entry}`
+      }
+
+      const result = _routeMatcher(cleanedEntry)
       if (!result) {
         throw new Error(
           `The provided path \`${entry}\` does not match the page: \`${page}\`.`
@@ -607,7 +620,10 @@ export async function buildStaticPaths(
     // For the object-provided path, we must make sure it specifies all
     // required keys.
     else {
-      const invalidKeys = Object.keys(entry).filter((key) => key !== 'params')
+      const invalidKeys = Object.keys(entry).filter(
+        (key) => key !== 'params' && key !== 'locale'
+      )
+
       if (invalidKeys.length) {
         throw new Error(
           `Additional keys were returned from \`getStaticPaths\` in page "${page}". ` +
@@ -657,7 +673,14 @@ export async function buildStaticPaths(
           .replace(/(?!^)\/$/, '')
       })
 
-      prerenderPaths?.add(builtPage)
+      if (entry.locale && !locales?.includes(entry.locale)) {
+        throw new Error(
+          `Invalid locale returned from getStaticPaths for ${page}, the locale ${entry.locale} is not specified in next.config.js`
+        )
+      }
+      const curLocale = entry.locale || defaultLocale || ''
+
+      prerenderPaths?.add(`${curLocale ? `/${curLocale}` : ''}${builtPage}`)
     }
   })
 
@@ -667,7 +690,9 @@ export async function buildStaticPaths(
 export async function isPageStatic(
   page: string,
   serverBundle: string,
-  runtimeEnvConfig: any
+  runtimeEnvConfig: any,
+  locales?: string[],
+  defaultLocale?: string
 ): Promise<{
   isStatic?: boolean
   isAmpOnly?: boolean
@@ -755,7 +780,12 @@ export async function isPageStatic(
       ;({
         paths: prerenderRoutes,
         fallback: prerenderFallback,
-      } = await buildStaticPaths(page, mod.getStaticPaths))
+      } = await buildStaticPaths(
+        page,
+        mod.getStaticPaths,
+        locales,
+        defaultLocale
+      ))
     }
 
     const config = mod.config || {}

packages/next/build/webpack-config.ts

@@ -986,6 +986,9 @@ export default async function getBaseWebpackConfig(
         ),
         'process.env.__NEXT_ROUTER_BASEPATH': JSON.stringify(config.basePath),
         'process.env.__NEXT_HAS_REWRITES': JSON.stringify(hasRewrites),
+        'process.env.__NEXT_i18n_SUPPORT': JSON.stringify(
+          !!config.experimental.i18n
+        ),
         ...(isServer
           ? {
               // Fix bad-actors in the npm ecosystem (e.g. `node-formidable`)

packages/next/build/webpack/loaders/next-serverless-loader.ts

@@ -28,6 +28,7 @@ export type ServerlessLoaderQuery = {
   runtimeConfig: string
   previewProps: string
   loadedEnvFiles: string
+  i18n: string
 }
 
 const vercelHeader = 'x-vercel-id'
@@ -49,6 +50,7 @@ const nextServerlessLoader: loader.Loader = function () {
     runtimeConfig,
     previewProps,
     loadedEnvFiles,
+    i18n,
   }: ServerlessLoaderQuery =
     typeof this.query === 'string' ? parse(this.query.substr(1)) : this.query
 
@@ -66,6 +68,8 @@ const nextServerlessLoader: loader.Loader = function () {
     JSON.parse(previewProps) as __ApiPreviewProps
   )
 
+  const i18nEnabled = !!i18n
+
   const defaultRouteRegex = pageIsDynamicRoute
     ? `
       const defaultRouteRegex = getRouteRegex("${page}")
@@ -212,6 +216,58 @@ const nextServerlessLoader: loader.Loader = function () {
   `
     : ''
 
+  const handleLocale = i18nEnabled
+    ? `
+      // get pathname from URL with basePath stripped for locale detection
+      const i18n = ${i18n}
+      const accept = require('@hapi/accept')
+      const { detectLocaleCookie } = require('next/dist/next-server/lib/i18n/detect-locale-cookie')
+      const { normalizeLocalePath } = require('next/dist/next-server/lib/i18n/normalize-locale-path')
+      let detectedLocale = detectLocaleCookie(req, i18n.locales)
+
+      if (!detectedLocale) {
+        detectedLocale = accept.language(
+          req.headers['accept-language'],
+          i18n.locales
+        ) || i18n.defaultLocale
+      }
+
+      if (
+        !nextStartMode &&
+        i18n.localeDetection !== false &&
+        denormalizePagePath(parsedUrl.pathname || '/') === '/'
+      ) {
+        res.setHeader(
+          'Location',
+          formatUrl({
+            // make sure to include any query values when redirecting
+            ...parsedUrl,
+            pathname: \`/\${detectedLocale}\`,
+          })
+        )
+        res.statusCode = 307
+        res.end()
+      }
+
+      // TODO: domain based locales (domain to locale mapping needs to be provided in next.config.js)
+      const localePathResult = normalizeLocalePath(parsedUrl.pathname, i18n.locales)
+
+      if (localePathResult.detectedLocale) {
+        detectedLocale = localePathResult.detectedLocale
+        req.url = formatUrl({
+          ...parsedUrl,
+          pathname: localePathResult.pathname,
+        })
+        parsedUrl.pathname = localePathResult.pathname
+      }
+
+      detectedLocale = detectedLocale || i18n.defaultLocale
+    `
+    : `
+      const i18n = {}
+      const detectedLocale = undefined
+    `
+
   if (page.match(API_ROUTE)) {
     return `
       import initServer from 'next-plugin-loader?middleware=on-init-server!'
@@ -305,6 +361,7 @@ const nextServerlessLoader: loader.Loader = function () {
     const { renderToHTML } = require('next/dist/next-server/server/render');
     const { tryGetPreviewData } = require('next/dist/next-server/server/api-utils');
     const { denormalizePagePath } = require('next/dist/next-server/server/denormalize-page-path')
+    const { setLazyProp, getCookieParser } = require('next/dist/next-server/server/api-utils')
     const {sendPayload} = require('next/dist/next-server/server/send-payload');
     const buildManifest = require('${buildManifest}');
     const reactLoadableManifest = require('${reactLoadableManifest}');
@@ -338,6 +395,9 @@ const nextServerlessLoader: loader.Loader = function () {
     export const _app = App
     export async function renderReqToHTML(req, res, renderMode, _renderOpts, _params) {
       const fromExport = renderMode === 'export' || renderMode === true;
+      const nextStartMode = renderMode === 'passthrough'
+
+      setLazyProp({ req }, 'cookies', getCookieParser(req))
 
       const options = {
         App,
@@ -388,12 +448,16 @@ const nextServerlessLoader: loader.Loader = function () {
           routeNoAssetPath = parsedUrl.pathname
         }
 
+        ${handleLocale}
+
         const renderOpts = Object.assign(
           {
             Component,
             pageConfig: config,
             nextExport: fromExport,
             isDataReq: _nextData,
+            locale: detectedLocale,
+            locales: i18n.locales,
           },
           options,
         )

packages/next/client/index.tsx

@@ -11,7 +11,11 @@ import type {
   AppProps,
   PrivateRouteInfo,
 } from '../next-server/lib/router/router'
-import { delBasePath, hasBasePath } from '../next-server/lib/router/router'
+import {
+  delBasePath,
+  hasBasePath,
+  delLocale,
+} from '../next-server/lib/router/router'
 import { isDynamicRoute } from '../next-server/lib/router/utils/is-dynamic'
 import * as querystring from '../next-server/lib/router/utils/querystring'
 import * as envConfig from '../next-server/lib/runtime-config'
@@ -60,8 +64,11 @@ const {
   dynamicIds,
   isFallback,
   head: initialHeadData,
+  locales,
 } = data
 
+let { locale } = data
+
 const prefix = assetPrefix || ''
 
 // With dynamic assetPrefix it's no longer possible to set assetPrefix at the build time
@@ -80,6 +87,23 @@ if (hasBasePath(asPath)) {
   asPath = delBasePath(asPath)
 }
 
+asPath = delLocale(asPath, locale)
+
+if (process.env.__NEXT_i18n_SUPPORT) {
+  const {
+    normalizeLocalePath,
+  } = require('../next-server/lib/i18n/normalize-locale-path')
+
+  if (isFallback && locales) {
+    const localePathResult = normalizeLocalePath(asPath, locales)
+
+    if (localePathResult.detectedLocale) {
+      asPath = asPath.substr(localePathResult.detectedLocale.length + 1)
+      locale = localePathResult.detectedLocale
+    }
+  }
+}
+
 type RegisterFn = (input: [string, () => void]) => void
 
 const pageLoader = new PageLoader(buildId, prefix, page)
@@ -291,6 +315,8 @@ export default async (opts: { webpackHMR?: any } = {}) => {
     isFallback: Boolean(isFallback),
     subscription: ({ Component, styleSheets, props, err }, App) =>
       render({ App, Component, styleSheets, props, err }),
+    locale,
+    locales,
   })
 
   // call init-client middleware

packages/next/client/link.tsx

@@ -2,6 +2,7 @@ import React, { Children } from 'react'
 import { UrlObject } from 'url'
 import {
   addBasePath,
+  addLocale,
   isLocalURL,
   NextRouter,
   PrefetchOptions,
@@ -331,7 +332,7 @@ function Link(props: React.PropsWithChildren<LinkProps>) {
   // If child is an <a> tag and doesn't have a href attribute, or if the 'passHref' property is
   // defined, we specify the current 'href', so that repetition is not needed by the user
   if (props.passHref || (child.type === 'a' && !('href' in child.props))) {
-    childProps.href = addBasePath(as)
+    childProps.href = addBasePath(addLocale(as, router && router.locale))
   }
 
   return React.cloneElement(child, childProps)

packages/next/client/page-loader.ts

@@ -7,6 +7,7 @@ import {
   addBasePath,
   markLoadingError,
   interpolateAs,
+  addLocale,
 } from '../next-server/lib/router/router'
 
 import getAssetPathFromRoute from '../next-server/lib/router/utils/get-asset-path-from-route'
@@ -202,13 +203,13 @@ export default class PageLoader {
    * @param {string} href the route href (file-system path)
    * @param {string} asPath the URL as shown in browser (virtual path); used for dynamic routes
    */
-  getDataHref(href: string, asPath: string, ssg: boolean) {
+  getDataHref(href: string, asPath: string, ssg: boolean, locale?: string) {
     const { pathname: hrefPathname, query, search } = parseRelativeUrl(href)
     const { pathname: asPathname } = parseRelativeUrl(asPath)
     const route = normalizeRoute(hrefPathname)
 
     const getHrefForSlug = (path: string) => {
-      const dataRoute = getAssetPathFromRoute(path, '.json')
+      const dataRoute = addLocale(getAssetPathFromRoute(path, '.json'), locale)
       return addBasePath(
         `/_next/data/${this.buildId}${dataRoute}${ssg ? '' : search}`
       )

packages/next/client/router.ts

@@ -37,6 +37,8 @@ const urlPropertyFields = [
   'components',
   'isFallback',
   'basePath',
+  'locale',
+  'locales',
 ]
 const routerEvents = [
   'routeChangeStart',
@@ -144,7 +146,10 @@ export function makePublicRouterInstance(router: Router): NextRouter {
 
   for (const property of urlPropertyFields) {
     if (typeof _router[property] === 'object') {
-      instance[property] = Object.assign({}, _router[property]) // makes sure query is not stateful
+      instance[property] = Object.assign(
+        Array.isArray(_router[property]) ? [] : {},
+        _router[property]
+      ) // makes sure query is not stateful
       continue
     }
 

packages/next/export/index.ts

@@ -298,6 +298,8 @@ export default async function exportApp(
     ampValidatorPath: nextConfig.experimental.amp?.validator || undefined,
     ampSkipValidation: nextConfig.experimental.amp?.skipValidation || false,
     ampOptimizerConfig: nextConfig.experimental.amp?.optimizer || undefined,
+    locales: nextConfig.experimental.i18n?.locales,
+    locale: nextConfig.experimental.i18n?.defaultLocale,
   }
 
   const { serverRuntimeConfig, publicRuntimeConfig } = nextConfig