Add auto static/dynamic (#7293)

* Add automatic exporting of pages with no getInitialProps

* Add support for exporting serverless to static
and serving the html files during next start

* Fix missing runtimeEnv when requiring page, re-add warning
when trying to export with serverless, and update tests

* Update flying-shuttle test

* revert un-used pagesManifest change

* remove query.amp RegExp test

* Fix windows backslashes not being replaced

* Re-enable serverless support for next start

* bump

* Fix getInitialProps check

* Fix incorrect error check

* Re-add check for reserved pages

* Fix static check

* Update to ignore /api pages and clean up some tests

* Re-add needed next.config for test and correct behavior

* Update RegExp for ignored pages for auto-static

* Add checking for custom getInitialProps in pages/_app

* Update isPageStatic logic to only use default export

* Re-add retrying to CircleCi

* Update query during dev to only have values
available during export for static pages

* Fix test

* Add warning when page without default export is
found and make sure to update pages-manifest
correctly in flying-shuttle mode

* Fix backslashes not being replaced

* Integrate auto-static with flying-shuttle
and make sure AMP is handled in flying-shuttle

* Add autoExport for opting in

packages/next-server/lib/constants.ts

@@ -7,6 +7,7 @@ export const BUILD_MANIFEST = 'build-manifest.json'
 export const REACT_LOADABLE_MANIFEST = 'react-loadable-manifest.json'
 export const CHUNK_GRAPH_MANIFEST = 'compilation-modules.json'
 export const SERVER_DIRECTORY = 'server'
+export const SERVERLESS_DIRECTORY = 'serverless'
 export const CONFIG_FILE = 'next.config.js'
 export const BUILD_ID_FILE = 'BUILD_ID'
 export const BLOCKED_PAGES = [
@@ -27,4 +28,5 @@ export const CLIENT_STATIC_FILES_RUNTIME_WEBPACK = `${CLIENT_STATIC_FILES_RUNTIM
 export const IS_BUNDLED_PAGE_REGEX = /^static[/\\][^/\\]+[/\\]pages.*\.js$/
 // matches static/<buildid>/pages/:page*.js
 export const ROUTE_NAME_REGEX = /^static[/\\][^/\\]+[/\\]pages[/\\](.*)\.js$/
+export const SERVERLESS_ROUTE_NAME_REGEX = /^pages[/\\](.*)\.js$/
 export const HEAD_BUILD_ID_FILE = `${CLIENT_STATIC_FILES_PATH}/HEAD_BUILD_ID`

packages/next-server/server/config.ts

@@ -27,6 +27,7 @@ const defaultConfig: {[key: string]: any} = {
       (Number(process.env.CIRCLE_NODE_TOTAL) ||
         (os.cpus() || { length: 1 }).length) - 1,
     ),
+    autoExport: false,
     ampBindInitData: false,
     exportTrailingSlash: true,
     terserLoader: false,

packages/next-server/server/load-components.ts

@@ -1,4 +1,4 @@
-import {BUILD_MANIFEST, CLIENT_STATIC_FILES_PATH, REACT_LOADABLE_MANIFEST, SERVER_DIRECTORY} from '../lib/constants';
+import {BUILD_MANIFEST, CLIENT_STATIC_FILES_PATH, REACT_LOADABLE_MANIFEST, SERVER_DIRECTORY, SERVERLESS_DIRECTORY} from '../lib/constants';
 import { join } from 'path';
 
 import { requirePage } from './require';
@@ -7,14 +7,18 @@ export function interopDefault(mod: any) {
   return mod.default || mod
 }
 
-export async function loadComponents(distDir: string, buildId: string, pathname: string) {
+export async function loadComponents(distDir: string, buildId: string, pathname: string, serverless: boolean) {
+  if (serverless) {
+    const Component = await requirePage(pathname, distDir, serverless)
+    return { Component }
+  }
   const documentPath = join(distDir, SERVER_DIRECTORY, CLIENT_STATIC_FILES_PATH, buildId, 'pages', '_document')
   const appPath = join(distDir, SERVER_DIRECTORY, CLIENT_STATIC_FILES_PATH, buildId, 'pages', '_app')
 
   const [buildManifest, reactLoadableManifest, Component, Document, App] = await Promise.all([
     require(join(distDir, BUILD_MANIFEST)),
     require(join(distDir, REACT_LOADABLE_MANIFEST)),
-    interopDefault(requirePage(pathname, distDir)),
+    interopDefault(requirePage(pathname, distDir, serverless)),
     interopDefault(require(documentPath)),
     interopDefault(require(appPath)),
   ])

packages/next-server/server/next-server.ts

@@ -72,14 +72,8 @@ export default class Server {
       publicRuntimeConfig,
       assetPrefix,
       generateEtags,
-      target,
     } = this.nextConfig
 
-    if (process.env.NODE_ENV === 'production' && target !== 'server')
-      throw new Error(
-        'Cannot start server when target is not server. https://err.sh/zeit/next.js/next-start-serverless',
-      )
-
     this.buildId = this.readBuildId()
     this.renderOpts = {
       ampBindInitData: this.nextConfig.experimental.ampBindInitData,
@@ -257,7 +251,7 @@ export default class Server {
    * @param pathname path of request
    */
   private resolveApiRequest(pathname: string) {
-    return getPagePath(pathname, this.distDir)
+    return getPagePath(pathname, this.distDir, this.nextConfig.target === 'serverless')
   }
 
   private generatePublicRoutes(): Route[] {
@@ -353,7 +347,25 @@ export default class Server {
     query: ParsedUrlQuery = {},
     opts: any,
   ) {
-    const result = await loadComponents(this.distDir, this.buildId, pathname)
+    const serverless = this.nextConfig.target === 'serverless'
+    // try serving a static AMP version first
+    if (query.amp) {
+      try {
+        const result = await loadComponents(this.distDir, this.buildId, (pathname === '/' ? '/index' : pathname) + '.amp', serverless)
+        if (typeof result.Component === 'string') return result.Component
+      } catch (err) {
+        if (err.code !== 'ENOENT') throw err
+      }
+    }
+    const result = await loadComponents(this.distDir, this.buildId, pathname, serverless)
+    // handle static page
+    if (typeof result.Component === 'string') return result.Component
+    // handle serverless
+    if (typeof result.Component === 'object' &&
+      typeof result.Component.renderReqToHTML === 'function'
+    ) {
+      return result.Component.renderReqToHTML(req, res)
+    }
     return renderToHTML(req, res, pathname, query, { ...result, ...opts })
   }
 

packages/next-server/server/render.tsx

@@ -252,6 +252,16 @@ export async function renderToHTML(
         `The default export is not a React Component in page: "/_document"`,
       )
     }
+
+    const isStaticPage = typeof (Component as any).getInitialProps !== 'function'
+    const defaultAppGetInitialProps = App.getInitialProps === (App as any).origGetInitialProps
+
+    if (isStaticPage && defaultAppGetInitialProps) {
+      // remove query values except ones that will be set during export
+      query = {
+        amp: query.amp,
+      }
+    }
   }
 
   // @ts-ignore url will always be set
@@ -304,7 +314,7 @@ export async function renderToHTML(
 
   const ampMode = {
     enabled: false,
-    hasQuery: Boolean(query.amp && /^(y|yes|true|1)/i.test(query.amp.toString())),
+    hasQuery: Boolean(query.amp),
   }
 
   if (ampBindInitData) {

packages/next-server/server/require.ts

@@ -1,15 +1,19 @@
+import fs from 'fs'
 import {join} from 'path'
-import {PAGES_MANIFEST, SERVER_DIRECTORY} from '../lib/constants'
+import {promisify} from 'util'
+import {PAGES_MANIFEST, SERVER_DIRECTORY, SERVERLESS_DIRECTORY} from '../lib/constants'
 import { normalizePagePath } from './normalize-page-path'
 
+const readFile = promisify(fs.readFile)
+
 export function pageNotFoundError(page: string): Error {
   const err: any = new Error(`Cannot find module for page: ${page}`)
   err.code = 'ENOENT'
   return err
 }
 
-export function getPagePath(page: string, distDir: string): string {
-  const serverBuildPath = join(distDir, SERVER_DIRECTORY)
+export function getPagePath(page: string, distDir: string, serverless: boolean): string {
+  const serverBuildPath = join(distDir, serverless ? SERVERLESS_DIRECTORY : SERVER_DIRECTORY)
   const pagesManifest = require(join(serverBuildPath, PAGES_MANIFEST))
 
   try {
@@ -32,7 +36,10 @@ export function getPagePath(page: string, distDir: string): string {
   return join(serverBuildPath, pagesManifest[page])
 }
 
-export function requirePage(page: string, distDir: string): any {
-  const pagePath = getPagePath(page, distDir)
+export function requirePage(page: string, distDir: string, serverless: boolean): any {
+  const pagePath = getPagePath(page, distDir, serverless)
+  if (pagePath.endsWith('.html')) {
+    return readFile(pagePath, 'utf8')
+  }
   return require(pagePath)
 }

packages/next/build/flying-shuttle.ts

@@ -9,6 +9,7 @@ import { promisify } from 'util'
 
 import { recursiveDelete } from '../lib/recursive-delete'
 import * as Log from './output/log'
+import { PageInfo } from './utils';
 
 const FILE_BUILD_ID = 'HEAD_BUILD_ID'
 const FILE_UPDATED_AT = 'UPDATED_AT'
@@ -220,6 +221,29 @@ export class FlyingShuttle {
     return (this._shuttleBuildId = contents)
   }
 
+  getPageInfos = async (): Promise<Map<string, PageInfo>> => {
+    const pageInfos: Map<string, PageInfo> = new Map()
+    const pagesManifest = JSON.parse(await fsReadFile(
+      path.join(
+        this.shuttleDirectory, DIR_FILES_NAME, 'serverless/pages-manifest.json'
+      ),
+      'utf8'
+    ))
+    Object.keys(pagesManifest).forEach(pg => {
+      const path = pagesManifest[pg]
+      const isStatic: boolean = path.endsWith('html')
+      let isAmp = Boolean(pagesManifest[pg + '.amp'])
+      if (pg === '/') isAmp = Boolean(pagesManifest['/index.amp'])
+      pageInfos.set(pg, {
+        isAmp,
+        size: 0,
+        static: isStatic,
+        serverBundle: path
+      })
+    })
+    return pageInfos
+  }
+
   getUnchangedPages = async () => {
     const manifestPath = path.join(this.shuttleDirectory, CHUNK_GRAPH_MANIFEST)
     const manifest = require(manifestPath) as ChunkGraphManifest
@@ -276,8 +300,36 @@ export class FlyingShuttle {
     return unchangedPages
   }
 
-  restorePage = async (page: string): Promise<boolean> => {
+  mergePagesManifest = async (): Promise<void> => {
+    const savedPagesManifest = path.join(
+      this.shuttleDirectory, DIR_FILES_NAME, 'serverless/pages-manifest.json'
+    )
+    if (!(await fsExists(savedPagesManifest))) return
+
+    const saved = JSON.parse(await fsReadFile(
+      savedPagesManifest,
+      'utf8'
+    ))
+    const currentPagesManifest = path.join(
+      this.distDirectory, 'serverless/pages-manifest.json'
+    )
+    const current = JSON.parse(await fsReadFile(
+      currentPagesManifest,
+      'utf8'
+    ))
+
+    await fsWriteFile(currentPagesManifest, JSON.stringify({
+      ...saved,
+      ...current,
+    }))
+  }
+
+  restorePage = async (
+    page: string,
+    pageInfo: PageInfo = {} as PageInfo
+  ): Promise<boolean> => {
     await this._restoreSema.acquire()
+
     try {
       const manifestPath = path.join(
         this.shuttleDirectory,
@@ -293,10 +345,9 @@ export class FlyingShuttle {
 
       const serverless = path.join(
         'serverless/pages',
-        `${page === '/' ? 'index' : page}.js`
+        `${page === '/' ? 'index' : page}.${pageInfo.static ? 'html' : 'js'}`
       )
       const files = [serverless, ...pageChunks[page]]
-
       const filesExists = await Promise.all(
         files
           .map(f => path.join(this.shuttleDirectory, DIR_FILES_NAME, f))
@@ -366,7 +417,7 @@ export class FlyingShuttle {
     }
   }
 
-  save = async () => {
+  save = async (staticPages: Set<string>, pageInfos: Map<string, PageInfo>) => {
     Log.wait('docking flying shuttle')
 
     await recursiveDelete(this.shuttleDirectory)
@@ -419,10 +470,30 @@ export class FlyingShuttle {
     const usedChunks = new Set()
     const pages = Object.keys(storeManifest.pageChunks)
     pages.forEach(page => {
-      storeManifest.pageChunks[page].forEach(file => usedChunks.add(file))
+      const info = pageInfos.get(page) || {} as PageInfo
+
+      storeManifest.pageChunks[page].forEach((file, idx) => {
+        if (info.isAmp) {
+          // AMP pages don't have client bundles
+          storeManifest.pageChunks[page] = []
+          return
+        }
+        usedChunks.add(file)
+      })
       usedChunks.add(
-        path.join('serverless/pages', `${page === '/' ? 'index' : page}.js`)
+        path.join('serverless/pages', `${
+          page === '/' ? 'index' : page
+        }.${staticPages.has(page) ? 'html' : 'js'}`)
       )
+      const ampPage = (page === '/' ? '/index' : page) + '.amp'
+
+      if (staticPages.has(ampPage)) {
+        storeManifest.pages[ampPage] = []
+        storeManifest.pageChunks[ampPage] = []
+        usedChunks.add(
+          path.join('serverless/pages', `${ampPage}.html`)
+        )
+      }
     })
 
     await fsWriteFile(
@@ -442,6 +513,11 @@ export class FlyingShuttle {
       })
     )
 
+    await fsCopyFile(
+      path.join(this.distDirectory, 'serverless/pages-manifest.json'),
+      path.join(this.shuttleDirectory, DIR_FILES_NAME, 'serverless/pages-manifest.json')
+    )
+
     Log.info(`flying shuttle payload: ${usedChunks.size + 2} files`)
     Log.ready('flying shuttle docked')