Experimental: Serverless Trace target (#8246)

* Experimental: Serverless Trace target
The Serverless Trace target produces Serverless-handler wrapped entrypoints, but does not bundle all of `node_modules`.

This behavior increases bundling performance to be more akin to `target: 'server'`.

This mode is expected to be used with smart platforms (like [ZEIT Now](https://zeit.co/now) that can trace a program to its minimum dependencies.

* Use more generic variables

* Add asset relocator for production mode of serverless trace

* Verify Firebase compatiblity

* Revert "Add asset relocator for production mode of serverless trace"

This reverts commit 8404f1d.

* Add serverless trace tests

* Add _isLikeServerless helper

* Make constants

* Fix export

* Update packages/next-server/server/config.ts

Co-Authored-By: JJ Kasper <jj@jjsweb.site>

* Use a global helper for is like serverless

* Update import for isTargetLikeServerless

* Update packages/next/build/index.ts

Co-Authored-By: JJ Kasper <jj@jjsweb.site>

package.json

@@ -81,6 +81,7 @@
  "execa": "2.0.3",
  "express": "4.17.0",
  "faunadb": "2.6.1",
+ "firebase": "6.3.4",
  "fs-extra": "7.0.1",
  "get-port": "5.0.0",
  "isomorphic-unfetch": "3.0.0",

packages/next-server/server/config.ts

@@ -1,9 +1,10 @@
-import os from 'os'
 import findUp from 'find-up'
+import os from 'os'
+
 import { CONFIG_FILE } from '../lib/constants'
 import { execOnce } from '../lib/utils'
 
-const targets = ['server', 'serverless']
+const targets = ['server', 'serverless', 'experimental-serverless-trace']
 
 const defaultConfig: { [key: string]: any } = {
  env: [],
@@ -120,10 +121,12 @@ export default function loadConfig(
  }
 
  if (
- userConfig.target === 'serverless' &&
+ userConfig.target &&
+ userConfig.target !== 'server' &&
  userConfig.publicRuntimeConfig &&
  Object.keys(userConfig.publicRuntimeConfig).length !== 0
  ) {
+ // TODO: change error message tone to "Only compatible with [fat] server mode"
  throw new Error(
  'Cannot use publicRuntimeConfig with target=serverless https://err.sh/zeit/next.js/serverless-publicRuntimeConfig'
  )
@@ -134,3 +137,9 @@ export default function loadConfig(
 
  return defaultConfig
 }
+
+export function isTargetLikeServerless(target: string) {
+ const isServerless = target === 'serverless'
+ const isServerlessTrace = target === 'experimental-serverless-trace'
+ return isServerless || isServerlessTrace
+}

packages/next-server/server/next-server.ts

@@ -1,13 +1,12 @@
+import compression from 'compression'
 import fs from 'fs'
 import { IncomingMessage, ServerResponse } from 'http'
 import { join, resolve, sep } from 'path'
 import { parse as parseQs, ParsedUrlQuery } from 'querystring'
 import { parse as parseUrl, UrlWithParsedQuery } from 'url'
-import compression from 'compression'
 
 import {
  BUILD_ID_FILE,
- BUILD_MANIFEST,
  CLIENT_PUBLIC_FILES_PATH,
  CLIENT_STATIC_FILES_PATH,
  CLIENT_STATIC_FILES_RUNTIME,
@@ -25,12 +24,12 @@ import {
 import * as envConfig from '../lib/runtime-config'
 import { NextApiRequest, NextApiResponse } from '../lib/utils'
 import { apiResolver } from './api-utils'
-import loadConfig from './config'
+import loadConfig, { isTargetLikeServerless } from './config'
 import { recursiveReadDirSync } from './lib/recursive-readdir-sync'
 import { loadComponents, LoadComponentsReturnType } from './load-components'
 import { renderToHTML } from './render'
 import { getPagePath } from './require'
-import Router, { route, Route, RouteMatch, Params } from './router'
+import Router, { Params, route, Route, RouteMatch } from './router'
 import { sendHTML } from './send-html'
 import { serveStatic } from './serve-static'
 import { isBlockedPage, isInternalUrl } from './utils'
@@ -98,7 +97,9 @@ export default class Server {
  this.publicDir = join(this.dir, CLIENT_PUBLIC_FILES_PATH)
  this.pagesManifest = join(
  this.distDir,
- this.nextConfig.target || 'server',
+ this.nextConfig.target === 'server'
+ ? SERVER_DIRECTORY
+ : SERVERLESS_DIRECTORY,
  PAGES_MANIFEST
  )
 
@@ -131,7 +132,7 @@ export default class Server {
  this.renderOpts.runtimeConfig = publicRuntimeConfig
  }
 
- if (compress && this.nextConfig.target !== 'serverless') {
+ if (compress && this.nextConfig.target === 'server') {
  this.compression = compression() as Middleware
  }
 
@@ -319,7 +320,7 @@ export default class Server {
  return this.render404(req, res)
  }
 
- if (!this.renderOpts.dev && this.nextConfig.target === 'serverless') {
+ if (!this.renderOpts.dev && this._isLikeServerless) {
  const mod = require(resolverFunction)
  if (typeof mod.default === 'function') {
  return mod.default(req, res)
@@ -342,7 +343,7 @@ export default class Server {
  return getPagePath(
  pathname,
  this.distDir,
- this.nextConfig.target === 'serverless',
+ this._isLikeServerless,
  this.renderOpts.dev
  )
  }
@@ -352,9 +353,7 @@ export default class Server {
  const publicFiles = recursiveReadDirSync(this.publicDir)
  const serverBuildPath = join(
  this.distDir,
- this.nextConfig.target === 'serverless'
- ? SERVERLESS_DIRECTORY
- : SERVER_DIRECTORY
+ this._isLikeServerless ? SERVERLESS_DIRECTORY : SERVER_DIRECTORY
  )
  const pagesManifest = require(join(serverBuildPath, PAGES_MANIFEST))
 
@@ -458,8 +457,7 @@ export default class Server {
  pathname: string,
  query: ParsedUrlQuery = {}
  ) {
- const serverless =
- !this.renderOpts.dev && this.nextConfig.target === 'serverless'
+ const serverless = !this.renderOpts.dev && this._isLikeServerless
  // try serving a static AMP version first
  if (query.amp) {
  try {
@@ -708,4 +706,8 @@ export default class Server {
  throw err
  }
  }
+
+ private get _isLikeServerless(): boolean {
+ return isTargetLikeServerless(this.nextConfig.target)
+ }
 }

packages/next/build/entries.ts

@@ -1,6 +1,8 @@
+import { isTargetLikeServerless } from 'next-server/dist/server/config'
 import { join } from 'path'
 import { stringify } from 'querystring'
-import { PAGES_DIR_ALIAS, DOT_NEXT_ALIAS, API_ROUTE } from '../lib/constants'
+
+import { API_ROUTE, DOT_NEXT_ALIAS, PAGES_DIR_ALIAS } from '../lib/constants'
 import { ServerlessLoaderQuery } from './webpack/loaders/next-serverless-loader'
 
 type PagesMapping = {
@@ -45,7 +47,7 @@ type Entrypoints = {
 
 export function createEntrypoints(
  pages: PagesMapping,
- target: 'server' | 'serverless',
+ target: 'server' | 'serverless' | 'experimental-serverless-trace',
  buildId: string,
  dynamicBuildId: boolean,
  config: any
@@ -72,7 +74,9 @@ export function createEntrypoints(
 
  const bundlePath = join('static', buildId, 'pages', bundleFile)
 
- if (isApiRoute && target === 'serverless') {
+ const isLikeServerless = isTargetLikeServerless(target)
+
+ if (isApiRoute && isLikeServerless) {
  const serverlessLoaderOptions: ServerlessLoaderQuery = {
  page,
  absolutePagePath,
@@ -83,11 +87,7 @@ export function createEntrypoints(
  )}!`
  } else if (isApiRoute || target === 'server') {
  server[bundlePath] = [absolutePagePath]
- } else if (
- target === 'serverless' &&
- page !== '/_app' &&
- page !== '/_document'
- ) {
+ } else if (isLikeServerless && page !== '/_app' && page !== '/_document') {
  const serverlessLoaderOptions: ServerlessLoaderQuery = {
  page,
  absolutePagePath,

packages/next/build/index.ts

@@ -1,18 +1,25 @@
+import { Sema } from 'async-sema'
 import chalk from 'chalk'
+import fs from 'fs'
+import mkdirpOrig from 'mkdirp'
 import {
- SERVER_DIRECTORY,
- SERVERLESS_DIRECTORY,
- PAGES_MANIFEST,
  CHUNK_GRAPH_MANIFEST,
+ PAGES_MANIFEST,
  PHASE_PRODUCTION_BUILD,
+ SERVER_DIRECTORY,
+ SERVERLESS_DIRECTORY,
 } from 'next-server/constants'
-import loadConfig from 'next-server/next-config'
+import loadConfig, {
+ isTargetLikeServerless,
+} from 'next-server/dist/server/config'
 import nanoid from 'next/dist/compiled/nanoid/index.js'
 import path from 'path'
-import fs from 'fs'
 import { promisify } from 'util'
+import workerFarm from 'worker-farm'
+
 import formatWebpackMessages from '../client/dev/error-overlay/format-webpack-messages'
 import { recursiveDelete } from '../lib/recursive-delete'
+import { recursiveReadDir } from '../lib/recursive-readdir'
 import { verifyTypeScriptSetup } from '../lib/verifyTypeScriptSetup'
 import { CompilerResult, runCompiler } from './compiler'
 import { createEntrypoints, createPagesMapping } from './entries'
@@ -25,20 +32,16 @@ import {
  getFileForPage,
  getPageSizeInKb,
  getSpecifiedPages,
- printTreeView,
- PageInfo,
  hasCustomAppGetInitialProps,
+ PageInfo,
+ printTreeView,
 } from './utils'
 import getBaseWebpackConfig from './webpack-config'
 import {
  exportManifest,
  getPageChunks,
 } from './webpack/plugins/chunk-graph-plugin'
 import { writeBuildId } from './write-build-id'
-import { recursiveReadDir } from '../lib/recursive-readdir'
-import mkdirpOrig from 'mkdirp'
-import workerFarm from 'worker-farm'
-import { Sema } from 'async-sema'
 
 const fsUnlink = promisify(fs.unlink)
 const fsRmdir = promisify(fs.rmdir)
@@ -75,11 +78,13 @@ export default async function build(dir: string, conf = null): Promise<void> {
  isFlyingShuttle || process.env.__NEXT_BUILDER_EXPERIMENTAL_PAGE
  )
 
+ const isLikeServerless = isTargetLikeServerless(target)
+
  if (selectivePageBuilding && target !== 'serverless') {
  throw new Error(
  `Cannot use ${
  isFlyingShuttle ? 'flying shuttle' : '`now dev`'
- } without the serverless target.`
+ } without the \`serverless\` target.`
  )
  }
 
@@ -199,7 +204,8 @@ export default async function build(dir: string, conf = null): Promise<void> {
  ])
 
  let result: CompilerResult = { warnings: [], errors: [] }
- if (target === 'serverless') {
+ // TODO: why do we need this?? https://github.com/zeit/next.js/issues/8253
+ if (isLikeServerless) {
  const clientResult = await runCompiler(configs[0])
  // Fail build if clientResult contains errors
  if (clientResult.errors.length > 0) {
@@ -273,7 +279,7 @@ export default async function build(dir: string, conf = null): Promise<void> {
  const pageKeys = Object.keys(mappedPages)
  const manifestPath = path.join(
  distDir,
- target === 'serverless' ? SERVERLESS_DIRECTORY : SERVER_DIRECTORY,
+ isLikeServerless ? SERVERLESS_DIRECTORY : SERVER_DIRECTORY,
  PAGES_MANIFEST
  )
 
@@ -303,12 +309,12 @@ export default async function build(dir: string, conf = null): Promise<void> {
  const actualPage = page === '/' ? '/index' : page
  const size = await getPageSizeInKb(actualPage, distPath, buildId)
  const bundleRelative = path.join(
- target === 'serverless' ? 'pages' : `static/${buildId}/pages`,
+ isLikeServerless ? 'pages' : `static/${buildId}/pages`,
  actualPage + '.js'
  )
  const serverBundle = path.join(
  distPath,
- target === 'serverless' ? SERVERLESS_DIRECTORY : SERVER_DIRECTORY,
+ isLikeServerless ? SERVERLESS_DIRECTORY : SERVER_DIRECTORY,
  bundleRelative
  )
 
@@ -324,7 +330,7 @@ export default async function build(dir: string, conf = null): Promise<void> {
 
  if (nonReservedPage && customAppGetInitialProps === undefined) {
  customAppGetInitialProps = hasCustomAppGetInitialProps(
- target === 'serverless'
+ isLikeServerless
  ? serverBundle
  : path.join(
  distPath,
@@ -437,7 +443,7 @@ export default async function build(dir: string, conf = null): Promise<void> {
  for (const file of toMove) {
  const orig = path.join(exportOptions.outdir, file)
  const dest = path.join(serverDir, file)
- const relativeDest = (target === 'serverless'
+ const relativeDest = (isLikeServerless
  ? path.join('pages', file)
  : path.join('static', buildId, 'pages', file)
  ).replace(/\\/g, '/')
@@ -465,9 +471,5 @@ export default async function build(dir: string, conf = null): Promise<void> {
  await flyingShuttle.save(allStaticPages, pageInfos)
  }
 
- printTreeView(
- Object.keys(allMappedPages),
- allPageInfos,
- target === 'serverless'
- )
+ printTreeView(Object.keys(allMappedPages), allPageInfos, isLikeServerless)
 }

packages/next/build/webpack-config.ts

@@ -1,10 +1,10 @@
 import ForkTsCheckerWebpackPlugin from 'fork-ts-checker-webpack-plugin'
-import fs from 'fs'
 import {
  CLIENT_STATIC_FILES_RUNTIME_MAIN,
  CLIENT_STATIC_FILES_RUNTIME_WEBPACK,
  REACT_LOADABLE_MANIFEST,
  SERVER_DIRECTORY,
+ SERVERLESS_DIRECTORY,
 } from 'next-server/constants'
 import resolve from 'next/dist/compiled/resolve/index.js'
 import path from 'path'
@@ -25,14 +25,14 @@ import ChunkNamesPlugin from './webpack/plugins/chunk-names-plugin'
 import { importAutoDllPlugin } from './webpack/plugins/dll-import'
 import { HashedChunkIdsPlugin } from './webpack/plugins/hashed-chunk-ids-plugin'
 import { DropClientPage } from './webpack/plugins/next-drop-client-page-plugin'
+import NextEsmPlugin from './webpack/plugins/next-esm-plugin'
 import NextJsSsrImportPlugin from './webpack/plugins/nextjs-ssr-import'
 import NextJsSSRModuleCachePlugin from './webpack/plugins/nextjs-ssr-module-cache'
 import PagesManifestPlugin from './webpack/plugins/pages-manifest-plugin'
 import { ReactLoadablePlugin } from './webpack/plugins/react-loadable-plugin'
 import { ServerlessPlugin } from './webpack/plugins/serverless-plugin'
 import { SharedRuntimePlugin } from './webpack/plugins/shared-runtime-plugin'
 import { TerserPlugin } from './webpack/plugins/terser-webpack-plugin/src/index'
-import NextEsmPlugin from './webpack/plugins/next-esm-plugin'
 
 type ExcludesFalse = <T>(x: T | false) => x is T
 
@@ -79,7 +79,12 @@ export default async function getBaseWebpackConfig(
  .split(process.platform === 'win32' ? ';' : ':')
  .filter(p => !!p)
 
- const outputDir = target === 'serverless' ? 'serverless' : SERVER_DIRECTORY
+ const isServerless = target === 'serverless'
+ const isServerlessTrace = target === 'experimental-serverless-trace'
+ // Intentionally not using isTargetLikeServerless helper
+ const isLikeServerless = isServerless || isServerlessTrace
+
+ const outputDir = isLikeServerless ? SERVERLESS_DIRECTORY : SERVER_DIRECTORY
  const outputPath = path.join(distDir, isServer ? outputDir : '')
  const totalPages = Object.keys(entrypoints).length
  const clientEntries = !isServer
@@ -238,7 +243,7 @@ export default async function getBaseWebpackConfig(
  target: isServer ? 'node' : 'web',
  externals: !isServer
  ? undefined
- : target !== 'serverless'
+ : !isServerless
  ? [
  (context, request, callback) => {
  const notExternalModules = [
@@ -293,8 +298,8 @@ export default async function getBaseWebpackConfig(
  },
  ]
  : [
- // When the serverless target is used all node_modules will be compiled into the output bundles
- // So that the serverless bundles have 0 runtime dependencies
+ // When the 'serverless' target is used all node_modules will be compiled into the output bundles
+ // So that the 'serverless' bundles have 0 runtime dependencies
  'amp-toolbox-optimizer', // except this one
  (context, request, callback) => {
  if (
@@ -561,11 +566,14 @@ export default async function getBaseWebpackConfig(
  )
  },
  }),
- target === 'serverless' &&
- (isServer || selectivePageBuilding) &&
- new ServerlessPlugin(buildId, { isServer }),
- isServer && new PagesManifestPlugin(target === 'serverless'),
- target !== 'serverless' &&
+ isLikeServerless &&
+ new ServerlessPlugin(buildId, {
+ isServer,
+ isFlyingShuttle: selectivePageBuilding,
+ isTrace: isServerlessTrace,
+ }),
+ isServer && new PagesManifestPlugin(isLikeServerless),
+ target === 'server' &&
  isServer &&
  new NextJsSSRModuleCachePlugin({ outputPath }),
  isServer && new NextJsSsrImportPlugin(),