Turbopack build: Add devlow-bench

.github/workflows/build_and_test.yml

@@ -186,7 +186,8 @@ jobs:
           - '--turbopack=false'
           - '--turbopack=true'
         selector:
-          - '--scenario=heavy-npm-deps --page=homepage'
+          - '--scenario=heavy-npm-deps --page=homepage --mode=dev'
+          - '--scenario="heavy-npm-deps build" --page=homepage --mode=build'
     uses: ./.github/workflows/build_reusable.yml
     with:
       afterBuild: pnpm install && ./node_modules/.bin/devlow-bench ./scripts/devlow-bench.mjs --datadog=ubuntu-latest-16-core ${{ matrix.mode }} ${{ matrix.selector }}

bench/heavy-npm-deps/package.json

@@ -5,6 +5,10 @@
   "scripts": {
     "dev-turbopack": "next dev --turbo",
     "dev-webpack": "next dev",
+    "build-turbopack": "TURBOPACK=1 TURBOPACK_BUILD=1 next build",
+    "build-webpack": "next build",
+    "start-turbopack": "TURBOPACK=1 next start",
+    "start-webpack": "next start",
     "build-application": "next build",
     "start-application": "next start"
   },

scripts/devlow-bench.mjs

@@ -25,6 +25,212 @@ const GIT_BRANCH =
     return cmd.output
   })())
 
+const nextBuildWorkflow =
+  (benchmarkName, pages) =>
+  async ({ turbopack, page }) => {
+    const pageConfig =
+      typeof pages[page] === 'string' ? { url: pages[page] } : pages[page]
+    const cleanupTasks = []
+    try {
+      const env = {
+        PATH: process.env.PATH,
+        NODE: process.env.NODE,
+        HOSTNAME: process.env.HOSTNAME,
+        PWD: process.env.PWD,
+        // Disable otel initialization to prevent pending / hanging request to otel collector
+        OTEL_SDK_DISABLED: 'true',
+        NEXT_PUBLIC_OTEL_SENTRY: 'true',
+        NEXT_PUBLIC_OTEL_DEV_DISABLED: 'true',
+        NEXT_TRACE_UPLOAD_DISABLED: 'true',
+        // Enable next.js test mode to get HMR events
+        __NEXT_TEST_MODE: '1',
+      }
+
+      const benchmarkDir = resolve(REPO_ROOT, 'bench', benchmarkName)
+
+      // cleanup .next directory to remove persistent cache
+      await retry(() =>
+        rm(join(benchmarkDir, '.next'), { recursive: true, force: true })
+      )
+
+      await measureTime('cleanup', {
+        scenario: benchmarkName,
+        props: { turbopack, page },
+      })
+
+      const buildArgs = [turbopack ? 'build-turbopack' : 'build-webpack']
+      let buildShell = command('pnpm', buildArgs, {
+        cwd: benchmarkDir,
+        env,
+      })
+      await buildShell.ok()
+
+      await measureTime('build', {
+        scenario: benchmarkName,
+        props: { turbopack, page },
+      })
+
+      // startup browser
+      let session = await newBrowserSession({})
+      const closeSession = async () => {
+        if (session) {
+          await session.close()
+          session = null
+        }
+      }
+      cleanupTasks.push(closeSession)
+      await measureTime('browser startup', {
+        props: { turbopack, page },
+      })
+
+      // run command to start dev server
+      const startArgs = [turbopack ? 'start-turbopack' : 'start-webpack']
+      let shell = command('pnpm', startArgs, {
+        cwd: benchmarkDir,
+        env,
+      })
+      const killShell = async () => {
+        if (shell) {
+          await shell.kill()
+          shell = null
+        }
+      }
+      cleanupTasks.push(killShell)
+
+      // wait for server to be ready
+      const START_SERVER_REGEXP = /Local:\s+(?<url>.+)\n/
+      const {
+        groups: { url },
+      } = await shell.waitForOutput(START_SERVER_REGEXP)
+      await measureTime('server startup', { props: { turbopack, page } })
+      await shell.reportMemUsage('mem usage after startup', {
+        props: { turbopack, page },
+      })
+
+      // open page
+      const pageInstance = await session.hardNavigation(
+        'open page',
+        url + pageConfig.url
+      )
+      await shell.reportMemUsage('mem usage after open page')
+
+      let status = 0
+      try {
+        if (
+          await pageInstance.evaluate(
+            '!next.appDir && __NEXT_DATA__.page === "/404"'
+          )
+        ) {
+          status = 2
+        }
+      } catch (e) {
+        status = 2
+      }
+
+      try {
+        if (
+          !(await pageInstance.evaluate(
+            'next.appDir || __NEXT_DATA__.page && !__NEXT_DATA__.err'
+          ))
+        ) {
+          status = 1
+        }
+      } catch (e) {
+        status = 1
+      }
+
+      await reportMeasurement('page status', status, 'status code')
+
+      // reload page
+      await session.reload('reload page')
+
+      await reportMeasurement(
+        'console output',
+        shell.output.split(/\n/).length,
+        'lines'
+      )
+
+      if (turbopack) {
+        // close dev server and browser
+        await killShell()
+        await closeSession()
+      } else {
+        // wait for persistent cache to be written
+        const waitPromise = new Promise((resolve) => {
+          setTimeout(resolve, 5000)
+        })
+        const cacheLocation = join(
+          benchmarkDir,
+          '.next',
+          'cache',
+          'webpack',
+          'client-development'
+        )
+        await Promise.race([
+          waitForFile(join(cacheLocation, 'index.pack')),
+          waitForFile(join(cacheLocation, 'index.pack.gz')),
+        ])
+        await measureTime('cache created')
+        await waitPromise
+        await measureTime('waiting')
+
+        // close dev server and browser
+        await killShell()
+        await closeSession()
+      }
+
+      buildShell = command('pnpm', buildArgs, {
+        cwd: benchmarkDir,
+        env,
+      })
+      await buildShell.ok()
+
+      await measureTime('build with cache', {
+        scenario: benchmarkName,
+        props: { turbopack, page },
+      })
+
+      // startup new browser
+      session = await newBrowserSession({})
+      await measureTime('browser startup', {
+        props: { turbopack, page },
+      })
+
+      // run command to start dev server
+      shell = command('pnpm', startArgs, {
+        cwd: benchmarkDir,
+        env,
+      })
+
+      // wait for server to be ready
+      const {
+        groups: { url: url2 },
+      } = await shell.waitForOutput(START_SERVER_REGEXP)
+      await shell.reportMemUsage('mem usage after startup with cache')
+
+      // open page
+      await session.hardNavigation(
+        'open page with cache',
+        url2 + pageConfig.url
+      )
+
+      await reportMeasurement(
+        'console output with cache',
+        shell.output.split(/\n/).length,
+        'lines'
+      )
+      await shell.reportMemUsage('mem usage after open page with cache')
+    } catch (e) {
+      console.log('CAUGHT', e)
+      throw e
+    } finally {
+      // This must run in order
+      // eslint-disable-next-line no-await-in-loop
+      for (const task of cleanupTasks.reverse()) await task()
+      await measureTime('shutdown')
+    }
+  }
+
 const nextDevWorkflow =
   (benchmarkName, pages) =>
   async ({ turbopack, page }) => {
@@ -41,7 +247,7 @@ const nextDevWorkflow =
 
       await measureTime('cleanup', {
         scenario: benchmarkName,
-        props: { turbopack: null, page: null },
+        props: { turbopack, page },
       })
 
       // startup browser
@@ -54,15 +260,14 @@ const nextDevWorkflow =
       }
       cleanupTasks.push(closeSession)
       await measureTime('browser startup', {
-        props: { turbopack: null, page: null },
+        props: { turbopack, page },
       })
 
       const env = {
         PATH: process.env.PATH,
         NODE: process.env.NODE,
         HOSTNAME: process.env.HOSTNAME,
         PWD: process.env.PWD,
-        NODE_ENV: 'development',
         // Disable otel initialization to prevent pending / hanging request to otel collector
         OTEL_SDK_DISABLED: 'true',
         NEXT_PUBLIC_OTEL_SENTRY: 'true',
@@ -91,9 +296,9 @@ const nextDevWorkflow =
       const {
         groups: { url },
       } = await shell.waitForOutput(START_SERVER_REGEXP)
-      await measureTime('server startup', { props: { page: null } })
+      await measureTime('server startup', { props: { turbopack, page } })
       await shell.reportMemUsage('mem usage after startup', {
-        props: { page: null },
+        props: { turbopack, page },
       })
 
       // open page
@@ -304,7 +509,7 @@ const nextDevWorkflow =
       // startup new browser
       session = await newBrowserSession({})
       await measureTime('browser startup', {
-        props: { turbopack: null, page: null },
+        props: { turbopack, page },
       })
 
       // run command to start dev server
@@ -361,6 +566,15 @@ describe(
   nextDevWorkflow('heavy-npm-deps', pages)
 )
 
+describe(
+  'heavy-npm-deps build test',
+  {
+    turbopack: true,
+    page: Object.keys(pages),
+  },
+  nextBuildWorkflow('heavy-npm-deps', pages)
+)
+
 async function retry(fn) {
   let lastError
   for (let i = 100; i < 2000; i += 100) {