Turbopack build: Add devlow-bench (#70511)

Ensures Webpack build and Turbopack build results for the heavy-npm-deps
benchmark are uploaded to DataDog so that we can track them over time.

<!-- Thanks for opening a PR! Your contribution is much appreciated.
To make sure your PR is handled as smoothly as possible we request that
you follow the checklist sections below.
Choose the right checklist for the change(s) that you're making:

## For Contributors

### Improving Documentation

- Run `pnpm prettier-fix` to fix formatting issues before opening the
PR.
- Read the Docs Contribution Guide to ensure your contribution follows
the docs guidelines:
https://nextjs.org/docs/community/contribution-guide

### Adding or Updating Examples

- The "examples guidelines" are followed from our contributing doc
https://github.com/vercel/next.js/blob/canary/contributing/examples/adding-examples.md
- Make sure the linting passes by running `pnpm build && pnpm lint`. See
https://github.com/vercel/next.js/blob/canary/contributing/repository/linting.md

### Fixing a bug

- Related issues linked using `fixes #number`
- Tests added. See:
https://github.com/vercel/next.js/blob/canary/contributing/core/testing.md#writing-tests-for-nextjs
- Errors have a helpful link attached, see
https://github.com/vercel/next.js/blob/canary/contributing.md

### Adding a feature

- Implements an existing feature request or RFC. Make sure the feature
request has been accepted for implementation before opening a PR. (A
discussion must be opened, see
https://github.com/vercel/next.js/discussions/new?category=ideas)
- Related issues/discussions are linked using `fixes #number`
- e2e tests added
(https://github.com/vercel/next.js/blob/canary/contributing/core/testing.md#writing-tests-for-nextjs)
- Documentation added
- Telemetry added. In case of a feature if it's used or not.
- Errors have a helpful link attached, see
https://github.com/vercel/next.js/blob/canary/contributing.md


## For Maintainers

- Minimal description (aim for explaining to someone not on the team to
understand the PR)
- When linking to a Slack thread, you might want to share details of the
conclusion
- Link both the Linear (Fixes NEXT-xxx) and the GitHub issues
- Add review comments if necessary to explain to the reviewer the logic
behind a change

### What?

### Why?

### How?

Closes NEXT-
Fixes #

-->

.github/workflows/build_and_test.yml

@@ -186,7 +186,8 @@ jobs:
           - '--turbopack=false'
           - '--turbopack=true'
         selector:
-          - '--scenario=heavy-npm-deps --page=homepage'
+          - '--scenario=heavy-npm-deps-dev --page=homepage'
+          - '--scenario=heavy-npm-deps-build --page=homepage'
     uses: ./.github/workflows/build_reusable.yml
     with:
       afterBuild: pnpm install && ./node_modules/.bin/devlow-bench ./scripts/devlow-bench.mjs --datadog=ubuntu-latest-16-core ${{ matrix.mode }} ${{ matrix.selector }}

bench/heavy-npm-deps/package.json

@@ -5,6 +5,10 @@
   "scripts": {
     "dev-turbopack": "next dev --turbo",
     "dev-webpack": "next dev",
+    "build-turbopack": "TURBOPACK=1 TURBOPACK_BUILD=1 next build",
+    "build-webpack": "next build",
+    "start-turbopack": "TURBOPACK=1 next start",
+    "start-webpack": "next start",
     "build-application": "next build",
     "start-application": "next start"
   },

scripts/devlow-bench.mjs

@@ -25,6 +25,212 @@ const GIT_BRANCH =
     return cmd.output
   })())
 
+const nextBuildWorkflow =
+  (benchmarkName, pages) =>
+  async ({ turbopack, page }) => {
+    const pageConfig =
+      typeof pages[page] === 'string' ? { url: pages[page] } : pages[page]
+    const cleanupTasks = []
+    try {
+      const env = {
+        PATH: process.env.PATH,
+        NODE: process.env.NODE,
+        HOSTNAME: process.env.HOSTNAME,
+        PWD: process.env.PWD,
+        // Disable otel initialization to prevent pending / hanging request to otel collector
+        OTEL_SDK_DISABLED: 'true',
+        NEXT_PUBLIC_OTEL_SENTRY: 'true',
+        NEXT_PUBLIC_OTEL_DEV_DISABLED: 'true',
+        NEXT_TRACE_UPLOAD_DISABLED: 'true',
+        // Enable next.js test mode to get HMR events
+        __NEXT_TEST_MODE: '1',
+      }
+
+      const benchmarkDir = resolve(REPO_ROOT, 'bench', benchmarkName)
+
+      // cleanup .next directory to remove persistent cache
+      await retry(() =>
+        rm(join(benchmarkDir, '.next'), { recursive: true, force: true })
+      )
+
+      await measureTime('cleanup', {
+        scenario: benchmarkName,
+        props: { turbopack, page },
+      })
+
+      const buildArgs = [turbopack ? 'build-turbopack' : 'build-webpack']
+      let buildShell = command('pnpm', buildArgs, {
+        cwd: benchmarkDir,
+        env,
+      })
+      await buildShell.ok()
+
+      await measureTime('build', {
+        scenario: benchmarkName,
+        props: { turbopack, page },
+      })
+
+      // startup browser
+      let session = await newBrowserSession({})
+      const closeSession = async () => {
+        if (session) {
+          await session.close()
+          session = null
+        }
+      }
+      cleanupTasks.push(closeSession)
+      await measureTime('browser startup', {
+        props: { turbopack, page },
+      })
+
+      // run command to start dev server
+      const startArgs = [turbopack ? 'start-turbopack' : 'start-webpack']
+      let shell = command('pnpm', startArgs, {
+        cwd: benchmarkDir,
+        env,
+      })
+      const killShell = async () => {
+        if (shell) {
+          await shell.kill()
+          shell = null
+        }
+      }
+      cleanupTasks.push(killShell)
+
+      // wait for server to be ready
+      const START_SERVER_REGEXP = /Local:\s+(?<url>.+)\n/
+      const {
+        groups: { url },
+      } = await shell.waitForOutput(START_SERVER_REGEXP)
+      await measureTime('server startup', { props: { turbopack, page } })
+      await shell.reportMemUsage('mem usage after startup', {
+        props: { turbopack, page },
+      })
+
+      // open page
+      const pageInstance = await session.hardNavigation(
+        'open page',
+        url + pageConfig.url
+      )
+      await shell.reportMemUsage('mem usage after open page')
+
+      let status = 0
+      try {
+        if (
+          await pageInstance.evaluate(
+            '!next.appDir && __NEXT_DATA__.page === "/404"'
+          )
+        ) {
+          status = 2
+        }
+      } catch (e) {
+        status = 2
+      }
+
+      try {
+        if (
+          !(await pageInstance.evaluate(
+            'next.appDir || __NEXT_DATA__.page && !__NEXT_DATA__.err'
+          ))
+        ) {
+          status = 1
+        }
+      } catch (e) {
+        status = 1
+      }
+
+      await reportMeasurement('page status', status, 'status code')
+
+      // reload page
+      await session.reload('reload page')
+
+      await reportMeasurement(
+        'console output',
+        shell.output.split(/\n/).length,
+        'lines'
+      )
+
+      if (turbopack) {
+        // close dev server and browser
+        await killShell()
+        await closeSession()
+      } else {
+        // wait for persistent cache to be written
+        const waitPromise = new Promise((resolve) => {
+          setTimeout(resolve, 5000)
+        })
+        const cacheLocation = join(
+          benchmarkDir,
+          '.next',
+          'cache',
+          'webpack',
+          'client-development'
+        )
+        await Promise.race([
+          waitForFile(join(cacheLocation, 'index.pack')),
+          waitForFile(join(cacheLocation, 'index.pack.gz')),
+        ])
+        await measureTime('cache created')
+        await waitPromise
+        await measureTime('waiting')
+
+        // close dev server and browser
+        await killShell()
+        await closeSession()
+      }
+
+      buildShell = command('pnpm', buildArgs, {
+        cwd: benchmarkDir,
+        env,
+      })
+      await buildShell.ok()
+
+      await measureTime('build with cache', {
+        scenario: benchmarkName,
+        props: { turbopack, page },
+      })
+
+      // startup new browser
+      session = await newBrowserSession({})
+      await measureTime('browser startup', {
+        props: { turbopack, page },
+      })
+
+      // run command to start dev server
+      shell = command('pnpm', startArgs, {
+        cwd: benchmarkDir,
+        env,
+      })
+
+      // wait for server to be ready
+      const {
+        groups: { url: url2 },
+      } = await shell.waitForOutput(START_SERVER_REGEXP)
+      await shell.reportMemUsage('mem usage after startup with cache')
+
+      // open page
+      await session.hardNavigation(
+        'open page with cache',
+        url2 + pageConfig.url
+      )
+
+      await reportMeasurement(
+        'console output with cache',
+        shell.output.split(/\n/).length,
+        'lines'
+      )
+      await shell.reportMemUsage('mem usage after open page with cache')
+    } catch (e) {
+      console.log('CAUGHT', e)
+      throw e
+    } finally {
+      // This must run in order
+      // eslint-disable-next-line no-await-in-loop
+      for (const task of cleanupTasks.reverse()) await task()
+      await measureTime('shutdown')
+    }
+  }
+
 const nextDevWorkflow =
   (benchmarkName, pages) =>
   async ({ turbopack, page }) => {
@@ -41,7 +247,7 @@ const nextDevWorkflow =
 
       await measureTime('cleanup', {
         scenario: benchmarkName,
-        props: { turbopack: null, page: null },
+        props: { turbopack, page },
       })
 
       // startup browser
@@ -54,15 +260,14 @@ const nextDevWorkflow =
       }
       cleanupTasks.push(closeSession)
       await measureTime('browser startup', {
-        props: { turbopack: null, page: null },
+        props: { turbopack, page },
       })
 
       const env = {
         PATH: process.env.PATH,
         NODE: process.env.NODE,
         HOSTNAME: process.env.HOSTNAME,
         PWD: process.env.PWD,
-        NODE_ENV: 'development',
         // Disable otel initialization to prevent pending / hanging request to otel collector
         OTEL_SDK_DISABLED: 'true',
         NEXT_PUBLIC_OTEL_SENTRY: 'true',
@@ -91,9 +296,9 @@ const nextDevWorkflow =
       const {
         groups: { url },
       } = await shell.waitForOutput(START_SERVER_REGEXP)
-      await measureTime('server startup', { props: { page: null } })
+      await measureTime('server startup', { props: { turbopack, page } })
       await shell.reportMemUsage('mem usage after startup', {
-        props: { page: null },
+        props: { turbopack, page },
       })
 
       // open page
@@ -304,7 +509,7 @@ const nextDevWorkflow =
       // startup new browser
       session = await newBrowserSession({})
       await measureTime('browser startup', {
-        props: { turbopack: null, page: null },
+        props: { turbopack, page },
       })
 
       // run command to start dev server
@@ -353,14 +558,25 @@ const pages = {
 }
 
 describe(
-  'heavy-npm-deps dev test',
+  'heavy-npm-deps-dev',
   {
     turbopack: true,
+    mode: 'dev',
     page: Object.keys(pages),
   },
   nextDevWorkflow('heavy-npm-deps', pages)
 )
 
+describe(
+  'heavy-npm-deps-build',
+  {
+    turbopack: true,
+    mode: 'build',
+    page: Object.keys(pages),
+  },
+  nextBuildWorkflow('heavy-npm-deps', pages)
+)
+
 async function retry(fn) {
   let lastError
   for (let i = 100; i < 2000; i += 100) {