fix: compatability for next@16.1.0-canary.19 (#3300)

Co-authored-by: Mateusz Bocian <mrstork@users.noreply.github.com>

Author: pieh

.github/workflows/run-tests.yml

@@ -16,6 +16,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
+      os: ${{ steps.set-matrix.outputs.os }}
     steps:
       - name: Check PR labels
         if: github.event_name == 'pull_request'
@@ -28,25 +29,42 @@ jobs:
               repo: context.repo.repo,
               issue_number: context.payload.pull_request.number,
             });
-            const shouldTestAllVersions = labels.some(label => label.name === 'autorelease: pending' || label.name === 'test all versions');
-            if (shouldTestAllVersions) {
-              return 'all'
+
+            const runOnWindows = labels.some(label => label.name === 'test on windows');
+
+            let versionsToTest = 'latest';
+            if (labels.some(label => label.name === 'autorelease: pending' || label.name === 'test all versions')) {
+              versionsToTest = 'all';
+            } else if (labels.some(label => label.name === 'test latest and canary')) {
+              versionsToTest = 'latest-and-canary';
             }
 
-            return labels.some(label => label.name === 'test latest and canary') ? 'latest-and-canary' : 'latest'
+            return {
+              versionsToTest,
+              runOnWindows
+            }
       - name: Set Next.js versions to test
         id: set-matrix
-        # If this is the nightly build or a release PR then run the full matrix of versions
-        run: |
-          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
-            echo "matrix=${{ github.event.inputs.versions }}" >> $GITHUB_OUTPUT
-          elif [ "${{ github.event_name }}" = "schedule" ] || [ "${{ steps.check-labels.outputs.result }}" = "all" ]; then
-            echo "matrix=[\"latest\", \"canary\", \"14.2.15\", \"13.5.1\"]" >> $GITHUB_OUTPUT
-          elif [ "${{ steps.check-labels.outputs.result }}" = "latest-and-canary" ]; then
-            echo "matrix=[\"latest\", \"canary\"]" >> $GITHUB_OUTPUT
-          else
-            echo "matrix=[\"latest\"]" >> $GITHUB_OUTPUT
-          fi
+        uses: actions/github-script@v8
+        with:
+          script: |
+            const { versionsToTest, runOnWindows } = ${{ steps.check-labels.outputs.result }} ?? {}
+
+            if ('${{ github.event_name }}' === 'workflow_dispatch') {
+              core.setOutput('matrix', '${{ github.event.inputs.versions }}');
+            } else if ('${{ github.event_name }}' === 'schedule' || versionsToTest === 'all') {
+              core.setOutput('matrix', '["latest", "canary", "15.5.9", "14.2.35", "13.5.1"]');
+            } else if (versionsToTest === 'latest-and-canary') {
+              core.setOutput('matrix', '["latest", "canary"]');
+            } else {
+              core.setOutput('matrix', '["latest"]');
+            }
+
+            if (runOnWindows) {
+              core.setOutput('os', '["ubuntu-latest", "windows-2025"]');
+            } else {
+              core.setOutput('os', '["ubuntu-latest"]');
+            }
 
   e2e:
     needs: setup
@@ -122,7 +140,7 @@ jobs:
           NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
           NEXT_VERSION: ${{ matrix.version }}
           NEXT_RESOLVED_VERSION: ${{ steps.resolve-next-version.outputs.version }}
-          NODE_OPTIONS: --import ${{ github.workspace }}/tools/fetch-retry.mjs
+          NODE_OPTIONS: --import file://${{ github.workspace }}/tools/fetch-retry.mjs
       - name: Upload blob report to GitHub Actions Artifacts
         uses: actions/upload-artifact@v5
         if: always()
@@ -137,13 +155,16 @@ jobs:
       fail-fast: false
       matrix:
         shard: [1, 2, 3, 4, 5, 6, 7, 8]
-        os: [ubuntu-latest]
+        os: ${{ fromJson(needs.setup.outputs.os) }}
         version: ${{ fromJson(needs.setup.outputs.matrix) }}
         exclude:
           - os: windows-2025
             version: "13.5.1"
           - os: windows-2025
-            version: "14.2.15"
+            version: "14.2.35"
+          - os: windows-2025
+            version: "15.5.9"
+
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v6
@@ -171,17 +192,8 @@ jobs:
           node-version: ${{ steps.decide-node-version.outputs.version }}
           cache: "npm"
           cache-dependency-path: "**/package-lock.json"
-      - name: Prefer npm global on windows
-        if: runner.os == 'Windows'
-        # On Windows by default PATH prefers corepack bundled with Node.js
-        # This prepends npm global to PATH to ensure that npm installed global corepack is used instead
-        run: |
-          echo "$(npm config get prefix)" >> "$GITHUB_PATH"
-        shell: bash
       - name: setup pnpm/yarn
         run: |
-          # global corepack installation requires --force on Windows, otherwise EEXIST errors occur
-          npm install -g corepack --force
           corepack enable
         shell: bash
       - name: Install Deno
@@ -217,15 +229,15 @@ jobs:
         env:
           NEXT_VERSION: ${{ matrix.version }}
           NEXT_RESOLVED_VERSION: ${{ steps.resolve-next-version.outputs.version }}
-          NODE_OPTIONS: --import ${{ github.workspace }}/tools/fetch-retry.mjs
+          NODE_OPTIONS: --import file://${{ github.workspace }}/tools/fetch-retry.mjs
       - name: "Unit and integration tests"
         run: npm run test:ci:unit-and-integration -- --shard=${{ matrix.shard }}/8
         env:
           NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
           NEXT_VERSION: ${{ matrix.version }}
           NEXT_RESOLVED_VERSION: ${{ steps.resolve-next-version.outputs.version }}
           TEMP: ${{ github.workspace }}/..
-          NODE_OPTIONS: --import ${{ github.workspace }}/tools/fetch-retry.mjs
+          NODE_OPTIONS: --import file://${{ github.workspace }}/tools/fetch-retry.mjs
 
   smoke:
     if: always()
@@ -286,7 +298,7 @@ jobs:
           NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
           NEXT_VERSION: ${{ matrix.version }}
           NEXT_RESOLVED_VERSION: ${{ steps.resolve-next-version.outputs.version }}
-          NODE_OPTIONS: --import ${{ github.workspace }}/tools/fetch-retry.mjs
+          NODE_OPTIONS: --import file://${{ github.workspace }}/tools/fetch-retry.mjs
 
   merge-reports:
     if: always()

src/build/content/server.ts

@@ -10,8 +10,8 @@ import {
   writeFile,
 } from 'node:fs/promises'
 import { createRequire } from 'node:module'
-import { dirname, join, relative, sep } from 'node:path'
-import { join as posixJoin, sep as posixSep } from 'node:path/posix'
+import { dirname, join, sep } from 'node:path'
+import { join as posixJoin, relative as posixRelative, sep as posixSep } from 'node:path/posix'
 
 import { trace } from '@opentelemetry/api'
 import { wrapTracer } from '@opentelemetry/api/experimental'
@@ -26,7 +26,11 @@ import type { PluginContext, RequiredServerFilesManifest } from '../plugin-conte
 
 const tracer = wrapTracer(trace.getTracer('Next runtime'))
 
-const toPosixPath = (path: string) => path.split(sep).join(posixSep)
+const toPosixPath = (path: string) =>
+  path
+    .replace(/^\\+\?\\+/, '') // https://github.com/nodejs/node/blob/81e05e124f71b3050cd4e60c95017af975568413/lib/internal/fs/utils.js#L370-L372
+    .split(sep)
+    .join(posixSep)
 
 function isError(error: unknown): error is NodeJS.ErrnoException {
   return error instanceof Error
@@ -309,17 +313,21 @@ export const copyNextDependencies = async (ctx: PluginContext): Promise<void> =>
   await tracer.withActiveSpan('copyNextDependencies', async () => {
     const promises: Promise<void>[] = []
 
-    const nodeModulesLocationsInStandalone = new Set<string>()
+    const nodeModulesLocations = new Set<{ source: string; destination: string }>()
     const commonFilter = ctx.constants.IS_LOCAL ? undefined : nodeModulesFilter
 
-    const dotNextDir = join(ctx.standaloneDir, ctx.nextDistDir)
+    const dotNextDir = toPosixPath(join(ctx.standaloneDir, ctx.nextDistDir))
+
+    const standaloneRootDir = toPosixPath(ctx.standaloneRootDir)
+    const outputFileTracingRoot = toPosixPath(ctx.outputFileTracingRoot)
 
     await cp(ctx.standaloneRootDir, ctx.serverHandlerRootDir, {
       recursive: true,
       verbatimSymlinks: true,
       force: true,
-      filter: async (sourcePath: string) => {
-        if (sourcePath === dotNextDir) {
+      filter: async (sourcePath: string, destination: string) => {
+        const posixSourcePath = toPosixPath(sourcePath)
+        if (posixSourcePath === dotNextDir) {
           // copy all except the distDir (.next) folder as this is handled in a separate function
           // this will include the node_modules folder as well
           return false
@@ -328,18 +336,23 @@ export const copyNextDependencies = async (ctx: PluginContext): Promise<void> =>
         if (sourcePath.endsWith('node_modules')) {
           // keep track of node_modules as we might need to recreate symlinks
           // we are still copying them
-          nodeModulesLocationsInStandalone.add(sourcePath)
+          nodeModulesLocations.add({
+            source: posixSourcePath,
+            destination: toPosixPath(destination),
+          })
         }
 
         // finally apply common filter if defined
         return commonFilter?.(sourcePath) ?? true
       },
     })
 
-    for (const nodeModulesLocationInStandalone of nodeModulesLocationsInStandalone) {
-      const relativeToRoot = relative(ctx.standaloneRootDir, nodeModulesLocationInStandalone)
-      const locationInProject = join(ctx.outputFileTracingRoot, relativeToRoot)
-      const locationInServerHandler = join(ctx.serverHandlerRootDir, relativeToRoot)
+    for (const {
+      source: nodeModulesLocationInStandalone,
+      destination: locationInServerHandler,
+    } of nodeModulesLocations) {
+      const relativeToRoot = posixRelative(standaloneRootDir, nodeModulesLocationInStandalone)
+      const locationInProject = posixJoin(outputFileTracingRoot, relativeToRoot)
 
       promises.push(recreateNodeModuleSymlinks(locationInProject, locationInServerHandler))
     }

src/build/plugin-context.ts

@@ -1,7 +1,7 @@
 import { existsSync, readFileSync } from 'node:fs'
 import { readFile } from 'node:fs/promises'
 import { createRequire } from 'node:module'
-import { join, relative, resolve } from 'node:path'
+import { join, relative, resolve, sep } from 'node:path'
 import { join as posixJoin, relative as posixRelative } from 'node:path/posix'
 import { fileURLToPath } from 'node:url'
 
@@ -88,11 +88,39 @@ export class PluginContext {
    * The root directory for output file tracing. Paths inside standalone directory preserve paths of project, relative to this directory.
    */
   get outputFileTracingRoot(): string {
-    return (
+    // Up until https://github.com/vercel/next.js/pull/86812 we had direct access to computed value of it with following
+    const outputFileTracingRootFromRequiredServerFiles =
       this.requiredServerFiles.config.outputFileTracingRoot ??
       // fallback for older Next.js versions that don't have outputFileTracingRoot in the config, but had it in config.experimental
       this.requiredServerFiles.config.experimental.outputFileTracingRoot
-    )
+    if (outputFileTracingRootFromRequiredServerFiles) {
+      return outputFileTracingRootFromRequiredServerFiles
+    }
+
+    if (!this.relativeAppDir.includes('..')) {
+      // For newer Next.js versions outputFileTracingRoot is not written to the output directly anymore, but we can use appDir and relativeAppDir to compute it.
+      // This assumes that relative app dir will never contain '..' segments. Some monorepos support workspaces outside of the monorepo root (verified with pnpm)
+      // However Next.js itself have some limits on it:
+      //  - turbopack by default would throw "Module not found: Can't resolve '<name_of_package_outside_of_root>'"
+      //    forcing user to manually set `outputFileTracingRoot` in next.config which will impact `appDir` and `relativeAppDir` preserving the lack of '..' in `relativeAppDir`
+      //  - webpack case depends on wether dependency is marked as external or not:
+      //    - if it's marked as external then standalone while working locally, it would never work when someone tries to deploy it (and not just on Netlify, but also in fully self-hosted scenarios)
+      //      because parts of application would be outside of "standalone" directory
+      //    - if it's not marked as external it will be included in next.js produced chunks
+
+      const depth = this.relativeAppDir === '' ? 0 : this.relativeAppDir.split(sep).length
+
+      const computedOutputFileTracingRoot = resolve(
+        this.requiredServerFiles.appDir,
+        ...Array.from<string>({ length: depth }).fill('..'),
+      )
+      return computedOutputFileTracingRoot
+    }
+
+    // if relativeAppDir contains '..', we can't actually figure out the outputFileTracingRoot
+    // so best fallback is to just cwd() which won't work in wild edge cases, but there is no way of getting anything better
+    // if it's not correct it will cause build failures later when assembling a server handler function
+    return process.cwd()
   }
 
   /**

tests/e2e/middleware.test.ts

@@ -660,5 +660,5 @@ test("requests with x-middleware-subrequest don't skip middleware (GHSA-f82v-jwr
   expect(response.headers.get('x-test-used-middleware')).toBe('true')
 
   // ensure we are testing version before the fix for self hosted
-  expect(response.headers.get('x-test-used-next-version')).toBe('15.1.9')
+  expect(response.headers.get('x-test-used-next-version')).toBe('15.1.11')
 })

tests/fixtures/after/package.json

@@ -14,7 +14,7 @@
   },
   "test": {
     "dependencies": {
-      "next": ">=15.1.9"
+      "next": "15.1.11"
     }
   }
 }

tests/fixtures/middleware-subrequest-vuln/package.json

@@ -8,13 +8,13 @@
     "build": "next build"
   },
   "dependencies": {
-    "next": "15.1.9",
+    "next": "15.1.11",
     "react": "18.2.0",
     "react-dom": "18.2.0"
   },
   "test": {
     "dependencies": {
-      "next": "15.1.9"
+      "next": "15.1.11"
     }
   }
 }

tests/utils/fixture.ts

@@ -106,6 +106,10 @@ export const createFixture = async (fixture: string, ctx: FixtureTestContext) =>
     delete globalThis[Symbol.for('next-patch')]
   }
 
+  // due to changes in https://github.com/vercel/next.js/pull/86591 , this global is specific to instance of application and we to clean it up
+  // from any previous function invocations that might have run in the same process
+  delete globalThis[Symbol.for('next.server.manifests')]
+
   ctx.cwd = await mkdtemp(join(tmpdir(), 'opennextjs-netlify-'))
   vi.spyOn(process, 'cwd').mockReturnValue(ctx.cwd)
 

tests/utils/next-version-helpers.mjs

@@ -3,7 +3,7 @@
 import { readFile, writeFile } from 'node:fs/promises'
 
 import fg from 'fast-glob'
-import { coerce, gt, gte, satisfies, valid } from 'semver'
+import { coerce, gt, gte, parse as parseSemver, satisfies, valid } from 'semver'
 import { execaCommand } from 'execa'
 
 const FUTURE_NEXT_PATCH_VERSION = '16.999.0'
@@ -193,7 +193,7 @@ export async function setNextVersionInFixture(
         const nextPeerDependencies = JSON.parse(stdout)
 
         if (updateReact && nextVersionRequiresReact19(checkVersion)) {
-          // canaries started reporting peerDependencies as `^18.2.0 || 19.0.0-rc-<hash>-<date>`
+          // canaries started reporting peerDependencies as `^18.2.0 || 19.0.0-rc-<hash>-<date> || ^19.0.0`
           // with https://github.com/vercel/next.js/pull/70219 which is valid range for package managers
           // but not for @nx/next which checks dependencies and tries to assure that at least React 18 is used
           // but the check doesn't handle the alternative in version selector which thinks it's not valid:
@@ -205,14 +205,22 @@ export async function setNextVersionInFixture(
             .split('||')
             .map((alternative) => {
               const selector = alternative.trim()
-              const coerced = coerce(selector)?.format()
+              // we need to pick the highest version from alternatives and to handle
+              // comparison of both range selectors (^) and pinned prerelease version (-rc-<hash>-<date>)
+              // we need to use couple of tricks:
+              // 1. we do try to parse semver - this only works for pinned versions and will handle prereleases, it will return null for ranges
+              // 2. if parsing returns null, we coerce
+              // this will allow us to preserve prerelease identifiers for comparisons (as coercing prerelease version strip those)
+
+              const versionToCompare = (parseSemver(selector) ?? coerce(selector))?.format()
+
               return {
                 selector,
-                coerced,
+                versionToCompare,
               }
             })
             .sort((a, b) => {
-              return gt(a.coerced, b.coerced) ? -1 : 1
+              return gt(a.versionToCompare, b.versionToCompare) ? -1 : 1
             })[0].selector
 
           const reactVersion =