Flatten recursive wildcard exports in barrel optimization (#56489)

This PR flattens the recursive optimization logic of our barrel optimization loader. So now if there're any recursive `export * from ...`, they won't be created as multiple individual Webpack modules, but optimized in one module.

With this change, we are running SWC transform to get the export map directly inside the barrel loader (instead of a separate loader rule). And that map is recursively calculated and cached in memory.

I also published https://unpkg.com/browse/recursive-barrel@1.0.0/ to give this a test. It contains 4 levels of 10 `export *` expressions, which is 10,000 modules in total. Without this change, it takes ~30s to compile and with this change, it goes down to ~7s.

packages/next/src/build/swc/options.ts

@@ -321,7 +321,6 @@ export function getLoaderSWCOptions({
   hasServerComponents,
   isServerLayer,
   isServerActionsEnabled,
-  optimizeBarrelExports,
   bundleTarget,
 }: // This is not passed yet as "paths" resolving is handled by webpack currently.
 // resolvedBaseUrl,
@@ -348,9 +347,6 @@ export function getLoaderSWCOptions({
   hasServerComponents?: boolean
   isServerLayer: boolean
   isServerActionsEnabled?: boolean
-  optimizeBarrelExports?: {
-    wildcard: boolean
-  }
 }) {
   let baseOptions: any = getBaseSWCOptions({
     filename,
@@ -405,9 +401,6 @@ export function getLoaderSWCOptions({
       packages: optimizePackageImports,
     }
   }
-  if (optimizeBarrelExports) {
-    baseOptions.optimizeBarrelExports = optimizeBarrelExports
-  }
 
   const isNextDist = nextDistPath.test(filename)
 

packages/next/src/build/webpack-config.ts

@@ -1412,27 +1412,6 @@ export default async function getBaseWebpackConfig(
     },
     module: {
       rules: [
-        {
-          // This loader rule passes the resource to the SWC loader with
-          // `optimizeBarrelExports` enabled. This option makes the SWC to
-          // transform the original code to be a JSON of its export map, so
-          // the barrel loader can analyze it and only keep the needed ones.
-          test: /__barrel_transform__/,
-          use: ({ resourceQuery }: { resourceQuery: string }) => {
-            const isFromWildcardExport = /[&?]wildcard/.test(resourceQuery)
-
-            return [
-              getSwcLoader({
-                isServerLayer: false,
-                bundleTarget: 'client',
-                hasServerComponents: false,
-                optimizeBarrelExports: {
-                  wildcard: isFromWildcardExport,
-                },
-              }),
-            ]
-          },
-        },
         {
           // This loader rule works like a bridge between user's import and
           // the target module behind a package's barrel file. It reads SWC's
@@ -1443,14 +1422,18 @@ export default async function getBaseWebpackConfig(
             const names = (
               resourceQuery.match(/\?names=([^&]+)/)?.[1] || ''
             ).split(',')
-            const isFromWildcardExport = /[&?]wildcard/.test(resourceQuery)
 
             return [
               {
                 loader: 'next-barrel-loader',
                 options: {
                   names,
-                  wildcard: isFromWildcardExport,
+                  swcCacheDir: path.join(
+                    dir,
+                    config?.distDir ?? '.next',
+                    'cache',
+                    'swc'
+                  ),
                 },
                 // This is part of the request value to serve as the module key.
                 // The barrel loader are no-op re-exported modules keyed by

packages/next/src/build/webpack/loaders/next-barrel-loader.ts

@@ -86,35 +86,177 @@
 
 import type webpack from 'webpack'
 
+import path from 'path'
+import { transform } from '../../swc'
+
+// This is a in-memory cache for the mapping of barrel exports. This only applies
+// to the packages that we optimize. It will never change (e.g. upgrading packages)
+// during the lifetime of the server so we can safely cache it.
+// There is also no need to collect the cache for the same reason.
+const barrelTransformMappingCache = new Map<
+  string,
+  {
+    prefix: string
+    exportList: [string, string, string][]
+    wildcardExports: string[]
+  } | null
+>()
+
+async function getBarrelMapping(
+  resourcePath: string,
+  swcCacheDir: string,
+  resolve: (context: string, request: string) => Promise<string>,
+  fs: {
+    readFile: (
+      path: string,
+      callback: (err: any, data: string | Buffer | undefined) => void
+    ) => void
+  }
+) {
+  if (barrelTransformMappingCache.has(resourcePath)) {
+    return barrelTransformMappingCache.get(resourcePath)!
+  }
+
+  // This is a SWC transform specifically for `optimizeBarrelExports`. We don't
+  // care about other things but the export map only.
+  async function transpileSource(
+    filename: string,
+    source: string,
+    isWildcard: boolean
+  ) {
+    const isTypeScript = filename.endsWith('.ts') || filename.endsWith('.tsx')
+    return new Promise<string>((res) =>
+      transform(source, {
+        filename,
+        inputSourceMap: undefined,
+        sourceFileName: filename,
+        optimizeBarrelExports: {
+          wildcard: isWildcard,
+        },
+        jsc: {
+          parser: {
+            syntax: isTypeScript ? 'typescript' : 'ecmascript',
+            [isTypeScript ? 'tsx' : 'jsx']: true,
+          },
+          experimental: {
+            cacheRoot: swcCacheDir,
+          },
+        },
+      }).then((output) => {
+        res(output.code)
+      })
+    )
+  }
+
+  // Avoid circular `export *` dependencies
+  const visited = new Set<string>()
+  async function getMatches(file: string, isWildcard: boolean) {
+    if (visited.has(file)) {
+      return null
+    }
+    visited.add(file)
+
+    const source = await new Promise<string>((res, rej) => {
+      fs.readFile(file, (err, data) => {
+        if (err || data === undefined) {
+          rej(err)
+        } else {
+          res(data.toString())
+        }
+      })
+    })
+
+    const output = await transpileSource(file, source, isWildcard)
+
+    const matches = output.match(
+      /^([^]*)export (const|var) __next_private_export_map__ = ('[^']+'|"[^"]+")/
+    )
+    if (!matches) {
+      return null
+    }
+
+    const prefix = matches[1]
+    let exportList = JSON.parse(matches[3].slice(1, -1)) as [
+      string,
+      string,
+      string
+    ][]
+    const wildcardExports = [
+      ...output.matchAll(/export \* from "([^"]+)"/g),
+    ].map((match) => match[1])
+
+    // In the wildcard case, if the value is exported from another file, we
+    // redirect to that file (decl[0]). Otherwise, export from the current
+    // file itself.
+    if (isWildcard) {
+      for (const decl of exportList) {
+        decl[1] = file
+        decl[2] = decl[0]
+      }
+    }
+
+    // This recursively handles the wildcard exports (e.g. `export * from './a'`)
+    if (wildcardExports.length) {
+      await Promise.all(
+        wildcardExports.map(async (req) => {
+          const targetPath = await resolve(
+            path.dirname(file),
+            req.replace('__barrel_optimize__?names=__PLACEHOLDER__!=!', '')
+          )
+
+          const targetMatches = await getMatches(targetPath, true)
+          if (targetMatches) {
+            // Merge the export list
+            exportList = exportList.concat(targetMatches.exportList)
+          }
+        })
+      )
+    }
+
+    return {
+      prefix,
+      exportList,
+      wildcardExports,
+    }
+  }
+
+  const res = await getMatches(resourcePath, false)
+  barrelTransformMappingCache.set(resourcePath, res)
+
+  return res
+}
+
 const NextBarrelLoader = async function (
   this: webpack.LoaderContext<{
     names: string[]
-    wildcard: boolean
+    swcCacheDir: string
   }>
 ) {
   this.async()
-  const { names, wildcard } = this.getOptions()
-
-  const source = await new Promise<string>((resolve, reject) => {
-    this.loadModule(
-      `__barrel_transform__${wildcard ? '?wildcard' : ''}!=!${
-        this.resourcePath
-      }`,
-      (err, src) => {
-        if (err) {
-          reject(err)
-        } else {
-          resolve(src)
-        }
-      }
-    )
+  this.cacheable(true)
+
+  const { names, swcCacheDir } = this.getOptions()
+
+  // For barrel optimizations, we always prefer the "module" field over the
+  // "main" field because ESM handling is more robust with better tree-shaking.
+  const resolve = this.getResolve({
+    mainFields: ['module', 'main'],
   })
 
-  const matches = source.match(
-    /^([^]*)export const __next_private_export_map__ = ('[^']+'|"[^"]+")/
+  const mapping = await getBarrelMapping(
+    this.resourcePath,
+    swcCacheDir,
+    resolve,
+    this.fs
   )
 
-  if (!matches) {
+  // `resolve` adds all sub-paths to the dependency graph. However, we already
+  // cached the mapping and we assume them to not change. So, we can safely
+  // clear the dependencies here to avoid unnecessary watchers which turned out
+  // to be very expensive.
+  this.clearDependencies()
+
+  if (!mapping) {
     // This file isn't a barrel and we can't apply any optimizations. Let's re-export everything.
     // Since this loader accepts `names` and the request is keyed with `names`, we can't simply
     // return the original source here. That will create these imports with different names as
@@ -123,19 +265,12 @@ const NextBarrelLoader = async function (
     return
   }
 
-  const wildcardExports = [...source.matchAll(/export \* from "([^"]+)"/g)]
-
   // It needs to keep the prefix for comments and directives like "use client".
-  const prefix = matches[1]
-
-  const exportList = JSON.parse(matches[2].slice(1, -1)) as [
-    string,
-    string,
-    string
-  ][]
+  const prefix = mapping.prefix
+  const exportList = mapping.exportList
   const exportMap = new Map<string, [string, string]>()
-  for (const [name, path, orig] of exportList) {
-    exportMap.set(name, [path, orig])
+  for (const [name, filePath, orig] of exportList) {
+    exportMap.set(name, [filePath, orig])
   }
 
   let output = prefix
@@ -145,15 +280,6 @@ const NextBarrelLoader = async function (
     if (exportMap.has(name)) {
       const decl = exportMap.get(name)!
 
-      // In the wildcard case, if the value is exported from another file, we
-      // redirect to that file (decl[0]). Otherwise, export from the current
-      // file itself (this.resourcePath).
-      if (wildcard && !decl[0]) {
-        // E.g. the file contains `export const a = 1`
-        decl[0] = this.resourcePath
-        decl[1] = name
-      }
-
       if (decl[1] === '*') {
         output += `\nexport * as ${name} from ${JSON.stringify(decl[0])}`
       } else if (decl[1] === 'default') {
@@ -174,11 +300,9 @@ const NextBarrelLoader = async function (
 
   // These are from wildcard exports.
   if (missedNames.length > 0) {
-    for (const match of wildcardExports) {
-      const path = match[1]
-
+    for (const req of mapping.wildcardExports) {
       output += `\nexport * from ${JSON.stringify(
-        path.replace('__PLACEHOLDER__', missedNames.join(',') + '&wildcard')
+        req.replace('__PLACEHOLDER__', missedNames.join(',') + '&wildcard')
       )}`
     }
   }

packages/next/src/build/webpack/loaders/next-swc-loader.ts

@@ -45,9 +45,6 @@ export interface SWCLoaderOptions {
   bundleTarget: BundleType
   hasServerComponents?: boolean
   isServerLayer: boolean
-  optimizeBarrelExports?: {
-    wildcard: boolean
-  }
 }
 
 async function loaderTransform(
@@ -73,19 +70,11 @@ async function loaderTransform(
     swcCacheDir,
     hasServerComponents,
     isServerLayer,
-    optimizeBarrelExports,
     bundleTarget,
   } = loaderOptions
   const isPageFile = filename.startsWith(pagesDir)
   const relativeFilePathFromRoot = path.relative(rootDir, filename)
 
-  // For testing purposes
-  if (process.env.NEXT_TEST_MODE) {
-    if (loaderOptions.optimizeBarrelExports) {
-      console.log('optimizeBarrelExports:', filename)
-    }
-  }
-
   const swcOptions = getLoaderSWCOptions({
     pagesDir,
     appDir,
@@ -106,7 +95,6 @@ async function loaderTransform(
     hasServerComponents,
     isServerActionsEnabled: nextConfig?.experimental?.serverActions,
     isServerLayer,
-    optimizeBarrelExports,
     bundleTarget,
   })