From 9b671046b9b3779eede47473f76229869df2cee7 Mon Sep 17 00:00:00 2001 From: Shu Ding Date: Fri, 6 Oct 2023 17:39:15 +0200 Subject: [PATCH] Flatten recursive wildcard exports in barrel optimization (#56489) This PR flattens the recursive optimization logic of our barrel optimization loader. So now if there're any recursive `export * from ...`, they won't be created as multiple individual Webpack modules, but optimized in one module. With this change, we are running SWC transform to get the export map directly inside the barrel loader (instead of a separate loader rule). And that map is recursively calculated and cached in memory. I also published https://unpkg.com/browse/recursive-barrel@1.0.0/ to give this a test. It contains 4 levels of 10 `export *` expressions, which is 10,000 modules in total. Without this change, it takes ~30s to compile and with this change, it goes down to ~7s. --- packages/next/src/build/swc/options.ts | 7 - packages/next/src/build/webpack-config.ts | 29 +-- .../webpack/loaders/next-barrel-loader.ts | 210 ++++++++++++++---- .../build/webpack/loaders/next-swc-loader.ts | 12 - .../basic/barrel-optimization.test.ts | 37 ++- .../app/recursive-barrel-app/page.js | 7 + .../basic/barrel-optimization/next.config.js | 2 +- .../pages/recursive-barrel.js | 5 + 8 files changed, 213 insertions(+), 96 deletions(-) create mode 100644 test/development/basic/barrel-optimization/app/recursive-barrel-app/page.js create mode 100644 test/development/basic/barrel-optimization/pages/recursive-barrel.js diff --git a/packages/next/src/build/swc/options.ts b/packages/next/src/build/swc/options.ts index 66fc96dc66573..f92bd3c99fd6f 100644 --- a/packages/next/src/build/swc/options.ts +++ b/packages/next/src/build/swc/options.ts @@ -321,7 +321,6 @@ export function getLoaderSWCOptions({ hasServerComponents, isServerLayer, isServerActionsEnabled, - optimizeBarrelExports, bundleTarget, }: // This is not passed yet as "paths" resolving is handled by webpack currently. // resolvedBaseUrl, @@ -348,9 +347,6 @@ export function getLoaderSWCOptions({ hasServerComponents?: boolean isServerLayer: boolean isServerActionsEnabled?: boolean - optimizeBarrelExports?: { - wildcard: boolean - } }) { let baseOptions: any = getBaseSWCOptions({ filename, @@ -405,9 +401,6 @@ export function getLoaderSWCOptions({ packages: optimizePackageImports, } } - if (optimizeBarrelExports) { - baseOptions.optimizeBarrelExports = optimizeBarrelExports - } const isNextDist = nextDistPath.test(filename) diff --git a/packages/next/src/build/webpack-config.ts b/packages/next/src/build/webpack-config.ts index de5d19591b091..2681305d6fb49 100644 --- a/packages/next/src/build/webpack-config.ts +++ b/packages/next/src/build/webpack-config.ts @@ -1412,27 +1412,6 @@ export default async function getBaseWebpackConfig( }, module: { rules: [ - { - // This loader rule passes the resource to the SWC loader with - // `optimizeBarrelExports` enabled. This option makes the SWC to - // transform the original code to be a JSON of its export map, so - // the barrel loader can analyze it and only keep the needed ones. - test: /__barrel_transform__/, - use: ({ resourceQuery }: { resourceQuery: string }) => { - const isFromWildcardExport = /[&?]wildcard/.test(resourceQuery) - - return [ - getSwcLoader({ - isServerLayer: false, - bundleTarget: 'client', - hasServerComponents: false, - optimizeBarrelExports: { - wildcard: isFromWildcardExport, - }, - }), - ] - }, - }, { // This loader rule works like a bridge between user's import and // the target module behind a package's barrel file. It reads SWC's @@ -1443,14 +1422,18 @@ export default async function getBaseWebpackConfig( const names = ( resourceQuery.match(/\?names=([^&]+)/)?.[1] || '' ).split(',') - const isFromWildcardExport = /[&?]wildcard/.test(resourceQuery) return [ { loader: 'next-barrel-loader', options: { names, - wildcard: isFromWildcardExport, + swcCacheDir: path.join( + dir, + config?.distDir ?? '.next', + 'cache', + 'swc' + ), }, // This is part of the request value to serve as the module key. // The barrel loader are no-op re-exported modules keyed by diff --git a/packages/next/src/build/webpack/loaders/next-barrel-loader.ts b/packages/next/src/build/webpack/loaders/next-barrel-loader.ts index 404fd9ab5608d..49defba08c1bf 100644 --- a/packages/next/src/build/webpack/loaders/next-barrel-loader.ts +++ b/packages/next/src/build/webpack/loaders/next-barrel-loader.ts @@ -86,35 +86,177 @@ import type webpack from 'webpack' +import path from 'path' +import { transform } from '../../swc' + +// This is a in-memory cache for the mapping of barrel exports. This only applies +// to the packages that we optimize. It will never change (e.g. upgrading packages) +// during the lifetime of the server so we can safely cache it. +// There is also no need to collect the cache for the same reason. +const barrelTransformMappingCache = new Map< + string, + { + prefix: string + exportList: [string, string, string][] + wildcardExports: string[] + } | null +>() + +async function getBarrelMapping( + resourcePath: string, + swcCacheDir: string, + resolve: (context: string, request: string) => Promise, + fs: { + readFile: ( + path: string, + callback: (err: any, data: string | Buffer | undefined) => void + ) => void + } +) { + if (barrelTransformMappingCache.has(resourcePath)) { + return barrelTransformMappingCache.get(resourcePath)! + } + + // This is a SWC transform specifically for `optimizeBarrelExports`. We don't + // care about other things but the export map only. + async function transpileSource( + filename: string, + source: string, + isWildcard: boolean + ) { + const isTypeScript = filename.endsWith('.ts') || filename.endsWith('.tsx') + return new Promise((res) => + transform(source, { + filename, + inputSourceMap: undefined, + sourceFileName: filename, + optimizeBarrelExports: { + wildcard: isWildcard, + }, + jsc: { + parser: { + syntax: isTypeScript ? 'typescript' : 'ecmascript', + [isTypeScript ? 'tsx' : 'jsx']: true, + }, + experimental: { + cacheRoot: swcCacheDir, + }, + }, + }).then((output) => { + res(output.code) + }) + ) + } + + // Avoid circular `export *` dependencies + const visited = new Set() + async function getMatches(file: string, isWildcard: boolean) { + if (visited.has(file)) { + return null + } + visited.add(file) + + const source = await new Promise((res, rej) => { + fs.readFile(file, (err, data) => { + if (err || data === undefined) { + rej(err) + } else { + res(data.toString()) + } + }) + }) + + const output = await transpileSource(file, source, isWildcard) + + const matches = output.match( + /^([^]*)export (const|var) __next_private_export_map__ = ('[^']+'|"[^"]+")/ + ) + if (!matches) { + return null + } + + const prefix = matches[1] + let exportList = JSON.parse(matches[3].slice(1, -1)) as [ + string, + string, + string + ][] + const wildcardExports = [ + ...output.matchAll(/export \* from "([^"]+)"/g), + ].map((match) => match[1]) + + // In the wildcard case, if the value is exported from another file, we + // redirect to that file (decl[0]). Otherwise, export from the current + // file itself. + if (isWildcard) { + for (const decl of exportList) { + decl[1] = file + decl[2] = decl[0] + } + } + + // This recursively handles the wildcard exports (e.g. `export * from './a'`) + if (wildcardExports.length) { + await Promise.all( + wildcardExports.map(async (req) => { + const targetPath = await resolve( + path.dirname(file), + req.replace('__barrel_optimize__?names=__PLACEHOLDER__!=!', '') + ) + + const targetMatches = await getMatches(targetPath, true) + if (targetMatches) { + // Merge the export list + exportList = exportList.concat(targetMatches.exportList) + } + }) + ) + } + + return { + prefix, + exportList, + wildcardExports, + } + } + + const res = await getMatches(resourcePath, false) + barrelTransformMappingCache.set(resourcePath, res) + + return res +} + const NextBarrelLoader = async function ( this: webpack.LoaderContext<{ names: string[] - wildcard: boolean + swcCacheDir: string }> ) { this.async() - const { names, wildcard } = this.getOptions() - - const source = await new Promise((resolve, reject) => { - this.loadModule( - `__barrel_transform__${wildcard ? '?wildcard' : ''}!=!${ - this.resourcePath - }`, - (err, src) => { - if (err) { - reject(err) - } else { - resolve(src) - } - } - ) + this.cacheable(true) + + const { names, swcCacheDir } = this.getOptions() + + // For barrel optimizations, we always prefer the "module" field over the + // "main" field because ESM handling is more robust with better tree-shaking. + const resolve = this.getResolve({ + mainFields: ['module', 'main'], }) - const matches = source.match( - /^([^]*)export const __next_private_export_map__ = ('[^']+'|"[^"]+")/ + const mapping = await getBarrelMapping( + this.resourcePath, + swcCacheDir, + resolve, + this.fs ) - if (!matches) { + // `resolve` adds all sub-paths to the dependency graph. However, we already + // cached the mapping and we assume them to not change. So, we can safely + // clear the dependencies here to avoid unnecessary watchers which turned out + // to be very expensive. + this.clearDependencies() + + if (!mapping) { // This file isn't a barrel and we can't apply any optimizations. Let's re-export everything. // Since this loader accepts `names` and the request is keyed with `names`, we can't simply // return the original source here. That will create these imports with different names as @@ -123,19 +265,12 @@ const NextBarrelLoader = async function ( return } - const wildcardExports = [...source.matchAll(/export \* from "([^"]+)"/g)] - // It needs to keep the prefix for comments and directives like "use client". - const prefix = matches[1] - - const exportList = JSON.parse(matches[2].slice(1, -1)) as [ - string, - string, - string - ][] + const prefix = mapping.prefix + const exportList = mapping.exportList const exportMap = new Map() - for (const [name, path, orig] of exportList) { - exportMap.set(name, [path, orig]) + for (const [name, filePath, orig] of exportList) { + exportMap.set(name, [filePath, orig]) } let output = prefix @@ -145,15 +280,6 @@ const NextBarrelLoader = async function ( if (exportMap.has(name)) { const decl = exportMap.get(name)! - // In the wildcard case, if the value is exported from another file, we - // redirect to that file (decl[0]). Otherwise, export from the current - // file itself (this.resourcePath). - if (wildcard && !decl[0]) { - // E.g. the file contains `export const a = 1` - decl[0] = this.resourcePath - decl[1] = name - } - if (decl[1] === '*') { output += `\nexport * as ${name} from ${JSON.stringify(decl[0])}` } else if (decl[1] === 'default') { @@ -174,11 +300,9 @@ const NextBarrelLoader = async function ( // These are from wildcard exports. if (missedNames.length > 0) { - for (const match of wildcardExports) { - const path = match[1] - + for (const req of mapping.wildcardExports) { output += `\nexport * from ${JSON.stringify( - path.replace('__PLACEHOLDER__', missedNames.join(',') + '&wildcard') + req.replace('__PLACEHOLDER__', missedNames.join(',') + '&wildcard') )}` } } diff --git a/packages/next/src/build/webpack/loaders/next-swc-loader.ts b/packages/next/src/build/webpack/loaders/next-swc-loader.ts index 253ef9e39035a..6a18b06d9bd2b 100644 --- a/packages/next/src/build/webpack/loaders/next-swc-loader.ts +++ b/packages/next/src/build/webpack/loaders/next-swc-loader.ts @@ -45,9 +45,6 @@ export interface SWCLoaderOptions { bundleTarget: BundleType hasServerComponents?: boolean isServerLayer: boolean - optimizeBarrelExports?: { - wildcard: boolean - } } async function loaderTransform( @@ -73,19 +70,11 @@ async function loaderTransform( swcCacheDir, hasServerComponents, isServerLayer, - optimizeBarrelExports, bundleTarget, } = loaderOptions const isPageFile = filename.startsWith(pagesDir) const relativeFilePathFromRoot = path.relative(rootDir, filename) - // For testing purposes - if (process.env.NEXT_TEST_MODE) { - if (loaderOptions.optimizeBarrelExports) { - console.log('optimizeBarrelExports:', filename) - } - } - const swcOptions = getLoaderSWCOptions({ pagesDir, appDir, @@ -106,7 +95,6 @@ async function loaderTransform( hasServerComponents, isServerActionsEnabled: nextConfig?.experimental?.serverActions, isServerLayer, - optimizeBarrelExports, bundleTarget, }) diff --git a/test/development/basic/barrel-optimization.test.ts b/test/development/basic/barrel-optimization.test.ts index fe18298a59dcf..7523cac2c65d7 100644 --- a/test/development/basic/barrel-optimization.test.ts +++ b/test/development/basic/barrel-optimization.test.ts @@ -42,6 +42,7 @@ describe('optimizePackageImports', () => { '@headlessui/react': '1.7.17', '@heroicons/react': '2.0.18', '@visx/visx': '3.3.0', + 'recursive-barrel': '1.0.0', }, }) }) @@ -97,24 +98,40 @@ describe('optimizePackageImports', () => { } }) - it('should reuse the transformed barrel meta file from SWC', async () => { + it('app - should optimize recursive wildcard export mapping', async () => { let logs = '' next.on('stdout', (log) => { logs += log }) - const html = await next.render('/dedupe') + await next.render('/recursive-barrel-app') - // Ensure the icons are rendered - expect(html).toContain(' { + let logs = '' + next.on('stdout', (log) => { + logs += log + }) + + await next.render('/recursive-barrel') + + const modules = [...logs.matchAll(/\((\d+) modules\)/g)] + + expect(modules.length).toBeGreaterThanOrEqual(1) + for (const [, moduleCount] of modules) { + // Ensure that the number of modules is less than 1000 - otherwise we're + // importing the entire library. + expect(parseInt(moduleCount)).toBeLessThan(1000) + } }) it('should handle recursive wildcard exports', async () => { diff --git a/test/development/basic/barrel-optimization/app/recursive-barrel-app/page.js b/test/development/basic/barrel-optimization/app/recursive-barrel-app/page.js new file mode 100644 index 0000000000000..52d86626f0710 --- /dev/null +++ b/test/development/basic/barrel-optimization/app/recursive-barrel-app/page.js @@ -0,0 +1,7 @@ +'use client' + +import { b_8_7_6_4 } from 'recursive-barrel' + +export default function Page() { + return

{b_8_7_6_4}

+} diff --git a/test/development/basic/barrel-optimization/next.config.js b/test/development/basic/barrel-optimization/next.config.js index d966eb99681dc..0afa42db843da 100644 --- a/test/development/basic/barrel-optimization/next.config.js +++ b/test/development/basic/barrel-optimization/next.config.js @@ -1,5 +1,5 @@ module.exports = { experimental: { - optimizePackageImports: ['my-lib'], + optimizePackageImports: ['my-lib', 'recursive-barrel'], }, } diff --git a/test/development/basic/barrel-optimization/pages/recursive-barrel.js b/test/development/basic/barrel-optimization/pages/recursive-barrel.js new file mode 100644 index 0000000000000..8b2b67baf90e6 --- /dev/null +++ b/test/development/basic/barrel-optimization/pages/recursive-barrel.js @@ -0,0 +1,5 @@ +import { b_8_3_1_1 as v } from 'recursive-barrel' + +export default function Page() { + return

{v}

+}