From cf9c2201235028d05079ff85e2a1bd4a9fa71f22 Mon Sep 17 00:00:00 2001 From: Mahdi Boomeri Date: Thu, 15 Dec 2022 23:22:28 +0330 Subject: [PATCH 01/34] fix: keep both version of conflicting deps --- src/rollup/plugins/externals.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/rollup/plugins/externals.ts b/src/rollup/plugins/externals.ts index 1b018a1411..e66827da2d 100644 --- a/src/rollup/plugins/externals.ts +++ b/src/rollup/plugins/externals.ts @@ -201,6 +201,7 @@ export function externals(opts: NodeExternalsOptions): Plugin { const tracedPackages = new Map(); // name => pkgDir const ignoreDirs = []; const ignoreWarns = new Set(); + const conflictingPackages = []; for (const file of tracedFiles) { const { baseDir, pkgName } = parseNodeModulePath(file); if (!pkgName) { @@ -230,6 +231,7 @@ export function externals(opts: NodeExternalsOptions): Plugin { consola.warn(warn); ignoreWarns.add(warn); } + conflictingPackages.push(pkgName) } const [newerDir, olderDir] = isNewer @@ -242,7 +244,9 @@ export function externals(opts: NodeExternalsOptions): Plugin { ); } // Exclude older version files - ignoreDirs.push(olderDir + "/"); + // if (false) { // check nitro options to determain if pacakge needs to be excluded + // ignoreDirs.push(olderDir + '/') + // } pkgDir = newerDir; // Update for tracedPackages } @@ -269,8 +273,10 @@ export function externals(opts: NodeExternalsOptions): Plugin { return; } const src = resolve(opts.traceOptions.base, file); - const { pkgName, subpath } = parseNodeModulePath(file); - const dst = resolve(opts.outDir, `node_modules/${pkgName + subpath}`); + const { pkgName, subpath, baseDir } = parseNodeModulePath(file) + const version = await getPackageJson(resolve(baseDir, pkgName)).then(r => r.version) + const fullName = conflictingPackages.includes(pkgName) ? `${pkgName}@${version}` : pkgName + const dst = resolve(opts.outDir, `node_modules/${fullName + subpath}`) await fsp.mkdir(dirname(dst), { recursive: true }); try { await fsp.copyFile(src, dst); From b0fd6a4bf7b6321b3583c3449aca2d02f4045672 Mon Sep 17 00:00:00 2001 From: Mahdi Boomeri Date: Sat, 17 Dec 2022 22:44:39 +0330 Subject: [PATCH 02/34] feat: add the ability to opt-in and out of optimization --- src/rollup/config.ts | 1 + src/rollup/plugins/externals.ts | 34 ++++++++++++++++----------------- src/types/nitro.ts | 4 ++++ 3 files changed, 22 insertions(+), 17 deletions(-) diff --git a/src/rollup/config.ts b/src/rollup/config.ts index 3782f94d4b..c1d3804b67 100644 --- a/src/rollup/config.ts +++ b/src/rollup/config.ts @@ -336,6 +336,7 @@ export const plugins = [ "node", "import", ], + optimizeDeps: nitro.options.optimizeExternals, }) ) ); diff --git a/src/rollup/plugins/externals.ts b/src/rollup/plugins/externals.ts index e66827da2d..e1c51f189f 100644 --- a/src/rollup/plugins/externals.ts +++ b/src/rollup/plugins/externals.ts @@ -16,6 +16,10 @@ export interface NodeExternalsOptions { moduleDirectories?: string[]; exportConditions?: string[]; traceInclude?: string[]; + optimizeExternals?: { + include?: string[]; + exclude?: string[]; + }; } export function externals(opts: NodeExternalsOptions): Plugin { @@ -200,8 +204,9 @@ export function externals(opts: NodeExternalsOptions): Plugin { // Keep track of npm packages const tracedPackages = new Map(); // name => pkgDir const ignoreDirs = []; - const ignoreWarns = new Set(); - const conflictingPackages = []; + const ignoreLogs = new Set(); + const excludeOptimization = new Set(opts.optimizeExternals?.exclude ?? []); + const includeOptimization = new Set(opts.optimizeExternals?.include ?? []) for (const file of tracedFiles) { const { baseDir, pkgName } = parseNodeModulePath(file); if (!pkgName) { @@ -220,18 +225,13 @@ export function externals(opts: NodeExternalsOptions): Plugin { // Warn about major version differences const getMajor = (v: string) => v.split(".").find((s) => s !== "0"); - if (getMajor(v1) !== getMajor(v2)) { - const warn = - `Multiple major versions of package \`${pkgName}\` are being externalized. Picking latest version:\n\n` + - [ - ` ${isNewer ? "-" : "+"} ` + existingPkgDir + "@" + v1, - ` ${isNewer ? "+" : "-"} ` + pkgDir + "@" + v2, - ].join("\n"); - if (!ignoreWarns.has(warn)) { - consola.warn(warn); - ignoreWarns.add(warn); + if (getMajor(v1) !== getMajor(v2) && !includeOptimization.has(pkgName)) { + const log = `Multiple major versions of package \`${pkgName}\` are being externalized. Skipping optimization...`; + if (!ignoreLogs.has(log)) { + consola.info(log); + ignoreLogs.add(log); } - conflictingPackages.push(pkgName) + excludeOptimization.add(pkgName) } const [newerDir, olderDir] = isNewer @@ -244,9 +244,9 @@ export function externals(opts: NodeExternalsOptions): Plugin { ); } // Exclude older version files - // if (false) { // check nitro options to determain if pacakge needs to be excluded - // ignoreDirs.push(olderDir + '/') - // } + if (!includeOptimization.has(pkgName) && excludeOptimization.has(pkgName)) { + ignoreDirs.push(olderDir + '/') + } pkgDir = newerDir; // Update for tracedPackages } @@ -275,7 +275,7 @@ export function externals(opts: NodeExternalsOptions): Plugin { const src = resolve(opts.traceOptions.base, file); const { pkgName, subpath, baseDir } = parseNodeModulePath(file) const version = await getPackageJson(resolve(baseDir, pkgName)).then(r => r.version) - const fullName = conflictingPackages.includes(pkgName) ? `${pkgName}@${version}` : pkgName + const fullName = excludeOptimization.has(pkgName) ? `${pkgName}@${version}` : pkgName const dst = resolve(opts.outDir, `node_modules/${fullName + subpath}`) await fsp.mkdir(dirname(dst), { recursive: true }); try { diff --git a/src/types/nitro.ts b/src/types/nitro.ts index f0a43c2756..4340441d6a 100644 --- a/src/types/nitro.ts +++ b/src/types/nitro.ts @@ -216,6 +216,10 @@ export interface NitroOptions extends PresetOptions { }; noExternals: boolean; externals: NodeExternalsOptions; + optimizeExternals?: { + include?: string[]; + exclude?: string[]; + }; analyze: false | PluginVisualizerOptions; replace: Record string)>; commonJS?: RollupCommonJSOptions; From ff61c5db3393d946f54c1ad3a1c08dffd5dffa50 Mon Sep 17 00:00:00 2001 From: Mahdi Boomeri Date: Sat, 17 Dec 2022 23:02:42 +0330 Subject: [PATCH 03/34] fix minor issue --- src/rollup/config.ts | 2 +- src/rollup/plugins/externals.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rollup/config.ts b/src/rollup/config.ts index c1d3804b67..a9ec96fde1 100644 --- a/src/rollup/config.ts +++ b/src/rollup/config.ts @@ -336,7 +336,7 @@ export const plugins = [ "node", "import", ], - optimizeDeps: nitro.options.optimizeExternals, + optimizeExternals: nitro.options.optimizeExternals, }) ) ); diff --git a/src/rollup/plugins/externals.ts b/src/rollup/plugins/externals.ts index e1c51f189f..7525aca6a7 100644 --- a/src/rollup/plugins/externals.ts +++ b/src/rollup/plugins/externals.ts @@ -244,7 +244,7 @@ export function externals(opts: NodeExternalsOptions): Plugin { ); } // Exclude older version files - if (!includeOptimization.has(pkgName) && excludeOptimization.has(pkgName)) { + if (includeOptimization.has(pkgName) && !excludeOptimization.has(pkgName)) { ignoreDirs.push(olderDir + '/') } pkgDir = newerDir; // Update for tracedPackages From 553a32b37a2197c7caa72fe64f449ecf0de44f69 Mon Sep 17 00:00:00 2001 From: Mahdi Boomeri Date: Sat, 17 Dec 2022 23:21:49 +0330 Subject: [PATCH 04/34] added docs --- docs/content/3.config/index.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/content/3.config/index.md b/docs/content/3.config/index.md index 60e7c00e5c..da535c88de 100644 --- a/docs/content/3.config/index.md +++ b/docs/content/3.config/index.md @@ -340,3 +340,18 @@ Rollup specific option. ## `commonJS` Rollup specific option. Specifies additional configuration for the rollup CommonJS plugin. + +## `optimizeExternals` + +**⚠️ Caution! This is an advanced configuration. things can go wrong if misconfigured.** + +Options to give you control over optimization of externalized packages. + +```js +{ + optimizeExternals: { + include: [''], + exclude: [''], // Handled by default if version difference is major + } +} +``` From 7c9edf1ff3d0be4aec5c8bb036bd2584a97d1bb6 Mon Sep 17 00:00:00 2001 From: Mahdi Boomeri Date: Sat, 17 Dec 2022 23:24:52 +0330 Subject: [PATCH 05/34] prettier --- src/rollup/plugins/externals.ts | 34 +++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/src/rollup/plugins/externals.ts b/src/rollup/plugins/externals.ts index 7525aca6a7..0f7fea6462 100644 --- a/src/rollup/plugins/externals.ts +++ b/src/rollup/plugins/externals.ts @@ -205,8 +205,12 @@ export function externals(opts: NodeExternalsOptions): Plugin { const tracedPackages = new Map(); // name => pkgDir const ignoreDirs = []; const ignoreLogs = new Set(); - const excludeOptimization = new Set(opts.optimizeExternals?.exclude ?? []); - const includeOptimization = new Set(opts.optimizeExternals?.include ?? []) + const excludeOptimization = new Set( + opts.optimizeExternals?.exclude ?? [] + ); + const includeOptimization = new Set( + opts.optimizeExternals?.include ?? [] + ); for (const file of tracedFiles) { const { baseDir, pkgName } = parseNodeModulePath(file); if (!pkgName) { @@ -225,13 +229,16 @@ export function externals(opts: NodeExternalsOptions): Plugin { // Warn about major version differences const getMajor = (v: string) => v.split(".").find((s) => s !== "0"); - if (getMajor(v1) !== getMajor(v2) && !includeOptimization.has(pkgName)) { + if ( + getMajor(v1) !== getMajor(v2) && + !includeOptimization.has(pkgName) + ) { const log = `Multiple major versions of package \`${pkgName}\` are being externalized. Skipping optimization...`; if (!ignoreLogs.has(log)) { consola.info(log); ignoreLogs.add(log); } - excludeOptimization.add(pkgName) + excludeOptimization.add(pkgName); } const [newerDir, olderDir] = isNewer @@ -244,8 +251,11 @@ export function externals(opts: NodeExternalsOptions): Plugin { ); } // Exclude older version files - if (includeOptimization.has(pkgName) && !excludeOptimization.has(pkgName)) { - ignoreDirs.push(olderDir + '/') + if ( + includeOptimization.has(pkgName) && + !excludeOptimization.has(pkgName) + ) { + ignoreDirs.push(olderDir + "/"); } pkgDir = newerDir; // Update for tracedPackages } @@ -273,10 +283,14 @@ export function externals(opts: NodeExternalsOptions): Plugin { return; } const src = resolve(opts.traceOptions.base, file); - const { pkgName, subpath, baseDir } = parseNodeModulePath(file) - const version = await getPackageJson(resolve(baseDir, pkgName)).then(r => r.version) - const fullName = excludeOptimization.has(pkgName) ? `${pkgName}@${version}` : pkgName - const dst = resolve(opts.outDir, `node_modules/${fullName + subpath}`) + const { pkgName, subpath, baseDir } = parseNodeModulePath(file); + const version = await getPackageJson(resolve(baseDir, pkgName)).then( + (r) => r.version + ); + const fullName = excludeOptimization.has(pkgName) + ? `${pkgName}@${version}` + : pkgName; + const dst = resolve(opts.outDir, `node_modules/${fullName + subpath}`); await fsp.mkdir(dirname(dst), { recursive: true }); try { await fsp.copyFile(src, dst); From ea930c5ad556f1fe290cf6565370e2b68c445133 Mon Sep 17 00:00:00 2001 From: Mahdi Boomeri Date: Sat, 17 Dec 2022 23:42:08 +0330 Subject: [PATCH 06/34] fix breaking normal behavior --- src/rollup/plugins/externals.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/rollup/plugins/externals.ts b/src/rollup/plugins/externals.ts index 0f7fea6462..d5ed6e725e 100644 --- a/src/rollup/plugins/externals.ts +++ b/src/rollup/plugins/externals.ts @@ -251,10 +251,7 @@ export function externals(opts: NodeExternalsOptions): Plugin { ); } // Exclude older version files - if ( - includeOptimization.has(pkgName) && - !excludeOptimization.has(pkgName) - ) { + if (!excludeOptimization.has(pkgName)) { ignoreDirs.push(olderDir + "/"); } pkgDir = newerDir; // Update for tracedPackages From db3550a336e5cf144a68a42f51ec33bb4a8c8887 Mon Sep 17 00:00:00 2001 From: Mahdi Boomeri Date: Sat, 17 Dec 2022 23:45:18 +0330 Subject: [PATCH 07/34] fix docs grammar --- docs/content/3.config/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/3.config/index.md b/docs/content/3.config/index.md index da535c88de..7534b7fbfd 100644 --- a/docs/content/3.config/index.md +++ b/docs/content/3.config/index.md @@ -345,7 +345,7 @@ Rollup specific option. Specifies additional configuration for the rollup Common **⚠️ Caution! This is an advanced configuration. things can go wrong if misconfigured.** -Options to give you control over optimization of externalized packages. +Options to give you control over the optimization of externalized packages. ```js { From 9fcb2d2e80101425018df6187d276face98b9581 Mon Sep 17 00:00:00 2001 From: Mahdi Boomeri Date: Tue, 20 Dec 2022 17:59:25 +0330 Subject: [PATCH 08/34] feat: find parent package base on file path --- src/rollup/plugins/externals.ts | 57 +++++++++++++++++++++++++++++---- 1 file changed, 51 insertions(+), 6 deletions(-) diff --git a/src/rollup/plugins/externals.ts b/src/rollup/plugins/externals.ts index d5ed6e725e..508bff11c2 100644 --- a/src/rollup/plugins/externals.ts +++ b/src/rollup/plugins/externals.ts @@ -174,14 +174,13 @@ export function externals(opts: NodeExternalsOptions): Plugin { } // Trace files - let tracedFiles = await nodeFileTrace( + const fileTrace = await nodeFileTrace( [...trackedExternals], opts.traceOptions - ) - .then((r) => - [...r.fileList].map((f) => resolve(opts.traceOptions.base, f)) - ) - .then((r) => r.filter((file) => file.includes("node_modules"))); + ); + let tracedFiles = [...fileTrace.fileList] + .map((f) => resolve(opts.traceOptions.base, f)) + .filter((file) => file.includes("node_modules")); // Resolve symlinks tracedFiles = await Promise.all( @@ -201,6 +200,52 @@ export function externals(opts: NodeExternalsOptions): Plugin { return pkgJSON; }; + // Find parent base on file path + const getParent = async (pkgPath: string) => { + const { pkgName, baseDir } = parseNodeModulePath(pkgPath); + const pkgVersion = await getPackageJson(resolve(baseDir, pkgName)).then( + (r) => r.version + ); + + const possibleParents = [ + ...new Set( + [...fileTrace.reasons] + .filter((r) => !r[1].ignored) + .filter((r) => parseNodeModulePath(r[0]).pkgName === pkgName) + .flatMap((r) => + [...r[1].parents].filter( + (v) => parseNodeModulePath(v).pkgName !== pkgName + ) + ) // Remove self-refrencing + ), + ]; + + // Find the currect parent base on package.json dependency version + for (const possible of possibleParents) { + const thePath = resolve(opts.traceOptions.base, possible); + const { pkgName: currentPkgName, baseDir: currentBaseDir } = + parseNodeModulePath(await fsp.realpath(thePath)); + + const packageJson = await getPackageJson( + resolve(currentBaseDir, currentPkgName) + ); + const version = packageJson.dependencies[pkgName]; + + if (!version) { + return null; + } + + const v1 = semver.parse( + version.replace("^", "").replace("~", "") + ).major; + const v2 = semver.parse(pkgVersion).major; + if (v1 === v2) { + return currentPkgName; + } + } + return null; + }; + // Keep track of npm packages const tracedPackages = new Map(); // name => pkgDir const ignoreDirs = []; From 2b8fb3f163e784d143eaae65121ab5f666b4cf22 Mon Sep 17 00:00:00 2001 From: Mahdi Boomeri Date: Tue, 20 Dec 2022 18:02:57 +0330 Subject: [PATCH 09/34] refactor: use regex --- src/rollup/plugins/externals.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/rollup/plugins/externals.ts b/src/rollup/plugins/externals.ts index 508bff11c2..7f59577d37 100644 --- a/src/rollup/plugins/externals.ts +++ b/src/rollup/plugins/externals.ts @@ -235,9 +235,7 @@ export function externals(opts: NodeExternalsOptions): Plugin { return null; } - const v1 = semver.parse( - version.replace("^", "").replace("~", "") - ).major; + const v1 = semver.parse(version.replace(/\^|~/, "")).major; const v2 = semver.parse(pkgVersion).major; if (v1 === v2) { return currentPkgName; From baae86aa4932b5697ae470f1f8354552399374bf Mon Sep 17 00:00:00 2001 From: Mahdi Boomeri Date: Tue, 20 Dec 2022 18:07:46 +0330 Subject: [PATCH 10/34] fix naming and if condition --- src/rollup/plugins/externals.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/rollup/plugins/externals.ts b/src/rollup/plugins/externals.ts index 7f59577d37..e742bdf661 100644 --- a/src/rollup/plugins/externals.ts +++ b/src/rollup/plugins/externals.ts @@ -223,11 +223,11 @@ export function externals(opts: NodeExternalsOptions): Plugin { // Find the currect parent base on package.json dependency version for (const possible of possibleParents) { const thePath = resolve(opts.traceOptions.base, possible); - const { pkgName: currentPkgName, baseDir: currentBaseDir } = + const { pkgName: existingPkgName, baseDir: existingBaseDir } = parseNodeModulePath(await fsp.realpath(thePath)); const packageJson = await getPackageJson( - resolve(currentBaseDir, currentPkgName) + resolve(existingBaseDir, existingPkgName) ); const version = packageJson.dependencies[pkgName]; @@ -238,7 +238,7 @@ export function externals(opts: NodeExternalsOptions): Plugin { const v1 = semver.parse(version.replace(/\^|~/, "")).major; const v2 = semver.parse(pkgVersion).major; if (v1 === v2) { - return currentPkgName; + return existingPkgName; } } return null; @@ -296,8 +296,8 @@ export function externals(opts: NodeExternalsOptions): Plugin { // Exclude older version files if (!excludeOptimization.has(pkgName)) { ignoreDirs.push(olderDir + "/"); + pkgDir = newerDir; // Update for tracedPackages } - pkgDir = newerDir; // Update for tracedPackages } // Add to traced packages From 5bdfd5144cb984db17c37efd02d6ff5fd18019e2 Mon Sep 17 00:00:00 2001 From: Mahdi Boomeri Date: Tue, 20 Dec 2022 18:08:23 +0330 Subject: [PATCH 11/34] Revert writeFile changes --- src/rollup/plugins/externals.ts | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/rollup/plugins/externals.ts b/src/rollup/plugins/externals.ts index e742bdf661..2f54ab3450 100644 --- a/src/rollup/plugins/externals.ts +++ b/src/rollup/plugins/externals.ts @@ -323,14 +323,8 @@ export function externals(opts: NodeExternalsOptions): Plugin { return; } const src = resolve(opts.traceOptions.base, file); - const { pkgName, subpath, baseDir } = parseNodeModulePath(file); - const version = await getPackageJson(resolve(baseDir, pkgName)).then( - (r) => r.version - ); - const fullName = excludeOptimization.has(pkgName) - ? `${pkgName}@${version}` - : pkgName; - const dst = resolve(opts.outDir, `node_modules/${fullName + subpath}`); + const { pkgName, subpath } = parseNodeModulePath(file); + const dst = resolve(opts.outDir, `node_modules/${pkgName + subpath}`); await fsp.mkdir(dirname(dst), { recursive: true }); try { await fsp.copyFile(src, dst); From 981050e8a9ae309f10d7e3c71a26887230338f3a Mon Sep 17 00:00:00 2001 From: Mahdi Boomeri Date: Tue, 20 Dec 2022 21:38:05 +0330 Subject: [PATCH 12/34] refactor --- src/rollup/plugins/externals.ts | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/rollup/plugins/externals.ts b/src/rollup/plugins/externals.ts index 2f54ab3450..8db9725969 100644 --- a/src/rollup/plugins/externals.ts +++ b/src/rollup/plugins/externals.ts @@ -270,12 +270,16 @@ export function externals(opts: NodeExternalsOptions): Plugin { const v2 = await getPackageJson(pkgDir).then((r) => r.version); const isNewer = semver.gt(v2, v1); + const [newerDir, olderDir] = isNewer + ? [pkgDir, existingPkgDir] + : [existingPkgDir, pkgDir]; + // Warn about major version differences const getMajor = (v: string) => v.split(".").find((s) => s !== "0"); - if ( - getMajor(v1) !== getMajor(v2) && - !includeOptimization.has(pkgName) - ) { + const shouldOptimize = + getMajor(v1) !== getMajor(v2) && !includeOptimization.has(pkgName); + + if (shouldOptimize) { const log = `Multiple major versions of package \`${pkgName}\` are being externalized. Skipping optimization...`; if (!ignoreLogs.has(log)) { consola.info(log); @@ -283,18 +287,13 @@ export function externals(opts: NodeExternalsOptions): Plugin { } excludeOptimization.add(pkgName); } - - const [newerDir, olderDir] = isNewer - ? [pkgDir, existingPkgDir] - : [existingPkgDir, pkgDir]; // Try to map traced files from one package to another for minor/patch versions - if (getMajor(v1) === getMajor(v2)) { + if (!shouldOptimize && !excludeOptimization.has(pkgName)) { tracedFiles = tracedFiles.map((f) => f.startsWith(olderDir + "/") ? f.replace(olderDir, newerDir) : f ); - } - // Exclude older version files - if (!excludeOptimization.has(pkgName)) { + + // Exclude older version files ignoreDirs.push(olderDir + "/"); pkgDir = newerDir; // Update for tracedPackages } From f5267cc080db93f07c3573c57505694314982801 Mon Sep 17 00:00:00 2001 From: Mahdi Boomeri Date: Wed, 21 Dec 2022 00:12:07 +0330 Subject: [PATCH 13/34] feat: write the nested dependencies --- src/rollup/plugins/externals.ts | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/rollup/plugins/externals.ts b/src/rollup/plugins/externals.ts index 8db9725969..e1aded1d90 100644 --- a/src/rollup/plugins/externals.ts +++ b/src/rollup/plugins/externals.ts @@ -274,7 +274,7 @@ export function externals(opts: NodeExternalsOptions): Plugin { ? [pkgDir, existingPkgDir] : [existingPkgDir, pkgDir]; - // Warn about major version differences + // Warn about major version differences and exclude the package from being optimized const getMajor = (v: string) => v.split(".").find((s) => s !== "0"); const shouldOptimize = getMajor(v1) !== getMajor(v2) && !includeOptimization.has(pkgName); @@ -323,7 +323,22 @@ export function externals(opts: NodeExternalsOptions): Plugin { } const src = resolve(opts.traceOptions.base, file); const { pkgName, subpath } = parseNodeModulePath(file); - const dst = resolve(opts.outDir, `node_modules/${pkgName + subpath}`); + let dst = resolve(opts.outDir, `node_modules/${pkgName + subpath}`); + + if (excludeOptimization.has(pkgName)) { + const parent = await getParent(file); + + if (parent) { + dst = resolve( + opts.outDir, + "node_modules", + parent, + 'node_modules', + pkgName + subpath + ); + } + } + await fsp.mkdir(dirname(dst), { recursive: true }); try { await fsp.copyFile(src, dst); From 3a2bc12cb43cf299e776a36c86dc3e33cf0a92e3 Mon Sep 17 00:00:00 2001 From: Mahdi Boomeri Date: Wed, 21 Dec 2022 23:39:58 +0330 Subject: [PATCH 14/34] fix: use full version compare for getParent --- src/rollup/plugins/externals.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/rollup/plugins/externals.ts b/src/rollup/plugins/externals.ts index e1aded1d90..5e270e593f 100644 --- a/src/rollup/plugins/externals.ts +++ b/src/rollup/plugins/externals.ts @@ -235,8 +235,9 @@ export function externals(opts: NodeExternalsOptions): Plugin { return null; } - const v1 = semver.parse(version.replace(/\^|~/, "")).major; - const v2 = semver.parse(pkgVersion).major; + const v1 = semver.parse(version.replace(/\^|~/, "")).version; + const v2 = semver.parse(pkgVersion).version; + console.log(v1, v2) if (v1 === v2) { return existingPkgName; } From 7cf9b6069ae73edf4d4168cdbed600473ae72610 Mon Sep 17 00:00:00 2001 From: Mahdi Boomeri Date: Wed, 21 Dec 2022 23:41:15 +0330 Subject: [PATCH 15/34] prettier --- src/rollup/plugins/externals.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rollup/plugins/externals.ts b/src/rollup/plugins/externals.ts index 5e270e593f..76b614fd7e 100644 --- a/src/rollup/plugins/externals.ts +++ b/src/rollup/plugins/externals.ts @@ -237,7 +237,7 @@ export function externals(opts: NodeExternalsOptions): Plugin { const v1 = semver.parse(version.replace(/\^|~/, "")).version; const v2 = semver.parse(pkgVersion).version; - console.log(v1, v2) + console.log(v1, v2); if (v1 === v2) { return existingPkgName; } @@ -334,7 +334,7 @@ export function externals(opts: NodeExternalsOptions): Plugin { opts.outDir, "node_modules", parent, - 'node_modules', + "node_modules", pkgName + subpath ); } From 4ee11f056b41f262f369ed1c55c0865e2fccf533 Mon Sep 17 00:00:00 2001 From: Mahdi Boomeri Date: Wed, 21 Dec 2022 23:43:35 +0330 Subject: [PATCH 16/34] remove console log --- src/rollup/plugins/externals.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/rollup/plugins/externals.ts b/src/rollup/plugins/externals.ts index 76b614fd7e..cf36524acb 100644 --- a/src/rollup/plugins/externals.ts +++ b/src/rollup/plugins/externals.ts @@ -237,7 +237,6 @@ export function externals(opts: NodeExternalsOptions): Plugin { const v1 = semver.parse(version.replace(/\^|~/, "")).version; const v2 = semver.parse(pkgVersion).version; - console.log(v1, v2); if (v1 === v2) { return existingPkgName; } From bd54d7d970455356e987f7eb01de5743d051e831 Mon Sep 17 00:00:00 2001 From: Mahdi Boomeri Date: Thu, 22 Dec 2022 00:10:47 +0330 Subject: [PATCH 17/34] ensure including package.json --- src/rollup/plugins/externals.ts | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/rollup/plugins/externals.ts b/src/rollup/plugins/externals.ts index cf36524acb..457d7fe342 100644 --- a/src/rollup/plugins/externals.ts +++ b/src/rollup/plugins/externals.ts @@ -259,6 +259,7 @@ export function externals(opts: NodeExternalsOptions): Plugin { if (!pkgName) { continue; } + let pkgFullName = pkgName; let pkgDir = resolve(baseDir, pkgName); // Check for duplicate versions @@ -297,10 +298,22 @@ export function externals(opts: NodeExternalsOptions): Plugin { ignoreDirs.push(olderDir + "/"); pkgDir = newerDir; // Update for tracedPackages } + + // Make the package name unique if there are multiple versions + if (excludeOptimization.has(pkgName)) { + pkgFullName = `${pkgName}#${v2}`; + + // Add the version to previous dependency + const previousPkg = tracedPackages.get(pkgName); + if (previousPkg) { + tracedPackages.delete(pkgName); + tracedPackages.set(`${pkgName}#${v1}`, previousPkg); + } + } } // Add to traced packages - tracedPackages.set(pkgName, pkgDir); + tracedPackages.set(pkgFullName, pkgDir); } // Filter out files from ignored packages and dedup @@ -353,6 +366,9 @@ export function externals(opts: NodeExternalsOptions): Plugin { ); // Write an informative package.json + const bundledDependencies = [ + ...new Set([...tracedPackages.keys()].map((p) => p.split("#")[0])), // Merge duplicated dependencies + ]; await fsp.writeFile( resolve(opts.outDir, "package.json"), JSON.stringify( @@ -360,7 +376,7 @@ export function externals(opts: NodeExternalsOptions): Plugin { name: "nitro-output", version: "0.0.0", private: true, - bundledDependencies: [...tracedPackages.keys()], + bundledDependencies, }, null, 2 From 8d1801896fdf8a97dff7f40a7a64ef94c26713bf Mon Sep 17 00:00:00 2001 From: Mahdi Boomeri Date: Thu, 22 Dec 2022 00:24:11 +0330 Subject: [PATCH 18/34] fix the test --- test/fixture/_/node_modules/nitro-dep-b/package.json | 2 +- test/tests.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/fixture/_/node_modules/nitro-dep-b/package.json b/test/fixture/_/node_modules/nitro-dep-b/package.json index 7e30c8dd8f..d7235353b7 100644 --- a/test/fixture/_/node_modules/nitro-dep-b/package.json +++ b/test/fixture/_/node_modules/nitro-dep-b/package.json @@ -3,6 +3,6 @@ "version": "1.0.0", "exports": "./index.mjs", "dependencies": { - "nitro-lib": "2.0.0" + "nitro-lib": "2.0.1" } } diff --git a/test/tests.ts b/test/tests.ts index 6cbc11cc63..508ae93ae1 100644 --- a/test/tests.ts +++ b/test/tests.ts @@ -228,10 +228,10 @@ export function testNitro( it("resolve module version conflicts", async () => { const { data } = await callHandler({ url: "/modules" }); expect(data).toMatchObject({ - depA: "2.0.1", + depA: "1.0.0", depB: "2.0.1", - depLib: "2.0.1", - subpathLib: "2.0.1", + depLib: "2.0.0", + subpathLib: "2.0.0", }); }); From 3ffbb78bce767633adbd678bc40d5a7ce733277d Mon Sep 17 00:00:00 2001 From: Mahdi Boomeri Date: Thu, 22 Dec 2022 12:52:55 +0330 Subject: [PATCH 19/34] remove un-used code --- src/rollup/plugins/externals.ts | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/rollup/plugins/externals.ts b/src/rollup/plugins/externals.ts index 457d7fe342..7e9f4e8250 100644 --- a/src/rollup/plugins/externals.ts +++ b/src/rollup/plugins/externals.ts @@ -302,13 +302,6 @@ export function externals(opts: NodeExternalsOptions): Plugin { // Make the package name unique if there are multiple versions if (excludeOptimization.has(pkgName)) { pkgFullName = `${pkgName}#${v2}`; - - // Add the version to previous dependency - const previousPkg = tracedPackages.get(pkgName); - if (previousPkg) { - tracedPackages.delete(pkgName); - tracedPackages.set(`${pkgName}#${v1}`, previousPkg); - } } } From 8522a7606c230f916b926ebdadbd05ddf9affa9f Mon Sep 17 00:00:00 2001 From: Mahdi Boomeri Date: Fri, 23 Dec 2022 01:40:10 +0330 Subject: [PATCH 20/34] fix the ordering issue --- src/rollup/plugins/externals.ts | 84 ++++++++++++++++++++------------- 1 file changed, 52 insertions(+), 32 deletions(-) diff --git a/src/rollup/plugins/externals.ts b/src/rollup/plugins/externals.ts index 7e9f4e8250..53bbe68258 100644 --- a/src/rollup/plugins/externals.ts +++ b/src/rollup/plugins/externals.ts @@ -254,59 +254,79 @@ export function externals(opts: NodeExternalsOptions): Plugin { const includeOptimization = new Set( opts.optimizeExternals?.include ?? [] ); + + // Get's every tracked version of a conflicting dependency + const getAllVersions = (pkgName: string) => { + return [...tracedPackages] + .filter((p) => p[0][0] === pkgName) + .map((p) => p[0][1]); + }; + const hasConflict = (pkgName: string, pkgVersion: string) => { + const allVersions = getAllVersions(pkgName); + const shouldOptimize = allVersions.filter( + (v1) => semver.parse(v1).major !== semver.parse(pkgVersion).major + ); + + return shouldOptimize.length > 0 && !includeOptimization.has(pkgName); + }; + for (const file of tracedFiles) { const { baseDir, pkgName } = parseNodeModulePath(file); if (!pkgName) { continue; } - let pkgFullName = pkgName; - let pkgDir = resolve(baseDir, pkgName); + const pkgDir = resolve(baseDir, pkgName); + const pkgVersion = await getPackageJson(pkgDir).then((r) => r.version); + + // Exclude duplicate packages with major version differance + if (hasConflict(pkgName, pkgVersion)) { + const log = `Multiple major versions of package \`${pkgName}\` are being externalized. Skipping optimization...`; + if (!ignoreLogs.has(log)) { + consola.info(log); + ignoreLogs.add(log); + } + excludeOptimization.add(pkgName); + } + + // Add to traced packages + tracedPackages.set([pkgName, pkgVersion], pkgDir); + } + + for (const file of tracedFiles) { + const { baseDir, pkgName } = parseNodeModulePath(file); + if (!pkgName) { + continue; + } + const pkgDir = resolve(baseDir, pkgName); - // Check for duplicate versions const existingPkgDir = tracedPackages.get(pkgName); if (existingPkgDir && existingPkgDir !== pkgDir) { const v1 = await getPackageJson(existingPkgDir).then( (r) => r.version ); const v2 = await getPackageJson(pkgDir).then((r) => r.version); - const isNewer = semver.gt(v2, v1); - - const [newerDir, olderDir] = isNewer - ? [pkgDir, existingPkgDir] - : [existingPkgDir, pkgDir]; - - // Warn about major version differences and exclude the package from being optimized const getMajor = (v: string) => v.split(".").find((s) => s !== "0"); - const shouldOptimize = - getMajor(v1) !== getMajor(v2) && !includeOptimization.has(pkgName); - - if (shouldOptimize) { - const log = `Multiple major versions of package \`${pkgName}\` are being externalized. Skipping optimization...`; - if (!ignoreLogs.has(log)) { - consola.info(log); - ignoreLogs.add(log); - } - excludeOptimization.add(pkgName); - } + // Try to map traced files from one package to another for minor/patch versions - if (!shouldOptimize && !excludeOptimization.has(pkgName)) { + if ( + getMajor(v1) === getMajor(v2) && + !excludeOptimization.has(pkgName) + ) { + const isNewer = semver.gt(v2, v1); + + const [newerDir, olderDir] = isNewer + ? [pkgDir, existingPkgDir] + : [existingPkgDir, pkgDir]; + tracedFiles = tracedFiles.map((f) => f.startsWith(olderDir + "/") ? f.replace(olderDir, newerDir) : f ); // Exclude older version files ignoreDirs.push(olderDir + "/"); - pkgDir = newerDir; // Update for tracedPackages - } - - // Make the package name unique if there are multiple versions - if (excludeOptimization.has(pkgName)) { - pkgFullName = `${pkgName}#${v2}`; + tracedPackages.delete([pkgName, v2]); // Remove the older package } } - - // Add to traced packages - tracedPackages.set(pkgFullName, pkgDir); } // Filter out files from ignored packages and dedup @@ -360,7 +380,7 @@ export function externals(opts: NodeExternalsOptions): Plugin { // Write an informative package.json const bundledDependencies = [ - ...new Set([...tracedPackages.keys()].map((p) => p.split("#")[0])), // Merge duplicated dependencies + ...new Set([...tracedPackages.keys()].map((p) => p[0])), // Dedup conflicting packages ]; await fsp.writeFile( resolve(opts.outDir, "package.json"), From 9712e20d7d5c856ea0c106e3f41988d34d7bcf24 Mon Sep 17 00:00:00 2001 From: Mahdi Boomeri Date: Fri, 23 Dec 2022 02:01:50 +0330 Subject: [PATCH 21/34] fix minor patching --- src/rollup/plugins/externals.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/rollup/plugins/externals.ts b/src/rollup/plugins/externals.ts index 53bbe68258..d809669325 100644 --- a/src/rollup/plugins/externals.ts +++ b/src/rollup/plugins/externals.ts @@ -298,8 +298,14 @@ export function externals(opts: NodeExternalsOptions): Plugin { continue; } const pkgDir = resolve(baseDir, pkgName); + const pkgVersion = await getPackageJson(resolve(baseDir, pkgName)).then( + (r) => r.version + ); - const existingPkgDir = tracedPackages.get(pkgName); + const existingPkgVersion = getAllVersions(pkgName).find( + (v) => v !== pkgVersion + ); + const existingPkgDir = tracedPackages.get([pkgName, existingPkgVersion]); if (existingPkgDir && existingPkgDir !== pkgDir) { const v1 = await getPackageJson(existingPkgDir).then( (r) => r.version From 7ef6e304e7ccd87ce57c4e582499529e57fbc6b1 Mon Sep 17 00:00:00 2001 From: Mahdi Boomeri Date: Fri, 23 Dec 2022 14:10:59 +0330 Subject: [PATCH 22/34] fix include option not picking the latest version --- src/rollup/plugins/externals.ts | 38 ++++++++++++++++----------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/src/rollup/plugins/externals.ts b/src/rollup/plugins/externals.ts index d809669325..2b14b40286 100644 --- a/src/rollup/plugins/externals.ts +++ b/src/rollup/plugins/externals.ts @@ -258,8 +258,8 @@ export function externals(opts: NodeExternalsOptions): Plugin { // Get's every tracked version of a conflicting dependency const getAllVersions = (pkgName: string) => { return [...tracedPackages] - .filter((p) => p[0][0] === pkgName) - .map((p) => p[0][1]); + .filter((p) => p[0].split("#")[0] === pkgName) + .map((p) => p[0].split("#")[1]); }; const hasConflict = (pkgName: string, pkgVersion: string) => { const allVersions = getAllVersions(pkgName); @@ -289,7 +289,7 @@ export function externals(opts: NodeExternalsOptions): Plugin { } // Add to traced packages - tracedPackages.set([pkgName, pkgVersion], pkgDir); + tracedPackages.set(`${pkgName}#${pkgVersion}`, pkgDir); } for (const file of tracedFiles) { @@ -305,33 +305,31 @@ export function externals(opts: NodeExternalsOptions): Plugin { const existingPkgVersion = getAllVersions(pkgName).find( (v) => v !== pkgVersion ); - const existingPkgDir = tracedPackages.get([pkgName, existingPkgVersion]); - if (existingPkgDir && existingPkgDir !== pkgDir) { + const existingPkgDir = tracedPackages.get( + `${pkgName}#${existingPkgVersion}` + ); + if (existingPkgDir && existingPkgDir !== pkgDir && !excludeOptimization.has(pkgName)) { const v1 = await getPackageJson(existingPkgDir).then( (r) => r.version ); const v2 = await getPackageJson(pkgDir).then((r) => r.version); const getMajor = (v: string) => v.split(".").find((s) => s !== "0"); - // Try to map traced files from one package to another for minor/patch versions - if ( - getMajor(v1) === getMajor(v2) && - !excludeOptimization.has(pkgName) - ) { - const isNewer = semver.gt(v2, v1); - - const [newerDir, olderDir] = isNewer - ? [pkgDir, existingPkgDir] - : [existingPkgDir, pkgDir]; + const isNewer = semver.gt(v2, v1); + const [newerDir, olderDir] = isNewer + ? [pkgDir, existingPkgDir] + : [existingPkgDir, pkgDir]; + // Try to map traced files from one package to another for minor/patch versions + if (getMajor(v1) === getMajor(v2)) { tracedFiles = tracedFiles.map((f) => f.startsWith(olderDir + "/") ? f.replace(olderDir, newerDir) : f ); - - // Exclude older version files - ignoreDirs.push(olderDir + "/"); - tracedPackages.delete([pkgName, v2]); // Remove the older package } + + // Exclude older version files + ignoreDirs.push(olderDir + "/"); + tracedPackages.delete(`${pkgName}#${isNewer ? v1 : v2}`); // Remove the older package } } @@ -386,7 +384,7 @@ export function externals(opts: NodeExternalsOptions): Plugin { // Write an informative package.json const bundledDependencies = [ - ...new Set([...tracedPackages.keys()].map((p) => p[0])), // Dedup conflicting packages + ...new Set([...tracedPackages.keys()].map((p) => p.split("#")[0])), // Dedup conflicting packages ]; await fsp.writeFile( resolve(opts.outDir, "package.json"), From 05fa04af2da1d73cc50b224c2fa21257d1bc5d94 Mon Sep 17 00:00:00 2001 From: Mahdi Boomeri Date: Fri, 23 Dec 2022 14:21:47 +0330 Subject: [PATCH 23/34] prettier --- src/rollup/plugins/externals.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/rollup/plugins/externals.ts b/src/rollup/plugins/externals.ts index 2b14b40286..fd2f26587c 100644 --- a/src/rollup/plugins/externals.ts +++ b/src/rollup/plugins/externals.ts @@ -308,7 +308,11 @@ export function externals(opts: NodeExternalsOptions): Plugin { const existingPkgDir = tracedPackages.get( `${pkgName}#${existingPkgVersion}` ); - if (existingPkgDir && existingPkgDir !== pkgDir && !excludeOptimization.has(pkgName)) { + if ( + existingPkgDir && + existingPkgDir !== pkgDir && + !excludeOptimization.has(pkgName) + ) { const v1 = await getPackageJson(existingPkgDir).then( (r) => r.version ); From 50e45fa94ba370b680efe375fabf6858afb16660 Mon Sep 17 00:00:00 2001 From: Mahdi Boomeri Date: Fri, 23 Dec 2022 14:23:03 +0330 Subject: [PATCH 24/34] better comment --- src/rollup/plugins/externals.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rollup/plugins/externals.ts b/src/rollup/plugins/externals.ts index fd2f26587c..962461c4e0 100644 --- a/src/rollup/plugins/externals.ts +++ b/src/rollup/plugins/externals.ts @@ -245,7 +245,7 @@ export function externals(opts: NodeExternalsOptions): Plugin { }; // Keep track of npm packages - const tracedPackages = new Map(); // name => pkgDir + const tracedPackages = new Map(); // name#version => pkgDir const ignoreDirs = []; const ignoreLogs = new Set(); const excludeOptimization = new Set( From df2093d4deceee1f965ebfad2efd424cf130f1c5 Mon Sep 17 00:00:00 2001 From: Pooya Parsa Date: Fri, 23 Dec 2022 17:55:05 +0100 Subject: [PATCH 25/34] feat!: rewrite external copy with multi version hoisting support --- src/rollup/plugins/externals.ts | 337 +++++++++++++++----------------- 1 file changed, 156 insertions(+), 181 deletions(-) diff --git a/src/rollup/plugins/externals.ts b/src/rollup/plugins/externals.ts index 962461c4e0..da344697d8 100644 --- a/src/rollup/plugins/externals.ts +++ b/src/rollup/plugins/externals.ts @@ -165,7 +165,7 @@ export function externals(opts: NodeExternalsOptions): Plugin { return; } - // Force trace paths + // Manually traced paths for (const pkgName of opts.traceInclude || []) { const path = await this.resolve(pkgName); if (path?.id) { @@ -173,19 +173,11 @@ export function externals(opts: NodeExternalsOptions): Plugin { } } - // Trace files - const fileTrace = await nodeFileTrace( + // Trace used files using nft + const _fileTrace = await nodeFileTrace( [...trackedExternals], opts.traceOptions ); - let tracedFiles = [...fileTrace.fileList] - .map((f) => resolve(opts.traceOptions.base, f)) - .filter((file) => file.includes("node_modules")); - - // Resolve symlinks - tracedFiles = await Promise.all( - tracedFiles.map((file) => fsp.realpath(file)) - ); // Read package.json with cache const packageJSONCache = new Map(); // pkgDir => contents @@ -200,196 +192,179 @@ export function externals(opts: NodeExternalsOptions): Plugin { return pkgJSON; }; - // Find parent base on file path - const getParent = async (pkgPath: string) => { - const { pkgName, baseDir } = parseNodeModulePath(pkgPath); - const pkgVersion = await getPackageJson(resolve(baseDir, pkgName)).then( - (r) => r.version - ); - - const possibleParents = [ - ...new Set( - [...fileTrace.reasons] - .filter((r) => !r[1].ignored) - .filter((r) => parseNodeModulePath(r[0]).pkgName === pkgName) - .flatMap((r) => - [...r[1].parents].filter( - (v) => parseNodeModulePath(v).pkgName !== pkgName - ) - ) // Remove self-refrencing - ), - ]; - - // Find the currect parent base on package.json dependency version - for (const possible of possibleParents) { - const thePath = resolve(opts.traceOptions.base, possible); - const { pkgName: existingPkgName, baseDir: existingBaseDir } = - parseNodeModulePath(await fsp.realpath(thePath)); - - const packageJson = await getPackageJson( - resolve(existingBaseDir, existingPkgName) - ); - const version = packageJson.dependencies[pkgName]; - - if (!version) { - return null; - } + // Resolve traced files + type TracedFile = { + path: string; + subpath: string; + parents: string[]; - const v1 = semver.parse(version.replace(/\^|~/, "")).version; - const v2 = semver.parse(pkgVersion).version; - if (v1 === v2) { - return existingPkgName; - } - } - return null; + pkgPath: string; + pkgName: string; + pkgVersion: string; }; + const _resolveTracedPath = (p) => + fsp.realpath(resolve(opts.traceOptions.base, p)); + const tracedFiles: Record = Object.fromEntries( + await Promise.all( + [..._fileTrace.reasons.entries()].map(async ([_path, reasons]) => { + if (reasons.ignored) { + return; + } + const path = await _resolveTracedPath(_path); + if (!path.includes("node_modules")) { + return; + } + if (!(await isFile(path))) { + return; + } + const { baseDir, pkgName, subpath } = parseNodeModulePath(path); + const pkgPath = join(baseDir, pkgName); + const parents = await Promise.all( + [...reasons.parents].map((p) => _resolveTracedPath(p)) + ); + const tracedFile = { + path, + parents, - // Keep track of npm packages - const tracedPackages = new Map(); // name#version => pkgDir - const ignoreDirs = []; - const ignoreLogs = new Set(); - const excludeOptimization = new Set( - opts.optimizeExternals?.exclude ?? [] - ); - const includeOptimization = new Set( - opts.optimizeExternals?.include ?? [] + subpath, + pkgName, + pkgPath, + }; + return [path, tracedFile]; + }) + ).then((r) => r.filter(Boolean)) ); - // Get's every tracked version of a conflicting dependency - const getAllVersions = (pkgName: string) => { - return [...tracedPackages] - .filter((p) => p[0].split("#")[0] === pkgName) - .map((p) => p[0].split("#")[1]); - }; - const hasConflict = (pkgName: string, pkgVersion: string) => { - const allVersions = getAllVersions(pkgName); - const shouldOptimize = allVersions.filter( - (v1) => semver.parse(v1).major !== semver.parse(pkgVersion).major - ); - - return shouldOptimize.length > 0 && !includeOptimization.has(pkgName); + // Resolve traced packages + type TracedPackage = { + name: string; + versions: Record< + string, + { + path: string; + files: string[]; + } + >; }; - - for (const file of tracedFiles) { - const { baseDir, pkgName } = parseNodeModulePath(file); - if (!pkgName) { - continue; - } - const pkgDir = resolve(baseDir, pkgName); - const pkgVersion = await getPackageJson(pkgDir).then((r) => r.version); - - // Exclude duplicate packages with major version differance - if (hasConflict(pkgName, pkgVersion)) { - const log = `Multiple major versions of package \`${pkgName}\` are being externalized. Skipping optimization...`; - if (!ignoreLogs.has(log)) { - consola.info(log); - ignoreLogs.add(log); + const tracedPackages: Record = {}; + await Promise.all( + Object.values(tracedFiles).map(async (tracedFile) => { + const pkgJSON = await getPackageJson(tracedFile.pkgPath); + let tracedPackage = tracedPackages[pkgJSON.name]; + if (!tracedPackage) { + tracedPackage = { + name: pkgJSON.name, + versions: {}, + }; + tracedPackages[pkgJSON.name] = tracedPackage; } - excludeOptimization.add(pkgName); - } - - // Add to traced packages - tracedPackages.set(`${pkgName}#${pkgVersion}`, pkgDir); - } - - for (const file of tracedFiles) { - const { baseDir, pkgName } = parseNodeModulePath(file); - if (!pkgName) { - continue; - } - const pkgDir = resolve(baseDir, pkgName); - const pkgVersion = await getPackageJson(resolve(baseDir, pkgName)).then( - (r) => r.version - ); - - const existingPkgVersion = getAllVersions(pkgName).find( - (v) => v !== pkgVersion - ); - const existingPkgDir = tracedPackages.get( - `${pkgName}#${existingPkgVersion}` - ); - if ( - existingPkgDir && - existingPkgDir !== pkgDir && - !excludeOptimization.has(pkgName) - ) { - const v1 = await getPackageJson(existingPkgDir).then( - (r) => r.version - ); - const v2 = await getPackageJson(pkgDir).then((r) => r.version); - const getMajor = (v: string) => v.split(".").find((s) => s !== "0"); - - const isNewer = semver.gt(v2, v1); - const [newerDir, olderDir] = isNewer - ? [pkgDir, existingPkgDir] - : [existingPkgDir, pkgDir]; - - // Try to map traced files from one package to another for minor/patch versions - if (getMajor(v1) === getMajor(v2)) { - tracedFiles = tracedFiles.map((f) => - f.startsWith(olderDir + "/") ? f.replace(olderDir, newerDir) : f - ); + let tracedPackageVersion = tracedPackage.versions[pkgJSON.version]; + if (!tracedPackageVersion) { + tracedPackageVersion = { path: tracedFile.pkgPath, files: [] }; + tracedPackage.versions[pkgJSON.version] = tracedPackageVersion; } - - // Exclude older version files - ignoreDirs.push(olderDir + "/"); - tracedPackages.delete(`${pkgName}#${isNewer ? v1 : v2}`); // Remove the older package - } - } - - // Filter out files from ignored packages and dedup - tracedFiles = tracedFiles.filter( - (f) => !ignoreDirs.some((d) => f.startsWith(d)) + tracedPackageVersion.files.push(tracedFile.path); + tracedFile.pkgName = pkgJSON.name; + tracedFile.pkgVersion = pkgJSON.version; + }) ); - tracedFiles = [...new Set(tracedFiles)]; - // Ensure all package.json files are traced - for (const pkgDir of tracedPackages.values()) { - const pkgJSON = join(pkgDir, "package.json"); - if (!tracedFiles.includes(pkgJSON)) { - tracedFiles.push(pkgJSON); + const writePackage = async ( + name: string, + version: string, + outputName?: string + ) => { + // Find pkg + const pkg = tracedPackages[name]; + + // Copy files + for (const src of pkg.versions[version].files) { + const { subpath } = parseNodeModulePath(src); + const dst = join( + opts.outDir, + "node_modules", + outputName || pkg.name, + subpath + ); + await fsp.mkdir(dirname(dst), { recursive: true }); + await fsp.copyFile(src, dst); } - } - const writeFile = async (file: string) => { - if (!(await isFile(file))) { - return; - } - const src = resolve(opts.traceOptions.base, file); - const { pkgName, subpath } = parseNodeModulePath(file); - let dst = resolve(opts.outDir, `node_modules/${pkgName + subpath}`); - - if (excludeOptimization.has(pkgName)) { - const parent = await getParent(file); - - if (parent) { - dst = resolve( - opts.outDir, - "node_modules", - parent, - "node_modules", - pkgName + subpath - ); - } - } + // Copy package.json + const pkgJSONPath = join( + opts.outDir, + "node_modules", + outputName || pkg.name, + "package.json" + ); + await fsp.mkdir(dirname(pkgJSONPath), { recursive: true }); + await fsp.copyFile( + join(pkg.versions[version].path, "package.json"), + pkgJSONPath + ); + }; + const linkPackage = async (from: string, to: string) => { + const src = join(opts.outDir, "node_modules", from); + const dst = join(opts.outDir, "node_modules", to); await fsp.mkdir(dirname(dst), { recursive: true }); - try { - await fsp.copyFile(src, dst); - } catch { - consola.warn(`Could not resolve \`${src}\`. Skipping.`); - } + // TODO: Windows workaround with junctions? + await fsp.symlink(src, dst); }; - // Write traced files + // Write traced packages await Promise.all( - tracedFiles.map((file) => retry(() => writeFile(file), 3)) + Object.values(tracedPackages).map(async (tracedPackage) => { + const versions = Object.keys(tracedPackage.versions); // TODO: sort by semver + if (versions.length === 1) { + // Write the only version into node_modules/{name} + await writePackage(tracedPackage.name, versions[0]); + } else { + for (const version of versions) { + // Try to find parent packages + const versionFiles: TracedFile[] = tracedPackage.versions[ + version + ].files.map((path) => tracedFiles[path]); + const parentPkgs = [ + ...new Set( + versionFiles.flatMap((file) => + file.parents.flatMap( + (parentPath) => tracedFiles[parentPath].pkgName + ) + ) + ), + ]; + + if (parentPkgs.length === 0) { + // No parent packages, assume as the hoisted version + await writePackage(tracedPackage.name, version); + } else { + // Write alternative version into node_modules/{name}@{version} + await writePackage( + tracedPackage.name, + version, + `${tracedPackage.name}@${version}` + ); + // For each parent, link into node_modules/{parent}/node_modules/{name} + for (const parentPath of parentPkgs) { + await linkPackage( + `${tracedPackage.name}@${version}`, + `${parentPath}/node_modules/${tracedPackage.name}` + ); + } + } + } + } + }) ); // Write an informative package.json - const bundledDependencies = [ - ...new Set([...tracedPackages.keys()].map((p) => p.split("#")[0])), // Dedup conflicting packages - ]; + const bundledDependencies = Object.fromEntries( + Object.values(tracedPackages).map((pkg) => [ + pkg.name, + Object.keys(pkg.versions).join(" | "), + ]) + ); await fsp.writeFile( resolve(opts.outDir, "package.json"), JSON.stringify( From a1643c9c5eae9b3cf464e84c027609b108794bbf Mon Sep 17 00:00:00 2001 From: Pooya Parsa Date: Fri, 23 Dec 2022 18:07:28 +0100 Subject: [PATCH 26/34] remove new option --- docs/content/3.config/index.md | 37 ++++++--------------------------- src/rollup/config.ts | 1 - src/rollup/plugins/externals.ts | 4 ---- 3 files changed, 6 insertions(+), 36 deletions(-) diff --git a/docs/content/3.config/index.md b/docs/content/3.config/index.md index 6962a8b201..3898f1f767 100644 --- a/docs/content/3.config/index.md +++ b/docs/content/3.config/index.md @@ -1,17 +1,16 @@ --- title: Configuration aside: false -description: 'Customize your Nitro app with a configuration file!' +description: "Customize your Nitro app with a configuration file!" --- In order to customize nitro's behavior, we create a file named `nitro.config.ts`. ```js // nitro.config.ts -import { defineNitroConfig } from 'nitropack' +import { defineNitroConfig } from "nitropack"; -export default defineNitroConfig({ -}) +export default defineNitroConfig({}); ``` ## Config Reference @@ -40,7 +39,6 @@ Server runtime configuration. **Note:**: `nitro` namespace is reserved. - ## `experimental` @@ -171,8 +169,6 @@ An array of paths to nitro plugins. They will be executed by order on the first A map from dynamic virtual import names to their contents or an (async) function that returns it. - - ## `baseURL` @@ -219,11 +215,11 @@ Path to a custom runtime error handler. Replacing nitro's built-in error page. **Example:** ```js [nitro.config] -import { defineNitroConfig } from 'nitropack' +import { defineNitroConfig } from "nitropack"; export default defineNitroConfig({ - errorHandler: '~/error' -}) + errorHandler: "~/error", +}); ``` ```js [error.ts] @@ -242,7 +238,6 @@ Route options. It is a map from route pattern (following [unjs/radix3](https://g When `cache` option is set, handlers matching pattern will be automatically wrapped with `defineCachedEventHandler`. See [Cache API](/guide/introduction/cache) for all available cache options. (`swr: true|number` is shortcut for `cache: { swr: true, maxAge: number }`.) - **Example:** ```js @@ -267,8 +262,6 @@ Prerendered options. Any route specified will be fetched during the build and co If `crawlLinks` option is set to `true`, nitro starts with `/` by default (or all routes in `routes` array) and for HTML pages extracts `` tags and prerender them as well. - - ## `rootDir` @@ -297,8 +290,6 @@ nitro's temporary working directory for generating build-related files. Output directories for production bundle. - - ## `dev` @@ -335,7 +326,6 @@ Preview and deploy command hints are usually filled by deployment presets. A custom error handler function for development errors. - ## `rollupConfig` @@ -389,18 +379,3 @@ Rollup specific option. ## `commonJS` Rollup specific option. Specifies additional configuration for the rollup CommonJS plugin. - -## `optimizeExternals` - -**⚠️ Caution! This is an advanced configuration. things can go wrong if misconfigured.** - -Options to give you control over the optimization of externalized packages. - -```js -{ - optimizeExternals: { - include: [''], - exclude: [''], // Handled by default if version difference is major - } -} -``` diff --git a/src/rollup/config.ts b/src/rollup/config.ts index a9ec96fde1..3782f94d4b 100644 --- a/src/rollup/config.ts +++ b/src/rollup/config.ts @@ -336,7 +336,6 @@ export const plugins = [ "node", "import", ], - optimizeExternals: nitro.options.optimizeExternals, }) ) ); diff --git a/src/rollup/plugins/externals.ts b/src/rollup/plugins/externals.ts index da344697d8..62333a39f5 100644 --- a/src/rollup/plugins/externals.ts +++ b/src/rollup/plugins/externals.ts @@ -16,10 +16,6 @@ export interface NodeExternalsOptions { moduleDirectories?: string[]; exportConditions?: string[]; traceInclude?: string[]; - optimizeExternals?: { - include?: string[]; - exclude?: string[]; - }; } export function externals(opts: NodeExternalsOptions): Plugin { From b2453c8c0c1f09ca027cf9fcdff971ea39c8bd7d Mon Sep 17 00:00:00 2001 From: Pooya Parsa Date: Fri, 23 Dec 2022 18:08:40 +0100 Subject: [PATCH 27/34] fix main merge issues --- docs/content/3.config/index.md | 22 ++++++++++++++++------ src/types/nitro.ts | 4 ---- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/docs/content/3.config/index.md b/docs/content/3.config/index.md index 3898f1f767..295ecd612c 100644 --- a/docs/content/3.config/index.md +++ b/docs/content/3.config/index.md @@ -1,16 +1,17 @@ --- title: Configuration aside: false -description: "Customize your Nitro app with a configuration file!" +description: 'Customize your Nitro app with a configuration file!' --- In order to customize nitro's behavior, we create a file named `nitro.config.ts`. ```js // nitro.config.ts -import { defineNitroConfig } from "nitropack"; +import { defineNitroConfig } from 'nitropack' -export default defineNitroConfig({}); +export default defineNitroConfig({ +}) ``` ## Config Reference @@ -39,6 +40,7 @@ Server runtime configuration. **Note:**: `nitro` namespace is reserved. + ## `experimental` @@ -169,6 +171,8 @@ An array of paths to nitro plugins. They will be executed by order on the first A map from dynamic virtual import names to their contents or an (async) function that returns it. + + ## `baseURL` @@ -215,11 +219,11 @@ Path to a custom runtime error handler. Replacing nitro's built-in error page. **Example:** ```js [nitro.config] -import { defineNitroConfig } from "nitropack"; +import { defineNitroConfig } from 'nitropack' export default defineNitroConfig({ - errorHandler: "~/error", -}); + errorHandler: '~/error' +}) ``` ```js [error.ts] @@ -238,6 +242,7 @@ Route options. It is a map from route pattern (following [unjs/radix3](https://g When `cache` option is set, handlers matching pattern will be automatically wrapped with `defineCachedEventHandler`. See [Cache API](/guide/introduction/cache) for all available cache options. (`swr: true|number` is shortcut for `cache: { swr: true, maxAge: number }`.) + **Example:** ```js @@ -262,6 +267,8 @@ Prerendered options. Any route specified will be fetched during the build and co If `crawlLinks` option is set to `true`, nitro starts with `/` by default (or all routes in `routes` array) and for HTML pages extracts `` tags and prerender them as well. + + ## `rootDir` @@ -290,6 +297,8 @@ nitro's temporary working directory for generating build-related files. Output directories for production bundle. + + ## `dev` @@ -326,6 +335,7 @@ Preview and deploy command hints are usually filled by deployment presets. A custom error handler function for development errors. + ## `rollupConfig` diff --git a/src/types/nitro.ts b/src/types/nitro.ts index 1b8038fdfb..1fdfc1b80b 100644 --- a/src/types/nitro.ts +++ b/src/types/nitro.ts @@ -217,10 +217,6 @@ export interface NitroOptions extends PresetOptions { }; noExternals: boolean; externals: NodeExternalsOptions; - optimizeExternals?: { - include?: string[]; - exclude?: string[]; - }; analyze: false | PluginVisualizerOptions; replace: Record string)>; commonJS?: RollupCommonJSOptions; From 76278e79d391cd83d43aa634b2c5a6f2122d5bdb Mon Sep 17 00:00:00 2001 From: Pooya Parsa Date: Fri, 13 Jan 2023 21:46:51 +0100 Subject: [PATCH 28/34] add workaround for making windows working (not portable) --- src/rollup/plugins/externals.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rollup/plugins/externals.ts b/src/rollup/plugins/externals.ts index 62333a39f5..2e8d26b368 100644 --- a/src/rollup/plugins/externals.ts +++ b/src/rollup/plugins/externals.ts @@ -304,8 +304,8 @@ export function externals(opts: NodeExternalsOptions): Plugin { const src = join(opts.outDir, "node_modules", from); const dst = join(opts.outDir, "node_modules", to); await fsp.mkdir(dirname(dst), { recursive: true }); - // TODO: Windows workaround with junctions? - await fsp.symlink(src, dst); + // TODO: Warn for windows output which is not portable with junctions or find another solution + await fsp.symlink(src, dst, "junction"); }; // Write traced packages From 6fc167706c2a9e4a987f68e76d808d5085002f80 Mon Sep 17 00:00:00 2001 From: Pooya Parsa Date: Mon, 16 Jan 2023 13:01:06 +0100 Subject: [PATCH 29/34] remove unused imports --- src/rollup/plugins/externals.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/rollup/plugins/externals.ts b/src/rollup/plugins/externals.ts index 1c11c54fb3..cd230a163b 100644 --- a/src/rollup/plugins/externals.ts +++ b/src/rollup/plugins/externals.ts @@ -1,10 +1,8 @@ import { existsSync, promises as fsp } from "node:fs"; import { resolve, dirname, normalize, join, isAbsolute } from "pathe"; -import consola from "consola"; import { nodeFileTrace, NodeFileTraceOptions } from "@vercel/nft"; import type { Plugin } from "rollup"; import { resolvePath, isValidNodeImport, normalizeid } from "mlly"; -import semver from "semver"; import { isDirectory, retry } from "../../utils"; export interface NodeExternalsOptions { From e4c3d29dd0ee51009e7affda33cde78d1fc719ce Mon Sep 17 00:00:00 2001 From: Pooya Parsa Date: Mon, 16 Jan 2023 13:03:18 +0100 Subject: [PATCH 30/34] keep supporting legacy externals --- src/rollup/config.ts | 6 +- src/rollup/plugins/externals-legacy.ts | 338 +++++++++++++++++++++++++ src/types/nitro.ts | 1 + 3 files changed, 344 insertions(+), 1 deletion(-) create mode 100644 src/rollup/plugins/externals-legacy.ts diff --git a/src/rollup/config.ts b/src/rollup/config.ts index 3782f94d4b..4b29b4e854 100644 --- a/src/rollup/config.ts +++ b/src/rollup/config.ts @@ -25,6 +25,7 @@ import { replace } from "./plugins/replace"; import { virtual } from "./plugins/virtual"; import { dynamicRequire } from "./plugins/dynamic-require"; import { externals } from "./plugins/externals"; +import { externals as legacyExternals } from "./plugins/externals-legacy"; import { timing } from "./plugins/timing"; import { publicAssets } from "./plugins/public-assets"; import { serverAssets } from "./plugins/server-assets"; @@ -305,8 +306,11 @@ export const plugins = [ // Externals Plugin if (!nitro.options.noExternals) { + let externalsPlugin = nitro.options.experimental.legacyExternals + ? legacyExternals + : externals; rollupConfig.plugins.push( - externals( + externalsPlugin( defu(nitro.options.externals, { outDir: nitro.options.output.serverDir, moduleDirectories: nitro.options.nodeModulesDirs, diff --git a/src/rollup/plugins/externals-legacy.ts b/src/rollup/plugins/externals-legacy.ts new file mode 100644 index 0000000000..6b1c57e8dd --- /dev/null +++ b/src/rollup/plugins/externals-legacy.ts @@ -0,0 +1,338 @@ +import { existsSync, promises as fsp } from "node:fs"; +import { resolve, dirname, normalize, join, isAbsolute } from "pathe"; +import consola from "consola"; +import { nodeFileTrace, NodeFileTraceOptions } from "@vercel/nft"; +import type { Plugin } from "rollup"; +import { resolvePath, isValidNodeImport, normalizeid } from "mlly"; +import semver from "semver"; +import { isDirectory, retry } from "../../utils"; + +export interface NodeExternalsOptions { + inline?: string[]; + external?: string[]; + outDir?: string; + trace?: boolean; + traceOptions?: NodeFileTraceOptions; + moduleDirectories?: string[]; + exportConditions?: string[]; + traceInclude?: string[]; +} + +export function externals(opts: NodeExternalsOptions): Plugin { + const trackedExternals = new Set(); + + const _resolveCache = new Map(); + const _resolve = async (id: string) => { + let resolved = _resolveCache.get(id); + if (resolved) { + return resolved; + } + resolved = await resolvePath(id, { + conditions: opts.exportConditions, + url: opts.moduleDirectories, + }); + _resolveCache.set(id, resolved); + return resolved; + }; + + // Normalize options + opts.inline = (opts.inline || []).map((p) => normalize(p)); + opts.external = (opts.external || []).map((p) => normalize(p)); + + return { + name: "node-externals", + async resolveId(originalId, importer, options) { + // Skip internals + if ( + !originalId || + originalId.startsWith("\u0000") || + originalId.includes("?") || + originalId.startsWith("#") + ) { + return null; + } + + // Skip relative paths + if (originalId.startsWith(".")) { + return null; + } + + // Normalize path (windows) + const id = normalize(originalId); + + // Id without .../node_modules/ + const idWithoutNodeModules = id.split("node_modules/").pop(); + + // Check for explicit inlines + if ( + opts.inline.some( + (i) => id.startsWith(i) || idWithoutNodeModules.startsWith(i) + ) + ) { + return null; + } + + // Check for explicit externals + if ( + opts.external.some( + (i) => id.startsWith(i) || idWithoutNodeModules.startsWith(i) + ) + ) { + return { id, external: true }; + } + + // Resolve id using rollup resolver + const resolved = (await this.resolve(originalId, importer, { + ...options, + skipSelf: true, + })) || { id }; + + // Try resolving with mlly as fallback + if ( + !isAbsolute(resolved.id) || + !existsSync(resolved.id) || + (await isDirectory(resolved.id)) + ) { + resolved.id = await _resolve(resolved.id).catch(() => resolved.id); + } + + // Inline invalid node imports + if (!(await isValidNodeImport(resolved.id).catch(() => false))) { + return null; + } + + // Externalize with full path if trace is disabled + if (opts.trace === false) { + return { + ...resolved, + id: isAbsolute(resolved.id) ? normalizeid(resolved.id) : resolved.id, + external: true, + }; + } + + // -- Trace externals -- + + // Try to extract package name from path + const { pkgName, subpath } = parseNodeModulePath(resolved.id); + + // Inline if cannot detect package name + if (!pkgName) { + return null; + } + + // Normally package name should be same as originalId + // Edge cases: Subpath export and full paths + if (pkgName !== originalId) { + // Subpath export + if (!isAbsolute(originalId)) { + const fullPath = await _resolve(originalId); + trackedExternals.add(fullPath); + return { + id: originalId, + external: true, + }; + } + + // Absolute path, we are not sure about subpath to generate import statement + // Guess as main subpath export + const packageEntry = await _resolve(pkgName).catch(() => null); + if (packageEntry !== originalId) { + // Guess subpathexport + const guessedSubpath = pkgName + subpath.replace(/\.[a-z]+$/, ""); + const resolvedGuess = await _resolve(guessedSubpath).catch( + () => null + ); + if (resolvedGuess === originalId) { + trackedExternals.add(resolvedGuess); + return { + id: guessedSubpath, + external: true, + }; + } + // Inline since we cannot guess subpath + return null; + } + } + + trackedExternals.add(resolved.id); + return { + id: pkgName, + external: true, + }; + }, + async buildEnd() { + if (opts.trace === false) { + return; + } + + // Force trace paths + for (const pkgName of opts.traceInclude || []) { + const path = await this.resolve(pkgName); + if (path?.id) { + trackedExternals.add(path.id.replace(/\?.+/, "")); + } + } + + // Trace files + let tracedFiles = await nodeFileTrace( + [...trackedExternals], + opts.traceOptions + ) + .then((r) => + [...r.fileList].map((f) => resolve(opts.traceOptions.base, f)) + ) + .then((r) => r.filter((file) => file.includes("node_modules"))); + + // Resolve symlinks + tracedFiles = await Promise.all( + tracedFiles.map((file) => fsp.realpath(file)) + ); + + // Read package.json with cache + const packageJSONCache = new Map(); // pkgDir => contents + const getPackageJson = async (pkgDir: string) => { + if (packageJSONCache.has(pkgDir)) { + return packageJSONCache.get(pkgDir); + } + const pkgJSON = JSON.parse( + await fsp.readFile(resolve(pkgDir, "package.json"), "utf8") + ); + packageJSONCache.set(pkgDir, pkgJSON); + return pkgJSON; + }; + + // Keep track of npm packages + const tracedPackages = new Map(); // name => pkgDir + const ignoreDirs = []; + const ignoreWarns = new Set(); + for (const file of tracedFiles) { + const { baseDir, pkgName } = parseNodeModulePath(file); + if (!pkgName) { + continue; + } + let pkgDir = resolve(baseDir, pkgName); + + // Check for duplicate versions + const existingPkgDir = tracedPackages.get(pkgName); + if (existingPkgDir && existingPkgDir !== pkgDir) { + const v1 = await getPackageJson(existingPkgDir).then( + (r) => r.version + ); + const v2 = await getPackageJson(pkgDir).then((r) => r.version); + const isNewer = semver.gt(v2, v1); + + // Warn about major version differences + const getMajor = (v: string) => v.split(".").find((s) => s !== "0"); + if (getMajor(v1) !== getMajor(v2)) { + const warn = + `Multiple major versions of package \`${pkgName}\` are being externalized. Picking latest version:\n\n` + + [ + ` ${isNewer ? "-" : "+"} ` + existingPkgDir + "@" + v1, + ` ${isNewer ? "+" : "-"} ` + pkgDir + "@" + v2, + ].join("\n"); + if (!ignoreWarns.has(warn)) { + consola.warn(warn); + ignoreWarns.add(warn); + } + } + + const [newerDir, olderDir] = isNewer + ? [pkgDir, existingPkgDir] + : [existingPkgDir, pkgDir]; + // Try to map traced files from one package to another for minor/patch versions + if (getMajor(v1) === getMajor(v2)) { + tracedFiles = tracedFiles.map((f) => + f.startsWith(olderDir + "/") ? f.replace(olderDir, newerDir) : f + ); + } + // Exclude older version files + ignoreDirs.push(olderDir + "/"); + pkgDir = newerDir; // Update for tracedPackages + } + + // Add to traced packages + tracedPackages.set(pkgName, pkgDir); + } + + // Filter out files from ignored packages and dedup + tracedFiles = tracedFiles.filter( + (f) => !ignoreDirs.some((d) => f.startsWith(d)) + ); + tracedFiles = [...new Set(tracedFiles)]; + + // Ensure all package.json files are traced + for (const pkgDir of tracedPackages.values()) { + const pkgJSON = join(pkgDir, "package.json"); + if (!tracedFiles.includes(pkgJSON)) { + tracedFiles.push(pkgJSON); + } + } + + const writeFile = async (file: string) => { + if (!(await isFile(file))) { + return; + } + const src = resolve(opts.traceOptions.base, file); + const { pkgName, subpath } = parseNodeModulePath(file); + const dst = resolve(opts.outDir, `node_modules/${pkgName + subpath}`); + await fsp.mkdir(dirname(dst), { recursive: true }); + try { + await fsp.copyFile(src, dst); + } catch { + consola.warn(`Could not resolve \`${src}\`. Skipping.`); + } + }; + + // Write traced files + await Promise.all( + tracedFiles.map((file) => retry(() => writeFile(file), 3)) + ); + + // Write an informative package.json + await fsp.writeFile( + resolve(opts.outDir, "package.json"), + JSON.stringify( + { + name: "nitro-output", + version: "0.0.0", + private: true, + bundledDependencies: [...tracedPackages.keys()], + }, + null, + 2 + ), + "utf8" + ); + }, + }; +} + +function parseNodeModulePath(path: string) { + if (!path) { + return {}; + } + const match = /^(.+\/node_modules\/)([^/@]+|@[^/]+\/[^/]+)(\/?.*?)?$/.exec( + normalize(path) + ); + if (!match) { + return {}; + } + const [, baseDir, pkgName, subpath] = match; + return { + baseDir, + pkgName, + subpath, + }; +} + +async function isFile(file: string) { + try { + const stat = await fsp.stat(file); + return stat.isFile(); + } catch (err) { + if (err.code === "ENOENT") { + return false; + } + throw err; + } +} diff --git a/src/types/nitro.ts b/src/types/nitro.ts index 1fdfc1b80b..70f27e1861 100644 --- a/src/types/nitro.ts +++ b/src/types/nitro.ts @@ -174,6 +174,7 @@ export interface NitroOptions extends PresetOptions { noPublicDir: boolean; experimental?: { wasm?: boolean | RollupWasmOptions; + legacyExternals?: boolean; }; serverAssets: ServerAssetDir[]; publicAssets: PublicAssetDir[]; From f1c9ce474223158fbba84649dc855aa978694071 Mon Sep 17 00:00:00 2001 From: Pooya Parsa Date: Mon, 16 Jan 2023 13:05:02 +0100 Subject: [PATCH 31/34] fix lint --- src/rollup/config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rollup/config.ts b/src/rollup/config.ts index 4b29b4e854..61170d715a 100644 --- a/src/rollup/config.ts +++ b/src/rollup/config.ts @@ -306,7 +306,7 @@ export const plugins = [ // Externals Plugin if (!nitro.options.noExternals) { - let externalsPlugin = nitro.options.experimental.legacyExternals + const externalsPlugin = nitro.options.experimental.legacyExternals ? legacyExternals : externals; rollupConfig.plugins.push( From f47feb1bf2f6039363c5621a4d8b0fcaf6f00db9 Mon Sep 17 00:00:00 2001 From: Pooya Parsa Date: Mon, 16 Jan 2023 13:47:56 +0100 Subject: [PATCH 32/34] support nested deps + fixture --- src/rollup/plugins/externals.ts | 51 +++++++++++++------ .../node_modules/nitro-lib/index.mjs | 4 +- .../node_modules/nested-lib/index.mjs | 1 + .../node_modules/nested-lib/package.json | 5 ++ .../node_modules/nitro-lib/package.json | 5 +- .../node_modules/nitro-lib/index.mjs | 4 +- .../node_modules/nested-lib/index.mjs | 1 + .../node_modules/nested-lib/package.json | 5 ++ .../node_modules/nitro-lib/package.json | 3 ++ .../_/node_modules/nitro-lib/index.mjs | 4 +- .../node_modules/nested-lib/index.mjs | 1 + .../node_modules/nested-lib/package.json | 5 ++ .../_/node_modules/nitro-lib/subpath.mjs | 2 +- 13 files changed, 70 insertions(+), 21 deletions(-) create mode 100644 test/fixture/_/node_modules/nitro-dep-a/node_modules/nitro-lib/node_modules/nested-lib/index.mjs create mode 100644 test/fixture/_/node_modules/nitro-dep-a/node_modules/nitro-lib/node_modules/nested-lib/package.json create mode 100644 test/fixture/_/node_modules/nitro-dep-b/node_modules/nitro-lib/node_modules/nested-lib/index.mjs create mode 100644 test/fixture/_/node_modules/nitro-dep-b/node_modules/nitro-lib/node_modules/nested-lib/package.json create mode 100644 test/fixture/_/node_modules/nitro-lib/node_modules/nested-lib/index.mjs create mode 100644 test/fixture/_/node_modules/nitro-lib/node_modules/nested-lib/package.json diff --git a/src/rollup/plugins/externals.ts b/src/rollup/plugins/externals.ts index cd230a163b..4d6188197e 100644 --- a/src/rollup/plugins/externals.ts +++ b/src/rollup/plugins/externals.ts @@ -305,9 +305,35 @@ export function externals(opts: NodeExternalsOptions): Plugin { const linkPackage = async (from: string, to: string) => { const src = join(opts.outDir, "node_modules", from); const dst = join(opts.outDir, "node_modules", to); + if (existsSync(dst)) { + return; // TODO: Warn? + } await fsp.mkdir(dirname(dst), { recursive: true }); - // TODO: Warn for windows output which is not portable with junctions or find another solution - await fsp.symlink(src, dst, "junction"); + // TODO: Use copy for windows for portable output? + await fsp.symlink(src, dst, "junction").catch((err) => { + console.error("Cannot link", src, "to", dst, ":", err.message); + }); + }; + + // Utility to find package parents + const findPackageParents = (pkg: TracedPackage, version: string) => { + // Try to find parent packages + const versionFiles: TracedFile[] = pkg.versions[version].files.map( + (path) => tracedFiles[path] + ); + const parentPkgs = [ + ...new Set( + versionFiles.flatMap((file) => + file.parents.flatMap( + (parentPath) => + tracedFiles[parentPath].pkgName + + "@" + + tracedFiles[parentPath].pkgVersion + ) + ) + ), + ]; + return parentPkgs; }; // Write traced packages @@ -319,20 +345,7 @@ export function externals(opts: NodeExternalsOptions): Plugin { await writePackage(tracedPackage.name, versions[0]); } else { for (const version of versions) { - // Try to find parent packages - const versionFiles: TracedFile[] = tracedPackage.versions[ - version - ].files.map((path) => tracedFiles[path]); - const parentPkgs = [ - ...new Set( - versionFiles.flatMap((file) => - file.parents.flatMap( - (parentPath) => tracedFiles[parentPath].pkgName - ) - ) - ), - ]; - + const parentPkgs = findPackageParents(tracedPackage, version); if (parentPkgs.length === 0) { // No parent packages, assume as the hoisted version await writePackage(tracedPackage.name, version); @@ -349,6 +362,12 @@ export function externals(opts: NodeExternalsOptions): Plugin { `${tracedPackage.name}@${version}`, `${parentPath}/node_modules/${tracedPackage.name}` ); + await linkPackage( + `${tracedPackage.name}@${version}`, + `${parentPath.split("@")[0]}/node_modules/${ + tracedPackage.name + }` + ); } } } diff --git a/test/fixture/_/node_modules/nitro-dep-a/node_modules/nitro-lib/index.mjs b/test/fixture/_/node_modules/nitro-dep-a/node_modules/nitro-lib/index.mjs index 3d64381bce..0029f235ab 100644 --- a/test/fixture/_/node_modules/nitro-dep-a/node_modules/nitro-lib/index.mjs +++ b/test/fixture/_/node_modules/nitro-dep-a/node_modules/nitro-lib/index.mjs @@ -1 +1,3 @@ -export default '1.0.0'; +import nestedLib from 'nested-lib' + +export default 'nitro-lib@1.0.0+' + nestedLib diff --git a/test/fixture/_/node_modules/nitro-dep-a/node_modules/nitro-lib/node_modules/nested-lib/index.mjs b/test/fixture/_/node_modules/nitro-dep-a/node_modules/nitro-lib/node_modules/nested-lib/index.mjs new file mode 100644 index 0000000000..da8927e1e8 --- /dev/null +++ b/test/fixture/_/node_modules/nitro-dep-a/node_modules/nitro-lib/node_modules/nested-lib/index.mjs @@ -0,0 +1 @@ +export default 'nested-lib@1.0.0' diff --git a/test/fixture/_/node_modules/nitro-dep-a/node_modules/nitro-lib/node_modules/nested-lib/package.json b/test/fixture/_/node_modules/nitro-dep-a/node_modules/nitro-lib/node_modules/nested-lib/package.json new file mode 100644 index 0000000000..5993dae833 --- /dev/null +++ b/test/fixture/_/node_modules/nitro-dep-a/node_modules/nitro-lib/node_modules/nested-lib/package.json @@ -0,0 +1,5 @@ +{ + "name": "nested-lib", + "version": "1.0.0", + "exports": "./index.mjs" +} diff --git a/test/fixture/_/node_modules/nitro-dep-a/node_modules/nitro-lib/package.json b/test/fixture/_/node_modules/nitro-dep-a/node_modules/nitro-lib/package.json index dd674789ba..b8bd61c01d 100644 --- a/test/fixture/_/node_modules/nitro-dep-a/node_modules/nitro-lib/package.json +++ b/test/fixture/_/node_modules/nitro-dep-a/node_modules/nitro-lib/package.json @@ -1,5 +1,8 @@ { "name": "nitro-lib", "version": "1.0.0", - "exports": "./index.mjs" + "exports": "./index.mjs", + "dependencies": { + "nested-lib": "1.0.0" + } } diff --git a/test/fixture/_/node_modules/nitro-dep-b/node_modules/nitro-lib/index.mjs b/test/fixture/_/node_modules/nitro-dep-b/node_modules/nitro-lib/index.mjs index c9e08004a5..53aaa0be76 100644 --- a/test/fixture/_/node_modules/nitro-dep-b/node_modules/nitro-lib/index.mjs +++ b/test/fixture/_/node_modules/nitro-dep-b/node_modules/nitro-lib/index.mjs @@ -1 +1,3 @@ -export default '2.0.1'; +import nestedLib from 'nested-lib' + +export default 'nitro-lib@2.0.1+' + nestedLib diff --git a/test/fixture/_/node_modules/nitro-dep-b/node_modules/nitro-lib/node_modules/nested-lib/index.mjs b/test/fixture/_/node_modules/nitro-dep-b/node_modules/nitro-lib/node_modules/nested-lib/index.mjs new file mode 100644 index 0000000000..caace8b77b --- /dev/null +++ b/test/fixture/_/node_modules/nitro-dep-b/node_modules/nitro-lib/node_modules/nested-lib/index.mjs @@ -0,0 +1 @@ +export default 'nested-lib@2.0.1' diff --git a/test/fixture/_/node_modules/nitro-dep-b/node_modules/nitro-lib/node_modules/nested-lib/package.json b/test/fixture/_/node_modules/nitro-dep-b/node_modules/nitro-lib/node_modules/nested-lib/package.json new file mode 100644 index 0000000000..fbc00e77b8 --- /dev/null +++ b/test/fixture/_/node_modules/nitro-dep-b/node_modules/nitro-lib/node_modules/nested-lib/package.json @@ -0,0 +1,5 @@ +{ + "name": "nested-lib", + "version": "2.0.1", + "exports": "./index.mjs" +} diff --git a/test/fixture/_/node_modules/nitro-dep-b/node_modules/nitro-lib/package.json b/test/fixture/_/node_modules/nitro-dep-b/node_modules/nitro-lib/package.json index bfb99596eb..3396d6ee1d 100644 --- a/test/fixture/_/node_modules/nitro-dep-b/node_modules/nitro-lib/package.json +++ b/test/fixture/_/node_modules/nitro-dep-b/node_modules/nitro-lib/package.json @@ -4,5 +4,8 @@ "exports": { ".": "./index.mjs", "./subpath": "./subpath.mjs" + }, + "dependencies": { + "nested-lib": "2.0.0" } } diff --git a/test/fixture/_/node_modules/nitro-lib/index.mjs b/test/fixture/_/node_modules/nitro-lib/index.mjs index ff603d7009..5e07a96ab0 100644 --- a/test/fixture/_/node_modules/nitro-lib/index.mjs +++ b/test/fixture/_/node_modules/nitro-lib/index.mjs @@ -1 +1,3 @@ -export default '2.0.0'; +import nestedLib from 'nested-lib' + +export default 'nitro-lib@2.0.0+' + nestedLib diff --git a/test/fixture/_/node_modules/nitro-lib/node_modules/nested-lib/index.mjs b/test/fixture/_/node_modules/nitro-lib/node_modules/nested-lib/index.mjs new file mode 100644 index 0000000000..69cb6433e5 --- /dev/null +++ b/test/fixture/_/node_modules/nitro-lib/node_modules/nested-lib/index.mjs @@ -0,0 +1 @@ +export default 'nested-lib@2.0.0' diff --git a/test/fixture/_/node_modules/nitro-lib/node_modules/nested-lib/package.json b/test/fixture/_/node_modules/nitro-lib/node_modules/nested-lib/package.json new file mode 100644 index 0000000000..e8d4a98a0a --- /dev/null +++ b/test/fixture/_/node_modules/nitro-lib/node_modules/nested-lib/package.json @@ -0,0 +1,5 @@ +{ + "name": "nested-lib", + "version": "2.0.0", + "exports": "./index.mjs" +} diff --git a/test/fixture/_/node_modules/nitro-lib/subpath.mjs b/test/fixture/_/node_modules/nitro-lib/subpath.mjs index ff603d7009..c4ac86cb68 100644 --- a/test/fixture/_/node_modules/nitro-lib/subpath.mjs +++ b/test/fixture/_/node_modules/nitro-lib/subpath.mjs @@ -1 +1 @@ -export default '2.0.0'; +export default 'nitro-lib@2.0.0'; From d8d5fecf69fa1a3f568c882606eba9196316be11 Mon Sep 17 00:00:00 2001 From: Pooya Parsa Date: Mon, 16 Jan 2023 13:50:30 +0100 Subject: [PATCH 33/34] fix: support aliased packages --- src/rollup/plugins/externals.ts | 9 +++++---- test/fixture/_/node_modules/nitro-lib/package.json | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/rollup/plugins/externals.ts b/src/rollup/plugins/externals.ts index 4d6188197e..38f60de5a3 100644 --- a/src/rollup/plugins/externals.ts +++ b/src/rollup/plugins/externals.ts @@ -248,13 +248,14 @@ export function externals(opts: NodeExternalsOptions): Plugin { await Promise.all( Object.values(tracedFiles).map(async (tracedFile) => { const pkgJSON = await getPackageJson(tracedFile.pkgPath); - let tracedPackage = tracedPackages[pkgJSON.name]; + const pkgName = tracedFile.pkgName; // Use file path as name to support aliases + let tracedPackage = tracedPackages[pkgName]; if (!tracedPackage) { tracedPackage = { - name: pkgJSON.name, + name: pkgName, versions: {}, }; - tracedPackages[pkgJSON.name] = tracedPackage; + tracedPackages[pkgName] = tracedPackage; } let tracedPackageVersion = tracedPackage.versions[pkgJSON.version]; if (!tracedPackageVersion) { @@ -262,7 +263,7 @@ export function externals(opts: NodeExternalsOptions): Plugin { tracedPackage.versions[pkgJSON.version] = tracedPackageVersion; } tracedPackageVersion.files.push(tracedFile.path); - tracedFile.pkgName = pkgJSON.name; + tracedFile.pkgName = pkgName; tracedFile.pkgVersion = pkgJSON.version; }) ); diff --git a/test/fixture/_/node_modules/nitro-lib/package.json b/test/fixture/_/node_modules/nitro-lib/package.json index 81618a9dc9..3709569af3 100644 --- a/test/fixture/_/node_modules/nitro-lib/package.json +++ b/test/fixture/_/node_modules/nitro-lib/package.json @@ -1,5 +1,5 @@ { - "name": "nitro-lib", + "name": "nitro-lib-aliased-from-another-name", "version": "2.0.0", "exports": { ".": "./index.mjs", From 71641e02afe6654c6029c50bd42d19dd8f610a8c Mon Sep 17 00:00:00 2001 From: Pooya Parsa Date: Mon, 16 Jan 2023 13:51:51 +0100 Subject: [PATCH 34/34] update test --- test/tests.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/tests.ts b/test/tests.ts index de9a8838ce..aa3ffcf361 100644 --- a/test/tests.ts +++ b/test/tests.ts @@ -231,10 +231,10 @@ export function testNitro( it("resolve module version conflicts", async () => { const { data } = await callHandler({ url: "/modules" }); expect(data).toMatchObject({ - depA: "1.0.0", - depB: "2.0.1", - depLib: "2.0.0", - subpathLib: "2.0.0", + depA: "nitro-lib@1.0.0+nested-lib@1.0.0", + depB: "nitro-lib@2.0.1+nested-lib@2.0.1", + depLib: "nitro-lib@2.0.0+nested-lib@2.0.0", + subpathLib: "nitro-lib@2.0.0", }); });