From 1832d18d62bd0d99dc283afd3f23b6db49555d1c Mon Sep 17 00:00:00 2001 From: Marcin Szczepanski Date: Fri, 25 Aug 2023 14:00:03 +1000 Subject: [PATCH] Selective lazy compilation (#9166) Implement selective lazy compilation for `--lazy`, as well as `--lazy-exclude` --- packages/core/core/src/dumpGraphToGraphViz.js | 2 + .../core/src/requests/AssetGraphRequest.js | 54 ++++++- .../core/src/requests/BundleGraphRequest.js | 2 + packages/core/core/src/resolveOptions.js | 15 ++ packages/core/core/src/types.js | 2 + packages/core/core/test/test-utils.js | 2 + .../integration-tests/test/lazy-compile.js | 141 ++++++++++++++++++ packages/core/parcel/src/cli.js | 17 ++- packages/core/types/index.js | 2 + 9 files changed, 230 insertions(+), 7 deletions(-) diff --git a/packages/core/core/src/dumpGraphToGraphViz.js b/packages/core/core/src/dumpGraphToGraphViz.js index b509a61991b..9b7bcfffe4b 100644 --- a/packages/core/core/src/dumpGraphToGraphViz.js +++ b/packages/core/core/src/dumpGraphToGraphViz.js @@ -93,6 +93,7 @@ export default async function dumpGraphToGraphViz( if (node.value.isOptional) parts.push('optional'); if (node.value.specifierType === SpecifierType.url) parts.push('url'); if (node.hasDeferred) parts.push('deferred'); + if (node.deferred) parts.push('deferred'); if (node.excluded) parts.push('excluded'); if (parts.length) label += ' (' + parts.join(', ') + ')'; if (node.value.env) label += ` (${getEnvDescription(node.value.env)})`; @@ -171,6 +172,7 @@ export default async function dumpGraphToGraphViz( if (node.value.needsStableName) parts.push('stable name'); parts.push(node.value.name); parts.push('bb:' + (node.value.bundleBehavior ?? 'null')); + if (node.value.isPlaceholder) parts.push('placeholder'); if (parts.length) label += ' (' + parts.join(', ') + ')'; if (node.value.env) label += ` (${getEnvDescription(node.value.env)})`; } else if (node.type === 'request') { diff --git a/packages/core/core/src/requests/AssetGraphRequest.js b/packages/core/core/src/requests/AssetGraphRequest.js index 3372a649f48..4b809413a57 100644 --- a/packages/core/core/src/requests/AssetGraphRequest.js +++ b/packages/core/core/src/requests/AssetGraphRequest.js @@ -19,7 +19,7 @@ import type {Diagnostic} from '@parcel/diagnostic'; import invariant from 'assert'; import nullthrows from 'nullthrows'; -import {PromiseQueue, setEqual} from '@parcel/utils'; +import {PromiseQueue, setEqual, isGlobMatch} from '@parcel/utils'; import {hashString} from '@parcel/hash'; import ThrowableDiagnostic from '@parcel/diagnostic'; import {Priority} from '../types'; @@ -29,7 +29,7 @@ import createEntryRequest from './EntryRequest'; import createTargetRequest from './TargetRequest'; import createAssetRequest from './AssetRequest'; import createPathRequest from './PathRequest'; -import {type ProjectPath} from '../projectPath'; +import {type ProjectPath, fromProjectPathRelative} from '../projectPath'; import dumpGraphToGraphViz from '../dumpGraphToGraphViz'; import {propagateSymbols} from '../SymbolPropagation'; @@ -39,6 +39,8 @@ type AssetGraphRequestInput = {| optionsRef: SharedReference, name: string, shouldBuildLazily?: boolean, + lazyIncludes?: string[], + lazyExcludes?: string[], requestedAssetIds?: Set, |}; @@ -111,6 +113,8 @@ export class AssetGraphBuilder { name: string; cacheKey: string; shouldBuildLazily: boolean; + lazyIncludes: string[]; + lazyExcludes: string[]; requestedAssetIds: Set; isSingleChangeRebuild: boolean; assetGroupsWithRemovedParents: Set; @@ -127,6 +131,8 @@ export class AssetGraphBuilder { name, requestedAssetIds, shouldBuildLazily, + lazyIncludes, + lazyExcludes, } = input; let assetGraph = prevResult?.assetGraph ?? new AssetGraph(); assetGraph.safeToIncrementallyBundle = true; @@ -148,6 +154,8 @@ export class AssetGraphBuilder { this.name = name; this.requestedAssetIds = requestedAssetIds ?? new Set(); this.shouldBuildLazily = shouldBuildLazily ?? false; + this.lazyIncludes = lazyIncludes ?? []; + this.lazyExcludes = lazyExcludes ?? []; this.cacheKey = hashString( `${PARCEL_VERSION}${name}${JSON.stringify(entries) ?? ''}${options.mode}`, ); @@ -306,14 +314,43 @@ export class AssetGraphBuilder { let childNode = nullthrows(this.assetGraph.getNode(childNodeId)); if (node.type === 'asset' && childNode.type === 'dependency') { - if (this.requestedAssetIds.has(node.value.id)) { + // This logic will set `node.requested` to `true` if the node is in the list of requested asset ids + // (i.e. this is an entry of a (probably) placeholder bundle that wasn't previously requested) + // + // Otherwise, if this node either is explicitly not requested, or has had it's requested attribute deleted, + // it will determine whether this node is an "async child" - that is, is it a (probably) + // dynamic import(). If so, it will explicitly have it's `node.requested` set to `false` + // + // If it's not requested, but it's not an async child then it's `node.requested` is deleted (undefined) + + // by default with lazy compilation all nodes are lazy + let isNodeLazy = true; + + // For conditional lazy building - if this node matches the `lazyInclude` globs that means we want + // only those nodes to be treated as lazy - that means if this node does _NOT_ match that glob, then we + // also consider it not lazy (so it gets marked as requested). + if (this.lazyIncludes.length > 0) { + isNodeLazy = isGlobMatch( + fromProjectPathRelative(node.value.filePath), + this.lazyIncludes, + ); + } + // Excludes override includes, so a node is _not_ lazy if it is included in the exclude list. + if (this.lazyExcludes.length > 0 && isNodeLazy) { + isNodeLazy = !isGlobMatch( + fromProjectPathRelative(node.value.filePath), + this.lazyExcludes, + ); + } + + if (this.requestedAssetIds.has(node.value.id) || !isNodeLazy) { node.requested = true; } else if (!node.requested) { let isAsyncChild = this.assetGraph .getIncomingDependencies(node.value) .every(dep => dep.isEntry || dep.priority !== Priority.sync); if (isAsyncChild) { - node.requested = false; + node.requested = !isNodeLazy; } else { delete node.requested; } @@ -322,12 +359,21 @@ export class AssetGraphBuilder { let previouslyDeferred = childNode.deferred; childNode.deferred = node.requested === false; + // The child dependency node we're now evaluating should not be deferred if it's parent + // is explicitly not requested (requested = false, but not requested = undefined) + // + // if we weren't previously deferred but we are now, then this dependency node's parents should also + // be marked as deferred + // + // if we were previously deferred but we not longer are, then then all parents should no longer be + // deferred either if (!previouslyDeferred && childNode.deferred) { this.assetGraph.markParentsWithHasDeferred(childNodeId); } else if (previouslyDeferred && !childNode.deferred) { this.assetGraph.unmarkParentsWithHasDeferred(childNodeId); } + // We `shouldVisitChild` if the childNode is not deferred return !childNode.deferred; } } diff --git a/packages/core/core/src/requests/BundleGraphRequest.js b/packages/core/core/src/requests/BundleGraphRequest.js index 94e4ac32887..f7c137639f1 100644 --- a/packages/core/core/src/requests/BundleGraphRequest.js +++ b/packages/core/core/src/requests/BundleGraphRequest.js @@ -99,6 +99,8 @@ export default function createBundleGraphRequest( entries: options.entries, optionsRef, shouldBuildLazily: options.shouldBuildLazily, + lazyIncludes: options.lazyIncludes, + lazyExcludes: options.lazyExcludes, requestedAssetIds, }); let {assetGraph, changedAssets, assetRequests} = await api.runRequest( diff --git a/packages/core/core/src/resolveOptions.js b/packages/core/core/src/resolveOptions.js index fbae0e7e486..8d36b891bc3 100644 --- a/packages/core/core/src/resolveOptions.js +++ b/packages/core/core/src/resolveOptions.js @@ -104,6 +104,19 @@ export default async function resolveOptions( : undefined; let shouldBuildLazily = initialOptions.shouldBuildLazily ?? false; + let lazyIncludes = initialOptions.lazyIncludes ?? []; + if (lazyIncludes.length > 0 && !shouldBuildLazily) { + throw new Error( + 'Lazy includes can only be provided when lazy building is enabled', + ); + } + let lazyExcludes = initialOptions.lazyExcludes ?? []; + if (lazyExcludes.length > 0 && !shouldBuildLazily) { + throw new Error( + 'Lazy excludes can only be provided when lazy building is enabled', + ); + } + let shouldContentHash = initialOptions.shouldContentHash ?? initialOptions.mode === 'production'; if (shouldBuildLazily && shouldContentHash) { @@ -140,6 +153,8 @@ export default async function resolveOptions( shouldAutoInstall: initialOptions.shouldAutoInstall ?? false, hmrOptions: initialOptions.hmrOptions ?? null, shouldBuildLazily, + lazyIncludes, + lazyExcludes, shouldBundleIncrementally: initialOptions.shouldBundleIncrementally ?? true, shouldContentHash, serveOptions: initialOptions.serveOptions diff --git a/packages/core/core/src/types.js b/packages/core/core/src/types.js index f199d323f7d..f20b167e24c 100644 --- a/packages/core/core/src/types.js +++ b/packages/core/core/src/types.js @@ -270,6 +270,8 @@ export type ParcelOptions = {| shouldContentHash: boolean, serveOptions: ServerOptions | false, shouldBuildLazily: boolean, + lazyIncludes: string[], + lazyExcludes: string[], shouldBundleIncrementally: boolean, shouldAutoInstall: boolean, logLevel: LogLevel, diff --git a/packages/core/core/test/test-utils.js b/packages/core/core/test/test-utils.js index a02a1175b93..6ed4367e94d 100644 --- a/packages/core/core/test/test-utils.js +++ b/packages/core/core/test/test-utils.js @@ -25,6 +25,8 @@ export const DEFAULT_OPTIONS: ParcelOptions = { hmrOptions: undefined, shouldContentHash: true, shouldBuildLazily: false, + lazyIncludes: [], + lazyExcludes: [], shouldBundleIncrementally: true, serveOptions: false, mode: 'development', diff --git a/packages/core/integration-tests/test/lazy-compile.js b/packages/core/integration-tests/test/lazy-compile.js index 6e29ead1c27..ac431fb91f6 100644 --- a/packages/core/integration-tests/test/lazy-compile.js +++ b/packages/core/integration-tests/test/lazy-compile.js @@ -120,4 +120,145 @@ describe('lazy compile', function () { ]); subscription.unsubscribe(); }); + + it('should support includes for lazy compile', async () => { + const b = await bundler( + path.join(__dirname, '/integration/lazy-compile/index.js'), + { + shouldBuildLazily: true, + lazyIncludes: ['**/lazy-1*'], + mode: 'development', + shouldContentHash: false, + }, + ); + + await removeDistDirectory(); + + const subscription = await b.watch(); + let result = await getNextBuild(b); + + // Expect the bundle graph to only contain `parallel-lazy-1` but not `lazy-1`s children as we're only including lazy-1 in lazy compilation + // `parallel-lazy-1` which wasn't requested. + assertBundles(result.bundleGraph, [ + { + name: /^index.*/, + assets: ['index.js', 'bundle-url.js', 'cacheLoader.js', 'js-loader.js'], + }, + { + // This will be a placeholder, but that info isn't available in the BundleGraph + assets: ['lazy-1.js'], + }, + { + assets: ['parallel-lazy-1.js', 'esmodule-helpers.js'], + }, + { + assets: ['parallel-lazy-2.js'], + }, + ]); + + // ensure parallel-lazy was produced, as it isn't "included" in laziness.. + assert( + await distDirIncludes([ + 'index.js', + /^parallel-lazy-1\./, + /^parallel-lazy-2\./, + ]), + ); + + result = await result.requestBundle( + findBundle(result.bundleGraph, /lazy-1/), + ); + + // Since lazy-2 was not included it should've been built when lazy-1 was.. + assert( + await distDirIncludes([ + 'index.js', + /^parallel-lazy-1\./, + /^parallel-lazy-2\./, + /^lazy-1\./, + /^lazy-2\./, + ]), + ); + + subscription.unsubscribe(); + }); + + it('should support excludes for lazy compile', async () => { + const b = await bundler( + path.join(__dirname, '/integration/lazy-compile/index.js'), + { + shouldBuildLazily: true, + lazyExcludes: ['**/lazy-*'], + mode: 'development', + shouldContentHash: false, + }, + ); + + await removeDistDirectory(); + + const subscription = await b.watch(); + let result = await getNextBuild(b); + + result = await result.requestBundle( + findBundle(result.bundleGraph, /index.js/), + ); + + assertBundles(result.bundleGraph, [ + { + name: /^index.*/, + assets: ['index.js', 'bundle-url.js', 'cacheLoader.js', 'js-loader.js'], + }, + { + assets: ['lazy-1.js', 'esmodule-helpers.js'], + }, + { + assets: ['lazy-2.js'], + }, + { + // This will be a placeholder, but that info isn't available in the BundleGraph + assets: ['parallel-lazy-1.js'], + }, + ]); + + // lazy-* is _excluded_ from lazy compilation so it should have been built, but parallel-lazy should not have + + assert(await distDirIncludes(['index.js', /^lazy-1\./, /^lazy-2\./])); + + subscription.unsubscribe(); + }); + + it('should lazy compile properly when same module is used sync/async', async () => { + const b = await bundler( + path.join(__dirname, '/integration/lazy-compile/index-sync-async.js'), + { + shouldBuildLazily: true, + mode: 'development', + shouldContentHash: false, + }, + ); + + await removeDistDirectory(); + + const subscription = await b.watch(); + let result = await getNextBuild(b); + result = await result.requestBundle( + findBundle(result.bundleGraph, /^index-sync-async\./), + ); + result = await result.requestBundle( + findBundle(result.bundleGraph, /^uses-static-component\./), + ); + result = await result.requestBundle( + findBundle(result.bundleGraph, /^uses-static-component-async\./), + ); + result = await result.requestBundle( + findBundle(result.bundleGraph, /^static-component\./), + ); + + let output = await run(result.bundleGraph); + assert.deepEqual(await output.default(), [ + 'static component', + 'static component', + ]); + subscription.unsubscribe(); + }); }); diff --git a/packages/core/parcel/src/cli.js b/packages/core/parcel/src/cli.js index bdb738286c9..56c0ac72e18 100755 --- a/packages/core/parcel/src/cli.js +++ b/packages/core/parcel/src/cli.js @@ -139,8 +139,12 @@ let serve = program ) .option('--watch-for-stdin', 'exit when stdin closes') .option( - '--lazy', - 'Build async bundles on demand, when requested in the browser', + '--lazy [includes]', + 'Build async bundles on demand, when requested in the browser. Defaults to all async bundles, unless a comma separated list of source file globs is provided. Only async bundles whose entry points match these globs will be built lazily', + ) + .option( + '--lazy-exclude ', + 'Can only be used in combination with --lazy. Comma separated list of source file globs, async bundles whose entry points match these globs will not be built lazily', ) .action(runCommand); @@ -470,6 +474,11 @@ async function normalizeOptions( } let mode = command.name() === 'build' ? 'production' : 'development'; + + const normalizeIncludeExcludeList = (input?: string): string[] => { + if (typeof input !== 'string') return []; + return input.split(',').map(value => value.trim()); + }; return { shouldDisableCache: command.cache === false, cacheDir: command.cacheDir, @@ -483,7 +492,9 @@ async function normalizeOptions( logLevel: command.logLevel, shouldProfile: command.profile, shouldTrace: command.trace, - shouldBuildLazily: command.lazy, + shouldBuildLazily: typeof command.lazy !== 'undefined', + lazyIncludes: normalizeIncludeExcludeList(command.lazy), + lazyExcludes: normalizeIncludeExcludeList(command.lazyExclude), shouldBundleIncrementally: process.env.PARCEL_INCREMENTAL_BUNDLING === 'false' ? false : true, detailedReport: diff --git a/packages/core/types/index.js b/packages/core/types/index.js index 3c33c674a73..7a496997678 100644 --- a/packages/core/types/index.js +++ b/packages/core/types/index.js @@ -303,6 +303,8 @@ export type InitialParcelOptions = {| +shouldTrace?: boolean, +shouldPatchConsole?: boolean, +shouldBuildLazily?: boolean, + +lazyIncludes?: string[], + +lazyExcludes?: string[], +shouldBundleIncrementally?: boolean, +inputFS?: FileSystem,