Selective lazy compilation (#9166)

Implement selective lazy compilation for `--lazy`, as well as `--lazy-exclude`

packages/core/core/src/dumpGraphToGraphViz.js

@@ -93,6 +93,7 @@ export default async function dumpGraphToGraphViz(
         if (node.value.isOptional) parts.push('optional');
         if (node.value.specifierType === SpecifierType.url) parts.push('url');
         if (node.hasDeferred) parts.push('deferred');
+        if (node.deferred) parts.push('deferred');
         if (node.excluded) parts.push('excluded');
         if (parts.length) label += ' (' + parts.join(', ') + ')';
         if (node.value.env) label += ` (${getEnvDescription(node.value.env)})`;
@@ -171,6 +172,7 @@ export default async function dumpGraphToGraphViz(
         if (node.value.needsStableName) parts.push('stable name');
         parts.push(node.value.name);
         parts.push('bb:' + (node.value.bundleBehavior ?? 'null'));
+        if (node.value.isPlaceholder) parts.push('placeholder');
         if (parts.length) label += ' (' + parts.join(', ') + ')';
         if (node.value.env) label += ` (${getEnvDescription(node.value.env)})`;
       } else if (node.type === 'request') {

packages/core/core/src/requests/AssetGraphRequest.js

@@ -19,7 +19,7 @@ import type {Diagnostic} from '@parcel/diagnostic';
 
 import invariant from 'assert';
 import nullthrows from 'nullthrows';
-import {PromiseQueue, setEqual} from '@parcel/utils';
+import {PromiseQueue, setEqual, isGlobMatch} from '@parcel/utils';
 import {hashString} from '@parcel/hash';
 import ThrowableDiagnostic from '@parcel/diagnostic';
 import {Priority} from '../types';
@@ -29,7 +29,7 @@ import createEntryRequest from './EntryRequest';
 import createTargetRequest from './TargetRequest';
 import createAssetRequest from './AssetRequest';
 import createPathRequest from './PathRequest';
-import {type ProjectPath} from '../projectPath';
+import {type ProjectPath, fromProjectPathRelative} from '../projectPath';
 import dumpGraphToGraphViz from '../dumpGraphToGraphViz';
 import {propagateSymbols} from '../SymbolPropagation';
 
@@ -39,6 +39,8 @@ type AssetGraphRequestInput = {|
   optionsRef: SharedReference,
   name: string,
   shouldBuildLazily?: boolean,
+  lazyIncludes?: string[],
+  lazyExcludes?: string[],
   requestedAssetIds?: Set<string>,
 |};
 
@@ -111,6 +113,8 @@ export class AssetGraphBuilder {
   name: string;
   cacheKey: string;
   shouldBuildLazily: boolean;
+  lazyIncludes: string[];
+  lazyExcludes: string[];
   requestedAssetIds: Set<string>;
   isSingleChangeRebuild: boolean;
   assetGroupsWithRemovedParents: Set<NodeId>;
@@ -127,6 +131,8 @@ export class AssetGraphBuilder {
       name,
       requestedAssetIds,
       shouldBuildLazily,
+      lazyIncludes,
+      lazyExcludes,
     } = input;
     let assetGraph = prevResult?.assetGraph ?? new AssetGraph();
     assetGraph.safeToIncrementallyBundle = true;
@@ -148,6 +154,8 @@ export class AssetGraphBuilder {
     this.name = name;
     this.requestedAssetIds = requestedAssetIds ?? new Set();
     this.shouldBuildLazily = shouldBuildLazily ?? false;
+    this.lazyIncludes = lazyIncludes ?? [];
+    this.lazyExcludes = lazyExcludes ?? [];
     this.cacheKey = hashString(
       `${PARCEL_VERSION}${name}${JSON.stringify(entries) ?? ''}${options.mode}`,
     );
@@ -306,14 +314,43 @@ export class AssetGraphBuilder {
       let childNode = nullthrows(this.assetGraph.getNode(childNodeId));
 
       if (node.type === 'asset' && childNode.type === 'dependency') {
-        if (this.requestedAssetIds.has(node.value.id)) {
+        // This logic will set `node.requested` to `true` if the node is in the list of requested asset ids
+        // (i.e. this is an entry of a (probably) placeholder bundle that wasn't previously requested)
+        //
+        // Otherwise, if this node either is explicitly not requested, or has had it's requested attribute deleted,
+        // it will determine whether this node is an "async child" - that is, is it a (probably)
+        // dynamic import(). If so, it will explicitly have it's `node.requested` set to `false`
+        //
+        // If it's not requested, but it's not an async child then it's `node.requested` is deleted (undefined)
+
+        // by default with lazy compilation all nodes are lazy
+        let isNodeLazy = true;
+
+        // For conditional lazy building - if this node matches the `lazyInclude` globs that means we want
+        // only those nodes to be treated as lazy - that means if this node does _NOT_ match that glob, then we
+        // also consider it not lazy (so it gets marked as requested).
+        if (this.lazyIncludes.length > 0) {
+          isNodeLazy = isGlobMatch(
+            fromProjectPathRelative(node.value.filePath),
+            this.lazyIncludes,
+          );
+        }
+        // Excludes override includes, so a node is _not_ lazy if it is included in the exclude list.
+        if (this.lazyExcludes.length > 0 && isNodeLazy) {
+          isNodeLazy = !isGlobMatch(
+            fromProjectPathRelative(node.value.filePath),
+            this.lazyExcludes,
+          );
+        }
+
+        if (this.requestedAssetIds.has(node.value.id) || !isNodeLazy) {
           node.requested = true;
         } else if (!node.requested) {
           let isAsyncChild = this.assetGraph
             .getIncomingDependencies(node.value)
             .every(dep => dep.isEntry || dep.priority !== Priority.sync);
           if (isAsyncChild) {
-            node.requested = false;
+            node.requested = !isNodeLazy;
           } else {
             delete node.requested;
           }
@@ -322,12 +359,21 @@ export class AssetGraphBuilder {
         let previouslyDeferred = childNode.deferred;
         childNode.deferred = node.requested === false;
 
+        // The child dependency node we're now evaluating should not be deferred if it's parent
+        // is explicitly not requested (requested = false, but not requested = undefined)
+        //
+        // if we weren't previously deferred but we are now, then this dependency node's parents should also
+        // be marked as deferred
+        //
+        // if we were previously deferred but we not longer are, then then all parents should no longer be
+        // deferred either
         if (!previouslyDeferred && childNode.deferred) {
           this.assetGraph.markParentsWithHasDeferred(childNodeId);
         } else if (previouslyDeferred && !childNode.deferred) {
           this.assetGraph.unmarkParentsWithHasDeferred(childNodeId);
         }
 
+        // We `shouldVisitChild` if the childNode is not deferred
         return !childNode.deferred;
       }
     }

packages/core/core/src/requests/BundleGraphRequest.js

@@ -99,6 +99,8 @@ export default function createBundleGraphRequest(
         entries: options.entries,
         optionsRef,
         shouldBuildLazily: options.shouldBuildLazily,
+        lazyIncludes: options.lazyIncludes,
+        lazyExcludes: options.lazyExcludes,
         requestedAssetIds,
       });
       let {assetGraph, changedAssets, assetRequests} = await api.runRequest(

packages/core/core/src/resolveOptions.js

@@ -104,6 +104,19 @@ export default async function resolveOptions(
       : undefined;
 
   let shouldBuildLazily = initialOptions.shouldBuildLazily ?? false;
+  let lazyIncludes = initialOptions.lazyIncludes ?? [];
+  if (lazyIncludes.length > 0 && !shouldBuildLazily) {
+    throw new Error(
+      'Lazy includes can only be provided when lazy building is enabled',
+    );
+  }
+  let lazyExcludes = initialOptions.lazyExcludes ?? [];
+  if (lazyExcludes.length > 0 && !shouldBuildLazily) {
+    throw new Error(
+      'Lazy excludes can only be provided when lazy building is enabled',
+    );
+  }
+
   let shouldContentHash =
     initialOptions.shouldContentHash ?? initialOptions.mode === 'production';
   if (shouldBuildLazily && shouldContentHash) {
@@ -140,6 +153,8 @@ export default async function resolveOptions(
     shouldAutoInstall: initialOptions.shouldAutoInstall ?? false,
     hmrOptions: initialOptions.hmrOptions ?? null,
     shouldBuildLazily,
+    lazyIncludes,
+    lazyExcludes,
     shouldBundleIncrementally: initialOptions.shouldBundleIncrementally ?? true,
     shouldContentHash,
     serveOptions: initialOptions.serveOptions

packages/core/core/src/types.js

@@ -270,6 +270,8 @@ export type ParcelOptions = {|
   shouldContentHash: boolean,
   serveOptions: ServerOptions | false,
   shouldBuildLazily: boolean,
+  lazyIncludes: string[],
+  lazyExcludes: string[],
   shouldBundleIncrementally: boolean,
   shouldAutoInstall: boolean,
   logLevel: LogLevel,

packages/core/core/test/test-utils.js

@@ -25,6 +25,8 @@ export const DEFAULT_OPTIONS: ParcelOptions = {
   hmrOptions: undefined,
   shouldContentHash: true,
   shouldBuildLazily: false,
+  lazyIncludes: [],
+  lazyExcludes: [],
   shouldBundleIncrementally: true,
   serveOptions: false,
   mode: 'development',

packages/core/integration-tests/test/lazy-compile.js

@@ -120,4 +120,145 @@ describe('lazy compile', function () {
     ]);
     subscription.unsubscribe();
   });
+
+  it('should support includes for lazy compile', async () => {
+    const b = await bundler(
+      path.join(__dirname, '/integration/lazy-compile/index.js'),
+      {
+        shouldBuildLazily: true,
+        lazyIncludes: ['**/lazy-1*'],
+        mode: 'development',
+        shouldContentHash: false,
+      },
+    );
+
+    await removeDistDirectory();
+
+    const subscription = await b.watch();
+    let result = await getNextBuild(b);
+
+    // Expect the bundle graph to only contain `parallel-lazy-1` but not `lazy-1`s children as we're only including lazy-1 in lazy compilation
+    // `parallel-lazy-1` which wasn't requested.
+    assertBundles(result.bundleGraph, [
+      {
+        name: /^index.*/,
+        assets: ['index.js', 'bundle-url.js', 'cacheLoader.js', 'js-loader.js'],
+      },
+      {
+        // This will be a placeholder, but that info isn't available in the BundleGraph
+        assets: ['lazy-1.js'],
+      },
+      {
+        assets: ['parallel-lazy-1.js', 'esmodule-helpers.js'],
+      },
+      {
+        assets: ['parallel-lazy-2.js'],
+      },
+    ]);
+
+    // ensure parallel-lazy was produced, as it isn't "included" in laziness..
+    assert(
+      await distDirIncludes([
+        'index.js',
+        /^parallel-lazy-1\./,
+        /^parallel-lazy-2\./,
+      ]),
+    );
+
+    result = await result.requestBundle(
+      findBundle(result.bundleGraph, /lazy-1/),
+    );
+
+    // Since lazy-2 was not included it should've been built when lazy-1 was..
+    assert(
+      await distDirIncludes([
+        'index.js',
+        /^parallel-lazy-1\./,
+        /^parallel-lazy-2\./,
+        /^lazy-1\./,
+        /^lazy-2\./,
+      ]),
+    );
+
+    subscription.unsubscribe();
+  });
+
+  it('should support excludes for lazy compile', async () => {
+    const b = await bundler(
+      path.join(__dirname, '/integration/lazy-compile/index.js'),
+      {
+        shouldBuildLazily: true,
+        lazyExcludes: ['**/lazy-*'],
+        mode: 'development',
+        shouldContentHash: false,
+      },
+    );
+
+    await removeDistDirectory();
+
+    const subscription = await b.watch();
+    let result = await getNextBuild(b);
+
+    result = await result.requestBundle(
+      findBundle(result.bundleGraph, /index.js/),
+    );
+
+    assertBundles(result.bundleGraph, [
+      {
+        name: /^index.*/,
+        assets: ['index.js', 'bundle-url.js', 'cacheLoader.js', 'js-loader.js'],
+      },
+      {
+        assets: ['lazy-1.js', 'esmodule-helpers.js'],
+      },
+      {
+        assets: ['lazy-2.js'],
+      },
+      {
+        // This will be a placeholder, but that info isn't available in the BundleGraph
+        assets: ['parallel-lazy-1.js'],
+      },
+    ]);
+
+    // lazy-* is _excluded_ from lazy compilation so it should have been built, but parallel-lazy should not have
+
+    assert(await distDirIncludes(['index.js', /^lazy-1\./, /^lazy-2\./]));
+
+    subscription.unsubscribe();
+  });
+
+  it('should lazy compile properly when same module is used sync/async', async () => {
+    const b = await bundler(
+      path.join(__dirname, '/integration/lazy-compile/index-sync-async.js'),
+      {
+        shouldBuildLazily: true,
+        mode: 'development',
+        shouldContentHash: false,
+      },
+    );
+
+    await removeDistDirectory();
+
+    const subscription = await b.watch();
+    let result = await getNextBuild(b);
+    result = await result.requestBundle(
+      findBundle(result.bundleGraph, /^index-sync-async\./),
+    );
+    result = await result.requestBundle(
+      findBundle(result.bundleGraph, /^uses-static-component\./),
+    );
+    result = await result.requestBundle(
+      findBundle(result.bundleGraph, /^uses-static-component-async\./),
+    );
+    result = await result.requestBundle(
+      findBundle(result.bundleGraph, /^static-component\./),
+    );
+
+    let output = await run(result.bundleGraph);
+    assert.deepEqual(await output.default(), [
+      'static component',
+      'static component',
+    ]);
+    subscription.unsubscribe();
+  });
 });

packages/core/parcel/src/cli.js

@@ -139,8 +139,12 @@ let serve = program
   )
   .option('--watch-for-stdin', 'exit when stdin closes')
   .option(
-    '--lazy',
-    'Build async bundles on demand, when requested in the browser',
+    '--lazy [includes]',
+    'Build async bundles on demand, when requested in the browser. Defaults to all async bundles, unless a comma separated list of source file globs is provided. Only async bundles whose entry points match these globs will be built lazily',
+  )
+  .option(
+    '--lazy-exclude <excludes>',
+    'Can only be used in combination with --lazy. Comma separated list of source file globs, async bundles whose entry points match these globs will not be built lazily',
   )
   .action(runCommand);
 
@@ -470,6 +474,11 @@ async function normalizeOptions(
   }
 
   let mode = command.name() === 'build' ? 'production' : 'development';
+
+  const normalizeIncludeExcludeList = (input?: string): string[] => {
+    if (typeof input !== 'string') return [];
+    return input.split(',').map(value => value.trim());
+  };
   return {
     shouldDisableCache: command.cache === false,
     cacheDir: command.cacheDir,
@@ -483,7 +492,9 @@ async function normalizeOptions(
     logLevel: command.logLevel,
     shouldProfile: command.profile,
     shouldTrace: command.trace,
-    shouldBuildLazily: command.lazy,
+    shouldBuildLazily: typeof command.lazy !== 'undefined',
+    lazyIncludes: normalizeIncludeExcludeList(command.lazy),
+    lazyExcludes: normalizeIncludeExcludeList(command.lazyExclude),
     shouldBundleIncrementally:
       process.env.PARCEL_INCREMENTAL_BUNDLING === 'false' ? false : true,
     detailedReport:

packages/core/types/index.js

@@ -303,6 +303,8 @@ export type InitialParcelOptions = {|
   +shouldTrace?: boolean,
   +shouldPatchConsole?: boolean,
   +shouldBuildLazily?: boolean,
+  +lazyIncludes?: string[],
+  +lazyExcludes?: string[],
   +shouldBundleIncrementally?: boolean,
 
   +inputFS?: FileSystem,