Correctly detect where modules loop back to AMD from webpack

Fixes #504.

Removes earlyBootSet, because that was a hack for the lack of this feature. earlyBootSet never actually worked correctly anyway. I'm considering this a bugfix and not a breaking change, since nobody could reliably use that setting anyway.

packages/ember-auto-import/README.md

@@ -126,7 +126,6 @@ let app = new EmberApp(defaults, {
 Supported Options
 
 - `alias`: _object_, Map from imported names to substitute names that will be imported instead. This is a prefix match by default. To opt out of prefix-matching and only match exactly, add a `$` suffix to the pattern.
-- `earlyBootSet`: _function, returning an array of strings_, (only supported on ember-source >= 3.27.0) defaults to undefined, but when used, the function will receive the default earlyBootSet, which is a list of common modules found at the source of rare timing / package / ordering issues in the compatibility / cross-communication between requirejs and webpack. This is a temporary escape hatch to allow non-embroider apps to consume v2 addons at any place in their dependency graph to help ease transitioning to embroider as this problem doesn't occur once an app is using embroider. See [issue#504](https://github.com/ef4/ember-auto-import/issues/504) for details. Note that if any modules listed here belong to v2 addons, they will be removed from the set. To opt out of default behavior, return an empty array.
 - `exclude`: _list of strings, defaults to []_. Packages in this list will be ignored by ember-auto-import. Can be helpful if the package is already included another way (like a shim from some other Ember addon).
 - `forbidEval`: _boolean_, defaults to false. We use `eval` in development by default (because that is the fastest way to provide sourcemaps). If you need to comply with a strict Content Security Policy (CSP), you can set `forbidEval: true`. You will still get sourcemaps, they will just use a slower implementation.
 - `insertScriptsAt`: _string_, defaults to undefined. Optionally allows you to take manual control over where ember-auto-import's generated `<script>` tags will be inserted into your HTML and what attributes they will have. See "Customizing HTML Insertion" below.

packages/ember-auto-import/ts/auto-import.ts

@@ -151,7 +151,6 @@ export default class AutoImport implements AutoImportSharedAPI {
  publicAssetURL: this.rootPackage.publicAssetURL(),
  webpack,
  hasFastboot: this.rootPackage.isFastBootEnabled,
- earlyBootSet: this.rootPackage.earlyBootSet,
  v2Addons: this.v2Addons,
  rootPackage: this.rootPackage,
  });

packages/ember-auto-import/ts/bundler.ts

@@ -21,7 +21,6 @@ export interface BundlerOptions {
  browserslist: string;
  webpack: typeof webpack;
  hasFastboot: boolean;
- earlyBootSet: undefined | ((defaultModules: string[]) => string[]);
  v2Addons: Map<string, string>;
  rootPackage: Package;
 }
@@ -32,6 +31,7 @@ export interface BuildResult {
  // names (because users can also add more entrypoints to the webpack config)
  entrypoints: Map<BundleName | string, string[]>;
  lazyAssets: string[];
+ externalDepsFor(request: string): string[];
 }
 
 export type Bundler = Plugin & {

packages/ember-auto-import/ts/package.ts

@@ -30,7 +30,6 @@ export interface Options {
  alias?: { [fromName: string]: string };
  webpack?: Configuration;
  publicAssetURL?: string;
- earlyBootSet?: (defaultModules: string[]) => string[];
  styleLoaderOptions?: Record<string, unknown>;
  cssLoaderOptions?: Record<string, unknown>;
  miniCssExtractPluginOptions?: Record<string, unknown>;
@@ -414,17 +413,6 @@ export default class Package {
  );
  }
 
- /**
- * The function for defining the early boot set.
- * Used when we begin building entry files for webpack, so that we can query all packages listed
- * in the early boot set to check if they are v2 addons --if they are v2 addons,
- * we remove them from the early boot set, as this feature is for a rare compatibility circumstance that
- * only affects v1 addons consumed by v2 addons.
- */
- get earlyBootSet(): undefined | ((defaults: string[]) => string[]) {
- return this.isAddon ? undefined : this.autoImportOptions?.earlyBootSet;
- }
-
  get styleLoaderOptions(): Record<string, unknown> | undefined {
  // only apps (not addons) are allowed to set this
  return this.isAddon

packages/ember-auto-import/ts/tests/inserter-test.ts

@@ -54,6 +54,7 @@ Qmodule('inserter', function (hooks) {
  buildResult = {
  entrypoints: new Map(),
  lazyAssets: [],
+ externalDepsFor: () => [],
  };
  bundleConfig = new BundleConfig({
  app: {

packages/ember-auto-import/ts/webpack.ts

@@ -5,10 +5,11 @@ import type {
  Stats,
  RuleSetUseItem,
  WebpackPluginInstance,
+ Module,
 } from 'webpack';
-import { join, dirname } from 'path';
+import { join, dirname, resolve } from 'path';
 import { mergeWith, flatten, zip } from 'lodash';
-import { writeFileSync, realpathSync } from 'fs';
+import { writeFileSync, realpathSync, readFileSync } from 'fs';
 import { compile, registerHelper } from 'handlebars';
 import jsStringEscape from 'js-string-escape';
 import { BundleDependencies, ResolvedTemplateImport } from './splitter';
@@ -22,53 +23,9 @@ import { Memoize } from 'typescript-memoize';
 import makeDebug from 'debug';
 import { ensureDirSync, symlinkSync, existsSync } from 'fs-extra';
 import MiniCssExtractPlugin from 'mini-css-extract-plugin';
-import semver from 'semver';
 
 const debug = makeDebug('ember-auto-import:webpack');
 
-/**
- * Passed to and configuable with autoImport.earlyBootset
- * example:
- * ```js
- * // ember-cli-build.js
- * // ...
- * autoImport: {
- * earlyBootSet: (defaultModules) => {
- * return [
- * ...defaultModules,
- * 'my-package/my-module,
- * ];
- * }
- * }
- * ```
- *
- * Anything listed in the return value from this function that is from a v2 addon will be removed.
- * (Allowing each of these packages from the default set to be incrementally converted to v2 addons
- * without the need for this code to be updated)
- *
- */
-const DEFAULT_EARLY_BOOT_SET = Object.freeze([
- '@glimmer/tracking',
- '@glimmer/component',
- '@ember/service',
- '@ember/controller',
- '@ember/routing/route',
- '@ember/component',
-]);
-
-/**
- * @glimmer/tracking + @glimmer/component
- * are separate addons, yet included in ember-source (for now),
- * but we will be required to use the real glimmer packages before
- * ember-source is converted to v2 (else we implement more hacks at resolver time!)
- */
-const BOOT_SET_FROM_EMBER_SOURCE = Object.freeze([
- '@ember/service',
- '@ember/controller',
- '@ember/routing/route',
- '@ember/component',
-]);
-
 registerHelper('js-string-escape', jsStringEscape);
 registerHelper('join', function (list, connector) {
  return list.join(connector);
@@ -90,9 +47,8 @@ module.exports = (function(){
  {{! this is only used for synchronous importSync() using a template string }}
  return r('_eai_sync_' + specifier)(Array.prototype.slice.call(arguments, 1))
  };
- d('__v1-addons__early-boot-set__', [{{{v1EmberDeps}}}], function() {});
  {{#each staticImports as |module|}}
- d('{{js-string-escape module.specifier}}', ['__v1-addons__early-boot-set__'], function() { return require('{{js-string-escape module.specifier}}'); });
+ d('{{js-string-escape module.specifier}}', EAI_DISCOVERED_EXTERNALS('{{js-string-escape module.specifier}}'), function() { return require('{{js-string-escape module.specifier}}'); });
  {{/each}}
  {{#each dynamicImports as |module|}}
  d('_eai_dyn_{{js-string-escape module.specifier}}', [], function() { return import('{{js-string-escape module.specifier}}'); });
@@ -120,7 +76,6 @@ module.exports = (function(){
  staticTemplateImports: { key: string; args: string; template: string }[];
  dynamicTemplateImports: { key: string; args: string; template: string }[];
  publicAssetURL: string | undefined;
- v1EmberDeps: string;
 }) => string;
 
 // this goes in a file by itself so we can tell webpack not to parse it. That
@@ -384,6 +339,73 @@ export default class WebpackBundler extends Plugin implements Bundler {
  this.linkDeps(bundleDeps);
  let stats = await this.runWebpack();
  this.lastBuildResult = this.summarizeStats(stats, bundleDeps);
+ this.addDiscoveredExternals(this.lastBuildResult);
+ }
+
+ private addDiscoveredExternals(build: BuildResult) {
+ for (let assetFiles of build.entrypoints.values()) {
+ for (let assetFile of assetFiles) {
+ let inputSrc = readFileSync(
+ resolve(this.outputPath, assetFile),
+ 'utf8'
+ );
+ let outputSrc = inputSrc.replace(
+ /EAI_DISCOVERED_EXTERNALS\('([^']+)'\)/g,
+ (_substr: string, matched: string) => {
+ let deps = build.externalDepsFor(matched);
+ return '[' + deps.map((d) => `'${d}'`).join(',') + ']';
+ }
+ );
+ writeFileSync(resolve(this.outputPath, assetFile), outputSrc, 'utf8');
+ }
+ }
+ }
+
+ private externalDepsSearcher(
+ stats: Required<Stats>
+ ): (request: string) => string[] {
+ let externals = new Map<Module, Set<string>>();
+
+ function gatherExternals(
+ module: Module,
+ output = new Set<string>()
+ ): Set<string> {
+ if (externals.has(module)) {
+ for (let ext of externals.get(module)!) {
+ output.add(ext);
+ }
+ } else {
+ let ownExternals = new Set<string>();
+ externals.set(module, ownExternals);
+ for (let dep of module.dependencies) {
+ let nextModule = stats.compilation.moduleGraph.getModule(dep);
+ if (nextModule) {
+ if ((nextModule as any).externalType) {
+ ownExternals.add((dep as any).request);
+ } else {
+ gatherExternals(nextModule, ownExternals);
+ }
+ }
+ }
+ for (let o of ownExternals) {
+ output.add(o);
+ }
+ }
+ return output;
+ }
+
+ return (request: string): string[] => {
+ for (let module of stats.compilation.modules) {
+ for (let dep of module.dependencies) {
+ if ((dep as any).request === request) {
+ return [
+ ...gatherExternals(stats.compilation.moduleGraph.getModule(dep)),
+ ];
+ }
+ }
+ }
+ return [];
+ };
  }
 
  private summarizeStats(
@@ -405,6 +427,7 @@ export default class WebpackBundler extends Plugin implements Bundler {
  let output: BuildResult = {
  entrypoints: new Map(),
  lazyAssets: [] as string[],
+ externalDepsFor: this.externalDepsSearcher(_stats),
  };
  let nonLazyAssets: Set<string> = new Set();
  for (let id of Object.keys(entrypoints!)) {
@@ -436,103 +459,7 @@ export default class WebpackBundler extends Plugin implements Bundler {
  return output;
  }
 
- private getEarlyBootSet() {
- let result = this.opts.earlyBootSet
- ? this.opts.earlyBootSet([...DEFAULT_EARLY_BOOT_SET])
- : [];
-
- /**
- * Prior to ember-source 3.27, the modules were precompiled into a variant of requirejs/AMD.
- * As such, the early boot set will not support earlier than 3.27.
- */
- let host = this.opts.rootPackage;
- let emberSource = host.requestedRange('ember-source');
- let emberSourceVersion = semver.valid(emberSource);
-
- if (emberSourceVersion && semver.lt(emberSourceVersion, '3.27.0')) {
- if (this.opts.earlyBootSet) {
- throw new Error(
- 'autoImport.earlyBootSet is not supported for ember-source <= 3.27.0'
- );
- }
-
- result = [];
- }
-
- if (!Array.isArray(result)) {
- throw new Error(
- 'autoImport.earlyBootSet was used, but did not return an array. An array of strings is required'
- );
- }
-
- // Reminder: [/* empty array */].every(anything) is true
- if (!result.every((entry) => typeof entry === 'string')) {
- throw new Error(
- 'autoImport.earlyBootSet was used, but the returned array did contained data other than strings. Every element in the return array must be a string representing a module'
- );
- }
-
- /**
- * TODO: iterate over these and check their dependencies if any depend on a v2 addon
- * - when this situation occurs, check that v2 addon's dependencies if any of those are v1 addons,
- * - if so, log a warning, about potentially needing to add modules from that v1 addon to the early boot set
- */
- let v2Addons = this.opts.v2Addons.keys();
- let isEmberSourceV2 = this.opts.v2Addons.has('ember-source');
-
- function depNameForPath(modulePath: string) {
- if (modulePath.startsWith('@')) {
- let [scope, name] = modulePath.split('/');
-
- return `${scope}/${name}`;
- }
-
- return modulePath.split('/')[0];
- }
-
- function isFromEmberSource(modulePath: string) {
- return BOOT_SET_FROM_EMBER_SOURCE.some((fromEmber) =>
- modulePath.startsWith(fromEmber)
- );
- }
-
- result = result.filter((modulePath) => {
- if (isEmberSourceV2 && isFromEmberSource(modulePath)) {
- return false;
- }
-
- let depName = depNameForPath(modulePath);
-
- /**
- * If a dependency from the earlyBootSet is not actually included in the project,
- * don't include in the earlyBootSet emitted content.
- */
- if (!host.hasDependency(depName) && !isFromEmberSource(modulePath)) {
- return false;
- }
-
- for (let v2Addon of v2Addons) {
- // Omit modulePaths from v2 addons
- if (modulePath.startsWith(v2Addon)) {
- if (!DEFAULT_EARLY_BOOT_SET.includes(v2Addon)) {
- console.warn(
- `\`${modulePath}\` was included in the \`autoImport.earlyBootSet\` list, but belongs to a v2 addon. You can remove this entry from the earlyBootSet`
- );
- }
-
- return false;
- }
- }
-
- return true;
- });
-
- return result;
- }
-
  private writeEntryFile(name: string, deps: BundleDependencies) {
- let v1EmberDeps = this.getEarlyBootSet();
-
  writeFileSync(
  join(this.stagingDir, `${name}.cjs`),
  entryTemplate({
@@ -543,7 +470,6 @@ export default class WebpackBundler extends Plugin implements Bundler {
  staticTemplateImports:
  deps.staticTemplateImports.map(mapTemplateImports),
  publicAssetURL: this.opts.publicAssetURL,
- v1EmberDeps: v1EmberDeps.map((name) => `'${name}'`).join(','),
  })
  );
  }

test-scenarios/v2-addon-test.ts

@@ -504,19 +504,6 @@ Scenarios.fromProject(baseApp)
  project.linkDependency('webpack', { baseDir: __dirname });
 
  merge(project.files, {
- 'ember-cli-build.js': `
- const EmberApp = require('ember-cli/lib/broccoli/ember-app');
- module.exports = function (defaults) {
- let app = new EmberApp(defaults, {
- autoImport: {
- earlyBootSet(defaults) {
- return [...defaults, 'v1-addon'];
- }
- }
- });
- return app.toTree();
- };
- `,
  tests: {
  unit: {
  'dep-chain-test.js': `