From 4f164b5bfc3812eb43fa70f3a611ff203fd1e7d0 Mon Sep 17 00:00:00 2001
From: Edward Faulkner <edward@eaf4.com>
Date: Sun, 2 Apr 2023 21:15:47 -0400
Subject: [PATCH 1/4] Correctly detect where modules loop back to AMD from
 webpack

Fixes #504.

Removes earlyBootSet, because that was a hack for the lack of this feature. earlyBootSet never actually worked correctly anyway. I'm considering this a bugfix and not a breaking change, since nobody could reliably use that setting anyway.
---
 packages/ember-auto-import/README.md          |   1 -
 packages/ember-auto-import/ts/auto-import.ts  |   1 -
 packages/ember-auto-import/ts/bundler.ts      |   2 +-
 packages/ember-auto-import/ts/package.ts      |  12 -
 .../ts/tests/inserter-test.ts                 |   1 +
 packages/ember-auto-import/ts/webpack.ts      | 218 ++++++------------
 test-scenarios/v2-addon-test.ts               |  13 --
 7 files changed, 74 insertions(+), 174 deletions(-)

diff --git a/packages/ember-auto-import/README.md b/packages/ember-auto-import/README.md
index ab0d3ec8a..5e1e0c6d9 100644
--- a/packages/ember-auto-import/README.md
+++ b/packages/ember-auto-import/README.md
@@ -126,7 +126,6 @@ let app = new EmberApp(defaults, {
 Supported Options
 
 - `alias`: _object_, Map from imported names to substitute names that will be imported instead. This is a prefix match by default. To opt out of prefix-matching and only match exactly, add a `$` suffix to the pattern.
-- `earlyBootSet`: _function, returning an array of strings_, (only supported on ember-source >= 3.27.0) defaults to undefined, but when used, the function will receive the default earlyBootSet, which is a list of common modules found at the source of rare timing / package / ordering issues in the compatibility / cross-communication between requirejs and webpack. This is a temporary escape hatch to allow non-embroider apps to consume v2 addons at any place in their dependency graph to help ease transitioning to embroider as this problem doesn't occur once an app is using embroider. See [issue#504](https://github.com/ef4/ember-auto-import/issues/504) for details.  Note that if any modules listed here belong to v2 addons, they will be removed from the set. To opt out of default behavior, return an empty array.
 - `exclude`: _list of strings, defaults to []_. Packages in this list will be ignored by ember-auto-import. Can be helpful if the package is already included another way (like a shim from some other Ember addon).
 - `forbidEval`: _boolean_, defaults to false. We use `eval` in development by default (because that is the fastest way to provide sourcemaps). If you need to comply with a strict Content Security Policy (CSP), you can set `forbidEval: true`. You will still get sourcemaps, they will just use a slower implementation.
 - `insertScriptsAt`: _string_, defaults to undefined. Optionally allows you to take manual control over where ember-auto-import's generated `<script>` tags will be inserted into your HTML and what attributes they will have. See "Customizing HTML Insertion" below.
diff --git a/packages/ember-auto-import/ts/auto-import.ts b/packages/ember-auto-import/ts/auto-import.ts
index aa713a317..c31610a57 100644
--- a/packages/ember-auto-import/ts/auto-import.ts
+++ b/packages/ember-auto-import/ts/auto-import.ts
@@ -151,7 +151,6 @@ export default class AutoImport implements AutoImportSharedAPI {
       publicAssetURL: this.rootPackage.publicAssetURL(),
       webpack,
       hasFastboot: this.rootPackage.isFastBootEnabled,
-      earlyBootSet: this.rootPackage.earlyBootSet,
       v2Addons: this.v2Addons,
       rootPackage: this.rootPackage,
     });
diff --git a/packages/ember-auto-import/ts/bundler.ts b/packages/ember-auto-import/ts/bundler.ts
index c2980e9ed..77a58fbd8 100644
--- a/packages/ember-auto-import/ts/bundler.ts
+++ b/packages/ember-auto-import/ts/bundler.ts
@@ -21,7 +21,6 @@ export interface BundlerOptions {
   browserslist: string;
   webpack: typeof webpack;
   hasFastboot: boolean;
-  earlyBootSet: undefined | ((defaultModules: string[]) => string[]);
   v2Addons: Map<string, string>;
   rootPackage: Package;
 }
@@ -32,6 +31,7 @@ export interface BuildResult {
   // names (because users can also add more entrypoints to the webpack config)
   entrypoints: Map<BundleName | string, string[]>;
   lazyAssets: string[];
+  externalDepsFor(request: string): string[];
 }
 
 export type Bundler = Plugin & {
diff --git a/packages/ember-auto-import/ts/package.ts b/packages/ember-auto-import/ts/package.ts
index 9e2506f80..935d73ee6 100644
--- a/packages/ember-auto-import/ts/package.ts
+++ b/packages/ember-auto-import/ts/package.ts
@@ -30,7 +30,6 @@ export interface Options {
   alias?: { [fromName: string]: string };
   webpack?: Configuration;
   publicAssetURL?: string;
-  earlyBootSet?: (defaultModules: string[]) => string[];
   styleLoaderOptions?: Record<string, unknown>;
   cssLoaderOptions?: Record<string, unknown>;
   miniCssExtractPluginOptions?: Record<string, unknown>;
@@ -414,17 +413,6 @@ export default class Package {
     );
   }
 
-  /**
-   * The function for defining the early boot set.
-   * Used when we begin building entry files for webpack, so that we can query all packages listed
-   * in the early boot set to check if they are v2 addons --if they are v2 addons,
-   * we remove them from the early boot set, as this feature is for a rare compatibility circumstance that
-   * only affects v1 addons consumed by v2 addons.
-   */
-  get earlyBootSet(): undefined | ((defaults: string[]) => string[]) {
-    return this.isAddon ? undefined : this.autoImportOptions?.earlyBootSet;
-  }
-
   get styleLoaderOptions(): Record<string, unknown> | undefined {
     // only apps (not addons) are allowed to set this
     return this.isAddon
diff --git a/packages/ember-auto-import/ts/tests/inserter-test.ts b/packages/ember-auto-import/ts/tests/inserter-test.ts
index 855380337..38efcc1e1 100644
--- a/packages/ember-auto-import/ts/tests/inserter-test.ts
+++ b/packages/ember-auto-import/ts/tests/inserter-test.ts
@@ -54,6 +54,7 @@ Qmodule('inserter', function (hooks) {
     buildResult = {
       entrypoints: new Map(),
       lazyAssets: [],
+      externalDepsFor: () => [],
     };
     bundleConfig = new BundleConfig({
       app: {
diff --git a/packages/ember-auto-import/ts/webpack.ts b/packages/ember-auto-import/ts/webpack.ts
index c1bbad925..b4a5fa8b6 100644
--- a/packages/ember-auto-import/ts/webpack.ts
+++ b/packages/ember-auto-import/ts/webpack.ts
@@ -5,10 +5,11 @@ import type {
   Stats,
   RuleSetUseItem,
   WebpackPluginInstance,
+  Module,
 } from 'webpack';
-import { join, dirname } from 'path';
+import { join, dirname, resolve } from 'path';
 import { mergeWith, flatten, zip } from 'lodash';
-import { writeFileSync, realpathSync } from 'fs';
+import { writeFileSync, realpathSync, readFileSync } from 'fs';
 import { compile, registerHelper } from 'handlebars';
 import jsStringEscape from 'js-string-escape';
 import { BundleDependencies, ResolvedTemplateImport } from './splitter';
@@ -22,53 +23,9 @@ import { Memoize } from 'typescript-memoize';
 import makeDebug from 'debug';
 import { ensureDirSync, symlinkSync, existsSync } from 'fs-extra';
 import MiniCssExtractPlugin from 'mini-css-extract-plugin';
-import semver from 'semver';
 
 const debug = makeDebug('ember-auto-import:webpack');
 
-/**
- * Passed to and configuable with autoImport.earlyBootset
- * example:
- * ```js
- * // ember-cli-build.js
- * // ...
- * autoImport: {
- *   earlyBootSet: (defaultModules) => {
- *     return [
- *       ...defaultModules,
- *       'my-package/my-module,
- *     ];
- *   }
- * }
- * ```
- *
- * Anything listed in the return value from this function that is from a v2 addon will be removed.
- * (Allowing each of these packages from the default set to be incrementally converted to v2 addons
- * without the need for this code to be updated)
- *
- */
-const DEFAULT_EARLY_BOOT_SET = Object.freeze([
-  '@glimmer/tracking',
-  '@glimmer/component',
-  '@ember/service',
-  '@ember/controller',
-  '@ember/routing/route',
-  '@ember/component',
-]);
-
-/**
- * @glimmer/tracking + @glimmer/component
- * are separate addons, yet included in ember-source (for now),
- * but we will be required to use the real glimmer packages before
- * ember-source is converted to v2 (else we implement more hacks at resolver time!)
- */
-const BOOT_SET_FROM_EMBER_SOURCE = Object.freeze([
-  '@ember/service',
-  '@ember/controller',
-  '@ember/routing/route',
-  '@ember/component',
-]);
-
 registerHelper('js-string-escape', jsStringEscape);
 registerHelper('join', function (list, connector) {
   return list.join(connector);
@@ -90,9 +47,8 @@ module.exports = (function(){
     {{! this is only used for synchronous importSync() using a template string }}
     return r('_eai_sync_' + specifier)(Array.prototype.slice.call(arguments, 1))
   };
-  d('__v1-addons__early-boot-set__', [{{{v1EmberDeps}}}], function() {});
   {{#each staticImports as |module|}}
-    d('{{js-string-escape module.specifier}}', ['__v1-addons__early-boot-set__'], function() { return require('{{js-string-escape module.specifier}}'); });
+    d('{{js-string-escape module.specifier}}', EAI_DISCOVERED_EXTERNALS('{{js-string-escape module.specifier}}'), function() { return require('{{js-string-escape module.specifier}}'); });
   {{/each}}
   {{#each dynamicImports as |module|}}
     d('_eai_dyn_{{js-string-escape module.specifier}}', [], function() { return import('{{js-string-escape module.specifier}}'); });
@@ -120,7 +76,6 @@ module.exports = (function(){
   staticTemplateImports: { key: string; args: string; template: string }[];
   dynamicTemplateImports: { key: string; args: string; template: string }[];
   publicAssetURL: string | undefined;
-  v1EmberDeps: string;
 }) => string;
 
 // this goes in a file by itself so we can tell webpack not to parse it. That
@@ -384,6 +339,73 @@ export default class WebpackBundler extends Plugin implements Bundler {
     this.linkDeps(bundleDeps);
     let stats = await this.runWebpack();
     this.lastBuildResult = this.summarizeStats(stats, bundleDeps);
+    this.addDiscoveredExternals(this.lastBuildResult);
+  }
+
+  private addDiscoveredExternals(build: BuildResult) {
+    for (let assetFiles of build.entrypoints.values()) {
+      for (let assetFile of assetFiles) {
+        let inputSrc = readFileSync(
+          resolve(this.outputPath, assetFile),
+          'utf8'
+        );
+        let outputSrc = inputSrc.replace(
+          /EAI_DISCOVERED_EXTERNALS\('([^']+)'\)/g,
+          (_substr: string, matched: string) => {
+            let deps = build.externalDepsFor(matched);
+            return '[' + deps.map((d) => `'${d}'`).join(',') + ']';
+          }
+        );
+        writeFileSync(resolve(this.outputPath, assetFile), outputSrc, 'utf8');
+      }
+    }
+  }
+
+  private externalDepsSearcher(
+    stats: Required<Stats>
+  ): (request: string) => string[] {
+    let externals = new Map<Module, Set<string>>();
+
+    function gatherExternals(
+      module: Module,
+      output = new Set<string>()
+    ): Set<string> {
+      if (externals.has(module)) {
+        for (let ext of externals.get(module)!) {
+          output.add(ext);
+        }
+      } else {
+        let ownExternals = new Set<string>();
+        externals.set(module, ownExternals);
+        for (let dep of module.dependencies) {
+          let nextModule = stats.compilation.moduleGraph.getModule(dep);
+          if (nextModule) {
+            if ((nextModule as any).externalType) {
+              ownExternals.add((dep as any).request);
+            } else {
+              gatherExternals(nextModule, ownExternals);
+            }
+          }
+        }
+        for (let o of ownExternals) {
+          output.add(o);
+        }
+      }
+      return output;
+    }
+
+    return (request: string): string[] => {
+      for (let module of stats.compilation.modules) {
+        for (let dep of module.dependencies) {
+          if ((dep as any).request === request) {
+            return [
+              ...gatherExternals(stats.compilation.moduleGraph.getModule(dep)),
+            ];
+          }
+        }
+      }
+      return [];
+    };
   }
 
   private summarizeStats(
@@ -405,6 +427,7 @@ export default class WebpackBundler extends Plugin implements Bundler {
     let output: BuildResult = {
       entrypoints: new Map(),
       lazyAssets: [] as string[],
+      externalDepsFor: this.externalDepsSearcher(_stats),
     };
     let nonLazyAssets: Set<string> = new Set();
     for (let id of Object.keys(entrypoints!)) {
@@ -436,103 +459,7 @@ export default class WebpackBundler extends Plugin implements Bundler {
     return output;
   }
 
-  private getEarlyBootSet() {
-    let result = this.opts.earlyBootSet
-      ? this.opts.earlyBootSet([...DEFAULT_EARLY_BOOT_SET])
-      : [];
-
-    /**
-     * Prior to ember-source 3.27, the modules were precompiled into a variant of requirejs/AMD.
-     * As such, the early boot set will not support earlier than 3.27.
-     */
-    let host = this.opts.rootPackage;
-    let emberSource = host.requestedRange('ember-source');
-    let emberSourceVersion = semver.valid(emberSource);
-
-    if (emberSourceVersion && semver.lt(emberSourceVersion, '3.27.0')) {
-      if (this.opts.earlyBootSet) {
-        throw new Error(
-          'autoImport.earlyBootSet is not supported for ember-source <= 3.27.0'
-        );
-      }
-
-      result = [];
-    }
-
-    if (!Array.isArray(result)) {
-      throw new Error(
-        'autoImport.earlyBootSet was used, but did not return an array. An array of strings is required'
-      );
-    }
-
-    // Reminder: [/* empty array */].every(anything) is true
-    if (!result.every((entry) => typeof entry === 'string')) {
-      throw new Error(
-        'autoImport.earlyBootSet was used, but the returned array did contained data other than strings. Every element in the return array must be a string representing a module'
-      );
-    }
-
-    /**
-     * TODO: iterate over these and check their dependencies if any depend on a v2 addon
-     *       - when this situation occurs, check that v2 addon's dependencies if any of those are v1 addons,
-     *         - if so, log a warning, about potentially needing to add modules from that v1 addon to the early boot set
-     */
-    let v2Addons = this.opts.v2Addons.keys();
-    let isEmberSourceV2 = this.opts.v2Addons.has('ember-source');
-
-    function depNameForPath(modulePath: string) {
-      if (modulePath.startsWith('@')) {
-        let [scope, name] = modulePath.split('/');
-
-        return `${scope}/${name}`;
-      }
-
-      return modulePath.split('/')[0];
-    }
-
-    function isFromEmberSource(modulePath: string) {
-      return BOOT_SET_FROM_EMBER_SOURCE.some((fromEmber) =>
-        modulePath.startsWith(fromEmber)
-      );
-    }
-
-    result = result.filter((modulePath) => {
-      if (isEmberSourceV2 && isFromEmberSource(modulePath)) {
-        return false;
-      }
-
-      let depName = depNameForPath(modulePath);
-
-      /**
-       * If a dependency from the earlyBootSet is not actually included in the project,
-       * don't include in the earlyBootSet emitted content.
-       */
-      if (!host.hasDependency(depName) && !isFromEmberSource(modulePath)) {
-        return false;
-      }
-
-      for (let v2Addon of v2Addons) {
-        // Omit modulePaths from v2 addons
-        if (modulePath.startsWith(v2Addon)) {
-          if (!DEFAULT_EARLY_BOOT_SET.includes(v2Addon)) {
-            console.warn(
-              `\`${modulePath}\` was included in the \`autoImport.earlyBootSet\` list, but belongs to a v2 addon. You can remove this entry from the earlyBootSet`
-            );
-          }
-
-          return false;
-        }
-      }
-
-      return true;
-    });
-
-    return result;
-  }
-
   private writeEntryFile(name: string, deps: BundleDependencies) {
-    let v1EmberDeps = this.getEarlyBootSet();
-
     writeFileSync(
       join(this.stagingDir, `${name}.cjs`),
       entryTemplate({
@@ -543,7 +470,6 @@ export default class WebpackBundler extends Plugin implements Bundler {
         staticTemplateImports:
           deps.staticTemplateImports.map(mapTemplateImports),
         publicAssetURL: this.opts.publicAssetURL,
-        v1EmberDeps: v1EmberDeps.map((name) => `'${name}'`).join(','),
       })
     );
   }
diff --git a/test-scenarios/v2-addon-test.ts b/test-scenarios/v2-addon-test.ts
index d0fb4a08f..79753eb10 100644
--- a/test-scenarios/v2-addon-test.ts
+++ b/test-scenarios/v2-addon-test.ts
@@ -504,19 +504,6 @@ Scenarios.fromProject(baseApp)
     project.linkDependency('webpack', { baseDir: __dirname });
 
     merge(project.files, {
-      'ember-cli-build.js': `
-        const EmberApp = require('ember-cli/lib/broccoli/ember-app');
-        module.exports = function (defaults) {
-          let app = new EmberApp(defaults, {
-            autoImport: {
-              earlyBootSet(defaults) {
-                return [...defaults, 'v1-addon'];
-              }
-            }
-          });
-          return app.toTree();
-        };
-      `,
       tests: {
         unit: {
           'dep-chain-test.js': `

From dd7889508bc7dfc45bb6c7026e273eb329d729dc Mon Sep 17 00:00:00 2001
From: Edward Faulkner <edward@eaf4.com>
Date: Mon, 3 Apr 2023 11:43:27 -0400
Subject: [PATCH 2/4] update actions/cache to eliminate github warnings

---
 .github/workflows/nodejs.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml
index ee271b6b5..36cdfffbb 100644
--- a/.github/workflows/nodejs.yml
+++ b/.github/workflows/nodejs.yml
@@ -19,7 +19,7 @@ jobs:
       - uses: volta-cli/action@v4
         with:
           node-version: '14'
-      - uses: actions/cache@v2
+      - uses: actions/cache@v3
         with:
           path: ~/.npm
           key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
@@ -44,7 +44,7 @@ jobs:
       - uses: volta-cli/action@v4
         with:
           node-version: '14'
-      - uses: actions/cache@v2
+      - uses: actions/cache@v3
         with:
           path: ~/.npm
           key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
@@ -63,7 +63,7 @@ jobs:
       - uses: volta-cli/action@v4
         with:
           node-version: '14'
-      - uses: actions/cache@v2
+      - uses: actions/cache@v3
         with:
           path: ~/.npm
           key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
@@ -82,7 +82,7 @@ jobs:
   #     - uses: volta-cli/action@v4
   #       with:
   #         node-version: '14'
-  #     - uses: actions/cache@v2
+  #     - uses: actions/cache@v3
   #       with:
   #         path: ~/.npm
   #         key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}

From 6109581de33fc6426e15d89aa0ac1c074bc32fdd Mon Sep 17 00:00:00 2001
From: Edward Faulkner <edward@eaf4.com>
Date: Mon, 3 Apr 2023 13:39:03 -0400
Subject: [PATCH 3/4] match either quote style

In prod this gets rewritten, presumably by terser, so we need to match either one.
---
 packages/ember-auto-import/ts/webpack.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/ember-auto-import/ts/webpack.ts b/packages/ember-auto-import/ts/webpack.ts
index b4a5fa8b6..0c6262fdc 100644
--- a/packages/ember-auto-import/ts/webpack.ts
+++ b/packages/ember-auto-import/ts/webpack.ts
@@ -350,7 +350,7 @@ export default class WebpackBundler extends Plugin implements Bundler {
           'utf8'
         );
         let outputSrc = inputSrc.replace(
-          /EAI_DISCOVERED_EXTERNALS\('([^']+)'\)/g,
+          /EAI_DISCOVERED_EXTERNALS\(['"]([^'"]+)['"]\)/g,
           (_substr: string, matched: string) => {
             let deps = build.externalDepsFor(matched);
             return '[' + deps.map((d) => `'${d}'`).join(',') + ']';

From 8dddf0741a5b6372bd38efc6731a4905f77872e2 Mon Sep 17 00:00:00 2001
From: Edward Faulkner <edward@eaf4.com>
Date: Tue, 4 Apr 2023 08:55:17 -0400
Subject: [PATCH 4/4] only insert deps externalized by us

---
 packages/ember-auto-import/ts/webpack.ts | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/packages/ember-auto-import/ts/webpack.ts b/packages/ember-auto-import/ts/webpack.ts
index 0c6262fdc..9d37870b0 100644
--- a/packages/ember-auto-import/ts/webpack.ts
+++ b/packages/ember-auto-import/ts/webpack.ts
@@ -277,13 +277,15 @@ export default class WebpackBundler extends Plugin implements Bundler {
     };
   }
 
+  private externalizedByUs = new Set<string>();
+
   @Memoize()
   private get externalsHandler(): Configuration['externals'] {
     let packageCache = PackageCache.shared(
       'ember-auto-import',
       this.opts.appRoot
     );
-    return function (params, callback) {
+    return (params, callback) => {
       let { context, request } = params;
       if (!context || !request) {
         return callback();
@@ -304,6 +306,7 @@ export default class WebpackBundler extends Plugin implements Bundler {
       }
 
       if (pkg.meta.externals?.includes(name)) {
+        this.externalizedByUs.add(request);
         return callback(undefined, 'commonjs ' + request);
       }
 
@@ -317,6 +320,7 @@ export default class WebpackBundler extends Plugin implements Bundler {
         } else {
           // the package exists but it is a v1 ember addon, so it's not
           // resolvable at build time, so we externalize it.
+          this.externalizedByUs.add(request);
           return callback(undefined, 'commonjs ' + request);
         }
       } catch (err) {
@@ -324,6 +328,7 @@ export default class WebpackBundler extends Plugin implements Bundler {
           throw err;
         }
         // real package doesn't exist, so externalize it
+        this.externalizedByUs.add(request);
         return callback(undefined, 'commonjs ' + request);
       }
     };
@@ -352,7 +357,9 @@ export default class WebpackBundler extends Plugin implements Bundler {
         let outputSrc = inputSrc.replace(
           /EAI_DISCOVERED_EXTERNALS\(['"]([^'"]+)['"]\)/g,
           (_substr: string, matched: string) => {
-            let deps = build.externalDepsFor(matched);
+            let deps = build
+              .externalDepsFor(matched)
+              .filter((dep) => this.externalizedByUs.has(dep));
             return '[' + deps.map((d) => `'${d}'`).join(',') + ']';
           }
         );