Fix Babel-Loader Caching for ember-template-compiler

Today, when switching template-compiler versions, we can end up with global cache pollution. Although this can manifest many ways, it most commonly manifests when running ember-try across versions of ember with different template compilation semantics when also using inline templates.

To address this:

We now include the checksum of the template compiler in the inline-template-compiler babel configuration options.
This busts the cache, as any functioning caching babel subsystem (such as babel-loader) must consider option changes as cache invalidating scenarios. Today, it is in-fact the case that babel-loader operates as we expect, and with this change, caches are correctly invalidated.

We co-located the checksum with the template-compiler options, rather then a single global cache key to both simplify debugging and not prevent some more granular caching strategy from functioning optimally.

Finally this commit uses the existing caching infrastructure to load / generate cacheKey’s for the template-compiler. It changes that infrastructure slightly only to ensure a vm.script is create when absolutely needed.

This does not explicitly include integration tests due to my believe that this is already covered sufficiently:

1) today’s tests already include this (running yarn test currently fails without this fix)
2) type checking ensure we now always pass the key in
3) this is ultimately a configuration change of a babel-loader feature (which itself should test this)

packages/compat/src/compat-app.ts

@@ -35,6 +35,7 @@ import bind from 'bind-decorator';
 import { pathExistsSync } from 'fs-extra';
 import { tmpdir } from 'os';
 import { Options as AdjustImportsOptions } from '@embroider/core/src/babel-plugin-adjust-imports';
+import { getEmberExports } from '@embroider/core/src/load-ember-template-compiler';
 import semver from 'semver';
 
 interface TreeNames {
@@ -401,11 +402,15 @@ class CompatAppAdapter implements AppAdapter<TreeNames> {
  adjustImportsOptions: this.makeAdjustImportOptions(false),
  });
 
+ const compilerPath = resolveSync(this.templateCompilerPath(), { basedir: this.root });
+ const { cacheKey: compilerChecksum } = getEmberExports(compilerPath);
  // It's ok that this isn't a fully configured template compiler. We're only
  // using it to parse component snippets out of rules.
  resolver.astTransformer(
  new NodeTemplateCompiler({
- compilerPath: resolveSync(this.templateCompilerPath(), { basedir: this.root }),
+ compilerPath,
+ compilerChecksum,
+
  EmberENV: {},
  plugins: {},
  })

packages/compat/src/v1-addon.ts

@@ -26,6 +26,7 @@ import { isEmberAutoImportDynamic, isCompactReexports, isColocationPlugin } from
 import { ResolvedDep } from '@embroider/core/src/resolver';
 import TemplateCompilerBroccoliPlugin from './template-compiler-broccoli-plugin';
 import { fromPairs } from 'lodash';
+import { getEmberExports } from '@embroider/core/src/load-ember-template-compiler';
 
 const stockTreeNames = Object.freeze([
  'addon',
@@ -132,8 +133,11 @@ export default class V1Addon {
  // our macros don't run here in stage1
  options.plugins.ast = options.plugins.ast.filter((p: any) => !isEmbroiderMacrosPlugin(p));
  if (options.plugins.ast.length > 0) {
+ const { cacheKey: compilerChecksum } = getEmberExports(options.templateCompilerPath);
+
  return new NodeTemplateCompiler({
  compilerPath: options.templateCompilerPath,
+ compilerChecksum,
  EmberENV: {},
  plugins: options.plugins,
  resolver: this.templateResolver(),

packages/compat/tests/audit.test.ts

@@ -24,6 +24,7 @@ describe('audit', function () {
  let templateCompiler = templateCompilerModule(
  {
  compilerPath: emberTemplateCompilerPath(),
+ compilerChecksum: `mock-compiler-checksum${Math.random()}`,
  EmberENV: {},
  plugins: { ast: [] },
  resolver: new CompatResolver({

packages/compat/tests/resolver.test.ts

@@ -10,6 +10,7 @@ import Resolver from '../src/resolver';
 import { PackageRules } from '../src';
 
 const compilerPath = emberTemplateCompilerPath();
+const compilerChecksum = `mock-compiler-checksum${Math.random()}`;
 
 describe('compat-resolver', function () {
  let appDir: string;
@@ -45,7 +46,7 @@ describe('compat-resolver', function () {
  otherOptions.adjustImportsImports
  ),
  });
- let compiler = new NodeTemplateCompiler({ compilerPath, resolver, EmberENV, plugins });
+ let compiler = new NodeTemplateCompiler({ compilerPath, compilerChecksum, resolver, EmberENV, plugins });
  return function (relativePath: string, contents: string) {
  let moduleName = givenFile(relativePath);
  let { dependencies } = compiler.precompile(moduleName, contents);

packages/core/src/app.ts

@@ -37,6 +37,7 @@ import cloneDeep from 'lodash/cloneDeep';
 import type { Params as InlineBabelParams } from './babel-plugin-inline-hbs-node';
 import { PortableHint } from './portable';
 import escapeRegExp from 'escape-string-regexp';
+import { getEmberExports } from './load-ember-template-compiler';
 
 export type EmberENV = unknown;
 
@@ -898,9 +899,13 @@ export class AppBuilder<TreeNames> {
  plugins.ast.push(macroPlugin);
  }
 
+ const compilerPath = resolve.sync(this.adapter.templateCompilerPath(), { basedir: this.root });
+ const compilerChecksum = getEmberExports(compilerPath).cacheKey;
+
  return {
  plugins,
- compilerPath: resolve.sync(this.adapter.templateCompilerPath(), { basedir: this.root }),
+ compilerPath,
+ compilerChecksum,
  resolver: this.adapter.templateResolver(),
  EmberENV: config,
  };

packages/core/src/load-ember-template-compiler.ts

@@ -34,34 +34,39 @@ export function getEmberExports(templateCompilerPath: string): EmbersExports {
  let stat = statSync(templateCompilerPath);
 
  let source = patch(readFileSync(templateCompilerPath, 'utf8'), templateCompilerPath);
-
- // matches (essentially) what ember-cli-htmlbars does in https://git.io/Jtbpj
- let sandbox = {
- module: { require, exports: {} },
- require,
- };
- if (typeof globalThis === 'undefined') {
- // for Node 10 usage with Ember 3.27+ we have to define the `global` global
- // in order for ember-template-compiler.js to evaluate properly
- // due to this code https://git.io/Jtb7
- (sandbox as any).global = sandbox;
- }
-
- // using vm.createContext / vm.Script to ensure we evaluate in a fresh sandbox context
- // so that any global mutation done within ember-template-compiler.js does not leak out
- let context = createContext(sandbox);
- let script = new Script(source, { filename: templateCompilerPath });
-
- script.runInContext(context);
- let theExports: any = context.module.exports;
+ let theExports: any = undefined;
 
  // cacheKey, theExports
  let cacheKey = createHash('md5').update(source).digest('hex');
 
  entry = Object.freeze({
  value: {
  cacheKey,
- theExports,
+ get theExports() {
+ if (theExports) {
+ return theExports;
+ }
+
+ // matches (essentially) what ember-cli-htmlbars does in https://git.io/Jtbpj
+ let sandbox = {
+ module: { require, exports: {} },
+ require,
+ };
+
+ if (typeof globalThis === 'undefined') {
+ // for Node 10 usage with Ember 3.27+ we have to define the `global` global
+ // in order for ember-template-compiler.js to evaluate properly
+ // due to this code https://git.io/Jtb7
+ (sandbox as any).global = sandbox;
+ }
+ // using vm.createContext / vm.Script to ensure we evaluate in a fresh sandbox context
+ // so that any global mutation done within ember-template-compiler.js does not leak out
+ let context = createContext(sandbox);
+ let script = new Script(source, { filename: templateCompilerPath });
+
+ script.runInContext(context);
+ return (theExports = context.module.exports);
+ },
  },
  stat, // This is stored, so we can reload the templateCompiler if it changes mid-build.
  });

packages/core/src/template-compiler-node.ts

@@ -9,6 +9,7 @@ import adjustImportsPlugin from './babel-plugin-adjust-imports';
 
 export interface NodeTemplateCompilerParams {
  compilerPath: string;
+ compilerChecksum: string;
  resolver?: Resolver;
  EmberENV: unknown;
  plugins: Plugins;

packages/core/tests/inline-hbs.test.ts

@@ -82,6 +82,7 @@ describe('inline-hbs', () => {
  babelConfig() {
  let templateCompiler: NodeTemplateCompilerParams = {
  compilerPath: emberTemplateCompilerPath(),
+ compilerChecksum: `mock-compiler-checksum${Math.random()}`,
  EmberENV: {},
  plugins: {
  ast: [sampleTransform],
@@ -105,6 +106,7 @@ describe('inline-hbs', () => {
  babelConfig() {
  let templateCompiler: NodeTemplateCompilerParams = {
  compilerPath: emberTemplateCompilerPath(),
+ compilerChecksum: `mock-compiler-checksum${Math.random()}`,
  EmberENV: {},
  plugins: {
  ast: [],
@@ -128,6 +130,7 @@ describe('inline-hbs', () => {
  babelConfig(major: number) {
  let templateCompiler: NodeTemplateCompilerParams = {
  compilerPath: emberTemplateCompilerPath(),
+ compilerChecksum: `mock-compiler-checksum${Math.random()}`,
  EmberENV: {},
  plugins: {
  ast: [],

packages/macros/tests/glimmer/helpers.ts

@@ -1,8 +1,11 @@
 import { NodeTemplateCompiler } from '@embroider/core';
+import { getEmberExports } from '@embroider/core/src/load-ember-template-compiler';
 import { emberTemplateCompilerPath, Project } from '@embroider/test-support';
 import { MacrosConfig } from '../../src/node';
 import { join } from 'path';
+
 const compilerPath = emberTemplateCompilerPath();
+const { cacheKey: compilerChecksum } = getEmberExports(compilerPath);
 
 export { Project };
 
@@ -19,6 +22,7 @@ export function templateTests(createTests: CreateTestsWithConfig | CreateTests)
  setConfig(config);
  let compiler = new NodeTemplateCompiler({
  compilerPath,
+ compilerChecksum,
  EmberENV: {},
  plugins: {
  ast: plugins,

packages/webpack/src/ember-webpack.ts

@@ -463,7 +463,7 @@ const Webpack: PackagerConstructor<Options> = class Webpack implements Packager
  this.consoleWrite(stats.toString());
  }
  resolve(stats.toJson());
- } catch(e) {
+ } catch (e) {
  reject(e);
  }
  });