From 3607e849af3b243aa15e156a3abab46502f61814 Mon Sep 17 00:00:00 2001 From: Robert Jackson Date: Fri, 26 Feb 2021 11:23:34 -0500 Subject: [PATCH 1/6] Refactor template compiler patches * extract a helper for introspecting ember version from the source comment * makes it clearer what changes are needed for each Ember version --- packages/core/src/patch-template-compiler.ts | 244 +++++++++++-------- 1 file changed, 149 insertions(+), 95 deletions(-) diff --git a/packages/core/src/patch-template-compiler.ts b/packages/core/src/patch-template-compiler.ts index b653169ea..a62a0fc33 100644 --- a/packages/core/src/patch-template-compiler.ts +++ b/packages/core/src/patch-template-compiler.ts @@ -15,124 +15,178 @@ import { import { NodePath } from '@babel/traverse'; import { transform } from '@babel/core'; -export function patch(source: string, templateCompilerPath: string) { +function emberVersionGte(templateCompilerPath: string, source: string, major: number, minor: number): boolean { + // ember-template-compiler.js contains a comment that indicates what version it is for + // that looks like: + + /*! + * @overview Ember - JavaScript Application Framework + * @copyright Copyright 2011-2020 Tilde Inc. and contributors + * Portions Copyright 2006-2011 Strobe Inc. + * Portions Copyright 2008-2011 Apple Inc. All rights reserved. + * @license Licensed under MIT license + * See https://raw.github.com/emberjs/ember.js/master/LICENSE + * @version 3.25.1 + */ + + let version = source.match(/@version\s+([\d\.]+)/); + if (!version || !version[1]) { + throw new Error( + `Could not find version string in \`${templateCompilerPath}\`. Maybe we don't support your ember-source version?` + ); + } + + let numbers = version[1].split('.'); + let actualMajor = parseInt(numbers[0], 10); + let actualMinor = parseInt(numbers[1], 10); + + return actualMajor > major || (actualMajor === major && actualMinor >= minor); +} + +export function patch(source: string, templateCompilerPath: string): string { let replacedVar = false; + let patchedSource; - // patch applies to ember 3.12 through 3.16. The template compiler contains a - // comment with the version. - let needsPatch = /@version\s+3\.1[23456][^\d]/.test(source); + let needsAngleBracketPrinterFix = + emberVersionGte(templateCompilerPath, source, 3, 12) && !emberVersionGte(templateCompilerPath, source, 3, 17); - // here we are stripping off the first `var Ember;`. That one small change - // lets us crack open the file and get access to its internal loader, because - // we can give it our own predefined `Ember` variable instead, which it will - // use and put `Ember.__loader` onto. - // - // on ember 3.12 through 3.16 (which use variants of glimmer-vm 0.38.5) we - // also apply a patch to the printer in @glimmer/syntax to fix - // https://github.com/glimmerjs/glimmer-vm/pull/941/files because it can - // really bork apps under embroider, and we'd like to support at least all - // active LTS versions of ember. - let patchedSource = transform(source, { - plugins: [ - function () { - return { - visitor: { - VariableDeclarator(path: NodePath) { - let id = path.node.id; - if (id.type === 'Identifier' && id.name === 'Ember' && !replacedVar) { - replacedVar = true; - path.remove(); - } - }, - CallExpression: { - enter(path: NodePath, state: BabelState) { - if (!needsPatch) { - return; - } - let callee = path.get('callee'); - if (!callee.isIdentifier() || callee.node.name !== 'define') { - return; + if (needsAngleBracketPrinterFix) { + // here we are stripping off the first `var Ember;`. That one small change + // lets us crack open the file and get access to its internal loader, because + // we can give it our own predefined `Ember` variable instead, which it will + // use and put `Ember.__loader` onto. + // + // on ember 3.12 through 3.16 (which use variants of glimmer-vm 0.38.5) we + // also apply a patch to the printer in @glimmer/syntax to fix + // https://github.com/glimmerjs/glimmer-vm/pull/941/files because it can + // really bork apps under embroider, and we'd like to support at least all + // active LTS versions of ember. + patchedSource = transform(source, { + plugins: [ + function () { + return { + visitor: { + VariableDeclarator(path: NodePath) { + let id = path.node.id; + if (id.type === 'Identifier' && id.name === 'Ember' && !replacedVar) { + replacedVar = true; + path.remove(); } - let firstArg = path.get('arguments')[0]; - if (!firstArg.isStringLiteral() || firstArg.node.value !== '@glimmer/syntax') { - return; - } - state.definingGlimmerSyntax = path; }, - exit(path: NodePath, state: BabelState) { - if (state.definingGlimmerSyntax === path) { - state.definingGlimmerSyntax = false; - } + CallExpression: { + enter(path: NodePath, state: BabelState) { + let callee = path.get('callee'); + if (!callee.isIdentifier() || callee.node.name !== 'define') { + return; + } + let firstArg = path.get('arguments')[0]; + if (!firstArg.isStringLiteral() || firstArg.node.value !== '@glimmer/syntax') { + return; + } + state.definingGlimmerSyntax = path; + }, + exit(path: NodePath, state: BabelState) { + if (state.definingGlimmerSyntax === path) { + state.definingGlimmerSyntax = false; + } + }, }, - }, - FunctionDeclaration: { - enter(path: NodePath, state: BabelState) { - if (!state.definingGlimmerSyntax) { - return; - } - let id = path.get('id'); - if (id.isIdentifier() && id.node.name === 'build') { - state.declaringBuildFunction = path; - } + FunctionDeclaration: { + enter(path: NodePath, state: BabelState) { + if (!state.definingGlimmerSyntax) { + return; + } + let id = path.get('id'); + if (id.isIdentifier() && id.node.name === 'build') { + state.declaringBuildFunction = path; + } + }, + exit(path: NodePath, state: BabelState) { + if (state.declaringBuildFunction === path) { + state.declaringBuildFunction = false; + } + }, }, - exit(path: NodePath, state: BabelState) { - if (state.declaringBuildFunction === path) { - state.declaringBuildFunction = false; - } + SwitchCase: { + enter(path: NodePath, state: BabelState) { + if (!state.definingGlimmerSyntax) { + return; + } + let test = path.get('test'); + if (test.isStringLiteral() && test.node.value === 'ElementNode') { + state.caseElementNode = path; + } + }, + exit(path: NodePath, state: BabelState) { + if (state.caseElementNode === path) { + state.caseElementNode = false; + } + }, }, - }, - SwitchCase: { - enter(path: NodePath, state: BabelState) { - if (!state.definingGlimmerSyntax) { + IfStatement(path: NodePath, state: BabelState) { + if (!state.caseElementNode) { return; } let test = path.get('test'); - if (test.isStringLiteral() && test.node.value === 'ElementNode') { - state.caseElementNode = path; + // the place we want is the only if with a computed member + // expression predicate. + if (test.isMemberExpression() && test.node.computed) { + path.node.alternate = ifStatement( + memberExpression(identifier('ast'), identifier('selfClosing')), + blockStatement([ + expressionStatement( + callExpression(memberExpression(identifier('output'), identifier('push')), [ + stringLiteral(' />'), + ]) + ), + ]), + path.node.alternate + ); } }, - exit(path: NodePath, state: BabelState) { - if (state.caseElementNode === path) { - state.caseElementNode = false; + }, + }; + }, + ], + })!.code!; + } else { + // applies to < 3.12 and >= 3.17 + // + // here we are stripping off the first `var Ember;`. That one small change + // lets us crack open the file and get access to its internal loader, because + // we can give it our own predefined `Ember` variable instead, which it will + // use and put `Ember.__loader` onto. + patchedSource = transform(source, { + plugins: [ + function () { + return { + visitor: { + VariableDeclarator(path: NodePath) { + let id = path.node.id; + if (id.type === 'Identifier' && id.name === 'Ember' && !replacedVar) { + replacedVar = true; + path.remove(); } }, }, - IfStatement(path: NodePath, state: BabelState) { - if (!state.caseElementNode) { - return; - } - let test = path.get('test'); - // the place we want is the only if with a computed member - // expression predicate. - if (test.isMemberExpression() && test.node.computed) { - path.node.alternate = ifStatement( - memberExpression(identifier('ast'), identifier('selfClosing')), - blockStatement([ - expressionStatement( - callExpression(memberExpression(identifier('output'), identifier('push')), [stringLiteral(' />')]) - ), - ]), - path.node.alternate - ); - } - }, - }, - }; - }, - ], - })!.code!; + }; + }, + ], + })!.code!; + } if (!replacedVar) { throw new Error( `didn't find expected source in ${templateCompilerPath}. Maybe we don't support your ember-source version?` ); } + return ` - let module = { exports: {} }; - let Ember = {}; - ${patchedSource}; - module.exports.Ember = Ember; - return module.exports + let module = { exports: {} }; + let Ember = {}; + ${patchedSource}; + module.exports.Ember = Ember; + return module.exports `; } From 63554dcc8c3ebc7c0a7c542022957acc5b57e35d Mon Sep 17 00:00:00 2001 From: Robert Jackson Date: Fri, 26 Feb 2021 11:25:24 -0500 Subject: [PATCH 2/6] Use vm context to avoid global mutation during template compiler evaluation --- packages/core/src/patch-template-compiler.ts | 2 -- packages/core/src/template-compiler.ts | 23 +++++++++++++++++++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/packages/core/src/patch-template-compiler.ts b/packages/core/src/patch-template-compiler.ts index a62a0fc33..698d9e19f 100644 --- a/packages/core/src/patch-template-compiler.ts +++ b/packages/core/src/patch-template-compiler.ts @@ -182,11 +182,9 @@ export function patch(source: string, templateCompilerPath: string): string { } return ` - let module = { exports: {} }; let Ember = {}; ${patchedSource}; module.exports.Ember = Ember; - return module.exports `; } diff --git a/packages/core/src/template-compiler.ts b/packages/core/src/template-compiler.ts index 80c8134fa..672c90d38 100644 --- a/packages/core/src/template-compiler.ts +++ b/packages/core/src/template-compiler.ts @@ -12,6 +12,7 @@ import wrapLegacyHbsPluginIfNeeded from 'wrap-legacy-hbs-plugin-if-needed'; import { patch } from './patch-template-compiler'; import { Portable, PortableHint } from './portable'; import type { Params as InlineBabelParams } from './babel-plugin-inline-hbs'; +import { createContext, Script } from 'vm'; export interface Plugins { ast?: unknown[]; @@ -94,7 +95,26 @@ function getEmberExports(templateCompilerPath: string): EmbersExports { let stat = statSync(templateCompilerPath); let source = patch(readFileSync(templateCompilerPath, 'utf8'), templateCompilerPath); - let theExports: any = new Function(source)(); + + // matches (essentially) what ember-cli-htmlbars does in https://git.io/Jtbpj + let sandbox = { + module: { require, exports: {} }, + require, + }; + if (typeof globalThis === 'undefined') { + // for Node 10 usage with Ember 3.27+ we have to define the `global` global + // in order for ember-template-compiler.js to evaluate properly + // due to this code https://git.io/Jtb7s + (sandbox as any).global = sandbox; + } + + // using vm.createContext / vm.Script to ensure we evaluate in a fresh sandbox context + // so that any global mutation done within ember-template-compiler.js does not leak out + let context = createContext(sandbox); + let script = new Script(source, { filename: templateCompilerPath }); + + script.runInContext(context); + let theExports: any = context.module.exports; // cacheKey, theExports let cacheKey = createHash('md5').update(source).digest('hex'); @@ -349,6 +369,7 @@ function hasProperties(item: any) { return item && (typeof item === 'object' || typeof item === 'function'); } +// this matches the setup done by ember-cli-htmlbars: https://git.io/JtbN6 function initializeEmberENV(syntax: GlimmerSyntax, EmberENV: any) { if (!EmberENV) { return; From 35e753f121756216b3d57d462e99d71b956ac6ec Mon Sep 17 00:00:00 2001 From: Robert Jackson Date: Fri, 26 Feb 2021 11:56:16 -0500 Subject: [PATCH 3/6] Avoid monkey patch template compiler on Ember >= 3.27 New API exposed in https://github.com/emberjs/ember.js/pull/19426 enables us to avoid munging with the template compiler internals. --- packages/core/src/patch-template-compiler.ts | 5 ++ packages/core/src/template-compiler.ts | 55 ++++++++++++-------- 2 files changed, 39 insertions(+), 21 deletions(-) diff --git a/packages/core/src/patch-template-compiler.ts b/packages/core/src/patch-template-compiler.ts index 698d9e19f..81ba8ba03 100644 --- a/packages/core/src/patch-template-compiler.ts +++ b/packages/core/src/patch-template-compiler.ts @@ -44,6 +44,11 @@ function emberVersionGte(templateCompilerPath: string, source: string, major: nu } export function patch(source: string, templateCompilerPath: string): string { + if (emberVersionGte(templateCompilerPath, source, 3, 27)) { + // no modifications are needed after https://github.com/emberjs/ember.js/pull/19426 + return source; + } + let replacedVar = false; let patchedSource; diff --git a/packages/core/src/template-compiler.ts b/packages/core/src/template-compiler.ts index 672c90d38..8c1d734ca 100644 --- a/packages/core/src/template-compiler.ts +++ b/packages/core/src/template-compiler.ts @@ -68,14 +68,6 @@ type TemplateCompilerCacheEntry = { const CACHE = new Map(); -// Today the template compiler seems to not expose a public way to to source 2 source compilation of templates. -// because of this, we must resort to some hackery. -// -// TODO: expose a way to accomplish this via purely public API's. -// Today we use the following API's -// * glimmer/syntax's preprocess -// * glimmer/syntax's print -// * ember-template-compiler/lib/system/compile-options's defaultOptions function getEmberExports(templateCompilerPath: string): EmbersExports { let entry = CACHE.get(templateCompilerPath); @@ -138,19 +130,40 @@ function getEmberExports(templateCompilerPath: string): EmbersExports { function loadGlimmerSyntax(templateCompilerPath: string): GlimmerSyntax { let { theExports, cacheKey } = getEmberExports(templateCompilerPath); - // TODO: we should work to make this, or what it intends to accomplish, public API - let syntax = theExports.Ember.__loader.require('@glimmer/syntax'); - let compilerOptions = theExports.Ember.__loader.require('ember-template-compiler/lib/system/compile-options'); - - return { - print: syntax.print, - preprocess: syntax.preprocess, - defaultOptions: compilerOptions.default, - registerPlugin: compilerOptions.registerPlugin, - precompile: theExports.precompile, - _Ember: theExports._Ember, - cacheKey, - }; + // detect if we are using an Ember version with the exports we need + // (from https://github.com/emberjs/ember.js/pull/19426) + if (theExports._preprocess !== undefined) { + return { + print: theExports._print, + preprocess: theExports._preprocess, + defaultOptions: theExports.compileOptions, + registerPlugin: theExports.registerPlugin, + precompile: theExports.precompile, + _Ember: theExports._Ember, + cacheKey, + }; + } else { + // Older Ember versions (prior to 3.27) do not expose a public way to to source 2 source compilation of templates. + // because of this, we must resort to some hackery. + // + // We use the following API's (that we grab from Ember.__loader): + // + // * glimmer/syntax's preprocess + // * glimmer/syntax's print + // * ember-template-compiler/lib/system/compile-options's defaultOptions + let syntax = theExports.Ember.__loader.require('@glimmer/syntax'); + let compilerOptions = theExports.Ember.__loader.require('ember-template-compiler/lib/system/compile-options'); + + return { + print: syntax.print, + preprocess: syntax.preprocess, + defaultOptions: compilerOptions.default, + registerPlugin: compilerOptions.registerPlugin, + precompile: theExports.precompile, + _Ember: theExports._Ember, + cacheKey, + }; + } } export function templateCompilerModule(params: TemplateCompilerParams, hints: PortableHint[]) { From a670a394cbcdf9508319db2da7432a1e1ddcf131 Mon Sep 17 00:00:00 2001 From: Robert Jackson Date: Fri, 26 Feb 2021 12:17:49 -0500 Subject: [PATCH 4/6] Remove references to registerPlugin in template compiler API. `registerPlugin` / `unregisterPlugin` have been deprecated (they rely on global mutation, and are slated for removal in Ember 4.0). --- packages/core/src/template-compiler.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/core/src/template-compiler.ts b/packages/core/src/template-compiler.ts index 8c1d734ca..18710775c 100644 --- a/packages/core/src/template-compiler.ts +++ b/packages/core/src/template-compiler.ts @@ -36,7 +36,6 @@ interface GlimmerSyntax { preprocess(html: string, options?: PreprocessOptions): AST; print(ast: AST): string; defaultOptions(options: PreprocessOptions): PreprocessOptions; - registerPlugin(type: string, plugin: unknown): void; precompile( templateContents: string, options: { @@ -137,7 +136,6 @@ function loadGlimmerSyntax(templateCompilerPath: string): GlimmerSyntax { print: theExports._print, preprocess: theExports._preprocess, defaultOptions: theExports.compileOptions, - registerPlugin: theExports.registerPlugin, precompile: theExports.precompile, _Ember: theExports._Ember, cacheKey, @@ -158,7 +156,6 @@ function loadGlimmerSyntax(templateCompilerPath: string): GlimmerSyntax { print: syntax.print, preprocess: syntax.preprocess, defaultOptions: compilerOptions.default, - registerPlugin: compilerOptions.registerPlugin, precompile: theExports.precompile, _Ember: theExports._Ember, cacheKey, From 1a90cecea77fff3c1abd1210eb49f82e7483774b Mon Sep 17 00:00:00 2001 From: Robert Jackson Date: Fri, 26 Feb 2021 13:22:56 -0500 Subject: [PATCH 5/6] Update TemplateCompiler#applyTransforms to preserve entity encoding This does two main things: * removes processing of HTML entities * disables Handlebars whitespace removal for certain contexts These changes are important because it allows our source -> source transformation to remain stable in more circumstances. --- packages/core/src/template-compiler.ts | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/packages/core/src/template-compiler.ts b/packages/core/src/template-compiler.ts index 18710775c..a0a264be3 100644 --- a/packages/core/src/template-compiler.ts +++ b/packages/core/src/template-compiler.ts @@ -27,6 +27,22 @@ interface PreprocessOptions { moduleName: string; plugins?: Plugins; filename?: string; + + parseOptions?: { + srcName?: string; + ignoreStandalone?: boolean; + }; + + // added in Ember 3.17 (@glimmer/syntax@0.40.2) + mode?: 'codemod' | 'precompile'; + + // added in Ember 3.25 + strictMode?: boolean; + locals?: string[]; +} + +interface PrinterOptions { + entityEncoding?: 'transformed' | 'raw'; } // This just reflects the API we're extracting from ember-template-compiler.js, @@ -34,7 +50,7 @@ interface PreprocessOptions { // stable. interface GlimmerSyntax { preprocess(html: string, options?: PreprocessOptions): AST; - print(ast: AST): string; + print(ast: AST, options?: PrinterOptions): string; defaultOptions(options: PreprocessOptions): PreprocessOptions; precompile( templateContents: string, @@ -289,12 +305,16 @@ export class TemplateCompiler { }); } + // instructs glimmer-vm to preserve entity encodings (e.g. don't parse   -> ' ') + opts.mode = 'codemod'; + opts.filename = moduleName; opts.moduleName = this.params.resolver ? this.params.resolver.absPathToRuntimePath(moduleName) || moduleName : moduleName; let ast = this.syntax.preprocess(contents, opts); - return this.syntax.print(ast); + + return this.syntax.print(ast, { entityEncoding: 'raw' }); } parse(moduleName: string, contents: string): AST { From e6d4d812db9956e44224a82fe768973f77673a9d Mon Sep 17 00:00:00 2001 From: Robert Jackson Date: Fri, 26 Feb 2021 13:24:43 -0500 Subject: [PATCH 6/6] Use ember-template-compiler.js's `_buildCompileOptions` when possible This API was introduced in https://github.com/emberjs/ember.js/pull/19426 and allows us to avoid Ember's own AST transform plugins. The changes here are safe across all Ember version ranges (in both cases applyTransforms avoids running Ember's own transforms). --- packages/core/src/template-compiler.ts | 41 ++++++++++++++------------ 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/packages/core/src/template-compiler.ts b/packages/core/src/template-compiler.ts index a0a264be3..9f5f58da9 100644 --- a/packages/core/src/template-compiler.ts +++ b/packages/core/src/template-compiler.ts @@ -151,7 +151,7 @@ function loadGlimmerSyntax(templateCompilerPath: string): GlimmerSyntax { return { print: theExports._print, preprocess: theExports._preprocess, - defaultOptions: theExports.compileOptions, + defaultOptions: theExports._buildCompileOptions, precompile: theExports.precompile, _Ember: theExports._Ember, cacheKey, @@ -248,11 +248,14 @@ export class TemplateCompiler { let opts = this.syntax.defaultOptions({ contents, moduleName }); let plugins: Plugins = { - ...opts.plugins, + ...opts?.plugins, + ast: [ ...this.getReversedASTPlugins(this.params.plugins.ast!), this.params.resolver && this.params.resolver.astTransformer(this), - ...opts.plugins!.ast!, + + // Ember 3.27+ uses _buildCompileOptions will not add AST plugins to its result + ...(opts?.plugins?.ast ?? []), ].filter(Boolean), }; @@ -287,23 +290,23 @@ export class TemplateCompiler { // Applies all custom AST transforms and emits the results still as // handlebars. - applyTransforms(moduleName: string, contents: string) { + applyTransforms(moduleName: string, contents: string): string { let opts = this.syntax.defaultOptions({ contents, moduleName }); - if (opts.plugins && opts.plugins.ast) { - // the user-provided plugins come first in the list, and those are the - // only ones we want to run. The built-in plugins don't need to run here - // in stage1, it's better that they run in stage3 when the appropriate - // ember version is in charge. - // - // rather than slicing them off, we could choose instead to not call - // syntax.defaultOptions, but then we lose some of the compatibility - // normalization that it does on the user-provided plugins. - opts.plugins.ast = this.getReversedASTPlugins(this.params.plugins.ast!).map(plugin => { - // Although the precompile API does, this direct glimmer syntax api - // does not support these legacy plugins, so we must wrap them. - return wrapLegacyHbsPluginIfNeeded(plugin as any); - }); - } + + // the user-provided plugins come first in the list, and those are the + // only ones we want to run. The built-in plugins don't need to run here + // in stage1, it's better that they run in stage3 when the appropriate + // ember version is in charge. + // + // rather than slicing them off, we could choose instead to not call + // syntax.defaultOptions, but then we lose some of the compatibility + // normalization that it does on the user-provided plugins. + opts.plugins = opts.plugins || {}; // Ember 3.27+ won't add opts.plugins + opts.plugins.ast = this.getReversedASTPlugins(this.params.plugins.ast!).map(plugin => { + // Although the precompile API does, this direct glimmer syntax api + // does not support these legacy plugins, so we must wrap them. + return wrapLegacyHbsPluginIfNeeded(plugin as any); + }); // instructs glimmer-vm to preserve entity encodings (e.g. don't parse   -> ' ') opts.mode = 'codemod';