Allow AST plugins' output be cached based on external files

.eslintrc.js

@@ -25,6 +25,7 @@ module.exports = {
         'ember-cli-build.js',
         'ember-addon-main.js',
         'index.js',
+        'addDependencyTracker.js',
         'utils.js',
         'testem.js',
         'blueprints/*/index.js',

README.md

@@ -41,6 +41,55 @@ module.exports = {
 };
 ```
 
+#### Options for registering a `htmlbars-ast-plugin`
+
+* `name` - String. The name of the AST transform for debugging purposes.
+* `plugin` - A function of type [`ASTPluginBuilder`](https://github.com/glimmerjs/glimmer-vm/blob/master/packages/%40glimmer/syntax/lib/parser/tokenizer-event-handlers.ts#L329-L341).
+* `dependencyInvalidation` - Boolean. A flag that indicates the AST Plugin may, on a per-template basis, depend on other files that affect its output.
+* `cacheKey` - function that returns any JSON-compatible value - The value returned is used to invalidate the persistent cache across restarts, usually in the case of a dependency or configuration change.
+* `baseDir` - `() => string`. A function that returns the directory on disk of the npm module for the plugin. If provided, a basic cache invalidation is performed if any of the dependencies change (e.g. due to a npm install/upgrade).
+
+#### Implementing Dependency Invalidation in an AST Plugin
+
+Plugins that set the `dependencyInvalidation` option to `true` can provide function for the `plugin` of type `ASTDependencyPlugin` as given below.
+
+Note: the `plugin` function is invoked without a value for `this` in context.
+
+```ts
+import {ASTPluginBuilder, ASTPlugin} from "@glimmer/syntax/dist/types/lib/parser/tokenizer-event-handlers";
+
+export type ASTDependencyPlugin = ASTPluginWithDepsBuilder | ASTPluginBuilderWithDeps;
+
+export interface ASTPluginWithDepsBuilder {
+  (env: ASTPluginEnvironment): ASTPluginWithDeps;
+}
+
+export interface ASTPluginBuilderWithDeps extends ASTPluginBuilder {
+  /**
+   * @see {ASTPluginWithDeps.dependencies} below.
+   **/
+  dependencies(relativePath): string[];
+}
+
+export interface ASTPluginWithDeps extends ASTPlugin {
+  /**
+   * If this method exists, it is called with the relative path to the current
+   * file just before processing starts. Use this method to reset the
+   * dependency tracking state associated with the file.
+   */
+  resetDependencies?(relativePath: string): void;
+  /**
+   * This method is called just as the template finishes being processed.
+   *
+   * @param relativePath {string} A relative path to the file that may have dependencies.
+   * @return {string[]} paths to files that are a dependency for the given
+   * file. Any relative paths returned by this method are taken to be relative
+   * to the file that was processed.
+   */
+  dependencies(relativePath: string): string[];
+}
+```
+
 ### Precompile HTMLBars template strings within other addons
 
 ```javascript

addDependencyTracker.js

@@ -0,0 +1,69 @@
+function addDependencyTracker(plugin, enableInvalidation) {
+  if (plugin.prototype && plugin.prototype.transform) {
+    // we don't track dependencies for legacy plugins.
+    return plugin;
+  }
+  if (!enableInvalidation) {
+    // Dependency invalidation isn't enabled, so no need to track.
+    return plugin;
+  }
+  if (typeof plugin.getDependencies === "function") {
+    // Don't add the dependency tracker if it's already added
+    return plugin;
+  }
+  // This variable in our closure allows us to share dependencies from
+  // the ast plugin that we can't access with the ast plugin generator that
+  // we can access.
+  let lastDependencies = {};
+  let trackedPlugin = (env) => {
+    let templateStackDepth = 0;
+    let realPlugin = plugin(env);
+    let visitor = realPlugin.visitor;
+    let origProgram = visitor.Program;
+    let origEnter, origExit, origKeys;
+    if (origProgram) {
+      if (typeof origProgram === "function") {
+        origEnter = origProgram;
+        origExit = undefined;
+        origKeys = undefined;
+      } else {
+        origEnter = origProgram.enter;
+        origExit = origProgram.exit;
+        origKeys = origProgram.keys;
+      }
+    }
+    // Ideally we'd use visitor.Template but we still support versions of
+    // handlebars where the template node didn't exist yet. Templates can have
+    // nested Program nodes. Since we can't rely on the Template node yet, we
+    // have to keep a stack counter of Program nodes that we've seen so far.
+    visitor.Program = {
+      keys: origKeys,
+      enter: (node) => {
+        templateStackDepth++;
+        if (templateStackDepth === 1) {
+          if (realPlugin.resetDependencies) {
+            realPlugin.resetDependencies(env.meta.moduleName);
+          }
+          delete lastDependencies[env.meta.moduleName];
+        }
+        if (origEnter) origEnter(node);
+      },
+      exit: (node) => {
+        if (templateStackDepth === 1) {
+          if (realPlugin.dependencies) {
+            lastDependencies[env.meta.moduleName] = realPlugin.dependencies(env.meta.moduleName);
+          }
+        }
+        if (origExit) origExit(node)
+        templateStackDepth--;
+      }
+    };
+    return realPlugin;
+  };
+  trackedPlugin.getDependencies = (relativePath) => {
+    return lastDependencies[relativePath] || [];
+  }
+  return trackedPlugin;
+}
+
+module.exports = addDependencyTracker;

ember-addon-main.js

@@ -2,6 +2,7 @@
 
 const path = require('path');
 const utils = require('./utils');
+const addDependencyTracker = require("./addDependencyTracker");
 const hashForDep = require('hash-for-dep');
 
 module.exports = {
@@ -106,6 +107,8 @@ module.exports = {
         ast: pluginInfo.plugins
       },
 
+      dependencyInvalidation: pluginInfo.dependencyInvalidation,
+
       pluginCacheKey: pluginInfo.cacheKeys
     };
 
@@ -121,16 +124,18 @@ module.exports = {
     let pluginWrappers = this.parentRegistry.load('htmlbars-ast-plugin');
     let plugins = [];
     let cacheKeys = [];
+    let dependencyInvalidation = false;
 
     for (let i = 0; i < pluginWrappers.length; i++) {
       let wrapper = pluginWrappers[i];
+      dependencyInvalidation = dependencyInvalidation || wrapper.dependencyInvalidation;
+      plugins.push(addDependencyTracker(wrapper.plugin, wrapper.dependencyInvalidation));
 
-      plugins.push(wrapper.plugin);
 
       let providesBaseDir = typeof wrapper.baseDir === 'function';
       let augmentsCacheKey = typeof wrapper.cacheKey === 'function';
 
-      if (providesBaseDir || augmentsCacheKey) {
+      if (providesBaseDir || augmentsCacheKey || wrapper.dependencyInvalidation) {
         if (providesBaseDir) {
           let pluginHashForDep = hashForDep(wrapper.baseDir());
           cacheKeys.push(pluginHashForDep);
@@ -147,7 +152,8 @@ module.exports = {
 
     return {
       plugins: plugins,
-      cacheKeys: cacheKeys
+      cacheKeys: cacheKeys,
+      dependencyInvalidation: dependencyInvalidation,
     };
   }
 };

index.js

@@ -1,6 +1,7 @@
 'use strict';
 
 const fs = require('fs');
+const path = require('path');
 const utils = require('./utils');
 const Filter = require('broccoli-persistent-filter');
 const crypto = require('crypto');
@@ -36,6 +37,7 @@ class TemplateCompiler extends Filter {
 
     this.precompile = this.options.templateCompiler.precompile;
     this.registerPlugin = this.options.templateCompiler.registerPlugin;
+    this.unregisterPlugin = this.options.templateCompiler.unregisterPlugin;
 
     this.registerPlugins();
     this.initializeFeatures();
@@ -45,6 +47,13 @@ class TemplateCompiler extends Filter {
     return __dirname;
   }
 
+  registeredASTPlugins() {
+    // This is a super obtuse way to get access to the plugins we've registered
+    // it also returns other plugins that are registered by ember itself.
+    let options = this.options.templateCompiler.compileOptions();
+    return options.plugins && options.plugins.ast || [];
+  }
+
   registerPlugins() {
     let plugins = this.options.plugins;
 
@@ -56,6 +65,17 @@ class TemplateCompiler extends Filter {
       }
     }
   }
+  unregisterPlugins() {
+    let plugins = this.options.plugins;
+
+    if (plugins) {
+      for (let type in plugins) {
+        for (let i = 0, l = plugins[type].length; i < l; i++) {
+          this.unregisterPlugin(type, plugins[type][i]);
+        }
+      }
+    }
+  }
 
   initializeFeatures() {
     let EmberENV = this.options.EmberENV;
@@ -73,11 +93,26 @@ class TemplateCompiler extends Filter {
   }
 
   processString(string, relativePath) {
+    let srcDir = this.inputPaths[0];
+    let srcName = path.join(srcDir, relativePath);
     try {
-      return 'export default ' + utils.template(this.options.templateCompiler, stripBom(string), {
+      let result = 'export default ' + utils.template(this.options.templateCompiler, stripBom(string), {
         contents: string,
-        moduleName: relativePath
+        moduleName: relativePath,
+        parseOptions: {
+          srcName: srcName
+        }
       }) + ';';
+      if (this.options.dependencyInvalidation) {
+        let plugins = pluginsWithDependencies(this.registeredASTPlugins());
+        let dependencies = [];
+        for (let i = 0; i < plugins.length; i++) {
+          let pluginDeps = plugins[i].getDependencies(relativePath);
+          dependencies = dependencies.concat(pluginDeps);
+        }
+        this.dependencies.setDependencies(relativePath, dependencies);
+      }
+      return result;
     } catch(error) {
       rethrowBuildError(error);
     }
@@ -122,4 +157,14 @@ class TemplateCompiler extends Filter {
 TemplateCompiler.prototype.extensions = ['hbs', 'handlebars'];
 TemplateCompiler.prototype.targetExtension = 'js';
 
+function pluginsWithDependencies(registeredPlugins) {
+  let found = [];
+  for (let i = 0; i < registeredPlugins.length; i++) {
+    if (registeredPlugins[i].getDependencies) {
+      found.push(registeredPlugins[i]);
+    }
+  }
+  return found;
+}
+
 module.exports = TemplateCompiler;