Skip to content

Commit

Permalink
When --dependency_mode is set to STRICT or LOOSE, order dependencies …
Browse files Browse the repository at this point in the history 
…in a deterministic depth-first order from entry points.

Avoids a full parse on every input when CommonJS module processing is enabled.
ES6 and CommonJS modules no longer generate synthetic goog.require and goog.provide calls.
ES6 Module Transpilation no longer depends on Closure-Library primitives.
  • Loading branch information
@ChadKillingsworth
ChadKillingsworth committed Sep 4, 2017
1 parent 83d52f6 commit d040afa
Show file tree
Hide file tree
Showing 9 changed files with 532 additions and 362 deletions.
170 changes: 152 additions & 18 deletions src/com/google/javascript/jscomp/Compiler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,6 @@
import java.io.Serializable;
import java.util.AbstractSet;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
Expand Down Expand Up @@ -1749,7 +1748,19 @@ Node parseInputs() {
options.moduleResolutionMode,
processJsonInputs(inputs));
}
} else {
// Use an empty module loader if we're not actually dealing with modules.
this.moduleLoader = ModuleLoader.EMPTY;
}

if (options.getDependencyOptions().needsManagement()) {
findDependenciesFromEntryPoints(
options.getLanguageIn().toFeatureSet().has(Feature.MODULES),
options.processCommonJSModules,
options.transformAMDToCJSModules);
} else if (options.needsTranspilationFrom(FeatureSet.ES6_MODULES)
|| options.transformAMDToCJSModules
|| options.processCommonJSModules) {
if (options.getLanguageIn().toFeatureSet().has(Feature.MODULES)) {
parsePotentialModules(inputs);
}
Expand Down Expand Up @@ -1782,12 +1793,12 @@ Node parseInputs() {
}
}

if (!inputsToRewrite.isEmpty()) {
forceToEs6Modules(inputsToRewrite.values());
for (CompilerInput input : inputsToRewrite.values()) {
forceInputToPathBasedModule(
input,
options.getLanguageIn().toFeatureSet().has(Feature.MODULES),
options.processCommonJSModules);
}
} else {
// Use an empty module loader if we're not actually dealing with modules.
this.moduleLoader = ModuleLoader.EMPTY;
}

orderInputs();
Expand Down Expand Up @@ -1894,6 +1905,141 @@ void orderInputs() {
}
}

/**
* Find dependencies by recursively traversing each dependency of an input starting with the entry
* points. Causes a full parse of each file, but since the file is reachable by walking the graph,
* this would be required in later compilation passes regardless.
*
* <p>Inputs which are not reachable during graph traversal will be dropped.
*
* <p>If the dependency mode is set to LOOSE, inputs for which the deps package did not find a
* provide statement or detect as a module will be treated as entry points.
*/
void findDependenciesFromEntryPoints(
boolean supportEs6Modules, boolean supportCommonJSModules, boolean supportAmdModules) {
hoistUnorderedExterns();
List<CompilerInput> entryPoints = new ArrayList<>();
Map<String, CompilerInput> inputsByProvide = new HashMap<>();
Map<String, CompilerInput> inputsByIdentifier = new HashMap<>();
for (CompilerInput input : inputs) {
if (!options.getDependencyOptions().shouldDropMoochers() && input.getProvides().isEmpty()) {
entryPoints.add(input);
}
inputsByIdentifier.put(
ModuleIdentifier.forFile(input.getPath().toString()).toString(), input);
for (String provide : input.getProvides()) {
if (!provide.startsWith("module$")) {
inputsByProvide.put(provide, input);
}
}
}
for (ModuleIdentifier moduleIdentifier : options.getDependencyOptions().getEntryPoints()) {
CompilerInput input = inputsByIdentifier.get(moduleIdentifier.toString());
if (input != null) {
entryPoints.add(input);
}
}

Set<CompilerInput> workingInputSet = new HashSet<>(inputs);
List<CompilerInput> orderedInputs = new ArrayList<>();
for (CompilerInput entryPoint : entryPoints) {
orderedInputs.addAll(
depthFirstDependenciesFromInput(
entryPoint,
false,
workingInputSet,
inputsByIdentifier,
inputsByProvide,
supportEs6Modules,
supportCommonJSModules,
supportAmdModules));
}

// TODO(ChadKillingsworth) Move this into the standard compilation passes
if (supportCommonJSModules) {
for (CompilerInput input : orderedInputs) {
new ProcessCommonJSModules(this).process(null, input.getAstRoot(this), false);
}
}
}

/** For a given input, order it's dependencies in a depth first traversal */
List<CompilerInput> depthFirstDependenciesFromInput(
CompilerInput input,
boolean wasImportedByModule,
Set<CompilerInput> inputs,
Map<String, CompilerInput> inputsByIdentifier,
Map<String, CompilerInput> inputsByProvide,
boolean supportEs6Modules,
boolean supportCommonJSModules,
boolean supportAmdModules) {
List<CompilerInput> orderedInputs = new ArrayList<>();
if (!inputs.remove(input)) {
// It's possible for a module to be included as both a script
// and a module in the same compilation. In these cases, it should
// be forced to be a module.
if (wasImportedByModule && !input.isJsModule()) {
forceInputToPathBasedModule(input, supportEs6Modules, supportCommonJSModules);
}

return orderedInputs;
}

if (supportAmdModules) {
new TransformAMDToCJSModule(this).process(null, input.getAstRoot(this));
}

FindModuleDependencies findDeps =
new FindModuleDependencies(this, supportEs6Modules, supportCommonJSModules);
findDeps.process(input.getAstRoot(this));

// If this input was imported by another module, it is itself a module
// so we force it to be detected as such.
if (wasImportedByModule && !input.isJsModule()) {
forceInputToPathBasedModule(input, supportEs6Modules, supportCommonJSModules);
}

for (String requiredNamespace : input.getRequires()) {
CompilerInput requiredInput = null;
boolean requiredByModuleImport = false;
if (inputsByProvide.containsKey(requiredNamespace)) {
requiredInput = inputsByProvide.get(requiredNamespace);
} else if (inputsByIdentifier.containsKey(requiredNamespace)) {
requiredByModuleImport = true;
requiredInput = inputsByIdentifier.get(requiredNamespace);
}

if (requiredInput != null) {
orderedInputs.addAll(
depthFirstDependenciesFromInput(
requiredInput,
requiredByModuleImport,
inputs,
inputsByIdentifier,
inputsByProvide,
supportEs6Modules,
supportCommonJSModules,
supportAmdModules));
}
}
orderedInputs.add(input);
return orderedInputs;
}

private void forceInputToPathBasedModule(
CompilerInput input, boolean supportEs6Modules, boolean supportCommonJSModules) {

if (supportEs6Modules) {
FindModuleDependencies findDeps =
new FindModuleDependencies(this, supportEs6Modules, supportCommonJSModules);
findDeps.convertToEs6Module(input.getAstRoot(this));
input.markAsModule(true);
} else if (supportCommonJSModules) {
new ProcessCommonJSModules(this).process(null, input.getAstRoot(this), true);
input.markAsModule(true);
}
}

/**
* Hoists inputs with the @externs annotation and no provides or requires into the externs list.
*/
Expand Down Expand Up @@ -2018,18 +2164,6 @@ Map<String, String> processJsonInputs(List<CompilerInput> inputsToProcess) {
return rewriteJson.getPackageJsonMainEntries();
}

void forceToEs6Modules(Collection<CompilerInput> inputsToProcess) {
for (CompilerInput input : inputsToProcess) {
input.setCompiler(this);
input.addProvide(input.getPath().toModuleName());
Node root = input.getAstRoot(this);
if (root == null) {
continue;
}
Es6RewriteModules moduleRewriter = new Es6RewriteModules(this);
moduleRewriter.forceToEs6Module(root);
}
}

private List<CompilerInput> parsePotentialModules(List<CompilerInput> inputsToProcess) {
List<CompilerInput> filteredInputs = new ArrayList<>();
Expand Down
38 changes: 31 additions & 7 deletions src/com/google/javascript/jscomp/CompilerInput.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,9 @@ public class CompilerInput implements SourceAst, DependencyInfo {
private DependencyInfo dependencyInfo;
private final List<String> extraRequires = new ArrayList<>();
private final List<String> extraProvides = new ArrayList<>();
private final List<String> orderedRequires = new ArrayList<>();
private boolean hasFullParseDependencyInfo = false;
private boolean jsModule = false;

// An AbstractCompiler for doing parsing.
// We do not want to persist this across serialized state.
Expand Down Expand Up @@ -151,6 +154,10 @@ public void setCompiler(AbstractCompiler compiler) {
/** Gets a list of types depended on by this input. */
@Override
public Collection<String> getRequires() {
if (hasFullParseDependencyInfo) {
return orderedRequires;
}

return getDependencyInfo().getRequires();
}

Expand Down Expand Up @@ -182,19 +189,36 @@ Collection<String> getKnownProvides() {
extraProvides);
}

// TODO(nicksantos): Remove addProvide/addRequire/removeRequire once
// there is better support for discovering non-closure dependencies.

/**
* Registers a type that this input defines.
* Registers a type that this input defines. Includes both explicitly declared namespaces via
* goog.provide and goog.module calls as well as implicit namespaces provided by module rewriting.
*/
public void addProvide(String provide) {
extraProvides.add(provide);
}

/**
* Registers a type that this input depends on.
*/
/** Registers a type that this input depends on in the order seen in the file. */
public boolean addOrderedRequire(String require) {
if (!orderedRequires.contains(require)) {
orderedRequires.add(require);
return true;
}
return false;
}

public void setHasFullParseDependencyInfo(boolean hasFullParseDependencyInfo) {
this.hasFullParseDependencyInfo = hasFullParseDependencyInfo;
}

public boolean isJsModule() {
return jsModule;
}

public void markAsModule(boolean isModule) {
jsModule = isModule;
}

/** Registers a type that this input depends on. */
public void addRequire(String require) {
extraRequires.add(require);
}
Expand Down
59 changes: 7 additions & 52 deletions src/com/google/javascript/jscomp/Es6RewriteModules.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@

import com.google.common.base.Preconditions;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableSet;
import com.google.javascript.jscomp.NodeTraversal.AbstractPostOrderCallback;
import com.google.javascript.jscomp.deps.ModuleLoader;
import com.google.javascript.rhino.IR;
Expand Down Expand Up @@ -80,8 +79,6 @@ public final class Es6RewriteModules extends AbstractPostOrderCallback

private Set<String> alreadyRequired;

private Node googRequireInsertSpot;

/**
* Creates a new Es6RewriteModules instance which can be used to rewrite
* ES6 modules to a concatenable form.
Expand All @@ -101,29 +98,6 @@ public static boolean isEs6ModuleRoot(Node scriptNode) {
return scriptNode.hasChildren() && scriptNode.getFirstChild().isModuleBody();
}

/**
* Force rewriting of a file into an ES6 module, such as for imported files that contain no
* "import" or "export" statements. Fails if the file contains a goog.provide or goog.module.
*
* @return True, if the file is now an ES6 module. False, if the file must remain a script.
* TODO(blickly): Move this logic out of this pass, since it is independent of whether or
* not we are actually transpiling modules
*/
public boolean forceToEs6Module(Node root) {
if (isEs6ModuleRoot(root)) {
return true;
}
FindGoogProvideOrGoogModule finder = new FindGoogProvideOrGoogModule();
NodeTraversal.traverseEs6(compiler, root, finder);
if (finder.isFound()) {
return false;
}
Node moduleNode = new Node(Token.MODULE_BODY).srcref(root);
moduleNode.addChildrenToBack(root.removeChildren());
root.addChildToBack(moduleNode);
return true;
}

@Override
public void process(Node externs, Node root) {
for (Node file = root.getFirstChild(); file != null; file = file.getNext()) {
Expand Down Expand Up @@ -155,7 +129,6 @@ public void clearState() {
this.classes = new HashSet<>();
this.typedefs = new HashSet<>();
this.alreadyRequired = new HashSet<>();
this.googRequireInsertSpot = null;
}

/**
Expand Down Expand Up @@ -270,16 +243,7 @@ private void visitImport(NodeTraversal t, Node importDecl, Node parent) {
}
}

// Emit goog.require call for the module.
if (alreadyRequired.add(moduleName)) {
Node require = IR.exprResult(
IR.call(NodeUtil.newQName(compiler, "goog.require"), IR.string(moduleName)));
require.useSourceInfoIfMissingFromForTree(importDecl);
parent.addChildAfter(require, googRequireInsertSpot);
googRequireInsertSpot = require;
t.getInput().addRequire(moduleName);
}

alreadyRequired.add(moduleName);
parent.removeChild(importDecl);
t.reportCodeChange();
}
Expand Down Expand Up @@ -466,22 +430,13 @@ private void visitScript(NodeTraversal t, Node script) {
// Rename vars to not conflict in global scope.
NodeTraversal.traverseEs6(compiler, script, new RenameGlobalVars(moduleName));

// Add goog.provide call.
Node googProvide = IR.exprResult(
IR.call(NodeUtil.newQName(compiler, "goog.provide"),
IR.string(moduleName)));
script.addChildToFront(googProvide.useSourceInfoIfMissingFromForTree(script));
t.getInput().addProvide(moduleName);

JSDocInfoBuilder jsDocInfo = script.getJSDocInfo() == null
? new JSDocInfoBuilder(false)
: JSDocInfoBuilder.copyFrom(script.getJSDocInfo());
if (!jsDocInfo.isPopulatedWithFileOverview()) {
jsDocInfo.recordFileOverview("");
if (!exportMap.isEmpty()) {
Node moduleVar = IR.var(IR.name(moduleName), IR.objectlit());
JSDocInfoBuilder infoBuilder = new JSDocInfoBuilder(false);
infoBuilder.recordConstancy();
moduleVar.setJSDocInfo(infoBuilder.build());
script.addChildToFront(moduleVar.useSourceInfoIfMissingFromForTree(script));
}
// Don't check provides and requires, since most of them are auto-generated.
jsDocInfo.recordSuppressions(ImmutableSet.of("missingProvide", "missingRequire"));
script.setJSDocInfo(jsDocInfo.build());

exportMap.clear();
t.reportCodeChange();
Expand Down
Loading

0 comments on commit d040afa

Please sign in to comment.