[dart2js] Land accurate compute merging dependency algorithm.

Change-Id: If47a670631c4c4d63b0fd834a01dc30275b416d8
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/191241
Reviewed-by: Sigmund Cherem <sigmund@google.com>
Commit-Queue: Joshua Litt <joshualitt@google.com>

pkg/compiler/lib/src/deferred_load.dart

@@ -80,13 +80,6 @@ class OutputUnit implements Comparable<OutputUnit> {
     return name.compareTo(other.name);
   }
 
-  void merge(OutputUnit that) {
-    assert(this != that);
-    // We don't currently support merging code into the main output unit.
-    assert(!isMainOutput);
-    this.imports.addAll(that.imports);
-  }
-
   @override
   String toString() => "OutputUnit($name, $imports)";
 }

pkg/compiler/lib/src/js_emitter/startup_emitter/fragment_merger.dart

@@ -83,17 +83,20 @@ class PreFragment {
     this.lazyInitializers.addAll(that.lazyInitializers);
     this.nativeSupport.addAll(that.nativeSupport);
     this.successors.remove(that);
+    this.predecessors.remove(that);
     that.successors.forEach((fragment) {
+      if (fragment == this) return;
+      this.successors.add(fragment);
       fragment.predecessors.remove(that);
       fragment.predecessors.add(this);
     });
-    this.successors.addAll(that.successors);
-    that.predecessors.remove(this);
     that.predecessors.forEach((fragment) {
+      if (fragment == this) return;
+      this.predecessors.add(fragment);
       fragment.successors.remove(that);
       fragment.successors.add(this);
     });
-    this.predecessors.addAll(that.predecessors);
+    that.clearAll();
     this.size += that.size;
     return this;
   }
@@ -102,14 +105,15 @@ class PreFragment {
       Program program, Map<Fragment, FinalizedFragment> fragmentMap) {
     FinalizedFragment finalizedFragment;
     var seedFragment = fragments.first;
+    var seedOutputUnit = seedFragment.outputUnit;
 
     // If we only have a single fragment, then wen just finalize it by itself.
     // Otherwise, we finalize an entire group of fragments into a single
     // merged and finalized fragment.
     if (fragments.length == 1) {
       finalizedFragment = FinalizedFragment(
           seedFragment.outputFileName,
-          seedFragment.outputUnit,
+          [seedOutputUnit],
           seedFragment.libraries,
           classPrototypes.first,
           closurePrototypes.first,
@@ -122,21 +126,22 @@ class PreFragment {
           staticNonFinalFields.first,
           lazyInitializers.first,
           nativeSupport.first,
-          program.metadataTypesForOutputUnit(seedFragment.outputUnit));
+          program.metadataTypesForOutputUnit(seedOutputUnit));
       fragmentMap[seedFragment] = finalizedFragment;
     } else {
+      List<OutputUnit> outputUnits = [seedOutputUnit];
       List<Library> libraries = [];
       for (var fragment in fragments) {
-        if (seedFragment.outputUnit != fragment.outputUnit) {
-          program.mergeOutputUnitMetadata(
-              seedFragment.outputUnit, fragment.outputUnit);
-          seedFragment.outputUnit.merge(fragment.outputUnit);
+        var fragmentOutputUnit = fragment.outputUnit;
+        if (seedOutputUnit != fragmentOutputUnit) {
+          program.mergeOutputUnitMetadata(seedOutputUnit, fragmentOutputUnit);
+          outputUnits.add(fragmentOutputUnit);
         }
         libraries.addAll(fragment.libraries);
       }
       finalizedFragment = FinalizedFragment(
           seedFragment.outputFileName,
-          seedFragment.outputUnit,
+          outputUnits,
           libraries,
           js.Block(classPrototypes),
           js.Block(closurePrototypes),
@@ -149,7 +154,7 @@ class PreFragment {
           js.Block(staticNonFinalFields),
           js.Block(lazyInitializers),
           js.Block(nativeSupport),
-          program.metadataTypesForOutputUnit(seedFragment.outputUnit));
+          program.metadataTypesForOutputUnit(seedOutputUnit));
       for (var fragment in fragments) {
         fragmentMap[fragment] = finalizedFragment;
       }
@@ -183,11 +188,31 @@ class PreFragment {
     }
     return "${outputUnitStrings.join('+')}";
   }
+
+  /// Clears all [PreFragment] data structure and zeros out the size. Should be
+  /// used only after merging to GC internal data structures.
+  void clearAll() {
+    fragments.clear();
+    classPrototypes.clear();
+    closurePrototypes.clear();
+    inheritance.clear();
+    methodAliases.clear();
+    tearOffs.clear();
+    constants.clear();
+    typeRules.clear();
+    variances.clear();
+    staticNonFinalFields.clear();
+    lazyInitializers.clear();
+    nativeSupport.clear();
+    successors.clear();
+    predecessors.clear();
+    size = 0;
+  }
 }
 
 class FinalizedFragment {
   final String outputFileName;
-  final OutputUnit outputUnit;
+  final List<OutputUnit> outputUnits;
   final List<Library> libraries;
   final js.Statement classPrototypes;
   final js.Statement closurePrototypes;
@@ -204,7 +229,7 @@ class FinalizedFragment {
 
   FinalizedFragment(
       this.outputFileName,
-      this.outputUnit,
+      this.outputUnits,
       this.libraries,
       this.classPrototypes,
       this.closurePrototypes,
@@ -247,6 +272,12 @@ class FinalizedFragment {
         isEmptyStatement(lazyInitializers) &&
         isEmptyStatement(nativeSupport);
   }
+
+  // The 'main' [OutputUnit] for this [FinalizedFragment].
+  // TODO(joshualitt): Refactor this to more clearly disambiguate between
+  // [OutputUnits](units of deferred merging), fragments(units of emitted code),
+  // and files.
+  OutputUnit get canonicalOutputUnit => outputUnits.first;
 }
 
 class _Partition {
@@ -261,6 +292,7 @@ class _Partition {
 
 class FragmentMerger {
   final CompilerOptions _options;
+  int totalSize = 0;
 
   FragmentMerger(this._options);
 
@@ -284,16 +316,63 @@ class FragmentMerger {
     return loadMap;
   }
 
-  // Attaches predecessors to each PreFragment. We only care about
-  // direct predecessors.
-  static void attachDependencies(Map<String, List<Fragment>> programLoadMap,
-      Map<Fragment, PreFragment> fragmentMap) {
-    programLoadMap.forEach((loadId, fragments) {
-      for (int i = 0; i < fragments.length - 1; i++) {
-        var fragment = fragmentMap[fragments[i]];
-        var nextFragment = fragmentMap[fragments[i + 1]];
-        fragment.successors.add(nextFragment);
-        nextFragment.predecessors.add(fragment);
+  /// Given a list of OutputUnits sorted by their import entites,
+  /// returns a map of all the direct edges between output units.
+  Map<OutputUnit, Set<OutputUnit>> createDirectEdges(
+      List<OutputUnit> allOutputUnits) {
+    Map<OutputUnit, Set<OutputUnit>> backEdges = {};
+    for (int i = 0; i < allOutputUnits.length; i++) {
+      var a = allOutputUnits[i];
+      var aImports = a.imports;
+      for (int j = i + 1; j < allOutputUnits.length; j++) {
+        var b = allOutputUnits[j];
+        if (b.imports.containsAll(aImports)) {
+          backEdges[b] ??= {};
+
+          // Remove transitive edges from nodes that will reach 'b' from the
+          // edge we just added.
+          // Note: Because we add edges in order (starting from the smallest
+          // sets) we always add transitive edges before the last direct edge.
+          backEdges[b].removeWhere((c) => aImports.containsAll(c.imports));
+
+          // Create an edge to denote that 'b' must be loaded before 'a'.
+          backEdges[b].add(a);
+        }
+      }
+    }
+
+    Map<OutputUnit, Set<OutputUnit>> forwardEdges = {};
+    backEdges.forEach((b, edges) {
+      for (var a in edges) {
+        (forwardEdges[a] ??= {}).add(b);
+      }
+    });
+    return forwardEdges;
+  }
+
+  /// Attachs predecessors and successors to each PreFragment.
+  void attachDependencies(Map<Fragment, PreFragment> fragmentMap,
+      List<PreFragment> preDeferredFragments) {
+    // Create a map of OutputUnit to Fragment.
+    Map<OutputUnit, Fragment> outputUnitMap = {};
+    List<OutputUnit> allOutputUnits = [];
+    for (var preFragment in preDeferredFragments) {
+      var fragment = preFragment.fragments.single;
+      var outputUnit = fragment.outputUnit;
+      outputUnitMap[outputUnit] = fragment;
+      allOutputUnits.add(outputUnit);
+      totalSize += preFragment.size;
+    }
+    allOutputUnits.sort();
+
+    // Get a list of direct edges and then attach them to PreFragments.
+    var allEdges = createDirectEdges(allOutputUnits);
+    allEdges.forEach((outputUnit, edges) {
+      var predecessor = fragmentMap[outputUnitMap[outputUnit]];
+      for (var edge in edges) {
+        var successor = fragmentMap[outputUnitMap[edge]];
+        predecessor.successors.add(successor);
+        successor.predecessors.add(predecessor);
       }
     });
   }
@@ -315,12 +394,6 @@ class FragmentMerger {
     });
     int desiredNumberOfFragment = _options.mergeFragmentsThreshold;
 
-    // TODO(joshualitt): Precalculate totalSize when computing dependencies.
-    int totalSize = 0;
-    for (var preFragment in preDeferredFragments) {
-      totalSize += preFragment.size;
-    }
-
     int idealFragmentSize = (totalSize / desiredNumberOfFragment).ceil();
     List<_Partition> partitions = [];
     void add(PreFragment next) {

pkg/compiler/lib/src/js_emitter/startup_emitter/model_emitter.dart

@@ -98,6 +98,7 @@ class ModelEmitter {
   final JClosedWorld _closedWorld;
   final ConstantOrdering _constantOrdering;
   final SourceInformationStrategy _sourceInformationStrategy;
+  final FragmentMerger fragmentMerger;
 
   // The full code that is written to each hunk part-file.
   final Map<OutputUnit, CodeOutput> emittedOutputBuffers = {};
@@ -128,7 +129,8 @@ class ModelEmitter {
       this._sourceInformationStrategy,
       RecipeEncoder rtiRecipeEncoder,
       this._shouldGenerateSourceMap)
-      : _constantOrdering = new ConstantOrdering(_closedWorld.sorter) {
+      : _constantOrdering = new ConstantOrdering(_closedWorld.sorter),
+        fragmentMerger = FragmentMerger(_options) {
     this.constantEmitter = new ConstantEmitter(
         _options,
         _namer,
@@ -227,12 +229,11 @@ class ModelEmitter {
       }
     });
 
-    // Attach dependencies to each PreFragment.
-    FragmentMerger.attachDependencies(program.loadMap, preFragmentMap);
-
+    // If we are going to merge, then we attach dependencies to each PreFragment
+    // and merge.
     if (shouldMergeFragments) {
       preDeferredFragments = _task.measureSubtask('merge fragments', () {
-        FragmentMerger fragmentMerger = FragmentMerger(_options);
+        fragmentMerger.attachDependencies(preFragmentMap, preDeferredFragments);
         return fragmentMerger.mergeFragments(preDeferredFragments);
       });
     }
@@ -252,7 +253,7 @@ class ModelEmitter {
       if (fragmentCode != null) {
         deferredFragmentsCode[finalizedFragment] = fragmentCode;
       } else {
-        omittedOutputUnits.add(finalizedFragment.outputUnit);
+        omittedOutputUnits.addAll(finalizedFragment.outputUnits);
       }
     }
 
@@ -419,7 +420,9 @@ class ModelEmitter {
             hunkPrefix, deferredExtension, OutputType.jsPart),
         outputListeners);
 
-    emittedOutputBuffers[fragment.outputUnit] = output;
+    // TODO(joshualitt): This breaks dump_info when we merge, but fixing it will
+    // require updating the schema.
+    emittedOutputBuffers[fragment.canonicalOutputUnit] = output;
 
     // The [code] contains the function that must be invoked when the deferred
     // hunk is loaded.

pkg/compiler/test/deferred_loading/data/deferred_overlapping/main.dart

@@ -2,11 +2,21 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
-/*library: 
+/*spec.library: 
  output_units=[
-  f1: {units: [1{lib1, lib2}], usedBy: [2, 3], needs: []},
-  f2: {units: [2{lib1}], usedBy: [], needs: [1]},
-  f3: {units: [3{lib2}], usedBy: [], needs: [1]}],
+  f1: {units: [1{lib1, lib2}], usedBy: [], needs: []},
+  f2: {units: [2{lib1}], usedBy: [], needs: []},
+  f3: {units: [3{lib2}], usedBy: [], needs: []}],
+ steps=[
+  lib1=(f1, f2),
+  lib2=(f1, f3)]
+*/
+
+/*two-frag|three-frag.library: 
+ output_units=[
+  f1: {units: [1{lib1, lib2}], usedBy: [], needs: [2, 3]},
+  f2: {units: [2{lib1}], usedBy: [1], needs: []},
+  f3: {units: [3{lib2}], usedBy: [1], needs: []}],
  steps=[
   lib1=(f1, f2),
   lib2=(f1, f3)]

pkg/compiler/test/deferred_loading/data/dont_inline_deferred_constants/main.dart

@@ -2,20 +2,30 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
-/*spec|three-frag.library: 
+/*spec.library: 
  output_units=[
-  f1: {units: [2{lib1, lib2}], usedBy: [2, 3], needs: []},
-  f2: {units: [1{lib1}], usedBy: [], needs: [1]},
-  f3: {units: [3{lib2}], usedBy: [], needs: [1]}],
+  f1: {units: [2{lib1, lib2}], usedBy: [], needs: []},
+  f2: {units: [1{lib1}], usedBy: [], needs: []},
+  f3: {units: [3{lib2}], usedBy: [], needs: []}],
+ steps=[
+  lib1=(f1, f2),
+  lib2=(f1, f3)]
+*/
+
+/*three-frag.library: 
+ output_units=[
+  f1: {units: [2{lib1, lib2}], usedBy: [], needs: [2, 3]},
+  f2: {units: [1{lib1}], usedBy: [1], needs: []},
+  f3: {units: [3{lib2}], usedBy: [1], needs: []}],
  steps=[
   lib1=(f1, f2),
   lib2=(f1, f3)]
 */
 
 /*two-frag.library: 
  output_units=[
-  f1: {units: [2{lib1, lib2}, 3{lib2}], usedBy: [2], needs: []},
-  f2: {units: [1{lib1}], usedBy: [], needs: [1]}],
+  f1: {units: [2{lib1, lib2}, 3{lib2}], usedBy: [], needs: [2]},
+  f2: {units: [1{lib1}], usedBy: [1], needs: []}],
  steps=[
   lib1=(f1, f2),
   lib2=(f1)]

pkg/compiler/test/deferred_loading/data/instantiation1/main.dart

@@ -2,11 +2,21 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
-/*library: 
+/*spec.library: 
  output_units=[
-  f1: {units: [2{b, c}], usedBy: [2, 3], needs: []},
-  f2: {units: [1{b}], usedBy: [], needs: [1]},
-  f3: {units: [3{c}], usedBy: [], needs: [1]}],
+  f1: {units: [2{b, c}], usedBy: [], needs: []},
+  f2: {units: [1{b}], usedBy: [], needs: []},
+  f3: {units: [3{c}], usedBy: [], needs: []}],
+ steps=[
+  b=(f1, f2),
+  c=(f1, f3)]
+*/
+
+/*two-frag|three-frag.library: 
+ output_units=[
+  f1: {units: [2{b, c}], usedBy: [], needs: [2, 3]},
+  f2: {units: [1{b}], usedBy: [1], needs: []},
+  f3: {units: [3{c}], usedBy: [1], needs: []}],
  steps=[
   b=(f1, f2),
   c=(f1, f3)]