Reland "[dart2js] Stop emitting a dill with closed world."

This is a reland of commit b9e25af

Original change's description:
> [dart2js] Stop emitting a dill with closed world.
>
> All transformations to the dill are now performed with the CFE linker phase (phase0b). This means we no longer need to emit a dill as part of the closed world computation. This saves both time and memory resources associated with the cost of serialization during that phase.
>
> Change-Id: I28dde8a1eecd71e823880027505f62fc804fcca6
> Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/332821
> Reviewed-by: Sigmund Cherem <sigmund@google.com>
> Reviewed-by: Mayank Patke <fishythefish@google.com>
> Commit-Queue: Nate Biggs <natebiggs@google.com>

Change-Id: Ia693213dcdc949a8ef4b4204cc033130274d2381
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/336982
Reviewed-by: Mayank Patke <fishythefish@google.com>
Commit-Queue: Nate Biggs <natebiggs@google.com>

pkg/compiler/lib/src/compiler.dart

@@ -419,7 +419,8 @@ class Compiler {
             component = trimComponent(component, includedLibraries);
           }
         }
-        serializationTask.serializeComponent(component);
+        serializationTask.serializeComponent(component,
+            includeSourceBytes: false);
       }
       return output.withNewComponent(component);
     } else {
@@ -525,9 +526,6 @@ class Compiler {
       List<Uri> libraries = output.libraries!;
       closedWorld = computeClosedWorld(component, rootLibraryUri, libraries);
       if (stage == Dart2JSStage.closedWorld && closedWorld != null) {
-        serializationTask.serializeComponent(
-            closedWorld.elementMap.programEnv.mainComponent,
-            includeSourceBytes: false);
         serializationTask.serializeClosedWorld(closedWorld, indices);
       } else if (options.testMode && closedWorld != null) {
         closedWorld = closedWorldTestMode(closedWorld);

pkg/compiler/lib/src/dart2js.dart

@@ -751,17 +751,15 @@ Future<api.CompilationResult> compile(List<String> argv,
       case Dart2JSStage.cfe:
       case Dart2JSStage.allFromDill:
       case Dart2JSStage.cfeFromDill:
-      case Dart2JSStage.closedWorld:
         final sourceCharCount =
             _formatCharacterCount(inputProvider.sourceBytesFromDill);
         inputName = 'input bytes ($sourceCharCount characters source)';
         inputSize = inputProvider.bytesRead;
         summary = 'Dart file $input ';
         break;
+      case Dart2JSStage.closedWorld:
       case Dart2JSStage.deferredLoadIds:
-        final sourceCharCount =
-            _formatCharacterCount(inputProvider.sourceBytesFromDill);
-        inputName = 'input bytes ($sourceCharCount characters source)';
+        inputName = 'input bytes';
         inputSize = inputProvider.bytesRead;
         summary = 'Dart file $input ';
         break;
@@ -833,12 +831,11 @@ Future<api.CompilationResult> compile(List<String> argv,
         processName = 'Serialized';
         outputName = 'bytes data';
         outputSize = outputProvider.totalDataWritten;
-        String output = fe.relativizeUri(Uri.base, out!, Platform.isWindows);
         String dataOutput = fe.relativizeUri(
             Uri.base,
             compilerOptions.dataOutputUriForStage(compilerOptions.stage),
             Platform.isWindows);
-        summary += 'serialized to dill and data: ${output} and ${dataOutput}.';
+        summary += 'serialized to data: ${dataOutput}.';
         break;
       case Dart2JSStage.deferredLoadIds:
         processName = 'Serialized';

pkg/compiler/lib/src/options.dart

@@ -37,7 +37,7 @@ enum Dart2JSStage {
       emitsKernel: false,
       emitsJs: false),
   closedWorld('closed-world',
-      dataOutputName: 'world.data', emitsKernel: true, emitsJs: false),
+      dataOutputName: 'world.data', emitsKernel: false, emitsJs: false),
   globalInference('global-inference',
       dataOutputName: 'global.data', emitsKernel: false, emitsJs: false),
   codegenAndJsEmitter('codegen-emit-js', emitsKernel: false, emitsJs: true),

pkg/compiler/test/end_to_end/command_line_test.dart

@@ -97,40 +97,37 @@ main() {
 
     // Run closed world only
     await test(['${Flags.stage}=closed-world', 'foo.dill'],
-        writeClosedWorld: 'world.data', out: 'out.dill');
+        writeClosedWorld: 'world.data');
     await test(['${Flags.stage}=closed-world', '${Flags.inputDill}=foo.dill'],
-        writeClosedWorld: 'world.data', out: 'out.dill');
-    await test(['${Flags.stage}=closed-world', '--out=out1.dill', 'foo.dill'],
-        writeClosedWorld: 'world.data', out: 'out1.dill');
+        writeClosedWorld: 'world.data');
+    await test(['${Flags.stage}=closed-world', 'foo.dill'],
+        writeClosedWorld: 'world.data');
     await test([
       '${Flags.stage}=closed-world',
       '${Flags.writeClosedWorld}=world1.data',
       'foo.dill'
-    ], writeClosedWorld: 'world1.data', out: 'out.dill');
+    ], writeClosedWorld: 'world1.data');
     await test([
       '${Flags.writeClosedWorld}=world1.data',
       'foo.dill',
-      '--out=out1.dill'
-    ], out: 'out1.dill', writeClosedWorld: 'world1.data');
-    await test(
-        ['${Flags.writeClosedWorld}=world1.data', 'foo.dill', '-oout1.dill'],
-        out: 'out1.dill', writeClosedWorld: 'world1.data');
+    ], writeClosedWorld: 'world1.data');
+    await test(['${Flags.writeClosedWorld}=world1.data', 'foo.dill'],
+        writeClosedWorld: 'world1.data');
     await test(
         ['${Flags.writeClosedWorld}=world1.data', 'foo.dill', '--out=prefix-'],
-        out: 'prefix-', writeClosedWorld: 'world1.data');
+        writeClosedWorld: 'world1.data');
     await test([
       '${Flags.writeClosedWorld}=world1.data',
       'foo.dill',
       '--out=/some/path/prefix-'
-    ], out: '/some/path/prefix-', writeClosedWorld: 'world1.data');
+    ], writeClosedWorld: 'world1.data');
     await test(['foo.dill', '${Flags.stage}=closed-world', '--out=/some/path/'],
-        writeClosedWorld: '/some/path/world.data', out: '/some/path/out.dill');
+        writeClosedWorld: '/some/path/world.data');
     await test(['foo.dill', '${Flags.stage}=closed-world', '--out=prefix-'],
-        writeClosedWorld: 'prefix-world.data', out: 'prefix-out.dill');
+        writeClosedWorld: 'prefix-world.data');
     await test(
         ['foo.dill', '${Flags.stage}=closed-world', '--out=/some/path/prefix-'],
-        writeClosedWorld: '/some/path/prefix-world.data',
-        out: '/some/path/prefix-out.dill');
+        writeClosedWorld: '/some/path/prefix-world.data');
 
     // Run global inference only
     await test(['${Flags.stage}=global-inference', 'foo.dill'],

pkg/compiler/test/serialization/on_disk_split_test.dart

@@ -14,7 +14,6 @@ main(List<String> args) {
   asyncTest(() async {
     Directory dir = await Directory.systemTemp.createTemp('on_disk');
     Uri dillUri = dir.uri.resolve('out.dill');
-    Uri modifiedDillUri = dir.uri.resolve('modified.dill');
     Uri closedWorldUri = dir.uri.resolve('world.data');
     Uri globalInferenceUri = dir.uri.resolve('global.data');
     Uri outUri = dir.uri.resolve('out.js');
@@ -38,11 +37,10 @@ main(List<String> args) {
           'pkg/compiler/test/codesize/swarm/swarm.dart',
           '${Flags.inputDill}=$dillUri',
           '${Flags.writeClosedWorld}=$closedWorldUri',
-          '--out=${modifiedDillUri}',
         ] +
         commonArgs);
     await internalMain([
-          '$modifiedDillUri',
+          '$dillUri',
           '${Flags.readClosedWorld}=$closedWorldUri',
           '${Flags.writeData}=$globalInferenceUri',
           '--out=${outUri}',

pkg/compiler/test/serialization/serialization_diff_helper.dart

@@ -23,7 +23,6 @@ Future<CompiledOutput> compileWithSerialization(
     required Map<String, dynamic> memorySourceFiles,
     required List<String> options}) async {
   final cfeDillUri = 'memory:cfe.dill';
-  final worldDillUri = 'memory:out.dill';
   final closedWorldUri = 'memory:world.data';
   final globalDataUri = 'memory:global.data';
   final codegenUri = 'memory:codegen';
@@ -46,19 +45,18 @@ Future<CompiledOutput> compileWithSerialization(
   await compile([...options, '--out=$cfeDillUri', Flags.cfeOnly]);
   await compile([
     ...options,
-    '--out=$worldDillUri',
     '${Flags.inputDill}=$cfeDillUri',
     '${Flags.writeClosedWorld}=$closedWorldUri'
   ]);
   await compile([
     ...options,
-    '${Flags.inputDill}=$worldDillUri',
+    '${Flags.inputDill}=$cfeDillUri',
     '${Flags.readClosedWorld}=$closedWorldUri',
     '${Flags.writeData}=$globalDataUri'
   ]);
   await compile([
     ...options,
-    '${Flags.inputDill}=$worldDillUri',
+    '${Flags.inputDill}=$cfeDillUri',
     '${Flags.readClosedWorld}=$closedWorldUri',
     '${Flags.readData}=$globalDataUri',
     '${Flags.writeCodegen}=$codegenUri',
@@ -67,7 +65,7 @@ Future<CompiledOutput> compileWithSerialization(
   ]);
   await compile([
     ...options,
-    '${Flags.inputDill}=$worldDillUri',
+    '${Flags.inputDill}=$cfeDillUri',
     '${Flags.readClosedWorld}=$closedWorldUri',
     '${Flags.readData}=$globalDataUri',
     '${Flags.writeCodegen}=$codegenUri',
@@ -76,7 +74,7 @@ Future<CompiledOutput> compileWithSerialization(
   ]);
   final output = await compile([
     ...options,
-    '${Flags.inputDill}=$worldDillUri',
+    '${Flags.inputDill}=$cfeDillUri',
     '${Flags.readClosedWorld}=$closedWorldUri',
     '${Flags.readData}=$globalDataUri',
     '${Flags.readCodegen}=$codegenUri',

pkg/compiler/test/serialization/serialization_test_helper.dart

@@ -159,7 +159,6 @@ runTest(
   File(cfeDillFileUri.path)
       .writeAsBytesSync(cfeDillCollector.binaryOutputMap[cfeDillFileUri]!.list);
 
-  var dillUri = dir.uri.resolve('out.dill');
   var closedWorldUri = Uri.parse('world.data');
   OutputCollector collector3a = OutputCollector();
   CompilationResult result3a = await runCompiler(
@@ -169,7 +168,6 @@ runTest(
       librariesSpecificationUri: librariesSpecificationUri,
       options: options +
           [
-            '--out=$dillUri',
             '${Flags.inputDill}=$cfeDillFileUri',
             '${Flags.writeClosedWorld}=$closedWorldUri'
           ],
@@ -178,15 +176,11 @@ runTest(
         compiler.forceSerializationForTesting = true;
       });
   Expect.isTrue(result3a.isSuccess);
-  Expect.isTrue(collector3a.binaryOutputMap.containsKey(dillUri));
   Expect.isTrue(collector3a.binaryOutputMap.containsKey(closedWorldUri));
 
-  final dillFileUri = dir.uri.resolve('out.dill');
   final closedWorldFileUri = dir.uri.resolve('world.data');
   final globalDataUri = Uri.parse('global.data');
-  final dillBytes = collector3a.binaryOutputMap[dillUri]!.list;
   final closedWorldBytes = collector3a.binaryOutputMap[closedWorldUri]!.list;
-  File(dillFileUri.path).writeAsBytesSync(dillBytes);
   File(closedWorldFileUri.path).writeAsBytesSync(closedWorldBytes);
   OutputCollector collector3b = OutputCollector();
   CompilationResult result3b = await runCompiler(
@@ -196,7 +190,7 @@ runTest(
       librariesSpecificationUri: librariesSpecificationUri,
       options: commonOptions +
           [
-            '${Flags.inputDill}=$dillFileUri',
+            '${Flags.inputDill}=$cfeDillFileUri',
             '${Flags.readClosedWorld}=$closedWorldFileUri',
             '${Flags.writeData}=$globalDataUri'
           ],
@@ -232,7 +226,7 @@ runTest(
       librariesSpecificationUri: librariesSpecificationUri,
       options: commonOptions +
           [
-            '${Flags.inputDill}=$dillFileUri',
+            '${Flags.inputDill}=$cfeDillFileUri',
             '${Flags.readClosedWorld}=$closedWorldFileUri',
             '${Flags.readData}=$globalDataFileUri',
             '--out=$jsOutUri'

pkg/compiler/tool/modular_test_suite_helper.dart

@@ -29,7 +29,6 @@ const dillSummaryId = DataId("summary.dill");
 const dillId = DataId("full.dill");
 const fullDillId = DataId("concatenate.dill");
 const closedWorldId = DataId("world");
-const globalUpdatedDillId = DataId("global.dill");
 const globalDataId = DataId("global.data");
 const codeId = ShardsDataId("code", 2);
 const codeId0 = ShardDataId(codeId, 0);
@@ -266,7 +265,7 @@ class ComputeClosedWorldStep extends IOModularStep {
   List<DataId> get dependencies => [fullDillId];
 
   @override
-  List<DataId> get resultData => const [closedWorldId, globalUpdatedDillId];
+  List<DataId> get resultData => const [closedWorldId];
 
   @override
   bool get needsSources => false;
@@ -297,7 +296,6 @@ class ComputeClosedWorldStep extends IOModularStep {
       for (String flag in flags) '--enable-experiment=$flag',
       '${Flags.writeClosedWorld}=${toUri(module, closedWorldId)}',
       Flags.noClosedWorldInData,
-      '--out=${toUri(module, globalUpdatedDillId)}',
     ];
     var result =
         await _runProcess(Platform.resolvedExecutable, args, root.toFilePath());
@@ -321,11 +319,10 @@ class GlobalAnalysisStep extends IOModularStep {
   bool get needsSources => false;
 
   @override
-  List<DataId> get dependencyDataNeeded => const [globalUpdatedDillId];
+  List<DataId> get dependencyDataNeeded => const [fullDillId];
 
   @override
-  List<DataId> get moduleDataNeeded =>
-      const [closedWorldId, globalUpdatedDillId];
+  List<DataId> get moduleDataNeeded => const [closedWorldId, fullDillId];
 
   @override
   bool get onlyOnMain => true;
@@ -342,7 +339,7 @@ class GlobalAnalysisStep extends IOModularStep {
       if (_options.useSdk) '--invoker=modular_test',
       Flags.soundNullSafety,
       '${Flags.entryUri}=$fakeRoot${module.mainSource}',
-      '${Flags.inputDill}=${toUri(module, globalUpdatedDillId)}',
+      '${Flags.inputDill}=${toUri(module, fullDillId)}',
       for (String flag in flags) '--enable-experiment=$flag',
       '${Flags.readClosedWorld}=${toUri(module, closedWorldId)}',
       '${Flags.writeData}=${toUri(module, globalDataId)}',
@@ -381,7 +378,7 @@ class Dart2jsCodegenStep extends IOModularStep {
 
   @override
   List<DataId> get moduleDataNeeded =>
-      const [globalUpdatedDillId, closedWorldId, globalDataId];
+      const [fullDillId, closedWorldId, globalDataId];
 
   @override
   bool get onlyOnMain => true;
@@ -397,7 +394,7 @@ class Dart2jsCodegenStep extends IOModularStep {
       if (_options.useSdk) '--invoker=modular_test',
       Flags.soundNullSafety,
       '${Flags.entryUri}=$fakeRoot${module.mainSource}',
-      '${Flags.inputDill}=${toUri(module, globalUpdatedDillId)}',
+      '${Flags.inputDill}=${toUri(module, fullDillId)}',
       for (String flag in flags) '--enable-experiment=$flag',
       '${Flags.readClosedWorld}=${toUri(module, closedWorldId)}',
       '${Flags.readData}=${toUri(module, globalDataId)}',
@@ -430,13 +427,8 @@ class Dart2jsEmissionStep extends IOModularStep {
   List<DataId> get dependencyDataNeeded => const [];
 
   @override
-  List<DataId> get moduleDataNeeded => const [
-        globalUpdatedDillId,
-        closedWorldId,
-        globalDataId,
-        codeId0,
-        codeId1
-      ];
+  List<DataId> get moduleDataNeeded =>
+      const [fullDillId, closedWorldId, globalDataId, codeId0, codeId1];
 
   @override
   bool get onlyOnMain => true;
@@ -452,7 +444,7 @@ class Dart2jsEmissionStep extends IOModularStep {
       if (_options.useSdk) '--invoker=modular_test',
       Flags.soundNullSafety,
       '${Flags.entryUri}=$fakeRoot${module.mainSource}',
-      '${Flags.inputDill}=${toUri(module, globalUpdatedDillId)}',
+      '${Flags.inputDill}=${toUri(module, fullDillId)}',
       for (String flag in flags) '${Flags.enableLanguageExperiments}=$flag',
       '${Flags.readClosedWorld}=${toUri(module, closedWorldId)}',
       '${Flags.readData}=${toUri(module, globalDataId)}',
@@ -487,7 +479,7 @@ class Dart2jsDumpInfoStep extends IOModularStep {
 
   @override
   List<DataId> get moduleDataNeeded => const [
-        globalUpdatedDillId,
+        fullDillId,
         closedWorldId,
         globalDataId,
         codeId0,
@@ -509,7 +501,7 @@ class Dart2jsDumpInfoStep extends IOModularStep {
       if (_options.useSdk) '--invoker=modular_test',
       Flags.soundNullSafety,
       '${Flags.entryUri}=$fakeRoot${module.mainSource}',
-      '${Flags.inputDill}=${toUri(module, globalUpdatedDillId)}',
+      '${Flags.inputDill}=${toUri(module, fullDillId)}',
       for (String flag in flags) '${Flags.enableLanguageExperiments}=$flag',
       '${Flags.readClosedWorld}=${toUri(module, closedWorldId)}',
       '${Flags.readData}=${toUri(module, globalDataId)}',