[vm/kernel] Add --from-dill option to kernel compilers

alexmarkov · commit-bot@chromium.org · commit ecc7a2110667 · 2020-06-08T18:16:33.000Z
This option can be used to split AOT kernel generation into two rounds: * The 1st round uses CFE to parse source files and generate intermediate dill file in *non-AOT* mode. * The 2nd round loads intermediate dill file and applies AOT global transformations, then generates final dill file. Each round is faster than one-shot generation of a dill file in AOT mode. In order to use this 2-round compilation, command gen_kernel <other-options> --aot -o output.dill main.dart should be replaced with the following 2 commands: gen_kernel <other-options> --no-aot -o intermediate.dill main.dart gen_kernel <other-options> --aot --from-dill intermediate.dill -o output.dill main.dart Bug: b/158101962 Change-Id: I2d4deef45b50eab11df47f4a372598e6cf1d0edf Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/150272 Reviewed-by: Martin Kustermann <kustermann@google.com> Reviewed-by: Alexander Aprelev <aam@google.com> Commit-Queue: Alexander Markov <alexmarkov@google.com>
diff --git a/pkg/frontend_server/lib/frontend_server.dart b/pkg/frontend_server/lib/frontend_server.dart
@@ -75,6 +75,9 @@ ArgParser argParser = ArgParser(allowTrailingOptions: true)
       defaultsTo: true)
   ..addOption('import-dill',
       help: 'Import libraries from existing dill file', defaultsTo: null)
+  ..addOption('from-dill',
+      help: 'Read existing dill file instead of compiling from sources',
+      defaultsTo: null)
   ..addOption('output-dill',
       help: 'Output path for the generated dill', defaultsTo: null)
   ..addOption('output-incremental-dill',
@@ -452,6 +455,13 @@ class FrontendCompiler implements CompilerInterface {
       }
     }
 
+    if (options['incremental']) {
+      if (options['from-dill'] != null) {
+        print('Error: --from-dill option cannot be used with --incremental');
+        return false;
+      }
+    }
+
     if (options['null-safety'] == null &&
         compilerOptions.experimentalFlags[ExperimentalFlag.nonNullable]) {
       await autoDetectNullSafetyMode(_mainSource, compilerOptions);
@@ -533,7 +543,8 @@ class FrontendCompiler implements CompilerInterface {
           enableAsserts: options['enable-asserts'],
           useProtobufTreeShaker: options['protobuf-tree-shaker'],
           minimalKernel: options['minimal-kernel'],
-          treeShakeWriteOnlyFields: options['tree-shake-write-only-fields']));
+          treeShakeWriteOnlyFields: options['tree-shake-write-only-fields'],
+          fromDillFile: options['from-dill']));
     }
     if (results.component != null) {
       transformer?.transform(results.component);
diff --git a/pkg/vm/lib/kernel_front_end.dart b/pkg/vm/lib/kernel_front_end.dart
@@ -44,6 +44,7 @@ import 'package:kernel/class_hierarchy.dart' show ClassHierarchy;
 import 'package:kernel/ast.dart' show Component, Library, Reference;
 import 'package:kernel/binary/ast_to_binary.dart' show BinaryPrinter;
 import 'package:kernel/core_types.dart' show CoreTypes;
+import 'package:kernel/kernel.dart' show loadComponentFromBinary;
 import 'package:kernel/target/targets.dart' show Target, TargetFlags, getTarget;
 
 import 'bytecode/bytecode_serialization.dart' show BytecodeSizeStatistics;
@@ -80,6 +81,9 @@ void declareCompilerOptions(ArgParser args) {
           'Produce kernel file for AOT compilation (enables global transformations).',
       defaultsTo: false);
   args.addOption('depfile', help: 'Path to output Ninja depfile');
+  args.addOption('from-dill',
+      help: 'Read existing dill file instead of compiling from sources',
+      defaultsTo: null);
   args.addFlag('link-platform',
       help: 'Include platform into resulting kernel file.', defaultsTo: true);
   args.addFlag('minimal-kernel',
@@ -165,6 +169,7 @@ Future<int> runCompiler(ArgResults options, String usage) async {
   final String targetName = options['target'];
   final String fileSystemScheme = options['filesystem-scheme'];
   final String depfile = options['depfile'];
+  final String fromDillFile = options['from-dill'];
   final List<String> fileSystemRoots = options['filesystem-root'];
   final bool aot = options['aot'];
   final bool tfa = options['tfa'];
@@ -260,7 +265,8 @@ Future<int> runCompiler(ArgResults options, String usage) async {
       dropAST: dropAST && !splitOutputByPackages,
       useProtobufTreeShaker: useProtobufTreeShaker,
       minimalKernel: minimalKernel,
-      treeShakeWriteOnlyFields: treeShakeWriteOnlyFields);
+      treeShakeWriteOnlyFields: treeShakeWriteOnlyFields,
+      fromDillFile: fromDillFile);
 
   errorPrinter.printCompilationMessages();
 
@@ -335,7 +341,8 @@ Future<KernelCompilationResults> compileToKernel(
     bool dropAST: false,
     bool useProtobufTreeShaker: false,
     bool minimalKernel: false,
-    bool treeShakeWriteOnlyFields: false}) async {
+    bool treeShakeWriteOnlyFields: false,
+    String fromDillFile: null}) async {
   // Replace error handler to detect if there are compilation errors.
   final errorDetector =
       new ErrorDetector(previousErrorHandler: options.onDiagnostic);
@@ -344,7 +351,13 @@ Future<KernelCompilationResults> compileToKernel(
   options.environmentDefines =
       options.target.updateEnvironmentDefines(environmentDefines);
 
-  CompilerResult compilerResult = await kernelForProgram(source, options);
+  CompilerResult compilerResult;
+  if (fromDillFile != null) {
+    compilerResult =
+        await loadKernel(options.fileSystem, resolveInputUri(fromDillFile));
+  } else {
+    compilerResult = await kernelForProgram(source, options);
+  }
   Component component = compilerResult?.component;
   Iterable<Uri> compiledSources = component?.uriToSource?.keys;
 
@@ -859,5 +872,25 @@ Future<void> createFarManifest(
   await packageManifest.close();
 }
 
+class CompilerResultLoadedFromKernel implements CompilerResult {
+  final Component component;
+  final Component sdkComponent = Component();
+
+  CompilerResultLoadedFromKernel(this.component);
+
+  List<int> get summary => null;
+  List<Component> get loadedComponents => const <Component>[];
+  List<Uri> get deps => const <Uri>[];
+  CoreTypes get coreTypes => null;
+  ClassHierarchy get classHierarchy => null;
+}
+
+Future<CompilerResult> loadKernel(
+    FileSystem fileSystem, Uri dillFileUri) async {
+  final component = loadComponentFromBinary(
+      (await asFileUri(fileSystem, dillFileUri)).toFilePath());
+  return CompilerResultLoadedFromKernel(component);
+}
+
 // Used by kernel_front_end_test.dart
 main() {}
diff --git a/runtime/tests/vm/dart/split_aot_compilation_test.dart b/runtime/tests/vm/dart/split_aot_compilation_test.dart
@@ -0,0 +1,54 @@
+// Copyright (c) 2020, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+// OtherResources=splay_test.dart
+
+// Tests AOT compilation split into 2 steps using '--from-dill' option.
+
+import 'dart:io' show Platform;
+
+import 'package:path/path.dart' as path;
+import 'snapshot_test_helper.dart';
+
+main() async {
+  final testScriptUri = Platform.script.resolve('splay_test.dart');
+
+  await withTempDir((String temp) async {
+    final intermediateDillPath = path.join(temp, 'intermediate.dill');
+    final outputDillPath = path.join(temp, 'output.dill');
+    final snapshotPath = path.join(temp, 'aot.snapshot');
+
+    final extraGenKernelOptions = Platform.executableArguments
+        .where((arg) =>
+            arg.startsWith('--enable-experiment=') ||
+            arg == '--null-safety' ||
+            arg == '--no-null-safety')
+        .toList();
+
+    await runGenKernel('BUILD INTERMEDIATE DILL FILE', [
+      ...extraGenKernelOptions,
+      '--no-aot',
+      '--link-platform',
+      '--output=$intermediateDillPath',
+      testScriptUri.toFilePath(),
+    ]);
+
+    await runGenKernel('BUILD FINAL DILL FILE', [
+      ...extraGenKernelOptions,
+      '--aot',
+      '--from-dill=$intermediateDillPath',
+      '--link-platform',
+      '--output=$outputDillPath',
+      testScriptUri.toFilePath(),
+    ]);
+
+    await runGenSnapshot('GENERATE SNAPSHOT', [
+      '--snapshot-kind=app-aot-elf',
+      '--elf=$snapshotPath',
+      outputDillPath,
+    ]);
+
+    await runBinary('RUN SNAPSHOT', dartPrecompiledRuntime, [snapshotPath]);
+  });
+}
diff --git a/runtime/tests/vm/dart_2/split_aot_compilation_test.dart b/runtime/tests/vm/dart_2/split_aot_compilation_test.dart
@@ -0,0 +1,54 @@
+// Copyright (c) 2020, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+// OtherResources=splay_test.dart
+
+// Tests AOT compilation split into 2 steps using '--from-dill' option.
+
+import 'dart:io' show Platform;
+
+import 'package:path/path.dart' as path;
+import 'snapshot_test_helper.dart';
+
+main() async {
+  final testScriptUri = Platform.script.resolve('splay_test.dart');
+
+  await withTempDir((String temp) async {
+    final intermediateDillPath = path.join(temp, 'intermediate.dill');
+    final outputDillPath = path.join(temp, 'output.dill');
+    final snapshotPath = path.join(temp, 'aot.snapshot');
+
+    final extraGenKernelOptions = Platform.executableArguments
+        .where((arg) =>
+            arg.startsWith('--enable-experiment=') ||
+            arg == '--null-safety' ||
+            arg == '--no-null-safety')
+        .toList();
+
+    await runGenKernel('BUILD INTERMEDIATE DILL FILE', [
+      ...extraGenKernelOptions,
+      '--no-aot',
+      '--link-platform',
+      '--output=$intermediateDillPath',
+      testScriptUri.toFilePath(),
+    ]);
+
+    await runGenKernel('BUILD FINAL DILL FILE', [
+      ...extraGenKernelOptions,
+      '--aot',
+      '--from-dill=$intermediateDillPath',
+      '--link-platform',
+      '--output=$outputDillPath',
+      testScriptUri.toFilePath(),
+    ]);
+
+    await runGenSnapshot('GENERATE SNAPSHOT', [
+      '--snapshot-kind=app-aot-elf',
+      '--elf=$snapshotPath',
+      outputDillPath,
+    ]);
+
+    await runBinary('RUN SNAPSHOT', dartPrecompiledRuntime, [snapshotPath]);
+  });
+}
diff --git a/runtime/tests/vm/vm.status b/runtime/tests/vm/vm.status
@@ -76,8 +76,10 @@ dart_2/transferable_throws_oom_test: SkipByDesign # This test tries to allocate
 [ $builder_tag == crossword || $builder_tag == crossword_ast ]
 dart/emit_aot_size_info_flag_test: SkipByDesign # The test itself cannot determine the location of gen_snapshot (only tools/test.py knows where it is).
 dart/bytecode_with_ast_in_aot_test: SkipByDesign # The test doesn't know location of cross-platform gen_snapshot
+dart/split_aot_compilation_test: SkipByDesign # The test doesn't know location of cross-platform gen_snapshot
 dart_2/emit_aot_size_info_flag_test: SkipByDesign # The test itself cannot determine the location of gen_snapshot (only tools/test.py knows where it is).
 dart_2/bytecode_with_ast_in_aot_test: SkipByDesign # The test doesn't know location of cross-platform gen_snapshot
+dart_2/split_aot_compilation_test: SkipByDesign # The test doesn't know location of cross-platform gen_snapshot
 
 [ $builder_tag == obfuscated ]
 dart/causal_stacks/async_throws_stack_lazy_test: SkipByDesign # Asserts exact stacktrace output.
@@ -396,9 +398,11 @@ dart_2/redirection_type_shuffling_test/none: RuntimeError
 dart/emit_aot_size_info_flag_test: SkipByDesign # This test is for VM AOT only and is quite slow (so we don't run it in debug mode).
 dart/use_bare_instructions_flag_test: SkipByDesign # This test is for VM AOT only and is quite slow (so we don't run it in debug mode).
 dart/bytecode_with_ast_in_aot_test: SkipByDesign # This test is for VM AOT only and is quite slow (so we don't run it in debug mode).
+dart/split_aot_compilation_test: SkipByDesign # This test is for VM AOT only and is quite slow (so we don't run it in debug mode).
 dart_2/emit_aot_size_info_flag_test: SkipByDesign # This test is for VM AOT only and is quite slow (so we don't run it in debug mode).
 dart_2/use_bare_instructions_flag_test: SkipByDesign # This test is for VM AOT only and is quite slow (so we don't run it in debug mode).
 dart_2/bytecode_with_ast_in_aot_test: SkipByDesign # This test is for VM AOT only and is quite slow (so we don't run it in debug mode).
+dart_2/split_aot_compilation_test: SkipByDesign # This test is for VM AOT only and is quite slow (so we don't run it in debug mode).
 
 # It makes no sense to run any test that uses spawnURI under the simulator
 # as that would involve running CFE (the front end) in simulator mode
