[vm/bytecode] Declare members in bytecode

This change replaces kernel AST declarations of fields and functions
with bytecode declarations.

Size of dilp files is reduced by 11-12%.

Startup latency:
Time to the first full frame: 1.945s -> 1.687s
FinalizeClass: 554ms -> 277ms
FinishClassLoading: 296ms -> 156ms

There are following regressions in bytecode mode, which will be fixed
in future:

* dart:mirrors are not supported yet (implementation of mirrors relies
  on reading kernel AST in certain cases).

  As the result, lib_2/mirrors/* tests fail.

* native extensions are not supported yet (annotations on libraries
  and classes in AST are cleaned up as they could reference members
  which are now removed from AST).

  As the result, standalone_2/entrypoints_verification_test test fails.

* language_2/spread_collections/const_error_test/* tests fail
  due to dart-lang/sdk#36286.

Change-Id: I5130f401fd7b84038b136136e7ccc1a6e51b6cea
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/97561
Commit-Queue: Alexander Markov <alexmarkov@google.com>
Reviewed-by: Ryan Macnak <rmacnak@google.com>

pkg/kernel/lib/text/ast_to_text.dart

@@ -477,6 +477,9 @@ class Printer extends Visitor<Null> {
         }
         writeWord('external');
       }
+      if (showMetadata) {
+        inner.writeMetadata(library);
+      }
       writeAnnotationList(library.annotations);
       writeWord('library');
       if (library.name != null) {

pkg/vm/lib/bytecode/ast_remover.dart

@@ -11,64 +11,167 @@ import '../metadata/bytecode.dart';
 /// Can preserve removed AST and restore it if needed.
 class ASTRemover extends Transformer {
   final BytecodeMetadataRepository metadata;
-  final droppedAST = <Member, dynamic>{};
+  final stashes = <Node, _Stash>{};
 
   ASTRemover(Component component)
-      : metadata = component.metadata[new BytecodeMetadataRepository().tag];
+      : metadata = component.metadata[new BytecodeMetadataRepository().tag] {
+    stashes[component] = new _ComponentStash(component.mainMethod);
+    component.mainMethod = null;
+  }
 
   @override
-  TreeNode defaultMember(Member node) {
-    if (_hasBytecode(node)) {
-      if (node is Field) {
-        droppedAST[node] = node.initializer;
-        node.initializer = null;
-      } else if (node is Constructor) {
-        droppedAST[node] =
-            new _DroppedConstructor(node.initializers, node.function.body);
-        node.initializers = <Initializer>[];
-        node.function.body = null;
-      } else if (node.function != null) {
-        droppedAST[node] = node.function.body;
-        node.function.body = null;
-      }
-    }
-
-    // Instance field initializers do not form separate functions, and bytecode
-    // is not attached to instance fields (it is included into constructors).
-    // When VM reads a constructor from kernel, it also reads and translates
-    // instance field initializers. So, their ASTs can be dropped only if
-    // bytecode was generated for all generative constructors.
-    if (node is Field && !node.isStatic && node.initializer != null) {
-      if (node.enclosingClass.constructors.every(_hasBytecode)) {
-        droppedAST[node] = node.initializer;
-        node.initializer = null;
-      }
-    }
+  visitLibrary(Library node) {
+    stashes[node] = new _LibraryStash(
+        new List<Expression>.from(node.annotations),
+        new List<Field>.from(node.fields),
+        new List<Procedure>.from(node.procedures),
+        new List<Reference>.from(node.additionalExports));
+
+    node.annotations.clear();
+    node.fields.clear();
+    node.procedures.clear();
+    node.additionalExports.clear();
+
+    super.visitLibrary(node);
 
     return node;
   }
 
-  bool _hasBytecode(Member node) =>
-      metadata != null && metadata.mapping.containsKey(node);
+  @override
+  visitLibraryDependency(LibraryDependency node) {
+    stashes[node] = new _LibraryDependencyStash(
+        new List<Expression>.from(node.annotations));
+
+    node.annotations.clear();
+
+    super.visitLibraryDependency(node);
+
+    return node;
+  }
+
+  // Still referenced from function types which may appear in class supertypes.
+  @override
+  visitTypedef(Typedef node) {
+    stashes[node] = new _TypedefStash(node.annotations);
+
+    node.annotations = const <Expression>[];
+
+    super.visitTypedef(node);
+
+    // TODO(alexmarkov): fix Typedef visitor to visit these fields.
+    transformList(node.positionalParameters, this, node);
+    transformList(node.namedParameters, this, node);
+
+    return node;
+  }
+
+  // May appear in typedefs.
+  @override
+  visitVariableDeclaration(VariableDeclaration node) {
+    stashes[node] = new _VariableDeclarationStash(node.annotations);
+
+    node.annotations = const <Expression>[];
+
+    super.visitVariableDeclaration(node);
+
+    return node;
+  }
+
+  @override
+  visitClass(Class node) {
+    stashes[node] = new _ClassStash(
+        node.annotations,
+        new List<Field>.from(node.fields),
+        new List<Procedure>.from(node.procedures),
+        new List<Constructor>.from(node.constructors));
+
+    node.annotations = const <Expression>[];
+    node.fields.clear();
+    node.procedures.clear();
+    node.constructors.clear();
+
+    super.visitClass(node);
+
+    return node;
+  }
 
   void restoreAST() {
-    droppedAST.forEach((Member node, dynamic dropped) {
-      if (node is Field) {
-        node.initializer = dropped;
-      } else if (node is Constructor) {
-        _DroppedConstructor droppedConstructor = dropped;
-        node.initializers = droppedConstructor.initializers;
-        node.function.body = droppedConstructor.body;
+    stashes.forEach((Node node, _Stash stash) {
+      if (node is Component) {
+        _ComponentStash componentStash = stash as _ComponentStash;
+        node.mainMethod = componentStash.mainMethod;
+      } else if (node is Library) {
+        _LibraryStash libraryStash = stash as _LibraryStash;
+        node.annotations.addAll(libraryStash.annotations);
+        node.fields.addAll(libraryStash.fields);
+        node.procedures.addAll(libraryStash.procedures);
+        node.additionalExports.addAll(libraryStash.additionalExports);
+      } else if (node is LibraryDependency) {
+        _LibraryDependencyStash libraryDependencyStash =
+            stash as _LibraryDependencyStash;
+        node.annotations.addAll(libraryDependencyStash.annotations);
+      } else if (node is Typedef) {
+        _TypedefStash typedefStash = stash as _TypedefStash;
+        node.annotations = typedefStash.annotations;
+      } else if (node is VariableDeclaration) {
+        _VariableDeclarationStash variableDeclarationStash =
+            stash as _VariableDeclarationStash;
+        node.annotations = variableDeclarationStash.annotations;
+      } else if (node is Class) {
+        _ClassStash classStash = stash as _ClassStash;
+        node.annotations = classStash.annotations;
+        node.fields.addAll(classStash.fields);
+        node.procedures.addAll(classStash.procedures);
+        node.constructors.addAll(classStash.constructors);
       } else {
-        node.function.body = dropped;
+        throw 'Unexpected ${node.runtimeType} $node';
       }
     });
   }
 }
 
-class _DroppedConstructor {
-  final List<Initializer> initializers;
-  final Statement body;
+abstract class _Stash {}
+
+class _ClassStash extends _Stash {
+  final List<Expression> annotations;
+  final List<Field> fields;
+  final List<Procedure> procedures;
+  final List<Constructor> constructors;
+
+  _ClassStash(
+      this.annotations, this.fields, this.procedures, this.constructors);
+}
+
+class _LibraryStash extends _Stash {
+  final List<Expression> annotations;
+  final List<Field> fields;
+  final List<Procedure> procedures;
+  final List<Reference> additionalExports;
+
+  _LibraryStash(
+      this.annotations, this.fields, this.procedures, this.additionalExports);
+}
+
+class _LibraryDependencyStash extends _Stash {
+  final List<Expression> annotations;
+
+  _LibraryDependencyStash(this.annotations);
+}
+
+class _TypedefStash extends _Stash {
+  final List<Expression> annotations;
+
+  _TypedefStash(this.annotations);
+}
+
+class _VariableDeclarationStash extends _Stash {
+  final List<Expression> annotations;
+
+  _VariableDeclarationStash(this.annotations);
+}
+
+class _ComponentStash extends _Stash {
+  final Procedure mainMethod;
 
-  _DroppedConstructor(this.initializers, this.body);
+  _ComponentStash(this.mainMethod);
 }

pkg/vm/lib/bytecode/bytecode_serialization.dart

@@ -29,15 +29,17 @@ class BufferedWriter {
   final int formatVersion;
   final StringWriter stringWriter;
   final ObjectWriter objectWriter;
+  final LinkWriter linkWriter;
   final BytesBuilder bytes = new BytesBuilder();
   final int baseOffset;
 
-  BufferedWriter(this.formatVersion, this.stringWriter, this.objectWriter,
+  BufferedWriter(
+      this.formatVersion, this.stringWriter, this.objectWriter, this.linkWriter,
       {this.baseOffset: 0});
 
   factory BufferedWriter.fromWriter(BufferedWriter writer) =>
-      new BufferedWriter(
-          writer.formatVersion, writer.stringWriter, writer.objectWriter);
+      new BufferedWriter(writer.formatVersion, writer.stringWriter,
+          writer.objectWriter, writer.linkWriter);
 
   List<int> takeBytes() => bytes.takeBytes();
 
@@ -110,6 +112,11 @@ class BufferedWriter {
     }
   }
 
+  void writeLinkOffset(Object target) {
+    final offset = linkWriter.getOffset(target);
+    writePackedUInt30(offset);
+  }
+
   void align(int alignment) {
     assert(alignment & (alignment - 1) == 0);
     int offs = baseOffset + offset;
@@ -124,14 +131,15 @@ class BufferedReader {
   int formatVersion;
   StringReader stringReader;
   ObjectReader objectReader;
+  LinkReader linkReader;
   final List<int> bytes;
   final int baseOffset;
 
   /// Position within [bytes], already includes [baseOffset].
   int _pos;
 
-  BufferedReader(
-      this.formatVersion, this.stringReader, this.objectReader, this.bytes,
+  BufferedReader(this.formatVersion, this.stringReader, this.objectReader,
+      this.linkReader, this.bytes,
       {this.baseOffset: 0})
       : _pos = baseOffset {
     assert((0 <= _pos) && (_pos <= bytes.length));
@@ -221,6 +229,11 @@ class BufferedReader {
     return result;
   }
 
+  T readLinkOffset<T>() {
+    final offset = readPackedUInt30();
+    return linkReader.get<T>(offset);
+  }
+
   void align(int alignment) {
     assert(alignment & (alignment - 1) == 0);
     _pos = ((_pos + alignment - 1) & -alignment);
@@ -384,6 +397,37 @@ class NamedEntryStatistics {
       "  (count: ${count.toString().padLeft(8)})";
 }
 
+class LinkWriter {
+  final _map = <Object, int>{};
+
+  void put(Object target, int offset) {
+    _map[target] = offset;
+  }
+
+  int getOffset(Object target) {
+    return _map[target] ??
+        (throw 'Offset of ${target.runtimeType} $target is not set');
+  }
+}
+
+class LinkReader {
+  final _map = <Type, Map<int, Object>>{};
+
+  void setOffset<T>(T target, int offset) {
+    final offsetToObject = (_map[T] ??= <int, Object>{});
+    final previous = offsetToObject[offset];
+    if (previous != null) {
+      throw 'Unable to associate offset $T/$offset with ${target.runtimeType} $target.'
+          ' It is already associated with ${previous.runtimeType} $previous';
+    }
+    offsetToObject[offset] = target;
+  }
+
+  T get<T>(int offset) {
+    return _map[T][offset] ?? (throw 'No object at offset $T/$offset');
+  }
+}
+
 class BytecodeSizeStatistics {
   static int componentSize = 0;
   static int objectTableSize = 0;

pkg/vm/lib/bytecode/dbc.dart

@@ -8,16 +8,14 @@ library vm.bytecode.dbc;
 
 /// Version of bytecode format, produced by default.
 /// Before bumping current bytecode version format, make sure that
-/// all users have switched to a VM which is able to consume next
+/// all users have switched to a VM which is able to consume new
 /// version of bytecode.
-const int stableBytecodeFormatVersion = 2;
+const int currentBytecodeFormatVersion = 3;
 
-/// Version of bleeding edge bytecode format.
+/// Version of experimental / bleeding edge bytecode format.
 /// Produced by bytecode generator when --use-future-bytecode-format
 /// option is enabled.
-/// Should match kMaxSupportedBytecodeFormatVersion in
-/// runtime/vm/constants_kbc.h.
-const int futureBytecodeFormatVersion = stableBytecodeFormatVersion + 1;
+const int futureBytecodeFormatVersion = currentBytecodeFormatVersion + 1;
 
 /// Alignment of bytecode instructions.
 const int bytecodeInstructionsAlignment = 4;