[vm/bytecode] Check for overflow of bytecode limits

Change-Id: I452596fb6f0fa72d76f26981adae2910ba1a179c
Reviewed-on: https://dart-review.googlesource.com/c/81336
Reviewed-by: Régis Crelier <regis@google.com>
Commit-Queue: Alexander Markov <alexmarkov@google.com>

Author: alexmarkov

pkg/vm/lib/bytecode/constant_pool.dart

@@ -9,6 +9,8 @@ import 'dart:typed_data';
 import 'package:kernel/ast.dart' hide MapEntry;
 import 'package:kernel/text/ast_to_text.dart' show Printer;
 
+import 'dbc.dart' show constantPoolIndexLimit, BytecodeLimitExceededException;
+
 /*
 
 In kernel binary, constant pool is encoded in the following way
@@ -1166,6 +1168,9 @@ class ConstantPool {
   int add(ConstantPoolEntry entry) {
     return _canonicalizationCache.putIfAbsent(entry, () {
       int index = entries.length;
+      if (index >= constantPoolIndexLimit) {
+        throw new ConstantPoolIndexOverflowException();
+      }
       _addEntry(entry);
       return index;
     });
@@ -1252,3 +1257,6 @@ class ConstantPool {
 
 int _combineHashes(int hash1, int hash2) =>
     (((hash1 * 31) & 0x3fffffff) + hash2) & 0x3fffffff;
+
+class ConstantPoolIndexOverflowException
+    extends BytecodeLimitExceededException {}

pkg/vm/lib/bytecode/dbc.dart

@@ -303,3 +303,17 @@ enum SpecialIndex {
 }
 
 bool isJump(Opcode opcode) => BytecodeFormats[opcode].encoding == Encoding.kT;
+
+// Bytecode instructions reference constant pool indices using
+// unsigned 16-bit operands.
+const int constantPoolIndexLimit = 1 << 16;
+
+// Local variables are referenced using 16-bit signed operands.
+const int localVariableIndexLimit = 1 << 15;
+
+// Captured variables are referenced using 16-bit unsigned operands.
+const int capturedVariableIndexLimit = 1 << 16;
+
+// Base class for exceptions thrown when certain limit of bytecode
+// format is exceeded.
+abstract class BytecodeLimitExceededException {}

pkg/vm/lib/bytecode/gen_bytecode.dart

@@ -141,35 +141,41 @@ class BytecodeGenerator extends RecursiveVisitor<Null> {
     if (node.isAbstract) {
       return;
     }
-    if (node is Field) {
-      if (node.isStatic && !_hasTrivialInitializer(node)) {
+    try {
+      if (node is Field) {
+        if (node.isStatic && !_hasTrivialInitializer(node)) {
+          start(node);
+          if (node.isConst) {
+            _genPushConstExpr(node.initializer);
+          } else {
+            node.initializer.accept(this);
+          }
+          _genReturnTOS();
+          end(node);
+        }
+      } else if ((node is Procedure && !node.isRedirectingFactoryConstructor) ||
+          (node is Constructor)) {
         start(node);
-        if (node.isConst) {
-          _genPushConstExpr(node.initializer);
+        if (node is Constructor) {
+          _genConstructorInitializers(node);
+        }
+        if (node.isExternal) {
+          final String nativeName = getExternalName(node);
+          if (nativeName == null) {
+            return;
+          }
+          _genNativeCall(nativeName);
         } else {
-          node.initializer.accept(this);
+          node.function?.body?.accept(this);
+          // BytecodeAssembler eliminates this bytecode if it is unreachable.
+          asm.emitPushNull();
         }
         _genReturnTOS();
         end(node);
       }
-    } else if ((node is Procedure && !node.isRedirectingFactoryConstructor) ||
-        (node is Constructor)) {
-      start(node);
-      if (node is Constructor) {
-        _genConstructorInitializers(node);
-      }
-      if (node.isExternal) {
-        final String nativeName = getExternalName(node);
-        if (nativeName == null) {
-          return;
-        }
-        _genNativeCall(nativeName);
-      } else {
-        node.function?.body?.accept(this);
-        // BytecodeAssembler eliminates this bytecode if it is unreachable.
-        asm.emitPushNull();
-      }
-      _genReturnTOS();
+    } on BytecodeLimitExceededException {
+      // Do not generate bytecode and fall back to using kernel AST.
+      hasErrors = true;
       end(node);
     }
   }

pkg/vm/lib/bytecode/local_vars.dart

@@ -832,6 +832,9 @@ class _Allocator extends RecursiveVisitor<Null> {
 
     if (v.isCaptured) {
       v.index = _currentScope.contextOwner.contextUsed++;
+      if (v.index >= capturedVariableIndexLimit) {
+        throw new LocalVariableIndexOverflowException();
+      }
       v.originalParamSlotIndex = paramSlotIndex;
       return;
     }
@@ -843,6 +846,9 @@ class _Allocator extends RecursiveVisitor<Null> {
       v.index = paramSlotIndex;
     } else {
       v.index = _currentScope.localsUsed++;
+      if (v.index >= localVariableIndexLimit) {
+        throw new LocalVariableIndexOverflowException();
+      }
     }
     _updateFrameSize();
   }
@@ -1139,3 +1145,6 @@ class _Allocator extends RecursiveVisitor<Null> {
     _visit(node, temps: 3);
   }
 }
+
+class LocalVariableIndexOverflowException
+    extends BytecodeLimitExceededException {}