Support digit separators

Work towards dart-lang/language#2

The feature is well-specified at the issue, but I will also follow
up with a specification to check into the language repo.

This change implements the feature more-or-less from front to back
(because the back is very close to the front in this case :P; no
"backend" work in the VM, etc). Digit separators are made available
via a new experiment, `digit-separators`.

Care is taken to report a single error when an underscore appears in
an unexpected position (see new `separators_error_test.dart`).

Three test files are added:

* `separators_test.dart` is run with the experiment enabled, and has
  no compile-time errors.
* `separators_error_test.dart` is run with the experiment enabled, and
  has many compile-time errors.
* `separators_error_no_experiment_test.dart` is run with the
  experiment _disabled_.

Change-Id: I7f1b1305d28b708b5ddf83f26188cd6e9ce3dd58
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/365181
Commit-Queue: Samuel Rawlins <srawlins@google.com>
Reviewed-by: Lasse Nielsen <lrn@google.com>
Reviewed-by: Kevin Moore <kevmoo@google.com>
Reviewed-by: Jens Johansen <jensj@google.com>

Author: srawlins

pkg/_fe_analyzer_shared/lib/src/experiments/flags.dart

@@ -45,6 +45,13 @@ enum ExperimentalFlag {
       experimentEnabledVersion: const Version(2, 0),
       experimentReleasedVersion: const Version(2, 0)),
 
+  digitSeparators(
+      name: 'digit-separators',
+      isEnabledByDefault: false,
+      isExpired: false,
+      experimentEnabledVersion: defaultLanguageVersion,
+      experimentReleasedVersion: defaultLanguageVersion),
+
   enhancedEnums(
       name: 'enhanced-enums',
       isEnabledByDefault: true,

pkg/_fe_analyzer_shared/lib/src/messages/codes_generated.dart

@@ -16803,6 +16803,19 @@ Message _withArgumentsUnexpectedModifierInNonNnbd(Token token) {
   );
 }
 
+// DO NOT EDIT. THIS FILE IS GENERATED. SEE TOP OF FILE.
+const Code<Null> codeUnexpectedSeparatorInNumber =
+    messageUnexpectedSeparatorInNumber;
+
+// DO NOT EDIT. THIS FILE IS GENERATED. SEE TOP OF FILE.
+const MessageCode messageUnexpectedSeparatorInNumber = const MessageCode(
+  "UnexpectedSeparatorInNumber",
+  analyzerCodes: <String>["UNEXPECTED_SEPARATOR_IN_NUMBER"],
+  problemMessage:
+      r"""Digit separators ('_') in a number literal can only be placed between two digits.""",
+  correctionMessage: r"""Try removing the '_'.""",
+);
+
 // DO NOT EDIT. THIS FILE IS GENERATED. SEE TOP OF FILE.
 const Code<Null> codeUnexpectedSuperParametersInGenerativeConstructors =
     messageUnexpectedSuperParametersInGenerativeConstructors;

pkg/_fe_analyzer_shared/lib/src/parser/forwarding_listener.dart

@@ -1632,11 +1632,21 @@ class ForwardingListener implements Listener {
     listener?.handleLiteralDouble(token);
   }
 
+  @override
+  void handleLiteralDoubleWithSeparators(Token token) {
+    listener?.handleLiteralDoubleWithSeparators(token);
+  }
+
   @override
   void handleLiteralInt(Token token) {
     listener?.handleLiteralInt(token);
   }
 
+  @override
+  void handleLiteralIntWithSeparators(Token token) {
+    listener?.handleLiteralIntWithSeparators(token);
+  }
+
   @override
   void handleLiteralList(
       int count, Token beginToken, Token? constKeyword, Token endToken) {

pkg/_fe_analyzer_shared/lib/src/parser/identifier_context.dart

@@ -334,10 +334,13 @@ bool looksLikeExpressionStart(Token next) =>
     next.isIdentifier ||
     next.isKeyword && !looksLikeStatementStart(next) ||
     next.type == TokenType.DOUBLE ||
+    next.type == TokenType.DOUBLE_WITH_SEPARATORS ||
     next.type == TokenType.HASH ||
     next.type == TokenType.HEXADECIMAL ||
+    next.type == TokenType.HEXADECIMAL_WITH_SEPARATORS ||
     next.type == TokenType.IDENTIFIER ||
     next.type == TokenType.INT ||
+    next.type == TokenType.INT_WITH_SEPARATORS ||
     next.type == TokenType.STRING ||
     optional('{', next) ||
     optional('(', next) ||
@@ -358,10 +361,13 @@ bool looksLikeExpressionStart(Token next) =>
 bool looksLikePatternStart(Token next) =>
     next.isIdentifier ||
     next.type == TokenType.DOUBLE ||
+    next.type == TokenType.DOUBLE_WITH_SEPARATORS ||
     next.type == TokenType.HASH ||
     next.type == TokenType.HEXADECIMAL ||
+    next.type == TokenType.HEXADECIMAL_WITH_SEPARATORS ||
     next.type == TokenType.IDENTIFIER ||
     next.type == TokenType.INT ||
+    next.type == TokenType.INT_WITH_SEPARATORS ||
     next.type == TokenType.STRING ||
     optional('null', next) ||
     optional('false', next) ||

pkg/_fe_analyzer_shared/lib/src/parser/listener.dart

@@ -1964,20 +1964,30 @@ class Listener implements UnescapeErrorListener {
     logEvent("Assert");
   }
 
-  /** Called with either the token containing a double literal, or
-    * an immediately preceding "unary plus" token.
-    */
+  /// Called with either the token containing a double literal, or an
+  /// immediately preceding "unary minus" token.
   void handleLiteralDouble(Token token) {
     logEvent("LiteralDouble");
   }
 
-  /** Called with either the token containing an integer literal,
-    * or an immediately preceding "unary plus" token.
-    */
+  /// Called with either the token containing a double literal with separators,
+  /// or an immediately preceding "unary minus" token.
+  void handleLiteralDoubleWithSeparators(Token token) {
+    logEvent("LiteralDoubleWithSeparators");
+  }
+
+  /// Called with either the token containing an integer literal, or an
+  /// immediately preceding "unary minus" token.
   void handleLiteralInt(Token token) {
     logEvent("LiteralInt");
   }
 
+  /// Called with either the token containing an integer literal with
+  /// separators, or an immediately preceding "unary minus" token.
+  void handleLiteralIntWithSeparators(Token token) {
+    logEvent("LiteralIntWithSeparators");
+  }
+
   void handleLiteralList(
       int count, Token leftBracket, Token? constKeyword, Token rightBracket) {
     logEvent("LiteralList");

pkg/_fe_analyzer_shared/lib/src/parser/parser_impl.dart

@@ -6544,13 +6544,22 @@ class Parser {
         reportRecoverableError(
             next, codes.messageInvalidConstantPatternConstPrefix);
       }
-      return parseLiteralInt(token);
+      if (identical(next.type, TokenType.INT_WITH_SEPARATORS) ||
+          identical(next.type, TokenType.HEXADECIMAL_WITH_SEPARATORS)) {
+        return parseLiteralIntWithSeparators(token);
+      } else {
+        return parseLiteralInt(token);
+      }
     } else if (kind == DOUBLE_TOKEN) {
       if (constantPatternContext == ConstantPatternContext.explicit) {
         reportRecoverableError(
             next, codes.messageInvalidConstantPatternConstPrefix);
       }
-      return parseLiteralDouble(token);
+      if (identical(next.type, TokenType.DOUBLE_WITH_SEPARATORS)) {
+        return parseLiteralDoubleWithSeparators(token);
+      } else {
+        return parseLiteralDouble(token);
+      }
     } else if (kind == STRING_TOKEN) {
       if (constantPatternContext == ConstantPatternContext.explicit) {
         reportRecoverableError(
@@ -7441,6 +7450,14 @@ class Parser {
     return token;
   }
 
+  Token parseLiteralIntWithSeparators(Token token) {
+    token = token.next!;
+    assert(identical(token.kind, INT_TOKEN) ||
+        identical(token.kind, HEXADECIMAL_TOKEN));
+    listener.handleLiteralIntWithSeparators(token);
+    return token;
+  }
+
   /// ```
   /// doubleLiteral:
   ///   double
@@ -7453,6 +7470,13 @@ class Parser {
     return token;
   }
 
+  Token parseLiteralDoubleWithSeparators(Token token) {
+    token = token.next!;
+    assert(identical(token.kind, DOUBLE_TOKEN));
+    listener.handleLiteralDoubleWithSeparators(token);
+    return token;
+  }
+
   /// ```
   /// stringLiteral:
   ///   (multilineString | singleLineString)+

pkg/_fe_analyzer_shared/lib/src/parser/util.dart

@@ -4,6 +4,8 @@
 
 library _fe_analyzer_shared.parser.util;
 
+import 'dart:typed_data';
+
 import '../messages/codes.dart' show noLength;
 
 import '../scanner/scanner.dart' show Token;
@@ -216,6 +218,38 @@ Token splitGtFromGtGtGtEq(Token token) {
       ..next = token.next);
 }
 
+/// Strips separator characters (underscore) from [source].
+///
+/// No validation is performed on [source]; it could be a valid int, a valid
+/// double, or invalid.
+String stripSeparators(String source) {
+  Uint8List list = _separatorStripBuffer;
+  if (list.length < source.length - 1) {
+    // Looking at a very long number. Allocate a new buffer.
+    // We only strip separators after finding that there is at least one
+    // separator, so the length can be reduced by at least one character.
+    list = new Uint8List(source.length - 1);
+    if (list.length < 128) {
+      // Store the new, larger list as the reusable buffer.
+      _separatorStripBuffer = list;
+    }
+  }
+
+  int writeIndex = 0;
+  for (int i = 0; i < source.length; i++) {
+    int char = source.codeUnitAt(i);
+    if (char != 0x5f /* _ */) list[writeIndex++] = char;
+  }
+  return new String.fromCharCodes(list, 0, writeIndex);
+}
+
+/// A reusable buffer for stripping separators from number literals.
+///
+/// The majority of number literals fit in 24 characters. A maximal double with
+/// no unnecessary leading or trailing zeros is 17 digits, one decimal point,
+/// one 'e', two '-'s, and three exponent digits: 24 characters.
+Uint8List _separatorStripBuffer = new Uint8List(24);
+
 /// Return a synthetic `>` followed by [next].
 /// Call [Token.setNext] to add the token to the stream.
 Token syntheticGt(Token next) {