Update AstBuilder to build AST with nullable types

... as part of adding NNBD as outlined in
dart-lang/language#110

This CL updates the parser to recognize two specialized comments at the
beginning of a library of the form '//@NNBD' and '//@NNBD*' for use
by the analyzer to aid developers when converting their libraries
to be non-nullable by default.

In addition, the AstBuilder now populates the generated analyzer AST
with nullable type information.

Change-Id: I80d221dd138973aa32f05bde631245d9ac6ee10f
Reviewed-on: https://dart-review.googlesource.com/c/87540
Reviewed-by: Brian Wilkerson <brianwilkerson@google.com>
Commit-Queue: Dan Rubel <danrubel@google.com>

pkg/analyzer/lib/src/fasta/ast_builder.dart

@@ -96,6 +96,8 @@ class AstBuilder extends StackListener {
 
   bool parseFunctionBodies = true;
 
+  bool showNullableTypeErrors = true;
+
   AstBuilder(ErrorReporter errorReporter, this.fileUri, this.isFullAst,
       [Uri uri])
       : this.errorReporter = new FastaErrorReporter(errorReporter),
@@ -263,6 +265,11 @@ class AstBuilder extends StackListener {
     scriptTag = ast.scriptTag(token);
   }
 
+  @override
+  void handleNNBD(bool isStar) {
+    showNullableTypeErrors = !isStar;
+  }
+
   void handleStringJuxtaposition(int literalCount) {
     debugEvent("StringJuxtaposition");
 
@@ -962,11 +969,13 @@ class AstBuilder extends StackListener {
   @override
   void handleType(Token beginToken, Token questionMark) {
     debugEvent("Type");
-    reportErrorIfNullableType(questionMark);
+    if (showNullableTypeErrors) {
+      reportErrorIfNullableType(questionMark);
+    }
 
     TypeArgumentList arguments = pop();
     Identifier name = pop();
-    push(ast.typeName(name, arguments));
+    push(ast.typeName(name, arguments, question: questionMark));
   }
 
   @override
@@ -1121,13 +1130,16 @@ class AstBuilder extends StackListener {
   void endFunctionType(Token functionToken, Token questionMark) {
     assert(optional('Function', functionToken));
     debugEvent("FunctionType");
-    reportErrorIfNullableType(questionMark);
+    if (showNullableTypeErrors) {
+      reportErrorIfNullableType(questionMark);
+    }
 
     FormalParameterList parameters = pop();
     TypeAnnotation returnType = pop();
     TypeParameterList typeParameters = pop();
     push(ast.genericFunctionType(
-        returnType, functionToken, typeParameters, parameters));
+        returnType, functionToken, typeParameters, parameters,
+        question: questionMark));
   }
 
   void handleFormalParameterWithoutValue(Token token) {

pkg/analyzer/test/generated/parser_fasta_test.dart

@@ -15,6 +15,7 @@ import 'package:analyzer/src/generated/parser.dart' as analyzer;
 import 'package:analyzer/src/generated/utilities_dart.dart';
 import 'package:analyzer/src/string_source.dart';
 import 'package:front_end/src/fasta/parser/parser.dart' as fasta;
+import 'package:front_end/src/fasta/parser/forwarding_listener.dart' as fasta;
 import 'package:front_end/src/fasta/scanner.dart'
     show ScannerResult, scanString;
 import 'package:front_end/src/fasta/scanner/error_token.dart' show ErrorToken;
@@ -1419,4 +1420,41 @@ mixin A {
     MixinDeclaration declaration = parseFullCompilationUnitMember();
     expectCommentText(declaration.documentationComment, '/// Doc');
   }
+
+  void test_parseNNDB() {
+    void testNNBD(String comments, {bool nnbd, bool star}) {
+      final listener = new NNBDTestListener();
+      String code = '''
+$comments
+main() {}''';
+      ScannerResult result = scanString(code, includeComments: true);
+      new fasta.Parser(listener).parseUnit(result.tokens);
+
+      expect(listener.isNNBD, nnbd ?? isNull);
+      expect(listener.isStar, star ?? isNull);
+    }
+
+    testNNBD('', nnbd: false);
+    testNNBD('#!/bin/dart', nnbd: false);
+    testNNBD('//@NNBDx', nnbd: false);
+    testNNBD('//@NNBD', nnbd: true, star: false);
+    testNNBD('// @NNBD', nnbd: true, star: false);
+    testNNBD('//@NNBD*', nnbd: true, star: true);
+    testNNBD('// @NNBD*', nnbd: true, star: true);
+    testNNBD('''#!/bin/dart
+// @NNBD*
+/* more comments */
+/// and dartdoc''', nnbd: true, star: true);
+  }
+}
+
+class NNBDTestListener extends fasta.ForwardingListener {
+  bool isNNBD = false;
+  bool isStar;
+
+  @override
+  void handleNNBD(bool isStar) {
+    this.isNNBD = true;
+    this.isStar = isStar;
+  }
 }

pkg/front_end/lib/src/fasta/parser/forwarding_listener.dart

@@ -1204,6 +1204,11 @@ class ForwardingListener implements Listener {
     listener?.handleForInitializerLocalVariableDeclaration(token);
   }
 
+  @override
+  void handleNNBD(bool isStar) {
+    listener?.handleNNBD(isStar);
+  }
+
   @override
   void handleNoFieldInitializer(Token token) {
     listener?.handleNoFieldInitializer(token);

pkg/front_end/lib/src/fasta/parser/listener.dart

@@ -1357,4 +1357,13 @@ class Listener implements UnescapeErrorListener {
   /// This event is generated by the parser when the parser's
   /// `parseOneCommentReference` method is called.
   void handleNoCommentReference() {}
+
+  /// A comment of the form `//@NNBD` or `//@NNBD*` has been encountered
+  /// at the beginning of the file indicating that this particular library
+  /// is in the process of being converted to non-nullable by default.
+  ///
+  /// If [isStar] is `true`, then the library should be considered
+  /// old style nullable but the `?` indicating that a type is nullable
+  /// should be ignored and no errors generated if they are present.
+  void handleNNBD(bool isStar) {}
 }

pkg/front_end/lib/src/fasta/parser/parser.dart

@@ -15,6 +15,7 @@ import '../../scanner/token.dart'
         ASSIGNMENT_PRECEDENCE,
         BeginToken,
         CASCADE_PRECEDENCE,
+        CommentToken,
         EQUALITY_PRECEDENCE,
         Keyword,
         POSTFIX_PRECEDENCE,
@@ -343,6 +344,11 @@ class Parser {
     int count = 0;
     DirectiveContext directiveState = new DirectiveContext();
     token = syntheticPreviousToken(token);
+    if (identical(token.next.type, TokenType.SCRIPT_TAG)) {
+      directiveState?.checkScriptTag(this, token.next);
+      token = parseScript(token);
+    }
+    parseNNBD(token);
     while (!token.next.isEof) {
       final Token start = token.next;
       token = parseTopLevelDeclarationImpl(token, directiveState);
@@ -369,6 +375,24 @@ class Parser {
     return token;
   }
 
+  /// Look for a comment of the form `//@NNBD` or `//@NNBD*`
+  /// indicating that this file is in process of being converted
+  /// to be non-nullable by default.
+  void parseNNBD(Token token) {
+    Token comment = token.next.precedingComments;
+    if (comment is CommentToken) {
+      String text = comment.lexeme;
+      if (text.startsWith('//')) {
+        text = text.substring(2).trim();
+        if (text == '@NNBD') {
+          listener.handleNNBD(false);
+        } else if (text == '@NNBD*') {
+          listener.handleNNBD(true);
+        }
+      }
+    }
+  }
+
   /// This method exists for analyzer compatibility only
   /// and will be removed once analyzer/fasta integration is complete.
   ///
@@ -459,10 +483,6 @@ class Parser {
   /// ```
   Token parseTopLevelDeclarationImpl(
       Token token, DirectiveContext directiveState) {
-    if (identical(token.next.type, TokenType.SCRIPT_TAG)) {
-      directiveState?.checkScriptTag(this, token.next);
-      return parseScript(token);
-    }
     token = parseMetadataStar(token);
     Token next = token.next;
     if (next.isTopLevelKeyword) {