Lay the groundwork for comparing language_2 compilation results between analyzer and CFE.

This CL just adds the compiler=compareAnalyzerCfe option to the test
framework, and updates the status files so that no language_2 failures
are expected when this "compiler" is used.  A placeholder executable
has been added to sdk/bin, but it is not wired up yet--that will be
done in a future CL.

In a future CL I will wire up the placeholder executable to the code
in pkg/analyzer_fe_comparison, and fill in the methods in
CompareAnalyzerCfeCommandOutput to process the output accordingly.

Change-Id: I27439de4ac609cb8b003f157a0ff88d181bfbcae
Reviewed-on: https://dart-review.googlesource.com/73561
Commit-Queue: Paul Berry <paulberry@google.com>
Reviewed-by: Bob Nystrom <rnystrom@google.com>

Author: stereotype441

pkg/analyzer_fe_comparison/bin/compare_sdk_tests

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# Copyright (c) 2018, the Dart project authors.  Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+# TODO(paulberry)
+echo "Placeholder text"

pkg/smith/lib/configuration.dart

@@ -584,6 +584,7 @@ class Compiler extends NamedEnum {
   static const precompiler = const Compiler._('precompiler');
   static const dart2js = const Compiler._('dart2js');
   static const dart2analyzer = const Compiler._('dart2analyzer');
+  static const compareAnalyzerCfe = const Compiler._('compare_analyzer_cfe');
   static const dartdevc = const Compiler._('dartdevc');
   static const dartdevk = const Compiler._('dartdevk');
   static const appJit = const Compiler._('app_jit');
@@ -601,6 +602,7 @@ class Compiler extends NamedEnum {
     precompiler,
     dart2js,
     dart2analyzer,
+    compareAnalyzerCfe,
     dartdevc,
     dartdevk,
     appJit,
@@ -653,6 +655,7 @@ class Compiler extends NamedEnum {
         ];
 
       case Compiler.dart2analyzer:
+      case Compiler.compareAnalyzerCfe:
         return const [Runtime.none];
       case Compiler.appJit:
       case Compiler.appJitk:
@@ -683,6 +686,7 @@ class Compiler extends NamedEnum {
       case Compiler.dartdevk:
         return Runtime.chrome;
       case Compiler.dart2analyzer:
+      case Compiler.compareAnalyzerCfe:
         return Runtime.none;
       case Compiler.appJit:
       case Compiler.appJitk:
@@ -705,6 +709,7 @@ class Compiler extends NamedEnum {
   Mode get defaultMode {
     switch (this) {
       case Compiler.dart2analyzer:
+      case Compiler.compareAnalyzerCfe:
       case Compiler.dart2js:
       case Compiler.dartdevc:
       case Compiler.dartdevk:

tests/language_2/language_2.status

@@ -2,15 +2,16 @@
 # for details. All rights reserved. Use of this source code is governed by a
 # BSD-style license that can be found in the LICENSE file.
 
+[ $compiler == app_jit ]
+deferred_inheritance_constraints_test/redirecting_constructor: Crash
+
+[ $compiler != compare_analyzer_cfe ]
 bug34235_test/01: MissingCompileTimeError # Issue 34235
 mixin_constructor_forwarding/const_constructor_test/none: CompileTimeError # Issue 32223
 mixin_constructor_forwarding/const_constructor_with_field_test/none: CompileTimeError # Issue 32223
 mixin_constructor_forwarding/optional_named_parameters_test/none: CompileTimeError # Issue 31543
 mixin_constructor_forwarding/optional_positional_parameters_test/none: CompileTimeError # Issue 31543
 
-[ $compiler == app_jit ]
-deferred_inheritance_constraints_test/redirecting_constructor: Crash
-
 [ $compiler == dart2analyzer ]
 double_literals/*: Skip # https://github.com/dart-lang/sdk/issues/34360
 
@@ -58,9 +59,16 @@ partial_instantiation_static_bounds_check_test/03: MissingCompileTimeError # Iss
 [ $compiler != app_jitk && $compiler != dartk && $compiler != dartkb && $compiler != dartkp && $mode == debug && $runtime == vm ]
 built_in_identifier_type_annotation_test/set: Crash # Not supported by legacy VM front-end.
 
-[ $compiler != dart2analyzer && $compiler != dart2js && $compiler != dartdevc && !$fasta && $strong ]
+[ $compiler != compare_analyzer_cfe && $compiler != dart2analyzer && $compiler != dart2js && $compiler != dartdevc && !$fasta && $strong ]
 type_promotion_functions_test: CompileTimeError # Issue 30895: This test requires a complete rewrite for 2.0.
 
+[ $compiler != compare_analyzer_cfe && $compiler != dart2js && !$fasta && $strong ]
+compile_time_constant_static5_test/11: CompileTimeError # Issue 30546
+compile_time_constant_static5_test/16: CompileTimeError # Issue 30546
+compile_time_constant_static5_test/21: CompileTimeError # Issue 30546
+compile_time_constant_static5_test/23: CompileTimeError # Issue 30546
+type_promotion_more_specific_test/04: CompileTimeError # Issue 30906.
+
 # Detection of compile-time errors that are related to constants can't be fully
 # done at the front end, because constants are evaluated at back ends.  So, some
 # errors aren't detected by fasta, but reported by back ends as compile-time
@@ -82,13 +90,6 @@ implicit_creation/implicit_const_not_default_values_test/e9: MissingCompileTimeE
 [ $compiler != dart2js && $compiler != dartdevc && !$checked ]
 function_type/*: Skip # Needs checked mode.
 
-[ $compiler != dart2js && !$fasta && $strong ]
-compile_time_constant_static5_test/11: CompileTimeError # Issue 30546
-compile_time_constant_static5_test/16: CompileTimeError # Issue 30546
-compile_time_constant_static5_test/21: CompileTimeError # Issue 30546
-compile_time_constant_static5_test/23: CompileTimeError # Issue 30546
-type_promotion_more_specific_test/04: CompileTimeError # Issue 30906.
-
 [ $compiler != dart2js && !$fasta && !$strong ]
 implicit_creation/implicit_new_constructor_generic_test: Fail # No support for implicit creation.
 implicit_creation/implicit_new_constructor_test: Fail # No support for implicit creation.

tools/testing/dart/command.dart

@@ -51,6 +51,12 @@ class Command {
     return new AnalysisCommand._(executable, arguments, environmentOverrides);
   }
 
+  static Command compareAnalyzerCfe(String executable, List<String> arguments,
+      Map<String, String> environmentOverrides) {
+    return new CompareAnalyzerCfeCommand._(
+        executable, arguments, environmentOverrides);
+  }
+
   static Command specParse(String executable, List<String> arguments,
       Map<String, String> environmentOverrides) {
     return new SpecParseCommand._(executable, arguments, environmentOverrides);
@@ -429,6 +435,13 @@ class AnalysisCommand extends ProcessCommand {
       : super._('dart2analyzer', executable, arguments, environmentOverrides);
 }
 
+class CompareAnalyzerCfeCommand extends ProcessCommand {
+  CompareAnalyzerCfeCommand._(String executable, List<String> arguments,
+      Map<String, String> environmentOverrides)
+      : super._('compare_analyzer_cfe', executable, arguments,
+            environmentOverrides);
+}
+
 class SpecParseCommand extends ProcessCommand {
   SpecParseCommand._(String executable, List<String> arguments,
       Map<String, String> environmentOverrides)

tools/testing/dart/command_output.dart

@@ -566,6 +566,41 @@ class AnalysisCommandOutput extends CommandOutput {
   }
 }
 
+class CompareAnalyzerCfeCommandOutput extends CommandOutput {
+  CompareAnalyzerCfeCommandOutput(
+      Command command,
+      int exitCode,
+      bool timedOut,
+      List<int> stdout,
+      List<int> stderr,
+      Duration time,
+      bool compilationSkipped)
+      : super(command, exitCode, timedOut, stdout, stderr, time,
+            compilationSkipped, 0);
+
+  Expectation result(TestCase testCase) {
+    // Handle crashes and timeouts first
+    if (hasCrashed) return Expectation.crash;
+    if (hasTimedOut) return Expectation.timeout;
+    if (hasNonUtf8) return Expectation.nonUtf8Error;
+
+    // TODO(paulberry): parse command output
+    return Expectation.pass;
+  }
+
+  /// Cloned code from member result(), with changes.
+  /// Delete existing result() function and rename, when status files are gone.
+  Expectation realResult(TestCase testCase) {
+    // Handle crashes and timeouts first
+    if (hasCrashed) return Expectation.crash;
+    if (hasTimedOut) return Expectation.timeout;
+    if (hasNonUtf8) return Expectation.nonUtf8Error;
+
+    // TODO(paulberry): parse command output
+    return Expectation.pass;
+  }
+}
+
 class SpecParseCommandOutput extends CommandOutput {
   SpecParseCommandOutput(
       Command command,
@@ -984,6 +1019,9 @@ CommandOutput createCommandOutput(Command command, int exitCode, bool timedOut,
   if (command is AnalysisCommand) {
     return new AnalysisCommandOutput(
         command, exitCode, timedOut, stdout, stderr, time, compilationSkipped);
+  } else if (command is CompareAnalyzerCfeCommand) {
+    return new CompareAnalyzerCfeCommandOutput(
+        command, exitCode, timedOut, stdout, stderr, time, compilationSkipped);
   } else if (command is SpecParseCommand) {
     return new SpecParseCommandOutput(
         command, exitCode, timedOut, stdout, stderr, time, compilationSkipped);

tools/testing/dart/compiler_configuration.dart

@@ -51,6 +51,9 @@ abstract class CompilerConfiguration {
       case Compiler.dart2analyzer:
         return new AnalyzerCompilerConfiguration(configuration);
 
+      case Compiler.compareAnalyzerCfe:
+        return new CompareAnalyzerCfeCompilerConfiguration(configuration);
+
       case Compiler.dart2js:
         return new Dart2jsCompilerConfiguration(configuration);
 
@@ -962,6 +965,46 @@ class AnalyzerCompilerConfiguration extends CompilerConfiguration {
   }
 }
 
+/// Configuration for compareAnalyzerCfe.
+class CompareAnalyzerCfeCompilerConfiguration extends CompilerConfiguration {
+  CompareAnalyzerCfeCompilerConfiguration(TestConfiguration configuration)
+      : super._subclass(configuration);
+
+  int get timeoutMultiplier => 4;
+
+  String computeCompilerPath() {
+    String suffix = executableScriptSuffix;
+    if (_useSdk) {
+      throw "--use-sdk cannot be used with compiler compare_analyzer_cfe";
+    }
+    return 'pkg/analyzer_fe_comparison/bin/compare_sdk_tests$suffix';
+  }
+
+  CommandArtifact computeCompilationArtifact(String tempDir,
+      List<String> arguments, Map<String, String> environmentOverrides) {
+    arguments = arguments.toList();
+    if (!previewDart2) {
+      throw new ArgumentError('--no-preview-dart-2 not supported');
+    }
+
+    // Since this is not a real compilation, no artifacts are produced.
+    return new CommandArtifact([
+      Command.compareAnalyzerCfe(
+          computeCompilerPath(), arguments, environmentOverrides)
+    ], null, null);
+  }
+
+  List<String> computeRuntimeArguments(
+      RuntimeConfiguration runtimeConfiguration,
+      TestInformation info,
+      List<String> vmOptions,
+      List<String> sharedOptions,
+      List<String> originalArguments,
+      CommandArtifact artifact) {
+    return <String>[];
+  }
+}
+
 /// Configuration for spec_parser.
 class SpecParserCompilerConfiguration extends CompilerConfiguration {
   SpecParserCompilerConfiguration(TestConfiguration configuration)

tools/testing/dart/options.dart

@@ -96,6 +96,7 @@ none:          Do not compile the Dart code.
 precompiler:   Compile into AOT snapshot before running the test.
 dart2js:       Compile to JavaScript using dart2js.
 dart2analyzer: Perform static analysis on Dart code using the analyzer.
+compareAnalyzerCfe: Compare analyzer and common front end representations.
 app_jit:       Compile the Dart code into an app snapshot.
 app_jitk:      Compile the Dart code into Kernel and then into an app snapshot.
 dartk:         Compile the Dart code into Kernel before running test.

tools/testing/dart/status_reporter.dart

@@ -13,6 +13,12 @@ final _combinations = {
       'archs': ['x64'],
       'compiler': 'dart2analyzer'
     },
+    {
+      'runtimes': ['none'],
+      'modes': ['release'],
+      'archs': ['x64'],
+      'compiler': 'compare_analyzer_cfe'
+    },
     {
       'runtimes': ['vm'],
       'modes': ['debug', 'release'],