[integration_test] Show stack trace of widget test errors on the platform side (#2967)

* [integration_test] Show stack trace of widget test errors on the platform side

We keep things simple for now, because this might change once we have a proper test reporter.

Author: jiahaog

packages/integration_test/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.8.1
+
+* Show stack trace of widget test errors on the platform side
+
 ## 0.8.0
 
 * Rename plugin to integration_test.

packages/integration_test/android/src/main/java/dev/flutter/plugins/integration_test/FlutterTestRunner.java

@@ -77,7 +77,7 @@ public void run(RunNotifier notifier) {
       Description d = Description.createTestDescription(testClass, name);
       notifier.fireTestStarted(d);
       String outcome = results.get(name);
-      if (outcome.equals("failed")) {
+      if (!outcome.equals("success")) {
         Exception dummyException = new Exception(outcome);
         notifier.fireTestFailure(new Failure(d, dummyException));
       }

packages/integration_test/ios/Classes/IntegrationTestIosTest.m

@@ -26,7 +26,7 @@ - (BOOL)testIntegrationTest:(NSString **)testResult {
       NSLog(@"%@ passed.", test);
       [passedTests addObject:test];
     } else {
-      NSLog(@"%@ failed.", test);
+      NSLog(@"%@ failed: %@", test, result);
       [failedTests addObject:test];
     }
   }

packages/integration_test/lib/common.dart

@@ -78,7 +78,7 @@ class Response {
     }
 
     _failureDetails.forEach((Failure f) {
-      list.add(f.toString());
+      list.add(f.toJson());
     });
 
     return list;
@@ -107,14 +107,16 @@ class Failure {
   Failure(this.methodName, this.details);
 
   /// Serializes the object to JSON.
-  @override
-  String toString() {
+  String toJson() {
     return json.encode(<String, String>{
       'methodName': methodName,
       'details': details,
     });
   }
 
+  @override
+  String toString() => toJson();
+
   /// Decode a JSON string to create a Failure object.
   static Failure fromJsonString(String jsonString) {
     Map<String, dynamic> failure = json.decode(jsonString);

packages/integration_test/lib/integration_test.dart

@@ -13,6 +13,8 @@ import 'package:flutter/widgets.dart';
 import 'common.dart';
 import '_extension_io.dart' if (dart.library.html) '_extension_web.dart';
 
+const String _success = 'success';
+
 /// A subclass of [LiveTestWidgetsFlutterBinding] that reports tests results
 /// on a channel to adapt them to native instrumentation test format.
 class IntegrationTestWidgetsFlutterBinding
@@ -33,7 +35,14 @@ class IntegrationTestWidgetsFlutterBinding
         }
         await _channel.invokeMethod<void>(
           'allTestsFinished',
-          <String, dynamic>{'results': results},
+          <String, dynamic>{
+            'results': results.map((name, result) {
+              if (result is Failure) {
+                return MapEntry(name, result.details);
+              }
+              return MapEntry(name, result);
+            })
+          },
         );
       } on MissingPluginException {
         print('Warning: integration_test test plugin was not detected.');
@@ -46,8 +55,7 @@ class IntegrationTestWidgetsFlutterBinding
     final TestExceptionReporter oldTestExceptionReporter = reportTestException;
     reportTestException =
         (FlutterErrorDetails details, String testDescription) {
-      results[testDescription] = 'failed';
-      _failureMethodsDetails.add(Failure(testDescription, details.toString()));
+      results[testDescription] = Failure(testDescription, details.toString());
       if (!_allTestsPassed.isCompleted) {
         _allTestsPassed.complete(false);
       }
@@ -95,17 +103,11 @@ class IntegrationTestWidgetsFlutterBinding
 
   final Completer<bool> _allTestsPassed = Completer<bool>();
 
-  /// Stores failure details.
-  ///
-  /// Failed test method's names used as key.
-  final List<Failure> _failureMethodsDetails = List<Failure>();
-
   /// Similar to [WidgetsFlutterBinding.ensureInitialized].
   ///
   /// Returns an instance of the [IntegrationTestWidgetsFlutterBinding], creating and
   /// initializing it if necessary.
   static WidgetsBinding ensureInitialized() {
-    print('TESTING123 ensuring init');
     if (WidgetsBinding.instance == null) {
       IntegrationTestWidgetsFlutterBinding();
     }
@@ -118,10 +120,12 @@ class IntegrationTestWidgetsFlutterBinding
 
   /// Test results that will be populated after the tests have completed.
   ///
-  /// Keys are the test descriptions, and values are either `success` or
-  /// `failed`.
+  /// Keys are the test descriptions, and values are either [_success] or
+  /// a [Failure].
   @visibleForTesting
-  Map<String, String> results = <String, String>{};
+  Map<String, Object> results = <String, Object>{};
+
+  List<Failure> get _failures => results.values.whereType<Failure>().toList();
 
   /// The extra data for the reported result.
   ///
@@ -143,7 +147,7 @@ class IntegrationTestWidgetsFlutterBinding
           'message': allTestsPassed
               ? Response.allTestsPassed(data: reportData).toJson()
               : Response.someTestsFailed(
-                  _failureMethodsDetails,
+                  _failures,
                   data: reportData,
                 ).toJson(),
         };
@@ -185,6 +189,6 @@ class IntegrationTestWidgetsFlutterBinding
       description: description,
       timeout: timeout,
     );
-    results[description] ??= 'success';
+    results[description] ??= _success;
   }
 }

packages/integration_test/pubspec.yaml

@@ -1,6 +1,6 @@
 name: integration_test
 description: Runs tests that use the flutter_test API as integration tests.
-version: 0.8.0
+version: 0.8.1
 homepage: https://github.com/flutter/plugins/tree/master/packages/integration_test
 
 environment:

packages/integration_test/test/binding_fail_test.dart

@@ -8,6 +8,7 @@ import 'package:flutter_test/flutter_test.dart';
 const String _flutterBin = 'flutter';
 const String _integrationResultsPrefix =
     'IntegrationTestWidgetsFlutterBinding test results:';
+const String _failureExcerpt = 'Expected: <false>\\n  Actual: <true>';
 
 void main() async {
   group('Integration binding result', () {
@@ -27,24 +28,20 @@ void main() async {
       final Map<String, dynamic> results =
           await _runTest('test/data/fail_test_script.dart');
 
+      expect(results, hasLength(2));
       expect(
-          results,
-          equals({
-            'failing test 1': 'failed',
-            'failing test 2': 'failed',
-          }));
+          results, containsPair('failing test 1', contains(_failureExcerpt)));
+      expect(
+          results, containsPair('failing test 2', contains(_failureExcerpt)));
     });
 
     test('when one test passes, then another fails', () async {
       final Map<String, dynamic> results =
           await _runTest('test/data/pass_then_fail_test_script.dart');
 
-      expect(
-          results,
-          equals({
-            'passing test': 'success',
-            'failing test': 'failed',
-          }));
+      expect(results, hasLength(2));
+      expect(results, containsPair('passing test', equals('success')));
+      expect(results, containsPair('failing test', contains(_failureExcerpt)));
     });
   });
 }