feat(test): list failing tests (#397)

lib/src/cli/cli.dart

@@ -5,6 +5,7 @@ import 'package:lcov_parser/lcov_parser.dart';
 import 'package:mason/mason.dart';
 import 'package:path/path.dart' as p;
 import 'package:pubspec_parse/pubspec_parse.dart';
+import 'package:stack_trace/stack_trace.dart';
 import 'package:universal_io/io.dart';
 import 'package:very_good_cli/src/commands/test/templates/test_runner_bundle.dart';
 import 'package:very_good_test_runner/very_good_test_runner.dart';

lib/src/cli/flutter_cli.dart

@@ -277,14 +277,14 @@ Future<int> _flutterTest({
  final suites = <int, TestSuite>{};
  final groups = <int, TestGroup>{};
  final tests = <int, Test>{};
+ final failedTestErrorMessages = <int, String>{};
 
  var successCount = 0;
  var skipCount = 0;
- var failureCount = 0;
 
  String computeStats() {
  final passingTests = successCount.formatSuccess();
- final failingTests = failureCount.formatFailure();
+ final failingTests = failedTestErrorMessages.length.formatFailure();
  final skippedTests = skipCount.formatSkipped();
  final result = [passingTests, failingTests, skippedTests]
  ..removeWhere((element) => element.isEmpty);
@@ -327,9 +327,20 @@ Future<int> _flutterTest({
 
  if (event is ErrorTestEvent) {
  stderr('$clearLine${event.error}');
+
  if (event.stackTrace.trim().isNotEmpty) {
  stderr('$clearLine${event.stackTrace}');
  }
+
+ final traceLocation = _getTraceLocation(stackTrace: event.stackTrace);
+
+ // When failing to recover the location from the stack trace,
+ // save a short description of the error
+ final testErrorDescription = traceLocation ??
+ event.error.replaceAll('\n', ' ').truncated(_lineLength);
+
+ final prefix = event.isFailure ? '[FAILED]' : '[ERROR]';
+ failedTestErrorMessages[event.testID] = '$prefix $testErrorDescription';
  }
 
  if (event is TestDoneEvent) {
@@ -347,7 +358,6 @@ Future<int> _flutterTest({
  successCount++;
  } else {
  stderr('$clearLine${test.name} ${suite.path} (FAILED)');
- failureCount++;
  }
 
  final timeElapsed = Duration(milliseconds: event.time).formatted();
@@ -366,6 +376,21 @@ Future<int> _flutterTest({
  : lightRed.wrap('Some tests failed.')!;
 
  stdout('$clearLine${darkGray.wrap(timeElapsed)} $stats: $summary\n');
+
+ if (event.success != true) {
+ assert(
+ failedTestErrorMessages.isNotEmpty,
+ 'Invalid state: test event report as failed but no failed tests '
+ 'were gathered',
+ );
+ final title = styleBold.wrap('Failing Tests:');
+
+ final lines = StringBuffer('$clearLine$title\n');
+ for (final errorMessage in failedTestErrorMessages.values) {
+ lines.writeln('$clearLine - $errorMessage');
+ }
+ stderr(lines.toString());
+ }
  }
 
  if (event is ExitTestEvent) {
@@ -436,3 +461,22 @@ extension on String {
  return '...$truncated';
  }
 }
+
+String? _getTraceLocation({
+ required String stackTrace,
+}) {
+ final trace = Trace.parse(stackTrace);
+ if (trace.frames.isEmpty) {
+ return null;
+ }
+
+ final lastFrame = trace.frames.last;
+
+ final library = lastFrame.library;
+ final line = lastFrame.line;
+ final column = lastFrame.column;
+
+ if (line == null) return library;
+ if (column == null) return '$library:$line';
+ return '$library:$line:$column';
+}

pubspec.yaml

@@ -16,6 +16,7 @@ dependencies:
  path: ^1.8.0
  pub_updater: ^0.2.1
  pubspec_parse: ^1.2.0
+ stack_trace: 1.10.0
  universal_io: ^2.0.4
  usage: ^4.0.2
  very_good_analysis: ^2.4.0

test/src/cli/flutter_cli_test.dart

@@ -122,6 +122,32 @@ void main() {
  });
 }''';
 
+const registerExceptionNoStackTraceContents = '''
+import 'package:test/test.dart';
+import 'package:stack_trace/stack_trace.dart' as stack_trace;
+void main() {
+ test('example', () {
+ print('EXCEPTION');
+ registerException(
+ 'fake error',
+ stack_trace.Chain(<stack_trace.Trace>[]),
+ );
+ });
+}''';
+
+String registerExceptionCustomStackTraceContents(String stackTrace) => '''
+import 'package:test/test.dart';
+import 'package:stack_trace/stack_trace.dart' as stack_trace;
+void main() {
+ test('example', () {
+ print('EXCEPTION');
+ registerException(
+ 'fake error',
+ stack_trace.Trace.parse('$stackTrace'),
+ );
+ });
+}''';
+
 const skippedTestContents = '''
 import 'package:test/test.dart';
 void main() {
@@ -381,6 +407,11 @@ void main() {
  verify(
  () => logger.write(any(that: contains('Some tests failed.'))),
  ).called(1);
+ verify(
+ () => logger.err(
+ any(that: contains('- [FAILED] test/example_test.dart:4:5')),
+ ),
+ ).called(1);
  });
 
  test('completes when there is a test directory (skipping)', () async {
@@ -534,8 +565,94 @@ void main() {
  verify(
  () => logger.write(any(that: contains('-1: Some tests failed.'))),
  ).called(1);
+ verify(
+ () => logger
+ .err(any(that: contains('- [ERROR] test/example_test.dart:5:5'))),
+ ).called(1);
  });
 
+ test(
+ 'completes when there is a test directory (exception w/o trace)',
+ () async {
+ final directory = Directory.systemTemp.createTempSync();
+ final testDirectory = Directory(p.join(directory.path, 'test'))
+ ..createSync();
+ File(p.join(directory.path, 'pubspec.yaml'))
+ .writeAsStringSync(pubspec);
+ File(
+ p.join(testDirectory.path, 'example_test.dart'),
+ ).writeAsStringSync(registerExceptionNoStackTraceContents);
+ await expectLater(
+ Flutter.test(
+ cwd: directory.path,
+ stdout: logger.write,
+ stderr: logger.err,
+ ),
+ completion(equals([ExitCode.unavailable.code])),
+ );
+ verify(
+ () => logger.write(
+ any(
+ that: contains(
+ 'Running "flutter test" in ${p.dirname(directory.path)}',
+ ),
+ ),
+ ),
+ ).called(1);
+ verify(() => logger.err(any(that: contains('EXCEPTION')))).called(1);
+ verify(
+ () => logger.write(any(that: contains('-1: Some tests failed.'))),
+ ).called(1);
+ verify(
+ () => logger.err(any(that: contains('- [ERROR] fake error'))),
+ ).called(1);
+ },
+ );
+
+ test(
+ 'completes when there is a test directory (exception w/ custom trace)',
+ () async {
+ final directory = Directory.systemTemp.createTempSync();
+ final testDirectory = Directory(p.join(directory.path, 'test'))
+ ..createSync();
+ File(p.join(directory.path, 'pubspec.yaml'))
+ .writeAsStringSync(pubspec);
+ File(
+ p.join(testDirectory.path, 'example_test.dart'),
+ ).writeAsStringSync(
+ registerExceptionCustomStackTraceContents(
+ 'test/example_test.dart 4 main',
+ ),
+ );
+ await expectLater(
+ Flutter.test(
+ cwd: directory.path,
+ stdout: logger.write,
+ stderr: logger.err,
+ ),
+ completion(equals([ExitCode.unavailable.code])),
+ );
+ verify(
+ () => logger.write(
+ any(
+ that: contains(
+ 'Running "flutter test" in ${p.dirname(directory.path)}',
+ ),
+ ),
+ ),
+ ).called(1);
+ verify(() => logger.err(any(that: contains('EXCEPTION')))).called(1);
+ verify(
+ () => logger.write(any(that: contains('-1: Some tests failed.'))),
+ ).called(1);
+ verify(
+ () => logger.err(
+ any(that: contains('- [ERROR] test/example_test.dart:4')),
+ ),
+ ).called(1);
+ },
+ );
+
  test('completes and truncates really long test name', () async {
  final directory = Directory.systemTemp.createTempSync();
  final testDirectory = Directory(p.join(directory.path, 'test'))