diff --git a/README.md b/README.md index f763ed56..cb65e84f 100644 --- a/README.md +++ b/README.md @@ -155,8 +155,10 @@ very_good test -r Run tests in a Dart or Flutter project. Usage: very_good test [arguments] --h, --help Print this usage information. --r, --recursive Run tests recursively for all nested packages. +-h, --help Print this usage information. +-r, --recursive Run tests recursively for all nested packages. + --coverage Whether to collect coverage information. + --min-coverage Whether to enforce a minimum coverage percentage. Run "very_good help" to see global options. ``` diff --git a/lib/src/cli/cli.dart b/lib/src/cli/cli.dart index 12b6508f..1dff6083 100644 --- a/lib/src/cli/cli.dart +++ b/lib/src/cli/cli.dart @@ -1,5 +1,6 @@ import 'dart:async'; +import 'package:lcov_parser/lcov_parser.dart'; import 'package:mason/mason.dart'; import 'package:path/path.dart' as p; import 'package:universal_io/io.dart'; diff --git a/lib/src/cli/flutter_cli.dart b/lib/src/cli/flutter_cli.dart index edbf7ff8..ca715740 100644 --- a/lib/src/cli/flutter_cli.dart +++ b/lib/src/cli/flutter_cli.dart @@ -4,6 +4,49 @@ part of 'cli.dart'; /// is executed without a `pubspec.yaml`. class PubspecNotFound implements Exception {} +/// {@template coverage_not_met} +/// Thrown when `flutter test ---coverage --min-coverage` +/// does not meet the provided minimum coverage threshold. +/// {@endtemplate} +class MinCoverageNotMet implements Exception { + /// {@macro coverage_not_met} + const MinCoverageNotMet(this.coverage); + + /// The measured coverage percentage (total hits / total found * 100). + final double coverage; +} + +/// Thrown when `flutter test ---coverage --min-coverage value` +/// does not generate the coverage file within the timeout threshold. +class GenerateCoverageTimeout implements Exception { + @override + String toString() => 'Timed out waiting for coverage to be generated.'; +} + +class _CoverageMetrics { + const _CoverageMetrics._({this.totalHits = 0, this.totalFound = 0}); + + /// Generate coverage metrics from a list of lcov records. + factory _CoverageMetrics.fromLcovRecords(List records) { + return records.fold<_CoverageMetrics>( + const _CoverageMetrics._(), + (current, record) { + final found = record.lines?.found ?? 0; + final hit = record.lines?.hit ?? 0; + return _CoverageMetrics._( + totalFound: current.totalFound + found, + totalHits: current.totalHits + hit, + ); + }, + ); + } + + final int totalHits; + final int totalFound; + + double get percentage => totalFound < 1 ? 0 : (totalHits / totalFound * 100); +} + /// Flutter CLI class Flutter { /// Determine whether flutter is installed. @@ -62,15 +105,25 @@ class Flutter { static Future test({ String cwd = '.', bool recursive = false, + bool collectCoverage = false, + double? minCoverage, void Function(String)? stdout, void Function(String)? stderr, - }) { - return _runCommand( + }) async { + final lcovPath = p.join(cwd, 'coverage', 'lcov.info'); + final lcovFile = File(lcovPath); + + if (collectCoverage && lcovFile.existsSync()) { + await lcovFile.delete(); + } + + await _runCommand( cmd: (cwd) { void noop(String? _) {} stdout?.call('Running "flutter test" in $cwd...\n'); return _flutterTest( cwd: cwd, + collectCoverage: collectCoverage, stdout: stdout ?? noop, stderr: stderr ?? noop, ); @@ -78,6 +131,14 @@ class Flutter { cwd: cwd, recursive: recursive, ); + + if (collectCoverage) await lcovFile.ensureCreated(); + if (minCoverage != null) { + final records = await Parser.parse(lcovPath); + final coverageMetrics = _CoverageMetrics.fromLcovRecords(records); + final coverage = coverageMetrics.percentage; + if (coverage < minCoverage) throw MinCoverageNotMet(coverage); + } } } @@ -110,6 +171,7 @@ Future _runCommand({ Future _flutterTest({ String cwd = '.', + bool collectCoverage = false, required void Function(String) stdout, required void Function(String) stderr, }) { @@ -142,7 +204,13 @@ Future _flutterTest({ }, ); - flutterTest(workingDirectory: cwd, runInShell: true).listen( + flutterTest( + workingDirectory: cwd, + arguments: [ + if (collectCoverage) '--coverage', + ], + runInShell: true, + ).listen( (event) { if (event.shouldCancelTimer()) timerSubscription.cancel(); if (event is SuiteTestEvent) suites[event.suite.id] = event.suite; @@ -217,6 +285,20 @@ final int _lineLength = () { } }(); +extension on File { + Future ensureCreated({ + Duration timeout = const Duration(seconds: 1), + Duration interval = const Duration(milliseconds: 50), + }) async { + var elapsedTime = Duration.zero; + while (!existsSync()) { + await Future.delayed(interval); + elapsedTime += interval; + if (elapsedTime >= timeout) throw GenerateCoverageTimeout(); + } + } +} + extension on TestEvent { bool shouldCancelTimer() { final event = this; diff --git a/lib/src/commands/test.dart b/lib/src/commands/test.dart index 7fc545a4..0b4f24c8 100644 --- a/lib/src/commands/test.dart +++ b/lib/src/commands/test.dart @@ -12,12 +12,22 @@ import 'package:very_good_cli/src/cli/cli.dart'; class TestCommand extends Command { /// {@macro test_command} TestCommand({Logger? logger}) : _logger = logger ?? Logger() { - argParser.addFlag( - 'recursive', - abbr: 'r', - help: 'Run tests recursively for all nested packages.', - negatable: false, - ); + argParser + ..addFlag( + 'recursive', + abbr: 'r', + help: 'Run tests recursively for all nested packages.', + negatable: false, + ) + ..addFlag( + 'coverage', + help: 'Whether to collect coverage information.', + negatable: false, + ) + ..addOption( + 'min-coverage', + help: 'Whether to enforce a minimum coverage percentage.', + ); } final Logger _logger; @@ -43,6 +53,10 @@ class TestCommand extends Command { final recursive = _argResults['recursive'] as bool; final target = _argResults.rest.length == 1 ? _argResults.rest[0] : '.'; final targetPath = path.normalize(Directory(target).absolute.path); + final collectCoverage = _argResults['coverage'] as bool; + final minCoverage = double.tryParse( + _argResults['min-coverage'] as String? ?? '', + ); final isFlutterInstalled = await Flutter.installed(); if (isFlutterInstalled) { try { @@ -51,10 +65,17 @@ class TestCommand extends Command { recursive: recursive, stdout: _logger.write, stderr: _logger.err, + collectCoverage: collectCoverage, + minCoverage: minCoverage, ); } on PubspecNotFound catch (_) { _logger.err('Could not find a pubspec.yaml in $targetPath'); return ExitCode.noInput.code; + } on MinCoverageNotMet catch (e) { + _logger.err( + '''Expected coverage >= ${minCoverage!.toStringAsFixed(2)}% but actual is ${e.coverage.toStringAsFixed(2)}%.''', + ); + return ExitCode.unavailable.code; } catch (error) { _logger.err('$error'); return ExitCode.unavailable.code; diff --git a/pubspec.yaml b/pubspec.yaml index 46f2afd2..5dabcd43 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -8,6 +8,7 @@ environment: dependencies: args: ^2.1.0 + lcov_parser: ^0.1.2 mason: ">=0.1.0-dev.9 <0.1.0-dev.10" mason_logger: ^0.1.0-dev.6 meta: ^1.3.0 diff --git a/test/src/cli/flutter_cli_test.dart b/test/src/cli/flutter_cli_test.dart index 3129b92a..6dee072f 100644 --- a/test/src/cli/flutter_cli_test.dart +++ b/test/src/cli/flutter_cli_test.dart @@ -5,6 +5,33 @@ import 'package:test/test.dart'; import 'package:universal_io/io.dart'; import 'package:very_good_cli/src/cli/cli.dart'; +const calculatorContents = ''' +class Calculator { + int add(int x, int y) => x + y; + int subtract(int x, int y) => x - y; +}'''; + +const calculatorTestContents = ''' +import 'package:test/test.dart'; +import 'package:example/calculator.dart'; + +void main() { + test('...', () { + expect(Calculator().add(1, 2), equals(3)); + expect(Calculator().subtract(43, 1), equals(42)); + }); +}'''; + +const calculatorTestContentsMissingCoverage = ''' +import 'package:test/test.dart'; +import 'package:example/calculator.dart'; + +void main() { + test('...', () { + expect(Calculator().add(1, 2), equals(3)); + }); +}'''; + const testContents = ''' import 'package:test/test.dart'; @@ -191,6 +218,13 @@ void main() { logger = MockLogger(); }); + test('GenerateCoverageTimeout toString()', () { + expect( + GenerateCoverageTimeout().toString(), + equals('Timed out waiting for coverage to be generated.'), + ); + }); + test('throws when there is no pubspec.yaml', () { expectLater( Flutter.test(cwd: Directory.systemTemp.path), @@ -496,6 +530,234 @@ void main() { () => logger.write(any(that: contains('+1: All tests passed!'))), ).called(1); }); + + test('completes w/coverage', () async { + final directory = Directory.systemTemp.createTempSync(); + final libDirectory = Directory(p.join(directory.path, 'lib')) + ..createSync(); + final testDirectory = Directory(p.join(directory.path, 'test')) + ..createSync(); + File(p.join(directory.path, 'pubspec.yaml')).writeAsStringSync(pubspec); + File( + p.join(libDirectory.path, 'calculator.dart'), + ).writeAsStringSync(calculatorContents); + File( + p.join(testDirectory.path, 'calculator_test.dart'), + ).writeAsStringSync(calculatorTestContents); + await expectLater( + Flutter.test( + cwd: directory.path, + stdout: logger.write, + stderr: logger.err, + collectCoverage: true, + ), + completes, + ); + verify( + () => logger.write( + any(that: contains('Running "flutter test" in ${directory.path}')), + ), + ).called(1); + verify( + () => logger.write(any(that: contains('+1: All tests passed!'))), + ).called(1); + expect( + File(p.join(directory.path, 'coverage', 'lcov.info')).existsSync(), + isTrue, + ); + }); + + test('overwrites previous coverage file', () async { + final directory = Directory.systemTemp.createTempSync(); + final coverageDirectory = Directory(p.join(directory.path, 'coverage')) + ..createSync(); + final libDirectory = Directory(p.join(directory.path, 'lib')) + ..createSync(); + final testDirectory = Directory(p.join(directory.path, 'test')) + ..createSync(); + File(p.join(coverageDirectory.path, 'lcov.info')) + .writeAsStringSync('HI'); + File(p.join(directory.path, 'pubspec.yaml')).writeAsStringSync(pubspec); + File( + p.join(libDirectory.path, 'calculator.dart'), + ).writeAsStringSync(calculatorContents); + File( + p.join(testDirectory.path, 'calculator_test.dart'), + ).writeAsStringSync(calculatorTestContents); + await expectLater( + Flutter.test( + cwd: directory.path, + stdout: logger.write, + stderr: logger.err, + collectCoverage: true, + ), + completes, + ); + verify( + () => logger.write( + any(that: contains('Running "flutter test" in ${directory.path}')), + ), + ).called(1); + verify( + () => logger.write(any(that: contains('+1: All tests passed!'))), + ).called(1); + expect( + File( + p.join(directory.path, 'coverage', 'lcov.info'), + ).readAsStringSync(), + isNot(equals('HI')), + ); + }); + + test('completes w/coverage and --min-coverage 100', () async { + final directory = Directory.systemTemp.createTempSync(); + final libDirectory = Directory(p.join(directory.path, 'lib')) + ..createSync(); + final testDirectory = Directory(p.join(directory.path, 'test')) + ..createSync(); + File(p.join(directory.path, 'pubspec.yaml')).writeAsStringSync(pubspec); + File( + p.join(libDirectory.path, 'calculator.dart'), + ).writeAsStringSync(calculatorContents); + File( + p.join(testDirectory.path, 'calculator_test.dart'), + ).writeAsStringSync(calculatorTestContents); + await expectLater( + Flutter.test( + cwd: directory.path, + stdout: logger.write, + stderr: logger.err, + collectCoverage: true, + minCoverage: 100, + ), + completes, + ); + verify( + () => logger.write( + any(that: contains('Running "flutter test" in ${directory.path}')), + ), + ).called(1); + verify( + () => logger.write(any(that: contains('+1: All tests passed!'))), + ).called(1); + expect( + File(p.join(directory.path, 'coverage', 'lcov.info')).existsSync(), + isTrue, + ); + }); + + test('throws when --min-coverage 100 not met (50%)', () async { + final directory = Directory.systemTemp.createTempSync(); + final libDirectory = Directory(p.join(directory.path, 'lib')) + ..createSync(); + final testDirectory = Directory(p.join(directory.path, 'test')) + ..createSync(); + File(p.join(directory.path, 'pubspec.yaml')).writeAsStringSync(pubspec); + File( + p.join(libDirectory.path, 'calculator.dart'), + ).writeAsStringSync(calculatorContents); + File( + p.join(testDirectory.path, 'calculator_test.dart'), + ).writeAsStringSync(calculatorTestContentsMissingCoverage); + await expectLater( + Flutter.test( + cwd: directory.path, + stdout: logger.write, + stderr: logger.err, + collectCoverage: true, + minCoverage: 100, + ), + throwsA( + isA().having((e) => e.coverage, 'coverage', 50), + ), + ); + verify( + () => logger.write( + any(that: contains('Running "flutter test" in ${directory.path}')), + ), + ).called(1); + verify( + () => logger.write(any(that: contains('+1: All tests passed!'))), + ).called(1); + expect( + File(p.join(directory.path, 'coverage', 'lcov.info')).existsSync(), + isTrue, + ); + }); + + test('passes when --min-coverage 50 met (50%)', () async { + final directory = Directory.systemTemp.createTempSync(); + final libDirectory = Directory(p.join(directory.path, 'lib')) + ..createSync(); + final testDirectory = Directory(p.join(directory.path, 'test')) + ..createSync(); + File(p.join(directory.path, 'pubspec.yaml')).writeAsStringSync(pubspec); + File( + p.join(libDirectory.path, 'calculator.dart'), + ).writeAsStringSync(calculatorContents); + File( + p.join(testDirectory.path, 'calculator_test.dart'), + ).writeAsStringSync(calculatorTestContentsMissingCoverage); + await expectLater( + Flutter.test( + cwd: directory.path, + stdout: logger.write, + stderr: logger.err, + collectCoverage: true, + minCoverage: 50, + ), + completes, + ); + verify( + () => logger.write( + any(that: contains('Running "flutter test" in ${directory.path}')), + ), + ).called(1); + verify( + () => logger.write(any(that: contains('+1: All tests passed!'))), + ).called(1); + expect( + File(p.join(directory.path, 'coverage', 'lcov.info')).existsSync(), + isTrue, + ); + }); + + test('passes when --min-coverage 49 met (50%)', () async { + final directory = Directory.systemTemp.createTempSync(); + final libDirectory = Directory(p.join(directory.path, 'lib')) + ..createSync(); + final testDirectory = Directory(p.join(directory.path, 'test')) + ..createSync(); + File(p.join(directory.path, 'pubspec.yaml')).writeAsStringSync(pubspec); + File( + p.join(libDirectory.path, 'calculator.dart'), + ).writeAsStringSync(calculatorContents); + File( + p.join(testDirectory.path, 'calculator_test.dart'), + ).writeAsStringSync(calculatorTestContentsMissingCoverage); + await expectLater( + Flutter.test( + cwd: directory.path, + stdout: logger.write, + stderr: logger.err, + collectCoverage: true, + minCoverage: 49, + ), + completes, + ); + verify( + () => logger.write( + any(that: contains('Running "flutter test" in ${directory.path}')), + ), + ).called(1); + verify( + () => logger.write(any(that: contains('+1: All tests passed!'))), + ).called(1); + expect( + File(p.join(directory.path, 'coverage', 'lcov.info')).existsSync(), + isTrue, + ); + }); }); }); } diff --git a/test/src/commands/test_test.dart b/test/src/commands/test_test.dart index 7d0694a6..89d9a5e7 100644 --- a/test/src/commands/test_test.dart +++ b/test/src/commands/test_test.dart @@ -20,8 +20,10 @@ const expectedTestUsage = [ 'Run tests in a Dart or Flutter project.\n' '\n' 'Usage: very_good test [arguments]\n' - '-h, --help Print this usage information.\n' - '-r, --recursive Run tests recursively for all nested packages.\n' + '-h, --help Print this usage information.\n' + '-r, --recursive Run tests recursively for all nested packages.\n' + ' --coverage Whether to collect coverage information.\n' + ''' --min-coverage Whether to enforce a minimum coverage percentage.\n''' '\n' 'Run "very_good help" to see global options.', ]; @@ -130,6 +132,90 @@ void main() { }), ); + test( + 'completes normally --coverage', + withRunner((commandRunner, logger, printLogs) async { + final directory = Directory.systemTemp.createTempSync(); + final testDirectory = Directory(path.join(directory.path, 'test')) + ..createSync(); + File( + path.join(directory.path, 'pubspec.yaml'), + ).writeAsStringSync(pubspecContent()); + File( + path.join(testDirectory.path, 'example_test.dart'), + ).writeAsStringSync(testContent); + final result = await commandRunner.run( + ['test', '--coverage', directory.path], + ); + expect(result, equals(ExitCode.success.code)); + verify(() { + logger.write( + any(that: contains('Running "flutter test" in')), + ); + }).called(1); + verify(() { + logger.write(any(that: contains('All tests passed'))); + }).called(1); + }), + ); + + test( + 'completes normally --coverage --min-coverage 0', + withRunner((commandRunner, logger, printLogs) async { + final directory = Directory.systemTemp.createTempSync(); + final testDirectory = Directory(path.join(directory.path, 'test')) + ..createSync(); + File( + path.join(directory.path, 'pubspec.yaml'), + ).writeAsStringSync(pubspecContent()); + File( + path.join(testDirectory.path, 'example_test.dart'), + ).writeAsStringSync(testContent); + final result = await commandRunner.run( + ['test', '--coverage', '--min-coverage', '0', directory.path], + ); + expect(result, equals(ExitCode.success.code)); + verify(() { + logger.write( + any(that: contains('Running "flutter test" in')), + ); + }).called(1); + verify(() { + logger.write(any(that: contains('All tests passed'))); + }).called(1); + }), + ); + + test( + 'fails when coverage not met --coverage --min-coverage 100', + withRunner((commandRunner, logger, printLogs) async { + final directory = Directory.systemTemp.createTempSync(); + final testDirectory = Directory(path.join(directory.path, 'test')) + ..createSync(); + File( + path.join(directory.path, 'pubspec.yaml'), + ).writeAsStringSync(pubspecContent()); + File( + path.join(testDirectory.path, 'example_test.dart'), + ).writeAsStringSync(testContent); + final result = await commandRunner.run( + ['test', '--coverage', '--min-coverage', '100', directory.path], + ); + expect(result, equals(ExitCode.unavailable.code)); + verify(() { + logger.write( + any(that: contains('Running "flutter test" in')), + ); + }).called(1); + verify(() { + logger.write(any(that: contains('All tests passed'))); + }).called(1); + verify( + () => logger.err('Expected coverage >= 100.00% but actual is 0.00%.'), + ).called(1); + }), + ); + test( 'completes normally ' 'when pubspec.yaml and tests exist (recursive)',