From 6f3bc24124d06c3bb0d078c8443db6a6b6334708 Mon Sep 17 00:00:00 2001 From: FMorschel <52160996+FMorschel@users.noreply.github.com> Date: Fri, 26 Sep 2025 10:01:02 -0300 Subject: [PATCH 1/2] solo skip shouldn't be reported by failures-only or compact reporters --- .../test/runner/compact_reporter_test.dart | 82 +++++++++++++++++++ .../runner/failures_only_reporter_test.dart | 51 ++++++++++++ pkgs/test_api/lib/src/backend/declarer.dart | 1 + pkgs/test_api/lib/src/backend/metadata.dart | 23 +++++- .../lib/src/runner/reporter/compact.dart | 3 +- .../src/runner/reporter/failures_only.dart | 1 + 6 files changed, 159 insertions(+), 2 deletions(-) diff --git a/pkgs/test/test/runner/compact_reporter_test.dart b/pkgs/test/test/runner/compact_reporter_test.dart index 842b65531..7ad63f376 100644 --- a/pkgs/test/test/runner/compact_reporter_test.dart +++ b/pkgs/test/test/runner/compact_reporter_test.dart @@ -467,6 +467,88 @@ void main() { }); }); + group('solo:', () { + test('displays non-solo skipped tests separately', () { + return _expectReport( + ''' + test('skip 1', () {}); + test('skip 2', () {}); + test('solo 3', () {}, solo: true);''', + ''' + +0: loading test.dart + +0: skip 1 + +0 ~1: skip 1 + +0 ~1: skip 2 + +0 ~2: skip 2 + +0 ~2: solo 3 + +1 ~2: solo 3 + +1 ~2: All tests passed!''', + ); + }); + + test('displays a non-solo skipped group', () { + return _expectReport( + ''' + group('skip', () { + test('test 1', () {}); + test('test 2', () {}); + test('test 3', () {}); + }); + group('solo', () { + test('test 4', () {}); + test('test 5', () {}); + test('test 6', () {}); + }, solo: true);''', + ''' + +0: loading test.dart + +0: skip + +0 ~1: skip + +0 ~1: solo test 4 + +1 ~1: solo test 4 + +1 ~1: solo test 5 + +2 ~1: solo test 5 + +2 ~1: solo test 6 + +3 ~1: solo test 6 + +3 ~1: All tests passed!''', + ); + }); + + test('runs solo tests along with successful and failing tests', () { + return _expectReport( + ''' + test('failure 1', () => throw TestFailure('oh no'), solo: true); + test('skip 1', () {}); + test('success 1', () {}, solo: true); + test('failure 2', () => throw TestFailure('oh no'), solo: true); + test('skip 2', () {}); + test('success 2', () {}, solo: true);''', + ''' + +0: loading test.dart + +0: failure 1 + +0 -1: failure 1 [E] + oh no + test.dart 6:35 main. + + + +0 -1: skip 1 + +0 ~1 -1: skip 1 + +0 ~1 -1: success 1 + +1 ~1 -1: success 1 + +1 ~1 -1: failure 2 + +1 ~1 -2: failure 2 [E] + oh no + test.dart 9:35 main. + + + +1 ~1 -2: skip 2 + +1 ~2 -2: skip 2 + +1 ~2 -2: success 2 + +2 ~2 -2: success 2 + +2 ~2 -2: Some tests failed.''', + ); + }); + }); + test('Directs users to enable stack trace chaining if disabled', () async { await _expectReport( '''test('failure 1', () => throw TestFailure('oh no'));''', diff --git a/pkgs/test/test/runner/failures_only_reporter_test.dart b/pkgs/test/test/runner/failures_only_reporter_test.dart index 2b56af47d..94aeb9e99 100644 --- a/pkgs/test/test/runner/failures_only_reporter_test.dart +++ b/pkgs/test/test/runner/failures_only_reporter_test.dart @@ -230,6 +230,57 @@ void main() { }); }); + group('solo:', () { + test('displays non-solo skipped tests separately', () { + return _expectReport( + ''' + test('skip 1', () {}); + test('skip 2', () {}); + test('solo 3', () {}, solo: true);''', + '+1 ~2: All tests passed!', + ); + }); + + test('displays a non-solo skipped group', () { + return _expectReport( + ''' + group('skip', () { + test('test 1', () {}); + test('test 2', () {}); + test('test 3', () {}); + }); + group('solo', () { + test('test 4', () {}); + test('test 5', () {}); + test('test 6', () {}); + }, solo: true);''', + '+3 ~1: All tests passed!', + ); + }); + + test('runs solo tests along with successful and failing tests', () { + return _expectReport( + ''' + test('failure 1', () => throw TestFailure('oh no'), solo: true); + test('skip 1', () {}); + test('success 1', () {}, solo: true); + test('failure 2', () => throw TestFailure('oh no'), solo: true); + test('skip 2', () {}); + test('success 2', () {}, solo: true);''', + ''' + +0 -1: failure 1 [E] + oh no + test.dart 6:35 main. + + +1 ~1 -2: failure 2 [E] + oh no + test.dart 9:35 main. + + +2 ~2 -2: Some tests failed.''', + ); + }); + }); + test('Directs users to enable stack trace chaining if disabled', () async { await _expectReport( '''test('failure 1', () => throw TestFailure('oh no'));''', diff --git a/pkgs/test_api/lib/src/backend/declarer.dart b/pkgs/test_api/lib/src/backend/declarer.dart index 7f9e4df3b..cbaa8c0cb 100644 --- a/pkgs/test_api/lib/src/backend/declarer.dart +++ b/pkgs/test_api/lib/src/backend/declarer.dart @@ -378,6 +378,7 @@ class Declarer { entry.name, entry.metadata.change( skip: true, + soloSkip: true, skipReason: 'does not have "solo"', ), () {}, diff --git a/pkgs/test_api/lib/src/backend/metadata.dart b/pkgs/test_api/lib/src/backend/metadata.dart index bc3d38a74..88c70b92a 100644 --- a/pkgs/test_api/lib/src/backend/metadata.dart +++ b/pkgs/test_api/lib/src/backend/metadata.dart @@ -33,6 +33,9 @@ final class Metadata { bool get skip => _skip ?? false; final bool? _skip; + bool get soloSkip => _soloSkip; + final bool _soloSkip; + /// The reason the test or suite should be skipped, if given. final String? skipReason; @@ -163,6 +166,7 @@ final class Metadata { PlatformSelector? testOn, Timeout? timeout, bool? skip, + bool? soloSkip, bool? verboseTrace, bool? chainStackTraces, int? retry, @@ -177,6 +181,7 @@ final class Metadata { testOn: testOn, timeout: timeout, skip: skip, + soloSkip: soloSkip, verboseTrace: verboseTrace, chainStackTraces: chainStackTraces, retry: retry, @@ -213,6 +218,7 @@ final class Metadata { PlatformSelector? testOn, Timeout? timeout, bool? skip, + bool? soloSkip, this.skipReason, bool? verboseTrace, bool? chainStackTraces, @@ -221,9 +227,14 @@ final class Metadata { Map? onPlatform, Map? forTag, this.languageVersionComment, - }) : testOn = testOn ?? PlatformSelector.all, + }) : assert( + soloSkip == null || skip == null || soloSkip == skip, + 'soloSkip and skip cannot be contradictory.', + ), + testOn = testOn ?? PlatformSelector.all, timeout = timeout ?? const Timeout.factor(1), _skip = skip, + _soloSkip = soloSkip ?? false, _verboseTrace = verboseTrace, _chainStackTraces = chainStackTraces, _retry = retry, @@ -255,6 +266,7 @@ final class Metadata { : PlatformSelector.parse(testOn), timeout = timeout ?? const Timeout.factor(1), _skip = skip == null ? null : skip != false, + _soloSkip = false, _verboseTrace = verboseTrace, _chainStackTraces = chainStackTraces, _retry = retry, @@ -279,6 +291,7 @@ final class Metadata { : PlatformSelector.parse(serialized['testOn'] as String), timeout = _deserializeTimeout(serialized['timeout']), _skip = serialized['skip'] as bool?, + _soloSkip = serialized['soloSkip'] as bool? ?? false, skipReason = serialized['skipReason'] as String?, _verboseTrace = serialized['verboseTrace'] as bool?, _chainStackTraces = serialized['chainStackTraces'] as bool?, @@ -370,6 +383,7 @@ final class Metadata { PlatformSelector? testOn, Timeout? timeout, bool? skip, + bool? soloSkip, bool? verboseTrace, bool? chainStackTraces, int? retry, @@ -379,9 +393,14 @@ final class Metadata { Map? forTag, String? languageVersionComment, }) { + assert( + soloSkip == null || skip == null || soloSkip == skip, + 'soloSkip and skip cannot be contradictory.', + ); testOn ??= this.testOn; timeout ??= this.timeout; skip ??= _skip; + soloSkip ??= _soloSkip; verboseTrace ??= _verboseTrace; chainStackTraces ??= _chainStackTraces; retry ??= _retry; @@ -394,6 +413,7 @@ final class Metadata { testOn: testOn, timeout: timeout, skip: skip, + soloSkip: soloSkip, verboseTrace: verboseTrace, chainStackTraces: chainStackTraces, skipReason: skipReason, @@ -431,6 +451,7 @@ final class Metadata { 'testOn': testOn == PlatformSelector.all ? null : testOn.toString(), 'timeout': _serializeTimeout(timeout), 'skip': _skip, + 'soloSkip': _soloSkip, 'skipReason': skipReason, 'verboseTrace': _verboseTrace, 'chainStackTraces': _chainStackTraces, diff --git a/pkgs/test_core/lib/src/runner/reporter/compact.dart b/pkgs/test_core/lib/src/runner/reporter/compact.dart index 3095ede2b..101694d1a 100644 --- a/pkgs/test_core/lib/src/runner/reporter/compact.dart +++ b/pkgs/test_core/lib/src/runner/reporter/compact.dart @@ -11,8 +11,8 @@ import 'package:test_api/src/backend/message.dart'; // ignore: implementation_im import 'package:test_api/src/backend/state.dart'; // ignore: implementation_imports import '../../util/io.dart'; -import '../../util/pretty_print.dart'; import '../../util/pretty_print.dart' as utils; +import '../../util/pretty_print.dart'; import '../engine.dart'; import '../load_exception.dart'; import '../load_suite.dart'; @@ -228,6 +228,7 @@ class CompactReporter implements Reporter { _subscriptions.add( liveTest.onMessage.listen((message) { + if (liveTest.test.metadata.soloSkip) return; _progressLine(_description(liveTest), truncate: false); if (!_printedNewline) _sink.writeln(''); _printedNewline = true; diff --git a/pkgs/test_core/lib/src/runner/reporter/failures_only.dart b/pkgs/test_core/lib/src/runner/reporter/failures_only.dart index 1549da52b..b4e9f9aa6 100644 --- a/pkgs/test_core/lib/src/runner/reporter/failures_only.dart +++ b/pkgs/test_core/lib/src/runner/reporter/failures_only.dart @@ -161,6 +161,7 @@ class FailuresOnlyReporter implements Reporter { _subscriptions.add( liveTest.onMessage.listen((message) { + if (liveTest.test.metadata.soloSkip) return; // TODO - Should this suppress output? Behave like printOnFailure? _progressLine(_description(liveTest)); var text = message.text; From 2f1b665a03d32401cc4025d3bb486e8bc06f3777 Mon Sep 17 00:00:00 2001 From: FMorschel <52160996+FMorschel@users.noreply.github.com> Date: Tue, 7 Oct 2025 14:12:03 -0300 Subject: [PATCH 2/2] Remove soloSkip metadata and update reporters to use skip flag --- pkgs/test_api/lib/src/backend/declarer.dart | 1 - pkgs/test_api/lib/src/backend/metadata.dart | 23 +------------------ .../lib/src/runner/reporter/compact.dart | 2 +- .../src/runner/reporter/failures_only.dart | 2 +- 4 files changed, 3 insertions(+), 25 deletions(-) diff --git a/pkgs/test_api/lib/src/backend/declarer.dart b/pkgs/test_api/lib/src/backend/declarer.dart index cbaa8c0cb..7f9e4df3b 100644 --- a/pkgs/test_api/lib/src/backend/declarer.dart +++ b/pkgs/test_api/lib/src/backend/declarer.dart @@ -378,7 +378,6 @@ class Declarer { entry.name, entry.metadata.change( skip: true, - soloSkip: true, skipReason: 'does not have "solo"', ), () {}, diff --git a/pkgs/test_api/lib/src/backend/metadata.dart b/pkgs/test_api/lib/src/backend/metadata.dart index 88c70b92a..bc3d38a74 100644 --- a/pkgs/test_api/lib/src/backend/metadata.dart +++ b/pkgs/test_api/lib/src/backend/metadata.dart @@ -33,9 +33,6 @@ final class Metadata { bool get skip => _skip ?? false; final bool? _skip; - bool get soloSkip => _soloSkip; - final bool _soloSkip; - /// The reason the test or suite should be skipped, if given. final String? skipReason; @@ -166,7 +163,6 @@ final class Metadata { PlatformSelector? testOn, Timeout? timeout, bool? skip, - bool? soloSkip, bool? verboseTrace, bool? chainStackTraces, int? retry, @@ -181,7 +177,6 @@ final class Metadata { testOn: testOn, timeout: timeout, skip: skip, - soloSkip: soloSkip, verboseTrace: verboseTrace, chainStackTraces: chainStackTraces, retry: retry, @@ -218,7 +213,6 @@ final class Metadata { PlatformSelector? testOn, Timeout? timeout, bool? skip, - bool? soloSkip, this.skipReason, bool? verboseTrace, bool? chainStackTraces, @@ -227,14 +221,9 @@ final class Metadata { Map? onPlatform, Map? forTag, this.languageVersionComment, - }) : assert( - soloSkip == null || skip == null || soloSkip == skip, - 'soloSkip and skip cannot be contradictory.', - ), - testOn = testOn ?? PlatformSelector.all, + }) : testOn = testOn ?? PlatformSelector.all, timeout = timeout ?? const Timeout.factor(1), _skip = skip, - _soloSkip = soloSkip ?? false, _verboseTrace = verboseTrace, _chainStackTraces = chainStackTraces, _retry = retry, @@ -266,7 +255,6 @@ final class Metadata { : PlatformSelector.parse(testOn), timeout = timeout ?? const Timeout.factor(1), _skip = skip == null ? null : skip != false, - _soloSkip = false, _verboseTrace = verboseTrace, _chainStackTraces = chainStackTraces, _retry = retry, @@ -291,7 +279,6 @@ final class Metadata { : PlatformSelector.parse(serialized['testOn'] as String), timeout = _deserializeTimeout(serialized['timeout']), _skip = serialized['skip'] as bool?, - _soloSkip = serialized['soloSkip'] as bool? ?? false, skipReason = serialized['skipReason'] as String?, _verboseTrace = serialized['verboseTrace'] as bool?, _chainStackTraces = serialized['chainStackTraces'] as bool?, @@ -383,7 +370,6 @@ final class Metadata { PlatformSelector? testOn, Timeout? timeout, bool? skip, - bool? soloSkip, bool? verboseTrace, bool? chainStackTraces, int? retry, @@ -393,14 +379,9 @@ final class Metadata { Map? forTag, String? languageVersionComment, }) { - assert( - soloSkip == null || skip == null || soloSkip == skip, - 'soloSkip and skip cannot be contradictory.', - ); testOn ??= this.testOn; timeout ??= this.timeout; skip ??= _skip; - soloSkip ??= _soloSkip; verboseTrace ??= _verboseTrace; chainStackTraces ??= _chainStackTraces; retry ??= _retry; @@ -413,7 +394,6 @@ final class Metadata { testOn: testOn, timeout: timeout, skip: skip, - soloSkip: soloSkip, verboseTrace: verboseTrace, chainStackTraces: chainStackTraces, skipReason: skipReason, @@ -451,7 +431,6 @@ final class Metadata { 'testOn': testOn == PlatformSelector.all ? null : testOn.toString(), 'timeout': _serializeTimeout(timeout), 'skip': _skip, - 'soloSkip': _soloSkip, 'skipReason': skipReason, 'verboseTrace': _verboseTrace, 'chainStackTraces': _chainStackTraces, diff --git a/pkgs/test_core/lib/src/runner/reporter/compact.dart b/pkgs/test_core/lib/src/runner/reporter/compact.dart index 101694d1a..087a10f2e 100644 --- a/pkgs/test_core/lib/src/runner/reporter/compact.dart +++ b/pkgs/test_core/lib/src/runner/reporter/compact.dart @@ -228,7 +228,7 @@ class CompactReporter implements Reporter { _subscriptions.add( liveTest.onMessage.listen((message) { - if (liveTest.test.metadata.soloSkip) return; + if (liveTest.test.metadata.skip) return; _progressLine(_description(liveTest), truncate: false); if (!_printedNewline) _sink.writeln(''); _printedNewline = true; diff --git a/pkgs/test_core/lib/src/runner/reporter/failures_only.dart b/pkgs/test_core/lib/src/runner/reporter/failures_only.dart index b4e9f9aa6..dc383bc20 100644 --- a/pkgs/test_core/lib/src/runner/reporter/failures_only.dart +++ b/pkgs/test_core/lib/src/runner/reporter/failures_only.dart @@ -161,7 +161,7 @@ class FailuresOnlyReporter implements Reporter { _subscriptions.add( liveTest.onMessage.listen((message) { - if (liveTest.test.metadata.soloSkip) return; + if (liveTest.test.metadata.skip) return; // TODO - Should this suppress output? Behave like printOnFailure? _progressLine(_description(liveTest)); var text = message.text;