Skip to content

Commit 76241c5

Browse files
epdenoudensebastianbergmann
epdenouden
authored and
sebastianbergmann
committed
Breakfast: rerun defects first
1 parent ac9e3e7 commit 76241c5

40 files changed

+1570
-95
lines changed

‎.gitignore

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,4 @@
1717
/tests/TextUI/*.out
1818
/tests/TextUI/*.php
1919
/vendor
20-
20+
/.phpunit.result.cache

‎phpunit.xml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
<phpunit xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
33
xsi:noNamespaceSchemaLocation="phpunit.xsd"
44
bootstrap="tests/bootstrap.php"
5+
cacheResult="true"
56
verbose="true">
67
<testsuites>
78
<testsuite name="small">

‎phpunit.xsd

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,8 +167,13 @@
167167
<xs:simpleType name="executionOrderType">
168168
<xs:restriction base="xs:string">
169169
<xs:enumeration value="default"/>
170-
<xs:enumeration value="reverse"/>
170+
<xs:enumeration value="defects"/>
171+
<xs:enumeration value="depends"/>
172+
<xs:enumeration value="depends,defects"/>
171173
<xs:enumeration value="random"/>
174+
<xs:enumeration value="reverse"/>
175+
<xs:enumeration value="depends,random"/>
176+
<xs:enumeration value="depends,reverse"/>
172177
</xs:restriction>
173178
</xs:simpleType>
174179
<xs:complexType name="fileFilterType">
@@ -216,6 +221,8 @@
216221
<xs:attribute name="backupGlobals" type="xs:boolean" default="false"/>
217222
<xs:attribute name="backupStaticAttributes" type="xs:boolean" default="false"/>
218223
<xs:attribute name="bootstrap" type="xs:anyURI"/>
224+
<xs:attribute name="cacheResult" type="xs:boolean"/>
225+
<xs:attribute name="cacheResultFile" type="xs:anyURI"/>
219226
<xs:attribute name="cacheTokens" type="xs:boolean"/>
220227
<xs:attribute name="colors" type="xs:boolean" default="false"/>
221228
<xs:attribute name="columns" type="columnsType" default="80"/>
@@ -228,6 +235,7 @@
228235
<xs:attribute name="printerClass" type="xs:string" default="PHPUnit\TextUI\ResultPrinter"/>
229236
<xs:attribute name="printerFile" type="xs:anyURI"/>
230237
<xs:attribute name="processIsolation" type="xs:boolean" default="false"/>
238+
<xs:attribute name="stopOnDefect" type="xs:boolean" default="false"/>
231239
<xs:attribute name="stopOnError" type="xs:boolean" default="false"/>
232240
<xs:attribute name="stopOnFailure" type="xs:boolean" default="false"/>
233241
<xs:attribute name="stopOnWarning" type="xs:boolean" default="false"/>

‎src/Framework/TestResult.php

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,11 @@ class TestResult implements Countable
174174
*/
175175
protected $stopOnSkipped = false;
176176

177+
/**
178+
* @var bool
179+
*/
180+
protected $stopOnDefect = false;
181+
177182
/**
178183
* @var bool
179184
*/
@@ -229,7 +234,7 @@ public function addError(Test $test, Throwable $t, float $time): void
229234
$test->markAsRisky();
230235
}
231236

232-
if ($this->stopOnRisky) {
237+
if ($this->stopOnRisky || $this->stopOnDefect) {
233238
$this->stop();
234239
}
235240
} elseif ($t instanceof IncompleteTest) {
@@ -274,7 +279,7 @@ public function addError(Test $test, Throwable $t, float $time): void
274279
*/
275280
public function addWarning(Test $test, Warning $e, float $time): void
276281
{
277-
if ($this->stopOnWarning) {
282+
if ($this->stopOnWarning || $this->stopOnDefect) {
278283
$this->stop();
279284
}
280285

@@ -301,7 +306,7 @@ public function addFailure(Test $test, AssertionFailedError $e, float $time): vo
301306
$test->markAsRisky();
302307
}
303308

304-
if ($this->stopOnRisky) {
309+
if ($this->stopOnRisky || $this->stopOnDefect) {
305310
$this->stop();
306311
}
307312
} elseif ($e instanceof IncompleteTest) {
@@ -322,7 +327,7 @@ public function addFailure(Test $test, AssertionFailedError $e, float $time): vo
322327
$this->failures[] = new TestFailure($test, $e);
323328
$notifyMethod = 'addFailure';
324329

325-
if ($this->stopOnFailure) {
330+
if ($this->stopOnFailure || $this->stopOnDefect) {
326331
$this->stop();
327332
}
328333
}
@@ -1011,6 +1016,14 @@ public function stopOnSkipped(bool $flag): void
10111016
$this->stopOnSkipped = $flag;
10121017
}
10131018

1019+
/**
1020+
* Enables or disables the stopping for defects: error, failure, warning
1021+
*/
1022+
public function stopOnDefect(bool $flag): void
1023+
{
1024+
$this->stopOnDefect = $flag;
1025+
}
1026+
10141027
/**
10151028
* Returns the time spent running the tests.
10161029
*/

‎src/Runner/ResultCacheExtension.php

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
<?php
2+
/*
3+
* This file is part of PHPUnit.
4+
*
5+
* (c) Sebastian Bergmann <sebastian@phpunit.de>
6+
*
7+
* For the full copyright and license information, please view the LICENSE
8+
* file that was distributed with this source code.
9+
*/
10+
namespace PHPUnit\Runner;
11+
12+
final class ResultCacheExtension implements AfterSuccessfulTestHook, AfterSkippedTestHook, AfterRiskyTestHook, AfterIncompleteTestHook, AfterTestErrorHook, AfterTestWarningHook, AfterTestFailureHook, AfterLastTestHook
13+
{
14+
/**
15+
* @var TestResultCacheInterface
16+
*/
17+
private $cache;
18+
19+
public function __construct(TestResultCache $cache)
20+
{
21+
$this->cache = $cache;
22+
}
23+
24+
public function flush(): void
25+
{
26+
$this->cache->persist();
27+
}
28+
29+
public function executeAfterSuccessfulTest(string $test, float $time): void
30+
{
31+
$testName = $this->getTestName($test);
32+
$this->cache->setTime($testName, \round($time, 3));
33+
}
34+
35+
public function executeAfterIncompleteTest(string $test, string $message, float $time): void
36+
{
37+
$testName = $this->getTestName($test);
38+
$this->cache->setTime($testName, \round($time, 3));
39+
$this->cache->setState($testName, BaseTestRunner::STATUS_INCOMPLETE);
40+
}
41+
42+
public function executeAfterRiskyTest(string $test, string $message, float $time): void
43+
{
44+
$testName = $this->getTestName($test);
45+
$this->cache->setTime($testName, \round($time, 3));
46+
$this->cache->setState($testName, BaseTestRunner::STATUS_RISKY);
47+
}
48+
49+
public function executeAfterSkippedTest(string $test, string $message, float $time): void
50+
{
51+
$testName = $this->getTestName($test);
52+
$this->cache->setTime($testName, \round($time, 3));
53+
$this->cache->setState($testName, BaseTestRunner::STATUS_SKIPPED);
54+
}
55+
56+
public function executeAfterTestError(string $test, string $message, float $time): void
57+
{
58+
$testName = $this->getTestName($test);
59+
$this->cache->setTime($testName, \round($time, 3));
60+
$this->cache->setState($testName, BaseTestRunner::STATUS_ERROR);
61+
}
62+
63+
public function executeAfterTestFailure(string $test, string $message, float $time): void
64+
{
65+
$testName = $this->getTestName($test);
66+
$this->cache->setTime($testName, \round($time, 3));
67+
$this->cache->setState($testName, BaseTestRunner::STATUS_FAILURE);
68+
}
69+
70+
public function executeAfterTestWarning(string $test, string $message, float $time): void
71+
{
72+
$testName = $this->getTestName($test);
73+
$this->cache->setTime($testName, \round($time, 3));
74+
$this->cache->setState($testName, BaseTestRunner::STATUS_WARNING);
75+
}
76+
77+
public function executeAfterLastTest(): void
78+
{
79+
$this->flush();
80+
}
81+
82+
/**
83+
* @param string $test A long description format of the current test
84+
*
85+
* @return string The test name without TestSuiteClassName:: and @dataprovider details
86+
*/
87+
private function getTestName(string $test): string
88+
{
89+
$matches = [];
90+
91+
if (\preg_match('/^(?:\S+::)?(?<name>\S+)(?:(?<data> with data set (?:#\d+|"[^"]+"))\s\()?/', $test, $matches)) {
92+
$test = $matches['name'];
93+
94+
if (isset($matches['data'])) {
95+
$test .= $matches['data'];
96+
}
97+
}
98+
99+
return $test;
100+
}
101+
}

‎src/Runner/TestSuiteSorter.php

Lines changed: 99 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,27 +31,69 @@ final class TestSuiteSorter
3131
*/
3232
public const ORDER_REVERSED = 2;
3333

34+
/**
35+
* @var int
36+
*/
37+
public const ORDER_DEFECTS_FIRST = 3;
38+
39+
/**
40+
* List of sorting weights for all test result codes. A higher number gives higher priority.
41+
*/
42+
private const DEFECT_SORT_WEIGHT = [
43+
BaseTestRunner::STATUS_ERROR => 6,
44+
BaseTestRunner::STATUS_FAILURE => 5,
45+
BaseTestRunner::STATUS_WARNING => 4,
46+
BaseTestRunner::STATUS_INCOMPLETE => 3,
47+
BaseTestRunner::STATUS_RISKY => 2,
48+
BaseTestRunner::STATUS_SKIPPED => 1,
49+
BaseTestRunner::STATUS_UNKNOWN => 0
50+
];
51+
52+
/**
53+
* @var array<string, int> Associative array of (string => DEFECT_SORT_WEIGHT) elements
54+
*/
55+
private $defectSortOrder = [];
56+
57+
/**
58+
* @var TestResultCacheInterface
59+
*/
60+
private $cache;
61+
62+
public function __construct(?TestResultCacheInterface $cache = null)
63+
{
64+
$this->cache = $cache ?? new NullTestResultCache;
65+
}
66+
3467
/**
3568
* @throws Exception
3669
*/
37-
public function reorderTestsInSuite(Test $suite, int $order, bool $resolveDependencies): void
70+
public function reorderTestsInSuite(Test $suite, int $order, bool $resolveDependencies, int $orderDefects): void
3871
{
3972
if ($order !== self::ORDER_DEFAULT && $order !== self::ORDER_REVERSED && $order !== self::ORDER_RANDOMIZED) {
4073
throw new Exception(
4174
'$order must be one of TestSuiteSorter::ORDER_DEFAULT, TestSuiteSorter::ORDER_REVERSED, or TestSuiteSorter::ORDER_RANDOMIZED'
4275
);
4376
}
4477

45-
if ($suite instanceof TestSuite && !empty($suite->tests())) {
78+
if ($orderDefects !== self::ORDER_DEFAULT && $orderDefects !== self::ORDER_DEFECTS_FIRST) {
79+
throw new Exception(
80+
'$orderDefects must be one of TestSuiteSorter::ORDER_DEFAULT, TestSuiteSorter::ORDER_DEFECTS_FIRST'
81+
);
82+
}
83+
84+
if ($suite instanceof TestSuite) {
4685
foreach ($suite as $_suite) {
47-
$this->reorderTestsInSuite($_suite, $order, $resolveDependencies);
86+
$this->reorderTestsInSuite($_suite, $order, $resolveDependencies, $orderDefects);
4887
}
4988

50-
$this->sort($suite, $order, $resolveDependencies);
89+
if ($orderDefects === self::ORDER_DEFECTS_FIRST) {
90+
$this->addSuiteToDefectSortOrder($suite);
91+
}
92+
$this->sort($suite, $order, $resolveDependencies, $orderDefects);
5193
}
5294
}
5395

54-
private function sort(TestSuite $suite, int $order, bool $resolveDependencies): void
96+
private function sort(TestSuite $suite, int $order, bool $resolveDependencies, int $orderDefects): void
5597
{
5698
if (empty($suite->tests())) {
5799
return;
@@ -63,11 +105,29 @@ private function sort(TestSuite $suite, int $order, bool $resolveDependencies):
63105
$suite->setTests($this->randomize($suite->tests()));
64106
}
65107

108+
if ($orderDefects === self::ORDER_DEFECTS_FIRST && $this->cache !== null) {
109+
$suite->setTests($this->sortDefectsFirst($suite->tests()));
110+
}
111+
66112
if ($resolveDependencies && !($suite instanceof DataProviderTestSuite) && $this->suiteOnlyContainsTests($suite)) {
67113
$suite->setTests($this->resolveDependencies($suite->tests()));
68114
}
69115
}
70116

117+
private function addSuiteToDefectSortOrder(TestSuite $suite): void
118+
{
119+
$max = 0;
120+
121+
foreach ($suite->tests() as $test) {
122+
if (!isset($this->defectSortOrder[$test->getName()])) {
123+
$this->defectSortOrder[$test->getName()] = self::DEFECT_SORT_WEIGHT[$this->cache->getState($test->getName())];
124+
$max = \max($max, $this->defectSortOrder[$test->getName()]);
125+
}
126+
}
127+
128+
$this->defectSortOrder[$suite->getName()] = $max;
129+
}
130+
71131
private function suiteOnlyContainsTests(TestSuite $suite): bool
72132
{
73133
return \array_reduce($suite->tests(), function ($carry, $test) {
@@ -87,6 +147,40 @@ private function randomize(array $tests): array
87147
return $tests;
88148
}
89149

150+
private function sortDefectsFirst(array $tests): array
151+
{
152+
\usort($tests, function ($left, $right) {
153+
return $this->cmpDefectPriorityAndTime($left, $right);
154+
});
155+
156+
return $tests;
157+
}
158+
159+
/**
160+
* Comparator callback function to sort tests for "reach failure as fast as possible":
161+
* 1. sort tests by defect weight defined in self::DEFECT_SORT_WEIGHT
162+
* 2. when tests are equally defective, sort the fastest to the front
163+
* 3. do not reorder successful tests
164+
*/
165+
private function cmpDefectPriorityAndTime(Test $a, Test $b): int
166+
{
167+
$priorityA = $this->defectSortOrder[$a->getName()] ?? 0;
168+
$priorityB = $this->defectSortOrder[$b->getName()] ?? 0;
169+
170+
if ($priorityB <=> $priorityA) {
171+
// Sort defect weight descending
172+
return $priorityB <=> $priorityA;
173+
}
174+
175+
if ($priorityA || $priorityB) {
176+
// Sort test duration ascending
177+
return $this->cache->getTime($a->getName()) <=> $this->cache->getTime($b->getName());
178+
}
179+
180+
// do not change execution order
181+
return 0;
182+
}
183+
90184
/**
91185
* Reorder Tests within a TestCase in such a way as to resolve as many dependencies as possible.
92186
* The algorithm will leave the tests in original running order when it can.

0 commit comments

Comments
 (0)