Implement filtering tests based on XML input file

.psalm/baseline.xml

@@ -759,6 +759,16 @@
       <code>$this-&gt;filter</code>
     </PossiblyNullArgument>
   </file>
+  <file src="src/Runner/Filter/XmlTestsIterator.php">
+    <MissingThrowsDocblock occurrences="1">
+      <code>getName</code>
+    </MissingThrowsDocblock>
+  </file>
+  <file src="src/Runner/Hook/TestListenerAdapter.php">
+    <DeprecatedInterface occurrences="1">
+      <code>TestListenerAdapter</code>
+    </DeprecatedInterface>
+  </file>
   <file src="src/Runner/PhptTestCase.php">
     <InternalClass occurrences="2">
       <code>RawCodeCoverageData::fromXdebugWithoutPathCoverage([])</code>
@@ -889,7 +899,8 @@
       <code>$e</code>
     </InvalidArgument>
     <InvalidStringClass occurrences="1"/>
-    <MissingThrowsDocblock occurrences="11">
+    <MissingThrowsDocblock occurrences="12">
+      <code>addFilter</code>
       <code>addFilter</code>
       <code>addFilter</code>
       <code>addFilter</code>

src/Runner/Filter/XmlTestsIterator.php

@@ -0,0 +1,146 @@
+<?php declare(strict_types=1);
+/*
+ * This file is part of PHPUnit.
+ *
+ * (c) Sebastian Bergmann <sebastian@phpunit.de>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+namespace PHPUnit\Runner\Filter;
+
+use DOMElement;
+use DOMXPath;
+use Exception;
+use Generator;
+use PHPUnit\Framework\TestCase;
+use PHPUnit\Framework\TestSuite;
+use PHPUnit\Runner\PhptTestCase;
+use PHPUnit\Util\Xml\Loader as XmlLoader;
+use RecursiveFilterIterator;
+use RecursiveIterator;
+
+/**
+ * @internal This class is not covered by the backward compatibility promise for PHPUnit
+ */
+final class XmlTestsIterator extends RecursiveFilterIterator
+{
+    /**
+     * The filter is used as a fast look for
+     * - the class name
+     * - the method name plus its optional data set description.
+     *
+     * Example: `filter[class name][method name + data set] = true;`
+     *
+     * The accept() method then can use a fast isset() to check if a test should
+     * be included or not.
+     *
+     * This works equally for phpt tests, except we hardcode the class name.
+     *
+     * @var array<string,array<string,true>>
+     */
+    private array $filter;
+
+    /**
+     * @throws \PHPUnit\Util\Xml\Exception
+     *
+     * @return array<string,array<string,true>>
+     */
+    public static function createFilterFromXmlFile(string $xmlFile): array
+    {
+        $xml    = (new XmlLoader())->loadFile($xmlFile);
+        $xpath  = new DOMXPath($xml);
+        $filter = [];
+
+        foreach (self::extractTestCases($xpath) as [$className, $methodName, $dataSet]) {
+            if (!isset($filter[$className])) {
+                $filter[$className] = [];
+            }
+
+            if (!$dataSet) {
+                $filter[$className][$methodName] = true;
+
+                continue;
+            }
+
+            $name                      = "{$methodName} with data set {$dataSet}";
+            $filter[$className][$name] = true;
+        }
+
+        foreach (self::extractPhptFile($xpath) as $path) {
+            $filter[PhptTestCase::class][$path] = true;
+        }
+
+        return $filter;
+    }
+
+    /**
+     * @throws Exception
+     */
+    public function __construct(RecursiveIterator $iterator, array $filter)
+    {
+        parent::__construct($iterator);
+
+        $this->filter = $filter;
+    }
+
+    public function accept(): bool
+    {
+        $test = $this->getInnerIterator()->current();
+
+        if ($test instanceof TestSuite) {
+            return true;
+        }
+
+        /** @var TestCase $test */
+        $testClass = get_class($test);
+
+        if (!isset($this->filter[$testClass])) {
+            return false;
+        }
+
+        $name = $test->getName();
+
+        return isset($this->filter[$testClass][$name]);
+    }
+
+    private static function extractTestCases(DOMXPath $xpath): Generator
+    {
+        /** @var DOMElement $class */
+        foreach ($xpath->evaluate('/tests/testCaseClass') as $class) {
+            $className = $class->getAttribute('name');
+
+            if (!$className) {
+                continue;
+            }
+
+            /** @var DOMElement $method */
+            foreach ($xpath->evaluate('testCaseMethod', $class) as $method) {
+                $methodName = $method->getAttribute('name');
+
+                if (!$methodName) {
+                    continue;
+                }
+
+                $dataSet = $method->getAttribute('dataSet');
+
+                yield [$className, $methodName, $dataSet];
+            }
+        }
+    }
+
+    /**
+     * @return Generator<string>
+     */
+    private static function extractPhptFile(DOMXPath $xpath): Generator
+    {
+        /* @var DOMElement $phptFile */
+        foreach ($xpath->evaluate('/tests/phptFile') as $phptFile) {
+            $path = $phptFile->getAttribute('path');
+
+            if ($path) {
+                yield $path;
+            }
+        }
+    }
+}

src/TextUI/CliArguments/Builder.php

@@ -116,6 +116,7 @@ final class Builder
         'testsuite=',
         'verbose',
         'version',
+        'tests-xml=',
     ];
 
     private const SHORT_OPTIONS = 'd:c:hv';
@@ -225,6 +226,7 @@ public function fromParameters(array $parameters, array $additionalLongOptions):
         $useDefaultConfiguration                    = null;
         $verbose                                    = null;
         $version                                    = null;
+        $testsXml                                   = null;
 
         if (isset($options[1][0])) {
             $argument = $options[1][0];
@@ -752,6 +754,11 @@ public function fromParameters(array $parameters, array $additionalLongOptions):
 
                     break;
 
+                case '--tests-xml':
+                    $testsXml = $option[1];
+
+                    break;
+
                 default:
                     $unrecognizedOptions[str_replace('--', '', $option[0])] = $option[1];
             }
@@ -862,7 +869,8 @@ public function fromParameters(array $parameters, array $additionalLongOptions):
             $useDefaultConfiguration,
             $verbose,
             $version,
-            $coverageFilter
+            $coverageFilter,
+            $testsXml
         );
     }
 }

src/TextUI/CliArguments/Configuration.php

@@ -199,10 +199,12 @@ final class Configuration
 
     private ?bool $version = null;
 
+    private ?string $testsXml;
+
     /**
      * @param null|int|string $columns
      */
-    public function __construct(?string $argument, ?string $atLeastVersion, ?bool $backupGlobals, ?bool $backupStaticAttributes, ?bool $beStrictAboutChangesToGlobalState, ?bool $beStrictAboutResourceUsageDuringSmallTests, ?string $bootstrap, ?bool $cacheResult, ?string $cacheResultFile, ?bool $checkVersion, ?string $colors, $columns, ?string $configuration, ?string $coverageClover, ?string $coverageCobertura, ?string $coverageCrap4J, ?string $coverageHtml, ?string $coveragePhp, ?string $coverageText, ?bool $coverageTextShowUncoveredFiles, ?bool $coverageTextShowOnlySummary, ?string $coverageXml, ?bool $pathCoverage, ?string $coverageCacheDirectory, ?bool $warmCoverageCache, ?bool $debug, ?int $defaultTimeLimit, ?bool $disableCodeCoverageIgnore, ?bool $disallowTestOutput, ?bool $disallowTodoAnnotatedTests, ?bool $enforceTimeLimit, ?array $excludeGroups, ?int $executionOrder, ?int $executionOrderDefects, ?array $extensions, ?array $unavailableExtensions, ?bool $failOnEmptyTestSuite, ?bool $failOnIncomplete, ?bool $failOnRisky, ?bool $failOnSkipped, ?bool $failOnWarning, ?string $filter, ?bool $generateConfiguration, ?bool $migrateConfiguration, ?array $groups, ?array $testsCovering, ?array $testsUsing, ?bool $help, ?string $includePath, ?array $iniSettings, ?string $junitLogfile, ?bool $listGroups, ?bool $listSuites, ?bool $listTests, ?string $listTestsXml, ?bool $noCoverage, ?bool $noExtensions, ?bool $noInteraction, ?bool $noLogging, ?string $printer, ?bool $processIsolation, ?int $randomOrderSeed, ?int $repeat, ?bool $reportUselessTests, ?bool $resolveDependencies, ?bool $reverseList, ?bool $stderr, ?bool $strictCoverage, ?bool $stopOnDefect, ?bool $stopOnError, ?bool $stopOnFailure, ?bool $stopOnIncomplete, ?bool $stopOnRisky, ?bool $stopOnSkipped, ?bool $stopOnWarning, ?string $teamcityLogfile, ?array $testdoxExcludeGroups, ?array $testdoxGroups, ?string $testdoxHtmlFile, ?string $testdoxTextFile, ?string $testdoxXmlFile, ?array $testSuffixes, ?string $testSuite, array $unrecognizedOptions, ?string $unrecognizedOrderBy, ?bool $useDefaultConfiguration, ?bool $verbose, ?bool $version, ?array $coverageFilter)
+    public function __construct(?string $argument, ?string $atLeastVersion, ?bool $backupGlobals, ?bool $backupStaticAttributes, ?bool $beStrictAboutChangesToGlobalState, ?bool $beStrictAboutResourceUsageDuringSmallTests, ?string $bootstrap, ?bool $cacheResult, ?string $cacheResultFile, ?bool $checkVersion, ?string $colors, $columns, ?string $configuration, ?string $coverageClover, ?string $coverageCobertura, ?string $coverageCrap4J, ?string $coverageHtml, ?string $coveragePhp, ?string $coverageText, ?bool $coverageTextShowUncoveredFiles, ?bool $coverageTextShowOnlySummary, ?string $coverageXml, ?bool $pathCoverage, ?string $coverageCacheDirectory, ?bool $warmCoverageCache, ?bool $debug, ?int $defaultTimeLimit, ?bool $disableCodeCoverageIgnore, ?bool $disallowTestOutput, ?bool $disallowTodoAnnotatedTests, ?bool $enforceTimeLimit, ?array $excludeGroups, ?int $executionOrder, ?int $executionOrderDefects, ?array $extensions, ?array $unavailableExtensions, ?bool $failOnEmptyTestSuite, ?bool $failOnIncomplete, ?bool $failOnRisky, ?bool $failOnSkipped, ?bool $failOnWarning, ?string $filter, ?bool $generateConfiguration, ?bool $migrateConfiguration, ?array $groups, ?array $testsCovering, ?array $testsUsing, ?bool $help, ?string $includePath, ?array $iniSettings, ?string $junitLogfile, ?bool $listGroups, ?bool $listSuites, ?bool $listTests, ?string $listTestsXml, ?bool $noCoverage, ?bool $noExtensions, ?bool $noInteraction, ?bool $noLogging, ?string $printer, ?bool $processIsolation, ?int $randomOrderSeed, ?int $repeat, ?bool $reportUselessTests, ?bool $resolveDependencies, ?bool $reverseList, ?bool $stderr, ?bool $strictCoverage, ?bool $stopOnDefect, ?bool $stopOnError, ?bool $stopOnFailure, ?bool $stopOnIncomplete, ?bool $stopOnRisky, ?bool $stopOnSkipped, ?bool $stopOnWarning, ?string $teamcityLogfile, ?array $testdoxExcludeGroups, ?array $testdoxGroups, ?string $testdoxHtmlFile, ?string $testdoxTextFile, ?string $testdoxXmlFile, ?array $testSuffixes, ?string $testSuite, array $unrecognizedOptions, ?string $unrecognizedOrderBy, ?bool $useDefaultConfiguration, ?bool $verbose, ?bool $version, ?array $coverageFilter, ?string $testsXml)
     {
         $this->argument                                   = $argument;
         $this->atLeastVersion                             = $atLeastVersion;
@@ -293,6 +295,7 @@ public function __construct(?string $argument, ?string $atLeastVersion, ?bool $b
         $this->useDefaultConfiguration                    = $useDefaultConfiguration;
         $this->verbose                                    = $verbose;
         $this->version                                    = $version;
+        $this->testsXml                                   = $testsXml;
     }
 
     public function hasArgument(): bool
@@ -1795,4 +1798,21 @@ public function version(): bool
 
         return $this->version;
     }
+
+    public function hasTestsXml(): bool
+    {
+        return $this->testsXml !== null;
+    }
+
+    /**
+     * @throws Exception
+     */
+    public function testsXml(): string
+    {
+        if ($this->testsXml === null) {
+            throw new Exception;
+        }
+
+        return $this->testsXml;
+    }
 }

src/TextUI/CliArguments/Mapper.php

@@ -351,6 +351,10 @@ public function mapToLegacyArray(Configuration $arguments): array
             $result['randomOrderSeed'] = $arguments->randomOrderSeed();
         }
 
+        if ($arguments->hasTestsXml()) {
+            $result['testsXml'] = $arguments->testsXml();
+        }
+
         return $result;
     }
 }

src/TextUI/Help.php

@@ -71,6 +71,7 @@ final class Help
             ['arg' => '--list-tests', 'desc' => 'List available tests'],
             ['arg' => '--list-tests-xml <file>', 'desc' => 'List available tests in XML format'],
             ['arg' => '--filter <pattern>', 'desc' => 'Filter which tests to run'],
+            ['arg' => '--tests-xml <file>', 'desc' => 'Filter which tests to run in XML format'],
             ['arg' => '--test-suffix <suffixes>', 'desc' => 'Only search for test in files with specified suffix(es). Default: Test.php,.phpt'],
         ],
 

src/TextUI/TestRunner.php

@@ -39,6 +39,7 @@
 use PHPUnit\Runner\Filter\Factory;
 use PHPUnit\Runner\Filter\IncludeGroupFilterIterator;
 use PHPUnit\Runner\Filter\NameFilterIterator;
+use PHPUnit\Runner\Filter\XmlTestsIterator;
 use PHPUnit\Runner\Hook;
 use PHPUnit\Runner\NullTestResultCache;
 use PHPUnit\Runner\ResultCacheExtension;
@@ -113,6 +114,7 @@ public function __construct(CodeCoverageFilter $filter = null)
     /**
      * @throws \PHPUnit\Runner\Exception
      * @throws \PHPUnit\TextUI\XmlConfiguration\Exception
+     * @throws \PHPUnit\Util\Xml\Exception
      * @throws Exception
      */
     public function run(TestSuite $suite, array $arguments = [], array $warnings = [], bool $exit = true): TestResult
@@ -1046,13 +1048,17 @@ private function handleConfiguration(array &$arguments): void
         $arguments['verbose']                                         = $arguments['verbose'] ?? false;
     }
 
+    /**
+     * @throws \PHPUnit\Util\Xml\Exception
+     */
     private function processSuiteFilters(TestSuite $suite, array $arguments): void
     {
         if (!$arguments['filter'] &&
             empty($arguments['groups']) &&
             empty($arguments['excludeGroups']) &&
             empty($arguments['testsCovering']) &&
-            empty($arguments['testsUsing'])) {
+            empty($arguments['testsUsing']) &&
+            empty($arguments['testsXml'])) {
             return;
         }
 
@@ -1103,6 +1109,13 @@ static function (string $name): string {
             );
         }
 
+        if (!empty($arguments['testsXml'])) {
+            $filterFactory->addFilter(
+                new ReflectionClass(XmlTestsIterator::class),
+                XmlTestsIterator::createFilterFromXmlFile($arguments['testsXml']),
+            );
+        }
+
         $suite->injectFilter($filterFactory);
     }
 

tests/end-to-end/cli/_files/output-cli-help-color.txt

@@ -52,6 +52,7 @@
   --list-tests                List available tests
   --list-tests-xml <file>     List available tests in XML format
   --filter <pattern>          Filter which tests to run
+  --tests-xml <file>          Filter which tests to run in XML format
   --test-suffix <suffixes>    Only search for test in files with
                               specified suffix(es). Default:
                               Test.php,.phpt

tests/end-to-end/cli/_files/output-cli-usage.txt

@@ -42,6 +42,7 @@ Test Selection Options:
   --list-tests                List available tests
   --list-tests-xml <file>     List available tests in XML format
   --filter <pattern>          Filter which tests to run
+  --tests-xml <file>          Filter which tests to run in XML format
   --test-suffix <suffixes>    Only search for test in files with specified suffix(es). Default: Test.php,.phpt
 
 Test Execution Options:

tests/end-to-end/tests-xml-dataprovider.phpt

@@ -0,0 +1,48 @@
+--TEST--
+phpunit --list-tests-xml ../../_files/DataProviderTest.php
+--FILE--
+<?php declare(strict_types=1);
+$xml = tempnam(sys_get_temp_dir(), __FILE__);
+file_put_contents($xml, <<<XML
+<?xml version="1.0"?>
+<tests>
+  <!-- This class exists -->
+  <testCaseClass name="PHPUnit\TestFixture\DataProviderTest">
+    <testCaseMethod name="testAdd" groups="default" dataSet="#0"/>
+    <!-- This method does not exist -->
+    <testCaseMethod name="methodDoesNotExist"/>
+    <!-- name attribute missing -->
+    <testCaseMethod />
+  </testCaseClass>
+
+  <!-- name attribute missing -->
+  <testCaseClass>
+    <testCaseMethod name="testAdd" groups="default" dataSet="#0"/>
+  </testCaseClass>
+
+  <ignoredTag/>
+
+  <testCaseClass name="Class\Does\Not\Exist">
+    <testCaseMethod name="methodAlsoDoesNotExist"/>
+  </testCaseClass>
+</tests>
+XML
+);
+
+$_SERVER['argv'][1] = '--no-configuration';
+$_SERVER['argv'][2] = '--tests-xml';
+$_SERVER['argv'][3] = $xml;
+$_SERVER['argv'][4] = __DIR__ . '/../_files/DataProviderTest.php';
+
+require __DIR__ . '/../bootstrap.php';
+PHPUnit\TextUI\Command::main(false);
+
+unlink($xml);
+--EXPECTF--
+PHPUnit %s by Sebastian Bergmann and contributors.
+
+.                                                                   1 / 1 (100%)
+
+Time: %s, Memory: %s
+
+OK (1 test, 1 assertion)