diff --git a/src/Tools/Pagination/Paginator.php b/src/Tools/Pagination/Paginator.php index db1b34db715..c3cc0a38dbf 100644 --- a/src/Tools/Pagination/Paginator.php +++ b/src/Tools/Pagination/Paginator.php @@ -7,13 +7,20 @@ use ArrayIterator; use Countable; use Doctrine\Common\Collections\Collection; +use Doctrine\ORM\EntityManagerInterface; use Doctrine\ORM\Internal\SQLResultCasing; +use Doctrine\ORM\Mapping\ClassMetadata; use Doctrine\ORM\NoResultException; use Doctrine\ORM\Query; +use Doctrine\ORM\Query\AST\Join; +use Doctrine\ORM\Query\AST\JoinAssociationDeclaration; +use Doctrine\ORM\Query\AST\Node; +use Doctrine\ORM\Query\AST\SelectExpression; use Doctrine\ORM\Query\Parameter; use Doctrine\ORM\Query\Parser; use Doctrine\ORM\Query\ResultSetMapping; use Doctrine\ORM\QueryBuilder; +use Doctrine\Persistence\Mapping\MappingException; use IteratorAggregate; use Traversable; @@ -21,7 +28,11 @@ use function array_map; use function array_sum; use function assert; +use function count; +use function in_array; use function is_string; +use function reset; +use function str_starts_with; /** * The paginator can handle various complex scenarios with DQL. @@ -36,13 +47,26 @@ class Paginator implements Countable, IteratorAggregate public const HINT_ENABLE_DISTINCT = 'paginator.distinct.enable'; private readonly Query $query; - private bool|null $useOutputWalkers = null; - private int|null $count = null; + private bool|null $useResultQueryOutputWalker = null; + private bool|null $useCountQueryOutputWalker = null; + private int|null $count = null; + /** + * The auto-detection of queries style was added a lot later to this class, and this + * class historically was by default using the more complex queries style, which means that + * the simple queries style is potentially very under-tested in production systems. The purpose + * of this variable is to not introduce breaking changes until an impression is developed that + * the simple queries style has been battle-tested enough. + */ + private bool $queryStyleAutoDetectionEnabled = false; + private bool $queryCouldHaveToManyJoins = true; - /** @param bool $fetchJoinCollection Whether the query joins a collection (true by default). */ + /** + * @param bool $queryCouldProduceDuplicates Whether the query could produce partially duplicated records. One case + * when it does is when it joins a collection. + */ public function __construct( Query|QueryBuilder $query, - private readonly bool $fetchJoinCollection = true, + private readonly bool $queryCouldProduceDuplicates = true, ) { if ($query instanceof QueryBuilder) { $query = $query->getQuery(); @@ -51,6 +75,190 @@ public function __construct( $this->query = $query; } + /** + * Create an instance of Paginator with auto-detection of whether the provided + * query is suitable for simple (and fast) pagination queries, or whether a complex + * set of pagination queries has to be used. + */ + public static function newWithAutoDetection(Query|QueryBuilder $query): self + { + if ($query instanceof QueryBuilder) { + $query = $query->getQuery(); + } + + $queryAST = $query->getAST(); + [ + 'hasGroupByClause' => $queryHasGroupByClause, + 'hasHavingClause' => $queryHasHavingClause, + 'rootEntityHasSingleIdentifierFieldName' => $rootEntityHasSingleIdentifierFieldName, + 'couldProduceDuplicates' => $queryCouldProduceDuplicates, + 'couldHaveToManyJoins' => $queryCouldHaveToManyJoins, + ] = self::autoDetectQueryFeatures($query->getEntityManager(), $queryAST); + + $paginator = new self($query, $queryCouldProduceDuplicates); + + $paginator->queryStyleAutoDetectionEnabled = true; + $paginator->queryCouldHaveToManyJoins = $queryCouldHaveToManyJoins; + // The following is ensuring the conditions for when the CountWalker cannot be used. + $paginator->useCountQueryOutputWalker = $queryHasHavingClause !== false + || $rootEntityHasSingleIdentifierFieldName !== true + || ($queryCouldHaveToManyJoins && $queryHasGroupByClause !== false); + + return $paginator; + } + + /** + * @return array{ + * hasGroupByClause: bool|null, + * hasHavingClause: bool|null, + * rootEntityHasSingleIdentifierFieldName: bool|null, + * couldProduceDuplicates: bool, + * couldHaveToManyJoins: bool, + * } + */ + private static function autoDetectQueryFeatures(EntityManagerInterface $entityManager, Node $queryAST): array + { + $queryFeatures = [ + // Null means undetermined + 'hasGroupByClause' => null, + 'hasHavingClause' => null, + 'rootEntityHasSingleIdentifierFieldName' => null, + 'couldProduceDuplicates' => true, + 'couldHaveToManyJoins' => true, + ]; + + if (! $queryAST instanceof Query\AST\SelectStatement) { + return $queryFeatures; + } + + $queryFeatures['hasGroupByClause'] = $queryAST->groupByClause !== null; + $queryFeatures['hasHavingClause'] = $queryAST->havingClause !== null; + + $from = $queryAST->fromClause->identificationVariableDeclarations; + if (count($from) > 1) { + return $queryFeatures; + } + + $fromRoot = reset($from); + if (! $fromRoot instanceof Query\AST\IdentificationVariableDeclaration) { + return $queryFeatures; + } + + if (! $fromRoot->rangeVariableDeclaration) { + return $queryFeatures; + } + + $rootAlias = $fromRoot->rangeVariableDeclaration->aliasIdentificationVariable; + try { + $rootClassMetadata = $entityManager->getClassMetadata($fromRoot->rangeVariableDeclaration->abstractSchemaName); + $queryFeatures['rootEntityHasSingleIdentifierFieldName'] = (bool) $rootClassMetadata->getSingleIdentifierFieldName(); + } catch (MappingException) { + return $queryFeatures; + } + + $rootClassName = $fromRoot->rangeVariableDeclaration->abstractSchemaName; + + $aliasesToClassNameOrClassMetadataMap = []; + $aliasesToClassNameOrClassMetadataMap[$rootAlias] = $rootClassMetadata; + $toManyJoinsAliases = []; + + // Check the Joins list. + foreach ($fromRoot->joins as $join) { + if (! $join instanceof Join || ! $join->joinAssociationDeclaration instanceof JoinAssociationDeclaration) { + return $queryFeatures; + } + + $joinParentAlias = $join->joinAssociationDeclaration->joinAssociationPathExpression->identificationVariable; + $joinParentFieldName = $join->joinAssociationDeclaration->joinAssociationPathExpression->associationField; + $joinAlias = $join->joinAssociationDeclaration->aliasIdentificationVariable; + + // Every Join descending from a ToMany Join is "in principle" also a ToMany Join + if (in_array($joinParentAlias, $toManyJoinsAliases, true)) { + $toManyJoinsAliases[] = $joinAlias; + + continue; + } + + $parentClassMetadata = $aliasesToClassNameOrClassMetadataMap[$joinParentAlias] ?? null; + if (! $parentClassMetadata) { + return $queryFeatures; + } + + // Load entity class metadata. + if (is_string($parentClassMetadata)) { + try { + $parentClassMetadata = $entityManager->getClassMetadata($parentClassMetadata); + $aliasesToClassNameOrClassMetadataMap[$joinParentAlias] = $parentClassMetadata; + } catch (MappingException) { + return $queryFeatures; + } + } + + $parentJoinAssociationMapping = $parentClassMetadata->associationMappings[$joinParentFieldName] ?? null; + if (! $parentJoinAssociationMapping) { + return $queryFeatures; + } + + $aliasesToClassNameOrClassMetadataMap[$joinAlias] = $parentJoinAssociationMapping['targetEntity']; + + if (! ($parentJoinAssociationMapping['type'] & ClassMetadata::TO_MANY)) { + continue; + } + + // The Join is a ToMany Join. + $toManyJoinsAliases[] = $joinAlias; + } + + $queryFeatures['couldHaveToManyJoins'] = count($toManyJoinsAliases) > 0; + + // Check the Select list. + foreach ($queryAST->selectClause->selectExpressions as $selectExpression) { + if (! $selectExpression instanceof SelectExpression) { + return $queryFeatures; + } + + // Must not use any of the ToMany aliases + if (is_string($selectExpression->expression)) { + foreach ($toManyJoinsAliases as $toManyJoinAlias) { + if ( + $selectExpression->expression === $toManyJoinAlias + || str_starts_with($selectExpression->expression, $toManyJoinAlias . '.') + ) { + return $queryFeatures; + } + } + } + + // If it's a function, then it has to be one from the following list. Reason: in some databases, + // there are functions that "generate rows". + if ( + $selectExpression->expression instanceof Query\AST\Functions\FunctionNode + && ! in_array($selectExpression->expression::class, [ + Query\AST\Functions\CountFunction::class, + Query\AST\Functions\AvgFunction::class, + Query\AST\Functions\SumFunction::class, + Query\AST\Functions\MinFunction::class, + Query\AST\Functions\MaxFunction::class, + ], true) + ) { + return $queryFeatures; + } + } + + // If there are ToMany Joins, then the Select clause has to use the DISTINCT keyword. Note: the foreach + // above also ensures that the ToMany Joins are not in the Select list, which is relevant. + if ( + count($toManyJoinsAliases) > 0 + && ! $queryAST->selectClause->isDistinct + ) { + return $queryFeatures; + } + + $queryFeatures['couldProduceDuplicates'] = false; + + return $queryFeatures; + } + /** * Returns the query. */ @@ -60,31 +268,80 @@ public function getQuery(): Query } /** - * Returns whether the query joins a collection. + * @deprecated Use ::getQueryCouldProduceDuplicates() instead. * - * @return bool Whether the query joins a collection. + * Returns whether the query joins a collection. */ public function getFetchJoinCollection(): bool { - return $this->fetchJoinCollection; + return $this->queryCouldProduceDuplicates; + } + + /** + * Returns whether the query could produce partially duplicated records. + */ + public function getQueryCouldProduceDuplicates(): bool + { + return $this->queryCouldProduceDuplicates; } /** + * @deprecated Use the individual ::get*OutputWalker() + * * Returns whether the paginator will use an output walker. */ public function getUseOutputWalkers(): bool|null { - return $this->useOutputWalkers; + return $this->getUseResultQueryOutputWalker() && $this->getUseCountQueryOutputWalker(); } /** + * @deprecated Use the individual ::set*OutputWalker() + * * Sets whether the paginator will use an output walker. * * @return $this */ public function setUseOutputWalkers(bool|null $useOutputWalkers): static { - $this->useOutputWalkers = $useOutputWalkers; + $this->setUseResultQueryOutputWalker($useOutputWalkers); + $this->setUseCountQueryOutputWalker($useOutputWalkers); + + return $this; + } + + /** + * Returns whether the paginator will use an output walker for the result query. + */ + public function getUseResultQueryOutputWalker(): bool|null + { + return $this->useResultQueryOutputWalker; + } + + /** + * Sets whether the paginator will use an output walker for the result query. + */ + public function setUseResultQueryOutputWalker(bool|null $useResultQueryOutputWalker): static + { + $this->useResultQueryOutputWalker = $useResultQueryOutputWalker; + + return $this; + } + + /** + * Returns whether the paginator will use an output walker for the count query. + */ + public function getUseCountQueryOutputWalker(): bool|null + { + return $this->useCountQueryOutputWalker; + } + + /** + * Sets whether the paginator will use an output walker for the count query. + */ + public function setUseCountQueryOutputWalker(bool|null $useCountQueryOutputWalker): static + { + $this->useCountQueryOutputWalker = $useCountQueryOutputWalker; return $this; } @@ -112,7 +369,7 @@ public function getIterator(): Traversable $offset = $this->query->getFirstResult(); $length = $this->query->getMaxResults(); - if ($this->fetchJoinCollection && $length !== null) { + if ($this->queryCouldProduceDuplicates && $length !== null) { $subQuery = $this->cloneQuery($this->query); if ($this->useOutputWalker($subQuery)) { @@ -171,13 +428,18 @@ private function cloneQuery(Query $query): Query /** * Determines whether to use an output walker for the query. */ - private function useOutputWalker(Query $query): bool + private function useOutputWalker(Query $query, bool $forCountQuery = false): bool { - if ($this->useOutputWalkers === null) { - return (bool) $query->getHint(Query::HINT_CUSTOM_OUTPUT_WALKER) === false; + if (! $forCountQuery && $this->useResultQueryOutputWalker !== null) { + return $this->useResultQueryOutputWalker; + } + + if ($forCountQuery && $this->useCountQueryOutputWalker !== null) { + return $this->useCountQueryOutputWalker; } - return $this->useOutputWalkers; + // When a custom output walker already present, then do not use the Paginator's. + return $query->getHint(Query::HINT_CUSTOM_OUTPUT_WALKER) === false; } /** @@ -205,10 +467,20 @@ private function getCountQuery(): Query $countQuery = $this->cloneQuery($this->query); if (! $countQuery->hasHint(CountWalker::HINT_DISTINCT)) { - $countQuery->setHint(CountWalker::HINT_DISTINCT, true); + $hintDistinctDefaultTrue = true; + + // When not joining onto *ToMany relations, then use a simpler COUNT query in the CountWalker. + if ( + $this->queryStyleAutoDetectionEnabled + && ! $this->queryCouldHaveToManyJoins + ) { + $hintDistinctDefaultTrue = false; + } + + $countQuery->setHint(CountWalker::HINT_DISTINCT, $hintDistinctDefaultTrue); } - if ($this->useOutputWalker($countQuery)) { + if ($this->useOutputWalker($countQuery, forCountQuery: true)) { $platform = $countQuery->getEntityManager()->getConnection()->getDatabasePlatform(); // law of demeter win $rsm = new ResultSetMapping(); diff --git a/tests/Tests/ORM/Tools/Pagination/PaginatorTest.php b/tests/Tests/ORM/Tools/Pagination/PaginatorTest.php index 41e3981e3b2..da43c2e5fac 100644 --- a/tests/Tests/ORM/Tools/Pagination/PaginatorTest.php +++ b/tests/Tests/ORM/Tools/Pagination/PaginatorTest.php @@ -13,7 +13,9 @@ use Doctrine\ORM\Internal\Hydration\AbstractHydrator; use Doctrine\ORM\Query; use Doctrine\ORM\Query\QueryException; +use Doctrine\ORM\QueryBuilder; use Doctrine\ORM\Tools\Pagination\Paginator; +use Doctrine\Tests\Models\CMS\CmsUser; use Doctrine\Tests\OrmTestCase; use PHPUnit\Framework\MockObject\MockObject; @@ -110,6 +112,18 @@ public function testgetIteratorDoesCareAboutExtraParametersWithoutOutputWalkersW $this->createPaginatorWithExtraParametersWithoutOutputWalkers([[10]])->getIterator(); } + /** @todo-PR-11595 FINISH TESTS */ + public function testNewWithAutoDetection(): void + { + $queryBuilder = new QueryBuilder($this->em); + $queryBuilder->select('u'); + $queryBuilder->from(CmsUser::class, 'u'); + + $paginator = Paginator::newWithAutoDetection($queryBuilder); + + $this->assertFalse($paginator->getFetchJoinCollection()); + } + /** @param int[][] $willReturnRows */ private function createPaginatorWithExtraParametersWithoutOutputWalkers(array $willReturnRows): Paginator {