diff --git a/docs/en/reference/working-with-objects.rst b/docs/en/reference/working-with-objects.rst index f10c0ab091d..d88b814e8c4 100644 --- a/docs/en/reference/working-with-objects.rst +++ b/docs/en/reference/working-with-objects.rst @@ -782,6 +782,23 @@ and these associations are mapped as EAGER, they will automatically be loaded together with the entity being queried and is thus immediately available to your application. +Eager Loading can also be configured at runtime through +``AbstractQuery::setFetchMode`` in DQL or Native Queries. + +Eager loading for many-to-one and one-to-one associations is using either a +LEFT JOIN or a second query for fetching the related entity eagerly. + +Eager loading for many-to-one associations uses a second query to load +the collections for several entities at the same time. + +When many-to-many, one-to-one or one-to-many associations are eagerly loaded, +then the global batch size configuration is used to avoid IN(?) queries with +too many arguments. The default batch size is 100 and can be changed with +``Configuration::setEagerFetchBatchSize()``. + +For eagerly loaded Many-To-Many associations one query has to be made for each +collection. + By Lazy Loading ~~~~~~~~~~~~~~~ diff --git a/lib/Doctrine/ORM/Configuration.php b/lib/Doctrine/ORM/Configuration.php index c6eccc25405..a6c932724f7 100644 --- a/lib/Doctrine/ORM/Configuration.php +++ b/lib/Doctrine/ORM/Configuration.php @@ -1144,4 +1144,14 @@ public function isRejectIdCollisionInIdentityMapEnabled(): bool { return $this->_attributes['rejectIdCollisionInIdentityMap'] ?? false; } + + public function setEagerFetchBatchSize(int $batchSize = 100): void + { + $this->_attributes['fetchModeSubselectBatchSize'] = $batchSize; + } + + public function getEagerFetchBatchSize(): int + { + return $this->_attributes['fetchModeSubselectBatchSize'] ?? 100; + } } diff --git a/lib/Doctrine/ORM/Persisters/Entity/BasicEntityPersister.php b/lib/Doctrine/ORM/Persisters/Entity/BasicEntityPersister.php index 32ac1d4f531..f968e7b7162 100644 --- a/lib/Doctrine/ORM/Persisters/Entity/BasicEntityPersister.php +++ b/lib/Doctrine/ORM/Persisters/Entity/BasicEntityPersister.php @@ -1264,7 +1264,7 @@ protected function getSelectColumnsSQL() } $isAssocToOneInverseSide = $assoc['type'] & ClassMetadata::TO_ONE && ! $assoc['isOwningSide']; - $isAssocFromOneEager = $assoc['type'] !== ClassMetadata::MANY_TO_MANY && $assoc['fetch'] === ClassMetadata::FETCH_EAGER; + $isAssocFromOneEager = $assoc['type'] & ClassMetadata::TO_ONE && $assoc['fetch'] === ClassMetadata::FETCH_EAGER; if (! ($isAssocFromOneEager || $isAssocToOneInverseSide)) { continue; diff --git a/lib/Doctrine/ORM/Query/QueryException.php b/lib/Doctrine/ORM/Query/QueryException.php index be9b8ed750e..9604ff5e107 100644 --- a/lib/Doctrine/ORM/Query/QueryException.php +++ b/lib/Doctrine/ORM/Query/QueryException.php @@ -204,6 +204,14 @@ public static function iterateWithFetchJoinNotAllowed($assoc) ); } + public static function eagerFetchJoinWithNotAllowed(string $sourceEntity, string $fieldName): QueryException + { + return new self( + 'Associations with fetch-mode=EAGER may not be using WITH conditions in + "' . $sourceEntity . '#' . $fieldName . '".' + ); + } + public static function iterateWithMixedResultNotAllowed(): QueryException { return new self('Iterating a query with mixed results (using scalars) is not supported.'); diff --git a/lib/Doctrine/ORM/Query/SqlWalker.php b/lib/Doctrine/ORM/Query/SqlWalker.php index a677ca26710..d3ff78a2405 100644 --- a/lib/Doctrine/ORM/Query/SqlWalker.php +++ b/lib/Doctrine/ORM/Query/SqlWalker.php @@ -1047,7 +1047,9 @@ public function walkJoinAssociationDeclaration($joinAssociationDeclaration, $joi } } - $targetTableJoin = null; + if ($relation['fetch'] === ClassMetadata::FETCH_EAGER && $condExpr !== null) { + throw QueryException::eagerFetchJoinWithNotAllowed($assoc['sourceEntity'], $assoc['fieldName']); + } // This condition is not checking ClassMetadata::MANY_TO_ONE, because by definition it cannot // be the owning side and previously we ensured that $assoc is always the owning side of the associations. diff --git a/lib/Doctrine/ORM/UnitOfWork.php b/lib/Doctrine/ORM/UnitOfWork.php index 607957a48f6..b1fef6d2012 100644 --- a/lib/Doctrine/ORM/UnitOfWork.php +++ b/lib/Doctrine/ORM/UnitOfWork.php @@ -51,6 +51,7 @@ use Throwable; use UnexpectedValueException; +use function array_chunk; use function array_combine; use function array_diff_key; use function array_filter; @@ -314,6 +315,9 @@ class UnitOfWork implements PropertyChangedListener */ private $eagerLoadingEntities = []; + /** @var array> */ + private $eagerLoadingCollections = []; + /** @var bool */ protected $hasCache = false; @@ -2749,6 +2753,7 @@ public function clear($entityName = null) $this->pendingCollectionElementRemovals = $this->visitedCollections = $this->eagerLoadingEntities = + $this->eagerLoadingCollections = $this->orphanRemovals = []; } else { Deprecation::triggerIfCalledFromOutside( @@ -2938,6 +2943,10 @@ public function createEntity($className, array $data, &$hints = []) continue; } + if (! isset($hints['fetchMode'][$class->name][$field])) { + $hints['fetchMode'][$class->name][$field] = $assoc['fetch']; + } + $targetClass = $this->em->getClassMetadata($assoc['targetEntity']); switch (true) { @@ -3001,10 +3010,6 @@ public function createEntity($className, array $data, &$hints = []) break; } - if (! isset($hints['fetchMode'][$class->name][$field])) { - $hints['fetchMode'][$class->name][$field] = $assoc['fetch']; - } - // Foreign key is set // Check identity map first // FIXME: Can break easily with composite keys if join column values are in @@ -3098,9 +3103,13 @@ public function createEntity($className, array $data, &$hints = []) $reflField = $class->reflFields[$field]; $reflField->setValue($entity, $pColl); - if ($assoc['fetch'] === ClassMetadata::FETCH_EAGER) { - $this->loadCollection($pColl); - $pColl->takeSnapshot(); + if ($hints['fetchMode'][$class->name][$field] === ClassMetadata::FETCH_EAGER) { + if ($assoc['type'] === ClassMetadata::ONE_TO_MANY) { + $this->scheduleCollectionForBatchLoading($pColl, $class); + } elseif ($assoc['type'] === ClassMetadata::MANY_TO_MANY) { + $this->loadCollection($pColl); + $pColl->takeSnapshot(); + } } $this->originalEntityData[$oid][$field] = $pColl; @@ -3117,7 +3126,7 @@ public function createEntity($className, array $data, &$hints = []) /** @return void */ public function triggerEagerLoads() { - if (! $this->eagerLoadingEntities) { + if (! $this->eagerLoadingEntities && ! $this->eagerLoadingCollections) { return; } @@ -3130,11 +3139,69 @@ public function triggerEagerLoads() continue; } - $class = $this->em->getClassMetadata($entityName); + $class = $this->em->getClassMetadata($entityName); + $batches = array_chunk($ids, $this->em->getConfiguration()->getEagerFetchBatchSize()); - $this->getEntityPersister($entityName)->loadAll( - array_combine($class->identifier, [array_values($ids)]) - ); + foreach ($batches as $batchedIds) { + $this->getEntityPersister($entityName)->loadAll( + array_combine($class->identifier, [$batchedIds]) + ); + } + } + + $eagerLoadingCollections = $this->eagerLoadingCollections; // avoid recursion + $this->eagerLoadingCollections = []; + + foreach ($eagerLoadingCollections as $group) { + $this->eagerLoadCollections($group['items'], $group['mapping']); + } + } + + /** + * Load all data into the given collections, according to the specified mapping + * + * @param PersistentCollection[] $collections + * @param array $mapping + * @psalm-param array{targetEntity: class-string, sourceEntity: class-string, mappedBy: string, indexBy: string|null} $mapping + */ + private function eagerLoadCollections(array $collections, array $mapping): void + { + $targetEntity = $mapping['targetEntity']; + $class = $this->em->getClassMetadata($mapping['sourceEntity']); + $mappedBy = $mapping['mappedBy']; + + $batches = array_chunk($collections, $this->em->getConfiguration()->getEagerFetchBatchSize(), true); + + foreach ($batches as $collectionBatch) { + $entities = []; + + foreach ($collectionBatch as $collection) { + $entities[] = $collection->getOwner(); + } + + $found = $this->getEntityPersister($targetEntity)->loadAll([$mappedBy => $entities]); + + $targetClass = $this->em->getClassMetadata($targetEntity); + $targetProperty = $targetClass->getReflectionProperty($mappedBy); + + foreach ($found as $targetValue) { + $sourceEntity = $targetProperty->getValue($targetValue); + + $id = $this->identifierFlattener->flattenIdentifier($class, $class->getIdentifierValues($sourceEntity)); + $idHash = implode(' ', $id); + + if (isset($mapping['indexBy'])) { + $indexByProperty = $targetClass->getReflectionProperty($mapping['indexBy']); + $collectionBatch[$idHash]->hydrateSet($indexByProperty->getValue($targetValue), $targetValue); + } else { + $collectionBatch[$idHash]->add($targetValue); + } + } + } + + foreach ($collections as $association) { + $association->setInitialized(true); + $association->takeSnapshot(); } } @@ -3165,6 +3232,33 @@ public function loadCollection(PersistentCollection $collection) $collection->setInitialized(true); } + /** + * Schedule this collection for batch loading at the end of the UnitOfWork + */ + private function scheduleCollectionForBatchLoading(PersistentCollection $collection, ClassMetadata $sourceClass): void + { + $mapping = $collection->getMapping(); + $name = $mapping['sourceEntity'] . '#' . $mapping['fieldName']; + + if (! isset($this->eagerLoadingCollections[$name])) { + $this->eagerLoadingCollections[$name] = [ + 'items' => [], + 'mapping' => $mapping, + ]; + } + + $owner = $collection->getOwner(); + assert($owner !== null); + + $id = $this->identifierFlattener->flattenIdentifier( + $sourceClass, + $sourceClass->getIdentifierValues($owner) + ); + $idHash = implode(' ', $id); + + $this->eagerLoadingCollections[$name]['items'][$idHash] = $collection; + } + /** * Gets the identity map of the UnitOfWork. * diff --git a/tests/Doctrine/Tests/ORM/Functional/EagerFetchCollectionTest.php b/tests/Doctrine/Tests/ORM/Functional/EagerFetchCollectionTest.php new file mode 100644 index 00000000000..90cd1793a9d --- /dev/null +++ b/tests/Doctrine/Tests/ORM/Functional/EagerFetchCollectionTest.php @@ -0,0 +1,152 @@ +createSchemaForModels(EagerFetchOwner::class, EagerFetchChild::class); + } + + public function testEagerFetchMode(): void + { + $owner = $this->createOwnerWithChildren(2); + $owner2 = $this->createOwnerWithChildren(3); + + $this->_em->flush(); + $this->_em->clear(); + + $owner = $this->_em->find(EagerFetchOwner::class, $owner->id); + + $afterQueryCount = count($this->getQueryLog()->queries); + $this->assertCount(2, $owner->children); + + $this->assertQueryCount($afterQueryCount, 'The $owner->children collection should already be initialized by find EagerFetchOwner before.'); + + $this->assertCount(3, $this->_em->find(EagerFetchOwner::class, $owner2->id)->children); + + $this->_em->clear(); + + $beforeQueryCount = count($this->getQueryLog()->queries); + $owners = $this->_em->getRepository(EagerFetchOwner::class)->findAll(); + + $this->assertQueryCount($beforeQueryCount + 2, 'the findAll() + 1 subselect loading both collections of the two returned $owners'); + + $this->assertCount(2, $owners[0]->children); + $this->assertCount(3, $owners[1]->children); + + $this->assertQueryCount($beforeQueryCount + 2, 'both collections are already initialized and counting them does not make a difference in total query count'); + } + + public function testEagerFetchModeWithDQL(): void + { + $owner = $this->createOwnerWithChildren(2); + $owner2 = $this->createOwnerWithChildren(3); + + $this->_em->flush(); + $this->_em->clear(); + + $query = $this->_em->createQuery('SELECT o FROM ' . EagerFetchOwner::class . ' o'); + $query->setFetchMode(EagerFetchOwner::class, 'children', ORM\ClassMetadata::FETCH_EAGER); + + $beforeQueryCount = count($this->getQueryLog()->queries); + $owners = $query->getResult(); + $afterQueryCount = count($this->getQueryLog()->queries); + + $this->assertEquals($beforeQueryCount + 2, $afterQueryCount); + + $owners[0]->children->count(); + $owners[1]->children->count(); + + $anotherQueryCount = count($this->getQueryLog()->queries); + + $this->assertEquals($anotherQueryCount, $afterQueryCount); + } + + public function testSubselectFetchJoinWithNotAllowed(): void + { + $this->expectException(QueryException::class); + $this->expectExceptionMessage('Associations with fetch-mode=EAGER may not be using WITH conditions'); + + $query = $this->_em->createQuery('SELECT o, c FROM ' . EagerFetchOwner::class . ' o JOIN o.children c WITH c.id = 1'); + $query->getResult(); + } + + protected function createOwnerWithChildren(int $children): EagerFetchOwner + { + $owner = new EagerFetchOwner(); + $this->_em->persist($owner); + + for ($i = 0; $i < $children; $i++) { + $child = new EagerFetchChild(); + $child->owner = $owner; + + $owner->children->add($child); + + $this->_em->persist($child); + } + + return $owner; + } +} + +/** + * @ORM\Entity + */ +class EagerFetchOwner +{ + /** + * @ORM\Column(type="integer") + * @ORM\Id + * @ORM\GeneratedValue() + * + * @var int + */ + public $id; + + /** + * @ORM\OneToMany(targetEntity="EagerFetchChild", mappedBy="owner", fetch="EAGER") + * + * @var ArrayCollection + */ + public $children; + + public function __construct() + { + $this->children = new ArrayCollection(); + } +} + +/** + * @ORM\Entity + */ +class EagerFetchChild +{ + /** + * @ORM\Column(type="integer") + * @ORM\Id + * @ORM\GeneratedValue() + * + * @var int + */ + public $id; + + /** + * @ORM\ManyToOne(targetEntity="EagerFetchOwner", inversedBy="children") + * + * @var EagerFetchOwner + */ + public $owner; +} diff --git a/tests/Doctrine/Tests/ORM/Functional/Ticket/DDC2350Test.php b/tests/Doctrine/Tests/ORM/Functional/Ticket/DDC2350Test.php index 2f9e7c514bd..3fe53e6bc2f 100644 --- a/tests/Doctrine/Tests/ORM/Functional/Ticket/DDC2350Test.php +++ b/tests/Doctrine/Tests/ORM/Functional/Ticket/DDC2350Test.php @@ -44,11 +44,11 @@ public function testEagerCollectionsAreOnlyRetrievedOnce(): void $this->getQueryLog()->reset()->enable(); $user = $this->_em->find(DDC2350User::class, $user->id); - $this->assertQueryCount(1); + $this->assertQueryCount(2); self::assertCount(2, $user->reportedBugs); - $this->assertQueryCount(1); + $this->assertQueryCount(2); } } diff --git a/tests/Doctrine/Tests/ORM/Functional/Ticket/DDC440Test.php b/tests/Doctrine/Tests/ORM/Functional/Ticket/DDC440Test.php index 4c9f1289db6..8101a38d418 100644 --- a/tests/Doctrine/Tests/ORM/Functional/Ticket/DDC440Test.php +++ b/tests/Doctrine/Tests/ORM/Functional/Ticket/DDC440Test.php @@ -49,7 +49,7 @@ public function testOriginalEntityDataEmptyWhenProxyLoadedFromTwoAssociations(): $phone->setClient($client); $phone2 = new DDC440Phone(); - $phone->setId(2); + $phone2->setId(2); $phone2->setNumber('418 222-2222'); $phone2->setClient($client); @@ -88,10 +88,10 @@ public function testOriginalEntityDataEmptyWhenProxyLoadedFromTwoAssociations(): class DDC440Phone { /** - * @var int * @Column(name="id", type="integer") * @Id * @GeneratedValue(strategy="AUTO") + * @var int */ protected $id; diff --git a/tests/Doctrine/Tests/ORM/Functional/Ticket/GH10808Test.php b/tests/Doctrine/Tests/ORM/Functional/Ticket/GH10808Test.php index a2ff78f43a9..78966fe8073 100644 --- a/tests/Doctrine/Tests/ORM/Functional/Ticket/GH10808Test.php +++ b/tests/Doctrine/Tests/ORM/Functional/Ticket/GH10808Test.php @@ -39,8 +39,7 @@ public function testDQLDeferredEagerLoad(): void $query = $this->_em->createQuery( 'SELECT appointment from Doctrine\Tests\ORM\Functional\Ticket\GH10808Appointment appointment - JOIN appointment.child appointment_child - WITH appointment_child.id = 1' + JOIN appointment.child appointment_child' ); // By default, UnitOfWork::HINT_DEFEREAGERLOAD is set to 'true'