Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Sharding support implemented #691

Closed
wants to merge 2 commits into from
Closed

Sharding support implemented #691

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
152b762
Sharding support implemented
lotar Oct 17, 2013
8b7a127
Throw MongoDBException::shardKeyChange() only if non-upsert update
lotar Mar 12, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 8 additions & 6 deletions lib/Doctrine/ODM/MongoDB/Hydrator/HydratorFactory.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -250,7 +250,8 @@ private function generateHydratorClass(ClassMetadata $class, $hydratorClassName,
\$mongoId = \$reference['\$id'];
}
\$targetMetadata = \$this->dm->getClassMetadata(\$className);
\$id = \$targetMetadata->getPHPIdentifierValue(\$mongoId);
\$idType = \$targetMetadata->fieldMappings[\$targetMetadata->identifier]['type'];
\$id = Type::getType(\$idType)->convertToPHPValue(\$mongoId);
\$return = \$this->dm->getReference(\$className, \$id);
\$this->class->reflFields['%2\$s']->setValue(\$document, \$return);
\$hydratedData['%2\$s'] = \$return;
Expand Down Expand Up @@ -329,7 +330,7 @@ private function generateHydratorClass(ClassMetadata $class, $hydratorClassName,
\$embeddedMetadata = \$this->dm->getClassMetadata(\$className);
\$return = \$embeddedMetadata->newInstance();

\$embeddedData = \$this->dm->getHydratorFactory()->hydrate(\$return, \$embeddedDocument, \$hints);
\$embeddedData = \$this->dm->getHydratorFactory()->hydrate(\$embeddedMetadata, \$return, \$embeddedDocument, \$hints);
\$this->unitOfWork->registerManaged(\$return, null, \$embeddedData);
\$this->unitOfWork->setParentAssociation(\$return, \$this->class->fieldMappings['%2\$s'], \$document, '%1\$s');

Expand All @@ -356,6 +357,7 @@ private function generateHydratorClass(ClassMetadata $class, $hydratorClassName,
use Doctrine\ODM\MongoDB\Mapping\ClassMetadata;
use Doctrine\ODM\MongoDB\Hydrator\HydratorInterface;
use Doctrine\ODM\MongoDB\UnitOfWork;
use Doctrine\ODM\MongoDB\Types\Type;

/**
* THIS CLASS WAS GENERATED BY THE DOCTRINE ODM. DO NOT EDIT THIS FILE.
Expand All @@ -373,7 +375,7 @@ public function __construct(DocumentManager \$dm, UnitOfWork \$uow, ClassMetadat
\$this->class = \$class;
}

public function hydrate(\$document, \$data, array \$hints = array())
public function hydrate(ClassMetadata \$metadata, \$document, \$data, array \$hints = array())
{
\$hydratedData = array();
%s return \$hydratedData;
Expand Down Expand Up @@ -402,14 +404,14 @@ public function hydrate(\$document, \$data, array \$hints = array())
/**
* Hydrate array of MongoDB document data into the given document object.
*
* @param ClassMetadata $metadata The ClassMetadata instance for the passed $document.
* @param object $document The document object to hydrate the data into.
* @param array $data The array of document data.
* @param array $hints Any hints to account for during reconstitution/lookup of the document.
* @return array $values The array of hydrated values.
*/
public function hydrate($document, $data, array $hints = array())
public function hydrate(ClassMetadata $metadata, $document, $data, array $hints = array())
{
$metadata = $this->dm->getClassMetadata(get_class($document));
// Invoke preLoad lifecycle events and listeners
if (isset($metadata->lifecycleCallbacks[Events::preLoad])) {
$args = array(&$data);
Expand All @@ -428,7 +430,7 @@ public function hydrate($document, $data, array $hints = array())
}
}

$data = $this->getHydratorFor($metadata->name)->hydrate($document, $data, $hints);
$data = $this->getHydratorFor($metadata->name)->hydrate($metadata, $document, $data, $hints);
if ($document instanceof Proxy) {
$document->__isInitialized__ = true;
}
Expand Down
3 changes: 2 additions & 1 deletion lib/Doctrine/ODM/MongoDB/Hydrator/HydratorInterface.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,11 @@ interface HydratorInterface
/**
* Hydrate array of MongoDB document data into the given document object.
*
* @param ClassMetadata $metadata The ClassMetadata instance for the passed $document.
* @param object $document The document object to hydrate the data into.
* @param array $data The array of document data.
* @param array $hints Any hints to account for during reconstitution/lookup of the document.
* @return array $values The array of hydrated values.
*/
function hydrate($document, $data, array $hints = array());
function hydrate(ClassMetadata $metadata, $document, $data, array $hints = array());
}
34 changes: 29 additions & 5 deletions lib/Doctrine/ODM/MongoDB/Mapping/ClassMetadataInfo.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,10 +159,15 @@ class ClassMetadataInfo implements \Doctrine\Common\Persistence\Mapping\ClassMet
*/
public $collectionMax;

/**
* READ-ONLY: The field name of the document identifier.
*/
public $identifier;
/**
* READ-ONLY: The field name of the document identifier.
*/
public $identifier;

/**
* The array of shard keys.
*/
public $shardKeys = array();

/**
* READ-ONLY: The field that stores a file reference and indicates the
Expand Down Expand Up @@ -455,6 +460,24 @@ public function getIdentifier()
return $this->identifier;
}

public function getShardKeys()
{
return $this->shardKeys;
}

public function setShardKeys(array $shardKeys)
{
$this->shardKeys = $shardKeys;
}

public function addShardKey($key)
{
if (!is_string($key)) {
throw MappingException::invalidShardKeyValue($this->reflClass->getName());
}
array_push($this->shardKeys, $key);
}

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Indentation for these looks off. Are you using tabs instead of 4 spaces per level?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Might be tabs, will have to check, but that can be easily modified

/**
* Get identifier field names of this class.
*
Expand Down Expand Up @@ -1317,7 +1340,8 @@ public function getDatabaseIdentifierValue($id)
*/
public function setIdentifierValue($document, $id)
{
$id = $this->getPHPIdentifierValue($id);
$idType = $this->fieldMappings[$this->identifier]['type'];
$id = Type::getType($idType)->convertToPHPValue($id);
$this->reflFields[$this->identifier]->setValue($document, $id);
}

Expand Down
4 changes: 4 additions & 0 deletions lib/Doctrine/ODM/MongoDB/Mapping/Driver/AnnotationDriver.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,10 @@ public function loadMetadataForClass($className, ClassMetadata $class)
$mapping['alsoLoadFields'] = (array) $annot->value;
} elseif ($annot instanceof ODM\Version) {
$mapping['version'] = true;
} elseif (property_exists($annot, 'options')
&& isset($annot->options['isShardKey'])
&& $annot->options['isShardKey'] == true) {
$class->addShardKey($mapping['fieldName']);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If the order of calls to addShardKey() can change based on the parsing order for fields, that could easily break. Would it be better to define a shard key similar to a compound index at the class level?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've just looked for relevance in shard key order when querying but haven't found anything. LMK if I'm wrong, but AFAIK order of keys is irrelevant.

Also, in JS console look at the following scenario:

mongos> db.MessageThreads2.update({_id: 'testlotar', fu: 1}, {$set: {test:1}}, true)
can't upsert something without full valid shard key : { _id: "testlotar", fu: 1.0 }
mongos> db.MessageThreads2.update({_id: 'testlotar', fu: 1, tId:1}, {$set: {test:1}}, true)
mongos> db.MessageThreads2.findOne({_id : 'testlotar'})
{ "_id" : "testlotar", "fu" : 1, "tId" : 1, "test" : 1 }
mongos> db.MessageThreads2.remove({_id: 'testlotar'})
mongos> db.MessageThreads2.findOne({_id : 'testlotar'})
null
mongos> db.MessageThreads2.update({_id: 'testlotar', tId:1, fu:1}, {$set: {test:1}}, true)
mongos> db.MessageThreads2.findOne({_id : 'testlotar'})
{ "_id" : "testlotar", "fu" : 1, "tId" : 1, "test" : 1 }
mongos> db.MessageThreads2.remove({_id: 'testlotar'})
mongos> db.MessageThreads2.findOne({_id : 'testlotar'})
null

and the shard key is following:

mongos> db.MessageThreads2.getIndexKeys()
[
    {
        "tId" : 1,
        "fu" : 1
    }

As I wrote, if I'm wrong LMK and will reconsider, but otherwise don't see a point in it.

} elseif ($annot instanceof ODM\Lock) {
$mapping['lock'] = true;
}
Expand Down
5 changes: 5 additions & 0 deletions lib/Doctrine/ODM/MongoDB/Mapping/MappingException.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,11 @@ public static function duplicateFieldMapping($document, $fieldName)
return new self('Property "' . $fieldName . '" in "' . $document . '" was already declared, but it must be declared only once');
}

public static function invalidShardKeyValue($document)
{
return new self('Invalid shard key for ' . $document . '. Has to be string.');
}

/**
* Throws an exception that indicates that a class used in a discriminator map does not exist.
* An example would be an outdated (maybe renamed) classname.
Expand Down
10 changes: 10 additions & 0 deletions lib/Doctrine/ODM/MongoDB/MongoDBException.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,16 @@ public static function invalidDocumentState($state)
return new self(sprintf('Invalid document state "%s"', $state));
}

public static function shardKeyChange($key)
{
return new self(sprintf('Shard key "%s" value cannot be changed.', $key));
}

public static function shardKeyMissing($key)
{
return new self(sprintf('Shard key "%s" is missing.', $key));
}

public static function documentNotMappedToCollection($className)
{
return new self(sprintf('The "%s" document is not mapped to a MongoDB database collection.', $className));
Expand Down
90 changes: 79 additions & 11 deletions lib/Doctrine/ODM/MongoDB/Persisters/DocumentPersister.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,10 +203,12 @@ public function executeInserts(array $options = array())
$postInsertIds = array();
$inserts = array();
$upserts = array();
$shardCriteria = array();
foreach ($this->queuedInserts as $oid => $document) {
$upsert = $this->uow->isScheduledForUpsert($document);
if ($upsert) {
$data = $this->pb->prepareUpsertData($document);
$shardCriteria[$oid] = $this->getShardKeyQuery($document);
} else {
$data = $this->pb->prepareInsertData($document);
}
Expand Down Expand Up @@ -241,7 +243,11 @@ public function executeInserts(array $options = array())

foreach ($inserts as $oid => $data) {
$document = $this->queuedInserts[$oid];
$postInsertIds[] = array($this->class->getPHPIdentifierValue($data['_id']), $document);

$idType = $this->class->fieldMappings[$this->class->identifier]['type'];
$postInsertId = Type::getType($idType)->convertToPHPValue($data['_id']);

$postInsertIds[] = array($postInsertId, $document);
}
}

Expand All @@ -252,18 +258,35 @@ public function executeInserts(array $options = array())
$upsertOptions['upsert'] = true;
foreach ($upserts as $oid => $data) {
$criteria = array('_id' => $data[$this->cmd . 'set']['_id']);
$criteria = array_merge($criteria, $shardCriteria[$oid]);
unset($data[$this->cmd . 'set']['_id']);
// stupid php
if (empty($data[$this->cmd . 'set'])) {
$data[$this->cmd . 'set'] = new \stdClass;
}
$data = $this->stripShardKeyData($shardCriteria[$oid], $data);
$this->collection->update($criteria, $data, $upsertOptions);
}
}

return $postInsertIds;
}

public function stripShardKeyData(array $shardCriteria, array $data)
{
if (!empty($shardCriteria)) {
foreach ($data as $operation => $values) {
$keys = array_intersect_key($values, $shardCriteria);
if ($keys) {
foreach ($keys as $k => $one) {
unset($data[$operation][$k]);
}
}
}
}
return $data;
}

/**
* Updates the already persisted document if it has any new changesets.
*
Expand All @@ -280,6 +303,9 @@ public function update($document, array $options = array())

$id = $this->class->getDatabaseIdentifierValue($id);
$query = array('_id' => $id);
$shardKeys = $this->getShardKeyQuery($document);

$query = array_merge($query, $shardKeys);

// Include versioning logic to set the new version value in the database
// and to ensure the version has not changed since this document object instance
Expand Down Expand Up @@ -314,6 +340,7 @@ public function update($document, array $options = array())
}

unset($update[$this->cmd . 'set']['_id']);
$update = $this->stripShardKeyData($shardKeys, $update);
$result = $this->collection->update($query, $update, $options);

if (($this->class->isVersioned || $this->class->isLockable) && ! $result['n']) {
Expand All @@ -322,6 +349,48 @@ public function update($document, array $options = array())
}
}

/**
* @param $document
* @return array
* @throws MongoDBException
*/
public function getShardKeyQuery($document)
{
$shardKeysQueryPart = array();

$dcs = $this->uow->getDocumentChangeSet($document);
$data = $this->uow->getDocumentActualData($this->class, $document);

$md = $this->dm->getMetadataFactory()->getMetadataFor(get_class($document));
$keys = $md->shardKeys;

$fieldMappings = $this->dm->getClassMetadata(get_class($document))->fieldMappings;
foreach ($keys as $key) {
if ($key !== 'id') {
$queryKey = $fieldMappings[$key]['name'];
}

//If the document is new, we can ignore shard key value, otherwise throw exception
$isUpdate = $this->uow->isScheduledForUpdate($document);
if ($isUpdate && isset($dcs[$key]) && $dcs[$key][0] != $dcs[$key][1]) {
throw MongoDBException::shardKeyChange($key);
}
if (!isset($data[$key])) {
throw MongoDBException::shardKeyMissing($key);
}
if (preg_match('/[a-f0-9]{24}/i', $data[$key])) {
$data[$key] = new \MongoId($data[$key]);
}
if ($md->isIdentifier($key)) {
$shardKeysQueryPart['_id'] = $data[$key];
} else {
$shardKeysQueryPart[$queryKey] = $data[$key];
}

}
return $shardKeysQueryPart;
}

/**
* Removes document from mongo
*
Expand Down Expand Up @@ -356,7 +425,7 @@ public function refresh($id, $document)
{
$class = $this->dm->getClassMetadata(get_class($document));
$data = $this->collection->findOne(array('_id' => $id));
$data = $this->hydratorFactory->hydrate($document, $data);
$data = $this->hydratorFactory->hydrate($class, $document, $data);
$this->uow->setOriginalDocumentData($document, $data);
}

Expand Down Expand Up @@ -649,7 +718,7 @@ private function loadEmbedManyCollection(PersistentCollection $collection)
$embeddedMetadata = $this->dm->getClassMetadata($className);
$embeddedDocumentObject = $embeddedMetadata->newInstance();

$data = $this->hydratorFactory->hydrate($embeddedDocumentObject, $embeddedDocument);
$data = $this->hydratorFactory->hydrate($embeddedMetadata, $embeddedDocumentObject, $embeddedDocument);
$this->uow->registerManaged($embeddedDocumentObject, null, $data);
$this->uow->setParentAssociation($embeddedDocumentObject, $mapping, $owner, $mapping['name'] . '.' . $key);
if ($mapping['strategy'] === 'set') {
Expand Down Expand Up @@ -678,7 +747,12 @@ private function loadReferenceManyCollectionOwningSide(PersistentCollection $col
$className = $this->dm->getClassNameFromDiscriminatorValue($mapping, $reference);
$mongoId = $reference[$cmd . 'id'];
}
$id = $this->dm->getClassMetadata($className)->getPHPIdentifierValue($mongoId);

$class = $this->dm->getClassMetadata($className);

$idType = $class->fieldMappings[$class->identifier]['type'];
$id = Type::getType($idType)->convertToPHPValue($mongoId);

if ( ! $id) {
continue;
}
Expand Down Expand Up @@ -725,7 +799,7 @@ private function loadReferenceManyCollectionOwningSide(PersistentCollection $col
$documents = $cursor->toArray();
foreach ($documents as $documentData) {
$document = $this->uow->getById((string) $documentData['_id'], $class->rootDocumentName);
$data = $this->hydratorFactory->hydrate($document, $documentData);
$data = $this->hydratorFactory->hydrate($class, $document, $documentData);
$this->uow->setOriginalDocumentData($document, $data);
$document->__isInitialized__ = true;
if ($sorted) {
Expand Down Expand Up @@ -967,12 +1041,6 @@ private function prepareQueryElement($fieldName, $value = null, $class = null, $
return array($fieldName, $value);
}

// Prepare mapped, embedded objects
if ( ! empty($mapping['embedded']) && is_object($value) &&
! $this->dm->getMetadataFactory()->isTransient(get_class($value))) {
return array($fieldName, $this->pb->prepareEmbeddedDocumentValue($mapping, $value));
}

// No further preparation unless we're dealing with a simple reference
if (empty($mapping['reference']) || empty($mapping['simple'])) {
return array($fieldName, $value);
Expand Down
Loading