From c9b2e3c6e9824bb4093a74a6e17046f58bcd8c90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Tamarelle?= Date: Tue, 16 Apr 2024 21:39:19 +0200 Subject: [PATCH] PHPORM-99 Implement optimized lock and cache --- CHANGELOG.md | 1 + composer.json | 3 +- src/Cache/MongoLock.php | 134 +++++++++++++ src/Cache/MongoStore.php | 296 ++++++++++++++++++++++++++++ src/MongoDBServiceProvider.php | 26 +++ tests/Cache/MongoCacheStoreTest.php | 231 ++++++++++++++++++++++ tests/Cache/MongoLockTest.php | 99 ++++++++++ tests/TestCase.php | 6 + 8 files changed, 795 insertions(+), 1 deletion(-) create mode 100644 src/Cache/MongoLock.php create mode 100644 src/Cache/MongoStore.php create mode 100644 tests/Cache/MongoCacheStoreTest.php create mode 100644 tests/Cache/MongoLockTest.php diff --git a/CHANGELOG.md b/CHANGELOG.md index 55a84247e..f653604ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to this project will be documented in this file. * New aggregation pipeline builder by @GromNaN in [#2738](https://github.com/mongodb/laravel-mongodb/pull/2738) * Drop support for Composer 1.x by @GromNaN in [#2784](https://github.com/mongodb/laravel-mongodb/pull/2784) * Fix `artisan query:retry` command by @GromNaN in [#2838](https://github.com/mongodb/laravel-mongodb/pull/2838) +* Add `mongodb` cache and lock drivers by @GromNaN in [#2877](https://github.com/mongodb/laravel-mongodb/pull/2877) ## [4.2.0] - 2024-03-14 diff --git a/composer.json b/composer.json index 51c7e1e43..8c038819e 100644 --- a/composer.json +++ b/composer.json @@ -25,10 +25,11 @@ "php": "^8.1", "ext-mongodb": "^1.15", "composer-runtime-api": "^2.0.0", - "illuminate/support": "^10.0|^11", + "illuminate/cache": "^10.36|^11", "illuminate/container": "^10.0|^11", "illuminate/database": "^10.30|^11", "illuminate/events": "^10.0|^11", + "illuminate/support": "^10.0|^11", "mongodb/mongodb": "^1.15" }, "require-dev": { diff --git a/src/Cache/MongoLock.php b/src/Cache/MongoLock.php new file mode 100644 index 000000000..105a3df40 --- /dev/null +++ b/src/Cache/MongoLock.php @@ -0,0 +1,134 @@ + [ + ['$lte' => ['$expiration', $this->currentTime()]], + ['$eq' => ['$owner', $this->owner]], + ], + ]; + $result = $this->collection->findOneAndUpdate( + ['_id' => $this->name], + [ + [ + '$set' => [ + 'owner' => [ + '$cond' => [ + 'if' => $isExpiredOrAlreadyOwned, + 'then' => $this->owner, + 'else' => '$owner', + ], + ], + 'expiration' => [ + '$cond' => [ + 'if' => $isExpiredOrAlreadyOwned, + 'then' => $this->expiresAt(), + 'else' => '$expiration', + ], + ], + ], + ], + ], + [ + 'upsert' => true, + 'returnDocument' => FindOneAndUpdate::RETURN_DOCUMENT_AFTER, + 'projection' => ['owner' => 1], + ], + ); + + if (random_int(1, $this->lottery[1]) <= $this->lottery[0]) { + $this->collection->deleteMany(['expiration' => ['$lte' => $this->currentTime()]]); + } + + return $result['owner'] === $this->owner; + } + + /** + * Release the lock. + */ + #[Override] + public function release(): bool + { + $result = $this->collection + ->deleteOne([ + '_id' => $this->name, + 'owner' => $this->owner, + ]); + + return $result->getDeletedCount() > 0; + } + + /** + * Releases this lock in disregard of ownership. + */ + #[Override] + public function forceRelease(): void + { + $this->collection->deleteOne([ + '_id' => $this->name, + ]); + } + + /** + * Returns the owner value written into the driver for this lock. + */ + #[Override] + protected function getCurrentOwner(): ?string + { + return $this->collection->findOne( + [ + '_id' => $this->name, + 'expiration' => ['$gte' => $this->currentTime()], + ], + ['projection' => ['owner' => 1]], + )['owner'] ?? null; + } + + /** + * Get the UNIX timestamp indicating when the lock should expire. + */ + private function expiresAt(): int + { + $lockTimeout = $this->seconds > 0 ? $this->seconds : $this->defaultTimeoutInSeconds; + + return $this->currentTime() + $lockTimeout; + } +} diff --git a/src/Cache/MongoStore.php b/src/Cache/MongoStore.php new file mode 100644 index 000000000..4a01c9161 --- /dev/null +++ b/src/Cache/MongoStore.php @@ -0,0 +1,296 @@ +collection = $this->connection->getCollection($this->collectionName); + } + + /** + * Get a lock instance. + * + * @param string $name + * @param int $seconds + * @param string|null $owner + */ + #[Override] + public function lock($name, $seconds = 0, $owner = null): MongoLock + { + return new MongoLock( + ($this->lockConnection ?? $this->connection)->getCollection($this->lockCollectionName), + $this->prefix . $name, + $seconds, + $owner, + $this->lockLottery, + $this->defaultLockTimeoutInSeconds, + ); + } + + /** + * Restore a lock instance using the owner identifier. + */ + #[Override] + public function restoreLock($name, $owner): MongoLock + { + return $this->lock($name, 0, $owner); + } + + /** + * Store an item in the cache for a given number of seconds. + * + * @param string $key + * @param mixed $value + * @param int $seconds + */ + #[Override] + public function put($key, $value, $seconds): bool + { + $result = $this->collection->updateOne( + [ + '_id' => $this->prefix . $key, + ], + [ + '$set' => [ + 'value' => $this->serialize($value), + 'expiration' => $this->currentTime() + $seconds, + ], + ], + [ + 'upsert' => true, + + ], + ); + + return $result->getUpsertedCount() > 0 || $result->getModifiedCount() > 0; + } + + /** + * Store an item in the cache if the key doesn't exist. + * + * @param string $key + * @param mixed $value + * @param int $seconds + */ + public function add($key, $value, $seconds): bool + { + $result = $this->collection->updateOne( + [ + '_id' => $this->prefix . $key, + ], + [ + [ + '$set' => [ + 'value' => [ + '$cond' => [ + 'if' => ['$lte' => ['$expiration', $this->currentTime()]], + 'then' => $this->serialize($value), + 'else' => '$value', + ], + ], + 'expiration' => [ + '$cond' => [ + 'if' => ['$lte' => ['$expiration', $this->currentTime()]], + 'then' => $this->currentTime() + $seconds, + 'else' => '$expiration', + ], + ], + ], + ], + ], + ['upsert' => true], + ); + + return $result->getUpsertedCount() > 0 || $result->getModifiedCount() > 0; + } + + /** + * Retrieve an item from the cache by key. + * + * @param string $key + */ + #[Override] + public function get($key): mixed + { + $result = $this->collection->findOne( + ['_id' => $this->prefix . $key], + ['projection' => ['value' => 1, 'expiration' => 1]], + ); + + if (! $result) { + return null; + } + + if ($result['expiration'] <= $this->currentTime()) { + $this->forgetIfExpired($key); + + return null; + } + + return $this->unserialize($result['value']); + } + + /** + * Increment the value of an item in the cache. + * + * @param string $key + * @param int|float $value + */ + #[Override] + public function increment($key, $value = 1): int|float|false + { + $this->forgetIfExpired($key); + + $result = $this->collection->findOneAndUpdate( + [ + '_id' => $this->prefix . $key, + 'expiration' => ['$gte' => $this->currentTime()], + ], + [ + '$inc' => ['value' => $value], + ], + [ + 'returnDocument' => FindOneAndUpdate::RETURN_DOCUMENT_AFTER, + ], + ); + + if (! $result) { + return false; + } + + if ($result['expiration'] <= $this->currentTime()) { + $this->forgetIfExpired($key); + + return false; + } + + return $result['value']; + } + + /** + * Decrement the value of an item in the cache. + * + * @param string $key + * @param int|float $value + */ + #[Override] + public function decrement($key, $value = 1): int|float|false + { + return $this->increment($key, -1 * $value); + } + + /** + * Store an item in the cache indefinitely. + * + * @param string $key + * @param mixed $value + */ + #[Override] + public function forever($key, $value): bool + { + return $this->put($key, $value, self::TEN_YEARS_IN_SECONDS); + } + + /** + * Remove an item from the cache. + * + * @param string $key + */ + #[Override] + public function forget($key): bool + { + $result = $this->collection->deleteOne([ + '_id' => $this->prefix . $key, + ]); + + return $result->getDeletedCount() > 0; + } + + /** + * Remove an item from the cache if it is expired. + * + * @param string $key + */ + public function forgetIfExpired($key): bool + { + $result = $this->collection->deleteOne([ + '_id' => $this->prefix . $key, + 'expiration' => ['$lte' => $this->currentTime()], + ]); + + return $result->getDeletedCount() > 0; + } + + public function flush(): bool + { + $this->collection->deleteMany([]); + + return true; + } + + public function getPrefix(): string + { + return $this->prefix; + } + + private function serialize($value): string|int|float + { + // Don't serialize numbers, so they can be incremented + if (is_int($value) || is_float($value)) { + return $value; + } + + return serialize($value); + } + + private function unserialize($value): mixed + { + if (! is_string($value) || ! str_contains($value, ';')) { + return $value; + } + + return unserialize($value); + } +} diff --git a/src/MongoDBServiceProvider.php b/src/MongoDBServiceProvider.php index d7af0c714..50c042230 100644 --- a/src/MongoDBServiceProvider.php +++ b/src/MongoDBServiceProvider.php @@ -4,10 +4,16 @@ namespace MongoDB\Laravel; +use Illuminate\Cache\CacheManager; +use Illuminate\Cache\Repository; +use Illuminate\Foundation\Application; use Illuminate\Support\ServiceProvider; +use MongoDB\Laravel\Cache\MongoStore; use MongoDB\Laravel\Eloquent\Model; use MongoDB\Laravel\Queue\MongoConnector; +use function assert; + class MongoDBServiceProvider extends ServiceProvider { /** @@ -34,6 +40,26 @@ public function register() }); }); + // Add cache and lock drivers. + $this->app->resolving('cache', function (CacheManager $cache) { + $cache->extend('mongodb', function (Application $app, array $config): Repository { + // The closure is bound to the CacheManager + assert($this instanceof CacheManager); + + $store = new MongoStore( + $app['db']->connection($config['connection'] ?? null), + $config['collection'] ?? 'cache', + $this->getPrefix($config), + $app['db']->connection($config['lock_connection'] ?? $config['connection'] ?? null), + $config['lock_collection'] ?? ($config['collection'] ?? 'cache') . '_locks', + $config['lock_lottery'] ?? [2, 100], + $config['lock_timeout'] ?? 86400, + ); + + return $this->repository($store, $config); + }); + }); + // Add connector for queue support. $this->app->resolving('queue', function ($queue) { $queue->addConnector('mongodb', function () { diff --git a/tests/Cache/MongoCacheStoreTest.php b/tests/Cache/MongoCacheStoreTest.php new file mode 100644 index 000000000..4ee97e75a --- /dev/null +++ b/tests/Cache/MongoCacheStoreTest.php @@ -0,0 +1,231 @@ +getCollection($this->getCacheCollectionName()) + ->drop(); + + parent::tearDown(); + } + + public function testGetNullWhenItemDoesNotExist() + { + $store = $this->getStore(); + $this->assertNull($store->get('foo')); + } + + public function testValueCanStoreNewCache() + { + $store = $this->getStore(); + + $store->put('foo', 'bar', 60); + + $this->assertSame('bar', $store->get('foo')); + } + + public function testPutOperationShouldNotStoreExpired() + { + $store = $this->getStore(); + + $store->put('foo', 'bar', 0); + + $this->assertDatabaseMissing($this->getCacheCollectionName(), ['_id' => $this->withCachePrefix('foo')]); + } + + public function testValueCanUpdateExistCache() + { + $store = $this->getStore(); + + $store->put('foo', 'bar', 60); + $store->put('foo', 'new-bar', 60); + + $this->assertSame('new-bar', $store->get('foo')); + } + + public function testValueCanUpdateExistCacheInTransaction() + { + $store = $this->getStore(); + + $store->put('foo', 'bar', 60); + + // Transactions are not used in MongoStore + DB::beginTransaction(); + $store->put('foo', 'new-bar', 60); + DB::commit(); + + $this->assertSame('new-bar', $store->get('foo')); + } + + public function testAddOperationShouldNotStoreExpired() + { + $store = $this->getStore(); + + $result = $store->add('foo', 'bar', 0); + + $this->assertFalse($result); + $this->assertDatabaseMissing($this->getCacheCollectionName(), ['_id' => $this->withCachePrefix('foo')]); + } + + public function testAddOperationCanStoreNewCache() + { + $store = $this->getStore(); + + $result = $store->add('foo', 'bar', 60); + + $this->assertTrue($result); + $this->assertSame('bar', $store->get('foo')); + } + + public function testAddOperationShouldNotUpdateExistCache() + { + $store = $this->getStore(); + + $store->add('foo', 'bar', 60); + $result = $store->add('foo', 'new-bar', 60); + + $this->assertFalse($result); + $this->assertSame('bar', $store->get('foo')); + } + + public function testAddOperationShouldNotUpdateExistCacheInTransaction() + { + $store = $this->getStore(); + + $store->add('foo', 'bar', 60); + + DB::beginTransaction(); + $result = $store->add('foo', 'new-bar', 60); + DB::commit(); + + $this->assertFalse($result); + $this->assertSame('bar', $store->get('foo')); + } + + public function testAddOperationCanUpdateIfCacheExpired() + { + $store = $this->getStore(); + + $this->insertToCacheTable('foo', 'bar', 0); + $result = $store->add('foo', 'new-bar', 60); + + $this->assertTrue($result); + $this->assertSame('new-bar', $store->get('foo')); + } + + public function testAddOperationCanUpdateIfCacheExpiredInTransaction() + { + $store = $this->getStore(); + + $this->insertToCacheTable('foo', 'bar', 0); + + DB::beginTransaction(); + $result = $store->add('foo', 'new-bar', 60); + DB::commit(); + + $this->assertTrue($result); + $this->assertSame('new-bar', $store->get('foo')); + } + + public function testGetOperationReturnNullIfExpired() + { + $store = $this->getStore(); + + $this->insertToCacheTable('foo', 'bar', 0); + + $result = $store->get('foo'); + + $this->assertNull($result); + } + + public function testGetOperationCanDeleteExpired() + { + $store = $this->getStore(); + + $this->insertToCacheTable('foo', 'bar', 0); + + $store->get('foo'); + + $this->assertDatabaseMissing($this->getCacheCollectionName(), ['_id' => $this->withCachePrefix('foo')]); + } + + public function testForgetIfExpiredOperationCanDeleteExpired() + { + $store = $this->getStore(); + + $this->insertToCacheTable('foo', 'bar', 0); + + $store->forgetIfExpired('foo'); + + $this->assertDatabaseMissing($this->getCacheCollectionName(), ['_id' => $this->withCachePrefix('foo')]); + } + + public function testForgetIfExpiredOperationShouldNotDeleteUnExpired() + { + $store = $this->getStore(); + + $store->put('foo', 'bar', 60); + + $store->forgetIfExpired('foo'); + + $this->assertDatabaseHas($this->getCacheCollectionName(), ['_id' => $this->withCachePrefix('foo')]); + } + + public function testIncrementDecrement() + { + $store = $this->getStore(); + $this->assertFalse($store->increment('foo', 10)); + $this->assertFalse($store->decrement('foo', 10)); + + $store->put('foo', 3.5, 60); + $this->assertSame(13.5, $store->increment('foo', 10)); + $this->assertSame(12.0, $store->decrement('foo', 1.5)); + $store->forget('foo'); + + $this->insertToCacheTable('foo', 10, -5); + $this->assertFalse($store->increment('foo', 5)); + } + + protected function getStore(): Repository + { + $repository = Cache::store('mongodb'); + assert($repository instanceof Repository); + + return $repository; + } + + protected function getCacheCollectionName(): string + { + return config('cache.stores.mongodb.collection'); + } + + protected function withCachePrefix(string $key): string + { + return config('cache.prefix') . $key; + } + + protected function insertToCacheTable(string $key, $value, $ttl = 60) + { + DB::connection('mongodb') + ->getCollection($this->getCacheCollectionName()) + ->insertOne([ + '_id' => $this->withCachePrefix($key), + 'value' => $value, + 'expiration' => Carbon::now()->addSeconds($ttl)->getTimestamp(), + ]); + } +} diff --git a/tests/Cache/MongoLockTest.php b/tests/Cache/MongoLockTest.php new file mode 100644 index 000000000..d08ee899c --- /dev/null +++ b/tests/Cache/MongoLockTest.php @@ -0,0 +1,99 @@ +getCollection('foo_cache_locks')->drop(); + + parent::tearDown(); + } + + public function testLockCanBeAcquired() + { + $lock = $this->getCache()->lock('foo'); + $this->assertTrue($lock->get()); + $this->assertTrue($lock->get()); + + $otherLock = $this->getCache()->lock('foo'); + $this->assertFalse($otherLock->get()); + + $lock->release(); + + $otherLock = $this->getCache()->lock('foo'); + $this->assertTrue($otherLock->get()); + $this->assertTrue($otherLock->get()); + + $otherLock->release(); + } + + public function testLockCanBeForceReleased() + { + $lock = $this->getCache()->lock('foo'); + $this->assertTrue($lock->get()); + + $otherLock = $this->getCache()->lock('foo'); + $otherLock->forceRelease(); + $this->assertTrue($otherLock->get()); + + $otherLock->release(); + } + + public function testExpiredLockCanBeRetrieved() + { + $lock = $this->getCache()->lock('foo'); + $this->assertTrue($lock->get()); + DB::table('foo_cache_locks')->update(['expiration' => now()->subDays(1)->getTimestamp()]); + + $otherLock = $this->getCache()->lock('foo'); + $this->assertTrue($otherLock->get()); + + $otherLock->release(); + } + + public function testOwnedByCurrentProcess() + { + $lock = $this->getCache()->lock('foo'); + $this->assertFalse($lock->isOwnedByCurrentProcess()); + + $lock->acquire(); + $this->assertTrue($lock->isOwnedByCurrentProcess()); + + $otherLock = $this->getCache()->lock('foo'); + $this->assertFalse($otherLock->isOwnedByCurrentProcess()); + } + + public function testRestoreLock() + { + $lock = $this->getCache()->lock('foo'); + $lock->acquire(); + $this->assertInstanceOf(MongoLock::class, $lock); + + $owner = $lock->owner(); + + $resoredLock = $this->getCache()->restoreLock('foo', $owner); + $this->assertTrue($resoredLock->isOwnedByCurrentProcess()); + + $resoredLock->release(); + $this->assertFalse($resoredLock->isOwnedByCurrentProcess()); + } + + private function getCache(): Repository + { + $repository = Cache::driver('mongodb'); + + $this->assertInstanceOf(Repository::class, $repository); + + return $repository; + } +} diff --git a/tests/TestCase.php b/tests/TestCase.php index 9f3a76e00..e2be67a04 100644 --- a/tests/TestCase.php +++ b/tests/TestCase.php @@ -75,6 +75,12 @@ protected function getEnvironmentSetUp($app) $app['config']->set('auth.providers.users.model', User::class); $app['config']->set('cache.driver', 'array'); + $app['config']->set('cache.stores.mongodb', [ + 'driver' => 'mongodb', + 'connection' => 'mongodb', + 'collection' => 'foo_cache', + ]); + $app['config']->set('queue.default', 'database'); $app['config']->set('queue.connections.database', [ 'driver' => 'mongodb',