diff --git a/.travis.yml b/.travis.yml index 35c4258d6..f10e7b763 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,6 +30,8 @@ matrix: env: SYMFONY_VERSION=2.3.x-dev - php: 5.6 env: SYMFONY_VERSION=2.7.x-dev + - php: 5.6 + env: SYMFONY_VERSION=2.8.x-dev WITH_ENQUEUE=true - php: 7.1 env: SYMFONY_VERSION=3.0.x-dev - php: 7.1 @@ -37,7 +39,7 @@ matrix: - php: 7.1 env: SYMFONY_VERSION=3.2.x-dev - php: 7.1 - env: SYMFONY_VERSION=3.3.x-dev + env: SYMFONY_VERSION=3.3.x-dev WITH_ENQUEUE=true - php: 7.1 env: SYMFONY_VERSION=dev-master allow_failures: @@ -55,6 +57,7 @@ before_install: - if [ "${TRAVIS_PHP_VERSION}" != "hhvm" ] && [ "${TRAVIS_PHP_VERSION:0:3}" != "5.3" ]; then composer require --no-update --dev league/flysystem:~1.0; fi; - if [ "${TRAVIS_PHP_VERSION}" != "hhvm" ] && [ "${TRAVIS_PHP_VERSION:0:1}" != "7" ]; then yes "" | pecl -q install -f mongo; composer require --no-update --dev doctrine/mongodb-odm:~1.0; fi; - if [ "${TRAVIS_PHP_VERSION}" != "hhvm" ] && [ "${TRAVIS_PHP_VERSION:0:1}" == "7" ]; then yes "" | pecl -q install -f mongodb; travis_retry composer require --dev alcaeus/mongo-php-adapter:~1.0; composer require --no-update --dev doctrine/mongodb-odm:~1.0; fi + - if [ "$WITH_ENQUEUE" = true ] ; then composer require --no-update --dev enqueue/enqueue-bundle:^0.3.6; fi; install: - travis_retry composer update $COMPOSER_FLAGS diff --git a/Async/CacheResolved.php b/Async/CacheResolved.php new file mode 100644 index 000000000..d52bcb005 --- /dev/null +++ b/Async/CacheResolved.php @@ -0,0 +1,71 @@ +path = $path; + $this->uris = $uris; + } + + /** + * @return string + */ + public function getPath() + { + return $this->path; + } + + /** + * @return \string[] + */ + public function getUris() + { + return $this->uris; + } + + /** + * {@inheritdoc} + */ + public function jsonSerialize() + { + return array('path' => $this->path, 'uris' => $this->uris); + } + + /** + * @param string $json + * + * @return static + */ + public static function jsonDeserialize($json) + { + $data = JSON::decode($json); + + if (empty($data['path'])) { + throw new \LogicException('The message does not contain "path" but it is required.'); + } + + if (empty($data['uris'])) { + throw new \LogicException('The message uris must not be empty array.'); + } + + return new static($data['path'], $data['uris']); + } +} diff --git a/Async/ResolveCache.php b/Async/ResolveCache.php new file mode 100644 index 000000000..316b50c5f --- /dev/null +++ b/Async/ResolveCache.php @@ -0,0 +1,86 @@ +path = $path; + $this->filters = $filters; + $this->force = $force; + } + + /** + * @return string + */ + public function getPath() + { + return $this->path; + } + + /** + * @return null|\string[] + */ + public function getFilters() + { + return $this->filters; + } + + /** + * @return bool + */ + public function isForce() + { + return $this->force; + } + + /** + * {@inheritdoc} + */ + public function jsonSerialize() + { + return array('path' => $this->path, 'filters' => $this->filters, 'force' => $this->force); + } + + /** + * @param string $json + * + * @return static + */ + public static function jsonDeserialize($json) + { + $data = array_replace(array('path' => null, 'filters' => null, 'force' => false), JSON::decode($json)); + + if (false == $data['path']) { + throw new \LogicException('The message does not contain "path" but it is required.'); + } + + if (false == (is_null($data['filters']) || is_array($data['filters']))) { + throw new \LogicException('The message filters could be either null or array.'); + } + + return new static($data['path'], $data['filters'], $data['force']); + } +} diff --git a/Async/ResolveCacheProcessor.php b/Async/ResolveCacheProcessor.php new file mode 100644 index 000000000..d82e24066 --- /dev/null +++ b/Async/ResolveCacheProcessor.php @@ -0,0 +1,108 @@ +cacheManager = $cacheManager; + $this->filterManager = $filterManager; + $this->dataManager = $dataManager; + $this->producer = $producer; + } + + /** + * {@inheritdoc} + */ + public function process(PsrMessage $psrMessage, PsrContext $psrContext) + { + try { + $message = ResolveCache::jsonDeserialize($psrMessage->getBody()); + } catch (\Exception $e) { + return Result::reject($e->getMessage()); + } + + $filters = $message->getFilters() ?: array_keys($this->filterManager->getFilterConfiguration()->all()); + $path = $message->getPath(); + $results = array(); + foreach ($filters as $filter) { + if ($this->cacheManager->isStored($path, $filter) && $message->isForce()) { + $this->cacheManager->remove($path, $filter); + } + + if (false == $this->cacheManager->isStored($path, $filter)) { + $binary = $this->dataManager->find($filter, $path); + $this->cacheManager->store( + $this->filterManager->applyFilter($binary, $filter), + $path, + $filter + ); + } + + $results[$filter] = $this->cacheManager->resolve($path, $filter); + } + + $this->producer->send(Topics::CACHE_RESOLVED, new CacheResolved($path, $results)); + + return self::ACK; + } + + /** + * {@inheritdoc} + */ + public static function getSubscribedTopics() + { + return array( + Topics::RESOLVE_CACHE => array('queueName' => Topics::RESOLVE_CACHE, 'queueNameHardcoded' => true), + ); + } + + /** + * {@inheritdoc} + */ + public static function getSubscribedQueues() + { + return array(Topics::RESOLVE_CACHE); + } +} diff --git a/Async/Topics.php b/Async/Topics.php new file mode 100644 index 000000000..59973761e --- /dev/null +++ b/Async/Topics.php @@ -0,0 +1,9 @@ +end() ->end() ->end() + ->booleanNode('enqueue')->defaultFalse()->info('Enables integration with enqueue if set true. Allows resolve image caches in background by sending messages to MQ.')->end() ->end(); return $treeBuilder; diff --git a/DependencyInjection/LiipImagineExtension.php b/DependencyInjection/LiipImagineExtension.php index bfbfe058f..7f406c253 100644 --- a/DependencyInjection/LiipImagineExtension.php +++ b/DependencyInjection/LiipImagineExtension.php @@ -72,6 +72,10 @@ public function load(array $configs, ContainerBuilder $container) $loader = new XmlFileLoader($container, new FileLocator(__DIR__.'/../Resources/config')); $loader->load('imagine.xml'); + if ($config['enqueue']) { + $loader->load('enqueue.xml'); + } + $this->setFactories($container); if (interface_exists('Imagine\Image\Metadata\MetadataReaderInterface')) { diff --git a/Resources/config/enqueue.xml b/Resources/config/enqueue.xml new file mode 100644 index 000000000..fc5e04f0a --- /dev/null +++ b/Resources/config/enqueue.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + diff --git a/Resources/doc/resolve-cache-images-in-background.rst b/Resources/doc/resolve-cache-images-in-background.rst new file mode 100644 index 000000000..12aa82700 --- /dev/null +++ b/Resources/doc/resolve-cache-images-in-background.rst @@ -0,0 +1,98 @@ +Resolve cache images in background +================================== + +Overview +-------- + +By default the LiipImagineBundle processes the image on demand. +It does in resolve controller and saves the result, does a 301 redirect to the processed static image. +The approach has its benefits. +The most notable is simplicity. +Though there are some disadvantages: + +* It takes huge amount of time during the first request since we have to do a lot of things. +* The images are processed by web servers. It increases the overall load on them and may affect the site performance. +* The resolve controller url is different from the cached image one. + If there is nothing in the cache the page will contain the url to resolve controller. + The varnish may cache the page with those links to the resolve controller. + A browser keeps sending requests to it though there is no need for it after the first call. + +The bundle provides a solution. It utilize messaging pattern and works on top of `enqueue library`_. + +Step 1: Install EnqueueBundle +----------------------------- + +First, we have to `install EnqueueBundle`_. You have to basically use composer to install the bundle, +register it to AppKernel and adjust settings. Here's the most simplest configuration without any extra dependencies. +It is based on `filesystem transport`_. + +.. code-block:: yaml + + # app/config/config.yml + + enqueue: + transport: + default: fs + fs: + store_dir: '%kernel.root_dir%/../var/queues' + client: ~ + +Step 2: Configure LiipImagineBundle +----------------------------------- + +At this step we instruct LiipImagineBundle to load some extra stuff required to process images in background. + +.. code-block:: yaml + + # app/config/config.yml + + liip_imagine: + enqueue: true + +Step 3: Run consumers +--------------------- + +Before we can start using it we need a pool of consumers (at least one) to be working in background. +Here's how you can run it: + +.. code-block:: bash + + $ ./app/console enqueue:consume --setup-broker -vvv + +Step 4: Send resolve cache message +---------------------------------- + +You have to send a message in order to process images in background. +The message must contain the original image path (in terms of LiipImagineBundle). +If you do not define filters the background process will resolve cache for all available filters. +If the cache already exist the background process does recreate it by default +You can force cache to be recreated and in this case the cached image is removed and a new one replaces it. + +.. code-block:: php + + get('enqueue.producer'); + + // resolve all caches + $producer->send(Topics::RESOLVE_CACHE, new ResolveCache('the/path/img.png')); + + // resolve specific cache + $producer->send(Topics::RESOLVE_CACHE, new ResolveCache('the/path/img.png', array('fooFilter'))); + + // force resolve (removes the cache if exists) + $producer->send(Topics::RESOLVE_CACHE, new ResolveCache('the/path/img.png', null, true)); + + +.. _`enqueue library`: https://github.com/php-enqueue/enqueue-dev +.. _`install EnqueueBundle`: https://github.com/php-enqueue/enqueue-dev/blob/master/docs/bundle/quick_tour.md +.. _`filesystem transport`: https://github.com/php-enqueue/enqueue-dev/blob/master/docs/transport/filesystem.md \ No newline at end of file diff --git a/Tests/Async/CacheResolvedTest.php b/Tests/Async/CacheResolvedTest.php new file mode 100644 index 000000000..51b0bd50a --- /dev/null +++ b/Tests/Async/CacheResolvedTest.php @@ -0,0 +1,39 @@ + 'http://example.com/fooFilter/thePath', + 'barFilter' => 'http://example.com/barFilter/thePath', + )); + + $this->assertEquals( + '{"path":"thePath","uris":{"fooFilter":"http:\/\/example.com\/fooFilter\/thePath","barFilter":"http:\/\/example.com\/barFilter\/thePath"}}', + json_encode($message) + ); + } + + public function testCouldBeJsonDeSerialized() + { + $message = CacheResolved::jsonDeserialize('{"path":"thePath","uris":{"fooFilter":"http:\/\/example.com\/fooFilter\/thePath","barFilter":"http:\/\/example.com\/barFilter\/thePath"}}'); + + $this->assertInstanceOf('Liip\ImagineBundle\Async\CacheResolved', $message); + $this->assertEquals('thePath', $message->getPath()); + $this->assertEquals(array( + 'fooFilter' => 'http://example.com/fooFilter/thePath', + 'barFilter' => 'http://example.com/barFilter/thePath', + ), $message->getUris()); + } +} \ No newline at end of file diff --git a/Tests/Async/ResolveCacheProcessorTest.php b/Tests/Async/ResolveCacheProcessorTest.php new file mode 100644 index 000000000..d4bbada3a --- /dev/null +++ b/Tests/Async/ResolveCacheProcessorTest.php @@ -0,0 +1,344 @@ +assertTrue($rc->implementsInterface('Enqueue\Psr\PsrProcessor')); + } + + public function testShouldImplementTopicSubscriberInterface() + { + $rc = new \ReflectionClass('Liip\ImagineBundle\Async\ResolveCacheProcessor'); + + $this->assertTrue($rc->implementsInterface('Enqueue\Client\TopicSubscriberInterface')); + } + + public function testShouldImplementQueueSubscriberInterface() + { + $rc = new \ReflectionClass('Liip\ImagineBundle\Async\ResolveCacheProcessor'); + + $this->assertTrue($rc->implementsInterface('Enqueue\Consumption\QueueSubscriberInterface')); + } + + public function testShouldSubscribeToExpectedTopic() + { + $topics = ResolveCacheProcessor::getSubscribedTopics(); + + $this->assertInternalType('array', $topics); + $this->assertArrayHasKey(Topics::RESOLVE_CACHE, $topics); + $this->assertEquals(array( + 'queueName' => 'liip_imagine_resolve_cache', + 'queueNameHardcoded' => true, + ), $topics[Topics::RESOLVE_CACHE]); + } + + public function testShouldSubscribeToExpectedQueue() + { + $queues = ResolveCacheProcessor::getSubscribedQueues(); + + $this->assertInternalType('array', $queues); + $this->assertEquals(array('liip_imagine_resolve_cache'), $queues); + } + + public function testCouldBeConstructedWithExpectedArguments() + { + new ResolveCacheProcessor( + $this->createCacheManagerMock(), + $this->createFilterManagerMock(), + $this->createDataManagerMock(), + $this->createProducerMock() + ); + } + + public function testShouldRejectMessagesWithInvalidJsonBody() + { + $processor = new ResolveCacheProcessor( + $this->createCacheManagerMock(), + $this->createFilterManagerMock(), + $this->createDataManagerMock(), + $this->createProducerMock() + ); + + $message = new NullMessage(); + $message->setBody('[}'); + + $result = $processor->process($message, new NullContext()); + + $this->assertInstanceOf('Enqueue\Consumption\Result', $result); + $this->assertEquals(Result::REJECT, (string) $result); + $this->assertStringStartsWith('The malformed json given.', $result->getReason()); + } + + public function testShouldRejectMessagesWithoutPass() + { + $processor = new ResolveCacheProcessor( + $this->createCacheManagerMock(), + $this->createFilterManagerMock(), + $this->createDataManagerMock(), + $this->createProducerMock() + ); + + $message = new NullMessage(); + $message->setBody('{}'); + + $result = $processor->process($message, new NullContext()); + + $this->assertInstanceOf('Enqueue\Consumption\Result', $result); + $this->assertEquals(Result::REJECT, (string) $result); + $this->assertEquals('The message does not contain "path" but it is required.', $result->getReason()); + } + + public function testShouldResolveCacheIfNotStored() + { + $originalBinary = $this->createDummyBinary(); + $filteredBinary = $this->createDummyBinary(); + + $filterManagerMock = $this->createFilterManagerMock(); + $filterManagerMock + ->expects($this->once()) + ->method('getFilterConfiguration') + ->willReturn(new FilterConfiguration(array( + 'fooFilter' => array('fooFilterConfig'), + ))) + ; + $filterManagerMock + ->expects($this->once()) + ->method('applyFilter') + ->with($this->identicalTo($originalBinary), 'fooFilter') + ->willReturn($filteredBinary) + ; + + $cacheManagerMock = $this->createCacheManagerMock(); + $cacheManagerMock + ->expects($this->atLeastOnce()) + ->method('isStored') + ->willReturn(false) + ; + $cacheManagerMock + ->expects($this->once()) + ->method('store') + ->with( + $this->identicalTo($filteredBinary), + 'theImagePath', + 'fooFilter' + ) + ; + $cacheManagerMock + ->expects($this->once()) + ->method('resolve') + ->with('theImagePath', 'fooFilter') + ; + + $dataManagerMock = $this->createDataManagerMock(); + $dataManagerMock + ->expects($this->once()) + ->method('find') + ->with('fooFilter', 'theImagePath') + ->willReturn($originalBinary) + ; + + $processor = new ResolveCacheProcessor( + $cacheManagerMock, + $filterManagerMock, + $dataManagerMock, + $this->createProducerMock() + ); + + $message = new NullMessage(); + $message->setBody('{"path": "theImagePath"}'); + + $result = $processor->process($message, new NullContext()); + + $this->assertEquals(Result::ACK, $result); + } + + public function testShouldNotResolveCacheIfStoredAndNotForce() + { + $filterManagerMock = $this->createFilterManagerMock(); + $filterManagerMock + ->expects($this->once()) + ->method('getFilterConfiguration') + ->willReturn(new FilterConfiguration(array( + 'fooFilter' => array('fooFilterConfig'), + ))) + ; + $filterManagerMock + ->expects($this->never()) + ->method('applyFilter') + ; + + $cacheManagerMock = $this->createCacheManagerMock(); + $cacheManagerMock + ->expects($this->atLeastOnce()) + ->method('isStored') + ->willReturn(true) + ; + $cacheManagerMock + ->expects($this->never()) + ->method('store') + ; + $cacheManagerMock + ->expects($this->once()) + ->method('resolve') + ->with('theImagePath', 'fooFilter') + ->willReturn('fooFilterUri') + ; + + $dataManagerMock = $this->createDataManagerMock(); + $dataManagerMock + ->expects($this->never()) + ->method('find') + ; + + $processor = new ResolveCacheProcessor( + $cacheManagerMock, + $filterManagerMock, + $dataManagerMock, + $this->createProducerMock() + ); + + $message = new NullMessage(); + $message->setBody('{"path": "theImagePath"}'); + + $result = $processor->process($message, new NullContext()); + + $this->assertEquals(Result::ACK, $result); + } + + public function testShouldSendMessageOnSuccessResolve() + { + $filterManagerMock = $this->createFilterManagerMock(); + $filterManagerMock + ->expects($this->once()) + ->method('getFilterConfiguration') + ->willReturn(new FilterConfiguration(array( + 'fooFilter' => array('fooFilterConfig'), + 'barFilter' => array('barFilterConfig'), + 'bazFilter' => array('bazFilterConfig'), + ))) + ; + $filterManagerMock + ->expects($this->atLeastOnce()) + ->method('applyFilter') + ->willReturn($this->createDummyBinary()) + ; + + $cacheManagerMock = $this->createCacheManagerMock(); + $cacheManagerMock + ->expects($this->atLeastOnce()) + ->method('isStored') + ->willReturn(false) + ; + $cacheManagerMock + ->expects($this->atLeastOnce()) + ->method('store') + ; + $cacheManagerMock + ->expects($this->atLeastOnce()) + ->method('resolve') + ->willReturnCallback(function($path, $filter) { + return $path.$filter.'Uri'; + }) + ; + + $dataManagerMock = $this->createDataManagerMock(); + $dataManagerMock + ->expects($this->atLeastOnce()) + ->method('find') + ->willReturn($this->createDummyBinary()) + ; + + $testCase = $this; + $producerMock = $this->createProducerMock(); + $producerMock + ->expects($this->once()) + ->method('send') + ->with(Topics::CACHE_RESOLVED, $this->isInstanceOf('Liip\ImagineBundle\Async\CacheResolved')) + ->willReturnCallback(function($topic, CacheResolved $message) use ($testCase) { + $testCase->assertEquals('theImagePath', $message->getPath()); + $testCase->assertEquals(array( + 'fooFilter' => 'theImagePathfooFilterUri', + 'barFilter' => 'theImagePathbarFilterUri', + 'bazFilter' => 'theImagePathbazFilterUri', + ), $message->getUris()); + }); + + $processor = new ResolveCacheProcessor( + $cacheManagerMock, + $filterManagerMock, + $dataManagerMock, + $producerMock + ); + + $message = new NullMessage(); + $message->setBody('{"path": "theImagePath"}'); + + $result = $processor->process($message, new NullContext()); + + $this->assertEquals(Result::ACK, $result); + } + + /** + * @return \PHPUnit_Framework_MockObject_MockObject|CacheManager + */ + private function createCacheManagerMock() + { + return $this->createMock('Liip\ImagineBundle\Imagine\Cache\CacheManager', array(), array(), '', false); + } + + /** + * @return \PHPUnit_Framework_MockObject_MockObject|FilterManager + */ + private function createFilterManagerMock() + { + return $this->createMock('Liip\ImagineBundle\Imagine\Filter\FilterManager', array(), array(), '', false); + } + + /** + * @return \PHPUnit_Framework_MockObject_MockObject|DataManager + */ + private function createDataManagerMock() + { + return $this->createMock('Liip\ImagineBundle\Imagine\Data\DataManager', array(), array(), '', false); + } + + /** + * @return \PHPUnit_Framework_MockObject_MockObject|ProducerInterface + */ + private function createProducerMock() + { + return $this->createMock('Enqueue\Client\ProducerInterface'); + } + + /** + * @return Binary + */ + private function createDummyBinary() + { + return new Binary('theContent', 'image/png', 'png'); + } +} \ No newline at end of file diff --git a/Tests/Async/ResolveCacheTest.php b/Tests/Async/ResolveCacheTest.php new file mode 100644 index 000000000..e58f914d1 --- /dev/null +++ b/Tests/Async/ResolveCacheTest.php @@ -0,0 +1,87 @@ +assertEquals('{"path":"thePath","filters":null,"force":false}', json_encode($message)); + } + + public function testCouldBeJsonSerializedWithFilters() + { + $message = new ResolveCache('thePath', array('fooFilter', 'barFilter')); + + $this->assertEquals('{"path":"thePath","filters":["fooFilter","barFilter"],"force":false}', json_encode($message)); + } + + public function testCouldBeJsonSerializedWithFiltersAndForce() + { + $message = new ResolveCache('thePath', array('fooFilter', 'barFilter'), true); + + $this->assertEquals('{"path":"thePath","filters":["fooFilter","barFilter"],"force":true}', json_encode($message)); + } + + public function testCouldBeJsonDeSerializedWithoutFiltersAndForce() + { + $message = ResolveCache::jsonDeserialize('{"path":"thePath","filters":null,"force":false}'); + + $this->assertInstanceOf('Liip\ImagineBundle\Async\ResolveCache', $message); + $this->assertEquals('thePath', $message->getPath()); + $this->assertNull($message->getFilters()); + $this->assertFalse($message->isForce()); + } + + public function testCouldBeJsonDeSerializedWithFilters() + { + $message = ResolveCache::jsonDeserialize('{"path":"thePath","filters":["fooFilter","barFilter"],"force":false}'); + + $this->assertInstanceOf('Liip\ImagineBundle\Async\ResolveCache', $message); + $this->assertEquals('thePath', $message->getPath()); + $this->assertEquals(array('fooFilter', 'barFilter'), $message->getFilters()); + $this->assertFalse($message->isForce()); + } + + public function testCouldBeJsonDeSerializedWithFiltersAndForce() + { + $message = ResolveCache::jsonDeserialize('{"path":"thePath","filters":["fooFilter","barFilter"],"force":true}'); + + $this->assertInstanceOf('Liip\ImagineBundle\Async\ResolveCache', $message); + $this->assertEquals('thePath', $message->getPath()); + $this->assertEquals(array('fooFilter', 'barFilter'), $message->getFilters()); + $this->assertTrue($message->isForce()); + } + + public function testCouldBeJsonDeSerializedWithOnlyPath() + { + $message = ResolveCache::jsonDeserialize('{"path":"thePath"}'); + + $this->assertInstanceOf('Liip\ImagineBundle\Async\ResolveCache', $message); + $this->assertEquals('thePath', $message->getPath()); + $this->assertNull($message->getFilters()); + $this->assertFalse($message->isForce()); + } + + public function testThrowIfMessageMissingPathOnJsonDeserialize() + { + $this->setExpectedException('LogicException', 'The message does not contain "path" but it is required.'); + ResolveCache::jsonDeserialize('{}'); + } + + public function testThrowIfMessageContainsNotSupportedFilters() + { + $this->setExpectedException('LogicException', 'The message filters could be either null or array.'); + ResolveCache::jsonDeserialize('{"path": "aPath", "filters": "stringFilterIsNotAllowed"}'); + } +} \ No newline at end of file diff --git a/composer.json b/composer.json index 79e1f97a7..0f1c57cf9 100644 --- a/composer.json +++ b/composer.json @@ -62,7 +62,8 @@ "doctrine/mongodb-odm": "required to use mongodb-backed doctrine components", "league/flysystem": "required to use FlySystem data loader or cache resolver", "monolog/monolog": "A psr/log compatible logger is required to enable logging", - "twig/twig": "required to use the provided Twig extension. Version 1.12 or greater needed" + "twig/twig": "required to use the provided Twig extension. Version 1.12 or greater needed", + "enqueue/enqueue-bundle": "add if you like to process images in background" }, "minimum-stability": "dev", "config": {