From 8a2fc86ac9d83457193c40cbc006f5799495e2b0 Mon Sep 17 00:00:00 2001 From: Jim Schmid Date: Tue, 8 Oct 2013 20:31:29 +0200 Subject: [PATCH 01/47] Removed default value 'fineuploader' for frontend key in Configuration. This was a legacy decision which will be changed in 1.0.0. The first version of this Bundle only supported Fineuploader which would have been a BC break if 'fineuploader' was not the default value. --- DependencyInjection/Configuration.php | 1 - 1 file changed, 1 deletion(-) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index e18268a4..97b1eef5 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -38,7 +38,6 @@ public function getConfigTreeBuilder() ->children() ->enumNode('frontend') ->values(array('fineuploader', 'blueimp', 'uploadify', 'yui3', 'fancyupload', 'mooupload', 'plupload', 'dropzone', 'custom')) - ->defaultValue('fineuploader') ->end() ->arrayNode('custom_frontend') ->addDefaultsIfNotSet() From d64ade8872f7a3e06a7c94b9620a398ca766974b Mon Sep 17 00:00:00 2001 From: Jim Schmid Date: Tue, 8 Oct 2013 21:03:16 +0200 Subject: [PATCH 02/47] Removed a notice which was stating that there is a default value for fronend config. --- Resources/doc/frontend_fineuploader.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/Resources/doc/frontend_fineuploader.md b/Resources/doc/frontend_fineuploader.md index b0248573..50ea10d8 100644 --- a/Resources/doc/frontend_fineuploader.md +++ b/Resources/doc/frontend_fineuploader.md @@ -32,6 +32,4 @@ oneup_uploader: frontend: fineuploader ``` -> Actually, `fineuploader` is the default value, so you don't have to provide it manually. - Be sure to check out the [official manual](https://github.com/Widen/fine-uploader/blob/master/readme.md) for details on the configuration. From 73e39c5eee56d90370af664eb4d118d1ffef0e01 Mon Sep 17 00:00:00 2001 From: Jim Schmid Date: Tue, 8 Oct 2013 21:23:55 +0200 Subject: [PATCH 03/47] Made request passing in ValidationEvent mandatory. --- Controller/AbstractController.php | 2 +- Event/ValidationEvent.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Controller/AbstractController.php b/Controller/AbstractController.php index 30b05197..04397d2d 100644 --- a/Controller/AbstractController.php +++ b/Controller/AbstractController.php @@ -164,7 +164,7 @@ protected function dispatchPostEvents($uploaded, ResponseInterface $response, Re protected function validate(UploadedFile $file) { $dispatcher = $this->container->get('event_dispatcher'); - $event = new ValidationEvent($file, $this->config, $this->type, $this->container->get('request')); + $event = new ValidationEvent($file, $this->container->get('request'), $this->config, $this->type); $dispatcher->dispatch(UploadEvents::VALIDATION, $event); } diff --git a/Event/ValidationEvent.php b/Event/ValidationEvent.php index 0fa90b0f..8110b78c 100644 --- a/Event/ValidationEvent.php +++ b/Event/ValidationEvent.php @@ -13,7 +13,7 @@ class ValidationEvent extends Event protected $type; protected $request; - public function __construct(UploadedFile $file, array $config, $type, Request $request = null) + public function __construct(UploadedFile $file, Request $request, array $config, $type) { $this->file = $file; $this->config = $config; From 3cee34e31747db8f6d4f541c9d422d2fbf6fc3a5 Mon Sep 17 00:00:00 2001 From: Jim Schmid Date: Tue, 8 Oct 2013 21:30:36 +0200 Subject: [PATCH 04/47] Fixed "unused code" labels in Scrutinizer report. --- Controller/BlueimpController.php | 3 +-- Controller/DropzoneController.php | 2 +- Controller/FancyUploadController.php | 2 +- Controller/MooUploadController.php | 3 --- Controller/UploadifyController.php | 2 +- Controller/YUI3Controller.php | 2 +- Tests/Controller/AbstractControllerTest.php | 2 +- Uploader/Storage/FilesystemStorage.php | 3 --- Uploader/Storage/GaufretteStorage.php | 2 +- Uploader/Storage/OrphanageStorage.php | 3 --- 10 files changed, 7 insertions(+), 17 deletions(-) diff --git a/Controller/BlueimpController.php b/Controller/BlueimpController.php index f10f776c..64e1b880 100644 --- a/Controller/BlueimpController.php +++ b/Controller/BlueimpController.php @@ -61,7 +61,7 @@ protected function parseChunkedRequest(Request $request) $attachmentName = rawurldecode(preg_replace('/(^[^"]+")|("$)/', '', $request->headers->get('content-disposition'))); // split the header string to the appropriate parts - list($tmp, $startByte, $endByte, $totalBytes) = preg_split('/[^0-9]+/', $headerRange); + list(, $startByte, $endByte, $totalBytes) = preg_split('/[^0-9]+/', $headerRange); // getting information about chunks // note: We don't have a chance to get the last $total @@ -73,7 +73,6 @@ protected function parseChunkedRequest(Request $request) $size = ($endByte + 1 - $startByte); $last = ($endByte + 1) == $totalBytes; $index = $last ? \PHP_INT_MAX : floor($startByte / $size); - $total = ceil($totalBytes / $size); // it is possible, that two clients send a file with the // exact same filename, therefore we have to add the session diff --git a/Controller/DropzoneController.php b/Controller/DropzoneController.php index 17d6ba95..b7b6e946 100644 --- a/Controller/DropzoneController.php +++ b/Controller/DropzoneController.php @@ -18,7 +18,7 @@ public function upload() foreach ($files as $file) { try { - $uploaded = $this->handleUpload($file, $response, $request); + $this->handleUpload($file, $response, $request); } catch (UploadException $e) { $this->errorHandler->addException($response, $e); } diff --git a/Controller/FancyUploadController.php b/Controller/FancyUploadController.php index 8901b2ed..e36979e3 100644 --- a/Controller/FancyUploadController.php +++ b/Controller/FancyUploadController.php @@ -18,7 +18,7 @@ public function upload() foreach ($files as $file) { try { - $uploaded = $this->handleUpload($file, $response, $request); + $this->handleUpload($file, $response, $request); } catch (UploadException $e) { $this->errorHandler->addException($response, $e); } diff --git a/Controller/MooUploadController.php b/Controller/MooUploadController.php index 8b6cce5e..a33ee432 100644 --- a/Controller/MooUploadController.php +++ b/Controller/MooUploadController.php @@ -17,9 +17,6 @@ class MooUploadController extends AbstractChunkedController public function upload() { $request = $this->container->get('request'); - $dispatcher = $this->container->get('event_dispatcher'); - $translator = $this->container->get('translator'); - $response = new MooUploadResponse(); $headers = $request->headers; diff --git a/Controller/UploadifyController.php b/Controller/UploadifyController.php index a98f7e8a..8b8fcf37 100644 --- a/Controller/UploadifyController.php +++ b/Controller/UploadifyController.php @@ -18,7 +18,7 @@ public function upload() foreach ($files as $file) { try { - $uploaded = $this->handleUpload($file, $response, $request); + $this->handleUpload($file, $response, $request); } catch (UploadException $e) { $this->errorHandler->addException($response, $e); } diff --git a/Controller/YUI3Controller.php b/Controller/YUI3Controller.php index 8638ad61..05d3bcb9 100644 --- a/Controller/YUI3Controller.php +++ b/Controller/YUI3Controller.php @@ -18,7 +18,7 @@ public function upload() foreach ($files as $file) { try { - $uploaded = $this->handleUpload($file, $response, $request); + $this->handleUpload($file, $response, $request); } catch (UploadException $e) { $this->errorHandler->addException($response, $e); } diff --git a/Tests/Controller/AbstractControllerTest.php b/Tests/Controller/AbstractControllerTest.php index 8909345a..c584b7fb 100644 --- a/Tests/Controller/AbstractControllerTest.php +++ b/Tests/Controller/AbstractControllerTest.php @@ -18,7 +18,7 @@ public function setUp() $this->helper = $this->container->get('oneup_uploader.templating.uploader_helper'); $this->createdFiles = array(); - $routes = $this->container->get('router')->getRouteCollection()->all(); + $this->container->get('router')->getRouteCollection()->all(); } abstract protected function getConfigKey(); diff --git a/Uploader/Storage/FilesystemStorage.php b/Uploader/Storage/FilesystemStorage.php index ed7931ec..d6915e04 100644 --- a/Uploader/Storage/FilesystemStorage.php +++ b/Uploader/Storage/FilesystemStorage.php @@ -2,7 +2,6 @@ namespace Oneup\UploaderBundle\Uploader\Storage; -use Symfony\Component\Filesystem\Filesystem; use Symfony\Component\HttpFoundation\File\File; use Oneup\UploaderBundle\Uploader\Storage\StorageInterface; @@ -18,8 +17,6 @@ public function __construct($directory) public function upload(File $file, $name, $path = null) { - $filesystem = new Filesystem(); - $path = is_null($path) ? $name : sprintf('%s/%s', $path, $name); $path = sprintf('%s/%s', $this->directory, $path); diff --git a/Uploader/Storage/GaufretteStorage.php b/Uploader/Storage/GaufretteStorage.php index 2fbd3491..f2229d0e 100644 --- a/Uploader/Storage/GaufretteStorage.php +++ b/Uploader/Storage/GaufretteStorage.php @@ -43,7 +43,7 @@ public function upload(File $file, $name, $path = null) while (!$src->eof()) { $data = $src->read($this->bufferSize); - $written = $dst->write($data); + $dst->write($data); } $dst->close(); diff --git a/Uploader/Storage/OrphanageStorage.php b/Uploader/Storage/OrphanageStorage.php index d0787d0e..fcbbaf06 100644 --- a/Uploader/Storage/OrphanageStorage.php +++ b/Uploader/Storage/OrphanageStorage.php @@ -5,7 +5,6 @@ use Symfony\Component\HttpFoundation\Session\SessionInterface; use Symfony\Component\HttpFoundation\File\File; use Symfony\Component\Finder\Finder; -use Symfony\Component\Filesystem\Filesystem; use Oneup\UploaderBundle\Uploader\Storage\FilesystemStorage; use Oneup\UploaderBundle\Uploader\Storage\StorageInterface; @@ -38,8 +37,6 @@ public function upload(File $file, $name, $path = null) public function uploadFiles() { - $filesystem = new Filesystem(); - try { $files = $this->getFiles(); $return = array(); From 4b7e0aab49b5cfe268d47f760ead43a5b71fe251 Mon Sep 17 00:00:00 2001 From: Jim Schmid Date: Tue, 8 Oct 2013 22:46:59 +0200 Subject: [PATCH 05/47] Added Next Steps section to frontend documentations. --- Resources/doc/frontend_blueimp.md | 12 +++++++++++- Resources/doc/frontend_dropzone.md | 12 +++++++++++- Resources/doc/frontend_fancyupload.md | 16 +++++++++++++--- Resources/doc/frontend_fineuploader.md | 10 ++++++++++ Resources/doc/frontend_mooupload.md | 12 +++++++++++- Resources/doc/frontend_plupload.md | 12 +++++++++++- Resources/doc/frontend_uploadify.md | 14 ++++++++++++-- 7 files changed, 79 insertions(+), 9 deletions(-) diff --git a/Resources/doc/frontend_blueimp.md b/Resources/doc/frontend_blueimp.md index f6935c25..db30fdf7 100644 --- a/Resources/doc/frontend_blueimp.md +++ b/Resources/doc/frontend_blueimp.md @@ -41,4 +41,14 @@ security: main: pattern: ^/ anonymous: true -``` \ No newline at end of file +``` + +Next steps +---------- + +After this setup, you can move on and implement some of the more advanced features. A full list is available [here](https://github.com/1up-lab/OneupUploaderBundle/blob/master/Resources/doc/index.md#next-steps). + +* [Process uploaded files using custom logic](custom_logic.md) +* [Return custom data to frontend](response.md) +* [Include your own Namer](custom_namer.md) +* [Configuration Reference](configuration_reference.md) diff --git a/Resources/doc/frontend_dropzone.md b/Resources/doc/frontend_dropzone.md index 0be52b31..66fc3188 100644 --- a/Resources/doc/frontend_dropzone.md +++ b/Resources/doc/frontend_dropzone.md @@ -21,4 +21,14 @@ oneup_uploader: frontend: dropzone ``` -Be sure to check out the [official manual](http://www.dropzonejs.com/) for details on the configuration. \ No newline at end of file +Be sure to check out the [official manual](http://www.dropzonejs.com/) for details on the configuration. + +Next steps +---------- + +After this setup, you can move on and implement some of the more advanced features. A full list is available [here](https://github.com/1up-lab/OneupUploaderBundle/blob/master/Resources/doc/index.md#next-steps). + +* [Process uploaded files using custom logic](custom_logic.md) +* [Return custom data to frontend](response.md) +* [Include your own Namer](custom_namer.md) +* [Configuration Reference](configuration_reference.md) diff --git a/Resources/doc/frontend_fancyupload.md b/Resources/doc/frontend_fancyupload.md index 02bef21b..09896f0e 100644 --- a/Resources/doc/frontend_fancyupload.md +++ b/Resources/doc/frontend_fancyupload.md @@ -28,13 +28,13 @@ window.addEvent('domready', function() debug: true, target: 'demo-browse' }); - + $('demo-browse').addEvent('click', function() { swiffy.browse(); return false; }); - + $('demo-select-images').addEvent('change', function() { var filter = null; @@ -71,7 +71,7 @@ window.addEvent('domready', function() - +

Browse Files | @@ -106,3 +106,13 @@ oneup_uploader: ``` Be sure to check out the [official manual](http://digitarald.de/project/fancyupload/) for details on the configuration. + +Next steps +---------- + +After this setup, you can move on and implement some of the more advanced features. A full list is available [here](https://github.com/1up-lab/OneupUploaderBundle/blob/master/Resources/doc/index.md#next-steps). + +* [Process uploaded files using custom logic](custom_logic.md) +* [Return custom data to frontend](response.md) +* [Include your own Namer](custom_namer.md) +* [Configuration Reference](configuration_reference.md) diff --git a/Resources/doc/frontend_fineuploader.md b/Resources/doc/frontend_fineuploader.md index 50ea10d8..cd8a0198 100644 --- a/Resources/doc/frontend_fineuploader.md +++ b/Resources/doc/frontend_fineuploader.md @@ -33,3 +33,13 @@ oneup_uploader: ``` Be sure to check out the [official manual](https://github.com/Widen/fine-uploader/blob/master/readme.md) for details on the configuration. + +Next steps +---------- + +After this setup, you can move on and implement some of the more advanced features. A full list is available [here](https://github.com/1up-lab/OneupUploaderBundle/blob/master/Resources/doc/index.md#next-steps). + +* [Process uploaded files using custom logic](custom_logic.md) +* [Return custom data to frontend](response.md) +* [Include your own Namer](custom_namer.md) +* [Configuration Reference](configuration_reference.md) diff --git a/Resources/doc/frontend_mooupload.md b/Resources/doc/frontend_mooupload.md index 5c318680..6a1db2c8 100644 --- a/Resources/doc/frontend_mooupload.md +++ b/Resources/doc/frontend_mooupload.md @@ -32,4 +32,14 @@ oneup_uploader: frontend: mooupload ``` -Be sure to check out the [official manual](https://github.com/juanparati/MooUpload) for details on the configuration. \ No newline at end of file +Be sure to check out the [official manual](https://github.com/juanparati/MooUpload) for details on the configuration. + +Next steps +---------- + +After this setup, you can move on and implement some of the more advanced features. A full list is available [here](https://github.com/1up-lab/OneupUploaderBundle/blob/master/Resources/doc/index.md#next-steps). + +* [Process uploaded files using custom logic](custom_logic.md) +* [Return custom data to frontend](response.md) +* [Include your own Namer](custom_namer.md) +* [Configuration Reference](configuration_reference.md) diff --git a/Resources/doc/frontend_plupload.md b/Resources/doc/frontend_plupload.md index 34c1dc54..deb1ce0c 100644 --- a/Resources/doc/frontend_plupload.md +++ b/Resources/doc/frontend_plupload.md @@ -47,4 +47,14 @@ security: main: pattern: ^/ anonymous: true -``` \ No newline at end of file +``` + +Next steps +---------- + +After this setup, you can move on and implement some of the more advanced features. A full list is available [here](https://github.com/1up-lab/OneupUploaderBundle/blob/master/Resources/doc/index.md#next-steps). + +* [Process uploaded files using custom logic](custom_logic.md) +* [Return custom data to frontend](response.md) +* [Include your own Namer](custom_namer.md) +* [Configuration Reference](configuration_reference.md) diff --git a/Resources/doc/frontend_uploadify.md b/Resources/doc/frontend_uploadify.md index 072f22ca..ec7bcc99 100644 --- a/Resources/doc/frontend_uploadify.md +++ b/Resources/doc/frontend_uploadify.md @@ -16,7 +16,7 @@ $(document).ready(function() swf: "{{ asset('bundles/acmedemo/js/uploadify.swf') }}", uploader: "{{ oneup_uploader_endpoint('gallery') }}" }); - + }); @@ -34,4 +34,14 @@ oneup_uploader: frontend: uploadify ``` -Be sure to check out the [official manual](http://www.uploadify.com/documentation/) for details on the configuration. \ No newline at end of file +Be sure to check out the [official manual](http://www.uploadify.com/documentation/) for details on the configuration. + +Next steps +---------- + +After this setup, you can move on and implement some of the more advanced features. A full list is available [here](https://github.com/1up-lab/OneupUploaderBundle/blob/master/Resources/doc/index.md#next-steps). + +* [Process uploaded files using custom logic](custom_logic.md) +* [Return custom data to frontend](response.md) +* [Include your own Namer](custom_namer.md) +* [Configuration Reference](configuration_reference.md) From ad5577f64f954377521b8724ce7de3f7b4682987 Mon Sep 17 00:00:00 2001 From: mitom Date: Wed, 9 Oct 2013 21:48:18 +0200 Subject: [PATCH 06/47] gaufrette chunk storage --- Controller/AbstractChunkedController.php | 3 +- Controller/AbstractController.php | 7 +- DependencyInjection/Configuration.php | 13 +- .../OneupUploaderExtension.php | 154 ++++++++++-------- Event/ValidationEvent.php | 4 +- .../AllowedExtensionValidationListener.php | 2 +- .../AllowedMimetypeValidationListener.php | 3 +- .../DisallowedExtensionValidationListener.php | 2 +- .../DisallowedMimetypeValidationListener.php | 3 +- Resources/config/uploader.xml | 1 + Uploader/Chunk/ChunkManager.php | 87 ++-------- Uploader/Chunk/ChunkManagerInterface.php | 4 +- Uploader/File/FileInterface.php | 50 ++++++ Uploader/File/FilesystemFile.php | 15 ++ Uploader/File/GaufretteFile.php | 62 +++++++ Uploader/Naming/NamerInterface.php | 7 +- Uploader/Naming/UniqidNamer.php | 8 +- Uploader/Storage/ChunkStorageInterface.php | 19 +++ Uploader/Storage/FilesystemStorage.php | 117 ++++++++++++- Uploader/Storage/GaufretteStorage.php | 139 ++++++++++++++-- Uploader/Storage/OrphanageStorage.php | 3 +- Uploader/Storage/StorageInterface.php | 3 +- 22 files changed, 527 insertions(+), 179 deletions(-) create mode 100644 Uploader/File/FileInterface.php create mode 100644 Uploader/File/FilesystemFile.php create mode 100644 Uploader/File/GaufretteFile.php create mode 100644 Uploader/Storage/ChunkStorageInterface.php diff --git a/Controller/AbstractChunkedController.php b/Controller/AbstractChunkedController.php index 7f930d4f..2678a664 100644 --- a/Controller/AbstractChunkedController.php +++ b/Controller/AbstractChunkedController.php @@ -74,8 +74,7 @@ protected function handleChunkedUpload(UploadedFile $file, ResponseInterface $re $path = $assembled->getPath(); // create a temporary uploaded file to meet the interface restrictions - $uploadedFile = new UploadedFile($assembled->getPathname(), $assembled->getBasename(), null, $assembled->getSize(), null, true); - $uploaded = $this->handleUpload($uploadedFile, $response, $request); + $this->handleUpload($assembled, $response, $request); $chunkManager->cleanup($path); } diff --git a/Controller/AbstractController.php b/Controller/AbstractController.php index 04397d2d..aea039e4 100644 --- a/Controller/AbstractController.php +++ b/Controller/AbstractController.php @@ -2,6 +2,7 @@ namespace Oneup\UploaderBundle\Controller; +use Oneup\UploaderBundle\Uploader\File\FileInterface; use Symfony\Component\HttpFoundation\JsonResponse; use Symfony\Component\DependencyInjection\ContainerInterface; use Symfony\Component\HttpFoundation\File\UploadedFile; @@ -103,7 +104,7 @@ protected function getFiles(FileBag $bag) * @param response A response object. * @param request The request object. */ - protected function handleUpload(UploadedFile $file, ResponseInterface $response, Request $request) + protected function handleUpload(FileInterface $file, ResponseInterface $response, Request $request) { $this->validate($file); @@ -126,7 +127,7 @@ protected function handleUpload(UploadedFile $file, ResponseInterface $response, * @param response A response object. * @param request The request object. */ - protected function dispatchPreUploadEvent(UploadedFile $uploaded, ResponseInterface $response, Request $request) + protected function dispatchPreUploadEvent(FileInterface $uploaded, ResponseInterface $response, Request $request) { $dispatcher = $this->container->get('event_dispatcher'); @@ -161,7 +162,7 @@ protected function dispatchPostEvents($uploaded, ResponseInterface $response, Re } } - protected function validate(UploadedFile $file) + protected function validate(FileInterface $file) { $dispatcher = $this->container->get('event_dispatcher'); $event = new ValidationEvent($file, $this->container->get('request'), $this->config, $this->type); diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index 97b1eef5..ab37255b 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -18,7 +18,18 @@ public function getConfigTreeBuilder() ->addDefaultsIfNotSet() ->children() ->scalarNode('maxage')->defaultValue(604800)->end() - ->scalarNode('directory')->defaultNull()->end() + ->arrayNode('storage') + ->addDefaultsIfNotSet() + ->children() + ->enumNode('type') + ->values(array('filesystem', 'gaufrette')) + ->defaultValue('filesystem') + ->end() + ->scalarNode('filesystem')->defaultNull()->end() + ->scalarNode('directory')->defaultNull()->end() + ->scalarNode('sync_buffer_size')->defaultValue('100K')->end() + ->end() + ->end() ->booleanNode('load_distribution')->defaultTrue()->end() ->end() ->end() diff --git a/DependencyInjection/OneupUploaderExtension.php b/DependencyInjection/OneupUploaderExtension.php index fae90564..72445faf 100644 --- a/DependencyInjection/OneupUploaderExtension.php +++ b/DependencyInjection/OneupUploaderExtension.php @@ -13,11 +13,14 @@ class OneupUploaderExtension extends Extension { protected $storageServices = array(); + protected $container; + protected $config; public function load(array $configs, ContainerBuilder $container) { $configuration = new Configuration(); - $config = $this->processConfiguration($configuration, $configs); + $this->config = $this->processConfiguration($configuration, $configs); + $this->container = $container; $loader = new Loader\XmlFileLoader($container, new FileLocator(__DIR__.'/../Resources/config')); $loader->load('uploader.xml'); @@ -25,88 +28,45 @@ public function load(array $configs, ContainerBuilder $container) $loader->load('validators.xml'); $loader->load('errorhandler.xml'); - if ($config['twig']) { + if ($this->config['twig']) { $loader->load('twig.xml'); } - $config['chunks']['directory'] = is_null($config['chunks']['directory']) ? - sprintf('%s/uploader/chunks', $container->getParameter('kernel.cache_dir')) : - $this->normalizePath($config['chunks']['directory']) - ; + if ($this->config['chunks']['storage']['type'] === 'filesystem' && + !isset($this->config['chunks']['storage']['directory'])) { + $this->config['chunks']['storage']['directory'] = sprintf('%s/uploader/chunks', $container->getParameter('kernel.cache_dir')); + } elseif ($this->config['chunks']['storage']['type'] === 'gaufrette' ) { + // Force load distribution when using gaufrette chunk storage + $this->config['chunks']['load_distribution'] = true; + } + + // register the service for the chunk storage + $this->createStorageService($this->config['chunks']['storage'], 'chunk'); - $config['orphanage']['directory'] = is_null($config['orphanage']['directory']) ? + + $this->config['orphanage']['directory'] = is_null($this->config['orphanage']['directory']) ? sprintf('%s/uploader/orphanage', $container->getParameter('kernel.cache_dir')) : - $this->normalizePath($config['orphanage']['directory']) + $this->normalizePath($this->config['orphanage']['directory']) ; - $container->setParameter('oneup_uploader.chunks', $config['chunks']); - $container->setParameter('oneup_uploader.orphanage', $config['orphanage']); + $container->setParameter('oneup_uploader.chunks', $this->config['chunks']); + $container->setParameter('oneup_uploader.orphanage', $this->config['orphanage']); $controllers = array(); // handle mappings - foreach ($config['mappings'] as $key => $mapping) { + foreach ($this->config['mappings'] as $key => $mapping) { + if ($key === 'chunk') { + throw new InvalidArgumentException('"chunk" is a protected mapping name, please use a different one.'); + } + $mapping['max_size'] = $mapping['max_size'] < 0 ? $this->getMaxUploadSize($mapping['max_size']) : $mapping['max_size'] ; // create the storage service according to the configuration - $storageService = null; - - // if a service is given, return a reference to this service - // this allows a user to overwrite the storage layer if needed - if (!is_null($mapping['storage']['service'])) { - $storageService = new Reference($mapping['storage']['service']); - } else { - // no service was given, so we create one - $storageName = sprintf('oneup_uploader.storage.%s', $key); - - if ($mapping['storage']['type'] == 'filesystem') { - $mapping['storage']['directory'] = is_null($mapping['storage']['directory']) ? - sprintf('%s/../web/uploads/%s', $container->getParameter('kernel.root_dir'), $key) : - $this->normalizePath($mapping['storage']['directory']) - ; - - $container - ->register($storageName, sprintf('%%oneup_uploader.storage.%s.class%%', $mapping['storage']['type'])) - ->addArgument($mapping['storage']['directory']) - ; - } - - if ($mapping['storage']['type'] == 'gaufrette') { - if(!class_exists('Gaufrette\\Filesystem')) - throw new InvalidArgumentException('You have to install Gaufrette in order to use it as a storage service.'); - - if(strlen($mapping['storage']['filesystem']) <= 0) - throw new ServiceNotFoundException('Empty service name'); - - $container - ->register($storageName, sprintf('%%oneup_uploader.storage.%s.class%%', $mapping['storage']['type'])) - ->addArgument(new Reference($mapping['storage']['filesystem'])) - ->addArgument($this->getValueInBytes($mapping['storage']['sync_buffer_size'])) - ; - } - - $storageService = new Reference($storageName); - - if ($mapping['use_orphanage']) { - $orphanageName = sprintf('oneup_uploader.orphanage.%s', $key); - - // this mapping wants to use the orphanage, so create - // a masked filesystem for the controller - $container - ->register($orphanageName, '%oneup_uploader.orphanage.class%') - ->addArgument($storageService) - ->addArgument(new Reference('session')) - ->addArgument($config['orphanage']) - ->addArgument($key) - ; - - // switch storage of mapping to orphanage - $storageService = new Reference($orphanageName); - } - } + $storageService = $this->createStorageService($mapping['storage'], $key, isset($mapping['orphanage']) ? :null); if ($mapping['frontend'] != 'custom') { $controllerName = sprintf('oneup_uploader.controller.%s', $key); @@ -154,6 +114,68 @@ public function load(array $configs, ContainerBuilder $container) $container->setParameter('oneup_uploader.controllers', $controllers); } + protected function createStorageService($storage, $key, $orphanage = null) + { + $storageService = null; + + // if a service is given, return a reference to this service + // this allows a user to overwrite the storage layer if needed + if (isset($storage['service']) && !is_null($storage['service'])) { + $storageService = new Reference($storage['storage']['service']); + } else { + // no service was given, so we create one + $storageName = sprintf('oneup_uploader.storage.%s', $key); + + if ($storage['type'] == 'filesystem') { + $storage['directory'] = is_null($storage['directory']) ? + sprintf('%s/../web/uploads/%s', $this->container->getParameter('kernel.root_dir'), $key) : + $this->normalizePath($storage['directory']) + ; + + $this->container + ->register($storageName, sprintf('%%oneup_uploader.storage.%s.class%%', $storage['type'])) + ->addArgument($storage['directory']) + ; + } + + if ($storage['type'] == 'gaufrette') { + if(!class_exists('Gaufrette\\Filesystem')) + throw new InvalidArgumentException('You have to install Gaufrette in order to use it as a storage service.'); + + if(strlen($storage['filesystem']) <= 0) + throw new ServiceNotFoundException('Empty service name'); + + $this->container + ->register($storageName, sprintf('%%oneup_uploader.storage.%s.class%%', $storage['type'])) + ->addArgument(new Reference($storage['filesystem'])) + ->addArgument($this->getValueInBytes($storage['sync_buffer_size'])) + ; + } + + $storageService = new Reference($storageName); + + if ($orphanage) { + $orphanageName = sprintf('oneup_uploader.orphanage.%s', $key); + + // this mapping wants to use the orphanage, so create + // a masked filesystem for the controller + $this->container + ->register($orphanageName, '%oneup_uploader.orphanage.class%') + ->addArgument($storageService) + ->addArgument(new Reference('session')) + ->addArgument($this->config['orphanage']) + ->addArgument($key) + ; + + // switch storage of mapping to orphanage + $storageService = new Reference($orphanageName); + } + } + + return $storageService; + } + + protected function getMaxUploadSize($input) { $input = $this->getValueInBytes($input); diff --git a/Event/ValidationEvent.php b/Event/ValidationEvent.php index 8110b78c..15e1c18a 100644 --- a/Event/ValidationEvent.php +++ b/Event/ValidationEvent.php @@ -2,8 +2,8 @@ namespace Oneup\UploaderBundle\Event; +use Oneup\UploaderBundle\Uploader\File\FileInterface; use Symfony\Component\EventDispatcher\Event; -use Symfony\Component\HttpFoundation\File\UploadedFile; use Symfony\Component\HttpFoundation\Request; class ValidationEvent extends Event @@ -13,7 +13,7 @@ class ValidationEvent extends Event protected $type; protected $request; - public function __construct(UploadedFile $file, Request $request, array $config, $type) + public function __construct(FileInterface $file, Request $request, array $config, $type) { $this->file = $file; $this->config = $config; diff --git a/EventListener/AllowedExtensionValidationListener.php b/EventListener/AllowedExtensionValidationListener.php index 68c26e85..add50ca2 100644 --- a/EventListener/AllowedExtensionValidationListener.php +++ b/EventListener/AllowedExtensionValidationListener.php @@ -12,7 +12,7 @@ public function onValidate(ValidationEvent $event) $config = $event->getConfig(); $file = $event->getFile(); - $extension = pathinfo($file->getClientOriginalName(), PATHINFO_EXTENSION); + $extension = $file->getExtension(); if (count($config['allowed_extensions']) > 0 && !in_array($extension, $config['allowed_extensions'])) { throw new ValidationException('error.whitelist'); diff --git a/EventListener/AllowedMimetypeValidationListener.php b/EventListener/AllowedMimetypeValidationListener.php index 195f6bdc..66aa7b91 100644 --- a/EventListener/AllowedMimetypeValidationListener.php +++ b/EventListener/AllowedMimetypeValidationListener.php @@ -16,8 +16,7 @@ public function onValidate(ValidationEvent $event) return; } - $finfo = finfo_open(FILEINFO_MIME_TYPE); - $mimetype = finfo_file($finfo, $file->getRealpath()); + $mimetype = $file->getMimeType(); if (!in_array($mimetype, $config['allowed_mimetypes'])) { throw new ValidationException('error.whitelist'); diff --git a/EventListener/DisallowedExtensionValidationListener.php b/EventListener/DisallowedExtensionValidationListener.php index 9cdac58e..9f000de3 100644 --- a/EventListener/DisallowedExtensionValidationListener.php +++ b/EventListener/DisallowedExtensionValidationListener.php @@ -12,7 +12,7 @@ public function onValidate(ValidationEvent $event) $config = $event->getConfig(); $file = $event->getFile(); - $extension = pathinfo($file->getClientOriginalName(), PATHINFO_EXTENSION); + $extension = $file->getExtension(); if (count($config['disallowed_extensions']) > 0 && in_array($extension, $config['disallowed_extensions'])) { throw new ValidationException('error.blacklist'); diff --git a/EventListener/DisallowedMimetypeValidationListener.php b/EventListener/DisallowedMimetypeValidationListener.php index 452eff61..c5ace443 100644 --- a/EventListener/DisallowedMimetypeValidationListener.php +++ b/EventListener/DisallowedMimetypeValidationListener.php @@ -16,8 +16,7 @@ public function onValidate(ValidationEvent $event) return; } - $finfo = finfo_open(FILEINFO_MIME_TYPE); - $mimetype = finfo_file($finfo, $file->getRealpath()); + $mimetype = $file->getExtension(); if (in_array($mimetype, $config['disallowed_mimetypes'])) { throw new ValidationException('error.blacklist'); diff --git a/Resources/config/uploader.xml b/Resources/config/uploader.xml index 39502abd..3c953864 100644 --- a/Resources/config/uploader.xml +++ b/Resources/config/uploader.xml @@ -26,6 +26,7 @@ %oneup_uploader.chunks% + diff --git a/Uploader/Chunk/ChunkManager.php b/Uploader/Chunk/ChunkManager.php index 4f7733ba..39bd2d21 100644 --- a/Uploader/Chunk/ChunkManager.php +++ b/Uploader/Chunk/ChunkManager.php @@ -2,6 +2,7 @@ namespace Oneup\UploaderBundle\Uploader\Chunk; +use Oneup\UploaderBundle\Uploader\Storage\ChunkStorageInterface; use Symfony\Component\HttpFoundation\File\File; use Symfony\Component\HttpFoundation\File\UploadedFile; use Symfony\Component\Finder\Finder; @@ -11,102 +12,38 @@ class ChunkManager implements ChunkManagerInterface { - public function __construct($configuration) + protected $configuration; + protected $storage; + + public function __construct($configuration, ChunkStorageInterface $storage) { $this->configuration = $configuration; + $this->storage = $storage; } public function clear() { - $system = new Filesystem(); - $finder = new Finder(); - - try { - $finder->in($this->configuration['directory'])->date('<=' . -1 * (int) $this->configuration['maxage'] . 'seconds')->files(); - } catch (\InvalidArgumentException $e) { - // the finder will throw an exception of type InvalidArgumentException - // if the directory he should search in does not exist - // in that case we don't have anything to clean - return; - } - - foreach ($finder as $file) { - $system->remove($file); - } + $this->storage->clear($this->configuration['maxage']); } public function addChunk($uuid, $index, UploadedFile $chunk, $original) { - $filesystem = new Filesystem(); - $path = sprintf('%s/%s', $this->configuration['directory'], $uuid); - $name = sprintf('%s_%s', $index, $original); - - // create directory if it does not yet exist - if(!$filesystem->exists($path)) - $filesystem->mkdir(sprintf('%s/%s', $this->configuration['directory'], $uuid)); - - return $chunk->move($path, $name); + return $this->storage->addChunk($uuid, $index, $chunk, $original); } - public function assembleChunks(\IteratorAggregate $chunks, $removeChunk = true, $renameChunk = false) + public function assembleChunks($chunks, $removeChunk = true, $renameChunk = false) { - $iterator = $chunks->getIterator()->getInnerIterator(); - - $base = $iterator->current(); - $iterator->next(); - - while ($iterator->valid()) { - - $file = $iterator->current(); - - if (false === file_put_contents($base->getPathname(), file_get_contents($file->getPathname()), \FILE_APPEND | \LOCK_EX)) { - throw new \RuntimeException('Reassembling chunks failed.'); - } - - if ($removeChunk) { - $filesystem = new Filesystem(); - $filesystem->remove($file->getPathname()); - } - - $iterator->next(); - } - - $name = $base->getBasename(); - - if ($renameChunk) { - $name = preg_replace('/^(\d+)_/', '', $base->getBasename()); - } - - // remove the prefix added by self::addChunk - $assembled = new File($base->getRealPath()); - $assembled = $assembled->move($base->getPath(), $name); - - return $assembled; + return $this->storage->assembleChunks($chunks, $removeChunk, $renameChunk); } public function cleanup($path) { - // cleanup - $filesystem = new Filesystem(); - $filesystem->remove($path); - - return true; + return $this->storage->cleanup($path); } public function getChunks($uuid) { - $finder = new Finder(); - $finder - ->in(sprintf('%s/%s', $this->configuration['directory'], $uuid))->files()->sort(function(\SplFileInfo $a, \SplFileInfo $b) { - $t = explode('_', $a->getBasename()); - $s = explode('_', $b->getBasename()); - $t = (int) $t[0]; - $s = (int) $s[0]; - - return $s < $t; - }); - - return $finder; + return $this->storage->getChunks($uuid); } public function getLoadDistribution() diff --git a/Uploader/Chunk/ChunkManagerInterface.php b/Uploader/Chunk/ChunkManagerInterface.php index 0991a3e7..99ad60c9 100644 --- a/Uploader/Chunk/ChunkManagerInterface.php +++ b/Uploader/Chunk/ChunkManagerInterface.php @@ -21,13 +21,13 @@ public function addChunk($uuid, $index, UploadedFile $chunk, $original); /** * Assembles the given chunks and return the resulting file. * - * @param \IteratorAggregate $chunks + * @param $chunks * @param bool $removeChunk Remove the chunk file once its assembled. * @param bool $renameChunk Rename the chunk file once its assembled. * * @return File */ - public function assembleChunks(\IteratorAggregate $chunks, $removeChunk = true, $renameChunk = false); + public function assembleChunks($chunks, $removeChunk = true, $renameChunk = false); /** * Get chunks associated with the given uuid. diff --git a/Uploader/File/FileInterface.php b/Uploader/File/FileInterface.php new file mode 100644 index 00000000..47e42662 --- /dev/null +++ b/Uploader/File/FileInterface.php @@ -0,0 +1,50 @@ +file = $file; + parent::__construct($file->getPath(), $file->getClientOriginalName(), $file->getClientMimeType(), $file->getClientSize(), $file->getError(), true); + } +} \ No newline at end of file diff --git a/Uploader/File/GaufretteFile.php b/Uploader/File/GaufretteFile.php new file mode 100644 index 00000000..7771c4ab --- /dev/null +++ b/Uploader/File/GaufretteFile.php @@ -0,0 +1,62 @@ +getKey(), $filesystem); + $this->filesystem = $filesystem; + } + + /** + * Returns the size of the file + * + * !! WARNING !! + * Calling this loads the entire file into memory, + * in case of bigger files this could throw exceptions, + * and will have heavy performance footprint. + * !! ------- !! + * + * TODO mock/calculate the size if possible and use that instead? + */ + public function getSize() + { + return parent::getSize(); + } + + public function getPath() + { + return pathinfo($this->getKey(), PATHINFO_DIRNAME); + } + + public function getName() + { + return pathinfo($this->getKey(), PATHINFO_BASENAME); + } + + /** + * @return string + */ + public function getMimeType() + { + $finfo = finfo_open(FILEINFO_MIME_TYPE); + return finfo_file($finfo, $this->getKey()); + } + + public function getExtension() + { + return pathinfo($this->getKey(), PATHINFO_EXTENSION); + } + + public function getFilesystem() + { + return $this->filesystem; + } + +} \ No newline at end of file diff --git a/Uploader/Naming/NamerInterface.php b/Uploader/Naming/NamerInterface.php index d44b8177..173e0b8f 100644 --- a/Uploader/Naming/NamerInterface.php +++ b/Uploader/Naming/NamerInterface.php @@ -2,15 +2,16 @@ namespace Oneup\UploaderBundle\Uploader\Naming; -use Symfony\Component\HttpFoundation\File\UploadedFile; + +use Oneup\UploaderBundle\Uploader\File\FileInterface; interface NamerInterface { /** * Name a given file and return the name * - * @param UploadedFile $file + * @param FileInterface $file * @return string */ - public function name(UploadedFile $file); + public function name(FileInterface $file); } diff --git a/Uploader/Naming/UniqidNamer.php b/Uploader/Naming/UniqidNamer.php index 4c16b98f..80bc7411 100644 --- a/Uploader/Naming/UniqidNamer.php +++ b/Uploader/Naming/UniqidNamer.php @@ -2,13 +2,13 @@ namespace Oneup\UploaderBundle\Uploader\Naming; -use Symfony\Component\HttpFoundation\File\UploadedFile; -use Oneup\UploaderBundle\Uploader\Naming\NamerInterface; + +use Oneup\UploaderBundle\Uploader\File\FileInterface; class UniqidNamer implements NamerInterface { - public function name(UploadedFile $file) + public function name(FileInterface $file) { - return sprintf('%s.%s', uniqid(), $file->guessExtension()); + return sprintf('%s.%s', uniqid(), $file->getExtension()); } } diff --git a/Uploader/Storage/ChunkStorageInterface.php b/Uploader/Storage/ChunkStorageInterface.php new file mode 100644 index 00000000..b9fa1f63 --- /dev/null +++ b/Uploader/Storage/ChunkStorageInterface.php @@ -0,0 +1,19 @@ +directory = $directory; } - public function upload(File $file, $name, $path = null) + public function upload(FileInterface $file, $name, $path = null) { + if (!($file instanceof File)) { + throw new \InvalidArgumentException('file must be an instance of Symfony\Component\HttpFoundation\File\File'); + } + $path = is_null($path) ? $name : sprintf('%s/%s', $path, $name); $path = sprintf('%s/%s', $this->directory, $path); @@ -29,4 +37,107 @@ public function upload(File $file, $name, $path = null) return $file; } + + public function clear($maxAge) + { + $system = new Filesystem(); + $finder = new Finder(); + + try { + $finder->in($this->directory)->date('<=' . -1 * (int) $maxAge . 'seconds')->files(); + } catch (\InvalidArgumentException $e) { + // the finder will throw an exception of type InvalidArgumentException + // if the directory he should search in does not exist + // in that case we don't have anything to clean + return; + } + + foreach ($finder as $file) { + $system->remove($file); + } + } + + public function addChunk($uuid, $index, UploadedFile $chunk, $original) + { + $filesystem = new Filesystem(); + $path = sprintf('%s/%s', $this->directory, $uuid); + $name = sprintf('%s_%s', $index, $original); + + // create directory if it does not yet exist + if(!$filesystem->exists($path)) + $filesystem->mkdir(sprintf('%s/%s', $this->directory, $uuid)); + + return $chunk->move($path, $name); + } + + public function assembleChunks($chunks, $removeChunk, $renameChunk) + { + if (!($chunks instanceof \IteratorAggregate)) { + throw new \InvalidArgumentException('The first argument must implement \IteratorAggregate interface.'); + } + + $iterator = $chunks->getIterator()->getInnerIterator(); + + $base = $iterator->current(); + $iterator->next(); + + while ($iterator->valid()) { + + $file = $iterator->current(); + + if (false === file_put_contents($base->getPathname(), file_get_contents($file->getPathname()), \FILE_APPEND | \LOCK_EX)) { + throw new \RuntimeException('Reassembling chunks failed.'); + } + + if ($removeChunk) { + $filesystem = new Filesystem(); + $filesystem->remove($file->getPathname()); + } + + $iterator->next(); + } + + $name = $base->getBasename(); + + if ($renameChunk) { + // remove the prefix added by self::addChunk + $name = preg_replace('/^(\d+)_/', '', $base->getBasename()); + } + + $assembled = new File($base->getRealPath()); + $assembled = $assembled->move($base->getPath(), $name); + + // the file is only renamed before it is uploaded + if ($renameChunk) { + // create an file to meet interface restrictions + $assembled = new FilesystemFile(new UploadedFile($assembled->getPathname(), $assembled->getBasename(), null, $assembled->getSize(), null, true)); + } + + return $assembled; + } + + public function cleanup($path) + { + // cleanup + $filesystem = new Filesystem(); + $filesystem->remove($path); + + return true; + } + + public function getChunks($uuid) + { + $finder = new Finder(); + $finder + ->in(sprintf('%s/%s', $this->directory, $uuid))->files()->sort(function(\SplFileInfo $a, \SplFileInfo $b) { + $t = explode('_', $a->getBasename()); + $s = explode('_', $b->getBasename()); + $t = (int) $t[0]; + $s = (int) $s[0]; + + return $s < $t; + }); + + return $finder; + } } diff --git a/Uploader/Storage/GaufretteStorage.php b/Uploader/Storage/GaufretteStorage.php index f2229d0e..6c82693e 100644 --- a/Uploader/Storage/GaufretteStorage.php +++ b/Uploader/Storage/GaufretteStorage.php @@ -2,18 +2,21 @@ namespace Oneup\UploaderBundle\Uploader\Storage; -use Symfony\Component\HttpFoundation\File\File; +use Oneup\UploaderBundle\Uploader\File\FileInterface; +use Oneup\UploaderBundle\Uploader\File\GaufretteFile; use Gaufrette\Stream\Local as LocalStream; use Gaufrette\StreamMode; use Gaufrette\Filesystem; use Gaufrette\Adapter\MetadataSupporter; -use Oneup\UploaderBundle\Uploader\Storage\StorageInterface; +use Symfony\Component\HttpFoundation\File\UploadedFile; -class GaufretteStorage implements StorageInterface +class GaufretteStorage implements StorageInterface, ChunkStorageInterface { protected $filesystem; protected $bufferSize; + protected $unhandledChunk; + protected $chunkPrefix = 'chunks'; public function __construct(Filesystem $filesystem, $bufferSize) { @@ -21,25 +24,142 @@ public function __construct(Filesystem $filesystem, $bufferSize) $this->bufferSize = $bufferSize; } - public function upload(File $file, $name, $path = null) + public function upload(FileInterface $file, $name, $path = null) { $path = is_null($path) ? $name : sprintf('%s/%s', $path, $name); + if ($file instanceof GaufretteFile) { + if ($file->getFilesystem() == $this->filesystem) { + $file->getFilesystem()->rename($file->getKey(), $path); + + return $this->filesystem->get($path); + } + } + + $this->stream($file, $path, $name); + + return $this->filesystem->get($path); + } + + public function clear($maxAge) + { + $matches = $this->filesystem->listKeys($this->chunkPrefix); + + $limit = time()+$maxAge; + $toDelete = array(); + + // Collect the directories that are old, + // this also means the files inside are old + // but after the files are deleted the dirs + // would remain + foreach ($matches['dirs'] as $key) { + if ($limit < $this->filesystem->mtime($key)) { + $toDelete[] = $key; + } + } + // The same directory is returned for every file it contains + array_unique($toDelete); + foreach ($matches['keys'] as $key) { + if ($limit < $this->filesystem->mtime($key)) { + $this->filesystem->delete($key); + } + } + + foreach($toDelete as $key) { + // The filesystem will throw exceptions if + // a directory is not empty + try { + $this->filesystem->delete($key); + } catch (\Exception $e) { + //do nothing + } + } + } + + /** + * Only saves the information about the chunk to avoid moving it + * forth-and-back to reassemble it. Load distribution is enforced + * for gaufrette based chunk storage therefore assembleChunks will + * be called in the same request. + * + * @param $uuid + * @param $index + * @param UploadedFile $chunk + * @param $original + */ + public function addChunk($uuid, $index, UploadedFile $chunk, $original) + { + $this->unhandledChunk = array( + 'uuid' => $uuid, + 'index' => $index, + 'chunk' => $chunk, + 'original' => $original + ); + return; + } + + public function assembleChunks($chunks, $removeChunk, $renameChunk) + { + // the index is only added to be in sync with the filesystem storage + $path = $this->chunkPrefix.'/'.$this->unhandledChunk['uuid'].'/'; + $filename = $this->unhandledChunk['index'].'_'.$this->unhandledChunk['original']; + + if (empty($chunks)) { + $target = $filename; + } else { + /* + * The array only contains items with matching prefix until the filename + * therefore the order will be decided depending on the filename + * It is only case-insensitive to be overly-careful. + */ + sort($chunks, SORT_STRING | SORT_FLAG_CASE); + $target = pathinfo($chunks[0], PATHINFO_BASENAME); + } + + $this->stream($this->unhandledChunk['chunk'], $path, $target); + + if ($renameChunk) { + $name = preg_replace('/^(\d+)_/', '', $target); + $this->filesystem->rename($path.$target, $path.$name); + $target = $name; + } + $uploaded = $this->filesystem->get($path.$target); + + if (!$renameChunk) { + return $uploaded; + } + + return new GaufretteFile($uploaded, $this->filesystem); + } + + public function cleanup($path) + { + $this->filesystem->delete($path); + } + + public function getChunks($uuid) + { + return $this->filesystem->listKeys($this->chunkPrefix.'/'.$uuid)['keys']; + } + + protected function stream($file, $path, $name) + { if ($this->filesystem->getAdapter() instanceof MetadataSupporter) { $this->filesystem->getAdapter()->setMetadata($name, array('contentType' => $file->getMimeType())); } - $src = new LocalStream($file->getPathname()); - $dst = $this->filesystem->createStream($path); - + $path = $path.$name; // this is a somehow ugly workaround introduced // because the stream-mode is not able to create // subdirectories. if(!$this->filesystem->has($path)) $this->filesystem->write($path, '', true); + $src = new LocalStream($file->getPathname()); + $dst = $this->filesystem->createStream($path); + $src->open(new StreamMode('rb+')); - $dst->open(new StreamMode('wb+')); + $dst->open(new StreamMode('ab+')); while (!$src->eof()) { $data = $src->read($this->bufferSize); @@ -48,7 +168,6 @@ public function upload(File $file, $name, $path = null) $dst->close(); $src->close(); - - return $this->filesystem->get($path); } + } diff --git a/Uploader/Storage/OrphanageStorage.php b/Uploader/Storage/OrphanageStorage.php index fcbbaf06..a50a3bdc 100644 --- a/Uploader/Storage/OrphanageStorage.php +++ b/Uploader/Storage/OrphanageStorage.php @@ -2,6 +2,7 @@ namespace Oneup\UploaderBundle\Uploader\Storage; +use Oneup\UploaderBundle\Uploader\File\FileInterface; use Symfony\Component\HttpFoundation\Session\SessionInterface; use Symfony\Component\HttpFoundation\File\File; use Symfony\Component\Finder\Finder; @@ -27,7 +28,7 @@ public function __construct(StorageInterface $storage, SessionInterface $session $this->type = $type; } - public function upload(File $file, $name, $path = null) + public function upload(FileInterface $file, $name, $path = null) { if(!$this->session->isStarted()) throw new \RuntimeException('You need a running session in order to run the Orphanage.'); diff --git a/Uploader/Storage/StorageInterface.php b/Uploader/Storage/StorageInterface.php index 79fb3ae9..a9e2332d 100644 --- a/Uploader/Storage/StorageInterface.php +++ b/Uploader/Storage/StorageInterface.php @@ -2,6 +2,7 @@ namespace Oneup\UploaderBundle\Uploader\Storage; +use Oneup\UploaderBundle\Uploader\File\FileInterface; use Symfony\Component\HttpFoundation\File\File; interface StorageInterface @@ -13,5 +14,5 @@ interface StorageInterface * @param string $name * @param string $path */ - public function upload(File $file, $name, $path = null); + public function upload(FileInterface $file, $name, $path = null); } From 130bef29476e69601526ca47adbdc18ad8ee3f6f Mon Sep 17 00:00:00 2001 From: mitom Date: Thu, 10 Oct 2013 12:27:42 +0200 Subject: [PATCH 07/47] separated storages both in definition and configuration --- DependencyInjection/Configuration.php | 1 + .../OneupUploaderExtension.php | 86 ++++++----- Resources/config/uploader.xml | 4 +- Uploader/Chunk/ChunkManager.php | 5 +- Uploader/Chunk/ChunkManagerInterface.php | 6 +- .../Storage/ChunkStorageInterface.php | 5 +- Uploader/Chunk/Storage/FilesystemStorage.php | 123 +++++++++++++++ Uploader/Chunk/Storage/GaufretteStorage.php | 143 ++++++++++++++++++ Uploader/File/FileInterface.php | 15 +- Uploader/File/FilesystemFile.php | 4 +- Uploader/File/GaufretteFile.php | 13 +- Uploader/Gaufrette/StreamManager.php | 59 ++++++++ Uploader/Naming/NamerInterface.php | 1 - Uploader/Naming/UniqidNamer.php | 1 - Uploader/Storage/FilesystemStorage.php | 110 +------------- Uploader/Storage/GaufretteStorage.php | 140 +---------------- 16 files changed, 416 insertions(+), 300 deletions(-) rename Uploader/{ => Chunk}/Storage/ChunkStorageInterface.php (86%) create mode 100644 Uploader/Chunk/Storage/FilesystemStorage.php create mode 100644 Uploader/Chunk/Storage/GaufretteStorage.php create mode 100644 Uploader/Gaufrette/StreamManager.php diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index ab37255b..1cf84148 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -28,6 +28,7 @@ public function getConfigTreeBuilder() ->scalarNode('filesystem')->defaultNull()->end() ->scalarNode('directory')->defaultNull()->end() ->scalarNode('sync_buffer_size')->defaultValue('100K')->end() + ->scalarNode('prefix')->defaultValue('chunks')->end() ->end() ->end() ->booleanNode('load_distribution')->defaultTrue()->end() diff --git a/DependencyInjection/OneupUploaderExtension.php b/DependencyInjection/OneupUploaderExtension.php index 72445faf..6a8bff1d 100644 --- a/DependencyInjection/OneupUploaderExtension.php +++ b/DependencyInjection/OneupUploaderExtension.php @@ -32,17 +32,7 @@ public function load(array $configs, ContainerBuilder $container) $loader->load('twig.xml'); } - if ($this->config['chunks']['storage']['type'] === 'filesystem' && - !isset($this->config['chunks']['storage']['directory'])) { - $this->config['chunks']['storage']['directory'] = sprintf('%s/uploader/chunks', $container->getParameter('kernel.cache_dir')); - } elseif ($this->config['chunks']['storage']['type'] === 'gaufrette' ) { - // Force load distribution when using gaufrette chunk storage - $this->config['chunks']['load_distribution'] = true; - } - - // register the service for the chunk storage - $this->createStorageService($this->config['chunks']['storage'], 'chunk'); - + $this->createChunkStorageService(); $this->config['orphanage']['directory'] = is_null($this->config['orphanage']['directory']) ? sprintf('%s/uploader/orphanage', $container->getParameter('kernel.cache_dir')) : @@ -56,10 +46,6 @@ public function load(array $configs, ContainerBuilder $container) // handle mappings foreach ($this->config['mappings'] as $key => $mapping) { - if ($key === 'chunk') { - throw new InvalidArgumentException('"chunk" is a protected mapping name, please use a different one.'); - } - $mapping['max_size'] = $mapping['max_size'] < 0 ? $this->getMaxUploadSize($mapping['max_size']) : $mapping['max_size'] @@ -114,42 +100,57 @@ public function load(array $configs, ContainerBuilder $container) $container->setParameter('oneup_uploader.controllers', $controllers); } - protected function createStorageService($storage, $key, $orphanage = null) + protected function createChunkStorageService() + { + $config = &$this->config['chunks']['storage']; + + $storageClass = sprintf('%%oneup_uploader.chunks_storage.%s.class%%', $config['type']); + if ($config['type'] === 'filesystem') { + $config['directory'] = is_null($config['directory']) ? + sprintf('%s/uploader/chunks', $this->container->getParameter('kernel.cache_dir')) : + $this->normalizePath($config['directory']) + ; + + $this->container + ->register('oneup_uploader.chunks_storage', sprintf('%%oneup_uploader.chunks_storage.%s.class%%', $config['type'])) + ->addArgument($config['directory']) + ; + } else { + $this->registerGaufretteStorage('oneup_uploader.chunks_storage', $storageClass, $config['filesystem'], $config['sync_buffer_size'], $config['prefix']); + + // enforce load distribution when using gaufrette as chunk + // torage to avoid moving files forth-and-back + $this->config['chunks']['load_distribution'] = true; + } + } + + protected function createStorageService($config, $key, $orphanage = null) { $storageService = null; // if a service is given, return a reference to this service // this allows a user to overwrite the storage layer if needed - if (isset($storage['service']) && !is_null($storage['service'])) { - $storageService = new Reference($storage['storage']['service']); + if (!is_null($config['service'])) { + $storageService = new Reference($config['storage']['service']); } else { // no service was given, so we create one $storageName = sprintf('oneup_uploader.storage.%s', $key); + $storageClass = sprintf('%%oneup_uploader.storage.%s.class%%', $config['type']); - if ($storage['type'] == 'filesystem') { - $storage['directory'] = is_null($storage['directory']) ? + if ($config['type'] == 'filesystem') { + $config['directory'] = is_null($config['directory']) ? sprintf('%s/../web/uploads/%s', $this->container->getParameter('kernel.root_dir'), $key) : - $this->normalizePath($storage['directory']) + $this->normalizePath($config['directory']) ; $this->container - ->register($storageName, sprintf('%%oneup_uploader.storage.%s.class%%', $storage['type'])) - ->addArgument($storage['directory']) + ->register($storageName, $storageClass) + ->addArgument($config['directory']) ; } - if ($storage['type'] == 'gaufrette') { - if(!class_exists('Gaufrette\\Filesystem')) - throw new InvalidArgumentException('You have to install Gaufrette in order to use it as a storage service.'); - - if(strlen($storage['filesystem']) <= 0) - throw new ServiceNotFoundException('Empty service name'); - - $this->container - ->register($storageName, sprintf('%%oneup_uploader.storage.%s.class%%', $storage['type'])) - ->addArgument(new Reference($storage['filesystem'])) - ->addArgument($this->getValueInBytes($storage['sync_buffer_size'])) - ; + if ($config['type'] == 'gaufrette') { + $this->registerGaufretteStorage($storageName, $storageClass, $config['filesystem'], $config['sync_buffer_size']); } $storageService = new Reference($storageName); @@ -175,6 +176,21 @@ protected function createStorageService($storage, $key, $orphanage = null) return $storageService; } + protected function registerGaufretteStorage($key, $class, $filesystem, $buffer, $prefix = '') + { + if(!class_exists('Gaufrette\\Filesystem')) + throw new InvalidArgumentException('You have to install Gaufrette in order to use it as a chunk storage service.'); + + if(strlen($filesystem) <= 0) + throw new ServiceNotFoundException('Empty service name'); + + $this->container + ->register($key, $class) + ->addArgument(new Reference($filesystem)) + ->addArgument($this->getValueInBytes($buffer)) + ->addArgument($prefix) + ; + } protected function getMaxUploadSize($input) { diff --git a/Resources/config/uploader.xml b/Resources/config/uploader.xml index 3c953864..66c7e70d 100644 --- a/Resources/config/uploader.xml +++ b/Resources/config/uploader.xml @@ -5,6 +5,8 @@ Oneup\UploaderBundle\Uploader\Chunk\ChunkManager + Oneup\UploaderBundle\Uploader\Chunk\Storage\GaufretteStorage + Oneup\UploaderBundle\Uploader\Chunk\Storage\FilesystemStorage Oneup\UploaderBundle\Uploader\Naming\UniqidNamer Oneup\UploaderBundle\Routing\RouteLoader Oneup\UploaderBundle\Uploader\Storage\GaufretteStorage @@ -26,7 +28,7 @@ %oneup_uploader.chunks% - + diff --git a/Uploader/Chunk/ChunkManager.php b/Uploader/Chunk/ChunkManager.php index 39bd2d21..49050c0b 100644 --- a/Uploader/Chunk/ChunkManager.php +++ b/Uploader/Chunk/ChunkManager.php @@ -2,11 +2,8 @@ namespace Oneup\UploaderBundle\Uploader\Chunk; -use Oneup\UploaderBundle\Uploader\Storage\ChunkStorageInterface; -use Symfony\Component\HttpFoundation\File\File; +use Oneup\UploaderBundle\Uploader\Chunk\Storage\ChunkStorageInterface; use Symfony\Component\HttpFoundation\File\UploadedFile; -use Symfony\Component\Finder\Finder; -use Symfony\Component\Filesystem\Filesystem; use Oneup\UploaderBundle\Uploader\Chunk\ChunkManagerInterface; diff --git a/Uploader/Chunk/ChunkManagerInterface.php b/Uploader/Chunk/ChunkManagerInterface.php index 99ad60c9..c89d9bfe 100644 --- a/Uploader/Chunk/ChunkManagerInterface.php +++ b/Uploader/Chunk/ChunkManagerInterface.php @@ -21,9 +21,9 @@ public function addChunk($uuid, $index, UploadedFile $chunk, $original); /** * Assembles the given chunks and return the resulting file. * - * @param $chunks - * @param bool $removeChunk Remove the chunk file once its assembled. - * @param bool $renameChunk Rename the chunk file once its assembled. + * @param $chunks + * @param bool $removeChunk Remove the chunk file once its assembled. + * @param bool $renameChunk Rename the chunk file once its assembled. * * @return File */ diff --git a/Uploader/Storage/ChunkStorageInterface.php b/Uploader/Chunk/Storage/ChunkStorageInterface.php similarity index 86% rename from Uploader/Storage/ChunkStorageInterface.php rename to Uploader/Chunk/Storage/ChunkStorageInterface.php index b9fa1f63..2be1146a 100644 --- a/Uploader/Storage/ChunkStorageInterface.php +++ b/Uploader/Chunk/Storage/ChunkStorageInterface.php @@ -1,7 +1,6 @@ directory = $directory; + } + + public function clear($maxAge) + { + $system = new Filesystem(); + $finder = new Finder(); + + try { + $finder->in($this->directory)->date('<=' . -1 * (int) $maxAge . 'seconds')->files(); + } catch (\InvalidArgumentException $e) { + // the finder will throw an exception of type InvalidArgumentException + // if the directory he should search in does not exist + // in that case we don't have anything to clean + return; + } + + foreach ($finder as $file) { + $system->remove($file); + } + } + + public function addChunk($uuid, $index, UploadedFile $chunk, $original) + { + $filesystem = new Filesystem(); + $path = sprintf('%s/%s', $this->directory, $uuid); + $name = sprintf('%s_%s', $index, $original); + + // create directory if it does not yet exist + if(!$filesystem->exists($path)) + $filesystem->mkdir(sprintf('%s/%s', $this->directory, $uuid)); + + return $chunk->move($path, $name); + } + + public function assembleChunks($chunks, $removeChunk, $renameChunk) + { + if (!($chunks instanceof \IteratorAggregate)) { + throw new \InvalidArgumentException('The first argument must implement \IteratorAggregate interface.'); + } + + $iterator = $chunks->getIterator()->getInnerIterator(); + + $base = $iterator->current(); + $iterator->next(); + + while ($iterator->valid()) { + + $file = $iterator->current(); + + if (false === file_put_contents($base->getPathname(), file_get_contents($file->getPathname()), \FILE_APPEND | \LOCK_EX)) { + throw new \RuntimeException('Reassembling chunks failed.'); + } + + if ($removeChunk) { + $filesystem = new Filesystem(); + $filesystem->remove($file->getPathname()); + } + + $iterator->next(); + } + + $name = $base->getBasename(); + + if ($renameChunk) { + // remove the prefix added by self::addChunk + $name = preg_replace('/^(\d+)_/', '', $base->getBasename()); + } + + $assembled = new File($base->getRealPath()); + $assembled = $assembled->move($base->getPath(), $name); + + // the file is only renamed before it is uploaded + if ($renameChunk) { + // create an file to meet interface restrictions + $assembled = new FilesystemFile(new UploadedFile($assembled->getPathname(), $assembled->getBasename(), null, $assembled->getSize(), null, true)); + } + + return $assembled; + } + + public function cleanup($path) + { + // cleanup + $filesystem = new Filesystem(); + $filesystem->remove($path); + + return true; + } + + public function getChunks($uuid) + { + $finder = new Finder(); + $finder + ->in(sprintf('%s/%s', $this->directory, $uuid))->files()->sort(function(\SplFileInfo $a, \SplFileInfo $b) { + $t = explode('_', $a->getBasename()); + $s = explode('_', $b->getBasename()); + $t = (int) $t[0]; + $s = (int) $s[0]; + + return $s < $t; + }); + + return $finder; + } +} diff --git a/Uploader/Chunk/Storage/GaufretteStorage.php b/Uploader/Chunk/Storage/GaufretteStorage.php new file mode 100644 index 00000000..00b00319 --- /dev/null +++ b/Uploader/Chunk/Storage/GaufretteStorage.php @@ -0,0 +1,143 @@ +getAdapter() instanceof StreamFactory)) { + throw new \InvalidArgumentException('The filesystem used as chunk storage must implement StreamFactory'); + } + $this->filesystem = $filesystem; + $this->bufferSize = $bufferSize; + $this->prefix = $prefix; + } + + public function clear($maxAge) + { + $matches = $this->filesystem->listKeys($this->prefix); + + $limit = time()+$maxAge; + $toDelete = array(); + + // Collect the directories that are old, + // this also means the files inside are old + // but after the files are deleted the dirs + // would remain + foreach ($matches['dirs'] as $key) { + if ($limit < $this->filesystem->mtime($key)) { + $toDelete[] = $key; + } + } + // The same directory is returned for every file it contains + array_unique($toDelete); + foreach ($matches['keys'] as $key) { + if ($limit < $this->filesystem->mtime($key)) { + $this->filesystem->delete($key); + } + } + + foreach ($toDelete as $key) { + // The filesystem will throw exceptions if + // a directory is not empty + try { + $this->filesystem->delete($key); + } catch (\Exception $e) { + //do nothing + } + } + } + + /** + * Only saves the information about the chunk to avoid moving it + * forth-and-back to reassemble it. Load distribution is enforced + * for gaufrette based chunk storage therefore assembleChunks will + * be called in the same request. + * + * @param $uuid + * @param $index + * @param UploadedFile $chunk + * @param $original + */ + public function addChunk($uuid, $index, UploadedFile $chunk, $original) + { + $this->unhandledChunk = array( + 'uuid' => $uuid, + 'index' => $index, + 'chunk' => $chunk, + 'original' => $original + ); + + return; + } + + public function assembleChunks($chunks, $removeChunk, $renameChunk) + { + // the index is only added to be in sync with the filesystem storage + $path = $this->prefix.'/'.$this->unhandledChunk['uuid'].'/'; + $filename = $this->unhandledChunk['index'].'_'.$this->unhandledChunk['original']; + + if (empty($chunks)) { + $target = $filename; + $this->ensureRemotePathExists($path.$target); + } else { + /* + * The array only contains items with matching prefix until the filename + * therefore the order will be decided depending on the filename + * It is only case-insensitive to be overly-careful. + */ + sort($chunks, SORT_STRING | SORT_FLAG_CASE); + $target = pathinfo($chunks[0], PATHINFO_BASENAME); + } + + $dst = $this->filesystem->createStream($path.$target); + if ($this->unhandledChunk['index'] === 0) { + // if it's the first chunk overwrite the already existing part + // to avoid appending to earlier failed uploads + $this->openStream($dst, 'w'); + } else { + $this->openStream($dst, 'a'); + } + + + // Meet the interface requirements + $uploadedFile = new FilesystemFile($this->unhandledChunk['chunk']); + + $this->stream($uploadedFile, $dst); + + if ($renameChunk) { + $name = preg_replace('/^(\d+)_/', '', $target); + $this->filesystem->rename($path.$target, $path.$name); + $target = $name; + } + $uploaded = $this->filesystem->get($path.$target); + + if (!$renameChunk) { + return $uploaded; + } + + return new GaufretteFile($uploaded, $this->filesystem); + } + + public function cleanup($path) + { + $this->filesystem->delete($path); + } + + public function getChunks($uuid) + { + return $this->filesystem->listKeys($this->prefix.'/'.$uuid)['keys']; + } +} diff --git a/Uploader/File/FileInterface.php b/Uploader/File/FileInterface.php index 47e42662..511169d7 100644 --- a/Uploader/File/FileInterface.php +++ b/Uploader/File/FileInterface.php @@ -21,10 +21,17 @@ interface FileInterface public function getSize(); /** - * Returns the directory of the file without the filename + * Returns the path of the file * * @return string */ + public function getPathname(); + + /** + * Return the path of the file without the filename + * + * @return mixed + */ public function getPath(); /** @@ -35,11 +42,11 @@ public function getPath(); public function getMimeType(); /** - * Returns the filename of the file + * Returns the basename of the file * * @return string */ - public function getName(); + public function getBasename(); /** * Returns the guessed extension of the file @@ -47,4 +54,4 @@ public function getName(); * @return mixed */ public function getExtension(); -} \ No newline at end of file +} diff --git a/Uploader/File/FilesystemFile.php b/Uploader/File/FilesystemFile.php index eed8d67a..f30d0284 100644 --- a/Uploader/File/FilesystemFile.php +++ b/Uploader/File/FilesystemFile.php @@ -10,6 +10,6 @@ class FilesystemFile extends UploadedFile implements FileInterface public function __construct(UploadedFile $file) { $this->file = $file; - parent::__construct($file->getPath(), $file->getClientOriginalName(), $file->getClientMimeType(), $file->getClientSize(), $file->getError(), true); + parent::__construct($file->getPathname(), $file->getClientOriginalName(), $file->getClientMimeType(), $file->getClientSize(), $file->getError(), true); } -} \ No newline at end of file +} diff --git a/Uploader/File/GaufretteFile.php b/Uploader/File/GaufretteFile.php index 7771c4ab..fd0701a2 100644 --- a/Uploader/File/GaufretteFile.php +++ b/Uploader/File/GaufretteFile.php @@ -9,7 +9,8 @@ class GaufretteFile extends File implements FileInterface { protected $filesystem; - public function __construct(File $file, Filesystem $filesystem) { + public function __construct(File $file, Filesystem $filesystem) + { parent::__construct($file->getKey(), $filesystem); $this->filesystem = $filesystem; } @@ -30,12 +31,17 @@ public function getSize() return parent::getSize(); } + public function getPathname() + { + return $this->getKey(); + } + public function getPath() { return pathinfo($this->getKey(), PATHINFO_DIRNAME); } - public function getName() + public function getBasename() { return pathinfo($this->getKey(), PATHINFO_BASENAME); } @@ -46,6 +52,7 @@ public function getName() public function getMimeType() { $finfo = finfo_open(FILEINFO_MIME_TYPE); + return finfo_file($finfo, $this->getKey()); } @@ -59,4 +66,4 @@ public function getFilesystem() return $this->filesystem; } -} \ No newline at end of file +} diff --git a/Uploader/Gaufrette/StreamManager.php b/Uploader/Gaufrette/StreamManager.php new file mode 100644 index 00000000..3ae16c5b --- /dev/null +++ b/Uploader/Gaufrette/StreamManager.php @@ -0,0 +1,59 @@ +createStream(); + } + + return new LocalStream($file->getPathname()); + } + + protected function ensureRemotePathExists($path) + { + // this is a somehow ugly workaround introduced + // because the stream-mode is not able to create + // subdirectories. + if(!$this->filesystem->has($path)) + $this->filesystem->write($path, '', true); + } + + protected function openStream(Stream $stream, $mode) + { + // always use binary mode + $mode = $mode.'b+'; + + return $stream->open(new StreamMode($mode)); + } + + protected function stream(FileInterface $file, Stream $dst) + { + $src = $this->createSourceStream($file); + + // always use reading only for the source + $this->openStream($src, 'r'); + + while (!$src->eof()) { + $data = $src->read($this->bufferSize); + $dst->write($data); + } + + $dst->close(); + $src->close(); + } + +} diff --git a/Uploader/Naming/NamerInterface.php b/Uploader/Naming/NamerInterface.php index 173e0b8f..fabd0178 100644 --- a/Uploader/Naming/NamerInterface.php +++ b/Uploader/Naming/NamerInterface.php @@ -2,7 +2,6 @@ namespace Oneup\UploaderBundle\Uploader\Naming; - use Oneup\UploaderBundle\Uploader\File\FileInterface; interface NamerInterface diff --git a/Uploader/Naming/UniqidNamer.php b/Uploader/Naming/UniqidNamer.php index 80bc7411..34a71413 100644 --- a/Uploader/Naming/UniqidNamer.php +++ b/Uploader/Naming/UniqidNamer.php @@ -2,7 +2,6 @@ namespace Oneup\UploaderBundle\Uploader\Naming; - use Oneup\UploaderBundle\Uploader\File\FileInterface; class UniqidNamer implements NamerInterface diff --git a/Uploader/Storage/FilesystemStorage.php b/Uploader/Storage/FilesystemStorage.php index d443f3cf..c6c90012 100644 --- a/Uploader/Storage/FilesystemStorage.php +++ b/Uploader/Storage/FilesystemStorage.php @@ -3,14 +3,9 @@ namespace Oneup\UploaderBundle\Uploader\Storage; use Oneup\UploaderBundle\Uploader\File\FileInterface; -use Oneup\UploaderBundle\Uploader\File\FilesystemFile; -use Symfony\Component\Filesystem\Filesystem; -use Symfony\Component\Finder\Finder; use Symfony\Component\HttpFoundation\File\File; -use Symfony\Component\HttpFoundation\File\UploadedFile; - -class FilesystemStorage implements StorageInterface,ChunkStorageInterface +class FilesystemStorage implements StorageInterface { protected $directory; @@ -37,107 +32,4 @@ public function upload(FileInterface $file, $name, $path = null) return $file; } - - public function clear($maxAge) - { - $system = new Filesystem(); - $finder = new Finder(); - - try { - $finder->in($this->directory)->date('<=' . -1 * (int) $maxAge . 'seconds')->files(); - } catch (\InvalidArgumentException $e) { - // the finder will throw an exception of type InvalidArgumentException - // if the directory he should search in does not exist - // in that case we don't have anything to clean - return; - } - - foreach ($finder as $file) { - $system->remove($file); - } - } - - public function addChunk($uuid, $index, UploadedFile $chunk, $original) - { - $filesystem = new Filesystem(); - $path = sprintf('%s/%s', $this->directory, $uuid); - $name = sprintf('%s_%s', $index, $original); - - // create directory if it does not yet exist - if(!$filesystem->exists($path)) - $filesystem->mkdir(sprintf('%s/%s', $this->directory, $uuid)); - - return $chunk->move($path, $name); - } - - public function assembleChunks($chunks, $removeChunk, $renameChunk) - { - if (!($chunks instanceof \IteratorAggregate)) { - throw new \InvalidArgumentException('The first argument must implement \IteratorAggregate interface.'); - } - - $iterator = $chunks->getIterator()->getInnerIterator(); - - $base = $iterator->current(); - $iterator->next(); - - while ($iterator->valid()) { - - $file = $iterator->current(); - - if (false === file_put_contents($base->getPathname(), file_get_contents($file->getPathname()), \FILE_APPEND | \LOCK_EX)) { - throw new \RuntimeException('Reassembling chunks failed.'); - } - - if ($removeChunk) { - $filesystem = new Filesystem(); - $filesystem->remove($file->getPathname()); - } - - $iterator->next(); - } - - $name = $base->getBasename(); - - if ($renameChunk) { - // remove the prefix added by self::addChunk - $name = preg_replace('/^(\d+)_/', '', $base->getBasename()); - } - - $assembled = new File($base->getRealPath()); - $assembled = $assembled->move($base->getPath(), $name); - - // the file is only renamed before it is uploaded - if ($renameChunk) { - // create an file to meet interface restrictions - $assembled = new FilesystemFile(new UploadedFile($assembled->getPathname(), $assembled->getBasename(), null, $assembled->getSize(), null, true)); - } - - return $assembled; - } - - public function cleanup($path) - { - // cleanup - $filesystem = new Filesystem(); - $filesystem->remove($path); - - return true; - } - - public function getChunks($uuid) - { - $finder = new Finder(); - $finder - ->in(sprintf('%s/%s', $this->directory, $uuid))->files()->sort(function(\SplFileInfo $a, \SplFileInfo $b) { - $t = explode('_', $a->getBasename()); - $s = explode('_', $b->getBasename()); - $t = (int) $t[0]; - $s = (int) $s[0]; - - return $s < $t; - }); - - return $finder; - } } diff --git a/Uploader/Storage/GaufretteStorage.php b/Uploader/Storage/GaufretteStorage.php index 6c82693e..35d46b70 100644 --- a/Uploader/Storage/GaufretteStorage.php +++ b/Uploader/Storage/GaufretteStorage.php @@ -4,19 +4,12 @@ use Oneup\UploaderBundle\Uploader\File\FileInterface; use Oneup\UploaderBundle\Uploader\File\GaufretteFile; -use Gaufrette\Stream\Local as LocalStream; -use Gaufrette\StreamMode; use Gaufrette\Filesystem; use Gaufrette\Adapter\MetadataSupporter; +use Oneup\UploaderBundle\Uploader\Gaufrette\StreamManager; -use Symfony\Component\HttpFoundation\File\UploadedFile; - -class GaufretteStorage implements StorageInterface, ChunkStorageInterface +class GaufretteStorage extends StreamManager implements StorageInterface { - protected $filesystem; - protected $bufferSize; - protected $unhandledChunk; - protected $chunkPrefix = 'chunks'; public function __construct(Filesystem $filesystem, $bufferSize) { @@ -36,138 +29,17 @@ public function upload(FileInterface $file, $name, $path = null) } } - $this->stream($file, $path, $name); - - return $this->filesystem->get($path); - } - - public function clear($maxAge) - { - $matches = $this->filesystem->listKeys($this->chunkPrefix); - - $limit = time()+$maxAge; - $toDelete = array(); - - // Collect the directories that are old, - // this also means the files inside are old - // but after the files are deleted the dirs - // would remain - foreach ($matches['dirs'] as $key) { - if ($limit < $this->filesystem->mtime($key)) { - $toDelete[] = $key; - } - } - // The same directory is returned for every file it contains - array_unique($toDelete); - foreach ($matches['keys'] as $key) { - if ($limit < $this->filesystem->mtime($key)) { - $this->filesystem->delete($key); - } - } - - foreach($toDelete as $key) { - // The filesystem will throw exceptions if - // a directory is not empty - try { - $this->filesystem->delete($key); - } catch (\Exception $e) { - //do nothing - } - } - } - - /** - * Only saves the information about the chunk to avoid moving it - * forth-and-back to reassemble it. Load distribution is enforced - * for gaufrette based chunk storage therefore assembleChunks will - * be called in the same request. - * - * @param $uuid - * @param $index - * @param UploadedFile $chunk - * @param $original - */ - public function addChunk($uuid, $index, UploadedFile $chunk, $original) - { - $this->unhandledChunk = array( - 'uuid' => $uuid, - 'index' => $index, - 'chunk' => $chunk, - 'original' => $original - ); - return; - } - - public function assembleChunks($chunks, $removeChunk, $renameChunk) - { - // the index is only added to be in sync with the filesystem storage - $path = $this->chunkPrefix.'/'.$this->unhandledChunk['uuid'].'/'; - $filename = $this->unhandledChunk['index'].'_'.$this->unhandledChunk['original']; - - if (empty($chunks)) { - $target = $filename; - } else { - /* - * The array only contains items with matching prefix until the filename - * therefore the order will be decided depending on the filename - * It is only case-insensitive to be overly-careful. - */ - sort($chunks, SORT_STRING | SORT_FLAG_CASE); - $target = pathinfo($chunks[0], PATHINFO_BASENAME); - } - - $this->stream($this->unhandledChunk['chunk'], $path, $target); - - if ($renameChunk) { - $name = preg_replace('/^(\d+)_/', '', $target); - $this->filesystem->rename($path.$target, $path.$name); - $target = $name; - } - $uploaded = $this->filesystem->get($path.$target); - - if (!$renameChunk) { - return $uploaded; - } - - return new GaufretteFile($uploaded, $this->filesystem); - } - - public function cleanup($path) - { - $this->filesystem->delete($path); - } - - public function getChunks($uuid) - { - return $this->filesystem->listKeys($this->chunkPrefix.'/'.$uuid)['keys']; - } - - protected function stream($file, $path, $name) - { if ($this->filesystem->getAdapter() instanceof MetadataSupporter) { $this->filesystem->getAdapter()->setMetadata($name, array('contentType' => $file->getMimeType())); } - - $path = $path.$name; - // this is a somehow ugly workaround introduced - // because the stream-mode is not able to create - // subdirectories. - if(!$this->filesystem->has($path)) - $this->filesystem->write($path, '', true); - - $src = new LocalStream($file->getPathname()); + $this->ensureRemotePathExists($path.$name); $dst = $this->filesystem->createStream($path); - $src->open(new StreamMode('rb+')); - $dst->open(new StreamMode('ab+')); + $this->openStream($dst, 'w'); - while (!$src->eof()) { - $data = $src->read($this->bufferSize); - $dst->write($data); - } + $this->stream($file, $dst); - $dst->close(); - $src->close(); + return $this->filesystem->get($path); } } From f795b14669b1cb3e7855b7326aee646f1aae7570 Mon Sep 17 00:00:00 2001 From: mitom Date: Thu, 10 Oct 2013 12:32:55 +0200 Subject: [PATCH 08/47] moved the metadatasupporter check --- Uploader/Storage/GaufretteStorage.php | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Uploader/Storage/GaufretteStorage.php b/Uploader/Storage/GaufretteStorage.php index 35d46b70..c69e3b2d 100644 --- a/Uploader/Storage/GaufretteStorage.php +++ b/Uploader/Storage/GaufretteStorage.php @@ -21,6 +21,10 @@ public function upload(FileInterface $file, $name, $path = null) { $path = is_null($path) ? $name : sprintf('%s/%s', $path, $name); + if ($this->filesystem->getAdapter() instanceof MetadataSupporter) { + $this->filesystem->getAdapter()->setMetadata($name, array('contentType' => $file->getMimeType())); + } + if ($file instanceof GaufretteFile) { if ($file->getFilesystem() == $this->filesystem) { $file->getFilesystem()->rename($file->getKey(), $path); @@ -29,9 +33,6 @@ public function upload(FileInterface $file, $name, $path = null) } } - if ($this->filesystem->getAdapter() instanceof MetadataSupporter) { - $this->filesystem->getAdapter()->setMetadata($name, array('contentType' => $file->getMimeType())); - } $this->ensureRemotePathExists($path.$name); $dst = $this->filesystem->createStream($path); From f1f84bca6eae7b49f47e43ee21b2bf0a4605e2f8 Mon Sep 17 00:00:00 2001 From: mitom Date: Thu, 10 Oct 2013 13:46:35 +0200 Subject: [PATCH 09/47] better file size calculation for gaufrette files --- DependencyInjection/Configuration.php | 2 + .../OneupUploaderExtension.php | 19 ++++++-- Uploader/Chunk/Storage/GaufretteStorage.php | 6 ++- Uploader/File/GaufretteFile.php | 43 ++++++++++++++++--- Uploader/Storage/GaufretteStorage.php | 8 ++-- 5 files changed, 65 insertions(+), 13 deletions(-) diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index 1cf84148..c11e0148 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -27,6 +27,7 @@ public function getConfigTreeBuilder() ->end() ->scalarNode('filesystem')->defaultNull()->end() ->scalarNode('directory')->defaultNull()->end() + ->scalarNode('stream_wrapper')->defaultNull()->end() ->scalarNode('sync_buffer_size')->defaultValue('100K')->end() ->scalarNode('prefix')->defaultValue('chunks')->end() ->end() @@ -68,6 +69,7 @@ public function getConfigTreeBuilder() ->end() ->scalarNode('filesystem')->defaultNull()->end() ->scalarNode('directory')->defaultNull()->end() + ->scalarNode('stream_wrapper')->defaultNull()->end() ->scalarNode('sync_buffer_size')->defaultValue('100K')->end() ->end() ->end() diff --git a/DependencyInjection/OneupUploaderExtension.php b/DependencyInjection/OneupUploaderExtension.php index 6a8bff1d..5f5ece93 100644 --- a/DependencyInjection/OneupUploaderExtension.php +++ b/DependencyInjection/OneupUploaderExtension.php @@ -116,7 +116,13 @@ protected function createChunkStorageService() ->addArgument($config['directory']) ; } else { - $this->registerGaufretteStorage('oneup_uploader.chunks_storage', $storageClass, $config['filesystem'], $config['sync_buffer_size'], $config['prefix']); + $this->registerGaufretteStorage( + 'oneup_uploader.chunks_storage', + $storageClass, $config['filesystem'], + $config['sync_buffer_size'], + $config['stream_wrapper'], + $config['prefix'] + ); // enforce load distribution when using gaufrette as chunk // torage to avoid moving files forth-and-back @@ -150,7 +156,13 @@ protected function createStorageService($config, $key, $orphanage = null) } if ($config['type'] == 'gaufrette') { - $this->registerGaufretteStorage($storageName, $storageClass, $config['filesystem'], $config['sync_buffer_size']); + $this->registerGaufretteStorage( + $storageName, + $storageClass, + $config['filesystem'], + $config['sync_buffer_size'], + $config['stream_wrapper'] + ); } $storageService = new Reference($storageName); @@ -176,7 +188,7 @@ protected function createStorageService($config, $key, $orphanage = null) return $storageService; } - protected function registerGaufretteStorage($key, $class, $filesystem, $buffer, $prefix = '') + protected function registerGaufretteStorage($key, $class, $filesystem, $buffer, $streamWrapper = null, $prefix = '') { if(!class_exists('Gaufrette\\Filesystem')) throw new InvalidArgumentException('You have to install Gaufrette in order to use it as a chunk storage service.'); @@ -188,6 +200,7 @@ protected function registerGaufretteStorage($key, $class, $filesystem, $buffer, ->register($key, $class) ->addArgument(new Reference($filesystem)) ->addArgument($this->getValueInBytes($buffer)) + ->addArgument($streamWrapper) ->addArgument($prefix) ; } diff --git a/Uploader/Chunk/Storage/GaufretteStorage.php b/Uploader/Chunk/Storage/GaufretteStorage.php index 00b00319..98707b7e 100644 --- a/Uploader/Chunk/Storage/GaufretteStorage.php +++ b/Uploader/Chunk/Storage/GaufretteStorage.php @@ -14,8 +14,9 @@ class GaufretteStorage extends StreamManager implements ChunkStorageInterface { protected $unhandledChunk; protected $prefix; + protected $streamWrapperPrefix; - public function __construct(Filesystem $filesystem, $bufferSize, $prefix) + public function __construct(Filesystem $filesystem, $bufferSize, $streamWrapperPrefix, $prefix) { if (!($filesystem->getAdapter() instanceof StreamFactory)) { throw new \InvalidArgumentException('The filesystem used as chunk storage must implement StreamFactory'); @@ -23,6 +24,7 @@ public function __construct(Filesystem $filesystem, $bufferSize, $prefix) $this->filesystem = $filesystem; $this->bufferSize = $bufferSize; $this->prefix = $prefix; + $this->streamWrapperPrefix = $streamWrapperPrefix; } public function clear($maxAge) @@ -128,7 +130,7 @@ public function assembleChunks($chunks, $removeChunk, $renameChunk) return $uploaded; } - return new GaufretteFile($uploaded, $this->filesystem); + return new GaufretteFile($uploaded, $this->filesystem, $this->streamWrapperPrefix); } public function cleanup($path) diff --git a/Uploader/File/GaufretteFile.php b/Uploader/File/GaufretteFile.php index fd0701a2..0c73be5b 100644 --- a/Uploader/File/GaufretteFile.php +++ b/Uploader/File/GaufretteFile.php @@ -2,17 +2,21 @@ namespace Oneup\UploaderBundle\Uploader\File; +use Gaufrette\Adapter\StreamFactory; use Gaufrette\File; use Gaufrette\Filesystem; class GaufretteFile extends File implements FileInterface { protected $filesystem; + protected $streamWrapperPrefix; + protected $mimeType; - public function __construct(File $file, Filesystem $filesystem) + public function __construct(File $file, Filesystem $filesystem, $streamWrapperPrefix = null) { parent::__construct($file->getKey(), $filesystem); $this->filesystem = $filesystem; + $this->streamWrapperPrefix = $streamWrapperPrefix; } /** @@ -20,14 +24,28 @@ public function __construct(File $file, Filesystem $filesystem) * * !! WARNING !! * Calling this loads the entire file into memory, - * in case of bigger files this could throw exceptions, + * unless it is on a stream-capable filesystem. + * In case of bigger files this could throw exceptions, * and will have heavy performance footprint. * !! ------- !! * - * TODO mock/calculate the size if possible and use that instead? */ public function getSize() { + // This can only work on streamable files, so basically local files, + // still only perform it once even on local files to avoid bothering the filesystem.php g + if ($this->filesystem->getAdapter() instanceof StreamFactory && !$this->size) { + if ($this->streamWrapperPrefix) { + try { + $this->setSize(filesize($this->streamWrapperPrefix.$this->getKey())); + } catch (\Exception $e) { + // Fail gracefully if there was a problem with opening the file and + // let gaufrette load the file into memory allowing it to throw exceptions + // if that doesn't work either. + } + } + } + return parent::getSize(); } @@ -51,11 +69,26 @@ public function getBasename() */ public function getMimeType() { - $finfo = finfo_open(FILEINFO_MIME_TYPE); + // This can only work on streamable files, so basically local files, + // still only perform it once even on local files to avoid bothering the filesystem. + if ($this->filesystem->getAdapter() instanceof StreamFactory && !$this->mimeType) { + if ($this->streamWrapperPrefix) { + $finfo = finfo_open(FILEINFO_MIME_TYPE); + $this->mimeType = finfo_file($finfo, $this->streamWrapperPrefix.$this->getKey()); + finfo_close($finfo); + } + } - return finfo_file($finfo, $this->getKey()); + return $this->mimeType; } + /** + * Now that we may be able to get the mime-type the extension + * COULD be guessed based on that, but it would be even less + * accurate as mime-types can have multiple extensions + * + * @return mixed + */ public function getExtension() { return pathinfo($this->getKey(), PATHINFO_EXTENSION); diff --git a/Uploader/Storage/GaufretteStorage.php b/Uploader/Storage/GaufretteStorage.php index c69e3b2d..315b92b7 100644 --- a/Uploader/Storage/GaufretteStorage.php +++ b/Uploader/Storage/GaufretteStorage.php @@ -10,11 +10,13 @@ class GaufretteStorage extends StreamManager implements StorageInterface { + protected $streamWrapperPrefix; - public function __construct(Filesystem $filesystem, $bufferSize) + public function __construct(Filesystem $filesystem, $bufferSize, $streamWrapperPrefix) { $this->filesystem = $filesystem; $this->bufferSize = $bufferSize; + $this->streamWrapperPrefix = $streamWrapperPrefix; } public function upload(FileInterface $file, $name, $path = null) @@ -29,7 +31,7 @@ public function upload(FileInterface $file, $name, $path = null) if ($file->getFilesystem() == $this->filesystem) { $file->getFilesystem()->rename($file->getKey(), $path); - return $this->filesystem->get($path); + return new GaufretteFile($this->filesystem->get($path), $this->filesystem, $this->streamWrapperPrefix); } } @@ -40,7 +42,7 @@ public function upload(FileInterface $file, $name, $path = null) $this->stream($file, $dst); - return $this->filesystem->get($path); + return new GaufretteFile($this->filesystem->get($path), $this->filesystem, $this->streamWrapperPrefix); } } From 4769708e0d1ba25f354c309cf542768e14e7a730 Mon Sep 17 00:00:00 2001 From: mitom Date: Thu, 10 Oct 2013 14:09:58 +0200 Subject: [PATCH 10/47] fix non-chunked uploads and improve extension handling --- Controller/AbstractChunkedController.php | 1 - Controller/AbstractController.php | 11 +++++++++-- Uploader/File/FilesystemFile.php | 11 +++++++++-- Uploader/Storage/GaufretteStorage.php | 2 +- 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/Controller/AbstractChunkedController.php b/Controller/AbstractChunkedController.php index 2678a664..0b92d580 100644 --- a/Controller/AbstractChunkedController.php +++ b/Controller/AbstractChunkedController.php @@ -73,7 +73,6 @@ protected function handleChunkedUpload(UploadedFile $file, ResponseInterface $re $path = $assembled->getPath(); - // create a temporary uploaded file to meet the interface restrictions $this->handleUpload($assembled, $response, $request); $chunkManager->cleanup($path); diff --git a/Controller/AbstractController.php b/Controller/AbstractController.php index aea039e4..6d5f0b4f 100644 --- a/Controller/AbstractController.php +++ b/Controller/AbstractController.php @@ -3,6 +3,7 @@ namespace Oneup\UploaderBundle\Controller; use Oneup\UploaderBundle\Uploader\File\FileInterface; +use Oneup\UploaderBundle\Uploader\File\FilesystemFile; use Symfony\Component\HttpFoundation\JsonResponse; use Symfony\Component\DependencyInjection\ContainerInterface; use Symfony\Component\HttpFoundation\File\UploadedFile; @@ -100,12 +101,18 @@ protected function getFiles(FileBag $bag) * * Note: The return value differs when * - * @param UploadedFile The file to upload + * @param The file to upload * @param response A response object. * @param request The request object. */ - protected function handleUpload(FileInterface $file, ResponseInterface $response, Request $request) + protected function handleUpload($file, ResponseInterface $response, Request $request) { + // wrap the file if it is not done yet which can only happen + // if it wasn't a chunked upload, in which case it is definitely + // on the local filesystem. + if (!($file instanceof FileInterface)) { + $file = new FilesystemFile($file); + } $this->validate($file); $this->dispatchPreUploadEvent($file, $response, $request); diff --git a/Uploader/File/FilesystemFile.php b/Uploader/File/FilesystemFile.php index f30d0284..c24ba28c 100644 --- a/Uploader/File/FilesystemFile.php +++ b/Uploader/File/FilesystemFile.php @@ -6,10 +6,17 @@ class FilesystemFile extends UploadedFile implements FileInterface { - protected $file; public function __construct(UploadedFile $file) { - $this->file = $file; parent::__construct($file->getPathname(), $file->getClientOriginalName(), $file->getClientMimeType(), $file->getClientSize(), $file->getError(), true); + + } + + public function getExtension() + { + // If the file is in tmp, it has no extension, but the wrapper object + // will have the original extension, otherwise it is better to rely + // on the actual extension + return parent::getExtension() ? :$this->getClientOriginalExtension(); } } diff --git a/Uploader/Storage/GaufretteStorage.php b/Uploader/Storage/GaufretteStorage.php index 315b92b7..c376167e 100644 --- a/Uploader/Storage/GaufretteStorage.php +++ b/Uploader/Storage/GaufretteStorage.php @@ -35,7 +35,7 @@ public function upload(FileInterface $file, $name, $path = null) } } - $this->ensureRemotePathExists($path.$name); + $this->ensureRemotePathExists($path); $dst = $this->filesystem->createStream($path); $this->openStream($dst, 'w'); From 39c94c573d5ae718efa305944600fd65d0219573 Mon Sep 17 00:00:00 2001 From: mitom Date: Thu, 10 Oct 2013 14:54:23 +0200 Subject: [PATCH 11/47] fixed messy type hinting for more comprehensible structure --- Controller/AbstractController.php | 1 - Uploader/Storage/FilesystemStorage.php | 11 +++-------- .../Storage/FilesystemStorageInterface.php | 19 +++++++++++++++++++ Uploader/Storage/StorageInterface.php | 7 +++---- 4 files changed, 25 insertions(+), 13 deletions(-) create mode 100644 Uploader/Storage/FilesystemStorageInterface.php diff --git a/Controller/AbstractController.php b/Controller/AbstractController.php index 6d5f0b4f..03e14cb2 100644 --- a/Controller/AbstractController.php +++ b/Controller/AbstractController.php @@ -6,7 +6,6 @@ use Oneup\UploaderBundle\Uploader\File\FilesystemFile; use Symfony\Component\HttpFoundation\JsonResponse; use Symfony\Component\DependencyInjection\ContainerInterface; -use Symfony\Component\HttpFoundation\File\UploadedFile; use Symfony\Component\HttpFoundation\Request; use Symfony\Component\HttpFoundation\FileBag; diff --git a/Uploader/Storage/FilesystemStorage.php b/Uploader/Storage/FilesystemStorage.php index c6c90012..ae73615a 100644 --- a/Uploader/Storage/FilesystemStorage.php +++ b/Uploader/Storage/FilesystemStorage.php @@ -2,10 +2,9 @@ namespace Oneup\UploaderBundle\Uploader\Storage; -use Oneup\UploaderBundle\Uploader\File\FileInterface; -use Symfony\Component\HttpFoundation\File\File; +use Oneup\UploaderBundle\Uploader\File\FilesystemFile; -class FilesystemStorage implements StorageInterface +class FilesystemStorage implements FilesystemStorageInterface { protected $directory; @@ -14,12 +13,8 @@ public function __construct($directory) $this->directory = $directory; } - public function upload(FileInterface $file, $name, $path = null) + public function upload(FilesystemFile $file, $name, $path = null) { - if (!($file instanceof File)) { - throw new \InvalidArgumentException('file must be an instance of Symfony\Component\HttpFoundation\File\File'); - } - $path = is_null($path) ? $name : sprintf('%s/%s', $path, $name); $path = sprintf('%s/%s', $this->directory, $path); diff --git a/Uploader/Storage/FilesystemStorageInterface.php b/Uploader/Storage/FilesystemStorageInterface.php new file mode 100644 index 00000000..875ea4ae --- /dev/null +++ b/Uploader/Storage/FilesystemStorageInterface.php @@ -0,0 +1,19 @@ + Date: Thu, 10 Oct 2013 15:11:29 +0200 Subject: [PATCH 12/47] made scrutiziner happier --- Controller/AbstractChunkedController.php | 3 - .../OneupUploaderExtension.php | 101 ++++++++++-------- Uploader/Chunk/Storage/GaufretteStorage.php | 4 +- Uploader/File/GaufretteFile.php | 2 + 4 files changed, 58 insertions(+), 52 deletions(-) diff --git a/Controller/AbstractChunkedController.php b/Controller/AbstractChunkedController.php index 0b92d580..e9e05a8d 100644 --- a/Controller/AbstractChunkedController.php +++ b/Controller/AbstractChunkedController.php @@ -48,9 +48,6 @@ protected function handleChunkedUpload(UploadedFile $file, ResponseInterface $re $request = $this->container->get('request'); $chunkManager = $this->container->get('oneup_uploader.chunk_manager'); - // reset uploaded to always have a return value - $uploaded = null; - // get information about this chunked request list($last, $uuid, $index, $orig) = $this->parseChunkedRequest($request); diff --git a/DependencyInjection/OneupUploaderExtension.php b/DependencyInjection/OneupUploaderExtension.php index 5f5ece93..b5e2ab57 100644 --- a/DependencyInjection/OneupUploaderExtension.php +++ b/DependencyInjection/OneupUploaderExtension.php @@ -33,11 +33,7 @@ public function load(array $configs, ContainerBuilder $container) } $this->createChunkStorageService(); - - $this->config['orphanage']['directory'] = is_null($this->config['orphanage']['directory']) ? - sprintf('%s/uploader/orphanage', $container->getParameter('kernel.cache_dir')) : - $this->normalizePath($this->config['orphanage']['directory']) - ; + $this->processOrphanageConfig(); $container->setParameter('oneup_uploader.chunks', $this->config['chunks']); $container->setParameter('oneup_uploader.orphanage', $this->config['orphanage']); @@ -46,58 +42,71 @@ public function load(array $configs, ContainerBuilder $container) // handle mappings foreach ($this->config['mappings'] as $key => $mapping) { - $mapping['max_size'] = $mapping['max_size'] < 0 ? - $this->getMaxUploadSize($mapping['max_size']) : - $mapping['max_size'] - ; + $this->processMapping($key, $mapping); + } + + $container->setParameter('oneup_uploader.controllers', $controllers); + } + + protected function processOrphanageConfig() + { + $this->config['orphanage']['directory'] = is_null($this->config['orphanage']['directory']) ? + sprintf('%s/uploader/orphanage', $this->container->getParameter('kernel.cache_dir')) : + $this->normalizePath($this->config['orphanage']['directory']) + ; + } - // create the storage service according to the configuration - $storageService = $this->createStorageService($mapping['storage'], $key, isset($mapping['orphanage']) ? :null); + protected function processMapping($key, &$mapping) + { + $mapping['max_size'] = $mapping['max_size'] < 0 ? + $this->getMaxUploadSize($mapping['max_size']) : + $mapping['max_size'] + ; - if ($mapping['frontend'] != 'custom') { - $controllerName = sprintf('oneup_uploader.controller.%s', $key); - $controllerType = sprintf('%%oneup_uploader.controller.%s.class%%', $mapping['frontend']); - } else { - $customFrontend = $mapping['custom_frontend']; + // create the storage service according to the configuration + $storageService = $this->createStorageService($mapping['storage'], $key, isset($mapping['orphanage']) ? :null); - $controllerName = sprintf('oneup_uploader.controller.%s', $customFrontend['name']); - $controllerType = $customFrontend['class']; + if ($mapping['frontend'] != 'custom') { + $controllerName = sprintf('oneup_uploader.controller.%s', $key); + $controllerType = sprintf('%%oneup_uploader.controller.%s.class%%', $mapping['frontend']); + } else { + $customFrontend = $mapping['custom_frontend']; - if(empty($controllerName) || empty($controllerType)) - throw new ServiceNotFoundException('Empty controller class or name. If you really want to use a custom frontend implementation, be sure to provide a class and a name.'); - } + $controllerName = sprintf('oneup_uploader.controller.%s', $customFrontend['name']); + $controllerType = $customFrontend['class']; - $errorHandler = is_null($mapping['error_handler']) ? - new Reference('oneup_uploader.error_handler.'.$mapping['frontend']) : - new Reference($mapping['error_handler']); + if(empty($controllerName) || empty($controllerType)) + throw new ServiceNotFoundException('Empty controller class or name. If you really want to use a custom frontend implementation, be sure to provide a class and a name.'); + } - // create controllers based on mapping - $container - ->register($controllerName, $controllerType) + $errorHandler = is_null($mapping['error_handler']) ? + new Reference('oneup_uploader.error_handler.'.$mapping['frontend']) : + new Reference($mapping['error_handler']); - ->addArgument(new Reference('service_container')) - ->addArgument($storageService) - ->addArgument($errorHandler) - ->addArgument($mapping) - ->addArgument($key) + // create controllers based on mapping + $this->container + ->register($controllerName, $controllerType) - ->addTag('oneup_uploader.routable', array('type' => $key)) - ->setScope('request') - ; + ->addArgument(new Reference('service_container')) + ->addArgument($storageService) + ->addArgument($errorHandler) + ->addArgument($mapping) + ->addArgument($key) - if ($mapping['enable_progress'] || $mapping['enable_cancelation']) { - if (strnatcmp(phpversion(), '5.4.0') < 0) { - throw new InvalidArgumentException('You need to run PHP version 5.4.0 or above to use the progress/cancelation feature.'); - } - } + ->addTag('oneup_uploader.routable', array('type' => $key)) + ->setScope('request') + ; - $controllers[$key] = array($controllerName, array( - 'enable_progress' => $mapping['enable_progress'], - 'enable_cancelation' => $mapping['enable_cancelation'] - )); + if ($mapping['enable_progress'] || $mapping['enable_cancelation']) { + if (strnatcmp(phpversion(), '5.4.0') < 0) { + throw new InvalidArgumentException('You need to run PHP version 5.4.0 or above to use the progress/cancelation feature.'); + } } - $container->setParameter('oneup_uploader.controllers', $controllers); + $controllers[$key] = array($controllerName, array( + 'enable_progress' => $mapping['enable_progress'], + 'enable_cancelation' => $mapping['enable_cancelation'] + )); } protected function createChunkStorageService() @@ -130,7 +139,7 @@ protected function createChunkStorageService() } } - protected function createStorageService($config, $key, $orphanage = null) + protected function createStorageService(&$config, $key, $orphanage = null) { $storageService = null; diff --git a/Uploader/Chunk/Storage/GaufretteStorage.php b/Uploader/Chunk/Storage/GaufretteStorage.php index 98707b7e..4f5a2864 100644 --- a/Uploader/Chunk/Storage/GaufretteStorage.php +++ b/Uploader/Chunk/Storage/GaufretteStorage.php @@ -57,7 +57,7 @@ public function clear($maxAge) try { $this->filesystem->delete($key); } catch (\Exception $e) { - //do nothing + continue; } } } @@ -81,8 +81,6 @@ public function addChunk($uuid, $index, UploadedFile $chunk, $original) 'chunk' => $chunk, 'original' => $original ); - - return; } public function assembleChunks($chunks, $removeChunk, $renameChunk) diff --git a/Uploader/File/GaufretteFile.php b/Uploader/File/GaufretteFile.php index 0c73be5b..0b6904b7 100644 --- a/Uploader/File/GaufretteFile.php +++ b/Uploader/File/GaufretteFile.php @@ -42,6 +42,8 @@ public function getSize() // Fail gracefully if there was a problem with opening the file and // let gaufrette load the file into memory allowing it to throw exceptions // if that doesn't work either. + // Not empty to make the scrutiziner happy. + return parent::getSize(); } } } From bd400e893294d1f6fca31fb2de252c7eb94d7da4 Mon Sep 17 00:00:00 2001 From: mitom Date: Thu, 10 Oct 2013 16:11:07 +0200 Subject: [PATCH 13/47] broke configuration up even more fro scrutinizer --- .../OneupUploaderExtension.php | 46 +++++++++++++------ 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/DependencyInjection/OneupUploaderExtension.php b/DependencyInjection/OneupUploaderExtension.php index b5e2ab57..a86036f5 100644 --- a/DependencyInjection/OneupUploaderExtension.php +++ b/DependencyInjection/OneupUploaderExtension.php @@ -42,7 +42,7 @@ public function load(array $configs, ContainerBuilder $container) // handle mappings foreach ($this->config['mappings'] as $key => $mapping) { - $this->processMapping($key, $mapping); + $controllers[$key] = $this->processMapping($key, $mapping); } $container->setParameter('oneup_uploader.controllers', $controllers); @@ -62,15 +62,26 @@ protected function processMapping($key, &$mapping) $this->getMaxUploadSize($mapping['max_size']) : $mapping['max_size'] ; + $controllerName = $this->createController($key, $mapping); + $this->verifyPhpVersion($mapping); + + return array($controllerName, array( + 'enable_progress' => $mapping['enable_progress'], + 'enable_cancelation' => $mapping['enable_cancelation'] + )); + } + + protected function createController($key, $config) + { // create the storage service according to the configuration - $storageService = $this->createStorageService($mapping['storage'], $key, isset($mapping['orphanage']) ? :null); + $storageService = $this->createStorageService($config['storage'], $key, isset($config['orphanage']) ? :null); - if ($mapping['frontend'] != 'custom') { + if ($config['frontend'] != 'custom') { $controllerName = sprintf('oneup_uploader.controller.%s', $key); - $controllerType = sprintf('%%oneup_uploader.controller.%s.class%%', $mapping['frontend']); + $controllerType = sprintf('%%oneup_uploader.controller.%s.class%%', $config['frontend']); } else { - $customFrontend = $mapping['custom_frontend']; + $customFrontend = $config['custom_frontend']; $controllerName = sprintf('oneup_uploader.controller.%s', $customFrontend['name']); $controllerType = $customFrontend['class']; @@ -79,9 +90,7 @@ protected function processMapping($key, &$mapping) throw new ServiceNotFoundException('Empty controller class or name. If you really want to use a custom frontend implementation, be sure to provide a class and a name.'); } - $errorHandler = is_null($mapping['error_handler']) ? - new Reference('oneup_uploader.error_handler.'.$mapping['frontend']) : - new Reference($mapping['error_handler']); + $errorHandler = $this->createErrorHandler($config); // create controllers based on mapping $this->container @@ -90,23 +99,30 @@ protected function processMapping($key, &$mapping) ->addArgument(new Reference('service_container')) ->addArgument($storageService) ->addArgument($errorHandler) - ->addArgument($mapping) + ->addArgument($config) ->addArgument($key) ->addTag('oneup_uploader.routable', array('type' => $key)) ->setScope('request') ; - if ($mapping['enable_progress'] || $mapping['enable_cancelation']) { + return $controllerName; + } + + protected function createErrorHandler($config) + { + return is_null($config['error_handler']) ? + new Reference('oneup_uploader.error_handler.'.$config['frontend']) : + new Reference($config['error_handler']); + } + + protected function verifyPhpVersion($config) + { + if ($config['enable_progress'] || $config['enable_cancelation']) { if (strnatcmp(phpversion(), '5.4.0') < 0) { throw new InvalidArgumentException('You need to run PHP version 5.4.0 or above to use the progress/cancelation feature.'); } } - - $controllers[$key] = array($controllerName, array( - 'enable_progress' => $mapping['enable_progress'], - 'enable_cancelation' => $mapping['enable_cancelation'] - )); } protected function createChunkStorageService() From 85dd84a7577795dbad6d19ae364a8c158f07da24 Mon Sep 17 00:00:00 2001 From: mitom Date: Thu, 10 Oct 2013 16:36:49 +0200 Subject: [PATCH 14/47] had to lower interface requirements for the controllers --- Uploader/Storage/FilesystemStorage.php | 4 ++-- .../Storage/FilesystemStorageInterface.php | 19 ------------------- Uploader/Storage/GaufretteStorage.php | 2 +- Uploader/Storage/OrphanageStorage.php | 3 ++- Uploader/Storage/StorageInterface.php | 4 ++-- 5 files changed, 7 insertions(+), 25 deletions(-) delete mode 100644 Uploader/Storage/FilesystemStorageInterface.php diff --git a/Uploader/Storage/FilesystemStorage.php b/Uploader/Storage/FilesystemStorage.php index ae73615a..f2613bff 100644 --- a/Uploader/Storage/FilesystemStorage.php +++ b/Uploader/Storage/FilesystemStorage.php @@ -4,7 +4,7 @@ use Oneup\UploaderBundle\Uploader\File\FilesystemFile; -class FilesystemStorage implements FilesystemStorageInterface +class FilesystemStorage implements StorageInterface { protected $directory; @@ -13,7 +13,7 @@ public function __construct($directory) $this->directory = $directory; } - public function upload(FilesystemFile $file, $name, $path = null) + public function upload($file, $name, $path = null) { $path = is_null($path) ? $name : sprintf('%s/%s', $path, $name); $path = sprintf('%s/%s', $this->directory, $path); diff --git a/Uploader/Storage/FilesystemStorageInterface.php b/Uploader/Storage/FilesystemStorageInterface.php deleted file mode 100644 index 875ea4ae..00000000 --- a/Uploader/Storage/FilesystemStorageInterface.php +++ /dev/null @@ -1,19 +0,0 @@ -streamWrapperPrefix = $streamWrapperPrefix; } - public function upload(FileInterface $file, $name, $path = null) + public function upload($file, $name, $path = null) { $path = is_null($path) ? $name : sprintf('%s/%s', $path, $name); diff --git a/Uploader/Storage/OrphanageStorage.php b/Uploader/Storage/OrphanageStorage.php index a50a3bdc..ffdd2ae8 100644 --- a/Uploader/Storage/OrphanageStorage.php +++ b/Uploader/Storage/OrphanageStorage.php @@ -3,6 +3,7 @@ namespace Oneup\UploaderBundle\Uploader\Storage; use Oneup\UploaderBundle\Uploader\File\FileInterface; +use Oneup\UploaderBundle\Uploader\File\FilesystemFile; use Symfony\Component\HttpFoundation\Session\SessionInterface; use Symfony\Component\HttpFoundation\File\File; use Symfony\Component\Finder\Finder; @@ -28,7 +29,7 @@ public function __construct(StorageInterface $storage, SessionInterface $session $this->type = $type; } - public function upload(FileInterface $file, $name, $path = null) + public function upload($file, $name, $path = null) { if(!$this->session->isStarted()) throw new \RuntimeException('You need a running session in order to run the Orphanage.'); diff --git a/Uploader/Storage/StorageInterface.php b/Uploader/Storage/StorageInterface.php index d028b9b5..0c7e4b55 100644 --- a/Uploader/Storage/StorageInterface.php +++ b/Uploader/Storage/StorageInterface.php @@ -9,9 +9,9 @@ interface StorageInterface /** * Uploads a File instance to the configured storage. * - * @param FileInterface $file + * @param $file * @param string $name * @param string $path */ - public function upload(FileInterface $file, $name, $path = null); + public function upload($file, $name, $path = null); } From 7884373aed3344919d6aee3f95251307bbf24808 Mon Sep 17 00:00:00 2001 From: mitom Date: Thu, 10 Oct 2013 19:59:30 +0200 Subject: [PATCH 15/47] fixed existing tests and some mistakes revealed by them --- Tests/Controller/AbstractValidationTest.php | 2 +- Tests/Controller/BlueimpValidationTest.php | 2 +- .../FilesystemStorageTest.php} | 22 +++++++++---------- Tests/Uploader/Naming/UniqidNamerTest.php | 8 +++---- .../Storage/FilesystemStorageTest.php | 3 ++- .../Uploader/Storage/GaufretteStorageTest.php | 3 ++- .../Uploader/Storage/OrphanageStorageTest.php | 3 ++- Uploader/File/FilesystemFile.php | 14 +++++++----- Uploader/Storage/FilesystemStorage.php | 4 ++-- Uploader/Storage/GaufretteStorage.php | 4 ++-- Uploader/Storage/OrphanageStorage.php | 4 ++-- Uploader/Storage/StorageInterface.php | 8 +++---- 12 files changed, 40 insertions(+), 37 deletions(-) rename Tests/Uploader/Chunk/{ChunkManagerTest.php => Storage/FilesystemStorageTest.php} (80%) diff --git a/Tests/Controller/AbstractValidationTest.php b/Tests/Controller/AbstractValidationTest.php index 01b0a6dc..1a3c4b7f 100644 --- a/Tests/Controller/AbstractValidationTest.php +++ b/Tests/Controller/AbstractValidationTest.php @@ -56,7 +56,7 @@ public function testEvents() // event data $validationCount = 0; - $dispatcher->addListener(UploadEvents::VALIDATION, function(ValidationEvent $event) use (&$validationCount) { + $dispatcher->addListener(UploadEvents::VALIDATION, function() use (&$validationCount) { ++ $validationCount; }); diff --git a/Tests/Controller/BlueimpValidationTest.php b/Tests/Controller/BlueimpValidationTest.php index 4bbfa1c7..19fea99c 100644 --- a/Tests/Controller/BlueimpValidationTest.php +++ b/Tests/Controller/BlueimpValidationTest.php @@ -52,7 +52,7 @@ public function testEvents() // event data $validationCount = 0; - $dispatcher->addListener(UploadEvents::VALIDATION, function(ValidationEvent $event) use (&$validationCount) { + $dispatcher->addListener(UploadEvents::VALIDATION, function() use (&$validationCount) { ++ $validationCount; }); diff --git a/Tests/Uploader/Chunk/ChunkManagerTest.php b/Tests/Uploader/Chunk/Storage/FilesystemStorageTest.php similarity index 80% rename from Tests/Uploader/Chunk/ChunkManagerTest.php rename to Tests/Uploader/Chunk/Storage/FilesystemStorageTest.php index 4ce9c4aa..1df65ea3 100644 --- a/Tests/Uploader/Chunk/ChunkManagerTest.php +++ b/Tests/Uploader/Chunk/Storage/FilesystemStorageTest.php @@ -1,13 +1,12 @@ getManager($maxage); + $manager = $this->getStorage(); $numberOfFiles = 10; $finder = new Finder(); @@ -58,7 +57,7 @@ public function testChunkCleanup() $this->fillDirectory($numberOfFiles); $this->assertCount(10, $finder); - $manager->clear(); + $manager->clear($maxage); $this->assertTrue(is_dir($this->tmpDir)); $this->assertTrue(is_writeable($this->tmpDir)); @@ -75,18 +74,17 @@ public function testClearIfDirectoryDoesNotExist() $filesystem = new Filesystem(); $filesystem->remove($this->tmpDir); - $manager = $this->getManager(10); - $manager->clear(); + $manager = $this->getStorage(); + $manager->clear(10); // yey, no exception $this->assertTrue(true); } - protected function getManager($maxage) + protected function getStorage() { - return new ChunkManager(array( - 'directory' => $this->tmpDir, - 'maxage' => $maxage + return new FilesystemStorage(array( + 'directory' => $this->tmpDir )); } diff --git a/Tests/Uploader/Naming/UniqidNamerTest.php b/Tests/Uploader/Naming/UniqidNamerTest.php index 2ff1631a..d20df0ac 100644 --- a/Tests/Uploader/Naming/UniqidNamerTest.php +++ b/Tests/Uploader/Naming/UniqidNamerTest.php @@ -8,14 +8,14 @@ class UniqidNamerTest extends \PHPUnit_Framework_TestCase { public function testNamerReturnsName() { - $file = $this->getMockBuilder('Symfony\Component\HttpFoundation\File\UploadedFile') + $file = $this->getMockBuilder('Oneup\UploaderBundle\Uploader\File\FilesystemFile') ->disableOriginalConstructor() ->getMock() ; $file ->expects($this->any()) - ->method('guessExtension') + ->method('getExtension') ->will($this->returnValue('jpeg')) ; @@ -25,14 +25,14 @@ public function testNamerReturnsName() public function testNamerReturnsUniqueName() { - $file = $this->getMockBuilder('Symfony\Component\HttpFoundation\File\UploadedFile') + $file = $this->getMockBuilder('Oneup\UploaderBundle\Uploader\File\FilesystemFile') ->disableOriginalConstructor() ->getMock() ; $file ->expects($this->any()) - ->method('guessExtension') + ->method('getExtension') ->will($this->returnValue('jpeg')) ; diff --git a/Tests/Uploader/Storage/FilesystemStorageTest.php b/Tests/Uploader/Storage/FilesystemStorageTest.php index 6f3e0114..1ae9964d 100644 --- a/Tests/Uploader/Storage/FilesystemStorageTest.php +++ b/Tests/Uploader/Storage/FilesystemStorageTest.php @@ -2,6 +2,7 @@ namespace Oneup\UploaderBundle\Tests\Uploader\Storage; +use Oneup\UploaderBundle\Uploader\File\FilesystemFile; use Symfony\Component\Finder\Finder; use Symfony\Component\Filesystem\Filesystem; use Symfony\Component\HttpFoundation\File\UploadedFile; @@ -26,7 +27,7 @@ public function setUp() public function testUpload() { - $payload = new UploadedFile($this->file, 'grumpycat.jpeg', null, null, null, true); + $payload = new FilesystemFile(new UploadedFile($this->file, 'grumpycat.jpeg', null, null, null, true)); $storage = new FilesystemStorage($this->directory); $storage->upload($payload, 'notsogrumpyanymore.jpeg'); diff --git a/Tests/Uploader/Storage/GaufretteStorageTest.php b/Tests/Uploader/Storage/GaufretteStorageTest.php index f9ecfe68..008e6180 100644 --- a/Tests/Uploader/Storage/GaufretteStorageTest.php +++ b/Tests/Uploader/Storage/GaufretteStorageTest.php @@ -2,6 +2,7 @@ namespace Oneup\UploaderBundle\Tests\Uploader\Storage; +use Oneup\UploaderBundle\Uploader\File\FilesystemFile; use Symfony\Component\Finder\Finder; use Symfony\Component\Filesystem\Filesystem; use Symfony\Component\HttpFoundation\File\UploadedFile; @@ -33,7 +34,7 @@ public function setUp() public function testUpload() { - $payload = new UploadedFile($this->file, 'grumpycat.jpeg', null, null, null, true); + $payload = new FilesystemFile(new UploadedFile($this->file, 'grumpycat.jpeg', null, null, null, true)); $this->storage->upload($payload, 'notsogrumpyanymore.jpeg'); $finder = new Finder(); diff --git a/Tests/Uploader/Storage/OrphanageStorageTest.php b/Tests/Uploader/Storage/OrphanageStorageTest.php index bcaf5251..b2eed0a5 100644 --- a/Tests/Uploader/Storage/OrphanageStorageTest.php +++ b/Tests/Uploader/Storage/OrphanageStorageTest.php @@ -2,6 +2,7 @@ namespace Oneup\UploaderBundle\Tests\Uploader\Storage; +use Oneup\UploaderBundle\Uploader\File\FilesystemFile; use Symfony\Component\Finder\Finder; use Symfony\Component\Filesystem\Filesystem; use Symfony\Component\HttpFoundation\File\UploadedFile; @@ -39,7 +40,7 @@ public function setUp() fwrite($pointer, str_repeat('A', 1024), 1024); fclose($pointer); - $this->payloads[] = new UploadedFile($file, $i . 'grumpycat.jpeg', null, null, null, true); + $this->payloads[] = new FilesystemFile(new UploadedFile($file, $i . 'grumpycat.jpeg', null, null, null, true)); } // create underlying storage diff --git a/Uploader/File/FilesystemFile.php b/Uploader/File/FilesystemFile.php index c24ba28c..539df731 100644 --- a/Uploader/File/FilesystemFile.php +++ b/Uploader/File/FilesystemFile.php @@ -2,21 +2,23 @@ namespace Oneup\UploaderBundle\Uploader\File; +use Symfony\Component\HttpFoundation\File\File; use Symfony\Component\HttpFoundation\File\UploadedFile; class FilesystemFile extends UploadedFile implements FileInterface { - public function __construct(UploadedFile $file) + public function __construct(File $file) { - parent::__construct($file->getPathname(), $file->getClientOriginalName(), $file->getClientMimeType(), $file->getClientSize(), $file->getError(), true); + if ($file instanceof UploadedFile) { + parent::__construct($file->getPathname(), $file->getClientOriginalName(), $file->getClientMimeType(), $file->getClientSize(), $file->getError(), true); + } else { + parent::__construct($file->getPathname(), $file->getBasename(), $file->getMimeType(), $file->getSize(), 0, true); + } } public function getExtension() { - // If the file is in tmp, it has no extension, but the wrapper object - // will have the original extension, otherwise it is better to rely - // on the actual extension - return parent::getExtension() ? :$this->getClientOriginalExtension(); + return $this->getClientOriginalExtension(); } } diff --git a/Uploader/Storage/FilesystemStorage.php b/Uploader/Storage/FilesystemStorage.php index f2613bff..ce626311 100644 --- a/Uploader/Storage/FilesystemStorage.php +++ b/Uploader/Storage/FilesystemStorage.php @@ -2,7 +2,7 @@ namespace Oneup\UploaderBundle\Uploader\Storage; -use Oneup\UploaderBundle\Uploader\File\FilesystemFile; +use Oneup\UploaderBundle\Uploader\File\FileInterface; class FilesystemStorage implements StorageInterface { @@ -13,7 +13,7 @@ public function __construct($directory) $this->directory = $directory; } - public function upload($file, $name, $path = null) + public function upload(FileInterface $file, $name, $path = null) { $path = is_null($path) ? $name : sprintf('%s/%s', $path, $name); $path = sprintf('%s/%s', $this->directory, $path); diff --git a/Uploader/Storage/GaufretteStorage.php b/Uploader/Storage/GaufretteStorage.php index 1c0bc9c6..f1ae5bd8 100644 --- a/Uploader/Storage/GaufretteStorage.php +++ b/Uploader/Storage/GaufretteStorage.php @@ -12,14 +12,14 @@ class GaufretteStorage extends StreamManager implements StorageInterface { protected $streamWrapperPrefix; - public function __construct(Filesystem $filesystem, $bufferSize, $streamWrapperPrefix) + public function __construct(Filesystem $filesystem, $bufferSize, $streamWrapperPrefix = null) { $this->filesystem = $filesystem; $this->bufferSize = $bufferSize; $this->streamWrapperPrefix = $streamWrapperPrefix; } - public function upload($file, $name, $path = null) + public function upload(FileInterface $file, $name, $path = null) { $path = is_null($path) ? $name : sprintf('%s/%s', $path, $name); diff --git a/Uploader/Storage/OrphanageStorage.php b/Uploader/Storage/OrphanageStorage.php index ffdd2ae8..22656aad 100644 --- a/Uploader/Storage/OrphanageStorage.php +++ b/Uploader/Storage/OrphanageStorage.php @@ -29,7 +29,7 @@ public function __construct(StorageInterface $storage, SessionInterface $session $this->type = $type; } - public function upload($file, $name, $path = null) + public function upload(FileInterface $file, $name, $path = null) { if(!$this->session->isStarted()) throw new \RuntimeException('You need a running session in order to run the Orphanage.'); @@ -44,7 +44,7 @@ public function uploadFiles() $return = array(); foreach ($files as $file) { - $return[] = $this->storage->upload(new File($file->getPathname()), str_replace($this->getFindPath(), '', $file)); + $return[] = $this->storage->upload(new FilesystemFile(new File($file->getPathname())), str_replace($this->getFindPath(), '', $file)); } return $return; diff --git a/Uploader/Storage/StorageInterface.php b/Uploader/Storage/StorageInterface.php index 0c7e4b55..31665d83 100644 --- a/Uploader/Storage/StorageInterface.php +++ b/Uploader/Storage/StorageInterface.php @@ -9,9 +9,9 @@ interface StorageInterface /** * Uploads a File instance to the configured storage. * - * @param $file - * @param string $name - * @param string $path + * @param $file + * @param string $name + * @param string $path */ - public function upload($file, $name, $path = null); + public function upload(FileInterface $file, $name, $path = null); } From c5a87ee373d09696d40afacb9cf5b81e8b6b4702 Mon Sep 17 00:00:00 2001 From: mitom Date: Thu, 10 Oct 2013 20:22:09 +0200 Subject: [PATCH 16/47] don't test for sf2.1 --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 0acf73bd..fc9e984a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,6 @@ php: - 5.4 env: - - SYMFONY_VERSION=2.1.* - SYMFONY_VERSION=2.2.* - SYMFONY_VERSION=2.3.* From db9177712e989569bbfab1f33189425d6116e272 Mon Sep 17 00:00:00 2001 From: mitom Date: Fri, 11 Oct 2013 09:57:36 +0200 Subject: [PATCH 17/47] Update configuration_reference.md --- Resources/doc/configuration_reference.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Resources/doc/configuration_reference.md b/Resources/doc/configuration_reference.md index b75fee44..0a712c94 100644 --- a/Resources/doc/configuration_reference.md +++ b/Resources/doc/configuration_reference.md @@ -7,7 +7,13 @@ All available configuration options along with their default values are listed b oneup_uploader: chunks: maxage: 604800 - directory: ~ + storage: + type: filesystem + directory: ~ + filesystem: ~ + sync_buffer_size: 100K + stream_wrapper: ~ + prefix: 'chunks' load_distribution: true orphanage: maxage: 604800 @@ -26,6 +32,7 @@ oneup_uploader: type: filesystem filesystem: ~ directory: ~ + stream_wrapper: ~ sync_buffer_size: 100K allowed_extensions: [] disallowed_extensions: [] From 4798b107492cfd8c6c483a4a95d86058761975dc Mon Sep 17 00:00:00 2001 From: mitom Date: Fri, 11 Oct 2013 10:37:31 +0200 Subject: [PATCH 18/47] Update chunked_uploads.md --- Resources/doc/chunked_uploads.md | 37 +++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/Resources/doc/chunked_uploads.md b/Resources/doc/chunked_uploads.md index 4f5c5e1c..7b0dadce 100644 --- a/Resources/doc/chunked_uploads.md +++ b/Resources/doc/chunked_uploads.md @@ -31,11 +31,46 @@ You can configure the `ChunkManager` by using the following configuration parame oneup_uploader: chunks: maxage: 86400 - directory: %kernel.cache_dir%/uploader/chunks + storage: + directory: %kernel.cache_dir%/uploader/chunks ``` You can choose a custom directory to save the chunks temporarily while uploading by changing the parameter `directory`. +Since version 1.0 you can also use a Gaufrette filesystem as the chunk storage. To do this you must first +set up [Gaufrette](gaufrette_storage.md).There are however some additional things to keep in mind. +The configuration for the Gaufrette chunk storage should look as the following: +``` +oneup_uploader: + chunks: + maxage: 86400 + storage: + type: gaufrette + filesystem: gaufrette.gallery_filesystem + prefix: 'chunks' + stream_wrapper: 'gaufrette://gallery/' +``` + +As you can see there are is a new option, ```prefix```. It represents the directory +in *relative* to the filesystem's directory which the chunks are stored in. +Gaufrette won't allow it to be outside of the filesystem. + +> You can only use stream capable filesystems for the chunk storage, at the time of this writing +only the Local filesystem is capable of streaming directly. + +This will give you a better structured directory, +as the chunk's folders and the uploaded files won't mix with each other. +> You can set it to an empty string (```''```), if you don't need it. Otherwise it defaults to ```chunks```. + +The chunks will be read directly from the tmp and appended to the already existing part on the given filesystem, +resulting in only 1 read and 1 write operation. + +You can achieve the biggest improvement if you use the same filesystem as your storage, as if you do so, the assembled +file only has to be moved out of the chunk directory, which on the same filesystem takes almost not time. + +> The load_distribution is forcefully turned on, if you use gaufrette as the chunk storage. + + ## Clean up The ChunkManager can be forced to clean up old and orphanaged chunks by using the command provided by the OneupUploaderBundle. From a47326e29906cb017ed5c653439305241be1a819 Mon Sep 17 00:00:00 2001 From: mitom Date: Fri, 11 Oct 2013 10:45:53 +0200 Subject: [PATCH 19/47] Update gaufrette_storage.md --- Resources/doc/gaufrette_storage.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/Resources/doc/gaufrette_storage.md b/Resources/doc/gaufrette_storage.md index 6c933477..c03ba849 100644 --- a/Resources/doc/gaufrette_storage.md +++ b/Resources/doc/gaufrette_storage.md @@ -88,3 +88,27 @@ oneup_uploader: sync_buffer_size: 1M ``` +You may also specify the stream wrapper protocol for your filesystem: +```yml +# app/config/config.yml + +oneup_uploader: + mappings: + gallery: + storage: + type: gaufrette + filesystem: gaufrette.gallery_filesystem + stream_wrapper: gaufrette://gallery/ +``` + +> This is only useful if you are using a stream capable adapter, at the time of this writing only +the local adapter is capable of streaming directly. + +The first part (```gaufrette```) in the example above ```MUST``` be the same as ```knp_gaufrette.stream_wrapper.protocol```, +the second part (```gallery```) in the example, ```MUST``` be the key of the filesytem (```knp_gaufette.filesystems.key```). +It also must end with a slash (```/```). + +This is particularly useful if you want to get exact informations about your files. Gaufrette offers you every functionality +to do this without relying on the stream wrapper, however it will have to download the file and load it into memory +to operate on it. If ```stream_wrapper``` is specified the bundle will try to open the file as streams when such operation +is requested.(e.g. getting the size of the file, the mime-type based on content) From 7bb3f2af115ac4462c8cdb5d58c7ca64bb8f4a3c Mon Sep 17 00:00:00 2001 From: mitom Date: Fri, 11 Oct 2013 10:48:44 +0200 Subject: [PATCH 20/47] Update chunked_uploads.md --- Resources/doc/chunked_uploads.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Resources/doc/chunked_uploads.md b/Resources/doc/chunked_uploads.md index 7b0dadce..b6c560d7 100644 --- a/Resources/doc/chunked_uploads.md +++ b/Resources/doc/chunked_uploads.md @@ -51,8 +51,10 @@ oneup_uploader: stream_wrapper: 'gaufrette://gallery/' ``` +> Setting the stream_wrapper is heavily recommended for better performance, see the reasons in the [gaufrette configuration](gaufrette_storage.md#configure-your-mappings) + As you can see there are is a new option, ```prefix```. It represents the directory -in *relative* to the filesystem's directory which the chunks are stored in. + *relative* to the filesystem's directory which the chunks are stored in. Gaufrette won't allow it to be outside of the filesystem. > You can only use stream capable filesystems for the chunk storage, at the time of this writing @@ -68,7 +70,7 @@ resulting in only 1 read and 1 write operation. You can achieve the biggest improvement if you use the same filesystem as your storage, as if you do so, the assembled file only has to be moved out of the chunk directory, which on the same filesystem takes almost not time. -> The load_distribution is forcefully turned on, if you use gaufrette as the chunk storage. +> The ```load distribution``` is forcefully turned on, if you use gaufrette as the chunk storage. ## Clean up From 160c9a163d34b03c5413cee193743bc0407471f1 Mon Sep 17 00:00:00 2001 From: mitom Date: Fri, 11 Oct 2013 10:50:13 +0200 Subject: [PATCH 21/47] Update custom_namer.md --- Resources/doc/custom_namer.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Resources/doc/custom_namer.md b/Resources/doc/custom_namer.md index 4aaeabcb..9b59e584 100644 --- a/Resources/doc/custom_namer.md +++ b/Resources/doc/custom_namer.md @@ -12,19 +12,19 @@ First, create a custom namer which implements ```Oneup\UploaderBundle\Uploader\N namespace Acme\DemoBundle; -use Symfony\Component\HttpFoundation\File\UploadedFile; +use Oneup\UploaderBundle\Uploader\File\FileInterface; use Oneup\UploaderBundle\Uploader\Naming\NamerInterface; class CatNamer implements NamerInterface { - public function name(UploadedFile $file) + public function name(FileInterface $file) { return 'grumpycat.jpg'; } } ``` -To match the `NamerInterface` you have to implement the function `name()` which expects an `UploadedFile` and should return a string representing the name of the given file. The example above would name every file _grumpycat.jpg_ and is therefore not very useful. +To match the `NamerInterface` you have to implement the function `name()` which expects an `FileInterface` and should return a string representing the name of the given file. The example above would name every file _grumpycat.jpg_ and is therefore not very useful. Next, register your created namer as a service in your `services.xml` From 082f119827d52101398ec25abd4fc1278e7b1ad1 Mon Sep 17 00:00:00 2001 From: mitom Date: Fri, 11 Oct 2013 13:07:30 +0200 Subject: [PATCH 22/47] orphanage for gaufrette chunk storage --- .../OneupUploaderExtension.php | 16 +++- Resources/config/uploader.xml | 2 +- Tests/Controller/BlueimpValidationTest.php | 1 - ...php => FilesystemOrphanageStorageTest.php} | 9 +- Uploader/Chunk/Storage/GaufretteStorage.php | 18 +++- Uploader/Gaufrette/StreamManager.php | 2 +- Uploader/Orphanage/OrphanageManager.php | 8 ++ ...age.php => FilesystemOrphanageStorage.php} | 6 +- .../Storage/GaufretteOrphanageStorage.php | 93 +++++++++++++++++++ 9 files changed, 141 insertions(+), 14 deletions(-) rename Tests/Uploader/Storage/{OrphanageStorageTest.php => FilesystemOrphanageStorageTest.php} (89%) rename Uploader/Storage/{OrphanageStorage.php => FilesystemOrphanageStorage.php} (85%) create mode 100644 Uploader/Storage/GaufretteOrphanageStorage.php diff --git a/DependencyInjection/OneupUploaderExtension.php b/DependencyInjection/OneupUploaderExtension.php index a86036f5..b25c844e 100644 --- a/DependencyInjection/OneupUploaderExtension.php +++ b/DependencyInjection/OneupUploaderExtension.php @@ -50,8 +50,13 @@ public function load(array $configs, ContainerBuilder $container) protected function processOrphanageConfig() { - $this->config['orphanage']['directory'] = is_null($this->config['orphanage']['directory']) ? - sprintf('%s/uploader/orphanage', $this->container->getParameter('kernel.cache_dir')) : + if ($this->config['chunks']['storage']['type'] === 'filesystem') { + $defaultDir = sprintf('%s/uploader/orphanage', $this->container->getParameter('kernel.cache_dir')); + } else { + $defaultDir = 'orphanage'; + } + + $this->config['orphanage']['directory'] = is_null($this->config['orphanage']['directory']) ? $defaultDir: $this->normalizePath($this->config['orphanage']['directory']) ; } @@ -75,7 +80,7 @@ protected function processMapping($key, &$mapping) protected function createController($key, $config) { // create the storage service according to the configuration - $storageService = $this->createStorageService($config['storage'], $key, isset($config['orphanage']) ? :null); + $storageService = $this->createStorageService($config['storage'], $key, $config['use_orphanage']); if ($config['frontend'] != 'custom') { $controllerName = sprintf('oneup_uploader.controller.%s', $key); @@ -149,13 +154,15 @@ protected function createChunkStorageService() $config['prefix'] ); + $this->container->setParameter('oneup_uploader.orphanage.class', 'Oneup\UploaderBundle\Uploader\Storage\GaufretteOrphanageStorage'); + // enforce load distribution when using gaufrette as chunk // torage to avoid moving files forth-and-back $this->config['chunks']['load_distribution'] = true; } } - protected function createStorageService(&$config, $key, $orphanage = null) + protected function createStorageService(&$config, $key, $orphanage = false) { $storageService = null; @@ -201,6 +208,7 @@ protected function createStorageService(&$config, $key, $orphanage = null) ->register($orphanageName, '%oneup_uploader.orphanage.class%') ->addArgument($storageService) ->addArgument(new Reference('session')) + ->addArgument(new Reference('oneup_uploader.chunks_storage')) ->addArgument($this->config['orphanage']) ->addArgument($key) ; diff --git a/Resources/config/uploader.xml b/Resources/config/uploader.xml index 66c7e70d..c9709943 100644 --- a/Resources/config/uploader.xml +++ b/Resources/config/uploader.xml @@ -11,7 +11,7 @@ Oneup\UploaderBundle\Routing\RouteLoader Oneup\UploaderBundle\Uploader\Storage\GaufretteStorage Oneup\UploaderBundle\Uploader\Storage\FilesystemStorage - Oneup\UploaderBundle\Uploader\Storage\OrphanageStorage + Oneup\UploaderBundle\Uploader\Storage\FilesystemOrphanageStorage Oneup\UploaderBundle\Uploader\Orphanage\OrphanageManager Oneup\UploaderBundle\Controller\FineUploaderController Oneup\UploaderBundle\Controller\BlueimpController diff --git a/Tests/Controller/BlueimpValidationTest.php b/Tests/Controller/BlueimpValidationTest.php index 19fea99c..95667fdd 100644 --- a/Tests/Controller/BlueimpValidationTest.php +++ b/Tests/Controller/BlueimpValidationTest.php @@ -4,7 +4,6 @@ use Symfony\Component\HttpFoundation\File\UploadedFile; use Oneup\UploaderBundle\Tests\Controller\AbstractValidationTest; -use Oneup\UploaderBundle\Event\ValidationEvent; use Oneup\UploaderBundle\UploadEvents; class BlueimpValidationTest extends AbstractValidationTest diff --git a/Tests/Uploader/Storage/OrphanageStorageTest.php b/Tests/Uploader/Storage/FilesystemOrphanageStorageTest.php similarity index 89% rename from Tests/Uploader/Storage/OrphanageStorageTest.php rename to Tests/Uploader/Storage/FilesystemOrphanageStorageTest.php index b2eed0a5..2278b966 100644 --- a/Tests/Uploader/Storage/OrphanageStorageTest.php +++ b/Tests/Uploader/Storage/FilesystemOrphanageStorageTest.php @@ -3,16 +3,17 @@ namespace Oneup\UploaderBundle\Tests\Uploader\Storage; use Oneup\UploaderBundle\Uploader\File\FilesystemFile; +use Oneup\UploaderBundle\Uploader\Storage\FilesystemOrphanageStorage; +use Oneup\UploaderBundle\Uploader\Chunk\Storage\FilesystemStorage as FilesystemChunkStorage; use Symfony\Component\Finder\Finder; use Symfony\Component\Filesystem\Filesystem; use Symfony\Component\HttpFoundation\File\UploadedFile; use Symfony\Component\HttpFoundation\Session\Session; use Symfony\Component\HttpFoundation\Session\Storage\MockArraySessionStorage; -use Oneup\UploaderBundle\Uploader\Storage\OrphanageStorage; use Oneup\UploaderBundle\Uploader\Storage\FilesystemStorage; -class OrphanageStorageTest extends \PHPUnit_Framework_TestCase +class FilesystemOrphanageStorageTest extends \PHPUnit_Framework_TestCase { protected $tempDirectory; protected $realDirectory; @@ -45,6 +46,8 @@ public function setUp() // create underlying storage $this->storage = new FilesystemStorage($this->realDirectory); + // is ignored anyways + $chunkStorage = new FilesystemChunkStorage('/tmp/'); // create orphanage $session = new Session(new MockArraySessionStorage()); @@ -52,7 +55,7 @@ public function setUp() $config = array('directory' => $this->tempDirectory); - $this->orphanage = new OrphanageStorage($this->storage, $session, $config, 'cat'); + $this->orphanage = new FilesystemOrphanageStorage($this->storage, $session, $chunkStorage, $config, 'cat'); } public function testUpload() diff --git a/Uploader/Chunk/Storage/GaufretteStorage.php b/Uploader/Chunk/Storage/GaufretteStorage.php index 4f5a2864..cfd55e3b 100644 --- a/Uploader/Chunk/Storage/GaufretteStorage.php +++ b/Uploader/Chunk/Storage/GaufretteStorage.php @@ -27,9 +27,18 @@ public function __construct(Filesystem $filesystem, $bufferSize, $streamWrapperP $this->streamWrapperPrefix = $streamWrapperPrefix; } - public function clear($maxAge) + /** + * Clears files and folders older than $maxAge in $prefix + * $prefix must be passable so it can clean the orphanage too + * as it is forced to be the same filesystem. + * + * @param $maxAge + * @param null $prefix + */ + public function clear($maxAge, $prefix = null) { - $matches = $this->filesystem->listKeys($this->prefix); + $prefix = $prefix ? :$this->prefix; + $matches = $this->filesystem->listKeys($prefix); $limit = time()+$maxAge; $toDelete = array(); @@ -140,4 +149,9 @@ public function getChunks($uuid) { return $this->filesystem->listKeys($this->prefix.'/'.$uuid)['keys']; } + + public function getFilesystem() + { + return $this->filesystem; + } } diff --git a/Uploader/Gaufrette/StreamManager.php b/Uploader/Gaufrette/StreamManager.php index 3ae16c5b..58868b88 100644 --- a/Uploader/Gaufrette/StreamManager.php +++ b/Uploader/Gaufrette/StreamManager.php @@ -10,7 +10,7 @@ class StreamManager { protected $filesystem; - protected $buffersize; + public $buffersize; protected function createSourceStream(FileInterface $file) { diff --git a/Uploader/Orphanage/OrphanageManager.php b/Uploader/Orphanage/OrphanageManager.php index b09dd25c..6903db76 100644 --- a/Uploader/Orphanage/OrphanageManager.php +++ b/Uploader/Orphanage/OrphanageManager.php @@ -24,6 +24,14 @@ public function get($key) public function clear() { + // Really ugly solution to clearing the orphanage on gaufrette + $class = $this->container->getParameter('oneup_uploader.orphanage.class'); + if ($class === 'Oneup\UploaderBundle\Uploader\Storage\GaufretteOrphanageStorage') { + $chunkStorage = $this->container->get('oneup_uploader.chunks_storage '); + $chunkStorage->clear($this->config['maxage'], $this->config['directory']); + + return; + } $system = new Filesystem(); $finder = new Finder(); diff --git a/Uploader/Storage/OrphanageStorage.php b/Uploader/Storage/FilesystemOrphanageStorage.php similarity index 85% rename from Uploader/Storage/OrphanageStorage.php rename to Uploader/Storage/FilesystemOrphanageStorage.php index 22656aad..e0026cca 100644 --- a/Uploader/Storage/OrphanageStorage.php +++ b/Uploader/Storage/FilesystemOrphanageStorage.php @@ -2,6 +2,7 @@ namespace Oneup\UploaderBundle\Uploader\Storage; +use Oneup\UploaderBundle\Uploader\Chunk\Storage\ChunkStorageInterface; use Oneup\UploaderBundle\Uploader\File\FileInterface; use Oneup\UploaderBundle\Uploader\File\FilesystemFile; use Symfony\Component\HttpFoundation\Session\SessionInterface; @@ -12,17 +13,18 @@ use Oneup\UploaderBundle\Uploader\Storage\StorageInterface; use Oneup\UploaderBundle\Uploader\Storage\OrphanageStorageInterface; -class OrphanageStorage extends FilesystemStorage implements OrphanageStorageInterface +class FilesystemOrphanageStorage extends FilesystemStorage implements OrphanageStorageInterface { protected $storage; protected $session; protected $config; protected $type; - public function __construct(StorageInterface $storage, SessionInterface $session, $config, $type) + public function __construct(StorageInterface $storage, SessionInterface $session, ChunkStorageInterface $chunkStorage, $config, $type) { parent::__construct($config['directory']); + // We can just ignore the chunkstorage here, it's not needed to access the files $this->storage = $storage; $this->session = $session; $this->config = $config; diff --git a/Uploader/Storage/GaufretteOrphanageStorage.php b/Uploader/Storage/GaufretteOrphanageStorage.php new file mode 100644 index 00000000..40f83f95 --- /dev/null +++ b/Uploader/Storage/GaufretteOrphanageStorage.php @@ -0,0 +1,93 @@ +getFilesystem(), $chunkStorage->bufferSize, null, null); + + $this->storage = $storage; + $this->chunkStorage = $chunkStorage; + $this->session = $session; + $this->config = $config; + $this->type = $type; + } + + public function upload(FileInterface $file, $name, $path = null) + { + if(!$this->session->isStarted()) + throw new \RuntimeException('You need a running session in order to run the Orphanage.'); + + return parent::upload($file, $name, $this->getPath()); + } + + public function uploadFiles() + { + try { + $files = $this->getFiles(); + $return = array(); + + foreach ($files as $key => $file) { + try { + $return[] = $this->storage->upload($file, str_replace($this->getPath(), '', $key)); + } catch (\Exception $e) { + // don't delete the file, if the upload failed beaces + // 1, it may not exist and deleting would throw another exception; + // 2, don't lose it on network related errors + continue; + } + + $this->chunkStorage->getFilesystem()->delete($key); + } + + return $return; + } catch (\Exception $e) { + return array(); + } + } + + public function getFiles() + { + $keys = $this->chunkStorage->getFilesystem()->listKeys($this->getPath())['keys']; + $files = array(); + + foreach ($keys as $key) { + // gotta pass the filesystem to both as you can't get it out from one.. + $files[$key] = new GaufretteFile(new File($key, $this->chunkStorage->getFilesystem()), $this->chunkStorage->getFilesystem()); + } + + return $files; + } + + protected function getPath() + { + // the storage is initiated in the root of the filesystem, from where the orphanage directory + // should be relative. + return sprintf('%s/%s/%s', $this->config['directory'], $this->session->getId(), $this->type); + } + +} From 94d6d30f78c96faa59f3b90fc8994571d89d73b6 Mon Sep 17 00:00:00 2001 From: mitom Date: Fri, 11 Oct 2013 13:18:30 +0200 Subject: [PATCH 23/47] typo --- Uploader/Storage/GaufretteOrphanageStorage.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Uploader/Storage/GaufretteOrphanageStorage.php b/Uploader/Storage/GaufretteOrphanageStorage.php index 40f83f95..ec02d6b7 100644 --- a/Uploader/Storage/GaufretteOrphanageStorage.php +++ b/Uploader/Storage/GaufretteOrphanageStorage.php @@ -55,7 +55,7 @@ public function uploadFiles() try { $return[] = $this->storage->upload($file, str_replace($this->getPath(), '', $key)); } catch (\Exception $e) { - // don't delete the file, if the upload failed beaces + // don't delete the file, if the upload failed because // 1, it may not exist and deleting would throw another exception; // 2, don't lose it on network related errors continue; From 0223eea4a97a72fe27f6a157ca5c9306c7199149 Mon Sep 17 00:00:00 2001 From: mitom Date: Fri, 11 Oct 2013 13:25:01 +0200 Subject: [PATCH 24/47] Update orphanage.md --- Resources/doc/orphanage.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Resources/doc/orphanage.md b/Resources/doc/orphanage.md index 59478cde..b4051a23 100644 --- a/Resources/doc/orphanage.md +++ b/Resources/doc/orphanage.md @@ -67,6 +67,12 @@ oneup_uploader: You can choose a custom directory to save the orphans temporarily while uploading by changing the parameter `directory`. +If you are using a gaufrette filesystem as the chunk storage, the ```directory``` specified above should be +relative to the filesystem's root directory. It will detect if you are using a gaufrette chunk storage +and default to ```orphanage```. + +> The orphanage and the chunk storage are forced to be on the same filesystem. + ## Clean up The `OrphanageManager` can be forced to clean up orphans by using the command provided by the OneupUploaderBundle. @@ -79,4 +85,4 @@ The `Orphanage` will save uploaded files in a directory like the following: %kernel.cache_dir%/uploader/orphanage/{session_id}/uploaded_file.ext -It is currently not possible to change the part after `%kernel.cache_dir%/uploader/orphanage` dynamically. This has some implications. If a user will upload files through your `gallery` mapping, and choose not to submit the form, but instead start over with a new form handled by the `gallery` mapping, the newly uploaded files are going to be moved in the same directory. Therefore you will get both the files uploaded the first time and the second time if you trigger the `uploadFiles` method. \ No newline at end of file +It is currently not possible to change the part after `%kernel.cache_dir%/uploader/orphanage` dynamically. This has some implications. If a user will upload files through your `gallery` mapping, and choose not to submit the form, but instead start over with a new form handled by the `gallery` mapping, the newly uploaded files are going to be moved in the same directory. Therefore you will get both the files uploaded the first time and the second time if you trigger the `uploadFiles` method. From 155aca367c31eaed594f45ff88b8ce9de2adfd01 Mon Sep 17 00:00:00 2001 From: mitom Date: Fri, 11 Oct 2013 14:25:00 +0200 Subject: [PATCH 25/47] gaufrette orphanage fixes and tests --- .../FilesystemOrphanageStorageTest.php | 78 +------------------ .../Storage/GaufretteOrphanageStorage.php | 6 +- 2 files changed, 2 insertions(+), 82 deletions(-) diff --git a/Tests/Uploader/Storage/FilesystemOrphanageStorageTest.php b/Tests/Uploader/Storage/FilesystemOrphanageStorageTest.php index 2278b966..d6478483 100644 --- a/Tests/Uploader/Storage/FilesystemOrphanageStorageTest.php +++ b/Tests/Uploader/Storage/FilesystemOrphanageStorageTest.php @@ -5,7 +5,6 @@ use Oneup\UploaderBundle\Uploader\File\FilesystemFile; use Oneup\UploaderBundle\Uploader\Storage\FilesystemOrphanageStorage; use Oneup\UploaderBundle\Uploader\Chunk\Storage\FilesystemStorage as FilesystemChunkStorage; -use Symfony\Component\Finder\Finder; use Symfony\Component\Filesystem\Filesystem; use Symfony\Component\HttpFoundation\File\UploadedFile; use Symfony\Component\HttpFoundation\Session\Session; @@ -13,15 +12,8 @@ use Oneup\UploaderBundle\Uploader\Storage\FilesystemStorage; -class FilesystemOrphanageStorageTest extends \PHPUnit_Framework_TestCase +class FilesystemOrphanageStorageTest extends OrphanageTest { - protected $tempDirectory; - protected $realDirectory; - protected $orphanage; - protected $storage; - protected $payloads; - protected $numberOfPayloads; - public function setUp() { $this->numberOfPayloads = 5; @@ -57,72 +49,4 @@ public function setUp() $this->orphanage = new FilesystemOrphanageStorage($this->storage, $session, $chunkStorage, $config, 'cat'); } - - public function testUpload() - { - for ($i = 0; $i < $this->numberOfPayloads; $i ++) { - $this->orphanage->upload($this->payloads[$i], $i . 'notsogrumpyanymore.jpeg'); - } - - $finder = new Finder(); - $finder->in($this->tempDirectory)->files(); - $this->assertCount($this->numberOfPayloads, $finder); - - $finder = new Finder(); - $finder->in($this->realDirectory)->files(); - $this->assertCount(0, $finder); - } - - public function testUploadAndFetching() - { - for ($i = 0; $i < $this->numberOfPayloads; $i ++) { - $this->orphanage->upload($this->payloads[$i], $i . 'notsogrumpyanymore.jpeg'); - } - - $finder = new Finder(); - $finder->in($this->tempDirectory)->files(); - $this->assertCount($this->numberOfPayloads, $finder); - - $finder = new Finder(); - $finder->in($this->realDirectory)->files(); - $this->assertCount(0, $finder); - - $files = $this->orphanage->uploadFiles(); - - $this->assertTrue(is_array($files)); - $this->assertCount($this->numberOfPayloads, $files); - - $finder = new Finder(); - $finder->in($this->tempDirectory)->files(); - $this->assertCount(0, $finder); - - $finder = new Finder(); - $finder->in($this->realDirectory)->files(); - $this->assertCount($this->numberOfPayloads, $finder); - } - - public function testUploadAndFetchingIfDirectoryDoesNotExist() - { - $filesystem = new Filesystem(); - $filesystem->remove($this->tempDirectory); - - $files = $this->orphanage->uploadFiles(); - - $this->assertTrue(is_array($files)); - $this->assertCount(0, $files); - } - - public function testIfGetFilesMethodIsAccessible() - { - // since ticket #48, getFiles has to be public - $method = new \ReflectionMethod($this->orphanage, 'getFiles'); - $this->assertTrue($method->isPublic()); - } - - public function tearDown() - { - $filesystem = new Filesystem(); - $filesystem->remove($this->tempDirectory); - $filesystem->remove($this->realDirectory); - } } diff --git a/Uploader/Storage/GaufretteOrphanageStorage.php b/Uploader/Storage/GaufretteOrphanageStorage.php index ec02d6b7..e087c1b6 100644 --- a/Uploader/Storage/GaufretteOrphanageStorage.php +++ b/Uploader/Storage/GaufretteOrphanageStorage.php @@ -55,13 +55,9 @@ public function uploadFiles() try { $return[] = $this->storage->upload($file, str_replace($this->getPath(), '', $key)); } catch (\Exception $e) { - // don't delete the file, if the upload failed because - // 1, it may not exist and deleting would throw another exception; - // 2, don't lose it on network related errors + // well, we tried. continue; } - - $this->chunkStorage->getFilesystem()->delete($key); } return $return; From 71acfbea43e12eeac73d76c72be2f35ba2c11f33 Mon Sep 17 00:00:00 2001 From: mitom Date: Fri, 11 Oct 2013 14:27:09 +0200 Subject: [PATCH 26/47] added missing files --- .../Storage/GaufretteOrphanageStorageTest.php | 110 ++++++++++++++++++ Tests/Uploader/Storage/OrphanageTest.php | 85 ++++++++++++++ 2 files changed, 195 insertions(+) create mode 100644 Tests/Uploader/Storage/GaufretteOrphanageStorageTest.php create mode 100644 Tests/Uploader/Storage/OrphanageTest.php diff --git a/Tests/Uploader/Storage/GaufretteOrphanageStorageTest.php b/Tests/Uploader/Storage/GaufretteOrphanageStorageTest.php new file mode 100644 index 00000000..bc3a6710 --- /dev/null +++ b/Tests/Uploader/Storage/GaufretteOrphanageStorageTest.php @@ -0,0 +1,110 @@ +numberOfPayloads = 5; + $this->realDirectory = sys_get_temp_dir() . '/storage'; + $this->chunkDirectory = $this->realDirectory .'/' . $this->chunksKey; + $this->tempDirectory = $this->realDirectory . '/' . $this->orphanageKey; + $this->payloads = array(); + + $filesystem = new \Symfony\Component\Filesystem\Filesystem(); + $filesystem->mkdir($this->realDirectory); + $filesystem->mkdir($this->chunkDirectory); + $filesystem->mkdir($this->tempDirectory); + + $adapter = new Adapter($this->realDirectory, true); + $filesystem = new GaufretteFilesystem($adapter); + + $this->storage = new GaufretteStorage($filesystem, 100000); + + $chunkStorage = new GaufretteChunkStorage($filesystem, 100000, null, 'chunks'); + + // create orphanage + $session = new Session(new MockArraySessionStorage()); + $session->start(); + + $config = array('directory' => 'orphanage'); + + $this->orphanage = new GaufretteOrphanageStorage($this->storage, $session, $chunkStorage, $config, 'cat'); + + for ($i = 0; $i < $this->numberOfPayloads; $i ++) { + // create temporary file as if it was reassembled by the chunk manager + $file = tempnam($this->chunkDirectory, 'uploader'); + + $pointer = fopen($file, 'w+'); + fwrite($pointer, str_repeat('A', 1024), 1024); + fclose($pointer); + + //gaufrette needs the key relative to it's root + $fileKey = str_replace($this->realDirectory, '', $file); + + $this->payloads[] = new GaufretteFile(new File($fileKey, $filesystem), $filesystem); + } + } + + public function testUpload() + { + for ($i = 0; $i < $this->numberOfPayloads; $i ++) { + $this->orphanage->upload($this->payloads[$i], $i . 'notsogrumpyanymore.jpeg'); + } + + $finder = new Finder(); + $finder->in($this->tempDirectory)->files(); + $this->assertCount($this->numberOfPayloads, $finder); + + $finder = new Finder(); + // exclude the orphanage and the chunks + $finder->in($this->realDirectory)->exclude(array($this->orphanageKey, $this->chunksKey))->files(); + $this->assertCount(0, $finder); + } + + public function testUploadAndFetching() + { + for ($i = 0; $i < $this->numberOfPayloads; $i ++) { + $this->orphanage->upload($this->payloads[$i], $i . 'notsogrumpyanymore.jpeg'); + } + + $finder = new Finder(); + $finder->in($this->tempDirectory)->files(); + $this->assertCount($this->numberOfPayloads, $finder); + + $finder = new Finder(); + $finder->in($this->realDirectory)->exclude(array($this->orphanageKey, $this->chunksKey))->files(); + $this->assertCount(0, $finder); + + $files = $this->orphanage->uploadFiles(); + + $this->assertTrue(is_array($files)); + $this->assertCount($this->numberOfPayloads, $files); + + $finder = new Finder(); + $finder->in($this->tempDirectory)->files(); + $this->assertCount(0, $finder); + + $finder = new Finder(); + $finder->in($this->realDirectory)->files(); + $this->assertCount($this->numberOfPayloads, $finder); + } + +} diff --git a/Tests/Uploader/Storage/OrphanageTest.php b/Tests/Uploader/Storage/OrphanageTest.php new file mode 100644 index 00000000..bc536acd --- /dev/null +++ b/Tests/Uploader/Storage/OrphanageTest.php @@ -0,0 +1,85 @@ +numberOfPayloads; $i ++) { + $this->orphanage->upload($this->payloads[$i], $i . 'notsogrumpyanymore.jpeg'); + } + + $finder = new Finder(); + $finder->in($this->tempDirectory)->files(); + $this->assertCount($this->numberOfPayloads, $finder); + + $finder = new Finder(); + $finder->in($this->realDirectory)->files(); + $this->assertCount(0, $finder); + } + + public function testUploadAndFetching() + { + for ($i = 0; $i < $this->numberOfPayloads; $i ++) { + $this->orphanage->upload($this->payloads[$i], $i . 'notsogrumpyanymore.jpeg'); + } + + $finder = new Finder(); + $finder->in($this->tempDirectory)->files(); + $this->assertCount($this->numberOfPayloads, $finder); + + $finder = new Finder(); + $finder->in($this->realDirectory)->files(); + $this->assertCount(0, $finder); + + $files = $this->orphanage->uploadFiles(); + + $this->assertTrue(is_array($files)); + $this->assertCount($this->numberOfPayloads, $files); + + $finder = new Finder(); + $finder->in($this->tempDirectory)->files(); + $this->assertCount(0, $finder); + + $finder = new Finder(); + $finder->in($this->realDirectory)->files(); + $this->assertCount($this->numberOfPayloads, $finder); + } + + public function testUploadAndFetchingIfDirectoryDoesNotExist() + { + $filesystem = new Filesystem(); + $filesystem->remove($this->tempDirectory); + + $files = $this->orphanage->uploadFiles(); + + $this->assertTrue(is_array($files)); + $this->assertCount(0, $files); + } + + public function testIfGetFilesMethodIsAccessible() + { + // since ticket #48, getFiles has to be public + $method = new \ReflectionMethod($this->orphanage, 'getFiles'); + $this->assertTrue($method->isPublic()); + } + + public function tearDown() + { + $filesystem = new Filesystem(); + $filesystem->remove($this->tempDirectory); + $filesystem->remove($this->realDirectory); + } +} From 98f2a903a34bb4f1cfd4b7fd1f73bbde6f9ae5f4 Mon Sep 17 00:00:00 2001 From: mitom Date: Fri, 11 Oct 2013 15:08:59 +0200 Subject: [PATCH 27/47] fix for php5.3 --- Tests/Uploader/Storage/OrphanageTest.php | 1 - Uploader/Chunk/Storage/GaufretteStorage.php | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Tests/Uploader/Storage/OrphanageTest.php b/Tests/Uploader/Storage/OrphanageTest.php index bc536acd..836ab860 100644 --- a/Tests/Uploader/Storage/OrphanageTest.php +++ b/Tests/Uploader/Storage/OrphanageTest.php @@ -4,7 +4,6 @@ use Symfony\Component\Finder\Finder; use Symfony\Component\Filesystem\Filesystem; -use Symfony\Component\HttpFoundation\Session\Session; abstract class OrphanageTest extends \PHPUnit_Framework_Testcase { diff --git a/Uploader/Chunk/Storage/GaufretteStorage.php b/Uploader/Chunk/Storage/GaufretteStorage.php index cfd55e3b..6c4cd042 100644 --- a/Uploader/Chunk/Storage/GaufretteStorage.php +++ b/Uploader/Chunk/Storage/GaufretteStorage.php @@ -147,7 +147,9 @@ public function cleanup($path) public function getChunks($uuid) { - return $this->filesystem->listKeys($this->prefix.'/'.$uuid)['keys']; + $results = $this->filesystem->listKeys($this->prefix.'/'.$uuid); + + return $results['keys']; } public function getFilesystem() From 052148a91278b16bc592ce65ed9b1dddb5105ad6 Mon Sep 17 00:00:00 2001 From: mitom Date: Fri, 11 Oct 2013 16:01:43 +0200 Subject: [PATCH 28/47] gaufette chunk storage test --- .../Chunk/Storage/ChunkStorageTest.php | 73 ++++++++++++++++++ .../Chunk/Storage/FilesystemStorageTest.php | 76 +------------------ .../Chunk/Storage/GaufretteStorageTest.php | 43 +++++++++++ Uploader/Chunk/Storage/GaufretteStorage.php | 6 +- 4 files changed, 123 insertions(+), 75 deletions(-) create mode 100644 Tests/Uploader/Chunk/Storage/ChunkStorageTest.php create mode 100644 Tests/Uploader/Chunk/Storage/GaufretteStorageTest.php diff --git a/Tests/Uploader/Chunk/Storage/ChunkStorageTest.php b/Tests/Uploader/Chunk/Storage/ChunkStorageTest.php new file mode 100644 index 00000000..af1f32fc --- /dev/null +++ b/Tests/Uploader/Chunk/Storage/ChunkStorageTest.php @@ -0,0 +1,73 @@ +assertTrue(is_dir($this->tmpDir)); + $this->assertTrue(is_writeable($this->tmpDir)); + } + + public function testFillOfTmpDir() + { + $finder = new Finder(); + $finder->in($this->tmpDir); + + $numberOfFiles = 10; + + $this->fillDirectory($numberOfFiles); + $this->assertCount($numberOfFiles, $finder); + } + + public function testChunkCleanup() + { + // get a manager configured with a max-age of 5 minutes + $maxage = 5 * 60; + $numberOfFiles = 10; + + $finder = new Finder(); + $finder->in($this->tmpDir); + + $this->fillDirectory($numberOfFiles); + $this->assertCount($numberOfFiles, $finder); + + $this->storage->clear($maxage); + + $this->assertTrue(is_dir($this->tmpDir)); + $this->assertTrue(is_writeable($this->tmpDir)); + + $this->assertCount(5, $finder); + + foreach ($finder as $file) { + $this->assertGreaterThanOrEqual(time() - $maxage, filemtime($file)); + } + } + + public function testClearIfDirectoryDoesNotExist() + { + $filesystem = new Filesystem(); + $filesystem->remove($this->tmpDir); + + $this->storage->clear(10); + + // yey, no exception + $this->assertTrue(true); + } + + protected function fillDirectory($number) + { + $system = new Filesystem(); + + for ($i = 0; $i < $number; $i ++) { + $system->touch(sprintf('%s/%s', $this->tmpDir, uniqid()), time() - $i * 60); + } + } +} diff --git a/Tests/Uploader/Chunk/Storage/FilesystemStorageTest.php b/Tests/Uploader/Chunk/Storage/FilesystemStorageTest.php index 1df65ea3..8e54cc4d 100644 --- a/Tests/Uploader/Chunk/Storage/FilesystemStorageTest.php +++ b/Tests/Uploader/Chunk/Storage/FilesystemStorageTest.php @@ -2,11 +2,10 @@ namespace Oneup\UploaderBundle\Tests\Uploader\Chunk\Storage; -use Symfony\Component\Finder\Finder; use Symfony\Component\Filesystem\Filesystem; use Oneup\UploaderBundle\Uploader\Chunk\Storage\FilesystemStorage; -class FilesystemStorageTest extends \PHPUnit_Framework_TestCase +class FilesystemStorageTest extends ChunkStorageTest { protected $tmpDir; @@ -19,81 +18,14 @@ public function setUp() $system->mkdir($tmpDir); $this->tmpDir = $tmpDir; - } - - public function tearDown() - { - $system = new Filesystem(); - $system->remove($this->tmpDir); - } - - public function testExistanceOfTmpDir() - { - $this->assertTrue(is_dir($this->tmpDir)); - $this->assertTrue(is_writeable($this->tmpDir)); - } - - public function testFillOfTmpDir() - { - $finder = new Finder(); - $finder->in($this->tmpDir); - - $numberOfFiles = 10; - - $this->fillDirectory($numberOfFiles); - $this->assertCount($numberOfFiles, $finder); - } - - public function testChunkCleanup() - { - // get a manager configured with a max-age of 5 minutes - $maxage = 5 * 60; - $manager = $this->getStorage(); - $numberOfFiles = 10; - - $finder = new Finder(); - $finder->in($this->tmpDir); - - $this->fillDirectory($numberOfFiles); - $this->assertCount(10, $finder); - - $manager->clear($maxage); - - $this->assertTrue(is_dir($this->tmpDir)); - $this->assertTrue(is_writeable($this->tmpDir)); - - $this->assertCount(5, $finder); - - foreach ($finder as $file) { - $this->assertGreaterThanOrEqual(time() - $maxage, filemtime($file)); - } - } - - public function testClearIfDirectoryDoesNotExist() - { - $filesystem = new Filesystem(); - $filesystem->remove($this->tmpDir); - - $manager = $this->getStorage(); - $manager->clear(10); - - // yey, no exception - $this->assertTrue(true); - } - - protected function getStorage() - { - return new FilesystemStorage(array( + $this->storage = new FilesystemStorage(array( 'directory' => $this->tmpDir )); } - protected function fillDirectory($number) + public function tearDown() { $system = new Filesystem(); - - for ($i = 0; $i < $number; $i ++) { - $system->touch(sprintf('%s/%s', $this->tmpDir, uniqid()), time() - $i * 60); - } + $system->remove($this->tmpDir); } } diff --git a/Tests/Uploader/Chunk/Storage/GaufretteStorageTest.php b/Tests/Uploader/Chunk/Storage/GaufretteStorageTest.php new file mode 100644 index 00000000..375d4505 --- /dev/null +++ b/Tests/Uploader/Chunk/Storage/GaufretteStorageTest.php @@ -0,0 +1,43 @@ +mkdir($parentDir); + + $this->parentDir = $parentDir; + + $adapter = new Adapter($this->parentDir, true); + + $filesystem = new GaufretteFilesystem($adapter); + + $this->storage = new GaufretteStorage($filesystem, 100000, null, $this->chunkKey); + $this->tmpDir = $this->parentDir.'/'.$this->chunkKey; + + $system->mkdir($this->tmpDir); + + } + + public function tearDown() + { + $system = new Filesystem(); + $system->remove($this->parentDir); + } + +} diff --git a/Uploader/Chunk/Storage/GaufretteStorage.php b/Uploader/Chunk/Storage/GaufretteStorage.php index 6c4cd042..5c641d00 100644 --- a/Uploader/Chunk/Storage/GaufretteStorage.php +++ b/Uploader/Chunk/Storage/GaufretteStorage.php @@ -40,7 +40,7 @@ public function clear($maxAge, $prefix = null) $prefix = $prefix ? :$this->prefix; $matches = $this->filesystem->listKeys($prefix); - $limit = time()+$maxAge; + $now = time(); $toDelete = array(); // Collect the directories that are old, @@ -48,14 +48,14 @@ public function clear($maxAge, $prefix = null) // but after the files are deleted the dirs // would remain foreach ($matches['dirs'] as $key) { - if ($limit < $this->filesystem->mtime($key)) { + if ($maxAge <= $now-$this->filesystem->mtime($key)) { $toDelete[] = $key; } } // The same directory is returned for every file it contains array_unique($toDelete); foreach ($matches['keys'] as $key) { - if ($limit < $this->filesystem->mtime($key)) { + if ($maxAge <= $now-$this->filesystem->mtime($key)) { $this->filesystem->delete($key); } } From 529b01cac232a81430772f8c5d9ae09f0c1a8d74 Mon Sep 17 00:00:00 2001 From: mitom Date: Fri, 11 Oct 2013 16:51:37 +0200 Subject: [PATCH 29/47] file tests --- Tests/Uploader/File/FileTest.php | 43 +++++++++++++++++++++ Tests/Uploader/File/FilesystemFileTest.php | 30 +++++++++++++++ Tests/Uploader/File/GaufretteFileTest.php | 44 ++++++++++++++++++++++ 3 files changed, 117 insertions(+) create mode 100644 Tests/Uploader/File/FileTest.php create mode 100644 Tests/Uploader/File/FilesystemFileTest.php create mode 100644 Tests/Uploader/File/GaufretteFileTest.php diff --git a/Tests/Uploader/File/FileTest.php b/Tests/Uploader/File/FileTest.php new file mode 100644 index 00000000..2753089c --- /dev/null +++ b/Tests/Uploader/File/FileTest.php @@ -0,0 +1,43 @@ +assertEquals($this->pathname, $this->file->getPathname()); + } + + public function testGetPath() + { + $this->assertEquals($this->path, $this->file->getPath()); + } + + public function testGetBasename() + { + $this->assertEquals($this->basename, $this->file->getBasename()); + } + + public function testGetExtension() + { + $this->assertEquals($this->extension, $this->file->getExtension()); + } + + public function testGetSize() + { + $this->assertEquals($this->size, $this->file->getSize()); + } + + public function testGetMimeType() + { + $this->assertEquals($this->mimeType, $this->file->getMimeType()); + } +} diff --git a/Tests/Uploader/File/FilesystemFileTest.php b/Tests/Uploader/File/FilesystemFileTest.php new file mode 100644 index 00000000..0868cb91 --- /dev/null +++ b/Tests/Uploader/File/FilesystemFileTest.php @@ -0,0 +1,30 @@ +path = sys_get_temp_dir(). '/oneup_test_tmp'; + mkdir($this->path); + + $this->basename = 'test_file.txt'; + $this->pathname = $this->path .'/'. $this->basename; + $this->extension = 'txt'; + $this->size = 9; //something = 9 bytes + $this->mimeType = 'text/plain'; + + file_put_contents($this->pathname, 'something'); + + $this->file = new FilesystemFile(new UploadedFile($this->pathname, 'test_file.txt', null, null, null, true)); + } + + public function tearDown() + { + unlink($this->pathname); + rmdir($this->path); + } +} diff --git a/Tests/Uploader/File/GaufretteFileTest.php b/Tests/Uploader/File/GaufretteFileTest.php new file mode 100644 index 00000000..fc7e2f7b --- /dev/null +++ b/Tests/Uploader/File/GaufretteFileTest.php @@ -0,0 +1,44 @@ +set('oneup', $filesystem); + + StreamWrapper::register(); + + $this->storage = new GaufretteStorage($filesystem, 100000); + + $this->path = 'oneup_test_tmp'; + mkdir(sys_get_temp_dir().'/'.$this->path); + + $this->basename = 'test_file.txt'; + $this->pathname = $this->path .'/'. $this->basename; + $this->extension = 'txt'; + $this->size = 9; //something = 9 bytes + $this->mimeType = 'text/plain'; + + file_put_contents(sys_get_temp_dir() .'/' . $this->pathname, 'something'); + + $this->file = new GaufretteFile(new File($this->pathname, $filesystem), $filesystem, 'gaufrette://oneup/'); + } + + public function tearDown() + { + unlink(sys_get_temp_dir().'/'.$this->pathname); + rmdir(sys_get_temp_dir().'/'.$this->path); + } +} From 71d6118a7598701727b72b2d712ad231184a8778 Mon Sep 17 00:00:00 2001 From: mitom Date: Fri, 11 Oct 2013 17:14:24 +0200 Subject: [PATCH 30/47] another php5.3 fix --- Uploader/Storage/GaufretteOrphanageStorage.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Uploader/Storage/GaufretteOrphanageStorage.php b/Uploader/Storage/GaufretteOrphanageStorage.php index e087c1b6..ea5379fb 100644 --- a/Uploader/Storage/GaufretteOrphanageStorage.php +++ b/Uploader/Storage/GaufretteOrphanageStorage.php @@ -68,7 +68,8 @@ public function uploadFiles() public function getFiles() { - $keys = $this->chunkStorage->getFilesystem()->listKeys($this->getPath())['keys']; + $keys = $this->chunkStorage->getFilesystem()->listKeys($this->getPath()); + $keys = $keys['keys']; $files = array(); foreach ($keys as $key) { From 48b19bc18fb944fe8b05a70173bff94074b8f83b Mon Sep 17 00:00:00 2001 From: Jim Schmid Date: Fri, 11 Oct 2013 18:15:47 +0200 Subject: [PATCH 31/47] Removed extension validators. --- DependencyInjection/Configuration.php | 6 -- .../AllowedExtensionValidationListener.php | 21 ------- .../DisallowedExtensionValidationListener.php | 21 ------- Resources/config/validators.xml | 8 --- Resources/doc/configuration_reference.md | 2 - Tests/App/config/config.yml | 21 +++---- Tests/Controller/AbstractValidationTest.php | 40 +------------ Tests/Controller/BlueimpValidationTest.php | 56 +------------------ Tests/Controller/DropzoneValidationTest.php | 20 ------- .../Controller/FancyUploadValidationTest.php | 20 ------- .../Controller/FineUploaderValidationTest.php | 20 ------- Tests/Controller/PluploadValidationTest.php | 20 ------- Tests/Controller/UploadifyValidationTest.php | 20 ------- Tests/Controller/YUI3ValidationTest.php | 20 ------- 14 files changed, 10 insertions(+), 285 deletions(-) delete mode 100644 EventListener/AllowedExtensionValidationListener.php delete mode 100644 EventListener/DisallowedExtensionValidationListener.php diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index 97b1eef5..3ebfbec6 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -59,12 +59,6 @@ public function getConfigTreeBuilder() ->scalarNode('sync_buffer_size')->defaultValue('100K')->end() ->end() ->end() - ->arrayNode('allowed_extensions') - ->prototype('scalar')->end() - ->end() - ->arrayNode('disallowed_extensions') - ->prototype('scalar')->end() - ->end() ->arrayNode('allowed_mimetypes') ->prototype('scalar')->end() ->end() diff --git a/EventListener/AllowedExtensionValidationListener.php b/EventListener/AllowedExtensionValidationListener.php deleted file mode 100644 index 68c26e85..00000000 --- a/EventListener/AllowedExtensionValidationListener.php +++ /dev/null @@ -1,21 +0,0 @@ -getConfig(); - $file = $event->getFile(); - - $extension = pathinfo($file->getClientOriginalName(), PATHINFO_EXTENSION); - - if (count($config['allowed_extensions']) > 0 && !in_array($extension, $config['allowed_extensions'])) { - throw new ValidationException('error.whitelist'); - } - } -} diff --git a/EventListener/DisallowedExtensionValidationListener.php b/EventListener/DisallowedExtensionValidationListener.php deleted file mode 100644 index 9cdac58e..00000000 --- a/EventListener/DisallowedExtensionValidationListener.php +++ /dev/null @@ -1,21 +0,0 @@ -getConfig(); - $file = $event->getFile(); - - $extension = pathinfo($file->getClientOriginalName(), PATHINFO_EXTENSION); - - if (count($config['disallowed_extensions']) > 0 && in_array($extension, $config['disallowed_extensions'])) { - throw new ValidationException('error.blacklist'); - } - } -} diff --git a/Resources/config/validators.xml b/Resources/config/validators.xml index 5b2ffe00..4c3fc731 100644 --- a/Resources/config/validators.xml +++ b/Resources/config/validators.xml @@ -8,14 +8,6 @@ - - - - - - - - diff --git a/Resources/doc/configuration_reference.md b/Resources/doc/configuration_reference.md index b75fee44..07c420b9 100644 --- a/Resources/doc/configuration_reference.md +++ b/Resources/doc/configuration_reference.md @@ -27,8 +27,6 @@ oneup_uploader: filesystem: ~ directory: ~ sync_buffer_size: 100K - allowed_extensions: [] - disallowed_extensions: [] allowed_mimetypes: [] disallowed_mimetypes: [] error_handler: oneup_uploader.error_handler.noop diff --git a/Tests/App/config/config.yml b/Tests/App/config/config.yml index 619cebc7..a350ddb8 100644 --- a/Tests/App/config/config.yml +++ b/Tests/App/config/config.yml @@ -23,8 +23,7 @@ oneup_uploader: max_size: 256 storage: directory: %kernel.root_dir%/cache/%kernel.environment%/upload - allowed_extensions: [ "ok" ] - disallowed_extensions: [ "fail" ] + allowed_mimetypes: [ "image/jpg", "text/plain" ] disallowed_mimetypes: [ "image/gif" ] @@ -38,8 +37,7 @@ oneup_uploader: max_size: 256 storage: directory: %kernel.root_dir%/cache/%kernel.environment%/upload - allowed_extensions: [ "ok" ] - disallowed_extensions: [ "fail" ] + allowed_mimetypes: [ "image/jpg", "text/plain" ] disallowed_mimetypes: [ "image/gif" ] @@ -53,8 +51,7 @@ oneup_uploader: max_size: 256 storage: directory: %kernel.root_dir%/cache/%kernel.environment%/upload - allowed_extensions: [ "ok" ] - disallowed_extensions: [ "fail" ] + allowed_mimetypes: [ "image/jpg", "text/plain" ] disallowed_mimetypes: [ "image/gif" ] @@ -68,8 +65,7 @@ oneup_uploader: max_size: 256 storage: directory: %kernel.root_dir%/cache/%kernel.environment%/upload - allowed_extensions: [ "ok" ] - disallowed_extensions: [ "fail" ] + allowed_mimetypes: [ "image/jpg", "text/plain" ] disallowed_mimetypes: [ "image/gif" ] @@ -83,8 +79,7 @@ oneup_uploader: max_size: 256 storage: directory: %kernel.root_dir%/cache/%kernel.environment%/upload - allowed_extensions: [ "ok" ] - disallowed_extensions: [ "fail" ] + allowed_mimetypes: [ "image/jpg", "text/plain" ] disallowed_mimetypes: [ "image/gif" ] @@ -98,8 +93,7 @@ oneup_uploader: max_size: 256 storage: directory: %kernel.root_dir%/cache/%kernel.environment%/upload - allowed_extensions: [ "ok" ] - disallowed_extensions: [ "fail" ] + allowed_mimetypes: [ "image/jpg", "text/plain" ] disallowed_mimetypes: [ "image/gif" ] @@ -115,8 +109,7 @@ oneup_uploader: storage: directory: %kernel.root_dir%/cache/%kernel.environment%/upload error_handler: oneup_uploader.error_handler.blueimp - allowed_extensions: [ "ok" ] - disallowed_extensions: [ "fail" ] + allowed_mimetypes: [ "image/jpg", "text/plain" ] disallowed_mimetypes: [ "image/gif" ] diff --git a/Tests/Controller/AbstractValidationTest.php b/Tests/Controller/AbstractValidationTest.php index 01b0a6dc..8b56bfbd 100644 --- a/Tests/Controller/AbstractValidationTest.php +++ b/Tests/Controller/AbstractValidationTest.php @@ -7,8 +7,6 @@ abstract class AbstractValidationTest extends AbstractControllerTest { - abstract protected function getFileWithCorrectExtension(); - abstract protected function getFileWithIncorrectExtension(); abstract protected function getFileWithCorrectMimeType(); abstract protected function getFileWithIncorrectMimeType(); abstract protected function getOversizedFile(); @@ -27,26 +25,6 @@ public function testAgainstMaxSize() $this->assertCount(0, $this->getUploadedFiles()); } - public function testAgainstCorrectExtension() - { - // assemble a request - $client = $this->client; - $endpoint = $this->helper->endpoint($this->getConfigKey()); - - $client->request('POST', $endpoint, $this->getRequestParameters(), array($this->getFileWithCorrectExtension())); - $response = $client->getResponse(); - - $this->assertTrue($response->isSuccessful()); - $this->assertEquals($response->headers->get('Content-Type'), 'application/json'); - $this->assertCount(1, $this->getUploadedFiles()); - - foreach ($this->getUploadedFiles() as $file) { - $this->assertTrue($file->isFile()); - $this->assertTrue($file->isReadable()); - $this->assertEquals(128, $file->getSize()); - } - } - public function testEvents() { $client = $this->client; @@ -60,7 +38,7 @@ public function testEvents() ++ $validationCount; }); - $client->request('POST', $endpoint, $this->getRequestParameters(), array($this->getFileWithCorrectExtension())); + $client->request('POST', $endpoint, $this->getRequestParameters(), array($this->getFileWithCorrectMimeType())); $this->assertEquals(1, $validationCount); } @@ -82,25 +60,11 @@ public function testIfRequestIsAvailableInEvent() ++ $validationCount; }); - $client->request('POST', $endpoint, $this->getRequestParameters(), array($this->getFileWithCorrectExtension())); + $client->request('POST', $endpoint, $this->getRequestParameters(), array($this->getFileWithCorrectMimeType())); $this->assertEquals(1, $validationCount); } - public function testAgainstIncorrectExtension() - { - // assemble a request - $client = $this->client; - $endpoint = $this->helper->endpoint($this->getConfigKey()); - - $client->request('POST', $endpoint, $this->getRequestParameters(), array($this->getFileWithIncorrectExtension())); - $response = $client->getResponse(); - - //$this->assertTrue($response->isNotSuccessful()); - $this->assertEquals($response->headers->get('Content-Type'), 'application/json'); - $this->assertCount(0, $this->getUploadedFiles()); - } - public function testAgainstCorrectMimeType() { // assemble a request diff --git a/Tests/Controller/BlueimpValidationTest.php b/Tests/Controller/BlueimpValidationTest.php index 4bbfa1c7..c5b8af6e 100644 --- a/Tests/Controller/BlueimpValidationTest.php +++ b/Tests/Controller/BlueimpValidationTest.php @@ -23,26 +23,6 @@ public function testAgainstMaxSize() $this->assertCount(0, $this->getUploadedFiles()); } - public function testAgainstCorrectExtension() - { - // assemble a request - $client = $this->client; - $endpoint = $this->helper->endpoint($this->getConfigKey()); - - $client->request('POST', $endpoint, $this->getRequestParameters(), $this->getFileWithCorrectExtension(), array('HTTP_ACCEPT' => 'application/json')); - $response = $client->getResponse(); - - $this->assertTrue($response->isSuccessful()); - $this->assertEquals($response->headers->get('Content-Type'), 'application/json'); - $this->assertCount(1, $this->getUploadedFiles()); - - foreach ($this->getUploadedFiles() as $file) { - $this->assertTrue($file->isFile()); - $this->assertTrue($file->isReadable()); - $this->assertEquals(128, $file->getSize()); - } - } - public function testEvents() { $client = $this->client; @@ -56,25 +36,11 @@ public function testEvents() ++ $validationCount; }); - $client->request('POST', $endpoint, $this->getRequestParameters(), $this->getFileWithCorrectExtension()); + $client->request('POST', $endpoint, $this->getRequestParameters(), $this->getFileWithCorrectMimeType()); $this->assertEquals(1, $validationCount); } - public function testAgainstIncorrectExtension() - { - // assemble a request - $client = $this->client; - $endpoint = $this->helper->endpoint($this->getConfigKey()); - - $client->request('POST', $endpoint, $this->getRequestParameters(), $this->getFileWithIncorrectExtension(), array('HTTP_ACCEPT' => 'application/json')); - $response = $client->getResponse(); - - //$this->assertTrue($response->isNotSuccessful()); - $this->assertEquals($response->headers->get('Content-Type'), 'application/json'); - $this->assertCount(0, $this->getUploadedFiles()); - } - public function testAgainstCorrectMimeType() { // assemble a request @@ -131,26 +97,6 @@ protected function getOversizedFile() ))); } - protected function getFileWithCorrectExtension() - { - return array('files' => array(new UploadedFile( - $this->createTempFile(128), - 'cat.ok', - 'text/plain', - 128 - ))); - } - - protected function getFileWithIncorrectExtension() - { - return array('files' => array(new UploadedFile( - $this->createTempFile(128), - 'cat.fail', - 'text/plain', - 128 - ))); - } - protected function getFileWithCorrectMimeType() { return array('files' => array(new UploadedFile( diff --git a/Tests/Controller/DropzoneValidationTest.php b/Tests/Controller/DropzoneValidationTest.php index 73cb5ff0..abe7f061 100644 --- a/Tests/Controller/DropzoneValidationTest.php +++ b/Tests/Controller/DropzoneValidationTest.php @@ -27,26 +27,6 @@ protected function getOversizedFile() ); } - protected function getFileWithCorrectExtension() - { - return new UploadedFile( - $this->createTempFile(128), - 'cat.ok', - 'text/plain', - 128 - ); - } - - protected function getFileWithIncorrectExtension() - { - return new UploadedFile( - $this->createTempFile(128), - 'cat.fail', - 'text/plain', - 128 - ); - } - protected function getFileWithCorrectMimeType() { return new UploadedFile( diff --git a/Tests/Controller/FancyUploadValidationTest.php b/Tests/Controller/FancyUploadValidationTest.php index 7e489649..1f782288 100644 --- a/Tests/Controller/FancyUploadValidationTest.php +++ b/Tests/Controller/FancyUploadValidationTest.php @@ -27,26 +27,6 @@ protected function getOversizedFile() ); } - protected function getFileWithCorrectExtension() - { - return new UploadedFile( - $this->createTempFile(128), - 'cat.ok', - 'text/plain', - 128 - ); - } - - protected function getFileWithIncorrectExtension() - { - return new UploadedFile( - $this->createTempFile(128), - 'cat.fail', - 'text/plain', - 128 - ); - } - protected function getFileWithCorrectMimeType() { return new UploadedFile( diff --git a/Tests/Controller/FineUploaderValidationTest.php b/Tests/Controller/FineUploaderValidationTest.php index d1961e43..a3b1a7e0 100644 --- a/Tests/Controller/FineUploaderValidationTest.php +++ b/Tests/Controller/FineUploaderValidationTest.php @@ -27,26 +27,6 @@ protected function getOversizedFile() ); } - protected function getFileWithCorrectExtension() - { - return new UploadedFile( - $this->createTempFile(128), - 'cat.ok', - 'text/plain', - 128 - ); - } - - protected function getFileWithIncorrectExtension() - { - return new UploadedFile( - $this->createTempFile(128), - 'cat.fail', - 'text/plain', - 128 - ); - } - protected function getFileWithCorrectMimeType() { return new UploadedFile( diff --git a/Tests/Controller/PluploadValidationTest.php b/Tests/Controller/PluploadValidationTest.php index 61ffc55c..c9e9be98 100644 --- a/Tests/Controller/PluploadValidationTest.php +++ b/Tests/Controller/PluploadValidationTest.php @@ -27,26 +27,6 @@ protected function getOversizedFile() ); } - protected function getFileWithCorrectExtension() - { - return new UploadedFile( - $this->createTempFile(128), - 'cat.ok', - 'text/plain', - 128 - ); - } - - protected function getFileWithIncorrectExtension() - { - return new UploadedFile( - $this->createTempFile(128), - 'cat.fail', - 'text/plain', - 128 - ); - } - protected function getFileWithCorrectMimeType() { return new UploadedFile( diff --git a/Tests/Controller/UploadifyValidationTest.php b/Tests/Controller/UploadifyValidationTest.php index 0f106503..086267f0 100644 --- a/Tests/Controller/UploadifyValidationTest.php +++ b/Tests/Controller/UploadifyValidationTest.php @@ -27,26 +27,6 @@ protected function getOversizedFile() ); } - protected function getFileWithCorrectExtension() - { - return new UploadedFile( - $this->createTempFile(128), - 'cat.ok', - 'text/plain', - 128 - ); - } - - protected function getFileWithIncorrectExtension() - { - return new UploadedFile( - $this->createTempFile(128), - 'cat.fail', - 'text/plain', - 128 - ); - } - protected function getFileWithCorrectMimeType() { return new UploadedFile( diff --git a/Tests/Controller/YUI3ValidationTest.php b/Tests/Controller/YUI3ValidationTest.php index ca4919c4..2325c617 100644 --- a/Tests/Controller/YUI3ValidationTest.php +++ b/Tests/Controller/YUI3ValidationTest.php @@ -27,26 +27,6 @@ protected function getOversizedFile() ); } - protected function getFileWithCorrectExtension() - { - return new UploadedFile( - $this->createTempFile(128), - 'cat.ok', - 'text/plain', - 128 - ); - } - - protected function getFileWithIncorrectExtension() - { - return new UploadedFile( - $this->createTempFile(128), - 'cat.fail', - 'text/plain', - 128 - ); - } - protected function getFileWithCorrectMimeType() { return new UploadedFile( From e7cb8dcf98d16af8310bb56750d576247c7becdf Mon Sep 17 00:00:00 2001 From: Jim Schmid Date: Fri, 11 Oct 2013 19:19:02 +0200 Subject: [PATCH 32/47] Fixed some typos in documentation. --- Resources/doc/chunked_uploads.md | 20 ++++++++++---------- Resources/doc/gaufrette_storage.md | 20 ++++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/Resources/doc/chunked_uploads.md b/Resources/doc/chunked_uploads.md index b6c560d7..7eefd95d 100644 --- a/Resources/doc/chunked_uploads.md +++ b/Resources/doc/chunked_uploads.md @@ -38,7 +38,7 @@ oneup_uploader: You can choose a custom directory to save the chunks temporarily while uploading by changing the parameter `directory`. Since version 1.0 you can also use a Gaufrette filesystem as the chunk storage. To do this you must first -set up [Gaufrette](gaufrette_storage.md).There are however some additional things to keep in mind. +set up [Gaufrette](gaufrette_storage.md). There are however some additional things to keep in mind. The configuration for the Gaufrette chunk storage should look as the following: ``` oneup_uploader: @@ -46,14 +46,14 @@ oneup_uploader: maxage: 86400 storage: type: gaufrette - filesystem: gaufrette.gallery_filesystem + filesystem: gaufrette.gallery_filesystem prefix: 'chunks' stream_wrapper: 'gaufrette://gallery/' ``` -> Setting the stream_wrapper is heavily recommended for better performance, see the reasons in the [gaufrette configuration](gaufrette_storage.md#configure-your-mappings) +> Setting the `stream_wrapper` is heavily recommended for better performance, see the reasons in the [gaufrette configuration](gaufrette_storage.md#configure-your-mappings) -As you can see there are is a new option, ```prefix```. It represents the directory +As you can see there are is a new option, `prefix`. It represents the directory *relative* to the filesystem's directory which the chunks are stored in. Gaufrette won't allow it to be outside of the filesystem. @@ -61,16 +61,16 @@ Gaufrette won't allow it to be outside of the filesystem. only the Local filesystem is capable of streaming directly. This will give you a better structured directory, -as the chunk's folders and the uploaded files won't mix with each other. -> You can set it to an empty string (```''```), if you don't need it. Otherwise it defaults to ```chunks```. +as the chunk's folders and the uploaded files won't mix with each other. +> You can set it to an empty string (`''`), if you don't need it. Otherwise it defaults to `chunks`. -The chunks will be read directly from the tmp and appended to the already existing part on the given filesystem, +The chunks will be read directly from the temporary directory and appended to the already existing part on the given filesystem, resulting in only 1 read and 1 write operation. -You can achieve the biggest improvement if you use the same filesystem as your storage, as if you do so, the assembled -file only has to be moved out of the chunk directory, which on the same filesystem takes almost not time. +You can achieve the biggest improvement if you use the same filesystem as your storage. If you do so, the assembled +file only has to be moved out of the chunk directory, which takes no time on a local filesystem. -> The ```load distribution``` is forcefully turned on, if you use gaufrette as the chunk storage. +> The load distribution is forcefully turned on, if you use Gaufrette as the chunk storage. ## Clean up diff --git a/Resources/doc/gaufrette_storage.md b/Resources/doc/gaufrette_storage.md index c03ba849..2e74a3cd 100644 --- a/Resources/doc/gaufrette_storage.md +++ b/Resources/doc/gaufrette_storage.md @@ -53,7 +53,7 @@ knp_gaufrette: local: directory: %kernel.root_dir%/../web/uploads create: true - + filesystems: gallery: adapter: gallery @@ -71,7 +71,7 @@ oneup_uploader: gallery: storage: type: gaufrette - filesystem: gaufrette.gallery_filesystem + filesystem: gaufrette.gallery_filesystem ``` You can specify the buffer size used for syncing files from your filesystem to the gaufrette storage by changing the property `sync_buffer_size`. @@ -84,7 +84,7 @@ oneup_uploader: gallery: storage: type: gaufrette - filesystem: gaufrette.gallery_filesystem + filesystem: gaufrette.gallery_filesystem sync_buffer_size: 1M ``` @@ -97,18 +97,18 @@ oneup_uploader: gallery: storage: type: gaufrette - filesystem: gaufrette.gallery_filesystem + filesystem: gaufrette.gallery_filesystem stream_wrapper: gaufrette://gallery/ ``` -> This is only useful if you are using a stream capable adapter, at the time of this writing only +> This is only useful if you are using a stream-capable adapter. At the time of this writing, only the local adapter is capable of streaming directly. -The first part (```gaufrette```) in the example above ```MUST``` be the same as ```knp_gaufrette.stream_wrapper.protocol```, -the second part (```gallery```) in the example, ```MUST``` be the key of the filesytem (```knp_gaufette.filesystems.key```). -It also must end with a slash (```/```). +The first part (`gaufrette`) in the example above `MUST` be the same as `knp_gaufrette.stream_wrapper.protocol`, +the second part (`gallery`) in the example, `MUST` be the key of the filesytem (`knp_gaufette.filesystems.key`). +It also must end with a slash (`/`). This is particularly useful if you want to get exact informations about your files. Gaufrette offers you every functionality to do this without relying on the stream wrapper, however it will have to download the file and load it into memory -to operate on it. If ```stream_wrapper``` is specified the bundle will try to open the file as streams when such operation -is requested.(e.g. getting the size of the file, the mime-type based on content) +to operate on it. If `stream_wrapper` is specified, the bundle will try to open the file as streams when such operation +is requested. (e.g. getting the size of the file, the mime-type based on content) From 207d2f19d89f37ca61831ff1a79e2557aa9464f1 Mon Sep 17 00:00:00 2001 From: Jim Schmid Date: Fri, 11 Oct 2013 19:19:16 +0200 Subject: [PATCH 33/47] Added a documentation topic: How to use Chunked Uploads behind Load Balancers. This is WIP. --- Resources/doc/index.md | 1 + Resources/doc/load_balancers.md | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 Resources/doc/load_balancers.md diff --git a/Resources/doc/index.md b/Resources/doc/index.md index fe69b15d..430fc085 100644 --- a/Resources/doc/index.md +++ b/Resources/doc/index.md @@ -124,6 +124,7 @@ some more advanced features. * [Validate your uploads](custom_validator.md) * [General/Generic Events](events.md) * [Enable Session upload progress / upload cancelation](progress.md) +* [Use Chunked Uploads behind Load Balancers](load_balancers.md) * [Configuration Reference](configuration_reference.md) * [Testing this bundle](testing.md) diff --git a/Resources/doc/load_balancers.md b/Resources/doc/load_balancers.md new file mode 100644 index 00000000..0667b949 --- /dev/null +++ b/Resources/doc/load_balancers.md @@ -0,0 +1,6 @@ +Use Chunked Uploads behind Load Balancers +========================================= + +If you want to use Chunked Uploads behind load balancers that is not configured to use sticky sessions you'll eventually end up with a bunch of chunks on every instance and the bundle is not able to reassemble the file on the server. + +You can avoid this problem by using Gaufrette as an abstract filesystem. \ No newline at end of file From 50d14ccfdd25c3bc9ea97ee5d4d585ffc8f55242 Mon Sep 17 00:00:00 2001 From: Jim Schmid Date: Fri, 11 Oct 2013 19:29:27 +0200 Subject: [PATCH 34/47] Raised requirement of symfony/framework-bundle to >=2.2. --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index b4a4d153..a6be9f89 100644 --- a/composer.json +++ b/composer.json @@ -15,7 +15,7 @@ ], "require": { - "symfony/framework-bundle": "2.*", + "symfony/framework-bundle": ">=2.2", "symfony/finder": ">=2.2.0" }, From 47c9e32a13e4cec964845e6ad09708643b66fe79 Mon Sep 17 00:00:00 2001 From: mitom Date: Mon, 14 Oct 2013 11:18:00 +0200 Subject: [PATCH 35/47] remove file from gaufrette chunk storage after upload --- Uploader/Storage/GaufretteStorage.php | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Uploader/Storage/GaufretteStorage.php b/Uploader/Storage/GaufretteStorage.php index f1ae5bd8..7b9bc7e3 100644 --- a/Uploader/Storage/GaufretteStorage.php +++ b/Uploader/Storage/GaufretteStorage.php @@ -5,6 +5,7 @@ use Oneup\UploaderBundle\Uploader\File\FileInterface; use Oneup\UploaderBundle\Uploader\File\GaufretteFile; use Gaufrette\Filesystem; +use Symfony\Component\Filesystem\Filesystem as LocalFilesystem; use Gaufrette\Adapter\MetadataSupporter; use Oneup\UploaderBundle\Uploader\Gaufrette\StreamManager; @@ -39,9 +40,15 @@ public function upload(FileInterface $file, $name, $path = null) $dst = $this->filesystem->createStream($path); $this->openStream($dst, 'w'); - $this->stream($file, $dst); + if ($file instanceof GaufretteFile) { + $file->delete(); + } else { + $filesystem = new LocalFilesystem(); + $filesystem->remove($file->getPathname()); + } + return new GaufretteFile($this->filesystem->get($path), $this->filesystem, $this->streamWrapperPrefix); } From 01e33479db3496aee484fdae9823d4bbddeac930 Mon Sep 17 00:00:00 2001 From: mitom Date: Mon, 14 Oct 2013 12:28:36 +0200 Subject: [PATCH 36/47] avoid post-chunk upload events to fire with empty files --- Controller/AbstractChunkedController.php | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Controller/AbstractChunkedController.php b/Controller/AbstractChunkedController.php index e9e05a8d..2e803bb0 100644 --- a/Controller/AbstractChunkedController.php +++ b/Controller/AbstractChunkedController.php @@ -53,11 +53,16 @@ protected function handleChunkedUpload(UploadedFile $file, ResponseInterface $re $chunk = $chunkManager->addChunk($uuid, $index, $file, $orig); - $this->dispatchChunkEvents($chunk, $response, $request, $last); + if ($chunk) { + $this->dispatchChunkEvents($chunk, $response, $request, $last); + } if ($chunkManager->getLoadDistribution()) { $chunks = $chunkManager->getChunks($uuid); $assembled = $chunkManager->assembleChunks($chunks, true, $last); + if (!$chunk) { + $this->dispatchChunkEvents($assembled, $response, $request, $last); + } } // if all chunks collected and stored, proceed From e91f3a76464f340191cc3f3ccbecbe37d2f36971 Mon Sep 17 00:00:00 2001 From: Jim Schmid Date: Mon, 14 Oct 2013 19:56:20 +0200 Subject: [PATCH 37/47] Marked some tests skipped if directories mismatch. Strangly on my machine the function tempnam returns something different than expected. tempnam('/foo', 'bar'); returns a file located in /private/foo. The way this test works does not allow this behaviour. This is why we test this condition and mark the test skipped. --- Tests/Uploader/Storage/GaufretteOrphanageStorageTest.php | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Tests/Uploader/Storage/GaufretteOrphanageStorageTest.php b/Tests/Uploader/Storage/GaufretteOrphanageStorageTest.php index bc3a6710..418298cb 100644 --- a/Tests/Uploader/Storage/GaufretteOrphanageStorageTest.php +++ b/Tests/Uploader/Storage/GaufretteOrphanageStorageTest.php @@ -28,6 +28,10 @@ public function setUp() $this->tempDirectory = $this->realDirectory . '/' . $this->orphanageKey; $this->payloads = array(); + if (!$this->checkIfTempnameMatchesAfterCreation()) { + $this->markTestSkipped('Temporary directories do not match'); + } + $filesystem = new \Symfony\Component\Filesystem\Filesystem(); $filesystem->mkdir($this->realDirectory); $filesystem->mkdir($this->chunkDirectory); @@ -107,4 +111,8 @@ public function testUploadAndFetching() $this->assertCount($this->numberOfPayloads, $finder); } + public function checkIfTempnameMatchesAfterCreation() + { + return strpos(tempnam($this->chunkDirectory, 'uploader'), $this->chunkDirectory) === 0; + } } From 15a45904ca8452fb281020a49c732f9391031e04 Mon Sep 17 00:00:00 2001 From: Jim Schmid Date: Mon, 14 Oct 2013 20:05:14 +0200 Subject: [PATCH 38/47] Normalize stream wrapper path. This way it is not important to remember appending a / to the end of the stream_wrapper string. --- .../OneupUploaderExtension.php | 11 ++++++++++ .../OneupUploaderExtensionTest.php | 22 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/DependencyInjection/OneupUploaderExtension.php b/DependencyInjection/OneupUploaderExtension.php index b25c844e..c017bdcb 100644 --- a/DependencyInjection/OneupUploaderExtension.php +++ b/DependencyInjection/OneupUploaderExtension.php @@ -229,6 +229,8 @@ protected function registerGaufretteStorage($key, $class, $filesystem, $buffer, if(strlen($filesystem) <= 0) throw new ServiceNotFoundException('Empty service name'); + $streamWrapper = $this->normalizeStreamWrapper($streamWrapper); + $this->container ->register($key, $class) ->addArgument(new Reference($filesystem)) @@ -266,4 +268,13 @@ protected function normalizePath($input) { return rtrim($input, '/'); } + + protected function normalizeStreamWrapper($input) + { + if (is_null($input)) { + return null; + } + + return rtrim($input, '/') . '/'; + } } diff --git a/Tests/DependencyInjection/OneupUploaderExtensionTest.php b/Tests/DependencyInjection/OneupUploaderExtensionTest.php index 1b09fbf5..85ac094b 100644 --- a/Tests/DependencyInjection/OneupUploaderExtensionTest.php +++ b/Tests/DependencyInjection/OneupUploaderExtensionTest.php @@ -28,6 +28,28 @@ public function testValueToByteTransformer() $this->assertEquals(2147483648, $method->invoke($mock, '2G')); } + public function testNormalizationOfStreamWrapper() + { + $mock = $this->getMockBuilder('Oneup\UploaderBundle\DependencyInjection\OneupUploaderExtension') + ->disableOriginalConstructor() + ->getMock() + ; + + $method = new \ReflectionMethod( + 'Oneup\UploaderBundle\DependencyInjection\OneupUploaderExtension', + 'normalizeStreamWrapper' + ); + $method->setAccessible(true); + + $output1 = $method->invoke($mock, 'gaufrette://gallery'); + $output2 = $method->invoke($mock, 'gaufrette://gallery/'); + $output3 = $method->invoke($mock, null); + + $this->assertEquals('gaufrette://gallery/', $output1); + $this->assertEquals('gaufrette://gallery/', $output2); + $this->assertNull($output3); + } + public function testGetMaxUploadSize() { $mock = $this->getMockBuilder('Oneup\UploaderBundle\DependencyInjection\OneupUploaderExtension') From 988082eb6eda9f1f69182ed478a04f333c5766f5 Mon Sep 17 00:00:00 2001 From: Jim Schmid Date: Mon, 14 Oct 2013 20:35:08 +0200 Subject: [PATCH 39/47] Extended documentation about chunked uploads and Gaufrette. --- Resources/doc/chunked_uploads.md | 27 +++++++++++++---------- Resources/doc/load_balancers.md | 37 +++++++++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 12 deletions(-) diff --git a/Resources/doc/chunked_uploads.md b/Resources/doc/chunked_uploads.md index 7eefd95d..fd6d87a5 100644 --- a/Resources/doc/chunked_uploads.md +++ b/Resources/doc/chunked_uploads.md @@ -37,10 +37,13 @@ oneup_uploader: You can choose a custom directory to save the chunks temporarily while uploading by changing the parameter `directory`. -Since version 1.0 you can also use a Gaufrette filesystem as the chunk storage. To do this you must first -set up [Gaufrette](gaufrette_storage.md). There are however some additional things to keep in mind. +## Use Gaufrette to store chunk files + +You can also use a Gaufrette filesystem as the chunk storage. A possible use case is to use chunked uploads behind non-session sticky load balancers. +To do this you must first set up [Gaufrette](gaufrette_storage.md). There are however some additional things to keep in mind. The configuration for the Gaufrette chunk storage should look as the following: -``` + +```yaml oneup_uploader: chunks: maxage: 86400 @@ -51,27 +54,29 @@ oneup_uploader: stream_wrapper: 'gaufrette://gallery/' ``` -> Setting the `stream_wrapper` is heavily recommended for better performance, see the reasons in the [gaufrette configuration](gaufrette_storage.md#configure-your-mappings) +> :exclamation: Setting the `stream_wrapper` is heavily recommended for better performance, see the reasons in the [gaufrette configuration](gaufrette_storage.md#configure-your-mappings) -As you can see there are is a new option, `prefix`. It represents the directory +As you can see there are is an option, `prefix`. It represents the directory *relative* to the filesystem's directory which the chunks are stored in. Gaufrette won't allow it to be outside of the filesystem. - -> You can only use stream capable filesystems for the chunk storage, at the time of this writing -only the Local filesystem is capable of streaming directly. - This will give you a better structured directory, as the chunk's folders and the uploaded files won't mix with each other. -> You can set it to an empty string (`''`), if you don't need it. Otherwise it defaults to `chunks`. +You can set it to an empty string (`''`), if you don't need it. Otherwise it defaults to `chunks`. + +> :exclamation: You can only use stream capable filesystems for the chunk storage, at the time of this writing +only the Local filesystem is capable of streaming directly. The chunks will be read directly from the temporary directory and appended to the already existing part on the given filesystem, -resulting in only 1 read and 1 write operation. +resulting in only one single read and one single write operation. + +> :exclamation: Do not use a Gaufrette filesystem for the chunk storage and a local filesystem for the mapping. This is not possible to check during container setup and will throw unexpected errors at runtime! You can achieve the biggest improvement if you use the same filesystem as your storage. If you do so, the assembled file only has to be moved out of the chunk directory, which takes no time on a local filesystem. > The load distribution is forcefully turned on, if you use Gaufrette as the chunk storage. +See the [Use Chunked Uploads behind Load Balancers](load_balancers.md) section in the documentation for a full configuration example. ## Clean up diff --git a/Resources/doc/load_balancers.md b/Resources/doc/load_balancers.md index 0667b949..c1786007 100644 --- a/Resources/doc/load_balancers.md +++ b/Resources/doc/load_balancers.md @@ -3,4 +3,39 @@ Use Chunked Uploads behind Load Balancers If you want to use Chunked Uploads behind load balancers that is not configured to use sticky sessions you'll eventually end up with a bunch of chunks on every instance and the bundle is not able to reassemble the file on the server. -You can avoid this problem by using Gaufrette as an abstract filesystem. \ No newline at end of file +You can avoid this problem by using Gaufrette as an abstract filesystem. Check the following configuration as an example. + +```yaml +knp_gaufrette: + adapters: + gallery: + local: + directory: %kernel.root_dir%/../web/uploads + create: true + + filesystems: + gallery: + adapter: gallery + + stream_wrapper: ~ + +oneup_uploader: + chunks: + storage: + type: gaufrette + filesystem: gaufrette.gallery_filesystem + stream_wrapper: gaufrette://gallery/ + + mappings: + gallery: + frontend: fineuploader + storage: + type: gaufrette + filesystem: gaufrette.gallery_filesystem +``` + +> :exclamation: Event though it is possible to use two different Gaufrette filesystems - one for the the chunk storage - and one for the mapping, it is not recommended. + +> :exclamation: Do not use a Gaufrette filesystem for the chunk storage and a local filesystem one for the mapping. This is not possible to check during configuration and will throw unexpected errors! + +Using Gaufrette filesystems for chunked upload directories has some limitations. It is highly recommended to use a `Local` Gaufrette adapter as it is the only one that is able to `rename` a file but `move` it. Especially when working with bigger files this can have serious perfomance advantages as this way the file doesn't have to be moved entirely to memory! \ No newline at end of file From 5e789d8b759d1977643a5ac99b3291fd30d866f5 Mon Sep 17 00:00:00 2001 From: mitom Date: Mon, 14 Oct 2013 20:56:48 +0200 Subject: [PATCH 40/47] made docs a bit nicer with a check --- DependencyInjection/OneupUploaderExtension.php | 6 ++++++ Resources/doc/chunked_uploads.md | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/DependencyInjection/OneupUploaderExtension.php b/DependencyInjection/OneupUploaderExtension.php index c017bdcb..a3f55dc7 100644 --- a/DependencyInjection/OneupUploaderExtension.php +++ b/DependencyInjection/OneupUploaderExtension.php @@ -63,6 +63,12 @@ protected function processOrphanageConfig() protected function processMapping($key, &$mapping) { + if ($this->config['chunks']['storage']['type'] === 'gaufrette') { + if ($mapping['storage']['type'] !== 'gaufrette') { + throw new \InvalidArgumentException('If you use a gaufrette based chunk storage, you may only use gaufrette based storages too.'); + } + } + $mapping['max_size'] = $mapping['max_size'] < 0 ? $this->getMaxUploadSize($mapping['max_size']) : $mapping['max_size'] diff --git a/Resources/doc/chunked_uploads.md b/Resources/doc/chunked_uploads.md index fd6d87a5..2f53fb78 100644 --- a/Resources/doc/chunked_uploads.md +++ b/Resources/doc/chunked_uploads.md @@ -69,7 +69,7 @@ only the Local filesystem is capable of streaming directly. The chunks will be read directly from the temporary directory and appended to the already existing part on the given filesystem, resulting in only one single read and one single write operation. -> :exclamation: Do not use a Gaufrette filesystem for the chunk storage and a local filesystem for the mapping. This is not possible to check during container setup and will throw unexpected errors at runtime! +> :exclamation: If you use a gaufrette filesystem as chunk storage, you may only use gaufrette filesystems in your mappings too! You can achieve the biggest improvement if you use the same filesystem as your storage. If you do so, the assembled file only has to be moved out of the chunk directory, which takes no time on a local filesystem. From 147388f69ed1b5d5fa8b91e2c039fb7e8deb0093 Mon Sep 17 00:00:00 2001 From: mitom Date: Mon, 14 Oct 2013 21:09:25 +0200 Subject: [PATCH 41/47] Revert "made docs a bit nicer with a check" This reverts commit 5e789d8b759d1977643a5ac99b3291fd30d866f5. --- DependencyInjection/OneupUploaderExtension.php | 6 ------ Resources/doc/chunked_uploads.md | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/DependencyInjection/OneupUploaderExtension.php b/DependencyInjection/OneupUploaderExtension.php index a3f55dc7..c017bdcb 100644 --- a/DependencyInjection/OneupUploaderExtension.php +++ b/DependencyInjection/OneupUploaderExtension.php @@ -63,12 +63,6 @@ protected function processOrphanageConfig() protected function processMapping($key, &$mapping) { - if ($this->config['chunks']['storage']['type'] === 'gaufrette') { - if ($mapping['storage']['type'] !== 'gaufrette') { - throw new \InvalidArgumentException('If you use a gaufrette based chunk storage, you may only use gaufrette based storages too.'); - } - } - $mapping['max_size'] = $mapping['max_size'] < 0 ? $this->getMaxUploadSize($mapping['max_size']) : $mapping['max_size'] diff --git a/Resources/doc/chunked_uploads.md b/Resources/doc/chunked_uploads.md index 2f53fb78..fd6d87a5 100644 --- a/Resources/doc/chunked_uploads.md +++ b/Resources/doc/chunked_uploads.md @@ -69,7 +69,7 @@ only the Local filesystem is capable of streaming directly. The chunks will be read directly from the temporary directory and appended to the already existing part on the given filesystem, resulting in only one single read and one single write operation. -> :exclamation: If you use a gaufrette filesystem as chunk storage, you may only use gaufrette filesystems in your mappings too! +> :exclamation: Do not use a Gaufrette filesystem for the chunk storage and a local filesystem for the mapping. This is not possible to check during container setup and will throw unexpected errors at runtime! You can achieve the biggest improvement if you use the same filesystem as your storage. If you do so, the assembled file only has to be moved out of the chunk directory, which takes no time on a local filesystem. From 341c88053299b932efefbd1ae068b2cae9a267da Mon Sep 17 00:00:00 2001 From: Jim Schmid Date: Mon, 14 Oct 2013 21:31:37 +0200 Subject: [PATCH 42/47] Test against null to avoid object to boolean conversion. --- Controller/AbstractChunkedController.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Controller/AbstractChunkedController.php b/Controller/AbstractChunkedController.php index 2e803bb0..db4d784d 100644 --- a/Controller/AbstractChunkedController.php +++ b/Controller/AbstractChunkedController.php @@ -60,7 +60,8 @@ protected function handleChunkedUpload(UploadedFile $file, ResponseInterface $re if ($chunkManager->getLoadDistribution()) { $chunks = $chunkManager->getChunks($uuid); $assembled = $chunkManager->assembleChunks($chunks, true, $last); - if (!$chunk) { + + if (is_null($chunk)) { $this->dispatchChunkEvents($assembled, $response, $request, $last); } } From 009586389c5345fc96de2e59f598016de23f11a0 Mon Sep 17 00:00:00 2001 From: mitom Date: Mon, 14 Oct 2013 21:33:49 +0200 Subject: [PATCH 43/47] fixed event firing check --- Controller/AbstractChunkedController.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Controller/AbstractChunkedController.php b/Controller/AbstractChunkedController.php index 2e803bb0..4116b4c8 100644 --- a/Controller/AbstractChunkedController.php +++ b/Controller/AbstractChunkedController.php @@ -53,14 +53,14 @@ protected function handleChunkedUpload(UploadedFile $file, ResponseInterface $re $chunk = $chunkManager->addChunk($uuid, $index, $file, $orig); - if ($chunk) { + if (null !== $chunk) { $this->dispatchChunkEvents($chunk, $response, $request, $last); } if ($chunkManager->getLoadDistribution()) { $chunks = $chunkManager->getChunks($uuid); $assembled = $chunkManager->assembleChunks($chunks, true, $last); - if (!$chunk) { + if (null === $chunk) { $this->dispatchChunkEvents($assembled, $response, $request, $last); } } From b17bee4fbd2d39a603218f2959bb0389e7828e6a Mon Sep 17 00:00:00 2001 From: PaulM Date: Tue, 15 Oct 2013 11:11:53 +0200 Subject: [PATCH 44/47] Fix configuration for storage service creation --- DependencyInjection/OneupUploaderExtension.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DependencyInjection/OneupUploaderExtension.php b/DependencyInjection/OneupUploaderExtension.php index c017bdcb..f6720a6a 100644 --- a/DependencyInjection/OneupUploaderExtension.php +++ b/DependencyInjection/OneupUploaderExtension.php @@ -169,7 +169,7 @@ protected function createStorageService(&$config, $key, $orphanage = false) // if a service is given, return a reference to this service // this allows a user to overwrite the storage layer if needed if (!is_null($config['service'])) { - $storageService = new Reference($config['storage']['service']); + $storageService = new Reference($config['service']); } else { // no service was given, so we create one $storageName = sprintf('oneup_uploader.storage.%s', $key); From 996a7849633a5aeb2b35665b346e7c4ca28bccb4 Mon Sep 17 00:00:00 2001 From: Jim Schmid Date: Wed, 23 Oct 2013 15:03:40 +0200 Subject: [PATCH 45/47] Fixes scrutinizer-ci warnings. Do not test against: too_many_methods too_many_fields coupling_between_objects As this bundle provides a base Controller that is dependant on many coupled objects/interfaces, the coupling does not really make sense to test. --- .scrutinizer.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.scrutinizer.yml b/.scrutinizer.yml index 764db5df..dc9dd293 100644 --- a/.scrutinizer.yml +++ b/.scrutinizer.yml @@ -1,6 +1,13 @@ tools: php_code_sniffer: true php_cs_fixer: true - php_mess_detector: true + php_mess_detector: + config: + code_size_rules: + too_many_fields: false + too_many_methods: false + + design_rules: + coupling_between_objects: false php_analyzer: true sensiolabs_security_checker: true \ No newline at end of file From d2652914296766d29f7ecff02058f4777a708b74 Mon Sep 17 00:00:00 2001 From: Jim Schmid Date: Wed, 23 Oct 2013 15:17:17 +0200 Subject: [PATCH 46/47] Use 1.0 version of this bundle. --- Resources/doc/index.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Resources/doc/index.md b/Resources/doc/index.md index 430fc085..bf97ce0b 100644 --- a/Resources/doc/index.md +++ b/Resources/doc/index.md @@ -34,7 +34,7 @@ Add OneupUploaderBundle to your composer.json using the following construct: ```js { "require": { - "oneup/uploader-bundle": "0.9.*@dev" + "oneup/uploader-bundle": "1.0.*@dev" } } ``` @@ -71,7 +71,8 @@ This bundle was designed to just work out of the box. The only thing you have to oneup_uploader: mappings: - gallery: ~ + gallery: + frontend: blueimp # or any uploader you use in the frontend ``` To enable the dynamic routes, add the following to your routing configuration file. From 8e21e8100a4dba50a1f2175e822d9e77e73cb918 Mon Sep 17 00:00:00 2001 From: Jim Schmid Date: Wed, 23 Oct 2013 15:27:21 +0200 Subject: [PATCH 47/47] Added Upgrade Note about version 1.0. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index a8b708b1..0abe5c17 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ The entry point of the documentation can be found in the file `Resources/docs/in Upgrade Notes ------------- +* Version **v1.0.0** introduced some backward compatibility breaks. For a full list of changes, head to the [dedicated pull request](https://github.com/1up-lab/OneupUploaderBundle/pull/57). * If you're using chunked uploads consider upgrading from **v0.9.6** to **v0.9.7**. A critical issue was reported regarding the assembly of chunks. More information in ticket [#21](https://github.com/1up-lab/OneupUploaderBundle/issues/21#issuecomment-21560320). * Error management [changed](https://github.com/1up-lab/OneupUploaderBundle/pull/25) in Version **0.9.6**. You can now register an `ErrorHandler` per configured frontend. This comes bundled with some adjustments to the `blueimp` controller. More information is available in [the documentation](https://github.com/1up-lab/OneupUploaderBundle/blob/master/Resources/doc/custom_error_handler.md). * Event dispatching [changed](https://github.com/1up-lab/OneupUploaderBundle/commit/a408548b241f47af3539b2137c1817a21a51fde9) in Version **0.9.5**. The dispatching is now handled in the `upload*` functions. So if you have created your own implementation, be sure to remove the call to the `dispatchEvents` function, otherwise it will be called twice. Furthermore no `POST_UPLOAD` event will be fired anymore after uploading a chunk. You can get more information on this topic in the [documentation](https://github.com/1up-lab/OneupUploaderBundle/blob/master/Resources/doc/custom_logic.md#using-chunked-uploads).