diff --git a/appinfo/info.xml b/appinfo/info.xml index b3c5345e..07ed378e 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -92,7 +92,7 @@ The app does not send any sensitive data to cloud providers or similar services. https://raw.githubusercontent.com/nextcloud/recognize/main/screenshots/Logo.png https://raw.githubusercontent.com/nextcloud/recognize/main/screenshots/imagenet_examples.jpg - + diff --git a/lib/BackgroundJobs/ClassifierJob.php b/lib/BackgroundJobs/ClassifierJob.php index 6bd13e6b..144efbd9 100644 --- a/lib/BackgroundJobs/ClassifierJob.php +++ b/lib/BackgroundJobs/ClassifierJob.php @@ -71,15 +71,15 @@ protected function runClassifier(string $model, array $argument): void { try { $this->logger->debug('Running ' . $model . ' classifier'); $this->classify($files); - } catch(\RuntimeException $e) { + } catch (\RuntimeException $e) { $this->logger->warning('Temporary problem with ' . $model . ' classifier, trying again soon', ['exception' => $e]); - } catch(\ErrorException $e) { + } catch (\ErrorException $e) { $this->settingsService->setSetting($model.'.status', 'false'); $this->logger->warning('Problem with ' . $model . ' classifier', ['exception' => $e]); $this->logger->debug('Removing '.static::class.' with argument ' . var_export($argument, true) . 'from oc_jobs'); $this->jobList->remove(static::class, $argument); throw $e; - } catch(\Throwable $e) { + } catch (\Throwable $e) { $this->settingsService->setSetting($model.'.status', 'false'); throw $e; } diff --git a/lib/BackgroundJobs/StorageCrawlJob.php b/lib/BackgroundJobs/StorageCrawlJob.php index a63d5b4c..e8b1d5b5 100644 --- a/lib/BackgroundJobs/StorageCrawlJob.php +++ b/lib/BackgroundJobs/StorageCrawlJob.php @@ -77,7 +77,6 @@ protected function run($argument): void { } if (!in_array(ImagenetClassifier::MODEL_NAME, $models) && in_array(LandmarksClassifier::MODEL_NAME, $models)) { $tags = $this->tagManager->getTagsForFiles([$queueFile->getFileId()]); - /** @var \OCP\SystemTag\ISystemTag[] $fileTags */ $fileTags = $tags[$queueFile->getFileId()]; $landmarkTags = array_filter($fileTags, function ($tag) { return in_array($tag->getName(), LandmarksClassifier::PRECONDITION_TAGS); diff --git a/lib/Command/Classify.php b/lib/Command/Classify.php index 9df010f2..62f6ce93 100644 --- a/lib/Command/Classify.php +++ b/lib/Command/Classify.php @@ -16,11 +16,14 @@ use OCA\Recognize\Service\Logger; use OCA\Recognize\Service\SettingsService; use OCA\Recognize\Service\StorageService; +use OCA\Recognize\Service\TagManager; use OCP\Files\Config\ICachedMountInfo; use OCP\Files\Config\IUserMountCache; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Exception\ExceptionInterface; +use Symfony\Component\Console\Input\ArrayInput; use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Output\OutputInterface; class Classify extends Command { @@ -29,6 +32,7 @@ class Classify extends Command { public function __construct( private StorageService $storageService, + private TagManager $tagManager, private Logger $logger, ImagenetClassifier $imagenet, ClusteringFaceClassifier $faces, @@ -53,7 +57,8 @@ public function __construct( */ protected function configure() { $this->setName('recognize:classify') - ->setDescription('Classify all files with the current settings in one go (will likely take a long time)'); + ->setDescription('Classify all files with the current settings in one go (will likely take a long time)') + ->addOption('retry', null, InputOption::VALUE_NONE, "Only classify untagged images"); } /** @@ -68,7 +73,9 @@ protected function configure() { protected function execute(InputInterface $input, OutputInterface $output): int { $this->logger->setCliOutput($output); - $this->clearBackgroundJobs->run($input, $output); + // pop "retry" flag from parameters passed to clear background jobs + $clearBackgroundJobs = new ArrayInput([]); + $this->clearBackgroundJobs->run($clearBackgroundJobs, $output); $models = array_values(array_filter([ ClusteringFaceClassifier::MODEL_NAME, @@ -77,6 +84,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int MusicnnClassifier::MODEL_NAME, ], fn ($modelName) => $this->settings->getSetting($modelName . '.enabled') === 'true')); + $processedTag = $this->tagManager->getProcessedTag(); + foreach ($this->storageService->getMounts() as $mount) { $this->logger->info('Processing storage ' . $mount['storage_id'] . ' with root ID ' . $mount['override_root']); @@ -107,13 +116,23 @@ protected function execute(InputInterface $input, OutputInterface $output): int $queueFile->setUpdate(false); if ($file['image']) { - if (in_array(ImagenetClassifier::MODEL_NAME, $models)) { - $queues[ImagenetClassifier::MODEL_NAME][] = $queueFile; - } if (in_array(ClusteringFaceClassifier::MODEL_NAME, $models)) { $queues[ClusteringFaceClassifier::MODEL_NAME][] = $queueFile; } } + // if retry flag is set, skip other classifiers for tagged files + if ($input->getOption('retry')) { + $fileTags = $this->tagManager->getTagsForFiles([$lastFileId]); + // check if processed tag is already in the tags + if (in_array($processedTag, $fileTags[$lastFileId])) { + continue; + } + } + if ($file['image']) { + if (in_array(ImagenetClassifier::MODEL_NAME, $models)) { + $queues[ImagenetClassifier::MODEL_NAME][] = $queueFile; + } + } if ($file['video']) { if (in_array(MovinetClassifier::MODEL_NAME, $models)) { $queues[MovinetClassifier::MODEL_NAME][] = $queueFile; diff --git a/lib/Controller/AdminController.php b/lib/Controller/AdminController.php index e7fd7dc8..6c99eb02 100644 --- a/lib/Controller/AdminController.php +++ b/lib/Controller/AdminController.php @@ -136,7 +136,7 @@ public function hasJobs(string $task): JSONResponse { } $iterator = $this->jobList->getJobsIterator($tasks[$task], null, 0); $lastRun = []; - foreach($iterator as $job) { + foreach ($iterator as $job) { $lastRun[] = $job->getLastRun(); } $count = count($lastRun); diff --git a/lib/Migration/InstallDeps.php b/lib/Migration/InstallDeps.php index c6b2f0c4..7e477b23 100644 --- a/lib/Migration/InstallDeps.php +++ b/lib/Migration/InstallDeps.php @@ -90,7 +90,7 @@ public function run(IOutput $output): void { $this->runFfmpegInstall($binaryPath); $this->runTfjsGpuInstall($binaryPath); $this->setNiceBinaryPath(); - } catch(\Throwable $e) { + } catch (\Throwable $e) { $output->warning('Failed to automatically install dependencies for recognize. Check the recognize admin panel for potential problems.'); $this->logger->error('Failed to automatically install dependencies for recognize. Check the recognize admin panel for potential problems.', ['exception' => $e]); } diff --git a/lib/Service/TagManager.php b/lib/Service/TagManager.php index 21ebd10f..116e57dc 100644 --- a/lib/Service/TagManager.php +++ b/lib/Service/TagManager.php @@ -79,7 +79,7 @@ public function assignTags(int $fileId, array $tags): void { /** * @param array $fileIds - * @return array + * @return array> */ public function getTagsForFiles(array $fileIds): array { /** @var array $tagsByFile */ diff --git a/vendor-bin/php-scoper/composer.lock b/vendor-bin/php-scoper/composer.lock index a1f4ec10..d54df316 100644 --- a/vendor-bin/php-scoper/composer.lock +++ b/vendor-bin/php-scoper/composer.lock @@ -450,16 +450,16 @@ }, { "name": "symfony/console", - "version": "v6.4.6", + "version": "v6.4.11", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "a2708a5da5c87d1d0d52937bdeac625df659e11f" + "reference": "42686880adaacdad1835ee8fc2a9ec5b7bd63998" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/a2708a5da5c87d1d0d52937bdeac625df659e11f", - "reference": "a2708a5da5c87d1d0d52937bdeac625df659e11f", + "url": "https://api.github.com/repos/symfony/console/zipball/42686880adaacdad1835ee8fc2a9ec5b7bd63998", + "reference": "42686880adaacdad1835ee8fc2a9ec5b7bd63998", "shasum": "" }, "require": { @@ -524,7 +524,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v6.4.6" + "source": "https://github.com/symfony/console/tree/v6.4.11" }, "funding": [ { @@ -540,20 +540,20 @@ "type": "tidelift" } ], - "time": "2024-03-29T19:07:53+00:00" + "time": "2024-08-15T22:48:29+00:00" }, { "name": "symfony/deprecation-contracts", - "version": "v3.4.0", + "version": "v3.5.0", "source": { "type": "git", "url": "https://github.com/symfony/deprecation-contracts.git", - "reference": "7c3aff79d10325257a001fcf92d991f24fc967cf" + "reference": "0e0d29ce1f20deffb4ab1b016a7257c4f1e789a1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/7c3aff79d10325257a001fcf92d991f24fc967cf", - "reference": "7c3aff79d10325257a001fcf92d991f24fc967cf", + "url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/0e0d29ce1f20deffb4ab1b016a7257c4f1e789a1", + "reference": "0e0d29ce1f20deffb4ab1b016a7257c4f1e789a1", "shasum": "" }, "require": { @@ -562,7 +562,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-main": "3.4-dev" + "dev-main": "3.5-dev" }, "thanks": { "name": "symfony/contracts", @@ -591,7 +591,7 @@ "description": "A generic function and convention to trigger deprecation notices", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/deprecation-contracts/tree/v3.4.0" + "source": "https://github.com/symfony/deprecation-contracts/tree/v3.5.0" }, "funding": [ { @@ -607,20 +607,20 @@ "type": "tidelift" } ], - "time": "2023-05-23T14:45:45+00:00" + "time": "2024-04-18T09:32:20+00:00" }, { "name": "symfony/event-dispatcher-contracts", - "version": "v3.4.2", + "version": "v3.5.0", "source": { "type": "git", "url": "https://github.com/symfony/event-dispatcher-contracts.git", - "reference": "4e64b49bf370ade88e567de29465762e316e4224" + "reference": "8f93aec25d41b72493c6ddff14e916177c9efc50" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/event-dispatcher-contracts/zipball/4e64b49bf370ade88e567de29465762e316e4224", - "reference": "4e64b49bf370ade88e567de29465762e316e4224", + "url": "https://api.github.com/repos/symfony/event-dispatcher-contracts/zipball/8f93aec25d41b72493c6ddff14e916177c9efc50", + "reference": "8f93aec25d41b72493c6ddff14e916177c9efc50", "shasum": "" }, "require": { @@ -630,7 +630,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-main": "3.4-dev" + "dev-main": "3.5-dev" }, "thanks": { "name": "symfony/contracts", @@ -667,7 +667,7 @@ "standards" ], "support": { - "source": "https://github.com/symfony/event-dispatcher-contracts/tree/v3.4.2" + "source": "https://github.com/symfony/event-dispatcher-contracts/tree/v3.5.0" }, "funding": [ { @@ -683,20 +683,20 @@ "type": "tidelift" } ], - "time": "2024-01-23T14:51:35+00:00" + "time": "2024-04-18T09:32:20+00:00" }, { "name": "symfony/filesystem", - "version": "v6.4.6", + "version": "v6.4.9", "source": { "type": "git", "url": "https://github.com/symfony/filesystem.git", - "reference": "9919b5509ada52cc7f66f9a35c86a4a29955c9d3" + "reference": "b51ef8059159330b74a4d52f68e671033c0fe463" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/filesystem/zipball/9919b5509ada52cc7f66f9a35c86a4a29955c9d3", - "reference": "9919b5509ada52cc7f66f9a35c86a4a29955c9d3", + "url": "https://api.github.com/repos/symfony/filesystem/zipball/b51ef8059159330b74a4d52f68e671033c0fe463", + "reference": "b51ef8059159330b74a4d52f68e671033c0fe463", "shasum": "" }, "require": { @@ -704,6 +704,9 @@ "symfony/polyfill-ctype": "~1.8", "symfony/polyfill-mbstring": "~1.8" }, + "require-dev": { + "symfony/process": "^5.4|^6.4|^7.0" + }, "type": "library", "autoload": { "psr-4": { @@ -730,7 +733,7 @@ "description": "Provides basic utilities for the filesystem", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/filesystem/tree/v6.4.6" + "source": "https://github.com/symfony/filesystem/tree/v6.4.9" }, "funding": [ { @@ -746,20 +749,20 @@ "type": "tidelift" } ], - "time": "2024-03-21T19:36:20+00:00" + "time": "2024-06-28T09:49:33+00:00" }, { "name": "symfony/finder", - "version": "v6.4.0", + "version": "v6.4.11", "source": { "type": "git", "url": "https://github.com/symfony/finder.git", - "reference": "11d736e97f116ac375a81f96e662911a34cd50ce" + "reference": "d7eb6daf8cd7e9ac4976e9576b32042ef7253453" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/finder/zipball/11d736e97f116ac375a81f96e662911a34cd50ce", - "reference": "11d736e97f116ac375a81f96e662911a34cd50ce", + "url": "https://api.github.com/repos/symfony/finder/zipball/d7eb6daf8cd7e9ac4976e9576b32042ef7253453", + "reference": "d7eb6daf8cd7e9ac4976e9576b32042ef7253453", "shasum": "" }, "require": { @@ -794,7 +797,7 @@ "description": "Finds files and directories via an intuitive fluent interface", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/finder/tree/v6.4.0" + "source": "https://github.com/symfony/finder/tree/v6.4.11" }, "funding": [ { @@ -810,24 +813,24 @@ "type": "tidelift" } ], - "time": "2023-10-31T17:30:12+00:00" + "time": "2024-08-13T14:27:37+00:00" }, { "name": "symfony/polyfill-ctype", - "version": "v1.29.0", + "version": "v1.31.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-ctype.git", - "reference": "ef4d7e442ca910c4764bce785146269b30cb5fc4" + "reference": "a3cc8b044a6ea513310cbd48ef7333b384945638" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/ef4d7e442ca910c4764bce785146269b30cb5fc4", - "reference": "ef4d7e442ca910c4764bce785146269b30cb5fc4", + "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/a3cc8b044a6ea513310cbd48ef7333b384945638", + "reference": "a3cc8b044a6ea513310cbd48ef7333b384945638", "shasum": "" }, "require": { - "php": ">=7.1" + "php": ">=7.2" }, "provide": { "ext-ctype": "*" @@ -873,7 +876,7 @@ "portable" ], "support": { - "source": "https://github.com/symfony/polyfill-ctype/tree/v1.29.0" + "source": "https://github.com/symfony/polyfill-ctype/tree/v1.31.0" }, "funding": [ { @@ -889,24 +892,24 @@ "type": "tidelift" } ], - "time": "2024-01-29T20:11:03+00:00" + "time": "2024-09-09T11:45:10+00:00" }, { "name": "symfony/polyfill-intl-grapheme", - "version": "v1.29.0", + "version": "v1.31.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-intl-grapheme.git", - "reference": "32a9da87d7b3245e09ac426c83d334ae9f06f80f" + "reference": "b9123926e3b7bc2f98c02ad54f6a4b02b91a8abe" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-intl-grapheme/zipball/32a9da87d7b3245e09ac426c83d334ae9f06f80f", - "reference": "32a9da87d7b3245e09ac426c83d334ae9f06f80f", + "url": "https://api.github.com/repos/symfony/polyfill-intl-grapheme/zipball/b9123926e3b7bc2f98c02ad54f6a4b02b91a8abe", + "reference": "b9123926e3b7bc2f98c02ad54f6a4b02b91a8abe", "shasum": "" }, "require": { - "php": ">=7.1" + "php": ">=7.2" }, "suggest": { "ext-intl": "For best performance" @@ -951,7 +954,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-intl-grapheme/tree/v1.29.0" + "source": "https://github.com/symfony/polyfill-intl-grapheme/tree/v1.31.0" }, "funding": [ { @@ -967,24 +970,24 @@ "type": "tidelift" } ], - "time": "2024-01-29T20:11:03+00:00" + "time": "2024-09-09T11:45:10+00:00" }, { "name": "symfony/polyfill-intl-normalizer", - "version": "v1.29.0", + "version": "v1.31.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-intl-normalizer.git", - "reference": "bc45c394692b948b4d383a08d7753968bed9a83d" + "reference": "3833d7255cc303546435cb650316bff708a1c75c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-intl-normalizer/zipball/bc45c394692b948b4d383a08d7753968bed9a83d", - "reference": "bc45c394692b948b4d383a08d7753968bed9a83d", + "url": "https://api.github.com/repos/symfony/polyfill-intl-normalizer/zipball/3833d7255cc303546435cb650316bff708a1c75c", + "reference": "3833d7255cc303546435cb650316bff708a1c75c", "shasum": "" }, "require": { - "php": ">=7.1" + "php": ">=7.2" }, "suggest": { "ext-intl": "For best performance" @@ -1032,7 +1035,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-intl-normalizer/tree/v1.29.0" + "source": "https://github.com/symfony/polyfill-intl-normalizer/tree/v1.31.0" }, "funding": [ { @@ -1048,24 +1051,24 @@ "type": "tidelift" } ], - "time": "2024-01-29T20:11:03+00:00" + "time": "2024-09-09T11:45:10+00:00" }, { "name": "symfony/polyfill-mbstring", - "version": "v1.29.0", + "version": "v1.31.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-mbstring.git", - "reference": "9773676c8a1bb1f8d4340a62efe641cf76eda7ec" + "reference": "85181ba99b2345b0ef10ce42ecac37612d9fd341" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/9773676c8a1bb1f8d4340a62efe641cf76eda7ec", - "reference": "9773676c8a1bb1f8d4340a62efe641cf76eda7ec", + "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/85181ba99b2345b0ef10ce42ecac37612d9fd341", + "reference": "85181ba99b2345b0ef10ce42ecac37612d9fd341", "shasum": "" }, "require": { - "php": ">=7.1" + "php": ">=7.2" }, "provide": { "ext-mbstring": "*" @@ -1112,7 +1115,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.29.0" + "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.31.0" }, "funding": [ { @@ -1128,25 +1131,26 @@ "type": "tidelift" } ], - "time": "2024-01-29T20:11:03+00:00" + "time": "2024-09-09T11:45:10+00:00" }, { "name": "symfony/service-contracts", - "version": "v3.4.2", + "version": "v3.5.0", "source": { "type": "git", "url": "https://github.com/symfony/service-contracts.git", - "reference": "11bbf19a0fb7b36345861e85c5768844c552906e" + "reference": "bd1d9e59a81d8fa4acdcea3f617c581f7475a80f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/service-contracts/zipball/11bbf19a0fb7b36345861e85c5768844c552906e", - "reference": "11bbf19a0fb7b36345861e85c5768844c552906e", + "url": "https://api.github.com/repos/symfony/service-contracts/zipball/bd1d9e59a81d8fa4acdcea3f617c581f7475a80f", + "reference": "bd1d9e59a81d8fa4acdcea3f617c581f7475a80f", "shasum": "" }, "require": { "php": ">=8.1", - "psr/container": "^1.1|^2.0" + "psr/container": "^1.1|^2.0", + "symfony/deprecation-contracts": "^2.5|^3" }, "conflict": { "ext-psr": "<1.1|>=2" @@ -1154,7 +1158,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-main": "3.4-dev" + "dev-main": "3.5-dev" }, "thanks": { "name": "symfony/contracts", @@ -1194,7 +1198,7 @@ "standards" ], "support": { - "source": "https://github.com/symfony/service-contracts/tree/v3.4.2" + "source": "https://github.com/symfony/service-contracts/tree/v3.5.0" }, "funding": [ { @@ -1210,20 +1214,20 @@ "type": "tidelift" } ], - "time": "2023-12-19T21:51:00+00:00" + "time": "2024-04-18T09:32:20+00:00" }, { "name": "symfony/string", - "version": "v6.4.4", + "version": "v6.4.11", "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "4e465a95bdc32f49cf4c7f07f751b843bbd6dcd9" + "reference": "5bc3eb632cf9c8dbfd6529d89be9950d1518883b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/4e465a95bdc32f49cf4c7f07f751b843bbd6dcd9", - "reference": "4e465a95bdc32f49cf4c7f07f751b843bbd6dcd9", + "url": "https://api.github.com/repos/symfony/string/zipball/5bc3eb632cf9c8dbfd6529d89be9950d1518883b", + "reference": "5bc3eb632cf9c8dbfd6529d89be9950d1518883b", "shasum": "" }, "require": { @@ -1280,7 +1284,7 @@ "utf8" ], "support": { - "source": "https://github.com/symfony/string/tree/v6.4.4" + "source": "https://github.com/symfony/string/tree/v6.4.11" }, "funding": [ { @@ -1296,7 +1300,7 @@ "type": "tidelift" } ], - "time": "2024-02-01T13:16:41+00:00" + "time": "2024-08-12T09:55:28+00:00" }, { "name": "thecodingmachine/safe",