Merge pull request #1172 from nielstron/feat/retry

Add option to only retry non-tagged files in classify
nextcloud · Sep 13, 2024 · 53bc672 · 53bc672
2 parents 7c5538f + 6c8da50
commit 53bc672
Show file tree

Hide file tree

Showing 8 changed files with 109 additions and 87 deletions.
diff --git a/appinfo/info.xml b/appinfo/info.xml
@@ -92,7 +92,7 @@ The app does not send any sensitive data to cloud providers or similar services.
 	<screenshot>https://raw.githubusercontent.com/nextcloud/recognize/main/screenshots/Logo.png</screenshot>
 	<screenshot>https://raw.githubusercontent.com/nextcloud/recognize/main/screenshots/imagenet_examples.jpg</screenshot>
 	<dependencies>
-		<nextcloud min-version="31" max-version="31" />
+		<nextcloud min-version="30" max-version="31" />
 	</dependencies>
 
 	<repair-steps>

diff --git a/lib/BackgroundJobs/ClassifierJob.php b/lib/BackgroundJobs/ClassifierJob.php
@@ -71,15 +71,15 @@ protected function runClassifier(string $model, array $argument): void {
 		try {
 			$this->logger->debug('Running ' . $model . ' classifier');
 			$this->classify($files);
-		} catch(\RuntimeException $e) {
+		} catch (\RuntimeException $e) {
 			$this->logger->warning('Temporary problem with ' . $model . ' classifier, trying again soon', ['exception' => $e]);
-		} catch(\ErrorException $e) {
+		} catch (\ErrorException $e) {
 			$this->settingsService->setSetting($model.'.status', 'false');
 			$this->logger->warning('Problem with ' . $model . ' classifier', ['exception' => $e]);
 			$this->logger->debug('Removing '.static::class.' with argument ' . var_export($argument, true) . 'from oc_jobs');
 			$this->jobList->remove(static::class, $argument);
 			throw $e;
-		} catch(\Throwable $e) {
+		} catch (\Throwable $e) {
 			$this->settingsService->setSetting($model.'.status', 'false');
 			throw $e;
 		}

diff --git a/lib/BackgroundJobs/StorageCrawlJob.php b/lib/BackgroundJobs/StorageCrawlJob.php
@@ -77,7 +77,6 @@ protected function run($argument): void {
 					}
 					if (!in_array(ImagenetClassifier::MODEL_NAME, $models) && in_array(LandmarksClassifier::MODEL_NAME, $models)) {
 						$tags = $this->tagManager->getTagsForFiles([$queueFile->getFileId()]);
-						/** @var \OCP\SystemTag\ISystemTag[] $fileTags */
 						$fileTags = $tags[$queueFile->getFileId()];
 						$landmarkTags = array_filter($fileTags, function ($tag) {
 							return in_array($tag->getName(), LandmarksClassifier::PRECONDITION_TAGS);

diff --git a/lib/Command/Classify.php b/lib/Command/Classify.php
@@ -16,11 +16,14 @@
 use OCA\Recognize\Service\Logger;
 use OCA\Recognize\Service\SettingsService;
 use OCA\Recognize\Service\StorageService;
+use OCA\Recognize\Service\TagManager;
 use OCP\Files\Config\ICachedMountInfo;
 use OCP\Files\Config\IUserMountCache;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Exception\ExceptionInterface;
+use Symfony\Component\Console\Input\ArrayInput;
 use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
 
 class Classify extends Command {
@@ -29,6 +32,7 @@ class Classify extends Command {
 
 	public function __construct(
 		private StorageService $storageService,
+		private TagManager $tagManager,
 		private Logger $logger,
 		ImagenetClassifier $imagenet,
 		ClusteringFaceClassifier $faces,
@@ -53,7 +57,8 @@ public function __construct(
 	 */
 	protected function configure() {
 		$this->setName('recognize:classify')
-			->setDescription('Classify all files with the current settings in one go (will likely take a long time)');
+			->setDescription('Classify all files with the current settings in one go (will likely take a long time)')
+			->addOption('retry', null, InputOption::VALUE_NONE, "Only classify untagged images");
 	}
 
 	/**
@@ -68,7 +73,9 @@ protected function configure() {
 	protected function execute(InputInterface $input, OutputInterface $output): int {
 		$this->logger->setCliOutput($output);
 
-		$this->clearBackgroundJobs->run($input, $output);
+		// pop "retry" flag from parameters passed to clear background jobs
+		$clearBackgroundJobs = new ArrayInput([]);
+		$this->clearBackgroundJobs->run($clearBackgroundJobs, $output);
 
 		$models = array_values(array_filter([
 			ClusteringFaceClassifier::MODEL_NAME,
@@ -77,6 +84,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int
 			MusicnnClassifier::MODEL_NAME,
 		], fn ($modelName) => $this->settings->getSetting($modelName . '.enabled') === 'true'));
 
+		$processedTag = $this->tagManager->getProcessedTag();
+
 		foreach ($this->storageService->getMounts() as $mount) {
 			$this->logger->info('Processing storage ' . $mount['storage_id'] . ' with root ID ' . $mount['override_root']);
 
@@ -107,13 +116,23 @@ protected function execute(InputInterface $input, OutputInterface $output): int
 					$queueFile->setUpdate(false);
 
 					if ($file['image']) {
-						if (in_array(ImagenetClassifier::MODEL_NAME, $models)) {
-							$queues[ImagenetClassifier::MODEL_NAME][] = $queueFile;
-						}
 						if (in_array(ClusteringFaceClassifier::MODEL_NAME, $models)) {
 							$queues[ClusteringFaceClassifier::MODEL_NAME][] = $queueFile;
 						}
 					}
+					// if retry flag is set, skip other classifiers for tagged files
+					if ($input->getOption('retry')) {
+						$fileTags = $this->tagManager->getTagsForFiles([$lastFileId]);
+						// check if processed tag is already in the tags
+						if (in_array($processedTag, $fileTags[$lastFileId])) {
+							continue;
+						}
+					}
+					if ($file['image']) {
+						if (in_array(ImagenetClassifier::MODEL_NAME, $models)) {
+							$queues[ImagenetClassifier::MODEL_NAME][] = $queueFile;
+						}
+					}
 					if ($file['video']) {
 						if (in_array(MovinetClassifier::MODEL_NAME, $models)) {
 							$queues[MovinetClassifier::MODEL_NAME][] = $queueFile;

diff --git a/lib/Controller/AdminController.php b/lib/Controller/AdminController.php
@@ -136,7 +136,7 @@ public function hasJobs(string $task): JSONResponse {
 		}
 		$iterator = $this->jobList->getJobsIterator($tasks[$task], null, 0);
 		$lastRun = [];
-		foreach($iterator as $job) {
+		foreach ($iterator as $job) {
 			$lastRun[] = $job->getLastRun();
 		}
 		$count = count($lastRun);

diff --git a/lib/Migration/InstallDeps.php b/lib/Migration/InstallDeps.php
@@ -90,7 +90,7 @@ public function run(IOutput $output): void {
 			$this->runFfmpegInstall($binaryPath);
 			$this->runTfjsGpuInstall($binaryPath);
 			$this->setNiceBinaryPath();
-		} catch(\Throwable $e) {
+		} catch (\Throwable $e) {
 			$output->warning('Failed to automatically install dependencies for recognize. Check the recognize admin panel for potential problems.');
 			$this->logger->error('Failed to automatically install dependencies for recognize. Check the recognize admin panel for potential problems.', ['exception' => $e]);
 		}

diff --git a/lib/Service/TagManager.php b/lib/Service/TagManager.php
@@ -79,7 +79,7 @@ public function assignTags(int $fileId, array $tags): void {
 
 	/**
 	 * @param array $fileIds
-	 * @return array
+	 * @return array<array-key, array<array-key,ISystemTag>>
 	 */
 	public function getTagsForFiles(array $fileIds): array {
 		/** @var array<string, string[]> $tagsByFile */