diff --git a/.github/workflows/cypress.yml b/.github/workflows/cypress.yml index 27de58fce..4b9e3ee2b 100644 --- a/.github/workflows/cypress.yml +++ b/.github/workflows/cypress.yml @@ -19,6 +19,9 @@ jobs: - name: Checkout app uses: actions/checkout@v3 + - name: Install server dependencies + run: composer install + - name: Read package.json node and npm engines version uses: skjnldsv/read-package-engines-version-actions@v2.0 id: versions @@ -35,7 +38,7 @@ jobs: - name: Set up npm ${{ steps.versions.outputs.npmVersion }} run: npm i -g npm@"${{ steps.versions.outputs.npmVersion }}" - - name: Install dependencies & build app + - name: Install node dependencies & build app run: | npm ci TESTING=true npm run build --if-present diff --git a/.php-cs-fixer.dist.php b/.php-cs-fixer.dist.php index f7bbdd812..2dc85d25b 100644 --- a/.php-cs-fixer.dist.php +++ b/.php-cs-fixer.dist.php @@ -2,7 +2,7 @@ declare(strict_types=1); -require_once './vendor/autoload.php'; +require_once __DIR__ . '/vendor/autoload.php'; use Nextcloud\CodingStandard\Config; diff --git a/appinfo/info.xml b/appinfo/info.xml index 140167a40..8ea43379b 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -1,13 +1,13 @@ - + photos Photos Your memories under your control Your memories under your control 2.2.0 agpl - John Molakvoæ + John Molakvoæ Photos multimedia @@ -15,8 +15,8 @@ - https://github.com/nextcloud/photos - https://github.com/nextcloud/photos/issues + https://github.com/nextcloud/photos + https://github.com/nextcloud/photos/issues https://github.com/nextcloud/photos.git @@ -30,6 +30,11 @@ + + OCA\Photos\Command\UpdateReverseGeocodingFilesCommand + OCA\Photos\Command\MapMediaToLocationCommand + + OCA\Photos\Sabre\RootCollection @@ -39,4 +44,8 @@ OCA\Photos\Sabre\Album\PropFindPlugin - + + + OCA\Photos\Jobs\AutomaticLocationMapperJob + + \ No newline at end of file diff --git a/composer.json b/composer.json index 60fcf9716..757985b6d 100644 --- a/composer.json +++ b/composer.json @@ -21,5 +21,8 @@ "vimeo/psalm": "^4.22", "sabre/dav": "^4.2.1", "nextcloud/ocp": "dev-master" + }, + "require": { + "hexogen/kdtree": "^0.2.5" } } diff --git a/composer.lock b/composer.lock index a5c62653d..24f955cb5 100644 --- a/composer.lock +++ b/composer.lock @@ -4,8 +4,70 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "602bc2404448c321b5bbb50160f12007", - "packages": [], + "content-hash": "88bed2a916ac8f06153e69bb691c154c", + "packages": [ + { + "name": "hexogen/kdtree", + "version": "v0.2.5", + "source": { + "type": "git", + "url": "https://github.com/hexogen/kdtree.git", + "reference": "f739186638445990463762d467e07a8262228daa" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/hexogen/kdtree/zipball/f739186638445990463762d467e07a8262228daa", + "reference": "f739186638445990463762d467e07a8262228daa", + "shasum": "" + }, + "require": { + "php": "^7.1|^8.0" + }, + "require-dev": { + "league/csv": "^9.7.0", + "mockery/mockery": "dev-master", + "phpunit/phpunit": "^9.0", + "squizlabs/php_codesniffer": "^3.5.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.0-dev" + } + }, + "autoload": { + "psr-4": { + "Hexogen\\KDTree\\": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Volodymyr Basarab", + "email": "volodymyrbas@gmail.com", + "homepage": "https://github.com/hexogen", + "role": "Developer" + } + ], + "description": "file system KDTree index", + "homepage": "https://github.com/hexogen/kdtree", + "keywords": [ + "algorithms", + "data structures", + "hexogen", + "kdtree", + "search" + ], + "support": { + "issues": "https://github.com/hexogen/kdtree/issues", + "source": "https://github.com/hexogen/kdtree/tree/v0.2.5" + }, + "time": "2022-11-21T13:19:19+00:00" + } + ], "packages-dev": [ { "name": "amphp/amp", diff --git a/lib/Album/AlbumFile.php b/lib/Album/AlbumFile.php index d6f09caa2..7d084bfd2 100644 --- a/lib/Album/AlbumFile.php +++ b/lib/Album/AlbumFile.php @@ -23,19 +23,11 @@ namespace OCA\Photos\Album; -use OC\Metadata\FileMetadata; +use OCA\Photos\DB\PhotosFile; -class AlbumFile { - private int $fileId; - private string $name; - private string $mimeType; - private int $size; - private int $mtime; - private string $etag; +class AlbumFile extends PhotosFile { private int $added; private string $owner; - /** @var array */ - private array $metaData = []; public function __construct( int $fileId, @@ -47,52 +39,19 @@ public function __construct( int $added, string $owner ) { - $this->fileId = $fileId; - $this->name = $name; - $this->mimeType = $mimeType; - $this->size = $size; - $this->mtime = $mtime; - $this->etag = $etag; + parent::__construct( + $fileId, + $name, + $mimeType, + $size, + $mtime, + $etag + ); + $this->added = $added; $this->owner = $owner; } - public function getFileId(): int { - return $this->fileId; - } - - public function getName(): string { - return $this->name; - } - - public function getMimeType(): string { - return $this->mimeType; - } - - public function getSize(): int { - return $this->size; - } - - public function getMTime(): int { - return $this->mtime; - } - - public function getEtag(): string { - return $this->etag; - } - - public function setMetadata(string $key, FileMetadata $value): void { - $this->metaData[$key] = $value; - } - - public function hasMetadata(string $key): bool { - return isset($this->metaData[$key]); - } - - public function getMetadata(string $key): FileMetadata { - return $this->metaData[$key]; - } - public function getAdded(): int { return $this->added; } diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php index d4a67e359..04764bbcf 100644 --- a/lib/AppInfo/Application.php +++ b/lib/AppInfo/Application.php @@ -32,6 +32,7 @@ use OCA\Photos\Listener\TagListener; use OCA\Photos\Listener\GroupUserRemovedListener; use OCA\Photos\Listener\GroupDeletedListener; +use OCA\Photos\Listener\LocationManagerEventListener; use OCP\AppFramework\App; use OCP\AppFramework\Bootstrap\IBootContext; use OCP\AppFramework\Bootstrap\IBootstrap; @@ -40,6 +41,9 @@ use OCP\SystemTag\MapperEvent; use OCP\Group\Events\UserRemovedEvent; use OCP\Group\Events\GroupDeletedEvent; +use OCP\Files\Events\Node\NodeWrittenEvent; + +require_once __DIR__ . '/../../vendor/autoload.php'; class Application extends App implements IBootstrap { public const APP_ID = 'photos'; @@ -78,6 +82,9 @@ public function register(IRegistrationContext $context): void { $context->registerEventListener(GroupDeletedEvent::class, GroupDeletedListener::class); + // Priority of -1 to be triggered after event listeners populating metadata. + $context->registerEventListener(NodeWrittenEvent::class, LocationManagerEventListener::class, -1); + $context->registerEventListener(SabrePluginAuthInitEvent::class, SabrePluginAuthInitListener::class); $context->registerEventListener(MapperEvent::EVENT_ASSIGN, TagListener::class); diff --git a/lib/Command/MapMediaToLocationCommand.php b/lib/Command/MapMediaToLocationCommand.php new file mode 100644 index 000000000..5895b2919 --- /dev/null +++ b/lib/Command/MapMediaToLocationCommand.php @@ -0,0 +1,116 @@ + + * + * @author Louis Chemineau + * + * @license AGPL-3.0-or-later + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ +namespace OCA\Photos\Command; + +use OCP\IConfig; +use OCP\IUserManager; +use OCP\Files\IRootFolder; +use OCP\Files\Folder; +use OCA\Photos\Service\MediaLocationManager; +use Symfony\Component\Console\Command\Command; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Input\InputOption; +use Symfony\Component\Console\Output\OutputInterface; + +class MapMediaToLocationCommand extends Command { + public function __construct( + private IRootFolder $rootFolder, + private MediaLocationManager $mediaLocationManager, + private IConfig $config, + private IUserManager $userManager, + ) { + parent::__construct(); + } + + /** + * Configure the command + */ + protected function configure(): void { + $this->setName('photos:map-media-to-location') + ->setDescription('Reverse geocode media coordinates.') + ->addOption('user', 'u', InputOption::VALUE_REQUIRED, 'Limit the mapping to a user.', null); + } + + /** + * Execute the command + */ + protected function execute(InputInterface $input, OutputInterface $output): int { + if (!$this->config->getSystemValueBool('enable_file_metadata', true)) { + throw new \Exception('File metadata is not enabled.'); + } + + $userId = $input->getOption('user'); + if ($userId === null) { + $this->scanForAllUsers($output); + } else { + $this->scanFilesForUser($userId, $output); + } + + return 0; + } + + private function scanForAllUsers(OutputInterface $output): void { + $users = $this->userManager->search(''); + + $output->writeln("Scanning all users:"); + foreach ($users as $user) { + $this->scanFilesForUser($user->getUID(), $output); + } + } + + private function scanFilesForUser(string $userId, OutputInterface $output): void { + $userFolder = $this->rootFolder->getUserFolder($userId); + $output->write(" - Scanning files for $userId"); + $startTime = time(); + $count = $this->scanFolder($userFolder); + $timeElapse = time() - $startTime; + $output->writeln(" - $count files, $timeElapse sec"); + } + + private function scanFolder(Folder $folder): int { + $count = 0; + + // Do not scan share and other moveable mounts. + if ($folder->getMountPoint() instanceof \OC\Files\Mount\MoveableMount) { + return $count; + } + + foreach ($folder->getDirectoryListing() as $node) { + if ($node instanceof Folder) { + $count += $this->scanFolder($node); + continue; + } + + if (!str_starts_with($node->getMimeType(), 'image')) { + continue; + } + + $this->mediaLocationManager->setLocationForFile($node->getId()); + $count++; + } + + return $count; + } +} diff --git a/lib/Command/UpdateReverseGeocodingFilesCommand.php b/lib/Command/UpdateReverseGeocodingFilesCommand.php new file mode 100644 index 000000000..3d6250f3e --- /dev/null +++ b/lib/Command/UpdateReverseGeocodingFilesCommand.php @@ -0,0 +1,61 @@ + + * + * @author Louis Chemineau + * + * @license AGPL-3.0-or-later + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ +namespace OCA\Photos\Command; + +use OCA\Photos\Service\ReverseGeoCoderService; +use Symfony\Component\Console\Command\Command; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Output\OutputInterface; + +class UpdateReverseGeocodingFilesCommand extends Command { + public function __construct( + private ReverseGeoCoderService $rgcService, + ) { + parent::__construct(); + } + + /** + * Configure the command + */ + protected function configure(): void { + $this->setName('photos:update-1000-cities') + ->setDescription('Update the list of 1000 and more inhabitant cities'); + } + + /** + * Execute the command + */ + protected function execute(InputInterface $input, OutputInterface $output): int { + try { + $this->rgcService->buildKDTree(true); + } catch (\Exception $ex) { + $output->writeln('Failed to update reverse geocoding files'); + $output->writeln($ex->getMessage()); + return 1; + } + + return 0; + } +} diff --git a/lib/DB/Location/LocationFile.php b/lib/DB/Location/LocationFile.php new file mode 100644 index 000000000..4296ea0e4 --- /dev/null +++ b/lib/DB/Location/LocationFile.php @@ -0,0 +1,53 @@ + + * + * @author Louis Chemineau + * + * @license AGPL-3.0-or-later + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCA\Photos\DB\Location; + +use OCA\Photos\DB\PhotosFile; + +class LocationFile extends PhotosFile { + public function __construct( + int $fileId, + string $name, + string $mimeType, + int $size, + int $mtime, + string $etag, + private string $location, + ) { + parent::__construct( + $fileId, + $name, + $mimeType, + $size, + $mtime, + $etag, + ); + } + + public function getLocation(): string { + return $this->location; + } +} diff --git a/lib/DB/Location/LocationInfo.php b/lib/DB/Location/LocationInfo.php new file mode 100644 index 000000000..24b0c5dfd --- /dev/null +++ b/lib/DB/Location/LocationInfo.php @@ -0,0 +1,42 @@ + + * + * @author Louis Chemineau + * + * @license AGPL-3.0-or-later + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCA\Photos\DB\Location; + +class LocationInfo { + public function __construct( + private string $userId, + private string $location + ) { + } + + public function getUserId(): string { + return $this->userId; + } + + public function getLocation(): string { + return $this->location; + } +} diff --git a/lib/DB/Location/LocationMapper.php b/lib/DB/Location/LocationMapper.php new file mode 100644 index 000000000..e3b9cac39 --- /dev/null +++ b/lib/DB/Location/LocationMapper.php @@ -0,0 +1,127 @@ + + * + * @author Louis Chemineau + * + * @license AGPL-3.0-or-later + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCA\Photos\DB\Location; + +use Doctrine\DBAL\Exception\UniqueConstraintViolationException; +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\Files\IMimeTypeLoader; +use OCP\Files\IRootFolder; +use OCP\IDBConnection; + +class LocationMapper { + public const METADATA_TYPE = 'photos_location'; + + public function __construct( + private IDBConnection $connection, + private IMimeTypeLoader $mimeTypeLoader, + private IRootFolder $rootFolder, + ) { + } + + /** @return LocationInfo[] */ + public function findLocationsForUser(string $userId): array { + $mountId = $this->rootFolder + ->getUserFolder($userId) + ->getMountPoint() + ->getMountId(); + $mimepart = $this->mimeTypeLoader->getId('image'); + + $qb = $this->connection->getQueryBuilder(); + + $rows = $qb->selectDistinct('meta.metadata') + ->from('mounts', 'mount') + ->join('mount', 'filecache', 'file', $qb->expr()->eq('file.storage', 'mount.storage_id', IQueryBuilder::PARAM_INT)) + ->join('file', 'file_metadata', 'meta', $qb->expr()->eq('file.fileid', 'meta.id', IQueryBuilder::PARAM_INT)) + ->where($qb->expr()->eq('mount.id', $qb->createNamedParameter($mountId), IQueryBuilder::PARAM_INT)) + ->andWhere($qb->expr()->eq('file.mimepart', $qb->createNamedParameter($mimepart, IQueryBuilder::PARAM_INT))) + ->andWhere($qb->expr()->eq('meta.group_name', $qb->createNamedParameter(self::METADATA_TYPE))) + ->executeQuery() + ->fetchAll(); + + return array_map(fn ($row) => new LocationInfo($userId, $row['metadata']), $rows); + } + + /** @return LocationFile[] */ + public function findFilesForUserAndLocation(string $userId, string $location) { + $mountId = $this->rootFolder + ->getUserFolder($userId) + ->getMountPoint() + ->getMountId(); + $mimepart = $this->mimeTypeLoader->getId('image'); + + $qb = $this->connection->getQueryBuilder(); + + $rows = $qb->select('file.fileid', 'file.name', 'file.mimetype', 'file.size', 'file.mtime', 'file.etag', 'meta.metadata') + ->from('mounts', 'mount') + ->join('mount', 'filecache', 'file', $qb->expr()->eq('file.storage', 'mount.storage_id', IQueryBuilder::PARAM_INT)) + ->join('file', 'file_metadata', 'meta', $qb->expr()->eq('file.fileid', 'meta.id', IQueryBuilder::PARAM_INT)) + ->where($qb->expr()->eq('mount.id', $qb->createNamedParameter($mountId), IQueryBuilder::PARAM_INT)) + ->andWhere($qb->expr()->eq('file.mimepart', $qb->createNamedParameter($mimepart, IQueryBuilder::PARAM_INT))) + ->andWhere($qb->expr()->eq('meta.group_name', $qb->createNamedParameter(self::METADATA_TYPE))) + ->andWhere($qb->expr()->eq('meta.metadata', $qb->createNamedParameter($location))) + ->executeQuery() + ->fetchAll(); + + return array_map( + fn ($row) => new LocationFile( + (int)$row['fileid'], + $row['name'], + $this->mimeTypeLoader->getMimetypeById($row['mimetype']), + (int)$row['size'], + (int)$row['mtime'], + $row['etag'], + $row['metadata'] + ), + $rows, + ); + } + + public function setLocationForFile(string $location, int $fileId): void { + try { + $query = $this->connection->getQueryBuilder(); + $query->insert('file_metadata') + ->values([ + "id" => $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT), + "group_name" => $query->createNamedParameter(self::METADATA_TYPE), + "metadata" => $query->createNamedParameter($location), + ]) + ->executeStatement(); + } catch (\Exception $ex) { + if ($ex->getPrevious() instanceof UniqueConstraintViolationException) { + $this->updateLocationForFile($location, $fileId); + } + } + } + + public function updateLocationForFile(string $location, int $fileId): void { + $query = $this->connection->getQueryBuilder(); + $query->update('file_metadata') + ->set("metadata", $query->createNamedParameter($location)) + ->where($query->expr()->eq('id', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))) + ->andWhere($query->expr()->eq('group_name', $query->createNamedParameter(self::METADATA_TYPE))) + ->executeStatement(); + } +} diff --git a/lib/DB/PhotosFile.php b/lib/DB/PhotosFile.php new file mode 100644 index 000000000..7a3ab90a9 --- /dev/null +++ b/lib/DB/PhotosFile.php @@ -0,0 +1,79 @@ + + * + * @author Louis Chemineau + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCA\Photos\DB; + +use OC\Metadata\FileMetadata; + +class PhotosFile { + /** @var array */ + private array $metaData = []; + + public function __construct( + private int $fileId, + private string $name, + private string $mimeType, + private int $size, + private int $mtime, + private string $etag, + ) { + } + + public function getFileId(): int { + return $this->fileId; + } + + public function getName(): string { + return $this->name; + } + + public function getMimeType(): string { + return $this->mimeType; + } + + public function getSize(): int { + return $this->size; + } + + public function getMTime(): int { + return $this->mtime; + } + + public function getEtag(): string { + return $this->etag; + } + + public function setMetadata(string $key, FileMetadata $value): void { + $this->metaData[$key] = $value; + } + + public function hasMetadata(string $key): bool { + return isset($this->metaData[$key]); + } + + public function getMetadata(string $key): FileMetadata { + return $this->metaData[$key]; + } +} diff --git a/lib/Jobs/AutomaticLocationMapperJob.php b/lib/Jobs/AutomaticLocationMapperJob.php new file mode 100644 index 000000000..fdfc0eb26 --- /dev/null +++ b/lib/Jobs/AutomaticLocationMapperJob.php @@ -0,0 +1,114 @@ + + * + * @author Louis Chemineau + * + * @license AGPL-3.0-or-later + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCA\Photos\Jobs; + +use OCA\Photos\AppInfo\Application; +use OCA\Photos\Service\MediaLocationManager; +use OCP\AppFramework\Utility\ITimeFactory; +use OCP\BackgroundJob\TimedJob; +use OCP\Files\Folder; +use OCP\Files\IRootFolder; +use OCP\IConfig; +use OCP\IUserManager; + +class AutomaticLocationMapperJob extends TimedJob { + public function __construct( + ITimeFactory $time, + private IConfig $config, + private IRootFolder $rootFolder, + private IUserManager $userManager, + private MediaLocationManager $mediaLocationManager, + ) { + parent::__construct($time); + $this->mediaLocationManager = $mediaLocationManager; + + $this->setTimeSensitivity(\OCP\BackgroundJob\IJob::TIME_INSENSITIVE); + $this->setInterval(24 * 3600); + } + + protected function run($argument) { + $locationMappingDone = $this->config->getAppValue(Application::APP_ID, 'lastLocationMappingDone', 'false'); + + if ($locationMappingDone === 'true') { + return; + } + + $users = $this->userManager->search(''); + $lastMappedUser = $this->config->getAppValue(Application::APP_ID, 'lastLocationMappedUser', ''); + + if ($lastMappedUser === '') { + $lastMappedUser = $users[array_key_first($users)]->getUID(); + } + + $startTime = null; + foreach ($users as $user) { + if ($startTime === null) { + // Skip all user before lastMappedUser. + if ($lastMappedUser !== $user->getUID()) { + continue; + } + + $startTime = time(); + } + + // Stop if execution time is more than one hour. + if (time() - $startTime > 60 * 60) { + return; + } + + $this->scanFilesForUser($user->getUID()); + $this->config->setAppValue(Application::APP_ID, 'lastLocationMappedUser', $user->getUID()); + } + + $this->config->setAppValue(Application::APP_ID, 'lastLocationMappingDone', 'true'); + } + + private function scanFilesForUser(string $userId): void { + $userFolder = $this->rootFolder->getUserFolder($userId); + $this->scanFolder($userFolder); + } + + + private function scanFolder(Folder $folder): void { + // Do not scan share and other moveable mounts. + if ($folder->getMountPoint() instanceof \OC\Files\Mount\MoveableMount) { + return; + } + + foreach ($folder->getDirectoryListing() as $node) { + if ($node instanceof Folder) { + $this->scanFolder($node); + continue; + } + + if (!str_starts_with($node->getMimeType(), 'image')) { + continue; + } + + $this->mediaLocationManager->setLocationForFile($node->getId()); + } + } +} diff --git a/lib/Jobs/MapMediaToLocationJob.php b/lib/Jobs/MapMediaToLocationJob.php new file mode 100644 index 000000000..1f3a9f4f0 --- /dev/null +++ b/lib/Jobs/MapMediaToLocationJob.php @@ -0,0 +1,48 @@ + + * + * @author Louis Chemineau + * + * @license AGPL-3.0-or-later + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCA\Photos\Jobs; + +use OCA\Photos\Service\MediaLocationManager; +use OCP\AppFramework\Utility\ITimeFactory; +use OCP\BackgroundJob\QueuedJob; + +class MapMediaToLocationJob extends QueuedJob { + private MediaLocationManager $mediaLocationManager; + + public function __construct( + ITimeFactory $time, + MediaLocationManager $mediaLocationManager + ) { + parent::__construct($time); + $this->mediaLocationManager = $mediaLocationManager; + } + + protected function run($argument) { + [$fileId] = $argument; + + $this->mediaLocationManager->setLocationForFile($fileId); + } +} diff --git a/lib/Listener/LocationManagerEventListener.php b/lib/Listener/LocationManagerEventListener.php new file mode 100644 index 000000000..9a78243e8 --- /dev/null +++ b/lib/Listener/LocationManagerEventListener.php @@ -0,0 +1,71 @@ + + * + * @author Louis Chemineau + * + * @license AGPL-3.0-or-later + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCA\Photos\Listener; + +use OCA\Photos\Jobs\MapMediaToLocationJob; +use OCA\Photos\Service\MediaLocationManager; +use OCP\BackgroundJob\IJobList; +use OCP\IConfig; +use OCP\EventDispatcher\Event; +use OCP\EventDispatcher\IEventListener; +use OCP\Files\Events\Node\NodeWrittenEvent; + +/** + * Listener to create, update or remove location info from the database. + */ +class LocationManagerEventListener implements IEventListener { + public function __construct( + private MediaLocationManager $mediaLocationManager, + private IConfig $config, + private IJobList $jobList, + ) { + } + + public function handle(Event $event): void { + if (!$this->config->getSystemValueBool('enable_file_metadata', true)) { + return; + } + + if ($event instanceof NodeWrittenEvent) { + if (!$this->isCorrectPath($event->getNode()->getPath())) { + return; + } + + if (!str_starts_with($event->getNode()->getMimeType(), 'image')) { + return; + } + + $fileId = $event->getNode()->getId(); + + $this->jobList->add(MapMediaToLocationJob::class, [$fileId]); + } + } + + private function isCorrectPath(string $path): bool { + // TODO make this more dynamic, we have the same issue in other places + return !str_starts_with($path, 'appdata_') && !str_starts_with($path, 'files_versions/'); + } +} diff --git a/lib/Service/MediaLocationManager.php b/lib/Service/MediaLocationManager.php new file mode 100644 index 000000000..ddaa24d34 --- /dev/null +++ b/lib/Service/MediaLocationManager.php @@ -0,0 +1,76 @@ + + * + * @author Louis Chemineau + * + * @license AGPL-3.0-or-later + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCA\Photos\Service; + +use OC\Metadata\IMetadataManager; +use OCA\Photos\DB\Location\LocationMapper; + +class MediaLocationManager { + public function __construct( + private IMetadataManager $metadataManager, + private ReverseGeoCoderService $rgcService, + private LocationMapper $locationMapper, + ) { + } + + public function setLocationForFile(int $fileId): void { + $location = $this->getLocationForFile($fileId); + + if ($location === null) { + return; + } + + $this->locationMapper->setLocationForFile($location, $fileId); + } + + public function updateLocationForFile(int $fileId): void { + $location = $this->getLocationForFile($fileId); + + if ($location === null) { + return; + } + + $this->locationMapper->updateLocationForFile($location, $fileId); + } + + private function getLocationForFile(int $fileId): ?string { + $gpsMetadata = $this->metadataManager->fetchMetadataFor('gps', [$fileId])[$fileId]; + $metadata = $gpsMetadata->getMetadata(); + + if (count($metadata) === 0) { + return null; + } + + $latitude = $metadata['latitude']; + $longitude = $metadata['longitude']; + + if ($latitude === null || $longitude === null) { + return null; + } + + return $this->rgcService->getLocationForCoordinates($latitude, $longitude); + } +} diff --git a/lib/Service/ReverseGeoCoderService.php b/lib/Service/ReverseGeoCoderService.php new file mode 100644 index 000000000..94685b02f --- /dev/null +++ b/lib/Service/ReverseGeoCoderService.php @@ -0,0 +1,168 @@ + + * + * @author Louis Chemineau + * + * @license AGPL-3.0-or-later + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ +namespace OCA\Photos\Service; + +use OCP\Files\IAppData; +use OCP\Files\SimpleFS\ISimpleFolder; +use OCP\Files\NotFoundException; +use OCP\Http\Client\IClientService; +use Hexogen\KDTree\FSTreePersister; +use Hexogen\KDTree\FSKDTree; +use Hexogen\KDTree\KDTree; +use Hexogen\KDTree\Item; +use Hexogen\KDTree\ItemList; +use Hexogen\KDTree\ItemFactory; +use Hexogen\KDTree\NearestSearch; +use Hexogen\KDTree\Point; + +class ReverseGeoCoderService { + private ISimpleFolder $geoNameFolder; + private ?NearestSearch $fsSearcher = null; + /** @var array */ + private ?array $citiesMapping = null; + + public function __construct( + IAppData $appData, + private IClientService $clientService, + ) { + try { + $this->geoNameFolder = $appData->getFolder("geonames"); + } catch (NotFoundException $ex) { + $this->geoNameFolder = $appData->newFolder("geonames"); + } + } + + public function getLocationForCoordinates(float $latitude, float $longitude): string { + $this->loadKdTree(); + $result = $this->fsSearcher->search(new Point([$latitude, $longitude]), 1); + return $this->getLocationNameForLocationId($result[0]->getId()); + } + + private function getLocationNameForLocationId(int $locationId): string { + if ($this->citiesMapping === null) { + $this->downloadCities1000(); + $cities1000 = $this->loadCities1000(); + $this->citiesMapping = []; + foreach ($cities1000 as $city) { + $this->citiesMapping[$city['id']] = $city['name']; + } + } + + return $this->citiesMapping[$locationId]; + } + + private function downloadCities1000(bool $force = false): void { + if ($this->geoNameFolder->fileExists('cities1000.csv') && !$force) { + return; + } + + // Download zip file to a tmp file. + $response = $this->clientService->newClient()->get("https://download.geonames.org/export/dump/cities1000.zip"); + $tmpFile = tmpfile(); + $cities1000ZipTmpFileName = stream_get_meta_data($tmpFile)['uri']; + fclose($tmpFile); + file_put_contents($cities1000ZipTmpFileName, $response->getBody()); + + // Unzip the txt file into a stream. + $zip = new \ZipArchive; + $res = $zip->open($cities1000ZipTmpFileName); + if ($res !== true) { + throw new \Exception("Fail to unzip location file: $res", $res); + } + $cities1000TxtSteam = $zip->getStream('cities1000.txt'); + + // Dump the txt file info into a smaller csv file. + $destinationStream = $this->geoNameFolder->newFile('cities1000.csv')->write(); + + while (($fields = fgetcsv($cities1000TxtSteam, 0, " ")) !== false) { + $result = fputcsv( + $destinationStream, + [ + 'id' => (int)$fields[0], + 'name' => $fields[1], + 'latitude' => (float)$fields[4], + 'longitude' => (float)$fields[5], + ] + ); + + if ($result === false) { + throw new \Exception('Failed to write csv line to tmp stream'); + } + } + + $zip->close(); + } + + private function loadCities1000(): array { + $csvStream = $this->geoNameFolder->getFile('cities1000.csv')->read(); + $cities = []; + + while (($fields = fgetcsv($csvStream)) !== false) { + $cities[] = [ + 'id' => (int)$fields[0], + 'name' => $fields[1], + 'latitude' => (float)$fields[2], + 'longitude' => (float)$fields[3], + ]; + } + + return $cities; + } + + public function buildKDTree($force = false): void { + if ($this->geoNameFolder->fileExists('cities1000.bin') && !$force) { + return; + } + + $this->downloadCities1000($force); + $cities1000 = $this->loadCities1000(); + + $itemList = new ItemList(2); + foreach ($cities1000 as $city) { + $itemList->addItem(new Item($city['id'], [$city['latitude'], $city['longitude']])); + } + $tree = new KDTree($itemList); + + // Persiste KDTree in app data. + $persister = new FSTreePersister('/'); + $kdTreeTmpFileName = tempnam(sys_get_temp_dir(), "nextcloud_photos_"); + $persister->convert($tree, $kdTreeTmpFileName); + $kdTreeString = file_get_contents($kdTreeTmpFileName); + $this->geoNameFolder->newFile('cities1000.bin', $kdTreeString); + } + + private function loadKdTree(): void { + if ($this->fsSearcher !== null) { + return; + } + + $this->buildKDTree(); + $kdTreeFileContent = $this->geoNameFolder->getFile("cities1000.bin")->getContent(); + $kdTreeTmpFileName = tempnam(sys_get_temp_dir(), "nextcloud_photos_"); + file_put_contents($kdTreeTmpFileName, $kdTreeFileContent); + $fsTree = new FSKDTree($kdTreeTmpFileName, new ItemFactory()); + $this->fsSearcher = new NearestSearch($fsTree); + } +} diff --git a/package.json b/package.json index 25b70bc8f..4858b3a7c 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "photos", "description": "Your memories under your control", - "version": "2.2.0", + "version": "2.2.1", "author": "John Molakvoæ ", "contributors": [ "John Molakvoæ " diff --git a/tests/stub.phpstub b/tests/stub.phpstub index 038e42e2b..1cf150a3b 100644 --- a/tests/stub.phpstub +++ b/tests/stub.phpstub @@ -118,6 +118,7 @@ namespace Symfony\Component\Console\Question { namespace Symfony\Component\Console\Output { class OutputInterface { public const VERBOSITY_VERBOSE = 1; + public function write($messages, $newline = false, $options = 0); public function writeln(string $text, int $flat = 0) {} } } @@ -269,10 +270,10 @@ namespace OC\Files\Mount { protected $class; protected $storageId; protected $rootId = null; - + /** @var int|null */ protected $mountId; - + /** * @param string|\OCP\Files\Storage\IStorage $storage * @param string $mountpoint @@ -285,7 +286,7 @@ namespace OC\Files\Mount { public function __construct($storage, $mountpoint, $arguments = null, $loader = null, $mountOptions = null, $mountId = null) { throw new \Exception('stub'); } - + /** * get complete path to the mount point, relative to data/ * @@ -294,7 +295,7 @@ namespace OC\Files\Mount { public function getMountPoint() { throw new \Exception('stub'); } - + /** * Sets the mount point path, relative to data/ * @@ -303,28 +304,28 @@ namespace OC\Files\Mount { public function setMountPoint($mountPoint) { throw new \Exception('stub'); } - + /** * @return \OCP\Files\Storage\IStorage */ public function getStorage() { throw new \Exception('stub'); } - + /** * @return string */ public function getStorageId() { throw new \Exception('stub'); } - + /** * @return int */ public function getNumericStorageId() { throw new \Exception('stub'); } - + /** * @param string $path * @return string @@ -332,14 +333,14 @@ namespace OC\Files\Mount { public function getInternalPath($path) { throw new \Exception('stub'); } - + /** * @param callable $wrapper */ public function wrapStorage($wrapper) { throw new \Exception('stub'); } - + /** * Get a mount option * @@ -350,7 +351,7 @@ namespace OC\Files\Mount { public function getOption($name, $default) { throw new \Exception('stub'); } - + /** * Get all options for the mount * @@ -359,18 +360,18 @@ namespace OC\Files\Mount { public function getOptions() { throw new \Exception('stub'); } - + /** * @return int */ public function getStorageRootId() { throw new \Exception('stub'); } - + public function getMountId() { throw new \Exception('stub'); } - + public function getMountType() { throw new \Exception('stub'); } @@ -656,7 +657,7 @@ use OCP\DB\Types; /** * @method string getGroupName() * @method void setGroupName(string $groupName) - * @method string getMetadata() + * @method array getMetadata() * @method void setMetadata(array $metadata) * @see \OC\Core\Migrations\Version240000Date20220404230027 */ @@ -686,3 +687,25 @@ namespace OCA\DAV\Upload { namespace Doctrine\DBAL\Exception { class UniqueConstraintViolationException extends \Exception {} } + +namespace OC\Files\Mount; + +/** + * Defines the mount point to be (re)moved by the user + */ +interface MoveableMount { + /** + * Move the mount point to $target + * + * @param string $target the target mount point + * @return bool + */ + public function moveMount($target); + + /** + * Remove the mount points + * + * @return bool + */ + public function removeMount(); +}