Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add a metadata service to store file metadata #31839

Merged
merged 2 commits into from
Apr 14, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion 3rdparty
5 changes: 5 additions & 0 deletions apps/accessibility/composer/autoload.php
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,11 @@

// autoload.php @generated by Composer

if (PHP_VERSION_ID < 50600) {
echo 'Composer 2.3.0 dropped support for autoloading on PHP <5.6 and you are running '.PHP_VERSION.', please upgrade PHP or use Composer 2.2 LTS via "composer self-update --2.2". Aborting.'.PHP_EOL;
exit(1);
}

require_once __DIR__ . '/composer/autoload_real.php';

return ComposerAutoloaderInitAccessibility::getLoader();
2 changes: 1 addition & 1 deletion apps/accessibility/composer/composer/autoload_real.php
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ public static function getLoader()
spl_autoload_unregister(array('ComposerAutoloaderInitAccessibility', 'loadClassLoader'));

require __DIR__ . '/autoload_static.php';
\Composer\Autoload\ComposerStaticInitAccessibility::getInitializer($loader)();
call_user_func(\Composer\Autoload\ComposerStaticInitAccessibility::getInitializer($loader));

$loader->setClassMapAuthoritative(true);
$loader->register(true);
Expand Down
5 changes: 5 additions & 0 deletions apps/admin_audit/composer/autoload.php
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,11 @@

// autoload.php @generated by Composer

if (PHP_VERSION_ID < 50600) {
echo 'Composer 2.3.0 dropped support for autoloading on PHP <5.6 and you are running '.PHP_VERSION.', please upgrade PHP or use Composer 2.2 LTS via "composer self-update --2.2". Aborting.'.PHP_EOL;
exit(1);
}

require_once __DIR__ . '/composer/autoload_real.php';

return ComposerAutoloaderInitAdminAudit::getLoader();
2 changes: 1 addition & 1 deletion apps/admin_audit/composer/composer/autoload_real.php
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ public static function getLoader()
spl_autoload_unregister(array('ComposerAutoloaderInitAdminAudit', 'loadClassLoader'));

require __DIR__ . '/autoload_static.php';
\Composer\Autoload\ComposerStaticInitAdminAudit::getInitializer($loader)();
call_user_func(\Composer\Autoload\ComposerStaticInitAdminAudit::getInitializer($loader));

$loader->setClassMapAuthoritative(true);
$loader->register(true);
Expand Down
5 changes: 5 additions & 0 deletions apps/cloud_federation_api/composer/autoload.php
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,11 @@

// autoload.php @generated by Composer

if (PHP_VERSION_ID < 50600) {
echo 'Composer 2.3.0 dropped support for autoloading on PHP <5.6 and you are running '.PHP_VERSION.', please upgrade PHP or use Composer 2.2 LTS via "composer self-update --2.2". Aborting.'.PHP_EOL;
exit(1);
}

require_once __DIR__ . '/composer/autoload_real.php';

return ComposerAutoloaderInitCloudFederationAPI::getLoader();
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ public static function getLoader()
spl_autoload_unregister(array('ComposerAutoloaderInitCloudFederationAPI', 'loadClassLoader'));

require __DIR__ . '/autoload_static.php';
\Composer\Autoload\ComposerStaticInitCloudFederationAPI::getInitializer($loader)();
call_user_func(\Composer\Autoload\ComposerStaticInitCloudFederationAPI::getInitializer($loader));

$loader->setClassMapAuthoritative(true);
$loader->register(true);
Expand Down
5 changes: 5 additions & 0 deletions apps/comments/composer/autoload.php
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,11 @@

// autoload.php @generated by Composer

if (PHP_VERSION_ID < 50600) {
echo 'Composer 2.3.0 dropped support for autoloading on PHP <5.6 and you are running '.PHP_VERSION.', please upgrade PHP or use Composer 2.2 LTS via "composer self-update --2.2". Aborting.'.PHP_EOL;
exit(1);
}

require_once __DIR__ . '/composer/autoload_real.php';

return ComposerAutoloaderInitComments::getLoader();
2 changes: 1 addition & 1 deletion apps/comments/composer/composer/autoload_real.php
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ public static function getLoader()
spl_autoload_unregister(array('ComposerAutoloaderInitComments', 'loadClassLoader'));

require __DIR__ . '/autoload_static.php';
\Composer\Autoload\ComposerStaticInitComments::getInitializer($loader)();
call_user_func(\Composer\Autoload\ComposerStaticInitComments::getInitializer($loader));

$loader->setClassMapAuthoritative(true);
$loader->register(true);
Expand Down
5 changes: 5 additions & 0 deletions apps/contactsinteraction/composer/autoload.php
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,11 @@

// autoload.php @generated by Composer

if (PHP_VERSION_ID < 50600) {
echo 'Composer 2.3.0 dropped support for autoloading on PHP <5.6 and you are running '.PHP_VERSION.', please upgrade PHP or use Composer 2.2 LTS via "composer self-update --2.2". Aborting.'.PHP_EOL;
exit(1);
}

require_once __DIR__ . '/composer/autoload_real.php';

return ComposerAutoloaderInitContactsInteraction::getLoader();
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ public static function getLoader()
spl_autoload_unregister(array('ComposerAutoloaderInitContactsInteraction', 'loadClassLoader'));

require __DIR__ . '/autoload_static.php';
\Composer\Autoload\ComposerStaticInitContactsInteraction::getInitializer($loader)();
call_user_func(\Composer\Autoload\ComposerStaticInitContactsInteraction::getInitializer($loader));

$loader->setClassMapAuthoritative(true);
$loader->register(true);
Expand Down
5 changes: 5 additions & 0 deletions apps/dav/composer/autoload.php
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,11 @@

// autoload.php @generated by Composer

if (PHP_VERSION_ID < 50600) {
echo 'Composer 2.3.0 dropped support for autoloading on PHP <5.6 and you are running '.PHP_VERSION.', please upgrade PHP or use Composer 2.2 LTS via "composer self-update --2.2". Aborting.'.PHP_EOL;
exit(1);
}

require_once __DIR__ . '/composer/autoload_real.php';

return ComposerAutoloaderInitDAV::getLoader();
2 changes: 1 addition & 1 deletion apps/dav/composer/composer/autoload_real.php
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ public static function getLoader()
spl_autoload_unregister(array('ComposerAutoloaderInitDAV', 'loadClassLoader'));

require __DIR__ . '/autoload_static.php';
\Composer\Autoload\ComposerStaticInitDAV::getInitializer($loader)();
call_user_func(\Composer\Autoload\ComposerStaticInitDAV::getInitializer($loader));

$loader->setClassMapAuthoritative(true);
$loader->register(true);
Expand Down
4 changes: 2 additions & 2 deletions apps/dav/composer/composer/installed.php
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
'type' => 'library',
'install_path' => __DIR__ . '/../',
'aliases' => array(),
'reference' => 'e2c675724fc4ea50f1275bf0027b96f277c32578',
'reference' => '9586920c0ec4016864a2219e838fb272127822d8',
'name' => '__root__',
'dev' => false,
),
Expand All @@ -16,7 +16,7 @@
'type' => 'library',
'install_path' => __DIR__ . '/../',
'aliases' => array(),
'reference' => 'e2c675724fc4ea50f1275bf0027b96f277c32578',
'reference' => '9586920c0ec4016864a2219e838fb272127822d8',
'dev_requirement' => false,
),
),
Expand Down
5 changes: 5 additions & 0 deletions apps/dav/lib/Connector/Sabre/Directory.php
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@

use OC\Files\Mount\MoveableMount;
use OC\Files\View;
use OC\Metadata\FileMetadata;
use OC\Metadata\MetadataGroup;
use OCA\DAV\Connector\Sabre\Exception\FileLocked;
use OCA\DAV\Connector\Sabre\Exception\Forbidden;
use OCA\DAV\Connector\Sabre\Exception\InvalidPath;
Expand Down Expand Up @@ -73,6 +75,9 @@ class Directory extends \OCA\DAV\Connector\Sabre\Node implements \Sabre\DAV\ICol
*/
private $tree;

/** @var array<string, array<int, FileMetadata>> */
private array $metadata = [];

/**
* Sets up the node, expects a full path name
*
Expand Down
16 changes: 16 additions & 0 deletions apps/dav/lib/Connector/Sabre/File.php
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
use OC\Files\Filesystem;
use OC\Files\Stream\HashWrapper;
use OC\Files\View;
use OC\Metadata\FileMetadata;
use OCA\DAV\AppInfo\Application;
use OCA\DAV\Connector\Sabre\Exception\EntityTooLarge;
use OCA\DAV\Connector\Sabre\Exception\FileLocked;
Expand Down Expand Up @@ -80,6 +81,9 @@ class File extends Node implements IFile {

protected IL10N $l10n;

/** @var array<string, FileMetadata> */
private array $metadata = [];

/**
* Sets up the node, expects a full path name
*
Expand Down Expand Up @@ -757,4 +761,16 @@ public function hash(string $type) {
public function getNode(): \OCP\Files\File {
return $this->node;
}

public function getMetadata(string $group): FileMetadata {
return $this->metadata[$group];
}

public function setMetadata(string $group, FileMetadata $metadata): void {
$this->metadata[$group] = $metadata;
}

public function hasMetadata(string $group) {
return array_key_exists($group, $this->metadata);
}
}
53 changes: 53 additions & 0 deletions apps/dav/lib/Connector/Sabre/FilesPlugin.php
Original file line number Diff line number Diff line change
Expand Up @@ -34,13 +34,15 @@
namespace OCA\DAV\Connector\Sabre;

use OC\AppFramework\Http\Request;
use OC\Metadata\IMetadataManager;
use OCP\Constants;
use OCP\Files\ForbiddenException;
use OCP\Files\StorageNotAvailableException;
use OCP\IConfig;
use OCP\IPreview;
use OCP\IRequest;
use OCP\IUserSession;
use Psr\Log\LoggerInterface;
use Sabre\DAV\Exception\Forbidden;
use Sabre\DAV\Exception\NotFound;
use Sabre\DAV\IFile;
Expand All @@ -50,6 +52,7 @@
use Sabre\DAV\Tree;
use Sabre\HTTP\RequestInterface;
use Sabre\HTTP\ResponseInterface;
use Sabre\Uri;

class FilesPlugin extends ServerPlugin {

Expand Down Expand Up @@ -79,6 +82,7 @@ class FilesPlugin extends ServerPlugin {
public const SHARE_NOTE = '{http://nextcloud.org/ns}note';
public const SUBFOLDER_COUNT_PROPERTYNAME = '{http://nextcloud.org/ns}contained-folder-count';
public const SUBFILE_COUNT_PROPERTYNAME = '{http://nextcloud.org/ns}contained-file-count';
public const FILE_METADATA_SIZE = '{http://nextcloud.org/ns}file-metadata-size';

/**
* Reference to main server object
Expand Down Expand Up @@ -436,6 +440,29 @@ public function handleGetProperties(PropFind $propFind, \Sabre\DAV\INode $node)
$propFind->handle(self::UPLOAD_TIME_PROPERTYNAME, function () use ($node) {
return $node->getFileInfo()->getUploadTime();
});

if ($this->config->getSystemValueBool('enable_file_metadata', true)) {
$propFind->handle(self::FILE_METADATA_SIZE, function () use ($node) {
if (!str_starts_with($node->getFileInfo()->getMimetype(), 'image')) {
return json_encode((object)[]);
}

if ($node->hasMetadata('size')) {
$sizeMetadata = $node->getMetadata('size');
} else {
// This code path should not be called since we try to preload
// the metadata when loading the folder or the search results
// in one go
$metadataManager = \OC::$server->get(IMetadataManager::class);
$sizeMetadata = $metadataManager->fetchMetadataFor('size', [$node->getId()])[$node->getId()];

// TODO would be nice to display this in the profiler...
\OC::$server->get(LoggerInterface::class)->debug('Inefficient fetching of metadata');
}

return json_encode((object)$sizeMetadata->getMetadata());
});
}
}

if ($node instanceof Directory) {
Expand All @@ -448,6 +475,32 @@ public function handleGetProperties(PropFind $propFind, \Sabre\DAV\INode $node)
});

$requestProperties = $propFind->getRequestedProperties();

// TODO detect dynamically which metadata groups are requested and
// preload all of them and not just size
if ($this->config->getSystemValueBool('enable_file_metadata', true)
&& in_array(self::FILE_METADATA_SIZE, $requestProperties, true)) {
// Preloading of the metadata
$fileIds = [];
foreach ($node->getChildren() as $child) {
/** @var \OCP\Files\Node|Node $child */
if (str_starts_with($child->getFileInfo()->getMimeType(), 'image/')) {
/** @var File $child */
$fileIds[] = $child->getFileInfo()->getId();
}
}
/** @var IMetaDataManager $metadataManager */
$metadataManager = \OC::$server->get(IMetadataManager::class);
$preloadedMetadata = $metadataManager->fetchMetadataFor('size', $fileIds);
foreach ($node->getChildren() as $child) {
/** @var \OCP\Files\Node|Node $child */
if (str_starts_with($child->getFileInfo()->getMimeType(), 'image')) {
CarlSchwan marked this conversation as resolved.
Show resolved Hide resolved
/** @var File $child */
$child->setMetadata('size', $preloadedMetadata[$child->getFileInfo()->getId()]);
}
}
}

if (in_array(self::SUBFILE_COUNT_PROPERTYNAME, $requestProperties, true)
|| in_array(self::SUBFOLDER_COUNT_PROPERTYNAME, $requestProperties, true)) {
$nbFiles = 0;
Expand Down
Loading