Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add a metadata service to store file metadata #31839

Merged
merged 2 commits into from
Apr 14, 2022
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion 3rdparty
5 changes: 5 additions & 0 deletions apps/accessibility/composer/autoload.php
Original file line number Diff line number Diff line change
@@ -2,6 +2,11 @@

// autoload.php @generated by Composer

if (PHP_VERSION_ID < 50600) {
echo 'Composer 2.3.0 dropped support for autoloading on PHP <5.6 and you are running '.PHP_VERSION.', please upgrade PHP or use Composer 2.2 LTS via "composer self-update --2.2". Aborting.'.PHP_EOL;
exit(1);
}

require_once __DIR__ . '/composer/autoload_real.php';

return ComposerAutoloaderInitAccessibility::getLoader();
2 changes: 1 addition & 1 deletion apps/accessibility/composer/composer/autoload_real.php
Original file line number Diff line number Diff line change
@@ -27,7 +27,7 @@ public static function getLoader()
spl_autoload_unregister(array('ComposerAutoloaderInitAccessibility', 'loadClassLoader'));

require __DIR__ . '/autoload_static.php';
\Composer\Autoload\ComposerStaticInitAccessibility::getInitializer($loader)();
call_user_func(\Composer\Autoload\ComposerStaticInitAccessibility::getInitializer($loader));

$loader->setClassMapAuthoritative(true);
$loader->register(true);
5 changes: 5 additions & 0 deletions apps/admin_audit/composer/autoload.php
Original file line number Diff line number Diff line change
@@ -2,6 +2,11 @@

// autoload.php @generated by Composer

if (PHP_VERSION_ID < 50600) {
echo 'Composer 2.3.0 dropped support for autoloading on PHP <5.6 and you are running '.PHP_VERSION.', please upgrade PHP or use Composer 2.2 LTS via "composer self-update --2.2". Aborting.'.PHP_EOL;
exit(1);
}

require_once __DIR__ . '/composer/autoload_real.php';

return ComposerAutoloaderInitAdminAudit::getLoader();
2 changes: 1 addition & 1 deletion apps/admin_audit/composer/composer/autoload_real.php
Original file line number Diff line number Diff line change
@@ -27,7 +27,7 @@ public static function getLoader()
spl_autoload_unregister(array('ComposerAutoloaderInitAdminAudit', 'loadClassLoader'));

require __DIR__ . '/autoload_static.php';
\Composer\Autoload\ComposerStaticInitAdminAudit::getInitializer($loader)();
call_user_func(\Composer\Autoload\ComposerStaticInitAdminAudit::getInitializer($loader));

$loader->setClassMapAuthoritative(true);
$loader->register(true);
5 changes: 5 additions & 0 deletions apps/cloud_federation_api/composer/autoload.php
Original file line number Diff line number Diff line change
@@ -2,6 +2,11 @@

// autoload.php @generated by Composer

if (PHP_VERSION_ID < 50600) {
echo 'Composer 2.3.0 dropped support for autoloading on PHP <5.6 and you are running '.PHP_VERSION.', please upgrade PHP or use Composer 2.2 LTS via "composer self-update --2.2". Aborting.'.PHP_EOL;
exit(1);
}

require_once __DIR__ . '/composer/autoload_real.php';

return ComposerAutoloaderInitCloudFederationAPI::getLoader();
Original file line number Diff line number Diff line change
@@ -27,7 +27,7 @@ public static function getLoader()
spl_autoload_unregister(array('ComposerAutoloaderInitCloudFederationAPI', 'loadClassLoader'));

require __DIR__ . '/autoload_static.php';
\Composer\Autoload\ComposerStaticInitCloudFederationAPI::getInitializer($loader)();
call_user_func(\Composer\Autoload\ComposerStaticInitCloudFederationAPI::getInitializer($loader));

$loader->setClassMapAuthoritative(true);
$loader->register(true);
5 changes: 5 additions & 0 deletions apps/comments/composer/autoload.php
Original file line number Diff line number Diff line change
@@ -2,6 +2,11 @@

// autoload.php @generated by Composer

if (PHP_VERSION_ID < 50600) {
echo 'Composer 2.3.0 dropped support for autoloading on PHP <5.6 and you are running '.PHP_VERSION.', please upgrade PHP or use Composer 2.2 LTS via "composer self-update --2.2". Aborting.'.PHP_EOL;
exit(1);
}

require_once __DIR__ . '/composer/autoload_real.php';

return ComposerAutoloaderInitComments::getLoader();
2 changes: 1 addition & 1 deletion apps/comments/composer/composer/autoload_real.php
Original file line number Diff line number Diff line change
@@ -27,7 +27,7 @@ public static function getLoader()
spl_autoload_unregister(array('ComposerAutoloaderInitComments', 'loadClassLoader'));

require __DIR__ . '/autoload_static.php';
\Composer\Autoload\ComposerStaticInitComments::getInitializer($loader)();
call_user_func(\Composer\Autoload\ComposerStaticInitComments::getInitializer($loader));

$loader->setClassMapAuthoritative(true);
$loader->register(true);
5 changes: 5 additions & 0 deletions apps/contactsinteraction/composer/autoload.php
Original file line number Diff line number Diff line change
@@ -2,6 +2,11 @@

// autoload.php @generated by Composer

if (PHP_VERSION_ID < 50600) {
echo 'Composer 2.3.0 dropped support for autoloading on PHP <5.6 and you are running '.PHP_VERSION.', please upgrade PHP or use Composer 2.2 LTS via "composer self-update --2.2". Aborting.'.PHP_EOL;
exit(1);
}

require_once __DIR__ . '/composer/autoload_real.php';

return ComposerAutoloaderInitContactsInteraction::getLoader();
Original file line number Diff line number Diff line change
@@ -27,7 +27,7 @@ public static function getLoader()
spl_autoload_unregister(array('ComposerAutoloaderInitContactsInteraction', 'loadClassLoader'));

require __DIR__ . '/autoload_static.php';
\Composer\Autoload\ComposerStaticInitContactsInteraction::getInitializer($loader)();
call_user_func(\Composer\Autoload\ComposerStaticInitContactsInteraction::getInitializer($loader));

$loader->setClassMapAuthoritative(true);
$loader->register(true);
5 changes: 5 additions & 0 deletions apps/dav/composer/autoload.php
Original file line number Diff line number Diff line change
@@ -2,6 +2,11 @@

// autoload.php @generated by Composer

if (PHP_VERSION_ID < 50600) {
echo 'Composer 2.3.0 dropped support for autoloading on PHP <5.6 and you are running '.PHP_VERSION.', please upgrade PHP or use Composer 2.2 LTS via "composer self-update --2.2". Aborting.'.PHP_EOL;
exit(1);
}

require_once __DIR__ . '/composer/autoload_real.php';

return ComposerAutoloaderInitDAV::getLoader();
2 changes: 1 addition & 1 deletion apps/dav/composer/composer/autoload_real.php
Original file line number Diff line number Diff line change
@@ -27,7 +27,7 @@ public static function getLoader()
spl_autoload_unregister(array('ComposerAutoloaderInitDAV', 'loadClassLoader'));

require __DIR__ . '/autoload_static.php';
\Composer\Autoload\ComposerStaticInitDAV::getInitializer($loader)();
call_user_func(\Composer\Autoload\ComposerStaticInitDAV::getInitializer($loader));

$loader->setClassMapAuthoritative(true);
$loader->register(true);
4 changes: 2 additions & 2 deletions apps/dav/composer/composer/installed.php
Original file line number Diff line number Diff line change
@@ -5,7 +5,7 @@
'type' => 'library',
'install_path' => __DIR__ . '/../',
'aliases' => array(),
'reference' => 'e2c675724fc4ea50f1275bf0027b96f277c32578',
'reference' => '9586920c0ec4016864a2219e838fb272127822d8',
'name' => '__root__',
'dev' => false,
),
@@ -16,7 +16,7 @@
'type' => 'library',
'install_path' => __DIR__ . '/../',
'aliases' => array(),
'reference' => 'e2c675724fc4ea50f1275bf0027b96f277c32578',
'reference' => '9586920c0ec4016864a2219e838fb272127822d8',
'dev_requirement' => false,
),
),
5 changes: 5 additions & 0 deletions apps/dav/lib/Connector/Sabre/Directory.php
Original file line number Diff line number Diff line change
@@ -34,6 +34,8 @@

use OC\Files\Mount\MoveableMount;
use OC\Files\View;
use OC\Metadata\FileMetadata;
use OC\Metadata\MetadataGroup;
use OCA\DAV\Connector\Sabre\Exception\FileLocked;
use OCA\DAV\Connector\Sabre\Exception\Forbidden;
use OCA\DAV\Connector\Sabre\Exception\InvalidPath;
@@ -73,6 +75,9 @@ class Directory extends \OCA\DAV\Connector\Sabre\Node implements \Sabre\DAV\ICol
*/
private $tree;

/** @var array<string, array<int, FileMetadata>> */
private array $metadata = [];

/**
* Sets up the node, expects a full path name
*
16 changes: 16 additions & 0 deletions apps/dav/lib/Connector/Sabre/File.php
Original file line number Diff line number Diff line change
@@ -43,6 +43,7 @@
use OC\Files\Filesystem;
use OC\Files\Stream\HashWrapper;
use OC\Files\View;
use OC\Metadata\FileMetadata;
use OCA\DAV\AppInfo\Application;
use OCA\DAV\Connector\Sabre\Exception\EntityTooLarge;
use OCA\DAV\Connector\Sabre\Exception\FileLocked;
@@ -80,6 +81,9 @@ class File extends Node implements IFile {

protected IL10N $l10n;

/** @var array<string, FileMetadata> */
private array $metadata = [];

/**
* Sets up the node, expects a full path name
*
@@ -757,4 +761,16 @@ public function hash(string $type) {
public function getNode(): \OCP\Files\File {
return $this->node;
}

public function getMetadata(string $group): FileMetadata {
return $this->metadata[$group];
}

public function setMetadata(string $group, FileMetadata $metadata): void {
$this->metadata[$group] = $metadata;
}

public function hasMetadata(string $group) {
return array_key_exists($group, $this->metadata);
}
}
53 changes: 53 additions & 0 deletions apps/dav/lib/Connector/Sabre/FilesPlugin.php
Original file line number Diff line number Diff line change
@@ -34,13 +34,15 @@
namespace OCA\DAV\Connector\Sabre;

use OC\AppFramework\Http\Request;
use OC\Metadata\IMetadataManager;
use OCP\Constants;
use OCP\Files\ForbiddenException;
use OCP\Files\StorageNotAvailableException;
use OCP\IConfig;
use OCP\IPreview;
use OCP\IRequest;
use OCP\IUserSession;
use Psr\Log\LoggerInterface;
use Sabre\DAV\Exception\Forbidden;
use Sabre\DAV\Exception\NotFound;
use Sabre\DAV\IFile;
@@ -50,6 +52,7 @@
use Sabre\DAV\Tree;
use Sabre\HTTP\RequestInterface;
use Sabre\HTTP\ResponseInterface;
use Sabre\Uri;

class FilesPlugin extends ServerPlugin {

@@ -79,6 +82,7 @@ class FilesPlugin extends ServerPlugin {
public const SHARE_NOTE = '{http://nextcloud.org/ns}note';
public const SUBFOLDER_COUNT_PROPERTYNAME = '{http://nextcloud.org/ns}contained-folder-count';
public const SUBFILE_COUNT_PROPERTYNAME = '{http://nextcloud.org/ns}contained-file-count';
public const FILE_METADATA_SIZE = '{http://nextcloud.org/ns}file-metadata-size';

/**
* Reference to main server object
@@ -436,6 +440,29 @@ public function handleGetProperties(PropFind $propFind, \Sabre\DAV\INode $node)
$propFind->handle(self::UPLOAD_TIME_PROPERTYNAME, function () use ($node) {
return $node->getFileInfo()->getUploadTime();
});

if ($this->config->getSystemValueBool('enable_file_metadata', true)) {
$propFind->handle(self::FILE_METADATA_SIZE, function () use ($node) {
if (!str_starts_with($node->getFileInfo()->getMimetype(), 'image')) {
return json_encode((object)[]);
}

if ($node->hasMetadata('size')) {
$sizeMetadata = $node->getMetadata('size');
} else {
// This code path should not be called since we try to preload
// the metadata when loading the folder or the search results
// in one go
$metadataManager = \OC::$server->get(IMetadataManager::class);
$sizeMetadata = $metadataManager->fetchMetadataFor('size', [$node->getId()])[$node->getId()];

// TODO would be nice to display this in the profiler...
\OC::$server->get(LoggerInterface::class)->debug('Inefficient fetching of metadata');
}

return json_encode((object)$sizeMetadata->getMetadata());
});
}
}

if ($node instanceof Directory) {
@@ -448,6 +475,32 @@ public function handleGetProperties(PropFind $propFind, \Sabre\DAV\INode $node)
});

$requestProperties = $propFind->getRequestedProperties();

// TODO detect dynamically which metadata groups are requested and
// preload all of them and not just size
if ($this->config->getSystemValueBool('enable_file_metadata', true)
&& in_array(self::FILE_METADATA_SIZE, $requestProperties, true)) {
// Preloading of the metadata
$fileIds = [];
foreach ($node->getChildren() as $child) {
/** @var \OCP\Files\Node|Node $child */
if (str_starts_with($child->getFileInfo()->getMimeType(), 'image/')) {
/** @var File $child */
$fileIds[] = $child->getFileInfo()->getId();
}
}
/** @var IMetaDataManager $metadataManager */
$metadataManager = \OC::$server->get(IMetadataManager::class);
$preloadedMetadata = $metadataManager->fetchMetadataFor('size', $fileIds);
foreach ($node->getChildren() as $child) {
/** @var \OCP\Files\Node|Node $child */
if (str_starts_with($child->getFileInfo()->getMimeType(), 'image')) {
/** @var File $child */
$child->setMetadata('size', $preloadedMetadata[$child->getFileInfo()->getId()]);
}
}
}

if (in_array(self::SUBFILE_COUNT_PROPERTYNAME, $requestProperties, true)
|| in_array(self::SUBFOLDER_COUNT_PROPERTYNAME, $requestProperties, true)) {
$nbFiles = 0;
Loading