Skip to content

Commit 3b9493e

Browse files
CarlSchwanCarlSchwan
committed
feat(Db): Use SnowflakeId for previews
Allow to get an id for the storing the preview on disk before inserting the preview on the DB. Signed-off-by: Carl Schwan <carl.schwan@nextcloud.com>
1 parent e5342b7 commit 3b9493e

File tree

17 files changed

+211
-47
lines changed

17 files changed

+211
-47
lines changed

‎core/Migrations/Version33000Date20251023110529.php‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-or-later
8+
*/
9+
namespace OC\Core\Migrations;
10+
11+
use Closure;
12+
use OCP\DB\ISchemaWrapper;
13+
use OCP\Migration\Attributes\ModifyColumn;
14+
use OCP\Migration\IOutput;
15+
use OCP\Migration\SimpleMigrationStep;
16+
17+
/**
18+
* Migrate away from auto-increment
19+
*/
20+
#[ModifyColumn(table: 'preview_locations', name: 'id', description: 'Remove auto-increment')]
21+
#[ModifyColumn(table: 'previews', name: 'id', description: 'Remove auto-increment')]
22+
#[ModifyColumn(table: 'preview_versions', name: 'id', description: 'Remove auto-increment')]
23+
class Version33000Date20251023110529 extends SimpleMigrationStep {
24+
/**
25+
* @param Closure(): ISchemaWrapper $schemaClosure The `\Closure` returns a `ISchemaWrapper`
26+
*/
27+
public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
28+
$schema = $schemaClosure();
29+
30+
if ($schema->hasTable('preview_locations')) {
31+
$schema->dropAutoincrement('preview_locations', 'id');
32+
}
33+
34+
if ($schema->hasTable('preview_versions')) {
35+
$schema->dropAutoincrement('preview_versions', 'id');
36+
}
37+
38+
if ($schema->hasTable('previews')) {
39+
$schema->dropAutoincrement('previews', 'id');
40+
}
41+
42+
return $schema;
43+
}
44+
}

‎core/Migrations/Version33000Date20251023120529.php‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-or-later
8+
*/
9+
namespace OC\Core\Migrations;
10+
11+
use Closure;
12+
use OCP\DB\ISchemaWrapper;
13+
use OCP\IDBConnection;
14+
use OCP\Migration\Attributes\AddIndex;
15+
use OCP\Migration\Attributes\IndexType;
16+
use OCP\Migration\IOutput;
17+
use OCP\Migration\SimpleMigrationStep;
18+
19+
/**
20+
* Use unique index for preview_locations
21+
*/
22+
#[AddIndex(table: 'preview_locations', type: IndexType::UNIQUE)]
23+
class Version33000Date20251023120529 extends SimpleMigrationStep {
24+
public function __construct(
25+
private readonly IDBConnection $connection,
26+
) {
27+
}
28+
29+
/**
30+
* @param Closure(): ISchemaWrapper $schemaClosure The `\Closure` returns a `ISchemaWrapper`
31+
*/
32+
public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
33+
/** @var ISchemaWrapper $schema */
34+
$schema = $schemaClosure();
35+
36+
if ($schema->hasTable('preview_locations')) {
37+
$table = $schema->getTable('preview_locations');
38+
$table->addUniqueIndex(['bucket_name', 'object_store_name'], 'unique_bucket_store');
39+
}
40+
41+
return $schema;
42+
}
43+
44+
public function preSchemaChange(IOutput $output, \Closure $schemaClosure, array $options) {
45+
// This code should never be run on a production instance but this might be needed on daily/git version.
46+
$qb = $this->connection->getQueryBuilder();
47+
$qb->select('*')
48+
->from('preview_locations')
49+
->groupBy('bucket_name', 'object_store_name')
50+
->having('COUNT(*) > 1');
51+
52+
$result = $qb->executeQuery();
53+
while ($row = $result->fetch()) {
54+
// Iterate over all the rows with duplicated rows
55+
$id = $row['id'];
56+
57+
$qb = $this->connection->getQueryBuilder();
58+
$qb->select('id')
59+
->from('preview_locations')
60+
->where($qb->expr()->eq('bucket_name', $qb->createNamedParameter($row['bucket_name'])))
61+
->andWhere($qb->expr()->eq('object_store_name', $qb->createNamedParameter($row['object_store_name'])))
62+
->andWhere($qb->expr()->neq('id', $qb->createNamedParameter($row['id'])));
63+
64+
$result = $qb->executeQuery();
65+
while ($row = $result->fetch()) {
66+
// Update previews entries to the now de-duplicated id
67+
$qb = $this->connection->getQueryBuilder();
68+
$qb->update('previews')
69+
->set('location_id', $qb->createNamedParameter($id))
70+
->where($qb->expr()->eq('id', $qb->createNamedParameter($row['id'])));
71+
$qb->executeStatement();
72+
73+
$qb = $this->connection->getQueryBuilder();
74+
$qb->delete('preview_locations')
75+
->where($qb->expr()->eq('id', $qb->createNamedParameter($row['id'])));
76+
$qb->executeStatement();
77+
}
78+
}
79+
}
80+
}

‎lib/composer/composer/autoload_classmap.php‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1530,6 +1530,7 @@
15301530
'OC\\Core\\Migrations\\Version32000Date20250731062008' => $baseDir . '/core/Migrations/Version32000Date20250731062008.php',
15311531
'OC\\Core\\Migrations\\Version32000Date20250806110519' => $baseDir . '/core/Migrations/Version32000Date20250806110519.php',
15321532
'OC\\Core\\Migrations\\Version33000Date20250819110529' => $baseDir . '/core/Migrations/Version33000Date20250819110529.php',
1533+
'OC\\Core\\Migrations\\Version33000Date20251023110529' => $baseDir . '/core/Migrations/Version33000Date20251023110529.php',
15331534
'OC\\Core\\Notification\\CoreNotifier' => $baseDir . '/core/Notification/CoreNotifier.php',
15341535
'OC\\Core\\ResponseDefinitions' => $baseDir . '/core/ResponseDefinitions.php',
15351536
'OC\\Core\\Service\\CronService' => $baseDir . '/core/Service/CronService.php',

‎lib/composer/composer/autoload_static.php‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1571,6 +1571,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
15711571
'OC\\Core\\Migrations\\Version32000Date20250731062008' => __DIR__ . '/../../..' . '/core/Migrations/Version32000Date20250731062008.php',
15721572
'OC\\Core\\Migrations\\Version32000Date20250806110519' => __DIR__ . '/../../..' . '/core/Migrations/Version32000Date20250806110519.php',
15731573
'OC\\Core\\Migrations\\Version33000Date20250819110529' => __DIR__ . '/../../..' . '/core/Migrations/Version33000Date20250819110529.php',
1574+
'OC\\Core\\Migrations\\Version33000Date20251023110529' => __DIR__ . '/../../..' . '/core/Migrations/Version33000Date20251023110529.php',
15741575
'OC\\Core\\Notification\\CoreNotifier' => __DIR__ . '/../../..' . '/core/Notification/CoreNotifier.php',
15751576
'OC\\Core\\ResponseDefinitions' => __DIR__ . '/../../..' . '/core/ResponseDefinitions.php',
15761577
'OC\\Core\\Service\\CronService' => __DIR__ . '/../../..' . '/core/Service/CronService.php',

‎lib/private/AppFramework/Http/Dispatcher.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -204,7 +204,7 @@ private function executeController(Controller $controller, string $methodName):
204204
try {
205205
$response = \call_user_func_array([$controller, $methodName], $arguments);
206206
} catch (\TypeError $e) {
207-
// Only intercept TypeErrors occuring on the first line, meaning that the invocation of the controller method failed.
207+
// Only intercept TypeErrors occurring on the first line, meaning that the invocation of the controller method failed.
208208
// Any other TypeError happens inside the controller method logic and should be logged as normal.
209209
if ($e->getFile() === $this->reflector->getFile() && $e->getLine() === $this->reflector->getStartLine()) {
210210
$this->logger->debug('Failed to call controller method: ' . $e->getMessage(), ['exception' => $e]);

‎lib/private/DB/SchemaWrapper.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@
99

1010
namespace OC\DB;
1111

12-
use Doctrine\DBAL\Exception;
1312
use Doctrine\DBAL\Platforms\AbstractPlatform;
13+
use Doctrine\DBAL\Platforms\OraclePlatform;
1414
use Doctrine\DBAL\Schema\Schema;
1515
use Doctrine\DBAL\Schema\Table;
1616
use OCP\DB\ISchemaWrapper;

‎lib/private/Preview/Db/Preview.php‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,14 +17,16 @@
1717
/**
1818
* Preview entity mapped to the oc_previews and oc_preview_locations table.
1919
*
20+
* @method string getId()
21+
* @method void setId(string $id)
2022
* @method int getFileId() Get the file id of the original file.
2123
* @method void setFileId(int $fileId)
2224
* @method int getStorageId() Get the storage id of the original file.
2325
* @method void setStorageId(int $fileId)
2426
* @method int getOldFileId() Get the old location in the file-cache table, for legacy compatibility.
2527
* @method void setOldFileId(int $oldFileId)
26-
* @method int getLocationId() Get the location id in the preview_locations table. Only set when using an object store as primary storage.
27-
* @method void setLocationId(int $locationId)
28+
* @method string getLocationId() Get the location id in the preview_locations table. Only set when using an object store as primary storage.
29+
* @method void setLocationId(string $locationId)
2830
* @method string|null getBucketName() Get the bucket name where the preview is stored. This is stored in the preview_locations table.
2931
* @method string|null getObjectStoreName() Get the object store name where the preview is stored. This is stored in the preview_locations table.
3032
* @method int getWidth() Get the width of the preview.
@@ -46,7 +48,7 @@
4648
* @method string getEtag() Get the etag of the preview.
4749
* @method void setEtag(string $etag)
4850
* @method string|null getVersion() Get the version for files_versions_s3
49-
* @method void setVersionId(int $versionId)
51+
* @method void setVersionId(string $versionId)
5052
* @method bool|null getIs() Get the version for files_versions_s3
5153
* @method bool isEncrypted() Get whether the preview is encrypted. At the moment every preview is unencrypted.
5254
* @method void setEncrypted(bool $encrypted)
@@ -57,7 +59,7 @@ class Preview extends Entity {
5759
protected ?int $fileId = null;
5860
protected ?int $oldFileId = null;
5961
protected ?int $storageId = null;
60-
protected ?int $locationId = null;
62+
protected ?string $locationId = null;
6163
protected ?string $bucketName = null;
6264
protected ?string $objectStoreName = null;
6365
protected ?int $width = null;
@@ -72,14 +74,15 @@ class Preview extends Entity {
7274
protected ?bool $cropped = null;
7375
protected ?string $etag = null;
7476
protected ?string $version = null;
75-
protected ?int $versionId = null;
77+
protected ?string $versionId = null;
7678
protected ?bool $encrypted = null;
7779

7880
public function __construct() {
81+
$this->addType('id', Types::STRING);
7982
$this->addType('fileId', Types::BIGINT);
8083
$this->addType('storageId', Types::BIGINT);
8184
$this->addType('oldFileId', Types::BIGINT);
82-
$this->addType('locationId', Types::BIGINT);
85+
$this->addType('locationId', Types::STRING);
8386
$this->addType('width', Types::INTEGER);
8487
$this->addType('height', Types::INTEGER);
8588
$this->addType('mimetypeId', Types::INTEGER);

‎lib/private/Preview/Db/PreviewMapper.php‎

Lines changed: 38 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
use OCP\DB\QueryBuilder\IQueryBuilder;
1616
use OCP\Files\IMimeTypeLoader;
1717
use OCP\IDBConnection;
18+
use OCP\Snowflake\IGenerator;
1819
use Override;
1920

2021
/**
@@ -29,6 +30,7 @@ class PreviewMapper extends QBMapper {
2930
public function __construct(
3031
IDBConnection $db,
3132
private readonly IMimeTypeLoader $mimeTypeLoader,
33+
private readonly IGenerator $snowflake,
3234
) {
3335
parent::__construct($db, self::TABLE_NAME, Preview::class);
3436
}
@@ -50,13 +52,15 @@ public function insert(Entity $entity): Entity {
5052

5153
if ($preview->getVersion() !== null && $preview->getVersion() !== '') {
5254
$qb = $this->db->getQueryBuilder();
55+
$id = $this->snowflake->nextId();
5356
$qb->insert(self::VERSION_TABLE_NAME)
5457
->values([
58+
'id' => $id,
5559
'version' => $preview->getVersion(),
5660
'file_id' => $preview->getFileId(),
5761
])
5862
->executeStatement();
59-
$entity->setVersionId($qb->getLastInsertId());
63+
$entity->setVersionId($id);
6064
}
6165
return parent::insert($preview);
6266
}
@@ -148,7 +152,13 @@ protected function joinLocation(IQueryBuilder $qb): IQueryBuilder {
148152
));
149153
}
150154

151-
public function getLocationId(string $bucket, string $objectStore): int {
155+
/**
156+
* Get the location id corresponding to the $bucket and $objectStore. Create one
157+
* if not existing yet.
158+
*
159+
* @throws Exception
160+
*/
161+
public function getLocationId(string $bucket, string $objectStore): string {
152162
$qb = $this->db->getQueryBuilder();
153163
$result = $qb->select('id')
154164
->from(self::LOCATION_TABLE_NAME)
@@ -157,14 +167,33 @@ public function getLocationId(string $bucket, string $objectStore): int {
157167
->executeQuery();
158168
$data = $result->fetchOne();
159169
if ($data) {
160-
return $data;
170+
return (string)$data;
161171
} else {
162-
$qb->insert(self::LOCATION_TABLE_NAME)
163-
->values([
164-
'bucket_name' => $qb->createNamedParameter($bucket),
165-
'object_store_name' => $qb->createNamedParameter($objectStore),
166-
])->executeStatement();
167-
return $qb->getLastInsertId();
172+
try {
173+
$id = $this->snowflake->nextId();
174+
$qb->insert(self::LOCATION_TABLE_NAME)
175+
->values([
176+
'id' => $qb->createNamedParameter($id),
177+
'bucket_name' => $qb->createNamedParameter($bucket),
178+
'object_store_name' => $qb->createNamedParameter($objectStore),
179+
])->executeStatement();
180+
return $id;
181+
} catch (Exception $e) {
182+
if ($e->getReason() === Exception::REASON_UNIQUE_CONSTRAINT_VIOLATION) {
183+
// Fetch again as there seems to be another entry added meanwhile
184+
$result = $qb->select('id')
185+
->from(self::LOCATION_TABLE_NAME)
186+
->where($qb->expr()->eq('bucket_name', $qb->createNamedParameter($bucket)))
187+
->andWhere($qb->expr()->eq('object_store_name', $qb->createNamedParameter($objectStore)))
188+
->executeQuery();
189+
$data = $result->fetchOne();
190+
if ($data) {
191+
return (string)$data;
192+
}
193+
}
194+
195+
throw $e;
196+
}
168197
}
169198
}
170199

‎lib/private/Preview/Generator.php‎

Lines changed: 16 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
use OCP\IStreamImage;
2424
use OCP\Preview\BeforePreviewFetchedEvent;
2525
use OCP\Preview\IVersionedPreviewFile;
26+
use OCP\Snowflake\IGenerator;
2627
use Psr\Log\LoggerInterface;
2728

2829
class Generator {
@@ -37,6 +38,7 @@ public function __construct(
3738
private LoggerInterface $logger,
3839
private PreviewMapper $previewMapper,
3940
private StorageFactory $storageFactory,
41+
private IGenerator $snowflakeGenerator,
4042
) {
4143
}
4244

@@ -348,6 +350,7 @@ private function generateProviderPreview(File $file, int $width, int $height, bo
348350

349351
try {
350352
$previewEntry = new Preview();
353+
$previewEntry->setId($this->snowflakeGenerator->nextId());
351354
$previewEntry->setFileId($file->getId());
352355
$previewEntry->setStorageId($file->getMountPoint()->getNumericStorageId());
353356
$previewEntry->setSourceMimeType($file->getMimeType());
@@ -360,7 +363,6 @@ private function generateProviderPreview(File $file, int $width, int $height, bo
360363
$previewEntry->setMimetype($preview->dataMimeType());
361364
$previewEntry->setEtag($file->getEtag());
362365
$previewEntry->setMtime((new \DateTime())->getTimestamp());
363-
$previewEntry->setSize(0);
364366
return $this->savePreview($previewEntry, $preview);
365367
} catch (NotPermittedException) {
366368
throw new NotFoundException();
@@ -502,6 +504,7 @@ private function generatePreview(
502504
}
503505

504506
$previewEntry = new Preview();
507+
$previewEntry->setId($this->snowflakeGenerator->nextId());
505508
$previewEntry->setFileId($file->getId());
506509
$previewEntry->setStorageId($file->getMountPoint()->getNumericStorageId());
507510
$previewEntry->setWidth($width);
@@ -514,7 +517,6 @@ private function generatePreview(
514517
$previewEntry->setMimeType($preview->dataMimeType());
515518
$previewEntry->setEtag($file->getEtag());
516519
$previewEntry->setMtime((new \DateTime())->getTimestamp());
517-
$previewEntry->setSize(0);
518520
if ($cacheResult) {
519521
$previewEntry = $this->savePreview($previewEntry, $preview);
520522
return new PreviewFile($previewEntry, $this->storageFactory, $this->previewMapper);
@@ -530,26 +532,20 @@ private function generatePreview(
530532
* @throws \OCP\DB\Exception
531533
*/
532534
public function savePreview(Preview $previewEntry, IImage $preview): Preview {
533-
$previewEntry = $this->previewMapper->insert($previewEntry);
534-
535535
// we need to save to DB first
536-
try {
537-
if ($preview instanceof IStreamImage) {
538-
$size = $this->storageFactory->writePreview($previewEntry, $preview->resource());
539-
} else {
540-
$stream = fopen('php://temp', 'w+');
541-
fwrite($stream, $preview->data());
542-
rewind($stream);
543-
$size = $this->storageFactory->writePreview($previewEntry, $stream);
544-
}
545-
if (!$size) {
546-
throw new \RuntimeException('Unable to write preview file');
547-
}
548-
} catch (\Exception $e) {
549-
$this->previewMapper->delete($previewEntry);
550-
throw $e;
536+
if ($preview instanceof IStreamImage) {
537+
$size = $this->storageFactory->writePreview($previewEntry, $preview->resource());
538+
} else {
539+
$stream = fopen('php://temp', 'w+');
540+
fwrite($stream, $preview->data());
541+
rewind($stream);
542+
$size = $this->storageFactory->writePreview($previewEntry, $stream);
543+
}
544+
if (!$size) {
545+
throw new \RuntimeException('Unable to write preview file');
551546
}
552547
$previewEntry->setSize($size);
553-
return $this->previewMapper->update($previewEntry);
548+
549+
return $this->previewMapper->insert($previewEntry);
554550
}
555551
}

0 commit comments

Comments
 (0)