Skip to content

Commit 52a9940

Browse files
skjnldsvskjnldsv
authored
Merge pull request #55728 from nextcloud/feat/snowflakeIds
2 parents 445cce4 + 336cc3f commit 52a9940

32 files changed

+830
-52
lines changed

‎.github/workflows/phpunit-32bits.yml‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,10 @@ name: PHPUnit 32bits
55
on:
66
pull_request:
77
paths:
8-
- 'version.php'
9-
- '.github/workflows/phpunit-32bits.yml'
10-
- 'tests/phpunit-autotest.xml'
8+
- "version.php"
9+
- ".github/workflows/phpunit-32bits.yml"
10+
- "tests/phpunit-autotest.xml"
11+
- "lib/private/Snowflake/*"
1112
workflow_dispatch:
1213
schedule:
1314
- cron: "15 1 * * 1-6"
@@ -30,7 +31,7 @@ jobs:
3031
strategy:
3132
fail-fast: false
3233
matrix:
33-
php-versions: ['8.2', '8.3', '8.4']
34+
php-versions: ["8.2", "8.3", "8.4"]
3435

3536
steps:
3637
- name: Checkout server
@@ -51,8 +52,7 @@ jobs:
5152
extensions: bz2, ctype, curl, dom, fileinfo, gd, iconv, imagick, intl, json, libxml, mbstring, openssl, pcntl, posix, redis, session, simplexml, xmlreader, xmlwriter, zip, zlib, sqlite, pdo_sqlite, apcu, ldap
5253
coverage: none
5354
ini-file: development
54-
ini-values:
55-
apc.enabled=on, apc.enable_cli=on, disable_functions= # https://github.com/shivammathur/setup-php/discussions/573
55+
ini-values: apc.enabled=on, apc.enable_cli=on, disable_functions= # https://github.com/shivammathur/setup-php/discussions/573
5656
env:
5757
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
5858

‎apps/settings/lib/SetupChecks/PhpModules.php‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ class PhpModules implements ISetupCheck {
3131
'zlib',
3232
];
3333
protected const RECOMMENDED_MODULES = [
34+
'apcu',
3435
'exif',
3536
'gmp',
3637
'intl',

‎composer.json‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,9 @@
1111
}
1212
},
1313
"autoload": {
14-
"exclude-from-classmap": ["**/bamarni/composer-bin-plugin/**"],
14+
"exclude-from-classmap": [
15+
"**/bamarni/composer-bin-plugin/**"
16+
],
1517
"files": [
1618
"lib/public/Log/functions.php"
1719
],
@@ -25,6 +27,7 @@
2527
},
2628
"require": {
2729
"php": "^8.2",
30+
"ext-apcu": "*",
2831
"ext-ctype": "*",
2932
"ext-curl": "*",
3033
"ext-dom": "*",

‎core/BackgroundJobs/MovePreviewJob.php‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
use OCP\IAppConfig;
2727
use OCP\IConfig;
2828
use OCP\IDBConnection;
29+
use OCP\Snowflake\IGenerator;
2930
use Override;
3031
use Psr\Log\LoggerInterface;
3132

@@ -44,6 +45,7 @@ public function __construct(
4445
private readonly IMimeTypeDetector $mimeTypeDetector,
4546
private readonly IMimeTypeLoader $mimeTypeLoader,
4647
private readonly LoggerInterface $logger,
48+
private readonly IGenerator $generator,
4749
IAppDataFactory $appDataFactory,
4850
) {
4951
parent::__construct($time);
@@ -136,6 +138,7 @@ private function processPreviews(int $fileId, bool $flatPath): void {
136138
$path = $fileId . '/' . $previewFile->getName();
137139
/** @var SimpleFile $previewFile */
138140
$preview = Preview::fromPath($path, $this->mimeTypeDetector);
141+
$preview->setId($this->generator->nextId());
139142
if (!$preview) {
140143
$this->logger->error('Unable to import old preview at path.');
141144
continue;

‎core/Command/SnowflakeDecodeId.php‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-only
8+
*/
9+
namespace OC\Core\Command;
10+
11+
use OC\Snowflake\Decoder;
12+
use Symfony\Component\Console\Helper\Table;
13+
use Symfony\Component\Console\Input\InputArgument;
14+
use Symfony\Component\Console\Input\InputInterface;
15+
use Symfony\Component\Console\Output\OutputInterface;
16+
17+
class SnowflakeDecodeId extends Base {
18+
protected function configure(): void {
19+
parent::configure();
20+
21+
$this
22+
->setName('decode-snowflake')
23+
->setDescription('Decode Snowflake IDs used by Nextcloud')
24+
->addArgument('snowflake-id', InputArgument::REQUIRED, 'Nextcloud Snowflake ID to decode');
25+
}
26+
27+
protected function execute(InputInterface $input, OutputInterface $output): int {
28+
$snowflakeId = $input->getArgument('snowflake-id');
29+
$data = (new Decoder)->decode($snowflakeId);
30+
31+
$rows = [
32+
['Snowflake ID', $snowflakeId],
33+
['Seconds', $data['seconds']],
34+
['Milliseconds', $data['milliseconds']],
35+
['Created from CLI', $data['isCli'] ? 'yes' : 'no'],
36+
['Server ID', $data['serverId']],
37+
['Sequence ID', $data['sequenceId']],
38+
['Creation timestamp', $data['createdAt']->format('U.v')],
39+
['Creation date', $data['createdAt']->format('Y-m-d H:i:s.v')],
40+
];
41+
42+
$table = new Table($output);
43+
$table->setRows($rows);
44+
$table->render();
45+
46+
return Base::SUCCESS;
47+
}
48+
}

‎core/Migrations/Version33000Date20251023110529.php‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-or-later
8+
*/
9+
namespace OC\Core\Migrations;
10+
11+
use Closure;
12+
use OCP\DB\ISchemaWrapper;
13+
use OCP\Migration\Attributes\ModifyColumn;
14+
use OCP\Migration\IOutput;
15+
use OCP\Migration\SimpleMigrationStep;
16+
17+
/**
18+
* Migrate away from auto-increment
19+
*/
20+
#[ModifyColumn(table: 'preview_locations', name: 'id', description: 'Remove auto-increment')]
21+
#[ModifyColumn(table: 'previews', name: 'id', description: 'Remove auto-increment')]
22+
#[ModifyColumn(table: 'preview_versions', name: 'id', description: 'Remove auto-increment')]
23+
class Version33000Date20251023110529 extends SimpleMigrationStep {
24+
/**
25+
* @param Closure(): ISchemaWrapper $schemaClosure The `\Closure` returns a `ISchemaWrapper`
26+
*/
27+
public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
28+
$schema = $schemaClosure();
29+
30+
if ($schema->hasTable('preview_locations')) {
31+
$schema->dropAutoincrementColumn('preview_locations', 'id');
32+
}
33+
34+
if ($schema->hasTable('preview_versions')) {
35+
$schema->dropAutoincrementColumn('preview_versions', 'id');
36+
}
37+
38+
if ($schema->hasTable('previews')) {
39+
$schema->dropAutoincrementColumn('previews', 'id');
40+
}
41+
42+
return $schema;
43+
}
44+
}

‎core/Migrations/Version33000Date20251023120529.php‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-or-later
8+
*/
9+
namespace OC\Core\Migrations;
10+
11+
use Closure;
12+
use OCP\DB\ISchemaWrapper;
13+
use OCP\IDBConnection;
14+
use OCP\Migration\Attributes\AddIndex;
15+
use OCP\Migration\Attributes\IndexType;
16+
use OCP\Migration\IOutput;
17+
use OCP\Migration\SimpleMigrationStep;
18+
19+
/**
20+
* Use unique index for preview_locations
21+
*/
22+
#[AddIndex(table: 'preview_locations', type: IndexType::UNIQUE)]
23+
class Version33000Date20251023120529 extends SimpleMigrationStep {
24+
public function __construct(
25+
private readonly IDBConnection $connection,
26+
) {
27+
}
28+
29+
/**
30+
* @param Closure(): ISchemaWrapper $schemaClosure The `\Closure` returns a `ISchemaWrapper`
31+
*/
32+
public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
33+
/** @var ISchemaWrapper $schema */
34+
$schema = $schemaClosure();
35+
36+
if ($schema->hasTable('preview_locations')) {
37+
$table = $schema->getTable('preview_locations');
38+
$table->addUniqueIndex(['bucket_name', 'object_store_name'], 'unique_bucket_store');
39+
}
40+
41+
return $schema;
42+
}
43+
44+
public function preSchemaChange(IOutput $output, \Closure $schemaClosure, array $options) {
45+
// This shouldn't run on a production instance, only daily
46+
$qb = $this->connection->getQueryBuilder();
47+
$qb->select('*')
48+
->from('preview_locations');
49+
$result = $qb->executeQuery();
50+
51+
$set = [];
52+
53+
while ($row = $result->fetch()) {
54+
// Iterate over all the rows with duplicated rows
55+
$id = $row['id'];
56+
57+
if (isset($set[$row['bucket_name'] . '_' . $row['object_store_name']])) {
58+
// duplicate
59+
$authoritativeId = $set[$row['bucket_name'] . '_' . $row['object_store_name']];
60+
$qb = $this->connection->getQueryBuilder();
61+
$qb->select('id')
62+
->from('preview_locations')
63+
->where($qb->expr()->eq('bucket_name', $qb->createNamedParameter($row['bucket_name'])))
64+
->andWhere($qb->expr()->eq('object_store_name', $qb->createNamedParameter($row['object_store_name'])))
65+
->andWhere($qb->expr()->neq('id', $qb->createNamedParameter($authoritativeId)));
66+
67+
$result = $qb->executeQuery();
68+
while ($row = $result->fetch()) {
69+
// Update previews entries to the now de-duplicated id
70+
$qb = $this->connection->getQueryBuilder();
71+
$qb->update('previews')
72+
->set('location_id', $qb->createNamedParameter($id))
73+
->where($qb->expr()->eq('id', $qb->createNamedParameter($row['id'])));
74+
$qb->executeStatement();
75+
76+
$qb = $this->connection->getQueryBuilder();
77+
$qb->delete('preview_locations')
78+
->where($qb->expr()->eq('id', $qb->createNamedParameter($row['id'])));
79+
$qb->executeStatement();
80+
}
81+
break;
82+
}
83+
$set[$row['bucket_name'] . '_' . $row['object_store_name']] = $row['id'];
84+
}
85+
}
86+
}

‎core/register_command.php‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@
8585
use OC\Core\Command\Security\ListCertificates;
8686
use OC\Core\Command\Security\RemoveCertificate;
8787
use OC\Core\Command\SetupChecks;
88+
use OC\Core\Command\SnowflakeDecodeId;
8889
use OC\Core\Command\Status;
8990
use OC\Core\Command\SystemTag\Edit;
9091
use OC\Core\Command\TaskProcessing\EnabledCommand;
@@ -246,6 +247,7 @@
246247
$application->add(Server::get(BruteforceAttempts::class));
247248
$application->add(Server::get(BruteforceResetAttempts::class));
248249
$application->add(Server::get(SetupChecks::class));
250+
$application->add(Server::get(SnowflakeDecodeId::class));
249251
$application->add(Server::get(Get::class));
250252

251253
$application->add(Server::get(GetCommand::class));

‎lib/composer/composer/autoload_classmap.php‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -822,6 +822,8 @@
822822
'OCP\\Share_Backend' => $baseDir . '/lib/public/Share_Backend.php',
823823
'OCP\\Share_Backend_Collection' => $baseDir . '/lib/public/Share_Backend_Collection.php',
824824
'OCP\\Share_Backend_File_Dependent' => $baseDir . '/lib/public/Share_Backend_File_Dependent.php',
825+
'OCP\\Snowflake\\IDecoder' => $baseDir . '/lib/public/Snowflake/IDecoder.php',
826+
'OCP\\Snowflake\\IGenerator' => $baseDir . '/lib/public/Snowflake/IGenerator.php',
825827
'OCP\\SpeechToText\\Events\\AbstractTranscriptionEvent' => $baseDir . '/lib/public/SpeechToText/Events/AbstractTranscriptionEvent.php',
826828
'OCP\\SpeechToText\\Events\\TranscriptionFailedEvent' => $baseDir . '/lib/public/SpeechToText/Events/TranscriptionFailedEvent.php',
827829
'OCP\\SpeechToText\\Events\\TranscriptionSuccessfulEvent' => $baseDir . '/lib/public/SpeechToText/Events/TranscriptionSuccessfulEvent.php',
@@ -1352,6 +1354,7 @@
13521354
'OC\\Core\\Command\\Security\\ListCertificates' => $baseDir . '/core/Command/Security/ListCertificates.php',
13531355
'OC\\Core\\Command\\Security\\RemoveCertificate' => $baseDir . '/core/Command/Security/RemoveCertificate.php',
13541356
'OC\\Core\\Command\\SetupChecks' => $baseDir . '/core/Command/SetupChecks.php',
1357+
'OC\\Core\\Command\\SnowflakeDecodeId' => $baseDir . '/core/Command/SnowflakeDecodeId.php',
13551358
'OC\\Core\\Command\\Status' => $baseDir . '/core/Command/Status.php',
13561359
'OC\\Core\\Command\\SystemTag\\Add' => $baseDir . '/core/Command/SystemTag/Add.php',
13571360
'OC\\Core\\Command\\SystemTag\\Delete' => $baseDir . '/core/Command/SystemTag/Delete.php',
@@ -1527,6 +1530,8 @@
15271530
'OC\\Core\\Migrations\\Version32000Date20250731062008' => $baseDir . '/core/Migrations/Version32000Date20250731062008.php',
15281531
'OC\\Core\\Migrations\\Version32000Date20250806110519' => $baseDir . '/core/Migrations/Version32000Date20250806110519.php',
15291532
'OC\\Core\\Migrations\\Version33000Date20250819110529' => $baseDir . '/core/Migrations/Version33000Date20250819110529.php',
1533+
'OC\\Core\\Migrations\\Version33000Date20251023110529' => $baseDir . '/core/Migrations/Version33000Date20251023110529.php',
1534+
'OC\\Core\\Migrations\\Version33000Date20251023120529' => $baseDir . '/core/Migrations/Version33000Date20251023120529.php',
15301535
'OC\\Core\\Notification\\CoreNotifier' => $baseDir . '/core/Notification/CoreNotifier.php',
15311536
'OC\\Core\\ResponseDefinitions' => $baseDir . '/core/ResponseDefinitions.php',
15321537
'OC\\Core\\Service\\CronService' => $baseDir . '/core/Service/CronService.php',
@@ -2103,6 +2108,8 @@
21032108
'OC\\Share\\Constants' => $baseDir . '/lib/private/Share/Constants.php',
21042109
'OC\\Share\\Helper' => $baseDir . '/lib/private/Share/Helper.php',
21052110
'OC\\Share\\Share' => $baseDir . '/lib/private/Share/Share.php',
2111+
'OC\\Snowflake\\Decoder' => $baseDir . '/lib/private/Snowflake/Decoder.php',
2112+
'OC\\Snowflake\\Generator' => $baseDir . '/lib/private/Snowflake/Generator.php',
21062113
'OC\\SpeechToText\\SpeechToTextManager' => $baseDir . '/lib/private/SpeechToText/SpeechToTextManager.php',
21072114
'OC\\SpeechToText\\TranscriptionJob' => $baseDir . '/lib/private/SpeechToText/TranscriptionJob.php',
21082115
'OC\\StreamImage' => $baseDir . '/lib/private/StreamImage.php',

‎lib/composer/composer/autoload_static.php‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -863,6 +863,8 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
863863
'OCP\\Share_Backend' => __DIR__ . '/../../..' . '/lib/public/Share_Backend.php',
864864
'OCP\\Share_Backend_Collection' => __DIR__ . '/../../..' . '/lib/public/Share_Backend_Collection.php',
865865
'OCP\\Share_Backend_File_Dependent' => __DIR__ . '/../../..' . '/lib/public/Share_Backend_File_Dependent.php',
866+
'OCP\\Snowflake\\IDecoder' => __DIR__ . '/../../..' . '/lib/public/Snowflake/IDecoder.php',
867+
'OCP\\Snowflake\\IGenerator' => __DIR__ . '/../../..' . '/lib/public/Snowflake/IGenerator.php',
866868
'OCP\\SpeechToText\\Events\\AbstractTranscriptionEvent' => __DIR__ . '/../../..' . '/lib/public/SpeechToText/Events/AbstractTranscriptionEvent.php',
867869
'OCP\\SpeechToText\\Events\\TranscriptionFailedEvent' => __DIR__ . '/../../..' . '/lib/public/SpeechToText/Events/TranscriptionFailedEvent.php',
868870
'OCP\\SpeechToText\\Events\\TranscriptionSuccessfulEvent' => __DIR__ . '/../../..' . '/lib/public/SpeechToText/Events/TranscriptionSuccessfulEvent.php',
@@ -1393,6 +1395,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
13931395
'OC\\Core\\Command\\Security\\ListCertificates' => __DIR__ . '/../../..' . '/core/Command/Security/ListCertificates.php',
13941396
'OC\\Core\\Command\\Security\\RemoveCertificate' => __DIR__ . '/../../..' . '/core/Command/Security/RemoveCertificate.php',
13951397
'OC\\Core\\Command\\SetupChecks' => __DIR__ . '/../../..' . '/core/Command/SetupChecks.php',
1398+
'OC\\Core\\Command\\SnowflakeDecodeId' => __DIR__ . '/../../..' . '/core/Command/SnowflakeDecodeId.php',
13961399
'OC\\Core\\Command\\Status' => __DIR__ . '/../../..' . '/core/Command/Status.php',
13971400
'OC\\Core\\Command\\SystemTag\\Add' => __DIR__ . '/../../..' . '/core/Command/SystemTag/Add.php',
13981401
'OC\\Core\\Command\\SystemTag\\Delete' => __DIR__ . '/../../..' . '/core/Command/SystemTag/Delete.php',
@@ -1568,6 +1571,8 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
15681571
'OC\\Core\\Migrations\\Version32000Date20250731062008' => __DIR__ . '/../../..' . '/core/Migrations/Version32000Date20250731062008.php',
15691572
'OC\\Core\\Migrations\\Version32000Date20250806110519' => __DIR__ . '/../../..' . '/core/Migrations/Version32000Date20250806110519.php',
15701573
'OC\\Core\\Migrations\\Version33000Date20250819110529' => __DIR__ . '/../../..' . '/core/Migrations/Version33000Date20250819110529.php',
1574+
'OC\\Core\\Migrations\\Version33000Date20251023110529' => __DIR__ . '/../../..' . '/core/Migrations/Version33000Date20251023110529.php',
1575+
'OC\\Core\\Migrations\\Version33000Date20251023120529' => __DIR__ . '/../../..' . '/core/Migrations/Version33000Date20251023120529.php',
15711576
'OC\\Core\\Notification\\CoreNotifier' => __DIR__ . '/../../..' . '/core/Notification/CoreNotifier.php',
15721577
'OC\\Core\\ResponseDefinitions' => __DIR__ . '/../../..' . '/core/ResponseDefinitions.php',
15731578
'OC\\Core\\Service\\CronService' => __DIR__ . '/../../..' . '/core/Service/CronService.php',
@@ -2144,6 +2149,8 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
21442149
'OC\\Share\\Constants' => __DIR__ . '/../../..' . '/lib/private/Share/Constants.php',
21452150
'OC\\Share\\Helper' => __DIR__ . '/../../..' . '/lib/private/Share/Helper.php',
21462151
'OC\\Share\\Share' => __DIR__ . '/../../..' . '/lib/private/Share/Share.php',
2152+
'OC\\Snowflake\\Decoder' => __DIR__ . '/../../..' . '/lib/private/Snowflake/Decoder.php',
2153+
'OC\\Snowflake\\Generator' => __DIR__ . '/../../..' . '/lib/private/Snowflake/Generator.php',
21472154
'OC\\SpeechToText\\SpeechToTextManager' => __DIR__ . '/../../..' . '/lib/private/SpeechToText/SpeechToTextManager.php',
21482155
'OC\\SpeechToText\\TranscriptionJob' => __DIR__ . '/../../..' . '/lib/private/SpeechToText/TranscriptionJob.php',
21492156
'OC\\StreamImage' => __DIR__ . '/../../..' . '/lib/private/StreamImage.php',

0 commit comments

Comments
 (0)