Skip to content

Commit 1e4b8c8

Browse files
CarlSchwanCarlSchwan
committed
feat(Db): Add SnowflakeId generator
Use generator from godruoyi/php-snowflake adapted for our needs. With a IMemCache as sequence generator when available or a random sequence generator otherwise. Implemented for the Preview table where this allow to save the preview on the filesystem as we know the id now before inserting it to the DB. Signed-off-by: Carl Schwan <carl.schwan@nextcloud.com>
1 parent 28eb887 commit 1e4b8c8

File tree

13 files changed

+365
-39
lines changed

13 files changed

+365
-39
lines changed

‎core/Migrations/Version33000Date20250819110529.php‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,23 +31,23 @@ public function changeSchema(IOutput $output, Closure $schemaClosure, array $opt
3131

3232
if (!$schema->hasTable('preview_locations')) {
3333
$table = $schema->createTable('preview_locations');
34-
$table->addColumn('id', Types::BIGINT, ['autoincrement' => true, 'notnull' => true, 'length' => 20, 'unsigned' => true]);
34+
$table->addColumn('id', Types::BIGINT, ['notnull' => true, 'length' => 20, 'unsigned' => true]);
3535
$table->addColumn('bucket_name', Types::STRING, ['notnull' => true, 'length' => 40]);
3636
$table->addColumn('object_store_name', Types::STRING, ['notnull' => true, 'length' => 40]);
3737
$table->setPrimaryKey(['id']);
3838
}
3939

4040
if (!$schema->hasTable('preview_versions')) {
4141
$table = $schema->createTable('preview_versions');
42-
$table->addColumn('id', Types::BIGINT, ['autoincrement' => true, 'notnull' => true, 'length' => 20, 'unsigned' => true]);
42+
$table->addColumn('id', Types::BIGINT, ['notnull' => true, 'length' => 20, 'unsigned' => true]);
4343
$table->addColumn('file_id', Types::BIGINT, ['notnull' => true, 'length' => 20, 'unsigned' => true]);
4444
$table->addColumn('version', Types::STRING, ['notnull' => true, 'default' => '', 'length' => 1024]);
4545
$table->setPrimaryKey(['id']);
4646
}
4747

4848
if (!$schema->hasTable('previews')) {
4949
$table = $schema->createTable('previews');
50-
$table->addColumn('id', Types::BIGINT, ['autoincrement' => true, 'notnull' => true, 'length' => 20, 'unsigned' => true]);
50+
$table->addColumn('id', Types::BIGINT, ['notnull' => true, 'length' => 20, 'unsigned' => true]);
5151
$table->addColumn('file_id', Types::BIGINT, ['notnull' => true, 'length' => 20, 'unsigned' => true]);
5252
$table->addColumn('storage_id', Types::BIGINT, ['notnull' => true, 'length' => 20, 'unsigned' => true]);
5353
$table->addColumn('old_file_id', Types::BIGINT, ['notnull' => false, 'length' => 20, 'unsigned' => true]);

‎lib/composer/composer/autoload_classmap.php‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@
7676
'OCP\\AppFramework\\Db\\DoesNotExistException' => $baseDir . '/lib/public/AppFramework/Db/DoesNotExistException.php',
7777
'OCP\\AppFramework\\Db\\Entity' => $baseDir . '/lib/public/AppFramework/Db/Entity.php',
7878
'OCP\\AppFramework\\Db\\IMapperException' => $baseDir . '/lib/public/AppFramework/Db/IMapperException.php',
79+
'OCP\\AppFramework\\Db\\ISnowflake' => $baseDir . '/lib/public/AppFramework/Db/ISnowflake.php',
7980
'OCP\\AppFramework\\Db\\MultipleObjectsReturnedException' => $baseDir . '/lib/public/AppFramework/Db/MultipleObjectsReturnedException.php',
8081
'OCP\\AppFramework\\Db\\QBMapper' => $baseDir . '/lib/public/AppFramework/Db/QBMapper.php',
8182
'OCP\\AppFramework\\Db\\TTransactional' => $baseDir . '/lib/public/AppFramework/Db/TTransactional.php',
@@ -1036,6 +1037,9 @@
10361037
'OC\\AppFramework\\Bootstrap\\ServiceAliasRegistration' => $baseDir . '/lib/private/AppFramework/Bootstrap/ServiceAliasRegistration.php',
10371038
'OC\\AppFramework\\Bootstrap\\ServiceFactoryRegistration' => $baseDir . '/lib/private/AppFramework/Bootstrap/ServiceFactoryRegistration.php',
10381039
'OC\\AppFramework\\Bootstrap\\ServiceRegistration' => $baseDir . '/lib/private/AppFramework/Bootstrap/ServiceRegistration.php',
1040+
'OC\\AppFramework\\Db\\NextcloudSequenceResolver' => $baseDir . '/lib/private/AppFramework/Db/NextcloudSequenceResolver.php',
1041+
'OC\\AppFramework\\Db\\Snowflake' => $baseDir . '/lib/private/AppFramework/Db/Snowflake.php',
1042+
'OC\\AppFramework\\Db\\SnowflakeGenerator' => $baseDir . '/lib/private/AppFramework/Db/SnowflakeGenerator.php',
10391043
'OC\\AppFramework\\DependencyInjection\\DIContainer' => $baseDir . '/lib/private/AppFramework/DependencyInjection/DIContainer.php',
10401044
'OC\\AppFramework\\Http' => $baseDir . '/lib/private/AppFramework/Http.php',
10411045
'OC\\AppFramework\\Http\\Dispatcher' => $baseDir . '/lib/private/AppFramework/Http/Dispatcher.php',

‎lib/composer/composer/autoload_static.php‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,32 +11,32 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
1111
);
1212

1313
public static $prefixLengthsPsr4 = array (
14-
'O' =>
14+
'O' =>
1515
array (
1616
'OC\\Core\\' => 8,
1717
'OC\\' => 3,
1818
'OCP\\' => 4,
1919
),
20-
'N' =>
20+
'N' =>
2121
array (
2222
'NCU\\' => 4,
2323
),
2424
);
2525

2626
public static $prefixDirsPsr4 = array (
27-
'OC\\Core\\' =>
27+
'OC\\Core\\' =>
2828
array (
2929
0 => __DIR__ . '/../../..' . '/core',
3030
),
31-
'OC\\' =>
31+
'OC\\' =>
3232
array (
3333
0 => __DIR__ . '/../../..' . '/lib/private',
3434
),
35-
'OCP\\' =>
35+
'OCP\\' =>
3636
array (
3737
0 => __DIR__ . '/../../..' . '/lib/public',
3838
),
39-
'NCU\\' =>
39+
'NCU\\' =>
4040
array (
4141
0 => __DIR__ . '/../../..' . '/lib/unstable',
4242
),
@@ -117,6 +117,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
117117
'OCP\\AppFramework\\Db\\DoesNotExistException' => __DIR__ . '/../../..' . '/lib/public/AppFramework/Db/DoesNotExistException.php',
118118
'OCP\\AppFramework\\Db\\Entity' => __DIR__ . '/../../..' . '/lib/public/AppFramework/Db/Entity.php',
119119
'OCP\\AppFramework\\Db\\IMapperException' => __DIR__ . '/../../..' . '/lib/public/AppFramework/Db/IMapperException.php',
120+
'OCP\\AppFramework\\Db\\ISnowflake' => __DIR__ . '/../../..' . '/lib/public/AppFramework/Db/ISnowflake.php',
120121
'OCP\\AppFramework\\Db\\MultipleObjectsReturnedException' => __DIR__ . '/../../..' . '/lib/public/AppFramework/Db/MultipleObjectsReturnedException.php',
121122
'OCP\\AppFramework\\Db\\QBMapper' => __DIR__ . '/../../..' . '/lib/public/AppFramework/Db/QBMapper.php',
122123
'OCP\\AppFramework\\Db\\TTransactional' => __DIR__ . '/../../..' . '/lib/public/AppFramework/Db/TTransactional.php',
@@ -1077,6 +1078,9 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
10771078
'OC\\AppFramework\\Bootstrap\\ServiceAliasRegistration' => __DIR__ . '/../../..' . '/lib/private/AppFramework/Bootstrap/ServiceAliasRegistration.php',
10781079
'OC\\AppFramework\\Bootstrap\\ServiceFactoryRegistration' => __DIR__ . '/../../..' . '/lib/private/AppFramework/Bootstrap/ServiceFactoryRegistration.php',
10791080
'OC\\AppFramework\\Bootstrap\\ServiceRegistration' => __DIR__ . '/../../..' . '/lib/private/AppFramework/Bootstrap/ServiceRegistration.php',
1081+
'OC\\AppFramework\\Db\\NextcloudSequenceResolver' => __DIR__ . '/../../..' . '/lib/private/AppFramework/Db/NextcloudSequenceResolver.php',
1082+
'OC\\AppFramework\\Db\\Snowflake' => __DIR__ . '/../../..' . '/lib/private/AppFramework/Db/Snowflake.php',
1083+
'OC\\AppFramework\\Db\\SnowflakeGenerator' => __DIR__ . '/../../..' . '/lib/private/AppFramework/Db/SnowflakeGenerator.php',
10801084
'OC\\AppFramework\\DependencyInjection\\DIContainer' => __DIR__ . '/../../..' . '/lib/private/AppFramework/DependencyInjection/DIContainer.php',
10811085
'OC\\AppFramework\\Http' => __DIR__ . '/../../..' . '/lib/private/AppFramework/Http.php',
10821086
'OC\\AppFramework\\Http\\Dispatcher' => __DIR__ . '/../../..' . '/lib/private/AppFramework/Http/Dispatcher.php',

‎lib/private/AppFramework/Db/NextcloudSequenceResolver.php‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH
7+
* SPDX-FileContributor: Carl Schwan
8+
* SPDX-License-Identifier: AGPL-3.0-or-later
9+
*/
10+
11+
namespace OC\AppFramework\Db;
12+
13+
use OCP\ICacheFactory;
14+
use OCP\IMemcache;
15+
16+
class NextcloudSequenceResolver {
17+
private ?IMemCache $localCache = null;
18+
19+
public function __construct(
20+
ICacheFactory $cache,
21+
) {
22+
$localCache = $cache->createLocal('snowflake');
23+
24+
if ($localCache instanceof IMemcache) {
25+
$this->localCache = $localCache;
26+
}
27+
}
28+
29+
public function isAvailable(): bool {
30+
return $this->localCache instanceof IMemcache;
31+
}
32+
33+
public function sequence(int $currentTime): int {
34+
if ($this->localCache->add((string)$currentTime, 1, 10)) {
35+
return 0;
36+
}
37+
38+
return $this->localCache->inc((string)$currentTime, 1) | 0;
39+
}
40+
}

‎lib/private/AppFramework/Db/Snowflake.php‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* This file is based on tourze/symfony-snowflake-bundle
7+
* SPDX-FileCopyrightText: tourze <https://github.com/tourze>
8+
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH
9+
* SPDX-FileContributor: Carl Schwan
10+
* SPDX-License-Identifier: MIT
11+
*/
12+
13+
namespace OC\AppFramework\Db;
14+
15+
use OCP\AppFramework\Db\ISnowflake;
16+
17+
class Snowflake implements ISnowflake {
18+
19+
/**
20+
* @var SnowflakeGenerator[]
21+
*/
22+
private static array $generators = [];
23+
24+
public static function getGenerator(int $datacenter, int $workerId, NextcloudSequenceResolver $resolver): SnowflakeGenerator {
25+
$key = "{$datacenter}-{$workerId}";
26+
if (!isset(self::$generators[$key])) {
27+
$generator = new SnowflakeGenerator(
28+
$datacenter,
29+
$workerId,
30+
$resolver,
31+
);
32+
$generator->setStartTimeStamp(strtotime('2025-01-01') * 1000);
33+
self::$generators[$key] = $generator;
34+
}
35+
return self::$generators[$key];
36+
}
37+
38+
public static function generateWorkerId(string $hostname, int $maxWorkerId = 31): int {
39+
$hash = crc32($hostname);
40+
return $hash % ($maxWorkerId + 1);
41+
}
42+
43+
private SnowflakeGenerator $generator;
44+
45+
public function __construct(
46+
NextcloudSequenceResolver $nextcloudSequenceResolver,
47+
) {
48+
$this->generator = static::getGenerator(
49+
-1, // ATM set randomely
50+
self::generateWorkerId(gethostname()),
51+
$nextcloudSequenceResolver
52+
);
53+
}
54+
55+
public function id(): string {
56+
return $this->generator->id();
57+
}
58+
59+
public function parseId(string $id, bool $transform = false): array {
60+
return SnowflakeGenerator::parseId($id, $transform);
61+
}
62+
}

‎lib/private/AppFramework/Db/SnowflakeGenerator.php‎

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* This file is based on the package: godruoyi/php-snowflake.
7+
* SPDX-FileCopyrightText: 2024 Godruoyi <g@godruoyi.com>
8+
* SPDX-License-Identifier: MIT
9+
*/
10+
11+
namespace OC\AppFramework\Db;
12+
13+
class SnowflakeGenerator {
14+
public const MAX_TIMESTAMP_LENGTH = 41;
15+
public const MAX_DATACENTER_LENGTH = 5;
16+
public const MAX_WORKID_LENGTH = 5;
17+
public const MAX_SEQUENCE_LENGTH = 12;
18+
public const MAX_SEQUENCE_SIZE = (-1 ^ (-1 << self::MAX_SEQUENCE_LENGTH));
19+
20+
/**
21+
* The data center id.
22+
*/
23+
protected int $datacenter;
24+
25+
/**
26+
* The worker id.
27+
*/
28+
protected int $workerId;
29+
30+
/**
31+
* The start timestamp.
32+
*/
33+
protected ?int $startTime = null;
34+
35+
/**
36+
* The last timestamp for the random generator.
37+
*/
38+
protected int $lastTimeStamp = -1;
39+
40+
/**
41+
* The sequence number for the random generator.
42+
*/
43+
protected int $sequence = 0;
44+
45+
/**
46+
* Build Snowflake Instance.
47+
*/
48+
public function __construct(
49+
int $datacenter,
50+
int $workerId,
51+
private readonly NextcloudSequenceResolver $sequenceResolver,
52+
) {
53+
$maxDataCenter = -1 ^ (-1 << self::MAX_DATACENTER_LENGTH);
54+
$maxWorkId = -1 ^ (-1 << self::MAX_WORKID_LENGTH);
55+
56+
// If not set datacenter or workid, we will set a default value to use.
57+
$this->datacenter = $datacenter < 0 || $datacenter > $maxDataCenter ? random_int(0, 31) : $datacenter;
58+
$this->workerId = $workerId < 0 || $workerId > $maxWorkId ? random_int(0, 31) : $workerId;
59+
}
60+
61+
/**
62+
* Get snowflake id.
63+
*/
64+
public function id(): string {
65+
$currentTime = $this->getCurrentMillisecond();
66+
while (($sequence = $this->callResolver($currentTime)) > (-1 ^ (-1 << self::MAX_SEQUENCE_LENGTH))) {
67+
usleep(1);
68+
$currentTime = $this->getCurrentMillisecond();
69+
}
70+
71+
$workerLeftMoveLength = self::MAX_SEQUENCE_LENGTH;
72+
$datacenterLeftMoveLength = self::MAX_WORKID_LENGTH + $workerLeftMoveLength;
73+
$timestampLeftMoveLength = self::MAX_DATACENTER_LENGTH + $datacenterLeftMoveLength;
74+
75+
return (string)((($currentTime - $this->getStartTimeStamp()) << $timestampLeftMoveLength)
76+
| ($this->datacenter << $datacenterLeftMoveLength)
77+
| ($this->workerId << $workerLeftMoveLength)
78+
| ($sequence));
79+
}
80+
81+
/**
82+
* Parse snowflake id.
83+
*/
84+
public static function parseId(string $id, bool $transform = false): array {
85+
$id = decbin((int)$id);
86+
87+
$data = [
88+
'timestamp' => substr($id, 0, -22),
89+
'sequence' => substr($id, -12),
90+
'workerid' => substr($id, -17, 5),
91+
'datacenter' => substr($id, -22, 5),
92+
];
93+
94+
return $transform ? array_map(static function ($value) {
95+
return bindec($value);
96+
}, $data) : $data;
97+
}
98+
99+
/**
100+
* Get current millisecond time.
101+
*/
102+
public function getCurrentMillisecond(): int {
103+
return (int)floor(microtime(true) * 1000) | 0;
104+
}
105+
106+
/**
107+
* Set start time (millisecond).
108+
* @throw \InvalidArgumentException
109+
*/
110+
public function setStartTimeStamp(int $millisecond): self {
111+
$missTime = $this->getCurrentMillisecond() - $millisecond;
112+
113+
if ($missTime < 0) {
114+
throw new \InvalidArgumentException('The start time cannot be greater than the current time');
115+
}
116+
117+
$maxTimeDiff = -1 ^ (-1 << self::MAX_TIMESTAMP_LENGTH);
118+
119+
if ($missTime > $maxTimeDiff) {
120+
throw new \InvalidArgumentException(sprintf('The current microtime - starttime is not allowed to exceed -1 ^ (-1 << %d), You can reset the start time to fix this', self::MAX_TIMESTAMP_LENGTH));
121+
}
122+
123+
$this->startTime = $millisecond;
124+
125+
return $this;
126+
}
127+
128+
/**
129+
* Get start timestamp (millisecond), If not set default to 2019-08-08 08:08:08.
130+
*/
131+
public function getStartTimeStamp(): float|int {
132+
if (! is_null($this->startTime)) {
133+
return $this->startTime;
134+
}
135+
136+
// We set a default start time if you not set.
137+
$defaultTime = '2019-08-08 08:08:08';
138+
139+
return strtotime($defaultTime) * 1000;
140+
}
141+
142+
/**
143+
* Call resolver.
144+
*/
145+
protected function callResolver(int $currentTime): int {
146+
// Memcache based resolver
147+
if ($this->sequenceResolver->isAvailable()) {
148+
return $this->sequenceResolver->sequence($currentTime);
149+
}
150+
151+
// random fallback
152+
if ($this->lastTimeStamp === $currentTime) {
153+
$this->sequence++;
154+
$this->lastTimeStamp = $currentTime;
155+
156+
return $this->sequence;
157+
}
158+
159+
$this->sequence = crc32(uniqid((string)random_int(0, PHP_INT_MAX), true)) % self::MAX_SEQUENCE_SIZE;
160+
$this->lastTimeStamp = $currentTime;
161+
162+
return $this->sequence;
163+
}
164+
}

‎lib/private/AppFramework/Http/Dispatcher.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -204,7 +204,7 @@ private function executeController(Controller $controller, string $methodName):
204204
try {
205205
$response = \call_user_func_array([$controller, $methodName], $arguments);
206206
} catch (\TypeError $e) {
207-
// Only intercept TypeErrors occuring on the first line, meaning that the invocation of the controller method failed.
207+
// Only intercept TypeErrors occurring on the first line, meaning that the invocation of the controller method failed.
208208
// Any other TypeError happens inside the controller method logic and should be logged as normal.
209209
if ($e->getFile() === $this->reflector->getFile() && $e->getLine() === $this->reflector->getStartLine()) {
210210
$this->logger->debug('Failed to call controller method: ' . $e->getMessage(), ['exception' => $e]);

0 commit comments

Comments
 (0)