Skip to content

Commit ff2de9d

Browse files
committed
feat(database): introduce Snowflake IDs generator
Signed-off-by: Benjamin Gaussorgues <benjamin.gaussorgues@nextcloud.com>
1 parent 84be993 commit ff2de9d

File tree

8 files changed

+398
-6
lines changed

8 files changed

+398
-6
lines changed

lib/composer/composer/autoload_classmap.php

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -628,6 +628,8 @@
628628
'OCP\\IRequestId' => $baseDir . '/lib/public/IRequestId.php',
629629
'OCP\\IServerContainer' => $baseDir . '/lib/public/IServerContainer.php',
630630
'OCP\\ISession' => $baseDir . '/lib/public/ISession.php',
631+
'OCP\\ISnowflakeId' => $baseDir . '/lib/public/ISnowflakeId.php',
632+
'OCP\\ISnowflakeIdGenerator' => $baseDir . '/lib/public/ISnowflakeIdGenerator.php',
631633
'OCP\\IStreamImage' => $baseDir . '/lib/public/IStreamImage.php',
632634
'OCP\\ITagManager' => $baseDir . '/lib/public/ITagManager.php',
633635
'OCP\\ITags' => $baseDir . '/lib/public/ITags.php',
@@ -1871,6 +1873,7 @@
18711873
'OC\\OCS\\CoreCapabilities' => $baseDir . '/lib/private/OCS/CoreCapabilities.php',
18721874
'OC\\OCS\\DiscoveryService' => $baseDir . '/lib/private/OCS/DiscoveryService.php',
18731875
'OC\\OCS\\Provider' => $baseDir . '/lib/private/OCS/Provider.php',
1876+
'OC\\OpenMetrics\\Exporters\\RunningJobs' => $baseDir . '/lib/private/OpenMetrics/Exporters/RunningJobs.php',
18741877
'OC\\PhoneNumberUtil' => $baseDir . '/lib/private/PhoneNumberUtil.php',
18751878
'OC\\PreviewManager' => $baseDir . '/lib/private/PreviewManager.php',
18761879
'OC\\PreviewNotAvailableException' => $baseDir . '/lib/private/PreviewNotAvailableException.php',
@@ -2103,6 +2106,8 @@
21032106
'OC\\Share\\Constants' => $baseDir . '/lib/private/Share/Constants.php',
21042107
'OC\\Share\\Helper' => $baseDir . '/lib/private/Share/Helper.php',
21052108
'OC\\Share\\Share' => $baseDir . '/lib/private/Share/Share.php',
2109+
'OC\\SnowflakeId' => $baseDir . '/lib/private/SnowflakeId.php',
2110+
'OC\\SnowflakeIdGenerator' => $baseDir . '/lib/private/SnowflakeIdGenerator.php',
21062111
'OC\\SpeechToText\\SpeechToTextManager' => $baseDir . '/lib/private/SpeechToText/SpeechToTextManager.php',
21072112
'OC\\SpeechToText\\TranscriptionJob' => $baseDir . '/lib/private/SpeechToText/TranscriptionJob.php',
21082113
'OC\\StreamImage' => $baseDir . '/lib/private/StreamImage.php',

lib/composer/composer/autoload_static.php

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,32 +11,32 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
1111
);
1212

1313
public static $prefixLengthsPsr4 = array (
14-
'O' =>
14+
'O' =>
1515
array (
1616
'OC\\Core\\' => 8,
1717
'OC\\' => 3,
1818
'OCP\\' => 4,
1919
),
20-
'N' =>
20+
'N' =>
2121
array (
2222
'NCU\\' => 4,
2323
),
2424
);
2525

2626
public static $prefixDirsPsr4 = array (
27-
'OC\\Core\\' =>
27+
'OC\\Core\\' =>
2828
array (
2929
0 => __DIR__ . '/../../..' . '/core',
3030
),
31-
'OC\\' =>
31+
'OC\\' =>
3232
array (
3333
0 => __DIR__ . '/../../..' . '/lib/private',
3434
),
35-
'OCP\\' =>
35+
'OCP\\' =>
3636
array (
3737
0 => __DIR__ . '/../../..' . '/lib/public',
3838
),
39-
'NCU\\' =>
39+
'NCU\\' =>
4040
array (
4141
0 => __DIR__ . '/../../..' . '/lib/unstable',
4242
),
@@ -669,6 +669,8 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
669669
'OCP\\IRequestId' => __DIR__ . '/../../..' . '/lib/public/IRequestId.php',
670670
'OCP\\IServerContainer' => __DIR__ . '/../../..' . '/lib/public/IServerContainer.php',
671671
'OCP\\ISession' => __DIR__ . '/../../..' . '/lib/public/ISession.php',
672+
'OCP\\ISnowflakeId' => __DIR__ . '/../../..' . '/lib/public/ISnowflakeId.php',
673+
'OCP\\ISnowflakeIdGenerator' => __DIR__ . '/../../..' . '/lib/public/ISnowflakeIdGenerator.php',
672674
'OCP\\IStreamImage' => __DIR__ . '/../../..' . '/lib/public/IStreamImage.php',
673675
'OCP\\ITagManager' => __DIR__ . '/../../..' . '/lib/public/ITagManager.php',
674676
'OCP\\ITags' => __DIR__ . '/../../..' . '/lib/public/ITags.php',
@@ -1912,6 +1914,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
19121914
'OC\\OCS\\CoreCapabilities' => __DIR__ . '/../../..' . '/lib/private/OCS/CoreCapabilities.php',
19131915
'OC\\OCS\\DiscoveryService' => __DIR__ . '/../../..' . '/lib/private/OCS/DiscoveryService.php',
19141916
'OC\\OCS\\Provider' => __DIR__ . '/../../..' . '/lib/private/OCS/Provider.php',
1917+
'OC\\OpenMetrics\\Exporters\\RunningJobs' => __DIR__ . '/../../..' . '/lib/private/OpenMetrics/Exporters/RunningJobs.php',
19151918
'OC\\PhoneNumberUtil' => __DIR__ . '/../../..' . '/lib/private/PhoneNumberUtil.php',
19161919
'OC\\PreviewManager' => __DIR__ . '/../../..' . '/lib/private/PreviewManager.php',
19171920
'OC\\PreviewNotAvailableException' => __DIR__ . '/../../..' . '/lib/private/PreviewNotAvailableException.php',
@@ -2144,6 +2147,8 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
21442147
'OC\\Share\\Constants' => __DIR__ . '/../../..' . '/lib/private/Share/Constants.php',
21452148
'OC\\Share\\Helper' => __DIR__ . '/../../..' . '/lib/private/Share/Helper.php',
21462149
'OC\\Share\\Share' => __DIR__ . '/../../..' . '/lib/private/Share/Share.php',
2150+
'OC\\SnowflakeId' => __DIR__ . '/../../..' . '/lib/private/SnowflakeId.php',
2151+
'OC\\SnowflakeIdGenerator' => __DIR__ . '/../../..' . '/lib/private/SnowflakeIdGenerator.php',
21472152
'OC\\SpeechToText\\SpeechToTextManager' => __DIR__ . '/../../..' . '/lib/private/SpeechToText/SpeechToTextManager.php',
21482153
'OC\\SpeechToText\\TranscriptionJob' => __DIR__ . '/../../..' . '/lib/private/SpeechToText/TranscriptionJob.php',
21492154
'OC\\StreamImage' => __DIR__ . '/../../..' . '/lib/private/StreamImage.php',

lib/private/SnowflakeId.php

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-only
8+
*/
9+
10+
namespace OC;
11+
12+
use OCP\ISnowflakeId;
13+
use Override;
14+
15+
/**
16+
* Nextcloud Snowflake ID
17+
*
18+
* Get information about Snowflake Id
19+
*
20+
* @since 33.0.0
21+
*/
22+
final class SnowflakeId implements ISnowflakeId {
23+
private int $seconds = 0;
24+
private int $milliseconds = 0;
25+
private bool $isCli = false;
26+
/** @var int<0, 511> */
27+
private int $serverId = 0;
28+
/** @var int<0, 4095> */
29+
private int $sequenceId = 0;
30+
31+
public function __construct(
32+
private readonly int|float $id,
33+
) {
34+
}
35+
36+
private function decode(): void {
37+
if ($this->seconds !== 0) {
38+
return;
39+
}
40+
41+
PHP_INT_SIZE === 9 // FIXME 8
42+
? $this->decode64bits()
43+
: $this->decode32bits();
44+
}
45+
46+
private function decode64bits(): void {
47+
$id = (int)$this->id;
48+
$firstHalf = $id >> 32;
49+
$secondHalf = $id & 0xFFFFFFFF;
50+
51+
// First half without first bit is seconds
52+
$this->seconds = $firstHalf & 0x7FFFFFFF;
53+
54+
printf(
55+
"Debug: %08x %08x\n",
56+
$firstHalf,
57+
$secondHalf,
58+
);
59+
60+
// Decode second half
61+
$this->milliseconds = $secondHalf >> 22;
62+
$this->serverId = ($secondHalf >> 13) & 0x1FF;
63+
$this->isCli = (bool)(($secondHalf >> 12) & 0x1);
64+
$this->sequenceId = $secondHalf & 0xFFF;
65+
}
66+
67+
private function decode32bits(): void {
68+
$id = str_pad(base_convert(number_format($this->id, 0, '', ''), 10, 16), 16, '0', STR_PAD_LEFT);
69+
$firstQuarter = hexdec(substr($id, 0, 4));
70+
$secondQuarter = hexdec(substr($id, 4, 4));
71+
$thirdQuarter = hexdec(substr($id, 8, 4));
72+
$fourthQuarter = hexdec(substr($id, 12, 4));
73+
74+
printf("\nDebug: %s\n", number_format($this->id, 0, '', ''));
75+
printf("Debug: %s\n", $id);
76+
printf(
77+
"Debug: %04x %04x %04x %04x\n",
78+
$firstQuarter,
79+
$secondQuarter,
80+
$thirdQuarter,
81+
$fourthQuarter,
82+
);
83+
84+
$this->seconds = (($firstQuarter & 0x7FFF) << 16) | ($secondQuarter & 0xFFFF);
85+
86+
$this->milliseconds = ($thirdQuarter >> 6) & 0x3FF;
87+
88+
$this->serverId = (($thirdQuarter & 0x3F) << 3) | (($fourthQuarter >> 13) & 0x7);
89+
$this->isCli = (bool)(($fourthQuarter >> 12) & 0x1);
90+
$this->sequenceId = $fourthQuarter & 0xFFF;
91+
}
92+
93+
#[Override]
94+
public function isCli(): bool {
95+
return $this->isCli;
96+
}
97+
98+
#[Override]
99+
public function numeric(): int|float {
100+
return $this->id;
101+
}
102+
103+
#[Override]
104+
public function seconds(): int {
105+
$this->decode();
106+
return $this->seconds;
107+
}
108+
109+
#[Override]
110+
public function milliseconds(): int {
111+
$this->decode();
112+
return $this->milliseconds;
113+
}
114+
115+
#[Override]
116+
public function createdAt(): float {
117+
$this->decode();
118+
return $this->seconds + self::TS_OFFSET + ($this->milliseconds / 1000);
119+
}
120+
121+
#[Override]
122+
public function serverId(): int {
123+
$this->decode();
124+
return $this->serverId;
125+
}
126+
127+
#[Override]
128+
public function sequenceId(): int {
129+
$this->decode();
130+
return $this->sequenceId;
131+
}
132+
}
Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-only
8+
*/
9+
10+
namespace OC;
11+
12+
/**
13+
* Nextcloud Snowflake ID generator
14+
*
15+
* Generates unique ID for database
16+
*
17+
* @since 33.0.0
18+
*/
19+
final class SnowflakeIdGenerator {
20+
public function __invoke(): int|float {
21+
// Time related
22+
$microtime = microtime(true);
23+
$seconds = ((int)$microtime) - SnowflakeId::TS_OFFSET;
24+
$milliseconds = ((int)($microtime * 1000)) % 1000;
25+
26+
$serverId = $this->getServerId() & 0x1FF; // Keep 9 bits
27+
$isCli = (int)$this->isCli(); // 1 bit
28+
$sequenceId = $this->getSequenceId($seconds, $milliseconds); // 12 bits
29+
if ($sequenceId > 0xFFF) {
30+
// Throttle a bit, wait for next millisecond
31+
usleep(1000);
32+
return $this();
33+
}
34+
35+
$firstHalf = $seconds & 0x7FFFFFFF;
36+
$secondHalf = (($milliseconds & 0x3FF) << 22) | ($serverId << 13) | ($isCli << 12) | $sequenceId;
37+
if (PHP_INT_SIZE === 8) {
38+
return $firstHalf << 32 | $secondHalf;
39+
}
40+
41+
// Fallback for 32 bits systems
42+
return hexdec(bin2hex(pack('LL', $firstHalf, $secondHalf)));
43+
}
44+
45+
private function getServerId(): int {
46+
return crc32(gethostname() ?: random_bytes(8));
47+
}
48+
49+
private function isCli() {
50+
return PHP_SAPI === 'cli';
51+
}
52+
53+
private function getSequenceId(int $seconds, int $milliseconds): int {
54+
if ($this->isCli()) {
55+
// APCu cache isn’t shared between CLI processes
56+
return random_int(0, 0xFFF - 1);
57+
}
58+
59+
if (function_exists('apcu_inc')) {
60+
$key = 'sequence:' . $seconds . ':' . $milliseconds;
61+
$sequenceId = apcu_inc($key, ttl: 1);
62+
if ($sequenceId === false) {
63+
throw new \Exception('Failed to generate SnowflakeId with APCu');
64+
}
65+
66+
return $sequenceId;
67+
}
68+
69+
// TODO Implement file fallback?
70+
throw new \Exception('Failed to get sequence Id');
71+
}
72+
}

lib/public/ISnowflakeId.php

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-only
8+
*/
9+
10+
namespace OCP;
11+
12+
/**
13+
* Nextcloud ID generator
14+
*
15+
* Generates unique ID
16+
* @since 33.0.0
17+
*/
18+
interface ISnowflakeId {
19+
public const TS_OFFSET = 1759276800; // 2025-10-01 00:00:00
20+
21+
/**
22+
* Returns sequence ID as int (64 bits servers) or float (32 bits servers)
23+
*
24+
* This method is suitable to store Sequence Id in database.
25+
* Use BIGINT (8 bytes)
26+
*
27+
* @since 33.0
28+
*/
29+
public function numeric(): int|float;
30+
31+
/**
32+
* Returns whether the SequenceId was created in CLI or not (eg. FPM, Apache)
33+
*
34+
* @since 33.0
35+
*/
36+
public function isCli(): bool;
37+
38+
/**
39+
* Returns the number of seconds after self::TS_OFFSET
40+
*
41+
* Creation time of the sequence ID since self::TS_OFFSET
42+
*
43+
* @since 33.0
44+
*/
45+
public function seconds(): int;
46+
47+
/**
48+
* Returns the number of milliseconds of creation
49+
*
50+
* @since 33.0
51+
*/
52+
public function milliseconds(): int;
53+
54+
/**
55+
* Returns full millisecond creation timestamp
56+
*
57+
* @since 33.0
58+
*/
59+
public function createdAt(): float;
60+
61+
/**
62+
* Returns server ID (encoded on 9 bits)
63+
*
64+
* @return int<0, 511>
65+
* @since 33.0
66+
*/
67+
public function serverId(): int;
68+
69+
/**
70+
* Returns sequence ID (encoded on 12 bits)
71+
*
72+
* @return int<0, 4095>
73+
* @since 33.0
74+
*/
75+
public function sequenceId(): int;
76+
}
Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-only
8+
*/
9+
10+
namespace OCP;
11+
12+
/**
13+
* Nextcloud ID generator
14+
*
15+
* Generates unique ID
16+
* @since 33.0.0
17+
*/
18+
interface ISnowflakeIdGenerator {
19+
public function __invoke(): int|float;
20+
}

0 commit comments

Comments
 (0)