diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index d16c6a0951e24..2f031d2fc117d 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -2110,8 +2110,11 @@ 'OC\\Share\\Constants' => $baseDir . '/lib/private/Share/Constants.php', 'OC\\Share\\Helper' => $baseDir . '/lib/private/Share/Helper.php', 'OC\\Share\\Share' => $baseDir . '/lib/private/Share/Share.php', + 'OC\\Snowflake\\APCuSequence' => $baseDir . '/lib/private/Snowflake/APCuSequence.php', 'OC\\Snowflake\\Decoder' => $baseDir . '/lib/private/Snowflake/Decoder.php', + 'OC\\Snowflake\\FileSequence' => $baseDir . '/lib/private/Snowflake/FileSequence.php', 'OC\\Snowflake\\Generator' => $baseDir . '/lib/private/Snowflake/Generator.php', + 'OC\\Snowflake\\ISequence' => $baseDir . '/lib/private/Snowflake/ISequence.php', 'OC\\SpeechToText\\SpeechToTextManager' => $baseDir . '/lib/private/SpeechToText/SpeechToTextManager.php', 'OC\\SpeechToText\\TranscriptionJob' => $baseDir . '/lib/private/SpeechToText/TranscriptionJob.php', 'OC\\StreamImage' => $baseDir . '/lib/private/StreamImage.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index 4fd57cdbe78ba..a09b068bec478 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -21,10 +21,6 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 array ( 'NCU\\' => 4, ), - 'B' => - array ( - 'Bamarni\\Composer\\Bin\\' => 21, - ), ); public static $prefixDirsPsr4 = array ( @@ -44,10 +40,6 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 array ( 0 => __DIR__ . '/../../..' . '/lib/unstable', ), - 'Bamarni\\Composer\\Bin\\' => - array ( - 0 => __DIR__ . '/..' . '/bamarni/composer-bin-plugin/src', - ), ); public static $fallbackDirsPsr4 = array ( @@ -2159,8 +2151,11 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Share\\Constants' => __DIR__ . '/../../..' . '/lib/private/Share/Constants.php', 'OC\\Share\\Helper' => __DIR__ . '/../../..' . '/lib/private/Share/Helper.php', 'OC\\Share\\Share' => __DIR__ . '/../../..' . '/lib/private/Share/Share.php', + 'OC\\Snowflake\\APCuSequence' => __DIR__ . '/../../..' . '/lib/private/Snowflake/APCuSequence.php', 'OC\\Snowflake\\Decoder' => __DIR__ . '/../../..' . '/lib/private/Snowflake/Decoder.php', + 'OC\\Snowflake\\FileSequence' => __DIR__ . '/../../..' . '/lib/private/Snowflake/FileSequence.php', 'OC\\Snowflake\\Generator' => __DIR__ . '/../../..' . '/lib/private/Snowflake/Generator.php', + 'OC\\Snowflake\\ISequence' => __DIR__ . '/../../..' . '/lib/private/Snowflake/ISequence.php', 'OC\\SpeechToText\\SpeechToTextManager' => __DIR__ . '/../../..' . '/lib/private/SpeechToText/SpeechToTextManager.php', 'OC\\SpeechToText\\TranscriptionJob' => __DIR__ . '/../../..' . '/lib/private/SpeechToText/TranscriptionJob.php', 'OC\\StreamImage' => __DIR__ . '/../../..' . '/lib/private/StreamImage.php', diff --git a/lib/private/Server.php b/lib/private/Server.php index 01b6ec0e22b69..48b6f2cc88076 100644 --- a/lib/private/Server.php +++ b/lib/private/Server.php @@ -114,8 +114,11 @@ use OC\SetupCheck\SetupCheckManager; use OC\Share20\ProviderFactory; use OC\Share20\ShareHelper; +use OC\Snowflake\APCuSequence; use OC\Snowflake\Decoder; +use OC\Snowflake\FileSequence; use OC\Snowflake\Generator; +use OC\Snowflake\ISequence; use OC\SpeechToText\SpeechToTextManager; use OC\SystemTag\ManagerFactory as SystemTagManagerFactory; use OC\Talk\Broker; @@ -1250,6 +1253,16 @@ public function __construct($webRoot, \OC\Config $config) { $this->registerAlias(ISignatureManager::class, SignatureManager::class); $this->registerAlias(IGenerator::class, Generator::class); + $this->registerService(ISequence::class, function (ContainerInterface $c): ISequence { + if (PHP_SAPI !== 'cli') { + $sequence = $c->get(APCuSequence::class); + if ($sequence->isAvailable()) { + return $sequence; + } + } + + return $c->get(FileSequence::class); + }, false); $this->registerAlias(IDecoder::class, Decoder::class); $this->connectDispatcher(); diff --git a/lib/private/Snowflake/APCuSequence.php b/lib/private/Snowflake/APCuSequence.php new file mode 100644 index 0000000000000..5f018c2a804a9 --- /dev/null +++ b/lib/private/Snowflake/APCuSequence.php @@ -0,0 +1,36 @@ +workDir = $tempManager->getTemporaryFolder('.snowflakes'); + } + + #[Override] + public function isAvailable(): bool { + return true; + } + + #[Override] + public function nextId(int $serverId, int $seconds, int $milliseconds): int { + // Open lock file + $filePath = $this->getFilePath($milliseconds % self::NB_FILES); + $fp = fopen($filePath, 'c+'); + if ($fp === false) { + throw new \Exception('Unable to open sequence ID file: ' . $filePath); + } + if (!flock($fp, LOCK_EX)) { + throw new \Exception('Unable to acquire lock on sequence ID file: ' . $filePath); + } + + // Read content + $content = (string)fgets($fp); + $locks = $content === '' + ? [] + : json_decode($content, true, 3, JSON_THROW_ON_ERROR); + + // Generate new ID + if (isset($locks[$seconds])) { + if (isset($locks[$seconds][$milliseconds])) { + ++$locks[$seconds][$milliseconds]; + } else { + $locks[$seconds][$milliseconds] = 0; + } + } else { + $locks[$seconds] = [ + $milliseconds => 0 + ]; + } + + // Clean old sequence IDs + $cleanBefore = $seconds - self::SEQUENCE_TTL; + $locks = array_filter($locks, static function ($key) use ($cleanBefore) { + return $key >= $cleanBefore; + }, ARRAY_FILTER_USE_KEY); + + // Write data + ftruncate($fp, 0); + $content = json_encode($locks, JSON_THROW_ON_ERROR); + rewind($fp); + fwrite($fp, $content); + fsync($fp); + + // Release lock + fclose($fp); + + return $locks[$seconds][$milliseconds]; + } + + private function getFilePath(int $fileId): string { + return $this->workDir . sprintf(self::LOCK_FILE_FORMAT, $fileId); + } +} diff --git a/lib/private/Snowflake/Generator.php b/lib/private/Snowflake/Generator.php index f378482a315dd..7f4c91e105e86 100644 --- a/lib/private/Snowflake/Generator.php +++ b/lib/private/Snowflake/Generator.php @@ -23,17 +23,18 @@ final class Generator implements IGenerator { public function __construct( private readonly ITimeFactory $timeFactory, + private readonly ISequence $sequenceGenerator, ) { } #[Override] public function nextId(): string { - // Time related + // Relative time [$seconds, $milliseconds] = $this->getCurrentTime(); $serverId = $this->getServerId() & 0x1FF; // Keep 9 bits $isCli = (int)$this->isCli(); // 1 bit - $sequenceId = $this->getSequenceId($seconds, $milliseconds, $serverId); // 12 bits + $sequenceId = $this->sequenceGenerator->nextId($seconds, $milliseconds, $serverId); // 12 bits if ($sequenceId > 0xFFF || $sequenceId === false) { // Throttle a bit, wait for next millisecond usleep(1000); @@ -106,33 +107,4 @@ private function getServerId(): int { private function isCli(): bool { return PHP_SAPI === 'cli'; } - - /** - * Generates sequence ID from APCu (general case) or random if APCu disabled or CLI - * - * @return int|false Sequence ID or false if APCu not ready - * @throws \Exception if there is an error with APCu - */ - private function getSequenceId(int $seconds, int $milliseconds, int $serverId): int|false { - $key = 'seq:' . $seconds . ':' . $milliseconds; - - // Use APCu as fastest local cache, but not shared between processes in CLI - if (!$this->isCli() && function_exists('apcu_enabled') && apcu_enabled()) { - if ((int)apcu_cache_info(true)['creation_time'] === $seconds) { - // APCu cache was just started - // It means a sequence was maybe deleted - return false; - } - - $sequenceId = apcu_inc($key, success: $success, ttl: 1); - if ($success === true) { - return $sequenceId; - } - - throw new \Exception('Failed to generate SnowflakeId with APCu'); - } - - // Otherwise, just return a random number - return random_int(0, 0xFFF - 1); - } } diff --git a/lib/private/Snowflake/ISequence.php b/lib/private/Snowflake/ISequence.php new file mode 100644 index 0000000000000..e5e1f6e414d32 --- /dev/null +++ b/lib/private/Snowflake/ISequence.php @@ -0,0 +1,25 @@ +sequence = new APCuSequence(); + } +} diff --git a/tests/lib/Snowflake/FileSequenceTest.php b/tests/lib/Snowflake/FileSequenceTest.php new file mode 100644 index 0000000000000..82d2f9fefbeca --- /dev/null +++ b/tests/lib/Snowflake/FileSequenceTest.php @@ -0,0 +1,35 @@ +createMock(ITempManager::class); + $this->path = uniqid(sys_get_temp_dir() . '/php_test_seq_', true); + mkdir($this->path); + $tempManager->method('getTemporaryFolder')->willReturn($this->path); + $this->sequence = new FileSequence($tempManager); + } + + public function tearDown():void { + foreach (glob($this->path . '/*') as $file) { + unlink($file); + } + rmdir($this->path); + } +} diff --git a/tests/lib/Snowflake/GeneratorTest.php b/tests/lib/Snowflake/GeneratorTest.php index 748d0f190422a..1e2369ee5c98a 100644 --- a/tests/lib/Snowflake/GeneratorTest.php +++ b/tests/lib/Snowflake/GeneratorTest.php @@ -12,9 +12,11 @@ use OC\AppFramework\Utility\TimeFactory; use OC\Snowflake\Decoder; use OC\Snowflake\Generator; +use OC\Snowflake\ISequence; use OCP\AppFramework\Utility\ITimeFactory; use OCP\Snowflake\IGenerator; use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\MockObject\MockObject; use Test\TestCase; /** @@ -22,12 +24,17 @@ */ class GeneratorTest extends TestCase { private Decoder $decoder; + private ISequence&MockObject $sequence; public function setUp():void { $this->decoder = new Decoder(); + $this->sequence = $this->createMock(ISequence::class); + $this->sequence->method('isAvailable')->willReturn(true); + $this->sequence->method('nextId')->willReturn(421); } + public function testGenerator(): void { - $generator = new Generator(new TimeFactory()); + $generator = new Generator(new TimeFactory(), $this->sequence); $snowflakeId = $generator->nextId(); $data = $this->decoder->decode($generator->nextId()); @@ -53,7 +60,7 @@ public function testGeneratorWithFixedTime(string $date, int $expectedSeconds, i $timeFactory = $this->createMock(ITimeFactory::class); $timeFactory->method('now')->willReturn($dt); - $generator = new Generator($timeFactory); + $generator = new Generator($timeFactory, $this->sequence); $data = $this->decoder->decode($generator->nextId()); $this->assertEquals($expectedSeconds, ($data['createdAt']->format('U') - IGenerator::TS_OFFSET)); diff --git a/tests/lib/Snowflake/ISequenceBase.php b/tests/lib/Snowflake/ISequenceBase.php new file mode 100644 index 0000000000000..bff37ca2d6d0b --- /dev/null +++ b/tests/lib/Snowflake/ISequenceBase.php @@ -0,0 +1,45 @@ +sequence->isAvailable()) { + $this->markTestSkipped('Sequence ID generator ' . get_class($this->sequence) . 'is’nt available. Skip'); + } + + $nb = 1000; + $ids = []; + $server = 42; + for ($i = 0; $i < $nb; ++$i) { + $time = explode('.', (string)microtime(true)); + $seconds = (int)$time[0]; + $milliseconds = str_pad(substr($time[1] ?? '0', 0, 3), 3, '0'); + $id = $this->sequence->nextId($server, $seconds, (int)$milliseconds); + $ids[] = sprintf('%d_%s_%d', $seconds, $milliseconds, $id); + usleep(100); + } + + // Is it unique? + $this->assertCount($nb, array_unique($ids)); + // Is it sequential? + $sortedIds = $ids; + natsort($sortedIds); + $this->assertSame($sortedIds, $ids); + } +}