chore: Switch to rocksdb fork with precompiled Linux ARM build

.changeset/strange-otters-join.md

@@ -0,0 +1,5 @@
+---
+'@farcaster/hubble': patch
+---
+
+Switch from `rocksdb` to `@farcaster/rocksdb` NPM package

apps/hubble/package.json

@@ -50,6 +50,7 @@
  "@chainsafe/libp2p-gossipsub": "6.1.0",
  "@chainsafe/libp2p-noise": "^11.0.0 ",
  "@farcaster/hub-nodejs": "^0.7.2",
+ "@farcaster/rocksdb": "^5.5.0",
  "@grpc/grpc-js": "~1.8.7",
  "@libp2p/interface-connection": "^3.0.2",
  "@libp2p/interface-peer-id": "^2.0.0",
@@ -68,7 +69,6 @@
  "node-cron": "~3.0.2",
  "pino": "~8.11.0",
  "rate-limiter-flexible": "^2.4.1",
- "rocksdb": "~5.2.1",
  "rwlock": "~5.0.0",
  "tiny-typed-emitter": "~2.1.0",
  "tsx": "~3.12.5"

apps/hubble/src/rocksdb.d.ts

@@ -1,5 +1,5 @@
-// Type definitions for rocksdb 3.0
-// Project: https://github.com/Level/rocksdb
+// Type definitions for @farcaster/rocksdb 3.0
+// Project: https://github.com/farcasterxyz/rocksdb
 // Definitions by: Meirion Hughes <https://github.com/MeirionHughes>
 // Daniel Byrne <https://github.com/danwbyrne>
 // Paul Fletcher-Hill <https://github.com/pfletcherhill>
@@ -8,87 +8,92 @@
 
 /// <reference types="node" />
 
-import {
- AbstractBatch,
- AbstractChainedBatch,
- AbstractGetOptions,
- AbstractIterator,
- AbstractIteratorOptions,
- AbstractLevelDOWN,
- AbstractOpenOptions,
- AbstractOptions,
- ErrorCallback,
- ErrorValueCallback,
-} from 'abstract-leveldown';
-
-interface RocksDB extends AbstractLevelDOWN<RocksDB.Bytes, RocksDB.Bytes> {
- open(cb: ErrorCallback): void;
- open(options: RocksDB.OpenOptions, cb: ErrorCallback): void;
-
- get(key: RocksDB.Bytes, cb: ErrorValueCallback<RocksDB.Bytes>): void;
- get(key: RocksDB.Bytes, options: RocksDB.GetOptions, cb: ErrorValueCallback<RocksDB.Bytes>): void;
-
- put(key: RocksDB.Bytes, value: RocksDB.Bytes, cb: ErrorCallback): void;
- put(key: RocksDB.Bytes, value: RocksDB.Bytes, options: RocksDB.PutOptions, cb: ErrorCallback): void;
-
- del(key: RocksDB.Bytes, cb: ErrorCallback): void;
- del(key: RocksDB.Bytes, options: RocksDB.DelOptions, cb: ErrorCallback): void;
-
- batch(): AbstractChainedBatch<RocksDB.Bytes, RocksDB.Bytes>;
- batch(array: AbstractBatch[], cb: ErrorCallback): AbstractChainedBatch<RocksDB.Bytes, RocksDB.Bytes>;
- batch(
- array: AbstractBatch[],
- options: RocksDB.BatchOptions,
- cb: ErrorCallback
- ): AbstractChainedBatch<RocksDB.Bytes, RocksDB.Bytes>;
-
- approximateSize(start: RocksDB.Bytes, end: RocksDB.Bytes, cb: RocksDB.ErrorSizeCallback): void;
- compactRange(start: RocksDB.Bytes, end: RocksDB.Bytes, cb: ErrorCallback): void;
- getProperty(property: string): string;
- iterator(options?: RocksDB.IteratorOptions): RocksDB.Iterator;
-}
-
-declare namespace RocksDB {
- type Bytes = string | Buffer;
- type ErrorSizeCallback = (err: Error | undefined, size: number) => void;
- // eslint-disable-next-line @typescript-eslint/no-empty-interface
- interface OpenOptions extends AbstractOpenOptions {}
-
- interface GetOptions extends AbstractGetOptions {
- fillCache?: boolean | undefined;
- }
-
- interface PutOptions extends AbstractOptions {
- sync?: boolean | undefined;
- }
-
- interface DelOptions extends AbstractOptions {
- sync?: boolean | undefined;
+// If we ever start using the upstream rocksdb package again, remove this
+// wrapper `module` declaration. It's only needed because our forked package is
+// scoped to @farcaster.
+declare module '@farcaster/rocksdb' {
+ import {
+ AbstractBatch,
+ AbstractChainedBatch,
+ AbstractGetOptions,
+ AbstractIterator,
+ AbstractIteratorOptions,
+ AbstractLevelDOWN,
+ AbstractOpenOptions,
+ AbstractOptions,
+ ErrorCallback,
+ ErrorValueCallback,
+ } from 'abstract-leveldown';
+
+ interface RocksDB extends AbstractLevelDOWN<RocksDB.Bytes, RocksDB.Bytes> {
+ open(cb: ErrorCallback): void;
+ open(options: RocksDB.OpenOptions, cb: ErrorCallback): void;
+
+ get(key: RocksDB.Bytes, cb: ErrorValueCallback<RocksDB.Bytes>): void;
+ get(key: RocksDB.Bytes, options: RocksDB.GetOptions, cb: ErrorValueCallback<RocksDB.Bytes>): void;
+
+ put(key: RocksDB.Bytes, value: RocksDB.Bytes, cb: ErrorCallback): void;
+ put(key: RocksDB.Bytes, value: RocksDB.Bytes, options: RocksDB.PutOptions, cb: ErrorCallback): void;
+
+ del(key: RocksDB.Bytes, cb: ErrorCallback): void;
+ del(key: RocksDB.Bytes, options: RocksDB.DelOptions, cb: ErrorCallback): void;
+
+ batch(): AbstractChainedBatch<RocksDB.Bytes, RocksDB.Bytes>;
+ batch(array: AbstractBatch[], cb: ErrorCallback): AbstractChainedBatch<RocksDB.Bytes, RocksDB.Bytes>;
+ batch(
+ array: AbstractBatch[],
+ options: RocksDB.BatchOptions,
+ cb: ErrorCallback
+ ): AbstractChainedBatch<RocksDB.Bytes, RocksDB.Bytes>;
+
+ approximateSize(start: RocksDB.Bytes, end: RocksDB.Bytes, cb: RocksDB.ErrorSizeCallback): void;
+ compactRange(start: RocksDB.Bytes, end: RocksDB.Bytes, cb: ErrorCallback): void;
+ getProperty(property: string): string;
+ iterator(options?: RocksDB.IteratorOptions): RocksDB.Iterator;
  }
 
- interface BatchOptions extends AbstractOptions {
- sync?: boolean | undefined;
+ declare namespace RocksDB {
+ type Bytes = string | Buffer;
+ type ErrorSizeCallback = (err: Error | undefined, size: number) => void;
+ // eslint-disable-next-line @typescript-eslint/no-empty-interface
+ interface OpenOptions extends AbstractOpenOptions {}
+
+ interface GetOptions extends AbstractGetOptions {
+ fillCache?: boolean | undefined;
+ }
+
+ interface PutOptions extends AbstractOptions {
+ sync?: boolean | undefined;
+ }
+
+ interface DelOptions extends AbstractOptions {
+ sync?: boolean | undefined;
+ }
+
+ interface BatchOptions extends AbstractOptions {
+ sync?: boolean | undefined;
+ }
+
+ interface IteratorOptions extends AbstractIteratorOptions<Bytes> {
+ fillCache?: boolean | undefined;
+ }
+
+ interface Iterator extends AbstractIterator<Bytes, Bytes> {
+ seek(key: Bytes): void;
+ binding: any;
+ cache: any;
+ finished: any;
+ fastFuture: any;
+ }
+
+ interface Constructor {
+ new (location: string): RocksDB;
+ (location: string): RocksDB;
+ destroy(location: string, cb: ErrorCallback): void;
+ repair(location: string, cb: ErrorCallback): void;
+ }
  }
 
- interface IteratorOptions extends AbstractIteratorOptions<Bytes> {
- fillCache?: boolean | undefined;
- }
-
- interface Iterator extends AbstractIterator<Bytes, Bytes> {
- seek(key: Bytes): void;
- binding: any;
- cache: any;
- finished: any;
- fastFuture: any;
- }
-
- interface Constructor {
- new (location: string): RocksDB;
- (location: string): RocksDB;
- destroy(location: string, cb: ErrorCallback): void;
- repair(location: string, cb: ErrorCallback): void;
- }
+ declare const RocksDB: RocksDB.Constructor;
+ export default RocksDB;
 }
-
-declare const RocksDB: RocksDB.Constructor;
-export default RocksDB;

apps/hubble/src/storage/db/rocksdb.ts

@@ -1,7 +1,7 @@
 import { bytesIncrement, HubError, isHubError } from '@farcaster/hub-nodejs';
 import { AbstractBatch, AbstractChainedBatch, AbstractIterator } from 'abstract-leveldown';
 import { mkdir } from 'fs';
-import AbstractRocksDB from 'rocksdb';
+import AbstractRocksDB from '@farcaster/rocksdb';
 
 export const DB_DIRECTORY = '.rocks';
 const DB_NAME_DEFAULT = 'farcaster';

yarn.lock

@@ -1141,6 +1141,15 @@
  dependencies:
  lodash.mergewith "^4.6.2"
 
+"@farcaster/rocksdb@^5.5.0":
+ version "5.5.0"
+ resolved "https://registry.npmjs.org/@farcaster/rocksdb/-/rocksdb-5.5.0.tgz#c352480a7d6e08c2eee8e1690dc0a49e5a72c8fd"
+ integrity sha512-E5xavqAOrS9yXV2zw2lEyv7j4w1jZd0UNtsi5C9tqk5Kw9voCojA/zTgb5WugEqHAT6Bt/szahhnNuWxbo9NPg==
+ dependencies:
+ abstract-leveldown "^7.2.0"
+ napi-macros "^2.0.0"
+ node-gyp-build "^4.3.0"
+
 "@grpc/grpc-js@^1.8.13":
  version "1.8.13"
  resolved "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.8.13.tgz#e775685962909b76f8d4b813833c3d123867165b"
@@ -6988,15 +6997,6 @@ rimraf@^4.1.2:
  dependencies:
  glob "^9.2.0"
 
-rocksdb@~5.2.1:
- version "5.2.1"
- resolved "https://registry.npmjs.org/rocksdb/-/rocksdb-5.2.1.tgz#40fd54b798f8843290521313462f4347b702e60d"
- integrity sha512-SN9RRLfqDEGfZ6MgfjpQ4DIn3+TdN5ECKYHgeuPKFy6Yw0I9hYyE46giLidngjZU47JXUmWXJDZuU1CyEcf4Fw==
- dependencies:
- abstract-leveldown "^7.2.0"
- napi-macros "^2.0.0"
- node-gyp-build "^4.3.0"
-
 rollup@^3.2.5:
  version "3.19.1"
  resolved "https://registry.npmjs.org/rollup/-/rollup-3.19.1.tgz#2b3a31ac1ff9f3afab2e523fa687fef5b0ee20fc"