fix: storage cache progress bar (#1323)

farcasterxyz · Sep 2, 2023 · 7fd1f94 · 7fd1f94
1 parent 6c12fee
commit 7fd1f94
Show file tree

Hide file tree

Showing 3 changed files with 32 additions and 2 deletions.
diff --git a/.changeset/late-crabs-switch.md b/.changeset/late-crabs-switch.md
@@ -0,0 +1,5 @@
+---
+"@farcaster/hubble": patch
+---
+
+fix: Add progress bar for storage cache
diff --git a/apps/hubble/package.json b/apps/hubble/package.json
@@ -18,7 +18,8 @@
     "bench": "NODE_OPTIONS='--max-old-space-size=8192' tsx src/test/bench",
     "copy:rust": "mkdir -p build/addon && cp src/addon/index.node build/addon/index.node",
     "build:rust": "cargo-cp-artifact -ac addon src/addon/index.node -- cargo build --manifest-path ./src/addon/Cargo.toml --message-format=json-render-diagnostics --release",
-    "build": "yarn clean && yarn build:rust && tsc --project ./tsconfig.json && yarn copy:rust",
+    "build:ts": "tsc --project ./tsconfig.json",
+    "build": "yarn clean && yarn build:rust && yarn build:ts && yarn copy:rust",
     "clean": "rimraf ./build && cargo clean --manifest-path ./src/addon/Cargo.toml",
     "dev": "yarn start | yarn pino-pretty",
     "lint": "yarn lint:customjs && rome format src/ --write && rome check src/ --apply",

diff --git a/apps/hubble/src/storage/stores/storageCache.ts b/apps/hubble/src/storage/stores/storageCache.ts
@@ -14,11 +14,12 @@ import {
 } from "@farcaster/hub-nodejs";
 import { err, ok } from "neverthrow";
 import RocksDB from "../db/rocksdb.js";
-import { FID_BYTES, RootPrefix, UserMessagePostfix, UserMessagePostfixMax } from "../db/types.js";
+import { FID_BYTES, OnChainEventPostfix, RootPrefix, UserMessagePostfix, UserMessagePostfixMax } from "../db/types.js";
 import { logger } from "../../utils/logger.js";
 import { makeFidKey, makeMessagePrimaryKey, makeTsHash, typeToSetPostfix } from "../db/message.js";
 import { bytesCompare, getFarcasterTime, HubAsyncResult } from "@farcaster/core";
 import { forEachOnChainEvent } from "../db/onChainEvent.js";
+import { addProgressBar } from "../../utils/progressBars.js";
 
 const makeKey = (fid: number, set: UserMessagePostfix): string => {
   return Buffer.concat([makeFidKey(fid), Buffer.from([set])]).toString("hex");
@@ -50,19 +51,38 @@ export class StorageCache {
 
     const start = Date.now();
 
+    let totalFids = 0;
+
+    await this._db.forEachIteratorByPrefix(
+      Buffer.concat([Buffer.from([RootPrefix.OnChainEvent, OnChainEventPostfix.IdRegisterByFid])]),
+      async () => {
+        totalFids++;
+      },
+      { keys: false, values: false },
+    );
+
+    const progressBar = addProgressBar("Syncing storage cache", totalFids * 2);
+
+    let lastFid = 0;
     const prefix = Buffer.from([RootPrefix.User]);
     await this._db.forEachIteratorByPrefix(
       prefix,
       async (key) => {
         const postfix = (key as Buffer).readUint8(1 + FID_BYTES);
         if (postfix < UserMessagePostfixMax) {
           const lookupKey = (key as Buffer).subarray(1, 1 + FID_BYTES + 1).toString("hex");
+          const fid = (key as Buffer).subarray(1, 1 + FID_BYTES).readUInt32BE();
           const count = usage.get(lookupKey) ?? 0;
           if (this._earliestTsHashes.get(lookupKey) === undefined) {
             const tsHash = Uint8Array.from((key as Buffer).subarray(1 + FID_BYTES + 1));
             this._earliestTsHashes.set(lookupKey, tsHash);
           }
           usage.set(lookupKey, count + 1);
+
+          if (lastFid !== fid) {
+            progressBar?.increment();
+            lastFid = fid;
+          }
         }
       },
       { values: false },
@@ -84,10 +104,14 @@ export class StorageCache {
                 ? existingSlot?.invalidateAt ?? rentEventBody.expiry
                 : rentEventBody.expiry,
           });
+          progressBar?.increment();
         }
       });
     }
 
+    progressBar?.update(progressBar?.getTotal());
+    progressBar?.stop();
+
     this._counts = usage;
     this._earliestTsHashes = new Map();
     log.info({ timeTakenMs: Date.now() - start }, "storage cache synced");