chore: benchmark bls signature deserialization

packages/beacon-node/src/chain/bls/multithread/index.ts

@@ -391,7 +391,7 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier {
         try {
           // Note: This can throw, must be handled per-job.
           // Pubkey and signature aggregation is defered here
-          workReq = jobItemWorkReq(job, this.format);
+          workReq = jobItemWorkReq(job, this.format, this.metrics);
         } catch (e) {
           this.metrics?.blsThreadPool.errorAggregateSignatureSetsCount.inc({type: job.type});
 

packages/beacon-node/src/chain/bls/multithread/jobItem.ts

@@ -4,6 +4,7 @@ import {ISignatureSet, SignatureSetType} from "@lodestar/state-transition";
 import {VerifySignatureOpts} from "../interface.js";
 import {getAggregatedPubkey} from "../utils.js";
 import {LinkedList} from "../../../util/array.js";
+import {Metrics} from "../../../metrics/metrics.js";
 import {BlsWorkReq} from "./types.js";
 
 export type JobQueueItem = JobQueueItemDefault | JobQueueItemSameMessage;
@@ -48,7 +49,7 @@ export function jobItemSigSets(job: JobQueueItem): number {
  * Prepare BlsWorkReq from JobQueueItem
  * WARNING: May throw with untrusted user input
  */
-export function jobItemWorkReq(job: JobQueueItem, format: PointFormat): BlsWorkReq {
+export function jobItemWorkReq(job: JobQueueItem, format: PointFormat, metrics: Metrics | null): BlsWorkReq {
   switch (job.type) {
     case JobQueueItemType.default:
       return {
@@ -60,20 +61,29 @@ export function jobItemWorkReq(job: JobQueueItem, format: PointFormat): BlsWorkR
           message: set.signingRoot,
         })),
       };
-    case JobQueueItemType.sameMessage:
+    case JobQueueItemType.sameMessage: {
+      // validate signature = true, this is slow code on main thread so should only run with network thread mode (useWorker=true)
+      // For a node subscribing to all subnets, with 1 signature per validator per epoch it takes around 80s
+      // to deserialize 750_000 signatures per epoch
+      // cpu profile on main thread has 250s idle so this only works until we reach 3M validators
+      // However, for normal node with only 2 to 7 subnet subscriptions per epoch this works until 27M validators
+      // and not a problem in the near future
+      // this is monitored on v1.11.0 https://github.com/ChainSafe/lodestar/pull/5912#issuecomment-1700320307
+      const timer = metrics?.blsThreadPool.signatureDeserializationMainThreadDuration.startTimer();
+      const signatures = job.sets.map((set) => bls.Signature.fromBytes(set.signature, CoordType.affine, true));
+      timer?.();
+
       return {
         opts: job.opts,
         sets: [
           {
             publicKey: bls.PublicKey.aggregate(job.sets.map((set) => set.publicKey)).toBytes(format),
-            signature: bls.Signature.aggregate(
-              // validate signature = true
-              job.sets.map((set) => bls.Signature.fromBytes(set.signature, CoordType.affine, true))
-            ).toBytes(format),
+            signature: bls.Signature.aggregate(signatures).toBytes(format),
             message: job.message,
           },
         ],
       };
+    }
   }
 }
 

packages/beacon-node/src/metrics/metrics/lodestar.ts

@@ -447,6 +447,10 @@ export function createLodestarMetrics(
         name: "lodestar_bls_thread_pool_batchable_sig_sets_total",
         help: "Count of total batchable signature sets",
       }),
+      signatureDeserializationMainThreadDuration: register.gauge({
+        name: "lodestar_bls_thread_pool_signature_deserialization_main_thread_time_seconds",
+        help: "Total time spent deserializing signatures on main thread",
+      }),
     },
 
     // BLS time on single thread mode

packages/beacon-node/test/perf/bls/bls.test.ts

@@ -77,6 +77,21 @@ describe("BLS ops", function () {
     });
   }
 
+  // this is total time we deserialize all signatures of validators per epoch
+  // ideally we want to track 700_000, 1_400_000, 2_100_000 validators but it takes too long
+  for (const numValidators of [10_000, 100_000]) {
+    const signatures = linspace(0, numValidators - 1).map((i) => getSet(i % 256).signature);
+    itBench({
+      id: `BLS deserializing ${numValidators} signatures`,
+      fn: () => {
+        for (const signature of signatures) {
+          // true = validate signature
+          bls.Signature.fromBytes(signature, CoordType.affine, true);
+        }
+      },
+    });
+  }
+
   // An aggregate and proof object has 3 signatures.
   // We may want to bundle up to 32 sets in a single batch.
   // TODO: figure out why it does not work with 256 or more