WIP block processing

packages/beacon-node/src/chain/blocks/verifyBlocksStateTransitionOnly.ts

@@ -1,4 +1,10 @@
-import {CachedBeaconStateAllForks, stateTransition} from "@lodestar/state-transition";
+import {
+  CachedBeaconStateAllForks,
+  stateTransition,
+  BlockExternalData,
+  ExecutionPayloadStatus,
+  DataAvailableStatus,
+} from "@lodestar/state-transition";
 import {allForks} from "@lodestar/types";
 import {ErrorAborted, sleep} from "@lodestar/utils";
 import {IMetrics} from "../../metrics/index.js";
@@ -30,12 +36,19 @@ export async function verifyBlocksStateTransitionOnly(
     const block = blocks[i];
     const preState = i === 0 ? preState0 : postStates[i - 1];
 
+    const externalData: BlockExternalData = {
+      executionPayloadStatus: ExecutionPayloadStatus.valid,
+      // TODO EIP-4844: Actually validate data
+      dataAvailableStatus: DataAvailableStatus.available,
+    };
+
     // STFN - per_slot_processing() + per_block_processing()
     // NOTE: `regen.getPreState()` should have dialed forward the state already caching checkpoint states
     const useBlsBatchVerify = !opts?.disableBlsBatchVerify;
     const postState = stateTransition(
       preState,
       block,
+      externalData,
       {
         // false because it's verified below with better error typing
         verifyStateRoot: false,

packages/beacon-node/src/db/interface.ts

@@ -14,6 +14,8 @@ import {
   SyncCommitteeRepository,
   SyncCommitteeWitnessRepository,
   BackfilledRanges,
+  BlobsSidecarRepository,
+  BlobsSidecarArchiveRepository,
 } from "./repositories/index.js";
 import {PreGenesisState, PreGenesisStateLastProcessedBlock} from "./single/index.js";
 
@@ -25,9 +27,11 @@ import {PreGenesisState, PreGenesisStateLastProcessedBlock} from "./single/index
 export interface IBeaconDb {
   // unfinalized blocks
   block: BlockRepository;
+  blobsSidecar: BlobsSidecarRepository;
 
   // finalized blocks
   blockArchive: BlockArchiveRepository;
+  blobsSidecarArchive: BlobsSidecarArchiveRepository;
 
   // finalized states
   stateArchive: StateArchiveRepository;

packages/beacon-node/src/db/repositories/blobsSidecar.ts

@@ -0,0 +1,31 @@
+import {IChainForkConfig} from "@lodestar/config";
+import {Bucket, Db, Repository} from "@lodestar/db";
+import {allForks, eip4844, ssz} from "@lodestar/types";
+import {getSignedBlockTypeFromBytes} from "../../util/multifork.js";
+
+/**
+ * BlobsSidecar by block root (= hash_tree_root(SignedBeaconBlockAndBlobsSidecar.beacon_block.message))
+ *
+ * Used to store unfinalized BlobsSidecar
+ */
+export class BlobsSidecarRepository extends Repository<Uint8Array, eip4844.BlobsSidecar> {
+  constructor(config: IChainForkConfig, db: Db) {
+    const type = ssz.phase0.SignedBeaconBlock; // Pick some type but won't be used
+    super(config, db, Bucket.allForks_block, type);
+  }
+
+  /**
+   * Id is hashTreeRoot of unsigned BeaconBlock
+   */
+  getId(value: allForks.SignedBeaconBlock): Uint8Array {
+    return this.config.getForkTypes(value.message.slot).BeaconBlock.hashTreeRoot(value.message);
+  }
+
+  encodeValue(value: allForks.SignedBeaconBlock): Buffer {
+    return this.config.getForkTypes(value.message.slot).SignedBeaconBlock.serialize(value) as Buffer;
+  }
+
+  decodeValue(data: Buffer): allForks.SignedBeaconBlock {
+    return getSignedBlockTypeFromBytes(this.config, data).deserialize(data);
+  }
+}

packages/beacon-node/src/db/repositories/blobsSidecarArchive.ts

@@ -0,0 +1,162 @@
+import all from "it-all";
+import {IChainForkConfig} from "@lodestar/config";
+import {Db, Repository, IKeyValue, IFilterOptions, Bucket} from "@lodestar/db";
+import {Slot, Root, allForks, ssz, eip4844} from "@lodestar/types";
+import {bytesToInt} from "@lodestar/utils";
+import {getSignedBlockTypeFromBytes} from "../../util/multifork.js";
+import {getRootIndexKey, getParentRootIndexKey} from "./blockArchiveIndex.js";
+import {deleteParentRootIndex, deleteRootIndex, storeParentRootIndex, storeRootIndex} from "./blockArchiveIndex.js";
+
+export interface IBlockFilterOptions extends IFilterOptions<Slot> {
+  step?: number;
+}
+
+export type BlockArchiveBatchPutBinaryItem = IKeyValue<Slot, Uint8Array> & {
+  slot: Slot;
+  blockRoot: Root;
+  parentRoot: Root;
+};
+
+/**
+ * Stores finalized blocks. Block slot is identifier.
+ */
+export class BlobsSidecarArchiveRepository extends Repository<Slot, eip4844.BlobsSidecar> {
+  constructor(config: IChainForkConfig, db: Db) {
+    const type = ssz.phase0.SignedBeaconBlock; // Pick some type but won't be used
+    super(config, db, Bucket.allForks_blockArchive, type);
+  }
+
+  // Overrides for multi-fork
+
+  encodeValue(value: allForks.SignedBeaconBlock): Uint8Array {
+    return this.config.getForkTypes(value.message.slot).SignedBeaconBlock.serialize(value) as Uint8Array;
+  }
+
+  decodeValue(data: Uint8Array): allForks.SignedBeaconBlock {
+    return getSignedBlockTypeFromBytes(this.config, data).deserialize(data);
+  }
+
+  // Handle key as slot
+
+  getId(value: allForks.SignedBeaconBlock): Slot {
+    return value.message.slot;
+  }
+
+  decodeKey(data: Uint8Array): number {
+    return bytesToInt((super.decodeKey(data) as unknown) as Uint8Array, "be");
+  }
+
+  // Overrides to index
+
+  async put(key: Slot, value: allForks.SignedBeaconBlock): Promise<void> {
+    const blockRoot = this.config.getForkTypes(value.message.slot).BeaconBlock.hashTreeRoot(value.message);
+    const slot = value.message.slot;
+    await Promise.all([
+      super.put(key, value),
+      storeRootIndex(this.db, slot, blockRoot),
+      storeParentRootIndex(this.db, slot, value.message.parentRoot),
+    ]);
+  }
+
+  async batchPut(items: IKeyValue<Slot, allForks.SignedBeaconBlock>[]): Promise<void> {
+    await Promise.all([
+      super.batchPut(items),
+      Array.from(items).map((item) => {
+        const slot = item.value.message.slot;
+        const blockRoot = this.config.getForkTypes(slot).BeaconBlock.hashTreeRoot(item.value.message);
+        return storeRootIndex(this.db, slot, blockRoot);
+      }),
+      Array.from(items).map((item) => {
+        const slot = item.value.message.slot;
+        const parentRoot = item.value.message.parentRoot;
+        return storeParentRootIndex(this.db, slot, parentRoot);
+      }),
+    ]);
+  }
+
+  async batchPutBinary(items: BlockArchiveBatchPutBinaryItem[]): Promise<void> {
+    await Promise.all([
+      super.batchPutBinary(items),
+      Array.from(items).map((item) => storeRootIndex(this.db, item.slot, item.blockRoot)),
+      Array.from(items).map((item) => storeParentRootIndex(this.db, item.slot, item.parentRoot)),
+    ]);
+  }
+
+  async remove(value: allForks.SignedBeaconBlock): Promise<void> {
+    await Promise.all([
+      super.remove(value),
+      deleteRootIndex(this.db, this.config.getForkTypes(value.message.slot).SignedBeaconBlock, value),
+      deleteParentRootIndex(this.db, value),
+    ]);
+  }
+
+  async batchRemove(values: allForks.SignedBeaconBlock[]): Promise<void> {
+    await Promise.all([
+      super.batchRemove(values),
+      Array.from(values).map((value) =>
+        deleteRootIndex(this.db, this.config.getForkTypes(value.message.slot).SignedBeaconBlock, value)
+      ),
+      Array.from(values).map((value) => deleteParentRootIndex(this.db, value)),
+    ]);
+  }
+
+  async *valuesStream(opts?: IBlockFilterOptions): AsyncIterable<allForks.SignedBeaconBlock> {
+    const firstSlot = this.getFirstSlot(opts);
+    const valuesStream = super.valuesStream(opts);
+    const step = (opts && opts.step) || 1;
+
+    for await (const value of valuesStream) {
+      if ((value.message.slot - firstSlot) % step === 0) {
+        yield value;
+      }
+    }
+  }
+
+  async values(opts?: IBlockFilterOptions): Promise<allForks.SignedBeaconBlock[]> {
+    return all(this.valuesStream(opts));
+  }
+
+  // INDEX
+
+  async getByRoot(root: Root): Promise<allForks.SignedBeaconBlock | null> {
+    const slot = await this.getSlotByRoot(root);
+    return slot !== null ? await this.get(slot) : null;
+  }
+
+  async getBinaryEntryByRoot(root: Root): Promise<IKeyValue<Slot, Buffer> | null> {
+    const slot = await this.getSlotByRoot(root);
+    return slot !== null ? ({key: slot, value: await this.getBinary(slot)} as IKeyValue<Slot, Buffer>) : null;
+  }
+
+  async getByParentRoot(root: Root): Promise<allForks.SignedBeaconBlock | null> {
+    const slot = await this.getSlotByParentRoot(root);
+    return slot !== null ? await this.get(slot) : null;
+  }
+
+  async getSlotByRoot(root: Root): Promise<Slot | null> {
+    return this.parseSlot(await this.db.get(getRootIndexKey(root)));
+  }
+
+  async getSlotByParentRoot(root: Root): Promise<Slot | null> {
+    return this.parseSlot(await this.db.get(getParentRootIndexKey(root)));
+  }
+
+  private parseSlot(slotBytes: Uint8Array | null): Slot | null {
+    if (!slotBytes) return null;
+    const slot = bytesToInt(slotBytes, "be");
+    // TODO: Is this necessary? How can bytesToInt return a non-integer?
+    return Number.isInteger(slot) ? slot : null;
+  }
+
+  private getFirstSlot(opts?: IBlockFilterOptions): Slot {
+    const dbFilterOpts = this.dbFilterOptions(opts);
+    const firstSlot = dbFilterOpts.gt
+      ? this.decodeKey(dbFilterOpts.gt) + 1
+      : dbFilterOpts.gte
+      ? this.decodeKey(dbFilterOpts.gte)
+      : null;
+    if (firstSlot === null) throw Error("specify opts.gt or opts.gte");
+
+    return firstSlot;
+  }
+}

packages/beacon-node/src/db/repositories/index.ts

@@ -1,4 +1,6 @@
 export {BlockRepository} from "./block.js";
+export {BlobsSidecarRepository} from "./blobsSidecar.js";
+export {BlobsSidecarArchiveRepository} from "./blobsSidecarArchive.js";
 export {BlockArchiveBatchPutBinaryItem, BlockArchiveRepository, IBlockFilterOptions} from "./blockArchive.js";
 export {StateArchiveRepository} from "./stateArchive.js";
 

packages/beacon-node/src/network/reqresp/handlers/beaconBlockAndBlobsSidecarByRoot.ts

@@ -0,0 +1,70 @@
+import {eip4844} from "@lodestar/types";
+import {toHex} from "@lodestar/utils";
+import {IBeaconChain} from "../../../chain/index.js";
+import {IBeaconDb} from "../../../db/index.js";
+import {getSlotFromBytes} from "../../../util/multifork.js";
+import {ReqRespBlockResponse} from "../types.js";
+
+export async function* onBeaconBlockAndBlobsSidecarByRoot(
+  requestBody: eip4844.BeaconBlockAndBlobsSidecarByRootRequest,
+  chain: IBeaconChain,
+  db: IBeaconDb
+): AsyncIterable<ReqRespBlockResponse> {
+  const finalizedSlot = chain.forkChoice.getFinalizedBlock().slot;
+
+  for (const blockRoot of requestBody) {
+    const blockRootHex = toHex(blockRoot);
+    const summary = chain.forkChoice.getBlockHex(blockRootHex);
+
+    // NOTE: Only support non-finalized blocks.
+    // SPEC: Clients MUST support requesting blocks and sidecars since the latest finalized epoch.
+    // https://github.com/ethereum/consensus-specs/blob/11a037fd9227e29ee809c9397b09f8cc3383a8c0/specs/eip4844/p2p-interface.md#beaconblockandblobssidecarbyroot-v1
+    if (!summary || summary.slot <= finalizedSlot) {
+      // TODO: Should accept the finalized block? Is the finalized block in the archive DB or hot DB?
+      continue;
+    }
+
+    // finalized block has summary in forkchoice but it stays in blockArchive db
+    const blockBytes = await db.block.getBinary(blockRoot);
+    if (!blockBytes) {
+      throw Error(`Inconsistent state, block known to fork-choice not in db ${blockRootHex}`);
+    }
+
+    const blobsSidecarBytes = await db.blobsSidecar.getBinary(blockRoot);
+    if (!blobsSidecarBytes) {
+      throw Error(`Inconsistent state, blobsSidecar known to fork-choice not in db ${blockRootHex}`);
+    }
+
+    yield {
+      bytes: signedBeaconBlockAndBlobsSidecarFromBytes(blockBytes, blobsSidecarBytes),
+      slot: getSlotFromBytes(blockBytes),
+    };
+  }
+}
+
+/**
+ * Construct a valid SSZ serialized container from its properties also serialized.
+ * ```
+ * class SignedBeaconBlockAndBlobsSidecar(Container):
+ *   beacon_block: SignedBeaconBlock
+ *   blobs_sidecar: BlobsSidecar
+ * ```
+ */
+function signedBeaconBlockAndBlobsSidecarFromBytes(blockBytes: Uint8Array, blobsSidecarBytes: Uint8Array): Uint8Array {
+  const totalLen = 4 + 4 + blockBytes.length + blobsSidecarBytes.length;
+  const arrayBuffer = new ArrayBuffer(totalLen);
+  const dataView = new DataView(arrayBuffer);
+  const uint8Array = new Uint8Array(arrayBuffer);
+
+  const blockOffset = 8;
+  const blobsOffset = 8 + blockBytes.length;
+
+  // Write offsets
+  dataView.setUint32(0, blockOffset);
+  dataView.setUint32(4, blobsOffset);
+
+  uint8Array.set(blockBytes, blockOffset);
+  uint8Array.set(blobsSidecarBytes, blobsOffset);
+
+  return uint8Array;
+}