From d2688155eb9849e588ee40f67da39eb095e132e4 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Wed, 16 Nov 2022 18:57:22 +0100 Subject: [PATCH] WIP block processing --- .../blocks/verifyBlocksStateTransitionOnly.ts | 15 +- packages/beacon-node/src/db/interface.ts | 4 + .../src/db/repositories/blobsSidecar.ts | 31 ++++ .../db/repositories/blobsSidecarArchive.ts | 162 ++++++++++++++++++ .../beacon-node/src/db/repositories/index.ts | 2 + .../beaconBlockAndBlobsSidecarByRoot.ts | 70 ++++++++ .../reqresp/handlers/blobsSidecarsByRange.ts | 76 ++++++++ .../src/network/reqresp/handlers/index.ts | 14 +- .../src/network/reqresp/interface.ts | 15 +- .../src/network/reqresp/reqResp.ts | 34 +++- .../beacon-node/src/network/reqresp/types.ts | 28 ++- .../src/block/externalData.ts | 16 ++ packages/state-transition/src/block/index.ts | 20 ++- .../src/block/processExecutionPayload.ts | 17 +- packages/state-transition/src/index.ts | 1 + .../state-transition/src/stateTransition.ts | 4 +- packages/types/src/eip4844/sszTypes.ts | 31 +++- packages/types/src/eip4844/types.ts | 3 + 18 files changed, 519 insertions(+), 24 deletions(-) create mode 100644 packages/beacon-node/src/db/repositories/blobsSidecar.ts create mode 100644 packages/beacon-node/src/db/repositories/blobsSidecarArchive.ts create mode 100644 packages/beacon-node/src/network/reqresp/handlers/beaconBlockAndBlobsSidecarByRoot.ts create mode 100644 packages/beacon-node/src/network/reqresp/handlers/blobsSidecarsByRange.ts create mode 100644 packages/state-transition/src/block/externalData.ts diff --git a/packages/beacon-node/src/chain/blocks/verifyBlocksStateTransitionOnly.ts b/packages/beacon-node/src/chain/blocks/verifyBlocksStateTransitionOnly.ts index e1c6b2031df8..c8d42963490a 100644 --- a/packages/beacon-node/src/chain/blocks/verifyBlocksStateTransitionOnly.ts +++ b/packages/beacon-node/src/chain/blocks/verifyBlocksStateTransitionOnly.ts @@ -1,4 +1,10 @@ -import {CachedBeaconStateAllForks, stateTransition} from "@lodestar/state-transition"; +import { + CachedBeaconStateAllForks, + stateTransition, + BlockExternalData, + ExecutionPayloadStatus, + DataAvailableStatus, +} from "@lodestar/state-transition"; import {allForks} from "@lodestar/types"; import {ErrorAborted, sleep} from "@lodestar/utils"; import {IMetrics} from "../../metrics/index.js"; @@ -30,12 +36,19 @@ export async function verifyBlocksStateTransitionOnly( const block = blocks[i]; const preState = i === 0 ? preState0 : postStates[i - 1]; + const externalData: BlockExternalData = { + executionPayloadStatus: ExecutionPayloadStatus.valid, + // TODO EIP-4844: Actually validate data + dataAvailableStatus: DataAvailableStatus.available, + }; + // STFN - per_slot_processing() + per_block_processing() // NOTE: `regen.getPreState()` should have dialed forward the state already caching checkpoint states const useBlsBatchVerify = !opts?.disableBlsBatchVerify; const postState = stateTransition( preState, block, + externalData, { // false because it's verified below with better error typing verifyStateRoot: false, diff --git a/packages/beacon-node/src/db/interface.ts b/packages/beacon-node/src/db/interface.ts index f39c39ebaba9..59b9c3630805 100644 --- a/packages/beacon-node/src/db/interface.ts +++ b/packages/beacon-node/src/db/interface.ts @@ -14,6 +14,8 @@ import { SyncCommitteeRepository, SyncCommitteeWitnessRepository, BackfilledRanges, + BlobsSidecarRepository, + BlobsSidecarArchiveRepository, } from "./repositories/index.js"; import {PreGenesisState, PreGenesisStateLastProcessedBlock} from "./single/index.js"; @@ -25,9 +27,11 @@ import {PreGenesisState, PreGenesisStateLastProcessedBlock} from "./single/index export interface IBeaconDb { // unfinalized blocks block: BlockRepository; + blobsSidecar: BlobsSidecarRepository; // finalized blocks blockArchive: BlockArchiveRepository; + blobsSidecarArchive: BlobsSidecarArchiveRepository; // finalized states stateArchive: StateArchiveRepository; diff --git a/packages/beacon-node/src/db/repositories/blobsSidecar.ts b/packages/beacon-node/src/db/repositories/blobsSidecar.ts new file mode 100644 index 000000000000..900be1230438 --- /dev/null +++ b/packages/beacon-node/src/db/repositories/blobsSidecar.ts @@ -0,0 +1,31 @@ +import {IChainForkConfig} from "@lodestar/config"; +import {Bucket, Db, Repository} from "@lodestar/db"; +import {allForks, eip4844, ssz} from "@lodestar/types"; +import {getSignedBlockTypeFromBytes} from "../../util/multifork.js"; + +/** + * BlobsSidecar by block root (= hash_tree_root(SignedBeaconBlockAndBlobsSidecar.beacon_block.message)) + * + * Used to store unfinalized BlobsSidecar + */ +export class BlobsSidecarRepository extends Repository { + constructor(config: IChainForkConfig, db: Db) { + const type = ssz.phase0.SignedBeaconBlock; // Pick some type but won't be used + super(config, db, Bucket.allForks_block, type); + } + + /** + * Id is hashTreeRoot of unsigned BeaconBlock + */ + getId(value: allForks.SignedBeaconBlock): Uint8Array { + return this.config.getForkTypes(value.message.slot).BeaconBlock.hashTreeRoot(value.message); + } + + encodeValue(value: allForks.SignedBeaconBlock): Buffer { + return this.config.getForkTypes(value.message.slot).SignedBeaconBlock.serialize(value) as Buffer; + } + + decodeValue(data: Buffer): allForks.SignedBeaconBlock { + return getSignedBlockTypeFromBytes(this.config, data).deserialize(data); + } +} diff --git a/packages/beacon-node/src/db/repositories/blobsSidecarArchive.ts b/packages/beacon-node/src/db/repositories/blobsSidecarArchive.ts new file mode 100644 index 000000000000..45e276caeb0d --- /dev/null +++ b/packages/beacon-node/src/db/repositories/blobsSidecarArchive.ts @@ -0,0 +1,162 @@ +import all from "it-all"; +import {IChainForkConfig} from "@lodestar/config"; +import {Db, Repository, IKeyValue, IFilterOptions, Bucket} from "@lodestar/db"; +import {Slot, Root, allForks, ssz, eip4844} from "@lodestar/types"; +import {bytesToInt} from "@lodestar/utils"; +import {getSignedBlockTypeFromBytes} from "../../util/multifork.js"; +import {getRootIndexKey, getParentRootIndexKey} from "./blockArchiveIndex.js"; +import {deleteParentRootIndex, deleteRootIndex, storeParentRootIndex, storeRootIndex} from "./blockArchiveIndex.js"; + +export interface IBlockFilterOptions extends IFilterOptions { + step?: number; +} + +export type BlockArchiveBatchPutBinaryItem = IKeyValue & { + slot: Slot; + blockRoot: Root; + parentRoot: Root; +}; + +/** + * Stores finalized blocks. Block slot is identifier. + */ +export class BlobsSidecarArchiveRepository extends Repository { + constructor(config: IChainForkConfig, db: Db) { + const type = ssz.phase0.SignedBeaconBlock; // Pick some type but won't be used + super(config, db, Bucket.allForks_blockArchive, type); + } + + // Overrides for multi-fork + + encodeValue(value: allForks.SignedBeaconBlock): Uint8Array { + return this.config.getForkTypes(value.message.slot).SignedBeaconBlock.serialize(value) as Uint8Array; + } + + decodeValue(data: Uint8Array): allForks.SignedBeaconBlock { + return getSignedBlockTypeFromBytes(this.config, data).deserialize(data); + } + + // Handle key as slot + + getId(value: allForks.SignedBeaconBlock): Slot { + return value.message.slot; + } + + decodeKey(data: Uint8Array): number { + return bytesToInt((super.decodeKey(data) as unknown) as Uint8Array, "be"); + } + + // Overrides to index + + async put(key: Slot, value: allForks.SignedBeaconBlock): Promise { + const blockRoot = this.config.getForkTypes(value.message.slot).BeaconBlock.hashTreeRoot(value.message); + const slot = value.message.slot; + await Promise.all([ + super.put(key, value), + storeRootIndex(this.db, slot, blockRoot), + storeParentRootIndex(this.db, slot, value.message.parentRoot), + ]); + } + + async batchPut(items: IKeyValue[]): Promise { + await Promise.all([ + super.batchPut(items), + Array.from(items).map((item) => { + const slot = item.value.message.slot; + const blockRoot = this.config.getForkTypes(slot).BeaconBlock.hashTreeRoot(item.value.message); + return storeRootIndex(this.db, slot, blockRoot); + }), + Array.from(items).map((item) => { + const slot = item.value.message.slot; + const parentRoot = item.value.message.parentRoot; + return storeParentRootIndex(this.db, slot, parentRoot); + }), + ]); + } + + async batchPutBinary(items: BlockArchiveBatchPutBinaryItem[]): Promise { + await Promise.all([ + super.batchPutBinary(items), + Array.from(items).map((item) => storeRootIndex(this.db, item.slot, item.blockRoot)), + Array.from(items).map((item) => storeParentRootIndex(this.db, item.slot, item.parentRoot)), + ]); + } + + async remove(value: allForks.SignedBeaconBlock): Promise { + await Promise.all([ + super.remove(value), + deleteRootIndex(this.db, this.config.getForkTypes(value.message.slot).SignedBeaconBlock, value), + deleteParentRootIndex(this.db, value), + ]); + } + + async batchRemove(values: allForks.SignedBeaconBlock[]): Promise { + await Promise.all([ + super.batchRemove(values), + Array.from(values).map((value) => + deleteRootIndex(this.db, this.config.getForkTypes(value.message.slot).SignedBeaconBlock, value) + ), + Array.from(values).map((value) => deleteParentRootIndex(this.db, value)), + ]); + } + + async *valuesStream(opts?: IBlockFilterOptions): AsyncIterable { + const firstSlot = this.getFirstSlot(opts); + const valuesStream = super.valuesStream(opts); + const step = (opts && opts.step) || 1; + + for await (const value of valuesStream) { + if ((value.message.slot - firstSlot) % step === 0) { + yield value; + } + } + } + + async values(opts?: IBlockFilterOptions): Promise { + return all(this.valuesStream(opts)); + } + + // INDEX + + async getByRoot(root: Root): Promise { + const slot = await this.getSlotByRoot(root); + return slot !== null ? await this.get(slot) : null; + } + + async getBinaryEntryByRoot(root: Root): Promise | null> { + const slot = await this.getSlotByRoot(root); + return slot !== null ? ({key: slot, value: await this.getBinary(slot)} as IKeyValue) : null; + } + + async getByParentRoot(root: Root): Promise { + const slot = await this.getSlotByParentRoot(root); + return slot !== null ? await this.get(slot) : null; + } + + async getSlotByRoot(root: Root): Promise { + return this.parseSlot(await this.db.get(getRootIndexKey(root))); + } + + async getSlotByParentRoot(root: Root): Promise { + return this.parseSlot(await this.db.get(getParentRootIndexKey(root))); + } + + private parseSlot(slotBytes: Uint8Array | null): Slot | null { + if (!slotBytes) return null; + const slot = bytesToInt(slotBytes, "be"); + // TODO: Is this necessary? How can bytesToInt return a non-integer? + return Number.isInteger(slot) ? slot : null; + } + + private getFirstSlot(opts?: IBlockFilterOptions): Slot { + const dbFilterOpts = this.dbFilterOptions(opts); + const firstSlot = dbFilterOpts.gt + ? this.decodeKey(dbFilterOpts.gt) + 1 + : dbFilterOpts.gte + ? this.decodeKey(dbFilterOpts.gte) + : null; + if (firstSlot === null) throw Error("specify opts.gt or opts.gte"); + + return firstSlot; + } +} diff --git a/packages/beacon-node/src/db/repositories/index.ts b/packages/beacon-node/src/db/repositories/index.ts index 1b39cf5292a5..8177914f73df 100644 --- a/packages/beacon-node/src/db/repositories/index.ts +++ b/packages/beacon-node/src/db/repositories/index.ts @@ -1,4 +1,6 @@ export {BlockRepository} from "./block.js"; +export {BlobsSidecarRepository} from "./blobsSidecar.js"; +export {BlobsSidecarArchiveRepository} from "./blobsSidecarArchive.js"; export {BlockArchiveBatchPutBinaryItem, BlockArchiveRepository, IBlockFilterOptions} from "./blockArchive.js"; export {StateArchiveRepository} from "./stateArchive.js"; diff --git a/packages/beacon-node/src/network/reqresp/handlers/beaconBlockAndBlobsSidecarByRoot.ts b/packages/beacon-node/src/network/reqresp/handlers/beaconBlockAndBlobsSidecarByRoot.ts new file mode 100644 index 000000000000..386b15a27bc2 --- /dev/null +++ b/packages/beacon-node/src/network/reqresp/handlers/beaconBlockAndBlobsSidecarByRoot.ts @@ -0,0 +1,70 @@ +import {eip4844} from "@lodestar/types"; +import {toHex} from "@lodestar/utils"; +import {IBeaconChain} from "../../../chain/index.js"; +import {IBeaconDb} from "../../../db/index.js"; +import {getSlotFromBytes} from "../../../util/multifork.js"; +import {ReqRespBlockResponse} from "../types.js"; + +export async function* onBeaconBlockAndBlobsSidecarByRoot( + requestBody: eip4844.BeaconBlockAndBlobsSidecarByRootRequest, + chain: IBeaconChain, + db: IBeaconDb +): AsyncIterable { + const finalizedSlot = chain.forkChoice.getFinalizedBlock().slot; + + for (const blockRoot of requestBody) { + const blockRootHex = toHex(blockRoot); + const summary = chain.forkChoice.getBlockHex(blockRootHex); + + // NOTE: Only support non-finalized blocks. + // SPEC: Clients MUST support requesting blocks and sidecars since the latest finalized epoch. + // https://github.com/ethereum/consensus-specs/blob/11a037fd9227e29ee809c9397b09f8cc3383a8c0/specs/eip4844/p2p-interface.md#beaconblockandblobssidecarbyroot-v1 + if (!summary || summary.slot <= finalizedSlot) { + // TODO: Should accept the finalized block? Is the finalized block in the archive DB or hot DB? + continue; + } + + // finalized block has summary in forkchoice but it stays in blockArchive db + const blockBytes = await db.block.getBinary(blockRoot); + if (!blockBytes) { + throw Error(`Inconsistent state, block known to fork-choice not in db ${blockRootHex}`); + } + + const blobsSidecarBytes = await db.blobsSidecar.getBinary(blockRoot); + if (!blobsSidecarBytes) { + throw Error(`Inconsistent state, blobsSidecar known to fork-choice not in db ${blockRootHex}`); + } + + yield { + bytes: signedBeaconBlockAndBlobsSidecarFromBytes(blockBytes, blobsSidecarBytes), + slot: getSlotFromBytes(blockBytes), + }; + } +} + +/** + * Construct a valid SSZ serialized container from its properties also serialized. + * ``` + * class SignedBeaconBlockAndBlobsSidecar(Container): + * beacon_block: SignedBeaconBlock + * blobs_sidecar: BlobsSidecar + * ``` + */ +function signedBeaconBlockAndBlobsSidecarFromBytes(blockBytes: Uint8Array, blobsSidecarBytes: Uint8Array): Uint8Array { + const totalLen = 4 + 4 + blockBytes.length + blobsSidecarBytes.length; + const arrayBuffer = new ArrayBuffer(totalLen); + const dataView = new DataView(arrayBuffer); + const uint8Array = new Uint8Array(arrayBuffer); + + const blockOffset = 8; + const blobsOffset = 8 + blockBytes.length; + + // Write offsets + dataView.setUint32(0, blockOffset); + dataView.setUint32(4, blobsOffset); + + uint8Array.set(blockBytes, blockOffset); + uint8Array.set(blobsSidecarBytes, blobsOffset); + + return uint8Array; +} diff --git a/packages/beacon-node/src/network/reqresp/handlers/blobsSidecarsByRange.ts b/packages/beacon-node/src/network/reqresp/handlers/blobsSidecarsByRange.ts new file mode 100644 index 000000000000..c680e800ca6c --- /dev/null +++ b/packages/beacon-node/src/network/reqresp/handlers/blobsSidecarsByRange.ts @@ -0,0 +1,76 @@ +import {GENESIS_SLOT, MAX_REQUEST_BLOCKS} from "@lodestar/params"; +import {eip4844, Slot} from "@lodestar/types"; +import {fromHex} from "@lodestar/utils"; +import {IBeaconChain} from "../../../chain/index.js"; +import {IBeaconDb} from "../../../db/index.js"; +import {RespStatus} from "../../../constants/index.js"; +import {ResponseError} from "../response/index.js"; + +/** This type helps response to beacon_block_by_range and beacon_block_by_root more efficiently */ +export type ReqRespBlobsResponse = { + /** Deserialized data of allForks.SignedBeaconBlock */ + bytes: Uint8Array; + slot: Slot; +}; + +// TODO: Unit test + +export async function* onBlobsSidecarsByRange( + requestBody: eip4844.BlobsSidecarsByRangeRequest, + chain: IBeaconChain, + db: IBeaconDb +): AsyncIterable { + const {startSlot} = requestBody; + let {count} = requestBody; + + if (count < 1) { + throw new ResponseError(RespStatus.INVALID_REQUEST, "count < 1"); + } + // TODO: validate against MIN_EPOCHS_FOR_BLOCK_REQUESTS + if (startSlot < GENESIS_SLOT) { + throw new ResponseError(RespStatus.INVALID_REQUEST, "startSlot < genesis"); + } + + if (count > MAX_REQUEST_BLOCKS) { + count = MAX_REQUEST_BLOCKS; + } + + const endSlot = startSlot + count; + + // SPEC: Clients MUST respond with blobs sidecars from their view of the current fork choice -- that is, blobs + // sidecars as included by blocks from the single chain defined by the current head. Of note, blocks from slots + // before the finalization MUST lead to the finalized block reported in the Status handshake. + // https://github.com/ethereum/consensus-specs/blob/11a037fd9227e29ee809c9397b09f8cc3383a8c0/specs/eip4844/p2p-interface.md#blobssidecarsbyrange-v1 + + const finalizedSlot = chain.forkChoice.getFinalizedBlock().slot; + + // Finalized tram of blobs + // TODO EIP-4844: Should the finalized block be included here or below? + + if (endSlot <= finalizedSlot) { + // Chain of blobs won't change + for await (const {key, value} of db.blobsSidecarArchive.binaryEntriesStream({gte: startSlot, lt: endSlot})) { + yield {bytes: value, slot: db.blockArchive.decodeKey(key)}; + } + } + + // Non-finalized tram of blobs + + if (endSlot > finalizedSlot) { + const headRoot = chain.forkChoice.getHeadRoot(); + // TODO EIP-4844: forkChoice should mantain an array of canonical blocks, and change only on reorg + const headChain = chain.forkChoice.getAllAncestorBlocks(headRoot); + + for (const blockSummary of headChain) { + // Must include only blocks with slot < endSlot + if (blockSummary.slot >= endSlot) { + break; + } + + const blockBytes = await db.blobsSidecar.getBinary(fromHex(blockSummary.blockRoot)); + if (blockBytes) { + yield {bytes: blockBytes, slot: blockSummary.slot}; + } + } + } +} diff --git a/packages/beacon-node/src/network/reqresp/handlers/index.ts b/packages/beacon-node/src/network/reqresp/handlers/index.ts index 5139bdd43523..d485d4e83e00 100644 --- a/packages/beacon-node/src/network/reqresp/handlers/index.ts +++ b/packages/beacon-node/src/network/reqresp/handlers/index.ts @@ -1,9 +1,11 @@ -import {altair, phase0, Root} from "@lodestar/types"; +import {altair, eip4844, phase0, Root} from "@lodestar/types"; import {IBeaconChain} from "../../../chain/index.js"; import {IBeaconDb} from "../../../db/index.js"; import {ReqRespBlockResponse} from "../types.js"; import {onBeaconBlocksByRange} from "./beaconBlocksByRange.js"; import {onBeaconBlocksByRoot} from "./beaconBlocksByRoot.js"; +import {onBeaconBlockAndBlobsSidecarByRoot} from "./beaconBlockAndBlobsSidecarByRoot.js"; +import {onBlobsSidecarsByRange} from "./blobsSidecarsByRange.js"; import {onLightClientBootstrap} from "./lightClientBootstrap.js"; import {onLightClientUpdatesByRange} from "./lightClientUpdatesByRange.js"; import {onLightClientFinalityUpdate} from "./lightClientFinalityUpdate.js"; @@ -13,6 +15,10 @@ export type ReqRespHandlers = { onStatus(): AsyncIterable; onBeaconBlocksByRange(req: phase0.BeaconBlocksByRangeRequest): AsyncIterable; onBeaconBlocksByRoot(req: phase0.BeaconBlocksByRootRequest): AsyncIterable; + onBeaconBlockAndBlobsSidecarByRoot( + req: eip4844.BeaconBlockAndBlobsSidecarByRootRequest + ): AsyncIterable; + onBlobsSidecarsByRange(req: eip4844.BlobsSidecarsByRangeRequest): AsyncIterable; onLightClientBootstrap(req: Root): AsyncIterable; onLightClientUpdatesByRange(req: altair.LightClientUpdatesByRange): AsyncIterable; onLightClientFinalityUpdate(): AsyncIterable; @@ -34,6 +40,12 @@ export function getReqRespHandlers({db, chain}: {db: IBeaconDb; chain: IBeaconCh async *onBeaconBlocksByRoot(req) { yield* onBeaconBlocksByRoot(req, chain, db); }, + async *onBeaconBlockAndBlobsSidecarByRoot(req) { + yield* onBeaconBlockAndBlobsSidecarByRoot(req, chain, db); + }, + async *onBlobsSidecarsByRange(req) { + yield* onBlobsSidecarsByRange(req, chain, db); + }, async *onLightClientBootstrap(req) { yield* onLightClientBootstrap(req, chain); }, diff --git a/packages/beacon-node/src/network/reqresp/interface.ts b/packages/beacon-node/src/network/reqresp/interface.ts index 296591775587..c6c5742472c8 100644 --- a/packages/beacon-node/src/network/reqresp/interface.ts +++ b/packages/beacon-node/src/network/reqresp/interface.ts @@ -2,7 +2,7 @@ import {Libp2p} from "libp2p"; import {PeerId} from "@libp2p/interface-peer-id"; import {ForkName} from "@lodestar/params"; import {IBeaconConfig} from "@lodestar/config"; -import {allForks, altair, phase0} from "@lodestar/types"; +import {allForks, altair, eip4844, phase0} from "@lodestar/types"; import {ILogger} from "@lodestar/utils"; import {IPeerRpcScoreStore} from "../peers/index.js"; import {MetadataController} from "../metadata.js"; @@ -24,6 +24,19 @@ export interface IReqResp { request: phase0.BeaconBlocksByRangeRequest ): Promise; beaconBlocksByRoot(peerId: PeerId, request: phase0.BeaconBlocksByRootRequest): Promise; + /** + * Requests blocks by block root (= hash_tree_root(SignedBeaconBlockAndBlobsSidecar.beacon_block.message)) + * + * BeaconBlockAndBlobsSidecarByRoot is primarily used to recover recent blocks and sidecars (e.g. when receiving a + * block or attestation whose parent is unknown) + * + * https://github.com/ethereum/consensus-specs/blob/11a037fd9227e29ee809c9397b09f8cc3383a8c0/specs/eip4844/p2p-interface.md#beaconblockandblobssidecarbyroot-v1 + */ + beaconBlockAndBlobsSidecarByRoot( + peerId: PeerId, + request: eip4844.BeaconBlockAndBlobsSidecarByRootRequest + ): Promise; + blobsSidecarsByRange(peerId: PeerId, request: eip4844.BlobsSidecarsByRangeRequest): Promise; pruneOnPeerDisconnect(peerId: PeerId): void; lightClientBootstrap(peerId: PeerId, request: Uint8Array): Promise; lightClientOptimisticUpdate(peerId: PeerId): Promise; diff --git a/packages/beacon-node/src/network/reqresp/reqResp.ts b/packages/beacon-node/src/network/reqresp/reqResp.ts index ed89583151ef..4578d0886f79 100644 --- a/packages/beacon-node/src/network/reqresp/reqResp.ts +++ b/packages/beacon-node/src/network/reqresp/reqResp.ts @@ -4,7 +4,7 @@ import {PeerId} from "@libp2p/interface-peer-id"; import {Connection, Stream} from "@libp2p/interface-connection"; import {ForkName} from "@lodestar/params"; import {IBeaconConfig} from "@lodestar/config"; -import {allForks, altair, phase0} from "@lodestar/types"; +import {allForks, altair, eip4844, phase0} from "@lodestar/types"; import {ILogger} from "@lodestar/utils"; import {RespStatus, timeoutOptions} from "../../constants/index.js"; import {PeersData} from "../peers/peersData.js"; @@ -138,6 +138,32 @@ export class ReqResp implements IReqResp { ); } + async beaconBlockAndBlobsSidecarByRoot( + peerId: PeerId, + request: eip4844.BeaconBlockAndBlobsSidecarByRootRequest + ): Promise { + return await this.sendRequest( + peerId, + Method.BeaconBlockAndBlobsSidecarByRoot, + [Version.V1], + request, + request.length + ); + } + + async blobsSidecarsByRange( + peerId: PeerId, + request: eip4844.BlobsSidecarsByRangeRequest + ): Promise { + return await this.sendRequest( + peerId, + Method.BlobsSidecarsByRange, + [Version.V1], + request, + request.count + ); + } + pruneOnPeerDisconnect(peerId: PeerId): void { this.inboundRateLimiter.prune(peerId); } @@ -298,6 +324,12 @@ export class ReqResp implements IReqResp { case Method.BeaconBlocksByRoot: yield* this.reqRespHandlers.onBeaconBlocksByRoot(requestTyped.body); break; + case Method.BeaconBlockAndBlobsSidecarByRoot: + yield* this.reqRespHandlers.onBeaconBlockAndBlobsSidecarByRoot(requestTyped.body); + break; + case Method.BlobsSidecarsByRange: + yield* this.reqRespHandlers.onBlobsSidecarsByRange(requestTyped.body); + break; case Method.LightClientBootstrap: yield* this.reqRespHandlers.onLightClientBootstrap(requestTyped.body); break; diff --git a/packages/beacon-node/src/network/reqresp/types.ts b/packages/beacon-node/src/network/reqresp/types.ts index 25c1356dd246..01f229097273 100644 --- a/packages/beacon-node/src/network/reqresp/types.ts +++ b/packages/beacon-node/src/network/reqresp/types.ts @@ -1,5 +1,5 @@ import {ForkName} from "@lodestar/params"; -import {allForks, phase0, ssz, Slot, altair, Root} from "@lodestar/types"; +import {allForks, phase0, ssz, Slot, altair, Root, eip4844} from "@lodestar/types"; export const protocolPrefix = "/eth2/beacon_chain/req"; @@ -12,6 +12,8 @@ export enum Method { Metadata = "metadata", BeaconBlocksByRange = "beacon_blocks_by_range", BeaconBlocksByRoot = "beacon_blocks_by_root", + BeaconBlockAndBlobsSidecarByRoot = "beacon_block_and_blobs_sidecar_by_root", + BlobsSidecarsByRange = "blobs_sidecars_by_range", LightClientBootstrap = "light_client_bootstrap", LightClientUpdate = "light_client_updates_by_range", LightClientFinalityUpdate = "light_client_finality_update", @@ -48,6 +50,7 @@ export const protocolsSupported: [Method, Version, Encoding][] = [ [Method.BeaconBlocksByRange, Version.V2, Encoding.SSZ_SNAPPY], [Method.BeaconBlocksByRoot, Version.V1, Encoding.SSZ_SNAPPY], [Method.BeaconBlocksByRoot, Version.V2, Encoding.SSZ_SNAPPY], + // TODO EIP-4844: Wait to have control on when this routes are available [Method.LightClientBootstrap, Version.V1, Encoding.SSZ_SNAPPY], [Method.LightClientUpdate, Version.V1, Encoding.SSZ_SNAPPY], [Method.LightClientFinalityUpdate, Version.V1, Encoding.SSZ_SNAPPY], @@ -61,6 +64,8 @@ export const isSingleResponseChunkByMethod: {[K in Method]: boolean} = { [Method.Metadata]: true, [Method.BeaconBlocksByRange]: false, // A stream, 0 or more response chunks [Method.BeaconBlocksByRoot]: false, + [Method.BlobsSidecarsByRange]: false, + [Method.BeaconBlockAndBlobsSidecarByRoot]: false, [Method.LightClientBootstrap]: true, [Method.LightClientUpdate]: false, [Method.LightClientFinalityUpdate]: true, @@ -87,6 +92,8 @@ export function contextBytesTypeByProtocol(protocol: Protocol): ContextBytesType case Method.LightClientUpdate: case Method.LightClientFinalityUpdate: case Method.LightClientOptimisticUpdate: + case Method.BlobsSidecarsByRange: + case Method.BeaconBlockAndBlobsSidecarByRoot: return ContextBytesType.ForkDigest; case Method.BeaconBlocksByRange: case Method.BeaconBlocksByRoot: @@ -121,6 +128,10 @@ export function getRequestSzzTypeByMethod(method: Method) { return ssz.Root; case Method.LightClientUpdate: return ssz.altair.LightClientUpdatesByRange; + case Method.BlobsSidecarsByRange: + return ssz.eip4844.BlobsSidecarsByRangeRequest; + case Method.BeaconBlockAndBlobsSidecarByRoot: + return ssz.eip4844.BeaconBlockAndBlobsSidecarByRootRequest; } } @@ -131,6 +142,8 @@ export type RequestBodyByMethod = { [Method.Metadata]: null; [Method.BeaconBlocksByRange]: phase0.BeaconBlocksByRangeRequest; [Method.BeaconBlocksByRoot]: phase0.BeaconBlocksByRootRequest; + [Method.BlobsSidecarsByRange]: eip4844.BlobsSidecarsByRangeRequest; + [Method.BeaconBlockAndBlobsSidecarByRoot]: eip4844.BeaconBlockAndBlobsSidecarByRootRequest; [Method.LightClientBootstrap]: Root; [Method.LightClientUpdate]: altair.LightClientUpdatesByRange; [Method.LightClientFinalityUpdate]: null; @@ -156,6 +169,10 @@ export function getResponseSzzTypeByMethod(protocol: Protocol, forkName: ForkNam case Method.BeaconBlocksByRoot: // SignedBeaconBlock type is changed in altair return ssz[forkName].SignedBeaconBlock; + case Method.BlobsSidecarsByRange: + return ssz.eip4844.BlobsSidecar; + case Method.BeaconBlockAndBlobsSidecarByRoot: + return ssz.eip4844.SignedBeaconBlockAndBlobsSidecar; case Method.LightClientBootstrap: return ssz.altair.LightClientBootstrap; case Method.LightClientUpdate: @@ -184,6 +201,11 @@ export function getOutgoingSerializerByMethod(protocol: Protocol): OutgoingSeria case Method.BeaconBlocksByRange: case Method.BeaconBlocksByRoot: return reqRespBlockResponseSerializer; + // TODO EIP-4844: Optimize responding bytes + case Method.BlobsSidecarsByRange: + return ssz.eip4844.BlobsSidecar; + case Method.BeaconBlockAndBlobsSidecarByRoot: + return ssz.eip4844.SignedBeaconBlockAndBlobsSidecar; case Method.LightClientBootstrap: return ssz.altair.LightClientBootstrap; case Method.LightClientUpdate: @@ -211,12 +233,16 @@ type CommonResponseBodyByMethod = { export type OutgoingResponseBodyByMethod = CommonResponseBodyByMethod & { [Method.BeaconBlocksByRange]: ReqRespBlockResponse; [Method.BeaconBlocksByRoot]: ReqRespBlockResponse; + [Method.BlobsSidecarsByRange]: eip4844.BlobsSidecar; + [Method.BeaconBlockAndBlobsSidecarByRoot]: eip4844.SignedBeaconBlockAndBlobsSidecar; }; // p2p protocol in the spec export type IncomingResponseBodyByMethod = CommonResponseBodyByMethod & { [Method.BeaconBlocksByRange]: allForks.SignedBeaconBlock; [Method.BeaconBlocksByRoot]: allForks.SignedBeaconBlock; + [Method.BlobsSidecarsByRange]: eip4844.BlobsSidecar; + [Method.BeaconBlockAndBlobsSidecarByRoot]: eip4844.SignedBeaconBlockAndBlobsSidecar; }; // Helper types to generically define the arguments of the encoder functions diff --git a/packages/state-transition/src/block/externalData.ts b/packages/state-transition/src/block/externalData.ts new file mode 100644 index 000000000000..b0a450a35498 --- /dev/null +++ b/packages/state-transition/src/block/externalData.ts @@ -0,0 +1,16 @@ +export enum ExecutionPayloadStatus { + preMerge = "preMerge", + invalid = "invalid", + valid = "valid", +} + +export enum DataAvailableStatus { + preEIP4844 = "preEIP4844", + notAvailable = "notAvailable", + available = "available", +} + +export interface BlockExternalData { + executionPayloadStatus: ExecutionPayloadStatus; + dataAvailableStatus: DataAvailableStatus; +} diff --git a/packages/state-transition/src/block/index.ts b/packages/state-transition/src/block/index.ts index ee4e8927e48b..e32713a109d6 100644 --- a/packages/state-transition/src/block/index.ts +++ b/packages/state-transition/src/block/index.ts @@ -1,8 +1,7 @@ import {ForkSeq} from "@lodestar/params"; import {allForks, altair, eip4844} from "@lodestar/types"; -import {ExecutionEngine} from "../util/executionEngine.js"; import {getFullOrBlindedPayload, isExecutionEnabled} from "../util/execution.js"; -import {CachedBeaconStateAllForks, CachedBeaconStateBellatrix, CachedBeaconStateEip4844} from "../types.js"; +import {CachedBeaconStateAllForks, CachedBeaconStateBellatrix} from "../types.js"; import {processExecutionPayload} from "./processExecutionPayload.js"; import {processSyncAggregate} from "./processSyncCommittee.js"; import {processBlockHeader} from "./processBlockHeader.js"; @@ -10,6 +9,7 @@ import {processEth1Data} from "./processEth1Data.js"; import {processOperations} from "./processOperations.js"; import {processRandao} from "./processRandao.js"; import {processBlobKzgCommitments} from "./processBlobKzgCommitments.js"; +import {BlockExternalData, DataAvailableStatus} from "./externalData.js"; // Spec tests export {processBlockHeader, processExecutionPayload, processRandao, processEth1Data, processSyncAggregate}; @@ -17,13 +17,14 @@ export * from "./processOperations.js"; export * from "./initiateValidatorExit.js"; export * from "./isValidIndexedAttestation.js"; +export * from "./externalData.js"; export function processBlock( fork: ForkSeq, state: CachedBeaconStateAllForks, block: allForks.FullOrBlindedBeaconBlock, - verifySignatures = true, - executionEngine: ExecutionEngine | null + externalData: BlockExternalData, + verifySignatures = true ): void { processBlockHeader(state, block); @@ -33,7 +34,7 @@ export function processBlock( const fullOrBlindedPayload = getFullOrBlindedPayload(block); if (isExecutionEnabled(state as CachedBeaconStateBellatrix, block)) { - processExecutionPayload(state as CachedBeaconStateBellatrix, fullOrBlindedPayload, executionEngine); + processExecutionPayload(state as CachedBeaconStateBellatrix, fullOrBlindedPayload, externalData); } } @@ -49,6 +50,15 @@ export function processBlock( // New in EIP-4844, note: Can sync optimistically without this condition, see note on `is_data_available` // NOTE: Ommitted and should be verified beforehand + // assert is_data_available(block.slot, hash_tree_root(block), block.body.blob_kzg_commitments) + switch (externalData.dataAvailableStatus) { + case DataAvailableStatus.preEIP4844: + throw Error("dataAvailableStatus preEIP4844"); + case DataAvailableStatus.notAvailable: + throw Error("dataAvailableStatus notAvailable"); + case DataAvailableStatus.available: + break; // ok + } } } diff --git a/packages/state-transition/src/block/processExecutionPayload.ts b/packages/state-transition/src/block/processExecutionPayload.ts index d29742235c15..d3bca00bf440 100644 --- a/packages/state-transition/src/block/processExecutionPayload.ts +++ b/packages/state-transition/src/block/processExecutionPayload.ts @@ -2,13 +2,13 @@ import {ssz, allForks} from "@lodestar/types"; import {toHexString, byteArrayEquals} from "@chainsafe/ssz"; import {CachedBeaconStateBellatrix, CachedBeaconStateCapella} from "../types.js"; import {getRandaoMix} from "../util/index.js"; -import {ExecutionEngine} from "../util/executionEngine.js"; import {isExecutionPayload, isMergeTransitionComplete, isCapellaPayload} from "../util/execution.js"; +import {BlockExternalData, ExecutionPayloadStatus} from "./externalData.js"; export function processExecutionPayload( state: CachedBeaconStateBellatrix | CachedBeaconStateCapella, payload: allForks.FullOrBlindedExecutionPayload, - executionEngine: ExecutionEngine | null + externalData: BlockExternalData ): void { // Verify consistency of the parent hash, block number, base fee per gas and gas limit // with respect to the previous execution payload header @@ -46,8 +46,17 @@ export function processExecutionPayload( // if executionEngine is null, executionEngine.onPayload MUST be called after running processBlock to get the // correct randao mix. Since executionEngine will be an async call in most cases it is called afterwards to keep // the state transition sync - if (isExecutionPayload(payload) && executionEngine && !executionEngine.notifyNewPayload(payload)) { - throw Error("Invalid execution payload"); + // + // Equivalent to `assert executionEngine.notifyNewPayload(payload)` + if (isExecutionPayload(payload)) { + switch (externalData.executionPayloadStatus) { + case ExecutionPayloadStatus.preMerge: + throw Error("executionPayloadStatus preMerge"); + case ExecutionPayloadStatus.invalid: + throw Error("Invalid execution payload"); + case ExecutionPayloadStatus.valid: + break; // ok + } } const transactionsRoot = isExecutionPayload(payload) diff --git a/packages/state-transition/src/index.ts b/packages/state-transition/src/index.ts index 8b9422aacfd9..6affd75199bc 100644 --- a/packages/state-transition/src/index.ts +++ b/packages/state-transition/src/index.ts @@ -38,6 +38,7 @@ export { export {isValidVoluntaryExit} from "./block/processVoluntaryExit.js"; export {assertValidProposerSlashing} from "./block/processProposerSlashing.js"; export {assertValidAttesterSlashing} from "./block/processAttesterSlashing.js"; +export {ExecutionPayloadStatus, DataAvailableStatus, BlockExternalData} from "./block/externalData.js"; // BeaconChain, to prepare new blocks export {becomesNewEth1Data} from "./block/processEth1Data.js"; diff --git a/packages/state-transition/src/stateTransition.ts b/packages/state-transition/src/stateTransition.ts index 09a29e9f2d1e..cc5614780c6a 100644 --- a/packages/state-transition/src/stateTransition.ts +++ b/packages/state-transition/src/stateTransition.ts @@ -22,6 +22,7 @@ import { } from "./slot/index.js"; import {processBlock} from "./block/index.js"; import {processEpoch} from "./epoch/index.js"; +import {BlockExternalData} from "./block/externalData.js"; // Multifork capable state transition @@ -37,6 +38,7 @@ export type StateTransitionOpts = EpochProcessOpts & { export function stateTransition( state: CachedBeaconStateAllForks, signedBlock: allForks.FullOrBlindedSignedBeaconBlock, + externalData: BlockExternalData, options?: StateTransitionOpts, metrics?: IBeaconStateTransitionMetrics | null ): CachedBeaconStateAllForks { @@ -67,7 +69,7 @@ export function stateTransition( const timer = metrics?.stfnProcessBlock.startTimer(); try { - processBlock(fork, postState, block, verifySignatures, null); + processBlock(fork, postState, block, externalData, verifySignatures); } finally { timer?.(); } diff --git a/packages/types/src/eip4844/sszTypes.ts b/packages/types/src/eip4844/sszTypes.ts index 8848ed76799a..b057470e6371 100644 --- a/packages/types/src/eip4844/sszTypes.ts +++ b/packages/types/src/eip4844/sszTypes.ts @@ -1,5 +1,10 @@ import {ContainerType, ListCompositeType, ByteVectorType} from "@chainsafe/ssz"; -import {HISTORICAL_ROOTS_LIMIT, FIELD_ELEMENTS_PER_BLOB, MAX_BLOBS_PER_BLOCK} from "@lodestar/params"; +import { + HISTORICAL_ROOTS_LIMIT, + FIELD_ELEMENTS_PER_BLOB, + MAX_BLOBS_PER_BLOCK, + MAX_REQUEST_BLOCKS, +} from "@lodestar/params"; import {ssz as primitiveSsz} from "../primitive/index.js"; import {ssz as phase0Ssz} from "../phase0/index.js"; import {ssz as altairSsz} from "../altair/index.js"; @@ -18,6 +23,7 @@ export const BLSFieldElement = Bytes32; export const KZGCommitment = Bytes48; export const KZGProof = Bytes48; +// TODO EIP-4844: Not sure where these should go const BYTES_PER_FIELD_ELEMENT = 32; // Beacon chain @@ -30,14 +36,8 @@ export const Blobs = new ListCompositeType(Blob, MAX_BLOBS_PER_BLOCK); export const VersionedHash = Bytes32; export const BlobKzgCommitments = new ListCompositeType(KZGCommitment, MAX_BLOBS_PER_BLOCK); -const excessDataGas = UintBn256; - // Constants -// TODO EIP-4844: Not sure where these should go -// export const BLOB_TX_TYPE uint8(0x05) -// VERSIONED_HASH_VERSION_KZG Bytes1(0x01) - // Validator types // https://github.com/ethereum/consensus-specs/blob/dev/specs/eip4844/validator.md @@ -66,13 +66,26 @@ export const PolynomialAndCommitment = new ContainerType( {typeName: "PolynomialAndCommitment", jsonCase: "eth2"} ); +// ReqResp types +// ============= + +export const BlobsSidecarsByRangeRequest = new ContainerType( + { + startSlot: Slot, + count: UintNum64, + }, + {typeName: "BlobsSidecarsByRangeRequest", jsonCase: "eth2"} +); + +export const BeaconBlockAndBlobsSidecarByRootRequest = new ListCompositeType(Root, MAX_REQUEST_BLOCKS); + // Beacon Chain types // https://github.com/ethereum/consensus-specs/blob/dev/specs/eip4844/beacon-chain.md#containers export const ExecutionPayload = new ContainerType( { ...capellaSsz.ExecutionPayload.fields, - excessDataGas, // New in EIP-4844 + excessDataGas: UintBn256, // New in EIP-4844 }, {typeName: "ExecutionPayload", jsonCase: "eth2"} ); @@ -80,7 +93,7 @@ export const ExecutionPayload = new ContainerType( export const ExecutionPayloadHeader = new ContainerType( { ...capellaSsz.ExecutionPayloadHeader.fields, - excessDataGas, // New in EIP-4844 + excessDataGas: UintBn256, // New in EIP-4844 }, {typeName: "ExecutionPayloadHeader", jsonCase: "eth2"} ); diff --git a/packages/types/src/eip4844/types.ts b/packages/types/src/eip4844/types.ts index a3ee799cd9f0..21e7a4bcf262 100644 --- a/packages/types/src/eip4844/types.ts +++ b/packages/types/src/eip4844/types.ts @@ -11,6 +11,9 @@ export type Polynomial = ValueOf; export type PolynomialAndCommitment = ValueOf; export type BLSFieldElement = ValueOf; +export type BlobsSidecarsByRangeRequest = ValueOf; +export type BeaconBlockAndBlobsSidecarByRootRequest = ValueOf; + export type ExecutionPayload = ValueOf; export type ExecutionPayloadHeader = ValueOf;