From 3b24f64b6ef8c4ee073c3f415c8d24943eb52ac7 Mon Sep 17 00:00:00 2001 From: harkamal Date: Wed, 14 Aug 2024 00:03:02 +0530 Subject: [PATCH 01/10] blockchain: allow optimistic block insertion in blockchain --- packages/blockchain/src/blockchain.ts | 140 +++++++++++++++----------- 1 file changed, 79 insertions(+), 61 deletions(-) diff --git a/packages/blockchain/src/blockchain.ts b/packages/blockchain/src/blockchain.ts index 89db94edda..0df81d7410 100644 --- a/packages/blockchain/src/blockchain.ts +++ b/packages/blockchain/src/blockchain.ts @@ -272,8 +272,8 @@ export class Blockchain implements BlockchainInterface { * heads/hashes are overwritten. * @param block - The block to be added to the blockchain */ - async putBlock(block: Block) { - await this._putBlockOrHeader(block) + async putBlock(block: Block, optimistic: boolean = false) { + await this._putBlockOrHeader(block, optimistic) } /** @@ -344,7 +344,7 @@ export class Blockchain implements BlockchainInterface { * header using the iterator method. * @hidden */ - private async _putBlockOrHeader(item: Block | BlockHeader) { + private async _putBlockOrHeader(item: Block | BlockHeader, optimistic: boolean = false) { await this.runWithLock(async () => { // Save the current sane state incase _putBlockOrHeader midway with some // dirty changes in head trackers @@ -367,15 +367,10 @@ export class Blockchain implements BlockchainInterface { throw new Error( 'Cannot put a different genesis block than current blockchain genesis: create a new Blockchain', ) + // genesis block is not optimistic + optimistic = false } - const { header } = block - const blockHash = header.hash() - const blockNumber = header.number - let td = header.difficulty - const currentTd = { header: BIGINT_0, block: BIGINT_0 } - let dbOps: DBOp[] = [] - if (block.common.chainId() !== this.common.chainId()) { throw new Error( `Chain mismatch while trying to put block or header. Chain ID of block: ${block.common.chainId}, chain ID of blockchain : ${this.common.chainId}`, @@ -391,67 +386,90 @@ export class Blockchain implements BlockchainInterface { await this.consensus!.validateConsensus(block) } - // set total difficulty in the current context scope - if (this._headHeaderHash) { - currentTd.header = await this.getTotalDifficulty(this._headHeaderHash) - } - if (this._headBlockHash) { - currentTd.block = await this.getTotalDifficulty(this._headBlockHash) - } - // calculate the total difficulty of the new block - const parentTd = await this.getParentTD(header) - if (!block.isGenesis()) { - td += parentTd - } - - // save total difficulty to the database - dbOps = dbOps.concat(DBSetTD(td, blockNumber, blockHash)) + const { header } = block + const blockHash = header.hash() + const blockNumber = header.number - // save header/block to the database, but save the input not our wrapper block - dbOps = dbOps.concat(DBSetBlockOrHeader(item)) + let td = header.difficulty + try { + const parentTd = await this.getParentTD(header) + if (!block.isGenesis()) { + td += parentTd + } + // since its linked its no more optimistic + optimistic = false + } catch (e) { + // opimistic insertion does care about td + if (!optimistic) { + throw e + } + } - let commonAncestor: undefined | BlockHeader - let ancestorHeaders: undefined | BlockHeader[] - // if total difficulty is higher than current, add it to canonical chain - if ( - block.isGenesis() || - td > currentTd.header || - block.common.consensusType() === ConsensusType.ProofOfStake - ) { - const foundCommon = await this.findCommonAncestor(header) - commonAncestor = foundCommon.commonAncestor - ancestorHeaders = foundCommon.ancestorHeaders + let dbOps: DBOp[] = [] + if (optimistic) { + dbOps = dbOps.concat(DBSetBlockOrHeader(item)) + dbOps.push(DBSetHashToNumber(blockHash, blockNumber)) + await this.dbManager.batch(dbOps) + } else { + const currentTd = { header: BIGINT_0, block: BIGINT_0 } - this._headHeaderHash = blockHash - if (item instanceof Block) { - this._headBlockHash = blockHash + // set total difficulty in the current context scope + if (this._headHeaderHash) { + currentTd.header = await this.getTotalDifficulty(this._headHeaderHash) } - if (this._hardforkByHeadBlockNumber) { - await this.checkAndTransitionHardForkByNumber(blockNumber, header.timestamp) + if (this._headBlockHash) { + currentTd.block = await this.getTotalDifficulty(this._headBlockHash) } - // delete higher number assignments and overwrite stale canonical chain - await this._deleteCanonicalChainReferences(blockNumber + BIGINT_1, blockHash, dbOps) - // from the current header block, check the blockchain in reverse (i.e. - // traverse `parentHash`) until `numberToHash` matches the current - // number/hash in the canonical chain also: overwrite any heads if these - // heads are stale in `_heads` and `_headBlockHash` - await this._rebuildCanonical(header, dbOps) - } else { - // the TD is lower than the current highest TD so we will add the block - // to the DB, but will not mark it as the canonical chain. - if (td > currentTd.block && item instanceof Block) { - this._headBlockHash = blockHash + // save total difficulty to the database + dbOps = dbOps.concat(DBSetTD(td, blockNumber, blockHash)) + + // save header/block to the database, but save the input not our wrapper block + dbOps = dbOps.concat(DBSetBlockOrHeader(item)) + + let commonAncestor: undefined | BlockHeader + let ancestorHeaders: undefined | BlockHeader[] + // if total difficulty is higher than current, add it to canonical chain + if ( + block.isGenesis() || + td > currentTd.header || + block.common.consensusType() === ConsensusType.ProofOfStake + ) { + const foundCommon = await this.findCommonAncestor(header) + commonAncestor = foundCommon.commonAncestor + ancestorHeaders = foundCommon.ancestorHeaders + + this._headHeaderHash = blockHash + if (item instanceof Block) { + this._headBlockHash = blockHash + } + if (this._hardforkByHeadBlockNumber) { + await this.checkAndTransitionHardForkByNumber(blockNumber, header.timestamp) + } + + // delete higher number assignments and overwrite stale canonical chain + await this._deleteCanonicalChainReferences(blockNumber + BIGINT_1, blockHash, dbOps) + // from the current header block, check the blockchain in reverse (i.e. + // traverse `parentHash`) until `numberToHash` matches the current + // number/hash in the canonical chain also: overwrite any heads if these + // heads are stale in `_heads` and `_headBlockHash` + await this._rebuildCanonical(header, dbOps) + } else { + // the TD is lower than the current highest TD so we will add the block + // to the DB, but will not mark it as the canonical chain. + if (td > currentTd.block && item instanceof Block) { + this._headBlockHash = blockHash + } + // save hash to number lookup info even if rebuild not needed + dbOps.push(DBSetHashToNumber(blockHash, blockNumber)) } - // save hash to number lookup info even if rebuild not needed - dbOps.push(DBSetHashToNumber(blockHash, blockNumber)) - } - const ops = dbOps.concat(this._saveHeadOps()) - await this.dbManager.batch(ops) + const ops = dbOps.concat(this._saveHeadOps()) + await this.dbManager.batch(ops) - await this.consensus?.newBlock(block, commonAncestor, ancestorHeaders) + await this.consensus?.newBlock(block, commonAncestor, ancestorHeaders) + } } catch (e) { // restore head to the previously sane state this._heads = oldHeads From a69d63f54e6de544cab2155a880b6ab4504d1af1 Mon Sep 17 00:00:00 2001 From: harkamal Date: Wed, 14 Aug 2024 16:01:05 +0530 Subject: [PATCH 02/10] add and use optimistic number to hash key/dbop to avoid conflict with number to ahash and rebuild canonical while backfilling --- packages/blockchain/src/blockchain.ts | 4 ++-- packages/blockchain/src/db/constants.ts | 5 +++++ packages/blockchain/src/db/manager.ts | 18 ++++++++++++++++-- packages/blockchain/src/db/operation.ts | 7 +++++++ 4 files changed, 30 insertions(+), 4 deletions(-) diff --git a/packages/blockchain/src/blockchain.ts b/packages/blockchain/src/blockchain.ts index 0df81d7410..55a04f3410 100644 --- a/packages/blockchain/src/blockchain.ts +++ b/packages/blockchain/src/blockchain.ts @@ -362,13 +362,13 @@ export class Blockchain implements BlockchainInterface { if (isGenesis) { if (equalsBytes(this.genesisBlock.hash(), block.hash())) { // Try to re-put the existing genesis block, accept this + optimistic = false return } throw new Error( 'Cannot put a different genesis block than current blockchain genesis: create a new Blockchain', ) // genesis block is not optimistic - optimistic = false } if (block.common.chainId() !== this.common.chainId()) { @@ -410,10 +410,10 @@ export class Blockchain implements BlockchainInterface { if (optimistic) { dbOps = dbOps.concat(DBSetBlockOrHeader(item)) dbOps.push(DBSetHashToNumber(blockHash, blockNumber)) + dbOps.push(DBOp.set(DBTarget.OptimisticNumberToHash, blockHash, { blockNumber })) await this.dbManager.batch(dbOps) } else { const currentTd = { header: BIGINT_0, block: BIGINT_0 } - // set total difficulty in the current context scope if (this._headHeaderHash) { currentTd.header = await this.getTotalDifficulty(this._headHeaderHash) diff --git a/packages/blockchain/src/db/constants.ts b/packages/blockchain/src/db/constants.ts index 5b365937ac..dbe6990fcc 100644 --- a/packages/blockchain/src/db/constants.ts +++ b/packages/blockchain/src/db/constants.ts @@ -28,6 +28,7 @@ const TD_SUFFIX = utf8ToBytes('t') * headerPrefix + number + numSuffix -> hash */ const NUM_SUFFIX = utf8ToBytes('n') +const OPTIMISTIC_NUM_SUFFIX = utf8ToBytes('o') /** * blockHashPrefix + hash -> number @@ -55,6 +56,9 @@ const bodyKey = (n: bigint, hash: Uint8Array) => concatBytes(BODY_PREFIX, bytesB const numberToHashKey = (n: bigint) => concatBytes(HEADER_PREFIX, bytesBE8(n), NUM_SUFFIX) +const optimisticNumberToHashKey = (n: bigint) => + concatBytes(HEADER_PREFIX, bytesBE8(n), OPTIMISTIC_NUM_SUFFIX) + const hashToNumberKey = (hash: Uint8Array) => concatBytes(BLOCK_HASH_PREFIX, hash) /** @@ -69,5 +73,6 @@ export { headerKey, HEADS_KEY, numberToHashKey, + optimisticNumberToHashKey, tdKey, } diff --git a/packages/blockchain/src/db/manager.ts b/packages/blockchain/src/db/manager.ts index 42f826c464..9f1f0f21c6 100644 --- a/packages/blockchain/src/db/manager.ts +++ b/packages/blockchain/src/db/manager.ts @@ -84,7 +84,10 @@ export class DBManager { * Fetches a block (header and body) given a block id, * which can be either its hash or its number. */ - async getBlock(blockId: Uint8Array | bigint | number): Promise { + async getBlock( + blockId: Uint8Array | bigint | number, + optimistic: boolean = false, + ): Promise { if (typeof blockId === 'number' && Number.isInteger(blockId)) { blockId = BigInt(blockId) } @@ -97,7 +100,13 @@ export class DBManager { number = await this.hashToNumber(blockId) } else if (typeof blockId === 'bigint') { number = blockId - hash = await this.numberToHash(blockId) + if (optimistic) { + hash = await this.optimisticNumberToHash(blockId) + } + // hash will be undefined if it no optimistic lookup was done or if that was not successful + if (hash === undefined) { + hash = await this.numberToHash(blockId) + } } else { throw new Error('Unknown blockId type') } @@ -190,6 +199,11 @@ export class DBManager { return value } + async optimisticNumberToHash(blockNumber: bigint): Promise { + const value = await this.get(DBTarget.OptimisticNumberToHash, { blockNumber }) + return value + } + /** * Fetches a key from the db. If `opts.cache` is specified * it first tries to load from cache, and on cache miss will diff --git a/packages/blockchain/src/db/operation.ts b/packages/blockchain/src/db/operation.ts index a861b9605f..5358d4c3bf 100644 --- a/packages/blockchain/src/db/operation.ts +++ b/packages/blockchain/src/db/operation.ts @@ -8,6 +8,7 @@ import { hashToNumberKey, headerKey, numberToHashKey, + optimisticNumberToHashKey, tdKey, } from './constants.js' @@ -25,6 +26,7 @@ export enum DBTarget { CliqueSignerStates, CliqueVotes, CliqueBlockSigners, + OptimisticNumberToHash, } /** @@ -88,6 +90,11 @@ export class DBOp { this.cacheString = 'numberToHash' break } + case DBTarget.OptimisticNumberToHash: { + this.baseDBOp.key = optimisticNumberToHashKey(key!.blockNumber!) + this.cacheString = 'optimisticNumberToHash' + break + } case DBTarget.TotalDifficulty: { this.baseDBOp.key = tdKey(key!.blockNumber!, key!.blockHash!) this.cacheString = 'td' From e640849168b59fd15d13e2d3e08d60d89e3455ea Mon Sep 17 00:00:00 2001 From: harkamal Date: Sat, 17 Aug 2024 18:36:25 +0530 Subject: [PATCH 03/10] add and handle further modifications and flags to handle skeleton functionality --- packages/blockchain/src/blockchain.ts | 32 +++++---- packages/blockchain/src/db/manager.ts | 30 ++++++-- packages/blockchain/src/db/operation.ts | 2 + packages/blockchain/src/types.ts | 5 +- packages/client/src/service/skeleton.ts | 94 +++---------------------- 5 files changed, 58 insertions(+), 105 deletions(-) diff --git a/packages/blockchain/src/blockchain.ts b/packages/blockchain/src/blockchain.ts index 55a04f3410..47d8efef77 100644 --- a/packages/blockchain/src/blockchain.ts +++ b/packages/blockchain/src/blockchain.ts @@ -25,6 +25,7 @@ import { import { DBManager } from './db/manager.js' import { DBTarget } from './db/operation.js' +import type { OptimisticOpts } from './db/operation.js' import type { BlockchainEvents, BlockchainInterface, @@ -272,8 +273,8 @@ export class Blockchain implements BlockchainInterface { * heads/hashes are overwritten. * @param block - The block to be added to the blockchain */ - async putBlock(block: Block, optimistic: boolean = false) { - await this._putBlockOrHeader(block, optimistic) + async putBlock(block: Block, opts?: OptimisticOpts) { + await this._putBlockOrHeader(block, opts) } /** @@ -344,7 +345,7 @@ export class Blockchain implements BlockchainInterface { * header using the iterator method. * @hidden */ - private async _putBlockOrHeader(item: Block | BlockHeader, optimistic: boolean = false) { + private async _putBlockOrHeader(item: Block | BlockHeader, optimisticOpts?: OptimisticOpts) { await this.runWithLock(async () => { // Save the current sane state incase _putBlockOrHeader midway with some // dirty changes in head trackers @@ -362,13 +363,13 @@ export class Blockchain implements BlockchainInterface { if (isGenesis) { if (equalsBytes(this.genesisBlock.hash(), block.hash())) { // Try to re-put the existing genesis block, accept this - optimistic = false + // genesis block is not optimistic + optimisticOpts = undefined return } throw new Error( 'Cannot put a different genesis block than current blockchain genesis: create a new Blockchain', ) - // genesis block is not optimistic } if (block.common.chainId() !== this.common.chainId()) { @@ -377,12 +378,12 @@ export class Blockchain implements BlockchainInterface { ) } - if (this._validateBlocks && !isGenesis && item instanceof Block) { + if (this._validateBlocks && !isGenesis && item instanceof Block && optimisticOpts === undefined) { // this calls into `getBlock`, which is why we cannot lock yet await this.validateBlock(block) } - if (this._validateConsensus) { + if (this._validateConsensus && optimisticOpts === undefined) { await this.consensus!.validateConsensus(block) } @@ -397,20 +398,20 @@ export class Blockchain implements BlockchainInterface { if (!block.isGenesis()) { td += parentTd } - // since its linked its no more optimistic - optimistic = false } catch (e) { // opimistic insertion does care about td - if (!optimistic) { + if (optimisticOpts === undefined) { throw e } } let dbOps: DBOp[] = [] - if (optimistic) { + if (optimisticOpts !== undefined) { dbOps = dbOps.concat(DBSetBlockOrHeader(item)) dbOps.push(DBSetHashToNumber(blockHash, blockNumber)) - dbOps.push(DBOp.set(DBTarget.OptimisticNumberToHash, blockHash, { blockNumber })) + if (optimisticOpts.fcUed) { + dbOps.push(DBOp.set(DBTarget.OptimisticNumberToHash, blockHash, { blockNumber })) + } await this.dbManager.batch(dbOps) } else { const currentTd = { header: BIGINT_0, block: BIGINT_0 } @@ -676,13 +677,16 @@ export class Blockchain implements BlockchainInterface { * this will be immediately looked up, otherwise it will wait until we have * unlocked the DB */ - async getBlock(blockId: Uint8Array | number | bigint): Promise { + async getBlock( + blockId: Uint8Array | number | bigint, + optimisticOpts?: OptimisticOpts, + ): Promise { // cannot wait for a lock here: it is used both in `validate` of `Block` // (calls `getBlock` to get `parentHash`) it is also called from `runBlock` // in the `VM` if we encounter a `BLOCKHASH` opcode: then a bigint is used we // need to then read the block from the canonical chain Q: is this safe? We // know it is OK if we call it from the iterator... (runBlock) - const block = await this.dbManager.getBlock(blockId) + const block = await this.dbManager.getBlock(blockId, optimisticOpts) if (block === undefined) { if (typeof blockId === 'object') { diff --git a/packages/blockchain/src/db/manager.ts b/packages/blockchain/src/db/manager.ts index 9f1f0f21c6..fd2f5b9a67 100644 --- a/packages/blockchain/src/db/manager.ts +++ b/packages/blockchain/src/db/manager.ts @@ -12,7 +12,7 @@ import { import { Cache } from './cache.js' import { DBOp, DBTarget } from './operation.js' -import type { DatabaseKey } from './operation.js' +import type { DatabaseKey, OptimisticOpts } from './operation.js' import type { Block, BlockBodyBytes, BlockBytes, BlockOptions } from '@ethereumjs/block' import type { Common } from '@ethereumjs/common' import type { BatchDBOp, DB, DBObject, DelBatch, PutBatch } from '@ethereumjs/util' @@ -47,6 +47,7 @@ export class DBManager { body: new Cache({ max: 256 }), numberToHash: new Cache({ max: 2048 }), hashToNumber: new Cache({ max: 2048 }), + optimisticNumberToHash: new Cache({ max: 2048 }), } } @@ -86,7 +87,7 @@ export class DBManager { */ async getBlock( blockId: Uint8Array | bigint | number, - optimistic: boolean = false, + optimisticOpts?: OptimisticOpts, ): Promise { if (typeof blockId === 'number' && Number.isInteger(blockId)) { blockId = BigInt(blockId) @@ -98,13 +99,30 @@ export class DBManager { if (blockId instanceof Uint8Array) { hash = blockId number = await this.hashToNumber(blockId) + if (number === undefined) { + return undefined + } + + if (optimisticOpts?.fcUed === true) { + let optimisticHash = await this.optimisticNumberToHash(number) + if (optimisticHash === undefined && optimisticOpts.linked === true) { + optimisticHash = await this.numberToHash(number) + } + if (optimisticHash === undefined || !equalsBytes(optimisticHash, hash)) { + return undefined + } + } } else if (typeof blockId === 'bigint') { number = blockId - if (optimistic) { + if (optimisticOpts !== undefined) { + if (!optimisticOpts.fcUed) { + throw Error(`Invalid fcUed optimistic block by number lookup`) + } hash = await this.optimisticNumberToHash(blockId) - } - // hash will be undefined if it no optimistic lookup was done or if that was not successful - if (hash === undefined) { + if (hash === undefined && optimisticOpts.linked === true) { + hash = await this.numberToHash(blockId) + } + } else { hash = await this.numberToHash(blockId) } } else { diff --git a/packages/blockchain/src/db/operation.ts b/packages/blockchain/src/db/operation.ts index 5358d4c3bf..5dd3e40d54 100644 --- a/packages/blockchain/src/db/operation.ts +++ b/packages/blockchain/src/db/operation.ts @@ -14,6 +14,8 @@ import { import type { CacheMap } from './manager.js' +export type OptimisticOpts = { fcUed: boolean; linked?: boolean } + export enum DBTarget { Heads, HeadHeader, diff --git a/packages/blockchain/src/types.ts b/packages/blockchain/src/types.ts index 0eee200bf8..69843c1c3e 100644 --- a/packages/blockchain/src/types.ts +++ b/packages/blockchain/src/types.ts @@ -1,3 +1,4 @@ +import type { OptimisticOpts } from './db/operation.js' import type { Blockchain } from './index.js' import type { Block, BlockHeader } from '@ethereumjs/block' import type { Common, ConsensusAlgorithm } from '@ethereumjs/common' @@ -16,7 +17,7 @@ export interface BlockchainInterface { * * @param block - The block to be added to the blockchain. */ - putBlock(block: Block): Promise + putBlock(block: Block, optimisticOpts?: OptimisticOpts): Promise /** * Deletes a block from the blockchain. All child blocks in the chain are @@ -29,7 +30,7 @@ export interface BlockchainInterface { /** * Returns a block by its hash or number. */ - getBlock(blockId: Uint8Array | number | bigint): Promise + getBlock(blockId: Uint8Array | number | bigint, optimisticOpts?: OptimisticOpts): Promise /** * Iterates through blocks starting at the specified iterator head and calls diff --git a/packages/client/src/service/skeleton.ts b/packages/client/src/service/skeleton.ts index 859a6454b3..f6bfdb0bd3 100644 --- a/packages/client/src/service/skeleton.ts +++ b/packages/client/src/service/skeleton.ts @@ -1362,21 +1362,7 @@ export class Skeleton extends MetaDBManager { */ private async putBlock(block: Block, onlyUnfinalized: boolean = false): Promise { // Serialize the block with its hardfork so that its easy to load the block latter - const rlp = this.serialize({ hardfork: block.common.hardfork(), blockRLP: block.serialize() }) - await this.put(DBKey.SkeletonUnfinalizedBlockByHash, block.hash(), rlp) - - if (!onlyUnfinalized) { - await this.put(DBKey.SkeletonBlock, bigIntToBytes(block.header.number), rlp) - // this is duplication of the unfinalized blocks but for now an easy reference - // will be pruned on finalization changes. this could be simplified and deduped - // but will anyway will move into blockchain class and db on upcoming skeleton refactor - await this.put( - DBKey.SkeletonBlockHashToNumber, - block.hash(), - bigIntToBytes(block.header.number), - ) - } - + await this.chain.blockchain.putBlock(block, { fcUed: !onlyUnfinalized }) return true } @@ -1395,22 +1381,10 @@ export class Skeleton extends MetaDBManager { * Gets a block from the skeleton or canonical db by number. */ async getBlock(number: bigint, onlyCanonical = false): Promise { - try { - const skeletonBlockRlp = await this.get(DBKey.SkeletonBlock, bigIntToBytes(number)) - if (skeletonBlockRlp === null) { - throw Error(`SkeletonBlock rlp lookup failed for ${number} onlyCanonical=${onlyCanonical}`) - } - return this.skeletonBlockRlpToBlock(skeletonBlockRlp) - } catch (error: any) { - // If skeleton is linked, it probably has deleted the block and put it into the chain - if (onlyCanonical && !this.status.linked) return undefined - // As a fallback, try to get the block from the canonical chain in case it is available there - try { - return await this.chain.getBlock(number) - } catch (error) { - return undefined - } - } + return this.chain.blockchain.dbManager.getBlock(number, { + fcUed: onlyCanonical, + linked: this.status.linked, + }) } /** @@ -1420,63 +1394,17 @@ export class Skeleton extends MetaDBManager { hash: Uint8Array, onlyCanonical: boolean = false, ): Promise { - const number = await this.get(DBKey.SkeletonBlockHashToNumber, hash) - if (number) { - const block = await this.getBlock(bytesToBigInt(number), onlyCanonical) - if (block !== undefined && equalsBytes(block.hash(), hash)) { - return block - } - } - - if (onlyCanonical === true && !this.status.linked) { - return undefined - } - - let block = onlyCanonical === false ? await this.getUnfinalizedBlock(hash) : undefined - if (block === undefined && (onlyCanonical === false || this.status.linked)) { - block = await this.chain.getBlock(hash).catch((_e) => undefined) - } - - if (onlyCanonical === false) { - return block - } else { - if (this.status.linked && block !== undefined) { - const canBlock = await this.chain.getBlock(block.header.number).catch((_e) => undefined) - if (canBlock !== undefined && equalsBytes(canBlock.hash(), block.hash())) { - // block is canonical - return block - } - } - - // no canonical block found or the block was not canonical - return undefined - } - } - - async getUnfinalizedBlock(hash: Uint8Array): Promise { - try { - const skeletonBlockRlp = await this.get(DBKey.SkeletonUnfinalizedBlockByHash, hash) - if (skeletonBlockRlp === null) { - throw Error(`SkeletonUnfinalizedBlockByHash rlp lookup failed for hash=${short(hash)}`) - } - return this.skeletonBlockRlpToBlock(skeletonBlockRlp) - } catch (_e) { - return undefined - } + return this.chain.blockchain.dbManager.getBlock(hash, { + fcUed: !onlyCanonical, + linked: this.status.linked, + }) } /** * Deletes a skeleton block from the db by number */ - async deleteBlock(block: Block): Promise { - try { - await this.delete(DBKey.SkeletonBlock, bigIntToBytes(block.header.number)) - await this.delete(DBKey.SkeletonBlockHashToNumber, block.hash()) - await this.delete(DBKey.SkeletonUnfinalizedBlockByHash, block.hash()) - return true - } catch (error: any) { - return false - } + async deleteBlock(_block: Block): Promise { + return true } /** From bbb140b92a1980f272a2cf8ae33dc91006bf80db Mon Sep 17 00:00:00 2001 From: harkamal Date: Sun, 18 Aug 2024 16:56:46 +0530 Subject: [PATCH 04/10] debug and fix skeleton tests --- packages/client/src/service/skeleton.ts | 35 +++++++--------- packages/client/test/sync/skeleton.spec.ts | 49 +++++++++++++--------- 2 files changed, 44 insertions(+), 40 deletions(-) diff --git a/packages/client/src/service/skeleton.ts b/packages/client/src/service/skeleton.ts index f6bfdb0bd3..9d547f8166 100644 --- a/packages/client/src/service/skeleton.ts +++ b/packages/client/src/service/skeleton.ts @@ -233,15 +233,6 @@ export class Skeleton extends MetaDBManager { return this.started > 0 } - async isLastAnnouncement(): Promise { - const subchain0 = this.status.progress.subchains[0] - if (subchain0 !== undefined) { - return this.getBlock(subchain0.head + BIGINT_1) !== undefined - } else { - return true - } - } - /** * Try fast forwarding the chain head to the number */ @@ -316,7 +307,7 @@ export class Skeleton extends MetaDBManager { } else if (lastchain.head >= number) { // Check if its duplicate announcement, if not trim the head and let the match run // post this if block - const mayBeDupBlock = await this.getBlock(number) + const mayBeDupBlock = await this.getBlock(number, true) if (mayBeDupBlock !== undefined && equalsBytes(mayBeDupBlock.header.hash(), head.hash())) { this.config.logger.debug( `Skeleton duplicate ${force ? 'setHead' : 'announcement'} tail=${lastchain.tail} head=${ @@ -359,7 +350,7 @@ export class Skeleton extends MetaDBManager { return true } } - const parent = await this.getBlock(number - BIGINT_1) + const parent = await this.getBlock(number - BIGINT_1, true) if (parent === undefined || !equalsBytes(parent.hash(), head.header.parentHash)) { if (force) { this.config.logger.warn( @@ -818,7 +809,7 @@ export class Skeleton extends MetaDBManager { const syncedBlock = await this.getBlock( syncedHeight, // need to debug why this flag causes to return undefined when chain gets synced - //, true + true, ) if ( syncedBlock !== undefined && @@ -859,7 +850,7 @@ export class Skeleton extends MetaDBManager { async headHash(): Promise { const subchain = this.bounds() if (subchain !== undefined) { - const headBlock = await this.getBlock(subchain.head) + const headBlock = await this.getBlock(subchain.head, true) if (headBlock) { return headBlock.hash() } @@ -998,7 +989,7 @@ export class Skeleton extends MetaDBManager { } else { // Critical error, we expect new incoming blocks to extend the canonical // subchain which is the [0]'th - const tailBlock = await this.getBlock(this.status.progress.subchains[0].tail) + const tailBlock = await this.getBlock(this.status.progress.subchains[0].tail, true) this.config.logger.warn( `Blocks don't extend canonical subchain tail=${ this.status.progress.subchains[0].tail @@ -1112,7 +1103,7 @@ export class Skeleton extends MetaDBManager { for (let i = 0; i < maxItems; i++) { const tailBlock = await this.getBlockByHash(next) - if (tailBlock === undefined) { + if (tailBlock === undefined || tailBlock.header.number <= BIGINT_1) { break } else { blocks.push(tailBlock) @@ -1202,7 +1193,7 @@ export class Skeleton extends MetaDBManager { while (!this.status.canonicalHeadReset && canonicalHead < subchain.head) { // Get next block const number = canonicalHead + BIGINT_1 - const block = await this.getBlock(number) + const block = await this.getBlock(number, true) if (block === undefined) { // This can happen @@ -1381,10 +1372,12 @@ export class Skeleton extends MetaDBManager { * Gets a block from the skeleton or canonical db by number. */ async getBlock(number: bigint, onlyCanonical = false): Promise { - return this.chain.blockchain.dbManager.getBlock(number, { - fcUed: onlyCanonical, - linked: this.status.linked, - }) + return this.chain.blockchain.dbManager + .getBlock(number, { + fcUed: onlyCanonical, + linked: this.status.linked, + }) + .catch((_e) => undefined) } /** @@ -1395,7 +1388,7 @@ export class Skeleton extends MetaDBManager { onlyCanonical: boolean = false, ): Promise { return this.chain.blockchain.dbManager.getBlock(hash, { - fcUed: !onlyCanonical, + fcUed: onlyCanonical, linked: this.status.linked, }) } diff --git a/packages/client/test/sync/skeleton.spec.ts b/packages/client/test/sync/skeleton.spec.ts index a26853cfa8..64c48c5d08 100644 --- a/packages/client/test/sync/skeleton.spec.ts +++ b/packages/client/test/sync/skeleton.spec.ts @@ -25,14 +25,23 @@ type Subchain = { } const common = new Common({ chain: Mainnet }) -const block49 = createBlock({ header: { number: 49 } }, { common }) -const block49B = createBlock({ header: { number: 49, extraData: utf8ToBytes('B') } }, { common }) -const block50 = createBlock({ header: { number: 50, parentHash: block49.hash() } }, { common }) +const block49 = createBlock({ header: { number: 49 } }, { common, setHardfork: true }) +const block49B = createBlock( + { header: { number: 49, extraData: utf8ToBytes('B') } }, + { common, setHardfork: true }, +) +const block50 = createBlock( + { header: { number: 50, parentHash: block49.hash() } }, + { common, setHardfork: true }, +) const block50B = createBlock( { header: { number: 50, parentHash: block49.hash(), gasLimit: 999 } }, - { common }, + { common, setHardfork: true }, +) +const block51 = createBlock( + { header: { number: 51, parentHash: block50.hash() } }, + { common, setHardfork: true }, ) -const block51 = createBlock({ header: { number: 51, parentHash: block50.hash() } }, { common }) describe('[Skeleton]/ startup scenarios ', () => { it('starts the chain when starting the skeleton', async () => { @@ -594,20 +603,20 @@ describe('[Skeleton] / setHead', async () => { 'canonical height should change when setHead is set with force=true', ) - // unlink the skeleton for the below check to check all blocks cleared + // unlink the skeleton for the below check and still find the blocks in blokchain skeleton['status'].linked = false for (const block of [block1, block2, block3, block4, block5]) { assert.equal( - (await skeleton.getBlock(block.header.number, true))?.hash(), - undefined, - `skeleton block number=${block.header.number} should be cleaned up after filling canonical chain`, + (await skeleton.getBlock(block.header.number, true))?.hash() !== undefined, + true, + `skeleton block number=${block.header.number} should be available even afer filling canonical chain`, ) assert.equal( - (await skeleton.getBlockByHash(block.hash(), true))?.hash(), - undefined, + (await skeleton.getBlockByHash(block.hash(), true))?.hash() !== undefined, + true, `skeleton block hash=${short( block.hash(), - )} should be cleaned up after filling canonical chain`, + )} should be available even after filling canonical chain`, ) } }) @@ -677,16 +686,16 @@ describe('[Skeleton] / setHead', async () => { skeleton['status'].linked = false for (const block of [block3, block4, block5]) { assert.equal( - (await skeleton.getBlock(block.header.number, true))?.hash(), - undefined, - `skeleton block number=${block.header.number} should be cleaned up after filling canonical chain`, + (await skeleton.getBlock(block.header.number, true))?.hash() !== undefined, + true, + `skeleton block number=${block.header.number} should still be available after filling canonical chain`, ) assert.equal( - (await skeleton.getBlockByHash(block.hash(), true))?.hash(), - undefined, + (await skeleton.getBlockByHash(block.hash(), true))?.hash() !== undefined, + true, `skeleton block hash=${short( block.hash(), - )} should be cleaned up after filling canonical chain`, + )} should still be available after filling canonical chain`, ) } // restore linkedStatus @@ -707,9 +716,9 @@ describe('[Skeleton] / setHead', async () => { await skeleton.setHead(block41, false) await skeleton.setHead(block51, false) - // should link the chains including the 41, 51 block backfilled from the unfinalized await skeleton.forkchoiceUpdate(block61) + assert.equal( skeleton['status'].progress.subchains[0]?.head, BigInt(6), @@ -856,7 +865,9 @@ describe('[Skeleton] / setHead', async () => { await skeleton.open() await skeleton.initSync(block4InvalidPoS) + // the following putBlocks will fail ad block3 is pow block await skeleton.putBlocks([block3PoW, block2]) + assert.equal(chain.blocks.height, BigInt(0), 'canonical height should be at genesis') await skeleton.putBlocks([block1]) await wait(200) From 23fd13fa8c7f8329b367c59ce0aba586e7087972 Mon Sep 17 00:00:00 2001 From: harkamal Date: Sun, 18 Aug 2024 20:04:40 +0530 Subject: [PATCH 05/10] further debug and fix the beacon sync sim test and get it working --- packages/blockchain/src/blockchain.ts | 7 ++++++- packages/client/test/sync/skeleton.spec.ts | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/blockchain/src/blockchain.ts b/packages/blockchain/src/blockchain.ts index 47d8efef77..8c5b5e5f2e 100644 --- a/packages/blockchain/src/blockchain.ts +++ b/packages/blockchain/src/blockchain.ts @@ -378,7 +378,12 @@ export class Blockchain implements BlockchainInterface { ) } - if (this._validateBlocks && !isGenesis && item instanceof Block && optimisticOpts === undefined) { + if ( + this._validateBlocks && + !isGenesis && + item instanceof Block && + optimisticOpts === undefined + ) { // this calls into `getBlock`, which is why we cannot lock yet await this.validateBlock(block) } diff --git a/packages/client/test/sync/skeleton.spec.ts b/packages/client/test/sync/skeleton.spec.ts index 64c48c5d08..4ff4a98df4 100644 --- a/packages/client/test/sync/skeleton.spec.ts +++ b/packages/client/test/sync/skeleton.spec.ts @@ -242,6 +242,7 @@ describe('[Skeleton] / initSync', async () => { storageCache: 1000, }) const chain = await Chain.create({ config }) + ;(chain.blockchain as any)._validateBlocks = false const skeleton = new Skeleton({ chain, config, metaDB: new MemoryLevel() }) await skeleton.open() @@ -359,6 +360,7 @@ describe('[Skeleton] / setHead', async () => { storageCache: 1000, }) const chain = await Chain.create({ config }) + ;(chain.blockchain as any)._validateBlocks = false const skeleton = new Skeleton({ chain, config, metaDB: new MemoryLevel() }) await skeleton.open() for (const block of testCase.blocks ?? []) { From c7f2ae283b8200226e0e07392667567d46633515 Mon Sep 17 00:00:00 2001 From: harkamal Date: Mon, 26 Aug 2024 22:53:54 +0530 Subject: [PATCH 06/10] refactor the entire approach to simply allow building normal index as optimistic without moving head but track completion by total difficulty index --- packages/blockchain/src/blockchain.ts | 223 ++++++++++++++------- packages/blockchain/src/db/constants.ts | 6 - packages/blockchain/src/db/manager.ts | 52 ++--- packages/blockchain/src/db/operation.ts | 9 +- packages/blockchain/src/types.ts | 6 +- packages/client/src/service/skeleton.ts | 106 ++++++++-- packages/client/src/util/metaDBManager.ts | 1 + packages/client/test/sync/skeleton.spec.ts | 149 +++++++++++--- 8 files changed, 373 insertions(+), 179 deletions(-) diff --git a/packages/blockchain/src/blockchain.ts b/packages/blockchain/src/blockchain.ts index 8c5b5e5f2e..1daf00640c 100644 --- a/packages/blockchain/src/blockchain.ts +++ b/packages/blockchain/src/blockchain.ts @@ -4,6 +4,7 @@ import { AsyncEventEmitter, BIGINT_0, BIGINT_1, + BIGINT_2, BIGINT_8, KECCAK256_RLP, Lock, @@ -25,7 +26,7 @@ import { import { DBManager } from './db/manager.js' import { DBTarget } from './db/operation.js' -import type { OptimisticOpts } from './db/operation.js' +import type { PutOpts } from './db/operation.js' import type { BlockchainEvents, BlockchainInterface, @@ -273,7 +274,7 @@ export class Blockchain implements BlockchainInterface { * heads/hashes are overwritten. * @param block - The block to be added to the blockchain */ - async putBlock(block: Block, opts?: OptimisticOpts) { + async putBlock(block: Block, opts?: PutOpts) { await this._putBlockOrHeader(block, opts) } @@ -345,7 +346,7 @@ export class Blockchain implements BlockchainInterface { * header using the iterator method. * @hidden */ - private async _putBlockOrHeader(item: Block | BlockHeader, optimisticOpts?: OptimisticOpts) { + private async _putBlockOrHeader(item: Block | BlockHeader, opts?: PutOpts) { await this.runWithLock(async () => { // Save the current sane state incase _putBlockOrHeader midway with some // dirty changes in head trackers @@ -363,8 +364,6 @@ export class Blockchain implements BlockchainInterface { if (isGenesis) { if (equalsBytes(this.genesisBlock.hash(), block.hash())) { // Try to re-put the existing genesis block, accept this - // genesis block is not optimistic - optimisticOpts = undefined return } throw new Error( @@ -378,97 +377,164 @@ export class Blockchain implements BlockchainInterface { ) } - if ( - this._validateBlocks && - !isGenesis && - item instanceof Block && - optimisticOpts === undefined - ) { - // this calls into `getBlock`, which is why we cannot lock yet - await this.validateBlock(block) - } - - if (this._validateConsensus && optimisticOpts === undefined) { - await this.consensus!.validateConsensus(block) - } - - // calculate the total difficulty of the new block const { header } = block const blockHash = header.hash() const blockNumber = header.number - let td = header.difficulty - try { - const parentTd = await this.getParentTD(header) - if (!block.isGenesis()) { - td += parentTd - } - } catch (e) { - // opimistic insertion does care about td - if (optimisticOpts === undefined) { - throw e + // check if head is still canonical i.e. if this is a block insertion on tail or reinsertion + // on the same canonical chain + let isHeadChainStillCanonical + if (opts?.notCanonical !== true) { + const childHeaderHash = await this.dbManager.numberToHash(blockNumber + BIGINT_1) + if (childHeaderHash !== undefined) { + const childHeader = await this.dbManager + .getHeaderSafe(childHeaderHash, blockNumber + BIGINT_1) + .catch((_e) => undefined) + isHeadChainStillCanonical = + childHeader !== undefined && equalsBytes(childHeader.parentHash, blockHash) + } else { + isHeadChainStillCanonical = false } + } else { + isHeadChainStillCanonical = true } let dbOps: DBOp[] = [] - if (optimisticOpts !== undefined) { - dbOps = dbOps.concat(DBSetBlockOrHeader(item)) - dbOps.push(DBSetHashToNumber(blockHash, blockNumber)) - if (optimisticOpts.fcUed) { - dbOps.push(DBOp.set(DBTarget.OptimisticNumberToHash, blockHash, { blockNumber })) + dbOps = dbOps.concat(DBSetBlockOrHeader(item)) + dbOps.push(DBSetHashToNumber(blockHash, blockNumber)) + + let parentTd = await this.dbManager + .getTotalDifficultySafe(header.parentHash, BigInt(blockNumber) - BIGINT_1) + .catch((_e) => undefined) + const isOptimistic = parentTd === undefined + parentTd = parentTd ?? opts?.parentTd + const isComplete = parentTd !== undefined + + if (!isOptimistic) { + if (this._validateBlocks && !isGenesis && item instanceof Block) { + // this calls into `getBlock`, which is why we cannot lock yet + await this.validateBlock(block) + } + } + + // 1. if notCanonical is explicit true then just dump the block + // 2. if notCanonical is explicit false then apply that even for the pow/poa blocks + // if they are optimistic, i.e. can't apply the normal rule + // 3. if notCanonical is not defined, then apply normal rules + if (opts?.notCanonical === true) { + if (parentTd !== undefined) { + const td = header.difficulty + parentTd + dbOps = dbOps.concat(DBSetTD(td, blockNumber, blockHash)) } await this.dbManager.batch(dbOps) } else { - const currentTd = { header: BIGINT_0, block: BIGINT_0 } - // set total difficulty in the current context scope - if (this._headHeaderHash) { - currentTd.header = await this.getTotalDifficulty(this._headHeaderHash) - } - if (this._headBlockHash) { - currentTd.block = await this.getTotalDifficulty(this._headBlockHash) - } - - // save total difficulty to the database - dbOps = dbOps.concat(DBSetTD(td, blockNumber, blockHash)) + let updatesHead = block.isGenesis() + let updatesHeadBlock + if (parentTd === undefined) { + // if the block is pow and optimistic, and has not been explicity marked canonical, then + // throw error since pow blocks can't be optimistically added without expicit instruction about + // their canonicality + if ( + !block.isGenesis() && + block.common.consensusType() !== ConsensusType.ProofOfStake && + opts?.notCanonical !== false + ) { + throw Error( + `Invalid parentTd=${parentTd} for consensus=${block.common.consensusType()} putBlockOrHeader`, + ) + } + updatesHead = true + updatesHeadBlock = updatesHead && isComplete + } else { + const currentTd = { header: BIGINT_0, block: BIGINT_0 } + // set total difficulty in the current context scope + if (this._headHeaderHash) { + currentTd.header = await this.getTotalDifficulty(this._headHeaderHash) + } + if (this._headBlockHash) { + currentTd.block = await this.getTotalDifficulty(this._headBlockHash) + } - // save header/block to the database, but save the input not our wrapper block - dbOps = dbOps.concat(DBSetBlockOrHeader(item)) + const td = parentTd + header.difficulty + dbOps = dbOps.concat(DBSetTD(td, blockNumber, blockHash)) + updatesHead = + block.isGenesis() || + td > currentTd.header || + block.common.consensusType() === ConsensusType.ProofOfStake + updatesHeadBlock = + isComplete && + (updatesHead || + td > currentTd.block || + block.common.consensusType() === ConsensusType.ProofOfStake) + } let commonAncestor: undefined | BlockHeader let ancestorHeaders: undefined | BlockHeader[] // if total difficulty is higher than current, add it to canonical chain - if ( - block.isGenesis() || - td > currentTd.header || - block.common.consensusType() === ConsensusType.ProofOfStake - ) { - const foundCommon = await this.findCommonAncestor(header) - commonAncestor = foundCommon.commonAncestor - ancestorHeaders = foundCommon.ancestorHeaders - - this._headHeaderHash = blockHash - if (item instanceof Block) { - this._headBlockHash = blockHash - } + if (updatesHead) { if (this._hardforkByHeadBlockNumber) { await this.checkAndTransitionHardForkByNumber(blockNumber, header.timestamp) } + if (this._validateConsensus) { + await this.consensus!.validateConsensus(block) + } + + if (!isOptimistic) { + const foundCommon = await this.findCommonAncestor(header) + commonAncestor = foundCommon.commonAncestor + ancestorHeaders = foundCommon.ancestorHeaders + } + // delete higher number assignments and overwrite stale canonical chain - await this._deleteCanonicalChainReferences(blockNumber + BIGINT_1, blockHash, dbOps) - // from the current header block, check the blockchain in reverse (i.e. - // traverse `parentHash`) until `numberToHash` matches the current - // number/hash in the canonical chain also: overwrite any heads if these - // heads are stale in `_heads` and `_headBlockHash` - await this._rebuildCanonical(header, dbOps) + if (isComplete) { + this._headHeaderHash = blockHash + if (item instanceof Block) { + this._headBlockHash = blockHash + } + + await this._deleteCanonicalChainReferences(blockNumber + BIGINT_1, blockHash, dbOps) + // from the current header block, check the blockchain in reverse (i.e. + // traverse `parentHash`) until `numberToHash` matches the current + // number/hash in the canonical chain also: overwrite any heads if these + // heads are stale in `_heads` and `_headBlockHash` + await this._rebuildCanonical(header, dbOps) + } else { + // reset to blockNumber - 2, blockNumber - 1 is parent and that is not present else + // we would have isComplete true + const headHeader = await this._getHeader( + this._headHeaderHash ?? this.genesisBlock.hash(), + ) + let resetToNumber = blockNumber - BIGINT_2 + if (resetToNumber < BIGINT_0) { + resetToNumber = BIGINT_0 + } + if (headHeader.number >= resetToNumber) { + let resetToHash = await this.dbManager.numberToHash(resetToNumber) + if (resetToHash === undefined) { + resetToHash = this.genesisBlock.hash() + resetToNumber = BIGINT_0 + } + + this._headHeaderHash = resetToHash + if (item instanceof Block) { + this._headBlockHash = resetToHash + } + await this._deleteCanonicalChainReferences( + resetToNumber + BIGINT_1, + resetToHash, + dbOps, + ) + // save this number to hash + } + } + dbOps.push(DBOp.set(DBTarget.NumberToHash, blockHash, { blockNumber })) } else { // the TD is lower than the current highest TD so we will add the block // to the DB, but will not mark it as the canonical chain. - if (td > currentTd.block && item instanceof Block) { + if (updatesHeadBlock && item instanceof Block) { this._headBlockHash = blockHash } - // save hash to number lookup info even if rebuild not needed - dbOps.push(DBSetHashToNumber(blockHash, blockNumber)) } const ops = dbOps.concat(this._saveHeadOps()) @@ -682,16 +748,13 @@ export class Blockchain implements BlockchainInterface { * this will be immediately looked up, otherwise it will wait until we have * unlocked the DB */ - async getBlock( - blockId: Uint8Array | number | bigint, - optimisticOpts?: OptimisticOpts, - ): Promise { + async getBlock(blockId: Uint8Array | number | bigint): Promise { // cannot wait for a lock here: it is used both in `validate` of `Block` // (calls `getBlock` to get `parentHash`) it is also called from `runBlock` // in the `VM` if we encounter a `BLOCKHASH` opcode: then a bigint is used we // need to then read the block from the canonical chain Q: is this safe? We // know it is OK if we call it from the iterator... (runBlock) - const block = await this.dbManager.getBlock(blockId, optimisticOpts) + const block = await this.dbManager.getBlock(blockId) if (block === undefined) { if (typeof blockId === 'object') { @@ -1074,6 +1137,10 @@ export class Blockchain implements BlockchainInterface { hash = await this.safeNumberToHash(blockNumber) while (hash !== false) { + const blockTd = await this.dbManager.getTotalDifficultySafe(hash, blockNumber) + if (blockTd === undefined) { + return + } ops.push(DBOp.del(DBTarget.NumberToHash, { blockNumber })) if (this.events.listenerCount('deletedCanonicalBlocks') > 0) { @@ -1174,10 +1241,12 @@ export class Blockchain implements BlockchainInterface { staleHeadBlock = true } - header = await this._getHeader(header.parentHash, --currentNumber) - if (header === undefined) { + const parentHeader = await this.dbManager.getHeaderSafe(header.parentHash, --currentNumber) + if (parentHeader === undefined) { staleHeads = [] break + } else { + header = parentHeader } } // When the stale hash is equal to the blockHash of the provided header, diff --git a/packages/blockchain/src/db/constants.ts b/packages/blockchain/src/db/constants.ts index dbe6990fcc..a15968b8db 100644 --- a/packages/blockchain/src/db/constants.ts +++ b/packages/blockchain/src/db/constants.ts @@ -28,8 +28,6 @@ const TD_SUFFIX = utf8ToBytes('t') * headerPrefix + number + numSuffix -> hash */ const NUM_SUFFIX = utf8ToBytes('n') -const OPTIMISTIC_NUM_SUFFIX = utf8ToBytes('o') - /** * blockHashPrefix + hash -> number */ @@ -56,9 +54,6 @@ const bodyKey = (n: bigint, hash: Uint8Array) => concatBytes(BODY_PREFIX, bytesB const numberToHashKey = (n: bigint) => concatBytes(HEADER_PREFIX, bytesBE8(n), NUM_SUFFIX) -const optimisticNumberToHashKey = (n: bigint) => - concatBytes(HEADER_PREFIX, bytesBE8(n), OPTIMISTIC_NUM_SUFFIX) - const hashToNumberKey = (hash: Uint8Array) => concatBytes(BLOCK_HASH_PREFIX, hash) /** @@ -73,6 +68,5 @@ export { headerKey, HEADS_KEY, numberToHashKey, - optimisticNumberToHashKey, tdKey, } diff --git a/packages/blockchain/src/db/manager.ts b/packages/blockchain/src/db/manager.ts index fd2f5b9a67..0235e60201 100644 --- a/packages/blockchain/src/db/manager.ts +++ b/packages/blockchain/src/db/manager.ts @@ -12,7 +12,7 @@ import { import { Cache } from './cache.js' import { DBOp, DBTarget } from './operation.js' -import type { DatabaseKey, OptimisticOpts } from './operation.js' +import type { DatabaseKey } from './operation.js' import type { Block, BlockBodyBytes, BlockBytes, BlockOptions } from '@ethereumjs/block' import type { Common } from '@ethereumjs/common' import type { BatchDBOp, DB, DBObject, DelBatch, PutBatch } from '@ethereumjs/util' @@ -47,7 +47,6 @@ export class DBManager { body: new Cache({ max: 256 }), numberToHash: new Cache({ max: 2048 }), hashToNumber: new Cache({ max: 2048 }), - optimisticNumberToHash: new Cache({ max: 2048 }), } } @@ -85,10 +84,7 @@ export class DBManager { * Fetches a block (header and body) given a block id, * which can be either its hash or its number. */ - async getBlock( - blockId: Uint8Array | bigint | number, - optimisticOpts?: OptimisticOpts, - ): Promise { + async getBlock(blockId: Uint8Array | bigint | number): Promise { if (typeof blockId === 'number' && Number.isInteger(blockId)) { blockId = BigInt(blockId) } @@ -102,29 +98,9 @@ export class DBManager { if (number === undefined) { return undefined } - - if (optimisticOpts?.fcUed === true) { - let optimisticHash = await this.optimisticNumberToHash(number) - if (optimisticHash === undefined && optimisticOpts.linked === true) { - optimisticHash = await this.numberToHash(number) - } - if (optimisticHash === undefined || !equalsBytes(optimisticHash, hash)) { - return undefined - } - } } else if (typeof blockId === 'bigint') { number = blockId - if (optimisticOpts !== undefined) { - if (!optimisticOpts.fcUed) { - throw Error(`Invalid fcUed optimistic block by number lookup`) - } - hash = await this.optimisticNumberToHash(blockId) - if (hash === undefined && optimisticOpts.linked === true) { - hash = await this.numberToHash(blockId) - } - } else { - hash = await this.numberToHash(blockId) - } + hash = await this.numberToHash(blockId) } else { throw new Error('Unknown blockId type') } @@ -190,6 +166,15 @@ export class DBManager { return createBlockHeaderFromBytesArray(headerValues as Uint8Array[], opts) } + async getHeaderSafe(blockHash: Uint8Array, blockNumber: bigint) { + const encodedHeader = await this.get(DBTarget.Header, { blockHash, blockNumber }) + + const opts: BlockOptions = { common: this.common, setHardfork: true } + return encodedHeader !== undefined + ? createBlockHeaderFromBytesArray(RLP.decode(encodedHeader) as Uint8Array[], opts) + : undefined + } + /** * Fetches total difficulty for a block given its hash and number. */ @@ -198,6 +183,14 @@ export class DBManager { return bytesToBigInt(RLP.decode(td) as Uint8Array) } + async getTotalDifficultySafe( + blockHash: Uint8Array, + blockNumber: bigint, + ): Promise { + const td = await this.get(DBTarget.TotalDifficulty, { blockHash, blockNumber }) + return td !== undefined ? bytesToBigInt(RLP.decode(td) as Uint8Array) : undefined + } + /** * Performs a block hash to block number lookup. */ @@ -217,11 +210,6 @@ export class DBManager { return value } - async optimisticNumberToHash(blockNumber: bigint): Promise { - const value = await this.get(DBTarget.OptimisticNumberToHash, { blockNumber }) - return value - } - /** * Fetches a key from the db. If `opts.cache` is specified * it first tries to load from cache, and on cache miss will diff --git a/packages/blockchain/src/db/operation.ts b/packages/blockchain/src/db/operation.ts index 5dd3e40d54..1c7a068e28 100644 --- a/packages/blockchain/src/db/operation.ts +++ b/packages/blockchain/src/db/operation.ts @@ -8,13 +8,12 @@ import { hashToNumberKey, headerKey, numberToHashKey, - optimisticNumberToHashKey, tdKey, } from './constants.js' import type { CacheMap } from './manager.js' -export type OptimisticOpts = { fcUed: boolean; linked?: boolean } +export type PutOpts = { notCanonical?: boolean; parentTd?: bigint } export enum DBTarget { Heads, @@ -28,7 +27,6 @@ export enum DBTarget { CliqueSignerStates, CliqueVotes, CliqueBlockSigners, - OptimisticNumberToHash, } /** @@ -92,11 +90,6 @@ export class DBOp { this.cacheString = 'numberToHash' break } - case DBTarget.OptimisticNumberToHash: { - this.baseDBOp.key = optimisticNumberToHashKey(key!.blockNumber!) - this.cacheString = 'optimisticNumberToHash' - break - } case DBTarget.TotalDifficulty: { this.baseDBOp.key = tdKey(key!.blockNumber!, key!.blockHash!) this.cacheString = 'td' diff --git a/packages/blockchain/src/types.ts b/packages/blockchain/src/types.ts index 69843c1c3e..7d2f6f3804 100644 --- a/packages/blockchain/src/types.ts +++ b/packages/blockchain/src/types.ts @@ -1,4 +1,4 @@ -import type { OptimisticOpts } from './db/operation.js' +import type { PutOpts } from './db/operation.js' import type { Blockchain } from './index.js' import type { Block, BlockHeader } from '@ethereumjs/block' import type { Common, ConsensusAlgorithm } from '@ethereumjs/common' @@ -17,7 +17,7 @@ export interface BlockchainInterface { * * @param block - The block to be added to the blockchain. */ - putBlock(block: Block, optimisticOpts?: OptimisticOpts): Promise + putBlock(block: Block, opts?: PutOpts): Promise /** * Deletes a block from the blockchain. All child blocks in the chain are @@ -30,7 +30,7 @@ export interface BlockchainInterface { /** * Returns a block by its hash or number. */ - getBlock(blockId: Uint8Array | number | bigint, optimisticOpts?: OptimisticOpts): Promise + getBlock(blockId: Uint8Array | number | bigint): Promise /** * Iterates through blocks starting at the specified iterator head and calls diff --git a/packages/client/src/service/skeleton.ts b/packages/client/src/service/skeleton.ts index 9d547f8166..eabbb6af8b 100644 --- a/packages/client/src/service/skeleton.ts +++ b/packages/client/src/service/skeleton.ts @@ -208,8 +208,12 @@ export class Skeleton extends MetaDBManager { // if its genesis we are linked if (tail === BIGINT_0) return true if (tail <= this.chain.blocks.height + BIGINT_1) { - const nextBlock = await this.chain.getBlock(tail - BIGINT_1) - const linked = equalsBytes(next, nextBlock.hash()) + // we here want the non optimistic block from canonical chain + const nextBlock = await this.chain.blockchain.getBlock(tail - BIGINT_1) + const tailTD = await this.chain.blockchain + .getTotalDifficulty(nextBlock.hash(), tail - BIGINT_1) + .catch((_e) => undefined) + const linked = equalsBytes(next, nextBlock.hash()) && tailTD !== undefined if (linked && this.status.progress.subchains.length > 1) { // Remove all other subchains as no more relevant const junkedSubChains = this.status.progress.subchains.splice(1) @@ -236,22 +240,39 @@ export class Skeleton extends MetaDBManager { /** * Try fast forwarding the chain head to the number */ - private async fastForwardHead(lastchain: SkeletonSubchain, target: bigint) { + private async fastForwardHead( + lastchain: SkeletonSubchain, + target: bigint, + targetHash: Uint8Array, + ) { const head = lastchain.head let headBlock = await this.getBlock(head, true) if (headBlock === undefined) { return } + const fastForwardBlocks = [] + for (let newHead = head + BIGINT_1; newHead <= target; newHead += BIGINT_1) { - const newBlock = await this.getBlock(newHead, true) + const newBlockHash = await this.get(DBKey.SkeletonForwardNumberToHash, bigIntToBytes(newHead)) + const newBlock = newBlockHash + ? await this.chain.blockchain.getBlock(newBlockHash).catch((_e) => undefined) + : undefined if (newBlock === undefined || !equalsBytes(newBlock.header.parentHash, headBlock.hash())) { // Head can't be updated forward break } headBlock = newBlock + fastForwardBlocks.push(newBlock) + } + + if (equalsBytes(headBlock.hash(), targetHash)) { + for (const block of fastForwardBlocks) { + await this.putBlock(block) + } + lastchain.head = headBlock.header.number } - lastchain.head = headBlock.header.number + this.config.logger.debug( `lastchain head fast forwarded from=${head} to=${lastchain.head} tail=${lastchain.tail}`, ) @@ -335,7 +356,7 @@ export class Skeleton extends MetaDBManager { } } else if (lastchain.head + BIGINT_1 < number) { if (force) { - await this.fastForwardHead(lastchain, number - BIGINT_1) + await this.fastForwardHead(lastchain, number - BIGINT_1, head.header.parentHash) // If its still less than number then its gapped head if (lastchain.head + BIGINT_1 < number) { this.config.logger.debug( @@ -547,7 +568,7 @@ export class Skeleton extends MetaDBManager { } // only add to unfinalized cache if this is announcement and before canonical head - await this.putBlock(head, !force && head.header.number <= subchain0Head) + await this.putBlock(head, !force) if (init) { await this.trySubChainsMerge() @@ -1153,7 +1174,6 @@ export class Skeleton extends MetaDBManager { async fillCanonicalChain() { if (this.filling) return this.filling = true - let canonicalHead = this.chain.blocks.height const subchain = this.status.progress.subchains[0]! if (this.status.canonicalHeadReset) { @@ -1172,7 +1192,7 @@ export class Skeleton extends MetaDBManager { `Resetting canonicalHead for fillCanonicalChain from=${canonicalHead} to=${newHead}`, ) canonicalHead = newHead - await this.chain.resetCanonicalHead(canonicalHead) + // await this.chain.resetCanonicalHead(canonicalHead) } // update in lock so as to not conflict/overwrite setHead/putBlock updates await this.runWithLock(async () => { @@ -1353,7 +1373,24 @@ export class Skeleton extends MetaDBManager { */ private async putBlock(block: Block, onlyUnfinalized: boolean = false): Promise { // Serialize the block with its hardfork so that its easy to load the block latter - await this.chain.blockchain.putBlock(block, { fcUed: !onlyUnfinalized }) + this.config.logger.debug( + `blockchain putBlock number=${block.header.number} hash=${short(block.hash())} onlyUnfinalized=${onlyUnfinalized}`, + ) + await this.chain.blockchain.putBlock(block, { notCanonical: onlyUnfinalized }) + + if (onlyUnfinalized) { + // save the forward annoucement for fast forwarding if lucky + const subchain0 = this.status.progress.subchains[0] + if (subchain0 === undefined || block.header.number > subchain0.head) { + await this.put( + DBKey.SkeletonForwardNumberToHash, + bigIntToBytes(block.header.number), + block.hash(), + ) + } + } else { + await this.chain.update() + } return true } @@ -1371,13 +1408,33 @@ export class Skeleton extends MetaDBManager { /** * Gets a block from the skeleton or canonical db by number. */ - async getBlock(number: bigint, onlyCanonical = false): Promise { - return this.chain.blockchain.dbManager - .getBlock(number, { - fcUed: onlyCanonical, - linked: this.status.linked, - }) - .catch((_e) => undefined) + async getBlock(number: bigint, fcUed: boolean = false): Promise { + const subchain0 = this.status.progress.subchains[0] + this.config.logger.debug( + `getBlock subchain0: head=${subchain0.head} tail=${subchain0.tail} next=${short(subchain0.next ?? 'na')} number=${number}`, + ) + + let block + if ( + !this.status.linked && + (subchain0 === undefined || number < subchain0.tail || number > subchain0.head) + ) { + block = undefined + } else { + block = await this.chain.blockchain.dbManager.getBlock(number).catch((_e) => undefined) + } + + if (block === undefined && fcUed === false) { + const blockHash = await this.get(DBKey.SkeletonForwardNumberToHash, bigIntToBytes(number)) + block = blockHash + ? await this.chain.blockchain.getBlock(blockHash).catch((_e) => undefined) + : undefined + } + + this.config.logger.debug( + `found block number=${number} with hash=${block ? short(block.hash()) : undefined}`, + ) + return block } /** @@ -1387,10 +1444,17 @@ export class Skeleton extends MetaDBManager { hash: Uint8Array, onlyCanonical: boolean = false, ): Promise { - return this.chain.blockchain.dbManager.getBlock(hash, { - fcUed: onlyCanonical, - linked: this.status.linked, - }) + const block = await this.chain.blockchain.dbManager.getBlock(hash).catch((_e) => undefined) + if (onlyCanonical && block !== undefined) { + const canonicalBlock = await this.getBlock(block.header.number) + if (canonicalBlock === undefined || !equalsBytes(block.hash(), canonicalBlock.hash())) { + return undefined + } else { + return canonicalBlock + } + } else { + return block + } } /** diff --git a/packages/client/src/util/metaDBManager.ts b/packages/client/src/util/metaDBManager.ts index a553cc8c44..9b04a38d13 100644 --- a/packages/client/src/util/metaDBManager.ts +++ b/packages/client/src/util/metaDBManager.ts @@ -21,6 +21,7 @@ export enum DBKey { SkeletonStatus, SkeletonUnfinalizedBlockByHash, Preimage, + SkeletonForwardNumberToHash, } export interface MetaDBManagerOptions { diff --git a/packages/client/test/sync/skeleton.spec.ts b/packages/client/test/sync/skeleton.spec.ts index 4ff4a98df4..42cff8c1a0 100644 --- a/packages/client/test/sync/skeleton.spec.ts +++ b/packages/client/test/sync/skeleton.spec.ts @@ -1,11 +1,6 @@ import { createBlock } from '@ethereumjs/block' -import { - Common, - Mainnet, - createCommonFromGethGenesis, - createCustomCommon, -} from '@ethereumjs/common' -import { equalsBytes, utf8ToBytes } from '@ethereumjs/util' +import { Mainnet, createCommonFromGethGenesis, createCustomCommon } from '@ethereumjs/common' +import { equalsBytes, hexToBytes, utf8ToBytes } from '@ethereumjs/util' import { MemoryLevel } from 'memory-level' import { assert, describe, it } from 'vitest' @@ -24,10 +19,14 @@ type Subchain = { tail: bigint } -const common = new Common({ chain: Mainnet }) -const block49 = createBlock({ header: { number: 49 } }, { common, setHardfork: true }) +const common = createCustomCommon(mergeGenesisParams, Mainnet, { name: 'post-merge' }) +const block48 = createBlock({ header: { number: 48 } }, { common, setHardfork: true }) +const block49 = createBlock( + { header: { number: 49, parentHash: block48.hash() } }, + { common, setHardfork: true }, +) const block49B = createBlock( - { header: { number: 49, extraData: utf8ToBytes('B') } }, + { header: { number: 49, parentHash: block48.hash(), extraData: utf8ToBytes('B') } }, { common, setHardfork: true }, ) const block50 = createBlock( @@ -408,6 +407,7 @@ describe('[Skeleton] / setHead', async () => { assert.ok(true, `test ${testCaseIndex}: subchain[${i}] matched`) } } + skeleton.logSyncStatus('specLog') }) } @@ -436,7 +436,7 @@ describe('[Skeleton] / setHead', async () => { }) it('should init/setHead properly from genesis', async () => { - const config = new Config({ common }) + const config = new Config({ common, logger: getLogger({ logLevel: 'debug' }) }) const chain = await Chain.create({ config }) ;(chain.blockchain as any)._validateBlocks = false const skeleton = new Skeleton({ chain, config, metaDB: new MemoryLevel() }) @@ -451,8 +451,18 @@ describe('[Skeleton] / setHead', async () => { { header: { number: 2, parentHash: block1.hash(), difficulty: 100 } }, { common, setHardfork: true }, ) + + // block3 from an alternate chain const block3 = createBlock( - { header: { number: 3, difficulty: 100 } }, + { + header: { + number: 3, + difficulty: 0, + parentHash: hexToBytes( + '0xa321d27cd2743617c1c1b0d7ecb607dd14febcdfca8f01b79c3f0249505ea069', + ), + }, + }, { common, setHardfork: true }, ) @@ -534,11 +544,17 @@ describe('[Skeleton] / setHead', async () => { 'head should be set to second block', ) assert.equal(skeleton.isLinked(), true, 'subchain status should stay linked') + skeleton.logSyncStatus('specLog') reorg = await skeleton.setHead(block3, true) + skeleton.logSyncStatus('specLog') assert.equal(reorg, true, 'should not extend on invalid third block') // since its not a forced update so shouldn't affect subchain status - assert.equal(skeleton['status'].progress.subchains.length, 2, 'new subchain should be created') + assert.equal( + skeleton['status'].progress.subchains.length, + 1, + 'new subchain should be created and previous truncated', + ) assert.equal( skeleton['status'].progress.subchains[0].head, BigInt(3), @@ -563,16 +579,18 @@ describe('[Skeleton] / setHead', async () => { { header: { number: 2, parentHash: block1.hash(), difficulty: 100 } }, { common, setHardfork: true }, ) + + // block3 onwards is POS const block3 = createBlock( - { header: { number: 3, parentHash: block2.hash(), difficulty: 100 } }, + { header: { number: 3, parentHash: block2.hash() } }, { common, setHardfork: true }, ) const block4 = createBlock( - { header: { number: 4, parentHash: block3.hash(), difficulty: 100 } }, + { header: { number: 4, parentHash: block3.hash() } }, { common, setHardfork: true }, ) const block5 = createBlock( - { header: { number: 5, parentHash: block4.hash(), difficulty: 100 } }, + { header: { number: 5, parentHash: block4.hash() } }, { common, setHardfork: true }, ) @@ -588,7 +606,9 @@ describe('[Skeleton] / setHead', async () => { BigInt(4), 'canonical height should update after being linked', ) + skeleton.logSyncStatus('specLog') await skeleton.setHead(block5, false) + skeleton.logSyncStatus('specLog') await wait(200) assert.equal( chain.blocks.height, @@ -621,10 +641,16 @@ describe('[Skeleton] / setHead', async () => { )} should be available even after filling canonical chain`, ) } + + skeleton.logSyncStatus('specLog') }) it('should fill the canonical chain after being linked to a canonical block past genesis', async () => { - const config = new Config({ common, engineNewpayloadMaxExecute: 10 }) + const config = new Config({ + common, + engineNewpayloadMaxExecute: 10, + logger: getLogger({ logLevel: 'debug' }), + }) const chain = await Chain.create({ config }) ;(chain.blockchain as any)._validateBlocks = false @@ -643,23 +669,26 @@ describe('[Skeleton] / setHead', async () => { { common, setHardfork: true }, ) const block3 = createBlock( - { header: { number: 3, parentHash: block2.hash(), difficulty: 100 } }, + { header: { number: 3, parentHash: block2.hash() } }, { common, setHardfork: true }, ) const block4 = createBlock( - { header: { number: 4, parentHash: block3.hash(), difficulty: 100 } }, + { header: { number: 4, parentHash: block3.hash() } }, { common, setHardfork: true }, ) const block5 = createBlock( - { header: { number: 5, parentHash: block4.hash(), difficulty: 100 } }, + { header: { number: 5, parentHash: block4.hash() } }, { common, setHardfork: true }, ) await chain.putBlocks([block1, block2]) + skeleton.logSyncStatus('specLog') await skeleton.initSync(block4) + skeleton.logSyncStatus('specLog') assert.equal(chain.blocks.height, BigInt(2), 'canonical height should be at block 2') await skeleton.putBlocks([block3]) await wait(200) + skeleton.logSyncStatus('specLog') assert.equal( chain.blocks.height, BigInt(4), @@ -704,15 +733,15 @@ describe('[Skeleton] / setHead', async () => { skeleton['status'].linked = prevLinked const block41 = createBlock( - { header: { number: 4, parentHash: block3.hash(), difficulty: 101 } }, + { header: { number: 4, parentHash: block3.hash() } }, { common, setHardfork: true }, ) const block51 = createBlock( - { header: { number: 5, parentHash: block41.hash(), difficulty: 100 } }, + { header: { number: 5, parentHash: block41.hash() } }, { common, setHardfork: true }, ) const block61 = createBlock( - { header: { number: 6, parentHash: block51.hash(), difficulty: 100 } }, + { header: { number: 6, parentHash: block51.hash() } }, { common, setHardfork: true }, ) @@ -728,22 +757,22 @@ describe('[Skeleton] / setHead', async () => { ) assert.equal( skeleton['status'].progress.subchains[0]?.tail, - BigInt(4), + BigInt(3), 'tail should be backfilled', ) assert.equal(skeleton['status'].linked, true, 'should be linked') assert.equal(chain.blocks.height, BigInt(6), 'all blocks should be in chain') const block71 = createBlock( - { header: { number: 7, parentHash: block61.hash(), difficulty: 100 } }, + { header: { number: 7, parentHash: block61.hash() } }, { common, setHardfork: true }, ) const block81 = createBlock( - { header: { number: 8, parentHash: block71.hash(), difficulty: 100 } }, + { header: { number: 8, parentHash: block71.hash() } }, { common, setHardfork: true }, ) const block91 = createBlock( - { header: { number: 9, parentHash: block81.hash(), difficulty: 100 } }, + { header: { number: 9, parentHash: block81.hash() } }, { common, setHardfork: true }, ) @@ -771,7 +800,7 @@ describe('[Skeleton] / setHead', async () => { ) assert.equal( skeleton['status'].progress.subchains[0]?.tail, - BigInt(7), + BigInt(8), 'tail should be backfilled', ) assert.equal(skeleton['status'].linked, true, 'should be linked') @@ -786,11 +815,11 @@ describe('[Skeleton] / setHead', async () => { // do a very common reorg that happens in a network: reorged head block const block92 = createBlock( - { header: { number: 9, parentHash: block81.hash(), difficulty: 101 } }, + { header: { number: 9, parentHash: block81.hash(), gasLimit: 999 } }, { common, setHardfork: true }, ) const block102 = createBlock( - { header: { number: 10, parentHash: block92.hash(), difficulty: 100 } }, + { header: { number: 10, parentHash: block92.hash(), gasLimit: 999 } }, { common, setHardfork: true }, ) @@ -828,6 +857,7 @@ describe('[Skeleton] / setHead', async () => { common, accountCache: 10000, storageCache: 1000, + logger: getLogger({ logLevel: 'debug' }), }) const chain = await Chain.create({ config }) ;(chain.blockchain as any)._validateBlocks = false @@ -868,10 +898,15 @@ describe('[Skeleton] / setHead', async () => { await skeleton.initSync(block4InvalidPoS) // the following putBlocks will fail ad block3 is pow block - await skeleton.putBlocks([block3PoW, block2]) + try { + await skeleton.putBlocks([block3PoW]) + assert.fail('should have failed putting invalid pow') + // eslint-disable-next-line no-empty + } catch (_e) {} assert.equal(chain.blocks.height, BigInt(0), 'canonical height should be at genesis') - await skeleton.putBlocks([block1]) + skeleton.logSyncStatus('specLog') + await skeleton.putBlocks([block2, block1]) await wait(200) assert.equal( chain.blocks.height, @@ -915,4 +950,54 @@ describe('[Skeleton] / setHead', async () => { await wait(200) assert.equal(skeleton.bounds().head, BigInt(5), 'should update to new height') }) + + it('should fast forward the chain after annoucement', async () => { + const config = new Config({ common, logger: getLogger({ logLevel: 'debug' }) }) + const chain = await Chain.create({ config }) + ;(chain.blockchain as any)._validateBlocks = false + const skeleton = new Skeleton({ chain, config, metaDB: new MemoryLevel() }) + await chain.open() + + const genesis = await chain.getBlock(BigInt(0)) + const block1 = createBlock( + { header: { number: 1, parentHash: genesis.hash(), difficulty: 100 } }, + { common, setHardfork: true }, + ) + const block2 = createBlock( + { header: { number: 2, parentHash: block1.hash(), difficulty: 100 } }, + { common, setHardfork: true }, + ) + + // block3 onwards is POS + const block3 = createBlock( + { header: { number: 3, parentHash: block2.hash() } }, + { common, setHardfork: true }, + ) + const block4 = createBlock( + { header: { number: 4, parentHash: block3.hash() } }, + { common, setHardfork: true }, + ) + const block5 = createBlock( + { header: { number: 5, parentHash: block4.hash() } }, + { common, setHardfork: true }, + ) + + await skeleton.open() + + await skeleton.initSync(block2) + await skeleton.setHead(block3, false) + await skeleton.setHead(block4, false) + await skeleton.setHead(block5, true) + assert.equal( + skeleton['status'].progress.subchains[0]?.head, + BigInt(5), + 'subchain should be fastforwarded', + ) + assert.equal( + skeleton['status'].progress.subchains[0]?.tail, + BigInt(2), + 'subchain should be fastforwarded', + ) + skeleton.logSyncStatus('specLog') + }) }) From d9e329744bf38f2b30f21ff48dd3be0ba0b9dc5b Mon Sep 17 00:00:00 2001 From: harkamal Date: Mon, 2 Sep 2024 16:36:24 +0530 Subject: [PATCH 07/10] modify putopts to be more understanable and add the optimistic block insertion test --- packages/blockchain/src/blockchain.ts | 12 +- packages/blockchain/src/db/operation.ts | 2 +- .../blockchain/test/4444Optimistic.spec.ts | 117 ++++++++++++++++++ packages/client/src/service/skeleton.ts | 2 +- 4 files changed, 125 insertions(+), 8 deletions(-) create mode 100644 packages/blockchain/test/4444Optimistic.spec.ts diff --git a/packages/blockchain/src/blockchain.ts b/packages/blockchain/src/blockchain.ts index 1daf00640c..a1e8b76c3b 100644 --- a/packages/blockchain/src/blockchain.ts +++ b/packages/blockchain/src/blockchain.ts @@ -384,7 +384,7 @@ export class Blockchain implements BlockchainInterface { // check if head is still canonical i.e. if this is a block insertion on tail or reinsertion // on the same canonical chain let isHeadChainStillCanonical - if (opts?.notCanonical !== true) { + if (opts?.canonical !== false) { const childHeaderHash = await this.dbManager.numberToHash(blockNumber + BIGINT_1) if (childHeaderHash !== undefined) { const childHeader = await this.dbManager @@ -417,11 +417,11 @@ export class Blockchain implements BlockchainInterface { } } - // 1. if notCanonical is explicit true then just dump the block - // 2. if notCanonical is explicit false then apply that even for the pow/poa blocks + // 1. if canonical is explicit false then just dump the block + // 2. if canonical is explicit true then apply that even for the pow/poa blocks // if they are optimistic, i.e. can't apply the normal rule - // 3. if notCanonical is not defined, then apply normal rules - if (opts?.notCanonical === true) { + // 3. if canonical is not defined, then apply normal rules + if (opts?.canonical === false) { if (parentTd !== undefined) { const td = header.difficulty + parentTd dbOps = dbOps.concat(DBSetTD(td, blockNumber, blockHash)) @@ -437,7 +437,7 @@ export class Blockchain implements BlockchainInterface { if ( !block.isGenesis() && block.common.consensusType() !== ConsensusType.ProofOfStake && - opts?.notCanonical !== false + opts?.canonical === undefined ) { throw Error( `Invalid parentTd=${parentTd} for consensus=${block.common.consensusType()} putBlockOrHeader`, diff --git a/packages/blockchain/src/db/operation.ts b/packages/blockchain/src/db/operation.ts index 1c7a068e28..47ac98597e 100644 --- a/packages/blockchain/src/db/operation.ts +++ b/packages/blockchain/src/db/operation.ts @@ -13,7 +13,7 @@ import { import type { CacheMap } from './manager.js' -export type PutOpts = { notCanonical?: boolean; parentTd?: bigint } +export type PutOpts = { canonical?: boolean; parentTd?: bigint } export enum DBTarget { Heads, diff --git a/packages/blockchain/test/4444Optimistic.spec.ts b/packages/blockchain/test/4444Optimistic.spec.ts new file mode 100644 index 0000000000..88f37ac659 --- /dev/null +++ b/packages/blockchain/test/4444Optimistic.spec.ts @@ -0,0 +1,117 @@ +import { createBlock } from '@ethereumjs/block' +import { Mainnet, createCustomCommon } from '@ethereumjs/common' +import { equalsBytes } from '@ethereumjs/util' +import { assert, describe, expect, it } from 'vitest' + +import mergeGenesisParams from '../../client/test/testdata/common/mergeTestnet.json' +import { createBlockchain } from '../src/index.js' + +describe('[Blockchain]: 4444/optimistic spec', () => { + const common = createCustomCommon(mergeGenesisParams, Mainnet, { name: 'post-merge' }) + const genesisBlock = createBlock({ header: { extraData: new Uint8Array(97) } }, { common }) + + const block1 = createBlock( + { header: { number: 1, parentHash: genesisBlock.hash(), difficulty: 100 } }, + { common }, + ) + const block2 = createBlock( + { header: { number: 2, parentHash: block1.hash(), difficulty: 100 } }, + { common }, + ) + const block3PoW = createBlock( + { header: { number: 3, parentHash: block2.hash(), difficulty: 100 } }, + { common }, + ) + const block3PoS = createBlock( + { header: { number: 3, parentHash: block2.hash() } }, + { common, setHardfork: true }, + ) + const block4InvalidPoS = createBlock( + { header: { number: 4, parentHash: block3PoW.hash() } }, + { common, setHardfork: true }, + ) + const block4 = createBlock( + { header: { number: 4, parentHash: block3PoS.hash() } }, + { common, setHardfork: true }, + ) + const block5 = createBlock( + { header: { number: 5, parentHash: block4.hash() } }, + { common, setHardfork: true }, + ) + const block6 = createBlock( + { header: { number: 5, parentHash: block5.hash() } }, + { common, setHardfork: true }, + ) + const block7 = createBlock( + { header: { number: 5, parentHash: block6.hash() } }, + { common, setHardfork: true }, + ) + const block8 = createBlock( + { header: { number: 5, parentHash: block7.hash() } }, + { common, setHardfork: true }, + ) + + const block51 = createBlock( + { header: { number: 5, parentHash: block4.hash(), gasLimit: 999 } }, + { common, setHardfork: true }, + ) + const block61 = createBlock( + { header: { number: 5, parentHash: block51.hash() } }, + { common, setHardfork: true }, + ) + const block71 = createBlock( + { header: { number: 5, parentHash: block61.hash() } }, + { common, setHardfork: true }, + ) + const block81 = createBlock( + { header: { number: 5, parentHash: block71.hash() } }, + { common, setHardfork: true }, + ) + + it('should be able to insert block optimistically', async () => { + const blockchain = await createBlockchain({ genesisBlock, common, validateBlocks: false }) + await blockchain.putBlock(block51) + await blockchain.putBlock(block4) + await blockchain.putBlock(block3PoS) + + let dbBlock = await blockchain.getBlock(block51.hash()).catch((_e) => undefined) + assert.equal(dbBlock !== undefined, true, 'optimistic block by hash should be found') + dbBlock = await blockchain.getBlock(block51.header.number).catch((_e) => undefined) + assert.equal(dbBlock !== undefined, true, 'optimistic block by number should be found') + + // pow block should be allowed to inserted only in backfill mode + await blockchain.putBlock(block2).catch((_e) => undefined) + dbBlock = await blockchain.getBlock(block2.header.number).catch((_e) => undefined) + assert.equal( + dbBlock === undefined, + true, + 'pow block should not be inserted with complete ancestor in blockchain', + ) + + await blockchain.putBlock(block2, { canonical: true }) + dbBlock = await blockchain.getBlock(block2.header.number).catch((_e) => undefined) + assert.equal( + dbBlock !== undefined, + true, + 'pow block should be inserted in optimistic block if marked canonical', + ) + + let headBlock = await blockchain.getCanonicalHeadBlock() + assert.equal(headBlock.header.number, genesisBlock.header.number, 'head still at genesis block') + + await blockchain.putBlock(block1, { canonical: true }) + headBlock = await blockchain.getCanonicalHeadBlock() + assert.equal(headBlock.header.number, block1.header.number, 'head should move') + + for (const block of [block2, block3PoS, block4, block5]) { + dbBlock = await blockchain.getBlock(block.header.number).catch((_e) => undefined) + assert.equal(dbBlock !== undefined, true, 'the forward index should still be untouched') + } + + for (const block of [block2, block3PoS, block4, block5]) { + await blockchain.putBlock(block).catch((_e) => undefined) + } + headBlock = await blockchain.getCanonicalHeadBlock() + assert.equal(headBlock.header.number, block5.header.number, 'head should move') + }) +}) diff --git a/packages/client/src/service/skeleton.ts b/packages/client/src/service/skeleton.ts index eabbb6af8b..98fbd09b8c 100644 --- a/packages/client/src/service/skeleton.ts +++ b/packages/client/src/service/skeleton.ts @@ -1376,7 +1376,7 @@ export class Skeleton extends MetaDBManager { this.config.logger.debug( `blockchain putBlock number=${block.header.number} hash=${short(block.hash())} onlyUnfinalized=${onlyUnfinalized}`, ) - await this.chain.blockchain.putBlock(block, { notCanonical: onlyUnfinalized }) + await this.chain.blockchain.putBlock(block, { canonical: !onlyUnfinalized }) if (onlyUnfinalized) { // save the forward annoucement for fast forwarding if lucky From 7bd20c22c7a92658e60f1a6ad1a5e38c2a8ebe30 Mon Sep 17 00:00:00 2001 From: harkamal Date: Tue, 3 Sep 2024 22:49:42 +0530 Subject: [PATCH 08/10] add 4444 and optimistic spec debug and get it working --- packages/blockchain/src/blockchain.ts | 36 +-- packages/blockchain/src/db/manager.ts | 3 + .../blockchain/test/4444Optimistic.spec.ts | 243 ++++++++++++++++-- 3 files changed, 239 insertions(+), 43 deletions(-) diff --git a/packages/blockchain/src/blockchain.ts b/packages/blockchain/src/blockchain.ts index a1e8b76c3b..bac5292857 100644 --- a/packages/blockchain/src/blockchain.ts +++ b/packages/blockchain/src/blockchain.ts @@ -383,21 +383,21 @@ export class Blockchain implements BlockchainInterface { // check if head is still canonical i.e. if this is a block insertion on tail or reinsertion // on the same canonical chain - let isHeadChainStillCanonical - if (opts?.canonical !== false) { - const childHeaderHash = await this.dbManager.numberToHash(blockNumber + BIGINT_1) - if (childHeaderHash !== undefined) { - const childHeader = await this.dbManager - .getHeaderSafe(childHeaderHash, blockNumber + BIGINT_1) - .catch((_e) => undefined) - isHeadChainStillCanonical = - childHeader !== undefined && equalsBytes(childHeader.parentHash, blockHash) - } else { - isHeadChainStillCanonical = false - } - } else { - isHeadChainStillCanonical = true - } + // let isHeadChainStillCanonical + // if (opts?.canonical !== false) { + // const childHeaderHash = await this.dbManager.numberToHash(blockNumber + BIGINT_1) + // if (childHeaderHash !== undefined) { + // const childHeader = await this.dbManager + // .getHeaderSafe(childHeaderHash, blockNumber + BIGINT_1) + // .catch((_e) => undefined) + // isHeadChainStillCanonical = + // childHeader !== undefined && equalsBytes(childHeader.parentHash, blockHash) + // } else { + // isHeadChainStillCanonical = false + // } + // } else { + // isHeadChainStillCanonical = true + // } let dbOps: DBOp[] = [] dbOps = dbOps.concat(DBSetBlockOrHeader(item)) @@ -421,7 +421,7 @@ export class Blockchain implements BlockchainInterface { // 2. if canonical is explicit true then apply that even for the pow/poa blocks // if they are optimistic, i.e. can't apply the normal rule // 3. if canonical is not defined, then apply normal rules - if (opts?.canonical === false) { + if (opts?.canonical === false || (opts?.canonical === undefined && !isComplete)) { if (parentTd !== undefined) { const td = header.difficulty + parentTd dbOps = dbOps.concat(DBSetTD(td, blockNumber, blockHash)) @@ -1241,7 +1241,9 @@ export class Blockchain implements BlockchainInterface { staleHeadBlock = true } - const parentHeader = await this.dbManager.getHeaderSafe(header.parentHash, --currentNumber) + const parentHeader = await this.dbManager + .getHeader(header.parentHash, --currentNumber) + .catch((_e) => undefined) if (parentHeader === undefined) { staleHeads = [] break diff --git a/packages/blockchain/src/db/manager.ts b/packages/blockchain/src/db/manager.ts index 0235e60201..015e06a97e 100644 --- a/packages/blockchain/src/db/manager.ts +++ b/packages/blockchain/src/db/manager.ts @@ -180,6 +180,9 @@ export class DBManager { */ async getTotalDifficulty(blockHash: Uint8Array, blockNumber: bigint): Promise { const td = await this.get(DBTarget.TotalDifficulty, { blockHash, blockNumber }) + if (td === undefined) { + throw Error(`totalDifficulty not found`) + } return bytesToBigInt(RLP.decode(td) as Uint8Array) } diff --git a/packages/blockchain/test/4444Optimistic.spec.ts b/packages/blockchain/test/4444Optimistic.spec.ts index 88f37ac659..ef9015d325 100644 --- a/packages/blockchain/test/4444Optimistic.spec.ts +++ b/packages/blockchain/test/4444Optimistic.spec.ts @@ -1,7 +1,7 @@ import { createBlock } from '@ethereumjs/block' import { Mainnet, createCustomCommon } from '@ethereumjs/common' import { equalsBytes } from '@ethereumjs/util' -import { assert, describe, expect, it } from 'vitest' +import { assert, describe, it } from 'vitest' import mergeGenesisParams from '../../client/test/testdata/common/mergeTestnet.json' import { createBlockchain } from '../src/index.js' @@ -12,24 +12,16 @@ describe('[Blockchain]: 4444/optimistic spec', () => { const block1 = createBlock( { header: { number: 1, parentHash: genesisBlock.hash(), difficulty: 100 } }, - { common }, + { common, setHardfork: true }, ) const block2 = createBlock( { header: { number: 2, parentHash: block1.hash(), difficulty: 100 } }, - { common }, - ) - const block3PoW = createBlock( - { header: { number: 3, parentHash: block2.hash(), difficulty: 100 } }, - { common }, + { common, setHardfork: true }, ) const block3PoS = createBlock( { header: { number: 3, parentHash: block2.hash() } }, { common, setHardfork: true }, ) - const block4InvalidPoS = createBlock( - { header: { number: 4, parentHash: block3PoW.hash() } }, - { common, setHardfork: true }, - ) const block4 = createBlock( { header: { number: 4, parentHash: block3PoS.hash() } }, { common, setHardfork: true }, @@ -39,15 +31,15 @@ describe('[Blockchain]: 4444/optimistic spec', () => { { common, setHardfork: true }, ) const block6 = createBlock( - { header: { number: 5, parentHash: block5.hash() } }, + { header: { number: 6, parentHash: block5.hash() } }, { common, setHardfork: true }, ) const block7 = createBlock( - { header: { number: 5, parentHash: block6.hash() } }, + { header: { number: 7, parentHash: block6.hash() } }, { common, setHardfork: true }, ) const block8 = createBlock( - { header: { number: 5, parentHash: block7.hash() } }, + { header: { number: 8, parentHash: block7.hash() } }, { common, setHardfork: true }, ) @@ -56,27 +48,169 @@ describe('[Blockchain]: 4444/optimistic spec', () => { { common, setHardfork: true }, ) const block61 = createBlock( - { header: { number: 5, parentHash: block51.hash() } }, + { header: { number: 6, parentHash: block51.hash() } }, { common, setHardfork: true }, ) const block71 = createBlock( - { header: { number: 5, parentHash: block61.hash() } }, + { header: { number: 7, parentHash: block61.hash() } }, { common, setHardfork: true }, ) const block81 = createBlock( - { header: { number: 5, parentHash: block71.hash() } }, + { header: { number: 8, parentHash: block71.hash() } }, { common, setHardfork: true }, ) + it('should allow optimistic non canonical insertion unless specified explictly', async () => { + let dbBlock + const blockchain = await createBlockchain({ genesisBlock, common, validateBlocks: false }) + + // randomly insert the blocks with gaps will be inserted as non canonical + for (const block of [block4, block3PoS, block7, block2]) { + // without explicit flag their insertion will fail as parent not present and canonicality + // can't be established + await blockchain.putBlock(block).catch((_e) => { + undefined + }) + dbBlock = await blockchain.getBlock(block.hash()).catch((_e) => undefined) + assert.equal(dbBlock !== undefined, true, 'block should be inserted') + + // however the block by number should still not fetch it + dbBlock = await blockchain.getBlock(block.header.number).catch((_e) => undefined) + assert.equal(dbBlock === undefined, true, 'block number index should not exit') + } + const headBlock = await blockchain.getCanonicalHeadBlock() + assert.equal( + headBlock.header.number, + genesisBlock.header.number, + 'head should still be at genesis', + ) + }) + + it('should allow explicit non canonical insertion', async () => { + let dbBlock, headBlock + const blockchain = await createBlockchain({ genesisBlock, common, validateBlocks: false }) + + for (const block of [block1, block2, block3PoS, block4, block5, block6, block7, block8]) { + await blockchain.putBlock(block, { canonical: false }).catch((_e) => { + undefined + }) + } + + for (const block of [block1, block2, block3PoS, block4, block5, block6, block7, block8]) { + dbBlock = await blockchain.getBlock(block.hash()).catch((_e) => undefined) + assert.equal(dbBlock !== undefined, true, 'block should be inserted') + + const td = await blockchain.getTotalDifficulty(block.hash()).catch((_e) => undefined) + assert.equal(td !== undefined, true, 'block should be marked complete') + + // however the block by number should still not fetch it + dbBlock = await blockchain.getBlock(block.header.number).catch((_e) => undefined) + assert.equal(dbBlock === undefined, true, 'block number index should not exit') + } + + // head should not move + headBlock = await blockchain.getCanonicalHeadBlock() + assert.equal( + headBlock.header.number, + genesisBlock.header.number, + 'head should still be at genesis', + ) + + // however normal insertion should respect canonicality updates + for (const block of [block1, block2, block3PoS, block4, block5, block6, block7, block8]) { + await blockchain.putBlock(block).catch((_e) => { + undefined + }) + } + headBlock = await blockchain.getCanonicalHeadBlock() + assert.equal(headBlock.header.number, block8.header.number, 'head should still be at genesis') + + for (const block of [block1, block2, block3PoS, block4, block5, block6, block7, block8]) { + // however the block by number should still not fetch it + dbBlock = await blockchain.getBlock(block.header.number).catch((_e) => undefined) + assert.equal(dbBlock !== undefined, true, 'block number index should not exit') + } + + // since block61 doesn't has the parent block51 inserted yet, they will still be inserted as non + // canonical + for (const block of [block61, block71]) { + await blockchain.putBlock(block).catch((_e) => { + undefined + }) + } + // verify + headBlock = await blockchain.getCanonicalHeadBlock() + assert.equal(headBlock.header.number, block8.header.number, 'head should still be at genesis') + + for (const block of [block61, block71]) { + // however the block by number should still not fetch it + dbBlock = await blockchain.getBlock(block.header.number).catch((_e) => undefined) + assert.equal( + equalsBytes(dbBlock.hash(), block.hash()), + false, + 'block number index should not exit', + ) + + const td = await blockchain.getTotalDifficulty(block.hash()).catch((_e) => undefined) + assert.equal(td, undefined, 'block should be not be marked complete') + } + + // insert the side chain blocks as non canonical and they should not impact the current canonical chain + for (const block of [block51, block61, block71]) { + await blockchain.putBlock(block, { canonical: false }).catch((_e) => { + undefined + }) + } + + headBlock = await blockchain.getCanonicalHeadBlock() + assert.equal(headBlock.header.number, block8.header.number, 'head should still be at block8') + assert.equal( + equalsBytes(headBlock.hash(), block8.hash()), + true, + 'head should still be at block 8', + ) + + for (const block of [block1, block2, block3PoS, block4, block5, block6, block7, block8]) { + // however the block by number should still not fetch it + dbBlock = await blockchain.getBlock(block.header.number).catch((_e) => undefined) + assert.equal(dbBlock !== undefined, true, 'block number index should not exit') + + const td = await blockchain.getTotalDifficulty(block.hash()).catch((_e) => undefined) + assert.equal(td !== undefined, true, 'block should be marked complete') + } + + // lets make the side chain canonical + await blockchain.putBlock(block81).catch((_e) => undefined) + headBlock = await blockchain.getCanonicalHeadBlock() + assert.equal(headBlock.header.number, block8.header.number, 'head should still be at block8') + assert.equal( + equalsBytes(headBlock.hash(), block81.hash()), + true, + 'head should still be at block 8', + ) + + // alternate chain should now be canonical + for (const block of [block1, block2, block3PoS, block4, block51, block61, block71, block81]) { + // however the block by number should still not fetch it + dbBlock = await blockchain.getBlock(block.header.number).catch((_e) => undefined) + assert.equal(dbBlock !== undefined, true, 'block number index should not exit') + assert.equal( + equalsBytes(dbBlock.hash(), block.hash()), + true, + 'head should still be at block 8', + ) + } + }) + it('should be able to insert block optimistically', async () => { const blockchain = await createBlockchain({ genesisBlock, common, validateBlocks: false }) - await blockchain.putBlock(block51) - await blockchain.putBlock(block4) - await blockchain.putBlock(block3PoS) + await blockchain.putBlock(block5, { canonical: true }) + await blockchain.putBlock(block4, { canonical: true }) + await blockchain.putBlock(block3PoS, { canonical: true }) - let dbBlock = await blockchain.getBlock(block51.hash()).catch((_e) => undefined) + let dbBlock = await blockchain.getBlock(block5.hash()).catch((_e) => undefined) assert.equal(dbBlock !== undefined, true, 'optimistic block by hash should be found') - dbBlock = await blockchain.getBlock(block51.header.number).catch((_e) => undefined) + dbBlock = await blockchain.getBlock(block5.header.number).catch((_e) => undefined) assert.equal(dbBlock !== undefined, true, 'optimistic block by number should be found') // pow block should be allowed to inserted only in backfill mode @@ -85,7 +219,7 @@ describe('[Blockchain]: 4444/optimistic spec', () => { assert.equal( dbBlock === undefined, true, - 'pow block should not be inserted with complete ancestor in blockchain', + 'pow block should not be inserted as a non canonical block', ) await blockchain.putBlock(block2, { canonical: true }) @@ -99,7 +233,7 @@ describe('[Blockchain]: 4444/optimistic spec', () => { let headBlock = await blockchain.getCanonicalHeadBlock() assert.equal(headBlock.header.number, genesisBlock.header.number, 'head still at genesis block') - await blockchain.putBlock(block1, { canonical: true }) + await blockchain.putBlock(block1) headBlock = await blockchain.getCanonicalHeadBlock() assert.equal(headBlock.header.number, block1.header.number, 'head should move') @@ -108,10 +242,67 @@ describe('[Blockchain]: 4444/optimistic spec', () => { assert.equal(dbBlock !== undefined, true, 'the forward index should still be untouched') } - for (const block of [block2, block3PoS, block4, block5]) { + for (const block of [block2, block3PoS, block4, block51]) { await blockchain.putBlock(block).catch((_e) => undefined) } headBlock = await blockchain.getCanonicalHeadBlock() - assert.equal(headBlock.header.number, block5.header.number, 'head should move') + assert.equal( + headBlock.header.number, + block5.header.number, + 'head should move along the new canonical chain', + ) + assert.equal( + equalsBytes(headBlock.hash(), block51.hash()), + true, + 'head should move along the new canonical chain', + ) + + for (const blockNumber of Array.from({ length: 5 }, (_v, i) => i + 1)) { + dbBlock = await blockchain.getBlock(blockNumber).catch((_e) => undefined) + assert.equal(dbBlock !== undefined, true, 'the blocks should be available from index') + } + }) + + it('should be able to specify a 4444 checkpoint and forward fill', async () => { + const blockchain = await createBlockchain({ genesisBlock, common, validateBlocks: false }) + + let dbBlock, headBlock + await blockchain.putBlock(block2).catch((_e) => undefined) + dbBlock = await blockchain.getBlock(block2.header.number).catch((_e) => undefined) + assert.equal(dbBlock === undefined, true, 'pow block2 should not be inserted without parents') + + headBlock = await blockchain.getCanonicalHeadBlock() + assert.equal(headBlock.header.number, genesisBlock.header.number, 'head should be at genesis') + + // insert block2 as checkpoint without needing block1 by specifying parent td which could be anything + // > difficulty of the checkpoint's parent + await blockchain.putBlock(block2, { parentTd: 102n }) + headBlock = await blockchain.getCanonicalHeadBlock() + assert.equal( + headBlock.header.number, + block2.header.number, + 'head should be at the new checkpoint', + ) + dbBlock = await blockchain.getBlock(block2.header.number).catch((_e) => undefined) + assert.equal(dbBlock !== undefined, true, 'pow block2 should be found with number') + + for (const block of [block3PoS, block4, block5, block6, block7, block8]) { + await blockchain.putBlock(block).catch((_e) => { + undefined + }) + } + + headBlock = await blockchain.getCanonicalHeadBlock() + assert.equal(headBlock.header.number, block8.header.number, 'head should be at block5') + + // block1 would not be available + dbBlock = await blockchain.getBlock(1n).catch((_e) => {}) + assert.equal(dbBlock === undefined, true, 'block 1 should be unavailable') + + // check from 2...8 by number + for (const blockNumber of Array.from({ length: 6 }, (_v, i) => i + 2)) { + dbBlock = await blockchain.getBlock(blockNumber).catch((_e) => {}) + assert.equal(dbBlock !== undefined, true, 'the blocks should be available from index') + } }) }) From e4253d607369161394e507af4d052fe3b09797f3 Mon Sep 17 00:00:00 2001 From: harkamal Date: Sun, 15 Sep 2024 00:12:35 +0530 Subject: [PATCH 09/10] add 4444 and optimisitc examples --- packages/blockchain/examples/4444.ts | 87 +++++++++++++ packages/blockchain/examples/optimistic.ts | 143 +++++++++++++++++++++ 2 files changed, 230 insertions(+) create mode 100644 packages/blockchain/examples/4444.ts create mode 100644 packages/blockchain/examples/optimistic.ts diff --git a/packages/blockchain/examples/4444.ts b/packages/blockchain/examples/4444.ts new file mode 100644 index 0000000000..7115c2fc3a --- /dev/null +++ b/packages/blockchain/examples/4444.ts @@ -0,0 +1,87 @@ +import { createBlock } from '@ethereumjs/block' +import { createBlockchain } from '@ethereumjs/blockchain' +import { Common, Hardfork, Mainnet } from '@ethereumjs/common' +import { bytesToHex, hexToBytes } from '@ethereumjs/util' + +const main = async () => { + const common = new Common({ chain: Mainnet, hardfork: Hardfork.Shanghai }) + // Use the safe static constructor which awaits the init method + const blockchain = await createBlockchain({ + validateBlocks: false, // Skipping validation so we can make a simple chain without having to provide complete blocks + validateConsensus: false, + common, + }) + + // We are using this to create minimal post merge blocks between shanghai and cancun in line with the + // block hardfork configuration of mainnet + const chainTTD = BigInt('58750000000000000000000') + const shanghaiTimestamp = 1681338455 + + // We use minimal data to provide a sequence of blocks (increasing number, difficulty, and then setting parent hash to previous block) + const block1 = createBlock( + { + header: { + // 15537393n is terminal block in mainnet config + number: 15537393n + 500n, + // Could be any parenthash other than 0x00..00 as we will set this block as a TRUSTED 4444 anchor + // instead of genesis to build blockchain on top of. One could use any criteria to set a block + // as trusted 4444 anchor + parentHash: hexToBytes(`0x${'20'.repeat(32)}`), + timestamp: shanghaiTimestamp + 12 * 500, + }, + }, + { common, setHardfork: true }, + ) + const block2 = createBlock( + { + header: { + number: block1.header.number + 1n, + parentHash: block1.header.hash(), + timestamp: shanghaiTimestamp + 12 * 501, + }, + }, + { common, setHardfork: true }, + ) + const block3 = createBlock( + { + header: { + number: block2.header.number + 1n, + parentHash: block2.header.hash(), + timestamp: shanghaiTimestamp + 12 * 502, + }, + }, + { common, setHardfork: true }, + ) + + let headBlock, blockByHash, blockByNumber + headBlock = await blockchain.getCanonicalHeadBlock() + console.log( + `Blockchain ${blockchain.consensus.algorithm} Head: ${headBlock.header.number} ${bytesToHex(headBlock.hash())}`, + ) + // Blockchain casper Head: 0 0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3 + + // 1. We can put any post merge block as 4444 anchor by using TTD as parentTD + // 2. For pre-merge blocks its prudent to supply correct parentTD so as to respect the + // hardfork configuration as well as to determine the canonicality of the chain on future putBlocks + await blockchain.putBlock(block1, { parentTd: chainTTD }) + headBlock = await blockchain.getCanonicalHeadBlock() + console.log( + `Blockchain ${blockchain.consensus.algorithm} Head: ${headBlock.header.number} ${bytesToHex(headBlock.hash())}`, + ) + // Blockchain casper Head: 15537893 0x26cb3bfda75027016c17d737fdabe56f412925311b42178a675da88a41bbb7e7 + + await blockchain.putBlock(block2) + headBlock = await blockchain.getCanonicalHeadBlock() + console.log( + `Blockchain ${blockchain.consensus.algorithm} Head: ${headBlock.header.number} ${bytesToHex(headBlock.hash())}`, + ) + // Blockchain casper Head: 15537894 0x6c33728cd8aa21db683d94418fec1f7ee1cfdaa9b77781762ec832da40ec3a7c + + await blockchain.putBlock(block3) + headBlock = await blockchain.getCanonicalHeadBlock() + console.log( + `Blockchain ${blockchain.consensus.algorithm} Head: ${headBlock.header.number} ${bytesToHex(headBlock.hash())}`, + ) + // Blockchain casper Head: 15537895 0x4263ec367ce44e4092b79ea240f132250d0d341639afbaf8c0833fbdd6160d0f +} +void main() diff --git a/packages/blockchain/examples/optimistic.ts b/packages/blockchain/examples/optimistic.ts new file mode 100644 index 0000000000..3c4934c312 --- /dev/null +++ b/packages/blockchain/examples/optimistic.ts @@ -0,0 +1,143 @@ +import { createBlock } from '@ethereumjs/block' +import { createBlockchain } from '@ethereumjs/blockchain' +import { Common, Hardfork, Mainnet } from '@ethereumjs/common' +import { bytesToHex, hexToBytes } from '@ethereumjs/util' + +const main = async () => { + const common = new Common({ chain: Mainnet, hardfork: Hardfork.Shanghai }) + // Use the safe static constructor which awaits the init method + const blockchain = await createBlockchain({ + validateBlocks: false, // Skipping validation so we can make a simple chain without having to provide complete blocks + validateConsensus: false, + common, + }) + + // We are using this to create minimal post merge blocks between shanghai and cancun in line with the + // block hardfork configuration of mainnet + const chainTTD = BigInt('58750000000000000000000') + const shanghaiTimestamp = 1681338455 + + // We use minimal data to construct random block sequence post merge/paris to worry not much about + // td's pre-merge while constructing chain + const block1 = createBlock( + { + header: { + // 15537393n is terminal block in mainnet config + number: 15537393n + 500n, + // Could be any parenthash other than 0x00..00 as we will set this block as a TRUSTED 4444 anchor + // instead of genesis to build blockchain on top of. One could use any criteria to set a block + // as trusted 4444 anchor + parentHash: hexToBytes(`0x${'20'.repeat(32)}`), + timestamp: shanghaiTimestamp + 12 * 500, + }, + }, + { common, setHardfork: true }, + ) + const block2 = createBlock( + { + header: { + number: block1.header.number + 1n, + parentHash: block1.header.hash(), + timestamp: shanghaiTimestamp + 12 * 501, + }, + }, + { common, setHardfork: true }, + ) + const block3 = createBlock( + { + header: { + number: block2.header.number + 1n, + parentHash: block2.header.hash(), + timestamp: shanghaiTimestamp + 12 * 502, + }, + }, + { common, setHardfork: true }, + ) + + let headBlock, blockByHash, blockByNumber + headBlock = await blockchain.getCanonicalHeadBlock() + console.log( + `Blockchain ${blockchain.consensus.algorithm} Head: ${headBlock.header.number} ${bytesToHex(headBlock.hash())}`, + ) + // Blockchain casper Head: 0 0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3 + + // allows any block > head + 1 to be put as non canonical block + await blockchain.putBlock(block3, { canonical: true }) + headBlock = await blockchain.getCanonicalHeadBlock() + blockByHash = await blockchain.getBlock(block3.hash()).catch((e) => null) + blockByNumber = await blockchain.getBlock(block3.header.number).catch((e) => null) + console.log( + `putBlock ${block3.header.number} ${bytesToHex(block3.hash())} byHash: ${blockByHash ? true : false} byNumber: ${blockByNumber ? true : false} headBlock=${headBlock.header.number}`, + ) + // putBlock 15537895 0x4263ec367ce44e4092b79ea240f132250d0d341639afbaf8c0833fbdd6160d0f byHash: true byNumber: true headBlock=0 + + let hasBlock1, hasBlock2, hasBlock3 + hasBlock1 = (await blockchain.getBlock(block1.header.number).catch((e) => null)) ? true : false + hasBlock2 = (await blockchain.getBlock(block2.header.number).catch((e) => null)) ? true : false + hasBlock3 = (await blockchain.getBlock(block3.header.number).catch((e) => null)) ? true : false + console.log( + `canonicality: head=${headBlock.header.number}, 0 ... ${block1.header.number}=${hasBlock1} ${block2.header.number}=${hasBlock2} ${block3.header.number}=${hasBlock3} `, + ) + // canonicality: head=0, 0 ... 15537893=false 15537894=false 15537895=true + + await blockchain.putBlock(block2, { canonical: true }) + headBlock = await blockchain.getCanonicalHeadBlock() + blockByHash = await blockchain.getBlock(block2.hash()).catch((e) => null) + blockByNumber = await blockchain.getBlock(block2.header.number).catch((e) => null) + console.log( + `putBlock ${block2.header.number} ${bytesToHex(block2.hash())} byHash: ${blockByHash ? true : false} byNumber: ${blockByNumber ? true : false} headBlock=${headBlock.header.number}`, + ) + // putBlock 15537894 0x6c33728cd8aa21db683d94418fec1f7ee1cfdaa9b77781762ec832da40ec3a7c byHash: true byNumber: true headBlock=0 + + hasBlock1 = (await blockchain.getBlock(block1.header.number).catch((e) => null)) ? true : false + hasBlock2 = (await blockchain.getBlock(block2.header.number).catch((e) => null)) ? true : false + hasBlock3 = (await blockchain.getBlock(block3.header.number).catch((e) => null)) ? true : false + console.log( + `canonicality: head=${headBlock.header.number}, 0 ... ${block1.header.number}=${hasBlock1} ${block2.header.number}=${hasBlock2} ${block3.header.number}=${hasBlock3} `, + ) + // canonicality: head=0, 0 ... 15537893=false 15537894=true 15537895=true + + // 1. We can put any post merge block as 4444 anchor by using TTD as parentTD + // 2. For pre-merge blocks its prudent to supply correct parentTD so as to respect the + // hardfork configuration as well as to determine the canonicality of the chain on future putBlocks + await blockchain.putBlock(block1, { parentTd: chainTTD }) + headBlock = await blockchain.getCanonicalHeadBlock() + hasBlock1 = (await blockchain.getBlock(block1.header.number).catch((e) => null)) ? true : false + hasBlock2 = (await blockchain.getBlock(block2.header.number).catch((e) => null)) ? true : false + hasBlock3 = (await blockchain.getBlock(block3.header.number).catch((e) => null)) ? true : false + console.log( + `canonicality: head=${headBlock.header.number}, 0 ... ${block1.header.number}=${hasBlock1} ${block2.header.number}=${hasBlock2} ${block3.header.number}=${hasBlock3} `, + ) + // canonicality: head=15537893, 0 ... 15537893=true 15537894=true 15537895=true + + await blockchain.putBlock(block2) + headBlock = await blockchain.getCanonicalHeadBlock() + console.log( + `Blockchain ${blockchain.consensus.algorithm} Head: ${headBlock.header.number} ${bytesToHex(headBlock.hash())}`, + ) + // Blockchain casper Head: 15537894 0x6c33728cd8aa21db683d94418fec1f7ee1cfdaa9b77781762ec832da40ec3a7c + + hasBlock1 = (await blockchain.getBlock(block1.header.number).catch((e) => null)) ? true : false + hasBlock2 = (await blockchain.getBlock(block2.header.number).catch((e) => null)) ? true : false + hasBlock3 = (await blockchain.getBlock(block3.header.number).catch((e) => null)) ? true : false + console.log( + `canonicality: head=${headBlock.header.number}, 0 ... ${block1.header.number}=${hasBlock1} ${block2.header.number}=${hasBlock2} ${block3.header.number}=${hasBlock3} `, + ) + // canonicality: head=15537894, 0 ... 15537893=true 15537894=true 15537895=true + + await blockchain.putBlock(block3) + headBlock = await blockchain.getCanonicalHeadBlock() + console.log( + `Blockchain ${blockchain.consensus.algorithm} Head: ${headBlock.header.number} ${bytesToHex(headBlock.hash())}`, + ) + // Blockchain casper Head: 15537895 0x4263ec367ce44e4092b79ea240f132250d0d341639afbaf8c0833fbdd6160d0f + + hasBlock1 = (await blockchain.getBlock(block1.header.number).catch((e) => null)) ? true : false + hasBlock2 = (await blockchain.getBlock(block2.header.number).catch((e) => null)) ? true : false + hasBlock3 = (await blockchain.getBlock(block3.header.number).catch((e) => null)) ? true : false + console.log( + `canonicality: head=${headBlock.header.number}, 0 ... ${block1.header.number}=${hasBlock1} ${block2.header.number}=${hasBlock2} ${block3.header.number}=${hasBlock3} `, + ) + // canonicality: head=15537895, 0 ... 15537893=true 15537894=true 15537895=true +} +void main() From 3bbed088a9912dbaa327ce8a6c759bdc20564c6d Mon Sep 17 00:00:00 2001 From: harkamal Date: Tue, 17 Sep 2024 18:29:33 +0530 Subject: [PATCH 10/10] remove skipping validation --- packages/blockchain/examples/4444.ts | 2 -- packages/blockchain/examples/optimistic.ts | 2 -- 2 files changed, 4 deletions(-) diff --git a/packages/blockchain/examples/4444.ts b/packages/blockchain/examples/4444.ts index 7115c2fc3a..ca4e2d0ede 100644 --- a/packages/blockchain/examples/4444.ts +++ b/packages/blockchain/examples/4444.ts @@ -7,8 +7,6 @@ const main = async () => { const common = new Common({ chain: Mainnet, hardfork: Hardfork.Shanghai }) // Use the safe static constructor which awaits the init method const blockchain = await createBlockchain({ - validateBlocks: false, // Skipping validation so we can make a simple chain without having to provide complete blocks - validateConsensus: false, common, }) diff --git a/packages/blockchain/examples/optimistic.ts b/packages/blockchain/examples/optimistic.ts index 3c4934c312..568047cf2a 100644 --- a/packages/blockchain/examples/optimistic.ts +++ b/packages/blockchain/examples/optimistic.ts @@ -7,8 +7,6 @@ const main = async () => { const common = new Common({ chain: Mainnet, hardfork: Hardfork.Shanghai }) // Use the safe static constructor which awaits the init method const blockchain = await createBlockchain({ - validateBlocks: false, // Skipping validation so we can make a simple chain without having to provide complete blocks - validateConsensus: false, common, })