blockchain: allow optimistic block insertion in blockchain

packages/blockchain/src/blockchain.ts

@@ -25,6 +25,7 @@ import {
 import { DBManager } from './db/manager.js'
 import { DBTarget } from './db/operation.js'
 
+import type { OptimisticOpts } from './db/operation.js'
 import type {
   BlockchainEvents,
   BlockchainInterface,
@@ -272,8 +273,8 @@ export class Blockchain implements BlockchainInterface {
    * heads/hashes are overwritten.
    * @param block - The block to be added to the blockchain
    */
-  async putBlock(block: Block) {
-    await this._putBlockOrHeader(block)
+  async putBlock(block: Block, opts?: OptimisticOpts) {
+    await this._putBlockOrHeader(block, opts)
   }
 
   /**
@@ -344,7 +345,7 @@ export class Blockchain implements BlockchainInterface {
    * header using the iterator method.
    * @hidden
    */
-  private async _putBlockOrHeader(item: Block | BlockHeader) {
+  private async _putBlockOrHeader(item: Block | BlockHeader, optimisticOpts?: OptimisticOpts) {
     await this.runWithLock<void>(async () => {
       // Save the current sane state incase _putBlockOrHeader midway with some
       // dirty changes in head trackers
@@ -362,96 +363,119 @@ export class Blockchain implements BlockchainInterface {
         if (isGenesis) {
           if (equalsBytes(this.genesisBlock.hash(), block.hash())) {
             // Try to re-put the existing genesis block, accept this
+            // genesis block is not optimistic
+            optimisticOpts = undefined
             return
           }
           throw new Error(
             'Cannot put a different genesis block than current blockchain genesis: create a new Blockchain',
           )
         }
 
-        const { header } = block
-        const blockHash = header.hash()
-        const blockNumber = header.number
-        let td = header.difficulty
-        const currentTd = { header: BIGINT_0, block: BIGINT_0 }
-        let dbOps: DBOp[] = []
-
         if (block.common.chainId() !== this.common.chainId()) {
           throw new Error(
             `Chain mismatch while trying to put block or header. Chain ID of block: ${block.common.chainId}, chain ID of blockchain : ${this.common.chainId}`,
           )
         }
 
-        if (this._validateBlocks && !isGenesis && item instanceof Block) {
+        if (
+          this._validateBlocks &&
+          !isGenesis &&
+          item instanceof Block &&
+          optimisticOpts === undefined
+        ) {
           // this calls into `getBlock`, which is why we cannot lock yet
           await this.validateBlock(block)
         }
 
-        if (this._validateConsensus) {
+        if (this._validateConsensus && optimisticOpts === undefined) {
           await this.consensus!.validateConsensus(block)
         }
 
-        // set total difficulty in the current context scope
-        if (this._headHeaderHash) {
-          currentTd.header = await this.getTotalDifficulty(this._headHeaderHash)
-        }
-        if (this._headBlockHash) {
-          currentTd.block = await this.getTotalDifficulty(this._headBlockHash)
-        }
-
         // calculate the total difficulty of the new block
-        const parentTd = await this.getParentTD(header)
-        if (!block.isGenesis()) {
-          td += parentTd
-        }
-
-        // save total difficulty to the database
-        dbOps = dbOps.concat(DBSetTD(td, blockNumber, blockHash))
-
-        // save header/block to the database, but save the input not our wrapper block
-        dbOps = dbOps.concat(DBSetBlockOrHeader(item))
-
-        let commonAncestor: undefined | BlockHeader
-        let ancestorHeaders: undefined | BlockHeader[]
-        // if total difficulty is higher than current, add it to canonical chain
-        if (
-          block.isGenesis() ||
-          td > currentTd.header ||
-          block.common.consensusType() === ConsensusType.ProofOfStake
-        ) {
-          const foundCommon = await this.findCommonAncestor(header)
-          commonAncestor = foundCommon.commonAncestor
-          ancestorHeaders = foundCommon.ancestorHeaders
+        const { header } = block
+        const blockHash = header.hash()
+        const blockNumber = header.number
 
-          this._headHeaderHash = blockHash
-          if (item instanceof Block) {
-            this._headBlockHash = blockHash
+        let td = header.difficulty
+        try {
+          const parentTd = await this.getParentTD(header)
+          if (!block.isGenesis()) {
+            td += parentTd
           }
-          if (this._hardforkByHeadBlockNumber) {
-            await this.checkAndTransitionHardForkByNumber(blockNumber, header.timestamp)
+        } catch (e) {
+          // opimistic insertion does care about td
+          if (optimisticOpts === undefined) {
+            throw e
           }
+        }
 
-          // delete higher number assignments and overwrite stale canonical chain
-          await this._deleteCanonicalChainReferences(blockNumber + BIGINT_1, blockHash, dbOps)
-          // from the current header block, check the blockchain in reverse (i.e.
-          // traverse `parentHash`) until `numberToHash` matches the current
-          // number/hash in the canonical chain also: overwrite any heads if these
-          // heads are stale in `_heads` and `_headBlockHash`
-          await this._rebuildCanonical(header, dbOps)
+        let dbOps: DBOp[] = []
+        if (optimisticOpts !== undefined) {
+          dbOps = dbOps.concat(DBSetBlockOrHeader(item))
+          dbOps.push(DBSetHashToNumber(blockHash, blockNumber))
+          if (optimisticOpts.fcUed) {
+            dbOps.push(DBOp.set(DBTarget.OptimisticNumberToHash, blockHash, { blockNumber }))
+          }
+          await this.dbManager.batch(dbOps)
         } else {
-          // the TD is lower than the current highest TD so we will add the block
-          // to the DB, but will not mark it as the canonical chain.
-          if (td > currentTd.block && item instanceof Block) {
-            this._headBlockHash = blockHash
+          const currentTd = { header: BIGINT_0, block: BIGINT_0 }
+          // set total difficulty in the current context scope
+          if (this._headHeaderHash) {
+            currentTd.header = await this.getTotalDifficulty(this._headHeaderHash)
+          }
+          if (this._headBlockHash) {
+            currentTd.block = await this.getTotalDifficulty(this._headBlockHash)
           }
-          // save hash to number lookup info even if rebuild not needed
-          dbOps.push(DBSetHashToNumber(blockHash, blockNumber))
-        }
 
-        const ops = dbOps.concat(this._saveHeadOps())
-        await this.dbManager.batch(ops)
+          // save total difficulty to the database
+          dbOps = dbOps.concat(DBSetTD(td, blockNumber, blockHash))
+
+          // save header/block to the database, but save the input not our wrapper block
+          dbOps = dbOps.concat(DBSetBlockOrHeader(item))
+
+          let commonAncestor: undefined | BlockHeader
+          let ancestorHeaders: undefined | BlockHeader[]
+          // if total difficulty is higher than current, add it to canonical chain
+          if (
+            block.isGenesis() ||
+            td > currentTd.header ||
+            block.common.consensusType() === ConsensusType.ProofOfStake
+          ) {
+            const foundCommon = await this.findCommonAncestor(header)
+            commonAncestor = foundCommon.commonAncestor
+            ancestorHeaders = foundCommon.ancestorHeaders
+
+            this._headHeaderHash = blockHash
+            if (item instanceof Block) {
+              this._headBlockHash = blockHash
+            }
+            if (this._hardforkByHeadBlockNumber) {
+              await this.checkAndTransitionHardForkByNumber(blockNumber, header.timestamp)
+            }
 
-        await this.consensus?.newBlock(block, commonAncestor, ancestorHeaders)
+            // delete higher number assignments and overwrite stale canonical chain
+            await this._deleteCanonicalChainReferences(blockNumber + BIGINT_1, blockHash, dbOps)
+            // from the current header block, check the blockchain in reverse (i.e.
+            // traverse `parentHash`) until `numberToHash` matches the current
+            // number/hash in the canonical chain also: overwrite any heads if these
+            // heads are stale in `_heads` and `_headBlockHash`
+            await this._rebuildCanonical(header, dbOps)
+          } else {
+            // the TD is lower than the current highest TD so we will add the block
+            // to the DB, but will not mark it as the canonical chain.
+            if (td > currentTd.block && item instanceof Block) {
+              this._headBlockHash = blockHash
+            }
+            // save hash to number lookup info even if rebuild not needed
+            dbOps.push(DBSetHashToNumber(blockHash, blockNumber))
+          }
+
+          const ops = dbOps.concat(this._saveHeadOps())
+          await this.dbManager.batch(ops)
+
+          await this.consensus?.newBlock(block, commonAncestor, ancestorHeaders)
+        }
       } catch (e) {
         // restore head to the previously sane state
         this._heads = oldHeads
@@ -658,13 +682,16 @@ export class Blockchain implements BlockchainInterface {
    * this will be immediately looked up, otherwise it will wait until we have
    * unlocked the DB
    */
-  async getBlock(blockId: Uint8Array | number | bigint): Promise<Block> {
+  async getBlock(
+    blockId: Uint8Array | number | bigint,
+    optimisticOpts?: OptimisticOpts,
+  ): Promise<Block> {
     // cannot wait for a lock here: it is used both in `validate` of `Block`
     // (calls `getBlock` to get `parentHash`) it is also called from `runBlock`
     // in the `VM` if we encounter a `BLOCKHASH` opcode: then a bigint is used we
     // need to then read the block from the canonical chain Q: is this safe? We
     // know it is OK if we call it from the iterator... (runBlock)
-    const block = await this.dbManager.getBlock(blockId)
+    const block = await this.dbManager.getBlock(blockId, optimisticOpts)
 
     if (block === undefined) {
       if (typeof blockId === 'object') {

packages/blockchain/src/db/constants.ts

@@ -28,6 +28,7 @@ const TD_SUFFIX = utf8ToBytes('t')
  * headerPrefix + number + numSuffix -> hash
  */
 const NUM_SUFFIX = utf8ToBytes('n')
+const OPTIMISTIC_NUM_SUFFIX = utf8ToBytes('o')
 
 /**
  * blockHashPrefix + hash -> number
@@ -55,6 +56,9 @@ const bodyKey = (n: bigint, hash: Uint8Array) => concatBytes(BODY_PREFIX, bytesB
 
 const numberToHashKey = (n: bigint) => concatBytes(HEADER_PREFIX, bytesBE8(n), NUM_SUFFIX)
 
+const optimisticNumberToHashKey = (n: bigint) =>
+  concatBytes(HEADER_PREFIX, bytesBE8(n), OPTIMISTIC_NUM_SUFFIX)
+
 const hashToNumberKey = (hash: Uint8Array) => concatBytes(BLOCK_HASH_PREFIX, hash)
 
 /**
@@ -69,5 +73,6 @@ export {
   headerKey,
   HEADS_KEY,
   numberToHashKey,
+  optimisticNumberToHashKey,
   tdKey,
 }

packages/blockchain/src/db/manager.ts

@@ -12,7 +12,7 @@ import {
 import { Cache } from './cache.js'
 import { DBOp, DBTarget } from './operation.js'
 
-import type { DatabaseKey } from './operation.js'
+import type { DatabaseKey, OptimisticOpts } from './operation.js'
 import type { Block, BlockBodyBytes, BlockBytes, BlockOptions } from '@ethereumjs/block'
 import type { Common } from '@ethereumjs/common'
 import type { BatchDBOp, DB, DBObject, DelBatch, PutBatch } from '@ethereumjs/util'
@@ -47,6 +47,7 @@ export class DBManager {
       body: new Cache({ max: 256 }),
       numberToHash: new Cache({ max: 2048 }),
       hashToNumber: new Cache({ max: 2048 }),
+      optimisticNumberToHash: new Cache({ max: 2048 }),
     }
   }
 
@@ -84,7 +85,10 @@ export class DBManager {
    * Fetches a block (header and body) given a block id,
    * which can be either its hash or its number.
    */
-  async getBlock(blockId: Uint8Array | bigint | number): Promise<Block | undefined> {
+  async getBlock(
+    blockId: Uint8Array | bigint | number,
+    optimisticOpts?: OptimisticOpts,
+  ): Promise<Block | undefined> {
     if (typeof blockId === 'number' && Number.isInteger(blockId)) {
       blockId = BigInt(blockId)
     }
@@ -95,9 +99,32 @@ export class DBManager {
     if (blockId instanceof Uint8Array) {
       hash = blockId
       number = await this.hashToNumber(blockId)
+      if (number === undefined) {
+        return undefined
+      }
+
+      if (optimisticOpts?.fcUed === true) {
+        let optimisticHash = await this.optimisticNumberToHash(number)
+        if (optimisticHash === undefined && optimisticOpts.linked === true) {
+          optimisticHash = await this.numberToHash(number)
+        }
+        if (optimisticHash === undefined || !equalsBytes(optimisticHash, hash)) {
+          return undefined
+        }
+      }
     } else if (typeof blockId === 'bigint') {
       number = blockId
-      hash = await this.numberToHash(blockId)
+      if (optimisticOpts !== undefined) {
+        if (!optimisticOpts.fcUed) {
+          throw Error(`Invalid fcUed optimistic block by number lookup`)
+        }
+        hash = await this.optimisticNumberToHash(blockId)
+        if (hash === undefined && optimisticOpts.linked === true) {
+          hash = await this.numberToHash(blockId)
+        }
+      } else {
+        hash = await this.numberToHash(blockId)
+      }
     } else {
       throw new Error('Unknown blockId type')
     }
@@ -190,6 +217,11 @@ export class DBManager {
     return value
   }
 
+  async optimisticNumberToHash(blockNumber: bigint): Promise<Uint8Array | undefined> {
+    const value = await this.get(DBTarget.OptimisticNumberToHash, { blockNumber })
+    return value
+  }
+
   /**
    * Fetches a key from the db. If `opts.cache` is specified
    * it first tries to load from cache, and on cache miss will

packages/blockchain/src/db/operation.ts

@@ -8,11 +8,14 @@ import {
   hashToNumberKey,
   headerKey,
   numberToHashKey,
+  optimisticNumberToHashKey,
   tdKey,
 } from './constants.js'
 
 import type { CacheMap } from './manager.js'
 
+export type OptimisticOpts = { fcUed: boolean; linked?: boolean }
+
 export enum DBTarget {
   Heads,
   HeadHeader,
@@ -25,6 +28,7 @@ export enum DBTarget {
   CliqueSignerStates,
   CliqueVotes,
   CliqueBlockSigners,
+  OptimisticNumberToHash,
 }
 
 /**
@@ -88,6 +92,11 @@ export class DBOp {
         this.cacheString = 'numberToHash'
         break
       }
+      case DBTarget.OptimisticNumberToHash: {
+        this.baseDBOp.key = optimisticNumberToHashKey(key!.blockNumber!)
+        this.cacheString = 'optimisticNumberToHash'
+        break
+      }
       case DBTarget.TotalDifficulty: {
         this.baseDBOp.key = tdKey(key!.blockNumber!, key!.blockHash!)
         this.cacheString = 'td'

packages/blockchain/src/types.ts

@@ -1,3 +1,4 @@
+import type { OptimisticOpts } from './db/operation.js'
 import type { Blockchain } from './index.js'
 import type { Block, BlockHeader } from '@ethereumjs/block'
 import type { Common, ConsensusAlgorithm } from '@ethereumjs/common'
@@ -16,7 +17,7 @@ export interface BlockchainInterface {
    *
    * @param block - The block to be added to the blockchain.
    */
-  putBlock(block: Block): Promise<void>
+  putBlock(block: Block, optimisticOpts?: OptimisticOpts): Promise<void>
 
   /**
    * Deletes a block from the blockchain. All child blocks in the chain are
@@ -29,7 +30,7 @@ export interface BlockchainInterface {
   /**
    * Returns a block by its hash or number.
    */
-  getBlock(blockId: Uint8Array | number | bigint): Promise<Block>
+  getBlock(blockId: Uint8Array | number | bigint, optimisticOpts?: OptimisticOpts): Promise<Block>
 
   /**
    * Iterates through blocks starting at the specified iterator head and calls