From bf68c5e193e15ddaef25c56cc672bbe0e1af1ffc Mon Sep 17 00:00:00 2001 From: Jochem Brouwer Date: Sun, 27 Sep 2020 19:12:56 +0200 Subject: [PATCH 01/20] blokchain: add reorg test --- packages/blockchain/test/reorg.ts | 66 +++++++++++++++++++++++++++++++ packages/blockchain/test/util.ts | 28 +++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 packages/blockchain/test/reorg.ts diff --git a/packages/blockchain/test/reorg.ts b/packages/blockchain/test/reorg.ts new file mode 100644 index 0000000000..0eb366b43d --- /dev/null +++ b/packages/blockchain/test/reorg.ts @@ -0,0 +1,66 @@ +import Common from '@ethereumjs/common' +import { Block, BlockHeader } from '@ethereumjs/block' +import { BN, toBuffer, bufferToInt } from 'ethereumjs-util' +import * as test from 'tape' +import Blockchain from '../src' +import { generateBlocks, generateConsecutiveBlock } from './util' +import * as testData from './testdata.json' + +const level = require('level-mem') + +const genesis = generateBlocks(1)[0] +genesis.header.difficulty = Buffer.from('02000', 'hex') // minimum difficulty + +test('reorg tests', (t) => { + t.test( + 'should correctly reorg the chain if the total difficulty is higher on a lower block number than the current head block', + async (st) => { + const common = new Common({ chain: 'mainnet', hardfork: 'muirGlacier' }) + const blockchain = new Blockchain({ validateBlocks: true, validatePow: false, common }) + await blockchain.putBlock(genesis, true) + + let blocks_lowTD: Block[] = [] + let blocks_highTD: Block[] = [] + + blocks_lowTD.push(generateConsecutiveBlock(genesis, 0)) + + let TD_Low = new BN(genesis.header.difficulty).add(new BN(blocks_lowTD[0].header.difficulty)) + let TD_High = new BN(genesis.header.difficulty) + + // Keep generating blocks until the Total Difficulty (TD) of the High TD chain is higher than the TD of the Low TD chain + // This means that the block number of the high TD chain is 1 lower than the low TD chain + + while (TD_High.cmp(TD_Low) == -1) { + blocks_lowTD.push(generateConsecutiveBlock(blocks_lowTD[blocks_lowTD.length - 1], 0)) + blocks_highTD.push( + generateConsecutiveBlock(blocks_highTD[blocks_highTD.length - 1] || genesis, 1), + ) + + TD_Low.iadd(new BN(blocks_lowTD[blocks_lowTD.length - 1].header.difficulty)) + TD_High.iadd(new BN(blocks_highTD[blocks_highTD.length - 1].header.difficulty)) + } + + // sanity check + const lowTDBlock = blocks_lowTD[blocks_lowTD.length - 1] + const highTDBlock = blocks_highTD[blocks_highTD.length - 1] + + let number_lowTD = new BN(lowTDBlock.header.number) + let number_highTD = new BN(highTDBlock.header.number) + + // ensure that the block number is indeed higher on the low TD chain + t.ok(number_lowTD.cmp(number_highTD) == 1) + + await blockchain.putBlocks(blocks_lowTD) + + let head_lowTD = await blockchain.getHead() + + await blockchain.putBlocks(blocks_highTD) + + let head_highTD = await blockchain.getHead() + + t.ok(head_lowTD.hash().equals(lowTDBlock.hash())) + t.ok(head_highTD.hash().equals(highTDBlock.hash())) + st.end() + }, + ) +}) diff --git a/packages/blockchain/test/util.ts b/packages/blockchain/test/util.ts index b1f3b8d785..3e59e13807 100644 --- a/packages/blockchain/test/util.ts +++ b/packages/blockchain/test/util.ts @@ -54,6 +54,34 @@ export const generateBlockchain = async (numberOfBlocks: number, genesis?: Block blocks, error: null, } +/** + * + * @param parentBlock parent block to generate the consecutive block on top of + * @param difficultyChangeFactor this integer can be any value, but will only return unique blocks between [-99, 1] (this is due to difficulty calculation). 1 will increase the difficulty, 0 will keep the difficulty constant any any negative number will decrease the difficulty + */ + +export const generateConsecutiveBlock = ( + parentBlock: Block, + difficultyChangeFactor: number, +): Block => { + const common = new Common({ chain: 'mainnet', hardfork: 'muirGlacier' }) + const block = new Block(undefined, { common }) + block.header.number = toBuffer(new BN(parentBlock.header.number).add(new BN(1))) + block.header.parentHash = parentBlock.hash() + block.header.gasLimit = toBuffer(8000000) + if (difficultyChangeFactor > 1) { + difficultyChangeFactor = 1 + } + block.header.timestamp = toBuffer( + bufferToInt(parentBlock.header.timestamp) + (10 + -difficultyChangeFactor * 9), + ) + block.header.difficulty = toBuffer(block.header.canonicalDifficulty(parentBlock)) + return new Block( + { + header: block.header, + }, + { common }, + ) } export const isConsecutive = (blocks: Block[]) => { From 784b46e9d9e002d3923895763a91b783430b673c Mon Sep 17 00:00:00 2001 From: Jochem Brouwer Date: Thu, 15 Oct 2020 21:37:57 +0200 Subject: [PATCH 02/20] vm: re-check skipped tests --- packages/vm/tests/BlockchainTestsRunner.ts | 6 ------ packages/vm/tests/config.ts | 21 +++------------------ 2 files changed, 3 insertions(+), 24 deletions(-) diff --git a/packages/vm/tests/BlockchainTestsRunner.ts b/packages/vm/tests/BlockchainTestsRunner.ts index 0a4624d209..164b7a4a2e 100644 --- a/packages/vm/tests/BlockchainTestsRunner.ts +++ b/packages/vm/tests/BlockchainTestsRunner.ts @@ -109,12 +109,6 @@ export default async function runBlockchainTest(options: any, testData: any, t: continue } - if (currentBlock.lt(lastBlock)) { - // "re-org": rollback the blockchain to currentBlock (i.e. delete that block number in the blockchain plus the children) - t.fail('re-orgs are not supported by the test suite') - return - } - try { // check if we should update common. const newFork = common.setHardforkByBlockNumber(currentBlock) diff --git a/packages/vm/tests/config.ts b/packages/vm/tests/config.ts index 6059e98317..11148a5ed0 100644 --- a/packages/vm/tests/config.ts +++ b/packages/vm/tests/config.ts @@ -11,26 +11,11 @@ export const DEFAULT_FORK_CONFIG = 'Istanbul' export const SKIP_BROKEN = [ 'ForkStressTest', // Only BlockchainTest, temporary till fixed (2020-05-23) 'ChainAtoChainB', // Only BlockchainTest, temporary, along expectException fixes (2020-05-23) - 'sha3_bigOffset', // SHA3: Only BlockchainTest, unclear SHA3 test situation (2020-05-28) (https://github.com/ethereumjs/ethereumjs-vm/pull/743#issuecomment-635116418) - 'sha3_memSizeNoQuadraticCost', // SHA3: See also: - 'sha3_memSizeQuadraticCost', // SHA3: https://github.com/ethereumjs/ethereumjs-vm/pull/743#issuecomment-635116418 - 'sha3_bigSize', // SHA3 - // these tests need "re-org" support in blockchain + // In these tests, we have access to two forked chains. Their total difficulty is equal. There are errors in the second chain, but we have no reason to execute this chain if the TD remains equal. 'blockChainFrontierWithLargerTDvsHomesteadBlockchain2_FrontierToHomesteadAt5', 'blockChainFrontierWithLargerTDvsHomesteadBlockchain_FrontierToHomesteadAt5', 'HomesteadOverrideFrontier_FrontierToHomesteadAt5', - 'DaoTransactions_HomesteadToDaoAt5', - 'RPC_API_Test', - 'lotsOfBranchesOverrideAtTheEnd', - 'lotsOfBranchesOverrideAtTheMiddle', - 'newChainFrom4Block', - 'newChainFrom5Block', - 'newChainFrom6Block', - 'sideChainWithMoreTransactions', - 'sideChainWithMoreTransactions2', - 'sideChainWithNewMaxDifficultyStartingFromBlock3AfterBlock4', - 'uncleBlockAtBlock3afterBlock4', ] /** @@ -405,10 +390,10 @@ export function getSkipTests(choices: string, defaultChoice: string): string[] { skipTests = skipTests.concat(SKIP_BROKEN) } if (all || choicesList.includes('permanent')) { - skipTests = skipTests.concat(SKIP_PERMANENT) + //skipTests = skipTests.concat(SKIP_PERMANENT) } if (all || choicesList.includes('slow')) { - skipTests = skipTests.concat(SKIP_SLOW) + //skipTests = skipTests.concat(SKIP_SLOW) } } return skipTests From 5b508bd37fa88b0697c3035dd3913e605ef91dc5 Mon Sep 17 00:00:00 2001 From: Jochem Brouwer Date: Thu, 15 Oct 2020 22:01:39 +0200 Subject: [PATCH 03/20] blockchain: fix tests vm: fix skipped tests --- packages/blockchain/test/util.ts | 34 +++++++++++++++++++------------- packages/vm/tests/config.ts | 4 ++-- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/packages/blockchain/test/util.ts b/packages/blockchain/test/util.ts index 3e59e13807..4920e92946 100644 --- a/packages/blockchain/test/util.ts +++ b/packages/blockchain/test/util.ts @@ -1,5 +1,5 @@ -import { BN, rlp } from 'ethereumjs-util' -import { Block } from '@ethereumjs/block' +import { BN, rlp, toBuffer } from 'ethereumjs-util' +import { Block, BlockHeader } from '@ethereumjs/block' import Common from '@ethereumjs/common' import Blockchain from '../src' const level = require('level-mem') @@ -64,24 +64,30 @@ export const generateConsecutiveBlock = ( parentBlock: Block, difficultyChangeFactor: number, ): Block => { - const common = new Common({ chain: 'mainnet', hardfork: 'muirGlacier' }) - const block = new Block(undefined, { common }) - block.header.number = toBuffer(new BN(parentBlock.header.number).add(new BN(1))) - block.header.parentHash = parentBlock.hash() - block.header.gasLimit = toBuffer(8000000) if (difficultyChangeFactor > 1) { difficultyChangeFactor = 1 } - block.header.timestamp = toBuffer( - bufferToInt(parentBlock.header.timestamp) + (10 + -difficultyChangeFactor * 9), - ) - block.header.difficulty = toBuffer(block.header.canonicalDifficulty(parentBlock)) - return new Block( + const common = new Common({ chain: 'mainnet', hardfork: 'muirGlacier' }) + const tmpHeader = BlockHeader.fromHeaderData({ + number: new BN(parentBlock.header.number).add(new BN(1)), + timestamp: parentBlock.header.timestamp.addn(10 + -difficultyChangeFactor * 9), + }) + const header = BlockHeader.fromHeaderData( { - header: block.header, + number: new BN(parentBlock.header.number).add(new BN(1)), + parentHash: parentBlock.hash(), + gasLimit: new BN(8000000), + timestamp: parentBlock.header.timestamp.addn(10 + -difficultyChangeFactor * 9), + difficulty: new BN(tmpHeader.canonicalDifficulty(parentBlock)), + }, + { + common, }, - { common }, ) + + const block = new Block(header, undefined, undefined, { common }) + + return block } export const isConsecutive = (blocks: Block[]) => { diff --git a/packages/vm/tests/config.ts b/packages/vm/tests/config.ts index 11148a5ed0..dc31c4dfe6 100644 --- a/packages/vm/tests/config.ts +++ b/packages/vm/tests/config.ts @@ -390,10 +390,10 @@ export function getSkipTests(choices: string, defaultChoice: string): string[] { skipTests = skipTests.concat(SKIP_BROKEN) } if (all || choicesList.includes('permanent')) { - //skipTests = skipTests.concat(SKIP_PERMANENT) + skipTests = skipTests.concat(SKIP_PERMANENT) } if (all || choicesList.includes('slow')) { - //skipTests = skipTests.concat(SKIP_SLOW) + skipTests = skipTests.concat(SKIP_SLOW) } } return skipTests From 73f155cd11f4d400363c412bbe59ccc374d8cb2a Mon Sep 17 00:00:00 2001 From: Jochem Brouwer Date: Fri, 16 Oct 2020 17:59:07 +0200 Subject: [PATCH 04/20] block: add static getCanonicalDifficulty method --- packages/block/src/header.ts | 59 ++++++++++++++++++++++++------------ 1 file changed, 40 insertions(+), 19 deletions(-) diff --git a/packages/block/src/header.ts b/packages/block/src/header.ts index 19d332005e..7838005f64 100644 --- a/packages/block/src/header.ts +++ b/packages/block/src/header.ts @@ -254,28 +254,36 @@ export class BlockHeader { } /** - * Returns the canonical difficulty for this block. - * - * @param parentBlockHeader - the header from the parent `Block` of this header + * @param parentBlock - The `parentBlock` where we wish to build upon + * @param timestamp - The `BN` timestamp which we should calculate the difficulty for + * @param number - The `BN` block number. Defaults to the parent blocks' `number` + 1 + * @param common - The `Common` to use: defaults to the parent blocks` `Common` */ - canonicalDifficulty(parentBlockHeader: BlockHeader): BN { - const hardfork = this._getHardfork() - const blockTs = this.timestamp - const { timestamp: parentTs, difficulty: parentDif } = parentBlockHeader + + public static getCanonicalDifficulty( + parentHeader: BlockHeader, + timestamp: BN, + number?: BN, + common?: Common, + ): BN { + const blockTs = timestamp.clone() + const { timestamp: parentTs, difficulty: parentDif } = parentHeader + const usedCommon = common || parentHeader._common + let num = (number || parentHeader.number).clone().addn(1) + const hardfork = usedCommon.hardfork() || usedCommon.activeHardfork(num.toNumber()) const minimumDifficulty = new BN( - this._common.paramByHardfork('pow', 'minimumDifficulty', hardfork) + usedCommon.paramByHardfork('pow', 'minimumDifficulty', hardfork), ) const offset = parentDif.div( - new BN(this._common.paramByHardfork('pow', 'difficultyBoundDivisor', hardfork)) + new BN(usedCommon.paramByHardfork('pow', 'difficultyBoundDivisor', hardfork)), ) - let num = this.number.clone() // We use a ! here as TS cannot follow this hardfork-dependent logic, but it always gets assigned let dif!: BN - if (this._common.hardforkGteHardfork(hardfork, 'byzantium')) { + if (usedCommon.hardforkGteHardfork(hardfork, 'byzantium')) { // max((2 if len(parent.uncles) else 1) - ((timestamp - parent.timestamp) // 9), -99) (EIP100) - const uncleAddend = parentBlockHeader.uncleHash.equals(KECCAK256_RLP_ARRAY) ? 1 : 2 + const uncleAddend = parentHeader.uncleHash.equals(KECCAK256_RLP_ARRAY) ? 1 : 2 let a = blockTs.sub(parentTs).idivn(9).ineg().iaddn(uncleAddend) const cutoff = new BN(-99) // MAX(cutoff, a) @@ -285,25 +293,25 @@ export class BlockHeader { dif = parentDif.add(offset.mul(a)) } - if (this._common.hardforkGteHardfork(hardfork, 'muirGlacier')) { + if (usedCommon.hardforkGteHardfork(hardfork, 'muirGlacier')) { // Istanbul/Berlin difficulty bomb delay (EIP2384) num.isubn(9000000) if (num.ltn(0)) { num = new BN(0) } - } else if (this._common.hardforkGteHardfork(hardfork, 'constantinople')) { + } else if (usedCommon.hardforkGteHardfork(hardfork, 'constantinople')) { // Constantinople difficulty bomb delay (EIP1234) num.isubn(5000000) if (num.ltn(0)) { num = new BN(0) } - } else if (this._common.hardforkGteHardfork(hardfork, 'byzantium')) { + } else if (usedCommon.hardforkGteHardfork(hardfork, 'byzantium')) { // Byzantium difficulty bomb delay (EIP649) num.isubn(3000000) if (num.ltn(0)) { num = new BN(0) } - } else if (this._common.hardforkGteHardfork(hardfork, 'homestead')) { + } else if (usedCommon.hardforkGteHardfork(hardfork, 'homestead')) { // 1 - (block_timestamp - parent_timestamp) // 10 let a = blockTs.sub(parentTs).idivn(10).ineg().iaddn(1) const cutoff = new BN(-99) @@ -315,9 +323,8 @@ export class BlockHeader { } else { // pre-homestead if ( - parentTs - .addn(this._common.paramByHardfork('pow', 'durationLimit', hardfork)) - .cmp(blockTs) === 1 + parentTs.addn(usedCommon.paramByHardfork('pow', 'durationLimit', hardfork)).cmp(blockTs) === + 1 ) { dif = offset.add(parentDif) } else { @@ -337,6 +344,20 @@ export class BlockHeader { return dif } + /** + * Returns the canonical difficulty for this block. + * + * @param parentBlock - the parent `Block` of this header + */ + canonicalDifficulty(parentHeader: BlockHeader): BN { + return BlockHeader.getCanonicalDifficulty( + parentHeader, + this.timestamp, + this.number, + this._common, + ) + } + /** * Checks that the block's `difficulty` matches the canonical difficulty. * From 671de5e9a141c3efcc0a11e4f601dd8188c7e907 Mon Sep 17 00:00:00 2001 From: Jochem Brouwer Date: Tue, 20 Oct 2020 16:41:29 +0200 Subject: [PATCH 05/20] blockchain: throw early on errors --- packages/blockchain/src/index.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/blockchain/src/index.ts b/packages/blockchain/src/index.ts index 4b5029779b..9ccbc9ea90 100644 --- a/packages/blockchain/src/index.ts +++ b/packages/blockchain/src/index.ts @@ -169,11 +169,11 @@ export default class Blockchain implements BlockchainInterface { try { genesisHash = await this.dbManager.numberToHash(new BN(0)) } catch (error) { - await this._setCanonicalGenesisBlock() - genesisHash = this._genesis if (error.type !== 'NotFoundError') { throw error } + await this._setCanonicalGenesisBlock() + genesisHash = this._genesis } // load verified iterator heads @@ -181,10 +181,10 @@ export default class Blockchain implements BlockchainInterface { const heads = await this.dbManager.getHeads() this._heads = heads } catch (error) { - this._heads = {} if (error.type !== 'NotFoundError') { throw error } + this._heads = {} } // load headerchain head @@ -192,10 +192,10 @@ export default class Blockchain implements BlockchainInterface { const hash = await this.dbManager.getHeadHeader() this._headHeader = hash } catch (error) { - this._headHeader = genesisHash if (error.type !== 'NotFoundError') { throw error } + this._headHeader = genesisHash } // load blockchain head @@ -203,10 +203,10 @@ export default class Blockchain implements BlockchainInterface { const hash = await this.dbManager.getHeadBlock() this._headBlock = hash } catch (error) { - this._headBlock = genesisHash if (error.type !== 'NotFoundError') { throw error } + this._headBlock = genesisHash } this._initDone = true From b17439d725cf5a8ca8729f919e0ee70b1d37ff6c Mon Sep 17 00:00:00 2001 From: Jochem Brouwer Date: Wed, 21 Oct 2020 18:18:05 +0200 Subject: [PATCH 06/20] do not track dist.browser folder --- packages/block/.gitignore | 1 + packages/common/.gitignore | 1 + packages/ethash/.gitignore | 1 + packages/tx/.gitignore | 1 + 4 files changed, 4 insertions(+) diff --git a/packages/block/.gitignore b/packages/block/.gitignore index c6ff137115..04c72ae07a 100644 --- a/packages/block/.gitignore +++ b/packages/block/.gitignore @@ -1 +1,2 @@ test-build +dist.browser \ No newline at end of file diff --git a/packages/common/.gitignore b/packages/common/.gitignore index e69de29bb2..55bc9c30df 100644 --- a/packages/common/.gitignore +++ b/packages/common/.gitignore @@ -0,0 +1 @@ +dist.browser \ No newline at end of file diff --git a/packages/ethash/.gitignore b/packages/ethash/.gitignore index e69de29bb2..55bc9c30df 100644 --- a/packages/ethash/.gitignore +++ b/packages/ethash/.gitignore @@ -0,0 +1 @@ +dist.browser \ No newline at end of file diff --git a/packages/tx/.gitignore b/packages/tx/.gitignore index 2a2abe0df6..e10571555c 100644 --- a/packages/tx/.gitignore +++ b/packages/tx/.gitignore @@ -1 +1,2 @@ test-build/ +dist.browser \ No newline at end of file From 7b13006a60b53e93ce330c460fa9148f436c6156 Mon Sep 17 00:00:00 2001 From: Jochem Brouwer Date: Wed, 21 Oct 2020 20:41:05 +0200 Subject: [PATCH 07/20] blockchain: add DBInterface, fix test --- packages/blockchain/src/db/DB.ts | 0 packages/blockchain/src/db/DBInterface.ts | 18 ++++++ packages/blockchain/test/reorg.ts | 72 +++++++++++++++-------- packages/blockchain/test/util.ts | 9 +-- 4 files changed, 69 insertions(+), 30 deletions(-) create mode 100644 packages/blockchain/src/db/DB.ts create mode 100644 packages/blockchain/src/db/DBInterface.ts diff --git a/packages/blockchain/src/db/DB.ts b/packages/blockchain/src/db/DB.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/blockchain/src/db/DBInterface.ts b/packages/blockchain/src/db/DBInterface.ts new file mode 100644 index 0000000000..baed59cfe6 --- /dev/null +++ b/packages/blockchain/src/db/DBInterface.ts @@ -0,0 +1,18 @@ +import { BN } from 'ethereumjs-util' +import { Block, BlockHeader } from '@ethereumjs/block' + +export type BlockIdentifier = BN | Buffer // either a Block number or a BlockHash + +export interface BlockchainDB { + putTD(blockIdentifier: BlockIdentifier): Promise + getTD(blockIdentifier: BlockIdentifier): Promise + + putBlock(block: Block): Promise + getBlock(blockIdentifier: BlockIdentifier): Promise + + putBlockHeader(header: BlockHeader): Promise + getBlockHeader(blockIdentifier: BlockIdentifier): Promise + + getNumberByHash(hash: Buffer): Promise + getHashByNumber(number: BN): Promise +} diff --git a/packages/blockchain/test/reorg.ts b/packages/blockchain/test/reorg.ts index 0eb366b43d..e38a465354 100644 --- a/packages/blockchain/test/reorg.ts +++ b/packages/blockchain/test/reorg.ts @@ -1,17 +1,19 @@ import Common from '@ethereumjs/common' -import { Block, BlockHeader } from '@ethereumjs/block' -import { BN, toBuffer, bufferToInt } from 'ethereumjs-util' -import * as test from 'tape' +import { Block } from '@ethereumjs/block' +import { BN } from 'ethereumjs-util' +import tape from 'tape' import Blockchain from '../src' -import { generateBlocks, generateConsecutiveBlock } from './util' -import * as testData from './testdata.json' - -const level = require('level-mem') - -const genesis = generateBlocks(1)[0] -genesis.header.difficulty = Buffer.from('02000', 'hex') // minimum difficulty +import { generateConsecutiveBlock } from './util' + +const genesis = Block.fromBlockData({ + header: { + number: new BN(0), + difficulty: new BN(0x020000), + gasLimit: new BN(8000000), + }, +}) -test('reorg tests', (t) => { +tape('reorg tests', (t) => { t.test( 'should correctly reorg the chain if the total difficulty is higher on a lower block number than the current head block', async (st) => { @@ -19,13 +21,15 @@ test('reorg tests', (t) => { const blockchain = new Blockchain({ validateBlocks: true, validatePow: false, common }) await blockchain.putBlock(genesis, true) - let blocks_lowTD: Block[] = [] - let blocks_highTD: Block[] = [] + const blocks_lowTD: Block[] = [] + const blocks_highTD: Block[] = [] blocks_lowTD.push(generateConsecutiveBlock(genesis, 0)) - let TD_Low = new BN(genesis.header.difficulty).add(new BN(blocks_lowTD[0].header.difficulty)) - let TD_High = new BN(genesis.header.difficulty) + const TD_Low = new BN(genesis.header.difficulty).add( + new BN(blocks_lowTD[0].header.difficulty) + ) + const TD_High = new BN(genesis.header.difficulty) // Keep generating blocks until the Total Difficulty (TD) of the High TD chain is higher than the TD of the Low TD chain // This means that the block number of the high TD chain is 1 lower than the low TD chain @@ -33,7 +37,7 @@ test('reorg tests', (t) => { while (TD_High.cmp(TD_Low) == -1) { blocks_lowTD.push(generateConsecutiveBlock(blocks_lowTD[blocks_lowTD.length - 1], 0)) blocks_highTD.push( - generateConsecutiveBlock(blocks_highTD[blocks_highTD.length - 1] || genesis, 1), + generateConsecutiveBlock(blocks_highTD[blocks_highTD.length - 1] || genesis, 1) ) TD_Low.iadd(new BN(blocks_lowTD[blocks_lowTD.length - 1].header.difficulty)) @@ -44,23 +48,39 @@ test('reorg tests', (t) => { const lowTDBlock = blocks_lowTD[blocks_lowTD.length - 1] const highTDBlock = blocks_highTD[blocks_highTD.length - 1] - let number_lowTD = new BN(lowTDBlock.header.number) - let number_highTD = new BN(highTDBlock.header.number) - - // ensure that the block number is indeed higher on the low TD chain - t.ok(number_lowTD.cmp(number_highTD) == 1) + const number_lowTD = new BN(lowTDBlock.header.number) + const number_highTD = new BN(highTDBlock.header.number) + + // ensure that the block difficulty is higher on the highTD chain when compared to the low TD chain + t.ok( + number_lowTD.cmp(number_highTD) == 1, + 'low TD should have a lower TD than the reported high TD' + ) + t.ok( + blocks_lowTD[blocks_lowTD.length - 1].header.number.gt( + blocks_highTD[blocks_highTD.length - 1].header.number + ), + 'low TD block should have a higher number than high TD block' + ) await blockchain.putBlocks(blocks_lowTD) - let head_lowTD = await blockchain.getHead() + const head_lowTD = await blockchain.getHead() await blockchain.putBlocks(blocks_highTD) - let head_highTD = await blockchain.getHead() + const head_highTD = await blockchain.getHead() + + t.ok( + head_lowTD.hash().equals(lowTDBlock.hash()), + 'head on the low TD chain should equal the low TD block' + ) + t.ok( + head_highTD.hash().equals(highTDBlock.hash()), + 'head on the high TD chain should equal the high TD block' + ) - t.ok(head_lowTD.hash().equals(lowTDBlock.hash())) - t.ok(head_highTD.hash().equals(highTDBlock.hash())) st.end() - }, + } ) }) diff --git a/packages/blockchain/test/util.ts b/packages/blockchain/test/util.ts index 4920e92946..3e4cffc93e 100644 --- a/packages/blockchain/test/util.ts +++ b/packages/blockchain/test/util.ts @@ -1,4 +1,4 @@ -import { BN, rlp, toBuffer } from 'ethereumjs-util' +import { BN, rlp } from 'ethereumjs-util' import { Block, BlockHeader } from '@ethereumjs/block' import Common from '@ethereumjs/common' import Blockchain from '../src' @@ -54,6 +54,7 @@ export const generateBlockchain = async (numberOfBlocks: number, genesis?: Block blocks, error: null, } +} /** * * @param parentBlock parent block to generate the consecutive block on top of @@ -62,7 +63,7 @@ export const generateBlockchain = async (numberOfBlocks: number, genesis?: Block export const generateConsecutiveBlock = ( parentBlock: Block, - difficultyChangeFactor: number, + difficultyChangeFactor: number ): Block => { if (difficultyChangeFactor > 1) { difficultyChangeFactor = 1 @@ -78,11 +79,11 @@ export const generateConsecutiveBlock = ( parentHash: parentBlock.hash(), gasLimit: new BN(8000000), timestamp: parentBlock.header.timestamp.addn(10 + -difficultyChangeFactor * 9), - difficulty: new BN(tmpHeader.canonicalDifficulty(parentBlock)), + difficulty: new BN(tmpHeader.canonicalDifficulty(parentBlock.header)), }, { common, - }, + } ) const block = new Block(header, undefined, undefined, { common }) From 81939e105d5caceb93efe923e4804dfa800597ce Mon Sep 17 00:00:00 2001 From: Jochem Brouwer Date: Thu, 22 Oct 2020 18:10:27 +0200 Subject: [PATCH 08/20] stash --- .../src/db/BlockchainDBInterface.ts | 24 +++++++++++++++++++ packages/blockchain/src/db/DBInterface.ts | 18 -------------- 2 files changed, 24 insertions(+), 18 deletions(-) create mode 100644 packages/blockchain/src/db/BlockchainDBInterface.ts delete mode 100644 packages/blockchain/src/db/DBInterface.ts diff --git a/packages/blockchain/src/db/BlockchainDBInterface.ts b/packages/blockchain/src/db/BlockchainDBInterface.ts new file mode 100644 index 0000000000..3a2441a570 --- /dev/null +++ b/packages/blockchain/src/db/BlockchainDBInterface.ts @@ -0,0 +1,24 @@ +import { BN } from 'ethereumjs-util' +import { Block, BlockHeader } from '@ethereumjs/block' + +export type BlockIdentifier = BN | Buffer // either a Block number or a BlockHash + +export interface BlockchainDB { + // put a total difficulty for a given Block Hash (a block by block hash always has the same total difficulty) + putTD(blockHash: Buffer): Promise + getTD(blockHash: BlockIdentifier): Promise + + // put block. this should also put the block header in the database. + putBlock(block: Block): Promise + // return a block either by the block hash or take x-th block from the canonical chain + getBlock(blockIdentifier: BlockIdentifier): Promise + + putBlockHeader(header: BlockHeader): Promise + // return a block header by the block hash or take the x-th block from the canonical chain + getBlockHeader(blockIdentifier: BlockIdentifier): Promise + + // return the block number given a hash + getNumberByHash(hash: Buffer): Promise + // return the hash given by a block number (this returns the current block hash in the canonical chain) + getHashByNumber(number: BN): Promise +} diff --git a/packages/blockchain/src/db/DBInterface.ts b/packages/blockchain/src/db/DBInterface.ts deleted file mode 100644 index baed59cfe6..0000000000 --- a/packages/blockchain/src/db/DBInterface.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { BN } from 'ethereumjs-util' -import { Block, BlockHeader } from '@ethereumjs/block' - -export type BlockIdentifier = BN | Buffer // either a Block number or a BlockHash - -export interface BlockchainDB { - putTD(blockIdentifier: BlockIdentifier): Promise - getTD(blockIdentifier: BlockIdentifier): Promise - - putBlock(block: Block): Promise - getBlock(blockIdentifier: BlockIdentifier): Promise - - putBlockHeader(header: BlockHeader): Promise - getBlockHeader(blockIdentifier: BlockIdentifier): Promise - - getNumberByHash(hash: Buffer): Promise - getHashByNumber(number: BN): Promise -} From 046e4c85112d39efa164a5ba883d91d8b0284075 Mon Sep 17 00:00:00 2001 From: Jochem Brouwer Date: Fri, 23 Oct 2020 22:07:35 +0200 Subject: [PATCH 09/20] blockchain: refactor DBManager, create databaseOperation factory --- .../src/db/BlockchainDBInterface.ts | 24 --- packages/blockchain/src/db/DB.ts | 0 packages/blockchain/src/{ => db}/cache.ts | 0 .../blockchain/src/db/databaseOperation.ts | 143 ++++++++++++++++++ .../src/{util.ts => db/dbConstants.ts} | 0 packages/blockchain/src/{ => db}/dbManager.ts | 93 ++++++------ packages/blockchain/src/index.ts | 13 +- 7 files changed, 189 insertions(+), 84 deletions(-) delete mode 100644 packages/blockchain/src/db/BlockchainDBInterface.ts delete mode 100644 packages/blockchain/src/db/DB.ts rename packages/blockchain/src/{ => db}/cache.ts (100%) create mode 100644 packages/blockchain/src/db/databaseOperation.ts rename packages/blockchain/src/{util.ts => db/dbConstants.ts} (100%) rename packages/blockchain/src/{ => db}/dbManager.ts (62%) diff --git a/packages/blockchain/src/db/BlockchainDBInterface.ts b/packages/blockchain/src/db/BlockchainDBInterface.ts deleted file mode 100644 index 3a2441a570..0000000000 --- a/packages/blockchain/src/db/BlockchainDBInterface.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { BN } from 'ethereumjs-util' -import { Block, BlockHeader } from '@ethereumjs/block' - -export type BlockIdentifier = BN | Buffer // either a Block number or a BlockHash - -export interface BlockchainDB { - // put a total difficulty for a given Block Hash (a block by block hash always has the same total difficulty) - putTD(blockHash: Buffer): Promise - getTD(blockHash: BlockIdentifier): Promise - - // put block. this should also put the block header in the database. - putBlock(block: Block): Promise - // return a block either by the block hash or take x-th block from the canonical chain - getBlock(blockIdentifier: BlockIdentifier): Promise - - putBlockHeader(header: BlockHeader): Promise - // return a block header by the block hash or take the x-th block from the canonical chain - getBlockHeader(blockIdentifier: BlockIdentifier): Promise - - // return the block number given a hash - getNumberByHash(hash: Buffer): Promise - // return the hash given by a block number (this returns the current block hash in the canonical chain) - getHashByNumber(number: BN): Promise -} diff --git a/packages/blockchain/src/db/DB.ts b/packages/blockchain/src/db/DB.ts deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packages/blockchain/src/cache.ts b/packages/blockchain/src/db/cache.ts similarity index 100% rename from packages/blockchain/src/cache.ts rename to packages/blockchain/src/db/cache.ts diff --git a/packages/blockchain/src/db/databaseOperation.ts b/packages/blockchain/src/db/databaseOperation.ts new file mode 100644 index 0000000000..c294fcb429 --- /dev/null +++ b/packages/blockchain/src/db/databaseOperation.ts @@ -0,0 +1,143 @@ +import BN = require('bn.js') +import { + HEADS_KEY, + HEAD_HEADER_KEY, + HEAD_BLOCK_KEY, + tdKey, + headerKey, + bodyKey, + numberToHashKey, + hashToNumberKey, +} from './dbConstants' + +import { CacheMap } from './dbManager' + +export enum DatabaseOperationTarget { + Heads, + HeadHeader, + HeadBlock, + HashToNumber, + NumberToHash, + TotalDifficulty, + Body, + Header + } + + /** + * @hidden + */ +export interface DBOp { + type?: String + key: Buffer | string + keyEncoding: String + valueEncoding?: String + value?: Buffer | object +} + +// a Database Key is identified by a block hash, a block number, or both +export type DatabaseKey = { + blockNumber?: BN, + blockHash?: Buffer +} + + + + + +export class DatabaseOperation { + + public operationTarget: DatabaseOperationTarget + public baseDBOp: DBOp + public cacheString: string | undefined + + private constructor(operationTarget: DatabaseOperationTarget, key?: DatabaseKey) { + + this.operationTarget = operationTarget + + this.baseDBOp = { + key: '', + keyEncoding: 'binary', + } + + switch(operationTarget) { + case DatabaseOperationTarget.Heads: { + this.baseDBOp.key = HEADS_KEY + break + } + case DatabaseOperationTarget.HeadHeader: { + this.baseDBOp.key = HEAD_HEADER_KEY + break + } + case DatabaseOperationTarget.HeadBlock: { + this.baseDBOp.key = HEAD_BLOCK_KEY + break + } + case DatabaseOperationTarget.HashToNumber: { + this.baseDBOp.key = hashToNumberKey(key!.blockHash!) + this.cacheString = "hashToNumber" + break + } + case DatabaseOperationTarget.NumberToHash: { + this.baseDBOp.key = numberToHashKey(key!.blockNumber!) + this.cacheString = "numberToHash" + break + } + case DatabaseOperationTarget.TotalDifficulty: { + this.baseDBOp.key = tdKey(key!.blockNumber!, key!.blockHash!) + this.cacheString = "td" + break + } + case DatabaseOperationTarget.Body: { + this.baseDBOp.key = bodyKey(key!.blockNumber!, key!.blockHash!) + this.cacheString = "body" + break + } + case DatabaseOperationTarget.Header: { + this.baseDBOp.key = headerKey(key!.blockNumber!, key!.blockHash!) + this.cacheString = "header" + break + } + } + } + + public static get(operationTarget: DatabaseOperationTarget, key?: DatabaseKey): DatabaseOperation { + return new DatabaseOperation(operationTarget, key) + } + + // set operation: note: value/key is not in default order + public static set(operationTarget: DatabaseOperationTarget, value: Buffer | object, key?: DatabaseKey): DatabaseOperation { + const databaseOperation = new DatabaseOperation(operationTarget, key) + databaseOperation.baseDBOp.value = value + databaseOperation.baseDBOp.type = 'put' + + if (operationTarget == DatabaseOperationTarget.Heads) { + databaseOperation.baseDBOp.valueEncoding = 'json' + } else { + databaseOperation.baseDBOp.valueEncoding = 'binary' + } + + return databaseOperation + } + + public static del(operationTarget: DatabaseOperationTarget, key?: DatabaseKey): DatabaseOperation { + const databaseOperation = new DatabaseOperation(operationTarget, key) + databaseOperation.baseDBOp.type = 'del' + return databaseOperation + } + + + public updateCache(cacheMap: CacheMap) { + if (this.cacheString && cacheMap[this.cacheString] && Buffer.isBuffer(this.baseDBOp.value)) { + if (this.baseDBOp.type == 'put') { + cacheMap[this.cacheString].set(this.baseDBOp.key, this.baseDBOp.value) + } else if (this.baseDBOp.type == 'del') { + cacheMap[this.cacheString].del(this.baseDBOp.key) + } else { + throw(new Error("unsupported db operation on cache")) + } + } + } + + + +} \ No newline at end of file diff --git a/packages/blockchain/src/util.ts b/packages/blockchain/src/db/dbConstants.ts similarity index 100% rename from packages/blockchain/src/util.ts rename to packages/blockchain/src/db/dbConstants.ts diff --git a/packages/blockchain/src/dbManager.ts b/packages/blockchain/src/db/dbManager.ts similarity index 62% rename from packages/blockchain/src/dbManager.ts rename to packages/blockchain/src/db/dbManager.ts index 9a2fc653e6..d9415f57e6 100644 --- a/packages/blockchain/src/dbManager.ts +++ b/packages/blockchain/src/db/dbManager.ts @@ -9,32 +9,24 @@ import { } from '@ethereumjs/block' import Common from '@ethereumjs/common' import Cache from './cache' +import { DatabaseKey, DatabaseOperation, DatabaseOperationTarget, DBOp } from './databaseOperation' + import { HEADS_KEY, HEAD_HEADER_KEY, HEAD_BLOCK_KEY, - hashToNumberKey, - numberToHashKey, + tdKey, - bodyKey, headerKey, -} from './util' + bodyKey, + numberToHashKey, + hashToNumberKey, +} from './dbConstants' import type { LevelUp } from 'levelup' const level = require('level-mem') -/** - * @hidden - */ -export interface DBOp { - type: String - key: Buffer | String - keyEncoding: String - valueEncoding?: String - value?: Buffer | object -} - /** * @hidden */ @@ -44,13 +36,15 @@ export interface GetOpts { cache?: string } +export type CacheMap = { [key: string]: Cache } + /** * Abstraction over a DB to facilitate storing/fetching blockchain-related * data, such as blocks and headers, indices, and the head block. * @hidden */ export class DBManager { - _cache: { [k: string]: Cache } + _cache: CacheMap _common: Common _db: LevelUp @@ -70,7 +64,7 @@ export class DBManager { * Fetches iterator heads from the db. */ async getHeads(): Promise<{ [key: string]: Buffer }> { - const heads = await this.get(HEADS_KEY, { valueEncoding: 'json' }) + const heads = await this.get(DatabaseOperationTarget.Heads) Object.keys(heads).forEach((key) => { heads[key] = Buffer.from(heads[key]) }) @@ -81,14 +75,14 @@ export class DBManager { * Fetches header of the head block. */ async getHeadHeader(): Promise { - return this.get(HEAD_HEADER_KEY) + return this.get(DatabaseOperationTarget.HeadHeader) } /** * Fetches head block. */ async getHeadBlock(): Promise { - return this.get(HEAD_BLOCK_KEY) + return this.get(DatabaseOperationTarget.HeadBlock) } /** @@ -129,18 +123,16 @@ export class DBManager { /** * Fetches body of a block given its hash and number. */ - async getBody(hash: Buffer, number: BN): Promise { - const key = bodyKey(number, hash) - const body = await this.get(key, { cache: 'body' }) + async getBody(blockHash: Buffer, blockNumber: BN): Promise { + const body = await this.get(DatabaseOperationTarget.Body, { blockHash, blockNumber }) return (rlp.decode(body) as any) as BlockBodyBuffer } /** * Fetches header of a block given its hash and number. */ - async getHeader(hash: Buffer, number: BN) { - const key = headerKey(number, hash) - const encodedHeader = await this.get(key, { cache: 'header' }) + async getHeader(blockHash: Buffer, blockNumber: BN) { + const encodedHeader = await this.get(DatabaseOperationTarget.Header, { blockHash, blockNumber }) const opts = { common: this._common } return BlockHeader.fromRLPSerializedHeader(encodedHeader, opts) } @@ -148,31 +140,28 @@ export class DBManager { /** * Fetches total difficulty for a block given its hash and number. */ - async getTd(hash: Buffer, number: BN): Promise { - const key = tdKey(number, hash) - const td = await this.get(key, { cache: 'td' }) + async getTd(blockHash: Buffer, blockNumber: BN): Promise { + const td = await this.get(DatabaseOperationTarget.TotalDifficulty, { blockHash, blockNumber }) return new BN(rlp.decode(td)) } /** * Performs a block hash to block number lookup. */ - async hashToNumber(hash: Buffer): Promise { - const key = hashToNumberKey(hash) - const value = await this.get(key, { cache: 'hashToNumber' }) + async hashToNumber(blockHash: Buffer): Promise { + const value = await this.get(DatabaseOperationTarget.HashToNumber, { blockHash }) return new BN(value) } /** * Performs a block number to block hash lookup. */ - async numberToHash(number: BN): Promise { - if (number.ltn(0)) { + async numberToHash(blockNumber: BN): Promise { + if (blockNumber.ltn(0)) { throw new level.errors.NotFoundError() } - const key = numberToHashKey(number) - return this.get(key, { cache: 'numberToHash' }) + return this.get(DatabaseOperationTarget.NumberToHash, { blockNumber }) } /** @@ -180,33 +169,39 @@ export class DBManager { * it first tries to load from cache, and on cache miss will * try to put the fetched item on cache afterwards. */ - async get(key: string | Buffer, opts: GetOpts = {}): Promise { - const dbOpts = { - keyEncoding: opts.keyEncoding || 'binary', - valueEncoding: opts.valueEncoding || 'binary', - } + async get(databaseOperationTarget: DatabaseOperationTarget, key?: DatabaseKey): Promise { + const databaseGetOperation = DatabaseOperation.get(databaseOperationTarget, key) + - if (opts.cache) { - if (!this._cache[opts.cache]) { - throw new Error(`Invalid cache: ${opts.cache}`) + const cacheString = databaseGetOperation.cacheString + const dbKey = databaseGetOperation.baseDBOp.key + const dbOpts = databaseGetOperation.baseDBOp + + if (cacheString) { + if (!this._cache[cacheString]) { + throw new Error(`Invalid cache: ${cacheString}`) } - let value = this._cache[opts.cache].get(key) + let value = this._cache[cacheString].get(dbKey) if (!value) { - value = await this._db.get(key, dbOpts) - this._cache[opts.cache].set(key, value) + value = await this._db.get(dbKey, dbOpts) + this._cache[cacheString].set(dbKey, value) } return value } - return this._db.get(key, dbOpts) + return this._db.get(dbKey, dbOpts) } /** * Performs a batch operation on db. */ - async batch(ops: DBOp[]) { - return this._db.batch(ops as any) + async batch(ops: DatabaseOperation[]) { + const convertedOps: DBOp[] = ops.map((op) => op.baseDBOp) + // update the cache of each operation + ops.map((op) => op.updateCache(this._cache)) + + return this._db.batch(convertedOps as any) } } diff --git a/packages/blockchain/src/index.ts b/packages/blockchain/src/index.ts index 9ccbc9ea90..9678fadc43 100644 --- a/packages/blockchain/src/index.ts +++ b/packages/blockchain/src/index.ts @@ -3,17 +3,8 @@ import { BN, rlp } from 'ethereumjs-util' import { Block, BlockHeader } from '@ethereumjs/block' import Ethash from '@ethereumjs/ethash' import Common from '@ethereumjs/common' -import { DBManager, DBOp } from './dbManager' -import { - HEAD_BLOCK_KEY, - HEAD_HEADER_KEY, - bufBE8, - hashToNumberKey, - headerKey, - bodyKey, - numberToHashKey, - tdKey, -} from './util' +import { DBManager } from './dbManager' + import type { LevelUp } from 'levelup' From bd52c9b985fd2463d184ef6ae1440f03f00445c8 Mon Sep 17 00:00:00 2001 From: Jochem Brouwer Date: Fri, 23 Oct 2020 22:27:54 +0200 Subject: [PATCH 10/20] blockchain: include new DB changes --- packages/blockchain/src/db/dbManager.ts | 14 +-- packages/blockchain/src/index.ts | 155 ++++++++---------------- 2 files changed, 49 insertions(+), 120 deletions(-) diff --git a/packages/blockchain/src/db/dbManager.ts b/packages/blockchain/src/db/dbManager.ts index d9415f57e6..2543ba07f6 100644 --- a/packages/blockchain/src/db/dbManager.ts +++ b/packages/blockchain/src/db/dbManager.ts @@ -11,18 +11,6 @@ import Common from '@ethereumjs/common' import Cache from './cache' import { DatabaseKey, DatabaseOperation, DatabaseOperationTarget, DBOp } from './databaseOperation' -import { - HEADS_KEY, - HEAD_HEADER_KEY, - HEAD_BLOCK_KEY, - - tdKey, - headerKey, - bodyKey, - numberToHashKey, - hashToNumberKey, -} from './dbConstants' - import type { LevelUp } from 'levelup' const level = require('level-mem') @@ -199,7 +187,7 @@ export class DBManager { */ async batch(ops: DatabaseOperation[]) { const convertedOps: DBOp[] = ops.map((op) => op.baseDBOp) - // update the cache of each operation + // update the current cache for each operation ops.map((op) => op.updateCache(this._cache)) return this._db.batch(convertedOps as any) diff --git a/packages/blockchain/src/index.ts b/packages/blockchain/src/index.ts index 9678fadc43..5ad698561a 100644 --- a/packages/blockchain/src/index.ts +++ b/packages/blockchain/src/index.ts @@ -3,7 +3,9 @@ import { BN, rlp } from 'ethereumjs-util' import { Block, BlockHeader } from '@ethereumjs/block' import Ethash from '@ethereumjs/ethash' import Common from '@ethereumjs/common' -import { DBManager } from './dbManager' +import { DBManager } from './db/dbManager' +import { DatabaseOperationTarget, DatabaseOperation } from './db/databaseOperation' +import { bufBE8 } from './db/dbConstants' import type { LevelUp } from 'levelup' @@ -347,10 +349,11 @@ export default class Blockchain implements BlockchainInterface { const hash = block.hash() const { header } = block - const { number } = header + const blockHash = header.hash() + const blockNumber = header.number const td = header.difficulty.clone() const currentTd = { header: new BN(0), block: new BN(0) } - const dbOps: DBOp[] = [] + const dbOps: DatabaseOperation[] = [] if (block._common.chainId() !== this._common.chainId()) { throw new Error('Chain mismatch while trying to put block or header') @@ -377,7 +380,7 @@ export default class Blockchain implements BlockchainInterface { } // calculate the total difficulty of the new block - const parentTd = await this._getTd(header.parentHash, number.subn(1)) + const parentTd = await this._getTd(header.parentHash, blockNumber.subn(1)) td.iadd(parentTd) } @@ -387,23 +390,18 @@ export default class Blockchain implements BlockchainInterface { const valueEncoding = 'binary' // save block and total difficulty to the database - let key = tdKey(number, hash) - let value = rlp.encode(td) - dbOps.push({ type, key, value, keyEncoding, valueEncoding }) - this.dbManager._cache.td.set(key, value) + const TDValue = rlp.encode(td) + dbOps.push(DatabaseOperation.set(DatabaseOperationTarget.TotalDifficulty, TDValue, { blockNumber, blockHash })) // save header - key = headerKey(number, hash) - value = header.serialize() - dbOps.push({ type, key, value, keyEncoding, valueEncoding }) - this.dbManager._cache.header.set(key, value) + + const headerValue = header.serialize() + dbOps.push(DatabaseOperation.set(DatabaseOperationTarget.Header, headerValue, { blockNumber, blockHash })) // store body if it exists if (isGenesis || block.transactions.length || block.uncleHeaders.length) { - key = bodyKey(number, hash) - value = rlp.encode(block.raw().slice(1)) - dbOps.push({ type, key, value, keyEncoding, valueEncoding }) - this.dbManager._cache.body.set(key, value) + const bodyValue = rlp.encode(block.raw().slice(1)) + dbOps.push(DatabaseOperation.set(DatabaseOperationTarget.Body, bodyValue, { blockNumber, blockHash })) } // if total difficulty is higher than current, add it to canonical chain @@ -417,17 +415,15 @@ export default class Blockchain implements BlockchainInterface { } // delete higher number assignments and overwrite stale canonical chain - await this._deleteStaleAssignments(number.addn(1), hash, dbOps) + await this._deleteStaleAssignments(blockNumber.addn(1), hash, dbOps) await this._rebuildCanonical(header, dbOps) } else { if (td.gt(currentTd.block) && item instanceof Block) { this._headBlock = hash } // save hash to number lookup info even if rebuild not needed - key = hashToNumberKey(hash) - value = bufBE8(number) - dbOps.push({ type, key, keyEncoding, valueEncoding, value }) - this.dbManager._cache.hashToNumber.set(key, value) + const blockNumber8Byte = bufBE8(blockNumber) + dbOps.push(DatabaseOperation.set(DatabaseOperationTarget.HashToNumber, blockNumber8Byte, { blockHash })) } } @@ -534,29 +530,11 @@ export default class Blockchain implements BlockchainInterface { /** * @hidden */ - _saveHeadOps(): DBOp[] { + _saveHeadOps(): DatabaseOperation[] { return [ - { - type: 'put', - key: 'heads', - keyEncoding: 'binary', - valueEncoding: 'json', - value: this._heads, - }, - { - type: 'put', - key: HEAD_HEADER_KEY, - keyEncoding: 'binary', - valueEncoding: 'binary', - value: this._headHeader!, - }, - { - type: 'put', - key: HEAD_BLOCK_KEY, - keyEncoding: 'binary', - valueEncoding: 'binary', - value: this._headBlock!, - }, + DatabaseOperation.set(DatabaseOperationTarget.Heads, this._heads), + DatabaseOperation.set(DatabaseOperationTarget.HeadHeader, this._headHeader!), + DatabaseOperation.set(DatabaseOperationTarget.HeadBlock, this._headBlock!) ] } @@ -572,10 +550,10 @@ export default class Blockchain implements BlockchainInterface { * * @hidden */ - async _deleteStaleAssignments(number: BN, headHash: Buffer, ops: DBOp[]) { + async _deleteStaleAssignments(blockNumber: BN, headHash: Buffer, ops: DatabaseOperation[]) { let hash: Buffer try { - hash = await this.dbManager.numberToHash(number) + hash = await this.dbManager.numberToHash(blockNumber) } catch (error) { if (error.type !== 'NotFoundError') { throw error @@ -583,12 +561,7 @@ export default class Blockchain implements BlockchainInterface { return } - const type = 'del' - const key = numberToHashKey(number) - const keyEncoding = 'binary' - - ops.push({ type, key, keyEncoding }) - this.dbManager._cache.numberToHash.del(key) + ops.push(DatabaseOperation.del(DatabaseOperationTarget.NumberToHash, { blockNumber })) // reset stale iterator heads to current canonical head Object.keys(this._heads).forEach((name) => { @@ -602,7 +575,7 @@ export default class Blockchain implements BlockchainInterface { this._headBlock = headHash } - await this._deleteStaleAssignments(number.addn(1), headHash, ops) + await this._deleteStaleAssignments(blockNumber.addn(1), headHash, ops) } /** @@ -610,43 +583,34 @@ export default class Blockchain implements BlockchainInterface { * * @hidden */ - async _rebuildCanonical(header: BlockHeader, ops: DBOp[]) { - const hash = header.hash() - const { number } = header + async _rebuildCanonical(header: BlockHeader, ops: DatabaseOperation[]) { + const blockHash = header.hash() + const blockNumber = header.number const saveLookups = async (hash: Buffer, number: BN) => { - const type = 'put' - const keyEncoding = 'binary' - const valueEncoding = 'binary' - - let key = numberToHashKey(number) - let value = hash - ops.push({ type, key, keyEncoding, valueEncoding, value }) - this.dbManager._cache.numberToHash.set(key, value) + ops.push(DatabaseOperation.set(DatabaseOperationTarget.NumberToHash, blockHash, { blockNumber })) - key = hashToNumberKey(hash) - value = bufBE8(number) - ops.push({ type, key, keyEncoding, valueEncoding, value }) - this.dbManager._cache.hashToNumber.set(key, value) + const blockNumber8Bytes = bufBE8(number) + ops.push(DatabaseOperation.set(DatabaseOperationTarget.HashToNumber, blockNumber8Bytes, { blockHash })) } // handle genesis block - if (number.isZero()) { - await saveLookups(hash, number) + if (blockNumber.isZero()) { + await saveLookups(blockHash, blockNumber) return } let staleHash: Buffer | null = null try { - staleHash = await this.dbManager.numberToHash(number) + staleHash = await this.dbManager.numberToHash(blockNumber) } catch (error) { if (error.type !== 'NotFoundError') { throw error } } - if (!staleHash || !hash.equals(staleHash)) { - await saveLookups(hash, number) + if (!staleHash || !blockHash.equals(staleHash)) { + await saveLookups(blockHash, blockNumber) // flag stale head for reset Object.keys(this._heads).forEach((name) => { @@ -661,7 +625,7 @@ export default class Blockchain implements BlockchainInterface { } try { - const parentHeader = await this._getHeader(header.parentHash, number.subn(1)) + const parentHeader = await this._getHeader(header.parentHash, blockNumber.subn(1)) await this._rebuildCanonical(parentHeader, ops) } catch (error) { this._staleHeads = [] @@ -672,12 +636,12 @@ export default class Blockchain implements BlockchainInterface { } else { // set stale heads to last previously valid canonical block this._staleHeads.forEach((name: string) => { - this._heads[name] = hash + this._heads[name] = blockHash }) this._staleHeads = [] // set stale headBlock to last previously valid canonical block if (this._staleHeadBlock) { - this._headBlock = hash + this._headBlock = blockHash this._staleHeadBlock = false } } @@ -708,7 +672,7 @@ export default class Blockchain implements BlockchainInterface { * @hidden */ async _delBlock(blockHash: Buffer) { - const dbOps: DBOp[] = [] + const dbOps: DatabaseOperation[] = [] // get header const header = await this._getHeader(blockHash) @@ -742,50 +706,27 @@ export default class Blockchain implements BlockchainInterface { /** * @hidden */ - async _delChild(hash: Buffer, number: BN, headHash: Buffer | null, ops: DBOp[]) { + async _delChild(blockHash: Buffer, blockNumber: BN, headHash: Buffer | null, ops: DatabaseOperation[]) { // delete header, body, hash to number mapping and td - ops.push({ - type: 'del', - key: headerKey(number, hash), - keyEncoding: 'binary', - }) - this.dbManager._cache.header.del(headerKey(number, hash)) - - ops.push({ - type: 'del', - key: bodyKey(number, hash), - keyEncoding: 'binary', - }) - this.dbManager._cache.body.del(bodyKey(number, hash)) - - ops.push({ - type: 'del', - key: hashToNumberKey(hash), - keyEncoding: 'binary', - }) - this.dbManager._cache.hashToNumber.del(hashToNumberKey(hash)) - - ops.push({ - type: 'del', - key: tdKey(number, hash), - keyEncoding: 'binary', - }) - this.dbManager._cache.td.del(tdKey(number, hash)) + ops.push(DatabaseOperation.del(DatabaseOperationTarget.Header, { blockHash, blockNumber })) + ops.push(DatabaseOperation.del(DatabaseOperationTarget.Body, { blockHash, blockNumber })) + ops.push(DatabaseOperation.del(DatabaseOperationTarget.HashToNumber, { blockHash })) + ops.push(DatabaseOperation.del(DatabaseOperationTarget.TotalDifficulty, { blockHash, blockNumber })) if (!headHash) { return } - if (this._headHeader?.equals(hash)) { + if (this._headHeader?.equals(blockHash)) { this._headHeader = headHash } - if (this._headBlock?.equals(hash)) { + if (this._headBlock?.equals(blockHash)) { this._headBlock = headHash } try { - const childHeader = await this._getCanonicalHeader(number.addn(1)) + const childHeader = await this._getCanonicalHeader(blockNumber.addn(1)) await this._delChild(childHeader.hash(), childHeader.number, headHash, ops) } catch (error) { if (error.type !== 'NotFoundError') { From a2ad1d1bdd707676eb5450a8556131a492000302 Mon Sep 17 00:00:00 2001 From: Jochem Brouwer Date: Fri, 23 Oct 2020 22:29:26 +0200 Subject: [PATCH 11/20] blockchain: lint --- .../blockchain/src/db/databaseOperation.ts | 232 +++++++++--------- packages/blockchain/src/db/dbManager.ts | 1 - packages/blockchain/src/index.ts | 53 ++-- 3 files changed, 154 insertions(+), 132 deletions(-) diff --git a/packages/blockchain/src/db/databaseOperation.ts b/packages/blockchain/src/db/databaseOperation.ts index c294fcb429..9c40a5c778 100644 --- a/packages/blockchain/src/db/databaseOperation.ts +++ b/packages/blockchain/src/db/databaseOperation.ts @@ -1,143 +1,143 @@ import BN = require('bn.js') import { - HEADS_KEY, - HEAD_HEADER_KEY, - HEAD_BLOCK_KEY, - tdKey, - headerKey, - bodyKey, - numberToHashKey, - hashToNumberKey, + HEADS_KEY, + HEAD_HEADER_KEY, + HEAD_BLOCK_KEY, + tdKey, + headerKey, + bodyKey, + numberToHashKey, + hashToNumberKey, } from './dbConstants' import { CacheMap } from './dbManager' export enum DatabaseOperationTarget { - Heads, - HeadHeader, - HeadBlock, - HashToNumber, - NumberToHash, - TotalDifficulty, - Body, - Header - } - - /** - * @hidden - */ + Heads, + HeadHeader, + HeadBlock, + HashToNumber, + NumberToHash, + TotalDifficulty, + Body, + Header, +} + +/** + * @hidden + */ export interface DBOp { - type?: String - key: Buffer | string - keyEncoding: String - valueEncoding?: String - value?: Buffer | object + type?: String + key: Buffer | string + keyEncoding: String + valueEncoding?: String + value?: Buffer | object } // a Database Key is identified by a block hash, a block number, or both export type DatabaseKey = { - blockNumber?: BN, - blockHash?: Buffer + blockNumber?: BN + blockHash?: Buffer } - - - - export class DatabaseOperation { + public operationTarget: DatabaseOperationTarget + public baseDBOp: DBOp + public cacheString: string | undefined - public operationTarget: DatabaseOperationTarget - public baseDBOp: DBOp - public cacheString: string | undefined - - private constructor(operationTarget: DatabaseOperationTarget, key?: DatabaseKey) { - - this.operationTarget = operationTarget + private constructor(operationTarget: DatabaseOperationTarget, key?: DatabaseKey) { + this.operationTarget = operationTarget - this.baseDBOp = { - key: '', - keyEncoding: 'binary', - } - - switch(operationTarget) { - case DatabaseOperationTarget.Heads: { - this.baseDBOp.key = HEADS_KEY - break - } - case DatabaseOperationTarget.HeadHeader: { - this.baseDBOp.key = HEAD_HEADER_KEY - break - } - case DatabaseOperationTarget.HeadBlock: { - this.baseDBOp.key = HEAD_BLOCK_KEY - break - } - case DatabaseOperationTarget.HashToNumber: { - this.baseDBOp.key = hashToNumberKey(key!.blockHash!) - this.cacheString = "hashToNumber" - break - } - case DatabaseOperationTarget.NumberToHash: { - this.baseDBOp.key = numberToHashKey(key!.blockNumber!) - this.cacheString = "numberToHash" - break - } - case DatabaseOperationTarget.TotalDifficulty: { - this.baseDBOp.key = tdKey(key!.blockNumber!, key!.blockHash!) - this.cacheString = "td" - break - } - case DatabaseOperationTarget.Body: { - this.baseDBOp.key = bodyKey(key!.blockNumber!, key!.blockHash!) - this.cacheString = "body" - break - } - case DatabaseOperationTarget.Header: { - this.baseDBOp.key = headerKey(key!.blockNumber!, key!.blockHash!) - this.cacheString = "header" - break - } - } + this.baseDBOp = { + key: '', + keyEncoding: 'binary', } - public static get(operationTarget: DatabaseOperationTarget, key?: DatabaseKey): DatabaseOperation { - return new DatabaseOperation(operationTarget, key) + switch (operationTarget) { + case DatabaseOperationTarget.Heads: { + this.baseDBOp.key = HEADS_KEY + break + } + case DatabaseOperationTarget.HeadHeader: { + this.baseDBOp.key = HEAD_HEADER_KEY + break + } + case DatabaseOperationTarget.HeadBlock: { + this.baseDBOp.key = HEAD_BLOCK_KEY + break + } + case DatabaseOperationTarget.HashToNumber: { + this.baseDBOp.key = hashToNumberKey(key!.blockHash!) + this.cacheString = 'hashToNumber' + break + } + case DatabaseOperationTarget.NumberToHash: { + this.baseDBOp.key = numberToHashKey(key!.blockNumber!) + this.cacheString = 'numberToHash' + break + } + case DatabaseOperationTarget.TotalDifficulty: { + this.baseDBOp.key = tdKey(key!.blockNumber!, key!.blockHash!) + this.cacheString = 'td' + break + } + case DatabaseOperationTarget.Body: { + this.baseDBOp.key = bodyKey(key!.blockNumber!, key!.blockHash!) + this.cacheString = 'body' + break + } + case DatabaseOperationTarget.Header: { + this.baseDBOp.key = headerKey(key!.blockNumber!, key!.blockHash!) + this.cacheString = 'header' + break + } } + } - // set operation: note: value/key is not in default order - public static set(operationTarget: DatabaseOperationTarget, value: Buffer | object, key?: DatabaseKey): DatabaseOperation { - const databaseOperation = new DatabaseOperation(operationTarget, key) - databaseOperation.baseDBOp.value = value - databaseOperation.baseDBOp.type = 'put' - - if (operationTarget == DatabaseOperationTarget.Heads) { - databaseOperation.baseDBOp.valueEncoding = 'json' - } else { - databaseOperation.baseDBOp.valueEncoding = 'binary' - } + public static get( + operationTarget: DatabaseOperationTarget, + key?: DatabaseKey + ): DatabaseOperation { + return new DatabaseOperation(operationTarget, key) + } - return databaseOperation + // set operation: note: value/key is not in default order + public static set( + operationTarget: DatabaseOperationTarget, + value: Buffer | object, + key?: DatabaseKey + ): DatabaseOperation { + const databaseOperation = new DatabaseOperation(operationTarget, key) + databaseOperation.baseDBOp.value = value + databaseOperation.baseDBOp.type = 'put' + + if (operationTarget == DatabaseOperationTarget.Heads) { + databaseOperation.baseDBOp.valueEncoding = 'json' + } else { + databaseOperation.baseDBOp.valueEncoding = 'binary' } - public static del(operationTarget: DatabaseOperationTarget, key?: DatabaseKey): DatabaseOperation { - const databaseOperation = new DatabaseOperation(operationTarget, key) - databaseOperation.baseDBOp.type = 'del' - return databaseOperation - } + return databaseOperation + } + public static del( + operationTarget: DatabaseOperationTarget, + key?: DatabaseKey + ): DatabaseOperation { + const databaseOperation = new DatabaseOperation(operationTarget, key) + databaseOperation.baseDBOp.type = 'del' + return databaseOperation + } - public updateCache(cacheMap: CacheMap) { - if (this.cacheString && cacheMap[this.cacheString] && Buffer.isBuffer(this.baseDBOp.value)) { - if (this.baseDBOp.type == 'put') { - cacheMap[this.cacheString].set(this.baseDBOp.key, this.baseDBOp.value) - } else if (this.baseDBOp.type == 'del') { - cacheMap[this.cacheString].del(this.baseDBOp.key) - } else { - throw(new Error("unsupported db operation on cache")) - } - } + public updateCache(cacheMap: CacheMap) { + if (this.cacheString && cacheMap[this.cacheString] && Buffer.isBuffer(this.baseDBOp.value)) { + if (this.baseDBOp.type == 'put') { + cacheMap[this.cacheString].set(this.baseDBOp.key, this.baseDBOp.value) + } else if (this.baseDBOp.type == 'del') { + cacheMap[this.cacheString].del(this.baseDBOp.key) + } else { + throw new Error('unsupported db operation on cache') + } } - - - -} \ No newline at end of file + } +} diff --git a/packages/blockchain/src/db/dbManager.ts b/packages/blockchain/src/db/dbManager.ts index 2543ba07f6..a2c60b5262 100644 --- a/packages/blockchain/src/db/dbManager.ts +++ b/packages/blockchain/src/db/dbManager.ts @@ -160,7 +160,6 @@ export class DBManager { async get(databaseOperationTarget: DatabaseOperationTarget, key?: DatabaseKey): Promise { const databaseGetOperation = DatabaseOperation.get(databaseOperationTarget, key) - const cacheString = databaseGetOperation.cacheString const dbKey = databaseGetOperation.baseDBOp.key const dbOpts = databaseGetOperation.baseDBOp diff --git a/packages/blockchain/src/index.ts b/packages/blockchain/src/index.ts index 5ad698561a..91b78823c2 100644 --- a/packages/blockchain/src/index.ts +++ b/packages/blockchain/src/index.ts @@ -7,7 +7,6 @@ import { DBManager } from './db/dbManager' import { DatabaseOperationTarget, DatabaseOperation } from './db/databaseOperation' import { bufBE8 } from './db/dbConstants' - import type { LevelUp } from 'levelup' const level = require('level-mem') @@ -385,23 +384,30 @@ export default class Blockchain implements BlockchainInterface { } const rebuildInfo = async () => { - const type = 'put' - const keyEncoding = 'binary' - const valueEncoding = 'binary' - // save block and total difficulty to the database const TDValue = rlp.encode(td) - dbOps.push(DatabaseOperation.set(DatabaseOperationTarget.TotalDifficulty, TDValue, { blockNumber, blockHash })) + dbOps.push( + DatabaseOperation.set(DatabaseOperationTarget.TotalDifficulty, TDValue, { + blockNumber, + blockHash, + }) + ) // save header - const headerValue = header.serialize() - dbOps.push(DatabaseOperation.set(DatabaseOperationTarget.Header, headerValue, { blockNumber, blockHash })) + dbOps.push( + DatabaseOperation.set(DatabaseOperationTarget.Header, headerValue, { + blockNumber, + blockHash, + }) + ) // store body if it exists if (isGenesis || block.transactions.length || block.uncleHeaders.length) { const bodyValue = rlp.encode(block.raw().slice(1)) - dbOps.push(DatabaseOperation.set(DatabaseOperationTarget.Body, bodyValue, { blockNumber, blockHash })) + dbOps.push( + DatabaseOperation.set(DatabaseOperationTarget.Body, bodyValue, { blockNumber, blockHash }) + ) } // if total difficulty is higher than current, add it to canonical chain @@ -423,7 +429,11 @@ export default class Blockchain implements BlockchainInterface { } // save hash to number lookup info even if rebuild not needed const blockNumber8Byte = bufBE8(blockNumber) - dbOps.push(DatabaseOperation.set(DatabaseOperationTarget.HashToNumber, blockNumber8Byte, { blockHash })) + dbOps.push( + DatabaseOperation.set(DatabaseOperationTarget.HashToNumber, blockNumber8Byte, { + blockHash, + }) + ) } } @@ -534,7 +544,7 @@ export default class Blockchain implements BlockchainInterface { return [ DatabaseOperation.set(DatabaseOperationTarget.Heads, this._heads), DatabaseOperation.set(DatabaseOperationTarget.HeadHeader, this._headHeader!), - DatabaseOperation.set(DatabaseOperationTarget.HeadBlock, this._headBlock!) + DatabaseOperation.set(DatabaseOperationTarget.HeadBlock, this._headBlock!), ] } @@ -588,10 +598,16 @@ export default class Blockchain implements BlockchainInterface { const blockNumber = header.number const saveLookups = async (hash: Buffer, number: BN) => { - ops.push(DatabaseOperation.set(DatabaseOperationTarget.NumberToHash, blockHash, { blockNumber })) + ops.push( + DatabaseOperation.set(DatabaseOperationTarget.NumberToHash, blockHash, { blockNumber }) + ) const blockNumber8Bytes = bufBE8(number) - ops.push(DatabaseOperation.set(DatabaseOperationTarget.HashToNumber, blockNumber8Bytes, { blockHash })) + ops.push( + DatabaseOperation.set(DatabaseOperationTarget.HashToNumber, blockNumber8Bytes, { + blockHash, + }) + ) } // handle genesis block @@ -706,12 +722,19 @@ export default class Blockchain implements BlockchainInterface { /** * @hidden */ - async _delChild(blockHash: Buffer, blockNumber: BN, headHash: Buffer | null, ops: DatabaseOperation[]) { + async _delChild( + blockHash: Buffer, + blockNumber: BN, + headHash: Buffer | null, + ops: DatabaseOperation[] + ) { // delete header, body, hash to number mapping and td ops.push(DatabaseOperation.del(DatabaseOperationTarget.Header, { blockHash, blockNumber })) ops.push(DatabaseOperation.del(DatabaseOperationTarget.Body, { blockHash, blockNumber })) ops.push(DatabaseOperation.del(DatabaseOperationTarget.HashToNumber, { blockHash })) - ops.push(DatabaseOperation.del(DatabaseOperationTarget.TotalDifficulty, { blockHash, blockNumber })) + ops.push( + DatabaseOperation.del(DatabaseOperationTarget.TotalDifficulty, { blockHash, blockNumber }) + ) if (!headHash) { return From 144546850e196993e6cc90a4fcd9765605ac38d3 Mon Sep 17 00:00:00 2001 From: Jochem Brouwer Date: Sat, 24 Oct 2020 17:13:01 +0200 Subject: [PATCH 12/20] blockchain: fix tests --- packages/blockchain/src/db/databaseOperation.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/blockchain/src/db/databaseOperation.ts b/packages/blockchain/src/db/databaseOperation.ts index 9c40a5c778..d01ca8d8af 100644 --- a/packages/blockchain/src/db/databaseOperation.ts +++ b/packages/blockchain/src/db/databaseOperation.ts @@ -51,11 +51,13 @@ export class DatabaseOperation { this.baseDBOp = { key: '', keyEncoding: 'binary', + valueEncoding: 'binary' } switch (operationTarget) { case DatabaseOperationTarget.Heads: { this.baseDBOp.key = HEADS_KEY + this.baseDBOp.valueEncoding = 'json' break } case DatabaseOperationTarget.HeadHeader: { From 13c6f01a4728250741d4473cfbe32243efaeba90 Mon Sep 17 00:00:00 2001 From: Jochem Brouwer Date: Sat, 24 Oct 2020 17:15:45 +0200 Subject: [PATCH 13/20] monorepo: lint --- packages/block/src/header.ts | 8 ++++---- packages/blockchain/src/db/databaseOperation.ts | 2 +- packages/vm/tests/BlockchainTestsRunner.ts | 2 -- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/packages/block/src/header.ts b/packages/block/src/header.ts index 7838005f64..bf34d37374 100644 --- a/packages/block/src/header.ts +++ b/packages/block/src/header.ts @@ -264,7 +264,7 @@ export class BlockHeader { parentHeader: BlockHeader, timestamp: BN, number?: BN, - common?: Common, + common?: Common ): BN { const blockTs = timestamp.clone() const { timestamp: parentTs, difficulty: parentDif } = parentHeader @@ -272,10 +272,10 @@ export class BlockHeader { let num = (number || parentHeader.number).clone().addn(1) const hardfork = usedCommon.hardfork() || usedCommon.activeHardfork(num.toNumber()) const minimumDifficulty = new BN( - usedCommon.paramByHardfork('pow', 'minimumDifficulty', hardfork), + usedCommon.paramByHardfork('pow', 'minimumDifficulty', hardfork) ) const offset = parentDif.div( - new BN(usedCommon.paramByHardfork('pow', 'difficultyBoundDivisor', hardfork)), + new BN(usedCommon.paramByHardfork('pow', 'difficultyBoundDivisor', hardfork)) ) // We use a ! here as TS cannot follow this hardfork-dependent logic, but it always gets assigned @@ -354,7 +354,7 @@ export class BlockHeader { parentHeader, this.timestamp, this.number, - this._common, + this._common ) } diff --git a/packages/blockchain/src/db/databaseOperation.ts b/packages/blockchain/src/db/databaseOperation.ts index d01ca8d8af..621cd98980 100644 --- a/packages/blockchain/src/db/databaseOperation.ts +++ b/packages/blockchain/src/db/databaseOperation.ts @@ -51,7 +51,7 @@ export class DatabaseOperation { this.baseDBOp = { key: '', keyEncoding: 'binary', - valueEncoding: 'binary' + valueEncoding: 'binary', } switch (operationTarget) { diff --git a/packages/vm/tests/BlockchainTestsRunner.ts b/packages/vm/tests/BlockchainTestsRunner.ts index 164b7a4a2e..a01c25b831 100644 --- a/packages/vm/tests/BlockchainTestsRunner.ts +++ b/packages/vm/tests/BlockchainTestsRunner.ts @@ -86,9 +86,7 @@ export default async function runBlockchainTest(options: any, testData: any, t: let currentFork = common.hardfork() let currentBlock = new BN(0) - let lastBlock = new BN(0) for (const raw of testData.blocks) { - lastBlock = currentBlock const paramFork = `expectException${options.forkConfigTestSuite}` // Two naming conventions in ethereum/tests to indicate "exception occurs on all HFs" semantics // Last checked: ethereumjs-testing v1.3.1 (2020-05-11) From 632101d20c0794092b1349d88bbcc9b0dc2dcedc Mon Sep 17 00:00:00 2001 From: Jochem Brouwer Date: Sun, 25 Oct 2020 20:06:55 +0100 Subject: [PATCH 14/20] blockchain: add docs, rename internal functions, make functions private, fix tests --- packages/blockchain/package.json | 2 +- packages/blockchain/src/index.ts | 171 ++++++++++++++++++------------ packages/blockchain/test/index.ts | 18 ++-- packages/blockchain/test/reorg.ts | 2 +- 4 files changed, 114 insertions(+), 79 deletions(-) diff --git a/packages/blockchain/package.json b/packages/blockchain/package.json index 68c0ca1aff..c212222f68 100644 --- a/packages/blockchain/package.json +++ b/packages/blockchain/package.json @@ -19,7 +19,7 @@ "tsc": "ethereumjs-config-tsc", "lint": "ethereumjs-config-lint", "lint:fix": "ethereumjs-config-lint-fix", - "test": "tape -r ts-node/register ./test/index.ts" + "test": "tape -r ts-node/register ./test/*.ts" }, "repository": { "type": "git", diff --git a/packages/blockchain/src/index.ts b/packages/blockchain/src/index.ts index 91b78823c2..15de795b82 100644 --- a/packages/blockchain/src/index.ts +++ b/packages/blockchain/src/index.ts @@ -83,10 +83,12 @@ export default class Blockchain implements BlockchainInterface { dbManager: DBManager ethash?: Ethash - private _genesis?: Buffer - private _headBlock?: Buffer - private _headHeader?: Buffer - private _heads: { [key: string]: Buffer } + private _genesis?: Buffer // the genesis hash of this blockchain + private _headBlockHash?: Buffer // the hash of the current head block + private _headHeaderHash?: Buffer // the hash of the current head header + private _heads: { [key: string]: Buffer } // a Map which stores the head of each key (for instance the "vm" key) + + // these internal variables are used in _rebuildCanonical and are used in subsequent (recursive) calls to _rebuildCanonical private _staleHeadBlock: boolean private _staleHeads: string[] @@ -143,7 +145,7 @@ export default class Blockchain implements BlockchainInterface { */ get meta() { return { - rawHead: this._headHeader, + rawHead: this._headHeaderHash, heads: this._heads, genesis: this._genesis, } @@ -156,7 +158,7 @@ export default class Blockchain implements BlockchainInterface { * * @hidden */ - async _init() { + private async _init() { let genesisHash try { genesisHash = await this.dbManager.numberToHash(new BN(0)) @@ -182,23 +184,23 @@ export default class Blockchain implements BlockchainInterface { // load headerchain head try { const hash = await this.dbManager.getHeadHeader() - this._headHeader = hash + this._headHeaderHash = hash } catch (error) { if (error.type !== 'NotFoundError') { throw error } - this._headHeader = genesisHash + this._headHeaderHash = genesisHash } // load blockchain head try { const hash = await this.dbManager.getHeadBlock() - this._headBlock = hash + this._headBlockHash = hash } catch (error) { if (error.type !== 'NotFoundError') { throw error } - this._headBlock = genesisHash + this._headBlockHash = genesisHash } this._initDone = true @@ -209,13 +211,13 @@ export default class Blockchain implements BlockchainInterface { * * @hidden */ - async _setCanonicalGenesisBlock() { + private async _setCanonicalGenesisBlock() { const common = new Common({ chain: this._common.chainId(), hardfork: 'chainstart', }) const genesis = Block.genesis({}, { common }) - await this._putBlockOrHeader(genesis, true) + await this._putBlockOrHeader(genesis) } /** @@ -224,7 +226,10 @@ export default class Blockchain implements BlockchainInterface { * @param genesis - The genesis block to be added */ async putGenesis(genesis: Block) { - await this.putBlock(genesis, true) + if (!genesis.isGenesis()) { + throw new Error('Supplied block is not a genesis block') + } + await this.putBlock(genesis) } /** @@ -237,13 +242,21 @@ export default class Blockchain implements BlockchainInterface { await this._init() } - // if the head is not found return the headHeader - const hash = this._heads[name] || this._headBlock - if (!hash) { - throw new Error('No head found.') - } + await this._lock.wait() + try { + // if the head is not found return the headHeader + const hash = this._heads[name] || this._headBlockHash + if (!hash) { + throw new Error('No head found.') + } - return this.getBlock(hash) + const block = this.getBlock(hash) + this._lock.release() + return block + } catch (error) { + this._lock.release() + throw error + } } /** @@ -254,11 +267,11 @@ export default class Blockchain implements BlockchainInterface { await this._init() } - if (!this._headHeader) { + if (!this._headHeaderHash) { throw new Error('No head header set') } - const block = await this.getBlock(this._headHeader) + const block = await this.getBlock(this._headHeaderHash) return block.header } @@ -270,11 +283,11 @@ export default class Blockchain implements BlockchainInterface { await this._init() } - if (!this._headBlock) { + if (!this._headBlockHash) { throw new Error('No head block set') } - return this.getBlock(this._headBlock) + return this.getBlock(this._headBlockHash) } /** @@ -293,7 +306,7 @@ export default class Blockchain implements BlockchainInterface { * * @param block - The block to be added to the blockchain */ - async putBlock(block: Block, isGenesis?: boolean) { + async putBlock(block: Block) { if (!this._initDone) { await this._init() } @@ -301,7 +314,7 @@ export default class Blockchain implements BlockchainInterface { await this._lock.wait() try { - await this._putBlockOrHeader(block, isGenesis) + await this._putBlockOrHeader(block) this._lock.release() } catch (error) { this._lock.release() @@ -343,10 +356,10 @@ export default class Blockchain implements BlockchainInterface { /** * @hidden */ - async _putBlockOrHeader(item: Block | BlockHeader, isGenesis?: boolean) { + async _putBlockOrHeader(item: Block | BlockHeader) { const block = item instanceof BlockHeader ? new Block(item) : item + const isGenesis = block.isGenesis() - const hash = block.hash() const { header } = block const blockHash = header.hash() const blockNumber = header.number @@ -358,7 +371,7 @@ export default class Blockchain implements BlockchainInterface { throw new Error('Chain mismatch while trying to put block or header') } - if (this._validateBlocks && !isGenesis && !block.isGenesis()) { + if (this._validateBlocks && !isGenesis) { await block.validate(this) } @@ -371,11 +384,11 @@ export default class Blockchain implements BlockchainInterface { if (!isGenesis) { // set total difficulty in the current context scope - if (this._headHeader) { - currentTd.header = await this._getTd(this._headHeader) + if (this._headHeaderHash) { + currentTd.header = await this._getTd(this._headHeaderHash) } - if (this._headBlock) { - currentTd.block = await this._getTd(this._headBlock) + if (this._headBlockHash) { + currentTd.block = await this._getTd(this._headBlockHash) } // calculate the total difficulty of the new block @@ -412,20 +425,21 @@ export default class Blockchain implements BlockchainInterface { // if total difficulty is higher than current, add it to canonical chain if (block.isGenesis() || td.gt(currentTd.header)) { - this._headHeader = hash + this._headHeaderHash = blockHash if (item instanceof Block) { - this._headBlock = hash + this._headBlockHash = blockHash } if (block.isGenesis()) { - this._genesis = hash + this._genesis = blockHash } // delete higher number assignments and overwrite stale canonical chain - await this._deleteStaleAssignments(blockNumber.addn(1), hash, dbOps) + await this._deleteCanonicalChainReferences(blockNumber.addn(1), blockHash, dbOps) await this._rebuildCanonical(header, dbOps) } else { + // the TD is lower than the current highest TD so we will add the block to the DB, but will not mark it as the canonical chain. if (td.gt(currentTd.block) && item instanceof Block) { - this._headBlock = hash + this._headBlockHash = blockHash } // save hash to number lookup info even if rebuild not needed const blockNumber8Byte = bufBE8(blockNumber) @@ -507,8 +521,8 @@ export default class Blockchain implements BlockchainInterface { /** * Given an ordered array, returns an array of hashes that are not in the blockchain yet. - * - * @param hashes - Ordered array of hashes + * Uses binary search to find out what hashes are missing. Therefore, the array needs to be ordered upon number. + * @param hashes - Ordered array of hashes (ordered on `number`). */ async selectNeededHashes(hashes: Array) { let max: number @@ -538,29 +552,38 @@ export default class Blockchain implements BlockchainInterface { } /** + * Builds the `DatabaseOperation[]` list which describes the DB operations to write the heads, head header hash and the head header block to the DB * @hidden */ - _saveHeadOps(): DatabaseOperation[] { + private _saveHeadOps(): DatabaseOperation[] { return [ DatabaseOperation.set(DatabaseOperationTarget.Heads, this._heads), - DatabaseOperation.set(DatabaseOperationTarget.HeadHeader, this._headHeader!), - DatabaseOperation.set(DatabaseOperationTarget.HeadBlock, this._headBlock!), + DatabaseOperation.set(DatabaseOperationTarget.HeadHeader, this._headHeaderHash!), + DatabaseOperation.set(DatabaseOperationTarget.HeadBlock, this._headBlockHash!), ] } /** + * Gets the `DatabaseOperation[]` list to save `_heads`, `_headHeaderHash` and `_headBlockHash` and writes these to the DB * @hidden */ - async _saveHeads() { + private async _saveHeads() { return this.dbManager.batch(this._saveHeadOps()) } /** - * Delete canonical number assignments for specified number and above - * + * Pushes DB operations to delete canonical number assignments for specified block number and above + * Also + * @param blockNumber - the block number from which we start deleting canonical chain assignments (including this block) + * @param headHash - the hash of the current canonical chain head. The _heads reference matching any hash of any of the deleted blocks will be set to this + * @param ops - the DatabaseOperation list to write DatabaseOperations to * @hidden */ - async _deleteStaleAssignments(blockNumber: BN, headHash: Buffer, ops: DatabaseOperation[]) { + private async _deleteCanonicalChainReferences( + blockNumber: BN, + headHash: Buffer, + ops: DatabaseOperation[] + ) { let hash: Buffer try { hash = await this.dbManager.numberToHash(blockNumber) @@ -581,28 +604,28 @@ export default class Blockchain implements BlockchainInterface { }) // reset stale headBlock to current canonical - if (this._headBlock?.equals(hash)) { - this._headBlock = headHash + if (this._headBlockHash?.equals(hash)) { + this._headBlockHash = headHash } - await this._deleteStaleAssignments(blockNumber.addn(1), headHash, ops) + await this._deleteCanonicalChainReferences(blockNumber.addn(1), headHash, ops) } /** - * Overwrites stale canonical number assignments. + * Given a `header`, rebuild the canonical chain using this header. * * @hidden */ - async _rebuildCanonical(header: BlockHeader, ops: DatabaseOperation[]) { + private async _rebuildCanonical(header: BlockHeader, ops: DatabaseOperation[]) { const blockHash = header.hash() const blockNumber = header.number - const saveLookups = async (hash: Buffer, number: BN) => { + const saveLookups = (blockHash: Buffer, blockNumber: BN) => { ops.push( DatabaseOperation.set(DatabaseOperationTarget.NumberToHash, blockHash, { blockNumber }) ) - const blockNumber8Bytes = bufBE8(number) + const blockNumber8Bytes = bufBE8(blockNumber) ops.push( DatabaseOperation.set(DatabaseOperationTarget.HashToNumber, blockNumber8Bytes, { blockHash, @@ -612,10 +635,11 @@ export default class Blockchain implements BlockchainInterface { // handle genesis block if (blockNumber.isZero()) { - await saveLookups(blockHash, blockNumber) + saveLookups(blockHash, blockNumber) return } + // track the staleHash: this is the hash currently in the DB which matches the block number of the provided header. let staleHash: Buffer | null = null try { staleHash = await this.dbManager.numberToHash(blockNumber) @@ -626,9 +650,9 @@ export default class Blockchain implements BlockchainInterface { } if (!staleHash || !blockHash.equals(staleHash)) { - await saveLookups(blockHash, blockNumber) + saveLookups(blockHash, blockNumber) - // flag stale head for reset + // mark each key `_heads` which is currently set to the hash in the DB as stale to overwrite this later. Object.keys(this._heads).forEach((name) => { if (staleHash && this._heads[name].equals(staleHash)) { this._staleHeads = this._staleHeads || [] @@ -636,7 +660,7 @@ export default class Blockchain implements BlockchainInterface { } }) // flag stale headBlock for reset - if (staleHash && this._headBlock?.equals(staleHash)) { + if (staleHash && this._headBlockHash?.equals(staleHash)) { this._staleHeadBlock = true } @@ -650,6 +674,7 @@ export default class Blockchain implements BlockchainInterface { } } } else { + // the stale hash is equal to the blockHash // set stale heads to last previously valid canonical block this._staleHeads.forEach((name: string) => { this._heads[name] = blockHash @@ -657,15 +682,15 @@ export default class Blockchain implements BlockchainInterface { this._staleHeads = [] // set stale headBlock to last previously valid canonical block if (this._staleHeadBlock) { - this._headBlock = blockHash + this._headBlockHash = blockHash this._staleHeadBlock = false } } } /** - * Deletes a block from the blockchain. All child blocks in the chain are deleted and any - * encountered heads are set to the parent block. + * Completely deletes a block from the blockchain including any references to this block. All child blocks in the chain are deleted and any + * encountered heads are set to the parent block. If the block was in the canonical chain, also update the canonical chain and set the head of the canonical chain to the parent block. * * @param blockHash - The hash of the block to be deleted */ @@ -687,7 +712,7 @@ export default class Blockchain implements BlockchainInterface { /** * @hidden */ - async _delBlock(blockHash: Buffer) { + private async _delBlock(blockHash: Buffer) { const dbOps: DatabaseOperation[] = [] // get header @@ -713,13 +738,20 @@ export default class Blockchain implements BlockchainInterface { // delete all number to hash mappings for deleted block number and above if (inCanonical) { - await this._deleteStaleAssignments(blockNumber, parentHash, dbOps) + await this._deleteCanonicalChainReferences(blockNumber, parentHash, dbOps) } await this.dbManager.batch(dbOps) } /** + * Updates the `DatabaseOperation` list to delete a block from the DB, identified by `blockHash` and `blockNumber`. Deletes fields from `Header`, `Body`, `HashToNumber` and `TotalDifficulty` tables. + * If child blocks of this current block are in the canonical chain, delete these as well. Does not actually commit these changes to the DB. + * Sets `_headHeaderHash` and `_headBlockHash` to `headHash` if any of these matches the current child to be deleted. + * @param blockHash - the block hash to delete + * @param blockNumber - the number corresponding to the block hash + * @param headHash - the current head of the chain (if null, do not update `_headHeaderHash` and `_headBlockHash`) + * @param ops - the `DatabaseOperation` list to add the delete operations to * @hidden */ async _delChild( @@ -740,12 +772,12 @@ export default class Blockchain implements BlockchainInterface { return } - if (this._headHeader?.equals(blockHash)) { - this._headHeader = headHash + if (this._headHeaderHash?.equals(blockHash)) { + this._headHeaderHash = headHash } - if (this._headBlock?.equals(blockHash)) { - this._headBlock = headHash + if (this._headBlockHash?.equals(blockHash)) { + this._headBlockHash = headHash } try { @@ -777,7 +809,7 @@ export default class Blockchain implements BlockchainInterface { /** * @hidden */ - async _iterator(name: string, onBlock: OnBlock) { + private async _iterator(name: string, onBlock: OnBlock) { const blockHash = this._heads[name] || this._genesis let lastBlock: Block | undefined @@ -788,7 +820,8 @@ export default class Blockchain implements BlockchainInterface { const number = await this.dbManager.hashToNumber(blockHash) const blockNumber = number.addn(1) - while (blockNumber) { + // eslint-disable-next-line no-constant-condition + while (true) { try { const block = await this.getBlock(blockNumber) @@ -812,6 +845,8 @@ export default class Blockchain implements BlockchainInterface { await this._saveHeads() } + /* Helper functions */ + /** * Gets a header by hash and number. Header can exist outside the canonical chain * diff --git a/packages/blockchain/test/index.ts b/packages/blockchain/test/index.ts index 7156f46c13..22624beeba 100644 --- a/packages/blockchain/test/index.ts +++ b/packages/blockchain/test/index.ts @@ -46,7 +46,7 @@ tape('blockchain test', (t) => { }) const badBlock = Block.fromBlockData({ header: { number: new BN(8) } }) try { - await blockchain.putBlock(badBlock, false) + await blockchain.putBlock(badBlock) } catch (error) { st.ok(error, 'returned with error') st.end() @@ -383,12 +383,12 @@ tape('blockchain test', (t) => { } const forkHeader = BlockHeader.fromHeaderData(headerData, { common }) - blockchain._heads['staletest'] = blockchain._headHeader + blockchain._heads['staletest'] = blockchain._headHeaderHash await blockchain.putHeader(forkHeader) st.ok(blockchain._heads['staletest'].equals(blocks[14].hash()), 'should update stale head') - st.ok(blockchain._headBlock.equals(blocks[14].hash()), 'should update stale headBlock') + st.ok(blockchain._headBlockHash.equals(blocks[14].hash()), 'should update stale headBlock') st.end() }) @@ -406,17 +406,17 @@ tape('blockchain test', (t) => { } const forkHeader = BlockHeader.fromHeaderData(headerData, { common }) - blockchain._heads['staletest'] = blockchain._headHeader + blockchain._heads['staletest'] = blockchain._headHeaderHash await blockchain.putHeader(forkHeader) st.ok(blockchain._heads['staletest'].equals(blocks[14].hash()), 'should update stale head') - st.ok(blockchain._headBlock.equals(blocks[14].hash()), 'should update stale headBlock') + st.ok(blockchain._headBlockHash.equals(blocks[14].hash()), 'should update stale headBlock') await blockchain.delBlock(forkHeader.hash()) - st.ok(blockchain._headHeader.equals(blocks[14].hash()), 'should reset headHeader') - st.ok(blockchain._headBlock.equals(blocks[14].hash()), 'should not change headBlock') + st.ok(blockchain._headHeaderHash.equals(blocks[14].hash()), 'should reset headHeader') + st.ok(blockchain._headBlockHash.equals(blocks[14].hash()), 'should not change headBlock') st.end() }) @@ -433,7 +433,7 @@ tape('blockchain test', (t) => { } await delNextBlock(9) - st.ok(blockchain._headHeader.equals(blocks[5].hash()), 'should have block 5 as head') + st.ok(blockchain._headHeaderHash.equals(blocks[5].hash()), 'should have block 5 as head') st.end() }) @@ -441,7 +441,7 @@ tape('blockchain test', (t) => { const { blockchain, blocks, error } = await generateBlockchain(25) st.error(error, 'no error') await blockchain.delBlock(blocks[1].hash()) - st.ok(blockchain._headHeader.equals(blocks[0].hash()), 'should have genesis as head') + st.ok(blockchain._headHeaderHash.equals(blocks[0].hash()), 'should have genesis as head') st.end() }) diff --git a/packages/blockchain/test/reorg.ts b/packages/blockchain/test/reorg.ts index e38a465354..fe571c27e5 100644 --- a/packages/blockchain/test/reorg.ts +++ b/packages/blockchain/test/reorg.ts @@ -19,7 +19,7 @@ tape('reorg tests', (t) => { async (st) => { const common = new Common({ chain: 'mainnet', hardfork: 'muirGlacier' }) const blockchain = new Blockchain({ validateBlocks: true, validatePow: false, common }) - await blockchain.putBlock(genesis, true) + await blockchain.putBlock(genesis) const blocks_lowTD: Block[] = [] const blocks_highTD: Block[] = [] From 1224cbc71cfa2cc8ef31c66c697169f551422a52 Mon Sep 17 00:00:00 2001 From: Jochem Brouwer Date: Sun, 25 Oct 2020 20:15:23 +0100 Subject: [PATCH 15/20] blockchain: fix race conditions on getters/iterator lint --- packages/blockchain/src/index.ts | 236 +++++++++++++++++++------------ 1 file changed, 147 insertions(+), 89 deletions(-) diff --git a/packages/blockchain/src/index.ts b/packages/blockchain/src/index.ts index 15de795b82..ea308b5418 100644 --- a/packages/blockchain/src/index.ts +++ b/packages/blockchain/src/index.ts @@ -267,27 +267,42 @@ export default class Blockchain implements BlockchainInterface { await this._init() } - if (!this._headHeaderHash) { - throw new Error('No head header set') - } + await this._lock.wait() + try { + if (!this._headHeaderHash) { + throw new Error('No head header set') + } - const block = await this.getBlock(this._headHeaderHash) - return block.header + const block = await this.getBlock(this._headHeaderHash) + this._lock.release() + return block.header + } catch (error) { + this._lock.release() + throw error + } } /** * Returns the latest full block in the canonical chain. */ - async getLatestBlock() { + async getLatestBlock(): Promise { if (!this._initDone) { await this._init() } - if (!this._headBlockHash) { - throw new Error('No head block set') - } + await this._lock.wait() + try { + if (!this._headBlockHash) { + throw new Error('No head block set') + } - return this.getBlock(this._headBlockHash) + const block = this.getBlock(this._headBlockHash) + this._lock.release() + return block + } catch (error) { + this._lock.release() + throw error + } } /** @@ -296,8 +311,15 @@ export default class Blockchain implements BlockchainInterface { * @param blocks - The blocks to be added to the blockchain */ async putBlocks(blocks: Block[]) { - for (let i = 0; i < blocks.length; i++) { - await this.putBlock(blocks[i]) + await this._lock.wait() + try { + for (let i = 0; i < blocks.length; i++) { + await this._putBlockOrHeader(blocks[i]) + } + this._lock.release() + } catch (error) { + this._lock.release() + throw error } } @@ -328,8 +350,15 @@ export default class Blockchain implements BlockchainInterface { * @param headers - The headers to be added to the blockchain */ async putHeaders(headers: Array) { - for (let i = 0; i < headers.length; i++) { - await this.putHeader(headers[i]) + await this._lock.wait() + try { + for (let i = 0; i < headers.length; i++) { + await this._putBlockOrHeader(headers[i]) + } + this._lock.release() + } catch (error) { + this._lock.release() + throw error } } @@ -467,13 +496,21 @@ export default class Blockchain implements BlockchainInterface { await this._init() } - return this._getBlock(blockId) + await this._lock.wait() + try { + const block = await this._getBlock(blockId) + this._lock.release() + return block + } catch (error) { + this._lock.release() + throw error + } } /** * @hidden */ - async _getBlock(blockId: Buffer | number | BN) { + private async _getBlock(blockId: Buffer | number | BN) { return this.dbManager.getBlock(blockId) } @@ -491,32 +528,39 @@ export default class Blockchain implements BlockchainInterface { skip: number, reverse: boolean ): Promise { - const blocks: Block[] = [] - let i = -1 - - const nextBlock = async (blockId: Buffer | BN | number): Promise => { - let block - try { - block = await this.getBlock(blockId) - } catch (error) { - if (error.type !== 'NotFoundError') { - throw error + await this._lock.wait() + try { + const blocks: Block[] = [] + let i = -1 + + const nextBlock = async (blockId: Buffer | BN | number): Promise => { + let block + try { + block = await this.getBlock(blockId) + } catch (error) { + if (error.type !== 'NotFoundError') { + throw error + } + return + } + i++ + const nextBlockNumber = block.header.number.addn(reverse ? -1 : 1) + if (i !== 0 && skip && i % (skip + 1) !== 0) { + return await nextBlock(nextBlockNumber) + } + blocks.push(block) + if (blocks.length < maxBlocks) { + await nextBlock(nextBlockNumber) } - return - } - i++ - const nextBlockNumber = block.header.number.addn(reverse ? -1 : 1) - if (i !== 0 && skip && i % (skip + 1) !== 0) { - return await nextBlock(nextBlockNumber) - } - blocks.push(block) - if (blocks.length < maxBlocks) { - await nextBlock(nextBlockNumber) } - } - await nextBlock(blockId) - return blocks + await nextBlock(blockId) + this._lock.release() + return blocks + } catch (error) { + this._lock.release() + throw error + } } /** @@ -524,31 +568,38 @@ export default class Blockchain implements BlockchainInterface { * Uses binary search to find out what hashes are missing. Therefore, the array needs to be ordered upon number. * @param hashes - Ordered array of hashes (ordered on `number`). */ - async selectNeededHashes(hashes: Array) { - let max: number - let mid: number - let min: number - - max = hashes.length - 1 - mid = min = 0 - - while (max >= min) { - let number - try { - number = await this.dbManager.hashToNumber(hashes[mid]) - } catch (error) { - if (error.type !== 'NotFoundError') { - throw error + async selectNeededHashes(hashes: Array): Promise { + await this._lock.wait() + try { + let max: number + let mid: number + let min: number + + max = hashes.length - 1 + mid = min = 0 + + while (max >= min) { + let number + try { + number = await this.dbManager.hashToNumber(hashes[mid]) + } catch (error) { + if (error.type !== 'NotFoundError') { + throw error + } } + if (number) { + min = mid + 1 + } else { + max = mid - 1 + } + mid = Math.floor((min + max) / 2) } - if (number) { - min = mid + 1 - } else { - max = mid - 1 - } - mid = Math.floor((min + max) / 2) + this._lock.release() + return hashes.slice(min) + } catch (error) { + this._lock.release() + throw error } - return hashes.slice(min) } /** @@ -754,7 +805,7 @@ export default class Blockchain implements BlockchainInterface { * @param ops - the `DatabaseOperation` list to add the delete operations to * @hidden */ - async _delChild( + private async _delChild( blockHash: Buffer, blockNumber: BN, headHash: Buffer | null, @@ -810,39 +861,46 @@ export default class Blockchain implements BlockchainInterface { * @hidden */ private async _iterator(name: string, onBlock: OnBlock) { - const blockHash = this._heads[name] || this._genesis - let lastBlock: Block | undefined - - if (!blockHash) { - return - } - - const number = await this.dbManager.hashToNumber(blockHash) - const blockNumber = number.addn(1) + await this._lock.wait() + try { + const blockHash = this._heads[name] || this._genesis + let lastBlock: Block | undefined - // eslint-disable-next-line no-constant-condition - while (true) { - try { - const block = await this.getBlock(blockNumber) + if (!blockHash) { + return + } - if (!block) { + const number = await this.dbManager.hashToNumber(blockHash) + const blockNumber = number.addn(1) + + // eslint-disable-next-line no-constant-condition + while (true) { + try { + const block = await this.getBlock(blockNumber) + + if (!block) { + break + } + + this._heads[name] = block.hash() + const reorg = lastBlock ? lastBlock.hash().equals(block.header.parentHash) : false + lastBlock = block + await onBlock(block, reorg) + blockNumber.iaddn(1) + } catch (error) { + if (error.type !== 'NotFoundError') { + throw error + } break } - - this._heads[name] = block.hash() - const reorg = lastBlock ? lastBlock.hash().equals(block.header.parentHash) : false - lastBlock = block - await onBlock(block, reorg) - blockNumber.iaddn(1) - } catch (error) { - if (error.type !== 'NotFoundError') { - throw error - } - break } - } - await this._saveHeads() + await this._saveHeads() + this._lock.release() + } catch (error) { + this._lock.release() + throw error + } } /* Helper functions */ From 861cab485ac2024759361e976c74af5066218edc Mon Sep 17 00:00:00 2001 From: Jochem Brouwer Date: Sun, 25 Oct 2020 21:34:14 +0100 Subject: [PATCH 16/20] blockchain: fix concurrency --- packages/blockchain/src/index.ts | 211 ++++++++++++++---------------- packages/blockchain/test/index.ts | 3 +- 2 files changed, 102 insertions(+), 112 deletions(-) diff --git a/packages/blockchain/src/index.ts b/packages/blockchain/src/index.ts index ea308b5418..8a99a3fa57 100644 --- a/packages/blockchain/src/index.ts +++ b/packages/blockchain/src/index.ts @@ -93,6 +93,7 @@ export default class Blockchain implements BlockchainInterface { private _staleHeads: string[] private _initDone: boolean + private _initPromise: Promise | undefined private _lock: Semaphore private _common: Common @@ -138,6 +139,9 @@ export default class Blockchain implements BlockchainInterface { this._lock = new Semaphore(1) this._initDone = false + + // eslint-disable-next-line @typescript-eslint/no-floating-promises + this._initPromise = this._init() } /** @@ -239,7 +243,7 @@ export default class Blockchain implements BlockchainInterface { */ async getHead(name = 'vm'): Promise { if (!this._initDone) { - await this._init() + await this._initPromise } await this._lock.wait() @@ -264,7 +268,7 @@ export default class Blockchain implements BlockchainInterface { */ async getLatestHeader(): Promise { if (!this._initDone) { - await this._init() + await this._initPromise } await this._lock.wait() @@ -273,7 +277,7 @@ export default class Blockchain implements BlockchainInterface { throw new Error('No head header set') } - const block = await this.getBlock(this._headHeaderHash) + const block = await this._getBlock(this._headHeaderHash) this._lock.release() return block.header } catch (error) { @@ -287,7 +291,7 @@ export default class Blockchain implements BlockchainInterface { */ async getLatestBlock(): Promise { if (!this._initDone) { - await this._init() + await this._initPromise } await this._lock.wait() @@ -296,7 +300,7 @@ export default class Blockchain implements BlockchainInterface { throw new Error('No head block set') } - const block = this.getBlock(this._headBlockHash) + const block = this._getBlock(this._headBlockHash) this._lock.release() return block } catch (error) { @@ -311,15 +315,11 @@ export default class Blockchain implements BlockchainInterface { * @param blocks - The blocks to be added to the blockchain */ async putBlocks(blocks: Block[]) { - await this._lock.wait() - try { - for (let i = 0; i < blocks.length; i++) { - await this._putBlockOrHeader(blocks[i]) - } - this._lock.release() - } catch (error) { - this._lock.release() - throw error + if (!this._initDone) { + await this._initPromise + } + for (let i = 0; i < blocks.length; i++) { + await this._putBlockOrHeader(blocks[i]) } } @@ -330,18 +330,10 @@ export default class Blockchain implements BlockchainInterface { */ async putBlock(block: Block) { if (!this._initDone) { - await this._init() + await this._initPromise } - await this._lock.wait() - - try { - await this._putBlockOrHeader(block) - this._lock.release() - } catch (error) { - this._lock.release() - throw error - } + await this._putBlockOrHeader(block) } /** @@ -350,15 +342,8 @@ export default class Blockchain implements BlockchainInterface { * @param headers - The headers to be added to the blockchain */ async putHeaders(headers: Array) { - await this._lock.wait() - try { - for (let i = 0; i < headers.length; i++) { - await this._putBlockOrHeader(headers[i]) - } - this._lock.release() - } catch (error) { - this._lock.release() - throw error + for (let i = 0; i < headers.length; i++) { + await this._putBlockOrHeader(headers[i]) } } @@ -369,23 +354,16 @@ export default class Blockchain implements BlockchainInterface { */ async putHeader(header: BlockHeader) { if (!this._initDone) { - await this._init() + await this._initPromise } - await this._lock.wait() - try { - await this._putBlockOrHeader(header) - this._lock.release() - } catch (error) { - this._lock.release() - throw error - } + await this._putBlockOrHeader(header) } /** * @hidden */ - async _putBlockOrHeader(item: Block | BlockHeader) { + private async _putBlockOrHeader(item: Block | BlockHeader) { const block = item instanceof BlockHeader ? new Block(item) : item const isGenesis = block.isGenesis() @@ -395,7 +373,6 @@ export default class Blockchain implements BlockchainInterface { const td = header.difficulty.clone() const currentTd = { header: new BN(0), block: new BN(0) } const dbOps: DatabaseOperation[] = [] - if (block._common.chainId() !== this._common.chainId()) { throw new Error('Chain mismatch while trying to put block or header') } @@ -411,79 +388,90 @@ export default class Blockchain implements BlockchainInterface { } } - if (!isGenesis) { - // set total difficulty in the current context scope - if (this._headHeaderHash) { - currentTd.header = await this._getTd(this._headHeaderHash) - } - if (this._headBlockHash) { - currentTd.block = await this._getTd(this._headBlockHash) - } - - // calculate the total difficulty of the new block - const parentTd = await this._getTd(header.parentHash, blockNumber.subn(1)) - td.iadd(parentTd) - } - - const rebuildInfo = async () => { - // save block and total difficulty to the database - const TDValue = rlp.encode(td) - dbOps.push( - DatabaseOperation.set(DatabaseOperationTarget.TotalDifficulty, TDValue, { - blockNumber, - blockHash, - }) - ) + try { + await this._lock.wait() + if (!isGenesis) { + // set total difficulty in the current context scope + if (this._headHeaderHash) { + currentTd.header = await this._getTd(this._headHeaderHash) + } + if (this._headBlockHash) { + currentTd.block = await this._getTd(this._headBlockHash) + } - // save header - const headerValue = header.serialize() - dbOps.push( - DatabaseOperation.set(DatabaseOperationTarget.Header, headerValue, { - blockNumber, - blockHash, - }) - ) + // calculate the total difficulty of the new block + const parentTd = await this._getTd(header.parentHash, blockNumber.subn(1)) + td.iadd(parentTd) + } - // store body if it exists - if (isGenesis || block.transactions.length || block.uncleHeaders.length) { - const bodyValue = rlp.encode(block.raw().slice(1)) + const rebuildInfo = async () => { + // save block and total difficulty to the database + const TDValue = rlp.encode(td) dbOps.push( - DatabaseOperation.set(DatabaseOperationTarget.Body, bodyValue, { blockNumber, blockHash }) + DatabaseOperation.set(DatabaseOperationTarget.TotalDifficulty, TDValue, { + blockNumber, + blockHash, + }) ) - } - - // if total difficulty is higher than current, add it to canonical chain - if (block.isGenesis() || td.gt(currentTd.header)) { - this._headHeaderHash = blockHash - if (item instanceof Block) { - this._headBlockHash = blockHash - } - if (block.isGenesis()) { - this._genesis = blockHash - } - // delete higher number assignments and overwrite stale canonical chain - await this._deleteCanonicalChainReferences(blockNumber.addn(1), blockHash, dbOps) - await this._rebuildCanonical(header, dbOps) - } else { - // the TD is lower than the current highest TD so we will add the block to the DB, but will not mark it as the canonical chain. - if (td.gt(currentTd.block) && item instanceof Block) { - this._headBlockHash = blockHash - } - // save hash to number lookup info even if rebuild not needed - const blockNumber8Byte = bufBE8(blockNumber) + // save header + const headerValue = header.serialize() dbOps.push( - DatabaseOperation.set(DatabaseOperationTarget.HashToNumber, blockNumber8Byte, { + DatabaseOperation.set(DatabaseOperationTarget.Header, headerValue, { + blockNumber, blockHash, }) ) + + // store body if it exists + if (isGenesis || block.transactions.length || block.uncleHeaders.length) { + const bodyValue = rlp.encode(block.raw().slice(1)) + dbOps.push( + DatabaseOperation.set(DatabaseOperationTarget.Body, bodyValue, { + blockNumber, + blockHash, + }) + ) + } + + // if total difficulty is higher than current, add it to canonical chain + if (block.isGenesis() || td.gt(currentTd.header)) { + this._headHeaderHash = blockHash + if (item instanceof Block) { + this._headBlockHash = blockHash + } + if (block.isGenesis()) { + this._genesis = blockHash + } + + // delete higher number assignments and overwrite stale canonical chain + + await this._deleteCanonicalChainReferences(blockNumber.addn(1), blockHash, dbOps) + await this._rebuildCanonical(header, dbOps) + } else { + // the TD is lower than the current highest TD so we will add the block to the DB, but will not mark it as the canonical chain. + if (td.gt(currentTd.block) && item instanceof Block) { + this._headBlockHash = blockHash + } + // save hash to number lookup info even if rebuild not needed + const blockNumber8Byte = bufBE8(blockNumber) + dbOps.push( + DatabaseOperation.set(DatabaseOperationTarget.HashToNumber, blockNumber8Byte, { + blockHash, + }) + ) + } } - } - await rebuildInfo() + await rebuildInfo() - const ops = dbOps.concat(this._saveHeadOps()) - await this.dbManager.batch(ops) + const ops = dbOps.concat(this._saveHeadOps()) + await this.dbManager.batch(ops) + this._lock.release() + } catch (e) { + this._lock.release() + throw e + } } /** @@ -493,7 +481,7 @@ export default class Blockchain implements BlockchainInterface { */ async getBlock(blockId: Buffer | number | BN): Promise { if (!this._initDone) { - await this._init() + await this._initPromise } await this._lock.wait() @@ -528,6 +516,9 @@ export default class Blockchain implements BlockchainInterface { skip: number, reverse: boolean ): Promise { + if (!this._initDone) { + await this._initPromise + } await this._lock.wait() try { const blocks: Block[] = [] @@ -536,7 +527,7 @@ export default class Blockchain implements BlockchainInterface { const nextBlock = async (blockId: Buffer | BN | number): Promise => { let block try { - block = await this.getBlock(blockId) + block = await this._getBlock(blockId) } catch (error) { if (error.type !== 'NotFoundError') { throw error @@ -747,7 +738,7 @@ export default class Blockchain implements BlockchainInterface { */ async delBlock(blockHash: Buffer) { if (!this._initDone) { - await this._init() + await this._initPromise } await this._lock.wait() @@ -851,7 +842,7 @@ export default class Blockchain implements BlockchainInterface { */ async iterator(name: string, onBlock: OnBlock) { if (!this._initDone) { - await this._init() + await this._initPromise } return this._iterator(name, onBlock) @@ -876,7 +867,7 @@ export default class Blockchain implements BlockchainInterface { // eslint-disable-next-line no-constant-condition while (true) { try { - const block = await this.getBlock(blockNumber) + const block = await this._getBlock(blockNumber) if (!block) { break diff --git a/packages/blockchain/test/index.ts b/packages/blockchain/test/index.ts index 22624beeba..8176f644a7 100644 --- a/packages/blockchain/test/index.ts +++ b/packages/blockchain/test/index.ts @@ -46,7 +46,7 @@ tape('blockchain test', (t) => { }) const badBlock = Block.fromBlockData({ header: { number: new BN(8) } }) try { - await blockchain.putBlock(badBlock) + await blockchain.putGenesis(badBlock) } catch (error) { st.ok(error, 'returned with error') st.end() @@ -564,7 +564,6 @@ tape('blockchain test', (t) => { } const header = BlockHeader.fromHeaderData(headerData) await blockchain.putHeader(header) - blockchain = new Blockchain({ db, validateBlocks: true, From fd19635363eca7232eaf1134aaa0d9a89579c58d Mon Sep 17 00:00:00 2001 From: Jochem Brouwer Date: Tue, 27 Oct 2020 17:10:36 +0100 Subject: [PATCH 17/20] block: fix static canonical functions --- packages/block/src/block.ts | 16 ++++++++++++++++ packages/block/src/header.ts | 33 +++++++++++++++------------------ 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/packages/block/src/block.ts b/packages/block/src/block.ts index cbd3c91dd6..3cb3cb34a2 100644 --- a/packages/block/src/block.ts +++ b/packages/block/src/block.ts @@ -246,6 +246,22 @@ export class Block { return this.header.canonicalDifficulty(parentBlock.header) } + /** + * Returns the canonical difficulty of any new block, given the timestamp of the consecutive block and (optionally) the block number and the common + * @param block - The block to get the canonical difficulty for + * @param timestamp - The timestamp of the consecutive block + * @param number - Optional, the number of the consecutive block + * @param common - Optional, the `common` to use in the difficulty calculation + */ + public static getCanonicalDifficulty( + block: Block, + timestamp: BN, + number?: BN, + common?: Common + ): BN { + return BlockHeader.getCanonicalDifficulty(block.header, timestamp, number, common) + } + /** * Checks that the block's `difficulty` matches the canonical difficulty. * diff --git a/packages/block/src/header.ts b/packages/block/src/header.ts index bf34d37374..57e576b1c2 100644 --- a/packages/block/src/header.ts +++ b/packages/block/src/header.ts @@ -254,36 +254,34 @@ export class BlockHeader { } /** - * @param parentBlock - The `parentBlock` where we wish to build upon + * @param header - The `header` where we wish to build upon * @param timestamp - The `BN` timestamp which we should calculate the difficulty for * @param number - The `BN` block number. Defaults to the parent blocks' `number` + 1 * @param common - The `Common` to use: defaults to the parent blocks` `Common` */ public static getCanonicalDifficulty( - parentHeader: BlockHeader, + header: BlockHeader, timestamp: BN, number?: BN, common?: Common ): BN { const blockTs = timestamp.clone() - const { timestamp: parentTs, difficulty: parentDif } = parentHeader - const usedCommon = common || parentHeader._common - let num = (number || parentHeader.number).clone().addn(1) - const hardfork = usedCommon.hardfork() || usedCommon.activeHardfork(num.toNumber()) - const minimumDifficulty = new BN( - usedCommon.paramByHardfork('pow', 'minimumDifficulty', hardfork) - ) + const { timestamp: parentTs, difficulty: parentDif } = header + common = common || header._common + let num = (number || header.number).clone().addn(1) + const hardfork = common.hardfork() || common.activeHardfork(num.toNumber()) + const minimumDifficulty = new BN(common.paramByHardfork('pow', 'minimumDifficulty', hardfork)) const offset = parentDif.div( - new BN(usedCommon.paramByHardfork('pow', 'difficultyBoundDivisor', hardfork)) + new BN(common.paramByHardfork('pow', 'difficultyBoundDivisor', hardfork)) ) // We use a ! here as TS cannot follow this hardfork-dependent logic, but it always gets assigned let dif!: BN - if (usedCommon.hardforkGteHardfork(hardfork, 'byzantium')) { + if (common.hardforkGteHardfork(hardfork, 'byzantium')) { // max((2 if len(parent.uncles) else 1) - ((timestamp - parent.timestamp) // 9), -99) (EIP100) - const uncleAddend = parentHeader.uncleHash.equals(KECCAK256_RLP_ARRAY) ? 1 : 2 + const uncleAddend = header.uncleHash.equals(KECCAK256_RLP_ARRAY) ? 1 : 2 let a = blockTs.sub(parentTs).idivn(9).ineg().iaddn(uncleAddend) const cutoff = new BN(-99) // MAX(cutoff, a) @@ -293,25 +291,25 @@ export class BlockHeader { dif = parentDif.add(offset.mul(a)) } - if (usedCommon.hardforkGteHardfork(hardfork, 'muirGlacier')) { + if (common.hardforkGteHardfork(hardfork, 'muirGlacier')) { // Istanbul/Berlin difficulty bomb delay (EIP2384) num.isubn(9000000) if (num.ltn(0)) { num = new BN(0) } - } else if (usedCommon.hardforkGteHardfork(hardfork, 'constantinople')) { + } else if (common.hardforkGteHardfork(hardfork, 'constantinople')) { // Constantinople difficulty bomb delay (EIP1234) num.isubn(5000000) if (num.ltn(0)) { num = new BN(0) } - } else if (usedCommon.hardforkGteHardfork(hardfork, 'byzantium')) { + } else if (common.hardforkGteHardfork(hardfork, 'byzantium')) { // Byzantium difficulty bomb delay (EIP649) num.isubn(3000000) if (num.ltn(0)) { num = new BN(0) } - } else if (usedCommon.hardforkGteHardfork(hardfork, 'homestead')) { + } else if (common.hardforkGteHardfork(hardfork, 'homestead')) { // 1 - (block_timestamp - parent_timestamp) // 10 let a = blockTs.sub(parentTs).idivn(10).ineg().iaddn(1) const cutoff = new BN(-99) @@ -323,8 +321,7 @@ export class BlockHeader { } else { // pre-homestead if ( - parentTs.addn(usedCommon.paramByHardfork('pow', 'durationLimit', hardfork)).cmp(blockTs) === - 1 + parentTs.addn(common.paramByHardfork('pow', 'durationLimit', hardfork)).cmp(blockTs) === 1 ) { dif = offset.add(parentDif) } else { From e3fe5562589d21d73bef5e613c8124b1a26c45c2 Mon Sep 17 00:00:00 2001 From: Jochem Brouwer Date: Tue, 27 Oct 2020 20:12:08 +0100 Subject: [PATCH 18/20] blockchain: add helper functions for initializing and locks --- packages/blockchain/src/index.ts | 155 ++++++++++--------------------- 1 file changed, 51 insertions(+), 104 deletions(-) diff --git a/packages/blockchain/src/index.ts b/packages/blockchain/src/index.ts index 8a99a3fa57..0095ae1c09 100644 --- a/packages/blockchain/src/index.ts +++ b/packages/blockchain/src/index.ts @@ -162,7 +162,7 @@ export default class Blockchain implements BlockchainInterface { * * @hidden */ - private async _init() { + private async _init(): Promise { let genesisHash try { genesisHash = await this.dbManager.numberToHash(new BN(0)) @@ -206,10 +206,35 @@ export default class Blockchain implements BlockchainInterface { } this._headBlockHash = genesisHash } - this._initDone = true } + /** + * Perform the `action` function after we have initialized this module and have acquired a lock + * @param action - the action function to run after initializing and acquiring a lock + * @hidden + */ + private async initAndLock(action: () => Promise): Promise { + await this._initPromise + return await this.runWithLock(action) + } + + /** + * Run a function after acquiring a lock. It is implied that we have already initialized the module (or we are calling this from the init function, like `_setCanonicalGenesisBlock`) + * @param action - function to run after acquiring a lock + * @hidden + */ + private async runWithLock(action: () => Promise): Promise { + try { + await this._lock.acquire() + const value = await action() + this._lock.release() + return value + } finally { + this._lock.release() + } + } + /** * Sets the default genesis block * @@ -242,71 +267,44 @@ export default class Blockchain implements BlockchainInterface { * @param name - Optional name of the state root head (default: 'vm') */ async getHead(name = 'vm'): Promise { - if (!this._initDone) { - await this._initPromise - } - - await this._lock.wait() - try { + return await this.initAndLock(async () => { // if the head is not found return the headHeader const hash = this._heads[name] || this._headBlockHash if (!hash) { throw new Error('No head found.') } - const block = this.getBlock(hash) - this._lock.release() + const block = await this._getBlock(hash) return block - } catch (error) { - this._lock.release() - throw error - } + }) } /** * Returns the latest header in the canonical chain. */ async getLatestHeader(): Promise { - if (!this._initDone) { - await this._initPromise - } - - await this._lock.wait() - try { + return await this.initAndLock(async () => { if (!this._headHeaderHash) { throw new Error('No head header set') } const block = await this._getBlock(this._headHeaderHash) - this._lock.release() return block.header - } catch (error) { - this._lock.release() - throw error - } + }) } /** * Returns the latest full block in the canonical chain. */ async getLatestBlock(): Promise { - if (!this._initDone) { - await this._initPromise - } - - await this._lock.wait() - try { + return this.initAndLock(async () => { if (!this._headBlockHash) { throw new Error('No head block set') } const block = this._getBlock(this._headBlockHash) - this._lock.release() return block - } catch (error) { - this._lock.release() - throw error - } + }) } /** @@ -315,9 +313,7 @@ export default class Blockchain implements BlockchainInterface { * @param blocks - The blocks to be added to the blockchain */ async putBlocks(blocks: Block[]) { - if (!this._initDone) { - await this._initPromise - } + await this._initPromise for (let i = 0; i < blocks.length; i++) { await this._putBlockOrHeader(blocks[i]) } @@ -329,10 +325,7 @@ export default class Blockchain implements BlockchainInterface { * @param block - The block to be added to the blockchain */ async putBlock(block: Block) { - if (!this._initDone) { - await this._initPromise - } - + await this._initPromise await this._putBlockOrHeader(block) } @@ -342,6 +335,7 @@ export default class Blockchain implements BlockchainInterface { * @param headers - The headers to be added to the blockchain */ async putHeaders(headers: Array) { + await this._initPromise for (let i = 0; i < headers.length; i++) { await this._putBlockOrHeader(headers[i]) } @@ -353,10 +347,7 @@ export default class Blockchain implements BlockchainInterface { * @param header - The header to be added to the blockchain */ async putHeader(header: BlockHeader) { - if (!this._initDone) { - await this._initPromise - } - + await this._initPromise await this._putBlockOrHeader(header) } @@ -378,6 +369,7 @@ export default class Blockchain implements BlockchainInterface { } if (this._validateBlocks && !isGenesis) { + // this calls into `getBlock`, which is why we cannot lock yet await block.validate(this) } @@ -388,8 +380,7 @@ export default class Blockchain implements BlockchainInterface { } } - try { - await this._lock.wait() + await this.runWithLock(async () => { if (!isGenesis) { // set total difficulty in the current context scope if (this._headHeaderHash) { @@ -467,11 +458,7 @@ export default class Blockchain implements BlockchainInterface { const ops = dbOps.concat(this._saveHeadOps()) await this.dbManager.batch(ops) - this._lock.release() - } catch (e) { - this._lock.release() - throw e - } + }) } /** @@ -480,19 +467,10 @@ export default class Blockchain implements BlockchainInterface { * @param blockId - The block's hash or number */ async getBlock(blockId: Buffer | number | BN): Promise { - if (!this._initDone) { - await this._initPromise - } - - await this._lock.wait() - try { + return await this.initAndLock(async () => { const block = await this._getBlock(blockId) - this._lock.release() return block - } catch (error) { - this._lock.release() - throw error - } + }) } /** @@ -516,11 +494,7 @@ export default class Blockchain implements BlockchainInterface { skip: number, reverse: boolean ): Promise { - if (!this._initDone) { - await this._initPromise - } - await this._lock.wait() - try { + return await this.initAndLock(async () => { const blocks: Block[] = [] let i = -1 @@ -546,12 +520,8 @@ export default class Blockchain implements BlockchainInterface { } await nextBlock(blockId) - this._lock.release() return blocks - } catch (error) { - this._lock.release() - throw error - } + }) } /** @@ -560,8 +530,7 @@ export default class Blockchain implements BlockchainInterface { * @param hashes - Ordered array of hashes (ordered on `number`). */ async selectNeededHashes(hashes: Array): Promise { - await this._lock.wait() - try { + return await this.initAndLock(async () => { let max: number let mid: number let min: number @@ -585,12 +554,8 @@ export default class Blockchain implements BlockchainInterface { } mid = Math.floor((min + max) / 2) } - this._lock.release() return hashes.slice(min) - } catch (error) { - this._lock.release() - throw error - } + }) } /** @@ -737,18 +702,9 @@ export default class Blockchain implements BlockchainInterface { * @param blockHash - The hash of the block to be deleted */ async delBlock(blockHash: Buffer) { - if (!this._initDone) { - await this._initPromise - } - - await this._lock.wait() - try { + await this.initAndLock(async () => { await this._delBlock(blockHash) - this._lock.release() - } catch (error) { - this._lock.release() - throw error - } + }) } /** @@ -841,10 +797,6 @@ export default class Blockchain implements BlockchainInterface { * @param onBlock - Function called on each block with params (block, reorg) */ async iterator(name: string, onBlock: OnBlock) { - if (!this._initDone) { - await this._initPromise - } - return this._iterator(name, onBlock) } @@ -852,8 +804,7 @@ export default class Blockchain implements BlockchainInterface { * @hidden */ private async _iterator(name: string, onBlock: OnBlock) { - await this._lock.wait() - try { + await this.initAndLock(async () => { const blockHash = this._heads[name] || this._genesis let lastBlock: Block | undefined @@ -887,11 +838,7 @@ export default class Blockchain implements BlockchainInterface { } await this._saveHeads() - this._lock.release() - } catch (error) { - this._lock.release() - throw error - } + }) } /* Helper functions */ From dcb1b3787e9dce6131259cb9faff6756599409ef Mon Sep 17 00:00:00 2001 From: Jochem Brouwer Date: Wed, 28 Oct 2020 15:47:25 +0100 Subject: [PATCH 19/20] blockchain: if non-canonical genesis block, provide this is constructor vm, blockchain: lint --- packages/blockchain/src/index.ts | 22 ++++++++++++++++--- packages/vm/tests/BlockchainTestsRunner.ts | 25 +++++++++++----------- 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/packages/blockchain/src/index.ts b/packages/blockchain/src/index.ts index 0095ae1c09..f93ba1b9b2 100644 --- a/packages/blockchain/src/index.ts +++ b/packages/blockchain/src/index.ts @@ -73,6 +73,12 @@ export interface BlockchainOptions { * provided, it defaults to `true`. */ validateBlocks?: boolean + + /** + * If a genesis block is present in the provided `db`, then this genesis block will be used. + * If there is no genesis block in the `db`, use this `genesisBlock`: if none is provided, use the standard genesis block. + */ + genesisBlock?: Block } /** @@ -141,7 +147,7 @@ export default class Blockchain implements BlockchainInterface { this._initDone = false // eslint-disable-next-line @typescript-eslint/no-floating-promises - this._initPromise = this._init() + this._initPromise = this._init(opts.genesisBlock) } /** @@ -162,7 +168,7 @@ export default class Blockchain implements BlockchainInterface { * * @hidden */ - private async _init(): Promise { + private async _init(genesisBlock?: Block): Promise { let genesisHash try { genesisHash = await this.dbManager.numberToHash(new BN(0)) @@ -170,7 +176,11 @@ export default class Blockchain implements BlockchainInterface { if (error.type !== 'NotFoundError') { throw error } - await this._setCanonicalGenesisBlock() + if (genesisBlock) { + await this._putBlockOrHeader(genesisBlock) + } else { + await this._setCanonicalGenesisBlock() + } genesisHash = this._genesis } @@ -358,6 +368,12 @@ export default class Blockchain implements BlockchainInterface { const block = item instanceof BlockHeader ? new Block(item) : item const isGenesis = block.isGenesis() + // we cannot overwrite the Genesis block after initializing the Blockchain + + if (this._initDone && isGenesis) { + throw new Error('Cannot put a new Genesis block: create a new Blockchain') + } + const { header } = block const blockHash = header.hash() const blockNumber = header.number diff --git a/packages/vm/tests/BlockchainTestsRunner.ts b/packages/vm/tests/BlockchainTestsRunner.ts index a01c25b831..9d16d5b7d7 100644 --- a/packages/vm/tests/BlockchainTestsRunner.ts +++ b/packages/vm/tests/BlockchainTestsRunner.ts @@ -34,11 +34,22 @@ export default async function runBlockchainTest(options: any, testData: any, t: const { common } = options common.setHardforkByBlockNumber(0) + // create and add genesis block + const header = formatBlockHeader(testData.genesisBlockHeader) + const blockData = { header } + const genesisBlock = Block.fromBlockData(blockData, { common }) + + if (testData.genesisRLP) { + const rlp = toBuffer(testData.genesisRLP) + t.ok(genesisBlock.serialize().equals(rlp), 'correct genesis RLP') + } + const blockchain = new Blockchain({ db: blockchainDB, common, validateBlocks: true, validatePow, + genesisBlock, }) if (validatePow) { @@ -58,22 +69,10 @@ export default async function runBlockchainTest(options: any, testData: any, t: common, }) - // create and add genesis block - const header = formatBlockHeader(testData.genesisBlockHeader) - const blockData = { header } - const genesis = Block.fromBlockData(blockData, { common }) - // set up pre-state await setupPreConditions(vm.stateManager._trie, testData) - t.ok(vm.stateManager._trie.root.equals(genesis.header.stateRoot), 'correct pre stateRoot') - - if (testData.genesisRLP) { - const rlp = toBuffer(testData.genesisRLP) - t.ok(genesis.serialize().equals(rlp), 'correct genesis RLP') - } - - await blockchain.putGenesis(genesis) + t.ok(vm.stateManager._trie.root.equals(genesisBlock.header.stateRoot), 'correct pre stateRoot') async function handleError(error: string | undefined, expectException: string) { if (expectException) { From 99b4ab2b278380f7356c9d3fad212bd71caea085 Mon Sep 17 00:00:00 2001 From: Jochem Brouwer Date: Wed, 28 Oct 2020 18:47:28 +0100 Subject: [PATCH 20/20] blockchain: genesis block options checks, static factory --- packages/blockchain/src/index.ts | 12 ++ packages/blockchain/test/index.ts | 182 ++++++++++---------- packages/blockchain/test/reorg.ts | 18 +- packages/blockchain/test/util.ts | 8 +- packages/vm/tests/api/index.spec.ts | 17 +- packages/vm/tests/api/runBlockchain.spec.ts | 39 +++-- packages/vm/tests/api/utils.ts | 13 +- 7 files changed, 163 insertions(+), 126 deletions(-) diff --git a/packages/blockchain/src/index.ts b/packages/blockchain/src/index.ts index f93ba1b9b2..8117552630 100644 --- a/packages/blockchain/src/index.ts +++ b/packages/blockchain/src/index.ts @@ -146,10 +146,22 @@ export default class Blockchain implements BlockchainInterface { this._lock = new Semaphore(1) this._initDone = false + if (opts.genesisBlock && !opts.genesisBlock.isGenesis()) { + throw 'supplied block is not a genesis block' + } + // eslint-disable-next-line @typescript-eslint/no-floating-promises this._initPromise = this._init(opts.genesisBlock) } + public static async create(opts: BlockchainOptions = {}) { + const blockchain = new Blockchain(opts) + await blockchain._initPromise!.catch((e) => { + throw e + }) + return blockchain + } + /** * Returns an object with metadata about the Blockchain. It's defined for backwards compatibility. */ diff --git a/packages/blockchain/test/index.ts b/packages/blockchain/test/index.ts index 8176f644a7..b280e6f413 100644 --- a/packages/blockchain/test/index.ts +++ b/packages/blockchain/test/index.ts @@ -29,24 +29,28 @@ tape('blockchain test', (t) => { }) t.test('should add a genesis block without errors', async (st) => { + const genesisBlock = Block.genesis() const blockchain = new Blockchain({ validateBlocks: true, validatePow: false, + genesisBlock, }) - const genesis = Block.genesis() - await blockchain.putGenesis(genesis) - st.ok(genesis.hash().equals(blockchain.meta.genesis!), 'genesis block hash should be correct') + await (blockchain)._initPromise + st.ok( + genesisBlock.hash().equals(blockchain.meta.genesis!), + 'genesis block hash should be correct' + ) st.end() }) t.test('should not validate a block incorrectly flagged as genesis', async (st) => { - const blockchain = new Blockchain({ - validateBlocks: true, - validatePow: false, - }) - const badBlock = Block.fromBlockData({ header: { number: new BN(8) } }) + const genesisBlock = Block.fromBlockData({ header: { number: new BN(8) } }) try { - await blockchain.putGenesis(badBlock) + await Blockchain.create({ + validateBlocks: true, + validatePow: false, + genesisBlock, + }) } catch (error) { st.ok(error, 'returned with error') st.end() @@ -64,16 +68,17 @@ tape('blockchain test', (t) => { }) t.test('should add 10 blocks, one at a time', async (st) => { + const blocks: Block[] = [] + const gasLimit = 8000000 + + const genesisBlock = Block.genesis({ header: { gasLimit } }) + blocks.push(genesisBlock) + const blockchain = new Blockchain({ validateBlocks: true, validatePow: false, + genesisBlock, }) - const blocks: Block[] = [] - const gasLimit = 8000000 - - const genesis = Block.genesis({ header: { gasLimit } }) - blocks.push(genesis) - await blockchain.putGenesis(genesis) const addNextBlock = async (number: number) => { const lastBlock = blocks[number - 1] @@ -103,23 +108,24 @@ tape('blockchain test', (t) => { }) t.test('should get block by number', async (st) => { + const blocks: Block[] = [] + const gasLimit = 8000000 + + const genesisBlock = Block.genesis({ header: { gasLimit } }) + blocks.push(genesisBlock) + const blockchain = new Blockchain({ validateBlocks: true, validatePow: false, + genesisBlock, }) - const blocks: Block[] = [] - const gasLimit = 8000000 - - const genesis = Block.genesis({ header: { gasLimit } }) - blocks.push(genesis) - await blockchain.putGenesis(genesis) const blockData = { header: { number: 1, - parentHash: genesis.hash(), - difficulty: genesis.canonicalDifficulty(genesis), - timestamp: genesis.header.timestamp.addn(1), + parentHash: genesisBlock.hash(), + difficulty: genesisBlock.canonicalDifficulty(genesisBlock), + timestamp: genesisBlock.header.timestamp.addn(1), gasLimit, }, } @@ -137,18 +143,17 @@ tape('blockchain test', (t) => { }) t.test('should get block by hash', async (st) => { + const gasLimit = 8000000 + const genesisBlock = Block.genesis({ header: { gasLimit } }) + const blockchain = new Blockchain({ validateBlocks: true, validatePow: false, + genesisBlock, }) - const gasLimit = 8000000 - - const genesis = Block.genesis({ header: { gasLimit } }) - await blockchain.putGenesis(genesis) - - const block = await blockchain.getBlock(genesis.hash()) + const block = await blockchain.getBlock(genesisBlock.hash()) if (block) { - st.ok(block.hash().equals(genesis.hash())) + st.ok(block.hash().equals(genesisBlock.hash())) } else { st.fail('block is not defined!') } @@ -446,12 +451,12 @@ tape('blockchain test', (t) => { }) t.test('should put one block at a time', async (st) => { + const blocks = generateBlocks(15) const blockchain = new Blockchain({ validateBlocks: true, validatePow: false, + genesisBlock: blocks[0], }) - const blocks = generateBlocks(15) - await blockchain.putGenesis(blocks[0]) await blockchain.putBlock(blocks[1]) await blockchain.putBlock(blocks[2]) await blockchain.putBlock(blocks[3]) @@ -459,14 +464,14 @@ tape('blockchain test', (t) => { }) t.test('should put multiple blocks at once', async (st) => { + const blocks: Block[] = [] + const genesisBlock = Block.genesis({ header: { gasLimit: 8000000 } }) + blocks.push(...generateBlocks(15, [genesisBlock])) const blockchain = new Blockchain({ validateBlocks: true, validatePow: false, + genesisBlock, }) - const blocks: Block[] = [] - const genesis = Block.genesis({ header: { gasLimit: 8000000 } }) - blocks.push(...generateBlocks(15, [genesis])) - await blockchain.putGenesis(genesis) await blockchain.putBlocks(blocks.slice(1)) st.end() }) @@ -489,12 +494,12 @@ tape('blockchain test', (t) => { }) t.test('should validate', async (st) => { + const genesisBlock = Block.genesis({ header: { gasLimit: 8000000 } }) const blockchain = new Blockchain({ validateBlocks: true, validatePow: false, + genesisBlock, }) - const genesis = Block.genesis({ header: { gasLimit: 8000000 } }) - await blockchain.putGenesis(genesis) const invalidBlock = Block.fromBlockData({ header: { number: 50 } }) try { @@ -507,17 +512,16 @@ tape('blockchain test', (t) => { }) t.test('should add block with body', async (st) => { + const genesisRlp = Buffer.from(testData.genesisRLP.slice(2), 'hex') + const genesisBlock = Block.fromRLPSerializedBlock(genesisRlp, { + initWithGenesisHeader: true, + }) const blockchain = new Blockchain({ validateBlocks: true, validatePow: false, + genesisBlock, }) - const genesisRlp = Buffer.from(testData.genesisRLP.slice(2), 'hex') - const genesis = Block.fromRLPSerializedBlock(genesisRlp, { - initWithGenesisHeader: true, - }) - await blockchain.putGenesis(genesis) - const blockRlp = Buffer.from(testData.blocks[0].rlp.slice(2), 'hex') const block = Block.fromRLPSerializedBlock(blockRlp) await blockchain.putBlock(block) @@ -546,21 +550,20 @@ tape('blockchain test', (t) => { const db = level() const gasLimit = 8000000 + const genesisBlock = Block.genesis({ header: { gasLimit } }) let blockchain = new Blockchain({ db, validateBlocks: true, validatePow: false, + genesisBlock, }) - const genesis = Block.genesis({ header: { gasLimit } }) - await blockchain.putGenesis(genesis) - const headerData = { number: 1, - parentHash: genesis.hash(), - difficulty: genesis.canonicalDifficulty(genesis), + parentHash: genesisBlock.hash(), + difficulty: genesisBlock.canonicalDifficulty(genesisBlock), gasLimit, - timestamp: genesis.header.timestamp.addn(1), + timestamp: genesisBlock.header.timestamp.addn(1), } const header = BlockHeader.fromHeaderData(headerData) await blockchain.putHeader(header) @@ -574,28 +577,28 @@ tape('blockchain test', (t) => { st.ok(latestHeader.hash().equals(header.hash()), 'should save headHeader') const latestBlock = await blockchain.getLatestBlock() - st.ok(latestBlock.hash().equals(genesis.hash()), 'should save headBlock') + st.ok(latestBlock.hash().equals(genesisBlock.hash()), 'should save headBlock') st.end() }) t.test('should get latest', async (st) => { - const blockchain = new Blockchain({ - validateBlocks: true, - validatePow: false, - }) const gasLimit = 8000000 const common = new Common({ chain: 'mainnet', hardfork: 'chainstart' }) const opts = { common } - const genesis = Block.genesis({ header: { gasLimit } }, opts) - await blockchain.putGenesis(genesis) + const genesisBlock = Block.genesis({ header: { gasLimit } }, opts) + const blockchain = new Blockchain({ + validateBlocks: true, + validatePow: false, + genesisBlock, + }) const blockData = { header: { number: 1, - parentHash: genesis.hash(), - difficulty: genesis.canonicalDifficulty(genesis), - timestamp: genesis.header.timestamp.addn(3), + parentHash: genesisBlock.hash(), + difficulty: genesisBlock.canonicalDifficulty(genesisBlock), + timestamp: genesisBlock.header.timestamp.addn(3), gasLimit, }, } @@ -603,9 +606,9 @@ tape('blockchain test', (t) => { const headerData1 = { number: 1, - parentHash: genesis.hash(), - difficulty: genesis.canonicalDifficulty(genesis), - timestamp: genesis.header.timestamp.addn(1), + parentHash: genesisBlock.hash(), + difficulty: genesisBlock.canonicalDifficulty(genesisBlock), + timestamp: genesisBlock.header.timestamp.addn(1), gasLimit, } const header1 = BlockHeader.fromHeaderData(headerData1, opts) @@ -627,7 +630,7 @@ tape('blockchain test', (t) => { st.ok(latestHeader.hash().equals(headers[1].hash()), 'should update latest header') const latestBlock = await blockchain.getLatestBlock() - st.ok(latestBlock.hash().equals(genesis.hash()), 'should not change latest block') + st.ok(latestBlock.hash().equals(genesisBlock.hash()), 'should not change latest block') await blockchain.putBlock(block) @@ -641,53 +644,52 @@ tape('blockchain test', (t) => { t.test('mismatched chains', async (st) => { const common = new Common({ chain: 'mainnet', hardfork: 'chainstart' }) - const blockchain = new Blockchain({ - common, - validateBlocks: true, - validatePow: false, - }) + const gasLimit = 8000000 - const genesis = Block.genesis({ header: { gasLimit } }, { common }) + const genesisBlock = Block.genesis({ header: { gasLimit } }, { common }) const blockData1 = { header: { number: 1, - parentHash: genesis.hash(), - difficulty: genesis.canonicalDifficulty(genesis), - timestamp: genesis.header.timestamp.addn(1), + parentHash: genesisBlock.hash(), + difficulty: genesisBlock.canonicalDifficulty(genesisBlock), + timestamp: genesisBlock.header.timestamp.addn(1), gasLimit, }, } const blockData2 = { ...blockData1, number: 2, - timestamp: genesis.header.timestamp.addn(2), + timestamp: genesisBlock.header.timestamp.addn(2), } const blocks = [ - genesis, + genesisBlock, Block.fromBlockData(blockData1, { common }), Block.fromBlockData(blockData2, { common: new Common({ chain: 'ropsten', hardfork: 'chainstart' }), }), ] - for (let i = 0; i < blocks.length; i++) { - if (i === 0) { - await blockchain.putGenesis(blocks[i]) + const blockchain = new Blockchain({ + common, + validateBlocks: true, + validatePow: false, + genesisBlock, + }) + + for (let i = 1; i < blocks.length; i++) { + let error + try { + await blockchain.putBlock(blocks[i]) + } catch (err) { + error = err + } + if (i === 2) { + st.ok(error.message.match('Chain mismatch'), 'should return chain mismatch error') } else { - let error - try { - await blockchain.putBlock(blocks[i]) - } catch (err) { - error = err - } - if (i === 2) { - st.ok(error.message.match('Chain mismatch'), 'should return chain mismatch error') - } else { - st.error(error, 'should not return mismatch error') - } + st.error(error, 'should not return mismatch error') } } st.end() diff --git a/packages/blockchain/test/reorg.ts b/packages/blockchain/test/reorg.ts index fe571c27e5..ba220c85fe 100644 --- a/packages/blockchain/test/reorg.ts +++ b/packages/blockchain/test/reorg.ts @@ -5,7 +5,7 @@ import tape from 'tape' import Blockchain from '../src' import { generateConsecutiveBlock } from './util' -const genesis = Block.fromBlockData({ +const genesisBlock = Block.fromBlockData({ header: { number: new BN(0), difficulty: new BN(0x020000), @@ -18,18 +18,22 @@ tape('reorg tests', (t) => { 'should correctly reorg the chain if the total difficulty is higher on a lower block number than the current head block', async (st) => { const common = new Common({ chain: 'mainnet', hardfork: 'muirGlacier' }) - const blockchain = new Blockchain({ validateBlocks: true, validatePow: false, common }) - await blockchain.putBlock(genesis) + const blockchain = new Blockchain({ + validateBlocks: true, + validatePow: false, + common, + genesisBlock, + }) const blocks_lowTD: Block[] = [] const blocks_highTD: Block[] = [] - blocks_lowTD.push(generateConsecutiveBlock(genesis, 0)) + blocks_lowTD.push(generateConsecutiveBlock(genesisBlock, 0)) - const TD_Low = new BN(genesis.header.difficulty).add( + const TD_Low = new BN(genesisBlock.header.difficulty).add( new BN(blocks_lowTD[0].header.difficulty) ) - const TD_High = new BN(genesis.header.difficulty) + const TD_High = new BN(genesisBlock.header.difficulty) // Keep generating blocks until the Total Difficulty (TD) of the High TD chain is higher than the TD of the Low TD chain // This means that the block number of the high TD chain is 1 lower than the low TD chain @@ -37,7 +41,7 @@ tape('reorg tests', (t) => { while (TD_High.cmp(TD_Low) == -1) { blocks_lowTD.push(generateConsecutiveBlock(blocks_lowTD[blocks_lowTD.length - 1], 0)) blocks_highTD.push( - generateConsecutiveBlock(blocks_highTD[blocks_highTD.length - 1] || genesis, 1) + generateConsecutiveBlock(blocks_highTD[blocks_highTD.length - 1] || genesisBlock, 1) ) TD_Low.iadd(new BN(blocks_lowTD[blocks_lowTD.length - 1].header.difficulty)) diff --git a/packages/blockchain/test/util.ts b/packages/blockchain/test/util.ts index 3e4cffc93e..35be9854e7 100644 --- a/packages/blockchain/test/util.ts +++ b/packages/blockchain/test/util.ts @@ -35,15 +35,15 @@ export const generateBlocks = (numberOfBlocks: number, existingBlocks?: Block[]) } export const generateBlockchain = async (numberOfBlocks: number, genesis?: Block): Promise => { + const existingBlocks: Block[] = genesis ? [genesis] : [] + const blocks = generateBlocks(numberOfBlocks, existingBlocks) + const blockchain = new Blockchain({ validateBlocks: true, validatePow: false, + genesisBlock: genesis || blocks[0], }) - const existingBlocks: Block[] = genesis ? [genesis] : [] - const blocks = generateBlocks(numberOfBlocks, existingBlocks) - try { - await blockchain.putGenesis(blocks[0]) await blockchain.putBlocks(blocks.slice(1)) } catch (error) { return { error } diff --git a/packages/vm/tests/api/index.spec.ts b/packages/vm/tests/api/index.spec.ts index 28c49d2392..3ee9454ff0 100644 --- a/packages/vm/tests/api/index.spec.ts +++ b/packages/vm/tests/api/index.spec.ts @@ -121,16 +121,15 @@ tape('VM with blockchain', (t) => { t.test('should run blockchain with mocked runBlock', async (st) => { const common = new Common({ chain: 'goerli' }) - const vm = setupVM({ common }) - await vm.init() - const genesisRlp = Buffer.from(testData.genesisRLP.slice(2), 'hex') - const genesis = Block.fromRLPSerializedBlock(genesisRlp, { common }) + const genesisBlock = Block.fromRLPSerializedBlock(genesisRlp, { common }) const blockRlp = Buffer.from(testData.blocks[0].rlp.slice(2), 'hex') const block = Block.fromRLPSerializedBlock(blockRlp, { common }) - await vm.blockchain.putGenesis(genesis) + const vm = setupVM({ common, genesisBlock }) + await vm.init() + st.equal(vm.blockchain.meta.genesis?.toString('hex'), testData.genesisBlockHeader.hash.slice(2)) await vm.blockchain.putBlock(block) @@ -152,16 +151,16 @@ tape('VM with blockchain', (t) => { t.test('should run blockchain with blocks', async (st) => { const common = new Common({ chain: 'goerli' }) - const vm = setupVM({ common }) - await vm.init() const genesisRlp = toBuffer(testData.genesisRLP) - const genesis = Block.fromRLPSerializedBlock(genesisRlp, { common }) + const genesisBlock = Block.fromRLPSerializedBlock(genesisRlp, { common }) const blockRlp = toBuffer(testData.blocks[0].rlp) const block = Block.fromRLPSerializedBlock(blockRlp, { common }) - await vm.blockchain.putGenesis(genesis) + const vm = setupVM({ common, genesisBlock }) + await vm.init() + st.equal(vm.blockchain.meta.genesis?.toString('hex'), testData.genesisBlockHeader.hash.slice(2)) await vm.blockchain.putBlock(block) diff --git a/packages/vm/tests/api/runBlockchain.spec.ts b/packages/vm/tests/api/runBlockchain.spec.ts index ba8d322937..b337d7c649 100644 --- a/packages/vm/tests/api/runBlockchain.spec.ts +++ b/packages/vm/tests/api/runBlockchain.spec.ts @@ -47,13 +47,20 @@ tape('runBlockchain', (t) => { t.test('should run with genesis block', async (st) => { try { const common = new Common({ chain: 'goerli', hardfork: 'chainstart' }) - const genesis = Block.genesis(undefined, { common }) + const genesisBlock = Block.genesis(undefined, { common }) + + const newBlockchain = new Blockchain({ + db: blockchainDB, + common, + validateBlocks: false, + validatePow: false, + genesisBlock, + }) - await blockchain.putGenesis(genesis) - st.ok(blockchain.meta.genesis, 'genesis should be set for blockchain') + st.ok(newBlockchain.meta.genesis, 'genesis should be set for blockchain') // @ts-ignore - await runBlockchain.bind(vm)(blockchain) + await runBlockchain.bind(vm)(newBlockchain) st.end() } catch (e) { st.end(e) @@ -73,28 +80,34 @@ tape('runBlockchain', (t) => { }) const common = new Common({ chain: 'goerli', hardfork: 'chainstart' }) - const genesis = Block.genesis(undefined, { common }) - await blockchain.putGenesis(genesis) + const genesisBlock = Block.genesis(undefined, { common }) + const newBlockchain = new Blockchain({ + db: blockchainDB, + common, + validateBlocks: false, + validatePow: false, + genesisBlock, + }) - const b1 = createBlock(genesis, 1, { common }) + const b1 = createBlock(genesisBlock, 1, { common }) const b2 = createBlock(b1, 2, { common }) const b3 = createBlock(b2, 3, { common }) - await blockchain.putBlock(b1) - await blockchain.putBlock(b2) - await blockchain.putBlock(b3) + await newBlockchain.putBlock(b1) + await newBlockchain.putBlock(b2) + await newBlockchain.putBlock(b3) - let head = await blockchain.getHead() + let head = await newBlockchain.getHead() st.deepEqual(head.hash(), b3.hash(), 'block3 should be the current head') try { // @ts-ignore - await runBlockchain.bind(vm)(blockchain) + await runBlockchain.bind(vm)(newBlockchain) st.fail('should have returned error') } catch (e) { st.equal(e.message, 'test') - head = await blockchain.getHead() + head = await newBlockchain.getHead() st.deepEqual(head.hash(), b2.hash(), 'should have removed invalid block from head') st.end() diff --git a/packages/vm/tests/api/utils.ts b/packages/vm/tests/api/utils.ts index d06669ef4c..92b4f38e25 100644 --- a/packages/vm/tests/api/utils.ts +++ b/packages/vm/tests/api/utils.ts @@ -2,6 +2,7 @@ import { Account, BN } from 'ethereumjs-util' import Blockchain from '@ethereumjs/blockchain' import VM from '../../lib/index' import { VMOpts } from '../../lib' +import { Block } from '@ethereumjs/block' const level = require('level-mem') @@ -9,11 +10,17 @@ export function createAccount(nonce: BN = new BN(0), balance: BN = new BN(0xfff3 return new Account(nonce, balance) } -export function setupVM(opts: VMOpts = {}) { +export function setupVM(opts: VMOpts & { genesisBlock?: Block } = {}) { const db = level() - const common = opts.common + const { common, genesisBlock } = opts if (!opts.blockchain) { - opts.blockchain = new Blockchain({ db, validateBlocks: false, validatePow: false, common }) + opts.blockchain = new Blockchain({ + db, + validateBlocks: false, + validatePow: false, + common, + genesisBlock, + }) } return new VM(opts) }