refactor(trie): abstract database implementation with an interface (#…

…1917)

packages/client/lib/execution/vmexecution.ts

@@ -8,7 +8,7 @@ import { ConsensusType, Hardfork } from '@ethereumjs/common'
 import VM from '@ethereumjs/vm'
 import { bufferToHex } from 'ethereumjs-util'
 import { DefaultStateManager } from '@ethereumjs/statemanager'
-import { SecureTrie as Trie } from 'merkle-patricia-tree'
+import { LevelDB, SecureTrie as Trie } from 'merkle-patricia-tree'
 import { short } from '../util'
 import { debugCodeReplayBlock } from '../util/debug'
 import { Event } from '../types'
@@ -36,7 +36,7 @@ export class VMExecution extends Execution {
     super(options)
 
     if (!this.config.vm) {
-      const trie = new Trie({ db: this.stateDB })
+      const trie = new Trie({ db: new LevelDB(this.stateDB) })
 
       const stateManager = new DefaultStateManager({
         common: this.config.execCommon,

packages/trie/benchmarks/checkpointing.ts

@@ -1,7 +1,7 @@
 import { pseudoRandomBytes } from 'crypto'
-import { CheckpointTrie } from '../dist'
+import { CheckpointTrie, DB } from '../dist'
 
-export const iterTest = async (numOfIter: number) => {
+export const iterTest = async (db: DB, numOfIter: number) => {
   const keys: Buffer[] = []
   const vals: Buffer[] = []
 
@@ -10,7 +10,7 @@ export const iterTest = async (numOfIter: number) => {
     vals.push(pseudoRandomBytes(32))
   }
 
-  const trie = new CheckpointTrie()
+  const trie = new CheckpointTrie({ db })
 
   for (let i = 0; i < numOfIter; i++) {
     trie.checkpoint()

packages/trie/benchmarks/index.ts

@@ -1,45 +1,50 @@
-import Benchmark = require('benchmark')
+import Benchmark from 'benchmark'
 import { runTrie } from './random'
 import { iterTest } from './checkpointing'
+import { MemoryDB, LevelDB } from '../dist'
 
 const suite = new Benchmark.Suite()
 
-// random.ts
-// Test ID is defined as: `pair_count`-`era_size`-`key_size`-`value_type`
-// where value_type = symmetric ? 'mir' : 'ran'
-// The standard secure-trie test is `1k-9-32-ran`
-// https://eth.wiki/en/fundamentals/benchmarks#results-1
-suite
-  .add('1k-3-32-ran', async () => {
-    await runTrie(3, false)
-  })
-  .add('1k-5-32-ran', async () => {
-    await runTrie(5, false)
-  })
-  .add('1k-9-32-ran', async () => {
-    await runTrie(9, false)
-  })
-  .add('1k-1k-32-ran', async () => {
-    await runTrie(1000, false)
-  })
-  .add('1k-1k-32-mir', async () => {
-    await runTrie(1000, true)
-  })
+for (const [name, DB] of Object.entries({ MemoryDB, LevelDB })) {
+  const db = new DB()
 
-// checkpointing.ts
-suite
-  .add('Checkpointing: 100 iterations', async () => {
-    await iterTest(100)
-  })
-  .add('Checkpointing: 500 iterations', async () => {
-    await iterTest(500)
-  })
-  .add('Checkpointing: 1000 iterations', async () => {
-    await iterTest(1000)
-  })
-  .add('Checkpointing: 5000 iterations', async () => {
-    await iterTest(5000)
-  })
+  // random.ts
+  // Test ID is defined as: `pair_count`-`era_size`-`key_size`-`value_type`
+  // where value_type = symmetric ? 'mir' : 'ran'
+  // The standard secure-trie test is `1k-9-32-ran`
+  // https://eth.wiki/en/fundamentals/benchmarks#results-1
+  suite
+    .add(`[${name}] 1k-3-32-ran`, async () => {
+      await runTrie(db, 3, false)
+    })
+    .add(`[${name}] 1k-5-32-ran`, async () => {
+      await runTrie(db, 5, false)
+    })
+    .add(`[${name}] 1k-9-32-ran`, async () => {
+      await runTrie(db, 9, false)
+    })
+    .add(`[${name}] 1k-1k-32-ran`, async () => {
+      await runTrie(db, 1000, false)
+    })
+    .add(`[${name}] 1k-1k-32-mir`, async () => {
+      await runTrie(db, 1000, true)
+    })
+
+  // checkpointing.ts
+  suite
+    .add(`[${name}] Checkpointing: 100 iterations`, async () => {
+      await iterTest(db, 100)
+    })
+    .add(`[${name}] Checkpointing: 500 iterations`, async () => {
+      await iterTest(db, 500)
+    })
+    .add(`[${name}] Checkpointing: 1000 iterations`, async () => {
+      await iterTest(db, 1000)
+    })
+    .add(`[${name}] Checkpointing: 5000 iterations`, async () => {
+      await iterTest(db, 5000)
+    })
+}
 
 suite
   .on('cycle', (event: any) => {

packages/trie/benchmarks/random.ts

@@ -1,6 +1,6 @@
 'use strict'
 import { keccak256 } from 'ethereum-cryptography/keccak'
-import { CheckpointTrie as Trie } from '../dist'
+import { CheckpointTrie as Trie, DB } from '../dist'
 
 // References:
 // https://eth.wiki/en/fundamentals/benchmarks#the-trie
@@ -9,8 +9,8 @@ import { CheckpointTrie as Trie } from '../dist'
 const ROUNDS = 1000
 const KEY_SIZE = 32
 
-export const runTrie = async (eraSize = 9, symmetric = false) => {
-  const trie = new Trie()
+export const runTrie = async (db: DB, eraSize = 9, symmetric = false) => {
+  const trie = new Trie({ db })
   let key = Buffer.alloc(KEY_SIZE)
 
   for (let i = 0; i <= ROUNDS; i++) {

packages/trie/src/baseTrie.ts

@@ -1,7 +1,7 @@
 import Semaphore from 'semaphore-async-await'
 import { keccak256 } from 'ethereum-cryptography/keccak'
 import { KECCAK256_RLP } from 'ethereumjs-util'
-import { DB, BatchDBOp, PutBatch } from './db'
+import { DB, BatchDBOp, PutBatch, MemoryDB } from './db'
 import { TrieReadStream as ReadStream } from './readStream'
 import { bufferToNibbles, matchingNibbleLength, doKeysMatch } from './util/nibbles'
 import { WalkController } from './util/walkController'
@@ -17,8 +17,6 @@ import {
   Nibbles,
 } from './trieNode'
 import { verifyRangeProof } from './verifyRangeProof'
-// eslint-disable-next-line implicit-dependencies/no-implicit
-import type { LevelUp } from 'levelup'
 
 export type Proof = Buffer[]
 
@@ -37,11 +35,9 @@ export type FoundNodeFunction = (
 
 export interface TrieOpts {
   /**
-   * A [levelup](https://github.com/Level/levelup) instance.
-   * By default (if the db is `null` or left undefined) creates an
-   * in-memory [memdown](https://github.com/Level/memdown) instance.
+   * A database instance.
    */
-  db?: LevelUp | null
+  db?: DB
   /**
    * A `Buffer` for the root of a previously stored trie
    */
@@ -72,15 +68,15 @@ export class Trie {
    * Create a new trie
    * @param opts Options for instantiating the trie
    */
-  constructor(opts: TrieOpts = {}) {
+  constructor(opts?: TrieOpts) {
     this.EMPTY_TRIE_ROOT = KECCAK256_RLP
     this.lock = new Semaphore(1)
 
-    this.db = opts.db ? new DB(opts.db) : new DB()
+    this.db = opts?.db ?? new MemoryDB()
     this._root = this.EMPTY_TRIE_ROOT
-    this._deleteFromDB = opts.deleteFromDB ?? false
+    this._deleteFromDB = opts?.deleteFromDB ?? false
 
-    if (opts.root) {
+    if (opts?.root) {
       this.root = opts.root
     }
   }
@@ -738,8 +734,7 @@ export class Trie {
    * Creates a new trie backed by the same db.
    */
   copy(): Trie {
-    const db = this.db.copy()
-    return new Trie({ db: db._leveldb, root: this.root, deleteFromDB: this._deleteFromDB })
+    return new Trie({ db: this.db.copy(), root: this.root, deleteFromDB: this._deleteFromDB })
   }
 
   /**

packages/trie/src/checkpointDb.ts

@@ -1,6 +1,5 @@
-import { DB, BatchDBOp, ENCODING_OPTS } from './db'
+import { DB, BatchDBOp } from './db'
 // eslint-disable-next-line implicit-dependencies/no-implicit
-import type { LevelUp } from 'levelup'
 
 export type Checkpoint = {
   // We cannot use a Buffer => Buffer map directly. If you create two Buffers with the same internal value,
@@ -13,16 +12,15 @@ export type Checkpoint = {
  * DB is a thin wrapper around the underlying levelup db,
  * which validates inputs and sets encoding type.
  */
-export class CheckpointDB extends DB {
+export class CheckpointDB implements DB {
   public checkpoints: Checkpoint[]
+  public db: DB
 
   /**
-   * Initialize a DB instance. If `leveldb` is not provided, DB
-   * defaults to an [in-memory store](https://github.com/Level/memdown).
-   * @param leveldb - An abstract-leveldown compliant store
+   * Initialize a DB instance.
    */
-  constructor(leveldb?: LevelUp | null) {
-    super(leveldb)
+  constructor(db: DB) {
+    this.db = db
     // Roots of trie at the moment of checkpoint
     this.checkpoints = []
   }
@@ -81,9 +79,7 @@ export class CheckpointDB extends DB {
   }
 
   /**
-   * Retrieves a raw value from leveldb.
-   * @param key
-   * @returns A Promise that resolves to `Buffer` if a value is found or `null` if no value is found.
+   * @inheritdoc
    */
   async get(key: Buffer): Promise<Buffer | null> {
     // Lookup the value in our cache. We return the latest checkpointed value (which should be the value on disk)
@@ -95,7 +91,7 @@ export class CheckpointDB extends DB {
     }
     // Nothing has been found in cache, look up from disk
 
-    const value = await super.get(key)
+    const value = await this.db.get(key)
     if (this.isCheckpoint) {
       // Since we are a checkpoint, put this value in cache, so future `get` calls will not look the key up again from disk.
       this.checkpoints[this.checkpoints.length - 1].keyValueMap.set(key.toString('binary'), value)
@@ -105,36 +101,32 @@ export class CheckpointDB extends DB {
   }
 
   /**
-   * Writes a value directly to leveldb.
-   * @param key The key as a `Buffer`
-   * @param value The value to be stored
+   * @inheritdoc
    */
   async put(key: Buffer, val: Buffer): Promise<void> {
     if (this.isCheckpoint) {
       // put value in cache
       this.checkpoints[this.checkpoints.length - 1].keyValueMap.set(key.toString('binary'), val)
     } else {
-      await super.put(key, val)
+      await this.db.put(key, val)
     }
   }
 
   /**
-   * Removes a raw value in the underlying leveldb.
-   * @param keys
+   * @inheritdoc
    */
   async del(key: Buffer): Promise<void> {
     if (this.isCheckpoint) {
       // delete the value in the current cache
       this.checkpoints[this.checkpoints.length - 1].keyValueMap.set(key.toString('binary'), null)
     } else {
       // delete the value on disk
-      await this._leveldb.del(key, ENCODING_OPTS)
+      await this.db.del(key)
     }
   }
 
   /**
-   * Performs a batch operation on db.
-   * @param opStack A stack of levelup operations
+   * @inheritdoc
    */
   async batch(opStack: BatchDBOp[]): Promise<void> {
     if (this.isCheckpoint) {
@@ -146,7 +138,14 @@ export class CheckpointDB extends DB {
         }
       }
     } else {
-      await super.batch(opStack)
+      await this.db.batch(opStack)
     }
   }
+
+  /**
+   * @inheritdoc
+   */
+  copy(): CheckpointDB {
+    return new CheckpointDB(this.db)
+  }
 }

packages/trie/src/checkpointTrie.ts

@@ -1,15 +1,18 @@
 import { Trie as BaseTrie, TrieOpts } from './baseTrie'
 import { CheckpointDB } from './checkpointDb'
+import { DB, MemoryDB } from './db'
 
 /**
  * Adds checkpointing to the {@link BaseTrie}
  */
 export class CheckpointTrie extends BaseTrie {
   db: CheckpointDB
+  dbStorage: DB
 
-  constructor(opts: TrieOpts = {}) {
+  constructor(opts?: TrieOpts) {
     super(opts)
-    this.db = new CheckpointDB(opts.db)
+    this.dbStorage = opts?.db ?? new MemoryDB()
+    this.db = new CheckpointDB(this.dbStorage)
   }
 
   /**
@@ -62,9 +65,8 @@ export class CheckpointTrie extends BaseTrie {
    * @param includeCheckpoints - If true and during a checkpoint, the copy will contain the checkpointing metadata and will use the same scratch as underlying db.
    */
   copy(includeCheckpoints = true): CheckpointTrie {
-    const db = this.db.copy()
     const trie = new CheckpointTrie({
-      db: db._leveldb,
+      db: this.dbStorage.copy(),
       root: this.root,
       deleteFromDB: (this as any)._deleteFromDB,
     })