diff --git a/yarn-project/acir-simulator/package.json b/yarn-project/acir-simulator/package.json index 9a3a0361d72..ccd51b8ba54 100644 --- a/yarn-project/acir-simulator/package.json +++ b/yarn-project/acir-simulator/package.json @@ -39,6 +39,7 @@ "tslib": "^2.4.0" }, "devDependencies": { + "@aztec/kv-store": "workspace:^", "@aztec/merkle-tree": "workspace:^", "@aztec/noir-contracts": "workspace:^", "@jest/globals": "^29.5.0", diff --git a/yarn-project/acir-simulator/src/client/private_execution.test.ts b/yarn-project/acir-simulator/src/client/private_execution.test.ts index 775433dbeb8..338810e9477 100644 --- a/yarn-project/acir-simulator/src/client/private_execution.test.ts +++ b/yarn-project/acir-simulator/src/client/private_execution.test.ts @@ -37,6 +37,7 @@ import { EthAddress } from '@aztec/foundation/eth-address'; import { Fr, GrumpkinScalar } from '@aztec/foundation/fields'; import { DebugLogger, createDebugLogger } from '@aztec/foundation/log'; import { FieldsOf } from '@aztec/foundation/types'; +import { AztecLmdbStore } from '@aztec/kv-store'; import { AppendOnlyTree, Pedersen, StandardTree, newTree } from '@aztec/merkle-tree'; import { ChildContractArtifact, @@ -50,8 +51,6 @@ import { import { jest } from '@jest/globals'; import { MockProxy, mock } from 'jest-mock-extended'; -import { default as levelup } from 'levelup'; -import { type MemDown, default as memdown } from 'memdown'; import { getFunctionSelector } from 'viem'; import { KeyPair } from '../acvm/index.js'; @@ -62,8 +61,6 @@ import { AcirSimulator } from './simulator.js'; jest.setTimeout(60_000); -const createMemDown = () => (memdown as any)() as MemDown; - describe('Private Execution test suite', () => { let oracle: MockProxy; let acirSimulator: AcirSimulator; @@ -136,7 +133,7 @@ describe('Private Execution test suite', () => { throw new Error(`Unknown tree ${name}`); } if (!trees[name]) { - const db = levelup(createMemDown()); + const db = await AztecLmdbStore.openTmp(); const pedersen = new Pedersen(); trees[name] = await newTree(StandardTree, db, pedersen, name, treeHeights[name]); } diff --git a/yarn-project/acir-simulator/tsconfig.json b/yarn-project/acir-simulator/tsconfig.json index da0e524126f..ce07b84cd9e 100644 --- a/yarn-project/acir-simulator/tsconfig.json +++ b/yarn-project/acir-simulator/tsconfig.json @@ -15,6 +15,9 @@ { "path": "../foundation" }, + { + "path": "../kv-store" + }, { "path": "../merkle-tree" }, diff --git a/yarn-project/archiver/src/archiver/kv_archiver_store/block_store.ts b/yarn-project/archiver/src/archiver/kv_archiver_store/block_store.ts index 3a2446fa0d5..db38f09866a 100644 --- a/yarn-project/archiver/src/archiver/kv_archiver_store/block_store.ts +++ b/yarn-project/archiver/src/archiver/kv_archiver_store/block_store.ts @@ -28,10 +28,10 @@ export class BlockStore { #log = createDebugLogger('aztec:archiver:block_store'); constructor(private db: AztecKVStore) { - this.#blocks = db.createMap('archiver_blocks'); + this.#blocks = db.openMap('archiver_blocks'); - this.#txIndex = db.createMap('archiver_tx_index'); - this.#contractIndex = db.createMap('archiver_contract_index'); + this.#txIndex = db.openMap('archiver_tx_index'); + this.#contractIndex = db.openMap('archiver_contract_index'); } /** diff --git a/yarn-project/archiver/src/archiver/kv_archiver_store/contract_class_store.ts b/yarn-project/archiver/src/archiver/kv_archiver_store/contract_class_store.ts index 6cd56d90e4d..686514aa1b7 100644 --- a/yarn-project/archiver/src/archiver/kv_archiver_store/contract_class_store.ts +++ b/yarn-project/archiver/src/archiver/kv_archiver_store/contract_class_store.ts @@ -9,7 +9,7 @@ export class ContractClassStore { #contractClasses: AztecMap; constructor(db: AztecKVStore) { - this.#contractClasses = db.createMap('archiver_contract_classes'); + this.#contractClasses = db.openMap('archiver_contract_classes'); } addContractClass(contractClass: ContractClassWithId): Promise { diff --git a/yarn-project/archiver/src/archiver/kv_archiver_store/contract_instance_store.ts b/yarn-project/archiver/src/archiver/kv_archiver_store/contract_instance_store.ts index 7842b54f2f4..fb020eb3c35 100644 --- a/yarn-project/archiver/src/archiver/kv_archiver_store/contract_instance_store.ts +++ b/yarn-project/archiver/src/archiver/kv_archiver_store/contract_instance_store.ts @@ -9,7 +9,7 @@ export class ContractInstanceStore { #contractInstances: AztecMap; constructor(db: AztecKVStore) { - this.#contractInstances = db.createMap('archiver_contract_instances'); + this.#contractInstances = db.openMap('archiver_contract_instances'); } addContractInstance(contractInstance: ContractInstanceWithAddress): Promise { diff --git a/yarn-project/archiver/src/archiver/kv_archiver_store/contract_store.ts b/yarn-project/archiver/src/archiver/kv_archiver_store/contract_store.ts index 055b25af20d..0c2f117ed92 100644 --- a/yarn-project/archiver/src/archiver/kv_archiver_store/contract_store.ts +++ b/yarn-project/archiver/src/archiver/kv_archiver_store/contract_store.ts @@ -14,7 +14,7 @@ export class ContractStore { #log = createDebugLogger('aztec:archiver:contract_store'); constructor(private db: AztecKVStore, blockStore: BlockStore) { - this.#extendedContractData = db.createMap('archiver_extended_contract_data'); + this.#extendedContractData = db.openMap('archiver_extended_contract_data'); this.#blockStore = blockStore; } diff --git a/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.test.ts b/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.test.ts index 2903ea6fe9c..520ac236d05 100644 --- a/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.test.ts +++ b/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.test.ts @@ -1,4 +1,3 @@ -import { EthAddress } from '@aztec/circuits.js'; import { AztecLmdbStore } from '@aztec/kv-store'; import { describeArchiverDataStore } from '../archiver_store_test_suite.js'; @@ -8,7 +7,7 @@ describe('KVArchiverDataStore', () => { let archiverStore: KVArchiverDataStore; beforeEach(async () => { - archiverStore = new KVArchiverDataStore(await AztecLmdbStore.create(EthAddress.random())); + archiverStore = new KVArchiverDataStore(await AztecLmdbStore.openTmp()); }); describeArchiverDataStore('ArchiverStore', () => archiverStore); diff --git a/yarn-project/archiver/src/archiver/kv_archiver_store/log_store.ts b/yarn-project/archiver/src/archiver/kv_archiver_store/log_store.ts index afab800fb48..b29e3d67818 100644 --- a/yarn-project/archiver/src/archiver/kv_archiver_store/log_store.ts +++ b/yarn-project/archiver/src/archiver/kv_archiver_store/log_store.ts @@ -23,8 +23,8 @@ export class LogStore { #log = createDebugLogger('aztec:archiver:log_store'); constructor(private db: AztecKVStore, private blockStore: BlockStore, logsMaxPageSize: number = 1000) { - this.#encryptedLogs = db.createMap('archiver_encrypted_logs'); - this.#unencryptedLogs = db.createMap('archiver_unencrypted_logs'); + this.#encryptedLogs = db.openMap('archiver_encrypted_logs'); + this.#unencryptedLogs = db.openMap('archiver_unencrypted_logs'); this.#logsMaxPageSize = logsMaxPageSize; } diff --git a/yarn-project/archiver/src/archiver/kv_archiver_store/message_store.ts b/yarn-project/archiver/src/archiver/kv_archiver_store/message_store.ts index 81a83e21560..642c90edde3 100644 --- a/yarn-project/archiver/src/archiver/kv_archiver_store/message_store.ts +++ b/yarn-project/archiver/src/archiver/kv_archiver_store/message_store.ts @@ -27,10 +27,10 @@ export class MessageStore { #log = createDebugLogger('aztec:archiver:message_store'); constructor(private db: AztecKVStore) { - this.#messages = db.createMap('archiver_l1_to_l2_messages'); - this.#pendingMessagesByFee = db.createCounter('archiver_messages_by_fee'); - this.#lastL1BlockAddingMessages = db.createSingleton('archiver_last_l1_block_adding_messages'); - this.#lastL1BlockCancellingMessages = db.createSingleton('archiver_last_l1_block_cancelling_messages'); + this.#messages = db.openMap('archiver_l1_to_l2_messages'); + this.#pendingMessagesByFee = db.openCounter('archiver_messages_by_fee'); + this.#lastL1BlockAddingMessages = db.openSingleton('archiver_last_l1_block_adding_messages'); + this.#lastL1BlockCancellingMessages = db.openSingleton('archiver_last_l1_block_cancelling_messages'); } /** diff --git a/yarn-project/aztec-node/package.json b/yarn-project/aztec-node/package.json index a2e602a03ff..2c0dd26296c 100644 --- a/yarn-project/aztec-node/package.json +++ b/yarn-project/aztec-node/package.json @@ -47,17 +47,11 @@ "@aztec/world-state": "workspace:^", "koa": "^2.14.2", "koa-router": "^12.0.0", - "levelup": "^5.1.1", - "lmdb": "^2.9.1", - "memdown": "^6.1.1", "tslib": "^2.4.0" }, "devDependencies": { "@jest/globals": "^29.5.0", "@types/jest": "^29.5.0", - "@types/leveldown": "^4.0.4", - "@types/levelup": "^5.1.2", - "@types/memdown": "^3.0.0", "@types/node": "^18.7.23", "jest": "^29.5.0", "ts-jest": "^29.1.0", diff --git a/yarn-project/aztec-node/src/aztec-node/db.ts b/yarn-project/aztec-node/src/aztec-node/db.ts deleted file mode 100644 index 9b5be428781..00000000000 --- a/yarn-project/aztec-node/src/aztec-node/db.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { LogFn } from '@aztec/foundation/log'; - -import { LevelDown, default as leveldown } from 'leveldown'; -import { LevelUp, default as levelup } from 'levelup'; -import { RootDatabase, open } from 'lmdb'; -import { MemDown, default as memdown } from 'memdown'; -import { mkdir } from 'node:fs/promises'; -import { join } from 'node:path'; - -import { AztecNodeConfig } from './config.js'; - -export const createMemDown = () => (memdown as any)() as MemDown; -export const createLevelDown = (path: string) => (leveldown as any)(path) as LevelDown; - -const DB_SUBDIR = 'aztec-node-db'; -const WORLD_STATE_SUBDIR = 'aztec-world-state-db'; -const NODE_METADATA_KEY = '@@aztec_node_metadata'; - -/** - * The metadata for an aztec node. - */ -type NodeMetadata = { - /** - * The address of the rollup contract on L1 - */ - rollupContractAddress: string; -}; - -/** - * Opens the database for the aztec node. If a data directory is specified, then this attempts to create it. - * @param config - The configuration to be used by the aztec node. - * @throws If `config.dataDirectory` is set and the directory cannot be created. - * @returns The database for the aztec node. - */ -export async function openDb( - config: AztecNodeConfig, - log: LogFn, -): Promise<[nodeDb: RootDatabase, worldStateDb: LevelUp]> { - const nodeMetadata: NodeMetadata = { - rollupContractAddress: config.l1Contracts.rollupAddress.toString(), - }; - - let nodeDb: RootDatabase; - let worldStateDb: LevelUp; - - if (config.dataDirectory) { - const nodeDir = join(config.dataDirectory, DB_SUBDIR); - const worldStateDir = join(config.dataDirectory, WORLD_STATE_SUBDIR); - // this throws if we don't have permissions to create the directory - await mkdir(nodeDir, { recursive: true }); - await mkdir(worldStateDir, { recursive: true }); - - log(`Opening aztec-node database at ${nodeDir}`); - nodeDb = open(nodeDir, {}); - - log(`Opening world-state database at ${worldStateDir}`); - worldStateDb = levelup(createLevelDown(worldStateDir)); - } else { - log('Opening temporary databases'); - // not passing a path will use a temp file that gets deleted when the process exits - nodeDb = open({}); - worldStateDb = levelup(createMemDown()); - } - - await checkNodeMetadataAndClear(nodeDb, worldStateDb, nodeMetadata, log); - return [nodeDb, worldStateDb]; -} - -/** - * Checks the node metadata and clears the database if the rollup contract address has changed. - * @param nodeDb - The database for the aztec node. - * @param nodeMetadata - The metadata for the aztec node. - */ -async function checkNodeMetadataAndClear( - nodeDb: RootDatabase, - worldStateDb: LevelUp, - nodeMetadata: NodeMetadata, - log: LogFn, -): Promise { - const metadataDB = nodeDb.openDB('metadata', {}); - try { - const existing = metadataDB.get(NODE_METADATA_KEY); - // if the rollup addresses are different, wipe the local database and start over - if (!existing || existing.rollupContractAddress !== nodeMetadata.rollupContractAddress) { - log('Rollup contract address has changed, clearing databases'); - await Promise.all([nodeDb.clearAsync(), worldStateDb.clear()]); - } - await metadataDB.put(NODE_METADATA_KEY, nodeMetadata); - } finally { - await metadataDB.close(); - } -} diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index 0c65a9c488d..5a7b08af067 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -39,7 +39,7 @@ import { computeGlobalsHash, computePublicDataTreeLeafSlot } from '@aztec/circui import { L1ContractAddresses, createEthereumChain } from '@aztec/ethereum'; import { AztecAddress } from '@aztec/foundation/aztec-address'; import { createDebugLogger } from '@aztec/foundation/log'; -import { AztecLmdbStore } from '@aztec/kv-store'; +import { AztecKVStore, AztecLmdbStore } from '@aztec/kv-store'; import { AztecKVTxPool, P2P, createP2PClient } from '@aztec/p2p'; import { GlobalVariableBuilder, @@ -56,10 +56,7 @@ import { getConfigEnvVars as getWorldStateConfig, } from '@aztec/world-state'; -import { LevelUp } from 'levelup'; - import { AztecNodeConfig } from './config.js'; -import { openDb } from './db.js'; /** * The aztec node. @@ -78,7 +75,7 @@ export class AztecNodeService implements AztecNode { protected readonly chainId: number, protected readonly version: number, protected readonly globalVariableBuilder: GlobalVariableBuilder, - protected readonly merkleTreesDb: LevelUp, + protected readonly merkleTreesDb: AztecKVStore, private log = createDebugLogger('aztec:node'), ) { const message = @@ -106,8 +103,7 @@ export class AztecNodeService implements AztecNode { } const log = createDebugLogger('aztec:node'); - const store = await AztecLmdbStore.create(config.l1Contracts.rollupAddress, config.dataDirectory); - const [_, worldStateDb] = await openDb(config, log); + const store = await AztecLmdbStore.open(config.l1Contracts.rollupAddress, config.dataDirectory); // first create and sync the archiver const archiverStore = new KVArchiverDataStore(store, config.maxLogs); @@ -121,14 +117,9 @@ export class AztecNodeService implements AztecNode { const p2pClient = await createP2PClient(store, config, new AztecKVTxPool(store), archiver); // now create the merkle trees and the world state synchronizer - const merkleTrees = await MerkleTrees.new(worldStateDb); + const merkleTrees = await MerkleTrees.new(store); const worldStateConfig: WorldStateConfig = getWorldStateConfig(); - const worldStateSynchronizer = await ServerWorldStateSynchronizer.new( - worldStateDb, - merkleTrees, - archiver, - worldStateConfig, - ); + const worldStateSynchronizer = new ServerWorldStateSynchronizer(store, merkleTrees, archiver, worldStateConfig); // start both and wait for them to sync from the block source await Promise.all([p2pClient.start(), worldStateSynchronizer.start()]); @@ -151,7 +142,7 @@ export class AztecNodeService implements AztecNode { ethereumChain.chainInfo.id, config.version, getGlobalVariableBuilder(config), - worldStateDb, + store, log, ); } @@ -285,8 +276,6 @@ export class AztecNodeService implements AztecNode { await this.p2pClient.stop(); await this.worldStateSynchronizer.stop(); await this.blockSource.stop(); - this.log('Closing Merkle Trees'); - await this.merkleTreesDb.close(); this.log.info(`Stopped`); } diff --git a/yarn-project/aztec-node/src/declaration.d.ts b/yarn-project/aztec-node/src/declaration.d.ts deleted file mode 100644 index d7367c50ba8..00000000000 --- a/yarn-project/aztec-node/src/declaration.d.ts +++ /dev/null @@ -1,16 +0,0 @@ -// eslint-disable-next-line @typescript-eslint/no-unused-vars -import { RootDatabaseOptionsWithPath } from 'lmdb'; - -// The problem is this snippet `nodeDb = open({});` in src/aztec-node/db.ts -// tsc compiles this code fine, but ts-jest can't. -// This is a mixture for two bugs: -// - the first in ts-jest, it gets confused by packages with mixed CJS and ESM type exports - https://github.com/kulshekhar/ts-jest/issues/4221 -// - the second in lmdb, it outputs different CJS and ESM types - https://github.com/kriszyp/lmdb-js/issues/243#issuecomment-1823585586 - -declare module 'lmdb' { - /* eslint-disable jsdoc/require-jsdoc */ - interface RootDatabaseOptionsWithPath { - path?: string; - } - /* eslint-enable jsdoc/require-jsdoc */ -} diff --git a/yarn-project/end-to-end/src/e2e_blacklist_token_contract.test.ts b/yarn-project/end-to-end/src/e2e_blacklist_token_contract.test.ts index f5904fdce5b..a434f91f090 100644 --- a/yarn-project/end-to-end/src/e2e_blacklist_token_contract.test.ts +++ b/yarn-project/end-to-end/src/e2e_blacklist_token_contract.test.ts @@ -14,18 +14,15 @@ import { computeAuthWitMessageHash, computeMessageSecretHash, } from '@aztec/aztec.js'; +import { AztecLmdbStore } from '@aztec/kv-store'; import { Pedersen, SparseTree, newTree } from '@aztec/merkle-tree'; import { SlowTreeContract, TokenBlacklistContract, TokenContract } from '@aztec/noir-contracts'; import { jest } from '@jest/globals'; -import levelup from 'levelup'; -import { type MemDown, default as memdown } from 'memdown'; import { setup } from './fixtures/utils.js'; import { TokenSimulator } from './simulators/token_simulator.js'; -export const createMemDown = () => (memdown as any)() as MemDown; - const TIMEOUT = 90_000; describe('e2e_blacklist_token_contract', () => { @@ -48,7 +45,7 @@ describe('e2e_blacklist_token_contract', () => { const getMembershipProof = async (index: bigint, includeUncommitted: boolean) => { return { index, - value: Fr.fromBuffer((await slowUpdateTreeSimulator.getLeafValue(index, includeUncommitted))!), + value: Fr.fromBuffer(slowUpdateTreeSimulator.getLeafValue(index, includeUncommitted)!), // eslint-disable-next-line camelcase sibling_path: (await slowUpdateTreeSimulator.getSiblingPath(index, includeUncommitted)).toFieldArray(), }; @@ -107,7 +104,7 @@ describe('e2e_blacklist_token_contract', () => { slowTree = await SlowTreeContract.deploy(wallets[0]).send().deployed(); const depth = 254; - slowUpdateTreeSimulator = await newTree(SparseTree, levelup(createMemDown()), new Pedersen(), 'test', depth); + slowUpdateTreeSimulator = await newTree(SparseTree, await AztecLmdbStore.openTmp(), new Pedersen(), 'test', depth); const deployTx = TokenBlacklistContract.deploy(wallets[0], accounts[0], slowTree.address).send({}); const receipt = await deployTx.wait(); diff --git a/yarn-project/end-to-end/src/e2e_slow_tree.test.ts b/yarn-project/end-to-end/src/e2e_slow_tree.test.ts index 6fc89230b02..00be8111f10 100644 --- a/yarn-project/end-to-end/src/e2e_slow_tree.test.ts +++ b/yarn-project/end-to-end/src/e2e_slow_tree.test.ts @@ -1,15 +1,11 @@ /* eslint-disable camelcase */ import { CheatCodes, DebugLogger, Fr, Wallet } from '@aztec/aztec.js'; +import { AztecLmdbStore } from '@aztec/kv-store'; import { Pedersen, SparseTree, newTree } from '@aztec/merkle-tree'; import { SlowTreeContract } from '@aztec/noir-contracts/SlowTree'; -import { default as levelup } from 'levelup'; -import { type MemDown, default as memdown } from 'memdown'; - import { setup } from './fixtures/utils.js'; -export const createMemDown = () => (memdown as any)() as MemDown; - describe('e2e_slow_tree', () => { let logger: DebugLogger; let wallet: Wallet; @@ -27,11 +23,17 @@ describe('e2e_slow_tree', () => { it('Messing around with noir slow tree', async () => { const depth = 254; - const slowUpdateTreeSimulator = await newTree(SparseTree, levelup(createMemDown()), new Pedersen(), 'test', depth); + const slowUpdateTreeSimulator = await newTree( + SparseTree, + await AztecLmdbStore.openTmp(), + new Pedersen(), + 'test', + depth, + ); const getMembershipProof = async (index: bigint, includeUncommitted: boolean) => { return { index, - value: Fr.fromBuffer((await slowUpdateTreeSimulator.getLeafValue(index, includeUncommitted))!), + value: Fr.fromBuffer(slowUpdateTreeSimulator.getLeafValue(index, includeUncommitted)!), // eslint-disable-next-line camelcase sibling_path: (await slowUpdateTreeSimulator.getSiblingPath(index, includeUncommitted)).toFieldArray(), }; diff --git a/yarn-project/end-to-end/src/integration_archiver_l1_to_l2.test.ts b/yarn-project/end-to-end/src/integration_archiver_l1_to_l2.test.ts index eb57e00eb21..25e501b8fdb 100644 --- a/yarn-project/end-to-end/src/integration_archiver_l1_to_l2.test.ts +++ b/yarn-project/end-to-end/src/integration_archiver_l1_to_l2.test.ts @@ -43,7 +43,7 @@ describe('archiver integration with l1 to l2 messages', () => { config.archiverPollingIntervalMS = 100; archiver = await Archiver.createAndSync( { ...config, l1Contracts: deployL1ContractsValues.l1ContractAddresses }, - new KVArchiverDataStore(await AztecLmdbStore.create(deployL1ContractsValues.l1ContractAddresses.rollupAddress)), + new KVArchiverDataStore(await AztecLmdbStore.open(deployL1ContractsValues.l1ContractAddresses.rollupAddress)), ); const walletClient = deployL1ContractsValues.walletClient; diff --git a/yarn-project/end-to-end/src/integration_l1_publisher.test.ts b/yarn-project/end-to-end/src/integration_l1_publisher.test.ts index be9d3ddad02..fc642ff2a80 100644 --- a/yarn-project/end-to-end/src/integration_l1_publisher.test.ts +++ b/yarn-project/end-to-end/src/integration_l1_publisher.test.ts @@ -28,6 +28,7 @@ import { } from '@aztec/circuits.js/factories'; import { createEthereumChain } from '@aztec/ethereum'; import { makeTuple, range } from '@aztec/foundation/array'; +import { AztecLmdbStore } from '@aztec/kv-store'; import { InboxAbi, OutboxAbi, RollupAbi } from '@aztec/l1-artifacts'; import { EmptyRollupProver, @@ -44,8 +45,6 @@ import { MerkleTreeOperations, MerkleTrees } from '@aztec/world-state'; import { beforeEach, describe, expect, it } from '@jest/globals'; import * as fs from 'fs'; -import { default as levelup } from 'levelup'; -import memdown from 'memdown'; import { Address, Chain, @@ -133,7 +132,7 @@ describe('L1Publisher integration', () => { publicClient, }); - builderDb = await MerkleTrees.new(levelup((memdown as any)())).then(t => t.asLatest()); + builderDb = await MerkleTrees.new(await AztecLmdbStore.openTmp()).then(t => t.asLatest()); const vks = getVerificationKeys(); const simulator = new RealRollupCircuitSimulator(); const prover = new EmptyRollupProver(); diff --git a/yarn-project/key-store/src/test_key_store.ts b/yarn-project/key-store/src/test_key_store.ts index f0af31ee6da..8a2b41dd59e 100644 --- a/yarn-project/key-store/src/test_key_store.ts +++ b/yarn-project/key-store/src/test_key_store.ts @@ -21,7 +21,7 @@ export class TestKeyStore implements KeyStore { #keys: AztecMap; constructor(private curve: Grumpkin, database: AztecKVStore) { - this.#keys = database.createMap('key_store'); + this.#keys = database.openMap('key_store'); } public async addAccount(privKey: GrumpkinPrivateKey): Promise { diff --git a/yarn-project/kv-store/src/interfaces/store.ts b/yarn-project/kv-store/src/interfaces/store.ts index 73a2901387c..2e2777f0e8a 100644 --- a/yarn-project/kv-store/src/interfaces/store.ts +++ b/yarn-project/kv-store/src/interfaces/store.ts @@ -11,34 +11,34 @@ export interface AztecKVStore { * @param name - The name of the map * @returns The map */ - createMap(name: string): AztecMap; + openMap(name: string): AztecMap; /** * Creates a new multi-map. * @param name - The name of the multi-map * @returns The multi-map */ - createMultiMap(name: string): AztecMultiMap; + openMultiMap(name: string): AztecMultiMap; /** * Creates a new array. * @param name - The name of the array * @returns The array */ - createArray(name: string): AztecArray; + openArray(name: string): AztecArray; /** * Creates a new singleton. * @param name - The name of the singleton * @returns The singleton */ - createSingleton(name: string): AztecSingleton; + openSingleton(name: string): AztecSingleton; /** * Creates a new count map. * @param name - name of the counter */ - createCounter(name: string): AztecCounter; + openCounter(name: string): AztecCounter; /** * Starts a transaction. All calls to read/write data while in a transaction are queued and executed atomically. diff --git a/yarn-project/kv-store/src/lmdb/store.ts b/yarn-project/kv-store/src/lmdb/store.ts index 8c3cebd0737..bbd866efe8c 100644 --- a/yarn-project/kv-store/src/lmdb/store.ts +++ b/yarn-project/kv-store/src/lmdb/store.ts @@ -39,7 +39,7 @@ export class AztecLmdbStore implements AztecKVStore { dupSort: true, }); - this.#rollupAddress = this.createSingleton('rollupAddress'); + this.#rollupAddress = this.openSingleton('rollupAddress'); } /** @@ -55,7 +55,7 @@ export class AztecLmdbStore implements AztecKVStore { * @param log - A logger to use. Optional * @returns The store */ - static async create( + static async open( rollupAddress: EthAddress, path?: string, log = createDebugLogger('aztec:kv-store:lmdb'), @@ -72,12 +72,16 @@ export class AztecLmdbStore implements AztecKVStore { return db; } + static openTmp(): Promise { + return AztecLmdbStore.open(EthAddress.random()); + } + /** * Creates a new AztecMap in the store. * @param name - Name of the map * @returns A new AztecMap */ - createMap(name: string): AztecMap { + openMap(name: string): AztecMap { return new LmdbAztecMap(this.#data, name); } @@ -86,11 +90,11 @@ export class AztecLmdbStore implements AztecKVStore { * @param name - Name of the map * @returns A new AztecMultiMap */ - createMultiMap(name: string): AztecMultiMap { + openMultiMap(name: string): AztecMultiMap { return new LmdbAztecMap(this.#multiMapData, name); } - createCounter>(name: string): AztecCounter { + openCounter>(name: string): AztecCounter { return new LmdbAztecCounter(this.#data, name); } @@ -99,7 +103,7 @@ export class AztecLmdbStore implements AztecKVStore { * @param name - Name of the array * @returns A new AztecArray */ - createArray(name: string): AztecArray { + openArray(name: string): AztecArray { return new LmdbAztecArray(this.#data, name); } @@ -108,7 +112,7 @@ export class AztecLmdbStore implements AztecKVStore { * @param name - Name of the singleton * @returns A new AztecSingleton */ - createSingleton(name: string): AztecSingleton { + openSingleton(name: string): AztecSingleton { return new LmdbAztecSingleton(this.#data, name); } diff --git a/yarn-project/merkle-tree/package.json b/yarn-project/merkle-tree/package.json index d9c347ed115..75bb7b0e2bf 100644 --- a/yarn-project/merkle-tree/package.json +++ b/yarn-project/merkle-tree/package.json @@ -34,9 +34,8 @@ "dependencies": { "@aztec/circuit-types": "workspace:^", "@aztec/foundation": "workspace:^", + "@aztec/kv-store": "workspace:^", "@aztec/types": "workspace:^", - "levelup": "^5.1.1", - "memdown": "^6.1.1", "sha256": "^0.2.0", "tslib": "^2.4.0" }, @@ -44,8 +43,6 @@ "@aztec/circuits.js": "workspace:^", "@jest/globals": "^29.5.0", "@types/jest": "^29.5.0", - "@types/levelup": "^5.1.2", - "@types/memdown": "^3.0.1", "@types/node": "^18.15.3", "@types/sha256": "^0.2.0", "jest": "^29.5.0", diff --git a/yarn-project/merkle-tree/src/interfaces/indexed_tree.ts b/yarn-project/merkle-tree/src/interfaces/indexed_tree.ts index 5e5531afacf..651f734f09e 100644 --- a/yarn-project/merkle-tree/src/interfaces/indexed_tree.ts +++ b/yarn-project/merkle-tree/src/interfaces/indexed_tree.ts @@ -1,8 +1,35 @@ -import { IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; +import { IndexedTreeLeaf, IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; import { SiblingPath } from '@aztec/types/membership'; import { AppendOnlyTree } from './append_only_tree.js'; +/** + * Factory for creating leaf preimages. + */ +export interface PreimageFactory { + /** + * Creates a new preimage from a leaf. + * @param leaf - Leaf to create a preimage from. + * @param nextKey - Next key of the leaf. + * @param nextIndex - Next index of the leaf. + */ + fromLeaf(leaf: IndexedTreeLeaf, nextKey: bigint, nextIndex: bigint): IndexedTreeLeafPreimage; + /** + * Creates a new preimage from a buffer. + * @param buffer - Buffer to create a preimage from. + */ + fromBuffer(buffer: Buffer): IndexedTreeLeafPreimage; + /** + * Creates an empty preimage. + */ + empty(): IndexedTreeLeafPreimage; + /** + * Creates a copy of a preimage. + * @param preimage - Preimage to be cloned. + */ + clone(preimage: IndexedTreeLeafPreimage): IndexedTreeLeafPreimage; +} + /** * All of the data to be return during batch insertion. */ @@ -56,7 +83,7 @@ export interface IndexedTree extends AppendOnlyTree { findIndexOfPreviousKey( newValue: bigint, includeUncommitted: boolean, - ): Promise< + ): | { /** * The index of the found leaf. @@ -67,8 +94,7 @@ export interface IndexedTree extends AppendOnlyTree { */ alreadyPresent: boolean; } - | undefined - >; + | undefined; /** * Gets the latest LeafPreimage copy. @@ -76,7 +102,7 @@ export interface IndexedTree extends AppendOnlyTree { * @param includeUncommitted - If true, the uncommitted changes are included in the search. * @returns A copy of the leaf preimage at the given index or undefined if the leaf was not found. */ - getLatestLeafPreimageCopy(index: bigint, includeUncommitted: boolean): Promise; + getLatestLeafPreimageCopy(index: bigint, includeUncommitted: boolean): IndexedTreeLeafPreimage | undefined; /** * Batch insert multiple leaves into the tree. diff --git a/yarn-project/merkle-tree/src/interfaces/merkle_tree.ts b/yarn-project/merkle-tree/src/interfaces/merkle_tree.ts index 257e5d79761..209656f885d 100644 --- a/yarn-project/merkle-tree/src/interfaces/merkle_tree.ts +++ b/yarn-project/merkle-tree/src/interfaces/merkle_tree.ts @@ -48,7 +48,7 @@ export interface MerkleTree extends SiblingPathSource { * @param index - The index of the leaf value to be returned. * @param includeUncommitted - Set to true to include uncommitted updates in the data set. */ - getLeafValue(index: bigint, includeUncommitted: boolean): Promise; + getLeafValue(index: bigint, includeUncommitted: boolean): Buffer | undefined; /** * Returns the index of a leaf given its value, or undefined if no leaf with that value is found. @@ -56,5 +56,5 @@ export interface MerkleTree extends SiblingPathSource { * @param includeUncommitted - Indicates whether to include uncommitted data. * @returns The index of the first leaf found with a given value (undefined if not found). */ - findLeafIndex(leaf: Buffer, includeUncommitted: boolean): Promise; + findLeafIndex(leaf: Buffer, includeUncommitted: boolean): bigint | undefined; } diff --git a/yarn-project/merkle-tree/src/load_tree.ts b/yarn-project/merkle-tree/src/load_tree.ts index 4d5bf606bb0..a4bf1e8853a 100644 --- a/yarn-project/merkle-tree/src/load_tree.ts +++ b/yarn-project/merkle-tree/src/load_tree.ts @@ -1,8 +1,7 @@ +import { AztecKVStore } from '@aztec/kv-store'; import { Hasher } from '@aztec/types/interfaces'; -import { LevelUp } from 'levelup'; - -import { TreeBase, decodeMeta } from './tree_base.js'; +import { TreeBase, getTreeMeta } from './tree_base.js'; /** * Creates a new tree and sets its root, depth and size based on the meta data which are associated with the name. @@ -12,15 +11,13 @@ import { TreeBase, decodeMeta } from './tree_base.js'; * @param name - Name of the tree. * @returns The newly created tree. */ -export async function loadTree( - c: new (db: LevelUp, hasher: Hasher, name: string, depth: number, size: bigint, root: Buffer) => T, - db: LevelUp, +export function loadTree( + c: new (store: AztecKVStore, hasher: Hasher, name: string, depth: number, size: bigint, root: Buffer) => T, + store: AztecKVStore, hasher: Hasher, name: string, ): Promise { - const meta: Buffer = await db.get(name); - const { root, depth, size } = decodeMeta(meta); - - const tree = new c(db, hasher, name, depth, size, root); - return tree; + const { root, depth, size } = getTreeMeta(store, name); + const tree = new c(store, hasher, name, depth, size, root); + return Promise.resolve(tree); } diff --git a/yarn-project/merkle-tree/src/new_tree.ts b/yarn-project/merkle-tree/src/new_tree.ts index 45e01ded402..5c354e21851 100644 --- a/yarn-project/merkle-tree/src/new_tree.ts +++ b/yarn-project/merkle-tree/src/new_tree.ts @@ -1,7 +1,6 @@ +import { AztecKVStore } from '@aztec/kv-store'; import { Hasher } from '@aztec/types/interfaces'; -import { LevelUp } from 'levelup'; - import { TreeBase } from './tree_base.js'; /** @@ -15,14 +14,14 @@ import { TreeBase } from './tree_base.js'; * @returns The newly created tree. */ export async function newTree( - c: new (db: LevelUp, hasher: Hasher, name: string, depth: number, size: bigint) => T, - db: LevelUp, + c: new (store: AztecKVStore, hasher: Hasher, name: string, depth: number, size: bigint) => T, + store: AztecKVStore, hasher: Hasher, name: string, depth: number, prefilledSize = 1, ): Promise { - const tree = new c(db, hasher, name, depth, 0n); + const tree = new c(store, hasher, name, depth, 0n); await tree.init(prefilledSize); return tree; } diff --git a/yarn-project/merkle-tree/src/snapshots/append_only_snapshot.test.ts b/yarn-project/merkle-tree/src/snapshots/append_only_snapshot.test.ts index b66eb2af22b..52ebdec437d 100644 --- a/yarn-project/merkle-tree/src/snapshots/append_only_snapshot.test.ts +++ b/yarn-project/merkle-tree/src/snapshots/append_only_snapshot.test.ts @@ -1,17 +1,16 @@ -import levelup, { LevelUp } from 'levelup'; +import { AztecKVStore, AztecLmdbStore } from '@aztec/kv-store'; import { Pedersen, StandardTree, newTree } from '../index.js'; -import { createMemDown } from '../test/utils/create_mem_down.js'; import { AppendOnlySnapshotBuilder } from './append_only_snapshot.js'; import { describeSnapshotBuilderTestSuite } from './snapshot_builder_test_suite.js'; describe('AppendOnlySnapshot', () => { let tree: StandardTree; let snapshotBuilder: AppendOnlySnapshotBuilder; - let db: LevelUp; + let db: AztecKVStore; beforeEach(async () => { - db = levelup(createMemDown()); + db = await AztecLmdbStore.openTmp(); const hasher = new Pedersen(); tree = await newTree(StandardTree, db, hasher, 'test', 4); snapshotBuilder = new AppendOnlySnapshotBuilder(db, tree, hasher); diff --git a/yarn-project/merkle-tree/src/snapshots/append_only_snapshot.ts b/yarn-project/merkle-tree/src/snapshots/append_only_snapshot.ts index 113a01d2bee..06d6e4b3194 100644 --- a/yarn-project/merkle-tree/src/snapshots/append_only_snapshot.ts +++ b/yarn-project/merkle-tree/src/snapshots/append_only_snapshot.ts @@ -1,23 +1,26 @@ +import { AztecKVStore, AztecMap } from '@aztec/kv-store'; import { Hasher } from '@aztec/types/interfaces'; import { SiblingPath } from '@aztec/types/membership'; -import { LevelUp } from 'levelup'; - import { AppendOnlyTree } from '../interfaces/append_only_tree.js'; import { TreeBase } from '../tree_base.js'; import { TreeSnapshot, TreeSnapshotBuilder } from './snapshot_builder.js'; // stores the last block that modified this node -const nodeModifiedAtBlockKey = (treeName: string, level: number, index: bigint) => - `snapshot:node:${treeName}:${level}:${index}:block`; +const nodeModifiedAtBlockKey = (level: number, index: bigint) => `node:${level}:${index}:modifiedAtBlock`; // stores the value of the node at the above block -const historicalNodeKey = (treeName: string, level: number, index: bigint) => - `snapshot:node:${treeName}:${level}:${index}:value`; +const historicalNodeKey = (level: number, index: bigint) => `node:${level}:${index}:value`; -// metadata for a snapshot -const snapshotRootKey = (treeName: string, block: number) => `snapshot:root:${treeName}:${block}`; -const snapshotNumLeavesKey = (treeName: string, block: number) => `snapshot:numLeaves:${treeName}:${block}`; +/** + * Metadata for a snapshot, per block + */ +type SnapshotMetadata = { + /** The tree root at the time */ + root: Buffer; + /** The number of filled leaves */ + numLeaves: bigint; +}; /** * A more space-efficient way of storing snapshots of AppendOnlyTrees that trades space need for slower @@ -35,91 +38,110 @@ const snapshotNumLeavesKey = (treeName: string, block: number) => `snapshot:numL * Worst case: O(H) database reads + O(H) hashes */ export class AppendOnlySnapshotBuilder implements TreeSnapshotBuilder { - constructor(private db: LevelUp, private tree: TreeBase & AppendOnlyTree, private hasher: Hasher) {} - async getSnapshot(block: number): Promise { - const meta = await this.#getSnapshotMeta(block); + #nodeValue: AztecMap, Buffer>; + #nodeLastModifiedByBlock: AztecMap, number>; + #snapshotMetadata: AztecMap; + + constructor(private db: AztecKVStore, private tree: TreeBase & AppendOnlyTree, private hasher: Hasher) { + const treeName = tree.getName(); + this.#nodeValue = db.openMap(`append_only_snapshot:${treeName}:node`); + this.#nodeLastModifiedByBlock = db.openMap(`append_ony_snapshot:${treeName}:block`); + this.#snapshotMetadata = db.openMap(`append_only_snapshot:${treeName}:snapshot_metadata`); + } + + getSnapshot(block: number): Promise { + const meta = this.#getSnapshotMeta(block); if (typeof meta === 'undefined') { - throw new Error(`Snapshot for tree ${this.tree.getName()} at block ${block} does not exist`); + return Promise.reject(new Error(`Snapshot for tree ${this.tree.getName()} at block ${block} does not exist`)); } - return new AppendOnlySnapshot(this.db, block, meta.numLeaves, meta.root, this.tree, this.hasher); + return Promise.resolve( + new AppendOnlySnapshot( + this.#nodeValue, + this.#nodeLastModifiedByBlock, + block, + meta.numLeaves, + meta.root, + this.tree, + this.hasher, + ), + ); } - async snapshot(block: number): Promise { - const meta = await this.#getSnapshotMeta(block); - if (typeof meta !== 'undefined') { - // no-op, we already have a snapshot - return new AppendOnlySnapshot(this.db, block, meta.numLeaves, meta.root, this.tree, this.hasher); - } - - const batch = this.db.batch(); - const root = this.tree.getRoot(false); - const depth = this.tree.getDepth(); - const treeName = this.tree.getName(); - const queue: [Buffer, number, bigint][] = [[root, 0, 0n]]; - - // walk the tree in BF and store latest nodes - while (queue.length > 0) { - const [node, level, index] = queue.shift()!; - - const historicalValue = await this.db.get(historicalNodeKey(treeName, level, index)).catch(() => undefined); - if (!historicalValue || !node.equals(historicalValue)) { - // we've never seen this node before or it's different than before - // update the historical tree and tag it with the block that modified it - batch.put(nodeModifiedAtBlockKey(treeName, level, index), String(block)); - batch.put(historicalNodeKey(treeName, level, index), node); - } else { - // if this node hasn't changed, that means, nothing below it has changed either - continue; - } - - if (level + 1 > depth) { - // short circuit if we've reached the leaf level - // otherwise getNode might throw if we ask for the children of a leaf - continue; + snapshot(block: number): Promise { + return this.db.transaction(() => { + const meta = this.#getSnapshotMeta(block); + if (typeof meta !== 'undefined') { + // no-op, we already have a snapshot + return new AppendOnlySnapshot( + this.#nodeValue, + this.#nodeLastModifiedByBlock, + block, + meta.numLeaves, + meta.root, + this.tree, + this.hasher, + ); } - // these could be undefined because zero hashes aren't stored in the tree - const [lhs, rhs] = await Promise.all([ - this.tree.getNode(level + 1, 2n * index), - this.tree.getNode(level + 1, 2n * index + 1n), - ]); - - if (lhs) { - queue.push([lhs, level + 1, 2n * index]); - } - - if (rhs) { - queue.push([rhs, level + 1, 2n * index + 1n]); + const root = this.tree.getRoot(false); + const depth = this.tree.getDepth(); + const queue: [Buffer, number, bigint][] = [[root, 0, 0n]]; + + // walk the tree in BF and store latest nodes + while (queue.length > 0) { + const [node, level, index] = queue.shift()!; + + const historicalValue = this.#nodeValue.get(historicalNodeKey(level, index)); + if (!historicalValue || !node.equals(historicalValue)) { + // we've never seen this node before or it's different than before + // update the historical tree and tag it with the block that modified it + void this.#nodeLastModifiedByBlock.set(nodeModifiedAtBlockKey(level, index), block); + void this.#nodeValue.set(historicalNodeKey(level, index), node); + } else { + // if this node hasn't changed, that means, nothing below it has changed either + continue; + } + + if (level + 1 > depth) { + // short circuit if we've reached the leaf level + // otherwise getNode might throw if we ask for the children of a leaf + continue; + } + + // these could be undefined because zero hashes aren't stored in the tree + const [lhs, rhs] = [this.tree.getNode(level + 1, 2n * index), this.tree.getNode(level + 1, 2n * index + 1n)]; + + if (lhs) { + queue.push([lhs, level + 1, 2n * index]); + } + + if (rhs) { + queue.push([rhs, level + 1, 2n * index + 1n]); + } } - } - - const numLeaves = this.tree.getNumLeaves(false); - batch.put(snapshotNumLeavesKey(treeName, block), String(numLeaves)); - batch.put(snapshotRootKey(treeName, block), root); - await batch.write(); - return new AppendOnlySnapshot(this.db, block, numLeaves, root, this.tree, this.hasher); + const numLeaves = this.tree.getNumLeaves(false); + void this.#snapshotMetadata.set(block, { + numLeaves, + root, + }); + + return new AppendOnlySnapshot( + this.#nodeValue, + this.#nodeLastModifiedByBlock, + block, + numLeaves, + root, + this.tree, + this.hasher, + ); + }); } - async #getSnapshotMeta(block: number): Promise< - | { - /** The root of the tree snapshot */ - root: Buffer; - /** The number of leaves in the tree snapshot */ - numLeaves: bigint; - } - | undefined - > { - try { - const treeName = this.tree.getName(); - const root = await this.db.get(snapshotRootKey(treeName, block)); - const numLeaves = BigInt(await this.db.get(snapshotNumLeavesKey(treeName, block))); - return { root, numLeaves }; - } catch (err) { - return undefined; - } + #getSnapshotMeta(block: number): SnapshotMetadata | undefined { + return this.#snapshotMetadata.get(block); } } @@ -128,7 +150,8 @@ export class AppendOnlySnapshotBuilder implements TreeSnapshotBuilder { */ class AppendOnlySnapshot implements TreeSnapshot { constructor( - private db: LevelUp, + private nodes: AztecMap, + private nodeHistory: AztecMap, private block: number, private leafCount: bigint, private historicalRoot: Buffer, @@ -136,7 +159,7 @@ class AppendOnlySnapshot implements TreeSnapshot { private hasher: Hasher, ) {} - public async getSiblingPath(index: bigint): Promise> { + public getSiblingPath(index: bigint): SiblingPath { const path: Buffer[] = []; const depth = this.tree.getDepth(); let level = depth; @@ -145,7 +168,7 @@ class AppendOnlySnapshot implements TreeSnapshot { const isRight = index & 0x01n; const siblingIndex = isRight ? index - 1n : index + 1n; - const sibling = await this.#getHistoricalNodeValue(level, siblingIndex); + const sibling = this.#getHistoricalNodeValue(level, siblingIndex); path.push(sibling); level -= 1; @@ -168,9 +191,9 @@ class AppendOnlySnapshot implements TreeSnapshot { return this.historicalRoot; } - async getLeafValue(index: bigint): Promise { + getLeafValue(index: bigint): Buffer | undefined { const leafLevel = this.getDepth(); - const blockNumber = await this.#getBlockNumberThatModifiedNode(leafLevel, index); + const blockNumber = this.#getBlockNumberThatModifiedNode(leafLevel, index); // leaf hasn't been set yet if (typeof blockNumber === 'undefined') { @@ -179,15 +202,15 @@ class AppendOnlySnapshot implements TreeSnapshot { // leaf was set some time in the past if (blockNumber <= this.block) { - return this.db.get(historicalNodeKey(this.tree.getName(), leafLevel, index)); + return this.nodes.get(historicalNodeKey(leafLevel, index)); } // leaf has been set but in a block in the future return undefined; } - async #getHistoricalNodeValue(level: number, index: bigint): Promise { - const blockNumber = await this.#getBlockNumberThatModifiedNode(level, index); + #getHistoricalNodeValue(level: number, index: bigint): Buffer { + const blockNumber = this.#getBlockNumberThatModifiedNode(level, index); // node has never been set if (typeof blockNumber === 'undefined') { @@ -196,7 +219,7 @@ class AppendOnlySnapshot implements TreeSnapshot { // node was set some time in the past if (blockNumber <= this.block) { - return this.db.get(historicalNodeKey(this.tree.getName(), level, index)); + return this.nodes.get(historicalNodeKey(level, index))!; } // the node has been modified since this snapshot was taken @@ -214,27 +237,22 @@ class AppendOnlySnapshot implements TreeSnapshot { return this.tree.getZeroHash(level); } - const [lhs, rhs] = await Promise.all([ + const [lhs, rhs] = [ this.#getHistoricalNodeValue(level + 1, 2n * index), this.#getHistoricalNodeValue(level + 1, 2n * index + 1n), - ]); + ]; return this.hasher.hash(lhs, rhs); } - async #getBlockNumberThatModifiedNode(level: number, index: bigint): Promise { - try { - const value: Buffer | string = await this.db.get(nodeModifiedAtBlockKey(this.tree.getName(), level, index)); - return parseInt(value.toString(), 10); - } catch (err) { - return undefined; - } + #getBlockNumberThatModifiedNode(level: number, index: bigint): number | undefined { + return this.nodeHistory.get(nodeModifiedAtBlockKey(level, index)); } - async findLeafIndex(value: Buffer): Promise { + findLeafIndex(value: Buffer): bigint | undefined { const numLeaves = this.getNumLeaves(); for (let i = 0n; i < numLeaves; i++) { - const currentValue = await this.getLeafValue(i); + const currentValue = this.getLeafValue(i); if (currentValue && currentValue.equals(value)) { return i; } diff --git a/yarn-project/merkle-tree/src/snapshots/base_full_snapshot.ts b/yarn-project/merkle-tree/src/snapshots/base_full_snapshot.ts index 8f49bb91582..5f1e250c569 100644 --- a/yarn-project/merkle-tree/src/snapshots/base_full_snapshot.ts +++ b/yarn-project/merkle-tree/src/snapshots/base_full_snapshot.ts @@ -1,17 +1,18 @@ +import { AztecKVStore, AztecMap } from '@aztec/kv-store'; import { SiblingPath } from '@aztec/types/membership'; -import { LevelUp, LevelUpChain } from 'levelup'; - import { TreeBase } from '../tree_base.js'; import { TreeSnapshot, TreeSnapshotBuilder } from './snapshot_builder.js'; -// key for a node's children -const snapshotChildKey = (node: Buffer, child: 0 | 1) => - Buffer.concat([Buffer.from('snapshot:node:'), node, Buffer.from(':' + child)]); - -// metadata for a snapshot -const snapshotRootKey = (treeName: string, block: number) => `snapshot:root:${treeName}:${block}`; -const snapshotNumLeavesKey = (treeName: string, block: number) => `snapshot:numLeaves:${treeName}:${block}`; +/** + * Metadata for a snapshot, per block + */ +type SnapshotMetadata = { + /** The tree root at the time */ + root: Buffer; + /** The number of filled leaves */ + numLeaves: bigint; +}; /** * Builds a full snapshot of a tree. This implementation works for any Merkle tree and stores @@ -32,104 +33,86 @@ const snapshotNumLeavesKey = (treeName: string, block: number) => `snapshot:numL export abstract class BaseFullTreeSnapshotBuilder implements TreeSnapshotBuilder { - constructor(protected db: LevelUp, protected tree: T) {} - - async snapshot(block: number): Promise { - const snapshotMetadata = await this.#getSnapshotMeta(block); - - if (snapshotMetadata) { - return this.openSnapshot(snapshotMetadata.root, snapshotMetadata.numLeaves); - } - - const batch = this.db.batch(); - const root = this.tree.getRoot(false); - const numLeaves = this.tree.getNumLeaves(false); - const depth = this.tree.getDepth(); - const queue: [Buffer, number, bigint][] = [[root, 0, 0n]]; - - // walk the tree breadth-first and store each of its nodes in the database - // for each node we save two keys - // :0 -> - // :1 -> - while (queue.length > 0) { - const [node, level, i] = queue.shift()!; - // check if the database already has a child for this tree - // if it does, then we know we've seen the whole subtree below it before - // and we don't have to traverse it anymore - // we use the left child here, but it could be anything that shows we've stored the node before - const exists: Buffer | undefined = await this.db.get(snapshotChildKey(node, 0)).catch(() => undefined); - if (exists) { - continue; - } + protected nodes: AztecMap; + protected snapshotMetadata: AztecMap; - if (level + 1 > depth) { - // short circuit if we've reached the leaf level - // otherwise getNode might throw if we ask for the children of a leaf - await this.handleLeaf(i, node, batch); - continue; - } - - const [lhs, rhs] = await Promise.all([ - this.tree.getNode(level + 1, 2n * i), - this.tree.getNode(level + 1, 2n * i + 1n), - ]); - - // we want the zero hash at the children's level, not the node's level - const zeroHash = this.tree.getZeroHash(level + 1); + constructor(protected db: AztecKVStore, protected tree: T) { + this.nodes = db.openMap(`full_snapshot:${tree.getName()}:node`); + this.snapshotMetadata = db.openMap(`full_snapshot:${tree.getName()}:metadata`); + } - batch.put(snapshotChildKey(node, 0), lhs ?? zeroHash); - batch.put(snapshotChildKey(node, 1), rhs ?? zeroHash); + snapshot(block: number): Promise { + return this.db.transaction(() => { + const snapshotMetadata = this.#getSnapshotMeta(block); - // enqueue the children only if they're not zero hashes - if (lhs) { - queue.push([lhs, level + 1, 2n * i]); + if (snapshotMetadata) { + return this.openSnapshot(snapshotMetadata.root, snapshotMetadata.numLeaves); } - if (rhs) { - queue.push([rhs, level + 1, 2n * i + 1n]); + const root = this.tree.getRoot(false); + const numLeaves = this.tree.getNumLeaves(false); + const depth = this.tree.getDepth(); + const queue: [Buffer, number, bigint][] = [[root, 0, 0n]]; + + // walk the tree breadth-first and store each of its nodes in the database + // for each node we save two keys + // :0 -> + // :1 -> + while (queue.length > 0) { + const [node, level, i] = queue.shift()!; + const nodeKey = node.toString('hex'); + // check if the database already has a child for this tree + // if it does, then we know we've seen the whole subtree below it before + // and we don't have to traverse it anymore + // we use the left child here, but it could be anything that shows we've stored the node before + if (this.nodes.has(nodeKey)) { + continue; + } + + if (level + 1 > depth) { + // short circuit if we've reached the leaf level + // otherwise getNode might throw if we ask for the children of a leaf + this.handleLeaf(i, node); + continue; + } + + const [lhs, rhs] = [this.tree.getNode(level + 1, 2n * i), this.tree.getNode(level + 1, 2n * i + 1n)]; + + // we want the zero hash at the children's level, not the node's level + const zeroHash = this.tree.getZeroHash(level + 1); + + void this.nodes.set(nodeKey, [lhs ?? zeroHash, rhs ?? zeroHash]); + // enqueue the children only if they're not zero hashes + if (lhs) { + queue.push([lhs, level + 1, 2n * i]); + } + + if (rhs) { + queue.push([rhs, level + 1, 2n * i + 1n]); + } } - } - batch.put(snapshotRootKey(this.tree.getName(), block), root); - batch.put(snapshotNumLeavesKey(this.tree.getName(), block), String(numLeaves)); - await batch.write(); - - return this.openSnapshot(root, numLeaves); + void this.snapshotMetadata.set(block, { root, numLeaves }); + return this.openSnapshot(root, numLeaves); + }); } - protected handleLeaf(_index: bigint, _node: Buffer, _batch: LevelUpChain) { - return Promise.resolve(); - } + protected handleLeaf(_index: bigint, _node: Buffer): void {} - async getSnapshot(version: number): Promise { - const snapshotMetadata = await this.#getSnapshotMeta(version); + getSnapshot(version: number): Promise { + const snapshotMetadata = this.#getSnapshotMeta(version); if (!snapshotMetadata) { - throw new Error(`Version ${version} does not exist for tree ${this.tree.getName()}`); + return Promise.reject(new Error(`Version ${version} does not exist for tree ${this.tree.getName()}`)); } - return this.openSnapshot(snapshotMetadata.root, snapshotMetadata.numLeaves); + return Promise.resolve(this.openSnapshot(snapshotMetadata.root, snapshotMetadata.numLeaves)); } protected abstract openSnapshot(root: Buffer, numLeaves: bigint): S; - async #getSnapshotMeta(block: number): Promise< - | { - /** The root of the tree snapshot */ - root: Buffer; - /** The number of leaves in the tree snapshot */ - numLeaves: bigint; - } - | undefined - > { - try { - const treeName = this.tree.getName(); - const root = await this.db.get(snapshotRootKey(treeName, block)); - const numLeaves = BigInt(await this.db.get(snapshotNumLeavesKey(treeName, block))); - return { root, numLeaves }; - } catch (err) { - return undefined; - } + #getSnapshotMeta(block: number): SnapshotMetadata | undefined { + return this.snapshotMetadata.get(block); } } @@ -138,16 +121,16 @@ export abstract class BaseFullTreeSnapshotBuilder, protected historicRoot: Buffer, protected numLeaves: bigint, protected tree: TreeBase, ) {} - async getSiblingPath(index: bigint): Promise> { + getSiblingPath(index: bigint): SiblingPath { const siblings: Buffer[] = []; - for await (const [_node, sibling] of this.pathFromRootToLeaf(index)) { + for (const [_node, sibling] of this.pathFromRootToLeaf(index)) { siblings.push(sibling); } @@ -158,9 +141,9 @@ export class BaseFullTreeSnapshot implements TreeSnapshot { return new SiblingPath(this.tree.getDepth() as N, siblings); } - async getLeafValue(index: bigint): Promise { + getLeafValue(index: bigint): Buffer | undefined { let leafNode: Buffer | undefined = undefined; - for await (const [node, _sibling] of this.pathFromRootToLeaf(index)) { + for (const [node, _sibling] of this.pathFromRootToLeaf(index)) { leafNode = node; } @@ -179,17 +162,17 @@ export class BaseFullTreeSnapshot implements TreeSnapshot { return this.numLeaves; } - protected async *pathFromRootToLeaf(leafIndex: bigint) { + protected *pathFromRootToLeaf(leafIndex: bigint) { const root = this.historicRoot; const pathFromRoot = this.#getPathFromRoot(leafIndex); let node: Buffer = root; for (let i = 0; i < pathFromRoot.length; i++) { // get both children. We'll need both anyway (one to keep track of, the other to walk down to) - const children: [Buffer, Buffer] = await Promise.all([ - this.db.get(snapshotChildKey(node, 0)), - this.db.get(snapshotChildKey(node, 1)), - ]).catch(() => [this.tree.getZeroHash(i + 1), this.tree.getZeroHash(i + 1)]); + const children: [Buffer, Buffer] = this.db.get(node.toString('hex')) ?? [ + this.tree.getZeroHash(i + 1), + this.tree.getZeroHash(i + 1), + ]; const next = children[pathFromRoot[i]]; const sibling = children[(pathFromRoot[i] + 1) % 2]; @@ -219,10 +202,10 @@ export class BaseFullTreeSnapshot implements TreeSnapshot { return path; } - async findLeafIndex(value: Buffer): Promise { + findLeafIndex(value: Buffer): bigint | undefined { const numLeaves = this.getNumLeaves(); for (let i = 0n; i < numLeaves; i++) { - const currentValue = await this.getLeafValue(i); + const currentValue = this.getLeafValue(i); if (currentValue && currentValue.equals(value)) { return i; } diff --git a/yarn-project/merkle-tree/src/snapshots/full_snapshot.test.ts b/yarn-project/merkle-tree/src/snapshots/full_snapshot.test.ts index 3f2cc2af791..4219dbd1c45 100644 --- a/yarn-project/merkle-tree/src/snapshots/full_snapshot.test.ts +++ b/yarn-project/merkle-tree/src/snapshots/full_snapshot.test.ts @@ -1,17 +1,16 @@ -import levelup, { LevelUp } from 'levelup'; +import { AztecKVStore, AztecLmdbStore } from '@aztec/kv-store'; import { Pedersen, StandardTree, newTree } from '../index.js'; -import { createMemDown } from '../test/utils/create_mem_down.js'; import { FullTreeSnapshotBuilder } from './full_snapshot.js'; import { describeSnapshotBuilderTestSuite } from './snapshot_builder_test_suite.js'; describe('FullSnapshotBuilder', () => { let tree: StandardTree; let snapshotBuilder: FullTreeSnapshotBuilder; - let db: LevelUp; + let db: AztecKVStore; beforeEach(async () => { - db = levelup(createMemDown()); + db = await AztecLmdbStore.openTmp(); tree = await newTree(StandardTree, db, new Pedersen(), 'test', 4); snapshotBuilder = new FullTreeSnapshotBuilder(db, tree); }); diff --git a/yarn-project/merkle-tree/src/snapshots/full_snapshot.ts b/yarn-project/merkle-tree/src/snapshots/full_snapshot.ts index c78d0ebb188..73cce3b05e7 100644 --- a/yarn-project/merkle-tree/src/snapshots/full_snapshot.ts +++ b/yarn-project/merkle-tree/src/snapshots/full_snapshot.ts @@ -21,6 +21,6 @@ export class FullTreeSnapshotBuilder implements TreeSnapshotBuilder { protected openSnapshot(root: Buffer, numLeaves: bigint): TreeSnapshot { - return new BaseFullTreeSnapshot(this.db, root, numLeaves, this.tree); + return new BaseFullTreeSnapshot(this.nodes, root, numLeaves, this.tree); } } diff --git a/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.test.ts b/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.test.ts index 664c83e605d..e1ba0b9e0f7 100644 --- a/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.test.ts +++ b/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.test.ts @@ -1,27 +1,25 @@ import { Fr, NullifierLeaf, NullifierLeafPreimage } from '@aztec/circuits.js'; +import { AztecKVStore, AztecLmdbStore } from '@aztec/kv-store'; import { Hasher } from '@aztec/types/interfaces'; -import levelup, { LevelUp } from 'levelup'; - import { Pedersen, newTree } from '../index.js'; import { StandardIndexedTreeWithAppend } from '../standard_indexed_tree/test/standard_indexed_tree_with_append.js'; -import { createMemDown } from '../test/utils/create_mem_down.js'; import { IndexedTreeSnapshotBuilder } from './indexed_tree_snapshot.js'; import { describeSnapshotBuilderTestSuite } from './snapshot_builder_test_suite.js'; class NullifierTree extends StandardIndexedTreeWithAppend { - constructor(db: levelup.LevelUp, hasher: Hasher, name: string, depth: number, size: bigint = 0n, root?: Buffer) { + constructor(db: AztecKVStore, hasher: Hasher, name: string, depth: number, size: bigint = 0n, root?: Buffer) { super(db, hasher, name, depth, size, NullifierLeafPreimage, NullifierLeaf, root); } } describe('IndexedTreeSnapshotBuilder', () => { - let db: LevelUp; + let db: AztecKVStore; let tree: StandardIndexedTreeWithAppend; let snapshotBuilder: IndexedTreeSnapshotBuilder; beforeEach(async () => { - db = levelup(createMemDown()); + db = await AztecLmdbStore.openTmp(); tree = await newTree(NullifierTree, db, new Pedersen(), 'test', 4); snapshotBuilder = new IndexedTreeSnapshotBuilder(db, tree, NullifierLeafPreimage); }); @@ -54,26 +52,26 @@ describe('IndexedTreeSnapshotBuilder', () => { await tree.appendLeaves([Buffer.from('d'), Buffer.from('e'), Buffer.from('f')]); await tree.commit(); - const expectedLeavesAtBlock2 = await Promise.all([ + const expectedLeavesAtBlock2 = [ tree.getLatestLeafPreimageCopy(0n, false), tree.getLatestLeafPreimageCopy(1n, false), tree.getLatestLeafPreimageCopy(2n, false), tree.getLatestLeafPreimageCopy(3n, false), tree.getLatestLeafPreimageCopy(4n, false), tree.getLatestLeafPreimageCopy(5n, false), - ]); + ]; await snapshotBuilder.snapshot(2); const snapshot1 = await snapshotBuilder.getSnapshot(1); - const actualLeavesAtBlock1 = await Promise.all([ + const actualLeavesAtBlock1 = [ snapshot1.getLatestLeafPreimageCopy(0n), snapshot1.getLatestLeafPreimageCopy(1n), snapshot1.getLatestLeafPreimageCopy(2n), snapshot1.getLatestLeafPreimageCopy(3n), snapshot1.getLatestLeafPreimageCopy(4n), snapshot1.getLatestLeafPreimageCopy(5n), - ]); + ]; expect(actualLeavesAtBlock1).toEqual(expectedLeavesAtBlock1); const snapshot2 = await snapshotBuilder.getSnapshot(2); @@ -94,12 +92,12 @@ describe('IndexedTreeSnapshotBuilder', () => { await tree.appendLeaves([Buffer.from('a'), Buffer.from('f'), Buffer.from('d')]); await tree.commit(); const snapshot = await snapshotBuilder.snapshot(1); - const historicalPrevValue = await tree.findIndexOfPreviousKey(2n, false); + const historicalPrevValue = tree.findIndexOfPreviousKey(2n, false); await tree.appendLeaves([Buffer.from('c'), Buffer.from('b'), Buffer.from('e')]); await tree.commit(); - await expect(snapshot.findIndexOfPreviousKey(2n)).resolves.toEqual(historicalPrevValue); + expect(snapshot.findIndexOfPreviousKey(2n)).toEqual(historicalPrevValue); }); }); }); diff --git a/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.ts b/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.ts index 28aeefdc953..8a787fd2067 100644 --- a/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.ts +++ b/yarn-project/merkle-tree/src/snapshots/indexed_tree_snapshot.ts @@ -1,33 +1,32 @@ import { IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; +import { AztecKVStore, AztecMap } from '@aztec/kv-store'; -import { LevelUp, LevelUpChain } from 'levelup'; - -import { IndexedTree } from '../interfaces/indexed_tree.js'; -import { PreimageFactory } from '../standard_indexed_tree/standard_indexed_tree.js'; +import { IndexedTree, PreimageFactory } from '../interfaces/indexed_tree.js'; import { TreeBase } from '../tree_base.js'; import { BaseFullTreeSnapshot, BaseFullTreeSnapshotBuilder } from './base_full_snapshot.js'; import { IndexedTreeSnapshot, TreeSnapshotBuilder } from './snapshot_builder.js'; -const snapshotLeafValue = (node: Buffer, index: bigint) => - Buffer.concat([Buffer.from('snapshot:leaf:'), node, Buffer.from(':' + index)]); +const snapshotLeafValue = (node: Buffer, index: bigint) => 'snapshot:leaf:' + node.toString('hex') + ':' + index; /** a */ export class IndexedTreeSnapshotBuilder extends BaseFullTreeSnapshotBuilder implements TreeSnapshotBuilder { - constructor(db: LevelUp, tree: IndexedTree & TreeBase, private leafPreimageBuilder: PreimageFactory) { - super(db, tree); + leaves: AztecMap; + constructor(store: AztecKVStore, tree: IndexedTree & TreeBase, private leafPreimageBuilder: PreimageFactory) { + super(store, tree); + this.leaves = store.openMap('indexed_tree_snapshot:' + tree.getName()); } protected openSnapshot(root: Buffer, numLeaves: bigint): IndexedTreeSnapshot { - return new IndexedTreeSnapshotImpl(this.db, root, numLeaves, this.tree, this.leafPreimageBuilder); + return new IndexedTreeSnapshotImpl(this.nodes, this.leaves, root, numLeaves, this.tree, this.leafPreimageBuilder); } - protected async handleLeaf(index: bigint, node: Buffer, batch: LevelUpChain) { - const leafPreimage = await this.tree.getLatestLeafPreimageCopy(index, false); + protected handleLeaf(index: bigint, node: Buffer) { + const leafPreimage = this.tree.getLatestLeafPreimageCopy(index, false); if (leafPreimage) { - batch.put(snapshotLeafValue(node, index), leafPreimage.toBuffer()); + void this.leaves.set(snapshotLeafValue(node, index), leafPreimage.toBuffer()); } } } @@ -35,7 +34,8 @@ export class IndexedTreeSnapshotBuilder /** A snapshot of an indexed tree at a particular point in time */ class IndexedTreeSnapshotImpl extends BaseFullTreeSnapshot implements IndexedTreeSnapshot { constructor( - db: LevelUp, + db: AztecMap, + private leaves: AztecMap, historicRoot: Buffer, numLeaves: bigint, tree: IndexedTree & TreeBase, @@ -44,14 +44,14 @@ class IndexedTreeSnapshotImpl extends BaseFullTreeSnapshot implements IndexedTre super(db, historicRoot, numLeaves, tree); } - async getLeafValue(index: bigint): Promise { - const leafPreimage = await this.getLatestLeafPreimageCopy(index); + getLeafValue(index: bigint): Buffer | undefined { + const leafPreimage = this.getLatestLeafPreimageCopy(index); return leafPreimage?.toBuffer(); } - async getLatestLeafPreimageCopy(index: bigint): Promise { - const leafNode = await super.getLeafValue(index); - const leafValue = await this.db.get(snapshotLeafValue(leafNode!, index)).catch(() => undefined); + getLatestLeafPreimageCopy(index: bigint): IndexedTreeLeafPreimage | undefined { + const leafNode = super.getLeafValue(index); + const leafValue = this.leaves.get(snapshotLeafValue(leafNode!, index)); if (leafValue) { return this.leafPreimageBuilder.fromBuffer(leafValue); } else { @@ -59,7 +59,7 @@ class IndexedTreeSnapshotImpl extends BaseFullTreeSnapshot implements IndexedTre } } - async findIndexOfPreviousKey(newValue: bigint): Promise<{ + findIndexOfPreviousKey(newValue: bigint): { /** * The index of the found leaf. */ @@ -68,13 +68,13 @@ class IndexedTreeSnapshotImpl extends BaseFullTreeSnapshot implements IndexedTre * A flag indicating if the corresponding leaf's value is equal to `newValue`. */ alreadyPresent: boolean; - }> { + } { const numLeaves = this.getNumLeaves(); const diff: bigint[] = []; for (let i = 0; i < numLeaves; i++) { // this is very inefficient - const storedLeaf = await this.getLatestLeafPreimageCopy(BigInt(i))!; + const storedLeaf = this.getLatestLeafPreimageCopy(BigInt(i))!; // The stored leaf can be undefined if it addresses an empty leaf // If the leaf is empty we do the same as if the leaf was larger @@ -99,8 +99,8 @@ class IndexedTreeSnapshotImpl extends BaseFullTreeSnapshot implements IndexedTre return { index: BigInt(minIndex), alreadyPresent: false }; } - async findLeafIndex(value: Buffer): Promise { - const index = await this.tree.findLeafIndex(value, false); + findLeafIndex(value: Buffer): bigint | undefined { + const index = this.tree.findLeafIndex(value, false); if (index !== undefined && index < this.getNumLeaves()) { return index; } diff --git a/yarn-project/merkle-tree/src/snapshots/snapshot_builder.ts b/yarn-project/merkle-tree/src/snapshots/snapshot_builder.ts index 872395e134e..98bbb9051d8 100644 --- a/yarn-project/merkle-tree/src/snapshots/snapshot_builder.ts +++ b/yarn-project/merkle-tree/src/snapshots/snapshot_builder.ts @@ -41,13 +41,13 @@ export interface TreeSnapshot { * Returns the value of a leaf at the specified index. * @param index - The index of the leaf value to be returned. */ - getLeafValue(index: bigint): Promise; + getLeafValue(index: bigint): Buffer | undefined; /** * Returns the sibling path for a requested leaf index. * @param index - The index of the leaf for which a sibling path is required. */ - getSiblingPath(index: bigint): Promise>; + getSiblingPath(index: bigint): SiblingPath; /** * Returns the index of a leaf given its value, or undefined if no leaf with that value is found. @@ -55,7 +55,7 @@ export interface TreeSnapshot { * @param value - The leaf value to look for. * @returns The index of the first leaf found with a given value (undefined if not found). */ - findLeafIndex(value: Buffer): Promise; + findLeafIndex(value: Buffer): bigint | undefined; } /** A snapshot of an indexed tree */ @@ -64,14 +64,14 @@ export interface IndexedTreeSnapshot extends TreeSnapshot { * Gets the historical data for a leaf * @param index - The index of the leaf to get the data for */ - getLatestLeafPreimageCopy(index: bigint): Promise; + getLatestLeafPreimageCopy(index: bigint): IndexedTreeLeafPreimage | undefined; /** * Finds the index of the largest leaf whose value is less than or equal to the provided value. * @param newValue - The new value to be inserted into the tree. * @returns The found leaf index and a flag indicating if the corresponding leaf's value is equal to `newValue`. */ - findIndexOfPreviousKey(newValue: bigint): Promise<{ + findIndexOfPreviousKey(newValue: bigint): { /** * The index of the found leaf. */ @@ -80,5 +80,5 @@ export interface IndexedTreeSnapshot extends TreeSnapshot { * A flag indicating if the corresponding leaf's value is equal to `newValue`. */ alreadyPresent: boolean; - }>; + }; } diff --git a/yarn-project/merkle-tree/src/snapshots/snapshot_builder_test_suite.ts b/yarn-project/merkle-tree/src/snapshots/snapshot_builder_test_suite.ts index f50ff1d69ae..85aa63f7b66 100644 --- a/yarn-project/merkle-tree/src/snapshots/snapshot_builder_test_suite.ts +++ b/yarn-project/merkle-tree/src/snapshots/snapshot_builder_test_suite.ts @@ -185,13 +185,13 @@ export function describeSnapshotBuilderTestSuite => { - return await newTree(SparseTree, levelUp, hasher, name, depth); +const createDb = async (db: AztecKVStore, hasher: Hasher, name: string, depth: number): Promise => { + return await newTree(SparseTree, db, hasher, name, depth); }; -const createFromName = async (levelUp: levelup.LevelUp, hasher: Hasher, name: string): Promise => { - return await loadTree(SparseTree, levelUp, hasher, name); +const createFromName = async (db: AztecKVStore, hasher: Hasher, name: string): Promise => { + return await loadTree(SparseTree, db, hasher, name); }; const TEST_TREE_DEPTH = 3; @@ -42,19 +36,19 @@ describe('SparseTreeSpecific', () => { }); it('throws when index is bigger than (2^DEPTH - 1) ', async () => { - const db = levelup(createMemDown()); + const db = await AztecLmdbStore.openTmp(); const depth = 32; const tree = await createDb(db, pedersen, 'test', depth); const index = 2n ** BigInt(depth); - await expect(tree.updateLeaf(Buffer.alloc(32), index)).rejects.toThrow(); + expect(() => tree.updateLeaf(Buffer.alloc(32), index)).toThrow(); }); it('updating non-empty leaf does not change tree size', async () => { const depth = 32; const maxIndex = 2 ** depth - 1; - const db = levelup(createMemDown()); + const db = await AztecLmdbStore.openTmp(); const tree = await createDb(db, pedersen, 'test', depth); const randomIndex = BigInt(Math.floor(Math.random() * maxIndex)); @@ -73,7 +67,7 @@ describe('SparseTreeSpecific', () => { const depth = 254; const maxIndex = 2 ** depth - 1; - const db = levelup(createMemDown()); + const db = await AztecLmdbStore.openTmp(); const tree = await createDb(db, pedersen, 'test', depth); const randomIndex = BigInt(Math.floor(Math.random() * maxIndex)); @@ -89,7 +83,7 @@ describe('SparseTreeSpecific', () => { }); it('should have correct root and sibling path after in a "non-append-only" way', async () => { - const db = levelup(createMemDown()); + const db = await AztecLmdbStore.openTmp(); const tree = await createDb(db, pedersen, 'test', 3); const level2ZeroHash = pedersen.hash(INITIAL_LEAF, INITIAL_LEAF); @@ -162,7 +156,7 @@ describe('SparseTreeSpecific', () => { const depth = 254; const maxIndex = 2 ** depth - 1; - const db = levelup(createMemDown()); + const db = await AztecLmdbStore.openTmp(); const tree = await createDb(db, pedersen, 'test', depth); const leaves = Array.from({ length: 1000 }).map(() => randomBytes(32)); diff --git a/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.ts b/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.ts index 138ca8f21e7..86f3c8506e4 100644 --- a/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.ts +++ b/yarn-project/merkle-tree/src/sparse_tree/sparse_tree.ts @@ -7,24 +7,23 @@ import { INITIAL_LEAF, TreeBase } from '../tree_base.js'; * A Merkle tree implementation that uses a LevelDB database to store the tree. */ export class SparseTree extends TreeBase implements UpdateOnlyTree { - #snapshotBuilder = new FullTreeSnapshotBuilder(this.db, this); - + #snapshotBuilder = new FullTreeSnapshotBuilder(this.store, this); /** * Updates a leaf in the tree. * @param leaf - New contents of the leaf. * @param index - Index of the leaf to be updated. */ - public async updateLeaf(leaf: Buffer, index: bigint): Promise { + public updateLeaf(leaf: Buffer, index: bigint): Promise { if (index > this.maxIndex) { throw Error(`Index out of bounds. Index ${index}, max index: ${this.maxIndex}.`); } const insertingZeroElement = leaf.equals(INITIAL_LEAF); - const originallyZeroElement = (await this.getLeafValue(index, true))?.equals(INITIAL_LEAF); + const originallyZeroElement = this.getLeafValue(index, true)?.equals(INITIAL_LEAF); if (insertingZeroElement && originallyZeroElement) { - return; + return Promise.resolve(); } - await this.addLeafToCacheAndHashToRoot(leaf, index); + this.addLeafToCacheAndHashToRoot(leaf, index); if (insertingZeroElement) { // Deleting element (originally non-zero and new value is zero) this.cachedSize = (this.cachedSize ?? this.size) - 1n; @@ -32,6 +31,8 @@ export class SparseTree extends TreeBase implements UpdateOnlyTree { // Inserting new element (originally zero and new value is non-zero) this.cachedSize = (this.cachedSize ?? this.size) + 1n; } + + return Promise.resolve(); } public snapshot(block: number): Promise { @@ -42,7 +43,7 @@ export class SparseTree extends TreeBase implements UpdateOnlyTree { return this.#snapshotBuilder.getSnapshot(block); } - public findLeafIndex(_value: Buffer, _includeUncommitted: boolean): Promise { + public findLeafIndex(_value: Buffer, _includeUncommitted: boolean): bigint | undefined { throw new Error('Finding leaf index is not supported for sparse trees'); } } diff --git a/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts b/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts index dcd56f31839..316bc2a62df 100644 --- a/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts +++ b/yarn-project/merkle-tree/src/standard_indexed_tree/standard_indexed_tree.ts @@ -1,57 +1,22 @@ import { TreeInsertionStats } from '@aztec/circuit-types/stats'; -import { toBigIntBE, toBufferBE } from '@aztec/foundation/bigint-buffer'; -import { createDebugLogger } from '@aztec/foundation/log'; +import { toBufferBE } from '@aztec/foundation/bigint-buffer'; import { Timer } from '@aztec/foundation/timer'; import { IndexedTreeLeaf, IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; +import { AztecKVStore, AztecMap } from '@aztec/kv-store'; import { Hasher } from '@aztec/types/interfaces'; import { SiblingPath } from '@aztec/types/membership'; -import { LevelUp } from 'levelup'; - -import { - BatchInsertionResult, - IndexedTree, - IndexedTreeSnapshot, - IndexedTreeSnapshotBuilder, - LowLeafWitnessData, -} from '../index.js'; +import { BatchInsertionResult, IndexedTree, LowLeafWitnessData, PreimageFactory } from '../interfaces/indexed_tree.js'; +import { IndexedTreeSnapshotBuilder } from '../snapshots/indexed_tree_snapshot.js'; +import { IndexedTreeSnapshot } from '../snapshots/snapshot_builder.js'; import { TreeBase } from '../tree_base.js'; -const log = createDebugLogger('aztec:standard-indexed-tree'); - -/** - * Factory for creating leaf preimages. - */ -export interface PreimageFactory { - /** - * Creates a new preimage from a leaf. - * @param leaf - Leaf to create a preimage from. - * @param nextKey - Next key of the leaf. - * @param nextIndex - Next index of the leaf. - */ - fromLeaf(leaf: IndexedTreeLeaf, nextKey: bigint, nextIndex: bigint): IndexedTreeLeafPreimage; - /** - * Creates a new preimage from a buffer. - * @param buffer - Buffer to create a preimage from. - */ - fromBuffer(buffer: Buffer): IndexedTreeLeafPreimage; - /** - * Creates an empty preimage. - */ - empty(): IndexedTreeLeafPreimage; - /** - * Creates a copy of a preimage. - * @param preimage - Preimage to be cloned. - */ - clone(preimage: IndexedTreeLeafPreimage): IndexedTreeLeafPreimage; -} - export const buildDbKeyForPreimage = (name: string, index: bigint) => { - return `${name}:leaf_by_index:${toBufferBE(index, 32).toString('hex')}`; + return `${name}:leaf_by_index:${toBufferBE(index, 32).toString('hex')}` as const; }; export const buildDbKeyForLeafIndex = (name: string, key: bigint) => { - return `${name}:leaf_index_by_leaf_key:${toBufferBE(key, 32).toString('hex')}`; + return `${name}:leaf_index_by_leaf_key:${toBufferBE(key, 32).toString('hex')}` as const; }; /** @@ -90,11 +55,14 @@ function getEmptyLowLeafWitness( * Standard implementation of an indexed tree. */ export class StandardIndexedTree extends TreeBase implements IndexedTree { - #snapshotBuilder = new IndexedTreeSnapshotBuilder(this.db, this, this.leafPreimageFactory); + #snapshotBuilder = new IndexedTreeSnapshotBuilder(this.store, this, this.leafPreimageFactory); + protected cachedLeafPreimages: { [key: string]: IndexedTreeLeafPreimage } = {}; + protected leaves: AztecMap, Buffer>; + protected leafIndex: AztecMap, bigint>; public constructor( - db: LevelUp, + store: AztecKVStore, hasher: Hasher, name: string, depth: number, @@ -103,7 +71,9 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { protected leafFactory: LeafFactory, root?: Buffer, ) { - super(db, hasher, name, depth, size, root); + super(store, hasher, name, depth, size, root); + this.leaves = store.openMap(`tree_${name}_leaves`); + this.leafIndex = store.openMap(`tree_${name}_leaf_index`); } /** @@ -140,8 +110,8 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { * @param includeUncommitted - Indicates whether to include uncommitted leaves in the computation. * @returns The value of the leaf at the given index or undefined if the leaf is empty. */ - public async getLeafValue(index: bigint, includeUncommitted: boolean): Promise { - const preimage = await this.getLatestLeafPreimageCopy(index, includeUncommitted); + public getLeafValue(index: bigint, includeUncommitted: boolean): Buffer | undefined { + const preimage = this.getLatestLeafPreimageCopy(index, includeUncommitted); return preimage && preimage.toBuffer(); } @@ -151,10 +121,10 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { * @param includeUncommitted - If true, the uncommitted changes are included in the search. * @returns The found leaf index and a flag indicating if the corresponding leaf's value is equal to `newValue`. */ - async findIndexOfPreviousKey( + findIndexOfPreviousKey( newKey: bigint, includeUncommitted: boolean, - ): Promise< + ): | { /** * The index of the found leaf. @@ -165,10 +135,9 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { */ alreadyPresent: boolean; } - | undefined - > { - let lowLeafIndex = await this.getDbLowLeafIndex(newKey); - let lowLeafPreimage = lowLeafIndex !== undefined ? await this.getDbPreimage(lowLeafIndex) : undefined; + | undefined { + let lowLeafIndex = this.getDbLowLeafIndex(newKey); + let lowLeafPreimage = lowLeafIndex !== undefined ? this.getDbPreimage(lowLeafIndex) : undefined; if (includeUncommitted) { const cachedLowLeafIndex = this.getCachedLowLeafIndex(newKey); @@ -213,36 +182,21 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { return undefined; } - private async getDbLowLeafIndex(key: bigint): Promise { - return await new Promise((resolve, reject) => { - let lowLeafIndex: bigint | undefined; - this.db - .createReadStream({ - gte: buildDbKeyForLeafIndex(this.getName(), 0n), - lte: buildDbKeyForLeafIndex(this.getName(), key), - limit: 1, - reverse: true, - }) - .on('data', data => { - lowLeafIndex = toBigIntBE(data.value); - }) - .on('close', function () {}) - .on('end', function () { - resolve(lowLeafIndex); - }) - .on('error', function () { - log.error('stream error'); - reject(); - }); - }); + private getDbLowLeafIndex(key: bigint): bigint | undefined { + const values = Array.from( + this.leafIndex.values({ + end: buildDbKeyForLeafIndex(this.getName(), key), + limit: 1, + reverse: true, + }), + ); + + return values[0]; } - private async getDbPreimage(index: bigint): Promise { - const dbPreimage = await this.db - .get(buildDbKeyForPreimage(this.getName(), index)) - .then(data => this.leafPreimageFactory.fromBuffer(data)) - .catch(() => undefined); - return dbPreimage; + private getDbPreimage(index: bigint): IndexedTreeLeafPreimage | undefined { + const value = this.leaves.get(buildDbKeyForPreimage(this.getName(), index)); + return value ? this.leafPreimageFactory.fromBuffer(value) : undefined; } private getCachedPreimage(index: bigint): IndexedTreeLeafPreimage | undefined { @@ -255,13 +209,10 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { * @param includeUncommitted - If true, the uncommitted changes are included in the search. * @returns A copy of the leaf preimage at the given index or undefined if the leaf was not found. */ - public async getLatestLeafPreimageCopy( - index: bigint, - includeUncommitted: boolean, - ): Promise { + public getLatestLeafPreimageCopy(index: bigint, includeUncommitted: boolean): IndexedTreeLeafPreimage | undefined { const preimage = !includeUncommitted - ? await this.getDbPreimage(index) - : this.getCachedPreimage(index) ?? (await this.getDbPreimage(index)); + ? this.getDbPreimage(index) + : this.getCachedPreimage(index) ?? this.getDbPreimage(index); return preimage && this.leafPreimageFactory.clone(preimage); } @@ -271,17 +222,15 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { * @param includeUncommitted - Indicates whether to include uncommitted data. * @returns The index of the first leaf found with a given value (undefined if not found). */ - public async findLeafIndex(value: Buffer, includeUncommitted: boolean): Promise { + public findLeafIndex(value: Buffer, includeUncommitted: boolean): bigint | undefined { const leaf = this.leafFactory.fromBuffer(value); - let index = await this.db - .get(buildDbKeyForLeafIndex(this.getName(), leaf.getKey())) - .then(data => toBigIntBE(data)) - .catch(() => undefined); + let index = this.leafIndex.get(buildDbKeyForLeafIndex(this.getName(), leaf.getKey())); if (includeUncommitted && index === undefined) { const cachedIndex = this.getCachedLeafIndex(leaf.getKey()); index = cachedIndex; } + return index; } @@ -314,24 +263,24 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { // Make the last leaf point to the first leaf leaves[prefilledSize - 1] = this.leafPreimageFactory.fromLeaf(leaves[prefilledSize - 1].asLeaf(), 0n, 0n); - await this.encodeAndAppendLeaves(leaves, true); + this.encodeAndAppendLeaves(leaves, true); await this.commit(); } /** * Commits all the leaves to the database and removes them from a cache. */ - private async commitLeaves(): Promise { - const batch = this.db.batch(); - const keys = Object.getOwnPropertyNames(this.cachedLeafPreimages); - for (const key of keys) { - const leaf = this.cachedLeafPreimages[key]; - const index = BigInt(key); - batch.put(buildDbKeyForPreimage(this.getName(), index), leaf.toBuffer()); - batch.put(buildDbKeyForLeafIndex(this.getName(), leaf.getKey()), toBufferBE(index, 32)); - } - await batch.write(); - this.clearCachedLeaves(); + private commitLeaves(): Promise { + return this.store.transaction(() => { + const keys = Object.getOwnPropertyNames(this.cachedLeafPreimages); + for (const key of keys) { + const leaf = this.cachedLeafPreimages[key]; + const index = BigInt(key); + void this.leaves.set(buildDbKeyForPreimage(this.getName(), index), leaf.toBuffer()); + void this.leafIndex.set(buildDbKeyForLeafIndex(this.getName(), leaf.getKey()), index); + } + this.clearCachedLeaves(); + }); } /** @@ -346,14 +295,14 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { * @param preimage - New contents of the leaf. * @param index - Index of the leaf to be updated. */ - protected async updateLeaf(preimage: IndexedTreeLeafPreimage, index: bigint) { + protected updateLeaf(preimage: IndexedTreeLeafPreimage, index: bigint) { if (index > this.maxIndex) { throw Error(`Index out of bounds. Index ${index}, max index: ${this.maxIndex}.`); } this.cachedLeafPreimages[index.toString()] = preimage; const encodedLeaf = this.encodeLeaf(preimage, true); - await this.addLeafToCacheAndHashToRoot(encodedLeaf, index); + this.addLeafToCacheAndHashToRoot(encodedLeaf, index); const numLeaves = this.getNumLeaves(true); if (index >= numLeaves) { this.cachedSize = index + 1n; @@ -542,7 +491,7 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { insertedKeys.set(newLeaf.getKey(), true); } - const indexOfPrevious = await this.findIndexOfPreviousKey(newLeaf.getKey(), true); + const indexOfPrevious = this.findIndexOfPreviousKey(newLeaf.getKey(), true); if (indexOfPrevious === undefined) { return { lowLeavesWitnessData: undefined, @@ -555,7 +504,7 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { const isUpdate = indexOfPrevious.alreadyPresent; // get the low leaf (existence checked in getting index) - const lowLeafPreimage = (await this.getLatestLeafPreimageCopy(indexOfPrevious.index, true))!; + const lowLeafPreimage = this.getLatestLeafPreimageCopy(indexOfPrevious.index, true)!; const siblingPath = await this.getSiblingPath(BigInt(indexOfPrevious.index), true); const witness: LowLeafWitnessData = { @@ -576,7 +525,7 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { lowLeafPreimage.getNextIndex(), ); - await this.updateLeaf(newLowLeafPreimage, indexOfPrevious.index); + this.updateLeaf(newLowLeafPreimage, indexOfPrevious.index); pendingInsertionSubtree[originalIndex] = this.leafPreimageFactory.empty(); } else { @@ -586,7 +535,7 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { startInsertionIndex + BigInt(originalIndex), ); - await this.updateLeaf(newLowLeafPreimage, indexOfPrevious.index); + this.updateLeaf(newLowLeafPreimage, indexOfPrevious.index); const currentPendingPreimageLeaf = this.leafPreimageFactory.fromLeaf( newLeaf, @@ -606,7 +555,7 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { // Perform batch insertion of new pending values // Note: In this case we set `hash0Leaf` param to false because batch insertion algorithm use forced null leaf // inclusion. See {@link encodeLeaf} for a more through param explanation. - await this.encodeAndAppendLeaves(pendingInsertionSubtree, false); + this.encodeAndAppendLeaves(pendingInsertionSubtree, false); this.log(`Inserted ${leaves.length} leaves into ${this.getName()} tree`, { eventName: 'tree-insertion', @@ -641,8 +590,8 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { return this.#snapshotBuilder.snapshot(blockNumber); } - getSnapshot(block: number): Promise { - return this.#snapshotBuilder.getSnapshot(block); + getSnapshot(blockNumber: number): Promise { + return this.#snapshotBuilder.getSnapshot(blockNumber); } /** @@ -651,7 +600,7 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { * @param hash0Leaf - Indicates whether 0 value leaf should be hashed. See {@link encodeLeaf}. * @returns Empty promise */ - private async encodeAndAppendLeaves(preimages: IndexedTreeLeafPreimage[], hash0Leaf: boolean): Promise { + private encodeAndAppendLeaves(preimages: IndexedTreeLeafPreimage[], hash0Leaf: boolean): void { const startInsertionIndex = this.getNumLeaves(true); const hashedLeaves = preimages.map((preimage, i) => { @@ -659,7 +608,7 @@ export class StandardIndexedTree extends TreeBase implements IndexedTree { return this.encodeLeaf(preimage, hash0Leaf); }); - await super.appendLeaves(hashedLeaves); + super.appendLeaves(hashedLeaves); } /** diff --git a/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree.test.ts b/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree.test.ts index 65d50a8eb49..1e793daf656 100644 --- a/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree.test.ts +++ b/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree.test.ts @@ -6,34 +6,32 @@ import { PublicDataTreeLeafPreimage, } from '@aztec/circuits.js'; import { toBufferBE } from '@aztec/foundation/bigint-buffer'; +import { AztecKVStore, AztecLmdbStore } from '@aztec/kv-store'; import { Hasher } from '@aztec/types/interfaces'; import { SiblingPath } from '@aztec/types/membership'; -import { default as levelup } from 'levelup'; - import { INITIAL_LEAF, MerkleTree, Pedersen, loadTree, newTree } from '../../index.js'; import { treeTestSuite } from '../../test/test_suite.js'; -import { createMemDown } from '../../test/utils/create_mem_down.js'; import { StandardIndexedTreeWithAppend } from './standard_indexed_tree_with_append.js'; class NullifierTree extends StandardIndexedTreeWithAppend { - constructor(db: levelup.LevelUp, hasher: Hasher, name: string, depth: number, size: bigint = 0n, root?: Buffer) { - super(db, hasher, name, depth, size, NullifierLeafPreimage, NullifierLeaf, root); + constructor(store: AztecKVStore, hasher: Hasher, name: string, depth: number, size: bigint = 0n, root?: Buffer) { + super(store, hasher, name, depth, size, NullifierLeafPreimage, NullifierLeaf, root); } } class PublicDataTree extends StandardIndexedTreeWithAppend { - constructor(db: levelup.LevelUp, hasher: Hasher, name: string, depth: number, size: bigint = 0n, root?: Buffer) { - super(db, hasher, name, depth, size, PublicDataTreeLeafPreimage, PublicDataTreeLeaf, root); + constructor(store: AztecKVStore, hasher: Hasher, name: string, depth: number, size: bigint = 0n, root?: Buffer) { + super(store, hasher, name, depth, size, PublicDataTreeLeafPreimage, PublicDataTreeLeaf, root); } } -const createDb = async (levelUp: levelup.LevelUp, hasher: Hasher, name: string, depth: number, prefilledSize = 1) => { - return await newTree(NullifierTree, levelUp, hasher, name, depth, prefilledSize); +const createDb = async (store: AztecKVStore, hasher: Hasher, name: string, depth: number, prefilledSize = 1) => { + return await newTree(NullifierTree, store, hasher, name, depth, prefilledSize); }; -const createFromName = async (levelUp: levelup.LevelUp, hasher: Hasher, name: string) => { - return await loadTree(NullifierTree, levelUp, hasher, name); +const createFromName = async (store: AztecKVStore, hasher: Hasher, name: string) => { + return await loadTree(NullifierTree, store, hasher, name); }; const createNullifierTreeLeafHashInputs = (value: number, nextIndex: number, nextValue: number) => { @@ -77,7 +75,7 @@ describe('StandardIndexedTreeSpecific', () => { it('produces the correct roots and sibling paths', async () => { // Create a depth-3 indexed merkle tree - const db = levelup(createMemDown()); + const db = await AztecLmdbStore.openTmp(); const tree = await createDb(db, pedersen, 'test', 3); /** @@ -274,8 +272,7 @@ describe('StandardIndexedTreeSpecific', () => { it('Can append empty leaves and handle insertions', async () => { // Create a depth-3 indexed merkle tree - const db = levelup(createMemDown()); - const tree = await createDb(db, pedersen, 'test', 3); + const tree = await createDb(await AztecLmdbStore.openTmp(), pedersen, 'test', 3); /** * Initial state: @@ -492,8 +489,8 @@ describe('StandardIndexedTreeSpecific', () => { const SUBTREE_HEIGHT = 5; // originally from BaseRollupInputs.NULLIFIER_SUBTREE_HEIGHT // Create a depth-3 indexed merkle tree - const appendTree = await createDb(levelup(createMemDown()), pedersen, 'test', TREE_HEIGHT, INITIAL_TREE_SIZE); - const insertTree = await createDb(levelup(createMemDown()), pedersen, 'test', TREE_HEIGHT, INITIAL_TREE_SIZE); + const appendTree = await createDb(await AztecLmdbStore.openTmp(), pedersen, 'test', TREE_HEIGHT, INITIAL_TREE_SIZE); + const insertTree = await createDb(await AztecLmdbStore.openTmp(), pedersen, 'test', TREE_HEIGHT, INITIAL_TREE_SIZE); await appendTree.appendLeaves(leaves); await insertTree.batchInsert(leaves, SUBTREE_HEIGHT); @@ -504,25 +501,25 @@ describe('StandardIndexedTreeSpecific', () => { }); it('should be able to find indexes of leaves', async () => { - const db = levelup(createMemDown()); + const db = await AztecLmdbStore.openTmp(); const tree = await createDb(db, pedersen, 'test', 3); const values = [Buffer.alloc(32, 1), Buffer.alloc(32, 2)]; await tree.appendLeaves([values[0]]); - expect(await tree.findLeafIndex(values[0], true)).toBeDefined(); - expect(await tree.findLeafIndex(values[0], false)).toBe(undefined); - expect(await tree.findLeafIndex(values[1], true)).toBe(undefined); + expect(tree.findLeafIndex(values[0], true)).toBeDefined(); + expect(tree.findLeafIndex(values[0], false)).toBe(undefined); + expect(tree.findLeafIndex(values[1], true)).toBe(undefined); await tree.commit(); - expect(await tree.findLeafIndex(values[0], false)).toBeDefined(); + expect(tree.findLeafIndex(values[0], false)).toBeDefined(); }); describe('Updatable leaves', () => { it('should be able to upsert leaves', async () => { // Create a depth-3 indexed merkle tree - const db = levelup(createMemDown()); + const db = await AztecLmdbStore.openTmp(); const tree = await newTree(PublicDataTree, db, pedersen, 'test', 3, 1); /** @@ -632,7 +629,7 @@ describe('StandardIndexedTreeSpecific', () => { const INITIAL_TREE_SIZE = 8; const SUBTREE_HEIGHT = 5; - const db = levelup(createMemDown()); + const db = await AztecLmdbStore.openTmp(); const appendTree = await newTree(PublicDataTree, db, pedersen, 'test', TREE_HEIGHT, INITIAL_TREE_SIZE); const insertTree = await newTree(PublicDataTree, db, pedersen, 'test', TREE_HEIGHT, INITIAL_TREE_SIZE); diff --git a/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree_with_append.ts b/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree_with_append.ts index 3fd3a123084..60ff4a9ecca 100644 --- a/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree_with_append.ts +++ b/yarn-project/merkle-tree/src/standard_indexed_tree/test/standard_indexed_tree_with_append.ts @@ -12,10 +12,12 @@ export class StandardIndexedTreeWithAppend extends StandardIndexedTree { * @returns Empty promise. * @remarks This method is inefficient and is here mostly for testing. Use batchInsert instead. */ - public async appendLeaves(leaves: Buffer[]): Promise { + public appendLeaves(leaves: Buffer[]): Promise { for (const leaf of leaves) { - await this.appendLeaf(leaf); + this.appendLeaf(leaf); } + + return Promise.resolve(); } private appendEmptyLeaf() { @@ -31,7 +33,7 @@ export class StandardIndexedTreeWithAppend extends StandardIndexedTree { * @param leaf - The leaf to append. * @returns Empty promise. */ - private async appendLeaf(leaf: Buffer): Promise { + private appendLeaf(leaf: Buffer): void { const newLeaf = this.leafFactory.fromBuffer(leaf); // Special case when appending zero @@ -40,13 +42,13 @@ export class StandardIndexedTreeWithAppend extends StandardIndexedTree { return; } - const lowLeafIndex = await this.findIndexOfPreviousKey(newLeaf.getKey(), true); + const lowLeafIndex = this.findIndexOfPreviousKey(newLeaf.getKey(), true); if (lowLeafIndex === undefined) { throw new Error(`Previous leaf not found!`); } const isUpdate = lowLeafIndex.alreadyPresent; - const lowLeafPreimage = (await this.getLatestLeafPreimageCopy(lowLeafIndex.index, true))!; + const lowLeafPreimage = this.getLatestLeafPreimageCopy(lowLeafIndex.index, true)!; const currentSize = this.getNumLeaves(true); if (isUpdate) { @@ -57,7 +59,7 @@ export class StandardIndexedTreeWithAppend extends StandardIndexedTree { lowLeafPreimage.getNextIndex(), ); - await this.updateLeaf(newLowLeafPreimage, BigInt(lowLeafIndex.index)); + this.updateLeaf(newLowLeafPreimage, BigInt(lowLeafIndex.index)); this.appendEmptyLeaf(); } else { const newLeafPreimage = this.leafPreimageFactory.fromLeaf( @@ -72,8 +74,8 @@ export class StandardIndexedTreeWithAppend extends StandardIndexedTree { newLeaf.getKey(), BigInt(currentSize), ); - await this.updateLeaf(newLowLeafPreimage, BigInt(lowLeafIndex.index)); - await this.updateLeaf(newLeafPreimage, currentSize); + this.updateLeaf(newLowLeafPreimage, BigInt(lowLeafIndex.index)); + this.updateLeaf(newLeafPreimage, currentSize); } } } diff --git a/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts b/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts index 34d8cc3b514..7da886753aa 100644 --- a/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts +++ b/yarn-project/merkle-tree/src/standard_tree/standard_tree.test.ts @@ -1,23 +1,21 @@ import { randomBytes } from '@aztec/foundation/crypto'; +import { AztecKVStore, AztecLmdbStore } from '@aztec/kv-store'; import { Hasher } from '@aztec/types/interfaces'; -import { default as levelup } from 'levelup'; - import { loadTree } from '../load_tree.js'; import { newTree } from '../new_tree.js'; import { standardBasedTreeTestSuite } from '../test/standard_based_test_suite.js'; import { treeTestSuite } from '../test/test_suite.js'; -import { createMemDown } from '../test/utils/create_mem_down.js'; import { PedersenWithCounter } from '../test/utils/pedersen_with_counter.js'; import { INITIAL_LEAF } from '../tree_base.js'; import { StandardTree } from './standard_tree.js'; -const createDb = async (levelUp: levelup.LevelUp, hasher: Hasher, name: string, depth: number) => { - return await newTree(StandardTree, levelUp, hasher, name, depth); +const createDb = async (store: AztecKVStore, hasher: Hasher, name: string, depth: number) => { + return await newTree(StandardTree, store, hasher, name, depth); }; -const createFromName = async (levelUp: levelup.LevelUp, hasher: Hasher, name: string) => { - return await loadTree(StandardTree, levelUp, hasher, name); +const createFromName = async (store: AztecKVStore, hasher: Hasher, name: string) => { + return await loadTree(StandardTree, store, hasher, name); }; treeTestSuite('StandardTree', createDb, createFromName); @@ -35,7 +33,7 @@ describe('StandardTree_batchAppend', () => { }); it('correctly computes root when batch appending and calls hash function expected num times', async () => { - const db = levelup(createMemDown()); + const db = await AztecLmdbStore.openTmp(); const tree = await createDb(db, pedersen, 'test', 3); const leaves = Array.from({ length: 5 }, _ => randomBytes(32)); @@ -71,18 +69,18 @@ describe('StandardTree_batchAppend', () => { }); it('should be able to find indexes of leaves', async () => { - const db = levelup(createMemDown()); + const db = await AztecLmdbStore.openTmp(); const tree = await createDb(db, pedersen, 'test', 3); const values = [Buffer.alloc(32, 1), Buffer.alloc(32, 2)]; await tree.appendLeaves([values[0]]); - expect(await tree.findLeafIndex(values[0], true)).toBeDefined(); - expect(await tree.findLeafIndex(values[0], false)).toBe(undefined); - expect(await tree.findLeafIndex(values[1], true)).toBe(undefined); + expect(tree.findLeafIndex(values[0], true)).toBeDefined(); + expect(tree.findLeafIndex(values[0], false)).toBe(undefined); + expect(tree.findLeafIndex(values[1], true)).toBe(undefined); await tree.commit(); - expect(await tree.findLeafIndex(values[0], false)).toBeDefined(); + expect(tree.findLeafIndex(values[0], false)).toBeDefined(); }); }); diff --git a/yarn-project/merkle-tree/src/standard_tree/standard_tree.ts b/yarn-project/merkle-tree/src/standard_tree/standard_tree.ts index cc587ee3a3d..c1dc29cda69 100644 --- a/yarn-project/merkle-tree/src/standard_tree/standard_tree.ts +++ b/yarn-project/merkle-tree/src/standard_tree/standard_tree.ts @@ -1,25 +1,26 @@ import { TreeInsertionStats } from '@aztec/circuit-types/stats'; import { Timer } from '@aztec/foundation/timer'; -import { AppendOnlySnapshotBuilder, TreeSnapshot } from '../index.js'; import { AppendOnlyTree } from '../interfaces/append_only_tree.js'; +import { AppendOnlySnapshotBuilder } from '../snapshots/append_only_snapshot.js'; +import { TreeSnapshot } from '../snapshots/snapshot_builder.js'; import { TreeBase } from '../tree_base.js'; /** * A Merkle tree implementation that uses a LevelDB database to store the tree. */ export class StandardTree extends TreeBase implements AppendOnlyTree { - #snapshotBuilder = new AppendOnlySnapshotBuilder(this.db, this, this.hasher); + #snapshotBuilder = new AppendOnlySnapshotBuilder(this.store, this, this.hasher); /** * Appends the given leaves to the tree. * @param leaves - The leaves to append. * @returns Empty promise. */ - public async appendLeaves(leaves: Buffer[]): Promise { + public appendLeaves(leaves: Buffer[]): Promise { this.hasher.reset(); const timer = new Timer(); - await super.appendLeaves(leaves); + super.appendLeaves(leaves); this.log(`Inserted ${leaves.length} leaves into ${this.getName()} tree`, { eventName: 'tree-insertion', duration: timer.ms(), @@ -29,19 +30,21 @@ export class StandardTree extends TreeBase implements AppendOnlyTree { treeType: 'append-only', ...this.hasher.stats(), } satisfies TreeInsertionStats); + + return Promise.resolve(); } - public snapshot(block: number): Promise { - return this.#snapshotBuilder.snapshot(block); + public snapshot(blockNumber: number): Promise { + return this.#snapshotBuilder.snapshot(blockNumber); } - public getSnapshot(block: number): Promise { - return this.#snapshotBuilder.getSnapshot(block); + public getSnapshot(blockNumber: number): Promise { + return this.#snapshotBuilder.getSnapshot(blockNumber); } - public async findLeafIndex(value: Buffer, includeUncommitted: boolean): Promise { + public findLeafIndex(value: Buffer, includeUncommitted: boolean): bigint | undefined { for (let i = 0n; i < this.getNumLeaves(includeUncommitted); i++) { - const currentValue = await this.getLeafValue(i, includeUncommitted); + const currentValue = this.getLeafValue(i, includeUncommitted); if (currentValue && currentValue.equals(value)) { return i; } diff --git a/yarn-project/merkle-tree/src/test/standard_based_test_suite.ts b/yarn-project/merkle-tree/src/test/standard_based_test_suite.ts index 29881831ee2..bfd95e7b349 100644 --- a/yarn-project/merkle-tree/src/test/standard_based_test_suite.ts +++ b/yarn-project/merkle-tree/src/test/standard_based_test_suite.ts @@ -1,21 +1,20 @@ +import { AztecKVStore, AztecLmdbStore } from '@aztec/kv-store'; import { Hasher } from '@aztec/types/interfaces'; import { SiblingPath } from '@aztec/types/membership'; import { randomBytes } from 'crypto'; -import { default as levelup } from 'levelup'; import { INITIAL_LEAF, Pedersen } from '../index.js'; import { AppendOnlyTree } from '../interfaces/append_only_tree.js'; import { UpdateOnlyTree } from '../interfaces/update_only_tree.js'; import { appendLeaves } from './utils/append_leaves.js'; -import { createMemDown } from './utils/create_mem_down.js'; const TEST_TREE_DEPTH = 2; export const standardBasedTreeTestSuite = ( testName: string, createDb: ( - levelup: levelup.LevelUp, + store: AztecKVStore, hasher: Hasher, name: string, depth: number, @@ -36,21 +35,21 @@ export const standardBasedTreeTestSuite = ( }); it('should have correct empty tree root for depth 32', async () => { - const db = levelup(createMemDown()); + const db = await AztecLmdbStore.openTmp(); const tree = await createDb(db, pedersen, 'test', 32); const root = tree.getRoot(false); expect(root.toString('hex')).toEqual('16642d9ccd8346c403aa4c3fa451178b22534a27035cdaa6ec34ae53b29c50cb'); }); it('should throw when appending beyond max index', async () => { - const db = levelup(createMemDown()); + const db = await AztecLmdbStore.openTmp(); const tree = await createDb(db, pedersen, 'test', 2); const leaves = Array.from({ length: 5 }, _ => randomBytes(32)); await expect(appendLeaves(tree, leaves)).rejects.toThrow(); }); it('should have correct root and sibling paths', async () => { - const db = levelup(createMemDown()); + const db = await AztecLmdbStore.openTmp(); const tree = await createDb(db, pedersen, 'test', 2); const level1ZeroHash = pedersen.hash(INITIAL_LEAF, INITIAL_LEAF); diff --git a/yarn-project/merkle-tree/src/test/test_suite.ts b/yarn-project/merkle-tree/src/test/test_suite.ts index 31ee72a9b0e..fd2d5b038fb 100644 --- a/yarn-project/merkle-tree/src/test/test_suite.ts +++ b/yarn-project/merkle-tree/src/test/test_suite.ts @@ -1,13 +1,11 @@ +import { AztecKVStore, AztecLmdbStore } from '@aztec/kv-store'; import { Hasher } from '@aztec/types/interfaces'; import { SiblingPath } from '@aztec/types/membership'; -import { default as levelup } from 'levelup'; - import { Pedersen } from '../index.js'; import { AppendOnlyTree } from '../interfaces/append_only_tree.js'; import { UpdateOnlyTree } from '../interfaces/update_only_tree.js'; import { appendLeaves } from './utils/append_leaves.js'; -import { createMemDown } from './utils/create_mem_down.js'; const expectSameTrees = async ( tree1: AppendOnlyTree | UpdateOnlyTree, @@ -28,12 +26,12 @@ const expectSameTrees = async ( export const treeTestSuite = ( testName: string, createDb: ( - levelup: levelup.LevelUp, + store: AztecKVStore, hasher: Hasher, name: string, depth: number, ) => Promise, - createFromName: (levelup: levelup.LevelUp, hasher: Hasher, name: string) => Promise, + createFromName: (store: AztecKVStore, hasher: Hasher, name: string) => Promise, ) => { describe(testName, () => { const values: Buffer[] = []; @@ -52,12 +50,10 @@ export const treeTestSuite = ( }); it('should revert changes on rollback', async () => { - const levelDownEmpty = createMemDown(); - const dbEmpty = levelup(levelDownEmpty); + const dbEmpty = await AztecLmdbStore.openTmp(); const emptyTree = await createDb(dbEmpty, pedersen, 'test', 10); - const levelDown = createMemDown(); - const db = levelup(levelDown); + const db = await AztecLmdbStore.openTmp(); const tree = await createDb(db, pedersen, 'test2', 10); await appendLeaves(tree, values.slice(0, 4)); @@ -89,12 +85,10 @@ export const treeTestSuite = ( }); it('should not revert changes after commit', async () => { - const levelDownEmpty = createMemDown(); - const dbEmpty = levelup(levelDownEmpty); + const dbEmpty = await AztecLmdbStore.openTmp(); const emptyTree = await createDb(dbEmpty, pedersen, 'test', 10); - const levelDown = createMemDown(); - const db = levelup(levelDown); + const db = await AztecLmdbStore.openTmp(); const tree = await createDb(db, pedersen, 'test2', 10); await appendLeaves(tree, values.slice(0, 4)); @@ -110,14 +104,12 @@ export const treeTestSuite = ( }); it('should be able to restore from previous committed data', async () => { - const levelDown = createMemDown(); - const db = levelup(levelDown); + const db = await AztecLmdbStore.openTmp(); const tree = await createDb(db, pedersen, 'test', 10); await appendLeaves(tree, values.slice(0, 4)); await tree.commit(); - const db2 = levelup(levelDown); - const tree2 = await createFromName(db2, pedersen, 'test'); + const tree2 = await createFromName(db, pedersen, 'test'); // both committed and uncommitted should be equal to the restored data expect(tree.getRoot(true)).toEqual(tree2.getRoot(true)); @@ -129,7 +121,7 @@ export const treeTestSuite = ( }); it('should throw an error if previous data does not exist for the given name', async () => { - const db = levelup(createMemDown()); + const db = await AztecLmdbStore.openTmp(); await expect( (async () => { await createFromName(db, pedersen, 'a_whole_new_tree'); @@ -138,7 +130,7 @@ export const treeTestSuite = ( }); it('should serialize sibling path data to a buffer and be able to deserialize it back', async () => { - const db = levelup(createMemDown()); + const db = await AztecLmdbStore.openTmp(); const tree = await createDb(db, pedersen, 'test', 10); await appendLeaves(tree, values.slice(0, 1)); diff --git a/yarn-project/merkle-tree/src/test/utils/create_mem_down.ts b/yarn-project/merkle-tree/src/test/utils/create_mem_down.ts deleted file mode 100644 index 0be5ac35604..00000000000 --- a/yarn-project/merkle-tree/src/test/utils/create_mem_down.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { type MemDown, default as memdown } from 'memdown'; - -export const createMemDown = () => (memdown as any)() as MemDown; diff --git a/yarn-project/merkle-tree/src/tree_base.ts b/yarn-project/merkle-tree/src/tree_base.ts index 36f9b12b444..5a24ec81280 100644 --- a/yarn-project/merkle-tree/src/tree_base.ts +++ b/yarn-project/merkle-tree/src/tree_base.ts @@ -1,10 +1,9 @@ import { toBigIntLE, toBufferLE } from '@aztec/foundation/bigint-buffer'; import { DebugLogger, createDebugLogger } from '@aztec/foundation/log'; +import { AztecKVStore, AztecMap, AztecSingleton } from '@aztec/kv-store'; import { Hasher } from '@aztec/types/interfaces'; import { SiblingPath } from '@aztec/types/membership'; -import { LevelUp, LevelUpChain } from 'levelup'; - import { HasherWithStats } from './hasher_with_stats.js'; import { MerkleTree } from './interfaces/merkle_tree.js'; @@ -17,7 +16,7 @@ const encodeMeta = (root: Buffer, depth: number, size: bigint) => { data.writeUInt32LE(depth, 32); return Buffer.concat([data, toBufferLE(size, 32)]); }; -export const decodeMeta = (meta: Buffer) => { +const decodeMeta = (meta: Buffer) => { const root = meta.subarray(0, 32); const depth = meta.readUInt32LE(32); const size = toBigIntLE(meta.subarray(36)); @@ -28,6 +27,18 @@ export const decodeMeta = (meta: Buffer) => { }; }; +const openTreeMetaSingleton = (store: AztecKVStore, treeName: string): AztecSingleton => + store.openSingleton(`merkle_tree_${treeName}_meta`); + +export const getTreeMeta = (store: AztecKVStore, treeName: string) => { + const singleton = openTreeMetaSingleton(store, treeName); + const val = singleton.get(); + if (!val) { + throw new Error(); + } + return decodeMeta(val); +}; + export const INITIAL_LEAF = Buffer.from('0000000000000000000000000000000000000000000000000000000000000000', 'hex'); /** @@ -40,11 +51,13 @@ export abstract class TreeBase implements MerkleTree { private zeroHashes: Buffer[] = []; private cache: { [key: string]: Buffer } = {}; protected log: DebugLogger; - protected hasher: HasherWithStats; + private nodes: AztecMap; + private meta: AztecSingleton; + public constructor( - protected db: LevelUp, + protected store: AztecKVStore, hasher: Hasher, private name: string, private depth: number, @@ -56,6 +69,8 @@ export abstract class TreeBase implements MerkleTree { } this.hasher = new HasherWithStats(hasher); + this.nodes = store.openMap('merkle_tree_' + name); + this.meta = openTreeMetaSingleton(store, name); // Compute the zero values at each layer. let current = INITIAL_LEAF; @@ -111,34 +126,36 @@ export abstract class TreeBase implements MerkleTree { * @returns A sibling path for the element at the given index. * Note: The sibling path is an array of sibling hashes, with the lowest hash (leaf hash) first, and the highest hash last. */ - public async getSiblingPath(index: bigint, includeUncommitted: boolean): Promise> { + public getSiblingPath(index: bigint, includeUncommitted: boolean): Promise> { const path: Buffer[] = []; let level = this.depth; while (level > 0) { const isRight = index & 0x01n; - const sibling = await this.getLatestValueAtIndex(level, isRight ? index - 1n : index + 1n, includeUncommitted); + const sibling = this.getLatestValueAtIndex(level, isRight ? index - 1n : index + 1n, includeUncommitted); path.push(sibling); level -= 1; index >>= 1n; } - return new SiblingPath(this.depth as N, path); + return Promise.resolve(new SiblingPath(this.depth as N, path)); } /** * Commits the changes to the database. * @returns Empty promise. */ - public async commit(): Promise { - const batch = this.db.batch(); - const keys = Object.getOwnPropertyNames(this.cache); - for (const key of keys) { - batch.put(key, this.cache[key]); - } - this.size = this.getNumLeaves(true); - this.root = this.getRoot(true); - await this.writeMeta(batch); - await batch.write(); - this.clearCache(); + public commit(): Promise { + return this.store.transaction(() => { + const keys = Object.getOwnPropertyNames(this.cache); + for (const key of keys) { + void this.nodes.set(key, this.cache[key]); + } + this.size = this.getNumLeaves(true); + this.root = this.getRoot(true); + + this.clearCache(); + + void this.writeMeta(); + }); } /** @@ -156,11 +173,11 @@ export abstract class TreeBase implements MerkleTree { * @param includeUncommitted - Indicates whether to include uncommitted changes. * @returns Leaf value at the given index or undefined. */ - public getLeafValue(index: bigint, includeUncommitted: boolean): Promise { + public getLeafValue(index: bigint, includeUncommitted: boolean): Buffer | undefined { return this.getLatestValueAtIndex(this.depth, index, includeUncommitted); } - public getNode(level: number, index: bigint): Promise { + public getNode(level: number, index: bigint): Buffer | undefined { if (level < 0 || level > this.depth) { throw Error('Invalid level: ' + level); } @@ -193,14 +210,14 @@ export abstract class TreeBase implements MerkleTree { * @param leaf - Leaf to add to cache. * @param index - Index of the leaf (used to derive the cache key). */ - protected async addLeafToCacheAndHashToRoot(leaf: Buffer, index: bigint) { + protected addLeafToCacheAndHashToRoot(leaf: Buffer, index: bigint) { const key = indexToKeyHash(this.name, this.depth, index); let current = leaf; this.cache[key] = current; let level = this.depth; while (level > 0) { const isRight = index & 0x01n; - const sibling = await this.getLatestValueAtIndex(level, isRight ? index - 1n : index + 1n, true); + const sibling = this.getLatestValueAtIndex(level, isRight ? index - 1n : index + 1n, true); const lhs = isRight ? sibling : current; const rhs = isRight ? current : sibling; current = this.hasher.hash(lhs, rhs); @@ -219,12 +236,12 @@ export abstract class TreeBase implements MerkleTree { * @returns The latest value at the given index. * Note: If the value is not in the cache, it will be fetched from the database. */ - private async getLatestValueAtIndex(level: number, index: bigint, includeUncommitted: boolean): Promise { + private getLatestValueAtIndex(level: number, index: bigint, includeUncommitted: boolean): Buffer { const key = indexToKeyHash(this.name, level, index); if (includeUncommitted && this.cache[key] !== undefined) { return this.cache[key]; } - const committed = await this.dbGet(key); + const committed = this.dbGet(key); if (committed !== undefined) { return committed; } @@ -236,8 +253,8 @@ export abstract class TreeBase implements MerkleTree { * @param key - The key to by which to get the value. * @returns A value from the db based on the key. */ - private async dbGet(key: string): Promise { - return await this.db.get(key).catch(() => {}); + private dbGet(key: string): Buffer | undefined { + return this.nodes.get(key); } /** @@ -255,13 +272,9 @@ export abstract class TreeBase implements MerkleTree { * Writes meta data to the provided batch. * @param batch - The batch to which to write the meta data. */ - protected async writeMeta(batch?: LevelUpChain) { + protected writeMeta() { const data = encodeMeta(this.getRoot(true), this.depth, this.getNumLeaves(true)); - if (batch) { - batch.put(this.name, data); - } else { - await this.db.put(this.name, data); - } + return this.meta.set(data); } /** @@ -279,7 +292,7 @@ export abstract class TreeBase implements MerkleTree { * `getLatestValueAtIndex` will return a value from cache (because at least one of the 2 children was * touched in previous iteration). */ - protected async appendLeaves(leaves: Buffer[]): Promise { + protected appendLeaves(leaves: Buffer[]): void { const numLeaves = this.getNumLeaves(true); if (numLeaves + BigInt(leaves.length) - 1n > this.maxIndex) { throw Error(`Can't append beyond max index. Max index: ${this.maxIndex}`); @@ -300,8 +313,8 @@ export abstract class TreeBase implements MerkleTree { lastIndex >>= 1n; // 3.Iterate over all the affected nodes at this level and update them for (let index = firstIndex; index <= lastIndex; index++) { - const lhs = await this.getLatestValueAtIndex(level, index * 2n, true); - const rhs = await this.getLatestValueAtIndex(level, index * 2n + 1n, true); + const lhs = this.getLatestValueAtIndex(level, index * 2n, true); + const rhs = this.getLatestValueAtIndex(level, index * 2n + 1n, true); const cacheKey = indexToKeyHash(this.name, level - 1, index); this.cache[cacheKey] = this.hasher.hash(lhs, rhs); } @@ -317,5 +330,5 @@ export abstract class TreeBase implements MerkleTree { * @param includeUncommitted - Indicates whether to include uncommitted data. * @returns The index of the first leaf found with a given value (undefined if not found). */ - abstract findLeafIndex(value: Buffer, includeUncommitted: boolean): Promise; + abstract findLeafIndex(value: Buffer, includeUncommitted: boolean): bigint | undefined; } diff --git a/yarn-project/merkle-tree/tsconfig.json b/yarn-project/merkle-tree/tsconfig.json index 413464d606d..631083e52f4 100644 --- a/yarn-project/merkle-tree/tsconfig.json +++ b/yarn-project/merkle-tree/tsconfig.json @@ -12,6 +12,9 @@ { "path": "../foundation" }, + { + "path": "../kv-store" + }, { "path": "../types" }, diff --git a/yarn-project/noir-protocol-circuits/package.json b/yarn-project/noir-protocol-circuits/package.json index 358e89ccc14..9d1845d8ee8 100644 --- a/yarn-project/noir-protocol-circuits/package.json +++ b/yarn-project/noir-protocol-circuits/package.json @@ -39,6 +39,7 @@ }, "devDependencies": { "@aztec/circuit-types": "workspace:^", + "@aztec/kv-store": "workspace:^", "@aztec/merkle-tree": "workspace:^", "@jest/globals": "^29.5.0", "@types/jest": "^29.5.0", diff --git a/yarn-project/noir-protocol-circuits/src/noir_test_gen.test.ts b/yarn-project/noir-protocol-circuits/src/noir_test_gen.test.ts index 149a6db9575..765ab453388 100644 --- a/yarn-project/noir-protocol-circuits/src/noir_test_gen.test.ts +++ b/yarn-project/noir-protocol-circuits/src/noir_test_gen.test.ts @@ -16,11 +16,9 @@ import { computeFunctionTreeRoot, } from '@aztec/circuits.js/abis'; import { Fr } from '@aztec/foundation/fields'; +import { AztecLmdbStore } from '@aztec/kv-store'; import { Pedersen, StandardTree } from '@aztec/merkle-tree'; -import { default as levelup } from 'levelup'; -import memdown from 'memdown'; - describe('Data generation for noir tests', () => { const defaultContract = { address: AztecAddress.fromField(new Fr(12345)), @@ -73,7 +71,7 @@ describe('Data generation for noir tests', () => { return contractLeaf.toBuffer(); }); - const db = levelup((memdown as any)()); + const db = await AztecLmdbStore.openTmp(); const tree = new StandardTree( db, new Pedersen(), @@ -94,7 +92,7 @@ describe('Data generation for noir tests', () => { const indexes = new Array(128).fill(null).map((_, i) => BigInt(i)); const leaves = indexes.map(i => new Fr(i + 1n).toBuffer()); - const db = levelup((memdown as any)()); + const db = await AztecLmdbStore.openTmp(); const noteHashTree = new StandardTree( db, diff --git a/yarn-project/noir-protocol-circuits/tsconfig.json b/yarn-project/noir-protocol-circuits/tsconfig.json index 917d509e475..a2e4cd63239 100644 --- a/yarn-project/noir-protocol-circuits/tsconfig.json +++ b/yarn-project/noir-protocol-circuits/tsconfig.json @@ -21,6 +21,9 @@ { "path": "../circuit-types" }, + { + "path": "../kv-store" + }, { "path": "../merkle-tree" } diff --git a/yarn-project/p2p/src/client/p2p_client.test.ts b/yarn-project/p2p/src/client/p2p_client.test.ts index 4de4f519f13..0ca27e09752 100644 --- a/yarn-project/p2p/src/client/p2p_client.test.ts +++ b/yarn-project/p2p/src/client/p2p_client.test.ts @@ -1,5 +1,4 @@ import { L2BlockSource, mockTx } from '@aztec/circuit-types'; -import { EthAddress } from '@aztec/circuits.js'; import { AztecKVStore, AztecLmdbStore } from '@aztec/kv-store'; import { expect, jest } from '@jest/globals'; @@ -42,7 +41,7 @@ describe('In-Memory P2P Client', () => { blockSource = new MockBlockSource(); - kvStore = await AztecLmdbStore.create(EthAddress.random()); + kvStore = await AztecLmdbStore.openTmp(); client = new P2PClient(kvStore, blockSource, txPool, p2pService); }); diff --git a/yarn-project/p2p/src/client/p2p_client.ts b/yarn-project/p2p/src/client/p2p_client.ts index 6c9789259ac..3d85307eb95 100644 --- a/yarn-project/p2p/src/client/p2p_client.ts +++ b/yarn-project/p2p/src/client/p2p_client.ts @@ -134,7 +134,7 @@ export class P2PClient implements P2P { ) { const { p2pBlockCheckIntervalMS: checkInterval, l2QueueSize } = getP2PConfigEnvVars(); this.blockDownloader = new L2BlockDownloader(l2BlockSource, l2QueueSize, checkInterval); - this.synchedBlockNumber = store.createSingleton('p2p_pool_last_l2_block'); + this.synchedBlockNumber = store.openSingleton('p2p_pool_last_l2_block'); } /** diff --git a/yarn-project/p2p/src/tx_pool/aztec_kv_tx_pool.test.ts b/yarn-project/p2p/src/tx_pool/aztec_kv_tx_pool.test.ts index 126779f9dd5..fe030abe6d7 100644 --- a/yarn-project/p2p/src/tx_pool/aztec_kv_tx_pool.test.ts +++ b/yarn-project/p2p/src/tx_pool/aztec_kv_tx_pool.test.ts @@ -1,4 +1,3 @@ -import { EthAddress } from '@aztec/circuits.js'; import { AztecLmdbStore } from '@aztec/kv-store'; import { AztecKVTxPool } from './aztec_kv_tx_pool.js'; @@ -7,7 +6,7 @@ import { describeTxPool } from './tx_pool_test_suite.js'; describe('In-Memory TX pool', () => { let txPool: AztecKVTxPool; beforeEach(async () => { - txPool = new AztecKVTxPool(await AztecLmdbStore.create(EthAddress.random())); + txPool = new AztecKVTxPool(await AztecLmdbStore.openTmp()); }); describeTxPool(() => txPool); diff --git a/yarn-project/p2p/src/tx_pool/aztec_kv_tx_pool.ts b/yarn-project/p2p/src/tx_pool/aztec_kv_tx_pool.ts index 1d8d723ab5d..fd1eba930e2 100644 --- a/yarn-project/p2p/src/tx_pool/aztec_kv_tx_pool.ts +++ b/yarn-project/p2p/src/tx_pool/aztec_kv_tx_pool.ts @@ -24,7 +24,7 @@ export class AztecKVTxPool implements TxPool { * @param log - A logger. */ constructor(store: AztecKVStore, log = createDebugLogger('aztec:tx_pool')) { - this.#txs = store.createMap('txs'); + this.#txs = store.openMap('txs'); this.#store = store; this.#log = log; } diff --git a/yarn-project/pxe/src/database/kv_pxe_database.test.ts b/yarn-project/pxe/src/database/kv_pxe_database.test.ts index a9054af0719..27e0da25f37 100644 --- a/yarn-project/pxe/src/database/kv_pxe_database.test.ts +++ b/yarn-project/pxe/src/database/kv_pxe_database.test.ts @@ -1,4 +1,3 @@ -import { EthAddress } from '@aztec/circuits.js'; import { AztecLmdbStore } from '@aztec/kv-store'; import { KVPxeDatabase } from './kv_pxe_database.js'; @@ -8,7 +7,7 @@ describe('KVPxeDatabase', () => { let database: KVPxeDatabase; beforeEach(async () => { - database = new KVPxeDatabase(await AztecLmdbStore.create(EthAddress.random())); + database = new KVPxeDatabase(await AztecLmdbStore.openTmp()); }); describePxeDatabase(() => database); diff --git a/yarn-project/pxe/src/database/kv_pxe_database.ts b/yarn-project/pxe/src/database/kv_pxe_database.ts index 2b1956efa98..fd32c595c34 100644 --- a/yarn-project/pxe/src/database/kv_pxe_database.ts +++ b/yarn-project/pxe/src/database/kv_pxe_database.ts @@ -46,28 +46,29 @@ export class KVPxeDatabase implements PxeDatabase { constructor(db: AztecKVStore) { this.#db = db; - this.#addresses = db.createArray('addresses'); - this.#addressIndex = db.createMap('address_index'); + this.#addresses = db.openArray('addresses'); + this.#addressIndex = db.openMap('address_index'); - this.#authWitnesses = db.createMap('auth_witnesses'); - this.#capsules = db.createArray('capsules'); - this.#contracts = db.createMap('contracts'); - this.#contractArtifacts = db.createMap('contract_artifacts'); - this.#contractInstances = db.createMap('contracts_instances'); + this.#authWitnesses = db.openMap('auth_witnesses'); + this.#capsules = db.openArray('capsules'); + this.#contracts = db.openMap('contracts'); - this.#synchronizedBlock = db.createSingleton('block_header'); - this.#syncedBlockPerPublicKey = db.createMap('synced_block_per_public_key'); + this.#contractArtifacts = db.openMap('contract_artifacts'); + this.#contractInstances = db.openMap('contracts_instances'); - this.#notes = db.createArray('notes'); - this.#nullifiedNotes = db.createMap('nullified_notes'); + this.#synchronizedBlock = db.openSingleton('block_header'); + this.#syncedBlockPerPublicKey = db.openMap('synced_block_per_public_key'); - this.#notesByContract = db.createMultiMap('notes_by_contract'); - this.#notesByStorageSlot = db.createMultiMap('notes_by_storage_slot'); - this.#notesByTxHash = db.createMultiMap('notes_by_tx_hash'); - this.#notesByOwner = db.createMultiMap('notes_by_owner'); + this.#notes = db.openArray('notes'); + this.#nullifiedNotes = db.openMap('nullified_notes'); - this.#deferredNotes = db.createArray('deferred_notes'); - this.#deferredNotesByContract = db.createMultiMap('deferred_notes_by_contract'); + this.#notesByContract = db.openMultiMap('notes_by_contract'); + this.#notesByStorageSlot = db.openMultiMap('notes_by_storage_slot'); + this.#notesByTxHash = db.openMultiMap('notes_by_tx_hash'); + this.#notesByOwner = db.openMultiMap('notes_by_owner'); + + this.#deferredNotes = db.openArray('deferred_notes'); + this.#deferredNotesByContract = db.openMultiMap('deferred_notes_by_contract'); } public async addContractArtifact(id: Fr, contract: ContractArtifact): Promise { diff --git a/yarn-project/pxe/src/note_processor/note_processor.test.ts b/yarn-project/pxe/src/note_processor/note_processor.test.ts index 64d91fb4dcc..0b97ebf0326 100644 --- a/yarn-project/pxe/src/note_processor/note_processor.test.ts +++ b/yarn-project/pxe/src/note_processor/note_processor.test.ts @@ -12,7 +12,7 @@ import { Note, TxL2Logs, } from '@aztec/circuit-types'; -import { EthAddress, Fr, MAX_NEW_COMMITMENTS_PER_TX } from '@aztec/circuits.js'; +import { Fr, MAX_NEW_COMMITMENTS_PER_TX } from '@aztec/circuits.js'; import { Grumpkin } from '@aztec/circuits.js/barretenberg'; import { pedersenHash } from '@aztec/foundation/crypto'; import { Point } from '@aztec/foundation/fields'; @@ -119,7 +119,7 @@ describe('Note Processor', () => { }); beforeEach(async () => { - database = new KVPxeDatabase(await AztecLmdbStore.create(EthAddress.random())); + database = new KVPxeDatabase(await AztecLmdbStore.openTmp()); addNotesSpy = jest.spyOn(database, 'addNotes'); aztecNode = mock(); diff --git a/yarn-project/pxe/src/pxe_service/create_pxe_service.ts b/yarn-project/pxe/src/pxe_service/create_pxe_service.ts index 2b52e0dfc3b..00d800f1e7a 100644 --- a/yarn-project/pxe/src/pxe_service/create_pxe_service.ts +++ b/yarn-project/pxe/src/pxe_service/create_pxe_service.ts @@ -35,11 +35,8 @@ export async function createPXEService( const keyStorePath = config.dataDirectory ? join(config.dataDirectory, 'pxe_key_store') : undefined; const l1Contracts = await aztecNode.getL1ContractAddresses(); - const keyStore = new TestKeyStore( - new Grumpkin(), - await AztecLmdbStore.create(l1Contracts.rollupAddress, keyStorePath), - ); - const db = new KVPxeDatabase(await AztecLmdbStore.create(l1Contracts.rollupAddress, pxeDbPath)); + const keyStore = new TestKeyStore(new Grumpkin(), await AztecLmdbStore.open(l1Contracts.rollupAddress, keyStorePath)); + const db = new KVPxeDatabase(await AztecLmdbStore.open(l1Contracts.rollupAddress, pxeDbPath)); const server = new PXEService(keyStore, aztecNode, db, config, logSuffix); diff --git a/yarn-project/pxe/src/pxe_service/test/pxe_service.test.ts b/yarn-project/pxe/src/pxe_service/test/pxe_service.test.ts index 617fdd3de96..7caf5668e35 100644 --- a/yarn-project/pxe/src/pxe_service/test/pxe_service.test.ts +++ b/yarn-project/pxe/src/pxe_service/test/pxe_service.test.ts @@ -14,7 +14,7 @@ import { PXEService } from '../pxe_service.js'; import { pxeTestSuite } from './pxe_test_suite.js'; async function createPXEService(): Promise { - const kvStore = await AztecLmdbStore.create(EthAddress.random()); + const kvStore = await AztecLmdbStore.openTmp(); const keyStore = new TestKeyStore(new Grumpkin(), kvStore); const node = mock(); const db = new KVPxeDatabase(kvStore); @@ -46,7 +46,7 @@ describe('PXEService', () => { let config: PXEServiceConfig; beforeEach(async () => { - const kvStore = await AztecLmdbStore.create(EthAddress.random()); + const kvStore = await AztecLmdbStore.openTmp(); keyStore = new TestKeyStore(new Grumpkin(), kvStore); node = mock(); db = new KVPxeDatabase(kvStore); diff --git a/yarn-project/pxe/src/synchronizer/synchronizer.test.ts b/yarn-project/pxe/src/synchronizer/synchronizer.test.ts index 19f10de11c8..44754f4c543 100644 --- a/yarn-project/pxe/src/synchronizer/synchronizer.test.ts +++ b/yarn-project/pxe/src/synchronizer/synchronizer.test.ts @@ -1,5 +1,5 @@ import { AztecNode, INITIAL_L2_BLOCK_NUM, L2Block, MerkleTreeId } from '@aztec/circuit-types'; -import { BlockHeader, CompleteAddress, EthAddress, Fr, GrumpkinScalar } from '@aztec/circuits.js'; +import { BlockHeader, CompleteAddress, Fr, GrumpkinScalar } from '@aztec/circuits.js'; import { Grumpkin } from '@aztec/circuits.js/barretenberg'; import { SerialQueue } from '@aztec/foundation/fifo'; import { TestKeyStore } from '@aztec/key-store'; @@ -32,7 +32,7 @@ describe('Synchronizer', () => { }; aztecNode = mock(); - database = new KVPxeDatabase(await AztecLmdbStore.create(EthAddress.random())); + database = new KVPxeDatabase(await AztecLmdbStore.openTmp()); jobQueue = new SerialQueue(); synchronizer = new TestSynchronizer(aztecNode, database, jobQueue); }); @@ -121,7 +121,7 @@ describe('Synchronizer', () => { expect(await synchronizer.isGlobalStateSynchronized()).toBe(true); // Manually adding account to database so that we can call synchronizer.isAccountStateSynchronized - const keyStore = new TestKeyStore(new Grumpkin(), await AztecLmdbStore.create(EthAddress.random())); + const keyStore = new TestKeyStore(new Grumpkin(), await AztecLmdbStore.openTmp()); const addAddress = async (startingBlockNum: number) => { const privateKey = GrumpkinScalar.random(); await keyStore.addAccount(privateKey); diff --git a/yarn-project/sequencer-client/package.json b/yarn-project/sequencer-client/package.json index 8c11e1f877f..40bf5fd98ee 100644 --- a/yarn-project/sequencer-client/package.json +++ b/yarn-project/sequencer-client/package.json @@ -49,6 +49,7 @@ "viem": "^1.2.5" }, "devDependencies": { + "@aztec/kv-store": "workspace:^", "@jest/globals": "^29.5.0", "@types/jest": "^29.5.0", "@types/levelup": "^5.1.2", diff --git a/yarn-project/sequencer-client/src/block_builder/solo_block_builder.test.ts b/yarn-project/sequencer-client/src/block_builder/solo_block_builder.test.ts index 3ab5e249e60..90b42751fb7 100644 --- a/yarn-project/sequencer-client/src/block_builder/solo_block_builder.test.ts +++ b/yarn-project/sequencer-client/src/block_builder/solo_block_builder.test.ts @@ -49,10 +49,10 @@ import { makeTuple, range } from '@aztec/foundation/array'; import { toBufferBE } from '@aztec/foundation/bigint-buffer'; import { times } from '@aztec/foundation/collection'; import { to2Fields } from '@aztec/foundation/serialize'; +import { AztecLmdbStore } from '@aztec/kv-store'; import { MerkleTreeOperations, MerkleTrees } from '@aztec/world-state'; import { MockProxy, mock } from 'jest-mock-extended'; -import { default as levelup } from 'levelup'; import { type MemDown, default as memdown } from 'memdown'; import { VerificationKeys, getVerificationKeys } from '../mocks/verification_keys.js'; @@ -96,8 +96,8 @@ describe('sequencer/solo_block_builder', () => { blockNumber = 3; globalVariables = new GlobalVariables(chainId, version, new Fr(blockNumber), Fr.ZERO); - builderDb = await MerkleTrees.new(levelup(createMemDown())).then(t => t.asLatest()); - expectsDb = await MerkleTrees.new(levelup(createMemDown())).then(t => t.asLatest()); + builderDb = await MerkleTrees.new(await AztecLmdbStore.openTmp()).then(t => t.asLatest()); + expectsDb = await MerkleTrees.new(await AztecLmdbStore.openTmp()).then(t => t.asLatest()); vks = getVerificationKeys(); simulator = mock(); prover = mock(); diff --git a/yarn-project/sequencer-client/tsconfig.json b/yarn-project/sequencer-client/tsconfig.json index 92a83cff65d..7a1f29b1488 100644 --- a/yarn-project/sequencer-client/tsconfig.json +++ b/yarn-project/sequencer-client/tsconfig.json @@ -38,6 +38,9 @@ }, { "path": "../world-state" + }, + { + "path": "../kv-store" } ], "include": ["src"] diff --git a/yarn-project/world-state/package.json b/yarn-project/world-state/package.json index 7fc61040e0e..e65cae9fcc1 100644 --- a/yarn-project/world-state/package.json +++ b/yarn-project/world-state/package.json @@ -33,10 +33,9 @@ "@aztec/circuit-types": "workspace:^", "@aztec/circuits.js": "workspace:^", "@aztec/foundation": "workspace:^", + "@aztec/kv-store": "workspace:^", "@aztec/merkle-tree": "workspace:^", "@aztec/types": "workspace:^", - "levelup": "^5.1.1", - "memdown": "^6.1.1", "tslib": "^2.4.0" }, "devDependencies": { diff --git a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.test.ts b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.test.ts index 65addf22ff3..8c0f0e23351 100644 --- a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.test.ts +++ b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.test.ts @@ -2,13 +2,12 @@ import { L2Block, L2BlockSource, MerkleTreeId } from '@aztec/circuit-types'; import { Fr } from '@aztec/circuits.js'; import { createDebugLogger } from '@aztec/foundation/log'; import { sleep } from '@aztec/foundation/sleep'; +import { AztecKVStore, AztecLmdbStore } from '@aztec/kv-store'; import { INITIAL_LEAF, Pedersen } from '@aztec/merkle-tree'; import { SiblingPath } from '@aztec/types/membership'; import { jest } from '@jest/globals'; import { mock } from 'jest-mock-extended'; -import levelup from 'levelup'; -import { default as memdown } from 'memdown'; import { MerkleTreeDb, MerkleTrees, WorldStateConfig } from '../index.js'; import { ServerWorldStateSynchronizer } from './server_world_state_synchronizer.js'; @@ -33,30 +32,10 @@ const getMockBlock = (blockNumber: number, newContractsCommitments?: Buffer[]) = return block; }; -const createMockDb = () => levelup((memdown as any)()); - -const createSynchronizer = async ( - db: levelup.LevelUp, - merkleTreeDb: any, - rollupSource: any, - blockCheckInterval = 100, -) => { - const worldStateConfig: WorldStateConfig = { - worldStateBlockCheckIntervalMS: blockCheckInterval, - l2QueueSize: 1000, - }; - - return await ServerWorldStateSynchronizer.new( - db, - merkleTreeDb as MerkleTrees, - rollupSource as L2BlockSource, - worldStateConfig, - ); -}; - const log = createDebugLogger('aztec:server_world_state_synchronizer_test'); describe('server_world_state_synchronizer', () => { + let db: AztecKVStore; const rollupSource = mock({ getBlockNumber: jest.fn(getLatestBlockNumber), getBlocks: jest.fn(consumeNextBlocks), @@ -111,12 +90,30 @@ describe('server_world_state_synchronizer', () => { expect(status.syncedToL2Block).toBe(LATEST_BLOCK_NUMBER + count); }; - it('can be constructed', async () => { - await expect(createSynchronizer(createMockDb(), merkleTreeDb, rollupSource)).resolves.toBeTruthy(); + const createSynchronizer = (blockCheckInterval = 100) => { + const worldStateConfig: WorldStateConfig = { + worldStateBlockCheckIntervalMS: blockCheckInterval, + l2QueueSize: 1000, + }; + + return new ServerWorldStateSynchronizer( + db, + merkleTreeDb as any as MerkleTrees, + rollupSource as L2BlockSource, + worldStateConfig, + ); + }; + + beforeEach(async () => { + db = await AztecLmdbStore.openTmp(); + }); + + it('can be constructed', () => { + expect(createSynchronizer()).toBeTruthy(); }); it('updates sync progress', async () => { - const server = await createSynchronizer(createMockDb(), merkleTreeDb, rollupSource); + const server = createSynchronizer(); // test initial state let status = await server.status(); @@ -165,7 +162,7 @@ describe('server_world_state_synchronizer', () => { }); it('enables blocking until synced', async () => { - const server = await createSynchronizer(createMockDb(), merkleTreeDb, rollupSource); + const server = createSynchronizer(); let currentBlockNumber = 0; const newBlocks = async () => { @@ -196,7 +193,7 @@ describe('server_world_state_synchronizer', () => { }); it('handles multiple calls to start', async () => { - const server = await createSynchronizer(createMockDb(), merkleTreeDb, rollupSource); + const server = createSynchronizer(); let currentBlockNumber = 0; const newBlocks = async () => { @@ -223,7 +220,7 @@ describe('server_world_state_synchronizer', () => { }); it('immediately syncs if no new blocks', async () => { - const server = await createSynchronizer(createMockDb(), merkleTreeDb, rollupSource); + const server = createSynchronizer(); rollupSource.getBlockNumber.mockImplementationOnce(() => { return Promise.resolve(0); }); @@ -241,7 +238,7 @@ describe('server_world_state_synchronizer', () => { }); it("can't be started if already stopped", async () => { - const server = await createSynchronizer(createMockDb(), merkleTreeDb, rollupSource); + const server = createSynchronizer(); rollupSource.getBlockNumber.mockImplementationOnce(() => { return Promise.resolve(0); }); @@ -256,7 +253,7 @@ describe('server_world_state_synchronizer', () => { it('adds the received L2 blocks', async () => { merkleTreeDb.handleL2Block.mockClear(); - const server = await createSynchronizer(createMockDb(), merkleTreeDb, rollupSource); + const server = createSynchronizer(); const totalBlocks = LATEST_BLOCK_NUMBER + 1; nextBlocks = Array(totalBlocks) .fill(0) @@ -269,7 +266,7 @@ describe('server_world_state_synchronizer', () => { }); it('can immediately sync to latest', async () => { - const server = await createSynchronizer(createMockDb(), merkleTreeDb, rollupSource, 10000); + const server = createSynchronizer(10000); await performInitialSync(server); @@ -297,7 +294,7 @@ describe('server_world_state_synchronizer', () => { }); it('can immediately sync to a minimum block number', async () => { - const server = await createSynchronizer(createMockDb(), merkleTreeDb, rollupSource, 10000); + const server = createSynchronizer(10000); await performInitialSync(server); @@ -322,7 +319,7 @@ describe('server_world_state_synchronizer', () => { }); it('can immediately sync to a minimum block in the past', async () => { - const server = await createSynchronizer(createMockDb(), merkleTreeDb, rollupSource, 10000); + const server = createSynchronizer(10000); await performInitialSync(server); // syncing to a block in the past should succeed @@ -344,7 +341,7 @@ describe('server_world_state_synchronizer', () => { }); it('throws if you try to sync to an unavailable block', async () => { - const server = await createSynchronizer(createMockDb(), merkleTreeDb, rollupSource, 10000); + const server = createSynchronizer(); await performInitialSync(server); @@ -370,7 +367,7 @@ describe('server_world_state_synchronizer', () => { }); it('throws if you try to immediate sync when not running', async () => { - const server = await createSynchronizer(createMockDb(), merkleTreeDb, rollupSource, 10000); + const server = createSynchronizer(10000); // test initial state const status = await server.status(); @@ -386,13 +383,12 @@ describe('server_world_state_synchronizer', () => { }); it('restores the last synced block', async () => { - const db = createMockDb(); - const initialServer = await createSynchronizer(db, merkleTreeDb, rollupSource, 10000); + const initialServer = createSynchronizer(10000); await performInitialSync(initialServer); await initialServer.stop(); - const server = await createSynchronizer(db, merkleTreeDb, rollupSource, 10000); + const server = createSynchronizer(10000); const status = await server.status(); expect(status).toEqual({ state: WorldStateRunningState.IDLE, @@ -401,13 +397,12 @@ describe('server_world_state_synchronizer', () => { }); it('starts syncing from the last block', async () => { - const db = createMockDb(); - const initialServer = await createSynchronizer(db, merkleTreeDb, rollupSource, 10000); + const initialServer = createSynchronizer(10000); await performInitialSync(initialServer); await initialServer.stop(); - const server = await createSynchronizer(db, merkleTreeDb, rollupSource, 10000); + const server = createSynchronizer(10000); await performSubsequentSync(server, 2); await server.stop(); }); diff --git a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts index ee0349ade38..3c25882bfed 100644 --- a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts +++ b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts @@ -3,8 +3,7 @@ import { L2BlockHandledStats } from '@aztec/circuit-types/stats'; import { SerialQueue } from '@aztec/foundation/fifo'; import { createDebugLogger } from '@aztec/foundation/log'; import { elapsed } from '@aztec/foundation/timer'; - -import { LevelUp } from 'levelup'; +import { AztecKVStore, AztecSingleton } from '@aztec/kv-store'; import { HandleL2BlockResult, MerkleTreeOperations, MerkleTrees } from '../world-state-db/index.js'; import { MerkleTreeOperationsFacade } from '../world-state-db/merkle_tree_operations_facade.js'; @@ -12,15 +11,12 @@ import { MerkleTreeSnapshotOperationsFacade } from '../world-state-db/merkle_tre import { WorldStateConfig } from './config.js'; import { WorldStateRunningState, WorldStateStatus, WorldStateSynchronizer } from './world_state_synchronizer.js'; -const DB_KEY_BLOCK_NUMBER = 'latestBlockNumber'; - /** * Synchronizes the world state with the L2 blocks from a L2BlockSource. * The synchronizer will download the L2 blocks from the L2BlockSource and insert the new commitments into the merkle * tree. */ export class ServerWorldStateSynchronizer implements WorldStateSynchronizer { - private currentL2BlockNum = 0; private latestBlockNumberAtStart = 0; private l2BlockDownloader: L2BlockDownloader; @@ -30,14 +26,16 @@ export class ServerWorldStateSynchronizer implements WorldStateSynchronizer { private stopping = false; private runningPromise: Promise = Promise.resolve(); private currentState: WorldStateRunningState = WorldStateRunningState.IDLE; + private blockNumber: AztecSingleton; - private constructor( - private db: LevelUp, + constructor( + store: AztecKVStore, private merkleTreeDb: MerkleTrees, private l2BlockSource: L2BlockSource, config: WorldStateConfig, private log = createDebugLogger('aztec:world_state'), ) { + this.blockNumber = store.openSingleton('world_state_synch_last_block_number'); this.l2BlockDownloader = new L2BlockDownloader( l2BlockSource, config.l2QueueSize, @@ -57,22 +55,6 @@ export class ServerWorldStateSynchronizer implements WorldStateSynchronizer { return new MerkleTreeSnapshotOperationsFacade(this.merkleTreeDb, blockNumber); } - public static async new( - db: LevelUp, - merkleTreeDb: MerkleTrees, - l2BlockSource: L2BlockSource, - config: WorldStateConfig, - log = createDebugLogger('aztec:world_state'), - ) { - const server = new ServerWorldStateSynchronizer(db, merkleTreeDb, l2BlockSource, config, log); - await server.#init(); - return server; - } - - async #init() { - await this.restoreCurrentL2BlockNumber(); - } - public async start() { if (this.currentState === WorldStateRunningState.STOPPED) { throw new Error('Synchronizer already stopped'); @@ -123,11 +105,13 @@ export class ServerWorldStateSynchronizer implements WorldStateSynchronizer { await this.merkleTreeDb.stop(); this.log('Awaiting promise'); await this.runningPromise; - this.log('Commiting current block number'); - await this.commitCurrentL2BlockNumber(); this.setCurrentState(WorldStateRunningState.STOPPED); } + private get currentL2BlockNum(): number { + return this.blockNumber.get() ?? 0; + } + public status(): Promise { const status = { syncedToL2Block: this.currentL2BlockNum, @@ -184,7 +168,6 @@ export class ServerWorldStateSynchronizer implements WorldStateSynchronizer { // This request for blocks will timeout after 1 second if no blocks are received const blocks = await this.l2BlockDownloader.getBlocks(1); await this.handleL2Blocks(blocks); - await this.commitCurrentL2BlockNumber(); } /** @@ -210,11 +193,9 @@ export class ServerWorldStateSynchronizer implements WorldStateSynchronizer { */ private async handleL2Block(l2Block: L2Block): Promise { const result = await this.merkleTreeDb.handleL2Block(l2Block); - this.currentL2BlockNum = l2Block.number; - if ( - this.currentState === WorldStateRunningState.SYNCHING && - this.currentL2BlockNum >= this.latestBlockNumberAtStart - ) { + await this.blockNumber.set(l2Block.number); + + if (this.currentState === WorldStateRunningState.SYNCHING && l2Block.number >= this.latestBlockNumberAtStart) { this.setCurrentState(WorldStateRunningState.RUNNING); if (this.syncResolve !== undefined) { this.syncResolve(); @@ -231,22 +212,4 @@ export class ServerWorldStateSynchronizer implements WorldStateSynchronizer { this.currentState = newState; this.log(`Moved to state ${WorldStateRunningState[this.currentState]}`); } - - private async commitCurrentL2BlockNumber() { - const hex = this.currentL2BlockNum.toString(16); - const encoded = Buffer.from(hex.length % 2 === 1 ? '0' + hex : hex, 'hex'); - - await this.db.put(DB_KEY_BLOCK_NUMBER, encoded); - } - - private async restoreCurrentL2BlockNumber() { - try { - const encoded: Buffer = await this.db.get(DB_KEY_BLOCK_NUMBER); - this.currentL2BlockNum = parseInt(encoded.toString('hex'), 16); - this.log.debug(`Restored current L2 block number ${this.currentL2BlockNum} from db`); - } catch (err) { - this.log.debug('No current L2 block number found in db, starting from 0'); - this.currentL2BlockNum = 0; - } - } } diff --git a/yarn-project/world-state/src/world-state-db/merkle_trees.ts b/yarn-project/world-state/src/world-state-db/merkle_trees.ts index b8a0d5c44ee..59d11217e8b 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_trees.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_trees.ts @@ -21,6 +21,7 @@ import { Committable } from '@aztec/foundation/committable'; import { SerialQueue } from '@aztec/foundation/fifo'; import { createDebugLogger } from '@aztec/foundation/log'; import { IndexedTreeLeafPreimage } from '@aztec/foundation/trees'; +import { AztecKVStore, AztecSingleton } from '@aztec/kv-store'; import { AppendOnlyTree, BatchInsertionResult, @@ -35,8 +36,6 @@ import { import { Hasher } from '@aztec/types/interfaces'; import { SiblingPath } from '@aztec/types/membership'; -import { default as levelup } from 'levelup'; - import { INITIAL_NULLIFIER_TREE_SIZE, INITIAL_PUBLIC_DATA_TREE_SIZE, MerkleTreeDb } from './merkle_tree_db.js'; import { CurrentTreeRoots, @@ -63,8 +62,8 @@ const LAST_GLOBAL_VARS_HASH = 'lastGlobalVarsHash'; * The nullifier tree is an indexed tree. */ class NullifierTree extends StandardIndexedTree { - constructor(db: levelup.LevelUp, hasher: Hasher, name: string, depth: number, size: bigint = 0n, root?: Buffer) { - super(db, hasher, name, depth, size, NullifierLeafPreimage, NullifierLeaf, root); + constructor(store: AztecKVStore, hasher: Hasher, name: string, depth: number, size: bigint = 0n, root?: Buffer) { + super(store, hasher, name, depth, size, NullifierLeafPreimage, NullifierLeaf, root); } } @@ -72,8 +71,8 @@ class NullifierTree extends StandardIndexedTree { * The public data tree is an indexed tree. */ class PublicDataTree extends StandardIndexedTree { - constructor(db: levelup.LevelUp, hasher: Hasher, name: string, depth: number, size: bigint = 0n, root?: Buffer) { - super(db, hasher, name, depth, size, PublicDataTreeLeafPreimage, PublicDataTreeLeaf, root); + constructor(store: AztecKVStore, hasher: Hasher, name: string, depth: number, size: bigint = 0n, root?: Buffer) { + super(store, hasher, name, depth, size, PublicDataTreeLeafPreimage, PublicDataTreeLeaf, root); } } @@ -85,8 +84,11 @@ export class MerkleTrees implements MerkleTreeDb { private latestGlobalVariablesHash: Committable; private jobQueue = new SerialQueue(); - constructor(private db: levelup.LevelUp, private log = createDebugLogger('aztec:merkle_trees')) { + #globalVariablesHash: AztecSingleton; + + constructor(private store: AztecKVStore, private log = createDebugLogger('aztec:merkle_trees')) { this.latestGlobalVariablesHash = new Committable(Fr.ZERO); + this.#globalVariablesHash = store.openSingleton(LAST_GLOBAL_VARS_HASH); } /** @@ -100,14 +102,14 @@ export class MerkleTrees implements MerkleTreeDb { const hasher = new Pedersen(); const contractTree: AppendOnlyTree = await initializeTree( StandardTree, - this.db, + this.store, hasher, `${MerkleTreeId[MerkleTreeId.CONTRACT_TREE]}`, CONTRACT_TREE_HEIGHT, ); const nullifierTree = await initializeTree( NullifierTree, - this.db, + this.store, hasher, `${MerkleTreeId[MerkleTreeId.NULLIFIER_TREE]}`, NULLIFIER_TREE_HEIGHT, @@ -115,14 +117,14 @@ export class MerkleTrees implements MerkleTreeDb { ); const noteHashTree: AppendOnlyTree = await initializeTree( StandardTree, - this.db, + this.store, hasher, `${MerkleTreeId[MerkleTreeId.NOTE_HASH_TREE]}`, NOTE_HASH_TREE_HEIGHT, ); const publicDataTree = await initializeTree( PublicDataTree, - this.db, + this.store, hasher, `${MerkleTreeId[MerkleTreeId.PUBLIC_DATA_TREE]}`, PUBLIC_DATA_TREE_HEIGHT, @@ -130,14 +132,14 @@ export class MerkleTrees implements MerkleTreeDb { ); const l1Tol2MessageTree: AppendOnlyTree = await initializeTree( StandardTree, - this.db, + this.store, hasher, `${MerkleTreeId[MerkleTreeId.L1_TO_L2_MESSAGE_TREE]}`, L1_TO_L2_MSG_TREE_HEIGHT, ); const archive: AppendOnlyTree = await initializeTree( StandardTree, - this.db, + this.store, hasher, `${MerkleTreeId[MerkleTreeId.ARCHIVE]}`, ARCHIVE_HEIGHT, @@ -161,15 +163,14 @@ export class MerkleTrees implements MerkleTreeDb { /** * Method to asynchronously create and initialize a MerkleTrees instance. - * @param db - The db instance to use for data persistance. + * @param store - The db instance to use for data persistance. * @returns - A fully initialized MerkleTrees instance. */ - public static async new(db: levelup.LevelUp) { - const merkleTrees = new MerkleTrees(db); - const globalVariablesHash: Buffer | undefined = await db.get(LAST_GLOBAL_VARS_HASH).catch(() => undefined); - await merkleTrees.init( - globalVariablesHash ? { globalVariablesHash: Fr.fromBuffer(globalVariablesHash) } : undefined, - ); + public static async new(store: AztecKVStore) { + const merkleTrees = new MerkleTrees(store); + const globalVariablesHash = store.openSingleton(LAST_GLOBAL_VARS_HASH); + const val = globalVariablesHash.get(); + await merkleTrees.init(val ? { globalVariablesHash: Fr.fromBuffer(val) } : undefined); return merkleTrees; } @@ -280,7 +281,7 @@ export class MerkleTrees implements MerkleTreeDb { index: bigint, includeUncommitted: boolean, ): Promise { - return await this.synchronize(() => this.trees[treeId].getLeafValue(index, includeUncommitted)); + return await this.synchronize(() => Promise.resolve(this.trees[treeId].getLeafValue(index, includeUncommitted))); } /** @@ -348,7 +349,9 @@ export class MerkleTrees implements MerkleTreeDb { } | undefined > { - return await this.synchronize(() => this._getIndexedTree(treeId).findIndexOfPreviousKey(value, includeUncommitted)); + return await this.synchronize(() => + Promise.resolve(this._getIndexedTree(treeId).findIndexOfPreviousKey(value, includeUncommitted)), + ); } /** @@ -364,7 +367,7 @@ export class MerkleTrees implements MerkleTreeDb { includeUncommitted: boolean, ): Promise { return await this.synchronize(() => - this._getIndexedTree(treeId).getLatestLeafPreimageCopy(index, includeUncommitted), + Promise.resolve(this._getIndexedTree(treeId).getLatestLeafPreimageCopy(index, includeUncommitted)), ); } @@ -380,9 +383,9 @@ export class MerkleTrees implements MerkleTreeDb { value: Buffer, includeUncommitted: boolean, ): Promise { - return await this.synchronize(async () => { + return await this.synchronize(() => { const tree = this.trees[treeId]; - return await tree.findLeafIndex(value, includeUncommitted); + return Promise.resolve(tree.findLeafIndex(value, includeUncommitted)); }); } @@ -523,7 +526,7 @@ export class MerkleTrees implements MerkleTreeDb { await tree.commit(); } this.latestGlobalVariablesHash.commit(); - await this.db.put(LAST_GLOBAL_VARS_HASH, this.latestGlobalVariablesHash.get().toBuffer()); + await this.#globalVariablesHash.set(this.latestGlobalVariablesHash.get().toBuffer()); } /** diff --git a/yarn-project/world-state/tsconfig.json b/yarn-project/world-state/tsconfig.json index 5550de6775f..0088e438260 100644 --- a/yarn-project/world-state/tsconfig.json +++ b/yarn-project/world-state/tsconfig.json @@ -15,6 +15,9 @@ { "path": "../foundation" }, + { + "path": "../kv-store" + }, { "path": "../merkle-tree" }, diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index a3ed1d89239..e678a1dcc0a 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -87,6 +87,7 @@ __metadata: "@aztec/circuit-types": "workspace:^" "@aztec/circuits.js": "workspace:^" "@aztec/foundation": "workspace:^" + "@aztec/kv-store": "workspace:^" "@aztec/merkle-tree": "workspace:^" "@aztec/noir-contracts": "workspace:^" "@jest/globals": ^29.5.0 @@ -180,16 +181,10 @@ __metadata: "@aztec/world-state": "workspace:^" "@jest/globals": ^29.5.0 "@types/jest": ^29.5.0 - "@types/leveldown": ^4.0.4 - "@types/levelup": ^5.1.2 - "@types/memdown": ^3.0.0 "@types/node": ^18.7.23 jest: ^29.5.0 koa: ^2.14.2 koa-router: ^12.0.0 - levelup: ^5.1.1 - lmdb: ^2.9.1 - memdown: ^6.1.1 ts-jest: ^29.1.0 ts-node: ^10.9.1 tslib: ^2.4.0 @@ -582,16 +577,13 @@ __metadata: "@aztec/circuit-types": "workspace:^" "@aztec/circuits.js": "workspace:^" "@aztec/foundation": "workspace:^" + "@aztec/kv-store": "workspace:^" "@aztec/types": "workspace:^" "@jest/globals": ^29.5.0 "@types/jest": ^29.5.0 - "@types/levelup": ^5.1.2 - "@types/memdown": ^3.0.1 "@types/node": ^18.15.3 "@types/sha256": ^0.2.0 jest: ^29.5.0 - levelup: ^5.1.1 - memdown: ^6.1.1 sha256: ^0.2.0 ts-jest: ^29.1.0 ts-node: ^10.9.1 @@ -657,6 +649,7 @@ __metadata: "@aztec/circuit-types": "workspace:^" "@aztec/circuits.js": "workspace:^" "@aztec/foundation": "workspace:^" + "@aztec/kv-store": "workspace:^" "@aztec/merkle-tree": "workspace:^" "@aztec/noir-compiler": "workspace:^" "@aztec/types": "workspace:^" @@ -813,6 +806,7 @@ __metadata: "@aztec/circuits.js": "workspace:^" "@aztec/ethereum": "workspace:^" "@aztec/foundation": "workspace:^" + "@aztec/kv-store": "workspace:^" "@aztec/l1-artifacts": "workspace:^" "@aztec/merkle-tree": "workspace:^" "@aztec/noir-protocol-circuits": "workspace:^" @@ -878,6 +872,7 @@ __metadata: "@aztec/circuit-types": "workspace:^" "@aztec/circuits.js": "workspace:^" "@aztec/foundation": "workspace:^" + "@aztec/kv-store": "workspace:^" "@aztec/merkle-tree": "workspace:^" "@aztec/types": "workspace:^" "@jest/globals": ^29.5.0 @@ -887,7 +882,6 @@ __metadata: "@types/node": ^18.7.23 jest: ^29.5.0 jest-mock-extended: ^3.0.5 - levelup: ^5.1.1 memdown: ^6.1.1 ts-jest: ^29.1.0 ts-node: ^10.9.1 @@ -3469,16 +3463,6 @@ __metadata: languageName: node linkType: hard -"@types/leveldown@npm:^4.0.4": - version: 4.0.4 - resolution: "@types/leveldown@npm:4.0.4" - dependencies: - "@types/abstract-leveldown": "*" - "@types/node": "*" - checksum: 630b2d2d1c48f83d14ab0f6c03ad2af1c427675c3692873c4fd3d673bde4140eabc028ce5736ad3d76aeea20769cf53df6f83468a4f0cf28f6d04dbb435edf48 - languageName: node - linkType: hard - "@types/levelup@npm:^5.1.2": version: 5.1.2 resolution: "@types/levelup@npm:5.1.2"