refactor: archiver store (AztecProtocol#3966)

This PR refactors the archiver store to use `@aztec/kv-store` instead of
lmdb directly. It is stacked on top of AztecProtocol#4028.

The diff looks massive but all it does is split the archiver store into
individual pieces to better separate them conceptually (blocks, logs,
contracts, messages) and replace direct uses of lmdb with the data
structures from kv-store.

yarn-project/archiver/package.json

@@ -39,6 +39,7 @@
     "@aztec/circuits.js": "workspace:^",
     "@aztec/ethereum": "workspace:^",
     "@aztec/foundation": "workspace:^",
+    "@aztec/kv-store": "workspace:^",
     "@aztec/l1-artifacts": "workspace:^",
     "@types/lodash.omit": "^4.5.7",
     "debug": "^4.3.4",

yarn-project/archiver/src/archiver/index.ts

@@ -1,5 +1,5 @@
 export * from './archiver.js';
 export * from './config.js';
 export { MemoryArchiverStore } from './memory_archiver_store/memory_archiver_store.js';
-export { LMDBArchiverStore } from './lmdb_archiver_store.js';
 export { ArchiverDataStore } from './archiver_store.js';
+export { KVArchiverDataStore } from './kv_archiver_store/kv_archiver_store.js';

yarn-project/archiver/src/archiver/kv_archiver_store/block_store.ts

@@ -0,0 +1,170 @@
+import { INITIAL_L2_BLOCK_NUM, L2Block, L2Tx, TxHash } from '@aztec/circuit-types';
+import { AztecAddress } from '@aztec/circuits.js';
+import { createDebugLogger } from '@aztec/foundation/log';
+import { AztecKVStore, AztecMap, Range } from '@aztec/kv-store';
+
+/* eslint-disable */
+type BlockIndexValue = [blockNumber: number, index: number];
+
+type BlockContext = {
+  blockNumber: number;
+  l1BlockNumber: bigint;
+  block: Buffer;
+  blockHash: Buffer;
+};
+/* eslint-enable */
+
+/**
+ * LMDB implementation of the ArchiverDataStore interface.
+ */
+export class BlockStore {
+  /** Map block number to block data */
+  #blocks: AztecMap<number, BlockContext>;
+
+  /** Index mapping transaction hash (as a string) to its location in a block */
+  #txIndex: AztecMap<string, BlockIndexValue>;
+
+  /** Index mapping a contract's address (as a string) to its location in a block */
+  #contractIndex: AztecMap<string, BlockIndexValue>;
+
+  #log = createDebugLogger('aztec:archiver:block_store');
+
+  constructor(private db: AztecKVStore) {
+    this.#blocks = db.createMap('archiver_blocks');
+
+    this.#txIndex = db.createMap('archiver_tx_index');
+    this.#contractIndex = db.createMap('archiver_contract_index');
+  }
+
+  /**
+   * Append new blocks to the store's list.
+   * @param blocks - The L2 blocks to be added to the store.
+   * @returns True if the operation is successful.
+   */
+  addBlocks(blocks: L2Block[]): Promise<boolean> {
+    return this.db.transaction(() => {
+      for (const block of blocks) {
+        void this.#blocks.set(block.number, {
+          blockNumber: block.number,
+          block: block.toBuffer(),
+          l1BlockNumber: block.getL1BlockNumber(),
+          blockHash: block.getBlockHash(),
+        });
+
+        for (const [i, tx] of block.getTxs().entries()) {
+          if (tx.txHash.isZero()) {
+            continue;
+          }
+          void this.#txIndex.set(tx.txHash.toString(), [block.number, i]);
+        }
+
+        for (const [i, contractData] of block.newContractData.entries()) {
+          if (contractData.contractAddress.isZero()) {
+            continue;
+          }
+
+          void this.#contractIndex.set(contractData.contractAddress.toString(), [block.number, i]);
+        }
+      }
+
+      return true;
+    });
+  }
+
+  /**
+   * Gets up to `limit` amount of L2 blocks starting from `from`.
+   * @param start - Number of the first block to return (inclusive).
+   * @param limit - The number of blocks to return.
+   * @returns The requested L2 blocks, without logs attached
+   */
+  *getBlocks(start: number, limit: number): IterableIterator<L2Block> {
+    for (const blockCtx of this.#blocks.values(this.#computeBlockRange(start, limit))) {
+      yield L2Block.fromBuffer(blockCtx.block, blockCtx.blockHash);
+    }
+  }
+
+  /**
+   * Gets an L2 block.
+   * @param blockNumber - The number of the block to return.
+   * @returns The requested L2 block, without logs attached
+   */
+  getBlock(blockNumber: number): L2Block | undefined {
+    const blockCtx = this.#blocks.get(blockNumber);
+    if (!blockCtx || !blockCtx.block) {
+      return undefined;
+    }
+
+    const block = L2Block.fromBuffer(blockCtx.block, blockCtx.blockHash);
+
+    return block;
+  }
+
+  /**
+   * Gets an l2 tx.
+   * @param txHash - The txHash of the l2 tx.
+   * @returns The requested L2 tx.
+   */
+  getL2Tx(txHash: TxHash): L2Tx | undefined {
+    const [blockNumber, txIndex] = this.getL2TxLocation(txHash) ?? [];
+    if (typeof blockNumber !== 'number' || typeof txIndex !== 'number') {
+      return undefined;
+    }
+
+    const block = this.getBlock(blockNumber);
+    return block?.getTx(txIndex);
+  }
+
+  /**
+   * Looks up which block included the requested L2 tx.
+   * @param txHash - The txHash of the l2 tx.
+   * @returns The block number and index of the tx.
+   */
+  getL2TxLocation(txHash: TxHash): [blockNumber: number, txIndex: number] | undefined {
+    return this.#txIndex.get(txHash.toString());
+  }
+
+  /**
+   * Looks up which block deployed a particular contract.
+   * @param contractAddress - The address of the contract to look up.
+   * @returns The block number and index of the contract.
+   */
+  getContractLocation(contractAddress: AztecAddress): [blockNumber: number, index: number] | undefined {
+    return this.#contractIndex.get(contractAddress.toString());
+  }
+
+  /**
+   * Gets the number of the latest L2 block processed.
+   * @returns The number of the latest L2 block processed.
+   */
+  getBlockNumber(): number {
+    const [lastBlockNumber] = this.#blocks.keys({ reverse: true, limit: 1 });
+    return typeof lastBlockNumber === 'number' ? lastBlockNumber : INITIAL_L2_BLOCK_NUM - 1;
+  }
+
+  /**
+   * Gets the most recent L1 block processed.
+   * @returns The L1 block that published the latest L2 block
+   */
+  getL1BlockNumber(): bigint {
+    const [lastBlock] = this.#blocks.values({ reverse: true, limit: 1 });
+    if (!lastBlock) {
+      return 0n;
+    } else {
+      return lastBlock.l1BlockNumber;
+    }
+  }
+
+  #computeBlockRange(start: number, limit: number): Required<Pick<Range<number>, 'start' | 'end'>> {
+    if (limit < 1) {
+      throw new Error(`Invalid limit: ${limit}`);
+    }
+
+    if (start < INITIAL_L2_BLOCK_NUM) {
+      this.#log(`Clamping start block ${start} to ${INITIAL_L2_BLOCK_NUM}`);
+      start = INITIAL_L2_BLOCK_NUM;
+    }
+
+    const end = start + limit;
+    return { start, end };
+  }
+}

yarn-project/archiver/src/archiver/kv_archiver_store/contract_store.ts

@@ -0,0 +1,93 @@
+import { ContractData, ExtendedContractData } from '@aztec/circuit-types';
+import { AztecAddress } from '@aztec/foundation/aztec-address';
+import { createDebugLogger } from '@aztec/foundation/log';
+import { AztecKVStore, AztecMap } from '@aztec/kv-store';
+
+import { BlockStore } from './block_store.js';
+
+/**
+ * LMDB implementation of the ArchiverDataStore interface.
+ */
+export class ContractStore {
+  #blockStore: BlockStore;
+  #extendedContractData: AztecMap<number, Buffer[]>;
+  #log = createDebugLogger('aztec:archiver:contract_store');
+
+  constructor(private db: AztecKVStore, blockStore: BlockStore) {
+    this.#extendedContractData = db.createMap('archiver_extended_contract_data');
+    this.#blockStore = blockStore;
+  }
+
+  /**
+   * Add new extended contract data from an L2 block to the store's list.
+   * @param data - List of contracts' data to be added.
+   * @param blockNum - Number of the L2 block the contract data was deployed in.
+   * @returns True if the operation is successful.
+   */
+  addExtendedContractData(data: ExtendedContractData[], blockNum: number): Promise<boolean> {
+    return this.#extendedContractData.swap(blockNum, (existingData = []) => {
+      existingData.push(...data.map(d => d.toBuffer()));
+      return existingData;
+    });
+  }
+
+  /**
+   * Get the extended contract data for this contract.
+   * @param contractAddress - The contract data address.
+   * @returns The extended contract data or undefined if not found.
+   */
+  getExtendedContractData(contractAddress: AztecAddress): ExtendedContractData | undefined {
+    const [blockNumber, _] = this.#blockStore.getContractLocation(contractAddress) ?? [];
+
+    if (typeof blockNumber !== 'number') {
+      return undefined;
+    }
+
+    for (const contract of this.#extendedContractData.get(blockNumber) ?? []) {
+      const extendedContractData = ExtendedContractData.fromBuffer(contract);
+      if (extendedContractData.contractData.contractAddress.equals(contractAddress)) {
+        return extendedContractData;
+      }
+    }
+
+    return undefined;
+  }
+
+  /**
+   * Lookup all extended contract data in an L2 block.
+   * @param blockNumber - The block number to get all contract data from.
+   * @returns All extended contract data in the block (if found).
+   */
+  getExtendedContractDataInBlock(blockNumber: number): Array<ExtendedContractData> {
+    return (this.#extendedContractData.get(blockNumber) ?? []).map(contract =>
+      ExtendedContractData.fromBuffer(contract),
+    );
+  }
+
+  /**
+   * Get basic info for an L2 contract.
+   * Contains contract address & the ethereum portal address.
+   * @param contractAddress - The contract data address.
+   * @returns ContractData with the portal address (if we didn't throw an error).
+   */
+  getContractData(contractAddress: AztecAddress): ContractData | undefined {
+    const [blockNumber, index] = this.#blockStore.getContractLocation(contractAddress) ?? [];
+    if (typeof blockNumber !== 'number' || typeof index !== 'number') {
+      return undefined;
+    }
+
+    const block = this.#blockStore.getBlock(blockNumber);
+    return block?.newContractData[index];
+  }
+
+  /**
+   * Get basic info for an all L2 contracts deployed in a block.
+   * Contains contract address & the ethereum portal address.
+   * @param blockNumber - Number of the L2 block where contracts were deployed.
+   * @returns ContractData with the portal address (if we didn't throw an error).
+   */
+  getContractDataInBlock(blockNumber: number): ContractData[] {
+    const block = this.#blockStore.getBlock(blockNumber);
+    return block?.newContractData ?? [];
+  }
+}

yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.test.ts

@@ -0,0 +1,15 @@
+import { EthAddress } from '@aztec/circuits.js';
+import { AztecLmdbStore } from '@aztec/kv-store';
+
+import { describeArchiverDataStore } from '../archiver_store_test_suite.js';
+import { KVArchiverDataStore } from './kv_archiver_store.js';
+
+describe('KVArchiverDataStore', () => {
+  let archiverStore: KVArchiverDataStore;
+
+  beforeEach(async () => {
+    archiverStore = new KVArchiverDataStore(await AztecLmdbStore.create(EthAddress.random()));
+  });
+
+  describeArchiverDataStore('ArchiverStore', () => archiverStore);
+});