From 7cd3ec5391d015cd27b8954db8fb1a0524f55674 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Tue, 12 Nov 2024 18:11:56 -0300 Subject: [PATCH 01/14] feat: PXE handles reorgs - Archiver and node return L2 block number and hash for `getTxEffect` - PXE database stores L2 block number and hash for each note - PXE database exposes a method for pruning notes after a given block number - PXE synchronizer uses L2 block stream to detect reorgs and clean up notes --- .../archiver/src/archiver/archiver.ts | 5 +- .../archiver/src/archiver/archiver_store.ts | 3 +- .../archiver/kv_archiver_store/block_store.ts | 14 +- .../kv_archiver_store/kv_archiver_store.ts | 3 +- .../memory_archiver_store.ts | 10 +- .../archiver/src/test/mock_l2_block_source.ts | 10 +- .../aztec-node/src/aztec-node/server.ts | 13 +- yarn-project/aztec.js/src/contract/sent_tx.ts | 2 +- .../aztec.js/src/wallet/base_wallet.ts | 3 +- yarn-project/circuit-types/src/in_block.ts | 36 +++++ yarn-project/circuit-types/src/index.ts | 1 + .../src/interfaces/archiver.test.ts | 7 +- .../circuit-types/src/interfaces/archiver.ts | 3 +- .../src/interfaces/aztec-node.test.ts | 30 +++- .../src/interfaces/aztec-node.ts | 20 ++- .../circuit-types/src/interfaces/pxe.test.ts | 11 +- .../circuit-types/src/interfaces/pxe.ts | 5 +- .../l2_block_downloader/l2_block_stream.ts | 10 +- .../circuit-types/src/l2_block_source.ts | 4 +- yarn-project/cli/src/utils/inspect.ts | 7 +- .../end-to-end/src/e2e_block_building.test.ts | 58 ++++---- .../end-to-end/src/e2e_event_logs.test.ts | 2 +- yarn-project/kv-store/package.json | 7 +- yarn-project/kv-store/src/stores/index.ts | 1 + .../kv-store/src/stores/l2_tips_store.test.ts | 71 ++++++++++ .../kv-store/src/stores/l2_tips_store.ts | 69 ++++++++++ yarn-project/kv-store/tsconfig.json | 6 + .../src/database/incoming_note_dao.test.ts | 32 +---- .../pxe/src/database/incoming_note_dao.ts | 47 ++++++- .../pxe/src/database/kv_pxe_database.ts | 35 +++++ .../src/database/outgoing_note_dao.test.ts | 20 +-- .../pxe/src/database/outgoing_note_dao.ts | 44 +++++- yarn-project/pxe/src/database/pxe_database.ts | 12 ++ .../src/database/pxe_database_test_suite.ts | 19 ++- yarn-project/pxe/src/kernel_oracle/index.ts | 2 +- .../produce_note_daos.ts | 8 ++ .../produce_note_daos_for_key.ts | 6 +- .../pxe/src/pxe_service/create_pxe_service.ts | 7 +- .../pxe/src/pxe_service/pxe_service.ts | 29 ++-- .../src/pxe_service/test/pxe_service.test.ts | 12 +- .../pxe/src/simulator_oracle/index.ts | 9 +- .../simulator_oracle/simulator_oracle.test.ts | 3 +- .../pxe/src/synchronizer/synchronizer.test.ts | 96 ++++--------- .../pxe/src/synchronizer/synchronizer.ts | 128 ++++++++---------- yarn-project/yarn.lock | 2 + 45 files changed, 627 insertions(+), 295 deletions(-) create mode 100644 yarn-project/circuit-types/src/in_block.ts create mode 100644 yarn-project/kv-store/src/stores/index.ts create mode 100644 yarn-project/kv-store/src/stores/l2_tips_store.test.ts create mode 100644 yarn-project/kv-store/src/stores/l2_tips_store.ts diff --git a/yarn-project/archiver/src/archiver/archiver.ts b/yarn-project/archiver/src/archiver/archiver.ts index a2a10d8c7a2..cd89bbc63d4 100644 --- a/yarn-project/archiver/src/archiver/archiver.ts +++ b/yarn-project/archiver/src/archiver/archiver.ts @@ -2,6 +2,7 @@ import { type EncryptedL2Log, type FromLogType, type GetUnencryptedLogsResponse, + type InBlock, type InboxLeaf, type L1ToL2MessageSource, type L2Block, @@ -589,7 +590,7 @@ export class Archiver implements ArchiveSource { } } - public getTxEffect(txHash: TxHash): Promise { + public getTxEffect(txHash: TxHash) { return this.store.getTxEffect(txHash); } @@ -933,7 +934,7 @@ class ArchiverStoreHelper getBlockHeaders(from: number, limit: number): Promise { return this.store.getBlockHeaders(from, limit); } - getTxEffect(txHash: TxHash): Promise { + getTxEffect(txHash: TxHash): Promise | undefined> { return this.store.getTxEffect(txHash); } getSettledTxReceipt(txHash: TxHash): Promise { diff --git a/yarn-project/archiver/src/archiver/archiver_store.ts b/yarn-project/archiver/src/archiver/archiver_store.ts index 4eb2c80ccc0..5ca7e878d86 100644 --- a/yarn-project/archiver/src/archiver/archiver_store.ts +++ b/yarn-project/archiver/src/archiver/archiver_store.ts @@ -1,6 +1,7 @@ import { type FromLogType, type GetUnencryptedLogsResponse, + type InBlock, type InboxLeaf, type L2Block, type L2BlockL2Logs, @@ -79,7 +80,7 @@ export interface ArchiverDataStore { * @param txHash - The txHash of the tx corresponding to the tx effect. * @returns The requested tx effect (or undefined if not found). */ - getTxEffect(txHash: TxHash): Promise; + getTxEffect(txHash: TxHash): Promise | undefined>; /** * Gets a receipt of a settled tx. diff --git a/yarn-project/archiver/src/archiver/kv_archiver_store/block_store.ts b/yarn-project/archiver/src/archiver/kv_archiver_store/block_store.ts index a4a3ba3281c..85bbacc0369 100644 --- a/yarn-project/archiver/src/archiver/kv_archiver_store/block_store.ts +++ b/yarn-project/archiver/src/archiver/kv_archiver_store/block_store.ts @@ -1,4 +1,4 @@ -import { Body, L2Block, type TxEffect, type TxHash, TxReceipt } from '@aztec/circuit-types'; +import { Body, type InBlock, L2Block, type TxEffect, type TxHash, TxReceipt } from '@aztec/circuit-types'; import { AppendOnlyTreeSnapshot, type AztecAddress, Header, INITIAL_L2_BLOCK_NUM } from '@aztec/circuits.js'; import { createDebugLogger } from '@aztec/foundation/log'; import { type AztecKVStore, type AztecMap, type AztecSingleton, type Range } from '@aztec/kv-store'; @@ -170,14 +170,22 @@ export class BlockStore { * @param txHash - The txHash of the tx corresponding to the tx effect. * @returns The requested tx effect (or undefined if not found). */ - getTxEffect(txHash: TxHash): TxEffect | undefined { + getTxEffect(txHash: TxHash): InBlock | undefined { const [blockNumber, txIndex] = this.getTxLocation(txHash) ?? []; if (typeof blockNumber !== 'number' || typeof txIndex !== 'number') { return undefined; } const block = this.getBlock(blockNumber); - return block?.data.body.txEffects[txIndex]; + if (!block) { + return undefined; + } + + return { + data: block.data.body.txEffects[txIndex], + l2BlockNumber: block.data.number, + l2BlockHash: block.data.hash().toString(), + }; } /** diff --git a/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts b/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts index 8cbb627a5ce..890a827288c 100644 --- a/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts +++ b/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts @@ -6,7 +6,6 @@ import { type L2BlockL2Logs, type LogFilter, type LogType, - type TxEffect, type TxHash, type TxReceipt, type TxScopedL2Log, @@ -159,7 +158,7 @@ export class KVArchiverDataStore implements ArchiverDataStore { * @param txHash - The txHash of the tx corresponding to the tx effect. * @returns The requested tx effect (or undefined if not found). */ - getTxEffect(txHash: TxHash): Promise { + getTxEffect(txHash: TxHash) { return Promise.resolve(this.#blockStore.getTxEffect(txHash)); } diff --git a/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts b/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts index e49ab8eccc2..441af8aa256 100644 --- a/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts +++ b/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts @@ -6,6 +6,7 @@ import { ExtendedUnencryptedL2Log, type FromLogType, type GetUnencryptedLogsResponse, + type InBlock, type InboxLeaf, type L2Block, type L2BlockL2Logs, @@ -17,6 +18,7 @@ import { TxReceipt, TxScopedL2Log, type UnencryptedL2BlockL2Logs, + wrapInBlock, } from '@aztec/circuit-types'; import { type ContractClassPublic, @@ -50,7 +52,7 @@ export class MemoryArchiverStore implements ArchiverDataStore { /** * An array containing all the tx effects in the L2 blocks that have been fetched so far. */ - private txEffects: TxEffect[] = []; + private txEffects: InBlock[] = []; private noteEncryptedLogsPerBlock: Map = new Map(); @@ -181,7 +183,7 @@ export class MemoryArchiverStore implements ArchiverDataStore { this.lastL1BlockNewBlocks = blocks[blocks.length - 1].l1.blockNumber; this.l2Blocks.push(...blocks); - this.txEffects.push(...blocks.flatMap(b => b.data.body.txEffects)); + this.txEffects.push(...blocks.flatMap(b => b.data.body.txEffects.map(txEffect => wrapInBlock(txEffect, b.data)))); return Promise.resolve(true); } @@ -365,8 +367,8 @@ export class MemoryArchiverStore implements ArchiverDataStore { * @param txHash - The txHash of the tx effect. * @returns The requested tx effect. */ - public getTxEffect(txHash: TxHash): Promise { - const txEffect = this.txEffects.find(tx => tx.txHash.equals(txHash)); + public getTxEffect(txHash: TxHash): Promise | undefined> { + const txEffect = this.txEffects.find(tx => tx.data.txHash.equals(txHash)); return Promise.resolve(txEffect); } diff --git a/yarn-project/archiver/src/test/mock_l2_block_source.ts b/yarn-project/archiver/src/test/mock_l2_block_source.ts index 86ffce4ae50..2ab843cb42a 100644 --- a/yarn-project/archiver/src/test/mock_l2_block_source.ts +++ b/yarn-project/archiver/src/test/mock_l2_block_source.ts @@ -119,8 +119,14 @@ export class MockL2BlockSource implements L2BlockSource { * @returns The requested tx effect. */ public getTxEffect(txHash: TxHash) { - const txEffect = this.l2Blocks.flatMap(b => b.body.txEffects).find(tx => tx.txHash.equals(txHash)); - return Promise.resolve(txEffect); + const match = this.l2Blocks + .flatMap(b => b.body.txEffects.map(tx => [tx, b] as const)) + .find(([tx]) => tx.txHash.equals(txHash)); + if (!match) { + return Promise.resolve(undefined); + } + const [txEffect, block] = match; + return Promise.resolve({ data: txEffect, l2BlockNumber: block.number, l2BlockHash: block.hash().toString() }); } /** diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index eec371b96fb..c746ba30534 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -6,6 +6,7 @@ import { type EpochProofQuote, type FromLogType, type GetUnencryptedLogsResponse, + type InBlock, type L1ToL2MessageSource, type L2Block, type L2BlockL2Logs, @@ -117,14 +118,18 @@ export class AztecNodeService implements AztecNode { this.log.info(message); } - addEpochProofQuote(quote: EpochProofQuote): Promise { + public addEpochProofQuote(quote: EpochProofQuote): Promise { return Promise.resolve(this.p2pClient.addEpochProofQuote(quote)); } - getEpochProofQuotes(epoch: bigint): Promise { + public getEpochProofQuotes(epoch: bigint): Promise { return this.p2pClient.getEpochProofQuotes(epoch); } + public getL2Tips() { + return this.blockSource.getL2Tips(); + } + /** * initializes the Aztec Node, wait for component to sync. * @param config - The configuration to be used by the aztec node. @@ -372,7 +377,7 @@ export class AztecNodeService implements AztecNode { return txReceipt; } - public getTxEffect(txHash: TxHash): Promise { + public getTxEffect(txHash: TxHash): Promise | undefined> { return this.blockSource.getTxEffect(txHash); } @@ -708,7 +713,7 @@ export class AztecNodeService implements AztecNode { * Returns the currently committed block header, or the initial header if no blocks have been produced. * @returns The current committed block header. */ - public async getHeader(blockNumber: L2BlockNumber = 'latest'): Promise
{ + public async getBlockHeader(blockNumber: L2BlockNumber = 'latest'): Promise
{ return ( (await this.getBlock(blockNumber === 'latest' ? -1 : blockNumber))?.header ?? this.worldStateSynchronizer.getCommitted().getInitialHeader() diff --git a/yarn-project/aztec.js/src/contract/sent_tx.ts b/yarn-project/aztec.js/src/contract/sent_tx.ts index 7b85b3fd853..d86f80528fb 100644 --- a/yarn-project/aztec.js/src/contract/sent_tx.ts +++ b/yarn-project/aztec.js/src/contract/sent_tx.ts @@ -78,7 +78,7 @@ export class SentTx { } if (opts?.debug) { const txHash = await this.getTxHash(); - const tx = (await this.pxe.getTxEffect(txHash))!; + const { data: tx } = (await this.pxe.getTxEffect(txHash))!; receipt.debugInfo = { noteHashes: tx.noteHashes, nullifiers: tx.nullifiers, diff --git a/yarn-project/aztec.js/src/wallet/base_wallet.ts b/yarn-project/aztec.js/src/wallet/base_wallet.ts index 00cc9127a38..94d7e479504 100644 --- a/yarn-project/aztec.js/src/wallet/base_wallet.ts +++ b/yarn-project/aztec.js/src/wallet/base_wallet.ts @@ -13,7 +13,6 @@ import { type SiblingPath, type SyncStatus, type Tx, - type TxEffect, type TxExecutionRequest, type TxHash, type TxProvingResult, @@ -122,7 +121,7 @@ export abstract class BaseWallet implements Wallet { sendTx(tx: Tx): Promise { return this.pxe.sendTx(tx); } - getTxEffect(txHash: TxHash): Promise { + getTxEffect(txHash: TxHash) { return this.pxe.getTxEffect(txHash); } getTxReceipt(txHash: TxHash): Promise { diff --git a/yarn-project/circuit-types/src/in_block.ts b/yarn-project/circuit-types/src/in_block.ts new file mode 100644 index 00000000000..5e5205b4237 --- /dev/null +++ b/yarn-project/circuit-types/src/in_block.ts @@ -0,0 +1,36 @@ +import { Fr } from '@aztec/circuits.js'; +import { schemas } from '@aztec/foundation/schemas'; + +import { type ZodTypeAny, z } from 'zod'; + +import { type L2Block } from './l2_block.js'; + +export type InBlock = { + l2BlockNumber: number; + l2BlockHash: string; + data: T; +}; + +export function randomInBlock(data: T): InBlock { + return { + data, + l2BlockNumber: Math.floor(Math.random() * 1000), + l2BlockHash: Fr.random().toString(), + }; +} + +export function wrapInBlock(data: T, block: L2Block): InBlock { + return { + data, + l2BlockNumber: block.number, + l2BlockHash: block.hash().toString(), + }; +} + +export function inBlockSchemaFor(schema: T) { + return z.object({ + data: schema, + l2BlockNumber: schemas.Integer, + l2BlockHash: z.string(), + }); +} diff --git a/yarn-project/circuit-types/src/index.ts b/yarn-project/circuit-types/src/index.ts index 86eade75e74..2040dedbbb3 100644 --- a/yarn-project/circuit-types/src/index.ts +++ b/yarn-project/circuit-types/src/index.ts @@ -22,3 +22,4 @@ export * from './simulation_error.js'; export * from './tx/index.js'; export * from './tx_effect.js'; export * from './tx_execution_request.js'; +export * from './in_block.js'; diff --git a/yarn-project/circuit-types/src/interfaces/archiver.test.ts b/yarn-project/circuit-types/src/interfaces/archiver.test.ts index b6b411c3177..70a7ab772f4 100644 --- a/yarn-project/circuit-types/src/interfaces/archiver.test.ts +++ b/yarn-project/circuit-types/src/interfaces/archiver.test.ts @@ -20,6 +20,7 @@ import { readFileSync } from 'fs'; import omit from 'lodash.omit'; import { resolve } from 'path'; +import { type InBlock } from '../in_block.js'; import { L2Block } from '../l2_block.js'; import { type L2Tips } from '../l2_block_source.js'; import { ExtendedUnencryptedL2Log } from '../logs/extended_unencrypted_l2_log.js'; @@ -106,7 +107,7 @@ describe('ArchiverApiSchema', () => { it('getTxEffect', async () => { const result = await context.client.getTxEffect(new TxHash(Buffer.alloc(32, 1))); - expect(result).toBeInstanceOf(TxEffect); + expect(result!.data).toBeInstanceOf(TxEffect); }); it('getSettledTxReceipt', async () => { @@ -261,9 +262,9 @@ class MockArchiver implements ArchiverApi { getBlocks(from: number, _limit: number, _proven?: boolean | undefined): Promise { return Promise.resolve([L2Block.random(from)]); } - getTxEffect(_txHash: TxHash): Promise { + getTxEffect(_txHash: TxHash): Promise | undefined> { expect(_txHash).toBeInstanceOf(TxHash); - return Promise.resolve(TxEffect.empty()); + return Promise.resolve({ l2BlockNumber: 1, l2BlockHash: '0x12', data: TxEffect.random() }); } getSettledTxReceipt(txHash: TxHash): Promise { expect(txHash).toBeInstanceOf(TxHash); diff --git a/yarn-project/circuit-types/src/interfaces/archiver.ts b/yarn-project/circuit-types/src/interfaces/archiver.ts index 0dc118875aa..686b512a65e 100644 --- a/yarn-project/circuit-types/src/interfaces/archiver.ts +++ b/yarn-project/circuit-types/src/interfaces/archiver.ts @@ -10,6 +10,7 @@ import { type ApiSchemaFor, optional, schemas } from '@aztec/foundation/schemas' import { z } from 'zod'; +import { inBlockSchemaFor } from '../in_block.js'; import { L2Block } from '../l2_block.js'; import { type L2BlockSource, L2TipsSchema } from '../l2_block_source.js'; import { GetUnencryptedLogsResponseSchema, TxScopedL2Log } from '../logs/get_logs_response.js'; @@ -42,7 +43,7 @@ export const ArchiverApiSchema: ApiSchemaFor = { .function() .args(schemas.Integer, schemas.Integer, optional(z.boolean())) .returns(z.array(L2Block.schema)), - getTxEffect: z.function().args(TxHash.schema).returns(TxEffect.schema.optional()), + getTxEffect: z.function().args(TxHash.schema).returns(inBlockSchemaFor(TxEffect.schema).optional()), getSettledTxReceipt: z.function().args(TxHash.schema).returns(TxReceipt.schema.optional()), getL2SlotNumber: z.function().args().returns(schemas.BigInt), getL2EpochNumber: z.function().args().returns(schemas.BigInt), diff --git a/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts b/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts index d415249c3f4..1c0f10fbf0f 100644 --- a/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts +++ b/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts @@ -28,7 +28,9 @@ import omit from 'lodash.omit'; import times from 'lodash.times'; import { resolve } from 'path'; +import { type InBlock } from '../in_block.js'; import { L2Block } from '../l2_block.js'; +import { type L2Tips } from '../l2_block_source.js'; import { ExtendedUnencryptedL2Log } from '../logs/extended_unencrypted_l2_log.js'; import { type GetUnencryptedLogsResponse, TxScopedL2Log } from '../logs/get_logs_response.js'; import { @@ -80,6 +82,15 @@ describe('AztecNodeApiSchema', () => { expect([...tested].sort()).toEqual(all.sort()); }); + it('getL2Tips', async () => { + const result = await context.client.getL2Tips(); + expect(result).toEqual({ + latest: { number: 1, hash: `0x01` }, + proven: { number: 1, hash: `0x01` }, + finalized: { number: 1, hash: `0x01` }, + }); + }); + it('findLeavesIndexes', async () => { const response = await context.client.findLeavesIndexes(1, MerkleTreeId.ARCHIVE, [Fr.random(), Fr.random()]); expect(response).toEqual([1n, undefined]); @@ -231,7 +242,7 @@ describe('AztecNodeApiSchema', () => { it('getTxEffect', async () => { const response = await context.client.getTxEffect(TxHash.random()); - expect(response).toBeInstanceOf(TxEffect); + expect(response!.data).toBeInstanceOf(TxEffect); }); it('getPendingTxs', async () => { @@ -254,8 +265,8 @@ describe('AztecNodeApiSchema', () => { expect(response).toBeInstanceOf(Fr); }); - it('getHeader', async () => { - const response = await context.client.getHeader(); + it('getBlockHeader', async () => { + const response = await context.client.getBlockHeader(); expect(response).toBeInstanceOf(Header); }); @@ -318,6 +329,13 @@ describe('AztecNodeApiSchema', () => { class MockAztecNode implements AztecNode { constructor(private artifact: ContractArtifact) {} + getL2Tips(): Promise { + return Promise.resolve({ + latest: { number: 1, hash: `0x01` }, + proven: { number: 1, hash: `0x01` }, + finalized: { number: 1, hash: `0x01` }, + }); + } findLeavesIndexes( blockNumber: number | 'latest', treeId: MerkleTreeId, @@ -472,9 +490,9 @@ class MockAztecNode implements AztecNode { expect(txHash).toBeInstanceOf(TxHash); return Promise.resolve(TxReceipt.empty()); } - getTxEffect(txHash: TxHash): Promise { + getTxEffect(txHash: TxHash): Promise | undefined> { expect(txHash).toBeInstanceOf(TxHash); - return Promise.resolve(TxEffect.random()); + return Promise.resolve({ l2BlockNumber: 1, l2BlockHash: '0x12', data: TxEffect.random() }); } getPendingTxs(): Promise { return Promise.resolve([Tx.random()]); @@ -491,7 +509,7 @@ class MockAztecNode implements AztecNode { expect(slot).toBeInstanceOf(Fr); return Promise.resolve(Fr.random()); } - getHeader(_blockNumber?: number | 'latest' | undefined): Promise
{ + getBlockHeader(_blockNumber?: number | 'latest' | undefined): Promise
{ return Promise.resolve(Header.empty()); } simulatePublicCalls(tx: Tx): Promise { diff --git a/yarn-project/circuit-types/src/interfaces/aztec-node.ts b/yarn-project/circuit-types/src/interfaces/aztec-node.ts index 91b7dd8d0ca..252757521fe 100644 --- a/yarn-project/circuit-types/src/interfaces/aztec-node.ts +++ b/yarn-project/circuit-types/src/interfaces/aztec-node.ts @@ -21,7 +21,9 @@ import { type ApiSchemaFor, optional, schemas } from '@aztec/foundation/schemas' import { z } from 'zod'; +import { type InBlock, inBlockSchemaFor } from '../in_block.js'; import { L2Block } from '../l2_block.js'; +import { type L2BlockSource, type L2Tips, L2TipsSchema } from '../l2_block_source.js'; import { type FromLogType, type GetUnencryptedLogsResponse, @@ -48,7 +50,14 @@ import { type ProverCoordination, ProverCoordinationApiSchema } from './prover-c * The aztec node. * We will probably implement the additional interfaces by means other than Aztec Node as it's currently a privacy leak */ -export interface AztecNode extends ProverCoordination { +export interface AztecNode + extends ProverCoordination, + Pick { + /** + * Returns the tips of the L2 chain. + */ + getL2Tips(): Promise; + /** * Find the indexes of the given leaves in the given tree. * @param blockNumber - The block number at which to get the data or 'latest' for latest data @@ -298,7 +307,7 @@ export interface AztecNode extends ProverCoordination { * @param txHash - The hash of a transaction which resulted in the returned tx effect. * @returns The requested tx effect. */ - getTxEffect(txHash: TxHash): Promise; + getTxEffect(txHash: TxHash): Promise | undefined>; /** * Method to retrieve pending txs. @@ -336,7 +345,7 @@ export interface AztecNode extends ProverCoordination { * Returns the currently committed block header. * @returns The current committed block header. */ - getHeader(blockNumber?: L2BlockNumber): Promise
; + getBlockHeader(blockNumber?: L2BlockNumber): Promise
; /** * Simulates the public part of a transaction with the current state. @@ -396,6 +405,7 @@ export interface AztecNode extends ProverCoordination { export const AztecNodeApiSchema: ApiSchemaFor = { ...ProverCoordinationApiSchema, + getL2Tips: z.function().args().returns(L2TipsSchema), findLeavesIndexes: z .function() .args(L2BlockNumberSchema, z.nativeEnum(MerkleTreeId), z.array(schemas.Fr)) @@ -485,7 +495,7 @@ export const AztecNodeApiSchema: ApiSchemaFor = { getTxReceipt: z.function().args(TxHash.schema).returns(TxReceipt.schema), - getTxEffect: z.function().args(TxHash.schema).returns(TxEffect.schema.optional()), + getTxEffect: z.function().args(TxHash.schema).returns(inBlockSchemaFor(TxEffect.schema).optional()), getPendingTxs: z.function().returns(z.array(Tx.schema)), @@ -495,7 +505,7 @@ export const AztecNodeApiSchema: ApiSchemaFor = { getPublicStorageAt: z.function().args(schemas.AztecAddress, schemas.Fr, L2BlockNumberSchema).returns(schemas.Fr), - getHeader: z.function().args(optional(L2BlockNumberSchema)).returns(Header.schema), + getBlockHeader: z.function().args(optional(L2BlockNumberSchema)).returns(Header.schema), simulatePublicCalls: z.function().args(Tx.schema).returns(PublicSimulationOutput.schema), diff --git a/yarn-project/circuit-types/src/interfaces/pxe.test.ts b/yarn-project/circuit-types/src/interfaces/pxe.test.ts index 8aefa059edb..e2aa6c1cca5 100644 --- a/yarn-project/circuit-types/src/interfaces/pxe.test.ts +++ b/yarn-project/circuit-types/src/interfaces/pxe.test.ts @@ -30,6 +30,7 @@ import times from 'lodash.times'; import { resolve } from 'path'; import { AuthWitness } from '../auth_witness.js'; +import { type InBlock } from '../in_block.js'; import { L2Block } from '../l2_block.js'; import { ExtendedUnencryptedL2Log, type GetUnencryptedLogsResponse, type LogFilter } from '../logs/index.js'; import { type IncomingNotesFilter } from '../notes/incoming_notes_filter.js'; @@ -178,8 +179,10 @@ describe('PXESchema', () => { }); it('getTxEffect', async () => { - const result = await context.client.getTxEffect(TxHash.random()); - expect(result).toBeInstanceOf(TxEffect); + const { l2BlockHash, l2BlockNumber, data } = (await context.client.getTxEffect(TxHash.random()))!; + expect(data).toBeInstanceOf(TxEffect); + expect(l2BlockHash).toMatch(/0x[a-fA-F0-9]{64}/); + expect(l2BlockNumber).toBe(1); }); it('getPublicStorageAt', async () => { @@ -401,9 +404,9 @@ class MockPXE implements PXE { expect(txHash).toBeInstanceOf(TxHash); return Promise.resolve(TxReceipt.empty()); } - getTxEffect(txHash: TxHash): Promise { + getTxEffect(txHash: TxHash): Promise | undefined> { expect(txHash).toBeInstanceOf(TxHash); - return Promise.resolve(TxEffect.random()); + return Promise.resolve({ data: TxEffect.random(), l2BlockHash: Fr.random().toString(), l2BlockNumber: 1 }); } getPublicStorageAt(contract: AztecAddress, slot: Fr): Promise { expect(contract).toBeInstanceOf(AztecAddress); diff --git a/yarn-project/circuit-types/src/interfaces/pxe.ts b/yarn-project/circuit-types/src/interfaces/pxe.ts index 4bfa62f74bb..a2a6f6940fe 100644 --- a/yarn-project/circuit-types/src/interfaces/pxe.ts +++ b/yarn-project/circuit-types/src/interfaces/pxe.ts @@ -27,6 +27,7 @@ import { AbiDecodedSchema, type ApiSchemaFor, type ZodFor, optional, schemas } f import { z } from 'zod'; import { AuthWitness } from '../auth_witness.js'; +import { type InBlock, inBlockSchemaFor } from '../in_block.js'; import { L2Block } from '../l2_block.js'; import { type GetUnencryptedLogsResponse, @@ -216,7 +217,7 @@ export interface PXE { * @param txHash - The hash of a transaction which resulted in the returned tx effect. * @returns The requested tx effect. */ - getTxEffect(txHash: TxHash): Promise; + getTxEffect(txHash: TxHash): Promise | undefined>; /** * Gets the storage value at the given contract storage slot. @@ -500,7 +501,7 @@ export const PXESchema: ApiSchemaFor = { getTxEffect: z .function() .args(TxHash.schema) - .returns(z.union([TxEffect.schema, z.undefined()])), + .returns(z.union([inBlockSchemaFor(TxEffect.schema), z.undefined()])), getPublicStorageAt: z.function().args(schemas.AztecAddress, schemas.Fr).returns(schemas.Fr), getIncomingNotes: z.function().args(IncomingNotesFilterSchema).returns(z.array(UniqueNote.schema)), getL1ToL2MembershipWitness: z diff --git a/yarn-project/circuit-types/src/l2_block_downloader/l2_block_stream.ts b/yarn-project/circuit-types/src/l2_block_downloader/l2_block_stream.ts index 7a1d179dd16..50608b6c0ea 100644 --- a/yarn-project/circuit-types/src/l2_block_downloader/l2_block_stream.ts +++ b/yarn-project/circuit-types/src/l2_block_downloader/l2_block_stream.ts @@ -12,14 +12,15 @@ export class L2BlockStream { private readonly log = createDebugLogger('aztec:l2_block_stream'); constructor( - private l2BlockSource: L2BlockSource, + private l2BlockSource: Pick, private localData: L2BlockStreamLocalDataProvider, private handler: L2BlockStreamEventHandler, private opts: { proven?: boolean; pollIntervalMS?: number; batchSize?: number; - }, + startingBlock?: number; + } = {}, ) { this.runningPromise = new RunningPromise(() => this.work(), this.opts.pollIntervalMS ?? 1000); } @@ -70,6 +71,11 @@ export class L2BlockStream { await this.emitEvent({ type: 'chain-pruned', blockNumber: latestBlockNumber }); } + // If we are just starting, use the starting block number from the options. + if (latestBlockNumber === 0 && this.opts.startingBlock !== undefined) { + latestBlockNumber = Math.max(this.opts.startingBlock - 1, 0); + } + // Request new blocks from the source. while (latestBlockNumber < sourceTips.latest.number) { const from = latestBlockNumber + 1; diff --git a/yarn-project/circuit-types/src/l2_block_source.ts b/yarn-project/circuit-types/src/l2_block_source.ts index 4dac6da0db9..6f749b28189 100644 --- a/yarn-project/circuit-types/src/l2_block_source.ts +++ b/yarn-project/circuit-types/src/l2_block_source.ts @@ -2,6 +2,7 @@ import { type EthAddress, type Header } from '@aztec/circuits.js'; import { z } from 'zod'; +import { type InBlock } from './in_block.js'; import { type L2Block } from './l2_block.js'; import { type TxHash } from './tx/tx_hash.js'; import { type TxReceipt } from './tx/tx_receipt.js'; @@ -69,7 +70,7 @@ export interface L2BlockSource { * @param txHash - The hash of a transaction which resulted in the returned tx effect. * @returns The requested tx effect. */ - getTxEffect(txHash: TxHash): Promise; + getTxEffect(txHash: TxHash): Promise | undefined>; /** * Gets a receipt of a settled tx. @@ -121,6 +122,7 @@ export type L2Tips = Record; /** Identifies a block by number and hash. */ export type L2BlockId = z.infer; +// TODO(palla/schemas): This package should know what is the block hash of the genesis block 0. const L2BlockIdSchema = z.union([ z.object({ number: z.literal(0), diff --git a/yarn-project/cli/src/utils/inspect.ts b/yarn-project/cli/src/utils/inspect.ts index 445e48c3bd1..855c38b9ef5 100644 --- a/yarn-project/cli/src/utils/inspect.ts +++ b/yarn-project/cli/src/utils/inspect.ts @@ -38,22 +38,23 @@ export async function inspectTx( log: LogFn, opts: { includeBlockInfo?: boolean; artifactMap?: ArtifactMap } = {}, ) { - const [receipt, effects, notes] = await Promise.all([ + const [receipt, effectsInBlock, notes] = await Promise.all([ pxe.getTxReceipt(txHash), pxe.getTxEffect(txHash), pxe.getIncomingNotes({ txHash, status: NoteStatus.ACTIVE_OR_NULLIFIED }), ]); // Base tx data log(`Tx ${txHash.toString()}`); - log(` Status: ${receipt.status} ${effects ? `(${effects.revertCode.getDescription()})` : ''}`); + log(` Status: ${receipt.status} ${effectsInBlock ? `(${effectsInBlock.data.revertCode.getDescription()})` : ''}`); if (receipt.error) { log(` Error: ${receipt.error}`); } - if (!effects) { + if (!effectsInBlock) { return; } + const effects = effectsInBlock.data; const artifactMap = opts?.artifactMap ?? (await getKnownArtifacts(pxe)); if (opts.includeBlockInfo) { diff --git a/yarn-project/end-to-end/src/e2e_block_building.test.ts b/yarn-project/end-to-end/src/e2e_block_building.test.ts index 88bfd6a1b23..41a27b70a92 100644 --- a/yarn-project/end-to-end/src/e2e_block_building.test.ts +++ b/yarn-project/end-to-end/src/e2e_block_building.test.ts @@ -1,5 +1,4 @@ import { getSchnorrAccount } from '@aztec/accounts/schnorr'; -import { createAccount } from '@aztec/accounts/testing'; import { type AztecAddress, type AztecNode, @@ -23,7 +22,6 @@ import { poseidon2HashWithSeparator } from '@aztec/foundation/crypto'; import { StatefulTestContract, StatefulTestContractArtifact } from '@aztec/noir-contracts.js'; import { TestContract } from '@aztec/noir-contracts.js/Test'; import { TokenContract } from '@aztec/noir-contracts.js/Token'; -import { createPXEService, getPXEServiceConfig } from '@aztec/pxe'; import 'jest-extended'; @@ -439,49 +437,63 @@ describe('e2e_block_building', () => { await cheatCodes.rollup.advanceToNextEpoch(); await cheatCodes.rollup.markAsProven(); - // Send a tx to the contract that updates the public data tree, this should take the first slot + // Send a tx to the contract that creates a note. This tx will be reorgd but re-included, + // since it is being built against a proven block number. logger.info('Sending initial tx'); - const tx1 = await contract.methods.increment_public_value(ownerAddress, 20).send().wait(); + const tx1 = await contract.methods.create_note(ownerAddress, ownerAddress, 20).send().wait(); expect(tx1.blockNumber).toEqual(initialBlockNumber + 1); - expect(await contract.methods.get_public_value(ownerAddress).simulate()).toEqual(20n); + expect(await contract.methods.summed_values(ownerAddress).simulate()).toEqual(21n); + + // And send a second one, which won't be re-included. + logger.info('Sending second tx'); + const tx2 = await contract.methods.create_note(ownerAddress, ownerAddress, 30).send().wait(); + expect(tx2.blockNumber).toEqual(initialBlockNumber + 2); + expect(await contract.methods.summed_values(ownerAddress).simulate()).toEqual(51n); // Now move to a new epoch and past the proof claim window to cause a reorg logger.info('Advancing past the proof claim window'); await cheatCodes.rollup.advanceToNextEpoch(); await cheatCodes.rollup.advanceSlots(aztecEpochProofClaimWindowInL2Slots + 1); // off-by-one? - // Wait a bit before spawning a new pxe - await sleep(2000); + // Wait until the sequencer kicks out tx1 + logger.info(`Waiting for node to prune tx1`); + await retryUntil( + async () => (await aztecNode.getTxReceipt(tx1.txHash)).status === TxStatus.PENDING, + 'wait for pruning', + 15, + 1, + ); - // tx1 is valid because it was build against a proven block number - // the sequencer will bring it back on chain + // And wait until it is brought back tx1 + logger.info(`Waiting for node to re-include tx1`); await retryUntil( async () => (await aztecNode.getTxReceipt(tx1.txHash)).status === TxStatus.SUCCESS, 'wait for re-inclusion', - 60, + 15, 1, ); + // Tx1 should have been mined in a block with the same number but different hash now const newTx1Receipt = await aztecNode.getTxReceipt(tx1.txHash); expect(newTx1Receipt.blockNumber).toEqual(tx1.blockNumber); expect(newTx1Receipt.blockHash).not.toEqual(tx1.blockHash); - // Send another tx which should be mined a block that is built on the reorg'd chain - // We need to send it from a new pxe since pxe doesn't detect reorgs (yet) - logger.info(`Creating new PXE service`); - const pxeServiceConfig = { ...getPXEServiceConfig() }; - const newPxe = await createPXEService(aztecNode, pxeServiceConfig); - const newWallet = await createAccount(newPxe); + // PXE should have cleared out the 30-note from tx2, but reapplied the 20-note from tx1 + expect(await contract.methods.summed_values(ownerAddress).simulate()).toEqual(21n); - // TODO: Contract.at should automatically register the instance in the pxe - logger.info(`Registering contract at ${contract.address} in new pxe`); - await newPxe.registerContract({ instance: contract.instance, artifact: StatefulTestContractArtifact }); - const contractFromNewPxe = await StatefulTestContract.at(contract.address, newWallet); + // PXE should be synced to the block number on the new chain + await retryUntil( + async () => (await pxe.getSyncStatus()).blocks === newTx1Receipt.blockNumber, + 'wait for pxe block header sync', + 15, + 1, + ); + // And we should be able to send a new tx on the new chain logger.info('Sending new tx on reorgd chain'); - const tx2 = await contractFromNewPxe.methods.increment_public_value(ownerAddress, 10).send().wait(); - expect(await contractFromNewPxe.methods.get_public_value(ownerAddress).simulate()).toEqual(30n); - expect(tx2.blockNumber).toEqual(initialBlockNumber + 3); + const tx3 = await contract.methods.create_note(ownerAddress, ownerAddress, 10).send().wait(); + expect(await contract.methods.summed_values(ownerAddress).simulate()).toEqual(31n); + expect(tx3.blockNumber).toBeGreaterThanOrEqual(newTx1Receipt.blockNumber! + 1); }); }); }); diff --git a/yarn-project/end-to-end/src/e2e_event_logs.test.ts b/yarn-project/end-to-end/src/e2e_event_logs.test.ts index ab8d4f88785..162d53d6eb0 100644 --- a/yarn-project/end-to-end/src/e2e_event_logs.test.ts +++ b/yarn-project/end-to-end/src/e2e_event_logs.test.ts @@ -48,7 +48,7 @@ describe('Logs', () => { const txEffect = await node.getTxEffect(tx.txHash); - const encryptedLogs = txEffect!.encryptedLogs.unrollLogs(); + const encryptedLogs = txEffect!.data.encryptedLogs.unrollLogs(); expect(encryptedLogs.length).toBe(3); const decryptedEvent0 = L1EventPayload.decryptAsIncoming(encryptedLogs[0], wallets[0].getEncryptionSecret())!; diff --git a/yarn-project/kv-store/package.json b/yarn-project/kv-store/package.json index ae05345eeaa..f216507a2da 100644 --- a/yarn-project/kv-store/package.json +++ b/yarn-project/kv-store/package.json @@ -5,7 +5,8 @@ "exports": { ".": "./dest/interfaces/index.js", "./lmdb": "./dest/lmdb/index.js", - "./utils": "./dest/utils.js" + "./utils": "./dest/utils.js", + "./stores": "./dest/stores/index.js" }, "scripts": { "build": "yarn clean && tsc -b", @@ -55,10 +56,12 @@ ] }, "dependencies": { + "@aztec/circuit-types": "workspace:^", "@aztec/foundation": "workspace:^", "lmdb": "^3.0.6" }, "devDependencies": { + "@aztec/circuits.js": "workspace:^", "@jest/globals": "^29.5.0", "@types/jest": "^29.5.0", "@types/node": "^18.7.23", @@ -75,4 +78,4 @@ "engines": { "node": ">=18" } -} +} \ No newline at end of file diff --git a/yarn-project/kv-store/src/stores/index.ts b/yarn-project/kv-store/src/stores/index.ts new file mode 100644 index 00000000000..c279b4ba628 --- /dev/null +++ b/yarn-project/kv-store/src/stores/index.ts @@ -0,0 +1 @@ +export * from './l2_tips_store.js'; diff --git a/yarn-project/kv-store/src/stores/l2_tips_store.test.ts b/yarn-project/kv-store/src/stores/l2_tips_store.test.ts new file mode 100644 index 00000000000..2b820aaf432 --- /dev/null +++ b/yarn-project/kv-store/src/stores/l2_tips_store.test.ts @@ -0,0 +1,71 @@ +import { type L2Block } from '@aztec/circuit-types'; +import { Fr, type Header } from '@aztec/circuits.js'; +import { times } from '@aztec/foundation/collection'; +import { type AztecKVStore } from '@aztec/kv-store'; +import { openTmpStore } from '@aztec/kv-store/utils'; + +import { L2TipsStore } from './l2_tips_store.js'; + +describe('L2TipsStore', () => { + let kvStore: AztecKVStore; + let tipsStore: L2TipsStore; + + beforeEach(() => { + kvStore = openTmpStore(true); + tipsStore = new L2TipsStore(kvStore, 'test'); + }); + + const makeBlock = (number: number): L2Block => + ({ number, header: { hash: () => new Fr(number) } as Header } as L2Block); + + const makeTip = (number: number) => ({ number, hash: number === 0 ? undefined : new Fr(number).toString() }); + + const makeTips = (latest: number, proven: number, finalized: number) => ({ + latest: makeTip(latest), + proven: makeTip(proven), + finalized: makeTip(finalized), + }); + + it('returns zero if no tips are stored', async () => { + const tips = await tipsStore.getL2Tips(); + expect(tips).toEqual(makeTips(0, 0, 0)); + }); + + it('stores chain tips', async () => { + await tipsStore.handleBlockStreamEvent({ type: 'blocks-added', blocks: times(20, i => makeBlock(i + 1)) }); + + await tipsStore.handleBlockStreamEvent({ type: 'chain-finalized', blockNumber: 5 }); + await tipsStore.handleBlockStreamEvent({ type: 'chain-proven', blockNumber: 8 }); + await tipsStore.handleBlockStreamEvent({ type: 'chain-pruned', blockNumber: 10 }); + + const tips = await tipsStore.getL2Tips(); + expect(tips).toEqual(makeTips(10, 8, 5)); + }); + + it('sets latest tip from blocks added', async () => { + await tipsStore.handleBlockStreamEvent({ type: 'blocks-added', blocks: times(3, i => makeBlock(i + 1)) }); + + const tips = await tipsStore.getL2Tips(); + expect(tips).toEqual(makeTips(3, 0, 0)); + + expect(await tipsStore.getL2BlockHash(1)).toEqual(new Fr(1).toString()); + expect(await tipsStore.getL2BlockHash(2)).toEqual(new Fr(2).toString()); + expect(await tipsStore.getL2BlockHash(3)).toEqual(new Fr(3).toString()); + }); + + it('clears block hashes when setting finalized chain', async () => { + await tipsStore.handleBlockStreamEvent({ type: 'blocks-added', blocks: times(5, i => makeBlock(i + 1)) }); + await tipsStore.handleBlockStreamEvent({ type: 'chain-proven', blockNumber: 3 }); + await tipsStore.handleBlockStreamEvent({ type: 'chain-finalized', blockNumber: 3 }); + + const tips = await tipsStore.getL2Tips(); + expect(tips).toEqual(makeTips(5, 3, 3)); + + expect(await tipsStore.getL2BlockHash(1)).toBeUndefined(); + expect(await tipsStore.getL2BlockHash(2)).toBeUndefined(); + + expect(await tipsStore.getL2BlockHash(3)).toEqual(new Fr(3).toString()); + expect(await tipsStore.getL2BlockHash(4)).toEqual(new Fr(4).toString()); + expect(await tipsStore.getL2BlockHash(5)).toEqual(new Fr(5).toString()); + }); +}); diff --git a/yarn-project/kv-store/src/stores/l2_tips_store.ts b/yarn-project/kv-store/src/stores/l2_tips_store.ts new file mode 100644 index 00000000000..36180272967 --- /dev/null +++ b/yarn-project/kv-store/src/stores/l2_tips_store.ts @@ -0,0 +1,69 @@ +import { + type L2BlockId, + type L2BlockStreamEvent, + type L2BlockStreamEventHandler, + type L2BlockStreamLocalDataProvider, + type L2BlockTag, + type L2Tips, +} from '@aztec/circuit-types'; + +import { type AztecMap } from '../interfaces/map.js'; +import { type AztecKVStore } from '../interfaces/store.js'; + +/** Stores currently synced L2 tips and unfinalized block hashes. */ +export class L2TipsStore implements L2BlockStreamEventHandler, L2BlockStreamLocalDataProvider { + private readonly l2TipsStore: AztecMap; + private readonly l2BlockHashesStore: AztecMap; + + constructor(store: AztecKVStore, namespace: string) { + this.l2TipsStore = store.openMap([namespace, 'l2_tips'].join('_')); + this.l2BlockHashesStore = store.openMap([namespace, 'l2_block_hashes'].join('_')); + } + + public getL2BlockHash(number: number): Promise { + return Promise.resolve(this.l2BlockHashesStore.get(number)); + } + + public getL2Tips(): Promise { + return Promise.resolve({ + latest: this.getL2Tip('latest'), + finalized: this.getL2Tip('finalized'), + proven: this.getL2Tip('proven'), + }); + } + + private getL2Tip(tag: L2BlockTag): L2BlockId { + const blockNumber = this.l2TipsStore.get(tag); + if (blockNumber === undefined) { + return { number: 0, hash: undefined }; + } + const blockHash = this.l2BlockHashesStore.get(blockNumber); + if (!blockHash) { + throw new Error(`Block hash not found for block number ${blockNumber}`); + } + return { number: blockNumber, hash: blockHash }; + } + + public async handleBlockStreamEvent(event: L2BlockStreamEvent): Promise { + switch (event.type) { + case 'blocks-added': + await this.l2TipsStore.set('latest', event.blocks.at(-1)!.number); + for (const block of event.blocks) { + await this.l2BlockHashesStore.set(block.number, block.header.hash().toString()); + } + break; + case 'chain-pruned': + await this.l2TipsStore.set('latest', event.blockNumber); + break; + case 'chain-proven': + await this.l2TipsStore.set('proven', event.blockNumber); + break; + case 'chain-finalized': + await this.l2TipsStore.set('finalized', event.blockNumber); + for (const key of this.l2BlockHashesStore.keys({ end: event.blockNumber })) { + await this.l2BlockHashesStore.delete(key); + } + break; + } + } +} diff --git a/yarn-project/kv-store/tsconfig.json b/yarn-project/kv-store/tsconfig.json index 63f8ab3e9f7..d2bc1df097f 100644 --- a/yarn-project/kv-store/tsconfig.json +++ b/yarn-project/kv-store/tsconfig.json @@ -6,8 +6,14 @@ "tsBuildInfoFile": ".tsbuildinfo" }, "references": [ + { + "path": "../circuit-types" + }, { "path": "../foundation" + }, + { + "path": "../circuits.js" } ], "include": ["src"] diff --git a/yarn-project/pxe/src/database/incoming_note_dao.test.ts b/yarn-project/pxe/src/database/incoming_note_dao.test.ts index a3cb0d5345c..1df9103e08f 100644 --- a/yarn-project/pxe/src/database/incoming_note_dao.test.ts +++ b/yarn-project/pxe/src/database/incoming_note_dao.test.ts @@ -1,38 +1,8 @@ -import { Note, randomTxHash } from '@aztec/circuit-types'; -import { AztecAddress, Fr, Point } from '@aztec/circuits.js'; -import { NoteSelector } from '@aztec/foundation/abi'; - import { IncomingNoteDao } from './incoming_note_dao.js'; -export const randomIncomingNoteDao = ({ - note = Note.random(), - contractAddress = AztecAddress.random(), - txHash = randomTxHash(), - storageSlot = Fr.random(), - noteTypeId = NoteSelector.random(), - nonce = Fr.random(), - noteHash = Fr.random(), - siloedNullifier = Fr.random(), - index = Fr.random().toBigInt(), - addressPoint = Point.random(), -}: Partial = {}) => { - return new IncomingNoteDao( - note, - contractAddress, - storageSlot, - noteTypeId, - txHash, - nonce, - noteHash, - siloedNullifier, - index, - addressPoint, - ); -}; - describe('Incoming Note DAO', () => { it('convert to and from buffer', () => { - const note = randomIncomingNoteDao(); + const note = IncomingNoteDao.random(); const buf = note.toBuffer(); expect(IncomingNoteDao.fromBuffer(buf)).toEqual(note); }); diff --git a/yarn-project/pxe/src/database/incoming_note_dao.ts b/yarn-project/pxe/src/database/incoming_note_dao.ts index cbd344b135c..d2dc2d38815 100644 --- a/yarn-project/pxe/src/database/incoming_note_dao.ts +++ b/yarn-project/pxe/src/database/incoming_note_dao.ts @@ -1,4 +1,4 @@ -import { type L1NotePayload, Note, TxHash } from '@aztec/circuit-types'; +import { type L1NotePayload, Note, TxHash, randomTxHash } from '@aztec/circuit-types'; import { AztecAddress, Fr, Point, type PublicKey } from '@aztec/circuits.js'; import { NoteSelector } from '@aztec/foundation/abi'; import { toBigIntBE } from '@aztec/foundation/bigint-buffer'; @@ -22,6 +22,10 @@ export class IncomingNoteDao implements NoteData { public noteTypeId: NoteSelector, /** The hash of the tx the note was created in. */ public txHash: TxHash, + /** The L2 block number in which the tx with this note was included. */ + public l2BlockNumber: number, + /** The L2 block hash in which the tx with this note was included. */ + public l2BlockHash: string, /** The nonce of the note. */ public nonce: Fr, /** @@ -44,6 +48,8 @@ export class IncomingNoteDao implements NoteData { note: Note, payload: L1NotePayload, noteInfo: NoteInfo, + l2BlockNumber: number, + l2BlockHash: string, dataStartIndexForTx: number, addressPoint: PublicKey, ) { @@ -54,6 +60,8 @@ export class IncomingNoteDao implements NoteData { payload.storageSlot, payload.noteTypeId, noteInfo.txHash, + l2BlockNumber, + l2BlockHash, noteInfo.nonce, noteInfo.noteHash, noteInfo.siloedNullifier, @@ -69,6 +77,8 @@ export class IncomingNoteDao implements NoteData { this.storageSlot, this.noteTypeId, this.txHash.buffer, + this.l2BlockNumber, + Fr.fromString(this.l2BlockHash), this.nonce, this.noteHash, this.siloedNullifier, @@ -76,6 +86,7 @@ export class IncomingNoteDao implements NoteData { this.addressPoint, ]); } + static fromBuffer(buffer: Buffer | BufferReader) { const reader = BufferReader.asReader(buffer); @@ -84,6 +95,8 @@ export class IncomingNoteDao implements NoteData { const storageSlot = Fr.fromBuffer(reader); const noteTypeId = reader.readObject(NoteSelector); const txHash = reader.readObject(TxHash); + const l2BlockNumber = reader.readNumber(); + const l2BlockHash = Fr.fromBuffer(reader).toString(); const nonce = Fr.fromBuffer(reader); const noteHash = Fr.fromBuffer(reader); const siloedNullifier = Fr.fromBuffer(reader); @@ -96,6 +109,8 @@ export class IncomingNoteDao implements NoteData { storageSlot, noteTypeId, txHash, + l2BlockNumber, + l2BlockHash, nonce, noteHash, siloedNullifier, @@ -122,4 +137,34 @@ export class IncomingNoteDao implements NoteData { const noteSize = 4 + this.note.items.length * Fr.SIZE_IN_BYTES; return noteSize + AztecAddress.SIZE_IN_BYTES + Fr.SIZE_IN_BYTES * 4 + TxHash.SIZE + Point.SIZE_IN_BYTES + indexSize; } + + static random({ + note = Note.random(), + contractAddress = AztecAddress.random(), + txHash = randomTxHash(), + storageSlot = Fr.random(), + noteTypeId = NoteSelector.random(), + nonce = Fr.random(), + l2BlockNumber = Math.floor(Math.random() * 1000), + l2BlockHash = Fr.random().toString(), + noteHash = Fr.random(), + siloedNullifier = Fr.random(), + index = Fr.random().toBigInt(), + addressPoint = Point.random(), + }: Partial = {}) { + return new IncomingNoteDao( + note, + contractAddress, + storageSlot, + noteTypeId, + txHash, + l2BlockNumber, + l2BlockHash, + nonce, + noteHash, + siloedNullifier, + index, + addressPoint, + ); + } } diff --git a/yarn-project/pxe/src/database/kv_pxe_database.ts b/yarn-project/pxe/src/database/kv_pxe_database.ts index 2fea8a0452b..90ccc3796e3 100644 --- a/yarn-project/pxe/src/database/kv_pxe_database.ts +++ b/yarn-project/pxe/src/database/kv_pxe_database.ts @@ -214,6 +214,41 @@ export class KVPxeDatabase implements PxeDatabase { }); } + public removeNotesAfter(blockNumber: number): Promise { + return this.db.transaction(() => { + for (const note of this.#notes.values()) { + const noteDao = IncomingNoteDao.fromBuffer(note); + if (noteDao.l2BlockNumber > blockNumber) { + const noteIndex = toBufferBE(noteDao.index, 32).toString('hex'); + void this.#notes.delete(noteIndex); + void this.#nullifierToNoteId.delete(noteDao.siloedNullifier.toString()); + for (const scope of this.#scopes.entries()) { + void this.#notesByAddressPointAndScope.get(scope)!.deleteValue(noteDao.addressPoint.toString(), noteIndex); + void this.#notesByTxHashAndScope.get(scope)!.deleteValue(noteDao.txHash.toString(), noteIndex); + void this.#notesByContractAndScope.get(scope)!.deleteValue(noteDao.contractAddress.toString(), noteIndex); + void this.#notesByStorageSlotAndScope.get(scope)!.deleteValue(noteDao.storageSlot.toString(), noteIndex); + } + } + } + + for (const note of this.#outgoingNotes.values()) { + const noteDao = OutgoingNoteDao.fromBuffer(note); + if (noteDao.l2BlockNumber > blockNumber) { + const noteIndex = toBufferBE(noteDao.index, 32).toString('hex'); + void this.#outgoingNotes.delete(noteIndex); + void this.#outgoingNotesByContract.deleteValue(noteDao.contractAddress.toString(), noteIndex); + void this.#outgoingNotesByStorageSlot.deleteValue(noteDao.storageSlot.toString(), noteIndex); + void this.#outgoingNotesByTxHash.deleteValue(noteDao.txHash.toString(), noteIndex); + void this.#outgoingNotesByOvpkM.deleteValue(noteDao.ovpkM.toString(), noteIndex); + } + } + }); + } + + public async unnullifyNotesAfter(_blockNumber: number): Promise { + // TODO: Implementme. Requires tracking when a note was nullified. + } + getIncomingNotes(filter: IncomingNotesFilter): Promise { const publicKey: PublicKey | undefined = filter.owner ? computePoint(filter.owner) : undefined; diff --git a/yarn-project/pxe/src/database/outgoing_note_dao.test.ts b/yarn-project/pxe/src/database/outgoing_note_dao.test.ts index 04b2b2201f7..0c293ba13eb 100644 --- a/yarn-project/pxe/src/database/outgoing_note_dao.test.ts +++ b/yarn-project/pxe/src/database/outgoing_note_dao.test.ts @@ -1,26 +1,8 @@ -import { Note, randomTxHash } from '@aztec/circuit-types'; -import { AztecAddress, Fr, Point } from '@aztec/circuits.js'; -import { NoteSelector } from '@aztec/foundation/abi'; - import { OutgoingNoteDao } from './outgoing_note_dao.js'; -export const randomOutgoingNoteDao = ({ - note = Note.random(), - contractAddress = AztecAddress.random(), - txHash = randomTxHash(), - storageSlot = Fr.random(), - noteTypeId = NoteSelector.random(), - nonce = Fr.random(), - noteHash = Fr.random(), - index = Fr.random().toBigInt(), - ovpkM = Point.random(), -}: Partial = {}) => { - return new OutgoingNoteDao(note, contractAddress, storageSlot, noteTypeId, txHash, nonce, noteHash, index, ovpkM); -}; - describe('Outgoing Note DAO', () => { it('convert to and from buffer', () => { - const note = randomOutgoingNoteDao(); + const note = OutgoingNoteDao.random(); const buf = note.toBuffer(); expect(OutgoingNoteDao.fromBuffer(buf)).toEqual(note); }); diff --git a/yarn-project/pxe/src/database/outgoing_note_dao.ts b/yarn-project/pxe/src/database/outgoing_note_dao.ts index 04bb7d4835c..386b23ecd57 100644 --- a/yarn-project/pxe/src/database/outgoing_note_dao.ts +++ b/yarn-project/pxe/src/database/outgoing_note_dao.ts @@ -1,4 +1,4 @@ -import { type L1NotePayload, Note, TxHash } from '@aztec/circuit-types'; +import { type L1NotePayload, Note, TxHash, randomTxHash } from '@aztec/circuit-types'; import { AztecAddress, Fr, Point, type PublicKey } from '@aztec/circuits.js'; import { NoteSelector } from '@aztec/foundation/abi'; import { toBigIntBE } from '@aztec/foundation/bigint-buffer'; @@ -21,6 +21,10 @@ export class OutgoingNoteDao { public noteTypeId: NoteSelector, /** The hash of the tx the note was created in. */ public txHash: TxHash, + /** The L2 block number in which the tx with this note was included. */ + public l2BlockNumber: number, + /** The L2 block hash in which the tx with this note was included. */ + public l2BlockHash: string, /** The nonce of the note. */ public nonce: Fr, /** @@ -38,6 +42,8 @@ export class OutgoingNoteDao { note: Note, payload: L1NotePayload, noteInfo: NoteInfo, + l2BlockNumber: number, + l2BlockHash: string, dataStartIndexForTx: number, ovpkM: PublicKey, ) { @@ -48,6 +54,8 @@ export class OutgoingNoteDao { payload.storageSlot, payload.noteTypeId, noteInfo.txHash, + l2BlockNumber, + l2BlockHash, noteInfo.nonce, noteInfo.noteHash, noteHashIndexInTheWholeTree, @@ -62,6 +70,8 @@ export class OutgoingNoteDao { this.storageSlot, this.noteTypeId, this.txHash.buffer, + this.l2BlockNumber, + Fr.fromString(this.l2BlockHash), this.nonce, this.noteHash, this.index, @@ -76,6 +86,8 @@ export class OutgoingNoteDao { const storageSlot = Fr.fromBuffer(reader); const noteTypeId = reader.readObject(NoteSelector); const txHash = new TxHash(reader.readBytes(TxHash.SIZE)); + const l2BlockNumber = reader.readNumber(); + const l2BlockHash = Fr.fromBuffer(reader).toString(); const nonce = Fr.fromBuffer(reader); const noteHash = Fr.fromBuffer(reader); const index = toBigIntBE(reader.readBytes(32)); @@ -87,6 +99,8 @@ export class OutgoingNoteDao { storageSlot, noteTypeId, txHash, + l2BlockNumber, + l2BlockHash, nonce, noteHash, index, @@ -111,4 +125,32 @@ export class OutgoingNoteDao { const noteSize = 4 + this.note.items.length * Fr.SIZE_IN_BYTES; return noteSize + AztecAddress.SIZE_IN_BYTES + Fr.SIZE_IN_BYTES * 2 + TxHash.SIZE + Point.SIZE_IN_BYTES; } + + static random({ + note = Note.random(), + contractAddress = AztecAddress.random(), + txHash = randomTxHash(), + storageSlot = Fr.random(), + noteTypeId = NoteSelector.random(), + nonce = Fr.random(), + l2BlockNumber = Math.floor(Math.random() * 1000), + l2BlockHash = Fr.random().toString(), + noteHash = Fr.random(), + index = Fr.random().toBigInt(), + ovpkM = Point.random(), + }: Partial = {}) { + return new OutgoingNoteDao( + note, + contractAddress, + storageSlot, + noteTypeId, + txHash, + l2BlockNumber, + l2BlockHash, + nonce, + noteHash, + index, + ovpkM, + ); + } } diff --git a/yarn-project/pxe/src/database/pxe_database.ts b/yarn-project/pxe/src/database/pxe_database.ts index d4cdb12bcec..58e39901da3 100644 --- a/yarn-project/pxe/src/database/pxe_database.ts +++ b/yarn-project/pxe/src/database/pxe_database.ts @@ -214,4 +214,16 @@ export interface PxeDatabase extends ContractArtifactDatabase, ContractInstanceD * @param appTaggingSecrets - The app siloed tagging secrets. */ setTaggingSecretsIndexesAsRecipient(indexedTaggingSecrets: IndexedTaggingSecret[]): Promise; + + /** + * Deletes all notes synched after this block number. + * @param blockNumber - All notes strictly after this block number are removed. + */ + removeNotesAfter(blockNumber: number): Promise; + + /** + * Restores notes nullified after the given block. + * @param blockNumber - All nullifiers strictly after this block are removed. + */ + unnullifyNotesAfter(blockNumber: number): Promise; } diff --git a/yarn-project/pxe/src/database/pxe_database_test_suite.ts b/yarn-project/pxe/src/database/pxe_database_test_suite.ts index 1fefa614bad..b0dfd5272d6 100644 --- a/yarn-project/pxe/src/database/pxe_database_test_suite.ts +++ b/yarn-project/pxe/src/database/pxe_database_test_suite.ts @@ -12,10 +12,8 @@ import { randomInt } from '@aztec/foundation/crypto'; import { Fr, Point } from '@aztec/foundation/fields'; import { BenchmarkingContractArtifact } from '@aztec/noir-contracts.js/Benchmarking'; -import { type IncomingNoteDao } from './incoming_note_dao.js'; -import { randomIncomingNoteDao } from './incoming_note_dao.test.js'; -import { type OutgoingNoteDao } from './outgoing_note_dao.js'; -import { randomOutgoingNoteDao } from './outgoing_note_dao.test.js'; +import { IncomingNoteDao } from './incoming_note_dao.js'; +import { OutgoingNoteDao } from './outgoing_note_dao.js'; import { type PxeDatabase } from './pxe_database.js'; /** @@ -121,11 +119,12 @@ export function describePxeDatabase(getDatabase: () => PxeDatabase) { storageSlots = Array.from({ length: 2 }).map(() => Fr.random()); notes = Array.from({ length: 10 }).map((_, i) => - randomIncomingNoteDao({ + IncomingNoteDao.random({ contractAddress: contractAddresses[i % contractAddresses.length], storageSlot: storageSlots[i % storageSlots.length], addressPoint: computePoint(owners[i % owners.length].address), index: BigInt(i), + l2BlockNumber: i, }), ); @@ -260,6 +259,14 @@ export function describePxeDatabase(getDatabase: () => PxeDatabase) { }), ).resolves.toEqual([]); }); + + it('removes notes after a given block', async () => { + await database.addNotes(notes, [], owners[0].address); + + await database.removeNotesAfter(5); + const result = await database.getIncomingNotes({ scopes: [owners[0].address] }); + expect(new Set(result)).toEqual(new Set(notes.slice(0, 6))); + }); }); describe('outgoing notes', () => { @@ -307,7 +314,7 @@ export function describePxeDatabase(getDatabase: () => PxeDatabase) { storageSlots = Array.from({ length: 2 }).map(() => Fr.random()); notes = Array.from({ length: 10 }).map((_, i) => - randomOutgoingNoteDao({ + OutgoingNoteDao.random({ contractAddress: contractAddresses[i % contractAddresses.length], storageSlot: storageSlots[i % storageSlots.length], ovpkM: owners[i % owners.length].publicKeys.masterOutgoingViewingPublicKey, diff --git a/yarn-project/pxe/src/kernel_oracle/index.ts b/yarn-project/pxe/src/kernel_oracle/index.ts index d2115d2cc40..a66ec8db465 100644 --- a/yarn-project/pxe/src/kernel_oracle/index.ts +++ b/yarn-project/pxe/src/kernel_oracle/index.ts @@ -70,7 +70,7 @@ export class KernelOracle implements ProvingDataOracle { } async getNoteHashTreeRoot(): Promise { - const header = await this.node.getHeader(this.blockNumber); + const header = await this.node.getBlockHeader(this.blockNumber); return header.state.partial.noteHashTree.root; } diff --git a/yarn-project/pxe/src/note_decryption_utils/produce_note_daos.ts b/yarn-project/pxe/src/note_decryption_utils/produce_note_daos.ts index 8e857d51330..fc3e1918ce1 100644 --- a/yarn-project/pxe/src/note_decryption_utils/produce_note_daos.ts +++ b/yarn-project/pxe/src/note_decryption_utils/produce_note_daos.ts @@ -34,6 +34,8 @@ export async function produceNoteDaos( ovpkM: PublicKey | undefined, payload: L1NotePayload, txHash: TxHash, + l2BlockNumber: number, + l2BlockHash: string, noteHashes: Fr[], dataStartIndexForTx: number, excludedIndices: Set, @@ -56,6 +58,8 @@ export async function produceNoteDaos( addressPoint, payload, txHash, + l2BlockNumber, + l2BlockHash, noteHashes, dataStartIndexForTx, excludedIndices, @@ -74,6 +78,8 @@ export async function produceNoteDaos( incomingNote.storageSlot, incomingNote.noteTypeId, incomingNote.txHash, + incomingNote.l2BlockNumber, + incomingNote.l2BlockHash, incomingNote.nonce, incomingNote.noteHash, incomingNote.index, @@ -86,6 +92,8 @@ export async function produceNoteDaos( ovpkM, payload, txHash, + l2BlockNumber, + l2BlockHash, noteHashes, dataStartIndexForTx, excludedIndices, diff --git a/yarn-project/pxe/src/note_decryption_utils/produce_note_daos_for_key.ts b/yarn-project/pxe/src/note_decryption_utils/produce_note_daos_for_key.ts index 9e530b387d1..eeeb6c9ee9e 100644 --- a/yarn-project/pxe/src/note_decryption_utils/produce_note_daos_for_key.ts +++ b/yarn-project/pxe/src/note_decryption_utils/produce_note_daos_for_key.ts @@ -13,6 +13,8 @@ export async function produceNoteDaosForKey( pkM: PublicKey, payload: L1NotePayload, txHash: TxHash, + l2BlockNumber: number, + l2BlockHash: string, noteHashes: Fr[], dataStartIndexForTx: number, excludedIndices: Set, @@ -21,6 +23,8 @@ export async function produceNoteDaosForKey( note: Note, payload: L1NotePayload, noteInfo: NoteInfo, + l2BlockNumber: number, + l2BlockHash: string, dataStartIndexForTx: number, pkM: PublicKey, ) => T, @@ -44,7 +48,7 @@ export async function produceNoteDaosForKey( ); excludedIndices?.add(noteInfo.noteHashIndex); - noteDao = daoConstructor(note, payload, noteInfo, dataStartIndexForTx, pkM); + noteDao = daoConstructor(note, payload, noteInfo, l2BlockNumber, l2BlockHash, dataStartIndexForTx, pkM); } catch (e) { logger.error(`Could not process note because of "${e}". Discarding note...`); } diff --git a/yarn-project/pxe/src/pxe_service/create_pxe_service.ts b/yarn-project/pxe/src/pxe_service/create_pxe_service.ts index bb608b93873..c4e4763e131 100644 --- a/yarn-project/pxe/src/pxe_service/create_pxe_service.ts +++ b/yarn-project/pxe/src/pxe_service/create_pxe_service.ts @@ -3,6 +3,7 @@ import { type AztecNode, type PrivateKernelProver } from '@aztec/circuit-types'; import { randomBytes } from '@aztec/foundation/crypto'; import { createDebugLogger } from '@aztec/foundation/log'; import { KeyStore } from '@aztec/key-store'; +import { L2TipsStore } from '@aztec/kv-store/stores'; import { createStore } from '@aztec/kv-store/utils'; import { type PXEServiceConfig } from '../config/index.js'; @@ -35,10 +36,12 @@ export async function createPXEService( const keyStore = new KeyStore( await createStore('pxe_key_store', storeConfig, createDebugLogger('aztec:pxe:keystore:lmdb')), ); - const db = new KVPxeDatabase(await createStore('pxe_data', storeConfig, createDebugLogger('aztec:pxe:data:lmdb'))); + const store = await createStore('pxe_data', storeConfig, createDebugLogger('aztec:pxe:data:lmdb')); + const db = new KVPxeDatabase(store); + const tips = new L2TipsStore(store, 'pxe'); const prover = proofCreator ?? (await createProver(config, logSuffix)); - const server = new PXEService(keyStore, aztecNode, db, prover, config, logSuffix); + const server = new PXEService(keyStore, aztecNode, db, tips, prover, config, logSuffix); await server.start(); return server; } diff --git a/yarn-project/pxe/src/pxe_service/pxe_service.ts b/yarn-project/pxe/src/pxe_service/pxe_service.ts index 62e96081780..cc73dfcd8dd 100644 --- a/yarn-project/pxe/src/pxe_service/pxe_service.ts +++ b/yarn-project/pxe/src/pxe_service/pxe_service.ts @@ -6,6 +6,7 @@ import { type ExtendedNote, type FunctionCall, type GetUnencryptedLogsResponse, + type InBlock, type IncomingNotesFilter, L1EventPayload, type L2Block, @@ -22,7 +23,6 @@ import { type SiblingPath, SimulationError, type Tx, - type TxEffect, type TxExecutionRequest, type TxHash, TxProvingResult, @@ -58,6 +58,7 @@ import { Fr, type Point } from '@aztec/foundation/fields'; import { type DebugLogger, createDebugLogger } from '@aztec/foundation/log'; import { SerialQueue } from '@aztec/foundation/queue'; import { type KeyStore } from '@aztec/key-store'; +import { type L2TipsStore } from '@aztec/kv-store/stores'; import { ProtocolContractAddress, getCanonicalProtocolContract, @@ -93,12 +94,13 @@ export class PXEService implements PXE { private keyStore: KeyStore, private node: AztecNode, private db: PxeDatabase, + tipsStore: L2TipsStore, private proofCreator: PrivateKernelProver, - private config: PXEServiceConfig, + config: PXEServiceConfig, logSuffix?: string, ) { this.log = createDebugLogger(logSuffix ? `aztec:pxe_service_${logSuffix}` : `aztec:pxe_service`); - this.synchronizer = new Synchronizer(node, db, this.jobQueue, logSuffix); + this.synchronizer = new Synchronizer(node, db, tipsStore, config, logSuffix); this.contractDataOracle = new ContractDataOracle(db); this.simulator = getAcirSimulator(db, node, keyStore, this.contractDataOracle); this.packageVersion = getPackageInfo().version; @@ -112,8 +114,7 @@ export class PXEService implements PXE { * @returns A promise that resolves when the server has started successfully. */ public async start() { - const { l2BlockPollingIntervalMS } = this.config; - await this.synchronizer.start(1, l2BlockPollingIntervalMS); + await this.synchronizer.start(); await this.#registerProtocolContracts(); const info = await this.getNodeInfo(); this.log.info(`Started PXE connected to chain ${info.l1ChainId} version ${info.protocolVersion}`); @@ -350,7 +351,7 @@ export class PXEService implements PXE { throw new Error(`Unknown account: ${note.owner.toString()}`); } - const nonces = await this.#getNoteNonces(note); + const { data: nonces, l2BlockNumber, l2BlockHash } = await this.#getNoteNonces(note); if (nonces.length === 0) { throw new Error(`Cannot find the note in tx: ${note.txHash}.`); } @@ -385,6 +386,8 @@ export class PXEService implements PXE { note.storageSlot, note.noteTypeId, note.txHash, + l2BlockNumber, + l2BlockHash, nonce, noteHash, siloedNullifier, @@ -397,7 +400,7 @@ export class PXEService implements PXE { } public async addNullifiedNote(note: ExtendedNote) { - const nonces = await this.#getNoteNonces(note); + const { data: nonces, l2BlockHash, l2BlockNumber } = await this.#getNoteNonces(note); if (nonces.length === 0) { throw new Error(`Cannot find the note in tx: ${note.txHash}.`); } @@ -428,6 +431,8 @@ export class PXEService implements PXE { note.storageSlot, note.noteTypeId, note.txHash, + l2BlockNumber, + l2BlockHash, nonce, noteHash, Fr.ZERO, // We are not able to derive @@ -444,15 +449,15 @@ export class PXEService implements PXE { * @returns The nonces of the note. * @remarks More than a single nonce may be returned since there might be more than one nonce for a given note. */ - async #getNoteNonces(note: ExtendedNote): Promise { + async #getNoteNonces(note: ExtendedNote): Promise> { const tx = await this.node.getTxEffect(note.txHash); if (!tx) { throw new Error(`Unknown tx: ${note.txHash}`); } const nonces: Fr[] = []; - const firstNullifier = tx.nullifiers[0]; - const hashes = tx.noteHashes; + const firstNullifier = tx.data.nullifiers[0]; + const hashes = tx.data.noteHashes; for (let i = 0; i < hashes.length; ++i) { const hash = hashes[i]; if (hash.equals(Fr.ZERO)) { @@ -473,7 +478,7 @@ export class PXEService implements PXE { } } - return nonces; + return { l2BlockHash: tx.l2BlockHash, l2BlockNumber: tx.l2BlockNumber, data: nonces }; } public async getBlock(blockNumber: number): Promise { @@ -587,7 +592,7 @@ export class PXEService implements PXE { return this.node.getTxReceipt(txHash); } - public getTxEffect(txHash: TxHash): Promise { + public getTxEffect(txHash: TxHash) { return this.node.getTxEffect(txHash); } diff --git a/yarn-project/pxe/src/pxe_service/test/pxe_service.test.ts b/yarn-project/pxe/src/pxe_service/test/pxe_service.test.ts index ca0d00de0db..90d3bb2ce0d 100644 --- a/yarn-project/pxe/src/pxe_service/test/pxe_service.test.ts +++ b/yarn-project/pxe/src/pxe_service/test/pxe_service.test.ts @@ -1,8 +1,9 @@ -import { type AztecNode, type PXE, TxEffect, mockTx } from '@aztec/circuit-types'; +import { type AztecNode, type PXE, TxEffect, mockTx, randomInBlock } from '@aztec/circuit-types'; import { INITIAL_L2_BLOCK_NUM } from '@aztec/circuits.js/constants'; import { type L1ContractAddresses } from '@aztec/ethereum'; import { EthAddress } from '@aztec/foundation/eth-address'; import { KeyStore } from '@aztec/key-store'; +import { L2TipsStore } from '@aztec/kv-store/stores'; import { openTmpStore } from '@aztec/kv-store/utils'; import { type MockProxy, mock } from 'jest-mock-extended'; @@ -19,6 +20,7 @@ function createPXEService(): Promise { const keyStore = new KeyStore(kvStore); const node = mock(); const db = new KVPxeDatabase(kvStore); + const tips = new L2TipsStore(kvStore, 'pxe'); const config: PXEServiceConfig = { l2BlockPollingIntervalMS: 100, l2StartingBlock: INITIAL_L2_BLOCK_NUM }; // Setup the relevant mocks @@ -39,7 +41,7 @@ function createPXEService(): Promise { }; node.getL1ContractAddresses.mockResolvedValue(mockedContracts); - return Promise.resolve(new PXEService(keyStore, node, db, new TestPrivateKernelProver(), config)); + return Promise.resolve(new PXEService(keyStore, node, db, tips, new TestPrivateKernelProver(), config)); } pxeTestSuite('PXEService', createPXEService); @@ -49,11 +51,13 @@ describe('PXEService', () => { let node: MockProxy; let db: PxeDatabase; let config: PXEServiceConfig; + let tips: L2TipsStore; beforeEach(() => { const kvStore = openTmpStore(); keyStore = new KeyStore(kvStore); node = mock(); + tips = new L2TipsStore(kvStore, 'pxe'); db = new KVPxeDatabase(kvStore); config = { l2BlockPollingIntervalMS: 100, l2StartingBlock: INITIAL_L2_BLOCK_NUM, proverEnabled: false }; }); @@ -62,9 +66,9 @@ describe('PXEService', () => { const settledTx = TxEffect.random(); const duplicateTx = mockTx(); - node.getTxEffect.mockResolvedValue(settledTx); + node.getTxEffect.mockResolvedValue(randomInBlock(settledTx)); - const pxe = new PXEService(keyStore, node, db, new TestPrivateKernelProver(), config); + const pxe = new PXEService(keyStore, node, db, tips, new TestPrivateKernelProver(), config); await expect(pxe.sendTx(duplicateTx)).rejects.toThrow(/A settled tx with equal hash/); }); }); diff --git a/yarn-project/pxe/src/simulator_oracle/index.ts b/yarn-project/pxe/src/simulator_oracle/index.ts index 73cf685b016..ed3e32df19d 100644 --- a/yarn-project/pxe/src/simulator_oracle/index.ts +++ b/yarn-project/pxe/src/simulator_oracle/index.ts @@ -1,5 +1,6 @@ import { type AztecNode, + InBlock, L1NotePayload, type L2Block, type L2BlockNumber, @@ -469,7 +470,7 @@ export class SimulatorOracle implements DBOracle { const incomingNotes: IncomingNoteDao[] = []; const outgoingNotes: OutgoingNoteDao[] = []; - const txEffectsCache = new Map(); + const txEffectsCache = new Map | undefined>(); for (const scopedLog of scopedLogs) { const incomingNotePayload = L1NotePayload.decryptAsIncoming( @@ -513,8 +514,10 @@ export class SimulatorOracle implements DBOracle { incomingNotePayload ? computePoint(recipient) : undefined, outgoingNotePayload ? recipientCompleteAddress.publicKeys.masterOutgoingViewingPublicKey : undefined, payload!, - txEffect.txHash, - txEffect.noteHashes, + txEffect.data.txHash, + txEffect.l2BlockNumber, + txEffect.l2BlockHash, + txEffect.data.noteHashes, scopedLog.dataStartIndexForTx, excludedIndices.get(scopedLog.txHash.toString())!, this.log, diff --git a/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts b/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts index f4630610b6e..bd51c6a7a2f 100644 --- a/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts +++ b/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts @@ -7,6 +7,7 @@ import { type TxEffect, TxHash, TxScopedL2Log, + randomInBlock, } from '@aztec/circuit-types'; import { AztecAddress, @@ -423,7 +424,7 @@ describe('Simulator oracle', () => { }); aztecNode.getTxEffect.mockImplementation(txHash => { - return Promise.resolve(txEffectsMap[txHash.toString()] as TxEffect); + return Promise.resolve(randomInBlock(txEffectsMap[txHash.toString()] as TxEffect)); }); aztecNode.findLeavesIndexes.mockImplementation((_blockNumber, _treeId, leafValues) => Promise.resolve( diff --git a/yarn-project/pxe/src/synchronizer/synchronizer.test.ts b/yarn-project/pxe/src/synchronizer/synchronizer.test.ts index 8b0c6810eda..ffde1cef571 100644 --- a/yarn-project/pxe/src/synchronizer/synchronizer.test.ts +++ b/yarn-project/pxe/src/synchronizer/synchronizer.test.ts @@ -1,94 +1,54 @@ -import { type AztecNode, L2Block } from '@aztec/circuit-types'; -import { type Header } from '@aztec/circuits.js'; -import { makeHeader } from '@aztec/circuits.js/testing'; -import { randomInt } from '@aztec/foundation/crypto'; -import { SerialQueue } from '@aztec/foundation/queue'; +import { type AztecNode, L2Block, type L2BlockStream } from '@aztec/circuit-types'; +import { L2TipsStore } from '@aztec/kv-store/stores'; import { openTmpStore } from '@aztec/kv-store/utils'; +import { jest } from '@jest/globals'; import { type MockProxy, mock } from 'jest-mock-extended'; +import times from 'lodash.times'; import { type PxeDatabase } from '../database/index.js'; import { KVPxeDatabase } from '../database/kv_pxe_database.js'; import { Synchronizer } from './synchronizer.js'; describe('Synchronizer', () => { - let aztecNode: MockProxy; let database: PxeDatabase; - let synchronizer: TestSynchronizer; - let jobQueue: SerialQueue; - const initialSyncBlockNumber = 3; - let headerBlock3: Header; - - beforeEach(() => { - headerBlock3 = makeHeader(randomInt(1000), initialSyncBlockNumber, initialSyncBlockNumber); + let synchronizer: Synchronizer; + let tipsStore: L2TipsStore; // eslint-disable-line @typescript-eslint/no-unused-vars - aztecNode = mock(); - database = new KVPxeDatabase(openTmpStore()); - jobQueue = new SerialQueue(); - synchronizer = new TestSynchronizer(aztecNode, database, jobQueue); - }); - - it('sets header from aztec node on initial sync', async () => { - aztecNode.getBlockNumber.mockResolvedValue(initialSyncBlockNumber); - aztecNode.getHeader.mockResolvedValue(headerBlock3); + let aztecNode: MockProxy; + let blockStream: MockProxy; - await synchronizer.initialSync(); + const TestSynchronizer = class extends Synchronizer { + protected override createBlockStream(): L2BlockStream { + return blockStream; + } + }; - expect(database.getHeader()).toEqual(headerBlock3); + beforeEach(() => { + const store = openTmpStore(); + blockStream = mock(); + aztecNode = mock(); + database = new KVPxeDatabase(store); + tipsStore = new L2TipsStore(store, 'pxe'); + synchronizer = new TestSynchronizer(aztecNode, database, tipsStore); }); it('sets header from latest block', async () => { const block = L2Block.random(1, 4); - aztecNode.getLogs.mockResolvedValueOnce([block.body.encryptedLogs]).mockResolvedValue([block.body.unencryptedLogs]); - aztecNode.getBlocks.mockResolvedValue([block]); - - await synchronizer.work(); + await synchronizer.handleBlockStreamEvent({ type: 'blocks-added', blocks: [block] }); const obtainedHeader = database.getHeader(); expect(obtainedHeader).toEqual(block.header); }); - it('overrides header from initial sync once current block number is larger', async () => { - // Initial sync is done on block with height 3 - aztecNode.getBlockNumber.mockResolvedValue(initialSyncBlockNumber); - aztecNode.getHeader.mockResolvedValue(headerBlock3); - - await synchronizer.initialSync(); - const header0 = database.getHeader(); - expect(header0).toEqual(headerBlock3); - - // We then process block with height 1, this should not change the header - const block1 = L2Block.random(1, 4); - - aztecNode.getLogs - .mockResolvedValueOnce([block1.body.encryptedLogs]) - .mockResolvedValue([block1.body.unencryptedLogs]); - - aztecNode.getBlocks.mockResolvedValue([block1]); + it('removes notes from db on a reorg', async () => { + const removeNotesAfter = jest.spyOn(database, 'removeNotesAfter').mockImplementation(() => Promise.resolve()); + const unnullifyNotesAfter = jest.spyOn(database, 'unnullifyNotesAfter').mockImplementation(() => Promise.resolve()); - await synchronizer.work(); - const header1 = database.getHeader(); - expect(header1).toEqual(headerBlock3); - expect(header1).not.toEqual(block1.header); + await synchronizer.handleBlockStreamEvent({ type: 'blocks-added', blocks: times(5, L2Block.random) }); + await synchronizer.handleBlockStreamEvent({ type: 'chain-pruned', blockNumber: 3 }); - // But they should change when we process block with height 5 - const block5 = L2Block.random(5, 4); - - aztecNode.getBlocks.mockResolvedValue([block5]); - - await synchronizer.work(); - const header5 = database.getHeader(); - expect(header5).not.toEqual(headerBlock3); - expect(header5).toEqual(block5.header); + expect(removeNotesAfter).toHaveBeenCalledWith(3); + expect(unnullifyNotesAfter).toHaveBeenCalledWith(3); }); }); - -class TestSynchronizer extends Synchronizer { - public override work(limit = 1) { - return super.work(limit); - } - - public override initialSync(): Promise { - return super.initialSync(); - } -} diff --git a/yarn-project/pxe/src/synchronizer/synchronizer.ts b/yarn-project/pxe/src/synchronizer/synchronizer.ts index 3ed458b2db1..63fdc36843e 100644 --- a/yarn-project/pxe/src/synchronizer/synchronizer.ts +++ b/yarn-project/pxe/src/synchronizer/synchronizer.ts @@ -1,9 +1,14 @@ -import { type AztecNode, type L2Block } from '@aztec/circuit-types'; +import { + type AztecNode, + L2BlockStream, + type L2BlockStreamEvent, + type L2BlockStreamEventHandler, +} from '@aztec/circuit-types'; import { INITIAL_L2_BLOCK_NUM } from '@aztec/circuits.js'; import { type DebugLogger, createDebugLogger } from '@aztec/foundation/log'; -import { type SerialQueue } from '@aztec/foundation/queue'; -import { RunningPromise } from '@aztec/foundation/running-promise'; +import { type L2TipsStore } from '@aztec/kv-store/stores'; +import { type PXEConfig } from '../config/index.js'; import { type PxeDatabase } from '../database/index.js'; /** @@ -13,14 +18,48 @@ import { type PxeDatabase } from '../database/index.js'; * details, and fetch transactions by hash. The Synchronizer ensures that it maintains the note processors * in sync with the blockchain while handling retries and errors gracefully. */ -export class Synchronizer { - private runningPromise?: RunningPromise; +export class Synchronizer implements L2BlockStreamEventHandler { private running = false; private initialSyncBlockNumber = INITIAL_L2_BLOCK_NUM - 1; private log: DebugLogger; + protected readonly blockStream: L2BlockStream; - constructor(private node: AztecNode, private db: PxeDatabase, private jobQueue: SerialQueue, logSuffix = '') { + constructor( + private node: AztecNode, + private db: PxeDatabase, + private l2TipsStore: L2TipsStore, + config: Partial> = {}, + logSuffix?: string, + ) { this.log = createDebugLogger(logSuffix ? `aztec:pxe_synchronizer_${logSuffix}` : 'aztec:pxe_synchronizer'); + this.blockStream = this.createBlockStream(config); + } + + protected createBlockStream(config: Partial>) { + return new L2BlockStream(this.node, this.l2TipsStore, this, { + pollIntervalMS: config.l2BlockPollingIntervalMS, + startingBlock: config.l2StartingBlock, + }); + } + + /** Handle events emitted by the block stream. */ + public async handleBlockStreamEvent(event: L2BlockStreamEvent): Promise { + await this.l2TipsStore.handleBlockStreamEvent(event); + + switch (event.type) { + case 'blocks-added': + this.log.verbose(`Processing blocks ${event.blocks[0].number} to ${event.blocks.at(-1)!.number}`); + await this.db.setHeader(event.blocks.at(-1)!.header); + break; + case 'chain-pruned': + this.log.info(`Pruning data after block ${event.blockNumber} due to reorg`); + // We first unnullify and then remove so that unnullified notes that were created after the block number end up deleted. + await this.db.unnullifyNotesAfter(event.blockNumber); + await this.db.removeNotesAfter(event.blockNumber); + // Update the header to the last block. + await this.db.setHeader(await this.node.getBlockHeader(event.blockNumber)); + break; + } } /** @@ -31,79 +70,21 @@ export class Synchronizer { * @param limit - The maximum number of encrypted, unencrypted logs and blocks to fetch in each iteration. * @param retryInterval - The time interval (in ms) to wait before retrying if no data is available. */ - public async start(limit = 1, retryInterval = 1000) { + public async start() { if (this.running) { return; } this.running = true; - await this.jobQueue.put(() => this.initialSync()); + // REFACTOR: We should know the header of the genesis block without having to request it from the node. + await this.db.setHeader(await this.node.getBlockHeader(0)); + + await this.trigger(); this.log.info('Initial sync complete'); - this.runningPromise = new RunningPromise(() => this.sync(limit), retryInterval); - this.runningPromise.start(); + this.blockStream.start(); this.log.debug('Started loop'); } - protected async initialSync() { - // fast forward to the latest block - const latestHeader = await this.node.getHeader(); - this.initialSyncBlockNumber = Number(latestHeader.globalVariables.blockNumber.toBigInt()); - await this.db.setHeader(latestHeader); - } - - /** - * Fetches encrypted logs and blocks from the Aztec node and processes them for all note processors. - * If needed, catches up note processors that are lagging behind the main sync, e.g. because we just added a new account. - * - * Uses the job queue to ensure that - * - sync does not overlap with pxe simulations. - * - one sync is running at a time. - * - * @param limit - The maximum number of encrypted, unencrypted logs and blocks to fetch in each iteration. - * @returns a promise that resolves when the sync is complete - */ - protected sync(limit: number) { - return this.jobQueue.put(async () => { - let moreWork = true; - // keep external this.running flag to interrupt greedy sync - while (moreWork && this.running) { - moreWork = await this.work(limit); - } - }); - } - - /** - * Fetches encrypted logs and blocks from the Aztec node and processes them for all note processors. - * - * @param limit - The maximum number of encrypted, unencrypted logs and blocks to fetch in each iteration. - * @returns true if there could be more work, false if we're caught up or there was an error. - */ - protected async work(limit = 1): Promise { - const from = this.getSynchedBlockNumber() + 1; - try { - const blocks = await this.node.getBlocks(from, limit); - if (blocks.length === 0) { - return false; - } - - // Update latest tree roots from the most recent block - const latestBlock = blocks[blocks.length - 1]; - await this.setHeaderFromBlock(latestBlock); - return true; - } catch (err) { - this.log.error(`Error in synchronizer work`, err); - return false; - } - } - - private async setHeaderFromBlock(latestBlock: L2Block) { - if (latestBlock.number < this.initialSyncBlockNumber) { - return; - } - - await this.db.setHeader(latestBlock.header); - } - /** * Stops the synchronizer gracefully, interrupting any ongoing sleep and waiting for the current * iteration to complete before setting the running state to false. Once stopped, the synchronizer @@ -113,10 +94,15 @@ export class Synchronizer { */ public async stop() { this.running = false; - await this.runningPromise?.stop(); + await this.blockStream.stop(); this.log.info('Stopped'); } + /** Triggers a single run. */ + public async trigger() { + await this.blockStream.sync(); + } + private getSynchedBlockNumber() { return this.db.getBlockNumber() ?? this.initialSyncBlockNumber; } diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index 449d4424f8e..68d0b7b6269 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -763,6 +763,8 @@ __metadata: version: 0.0.0-use.local resolution: "@aztec/kv-store@workspace:kv-store" dependencies: + "@aztec/circuit-types": "workspace:^" + "@aztec/circuits.js": "workspace:^" "@aztec/foundation": "workspace:^" "@jest/globals": ^29.5.0 "@types/jest": ^29.5.0 From b09c6e5d244dddf445c815ddfd7012cc3dd31651 Mon Sep 17 00:00:00 2001 From: thunkar Date: Tue, 19 Nov 2024 06:49:13 +0000 Subject: [PATCH 02/14] implemented unnullify --- .../pxe/src/database/kv_pxe_database.ts | 46 ++++++++++++++++++- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/yarn-project/pxe/src/database/kv_pxe_database.ts b/yarn-project/pxe/src/database/kv_pxe_database.ts index 90ccc3796e3..d375fb76520 100644 --- a/yarn-project/pxe/src/database/kv_pxe_database.ts +++ b/yarn-project/pxe/src/database/kv_pxe_database.ts @@ -7,6 +7,7 @@ import { type IndexedTaggingSecret, type PublicKey, SerializableContractInstance, + computeAddress, computePoint, } from '@aztec/circuits.js'; import { type ContractArtifact } from '@aztec/foundation/abi'; @@ -39,11 +40,13 @@ export class KVPxeDatabase implements PxeDatabase { #notes: AztecMap; #nullifiedNotes: AztecMap; #nullifierToNoteId: AztecMap; + #nullifiersByBlockNumber: AztecMultiMap; #nullifiedNotesByContract: AztecMultiMap; #nullifiedNotesByStorageSlot: AztecMultiMap; #nullifiedNotesByTxHash: AztecMultiMap; #nullifiedNotesByAddressPoint: AztecMultiMap; + #nullifiedNotesByNullifier: AztecMap; #syncedBlockPerPublicKey: AztecMap; #contractArtifacts: AztecMap; #contractInstances: AztecMap; @@ -87,11 +90,13 @@ export class KVPxeDatabase implements PxeDatabase { this.#notes = db.openMap('notes'); this.#nullifiedNotes = db.openMap('nullified_notes'); this.#nullifierToNoteId = db.openMap('nullifier_to_note'); + this.#nullifiersByBlockNumber = db.openMultiMap('nullifier_to_block_number'); this.#nullifiedNotesByContract = db.openMultiMap('nullified_notes_by_contract'); this.#nullifiedNotesByStorageSlot = db.openMultiMap('nullified_notes_by_storage_slot'); this.#nullifiedNotesByTxHash = db.openMultiMap('nullified_notes_by_tx_hash'); this.#nullifiedNotesByAddressPoint = db.openMultiMap('nullified_notes_by_address_point'); + this.#nullifiedNotesByNullifier = db.openMap('nullified_notes_by_nullifier'); this.#outgoingNotes = db.openMap('outgoing_notes'); this.#outgoingNotesByContract = db.openMultiMap('outgoing_notes_by_contract'); @@ -245,8 +250,43 @@ export class KVPxeDatabase implements PxeDatabase { }); } - public async unnullifyNotesAfter(_blockNumber: number): Promise { - // TODO: Implementme. Requires tracking when a note was nullified. + public async unnullifyNotesAfter(blockNumber: number): Promise { + const nullifiersToUndo = Array.from(await this.#nullifiersByBlockNumber.getValues(blockNumber)); + const notesIndexesToReinsert = await this.db.transaction(() => + nullifiersToUndo.map(nullifier => this.#nullifiedNotesByNullifier.get(nullifier)), + ); + const nullifiedNoteBuffers = await this.db.transaction(() => { + return notesIndexesToReinsert + .filter(noteIndex => noteIndex != undefined) + .map(noteIndex => this.#nullifiedNotes.get(noteIndex!)); + }); + const noteDaos = nullifiedNoteBuffers + .filter(buffer => buffer != undefined) + .map(buffer => IncomingNoteDao.fromBuffer(buffer!)); + + await this.db.transaction(() => { + for (const dao of noteDaos) { + const noteIndex = toBufferBE(dao.index, 32).toString('hex'); + void this.#notes.set(noteIndex, dao.toBuffer()); + void this.#nullifierToNoteId.set(dao.siloedNullifier.toString(), noteIndex); + + // TODO: check if is this a good assumption + const scope = new AztecAddress(dao.addressPoint.x); + + void this.#notesByContractAndScope.get(scope.toString())!.set(dao.contractAddress.toString(), noteIndex); + void this.#notesByStorageSlotAndScope.get(scope.toString())!.set(dao.storageSlot.toString(), noteIndex); + void this.#notesByTxHashAndScope.get(scope.toString())!.set(dao.txHash.toString(), noteIndex); + void this.#notesByAddressPointAndScope.get(scope.toString())!.set(dao.addressPoint.toString(), noteIndex); + + void this.#nullifiedNotes.delete(noteIndex); + void this.#nullifiersByBlockNumber.deleteValue(dao.l2BlockNumber, dao.siloedNullifier.toString()); + void this.#nullifiedNotesByContract.deleteValue(dao.contractAddress.toString(), noteIndex); + void this.#nullifiedNotesByStorageSlot.deleteValue(dao.storageSlot.toString(), noteIndex); + void this.#nullifiedNotesByTxHash.deleteValue(dao.txHash.toString(), noteIndex); + void this.#nullifiedNotesByAddressPoint.deleteValue(dao.addressPoint.toString(), noteIndex); + void this.#nullifiedNotesByNullifier.delete(dao.siloedNullifier.toString()); + } + }); } getIncomingNotes(filter: IncomingNotesFilter): Promise { @@ -424,10 +464,12 @@ export class KVPxeDatabase implements PxeDatabase { } void this.#nullifiedNotes.set(noteIndex, note.toBuffer()); + void this.#nullifiersByBlockNumber.set(note.l2BlockNumber, nullifier.toString()); void this.#nullifiedNotesByContract.set(note.contractAddress.toString(), noteIndex); void this.#nullifiedNotesByStorageSlot.set(note.storageSlot.toString(), noteIndex); void this.#nullifiedNotesByTxHash.set(note.txHash.toString(), noteIndex); void this.#nullifiedNotesByAddressPoint.set(note.addressPoint.toString(), noteIndex); + void this.#nullifiedNotesByNullifier.set(nullifier.toString(), noteIndex); void this.#nullifierToNoteId.delete(nullifier.toString()); } From fa2e521d13080db82aeed040d63938b058ee5738 Mon Sep 17 00:00:00 2001 From: thunkar Date: Tue, 19 Nov 2024 06:56:27 +0000 Subject: [PATCH 03/14] ensure to unnullify after provided block --- yarn-project/pxe/src/database/kv_pxe_database.ts | 9 +++++++-- yarn-project/pxe/src/simulator_oracle/index.ts | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/yarn-project/pxe/src/database/kv_pxe_database.ts b/yarn-project/pxe/src/database/kv_pxe_database.ts index d375fb76520..61f23f35283 100644 --- a/yarn-project/pxe/src/database/kv_pxe_database.ts +++ b/yarn-project/pxe/src/database/kv_pxe_database.ts @@ -7,7 +7,6 @@ import { type IndexedTaggingSecret, type PublicKey, SerializableContractInstance, - computeAddress, computePoint, } from '@aztec/circuits.js'; import { type ContractArtifact } from '@aztec/foundation/abi'; @@ -251,7 +250,13 @@ export class KVPxeDatabase implements PxeDatabase { } public async unnullifyNotesAfter(blockNumber: number): Promise { - const nullifiersToUndo = Array.from(await this.#nullifiersByBlockNumber.getValues(blockNumber)); + const nullifiersToUndo: string[] = []; + let currentBlockNumber = blockNumber; + const maxBlockNumber = this.getBlockNumber() ?? currentBlockNumber; + do { + nullifiersToUndo.push(...this.#nullifiersByBlockNumber.getValues(currentBlockNumber)); + currentBlockNumber++; + } while (currentBlockNumber < maxBlockNumber); const notesIndexesToReinsert = await this.db.transaction(() => nullifiersToUndo.map(nullifier => this.#nullifiedNotesByNullifier.get(nullifier)), ); diff --git a/yarn-project/pxe/src/simulator_oracle/index.ts b/yarn-project/pxe/src/simulator_oracle/index.ts index ed3e32df19d..2cd907f75a2 100644 --- a/yarn-project/pxe/src/simulator_oracle/index.ts +++ b/yarn-project/pxe/src/simulator_oracle/index.ts @@ -1,6 +1,6 @@ import { type AztecNode, - InBlock, + type InBlock, L1NotePayload, type L2Block, type L2BlockNumber, From ce4b168083a7dbb7659656bf6ee3076180bbf656 Mon Sep 17 00:00:00 2001 From: thunkar Date: Tue, 19 Nov 2024 13:18:02 +0000 Subject: [PATCH 04/14] initial approach to blockNumber in leaf indexes --- .../src/archiver/archiver_store_test_suite.ts | 26 +++++----- .../aztec-node/src/aztec-node/server.ts | 28 +++++++++++ .../src/interfaces/aztec-node.test.ts | 21 +++++++- .../src/interfaces/aztec-node.ts | 12 +++++ .../src/logs/get_logs_response.ts | 10 ++++ .../pxe/src/database/kv_pxe_database.ts | 20 +++++--- yarn-project/pxe/src/database/pxe_database.ts | 4 +- .../src/database/pxe_database_test_suite.ts | 50 +++++++++++++++++-- .../pxe/src/simulator_oracle/index.ts | 27 +++++++--- .../pxe/src/synchronizer/synchronizer.test.ts | 3 ++ 10 files changed, 168 insertions(+), 33 deletions(-) diff --git a/yarn-project/archiver/src/archiver/archiver_store_test_suite.ts b/yarn-project/archiver/src/archiver/archiver_store_test_suite.ts index cd30af56a78..05a66280d08 100644 --- a/yarn-project/archiver/src/archiver/archiver_store_test_suite.ts +++ b/yarn-project/archiver/src/archiver/archiver_store_test_suite.ts @@ -1,4 +1,4 @@ -import { InboxLeaf, L2Block, LogId, LogType, TxHash } from '@aztec/circuit-types'; +import { InboxLeaf, L2Block, LogId, LogType, TxHash, wrapInBlock } from '@aztec/circuit-types'; import '@aztec/circuit-types/jest'; import { AztecAddress, @@ -191,14 +191,14 @@ export function describeArchiverDataStore(testName: string, getStore: () => Arch }); it.each([ - () => blocks[0].data.body.txEffects[0], - () => blocks[9].data.body.txEffects[3], - () => blocks[3].data.body.txEffects[1], - () => blocks[5].data.body.txEffects[2], - () => blocks[1].data.body.txEffects[0], + () => wrapInBlock(blocks[0].data.body.txEffects[0], blocks[0].data), + () => wrapInBlock(blocks[9].data.body.txEffects[3], blocks[9].data), + () => wrapInBlock(blocks[3].data.body.txEffects[1], blocks[3].data), + () => wrapInBlock(blocks[5].data.body.txEffects[2], blocks[5].data), + () => wrapInBlock(blocks[1].data.body.txEffects[0], blocks[1].data), ])('retrieves a previously stored transaction', async getExpectedTx => { const expectedTx = getExpectedTx(); - const actualTx = await store.getTxEffect(expectedTx.txHash); + const actualTx = await store.getTxEffect(expectedTx.data.txHash); expect(actualTx).toEqual(expectedTx); }); @@ -207,16 +207,16 @@ export function describeArchiverDataStore(testName: string, getStore: () => Arch }); it.each([ - () => blocks[0].data.body.txEffects[0], - () => blocks[9].data.body.txEffects[3], - () => blocks[3].data.body.txEffects[1], - () => blocks[5].data.body.txEffects[2], - () => blocks[1].data.body.txEffects[0], + () => wrapInBlock(blocks[0].data.body.txEffects[0], blocks[0].data), + () => wrapInBlock(blocks[9].data.body.txEffects[3], blocks[9].data), + () => wrapInBlock(blocks[3].data.body.txEffects[1], blocks[3].data), + () => wrapInBlock(blocks[5].data.body.txEffects[2], blocks[5].data), + () => wrapInBlock(blocks[1].data.body.txEffects[0], blocks[1].data), ])('tries to retrieves a previously stored transaction after deleted', async getExpectedTx => { await store.unwindBlocks(blocks.length, blocks.length); const expectedTx = getExpectedTx(); - const actualTx = await store.getTxEffect(expectedTx.txHash); + const actualTx = await store.getTxEffect(expectedTx.data.txHash); expect(actualTx).toEqual(undefined); }); diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index 192cd8bf9ca..fa8cdb5bc42 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -33,6 +33,7 @@ import { type TxValidator, type WorldStateSynchronizer, tryStop, + wrapInBlock, } from '@aztec/circuit-types'; import { type ARCHIVE_HEIGHT, @@ -431,6 +432,33 @@ export class AztecNodeService implements AztecNode { return await Promise.all(leafValues.map(leafValue => committedDb.findLeafIndex(treeId, leafValue.toBuffer()))); } + public async findLeavesIndexesWithApproxBlockNumber( + maxBlockNumber: L2BlockNumber, + minBlockNumber: number, + treeId: MerkleTreeId, + leafValues: Fr[], + ): Promise<(InBlock | undefined)[]> { + const initialBlockNumber = maxBlockNumber === 'latest' ? await this.getBlockNumber() : maxBlockNumber; + return await Promise.all( + leafValues.map(async leafValue => { + let blockNumber = initialBlockNumber; + let treeSnapshot = this.worldStateSynchronizer.getSnapshot(initialBlockNumber); + let index = await treeSnapshot.findLeafIndex(treeId, leafValue.toBuffer()); + if (!index) { + return undefined; + } + let meta = await treeSnapshot.getTreeInfo(treeId); + while (meta.size > index && blockNumber >= minBlockNumber) { + blockNumber -= 1; + treeSnapshot = this.worldStateSynchronizer.getSnapshot(blockNumber); + meta = await treeSnapshot.getTreeInfo(treeId); + } + const block = await this.getBlock(blockNumber + 1); + return wrapInBlock(index, block!); + }), + ); + } + /** * Returns a sibling path for the given index in the nullifier tree. * @param blockNumber - The block number at which to get the data. diff --git a/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts b/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts index 1c0f10fbf0f..89f9e7714b5 100644 --- a/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts +++ b/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts @@ -28,7 +28,7 @@ import omit from 'lodash.omit'; import times from 'lodash.times'; import { resolve } from 'path'; -import { type InBlock } from '../in_block.js'; +import { type InBlock, wrapInBlock } from '../in_block.js'; import { L2Block } from '../l2_block.js'; import { type L2Tips } from '../l2_block_source.js'; import { ExtendedUnencryptedL2Log } from '../logs/extended_unencrypted_l2_log.js'; @@ -96,6 +96,14 @@ describe('AztecNodeApiSchema', () => { expect(response).toEqual([1n, undefined]); }); + it('findLeavesIndexesWithApproxBlockNumber', async () => { + const response = await context.client.findLeavesIndexesWithApproxBlockNumber(1, 1, MerkleTreeId.ARCHIVE, [ + Fr.random(), + Fr.random(), + ]); + expect(response).toEqual([{ data: 1n, l2BlockNumber: 1, l2BlockHash: expect.any(String) }, undefined]); + }); + it('getNullifierSiblingPath', async () => { const response = await context.client.getNullifierSiblingPath(1, 1n); expect(response).toBeInstanceOf(SiblingPath); @@ -346,6 +354,17 @@ class MockAztecNode implements AztecNode { expect(leafValues[1]).toBeInstanceOf(Fr); return Promise.resolve([1n, undefined]); } + findLeavesIndexesWithApproxBlockNumber( + maxBlockNumber: number | 'latest', + minBlockNumber: number, + treeId: MerkleTreeId, + leafValues: Fr[], + ): Promise<(InBlock | undefined)[]> { + expect(leafValues).toHaveLength(2); + expect(leafValues[0]).toBeInstanceOf(Fr); + expect(leafValues[1]).toBeInstanceOf(Fr); + return Promise.resolve([wrapInBlock(1n, L2Block.random(1)), undefined]); + } getNullifierSiblingPath( blockNumber: number | 'latest', leafIndex: bigint, diff --git a/yarn-project/circuit-types/src/interfaces/aztec-node.ts b/yarn-project/circuit-types/src/interfaces/aztec-node.ts index 252757521fe..c1d071be9df 100644 --- a/yarn-project/circuit-types/src/interfaces/aztec-node.ts +++ b/yarn-project/circuit-types/src/interfaces/aztec-node.ts @@ -71,6 +71,13 @@ export interface AztecNode leafValues: Fr[], ): Promise<(bigint | undefined)[]>; + findLeavesIndexesWithApproxBlockNumber( + maxBlockNumber: L2BlockNumber, + minBlockNumber: number, + treeId: MerkleTreeId, + leafValues: Fr[], + ): Promise<(InBlock | undefined)[]>; + /** * Returns a sibling path for the given index in the nullifier tree. * @param blockNumber - The block number at which to get the data. @@ -411,6 +418,11 @@ export const AztecNodeApiSchema: ApiSchemaFor = { .args(L2BlockNumberSchema, z.nativeEnum(MerkleTreeId), z.array(schemas.Fr)) .returns(z.array(optional(schemas.BigInt))), + findLeavesIndexesWithApproxBlockNumber: z + .function() + .args(L2BlockNumberSchema, z.number(), z.nativeEnum(MerkleTreeId), z.array(schemas.Fr)) + .returns(z.array(optional(inBlockSchemaFor(schemas.BigInt)))), + getNullifierSiblingPath: z .function() .args(L2BlockNumberSchema, schemas.BigInt) diff --git a/yarn-project/circuit-types/src/logs/get_logs_response.ts b/yarn-project/circuit-types/src/logs/get_logs_response.ts index c62b5af965c..6f1d156be0b 100644 --- a/yarn-project/circuit-types/src/logs/get_logs_response.ts +++ b/yarn-project/circuit-types/src/logs/get_logs_response.ts @@ -84,4 +84,14 @@ export class TxScopedL2Log { static random() { return new TxScopedL2Log(TxHash.random(), 1, 1, false, Fr.random().toBuffer()); } + + equals(other: TxScopedL2Log) { + return ( + this.txHash.equals(other.txHash) && + this.dataStartIndexForTx === other.dataStartIndexForTx && + this.blockNumber === other.blockNumber && + this.isFromPublic === other.isFromPublic && + this.logData.equals(other.logData) + ); + } } diff --git a/yarn-project/pxe/src/database/kv_pxe_database.ts b/yarn-project/pxe/src/database/kv_pxe_database.ts index 61f23f35283..2f8336b8d81 100644 --- a/yarn-project/pxe/src/database/kv_pxe_database.ts +++ b/yarn-project/pxe/src/database/kv_pxe_database.ts @@ -1,4 +1,10 @@ -import { type IncomingNotesFilter, MerkleTreeId, NoteStatus, type OutgoingNotesFilter } from '@aztec/circuit-types'; +import { + InBlock, + type IncomingNotesFilter, + MerkleTreeId, + NoteStatus, + type OutgoingNotesFilter, +} from '@aztec/circuit-types'; import { AztecAddress, CompleteAddress, @@ -253,10 +259,11 @@ export class KVPxeDatabase implements PxeDatabase { const nullifiersToUndo: string[] = []; let currentBlockNumber = blockNumber; const maxBlockNumber = this.getBlockNumber() ?? currentBlockNumber; - do { + while (currentBlockNumber <= maxBlockNumber) { nullifiersToUndo.push(...this.#nullifiersByBlockNumber.getValues(currentBlockNumber)); currentBlockNumber++; - } while (currentBlockNumber < maxBlockNumber); + } + const notesIndexesToReinsert = await this.db.transaction(() => nullifiersToUndo.map(nullifier => this.#nullifiedNotesByNullifier.get(nullifier)), ); @@ -430,7 +437,7 @@ export class KVPxeDatabase implements PxeDatabase { return Promise.resolve(notes); } - removeNullifiedNotes(nullifiers: Fr[], accountAddressPoint: PublicKey): Promise { + removeNullifiedNotes(nullifiers: InBlock[], accountAddressPoint: PublicKey): Promise { if (nullifiers.length === 0) { return Promise.resolve([]); } @@ -438,7 +445,8 @@ export class KVPxeDatabase implements PxeDatabase { return this.#db.transaction(() => { const nullifiedNotes: IncomingNoteDao[] = []; - for (const nullifier of nullifiers) { + for (const blockScopedNullifier of nullifiers) { + const { data: nullifier, l2BlockNumber: blockNumber } = blockScopedNullifier; const noteIndex = this.#nullifierToNoteId.get(nullifier.toString()); if (!noteIndex) { continue; @@ -469,7 +477,7 @@ export class KVPxeDatabase implements PxeDatabase { } void this.#nullifiedNotes.set(noteIndex, note.toBuffer()); - void this.#nullifiersByBlockNumber.set(note.l2BlockNumber, nullifier.toString()); + void this.#nullifiersByBlockNumber.set(blockNumber, nullifier.toString()); void this.#nullifiedNotesByContract.set(note.contractAddress.toString(), noteIndex); void this.#nullifiedNotesByStorageSlot.set(note.storageSlot.toString(), noteIndex); void this.#nullifiedNotesByTxHash.set(note.txHash.toString(), noteIndex); diff --git a/yarn-project/pxe/src/database/pxe_database.ts b/yarn-project/pxe/src/database/pxe_database.ts index 58e39901da3..09746a55ba1 100644 --- a/yarn-project/pxe/src/database/pxe_database.ts +++ b/yarn-project/pxe/src/database/pxe_database.ts @@ -1,4 +1,4 @@ -import { type IncomingNotesFilter, type OutgoingNotesFilter } from '@aztec/circuit-types'; +import { InBlock, type IncomingNotesFilter, type OutgoingNotesFilter } from '@aztec/circuit-types'; import { type CompleteAddress, type ContractInstanceWithAddress, @@ -96,7 +96,7 @@ export interface PxeDatabase extends ContractArtifactDatabase, ContractInstanceD * @param account - A PublicKey instance representing the account for which the records are being removed. * @returns Removed notes. */ - removeNullifiedNotes(nullifiers: Fr[], account: PublicKey): Promise; + removeNullifiedNotes(nullifiers: InBlock[], account: PublicKey): Promise; /** * Gets the most recently processed block number. diff --git a/yarn-project/pxe/src/database/pxe_database_test_suite.ts b/yarn-project/pxe/src/database/pxe_database_test_suite.ts index b0dfd5272d6..a3f334684f5 100644 --- a/yarn-project/pxe/src/database/pxe_database_test_suite.ts +++ b/yarn-project/pxe/src/database/pxe_database_test_suite.ts @@ -2,6 +2,7 @@ import { type IncomingNotesFilter, NoteStatus, type OutgoingNotesFilter, randomT import { AztecAddress, CompleteAddress, + Header, INITIAL_L2_BLOCK_NUM, PublicKeys, SerializableContractInstance, @@ -156,7 +157,11 @@ export function describePxeDatabase(getDatabase: () => PxeDatabase) { // Nullify all notes and use the same filter as other test cases for (const owner of owners) { const notesToNullify = notes.filter(note => note.addressPoint.equals(computePoint(owner.address))); - const nullifiers = notesToNullify.map(note => note.siloedNullifier); + const nullifiers = notesToNullify.map(note => ({ + data: note.siloedNullifier, + l2BlockNumber: note.l2BlockNumber, + l2BlockHash: note.l2BlockHash, + })); await expect(database.removeNullifiedNotes(nullifiers, computePoint(owner.address))).resolves.toEqual( notesToNullify, ); @@ -171,7 +176,11 @@ export function describePxeDatabase(getDatabase: () => PxeDatabase) { await database.addNotes(notes, []); const notesToNullify = notes.filter(note => note.addressPoint.equals(computePoint(owners[0].address))); - const nullifiers = notesToNullify.map(note => note.siloedNullifier); + const nullifiers = notesToNullify.map(note => ({ + data: note.siloedNullifier, + l2BlockNumber: note.l2BlockNumber, + l2BlockHash: note.l2BlockHash, + })); await expect(database.removeNullifiedNotes(nullifiers, notesToNullify[0].addressPoint)).resolves.toEqual( notesToNullify, ); @@ -183,11 +192,35 @@ export function describePxeDatabase(getDatabase: () => PxeDatabase) { expect(actualNotesWithActive).toEqual(notes.filter(note => !notesToNullify.includes(note))); }); + it('handles note unnullification', async () => { + await database.setHeader(makeHeader(randomInt(1000), 100, 0 /** slot number */)); + await database.addNotes(notes, []); + + const notesToNullify = notes.filter(note => note.addressPoint.equals(computePoint(owners[0].address))); + const nullifiers = notesToNullify.map(note => ({ + data: note.siloedNullifier, + l2BlockNumber: 100, + l2BlockHash: Fr.random().toString(), + })); + await expect(database.removeNullifiedNotes(nullifiers, notesToNullify[0].addressPoint)).resolves.toEqual( + notesToNullify, + ); + await expect(database.unnullifyNotesAfter(98)).resolves.toEqual(undefined); + + const result = await database.getIncomingNotes({ status: NoteStatus.ACTIVE, owner: owners[0].address }); + + expect(result.sort()).toEqual([...notesToNullify].sort()); + }); + it('returns active and nullified notes when requesting either', async () => { await database.addNotes(notes, []); const notesToNullify = notes.filter(note => note.addressPoint.equals(computePoint(owners[0].address))); - const nullifiers = notesToNullify.map(note => note.siloedNullifier); + const nullifiers = notesToNullify.map(note => ({ + data: note.siloedNullifier, + l2BlockNumber: note.l2BlockNumber, + l2BlockHash: note.l2BlockHash, + })); await expect(database.removeNullifiedNotes(nullifiers, notesToNullify[0].addressPoint)).resolves.toEqual( notesToNullify, ); @@ -245,7 +278,16 @@ export function describePxeDatabase(getDatabase: () => PxeDatabase) { ).resolves.toEqual([notes[0]]); await expect( - database.removeNullifiedNotes([notes[0].siloedNullifier], computePoint(owners[0].address)), + database.removeNullifiedNotes( + [ + { + data: notes[0].siloedNullifier, + l2BlockHash: notes[0].l2BlockHash, + l2BlockNumber: notes[0].l2BlockNumber, + }, + ], + computePoint(owners[0].address), + ), ).resolves.toEqual([notes[0]]); await expect( diff --git a/yarn-project/pxe/src/simulator_oracle/index.ts b/yarn-project/pxe/src/simulator_oracle/index.ts index 2cd907f75a2..251f47a5891 100644 --- a/yarn-project/pxe/src/simulator_oracle/index.ts +++ b/yarn-project/pxe/src/simulator_oracle/index.ts @@ -442,7 +442,12 @@ export class SimulatorOracle implements DBOracle { result.set( recipient.toString(), - logs.filter(log => log.blockNumber <= maxBlockNumber), + // Remove logs with a block number higher than the max block number + // Duplicates are likely to happen due to the sliding window, so we also filter them out + logs.filter( + (log, index, self) => + log.blockNumber <= maxBlockNumber && index === self.findIndex(otherLog => otherLog.equals(log)), + ), ); } return result; @@ -560,15 +565,23 @@ export class SimulatorOracle implements DBOracle { } const nullifiedNotes: IncomingNoteDao[] = []; const currentNotesForRecipient = await this.db.getIncomingNotes({ owner: recipient }); - const nullifierIndexes = await this.aztecNode.findLeavesIndexes( - 'latest', + const nullifiersToCheck = currentNotesForRecipient.map(note => note.siloedNullifier); + const currentBlockNumber = await this.getBlockNumber(); + const nullifierIndexes = await this.aztecNode.findLeavesIndexesWithApproxBlockNumber( + currentBlockNumber, + // Epoch size, we don't care about accurate block numbers in finalized nullifiers + currentBlockNumber - 2 * 32, MerkleTreeId.NULLIFIER_TREE, - currentNotesForRecipient.map(note => note.siloedNullifier), + nullifiersToCheck, ); - const foundNullifiers = currentNotesForRecipient - .filter((_, i) => nullifierIndexes[i] !== undefined) - .map(note => note.siloedNullifier); + const foundNullifiers = nullifiersToCheck + .map((nullifier, i) => { + if (nullifierIndexes[i] !== undefined) { + return { ...nullifierIndexes[i], ...{ data: nullifier } } as InBlock; + } + }) + .filter(nullifier => nullifier !== undefined) as InBlock[]; await this.db.removeNullifiedNotes(foundNullifiers, computePoint(recipient)); nullifiedNotes.forEach(noteDao => { diff --git a/yarn-project/pxe/src/synchronizer/synchronizer.test.ts b/yarn-project/pxe/src/synchronizer/synchronizer.test.ts index ffde1cef571..c504e82d279 100644 --- a/yarn-project/pxe/src/synchronizer/synchronizer.test.ts +++ b/yarn-project/pxe/src/synchronizer/synchronizer.test.ts @@ -44,6 +44,9 @@ describe('Synchronizer', () => { it('removes notes from db on a reorg', async () => { const removeNotesAfter = jest.spyOn(database, 'removeNotesAfter').mockImplementation(() => Promise.resolve()); const unnullifyNotesAfter = jest.spyOn(database, 'unnullifyNotesAfter').mockImplementation(() => Promise.resolve()); + aztecNode.getBlockHeader.mockImplementation(blockNumber => + Promise.resolve(L2Block.random(blockNumber as number).header), + ); await synchronizer.handleBlockStreamEvent({ type: 'blocks-added', blocks: times(5, L2Block.random) }); await synchronizer.handleBlockStreamEvent({ type: 'chain-pruned', blockNumber: 3 }); From da4a91a1ea013ca76791a68f79df65c45e64d6c4 Mon Sep 17 00:00:00 2001 From: thunkar Date: Tue, 19 Nov 2024 13:26:34 +0000 Subject: [PATCH 05/14] tsconfig --- yarn-project/kv-store/tsconfig.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yarn-project/kv-store/tsconfig.json b/yarn-project/kv-store/tsconfig.json index 1e79bc8359c..4a640f19e71 100644 --- a/yarn-project/kv-store/tsconfig.json +++ b/yarn-project/kv-store/tsconfig.json @@ -7,13 +7,13 @@ }, "references": [ { - "path": "../circuit-types" + "path": "../ethereum" }, { "path": "../foundation" }, { - "path": "../ethereum" + "path": "../circuits.js" } ], "include": ["src"] From aea45789f86eda86353151d0c67522adf3ad8ed5 Mon Sep 17 00:00:00 2001 From: thunkar Date: Tue, 19 Nov 2024 14:09:48 +0000 Subject: [PATCH 06/14] fix --- yarn-project/kv-store/package.json | 2 +- yarn-project/yarn.lock | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/yarn-project/kv-store/package.json b/yarn-project/kv-store/package.json index c0391ce5469..1df8f86b53f 100644 --- a/yarn-project/kv-store/package.json +++ b/yarn-project/kv-store/package.json @@ -78,4 +78,4 @@ "engines": { "node": ">=18" } -} \ No newline at end of file +} diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index 196a74d9b03..5c7b2127d49 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -763,6 +763,7 @@ __metadata: version: 0.0.0-use.local resolution: "@aztec/kv-store@workspace:kv-store" dependencies: + "@aztec/circuits.js": "workspace:^" "@aztec/ethereum": "workspace:^" "@aztec/foundation": "workspace:^" "@jest/globals": ^29.5.0 From fa28dab7a839c7ec51f8ec59739c6df5687ea678 Mon Sep 17 00:00:00 2001 From: thunkar Date: Tue, 19 Nov 2024 14:16:51 +0000 Subject: [PATCH 07/14] bad merge --- yarn-project/kv-store/package.json | 2 ++ yarn-project/kv-store/tsconfig.json | 3 +++ yarn-project/yarn.lock | 1 + 3 files changed, 6 insertions(+) diff --git a/yarn-project/kv-store/package.json b/yarn-project/kv-store/package.json index 1df8f86b53f..bacc49e1a38 100644 --- a/yarn-project/kv-store/package.json +++ b/yarn-project/kv-store/package.json @@ -6,6 +6,7 @@ ".": "./dest/interfaces/index.js", "./lmdb": "./dest/lmdb/index.js", "./utils": "./dest/utils.js", + "./stores": "./dest/stores/index.js", "./config": "./dest/config.js" }, "scripts": { @@ -56,6 +57,7 @@ ] }, "dependencies": { + "@aztec/circuit-types": "workspace:^", "@aztec/ethereum": "workspace:^", "@aztec/foundation": "workspace:^", "lmdb": "^3.0.6" diff --git a/yarn-project/kv-store/tsconfig.json b/yarn-project/kv-store/tsconfig.json index 4a640f19e71..bd860591ecb 100644 --- a/yarn-project/kv-store/tsconfig.json +++ b/yarn-project/kv-store/tsconfig.json @@ -6,6 +6,9 @@ "tsBuildInfoFile": ".tsbuildinfo" }, "references": [ + { + "path": "../circuit-types" + }, { "path": "../ethereum" }, diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index 5c7b2127d49..c66a71ea45e 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -763,6 +763,7 @@ __metadata: version: 0.0.0-use.local resolution: "@aztec/kv-store@workspace:kv-store" dependencies: + "@aztec/circuit-types": "workspace:^" "@aztec/circuits.js": "workspace:^" "@aztec/ethereum": "workspace:^" "@aztec/foundation": "workspace:^" From b4d9f6052a62ab245f1b926fa2eb0283bffa7ef8 Mon Sep 17 00:00:00 2001 From: thunkar Date: Wed, 20 Nov 2024 08:40:42 +0000 Subject: [PATCH 08/14] nullifier source approach --- .../archiver/src/archiver/archiver.ts | 24 +++++- .../archiver/src/archiver/archiver_store.ts | 17 ++++ .../src/archiver/archiver_store_test_suite.ts | 54 +++++++++++++ .../kv_archiver_store/kv_archiver_store.ts | 21 +++++ .../kv_archiver_store/nullifier_store.ts | 78 +++++++++++++++++++ .../memory_archiver_store.ts | 48 ++++++++++++ .../aztec-node/src/aztec-node/server.test.ts | 4 + .../aztec-node/src/aztec-node/server.ts | 34 +++----- yarn-project/circuit-types/src/index.ts | 1 + .../src/interfaces/archiver.test.ts | 13 +++- .../circuit-types/src/interfaces/archiver.ts | 7 +- .../src/interfaces/aztec-node.test.ts | 28 ++++--- .../src/interfaces/aztec-node.ts | 19 +++-- .../src/nullifier_with_block_source.ts | 7 ++ .../pxe/src/database/kv_pxe_database.ts | 2 +- yarn-project/pxe/src/database/pxe_database.ts | 2 +- .../src/database/pxe_database_test_suite.ts | 1 - .../pxe/src/pxe_service/create_pxe_service.ts | 8 +- .../src/pxe_service/test/pxe_service.test.ts | 1 + .../pxe/src/simulator_oracle/index.ts | 8 +- .../simulator_oracle/simulator_oracle.test.ts | 6 +- 21 files changed, 318 insertions(+), 65 deletions(-) create mode 100644 yarn-project/archiver/src/archiver/kv_archiver_store/nullifier_store.ts create mode 100644 yarn-project/circuit-types/src/nullifier_with_block_source.ts diff --git a/yarn-project/archiver/src/archiver/archiver.ts b/yarn-project/archiver/src/archiver/archiver.ts index cd89bbc63d4..e31c9778208 100644 --- a/yarn-project/archiver/src/archiver/archiver.ts +++ b/yarn-project/archiver/src/archiver/archiver.ts @@ -13,6 +13,7 @@ import { type L2Tips, type LogFilter, type LogType, + type NullifierWithBlockSource, type TxEffect, type TxHash, type TxReceipt, @@ -74,7 +75,11 @@ import { type L1Published } from './structs/published.js'; /** * Helper interface to combine all sources this archiver implementation provides. */ -export type ArchiveSource = L2BlockSource & L2LogsSource & ContractDataSource & L1ToL2MessageSource; +export type ArchiveSource = L2BlockSource & + L2LogsSource & + ContractDataSource & + L1ToL2MessageSource & + NullifierWithBlockSource; /** * Pulls L2 blocks in a non-blocking manner and provides interface for their retrieval. @@ -644,6 +649,17 @@ export class Archiver implements ArchiveSource { return this.store.getLogsByTags(tags); } + /** + * Returns the provided nullifier indexes scoped to the block + * they were first included in, or undefined if they're not present in the tree + * @param blockNumber Max block number to search for the nullifiers + * @param nullifiers Nullifiers to get + * @returns The block scoped indexes of the provided nullifiers, or undefined if the nullifier doesn't exist in the tree + */ + findNullifiersIndexesWithBlock(blockNumber: number, nullifiers: Fr[]): Promise<(InBlock | undefined)[]> { + return this.store.findNullifiersIndexesWithBlock(blockNumber, nullifiers); + } + /** * Gets unencrypted logs based on the provided filter. * @param filter - The filter to apply to the logs. @@ -765,6 +781,8 @@ class ArchiverStoreHelper ArchiverDataStore, | 'addLogs' | 'deleteLogs' + | 'addNullifiers' + | 'deleteNullifiers' | 'addContractClasses' | 'deleteContractClasses' | 'addContractInstances' @@ -895,6 +913,7 @@ class ArchiverStoreHelper ).every(Boolean); }), )), + this.store.addNullifiers(blocks.map(block => block.data)), this.store.addBlocks(blocks), ].every(Boolean); } @@ -959,6 +978,9 @@ class ArchiverStoreHelper getLogsByTags(tags: Fr[]): Promise { return this.store.getLogsByTags(tags); } + findNullifiersIndexesWithBlock(blockNumber: number, nullifiers: Fr[]): Promise<(InBlock | undefined)[]> { + return this.store.findNullifiersIndexesWithBlock(blockNumber, nullifiers); + } getUnencryptedLogs(filter: LogFilter): Promise { return this.store.getUnencryptedLogs(filter); } diff --git a/yarn-project/archiver/src/archiver/archiver_store.ts b/yarn-project/archiver/src/archiver/archiver_store.ts index 5ca7e878d86..12a6d0e1f96 100644 --- a/yarn-project/archiver/src/archiver/archiver_store.ts +++ b/yarn-project/archiver/src/archiver/archiver_store.ts @@ -97,6 +97,23 @@ export interface ArchiverDataStore { addLogs(blocks: L2Block[]): Promise; deleteLogs(blocks: L2Block[]): Promise; + /** + * Append new nullifiers to the store's list. + * @param blocks - The blocks for which to add the nullifiers. + * @returns True if the operation is successful. + */ + addNullifiers(blocks: L2Block[]): Promise; + deleteNullifiers(blocks: L2Block[]): Promise; + + /** + * Returns the provided nullifier indexes scoped to the block + * they were first included in, or undefined if they're not present in the tree + * @param blockNumber Max block number to search for the nullifiers + * @param nullifiers Nullifiers to get + * @returns The block scoped indexes of the provided nullifiers, or undefined if the nullifier doesn't exist in the tree + */ + findNullifiersIndexesWithBlock(blockNumber: number, nullifiers: Fr[]): Promise<(InBlock | undefined)[]>; + /** * Append L1 to L2 messages to the store. * @param messages - The L1 to L2 messages to be added to the store and the last processed L1 block. diff --git a/yarn-project/archiver/src/archiver/archiver_store_test_suite.ts b/yarn-project/archiver/src/archiver/archiver_store_test_suite.ts index 05a66280d08..9253a99fede 100644 --- a/yarn-project/archiver/src/archiver/archiver_store_test_suite.ts +++ b/yarn-project/archiver/src/archiver/archiver_store_test_suite.ts @@ -7,6 +7,7 @@ import { Fr, INITIAL_L2_BLOCK_NUM, L1_TO_L2_MSG_SUBTREE_HEIGHT, + MAX_NULLIFIERS_PER_TX, SerializableContractInstance, } from '@aztec/circuits.js'; import { @@ -705,5 +706,58 @@ export function describeArchiverDataStore(testName: string, getStore: () => Arch } }); }); + + describe('findNullifiersIndexesWithBlock', () => { + let blocks: L2Block[]; + const numBlocks = 10; + const nullifiersPerBlock = new Map(); + + beforeEach(() => { + blocks = times(numBlocks, (index: number) => L2Block.random(index + 1, 1)); + + blocks.forEach((block, blockIndex) => { + nullifiersPerBlock.set( + blockIndex, + block.body.txEffects.flatMap(txEffect => txEffect.nullifiers), + ); + }); + }); + + it('returns wrapped nullifiers with blocks if they exist', async () => { + await store.addNullifiers(blocks); + const nullifiersToRetrieve = [...nullifiersPerBlock.get(0)!, ...nullifiersPerBlock.get(5)!, Fr.random()]; + const blockScopedNullifiers = await store.findNullifiersIndexesWithBlock(10, nullifiersToRetrieve); + + expect(blockScopedNullifiers).toHaveLength(nullifiersToRetrieve.length); + const [undefinedNullifier] = blockScopedNullifiers.slice(-1); + const realNullifiers = blockScopedNullifiers.slice(0, -1); + realNullifiers.forEach((blockScopedNullifier, index) => { + expect(blockScopedNullifier).not.toBeUndefined(); + const { data, l2BlockNumber } = blockScopedNullifier!; + expect(data).toEqual(expect.any(BigInt)); + expect(l2BlockNumber).toEqual(index < MAX_NULLIFIERS_PER_TX ? 1 : 6); + }); + expect(undefinedNullifier).toBeUndefined(); + }); + + it('returns wrapped nullifiers filtering by blockNumber', async () => { + await store.addNullifiers(blocks); + const nullifiersToRetrieve = [...nullifiersPerBlock.get(0)!, ...nullifiersPerBlock.get(5)!]; + const blockScopedNullifiers = await store.findNullifiersIndexesWithBlock(5, nullifiersToRetrieve); + + expect(blockScopedNullifiers).toHaveLength(nullifiersToRetrieve.length); + const undefinedNullifiers = blockScopedNullifiers.slice(-MAX_NULLIFIERS_PER_TX); + const realNullifiers = blockScopedNullifiers.slice(0, -MAX_NULLIFIERS_PER_TX); + realNullifiers.forEach(blockScopedNullifier => { + expect(blockScopedNullifier).not.toBeUndefined(); + const { data, l2BlockNumber } = blockScopedNullifier!; + expect(data).toEqual(expect.any(BigInt)); + expect(l2BlockNumber).toEqual(1); + }); + undefinedNullifiers.forEach(undefinedNullifier => { + expect(undefinedNullifier).toBeUndefined(); + }); + }); + }); }); } diff --git a/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts b/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts index 890a827288c..dcbf3d3691e 100644 --- a/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts +++ b/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts @@ -1,6 +1,7 @@ import { type FromLogType, type GetUnencryptedLogsResponse, + type InBlock, type InboxLeaf, type L2Block, type L2BlockL2Logs, @@ -32,6 +33,7 @@ import { ContractClassStore } from './contract_class_store.js'; import { ContractInstanceStore } from './contract_instance_store.js'; import { LogStore } from './log_store.js'; import { MessageStore } from './message_store.js'; +import { NullifierStore } from './nullifier_store.js'; /** * LMDB implementation of the ArchiverDataStore interface. @@ -39,6 +41,7 @@ import { MessageStore } from './message_store.js'; export class KVArchiverDataStore implements ArchiverDataStore { #blockStore: BlockStore; #logStore: LogStore; + #nullifierStore: NullifierStore; #messageStore: MessageStore; #contractClassStore: ContractClassStore; #contractInstanceStore: ContractInstanceStore; @@ -53,6 +56,7 @@ export class KVArchiverDataStore implements ArchiverDataStore { this.#contractClassStore = new ContractClassStore(db); this.#contractInstanceStore = new ContractInstanceStore(db); this.#contractArtifactStore = new ContractArtifactsStore(db); + this.#nullifierStore = new NullifierStore(db); } getContractArtifact(address: AztecAddress): Promise { @@ -184,6 +188,23 @@ export class KVArchiverDataStore implements ArchiverDataStore { return this.#logStore.deleteLogs(blocks); } + /** + * Append new nullifiers to the store's list. + * @param blocks - The blocks for which to add the nullifiers. + * @returns True if the operation is successful. + */ + addNullifiers(blocks: L2Block[]): Promise { + return this.#nullifierStore.addNullifiers(blocks); + } + + deleteNullifiers(blocks: L2Block[]): Promise { + return this.#nullifierStore.deleteNullifiers(blocks); + } + + findNullifiersIndexesWithBlock(blockNumber: number, nullifiers: Fr[]): Promise<(InBlock | undefined)[]> { + return this.#nullifierStore.findNullifiersIndexesWithBlock(blockNumber, nullifiers); + } + getTotalL1ToL2MessageCount(): Promise { return Promise.resolve(this.#messageStore.getTotalL1ToL2MessageCount()); } diff --git a/yarn-project/archiver/src/archiver/kv_archiver_store/nullifier_store.ts b/yarn-project/archiver/src/archiver/kv_archiver_store/nullifier_store.ts new file mode 100644 index 00000000000..716dedc5a0a --- /dev/null +++ b/yarn-project/archiver/src/archiver/kv_archiver_store/nullifier_store.ts @@ -0,0 +1,78 @@ +import { type InBlock, type L2Block } from '@aztec/circuit-types'; +import { type Fr, MAX_NULLIFIERS_PER_TX } from '@aztec/circuits.js'; +import { createDebugLogger } from '@aztec/foundation/log'; +import { type AztecKVStore, type AztecMap } from '@aztec/kv-store'; + +export class NullifierStore { + #nullifiersToBlockNumber: AztecMap; + #nullifiersToBlockHash: AztecMap; + #nullifiersToIndex: AztecMap; + #log = createDebugLogger('aztec:archiver:log_store'); + + constructor(private db: AztecKVStore) { + this.#nullifiersToBlockNumber = db.openMap('archiver_nullifiers_to_block_number'); + this.#nullifiersToBlockHash = db.openMap('archiver_nullifiers_to_block_hash'); + this.#nullifiersToIndex = db.openMap('archiver_nullifiers_to_index'); + } + + async addNullifiers(blocks: L2Block[]): Promise { + await this.db.transaction(() => { + blocks.forEach(block => { + const dataStartIndexForBlock = + block.header.state.partial.nullifierTree.nextAvailableLeafIndex - + block.body.numberOfTxsIncludingPadded * MAX_NULLIFIERS_PER_TX; + block.body.txEffects.forEach((txEffects, txIndex) => { + const dataStartIndexForTx = dataStartIndexForBlock + txIndex * MAX_NULLIFIERS_PER_TX; + txEffects.nullifiers.forEach((nullifier, nullifierIndex) => { + void this.#nullifiersToBlockNumber.set(nullifier.toString(), block.number); + void this.#nullifiersToBlockHash.set(nullifier.toString(), block.hash().toString()); + void this.#nullifiersToIndex.set(nullifier.toString(), dataStartIndexForTx + nullifierIndex); + }); + }); + }); + }); + return true; + } + + async deleteNullifiers(blocks: L2Block[]): Promise { + await this.db.transaction(() => { + for (const block of blocks) { + for (const nullifier of block.body.txEffects.flatMap(tx => tx.nullifiers)) { + void this.#nullifiersToBlockNumber.delete(nullifier.toString()); + void this.#nullifiersToBlockHash.delete(nullifier.toString()); + void this.#nullifiersToIndex.delete(nullifier.toString()); + } + } + }); + return true; + } + + async findNullifiersIndexesWithBlock( + blockNumber: number, + nullifiers: Fr[], + ): Promise<(InBlock | undefined)[]> { + const maybeNullifiers = await this.db.transaction(() => { + return nullifiers.map(nullifier => ({ + data: this.#nullifiersToIndex.get(nullifier.toString()), + l2BlockNumber: this.#nullifiersToBlockNumber.get(nullifier.toString()), + l2BlockHash: this.#nullifiersToBlockHash.get(nullifier.toString()), + })); + }); + return maybeNullifiers.map(({ data, l2BlockNumber, l2BlockHash }) => { + if ( + data === undefined || + l2BlockNumber === undefined || + l2BlockHash === undefined || + l2BlockNumber > blockNumber + ) { + return undefined; + } else { + return { + data: BigInt(data), + l2BlockNumber, + l2BlockHash, + } as InBlock; + } + }); + } +} diff --git a/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts b/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts index 441af8aa256..7f5418f79a6 100644 --- a/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts +++ b/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts @@ -29,6 +29,7 @@ import { type Header, INITIAL_L2_BLOCK_NUM, MAX_NOTE_HASHES_PER_TX, + MAX_NULLIFIERS_PER_TX, type UnconstrainedFunctionWithMembershipProof, } from '@aztec/circuits.js'; import { type ContractArtifact } from '@aztec/foundation/abi'; @@ -66,6 +67,8 @@ export class MemoryArchiverStore implements ArchiverDataStore { private contractClassLogsPerBlock: Map = new Map(); + private blockScopedNullifiers: Map = new Map(); + /** * Contains all L1 to L2 messages. */ @@ -299,6 +302,51 @@ export class MemoryArchiverStore implements ArchiverDataStore { return Promise.resolve(true); } + addNullifiers(blocks: L2Block[]): Promise { + blocks.forEach(block => { + const dataStartIndexForBlock = + block.header.state.partial.nullifierTree.nextAvailableLeafIndex - + block.body.numberOfTxsIncludingPadded * MAX_NULLIFIERS_PER_TX; + block.body.txEffects.forEach((txEffects, txIndex) => { + const dataStartIndexForTx = dataStartIndexForBlock + txIndex * MAX_NULLIFIERS_PER_TX; + txEffects.nullifiers.forEach((nullifier, nullifierIndex) => { + this.blockScopedNullifiers.set(nullifier.toString(), { + index: BigInt(dataStartIndexForTx + nullifierIndex), + blockNumber: block.number, + blockHash: block.hash().toString(), + }); + }); + }); + }); + return Promise.resolve(true); + } + + deleteNullifiers(blocks: L2Block[]): Promise { + blocks.forEach(block => { + block.body.txEffects.forEach(txEffect => { + txEffect.nullifiers.forEach(nullifier => { + this.blockScopedNullifiers.delete(nullifier.toString()); + }); + }); + }); + return Promise.resolve(true); + } + + findNullifiersIndexesWithBlock(blockNumber: number, nullifiers: Fr[]): Promise<(InBlock | undefined)[]> { + const blockScopedNullifiers = nullifiers.map(nullifier => { + const nullifierData = this.blockScopedNullifiers.get(nullifier.toString()); + if (nullifierData !== undefined && nullifierData.blockNumber <= blockNumber) { + return { + data: nullifierData.index, + l2BlockHash: nullifierData.blockHash, + l2BlockNumber: nullifierData.blockNumber, + } as InBlock; + } + return undefined; + }); + return Promise.resolve(blockScopedNullifiers); + } + getTotalL1ToL2MessageCount(): Promise { return Promise.resolve(this.l1ToL2Messages.getTotalL1ToL2MessageCount()); } diff --git a/yarn-project/aztec-node/src/aztec-node/server.test.ts b/yarn-project/aztec-node/src/aztec-node/server.test.ts index 09b1a2e081a..29bbf20e274 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.test.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.test.ts @@ -6,6 +6,7 @@ import { type L2LogsSource, MerkleTreeId, type MerkleTreeReadOperations, + type NullifierWithBlockSource, type WorldStateSynchronizer, mockTxForRollup, } from '@aztec/circuit-types'; @@ -53,6 +54,8 @@ describe('aztec node', () => { // all txs use the same allowed FPC class const contractSource = mock(); + const nullifierWithBlockSource = mock(); + const aztecNodeConfig: AztecNodeConfig = getConfigEnvVars(); node = new AztecNodeService( @@ -72,6 +75,7 @@ describe('aztec node', () => { l2LogsSource, contractSource, l1ToL2MessageSource, + nullifierWithBlockSource, worldState, undefined, 12345, diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index fa8cdb5bc42..a048c9a0bd9 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -17,6 +17,7 @@ import { LogType, MerkleTreeId, NullifierMembershipWitness, + type NullifierWithBlockSource, type ProcessedTx, type ProverConfig, PublicDataWitness, @@ -98,6 +99,7 @@ export class AztecNodeService implements AztecNode { protected readonly unencryptedLogsSource: L2LogsSource, protected readonly contractDataSource: ContractDataSource, protected readonly l1ToL2MessageSource: L1ToL2MessageSource, + protected readonly nullifierSource: NullifierWithBlockSource, protected readonly worldStateSynchronizer: WorldStateSynchronizer, protected readonly sequencer: SequencerClient | undefined, protected readonly l1ChainId: number, @@ -193,6 +195,7 @@ export class AztecNodeService implements AztecNode { archiver, archiver, archiver, + archiver, worldStateSynchronizer, sequencer, ethereumChain.chainInfo.id, @@ -432,31 +435,14 @@ export class AztecNodeService implements AztecNode { return await Promise.all(leafValues.map(leafValue => committedDb.findLeafIndex(treeId, leafValue.toBuffer()))); } - public async findLeavesIndexesWithApproxBlockNumber( - maxBlockNumber: L2BlockNumber, - minBlockNumber: number, - treeId: MerkleTreeId, - leafValues: Fr[], + public async findNullifiersIndexesWithBlock( + blockNumber: L2BlockNumber, + nullifiers: Fr[], ): Promise<(InBlock | undefined)[]> { - const initialBlockNumber = maxBlockNumber === 'latest' ? await this.getBlockNumber() : maxBlockNumber; - return await Promise.all( - leafValues.map(async leafValue => { - let blockNumber = initialBlockNumber; - let treeSnapshot = this.worldStateSynchronizer.getSnapshot(initialBlockNumber); - let index = await treeSnapshot.findLeafIndex(treeId, leafValue.toBuffer()); - if (!index) { - return undefined; - } - let meta = await treeSnapshot.getTreeInfo(treeId); - while (meta.size > index && blockNumber >= minBlockNumber) { - blockNumber -= 1; - treeSnapshot = this.worldStateSynchronizer.getSnapshot(blockNumber); - meta = await treeSnapshot.getTreeInfo(treeId); - } - const block = await this.getBlock(blockNumber + 1); - return wrapInBlock(index, block!); - }), - ); + if (blockNumber === 'latest') { + blockNumber = await this.getBlockNumber(); + } + return this.nullifierSource.findNullifiersIndexesWithBlock(blockNumber, nullifiers); } /** diff --git a/yarn-project/circuit-types/src/index.ts b/yarn-project/circuit-types/src/index.ts index 2040dedbbb3..eb25ca0b6b8 100644 --- a/yarn-project/circuit-types/src/index.ts +++ b/yarn-project/circuit-types/src/index.ts @@ -23,3 +23,4 @@ export * from './tx/index.js'; export * from './tx_effect.js'; export * from './tx_execution_request.js'; export * from './in_block.js'; +export * from './nullifier_with_block_source.js'; diff --git a/yarn-project/circuit-types/src/interfaces/archiver.test.ts b/yarn-project/circuit-types/src/interfaces/archiver.test.ts index 70a7ab772f4..deffffbc925 100644 --- a/yarn-project/circuit-types/src/interfaces/archiver.test.ts +++ b/yarn-project/circuit-types/src/interfaces/archiver.test.ts @@ -20,7 +20,7 @@ import { readFileSync } from 'fs'; import omit from 'lodash.omit'; import { resolve } from 'path'; -import { type InBlock } from '../in_block.js'; +import { type InBlock, randomInBlock } from '../in_block.js'; import { L2Block } from '../l2_block.js'; import { type L2Tips } from '../l2_block_source.js'; import { ExtendedUnencryptedL2Log } from '../logs/extended_unencrypted_l2_log.js'; @@ -144,6 +144,12 @@ describe('ArchiverApiSchema', () => { }); }); + it('findNullifiersIndexesWithBlock', async () => { + const nullifier = Fr.random(); + const result = await context.client.findNullifiersIndexesWithBlock(1, [nullifier]); + expect(result).toEqual({ data: expect.any(BigInt), l2BlockNumber: 1n, l2BlockHash: expect.any(String) }); + }); + it('getLogs(Encrypted)', async () => { const result = await context.client.getLogs(1, 1, LogType.ENCRYPTED); expect(result).toEqual([expect.any(EncryptedL2BlockL2Logs)]); @@ -291,6 +297,11 @@ class MockArchiver implements ArchiverApi { finalized: { number: 1, hash: `0x01` }, }); } + findNullifiersIndexesWithBlock(blockNumber: number, nullifiers: Fr[]): Promise<(InBlock | undefined)[]> { + expect(blockNumber).toEqual(1n); + expect(nullifiers[0]).toBeInstanceOf(Fr); + return Promise.resolve([randomInBlock(Fr.random().toBigInt())]); + } getLogs( _from: number, _limit: number, diff --git a/yarn-project/circuit-types/src/interfaces/archiver.ts b/yarn-project/circuit-types/src/interfaces/archiver.ts index 686b512a65e..9fafa9b6a7c 100644 --- a/yarn-project/circuit-types/src/interfaces/archiver.ts +++ b/yarn-project/circuit-types/src/interfaces/archiver.ts @@ -19,12 +19,13 @@ import { type L2LogsSource } from '../logs/l2_logs_source.js'; import { LogFilterSchema } from '../logs/log_filter.js'; import { LogType } from '../logs/log_type.js'; import { type L1ToL2MessageSource } from '../messaging/l1_to_l2_message_source.js'; +import { type NullifierWithBlockSource } from '../nullifier_with_block_source.js'; import { TxHash } from '../tx/tx_hash.js'; import { TxReceipt } from '../tx/tx_receipt.js'; import { TxEffect } from '../tx_effect.js'; export type ArchiverApi = Omit< - L2BlockSource & L2LogsSource & ContractDataSource & L1ToL2MessageSource, + L2BlockSource & L2LogsSource & ContractDataSource & L1ToL2MessageSource & NullifierWithBlockSource, 'start' | 'stop' >; @@ -58,6 +59,10 @@ export const ArchiverApiSchema: ApiSchemaFor = { .function() .args(z.array(schemas.Fr)) .returns(z.array(z.array(TxScopedL2Log.schema))), + findNullifiersIndexesWithBlock: z + .function() + .args(z.number(), z.array(schemas.Fr)) + .returns(z.array(optional(inBlockSchemaFor(schemas.BigInt)))), getUnencryptedLogs: z.function().args(LogFilterSchema).returns(GetUnencryptedLogsResponseSchema), getContractClassLogs: z.function().args(LogFilterSchema).returns(GetUnencryptedLogsResponseSchema), getPublicFunction: z diff --git a/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts b/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts index 89f9e7714b5..95a2813c47e 100644 --- a/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts +++ b/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts @@ -28,7 +28,7 @@ import omit from 'lodash.omit'; import times from 'lodash.times'; import { resolve } from 'path'; -import { type InBlock, wrapInBlock } from '../in_block.js'; +import { type InBlock, randomInBlock } from '../in_block.js'; import { L2Block } from '../l2_block.js'; import { type L2Tips } from '../l2_block_source.js'; import { ExtendedUnencryptedL2Log } from '../logs/extended_unencrypted_l2_log.js'; @@ -96,12 +96,12 @@ describe('AztecNodeApiSchema', () => { expect(response).toEqual([1n, undefined]); }); - it('findLeavesIndexesWithApproxBlockNumber', async () => { - const response = await context.client.findLeavesIndexesWithApproxBlockNumber(1, 1, MerkleTreeId.ARCHIVE, [ - Fr.random(), - Fr.random(), + it('findNullifierIndexesWithBlock', async () => { + const response = await context.client.findNullifiersIndexesWithBlock(1, [Fr.random(), Fr.random()]); + expect(response).toEqual([ + { data: 1n, l2BlockNumber: expect.any(Number), l2BlockHash: expect.any(String) }, + undefined, ]); - expect(response).toEqual([{ data: 1n, l2BlockNumber: 1, l2BlockHash: expect.any(String) }, undefined]); }); it('getNullifierSiblingPath', async () => { @@ -354,16 +354,14 @@ class MockAztecNode implements AztecNode { expect(leafValues[1]).toBeInstanceOf(Fr); return Promise.resolve([1n, undefined]); } - findLeavesIndexesWithApproxBlockNumber( - maxBlockNumber: number | 'latest', - minBlockNumber: number, - treeId: MerkleTreeId, - leafValues: Fr[], + findNullifiersIndexesWithBlock( + blockNumber: number | 'latest', + nullifiers: Fr[], ): Promise<(InBlock | undefined)[]> { - expect(leafValues).toHaveLength(2); - expect(leafValues[0]).toBeInstanceOf(Fr); - expect(leafValues[1]).toBeInstanceOf(Fr); - return Promise.resolve([wrapInBlock(1n, L2Block.random(1)), undefined]); + expect(nullifiers).toHaveLength(2); + expect(nullifiers[0]).toBeInstanceOf(Fr); + expect(nullifiers[1]).toBeInstanceOf(Fr); + return Promise.resolve([randomInBlock(1n), undefined]); } getNullifierSiblingPath( blockNumber: number | 'latest', diff --git a/yarn-project/circuit-types/src/interfaces/aztec-node.ts b/yarn-project/circuit-types/src/interfaces/aztec-node.ts index c1d071be9df..983f6b55934 100644 --- a/yarn-project/circuit-types/src/interfaces/aztec-node.ts +++ b/yarn-project/circuit-types/src/interfaces/aztec-node.ts @@ -71,11 +71,16 @@ export interface AztecNode leafValues: Fr[], ): Promise<(bigint | undefined)[]>; - findLeavesIndexesWithApproxBlockNumber( - maxBlockNumber: L2BlockNumber, - minBlockNumber: number, - treeId: MerkleTreeId, - leafValues: Fr[], + /** + * Returns the indexes of the given nullifiers in the nullifier tree, + * scoped to the block they were included in. + * @param blockNumber - The block number at which to get the data. + * @param nullifiers - The nullifiers to search for. + * @returns The block scoped indexes of the given nullifiers in the nullifier tree, or undefined if not found. + */ + findNullifiersIndexesWithBlock( + blockNumber: L2BlockNumber, + nullifiers: Fr[], ): Promise<(InBlock | undefined)[]>; /** @@ -418,9 +423,9 @@ export const AztecNodeApiSchema: ApiSchemaFor = { .args(L2BlockNumberSchema, z.nativeEnum(MerkleTreeId), z.array(schemas.Fr)) .returns(z.array(optional(schemas.BigInt))), - findLeavesIndexesWithApproxBlockNumber: z + findNullifiersIndexesWithBlock: z .function() - .args(L2BlockNumberSchema, z.number(), z.nativeEnum(MerkleTreeId), z.array(schemas.Fr)) + .args(L2BlockNumberSchema, z.array(schemas.Fr)) .returns(z.array(optional(inBlockSchemaFor(schemas.BigInt)))), getNullifierSiblingPath: z diff --git a/yarn-project/circuit-types/src/nullifier_with_block_source.ts b/yarn-project/circuit-types/src/nullifier_with_block_source.ts new file mode 100644 index 00000000000..d8716c6f3a4 --- /dev/null +++ b/yarn-project/circuit-types/src/nullifier_with_block_source.ts @@ -0,0 +1,7 @@ +import { type Fr } from '@aztec/circuits.js'; + +import { type InBlock } from './index.js'; + +export interface NullifierWithBlockSource { + findNullifiersIndexesWithBlock(blockNumber: number, nullifiers: Fr[]): Promise<(InBlock | undefined)[]>; +} diff --git a/yarn-project/pxe/src/database/kv_pxe_database.ts b/yarn-project/pxe/src/database/kv_pxe_database.ts index 2f8336b8d81..475897ef331 100644 --- a/yarn-project/pxe/src/database/kv_pxe_database.ts +++ b/yarn-project/pxe/src/database/kv_pxe_database.ts @@ -1,5 +1,5 @@ import { - InBlock, + type InBlock, type IncomingNotesFilter, MerkleTreeId, NoteStatus, diff --git a/yarn-project/pxe/src/database/pxe_database.ts b/yarn-project/pxe/src/database/pxe_database.ts index 09746a55ba1..f93d32240d3 100644 --- a/yarn-project/pxe/src/database/pxe_database.ts +++ b/yarn-project/pxe/src/database/pxe_database.ts @@ -1,4 +1,4 @@ -import { InBlock, type IncomingNotesFilter, type OutgoingNotesFilter } from '@aztec/circuit-types'; +import { type InBlock, type IncomingNotesFilter, type OutgoingNotesFilter } from '@aztec/circuit-types'; import { type CompleteAddress, type ContractInstanceWithAddress, diff --git a/yarn-project/pxe/src/database/pxe_database_test_suite.ts b/yarn-project/pxe/src/database/pxe_database_test_suite.ts index a3f334684f5..fb214faba15 100644 --- a/yarn-project/pxe/src/database/pxe_database_test_suite.ts +++ b/yarn-project/pxe/src/database/pxe_database_test_suite.ts @@ -2,7 +2,6 @@ import { type IncomingNotesFilter, NoteStatus, type OutgoingNotesFilter, randomT import { AztecAddress, CompleteAddress, - Header, INITIAL_L2_BLOCK_NUM, PublicKeys, SerializableContractInstance, diff --git a/yarn-project/pxe/src/pxe_service/create_pxe_service.ts b/yarn-project/pxe/src/pxe_service/create_pxe_service.ts index 5148121a22b..a3bdd43b105 100644 --- a/yarn-project/pxe/src/pxe_service/create_pxe_service.ts +++ b/yarn-project/pxe/src/pxe_service/create_pxe_service.ts @@ -40,9 +40,11 @@ export async function createPXEService( const keyStore = new KeyStore( await createStore('pxe_key_store', configWithContracts, createDebugLogger('aztec:pxe:keystore:lmdb')), ); - const db = new KVPxeDatabase( - await createStore('pxe_data', configWithContracts, createDebugLogger('aztec:pxe:data:lmdb')), - ); + + const store = await createStore('pxe_data', configWithContracts, createDebugLogger('aztec:pxe:data:lmdb')); + + const db = new KVPxeDatabase(store); + const tips = new L2TipsStore(store, 'pxe'); const prover = proofCreator ?? (await createProver(config, logSuffix)); const server = new PXEService(keyStore, aztecNode, db, tips, prover, config, logSuffix); diff --git a/yarn-project/pxe/src/pxe_service/test/pxe_service.test.ts b/yarn-project/pxe/src/pxe_service/test/pxe_service.test.ts index 4ce4e96c670..4acbc87e2a4 100644 --- a/yarn-project/pxe/src/pxe_service/test/pxe_service.test.ts +++ b/yarn-project/pxe/src/pxe_service/test/pxe_service.test.ts @@ -20,6 +20,7 @@ function createPXEService(): Promise { const keyStore = new KeyStore(kvStore); const node = mock(); const db = new KVPxeDatabase(kvStore); + const tips = new L2TipsStore(kvStore, 'pxe'); const config: PXEServiceConfig = { l2BlockPollingIntervalMS: 100, l2StartingBlock: INITIAL_L2_BLOCK_NUM, diff --git a/yarn-project/pxe/src/simulator_oracle/index.ts b/yarn-project/pxe/src/simulator_oracle/index.ts index 251f47a5891..9e6c6932210 100644 --- a/yarn-project/pxe/src/simulator_oracle/index.ts +++ b/yarn-project/pxe/src/simulator_oracle/index.ts @@ -567,13 +567,7 @@ export class SimulatorOracle implements DBOracle { const currentNotesForRecipient = await this.db.getIncomingNotes({ owner: recipient }); const nullifiersToCheck = currentNotesForRecipient.map(note => note.siloedNullifier); const currentBlockNumber = await this.getBlockNumber(); - const nullifierIndexes = await this.aztecNode.findLeavesIndexesWithApproxBlockNumber( - currentBlockNumber, - // Epoch size, we don't care about accurate block numbers in finalized nullifiers - currentBlockNumber - 2 * 32, - MerkleTreeId.NULLIFIER_TREE, - nullifiersToCheck, - ); + const nullifierIndexes = await this.aztecNode.findNullifiersIndexesWithBlock(currentBlockNumber, nullifiersToCheck); const foundNullifiers = nullifiersToCheck .map((nullifier, i) => { diff --git a/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts b/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts index bd51c6a7a2f..9e160d98e82 100644 --- a/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts +++ b/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts @@ -426,11 +426,11 @@ describe('Simulator oracle', () => { aztecNode.getTxEffect.mockImplementation(txHash => { return Promise.resolve(randomInBlock(txEffectsMap[txHash.toString()] as TxEffect)); }); - aztecNode.findLeavesIndexes.mockImplementation((_blockNumber, _treeId, leafValues) => + aztecNode.findNullifiersIndexesWithBlock.mockImplementation((_blockNumber, requestedNullifiers) => Promise.resolve( - Array(leafValues.length - nullifiers) + Array(requestedNullifiers.length - nullifiers) .fill(undefined) - .concat(Array(nullifiers).fill(1n)), + .concat(Array(nullifiers).fill({ data: 1n, l2BlockNumber: 1n, l2BlockHash: '0x' })), ), ); return taggedLogs; From 28a727f975c4c459c8ee222e77bad97012391e77 Mon Sep 17 00:00:00 2001 From: thunkar Date: Wed, 20 Nov 2024 08:52:50 +0000 Subject: [PATCH 09/14] pr comments --- yarn-project/pxe/src/database/kv_pxe_database.ts | 2 +- yarn-project/pxe/src/database/pxe_database_test_suite.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/yarn-project/pxe/src/database/kv_pxe_database.ts b/yarn-project/pxe/src/database/kv_pxe_database.ts index 475897ef331..b67c292e1b3 100644 --- a/yarn-project/pxe/src/database/kv_pxe_database.ts +++ b/yarn-project/pxe/src/database/kv_pxe_database.ts @@ -257,7 +257,7 @@ export class KVPxeDatabase implements PxeDatabase { public async unnullifyNotesAfter(blockNumber: number): Promise { const nullifiersToUndo: string[] = []; - let currentBlockNumber = blockNumber; + let currentBlockNumber = blockNumber + 1; const maxBlockNumber = this.getBlockNumber() ?? currentBlockNumber; while (currentBlockNumber <= maxBlockNumber) { nullifiersToUndo.push(...this.#nullifiersByBlockNumber.getValues(currentBlockNumber)); diff --git a/yarn-project/pxe/src/database/pxe_database_test_suite.ts b/yarn-project/pxe/src/database/pxe_database_test_suite.ts index fb214faba15..465aa153d5f 100644 --- a/yarn-project/pxe/src/database/pxe_database_test_suite.ts +++ b/yarn-project/pxe/src/database/pxe_database_test_suite.ts @@ -198,7 +198,7 @@ export function describePxeDatabase(getDatabase: () => PxeDatabase) { const notesToNullify = notes.filter(note => note.addressPoint.equals(computePoint(owners[0].address))); const nullifiers = notesToNullify.map(note => ({ data: note.siloedNullifier, - l2BlockNumber: 100, + l2BlockNumber: 99, l2BlockHash: Fr.random().toString(), })); await expect(database.removeNullifiedNotes(nullifiers, notesToNullify[0].addressPoint)).resolves.toEqual( From 82333fef69b847bc8b6e2e9728df046ea2a3c2e5 Mon Sep 17 00:00:00 2001 From: thunkar Date: Wed, 20 Nov 2024 10:57:16 +0000 Subject: [PATCH 10/14] fixes and fmt --- yarn-project/aztec-node/src/aztec-node/server.ts | 1 - yarn-project/circuit-types/src/interfaces/archiver.test.ts | 7 +++---- .../circuit-types/src/interfaces/aztec-node.test.ts | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index af8e74bc338..40a08864975 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -34,7 +34,6 @@ import { type TxValidator, type WorldStateSynchronizer, tryStop, - wrapInBlock, } from '@aztec/circuit-types'; import { type ARCHIVE_HEIGHT, diff --git a/yarn-project/circuit-types/src/interfaces/archiver.test.ts b/yarn-project/circuit-types/src/interfaces/archiver.test.ts index deffffbc925..a44553a2e56 100644 --- a/yarn-project/circuit-types/src/interfaces/archiver.test.ts +++ b/yarn-project/circuit-types/src/interfaces/archiver.test.ts @@ -145,9 +145,8 @@ describe('ArchiverApiSchema', () => { }); it('findNullifiersIndexesWithBlock', async () => { - const nullifier = Fr.random(); - const result = await context.client.findNullifiersIndexesWithBlock(1, [nullifier]); - expect(result).toEqual({ data: expect.any(BigInt), l2BlockNumber: 1n, l2BlockHash: expect.any(String) }); + const result = await context.client.findNullifiersIndexesWithBlock(1, [Fr.random()]); + expect(result).toEqual({ data: expect.any(BigInt), l2BlockNumber: 1, l2BlockHash: expect.any(String) }); }); it('getLogs(Encrypted)', async () => { @@ -298,7 +297,7 @@ class MockArchiver implements ArchiverApi { }); } findNullifiersIndexesWithBlock(blockNumber: number, nullifiers: Fr[]): Promise<(InBlock | undefined)[]> { - expect(blockNumber).toEqual(1n); + expect(blockNumber).toEqual(1); expect(nullifiers[0]).toBeInstanceOf(Fr); return Promise.resolve([randomInBlock(Fr.random().toBigInt())]); } diff --git a/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts b/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts index 95a2813c47e..f3174066d1d 100644 --- a/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts +++ b/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts @@ -96,7 +96,7 @@ describe('AztecNodeApiSchema', () => { expect(response).toEqual([1n, undefined]); }); - it('findNullifierIndexesWithBlock', async () => { + it('findNullifiersIndexesWithBlock', async () => { const response = await context.client.findNullifiersIndexesWithBlock(1, [Fr.random(), Fr.random()]); expect(response).toEqual([ { data: 1n, l2BlockNumber: expect.any(Number), l2BlockHash: expect.any(String) }, From 9396141711e633f2cb099a782b0f299cd619180e Mon Sep 17 00:00:00 2001 From: thunkar Date: Wed, 20 Nov 2024 11:00:47 +0000 Subject: [PATCH 11/14] review comment --- yarn-project/pxe/src/database/kv_pxe_database.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/yarn-project/pxe/src/database/kv_pxe_database.ts b/yarn-project/pxe/src/database/kv_pxe_database.ts index b67c292e1b3..0c4367fd1d4 100644 --- a/yarn-project/pxe/src/database/kv_pxe_database.ts +++ b/yarn-project/pxe/src/database/kv_pxe_database.ts @@ -259,9 +259,8 @@ export class KVPxeDatabase implements PxeDatabase { const nullifiersToUndo: string[] = []; let currentBlockNumber = blockNumber + 1; const maxBlockNumber = this.getBlockNumber() ?? currentBlockNumber; - while (currentBlockNumber <= maxBlockNumber) { + for (let blockNumber = currentBlockNumber; currentBlockNumber <= maxBlockNumber; blockNumber++) { nullifiersToUndo.push(...this.#nullifiersByBlockNumber.getValues(currentBlockNumber)); - currentBlockNumber++; } const notesIndexesToReinsert = await this.db.transaction(() => From 40b82f78475be4cc810da03bcf2627a61fcc055f Mon Sep 17 00:00:00 2001 From: thunkar Date: Wed, 20 Nov 2024 11:04:58 +0000 Subject: [PATCH 12/14] pr comments --- yarn-project/pxe/src/database/kv_pxe_database.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yarn-project/pxe/src/database/kv_pxe_database.ts b/yarn-project/pxe/src/database/kv_pxe_database.ts index 0c4367fd1d4..1f534c5a53d 100644 --- a/yarn-project/pxe/src/database/kv_pxe_database.ts +++ b/yarn-project/pxe/src/database/kv_pxe_database.ts @@ -259,8 +259,8 @@ export class KVPxeDatabase implements PxeDatabase { const nullifiersToUndo: string[] = []; let currentBlockNumber = blockNumber + 1; const maxBlockNumber = this.getBlockNumber() ?? currentBlockNumber; - for (let blockNumber = currentBlockNumber; currentBlockNumber <= maxBlockNumber; blockNumber++) { - nullifiersToUndo.push(...this.#nullifiersByBlockNumber.getValues(currentBlockNumber)); + for (let i = currentBlockNumber; i <= maxBlockNumber; i++) { + nullifiersToUndo.push(...this.#nullifiersByBlockNumber.getValues(i)); } const notesIndexesToReinsert = await this.db.transaction(() => From e4e80ae27c2e2b725bdd43cc7f7d723e4a5b4d58 Mon Sep 17 00:00:00 2001 From: thunkar Date: Wed, 20 Nov 2024 12:02:33 +0000 Subject: [PATCH 13/14] restore note scopes --- .../pxe/src/database/kv_pxe_database.ts | 35 ++++++++++++++----- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/yarn-project/pxe/src/database/kv_pxe_database.ts b/yarn-project/pxe/src/database/kv_pxe_database.ts index 1f534c5a53d..4e767830a59 100644 --- a/yarn-project/pxe/src/database/kv_pxe_database.ts +++ b/yarn-project/pxe/src/database/kv_pxe_database.ts @@ -47,6 +47,7 @@ export class KVPxeDatabase implements PxeDatabase { #nullifierToNoteId: AztecMap; #nullifiersByBlockNumber: AztecMultiMap; + #nullifiedNotesToScope: AztecMultiMap; #nullifiedNotesByContract: AztecMultiMap; #nullifiedNotesByStorageSlot: AztecMultiMap; #nullifiedNotesByTxHash: AztecMultiMap; @@ -64,6 +65,7 @@ export class KVPxeDatabase implements PxeDatabase { #outgoingNotesByOvpkM: AztecMultiMap; #scopes: AztecSet; + #notesToScope: AztecMultiMap; #notesByContractAndScope: Map>; #notesByStorageSlotAndScope: Map>; #notesByTxHashAndScope: Map>; @@ -97,6 +99,7 @@ export class KVPxeDatabase implements PxeDatabase { this.#nullifierToNoteId = db.openMap('nullifier_to_note'); this.#nullifiersByBlockNumber = db.openMultiMap('nullifier_to_block_number'); + this.#nullifiedNotesToScope = db.openMultiMap('nullified_notes_to_scope'); this.#nullifiedNotesByContract = db.openMultiMap('nullified_notes_by_contract'); this.#nullifiedNotesByStorageSlot = db.openMultiMap('nullified_notes_by_storage_slot'); this.#nullifiedNotesByTxHash = db.openMultiMap('nullified_notes_by_tx_hash'); @@ -110,6 +113,7 @@ export class KVPxeDatabase implements PxeDatabase { this.#outgoingNotesByOvpkM = db.openMultiMap('outgoing_notes_by_ovpk_m'); this.#scopes = db.openSet('scopes'); + this.#notesToScope = db.openMultiMap('notes_to_scope'); this.#notesByContractAndScope = new Map>(); this.#notesByStorageSlotAndScope = new Map>(); this.#notesByTxHashAndScope = new Map>(); @@ -205,6 +209,7 @@ export class KVPxeDatabase implements PxeDatabase { // Had we stored them by their nullifier, they would be returned in random order const noteIndex = toBufferBE(dao.index, 32).toString('hex'); void this.#notes.set(noteIndex, dao.toBuffer()); + void this.#notesToScope.set(noteIndex, scope.toString()); void this.#nullifierToNoteId.set(dao.siloedNullifier.toString(), noteIndex); void this.#notesByContractAndScope.get(scope.toString())!.set(dao.contractAddress.toString(), noteIndex); @@ -231,6 +236,7 @@ export class KVPxeDatabase implements PxeDatabase { if (noteDao.l2BlockNumber > blockNumber) { const noteIndex = toBufferBE(noteDao.index, 32).toString('hex'); void this.#notes.delete(noteIndex); + void this.#notesToScope.delete(noteIndex); void this.#nullifierToNoteId.delete(noteDao.siloedNullifier.toString()); for (const scope of this.#scopes.entries()) { void this.#notesByAddressPointAndScope.get(scope)!.deleteValue(noteDao.addressPoint.toString(), noteIndex); @@ -281,15 +287,22 @@ export class KVPxeDatabase implements PxeDatabase { void this.#notes.set(noteIndex, dao.toBuffer()); void this.#nullifierToNoteId.set(dao.siloedNullifier.toString(), noteIndex); - // TODO: check if is this a good assumption - const scope = new AztecAddress(dao.addressPoint.x); + let scopes = Array.from(this.#nullifiedNotesToScope.getValues(noteIndex) ?? []); - void this.#notesByContractAndScope.get(scope.toString())!.set(dao.contractAddress.toString(), noteIndex); - void this.#notesByStorageSlotAndScope.get(scope.toString())!.set(dao.storageSlot.toString(), noteIndex); - void this.#notesByTxHashAndScope.get(scope.toString())!.set(dao.txHash.toString(), noteIndex); - void this.#notesByAddressPointAndScope.get(scope.toString())!.set(dao.addressPoint.toString(), noteIndex); + if (scopes.length === 0) { + scopes = [new AztecAddress(dao.addressPoint.x).toString()]; + } + + for (const scope of scopes) { + void this.#notesByContractAndScope.get(scope)!.set(dao.contractAddress.toString(), noteIndex); + void this.#notesByStorageSlotAndScope.get(scope)!.set(dao.storageSlot.toString(), noteIndex); + void this.#notesByTxHashAndScope.get(scope)!.set(dao.txHash.toString(), noteIndex); + void this.#notesByAddressPointAndScope.get(scope)!.set(dao.addressPoint.toString(), noteIndex); + void this.#notesToScope.set(noteIndex, scope); + } void this.#nullifiedNotes.delete(noteIndex); + void this.#nullifiedNotesToScope.delete(noteIndex); void this.#nullifiersByBlockNumber.deleteValue(dao.l2BlockNumber, dao.siloedNullifier.toString()); void this.#nullifiedNotesByContract.deleteValue(dao.contractAddress.toString(), noteIndex); void this.#nullifiedNotesByStorageSlot.deleteValue(dao.storageSlot.toString(), noteIndex); @@ -457,7 +470,7 @@ export class KVPxeDatabase implements PxeDatabase { // note doesn't exist. Maybe it got nullified already continue; } - + const noteScopes = this.#notesToScope.getValues(noteIndex) ?? []; const note = IncomingNoteDao.fromBuffer(noteBuffer); if (!note.addressPoint.equals(accountAddressPoint)) { // tried to nullify someone else's note @@ -467,14 +480,20 @@ export class KVPxeDatabase implements PxeDatabase { nullifiedNotes.push(note); void this.#notes.delete(noteIndex); + void this.#notesToScope.delete(noteIndex); - for (const scope in this.#scopes.entries()) { + for (const scope of this.#scopes.entries()) { void this.#notesByAddressPointAndScope.get(scope)!.deleteValue(accountAddressPoint.toString(), noteIndex); void this.#notesByTxHashAndScope.get(scope)!.deleteValue(note.txHash.toString(), noteIndex); void this.#notesByContractAndScope.get(scope)!.deleteValue(note.contractAddress.toString(), noteIndex); void this.#notesByStorageSlotAndScope.get(scope)!.deleteValue(note.storageSlot.toString(), noteIndex); } + if (noteScopes !== undefined) { + for (const scope of noteScopes) { + this.#nullifiedNotesToScope.set(noteIndex, scope); + } + } void this.#nullifiedNotes.set(noteIndex, note.toBuffer()); void this.#nullifiersByBlockNumber.set(blockNumber, nullifier.toString()); void this.#nullifiedNotesByContract.set(note.contractAddress.toString(), noteIndex); From 11e4f25d409f63ec0c787d2820534f1da7400732 Mon Sep 17 00:00:00 2001 From: thunkar Date: Wed, 20 Nov 2024 13:06:45 +0000 Subject: [PATCH 14/14] test and fmt --- .../circuit-types/src/interfaces/archiver.test.ts | 15 ++++++++++++--- yarn-project/pxe/src/database/kv_pxe_database.ts | 4 ++-- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/yarn-project/circuit-types/src/interfaces/archiver.test.ts b/yarn-project/circuit-types/src/interfaces/archiver.test.ts index a44553a2e56..fc3e30d9cc8 100644 --- a/yarn-project/circuit-types/src/interfaces/archiver.test.ts +++ b/yarn-project/circuit-types/src/interfaces/archiver.test.ts @@ -145,8 +145,15 @@ describe('ArchiverApiSchema', () => { }); it('findNullifiersIndexesWithBlock', async () => { - const result = await context.client.findNullifiersIndexesWithBlock(1, [Fr.random()]); - expect(result).toEqual({ data: expect.any(BigInt), l2BlockNumber: 1, l2BlockHash: expect.any(String) }); + const result = await context.client.findNullifiersIndexesWithBlock(1, [Fr.random(), Fr.random()]); + expect(result).toEqual([ + { + data: expect.any(BigInt), + l2BlockNumber: expect.any(Number), + l2BlockHash: expect.any(String), + }, + undefined, + ]); }); it('getLogs(Encrypted)', async () => { @@ -298,8 +305,10 @@ class MockArchiver implements ArchiverApi { } findNullifiersIndexesWithBlock(blockNumber: number, nullifiers: Fr[]): Promise<(InBlock | undefined)[]> { expect(blockNumber).toEqual(1); + expect(nullifiers).toHaveLength(2); expect(nullifiers[0]).toBeInstanceOf(Fr); - return Promise.resolve([randomInBlock(Fr.random().toBigInt())]); + expect(nullifiers[1]).toBeInstanceOf(Fr); + return Promise.resolve([randomInBlock(Fr.random().toBigInt()), undefined]); } getLogs( _from: number, diff --git a/yarn-project/pxe/src/database/kv_pxe_database.ts b/yarn-project/pxe/src/database/kv_pxe_database.ts index 4e767830a59..9421b66231d 100644 --- a/yarn-project/pxe/src/database/kv_pxe_database.ts +++ b/yarn-project/pxe/src/database/kv_pxe_database.ts @@ -263,7 +263,7 @@ export class KVPxeDatabase implements PxeDatabase { public async unnullifyNotesAfter(blockNumber: number): Promise { const nullifiersToUndo: string[] = []; - let currentBlockNumber = blockNumber + 1; + const currentBlockNumber = blockNumber + 1; const maxBlockNumber = this.getBlockNumber() ?? currentBlockNumber; for (let i = currentBlockNumber; i <= maxBlockNumber; i++) { nullifiersToUndo.push(...this.#nullifiersByBlockNumber.getValues(i)); @@ -491,7 +491,7 @@ export class KVPxeDatabase implements PxeDatabase { if (noteScopes !== undefined) { for (const scope of noteScopes) { - this.#nullifiedNotesToScope.set(noteIndex, scope); + void this.#nullifiedNotesToScope.set(noteIndex, scope); } } void this.#nullifiedNotes.set(noteIndex, note.toBuffer());