diff --git a/yarn-project/archiver/src/archiver/archiver.ts b/yarn-project/archiver/src/archiver/archiver.ts index b2d8887dd97..cd1c78cb3b2 100644 --- a/yarn-project/archiver/src/archiver/archiver.ts +++ b/yarn-project/archiver/src/archiver/archiver.ts @@ -2,6 +2,7 @@ import { type EncryptedL2Log, type FromLogType, type GetUnencryptedLogsResponse, + type InBlock, type InboxLeaf, type L1ToL2MessageSource, type L2Block, @@ -12,6 +13,7 @@ import { type L2Tips, type LogFilter, type LogType, + type NullifierWithBlockSource, type TxEffect, type TxHash, type TxReceipt, @@ -73,7 +75,11 @@ import { type L1Published } from './structs/published.js'; /** * Helper interface to combine all sources this archiver implementation provides. */ -export type ArchiveSource = L2BlockSource & L2LogsSource & ContractDataSource & L1ToL2MessageSource; +export type ArchiveSource = L2BlockSource & + L2LogsSource & + ContractDataSource & + L1ToL2MessageSource & + NullifierWithBlockSource; /** * Pulls L2 blocks in a non-blocking manner and provides interface for their retrieval. @@ -589,7 +595,7 @@ export class Archiver implements ArchiveSource { } } - public getTxEffect(txHash: TxHash): Promise { + public getTxEffect(txHash: TxHash) { return this.store.getTxEffect(txHash); } @@ -643,6 +649,17 @@ export class Archiver implements ArchiveSource { return this.store.getLogsByTags(tags); } + /** + * Returns the provided nullifier indexes scoped to the block + * they were first included in, or undefined if they're not present in the tree + * @param blockNumber Max block number to search for the nullifiers + * @param nullifiers Nullifiers to get + * @returns The block scoped indexes of the provided nullifiers, or undefined if the nullifier doesn't exist in the tree + */ + findNullifiersIndexesWithBlock(blockNumber: number, nullifiers: Fr[]): Promise<(InBlock | undefined)[]> { + return this.store.findNullifiersIndexesWithBlock(blockNumber, nullifiers); + } + /** * Gets unencrypted logs based on the provided filter. * @param filter - The filter to apply to the logs. @@ -770,6 +787,9 @@ class ArchiverStoreHelper ArchiverDataStore, | 'addLogs' | 'deleteLogs' + | 'addNullifiers' + | 'deleteNullifiers' + | 'addContractClasses' | 'deleteContractClasses' | 'addContractInstances' | 'deleteContractInstances' @@ -904,6 +924,7 @@ class ArchiverStoreHelper ).every(Boolean); }), )), + this.store.addNullifiers(blocks.map(block => block.data)), this.store.addBlocks(blocks), ].every(Boolean); } @@ -943,7 +964,7 @@ class ArchiverStoreHelper getBlockHeaders(from: number, limit: number): Promise { return this.store.getBlockHeaders(from, limit); } - getTxEffect(txHash: TxHash): Promise { + getTxEffect(txHash: TxHash): Promise | undefined> { return this.store.getTxEffect(txHash); } getSettledTxReceipt(txHash: TxHash): Promise { @@ -968,6 +989,9 @@ class ArchiverStoreHelper getLogsByTags(tags: Fr[]): Promise { return this.store.getLogsByTags(tags); } + findNullifiersIndexesWithBlock(blockNumber: number, nullifiers: Fr[]): Promise<(InBlock | undefined)[]> { + return this.store.findNullifiersIndexesWithBlock(blockNumber, nullifiers); + } getUnencryptedLogs(filter: LogFilter): Promise { return this.store.getUnencryptedLogs(filter); } diff --git a/yarn-project/archiver/src/archiver/archiver_store.ts b/yarn-project/archiver/src/archiver/archiver_store.ts index 4eb2c80ccc0..12a6d0e1f96 100644 --- a/yarn-project/archiver/src/archiver/archiver_store.ts +++ b/yarn-project/archiver/src/archiver/archiver_store.ts @@ -1,6 +1,7 @@ import { type FromLogType, type GetUnencryptedLogsResponse, + type InBlock, type InboxLeaf, type L2Block, type L2BlockL2Logs, @@ -79,7 +80,7 @@ export interface ArchiverDataStore { * @param txHash - The txHash of the tx corresponding to the tx effect. * @returns The requested tx effect (or undefined if not found). */ - getTxEffect(txHash: TxHash): Promise; + getTxEffect(txHash: TxHash): Promise | undefined>; /** * Gets a receipt of a settled tx. @@ -96,6 +97,23 @@ export interface ArchiverDataStore { addLogs(blocks: L2Block[]): Promise; deleteLogs(blocks: L2Block[]): Promise; + /** + * Append new nullifiers to the store's list. + * @param blocks - The blocks for which to add the nullifiers. + * @returns True if the operation is successful. + */ + addNullifiers(blocks: L2Block[]): Promise; + deleteNullifiers(blocks: L2Block[]): Promise; + + /** + * Returns the provided nullifier indexes scoped to the block + * they were first included in, or undefined if they're not present in the tree + * @param blockNumber Max block number to search for the nullifiers + * @param nullifiers Nullifiers to get + * @returns The block scoped indexes of the provided nullifiers, or undefined if the nullifier doesn't exist in the tree + */ + findNullifiersIndexesWithBlock(blockNumber: number, nullifiers: Fr[]): Promise<(InBlock | undefined)[]>; + /** * Append L1 to L2 messages to the store. * @param messages - The L1 to L2 messages to be added to the store and the last processed L1 block. diff --git a/yarn-project/archiver/src/archiver/archiver_store_test_suite.ts b/yarn-project/archiver/src/archiver/archiver_store_test_suite.ts index cd30af56a78..9253a99fede 100644 --- a/yarn-project/archiver/src/archiver/archiver_store_test_suite.ts +++ b/yarn-project/archiver/src/archiver/archiver_store_test_suite.ts @@ -1,4 +1,4 @@ -import { InboxLeaf, L2Block, LogId, LogType, TxHash } from '@aztec/circuit-types'; +import { InboxLeaf, L2Block, LogId, LogType, TxHash, wrapInBlock } from '@aztec/circuit-types'; import '@aztec/circuit-types/jest'; import { AztecAddress, @@ -7,6 +7,7 @@ import { Fr, INITIAL_L2_BLOCK_NUM, L1_TO_L2_MSG_SUBTREE_HEIGHT, + MAX_NULLIFIERS_PER_TX, SerializableContractInstance, } from '@aztec/circuits.js'; import { @@ -191,14 +192,14 @@ export function describeArchiverDataStore(testName: string, getStore: () => Arch }); it.each([ - () => blocks[0].data.body.txEffects[0], - () => blocks[9].data.body.txEffects[3], - () => blocks[3].data.body.txEffects[1], - () => blocks[5].data.body.txEffects[2], - () => blocks[1].data.body.txEffects[0], + () => wrapInBlock(blocks[0].data.body.txEffects[0], blocks[0].data), + () => wrapInBlock(blocks[9].data.body.txEffects[3], blocks[9].data), + () => wrapInBlock(blocks[3].data.body.txEffects[1], blocks[3].data), + () => wrapInBlock(blocks[5].data.body.txEffects[2], blocks[5].data), + () => wrapInBlock(blocks[1].data.body.txEffects[0], blocks[1].data), ])('retrieves a previously stored transaction', async getExpectedTx => { const expectedTx = getExpectedTx(); - const actualTx = await store.getTxEffect(expectedTx.txHash); + const actualTx = await store.getTxEffect(expectedTx.data.txHash); expect(actualTx).toEqual(expectedTx); }); @@ -207,16 +208,16 @@ export function describeArchiverDataStore(testName: string, getStore: () => Arch }); it.each([ - () => blocks[0].data.body.txEffects[0], - () => blocks[9].data.body.txEffects[3], - () => blocks[3].data.body.txEffects[1], - () => blocks[5].data.body.txEffects[2], - () => blocks[1].data.body.txEffects[0], + () => wrapInBlock(blocks[0].data.body.txEffects[0], blocks[0].data), + () => wrapInBlock(blocks[9].data.body.txEffects[3], blocks[9].data), + () => wrapInBlock(blocks[3].data.body.txEffects[1], blocks[3].data), + () => wrapInBlock(blocks[5].data.body.txEffects[2], blocks[5].data), + () => wrapInBlock(blocks[1].data.body.txEffects[0], blocks[1].data), ])('tries to retrieves a previously stored transaction after deleted', async getExpectedTx => { await store.unwindBlocks(blocks.length, blocks.length); const expectedTx = getExpectedTx(); - const actualTx = await store.getTxEffect(expectedTx.txHash); + const actualTx = await store.getTxEffect(expectedTx.data.txHash); expect(actualTx).toEqual(undefined); }); @@ -705,5 +706,58 @@ export function describeArchiverDataStore(testName: string, getStore: () => Arch } }); }); + + describe('findNullifiersIndexesWithBlock', () => { + let blocks: L2Block[]; + const numBlocks = 10; + const nullifiersPerBlock = new Map(); + + beforeEach(() => { + blocks = times(numBlocks, (index: number) => L2Block.random(index + 1, 1)); + + blocks.forEach((block, blockIndex) => { + nullifiersPerBlock.set( + blockIndex, + block.body.txEffects.flatMap(txEffect => txEffect.nullifiers), + ); + }); + }); + + it('returns wrapped nullifiers with blocks if they exist', async () => { + await store.addNullifiers(blocks); + const nullifiersToRetrieve = [...nullifiersPerBlock.get(0)!, ...nullifiersPerBlock.get(5)!, Fr.random()]; + const blockScopedNullifiers = await store.findNullifiersIndexesWithBlock(10, nullifiersToRetrieve); + + expect(blockScopedNullifiers).toHaveLength(nullifiersToRetrieve.length); + const [undefinedNullifier] = blockScopedNullifiers.slice(-1); + const realNullifiers = blockScopedNullifiers.slice(0, -1); + realNullifiers.forEach((blockScopedNullifier, index) => { + expect(blockScopedNullifier).not.toBeUndefined(); + const { data, l2BlockNumber } = blockScopedNullifier!; + expect(data).toEqual(expect.any(BigInt)); + expect(l2BlockNumber).toEqual(index < MAX_NULLIFIERS_PER_TX ? 1 : 6); + }); + expect(undefinedNullifier).toBeUndefined(); + }); + + it('returns wrapped nullifiers filtering by blockNumber', async () => { + await store.addNullifiers(blocks); + const nullifiersToRetrieve = [...nullifiersPerBlock.get(0)!, ...nullifiersPerBlock.get(5)!]; + const blockScopedNullifiers = await store.findNullifiersIndexesWithBlock(5, nullifiersToRetrieve); + + expect(blockScopedNullifiers).toHaveLength(nullifiersToRetrieve.length); + const undefinedNullifiers = blockScopedNullifiers.slice(-MAX_NULLIFIERS_PER_TX); + const realNullifiers = blockScopedNullifiers.slice(0, -MAX_NULLIFIERS_PER_TX); + realNullifiers.forEach(blockScopedNullifier => { + expect(blockScopedNullifier).not.toBeUndefined(); + const { data, l2BlockNumber } = blockScopedNullifier!; + expect(data).toEqual(expect.any(BigInt)); + expect(l2BlockNumber).toEqual(1); + }); + undefinedNullifiers.forEach(undefinedNullifier => { + expect(undefinedNullifier).toBeUndefined(); + }); + }); + }); }); } diff --git a/yarn-project/archiver/src/archiver/kv_archiver_store/block_store.ts b/yarn-project/archiver/src/archiver/kv_archiver_store/block_store.ts index a4a3ba3281c..85bbacc0369 100644 --- a/yarn-project/archiver/src/archiver/kv_archiver_store/block_store.ts +++ b/yarn-project/archiver/src/archiver/kv_archiver_store/block_store.ts @@ -1,4 +1,4 @@ -import { Body, L2Block, type TxEffect, type TxHash, TxReceipt } from '@aztec/circuit-types'; +import { Body, type InBlock, L2Block, type TxEffect, type TxHash, TxReceipt } from '@aztec/circuit-types'; import { AppendOnlyTreeSnapshot, type AztecAddress, Header, INITIAL_L2_BLOCK_NUM } from '@aztec/circuits.js'; import { createDebugLogger } from '@aztec/foundation/log'; import { type AztecKVStore, type AztecMap, type AztecSingleton, type Range } from '@aztec/kv-store'; @@ -170,14 +170,22 @@ export class BlockStore { * @param txHash - The txHash of the tx corresponding to the tx effect. * @returns The requested tx effect (or undefined if not found). */ - getTxEffect(txHash: TxHash): TxEffect | undefined { + getTxEffect(txHash: TxHash): InBlock | undefined { const [blockNumber, txIndex] = this.getTxLocation(txHash) ?? []; if (typeof blockNumber !== 'number' || typeof txIndex !== 'number') { return undefined; } const block = this.getBlock(blockNumber); - return block?.data.body.txEffects[txIndex]; + if (!block) { + return undefined; + } + + return { + data: block.data.body.txEffects[txIndex], + l2BlockNumber: block.data.number, + l2BlockHash: block.data.hash().toString(), + }; } /** diff --git a/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts b/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts index 8cbb627a5ce..dcbf3d3691e 100644 --- a/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts +++ b/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts @@ -1,12 +1,12 @@ import { type FromLogType, type GetUnencryptedLogsResponse, + type InBlock, type InboxLeaf, type L2Block, type L2BlockL2Logs, type LogFilter, type LogType, - type TxEffect, type TxHash, type TxReceipt, type TxScopedL2Log, @@ -33,6 +33,7 @@ import { ContractClassStore } from './contract_class_store.js'; import { ContractInstanceStore } from './contract_instance_store.js'; import { LogStore } from './log_store.js'; import { MessageStore } from './message_store.js'; +import { NullifierStore } from './nullifier_store.js'; /** * LMDB implementation of the ArchiverDataStore interface. @@ -40,6 +41,7 @@ import { MessageStore } from './message_store.js'; export class KVArchiverDataStore implements ArchiverDataStore { #blockStore: BlockStore; #logStore: LogStore; + #nullifierStore: NullifierStore; #messageStore: MessageStore; #contractClassStore: ContractClassStore; #contractInstanceStore: ContractInstanceStore; @@ -54,6 +56,7 @@ export class KVArchiverDataStore implements ArchiverDataStore { this.#contractClassStore = new ContractClassStore(db); this.#contractInstanceStore = new ContractInstanceStore(db); this.#contractArtifactStore = new ContractArtifactsStore(db); + this.#nullifierStore = new NullifierStore(db); } getContractArtifact(address: AztecAddress): Promise { @@ -159,7 +162,7 @@ export class KVArchiverDataStore implements ArchiverDataStore { * @param txHash - The txHash of the tx corresponding to the tx effect. * @returns The requested tx effect (or undefined if not found). */ - getTxEffect(txHash: TxHash): Promise { + getTxEffect(txHash: TxHash) { return Promise.resolve(this.#blockStore.getTxEffect(txHash)); } @@ -185,6 +188,23 @@ export class KVArchiverDataStore implements ArchiverDataStore { return this.#logStore.deleteLogs(blocks); } + /** + * Append new nullifiers to the store's list. + * @param blocks - The blocks for which to add the nullifiers. + * @returns True if the operation is successful. + */ + addNullifiers(blocks: L2Block[]): Promise { + return this.#nullifierStore.addNullifiers(blocks); + } + + deleteNullifiers(blocks: L2Block[]): Promise { + return this.#nullifierStore.deleteNullifiers(blocks); + } + + findNullifiersIndexesWithBlock(blockNumber: number, nullifiers: Fr[]): Promise<(InBlock | undefined)[]> { + return this.#nullifierStore.findNullifiersIndexesWithBlock(blockNumber, nullifiers); + } + getTotalL1ToL2MessageCount(): Promise { return Promise.resolve(this.#messageStore.getTotalL1ToL2MessageCount()); } diff --git a/yarn-project/archiver/src/archiver/kv_archiver_store/nullifier_store.ts b/yarn-project/archiver/src/archiver/kv_archiver_store/nullifier_store.ts new file mode 100644 index 00000000000..716dedc5a0a --- /dev/null +++ b/yarn-project/archiver/src/archiver/kv_archiver_store/nullifier_store.ts @@ -0,0 +1,78 @@ +import { type InBlock, type L2Block } from '@aztec/circuit-types'; +import { type Fr, MAX_NULLIFIERS_PER_TX } from '@aztec/circuits.js'; +import { createDebugLogger } from '@aztec/foundation/log'; +import { type AztecKVStore, type AztecMap } from '@aztec/kv-store'; + +export class NullifierStore { + #nullifiersToBlockNumber: AztecMap; + #nullifiersToBlockHash: AztecMap; + #nullifiersToIndex: AztecMap; + #log = createDebugLogger('aztec:archiver:log_store'); + + constructor(private db: AztecKVStore) { + this.#nullifiersToBlockNumber = db.openMap('archiver_nullifiers_to_block_number'); + this.#nullifiersToBlockHash = db.openMap('archiver_nullifiers_to_block_hash'); + this.#nullifiersToIndex = db.openMap('archiver_nullifiers_to_index'); + } + + async addNullifiers(blocks: L2Block[]): Promise { + await this.db.transaction(() => { + blocks.forEach(block => { + const dataStartIndexForBlock = + block.header.state.partial.nullifierTree.nextAvailableLeafIndex - + block.body.numberOfTxsIncludingPadded * MAX_NULLIFIERS_PER_TX; + block.body.txEffects.forEach((txEffects, txIndex) => { + const dataStartIndexForTx = dataStartIndexForBlock + txIndex * MAX_NULLIFIERS_PER_TX; + txEffects.nullifiers.forEach((nullifier, nullifierIndex) => { + void this.#nullifiersToBlockNumber.set(nullifier.toString(), block.number); + void this.#nullifiersToBlockHash.set(nullifier.toString(), block.hash().toString()); + void this.#nullifiersToIndex.set(nullifier.toString(), dataStartIndexForTx + nullifierIndex); + }); + }); + }); + }); + return true; + } + + async deleteNullifiers(blocks: L2Block[]): Promise { + await this.db.transaction(() => { + for (const block of blocks) { + for (const nullifier of block.body.txEffects.flatMap(tx => tx.nullifiers)) { + void this.#nullifiersToBlockNumber.delete(nullifier.toString()); + void this.#nullifiersToBlockHash.delete(nullifier.toString()); + void this.#nullifiersToIndex.delete(nullifier.toString()); + } + } + }); + return true; + } + + async findNullifiersIndexesWithBlock( + blockNumber: number, + nullifiers: Fr[], + ): Promise<(InBlock | undefined)[]> { + const maybeNullifiers = await this.db.transaction(() => { + return nullifiers.map(nullifier => ({ + data: this.#nullifiersToIndex.get(nullifier.toString()), + l2BlockNumber: this.#nullifiersToBlockNumber.get(nullifier.toString()), + l2BlockHash: this.#nullifiersToBlockHash.get(nullifier.toString()), + })); + }); + return maybeNullifiers.map(({ data, l2BlockNumber, l2BlockHash }) => { + if ( + data === undefined || + l2BlockNumber === undefined || + l2BlockHash === undefined || + l2BlockNumber > blockNumber + ) { + return undefined; + } else { + return { + data: BigInt(data), + l2BlockNumber, + l2BlockHash, + } as InBlock; + } + }); + } +} diff --git a/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts b/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts index e49ab8eccc2..7f5418f79a6 100644 --- a/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts +++ b/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts @@ -6,6 +6,7 @@ import { ExtendedUnencryptedL2Log, type FromLogType, type GetUnencryptedLogsResponse, + type InBlock, type InboxLeaf, type L2Block, type L2BlockL2Logs, @@ -17,6 +18,7 @@ import { TxReceipt, TxScopedL2Log, type UnencryptedL2BlockL2Logs, + wrapInBlock, } from '@aztec/circuit-types'; import { type ContractClassPublic, @@ -27,6 +29,7 @@ import { type Header, INITIAL_L2_BLOCK_NUM, MAX_NOTE_HASHES_PER_TX, + MAX_NULLIFIERS_PER_TX, type UnconstrainedFunctionWithMembershipProof, } from '@aztec/circuits.js'; import { type ContractArtifact } from '@aztec/foundation/abi'; @@ -50,7 +53,7 @@ export class MemoryArchiverStore implements ArchiverDataStore { /** * An array containing all the tx effects in the L2 blocks that have been fetched so far. */ - private txEffects: TxEffect[] = []; + private txEffects: InBlock[] = []; private noteEncryptedLogsPerBlock: Map = new Map(); @@ -64,6 +67,8 @@ export class MemoryArchiverStore implements ArchiverDataStore { private contractClassLogsPerBlock: Map = new Map(); + private blockScopedNullifiers: Map = new Map(); + /** * Contains all L1 to L2 messages. */ @@ -181,7 +186,7 @@ export class MemoryArchiverStore implements ArchiverDataStore { this.lastL1BlockNewBlocks = blocks[blocks.length - 1].l1.blockNumber; this.l2Blocks.push(...blocks); - this.txEffects.push(...blocks.flatMap(b => b.data.body.txEffects)); + this.txEffects.push(...blocks.flatMap(b => b.data.body.txEffects.map(txEffect => wrapInBlock(txEffect, b.data)))); return Promise.resolve(true); } @@ -297,6 +302,51 @@ export class MemoryArchiverStore implements ArchiverDataStore { return Promise.resolve(true); } + addNullifiers(blocks: L2Block[]): Promise { + blocks.forEach(block => { + const dataStartIndexForBlock = + block.header.state.partial.nullifierTree.nextAvailableLeafIndex - + block.body.numberOfTxsIncludingPadded * MAX_NULLIFIERS_PER_TX; + block.body.txEffects.forEach((txEffects, txIndex) => { + const dataStartIndexForTx = dataStartIndexForBlock + txIndex * MAX_NULLIFIERS_PER_TX; + txEffects.nullifiers.forEach((nullifier, nullifierIndex) => { + this.blockScopedNullifiers.set(nullifier.toString(), { + index: BigInt(dataStartIndexForTx + nullifierIndex), + blockNumber: block.number, + blockHash: block.hash().toString(), + }); + }); + }); + }); + return Promise.resolve(true); + } + + deleteNullifiers(blocks: L2Block[]): Promise { + blocks.forEach(block => { + block.body.txEffects.forEach(txEffect => { + txEffect.nullifiers.forEach(nullifier => { + this.blockScopedNullifiers.delete(nullifier.toString()); + }); + }); + }); + return Promise.resolve(true); + } + + findNullifiersIndexesWithBlock(blockNumber: number, nullifiers: Fr[]): Promise<(InBlock | undefined)[]> { + const blockScopedNullifiers = nullifiers.map(nullifier => { + const nullifierData = this.blockScopedNullifiers.get(nullifier.toString()); + if (nullifierData !== undefined && nullifierData.blockNumber <= blockNumber) { + return { + data: nullifierData.index, + l2BlockHash: nullifierData.blockHash, + l2BlockNumber: nullifierData.blockNumber, + } as InBlock; + } + return undefined; + }); + return Promise.resolve(blockScopedNullifiers); + } + getTotalL1ToL2MessageCount(): Promise { return Promise.resolve(this.l1ToL2Messages.getTotalL1ToL2MessageCount()); } @@ -365,8 +415,8 @@ export class MemoryArchiverStore implements ArchiverDataStore { * @param txHash - The txHash of the tx effect. * @returns The requested tx effect. */ - public getTxEffect(txHash: TxHash): Promise { - const txEffect = this.txEffects.find(tx => tx.txHash.equals(txHash)); + public getTxEffect(txHash: TxHash): Promise | undefined> { + const txEffect = this.txEffects.find(tx => tx.data.txHash.equals(txHash)); return Promise.resolve(txEffect); } diff --git a/yarn-project/archiver/src/test/mock_l2_block_source.ts b/yarn-project/archiver/src/test/mock_l2_block_source.ts index 86ffce4ae50..2ab843cb42a 100644 --- a/yarn-project/archiver/src/test/mock_l2_block_source.ts +++ b/yarn-project/archiver/src/test/mock_l2_block_source.ts @@ -119,8 +119,14 @@ export class MockL2BlockSource implements L2BlockSource { * @returns The requested tx effect. */ public getTxEffect(txHash: TxHash) { - const txEffect = this.l2Blocks.flatMap(b => b.body.txEffects).find(tx => tx.txHash.equals(txHash)); - return Promise.resolve(txEffect); + const match = this.l2Blocks + .flatMap(b => b.body.txEffects.map(tx => [tx, b] as const)) + .find(([tx]) => tx.txHash.equals(txHash)); + if (!match) { + return Promise.resolve(undefined); + } + const [txEffect, block] = match; + return Promise.resolve({ data: txEffect, l2BlockNumber: block.number, l2BlockHash: block.hash().toString() }); } /** diff --git a/yarn-project/aztec-node/src/aztec-node/server.test.ts b/yarn-project/aztec-node/src/aztec-node/server.test.ts index 09b1a2e081a..29bbf20e274 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.test.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.test.ts @@ -6,6 +6,7 @@ import { type L2LogsSource, MerkleTreeId, type MerkleTreeReadOperations, + type NullifierWithBlockSource, type WorldStateSynchronizer, mockTxForRollup, } from '@aztec/circuit-types'; @@ -53,6 +54,8 @@ describe('aztec node', () => { // all txs use the same allowed FPC class const contractSource = mock(); + const nullifierWithBlockSource = mock(); + const aztecNodeConfig: AztecNodeConfig = getConfigEnvVars(); node = new AztecNodeService( @@ -72,6 +75,7 @@ describe('aztec node', () => { l2LogsSource, contractSource, l1ToL2MessageSource, + nullifierWithBlockSource, worldState, undefined, 12345, diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index 058d5d6a770..13d1051a623 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -6,6 +6,7 @@ import { type EpochProofQuote, type FromLogType, type GetUnencryptedLogsResponse, + type InBlock, type L1ToL2MessageSource, type L2Block, type L2BlockL2Logs, @@ -16,6 +17,7 @@ import { LogType, MerkleTreeId, NullifierMembershipWitness, + type NullifierWithBlockSource, type ProcessedTx, type ProverConfig, PublicDataWitness, @@ -96,6 +98,7 @@ export class AztecNodeService implements AztecNode { protected readonly unencryptedLogsSource: L2LogsSource, protected readonly contractDataSource: ContractDataSource, protected readonly l1ToL2MessageSource: L1ToL2MessageSource, + protected readonly nullifierSource: NullifierWithBlockSource, protected readonly worldStateSynchronizer: WorldStateSynchronizer, protected readonly sequencer: SequencerClient | undefined, protected readonly l1ChainId: number, @@ -117,14 +120,18 @@ export class AztecNodeService implements AztecNode { this.log.info(message); } - addEpochProofQuote(quote: EpochProofQuote): Promise { + public addEpochProofQuote(quote: EpochProofQuote): Promise { return Promise.resolve(this.p2pClient.addEpochProofQuote(quote)); } - getEpochProofQuotes(epoch: bigint): Promise { + public getEpochProofQuotes(epoch: bigint): Promise { return this.p2pClient.getEpochProofQuotes(epoch); } + public getL2Tips() { + return this.blockSource.getL2Tips(); + } + /** * initializes the Aztec Node, wait for component to sync. * @param config - The configuration to be used by the aztec node. @@ -184,6 +191,7 @@ export class AztecNodeService implements AztecNode { archiver, archiver, archiver, + archiver, worldStateSynchronizer, sequencer, ethereumChain.chainInfo.id, @@ -369,7 +377,7 @@ export class AztecNodeService implements AztecNode { return txReceipt; } - public getTxEffect(txHash: TxHash): Promise { + public getTxEffect(txHash: TxHash): Promise | undefined> { return this.blockSource.getTxEffect(txHash); } @@ -423,6 +431,16 @@ export class AztecNodeService implements AztecNode { return await Promise.all(leafValues.map(leafValue => committedDb.findLeafIndex(treeId, leafValue.toBuffer()))); } + public async findNullifiersIndexesWithBlock( + blockNumber: L2BlockNumber, + nullifiers: Fr[], + ): Promise<(InBlock | undefined)[]> { + if (blockNumber === 'latest') { + blockNumber = await this.getBlockNumber(); + } + return this.nullifierSource.findNullifiersIndexesWithBlock(blockNumber, nullifiers); + } + /** * Returns a sibling path for the given index in the nullifier tree. * @param blockNumber - The block number at which to get the data. @@ -705,7 +723,7 @@ export class AztecNodeService implements AztecNode { * Returns the currently committed block header, or the initial header if no blocks have been produced. * @returns The current committed block header. */ - public async getHeader(blockNumber: L2BlockNumber = 'latest'): Promise
{ + public async getBlockHeader(blockNumber: L2BlockNumber = 'latest'): Promise
{ return ( (await this.getBlock(blockNumber === 'latest' ? -1 : blockNumber))?.header ?? this.worldStateSynchronizer.getCommitted().getInitialHeader() diff --git a/yarn-project/aztec.js/src/contract/sent_tx.ts b/yarn-project/aztec.js/src/contract/sent_tx.ts index 7b85b3fd853..d86f80528fb 100644 --- a/yarn-project/aztec.js/src/contract/sent_tx.ts +++ b/yarn-project/aztec.js/src/contract/sent_tx.ts @@ -78,7 +78,7 @@ export class SentTx { } if (opts?.debug) { const txHash = await this.getTxHash(); - const tx = (await this.pxe.getTxEffect(txHash))!; + const { data: tx } = (await this.pxe.getTxEffect(txHash))!; receipt.debugInfo = { noteHashes: tx.noteHashes, nullifiers: tx.nullifiers, diff --git a/yarn-project/aztec.js/src/wallet/base_wallet.ts b/yarn-project/aztec.js/src/wallet/base_wallet.ts index 00cc9127a38..94d7e479504 100644 --- a/yarn-project/aztec.js/src/wallet/base_wallet.ts +++ b/yarn-project/aztec.js/src/wallet/base_wallet.ts @@ -13,7 +13,6 @@ import { type SiblingPath, type SyncStatus, type Tx, - type TxEffect, type TxExecutionRequest, type TxHash, type TxProvingResult, @@ -122,7 +121,7 @@ export abstract class BaseWallet implements Wallet { sendTx(tx: Tx): Promise { return this.pxe.sendTx(tx); } - getTxEffect(txHash: TxHash): Promise { + getTxEffect(txHash: TxHash) { return this.pxe.getTxEffect(txHash); } getTxReceipt(txHash: TxHash): Promise { diff --git a/yarn-project/circuit-types/src/in_block.ts b/yarn-project/circuit-types/src/in_block.ts new file mode 100644 index 00000000000..5e5205b4237 --- /dev/null +++ b/yarn-project/circuit-types/src/in_block.ts @@ -0,0 +1,36 @@ +import { Fr } from '@aztec/circuits.js'; +import { schemas } from '@aztec/foundation/schemas'; + +import { type ZodTypeAny, z } from 'zod'; + +import { type L2Block } from './l2_block.js'; + +export type InBlock = { + l2BlockNumber: number; + l2BlockHash: string; + data: T; +}; + +export function randomInBlock(data: T): InBlock { + return { + data, + l2BlockNumber: Math.floor(Math.random() * 1000), + l2BlockHash: Fr.random().toString(), + }; +} + +export function wrapInBlock(data: T, block: L2Block): InBlock { + return { + data, + l2BlockNumber: block.number, + l2BlockHash: block.hash().toString(), + }; +} + +export function inBlockSchemaFor(schema: T) { + return z.object({ + data: schema, + l2BlockNumber: schemas.Integer, + l2BlockHash: z.string(), + }); +} diff --git a/yarn-project/circuit-types/src/index.ts b/yarn-project/circuit-types/src/index.ts index 00f71f992c1..829e43cf8e3 100644 --- a/yarn-project/circuit-types/src/index.ts +++ b/yarn-project/circuit-types/src/index.ts @@ -22,4 +22,6 @@ export * from './simulation_error.js'; export * from './tx/index.js'; export * from './tx_effect.js'; export * from './tx_execution_request.js'; +export * from './in_block.js'; +export * from './nullifier_with_block_source.js'; export * from './proving_error.js'; diff --git a/yarn-project/circuit-types/src/interfaces/archiver.test.ts b/yarn-project/circuit-types/src/interfaces/archiver.test.ts index 6003a86e427..6e9dacdd574 100644 --- a/yarn-project/circuit-types/src/interfaces/archiver.test.ts +++ b/yarn-project/circuit-types/src/interfaces/archiver.test.ts @@ -20,6 +20,7 @@ import { readFileSync } from 'fs'; import omit from 'lodash.omit'; import { resolve } from 'path'; +import { type InBlock, randomInBlock } from '../in_block.js'; import { L2Block } from '../l2_block.js'; import { type L2Tips } from '../l2_block_source.js'; import { ExtendedUnencryptedL2Log } from '../logs/extended_unencrypted_l2_log.js'; @@ -106,7 +107,7 @@ describe('ArchiverApiSchema', () => { it('getTxEffect', async () => { const result = await context.client.getTxEffect(new TxHash(Buffer.alloc(32, 1))); - expect(result).toBeInstanceOf(TxEffect); + expect(result!.data).toBeInstanceOf(TxEffect); }); it('getSettledTxReceipt', async () => { @@ -143,6 +144,18 @@ describe('ArchiverApiSchema', () => { }); }); + it('findNullifiersIndexesWithBlock', async () => { + const result = await context.client.findNullifiersIndexesWithBlock(1, [Fr.random(), Fr.random()]); + expect(result).toEqual([ + { + data: expect.any(BigInt), + l2BlockNumber: expect.any(Number), + l2BlockHash: expect.any(String), + }, + undefined, + ]); + }); + it('getLogs(Encrypted)', async () => { const result = await context.client.getLogs(1, 1, LogType.ENCRYPTED); expect(result).toEqual([expect.any(EncryptedL2BlockL2Logs)]); @@ -270,9 +283,9 @@ class MockArchiver implements ArchiverApi { getBlocks(from: number, _limit: number, _proven?: boolean | undefined): Promise { return Promise.resolve([L2Block.random(from)]); } - getTxEffect(_txHash: TxHash): Promise { + getTxEffect(_txHash: TxHash): Promise | undefined> { expect(_txHash).toBeInstanceOf(TxHash); - return Promise.resolve(TxEffect.empty()); + return Promise.resolve({ l2BlockNumber: 1, l2BlockHash: '0x12', data: TxEffect.random() }); } getSettledTxReceipt(txHash: TxHash): Promise { expect(txHash).toBeInstanceOf(TxHash); @@ -299,6 +312,13 @@ class MockArchiver implements ArchiverApi { finalized: { number: 1, hash: `0x01` }, }); } + findNullifiersIndexesWithBlock(blockNumber: number, nullifiers: Fr[]): Promise<(InBlock | undefined)[]> { + expect(blockNumber).toEqual(1); + expect(nullifiers).toHaveLength(2); + expect(nullifiers[0]).toBeInstanceOf(Fr); + expect(nullifiers[1]).toBeInstanceOf(Fr); + return Promise.resolve([randomInBlock(Fr.random().toBigInt()), undefined]); + } getLogs( _from: number, _limit: number, diff --git a/yarn-project/circuit-types/src/interfaces/archiver.ts b/yarn-project/circuit-types/src/interfaces/archiver.ts index 4e32edb2b8f..f5a212f5099 100644 --- a/yarn-project/circuit-types/src/interfaces/archiver.ts +++ b/yarn-project/circuit-types/src/interfaces/archiver.ts @@ -10,6 +10,7 @@ import { type ApiSchemaFor, optional, schemas } from '@aztec/foundation/schemas' import { z } from 'zod'; +import { inBlockSchemaFor } from '../in_block.js'; import { L2Block } from '../l2_block.js'; import { type L2BlockSource, L2TipsSchema } from '../l2_block_source.js'; import { GetUnencryptedLogsResponseSchema, TxScopedL2Log } from '../logs/get_logs_response.js'; @@ -18,12 +19,13 @@ import { type L2LogsSource } from '../logs/l2_logs_source.js'; import { LogFilterSchema } from '../logs/log_filter.js'; import { LogType } from '../logs/log_type.js'; import { type L1ToL2MessageSource } from '../messaging/l1_to_l2_message_source.js'; +import { type NullifierWithBlockSource } from '../nullifier_with_block_source.js'; import { TxHash } from '../tx/tx_hash.js'; import { TxReceipt } from '../tx/tx_receipt.js'; import { TxEffect } from '../tx_effect.js'; export type ArchiverApi = Omit< - L2BlockSource & L2LogsSource & ContractDataSource & L1ToL2MessageSource, + L2BlockSource & L2LogsSource & ContractDataSource & L1ToL2MessageSource & NullifierWithBlockSource, 'start' | 'stop' >; @@ -42,7 +44,7 @@ export const ArchiverApiSchema: ApiSchemaFor = { .function() .args(schemas.Integer, schemas.Integer, optional(z.boolean())) .returns(z.array(L2Block.schema)), - getTxEffect: z.function().args(TxHash.schema).returns(TxEffect.schema.optional()), + getTxEffect: z.function().args(TxHash.schema).returns(inBlockSchemaFor(TxEffect.schema).optional()), getSettledTxReceipt: z.function().args(TxHash.schema).returns(TxReceipt.schema.optional()), getL2SlotNumber: z.function().args().returns(schemas.BigInt), getL2EpochNumber: z.function().args().returns(schemas.BigInt), @@ -57,6 +59,10 @@ export const ArchiverApiSchema: ApiSchemaFor = { .function() .args(z.array(schemas.Fr)) .returns(z.array(z.array(TxScopedL2Log.schema))), + findNullifiersIndexesWithBlock: z + .function() + .args(z.number(), z.array(schemas.Fr)) + .returns(z.array(optional(inBlockSchemaFor(schemas.BigInt)))), getUnencryptedLogs: z.function().args(LogFilterSchema).returns(GetUnencryptedLogsResponseSchema), getContractClassLogs: z.function().args(LogFilterSchema).returns(GetUnencryptedLogsResponseSchema), getPublicFunction: z diff --git a/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts b/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts index 2e810f65b3f..7e33bc9400d 100644 --- a/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts +++ b/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts @@ -28,7 +28,9 @@ import omit from 'lodash.omit'; import times from 'lodash.times'; import { resolve } from 'path'; +import { type InBlock, randomInBlock } from '../in_block.js'; import { L2Block } from '../l2_block.js'; +import { type L2Tips } from '../l2_block_source.js'; import { ExtendedUnencryptedL2Log } from '../logs/extended_unencrypted_l2_log.js'; import { type GetUnencryptedLogsResponse, TxScopedL2Log } from '../logs/get_logs_response.js'; import { @@ -80,11 +82,28 @@ describe('AztecNodeApiSchema', () => { expect([...tested].sort()).toEqual(all.sort()); }); + it('getL2Tips', async () => { + const result = await context.client.getL2Tips(); + expect(result).toEqual({ + latest: { number: 1, hash: `0x01` }, + proven: { number: 1, hash: `0x01` }, + finalized: { number: 1, hash: `0x01` }, + }); + }); + it('findLeavesIndexes', async () => { const response = await context.client.findLeavesIndexes(1, MerkleTreeId.ARCHIVE, [Fr.random(), Fr.random()]); expect(response).toEqual([1n, undefined]); }); + it('findNullifiersIndexesWithBlock', async () => { + const response = await context.client.findNullifiersIndexesWithBlock(1, [Fr.random(), Fr.random()]); + expect(response).toEqual([ + { data: 1n, l2BlockNumber: expect.any(Number), l2BlockHash: expect.any(String) }, + undefined, + ]); + }); + it('getNullifierSiblingPath', async () => { const response = await context.client.getNullifierSiblingPath(1, 1n); expect(response).toBeInstanceOf(SiblingPath); @@ -231,7 +250,7 @@ describe('AztecNodeApiSchema', () => { it('getTxEffect', async () => { const response = await context.client.getTxEffect(TxHash.random()); - expect(response).toBeInstanceOf(TxEffect); + expect(response!.data).toBeInstanceOf(TxEffect); }); it('getPendingTxs', async () => { @@ -254,8 +273,8 @@ describe('AztecNodeApiSchema', () => { expect(response).toBeInstanceOf(Fr); }); - it('getHeader', async () => { - const response = await context.client.getHeader(); + it('getBlockHeader', async () => { + const response = await context.client.getBlockHeader(); expect(response).toBeInstanceOf(Header); }); @@ -323,6 +342,13 @@ describe('AztecNodeApiSchema', () => { class MockAztecNode implements AztecNode { constructor(private artifact: ContractArtifact) {} + getL2Tips(): Promise { + return Promise.resolve({ + latest: { number: 1, hash: `0x01` }, + proven: { number: 1, hash: `0x01` }, + finalized: { number: 1, hash: `0x01` }, + }); + } findLeavesIndexes( blockNumber: number | 'latest', treeId: MerkleTreeId, @@ -333,6 +359,15 @@ class MockAztecNode implements AztecNode { expect(leafValues[1]).toBeInstanceOf(Fr); return Promise.resolve([1n, undefined]); } + findNullifiersIndexesWithBlock( + blockNumber: number | 'latest', + nullifiers: Fr[], + ): Promise<(InBlock | undefined)[]> { + expect(nullifiers).toHaveLength(2); + expect(nullifiers[0]).toBeInstanceOf(Fr); + expect(nullifiers[1]).toBeInstanceOf(Fr); + return Promise.resolve([randomInBlock(1n), undefined]); + } getNullifierSiblingPath( blockNumber: number | 'latest', leafIndex: bigint, @@ -477,9 +512,9 @@ class MockAztecNode implements AztecNode { expect(txHash).toBeInstanceOf(TxHash); return Promise.resolve(TxReceipt.empty()); } - getTxEffect(txHash: TxHash): Promise { + getTxEffect(txHash: TxHash): Promise | undefined> { expect(txHash).toBeInstanceOf(TxHash); - return Promise.resolve(TxEffect.random()); + return Promise.resolve({ l2BlockNumber: 1, l2BlockHash: '0x12', data: TxEffect.random() }); } getPendingTxs(): Promise { return Promise.resolve([Tx.random()]); @@ -496,7 +531,7 @@ class MockAztecNode implements AztecNode { expect(slot).toBeInstanceOf(Fr); return Promise.resolve(Fr.random()); } - getHeader(_blockNumber?: number | 'latest' | undefined): Promise
{ + getBlockHeader(_blockNumber?: number | 'latest' | undefined): Promise
{ return Promise.resolve(Header.empty()); } simulatePublicCalls(tx: Tx): Promise { diff --git a/yarn-project/circuit-types/src/interfaces/aztec-node.ts b/yarn-project/circuit-types/src/interfaces/aztec-node.ts index b2a6c6a3149..deeae772391 100644 --- a/yarn-project/circuit-types/src/interfaces/aztec-node.ts +++ b/yarn-project/circuit-types/src/interfaces/aztec-node.ts @@ -21,7 +21,9 @@ import { type ApiSchemaFor, optional, schemas } from '@aztec/foundation/schemas' import { z } from 'zod'; +import { type InBlock, inBlockSchemaFor } from '../in_block.js'; import { L2Block } from '../l2_block.js'; +import { type L2BlockSource, type L2Tips, L2TipsSchema } from '../l2_block_source.js'; import { type FromLogType, type GetUnencryptedLogsResponse, @@ -48,7 +50,14 @@ import { type ProverCoordination, ProverCoordinationApiSchema } from './prover-c * The aztec node. * We will probably implement the additional interfaces by means other than Aztec Node as it's currently a privacy leak */ -export interface AztecNode extends ProverCoordination { +export interface AztecNode + extends ProverCoordination, + Pick { + /** + * Returns the tips of the L2 chain. + */ + getL2Tips(): Promise; + /** * Find the indexes of the given leaves in the given tree. * @param blockNumber - The block number at which to get the data or 'latest' for latest data @@ -62,6 +71,18 @@ export interface AztecNode extends ProverCoordination { leafValues: Fr[], ): Promise<(bigint | undefined)[]>; + /** + * Returns the indexes of the given nullifiers in the nullifier tree, + * scoped to the block they were included in. + * @param blockNumber - The block number at which to get the data. + * @param nullifiers - The nullifiers to search for. + * @returns The block scoped indexes of the given nullifiers in the nullifier tree, or undefined if not found. + */ + findNullifiersIndexesWithBlock( + blockNumber: L2BlockNumber, + nullifiers: Fr[], + ): Promise<(InBlock | undefined)[]>; + /** * Returns a sibling path for the given index in the nullifier tree. * @param blockNumber - The block number at which to get the data. @@ -298,7 +319,7 @@ export interface AztecNode extends ProverCoordination { * @param txHash - The hash of a transaction which resulted in the returned tx effect. * @returns The requested tx effect. */ - getTxEffect(txHash: TxHash): Promise; + getTxEffect(txHash: TxHash): Promise | undefined>; /** * Method to retrieve pending txs. @@ -336,7 +357,7 @@ export interface AztecNode extends ProverCoordination { * Returns the currently committed block header. * @returns The current committed block header. */ - getHeader(blockNumber?: L2BlockNumber): Promise
; + getBlockHeader(blockNumber?: L2BlockNumber): Promise
; /** * Simulates the public part of a transaction with the current state. @@ -403,11 +424,17 @@ export interface AztecNode extends ProverCoordination { export const AztecNodeApiSchema: ApiSchemaFor = { ...ProverCoordinationApiSchema, + getL2Tips: z.function().args().returns(L2TipsSchema), findLeavesIndexes: z .function() .args(L2BlockNumberSchema, z.nativeEnum(MerkleTreeId), z.array(schemas.Fr)) .returns(z.array(optional(schemas.BigInt))), + findNullifiersIndexesWithBlock: z + .function() + .args(L2BlockNumberSchema, z.array(schemas.Fr)) + .returns(z.array(optional(inBlockSchemaFor(schemas.BigInt)))), + getNullifierSiblingPath: z .function() .args(L2BlockNumberSchema, schemas.BigInt) @@ -492,7 +519,7 @@ export const AztecNodeApiSchema: ApiSchemaFor = { getTxReceipt: z.function().args(TxHash.schema).returns(TxReceipt.schema), - getTxEffect: z.function().args(TxHash.schema).returns(TxEffect.schema.optional()), + getTxEffect: z.function().args(TxHash.schema).returns(inBlockSchemaFor(TxEffect.schema).optional()), getPendingTxs: z.function().returns(z.array(Tx.schema)), @@ -502,7 +529,7 @@ export const AztecNodeApiSchema: ApiSchemaFor = { getPublicStorageAt: z.function().args(schemas.AztecAddress, schemas.Fr, L2BlockNumberSchema).returns(schemas.Fr), - getHeader: z.function().args(optional(L2BlockNumberSchema)).returns(Header.schema), + getBlockHeader: z.function().args(optional(L2BlockNumberSchema)).returns(Header.schema), simulatePublicCalls: z.function().args(Tx.schema).returns(PublicSimulationOutput.schema), diff --git a/yarn-project/circuit-types/src/interfaces/pxe.test.ts b/yarn-project/circuit-types/src/interfaces/pxe.test.ts index 8aefa059edb..e2aa6c1cca5 100644 --- a/yarn-project/circuit-types/src/interfaces/pxe.test.ts +++ b/yarn-project/circuit-types/src/interfaces/pxe.test.ts @@ -30,6 +30,7 @@ import times from 'lodash.times'; import { resolve } from 'path'; import { AuthWitness } from '../auth_witness.js'; +import { type InBlock } from '../in_block.js'; import { L2Block } from '../l2_block.js'; import { ExtendedUnencryptedL2Log, type GetUnencryptedLogsResponse, type LogFilter } from '../logs/index.js'; import { type IncomingNotesFilter } from '../notes/incoming_notes_filter.js'; @@ -178,8 +179,10 @@ describe('PXESchema', () => { }); it('getTxEffect', async () => { - const result = await context.client.getTxEffect(TxHash.random()); - expect(result).toBeInstanceOf(TxEffect); + const { l2BlockHash, l2BlockNumber, data } = (await context.client.getTxEffect(TxHash.random()))!; + expect(data).toBeInstanceOf(TxEffect); + expect(l2BlockHash).toMatch(/0x[a-fA-F0-9]{64}/); + expect(l2BlockNumber).toBe(1); }); it('getPublicStorageAt', async () => { @@ -401,9 +404,9 @@ class MockPXE implements PXE { expect(txHash).toBeInstanceOf(TxHash); return Promise.resolve(TxReceipt.empty()); } - getTxEffect(txHash: TxHash): Promise { + getTxEffect(txHash: TxHash): Promise | undefined> { expect(txHash).toBeInstanceOf(TxHash); - return Promise.resolve(TxEffect.random()); + return Promise.resolve({ data: TxEffect.random(), l2BlockHash: Fr.random().toString(), l2BlockNumber: 1 }); } getPublicStorageAt(contract: AztecAddress, slot: Fr): Promise { expect(contract).toBeInstanceOf(AztecAddress); diff --git a/yarn-project/circuit-types/src/interfaces/pxe.ts b/yarn-project/circuit-types/src/interfaces/pxe.ts index 4bfa62f74bb..a2a6f6940fe 100644 --- a/yarn-project/circuit-types/src/interfaces/pxe.ts +++ b/yarn-project/circuit-types/src/interfaces/pxe.ts @@ -27,6 +27,7 @@ import { AbiDecodedSchema, type ApiSchemaFor, type ZodFor, optional, schemas } f import { z } from 'zod'; import { AuthWitness } from '../auth_witness.js'; +import { type InBlock, inBlockSchemaFor } from '../in_block.js'; import { L2Block } from '../l2_block.js'; import { type GetUnencryptedLogsResponse, @@ -216,7 +217,7 @@ export interface PXE { * @param txHash - The hash of a transaction which resulted in the returned tx effect. * @returns The requested tx effect. */ - getTxEffect(txHash: TxHash): Promise; + getTxEffect(txHash: TxHash): Promise | undefined>; /** * Gets the storage value at the given contract storage slot. @@ -500,7 +501,7 @@ export const PXESchema: ApiSchemaFor = { getTxEffect: z .function() .args(TxHash.schema) - .returns(z.union([TxEffect.schema, z.undefined()])), + .returns(z.union([inBlockSchemaFor(TxEffect.schema), z.undefined()])), getPublicStorageAt: z.function().args(schemas.AztecAddress, schemas.Fr).returns(schemas.Fr), getIncomingNotes: z.function().args(IncomingNotesFilterSchema).returns(z.array(UniqueNote.schema)), getL1ToL2MembershipWitness: z diff --git a/yarn-project/circuit-types/src/l2_block_downloader/l2_block_stream.ts b/yarn-project/circuit-types/src/l2_block_downloader/l2_block_stream.ts index 7a1d179dd16..50608b6c0ea 100644 --- a/yarn-project/circuit-types/src/l2_block_downloader/l2_block_stream.ts +++ b/yarn-project/circuit-types/src/l2_block_downloader/l2_block_stream.ts @@ -12,14 +12,15 @@ export class L2BlockStream { private readonly log = createDebugLogger('aztec:l2_block_stream'); constructor( - private l2BlockSource: L2BlockSource, + private l2BlockSource: Pick, private localData: L2BlockStreamLocalDataProvider, private handler: L2BlockStreamEventHandler, private opts: { proven?: boolean; pollIntervalMS?: number; batchSize?: number; - }, + startingBlock?: number; + } = {}, ) { this.runningPromise = new RunningPromise(() => this.work(), this.opts.pollIntervalMS ?? 1000); } @@ -70,6 +71,11 @@ export class L2BlockStream { await this.emitEvent({ type: 'chain-pruned', blockNumber: latestBlockNumber }); } + // If we are just starting, use the starting block number from the options. + if (latestBlockNumber === 0 && this.opts.startingBlock !== undefined) { + latestBlockNumber = Math.max(this.opts.startingBlock - 1, 0); + } + // Request new blocks from the source. while (latestBlockNumber < sourceTips.latest.number) { const from = latestBlockNumber + 1; diff --git a/yarn-project/circuit-types/src/l2_block_source.ts b/yarn-project/circuit-types/src/l2_block_source.ts index 4dac6da0db9..6f749b28189 100644 --- a/yarn-project/circuit-types/src/l2_block_source.ts +++ b/yarn-project/circuit-types/src/l2_block_source.ts @@ -2,6 +2,7 @@ import { type EthAddress, type Header } from '@aztec/circuits.js'; import { z } from 'zod'; +import { type InBlock } from './in_block.js'; import { type L2Block } from './l2_block.js'; import { type TxHash } from './tx/tx_hash.js'; import { type TxReceipt } from './tx/tx_receipt.js'; @@ -69,7 +70,7 @@ export interface L2BlockSource { * @param txHash - The hash of a transaction which resulted in the returned tx effect. * @returns The requested tx effect. */ - getTxEffect(txHash: TxHash): Promise; + getTxEffect(txHash: TxHash): Promise | undefined>; /** * Gets a receipt of a settled tx. @@ -121,6 +122,7 @@ export type L2Tips = Record; /** Identifies a block by number and hash. */ export type L2BlockId = z.infer; +// TODO(palla/schemas): This package should know what is the block hash of the genesis block 0. const L2BlockIdSchema = z.union([ z.object({ number: z.literal(0), diff --git a/yarn-project/circuit-types/src/logs/get_logs_response.ts b/yarn-project/circuit-types/src/logs/get_logs_response.ts index c62b5af965c..6f1d156be0b 100644 --- a/yarn-project/circuit-types/src/logs/get_logs_response.ts +++ b/yarn-project/circuit-types/src/logs/get_logs_response.ts @@ -84,4 +84,14 @@ export class TxScopedL2Log { static random() { return new TxScopedL2Log(TxHash.random(), 1, 1, false, Fr.random().toBuffer()); } + + equals(other: TxScopedL2Log) { + return ( + this.txHash.equals(other.txHash) && + this.dataStartIndexForTx === other.dataStartIndexForTx && + this.blockNumber === other.blockNumber && + this.isFromPublic === other.isFromPublic && + this.logData.equals(other.logData) + ); + } } diff --git a/yarn-project/circuit-types/src/nullifier_with_block_source.ts b/yarn-project/circuit-types/src/nullifier_with_block_source.ts new file mode 100644 index 00000000000..d8716c6f3a4 --- /dev/null +++ b/yarn-project/circuit-types/src/nullifier_with_block_source.ts @@ -0,0 +1,7 @@ +import { type Fr } from '@aztec/circuits.js'; + +import { type InBlock } from './index.js'; + +export interface NullifierWithBlockSource { + findNullifiersIndexesWithBlock(blockNumber: number, nullifiers: Fr[]): Promise<(InBlock | undefined)[]>; +} diff --git a/yarn-project/cli/src/utils/inspect.ts b/yarn-project/cli/src/utils/inspect.ts index 445e48c3bd1..855c38b9ef5 100644 --- a/yarn-project/cli/src/utils/inspect.ts +++ b/yarn-project/cli/src/utils/inspect.ts @@ -38,22 +38,23 @@ export async function inspectTx( log: LogFn, opts: { includeBlockInfo?: boolean; artifactMap?: ArtifactMap } = {}, ) { - const [receipt, effects, notes] = await Promise.all([ + const [receipt, effectsInBlock, notes] = await Promise.all([ pxe.getTxReceipt(txHash), pxe.getTxEffect(txHash), pxe.getIncomingNotes({ txHash, status: NoteStatus.ACTIVE_OR_NULLIFIED }), ]); // Base tx data log(`Tx ${txHash.toString()}`); - log(` Status: ${receipt.status} ${effects ? `(${effects.revertCode.getDescription()})` : ''}`); + log(` Status: ${receipt.status} ${effectsInBlock ? `(${effectsInBlock.data.revertCode.getDescription()})` : ''}`); if (receipt.error) { log(` Error: ${receipt.error}`); } - if (!effects) { + if (!effectsInBlock) { return; } + const effects = effectsInBlock.data; const artifactMap = opts?.artifactMap ?? (await getKnownArtifacts(pxe)); if (opts.includeBlockInfo) { diff --git a/yarn-project/end-to-end/src/e2e_block_building.test.ts b/yarn-project/end-to-end/src/e2e_block_building.test.ts index 88bfd6a1b23..41a27b70a92 100644 --- a/yarn-project/end-to-end/src/e2e_block_building.test.ts +++ b/yarn-project/end-to-end/src/e2e_block_building.test.ts @@ -1,5 +1,4 @@ import { getSchnorrAccount } from '@aztec/accounts/schnorr'; -import { createAccount } from '@aztec/accounts/testing'; import { type AztecAddress, type AztecNode, @@ -23,7 +22,6 @@ import { poseidon2HashWithSeparator } from '@aztec/foundation/crypto'; import { StatefulTestContract, StatefulTestContractArtifact } from '@aztec/noir-contracts.js'; import { TestContract } from '@aztec/noir-contracts.js/Test'; import { TokenContract } from '@aztec/noir-contracts.js/Token'; -import { createPXEService, getPXEServiceConfig } from '@aztec/pxe'; import 'jest-extended'; @@ -439,49 +437,63 @@ describe('e2e_block_building', () => { await cheatCodes.rollup.advanceToNextEpoch(); await cheatCodes.rollup.markAsProven(); - // Send a tx to the contract that updates the public data tree, this should take the first slot + // Send a tx to the contract that creates a note. This tx will be reorgd but re-included, + // since it is being built against a proven block number. logger.info('Sending initial tx'); - const tx1 = await contract.methods.increment_public_value(ownerAddress, 20).send().wait(); + const tx1 = await contract.methods.create_note(ownerAddress, ownerAddress, 20).send().wait(); expect(tx1.blockNumber).toEqual(initialBlockNumber + 1); - expect(await contract.methods.get_public_value(ownerAddress).simulate()).toEqual(20n); + expect(await contract.methods.summed_values(ownerAddress).simulate()).toEqual(21n); + + // And send a second one, which won't be re-included. + logger.info('Sending second tx'); + const tx2 = await contract.methods.create_note(ownerAddress, ownerAddress, 30).send().wait(); + expect(tx2.blockNumber).toEqual(initialBlockNumber + 2); + expect(await contract.methods.summed_values(ownerAddress).simulate()).toEqual(51n); // Now move to a new epoch and past the proof claim window to cause a reorg logger.info('Advancing past the proof claim window'); await cheatCodes.rollup.advanceToNextEpoch(); await cheatCodes.rollup.advanceSlots(aztecEpochProofClaimWindowInL2Slots + 1); // off-by-one? - // Wait a bit before spawning a new pxe - await sleep(2000); + // Wait until the sequencer kicks out tx1 + logger.info(`Waiting for node to prune tx1`); + await retryUntil( + async () => (await aztecNode.getTxReceipt(tx1.txHash)).status === TxStatus.PENDING, + 'wait for pruning', + 15, + 1, + ); - // tx1 is valid because it was build against a proven block number - // the sequencer will bring it back on chain + // And wait until it is brought back tx1 + logger.info(`Waiting for node to re-include tx1`); await retryUntil( async () => (await aztecNode.getTxReceipt(tx1.txHash)).status === TxStatus.SUCCESS, 'wait for re-inclusion', - 60, + 15, 1, ); + // Tx1 should have been mined in a block with the same number but different hash now const newTx1Receipt = await aztecNode.getTxReceipt(tx1.txHash); expect(newTx1Receipt.blockNumber).toEqual(tx1.blockNumber); expect(newTx1Receipt.blockHash).not.toEqual(tx1.blockHash); - // Send another tx which should be mined a block that is built on the reorg'd chain - // We need to send it from a new pxe since pxe doesn't detect reorgs (yet) - logger.info(`Creating new PXE service`); - const pxeServiceConfig = { ...getPXEServiceConfig() }; - const newPxe = await createPXEService(aztecNode, pxeServiceConfig); - const newWallet = await createAccount(newPxe); + // PXE should have cleared out the 30-note from tx2, but reapplied the 20-note from tx1 + expect(await contract.methods.summed_values(ownerAddress).simulate()).toEqual(21n); - // TODO: Contract.at should automatically register the instance in the pxe - logger.info(`Registering contract at ${contract.address} in new pxe`); - await newPxe.registerContract({ instance: contract.instance, artifact: StatefulTestContractArtifact }); - const contractFromNewPxe = await StatefulTestContract.at(contract.address, newWallet); + // PXE should be synced to the block number on the new chain + await retryUntil( + async () => (await pxe.getSyncStatus()).blocks === newTx1Receipt.blockNumber, + 'wait for pxe block header sync', + 15, + 1, + ); + // And we should be able to send a new tx on the new chain logger.info('Sending new tx on reorgd chain'); - const tx2 = await contractFromNewPxe.methods.increment_public_value(ownerAddress, 10).send().wait(); - expect(await contractFromNewPxe.methods.get_public_value(ownerAddress).simulate()).toEqual(30n); - expect(tx2.blockNumber).toEqual(initialBlockNumber + 3); + const tx3 = await contract.methods.create_note(ownerAddress, ownerAddress, 10).send().wait(); + expect(await contract.methods.summed_values(ownerAddress).simulate()).toEqual(31n); + expect(tx3.blockNumber).toBeGreaterThanOrEqual(newTx1Receipt.blockNumber! + 1); }); }); }); diff --git a/yarn-project/end-to-end/src/e2e_event_logs.test.ts b/yarn-project/end-to-end/src/e2e_event_logs.test.ts index ab8d4f88785..162d53d6eb0 100644 --- a/yarn-project/end-to-end/src/e2e_event_logs.test.ts +++ b/yarn-project/end-to-end/src/e2e_event_logs.test.ts @@ -48,7 +48,7 @@ describe('Logs', () => { const txEffect = await node.getTxEffect(tx.txHash); - const encryptedLogs = txEffect!.encryptedLogs.unrollLogs(); + const encryptedLogs = txEffect!.data.encryptedLogs.unrollLogs(); expect(encryptedLogs.length).toBe(3); const decryptedEvent0 = L1EventPayload.decryptAsIncoming(encryptedLogs[0], wallets[0].getEncryptionSecret())!; diff --git a/yarn-project/kv-store/package.json b/yarn-project/kv-store/package.json index 870afec3c63..bacc49e1a38 100644 --- a/yarn-project/kv-store/package.json +++ b/yarn-project/kv-store/package.json @@ -6,6 +6,7 @@ ".": "./dest/interfaces/index.js", "./lmdb": "./dest/lmdb/index.js", "./utils": "./dest/utils.js", + "./stores": "./dest/stores/index.js", "./config": "./dest/config.js" }, "scripts": { @@ -56,11 +57,13 @@ ] }, "dependencies": { + "@aztec/circuit-types": "workspace:^", "@aztec/ethereum": "workspace:^", "@aztec/foundation": "workspace:^", "lmdb": "^3.0.6" }, "devDependencies": { + "@aztec/circuits.js": "workspace:^", "@jest/globals": "^29.5.0", "@types/jest": "^29.5.0", "@types/node": "^18.7.23", diff --git a/yarn-project/kv-store/src/stores/index.ts b/yarn-project/kv-store/src/stores/index.ts new file mode 100644 index 00000000000..c279b4ba628 --- /dev/null +++ b/yarn-project/kv-store/src/stores/index.ts @@ -0,0 +1 @@ +export * from './l2_tips_store.js'; diff --git a/yarn-project/kv-store/src/stores/l2_tips_store.test.ts b/yarn-project/kv-store/src/stores/l2_tips_store.test.ts new file mode 100644 index 00000000000..2b820aaf432 --- /dev/null +++ b/yarn-project/kv-store/src/stores/l2_tips_store.test.ts @@ -0,0 +1,71 @@ +import { type L2Block } from '@aztec/circuit-types'; +import { Fr, type Header } from '@aztec/circuits.js'; +import { times } from '@aztec/foundation/collection'; +import { type AztecKVStore } from '@aztec/kv-store'; +import { openTmpStore } from '@aztec/kv-store/utils'; + +import { L2TipsStore } from './l2_tips_store.js'; + +describe('L2TipsStore', () => { + let kvStore: AztecKVStore; + let tipsStore: L2TipsStore; + + beforeEach(() => { + kvStore = openTmpStore(true); + tipsStore = new L2TipsStore(kvStore, 'test'); + }); + + const makeBlock = (number: number): L2Block => + ({ number, header: { hash: () => new Fr(number) } as Header } as L2Block); + + const makeTip = (number: number) => ({ number, hash: number === 0 ? undefined : new Fr(number).toString() }); + + const makeTips = (latest: number, proven: number, finalized: number) => ({ + latest: makeTip(latest), + proven: makeTip(proven), + finalized: makeTip(finalized), + }); + + it('returns zero if no tips are stored', async () => { + const tips = await tipsStore.getL2Tips(); + expect(tips).toEqual(makeTips(0, 0, 0)); + }); + + it('stores chain tips', async () => { + await tipsStore.handleBlockStreamEvent({ type: 'blocks-added', blocks: times(20, i => makeBlock(i + 1)) }); + + await tipsStore.handleBlockStreamEvent({ type: 'chain-finalized', blockNumber: 5 }); + await tipsStore.handleBlockStreamEvent({ type: 'chain-proven', blockNumber: 8 }); + await tipsStore.handleBlockStreamEvent({ type: 'chain-pruned', blockNumber: 10 }); + + const tips = await tipsStore.getL2Tips(); + expect(tips).toEqual(makeTips(10, 8, 5)); + }); + + it('sets latest tip from blocks added', async () => { + await tipsStore.handleBlockStreamEvent({ type: 'blocks-added', blocks: times(3, i => makeBlock(i + 1)) }); + + const tips = await tipsStore.getL2Tips(); + expect(tips).toEqual(makeTips(3, 0, 0)); + + expect(await tipsStore.getL2BlockHash(1)).toEqual(new Fr(1).toString()); + expect(await tipsStore.getL2BlockHash(2)).toEqual(new Fr(2).toString()); + expect(await tipsStore.getL2BlockHash(3)).toEqual(new Fr(3).toString()); + }); + + it('clears block hashes when setting finalized chain', async () => { + await tipsStore.handleBlockStreamEvent({ type: 'blocks-added', blocks: times(5, i => makeBlock(i + 1)) }); + await tipsStore.handleBlockStreamEvent({ type: 'chain-proven', blockNumber: 3 }); + await tipsStore.handleBlockStreamEvent({ type: 'chain-finalized', blockNumber: 3 }); + + const tips = await tipsStore.getL2Tips(); + expect(tips).toEqual(makeTips(5, 3, 3)); + + expect(await tipsStore.getL2BlockHash(1)).toBeUndefined(); + expect(await tipsStore.getL2BlockHash(2)).toBeUndefined(); + + expect(await tipsStore.getL2BlockHash(3)).toEqual(new Fr(3).toString()); + expect(await tipsStore.getL2BlockHash(4)).toEqual(new Fr(4).toString()); + expect(await tipsStore.getL2BlockHash(5)).toEqual(new Fr(5).toString()); + }); +}); diff --git a/yarn-project/kv-store/src/stores/l2_tips_store.ts b/yarn-project/kv-store/src/stores/l2_tips_store.ts new file mode 100644 index 00000000000..36180272967 --- /dev/null +++ b/yarn-project/kv-store/src/stores/l2_tips_store.ts @@ -0,0 +1,69 @@ +import { + type L2BlockId, + type L2BlockStreamEvent, + type L2BlockStreamEventHandler, + type L2BlockStreamLocalDataProvider, + type L2BlockTag, + type L2Tips, +} from '@aztec/circuit-types'; + +import { type AztecMap } from '../interfaces/map.js'; +import { type AztecKVStore } from '../interfaces/store.js'; + +/** Stores currently synced L2 tips and unfinalized block hashes. */ +export class L2TipsStore implements L2BlockStreamEventHandler, L2BlockStreamLocalDataProvider { + private readonly l2TipsStore: AztecMap; + private readonly l2BlockHashesStore: AztecMap; + + constructor(store: AztecKVStore, namespace: string) { + this.l2TipsStore = store.openMap([namespace, 'l2_tips'].join('_')); + this.l2BlockHashesStore = store.openMap([namespace, 'l2_block_hashes'].join('_')); + } + + public getL2BlockHash(number: number): Promise { + return Promise.resolve(this.l2BlockHashesStore.get(number)); + } + + public getL2Tips(): Promise { + return Promise.resolve({ + latest: this.getL2Tip('latest'), + finalized: this.getL2Tip('finalized'), + proven: this.getL2Tip('proven'), + }); + } + + private getL2Tip(tag: L2BlockTag): L2BlockId { + const blockNumber = this.l2TipsStore.get(tag); + if (blockNumber === undefined) { + return { number: 0, hash: undefined }; + } + const blockHash = this.l2BlockHashesStore.get(blockNumber); + if (!blockHash) { + throw new Error(`Block hash not found for block number ${blockNumber}`); + } + return { number: blockNumber, hash: blockHash }; + } + + public async handleBlockStreamEvent(event: L2BlockStreamEvent): Promise { + switch (event.type) { + case 'blocks-added': + await this.l2TipsStore.set('latest', event.blocks.at(-1)!.number); + for (const block of event.blocks) { + await this.l2BlockHashesStore.set(block.number, block.header.hash().toString()); + } + break; + case 'chain-pruned': + await this.l2TipsStore.set('latest', event.blockNumber); + break; + case 'chain-proven': + await this.l2TipsStore.set('proven', event.blockNumber); + break; + case 'chain-finalized': + await this.l2TipsStore.set('finalized', event.blockNumber); + for (const key of this.l2BlockHashesStore.keys({ end: event.blockNumber })) { + await this.l2BlockHashesStore.delete(key); + } + break; + } + } +} diff --git a/yarn-project/kv-store/tsconfig.json b/yarn-project/kv-store/tsconfig.json index 18fc3bcf3f2..bd860591ecb 100644 --- a/yarn-project/kv-store/tsconfig.json +++ b/yarn-project/kv-store/tsconfig.json @@ -7,10 +7,16 @@ }, "references": [ { - "path": "../foundation" + "path": "../circuit-types" }, { "path": "../ethereum" + }, + { + "path": "../foundation" + }, + { + "path": "../circuits.js" } ], "include": ["src"] diff --git a/yarn-project/pxe/src/database/incoming_note_dao.test.ts b/yarn-project/pxe/src/database/incoming_note_dao.test.ts index a3cb0d5345c..1df9103e08f 100644 --- a/yarn-project/pxe/src/database/incoming_note_dao.test.ts +++ b/yarn-project/pxe/src/database/incoming_note_dao.test.ts @@ -1,38 +1,8 @@ -import { Note, randomTxHash } from '@aztec/circuit-types'; -import { AztecAddress, Fr, Point } from '@aztec/circuits.js'; -import { NoteSelector } from '@aztec/foundation/abi'; - import { IncomingNoteDao } from './incoming_note_dao.js'; -export const randomIncomingNoteDao = ({ - note = Note.random(), - contractAddress = AztecAddress.random(), - txHash = randomTxHash(), - storageSlot = Fr.random(), - noteTypeId = NoteSelector.random(), - nonce = Fr.random(), - noteHash = Fr.random(), - siloedNullifier = Fr.random(), - index = Fr.random().toBigInt(), - addressPoint = Point.random(), -}: Partial = {}) => { - return new IncomingNoteDao( - note, - contractAddress, - storageSlot, - noteTypeId, - txHash, - nonce, - noteHash, - siloedNullifier, - index, - addressPoint, - ); -}; - describe('Incoming Note DAO', () => { it('convert to and from buffer', () => { - const note = randomIncomingNoteDao(); + const note = IncomingNoteDao.random(); const buf = note.toBuffer(); expect(IncomingNoteDao.fromBuffer(buf)).toEqual(note); }); diff --git a/yarn-project/pxe/src/database/incoming_note_dao.ts b/yarn-project/pxe/src/database/incoming_note_dao.ts index cbd344b135c..d2dc2d38815 100644 --- a/yarn-project/pxe/src/database/incoming_note_dao.ts +++ b/yarn-project/pxe/src/database/incoming_note_dao.ts @@ -1,4 +1,4 @@ -import { type L1NotePayload, Note, TxHash } from '@aztec/circuit-types'; +import { type L1NotePayload, Note, TxHash, randomTxHash } from '@aztec/circuit-types'; import { AztecAddress, Fr, Point, type PublicKey } from '@aztec/circuits.js'; import { NoteSelector } from '@aztec/foundation/abi'; import { toBigIntBE } from '@aztec/foundation/bigint-buffer'; @@ -22,6 +22,10 @@ export class IncomingNoteDao implements NoteData { public noteTypeId: NoteSelector, /** The hash of the tx the note was created in. */ public txHash: TxHash, + /** The L2 block number in which the tx with this note was included. */ + public l2BlockNumber: number, + /** The L2 block hash in which the tx with this note was included. */ + public l2BlockHash: string, /** The nonce of the note. */ public nonce: Fr, /** @@ -44,6 +48,8 @@ export class IncomingNoteDao implements NoteData { note: Note, payload: L1NotePayload, noteInfo: NoteInfo, + l2BlockNumber: number, + l2BlockHash: string, dataStartIndexForTx: number, addressPoint: PublicKey, ) { @@ -54,6 +60,8 @@ export class IncomingNoteDao implements NoteData { payload.storageSlot, payload.noteTypeId, noteInfo.txHash, + l2BlockNumber, + l2BlockHash, noteInfo.nonce, noteInfo.noteHash, noteInfo.siloedNullifier, @@ -69,6 +77,8 @@ export class IncomingNoteDao implements NoteData { this.storageSlot, this.noteTypeId, this.txHash.buffer, + this.l2BlockNumber, + Fr.fromString(this.l2BlockHash), this.nonce, this.noteHash, this.siloedNullifier, @@ -76,6 +86,7 @@ export class IncomingNoteDao implements NoteData { this.addressPoint, ]); } + static fromBuffer(buffer: Buffer | BufferReader) { const reader = BufferReader.asReader(buffer); @@ -84,6 +95,8 @@ export class IncomingNoteDao implements NoteData { const storageSlot = Fr.fromBuffer(reader); const noteTypeId = reader.readObject(NoteSelector); const txHash = reader.readObject(TxHash); + const l2BlockNumber = reader.readNumber(); + const l2BlockHash = Fr.fromBuffer(reader).toString(); const nonce = Fr.fromBuffer(reader); const noteHash = Fr.fromBuffer(reader); const siloedNullifier = Fr.fromBuffer(reader); @@ -96,6 +109,8 @@ export class IncomingNoteDao implements NoteData { storageSlot, noteTypeId, txHash, + l2BlockNumber, + l2BlockHash, nonce, noteHash, siloedNullifier, @@ -122,4 +137,34 @@ export class IncomingNoteDao implements NoteData { const noteSize = 4 + this.note.items.length * Fr.SIZE_IN_BYTES; return noteSize + AztecAddress.SIZE_IN_BYTES + Fr.SIZE_IN_BYTES * 4 + TxHash.SIZE + Point.SIZE_IN_BYTES + indexSize; } + + static random({ + note = Note.random(), + contractAddress = AztecAddress.random(), + txHash = randomTxHash(), + storageSlot = Fr.random(), + noteTypeId = NoteSelector.random(), + nonce = Fr.random(), + l2BlockNumber = Math.floor(Math.random() * 1000), + l2BlockHash = Fr.random().toString(), + noteHash = Fr.random(), + siloedNullifier = Fr.random(), + index = Fr.random().toBigInt(), + addressPoint = Point.random(), + }: Partial = {}) { + return new IncomingNoteDao( + note, + contractAddress, + storageSlot, + noteTypeId, + txHash, + l2BlockNumber, + l2BlockHash, + nonce, + noteHash, + siloedNullifier, + index, + addressPoint, + ); + } } diff --git a/yarn-project/pxe/src/database/kv_pxe_database.ts b/yarn-project/pxe/src/database/kv_pxe_database.ts index 2fea8a0452b..9421b66231d 100644 --- a/yarn-project/pxe/src/database/kv_pxe_database.ts +++ b/yarn-project/pxe/src/database/kv_pxe_database.ts @@ -1,4 +1,10 @@ -import { type IncomingNotesFilter, MerkleTreeId, NoteStatus, type OutgoingNotesFilter } from '@aztec/circuit-types'; +import { + type InBlock, + type IncomingNotesFilter, + MerkleTreeId, + NoteStatus, + type OutgoingNotesFilter, +} from '@aztec/circuit-types'; import { AztecAddress, CompleteAddress, @@ -39,11 +45,14 @@ export class KVPxeDatabase implements PxeDatabase { #notes: AztecMap; #nullifiedNotes: AztecMap; #nullifierToNoteId: AztecMap; + #nullifiersByBlockNumber: AztecMultiMap; + #nullifiedNotesToScope: AztecMultiMap; #nullifiedNotesByContract: AztecMultiMap; #nullifiedNotesByStorageSlot: AztecMultiMap; #nullifiedNotesByTxHash: AztecMultiMap; #nullifiedNotesByAddressPoint: AztecMultiMap; + #nullifiedNotesByNullifier: AztecMap; #syncedBlockPerPublicKey: AztecMap; #contractArtifacts: AztecMap; #contractInstances: AztecMap; @@ -56,6 +65,7 @@ export class KVPxeDatabase implements PxeDatabase { #outgoingNotesByOvpkM: AztecMultiMap; #scopes: AztecSet; + #notesToScope: AztecMultiMap; #notesByContractAndScope: Map>; #notesByStorageSlotAndScope: Map>; #notesByTxHashAndScope: Map>; @@ -87,11 +97,14 @@ export class KVPxeDatabase implements PxeDatabase { this.#notes = db.openMap('notes'); this.#nullifiedNotes = db.openMap('nullified_notes'); this.#nullifierToNoteId = db.openMap('nullifier_to_note'); + this.#nullifiersByBlockNumber = db.openMultiMap('nullifier_to_block_number'); + this.#nullifiedNotesToScope = db.openMultiMap('nullified_notes_to_scope'); this.#nullifiedNotesByContract = db.openMultiMap('nullified_notes_by_contract'); this.#nullifiedNotesByStorageSlot = db.openMultiMap('nullified_notes_by_storage_slot'); this.#nullifiedNotesByTxHash = db.openMultiMap('nullified_notes_by_tx_hash'); this.#nullifiedNotesByAddressPoint = db.openMultiMap('nullified_notes_by_address_point'); + this.#nullifiedNotesByNullifier = db.openMap('nullified_notes_by_nullifier'); this.#outgoingNotes = db.openMap('outgoing_notes'); this.#outgoingNotesByContract = db.openMultiMap('outgoing_notes_by_contract'); @@ -100,6 +113,7 @@ export class KVPxeDatabase implements PxeDatabase { this.#outgoingNotesByOvpkM = db.openMultiMap('outgoing_notes_by_ovpk_m'); this.#scopes = db.openSet('scopes'); + this.#notesToScope = db.openMultiMap('notes_to_scope'); this.#notesByContractAndScope = new Map>(); this.#notesByStorageSlotAndScope = new Map>(); this.#notesByTxHashAndScope = new Map>(); @@ -195,6 +209,7 @@ export class KVPxeDatabase implements PxeDatabase { // Had we stored them by their nullifier, they would be returned in random order const noteIndex = toBufferBE(dao.index, 32).toString('hex'); void this.#notes.set(noteIndex, dao.toBuffer()); + void this.#notesToScope.set(noteIndex, scope.toString()); void this.#nullifierToNoteId.set(dao.siloedNullifier.toString(), noteIndex); void this.#notesByContractAndScope.get(scope.toString())!.set(dao.contractAddress.toString(), noteIndex); @@ -214,6 +229,90 @@ export class KVPxeDatabase implements PxeDatabase { }); } + public removeNotesAfter(blockNumber: number): Promise { + return this.db.transaction(() => { + for (const note of this.#notes.values()) { + const noteDao = IncomingNoteDao.fromBuffer(note); + if (noteDao.l2BlockNumber > blockNumber) { + const noteIndex = toBufferBE(noteDao.index, 32).toString('hex'); + void this.#notes.delete(noteIndex); + void this.#notesToScope.delete(noteIndex); + void this.#nullifierToNoteId.delete(noteDao.siloedNullifier.toString()); + for (const scope of this.#scopes.entries()) { + void this.#notesByAddressPointAndScope.get(scope)!.deleteValue(noteDao.addressPoint.toString(), noteIndex); + void this.#notesByTxHashAndScope.get(scope)!.deleteValue(noteDao.txHash.toString(), noteIndex); + void this.#notesByContractAndScope.get(scope)!.deleteValue(noteDao.contractAddress.toString(), noteIndex); + void this.#notesByStorageSlotAndScope.get(scope)!.deleteValue(noteDao.storageSlot.toString(), noteIndex); + } + } + } + + for (const note of this.#outgoingNotes.values()) { + const noteDao = OutgoingNoteDao.fromBuffer(note); + if (noteDao.l2BlockNumber > blockNumber) { + const noteIndex = toBufferBE(noteDao.index, 32).toString('hex'); + void this.#outgoingNotes.delete(noteIndex); + void this.#outgoingNotesByContract.deleteValue(noteDao.contractAddress.toString(), noteIndex); + void this.#outgoingNotesByStorageSlot.deleteValue(noteDao.storageSlot.toString(), noteIndex); + void this.#outgoingNotesByTxHash.deleteValue(noteDao.txHash.toString(), noteIndex); + void this.#outgoingNotesByOvpkM.deleteValue(noteDao.ovpkM.toString(), noteIndex); + } + } + }); + } + + public async unnullifyNotesAfter(blockNumber: number): Promise { + const nullifiersToUndo: string[] = []; + const currentBlockNumber = blockNumber + 1; + const maxBlockNumber = this.getBlockNumber() ?? currentBlockNumber; + for (let i = currentBlockNumber; i <= maxBlockNumber; i++) { + nullifiersToUndo.push(...this.#nullifiersByBlockNumber.getValues(i)); + } + + const notesIndexesToReinsert = await this.db.transaction(() => + nullifiersToUndo.map(nullifier => this.#nullifiedNotesByNullifier.get(nullifier)), + ); + const nullifiedNoteBuffers = await this.db.transaction(() => { + return notesIndexesToReinsert + .filter(noteIndex => noteIndex != undefined) + .map(noteIndex => this.#nullifiedNotes.get(noteIndex!)); + }); + const noteDaos = nullifiedNoteBuffers + .filter(buffer => buffer != undefined) + .map(buffer => IncomingNoteDao.fromBuffer(buffer!)); + + await this.db.transaction(() => { + for (const dao of noteDaos) { + const noteIndex = toBufferBE(dao.index, 32).toString('hex'); + void this.#notes.set(noteIndex, dao.toBuffer()); + void this.#nullifierToNoteId.set(dao.siloedNullifier.toString(), noteIndex); + + let scopes = Array.from(this.#nullifiedNotesToScope.getValues(noteIndex) ?? []); + + if (scopes.length === 0) { + scopes = [new AztecAddress(dao.addressPoint.x).toString()]; + } + + for (const scope of scopes) { + void this.#notesByContractAndScope.get(scope)!.set(dao.contractAddress.toString(), noteIndex); + void this.#notesByStorageSlotAndScope.get(scope)!.set(dao.storageSlot.toString(), noteIndex); + void this.#notesByTxHashAndScope.get(scope)!.set(dao.txHash.toString(), noteIndex); + void this.#notesByAddressPointAndScope.get(scope)!.set(dao.addressPoint.toString(), noteIndex); + void this.#notesToScope.set(noteIndex, scope); + } + + void this.#nullifiedNotes.delete(noteIndex); + void this.#nullifiedNotesToScope.delete(noteIndex); + void this.#nullifiersByBlockNumber.deleteValue(dao.l2BlockNumber, dao.siloedNullifier.toString()); + void this.#nullifiedNotesByContract.deleteValue(dao.contractAddress.toString(), noteIndex); + void this.#nullifiedNotesByStorageSlot.deleteValue(dao.storageSlot.toString(), noteIndex); + void this.#nullifiedNotesByTxHash.deleteValue(dao.txHash.toString(), noteIndex); + void this.#nullifiedNotesByAddressPoint.deleteValue(dao.addressPoint.toString(), noteIndex); + void this.#nullifiedNotesByNullifier.delete(dao.siloedNullifier.toString()); + } + }); + } + getIncomingNotes(filter: IncomingNotesFilter): Promise { const publicKey: PublicKey | undefined = filter.owner ? computePoint(filter.owner) : undefined; @@ -350,7 +449,7 @@ export class KVPxeDatabase implements PxeDatabase { return Promise.resolve(notes); } - removeNullifiedNotes(nullifiers: Fr[], accountAddressPoint: PublicKey): Promise { + removeNullifiedNotes(nullifiers: InBlock[], accountAddressPoint: PublicKey): Promise { if (nullifiers.length === 0) { return Promise.resolve([]); } @@ -358,7 +457,8 @@ export class KVPxeDatabase implements PxeDatabase { return this.#db.transaction(() => { const nullifiedNotes: IncomingNoteDao[] = []; - for (const nullifier of nullifiers) { + for (const blockScopedNullifier of nullifiers) { + const { data: nullifier, l2BlockNumber: blockNumber } = blockScopedNullifier; const noteIndex = this.#nullifierToNoteId.get(nullifier.toString()); if (!noteIndex) { continue; @@ -370,7 +470,7 @@ export class KVPxeDatabase implements PxeDatabase { // note doesn't exist. Maybe it got nullified already continue; } - + const noteScopes = this.#notesToScope.getValues(noteIndex) ?? []; const note = IncomingNoteDao.fromBuffer(noteBuffer); if (!note.addressPoint.equals(accountAddressPoint)) { // tried to nullify someone else's note @@ -380,19 +480,27 @@ export class KVPxeDatabase implements PxeDatabase { nullifiedNotes.push(note); void this.#notes.delete(noteIndex); + void this.#notesToScope.delete(noteIndex); - for (const scope in this.#scopes.entries()) { + for (const scope of this.#scopes.entries()) { void this.#notesByAddressPointAndScope.get(scope)!.deleteValue(accountAddressPoint.toString(), noteIndex); void this.#notesByTxHashAndScope.get(scope)!.deleteValue(note.txHash.toString(), noteIndex); void this.#notesByContractAndScope.get(scope)!.deleteValue(note.contractAddress.toString(), noteIndex); void this.#notesByStorageSlotAndScope.get(scope)!.deleteValue(note.storageSlot.toString(), noteIndex); } + if (noteScopes !== undefined) { + for (const scope of noteScopes) { + void this.#nullifiedNotesToScope.set(noteIndex, scope); + } + } void this.#nullifiedNotes.set(noteIndex, note.toBuffer()); + void this.#nullifiersByBlockNumber.set(blockNumber, nullifier.toString()); void this.#nullifiedNotesByContract.set(note.contractAddress.toString(), noteIndex); void this.#nullifiedNotesByStorageSlot.set(note.storageSlot.toString(), noteIndex); void this.#nullifiedNotesByTxHash.set(note.txHash.toString(), noteIndex); void this.#nullifiedNotesByAddressPoint.set(note.addressPoint.toString(), noteIndex); + void this.#nullifiedNotesByNullifier.set(nullifier.toString(), noteIndex); void this.#nullifierToNoteId.delete(nullifier.toString()); } diff --git a/yarn-project/pxe/src/database/outgoing_note_dao.test.ts b/yarn-project/pxe/src/database/outgoing_note_dao.test.ts index 04b2b2201f7..0c293ba13eb 100644 --- a/yarn-project/pxe/src/database/outgoing_note_dao.test.ts +++ b/yarn-project/pxe/src/database/outgoing_note_dao.test.ts @@ -1,26 +1,8 @@ -import { Note, randomTxHash } from '@aztec/circuit-types'; -import { AztecAddress, Fr, Point } from '@aztec/circuits.js'; -import { NoteSelector } from '@aztec/foundation/abi'; - import { OutgoingNoteDao } from './outgoing_note_dao.js'; -export const randomOutgoingNoteDao = ({ - note = Note.random(), - contractAddress = AztecAddress.random(), - txHash = randomTxHash(), - storageSlot = Fr.random(), - noteTypeId = NoteSelector.random(), - nonce = Fr.random(), - noteHash = Fr.random(), - index = Fr.random().toBigInt(), - ovpkM = Point.random(), -}: Partial = {}) => { - return new OutgoingNoteDao(note, contractAddress, storageSlot, noteTypeId, txHash, nonce, noteHash, index, ovpkM); -}; - describe('Outgoing Note DAO', () => { it('convert to and from buffer', () => { - const note = randomOutgoingNoteDao(); + const note = OutgoingNoteDao.random(); const buf = note.toBuffer(); expect(OutgoingNoteDao.fromBuffer(buf)).toEqual(note); }); diff --git a/yarn-project/pxe/src/database/outgoing_note_dao.ts b/yarn-project/pxe/src/database/outgoing_note_dao.ts index 04bb7d4835c..386b23ecd57 100644 --- a/yarn-project/pxe/src/database/outgoing_note_dao.ts +++ b/yarn-project/pxe/src/database/outgoing_note_dao.ts @@ -1,4 +1,4 @@ -import { type L1NotePayload, Note, TxHash } from '@aztec/circuit-types'; +import { type L1NotePayload, Note, TxHash, randomTxHash } from '@aztec/circuit-types'; import { AztecAddress, Fr, Point, type PublicKey } from '@aztec/circuits.js'; import { NoteSelector } from '@aztec/foundation/abi'; import { toBigIntBE } from '@aztec/foundation/bigint-buffer'; @@ -21,6 +21,10 @@ export class OutgoingNoteDao { public noteTypeId: NoteSelector, /** The hash of the tx the note was created in. */ public txHash: TxHash, + /** The L2 block number in which the tx with this note was included. */ + public l2BlockNumber: number, + /** The L2 block hash in which the tx with this note was included. */ + public l2BlockHash: string, /** The nonce of the note. */ public nonce: Fr, /** @@ -38,6 +42,8 @@ export class OutgoingNoteDao { note: Note, payload: L1NotePayload, noteInfo: NoteInfo, + l2BlockNumber: number, + l2BlockHash: string, dataStartIndexForTx: number, ovpkM: PublicKey, ) { @@ -48,6 +54,8 @@ export class OutgoingNoteDao { payload.storageSlot, payload.noteTypeId, noteInfo.txHash, + l2BlockNumber, + l2BlockHash, noteInfo.nonce, noteInfo.noteHash, noteHashIndexInTheWholeTree, @@ -62,6 +70,8 @@ export class OutgoingNoteDao { this.storageSlot, this.noteTypeId, this.txHash.buffer, + this.l2BlockNumber, + Fr.fromString(this.l2BlockHash), this.nonce, this.noteHash, this.index, @@ -76,6 +86,8 @@ export class OutgoingNoteDao { const storageSlot = Fr.fromBuffer(reader); const noteTypeId = reader.readObject(NoteSelector); const txHash = new TxHash(reader.readBytes(TxHash.SIZE)); + const l2BlockNumber = reader.readNumber(); + const l2BlockHash = Fr.fromBuffer(reader).toString(); const nonce = Fr.fromBuffer(reader); const noteHash = Fr.fromBuffer(reader); const index = toBigIntBE(reader.readBytes(32)); @@ -87,6 +99,8 @@ export class OutgoingNoteDao { storageSlot, noteTypeId, txHash, + l2BlockNumber, + l2BlockHash, nonce, noteHash, index, @@ -111,4 +125,32 @@ export class OutgoingNoteDao { const noteSize = 4 + this.note.items.length * Fr.SIZE_IN_BYTES; return noteSize + AztecAddress.SIZE_IN_BYTES + Fr.SIZE_IN_BYTES * 2 + TxHash.SIZE + Point.SIZE_IN_BYTES; } + + static random({ + note = Note.random(), + contractAddress = AztecAddress.random(), + txHash = randomTxHash(), + storageSlot = Fr.random(), + noteTypeId = NoteSelector.random(), + nonce = Fr.random(), + l2BlockNumber = Math.floor(Math.random() * 1000), + l2BlockHash = Fr.random().toString(), + noteHash = Fr.random(), + index = Fr.random().toBigInt(), + ovpkM = Point.random(), + }: Partial = {}) { + return new OutgoingNoteDao( + note, + contractAddress, + storageSlot, + noteTypeId, + txHash, + l2BlockNumber, + l2BlockHash, + nonce, + noteHash, + index, + ovpkM, + ); + } } diff --git a/yarn-project/pxe/src/database/pxe_database.ts b/yarn-project/pxe/src/database/pxe_database.ts index d4cdb12bcec..f93d32240d3 100644 --- a/yarn-project/pxe/src/database/pxe_database.ts +++ b/yarn-project/pxe/src/database/pxe_database.ts @@ -1,4 +1,4 @@ -import { type IncomingNotesFilter, type OutgoingNotesFilter } from '@aztec/circuit-types'; +import { type InBlock, type IncomingNotesFilter, type OutgoingNotesFilter } from '@aztec/circuit-types'; import { type CompleteAddress, type ContractInstanceWithAddress, @@ -96,7 +96,7 @@ export interface PxeDatabase extends ContractArtifactDatabase, ContractInstanceD * @param account - A PublicKey instance representing the account for which the records are being removed. * @returns Removed notes. */ - removeNullifiedNotes(nullifiers: Fr[], account: PublicKey): Promise; + removeNullifiedNotes(nullifiers: InBlock[], account: PublicKey): Promise; /** * Gets the most recently processed block number. @@ -214,4 +214,16 @@ export interface PxeDatabase extends ContractArtifactDatabase, ContractInstanceD * @param appTaggingSecrets - The app siloed tagging secrets. */ setTaggingSecretsIndexesAsRecipient(indexedTaggingSecrets: IndexedTaggingSecret[]): Promise; + + /** + * Deletes all notes synched after this block number. + * @param blockNumber - All notes strictly after this block number are removed. + */ + removeNotesAfter(blockNumber: number): Promise; + + /** + * Restores notes nullified after the given block. + * @param blockNumber - All nullifiers strictly after this block are removed. + */ + unnullifyNotesAfter(blockNumber: number): Promise; } diff --git a/yarn-project/pxe/src/database/pxe_database_test_suite.ts b/yarn-project/pxe/src/database/pxe_database_test_suite.ts index 1fefa614bad..465aa153d5f 100644 --- a/yarn-project/pxe/src/database/pxe_database_test_suite.ts +++ b/yarn-project/pxe/src/database/pxe_database_test_suite.ts @@ -12,10 +12,8 @@ import { randomInt } from '@aztec/foundation/crypto'; import { Fr, Point } from '@aztec/foundation/fields'; import { BenchmarkingContractArtifact } from '@aztec/noir-contracts.js/Benchmarking'; -import { type IncomingNoteDao } from './incoming_note_dao.js'; -import { randomIncomingNoteDao } from './incoming_note_dao.test.js'; -import { type OutgoingNoteDao } from './outgoing_note_dao.js'; -import { randomOutgoingNoteDao } from './outgoing_note_dao.test.js'; +import { IncomingNoteDao } from './incoming_note_dao.js'; +import { OutgoingNoteDao } from './outgoing_note_dao.js'; import { type PxeDatabase } from './pxe_database.js'; /** @@ -121,11 +119,12 @@ export function describePxeDatabase(getDatabase: () => PxeDatabase) { storageSlots = Array.from({ length: 2 }).map(() => Fr.random()); notes = Array.from({ length: 10 }).map((_, i) => - randomIncomingNoteDao({ + IncomingNoteDao.random({ contractAddress: contractAddresses[i % contractAddresses.length], storageSlot: storageSlots[i % storageSlots.length], addressPoint: computePoint(owners[i % owners.length].address), index: BigInt(i), + l2BlockNumber: i, }), ); @@ -157,7 +156,11 @@ export function describePxeDatabase(getDatabase: () => PxeDatabase) { // Nullify all notes and use the same filter as other test cases for (const owner of owners) { const notesToNullify = notes.filter(note => note.addressPoint.equals(computePoint(owner.address))); - const nullifiers = notesToNullify.map(note => note.siloedNullifier); + const nullifiers = notesToNullify.map(note => ({ + data: note.siloedNullifier, + l2BlockNumber: note.l2BlockNumber, + l2BlockHash: note.l2BlockHash, + })); await expect(database.removeNullifiedNotes(nullifiers, computePoint(owner.address))).resolves.toEqual( notesToNullify, ); @@ -172,7 +175,11 @@ export function describePxeDatabase(getDatabase: () => PxeDatabase) { await database.addNotes(notes, []); const notesToNullify = notes.filter(note => note.addressPoint.equals(computePoint(owners[0].address))); - const nullifiers = notesToNullify.map(note => note.siloedNullifier); + const nullifiers = notesToNullify.map(note => ({ + data: note.siloedNullifier, + l2BlockNumber: note.l2BlockNumber, + l2BlockHash: note.l2BlockHash, + })); await expect(database.removeNullifiedNotes(nullifiers, notesToNullify[0].addressPoint)).resolves.toEqual( notesToNullify, ); @@ -184,11 +191,35 @@ export function describePxeDatabase(getDatabase: () => PxeDatabase) { expect(actualNotesWithActive).toEqual(notes.filter(note => !notesToNullify.includes(note))); }); + it('handles note unnullification', async () => { + await database.setHeader(makeHeader(randomInt(1000), 100, 0 /** slot number */)); + await database.addNotes(notes, []); + + const notesToNullify = notes.filter(note => note.addressPoint.equals(computePoint(owners[0].address))); + const nullifiers = notesToNullify.map(note => ({ + data: note.siloedNullifier, + l2BlockNumber: 99, + l2BlockHash: Fr.random().toString(), + })); + await expect(database.removeNullifiedNotes(nullifiers, notesToNullify[0].addressPoint)).resolves.toEqual( + notesToNullify, + ); + await expect(database.unnullifyNotesAfter(98)).resolves.toEqual(undefined); + + const result = await database.getIncomingNotes({ status: NoteStatus.ACTIVE, owner: owners[0].address }); + + expect(result.sort()).toEqual([...notesToNullify].sort()); + }); + it('returns active and nullified notes when requesting either', async () => { await database.addNotes(notes, []); const notesToNullify = notes.filter(note => note.addressPoint.equals(computePoint(owners[0].address))); - const nullifiers = notesToNullify.map(note => note.siloedNullifier); + const nullifiers = notesToNullify.map(note => ({ + data: note.siloedNullifier, + l2BlockNumber: note.l2BlockNumber, + l2BlockHash: note.l2BlockHash, + })); await expect(database.removeNullifiedNotes(nullifiers, notesToNullify[0].addressPoint)).resolves.toEqual( notesToNullify, ); @@ -246,7 +277,16 @@ export function describePxeDatabase(getDatabase: () => PxeDatabase) { ).resolves.toEqual([notes[0]]); await expect( - database.removeNullifiedNotes([notes[0].siloedNullifier], computePoint(owners[0].address)), + database.removeNullifiedNotes( + [ + { + data: notes[0].siloedNullifier, + l2BlockHash: notes[0].l2BlockHash, + l2BlockNumber: notes[0].l2BlockNumber, + }, + ], + computePoint(owners[0].address), + ), ).resolves.toEqual([notes[0]]); await expect( @@ -260,6 +300,14 @@ export function describePxeDatabase(getDatabase: () => PxeDatabase) { }), ).resolves.toEqual([]); }); + + it('removes notes after a given block', async () => { + await database.addNotes(notes, [], owners[0].address); + + await database.removeNotesAfter(5); + const result = await database.getIncomingNotes({ scopes: [owners[0].address] }); + expect(new Set(result)).toEqual(new Set(notes.slice(0, 6))); + }); }); describe('outgoing notes', () => { @@ -307,7 +355,7 @@ export function describePxeDatabase(getDatabase: () => PxeDatabase) { storageSlots = Array.from({ length: 2 }).map(() => Fr.random()); notes = Array.from({ length: 10 }).map((_, i) => - randomOutgoingNoteDao({ + OutgoingNoteDao.random({ contractAddress: contractAddresses[i % contractAddresses.length], storageSlot: storageSlots[i % storageSlots.length], ovpkM: owners[i % owners.length].publicKeys.masterOutgoingViewingPublicKey, diff --git a/yarn-project/pxe/src/kernel_oracle/index.ts b/yarn-project/pxe/src/kernel_oracle/index.ts index d2115d2cc40..a66ec8db465 100644 --- a/yarn-project/pxe/src/kernel_oracle/index.ts +++ b/yarn-project/pxe/src/kernel_oracle/index.ts @@ -70,7 +70,7 @@ export class KernelOracle implements ProvingDataOracle { } async getNoteHashTreeRoot(): Promise { - const header = await this.node.getHeader(this.blockNumber); + const header = await this.node.getBlockHeader(this.blockNumber); return header.state.partial.noteHashTree.root; } diff --git a/yarn-project/pxe/src/note_decryption_utils/produce_note_daos.ts b/yarn-project/pxe/src/note_decryption_utils/produce_note_daos.ts index 8e857d51330..fc3e1918ce1 100644 --- a/yarn-project/pxe/src/note_decryption_utils/produce_note_daos.ts +++ b/yarn-project/pxe/src/note_decryption_utils/produce_note_daos.ts @@ -34,6 +34,8 @@ export async function produceNoteDaos( ovpkM: PublicKey | undefined, payload: L1NotePayload, txHash: TxHash, + l2BlockNumber: number, + l2BlockHash: string, noteHashes: Fr[], dataStartIndexForTx: number, excludedIndices: Set, @@ -56,6 +58,8 @@ export async function produceNoteDaos( addressPoint, payload, txHash, + l2BlockNumber, + l2BlockHash, noteHashes, dataStartIndexForTx, excludedIndices, @@ -74,6 +78,8 @@ export async function produceNoteDaos( incomingNote.storageSlot, incomingNote.noteTypeId, incomingNote.txHash, + incomingNote.l2BlockNumber, + incomingNote.l2BlockHash, incomingNote.nonce, incomingNote.noteHash, incomingNote.index, @@ -86,6 +92,8 @@ export async function produceNoteDaos( ovpkM, payload, txHash, + l2BlockNumber, + l2BlockHash, noteHashes, dataStartIndexForTx, excludedIndices, diff --git a/yarn-project/pxe/src/note_decryption_utils/produce_note_daos_for_key.ts b/yarn-project/pxe/src/note_decryption_utils/produce_note_daos_for_key.ts index 9e530b387d1..eeeb6c9ee9e 100644 --- a/yarn-project/pxe/src/note_decryption_utils/produce_note_daos_for_key.ts +++ b/yarn-project/pxe/src/note_decryption_utils/produce_note_daos_for_key.ts @@ -13,6 +13,8 @@ export async function produceNoteDaosForKey( pkM: PublicKey, payload: L1NotePayload, txHash: TxHash, + l2BlockNumber: number, + l2BlockHash: string, noteHashes: Fr[], dataStartIndexForTx: number, excludedIndices: Set, @@ -21,6 +23,8 @@ export async function produceNoteDaosForKey( note: Note, payload: L1NotePayload, noteInfo: NoteInfo, + l2BlockNumber: number, + l2BlockHash: string, dataStartIndexForTx: number, pkM: PublicKey, ) => T, @@ -44,7 +48,7 @@ export async function produceNoteDaosForKey( ); excludedIndices?.add(noteInfo.noteHashIndex); - noteDao = daoConstructor(note, payload, noteInfo, dataStartIndexForTx, pkM); + noteDao = daoConstructor(note, payload, noteInfo, l2BlockNumber, l2BlockHash, dataStartIndexForTx, pkM); } catch (e) { logger.error(`Could not process note because of "${e}". Discarding note...`); } diff --git a/yarn-project/pxe/src/pxe_service/create_pxe_service.ts b/yarn-project/pxe/src/pxe_service/create_pxe_service.ts index 3fa9abe5b1e..a3bdd43b105 100644 --- a/yarn-project/pxe/src/pxe_service/create_pxe_service.ts +++ b/yarn-project/pxe/src/pxe_service/create_pxe_service.ts @@ -3,6 +3,7 @@ import { type AztecNode, type PrivateKernelProver } from '@aztec/circuit-types'; import { randomBytes } from '@aztec/foundation/crypto'; import { createDebugLogger } from '@aztec/foundation/log'; import { KeyStore } from '@aztec/key-store'; +import { L2TipsStore } from '@aztec/kv-store/stores'; import { createStore } from '@aztec/kv-store/utils'; import { type PXEServiceConfig } from '../config/index.js'; @@ -39,12 +40,14 @@ export async function createPXEService( const keyStore = new KeyStore( await createStore('pxe_key_store', configWithContracts, createDebugLogger('aztec:pxe:keystore:lmdb')), ); - const db = new KVPxeDatabase( - await createStore('pxe_data', configWithContracts, createDebugLogger('aztec:pxe:data:lmdb')), - ); + + const store = await createStore('pxe_data', configWithContracts, createDebugLogger('aztec:pxe:data:lmdb')); + + const db = new KVPxeDatabase(store); + const tips = new L2TipsStore(store, 'pxe'); const prover = proofCreator ?? (await createProver(config, logSuffix)); - const server = new PXEService(keyStore, aztecNode, db, prover, config, logSuffix); + const server = new PXEService(keyStore, aztecNode, db, tips, prover, config, logSuffix); await server.start(); return server; } diff --git a/yarn-project/pxe/src/pxe_service/pxe_service.ts b/yarn-project/pxe/src/pxe_service/pxe_service.ts index bbc78a4400f..9c6b809176e 100644 --- a/yarn-project/pxe/src/pxe_service/pxe_service.ts +++ b/yarn-project/pxe/src/pxe_service/pxe_service.ts @@ -6,6 +6,7 @@ import { type ExtendedNote, type FunctionCall, type GetUnencryptedLogsResponse, + type InBlock, type IncomingNotesFilter, L1EventPayload, type L2Block, @@ -22,7 +23,6 @@ import { type SiblingPath, SimulationError, type Tx, - type TxEffect, type TxExecutionRequest, type TxHash, TxProvingResult, @@ -58,6 +58,7 @@ import { Fr, type Point } from '@aztec/foundation/fields'; import { type DebugLogger, createDebugLogger } from '@aztec/foundation/log'; import { SerialQueue } from '@aztec/foundation/queue'; import { type KeyStore } from '@aztec/key-store'; +import { type L2TipsStore } from '@aztec/kv-store/stores'; import { ProtocolContractAddress, getCanonicalProtocolContract, @@ -93,12 +94,13 @@ export class PXEService implements PXE { private keyStore: KeyStore, private node: AztecNode, private db: PxeDatabase, + tipsStore: L2TipsStore, private proofCreator: PrivateKernelProver, - private config: PXEServiceConfig, + config: PXEServiceConfig, logSuffix?: string, ) { this.log = createDebugLogger(logSuffix ? `aztec:pxe_service_${logSuffix}` : `aztec:pxe_service`); - this.synchronizer = new Synchronizer(node, db, this.jobQueue, logSuffix); + this.synchronizer = new Synchronizer(node, db, tipsStore, config, logSuffix); this.contractDataOracle = new ContractDataOracle(db); this.simulator = getAcirSimulator(db, node, keyStore, this.contractDataOracle); this.packageVersion = getPackageInfo().version; @@ -112,8 +114,7 @@ export class PXEService implements PXE { * @returns A promise that resolves when the server has started successfully. */ public async start() { - const { l2BlockPollingIntervalMS } = this.config; - await this.synchronizer.start(1, l2BlockPollingIntervalMS); + await this.synchronizer.start(); await this.#registerProtocolContracts(); const info = await this.getNodeInfo(); this.log.info(`Started PXE connected to chain ${info.l1ChainId} version ${info.protocolVersion}`); @@ -356,7 +357,7 @@ export class PXEService implements PXE { throw new Error(`Unknown account: ${note.owner.toString()}`); } - const nonces = await this.#getNoteNonces(note); + const { data: nonces, l2BlockNumber, l2BlockHash } = await this.#getNoteNonces(note); if (nonces.length === 0) { throw new Error(`Cannot find the note in tx: ${note.txHash}.`); } @@ -391,6 +392,8 @@ export class PXEService implements PXE { note.storageSlot, note.noteTypeId, note.txHash, + l2BlockNumber, + l2BlockHash, nonce, noteHash, siloedNullifier, @@ -403,7 +406,7 @@ export class PXEService implements PXE { } public async addNullifiedNote(note: ExtendedNote) { - const nonces = await this.#getNoteNonces(note); + const { data: nonces, l2BlockHash, l2BlockNumber } = await this.#getNoteNonces(note); if (nonces.length === 0) { throw new Error(`Cannot find the note in tx: ${note.txHash}.`); } @@ -434,6 +437,8 @@ export class PXEService implements PXE { note.storageSlot, note.noteTypeId, note.txHash, + l2BlockNumber, + l2BlockHash, nonce, noteHash, Fr.ZERO, // We are not able to derive @@ -450,15 +455,15 @@ export class PXEService implements PXE { * @returns The nonces of the note. * @remarks More than a single nonce may be returned since there might be more than one nonce for a given note. */ - async #getNoteNonces(note: ExtendedNote): Promise { + async #getNoteNonces(note: ExtendedNote): Promise> { const tx = await this.node.getTxEffect(note.txHash); if (!tx) { throw new Error(`Unknown tx: ${note.txHash}`); } const nonces: Fr[] = []; - const firstNullifier = tx.nullifiers[0]; - const hashes = tx.noteHashes; + const firstNullifier = tx.data.nullifiers[0]; + const hashes = tx.data.noteHashes; for (let i = 0; i < hashes.length; ++i) { const hash = hashes[i]; if (hash.equals(Fr.ZERO)) { @@ -479,7 +484,7 @@ export class PXEService implements PXE { } } - return nonces; + return { l2BlockHash: tx.l2BlockHash, l2BlockNumber: tx.l2BlockNumber, data: nonces }; } public async getBlock(blockNumber: number): Promise { @@ -593,7 +598,7 @@ export class PXEService implements PXE { return this.node.getTxReceipt(txHash); } - public getTxEffect(txHash: TxHash): Promise { + public getTxEffect(txHash: TxHash) { return this.node.getTxEffect(txHash); } diff --git a/yarn-project/pxe/src/pxe_service/test/pxe_service.test.ts b/yarn-project/pxe/src/pxe_service/test/pxe_service.test.ts index c4fd6d2a627..4acbc87e2a4 100644 --- a/yarn-project/pxe/src/pxe_service/test/pxe_service.test.ts +++ b/yarn-project/pxe/src/pxe_service/test/pxe_service.test.ts @@ -1,8 +1,9 @@ -import { type AztecNode, type PXE, TxEffect, mockTx } from '@aztec/circuit-types'; +import { type AztecNode, type PXE, TxEffect, mockTx, randomInBlock } from '@aztec/circuit-types'; import { INITIAL_L2_BLOCK_NUM } from '@aztec/circuits.js/constants'; import { type L1ContractAddresses } from '@aztec/ethereum'; import { EthAddress } from '@aztec/foundation/eth-address'; import { KeyStore } from '@aztec/key-store'; +import { L2TipsStore } from '@aztec/kv-store/stores'; import { openTmpStore } from '@aztec/kv-store/utils'; import { type MockProxy, mock } from 'jest-mock-extended'; @@ -19,6 +20,7 @@ function createPXEService(): Promise { const keyStore = new KeyStore(kvStore); const node = mock(); const db = new KVPxeDatabase(kvStore); + const tips = new L2TipsStore(kvStore, 'pxe'); const config: PXEServiceConfig = { l2BlockPollingIntervalMS: 100, l2StartingBlock: INITIAL_L2_BLOCK_NUM, @@ -45,7 +47,7 @@ function createPXEService(): Promise { }; node.getL1ContractAddresses.mockResolvedValue(mockedContracts); - return Promise.resolve(new PXEService(keyStore, node, db, new TestPrivateKernelProver(), config)); + return Promise.resolve(new PXEService(keyStore, node, db, tips, new TestPrivateKernelProver(), config)); } pxeTestSuite('PXEService', createPXEService); @@ -55,11 +57,13 @@ describe('PXEService', () => { let node: MockProxy; let db: PxeDatabase; let config: PXEServiceConfig; + let tips: L2TipsStore; beforeEach(() => { const kvStore = openTmpStore(); keyStore = new KeyStore(kvStore); node = mock(); + tips = new L2TipsStore(kvStore, 'pxe'); db = new KVPxeDatabase(kvStore); config = { l2BlockPollingIntervalMS: 100, @@ -75,9 +79,9 @@ describe('PXEService', () => { const settledTx = TxEffect.random(); const duplicateTx = mockTx(); - node.getTxEffect.mockResolvedValue(settledTx); + node.getTxEffect.mockResolvedValue(randomInBlock(settledTx)); - const pxe = new PXEService(keyStore, node, db, new TestPrivateKernelProver(), config); + const pxe = new PXEService(keyStore, node, db, tips, new TestPrivateKernelProver(), config); await expect(pxe.sendTx(duplicateTx)).rejects.toThrow(/A settled tx with equal hash/); }); }); diff --git a/yarn-project/pxe/src/simulator_oracle/index.ts b/yarn-project/pxe/src/simulator_oracle/index.ts index 73cf685b016..9e6c6932210 100644 --- a/yarn-project/pxe/src/simulator_oracle/index.ts +++ b/yarn-project/pxe/src/simulator_oracle/index.ts @@ -1,5 +1,6 @@ import { type AztecNode, + type InBlock, L1NotePayload, type L2Block, type L2BlockNumber, @@ -441,7 +442,12 @@ export class SimulatorOracle implements DBOracle { result.set( recipient.toString(), - logs.filter(log => log.blockNumber <= maxBlockNumber), + // Remove logs with a block number higher than the max block number + // Duplicates are likely to happen due to the sliding window, so we also filter them out + logs.filter( + (log, index, self) => + log.blockNumber <= maxBlockNumber && index === self.findIndex(otherLog => otherLog.equals(log)), + ), ); } return result; @@ -469,7 +475,7 @@ export class SimulatorOracle implements DBOracle { const incomingNotes: IncomingNoteDao[] = []; const outgoingNotes: OutgoingNoteDao[] = []; - const txEffectsCache = new Map(); + const txEffectsCache = new Map | undefined>(); for (const scopedLog of scopedLogs) { const incomingNotePayload = L1NotePayload.decryptAsIncoming( @@ -513,8 +519,10 @@ export class SimulatorOracle implements DBOracle { incomingNotePayload ? computePoint(recipient) : undefined, outgoingNotePayload ? recipientCompleteAddress.publicKeys.masterOutgoingViewingPublicKey : undefined, payload!, - txEffect.txHash, - txEffect.noteHashes, + txEffect.data.txHash, + txEffect.l2BlockNumber, + txEffect.l2BlockHash, + txEffect.data.noteHashes, scopedLog.dataStartIndexForTx, excludedIndices.get(scopedLog.txHash.toString())!, this.log, @@ -557,15 +565,17 @@ export class SimulatorOracle implements DBOracle { } const nullifiedNotes: IncomingNoteDao[] = []; const currentNotesForRecipient = await this.db.getIncomingNotes({ owner: recipient }); - const nullifierIndexes = await this.aztecNode.findLeavesIndexes( - 'latest', - MerkleTreeId.NULLIFIER_TREE, - currentNotesForRecipient.map(note => note.siloedNullifier), - ); - - const foundNullifiers = currentNotesForRecipient - .filter((_, i) => nullifierIndexes[i] !== undefined) - .map(note => note.siloedNullifier); + const nullifiersToCheck = currentNotesForRecipient.map(note => note.siloedNullifier); + const currentBlockNumber = await this.getBlockNumber(); + const nullifierIndexes = await this.aztecNode.findNullifiersIndexesWithBlock(currentBlockNumber, nullifiersToCheck); + + const foundNullifiers = nullifiersToCheck + .map((nullifier, i) => { + if (nullifierIndexes[i] !== undefined) { + return { ...nullifierIndexes[i], ...{ data: nullifier } } as InBlock; + } + }) + .filter(nullifier => nullifier !== undefined) as InBlock[]; await this.db.removeNullifiedNotes(foundNullifiers, computePoint(recipient)); nullifiedNotes.forEach(noteDao => { diff --git a/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts b/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts index f4630610b6e..9e160d98e82 100644 --- a/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts +++ b/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts @@ -7,6 +7,7 @@ import { type TxEffect, TxHash, TxScopedL2Log, + randomInBlock, } from '@aztec/circuit-types'; import { AztecAddress, @@ -423,13 +424,13 @@ describe('Simulator oracle', () => { }); aztecNode.getTxEffect.mockImplementation(txHash => { - return Promise.resolve(txEffectsMap[txHash.toString()] as TxEffect); + return Promise.resolve(randomInBlock(txEffectsMap[txHash.toString()] as TxEffect)); }); - aztecNode.findLeavesIndexes.mockImplementation((_blockNumber, _treeId, leafValues) => + aztecNode.findNullifiersIndexesWithBlock.mockImplementation((_blockNumber, requestedNullifiers) => Promise.resolve( - Array(leafValues.length - nullifiers) + Array(requestedNullifiers.length - nullifiers) .fill(undefined) - .concat(Array(nullifiers).fill(1n)), + .concat(Array(nullifiers).fill({ data: 1n, l2BlockNumber: 1n, l2BlockHash: '0x' })), ), ); return taggedLogs; diff --git a/yarn-project/pxe/src/synchronizer/synchronizer.test.ts b/yarn-project/pxe/src/synchronizer/synchronizer.test.ts index 8b0c6810eda..c504e82d279 100644 --- a/yarn-project/pxe/src/synchronizer/synchronizer.test.ts +++ b/yarn-project/pxe/src/synchronizer/synchronizer.test.ts @@ -1,94 +1,57 @@ -import { type AztecNode, L2Block } from '@aztec/circuit-types'; -import { type Header } from '@aztec/circuits.js'; -import { makeHeader } from '@aztec/circuits.js/testing'; -import { randomInt } from '@aztec/foundation/crypto'; -import { SerialQueue } from '@aztec/foundation/queue'; +import { type AztecNode, L2Block, type L2BlockStream } from '@aztec/circuit-types'; +import { L2TipsStore } from '@aztec/kv-store/stores'; import { openTmpStore } from '@aztec/kv-store/utils'; +import { jest } from '@jest/globals'; import { type MockProxy, mock } from 'jest-mock-extended'; +import times from 'lodash.times'; import { type PxeDatabase } from '../database/index.js'; import { KVPxeDatabase } from '../database/kv_pxe_database.js'; import { Synchronizer } from './synchronizer.js'; describe('Synchronizer', () => { - let aztecNode: MockProxy; let database: PxeDatabase; - let synchronizer: TestSynchronizer; - let jobQueue: SerialQueue; - const initialSyncBlockNumber = 3; - let headerBlock3: Header; - - beforeEach(() => { - headerBlock3 = makeHeader(randomInt(1000), initialSyncBlockNumber, initialSyncBlockNumber); + let synchronizer: Synchronizer; + let tipsStore: L2TipsStore; // eslint-disable-line @typescript-eslint/no-unused-vars - aztecNode = mock(); - database = new KVPxeDatabase(openTmpStore()); - jobQueue = new SerialQueue(); - synchronizer = new TestSynchronizer(aztecNode, database, jobQueue); - }); - - it('sets header from aztec node on initial sync', async () => { - aztecNode.getBlockNumber.mockResolvedValue(initialSyncBlockNumber); - aztecNode.getHeader.mockResolvedValue(headerBlock3); + let aztecNode: MockProxy; + let blockStream: MockProxy; - await synchronizer.initialSync(); + const TestSynchronizer = class extends Synchronizer { + protected override createBlockStream(): L2BlockStream { + return blockStream; + } + }; - expect(database.getHeader()).toEqual(headerBlock3); + beforeEach(() => { + const store = openTmpStore(); + blockStream = mock(); + aztecNode = mock(); + database = new KVPxeDatabase(store); + tipsStore = new L2TipsStore(store, 'pxe'); + synchronizer = new TestSynchronizer(aztecNode, database, tipsStore); }); it('sets header from latest block', async () => { const block = L2Block.random(1, 4); - aztecNode.getLogs.mockResolvedValueOnce([block.body.encryptedLogs]).mockResolvedValue([block.body.unencryptedLogs]); - aztecNode.getBlocks.mockResolvedValue([block]); - - await synchronizer.work(); + await synchronizer.handleBlockStreamEvent({ type: 'blocks-added', blocks: [block] }); const obtainedHeader = database.getHeader(); expect(obtainedHeader).toEqual(block.header); }); - it('overrides header from initial sync once current block number is larger', async () => { - // Initial sync is done on block with height 3 - aztecNode.getBlockNumber.mockResolvedValue(initialSyncBlockNumber); - aztecNode.getHeader.mockResolvedValue(headerBlock3); - - await synchronizer.initialSync(); - const header0 = database.getHeader(); - expect(header0).toEqual(headerBlock3); - - // We then process block with height 1, this should not change the header - const block1 = L2Block.random(1, 4); - - aztecNode.getLogs - .mockResolvedValueOnce([block1.body.encryptedLogs]) - .mockResolvedValue([block1.body.unencryptedLogs]); - - aztecNode.getBlocks.mockResolvedValue([block1]); + it('removes notes from db on a reorg', async () => { + const removeNotesAfter = jest.spyOn(database, 'removeNotesAfter').mockImplementation(() => Promise.resolve()); + const unnullifyNotesAfter = jest.spyOn(database, 'unnullifyNotesAfter').mockImplementation(() => Promise.resolve()); + aztecNode.getBlockHeader.mockImplementation(blockNumber => + Promise.resolve(L2Block.random(blockNumber as number).header), + ); - await synchronizer.work(); - const header1 = database.getHeader(); - expect(header1).toEqual(headerBlock3); - expect(header1).not.toEqual(block1.header); + await synchronizer.handleBlockStreamEvent({ type: 'blocks-added', blocks: times(5, L2Block.random) }); + await synchronizer.handleBlockStreamEvent({ type: 'chain-pruned', blockNumber: 3 }); - // But they should change when we process block with height 5 - const block5 = L2Block.random(5, 4); - - aztecNode.getBlocks.mockResolvedValue([block5]); - - await synchronizer.work(); - const header5 = database.getHeader(); - expect(header5).not.toEqual(headerBlock3); - expect(header5).toEqual(block5.header); + expect(removeNotesAfter).toHaveBeenCalledWith(3); + expect(unnullifyNotesAfter).toHaveBeenCalledWith(3); }); }); - -class TestSynchronizer extends Synchronizer { - public override work(limit = 1) { - return super.work(limit); - } - - public override initialSync(): Promise { - return super.initialSync(); - } -} diff --git a/yarn-project/pxe/src/synchronizer/synchronizer.ts b/yarn-project/pxe/src/synchronizer/synchronizer.ts index 3ed458b2db1..63fdc36843e 100644 --- a/yarn-project/pxe/src/synchronizer/synchronizer.ts +++ b/yarn-project/pxe/src/synchronizer/synchronizer.ts @@ -1,9 +1,14 @@ -import { type AztecNode, type L2Block } from '@aztec/circuit-types'; +import { + type AztecNode, + L2BlockStream, + type L2BlockStreamEvent, + type L2BlockStreamEventHandler, +} from '@aztec/circuit-types'; import { INITIAL_L2_BLOCK_NUM } from '@aztec/circuits.js'; import { type DebugLogger, createDebugLogger } from '@aztec/foundation/log'; -import { type SerialQueue } from '@aztec/foundation/queue'; -import { RunningPromise } from '@aztec/foundation/running-promise'; +import { type L2TipsStore } from '@aztec/kv-store/stores'; +import { type PXEConfig } from '../config/index.js'; import { type PxeDatabase } from '../database/index.js'; /** @@ -13,14 +18,48 @@ import { type PxeDatabase } from '../database/index.js'; * details, and fetch transactions by hash. The Synchronizer ensures that it maintains the note processors * in sync with the blockchain while handling retries and errors gracefully. */ -export class Synchronizer { - private runningPromise?: RunningPromise; +export class Synchronizer implements L2BlockStreamEventHandler { private running = false; private initialSyncBlockNumber = INITIAL_L2_BLOCK_NUM - 1; private log: DebugLogger; + protected readonly blockStream: L2BlockStream; - constructor(private node: AztecNode, private db: PxeDatabase, private jobQueue: SerialQueue, logSuffix = '') { + constructor( + private node: AztecNode, + private db: PxeDatabase, + private l2TipsStore: L2TipsStore, + config: Partial> = {}, + logSuffix?: string, + ) { this.log = createDebugLogger(logSuffix ? `aztec:pxe_synchronizer_${logSuffix}` : 'aztec:pxe_synchronizer'); + this.blockStream = this.createBlockStream(config); + } + + protected createBlockStream(config: Partial>) { + return new L2BlockStream(this.node, this.l2TipsStore, this, { + pollIntervalMS: config.l2BlockPollingIntervalMS, + startingBlock: config.l2StartingBlock, + }); + } + + /** Handle events emitted by the block stream. */ + public async handleBlockStreamEvent(event: L2BlockStreamEvent): Promise { + await this.l2TipsStore.handleBlockStreamEvent(event); + + switch (event.type) { + case 'blocks-added': + this.log.verbose(`Processing blocks ${event.blocks[0].number} to ${event.blocks.at(-1)!.number}`); + await this.db.setHeader(event.blocks.at(-1)!.header); + break; + case 'chain-pruned': + this.log.info(`Pruning data after block ${event.blockNumber} due to reorg`); + // We first unnullify and then remove so that unnullified notes that were created after the block number end up deleted. + await this.db.unnullifyNotesAfter(event.blockNumber); + await this.db.removeNotesAfter(event.blockNumber); + // Update the header to the last block. + await this.db.setHeader(await this.node.getBlockHeader(event.blockNumber)); + break; + } } /** @@ -31,79 +70,21 @@ export class Synchronizer { * @param limit - The maximum number of encrypted, unencrypted logs and blocks to fetch in each iteration. * @param retryInterval - The time interval (in ms) to wait before retrying if no data is available. */ - public async start(limit = 1, retryInterval = 1000) { + public async start() { if (this.running) { return; } this.running = true; - await this.jobQueue.put(() => this.initialSync()); + // REFACTOR: We should know the header of the genesis block without having to request it from the node. + await this.db.setHeader(await this.node.getBlockHeader(0)); + + await this.trigger(); this.log.info('Initial sync complete'); - this.runningPromise = new RunningPromise(() => this.sync(limit), retryInterval); - this.runningPromise.start(); + this.blockStream.start(); this.log.debug('Started loop'); } - protected async initialSync() { - // fast forward to the latest block - const latestHeader = await this.node.getHeader(); - this.initialSyncBlockNumber = Number(latestHeader.globalVariables.blockNumber.toBigInt()); - await this.db.setHeader(latestHeader); - } - - /** - * Fetches encrypted logs and blocks from the Aztec node and processes them for all note processors. - * If needed, catches up note processors that are lagging behind the main sync, e.g. because we just added a new account. - * - * Uses the job queue to ensure that - * - sync does not overlap with pxe simulations. - * - one sync is running at a time. - * - * @param limit - The maximum number of encrypted, unencrypted logs and blocks to fetch in each iteration. - * @returns a promise that resolves when the sync is complete - */ - protected sync(limit: number) { - return this.jobQueue.put(async () => { - let moreWork = true; - // keep external this.running flag to interrupt greedy sync - while (moreWork && this.running) { - moreWork = await this.work(limit); - } - }); - } - - /** - * Fetches encrypted logs and blocks from the Aztec node and processes them for all note processors. - * - * @param limit - The maximum number of encrypted, unencrypted logs and blocks to fetch in each iteration. - * @returns true if there could be more work, false if we're caught up or there was an error. - */ - protected async work(limit = 1): Promise { - const from = this.getSynchedBlockNumber() + 1; - try { - const blocks = await this.node.getBlocks(from, limit); - if (blocks.length === 0) { - return false; - } - - // Update latest tree roots from the most recent block - const latestBlock = blocks[blocks.length - 1]; - await this.setHeaderFromBlock(latestBlock); - return true; - } catch (err) { - this.log.error(`Error in synchronizer work`, err); - return false; - } - } - - private async setHeaderFromBlock(latestBlock: L2Block) { - if (latestBlock.number < this.initialSyncBlockNumber) { - return; - } - - await this.db.setHeader(latestBlock.header); - } - /** * Stops the synchronizer gracefully, interrupting any ongoing sleep and waiting for the current * iteration to complete before setting the running state to false. Once stopped, the synchronizer @@ -113,10 +94,15 @@ export class Synchronizer { */ public async stop() { this.running = false; - await this.runningPromise?.stop(); + await this.blockStream.stop(); this.log.info('Stopped'); } + /** Triggers a single run. */ + public async trigger() { + await this.blockStream.sync(); + } + private getSynchedBlockNumber() { return this.db.getBlockNumber() ?? this.initialSyncBlockNumber; } diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index 196a74d9b03..c66a71ea45e 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -763,6 +763,8 @@ __metadata: version: 0.0.0-use.local resolution: "@aztec/kv-store@workspace:kv-store" dependencies: + "@aztec/circuit-types": "workspace:^" + "@aztec/circuits.js": "workspace:^" "@aztec/ethereum": "workspace:^" "@aztec/foundation": "workspace:^" "@jest/globals": ^29.5.0