From b5f6465a5c40b791149f7491f821c76ba78d3909 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Fri, 14 Feb 2025 17:12:48 +0000 Subject: [PATCH 1/5] Fix epoch monitoring --- .../aztec/src/cli/cmds/start_prover_node.ts | 2 +- .../end-to-end/src/e2e_epochs.test.ts | 6 +- .../src/e2e_prover/e2e_prover_test.ts | 2 +- yarn-project/end-to-end/src/fixtures/utils.ts | 2 +- .../src/monitors/epoch-monitor.test.ts | 182 ++++++++++++++++-- .../prover-node/src/monitors/epoch-monitor.ts | 146 +++++++++++++- yarn-project/prover-node/src/prover-node.ts | 6 +- 7 files changed, 314 insertions(+), 32 deletions(-) diff --git a/yarn-project/aztec/src/cli/cmds/start_prover_node.ts b/yarn-project/aztec/src/cli/cmds/start_prover_node.ts index 2fd64e3e65a..80b26a7c62b 100644 --- a/yarn-project/aztec/src/cli/cmds/start_prover_node.ts +++ b/yarn-project/aztec/src/cli/cmds/start_prover_node.ts @@ -101,6 +101,6 @@ export async function startProverNode( signalHandlers.push(proverNode.stop.bind(proverNode)); - proverNode.start(); + await proverNode.start(); return { config: proverConfig }; } diff --git a/yarn-project/end-to-end/src/e2e_epochs.test.ts b/yarn-project/end-to-end/src/e2e_epochs.test.ts index 67e0d7c35a8..02bd6d3d7a5 100644 --- a/yarn-project/end-to-end/src/e2e_epochs.test.ts +++ b/yarn-project/end-to-end/src/e2e_epochs.test.ts @@ -124,11 +124,15 @@ describe('e2e_epochs', () => { }; it('does not allow submitting proof after epoch end', async () => { - // Hold off prover tx until end of next epoch! + // Here we cause a re-org by not publishing the proof for epoch 0 until after the end of epoch 1 + // The proof will be rejected and a re-org will take place + + // Hold off prover tx until end epoch 1 const [epoch2Start] = getTimestampRangeForEpoch(2n, constants); proverDelayer.pauseNextTxUntilTimestamp(epoch2Start); logger.info(`Delayed prover tx until epoch 2 starts at ${epoch2Start}`); + // Wait until the start of epoch 1 and grab the block number await waitUntilEpochStarts(1); const blockNumberAtEndOfEpoch0 = Number(await rollup.getBlockNumber()); logger.info(`Starting epoch 1 after L2 block ${blockNumberAtEndOfEpoch0}`); diff --git a/yarn-project/end-to-end/src/e2e_prover/e2e_prover_test.ts b/yarn-project/end-to-end/src/e2e_prover/e2e_prover_test.ts index 021e4360b7f..a623b7f8e02 100644 --- a/yarn-project/end-to-end/src/e2e_prover/e2e_prover_test.ts +++ b/yarn-project/end-to-end/src/e2e_prover/e2e_prover_test.ts @@ -280,7 +280,7 @@ export class FullProverTest { archiver: archiver as Archiver, blobSinkClient, }); - this.proverNode.start(); + await this.proverNode.start(); this.logger.warn(`Proofs are now enabled`); return this; diff --git a/yarn-project/end-to-end/src/fixtures/utils.ts b/yarn-project/end-to-end/src/fixtures/utils.ts index 5437f99efb1..570dc4a5258 100644 --- a/yarn-project/end-to-end/src/fixtures/utils.ts +++ b/yarn-project/end-to-end/src/fixtures/utils.ts @@ -754,7 +754,7 @@ export async function createAndSyncProverNode( archiver: archiver as Archiver, l1TxUtils, }); - proverNode.start(); + await proverNode.start(); return proverNode; } diff --git a/yarn-project/prover-node/src/monitors/epoch-monitor.test.ts b/yarn-project/prover-node/src/monitors/epoch-monitor.test.ts index f9333c86150..7df4a5a94b7 100644 --- a/yarn-project/prover-node/src/monitors/epoch-monitor.test.ts +++ b/yarn-project/prover-node/src/monitors/epoch-monitor.test.ts @@ -1,4 +1,5 @@ -import { type L2BlockSource } from '@aztec/circuit-types'; +import { L2Block, type L2BlockSource, type L2Tips } from '@aztec/circuit-types'; +import { Fr } from '@aztec/circuits.js'; import { sleep } from '@aztec/foundation/sleep'; import { type MockProxy, mock } from 'jest-mock-extended'; @@ -9,15 +10,77 @@ describe('EpochMonitor', () => { let l2BlockSource: MockProxy; let handler: MockProxy; let epochMonitor: EpochMonitor; + let blocks: L2Block[] = []; let lastEpochComplete: bigint = 0n; - - beforeEach(() => { + const epochSize = 10; + const totalNumBlocks = 100; + + const randomBlockHash = () => Fr.random().toString(); + const mockBlock = (blockNum: number) => { + const block = L2Block.random(blockNum, 1, 1, 1, undefined, Math.round(Math.random() * 1000000)); + return block; + }; + + const l2Tips: L2Tips = { + latest: { number: 58, hash: randomBlockHash() }, + proven: { number: 39, hash: randomBlockHash() }, + finalized: { number: 20, hash: randomBlockHash() }, + }; + + const updateTipsAndBlocks = async (startIndex?: number) => { + const newBlocks = [...blocks]; + if (startIndex !== undefined) { + const numToAdd = totalNumBlocks - startIndex; + const toAdd = await Promise.all( + Array.from({ length: numToAdd }).map((x: unknown, index: number) => mockBlock(index + startIndex + 1)), + ); + newBlocks.splice(startIndex, 0, ...toAdd); + } + const latestHeader = newBlocks.find(x => x.number == l2Tips.latest.number)!; + const provenHeader = newBlocks.find(x => x.number == l2Tips.proven.number)!; + const latestHash = await latestHeader!.hash(); + const provenHash = await provenHeader!.hash(); + blocks = newBlocks; + l2Tips.latest.hash = latestHash.toString(); + l2Tips.proven.hash = provenHash.toString(); + }; + + beforeEach(async () => { handler = mock(); + await updateTipsAndBlocks(0); l2BlockSource = mock({ isEpochComplete(epochNumber) { return Promise.resolve(epochNumber <= lastEpochComplete); }, + getL2Tips() { + const tips: L2Tips = { + latest: { number: l2Tips.latest.number, hash: l2Tips.latest.hash! }, + proven: { number: l2Tips.proven.number, hash: l2Tips.proven.hash! }, + finalized: { number: l2Tips.finalized.number, hash: l2Tips.finalized.hash! }, + }; + return Promise.resolve(tips); + }, + getBlocks(from, limit, _proven) { + const index = blocks.findIndex(x => x.number == from); + if (index === undefined) { + return Promise.resolve([]); + } + const slice = blocks.slice(index, index + limit).filter(x => x.number <= l2Tips.latest.number); + return Promise.resolve(slice); + }, + getBlockHeader(blockNumber: number | 'latest') { + if (blockNumber === 'latest') { + blockNumber = l2Tips.latest.number; + } + return Promise.resolve(blocks.find(x => x.number == blockNumber)!.header); + }, + }); + + l2BlockSource.getBlocksForEpoch.mockImplementation((epochNumber: bigint) => { + const startBlock = blocks.findIndex(x => x.number == Number(epochNumber) * epochSize); + const slice = blocks.slice(startBlock, startBlock + epochSize); + return Promise.resolve(slice); }); epochMonitor = new EpochMonitor(l2BlockSource, { pollingIntervalMs: 10 }); @@ -28,50 +91,129 @@ describe('EpochMonitor', () => { }); it('triggers initial epoch sync', async () => { - l2BlockSource.getL2EpochNumber.mockResolvedValue(10n); - epochMonitor.start(handler); + l2BlockSource.getL2EpochNumber.mockResolvedValue(5n); + await epochMonitor.start(handler); await sleep(100); - expect(handler.handleEpochCompleted).toHaveBeenCalledWith(9n); + expect(handler.handleEpochCompleted).toHaveBeenCalledWith(4n); }); it('does not trigger initial epoch sync on epoch zero', async () => { l2BlockSource.getL2EpochNumber.mockResolvedValue(0n); - epochMonitor.start(handler); + await epochMonitor.start(handler); await sleep(100); expect(handler.handleEpochCompleted).not.toHaveBeenCalled(); }); it('triggers epoch completion', async () => { - lastEpochComplete = 9n; - l2BlockSource.getL2EpochNumber.mockResolvedValue(10n); - epochMonitor.start(handler); + lastEpochComplete = 4n; + l2BlockSource.getL2EpochNumber.mockResolvedValue(5n); + await epochMonitor.start(handler); await sleep(100); - expect(handler.handleEpochCompleted).toHaveBeenCalledWith(9n); + expect(handler.handleEpochCompleted).toHaveBeenCalledWith(4n); expect(handler.handleEpochCompleted).toHaveBeenCalledTimes(1); - lastEpochComplete = 10n; + lastEpochComplete = 5n; + l2Tips.latest.number = 65; + l2Tips.proven.number = 49; + await updateTipsAndBlocks(); await sleep(100); - expect(handler.handleEpochCompleted).toHaveBeenCalledWith(10n); + expect(handler.handleEpochCompleted).toHaveBeenCalledWith(5n); expect(handler.handleEpochCompleted).toHaveBeenCalledTimes(2); - lastEpochComplete = 11n; + lastEpochComplete = 6n; + l2Tips.latest.number = 77; + l2Tips.proven.number = 59; + await updateTipsAndBlocks(); await sleep(100); - expect(handler.handleEpochCompleted).toHaveBeenCalledWith(11n); + expect(handler.handleEpochCompleted).toHaveBeenCalledWith(6n); expect(handler.handleEpochCompleted).toHaveBeenCalledTimes(3); + + lastEpochComplete = 7n; + l2Tips.latest.number = 83; + // A partial epoch has been proven, nothing should be triggered + l2Tips.proven.number = 65; + await updateTipsAndBlocks(); + await sleep(100); + expect(handler.handleEpochCompleted).toHaveBeenCalledTimes(3); + + // Now the full epoch has been proven, should trigger + l2Tips.proven.number = 69; + await updateTipsAndBlocks(); + await sleep(100); + expect(handler.handleEpochCompleted).toHaveBeenCalledWith(7n); + expect(handler.handleEpochCompleted).toHaveBeenCalledTimes(4); }); it('triggers epoch completion if initial epoch was already complete', async () => { // this happens if we start the monitor on the very last slot of an epoch - lastEpochComplete = 10n; - l2BlockSource.getL2EpochNumber.mockResolvedValue(10n); - epochMonitor.start(handler); + lastEpochComplete = 5n; + l2BlockSource.getL2EpochNumber.mockResolvedValue(5n); + l2Tips.proven.number = 49; + await epochMonitor.start(handler); + + await sleep(100); + expect(handler.handleEpochCompleted).toHaveBeenCalledWith(4n); + expect(handler.handleEpochCompleted).toHaveBeenCalledWith(5n); + expect(handler.handleEpochCompleted).toHaveBeenCalledTimes(2); + }); + + it('handles prunes', async () => { + lastEpochComplete = 4n; + l2BlockSource.getL2EpochNumber.mockResolvedValue(5n); + await epochMonitor.start(handler); + + await sleep(100); + expect(handler.handleEpochCompleted).toHaveBeenCalledWith(4n); + expect(handler.handleEpochCompleted).toHaveBeenCalledTimes(1); + // We signal epoch 5 has completed, the pending chain is at 65 + // but the proven chain still hasn't moved + // Nothing should change + lastEpochComplete = 5n; + l2Tips.latest.number = 65; + l2Tips.proven.number = 39; + await updateTipsAndBlocks(); + await sleep(100); + expect(handler.handleEpochCompleted).toHaveBeenCalledWith(4n); + expect(handler.handleEpochCompleted).toHaveBeenCalledTimes(1); + + // The epoch is partially proven, still nothing happens + l2Tips.proven.number = 42; + await updateTipsAndBlocks(); + await sleep(100); + expect(handler.handleEpochCompleted).toHaveBeenCalledWith(4n); + expect(handler.handleEpochCompleted).toHaveBeenCalledTimes(1); + + // We now prune back to the proven chain, nothing should change + // We will create new set of blocks from the proven block forward + lastEpochComplete = 5n; + l2Tips.latest.number = 42; + l2Tips.proven.number = 42; + await updateTipsAndBlocks(l2Tips.proven.number); + await sleep(100); + expect(handler.handleEpochCompleted).toHaveBeenCalledWith(4n); + expect(handler.handleEpochCompleted).toHaveBeenCalledTimes(1); + + // Move the pending chain forward a few blocks + lastEpochComplete = 5n; + l2Tips.latest.number = 46; + l2Tips.proven.number = 42; + await updateTipsAndBlocks(); + await sleep(100); + expect(handler.handleEpochCompleted).toHaveBeenCalledWith(4n); + expect(handler.handleEpochCompleted).toHaveBeenCalledTimes(1); + // Epoch 6 now consists of blocks 43 - 48 + const startBlock = blocks.findIndex(x => x.number == 43); + const slice = blocks.slice(startBlock, startBlock + 5); + l2BlockSource.getBlocksForEpoch.mockResolvedValueOnce(slice); + lastEpochComplete = 6n; + l2Tips.latest.number = 48; + l2Tips.proven.number = 42; await sleep(100); - expect(handler.handleEpochCompleted).toHaveBeenCalledWith(9n); - expect(handler.handleEpochCompleted).toHaveBeenCalledWith(10n); + expect(handler.handleEpochCompleted).toHaveBeenCalledWith(6n); expect(handler.handleEpochCompleted).toHaveBeenCalledTimes(2); }); }); diff --git a/yarn-project/prover-node/src/monitors/epoch-monitor.ts b/yarn-project/prover-node/src/monitors/epoch-monitor.ts index 27fa6c00c33..af72f26d2f5 100644 --- a/yarn-project/prover-node/src/monitors/epoch-monitor.ts +++ b/yarn-project/prover-node/src/monitors/epoch-monitor.ts @@ -1,4 +1,11 @@ -import { type L2BlockSource } from '@aztec/circuit-types'; +import { + type L2BlockSource, + L2BlockStream, + type L2BlockStreamEvent, + type L2BlockStreamEventHandler, + type L2BlockStreamLocalDataProvider, + type L2Tips, +} from '@aztec/circuit-types'; import { createLogger } from '@aztec/foundation/log'; import { RunningPromise } from '@aztec/foundation/running-promise'; import { @@ -13,7 +20,15 @@ export interface EpochMonitorHandler { handleEpochCompleted(epochNumber: bigint): Promise; } -export class EpochMonitor implements Traceable { +const PROVEN_BLOCK_HISTORY = 64; + +type EpochStatus = { + epochNumber: bigint; + finalBlockNumber: number; + startBlockNumber: number; +}; + +export class EpochMonitor implements Traceable, L2BlockStreamEventHandler, L2BlockStreamLocalDataProvider { private runningPromise: RunningPromise; private log = createLogger('prover-node:epoch-monitor'); public readonly tracer: Tracer; @@ -22,6 +37,11 @@ export class EpochMonitor implements Traceable { private latestEpochNumber: bigint | undefined; + private epochs: EpochStatus[] = []; + private blockHashes = new Map(); + private l2Tips: L2Tips | undefined = undefined; + private blockStream: L2BlockStream | undefined = undefined; + constructor( private readonly l2BlockSource: L2BlockSource, private options: { pollingIntervalMs: number }, @@ -31,13 +51,23 @@ export class EpochMonitor implements Traceable { this.runningPromise = new RunningPromise(this.work.bind(this), this.log, this.options.pollingIntervalMs); } - public start(handler: EpochMonitorHandler) { + public async start(handler: EpochMonitorHandler): Promise { this.handler = handler; + this.blockHashes.clear(); + this.epochs = []; + await this.reset(); + this.blockStream = new L2BlockStream(this.l2BlockSource, this, this, this.log, { + proven: false, + pollIntervalMS: this.options.pollingIntervalMs, + startingBlock: this.l2Tips!.proven.number, + }); + this.blockStream.start(); this.runningPromise.start(); this.log.info('Started EpochMonitor', this.options); } public async stop() { + await this.blockStream?.stop(); await this.runningPromise.stop(); this.log.info('Stopped EpochMonitor'); } @@ -47,15 +77,121 @@ export class EpochMonitor implements Traceable { if (!this.latestEpochNumber) { const epochNumber = await this.l2BlockSource.getL2EpochNumber(); if (epochNumber > 0n) { - await this.handler?.handleEpochCompleted(epochNumber - 1n); + await this.addNewEpoch(epochNumber - 1n); } this.latestEpochNumber = epochNumber; return; } + // If the block source says this epoch is ready to go then store it as of interest and wait + // for the proven chain to advance to just before it's block range if (await this.l2BlockSource.isEpochComplete(this.latestEpochNumber)) { - await this.handler?.handleEpochCompleted(this.latestEpochNumber); + await this.addNewEpoch(this.latestEpochNumber); this.latestEpochNumber += 1n; } + + // If we have epochs of interest, then look for the proven chain advancing to just before the first epoch we are watching + if (this.epochs.length > 0 && this.l2Tips!.proven.number >= this.epochs[0].startBlockNumber - 1) { + // Epoch can be proven + const epoch = this.epochs.shift(); + this.log.verbose(`Triggering epoch completion ${epoch?.epochNumber}`); + await this.handler!.handleEpochCompleted(epoch!.epochNumber); + } + } + + private async addNewEpoch(epochNumber: bigint) { + // We are adding this new epoch of interest, provided it has blocks to prove + // The archiver has these blocks already, otherwise this epoch would not have been returned from getL2EpochNumber + const blocks = await this.l2BlockSource.getBlocksForEpoch(epochNumber); + if (blocks.length == 0) { + return; + } + + // Ensure we have block hashes for these + for (const block of blocks) { + const h = await block.hash(); + this.blockHashes.set(block.number, h.toString()); + } + + // Store the epoch data + const status: EpochStatus = { + epochNumber, + finalBlockNumber: blocks[blocks.length - 1].number, + startBlockNumber: blocks[0].number, + }; + this.epochs.push(status); + this.log.trace( + `Added epoch ${status.epochNumber}, block range: ${status.startBlockNumber} - ${status.finalBlockNumber}`, + ); + } + + private async reset(): Promise { + // Take the latest state from the block source + // Get the current L2 tips and refresh our blocks cache with the pending chain and proven history + this.l2Tips = await this.l2BlockSource.getL2Tips(); + const earliestBlock = Math.max(1, this.l2Tips.proven.number - PROVEN_BLOCK_HISTORY); + const pendingChainLength = this.l2Tips.latest.number - earliestBlock; + const blocks = await this.l2BlockSource.getBlocks(earliestBlock, pendingChainLength + 1); + for (const block of blocks) { + const h = await block.hash(); + this.blockHashes.set(block.number, h.toString()); + } + } + + private clearHistoricBlocks() { + // Remove block hashes older than our configured history + const earliestBlock = Math.max(0, this.l2Tips!.proven.number - PROVEN_BLOCK_HISTORY); + if (earliestBlock < 1) { + return; + } + const allToClear = Array.from(this.blockHashes.keys()).filter(k => k < earliestBlock); + allToClear.forEach(k => this.blockHashes.delete(k)); + } + + private async clearCacheAndResetState() { + this.blockHashes.clear(); + this.epochs = []; + await this.reset(); + } + + async handleBlockStreamEvent(event: L2BlockStreamEvent): Promise { + if (event.type === 'chain-pruned') { + this.log.trace(`Received chain pruned event for block ${event.blockNumber}`); + + // Completely reset our state to the current from the block source + await this.clearCacheAndResetState(); + } else if (event.type === 'blocks-added') { + this.log.trace(`Received blocks added`); + + // Here we store the hashes of new blocks and update out latest tip if necessary + for (const block of event.blocks) { + const h = await block.hash(); + this.blockHashes.set(block.number, h.toString()); + } + const latest = this.l2Tips!.latest.number; + if (event.blocks.length > 0) { + const lastBlock = event.blocks[event.blocks.length - 1]; + if (lastBlock.number > latest) { + this.l2Tips!.latest.hash = this.blockHashes.get(lastBlock.number); + this.l2Tips!.latest.number = lastBlock.number; + this.log.trace(`Received blocks added event up to block ${lastBlock.number}`); + } + } + } else if (event.type === 'chain-proven') { + this.log.trace(`Received chain proven event for block ${event.blockNumber}`); + + // Update our proven tip and clean any historic blocks + this.l2Tips!.proven.hash = undefined; + this.l2Tips!.proven.number = event.blockNumber; + this.clearHistoricBlocks(); + } + return Promise.resolve(); + } + + getL2BlockHash(number: number): Promise { + return Promise.resolve(this.blockHashes.get(number)); + } + getL2Tips(): Promise { + return Promise.resolve(this.l2Tips!); } } diff --git a/yarn-project/prover-node/src/prover-node.ts b/yarn-project/prover-node/src/prover-node.ts index eed1c08da00..827aa164b87 100644 --- a/yarn-project/prover-node/src/prover-node.ts +++ b/yarn-project/prover-node/src/prover-node.ts @@ -129,9 +129,9 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable * Starts the prover node so it periodically checks for unproven epochs in the unfinalised chain from L1 and * starts proving jobs for them. */ - start() { + async start() { this.txFetcher.start(); - this.epochsMonitor.start(this); + await this.epochsMonitor.start(this); this.log.info('Started ProverNode', this.options); } @@ -216,7 +216,7 @@ export class ProverNode implements EpochMonitorHandler, ProverNodeApi, Traceable // Fast forward world state to right before the target block and get a fork this.log.verbose(`Creating proving job for epoch ${epochNumber} for block range ${fromBlock} to ${toBlock}`); - await this.worldState.syncImmediate(fromBlock - 1); + await this.worldState.syncImmediate(toBlock); // Create a processor using the forked world state const publicProcessorFactory = new PublicProcessorFactory( From 464d867f961b20004d19b66ba54f68f9e3ed0e11 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Fri, 14 Feb 2025 17:23:25 +0000 Subject: [PATCH 2/5] Fix --- yarn-project/prover-node/src/prover-node.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn-project/prover-node/src/prover-node.test.ts b/yarn-project/prover-node/src/prover-node.test.ts index a9f220ec588..ee155bb1300 100644 --- a/yarn-project/prover-node/src/prover-node.test.ts +++ b/yarn-project/prover-node/src/prover-node.test.ts @@ -179,7 +179,7 @@ describe('prover-node', () => { it('starts a proof during initial sync', async () => { l2BlockSource.getL2EpochNumber.mockResolvedValue(11n); - proverNode.start(); + await proverNode.start(); await sleep(100); expect(jobs[0].epochNumber).toEqual(10n); @@ -191,7 +191,7 @@ describe('prover-node', () => { mockCoordination.getTxsByHash.mockResolvedValue([]); - proverNode.start(); + await proverNode.start(); await sleep(2000); expect(jobs).toHaveLength(0); }); @@ -200,7 +200,7 @@ describe('prover-node', () => { lastEpochComplete = 10n; l2BlockSource.getL2EpochNumber.mockResolvedValue(11n); - proverNode.start(); + await proverNode.start(); await sleep(100); lastEpochComplete = 11n; From 30535dd4f262724d1b1c2d31bab228d71db6280d Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Fri, 14 Feb 2025 17:49:17 +0000 Subject: [PATCH 3/5] WIP --- .../end-to-end/src/e2e_ignition.test.ts | 148 ++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 yarn-project/end-to-end/src/e2e_ignition.test.ts diff --git a/yarn-project/end-to-end/src/e2e_ignition.test.ts b/yarn-project/end-to-end/src/e2e_ignition.test.ts new file mode 100644 index 00000000000..f85272fa45f --- /dev/null +++ b/yarn-project/end-to-end/src/e2e_ignition.test.ts @@ -0,0 +1,148 @@ +import { type Logger, getTimestampRangeForEpoch, retryUntil } from '@aztec/aztec.js'; +import { ChainMonitor } from '@aztec/aztec.js/ethereum'; +// eslint-disable-next-line no-restricted-imports +import { type L1RollupConstants } from '@aztec/circuit-types'; +import { RollupContract } from '@aztec/ethereum/contracts'; +import { type DelayedTxUtils, type Delayer, waitUntilL1Timestamp } from '@aztec/ethereum/test'; +import { type ProverNodePublisher } from '@aztec/prover-node'; +import { type TestProverNode } from '@aztec/prover-node/test'; +import { type SequencerPublisher } from '@aztec/sequencer-client'; +import { type TestSequencerClient } from '@aztec/sequencer-client/test'; + +import { jest } from '@jest/globals'; +import { type PublicClient } from 'viem'; + +import { type EndToEndContext, setup } from './fixtures/utils.js'; + +jest.setTimeout(1000 * 60 * 10); + +// Tests building of epochs using fast block times and short epochs. +// Spawns an aztec node and a prover node with fake proofs. +// Sequencer is allowed to build empty blocks. +describe('e2e_ignition', () => { + let context: EndToEndContext; + let l1Client: PublicClient; + let rollup: RollupContract; + let constants: L1RollupConstants; + let logger: Logger; + let proverDelayer: Delayer; + let sequencerDelayer: Delayer; + let monitor: ChainMonitor; + + const EPOCH_DURATION_IN_L2_SLOTS = 4; + const L2_SLOT_DURATION_IN_L1_SLOTS = 2; + const L1_BLOCK_TIME_IN_S = process.env.L1_BLOCK_TIME ? parseInt(process.env.L1_BLOCK_TIME) : 2; + + beforeEach(async () => { + // Set up system without any account nor protocol contracts + // and with faster block times and shorter epochs. + context = await setup(0, { + assumeProvenThrough: undefined, + checkIntervalMs: 50, + archiverPollingIntervalMS: 50, + skipProtocolContracts: true, + salt: 1, + aztecEpochDuration: EPOCH_DURATION_IN_L2_SLOTS, + aztecSlotDuration: L1_BLOCK_TIME_IN_S * L2_SLOT_DURATION_IN_L1_SLOTS, + ethereumSlotDuration: L1_BLOCK_TIME_IN_S, + aztecProofSubmissionWindow: EPOCH_DURATION_IN_L2_SLOTS * 2 - 1, + minTxsPerBlock: 0, + realProofs: false, + startProverNode: true, + // This must be enough so that the tx from the prover is delayed properly, + // but not so much to hang the sequencer and timeout the teardown + txPropagationMaxQueryAttempts: 12, + }); + + logger = context.logger; + l1Client = context.deployL1ContractsValues.publicClient; + rollup = RollupContract.getFromConfig(context.config); + + // Loop that tracks L1 and L2 block numbers and logs whenever there's a new one. + monitor = new ChainMonitor(rollup, logger); + monitor.start(); + + // This is hideous. + // We ought to have a definite reference to the l1TxUtils that we're using in both places, provided by the test context. + proverDelayer = ( + ((context.proverNode as TestProverNode).publisher as ProverNodePublisher).l1TxUtils as DelayedTxUtils + ).delayer!; + sequencerDelayer = ( + ((context.sequencer as TestSequencerClient).sequencer.publisher as SequencerPublisher).l1TxUtils as DelayedTxUtils + ).delayer!; + expect(proverDelayer).toBeDefined(); + expect(sequencerDelayer).toBeDefined(); + + // Constants used for time calculation + constants = { + epochDuration: EPOCH_DURATION_IN_L2_SLOTS, + slotDuration: L1_BLOCK_TIME_IN_S * L2_SLOT_DURATION_IN_L1_SLOTS, + l1StartBlock: await rollup.getL1StartBlock(), + l1GenesisTime: await rollup.getL1GenesisTime(), + ethereumSlotDuration: L1_BLOCK_TIME_IN_S, + }; + + logger.info(`L2 genesis at L1 block ${constants.l1StartBlock} (timestamp ${constants.l1GenesisTime})`); + }); + + afterEach(async () => { + jest.restoreAllMocks(); + monitor.stop(); + await context.proverNode?.stop(); + await context.teardown(); + }); + + afterAll(async () => { + jest.restoreAllMocks(); + monitor.stop(); + await context.proverNode?.stop(); + await context.teardown(); + }); + + /** Waits until the epoch begins (ie until the immediately previous L1 block is mined). */ + const waitUntilEpochStarts = async (epoch: number) => { + const [start] = getTimestampRangeForEpoch(BigInt(epoch), constants); + logger.info(`Waiting until L1 timestamp ${start} is reached as the start of epoch ${epoch}`); + await waitUntilL1Timestamp(l1Client, start - BigInt(L1_BLOCK_TIME_IN_S)); + return start; + }; + + /** Waits until the given L2 block number is mined. */ + const waitUntilL2BlockNumber = async (target: number) => { + await retryUntil(() => Promise.resolve(target === monitor.l2BlockNumber), `Wait until L2 block ${target}`, 60, 0.1); + }; + + /** Waits until the given L2 block number is marked as proven. */ + const waitUntilProvenL2BlockNumber = async (t: number, timeout = 60) => { + await retryUntil( + () => Promise.resolve(t === monitor.l2ProvenBlockNumber), + `Wait proven L2 block ${t}`, + timeout, + 0.1, + ); + }; + + it('successfully proves 8 epochs', async () => { + const targetProvenEpochs = 4; + const targetProvenBlockNumber = targetProvenEpochs * EPOCH_DURATION_IN_L2_SLOTS; + + let provenBlockNumber = 0; + let epochNumber = 0; + while (provenBlockNumber < targetProvenBlockNumber) { + logger.info(`Waiting for the end of epoch ${epochNumber}`); + await waitUntilEpochStarts(epochNumber + 1); + const epochTargetBlockNumber = Number(await rollup.getBlockNumber()); + logger.info(`Epoch ${epochNumber} ended with PENDING block number ${epochTargetBlockNumber}`); + await waitUntilL2BlockNumber(epochTargetBlockNumber); + provenBlockNumber = epochTargetBlockNumber; + logger.info( + `Reached PENDING L2 block ${epochTargetBlockNumber}, proving should now start, waiting for PROVEN block to reach ${provenBlockNumber}`, + ); + await waitUntilProvenL2BlockNumber(provenBlockNumber, 120); + expect(Number(await rollup.getProvenBlockNumber())).toBe(provenBlockNumber); + logger.info(`Reached PROVEN block number ${provenBlockNumber}, epoch ${epochNumber} is now proven`); + epochNumber++; + } + logger.info('Test Succeeded'); + }); +}); From 1e5e41c8621a3f7904304b333f478afe9a643c36 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Fri, 14 Feb 2025 18:47:10 +0000 Subject: [PATCH 4/5] Fix test --- .../prover-node/src/prover-node.test.ts | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/yarn-project/prover-node/src/prover-node.test.ts b/yarn-project/prover-node/src/prover-node.test.ts index ee155bb1300..282247ff3ea 100644 --- a/yarn-project/prover-node/src/prover-node.test.ts +++ b/yarn-project/prover-node/src/prover-node.test.ts @@ -5,6 +5,7 @@ import { type L1ToL2MessageSource, L2Block, type L2BlockSource, + type L2Tips, type MerkleTreeWriteOperations, type ProverCoordination, type Tx, @@ -177,7 +178,16 @@ describe('prover-node', () => { }); it('starts a proof during initial sync', async () => { + const blocks = await Promise.all([L2Block.random(10)]); l2BlockSource.getL2EpochNumber.mockResolvedValue(11n); + l2BlockSource.getBlocksForEpoch.mockResolvedValue(blocks); + const tips: L2Tips = { + latest: { number: 10, hash: '' }, + proven: { number: 9, hash: '' }, + finalized: { number: 8, hash: '' }, + }; + + l2BlockSource.getL2Tips.mockResolvedValue(tips); await proverNode.start(); await sleep(100); @@ -196,13 +206,34 @@ describe('prover-node', () => { expect(jobs).toHaveLength(0); }); - it('starts a proof when a new epoch is completed', async () => { + it('starts a proof when a new epoch is ready', async () => { + const blocks = await Promise.all([L2Block.random(10), L2Block.random(11), L2Block.random(12)]); lastEpochComplete = 10n; l2BlockSource.getL2EpochNumber.mockResolvedValue(11n); + l2BlockSource.getBlocksForEpoch.mockResolvedValueOnce([blocks[0]]); + l2BlockSource.getBlockHeader.mockResolvedValue(blocks[1].header); + l2BlockSource.getBlocks.mockResolvedValue(blocks); + const tips1: L2Tips = { + latest: { number: 11, hash: (await blocks[1].header.hash()).toString() }, + proven: { number: 9, hash: '' }, + finalized: { number: 8, hash: '' }, + }; + + l2BlockSource.getL2Tips.mockResolvedValue(tips1); await proverNode.start(); await sleep(100); + // Now progress the chain by an epoch + l2BlockSource.getBlocksForEpoch.mockResolvedValueOnce([blocks[1]]); + const tips2: L2Tips = { + latest: { number: 12, hash: (await blocks[2].header.hash()).toString() }, + proven: { number: 10, hash: '' }, + finalized: { number: 8, hash: '' }, + }; + + l2BlockSource.getL2Tips.mockResolvedValue(tips2); + lastEpochComplete = 11n; await sleep(100); From b1cdfb9d210013e5ea94608662cc236b8eb6aeaa Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Mon, 17 Feb 2025 15:32:20 +0000 Subject: [PATCH 5/5] Updated test --- .../aztec-node/src/aztec-node/server.ts | 6 ++ .../src/interfaces/aztec-node.test.ts | 5 ++ .../src/interfaces/aztec-node.ts | 8 +++ .../src/interfaces/world_state.ts | 23 ++++++- .../end-to-end/src/e2e_ignition.test.ts | 67 ++++++++++++------- .../src/sequencer/sequencer.ts | 7 +- .../server_world_state_synchronizer.ts | 14 +++- 7 files changed, 101 insertions(+), 29 deletions(-) diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index 98c8618d1ff..c0016291fe2 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -30,6 +30,7 @@ import { type TxScopedL2Log, TxStatus, type TxValidationResult, + type WorldStateSyncStatus, type WorldStateSynchronizer, tryStop, } from '@aztec/circuit-types'; @@ -122,6 +123,11 @@ export class AztecNodeService implements AztecNode, Traceable { this.log.info(`Aztec Node started on chain 0x${l1ChainId.toString(16)}`, config.l1Contracts); } + public async getWorldStateSyncStatus(): Promise { + const status = await this.worldStateSynchronizer.status(); + return status.syncSummary; + } + public getL2Tips() { return this.blockSource.getL2Tips(); } diff --git a/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts b/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts index 2e32485fffa..9474545e84d 100644 --- a/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts +++ b/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts @@ -53,6 +53,7 @@ import { type AztecNode, AztecNodeApiSchema } from './aztec-node.js'; import { type SequencerConfig } from './configs.js'; import { NullifierMembershipWitness } from './nullifier_membership_witness.js'; import { type ProverConfig } from './prover-client.js'; +import type { WorldStateSyncStatus } from './world_state.js'; describe('AztecNodeApiSchema', () => { let handler: MockAztecNode; @@ -355,6 +356,10 @@ describe('AztecNodeApiSchema', () => { class MockAztecNode implements AztecNode { constructor(private artifact: ContractArtifact) {} + getWorldStateSyncStatus(): Promise { + throw new Error('Method not implemented.'); + } + getL2Tips(): Promise { return Promise.resolve({ latest: { number: 1, hash: `0x01` }, diff --git a/yarn-project/circuit-types/src/interfaces/aztec-node.ts b/yarn-project/circuit-types/src/interfaces/aztec-node.ts index bd9e0383c35..30d98d36514 100644 --- a/yarn-project/circuit-types/src/interfaces/aztec-node.ts +++ b/yarn-project/circuit-types/src/interfaces/aztec-node.ts @@ -54,6 +54,7 @@ import { type L2BlockNumber, L2BlockNumberSchema } from './l2_block_number.js'; import { NullifierMembershipWitness } from './nullifier_membership_witness.js'; import { type ProverConfig, ProverConfigSchema } from './prover-client.js'; import { type ProverCoordination } from './prover-coordination.js'; +import { type WorldStateSyncStatus, WorldStateSyncStatusSchema } from './world_state.js'; /** * The aztec node. @@ -67,6 +68,11 @@ export interface AztecNode */ getL2Tips(): Promise; + /** + * Returns the sync status of the node's world state + */ + getWorldStateSyncStatus(): Promise; + /** * Find the indexes of the given leaves in the given tree. * @param blockNumber - The block number at which to get the data or 'latest' for latest data @@ -450,6 +456,8 @@ export interface AztecNode export const AztecNodeApiSchema: ApiSchemaFor = { getL2Tips: z.function().args().returns(L2TipsSchema), + getWorldStateSyncStatus: z.function().args().returns(WorldStateSyncStatusSchema), + findLeavesIndexes: z .function() .args(L2BlockNumberSchema, z.nativeEnum(MerkleTreeId), z.array(schemas.Fr)) diff --git a/yarn-project/circuit-types/src/interfaces/world_state.ts b/yarn-project/circuit-types/src/interfaces/world_state.ts index 4fd93acf259..7147c815407 100644 --- a/yarn-project/circuit-types/src/interfaces/world_state.ts +++ b/yarn-project/circuit-types/src/interfaces/world_state.ts @@ -1,4 +1,5 @@ -import { type L2BlockId } from '../l2_block_source.js'; +import { z } from 'zod'; + import type { MerkleTreeReadOperations, MerkleTreeWriteOperations } from './merkle_tree_operations.js'; /** @@ -11,6 +12,14 @@ export enum WorldStateRunningState { STOPPED, } +export interface WorldStateSyncStatus { + latestBlockNumber: number; + latestBlockHash: string; + finalisedBlockNumber: number; + oldestHistoricBlockNumber: number; + treesAreSynched: boolean; +} + /** * Defines the status of the world state synchronizer. */ @@ -20,9 +29,9 @@ export interface WorldStateSynchronizerStatus { */ state: WorldStateRunningState; /** - * The block number that the world state synchronizer is synced to. + * The block numbers that the world state synchronizer is synced to. */ - syncedToL2Block: L2BlockId; + syncSummary: WorldStateSyncStatus; } /** Provides writeable forks of the world state at a given block number. */ @@ -65,3 +74,11 @@ export interface WorldStateSynchronizer extends ForkMerkleTreeOperations { */ getCommitted(): MerkleTreeReadOperations; } + +export const WorldStateSyncStatusSchema = z.object({ + finalisedBlockNumber: z.number(), + latestBlockNumber: z.number(), + latestBlockHash: z.string(), + oldestHistoricBlockNumber: z.number(), + treesAreSynched: z.boolean(), +}) satisfies z.ZodType; diff --git a/yarn-project/end-to-end/src/e2e_ignition.test.ts b/yarn-project/end-to-end/src/e2e_ignition.test.ts index f85272fa45f..98a74c58441 100644 --- a/yarn-project/end-to-end/src/e2e_ignition.test.ts +++ b/yarn-project/end-to-end/src/e2e_ignition.test.ts @@ -1,13 +1,9 @@ -import { type Logger, getTimestampRangeForEpoch, retryUntil } from '@aztec/aztec.js'; +import { type Logger, getTimestampRangeForEpoch, retryUntil, sleep } from '@aztec/aztec.js'; import { ChainMonitor } from '@aztec/aztec.js/ethereum'; // eslint-disable-next-line no-restricted-imports -import { type L1RollupConstants } from '@aztec/circuit-types'; +import { type L1RollupConstants, type L2BlockNumber, MerkleTreeId } from '@aztec/circuit-types'; import { RollupContract } from '@aztec/ethereum/contracts'; -import { type DelayedTxUtils, type Delayer, waitUntilL1Timestamp } from '@aztec/ethereum/test'; -import { type ProverNodePublisher } from '@aztec/prover-node'; -import { type TestProverNode } from '@aztec/prover-node/test'; -import { type SequencerPublisher } from '@aztec/sequencer-client'; -import { type TestSequencerClient } from '@aztec/sequencer-client/test'; +import { waitUntilL1Timestamp } from '@aztec/ethereum/test'; import { jest } from '@jest/globals'; import { type PublicClient } from 'viem'; @@ -25,13 +21,14 @@ describe('e2e_ignition', () => { let rollup: RollupContract; let constants: L1RollupConstants; let logger: Logger; - let proverDelayer: Delayer; - let sequencerDelayer: Delayer; let monitor: ChainMonitor; const EPOCH_DURATION_IN_L2_SLOTS = 4; const L2_SLOT_DURATION_IN_L1_SLOTS = 2; const L1_BLOCK_TIME_IN_S = process.env.L1_BLOCK_TIME ? parseInt(process.env.L1_BLOCK_TIME) : 2; + const WORLD_STATE_BLOCK_HISTORY = 2; + const WORLD_STATE_BLOCK_CHECK_INTERVAL = 50; + const ARCHIVER_POLL_INTERVAL = 50; beforeEach(async () => { // Set up system without any account nor protocol contracts @@ -39,7 +36,8 @@ describe('e2e_ignition', () => { context = await setup(0, { assumeProvenThrough: undefined, checkIntervalMs: 50, - archiverPollingIntervalMS: 50, + archiverPollingIntervalMS: ARCHIVER_POLL_INTERVAL, + worldStateBlockCheckIntervalMS: WORLD_STATE_BLOCK_CHECK_INTERVAL, skipProtocolContracts: true, salt: 1, aztecEpochDuration: EPOCH_DURATION_IN_L2_SLOTS, @@ -52,6 +50,7 @@ describe('e2e_ignition', () => { // This must be enough so that the tx from the prover is delayed properly, // but not so much to hang the sequencer and timeout the teardown txPropagationMaxQueryAttempts: 12, + worldStateBlockHistory: WORLD_STATE_BLOCK_HISTORY, }); logger = context.logger; @@ -62,17 +61,6 @@ describe('e2e_ignition', () => { monitor = new ChainMonitor(rollup, logger); monitor.start(); - // This is hideous. - // We ought to have a definite reference to the l1TxUtils that we're using in both places, provided by the test context. - proverDelayer = ( - ((context.proverNode as TestProverNode).publisher as ProverNodePublisher).l1TxUtils as DelayedTxUtils - ).delayer!; - sequencerDelayer = ( - ((context.sequencer as TestSequencerClient).sequencer.publisher as SequencerPublisher).l1TxUtils as DelayedTxUtils - ).delayer!; - expect(proverDelayer).toBeDefined(); - expect(sequencerDelayer).toBeDefined(); - // Constants used for time calculation constants = { epochDuration: EPOCH_DURATION_IN_L2_SLOTS, @@ -122,8 +110,30 @@ describe('e2e_ignition', () => { ); }; - it('successfully proves 8 epochs', async () => { - const targetProvenEpochs = 4; + const waitForNodeToSync = async (blockNumber: number, type: 'finalised' | 'historic') => { + const waitTime = ARCHIVER_POLL_INTERVAL + WORLD_STATE_BLOCK_CHECK_INTERVAL; + let synched = false; + while (!synched) { + await sleep(waitTime); + const syncState = await context.aztecNode.getWorldStateSyncStatus(); + if (type === 'finalised') { + synched = syncState.finalisedBlockNumber >= blockNumber; + } else { + synched = syncState.oldestHistoricBlockNumber >= blockNumber; + } + } + }; + + const verifyHistoricBlock = async (blockNumber: L2BlockNumber, expectedSuccess: boolean) => { + const result = await context.aztecNode + .findBlockNumbersForIndexes(blockNumber, MerkleTreeId.NULLIFIER_TREE, [0n]) + .then(_ => true) + .catch(_ => false); + expect(result).toBe(expectedSuccess); + }; + + it('successfully proves all epochs', async () => { + const targetProvenEpochs = 8; const targetProvenBlockNumber = targetProvenEpochs * EPOCH_DURATION_IN_L2_SLOTS; let provenBlockNumber = 0; @@ -142,6 +152,17 @@ describe('e2e_ignition', () => { expect(Number(await rollup.getProvenBlockNumber())).toBe(provenBlockNumber); logger.info(`Reached PROVEN block number ${provenBlockNumber}, epoch ${epochNumber} is now proven`); epochNumber++; + + // Verify the state syncs + await waitForNodeToSync(provenBlockNumber, 'finalised'); + await verifyHistoricBlock(provenBlockNumber, true); + const expectedOldestHistoricBlock = provenBlockNumber - WORLD_STATE_BLOCK_HISTORY + 1; + const expectedBlockRemoved = expectedOldestHistoricBlock - 1; + await waitForNodeToSync(expectedOldestHistoricBlock, 'historic'); + await verifyHistoricBlock(Math.max(expectedOldestHistoricBlock, 1), true); + if (expectedBlockRemoved > 0) { + await verifyHistoricBlock(expectedBlockRemoved, false); + } } logger.info('Test Succeeded'); }); diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.ts index 58dd8d97cc8..7e365ebeea0 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.ts @@ -685,7 +685,12 @@ export class Sequencer { */ protected async getChainTip(): Promise<{ blockNumber: number; archive: Fr } | undefined> { const syncedBlocks = await Promise.all([ - this.worldState.status().then((s: WorldStateSynchronizerStatus) => s.syncedToL2Block), + this.worldState.status().then((s: WorldStateSynchronizerStatus) => { + return { + number: s.syncSummary.latestBlockNumber, + hash: s.syncSummary.latestBlockHash, + }; + }), this.l2BlockSource.getL2Tips().then(t => t.latest), this.p2pClient.getStatus().then(p2p => p2p.syncedToL2Block), this.l1ToL2MessageSource.getBlockNumber(), diff --git a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts index dd5d0337108..4fa832dd3b0 100644 --- a/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts +++ b/yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts @@ -12,6 +12,7 @@ import { type MerkleTreeReadOperations, type MerkleTreeWriteOperations, WorldStateRunningState, + type WorldStateSyncStatus, type WorldStateSynchronizer, type WorldStateSynchronizerStatus, } from '@aztec/circuit-types'; @@ -128,8 +129,16 @@ export class ServerWorldStateSynchronizer } public async status(): Promise { + const summary = await this.merkleTreeDb.getStatusSummary(); + const status: WorldStateSyncStatus = { + latestBlockNumber: Number(summary.unfinalisedBlockNumber), + latestBlockHash: (await this.getL2BlockHash(Number(summary.unfinalisedBlockNumber))) ?? '', + finalisedBlockNumber: Number(summary.finalisedBlockNumber), + oldestHistoricBlockNumber: Number(summary.oldestHistoricalBlock), + treesAreSynched: summary.treesAreSynched, + }; return { - syncedToL2Block: (await this.getL2Tips()).latest, + syncSummary: status, state: this.currentState, }; } @@ -281,7 +290,8 @@ export class ServerWorldStateSynchronizer return; } this.log.verbose(`Pruning historic blocks to ${newHistoricBlock}`); - await this.merkleTreeDb.removeHistoricalBlocks(newHistoricBlock); + const status = await this.merkleTreeDb.removeHistoricalBlocks(newHistoricBlock); + this.log.debug(`World state summary `, status.summary); } private handleChainProven(blockNumber: number) {