fix: Fix for detecting the correct blocks/epochs to prove

yarn-project/aztec-node/src/aztec-node/server.ts

@@ -30,6 +30,7 @@ import {
   type TxScopedL2Log,
   TxStatus,
   type TxValidationResult,
+  type WorldStateSyncStatus,
   type WorldStateSynchronizer,
   tryStop,
 } from '@aztec/circuit-types';
@@ -122,6 +123,11 @@ export class AztecNodeService implements AztecNode, Traceable {
     this.log.info(`Aztec Node started on chain 0x${l1ChainId.toString(16)}`, config.l1Contracts);
   }
 
+  public async getWorldStateSyncStatus(): Promise<WorldStateSyncStatus> {
+    const status = await this.worldStateSynchronizer.status();
+    return status.syncSummary;
+  }
+
   public getL2Tips() {
     return this.blockSource.getL2Tips();
   }

yarn-project/aztec/src/cli/cmds/start_prover_node.ts

@@ -101,6 +101,6 @@ export async function startProverNode(
 
   signalHandlers.push(proverNode.stop.bind(proverNode));
 
-  proverNode.start();
+  await proverNode.start();
   return { config: proverConfig };
 }

yarn-project/circuit-types/src/interfaces/aztec-node.test.ts

@@ -53,6 +53,7 @@ import { type AztecNode, AztecNodeApiSchema } from './aztec-node.js';
 import { type SequencerConfig } from './configs.js';
 import { NullifierMembershipWitness } from './nullifier_membership_witness.js';
 import { type ProverConfig } from './prover-client.js';
+import type { WorldStateSyncStatus } from './world_state.js';
 
 describe('AztecNodeApiSchema', () => {
   let handler: MockAztecNode;
@@ -355,6 +356,10 @@ describe('AztecNodeApiSchema', () => {
 class MockAztecNode implements AztecNode {
   constructor(private artifact: ContractArtifact) {}
 
+  getWorldStateSyncStatus(): Promise<WorldStateSyncStatus> {
+    throw new Error('Method not implemented.');
+  }
+
   getL2Tips(): Promise<L2Tips> {
     return Promise.resolve({
       latest: { number: 1, hash: `0x01` },

yarn-project/circuit-types/src/interfaces/aztec-node.ts

@@ -54,6 +54,7 @@ import { type L2BlockNumber, L2BlockNumberSchema } from './l2_block_number.js';
 import { NullifierMembershipWitness } from './nullifier_membership_witness.js';
 import { type ProverConfig, ProverConfigSchema } from './prover-client.js';
 import { type ProverCoordination } from './prover-coordination.js';
+import { type WorldStateSyncStatus, WorldStateSyncStatusSchema } from './world_state.js';
 
 /**
  * The aztec node.
@@ -67,6 +68,11 @@ export interface AztecNode
    */
   getL2Tips(): Promise<L2Tips>;
 
+  /**
+   * Returns the sync status of the node's world state
+   */
+  getWorldStateSyncStatus(): Promise<WorldStateSyncStatus>;
+
   /**
    * Find the indexes of the given leaves in the given tree.
    * @param blockNumber - The block number at which to get the data or 'latest' for latest data
@@ -450,6 +456,8 @@ export interface AztecNode
 export const AztecNodeApiSchema: ApiSchemaFor<AztecNode> = {
   getL2Tips: z.function().args().returns(L2TipsSchema),
 
+  getWorldStateSyncStatus: z.function().args().returns(WorldStateSyncStatusSchema),
+
   findLeavesIndexes: z
     .function()
     .args(L2BlockNumberSchema, z.nativeEnum(MerkleTreeId), z.array(schemas.Fr))

yarn-project/circuit-types/src/interfaces/world_state.ts

@@ -1,4 +1,5 @@
-import { type L2BlockId } from '../l2_block_source.js';
+import { z } from 'zod';
+
 import type { MerkleTreeReadOperations, MerkleTreeWriteOperations } from './merkle_tree_operations.js';
 
 /**
@@ -11,6 +12,14 @@ export enum WorldStateRunningState {
   STOPPED,
 }
 
+export interface WorldStateSyncStatus {
+  latestBlockNumber: number;
+  latestBlockHash: string;
+  finalisedBlockNumber: number;
+  oldestHistoricBlockNumber: number;
+  treesAreSynched: boolean;
+}
+
 /**
  * Defines the status of the world state synchronizer.
  */
@@ -20,9 +29,9 @@ export interface WorldStateSynchronizerStatus {
    */
   state: WorldStateRunningState;
   /**
-   * The block number that the world state synchronizer is synced to.
+   * The block numbers that the world state synchronizer is synced to.
    */
-  syncedToL2Block: L2BlockId;
+  syncSummary: WorldStateSyncStatus;
 }
 
 /** Provides writeable forks of the world state at a given block number. */
@@ -65,3 +74,11 @@ export interface WorldStateSynchronizer extends ForkMerkleTreeOperations {
    */
   getCommitted(): MerkleTreeReadOperations;
 }
+
+export const WorldStateSyncStatusSchema = z.object({
+  finalisedBlockNumber: z.number(),
+  latestBlockNumber: z.number(),
+  latestBlockHash: z.string(),
+  oldestHistoricBlockNumber: z.number(),
+  treesAreSynched: z.boolean(),
+}) satisfies z.ZodType<WorldStateSyncStatus>;

yarn-project/end-to-end/src/e2e_epochs.test.ts

@@ -124,11 +124,15 @@ describe('e2e_epochs', () => {
   };
 
   it('does not allow submitting proof after epoch end', async () => {
-    // Hold off prover tx until end of next epoch!
+    // Here we cause a re-org by not publishing the proof for epoch 0 until after the end of epoch 1
+    // The proof will be rejected and a re-org will take place
+
+    // Hold off prover tx until end epoch 1
     const [epoch2Start] = getTimestampRangeForEpoch(2n, constants);
     proverDelayer.pauseNextTxUntilTimestamp(epoch2Start);
     logger.info(`Delayed prover tx until epoch 2 starts at ${epoch2Start}`);
 
+    // Wait until the start of epoch 1 and grab the block number
     await waitUntilEpochStarts(1);
     const blockNumberAtEndOfEpoch0 = Number(await rollup.getBlockNumber());
     logger.info(`Starting epoch 1 after L2 block ${blockNumberAtEndOfEpoch0}`);

yarn-project/end-to-end/src/e2e_ignition.test.ts

@@ -0,0 +1,169 @@
+import { type Logger, getTimestampRangeForEpoch, retryUntil, sleep } from '@aztec/aztec.js';
+import { ChainMonitor } from '@aztec/aztec.js/ethereum';
+// eslint-disable-next-line no-restricted-imports
+import { type L1RollupConstants, type L2BlockNumber, MerkleTreeId } from '@aztec/circuit-types';
+import { RollupContract } from '@aztec/ethereum/contracts';
+import { waitUntilL1Timestamp } from '@aztec/ethereum/test';
+
+import { jest } from '@jest/globals';
+import { type PublicClient } from 'viem';
+
+import { type EndToEndContext, setup } from './fixtures/utils.js';
+
+jest.setTimeout(1000 * 60 * 10);
+
+// Tests building of epochs using fast block times and short epochs.
+// Spawns an aztec node and a prover node with fake proofs.
+// Sequencer is allowed to build empty blocks.
+describe('e2e_ignition', () => {
+  let context: EndToEndContext;
+  let l1Client: PublicClient;
+  let rollup: RollupContract;
+  let constants: L1RollupConstants;
+  let logger: Logger;
+  let monitor: ChainMonitor;
+
+  const EPOCH_DURATION_IN_L2_SLOTS = 4;
+  const L2_SLOT_DURATION_IN_L1_SLOTS = 2;
+  const L1_BLOCK_TIME_IN_S = process.env.L1_BLOCK_TIME ? parseInt(process.env.L1_BLOCK_TIME) : 2;
+  const WORLD_STATE_BLOCK_HISTORY = 2;
+  const WORLD_STATE_BLOCK_CHECK_INTERVAL = 50;
+  const ARCHIVER_POLL_INTERVAL = 50;
+
+  beforeEach(async () => {
+    // Set up system without any account nor protocol contracts
+    // and with faster block times and shorter epochs.
+    context = await setup(0, {
+      assumeProvenThrough: undefined,
+      checkIntervalMs: 50,
+      archiverPollingIntervalMS: ARCHIVER_POLL_INTERVAL,
+      worldStateBlockCheckIntervalMS: WORLD_STATE_BLOCK_CHECK_INTERVAL,
+      skipProtocolContracts: true,
+      salt: 1,
+      aztecEpochDuration: EPOCH_DURATION_IN_L2_SLOTS,
+      aztecSlotDuration: L1_BLOCK_TIME_IN_S * L2_SLOT_DURATION_IN_L1_SLOTS,
+      ethereumSlotDuration: L1_BLOCK_TIME_IN_S,
+      aztecProofSubmissionWindow: EPOCH_DURATION_IN_L2_SLOTS * 2 - 1,
+      minTxsPerBlock: 0,
+      realProofs: false,
+      startProverNode: true,
+      // This must be enough so that the tx from the prover is delayed properly,
+      // but not so much to hang the sequencer and timeout the teardown
+      txPropagationMaxQueryAttempts: 12,
+      worldStateBlockHistory: WORLD_STATE_BLOCK_HISTORY,
+    });
+
+    logger = context.logger;
+    l1Client = context.deployL1ContractsValues.publicClient;
+    rollup = RollupContract.getFromConfig(context.config);
+
+    // Loop that tracks L1 and L2 block numbers and logs whenever there's a new one.
+    monitor = new ChainMonitor(rollup, logger);
+    monitor.start();
+
+    // Constants used for time calculation
+    constants = {
+      epochDuration: EPOCH_DURATION_IN_L2_SLOTS,
+      slotDuration: L1_BLOCK_TIME_IN_S * L2_SLOT_DURATION_IN_L1_SLOTS,
+      l1StartBlock: await rollup.getL1StartBlock(),
+      l1GenesisTime: await rollup.getL1GenesisTime(),
+      ethereumSlotDuration: L1_BLOCK_TIME_IN_S,
+    };
+
+    logger.info(`L2 genesis at L1 block ${constants.l1StartBlock} (timestamp ${constants.l1GenesisTime})`);
+  });
+
+  afterEach(async () => {
+    jest.restoreAllMocks();
+    monitor.stop();
+    await context.proverNode?.stop();
+    await context.teardown();
+  });
+
+  afterAll(async () => {
+    jest.restoreAllMocks();
+    monitor.stop();
+    await context.proverNode?.stop();
+    await context.teardown();
+  });
+
+  /** Waits until the epoch begins (ie until the immediately previous L1 block is mined). */
+  const waitUntilEpochStarts = async (epoch: number) => {
+    const [start] = getTimestampRangeForEpoch(BigInt(epoch), constants);
+    logger.info(`Waiting until L1 timestamp ${start} is reached as the start of epoch ${epoch}`);
+    await waitUntilL1Timestamp(l1Client, start - BigInt(L1_BLOCK_TIME_IN_S));
+    return start;
+  };
+
+  /** Waits until the given L2 block number is mined. */
+  const waitUntilL2BlockNumber = async (target: number) => {
+    await retryUntil(() => Promise.resolve(target === monitor.l2BlockNumber), `Wait until L2 block ${target}`, 60, 0.1);
+  };
+
+  /** Waits until the given L2 block number is marked as proven. */
+  const waitUntilProvenL2BlockNumber = async (t: number, timeout = 60) => {
+    await retryUntil(
+      () => Promise.resolve(t === monitor.l2ProvenBlockNumber),
+      `Wait proven L2 block ${t}`,
+      timeout,
+      0.1,
+    );
+  };
+
+  const waitForNodeToSync = async (blockNumber: number, type: 'finalised' | 'historic') => {
+    const waitTime = ARCHIVER_POLL_INTERVAL + WORLD_STATE_BLOCK_CHECK_INTERVAL;
+    let synched = false;
+    while (!synched) {
+      await sleep(waitTime);
+      const syncState = await context.aztecNode.getWorldStateSyncStatus();
+      if (type === 'finalised') {
+        synched = syncState.finalisedBlockNumber >= blockNumber;
+      } else {
+        synched = syncState.oldestHistoricBlockNumber >= blockNumber;
+      }
+    }
+  };
+
+  const verifyHistoricBlock = async (blockNumber: L2BlockNumber, expectedSuccess: boolean) => {
+    const result = await context.aztecNode
+      .findBlockNumbersForIndexes(blockNumber, MerkleTreeId.NULLIFIER_TREE, [0n])
+      .then(_ => true)
+      .catch(_ => false);
+    expect(result).toBe(expectedSuccess);
+  };
+
+  it('successfully proves all epochs', async () => {
+    const targetProvenEpochs = 8;
+    const targetProvenBlockNumber = targetProvenEpochs * EPOCH_DURATION_IN_L2_SLOTS;
+
+    let provenBlockNumber = 0;
+    let epochNumber = 0;
+    while (provenBlockNumber < targetProvenBlockNumber) {
+      logger.info(`Waiting for the end of epoch ${epochNumber}`);
+      await waitUntilEpochStarts(epochNumber + 1);
+      const epochTargetBlockNumber = Number(await rollup.getBlockNumber());
+      logger.info(`Epoch ${epochNumber} ended with PENDING block number ${epochTargetBlockNumber}`);
+      await waitUntilL2BlockNumber(epochTargetBlockNumber);
+      provenBlockNumber = epochTargetBlockNumber;
+      logger.info(
+        `Reached PENDING L2 block ${epochTargetBlockNumber}, proving should now start, waiting for PROVEN block to reach ${provenBlockNumber}`,
+      );
+      await waitUntilProvenL2BlockNumber(provenBlockNumber, 120);
+      expect(Number(await rollup.getProvenBlockNumber())).toBe(provenBlockNumber);
+      logger.info(`Reached PROVEN block number ${provenBlockNumber}, epoch ${epochNumber} is now proven`);
+      epochNumber++;
+
+      // Verify the state syncs
+      await waitForNodeToSync(provenBlockNumber, 'finalised');
+      await verifyHistoricBlock(provenBlockNumber, true);
+      const expectedOldestHistoricBlock = provenBlockNumber - WORLD_STATE_BLOCK_HISTORY + 1;
+      const expectedBlockRemoved = expectedOldestHistoricBlock - 1;
+      await waitForNodeToSync(expectedOldestHistoricBlock, 'historic');
+      await verifyHistoricBlock(Math.max(expectedOldestHistoricBlock, 1), true);
+      if (expectedBlockRemoved > 0) {
+        await verifyHistoricBlock(expectedBlockRemoved, false);
+      }
+    }
+    logger.info('Test Succeeded');
+  });
+});

yarn-project/end-to-end/src/e2e_prover/e2e_prover_test.ts

@@ -280,7 +280,7 @@ export class FullProverTest {
       archiver: archiver as Archiver,
       blobSinkClient,
     });
-    this.proverNode.start();
+    await this.proverNode.start();
 
     this.logger.warn(`Proofs are now enabled`);
     return this;

yarn-project/end-to-end/src/fixtures/utils.ts

@@ -754,7 +754,7 @@ export async function createAndSyncProverNode(
     archiver: archiver as Archiver,
     l1TxUtils,
   });
-  proverNode.start();
+  await proverNode.start();
   return proverNode;
 }