feat: Validator deadline for reexecution

yarn-project/archiver/src/archiver/archiver.ts

@@ -274,7 +274,10 @@ export class Archiver implements ArchiveSource, Traceable {
     }
 
     if (initialRun) {
-      this.log.info(`Initial archiver sync to L1 block ${currentL1BlockNumber} complete.`);
+      this.log.info(`Initial archiver sync to L1 block ${currentL1BlockNumber} complete.`, {
+        l1BlockNumber: currentL1BlockNumber,
+        ...(await this.getL2Tips()),
+      });
     }
   }
 
@@ -364,6 +367,16 @@ export class Archiver implements ArchiveSource, Traceable {
 
     const updateProvenBlock = async () => {
       const localBlockForDestinationProvenBlockNumber = await this.getBlock(Number(provenBlockNumber));
+
+      // Sanity check. I've hit what seems to be a state where the proven block is set to a value greater than the latest
+      // synched block when requesting L2Tips from the archiver. This is the only place where the proven block is set.
+      const synched = await this.store.getSynchedL2BlockNumber();
+      if (localBlockForDestinationProvenBlockNumber && synched < localBlockForDestinationProvenBlockNumber?.number) {
+        this.log.error(
+          `Hit local block greater than last synched block: ${localBlockForDestinationProvenBlockNumber.number} > ${synched}`,
+        );
+      }
+
       if (
         localBlockForDestinationProvenBlockNumber &&
         provenArchive === localBlockForDestinationProvenBlockNumber.archive.root.toString()
@@ -798,11 +811,13 @@ export class Archiver implements ArchiveSource, Traceable {
     ] as const);
 
     if (latestBlockNumber > 0 && !latestBlockHeader) {
-      throw new Error('Failed to retrieve latest block header');
+      throw new Error(`Failed to retrieve latest block header for block ${latestBlockNumber}`);
     }
 
     if (provenBlockNumber > 0 && !provenBlockHeader) {
-      throw new Error('Failed to retrieve proven block header');
+      throw new Error(
+        `Failed to retrieve proven block header for block ${provenBlockNumber} (latest block is ${latestBlockNumber})`,
+      );
     }
 
     return {

yarn-project/aztec.js/src/utils/chain_monitor.ts

@@ -0,0 +1,75 @@
+import { type RollupContract } from '@aztec/ethereum';
+import { createLogger } from '@aztec/foundation/log';
+
+import { type PublicClient } from 'viem';
+
+/** Utility class that polls the chain on quick intervals and logs new L1 blocks, L2 blocks, and L2 proofs. */
+export class ChainMonitor {
+  private readonly l1Client: PublicClient;
+  private handle: NodeJS.Timeout | undefined;
+
+  /** Current L1 block number */
+  public l1BlockNumber!: number;
+  /** Current L2 block number */
+  public l2BlockNumber!: number;
+  /** Current L2 proven block number */
+  public l2ProvenBlockNumber!: number;
+
+  constructor(
+    private readonly rollup: RollupContract,
+    private logger = createLogger('aztecjs:utils:chain_monitor'),
+    private readonly intervalMs = 200,
+  ) {
+    this.l1Client = rollup.client;
+  }
+
+  start() {
+    if (this.handle) {
+      throw new Error('Chain monitor already started');
+    }
+    this.handle = setInterval(() => this.run(), this.intervalMs);
+  }
+
+  stop() {
+    if (this.handle) {
+      clearInterval(this.handle!);
+      this.handle = undefined;
+    }
+  }
+
+  async run() {
+    const newL1BlockNumber = Number(await this.l1Client.getBlockNumber({ cacheTime: 0 }));
+    if (this.l1BlockNumber === newL1BlockNumber) {
+      return;
+    }
+    this.l1BlockNumber = newL1BlockNumber;
+
+    const block = await this.l1Client.getBlock({ blockNumber: BigInt(newL1BlockNumber), includeTransactions: false });
+    const timestamp = block.timestamp;
+    const timestampString = new Date(Number(timestamp) * 1000).toTimeString().split(' ')[0];
+
+    let msg = `L1 block ${newL1BlockNumber} mined at ${timestampString}`;
+
+    const newL2BlockNumber = Number(await this.rollup.getBlockNumber());
+    if (this.l2BlockNumber !== newL2BlockNumber) {
+      const epochNumber = await this.rollup.getEpochNumber(BigInt(newL2BlockNumber));
+      msg += ` with new L2 block ${newL2BlockNumber} for epoch ${epochNumber}`;
+      this.l2BlockNumber = newL2BlockNumber;
+    }
+
+    const newL2ProvenBlockNumber = Number(await this.rollup.getProvenBlockNumber());
+    if (this.l2ProvenBlockNumber !== newL2ProvenBlockNumber) {
+      const epochNumber = await this.rollup.getEpochNumber(BigInt(newL2ProvenBlockNumber));
+      msg += ` with proof up to L2 block ${newL2ProvenBlockNumber} for epoch ${epochNumber}`;
+      this.l2ProvenBlockNumber = newL2ProvenBlockNumber;
+    }
+
+    this.logger.info(msg, {
+      l1Timestamp: timestamp,
+      l1BlockNumber: this.l1BlockNumber,
+      l2SlotNumber: await this.rollup.getSlotNumber(),
+      l2BlockNumber: this.l2BlockNumber,
+      l2ProvenBlockNumber: this.l2ProvenBlockNumber,
+    });
+  }
+}

yarn-project/aztec.js/src/utils/index.ts

@@ -8,3 +8,4 @@ export * from './node.js';
 export * from './anvil_test_watcher.js';
 export * from './field_compressed_string.js';
 export * from './portal_manager.js';
+export * from './chain_monitor.js';

yarn-project/end-to-end/package.json

@@ -54,6 +54,7 @@
     "@aztec/simulator": "workspace:^",
     "@aztec/telemetry-client": "workspace:^",
     "@aztec/types": "workspace:^",
+    "@aztec/validator-client": "workspace:^",
     "@aztec/world-state": "workspace:^",
     "@iarna/toml": "^2.2.5",
     "@jest/globals": "^29.5.0",

yarn-project/end-to-end/src/e2e_epochs.test.ts

@@ -1,4 +1,5 @@
 import { type Logger, getTimestampRangeForEpoch, retryUntil, sleep } from '@aztec/aztec.js';
+import { ChainMonitor } from '@aztec/aztec.js/utils';
 // eslint-disable-next-line no-restricted-imports
 import { type L1RollupConstants } from '@aztec/circuit-types';
 import { Proof } from '@aztec/circuits.js';
@@ -25,11 +26,7 @@ describe('e2e_epochs', () => {
   let logger: Logger;
   let proverDelayer: Delayer;
   let sequencerDelayer: Delayer;
-
-  let l2BlockNumber: number = 0;
-  let l2ProvenBlockNumber: number = 0;
-  let l1BlockNumber: number;
-  let handle: NodeJS.Timeout;
+  let monitor: ChainMonitor;
 
   const EPOCH_DURATION_IN_L2_SLOTS = 4;
   const L2_SLOT_DURATION_IN_L1_SLOTS = 2;
@@ -59,34 +56,8 @@ describe('e2e_epochs', () => {
     rollup = RollupContract.getFromConfig(context.config);
 
     // Loop that tracks L1 and L2 block numbers and logs whenever there's a new one.
-    // We could refactor this out to an utility if we want to use this in other tests.
-    handle = setInterval(async () => {
-      const newL1BlockNumber = Number(await l1Client.getBlockNumber({ cacheTime: 0 }));
-      if (l1BlockNumber === newL1BlockNumber) {
-        return;
-      }
-      const block = await l1Client.getBlock({ blockNumber: BigInt(newL1BlockNumber), includeTransactions: false });
-      const timestamp = block.timestamp;
-      l1BlockNumber = newL1BlockNumber;
-
-      let msg = `L1 block ${newL1BlockNumber} mined at ${timestamp}`;
-
-      const newL2BlockNumber = Number(await rollup.getBlockNumber());
-      if (l2BlockNumber !== newL2BlockNumber) {
-        const epochNumber = await rollup.getEpochNumber(BigInt(newL2BlockNumber));
-        msg += ` with new L2 block ${newL2BlockNumber} for epoch ${epochNumber}`;
-        l2BlockNumber = newL2BlockNumber;
-      }
-
-      const newL2ProvenBlockNumber = Number(await rollup.getProvenBlockNumber());
-
-      if (l2ProvenBlockNumber !== newL2ProvenBlockNumber) {
-        const epochNumber = await rollup.getEpochNumber(BigInt(newL2ProvenBlockNumber));
-        msg += ` with proof up to L2 block ${newL2ProvenBlockNumber} for epoch ${epochNumber}`;
-        l2ProvenBlockNumber = newL2ProvenBlockNumber;
-      }
-      logger.info(msg);
-    }, 200);
+    monitor = new ChainMonitor(rollup, logger);
+    monitor.start();
 
     proverDelayer = ((context.proverNode as TestProverNode).publisher as TestL1Publisher).delayer!;
     sequencerDelayer = ((context.sequencer as TestSequencerClient).sequencer.publisher as TestL1Publisher).delayer!;
@@ -106,8 +77,8 @@ describe('e2e_epochs', () => {
   });
 
   afterEach(async () => {
-    clearInterval(handle);
     jest.restoreAllMocks();
+    monitor.stop();
     await context.teardown();
   });
 
@@ -121,12 +92,12 @@ describe('e2e_epochs', () => {
 
   /** Waits until the given L2 block number is mined. */
   const waitUntilL2BlockNumber = async (target: number) => {
-    await retryUntil(() => Promise.resolve(target === l2BlockNumber), `Wait until L2 block ${target}`, 60, 0.1);
+    await retryUntil(() => Promise.resolve(target === monitor.l2BlockNumber), `Wait until L2 block ${target}`, 60, 0.1);
   };
 
   /** Waits until the given L2 block number is marked as proven. */
-  const waitUntilProvenL2BlockNumber = async (target: number) => {
-    await retryUntil(() => Promise.resolve(target === l2ProvenBlockNumber), `Wait proven L2 block ${target}`, 60, 0.1);
+  const waitUntilProvenL2BlockNumber = async (t: number) => {
+    await retryUntil(() => Promise.resolve(t === monitor.l2ProvenBlockNumber), `Wait proven L2 block ${t}`, 60, 0.1);
   };
 
   it('does not allow submitting proof after epoch end', async () => {
@@ -169,7 +140,7 @@ describe('e2e_epochs', () => {
     logger.info(`Starting epoch 1 after L2 block ${blockNumberAtEndOfEpoch0}`);
 
     await waitUntilProvenL2BlockNumber(blockNumberAtEndOfEpoch0);
-    expect(l2BlockNumber).toEqual(blockNumberAtEndOfEpoch0);
+    expect(monitor.l2BlockNumber).toEqual(blockNumberAtEndOfEpoch0);
     logger.info(`Test succeeded`);
   });
 
@@ -199,7 +170,7 @@ describe('e2e_epochs', () => {
     logger.info(`Starting epoch 2`);
 
     // No proof for epoch zero should have landed during epoch one
-    expect(l2ProvenBlockNumber).toEqual(0);
+    expect(monitor.l2ProvenBlockNumber).toEqual(0);
 
     // Wait until the prover job finalises (and a bit more) and check that it aborted and never attempted to submit a tx
     logger.info(`Awaiting finalise epoch`);

yarn-project/end-to-end/src/e2e_p2p/p2p_network.ts

@@ -1,7 +1,8 @@
 import { getSchnorrAccount } from '@aztec/accounts/schnorr';
 import { type AztecNodeConfig, type AztecNodeService } from '@aztec/aztec-node';
 import { type AccountWalletWithSecretKey } from '@aztec/aztec.js';
-import { L1TxUtils, getL1ContractsConfigEnvVars } from '@aztec/ethereum';
+import { ChainMonitor } from '@aztec/aztec.js/utils';
+import { L1TxUtils, RollupContract, getL1ContractsConfigEnvVars } from '@aztec/ethereum';
 import { EthCheatCodesWithState } from '@aztec/ethereum/test';
 import { type Logger, createLogger } from '@aztec/foundation/log';
 import { RollupAbi, TestERC20Abi } from '@aztec/l1-artifacts';
@@ -38,6 +39,7 @@ export class P2PNetworkTest {
   private baseAccount;
 
   public logger: Logger;
+  public monitor!: ChainMonitor;
 
   public ctx!: SubsystemsContext;
   public attesterPrivateKeys: `0x${string}`[] = [];
@@ -308,6 +310,9 @@ export class P2PNetworkTest {
         stallTimeMs: 1000,
       },
     );
+
+    this.monitor = new ChainMonitor(RollupContract.getFromL1ContractsValues(this.ctx.deployL1ContractsValues));
+    this.monitor.start();
   }
 
   async stopNodes(nodes: AztecNodeService[]) {
@@ -325,6 +330,7 @@ export class P2PNetworkTest {
   }
 
   async teardown() {
+    this.monitor.stop();
     await this.bootstrapNode.stop();
     await this.snapshotManager.teardown();
     if (this.cleanupInterval) {