chore: Handle exceptions in bot runner

yarn-project/aztec/terraform/bot/main.tf

@@ -99,6 +99,7 @@ resource "aws_ecs_task_definition" "aztec-bot" {
         { name = "BOT_NO_START", value = "true" },
         { name = "BOT_PXE_URL", value = "http://${var.DEPLOY_TAG}-aztec-pxe-1.local/${var.DEPLOY_TAG}/aztec-pxe-1/${var.API_KEY}" },
         { name = "BOT_TX_INTERVAL_SECONDS", value = 300 },
+        { name = "LOG_LEVEL", value = var.LOG_LEVEL },
         { name = "AZTEC_PORT", value = "80" },
         { name = "API_PREFIX", value = local.api_prefix },
       ]

yarn-project/aztec/terraform/bot/variables.tf

@@ -17,3 +17,8 @@ variable "BOT_API_KEY" {
 variable "BOT_PRIVATE_KEY" {
   type = string
 }
+
+variable "LOG_LEVEL" {
+  type    = string
+  default = "verbose"
+}

yarn-project/bot/src/bot.ts

@@ -34,12 +34,14 @@ export class Bot {
   }
 
   public async run() {
+    const logCtx = { runId: Date.now() * 1000 + Math.floor(Math.random() * 1000) };
     const { privateTransfersPerTx, publicTransfersPerTx, feePaymentMethod } = this.config;
     const { token, recipient, wallet } = this;
     const sender = wallet.getAddress();
 
     this.log.verbose(
       `Sending tx with ${feePaymentMethod} fee with ${privateTransfersPerTx} private and ${publicTransfersPerTx} public transfers`,
+      logCtx,
     );
 
     const calls: FunctionCall[] = [
@@ -52,11 +54,24 @@ export class Bot {
     const paymentMethod = feePaymentMethod === 'native' ? new NativeFeePaymentMethod(sender) : new NoFeePaymentMethod();
     const gasSettings = GasSettings.default();
     const opts: SendMethodOptions = { estimateGas: true, fee: { paymentMethod, gasSettings } };
-    const tx = new BatchCall(wallet, calls).send(opts);
-    this.log.verbose(`Sent tx ${tx.getTxHash()}`);
 
-    const receipt = await tx.wait();
-    this.log.info(`Tx ${receipt.txHash} mined in block ${receipt.blockNumber}`);
+    const batch = new BatchCall(wallet, calls);
+    this.log.verbose(`Creating batch execution request with ${calls.length} calls`, logCtx);
+    await batch.create(opts);
+
+    this.log.verbose(`Simulating transaction`, logCtx);
+    await batch.simulate();
+
+    this.log.verbose(`Proving transaction`, logCtx);
+    await batch.prove(opts);
+
+    this.log.verbose(`Sending tx`, logCtx);
+    const tx = batch.send(opts);
+
+    this.log.verbose(`Awaiting tx ${tx.getTxHash()} to be mined (timeout ${this.config.txMinedWaitSeconds}s)`, logCtx);
+    const receipt = await tx.wait({ timeout: this.config.txMinedWaitSeconds });
+
+    this.log.info(`Tx ${receipt.txHash} mined in block ${receipt.blockNumber}`, logCtx);
   }
 
   public async getBalances() {

yarn-project/bot/src/config.ts

@@ -20,6 +20,8 @@ export type BotConfig = {
   feePaymentMethod: 'native' | 'none';
   /** True to not automatically setup or start the bot on initialization. */
   noStart: boolean;
+  /** How long to wait for a tx to be mined before reporting an error. */
+  txMinedWaitSeconds: number;
 };
 
 export function getBotConfigFromEnv(): BotConfig {
@@ -32,6 +34,7 @@ export function getBotConfigFromEnv(): BotConfig {
     BOT_PRIVATE_TRANSFERS_PER_TX,
     BOT_PUBLIC_TRANSFERS_PER_TX,
     BOT_NO_START,
+    BOT_TX_MINED_WAIT_SECONDS,
   } = process.env;
   if (BOT_FEE_PAYMENT_METHOD && !['native', 'none'].includes(BOT_FEE_PAYMENT_METHOD)) {
     throw new Error(`Invalid bot fee payment method: ${BOT_FEE_PAYMENT_METHOD}`);
@@ -49,6 +52,7 @@ export function getBotConfigFromEnv(): BotConfig {
     publicTransfersPerTx: BOT_PUBLIC_TRANSFERS_PER_TX ? parseInt(BOT_PUBLIC_TRANSFERS_PER_TX) : undefined,
     feePaymentMethod: BOT_FEE_PAYMENT_METHOD ? (BOT_FEE_PAYMENT_METHOD as 'native' | 'none') : undefined,
     noStart: BOT_NO_START ? ['1', 'true'].includes(BOT_NO_START) : undefined,
+    txMinedWaitSeconds: BOT_TX_MINED_WAIT_SECONDS ? parseInt(BOT_TX_MINED_WAIT_SECONDS) : undefined,
   });
 }
 
@@ -63,6 +67,7 @@ export function getBotDefaultConfig(overrides: Partial<BotConfig> = {}): BotConf
     publicTransfersPerTx: 1,
     feePaymentMethod: 'none',
     noStart: false,
+    txMinedWaitSeconds: 180,
     ...compact(overrides),
   };
 }

yarn-project/bot/src/runner.ts

@@ -31,7 +31,7 @@ export class BotRunner {
     await this.setup();
     if (!this.interval) {
       this.log.info(`Starting bot with interval of ${this.config.txIntervalSeconds}s`);
-      this.interval = setInterval(() => this.run(), this.config.txIntervalSeconds * 1000);
+      this.interval = setInterval(() => this.#safeRun(), this.config.txIntervalSeconds * 1000);
     }
   }
 
@@ -80,14 +80,31 @@ export class BotRunner {
    */
   public async run() {
     if (!this.bot) {
+      this.log.error(`Trying to run with uninitialized bot`);
       throw new Error(`Bot is not initialized`);
     }
-    this.log.verbose(`Manually triggered bot run`);
-    const bot = await this.bot;
-    const promise = bot.run();
-    this.running.add(promise);
-    await promise;
-    this.running.delete(promise);
+
+    let bot;
+    try {
+      bot = await this.bot;
+    } catch (err) {
+      this.log.error(`Error awaiting bot set up: ${err}`);
+      throw err;
+    }
+
+    let promise;
+    try {
+      promise = bot.run();
+      this.running.add(promise);
+      await promise;
+    } catch (err) {
+      this.log.error(`Error running bot: ${err}`);
+      throw err;
+    } finally {
+      if (promise) {
+        this.running.delete(promise);
+      }
+    }
   }
 
   /** Returns the current configuration for the bot. */
@@ -96,7 +113,20 @@ export class BotRunner {
   }
 
   async #createBot() {
-    this.bot = Bot.create(this.config, { pxe: this.pxe });
-    await this.bot;
+    try {
+      this.bot = Bot.create(this.config, { pxe: this.pxe });
+      await this.bot;
+    } catch (err) {
+      this.log.error(`Error setting up bot: ${err}`);
+      throw err;
+    }
+  }
+
+  async #safeRun() {
+    try {
+      await this.run();
+    } catch (err) {
+      // Already logged in run()
+    }
   }
 }