diff --git a/src/commands/run.ts b/src/commands/run.ts index a03bedb..fa08059 100644 --- a/src/commands/run.ts +++ b/src/commands/run.ts @@ -9,7 +9,14 @@ import { ApiService } from "../utils/api"; */ export async function run(options: RunOptions) { const log = getLogger("commands:run"); - const { rpc, deploymentBlock, oneShot, disableApi, apiPort } = options; + const { + rpc, + deploymentBlock, + oneShot, + disableApi, + apiPort, + watchdogTimeout, + } = options; // Start the API server if it's not disabled if (!disableApi) { @@ -45,7 +52,7 @@ export async function run(options: RunOptions) { // Run the block watcher after warm up for each chain const runPromises = chainContexts.map(async (context) => { - return context.warmUp(oneShot); + return context.warmUp(watchdogTimeout, oneShot); }); // Run all the chain contexts diff --git a/src/domain/chainContext.ts b/src/domain/chainContext.ts index 5199a19..de084dc 100644 --- a/src/domain/chainContext.ts +++ b/src/domain/chainContext.ts @@ -15,7 +15,6 @@ import { composableCowContract, DBService, getLogger } from "../utils"; import { MetricsService } from "../utils/metrics"; const WATCHDOG_FREQUENCY = 5 * 1000; // 5 seconds -const WATCHDOG_KILL_THRESHOLD = 30 * 1000; // 30 seconds const MULTICALL3 = "0xcA11bde05977b3631167028862bE2a173976CA11"; @@ -92,10 +91,11 @@ export class ChainContext { /** * Warm up the chain watcher by fetching the latest block number and * checking if the chain is in sync. + * @param watchdogTimeout the timeout for the watchdog * @param oneShot if true, only warm up the chain watcher and return * @returns the run promises for what needs to be watched */ - public async warmUp(oneShot?: boolean) { + public async warmUp(watchdogTimeout: number, oneShot?: boolean) { const { provider, chainId } = this; const log = getLogger(`chainContext:warmUp:${chainId}`); const { lastProcessedBlock } = this.registry; @@ -225,7 +225,7 @@ export class ChainContext { } // Otherwise, run the block watcher - return await this.runBlockWatcher(); + return await this.runBlockWatcher(watchdogTimeout); } /** @@ -233,11 +233,12 @@ export class ChainContext { * 1. Check if there are any `ConditionalOrderCreated` events, and index these. * 2. Check if any orders want to create discrete orders. */ - private async runBlockWatcher() { + private async runBlockWatcher(watchdogTimeout: number) { const { provider, registry, chainId } = this; const log = getLogger(`chainContext:runBlockWatcher:${chainId}`); // Watch for new blocks - log.info("👀 Start block watcher"); + log.info(`👀 Start block watcher`); + log.debug(`Watchdog timeout: ${watchdogTimeout} seconds`); let lastBlockReceived = 0; let timeLastBlockProcessed = new Date().getTime(); provider.on("block", async (blockNumber: number) => { @@ -294,8 +295,10 @@ export class ChainContext { log.debug(`Time since last block processed: ${timeElapsed}ms`); // If we haven't received a block in 30 seconds, exit so that the process manager can restart us - if (timeElapsed >= WATCHDOG_KILL_THRESHOLD) { - log.error(`Watchdog timeout`); + if (timeElapsed >= watchdogTimeout * 1000) { + log.error( + `Watchdog timeout (RPC failed, or chain is stuck / not issuing blocks)` + ); await registry.storage.close(); process.exit(1); } diff --git a/src/index.ts b/src/index.ts index 5d73313..37b3382 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,10 @@ import "dotenv/config"; -import { program, Option } from "@commander-js/extra-typings"; +import { + program, + Option, + InvalidArgumentError, +} from "@commander-js/extra-typings"; import { ReplayTxOptions } from "./types"; import { dumpDb, replayBlock, replayTx, run } from "./commands"; import { initLogging } from "./utils"; @@ -25,10 +29,10 @@ async function main() { "--deployment-block ", "Block number at which the contracts were deployed" ) - .option( - "--page-size ", - "Number of blocks to fetch per page", - "5000" + .addOption( + new Option("--page-size ", "Number of blocks to fetch per page") + .default("5000") + .argParser(parseIntOption) ) .option("--dry-run", "Do not publish orders to the OrderBook API", false) .addOption( @@ -37,19 +41,32 @@ async function main() { .default(false) ) .option("--disable-api", "Disable the REST API", false) - .option("--api-port ", "Port for the REST API", "8080") + .addOption( + new Option("--api-port ", "Port for the REST API") + .default("8080") + .argParser(parseIntOption) + ) .option("--slack-webhook ", "Slack webhook URL") .option("--one-shot", "Run the watchtower once and exit", false) + .addOption( + new Option( + "--watchdog-timeout ", + "Watchdog timeout (in seconds)" + ) + .default("30") + .argParser(parseIntOption) + ) .addOption(logLevelOption) .action((options) => { const { logLevel } = options; + const [pageSize, apiPort, watchdogTimeout] = [ + options.pageSize, + options.apiPort, + options.watchdogTimeout, + ].map((value) => Number(value)); initLogging({ logLevel }); - const { - rpc, - deploymentBlock: deploymentBlockEnv, - pageSize: pageSizeEnv, - } = options; + const { rpc, deploymentBlock: deploymentBlockEnv } = options; // Ensure that the deployment blocks are all numbers const deploymentBlock = deploymentBlockEnv.map((block) => Number(block)); @@ -57,25 +74,13 @@ async function main() { throw new Error("Deployment blocks must be numbers"); } - // Ensure that pageSize is a number - const pageSize = Number(pageSizeEnv); - if (isNaN(pageSize)) { - throw new Error("Page size must be a number"); - } - - // Ensure that the port is a number - const apiPort = Number(options.apiPort); - if (isNaN(apiPort)) { - throw new Error("API port must be a number"); - } - // Ensure that the RPCs and deployment blocks are the same length if (rpc.length !== deploymentBlock.length) { throw new Error("RPC and deployment blocks must be the same length"); } // Run the watchtower - run({ ...options, deploymentBlock, pageSize, apiPort }); + run({ ...options, deploymentBlock, pageSize, apiPort, watchdogTimeout }); }); program @@ -134,6 +139,14 @@ async function main() { await program.parseAsync(); } +function parseIntOption(option: string) { + const parsed = Number(option); + if (isNaN(parsed)) { + throw new InvalidArgumentError(`${option} must be a number`); + } + return parsed.toString(); +} + main().catch((error) => { console.error(error); process.exit(1); diff --git a/src/types/index.ts b/src/types/index.ts index be090a1..a5e5d27 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -19,6 +19,7 @@ export interface RunOptions extends WatchtowerOptions { oneShot: boolean; disableApi: boolean; apiPort: number; + watchdogTimeout: number; } export type SingularRunOptions = Omit & {