From 68c5e2ee4d9b208f736a0f3624113a74b9f4111c Mon Sep 17 00:00:00 2001 From: Nazar Hussain Date: Fri, 30 Jun 2023 15:42:43 +0200 Subject: [PATCH 1/8] Fix the validator abort error --- packages/cli/src/cmds/validator/handler.ts | 106 +++++++++--------- .../cli/src/cmds/validator/signers/index.ts | 4 +- 2 files changed, 57 insertions(+), 53 deletions(-) diff --git a/packages/cli/src/cmds/validator/handler.ts b/packages/cli/src/cmds/validator/handler.ts index 69ead593d1e6..ec4b9694e936 100644 --- a/packages/cli/src/cmds/validator/handler.ts +++ b/packages/cli/src/cmds/validator/handler.ts @@ -16,6 +16,7 @@ import { MonitoringService, } from "@lodestar/beacon-node"; import {getNodeLogger} from "@lodestar/logger/node"; +import {isErrorAborted} from "@lodestar/utils"; import {getBeaconConfigFromArgs} from "../../config/index.js"; import {GlobalArgs} from "../../options/index.js"; import {YargsError, cleanOldLogFiles, getDefaultGraffiti, mkdir, parseLoggerArgs} from "../../util/index.js"; @@ -150,60 +151,63 @@ export async function validatorHandler(args: IValidatorCliArgs & GlobalArgs): Pr // This promise resolves once genesis is available. // It will wait for genesis, so this promise can be potentially very long - - const validator = await Validator.initializeFromBeaconNode( - { - db, - config, - slashingProtection, - api: args.beaconNodes, - logger, - processShutdownCallback, - signers, - abortController, - doppelgangerProtection, - afterBlockDelaySlotFraction: args.afterBlockDelaySlotFraction, - scAfterBlockDelaySlotFraction: args.scAfterBlockDelaySlotFraction, - disableAttestationGrouping: args.disableAttestationGrouping, - valProposerConfig, - distributed: args.distributed, - }, - metrics - ); - - onGracefulShutdownCbs.push(() => validator.close()); - - // Start keymanager API backend - // Only if keymanagerEnabled flag is set to true - if (args["keymanager"]) { - // if proposerSettingsFile provided disable the key proposerConfigWrite in keymanager - const proposerConfigWriteDisabled = args.proposerSettingsFile !== undefined; - if (proposerConfigWriteDisabled) { - logger.warn( - "Proposer data updates (feeRecipient/gasLimit etc) will not be available via Keymanager API as proposerSettingsFile has been set" - ); - } - - const keymanagerApi = new KeymanagerApi( - validator, - persistedKeysBackend, - abortController.signal, - proposerConfigWriteDisabled - ); - const keymanagerServer = new KeymanagerRestApiServer( + // During the wait time user can abort the process with Ctrl+C + // So we have to wrap this code with the try/catch + try { + const validator = await Validator.initializeFromBeaconNode( { - address: args["keymanager.address"], - port: args["keymanager.port"], - cors: args["keymanager.cors"], - isAuthEnabled: args["keymanager.authEnabled"], - headerLimit: args["keymanager.headerLimit"], - bodyLimit: args["keymanager.bodyLimit"], - tokenDir: dbPath, + db, + config, + slashingProtection, + api: args.beaconNodes, + logger, + processShutdownCallback, + signers, + abortController, + doppelgangerProtection, + afterBlockDelaySlotFraction: args.afterBlockDelaySlotFraction, + scAfterBlockDelaySlotFraction: args.scAfterBlockDelaySlotFraction, + disableAttestationGrouping: args.disableAttestationGrouping, + valProposerConfig, + distributed: args.distributed, }, - {config, logger, api: keymanagerApi, metrics: metrics ? metrics.keymanagerApiRest : null} + metrics ); - onGracefulShutdownCbs.push(() => keymanagerServer.close()); - await keymanagerServer.listen(); + onGracefulShutdownCbs.push(() => validator.close()); + // Start keymanager API backend + // Only if keymanagerEnabled flag is set to true + if (args["keymanager"]) { + // if proposerSettingsFile provided disable the key proposerConfigWrite in keymanager + const proposerConfigWriteDisabled = args.proposerSettingsFile !== undefined; + if (proposerConfigWriteDisabled) { + logger.warn( + "Proposer data updates (feeRecipient/gasLimit etc) will not be available via Keymanager API as proposerSettingsFile has been set" + ); + } + + const keymanagerApi = new KeymanagerApi( + validator, + persistedKeysBackend, + abortController.signal, + proposerConfigWriteDisabled + ); + const keymanagerServer = new KeymanagerRestApiServer( + { + address: args["keymanager.address"], + port: args["keymanager.port"], + cors: args["keymanager.cors"], + isAuthEnabled: args["keymanager.authEnabled"], + headerLimit: args["keymanager.headerLimit"], + bodyLimit: args["keymanager.bodyLimit"], + tokenDir: dbPath, + }, + {config, logger, api: keymanagerApi, metrics: metrics ? metrics.keymanagerApiRest : null} + ); + onGracefulShutdownCbs.push(() => keymanagerServer.close()); + await keymanagerServer.listen(); + } + } catch (err) { + if (!isErrorAborted(err)) throw err; } } diff --git a/packages/cli/src/cmds/validator/signers/index.ts b/packages/cli/src/cmds/validator/signers/index.ts index 57e726a3810f..ea654b8ba221 100644 --- a/packages/cli/src/cmds/validator/signers/index.ts +++ b/packages/cli/src/cmds/validator/signers/index.ts @@ -86,7 +86,7 @@ export async function getSignersFromArgs( const needle = showProgress({ total: keystoreDefinitions.length, frequencyMs: KEYSTORE_IMPORT_PROGRESS_MS, - signal: signal, + signal, progress: ({ratePerSec, percentage, current, total}) => { logger.info( `${percentage.toFixed(0)}% of keystores imported. current=${current} total=${total} rate=${( @@ -119,7 +119,7 @@ export async function getSignersFromArgs( const needle = showProgress({ total: keystoreDefinitions.length, frequencyMs: KEYSTORE_IMPORT_PROGRESS_MS, - signal: signal, + signal, progress: ({ratePerSec, percentage, current, total}) => { logger.info( `${percentage.toFixed(0)}% of local keystores imported. current=${current} total=${total} rate=${( From ba6b11664aae83d76e045533cebfad0f730bfe3a Mon Sep 17 00:00:00 2001 From: Nazar Hussain Date: Thu, 13 Jul 2023 18:07:47 +0200 Subject: [PATCH 2/8] Fix abort error --- packages/api/src/beacon/server/events.ts | 8 +++++++- packages/beacon-node/src/api/rest/base.ts | 5 +---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/api/src/beacon/server/events.ts b/packages/api/src/beacon/server/events.ts index ed0119dc8f3e..e92426235147 100644 --- a/packages/api/src/beacon/server/events.ts +++ b/packages/api/src/beacon/server/events.ts @@ -1,4 +1,5 @@ import {ChainForkConfig} from "@lodestar/config"; +import {ErrorAborted} from "@lodestar/utils"; import {Api, ReqTypes, routesData, getEventSerdes} from "../routes/events.js"; import {ServerRoutes} from "../../utils/server/index.js"; import {ServerApi} from "../../interfaces.js"; @@ -49,7 +50,12 @@ export function getRoutes(config: ChainForkConfig, api: ServerApi): ServerR // The client may disconnect and we need to clean the subscriptions. req.raw.once("close", () => resolve()); req.raw.once("end", () => resolve()); - req.raw.once("error", (err) => reject(err)); + req.raw.once("error", (err) => { + if ("code" in err && (err as unknown as {code: string}).code === "ECONNRESET") { + return reject(new ErrorAborted()); + } + return reject(err); + }); }); // api.eventstream will never stop, so no need to ever call `res.raw.end();` diff --git a/packages/beacon-node/src/api/rest/base.ts b/packages/beacon-node/src/api/rest/base.ts index fda16fffc8ce..57227be7956b 100644 --- a/packages/beacon-node/src/api/rest/base.ts +++ b/packages/beacon-node/src/api/rest/base.ts @@ -44,10 +44,7 @@ export class RestApiServer { private status = Status.Closed; - constructor( - private readonly opts: RestApiServerOpts, - modules: RestApiServerModules - ) { + constructor(private readonly opts: RestApiServerOpts, modules: RestApiServerModules) { // Apply opts defaults const {logger, metrics} = modules; From 11134a2d17e8a0156bb9439b24078c4ce2b8414f Mon Sep 17 00:00:00 2001 From: Nazar Hussain Date: Fri, 14 Jul 2023 11:01:25 +0200 Subject: [PATCH 3/8] Update the request handling for headers --- packages/cli/src/cmds/validator/handler.ts | 107 +++++++++++---------- 1 file changed, 55 insertions(+), 52 deletions(-) diff --git a/packages/cli/src/cmds/validator/handler.ts b/packages/cli/src/cmds/validator/handler.ts index ec4b9694e936..4b29ac52e272 100644 --- a/packages/cli/src/cmds/validator/handler.ts +++ b/packages/cli/src/cmds/validator/handler.ts @@ -153,61 +153,64 @@ export async function validatorHandler(args: IValidatorCliArgs & GlobalArgs): Pr // It will wait for genesis, so this promise can be potentially very long // During the wait time user can abort the process with Ctrl+C // So we have to wrap this code with the try/catch - try { - const validator = await Validator.initializeFromBeaconNode( + const validator = await Validator.initializeFromBeaconNode( + { + db, + config, + slashingProtection, + api: args.beaconNodes, + logger, + processShutdownCallback, + signers, + abortController, + doppelgangerProtection, + afterBlockDelaySlotFraction: args.afterBlockDelaySlotFraction, + scAfterBlockDelaySlotFraction: args.scAfterBlockDelaySlotFraction, + disableAttestationGrouping: args.disableAttestationGrouping, + valProposerConfig, + distributed: args.distributed, + }, + metrics + ).catch((err) => { + if (isErrorAborted(err)) { + throw new Error("Validator initialization aborted. Exiting."); + } else { + throw err; + } + }); + + onGracefulShutdownCbs.push(() => validator.close()); + // Start keymanager API backend + // Only if keymanagerEnabled flag is set to true + if (args["keymanager"]) { + // if proposerSettingsFile provided disable the key proposerConfigWrite in keymanager + const proposerConfigWriteDisabled = args.proposerSettingsFile !== undefined; + if (proposerConfigWriteDisabled) { + logger.warn( + "Proposer data updates (feeRecipient/gasLimit etc) will not be available via Keymanager API as proposerSettingsFile has been set" + ); + } + + const keymanagerApi = new KeymanagerApi( + validator, + persistedKeysBackend, + abortController.signal, + proposerConfigWriteDisabled + ); + const keymanagerServer = new KeymanagerRestApiServer( { - db, - config, - slashingProtection, - api: args.beaconNodes, - logger, - processShutdownCallback, - signers, - abortController, - doppelgangerProtection, - afterBlockDelaySlotFraction: args.afterBlockDelaySlotFraction, - scAfterBlockDelaySlotFraction: args.scAfterBlockDelaySlotFraction, - disableAttestationGrouping: args.disableAttestationGrouping, - valProposerConfig, - distributed: args.distributed, + address: args["keymanager.address"], + port: args["keymanager.port"], + cors: args["keymanager.cors"], + isAuthEnabled: args["keymanager.authEnabled"], + headerLimit: args["keymanager.headerLimit"], + bodyLimit: args["keymanager.bodyLimit"], + tokenDir: dbPath, }, - metrics + {config, logger, api: keymanagerApi, metrics: metrics ? metrics.keymanagerApiRest : null} ); - onGracefulShutdownCbs.push(() => validator.close()); - // Start keymanager API backend - // Only if keymanagerEnabled flag is set to true - if (args["keymanager"]) { - // if proposerSettingsFile provided disable the key proposerConfigWrite in keymanager - const proposerConfigWriteDisabled = args.proposerSettingsFile !== undefined; - if (proposerConfigWriteDisabled) { - logger.warn( - "Proposer data updates (feeRecipient/gasLimit etc) will not be available via Keymanager API as proposerSettingsFile has been set" - ); - } - - const keymanagerApi = new KeymanagerApi( - validator, - persistedKeysBackend, - abortController.signal, - proposerConfigWriteDisabled - ); - const keymanagerServer = new KeymanagerRestApiServer( - { - address: args["keymanager.address"], - port: args["keymanager.port"], - cors: args["keymanager.cors"], - isAuthEnabled: args["keymanager.authEnabled"], - headerLimit: args["keymanager.headerLimit"], - bodyLimit: args["keymanager.bodyLimit"], - tokenDir: dbPath, - }, - {config, logger, api: keymanagerApi, metrics: metrics ? metrics.keymanagerApiRest : null} - ); - onGracefulShutdownCbs.push(() => keymanagerServer.close()); - await keymanagerServer.listen(); - } - } catch (err) { - if (!isErrorAborted(err)) throw err; + onGracefulShutdownCbs.push(() => keymanagerServer.close()); + await keymanagerServer.listen(); } } From 476e7ea2892b565d5d100775bcbf4c1c99dcd1d8 Mon Sep 17 00:00:00 2001 From: Nazar Hussain Date: Fri, 14 Jul 2023 11:01:41 +0200 Subject: [PATCH 4/8] Update the request handling for headers --- packages/api/src/beacon/server/events.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/api/src/beacon/server/events.ts b/packages/api/src/beacon/server/events.ts index e92426235147..d35726a4f755 100644 --- a/packages/api/src/beacon/server/events.ts +++ b/packages/api/src/beacon/server/events.ts @@ -37,6 +37,9 @@ export function getRoutes(config: ChainForkConfig, api: ServerApi): ServerR await new Promise((resolve, reject) => { void api.eventstream(req.query.topics, controller.signal, (event) => { try { + // If the request is already aborted, we don't need to send any more events. + if (req.raw.destroyed) return; + const data = eventSerdes.toJson(event); res.raw.write(serializeSSEEvent({event: event.type, data})); } catch (e) { From e467e63ed00954670cb7f0bbe1649fa83cade7d8 Mon Sep 17 00:00:00 2001 From: Nazar Hussain Date: Fri, 14 Jul 2023 11:03:26 +0200 Subject: [PATCH 5/8] Improve the check on the error --- packages/api/src/beacon/server/events.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/beacon/server/events.ts b/packages/api/src/beacon/server/events.ts index d35726a4f755..0a20e57e908a 100644 --- a/packages/api/src/beacon/server/events.ts +++ b/packages/api/src/beacon/server/events.ts @@ -54,7 +54,7 @@ export function getRoutes(config: ChainForkConfig, api: ServerApi): ServerR req.raw.once("close", () => resolve()); req.raw.once("end", () => resolve()); req.raw.once("error", (err) => { - if ("code" in err && (err as unknown as {code: string}).code === "ECONNRESET") { + if ((err as unknown as {code: string}).code === "ECONNRESET") { return reject(new ErrorAborted()); } return reject(err); From 175fcb4ff6cdb5e68d9da9f10add8d6eb6e6d08c Mon Sep 17 00:00:00 2001 From: Nazar Hussain Date: Fri, 14 Jul 2023 11:10:53 +0200 Subject: [PATCH 6/8] Fix prittier error --- packages/beacon-node/src/api/rest/base.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/beacon-node/src/api/rest/base.ts b/packages/beacon-node/src/api/rest/base.ts index 57227be7956b..fda16fffc8ce 100644 --- a/packages/beacon-node/src/api/rest/base.ts +++ b/packages/beacon-node/src/api/rest/base.ts @@ -44,7 +44,10 @@ export class RestApiServer { private status = Status.Closed; - constructor(private readonly opts: RestApiServerOpts, modules: RestApiServerModules) { + constructor( + private readonly opts: RestApiServerOpts, + modules: RestApiServerModules + ) { // Apply opts defaults const {logger, metrics} = modules; From 7ce0c928f1a07c81e4ccd3b92cc50d0474ec3bf3 Mon Sep 17 00:00:00 2001 From: Nazar Hussain Date: Fri, 14 Jul 2023 13:04:06 +0200 Subject: [PATCH 7/8] Revert the changes for the validator handler --- packages/cli/src/cmds/validator/handler.ts | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/packages/cli/src/cmds/validator/handler.ts b/packages/cli/src/cmds/validator/handler.ts index 4b29ac52e272..b0642a9959aa 100644 --- a/packages/cli/src/cmds/validator/handler.ts +++ b/packages/cli/src/cmds/validator/handler.ts @@ -16,7 +16,6 @@ import { MonitoringService, } from "@lodestar/beacon-node"; import {getNodeLogger} from "@lodestar/logger/node"; -import {isErrorAborted} from "@lodestar/utils"; import {getBeaconConfigFromArgs} from "../../config/index.js"; import {GlobalArgs} from "../../options/index.js"; import {YargsError, cleanOldLogFiles, getDefaultGraffiti, mkdir, parseLoggerArgs} from "../../util/index.js"; @@ -151,8 +150,6 @@ export async function validatorHandler(args: IValidatorCliArgs & GlobalArgs): Pr // This promise resolves once genesis is available. // It will wait for genesis, so this promise can be potentially very long - // During the wait time user can abort the process with Ctrl+C - // So we have to wrap this code with the try/catch const validator = await Validator.initializeFromBeaconNode( { db, @@ -171,13 +168,7 @@ export async function validatorHandler(args: IValidatorCliArgs & GlobalArgs): Pr distributed: args.distributed, }, metrics - ).catch((err) => { - if (isErrorAborted(err)) { - throw new Error("Validator initialization aborted. Exiting."); - } else { - throw err; - } - }); + ); onGracefulShutdownCbs.push(() => validator.close()); // Start keymanager API backend From 513396f2a62ea3dec765e3d7ecea8d30bb971c86 Mon Sep 17 00:00:00 2001 From: Nazar Hussain Date: Fri, 14 Jul 2023 13:05:01 +0200 Subject: [PATCH 8/8] Revert the changes for the validator handler --- packages/cli/src/cmds/validator/handler.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/cli/src/cmds/validator/handler.ts b/packages/cli/src/cmds/validator/handler.ts index b0642a9959aa..69ead593d1e6 100644 --- a/packages/cli/src/cmds/validator/handler.ts +++ b/packages/cli/src/cmds/validator/handler.ts @@ -150,6 +150,7 @@ export async function validatorHandler(args: IValidatorCliArgs & GlobalArgs): Pr // This promise resolves once genesis is available. // It will wait for genesis, so this promise can be potentially very long + const validator = await Validator.initializeFromBeaconNode( { db, @@ -171,6 +172,7 @@ export async function validatorHandler(args: IValidatorCliArgs & GlobalArgs): Pr ); onGracefulShutdownCbs.push(() => validator.close()); + // Start keymanager API backend // Only if keymanagerEnabled flag is set to true if (args["keymanager"]) {