feat: warning log if primary beacon node is unhealthy (#6921)

* feat: add warning log if primary beacon node is unhealthy

* Fix api client stub type

* Improve http client stub

* Update all api client stubs

* Only sanitize primary node url once

packages/api/src/beacon/client/index.ts

@@ -23,7 +23,7 @@ type ClientModules = HttpClientModules & {
   httpClient?: IHttpClient;
 };
 
-export type ApiClient = {[K in keyof Endpoints]: ApiClientMethods<Endpoints[K]>};
+export type ApiClient = {[K in keyof Endpoints]: ApiClientMethods<Endpoints[K]>} & {httpClient: IHttpClient};
 
 /**
  * REST HTTP client for all routes
@@ -42,5 +42,6 @@ export function getClient(opts: HttpClientOptions, modules: ClientModules): ApiC
     node: node.getClient(config, httpClient),
     proof: proof.getClient(config, httpClient),
     validator: validator.getClient(config, httpClient),
+    httpClient,
   };
 }

packages/api/src/utils/client/httpClient.ts

@@ -50,6 +50,8 @@ export const defaultInit: Required<ExtraRequestInit> = {
 
 export interface IHttpClient {
   readonly baseUrl: string;
+  readonly urlsInits: UrlInitRequired[];
+  readonly urlsScore: number[];
 
   request<E extends Endpoint>(
     definition: RouteDefinitionExtra<E>,
@@ -71,14 +73,13 @@ export type HttpClientModules = {
 
 export class HttpClient implements IHttpClient {
   readonly urlsInits: UrlInitRequired[] = [];
+  readonly urlsScore: number[];
 
   private readonly signal: null | AbortSignal;
   private readonly fetch: typeof fetch;
   private readonly metrics: null | Metrics;
   private readonly logger: null | Logger;
 
-  private readonly urlsScore: number[];
-
   /**
    * Cache to keep track of routes per server that do not support SSZ. This cache will only be
    * populated if we receive a 415 error response from the server after sending a SSZ request body.

packages/beacon-node/test/utils/node/validator.ts

@@ -1,4 +1,5 @@
 import tmp from "tmp";
+import {vi} from "vitest";
 import type {SecretKey} from "@chainsafe/bls/types";
 import {LevelDbController} from "@lodestar/db";
 import {interopSecretKey} from "@lodestar/state-transition";
@@ -91,7 +92,7 @@ export async function getAndInitDevValidators({
 }
 
 export function getApiFromServerHandlers(api: BeaconApiMethods): ApiClient {
-  return mapValues(api, (apiModule) =>
+  const apiClient = mapValues(api, (apiModule) =>
     mapValues(apiModule, (api: (args: unknown, context: unknown) => PromiseLike<{data: unknown; meta: unknown}>) => {
       return async (args: unknown) => {
         try {
@@ -114,6 +115,15 @@ export function getApiFromServerHandlers(api: BeaconApiMethods): ApiClient {
       };
     })
   ) as ApiClient;
+
+  apiClient.httpClient = {
+    baseUrl: "",
+    request: vi.fn(),
+    urlsInits: [],
+    urlsScore: [],
+  };
+
+  return apiClient;
 }
 
 export function getNodeApiUrl(node: BeaconNode): string {

packages/validator/src/validator.ts

@@ -134,6 +134,21 @@ export class Validator {
       this.clock.start(this.controller.signal);
       this.chainHeaderTracker.start(this.controller.signal);
 
+      // Add notifier to warn user if primary node is unhealthy as there might
+      // not be any errors in the logs due to fallback nodes handling the requests
+      const {httpClient} = this.api;
+      if (httpClient.urlsInits.length > 1) {
+        const primaryNodeUrl = toSafePrintableUrl(httpClient.urlsInits[0].baseUrl);
+
+        this.clock?.runEveryEpoch(async () => {
+          // Only emit warning if URL score is 0 to prevent false positives
+          // if just a single request fails which might happen due to other reasons
+          if (httpClient.urlsScore[0] === 0) {
+            this.logger?.warn("Primary beacon node is unhealthy", {url: primaryNodeUrl});
+          }
+        });
+      }
+
       if (metrics) {
         this.db.setMetrics(metrics.db);
 

packages/validator/test/utils/apiStub.ts

@@ -1,7 +1,18 @@
 import {vi, Mocked} from "vitest";
-import {ApiClientMethods, ApiResponse, Endpoint, Endpoints, HttpStatusCode} from "@lodestar/api";
+import {ApiClientMethods, ApiResponse, Endpoint, Endpoints, HttpStatusCode, IHttpClient} from "@lodestar/api";
 
-export function getApiClientStub(): {[K in keyof Endpoints]: Mocked<ApiClientMethods<Endpoints[K]>>} {
+type ApiClientStub = {[K in keyof Endpoints]: Mocked<ApiClientMethods<Endpoints[K]>>} & {
+  httpClient: Mocked<IHttpClient>;
+};
+
+const httpClientStub: IHttpClient = {
+  baseUrl: "",
+  request: vi.fn(),
+  urlsInits: [],
+  urlsScore: [],
+};
+
+export function getApiClientStub(): ApiClientStub {
   return {
     beacon: {
       getStateValidators: vi.fn(),
@@ -25,7 +36,8 @@ export function getApiClientStub(): {[K in keyof Endpoints]: Mocked<ApiClientMet
       publishAggregateAndProofs: vi.fn(),
       submitBeaconCommitteeSelections: vi.fn(),
     },
-  } as unknown as {[K in keyof Endpoints]: Mocked<ApiClientMethods<Endpoints[K]>>};
+    httpClient: httpClientStub,
+  } as unknown as ApiClientStub;
 }
 
 export function mockApiResponse<T, M, E extends Endpoint<any, any, any, T, M>>({