From 5bb4d4f0c47a24e5411915d95042035edfe2281b Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Sat, 10 Jun 2023 15:44:33 +0700 Subject: [PATCH 1/7] feat: write network thread profile --- packages/api/src/beacon/routes/lodestar.ts | 6 ++++ .../src/api/impl/lodestar/index.ts | 5 ++++ .../src/network/core/networkCore.ts | 4 +++ .../src/network/core/networkCoreWorker.ts | 28 +++++++++++++++++++ .../network/core/networkCoreWorkerHandler.ts | 3 ++ .../beacon-node/src/network/core/types.ts | 2 ++ packages/beacon-node/src/network/interface.ts | 1 + packages/beacon-node/src/network/network.ts | 4 +++ 8 files changed, 53 insertions(+) diff --git a/packages/api/src/beacon/routes/lodestar.ts b/packages/api/src/beacon/routes/lodestar.ts index 4ca8523b6ab..85033e8036e 100644 --- a/packages/api/src/beacon/routes/lodestar.ts +++ b/packages/api/src/beacon/routes/lodestar.ts @@ -77,6 +77,8 @@ export type LodestarNodePeer = NodePeer & { export type Api = { /** Trigger to write a heapdump to disk at `dirpath`. May take > 1min */ writeHeapdump(dirpath?: string): Promise>; + /** Trigger to write 10m profile file of network thread to disk */ + writeNetworkThreadProfile(): Promise>; /** TODO: description */ getLatestWeakSubjectivityCheckpointEpoch(): Promise>; /** TODO: description */ @@ -125,6 +127,7 @@ export type Api = { */ export const routesData: RoutesData = { writeHeapdump: {url: "/eth/v1/lodestar/writeheapdump", method: "POST"}, + writeNetworkThreadProfile: {url: "/eth/v1/lodestar/writenetworkthreadprofile", method: "POST"}, getLatestWeakSubjectivityCheckpointEpoch: {url: "/eth/v1/lodestar/ws_epoch", method: "GET"}, getSyncChainsDebugState: {url: "/eth/v1/lodestar/sync-chains-debug-state", method: "GET"}, getGossipQueueItems: {url: "/eth/v1/lodestar/gossip-queue-items/:gossipType", method: "GET"}, @@ -145,6 +148,7 @@ export const routesData: RoutesData = { export type ReqTypes = { writeHeapdump: {query: {dirpath?: string}}; + writeNetworkThreadProfile: ReqEmpty; getLatestWeakSubjectivityCheckpointEpoch: ReqEmpty; getSyncChainsDebugState: ReqEmpty; getGossipQueueItems: {params: {gossipType: string}}; @@ -170,6 +174,7 @@ export function getReqSerializers(): ReqSerializers { parseReq: ({query}) => [query.dirpath], schema: {query: {dirpath: Schema.String}}, }, + writeNetworkThreadProfile: reqEmpty, getLatestWeakSubjectivityCheckpointEpoch: reqEmpty, getSyncChainsDebugState: reqEmpty, getGossipQueueItems: { @@ -212,6 +217,7 @@ export function getReqSerializers(): ReqSerializers { export function getReturnTypes(): ReturnTypes { return { writeHeapdump: sameType(), + writeNetworkThreadProfile: sameType(), getLatestWeakSubjectivityCheckpointEpoch: sameType(), getSyncChainsDebugState: jsonType("snake"), getGossipQueueItems: jsonType("snake"), diff --git a/packages/beacon-node/src/api/impl/lodestar/index.ts b/packages/beacon-node/src/api/impl/lodestar/index.ts index 4da8b723f2a..9bf368ac509 100644 --- a/packages/beacon-node/src/api/impl/lodestar/index.ts +++ b/packages/beacon-node/src/api/impl/lodestar/index.ts @@ -47,6 +47,11 @@ export function getLodestarApi({ } }, + async writeNetworkThreadProfile() { + const filepath = await network.takeProfile(); + return {data: {filepath}}; + }, + async getLatestWeakSubjectivityCheckpointEpoch() { const state = chain.getHeadState(); return {data: getLatestWeakSubjectivityCheckpointEpoch(config, state)}; diff --git a/packages/beacon-node/src/network/core/networkCore.ts b/packages/beacon-node/src/network/core/networkCore.ts index bba28b01518..a63ebc346a8 100644 --- a/packages/beacon-node/src/network/core/networkCore.ts +++ b/packages/beacon-node/src/network/core/networkCore.ts @@ -412,6 +412,10 @@ export class NetworkCore implements INetworkCore { return meshPeers; } + async takeProfile(): Promise { + throw new Error("Method not implemented, please configure network thread"); + } + /** * Handle subscriptions through fork transitions, @see FORK_EPOCH_LOOKAHEAD */ diff --git a/packages/beacon-node/src/network/core/networkCoreWorker.ts b/packages/beacon-node/src/network/core/networkCoreWorker.ts index 3d1dca5a371..daa7d563c31 100644 --- a/packages/beacon-node/src/network/core/networkCoreWorker.ts +++ b/packages/beacon-node/src/network/core/networkCoreWorker.ts @@ -1,9 +1,12 @@ import worker from "node:worker_threads"; +import inspector from "node:inspector"; +import fs from "node:fs"; import {createFromProtobuf} from "@libp2p/peer-id-factory"; import {expose} from "@chainsafe/threads/worker"; import {chainConfigFromJson, createBeaconConfig} from "@lodestar/config"; import {getNodeLogger} from "@lodestar/logger/node"; import type {WorkerModule} from "@chainsafe/threads/dist/types/worker.js"; +import {sleep} from "@lodestar/utils"; import {collectNodeJSMetrics, RegistryMetricCreator} from "../../metrics/index.js"; import {AsyncIterableBridgeCaller, AsyncIterableBridgeHandler} from "../../util/asyncIterableToEvents.js"; import {Clock} from "../../util/clock.js"; @@ -147,6 +150,31 @@ const libp2pWorkerApi: NetworkWorkerApi = { dumpGossipPeerScoreStats: () => core.dumpGossipPeerScoreStats(), dumpDiscv5KadValues: () => core.dumpDiscv5KadValues(), dumpMeshPeers: () => core.dumpMeshPeers(), + takeProfile: () => { + return new Promise((resolve, reject) => { + // Start the inspector and connect to it + const session = new inspector.Session(); + session.connect(); + + // Enable the Profiler + session.post("Profiler.enable", () => { + // Start Profiler + session.post("Profiler.start", async () => { + await sleep(10 * 60 * 1000); + session.post("Profiler.stop", (err, {profile}) => { + // Write profile to disk, upload, etc. + if (!err) { + const filePath = `network_thread_${new Date().toISOString()}.profile`; + fs.writeFileSync(filePath, JSON.stringify(profile)); + resolve(filePath); + } else { + reject(err); + } + }); + }); + }); + }); + }, }; expose(libp2pWorkerApi as WorkerModule); diff --git a/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts b/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts index acfd60603e8..49f6067c0bf 100644 --- a/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts +++ b/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts @@ -210,6 +210,9 @@ export class WorkerNetworkCore implements INetworkCore { dumpMeshPeers(): Promise> { return this.getApi().dumpMeshPeers(); } + takeProfile(): Promise { + return this.getApi().takeProfile(); + } private getApi(): NetworkWorkerApi { return this.modules.workerApi; diff --git a/packages/beacon-node/src/network/core/types.ts b/packages/beacon-node/src/network/core/types.ts index 15e77ea2026..a346b7cb76a 100644 --- a/packages/beacon-node/src/network/core/types.ts +++ b/packages/beacon-node/src/network/core/types.ts @@ -61,6 +61,7 @@ export interface INetworkCore extends INetworkCorePublic { close(): Promise; scrapeMetrics(): Promise; + takeProfile(): Promise; } /** @@ -99,6 +100,7 @@ export type NetworkWorkerApi = INetworkCorePublic & { close(): Promise; scrapeMetrics(): Promise; + takeProfile(): Promise; // TODO: ReqResp outgoing // TODO: ReqResp incoming diff --git a/packages/beacon-node/src/network/interface.ts b/packages/beacon-node/src/network/interface.ts index 3a0d2b00586..c7ac1f8468f 100644 --- a/packages/beacon-node/src/network/interface.ts +++ b/packages/beacon-node/src/network/interface.ts @@ -60,6 +60,7 @@ export interface INetwork extends INetworkCorePublic { // Debug dumpGossipQueue(gossipType: GossipType): Promise; + takeProfile(): Promise; } export type PeerDirection = Connection["stat"]["direction"]; diff --git a/packages/beacon-node/src/network/network.ts b/packages/beacon-node/src/network/network.ts index 4a9c2bcd24f..b4a0571352c 100644 --- a/packages/beacon-node/src/network/network.ts +++ b/packages/beacon-node/src/network/network.ts @@ -544,6 +544,10 @@ export class Network implements INetwork { return this.networkProcessor.dumpGossipQueue(gossipType); } + async takeProfile(): Promise { + return this.core.takeProfile(); + } + private onLightClientFinalityUpdate = async (finalityUpdate: allForks.LightClientFinalityUpdate): Promise => { // TODO: Review is OK to remove if (this.hasAttachedSyncCommitteeMember()) From b172d05590d7a444e692ac771bc67417083aabd4 Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Sat, 10 Jun 2023 17:18:06 +0700 Subject: [PATCH 2/7] fix: persist .cpuprofile file --- packages/beacon-node/src/network/core/networkCoreWorker.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/beacon-node/src/network/core/networkCoreWorker.ts b/packages/beacon-node/src/network/core/networkCoreWorker.ts index daa7d563c31..672b606d00f 100644 --- a/packages/beacon-node/src/network/core/networkCoreWorker.ts +++ b/packages/beacon-node/src/network/core/networkCoreWorker.ts @@ -164,7 +164,7 @@ const libp2pWorkerApi: NetworkWorkerApi = { session.post("Profiler.stop", (err, {profile}) => { // Write profile to disk, upload, etc. if (!err) { - const filePath = `network_thread_${new Date().toISOString()}.profile`; + const filePath = `network_thread_${new Date().toISOString()}.cpuprofile`; fs.writeFileSync(filePath, JSON.stringify(profile)); resolve(filePath); } else { From ff1e500afa330ab2646b03edeefceb2be3988950 Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Sun, 11 Jun 2023 06:09:45 +0700 Subject: [PATCH 3/7] fix: disconnect inspector session --- packages/beacon-node/src/network/core/networkCoreWorker.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/beacon-node/src/network/core/networkCoreWorker.ts b/packages/beacon-node/src/network/core/networkCoreWorker.ts index 672b606d00f..0990f0816fd 100644 --- a/packages/beacon-node/src/network/core/networkCoreWorker.ts +++ b/packages/beacon-node/src/network/core/networkCoreWorker.ts @@ -170,6 +170,10 @@ const libp2pWorkerApi: NetworkWorkerApi = { } else { reject(err); } + + // Detach from the inspector and close the session + session.post("Profiler.disable"); + session.disconnect(); }); }); }); From bb89ac1e286200da0ee5a5204cc0553930217cdc Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Mon, 12 Jun 2023 09:45:44 +0700 Subject: [PATCH 4/7] feat: support dirpath and refactor --- packages/api/src/beacon/routes/lodestar.ts | 12 ++++--- .../src/api/impl/lodestar/index.ts | 15 ++++++-- .../src/network/core/networkCore.ts | 2 +- .../src/network/core/networkCoreWorker.ts | 36 ++++--------------- .../network/core/networkCoreWorkerHandler.ts | 4 +-- .../beacon-node/src/network/core/types.ts | 4 +-- packages/beacon-node/src/network/interface.ts | 2 +- packages/beacon-node/src/network/network.ts | 4 +-- packages/beacon-node/src/util/profile.ts | 34 ++++++++++++++++++ .../onWorker/dataSerialization.test.ts | 2 ++ 10 files changed, 70 insertions(+), 45 deletions(-) create mode 100644 packages/beacon-node/src/util/profile.ts diff --git a/packages/api/src/beacon/routes/lodestar.ts b/packages/api/src/beacon/routes/lodestar.ts index 85033e8036e..2f973bc132e 100644 --- a/packages/api/src/beacon/routes/lodestar.ts +++ b/packages/api/src/beacon/routes/lodestar.ts @@ -77,8 +77,8 @@ export type LodestarNodePeer = NodePeer & { export type Api = { /** Trigger to write a heapdump to disk at `dirpath`. May take > 1min */ writeHeapdump(dirpath?: string): Promise>; - /** Trigger to write 10m profile file of network thread to disk */ - writeNetworkThreadProfile(): Promise>; + /** Trigger to write 10m network thread profile to disk */ + writeNetworkThreadProfile(dirpath?: string): Promise>; /** TODO: description */ getLatestWeakSubjectivityCheckpointEpoch(): Promise>; /** TODO: description */ @@ -148,7 +148,7 @@ export const routesData: RoutesData = { export type ReqTypes = { writeHeapdump: {query: {dirpath?: string}}; - writeNetworkThreadProfile: ReqEmpty; + writeNetworkThreadProfile: {query: {dirpath?: string}}; getLatestWeakSubjectivityCheckpointEpoch: ReqEmpty; getSyncChainsDebugState: ReqEmpty; getGossipQueueItems: {params: {gossipType: string}}; @@ -174,7 +174,11 @@ export function getReqSerializers(): ReqSerializers { parseReq: ({query}) => [query.dirpath], schema: {query: {dirpath: Schema.String}}, }, - writeNetworkThreadProfile: reqEmpty, + writeNetworkThreadProfile: { + writeReq: (dirpath) => ({query: {dirpath}}), + parseReq: ({query}) => [query.dirpath], + schema: {query: {dirpath: Schema.String}}, + }, getLatestWeakSubjectivityCheckpointEpoch: reqEmpty, getSyncChainsDebugState: reqEmpty, getGossipQueueItems: { diff --git a/packages/beacon-node/src/api/impl/lodestar/index.ts b/packages/beacon-node/src/api/impl/lodestar/index.ts index 9bf368ac509..80c4a342277 100644 --- a/packages/beacon-node/src/api/impl/lodestar/index.ts +++ b/packages/beacon-node/src/api/impl/lodestar/index.ts @@ -19,6 +19,7 @@ export function getLodestarApi({ sync, }: Pick): ServerApi { let writingHeapdump = false; + let writingNetworkProfile = false; return { async writeHeapdump(dirpath = ".") { @@ -47,9 +48,17 @@ export function getLodestarApi({ } }, - async writeNetworkThreadProfile() { - const filepath = await network.takeProfile(); - return {data: {filepath}}; + async writeNetworkThreadProfile(dirpath = ".") { + if (writingNetworkProfile) { + throw Error("Already writing network profile"); + } + + try { + const filepath = await network.writeNetworkThreadProfile(dirpath); + return {data: {filepath}}; + } finally { + writingNetworkProfile = false; + } }, async getLatestWeakSubjectivityCheckpointEpoch() { diff --git a/packages/beacon-node/src/network/core/networkCore.ts b/packages/beacon-node/src/network/core/networkCore.ts index a63ebc346a8..f69251bd303 100644 --- a/packages/beacon-node/src/network/core/networkCore.ts +++ b/packages/beacon-node/src/network/core/networkCore.ts @@ -412,7 +412,7 @@ export class NetworkCore implements INetworkCore { return meshPeers; } - async takeProfile(): Promise { + async writeNetworkThreadProfile(): Promise { throw new Error("Method not implemented, please configure network thread"); } diff --git a/packages/beacon-node/src/network/core/networkCoreWorker.ts b/packages/beacon-node/src/network/core/networkCoreWorker.ts index 0990f0816fd..9db9dfccd0b 100644 --- a/packages/beacon-node/src/network/core/networkCoreWorker.ts +++ b/packages/beacon-node/src/network/core/networkCoreWorker.ts @@ -1,18 +1,17 @@ import worker from "node:worker_threads"; -import inspector from "node:inspector"; import fs from "node:fs"; import {createFromProtobuf} from "@libp2p/peer-id-factory"; import {expose} from "@chainsafe/threads/worker"; import {chainConfigFromJson, createBeaconConfig} from "@lodestar/config"; import {getNodeLogger} from "@lodestar/logger/node"; import type {WorkerModule} from "@chainsafe/threads/dist/types/worker.js"; -import {sleep} from "@lodestar/utils"; import {collectNodeJSMetrics, RegistryMetricCreator} from "../../metrics/index.js"; import {AsyncIterableBridgeCaller, AsyncIterableBridgeHandler} from "../../util/asyncIterableToEvents.js"; import {Clock} from "../../util/clock.js"; import {wireEventsOnWorkerThread} from "../../util/workerEvents.js"; import {NetworkEventBus, NetworkEventData, networkEventDirection} from "../events.js"; import {peerIdToString} from "../../util/peerId.js"; +import {profileNodeJS} from "../../util/profile.js"; import {getNetworkCoreWorkerMetrics} from "./metrics.js"; import {NetworkWorkerApi, NetworkWorkerData} from "./types.js"; import {NetworkCore} from "./networkCore.js"; @@ -150,34 +149,11 @@ const libp2pWorkerApi: NetworkWorkerApi = { dumpGossipPeerScoreStats: () => core.dumpGossipPeerScoreStats(), dumpDiscv5KadValues: () => core.dumpDiscv5KadValues(), dumpMeshPeers: () => core.dumpMeshPeers(), - takeProfile: () => { - return new Promise((resolve, reject) => { - // Start the inspector and connect to it - const session = new inspector.Session(); - session.connect(); - - // Enable the Profiler - session.post("Profiler.enable", () => { - // Start Profiler - session.post("Profiler.start", async () => { - await sleep(10 * 60 * 1000); - session.post("Profiler.stop", (err, {profile}) => { - // Write profile to disk, upload, etc. - if (!err) { - const filePath = `network_thread_${new Date().toISOString()}.cpuprofile`; - fs.writeFileSync(filePath, JSON.stringify(profile)); - resolve(filePath); - } else { - reject(err); - } - - // Detach from the inspector and close the session - session.post("Profiler.disable"); - session.disconnect(); - }); - }); - }); - }); + writeProfile: async (dirpath = ".") => { + const profile = await profileNodeJS(); + const filePath = `${dirpath}/network_thread_${new Date().toISOString()}.cpuprofile`; + fs.writeFileSync(filePath, profile); + return filePath; }, }; diff --git a/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts b/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts index 49f6067c0bf..8cf6cb7248d 100644 --- a/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts +++ b/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts @@ -210,8 +210,8 @@ export class WorkerNetworkCore implements INetworkCore { dumpMeshPeers(): Promise> { return this.getApi().dumpMeshPeers(); } - takeProfile(): Promise { - return this.getApi().takeProfile(); + writeNetworkThreadProfile(dirpath?: string): Promise { + return this.getApi().writeProfile(dirpath); } private getApi(): NetworkWorkerApi { diff --git a/packages/beacon-node/src/network/core/types.ts b/packages/beacon-node/src/network/core/types.ts index a346b7cb76a..cea5e50b0d3 100644 --- a/packages/beacon-node/src/network/core/types.ts +++ b/packages/beacon-node/src/network/core/types.ts @@ -61,7 +61,7 @@ export interface INetworkCore extends INetworkCorePublic { close(): Promise; scrapeMetrics(): Promise; - takeProfile(): Promise; + writeNetworkThreadProfile(dirpath?: string): Promise; } /** @@ -100,7 +100,7 @@ export type NetworkWorkerApi = INetworkCorePublic & { close(): Promise; scrapeMetrics(): Promise; - takeProfile(): Promise; + writeProfile(dirpath?: string): Promise; // TODO: ReqResp outgoing // TODO: ReqResp incoming diff --git a/packages/beacon-node/src/network/interface.ts b/packages/beacon-node/src/network/interface.ts index c7ac1f8468f..b06211c77f5 100644 --- a/packages/beacon-node/src/network/interface.ts +++ b/packages/beacon-node/src/network/interface.ts @@ -60,7 +60,7 @@ export interface INetwork extends INetworkCorePublic { // Debug dumpGossipQueue(gossipType: GossipType): Promise; - takeProfile(): Promise; + writeNetworkThreadProfile(dirpath?: string): Promise; } export type PeerDirection = Connection["stat"]["direction"]; diff --git a/packages/beacon-node/src/network/network.ts b/packages/beacon-node/src/network/network.ts index b4a0571352c..90feb82b1eb 100644 --- a/packages/beacon-node/src/network/network.ts +++ b/packages/beacon-node/src/network/network.ts @@ -544,8 +544,8 @@ export class Network implements INetwork { return this.networkProcessor.dumpGossipQueue(gossipType); } - async takeProfile(): Promise { - return this.core.takeProfile(); + async writeNetworkThreadProfile(dirpath = "."): Promise { + return this.core.writeNetworkThreadProfile(dirpath); } private onLightClientFinalityUpdate = async (finalityUpdate: allForks.LightClientFinalityUpdate): Promise => { diff --git a/packages/beacon-node/src/util/profile.ts b/packages/beacon-node/src/util/profile.ts new file mode 100644 index 00000000000..e78147d4fb3 --- /dev/null +++ b/packages/beacon-node/src/util/profile.ts @@ -0,0 +1,34 @@ +import {sleep} from "@lodestar/utils"; + +const DEFAULT_PROFILE_DURATION = 10 * 60 * 1000; + +/** + * Take 10m profile of the current thread without promise tracking. + */ +export async function profileNodeJS(): Promise { + const inspector = await import("node:inspector"); + + // due to some typing issues, not able to use promisify here + return new Promise((resolve, reject) => { + // Start the inspector and connect to it + const session = new inspector.Session(); + session.connect(); + + session.post("Profiler.enable", () => { + session.post("Profiler.start", async () => { + await sleep(DEFAULT_PROFILE_DURATION); + session.post("Profiler.stop", (err, {profile}) => { + if (!err) { + resolve(JSON.stringify(profile)); + } else { + reject(err); + } + + // Detach from the inspector and close the session + session.post("Profiler.disable"); + session.disconnect(); + }); + }); + }); + }); +} diff --git a/packages/beacon-node/test/e2e/network/onWorker/dataSerialization.test.ts b/packages/beacon-node/test/e2e/network/onWorker/dataSerialization.test.ts index f39c0b1df3c..7dd8e3c6104 100644 --- a/packages/beacon-node/test/e2e/network/onWorker/dataSerialization.test.ts +++ b/packages/beacon-node/test/e2e/network/onWorker/dataSerialization.test.ts @@ -140,6 +140,7 @@ describe("data serialization through worker boundary", function () { publishGossip: ["test-topic", bytes, {allowPublishToZeroPeers: true, ignoreDuplicatePublishError: true}], close: [], scrapeMetrics: [], + writeProfile: [""], }; const lodestarPeer: routes.lodestar.LodestarNodePeer = { @@ -201,6 +202,7 @@ describe("data serialization through worker boundary", function () { publishGossip: 1, close: null, scrapeMetrics: "test-metrics", + writeProfile: "", }; type TestCase = {id: string; data: unknown; shouldFail?: boolean}; From 83c289db8354275ca1be9b6d05ac577321b5badb Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Mon, 12 Jun 2023 09:58:02 +0700 Subject: [PATCH 5/7] chore: fix lint in api --- packages/api/src/beacon/routes/lodestar.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/api/src/beacon/routes/lodestar.ts b/packages/api/src/beacon/routes/lodestar.ts index 2f973bc132e..2057b98e37b 100644 --- a/packages/api/src/beacon/routes/lodestar.ts +++ b/packages/api/src/beacon/routes/lodestar.ts @@ -78,7 +78,9 @@ export type Api = { /** Trigger to write a heapdump to disk at `dirpath`. May take > 1min */ writeHeapdump(dirpath?: string): Promise>; /** Trigger to write 10m network thread profile to disk */ - writeNetworkThreadProfile(dirpath?: string): Promise>; + writeNetworkThreadProfile( + dirpath?: string + ): Promise>; /** TODO: description */ getLatestWeakSubjectivityCheckpointEpoch(): Promise>; /** TODO: description */ From 3b9337a2d826ef81e575a348f7df14735e82f8ea Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Tue, 13 Jun 2023 13:57:05 +0700 Subject: [PATCH 6/7] feat: specify duration for the api --- packages/api/src/beacon/routes/lodestar.ts | 9 +++++---- packages/beacon-node/src/api/impl/lodestar/index.ts | 4 ++-- .../beacon-node/src/network/core/networkCoreWorker.ts | 9 ++++++--- .../src/network/core/networkCoreWorkerHandler.ts | 4 ++-- packages/beacon-node/src/network/core/types.ts | 4 ++-- packages/beacon-node/src/network/interface.ts | 2 +- packages/beacon-node/src/network/network.ts | 4 ++-- packages/beacon-node/src/util/profile.ts | 6 ++---- 8 files changed, 22 insertions(+), 20 deletions(-) diff --git a/packages/api/src/beacon/routes/lodestar.ts b/packages/api/src/beacon/routes/lodestar.ts index 2057b98e37b..a097ce5e558 100644 --- a/packages/api/src/beacon/routes/lodestar.ts +++ b/packages/api/src/beacon/routes/lodestar.ts @@ -79,6 +79,7 @@ export type Api = { writeHeapdump(dirpath?: string): Promise>; /** Trigger to write 10m network thread profile to disk */ writeNetworkThreadProfile( + duration?: number, dirpath?: string ): Promise>; /** TODO: description */ @@ -129,7 +130,7 @@ export type Api = { */ export const routesData: RoutesData = { writeHeapdump: {url: "/eth/v1/lodestar/writeheapdump", method: "POST"}, - writeNetworkThreadProfile: {url: "/eth/v1/lodestar/writenetworkthreadprofile", method: "POST"}, + writeNetworkThreadProfile: {url: "/eth/v1/lodestar/write_network_thread_profile", method: "POST"}, getLatestWeakSubjectivityCheckpointEpoch: {url: "/eth/v1/lodestar/ws_epoch", method: "GET"}, getSyncChainsDebugState: {url: "/eth/v1/lodestar/sync-chains-debug-state", method: "GET"}, getGossipQueueItems: {url: "/eth/v1/lodestar/gossip-queue-items/:gossipType", method: "GET"}, @@ -150,7 +151,7 @@ export const routesData: RoutesData = { export type ReqTypes = { writeHeapdump: {query: {dirpath?: string}}; - writeNetworkThreadProfile: {query: {dirpath?: string}}; + writeNetworkThreadProfile: {query: {duration?: number; dirpath?: string}}; getLatestWeakSubjectivityCheckpointEpoch: ReqEmpty; getSyncChainsDebugState: ReqEmpty; getGossipQueueItems: {params: {gossipType: string}}; @@ -177,8 +178,8 @@ export function getReqSerializers(): ReqSerializers { schema: {query: {dirpath: Schema.String}}, }, writeNetworkThreadProfile: { - writeReq: (dirpath) => ({query: {dirpath}}), - parseReq: ({query}) => [query.dirpath], + writeReq: (duration, dirpath) => ({query: {duration, dirpath}}), + parseReq: ({query}) => [query.duration, query.dirpath], schema: {query: {dirpath: Schema.String}}, }, getLatestWeakSubjectivityCheckpointEpoch: reqEmpty, diff --git a/packages/beacon-node/src/api/impl/lodestar/index.ts b/packages/beacon-node/src/api/impl/lodestar/index.ts index 80c4a342277..278c7e4112a 100644 --- a/packages/beacon-node/src/api/impl/lodestar/index.ts +++ b/packages/beacon-node/src/api/impl/lodestar/index.ts @@ -48,13 +48,13 @@ export function getLodestarApi({ } }, - async writeNetworkThreadProfile(dirpath = ".") { + async writeNetworkThreadProfile(durationMs?: number, dirpath?: string) { if (writingNetworkProfile) { throw Error("Already writing network profile"); } try { - const filepath = await network.writeNetworkThreadProfile(dirpath); + const filepath = await network.writeNetworkThreadProfile(durationMs, dirpath); return {data: {filepath}}; } finally { writingNetworkProfile = false; diff --git a/packages/beacon-node/src/network/core/networkCoreWorker.ts b/packages/beacon-node/src/network/core/networkCoreWorker.ts index 9db9dfccd0b..565653fbdbf 100644 --- a/packages/beacon-node/src/network/core/networkCoreWorker.ts +++ b/packages/beacon-node/src/network/core/networkCoreWorker.ts @@ -1,10 +1,12 @@ import worker from "node:worker_threads"; import fs from "node:fs"; +import path from "node:path"; import {createFromProtobuf} from "@libp2p/peer-id-factory"; import {expose} from "@chainsafe/threads/worker"; import {chainConfigFromJson, createBeaconConfig} from "@lodestar/config"; import {getNodeLogger} from "@lodestar/logger/node"; import type {WorkerModule} from "@chainsafe/threads/dist/types/worker.js"; +import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {collectNodeJSMetrics, RegistryMetricCreator} from "../../metrics/index.js"; import {AsyncIterableBridgeCaller, AsyncIterableBridgeHandler} from "../../util/asyncIterableToEvents.js"; import {Clock} from "../../util/clock.js"; @@ -34,6 +36,7 @@ if (!parentPort) throw Error("parentPort must be defined"); const config = createBeaconConfig(chainConfigFromJson(workerData.chainConfigJson), workerData.genesisValidatorsRoot); const peerId = await createFromProtobuf(workerData.peerIdProto); +const DEFAULT_PROFILE_DURATION = SLOTS_PER_EPOCH * config.SECONDS_PER_SLOT * 1000; // TODO: Pass options from main thread for logging // TODO: Logging won't be visible in file loggers @@ -149,9 +152,9 @@ const libp2pWorkerApi: NetworkWorkerApi = { dumpGossipPeerScoreStats: () => core.dumpGossipPeerScoreStats(), dumpDiscv5KadValues: () => core.dumpDiscv5KadValues(), dumpMeshPeers: () => core.dumpMeshPeers(), - writeProfile: async (dirpath = ".") => { - const profile = await profileNodeJS(); - const filePath = `${dirpath}/network_thread_${new Date().toISOString()}.cpuprofile`; + writeProfile: async (durationMs = DEFAULT_PROFILE_DURATION, dirpath = ".") => { + const profile = await profileNodeJS(durationMs); + const filePath = path.join(dirpath, `network_thread_${new Date().toISOString()}.cpuprofile`); fs.writeFileSync(filePath, profile); return filePath; }, diff --git a/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts b/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts index 8cf6cb7248d..d2963d5978b 100644 --- a/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts +++ b/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts @@ -210,8 +210,8 @@ export class WorkerNetworkCore implements INetworkCore { dumpMeshPeers(): Promise> { return this.getApi().dumpMeshPeers(); } - writeNetworkThreadProfile(dirpath?: string): Promise { - return this.getApi().writeProfile(dirpath); + writeNetworkThreadProfile(durationMs?: number, dirpath?: string): Promise { + return this.getApi().writeProfile(durationMs, dirpath); } private getApi(): NetworkWorkerApi { diff --git a/packages/beacon-node/src/network/core/types.ts b/packages/beacon-node/src/network/core/types.ts index cea5e50b0d3..bb66f4dab08 100644 --- a/packages/beacon-node/src/network/core/types.ts +++ b/packages/beacon-node/src/network/core/types.ts @@ -61,7 +61,7 @@ export interface INetworkCore extends INetworkCorePublic { close(): Promise; scrapeMetrics(): Promise; - writeNetworkThreadProfile(dirpath?: string): Promise; + writeNetworkThreadProfile(durationMs?: number, dirpath?: string): Promise; } /** @@ -100,7 +100,7 @@ export type NetworkWorkerApi = INetworkCorePublic & { close(): Promise; scrapeMetrics(): Promise; - writeProfile(dirpath?: string): Promise; + writeProfile(durationMs?: number, dirpath?: string): Promise; // TODO: ReqResp outgoing // TODO: ReqResp incoming diff --git a/packages/beacon-node/src/network/interface.ts b/packages/beacon-node/src/network/interface.ts index b06211c77f5..705e71ab645 100644 --- a/packages/beacon-node/src/network/interface.ts +++ b/packages/beacon-node/src/network/interface.ts @@ -60,7 +60,7 @@ export interface INetwork extends INetworkCorePublic { // Debug dumpGossipQueue(gossipType: GossipType): Promise; - writeNetworkThreadProfile(dirpath?: string): Promise; + writeNetworkThreadProfile(durationMs?: number, dirpath?: string): Promise; } export type PeerDirection = Connection["stat"]["direction"]; diff --git a/packages/beacon-node/src/network/network.ts b/packages/beacon-node/src/network/network.ts index 90feb82b1eb..e4cfbee6f14 100644 --- a/packages/beacon-node/src/network/network.ts +++ b/packages/beacon-node/src/network/network.ts @@ -544,8 +544,8 @@ export class Network implements INetwork { return this.networkProcessor.dumpGossipQueue(gossipType); } - async writeNetworkThreadProfile(dirpath = "."): Promise { - return this.core.writeNetworkThreadProfile(dirpath); + async writeNetworkThreadProfile(durationMs?: number, dirpath?: string): Promise { + return this.core.writeNetworkThreadProfile(durationMs, dirpath); } private onLightClientFinalityUpdate = async (finalityUpdate: allForks.LightClientFinalityUpdate): Promise => { diff --git a/packages/beacon-node/src/util/profile.ts b/packages/beacon-node/src/util/profile.ts index e78147d4fb3..09be059e3eb 100644 --- a/packages/beacon-node/src/util/profile.ts +++ b/packages/beacon-node/src/util/profile.ts @@ -1,11 +1,9 @@ import {sleep} from "@lodestar/utils"; -const DEFAULT_PROFILE_DURATION = 10 * 60 * 1000; - /** * Take 10m profile of the current thread without promise tracking. */ -export async function profileNodeJS(): Promise { +export async function profileNodeJS(durationMs: number): Promise { const inspector = await import("node:inspector"); // due to some typing issues, not able to use promisify here @@ -16,7 +14,7 @@ export async function profileNodeJS(): Promise { session.post("Profiler.enable", () => { session.post("Profiler.start", async () => { - await sleep(DEFAULT_PROFILE_DURATION); + await sleep(durationMs); session.post("Profiler.stop", (err, {profile}) => { if (!err) { resolve(JSON.stringify(profile)); From 8c3f8dffe9a7f3ae387cdca6f91c5669de5cfc87 Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Tue, 13 Jun 2023 14:07:39 +0700 Subject: [PATCH 7/7] chore: fix check-types in beacon-node --- .../test/e2e/network/onWorker/dataSerialization.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/beacon-node/test/e2e/network/onWorker/dataSerialization.test.ts b/packages/beacon-node/test/e2e/network/onWorker/dataSerialization.test.ts index 7dd8e3c6104..0dd895506f1 100644 --- a/packages/beacon-node/test/e2e/network/onWorker/dataSerialization.test.ts +++ b/packages/beacon-node/test/e2e/network/onWorker/dataSerialization.test.ts @@ -140,7 +140,7 @@ describe("data serialization through worker boundary", function () { publishGossip: ["test-topic", bytes, {allowPublishToZeroPeers: true, ignoreDuplicatePublishError: true}], close: [], scrapeMetrics: [], - writeProfile: [""], + writeProfile: [0, ""], }; const lodestarPeer: routes.lodestar.LodestarNodePeer = {