diff --git a/cmd/devnet/devnet/node.go b/cmd/devnet/devnet/node.go index 33f716aa3f3..55dd5e53ede 100644 --- a/cmd/devnet/devnet/node.go +++ b/cmd/devnet/devnet/node.go @@ -184,9 +184,7 @@ func (n *devnetNode) run(ctx *cli.Context) error { n.ethNode, err = enode.New(ctx.Context, n.nodeCfg, n.ethCfg, logger) - if metricsMux != nil { - diagnostics.Setup(ctx, metricsMux, n.ethNode) - } + diagnostics.Setup(ctx, n.ethNode, metricsMux) n.Lock() if n.startErr != nil { diff --git a/cmd/erigon/main.go b/cmd/erigon/main.go index b3dd55dcdb5..7686761ad3b 100644 --- a/cmd/erigon/main.go +++ b/cmd/erigon/main.go @@ -68,9 +68,7 @@ func runErigon(cliCtx *cli.Context) error { return err } - if metricsMux != nil { - diagnostics.Setup(cliCtx, metricsMux, ethNode) - } + diagnostics.Setup(cliCtx, ethNode, metricsMux) err = ethNode.Serve() if err != nil { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 1097ad128f0..f1c2e5b08e1 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -995,9 +995,24 @@ var ( Usage: "set the cors' allow origins", Value: cli.NewStringSlice(), } + DiagDisabledFlag = cli.BoolFlag{ + Name: "diagnostics.disabled", + Usage: "Disable diagnostics", + Value: false, + } + DiagEndpointAddrFlag = cli.StringFlag{ + Name: "diagnostics.endpoint.addr", + Usage: "Diagnostics HTTP server listening interface", + Value: "0.0.0.0", + } + DiagEndpointPortFlag = cli.UintFlag{ + Name: "diagnostics.endpoint.port", + Usage: "Diagnostics HTTP server listening port", + Value: 6060, + } ) -var MetricFlags = []cli.Flag{&MetricsEnabledFlag, &MetricsHTTPFlag, &MetricsPortFlag} +var MetricFlags = []cli.Flag{&MetricsEnabledFlag, &MetricsHTTPFlag, &MetricsPortFlag, &DiagDisabledFlag, &DiagEndpointAddrFlag, &DiagEndpointPortFlag} var DiagnosticsFlags = []cli.Flag{&DiagnosticsURLFlag, &DiagnosticsURLFlag, &DiagnosticsSessionsFlag} diff --git a/diagnostics/block_body_download_stats.go b/diagnostics/block_body_download_stats.go index 4903e1a8c99..a97c4a6493c 100644 --- a/diagnostics/block_body_download_stats.go +++ b/diagnostics/block_body_download_stats.go @@ -10,6 +10,10 @@ import ( ) func SetupBlockBodyDownload(metricsMux *http.ServeMux) { + if metricsMux == nil { + return + } + metricsMux.HandleFunc("/block_body_download", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Access-Control-Allow-Origin", "*") writeBlockBodyDownload(w, r) diff --git a/diagnostics/bodies_info.go b/diagnostics/bodies_info.go index 2a619ecbe46..795d23c38b2 100644 --- a/diagnostics/bodies_info.go +++ b/diagnostics/bodies_info.go @@ -8,6 +8,10 @@ import ( ) func SetupBodiesAccess(metricsMux *http.ServeMux, diag *diaglib.DiagnosticClient) { + if metricsMux == nil { + return + } + metricsMux.HandleFunc("/bodies", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Access-Control-Allow-Origin", "*") w.Header().Set("Content-Type", "application/json") diff --git a/diagnostics/bootnodes.go b/diagnostics/bootnodes.go index fba0982881e..00fd24c25ed 100644 --- a/diagnostics/bootnodes.go +++ b/diagnostics/bootnodes.go @@ -8,6 +8,10 @@ import ( ) func SetupBootnodesAccess(metricsMux *http.ServeMux, node *node.ErigonNode) { + if metricsMux == nil { + return + } + metricsMux.HandleFunc("/bootnodes", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Access-Control-Allow-Origin", "*") w.Header().Set("Content-Type", "application/json") diff --git a/diagnostics/cmd_line.go b/diagnostics/cmd_line.go index db4d9dcfdf5..a2050ca4397 100644 --- a/diagnostics/cmd_line.go +++ b/diagnostics/cmd_line.go @@ -8,6 +8,10 @@ import ( ) func SetupCmdLineAccess(metricsMux *http.ServeMux) { + if metricsMux == nil { + return + } + metricsMux.HandleFunc("/cmdline", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Access-Control-Allow-Origin", "*") w.Header().Set("Content-Type", "application/json") diff --git a/diagnostics/db.go b/diagnostics/db.go index 6769b29425e..e0c7a629561 100644 --- a/diagnostics/db.go +++ b/diagnostics/db.go @@ -16,6 +16,10 @@ import ( ) func SetupDbAccess(ctx *cli.Context, metricsMux *http.ServeMux) { + if metricsMux == nil { + return + } + var dataDir string if ctx.IsSet("datadir") { dataDir = ctx.String("datadir") diff --git a/diagnostics/flags.go b/diagnostics/flags.go index 9cdf0267031..cbcc11b3228 100644 --- a/diagnostics/flags.go +++ b/diagnostics/flags.go @@ -8,6 +8,10 @@ import ( ) func SetupFlagsAccess(ctx *cli.Context, metricsMux *http.ServeMux) { + if metricsMux == nil { + return + } + metricsMux.HandleFunc("/flags", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Access-Control-Allow-Origin", "*") w.Header().Set("Content-Type", "application/json") diff --git a/diagnostics/header_downloader_stats.go b/diagnostics/header_downloader_stats.go index a388d6fb4ae..0b9c4b48a76 100644 --- a/diagnostics/header_downloader_stats.go +++ b/diagnostics/header_downloader_stats.go @@ -10,6 +10,10 @@ import ( ) func SetupHeaderDownloadStats(metricsMux *http.ServeMux) { + if metricsMux == nil { + return + } + metricsMux.HandleFunc("/headers_download", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Access-Control-Allow-Origin", "*") writeHeaderDownload(w, r) diff --git a/diagnostics/headers.go b/diagnostics/headers.go index 4f63ef8343e..82066609368 100644 --- a/diagnostics/headers.go +++ b/diagnostics/headers.go @@ -8,6 +8,10 @@ import ( ) func SetupHeadersAccess(metricsMux *http.ServeMux, diag *diaglib.DiagnosticClient) { + if metricsMux == nil { + return + } + metricsMux.HandleFunc("/headers", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Access-Control-Allow-Origin", "*") w.Header().Set("Content-Type", "application/json") diff --git a/diagnostics/logs.go b/diagnostics/logs.go index 72196aa79a4..66889f8f68f 100644 --- a/diagnostics/logs.go +++ b/diagnostics/logs.go @@ -19,6 +19,10 @@ import ( ) func SetupLogsAccess(ctx *cli.Context, metricsMux *http.ServeMux) { + if metricsMux == nil { + return + } + dirPath := ctx.String(logging.LogDirPathFlag.Name) if dirPath == "" { datadir := ctx.String("datadir") diff --git a/diagnostics/mem.go b/diagnostics/mem.go index e1d25e210b7..1ad34adea56 100644 --- a/diagnostics/mem.go +++ b/diagnostics/mem.go @@ -8,6 +8,10 @@ import ( ) func SetupMemAccess(metricsMux *http.ServeMux) { + if metricsMux == nil { + return + } + metricsMux.HandleFunc("/mem", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Access-Control-Allow-Origin", "*") w.Header().Set("Content-Type", "application/json") diff --git a/diagnostics/nodeinfo.go b/diagnostics/nodeinfo.go index 198aa77d7d2..fc09c170436 100644 --- a/diagnostics/nodeinfo.go +++ b/diagnostics/nodeinfo.go @@ -8,6 +8,10 @@ import ( ) func SetupNodeInfoAccess(metricsMux *http.ServeMux, node *node.ErigonNode) { + if metricsMux == nil { + return + } + metricsMux.HandleFunc("/nodeinfo", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Access-Control-Allow-Origin", "*") writeNodeInfo(w, node) diff --git a/diagnostics/peers.go b/diagnostics/peers.go index e2a59e650c0..8f2d7847396 100644 --- a/diagnostics/peers.go +++ b/diagnostics/peers.go @@ -37,6 +37,10 @@ type PeerResponse struct { } func SetupPeersAccess(ctxclient *cli.Context, metricsMux *http.ServeMux, node *node.ErigonNode, diag *diaglib.DiagnosticClient) { + if metricsMux == nil { + return + } + metricsMux.HandleFunc("/peers", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Access-Control-Allow-Origin", "*") w.Header().Set("Content-Type", "application/json") diff --git a/diagnostics/setup.go b/diagnostics/setup.go index 6779d31c1b8..8808eb94185 100644 --- a/diagnostics/setup.go +++ b/diagnostics/setup.go @@ -1,6 +1,7 @@ package diagnostics import ( + "fmt" "net/http" "strings" @@ -8,33 +9,90 @@ import ( diaglib "github.com/ledgerwatch/erigon-lib/diagnostics" "github.com/ledgerwatch/erigon/turbo/node" + "github.com/ledgerwatch/log/v3" ) -func Setup(ctx *cli.Context, metricsMux *http.ServeMux, node *node.ErigonNode) { - debugMux := http.NewServeMux() +var ( + diagnosticsDisabledFlag = "diagnostics.disabled" + diagnosticsAddrFlag = "diagnostics.endpoint.addr" + diagnosticsPortFlag = "diagnostics.endpoint.port" + metricsHTTPFlag = "metrics.addr" + metricsPortFlag = "metrics.port" +) + +func Setup(ctx *cli.Context, node *node.ErigonNode, metricsMux *http.ServeMux) { + if ctx.Bool(diagnosticsDisabledFlag) { + return + } + + var diagMux *http.ServeMux + + diagHost := ctx.String(diagnosticsAddrFlag) + diagPort := ctx.Int(diagnosticsPortFlag) + diagAddress := fmt.Sprintf("%s:%d", diagHost, diagPort) + + metricsHost := ctx.String(metricsHTTPFlag) + metricsPort := ctx.Int(metricsPortFlag) + metricsAddress := fmt.Sprintf("%s:%d", metricsHost, metricsPort) - diagnostic := diaglib.NewDiagnosticClient(debugMux, node.Backend().DataDir()) + if diagAddress == metricsAddress { + diagMux = SetupDiagnosticsEndpoint(metricsMux, diagAddress) + } else { + diagMux = SetupDiagnosticsEndpoint(nil, diagAddress) + } + + diagnostic := diaglib.NewDiagnosticClient(diagMux, node.Backend().DataDir()) diagnostic.Setup() - metricsMux.HandleFunc("/debug/", func(w http.ResponseWriter, r *http.Request) { - r.URL.Path = strings.TrimPrefix(r.URL.Path, "/debug") - r.URL.RawPath = strings.TrimPrefix(r.URL.RawPath, "/debug") - debugMux.ServeHTTP(w, r) - }) + SetupEndpoints(ctx, node, diagMux, diagnostic) +} + +func SetupDiagnosticsEndpoint(metricsMux *http.ServeMux, addres string) *http.ServeMux { + diagMux := http.NewServeMux() - SetupLogsAccess(ctx, debugMux) - SetupDbAccess(ctx, debugMux) - SetupCmdLineAccess(debugMux) - SetupFlagsAccess(ctx, debugMux) - SetupVersionAccess(debugMux) - SetupBlockBodyDownload(debugMux) - SetupHeaderDownloadStats(debugMux) - SetupNodeInfoAccess(debugMux, node) - SetupPeersAccess(ctx, debugMux, node, diagnostic) - SetupBootnodesAccess(debugMux, node) - SetupStagesAccess(debugMux, diagnostic) - SetupMemAccess(debugMux) - SetupHeadersAccess(debugMux, diagnostic) - SetupBodiesAccess(debugMux, diagnostic) + if metricsMux != nil { + SetupMiddleMuxHandler(diagMux, metricsMux, "/debug/diag") + } else { + middleMux := http.NewServeMux() + SetupMiddleMuxHandler(diagMux, middleMux, "/debug/diag") + + diagServer := &http.Server{ + Addr: addres, + Handler: middleMux, + } + + go func() { + if err := diagServer.ListenAndServe(); err != nil { + log.Error("[Diagnostics] Failure in running diagnostics server", "err", err) + } + }() + + } + + return diagMux +} + +func SetupMiddleMuxHandler(mux *http.ServeMux, middleMux *http.ServeMux, path string) { + middleMux.HandleFunc(path+"/", func(w http.ResponseWriter, r *http.Request) { + r.URL.Path = strings.TrimPrefix(r.URL.Path, path) + r.URL.RawPath = strings.TrimPrefix(r.URL.RawPath, path) + mux.ServeHTTP(w, r) + }) +} +func SetupEndpoints(ctx *cli.Context, node *node.ErigonNode, diagMux *http.ServeMux, diagnostic *diaglib.DiagnosticClient) { + SetupLogsAccess(ctx, diagMux) + SetupDbAccess(ctx, diagMux) + SetupCmdLineAccess(diagMux) + SetupFlagsAccess(ctx, diagMux) + SetupVersionAccess(diagMux) + SetupBlockBodyDownload(diagMux) + SetupHeaderDownloadStats(diagMux) + SetupNodeInfoAccess(diagMux, node) + SetupPeersAccess(ctx, diagMux, node, diagnostic) + SetupBootnodesAccess(diagMux, node) + SetupStagesAccess(diagMux, diagnostic) + SetupMemAccess(diagMux) + SetupHeadersAccess(diagMux, diagnostic) + SetupBodiesAccess(diagMux, diagnostic) } diff --git a/diagnostics/snapshot_sync.go b/diagnostics/snapshot_sync.go index 4cfa90dbbcf..9100977ab5b 100644 --- a/diagnostics/snapshot_sync.go +++ b/diagnostics/snapshot_sync.go @@ -8,6 +8,10 @@ import ( ) func SetupStagesAccess(metricsMux *http.ServeMux, diag *diaglib.DiagnosticClient) { + if metricsMux == nil { + return + } + metricsMux.HandleFunc("/snapshot-sync", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Access-Control-Allow-Origin", "*") w.Header().Set("Content-Type", "application/json") diff --git a/diagnostics/version.go b/diagnostics/version.go index f54bfa73b64..6bf869e835f 100644 --- a/diagnostics/version.go +++ b/diagnostics/version.go @@ -10,6 +10,10 @@ import ( const Version = 3 func SetupVersionAccess(metricsMux *http.ServeMux) { + if metricsMux == nil { + return + } + metricsMux.HandleFunc("/version", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Access-Control-Allow-Origin", "*") w.Header().Set("Content-Type", "application/json") diff --git a/erigon-lib/common/metrics/metrics_enabled.go b/erigon-lib/common/metrics/metrics_enabled.go index dff5154390b..0f3b89bd6d6 100644 --- a/erigon-lib/common/metrics/metrics_enabled.go +++ b/erigon-lib/common/metrics/metrics_enabled.go @@ -28,6 +28,6 @@ type Config struct { //nolint:maligned var DefaultConfig = Config{ Enabled: false, EnabledExpensive: false, - HTTP: "127.0.0.1", + HTTP: "0.0.0.0", Port: 6060, } diff --git a/erigon-lib/go.sum b/erigon-lib/go.sum index 8448cc70560..b78fe520cb6 100644 --- a/erigon-lib/go.sum +++ b/erigon-lib/go.sum @@ -688,4 +688,4 @@ modernc.org/sqlite v1.26.0/go.mod h1:FL3pVXie73rg3Rii6V/u5BoHlSoyeZeIgKZEgHARyCU rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= zombiezen.com/go/sqlite v0.13.1 h1:qDzxyWWmMtSSEH5qxamqBFmqA2BLSSbtODi3ojaE02o= -zombiezen.com/go/sqlite v0.13.1/go.mod h1:Ht/5Rg3Ae2hoyh1I7gbWtWAl89CNocfqeb/aAMTkJr4= +zombiezen.com/go/sqlite v0.13.1/go.mod h1:Ht/5Rg3Ae2hoyh1I7gbWtWAl89CNocfqeb/aAMTkJr4= \ No newline at end of file diff --git a/turbo/app/snapshots_cmd.go b/turbo/app/snapshots_cmd.go index 5acf0333ff8..032c54ff52f 100644 --- a/turbo/app/snapshots_cmd.go +++ b/turbo/app/snapshots_cmd.go @@ -708,9 +708,7 @@ func doUploaderCommand(cliCtx *cli.Context) error { return err } - if metricsMux != nil { - diagnostics.Setup(cliCtx, metricsMux, ethNode) - } + diagnostics.Setup(cliCtx, ethNode, metricsMux) err = ethNode.Serve() if err != nil { diff --git a/turbo/app/support_cmd.go b/turbo/app/support_cmd.go index 9de80045fa0..64e5054bebb 100644 --- a/turbo/app/support_cmd.go +++ b/turbo/app/support_cmd.go @@ -185,7 +185,7 @@ func tunnel(ctx context.Context, cancel context.CancelFunc, sigs chan os.Signal, nodes := map[string]*node{} for _, debugURL := range debugURLs { - debugResponse, err := metricsClient.Get(debugURL + "/debug/nodeinfo") + debugResponse, err := metricsClient.Get(debugURL + "/debug/diag/nodeinfo") if err != nil { return err @@ -326,7 +326,7 @@ func tunnel(ctx context.Context, cancel context.CancelFunc, sigs chan os.Signal, queryString = "?" + nodeRequest.QueryParams.Encode() } - debugURL := node.debugURL + "/debug/" + requests[0].Method + queryString + debugURL := node.debugURL + "/debug/diag/" + requests[0].Method + queryString debugResponse, err := metricsClient.Get(debugURL) if err != nil { diff --git a/turbo/debug/flags.go b/turbo/debug/flags.go index 7ae0a844093..07affd9549e 100644 --- a/turbo/debug/flags.go +++ b/turbo/debug/flags.go @@ -23,6 +23,7 @@ import ( "net/http/pprof" //nolint:gosec "os" "path/filepath" + "strings" "github.com/ledgerwatch/erigon-lib/common/disk" "github.com/ledgerwatch/erigon-lib/common/mem" @@ -70,7 +71,7 @@ var ( pprofAddrFlag = cli.StringFlag{ Name: "pprof.addr", Usage: "pprof HTTP server listening interface", - Value: "127.0.0.1", + Value: "0.0.0.0", } cpuprofileFlag = cli.StringFlag{ Name: "pprof.cpuprofile", @@ -222,32 +223,56 @@ func Setup(ctx *cli.Context, rootLogger bool) (log.Logger, *http.ServeMux, error pprofPort := ctx.Int(pprofPortFlag.Name) address := fmt.Sprintf("%s:%d", pprofHost, pprofPort) if address == metricsAddress { - StartPProf(address, metricsMux) + metricsMux = StartPProf(address, metricsMux) } else { - StartPProf(address, nil) + metricsMux = StartPProf(address, nil) } } return logger, metricsMux, nil } -func StartPProf(address string, metricsMux *http.ServeMux) { +func StartPProf(address string, metricsMux *http.ServeMux) *http.ServeMux { cpuMsg := fmt.Sprintf("go tool pprof -lines -http=: http://%s/%s", address, "debug/pprof/profile?seconds=20") heapMsg := fmt.Sprintf("go tool pprof -lines -http=: http://%s/%s", address, "debug/pprof/heap") log.Info("Starting pprof server", "cpu", cpuMsg, "heap", heapMsg) + pprofMux := http.NewServeMux() + + pprofMux.HandleFunc("/", pprof.Index) + pprofMux.HandleFunc("/cmdline", pprof.Cmdline) + pprofMux.HandleFunc("/debug/pprof/profile", pprof.Profile) + pprofMux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + pprofMux.HandleFunc("/debug/pprof/trace", pprof.Trace) + if metricsMux == nil { + middleMux := http.NewServeMux() + middleMux.HandleFunc("/debug/pprof/", func(w http.ResponseWriter, r *http.Request) { + r.URL.Path = strings.TrimPrefix(r.URL.Path, "/debug/pprof") + r.URL.RawPath = strings.TrimPrefix(r.URL.RawPath, "/debug/pprof") + pprofMux.ServeHTTP(w, r) + }) + + pprofServer := &http.Server{ + Addr: address, + Handler: middleMux, + } + go func() { - if err := http.ListenAndServe(address, nil); err != nil { // nolint:gosec + if err := pprofServer.ListenAndServe(); err != nil { log.Error("Failure in running pprof server", "err", err) } }() + + return middleMux } else { - metricsMux.HandleFunc("/debug/pprof/", pprof.Index) - metricsMux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) - metricsMux.HandleFunc("/debug/pprof/profile", pprof.Profile) - metricsMux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) - metricsMux.HandleFunc("/debug/pprof/trace", pprof.Trace) + metricsMux.HandleFunc("/debug/pprof/", func(w http.ResponseWriter, r *http.Request) { + r.URL.Path = strings.TrimPrefix(r.URL.Path, "/debug/pprof") + r.URL.RawPath = strings.TrimPrefix(r.URL.RawPath, "/debug/pprof") + pprofMux.ServeHTTP(w, r) + }) + + return metricsMux } }