From e66945775d9ee27917b7667844c6b6d7b31abd4a Mon Sep 17 00:00:00 2001 From: Shoham Chakraborty Date: Wed, 14 Feb 2024 15:43:38 +0800 Subject: [PATCH 01/16] Add mem stats to dbg --- diagnostics/mem.go | 23 +++++++++++++++++++++++ erigon-lib/common/dbg/experiments.go | 18 ++++++++++++++++++ erigon-lib/diagnostics/entities.go | 6 ++++++ erigon-lib/go.mod | 10 +++++++++- go.mod | 9 ++++++++- turbo/snapshotsync/snapshotsync.go | 7 +++++++ 6 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 diagnostics/mem.go diff --git a/diagnostics/mem.go b/diagnostics/mem.go new file mode 100644 index 00000000000..2c22f162050 --- /dev/null +++ b/diagnostics/mem.go @@ -0,0 +1,23 @@ +package diagnostics + +import ( + "github.com/shirou/gopsutil/v3/process" + "os" +) + +func getMemUsage() (process.MemoryMapsStat, error) { + pid := os.Getpid() + proc, err := process.NewProcess(int32(pid)) + + if err != nil { + return process.MemoryMapsStat{}, err + } + + memoryMaps, err := proc.MemoryMaps(true) + + if err != nil { + return process.MemoryMapsStat{}, err + } + + return (*memoryMaps)[0], nil +} diff --git a/erigon-lib/common/dbg/experiments.go b/erigon-lib/common/dbg/experiments.go index e9df7ace44e..eaf7e87322c 100644 --- a/erigon-lib/common/dbg/experiments.go +++ b/erigon-lib/common/dbg/experiments.go @@ -17,6 +17,7 @@ package dbg import ( + "github.com/shirou/gopsutil/v3/process" "os" "runtime" "strconv" @@ -44,6 +45,23 @@ func ReadMemStats(m *runtime.MemStats) { } } +func GetMemUsage() (process.MemoryMapsStat, error) { + pid := os.Getpid() + proc, err := process.NewProcess(int32(pid)) + + if err != nil { + return process.MemoryMapsStat{}, err + } + + memoryMaps, err := proc.MemoryMaps(true) + + if err != nil { + return process.MemoryMapsStat{}, err + } + + return (*memoryMaps)[0], nil +} + var ( writeMap bool writeMapOnce sync.Once diff --git a/erigon-lib/diagnostics/entities.go b/erigon-lib/diagnostics/entities.go index f91ab9cea6e..a49edf89174 100644 --- a/erigon-lib/diagnostics/entities.go +++ b/erigon-lib/diagnostics/entities.go @@ -16,6 +16,8 @@ package diagnostics +import "github.com/shirou/gopsutil/v3/process" + type PeerStatisticsGetter interface { GetPeersStatistics() map[string]*PeerStatistics } @@ -73,6 +75,10 @@ type SnapshotSegmentIndexingFinishedUpdate struct { SegmentName string `json:"segmentName"` } +type MemUsage struct { + stats process.MemoryMapsStat `json:"stats"` +} + func (ti SnapshotDownloadStatistics) Type() Type { return TypeOf(ti) } diff --git a/erigon-lib/go.mod b/erigon-lib/go.mod index 95e0dd8f5ec..666ab178544 100644 --- a/erigon-lib/go.mod +++ b/erigon-lib/go.mod @@ -37,13 +37,14 @@ require ( github.com/prometheus/client_golang v1.17.0 github.com/prometheus/client_model v0.5.0 github.com/quasilyte/go-ruleguard/dsl v0.3.22 + github.com/shirou/gopsutil/v3 v3.24.1 github.com/spaolacci/murmur3 v1.1.0 github.com/stretchr/testify v1.8.4 github.com/tidwall/btree v1.6.0 golang.org/x/crypto v0.16.0 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 golang.org/x/sync v0.5.0 - golang.org/x/sys v0.15.0 + golang.org/x/sys v0.16.0 golang.org/x/time v0.5.0 google.golang.org/grpc v1.59.0 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0 @@ -96,11 +97,13 @@ require ( github.com/go-llsqlite/crawshaw v0.0.0-20230910110433-7e901377eb6c // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect github.com/godbus/dbus/v5 v5.0.4 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/uuid v1.3.1 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/huandu/xstrings v1.4.0 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect @@ -127,11 +130,16 @@ require ( github.com/pion/webrtc/v3 v3.1.42 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/prometheus/common v0.44.0 // indirect github.com/prometheus/procfs v0.11.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 // indirect + github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/sirupsen/logrus v1.9.0 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect + github.com/yusufpapurcu/wmi v1.2.3 // indirect go.etcd.io/bbolt v1.3.6 // indirect go.opentelemetry.io/otel v1.8.0 // indirect go.opentelemetry.io/otel/trace v1.8.0 // indirect diff --git a/go.mod b/go.mod index a18844cc939..0ea4d22da8c 100644 --- a/go.mod +++ b/go.mod @@ -75,6 +75,7 @@ require ( github.com/prysmaticlabs/gohashtree v0.0.3-alpha.0.20230502123415-aafd8b3ca202 github.com/quasilyte/go-ruleguard/dsl v0.3.22 github.com/rs/cors v1.10.1 + github.com/shirou/gopsutil/v3 v3.24.1 github.com/spf13/afero v1.9.5 github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 @@ -92,7 +93,7 @@ require ( golang.org/x/exp v0.0.0-20230905200255-921286631fa9 golang.org/x/net v0.19.0 golang.org/x/sync v0.5.0 - golang.org/x/sys v0.15.0 + golang.org/x/sys v0.16.0 golang.org/x/time v0.5.0 google.golang.org/grpc v1.59.0 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0 @@ -168,6 +169,7 @@ require ( github.com/go-llsqlite/crawshaw v0.0.0-20230910110433-7e901377eb6c // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/go-stack/stack v1.8.1 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect @@ -198,6 +200,7 @@ require ( github.com/libp2p/go-netroute v0.2.1 // indirect github.com/libp2p/go-reuseport v0.4.0 // indirect github.com/libp2p/go-yamux/v4 v4.0.1 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect @@ -242,6 +245,7 @@ require ( github.com/pion/webrtc/v3 v3.1.42 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/prometheus/client_golang v1.17.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.44.0 // indirect @@ -260,7 +264,10 @@ require ( github.com/sosodev/duration v1.1.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/supranational/blst v0.3.11 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect + github.com/yusufpapurcu/wmi v1.2.3 // indirect go.etcd.io/bbolt v1.3.6 // indirect go.opentelemetry.io/otel v1.8.0 // indirect go.opentelemetry.io/otel/trace v1.8.0 // indirect diff --git a/turbo/snapshotsync/snapshotsync.go b/turbo/snapshotsync/snapshotsync.go index a33b5e2efa1..736b23df937 100644 --- a/turbo/snapshotsync/snapshotsync.go +++ b/turbo/snapshotsync/snapshotsync.go @@ -185,6 +185,12 @@ Loop: } dbg.ReadMemStats(&m) + memStats, err := dbg.GetMemUsage() + + if err != nil { + log.Warn("Error reading memory stats", "err", err) + } + downloadTimeLeft := calculateTime(stats.BytesTotal-stats.BytesCompleted, stats.DownloadRate) suffix := "downloading" if stats.Progress > 0 && stats.DownloadRate == 0 { @@ -201,6 +207,7 @@ Loop: "files", stats.FilesTotal, "connections", stats.ConnectionsTotal, "alloc", common.ByteCount(m.Alloc), "sys", common.ByteCount(m.Sys), + "shared-usage-clean", common.ByteCount(memStats.SharedClean), ) } } From 80d35ac96b96535e1d9fd3fb4a1b5e4d6f307cf0 Mon Sep 17 00:00:00 2001 From: Shoham Chakraborty Date: Mon, 19 Feb 2024 10:55:24 +0800 Subject: [PATCH 02/16] Add mem stats to metrics server --- diagnostics/mem.go | 23 ++++++++++++++++++++++- diagnostics/setup.go | 1 + erigon-lib/go.sum | 30 +++++++++++++++++++++++++++--- go.sum | 26 ++++++++++++++++++++++++-- 4 files changed, 74 insertions(+), 6 deletions(-) diff --git a/diagnostics/mem.go b/diagnostics/mem.go index 2c22f162050..274b442c7f7 100644 --- a/diagnostics/mem.go +++ b/diagnostics/mem.go @@ -1,11 +1,32 @@ package diagnostics import ( + "encoding/json" "github.com/shirou/gopsutil/v3/process" + "net/http" "os" ) -func getMemUsage() (process.MemoryMapsStat, error) { +func SetupMemAccess(metricsMux *http.ServeMux) { + metricsMux.HandleFunc("/mem", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("Content-Type", "application/json") + writeMem(w) + }) +} + +func writeMem(w http.ResponseWriter) { + memStats, err := GetMemUsage() + + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + json.NewEncoder(w).Encode(memStats) +} + +func GetMemUsage() (process.MemoryMapsStat, error) { pid := os.Getpid() proc, err := process.NewProcess(int32(pid)) diff --git a/diagnostics/setup.go b/diagnostics/setup.go index 44fc74570fc..2773c9a74bb 100644 --- a/diagnostics/setup.go +++ b/diagnostics/setup.go @@ -31,5 +31,6 @@ func Setup(ctx *cli.Context, metricsMux *http.ServeMux, node *node.ErigonNode) { SetupPeersAccess(ctx, debugMux, node) SetupBootnodesAccess(debugMux, node) SetupStagesAccess(debugMux, diagnostic) + SetupMemAccess(debugMux) } diff --git a/erigon-lib/go.sum b/erigon-lib/go.sum index 005ce3b3ce1..b01abfd4449 100644 --- a/erigon-lib/go.sum +++ b/erigon-lib/go.sum @@ -200,6 +200,8 @@ github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= @@ -240,8 +242,11 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -299,6 +304,8 @@ github.com/ledgerwatch/log/v3 v3.9.0 h1:iDwrXe0PVwBC68Dd94YSsHbMgQ3ufsgjzXtFNFVZ github.com/ledgerwatch/log/v3 v3.9.0/go.mod h1:EiAY6upmI/6LkNhOVxb4eVsmsP11HZCnZ3PlJMjYiqE= github.com/ledgerwatch/secp256k1 v1.0.0 h1:Usvz87YoTG0uePIV8woOof5cQnLXGYa162rFf3YnwaQ= github.com/ledgerwatch/secp256k1 v1.0.0/go.mod h1:SPmqJFciiF/Q0mPt2jVs2dTr/1TZBTIA+kPMmKgBAak= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/matryer/moq v0.3.3 h1:pScMH9VyrdT4S93yiLpVyU8rCDqGQr24uOyBxmktG5Q= github.com/matryer/moq v0.3.3/go.mod h1:RJ75ZZZD71hejp39j4crZLsEDszGk6iH4v4YsWFKH4s= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= @@ -391,6 +398,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= @@ -429,6 +438,12 @@ github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417/go.mod h1:qe5TWALJ8/a1 github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= +github.com/shirou/gopsutil/v3 v3.24.1 h1:R3t6ondCEvmARp3wxODhXMTLC/klMa87h2PHUw5m7QI= +github.com/shirou/gopsutil/v3 v3.24.1/go.mod h1:UU7a2MSBQa+kW1uuDq8DeEBS8kmrnQwsv2b5O513rwU= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= @@ -460,11 +475,17 @@ github.com/tidwall/btree v1.6.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EU github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= @@ -555,6 +576,7 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -564,6 +586,7 @@ golang.org/x/sys v0.0.0-20200413165638-669c56c373c4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -579,9 +602,10 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/go.sum b/go.sum index fcabad51499..cbf99d7e7dc 100644 --- a/go.sum +++ b/go.sum @@ -342,6 +342,8 @@ github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU= github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -417,8 +419,11 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -577,6 +582,8 @@ github.com/libp2p/go-reuseport v0.4.0 h1:nR5KU7hD0WxXCJbmw7r2rhRYruNRl2koHw8fQsc github.com/libp2p/go-reuseport v0.4.0/go.mod h1:ZtI03j/wO5hZVDFo2jKywN6bYKWLOy8Se6DrI2E1cLU= github.com/libp2p/go-yamux/v4 v4.0.1 h1:FfDR4S1wj6Bw2Pqbc8Uz7pCxeRBPbwsBbEdfwiCypkQ= github.com/libp2p/go-yamux/v4 v4.0.1/go.mod h1:NWjl8ZTLOGlozrXSOZ/HlfG++39iKNnM5wwmtQP1YB4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd h1:br0buuQ854V8u83wA0rVZ8ttrq5CpaPZdvrK0LP2lOk= @@ -738,6 +745,8 @@ github.com/pkg/profile v1.6.0/go.mod h1:qBsxPvzyUincmltOk6iyRVxHYg4adc0OFOv72ZdL github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= @@ -808,6 +817,10 @@ github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5P github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= +github.com/shirou/gopsutil/v3 v3.24.1 h1:R3t6ondCEvmARp3wxODhXMTLC/klMa87h2PHUw5m7QI= +github.com/shirou/gopsutil/v3 v3.24.1/go.mod h1:UU7a2MSBQa+kW1uuDq8DeEBS8kmrnQwsv2b5O513rwU= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0= @@ -879,6 +892,10 @@ github.com/tidwall/btree v1.6.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EU github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/ugorji/go v1.1.13/go.mod h1:jxau1n+/wyTGLQoCkjok9r5zFa/FxT6eI5HiHKQszjc= github.com/ugorji/go/codec v1.1.13 h1:013LbFhocBoIqgHeIHKlV4JWYhqogATYWZhIcH0WHn4= github.com/ugorji/go/codec v1.1.13/go.mod h1:oNVt3Dq+FO91WNQ/9JnHKQP2QJxTzoN7wCBFCq1OeuU= @@ -905,6 +922,8 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= @@ -1102,6 +1121,7 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1130,6 +1150,7 @@ golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1156,8 +1177,9 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= From d8f3dadced0f80050e89b3b221d8e2ba06e91b86 Mon Sep 17 00:00:00 2001 From: Shoham Chakraborty Date: Mon, 19 Feb 2024 15:13:11 +0800 Subject: [PATCH 03/16] Add to prometheus --- diagnostics/mem.go | 34 ++++++++------------ erigon-lib/common/dbg/experiments.go | 18 ----------- erigon-lib/common/dbg/mem.go | 48 ++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 38 deletions(-) create mode 100644 erigon-lib/common/dbg/mem.go diff --git a/diagnostics/mem.go b/diagnostics/mem.go index 274b442c7f7..00a5d6819e1 100644 --- a/diagnostics/mem.go +++ b/diagnostics/mem.go @@ -2,9 +2,9 @@ package diagnostics import ( "encoding/json" - "github.com/shirou/gopsutil/v3/process" + "github.com/ledgerwatch/erigon-lib/common/dbg" "net/http" - "os" + "time" ) func SetupMemAccess(metricsMux *http.ServeMux) { @@ -13,10 +13,21 @@ func SetupMemAccess(metricsMux *http.ServeMux) { w.Header().Set("Content-Type", "application/json") writeMem(w) }) + + // update prometheus memory stats at least every 30 seconds + go func() { + ticker := time.NewTicker(30 * time.Second) + defer ticker.Stop() + + for { + dbg.GetMemUsage() + <-ticker.C + } + }() } func writeMem(w http.ResponseWriter) { - memStats, err := GetMemUsage() + memStats, err := dbg.GetMemUsage() if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) @@ -25,20 +36,3 @@ func writeMem(w http.ResponseWriter) { json.NewEncoder(w).Encode(memStats) } - -func GetMemUsage() (process.MemoryMapsStat, error) { - pid := os.Getpid() - proc, err := process.NewProcess(int32(pid)) - - if err != nil { - return process.MemoryMapsStat{}, err - } - - memoryMaps, err := proc.MemoryMaps(true) - - if err != nil { - return process.MemoryMapsStat{}, err - } - - return (*memoryMaps)[0], nil -} diff --git a/erigon-lib/common/dbg/experiments.go b/erigon-lib/common/dbg/experiments.go index eaf7e87322c..e9df7ace44e 100644 --- a/erigon-lib/common/dbg/experiments.go +++ b/erigon-lib/common/dbg/experiments.go @@ -17,7 +17,6 @@ package dbg import ( - "github.com/shirou/gopsutil/v3/process" "os" "runtime" "strconv" @@ -45,23 +44,6 @@ func ReadMemStats(m *runtime.MemStats) { } } -func GetMemUsage() (process.MemoryMapsStat, error) { - pid := os.Getpid() - proc, err := process.NewProcess(int32(pid)) - - if err != nil { - return process.MemoryMapsStat{}, err - } - - memoryMaps, err := proc.MemoryMaps(true) - - if err != nil { - return process.MemoryMapsStat{}, err - } - - return (*memoryMaps)[0], nil -} - var ( writeMap bool writeMapOnce sync.Once diff --git a/erigon-lib/common/dbg/mem.go b/erigon-lib/common/dbg/mem.go new file mode 100644 index 00000000000..c4f0dd00f6c --- /dev/null +++ b/erigon-lib/common/dbg/mem.go @@ -0,0 +1,48 @@ +package dbg + +import ( + "github.com/ledgerwatch/erigon-lib/metrics" + "github.com/shirou/gopsutil/v3/process" + "os" +) + +var ( + memRssGauge = metrics.NewGauge(`mem_rss`) + memSizeGauge = metrics.NewGauge(`mem_size`) + memPssGauge = metrics.NewGauge(`mem_pss`) + memSharedCleanGauge = metrics.NewGauge(`mem_shared_clean`) + memSharedDirtyGauge = metrics.NewGauge(`mem_shared_dirty`) + memPrivateCleanGauge = metrics.NewGauge(`mem_private_clean`) + memPrivateDirtyGauge = metrics.NewGauge(`mem_private_dirty`) + memReferencedGauge = metrics.NewGauge(`mem_referenced`) + memAnonymousGauge = metrics.NewGauge(`mem_anonymous`) + memSwapGauge = metrics.NewGauge(`mem_swap`) +) + +func GetMemUsage() (process.MemoryMapsStat, error) { + pid := os.Getpid() + proc, err := process.NewProcess(int32(pid)) + + if err != nil { + return process.MemoryMapsStat{}, err + } + + memoryMaps, err := proc.MemoryMaps(true) + + if err != nil { + return process.MemoryMapsStat{}, err + } + + memRssGauge.SetUint64((*memoryMaps)[0].Rss) + memSizeGauge.SetUint64((*memoryMaps)[0].Size) + memPssGauge.SetUint64((*memoryMaps)[0].Pss) + memSharedCleanGauge.SetUint64((*memoryMaps)[0].SharedClean) + memSharedDirtyGauge.SetUint64((*memoryMaps)[0].SharedDirty) + memPrivateCleanGauge.SetUint64((*memoryMaps)[0].PrivateClean) + memPrivateDirtyGauge.SetUint64((*memoryMaps)[0].PrivateDirty) + memReferencedGauge.SetUint64((*memoryMaps)[0].Referenced) + memAnonymousGauge.SetUint64((*memoryMaps)[0].Anonymous) + memSwapGauge.SetUint64((*memoryMaps)[0].Swap) + + return (*memoryMaps)[0], nil +} From 8dc379bf1451bf2228e14fe75d27772fb0466aea Mon Sep 17 00:00:00 2001 From: Shoham Chakraborty Date: Wed, 21 Feb 2024 10:33:55 +0800 Subject: [PATCH 04/16] Handle errors, split functions --- cmd/erigon/main.go | 2 +- diagnostics/mem.go | 18 ++++-------------- diagnostics/setup.go | 25 ++++++++++++++++++++++++- erigon-lib/common/dbg/mem.go | 26 ++++++++++++++------------ turbo/snapshotsync/snapshotsync.go | 2 +- 5 files changed, 44 insertions(+), 29 deletions(-) diff --git a/cmd/erigon/main.go b/cmd/erigon/main.go index b3dd55dcdb5..4ef2aa77fc0 100644 --- a/cmd/erigon/main.go +++ b/cmd/erigon/main.go @@ -69,7 +69,7 @@ func runErigon(cliCtx *cli.Context) error { } if metricsMux != nil { - diagnostics.Setup(cliCtx, metricsMux, ethNode) + diagnostics.Setup(cliCtx, metricsMux, ethNode, logger) } err = ethNode.Serve() diff --git a/diagnostics/mem.go b/diagnostics/mem.go index 00a5d6819e1..55ea9c3a32f 100644 --- a/diagnostics/mem.go +++ b/diagnostics/mem.go @@ -4,7 +4,6 @@ import ( "encoding/json" "github.com/ledgerwatch/erigon-lib/common/dbg" "net/http" - "time" ) func SetupMemAccess(metricsMux *http.ServeMux) { @@ -13,26 +12,17 @@ func SetupMemAccess(metricsMux *http.ServeMux) { w.Header().Set("Content-Type", "application/json") writeMem(w) }) - - // update prometheus memory stats at least every 30 seconds - go func() { - ticker := time.NewTicker(30 * time.Second) - defer ticker.Stop() - - for { - dbg.GetMemUsage() - <-ticker.C - } - }() } func writeMem(w http.ResponseWriter) { - memStats, err := dbg.GetMemUsage() + memStats, err := dbg.ReadVirtualMemStats() if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - json.NewEncoder(w).Encode(memStats) + if err := json.NewEncoder(w).Encode(memStats); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } } diff --git a/diagnostics/setup.go b/diagnostics/setup.go index 2773c9a74bb..201c69f3dcb 100644 --- a/diagnostics/setup.go +++ b/diagnostics/setup.go @@ -1,14 +1,17 @@ package diagnostics import ( + "github.com/ledgerwatch/erigon-lib/common/dbg" + "github.com/ledgerwatch/log/v3" "net/http" "strings" + "time" "github.com/ledgerwatch/erigon/turbo/node" "github.com/urfave/cli/v2" ) -func Setup(ctx *cli.Context, metricsMux *http.ServeMux, node *node.ErigonNode) { +func Setup(ctx *cli.Context, metricsMux *http.ServeMux, node *node.ErigonNode, logger log.Logger) { debugMux := http.NewServeMux() diagnostic := NewDiagnosticClient(ctx, debugMux, node) @@ -33,4 +36,24 @@ func Setup(ctx *cli.Context, metricsMux *http.ServeMux, node *node.ErigonNode) { SetupStagesAccess(debugMux, diagnostic) SetupMemAccess(debugMux) + // setup periodic logging and prometheus updates + go func() { + logEvery := time.NewTicker(180 * time.Second) + defer logEvery.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-logEvery.C: + memStats, err := dbg.ReadVirtualMemStats() + if err != nil { + logger.Warn("[mem] error reading virtual memory stats", "err", err) + } + + logger.Info("[mem] virtual memory stats", memStats) + dbg.UpdatePrometheusVirtualMemStats(memStats) + } + } + }() } diff --git a/erigon-lib/common/dbg/mem.go b/erigon-lib/common/dbg/mem.go index c4f0dd00f6c..16763660272 100644 --- a/erigon-lib/common/dbg/mem.go +++ b/erigon-lib/common/dbg/mem.go @@ -19,7 +19,7 @@ var ( memSwapGauge = metrics.NewGauge(`mem_swap`) ) -func GetMemUsage() (process.MemoryMapsStat, error) { +func ReadVirtualMemStats() (process.MemoryMapsStat, error) { pid := os.Getpid() proc, err := process.NewProcess(int32(pid)) @@ -33,16 +33,18 @@ func GetMemUsage() (process.MemoryMapsStat, error) { return process.MemoryMapsStat{}, err } - memRssGauge.SetUint64((*memoryMaps)[0].Rss) - memSizeGauge.SetUint64((*memoryMaps)[0].Size) - memPssGauge.SetUint64((*memoryMaps)[0].Pss) - memSharedCleanGauge.SetUint64((*memoryMaps)[0].SharedClean) - memSharedDirtyGauge.SetUint64((*memoryMaps)[0].SharedDirty) - memPrivateCleanGauge.SetUint64((*memoryMaps)[0].PrivateClean) - memPrivateDirtyGauge.SetUint64((*memoryMaps)[0].PrivateDirty) - memReferencedGauge.SetUint64((*memoryMaps)[0].Referenced) - memAnonymousGauge.SetUint64((*memoryMaps)[0].Anonymous) - memSwapGauge.SetUint64((*memoryMaps)[0].Swap) - return (*memoryMaps)[0], nil } + +func UpdatePrometheusVirtualMemStats(p process.MemoryMapsStat) { + memRssGauge.SetUint64(p.Rss) + memSizeGauge.SetUint64(p.Size) + memPssGauge.SetUint64(p.Pss) + memSharedCleanGauge.SetUint64(p.SharedClean) + memSharedDirtyGauge.SetUint64(p.SharedDirty) + memPrivateCleanGauge.SetUint64(p.PrivateClean) + memPrivateDirtyGauge.SetUint64(p.PrivateDirty) + memReferencedGauge.SetUint64(p.Referenced) + memAnonymousGauge.SetUint64(p.Anonymous) + memSwapGauge.SetUint64(p.Swap) +} diff --git a/turbo/snapshotsync/snapshotsync.go b/turbo/snapshotsync/snapshotsync.go index 736b23df937..235b3d51c99 100644 --- a/turbo/snapshotsync/snapshotsync.go +++ b/turbo/snapshotsync/snapshotsync.go @@ -185,7 +185,7 @@ Loop: } dbg.ReadMemStats(&m) - memStats, err := dbg.GetMemUsage() + memStats, err := dbg.ReadVirtualMemStats() if err != nil { log.Warn("Error reading memory stats", "err", err) From 20543c6e3df41ca00084b1057ca0b1f32a1029c9 Mon Sep 17 00:00:00 2001 From: Shoham Chakraborty Date: Wed, 21 Feb 2024 10:42:22 +0800 Subject: [PATCH 05/16] Update all usages of Setup --- cmd/devnet/devnet/node.go | 2 +- tests/testdata | 2 +- turbo/app/snapshots_cmd.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/devnet/devnet/node.go b/cmd/devnet/devnet/node.go index 4c372721a03..7a0f816b527 100644 --- a/cmd/devnet/devnet/node.go +++ b/cmd/devnet/devnet/node.go @@ -177,7 +177,7 @@ func (n *devnetNode) run(ctx *cli.Context) error { n.ethNode, err = enode.New(ctx.Context, n.nodeCfg, n.ethCfg, logger) if metricsMux != nil { - diagnostics.Setup(ctx, metricsMux, n.ethNode) + diagnostics.Setup(ctx, metricsMux, n.ethNode, logger) } n.Lock() diff --git a/tests/testdata b/tests/testdata index 06e276776bc..428f218d7d6 160000 --- a/tests/testdata +++ b/tests/testdata @@ -1 +1 @@ -Subproject commit 06e276776bc87817c38f6efb492bf6f4527fa904 +Subproject commit 428f218d7d6f4a52544e12684afbfe6e2882ffbf diff --git a/turbo/app/snapshots_cmd.go b/turbo/app/snapshots_cmd.go index 79d234ec572..65133d8ce72 100644 --- a/turbo/app/snapshots_cmd.go +++ b/turbo/app/snapshots_cmd.go @@ -682,7 +682,7 @@ func doUploaderCommand(cliCtx *cli.Context) error { } if metricsMux != nil { - diagnostics.Setup(cliCtx, metricsMux, ethNode) + diagnostics.Setup(cliCtx, metricsMux, ethNode, logger) } err = ethNode.Serve() From 0f3d967b8149f579f92c5eff137bf725ea6463a5 Mon Sep 17 00:00:00 2001 From: Shoham Chakraborty Date: Wed, 21 Feb 2024 11:09:56 +0800 Subject: [PATCH 06/16] Fix logging --- diagnostics/setup.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/diagnostics/setup.go b/diagnostics/setup.go index 201c69f3dcb..b3ca065c886 100644 --- a/diagnostics/setup.go +++ b/diagnostics/setup.go @@ -4,6 +4,7 @@ import ( "github.com/ledgerwatch/erigon-lib/common/dbg" "github.com/ledgerwatch/log/v3" "net/http" + "reflect" "strings" "time" @@ -51,7 +52,20 @@ func Setup(ctx *cli.Context, metricsMux *http.ServeMux, node *node.ErigonNode, l logger.Warn("[mem] error reading virtual memory stats", "err", err) } - logger.Info("[mem] virtual memory stats", memStats) + typ := reflect.TypeOf(memStats) + val := reflect.ValueOf(memStats) + + var slice []interface{} + for i := 0; i < typ.NumField(); i++ { + t := typ.Field(i).Name + if t == "Path" { // always empty for aggregated smap statistics + continue + } + + slice = append(slice, t, val.Field(i).Interface()) + } + + logger.Info("[mem] virtual memory stats", slice...) dbg.UpdatePrometheusVirtualMemStats(memStats) } } From 32f660ff6b839889060f47429b3fb1c768efa2f9 Mon Sep 17 00:00:00 2001 From: Shoham Chakraborty Date: Thu, 22 Feb 2024 10:51:04 +0800 Subject: [PATCH 07/16] Handle cross-platform discrepancies, move goroutine --- diagnostics/mem.go | 5 ++- diagnostics/setup.go | 41 +----------------- erigon-lib/common/mem/mem.go | 14 ++++++ .../common/{dbg/mem.go => mem/mem_linux.go} | 4 +- eth/backend.go | 43 +++++++++++++++++++ turbo/snapshotsync/snapshotsync.go | 7 --- 6 files changed, 65 insertions(+), 49 deletions(-) create mode 100644 erigon-lib/common/mem/mem.go rename erigon-lib/common/{dbg/mem.go => mem/mem_linux.go} (98%) diff --git a/diagnostics/mem.go b/diagnostics/mem.go index 55ea9c3a32f..73bb4e6f940 100644 --- a/diagnostics/mem.go +++ b/diagnostics/mem.go @@ -2,8 +2,9 @@ package diagnostics import ( "encoding/json" - "github.com/ledgerwatch/erigon-lib/common/dbg" "net/http" + + "github.com/ledgerwatch/erigon-lib/common/mem" ) func SetupMemAccess(metricsMux *http.ServeMux) { @@ -15,7 +16,7 @@ func SetupMemAccess(metricsMux *http.ServeMux) { } func writeMem(w http.ResponseWriter) { - memStats, err := dbg.ReadVirtualMemStats() + memStats, err := mem.ReadVirtualMemStats() if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) diff --git a/diagnostics/setup.go b/diagnostics/setup.go index b3ca065c886..8a1d81c82d1 100644 --- a/diagnostics/setup.go +++ b/diagnostics/setup.go @@ -1,15 +1,11 @@ package diagnostics import ( - "github.com/ledgerwatch/erigon-lib/common/dbg" + "github.com/ledgerwatch/erigon/turbo/node" "github.com/ledgerwatch/log/v3" + "github.com/urfave/cli/v2" "net/http" - "reflect" "strings" - "time" - - "github.com/ledgerwatch/erigon/turbo/node" - "github.com/urfave/cli/v2" ) func Setup(ctx *cli.Context, metricsMux *http.ServeMux, node *node.ErigonNode, logger log.Logger) { @@ -37,37 +33,4 @@ func Setup(ctx *cli.Context, metricsMux *http.ServeMux, node *node.ErigonNode, l SetupStagesAccess(debugMux, diagnostic) SetupMemAccess(debugMux) - // setup periodic logging and prometheus updates - go func() { - logEvery := time.NewTicker(180 * time.Second) - defer logEvery.Stop() - - for { - select { - case <-ctx.Done(): - return - case <-logEvery.C: - memStats, err := dbg.ReadVirtualMemStats() - if err != nil { - logger.Warn("[mem] error reading virtual memory stats", "err", err) - } - - typ := reflect.TypeOf(memStats) - val := reflect.ValueOf(memStats) - - var slice []interface{} - for i := 0; i < typ.NumField(); i++ { - t := typ.Field(i).Name - if t == "Path" { // always empty for aggregated smap statistics - continue - } - - slice = append(slice, t, val.Field(i).Interface()) - } - - logger.Info("[mem] virtual memory stats", slice...) - dbg.UpdatePrometheusVirtualMemStats(memStats) - } - } - }() } diff --git a/erigon-lib/common/mem/mem.go b/erigon-lib/common/mem/mem.go new file mode 100644 index 00000000000..b7763c6be28 --- /dev/null +++ b/erigon-lib/common/mem/mem.go @@ -0,0 +1,14 @@ +//go:build !linux + +package mem + +import ( + "errors" + "github.com/shirou/gopsutil/v3/process" +) + +func ReadVirtualMemStats() (process.MemoryMapsStat, error) { + return process.MemoryMapsStat{}, errors.New("unsupported platform") +} + +func UpdatePrometheusVirtualMemStats(p process.MemoryMapsStat) {} diff --git a/erigon-lib/common/dbg/mem.go b/erigon-lib/common/mem/mem_linux.go similarity index 98% rename from erigon-lib/common/dbg/mem.go rename to erigon-lib/common/mem/mem_linux.go index 16763660272..4baf39eadc6 100644 --- a/erigon-lib/common/dbg/mem.go +++ b/erigon-lib/common/mem/mem_linux.go @@ -1,4 +1,6 @@ -package dbg +//go:build linux + +package mem import ( "github.com/ledgerwatch/erigon-lib/metrics" diff --git a/eth/backend.go b/eth/backend.go index f389a9463c4..d9b3ed1ab8e 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -21,11 +21,13 @@ import ( "context" "errors" "fmt" + "github.com/ledgerwatch/erigon-lib/common/mem" "io/fs" "math/big" "net" "os" "path/filepath" + "reflect" "strconv" "strings" "sync" @@ -488,6 +490,47 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger }() } + // setup periodic logging and prometheus updates + go func() { + logEvery := time.NewTicker(180 * time.Second) + defer logEvery.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-logEvery.C: + memStats, err := mem.ReadVirtualMemStats() + if err != nil { + logger.Warn("[mem] error reading virtual memory stats", "err", err) + + if err.Error() == "unsupported platform" { + return // we do not want to pollute the log with repeated warnings + } else { + continue + } + } + + // convert to slice + typ := reflect.TypeOf(memStats) + val := reflect.ValueOf(memStats) + + var slice []interface{} + for i := 0; i < typ.NumField(); i++ { + t := typ.Field(i).Name + if t == "Path" { // always empty for aggregated smap statistics + continue + } + + slice = append(slice, t, val.Field(i).Interface()) + } + + logger.Info("[mem] virtual memory stats", slice...) + mem.UpdatePrometheusVirtualMemStats(memStats) + } + } + }() + var currentBlock *types.Block if err := chainKv.View(context.Background(), func(tx kv.Tx) error { currentBlock, err = blockReader.CurrentBlock(tx) diff --git a/turbo/snapshotsync/snapshotsync.go b/turbo/snapshotsync/snapshotsync.go index 235b3d51c99..a33b5e2efa1 100644 --- a/turbo/snapshotsync/snapshotsync.go +++ b/turbo/snapshotsync/snapshotsync.go @@ -185,12 +185,6 @@ Loop: } dbg.ReadMemStats(&m) - memStats, err := dbg.ReadVirtualMemStats() - - if err != nil { - log.Warn("Error reading memory stats", "err", err) - } - downloadTimeLeft := calculateTime(stats.BytesTotal-stats.BytesCompleted, stats.DownloadRate) suffix := "downloading" if stats.Progress > 0 && stats.DownloadRate == 0 { @@ -207,7 +201,6 @@ Loop: "files", stats.FilesTotal, "connections", stats.ConnectionsTotal, "alloc", common.ByteCount(m.Alloc), "sys", common.ByteCount(m.Sys), - "shared-usage-clean", common.ByteCount(memStats.SharedClean), ) } } From 9c65bf0e44022780446a0e29b8fd4b67e182026f Mon Sep 17 00:00:00 2001 From: Shoham Chakraborty Date: Thu, 22 Feb 2024 10:53:42 +0800 Subject: [PATCH 08/16] Rename metrics --- erigon-lib/common/mem/mem_linux.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/erigon-lib/common/mem/mem_linux.go b/erigon-lib/common/mem/mem_linux.go index 4baf39eadc6..f0b6ac28186 100644 --- a/erigon-lib/common/mem/mem_linux.go +++ b/erigon-lib/common/mem/mem_linux.go @@ -12,10 +12,10 @@ var ( memRssGauge = metrics.NewGauge(`mem_rss`) memSizeGauge = metrics.NewGauge(`mem_size`) memPssGauge = metrics.NewGauge(`mem_pss`) - memSharedCleanGauge = metrics.NewGauge(`mem_shared_clean`) - memSharedDirtyGauge = metrics.NewGauge(`mem_shared_dirty`) - memPrivateCleanGauge = metrics.NewGauge(`mem_private_clean`) - memPrivateDirtyGauge = metrics.NewGauge(`mem_private_dirty`) + memSharedCleanGauge = metrics.NewGauge(`mem_shared{type="clean"}"`) + memSharedDirtyGauge = metrics.NewGauge(`mem_shared{type="dirty"}`) + memPrivateCleanGauge = metrics.NewGauge(`mem_private{type="clean"}`) + memPrivateDirtyGauge = metrics.NewGauge(`mem_private{type="dirty"}`) memReferencedGauge = metrics.NewGauge(`mem_referenced`) memAnonymousGauge = metrics.NewGauge(`mem_anonymous`) memSwapGauge = metrics.NewGauge(`mem_swap`) @@ -24,13 +24,11 @@ var ( func ReadVirtualMemStats() (process.MemoryMapsStat, error) { pid := os.Getpid() proc, err := process.NewProcess(int32(pid)) - if err != nil { return process.MemoryMapsStat{}, err } memoryMaps, err := proc.MemoryMaps(true) - if err != nil { return process.MemoryMapsStat{}, err } From 0eeda281ad044b8e2aa0debc5bcfff3fbadcb0e3 Mon Sep 17 00:00:00 2001 From: Shoham Chakraborty Date: Thu, 22 Feb 2024 10:55:22 +0800 Subject: [PATCH 09/16] Remove unused changes --- erigon-lib/diagnostics/entities.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/erigon-lib/diagnostics/entities.go b/erigon-lib/diagnostics/entities.go index a49edf89174..f91ab9cea6e 100644 --- a/erigon-lib/diagnostics/entities.go +++ b/erigon-lib/diagnostics/entities.go @@ -16,8 +16,6 @@ package diagnostics -import "github.com/shirou/gopsutil/v3/process" - type PeerStatisticsGetter interface { GetPeersStatistics() map[string]*PeerStatistics } @@ -75,10 +73,6 @@ type SnapshotSegmentIndexingFinishedUpdate struct { SegmentName string `json:"segmentName"` } -type MemUsage struct { - stats process.MemoryMapsStat `json:"stats"` -} - func (ti SnapshotDownloadStatistics) Type() Type { return TypeOf(ti) } From 09d5b4205a982108cc422947e11065ababdab384 Mon Sep 17 00:00:00 2001 From: Shoham Chakraborty Date: Thu, 22 Feb 2024 12:44:46 +0800 Subject: [PATCH 10/16] Fix submodule commit --- tests/testdata | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testdata b/tests/testdata index 5bf1fff257c..853b1e03b10 160000 --- a/tests/testdata +++ b/tests/testdata @@ -1 +1 @@ -Subproject commit 5bf1fff257c0662bc526c27d7546d470025b3239 +Subproject commit 853b1e03b1078d370614002851ba1ee9803d9fcf From 6c8e7ab0f95c44e2073bd703c6fc36330037bc4d Mon Sep 17 00:00:00 2001 From: Shoham Chakraborty Date: Thu, 22 Feb 2024 12:47:59 +0800 Subject: [PATCH 11/16] Add log for exiting goroutine --- eth/backend.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/eth/backend.go b/eth/backend.go index 60d1586b77f..9932e8e7253 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -502,8 +502,9 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger if err != nil { logger.Warn("[mem] error reading virtual memory stats", "err", err) - if err.Error() == "unsupported platform" { - return // we do not want to pollute the log with repeated warnings + if err.Error() == "unsupported platform" { // do not pollute the log with repeated warnings + logger.Warn("[mem] platform unsupported, exiting virtual memory stats goroutine") + return } else { continue } From c4121d6f3778e95e44f72ce7a18cae031cd2f9bb Mon Sep 17 00:00:00 2001 From: Shoham Chakraborty Date: Thu, 22 Feb 2024 13:01:21 +0800 Subject: [PATCH 12/16] Handle logging using build tags, remove unused logger --- cmd/devnet/devnet/node.go | 2 +- cmd/erigon/main.go | 2 +- diagnostics/mem.go | 1 - diagnostics/setup.go | 9 +++--- erigon-lib/common/mem/mem.go | 3 ++ erigon-lib/common/mem/mem_linux.go | 44 +++++++++++++++++++++++++++-- eth/backend.go | 45 ++---------------------------- turbo/app/snapshots_cmd.go | 2 +- 8 files changed, 56 insertions(+), 52 deletions(-) diff --git a/cmd/devnet/devnet/node.go b/cmd/devnet/devnet/node.go index 22e34020329..33f716aa3f3 100644 --- a/cmd/devnet/devnet/node.go +++ b/cmd/devnet/devnet/node.go @@ -185,7 +185,7 @@ func (n *devnetNode) run(ctx *cli.Context) error { n.ethNode, err = enode.New(ctx.Context, n.nodeCfg, n.ethCfg, logger) if metricsMux != nil { - diagnostics.Setup(ctx, metricsMux, n.ethNode, logger) + diagnostics.Setup(ctx, metricsMux, n.ethNode) } n.Lock() diff --git a/cmd/erigon/main.go b/cmd/erigon/main.go index 4ef2aa77fc0..b3dd55dcdb5 100644 --- a/cmd/erigon/main.go +++ b/cmd/erigon/main.go @@ -69,7 +69,7 @@ func runErigon(cliCtx *cli.Context) error { } if metricsMux != nil { - diagnostics.Setup(cliCtx, metricsMux, ethNode, logger) + diagnostics.Setup(cliCtx, metricsMux, ethNode) } err = ethNode.Serve() diff --git a/diagnostics/mem.go b/diagnostics/mem.go index 73bb4e6f940..e1d25e210b7 100644 --- a/diagnostics/mem.go +++ b/diagnostics/mem.go @@ -17,7 +17,6 @@ func SetupMemAccess(metricsMux *http.ServeMux) { func writeMem(w http.ResponseWriter) { memStats, err := mem.ReadVirtualMemStats() - if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return diff --git a/diagnostics/setup.go b/diagnostics/setup.go index 8a1d81c82d1..d0ec5fbfa43 100644 --- a/diagnostics/setup.go +++ b/diagnostics/setup.go @@ -1,14 +1,15 @@ package diagnostics import ( - "github.com/ledgerwatch/erigon/turbo/node" - "github.com/ledgerwatch/log/v3" - "github.com/urfave/cli/v2" "net/http" "strings" + + "github.com/urfave/cli/v2" + + "github.com/ledgerwatch/erigon/turbo/node" ) -func Setup(ctx *cli.Context, metricsMux *http.ServeMux, node *node.ErigonNode, logger log.Logger) { +func Setup(ctx *cli.Context, metricsMux *http.ServeMux, node *node.ErigonNode) { debugMux := http.NewServeMux() diagnostic := NewDiagnosticClient(ctx, debugMux, node) diff --git a/erigon-lib/common/mem/mem.go b/erigon-lib/common/mem/mem.go index b7763c6be28..6775f04e059 100644 --- a/erigon-lib/common/mem/mem.go +++ b/erigon-lib/common/mem/mem.go @@ -4,6 +4,7 @@ package mem import ( "errors" + "github.com/shirou/gopsutil/v3/process" ) @@ -12,3 +13,5 @@ func ReadVirtualMemStats() (process.MemoryMapsStat, error) { } func UpdatePrometheusVirtualMemStats(p process.MemoryMapsStat) {} + +func LogVirtualMemStats(ctx context.Context, logger log.Logger) {} diff --git a/erigon-lib/common/mem/mem_linux.go b/erigon-lib/common/mem/mem_linux.go index fa02e99eceb..f7981cb1628 100644 --- a/erigon-lib/common/mem/mem_linux.go +++ b/erigon-lib/common/mem/mem_linux.go @@ -3,9 +3,15 @@ package mem import ( - "github.com/ledgerwatch/erigon-lib/metrics" - "github.com/shirou/gopsutil/v3/process" + "context" "os" + "reflect" + "time" + + "github.com/ledgerwatch/log/v3" + "github.com/shirou/gopsutil/v3/process" + + "github.com/ledgerwatch/erigon-lib/metrics" ) var ( @@ -48,3 +54,37 @@ func UpdatePrometheusVirtualMemStats(p process.MemoryMapsStat) { memAnonymousGauge.SetUint64(p.Anonymous) memSwapGauge.SetUint64(p.Swap) } + +func LogVirtualMemStats(ctx context.Context, logger log.Logger) { + logEvery := time.NewTicker(180 * time.Second) + defer logEvery.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-logEvery.C: + memStats, err := ReadVirtualMemStats() + if err != nil { + logger.Warn("[mem] error reading virtual memory stats", "err", err) + } + + // convert to slice + typ := reflect.TypeOf(memStats) + val := reflect.ValueOf(memStats) + + var slice []interface{} + for i := 0; i < typ.NumField(); i++ { + t := typ.Field(i).Name + if t == "Path" { // always empty for aggregated smap statistics + continue + } + + slice = append(slice, t, val.Field(i).Interface()) + } + + logger.Info("[mem] virtual memory stats", slice...) + UpdatePrometheusVirtualMemStats(memStats) + } + } +} diff --git a/eth/backend.go b/eth/backend.go index 9932e8e7253..5c16c92a00f 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -21,19 +21,19 @@ import ( "context" "errors" "fmt" - "github.com/ledgerwatch/erigon-lib/common/mem" "io/fs" "math/big" "net" "os" "path/filepath" - "reflect" "strconv" "strings" "sync" "sync/atomic" "time" + "github.com/ledgerwatch/erigon-lib/common/mem" + "github.com/erigontech/mdbx-go/mdbx" lru "github.com/hashicorp/golang-lru/arc/v2" "github.com/holiman/uint256" @@ -489,46 +489,7 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger } // setup periodic logging and prometheus updates - go func() { - logEvery := time.NewTicker(180 * time.Second) - defer logEvery.Stop() - - for { - select { - case <-ctx.Done(): - return - case <-logEvery.C: - memStats, err := mem.ReadVirtualMemStats() - if err != nil { - logger.Warn("[mem] error reading virtual memory stats", "err", err) - - if err.Error() == "unsupported platform" { // do not pollute the log with repeated warnings - logger.Warn("[mem] platform unsupported, exiting virtual memory stats goroutine") - return - } else { - continue - } - } - - // convert to slice - typ := reflect.TypeOf(memStats) - val := reflect.ValueOf(memStats) - - var slice []interface{} - for i := 0; i < typ.NumField(); i++ { - t := typ.Field(i).Name - if t == "Path" { // always empty for aggregated smap statistics - continue - } - - slice = append(slice, t, val.Field(i).Interface()) - } - - logger.Info("[mem] virtual memory stats", slice...) - mem.UpdatePrometheusVirtualMemStats(memStats) - } - } - }() + go mem.LogVirtualMemStats(ctx, logger) var currentBlock *types.Block if err := chainKv.View(context.Background(), func(tx kv.Tx) error { diff --git a/turbo/app/snapshots_cmd.go b/turbo/app/snapshots_cmd.go index 32de602b284..7416b991129 100644 --- a/turbo/app/snapshots_cmd.go +++ b/turbo/app/snapshots_cmd.go @@ -677,7 +677,7 @@ func doUploaderCommand(cliCtx *cli.Context) error { } if metricsMux != nil { - diagnostics.Setup(cliCtx, metricsMux, ethNode, logger) + diagnostics.Setup(cliCtx, metricsMux, ethNode) } err = ethNode.Serve() From 9b890aab27f786bec7553bd19e57b5c6879b2296 Mon Sep 17 00:00:00 2001 From: Shoham Chakraborty Date: Thu, 22 Feb 2024 13:27:33 +0800 Subject: [PATCH 13/16] Convert slice inside function --- erigon-lib/common/mem/mem_linux.go | 40 +++++++++++++++++++----------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/erigon-lib/common/mem/mem_linux.go b/erigon-lib/common/mem/mem_linux.go index f7981cb1628..c10b303ec35 100644 --- a/erigon-lib/common/mem/mem_linux.go +++ b/erigon-lib/common/mem/mem_linux.go @@ -14,6 +14,28 @@ import ( "github.com/ledgerwatch/erigon-lib/metrics" ) +type VirtualMemStat struct { + process.MemoryMapsStat +} + +// Fields converts VirtualMemStat to slice +func (m VirtualMemStat) Fields() []interface{} { + typ := reflect.TypeOf(m.MemoryMapsStat) + val := reflect.ValueOf(m.MemoryMapsStat) + + var s []interface{} + for i := 0; i < typ.NumField(); i++ { + t := typ.Field(i).Name + if t == "Path" { // always empty for aggregated smap statistics + continue + } + + s = append(s, t, val.Field(i).Interface()) + } + + return s +} + var ( memRssGauge = metrics.NewGauge(`mem_rss`) memSizeGauge = metrics.NewGauge(`mem_size`) @@ -67,23 +89,11 @@ func LogVirtualMemStats(ctx context.Context, logger log.Logger) { memStats, err := ReadVirtualMemStats() if err != nil { logger.Warn("[mem] error reading virtual memory stats", "err", err) + continue } - // convert to slice - typ := reflect.TypeOf(memStats) - val := reflect.ValueOf(memStats) - - var slice []interface{} - for i := 0; i < typ.NumField(); i++ { - t := typ.Field(i).Name - if t == "Path" { // always empty for aggregated smap statistics - continue - } - - slice = append(slice, t, val.Field(i).Interface()) - } - - logger.Info("[mem] virtual memory stats", slice...) + v := VirtualMemStat{memStats} + logger.Info("[mem] virtual memory stats", v.Fields()...) UpdatePrometheusVirtualMemStats(memStats) } } From 61474abd6ac7152c12e223d4ec72138935c55696 Mon Sep 17 00:00:00 2001 From: Shoham Chakraborty Date: Thu, 22 Feb 2024 13:46:33 +0800 Subject: [PATCH 14/16] Fix mac build --- erigon-lib/common/mem/mem.go | 1 + 1 file changed, 1 insertion(+) diff --git a/erigon-lib/common/mem/mem.go b/erigon-lib/common/mem/mem.go index 6775f04e059..0f67415c36a 100644 --- a/erigon-lib/common/mem/mem.go +++ b/erigon-lib/common/mem/mem.go @@ -3,6 +3,7 @@ package mem import ( + "context" "errors" "github.com/shirou/gopsutil/v3/process" From 147020ecc26caad4d1189f47306f7efc765ac5ab Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 27 Feb 2024 11:01:43 +0700 Subject: [PATCH 15/16] save --- erigon-lib/common/mem/mem.go | 1 + 1 file changed, 1 insertion(+) diff --git a/erigon-lib/common/mem/mem.go b/erigon-lib/common/mem/mem.go index 0f67415c36a..a3be29fa14e 100644 --- a/erigon-lib/common/mem/mem.go +++ b/erigon-lib/common/mem/mem.go @@ -6,6 +6,7 @@ import ( "context" "errors" + "github.com/ledgerwatch/log/v3" "github.com/shirou/gopsutil/v3/process" ) From 16bfa6811e911ce17c3473c7672cd16ade4cb0b5 Mon Sep 17 00:00:00 2001 From: Shoham Chakraborty Date: Tue, 27 Feb 2024 21:31:24 +0800 Subject: [PATCH 16/16] Fix lint --- erigon-lib/go.mod | 5 +---- erigon-lib/go.sum | 3 --- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/erigon-lib/go.mod b/erigon-lib/go.mod index fba09b93c2f..23b9735eb3a 100644 --- a/erigon-lib/go.mod +++ b/erigon-lib/go.mod @@ -42,7 +42,6 @@ require ( golang.org/x/exp v0.0.0-20230905200255-921286631fa9 golang.org/x/sync v0.6.0 golang.org/x/sys v0.16.0 - golang.org/x/sys v0.16.0 golang.org/x/time v0.5.0 google.golang.org/grpc v1.60.1 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0 @@ -114,11 +113,9 @@ require ( github.com/pion/webrtc/v3 v3.1.42 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect - github.com/prometheus/common v0.45.0 // indirect - github.com/prometheus/procfs v0.12.0 // indirect - github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect diff --git a/erigon-lib/go.sum b/erigon-lib/go.sum index 53180f4a891..8535fc6ea01 100644 --- a/erigon-lib/go.sum +++ b/erigon-lib/go.sum @@ -209,8 +209,6 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -533,7 +531,6 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=