From 344c18e254f630b6ae62ce65fc73009b21bb539b Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Mon, 22 May 2023 18:37:44 -0400 Subject: [PATCH 1/2] Fetch process resource stats as best-effort --- node/node.go | 1 + utils/resource/usage.go | 45 ++++++++++++++++++++++++++++++++++++----- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/node/node.go b/node/node.go index 948821ce4278..13bc4912ad05 100644 --- a/node/node.go +++ b/node/node.go @@ -1197,6 +1197,7 @@ func (n *Node) initVdrs() validators.Set { // Initialize [n.resourceManager]. func (n *Node) initResourceManager(reg prometheus.Registerer) error { n.resourceManager = resource.NewManager( + n.Log, n.Config.DatabaseConfig.Path, n.Config.SystemTrackerFrequency, n.Config.SystemTrackerCPUHalflife, diff --git a/utils/resource/usage.go b/utils/resource/usage.go index 2c83aa0db034..282524b2755b 100644 --- a/utils/resource/usage.go +++ b/utils/resource/usage.go @@ -8,8 +8,12 @@ import ( "sync" "time" + "github.com/shirou/gopsutil/cpu" "github.com/shirou/gopsutil/process" + "go.uber.org/zap" + + "github.com/ava-labs/avalanchego/utils/logging" "github.com/ava-labs/avalanchego/utils/storage" ) @@ -62,6 +66,8 @@ type Manager interface { } type manager struct { + log logging.Logger + processesLock sync.Mutex processes map[int]*proc @@ -78,8 +84,15 @@ type manager struct { onClose chan struct{} } -func NewManager(diskPath string, frequency, cpuHalflife, diskHalflife time.Duration) Manager { +func NewManager( + log logging.Logger, + diskPath string, + frequency time.Duration, + cpuHalflife time.Duration, + diskHalflife time.Duration, +) Manager { m := &manager{ + log: log, processes: make(map[int]*proc), onClose: make(chan struct{}), availableDiskBytes: math.MaxUint64, @@ -115,7 +128,10 @@ func (m *manager) TrackProcess(pid int) { return } - process := &proc{p: p} + process := &proc{ + p: p, + log: m.log, + } m.processesLock.Lock() m.processes[pid] = process @@ -149,6 +165,12 @@ func (m *manager) update(diskPath string, frequency, cpuHalflife, diskHalflife t currentScaledWriteUsage := newDiskWeight * currentWriteUsage availableBytes, getBytesErr := storage.AvailableBytes(diskPath) + if getBytesErr != nil { + m.log.Debug("failed to lookup resource", + zap.String("resource", "system disk"), + zap.Error(getBytesErr), + ) + } m.usageLock.Lock() m.cpuUsage = oldCPUWeight*m.cpuUsage + currentScaledCPUUsage @@ -193,7 +215,8 @@ func (m *manager) getActiveUsage(secondsSinceLastUpdate float64) (float64, float } type proc struct { - p *process.Process + p *process.Process + log logging.Logger initialized bool @@ -212,12 +235,24 @@ func (p *proc) getActiveUsage(secondsSinceLastUpdate float64) (float64, float64, // assume that the utilization is 0. times, err := p.p.Times() if err != nil { - return 0, 0, 0 + p.log.Debug("failed to lookup resource", + zap.String("resource", "process CPU"), + zap.Int32("pid", p.p.Pid), + zap.Error(err), + ) + times = &cpu.TimesStat{} } + // Note: IOCounters is not implemented on macos and therefore always returns + // an error on macos. io, err := p.p.IOCounters() if err != nil { - return 0, 0, 0 + p.log.Debug("failed to lookup resource", + zap.String("resource", "process IO"), + zap.Int32("pid", p.p.Pid), + zap.Error(err), + ) + io = &process.IOCountersStat{} } var ( From 47eeed019d1db9a9a03fa7b0aa3e794583281d16 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Mon, 22 May 2023 18:44:35 -0400 Subject: [PATCH 2/2] fix test --- vms/registry/vm_getter_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vms/registry/vm_getter_test.go b/vms/registry/vm_getter_test.go index ce659cf4716b..a9aef00e9cef 100644 --- a/vms/registry/vm_getter_test.go +++ b/vms/registry/vm_getter_test.go @@ -15,6 +15,7 @@ import ( "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/avalanchego/utils/filesystem" + "github.com/ava-labs/avalanchego/utils/logging" "github.com/ava-labs/avalanchego/utils/resource" "github.com/ava-labs/avalanchego/vms" ) @@ -149,7 +150,7 @@ func initVMGetterTest(t *testing.T) *vmGetterTestResources { FileReader: mockReader, Manager: mockManager, PluginDirectory: pluginDir, - CPUTracker: resource.NewManager("", time.Hour, time.Hour, time.Hour), + CPUTracker: resource.NewManager(logging.NoLog{}, "", time.Hour, time.Hour, time.Hour), }, )