From fadbf514d1b34635fb3b1a550b79541b3660b4e3 Mon Sep 17 00:00:00 2001 From: Alex Kristiansen Date: Wed, 13 May 2020 12:45:57 -0700 Subject: [PATCH 1/3] remove global loggers from libbeat metric and cloudid --- libbeat/cloudid/cloudid.go | 9 ++++--- libbeat/metric/system/memory/memory.go | 3 ++- libbeat/metric/system/process/process.go | 31 +++++++++++++++--------- 3 files changed, 27 insertions(+), 16 deletions(-) diff --git a/libbeat/cloudid/cloudid.go b/libbeat/cloudid/cloudid.go index 5e39dee27f1d..835c563c2c06 100644 --- a/libbeat/cloudid/cloudid.go +++ b/libbeat/cloudid/cloudid.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// package cloudid contains functions for parsing the cloud.id and cloud.auth +// Package cloudid contains functions for parsing the cloud.id and cloud.auth // settings and modifying the configuration to take them into account. package cloudid @@ -149,6 +149,7 @@ func (c *CloudID) decodeCloudAuth() error { // settings. func OverwriteSettings(cfg *common.Config) error { + logger := logp.NewLogger("OverwriteSettings") cloudID, _ := cfg.String("cloud.id", -1) cloudAuth, _ := cfg.String("cloud.auth", -1) @@ -157,9 +158,9 @@ func OverwriteSettings(cfg *common.Config) error { return nil } - logp.Debug("cloudid", "cloud.id: %s, cloud.auth: %s", cloudID, cloudAuth) + logger.Debugf("cloudid", "cloud.id: %s, cloud.auth: %s", cloudID, cloudAuth) if cloudID == "" { - return errors.New("cloud.auth specified but cloud.id is empty. Please specify both.") + return errors.New("cloud.auth specified but cloud.id is empty. Please specify both") } // cloudID overwrites @@ -168,7 +169,7 @@ func OverwriteSettings(cfg *common.Config) error { return errors.Errorf("Error decoding cloud.id: %v", err) } - logp.Info("Setting Elasticsearch and Kibana URLs based on the cloud id: output.elasticsearch.hosts=%s and setup.kibana.host=%s", cid.esURL, cid.kibURL) + logger.Infof("Setting Elasticsearch and Kibana URLs based on the cloud id: output.elasticsearch.hosts=%s and setup.kibana.host=%s", cid.esURL, cid.kibURL) esURLConfig, err := common.NewConfigFrom([]string{cid.ElasticsearchURL()}) if err != nil { diff --git a/libbeat/metric/system/memory/memory.go b/libbeat/metric/system/memory/memory.go index 9351ac08e9e5..a1e198128149 100644 --- a/libbeat/metric/system/memory/memory.go +++ b/libbeat/metric/system/memory/memory.go @@ -78,7 +78,8 @@ func GetSwap() (*SwapStat, error) { // this can provoke too big values for used swap. // Workaround this by assuming that all swap is free in that case. if swap.Free > swap.Total || swap.Used > swap.Total { - logp.Debug("memory", + logger := logp.NewLogger("GetSwap") + logger.Debugf("memory", "Unexpected values for swap memory - total: %v free: %v used: %v. Setting swap used to 0.", swap.Total, swap.Free, swap.Used) swap.Free = swap.Total diff --git a/libbeat/metric/system/process/process.go b/libbeat/metric/system/process/process.go index 7c5d70ebbc69..bc452658719e 100644 --- a/libbeat/metric/system/process/process.go +++ b/libbeat/metric/system/process/process.go @@ -76,6 +76,8 @@ type Stats struct { procRegexps []match.Matcher // List of regular expressions used to whitelist processes. envRegexps []match.Matcher // List of regular expressions used to whitelist env vars. + + logger *logp.Logger } // Ticks of CPU for a process @@ -217,15 +219,11 @@ func getProcEnv(pid int, out common.MapStr, filter func(v string) bool) error { return nil } +// GetProcMemPercentage returns process memory usage as a percent of total memory usage func GetProcMemPercentage(proc *Process, totalPhyMem uint64) float64 { // in unit tests, total_phymem is set to a value greater than zero if totalPhyMem == 0 { - memStat, err := memory.Get() - if err != nil { - logp.Warn("Getting memory details: %v", err) - return 0 - } - totalPhyMem = memStat.Mem.Total + return 0 } perc := (float64(proc.Mem.Resident) / float64(totalPhyMem)) @@ -233,6 +231,7 @@ func GetProcMemPercentage(proc *Process, totalPhyMem uint64) float64 { return common.Round(perc, 4) } +// Pids returns a list of PIDs func Pids() ([]int, error) { pids := sigar.ProcList{} err := pids.Get() @@ -273,6 +272,15 @@ func GetOwnResourceUsageTimeInMillis() (int64, int64, error) { } func (procStats *Stats) getProcessEvent(process *Process) common.MapStr { + + var totalPhyMem uint64 + baseMem, err := memory.Get() + if err != nil { + procStats.logger.Warnf("Getting memory details: %v", err) + } else { + totalPhyMem = baseMem.Mem.Total + } + proc := common.MapStr{ "pid": process.Pid, "ppid": process.Ppid, @@ -284,7 +292,7 @@ func (procStats *Stats) getProcessEvent(process *Process) common.MapStr { "size": process.Mem.Size, "rss": common.MapStr{ "bytes": process.Mem.Resident, - "pct": GetProcMemPercentage(process, 0 /* read total mem usage */), + "pct": GetProcMemPercentage(process, totalPhyMem), }, "share": process.Mem.Share, }, @@ -380,6 +388,7 @@ func (procStats *Stats) matchProcess(name string) bool { // Init initializes a Stats instance. It returns errors if the provided process regexes // cannot be compiled. func (procStats *Stats) Init() error { + procStats.logger = logp.NewLogger("Stats") procStats.ProcsMap = make(ProcsMap) if len(procStats.Procs) == 0 { @@ -431,7 +440,7 @@ func (procStats *Stats) Get() ([]common.MapStr, error) { procStats.ProcsMap = newProcs processes = procStats.includeTopProcesses(processes) - logp.Debug("processes", "Filtered top processes down to %d processes", len(processes)) + procStats.logger.Debugf("processes", "Filtered top processes down to %d processes", len(processes)) procs := make([]common.MapStr, 0, len(processes)) for _, process := range processes { @@ -472,18 +481,18 @@ func (procStats *Stats) getSingleProcess(pid int, newProcs ProcsMap) *Process { process, err := newProcess(pid, cmdline, env) if err != nil { - logp.Debug("processes", "Skip process pid=%d: %v", pid, err) + procStats.logger.Debugf("processes", "Skip process pid=%d: %v", pid, err) return nil } if !procStats.matchProcess(process.Name) { - logp.Debug("processes", "Process name does not matches the provided regex; pid=%d; name=%s: %v", pid, process.Name, err) + procStats.logger.Debugf("processes", "Process name does not matches the provided regex; pid=%d; name=%s: %v", pid, process.Name, err) return nil } err = process.getDetails(procStats.isWhitelistedEnvVar) if err != nil { - logp.Debug("processes", "Error getting details for process %s with pid=%d: %v", process.Name, process.Pid, err) + procStats.logger.Debugf("processes", "Error getting details for process %s with pid=%d: %v", process.Name, process.Pid, err) return nil } From df2b541e1884be8de99161ea6d44ef98d810a5c3 Mon Sep 17 00:00:00 2001 From: Alex Kristiansen Date: Thu, 14 May 2020 07:32:40 -0700 Subject: [PATCH 2/3] fixes --- libbeat/cloudid/cloudid.go | 4 ++-- libbeat/metric/system/memory/memory.go | 2 +- libbeat/metric/system/process/process.go | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/libbeat/cloudid/cloudid.go b/libbeat/cloudid/cloudid.go index 835c563c2c06..ed0512244439 100644 --- a/libbeat/cloudid/cloudid.go +++ b/libbeat/cloudid/cloudid.go @@ -149,7 +149,7 @@ func (c *CloudID) decodeCloudAuth() error { // settings. func OverwriteSettings(cfg *common.Config) error { - logger := logp.NewLogger("OverwriteSettings") + logger := logp.NewLogger("cloudid") cloudID, _ := cfg.String("cloud.id", -1) cloudAuth, _ := cfg.String("cloud.auth", -1) @@ -158,7 +158,7 @@ func OverwriteSettings(cfg *common.Config) error { return nil } - logger.Debugf("cloudid", "cloud.id: %s, cloud.auth: %s", cloudID, cloudAuth) + logger.Debugf("cloud.id: %s, cloud.auth: %s", cloudID, cloudAuth) if cloudID == "" { return errors.New("cloud.auth specified but cloud.id is empty. Please specify both") } diff --git a/libbeat/metric/system/memory/memory.go b/libbeat/metric/system/memory/memory.go index a1e198128149..72e351db384f 100644 --- a/libbeat/metric/system/memory/memory.go +++ b/libbeat/metric/system/memory/memory.go @@ -78,7 +78,7 @@ func GetSwap() (*SwapStat, error) { // this can provoke too big values for used swap. // Workaround this by assuming that all swap is free in that case. if swap.Free > swap.Total || swap.Used > swap.Total { - logger := logp.NewLogger("GetSwap") + logger := logp.NewLogger("memory") logger.Debugf("memory", "Unexpected values for swap memory - total: %v free: %v used: %v. Setting swap used to 0.", swap.Total, swap.Free, swap.Used) diff --git a/libbeat/metric/system/process/process.go b/libbeat/metric/system/process/process.go index bc452658719e..5855ec2b6c6e 100644 --- a/libbeat/metric/system/process/process.go +++ b/libbeat/metric/system/process/process.go @@ -221,7 +221,6 @@ func getProcEnv(pid int, out common.MapStr, filter func(v string) bool) error { // GetProcMemPercentage returns process memory usage as a percent of total memory usage func GetProcMemPercentage(proc *Process, totalPhyMem uint64) float64 { - // in unit tests, total_phymem is set to a value greater than zero if totalPhyMem == 0 { return 0 } From f2d7525b7a2ad201b085dbc6e51f452e80e7b3e8 Mon Sep 17 00:00:00 2001 From: Alex Kristiansen Date: Thu, 14 May 2020 07:38:27 -0700 Subject: [PATCH 3/3] remove global selector --- libbeat/metric/system/process/process.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/libbeat/metric/system/process/process.go b/libbeat/metric/system/process/process.go index 5855ec2b6c6e..03ed5085c44b 100644 --- a/libbeat/metric/system/process/process.go +++ b/libbeat/metric/system/process/process.go @@ -387,7 +387,7 @@ func (procStats *Stats) matchProcess(name string) bool { // Init initializes a Stats instance. It returns errors if the provided process regexes // cannot be compiled. func (procStats *Stats) Init() error { - procStats.logger = logp.NewLogger("Stats") + procStats.logger = logp.NewLogger("processes") procStats.ProcsMap = make(ProcsMap) if len(procStats.Procs) == 0 { @@ -439,7 +439,7 @@ func (procStats *Stats) Get() ([]common.MapStr, error) { procStats.ProcsMap = newProcs processes = procStats.includeTopProcesses(processes) - procStats.logger.Debugf("processes", "Filtered top processes down to %d processes", len(processes)) + procStats.logger.Debugf("Filtered top processes down to %d processes", len(processes)) procs := make([]common.MapStr, 0, len(processes)) for _, process := range processes { @@ -480,18 +480,18 @@ func (procStats *Stats) getSingleProcess(pid int, newProcs ProcsMap) *Process { process, err := newProcess(pid, cmdline, env) if err != nil { - procStats.logger.Debugf("processes", "Skip process pid=%d: %v", pid, err) + procStats.logger.Debugf("Skip process pid=%d: %v", pid, err) return nil } if !procStats.matchProcess(process.Name) { - procStats.logger.Debugf("processes", "Process name does not matches the provided regex; pid=%d; name=%s: %v", pid, process.Name, err) + procStats.logger.Debugf("Process name does not matches the provided regex; pid=%d; name=%s: %v", pid, process.Name, err) return nil } err = process.getDetails(procStats.isWhitelistedEnvVar) if err != nil { - procStats.logger.Debugf("processes", "Error getting details for process %s with pid=%d: %v", process.Name, process.Pid, err) + procStats.logger.Debugf("Error getting details for process %s with pid=%d: %v", process.Name, process.Pid, err) return nil }