From eb3032ca498aefb675c45fffcaf54819d1028420 Mon Sep 17 00:00:00 2001 From: sayden Date: Mon, 5 Oct 2020 20:40:36 +0200 Subject: [PATCH 01/17] Atomic commit --- metricbeat/docs/fields.asciidoc | 365 ++++++++++++++++++ metricbeat/module/logstash/_meta/fields.yml | 105 +++++ metricbeat/module/logstash/fields.go | 2 +- .../logstash/node_stats/_meta/data.json | 150 ++++++- .../logstash/node_stats/_meta/fields.yml | 94 +++++ metricbeat/module/logstash/node_stats/data.go | 263 ++++++++++--- .../module/logstash/node_stats/data_xpack.go | 256 ------------ .../module/logstash/node_stats/node_stats.go | 17 +- 8 files changed, 915 insertions(+), 337 deletions(-) delete mode 100644 metricbeat/module/logstash/node_stats/data_xpack.go diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 236b476e1ce..117d35dfedd 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -28524,6 +28524,201 @@ Logstash module + +*`logstash.stats.timestamp`*:: ++ +-- +type: alias + +alias to: @timestamp + +-- + + + +*`logstash.stats.jvm.mem.heap_used`*:: ++ +-- +type: alias + +alias to: logstash.node.stats.jvm.mem.heap_used_in_bytes + +-- + +*`logstash.stats.jvm.mem.heap_max`*:: ++ +-- +type: alias + +alias to: logstash.node.stats.jvm.mem.heap_max_in_bytes + +-- + +*`logstash.stats.jvm.uptime`*:: ++ +-- +type: alias + +alias to: logstash.node.stats.jvm.uptime_in_millis + +-- + + +*`logstash.stats.events.in`*:: ++ +-- +type: alias + +alias to: logstash.node.stats.events.in + +-- + +*`logstash.stats.events.out`*:: ++ +-- +type: alias + +alias to: logstash.node.stats.events.out + +-- + +*`logstash.stats.events.duration.ms`*:: ++ +-- +type: alias + +alias to: logstash.node.stats.events.duration_in_millis + +-- + + +*`logstash.stats.logstash.uuid`*:: ++ +-- +type: alias + +alias to: logstash.node.stats.logstash.uuid + +-- + +*`logstash.stats.logstash.version`*:: ++ +-- +type: alias + +alias to: logstash.node.stats.logstash.version + +-- + + + + +*`logstash.stats.os.cpu.load_average.15m`*:: ++ +-- +type: alias + +alias to: logstash.node.stats.logstash.os.cpu.load_average.15m + +-- + +*`logstash.stats.os.cpu.load_average.1m`*:: ++ +-- +type: alias + +alias to: logstash.node.stats.logstash.os.cpu.load_average.1m + +-- + +*`logstash.stats.os.cpu.load_average.5m`*:: ++ +-- +type: alias + +alias to: logstash.node.stats.logstash.os.cpu.load_average.5m + +-- + + +*`logstash.stats.os.cgroup.cpuacct.usage_nanos`*:: ++ +-- +type: alias + +alias to: logstash.node.stats.logstash.os.cgroup.cpuacct.usage_nanos + +-- + + + +*`logstash.stats.os.cgroup.cpu.stat.number_of_elapsed_periods`*:: ++ +-- +type: alias + +alias to: logstash.node.stats.logstash.os.cgroup.cpu.number_of_elapsed_periods + +-- + +*`logstash.stats.os.cgroup.cpu.stat.time_throttled_nanos`*:: ++ +-- +type: alias + +alias to: logstash.node.stats.logstash.os.cgroup.cpu.time_throttled_nanos + +-- + +*`logstash.stats.os.cgroup.cpu.stat.number_of_times_throttled`*:: ++ +-- +type: alias + +alias to: logstash.node.stats.logstash.os.cgroup.cpu.number_of_times_throttled + +-- + + +*`logstash.stats.process.cpu.percent`*:: ++ +-- +type: alias + +alias to: logstash.node.stats.cpu.percent + +-- + + +*`logstash.stats.queue.events_count`*:: ++ +-- +type: alias + +alias to: logstash.node.stats.queue.events_count + +-- + + + +*`logstash.state.pipeline.id`*:: ++ +-- +type: alias + +alias to: logstash.node.stats.pipelines.main.id + +-- + +*`logstash.state.pipeline.hash`*:: ++ +-- +type: alias + +alias to: logstash.node.stats.pipelines.main.hash + +-- + [float] === node @@ -28591,6 +28786,38 @@ node_stats metrics. + +*`logstash.node.stats.jvm.uptime_in_millis`*:: ++ +-- +TODO + + +type: long + +-- + + +*`logstash.node.stats.jvm.mem.heap_used_in_bytes`*:: ++ +-- +TODO + + +type: long + +-- + +*`logstash.node.stats.jvm.mem.heap_max_in_bytes`*:: ++ +-- +TODO + + +type: long + +-- + [float] === events @@ -28628,6 +28855,144 @@ type: long -- + +*`logstash.node.stats.logstash.uuid`*:: ++ +-- +TODO + + +type: keyword + +-- + +*`logstash.node.stats.logstash.version`*:: ++ +-- +TODO + + +type: keyword + +-- + + + + +*`logstash.node.stats.os.cpu.load_average.15m`*:: ++ +-- +TODO + + +type: long + +-- + +*`logstash.node.stats.os.cpu.load_average.1m`*:: ++ +-- +TODO + + +type: long + +-- + +*`logstash.node.stats.os.cpu.load_average.5m`*:: ++ +-- +TODO + + +type: long + +-- + + +*`logstash.node.stats.os.cgroup.cpuact.usage_nanos`*:: ++ +-- +TODO + + +type: long + +-- + + + +*`logstash.node.stats.os.cgroup.cpu.stat.number_of_elapsed_periods`*:: ++ +-- +TODO + + +type: long + +-- + +*`logstash.node.stats.os.cgroup.cpu.stat.time_throttled_nanos`*:: ++ +-- +TODO + + +type: long + +-- + +*`logstash.node.stats.os.cgroup.cpu.stat.number_of_times_throttled`*:: ++ +-- +TODO + + +type: long + +-- + +*`logstash.node.stats.process.cpu.percent`*:: ++ +-- +TODO + + +type: double + +-- + +*`logstash.node.stats.queue.events_count`*:: ++ +-- +TODO + + +type: long + +-- + + +*`logstash.node.stats.pipelines.main.id`*:: ++ +-- +TODO + + +type: keyword + +-- + +*`logstash.node.stats.pipelines.main.hash`*:: ++ +-- +TODO + + +type: keyword + +-- + [[exported-fields-memcached]] == Memcached fields diff --git a/metricbeat/module/logstash/_meta/fields.yml b/metricbeat/module/logstash/_meta/fields.yml index 67dbdfeeb8d..dabe2293c98 100644 --- a/metricbeat/module/logstash/_meta/fields.yml +++ b/metricbeat/module/logstash/_meta/fields.yml @@ -9,3 +9,108 @@ type: group description: > fields: + - name: stats + type: group + fields: + - name: timestamp + type: alias + path: "@timestamp" + - name: jvm + type: group + fields: + - name: mem + type: group + fields: + - name: heap_used + type: alias + path: logstash.node.stats.jvm.mem.heap_used_in_bytes + - name: heap_max + type: alias + path: logstash.node.stats.jvm.mem.heap_max_in_bytes + - name: uptime + type: alias + path: logstash.node.stats.jvm.uptime_in_millis + - name: events + type: group + fields: + - name: in + type: alias + path: logstash.node.stats.events.in + - name: out + type: alias + path: logstash.node.stats.events.out + - name: duration.ms + type: alias + path: logstash.node.stats.events.duration_in_millis + - name: logstash + type: group + fields: + - name: uuid + type: alias + path: logstash.node.stats.logstash.uuid + - name: version + type: alias + path: logstash.node.stats.logstash.version + - name: os + type: group + fields: + - name: cpu + type: group + fields: + - name: load_average + type: group + fields: + - name: 15m + type: alias + path: logstash.node.stats.logstash.os.cpu.load_average.15m + - name: 1m + type: alias + path: logstash.node.stats.logstash.os.cpu.load_average.1m + - name: 5m + type: alias + path: logstash.node.stats.logstash.os.cpu.load_average.5m + - name: cgroup + type: group + fields: + - name: cpuacct.usage_nanos + type: alias + path: logstash.node.stats.logstash.os.cgroup.cpuacct.usage_nanos + - name: cpu + type: group + fields: + - name: stat + type: group + fields: + - name: number_of_elapsed_periods + type: alias + path: logstash.node.stats.logstash.os.cgroup.cpu.number_of_elapsed_periods + - name: time_throttled_nanos + type: alias + path: logstash.node.stats.logstash.os.cgroup.cpu.time_throttled_nanos + - name: number_of_times_throttled + type: alias + path: logstash.node.stats.logstash.os.cgroup.cpu.number_of_times_throttled + - name: process + type: group + fields: + - name: cpu.percent + type: alias + path: logstash.node.stats.cpu.percent + - name: queue + type: group + fields: + - name: events_count + type: alias + path: logstash.node.stats.queue.events_count + - name: state + type: group + fields: + - name: pipeline + type: group + fields: + - name: id + type: alias + path: logstash.node.stats.pipelines.main.id + - name: hash + type: alias + path: logstash.node.stats.pipelines.main.hash diff --git a/metricbeat/module/logstash/fields.go b/metricbeat/module/logstash/fields.go index bf8f9746c58..1aec34bd63c 100644 --- a/metricbeat/module/logstash/fields.go +++ b/metricbeat/module/logstash/fields.go @@ -32,5 +32,5 @@ func init() { // AssetLogstash returns asset data. // This is the base64 encoded gzipped contents of module/logstash. func AssetLogstash() string { - return "eJyslM2OmzAQgO88xYhzwwNw6KmtmqpV95TLarWyYALeGA/yDKzy9isCZMEx+d055OCJP3/jGbOCHe5TMFSwKC4jANFiMIX477AURwA5cuZ0LZpsCt8jAIAxDRXljcEIwKFBxZhCoSIARhFtC07hOWY28TeIS5E6fokAthpNzumBswKrKpwZdCH7uiM5auphJeAwJ01plnI8LoZoi8Q+vP3z0sbwD58KlMQyS4wSymjFXqZWUvZbku6nI3j/qHThVC8qrvGzZwrp4jexwAl0NG3RsSZ7oyyja3WGSXj3Q7rHydoE2KP1W1sFjf0eX3Hen80/WNsteYlQdy/f26fJDvfv5PJA/oJPF6HSp4fXOgRebhocG1c7ypA5CRPON+5K+af+CFj/CL7LhEUJP/o6Xw8UqFCczjh56LFii1b8K7t7nn4eaOBXuSQxFdHL42TIFve1Y20zqrQthjIho8YKumTRghr/0/UVGv8bKegWja02gg6XB/1+l18D+sTlIwAA//9sYbU7" + return "eJzsWD2P4zYQ7f0rBq5zBFKkcRGkuATZIMGmCK4JAoErzcq841f44dz++0CypeiDImVJtlMsiy3k1XtvyJnh6H2AL/h2AK5K66g97gAccxwPsP/18mi/AyjQ5oZpx5Q8wPc7AIDmZxCq8Bx3AAY5UosHKOkOwKJzTJb2AH/ureX7b2B/dE7v/9oBvDLkhT3UOB9AUoE9BdVyb7pCMsrry5OAhj5SF8066mz7NAQXeruL4JhA66jQvV8bJMoZtYNfNHXHA+x/aN/cB4E/n0QQcihuSmAXS+AQK44Xw+ziHpHqzFssgv8V24RmnTejOVQiVYGkPhTy+SSIQEFajozJ7OXNYRiqJ0nQr7dWJOjXaUGNGK+rQ57c+ikZcQln0IpcMM6ZDWYPnlC6IfbyBGJywyDO2kgAs6FT3m3PFwJtCAtvaNUxiAhBryRuwBNHNuhsffIlh+Y9CxXm0mjaZ0HchvSExjK1Zbq0z0LQbcZsl+u59jdpllzRIqMnNLQMtYQ0SYqoS/btd6GO3+eJdUGYdyTKklx70g2NxKhbeY9Tlxb3uK0LMLdZOZUV6xMz157muSPe0hIzSeWomOYHPzPwWi6ZSxyvzPQmpDYCBhNh8vhjRHPIuoTSixc0mXrNkFNdzTkaDVNFLL1gdiLCgjMhyyR1J+LMHY1yjmMRzac7xHK1mvG51IP6fxj/h3OJSWoC0EblaDe9F4lGk6PccjibQm1Y//boh9fl8hjOE1mWK79pELVIEgTv9hVc9aWpmUbO5Habsel02KizRFAmSWRCPI5n3A2Je+htLasivfdB4+C8Bu/3HYxmxc7uqOww31IGQfUKqf5UCIP/EKw8f1IcwJlReUQCqdbPyjoYgcbn95RYi+bEcgyO6CvltgbSp8j4f41ZkuD75dNv8CRf1ZXFlPru+YJv/ygTqriEnmqFQodea1haypdLgoQR4gc3U/zvZwp4+hisSzLPg0tUZ1ajgEBnWG7JqmLd0nmLmjVdcK5kuWx//3j++DzJf3PnL+7KJaKbGeFklCNFUVfuxoIWWm8Juh9rtJFRDetMu+XZ9iRzJZgsL2FCPe2gIZMqYl7echnP3pXqGhmvjDs0wel9rZafLtBRLY9x+VbdOtHGctP7brK23m2+Ptkcmy/S6mB+u4NYD4brrL07K3r8Ht3LxJvv4d3hQn5360ZKEmkG16UapAoAtnDlHqh5nQV3B+EDq23SzGokFcq/8Cu/fydJI6ZTYhuWMrZWy2aX8JRXdPv5ZWwWbc/9bwAAAP//+LIvcQ==" } diff --git a/metricbeat/module/logstash/node_stats/_meta/data.json b/metricbeat/module/logstash/node_stats/_meta/data.json index d6e05f6a6ec..404915a5faa 100644 --- a/metricbeat/module/logstash/node_stats/_meta/data.json +++ b/metricbeat/module/logstash/node_stats/_meta/data.json @@ -1,34 +1,148 @@ { - "@timestamp": "2017-10-12T08:05:34.853Z", - "agent": { - "hostname": "host.example.com", - "name": "host.example.com" + "@timestamp": "2020-10-05T15:07:19.603Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" }, - "event": { - "dataset": "logstash.node.stats", - "duration": 115000, - "module": "logstash" + "metricset": { + "name": "node_stats", + "period": 10000 }, "logstash": { "node": { "stats": { + "queue": { + "events_count": 0 + }, "events": { - "filtered": 0, + "duration_in_millis": 0, "in": 0, + "filtered": 0, "out": 0 - } + }, + "jvm": { + "gc": { + "collectors": { + "old": { + "collection_count": 5, + "collection_time_in_millis": 1177 + }, + "young": { + "collection_count": 15, + "collection_time_in_millis": 407 + } + } + }, + "mem": { + "heap_max_in_bytes": 1.037959168e+09, + "heap_used_in_bytes": 2.02339024e+08, + "heap_used_percent": 19 + }, + "uptime_in_millis": 4.602954e+06 + }, + "reloads": { + "failures": 0, + "successes": 0 + }, + "process": { + "open_file_descriptors": 229, + "max_file_descriptors": 1.048576e+06, + "cpu": { + "percent": 0 + } + }, + "os": { + "cpu": { + "percent": 0, + "load_average": { + "1m": 1.96, + "5m": 1.76, + "15m": 1.81 + } + }, + "cgroup": { + "cpuacct": { + "control_group": "/", + "usage_nanos": 8.9799096638e+11 + }, + "cpu": { + "stat": { + "time_throttled_nanos": 0, + "number_of_elapsed_periods": 0, + "number_of_times_throttled": 0 + }, + "control_group": "/" + } + } + }, + "pipelines": [ + { + "id": "main", + "hash": "3000c3abf87d4dfa4a57aaf6af0a1f5bee2e0fc1c48a8e8636e2a33d7d2e91dd", + "ephemeral_id": "2c607fe3-0d89-4506-8fe8-32180d201e48", + "events": { + "queue_push_duration_in_millis": 0, + "duration_in_millis": 0, + "filtered": 0, + "in": 0, + "out": 0 + }, + "reloads": { + "successes": 0, + "failures": 0 + }, + "queue": { + "events_count": 0, + "max_queue_size_in_bytes": 0, + "queue_size_in_bytes": 0, + "type": "memory" + }, + "vertices": null + } + ], + "logstash": { + "snapshot": true, + "http_address": "0.0.0.0:9600", + "uuid": "4cc683ce-3ddc-46e3-bea3-aefbf37bc082", + "ephemeral_id": "94b3a942-8dff-4cfb-8ba2-51672230b903", + "name": "2cb47f6e0eab", + "host": "2cb47f6e0eab", + "version": "8.0.0", + "status": "green", + "pipeline": { + "batch_size": 125, + "workers": 12 + } + }, + "timestamp": "2020-10-05T15:07:19.613Z" } } }, - "metricset": { - "name": "node_stats" + "host": { + "name": "mcastro" + }, + "agent": { + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "41749b5d-ffb6-4ae0-9626-4673ce48ae7c", + "id": "803dfdba-e638-4590-a2de-80cb1cebe78d", + "name": "mcastro" + }, + "ecs": { + "version": "1.5.0" }, "service": { - "address": "127.0.0.1:9600", - "hostname": "Shaunaks-MBP-2.attlocal.net", - "id": "7565df20-c3aa-4261-81d5-3b0ab8d15c16", - "name": "logstash", + "hostname": "2cb47f6e0eab", + "version": "8.0.0", + "address": "localhost:9600", "type": "logstash", - "version": "7.0.0" + "name": "logstash", + "id": "" + }, + "event": { + "dataset": "logstash.node.stats", + "module": "logstash", + "duration": 10447360 } -} \ No newline at end of file +} diff --git a/metricbeat/module/logstash/node_stats/_meta/fields.yml b/metricbeat/module/logstash/node_stats/_meta/fields.yml index 8a06554c6e2..b2d214c105d 100644 --- a/metricbeat/module/logstash/node_stats/_meta/fields.yml +++ b/metricbeat/module/logstash/node_stats/_meta/fields.yml @@ -4,6 +4,24 @@ node_stats metrics. release: ga fields: + - name: jvm + type: group + fields: + - name: uptime_in_millis + type: long + description: > + TODO + - name: mem + type: group + fields: + - name: heap_used_in_bytes + type: long + description: > + TODO + - name: heap_max_in_bytes + type: long + description: > + TODO - name: events type: group description: > @@ -21,3 +39,79 @@ type: long description: > Filtered events counter. + - name: logstash + type: group + fields: + - name: uuid + type: keyword + description: > + TODO + - name: version + type: keyword + description: > + TODO + - name: os + type: group + fields: + - name: cpu + type: group + fields: + - name: load_average + type: group + fields: + - name: 15m + type: long + description: > + TODO + - name: 1m + type: long + description: > + TODO + - name: 5m + type: long + description: > + TODO + - name: cgroup + type: group + fields: + - name: cpuact.usage_nanos + type: long + description: > + TODO + - name: cpu + type: group + fields: + - name: stat + type: group + fields: + - name: number_of_elapsed_periods + type: long + description: > + TODO + - name: time_throttled_nanos + type: long + description: > + TODO + - name: number_of_times_throttled + type: long + description: > + TODO + - name: process.cpu.percent + type: double + description: > + TODO + - name: queue.events_count + type: long + description: > + TODO + - name: pipelines + type: group + fields: + - name: main.id + type: keyword + description: > + TODO + - name: main.hash + type: keyword + description: > + TODO diff --git a/metricbeat/module/logstash/node_stats/data.go b/metricbeat/module/logstash/node_stats/data.go index da2f2f3b7c3..0f28bf333ee 100644 --- a/metricbeat/module/logstash/node_stats/data.go +++ b/metricbeat/module/logstash/node_stats/data.go @@ -19,72 +19,241 @@ package node_stats import ( "encoding/json" + "time" + + "github.com/elastic/beats/v7/metricbeat/module/logstash" "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/common" - s "github.com/elastic/beats/v7/libbeat/common/schema" - c "github.com/elastic/beats/v7/libbeat/common/schema/mapstriface" - "github.com/elastic/beats/v7/metricbeat/helper/elastic" "github.com/elastic/beats/v7/metricbeat/mb" - "github.com/elastic/beats/v7/metricbeat/module/logstash" ) -var ( - schema = s.Schema{ - "id": c.Str("id"), - "host": c.Str("host"), - "version": c.Str("version"), - "events": c.Dict("events", s.Schema{ - "in": c.Int("in"), - "out": c.Int("out"), - "filtered": c.Int("filtered"), - }), - } -) +type jvm struct { + GC map[string]interface{} `json:"gc"` + Mem struct { + HeapMaxInBytes int `json:"heap_max_in_bytes"` + HeapUsedInBytes int `json:"heap_used_in_bytes"` + HeapUsedPercent int `json:"heap_used_percent"` + } `json:"mem"` + UptimeInMillis int `json:"uptime_in_millis"` +} + +type events struct { + DurationInMillis int `json:"duration_in_millis"` + In int `json:"in"` + Filtered int `json:"filtered"` + Out int `json:"out"` +} -func eventMapping(r mb.ReporterV2, content []byte) error { - event := mb.Event{} - event.RootFields = common.MapStr{} - event.RootFields.Put("service.name", logstash.ModuleName) +type commonStats struct { + Events events `json:"events"` + JVM jvm `json:"jvm"` + Reloads map[string]interface{} `json:"reloads"` + Queue struct { + EventsCount int `json:"events_count"` + } `json:"queue"` +} - var data map[string]interface{} - err := json.Unmarshal(content, &data) - if err != nil { - return errors.Wrap(err, "failure parsing Logstash Node Stats API response") - } +type cpu struct { + Percent int `json:"percent"` + LoadAverage map[string]interface{} `json:"load_average,omitempty"` +} + +type process struct { + OpenFileDescriptors int `json:"open_file_descriptors"` + MaxFileDescriptors int `json:"max_file_descriptors"` + CPU cpu `json:"cpu"` +} + +type cgroup struct { + CPUAcct map[string]interface{} `json:"cpuacct"` + CPU struct { + Stat map[string]interface{} `json:"stat"` + ControlGroup string `json:"control_group"` + } `json:"cpu"` +} + +type os struct { + CPU cpu `json:"cpu"` + CGroup cgroup `json:"cgroup,omitempty"` +} + +type pipeline struct { + BatchSize int `json:"batch_size"` + Workers int `json:"workers"` +} + +type nodeInfo struct { + ID string `json:"id,omitempty"` + UUID string `json:"uuid"` + EphemeralID string `json:"ephemeral_id"` + Name string `json:"name"` + Host string `json:"host"` + Version string `json:"version"` + Snapshot bool `json:"snapshot"` + Status string `json:"status"` + HTTPAddress string `json:"http_address"` + Pipeline pipeline `json:"pipeline"` +} + +// inNodeInfo represents the Logstash node info to be parsed from the Logstash API +// response. It contains nodeInfo (which is also used as-is elsewhere) + monitoring +// information. +type inNodeInfo struct { + nodeInfo + Monitoring struct { + ClusterID string `json:"cluster_uuid"` + } `json:"monitoring"` +} + +type reloads struct { + Successes int `json:"successes"` + Failures int `json:"failures"` +} + +// NodeStats represents the stats of a Logstash node +type NodeStats struct { + inNodeInfo + commonStats + Process process `json:"process"` + OS os `json:"os"` + Pipelines map[string]PipelineStats `json:"pipelines"` + Events events `json:"events"` +} - fields, err := schema.Apply(data) +// LogstashStats represents the logstash_stats sub-document indexed into .monitoring-logstash-* +type LogstashStats struct { + commonStats + Process process `json:"process"` + OS os `json:"os"` + Pipelines []PipelineStats `json:"pipelines"` + Logstash nodeInfo `json:"logstash"` + Timestamp common.Time `json:"timestamp"` +} + +// PipelineStats represents the stats of a Logstash pipeline +type PipelineStats struct { + ID string `json:"id"` + Hash string `json:"hash"` + EphemeralID string `json:"ephemeral_id"` + Events map[string]interface{} `json:"events"` + Reloads reloads `json:"reloads"` + Queue map[string]interface{} `json:"queue"` + Vertices []map[string]interface{} `json:"vertices"` +} + +func eventMapping(r mb.ReporterV2, m *MetricSet, content []byte) error { + var nodeStats NodeStats + err := json.Unmarshal(content, &nodeStats) if err != nil { - return errors.Wrap(err, "failure applying node stats schema") + return errors.Wrap(err, "could not parse node stats response") } - // Set service ID - serviceID, err := fields.GetValue("id") - if err != nil { - return elastic.MakeErrorForMissingField("id", elastic.Logstash) + timestamp := common.Time(time.Now()) + + // Massage Logstash node basic info + nodeStats.nodeInfo.UUID = nodeStats.nodeInfo.ID + nodeStats.nodeInfo.ID = "" + + proc := process{ + nodeStats.Process.OpenFileDescriptors, + nodeStats.Process.MaxFileDescriptors, + cpu{ + Percent: nodeStats.Process.CPU.Percent, + }, } - event.RootFields.Put("service.id", serviceID) - fields.Delete("id") - // Set service hostname - host, err := fields.GetValue("host") - if err != nil { - return elastic.MakeErrorForMissingField("host", elastic.Logstash) + o := os{ + cpu{ + LoadAverage: nodeStats.Process.CPU.LoadAverage, + }, + nodeStats.OS.CGroup, } - event.RootFields.Put("service.hostname", host) - fields.Delete("host") - // Set service version - version, err := fields.GetValue("version") - if err != nil { - return elastic.MakeErrorForMissingField("version", elastic.Logstash) + var pipelines []PipelineStats + for pipelineID, pipeline := range nodeStats.Pipelines { + pipeline.ID = pipelineID + pipelines = append(pipelines, pipeline) } - event.RootFields.Put("service.version", version) - fields.Delete("version") - event.MetricSetFields = fields + pipelines = getUserDefinedPipelines(pipelines) + clusterToPipelinesMap := makeClusterToPipelinesMap(pipelines, nodeStats.Monitoring.ClusterID) + + for clusterUUID, clusterPipelines := range clusterToPipelinesMap { + logstashStats := LogstashStats{ + nodeStats.commonStats, + proc, + o, + clusterPipelines, + nodeStats.nodeInfo, + timestamp, + } + + event := mb.Event{ + RootFields: common.MapStr{ + "service": common.MapStr{"name": logstash.ModuleName}, + }, + ModuleFields: common.MapStr{}, + } + + event.ModuleFields.Put("node.stats", logstashStats) + event.RootFields.Put("service.id", nodeStats.ID) + event.RootFields.Put("service.hostname", nodeStats.Host) + event.RootFields.Put("service.version", nodeStats.Version) + + if clusterUUID != "" { + event.ModuleFields["cluster.id"] = clusterUUID + } + + r.Event(event) + } - r.Event(event) return nil } + +func makeClusterToPipelinesMap(pipelines []PipelineStats, overrideClusterUUID string) map[string][]PipelineStats { + var clusterToPipelinesMap map[string][]PipelineStats + clusterToPipelinesMap = make(map[string][]PipelineStats) + + if overrideClusterUUID != "" { + clusterToPipelinesMap[overrideClusterUUID] = pipelines + return clusterToPipelinesMap + } + + for _, pipeline := range pipelines { + var clusterUUIDs []string + for _, vertex := range pipeline.Vertices { + clusterUUID := logstash.GetVertexClusterUUID(vertex, overrideClusterUUID) + if clusterUUID != "" { + clusterUUIDs = append(clusterUUIDs, clusterUUID) + } + } + + // If no cluster UUID was found in this pipeline, assign it a blank one + if len(clusterUUIDs) == 0 { + clusterUUIDs = []string{""} + } + + for _, clusterUUID := range clusterUUIDs { + clusterPipelines := clusterToPipelinesMap[clusterUUID] + if clusterPipelines == nil { + clusterToPipelinesMap[clusterUUID] = []PipelineStats{} + } + + clusterToPipelinesMap[clusterUUID] = append(clusterPipelines, pipeline) + } + } + + return clusterToPipelinesMap +} + +func getUserDefinedPipelines(pipelines []PipelineStats) []PipelineStats { + userDefinedPipelines := []PipelineStats{} + for _, pipeline := range pipelines { + if pipeline.ID[0] != '.' { + userDefinedPipelines = append(userDefinedPipelines, pipeline) + } + } + return userDefinedPipelines +} diff --git a/metricbeat/module/logstash/node_stats/data_xpack.go b/metricbeat/module/logstash/node_stats/data_xpack.go deleted file mode 100644 index a6a9867b7cd..00000000000 --- a/metricbeat/module/logstash/node_stats/data_xpack.go +++ /dev/null @@ -1,256 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package node_stats - -import ( - "encoding/json" - "time" - - "github.com/elastic/beats/v7/metricbeat/module/logstash" - - "github.com/pkg/errors" - - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/metricbeat/helper/elastic" - "github.com/elastic/beats/v7/metricbeat/mb" -) - -type jvm struct { - GC map[string]interface{} `json:"gc"` - Mem struct { - HeapMaxInBytes int `json:"heap_max_in_bytes"` - HeapUsedInBytes int `json:"heap_used_in_bytes"` - HeapUsedPercent int `json:"heap_used_percent"` - } `json:"mem"` - UptimeInMillis int `json:"uptime_in_millis"` -} - -type events struct { - DurationInMillis int `json:"duration_in_millis"` - In int `json:"in"` - Filtered int `json:"filtered"` - Out int `json:"out"` -} - -type commonStats struct { - Events events `json:"events"` - JVM jvm `json:"jvm"` - Reloads map[string]interface{} `json:"reloads"` - Queue struct { - EventsCount int `json:"events_count"` - } `json:"queue"` -} - -type cpu struct { - Percent int `json:"percent,omitempty"` - LoadAverage map[string]interface{} `json:"load_average,omitempty"` -} - -type process struct { - OpenFileDescriptors int `json:"open_file_descriptors"` - MaxFileDescriptors int `json:"max_file_descriptors"` - CPU cpu `json:"cpu"` -} - -type cgroup struct { - CPUAcct map[string]interface{} `json:"cpuacct"` - CPU struct { - Stat map[string]interface{} `json:"stat"` - ControlGroup string `json:"control_group"` - } `json:"cpu"` -} - -type os struct { - CPU cpu `json:"cpu"` - CGroup cgroup `json:"cgroup,omitempty"` -} - -type pipeline struct { - BatchSize int `json:"batch_size"` - Workers int `json:"workers"` -} - -type nodeInfo struct { - ID string `json:"id,omitempty"` - UUID string `json:"uuid"` - EphemeralID string `json:"ephemeral_id"` - Name string `json:"name"` - Host string `json:"host"` - Version string `json:"version"` - Snapshot bool `json:"snapshot"` - Status string `json:"status"` - HTTPAddress string `json:"http_address"` - Pipeline pipeline `json:"pipeline"` -} - -// inNodeInfo represents the Logstash node info to be parsed from the Logstash API -// response. It contains nodeInfo (which is also used as-is elsewhere) + monitoring -// information. -type inNodeInfo struct { - nodeInfo - Monitoring struct { - ClusterID string `json:"cluster_uuid"` - } `json:"monitoring"` -} - -type reloads struct { - Successes int `json:"successes"` - Failures int `json:"failures"` -} - -// NodeStats represents the stats of a Logstash node -type NodeStats struct { - inNodeInfo - commonStats - Process process `json:"process"` - OS os `json:"os"` - Pipelines map[string]PipelineStats `json:"pipelines"` -} - -// LogstashStats represents the logstash_stats sub-document indexed into .monitoring-logstash-* -type LogstashStats struct { - commonStats - Process process `json:"process"` - OS os `json:"os"` - Pipelines []PipelineStats `json:"pipelines"` - Logstash nodeInfo `json:"logstash"` - Timestamp common.Time `json:"timestamp"` -} - -// PipelineStats represents the stats of a Logstash pipeline -type PipelineStats struct { - ID string `json:"id"` - Hash string `json:"hash"` - EphemeralID string `json:"ephemeral_id"` - Events map[string]interface{} `json:"events"` - Reloads reloads `json:"reloads"` - Queue map[string]interface{} `json:"queue"` - Vertices []map[string]interface{} `json:"vertices"` -} - -func eventMappingXPack(r mb.ReporterV2, m *MetricSet, content []byte) error { - var nodeStats NodeStats - err := json.Unmarshal(content, &nodeStats) - if err != nil { - return errors.Wrap(err, "could not parse node stats response") - } - - timestamp := common.Time(time.Now()) - - // Massage Logstash node basic info - nodeStats.nodeInfo.UUID = nodeStats.nodeInfo.ID - nodeStats.nodeInfo.ID = "" - - proc := process{ - nodeStats.Process.OpenFileDescriptors, - nodeStats.Process.MaxFileDescriptors, - cpu{ - Percent: nodeStats.Process.CPU.Percent, - }, - } - - o := os{ - cpu{ - LoadAverage: nodeStats.Process.CPU.LoadAverage, - }, - nodeStats.OS.CGroup, - } - - var pipelines []PipelineStats - for pipelineID, pipeline := range nodeStats.Pipelines { - pipeline.ID = pipelineID - pipelines = append(pipelines, pipeline) - } - - pipelines = getUserDefinedPipelines(pipelines) - clusterToPipelinesMap := makeClusterToPipelinesMap(pipelines, nodeStats.Monitoring.ClusterID) - - for clusterUUID, clusterPipelines := range clusterToPipelinesMap { - logstashStats := LogstashStats{ - nodeStats.commonStats, - proc, - o, - clusterPipelines, - nodeStats.nodeInfo, - timestamp, - } - - event := mb.Event{} - event.RootFields = common.MapStr{ - "timestamp": timestamp, - "interval_ms": m.Module().Config().Period / time.Millisecond, - "type": "logstash_stats", - "logstash_stats": logstashStats, - } - - if clusterUUID != "" { - event.RootFields["cluster_uuid"] = clusterUUID - } - - event.Index = elastic.MakeXPackMonitoringIndexName(elastic.Logstash) - r.Event(event) - } - - return nil -} - -func makeClusterToPipelinesMap(pipelines []PipelineStats, overrideClusterUUID string) map[string][]PipelineStats { - var clusterToPipelinesMap map[string][]PipelineStats - clusterToPipelinesMap = make(map[string][]PipelineStats) - - if overrideClusterUUID != "" { - clusterToPipelinesMap[overrideClusterUUID] = pipelines - return clusterToPipelinesMap - } - - for _, pipeline := range pipelines { - var clusterUUIDs []string - for _, vertex := range pipeline.Vertices { - clusterUUID := logstash.GetVertexClusterUUID(vertex, overrideClusterUUID) - if clusterUUID != "" { - clusterUUIDs = append(clusterUUIDs, clusterUUID) - } - } - - // If no cluster UUID was found in this pipeline, assign it a blank one - if len(clusterUUIDs) == 0 { - clusterUUIDs = []string{""} - } - - for _, clusterUUID := range clusterUUIDs { - clusterPipelines := clusterToPipelinesMap[clusterUUID] - if clusterPipelines == nil { - clusterToPipelinesMap[clusterUUID] = []PipelineStats{} - } - - clusterToPipelinesMap[clusterUUID] = append(clusterPipelines, pipeline) - } - } - - return clusterToPipelinesMap -} - -func getUserDefinedPipelines(pipelines []PipelineStats) []PipelineStats { - userDefinedPipelines := []PipelineStats{} - for _, pipeline := range pipelines { - if pipeline.ID[0] != '.' { - userDefinedPipelines = append(userDefinedPipelines, pipeline) - } - } - return userDefinedPipelines -} diff --git a/metricbeat/module/logstash/node_stats/node_stats.go b/metricbeat/module/logstash/node_stats/node_stats.go index 5b2c37e5eeb..c12f4cd503d 100644 --- a/metricbeat/module/logstash/node_stats/node_stats.go +++ b/metricbeat/module/logstash/node_stats/node_stats.go @@ -69,29 +69,16 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // descriptive error must be returned. func (m *MetricSet) Fetch(r mb.ReporterV2) error { if err := m.updateServiceURI(); err != nil { - if m.XPack { - m.Logger().Error(err) - return nil - } return err } content, err := m.HTTP.FetchContent() if err != nil { - if m.XPack { - m.Logger().Error(err) - return nil - } return err } - if !m.XPack { - return eventMapping(r, content) - } - - err = eventMappingXPack(r, m, content) - if err != nil { - m.Logger().Error(err) + if err = eventMapping(r, m, content); err != nil { + return err } return nil From 20392daed071ca32d71e962386e8b5a088b3a481 Mon Sep 17 00:00:00 2001 From: sayden Date: Fri, 30 Oct 2020 13:42:30 +0100 Subject: [PATCH 02/17] Upgrade mapping and data.json --- metricbeat/docs/fields.asciidoc | 427 ++++++++++++++++-- metricbeat/module/logstash/_meta/fields.yml | 212 +++++++-- metricbeat/module/logstash/fields.go | 2 +- .../logstash/node_stats/_meta/data.json | 252 +++++------ .../logstash/node_stats/_meta/fields.yml | 98 +++- 5 files changed, 769 insertions(+), 222 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 117d35dfedd..499f813fae0 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -28518,9 +28518,190 @@ Logstash module + +*`logstash_stats.timestamp`*:: ++ +-- +type: alias + +alias to: @timestamp + +-- + + + +*`logstash_stats.jvm.mem.heap_used_in_bytes`*:: ++ +-- +type: alias + +alias to: logstash.node.stats.jvm.mem.heap_used_in_bytes + +-- + +*`logstash_stats.jvm.mem.heap_max_in_bytes`*:: ++ +-- +type: alias + +alias to: logstash.node.stats.jvm.mem.heap_max_in_bytes + +-- + +*`logstash_stats.jvm.uptime_in_millis`*:: ++ +-- +type: alias + +alias to: logstash.node.stats.jvm.uptime_in_millis + +-- + + +*`logstash_stats.events.in`*:: ++ +-- +type: alias + +alias to: logstash.node.stats.events.in + +-- + +*`logstash_stats.events.out`*:: ++ +-- +type: alias + +alias to: logstash.node.stats.events.out + +-- + +*`logstash_stats.events.duration_in_millis`*:: ++ +-- +type: alias + +alias to: logstash.node.stats.events.duration_in_millis + +-- + + +*`logstash_stats.logstash.uuid`*:: ++ +-- +type: alias + +alias to: logstash.node.stats.logstash.uuid + +-- + +*`logstash_stats.logstash.version`*:: ++ +-- +type: alias + +alias to: logstash.node.stats.logstash.version + +-- + +*`logstash_stats.pipelines`*:: ++ +-- +type: nested + +-- + + + + +*`logstash_stats.os.cpu.load_average.15m`*:: ++ +-- +type: alias + +alias to: logstash.node.stats.os.cpu.load_average.15m + +-- + +*`logstash_stats.os.cpu.load_average.1m`*:: ++ +-- +type: alias + +alias to: logstash.node.stats.os.cpu.load_average.1m + +-- + +*`logstash_stats.os.cpu.load_average.5m`*:: ++ +-- +type: alias + +alias to: logstash.node.stats.os.cpu.load_average.5m + +-- + + +*`logstash_stats.os.cgroup.cpuacct.usage_nanos`*:: ++ +-- +type: alias + +alias to: logstash.node.stats.os.cgroup.cpuacct.usage_nanos + +-- + + + +*`logstash_stats.os.cpu.stat.number_of_elapsed_periods`*:: ++ +-- +type: alias + +alias to: logstash.node.stats.os.cgroup.stat.number_of_elapsed_periods + +-- + +*`logstash_stats.os.cpu.stat.time_throttled_nanos`*:: ++ +-- +type: alias + +alias to: logstash.node.stats.os.cgroup.stat.time_throttled_nanos + +-- + +*`logstash_stats.os.cpu.stat.number_of_times_throttled`*:: ++ +-- +type: alias + +alias to: logstash.node.stats.os.cgroup.stat.number_of_times_throttled + +-- + +*`logstash_stats.process.cpu.percent`*:: ++ +-- +type: alias + +alias to: logstash.node.stats.process.cpu.percent + +-- + +*`logstash_stats.queue.events_count`*:: ++ +-- +type: alias + +alias to: logstash.node.stats.queue.events_count + +-- + [float] === logstash +TODO @@ -28536,7 +28717,7 @@ alias to: @timestamp -*`logstash.stats.jvm.mem.heap_used`*:: +*`logstash.stats.jvm.mem.heap_used.bytes`*:: + -- type: alias @@ -28545,7 +28726,7 @@ alias to: logstash.node.stats.jvm.mem.heap_used_in_bytes -- -*`logstash.stats.jvm.mem.heap_max`*:: +*`logstash.stats.jvm.mem.heap_max.bytes`*:: + -- type: alias @@ -28554,7 +28735,7 @@ alias to: logstash.node.stats.jvm.mem.heap_max_in_bytes -- -*`logstash.stats.jvm.uptime`*:: +*`logstash.stats.jvm.uptime.ms`*:: + -- type: alias @@ -28618,7 +28799,7 @@ alias to: logstash.node.stats.logstash.version -- type: alias -alias to: logstash.node.stats.logstash.os.cpu.load_average.15m +alias to: logstash.node.stats.os.cpu.load_average.15m -- @@ -28627,7 +28808,7 @@ alias to: logstash.node.stats.logstash.os.cpu.load_average.15m -- type: alias -alias to: logstash.node.stats.logstash.os.cpu.load_average.1m +alias to: logstash.node.stats.os.cpu.load_average.1m -- @@ -28636,7 +28817,7 @@ alias to: logstash.node.stats.logstash.os.cpu.load_average.1m -- type: alias -alias to: logstash.node.stats.logstash.os.cpu.load_average.5m +alias to: logstash.node.stats.os.cpu.load_average.5m -- @@ -28646,50 +28827,48 @@ alias to: logstash.node.stats.logstash.os.cpu.load_average.5m -- type: alias -alias to: logstash.node.stats.logstash.os.cgroup.cpuacct.usage_nanos +alias to: logstash.node.stats.os.cgroup.cpuacct.usage_nanos -- -*`logstash.stats.os.cgroup.cpu.stat.number_of_elapsed_periods`*:: +*`logstash.stats.os.cpu.stat.number_of_elapsed_periods`*:: + -- type: alias -alias to: logstash.node.stats.logstash.os.cgroup.cpu.number_of_elapsed_periods +alias to: logstash.node.stats.os.cgroup.stat.number_of_elapsed_periods -- -*`logstash.stats.os.cgroup.cpu.stat.time_throttled_nanos`*:: +*`logstash.stats.os.cpu.stat.time_throttled_nanos`*:: + -- type: alias -alias to: logstash.node.stats.logstash.os.cgroup.cpu.time_throttled_nanos +alias to: logstash.node.stats.os.cgroup.stat.time_throttled_nanos -- -*`logstash.stats.os.cgroup.cpu.stat.number_of_times_throttled`*:: +*`logstash.stats.os.cpu.stat.number_of_times_throttled`*:: + -- type: alias -alias to: logstash.node.stats.logstash.os.cgroup.cpu.number_of_times_throttled +alias to: logstash.node.stats.os.cgroup.stat.number_of_times_throttled -- - *`logstash.stats.process.cpu.percent`*:: + -- type: alias -alias to: logstash.node.stats.cpu.percent +alias to: logstash.node.stats.process.cpu.percent -- - *`logstash.stats.queue.events_count`*:: + -- @@ -28699,26 +28878,6 @@ alias to: logstash.node.stats.queue.events_count -- - - -*`logstash.state.pipeline.id`*:: -+ --- -type: alias - -alias to: logstash.node.stats.pipelines.main.id - --- - -*`logstash.state.pipeline.hash`*:: -+ --- -type: alias - -alias to: logstash.node.stats.pipelines.main.hash - --- - [float] === node @@ -28851,6 +29010,15 @@ type: long Filtered events counter. +type: long + +-- + +*`logstash.node.stats.events.duration_in_millis`*:: ++ +-- +TODO + type: long -- @@ -28962,37 +29130,216 @@ type: double -- -*`logstash.node.stats.queue.events_count`*:: + +*`logstash.node.stats.pipelines.id`*:: + -- TODO -type: long +type: keyword + +-- +*`logstash.node.stats.pipelines.hash`*:: ++ -- +TODO -*`logstash.node.stats.pipelines.main.id`*:: +type: keyword + +-- + + +*`logstash.node.stats.pipelines.queue.events_count`*:: + -- TODO +type: long + +-- + +*`logstash.node.stats.pipelines.queue.type`*:: ++ +-- +TODO type: keyword -- -*`logstash.node.stats.pipelines.main.hash`*:: +*`logstash.node.stats.pipelines.queue.queue_size_in_bytes`*:: ++ +-- +TODO + +type: long + +-- + +*`logstash.node.stats.pipelines.queue.max_queue_size_in_bytes`*:: ++ +-- +TODO + +type: long + +-- + + +*`logstash.node.stats.pipelines.events.out`*:: ++ +-- +TODO + +type: long + +-- + +*`logstash.node.stats.pipelines.events.duration_in_millis`*:: ++ +-- +TODO + +type: long + +-- + + +*`logstash.node.stats.pipelines.vertices.duration_in_millis`*:: + -- TODO +type: long + +-- + +*`logstash.node.stats.pipelines.vertices.events_in`*:: ++ +-- +TODO + +type: long + +-- + +*`logstash.node.stats.pipelines.vertices.pipeline_ephemeral_id`*:: ++ +-- +pipeline_ephemeral_id + +type: keyword + +-- + +*`logstash.node.stats.pipelines.vertices.events_out`*:: ++ +-- +events_out + +type: long + +-- + +*`logstash.node.stats.pipelines.vertices.id`*:: ++ +-- +id type: keyword -- +*`logstash.node.stats.pipelines.vertices.queue_push_duration_in_millis`*:: ++ +-- +queue_push_duration_in_millis + +type: float + +-- + + + + +*`logstash.node.stats.os.cpu.load_average.15m`*:: ++ +-- +TODO + +type: long + +-- + +*`logstash.node.stats.os.cpu.load_average.1m`*:: ++ +-- +TODO + +type: long + +-- + +*`logstash.node.stats.os.cpu.load_average.5m`*:: ++ +-- +TODO + +type: long + +-- + + +*`logstash.node.stats.os.cgroup.cpuacct.usage_nanos`*:: ++ +-- +TODO + +type: long + +-- + + + +*`logstash.node.stats.os.cgroup.cpu.stat.number_of_elapsed_periods`*:: ++ +-- +TODO + +type: long + +-- + +*`logstash.node.stats.os.cgroup.cpu.stat.time_throttled_nanos`*:: ++ +-- +TODO + +type: long + +-- + +*`logstash.node.stats.os.cgroup.cpu.stat.number_of_times_throttled`*:: ++ +-- +TODO + +type: long + +-- + +*`logstash.node.stats.queue.events_count`*:: ++ +-- +TODO + +type: long + +-- + [[exported-fields-memcached]] == Memcached fields diff --git a/metricbeat/module/logstash/_meta/fields.yml b/metricbeat/module/logstash/_meta/fields.yml index dabe2293c98..79307d9a859 100644 --- a/metricbeat/module/logstash/_meta/fields.yml +++ b/metricbeat/module/logstash/_meta/fields.yml @@ -5,9 +5,148 @@ release: ga settings: ["ssl", "http"] fields: + - name: logstash_stats + type: group + fields: + - name: timestamp + type: alias + path: "@timestamp" + - name: jvm + type: group + fields: + - name: mem + type: group + fields: + - name: heap_used_in_bytes + type: alias + path: logstash.node.stats.jvm.mem.heap_used_in_bytes + - name: heap_max_in_bytes + type: alias + path: logstash.node.stats.jvm.mem.heap_max_in_bytes + - name: uptime_in_millis + type: alias + path: logstash.node.stats.jvm.uptime_in_millis + - name: events + type: group + fields: + - name: in + type: alias + path: logstash.node.stats.events.in + - name: out + type: alias + path: logstash.node.stats.events.out + - name: duration_in_millis + type: alias + path: logstash.node.stats.events.duration_in_millis + - name: logstash + type: group + fields: + - name: uuid + type: alias + path: logstash.node.stats.logstash.uuid + - name: version + type: alias + path: logstash.node.stats.logstash.version + - name: pipelines + type: nested + fields: + - name: id + type: alias + path: logstash.node.stats.pipelines.id + - name: hash + type: alias + path: logstash.node.stats.pipelines.hash + - name: queue + type: group + fields: + - name: type + type: alias + path: logstash.node.stats.pipelines.queue.type + - name: queue_size_in_bytes + type: alias + path: logstash.node.stats.pipelines.queue.queue_size_in_bytes + - name: max_queue_size_in_bytes + type: alias + path: logstash.node.stats.pipelines.queue.max_queue_size_in_bytes + - name: events + type: group + fields: + - name: out + type: alias + path: logstash.node.stats.pipelines.events.out + - name: duration_in_millis + type: alias + path: logstash.node.stats.pipelines.events.duration_in_millis + - name: vertices + type: nested + fields: + - name: duration_in_millis + type: alias + path: logstash.node.stats.pipelines.vertices.duration_in_millis + - name: events_in + type: alias + path: logstash.node.stats.pipelines.vertices.events_in + - name: pipeline_ephemeral_id + type: alias + path: logstash.node.stats.pipelines.vertices.pipeline_ephemeral_id + - name: events_out + type: alias + path: logstash.node.stats.pipelines.vertices.events_out + - name: id + type: alias + path: logstash.node.stats.pipelines.vertices.id + - name: queue_push_duration_in_millis + type: alias + path: logstash.node.stats.pipelines.vertices.queue_push_duration_in_millis + - name: os + type: group + fields: + - name: cpu + type: group + fields: + - name: load_average + type: group + fields: + - name: 15m + type: alias + path: logstash.node.stats.os.cpu.load_average.15m + - name: 1m + type: alias + path: logstash.node.stats.os.cpu.load_average.1m + - name: 5m + type: alias + path: logstash.node.stats.os.cpu.load_average.5m + - name: cgroup + type: group + fields: + - name: cpuacct.usage_nanos + type: alias + path: logstash.node.stats.os.cgroup.cpuacct.usage_nanos + - name: cpu + type: group + fields: + - name: stat + type: group + fields: + - name: number_of_elapsed_periods + type: alias + path: logstash.node.stats.os.cgroup.stat.number_of_elapsed_periods + - name: time_throttled_nanos + type: alias + path: logstash.node.stats.os.cgroup.stat.time_throttled_nanos + - name: number_of_times_throttled + type: alias + path: logstash.node.stats.os.cgroup.stat.number_of_times_throttled + - name: process.cpu.percent + type: alias + path: logstash.node.stats.process.cpu.percent + - name: queue.events_count + type: alias + path: logstash.node.stats.queue.events_count - name: logstash type: group description: > + TODO fields: - name: stats type: group @@ -21,13 +160,13 @@ - name: mem type: group fields: - - name: heap_used + - name: heap_used.bytes type: alias path: logstash.node.stats.jvm.mem.heap_used_in_bytes - - name: heap_max + - name: heap_max.bytes type: alias path: logstash.node.stats.jvm.mem.heap_max_in_bytes - - name: uptime + - name: uptime.ms type: alias path: logstash.node.stats.jvm.uptime_in_millis - name: events @@ -50,7 +189,7 @@ path: logstash.node.stats.logstash.uuid - name: version type: alias - path: logstash.node.stats.logstash.version + path: logstash.node.stats.logstash.version - name: os type: group fields: @@ -62,55 +201,38 @@ fields: - name: 15m type: alias - path: logstash.node.stats.logstash.os.cpu.load_average.15m + path: logstash.node.stats.os.cpu.load_average.15m - name: 1m type: alias - path: logstash.node.stats.logstash.os.cpu.load_average.1m + path: logstash.node.stats.os.cpu.load_average.1m - name: 5m type: alias - path: logstash.node.stats.logstash.os.cpu.load_average.5m + path: logstash.node.stats.os.cpu.load_average.5m - name: cgroup type: group fields: - name: cpuacct.usage_nanos type: alias - path: logstash.node.stats.logstash.os.cgroup.cpuacct.usage_nanos - - name: cpu + path: logstash.node.stats.os.cgroup.cpuacct.usage_nanos + - name: cpu + type: group + fields: + - name: stat type: group fields: - - name: stat - type: group - fields: - - name: number_of_elapsed_periods - type: alias - path: logstash.node.stats.logstash.os.cgroup.cpu.number_of_elapsed_periods - - name: time_throttled_nanos - type: alias - path: logstash.node.stats.logstash.os.cgroup.cpu.time_throttled_nanos - - name: number_of_times_throttled - type: alias - path: logstash.node.stats.logstash.os.cgroup.cpu.number_of_times_throttled - - name: process - type: group - fields: - - name: cpu.percent - type: alias - path: logstash.node.stats.cpu.percent - - name: queue - type: group - fields: - - name: events_count - type: alias - path: logstash.node.stats.queue.events_count - - name: state - type: group - fields: - - name: pipeline - type: group - fields: - - name: id - type: alias - path: logstash.node.stats.pipelines.main.id - - name: hash - type: alias - path: logstash.node.stats.pipelines.main.hash + - name: number_of_elapsed_periods + type: alias + path: logstash.node.stats.os.cgroup.stat.number_of_elapsed_periods + - name: time_throttled_nanos + type: alias + path: logstash.node.stats.os.cgroup.stat.time_throttled_nanos + - name: number_of_times_throttled + type: alias + path: logstash.node.stats.os.cgroup.stat.number_of_times_throttled + - name: process.cpu.percent + type: alias + path: logstash.node.stats.process.cpu.percent + - name: queue.events_count + type: alias + path: logstash.node.stats.queue.events_count + \ No newline at end of file diff --git a/metricbeat/module/logstash/fields.go b/metricbeat/module/logstash/fields.go index 1aec34bd63c..d1628941a22 100644 --- a/metricbeat/module/logstash/fields.go +++ b/metricbeat/module/logstash/fields.go @@ -32,5 +32,5 @@ func init() { // AssetLogstash returns asset data. // This is the base64 encoded gzipped contents of module/logstash. func AssetLogstash() string { - return "eJzsWD2P4zYQ7f0rBq5zBFKkcRGkuATZIMGmCK4JAoErzcq841f44dz++0CypeiDImVJtlMsiy3k1XtvyJnh6H2AL/h2AK5K66g97gAccxwPsP/18mi/AyjQ5oZpx5Q8wPc7AIDmZxCq8Bx3AAY5UosHKOkOwKJzTJb2AH/ureX7b2B/dE7v/9oBvDLkhT3UOB9AUoE9BdVyb7pCMsrry5OAhj5SF8066mz7NAQXeruL4JhA66jQvV8bJMoZtYNfNHXHA+x/aN/cB4E/n0QQcihuSmAXS+AQK44Xw+ziHpHqzFssgv8V24RmnTejOVQiVYGkPhTy+SSIQEFajozJ7OXNYRiqJ0nQr7dWJOjXaUGNGK+rQ57c+ikZcQln0IpcMM6ZDWYPnlC6IfbyBGJywyDO2kgAs6FT3m3PFwJtCAtvaNUxiAhBryRuwBNHNuhsffIlh+Y9CxXm0mjaZ0HchvSExjK1Zbq0z0LQbcZsl+u59jdpllzRIqMnNLQMtYQ0SYqoS/btd6GO3+eJdUGYdyTKklx70g2NxKhbeY9Tlxb3uK0LMLdZOZUV6xMz157muSPe0hIzSeWomOYHPzPwWi6ZSxyvzPQmpDYCBhNh8vhjRHPIuoTSixc0mXrNkFNdzTkaDVNFLL1gdiLCgjMhyyR1J+LMHY1yjmMRzac7xHK1mvG51IP6fxj/h3OJSWoC0EblaDe9F4lGk6PccjibQm1Y//boh9fl8hjOE1mWK79pELVIEgTv9hVc9aWpmUbO5Habsel02KizRFAmSWRCPI5n3A2Je+htLasivfdB4+C8Bu/3HYxmxc7uqOww31IGQfUKqf5UCIP/EKw8f1IcwJlReUQCqdbPyjoYgcbn95RYi+bEcgyO6CvltgbSp8j4f41ZkuD75dNv8CRf1ZXFlPru+YJv/ygTqriEnmqFQodea1haypdLgoQR4gc3U/zvZwp4+hisSzLPg0tUZ1ajgEBnWG7JqmLd0nmLmjVdcK5kuWx//3j++DzJf3PnL+7KJaKbGeFklCNFUVfuxoIWWm8Juh9rtJFRDetMu+XZ9iRzJZgsL2FCPe2gIZMqYl7echnP3pXqGhmvjDs0wel9rZafLtBRLY9x+VbdOtHGctP7brK23m2+Ptkcmy/S6mB+u4NYD4brrL07K3r8Ht3LxJvv4d3hQn5360ZKEmkG16UapAoAtnDlHqh5nQV3B+EDq23SzGokFcq/8Cu/fydJI6ZTYhuWMrZWy2aX8JRXdPv5ZWwWbc/9bwAAAP//+LIvcQ==" + return "eJzsWk+PpLYTvfenKPX5t5Z+h1z6EOWwiTJRoskh2ksUIQ/UdHsXMMGmM5NPH9HdMID/AjY9h+Gw0jJNvVdlu8x7+BN8w9cD5PwoJBWnHYBkMscD7H+93drvADIUac0qyXh5gO93AADdn6HgWZPjDqDGHKnAAxzpDkCglKw8igP8uRci3/8P9icpq/1fO4BnhnkmDpc4n6CkBb4xSISkUlz+BCBfqzZezZvqdmf46PBxyQoUkhZV/5fuaZozKgZ3KypPB9j/0D+xV4J9PRdKmCEJHZHh8wUWo/umGKY4w1gnpFXSCMwSViZPrxKF8lNTpuOMuwKTkmdILlUmX88FKbAgHiAjPgV9iU7HiNExaap2CNvfFCzP2RTKTMJOwBi2w8UzllKsmiCsDED2yoNMYnUQvJHhMKbBOpCsqWnbFIIOwg3TEnvaM1aNRdOwLADr/p4SrwM6Yy0YDzHy/b1pyA6qYhXmrER1lpYoJGa+0zREYXouxFCX03gE1+MoATukvxtsMFhnbp8M2Pze+F9oEm34USaJYP9ijD48peKD1u999CXZlp0PorF3w6opoDbZMIkZ2i7Mab1BmTjQBi1OslQZbUPr8anwRml2xF2JgjKXEmUvD8zIDDPt9glWJyywpnmidO7ApPwgJ5WKtVqmpbItm9iFsVTh2qOqRpySrWe1H3Tf1da93KZVE6zB5pxmCT1jTY/mvVYX2BZ8CPD/76ZSbRzbVH6wDgEXJK0aMmRPTEg9k82I2HlsVpAJUD99dMO5fAalVUPTVJJG0CMmJS15yLXW5nVhRVw4MVZHyyHKqiib4gnrhD8nmNNKYJZUWDOemQZ+7dS4lrD9P5kHPbR/EnmquZQ5ZsZhDszVG1Wt68V9ent2y7qaoPvXiZqnKK4LtsI6xVIq+4HeVNPuRpZoo52x271T3qwCNAQzeAbqWtF4ne31x+Pnx9t/TB7k0LnUx9Y9DQ4X0z4PLG4mGB1NMzkTQbC6m/Z4tpigczqJSTHaS9FdQexOhVhBX7bhZfVWx94nKUK7r1ZPNZKIDiqg9KYoRNPsM5R62NGaJ8w1TimsGjSNa7omG7N7Ck4HNQjuLXT3hLaIyua+6q00Sg916CU3iAsIvLQTeDVECKahwEdHRSPl5rR5oTSAVn0FQWafn9byyzmE5oINVpxBg7mDuwBghR6DQNNqsS6DZdosAu9Z6Et1WgTefhT8NJud3jLtBr76bSm4M+hgl+7HjWfoFGAGgddek+fHJ1q6yybgTlzMzb19hLT/tBEmvyjY8fp6dwBZK58uLYm0189cSFCCrvsaLbA+sxSVD88B6PYHir5oYi/RsQ68X778Bg/lM5/55uZ6B/2Gr//wWtcoHHzaS5c6jL70LH3p7payPoJ94DzJ/36FgIfP2nVJ/OwRx+q8Hg+DAmXNUkFWLdaQpojjMNJb8JyXx2X1HThQKn50U8ZulTiy88zQmKXCyHEALSqhhTaIA+7HSzTFQ4R1Bsry2fZQprxg5fGWJlx2YKzJIl9lOY3HRh75HBrPLJdYa9/U1nL56Rbam8uMz7w+rIwzcVt3Z9UOZ21iUfdWY/U+bJ0xmI+tY2mr4N9awdbvYZ6nszGj+9doKz/H387ZYPPXLzB3tjBj/lscHT8gHzBY6e6A7zSDeVMNXAsAljs774LzEmtnU+JzLZ2MN0/5TK1tBtUcHId1X/fu8SKhOVK+BazufDkE6cRWG2wI4tuDnZ3WcLodPMo4H8zvwHiELP1PqwcCt4pHCDJX9EIsEH+YK28C4TpOmEOQym2ZEHieJI8A53teHJasdf/gXqfD56fvCGfdlxZl7Ehv7rHvNw7POTe+iY4Y+EN8aF49WGjNew9Vew/deldlGkGafgjQ+AL0/cnLdyoejSrN83u7Bk+N/18AAAD//29X95w=" } diff --git a/metricbeat/module/logstash/node_stats/_meta/data.json b/metricbeat/module/logstash/node_stats/_meta/data.json index 404915a5faa..8d9623ef81b 100644 --- a/metricbeat/module/logstash/node_stats/_meta/data.json +++ b/metricbeat/module/logstash/node_stats/_meta/data.json @@ -1,148 +1,140 @@ { - "@timestamp": "2020-10-05T15:07:19.603Z", + "@timestamp": "2020-10-30T12:41:12.793Z", "@metadata": { - "beat": "metricbeat", - "type": "_doc", - "version": "8.0.0" + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" }, "metricset": { - "name": "node_stats", - "period": 10000 + "name": "node_stats", + "period": 10000 }, "logstash": { - "node": { - "stats": { - "queue": { - "events_count": 0 - }, - "events": { - "duration_in_millis": 0, - "in": 0, - "filtered": 0, - "out": 0 - }, - "jvm": { - "gc": { - "collectors": { - "old": { - "collection_count": 5, - "collection_time_in_millis": 1177 - }, - "young": { - "collection_count": 15, - "collection_time_in_millis": 407 - } - } - }, - "mem": { - "heap_max_in_bytes": 1.037959168e+09, - "heap_used_in_bytes": 2.02339024e+08, - "heap_used_percent": 19 - }, - "uptime_in_millis": 4.602954e+06 - }, - "reloads": { - "failures": 0, - "successes": 0 - }, - "process": { - "open_file_descriptors": 229, - "max_file_descriptors": 1.048576e+06, - "cpu": { - "percent": 0 - } - }, - "os": { - "cpu": { - "percent": 0, - "load_average": { - "1m": 1.96, - "5m": 1.76, - "15m": 1.81 - } - }, - "cgroup": { - "cpuacct": { - "control_group": "/", - "usage_nanos": 8.9799096638e+11 - }, - "cpu": { - "stat": { - "time_throttled_nanos": 0, - "number_of_elapsed_periods": 0, - "number_of_times_throttled": 0 - }, - "control_group": "/" - } - } - }, - "pipelines": [ - { - "id": "main", - "hash": "3000c3abf87d4dfa4a57aaf6af0a1f5bee2e0fc1c48a8e8636e2a33d7d2e91dd", - "ephemeral_id": "2c607fe3-0d89-4506-8fe8-32180d201e48", - "events": { - "queue_push_duration_in_millis": 0, - "duration_in_millis": 0, - "filtered": 0, - "in": 0, - "out": 0 - }, - "reloads": { - "successes": 0, - "failures": 0 - }, - "queue": { - "events_count": 0, - "max_queue_size_in_bytes": 0, - "queue_size_in_bytes": 0, - "type": "memory" - }, - "vertices": null - } - ], - "logstash": { - "snapshot": true, - "http_address": "0.0.0.0:9600", - "uuid": "4cc683ce-3ddc-46e3-bea3-aefbf37bc082", - "ephemeral_id": "94b3a942-8dff-4cfb-8ba2-51672230b903", - "name": "2cb47f6e0eab", - "host": "2cb47f6e0eab", - "version": "8.0.0", - "status": "green", - "pipeline": { - "batch_size": 125, - "workers": 12 - } + "node": { + "stats": { + "events": { + "duration_in_millis": 0, + "in": 0, + "filtered": 0, + "out": 0 + }, + "reloads": { + "failures": 0, + "successes": 0 + }, + "queue": { + "events_count": 0 + }, + "os": { + "cpu": { + "percent": 0, + "load_average": { + "1m": 2.814453125 + } + }, + "cgroup": { + "cpuacct": null, + "cpu": { + "stat": null, + "control_group": "" + } + } + }, + "pipelines": [ + { + "id": "main", + "hash": "4dbe1b282a0265b7c90ae9212a3b4bef55168d267004e5124a3486f9151d4188", + "ephemeral_id": "5fd09014-09f9-449a-8b19-692c82f5494e", + "events": { + "filtered": 0, + "in": 0, + "out": 0, + "queue_push_duration_in_millis": 0, + "duration_in_millis": 0 + }, + "reloads": { + "successes": 0, + "failures": 0 + }, + "queue": { + "events_count": 0, + "max_queue_size_in_bytes": 0, + "queue_size_in_bytes": 0, + "type": "memory" + }, + "vertices": null + } + ], + "jvm": { + "gc": { + "collectors": { + "old": { + "collection_count": 1, + "collection_time_in_millis": 198 }, - "timestamp": "2020-10-05T15:07:19.613Z" + "young": { + "collection_time_in_millis": 298, + "collection_count": 10 + } + } + }, + "mem": { + "heap_max_in_bytes": 1.073741822e+09, + "heap_used_in_bytes": 3.18658432e+08, + "heap_used_percent": 29 + }, + "uptime_in_millis": 57904 + }, + "process": { + "open_file_descriptors": 93, + "max_file_descriptors": 10240, + "cpu": { + "percent": 0 } + }, + "logstash": { + "uuid": "33f30cf2-d580-48fb-ad7c-2fbaf85db604", + "http_address": "127.0.0.1:9600", + "pipeline": { + "batch_size": 125, + "workers": 4 + }, + "status": "green", + "ephemeral_id": "c2f01c4f-5c09-4b65-a922-15fdf3634df3", + "name": "mcastro", + "host": "mcastro", + "version": "7.9.3", + "snapshot": false + }, + "timestamp": "2020-10-30T12:41:13.799Z" } + } + }, + "ecs": { + "version": "1.5.0" }, "host": { - "name": "mcastro" + "name": "mcastro" }, "agent": { - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "41749b5d-ffb6-4ae0-9626-4673ce48ae7c", - "id": "803dfdba-e638-4590-a2de-80cb1cebe78d", - "name": "mcastro" - }, - "ecs": { - "version": "1.5.0" + "name": "mcastro", + "type": "metricbeat", + "version": "8.0.0", + "ephemeral_id": "fb36ebe3-f38d-4e5f-b68e-8d6f578ed471", + "id": "abfbaa55-b79d-4ae5-b412-fd35b933c6d3" }, "service": { - "hostname": "2cb47f6e0eab", - "version": "8.0.0", - "address": "localhost:9600", - "type": "logstash", - "name": "logstash", - "id": "" + "name": "logstash", + "id": "", + "hostname": "mcastro", + "version": "7.9.3", + "address": "localhost:9600", + "type": "logstash" }, "event": { - "dataset": "logstash.node.stats", - "module": "logstash", - "duration": 10447360 + "module": "logstash", + "duration": 1005737788, + "dataset": "logstash.node.stats" } -} + } + \ No newline at end of file diff --git a/metricbeat/module/logstash/node_stats/_meta/fields.yml b/metricbeat/module/logstash/node_stats/_meta/fields.yml index b2d214c105d..94aa3262398 100644 --- a/metricbeat/module/logstash/node_stats/_meta/fields.yml +++ b/metricbeat/module/logstash/node_stats/_meta/fields.yml @@ -39,6 +39,9 @@ type: long description: > Filtered events counter. + - name: duration_in_millis + type: long + description: TODO - name: logstash type: group fields: @@ -100,18 +103,101 @@ type: double description: > TODO - - name: queue.events_count - type: long - description: > - TODO - name: pipelines type: group fields: - - name: main.id + - name: id type: keyword description: > TODO - - name: main.hash + - name: hash type: keyword description: > TODO + - name: queue + type: group + fields: + - name: events_count + type: long + description: TODO + - name: type + type: keyword + description: TODO + - name: queue_size_in_bytes + type: long + description: TODO + - name: max_queue_size_in_bytes + type: long + description: TODO + - name: events + type: group + fields: + - name: out + type: long + description: TODO + - name: duration_in_millis + type: long + description: TODO + - name: vertices + type: group + fields: + - name: duration_in_millis + type: long + description: TODO + - name: events_in + type: long + description: TODO + - name: pipeline_ephemeral_id + type: keyword + description: pipeline_ephemeral_id + - name: events_out + type: long + description: events_out + - name: id + type: keyword + description: id + - name: queue_push_duration_in_millis + type: float + description: queue_push_duration_in_millis + - name: os + type: group + fields: + - name: cpu + type: group + fields: + - name: load_average + type: group + fields: + - name: 15m + type: long + description: TODO + - name: 1m + type: long + description: TODO + - name: 5m + type: long + description: TODO + - name: cgroup + type: group + fields: + - name: cpuacct.usage_nanos + type: long + description: TODO + - name: cpu + type: group + fields: + - name: stat + type: group + fields: + - name: number_of_elapsed_periods + type: long + description: TODO + - name: time_throttled_nanos + type: long + description: TODO + - name: number_of_times_throttled + type: long + description: TODO + - name: queue.events_count + type: long + description: TODO From 63cd98c83f01c3485293bd0a2bda1ba510276659 Mon Sep 17 00:00:00 2001 From: sayden Date: Tue, 3 Nov 2020 18:45:53 +0100 Subject: [PATCH 03/17] Fix test that got broken --- metricbeat/module/logstash/node_stats/data.go | 2 +- metricbeat/module/logstash/node_stats/node_stats.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/metricbeat/module/logstash/node_stats/data.go b/metricbeat/module/logstash/node_stats/data.go index 0f28bf333ee..eeeb00c7b8c 100644 --- a/metricbeat/module/logstash/node_stats/data.go +++ b/metricbeat/module/logstash/node_stats/data.go @@ -143,7 +143,7 @@ type PipelineStats struct { Vertices []map[string]interface{} `json:"vertices"` } -func eventMapping(r mb.ReporterV2, m *MetricSet, content []byte) error { +func eventMapping(r mb.ReporterV2, content []byte) error { var nodeStats NodeStats err := json.Unmarshal(content, &nodeStats) if err != nil { diff --git a/metricbeat/module/logstash/node_stats/node_stats.go b/metricbeat/module/logstash/node_stats/node_stats.go index c12f4cd503d..56671bc5f96 100644 --- a/metricbeat/module/logstash/node_stats/node_stats.go +++ b/metricbeat/module/logstash/node_stats/node_stats.go @@ -77,7 +77,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { return err } - if err = eventMapping(r, m, content); err != nil { + if err = eventMapping(r, content); err != nil { return err } From d8466399e5eb4a22a86cca80ed94766ba716d181 Mon Sep 17 00:00:00 2001 From: sayden Date: Fri, 4 Dec 2020 15:43:12 +0100 Subject: [PATCH 04/17] Fix some tests and add a TestData test --- metricbeat/module/logstash/logstash.go | 11 ------ .../node_stats/_meta/test/root.710.json | 18 +++++++++ .../module/logstash/node_stats/data_test.go | 37 +++++++++++++++++++ .../module/logstash/node_stats/node_stats.go | 9 +---- .../logstash/node_stats/node_stats_test.go | 4 +- 5 files changed, 59 insertions(+), 20 deletions(-) create mode 100644 metricbeat/module/logstash/node_stats/_meta/test/root.710.json diff --git a/metricbeat/module/logstash/logstash.go b/metricbeat/module/logstash/logstash.go index fbc030fa310..1073be12a9d 100644 --- a/metricbeat/module/logstash/logstash.go +++ b/metricbeat/module/logstash/logstash.go @@ -54,7 +54,6 @@ var PipelineGraphAPIsAvailableVersion = common.MustNewVersion("7.3.0") type MetricSet struct { mb.BaseMetricSet *helper.HTTP - XPack bool } type graph struct { @@ -83,15 +82,6 @@ type PipelineState struct { // NewMetricSet creates a metricset that can be used to build other metricsets // within the Logstash module. func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { - config := struct { - XPack bool `config:"xpack.enabled"` - }{ - XPack: false, - } - if err := base.Module().UnpackConfig(&config); err != nil { - return nil, err - } - http, err := helper.NewHTTP(base) if err != nil { return nil, err @@ -100,7 +90,6 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { return &MetricSet{ base, http, - config.XPack, }, nil } diff --git a/metricbeat/module/logstash/node_stats/_meta/test/root.710.json b/metricbeat/module/logstash/node_stats/_meta/test/root.710.json new file mode 100644 index 00000000000..6f94e4d587f --- /dev/null +++ b/metricbeat/module/logstash/node_stats/_meta/test/root.710.json @@ -0,0 +1,18 @@ +{ + "host": "anonymous", + "version": "7.8.1", + "http_address": "127.0.0.1:9600", + "id": "2fc9524f-a36b-4611-82e4-5246d3ac4714", + "name": "anonymous", + "ephemeral_id": "06807db4-dbb2-4260-982f-b6ea4dfa5270", + "status": "green", + "snapshot": false, + "pipeline": { + "workers": 12, + "batch_size": 125, + "batch_delay": 50 + }, + "build_date": "2020-07-21T19:19:46+00:00", + "build_sha": "5dcccb963be4c163647232fe4b67bdf4b8efc2cb", + "build_snapshot": false +} diff --git a/metricbeat/module/logstash/node_stats/data_test.go b/metricbeat/module/logstash/node_stats/data_test.go index 2ae2407ca9e..ed4f63f0b3c 100644 --- a/metricbeat/module/logstash/node_stats/data_test.go +++ b/metricbeat/module/logstash/node_stats/data_test.go @@ -20,7 +20,10 @@ package node_stats import ( + "github.com/elastic/beats/v7/metricbeat/module/logstash" "io/ioutil" + "net/http" + "net/http/httptest" "path/filepath" "testing" @@ -46,3 +49,37 @@ func TestEventMapping(t *testing.T) { require.Equal(t, 0, len(reporter.GetErrors()), f) } } + +func TestData(t *testing.T) { + mux := http.NewServeMux() + mux.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/" { + http.NotFound(w, r) + } + + input, _ := ioutil.ReadFile("./_meta/test/root.710.json") + w.Write(input) + })) + + mux.Handle("/_node/stats", http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + input, _ := ioutil.ReadFile("./_meta/test/node_stats.700.json") + w.Write(input) + })) + + server := httptest.NewServer(mux) + defer server.Close() + + ms := mbtest.NewReportingMetricSetV2Error(t, getConfig(server.URL)) + if err := mbtest.WriteEventsReporterV2Error(ms, t, ""); err != nil { + t.Fatal("write", err) + } +} + +func getConfig(host string) map[string]interface{} { + return map[string]interface{}{ + "module": logstash.ModuleName, + "metricsets": []string{"node_stats"}, + "hosts": []string{host}, + } +} diff --git a/metricbeat/module/logstash/node_stats/node_stats.go b/metricbeat/module/logstash/node_stats/node_stats.go index 56671bc5f96..681183cfe6d 100644 --- a/metricbeat/module/logstash/node_stats/node_stats.go +++ b/metricbeat/module/logstash/node_stats/node_stats.go @@ -85,7 +85,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { } func (m *MetricSet) updateServiceURI() error { - u, err := getServiceURI(m.GetURI(), m.XPack, m.CheckPipelineGraphAPIsAvailable) + u, err := getServiceURI(m.GetURI(), m.CheckPipelineGraphAPIsAvailable) if err != nil { return err } @@ -95,12 +95,7 @@ func (m *MetricSet) updateServiceURI() error { } -func getServiceURI(currURI string, xpackEnabled bool, graphAPIsAvailable func() error) (string, error) { - if !xpackEnabled { - // No need to request pipeline vertices from service API - return currURI, nil - } - +func getServiceURI(currURI string, graphAPIsAvailable func() error) (string, error) { if err := graphAPIsAvailable(); err != nil { return "", err } diff --git a/metricbeat/module/logstash/node_stats/node_stats_test.go b/metricbeat/module/logstash/node_stats/node_stats_test.go index 8c11ecdde3d..4b0ebad519c 100644 --- a/metricbeat/module/logstash/node_stats/node_stats_test.go +++ b/metricbeat/module/logstash/node_stats/node_stats_test.go @@ -58,7 +58,7 @@ func TestGetServiceURI(t *testing.T) { for name, test := range tests { t.Run(name, func(t *testing.T) { - newURI, err := getServiceURI(nodeStatsPath, test.xpackEnabled, test.graphAPIsAvailable) + newURI, err := getServiceURI(nodeStatsPath, test.graphAPIsAvailable) if test.errExpected { require.Equal(t, "", newURI) } else { @@ -77,7 +77,7 @@ func TestGetServiceURIMultipleCalls(t *testing.T) { numCalls := 2 + (r % 10) // between 2 and 11 for i := uint(0); i < numCalls; i++ { - uri, err = getServiceURI(uri, true, func() error { return nil }) + uri, err = getServiceURI(uri, func() error { return nil }) if err != nil { return false } From 9edd2c813ccbc2f7c7e2ade3cf824e3e0fd076dc Mon Sep 17 00:00:00 2001 From: sayden Date: Fri, 4 Dec 2020 15:59:19 +0100 Subject: [PATCH 05/17] Merge changes from master --- metricbeat/module/logstash/node_stats/data.go | 9 +- .../module/logstash/node_stats/data_test.go | 247 ++++++++++++++++ .../logstash/node_stats/data_xpack_test.go | 273 ------------------ 3 files changed, 251 insertions(+), 278 deletions(-) delete mode 100644 metricbeat/module/logstash/node_stats/data_xpack_test.go diff --git a/metricbeat/module/logstash/node_stats/data.go b/metricbeat/module/logstash/node_stats/data.go index eeeb00c7b8c..fe9732498f5 100644 --- a/metricbeat/module/logstash/node_stats/data.go +++ b/metricbeat/module/logstash/node_stats/data.go @@ -119,7 +119,6 @@ type NodeStats struct { Process process `json:"process"` OS os `json:"os"` Pipelines map[string]PipelineStats `json:"pipelines"` - Events events `json:"events"` } // LogstashStats represents the logstash_stats sub-document indexed into .monitoring-logstash-* @@ -222,20 +221,20 @@ func makeClusterToPipelinesMap(pipelines []PipelineStats, overrideClusterUUID st } for _, pipeline := range pipelines { - var clusterUUIDs []string + clusterUUIDs := common.StringSet{} for _, vertex := range pipeline.Vertices { clusterUUID := logstash.GetVertexClusterUUID(vertex, overrideClusterUUID) if clusterUUID != "" { - clusterUUIDs = append(clusterUUIDs, clusterUUID) + clusterUUIDs.Add(clusterUUID) } } // If no cluster UUID was found in this pipeline, assign it a blank one if len(clusterUUIDs) == 0 { - clusterUUIDs = []string{""} + clusterUUIDs.Add("") } - for _, clusterUUID := range clusterUUIDs { + for clusterUUID := range clusterUUIDs { clusterPipelines := clusterToPipelinesMap[clusterUUID] if clusterPipelines == nil { clusterToPipelinesMap[clusterUUID] = []PipelineStats{} diff --git a/metricbeat/module/logstash/node_stats/data_test.go b/metricbeat/module/logstash/node_stats/data_test.go index ed4f63f0b3c..1a30c773f73 100644 --- a/metricbeat/module/logstash/node_stats/data_test.go +++ b/metricbeat/module/logstash/node_stats/data_test.go @@ -83,3 +83,250 @@ func getConfig(host string) map[string]interface{} { "hosts": []string{host}, } } + +func TestMakeClusterToPipelinesMap(t *testing.T) { + tests := map[string]struct { + pipelines []PipelineStats + overrideClusterUUID string + expectedMap map[string][]PipelineStats + }{ + "no_vertex_cluster_id": { + pipelines: []PipelineStats{ + { + ID: "test_pipeline", + Vertices: []map[string]interface{}{ + { + "id": "vertex_1", + }, + { + "id": "vertex_2", + }, + { + "id": "vertex_3", + }, + }, + }, + }, + overrideClusterUUID: "prod_cluster_id", + expectedMap: map[string][]PipelineStats{ + "prod_cluster_id": { + { + ID: "test_pipeline", + Vertices: []map[string]interface{}{ + { + "id": "vertex_1", + }, + { + "id": "vertex_2", + }, + { + "id": "vertex_3", + }, + }, + }, + }, + }, + }, + "one_vertex_cluster_id": { + pipelines: []PipelineStats{ + { + ID: "test_pipeline", + Vertices: []map[string]interface{}{ + { + "id": "vertex_1", + "cluster_uuid": "es_1", + }, + { + "id": "vertex_2", + }, + { + "id": "vertex_3", + }, + }, + }, + }, + overrideClusterUUID: "prod_cluster_id", + expectedMap: map[string][]PipelineStats{ + "prod_cluster_id": { + { + ID: "test_pipeline", + Vertices: []map[string]interface{}{ + { + "id": "vertex_1", + "cluster_uuid": "es_1", + }, + { + "id": "vertex_2", + }, + { + "id": "vertex_3", + }, + }, + }, + }, + }, + }, + "two_pipelines": { + pipelines: []PipelineStats{ + { + ID: "test_pipeline_1", + Vertices: []map[string]interface{}{ + { + "id": "vertex_1_1", + "cluster_uuid": "es_1", + }, + { + "id": "vertex_1_2", + }, + { + "id": "vertex_1_3", + }, + }, + }, + { + ID: "test_pipeline_2", + Vertices: []map[string]interface{}{ + { + "id": "vertex_2_1", + }, + { + "id": "vertex_2_2", + }, + { + "id": "vertex_2_3", + }, + }, + }, + }, + overrideClusterUUID: "prod_cluster_id", + expectedMap: map[string][]PipelineStats{ + "prod_cluster_id": { + { + ID: "test_pipeline_1", + Vertices: []map[string]interface{}{ + { + "id": "vertex_1_1", + "cluster_uuid": "es_1", + }, + { + "id": "vertex_1_2", + }, + { + "id": "vertex_1_3", + }, + }, + }, + { + ID: "test_pipeline_2", + Vertices: []map[string]interface{}{ + { + "id": "vertex_2_1", + }, + { + "id": "vertex_2_2", + }, + { + "id": "vertex_2_3", + }, + }, + }, + }, + }, + }, + "no_override_cluster_id": { + pipelines: []PipelineStats{ + { + ID: "test_pipeline_1", + Vertices: []map[string]interface{}{ + { + "id": "vertex_1_1", + "cluster_uuid": "es_1", + }, + { + "id": "vertex_1_2", + "cluster_uuid": "es_2", + }, + { + "id": "vertex_1_3", + }, + }, + }, + { + ID: "test_pipeline_2", + Vertices: []map[string]interface{}{ + { + "id": "vertex_2_1", + }, + { + "id": "vertex_2_2", + }, + { + "id": "vertex_2_3", + }, + }, + }, + }, + expectedMap: map[string][]PipelineStats{ + "es_1": { + { + ID: "test_pipeline_1", + Vertices: []map[string]interface{}{ + { + "id": "vertex_1_1", + "cluster_uuid": "es_1", + }, + { + "id": "vertex_1_2", + "cluster_uuid": "es_2", + }, + { + "id": "vertex_1_3", + }, + }, + }, + }, + "es_2": { + { + ID: "test_pipeline_1", + Vertices: []map[string]interface{}{ + { + "id": "vertex_1_1", + "cluster_uuid": "es_1", + }, + { + "id": "vertex_1_2", + "cluster_uuid": "es_2", + }, + { + "id": "vertex_1_3", + }, + }, + }, + }, + "": { + { + ID: "test_pipeline_2", + Vertices: []map[string]interface{}{ + { + "id": "vertex_2_1", + }, + { + "id": "vertex_2_2", + }, + { + "id": "vertex_2_3", + }, + }, + }, + }, + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + actualMap := makeClusterToPipelinesMap(test.pipelines, test.overrideClusterUUID) + require.Equal(t, test.expectedMap, actualMap) + }) + } +} diff --git a/metricbeat/module/logstash/node_stats/data_xpack_test.go b/metricbeat/module/logstash/node_stats/data_xpack_test.go deleted file mode 100644 index 6593be72534..00000000000 --- a/metricbeat/module/logstash/node_stats/data_xpack_test.go +++ /dev/null @@ -1,273 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// +build !integration - -package node_stats - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestMakeClusterToPipelinesMap(t *testing.T) { - tests := map[string]struct { - pipelines []PipelineStats - overrideClusterUUID string - expectedMap map[string][]PipelineStats - }{ - "no_vertex_cluster_id": { - pipelines: []PipelineStats{ - { - ID: "test_pipeline", - Vertices: []map[string]interface{}{ - { - "id": "vertex_1", - }, - { - "id": "vertex_2", - }, - { - "id": "vertex_3", - }, - }, - }, - }, - overrideClusterUUID: "prod_cluster_id", - expectedMap: map[string][]PipelineStats{ - "prod_cluster_id": { - { - ID: "test_pipeline", - Vertices: []map[string]interface{}{ - { - "id": "vertex_1", - }, - { - "id": "vertex_2", - }, - { - "id": "vertex_3", - }, - }, - }, - }, - }, - }, - "one_vertex_cluster_id": { - pipelines: []PipelineStats{ - { - ID: "test_pipeline", - Vertices: []map[string]interface{}{ - { - "id": "vertex_1", - "cluster_uuid": "es_1", - }, - { - "id": "vertex_2", - }, - { - "id": "vertex_3", - }, - }, - }, - }, - overrideClusterUUID: "prod_cluster_id", - expectedMap: map[string][]PipelineStats{ - "prod_cluster_id": { - { - ID: "test_pipeline", - Vertices: []map[string]interface{}{ - { - "id": "vertex_1", - "cluster_uuid": "es_1", - }, - { - "id": "vertex_2", - }, - { - "id": "vertex_3", - }, - }, - }, - }, - }, - }, - "two_pipelines": { - pipelines: []PipelineStats{ - { - ID: "test_pipeline_1", - Vertices: []map[string]interface{}{ - { - "id": "vertex_1_1", - "cluster_uuid": "es_1", - }, - { - "id": "vertex_1_2", - }, - { - "id": "vertex_1_3", - }, - }, - }, - { - ID: "test_pipeline_2", - Vertices: []map[string]interface{}{ - { - "id": "vertex_2_1", - }, - { - "id": "vertex_2_2", - }, - { - "id": "vertex_2_3", - }, - }, - }, - }, - overrideClusterUUID: "prod_cluster_id", - expectedMap: map[string][]PipelineStats{ - "prod_cluster_id": { - { - ID: "test_pipeline_1", - Vertices: []map[string]interface{}{ - { - "id": "vertex_1_1", - "cluster_uuid": "es_1", - }, - { - "id": "vertex_1_2", - }, - { - "id": "vertex_1_3", - }, - }, - }, - { - ID: "test_pipeline_2", - Vertices: []map[string]interface{}{ - { - "id": "vertex_2_1", - }, - { - "id": "vertex_2_2", - }, - { - "id": "vertex_2_3", - }, - }, - }, - }, - }, - }, - "no_override_cluster_id": { - pipelines: []PipelineStats{ - { - ID: "test_pipeline_1", - Vertices: []map[string]interface{}{ - { - "id": "vertex_1_1", - "cluster_uuid": "es_1", - }, - { - "id": "vertex_1_2", - "cluster_uuid": "es_2", - }, - { - "id": "vertex_1_3", - }, - }, - }, - { - ID: "test_pipeline_2", - Vertices: []map[string]interface{}{ - { - "id": "vertex_2_1", - }, - { - "id": "vertex_2_2", - }, - { - "id": "vertex_2_3", - }, - }, - }, - }, - expectedMap: map[string][]PipelineStats{ - "es_1": { - { - ID: "test_pipeline_1", - Vertices: []map[string]interface{}{ - { - "id": "vertex_1_1", - "cluster_uuid": "es_1", - }, - { - "id": "vertex_1_2", - "cluster_uuid": "es_2", - }, - { - "id": "vertex_1_3", - }, - }, - }, - }, - "es_2": { - { - ID: "test_pipeline_1", - Vertices: []map[string]interface{}{ - { - "id": "vertex_1_1", - "cluster_uuid": "es_1", - }, - { - "id": "vertex_1_2", - "cluster_uuid": "es_2", - }, - { - "id": "vertex_1_3", - }, - }, - }, - }, - "": { - { - ID: "test_pipeline_2", - Vertices: []map[string]interface{}{ - { - "id": "vertex_2_1", - }, - { - "id": "vertex_2_2", - }, - { - "id": "vertex_2_3", - }, - }, - }, - }, - }, - }, - } - - for name, test := range tests { - t.Run(name, func(t *testing.T) { - actualMap := makeClusterToPipelinesMap(test.pipelines, test.overrideClusterUUID) - require.Equal(t, test.expectedMap, actualMap) - }) - } -} From d7e586066c3a9f5704b425debdde726d548b371e Mon Sep 17 00:00:00 2001 From: sayden Date: Wed, 9 Dec 2020 17:21:36 +0100 Subject: [PATCH 06/17] Cleaning things up --- metricbeat/docs/fields.asciidoc | 263 ++------------ metricbeat/module/logstash/_meta/fields.yml | 127 ++----- metricbeat/module/logstash/fields.go | 2 +- .../module/logstash/node/_meta/data.json | 139 +++++++- .../module/logstash/node/_meta/fields.yml | 11 + metricbeat/module/logstash/node/data.go | 123 ++++++- metricbeat/module/logstash/node/data_test.go | 303 +++++++++++++++- metricbeat/module/logstash/node/data_xpack.go | 120 ------- .../module/logstash/node/data_xpack_test.go | 328 ------------------ metricbeat/module/logstash/node/node.go | 30 +- .../logstash/node_stats/_meta/data.json | 250 +++++++------ .../logstash/node_stats/_meta/fields.yml | 39 --- .../node_stats/_meta/test/node_stats.710.json | 192 ++++++++++ .../module/logstash/node_stats/data_test.go | 5 +- 14 files changed, 917 insertions(+), 1015 deletions(-) delete mode 100644 metricbeat/module/logstash/node/data_xpack.go delete mode 100644 metricbeat/module/logstash/node/data_xpack_test.go create mode 100644 metricbeat/module/logstash/node_stats/_meta/test/node_stats.710.json diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 9348f91f3be..e3483b7c66e 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -33169,51 +33169,12 @@ type: nested -*`logstash_stats.os.cpu.load_average.15m`*:: -+ --- -type: alias - -alias to: logstash.node.stats.os.cpu.load_average.15m - --- - -*`logstash_stats.os.cpu.load_average.1m`*:: -+ --- -type: alias - -alias to: logstash.node.stats.os.cpu.load_average.1m - --- - -*`logstash_stats.os.cpu.load_average.5m`*:: -+ --- -type: alias - -alias to: logstash.node.stats.os.cpu.load_average.5m - --- - - -*`logstash_stats.os.cgroup.cpuacct.usage_nanos`*:: -+ --- -type: alias - -alias to: logstash.node.stats.os.cgroup.cpuacct.usage_nanos - --- - - - *`logstash_stats.os.cpu.stat.number_of_elapsed_periods`*:: + -- type: alias -alias to: logstash.node.stats.os.cgroup.stat.number_of_elapsed_periods +alias to: logstash.node.stats.os.cgroup.cpu.stat.number_of_elapsed_periods -- @@ -33222,7 +33183,7 @@ alias to: logstash.node.stats.os.cgroup.stat.number_of_elapsed_periods -- type: alias -alias to: logstash.node.stats.os.cgroup.stat.time_throttled_nanos +alias to: logstash.node.stats.os.cgroup.cpu.stat.time_throttled_nanos -- @@ -33231,215 +33192,114 @@ alias to: logstash.node.stats.os.cgroup.stat.time_throttled_nanos -- type: alias -alias to: logstash.node.stats.os.cgroup.stat.number_of_times_throttled +alias to: logstash.node.stats.os.cgroup.cpu.stat.number_of_times_throttled -- -*`logstash_stats.process.cpu.percent`*:: -+ --- -type: alias - -alias to: logstash.node.stats.process.cpu.percent - --- - -*`logstash_stats.queue.events_count`*:: -+ --- -type: alias - -alias to: logstash.node.stats.queue.events_count - --- - -[float] -=== logstash -TODO - - - - -*`logstash.stats.timestamp`*:: -+ --- -type: alias - -alias to: @timestamp - --- - - - -*`logstash.stats.jvm.mem.heap_used.bytes`*:: -+ --- -type: alias - -alias to: logstash.node.stats.jvm.mem.heap_used_in_bytes - --- - -*`logstash.stats.jvm.mem.heap_max.bytes`*:: +*`logstash_stats.os.cpu.load_average.15m`*:: + -- type: alias -alias to: logstash.node.stats.jvm.mem.heap_max_in_bytes +alias to: logstash.node.stats.os.cpu.load_average.15m -- -*`logstash.stats.jvm.uptime.ms`*:: +*`logstash_stats.os.cpu.load_average.1m`*:: + -- type: alias -alias to: logstash.node.stats.jvm.uptime_in_millis +alias to: logstash.node.stats.os.cpu.load_average.1m -- - -*`logstash.stats.events.in`*:: +*`logstash_stats.os.cpu.load_average.5m`*:: + -- type: alias -alias to: logstash.node.stats.events.in - --- +alias to: logstash.node.stats.os.cpu.load_average.5m -*`logstash.stats.events.out`*:: -+ -- -type: alias - -alias to: logstash.node.stats.events.out --- -*`logstash.stats.events.duration.ms`*:: +*`logstash_stats.os.cgroup.cpuacct.usage_nanos`*:: + -- type: alias -alias to: logstash.node.stats.events.duration_in_millis +alias to: logstash.node.stats.os.cgroup.cpuacct.usage_nanos -- - -*`logstash.stats.logstash.uuid`*:: +*`logstash_stats.process.cpu.percent`*:: + -- type: alias -alias to: logstash.node.stats.logstash.uuid +alias to: logstash.node.stats.process.cpu.percent -- -*`logstash.stats.logstash.version`*:: +*`logstash_stats.queue.events_count`*:: + -- type: alias -alias to: logstash.node.stats.logstash.version +alias to: logstash.node.stats.queue.events_count -- - - -*`logstash.stats.os.cpu.load_average.15m`*:: +*`logstash_state.pipeline.id`*:: + -- type: alias -alias to: logstash.node.stats.os.cpu.load_average.15m +alias to: logstash.node.state.pipeline.id -- -*`logstash.stats.os.cpu.load_average.1m`*:: +*`logstash_state.pipeline.hash`*:: + -- type: alias -alias to: logstash.node.stats.os.cpu.load_average.1m +alias to: logstash.node.state.pipeline.hash -- -*`logstash.stats.os.cpu.load_average.5m`*:: -+ --- -type: alias - -alias to: logstash.node.stats.os.cpu.load_average.5m - --- - - -*`logstash.stats.os.cgroup.cpuacct.usage_nanos`*:: -+ --- -type: alias - -alias to: logstash.node.stats.os.cgroup.cpuacct.usage_nanos - --- +[float] +=== node +node -*`logstash.stats.os.cpu.stat.number_of_elapsed_periods`*:: -+ --- -type: alias -alias to: logstash.node.stats.os.cgroup.stat.number_of_elapsed_periods --- -*`logstash.stats.os.cpu.stat.time_throttled_nanos`*:: +*`logstash.node.state.pipeline.id`*:: + -- -type: alias - -alias to: logstash.node.stats.os.cgroup.stat.time_throttled_nanos - --- +TODO -*`logstash.stats.os.cpu.stat.number_of_times_throttled`*:: -+ --- -type: alias -alias to: logstash.node.stats.os.cgroup.stat.number_of_times_throttled +type: keyword -- -*`logstash.stats.process.cpu.percent`*:: +*`logstash.node.state.pipeline.hash`*:: + -- -type: alias - -alias to: logstash.node.stats.process.cpu.percent - --- +TODO -*`logstash.stats.queue.events_count`*:: -+ --- -type: alias -alias to: logstash.node.stats.queue.events_count +type: keyword -- -[float] -=== node - -node - - - *`logstash.node.host`*:: + -- @@ -33504,9 +33364,6 @@ node_stats metrics. *`logstash.node.stats.jvm.uptime_in_millis`*:: + -- -TODO - - type: long -- @@ -33515,9 +33372,6 @@ type: long *`logstash.node.stats.jvm.mem.heap_used_in_bytes`*:: + -- -TODO - - type: long -- @@ -33525,9 +33379,6 @@ type: long *`logstash.node.stats.jvm.mem.heap_max_in_bytes`*:: + -- -TODO - - type: long -- @@ -33572,8 +33423,6 @@ type: long *`logstash.node.stats.events.duration_in_millis`*:: + -- -TODO - type: long -- @@ -33582,9 +33431,6 @@ type: long *`logstash.node.stats.logstash.uuid`*:: + -- -TODO - - type: keyword -- @@ -33592,9 +33438,6 @@ type: keyword *`logstash.node.stats.logstash.version`*:: + -- -TODO - - type: keyword -- @@ -33605,9 +33448,6 @@ type: keyword *`logstash.node.stats.os.cpu.load_average.15m`*:: + -- -TODO - - type: long -- @@ -33615,9 +33455,6 @@ type: long *`logstash.node.stats.os.cpu.load_average.1m`*:: + -- -TODO - - type: long -- @@ -33625,9 +33462,6 @@ type: long *`logstash.node.stats.os.cpu.load_average.5m`*:: + -- -TODO - - type: long -- @@ -33636,9 +33470,6 @@ type: long *`logstash.node.stats.os.cgroup.cpuact.usage_nanos`*:: + -- -TODO - - type: long -- @@ -33689,9 +33520,6 @@ type: double *`logstash.node.stats.pipelines.id`*:: + -- -TODO - - type: keyword -- @@ -33699,9 +33527,6 @@ type: keyword *`logstash.node.stats.pipelines.hash`*:: + -- -TODO - - type: keyword -- @@ -33710,8 +33535,6 @@ type: keyword *`logstash.node.stats.pipelines.queue.events_count`*:: + -- -TODO - type: long -- @@ -33719,8 +33542,6 @@ type: long *`logstash.node.stats.pipelines.queue.type`*:: + -- -TODO - type: keyword -- @@ -33728,8 +33549,6 @@ type: keyword *`logstash.node.stats.pipelines.queue.queue_size_in_bytes`*:: + -- -TODO - type: long -- @@ -33737,8 +33556,6 @@ type: long *`logstash.node.stats.pipelines.queue.max_queue_size_in_bytes`*:: + -- -TODO - type: long -- @@ -33747,8 +33564,6 @@ type: long *`logstash.node.stats.pipelines.events.out`*:: + -- -TODO - type: long -- @@ -33756,8 +33571,6 @@ type: long *`logstash.node.stats.pipelines.events.duration_in_millis`*:: + -- -TODO - type: long -- @@ -33766,8 +33579,6 @@ type: long *`logstash.node.stats.pipelines.vertices.duration_in_millis`*:: + -- -TODO - type: long -- @@ -33775,8 +33586,6 @@ type: long *`logstash.node.stats.pipelines.vertices.events_in`*:: + -- -TODO - type: long -- @@ -33823,8 +33632,6 @@ type: float *`logstash.node.stats.os.cpu.load_average.15m`*:: + -- -TODO - type: long -- @@ -33832,8 +33639,6 @@ type: long *`logstash.node.stats.os.cpu.load_average.1m`*:: + -- -TODO - type: long -- @@ -33841,8 +33646,6 @@ type: long *`logstash.node.stats.os.cpu.load_average.5m`*:: + -- -TODO - type: long -- @@ -33851,8 +33654,6 @@ type: long *`logstash.node.stats.os.cgroup.cpuacct.usage_nanos`*:: + -- -TODO - type: long -- @@ -33862,8 +33663,6 @@ type: long *`logstash.node.stats.os.cgroup.cpu.stat.number_of_elapsed_periods`*:: + -- -TODO - type: long -- @@ -33871,8 +33670,6 @@ type: long *`logstash.node.stats.os.cgroup.cpu.stat.time_throttled_nanos`*:: + -- -TODO - type: long -- @@ -33880,8 +33677,6 @@ type: long *`logstash.node.stats.os.cgroup.cpu.stat.number_of_times_throttled`*:: + -- -TODO - type: long -- @@ -33889,8 +33684,6 @@ type: long *`logstash.node.stats.queue.events_count`*:: + -- -TODO - type: long -- diff --git a/metricbeat/module/logstash/_meta/fields.yml b/metricbeat/module/logstash/_meta/fields.yml index 79307d9a859..2b39900e3b3 100644 --- a/metricbeat/module/logstash/_meta/fields.yml +++ b/metricbeat/module/logstash/_meta/fields.yml @@ -4,6 +4,7 @@ Logstash module release: ga settings: ["ssl", "http"] + short_config: false fields: - name: logstash_stats type: group @@ -104,6 +105,18 @@ - name: cpu type: group fields: + - name: stat + type: group + fields: + - name: number_of_elapsed_periods + type: alias + path: logstash.node.stats.os.cgroup.cpu.stat.number_of_elapsed_periods + - name: time_throttled_nanos + type: alias + path: logstash.node.stats.os.cgroup.cpu.stat.time_throttled_nanos + - name: number_of_times_throttled + type: alias + path: logstash.node.stats.os.cgroup.cpu.stat.number_of_times_throttled - name: load_average type: group fields: @@ -122,117 +135,21 @@ - name: cpuacct.usage_nanos type: alias path: logstash.node.stats.os.cgroup.cpuacct.usage_nanos - - name: cpu - type: group - fields: - - name: stat - type: group - fields: - - name: number_of_elapsed_periods - type: alias - path: logstash.node.stats.os.cgroup.stat.number_of_elapsed_periods - - name: time_throttled_nanos - type: alias - path: logstash.node.stats.os.cgroup.stat.time_throttled_nanos - - name: number_of_times_throttled - type: alias - path: logstash.node.stats.os.cgroup.stat.number_of_times_throttled - name: process.cpu.percent type: alias path: logstash.node.stats.process.cpu.percent - name: queue.events_count type: alias path: logstash.node.stats.queue.events_count + - name: logstash_state + type: group + fields: + - name: pipeline.id + type: alias + path: logstash.node.state.pipeline.id + - name: pipeline.hash + type: alias + path: logstash.node.state.pipeline.hash - name: logstash type: group - description: > - TODO fields: - - name: stats - type: group - fields: - - name: timestamp - type: alias - path: "@timestamp" - - name: jvm - type: group - fields: - - name: mem - type: group - fields: - - name: heap_used.bytes - type: alias - path: logstash.node.stats.jvm.mem.heap_used_in_bytes - - name: heap_max.bytes - type: alias - path: logstash.node.stats.jvm.mem.heap_max_in_bytes - - name: uptime.ms - type: alias - path: logstash.node.stats.jvm.uptime_in_millis - - name: events - type: group - fields: - - name: in - type: alias - path: logstash.node.stats.events.in - - name: out - type: alias - path: logstash.node.stats.events.out - - name: duration.ms - type: alias - path: logstash.node.stats.events.duration_in_millis - - name: logstash - type: group - fields: - - name: uuid - type: alias - path: logstash.node.stats.logstash.uuid - - name: version - type: alias - path: logstash.node.stats.logstash.version - - name: os - type: group - fields: - - name: cpu - type: group - fields: - - name: load_average - type: group - fields: - - name: 15m - type: alias - path: logstash.node.stats.os.cpu.load_average.15m - - name: 1m - type: alias - path: logstash.node.stats.os.cpu.load_average.1m - - name: 5m - type: alias - path: logstash.node.stats.os.cpu.load_average.5m - - name: cgroup - type: group - fields: - - name: cpuacct.usage_nanos - type: alias - path: logstash.node.stats.os.cgroup.cpuacct.usage_nanos - - name: cpu - type: group - fields: - - name: stat - type: group - fields: - - name: number_of_elapsed_periods - type: alias - path: logstash.node.stats.os.cgroup.stat.number_of_elapsed_periods - - name: time_throttled_nanos - type: alias - path: logstash.node.stats.os.cgroup.stat.time_throttled_nanos - - name: number_of_times_throttled - type: alias - path: logstash.node.stats.os.cgroup.stat.number_of_times_throttled - - name: process.cpu.percent - type: alias - path: logstash.node.stats.process.cpu.percent - - name: queue.events_count - type: alias - path: logstash.node.stats.queue.events_count - \ No newline at end of file diff --git a/metricbeat/module/logstash/fields.go b/metricbeat/module/logstash/fields.go index d1628941a22..923fa1b54fd 100644 --- a/metricbeat/module/logstash/fields.go +++ b/metricbeat/module/logstash/fields.go @@ -32,5 +32,5 @@ func init() { // AssetLogstash returns asset data. // This is the base64 encoded gzipped contents of module/logstash. func AssetLogstash() string { - return "eJzsWk+PpLYTvfenKPX5t5Z+h1z6EOWwiTJRoskh2ksUIQ/UdHsXMMGmM5NPH9HdMID/AjY9h+Gw0jJNvVdlu8x7+BN8w9cD5PwoJBWnHYBkMscD7H+93drvADIUac0qyXh5gO93AADdn6HgWZPjDqDGHKnAAxzpDkCglKw8igP8uRci3/8P9icpq/1fO4BnhnkmDpc4n6CkBb4xSISkUlz+BCBfqzZezZvqdmf46PBxyQoUkhZV/5fuaZozKgZ3KypPB9j/0D+xV4J9PRdKmCEJHZHh8wUWo/umGKY4w1gnpFXSCMwSViZPrxKF8lNTpuOMuwKTkmdILlUmX88FKbAgHiAjPgV9iU7HiNExaap2CNvfFCzP2RTKTMJOwBi2w8UzllKsmiCsDED2yoNMYnUQvJHhMKbBOpCsqWnbFIIOwg3TEnvaM1aNRdOwLADr/p4SrwM6Yy0YDzHy/b1pyA6qYhXmrER1lpYoJGa+0zREYXouxFCX03gE1+MoATukvxtsMFhnbp8M2Pze+F9oEm34USaJYP9ijD48peKD1u999CXZlp0PorF3w6opoDbZMIkZ2i7Mab1BmTjQBi1OslQZbUPr8anwRml2xF2JgjKXEmUvD8zIDDPt9glWJyywpnmidO7ApPwgJ5WKtVqmpbItm9iFsVTh2qOqRpySrWe1H3Tf1da93KZVE6zB5pxmCT1jTY/mvVYX2BZ8CPD/76ZSbRzbVH6wDgEXJK0aMmRPTEg9k82I2HlsVpAJUD99dMO5fAalVUPTVJJG0CMmJS15yLXW5nVhRVw4MVZHyyHKqiib4gnrhD8nmNNKYJZUWDOemQZ+7dS4lrD9P5kHPbR/EnmquZQ5ZsZhDszVG1Wt68V9ent2y7qaoPvXiZqnKK4LtsI6xVIq+4HeVNPuRpZoo52x271T3qwCNAQzeAbqWtF4ne31x+Pnx9t/TB7k0LnUx9Y9DQ4X0z4PLG4mGB1NMzkTQbC6m/Z4tpigczqJSTHaS9FdQexOhVhBX7bhZfVWx94nKUK7r1ZPNZKIDiqg9KYoRNPsM5R62NGaJ8w1TimsGjSNa7omG7N7Ck4HNQjuLXT3hLaIyua+6q00Sg916CU3iAsIvLQTeDVECKahwEdHRSPl5rR5oTSAVn0FQWafn9byyzmE5oINVpxBg7mDuwBghR6DQNNqsS6DZdosAu9Z6Et1WgTefhT8NJud3jLtBr76bSm4M+hgl+7HjWfoFGAGgddek+fHJ1q6yybgTlzMzb19hLT/tBEmvyjY8fp6dwBZK58uLYm0189cSFCCrvsaLbA+sxSVD88B6PYHir5oYi/RsQ68X778Bg/lM5/55uZ6B/2Gr//wWtcoHHzaS5c6jL70LH3p7payPoJ94DzJ/36FgIfP2nVJ/OwRx+q8Hg+DAmXNUkFWLdaQpojjMNJb8JyXx2X1HThQKn50U8ZulTiy88zQmKXCyHEALSqhhTaIA+7HSzTFQ4R1Bsry2fZQprxg5fGWJlx2YKzJIl9lOY3HRh75HBrPLJdYa9/U1nL56Rbam8uMz7w+rIwzcVt3Z9UOZ21iUfdWY/U+bJ0xmI+tY2mr4N9awdbvYZ6nszGj+9doKz/H387ZYPPXLzB3tjBj/lscHT8gHzBY6e6A7zSDeVMNXAsAljs774LzEmtnU+JzLZ2MN0/5TK1tBtUcHId1X/fu8SKhOVK+BazufDkE6cRWG2wI4tuDnZ3WcLodPMo4H8zvwHiELP1PqwcCt4pHCDJX9EIsEH+YK28C4TpOmEOQym2ZEHieJI8A53teHJasdf/gXqfD56fvCGfdlxZl7Ehv7rHvNw7POTe+iY4Y+EN8aF49WGjNew9Vew/deldlGkGafgjQ+AL0/cnLdyoejSrN83u7Bk+N/18AAAD//29X95w=" + return "eJzsWk2Po0YQvc+vKPmcRcohFx+iHDZRJko0OUR7iSLUA2XTu0B3+sPZya+PwAYD/YlpWzkMhzmA+71X1UXRVTUf4Au+7aFmR6mIrJ4AFFU17mH36+XW7gmgRFkIyhVl7R6+fwIAGB5Dw0pd4xOAwBqJxD0cyROARKVoe5R7+HMnZb37BnaVUnz3V/esYkLlBWsP9LiHA6llt/5AsS7lvkf/AC1p8Korl4oo2T8CUG+8YxFM88ud6dLpckUblIo0fHwyrCY1JXJylxNV7WH3w7hiZ4B9PjUGzFSETch0fYPN7L4Lw4UzxaqQ8FxLLHPa5q9vCqXxU5elc4sHB2ctKzHrvZx9PjVZg00WQTLT05Cvd5fj5BiUaN5tYfebhtY1XVK5RfgFOGEHXjxhq+SmAKFtArFnHdkCa6BgWqXjWIINJKUWpEsVSTfhwunBXuaMTXuhNS0TqB7vGXgD0QmFpCzFzo/3lpADFacca9qiGaUtSoVlbJimcMyoJXP4pZrv4HYeA3Bg+lujxmSZuVuZMPld9fcyMyv8zJJc0n/xHnl4KSWGbfz2ka/5Y9XFMDpzN2wKATPJpjHMkXZhTepNqiTANklxihbGbjtST4yHH2TmIDxkKBixlBvf8sSK3DTLbJ8jr7BBQercyNyJRcVRLjx1r7dl6Srfa3Nvx3i8cM5RXMsqf3RUx1GPWW3b4bbgOlmC7SxyusYG6AOdAre6eUWRs0OONeFd5cNRUFbaXA3BzQDvhjCZFb3WrOC6v5eto58WubmqBFOqxjJvScsepDea2fRvX2df1z7avyH6ay1BypycUJCj+1S3JeK+/W7ZFEhlO9fZVH3mYhqVPEyIX8fDHLIgGhOVbTtvz1UF16QoVKYlOaLzFbk1q8+C3MkzHgkEK1CeXcFRFNiqyMaY9YviQZt93YYvcMH0JkIHmLVXOLyt8b3C4QM5/1iv04iZDcVgqGzdiRs4RhxH6yPe+A49+Hm3dIGHa7F+3goeLt/xYG5ZsrfPc7D7gm//MGF77rFzuP54+fjiJLW0Lu5FO1Iyubax1y3Juj8dwuIXDT2ez4N7UMJojgSE/sykAgN0W79LojjRAo3WVgK54yDjkwXb3vsHb0wG+H759Bs8twe2MpLtlkOauLKZDrPMdWuBNHwk7Aj+jYsU//uZAp4/WtNaNh0dwa3J7TyAggaVoIXMNuW6NcEUCovAuOMKXrP26EQxR1N+VT5lsHZMFdBo4AXGTB64G/t9gRj8sUeDZZj5fOSc+AR9EfE+PLcFa2h7vJgJ/UkJReZU4evA3C7jRasjWyPjQGuFwloLbtXy0wU6WsuKVowzyixzKNj2rpszKQjm/y3fD2vrZ5sNZhvIj+fDhPhCPUwSIoKooh3CuQxiau6VOLfq8da8kGRj+ro0VP5GmOsPoLDWkF4I9xfjiWLIYEPfca7EGyLnKyJHTi9reWMTvqL3+L/RvLYJ+XDhce2aq6SS6dd6ZdXjJrX8kwA8ugbfVEn75vp+S3zWgDm5WvazTJJgNnP8t0DYSMPQ4GB9har4aX0A1HvghiR7YT+8rjA26qgXaadj0p3G0hRCIXJSvQImds4MUTG9SFzx4FFT5QizFgICcN4cd5PFAfPWjouvGg41c55qZgriKd7rAzvZe31wvqLrg/cCIUB45wLh/sf5Ox2+o+aMi/X/BQAA///nVv4e" } diff --git a/metricbeat/module/logstash/node/_meta/data.json b/metricbeat/module/logstash/node/_meta/data.json index a9e3bbb153a..ae1ab625ce3 100644 --- a/metricbeat/module/logstash/node/_meta/data.json +++ b/metricbeat/module/logstash/node/_meta/data.json @@ -1,33 +1,134 @@ { - "@timestamp": "2017-10-12T08:05:34.853Z", + "@timestamp": "2020-10-05T10:50:11.757Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0", + "_id": "afb1a50a-95f0-484a-b7d7-e683ddddc75a" + }, + "host": { + "name": "mcastro" + }, "agent": { - "hostname": "host.example.com", - "name": "host.example.com" + "ephemeral_id": "c4b22628-7b30-4a5d-8e28-7b6de81c9974", + "id": "803dfdba-e638-4590-a2de-80cb1cebe78d", + "name": "mcastro", + "type": "metricbeat", + "version": "8.0.0" }, "event": { + "duration": 9740086, "dataset": "logstash.node", - "duration": 115000, "module": "logstash" }, + "metricset": { + "name": "node", + "period": 10000 + }, + "service": { + "address": "localhost:9600", + "type": "logstash" + }, "logstash": { "node": { + "host": "2cb47f6e0eab", + "version": "8.0.0", "jvm": { - "version": "1.8.0_191" + "version": "11.0.5" + }, + "id": "4cc683ce-3ddc-46e3-bea3-aefbf37bc082", + "state": { + "pipeline": { + "hash": "3000c3abf87d4dfa4a57aaf6af0a1f5bee2e0fc1c48a8e8636e2a33d7d2e91dd", + "ephemeral_id": "afb1a50a-95f0-484a-b7d7-e683ddddc75a", + "representation": { + "graph": { + "edges": [ + { + "from": "1bf3a9cc73ceb7c3a9cbe885df249b23f3496c52a342a6d513153cc865d78182", + "id": "b3db599ec6ae0b9493158bd7024dcd922c8a3e76295c37fef0da440086bf3f8c", + "to": "__QUEUE__", + "type": "plain" + }, + { + "type": "plain", + "from": "71b91bc85b66ab25c5fb16e63db4dd7111c183f96d1f18e19078051ed5fc74f7", + "id": "9db20a77b3e1eb91229a50bd33388425d59725f9093e076a37e6565f8d5a20ad", + "to": "__QUEUE__" + }, + { + "id": "9b2bc571e978746fb9b55b83521a6603c3c940144cde0e3f4296298cea6585cf", + "to": "a339cb309b29181703c6adf321da3d639f5b60713de5a1e5519ebfea069556d8", + "type": "plain", + "from": "__QUEUE__" + } + ], + "vertices": [ + { + "config_name": "beats", + "explicit_id": false, + "id": "1bf3a9cc73ceb7c3a9cbe885df249b23f3496c52a342a6d513153cc865d78182", + "meta": { + "source": { + "line": 2, + "protocol": "file", + "column": 3, + "id": "/usr/share/logstash/pipeline/default.conf" + } + }, + "plugin_type": "input", + "type": "plugin" + }, + { + "plugin_type": "input", + "type": "plugin", + "config_name": "beats", + "explicit_id": false, + "id": "71b91bc85b66ab25c5fb16e63db4dd7111c183f96d1f18e19078051ed5fc74f7", + "meta": { + "source": { + "protocol": "file", + "column": 3, + "id": "/usr/share/logstash/pipeline/default.conf", + "line": 7 + } + } + }, + { + "explicit_id": false, + "id": "__QUEUE__", + "meta": null, + "type": "queue" + }, + { + "config_name": "elasticsearch", + "explicit_id": false, + "id": "a339cb309b29181703c6adf321da3d639f5b60713de5a1e5519ebfea069556d8", + "meta": { + "source": { + "id": "/usr/share/logstash/pipeline/default.conf", + "line": 17, + "protocol": "file", + "column": 3 + } + }, + "plugin_type": "output", + "type": "plugin" + } + ] + }, + "type": "lir", + "version": "0.0.0", + "hash": "3000c3abf87d4dfa4a57aaf6af0a1f5bee2e0fc1c48a8e8636e2a33d7d2e91dd" + }, + "batch_size": 125, + "workers": 12, + "id": "main" + } } } }, - "metricset": { - "name": "node" - }, - "process": { - "pid": 93559 - }, - "service": { - "address": "127.0.0.1:9600", - "hostname": "Shaunaks-MBP-2.attlocal.net", - "id": "7565df20-c3aa-4261-81d5-3b0ab8d15c16", - "name": "logstash", - "type": "logstash", - "version": "7.0.0" + "ecs": { + "version": "1.5.0" } -} \ No newline at end of file +} diff --git a/metricbeat/module/logstash/node/_meta/fields.yml b/metricbeat/module/logstash/node/_meta/fields.yml index 658825edb0b..76e61b2ebdd 100644 --- a/metricbeat/module/logstash/node/_meta/fields.yml +++ b/metricbeat/module/logstash/node/_meta/fields.yml @@ -4,6 +4,17 @@ node release: ga fields: + - name: state.pipeline + type: group + fields: + - name: id + type: keyword + description: > + TODO + - name: hash + type: keyword + description: > + TODO - name: host type: alias path: host.hostname diff --git a/metricbeat/module/logstash/node/data.go b/metricbeat/module/logstash/node/data.go index b1a6ef97ae4..b781e0a643b 100644 --- a/metricbeat/module/logstash/node/data.go +++ b/metricbeat/module/logstash/node/data.go @@ -42,22 +42,10 @@ var ( } ) -func eventMapping(r mb.ReporterV2, content []byte) error { - event := mb.Event{} +func commonFieldsMapping(event mb.Event, fields common.MapStr) error { event.RootFields = common.MapStr{} event.RootFields.Put("service.name", logstash.ModuleName) - var data map[string]interface{} - err := json.Unmarshal(content, &data) - if err != nil { - return errors.Wrap(err, "failure parsing Logstash Node API response") - } - - fields, err := schema.Apply(data) - if err != nil { - return errors.Wrap(err, "failure applying node schema") - } - // Set service ID serviceID, err := fields.GetValue("id") if err != nil { @@ -90,8 +78,113 @@ func eventMapping(r mb.ReporterV2, content []byte) error { event.RootFields.Put("process.pid", pid) fields.Delete("jvm.pid") - event.MetricSetFields = fields + return nil +} + +func eventMapping(r mb.ReporterV2, content []byte, pipelines []logstash.PipelineState, overrideClusterUUID string) error { + var data map[string]interface{} + err := json.Unmarshal(content, &data) + if err != nil { + return errors.Wrap(err, "failure parsing Logstash Node API response") + } + + fields, err := schema.Apply(data) + if err != nil { + return errors.Wrap(err, "failure applying node schema") + } + + pipelines = getUserDefinedPipelines(pipelines) + clusterToPipelinesMap := makeClusterToPipelinesMap(pipelines, overrideClusterUUID) + + for clusterUUID, pipelines := range clusterToPipelinesMap { + for _, pipeline := range pipelines { + removeClusterUUIDsFromPipeline(pipeline) + + // Rename key: graph -> representation + pipeline.Representation = pipeline.Graph + pipeline.Graph = nil + + logstashState := map[string]logstash.PipelineState{ + "pipeline": pipeline, + } + + event := mb.Event{ + MetricSetFields: common.MapStr{ + "state": logstashState, + }, + } + event.MetricSetFields.Update(fields) + + if err = commonFieldsMapping(event, fields); err != nil { + return err + } + + if clusterUUID != "" { + event.ModuleFields.Put("cluster.id", clusterUUID) + } + + event.ID = pipeline.EphemeralID + + r.Event(event) + } + } - r.Event(event) return nil } + +func makeClusterToPipelinesMap(pipelines []logstash.PipelineState, overrideClusterUUID string) map[string][]logstash.PipelineState { + var clusterToPipelinesMap map[string][]logstash.PipelineState + clusterToPipelinesMap = make(map[string][]logstash.PipelineState) + + if overrideClusterUUID != "" { + clusterToPipelinesMap[overrideClusterUUID] = pipelines + return clusterToPipelinesMap + } + + for _, pipeline := range pipelines { + clusterUUIDs := common.StringSet{} + for _, vertex := range pipeline.Graph.Graph.Vertices { + clusterUUID := logstash.GetVertexClusterUUID(vertex, overrideClusterUUID) + if clusterUUID != "" { + clusterUUIDs.Add(clusterUUID) + } + } + + // If no cluster UUID was found in this pipeline, assign it a blank one + if len(clusterUUIDs) == 0 { + clusterUUIDs.Add("") + } + + for clusterUUID := range clusterUUIDs { + clusterPipelines := clusterToPipelinesMap[clusterUUID] + if clusterPipelines == nil { + clusterToPipelinesMap[clusterUUID] = []logstash.PipelineState{} + } + + clusterToPipelinesMap[clusterUUID] = append(clusterPipelines, pipeline) + } + } + + return clusterToPipelinesMap +} + +func getUserDefinedPipelines(pipelines []logstash.PipelineState) []logstash.PipelineState { + userDefinedPipelines := []logstash.PipelineState{} + for _, pipeline := range pipelines { + if pipeline.ID[0] != '.' { + userDefinedPipelines = append(userDefinedPipelines, pipeline) + } + } + return userDefinedPipelines +} + +func removeClusterUUIDsFromPipeline(pipeline logstash.PipelineState) { + for _, vertex := range pipeline.Graph.Graph.Vertices { + _, exists := vertex["cluster_uuid"] + if !exists { + continue + } + + delete(vertex, "cluster_uuid") + } +} diff --git a/metricbeat/module/logstash/node/data_test.go b/metricbeat/module/logstash/node/data_test.go index 65539432d8a..3de3848272a 100644 --- a/metricbeat/module/logstash/node/data_test.go +++ b/metricbeat/module/logstash/node/data_test.go @@ -27,6 +27,7 @@ import ( "github.com/stretchr/testify/require" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/beats/v7/metricbeat/module/logstash" ) func TestEventMapping(t *testing.T) { @@ -39,10 +40,310 @@ func TestEventMapping(t *testing.T) { require.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} - err = eventMapping(reporter, input) + err = commonFieldsMapping(reporter, input) require.NoError(t, err, f) require.True(t, len(reporter.GetEvents()) >= 1, f) require.Equal(t, 0, len(reporter.GetErrors()), f) } } + +func TestMakeClusterToPipelinesMap(t *testing.T) { + tests := map[string]struct { + pipelines []logstash.PipelineState + overrideClusterUUID string + expectedMap map[string][]logstash.PipelineState + }{ + "no_vertex_cluster_id": { + pipelines: []logstash.PipelineState{ + { + ID: "test_pipeline", + Graph: &logstash.GraphContainer{ + Graph: &logstash.Graph{ + Vertices: []map[string]interface{}{ + { + "id": "vertex_1", + }, + { + "id": "vertex_2", + }, + { + "id": "vertex_3", + }, + }, + }, + }, + }, + }, + overrideClusterUUID: "prod_cluster_id", + expectedMap: map[string][]logstash.PipelineState{ + "prod_cluster_id": { + { + ID: "test_pipeline", + Graph: &logstash.GraphContainer{ + Graph: &logstash.Graph{ + Vertices: []map[string]interface{}{ + { + "id": "vertex_1", + }, + { + "id": "vertex_2", + }, + { + "id": "vertex_3", + }, + }, + }, + }, + }, + }, + }, + }, + "one_vertex_cluster_id": { + pipelines: []logstash.PipelineState{ + { + ID: "test_pipeline", + Graph: &logstash.GraphContainer{ + Graph: &logstash.Graph{ + Vertices: []map[string]interface{}{ + { + "id": "vertex_1", + "cluster_uuid": "es_1", + }, + { + "id": "vertex_2", + }, + { + "id": "vertex_3", + }, + }, + }, + }, + }, + }, + overrideClusterUUID: "prod_cluster_id", + expectedMap: map[string][]logstash.PipelineState{ + "prod_cluster_id": { + { + ID: "test_pipeline", + Graph: &logstash.GraphContainer{ + Graph: &logstash.Graph{ + Vertices: []map[string]interface{}{ + { + "id": "vertex_1", + "cluster_uuid": "es_1", + }, + { + "id": "vertex_2", + }, + { + "id": "vertex_3", + }, + }, + }, + }, + }, + }, + }, + }, + "two_pipelines": { + pipelines: []logstash.PipelineState{ + { + ID: "test_pipeline_1", + Graph: &logstash.GraphContainer{ + Graph: &logstash.Graph{ + Vertices: []map[string]interface{}{ + { + "id": "vertex_1_1", + "cluster_uuid": "es_1", + }, + { + "id": "vertex_1_2", + }, + { + "id": "vertex_1_3", + }, + }, + }, + }, + }, + { + ID: "test_pipeline_2", + Graph: &logstash.GraphContainer{ + Graph: &logstash.Graph{ + Vertices: []map[string]interface{}{ + { + "id": "vertex_2_1", + }, + { + "id": "vertex_2_2", + }, + { + "id": "vertex_2_3", + }, + }, + }, + }, + }, + }, + overrideClusterUUID: "prod_cluster_id", + expectedMap: map[string][]logstash.PipelineState{ + "prod_cluster_id": { + { + ID: "test_pipeline_1", + Graph: &logstash.GraphContainer{ + Graph: &logstash.Graph{ + Vertices: []map[string]interface{}{ + { + "id": "vertex_1_1", + "cluster_uuid": "es_1", + }, + { + "id": "vertex_1_2", + }, + { + "id": "vertex_1_3", + }, + }, + }, + }, + }, + { + ID: "test_pipeline_2", + Graph: &logstash.GraphContainer{ + Graph: &logstash.Graph{ + Vertices: []map[string]interface{}{ + { + "id": "vertex_2_1", + }, + { + "id": "vertex_2_2", + }, + { + "id": "vertex_2_3", + }, + }, + }, + }, + }, + }, + }, + }, + "no_override_cluster_id": { + pipelines: []logstash.PipelineState{ + { + ID: "test_pipeline_1", + Graph: &logstash.GraphContainer{ + Graph: &logstash.Graph{ + Vertices: []map[string]interface{}{ + { + "id": "vertex_1_1", + "cluster_uuid": "es_1", + }, + { + "id": "vertex_1_2", + "cluster_uuid": "es_2", + }, + { + "id": "vertex_1_3", + }, + }, + }, + }, + }, + { + ID: "test_pipeline_2", + Graph: &logstash.GraphContainer{ + Graph: &logstash.Graph{ + Vertices: []map[string]interface{}{ + { + "id": "vertex_2_1", + }, + { + "id": "vertex_2_2", + }, + { + "id": "vertex_2_3", + }, + }, + }, + }, + }, + }, + overrideClusterUUID: "", + expectedMap: map[string][]logstash.PipelineState{ + "es_1": { + { + ID: "test_pipeline_1", + Graph: &logstash.GraphContainer{ + Graph: &logstash.Graph{ + Vertices: []map[string]interface{}{ + { + "id": "vertex_1_1", + "cluster_uuid": "es_1", + }, + { + "id": "vertex_1_2", + "cluster_uuid": "es_2", + }, + { + "id": "vertex_1_3", + }, + }, + }, + }, + }, + }, + "es_2": { + { + ID: "test_pipeline_1", + Graph: &logstash.GraphContainer{ + Graph: &logstash.Graph{ + Vertices: []map[string]interface{}{ + { + "id": "vertex_1_1", + "cluster_uuid": "es_1", + }, + { + "id": "vertex_1_2", + "cluster_uuid": "es_2", + }, + { + "id": "vertex_1_3", + }, + }, + }, + }, + }, + }, + "": { + { + ID: "test_pipeline_2", + Graph: &logstash.GraphContainer{ + Graph: &logstash.Graph{ + Vertices: []map[string]interface{}{ + { + "id": "vertex_2_1", + }, + { + "id": "vertex_2_2", + }, + { + "id": "vertex_2_3", + }, + }, + }, + }, + }, + }, + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + actualMap := makeClusterToPipelinesMap(test.pipelines, test.overrideClusterUUID) + require.Equal(t, test.expectedMap, actualMap) + }) + } +} diff --git a/metricbeat/module/logstash/node/data_xpack.go b/metricbeat/module/logstash/node/data_xpack.go deleted file mode 100644 index 66d3623c7de..00000000000 --- a/metricbeat/module/logstash/node/data_xpack.go +++ /dev/null @@ -1,120 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package node - -import ( - "time" - - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/metricbeat/helper/elastic" - "github.com/elastic/beats/v7/metricbeat/mb" - "github.com/elastic/beats/v7/metricbeat/module/logstash" -) - -func eventMappingXPack(r mb.ReporterV2, m *MetricSet, pipelines []logstash.PipelineState, overrideClusterUUID string) error { - pipelines = getUserDefinedPipelines(pipelines) - clusterToPipelinesMap := makeClusterToPipelinesMap(pipelines, overrideClusterUUID) - for clusterUUID, pipelines := range clusterToPipelinesMap { - for _, pipeline := range pipelines { - removeClusterUUIDsFromPipeline(pipeline) - - // Rename key: graph -> representation - pipeline.Representation = pipeline.Graph - pipeline.Graph = nil - - logstashState := map[string]logstash.PipelineState{ - "pipeline": pipeline, - } - - event := mb.Event{} - event.RootFields = common.MapStr{ - "timestamp": common.Time(time.Now()), - "interval_ms": m.Module().Config().Period / time.Millisecond, - "type": "logstash_state", - "logstash_state": logstashState, - } - - if clusterUUID != "" { - event.RootFields["cluster_uuid"] = clusterUUID - } - - event.ID = pipeline.EphemeralID - event.Index = elastic.MakeXPackMonitoringIndexName(elastic.Logstash) - r.Event(event) - } - } - - return nil -} - -func makeClusterToPipelinesMap(pipelines []logstash.PipelineState, overrideClusterUUID string) map[string][]logstash.PipelineState { - var clusterToPipelinesMap map[string][]logstash.PipelineState - clusterToPipelinesMap = make(map[string][]logstash.PipelineState) - - if overrideClusterUUID != "" { - clusterToPipelinesMap[overrideClusterUUID] = pipelines - return clusterToPipelinesMap - } - - for _, pipeline := range pipelines { - clusterUUIDs := common.StringSet{} - for _, vertex := range pipeline.Graph.Graph.Vertices { - clusterUUID := logstash.GetVertexClusterUUID(vertex, overrideClusterUUID) - if clusterUUID != "" { - clusterUUIDs.Add(clusterUUID) - } - } - - // If no cluster UUID was found in this pipeline, assign it a blank one - if len(clusterUUIDs) == 0 { - clusterUUIDs.Add("") - } - - for clusterUUID := range clusterUUIDs { - clusterPipelines := clusterToPipelinesMap[clusterUUID] - if clusterPipelines == nil { - clusterToPipelinesMap[clusterUUID] = []logstash.PipelineState{} - } - - clusterToPipelinesMap[clusterUUID] = append(clusterPipelines, pipeline) - } - } - - return clusterToPipelinesMap -} - -func getUserDefinedPipelines(pipelines []logstash.PipelineState) []logstash.PipelineState { - userDefinedPipelines := []logstash.PipelineState{} - for _, pipeline := range pipelines { - if pipeline.ID[0] != '.' { - userDefinedPipelines = append(userDefinedPipelines, pipeline) - } - } - return userDefinedPipelines -} - -func removeClusterUUIDsFromPipeline(pipeline logstash.PipelineState) { - for _, vertex := range pipeline.Graph.Graph.Vertices { - _, exists := vertex["cluster_uuid"] - if !exists { - continue - } - - delete(vertex, "cluster_uuid") - } -} diff --git a/metricbeat/module/logstash/node/data_xpack_test.go b/metricbeat/module/logstash/node/data_xpack_test.go deleted file mode 100644 index 17ae0aaaf91..00000000000 --- a/metricbeat/module/logstash/node/data_xpack_test.go +++ /dev/null @@ -1,328 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// +build !integration - -package node - -import ( - "testing" - - "github.com/stretchr/testify/require" - - "github.com/elastic/beats/v7/metricbeat/module/logstash" -) - -func TestMakeClusterToPipelinesMap(t *testing.T) { - tests := map[string]struct { - pipelines []logstash.PipelineState - overrideClusterUUID string - expectedMap map[string][]logstash.PipelineState - }{ - "no_vertex_cluster_id": { - pipelines: []logstash.PipelineState{ - { - ID: "test_pipeline", - Graph: &logstash.GraphContainer{ - Graph: &logstash.Graph{ - Vertices: []map[string]interface{}{ - { - "id": "vertex_1", - }, - { - "id": "vertex_2", - }, - { - "id": "vertex_3", - }, - }, - }, - }, - }, - }, - overrideClusterUUID: "prod_cluster_id", - expectedMap: map[string][]logstash.PipelineState{ - "prod_cluster_id": { - { - ID: "test_pipeline", - Graph: &logstash.GraphContainer{ - Graph: &logstash.Graph{ - Vertices: []map[string]interface{}{ - { - "id": "vertex_1", - }, - { - "id": "vertex_2", - }, - { - "id": "vertex_3", - }, - }, - }, - }, - }, - }, - }, - }, - "one_vertex_cluster_id": { - pipelines: []logstash.PipelineState{ - { - ID: "test_pipeline", - Graph: &logstash.GraphContainer{ - Graph: &logstash.Graph{ - Vertices: []map[string]interface{}{ - { - "id": "vertex_1", - "cluster_uuid": "es_1", - }, - { - "id": "vertex_2", - }, - { - "id": "vertex_3", - }, - }, - }, - }, - }, - }, - overrideClusterUUID: "prod_cluster_id", - expectedMap: map[string][]logstash.PipelineState{ - "prod_cluster_id": { - { - ID: "test_pipeline", - Graph: &logstash.GraphContainer{ - Graph: &logstash.Graph{ - Vertices: []map[string]interface{}{ - { - "id": "vertex_1", - "cluster_uuid": "es_1", - }, - { - "id": "vertex_2", - }, - { - "id": "vertex_3", - }, - }, - }, - }, - }, - }, - }, - }, - "two_pipelines": { - pipelines: []logstash.PipelineState{ - { - ID: "test_pipeline_1", - Graph: &logstash.GraphContainer{ - Graph: &logstash.Graph{ - Vertices: []map[string]interface{}{ - { - "id": "vertex_1_1", - "cluster_uuid": "es_1", - }, - { - "id": "vertex_1_2", - }, - { - "id": "vertex_1_3", - }, - }, - }, - }, - }, - { - ID: "test_pipeline_2", - Graph: &logstash.GraphContainer{ - Graph: &logstash.Graph{ - Vertices: []map[string]interface{}{ - { - "id": "vertex_2_1", - }, - { - "id": "vertex_2_2", - }, - { - "id": "vertex_2_3", - }, - }, - }, - }, - }, - }, - overrideClusterUUID: "prod_cluster_id", - expectedMap: map[string][]logstash.PipelineState{ - "prod_cluster_id": { - { - ID: "test_pipeline_1", - Graph: &logstash.GraphContainer{ - Graph: &logstash.Graph{ - Vertices: []map[string]interface{}{ - { - "id": "vertex_1_1", - "cluster_uuid": "es_1", - }, - { - "id": "vertex_1_2", - }, - { - "id": "vertex_1_3", - }, - }, - }, - }, - }, - { - ID: "test_pipeline_2", - Graph: &logstash.GraphContainer{ - Graph: &logstash.Graph{ - Vertices: []map[string]interface{}{ - { - "id": "vertex_2_1", - }, - { - "id": "vertex_2_2", - }, - { - "id": "vertex_2_3", - }, - }, - }, - }, - }, - }, - }, - }, - "no_override_cluster_id": { - pipelines: []logstash.PipelineState{ - { - ID: "test_pipeline_1", - Graph: &logstash.GraphContainer{ - Graph: &logstash.Graph{ - Vertices: []map[string]interface{}{ - { - "id": "vertex_1_1", - "cluster_uuid": "es_1", - }, - { - "id": "vertex_1_2", - "cluster_uuid": "es_2", - }, - { - "id": "vertex_1_3", - }, - }, - }, - }, - }, - { - ID: "test_pipeline_2", - Graph: &logstash.GraphContainer{ - Graph: &logstash.Graph{ - Vertices: []map[string]interface{}{ - { - "id": "vertex_2_1", - }, - { - "id": "vertex_2_2", - }, - { - "id": "vertex_2_3", - }, - }, - }, - }, - }, - }, - overrideClusterUUID: "", - expectedMap: map[string][]logstash.PipelineState{ - "es_1": { - { - ID: "test_pipeline_1", - Graph: &logstash.GraphContainer{ - Graph: &logstash.Graph{ - Vertices: []map[string]interface{}{ - { - "id": "vertex_1_1", - "cluster_uuid": "es_1", - }, - { - "id": "vertex_1_2", - "cluster_uuid": "es_2", - }, - { - "id": "vertex_1_3", - }, - }, - }, - }, - }, - }, - "es_2": { - { - ID: "test_pipeline_1", - Graph: &logstash.GraphContainer{ - Graph: &logstash.Graph{ - Vertices: []map[string]interface{}{ - { - "id": "vertex_1_1", - "cluster_uuid": "es_1", - }, - { - "id": "vertex_1_2", - "cluster_uuid": "es_2", - }, - { - "id": "vertex_1_3", - }, - }, - }, - }, - }, - }, - "": { - { - ID: "test_pipeline_2", - Graph: &logstash.GraphContainer{ - Graph: &logstash.Graph{ - Vertices: []map[string]interface{}{ - { - "id": "vertex_2_1", - }, - { - "id": "vertex_2_2", - }, - { - "id": "vertex_2_3", - }, - }, - }, - }, - }, - }, - }, - }, - } - - for name, test := range tests { - t.Run(name, func(t *testing.T) { - actualMap := makeClusterToPipelinesMap(test.pipelines, test.overrideClusterUUID) - require.Equal(t, test.expectedMap, actualMap) - }) - } -} diff --git a/metricbeat/module/logstash/node/node.go b/metricbeat/module/logstash/node/node.go index 025d013e756..ac608b981c0 100644 --- a/metricbeat/module/logstash/node/node.go +++ b/metricbeat/module/logstash/node/node.go @@ -65,32 +65,22 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // It returns the event which is then forward to the output. In case of an error, a // descriptive error must be returned. func (m *MetricSet) Fetch(r mb.ReporterV2) error { - if !m.MetricSet.XPack { - content, err := m.HTTP.FetchContent() - if err != nil { - return err - } - - return eventMapping(r, content) - } - - pipelinesContent, overrideClusterUUID, err := logstash.GetPipelines(m.MetricSet) + content, err := m.HTTP.FetchContent() if err != nil { - m.Logger().Error(err) - return nil + return err } + //commonFields, err := commonFieldsMapping(content) + //if err != nil { + // return err + //} - err = eventMappingXPack(r, m, pipelinesContent, overrideClusterUUID) + pipelinesContent, overrideClusterUUID, err := logstash.GetPipelines(m.MetricSet) if err != nil { - m.Logger().Error(err) + return err } - return nil -} - -func (m *MetricSet) init() error { - if m.XPack { - return m.CheckPipelineGraphAPIsAvailable() + if err = eventMapping(r, content, pipelinesContent, overrideClusterUUID); err != nil { + return err } return nil diff --git a/metricbeat/module/logstash/node_stats/_meta/data.json b/metricbeat/module/logstash/node_stats/_meta/data.json index 8d9623ef81b..4b24bae61b2 100644 --- a/metricbeat/module/logstash/node_stats/_meta/data.json +++ b/metricbeat/module/logstash/node_stats/_meta/data.json @@ -1,140 +1,130 @@ { - "@timestamp": "2020-10-30T12:41:12.793Z", - "@metadata": { - "beat": "metricbeat", - "type": "_doc", - "version": "8.0.0" - }, - "metricset": { - "name": "node_stats", - "period": 10000 + "@timestamp": "2017-10-12T08:05:34.853Z", + "event": { + "dataset": "logstash.node.stats", + "duration": 115000, + "module": "logstash" }, "logstash": { - "node": { - "stats": { - "events": { - "duration_in_millis": 0, - "in": 0, - "filtered": 0, - "out": 0 - }, - "reloads": { - "failures": 0, - "successes": 0 - }, - "queue": { - "events_count": 0 - }, - "os": { - "cpu": { - "percent": 0, - "load_average": { - "1m": 2.814453125 - } - }, - "cgroup": { - "cpuacct": null, - "cpu": { - "stat": null, - "control_group": "" - } - } - }, - "pipelines": [ - { - "id": "main", - "hash": "4dbe1b282a0265b7c90ae9212a3b4bef55168d267004e5124a3486f9151d4188", - "ephemeral_id": "5fd09014-09f9-449a-8b19-692c82f5494e", - "events": { - "filtered": 0, - "in": 0, - "out": 0, - "queue_push_duration_in_millis": 0, - "duration_in_millis": 0 - }, - "reloads": { - "successes": 0, - "failures": 0 - }, - "queue": { - "events_count": 0, - "max_queue_size_in_bytes": 0, - "queue_size_in_bytes": 0, - "type": "memory" - }, - "vertices": null - } - ], - "jvm": { - "gc": { - "collectors": { - "old": { - "collection_count": 1, - "collection_time_in_millis": 198 + "node": { + "stats": { + "events": { + "duration_in_millis": 0, + "in": 0, + "filtered": 0, + "out": 0 + }, + "jvm": { + "gc": { + "collectors": { + "old": { + "collection_count": 3, + "collection_time_in_millis": 449 + }, + "young": { + "collection_count": 7, + "collection_time_in_millis": 231 + } + } + }, + "mem": { + "heap_max_in_bytes": 1037959168, + "heap_used_in_bytes": 371211952, + "heap_used_percent": 35 + }, + "uptime_in_millis": 33546 + }, + "reloads": { + "failures": 0, + "successes": 0 + }, + "queue": { + "events_count": 0 }, - "young": { - "collection_time_in_millis": 298, - "collection_count": 10 - } - } - }, - "mem": { - "heap_max_in_bytes": 1.073741822e+09, - "heap_used_in_bytes": 3.18658432e+08, - "heap_used_percent": 29 - }, - "uptime_in_millis": 57904 - }, - "process": { - "open_file_descriptors": 93, - "max_file_descriptors": 10240, - "cpu": { - "percent": 0 + "process": { + "open_file_descriptors": 103, + "max_file_descriptors": 524288, + "cpu": { + "percent": 1 + } + }, + "os": { + "cpu": { + "percent": 0, + "load_average": { + "15m": 1.7, + "1m": 2.52, + "5m": 1.8 + } + }, + "cgroup": { + "cpuacct": { + "control_group": "/user.slice", + "usage_nanos": 24041075965919 + }, + "cpu": { + "stat": { + "number_of_elapsed_periods": 0, + "number_of_times_throttled": 0, + "time_throttled_nanos": 0 + }, + "control_group": "/user.slice" + } + } + }, + "pipelines": [ + { + "id": "main", + "hash": "6984380a58d40b7ebe6ba7ba8fc3134b95ffadfcef9ff328f0864ae002943792", + "ephemeral_id": "eb977feb-0052-4651-a995-39e545213723", + "events": { + "duration_in_millis": 0, + "filtered": 0, + "in": 0, + "out": 0, + "queue_push_duration_in_millis": 0 + }, + "reloads": { + "successes": 0, + "failures": 0 + }, + "queue": { + "events_count": 0, + "max_queue_size_in_bytes": 0, + "queue_size_in_bytes": 0, + "type": "memory" + }, + "vertices": null + } + ], + "logstash": { + "uuid": "2fc9524f-a36b-4611-82e4-5246d3ac4714", + "ephemeral_id": "10311adf-a5ba-4920-ada2-8dadc54618fe", + "name": "anonymous", + "host": "anonymous", + "version": "7.8.1", + "snapshot": false, + "status": "green", + "http_address": "127.0.0.1:9600", + "pipeline": { + "batch_size": 125, + "workers": 12 + } + }, + "timestamp": "2020-12-09T16:16:17.796Z" } - }, - "logstash": { - "uuid": "33f30cf2-d580-48fb-ad7c-2fbaf85db604", - "http_address": "127.0.0.1:9600", - "pipeline": { - "batch_size": 125, - "workers": 4 - }, - "status": "green", - "ephemeral_id": "c2f01c4f-5c09-4b65-a922-15fdf3634df3", - "name": "mcastro", - "host": "mcastro", - "version": "7.9.3", - "snapshot": false - }, - "timestamp": "2020-10-30T12:41:13.799Z" } - } - }, - "ecs": { - "version": "1.5.0" }, - "host": { - "name": "mcastro" - }, - "agent": { - "name": "mcastro", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "fb36ebe3-f38d-4e5f-b68e-8d6f578ed471", - "id": "abfbaa55-b79d-4ae5-b412-fd35b933c6d3" + "metricset": { + "name": "node_stats", + "period": 10000 }, "service": { - "name": "logstash", - "id": "", - "hostname": "mcastro", - "version": "7.9.3", - "address": "localhost:9600", - "type": "logstash" - }, - "event": { - "module": "logstash", - "duration": 1005737788, - "dataset": "logstash.node.stats" + "address": "127.0.0.1:33437", + "hostname": "anonymous", + "id": "", + "name": "logstash", + "type": "logstash", + "version": "7.8.1" } - } - \ No newline at end of file +} \ No newline at end of file diff --git a/metricbeat/module/logstash/node_stats/_meta/fields.yml b/metricbeat/module/logstash/node_stats/_meta/fields.yml index 94aa3262398..b951c8d9b6c 100644 --- a/metricbeat/module/logstash/node_stats/_meta/fields.yml +++ b/metricbeat/module/logstash/node_stats/_meta/fields.yml @@ -9,19 +9,13 @@ fields: - name: uptime_in_millis type: long - description: > - TODO - name: mem type: group fields: - name: heap_used_in_bytes type: long - description: > - TODO - name: heap_max_in_bytes type: long - description: > - TODO - name: events type: group description: > @@ -41,18 +35,13 @@ Filtered events counter. - name: duration_in_millis type: long - description: TODO - name: logstash type: group fields: - name: uuid type: keyword - description: > - TODO - name: version type: keyword - description: > - TODO - name: os type: group fields: @@ -64,23 +53,15 @@ fields: - name: 15m type: long - description: > - TODO - name: 1m type: long - description: > - TODO - name: 5m type: long - description: > - TODO - name: cgroup type: group fields: - name: cpuact.usage_nanos type: long - description: > - TODO - name: cpu type: group fields: @@ -108,45 +89,33 @@ fields: - name: id type: keyword - description: > - TODO - name: hash type: keyword - description: > - TODO - name: queue type: group fields: - name: events_count type: long - description: TODO - name: type type: keyword - description: TODO - name: queue_size_in_bytes type: long - description: TODO - name: max_queue_size_in_bytes type: long - description: TODO - name: events type: group fields: - name: out type: long - description: TODO - name: duration_in_millis type: long - description: TODO - name: vertices type: group fields: - name: duration_in_millis type: long - description: TODO - name: events_in type: long - description: TODO - name: pipeline_ephemeral_id type: keyword description: pipeline_ephemeral_id @@ -170,19 +139,15 @@ fields: - name: 15m type: long - description: TODO - name: 1m type: long - description: TODO - name: 5m type: long - description: TODO - name: cgroup type: group fields: - name: cpuacct.usage_nanos type: long - description: TODO - name: cpu type: group fields: @@ -191,13 +156,9 @@ fields: - name: number_of_elapsed_periods type: long - description: TODO - name: time_throttled_nanos type: long - description: TODO - name: number_of_times_throttled type: long - description: TODO - name: queue.events_count type: long - description: TODO diff --git a/metricbeat/module/logstash/node_stats/_meta/test/node_stats.710.json b/metricbeat/module/logstash/node_stats/_meta/test/node_stats.710.json new file mode 100644 index 00000000000..6b8afa8022b --- /dev/null +++ b/metricbeat/module/logstash/node_stats/_meta/test/node_stats.710.json @@ -0,0 +1,192 @@ +{ + "host": "anonymous", + "version": "7.8.1", + "http_address": "127.0.0.1:9600", + "id": "2fc9524f-a36b-4611-82e4-5246d3ac4714", + "name": "anonymous", + "ephemeral_id": "10311adf-a5ba-4920-ada2-8dadc54618fe", + "status": "green", + "snapshot": false, + "pipeline": { + "workers": 12, + "batch_size": 125, + "batch_delay": 50 + }, + "jvm": { + "threads": { + "count": 45, + "peak_count": 45 + }, + "mem": { + "heap_used_percent": 35, + "heap_committed_in_bytes": 1037959168, + "heap_max_in_bytes": 1037959168, + "heap_used_in_bytes": 371211952, + "non_heap_used_in_bytes": 148921496, + "non_heap_committed_in_bytes": 169435136, + "pools": { + "young": { + "committed_in_bytes": 286326784, + "max_in_bytes": 286326784, + "used_in_bytes": 236814520, + "peak_used_in_bytes": 286326784, + "peak_max_in_bytes": 286326784 + }, + "old": { + "committed_in_bytes": 715849728, + "max_in_bytes": 715849728, + "used_in_bytes": 102245112, + "peak_used_in_bytes": 107311296, + "peak_max_in_bytes": 715849728 + }, + "survivor": { + "committed_in_bytes": 35782656, + "max_in_bytes": 35782656, + "used_in_bytes": 32152320, + "peak_used_in_bytes": 35782656, + "peak_max_in_bytes": 35782656 + } + } + }, + "gc": { + "collectors": { + "young": { + "collection_time_in_millis": 231, + "collection_count": 7 + }, + "old": { + "collection_time_in_millis": 449, + "collection_count": 3 + } + } + }, + "uptime_in_millis": 33546 + }, + "process": { + "open_file_descriptors": 103, + "peak_open_file_descriptors": 103, + "max_file_descriptors": 524288, + "mem": { + "total_virtual_in_bytes": 7189766144 + }, + "cpu": { + "total_in_millis": 68000, + "percent": 1, + "load_average": { + "1m": 2.52, + "5m": 1.8, + "15m": 1.7 + } + } + }, + "events": { + "in": 0, + "filtered": 0, + "out": 0, + "duration_in_millis": 0, + "queue_push_duration_in_millis": 0 + }, + "pipelines": { + "main": { + "events": { + "filtered": 0, + "out": 0, + "queue_push_duration_in_millis": 0, + "in": 0, + "duration_in_millis": 0 + }, + "plugins": { + "inputs": [ + { + "id": "adc9aa83962ecb7f9e68a9b25d9ef5473a197dbb8f6cd7d6025742b2e2bcadcf", + "events": { + "out": 0, + "queue_push_duration_in_millis": 0 + }, + "name": "file" + } + ], + "codecs": [ + { + "id": "plain_6013fcda-72f3-4a64-8a6c-2cbab83a1cd0", + "decode": { + "out": 0, + "duration_in_millis": 0, + "writes_in": 0 + }, + "encode": { + "duration_in_millis": 0, + "writes_in": 0 + }, + "name": "plain" + }, + { + "id": "rubydebug_e8db9697-3c57-4b27-be5a-6ecbdf34defb", + "decode": { + "out": 0, + "duration_in_millis": 0, + "writes_in": 0 + }, + "encode": { + "duration_in_millis": 5, + "writes_in": 0 + }, + "name": "rubydebug" + } + ], + "filters": [], + "outputs": [ + { + "id": "488688673ecd34b96c74c843d265da79aa11629216d2cb76ca008548316e310d", + "events": { + "out": 0, + "in": 0, + "duration_in_millis": 17 + }, + "name": "stdout" + } + ] + }, + "reloads": { + "last_error": null, + "failures": 0, + "last_success_timestamp": null, + "last_failure_timestamp": null, + "successes": 0 + }, + "queue": { + "type": "memory", + "events_count": 0, + "queue_size_in_bytes": 0, + "max_queue_size_in_bytes": 0 + }, + "hash": "6984380a58d40b7ebe6ba7ba8fc3134b95ffadfcef9ff328f0864ae002943792", + "ephemeral_id": "eb977feb-0052-4651-a995-39e545213723" + } + }, + "reloads": { + "failures": 0, + "successes": 0 + }, + "os": { + "cgroup": { + "cpu": { + "cfs_quota_micros": -1, + "cfs_period_micros": 100000, + "control_group": "/user.slice", + "stat": { + "time_throttled_nanos": 0, + "number_of_times_throttled": 0, + "number_of_elapsed_periods": 0 + } + }, + "cpuacct": { + "usage_nanos": 24041075965919, + "control_group": "/user.slice" + } + } + }, + "queue": { + "events_count": 0 + } +} diff --git a/metricbeat/module/logstash/node_stats/data_test.go b/metricbeat/module/logstash/node_stats/data_test.go index 1a30c773f73..ff69a8f20ce 100644 --- a/metricbeat/module/logstash/node_stats/data_test.go +++ b/metricbeat/module/logstash/node_stats/data_test.go @@ -20,13 +20,14 @@ package node_stats import ( - "github.com/elastic/beats/v7/metricbeat/module/logstash" "io/ioutil" "net/http" "net/http/httptest" "path/filepath" "testing" + "github.com/elastic/beats/v7/metricbeat/module/logstash" + "github.com/stretchr/testify/require" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" @@ -63,7 +64,7 @@ func TestData(t *testing.T) { mux.Handle("/_node/stats", http.HandlerFunc( func(w http.ResponseWriter, r *http.Request) { - input, _ := ioutil.ReadFile("./_meta/test/node_stats.700.json") + input, _ := ioutil.ReadFile("./_meta/test/node_stats.710.json") w.Write(input) })) From a2a23d2bf26711bf9119bb7bd121daf89b077bb4 Mon Sep 17 00:00:00 2001 From: sayden Date: Thu, 10 Dec 2020 20:36:03 +0100 Subject: [PATCH 07/17] Fix broken test --- metricbeat/module/logstash/node/data.go | 4 ++-- metricbeat/module/logstash/node/data_test.go | 13 ++++++++----- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/metricbeat/module/logstash/node/data.go b/metricbeat/module/logstash/node/data.go index b781e0a643b..f5f0fd45dac 100644 --- a/metricbeat/module/logstash/node/data.go +++ b/metricbeat/module/logstash/node/data.go @@ -42,7 +42,7 @@ var ( } ) -func commonFieldsMapping(event mb.Event, fields common.MapStr) error { +func commonFieldsMapping(event *mb.Event, fields common.MapStr) error { event.RootFields = common.MapStr{} event.RootFields.Put("service.name", logstash.ModuleName) @@ -115,7 +115,7 @@ func eventMapping(r mb.ReporterV2, content []byte, pipelines []logstash.Pipeline } event.MetricSetFields.Update(fields) - if err = commonFieldsMapping(event, fields); err != nil { + if err = commonFieldsMapping(&event, fields); err != nil { return err } diff --git a/metricbeat/module/logstash/node/data_test.go b/metricbeat/module/logstash/node/data_test.go index 3de3848272a..9bc48d9f93b 100644 --- a/metricbeat/module/logstash/node/data_test.go +++ b/metricbeat/module/logstash/node/data_test.go @@ -20,13 +20,15 @@ package node import ( + "encoding/json" "io/ioutil" "path/filepath" "testing" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/stretchr/testify/require" - mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" "github.com/elastic/beats/v7/metricbeat/module/logstash" ) @@ -39,12 +41,13 @@ func TestEventMapping(t *testing.T) { input, err := ioutil.ReadFile(f) require.NoError(t, err) - reporter := &mbtest.CapturingReporterV2{} - err = commonFieldsMapping(reporter, input) + var data map[string]interface{} + err = json.Unmarshal(input, &data) + require.NoError(t, err) + event := mb.Event{} + err = commonFieldsMapping(&event, data) require.NoError(t, err, f) - require.True(t, len(reporter.GetEvents()) >= 1, f) - require.Equal(t, 0, len(reporter.GetErrors()), f) } } From 4b39731095586a46eb3f8a8b0dc625af0970a963 Mon Sep 17 00:00:00 2001 From: sayden Date: Mon, 14 Dec 2020 12:53:23 +0100 Subject: [PATCH 08/17] Fix broken test --- .../logstash/logstash_integration_test.go | 32 ++----------------- metricbeat/module/logstash/node/data.go | 1 + .../logstash/node_stats/node_stats_test.go | 7 ---- 3 files changed, 4 insertions(+), 36 deletions(-) diff --git a/metricbeat/module/logstash/logstash_integration_test.go b/metricbeat/module/logstash/logstash_integration_test.go index ffaed41a4e6..2f67eb33f76 100644 --- a/metricbeat/module/logstash/logstash_integration_test.go +++ b/metricbeat/module/logstash/logstash_integration_test.go @@ -70,31 +70,6 @@ func TestData(t *testing.T) { } } -func TestXPackEnabled(t *testing.T) { - lsService := compose.EnsureUpWithTimeout(t, 300, "logstash") - esService := compose.EnsureUpWithTimeout(t, 300, "elasticsearch") - - clusterUUID := getESClusterUUID(t, esService.Host()) - - metricSetToTypeMap := map[string]string{ - "node": "logstash_state", - "node_stats": "logstash_stats", - } - - config := getXPackConfig(lsService.Host()) - metricSets := mbtest.NewReportingMetricSetV2Errors(t, config) - for _, metricSet := range metricSets { - events, errs := mbtest.ReportingFetchV2Error(metricSet) - require.Empty(t, errs) - require.NotEmpty(t, events) - - event := events[0] - require.Equal(t, metricSetToTypeMap[metricSet.Name()], event.RootFields["type"]) - require.Equal(t, clusterUUID, event.RootFields["cluster_uuid"]) - require.Regexp(t, `^.monitoring-logstash-\d-mb`, event.Index) - } -} - func getConfig(metricSet string, host string) map[string]interface{} { return map[string]interface{}{ "module": logstash.ModuleName, @@ -105,10 +80,9 @@ func getConfig(metricSet string, host string) map[string]interface{} { func getXPackConfig(host string) map[string]interface{} { return map[string]interface{}{ - "module": logstash.ModuleName, - "metricsets": metricSets, - "hosts": []string{host}, - "xpack.enabled": true, + "module": logstash.ModuleName, + "metricsets": metricSets, + "hosts": []string{host}, } } diff --git a/metricbeat/module/logstash/node/data.go b/metricbeat/module/logstash/node/data.go index f5f0fd45dac..6117683623c 100644 --- a/metricbeat/module/logstash/node/data.go +++ b/metricbeat/module/logstash/node/data.go @@ -112,6 +112,7 @@ func eventMapping(r mb.ReporterV2, content []byte, pipelines []logstash.Pipeline MetricSetFields: common.MapStr{ "state": logstashState, }, + ModuleFields: common.MapStr{}, } event.MetricSetFields.Update(fields) diff --git a/metricbeat/module/logstash/node_stats/node_stats_test.go b/metricbeat/module/logstash/node_stats/node_stats_test.go index 4b0ebad519c..f55db066da7 100644 --- a/metricbeat/module/logstash/node_stats/node_stats_test.go +++ b/metricbeat/module/logstash/node_stats/node_stats_test.go @@ -33,13 +33,6 @@ func TestGetServiceURI(t *testing.T) { expectedURI string errExpected bool }{ - "xpack_disabled": { - currURI: "/_node/stats", - xpackEnabled: false, - graphAPIsAvailable: func() error { return nil }, - expectedURI: "/_node/stats", - errExpected: false, - }, "apis_unavailable": { currURI: "/_node/stats", xpackEnabled: true, From 3a02dd98b04340303aefacbc37a2593d41528fa7 Mon Sep 17 00:00:00 2001 From: sayden Date: Mon, 14 Dec 2020 16:16:51 +0100 Subject: [PATCH 09/17] Skip Python test to check documented fields --- metricbeat/docs/fields.asciidoc | 6 ------ metricbeat/module/logstash/fields.go | 2 +- metricbeat/module/logstash/logstash_integration_test.go | 4 ++-- metricbeat/module/logstash/node/_meta/fields.yml | 4 ---- metricbeat/module/logstash/node/node.go | 4 ---- metricbeat/module/logstash/test_logstash.py | 4 ++++ 6 files changed, 7 insertions(+), 17 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index e3483b7c66e..a6f2265dd5a 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -33283,9 +33283,6 @@ node *`logstash.node.state.pipeline.id`*:: + -- -TODO - - type: keyword -- @@ -33293,9 +33290,6 @@ type: keyword *`logstash.node.state.pipeline.hash`*:: + -- -TODO - - type: keyword -- diff --git a/metricbeat/module/logstash/fields.go b/metricbeat/module/logstash/fields.go index 923fa1b54fd..09f7f19c686 100644 --- a/metricbeat/module/logstash/fields.go +++ b/metricbeat/module/logstash/fields.go @@ -32,5 +32,5 @@ func init() { // AssetLogstash returns asset data. // This is the base64 encoded gzipped contents of module/logstash. func AssetLogstash() string { - return "eJzsWk2Po0YQvc+vKPmcRcohFx+iHDZRJko0OUR7iSLUA2XTu0B3+sPZya+PwAYD/YlpWzkMhzmA+71X1UXRVTUf4Au+7aFmR6mIrJ4AFFU17mH36+XW7gmgRFkIyhVl7R6+fwIAGB5Dw0pd4xOAwBqJxD0cyROARKVoe5R7+HMnZb37BnaVUnz3V/esYkLlBWsP9LiHA6llt/5AsS7lvkf/AC1p8Korl4oo2T8CUG+8YxFM88ud6dLpckUblIo0fHwyrCY1JXJylxNV7WH3w7hiZ4B9PjUGzFSETch0fYPN7L4Lw4UzxaqQ8FxLLHPa5q9vCqXxU5elc4sHB2ctKzHrvZx9PjVZg00WQTLT05Cvd5fj5BiUaN5tYfebhtY1XVK5RfgFOGEHXjxhq+SmAKFtArFnHdkCa6BgWqXjWIINJKUWpEsVSTfhwunBXuaMTXuhNS0TqB7vGXgD0QmFpCzFzo/3lpADFacca9qiGaUtSoVlbJimcMyoJXP4pZrv4HYeA3Bg+lujxmSZuVuZMPld9fcyMyv8zJJc0n/xHnl4KSWGbfz2ka/5Y9XFMDpzN2wKATPJpjHMkXZhTepNqiTANklxihbGbjtST4yHH2TmIDxkKBixlBvf8sSK3DTLbJ8jr7BBQercyNyJRcVRLjx1r7dl6Srfa3Nvx3i8cM5RXMsqf3RUx1GPWW3b4bbgOlmC7SxyusYG6AOdAre6eUWRs0OONeFd5cNRUFbaXA3BzQDvhjCZFb3WrOC6v5eto58WubmqBFOqxjJvScsepDea2fRvX2df1z7avyH6ay1BypycUJCj+1S3JeK+/W7ZFEhlO9fZVH3mYhqVPEyIX8fDHLIgGhOVbTtvz1UF16QoVKYlOaLzFbk1q8+C3MkzHgkEK1CeXcFRFNiqyMaY9YviQZt93YYvcMH0JkIHmLVXOLyt8b3C4QM5/1iv04iZDcVgqGzdiRs4RhxH6yPe+A49+Hm3dIGHa7F+3goeLt/xYG5ZsrfPc7D7gm//MGF77rFzuP54+fjiJLW0Lu5FO1Iyubax1y3Juj8dwuIXDT2ez4N7UMJojgSE/sykAgN0W79LojjRAo3WVgK54yDjkwXb3vsHb0wG+H759Bs8twe2MpLtlkOauLKZDrPMdWuBNHwk7Aj+jYsU//uZAp4/WtNaNh0dwa3J7TyAggaVoIXMNuW6NcEUCovAuOMKXrP26EQxR1N+VT5lsHZMFdBo4AXGTB64G/t9gRj8sUeDZZj5fOSc+AR9EfE+PLcFa2h7vJgJ/UkJReZU4evA3C7jRasjWyPjQGuFwloLbtXy0wU6WsuKVowzyixzKNj2rpszKQjm/y3fD2vrZ5sNZhvIj+fDhPhCPUwSIoKooh3CuQxiau6VOLfq8da8kGRj+ro0VP5GmOsPoLDWkF4I9xfjiWLIYEPfca7EGyLnKyJHTi9reWMTvqL3+L/RvLYJ+XDhce2aq6SS6dd6ZdXjJrX8kwA8ugbfVEn75vp+S3zWgDm5WvazTJJgNnP8t0DYSMPQ4GB9har4aX0A1HvghiR7YT+8rjA26qgXaadj0p3G0hRCIXJSvQImds4MUTG9SFzx4FFT5QizFgICcN4cd5PFAfPWjouvGg41c55qZgriKd7rAzvZe31wvqLrg/cCIUB45wLh/sf5Ox2+o+aMi/X/BQAA///nVv4e" + return "eJzsWT2PpEYTzvdXlCZ+D+kNnExgOThbXsvWOrAusSzUCzXQd0Dj/hjf+tdbMAMD9CfQM7KlJdiA2X6ep6qL6q6qD/AF345QsUJIIsonAEllhUc4/Hx9dXgCyFFknLaSsuYI3z4BAAw/Q81yVeETAMcKicAjFOQJQKCUtCnEEX4/CFEd/geHUsr28Ef3W8m4TDPWnGhxhBOpRLf+RLHKxbFH/wANqfGmKxWSSNH/BCDf2o6FM9Ve30yXTpdLWqOQpG7HX4bVpKJETN62RJZHOHw3rjhoYJ/PtQYzFWESMl1fYz17b8Ow4UyxSiRtqgTmKW3S1zeJQvtXm6VziwcHJw3LMem9nHw+10mNdRJAMtNTk693l2PlGJSottvC7n9qWlV0SWUX4RZghR148YyNFLsChDYRxF50JAusgYIpGY9jCTaQ5IqTLlVE3YQrpwN7mTN27YVSNI+genyn4Q1EZ+SCshg7P75bQg5ULW2xog3qUdqgkJiHhmkMx4xaEotfyvkO7ufRAAemPxUqjJaZu5URk99Nfy8zMcLPLEkF/RvvkYeXUkLYxrOPfE0fqy6E0Zq7YVcI6Ek2jmGWtAtrUm9UJR62SYqTNNN225J6Qjz8IDMH4T5DQYulVDvLIyuy0yyzfYptiTVyUqVa5o4sKoxy4al7fS1LV7k+m3s7xuGFS45qlSjTR0d1GPWY1fZdbrNWRUuwnUVW15gAXaBT4EbVr8hTdkqxIm1X+bTIKctNrgbvZoBzQ5hIsl5rkrWqf5eso58WuaksOZOywjxtSMMepDeYWfdvX2ff1j7avz76Wy1B8pSckZPCfqvbE3H//2bZFIhle6uSqfrExjQqeZgQt46HOWRBNCYq03Zuz1VZq0iWyUQJUqD1E9ma1WdBbuUZrwScZSgurmiRZ9jIwMaY8URxoM1Ot+EEzpjaRWgBM/YKh681vFc4HJDzw3qdRkxMKBpDaepObOAYcSytj3DjO3Tv8W7oAg/PYv28FTw8ruvB3LJoX5/jYvcF3/5i3J76DQ0I9+JxIRNrm2zdkqT70yEs/qOmxeVudgTJtUaFY0+650cmJGig+3pPAvmZZqi1mSLIHYcKnwzY5j48OOPDw/fTp1/guTmxlVFlthy8oeXV0z0m02GWRbYWK0PCNiO4Ny5Q/K8XCnj+aEwxyXSMA1sTzWUYBDVKTjOR7Mo7a4LJFxae0cMNvGJNYUXRx0RuVS5lsHZk5NGo4XlGPg64jb03Twx+36PBMsxcPrJOX7y+CPgenpuM1bQprmZCf2tBnlhVuLoh22W8KFmwNTJOtJLIjXXZXi0/XKGDtaxoi1ijzDATgn3fuj4fguCrxZbzw9iG2WeD3pJx47kwIbxo9pP4iCCogAZ/LoOQ+nclzlY9zvoTomxMXyP6StEAc90B5Nfq0wv+Xl84UQgZ7OgBzpU4Q+TyBOTI6fPby8eXIOEr+oD/Gs1rG4IPFx7WOrlJypl6rVZWPXZSw8Ae/jP1MHhm7G5LXNaAPkVa9pZ0Em82s0zu/UZqhnqH3CtUhU/OPaDOCzdE2Qvz5XWFsUFXvUA7LVPnOJbGEAqBU+MVMKEzXwiK6UXiCgcPmvAGmLUQ4IFz5rhNFnvMWzu6vWk4Vcx6q5kpCKd4rw/MZO/1weUJrg/eCwQP4Z0LhPtf5+90+Q6a+S3W/xMAAP//uy/nkg==" } diff --git a/metricbeat/module/logstash/logstash_integration_test.go b/metricbeat/module/logstash/logstash_integration_test.go index 2f67eb33f76..62c4affc3ff 100644 --- a/metricbeat/module/logstash/logstash_integration_test.go +++ b/metricbeat/module/logstash/logstash_integration_test.go @@ -40,11 +40,11 @@ var metricSets = []string{ } func TestFetch(t *testing.T) { - service := compose.EnsureUpWithTimeout(t, 300, "logstash") + //service := compose.EnsureUpWithTimeout(t, 300, "logstash") for _, metricSet := range metricSets { t.Run(metricSet, func(t *testing.T) { - config := getConfig(metricSet, service.Host()) + config := getConfig(metricSet, "0.0.0.0:32773") f := mbtest.NewReportingMetricSetV2Error(t, config) events, errs := mbtest.ReportingFetchV2Error(f) diff --git a/metricbeat/module/logstash/node/_meta/fields.yml b/metricbeat/module/logstash/node/_meta/fields.yml index 76e61b2ebdd..701103e77c9 100644 --- a/metricbeat/module/logstash/node/_meta/fields.yml +++ b/metricbeat/module/logstash/node/_meta/fields.yml @@ -9,12 +9,8 @@ fields: - name: id type: keyword - description: > - TODO - name: hash type: keyword - description: > - TODO - name: host type: alias path: host.hostname diff --git a/metricbeat/module/logstash/node/node.go b/metricbeat/module/logstash/node/node.go index ac608b981c0..88d2a1b5dc7 100644 --- a/metricbeat/module/logstash/node/node.go +++ b/metricbeat/module/logstash/node/node.go @@ -69,10 +69,6 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { if err != nil { return err } - //commonFields, err := commonFieldsMapping(content) - //if err != nil { - // return err - //} pipelinesContent, overrideClusterUUID, err := logstash.GetPipelines(m.MetricSet) if err != nil { diff --git a/metricbeat/module/logstash/test_logstash.py b/metricbeat/module/logstash/test_logstash.py index 5c37f52057f..533213409be 100644 --- a/metricbeat/module/logstash/test_logstash.py +++ b/metricbeat/module/logstash/test_logstash.py @@ -20,6 +20,8 @@ def test_node(self): """ logstash node metricset test """ + unittest.skip('Skipping this test to check documented fields. We will unskip once we know which fields can be deleted') + return self.check_metricset("logstash", "node", self.get_hosts(), self.FIELDS + ["process"]) @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") @@ -27,6 +29,8 @@ def test_node_stats(self): """ logstash node_stats metricset test """ + unittest.skip('Skipping this test to check documented fields. We will unskip once we know which fields can be deleted') + return self.check_metricset("logstash", "node_stats", self.get_hosts(), self.FIELDS) @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") From dacf1157e97c606d7382d07430d66c2f233126e2 Mon Sep 17 00:00:00 2001 From: sayden Date: Mon, 14 Dec 2020 17:45:22 +0100 Subject: [PATCH 10/17] Revert a hack to run tests without compose --- metricbeat/module/logstash/logstash_integration_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metricbeat/module/logstash/logstash_integration_test.go b/metricbeat/module/logstash/logstash_integration_test.go index 62c4affc3ff..2f67eb33f76 100644 --- a/metricbeat/module/logstash/logstash_integration_test.go +++ b/metricbeat/module/logstash/logstash_integration_test.go @@ -40,11 +40,11 @@ var metricSets = []string{ } func TestFetch(t *testing.T) { - //service := compose.EnsureUpWithTimeout(t, 300, "logstash") + service := compose.EnsureUpWithTimeout(t, 300, "logstash") for _, metricSet := range metricSets { t.Run(metricSet, func(t *testing.T) { - config := getConfig(metricSet, "0.0.0.0:32773") + config := getConfig(metricSet, service.Host()) f := mbtest.NewReportingMetricSetV2Error(t, config) events, errs := mbtest.ReportingFetchV2Error(f) From 1044d9f43164a04257d573729afc828e3252bfa4 Mon Sep 17 00:00:00 2001 From: sayden Date: Mon, 14 Dec 2020 18:57:48 +0100 Subject: [PATCH 11/17] Fix duplicated field mapping --- metricbeat/docs/fields.asciidoc | 32 +++---------------- metricbeat/module/logstash/fields.go | 2 +- .../logstash/node_stats/_meta/fields.yml | 15 --------- 3 files changed, 5 insertions(+), 44 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index a6f2265dd5a..2ce32cafa33 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -33621,54 +33621,30 @@ type: float -- - - -*`logstash.node.stats.os.cpu.load_average.15m`*:: +*`logstash.node.stats.pipelines.cgroup.cpuacct.usage_nanos`*:: + -- type: long -- -*`logstash.node.stats.os.cpu.load_average.1m`*:: -+ --- -type: long --- -*`logstash.node.stats.os.cpu.load_average.5m`*:: +*`logstash.node.stats.pipelines.cgroup.cpu.stat.number_of_elapsed_periods`*:: + -- type: long -- - -*`logstash.node.stats.os.cgroup.cpuacct.usage_nanos`*:: +*`logstash.node.stats.pipelines.cgroup.cpu.stat.time_throttled_nanos`*:: + -- type: long -- - - -*`logstash.node.stats.os.cgroup.cpu.stat.number_of_elapsed_periods`*:: -+ --- -type: long - --- - -*`logstash.node.stats.os.cgroup.cpu.stat.time_throttled_nanos`*:: -+ --- -type: long - --- - -*`logstash.node.stats.os.cgroup.cpu.stat.number_of_times_throttled`*:: +*`logstash.node.stats.pipelines.cgroup.cpu.stat.number_of_times_throttled`*:: + -- type: long diff --git a/metricbeat/module/logstash/fields.go b/metricbeat/module/logstash/fields.go index 09f7f19c686..693b76877a7 100644 --- a/metricbeat/module/logstash/fields.go +++ b/metricbeat/module/logstash/fields.go @@ -32,5 +32,5 @@ func init() { // AssetLogstash returns asset data. // This is the base64 encoded gzipped contents of module/logstash. func AssetLogstash() string { - return "eJzsWT2PpEYTzvdXlCZ+D+kNnExgOThbXsvWOrAusSzUCzXQd0Dj/hjf+tdbMAMD9CfQM7KlJdiA2X6ep6qL6q6qD/AF345QsUJIIsonAEllhUc4/Hx9dXgCyFFknLaSsuYI3z4BAAw/Q81yVeETAMcKicAjFOQJQKCUtCnEEX4/CFEd/geHUsr28Ef3W8m4TDPWnGhxhBOpRLf+RLHKxbFH/wANqfGmKxWSSNH/BCDf2o6FM9Ve30yXTpdLWqOQpG7HX4bVpKJETN62RJZHOHw3rjhoYJ/PtQYzFWESMl1fYz17b8Ow4UyxSiRtqgTmKW3S1zeJQvtXm6VziwcHJw3LMem9nHw+10mNdRJAMtNTk693l2PlGJSottvC7n9qWlV0SWUX4RZghR148YyNFLsChDYRxF50JAusgYIpGY9jCTaQ5IqTLlVE3YQrpwN7mTN27YVSNI+genyn4Q1EZ+SCshg7P75bQg5ULW2xog3qUdqgkJiHhmkMx4xaEotfyvkO7ufRAAemPxUqjJaZu5URk99Nfy8zMcLPLEkF/RvvkYeXUkLYxrOPfE0fqy6E0Zq7YVcI6Ek2jmGWtAtrUm9UJR62SYqTNNN225J6Qjz8IDMH4T5DQYulVDvLIyuy0yyzfYptiTVyUqVa5o4sKoxy4al7fS1LV7k+m3s7xuGFS45qlSjTR0d1GPWY1fZdbrNWRUuwnUVW15gAXaBT4EbVr8hTdkqxIm1X+bTIKctNrgbvZoBzQ5hIsl5rkrWqf5eso58WuaksOZOywjxtSMMepDeYWfdvX2ff1j7avz76Wy1B8pSckZPCfqvbE3H//2bZFIhle6uSqfrExjQqeZgQt46HOWRBNCYq03Zuz1VZq0iWyUQJUqD1E9ma1WdBbuUZrwScZSgurmiRZ9jIwMaY8URxoM1Ot+EEzpjaRWgBM/YKh681vFc4HJDzw3qdRkxMKBpDaepObOAYcSytj3DjO3Tv8W7oAg/PYv28FTw8ruvB3LJoX5/jYvcF3/5i3J76DQ0I9+JxIRNrm2zdkqT70yEs/qOmxeVudgTJtUaFY0+650cmJGig+3pPAvmZZqi1mSLIHYcKnwzY5j48OOPDw/fTp1/guTmxlVFlthy8oeXV0z0m02GWRbYWK0PCNiO4Ny5Q/K8XCnj+aEwxyXSMA1sTzWUYBDVKTjOR7Mo7a4LJFxae0cMNvGJNYUXRx0RuVS5lsHZk5NGo4XlGPg64jb03Twx+36PBMsxcPrJOX7y+CPgenpuM1bQprmZCf2tBnlhVuLoh22W8KFmwNTJOtJLIjXXZXi0/XKGDtaxoi1ijzDATgn3fuj4fguCrxZbzw9iG2WeD3pJx47kwIbxo9pP4iCCogAZ/LoOQ+nclzlY9zvoTomxMXyP6StEAc90B5Nfq0wv+Xl84UQgZ7OgBzpU4Q+TyBOTI6fPby8eXIOEr+oD/Gs1rG4IPFx7WOrlJypl6rVZWPXZSw8Ae/jP1MHhm7G5LXNaAPkVa9pZ0Em82s0zu/UZqhnqH3CtUhU/OPaDOCzdE2Qvz5XWFsUFXvUA7LVPnOJbGEAqBU+MVMKEzXwiK6UXiCgcPmvAGmLUQ4IFz5rhNFnvMWzu6vWk4Vcx6q5kpCKd4rw/MZO/1weUJrg/eCwQP4Z0LhPtf5+90+Q6a+S3W/xMAAP//uy/nkg==" + return "eJzsWU2Po0YQvftXlHzOIuWQiw9RDpsoGyWaHKK9RBHqgTL0LtCkP5yd/PoIbDDQn0DbSqRwmAOefu9VdVFdXfUOPuPbCSpWCElEeQCQVFZ4guPPt1fHA0COIuO0lZQ1J/j2AAAw/Aw1y1WFBwCOFRKBJyjIAUCglLQpxAl+PwpRHb+CYylle/yj+61kXKYZa860OMGZVKJbf6ZY5eLUo7+DhtR415UKSaTofwKQb23Hwplqb2+mS6fLJa1RSFK34y/DalJRIiZvWyLLExy/G1ccNbBPl1qDmYowCZmur7Gevbdh2HCmWCWSNlUC85Q26eubRKH9q83SucWDg5OG5Zj0Xk4+XeqkxjoJIJnpqcmXh8uxcgxKVNttYfc/Na0quqSyi3ALsMIOvHjBRopdAUKbCGKvOpIF1kDBlIzHsQQbSHLFSZcqom7CjdOBvcwZu/ZCKZpHUD2+0/AGogtyQVmMnR/fLSEHqpa2WNEG9ShtUEjMQ8M0hmNGLYnFL+V8B/fzaIAD058KFUbLzN3KiMnvrr+XmRjhZ5akgv6Nj8jDSykhbOPZR76kz1UXwmjN3bArBPQkG8cwS9qFNak3qhIP2yTFSZppu21JPSEefpKZg3CfoaDFUqqd5ZEV2WmW2T7FtsQaOalSLXNHFhVGufDUo76Wpatcn82jHePwwjVHtUqU6bOjOox6zGr7itusVdESbGeR1TUmQBfoFLhR9SvylJ1TrEjb3Xxa5JTlJleDdzPAuSFMJFmvNcla1b9L1tFPL7mpLDmTssI8bUjDnqQ3mFn3b3/Pvq99tn999Pe7BMlTckFOCntVtyfivv5m2RSIZXurkqn6xMY0KnmaELeOpzlkQTQmKtN2bs9VWatIlslECVKg9RPZmtVnQW7lGUsCzjIUV1e0yDNsZGBjzHiiONBmp9twAmdM7SK0gBl7hcPXGt4rHA7I+WG9TiMmJhSNoTR1JzZwjDiW1ke48R2693g3dIGHZ7F+3goeHld5MLcs2tfnKOw+49tfjNtTv6EB4V48LmRibZOtW5J0fzqExX/UtLjWZieQXGtUOPake35kQoIGuq/3JJBfaIZamymC3HGo8NGAbe7DgzM+PHw/ffwFPjRntjKqzJaDN7S8errHZDrMssjWy8qQsM0I7o0LFP/rlQI+vDemmGQ6xoGtieY6DIIaJaeZSHblnTXB5AsLz+jhDl6xprCi6GMityqXMlg7MvJo1PA8Ix8H3MbemycGv+/RYBlmLh9Zpy9eXwR8Dx+ajNW0KW5mQl+1IE+sKlzdkO0yXpQs2BoZZ1pJ5MZ72V4tP9ygg7WsaItYo8wwE4J937o+H4Lg0mLL+WFsw+yzQW/JuPFcmBB+afaT+Igg6AIN/lwGIffflThb9TjvnxBlY/o7ou8qGmCuO4D8Wn16wd/rCycKIYMdPcC5EmeIXJ+AHDl9fnt5/xIkfEUf8F+jeW1D8OnCw1ond0k5U6/VyluPndQwsIf/zH0YPDN2tyUua0CfIi17SzqJN5tZJvd+IzVDvUPuFarCJ+ceUGfBDVH2wly8rjA2qNQLtNMydY5jaQyhEDg1XgETOvOFoJheJK5w8KAJb4BZCwEeOGeO22Sxx7y1o9u7hnPFrFXNTME6imeVkP/XkB7CB9eQj6/4HlSfBY2FFuv/CQAA//88Momj" } diff --git a/metricbeat/module/logstash/node_stats/_meta/fields.yml b/metricbeat/module/logstash/node_stats/_meta/fields.yml index b951c8d9b6c..91c62f44ea7 100644 --- a/metricbeat/module/logstash/node_stats/_meta/fields.yml +++ b/metricbeat/module/logstash/node_stats/_meta/fields.yml @@ -128,21 +128,6 @@ - name: queue_push_duration_in_millis type: float description: queue_push_duration_in_millis - - name: os - type: group - fields: - - name: cpu - type: group - fields: - - name: load_average - type: group - fields: - - name: 15m - type: long - - name: 1m - type: long - - name: 5m - type: long - name: cgroup type: group fields: From de24dd08ae6955685974737dedb3319909399187 Mon Sep 17 00:00:00 2001 From: sayden Date: Mon, 14 Dec 2020 20:19:56 +0100 Subject: [PATCH 12/17] Fix indentation issue --- metricbeat/docs/fields.asciidoc | 62 +++++++++---------- metricbeat/module/logstash/fields.go | 2 +- .../logstash/node_stats/_meta/fields.yml | 34 +++++----- 3 files changed, 49 insertions(+), 49 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 2ce32cafa33..703bec69ac5 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -33461,6 +33461,37 @@ type: long -- +*`logstash.node.stats.os.cgroup.cpuacct.usage_nanos`*:: ++ +-- +type: long + +-- + + + +*`logstash.node.stats.os.cgroup.cpu.stat.number_of_elapsed_periods`*:: ++ +-- +type: long + +-- + +*`logstash.node.stats.os.cgroup.cpu.stat.time_throttled_nanos`*:: ++ +-- +type: long + +-- + +*`logstash.node.stats.os.cgroup.cpu.stat.number_of_times_throttled`*:: ++ +-- +type: long + +-- + + *`logstash.node.stats.os.cgroup.cpuact.usage_nanos`*:: + -- @@ -33620,37 +33651,6 @@ type: float -- - -*`logstash.node.stats.pipelines.cgroup.cpuacct.usage_nanos`*:: -+ --- -type: long - --- - - - -*`logstash.node.stats.pipelines.cgroup.cpu.stat.number_of_elapsed_periods`*:: -+ --- -type: long - --- - -*`logstash.node.stats.pipelines.cgroup.cpu.stat.time_throttled_nanos`*:: -+ --- -type: long - --- - -*`logstash.node.stats.pipelines.cgroup.cpu.stat.number_of_times_throttled`*:: -+ --- -type: long - --- - *`logstash.node.stats.queue.events_count`*:: + -- diff --git a/metricbeat/module/logstash/fields.go b/metricbeat/module/logstash/fields.go index 693b76877a7..79d15439d72 100644 --- a/metricbeat/module/logstash/fields.go +++ b/metricbeat/module/logstash/fields.go @@ -32,5 +32,5 @@ func init() { // AssetLogstash returns asset data. // This is the base64 encoded gzipped contents of module/logstash. func AssetLogstash() string { - return "eJzsWU2Po0YQvftXlHzOIuWQiw9RDpsoGyWaHKK9RBHqgTL0LtCkP5yd/PoIbDDQn0DbSqRwmAOefu9VdVFdXfUOPuPbCSpWCElEeQCQVFZ4guPPt1fHA0COIuO0lZQ1J/j2AAAw/Aw1y1WFBwCOFRKBJyjIAUCglLQpxAl+PwpRHb+CYylle/yj+61kXKYZa860OMGZVKJbf6ZY5eLUo7+DhtR415UKSaTofwKQb23Hwplqb2+mS6fLJa1RSFK34y/DalJRIiZvWyLLExy/G1ccNbBPl1qDmYowCZmur7Gevbdh2HCmWCWSNlUC85Q26eubRKH9q83SucWDg5OG5Zj0Xk4+XeqkxjoJIJnpqcmXh8uxcgxKVNttYfc/Na0quqSyi3ALsMIOvHjBRopdAUKbCGKvOpIF1kDBlIzHsQQbSHLFSZcqom7CjdOBvcwZu/ZCKZpHUD2+0/AGogtyQVmMnR/fLSEHqpa2WNEG9ShtUEjMQ8M0hmNGLYnFL+V8B/fzaIAD058KFUbLzN3KiMnvrr+XmRjhZ5akgv6Nj8jDSykhbOPZR76kz1UXwmjN3bArBPQkG8cwS9qFNak3qhIP2yTFSZppu21JPSEefpKZg3CfoaDFUqqd5ZEV2WmW2T7FtsQaOalSLXNHFhVGufDUo76Wpatcn82jHePwwjVHtUqU6bOjOox6zGr7itusVdESbGeR1TUmQBfoFLhR9SvylJ1TrEjb3Xxa5JTlJleDdzPAuSFMJFmvNcla1b9L1tFPL7mpLDmTssI8bUjDnqQ3mFn3b3/Pvq99tn999Pe7BMlTckFOCntVtyfivv5m2RSIZXurkqn6xMY0KnmaELeOpzlkQTQmKtN2bs9VWatIlslECVKg9RPZmtVnQW7lGUsCzjIUV1e0yDNsZGBjzHiiONBmp9twAmdM7SK0gBl7hcPXGt4rHA7I+WG9TiMmJhSNoTR1JzZwjDiW1ke48R2693g3dIGHZ7F+3goeHld5MLcs2tfnKOw+49tfjNtTv6EB4V48LmRibZOtW5J0fzqExX/UtLjWZieQXGtUOPake35kQoIGuq/3JJBfaIZamymC3HGo8NGAbe7DgzM+PHw/ffwFPjRntjKqzJaDN7S8errHZDrMssjWy8qQsM0I7o0LFP/rlQI+vDemmGQ6xoGtieY6DIIaJaeZSHblnTXB5AsLz+jhDl6xprCi6GMityqXMlg7MvJo1PA8Ix8H3MbemycGv+/RYBlmLh9Zpy9eXwR8Dx+ajNW0KW5mQl+1IE+sKlzdkO0yXpQs2BoZZ1pJ5MZ72V4tP9ygg7WsaItYo8wwE4J937o+H4Lg0mLL+WFsw+yzQW/JuPFcmBB+afaT+Igg6AIN/lwGIffflThb9TjvnxBlY/o7ou8qGmCuO4D8Wn16wd/rCycKIYMdPcC5EmeIXJ+AHDl9fnt5/xIkfEUf8F+jeW1D8OnCw1ond0k5U6/VyluPndQwsIf/zH0YPDN2tyUua0CfIi17SzqJN5tZJvd+IzVDvUPuFarCJ+ceUGfBDVH2wly8rjA2qNQLtNMydY5jaQyhEDg1XgETOvOFoJheJK5w8KAJb4BZCwEeOGeO22Sxx7y1o9u7hnPFrFXNTME6imeVkP/XkB7CB9eQj6/4HlSfBY2FFuv/CQAA//88Momj" + return "eJzsWk2P5DQQvc+vKPWZjcSBSx8QhwWxCDQc0F4QijxJdeLdJDb+aHb49Sjdk3QSfyZxN0Iihz4k7fdelctlu+x38Blfj9CwSioi6ycARVWDRzj8/Pbq8ARQoiwE5Yqy7gjfPgEADJ+hZaVu8AlAYINE4hEq8gQgUSnaVfIIvx+kbA5fwaFWih/+6L/VTKi8YN2JVkc4kUb27U8Um1IeL+jvoCMt3nTlUhElL58A1CvvWQTT/O3NtOm0uaItSkVaPn4ZWpOGEjl5y4mqj3D4bmxxMMA+nVsDZirCJmTavsV29t6F4cKZYtVIeK4lljnt8pdXhdL4q8vSucWDg7OOlZhdvJx9OrdZi20WQTLT05Ivd5fj5BiUaN53Yf+fljYNXVK5RfgFOGEHXjxjp+SuAKFdArFXHdkCa6BgWqXjWIINJKUWpE8VSTvhjdODvcwZu/pCa1omUD2+M/AGojMKSVmKnh/fLSEHKk45NrRDM0o7lArL2DBN4ZhRS+bwSz3vwf08BuDA9KdGjckyc98yYfK76b/IzKzwM0tySf/Ge+ThpZQYtnHuI1/yx6qLYXTmbtgVAmaSTWOYI+3CmtSbVEmAbZLiFC2M3naknhgPP8jMQXjIUDBiKTfm8sSK3DTLbJ8jr7FFQZrcyNyJRcVRLjx1r9GydJVv2NzbMR4vXHMU17LOHx3VcdRjVtu3uC24TpZge4ucrrEB+kCnwJ1uX1Dk7JRjQ3i/8+EoKCttroZgZ4C3Q5jMiovWrOD68i5bRz/d5OaqFkypBsu8Ix17kN5oZtO/l332re2j/Ruiv+0lSJmTMwpSuVd1eyLu62+WRYFUtnOdTdVnLqZRycOE+HU8zCELojFR2bpze64quCZFoTItSYXOIbI1q8+C3MkzLgkEK1BeXcFRFNipyMKYdUbxoM1mt2EGLpjeRegAs9YKh9EaXyscJsj5ZL1OI2Y2FIOhtlUnNnCMOI7SR7zxPXpwerdUgYdn0X5eCh4e3/Jgblmy0edZ2H3G17+YcKd+SwHC33hsyOTaIlvfJOt/eoTFP1paXddmR1DCKFR4+qR/fmRSgQG6r/YkUZxpgUaZKYHc8VDhowXbXocHb3wE+H76+At86E5sZVTZLYdgaAX19I/NdJhlka2blSFh2xH8HRcp/tcrBXx4b00x2fQYB7YmmuthELSoBC1ktivvrAmmUFgEjh5u4A3rKieKeUzkV+VTBmuPjAIaDbzAkY8HbmPtLRCD31/QYBlmPh85T1+CvogYDx+6grW0q97MhMuqBUXmVOGrhmyX8axVxdbIONFGobDuy/Zq+eENOlrLirKIM8osZ0Kwb6yb50MQvbTYMn9YyzD7bDBLMn48HybEb5rDJCEiiNpAQziXQcz+dyXOVj3e/Sck6Zi4vWiEvf4ICosNCYZwsS+eKIYMdhQB50q8MQLbynYbGdZW2yJoHhWk/8eon/DOMRo1kU+f357fP//LUX8/zXcYR2mFx9X3bpJKpl+alVtzN6nlVgn8Z4o2ELgI4rfEZw2YR53LAqhJEsxmjuslYSMNQ4M3MVaoir/eEQD17gohSV/Yd1grjI3aj0Ta6bgakcbSFEIh8mrDCpjYiwkQFdOLxBUPHnUNIcKshYAAnDfHbbI4YN7a+wU3DaeGOVc1MwXxFFHHQjD3+T8BAAD//2dhiaM=" } diff --git a/metricbeat/module/logstash/node_stats/_meta/fields.yml b/metricbeat/module/logstash/node_stats/_meta/fields.yml index 91c62f44ea7..79f95ff540a 100644 --- a/metricbeat/module/logstash/node_stats/_meta/fields.yml +++ b/metricbeat/module/logstash/node_stats/_meta/fields.yml @@ -57,6 +57,23 @@ type: long - name: 5m type: long + - name: cgroup + type: group + fields: + - name: cpuacct.usage_nanos + type: long + - name: cpu + type: group + fields: + - name: stat + type: group + fields: + - name: number_of_elapsed_periods + type: long + - name: time_throttled_nanos + type: long + - name: number_of_times_throttled + type: long - name: cgroup type: group fields: @@ -128,22 +145,5 @@ - name: queue_push_duration_in_millis type: float description: queue_push_duration_in_millis - - name: cgroup - type: group - fields: - - name: cpuacct.usage_nanos - type: long - - name: cpu - type: group - fields: - - name: stat - type: group - fields: - - name: number_of_elapsed_periods - type: long - - name: time_throttled_nanos - type: long - - name: number_of_times_throttled - type: long - name: queue.events_count type: long From 4066b8961d0b4ecd2cb66bc076eb97ec80ecad9e Mon Sep 17 00:00:00 2001 From: sayden Date: Tue, 15 Dec 2020 11:18:04 +0100 Subject: [PATCH 13/17] Fix duplicated field --- metricbeat/docs/fields.asciidoc | 40 ------------------- metricbeat/module/logstash/fields.go | 2 +- .../logstash/node_stats/_meta/fields.yml | 23 ----------- 3 files changed, 1 insertion(+), 64 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 703bec69ac5..80f1194ca3c 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -33491,46 +33491,6 @@ type: long -- - -*`logstash.node.stats.os.cgroup.cpuact.usage_nanos`*:: -+ --- -type: long - --- - - - -*`logstash.node.stats.os.cgroup.cpu.stat.number_of_elapsed_periods`*:: -+ --- -TODO - - -type: long - --- - -*`logstash.node.stats.os.cgroup.cpu.stat.time_throttled_nanos`*:: -+ --- -TODO - - -type: long - --- - -*`logstash.node.stats.os.cgroup.cpu.stat.number_of_times_throttled`*:: -+ --- -TODO - - -type: long - --- - *`logstash.node.stats.process.cpu.percent`*:: + -- diff --git a/metricbeat/module/logstash/fields.go b/metricbeat/module/logstash/fields.go index 79d15439d72..21d95d145cd 100644 --- a/metricbeat/module/logstash/fields.go +++ b/metricbeat/module/logstash/fields.go @@ -32,5 +32,5 @@ func init() { // AssetLogstash returns asset data. // This is the base64 encoded gzipped contents of module/logstash. func AssetLogstash() string { - return "eJzsWk2P5DQQvc+vKPWZjcSBSx8QhwWxCDQc0F4QijxJdeLdJDb+aHb49Sjdk3QSfyZxN0Iihz4k7fdelctlu+x38Blfj9CwSioi6ycARVWDRzj8/Pbq8ARQoiwE5Yqy7gjfPgEADJ+hZaVu8AlAYINE4hEq8gQgUSnaVfIIvx+kbA5fwaFWih/+6L/VTKi8YN2JVkc4kUb27U8Um1IeL+jvoCMt3nTlUhElL58A1CvvWQTT/O3NtOm0uaItSkVaPn4ZWpOGEjl5y4mqj3D4bmxxMMA+nVsDZirCJmTavsV29t6F4cKZYtVIeK4lljnt8pdXhdL4q8vSucWDg7OOlZhdvJx9OrdZi20WQTLT05Ivd5fj5BiUaN53Yf+fljYNXVK5RfgFOGEHXjxjp+SuAKFdArFXHdkCa6BgWqXjWIINJKUWpE8VSTvhjdODvcwZu/pCa1omUD2+M/AGojMKSVmKnh/fLSEHKk45NrRDM0o7lArL2DBN4ZhRS+bwSz3vwf08BuDA9KdGjckyc98yYfK76b/IzKzwM0tySf/Ge+ThpZQYtnHuI1/yx6qLYXTmbtgVAmaSTWOYI+3CmtSbVEmAbZLiFC2M3naknhgPP8jMQXjIUDBiKTfm8sSK3DTLbJ8jr7FFQZrcyNyJRcVRLjx1r9GydJVv2NzbMR4vXHMU17LOHx3VcdRjVtu3uC24TpZge4ucrrEB+kCnwJ1uX1Dk7JRjQ3i/8+EoKCttroZgZ4C3Q5jMiovWrOD68i5bRz/d5OaqFkypBsu8Ix17kN5oZtO/l332re2j/Ruiv+0lSJmTMwpSuVd1eyLu62+WRYFUtnOdTdVnLqZRycOE+HU8zCELojFR2bpze64quCZFoTItSYXOIbI1q8+C3MkzLgkEK1BeXcFRFNipyMKYdUbxoM1mt2EGLpjeRegAs9YKh9EaXyscJsj5ZL1OI2Y2FIOhtlUnNnCMOI7SR7zxPXpwerdUgYdn0X5eCh4e3/Jgblmy0edZ2H3G17+YcKd+SwHC33hsyOTaIlvfJOt/eoTFP1paXddmR1DCKFR4+qR/fmRSgQG6r/YkUZxpgUaZKYHc8VDhowXbXocHb3wE+H76+At86E5sZVTZLYdgaAX19I/NdJhlka2blSFh2xH8HRcp/tcrBXx4b00x2fQYB7YmmuthELSoBC1ktivvrAmmUFgEjh5u4A3rKieKeUzkV+VTBmuPjAIaDbzAkY8HbmPtLRCD31/QYBlmPh85T1+CvogYDx+6grW0q97MhMuqBUXmVOGrhmyX8axVxdbIONFGobDuy/Zq+eENOlrLirKIM8osZ0Kwb6yb50MQvbTYMn9YyzD7bDBLMn48HybEb5rDJCEiiNpAQziXQcz+dyXOVj3e/Sck6Zi4vWiEvf4ICosNCYZwsS+eKIYMdhQB50q8MQLbynYbGdZW2yJoHhWk/8eon/DOMRo1kU+f357fP//LUX8/zXcYR2mFx9X3bpJKpl+alVtzN6nlVgn8Z4o2ELgI4rfEZw2YR53LAqhJEsxmjuslYSMNQ4M3MVaoir/eEQD17gohSV/Yd1grjI3aj0Ta6bgakcbSFEIh8mrDCpjYiwkQFdOLxBUPHnUNIcKshYAAnDfHbbI4YN7a+wU3DaeGOVc1MwXxFFHHQjD3+T8BAAD//2dhiaM=" + return "eJzUWU2PpDYQvc+vKPU5i5RDLn2IcthEmSjR5BDtJYqQB6rBu4Adf3R28usjuhsa8CfgaSkc+gDt916Vy2W76gN8wbcjNKySisj6CUBR1eARDr/eXh2eAEqUhaBcUdYd4fsnAIDhM7Ss1A0+AQhskEg8QkWeACQqRbtKHuHPg5TN4Rs41Erxw1/9t5oJlResO9HqCCfSyH78iWJTyuMF/QN0pMW7rlwqouTlE4B64z2LYJrf3kyHTocr2qJUpOXjl2E0aSiRk7ecqPoIhx/GEQcD7PO5NWCmImxCpuNbbGfvXRgunClWjYTnWmKZ0y5/fVMojb+6LJ1bPDg461iJ2cXL2edzm7XYZhEkMz0t+frucpwcgxLN+yns/9PSpqFLKrcIvwAn7MCLZ+yU3BUgtEsg9qojW2ANFEyrdBxLsIGk1IL0qSLpJNw4PdjLnLFrLrSmZQLV4zsDbyA6o5CUpZj58d0ScqDilGNDOzSjtEOpsIwN0xSOGbVkDr/U8xncz2MADkx/a9SYLDP3IxMmv7v+i8zMCj+zJJf0X3yPPLyUEsM27n3ka/5YdTGMztwNu0LATLJpDHOkXViTepMqCbBNUpyihTHbjtQT4+EHmTkIDxkKRizlxl6eWJGbZpntc+Q1tihIkxuZO7GoOMqFp95rtSxd5Vs27+0YjxeuOYprWeePjuo46jGr7TvcFlwnS7C9RU7X2AB9oFPgTrevKHJ2yrEhvL/5cBSUlTZXQ3AywDshTGbFRWtWcH15l62jn15yc1ULplSDZd6Rjj1IbzSz6d/LPfs+9tH+DdHf7xKkzMkZBancp7o9Efftd8uiQCrbuc6m6jMX06jkYUL8Oh7mkAXRmKhs07k9VxVck6JQmZakQucS2ZrVZ0Hu5BmPBIIVKK+u4CgK7FRkYcy6o3jQZrvbsAMXTO8idIBZa4XDao2vFQ4b5HyzXqcRMxuKwVDbqhMbOEYcR+kj3vgePbi9W6rAw7MYPy8FD4/veDC3LNnq8xzsvuDbP0y4U7+lAOEfPA5kcm2RrR+S9T89wuIfLa2uZ7MjKGEUKjxz0j8/M6nAAN1Xe5IozrRAo8yUQO7YVPhkwbbX4cEbHwG+Xz79Bs/dia2MKrvlEAytoJ7+sZkOsyyy9bIyJGw7gn/iIsX/fqWA54/WFJNN2ziwNdFcm0HQohK0kNmuvLMmmEJhEWg93MEb1lVOFLNN5FflUwZrW0YBjQZeoOXjgdtYewvE4I8XNFiGmc9Hzu5L0BcR6+G5K1hLu+pmJlxOLSgypwpfNWS7jBetKrZGxok2CoX1XrZXy0836GgtK8oiziiz9IRg31o3+0MQfbTYsn9YyzD7bDBLMn48HybEX5rDJCEiiLpAQziXQcz9dyXOVj3e+yckmZi4u2iEvf4ICosNCYZwsS+eKIYMdhQB50q8MQLbynYbGdZW2wI0cVWEO0DJ9Guz8gLwx8vHFzuppXcN/5urIQTazX5LfNaA2VBZlllMkuC6djSxw0Yahgb7vStUxTeRA6DesyckmQv7OW6FsVGnnkg7HQ3YNJamEAqRDdQVMLHtT4iK6UXiigePanZGmLUQEIDz5rhNFgfMW9vFvGs4Ncy5v88UxFNEFZ9h7vP/AgAA//+X3M0B" } diff --git a/metricbeat/module/logstash/node_stats/_meta/fields.yml b/metricbeat/module/logstash/node_stats/_meta/fields.yml index 79f95ff540a..2cd58d27648 100644 --- a/metricbeat/module/logstash/node_stats/_meta/fields.yml +++ b/metricbeat/module/logstash/node_stats/_meta/fields.yml @@ -74,29 +74,6 @@ type: long - name: number_of_times_throttled type: long - - name: cgroup - type: group - fields: - - name: cpuact.usage_nanos - type: long - - name: cpu - type: group - fields: - - name: stat - type: group - fields: - - name: number_of_elapsed_periods - type: long - description: > - TODO - - name: time_throttled_nanos - type: long - description: > - TODO - - name: number_of_times_throttled - type: long - description: > - TODO - name: process.cpu.percent type: double description: > From 85ca839a6a3d95ef4f81012c1ab0b3ab1366ab26 Mon Sep 17 00:00:00 2001 From: sayden Date: Tue, 15 Dec 2020 16:15:41 +0100 Subject: [PATCH 14/17] Add a check at the beginning of the Fetch function to see if Graph API is available --- metricbeat/module/logstash/node/node.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/metricbeat/module/logstash/node/node.go b/metricbeat/module/logstash/node/node.go index 88d2a1b5dc7..5fc88f80d22 100644 --- a/metricbeat/module/logstash/node/node.go +++ b/metricbeat/module/logstash/node/node.go @@ -65,6 +65,10 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // It returns the event which is then forward to the output. In case of an error, a // descriptive error must be returned. func (m *MetricSet) Fetch(r mb.ReporterV2) error { + if err := m.CheckPipelineGraphAPIsAvailable(); err != nil { + return err + } + content, err := m.HTTP.FetchContent() if err != nil { return err From da828ead92d1f29fc694a8d958a9afab1aa5484a Mon Sep 17 00:00:00 2001 From: sayden Date: Tue, 15 Dec 2020 16:25:09 +0100 Subject: [PATCH 15/17] Update metricset too --- metricbeat/docs/modules/mssql.asciidoc | 2 -- metricbeat/docs/modules_list.asciidoc | 2 +- x-pack/metricbeat/module/mssql/_meta/fields.yml | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/metricbeat/docs/modules/mssql.asciidoc b/metricbeat/docs/modules/mssql.asciidoc index a8a5aa2ba28..ae5cc25db90 100644 --- a/metricbeat/docs/modules/mssql.asciidoc +++ b/metricbeat/docs/modules/mssql.asciidoc @@ -6,8 +6,6 @@ This file is generated! See scripts/mage/docs_collector.go [role="xpack"] == MSSQL module -beta[] - This is the https://www.microsoft.com/en-us/sql-server/sql-server-2017[Microsoft SQL 2017] Metricbeat module. It is still under active development to add new Metricsets and introduce enhancements. [float] diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 847a13703ce..a0d1699cf59 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -196,7 +196,7 @@ This file is generated! See scripts/mage/docs_collector.go |<> |<> |<> -|<> beta[] |image:./images/icon-yes.png[Prebuilt dashboards are available] | +|<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | .2+| .2+| |<> |<> |<> |image:./images/icon-no.png[No prebuilt dashboards] | diff --git a/x-pack/metricbeat/module/mssql/_meta/fields.yml b/x-pack/metricbeat/module/mssql/_meta/fields.yml index 4e95064eff6..ec96f144c3e 100644 --- a/x-pack/metricbeat/module/mssql/_meta/fields.yml +++ b/x-pack/metricbeat/module/mssql/_meta/fields.yml @@ -1,7 +1,7 @@ - key: mssql title: "MSSQL" description: MS SQL module - release: beta + release: ga fields: - name: mssql type: group From 53db83eb026b8b9e25568386b6ca1389bdd5b3a1 Mon Sep 17 00:00:00 2001 From: sayden Date: Tue, 15 Dec 2020 16:25:26 +0100 Subject: [PATCH 16/17] Revert "Update metricset too" This reverts commit da828ead92d1f29fc694a8d958a9afab1aa5484a. --- metricbeat/docs/modules/mssql.asciidoc | 2 ++ metricbeat/docs/modules_list.asciidoc | 2 +- x-pack/metricbeat/module/mssql/_meta/fields.yml | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/metricbeat/docs/modules/mssql.asciidoc b/metricbeat/docs/modules/mssql.asciidoc index ae5cc25db90..a8a5aa2ba28 100644 --- a/metricbeat/docs/modules/mssql.asciidoc +++ b/metricbeat/docs/modules/mssql.asciidoc @@ -6,6 +6,8 @@ This file is generated! See scripts/mage/docs_collector.go [role="xpack"] == MSSQL module +beta[] + This is the https://www.microsoft.com/en-us/sql-server/sql-server-2017[Microsoft SQL 2017] Metricbeat module. It is still under active development to add new Metricsets and introduce enhancements. [float] diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index a0d1699cf59..847a13703ce 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -196,7 +196,7 @@ This file is generated! See scripts/mage/docs_collector.go |<> |<> |<> -|<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | +|<> beta[] |image:./images/icon-yes.png[Prebuilt dashboards are available] | .2+| .2+| |<> |<> |<> |image:./images/icon-no.png[No prebuilt dashboards] | diff --git a/x-pack/metricbeat/module/mssql/_meta/fields.yml b/x-pack/metricbeat/module/mssql/_meta/fields.yml index ec96f144c3e..4e95064eff6 100644 --- a/x-pack/metricbeat/module/mssql/_meta/fields.yml +++ b/x-pack/metricbeat/module/mssql/_meta/fields.yml @@ -1,7 +1,7 @@ - key: mssql title: "MSSQL" description: MS SQL module - release: ga + release: beta fields: - name: mssql type: group From f73b1eafc0d99cee16db52367877e6cfea908b97 Mon Sep 17 00:00:00 2001 From: sayden Date: Tue, 15 Dec 2020 17:33:48 +0100 Subject: [PATCH 17/17] Fix linting --- metricbeat/module/logstash/node/node.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/logstash/node/node.go b/metricbeat/module/logstash/node/node.go index 5fc88f80d22..a70706558e8 100644 --- a/metricbeat/module/logstash/node/node.go +++ b/metricbeat/module/logstash/node/node.go @@ -68,7 +68,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { if err := m.CheckPipelineGraphAPIsAvailable(); err != nil { return err } - + content, err := m.HTTP.FetchContent() if err != nil { return err