Skip to content

Commit

Permalink
Make logstash/node_stats metricset work for Stack Monitoring without …
Browse files Browse the repository at this point in the history
…xpack.enabled flag (elastic#21546)
  • Loading branch information
sayden authored Dec 15, 2020
1 parent a2597cd commit 3dce95a
Show file tree
Hide file tree
Showing 23 changed files with 2,069 additions and 1,167 deletions.
439 changes: 437 additions & 2 deletions metricbeat/docs/fields.asciidoc

Large diffs are not rendered by default.

146 changes: 145 additions & 1 deletion metricbeat/module/logstash/_meta/fields.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,152 @@
Logstash module
release: ga
settings: ["ssl", "http"]
short_config: false
fields:
- name: logstash_stats
type: group
fields:
- name: timestamp
type: alias
path: "@timestamp"
- name: jvm
type: group
fields:
- name: mem
type: group
fields:
- name: heap_used_in_bytes
type: alias
path: logstash.node.stats.jvm.mem.heap_used_in_bytes
- name: heap_max_in_bytes
type: alias
path: logstash.node.stats.jvm.mem.heap_max_in_bytes
- name: uptime_in_millis
type: alias
path: logstash.node.stats.jvm.uptime_in_millis
- name: events
type: group
fields:
- name: in
type: alias
path: logstash.node.stats.events.in
- name: out
type: alias
path: logstash.node.stats.events.out
- name: duration_in_millis
type: alias
path: logstash.node.stats.events.duration_in_millis
- name: logstash
type: group
fields:
- name: uuid
type: alias
path: logstash.node.stats.logstash.uuid
- name: version
type: alias
path: logstash.node.stats.logstash.version
- name: pipelines
type: nested
fields:
- name: id
type: alias
path: logstash.node.stats.pipelines.id
- name: hash
type: alias
path: logstash.node.stats.pipelines.hash
- name: queue
type: group
fields:
- name: type
type: alias
path: logstash.node.stats.pipelines.queue.type
- name: queue_size_in_bytes
type: alias
path: logstash.node.stats.pipelines.queue.queue_size_in_bytes
- name: max_queue_size_in_bytes
type: alias
path: logstash.node.stats.pipelines.queue.max_queue_size_in_bytes
- name: events
type: group
fields:
- name: out
type: alias
path: logstash.node.stats.pipelines.events.out
- name: duration_in_millis
type: alias
path: logstash.node.stats.pipelines.events.duration_in_millis
- name: vertices
type: nested
fields:
- name: duration_in_millis
type: alias
path: logstash.node.stats.pipelines.vertices.duration_in_millis
- name: events_in
type: alias
path: logstash.node.stats.pipelines.vertices.events_in
- name: pipeline_ephemeral_id
type: alias
path: logstash.node.stats.pipelines.vertices.pipeline_ephemeral_id
- name: events_out
type: alias
path: logstash.node.stats.pipelines.vertices.events_out
- name: id
type: alias
path: logstash.node.stats.pipelines.vertices.id
- name: queue_push_duration_in_millis
type: alias
path: logstash.node.stats.pipelines.vertices.queue_push_duration_in_millis
- name: os
type: group
fields:
- name: cpu
type: group
fields:
- name: stat
type: group
fields:
- name: number_of_elapsed_periods
type: alias
path: logstash.node.stats.os.cgroup.cpu.stat.number_of_elapsed_periods
- name: time_throttled_nanos
type: alias
path: logstash.node.stats.os.cgroup.cpu.stat.time_throttled_nanos
- name: number_of_times_throttled
type: alias
path: logstash.node.stats.os.cgroup.cpu.stat.number_of_times_throttled
- name: load_average
type: group
fields:
- name: 15m
type: alias
path: logstash.node.stats.os.cpu.load_average.15m
- name: 1m
type: alias
path: logstash.node.stats.os.cpu.load_average.1m
- name: 5m
type: alias
path: logstash.node.stats.os.cpu.load_average.5m
- name: cgroup
type: group
fields:
- name: cpuacct.usage_nanos
type: alias
path: logstash.node.stats.os.cgroup.cpuacct.usage_nanos
- name: process.cpu.percent
type: alias
path: logstash.node.stats.process.cpu.percent
- name: queue.events_count
type: alias
path: logstash.node.stats.queue.events_count
- name: logstash_state
type: group
fields:
- name: pipeline.id
type: alias
path: logstash.node.state.pipeline.id
- name: pipeline.hash
type: alias
path: logstash.node.state.pipeline.hash
- name: logstash
type: group
description: >
fields:
2 changes: 1 addition & 1 deletion metricbeat/module/logstash/fields.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

11 changes: 0 additions & 11 deletions metricbeat/module/logstash/logstash.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@ var PipelineGraphAPIsAvailableVersion = common.MustNewVersion("7.3.0")
type MetricSet struct {
mb.BaseMetricSet
*helper.HTTP
XPack bool
}

type Graph struct {
Expand Down Expand Up @@ -83,15 +82,6 @@ type PipelineState struct {
// NewMetricSet creates a metricset that can be used to build other metricsets
// within the Logstash module.
func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) {
config := struct {
XPack bool `config:"xpack.enabled"`
}{
XPack: false,
}
if err := base.Module().UnpackConfig(&config); err != nil {
return nil, err
}

http, err := helper.NewHTTP(base)
if err != nil {
return nil, err
Expand All @@ -100,7 +90,6 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) {
return &MetricSet{
base,
http,
config.XPack,
}, nil
}

Expand Down
32 changes: 3 additions & 29 deletions metricbeat/module/logstash/logstash_integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,31 +70,6 @@ func TestData(t *testing.T) {
}
}

func TestXPackEnabled(t *testing.T) {
lsService := compose.EnsureUpWithTimeout(t, 300, "logstash")
esService := compose.EnsureUpWithTimeout(t, 300, "elasticsearch")

clusterUUID := getESClusterUUID(t, esService.Host())

metricSetToTypeMap := map[string]string{
"node": "logstash_state",
"node_stats": "logstash_stats",
}

config := getXPackConfig(lsService.Host())
metricSets := mbtest.NewReportingMetricSetV2Errors(t, config)
for _, metricSet := range metricSets {
events, errs := mbtest.ReportingFetchV2Error(metricSet)
require.Empty(t, errs)
require.NotEmpty(t, events)

event := events[0]
require.Equal(t, metricSetToTypeMap[metricSet.Name()], event.RootFields["type"])
require.Equal(t, clusterUUID, event.RootFields["cluster_uuid"])
require.Regexp(t, `^.monitoring-logstash-\d-mb`, event.Index)
}
}

func getConfig(metricSet string, host string) map[string]interface{} {
return map[string]interface{}{
"module": logstash.ModuleName,
Expand All @@ -105,10 +80,9 @@ func getConfig(metricSet string, host string) map[string]interface{} {

func getXPackConfig(host string) map[string]interface{} {
return map[string]interface{}{
"module": logstash.ModuleName,
"metricsets": metricSets,
"hosts": []string{host},
"xpack.enabled": true,
"module": logstash.ModuleName,
"metricsets": metricSets,
"hosts": []string{host},
}
}

Expand Down
139 changes: 120 additions & 19 deletions metricbeat/module/logstash/node/_meta/data.json
Original file line number Diff line number Diff line change
@@ -1,33 +1,134 @@
{
"@timestamp": "2017-10-12T08:05:34.853Z",
"@timestamp": "2020-10-05T10:50:11.757Z",
"@metadata": {
"beat": "metricbeat",
"type": "_doc",
"version": "8.0.0",
"_id": "afb1a50a-95f0-484a-b7d7-e683ddddc75a"
},
"host": {
"name": "mcastro"
},
"agent": {
"hostname": "host.example.com",
"name": "host.example.com"
"ephemeral_id": "c4b22628-7b30-4a5d-8e28-7b6de81c9974",
"id": "803dfdba-e638-4590-a2de-80cb1cebe78d",
"name": "mcastro",
"type": "metricbeat",
"version": "8.0.0"
},
"event": {
"duration": 9740086,
"dataset": "logstash.node",
"duration": 115000,
"module": "logstash"
},
"metricset": {
"name": "node",
"period": 10000
},
"service": {
"address": "localhost:9600",
"type": "logstash"
},
"logstash": {
"node": {
"host": "2cb47f6e0eab",
"version": "8.0.0",
"jvm": {
"version": "1.8.0_191"
"version": "11.0.5"
},
"id": "4cc683ce-3ddc-46e3-bea3-aefbf37bc082",
"state": {
"pipeline": {
"hash": "3000c3abf87d4dfa4a57aaf6af0a1f5bee2e0fc1c48a8e8636e2a33d7d2e91dd",
"ephemeral_id": "afb1a50a-95f0-484a-b7d7-e683ddddc75a",
"representation": {
"graph": {
"edges": [
{
"from": "1bf3a9cc73ceb7c3a9cbe885df249b23f3496c52a342a6d513153cc865d78182",
"id": "b3db599ec6ae0b9493158bd7024dcd922c8a3e76295c37fef0da440086bf3f8c",
"to": "__QUEUE__",
"type": "plain"
},
{
"type": "plain",
"from": "71b91bc85b66ab25c5fb16e63db4dd7111c183f96d1f18e19078051ed5fc74f7",
"id": "9db20a77b3e1eb91229a50bd33388425d59725f9093e076a37e6565f8d5a20ad",
"to": "__QUEUE__"
},
{
"id": "9b2bc571e978746fb9b55b83521a6603c3c940144cde0e3f4296298cea6585cf",
"to": "a339cb309b29181703c6adf321da3d639f5b60713de5a1e5519ebfea069556d8",
"type": "plain",
"from": "__QUEUE__"
}
],
"vertices": [
{
"config_name": "beats",
"explicit_id": false,
"id": "1bf3a9cc73ceb7c3a9cbe885df249b23f3496c52a342a6d513153cc865d78182",
"meta": {
"source": {
"line": 2,
"protocol": "file",
"column": 3,
"id": "/usr/share/logstash/pipeline/default.conf"
}
},
"plugin_type": "input",
"type": "plugin"
},
{
"plugin_type": "input",
"type": "plugin",
"config_name": "beats",
"explicit_id": false,
"id": "71b91bc85b66ab25c5fb16e63db4dd7111c183f96d1f18e19078051ed5fc74f7",
"meta": {
"source": {
"protocol": "file",
"column": 3,
"id": "/usr/share/logstash/pipeline/default.conf",
"line": 7
}
}
},
{
"explicit_id": false,
"id": "__QUEUE__",
"meta": null,
"type": "queue"
},
{
"config_name": "elasticsearch",
"explicit_id": false,
"id": "a339cb309b29181703c6adf321da3d639f5b60713de5a1e5519ebfea069556d8",
"meta": {
"source": {
"id": "/usr/share/logstash/pipeline/default.conf",
"line": 17,
"protocol": "file",
"column": 3
}
},
"plugin_type": "output",
"type": "plugin"
}
]
},
"type": "lir",
"version": "0.0.0",
"hash": "3000c3abf87d4dfa4a57aaf6af0a1f5bee2e0fc1c48a8e8636e2a33d7d2e91dd"
},
"batch_size": 125,
"workers": 12,
"id": "main"
}
}
}
},
"metricset": {
"name": "node"
},
"process": {
"pid": 93559
},
"service": {
"address": "127.0.0.1:9600",
"hostname": "Shaunaks-MBP-2.attlocal.net",
"id": "7565df20-c3aa-4261-81d5-3b0ab8d15c16",
"name": "logstash",
"type": "logstash",
"version": "7.0.0"
"ecs": {
"version": "1.5.0"
}
}
}
Loading

0 comments on commit 3dce95a

Please sign in to comment.