Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add basic shard metricset to Elasticsearch #7006

Merged
merged 1 commit into from
May 14, 2018
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.asciidoc
Original file line number Diff line number Diff line change
@@ -232,6 +232,7 @@ https://github.com/elastic/beats/compare/v6.2.3...master[Check the HEAD diff]
- Add exchanges metricset to the RabbitMQ module {issue}6442[6442] {pull}6607[6607]
- Add Elasticsearch index_summary metricset. {pull}6918[6918]
- Add config option `management_path_prefix` for RabbitMQ module to configure management plugin path prefix {issue}6875[6875] {pull}7074[7074]
- Add shard metricset to Elasticsearch module. {pull}7006[7006]

*Packetbeat*

47 changes: 47 additions & 0 deletions metricbeat/docs/fields.asciidoc
Original file line number Diff line number Diff line change
@@ -3337,6 +3337,16 @@ type: keyword
Elasticsearch cluster id.


--

*`elasticsearch.cluster.state.id`*::
+
--
type: keyword

Elasticsearch state id.


--

[float]
@@ -3951,6 +3961,43 @@ format: bytes



--

[float]
== shard fields

shard fields



*`elasticsearch.shard.primary`*::
+
--
type: boolean

True if this is the primary shard.


--

*`elasticsearch.shard.number`*::
+
--
type: long

The number of this shard.


--

*`elasticsearch.shard.state`*::
+
--
type: keyword

The state of this shard.


--

[[exported-fields-etcd]]
13 changes: 9 additions & 4 deletions metricbeat/docs/modules/elasticsearch.asciidoc
Original file line number Diff line number Diff line change
@@ -23,10 +23,11 @@ in <<configuration-metricbeat>>. Here is an example configuration:
metricbeat.modules:
- module: elasticsearch
metricsets:
#- index
#- index_summary
- node
- node_stats
- node
- node_stats
#- index
#- index_summary
#- shard
period: 10s
hosts: ["localhost:9200"]
----
@@ -46,6 +47,8 @@ The following metricsets are available:

* <<metricbeat-metricset-elasticsearch-node_stats,node_stats>>

* <<metricbeat-metricset-elasticsearch-shard,shard>>

include::elasticsearch/index.asciidoc[]

include::elasticsearch/index_summary.asciidoc[]
@@ -54,3 +57,5 @@ include::elasticsearch/node.asciidoc[]

include::elasticsearch/node_stats.asciidoc[]

include::elasticsearch/shard.asciidoc[]

23 changes: 23 additions & 0 deletions metricbeat/docs/modules/elasticsearch/shard.asciidoc
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
////
This file is generated! See scripts/docs_collector.py
////

[[metricbeat-metricset-elasticsearch-shard]]
=== Elasticsearch shard metricset

experimental[]

include::../../../module/elasticsearch/shard/_meta/docs.asciidoc[]


==== Fields

For a description of each field in the metricset, see the
<<exported-fields-elasticsearch,exported fields>> section.

Here is an example document generated by this metricset:

[source,json]
----
include::../../../module/elasticsearch/shard/_meta/data.json[]
----
3 changes: 2 additions & 1 deletion metricbeat/docs/modules_list.asciidoc
Original file line number Diff line number Diff line change
@@ -33,10 +33,11 @@ This file is generated! See scripts/docs_collector.py
|<<metricbeat-module-dropwizard,Dropwizard>> beta[] |image:./images/icon-no.png[No prebuilt dashboards] |
.1+| .1+| |<<metricbeat-metricset-dropwizard-collector,collector>> beta[]
|<<metricbeat-module-elasticsearch,Elasticsearch>> beta[] |image:./images/icon-no.png[No prebuilt dashboards] |
.4+| .4+| |<<metricbeat-metricset-elasticsearch-index,index>> experimental[]
.5+| .5+| |<<metricbeat-metricset-elasticsearch-index,index>> experimental[]
|<<metricbeat-metricset-elasticsearch-index_summary,index_summary>> experimental[]
|<<metricbeat-metricset-elasticsearch-node,node>> beta[]
|<<metricbeat-metricset-elasticsearch-node_stats,node_stats>> beta[]
|<<metricbeat-metricset-elasticsearch-shard,shard>> experimental[]
|<<metricbeat-module-etcd,Etcd>> beta[] |image:./images/icon-no.png[No prebuilt dashboards] |
.3+| .3+| |<<metricbeat-metricset-etcd-leader,leader>> beta[]
|<<metricbeat-metricset-etcd-self,self>> beta[]
1 change: 1 addition & 0 deletions metricbeat/include/list.go
Original file line number Diff line number Diff line change
@@ -40,6 +40,7 @@ import (
_ "github.com/elastic/beats/metricbeat/module/elasticsearch/index_summary"
_ "github.com/elastic/beats/metricbeat/module/elasticsearch/node"
_ "github.com/elastic/beats/metricbeat/module/elasticsearch/node_stats"
_ "github.com/elastic/beats/metricbeat/module/elasticsearch/shard"
_ "github.com/elastic/beats/metricbeat/module/etcd"
_ "github.com/elastic/beats/metricbeat/module/etcd/leader"
_ "github.com/elastic/beats/metricbeat/module/etcd/self"
9 changes: 5 additions & 4 deletions metricbeat/metricbeat.reference.yml
Original file line number Diff line number Diff line change
@@ -196,10 +196,11 @@ metricbeat.modules:
#---------------------------- Elasticsearch Module ---------------------------
- module: elasticsearch
metricsets:
#- index
#- index_summary
- node
- node_stats
- node
- node_stats
#- index
#- index_summary
#- shard
period: 10s
hosts: ["localhost:9200"]

9 changes: 5 additions & 4 deletions metricbeat/module/elasticsearch/_meta/config.reference.yml
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
- module: elasticsearch
metricsets:
#- index
#- index_summary
- node
- node_stats
- node
- node_stats
#- index
#- index_summary
#- shard
period: 10s
hosts: ["localhost:9200"]
5 changes: 5 additions & 0 deletions metricbeat/module/elasticsearch/_meta/fields.yml
Original file line number Diff line number Diff line change
@@ -19,3 +19,8 @@
type: keyword
description: >
Elasticsearch cluster id.

- name: cluster.state.id
type: keyword
description: >
Elasticsearch state id.
Original file line number Diff line number Diff line change
@@ -17,13 +17,15 @@ import (
_ "github.com/elastic/beats/metricbeat/module/elasticsearch/index_summary"
_ "github.com/elastic/beats/metricbeat/module/elasticsearch/node"
_ "github.com/elastic/beats/metricbeat/module/elasticsearch/node_stats"
_ "github.com/elastic/beats/metricbeat/module/elasticsearch/shard"
)

var metricSets = []string{
"index",
"index_summary",
"node",
"node_stats",
"shard",
}

func TestFetch(t *testing.T) {
29 changes: 29 additions & 0 deletions metricbeat/module/elasticsearch/shard/_meta/data-xpack.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
{
"_index": ".monitoring-es-6-2018.04.29",
"_type": "doc",
"_id": "zCHJWMeqT1StL1M28ml_Vw:r4XD9O8eTrCHyN_GJswZ5A:heartbeat-6.0.0-rc1-2018.04.14:0:p",
"_score": 2.0136,
"_source": {
"cluster_uuid": "3zVAmPiRRNK6TYXeqCVbqg",
"timestamp": "2018-04-29T00:00:30.108Z",
"interval_ms": 10000,
"type": "shards",
"source_node": {
"uuid": "r4XD9O8eTrCHyN_GJswZ5A",
"host": "172.25.133.112",
"transport_address": "172.25.133.112:19608",
"ip": "172.25.133.112",
"name": "instance-0000000016",
"timestamp": "2018-04-29T00:00:30.073Z"
},
"state_uuid": "zCHJWMeqT1StL1M28ml_Vw",
"shard": {
"state": "STARTED",
"primary": true,
"node": "r4XD9O8eTrCHyN_GJswZ5A",
"relocating_node": null,
"shard": 0,
"index": "heartbeat-6.0.0-rc1-2018.04.14"
}
}
}
35 changes: 35 additions & 0 deletions metricbeat/module/elasticsearch/shard/_meta/data.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
{
"@timestamp": "2017-10-12T08:05:34.853Z",
"beat": {
"hostname": "host.example.com",
"name": "host.example.com"
},
"elasticsearch": {
"cluster": {
"name": "elasticsearch",
"state": {
"id": "dJaHX6fxSqSVMOsL4QZwwQ"
}
},
"index": {
"name": "filebeat-7.0.0-alpha1-2018.05.09"
},
"node": {
"name": "523zXyT6TRWiqXcQItnkyQ"
},
"shard": {
"primary": true,
"state": "STARTED"
}
},
"metricset": {
"host": "127.0.0.1:9200",
"module": "elasticsearch",
"name": "shard",
"namespace": "elasticsearch.shard",
"rtt": 115
},
"service": {
"name": "elasticsearch"
}
}
2 changes: 2 additions & 0 deletions metricbeat/module/elasticsearch/shard/_meta/docs.asciidoc
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
The `shard` metricset interrogates the
https://www.elastic.co/guide/en/elasticsearch/reference/6.2/cluster-state.html[Cluster State API endpoint] to fetch information about all shards.
18 changes: 18 additions & 0 deletions metricbeat/module/elasticsearch/shard/_meta/fields.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
- name: shard
type: group
description: >
shard fields
fields:
- name: primary
type: boolean
description: >
True if this is the primary shard.
- name: number
type: long
description: >
The number of this shard.

- name: state
type: keyword
description: >
The state of this shard.

Large diffs are not rendered by default.

63 changes: 63 additions & 0 deletions metricbeat/module/elasticsearch/shard/data.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
package shard

import (
"encoding/json"

"github.com/elastic/beats/libbeat/common"
s "github.com/elastic/beats/libbeat/common/schema"
c "github.com/elastic/beats/libbeat/common/schema/mapstriface"
"github.com/elastic/beats/metricbeat/mb"
)

var (
schema = s.Schema{
"state": c.Str("state"),
"primary": c.Bool("primary"),
"node": c.Str("node"),
"index": c.Str("index"),
"shard": c.Int("number"),
}
)

type stateStruct struct {
ClusterName string `json:"cluster_name"`
StateID string `json:"state_uuid"`
RoutingTable struct {
Indices map[string]struct {
Shards map[string][]map[string]interface{} `json:"shards"`
} `json:"indices"`
} `json:"routing_table"`
}

func eventsMapping(r mb.ReporterV2, content []byte) {
stateData := &stateStruct{}
err := json.Unmarshal(content, stateData)
if err != nil {
r.Error(err)
return
}

for _, index := range stateData.RoutingTable.Indices {
for _, shards := range index.Shards {
for _, shard := range shards {
event := mb.Event{}

fields, _ := schema.Apply(shard)

event.ModuleFields = common.MapStr{}
event.ModuleFields.Put("node.name", fields["node"])
delete(fields, "node")
event.ModuleFields.Put("index.name", fields["index"])
delete(fields, "index")
event.MetricSetFields = fields
event.ModuleFields.Put("cluster.state.id", stateData.StateID)
event.ModuleFields.Put("cluster.name", stateData.ClusterName)

event.RootFields = common.MapStr{}
event.RootFields.Put("service.name", "elasticsearch")

r.Event(event)
}
}
}
}
27 changes: 27 additions & 0 deletions metricbeat/module/elasticsearch/shard/data_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
package shard

import (
"io/ioutil"
"path/filepath"
"testing"

mbtest "github.com/elastic/beats/metricbeat/mb/testing"

"github.com/stretchr/testify/assert"
)

func TestStats(t *testing.T) {
files, err := filepath.Glob("./_meta/test/routing_table.*.json")
assert.NoError(t, err)

for _, f := range files {
input, err := ioutil.ReadFile(f)
assert.NoError(t, err)

reporter := &mbtest.CapturingReporterV2{}
eventsMapping(reporter, input)

assert.True(t, len(reporter.GetEvents()) >= 1)
assert.Equal(t, 0, len(reporter.GetErrors()))
}
}
56 changes: 56 additions & 0 deletions metricbeat/module/elasticsearch/shard/shard.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
package shard

import (
"github.com/elastic/beats/libbeat/common/cfgwarn"
"github.com/elastic/beats/metricbeat/helper"
"github.com/elastic/beats/metricbeat/mb"
"github.com/elastic/beats/metricbeat/mb/parse"
)

func init() {
mb.Registry.MustAddMetricSet("elasticsearch", "shard", New,
mb.WithHostParser(hostParser),
mb.DefaultMetricSet(),
mb.WithNamespace("elasticsearch.shard"),
)
}

var (
hostParser = parse.URLHostParserBuilder{
DefaultScheme: "http",
PathConfigKey: "path",
// Get the stats from the local node
DefaultPath: "_cluster/state/version,master_node,routing_table",
}.Build()
)

// MetricSet type defines all fields of the MetricSet
type MetricSet struct {
mb.BaseMetricSet
http *helper.HTTP
}

// New create a new instance of the MetricSet
func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
cfgwarn.Beta("The elasticsearch shard metricset is beta")

http, err := helper.NewHTTP(base)
if err != nil {
return nil, err
}
return &MetricSet{
BaseMetricSet: base,
http: http,
}, nil
}

// Fetch methods implements the data gathering and data conversion to the right format
func (m *MetricSet) Fetch(r mb.ReporterV2) {
content, err := m.http.FetchContent()
if err != nil {
r.Error(err)
return
}

eventsMapping(r, content)
}
1 change: 1 addition & 0 deletions metricbeat/module/elasticsearch/test_elasticsearch.py
Original file line number Diff line number Diff line change
@@ -20,6 +20,7 @@ class Test(metricbeat.BaseTest):
"index_summary",
"node_stats",
"node",
"shard"
])
@unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test")
def test_metricsets(self, metricset):