From 9a8e89e579d96e21a341818c3855f071f22492f4 Mon Sep 17 00:00:00 2001 From: Leandro Piccilli Date: Tue, 21 Aug 2018 17:05:13 +0200 Subject: [PATCH 1/4] Add Kibana input plugin --- plugins/inputs/all/all.go | 1 + plugins/inputs/kibana/README.md | 58 +++++++ plugins/inputs/kibana/kibana.go | 230 +++++++++++++++++++++++++ plugins/inputs/kibana/kibana_test.go | 66 +++++++ plugins/inputs/kibana/testdata_test.go | 199 +++++++++++++++++++++ 5 files changed, 554 insertions(+) create mode 100644 plugins/inputs/kibana/README.md create mode 100644 plugins/inputs/kibana/kibana.go create mode 100644 plugins/inputs/kibana/kibana_test.go create mode 100644 plugins/inputs/kibana/testdata_test.go diff --git a/plugins/inputs/all/all.go b/plugins/inputs/all/all.go index ac86fb87985e2..b735851959c46 100644 --- a/plugins/inputs/all/all.go +++ b/plugins/inputs/all/all.go @@ -56,6 +56,7 @@ import ( _ "github.com/influxdata/telegraf/plugins/inputs/kapacitor" _ "github.com/influxdata/telegraf/plugins/inputs/kernel" _ "github.com/influxdata/telegraf/plugins/inputs/kernel_vmstat" + _ "github.com/influxdata/telegraf/plugins/inputs/kibana" _ "github.com/influxdata/telegraf/plugins/inputs/kubernetes" _ "github.com/influxdata/telegraf/plugins/inputs/leofs" _ "github.com/influxdata/telegraf/plugins/inputs/linux_sysctl_fs" diff --git a/plugins/inputs/kibana/README.md b/plugins/inputs/kibana/README.md new file mode 100644 index 0000000000000..a1d0bdc445f24 --- /dev/null +++ b/plugins/inputs/kibana/README.md @@ -0,0 +1,58 @@ +# Kibana input plugin + +The [kibana](https://www.elastic.co/) plugin queries Kibana status API to +obtain the health status of Kibana and some useful metrics. + +### Configuration + +```toml +[[inputs.kibana]] + ## specify a list of one or more Kibana servers + # you can add username and password to your url to use basic authentication: + # servers = ["http://user:pass@localhost:5601"] + servers = ["http://localhost:5601"] + + ## Timeout for HTTP requests + http_timeout = "5s" + + ## Optional TLS Config + # tls_ca = "/etc/telegraf/ca.pem" + # tls_cert = "/etc/telegraf/cert.pem" + # tls_key = "/etc/telegraf/key.pem" + ## Use TLS but skip chain & host verification + # insecure_skip_verify = false +``` + +### Status mappings + +When reporting health (green/yellow/red), additional field `status_code` +is reported. Field contains mapping from status:string to status_code:int +with following rules: + +- `green` - 1 +- `yellow` - 2 +- `red` - 3 +- `unknown` - 0 + +### Measurements & Fields + +- kibana + - status: string (green, yellow, red) + - status_code: integer (1, 2, 3, 0) + - heap_max_bytes: integer + - heap_used_bytes: integer + - uptime_ms: integer + - response_time_avg_ms: integer + - response_time_max_ms: integer + - concurrent_connections: integer + +### Tags + +- name (Kibana reported name) +- uuid (Kibana reported UUID) +- version (Kibana version) +- server (Kibana server hostname or IP) + +### Example Output + +kibana,host=myhost,name=my-kibana,server=localhost:5601,uuid=00000000-0000-0000-0000-000000000000,version=6.3.2 concurrent_connections=0i,heap_max_bytes=136478720i,heap_used_bytes=119231088i,response_time_avg_ms=0i,response_time_max_ms=0i,status="green",status_code=1i,uptime_ms=2187428019i 1534864502000000000 diff --git a/plugins/inputs/kibana/kibana.go b/plugins/inputs/kibana/kibana.go new file mode 100644 index 0000000000000..59cf6a686a65e --- /dev/null +++ b/plugins/inputs/kibana/kibana.go @@ -0,0 +1,230 @@ +package kibana + +import ( + "encoding/json" + "fmt" + "net/http" + "regexp" + "strings" + "sync" + "time" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/internal" + "github.com/influxdata/telegraf/internal/tls" + "github.com/influxdata/telegraf/plugins/inputs" +) + +// mask for masking username/password from error messages +var mask = regexp.MustCompile(`https?:\/\/\S+:\S+@`) + +const statusPath = "/api/status" + +type kibanaStatus struct { + Name string `json:"name"` + UUID string `json:"uuid"` + Version version `json:"version"` + Status status `json:"status"` + Metrics metrics `json:"metrics"` +} + +type version struct { + Number string `json:"number"` + BuildHash string `json:"build_hash"` + BuildNumber int `json:"build_number"` + BuildSnapshot bool `json:"build_snapshot"` +} + +type status struct { + Overall overallStatus `json:"overall"` + Statuses interface{} `json:"statuses"` +} + +type overallStatus struct { + State string `json:"state"` +} + +type metrics struct { + UptimeInMillis int `json:"uptime_in_millis"` + ConcurrentConnections int `json:"concurrent_connections"` + ResponseTimes responseTimes `json:"response_times"` + Process process `json:"process"` +} + +type responseTimes struct { + AvgInMillis int `json:"avg_in_millis"` + MaxInMillis int `json:"max_in_millis"` +} + +type process struct { + Mem mem `json:"mem"` +} + +type mem struct { + HeapMaxInBytes int `json:"heap_max_in_bytes"` + HeapUsedInBytes int `json:"heap_used_in_bytes"` +} + +const sampleConfig = ` + ## specify a list of one or more Kibana servers + # you can add username and password to your url to use basic authentication: + # servers = ["http://user:pass@localhost:5601"] + servers = ["http://localhost:5601"] + + ## Timeout for HTTP requests + http_timeout = "5s" + + ## Optional TLS Config + # tls_ca = "/etc/telegraf/ca.pem" + # tls_cert = "/etc/telegraf/cert.pem" + # tls_key = "/etc/telegraf/key.pem" + ## Use TLS but skip chain & host verification + # insecure_skip_verify = false +` + +type Kibana struct { + Local bool + Servers []string + HttpTimeout internal.Duration + tls.ClientConfig + + client *http.Client + catMasterResponseTokens []string + isMaster bool +} + +func NewKibana() *Kibana { + return &Kibana{ + HttpTimeout: internal.Duration{Duration: time.Second * 5}, + } +} + +// perform status mapping +func mapHealthStatusToCode(s string) int { + switch strings.ToLower(s) { + case "green": + return 1 + case "yellow": + return 2 + case "red": + return 3 + } + return 0 +} + +// SampleConfig returns sample configuration for this plugin. +func (k *Kibana) SampleConfig() string { + return sampleConfig +} + +// Description returns the plugin description. +func (k *Kibana) Description() string { + return "Read status information from one or more Kibana servers" +} + +func (k *Kibana) Gather(acc telegraf.Accumulator) error { + if k.client == nil { + client, err := k.createHttpClient() + + if err != nil { + return err + } + k.client = client + } + + var wg sync.WaitGroup + wg.Add(len(k.Servers)) + + for _, serv := range k.Servers { + go func(baseUrl string, acc telegraf.Accumulator) { + defer wg.Done() + if err := k.gatherKibanaStatus(baseUrl, acc); err != nil { + acc.AddError(fmt.Errorf(mask.ReplaceAllString(err.Error(), "http(s)://XXX:XXX@"))) + return + } + }(serv, acc) + } + + wg.Wait() + return nil +} + +func (k *Kibana) createHttpClient() (*http.Client, error) { + tlsCfg, err := k.ClientConfig.TLSConfig() + if err != nil { + return nil, err + } + tr := &http.Transport{ + ResponseHeaderTimeout: k.HttpTimeout.Duration, + TLSClientConfig: tlsCfg, + } + client := &http.Client{ + Transport: tr, + Timeout: k.HttpTimeout.Duration, + } + + return client, nil +} + +func (k *Kibana) gatherKibanaStatus(baseUrl string, acc telegraf.Accumulator) error { + + kibanaStatus := &kibanaStatus{} + url := baseUrl + statusPath + + host, err := k.gatherJsonData(url, kibanaStatus) + if err != nil { + return err + } + + fields := make(map[string]interface{}) + tags := make(map[string]string) + + tags["name"] = kibanaStatus.Name + tags["uuid"] = kibanaStatus.UUID + tags["server"] = host + tags["version"] = kibanaStatus.Version.Number + + fields["status"] = kibanaStatus.Status.Overall.State + fields["status_code"] = mapHealthStatusToCode(kibanaStatus.Status.Overall.State) + + fields["uptime_ms"] = kibanaStatus.Metrics.UptimeInMillis + fields["concurrent_connections"] = kibanaStatus.Metrics.ConcurrentConnections + fields["heap_max_bytes"] = kibanaStatus.Metrics.Process.Mem.HeapMaxInBytes + fields["heap_used_bytes"] = kibanaStatus.Metrics.Process.Mem.HeapUsedInBytes + fields["response_time_avg_ms"] = kibanaStatus.Metrics.ResponseTimes.AvgInMillis + fields["response_time_max_ms"] = kibanaStatus.Metrics.ResponseTimes.MaxInMillis + + acc.AddFields("kibana", fields, tags) + + return nil +} + +func (k *Kibana) gatherJsonData(url string, v interface{}) (host string, err error) { + r, err := k.client.Get(url) + if err != nil { + return "", err + } + + host = r.Request.Host + + defer r.Body.Close() + if r.StatusCode != http.StatusOK { + // NOTE: we are not going to read/discard r.Body under the assumption we'd prefer + // to let the underlying transport close the connection and re-establish a new one for + // future calls. + return "", fmt.Errorf("kibana: API responded with status-code %d, expected %d", + r.StatusCode, http.StatusOK) + } + + if err = json.NewDecoder(r.Body).Decode(v); err != nil { + return host, err + } + + return host, nil +} + +func init() { + inputs.Add("kibana", func() telegraf.Input { + return NewKibana() + }) +} diff --git a/plugins/inputs/kibana/kibana_test.go b/plugins/inputs/kibana/kibana_test.go new file mode 100644 index 0000000000000..5c9e437a2d7e0 --- /dev/null +++ b/plugins/inputs/kibana/kibana_test.go @@ -0,0 +1,66 @@ +package kibana + +import ( + "io/ioutil" + "net/http" + "strings" + "testing" + + "github.com/influxdata/telegraf/testutil" +) + +func defaultTags() map[string]string { + return map[string]string{ + "name": "my-kibana", + "uuid": "00000000-0000-0000-0000-000000000000", + "server": "example.com:5601", + "version": "6.3.2", + } +} + +type transportMock struct { + statusCode int + body string +} + +func newTransportMock(statusCode int, body string) http.RoundTripper { + return &transportMock{ + statusCode: statusCode, + body: body, + } +} + +func (t *transportMock) RoundTrip(r *http.Request) (*http.Response, error) { + res := &http.Response{ + Header: make(http.Header), + Request: r, + StatusCode: t.statusCode, + } + res.Header.Set("Content-Type", "application/json") + res.Body = ioutil.NopCloser(strings.NewReader(t.body)) + return res, nil +} + +func checkKibanaStatusResult(t *testing.T, acc *testutil.Accumulator) { + tags := defaultTags() + acc.AssertContainsTaggedFields(t, "kibana", kibanaStatusExpected, tags) +} + +func TestGather(t *testing.T) { + ks := newKibanahWithClient() + ks.Servers = []string{"https://user:passwd@example.com:5601"} + ks.client.Transport = newTransportMock(http.StatusOK, kibanaStatusResponse) + + var acc testutil.Accumulator + if err := acc.GatherError(ks.Gather); err != nil { + t.Fatal(err) + } + + checkKibanaStatusResult(t, &acc) +} + +func newKibanahWithClient() *Kibana { + ks := NewKibana() + ks.client = &http.Client{} + return ks +} diff --git a/plugins/inputs/kibana/testdata_test.go b/plugins/inputs/kibana/testdata_test.go new file mode 100644 index 0000000000000..a9372027badfb --- /dev/null +++ b/plugins/inputs/kibana/testdata_test.go @@ -0,0 +1,199 @@ +package kibana + +const kibanaStatusResponse = ` +{ + "name": "my-kibana", + "uuid": "00000000-0000-0000-0000-000000000000", + "version": { + "number": "6.3.2", + "build_hash": "53d0c6758ac3fb38a3a1df198c1d4c87765e63f7", + "build_number": 17307, + "build_snapshot": false + }, + "status": { + "overall": { + "state": "green", + "title": "Green", + "nickname": "Looking good", + "icon": "success", + "since": "2018-07-27T07:37:42.567Z" + }, + "statuses": [{ + "id": "plugin:kibana@6.3.2", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2018-07-27T07:37:42.567Z" + }, + { + "id": "plugin:elasticsearch@6.3.2", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2018-07-28T10:07:04.920Z" + }, + { + "id": "plugin:xpack_main@6.3.2", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2018-07-28T10:07:02.393Z" + }, + { + "id": "plugin:searchprofiler@6.3.2", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2018-07-28T10:07:02.395Z" + }, + { + "id": "plugin:tilemap@6.3.2", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2018-07-28T10:07:02.396Z" + }, + { + "id": "plugin:watcher@6.3.2", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2018-07-28T10:07:02.397Z" + }, + { + "id": "plugin:license_management@6.3.2", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2018-07-27T07:37:42.668Z" + }, + { + "id": "plugin:index_management@6.3.2", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2018-07-28T10:07:02.399Z" + }, + { + "id": "plugin:timelion@6.3.2", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2018-07-27T07:37:42.912Z" + }, + { + "id": "plugin:logtrail@0.1.29", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2018-07-27T07:37:42.919Z" + }, + { + "id": "plugin:monitoring@6.3.2", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2018-07-27T07:37:42.922Z" + }, + { + "id": "plugin:grokdebugger@6.3.2", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2018-07-28T10:07:02.400Z" + }, + { + "id": "plugin:dashboard_mode@6.3.2", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2018-07-27T07:37:42.928Z" + }, + { + "id": "plugin:logstash@6.3.2", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2018-07-28T10:07:02.401Z" + }, + { + "id": "plugin:apm@6.3.2", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2018-07-27T07:37:42.950Z" + }, + { + "id": "plugin:console@6.3.2", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2018-07-27T07:37:42.958Z" + }, + { + "id": "plugin:console_extensions@6.3.2", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2018-07-27T07:37:42.961Z" + }, + { + "id": "plugin:metrics@6.3.2", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2018-07-27T07:37:42.965Z" + }, + { + "id": "plugin:reporting@6.3.2", + "state": "green", + "icon": "success", + "message": "Ready", + "since": "2018-07-28T10:07:02.402Z" + }] + }, + "metrics": { + "last_updated": "2018-08-21T11:24:25.823Z", + "collection_interval_in_millis": 5000, + "uptime_in_millis": 2173595336, + "process": { + "mem": { + "heap_max_in_bytes": 149954560, + "heap_used_in_bytes": 126274392 + } + }, + "os": { + "cpu": { + "load_average": { + "1m": 0.1806640625, + "5m": 0.49658203125, + "15m": 0.458984375 + } + } + }, + "response_times": { + "avg_in_millis": 12, + "max_in_millis": 123 + }, + "requests": { + "total": 2, + "disconnects": 0, + "status_codes": { + "200": 2 + } + }, + "concurrent_connections": 10 + } +} +` + +var kibanaStatusExpected = map[string]interface{}{ + "status": "green", + "status_code": 1, + "heap_max_bytes": 149954560, + "heap_used_bytes": 126274392, + "uptime_ms": 2173595336, + "response_time_avg_ms": 12, + "response_time_max_ms": 123, + "concurrent_connections": 10, +} From aca113d5bf2b3c84262a5ef4728d2a5f13e05619 Mon Sep 17 00:00:00 2001 From: Leandro Piccilli Date: Thu, 23 Aug 2018 00:34:21 +0200 Subject: [PATCH 2/4] Change metric fields from int to int64 --- plugins/inputs/kibana/kibana.go | 12 ++++++------ plugins/inputs/kibana/testdata_test.go | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/plugins/inputs/kibana/kibana.go b/plugins/inputs/kibana/kibana.go index 59cf6a686a65e..9a1cbc5902ff2 100644 --- a/plugins/inputs/kibana/kibana.go +++ b/plugins/inputs/kibana/kibana.go @@ -45,15 +45,15 @@ type overallStatus struct { } type metrics struct { - UptimeInMillis int `json:"uptime_in_millis"` - ConcurrentConnections int `json:"concurrent_connections"` + UptimeInMillis int64 `json:"uptime_in_millis"` + ConcurrentConnections int64 `json:"concurrent_connections"` ResponseTimes responseTimes `json:"response_times"` Process process `json:"process"` } type responseTimes struct { - AvgInMillis int `json:"avg_in_millis"` - MaxInMillis int `json:"max_in_millis"` + AvgInMillis int64 `json:"avg_in_millis"` + MaxInMillis int64 `json:"max_in_millis"` } type process struct { @@ -61,8 +61,8 @@ type process struct { } type mem struct { - HeapMaxInBytes int `json:"heap_max_in_bytes"` - HeapUsedInBytes int `json:"heap_used_in_bytes"` + HeapMaxInBytes int64 `json:"heap_max_in_bytes"` + HeapUsedInBytes int64 `json:"heap_used_in_bytes"` } const sampleConfig = ` diff --git a/plugins/inputs/kibana/testdata_test.go b/plugins/inputs/kibana/testdata_test.go index a9372027badfb..860aec9c7461e 100644 --- a/plugins/inputs/kibana/testdata_test.go +++ b/plugins/inputs/kibana/testdata_test.go @@ -190,10 +190,10 @@ const kibanaStatusResponse = ` var kibanaStatusExpected = map[string]interface{}{ "status": "green", "status_code": 1, - "heap_max_bytes": 149954560, - "heap_used_bytes": 126274392, - "uptime_ms": 2173595336, - "response_time_avg_ms": 12, - "response_time_max_ms": 123, - "concurrent_connections": 10, + "heap_max_bytes": int64(149954560), + "heap_used_bytes": int64(126274392), + "uptime_ms": int64(2173595336), + "response_time_avg_ms": int64(12), + "response_time_max_ms": int64(123), + "concurrent_connections": int64(10), } From 651bed4699c50804dfa9816da5709dcaddc7d334 Mon Sep 17 00:00:00 2001 From: Leandro Piccilli Date: Tue, 21 Aug 2018 17:05:13 +0200 Subject: [PATCH 3/4] Changes to http basic auth, cleanup & fixes --- plugins/inputs/kibana/README.md | 18 ++++--- plugins/inputs/kibana/kibana.go | 75 ++++++++++++-------------- plugins/inputs/kibana/kibana_test.go | 6 +-- plugins/inputs/kibana/testdata_test.go | 5 +- 4 files changed, 49 insertions(+), 55 deletions(-) diff --git a/plugins/inputs/kibana/README.md b/plugins/inputs/kibana/README.md index a1d0bdc445f24..6bcd9d2fcf8b7 100644 --- a/plugins/inputs/kibana/README.md +++ b/plugins/inputs/kibana/README.md @@ -7,13 +7,15 @@ obtain the health status of Kibana and some useful metrics. ```toml [[inputs.kibana]] - ## specify a list of one or more Kibana servers - # you can add username and password to your url to use basic authentication: - # servers = ["http://user:pass@localhost:5601"] + ## specify a list of one or more Kibana servers servers = ["http://localhost:5601"] ## Timeout for HTTP requests - http_timeout = "5s" + timeout = "5s" + + ## HTTP Basic Auth credentials + # username = "username" + # password = "pa$$word" ## Optional TLS Config # tls_ca = "/etc/telegraf/ca.pem" @@ -37,22 +39,22 @@ with following rules: ### Measurements & Fields - kibana - - status: string (green, yellow, red) - status_code: integer (1, 2, 3, 0) - heap_max_bytes: integer - heap_used_bytes: integer - uptime_ms: integer - - response_time_avg_ms: integer + - response_time_avg_ms: float - response_time_max_ms: integer - concurrent_connections: integer ### Tags +- status (Kibana health: green, yellow, red) - name (Kibana reported name) - uuid (Kibana reported UUID) - version (Kibana version) -- server (Kibana server hostname or IP) +- source (Kibana server hostname or IP) ### Example Output -kibana,host=myhost,name=my-kibana,server=localhost:5601,uuid=00000000-0000-0000-0000-000000000000,version=6.3.2 concurrent_connections=0i,heap_max_bytes=136478720i,heap_used_bytes=119231088i,response_time_avg_ms=0i,response_time_max_ms=0i,status="green",status_code=1i,uptime_ms=2187428019i 1534864502000000000 +kibana,host=myhost,name=my-kibana,source=localhost:5601,version=6.3.2 concurrent_connections=0i,heap_max_bytes=136478720i,heap_used_bytes=119231088i,response_time_avg_ms=0i,response_time_max_ms=0i,status="green",status_code=1i,uptime_ms=2187428019i 1534864502000000000 diff --git a/plugins/inputs/kibana/kibana.go b/plugins/inputs/kibana/kibana.go index 9a1cbc5902ff2..275928eeec511 100644 --- a/plugins/inputs/kibana/kibana.go +++ b/plugins/inputs/kibana/kibana.go @@ -4,7 +4,6 @@ import ( "encoding/json" "fmt" "net/http" - "regexp" "strings" "sync" "time" @@ -15,14 +14,10 @@ import ( "github.com/influxdata/telegraf/plugins/inputs" ) -// mask for masking username/password from error messages -var mask = regexp.MustCompile(`https?:\/\/\S+:\S+@`) - const statusPath = "/api/status" type kibanaStatus struct { Name string `json:"name"` - UUID string `json:"uuid"` Version version `json:"version"` Status status `json:"status"` Metrics metrics `json:"metrics"` @@ -52,8 +47,8 @@ type metrics struct { } type responseTimes struct { - AvgInMillis int64 `json:"avg_in_millis"` - MaxInMillis int64 `json:"max_in_millis"` + AvgInMillis float64 `json:"avg_in_millis"` + MaxInMillis int64 `json:"max_in_millis"` } type process struct { @@ -67,12 +62,14 @@ type mem struct { const sampleConfig = ` ## specify a list of one or more Kibana servers - # you can add username and password to your url to use basic authentication: - # servers = ["http://user:pass@localhost:5601"] servers = ["http://localhost:5601"] ## Timeout for HTTP requests - http_timeout = "5s" + timeout = "5s" + + ## HTTP Basic Auth credentials + # username = "username" + # password = "pa$$word" ## Optional TLS Config # tls_ca = "/etc/telegraf/ca.pem" @@ -83,19 +80,19 @@ const sampleConfig = ` ` type Kibana struct { - Local bool - Servers []string - HttpTimeout internal.Duration + Local bool + Servers []string + Username string + Password string + Timeout internal.Duration tls.ClientConfig - client *http.Client - catMasterResponseTokens []string - isMaster bool + client *http.Client } func NewKibana() *Kibana { return &Kibana{ - HttpTimeout: internal.Duration{Duration: time.Second * 5}, + Timeout: internal.Duration{Duration: time.Second * 5}, } } @@ -139,7 +136,7 @@ func (k *Kibana) Gather(acc telegraf.Accumulator) error { go func(baseUrl string, acc telegraf.Accumulator) { defer wg.Done() if err := k.gatherKibanaStatus(baseUrl, acc); err != nil { - acc.AddError(fmt.Errorf(mask.ReplaceAllString(err.Error(), "http(s)://XXX:XXX@"))) + acc.AddError(fmt.Errorf("[url=%s]: %s", baseUrl, err)) return } }(serv, acc) @@ -154,13 +151,12 @@ func (k *Kibana) createHttpClient() (*http.Client, error) { if err != nil { return nil, err } - tr := &http.Transport{ - ResponseHeaderTimeout: k.HttpTimeout.Duration, - TLSClientConfig: tlsCfg, - } + client := &http.Client{ - Transport: tr, - Timeout: k.HttpTimeout.Duration, + Transport: &http.Transport{ + TLSClientConfig: tlsCfg, + }, + Timeout: k.Timeout.Duration, } return client, nil @@ -180,11 +176,10 @@ func (k *Kibana) gatherKibanaStatus(baseUrl string, acc telegraf.Accumulator) er tags := make(map[string]string) tags["name"] = kibanaStatus.Name - tags["uuid"] = kibanaStatus.UUID - tags["server"] = host + tags["source"] = host tags["version"] = kibanaStatus.Version.Number + tags["status"] = kibanaStatus.Status.Overall.State - fields["status"] = kibanaStatus.Status.Overall.State fields["status_code"] = mapHealthStatusToCode(kibanaStatus.Status.Overall.State) fields["uptime_ms"] = kibanaStatus.Metrics.UptimeInMillis @@ -200,27 +195,25 @@ func (k *Kibana) gatherKibanaStatus(baseUrl string, acc telegraf.Accumulator) er } func (k *Kibana) gatherJsonData(url string, v interface{}) (host string, err error) { - r, err := k.client.Get(url) + + request, err := http.NewRequest("GET", url, nil) + + if (k.Username != "") || (k.Password != "") { + request.SetBasicAuth(k.Username, k.Password) + } + + response, err := k.client.Do(request) if err != nil { return "", err } - host = r.Request.Host - - defer r.Body.Close() - if r.StatusCode != http.StatusOK { - // NOTE: we are not going to read/discard r.Body under the assumption we'd prefer - // to let the underlying transport close the connection and re-establish a new one for - // future calls. - return "", fmt.Errorf("kibana: API responded with status-code %d, expected %d", - r.StatusCode, http.StatusOK) - } + defer response.Body.Close() - if err = json.NewDecoder(r.Body).Decode(v); err != nil { - return host, err + if err = json.NewDecoder(response.Body).Decode(v); err != nil { + return request.Host, err } - return host, nil + return request.Host, nil } func init() { diff --git a/plugins/inputs/kibana/kibana_test.go b/plugins/inputs/kibana/kibana_test.go index 5c9e437a2d7e0..ad5e32d290c34 100644 --- a/plugins/inputs/kibana/kibana_test.go +++ b/plugins/inputs/kibana/kibana_test.go @@ -12,9 +12,9 @@ import ( func defaultTags() map[string]string { return map[string]string{ "name": "my-kibana", - "uuid": "00000000-0000-0000-0000-000000000000", - "server": "example.com:5601", + "source": "example.com:5601", "version": "6.3.2", + "status": "green", } } @@ -48,7 +48,7 @@ func checkKibanaStatusResult(t *testing.T, acc *testutil.Accumulator) { func TestGather(t *testing.T) { ks := newKibanahWithClient() - ks.Servers = []string{"https://user:passwd@example.com:5601"} + ks.Servers = []string{"http://example.com:5601"} ks.client.Transport = newTransportMock(http.StatusOK, kibanaStatusResponse) var acc testutil.Accumulator diff --git a/plugins/inputs/kibana/testdata_test.go b/plugins/inputs/kibana/testdata_test.go index 860aec9c7461e..f4ecc9a583b90 100644 --- a/plugins/inputs/kibana/testdata_test.go +++ b/plugins/inputs/kibana/testdata_test.go @@ -172,7 +172,7 @@ const kibanaStatusResponse = ` } }, "response_times": { - "avg_in_millis": 12, + "avg_in_millis": 12.5, "max_in_millis": 123 }, "requests": { @@ -188,12 +188,11 @@ const kibanaStatusResponse = ` ` var kibanaStatusExpected = map[string]interface{}{ - "status": "green", "status_code": 1, "heap_max_bytes": int64(149954560), "heap_used_bytes": int64(126274392), "uptime_ms": int64(2173595336), - "response_time_avg_ms": int64(12), + "response_time_avg_ms": float64(12.5), "response_time_max_ms": int64(123), "concurrent_connections": int64(10), } From 862d7a2d40480ebb35d146cbcf64b58f4c069189 Mon Sep 17 00:00:00 2001 From: Leandro Piccilli Date: Fri, 24 Aug 2018 16:54:55 +0200 Subject: [PATCH 4/4] Add requests_per_second field --- plugins/inputs/kibana/README.md | 3 +++ plugins/inputs/kibana/kibana.go | 15 +++++++++++---- plugins/inputs/kibana/testdata_test.go | 1 + 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/plugins/inputs/kibana/README.md b/plugins/inputs/kibana/README.md index 6bcd9d2fcf8b7..7d885aed102a8 100644 --- a/plugins/inputs/kibana/README.md +++ b/plugins/inputs/kibana/README.md @@ -3,6 +3,8 @@ The [kibana](https://www.elastic.co/) plugin queries Kibana status API to obtain the health status of Kibana and some useful metrics. +This plugin has been tested and works on Kibana 6.x versions. + ### Configuration ```toml @@ -46,6 +48,7 @@ with following rules: - response_time_avg_ms: float - response_time_max_ms: integer - concurrent_connections: integer + - requests_per_sec: float ### Tags diff --git a/plugins/inputs/kibana/kibana.go b/plugins/inputs/kibana/kibana.go index 275928eeec511..0e21ad800d408 100644 --- a/plugins/inputs/kibana/kibana.go +++ b/plugins/inputs/kibana/kibana.go @@ -40,10 +40,12 @@ type overallStatus struct { } type metrics struct { - UptimeInMillis int64 `json:"uptime_in_millis"` - ConcurrentConnections int64 `json:"concurrent_connections"` - ResponseTimes responseTimes `json:"response_times"` - Process process `json:"process"` + UptimeInMillis int64 `json:"uptime_in_millis"` + ConcurrentConnections int64 `json:"concurrent_connections"` + CollectionIntervalInMilles int64 `json:"collection_interval_in_millis"` + ResponseTimes responseTimes `json:"response_times"` + Process process `json:"process"` + Requests requests `json:"requests"` } type responseTimes struct { @@ -55,6 +57,10 @@ type process struct { Mem mem `json:"mem"` } +type requests struct { + Total int64 `json:"total"` +} + type mem struct { HeapMaxInBytes int64 `json:"heap_max_in_bytes"` HeapUsedInBytes int64 `json:"heap_used_in_bytes"` @@ -188,6 +194,7 @@ func (k *Kibana) gatherKibanaStatus(baseUrl string, acc telegraf.Accumulator) er fields["heap_used_bytes"] = kibanaStatus.Metrics.Process.Mem.HeapUsedInBytes fields["response_time_avg_ms"] = kibanaStatus.Metrics.ResponseTimes.AvgInMillis fields["response_time_max_ms"] = kibanaStatus.Metrics.ResponseTimes.MaxInMillis + fields["requests_per_sec"] = float64(kibanaStatus.Metrics.Requests.Total) / float64(kibanaStatus.Metrics.CollectionIntervalInMilles) * 1000 acc.AddFields("kibana", fields, tags) diff --git a/plugins/inputs/kibana/testdata_test.go b/plugins/inputs/kibana/testdata_test.go index f4ecc9a583b90..ec393bb197ae9 100644 --- a/plugins/inputs/kibana/testdata_test.go +++ b/plugins/inputs/kibana/testdata_test.go @@ -195,4 +195,5 @@ var kibanaStatusExpected = map[string]interface{}{ "response_time_avg_ms": float64(12.5), "response_time_max_ms": int64(123), "concurrent_connections": int64(10), + "requests_per_sec": float64(0.4), }