From a2abba45ff718f310ac46ccaee24d1c56ffc6c99 Mon Sep 17 00:00:00 2001 From: chrismark Date: Thu, 5 Nov 2020 17:15:11 +0200 Subject: [PATCH 01/18] Wip Signed-off-by: chrismark --- metricbeat/module/nats/_meta/run.sh | 6 +- .../_meta/test/connectionsmetrics.json | 30 ++++- .../module/nats/connections/connections.go | 2 +- metricbeat/module/nats/connections/data.go | 97 ++++++++++++++-- metricbeat/module/nats/docker-compose.yml | 15 +++ metricbeat/module/nats/stats/data.go | 80 +------------ metricbeat/module/nats/util/util.go | 108 ++++++++++++++++++ 7 files changed, 245 insertions(+), 93 deletions(-) create mode 100644 metricbeat/module/nats/util/util.go diff --git a/metricbeat/module/nats/_meta/run.sh b/metricbeat/module/nats/_meta/run.sh index e98cf3fb3fb..bb942fb9759 100755 --- a/metricbeat/module/nats/_meta/run.sh +++ b/metricbeat/module/nats/_meta/run.sh @@ -6,7 +6,11 @@ # NATS 2.X if [ -x /opt/nats/nats-server ]; then - exec /opt/nats/nats-server -c /opt/nats/nats-server.conf + if [[ -z "${ROUTES}" ]]; then + exec /opt/nats/nats-server --cluster nats://0.0.0.0:6222 --http_port 8222 --port 4222 + else + exec /opt/nats/nats-server --cluster nats://0.0.0.0:6222 --http_port 8222 --port 4222 --routes nats://nats:6222 + fi fi # NATS 1.X diff --git a/metricbeat/module/nats/connections/_meta/test/connectionsmetrics.json b/metricbeat/module/nats/connections/_meta/test/connectionsmetrics.json index 4dec7c8eaac..ba34ee5f66a 100644 --- a/metricbeat/module/nats/connections/_meta/test/connectionsmetrics.json +++ b/metricbeat/module/nats/connections/_meta/test/connectionsmetrics.json @@ -1,5 +1,29 @@ { - "server_id": "bUAdpRFtMWddIBWw80Yd9D", - "now": "2018-12-28T12:33:53.026865597Z", - "total": 10 + "server_id": "NCA52DU3MRRJGSDZXWBKSAGCKICQBYFLTQDHHXWAWYYCXBZ33LF55P7H", + "now": "2020-11-05T10:55:51.553046034Z", + "num_connections": 1, + "total": 1, + "offset": 0, + "limit": 1024, + "connections": [ + { + "cid": 3, + "ip": "172.22.0.1", + "port": 33794, + "start": "2020-11-05T10:55:21.968199135Z", + "last_activity": "2020-11-05T10:55:51.552942651Z", + "rtt": "31ms", + "uptime": "29s", + "idle": "0s", + "pending_bytes": 0, + "in_msgs": 42429204, + "out_msgs": 0, + "in_bytes": 678867264, + "out_bytes": 0, + "subscriptions": 0, + "name": "NATS Benchmark", + "lang": "go", + "version": "1.11.0" + } + ] } diff --git a/metricbeat/module/nats/connections/connections.go b/metricbeat/module/nats/connections/connections.go index 4a158180bf4..0acf74df46c 100644 --- a/metricbeat/module/nats/connections/connections.go +++ b/metricbeat/module/nats/connections/connections.go @@ -87,7 +87,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { if err != nil { return errors.Wrap(err, "error in fetch") } - err = eventMapping(r, content) + err = eventsMapping(r, content) if err != nil { return errors.Wrap(err, "error in mapping") } diff --git a/metricbeat/module/nats/connections/data.go b/metricbeat/module/nats/connections/data.go index 1efe616cb76..3924a2b61d6 100644 --- a/metricbeat/module/nats/connections/data.go +++ b/metricbeat/module/nats/connections/data.go @@ -20,6 +20,8 @@ package connections import ( "encoding/json" + "github.com/elastic/beats/v7/metricbeat/module/nats/util" + "github.com/elastic/beats/v7/metricbeat/mb" "github.com/pkg/errors" @@ -35,29 +37,102 @@ var ( "time": c.Str("now"), }, } - connectionsSchema = s.Schema{ + emptyConnectionsSchema = s.Schema{ "total": c.Int("total"), } + connectionsSchema = s.Schema{ + "total": c.Int("total"), + "name": c.Str("name"), + "subscriptions": c.Int("subscriptions"), + "in": s.Object{ + "messages": c.Int("in_msgs"), + "bytes": c.Int("in_bytes"), + }, + "out": s.Object{ + "messages": c.Int("out_msgs"), + "bytes": c.Int("out_bytes"), + }, + "pending_bytes": c.Int("pending_bytes"), + "uptime": c.Str("uptime"), + "idle_time": c.Str("idle"), + } ) -func eventMapping(r mb.ReporterV2, content []byte) error { - var event mb.Event - var inInterface map[string]interface{} +// Connections stores connections related information +type Connections struct { + ConnectionNum int `json:"total"` + Now string `json:"now"` + ServerID string `json:"server_id"` + Connections []map[string]interface{} `json:"connections,omitempty"` +} - err := json.Unmarshal(content, &inInterface) +// eventMapping maps a subscription to a Metricbeat event using subscriptionsSchema +// to parse through each subscription under a presumed channel +func eventMapping(content map[string]interface{}, fieldsSchema s.Schema) (mb.Event, error) { + fields, err := fieldsSchema.Apply(content) if err != nil { - return errors.Wrap(err, "failure parsing NATS connections API response") + return mb.Event{}, errors.Wrap(err, "error applying connection schema") } - event.MetricSetFields, err = connectionsSchema.Apply(inInterface) + + err = util.UpdateDuration(fields, "uptime") if err != nil { - return errors.Wrap(err, "failure applying connections schema") + return mb.Event{}, errors.Wrap(err, "failure updating uptime key") + } + err = util.UpdateDuration(fields, "idle_time") + if err != nil { + return mb.Event{}, errors.Wrap(err, "failure updating idle_time key") } - event.ModuleFields, err = moduleSchema.Apply(inInterface) + moduleFields, err := moduleSchema.Apply(content) if err != nil { - return errors.Wrap(err, "failure applying module schema") + return mb.Event{}, errors.Wrap(err, "error applying module schema") + } + + event := mb.Event{ + MetricSetFields: fields, + ModuleFields: moduleFields, + } + return event, nil +} + +// eventsMapping maps the top-level channel metrics AND also per-channel metrics AND subscriptions +func eventsMapping(r mb.ReporterV2, content []byte) error { + var err error + connections := Connections{} + if err = json.Unmarshal(content, &connections); err != nil { + return errors.Wrap(err, "failure parsing NATS connections API response") + } + + for _, con := range connections.Connections { + var evt mb.Event + con["server_id"] = connections.ServerID + con["now"] = connections.Now + con["total"] = connections.ConnectionNum + evt, err = eventMapping(con, connectionsSchema) + if err != nil { + r.Error(errors.Wrap(err, "error mapping connection event")) + continue + } + if !r.Event(evt) { + return nil + } + } + // emit one event even if there are no connections + if len(connections.Connections) == 0 { + var evt mb.Event + con := make(map[string]interface{}) + con["server_id"] = connections.ServerID + con["now"] = connections.Now + con["total"] = connections.ConnectionNum + evt, err = eventMapping(con, emptyConnectionsSchema) + if err != nil { + r.Error(errors.Wrap(err, "error mapping connection event")) + return nil + } + if !r.Event(evt) { + return nil + } } - r.Event(event) return nil } diff --git a/metricbeat/module/nats/docker-compose.yml b/metricbeat/module/nats/docker-compose.yml index 36cbd026fc4..fd24a4d28b1 100644 --- a/metricbeat/module/nats/docker-compose.yml +++ b/metricbeat/module/nats/docker-compose.yml @@ -10,3 +10,18 @@ services: NATS_VERSION: ${NATS_VERSION:-2.0.4} ports: - 8222 + - 4222 + - 6222 + nats-1: + image: docker.elastic.co/integrations-ci/beats-nats:${NATS_VERSION:-2.0.4}-1 + build: + context: ./_meta + dockerfile: Dockerfile + args: + NATS_VERSION: ${NATS_VERSION:-2.0.4} + environment: + - ROUTES=1 + ports: + - 8222 + - 4222 + - 6222 diff --git a/metricbeat/module/nats/stats/data.go b/metricbeat/module/nats/stats/data.go index 594276d35be..3e3ac3274f5 100644 --- a/metricbeat/module/nats/stats/data.go +++ b/metricbeat/module/nats/stats/data.go @@ -20,16 +20,13 @@ package stats import ( "encoding/json" - "github.com/elastic/beats/v7/metricbeat/mb" - "github.com/pkg/errors" - "strconv" - "strings" - "github.com/elastic/beats/v7/libbeat/common" s "github.com/elastic/beats/v7/libbeat/common/schema" c "github.com/elastic/beats/v7/libbeat/common/schema/mapstriface" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/module/nats/util" ) var ( @@ -68,69 +65,6 @@ var ( } ) -// Converts uptime from formatted string to seconds -// input: "1y20d22h3m30s", output: 33343410 -func convertUptime(uptime string) (seconds int64, err error) { - - var split []string - var years, days, hours, minutes, secs int64 - if strings.Contains(uptime, "y") { - split = strings.Split(uptime, "y") - uptime = split[1] - years, err = strconv.ParseInt(split[0], 10, 64) - if err != nil { - err = errors.Wrap(err, "invalid years format in json data") - return - } - seconds += years * 31536000 - } - - if strings.Contains(uptime, "d") { - split = strings.Split(uptime, "d") - uptime = split[1] - days, err = strconv.ParseInt(split[0], 10, 64) - if err != nil { - err = errors.Wrap(err, "invalid days format in json data") - return - } - seconds += days * 86400 - } - - if strings.Contains(uptime, "h") { - split = strings.Split(uptime, "h") - uptime = split[1] - hours, err = strconv.ParseInt(split[0], 10, 64) - if err != nil { - err = errors.Wrap(err, "invalid hours format in json data") - return - } - seconds += hours * 3600 - } - - if strings.Contains(uptime, "m") { - split = strings.Split(uptime, "m") - uptime = split[1] - minutes, err = strconv.ParseInt(split[0], 10, 64) - if err != nil { - err = errors.Wrap(err, "invalid minutes format in json data") - return - } - seconds += minutes * 60 - } - - if strings.Contains(uptime, "s") { - split = strings.Split(uptime, "s") - uptime = split[1] - secs, err = strconv.ParseInt(split[0], 10, 64) - if err != nil { - err = errors.Wrap(err, "invalid seconds format in json data") - return - } - seconds += secs - } - return -} - func eventMapping(r mb.ReporterV2, content []byte) error { var event common.MapStr var inInterface map[string]interface{} @@ -144,15 +78,7 @@ func eventMapping(r mb.ReporterV2, content []byte) error { return errors.Wrap(err, "failure applying stats schema") } - uptime, err := event.GetValue("uptime") - if err != nil { - return errors.Wrap(err, "failure retrieving uptime key") - } - uptime, err = convertUptime(uptime.(string)) - if err != nil { - return errors.Wrap(err, "failure converting uptime from string to integer") - } - _, err = event.Put("uptime", uptime) + err = util.UpdateDuration(event, "uptime") if err != nil { return errors.Wrap(err, "failure updating uptime key") } diff --git a/metricbeat/module/nats/util/util.go b/metricbeat/module/nats/util/util.go new file mode 100644 index 00000000000..8fb34ad2928 --- /dev/null +++ b/metricbeat/module/nats/util/util.go @@ -0,0 +1,108 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package util + +import ( + "fmt" + "strconv" + "strings" + + "github.com/pkg/errors" + + "github.com/elastic/beats/v7/libbeat/common" +) + +// convertUptimeToSeconds converts uptime from formatted string to seconds +// input: "1y20d22h3m30s", output: 33343410 +func convertUptimeToSeconds(uptime string) (seconds int64, err error) { + + var split []string + var years, days, hours, minutes, secs int64 + if strings.Contains(uptime, "y") { + split = strings.Split(uptime, "y") + uptime = split[1] + years, err = strconv.ParseInt(split[0], 10, 64) + if err != nil { + err = errors.Wrap(err, "invalid years format in json data") + return + } + seconds += years * 31536000 + } + + if strings.Contains(uptime, "d") { + split = strings.Split(uptime, "d") + uptime = split[1] + days, err = strconv.ParseInt(split[0], 10, 64) + if err != nil { + err = errors.Wrap(err, "invalid days format in json data") + return + } + seconds += days * 86400 + } + + if strings.Contains(uptime, "h") { + split = strings.Split(uptime, "h") + uptime = split[1] + hours, err = strconv.ParseInt(split[0], 10, 64) + if err != nil { + err = errors.Wrap(err, "invalid hours format in json data") + return + } + seconds += hours * 3600 + } + + if strings.Contains(uptime, "m") { + split = strings.Split(uptime, "m") + uptime = split[1] + minutes, err = strconv.ParseInt(split[0], 10, 64) + if err != nil { + err = errors.Wrap(err, "invalid minutes format in json data") + return + } + seconds += minutes * 60 + } + + if strings.Contains(uptime, "s") { + split = strings.Split(uptime, "s") + uptime = split[1] + secs, err = strconv.ParseInt(split[0], 10, 64) + if err != nil { + err = errors.Wrap(err, "invalid seconds format in json data") + return + } + seconds += secs + } + return +} + +// UpdateDuration updates a duration in a common.MapStr from formatted string to seconds +func UpdateDuration(event common.MapStr, key string) error { + item, err := event.GetValue(key) + if err != nil { + return nil + } + itemConverted, err := convertUptimeToSeconds(item.(string)) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failure converting %v key from string to integer", key)) + } + _, err = event.Put(key, itemConverted) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failure updating %v key", key)) + } + return nil +} From 47917ae2d3a0f65ab494b21434c9f3ea9cd0798d Mon Sep 17 00:00:00 2001 From: chrismark Date: Mon, 9 Nov 2020 15:23:00 +0200 Subject: [PATCH 02/18] Add nats cluster for tests Signed-off-by: chrismark --- metricbeat/module/nats/_meta/Dockerfile | 7 +++++++ metricbeat/module/nats/_meta/run.sh | 12 ++++++++++-- metricbeat/module/nats/test_nats.py | 5 ++++- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/metricbeat/module/nats/_meta/Dockerfile b/metricbeat/module/nats/_meta/Dockerfile index f846c968973..24757de5400 100644 --- a/metricbeat/module/nats/_meta/Dockerfile +++ b/metricbeat/module/nats/_meta/Dockerfile @@ -1,10 +1,17 @@ ARG NATS_VERSION=2.0.4 FROM nats:$NATS_VERSION +# build stage +FROM golang:1.13-alpine3.11 AS build-env +RUN apk --no-cache add build-base git mercurial gcc +RUN cd src && go get -d github.com/nats-io/nats.go/ +RUN cd src/github.com/nats-io/nats.go/examples/nats-bench && git checkout tags/v1.10.0 && go build . + # create an enhanced container with nc command available since nats is based # on scratch image making healthcheck impossible FROM alpine:latest COPY --from=0 / /opt/nats +COPY --from=build-env /go/src/github.com/nats-io/nats.go/examples/nats-bench/nats-bench /nats-bench COPY run.sh /run.sh # Expose client, management, and cluster ports EXPOSE 4222 8222 6222 diff --git a/metricbeat/module/nats/_meta/run.sh b/metricbeat/module/nats/_meta/run.sh index bb942fb9759..534d3fcea85 100755 --- a/metricbeat/module/nats/_meta/run.sh +++ b/metricbeat/module/nats/_meta/run.sh @@ -9,13 +9,21 @@ if [ -x /opt/nats/nats-server ]; then if [[ -z "${ROUTES}" ]]; then exec /opt/nats/nats-server --cluster nats://0.0.0.0:6222 --http_port 8222 --port 4222 else - exec /opt/nats/nats-server --cluster nats://0.0.0.0:6222 --http_port 8222 --port 4222 --routes nats://nats:6222 + (/opt/nats/nats-server --cluster nats://0.0.0.0:6222 --http_port 8222 --port 4222 --routes nats://nats:6222) & + sleep 2 + while true; do /nats-bench -np 1 -n 100000000 -ms 16 foo; done fi fi # NATS 1.X if [ -x /opt/nats/gnatsd ]; then - exec /opt/nats/gnatsd -c /opt/nats/gnatsd.conf + if [[ -z "${ROUTES}" ]]; then + exec /opt/nats/gnatsd --cluster nats://0.0.0.0:6222 --http_port 8222 --port 4222 + else + (/opt/nats/gnatsd --cluster nats://0.0.0.0:6222 --http_port 8222 --port 4222 --routes nats://nats:6222) & + sleep 2 + while true; do /nats-bench -np 1 -n 100000000 -ms 16 foo; done + fi fi echo "Couldn't find the nats server binary" diff --git a/metricbeat/module/nats/test_nats.py b/metricbeat/module/nats/test_nats.py index 406ee61d66d..d77a5947a41 100644 --- a/metricbeat/module/nats/test_nats.py +++ b/metricbeat/module/nats/test_nats.py @@ -8,7 +8,7 @@ @metricbeat.parameterized_with_supported_versions class TestNats(metricbeat.BaseTest): - COMPOSE_SERVICES = ['nats'] + COMPOSE_SERVICES = ['nats', 'nats-1'] @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") def test_stats(self): @@ -109,3 +109,6 @@ def test_subscriptions(self): self.assertCountEqual(self.de_dot(NATS_FIELDS), evt.keys(), evt) self.assert_fields_are_documented(evt) + + def get_hosts(self): + return [self.compose_host("nats")] From a6b144a2f805e802e1c0fc9d4ce597c9dceef76a Mon Sep 17 00:00:00 2001 From: chrismark Date: Mon, 9 Nov 2020 15:42:33 +0200 Subject: [PATCH 03/18] Add routes metrics Signed-off-by: chrismark --- metricbeat/module/nats/connections/data.go | 5 +- .../nats/routes/_meta/test/routesmetrics.json | 38 +++++++- metricbeat/module/nats/routes/data.go | 87 ++++++++++++++++--- metricbeat/module/nats/routes/routes.go | 2 +- 4 files changed, 113 insertions(+), 19 deletions(-) diff --git a/metricbeat/module/nats/connections/data.go b/metricbeat/module/nats/connections/data.go index 3924a2b61d6..1aad7207845 100644 --- a/metricbeat/module/nats/connections/data.go +++ b/metricbeat/module/nats/connections/data.go @@ -66,8 +66,7 @@ type Connections struct { Connections []map[string]interface{} `json:"connections,omitempty"` } -// eventMapping maps a subscription to a Metricbeat event using subscriptionsSchema -// to parse through each subscription under a presumed channel +// eventMapping maps a connection to a Metricbeat event using connectionsSchema func eventMapping(content map[string]interface{}, fieldsSchema s.Schema) (mb.Event, error) { fields, err := fieldsSchema.Apply(content) if err != nil { @@ -96,7 +95,7 @@ func eventMapping(content map[string]interface{}, fieldsSchema s.Schema) (mb.Eve return event, nil } -// eventsMapping maps the top-level channel metrics AND also per-channel metrics AND subscriptions +// eventsMapping maps the top-level connections metrics AND also per-connection metrics func eventsMapping(r mb.ReporterV2, content []byte) error { var err error connections := Connections{} diff --git a/metricbeat/module/nats/routes/_meta/test/routesmetrics.json b/metricbeat/module/nats/routes/_meta/test/routesmetrics.json index f850791bc53..1d9c685696f 100644 --- a/metricbeat/module/nats/routes/_meta/test/routesmetrics.json +++ b/metricbeat/module/nats/routes/_meta/test/routesmetrics.json @@ -1,5 +1,37 @@ { - "server_id": "bUAdpRFtMWddIBWw80Yd9D", - "now": "2018-12-28T12:33:53.026865597Z", - "num_routes": 10 + "server_id": "NDIFJNKCKCL2I2ICEXF64LAOAFMW2TN667B4NOWEKF77U7WFQR7CMF7M", + "now": "2020-11-09T09:34:10.027121937Z", + "num_routes": 2, + "routes": [ + { + "rid": 1, + "remote_id": "NCYX4P3ZCXZT3UF4BBZUG46S7EU224XXXLUTUBUDOIAUAIR5WJV73BV5", + "did_solicit": false, + "is_configured": false, + "ip": "172.25.255.2", + "port": 60736, + "pending_size": 0, + "rtt": "1.646926ms", + "in_msgs": 0, + "out_msgs": 0, + "in_bytes": 0, + "out_bytes": 0, + "subscriptions": 0 + }, + { + "rid": 2, + "remote_id": "NAVXZD4IXIH5QYJM33LQYXO3DZGXCUZ4MU72XH5GSELJB3WJWYIGND2D", + "did_solicit": false, + "is_configured": false, + "ip": "172.25.255.3", + "port": 42898, + "pending_size": 0, + "rtt": "511µs", + "in_msgs": 0, + "out_msgs": 0, + "in_bytes": 0, + "out_bytes": 0, + "subscriptions": 0 + } + ] } diff --git a/metricbeat/module/nats/routes/data.go b/metricbeat/module/nats/routes/data.go index 83581a16a31..e41e2dfa8d6 100644 --- a/metricbeat/module/nats/routes/data.go +++ b/metricbeat/module/nats/routes/data.go @@ -35,28 +35,91 @@ var ( "time": c.Str("now"), }, } - routesSchema = s.Schema{ + emptyRoutesSchema = s.Schema{ "total": c.Int("num_routes"), } + routesSchema = s.Schema{ + "total": c.Int("num_routes"), + "remote_id": c.Str("remote_id"), + "subscriptions": c.Int("subscriptions"), + "in": s.Object{ + "messages": c.Int("in_msgs"), + "bytes": c.Int("in_bytes"), + }, + "out": s.Object{ + "messages": c.Int("out_msgs"), + "bytes": c.Int("out_bytes"), + }, + "pending_size": c.Int("pending_size"), + "port": c.Int("port"), + "ip": c.Str("ip"), + } ) -func eventMapping(r mb.ReporterV2, content []byte) error { - var event mb.Event - var inInterface map[string]interface{} +// Routes stores routes related information +type Routes struct { + RoutesNum int `json:"num_routes"` + Now string `json:"now"` + ServerID string `json:"server_id"` + Routes []map[string]interface{} `json:"routes,omitempty"` +} - err := json.Unmarshal(content, &inInterface) +// eventMapping maps a route to a Metricbeat event using routesSchema +func eventMapping(content map[string]interface{}, fieldsSchema s.Schema) (mb.Event, error) { + fields, err := fieldsSchema.Apply(content) if err != nil { - return errors.Wrap(err, "failure parsing Nats routes API response") + return mb.Event{}, errors.Wrap(err, "error applying routes schema") } - event.MetricSetFields, err = routesSchema.Apply(inInterface) + + moduleFields, err := moduleSchema.Apply(content) if err != nil { - return errors.Wrap(err, "failure applying routes schema") + return mb.Event{}, errors.Wrap(err, "error applying module schema") } - event.ModuleFields, err = moduleSchema.Apply(inInterface) - if err != nil { - return errors.Wrap(err, "failure applying module schema") + event := mb.Event{ + MetricSetFields: fields, + ModuleFields: moduleFields, + } + return event, nil +} + +// eventsMapping maps the top-level routes metrics AND also per-route metrics +func eventsMapping(r mb.ReporterV2, content []byte) error { + var err error + connections := Routes{} + if err = json.Unmarshal(content, &connections); err != nil { + return errors.Wrap(err, "failure parsing NATS connections API response") + } + + for _, con := range connections.Routes { + var evt mb.Event + con["server_id"] = connections.ServerID + con["now"] = connections.Now + con["num_routes"] = connections.RoutesNum + evt, err = eventMapping(con, routesSchema) + if err != nil { + r.Error(errors.Wrap(err, "error mapping connection event")) + continue + } + if !r.Event(evt) { + return nil + } + } + // emit one event even if there are no connections + if len(connections.Routes) == 0 { + var evt mb.Event + con := make(map[string]interface{}) + con["server_id"] = connections.ServerID + con["now"] = connections.Now + con["num_routes"] = connections.RoutesNum + evt, err = eventMapping(con, emptyRoutesSchema) + if err != nil { + r.Error(errors.Wrap(err, "error mapping connection event")) + return nil + } + if !r.Event(evt) { + return nil + } } - r.Event(event) return nil } diff --git a/metricbeat/module/nats/routes/routes.go b/metricbeat/module/nats/routes/routes.go index 91b3ad33930..8ead24cbadd 100644 --- a/metricbeat/module/nats/routes/routes.go +++ b/metricbeat/module/nats/routes/routes.go @@ -87,7 +87,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { if err != nil { return errors.Wrap(err, "error in fetch") } - err = eventMapping(r, content) + err = eventsMapping(r, content) if err != nil { return errors.Wrap(err, "error in mapping") } From b36c68f8afce2a2792f617944628e349a0fac4ca Mon Sep 17 00:00:00 2001 From: chrismark Date: Mon, 9 Nov 2020 15:44:02 +0200 Subject: [PATCH 04/18] fix unit tests Signed-off-by: chrismark --- metricbeat/module/nats/connections/connections_test.go | 2 +- metricbeat/module/nats/routes/routes_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/metricbeat/module/nats/connections/connections_test.go b/metricbeat/module/nats/connections/connections_test.go index 8c32d6c9291..a604aceea45 100644 --- a/metricbeat/module/nats/connections/connections_test.go +++ b/metricbeat/module/nats/connections/connections_test.go @@ -33,7 +33,7 @@ func TestEventMapping(t *testing.T) { content, err := ioutil.ReadFile("./_meta/test/connectionsmetrics.json") assert.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} - err = eventMapping(reporter, content) + err = eventsMapping(reporter, content) assert.NoError(t, err) event := reporter.GetEvents()[0] d, _ := event.MetricSetFields.GetValue("total") diff --git a/metricbeat/module/nats/routes/routes_test.go b/metricbeat/module/nats/routes/routes_test.go index a76b35c5e1d..348decc1af9 100644 --- a/metricbeat/module/nats/routes/routes_test.go +++ b/metricbeat/module/nats/routes/routes_test.go @@ -33,7 +33,7 @@ func TestEventMapping(t *testing.T) { content, err := ioutil.ReadFile("./_meta/test/routesmetrics.json") assert.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} - err = eventMapping(reporter, content) + err = eventsMapping(reporter, content) assert.NoError(t, err) event := reporter.GetEvents()[0] d, _ := event.MetricSetFields.GetValue("total") From 8ae80bd2689d18f504a888ec15fcbc6286e1427b Mon Sep 17 00:00:00 2001 From: chrismark Date: Mon, 9 Nov 2020 15:58:05 +0200 Subject: [PATCH 05/18] Fix tests Signed-off-by: chrismark --- metricbeat/module/nats/connections/connections_test.go | 2 +- metricbeat/module/nats/routes/routes_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/metricbeat/module/nats/connections/connections_test.go b/metricbeat/module/nats/connections/connections_test.go index a604aceea45..3f602281af4 100644 --- a/metricbeat/module/nats/connections/connections_test.go +++ b/metricbeat/module/nats/connections/connections_test.go @@ -37,7 +37,7 @@ func TestEventMapping(t *testing.T) { assert.NoError(t, err) event := reporter.GetEvents()[0] d, _ := event.MetricSetFields.GetValue("total") - assert.Equal(t, d, int64(10)) + assert.Equal(t, d, int64(1)) } func TestFetchEventContent(t *testing.T) { diff --git a/metricbeat/module/nats/routes/routes_test.go b/metricbeat/module/nats/routes/routes_test.go index 348decc1af9..9ce261d5804 100644 --- a/metricbeat/module/nats/routes/routes_test.go +++ b/metricbeat/module/nats/routes/routes_test.go @@ -37,7 +37,7 @@ func TestEventMapping(t *testing.T) { assert.NoError(t, err) event := reporter.GetEvents()[0] d, _ := event.MetricSetFields.GetValue("total") - assert.Equal(t, d, int64(10)) + assert.Equal(t, d, int64(2)) } func TestFetchEventContent(t *testing.T) { From f7ffe598da22922aad3554902939bfa090dec3cf Mon Sep 17 00:00:00 2001 From: chrismark Date: Mon, 9 Nov 2020 16:10:43 +0200 Subject: [PATCH 06/18] Fix docker-compose file Signed-off-by: chrismark --- metricbeat/module/nats/docker-compose.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/metricbeat/module/nats/docker-compose.yml b/metricbeat/module/nats/docker-compose.yml index fd24a4d28b1..42c8fba1f9d 100644 --- a/metricbeat/module/nats/docker-compose.yml +++ b/metricbeat/module/nats/docker-compose.yml @@ -10,8 +10,6 @@ services: NATS_VERSION: ${NATS_VERSION:-2.0.4} ports: - 8222 - - 4222 - - 6222 nats-1: image: docker.elastic.co/integrations-ci/beats-nats:${NATS_VERSION:-2.0.4}-1 build: @@ -23,5 +21,3 @@ services: - ROUTES=1 ports: - 8222 - - 4222 - - 6222 From 38cf01410557db851350c01b3b4dae8e4c432492 Mon Sep 17 00:00:00 2001 From: chrismark Date: Mon, 9 Nov 2020 16:43:04 +0200 Subject: [PATCH 07/18] Tune tests & update data.json Signed-off-by: chrismark --- .../module/nats/connections/_meta/data.json | 19 ++++++++++++++++--- .../connections_integration_test.go | 7 ++----- metricbeat/module/nats/docker-compose.yml | 2 -- metricbeat/module/nats/routes/_meta/data.json | 17 +++++++++++++++-- .../nats/routes/routes_integration_test.go | 7 ++----- .../nats/stats/stats_integration_test.go | 7 ++----- .../subscriptions_integration_test.go | 7 ++----- 7 files changed, 39 insertions(+), 27 deletions(-) diff --git a/metricbeat/module/nats/connections/_meta/data.json b/metricbeat/module/nats/connections/_meta/data.json index 74374d49c35..c374955d613 100644 --- a/metricbeat/module/nats/connections/_meta/data.json +++ b/metricbeat/module/nats/connections/_meta/data.json @@ -10,13 +10,26 @@ "name":"connections", "rtt":44269 }, - "nats":{ + "nats": { "server": { "id": "bUAdpRFtMWddIBWw80Yd9D", "time": "2018-12-28T12:33:53.026865597Z" }, - "connections":{ - "total": 10 + "connections": { + "idle_time": 0, + "in": { + "bytes": 678867264, + "messages": 42429204 + }, + "name": "NATS Benchmark", + "out": { + "bytes": 0, + "messages": 0 + }, + "pending_bytes": 0, + "subscriptions": 0, + "total": 1, + "uptime": 29 } }, "type":"metricsets" diff --git a/metricbeat/module/nats/connections/connections_integration_test.go b/metricbeat/module/nats/connections/connections_integration_test.go index a9758e42e69..e9a51aa8deb 100644 --- a/metricbeat/module/nats/connections/connections_integration_test.go +++ b/metricbeat/module/nats/connections/connections_integration_test.go @@ -29,11 +29,8 @@ import ( func TestData(t *testing.T) { service := compose.EnsureUp(t, "nats") - metricSet := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host())) - err := mbtest.WriteEventsReporterV2Error(metricSet, t, "./test_data.json") - if err != nil { - t.Fatal("write", err) - } + m := mbtest.NewFetcher(t, getConfig(service.Host())) + m.WriteEvents(t, "") } func TestFetch(t *testing.T) { diff --git a/metricbeat/module/nats/docker-compose.yml b/metricbeat/module/nats/docker-compose.yml index 42c8fba1f9d..f43cf272009 100644 --- a/metricbeat/module/nats/docker-compose.yml +++ b/metricbeat/module/nats/docker-compose.yml @@ -19,5 +19,3 @@ services: NATS_VERSION: ${NATS_VERSION:-2.0.4} environment: - ROUTES=1 - ports: - - 8222 diff --git a/metricbeat/module/nats/routes/_meta/data.json b/metricbeat/module/nats/routes/_meta/data.json index 7b3fcb83d6e..2f091f72953 100644 --- a/metricbeat/module/nats/routes/_meta/data.json +++ b/metricbeat/module/nats/routes/_meta/data.json @@ -15,8 +15,21 @@ "id": "bUAdpRFtMWddIBWw80Yd9D", "time": "2018-12-28T12:33:53.026865597Z" }, - "routes":{ - "total": 10 + "routes": { + "in": { + "bytes": 0, + "messages": 0 + }, + "ip": "172.25.255.2", + "out": { + "bytes": 0, + "messages": 0 + }, + "pending_size": 0, + "port": 60736, + "remote_id": "NCYX4P3ZCXZT3UF4BBZUG46S7EU224XXXLUTUBUDOIAUAIR5WJV73BV5", + "subscriptions": 0, + "total": 2 } }, "type":"metricsets" diff --git a/metricbeat/module/nats/routes/routes_integration_test.go b/metricbeat/module/nats/routes/routes_integration_test.go index a10e4be938e..2ffb1a64f2a 100644 --- a/metricbeat/module/nats/routes/routes_integration_test.go +++ b/metricbeat/module/nats/routes/routes_integration_test.go @@ -29,11 +29,8 @@ import ( func TestData(t *testing.T) { service := compose.EnsureUp(t, "nats") - metricSet := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host())) - err := mbtest.WriteEventsReporterV2Error(metricSet, t, "./test_data.json") - if err != nil { - t.Fatal("write", err) - } + m := mbtest.NewFetcher(t, getConfig(service.Host())) + m.WriteEvents(t, "") } func TestFetch(t *testing.T) { diff --git a/metricbeat/module/nats/stats/stats_integration_test.go b/metricbeat/module/nats/stats/stats_integration_test.go index b95447e9c9d..56b50fc01c3 100644 --- a/metricbeat/module/nats/stats/stats_integration_test.go +++ b/metricbeat/module/nats/stats/stats_integration_test.go @@ -29,11 +29,8 @@ import ( func TestData(t *testing.T) { service := compose.EnsureUp(t, "nats") - metricSet := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host())) - err := mbtest.WriteEventsReporterV2Error(metricSet, t, "./test_data.json") - if err != nil { - t.Fatal("write", err) - } + m := mbtest.NewFetcher(t, getConfig(service.Host())) + m.WriteEvents(t, "") } func TestFetch(t *testing.T) { diff --git a/metricbeat/module/nats/subscriptions/subscriptions_integration_test.go b/metricbeat/module/nats/subscriptions/subscriptions_integration_test.go index 909537b3475..0d0b03f71a4 100644 --- a/metricbeat/module/nats/subscriptions/subscriptions_integration_test.go +++ b/metricbeat/module/nats/subscriptions/subscriptions_integration_test.go @@ -29,11 +29,8 @@ import ( func TestData(t *testing.T) { service := compose.EnsureUp(t, "nats") - metricSet := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host())) - err := mbtest.WriteEventsReporterV2Error(metricSet, t, "./test_data.json") - if err != nil { - t.Fatal("write", err) - } + m := mbtest.NewFetcher(t, getConfig(service.Host())) + m.WriteEvents(t, "") } func TestFetch(t *testing.T) { From c9a88cdcf069dd70faf4f8dfd0e50f3b104af6e2 Mon Sep 17 00:00:00 2001 From: chrismark Date: Mon, 9 Nov 2020 17:30:11 +0200 Subject: [PATCH 08/18] Update fields Signed-off-by: chrismark --- metricbeat/docs/fields.asciidoc | 222 ++++++++++++++++++ .../module/nats/connections/_meta/fields.yml | 51 ++++ metricbeat/module/nats/fields.go | 2 +- .../module/nats/routes/_meta/fields.yml | 48 ++++ 4 files changed, 322 insertions(+), 1 deletion(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 9809f27f0b5..25f73a0adcc 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -35491,6 +35491,120 @@ type: integer -- +*`nats.connections.name`*:: ++ +-- +The name of the connection + + +type: keyword + +-- + +*`nats.connections.subscriptions`*:: ++ +-- +The number of subscriptions in this connection + + +type: integer + +-- + +*`nats.connections.pending_bytes`*:: ++ +-- +The number of pending bytes of this connection + + +type: long + +format: bytes + +-- + +*`nats.connections.uptime`*:: ++ +-- +The period the connection is up (sec) + + +type: long + +format: duration + +-- + +*`nats.connections.idle_time`*:: ++ +-- +The period the connection is idle (sec) + + +type: long + +format: duration + +-- + +[float] +=== in + +The amount of incoming data + + + +*`nats.connections.in.messages`*:: ++ +-- +The amount of incoming messages + + +type: long + +-- + +*`nats.connections.in.bytes`*:: ++ +-- +The amount of incoming bytes + + +type: long + +format: bytes + +-- + +[float] +=== out + +The amount of outgoing data + + + +*`nats.connections.out.messages`*:: ++ +-- +The amount of outgoing messages + + +type: long + +-- + +*`nats.connections.out.bytes`*:: ++ +-- +The amount of outgoing bytes + + +type: long + +format: bytes + +-- + [float] === routes @@ -35508,6 +35622,114 @@ type: integer -- +*`nats.routes.subscriptions`*:: ++ +-- +The number of subscriptions in this connection + + +type: integer + +-- + +*`nats.routes.remote_id`*:: ++ +-- +The remote id on which the route is connected to + + +type: keyword + +-- + +*`nats.routes.pending_size`*:: ++ +-- +The number of pending routes + + +type: long + +-- + +*`nats.routes.port`*:: ++ +-- +The port of the route + + +type: integer + +-- + +*`nats.routes.ip`*:: ++ +-- +The ip of the route + + +type: ip + +-- + +[float] +=== in + +The amount of incoming data + + + +*`nats.routes.in.messages`*:: ++ +-- +The amount of incoming messages + + +type: long + +-- + +*`nats.routes.in.bytes`*:: ++ +-- +The amount of incoming bytes + + +type: long + +format: bytes + +-- + +[float] +=== out + +The amount of outgoing data + + + +*`nats.routes.out.messages`*:: ++ +-- +The amount of outgoing messages + + +type: long + +-- + +*`nats.routes.out.bytes`*:: ++ +-- +The amount of outgoing bytes + + +type: long + +format: bytes + +-- + [float] === stats diff --git a/metricbeat/module/nats/connections/_meta/fields.yml b/metricbeat/module/nats/connections/_meta/fields.yml index b8ae24cf652..61d5bb67207 100644 --- a/metricbeat/module/nats/connections/_meta/fields.yml +++ b/metricbeat/module/nats/connections/_meta/fields.yml @@ -8,3 +8,54 @@ type: integer description: > The number of currently active clients + - name: name + type: keyword + description: > + The name of the connection + - name: subscriptions + type: integer + description: > + The number of subscriptions in this connection + - name: pending_bytes + type: long + format: bytes + description: > + The number of pending bytes of this connection + - name: uptime + type: long + format: duration + description: > + The period the connection is up (sec) + - name: idle_time + type: long + format: duration + description: > + The period the connection is idle (sec) + - name: in + type: group + description: > + The amount of incoming data + fields: + - name: messages + type: long + description: > + The amount of incoming messages + - name: bytes + type: long + format: bytes + description: > + The amount of incoming bytes + - name: out + type: group + description: > + The amount of outgoing data + fields: + - name: messages + type: long + description: > + The amount of outgoing messages + - name: bytes + type: long + format: bytes + description: > + The amount of outgoing bytes diff --git a/metricbeat/module/nats/fields.go b/metricbeat/module/nats/fields.go index ee76b5c6fab..d66285bf14b 100644 --- a/metricbeat/module/nats/fields.go +++ b/metricbeat/module/nats/fields.go @@ -32,5 +32,5 @@ func init() { // AssetNats returns asset data. // This is the base64 encoded gzipped contents of module/nats. func AssetNats() string { - return "eJzUmM+S2zYMxu9+CkxO7SH7AD50ptNecmgO3fS8oSlY5kQkFAD0xnn6DiXLlmjJlhtlt6ujKeP76QP4B3wPX/CwhmBUVgDqtMI1vPv4+6fHdyuAAsWyq9VRWMNvKwBo3oS/qIgVrgAYKzSCayjNCmDrsCpk3bz3HoLxeIqcHj3U6U2mWB9/GYmfns/pT5/BUlDjgoCoUSfqrIDujMIzMgKjKWDL5OHjWaJP0KcQ5D3ygytOIx3OFzw8E/d/n4BKz6cdHkPBhz+nRNR5vJApjOI8jcc2fooCtAWPys6CZTTp7QtRSyGgTUNyIdq3+obqH53XTX7PQVOGjWJx5OhrDHPfPXkG+qxKaqrBSEfqgmKJnI1d4e2yEaLfICejbGTGoNUBjFW3R7CVw6By4RhTVFzQrCbe2/GJsXSiyFjkTpzqWI0u6M/e8KLuxDqbYWfGikKZDWyJvdE1FJGHE2i2dTWyowL0PPedQKzhF0H76yihR/+wOQyLbBbk2J9mEB5LP+kSHyCKKZu1Iy3jUDNZFBkFtcQTkAsUWkWls6ZqRRr7+jzAMQhk+Thx1XGUSqypsHjaVmR0wsIa2WLIR+8w0dZx4KBcdbCZqk/jizBcS/ldVjYq1aHdBbC4WNr6RIyepkpv4eVjRKmjcPlEG19EZsobTzFoknfBknehTBuqyUtgZNGAwbSUlNfcGriWphmAVyAnFTuksQl/k+faavFjvJcRO1CK+ZRaKKEUtaT/e0JPkG8koSfe6YRKRc9p3ZLokX/OqpUk4CTRO6BRaLaCUa6dal5PP1RpKV537Djtie0+fme1MX59yo9GtxFnYHaojF8jivb7nbFimcDso0Z2o+O3QGfCZsBQOFF2m9g0DBTAU3BKnArwn78/PE7EuPYhkJ/Xv0++dmNu3flZcFHFO6fpnHLE6H/btMvD9uz12RuKe9H3hl+fPEHcCy5xI69P3lDci85E+T778uQJYgq873GnsmCvOAj7dnrq443DlCn9Y7EgX+wgy2y3bWygGttOV8CFzNDKiU42Dfv5/eqdHUOK/d+4vFG7+0lc6jwKmFYj9fRbiqFIhzEwA77xFtXYHT6I+z5+E7FIpyWxUhDUxrHUQDeiV3B2Tp94eNMIL9I8J9EuWWAYYYNp5eA0cXGPx4vaW/hbEyjqgzfflvXUm2/ORw9t/Pb0V8DmMBfI7PNCO17qUtxUudlzup89silxiuffAAAA//+DHv/E" + return "eJzsmjtzGzcQx3t9ih1XSWF9ABaZ8SSNi7iIlJoGgSUP4wNwXiwo058+g3tQ98A9KJGSmeEVKoTj/n+3i8M+yI/wDQ8rsIL9HQBrznEFH758enz4cAeg0EvSBWtnV/DHHQCUd8LfToUc7wAIcxQeV7ATdwBbjbnyq/K+j2CFwaPlePGhiHeSC0X9n4T9eH2NH/oK0lkW2nrwLFh71tIDZ4LhCQmBUCjYkjPw5VmiTdCm8Eh7pHutjisNzjc8PDlq/38EKl6PGdam4PNfYyKsDQ5klGBcpvFQ2Y9WwG3BIJOWIAlFvHsgKp21KOOSH4i2XT2j+mfj6zK+z0ZjhAWjqjnaGt3YN1c/Am1WdizyzkpDqi3jDqm3NsHbRMMGs0GKjpKBCC3nBxCS9R5B5hot+yRJ/JsEGW6HpSCiihdn2HJfUtyHzdGcv5Q7OiKgLXCm/RxZgVZpu1tvDoxpstzZXW9h68gIXkHqQych1+qVocqX88ih6L1vi1hVIDEwuQi3QNJO9cIM2kMo4DeP8vckpVY5rn8R0MgyhdrXSp8lCymEccFyDKa20pkYXiVY9J82cWi0mQx6L3aD3TXpwQWAE5Cjig1SarfP8ky9Kq/jHVpsQF3gywTUBd65Xz2gR8grCeiRt2uxgSQXukKvTPSlvevJ8YQ77RkJ1dAT15BfCY1jXOt+ffHK2qMyC1qBs/CUaZmVx34V3GcqVMBuMu97/XNxhnpRap+IWuEofVC9OFjRYlOSlcLpfNc/BGvVF5yNulggd0uv13Ma39LrjIP+Z+nVs+AzZte9oLPm1nfsdeqpx1yfY9Dcv2XzWDf9UdfRAULcgzHOXz49PkBBTqJPv7bS0QjkGWqD3O20FHklUrqvzQMUrIeRCkEWIUnlpchRrbe5E/3zpnFhgSTR9ldPcKIsQseDftKDZaG3To+f4GyFQ6mSH6r5F6rJoU5VBl0squ3iM6F0S/C3BH9L8CfyvFWCb4P63D3Fc8sHg3SZUytKwFGiNZp2tkwFSa6MOd2KvHCnRXtN2XHMiVUeP3G3EX5f90ujecQFmHBsYL8H9Nz+pie1WUYw26iBdHJ9DnQhbA8YlPZMehPK2aqzYJzV7ChuwH//+fwwYmPqQaA/7fk5etvMu3XiY8FgF2eaY51SY7SfbdzL3S+m3p+9pDgVfS/o/ckjxKngPmz8+5OXFKeik3P9PPv25BFiDHxuuvjKXrE7T7yaiWz9Xev4yPW5LPZIgwxynnRb2QZXYNXplvPYrkNz7Xm0adgv71dP7Bii7ZdxGcEyuxAXa4MeRKURe/qtC1bFYgxEhy/dogqZ4f3oqPgsnZYPOYNHrifrWIlO4GSa1yQ4jXTJ5jmKNsECQQgbLAfd8cXFPdY/UZnD3wrrAt8b8eO8PjXihzbBQGW/qv4UbA5LgcS+v9Hqn7O4sMn7zl7S/eyRxA7HeP4LAAD//x6BNsM=" } diff --git a/metricbeat/module/nats/routes/_meta/fields.yml b/metricbeat/module/nats/routes/_meta/fields.yml index 3b9db2536b3..b20a46844ad 100644 --- a/metricbeat/module/nats/routes/_meta/fields.yml +++ b/metricbeat/module/nats/routes/_meta/fields.yml @@ -8,3 +8,51 @@ type: integer description: > The number of registered routes + - name: subscriptions + type: integer + description: > + The number of subscriptions in this connection + - name: remote_id + type: keyword + description: > + The remote id on which the route is connected to + - name: pending_size + type: long + description: > + The number of pending routes + - name: port + type: integer + description: > + The port of the route + - name: ip + type: ip + description: > + The ip of the route + - name: in + type: group + description: > + The amount of incoming data + fields: + - name: messages + type: long + description: > + The amount of incoming messages + - name: bytes + type: long + format: bytes + description: > + The amount of incoming bytes + - name: out + type: group + description: > + The amount of outgoing data + fields: + - name: messages + type: long + description: > + The amount of outgoing messages + - name: bytes + type: long + format: bytes + description: > + The amount of outgoing bytes From f15854ba697e233270de66c98e1c3e10ae59ddc8 Mon Sep 17 00:00:00 2001 From: chrismark Date: Wed, 11 Nov 2020 09:32:46 +0200 Subject: [PATCH 09/18] Add changelog Signed-off-by: chrismark --- CHANGELOG.next.asciidoc | 1 + metricbeat/module/nats/connections/data.go | 6 ++---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index ad0b9a61045..de874c2a1e2 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -790,6 +790,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Map cloud data filed `cloud.account.id` to azure subscription. {pull}21483[21483] {issue}21381[21381] - Move s3_daily_storage and s3_request metricsets to use cloudwatch input. {pull}21703[21703] - Duplicate system.process.cmdline field with process.command_line ECS field name. {pull}22325[22325] +- Add connection and route details for nats metricbeat module. {pull}22445[22445] *Packetbeat* diff --git a/metricbeat/module/nats/connections/data.go b/metricbeat/module/nats/connections/data.go index 1aad7207845..d95272324bd 100644 --- a/metricbeat/module/nats/connections/data.go +++ b/metricbeat/module/nats/connections/data.go @@ -20,14 +20,12 @@ package connections import ( "encoding/json" - "github.com/elastic/beats/v7/metricbeat/module/nats/util" - - "github.com/elastic/beats/v7/metricbeat/mb" - "github.com/pkg/errors" s "github.com/elastic/beats/v7/libbeat/common/schema" c "github.com/elastic/beats/v7/libbeat/common/schema/mapstriface" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/module/nats/util" ) var ( From 7dc47a70e84912593a7d837e9c462b459cb04e3e Mon Sep 17 00:00:00 2001 From: chrismark Date: Wed, 11 Nov 2020 11:35:13 +0200 Subject: [PATCH 10/18] Change images version Signed-off-by: chrismark --- metricbeat/module/nats/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metricbeat/module/nats/docker-compose.yml b/metricbeat/module/nats/docker-compose.yml index f43cf272009..d4a3248a8ce 100644 --- a/metricbeat/module/nats/docker-compose.yml +++ b/metricbeat/module/nats/docker-compose.yml @@ -2,7 +2,7 @@ version: '2.3' services: nats: - image: docker.elastic.co/integrations-ci/beats-nats:${NATS_VERSION:-2.0.4}-1 + image: docker.elastic.co/integrations-ci/beats-nats:${NATS_VERSION:-2.0.4}-2 build: context: ./_meta dockerfile: Dockerfile @@ -11,7 +11,7 @@ services: ports: - 8222 nats-1: - image: docker.elastic.co/integrations-ci/beats-nats:${NATS_VERSION:-2.0.4}-1 + image: docker.elastic.co/integrations-ci/beats-nats:${NATS_VERSION:-2.0.4}-2 build: context: ./_meta dockerfile: Dockerfile From e9f4dd1bfe01ec5c7ec343cbfcc42cdff0d14d7f Mon Sep 17 00:00:00 2001 From: chrismark Date: Thu, 12 Nov 2020 13:30:07 +0200 Subject: [PATCH 11/18] Add connection and route metricsets for detailed metrics Signed-off-by: chrismark --- CHANGELOG.next.asciidoc | 2 +- metricbeat/docs/fields.asciidoc | 80 ++++++++----- metricbeat/docs/modules/nats.asciidoc | 21 +++- .../docs/modules/nats/connection.asciidoc | 21 ++++ metricbeat/docs/modules/nats/route.asciidoc | 21 ++++ metricbeat/docs/modules_list.asciidoc | 4 +- metricbeat/include/list_common.go | 2 + metricbeat/metricbeat.reference.yml | 10 +- .../module/nats/_meta/config.reference.yml | 10 +- metricbeat/module/nats/_meta/config.yml | 10 +- metricbeat/module/nats/_meta/docs.asciidoc | 3 +- metricbeat/module/nats/_meta/run.sh | 12 +- .../module/nats/connection/_meta/data.json | 35 ++++++ .../nats/connection/_meta/docs.asciidoc | 1 + .../module/nats/connection/_meta/fields.yml | 57 +++++++++ .../_meta/test/connectionsmetrics.json | 29 +++++ .../module/nats/connection/connection.go | 94 +++++++++++++++ .../connection/connection_integration_test.go | 56 +++++++++ .../module/nats/connection/connection_test.go | 63 ++++++++++ metricbeat/module/nats/connection/data.go | 113 ++++++++++++++++++ .../module/nats/connections/_meta/data.json | 15 +-- .../nats/connections/_meta/docs.asciidoc | 2 +- .../module/nats/connections/_meta/fields.yml | 51 -------- .../module/nats/connections/connections.go | 2 +- .../nats/connections/connections_test.go | 2 +- metricbeat/module/nats/connections/data.go | 95 ++------------- metricbeat/module/nats/docker-compose.yml | 12 +- metricbeat/module/nats/fields.go | 2 +- metricbeat/module/nats/route/_meta/data.json | 35 ++++++ .../module/nats/route/_meta/docs.asciidoc | 1 + metricbeat/module/nats/route/_meta/fields.yml | 54 +++++++++ .../nats/route/_meta/test/routesmetrics.json | 37 ++++++ metricbeat/module/nats/route/data.go | 103 ++++++++++++++++ metricbeat/module/nats/route/route.go | 95 +++++++++++++++ .../nats/route/route_integration_test.go | 56 +++++++++ metricbeat/module/nats/route/route_test.go | 63 ++++++++++ metricbeat/module/nats/routes/_meta/data.json | 13 -- .../module/nats/routes/_meta/docs.asciidoc | 2 +- .../module/nats/routes/_meta/fields.yml | 48 -------- metricbeat/module/nats/routes/data.go | 87 ++------------ metricbeat/module/nats/routes/routes.go | 2 +- metricbeat/module/nats/routes/routes_test.go | 2 +- .../module/nats/stats/_meta/docs.asciidoc | 2 +- .../nats/subscriptions/_meta/docs.asciidoc | 2 +- metricbeat/module/nats/test_nats.py | 52 +++++++- metricbeat/modules.d/nats.yml.disabled | 10 +- 46 files changed, 1137 insertions(+), 352 deletions(-) create mode 100644 metricbeat/docs/modules/nats/connection.asciidoc create mode 100644 metricbeat/docs/modules/nats/route.asciidoc create mode 100644 metricbeat/module/nats/connection/_meta/data.json create mode 100644 metricbeat/module/nats/connection/_meta/docs.asciidoc create mode 100644 metricbeat/module/nats/connection/_meta/fields.yml create mode 100644 metricbeat/module/nats/connection/_meta/test/connectionsmetrics.json create mode 100644 metricbeat/module/nats/connection/connection.go create mode 100644 metricbeat/module/nats/connection/connection_integration_test.go create mode 100644 metricbeat/module/nats/connection/connection_test.go create mode 100644 metricbeat/module/nats/connection/data.go create mode 100644 metricbeat/module/nats/route/_meta/data.json create mode 100644 metricbeat/module/nats/route/_meta/docs.asciidoc create mode 100644 metricbeat/module/nats/route/_meta/fields.yml create mode 100644 metricbeat/module/nats/route/_meta/test/routesmetrics.json create mode 100644 metricbeat/module/nats/route/data.go create mode 100644 metricbeat/module/nats/route/route.go create mode 100644 metricbeat/module/nats/route/route_integration_test.go create mode 100644 metricbeat/module/nats/route/route_test.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index de874c2a1e2..b5623c702f4 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -790,7 +790,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Map cloud data filed `cloud.account.id` to azure subscription. {pull}21483[21483] {issue}21381[21381] - Move s3_daily_storage and s3_request metricsets to use cloudwatch input. {pull}21703[21703] - Duplicate system.process.cmdline field with process.command_line ECS field name. {pull}22325[22325] -- Add connection and route details for nats metricbeat module. {pull}22445[22445] +- Add connection and route metricsets for nats metricbeat module to collect metrics per connection/route. {pull}22445[22445] *Packetbeat* diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 25f73a0adcc..9138ee28c87 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -35475,23 +35475,13 @@ type: date -- [float] -=== connections +=== connection Contains nats connection related metrics -*`nats.connections.total`*:: -+ --- -The number of currently active clients - - -type: integer - --- - -*`nats.connections.name`*:: +*`nats.connection.name`*:: + -- The name of the connection @@ -35501,7 +35491,7 @@ type: keyword -- -*`nats.connections.subscriptions`*:: +*`nats.connection.subscriptions`*:: + -- The number of subscriptions in this connection @@ -35511,7 +35501,7 @@ type: integer -- -*`nats.connections.pending_bytes`*:: +*`nats.connection.pending_bytes`*:: + -- The number of pending bytes of this connection @@ -35523,7 +35513,7 @@ format: bytes -- -*`nats.connections.uptime`*:: +*`nats.connection.uptime`*:: + -- The period the connection is up (sec) @@ -35535,7 +35525,7 @@ format: duration -- -*`nats.connections.idle_time`*:: +*`nats.connection.idle_time`*:: + -- The period the connection is idle (sec) @@ -35554,7 +35544,7 @@ The amount of incoming data -*`nats.connections.in.messages`*:: +*`nats.connection.in.messages`*:: + -- The amount of incoming messages @@ -35564,7 +35554,7 @@ type: long -- -*`nats.connections.in.bytes`*:: +*`nats.connection.in.bytes`*:: + -- The amount of incoming bytes @@ -35583,7 +35573,7 @@ The amount of outgoing data -*`nats.connections.out.messages`*:: +*`nats.connection.out.messages`*:: + -- The amount of outgoing messages @@ -35593,7 +35583,7 @@ type: long -- -*`nats.connections.out.bytes`*:: +*`nats.connection.out.bytes`*:: + -- The amount of outgoing bytes @@ -35606,23 +35596,30 @@ format: bytes -- [float] -=== routes +=== connections -Contains nats route related metrics +Contains nats connection related metrics -*`nats.routes.total`*:: +*`nats.connections.total`*:: + -- -The number of registered routes +The number of currently active clients type: integer -- -*`nats.routes.subscriptions`*:: +[float] +=== route + +Contains nats route related metrics + + + +*`nats.route.subscriptions`*:: + -- The number of subscriptions in this connection @@ -35632,7 +35629,7 @@ type: integer -- -*`nats.routes.remote_id`*:: +*`nats.route.remote_id`*:: + -- The remote id on which the route is connected to @@ -35642,7 +35639,7 @@ type: keyword -- -*`nats.routes.pending_size`*:: +*`nats.route.pending_size`*:: + -- The number of pending routes @@ -35652,7 +35649,7 @@ type: long -- -*`nats.routes.port`*:: +*`nats.route.port`*:: + -- The port of the route @@ -35662,7 +35659,7 @@ type: integer -- -*`nats.routes.ip`*:: +*`nats.route.ip`*:: + -- The ip of the route @@ -35679,7 +35676,7 @@ The amount of incoming data -*`nats.routes.in.messages`*:: +*`nats.route.in.messages`*:: + -- The amount of incoming messages @@ -35689,7 +35686,7 @@ type: long -- -*`nats.routes.in.bytes`*:: +*`nats.route.in.bytes`*:: + -- The amount of incoming bytes @@ -35708,7 +35705,7 @@ The amount of outgoing data -*`nats.routes.out.messages`*:: +*`nats.route.out.messages`*:: + -- The amount of outgoing messages @@ -35718,7 +35715,7 @@ type: long -- -*`nats.routes.out.bytes`*:: +*`nats.route.out.bytes`*:: + -- The amount of outgoing bytes @@ -35730,6 +35727,23 @@ format: bytes -- +[float] +=== routes + +Contains nats route related metrics + + + +*`nats.routes.total`*:: ++ +-- +The number of registered routes + + +type: integer + +-- + [float] === stats diff --git a/metricbeat/docs/modules/nats.asciidoc b/metricbeat/docs/modules/nats.asciidoc index 8328da8dce7..158e20033cd 100644 --- a/metricbeat/docs/modules/nats.asciidoc +++ b/metricbeat/docs/modules/nats.asciidoc @@ -7,7 +7,8 @@ This file is generated! See scripts/mage/docs_collector.go This is the Nats module. The Nats module uses https://nats.io/documentation/managing_the_server/monitoring/[Nats monitoring server APIs] to collect metrics. -The default metricsets are `stats`, `connections`, `routes` and `subscriptions`. +The default metricsets are `stats`, `connections`, `routes` and `subscriptions` while `connection` and `route` +metricsets can be enabled to collect detailed metrics per connection/route. [float] === Compatibility @@ -33,13 +34,21 @@ in <>. Here is an example configuration: ---- metricbeat.modules: - module: nats - metricsets: ["connections", "routes", "stats", "subscriptions"] + metricsets: + - "connections" + - "routes" + - "stats" + - "subscriptions" + - "connection" + - "route" period: 10s hosts: ["localhost:8222"] #stats.metrics_path: "/varz" #connections.metrics_path: "/connz" #routes.metrics_path: "/routez" #subscriptions.metrics_path: "/subsz" + #connection.metrics_path: "/connz" + #route.metrics_path: "/routez" ---- [float] @@ -47,16 +56,24 @@ metricbeat.modules: The following metricsets are available: +* <> + * <> +* <> + * <> * <> * <> +include::nats/connection.asciidoc[] + include::nats/connections.asciidoc[] +include::nats/route.asciidoc[] + include::nats/routes.asciidoc[] include::nats/stats.asciidoc[] diff --git a/metricbeat/docs/modules/nats/connection.asciidoc b/metricbeat/docs/modules/nats/connection.asciidoc new file mode 100644 index 00000000000..8ee655538a7 --- /dev/null +++ b/metricbeat/docs/modules/nats/connection.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-nats-connection]] +=== NATS connection metricset + +include::../../../module/nats/connection/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/nats/connection/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/nats/route.asciidoc b/metricbeat/docs/modules/nats/route.asciidoc new file mode 100644 index 00000000000..af28e678281 --- /dev/null +++ b/metricbeat/docs/modules/nats/route.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-nats-route]] +=== NATS route metricset + +include::../../../module/nats/route/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/nats/route/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 777a317471b..68476e1acaa 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -205,7 +205,9 @@ This file is generated! See scripts/mage/docs_collector.go |<> beta[] |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.4+| .4+| |<> +.6+| .6+| |<> +|<> +|<> |<> |<> |<> diff --git a/metricbeat/include/list_common.go b/metricbeat/include/list_common.go index b77bb57f9a6..a98e7d2ed87 100644 --- a/metricbeat/include/list_common.go +++ b/metricbeat/include/list_common.go @@ -117,7 +117,9 @@ import ( _ "github.com/elastic/beats/v7/metricbeat/module/mysql/query" _ "github.com/elastic/beats/v7/metricbeat/module/mysql/status" _ "github.com/elastic/beats/v7/metricbeat/module/nats" + _ "github.com/elastic/beats/v7/metricbeat/module/nats/connection" _ "github.com/elastic/beats/v7/metricbeat/module/nats/connections" + _ "github.com/elastic/beats/v7/metricbeat/module/nats/route" _ "github.com/elastic/beats/v7/metricbeat/module/nats/routes" _ "github.com/elastic/beats/v7/metricbeat/module/nats/stats" _ "github.com/elastic/beats/v7/metricbeat/module/nats/subscriptions" diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 20d33337ec1..ca59aa1ecda 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -671,13 +671,21 @@ metricbeat.modules: #--------------------------------- NATS Module --------------------------------- - module: nats - metricsets: ["connections", "routes", "stats", "subscriptions"] + metricsets: + - "connections" + - "routes" + - "stats" + - "subscriptions" + - "connection" + - "route" period: 10s hosts: ["localhost:8222"] #stats.metrics_path: "/varz" #connections.metrics_path: "/connz" #routes.metrics_path: "/routez" #subscriptions.metrics_path: "/subsz" + #connection.metrics_path: "/connz" + #route.metrics_path: "/routez" #-------------------------------- Nginx Module -------------------------------- - module: nginx diff --git a/metricbeat/module/nats/_meta/config.reference.yml b/metricbeat/module/nats/_meta/config.reference.yml index d27d8f1ab2a..13b20ea7fd8 100644 --- a/metricbeat/module/nats/_meta/config.reference.yml +++ b/metricbeat/module/nats/_meta/config.reference.yml @@ -1,8 +1,16 @@ - module: nats - metricsets: ["connections", "routes", "stats", "subscriptions"] + metricsets: + - "connections" + - "routes" + - "stats" + - "subscriptions" + - "connection" + - "route" period: 10s hosts: ["localhost:8222"] #stats.metrics_path: "/varz" #connections.metrics_path: "/connz" #routes.metrics_path: "/routez" #subscriptions.metrics_path: "/subsz" + #connection.metrics_path: "/connz" + #route.metrics_path: "/routez" diff --git a/metricbeat/module/nats/_meta/config.yml b/metricbeat/module/nats/_meta/config.yml index d27d8f1ab2a..13b20ea7fd8 100644 --- a/metricbeat/module/nats/_meta/config.yml +++ b/metricbeat/module/nats/_meta/config.yml @@ -1,8 +1,16 @@ - module: nats - metricsets: ["connections", "routes", "stats", "subscriptions"] + metricsets: + - "connections" + - "routes" + - "stats" + - "subscriptions" + - "connection" + - "route" period: 10s hosts: ["localhost:8222"] #stats.metrics_path: "/varz" #connections.metrics_path: "/connz" #routes.metrics_path: "/routez" #subscriptions.metrics_path: "/subsz" + #connection.metrics_path: "/connz" + #route.metrics_path: "/routez" diff --git a/metricbeat/module/nats/_meta/docs.asciidoc b/metricbeat/module/nats/_meta/docs.asciidoc index 900539ff40d..01a30e14046 100644 --- a/metricbeat/module/nats/_meta/docs.asciidoc +++ b/metricbeat/module/nats/_meta/docs.asciidoc @@ -1,6 +1,7 @@ This is the Nats module. The Nats module uses https://nats.io/documentation/managing_the_server/monitoring/[Nats monitoring server APIs] to collect metrics. -The default metricsets are `stats`, `connections`, `routes` and `subscriptions`. +The default metricsets are `stats`, `connections`, `routes` and `subscriptions` while `connection` and `route` +metricsets can be enabled to collect detailed metrics per connection/route. [float] === Compatibility diff --git a/metricbeat/module/nats/_meta/run.sh b/metricbeat/module/nats/_meta/run.sh index 534d3fcea85..a832c5ac7d3 100755 --- a/metricbeat/module/nats/_meta/run.sh +++ b/metricbeat/module/nats/_meta/run.sh @@ -7,23 +7,21 @@ # NATS 2.X if [ -x /opt/nats/nats-server ]; then if [[ -z "${ROUTES}" ]]; then - exec /opt/nats/nats-server --cluster nats://0.0.0.0:6222 --http_port 8222 --port 4222 + (/opt/nats/nats-server --cluster nats://0.0.0.0:6222 --http_port 8222 --port 4222) & else (/opt/nats/nats-server --cluster nats://0.0.0.0:6222 --http_port 8222 --port 4222 --routes nats://nats:6222) & - sleep 2 - while true; do /nats-bench -np 1 -n 100000000 -ms 16 foo; done fi + while true; do /nats-bench -np 1 -n 100000000 -ms 16 foo; done fi # NATS 1.X if [ -x /opt/nats/gnatsd ]; then - if [[ -z "${ROUTES}" ]]; then - exec /opt/nats/gnatsd --cluster nats://0.0.0.0:6222 --http_port 8222 --port 4222 + if [[ -z "${ROUTES}" ]]; then + (/opt/nats/gnatsd --cluster nats://0.0.0.0:6222 --http_port 8222 --port 4222) & else (/opt/nats/gnatsd --cluster nats://0.0.0.0:6222 --http_port 8222 --port 4222 --routes nats://nats:6222) & - sleep 2 - while true; do /nats-bench -np 1 -n 100000000 -ms 16 foo; done fi + while true; do /nats-bench -np 1 -n 100000000 -ms 16 foo; done fi echo "Couldn't find the nats server binary" diff --git a/metricbeat/module/nats/connection/_meta/data.json b/metricbeat/module/nats/connection/_meta/data.json new file mode 100644 index 00000000000..7e8bb84b77d --- /dev/null +++ b/metricbeat/module/nats/connection/_meta/data.json @@ -0,0 +1,35 @@ +{ + "@timestamp":"2016-05-23T08:05:34.853Z", + "beat":{ + "hostname":"beathost", + "name":"beathost" + }, + "metricset":{ + "host":"localhost", + "module":"nats", + "name":"connections", + "rtt":44269 + }, + "nats": { + "server": { + "id": "bUAdpRFtMWddIBWw80Yd9D", + "time": "2018-12-28T12:33:53.026865597Z" + }, + "connection": { + "idle_time": 0, + "in": { + "bytes": 678867264, + "messages": 42429204 + }, + "name": "NATS Benchmark", + "out": { + "bytes": 0, + "messages": 0 + }, + "pending_bytes": 0, + "subscriptions": 0, + "uptime": 29 + } + }, + "type":"metricsets" +} diff --git a/metricbeat/module/nats/connection/_meta/docs.asciidoc b/metricbeat/module/nats/connection/_meta/docs.asciidoc new file mode 100644 index 00000000000..3945ea045a4 --- /dev/null +++ b/metricbeat/module/nats/connection/_meta/docs.asciidoc @@ -0,0 +1 @@ +This is the connections metricset of the module nats collecting metrics per connection. diff --git a/metricbeat/module/nats/connection/_meta/fields.yml b/metricbeat/module/nats/connection/_meta/fields.yml new file mode 100644 index 00000000000..6ccbab9ca0c --- /dev/null +++ b/metricbeat/module/nats/connection/_meta/fields.yml @@ -0,0 +1,57 @@ +- name: connection + type: group + description: > + Contains nats connection related metrics + release: ga + fields: + - name: name + type: keyword + description: > + The name of the connection + - name: subscriptions + type: integer + description: > + The number of subscriptions in this connection + - name: pending_bytes + type: long + format: bytes + description: > + The number of pending bytes of this connection + - name: uptime + type: long + format: duration + description: > + The period the connection is up (sec) + - name: idle_time + type: long + format: duration + description: > + The period the connection is idle (sec) + - name: in + type: group + description: > + The amount of incoming data + fields: + - name: messages + type: long + description: > + The amount of incoming messages + - name: bytes + type: long + format: bytes + description: > + The amount of incoming bytes + - name: out + type: group + description: > + The amount of outgoing data + fields: + - name: messages + type: long + description: > + The amount of outgoing messages + - name: bytes + type: long + format: bytes + description: > + The amount of outgoing bytes diff --git a/metricbeat/module/nats/connection/_meta/test/connectionsmetrics.json b/metricbeat/module/nats/connection/_meta/test/connectionsmetrics.json new file mode 100644 index 00000000000..ba34ee5f66a --- /dev/null +++ b/metricbeat/module/nats/connection/_meta/test/connectionsmetrics.json @@ -0,0 +1,29 @@ +{ + "server_id": "NCA52DU3MRRJGSDZXWBKSAGCKICQBYFLTQDHHXWAWYYCXBZ33LF55P7H", + "now": "2020-11-05T10:55:51.553046034Z", + "num_connections": 1, + "total": 1, + "offset": 0, + "limit": 1024, + "connections": [ + { + "cid": 3, + "ip": "172.22.0.1", + "port": 33794, + "start": "2020-11-05T10:55:21.968199135Z", + "last_activity": "2020-11-05T10:55:51.552942651Z", + "rtt": "31ms", + "uptime": "29s", + "idle": "0s", + "pending_bytes": 0, + "in_msgs": 42429204, + "out_msgs": 0, + "in_bytes": 678867264, + "out_bytes": 0, + "subscriptions": 0, + "name": "NATS Benchmark", + "lang": "go", + "version": "1.11.0" + } + ] +} diff --git a/metricbeat/module/nats/connection/connection.go b/metricbeat/module/nats/connection/connection.go new file mode 100644 index 00000000000..c960496ece4 --- /dev/null +++ b/metricbeat/module/nats/connection/connection.go @@ -0,0 +1,94 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package connection + +import ( + "github.com/pkg/errors" + + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/metricbeat/helper" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/mb/parse" +) + +const ( + defaultScheme = "http" + defaultPath = "/connz" +) + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: defaultScheme, + DefaultPath: defaultPath, + PathConfigKey: "connection.metrics_path", + }.Build() +) + +// init registers the MetricSet with the central registry as soon as the program +// starts. The New function will be called later to instantiate an instance of +// the MetricSet for each host defined in the module's configuration. After the +// MetricSet has been created then Fetch will begin to be called periodically. +func init() { + mb.Registry.MustAddMetricSet("nats", "connection", New, + mb.WithHostParser(hostParser), + ) +} + +// MetricSet holds any configuration or state information. It must implement +// the mb.MetricSet interface. And this is best achieved by embedding +// mb.BaseMetricSet because it implements all of the required mb.MetricSet +// interface methods except for Fetch. +type MetricSet struct { + mb.BaseMetricSet + http *helper.HTTP + Log *logp.Logger +} + +// New creates a new instance of the MetricSet. New is responsible for unpacking +// any MetricSet specific configuration options if there are any. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + config := struct{}{} + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + + http, err := helper.NewHTTP(base) + if err != nil { + return nil, err + } + return &MetricSet{ + base, + http, + logp.NewLogger("nats"), + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(r mb.ReporterV2) error { + content, err := m.http.FetchContent() + if err != nil { + return errors.Wrap(err, "error in fetch") + } + err = eventsMapping(r, content) + if err != nil { + return errors.Wrap(err, "error in mapping") + } + return nil +} diff --git a/metricbeat/module/nats/connection/connection_integration_test.go b/metricbeat/module/nats/connection/connection_integration_test.go new file mode 100644 index 00000000000..f6acb058ffc --- /dev/null +++ b/metricbeat/module/nats/connection/connection_integration_test.go @@ -0,0 +1,56 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration + +package connection + +import ( + "testing" + + "github.com/elastic/beats/v7/libbeat/tests/compose" + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" +) + +func TestData(t *testing.T) { + service := compose.EnsureUp(t, "nats") + compose.EnsureUp(t, "nats-routes") + + m := mbtest.NewFetcher(t, getConfig(service.Host())) + m.WriteEvents(t, "") +} + +func TestFetch(t *testing.T) { + service := compose.EnsureUp(t, "nats") + compose.EnsureUp(t, "nats-routes") + + reporter := &mbtest.CapturingReporterV2{} + + metricSet := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host())) + metricSet.Fetch(reporter) + + e := mbtest.StandardizeEvent(metricSet, reporter.GetEvents()[0]) + t.Logf("%s/%s event: %+v", metricSet.Module().Name(), metricSet.Name(), e.Fields.StringToPrint()) +} + +func getConfig(host string) map[string]interface{} { + return map[string]interface{}{ + "module": "nats", + "metricsets": []string{"connection"}, + "hosts": []string{host}, + } +} diff --git a/metricbeat/module/nats/connection/connection_test.go b/metricbeat/module/nats/connection/connection_test.go new file mode 100644 index 00000000000..d6b95960aef --- /dev/null +++ b/metricbeat/module/nats/connection/connection_test.go @@ -0,0 +1,63 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package connection + +import ( + "io/ioutil" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" +) + +func TestEventMapping(t *testing.T) { + content, err := ioutil.ReadFile("./_meta/test/connectionsmetrics.json") + assert.NoError(t, err) + reporter := &mbtest.CapturingReporterV2{} + err = eventsMapping(reporter, content) + assert.NoError(t, err) +} + +func TestFetchEventContent(t *testing.T) { + absPath, _ := filepath.Abs("./_meta/test") + + response, _ := ioutil.ReadFile(absPath + "/connectionsmetrics.json") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Header().Set("Content-Type", "application/json;") + w.Write([]byte(response)) + })) + defer server.Close() + + config := map[string]interface{}{ + "module": "nats", + "metricsets": []string{"connection"}, + "hosts": []string{server.URL}, + } + reporter := &mbtest.CapturingReporterV2{} + + metricSet := mbtest.NewReportingMetricSetV2Error(t, config) + metricSet.Fetch(reporter) + + e := mbtest.StandardizeEvent(metricSet, reporter.GetEvents()[0]) + t.Logf("%s/%s event: %+v", metricSet.Module().Name(), metricSet.Name(), e.Fields.StringToPrint()) +} diff --git a/metricbeat/module/nats/connection/data.go b/metricbeat/module/nats/connection/data.go new file mode 100644 index 00000000000..a895d494ad6 --- /dev/null +++ b/metricbeat/module/nats/connection/data.go @@ -0,0 +1,113 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package connection + +import ( + "encoding/json" + + "github.com/pkg/errors" + + s "github.com/elastic/beats/v7/libbeat/common/schema" + c "github.com/elastic/beats/v7/libbeat/common/schema/mapstriface" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/module/nats/util" +) + +var ( + moduleSchema = s.Schema{ + "server": s.Object{ + "id": c.Str("server_id"), + "time": c.Str("now"), + }, + } + connectionsSchema = s.Schema{ + "name": c.Str("name"), + "subscriptions": c.Int("subscriptions"), + "in": s.Object{ + "messages": c.Int("in_msgs"), + "bytes": c.Int("in_bytes"), + }, + "out": s.Object{ + "messages": c.Int("out_msgs"), + "bytes": c.Int("out_bytes"), + }, + "pending_bytes": c.Int("pending_bytes"), + "uptime": c.Str("uptime"), + "idle_time": c.Str("idle"), + } +) + +// Connections stores connections related information +type Connections struct { + Now string `json:"now"` + ServerID string `json:"server_id"` + Connections []map[string]interface{} `json:"connections,omitempty"` +} + +// eventMapping maps a connection to a Metricbeat event using connectionsSchema +func eventMapping(content map[string]interface{}, fieldsSchema s.Schema) (mb.Event, error) { + fields, err := fieldsSchema.Apply(content) + if err != nil { + return mb.Event{}, errors.Wrap(err, "error applying connection schema") + } + + err = util.UpdateDuration(fields, "uptime") + if err != nil { + return mb.Event{}, errors.Wrap(err, "failure updating uptime key") + } + + err = util.UpdateDuration(fields, "idle_time") + if err != nil { + return mb.Event{}, errors.Wrap(err, "failure updating idle_time key") + } + + moduleFields, err := moduleSchema.Apply(content) + if err != nil { + return mb.Event{}, errors.Wrap(err, "error applying module schema") + } + + event := mb.Event{ + MetricSetFields: fields, + ModuleFields: moduleFields, + } + return event, nil +} + +// eventsMapping maps per-connection metrics +func eventsMapping(r mb.ReporterV2, content []byte) error { + var err error + connections := Connections{} + if err = json.Unmarshal(content, &connections); err != nil { + return errors.Wrap(err, "failure parsing NATS connections API response") + } + + for _, con := range connections.Connections { + var evt mb.Event + con["server_id"] = connections.ServerID + con["now"] = connections.Now + evt, err = eventMapping(con, connectionsSchema) + if err != nil { + r.Error(errors.Wrap(err, "error mapping connection event")) + continue + } + if !r.Event(evt) { + return nil + } + } + return nil +} diff --git a/metricbeat/module/nats/connections/_meta/data.json b/metricbeat/module/nats/connections/_meta/data.json index c374955d613..932a40be6e4 100644 --- a/metricbeat/module/nats/connections/_meta/data.json +++ b/metricbeat/module/nats/connections/_meta/data.json @@ -16,20 +16,7 @@ "time": "2018-12-28T12:33:53.026865597Z" }, "connections": { - "idle_time": 0, - "in": { - "bytes": 678867264, - "messages": 42429204 - }, - "name": "NATS Benchmark", - "out": { - "bytes": 0, - "messages": 0 - }, - "pending_bytes": 0, - "subscriptions": 0, - "total": 1, - "uptime": 29 + "total": 1 } }, "type":"metricsets" diff --git a/metricbeat/module/nats/connections/_meta/docs.asciidoc b/metricbeat/module/nats/connections/_meta/docs.asciidoc index 1d48f9d5719..03d16c0c569 100644 --- a/metricbeat/module/nats/connections/_meta/docs.asciidoc +++ b/metricbeat/module/nats/connections/_meta/docs.asciidoc @@ -1 +1 @@ -This is the connections metricset of the module nats. +This is the connections metricset of the module nats collecting top level metrics about connections. diff --git a/metricbeat/module/nats/connections/_meta/fields.yml b/metricbeat/module/nats/connections/_meta/fields.yml index 61d5bb67207..b8ae24cf652 100644 --- a/metricbeat/module/nats/connections/_meta/fields.yml +++ b/metricbeat/module/nats/connections/_meta/fields.yml @@ -8,54 +8,3 @@ type: integer description: > The number of currently active clients - - name: name - type: keyword - description: > - The name of the connection - - name: subscriptions - type: integer - description: > - The number of subscriptions in this connection - - name: pending_bytes - type: long - format: bytes - description: > - The number of pending bytes of this connection - - name: uptime - type: long - format: duration - description: > - The period the connection is up (sec) - - name: idle_time - type: long - format: duration - description: > - The period the connection is idle (sec) - - name: in - type: group - description: > - The amount of incoming data - fields: - - name: messages - type: long - description: > - The amount of incoming messages - - name: bytes - type: long - format: bytes - description: > - The amount of incoming bytes - - name: out - type: group - description: > - The amount of outgoing data - fields: - - name: messages - type: long - description: > - The amount of outgoing messages - - name: bytes - type: long - format: bytes - description: > - The amount of outgoing bytes diff --git a/metricbeat/module/nats/connections/connections.go b/metricbeat/module/nats/connections/connections.go index 0acf74df46c..4a158180bf4 100644 --- a/metricbeat/module/nats/connections/connections.go +++ b/metricbeat/module/nats/connections/connections.go @@ -87,7 +87,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { if err != nil { return errors.Wrap(err, "error in fetch") } - err = eventsMapping(r, content) + err = eventMapping(r, content) if err != nil { return errors.Wrap(err, "error in mapping") } diff --git a/metricbeat/module/nats/connections/connections_test.go b/metricbeat/module/nats/connections/connections_test.go index 3f602281af4..e55bf173071 100644 --- a/metricbeat/module/nats/connections/connections_test.go +++ b/metricbeat/module/nats/connections/connections_test.go @@ -33,7 +33,7 @@ func TestEventMapping(t *testing.T) { content, err := ioutil.ReadFile("./_meta/test/connectionsmetrics.json") assert.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} - err = eventsMapping(reporter, content) + err = eventMapping(reporter, content) assert.NoError(t, err) event := reporter.GetEvents()[0] d, _ := event.MetricSetFields.GetValue("total") diff --git a/metricbeat/module/nats/connections/data.go b/metricbeat/module/nats/connections/data.go index d95272324bd..f08f7b73ec3 100644 --- a/metricbeat/module/nats/connections/data.go +++ b/metricbeat/module/nats/connections/data.go @@ -25,7 +25,6 @@ import ( s "github.com/elastic/beats/v7/libbeat/common/schema" c "github.com/elastic/beats/v7/libbeat/common/schema/mapstriface" "github.com/elastic/beats/v7/metricbeat/mb" - "github.com/elastic/beats/v7/metricbeat/module/nats/util" ) var ( @@ -35,101 +34,29 @@ var ( "time": c.Str("now"), }, } - emptyConnectionsSchema = s.Schema{ - "total": c.Int("total"), - } connectionsSchema = s.Schema{ - "total": c.Int("total"), - "name": c.Str("name"), - "subscriptions": c.Int("subscriptions"), - "in": s.Object{ - "messages": c.Int("in_msgs"), - "bytes": c.Int("in_bytes"), - }, - "out": s.Object{ - "messages": c.Int("out_msgs"), - "bytes": c.Int("out_bytes"), - }, - "pending_bytes": c.Int("pending_bytes"), - "uptime": c.Str("uptime"), - "idle_time": c.Str("idle"), + "total": c.Int("total"), } ) -// Connections stores connections related information -type Connections struct { - ConnectionNum int `json:"total"` - Now string `json:"now"` - ServerID string `json:"server_id"` - Connections []map[string]interface{} `json:"connections,omitempty"` -} +func eventMapping(r mb.ReporterV2, content []byte) error { + var event mb.Event + var inInterface map[string]interface{} -// eventMapping maps a connection to a Metricbeat event using connectionsSchema -func eventMapping(content map[string]interface{}, fieldsSchema s.Schema) (mb.Event, error) { - fields, err := fieldsSchema.Apply(content) + err := json.Unmarshal(content, &inInterface) if err != nil { - return mb.Event{}, errors.Wrap(err, "error applying connection schema") + return errors.Wrap(err, "failure parsing NATS connections API response") } - - err = util.UpdateDuration(fields, "uptime") + event.MetricSetFields, err = connectionsSchema.Apply(inInterface) if err != nil { - return mb.Event{}, errors.Wrap(err, "failure updating uptime key") - } + return errors.Wrap(err, "failure applying connections schema") - err = util.UpdateDuration(fields, "idle_time") - if err != nil { - return mb.Event{}, errors.Wrap(err, "failure updating idle_time key") } - moduleFields, err := moduleSchema.Apply(content) + event.ModuleFields, err = moduleSchema.Apply(inInterface) if err != nil { - return mb.Event{}, errors.Wrap(err, "error applying module schema") - } - - event := mb.Event{ - MetricSetFields: fields, - ModuleFields: moduleFields, - } - return event, nil -} - -// eventsMapping maps the top-level connections metrics AND also per-connection metrics -func eventsMapping(r mb.ReporterV2, content []byte) error { - var err error - connections := Connections{} - if err = json.Unmarshal(content, &connections); err != nil { - return errors.Wrap(err, "failure parsing NATS connections API response") - } - - for _, con := range connections.Connections { - var evt mb.Event - con["server_id"] = connections.ServerID - con["now"] = connections.Now - con["total"] = connections.ConnectionNum - evt, err = eventMapping(con, connectionsSchema) - if err != nil { - r.Error(errors.Wrap(err, "error mapping connection event")) - continue - } - if !r.Event(evt) { - return nil - } - } - // emit one event even if there are no connections - if len(connections.Connections) == 0 { - var evt mb.Event - con := make(map[string]interface{}) - con["server_id"] = connections.ServerID - con["now"] = connections.Now - con["total"] = connections.ConnectionNum - evt, err = eventMapping(con, emptyConnectionsSchema) - if err != nil { - r.Error(errors.Wrap(err, "error mapping connection event")) - return nil - } - if !r.Event(evt) { - return nil - } + return errors.Wrap(err, "failure applying module schema") } + r.Event(event) return nil } diff --git a/metricbeat/module/nats/docker-compose.yml b/metricbeat/module/nats/docker-compose.yml index d4a3248a8ce..54340cf3518 100644 --- a/metricbeat/module/nats/docker-compose.yml +++ b/metricbeat/module/nats/docker-compose.yml @@ -2,7 +2,7 @@ version: '2.3' services: nats: - image: docker.elastic.co/integrations-ci/beats-nats:${NATS_VERSION:-2.0.4}-2 + image: docker.elastic.co/integrations-ci/beats-nats:${NATS_VERSION:-2.0.4}-3 build: context: ./_meta dockerfile: Dockerfile @@ -10,12 +10,8 @@ services: NATS_VERSION: ${NATS_VERSION:-2.0.4} ports: - 8222 - nats-1: - image: docker.elastic.co/integrations-ci/beats-nats:${NATS_VERSION:-2.0.4}-2 - build: - context: ./_meta - dockerfile: Dockerfile - args: - NATS_VERSION: ${NATS_VERSION:-2.0.4} + nats-routes: + extends: + service: nats environment: - ROUTES=1 diff --git a/metricbeat/module/nats/fields.go b/metricbeat/module/nats/fields.go index d66285bf14b..0d2a16c63dd 100644 --- a/metricbeat/module/nats/fields.go +++ b/metricbeat/module/nats/fields.go @@ -32,5 +32,5 @@ func init() { // AssetNats returns asset data. // This is the base64 encoded gzipped contents of module/nats. func AssetNats() string { - return "eJzsmjtzGzcQx3t9ih1XSWF9ABaZ8SSNi7iIlJoGgSUP4wNwXiwo058+g3tQ98A9KJGSmeEVKoTj/n+3i8M+yI/wDQ8rsIL9HQBrznEFH758enz4cAeg0EvSBWtnV/DHHQCUd8LfToUc7wAIcxQeV7ATdwBbjbnyq/K+j2CFwaPlePGhiHeSC0X9n4T9eH2NH/oK0lkW2nrwLFh71tIDZ4LhCQmBUCjYkjPw5VmiTdCm8Eh7pHutjisNzjc8PDlq/38EKl6PGdam4PNfYyKsDQ5klGBcpvFQ2Y9WwG3BIJOWIAlFvHsgKp21KOOSH4i2XT2j+mfj6zK+z0ZjhAWjqjnaGt3YN1c/Am1WdizyzkpDqi3jDqm3NsHbRMMGs0GKjpKBCC3nBxCS9R5B5hot+yRJ/JsEGW6HpSCiihdn2HJfUtyHzdGcv5Q7OiKgLXCm/RxZgVZpu1tvDoxpstzZXW9h68gIXkHqQych1+qVocqX88ih6L1vi1hVIDEwuQi3QNJO9cIM2kMo4DeP8vckpVY5rn8R0MgyhdrXSp8lCymEccFyDKa20pkYXiVY9J82cWi0mQx6L3aD3TXpwQWAE5Cjig1SarfP8ky9Kq/jHVpsQF3gywTUBd65Xz2gR8grCeiRt2uxgSQXukKvTPSlvevJ8YQ77RkJ1dAT15BfCY1jXOt+ffHK2qMyC1qBs/CUaZmVx34V3GcqVMBuMu97/XNxhnpRap+IWuEofVC9OFjRYlOSlcLpfNc/BGvVF5yNulggd0uv13Ma39LrjIP+Z+nVs+AzZte9oLPm1nfsdeqpx1yfY9Dcv2XzWDf9UdfRAULcgzHOXz49PkBBTqJPv7bS0QjkGWqD3O20FHklUrqvzQMUrIeRCkEWIUnlpchRrbe5E/3zpnFhgSTR9ldPcKIsQseDftKDZaG3To+f4GyFQ6mSH6r5F6rJoU5VBl0squ3iM6F0S/C3BH9L8CfyvFWCb4P63D3Fc8sHg3SZUytKwFGiNZp2tkwFSa6MOd2KvHCnRXtN2XHMiVUeP3G3EX5f90ujecQFmHBsYL8H9Nz+pie1WUYw26iBdHJ9DnQhbA8YlPZMehPK2aqzYJzV7ChuwH//+fwwYmPqQaA/7fk5etvMu3XiY8FgF2eaY51SY7SfbdzL3S+m3p+9pDgVfS/o/ckjxKngPmz8+5OXFKeik3P9PPv25BFiDHxuuvjKXrE7T7yaiWz9Xev4yPW5LPZIgwxynnRb2QZXYNXplvPYrkNz7Xm0adgv71dP7Bii7ZdxGcEyuxAXa4MeRKURe/qtC1bFYgxEhy/dogqZ4f3oqPgsnZYPOYNHrifrWIlO4GSa1yQ4jXTJ5jmKNsECQQgbLAfd8cXFPdY/UZnD3wrrAt8b8eO8PjXihzbBQGW/qv4UbA5LgcS+v9Hqn7O4sMn7zl7S/eyRxA7HeP4LAAD//x6BNsM=" + return "eJzsms1y4zYMx+95Csye2sPmAXzozE572UNzaNKzlyZhi7MiqQVBZ71P36E+HH1QluzYidOxjpYM/ASQxB+wP8N33C3ACvZ3AKw5xwV8evjy9PjpDkChl6QL1s4u4I87ACifhL+dCjneARDmKDwuYCPuANYac+UX5XOfwQqDe8vx4l0RnyQXivqThP14fYtf+gbSWRbaevAsWHvW0gNnguEZCYFQKFiTM/Dw4qJN0KbwSFuke632dxqc77h7dtT+fAQqXk8Z1qbg619jTlgbHLhRgnGej8fKfrQCbg0GmbQESSji0wOn0lmLsnMrFekJp382oS7T+2IzJlgwqhrDt77UTX1z9RPQXQqdwBzKwQRuk4toMQaJM0wHopWcsNqb80kKbRk3SCdQBLNCihwdJ6AtcKb9FFmBVmm7Wa52jGmy3NlN78bakRG8gNSXjkKuvVeGqlhOI4eit8hnsapAYmByFm6BpJ3qpRm0h1DAbx7l70lKrXJcXgloZDmE2veV3sEzKYRxwXJMprbSmZheJVj03zaxVdtMBr0Xm8HqOhjBGYAHIEc9Nkip1T7Jc2irvI53aLEBdYEvk1AXeOOuPaF7yA+S0D1v1+KwxPqrr7HsWOSXKm8yEKHlfAdCst4iyFyj5WHEyAUeaqCTY1WaO2uYrlcNEBrHuNR9NfRKpVSZBa3AWXjOtMzKIlVF9oUKFbA7qFK8/jW7np4kREqk9KlaOEofqycnK1psBGR/0baqc//Irr2ecJLrYoa7mxj4OLXjJgYmAvQ/EwOD4+nqattFJQDhRntGQtWPxL60suAzxmcr6KzRecfWtZ4cTbWtBs39W84CalUX/TraQYibNKb64cvTIxTkJPr0uSYdjUCeYaHlbqOlyCsnZfjaPEDBehiRULIISSovRY5quc6d6B/ITQgLJIm2f/eIIMoidCLoD0aw3KrLdH8BZ1NWpZd8V80QUQ1UOwzk58Wy2j4+Ep5uCuimgG4K6Eiet1JAbVCfu+d4bvlgkC5zakUXsHfRmj04W5aCJFfGnO7VTlxp0V4jO/Y1sarjR642wh/LvjSaRpyBCfsO/0dAz+1fy1KLZQSzjRpIJ+9Pgc6E7QGD0p5Jr0I5C3MWjLOaHcUF+O8/Xx9HbBx6Eejr9V+jj03srSNfCwarONMcdUqN0X638Sh3J4/vz15SHIu+FfT+5BHiWHAfVv79yUuKY9HJuX6dfXvyCDEGPjV+fWWv2B24fpieuh6mj8+kX2SxRxpUkPOU28o2uAKrTrccWHcDmmvPo03Ddn6/emTHEG2fxmUEy+xCXKwNehCVj9jTr12wKooxEB2+dIsqZIb3o7P0s3RaPuQMHrn+6QErpwdwMs1LEpxGumTzHJ02yQJBCCssfwmIGxe3WP/NZwp/LawLfG/Ez/PG1Iif2gQDlf1K/SlY7eYCiW1/odV/CXJhlfeDPaf72SKJDY7x/BcAAP//jnKOlQ==" } diff --git a/metricbeat/module/nats/route/_meta/data.json b/metricbeat/module/nats/route/_meta/data.json new file mode 100644 index 00000000000..4858915bf0a --- /dev/null +++ b/metricbeat/module/nats/route/_meta/data.json @@ -0,0 +1,35 @@ +{ + "@timestamp":"2016-05-23T08:05:34.853Z", + "beat":{ + "hostname":"beathost", + "name":"beathost" + }, + "metricset":{ + "host":"localhost", + "module":"nats", + "name":"routes", + "rtt":44269 + }, + "nats":{ + "server": { + "id": "bUAdpRFtMWddIBWw80Yd9D", + "time": "2018-12-28T12:33:53.026865597Z" + }, + "routes": { + "in": { + "bytes": 0, + "messages": 0 + }, + "ip": "172.25.255.2", + "out": { + "bytes": 0, + "messages": 0 + }, + "pending_size": 0, + "port": 60736, + "remote_id": "NCYX4P3ZCXZT3UF4BBZUG46S7EU224XXXLUTUBUDOIAUAIR5WJV73BV5", + "subscriptions": 0, + } + }, + "type":"metricsets" +} diff --git a/metricbeat/module/nats/route/_meta/docs.asciidoc b/metricbeat/module/nats/route/_meta/docs.asciidoc new file mode 100644 index 00000000000..eec8b00ea7c --- /dev/null +++ b/metricbeat/module/nats/route/_meta/docs.asciidoc @@ -0,0 +1 @@ +This is the route metricset of the module nats collecting metrcis per connection. diff --git a/metricbeat/module/nats/route/_meta/fields.yml b/metricbeat/module/nats/route/_meta/fields.yml new file mode 100644 index 00000000000..b60e5d7c199 --- /dev/null +++ b/metricbeat/module/nats/route/_meta/fields.yml @@ -0,0 +1,54 @@ +- name: route + type: group + description: > + Contains nats route related metrics + release: ga + fields: + - name: subscriptions + type: integer + description: > + The number of subscriptions in this connection + - name: remote_id + type: keyword + description: > + The remote id on which the route is connected to + - name: pending_size + type: long + description: > + The number of pending routes + - name: port + type: integer + description: > + The port of the route + - name: ip + type: ip + description: > + The ip of the route + - name: in + type: group + description: > + The amount of incoming data + fields: + - name: messages + type: long + description: > + The amount of incoming messages + - name: bytes + type: long + format: bytes + description: > + The amount of incoming bytes + - name: out + type: group + description: > + The amount of outgoing data + fields: + - name: messages + type: long + description: > + The amount of outgoing messages + - name: bytes + type: long + format: bytes + description: > + The amount of outgoing bytes diff --git a/metricbeat/module/nats/route/_meta/test/routesmetrics.json b/metricbeat/module/nats/route/_meta/test/routesmetrics.json new file mode 100644 index 00000000000..1d9c685696f --- /dev/null +++ b/metricbeat/module/nats/route/_meta/test/routesmetrics.json @@ -0,0 +1,37 @@ +{ + "server_id": "NDIFJNKCKCL2I2ICEXF64LAOAFMW2TN667B4NOWEKF77U7WFQR7CMF7M", + "now": "2020-11-09T09:34:10.027121937Z", + "num_routes": 2, + "routes": [ + { + "rid": 1, + "remote_id": "NCYX4P3ZCXZT3UF4BBZUG46S7EU224XXXLUTUBUDOIAUAIR5WJV73BV5", + "did_solicit": false, + "is_configured": false, + "ip": "172.25.255.2", + "port": 60736, + "pending_size": 0, + "rtt": "1.646926ms", + "in_msgs": 0, + "out_msgs": 0, + "in_bytes": 0, + "out_bytes": 0, + "subscriptions": 0 + }, + { + "rid": 2, + "remote_id": "NAVXZD4IXIH5QYJM33LQYXO3DZGXCUZ4MU72XH5GSELJB3WJWYIGND2D", + "did_solicit": false, + "is_configured": false, + "ip": "172.25.255.3", + "port": 42898, + "pending_size": 0, + "rtt": "511µs", + "in_msgs": 0, + "out_msgs": 0, + "in_bytes": 0, + "out_bytes": 0, + "subscriptions": 0 + } + ] +} diff --git a/metricbeat/module/nats/route/data.go b/metricbeat/module/nats/route/data.go new file mode 100644 index 00000000000..134ad352e5a --- /dev/null +++ b/metricbeat/module/nats/route/data.go @@ -0,0 +1,103 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package route + +import ( + "encoding/json" + + "github.com/elastic/beats/v7/metricbeat/mb" + + "github.com/pkg/errors" + + s "github.com/elastic/beats/v7/libbeat/common/schema" + c "github.com/elastic/beats/v7/libbeat/common/schema/mapstriface" +) + +var ( + moduleSchema = s.Schema{ + "server": s.Object{ + "id": c.Str("server_id"), + "time": c.Str("now"), + }, + } + routesSchema = s.Schema{ + "remote_id": c.Str("remote_id"), + "subscriptions": c.Int("subscriptions"), + "in": s.Object{ + "messages": c.Int("in_msgs"), + "bytes": c.Int("in_bytes"), + }, + "out": s.Object{ + "messages": c.Int("out_msgs"), + "bytes": c.Int("out_bytes"), + }, + "pending_size": c.Int("pending_size"), + "port": c.Int("port"), + "ip": c.Str("ip"), + } +) + +// Routes stores routes related information +type Routes struct { + Now string `json:"now"` + ServerID string `json:"server_id"` + Routes []map[string]interface{} `json:"routes,omitempty"` +} + +// eventMapping maps a route to a Metricbeat event using routesSchema +func eventMapping(content map[string]interface{}, fieldsSchema s.Schema) (mb.Event, error) { + fields, err := fieldsSchema.Apply(content) + if err != nil { + return mb.Event{}, errors.Wrap(err, "error applying routes schema") + } + + moduleFields, err := moduleSchema.Apply(content) + if err != nil { + return mb.Event{}, errors.Wrap(err, "error applying module schema") + } + + event := mb.Event{ + MetricSetFields: fields, + ModuleFields: moduleFields, + } + return event, nil +} + +// eventsMapping maps per-route metrics +func eventsMapping(r mb.ReporterV2, content []byte) error { + var err error + connections := Routes{} + if err = json.Unmarshal(content, &connections); err != nil { + return errors.Wrap(err, "failure parsing NATS connections API response") + } + + for _, con := range connections.Routes { + var evt mb.Event + con["server_id"] = connections.ServerID + con["now"] = connections.Now + evt, err = eventMapping(con, routesSchema) + if err != nil { + r.Error(errors.Wrap(err, "error mapping connection event")) + continue + } + if !r.Event(evt) { + return nil + } + } + return nil +} diff --git a/metricbeat/module/nats/route/route.go b/metricbeat/module/nats/route/route.go new file mode 100644 index 00000000000..fe2109c2332 --- /dev/null +++ b/metricbeat/module/nats/route/route.go @@ -0,0 +1,95 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package route + +import ( + "github.com/pkg/errors" + + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/metricbeat/helper" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/mb/parse" +) + +const ( + defaultScheme = "http" + defaultPath = "/routez" +) + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: defaultScheme, + DefaultPath: defaultPath, + PathConfigKey: "route.metrics_path", + }.Build() +) + +// init registers the MetricSet with the central registry as soon as the program +// starts. The New function will be called later to instantiate an instance of +// the MetricSet for each host defined in the module's configuration. After the +// MetricSet has been created then Fetch will begin to be called periodically. +func init() { + mb.Registry.MustAddMetricSet("nats", "route", New, + mb.WithHostParser(hostParser), + ) +} + +// MetricSet holds any configuration or state information. It must implement +// the mb.MetricSet interface. And this is best achieved by embedding +// mb.BaseMetricSet because it implements all of the required mb.MetricSet +// interface methods except for Fetch. +type MetricSet struct { + mb.BaseMetricSet + http *helper.HTTP + Log *logp.Logger +} + +// New creates a new instance of the MetricSet. New is responsible for unpacking +// any MetricSet specific configuration options if there are any. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + config := struct{}{} + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + + http, err := helper.NewHTTP(base) + if err != nil { + return nil, err + } + return &MetricSet{ + base, + http, + logp.NewLogger("nats"), + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(r mb.ReporterV2) error { + content, err := m.http.FetchContent() + if err != nil { + return errors.Wrap(err, "error in fetch") + } + err = eventsMapping(r, content) + if err != nil { + return errors.Wrap(err, "error in mapping") + } + + return nil +} diff --git a/metricbeat/module/nats/route/route_integration_test.go b/metricbeat/module/nats/route/route_integration_test.go new file mode 100644 index 00000000000..f54a0ff3f6e --- /dev/null +++ b/metricbeat/module/nats/route/route_integration_test.go @@ -0,0 +1,56 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration + +package route + +import ( + "testing" + + "github.com/elastic/beats/v7/libbeat/tests/compose" + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" +) + +func TestData(t *testing.T) { + service := compose.EnsureUp(t, "nats") + compose.EnsureUp(t, "nats-routes") + + m := mbtest.NewFetcher(t, getConfig(service.Host())) + m.WriteEvents(t, "") +} + +func TestFetch(t *testing.T) { + service := compose.EnsureUp(t, "nats") + compose.EnsureUp(t, "nats-routes") + + reporter := &mbtest.CapturingReporterV2{} + + metricSet := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host())) + metricSet.Fetch(reporter) + + e := mbtest.StandardizeEvent(metricSet, reporter.GetEvents()[0]) + t.Logf("%s/%s event: %+v", metricSet.Module().Name(), metricSet.Name(), e.Fields.StringToPrint()) +} + +func getConfig(host string) map[string]interface{} { + return map[string]interface{}{ + "module": "nats", + "metricsets": []string{"route"}, + "hosts": []string{host}, + } +} diff --git a/metricbeat/module/nats/route/route_test.go b/metricbeat/module/nats/route/route_test.go new file mode 100644 index 00000000000..72f5f2f3794 --- /dev/null +++ b/metricbeat/module/nats/route/route_test.go @@ -0,0 +1,63 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package route + +import ( + "io/ioutil" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" +) + +func TestEventMapping(t *testing.T) { + content, err := ioutil.ReadFile("./_meta/test/routesmetrics.json") + assert.NoError(t, err) + reporter := &mbtest.CapturingReporterV2{} + err = eventsMapping(reporter, content) + assert.NoError(t, err) +} + +func TestFetchEventContent(t *testing.T) { + absPath, _ := filepath.Abs("./_meta/test") + + response, _ := ioutil.ReadFile(absPath + "/routesmetrics.json") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Header().Set("Content-Type", "application/json;") + w.Write([]byte(response)) + })) + defer server.Close() + + config := map[string]interface{}{ + "module": "nats", + "metricsets": []string{"route"}, + "hosts": []string{server.URL}, + } + reporter := &mbtest.CapturingReporterV2{} + + metricSet := mbtest.NewReportingMetricSetV2Error(t, config) + metricSet.Fetch(reporter) + + e := mbtest.StandardizeEvent(metricSet, reporter.GetEvents()[0]) + t.Logf("%s/%s event: %+v", metricSet.Module().Name(), metricSet.Name(), e.Fields.StringToPrint()) +} diff --git a/metricbeat/module/nats/routes/_meta/data.json b/metricbeat/module/nats/routes/_meta/data.json index 2f091f72953..8c1748713b4 100644 --- a/metricbeat/module/nats/routes/_meta/data.json +++ b/metricbeat/module/nats/routes/_meta/data.json @@ -16,19 +16,6 @@ "time": "2018-12-28T12:33:53.026865597Z" }, "routes": { - "in": { - "bytes": 0, - "messages": 0 - }, - "ip": "172.25.255.2", - "out": { - "bytes": 0, - "messages": 0 - }, - "pending_size": 0, - "port": 60736, - "remote_id": "NCYX4P3ZCXZT3UF4BBZUG46S7EU224XXXLUTUBUDOIAUAIR5WJV73BV5", - "subscriptions": 0, "total": 2 } }, diff --git a/metricbeat/module/nats/routes/_meta/docs.asciidoc b/metricbeat/module/nats/routes/_meta/docs.asciidoc index 85ddb92bb40..907401be3a7 100644 --- a/metricbeat/module/nats/routes/_meta/docs.asciidoc +++ b/metricbeat/module/nats/routes/_meta/docs.asciidoc @@ -1 +1 @@ -This is the routes metricset of the module nats. +This is the routes metricset of the module nats collecting top level metrics about routes. diff --git a/metricbeat/module/nats/routes/_meta/fields.yml b/metricbeat/module/nats/routes/_meta/fields.yml index b20a46844ad..3b9db2536b3 100644 --- a/metricbeat/module/nats/routes/_meta/fields.yml +++ b/metricbeat/module/nats/routes/_meta/fields.yml @@ -8,51 +8,3 @@ type: integer description: > The number of registered routes - - name: subscriptions - type: integer - description: > - The number of subscriptions in this connection - - name: remote_id - type: keyword - description: > - The remote id on which the route is connected to - - name: pending_size - type: long - description: > - The number of pending routes - - name: port - type: integer - description: > - The port of the route - - name: ip - type: ip - description: > - The ip of the route - - name: in - type: group - description: > - The amount of incoming data - fields: - - name: messages - type: long - description: > - The amount of incoming messages - - name: bytes - type: long - format: bytes - description: > - The amount of incoming bytes - - name: out - type: group - description: > - The amount of outgoing data - fields: - - name: messages - type: long - description: > - The amount of outgoing messages - - name: bytes - type: long - format: bytes - description: > - The amount of outgoing bytes diff --git a/metricbeat/module/nats/routes/data.go b/metricbeat/module/nats/routes/data.go index e41e2dfa8d6..83581a16a31 100644 --- a/metricbeat/module/nats/routes/data.go +++ b/metricbeat/module/nats/routes/data.go @@ -35,91 +35,28 @@ var ( "time": c.Str("now"), }, } - emptyRoutesSchema = s.Schema{ - "total": c.Int("num_routes"), - } routesSchema = s.Schema{ - "total": c.Int("num_routes"), - "remote_id": c.Str("remote_id"), - "subscriptions": c.Int("subscriptions"), - "in": s.Object{ - "messages": c.Int("in_msgs"), - "bytes": c.Int("in_bytes"), - }, - "out": s.Object{ - "messages": c.Int("out_msgs"), - "bytes": c.Int("out_bytes"), - }, - "pending_size": c.Int("pending_size"), - "port": c.Int("port"), - "ip": c.Str("ip"), + "total": c.Int("num_routes"), } ) -// Routes stores routes related information -type Routes struct { - RoutesNum int `json:"num_routes"` - Now string `json:"now"` - ServerID string `json:"server_id"` - Routes []map[string]interface{} `json:"routes,omitempty"` -} +func eventMapping(r mb.ReporterV2, content []byte) error { + var event mb.Event + var inInterface map[string]interface{} -// eventMapping maps a route to a Metricbeat event using routesSchema -func eventMapping(content map[string]interface{}, fieldsSchema s.Schema) (mb.Event, error) { - fields, err := fieldsSchema.Apply(content) + err := json.Unmarshal(content, &inInterface) if err != nil { - return mb.Event{}, errors.Wrap(err, "error applying routes schema") + return errors.Wrap(err, "failure parsing Nats routes API response") } - - moduleFields, err := moduleSchema.Apply(content) + event.MetricSetFields, err = routesSchema.Apply(inInterface) if err != nil { - return mb.Event{}, errors.Wrap(err, "error applying module schema") + return errors.Wrap(err, "failure applying routes schema") } - event := mb.Event{ - MetricSetFields: fields, - ModuleFields: moduleFields, - } - return event, nil -} - -// eventsMapping maps the top-level routes metrics AND also per-route metrics -func eventsMapping(r mb.ReporterV2, content []byte) error { - var err error - connections := Routes{} - if err = json.Unmarshal(content, &connections); err != nil { - return errors.Wrap(err, "failure parsing NATS connections API response") - } - - for _, con := range connections.Routes { - var evt mb.Event - con["server_id"] = connections.ServerID - con["now"] = connections.Now - con["num_routes"] = connections.RoutesNum - evt, err = eventMapping(con, routesSchema) - if err != nil { - r.Error(errors.Wrap(err, "error mapping connection event")) - continue - } - if !r.Event(evt) { - return nil - } - } - // emit one event even if there are no connections - if len(connections.Routes) == 0 { - var evt mb.Event - con := make(map[string]interface{}) - con["server_id"] = connections.ServerID - con["now"] = connections.Now - con["num_routes"] = connections.RoutesNum - evt, err = eventMapping(con, emptyRoutesSchema) - if err != nil { - r.Error(errors.Wrap(err, "error mapping connection event")) - return nil - } - if !r.Event(evt) { - return nil - } + event.ModuleFields, err = moduleSchema.Apply(inInterface) + if err != nil { + return errors.Wrap(err, "failure applying module schema") } + r.Event(event) return nil } diff --git a/metricbeat/module/nats/routes/routes.go b/metricbeat/module/nats/routes/routes.go index 8ead24cbadd..91b3ad33930 100644 --- a/metricbeat/module/nats/routes/routes.go +++ b/metricbeat/module/nats/routes/routes.go @@ -87,7 +87,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { if err != nil { return errors.Wrap(err, "error in fetch") } - err = eventsMapping(r, content) + err = eventMapping(r, content) if err != nil { return errors.Wrap(err, "error in mapping") } diff --git a/metricbeat/module/nats/routes/routes_test.go b/metricbeat/module/nats/routes/routes_test.go index 9ce261d5804..bf987019885 100644 --- a/metricbeat/module/nats/routes/routes_test.go +++ b/metricbeat/module/nats/routes/routes_test.go @@ -33,7 +33,7 @@ func TestEventMapping(t *testing.T) { content, err := ioutil.ReadFile("./_meta/test/routesmetrics.json") assert.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} - err = eventsMapping(reporter, content) + err = eventMapping(reporter, content) assert.NoError(t, err) event := reporter.GetEvents()[0] d, _ := event.MetricSetFields.GetValue("total") diff --git a/metricbeat/module/nats/stats/_meta/docs.asciidoc b/metricbeat/module/nats/stats/_meta/docs.asciidoc index 1e468233bf1..3ce6026a5b6 100644 --- a/metricbeat/module/nats/stats/_meta/docs.asciidoc +++ b/metricbeat/module/nats/stats/_meta/docs.asciidoc @@ -1 +1 @@ -This is the stats metricset of the module nats. +This is the stats metricset of the module nats collecting generic stats. diff --git a/metricbeat/module/nats/subscriptions/_meta/docs.asciidoc b/metricbeat/module/nats/subscriptions/_meta/docs.asciidoc index a0601ada457..adaa097d75c 100644 --- a/metricbeat/module/nats/subscriptions/_meta/docs.asciidoc +++ b/metricbeat/module/nats/subscriptions/_meta/docs.asciidoc @@ -1 +1 @@ -This is the subscriptions metricset of the module nats. +This is the subscriptions metricset of the module nats collecting metrics about subscriptions. diff --git a/metricbeat/module/nats/test_nats.py b/metricbeat/module/nats/test_nats.py index d77a5947a41..810f4afdb88 100644 --- a/metricbeat/module/nats/test_nats.py +++ b/metricbeat/module/nats/test_nats.py @@ -8,7 +8,7 @@ @metricbeat.parameterized_with_supported_versions class TestNats(metricbeat.BaseTest): - COMPOSE_SERVICES = ['nats', 'nats-1'] + COMPOSE_SERVICES = ['nats', 'nats-routes'] @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") def test_stats(self): @@ -60,6 +60,31 @@ def test_connections(self): self.assert_fields_are_documented(evt) + @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") + def test_connection(self): + """ + nats connection test + """ + self.render_config_template(modules=[{ + "name": "nats", + "metricsets": ["connection"], + "hosts": self.get_hosts(), + "period": "5s", + "connections.metrics_path": "/connz" + }]) + proc = self.start_beat() + self.wait_until(lambda: self.output_lines() > 0) + proc.check_kill_and_wait() + self.assert_no_logged_warnings() + + output = self.read_output_json() + self.assertEqual(len(output), 1) + evt = output[0] + + self.assertCountEqual(self.de_dot(NATS_FIELDS), evt.keys(), evt) + + self.assert_fields_are_documented(evt) + @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") def test_routes(self): """ @@ -85,6 +110,31 @@ def test_routes(self): self.assert_fields_are_documented(evt) + @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") + def test_route(self): + """ + nats route test + """ + self.render_config_template(modules=[{ + "name": "nats", + "metricsets": ["route"], + "hosts": self.get_hosts(), + "period": "5s", + "routes.metrics_path": "/routez" + }]) + proc = self.start_beat() + self.wait_until(lambda: self.output_lines() > 0) + proc.check_kill_and_wait() + self.assert_no_logged_warnings() + + output = self.read_output_json() + self.assertEqual(len(output), 1) + evt = output[0] + + self.assertCountEqual(self.de_dot(NATS_FIELDS), evt.keys(), evt) + + self.assert_fields_are_documented(evt) + @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") def test_subscriptions(self): """ diff --git a/metricbeat/modules.d/nats.yml.disabled b/metricbeat/modules.d/nats.yml.disabled index abe095102f5..73b542cc507 100644 --- a/metricbeat/modules.d/nats.yml.disabled +++ b/metricbeat/modules.d/nats.yml.disabled @@ -2,10 +2,18 @@ # Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-nats.html - module: nats - metricsets: ["connections", "routes", "stats", "subscriptions"] + metricsets: + - "connections" + - "routes" + - "stats" + - "subscriptions" + - "connection" + - "route" period: 10s hosts: ["localhost:8222"] #stats.metrics_path: "/varz" #connections.metrics_path: "/connz" #routes.metrics_path: "/routez" #subscriptions.metrics_path: "/subsz" + #connection.metrics_path: "/connz" + #route.metrics_path: "/routez" From 3f4ab753565d4d8f5270562678dcfce136922e76 Mon Sep 17 00:00:00 2001 From: chrismark Date: Thu, 12 Nov 2020 13:38:27 +0200 Subject: [PATCH 12/18] fix identation Signed-off-by: chrismark --- metricbeat/module/nats/_meta/run.sh | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/metricbeat/module/nats/_meta/run.sh b/metricbeat/module/nats/_meta/run.sh index a832c5ac7d3..1e25ec87722 100755 --- a/metricbeat/module/nats/_meta/run.sh +++ b/metricbeat/module/nats/_meta/run.sh @@ -6,22 +6,22 @@ # NATS 2.X if [ -x /opt/nats/nats-server ]; then - if [[ -z "${ROUTES}" ]]; then - (/opt/nats/nats-server --cluster nats://0.0.0.0:6222 --http_port 8222 --port 4222) & - else - (/opt/nats/nats-server --cluster nats://0.0.0.0:6222 --http_port 8222 --port 4222 --routes nats://nats:6222) & - fi - while true; do /nats-bench -np 1 -n 100000000 -ms 16 foo; done + if [[ -z "${ROUTES}" ]]; then + (/opt/nats/nats-server --cluster nats://0.0.0.0:6222 --http_port 8222 --port 4222) & + else + (/opt/nats/nats-server --cluster nats://0.0.0.0:6222 --http_port 8222 --port 4222 --routes nats://nats:6222) & + fi + while true; do /nats-bench -np 1 -n 100000000 -ms 16 foo; done fi # NATS 1.X if [ -x /opt/nats/gnatsd ]; then - if [[ -z "${ROUTES}" ]]; then - (/opt/nats/gnatsd --cluster nats://0.0.0.0:6222 --http_port 8222 --port 4222) & - else - (/opt/nats/gnatsd --cluster nats://0.0.0.0:6222 --http_port 8222 --port 4222 --routes nats://nats:6222) & - fi - while true; do /nats-bench -np 1 -n 100000000 -ms 16 foo; done + if [[ -z "${ROUTES}" ]]; then + (/opt/nats/gnatsd --cluster nats://0.0.0.0:6222 --http_port 8222 --port 4222) & + else + (/opt/nats/gnatsd --cluster nats://0.0.0.0:6222 --http_port 8222 --port 4222 --routes nats://nats:6222) & + fi + while true; do /nats-bench -np 1 -n 100000000 -ms 16 foo; done fi echo "Couldn't find the nats server binary" From 9e3ca07b693f39fd7ea99905ec50621ac7bd0bd7 Mon Sep 17 00:00:00 2001 From: chrismark Date: Thu, 12 Nov 2020 14:25:09 +0200 Subject: [PATCH 13/18] update x-pack Signed-off-by: chrismark --- x-pack/metricbeat/metricbeat.reference.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 1942bdefca7..85850cf5575 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -1027,13 +1027,21 @@ metricbeat.modules: #--------------------------------- NATS Module --------------------------------- - module: nats - metricsets: ["connections", "routes", "stats", "subscriptions"] + metricsets: + - "connections" + - "routes" + - "stats" + - "subscriptions" + - "connection" + - "route" period: 10s hosts: ["localhost:8222"] #stats.metrics_path: "/varz" #connections.metrics_path: "/connz" #routes.metrics_path: "/routez" #subscriptions.metrics_path: "/subsz" + #connection.metrics_path: "/connz" + #route.metrics_path: "/routez" #-------------------------------- Nginx Module -------------------------------- - module: nginx From 1fbf464ec93470dca72735ea5a9a39913d2e15fb Mon Sep 17 00:00:00 2001 From: chrismark Date: Thu, 12 Nov 2020 16:43:02 +0200 Subject: [PATCH 14/18] use time of server for timestamp Signed-off-by: chrismark --- metricbeat/docs/fields.asciidoc | 10 ------- metricbeat/docs/modules/nats.asciidoc | 4 +-- metricbeat/metricbeat.reference.yml | 4 +-- .../module/nats/_meta/config.reference.yml | 4 +-- metricbeat/module/nats/_meta/config.yml | 12 ++++----- metricbeat/module/nats/_meta/fields.yml | 4 --- metricbeat/module/nats/connection/data.go | 4 +++ metricbeat/module/nats/connections/data.go | 13 +++++++--- metricbeat/module/nats/fields.go | 2 +- metricbeat/module/nats/route/data.go | 4 +++ metricbeat/module/nats/routes/data.go | 13 +++++++--- metricbeat/module/nats/stats/data.go | 26 ++++++++++++------- metricbeat/modules.d/nats.yml.disabled | 12 ++++----- x-pack/metricbeat/metricbeat.reference.yml | 4 +-- 14 files changed, 66 insertions(+), 50 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 9138ee28c87..8e88500f0d0 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -35464,16 +35464,6 @@ type: keyword -- -*`nats.server.time`*:: -+ --- -Server time of metric creation - - -type: date - --- - [float] === connection diff --git a/metricbeat/docs/modules/nats.asciidoc b/metricbeat/docs/modules/nats.asciidoc index 158e20033cd..91c45b0c32f 100644 --- a/metricbeat/docs/modules/nats.asciidoc +++ b/metricbeat/docs/modules/nats.asciidoc @@ -39,8 +39,8 @@ metricbeat.modules: - "routes" - "stats" - "subscriptions" - - "connection" - - "route" + #- "connection" + #- "route" period: 10s hosts: ["localhost:8222"] #stats.metrics_path: "/varz" diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index ca59aa1ecda..ab3eb3f4380 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -676,8 +676,8 @@ metricbeat.modules: - "routes" - "stats" - "subscriptions" - - "connection" - - "route" + #- "connection" + #- "route" period: 10s hosts: ["localhost:8222"] #stats.metrics_path: "/varz" diff --git a/metricbeat/module/nats/_meta/config.reference.yml b/metricbeat/module/nats/_meta/config.reference.yml index 13b20ea7fd8..a9440c6de3f 100644 --- a/metricbeat/module/nats/_meta/config.reference.yml +++ b/metricbeat/module/nats/_meta/config.reference.yml @@ -4,8 +4,8 @@ - "routes" - "stats" - "subscriptions" - - "connection" - - "route" + #- "connection" + #- "route" period: 10s hosts: ["localhost:8222"] #stats.metrics_path: "/varz" diff --git a/metricbeat/module/nats/_meta/config.yml b/metricbeat/module/nats/_meta/config.yml index 13b20ea7fd8..540b262a298 100644 --- a/metricbeat/module/nats/_meta/config.yml +++ b/metricbeat/module/nats/_meta/config.yml @@ -1,11 +1,11 @@ - module: nats metricsets: - - "connections" - - "routes" - - "stats" - - "subscriptions" - - "connection" - - "route" + #- "connections" + #- "routes" + #- "stats" + #- "subscriptions" + #- "connection" + #- "route" period: 10s hosts: ["localhost:8222"] #stats.metrics_path: "/varz" diff --git a/metricbeat/module/nats/_meta/fields.yml b/metricbeat/module/nats/_meta/fields.yml index cfe8cfe1bc4..13e3f263d0a 100644 --- a/metricbeat/module/nats/_meta/fields.yml +++ b/metricbeat/module/nats/_meta/fields.yml @@ -13,7 +13,3 @@ type: keyword description: > The server ID - - name: server.time - type: date - description: > - Server time of metric creation diff --git a/metricbeat/module/nats/connection/data.go b/metricbeat/module/nats/connection/data.go index a895d494ad6..0eb2f0a39ea 100644 --- a/metricbeat/module/nats/connection/data.go +++ b/metricbeat/module/nats/connection/data.go @@ -19,6 +19,7 @@ package connection import ( "encoding/json" + "time" "github.com/pkg/errors" @@ -81,9 +82,12 @@ func eventMapping(content map[string]interface{}, fieldsSchema s.Schema) (mb.Eve return mb.Event{}, errors.Wrap(err, "error applying module schema") } + timestamp, _ := moduleFields.GetValue("now") + moduleFields.Delete("now") event := mb.Event{ MetricSetFields: fields, ModuleFields: moduleFields, + Timestamp: timestamp.(time.Time), } return event, nil } diff --git a/metricbeat/module/nats/connections/data.go b/metricbeat/module/nats/connections/data.go index f08f7b73ec3..73884c91dec 100644 --- a/metricbeat/module/nats/connections/data.go +++ b/metricbeat/module/nats/connections/data.go @@ -19,6 +19,7 @@ package connections import ( "encoding/json" + "time" "github.com/pkg/errors" @@ -40,23 +41,29 @@ var ( ) func eventMapping(r mb.ReporterV2, content []byte) error { - var event mb.Event var inInterface map[string]interface{} err := json.Unmarshal(content, &inInterface) if err != nil { return errors.Wrap(err, "failure parsing NATS connections API response") } - event.MetricSetFields, err = connectionsSchema.Apply(inInterface) + metricSetFields, err := connectionsSchema.Apply(inInterface) if err != nil { return errors.Wrap(err, "failure applying connections schema") } - event.ModuleFields, err = moduleSchema.Apply(inInterface) + moduleFields, err := moduleSchema.Apply(inInterface) if err != nil { return errors.Wrap(err, "failure applying module schema") } + timestamp, _ := moduleFields.GetValue("now") + moduleFields.Delete("now") + event := mb.Event{ + MetricSetFields: metricSetFields, + ModuleFields: moduleFields, + Timestamp: timestamp.(time.Time), + } r.Event(event) return nil } diff --git a/metricbeat/module/nats/fields.go b/metricbeat/module/nats/fields.go index 0d2a16c63dd..357a25c578c 100644 --- a/metricbeat/module/nats/fields.go +++ b/metricbeat/module/nats/fields.go @@ -32,5 +32,5 @@ func init() { // AssetNats returns asset data. // This is the base64 encoded gzipped contents of module/nats. func AssetNats() string { - return "eJzsms1y4zYMx+95Csye2sPmAXzozE572UNzaNKzlyZhi7MiqQVBZ71P36E+HH1QluzYidOxjpYM/ASQxB+wP8N33C3ACvZ3AKw5xwV8evjy9PjpDkChl6QL1s4u4I87ACifhL+dCjneARDmKDwuYCPuANYac+UX5XOfwQqDe8vx4l0RnyQXivqThP14fYtf+gbSWRbaevAsWHvW0gNnguEZCYFQKFiTM/Dw4qJN0KbwSFuke632dxqc77h7dtT+fAQqXk8Z1qbg619jTlgbHLhRgnGej8fKfrQCbg0GmbQESSji0wOn0lmLsnMrFekJp382oS7T+2IzJlgwqhrDt77UTX1z9RPQXQqdwBzKwQRuk4toMQaJM0wHopWcsNqb80kKbRk3SCdQBLNCihwdJ6AtcKb9FFmBVmm7Wa52jGmy3NlN78bakRG8gNSXjkKuvVeGqlhOI4eit8hnsapAYmByFm6BpJ3qpRm0h1DAbx7l70lKrXJcXgloZDmE2veV3sEzKYRxwXJMprbSmZheJVj03zaxVdtMBr0Xm8HqOhjBGYAHIEc9Nkip1T7Jc2irvI53aLEBdYEvk1AXeOOuPaF7yA+S0D1v1+KwxPqrr7HsWOSXKm8yEKHlfAdCst4iyFyj5WHEyAUeaqCTY1WaO2uYrlcNEBrHuNR9NfRKpVSZBa3AWXjOtMzKIlVF9oUKFbA7qFK8/jW7np4kREqk9KlaOEofqycnK1psBGR/0baqc//Irr2ecJLrYoa7mxj4OLXjJgYmAvQ/EwOD4+nqattFJQDhRntGQtWPxL60suAzxmcr6KzRecfWtZ4cTbWtBs39W84CalUX/TraQYibNKb64cvTIxTkJPr0uSYdjUCeYaHlbqOlyCsnZfjaPEDBehiRULIISSovRY5quc6d6B/ITQgLJIm2f/eIIMoidCLoD0aw3KrLdH8BZ1NWpZd8V80QUQ1UOwzk58Wy2j4+Ep5uCuimgG4K6Eiet1JAbVCfu+d4bvlgkC5zakUXsHfRmj04W5aCJFfGnO7VTlxp0V4jO/Y1sarjR642wh/LvjSaRpyBCfsO/0dAz+1fy1KLZQSzjRpIJ+9Pgc6E7QGD0p5Jr0I5C3MWjLOaHcUF+O8/Xx9HbBx6Eejr9V+jj03srSNfCwarONMcdUqN0X638Sh3J4/vz15SHIu+FfT+5BHiWHAfVv79yUuKY9HJuX6dfXvyCDEGPjV+fWWv2B24fpieuh6mj8+kX2SxRxpUkPOU28o2uAKrTrccWHcDmmvPo03Ddn6/emTHEG2fxmUEy+xCXKwNehCVj9jTr12wKooxEB2+dIsqZIb3o7P0s3RaPuQMHrn+6QErpwdwMs1LEpxGumTzHJ02yQJBCCssfwmIGxe3WP/NZwp/LawLfG/Ez/PG1Iif2gQDlf1K/SlY7eYCiW1/odV/CXJhlfeDPaf72SKJDY7x/BcAAP//jnKOlQ==" + return "eJzsms1y4zYMx+95Csye2sPmAXzozE572UNzaNKzlyZhi7MiqQVBZ71P36E+HH1QluzYidOxjpYE/AiQxB+UP8N33C3ACvZ3AKw5xwV8evjy9PjpDkChl6QL1s4u4I87ACifhL+dCjneARDmKDwuYCPuANYac+UX5XOfwQqDe8vx4l0RnyQXivqXhP14fYsvfQPpLAttPXgWrD1r6YEzwfCMhEAoFKzJGXh4cdEmaFN4pC3SvVb7Ow3Od9w9O2r/PgIVr6cMa1Pw9a+BE+msRRnfG3hpD3rCx5/NqMtIv9iMsRaMCgwyaelbL3Wz0Fz9WHSzYrBzYzwcE7hNWKJFcGvgDNOBaCUjrPbmfJJCW8YN0gkUwayQIkfHCWgLnGk/RVagVdpulqsdY5osd3bTu7F2ZAQvIPXSUci198pQFctp5FCwHsnlAVYVSAxMzsItkLRTvTSD9hAK+M2j/D1JqVWOyysBjSyHUPu+0it4JoUwLliOydRWOhPTqwSL/mgTS7XNZNB7sRnMroMRnAF4AHLUY4OUmu2TPIeWyut4hxYbUBf4Mgl1gTfu2hO6h/wgCd3zdi0OS6y/+hrLjkV+qfImAxFazncgJOstgsw1Wh5GjFxgPF+sSnNnDdP1qgFC4xiXuq+GXqmUKrOgFTgLz5mWWVmkqsi+UKECdgdVite/ZtfTk4RIiZTeVQtH6W315GRFi42A7E/aVnXub9m11xN2cl3McHcTAx+ndtzEwESA/mdiYLA9XV1tu6gEINxoz0io+pHYl1YWfMb4bAWdNTrv2LrWhzhTbatBc/+WZwG1qot+He0gxEUaU/3w5ekRCnISfXpfk45GIM8w0XK30VLklZMyfG0eoGA9jEgoWYQklZciR7Vc5070N+QmhAWSRNu/e0QQZRE6EfQHI1gu1WW6v4CzKavSS74DSVguo75qh4H8vFhW29tHwtNNAd0U0E0BHcnzVgqoDepz9xz3LR8M0mV2regC9i5aZw/OlqUgyZUxp3u1E2datNfIjn1NrOr4kbON8MeyL42mEWdgwr7D/xHQc/vDVWqyjGC2UQPp5P0p0JmwPWBQ2jPpVSjPwpwF46xmR3EC/vvP18cRG4cGAn29/mv0sYm1deSwYDCLM81Rp9QY7bGNR7l78vj+7CXFsehbQe9PHiGOBfdh5d+fvKQ4Fp2c69fZtyePEGPgU8evr+wVuweuH6anrg/Tx8+kX2SxRxpUkPOU28o2uAKrTrc8sO4GNNeeR5uG7fx+9ciOIdo+jcsIltmFuFgb9CAqH7GnX7tgVRRjIDp86RZVyAzvR8/Sz9Jp+ZAzeOT60wNWTg/gZJqXJDiNdMnmOTptkgWCEFZYfgmICxe3WP/jZgp/LawLfG/Ez/PG1Iif2gQDlf1K/SlY7eYCiW1/olVAyoVV3g/2nO5niyQ2OMbzXwAAAP//PYJuZw==" } diff --git a/metricbeat/module/nats/route/data.go b/metricbeat/module/nats/route/data.go index 134ad352e5a..70f10f3474d 100644 --- a/metricbeat/module/nats/route/data.go +++ b/metricbeat/module/nats/route/data.go @@ -19,6 +19,7 @@ package route import ( "encoding/json" + "time" "github.com/elastic/beats/v7/metricbeat/mb" @@ -71,9 +72,12 @@ func eventMapping(content map[string]interface{}, fieldsSchema s.Schema) (mb.Eve return mb.Event{}, errors.Wrap(err, "error applying module schema") } + timestamp, _ := moduleFields.GetValue("now") + moduleFields.Delete("now") event := mb.Event{ MetricSetFields: fields, ModuleFields: moduleFields, + Timestamp: timestamp.(time.Time), } return event, nil } diff --git a/metricbeat/module/nats/routes/data.go b/metricbeat/module/nats/routes/data.go index 83581a16a31..fe274873d27 100644 --- a/metricbeat/module/nats/routes/data.go +++ b/metricbeat/module/nats/routes/data.go @@ -19,6 +19,7 @@ package routes import ( "encoding/json" + "time" "github.com/elastic/beats/v7/metricbeat/mb" @@ -41,22 +42,28 @@ var ( ) func eventMapping(r mb.ReporterV2, content []byte) error { - var event mb.Event var inInterface map[string]interface{} err := json.Unmarshal(content, &inInterface) if err != nil { return errors.Wrap(err, "failure parsing Nats routes API response") } - event.MetricSetFields, err = routesSchema.Apply(inInterface) + metricSetFields, err := routesSchema.Apply(inInterface) if err != nil { return errors.Wrap(err, "failure applying routes schema") } - event.ModuleFields, err = moduleSchema.Apply(inInterface) + moduleFields, err := moduleSchema.Apply(inInterface) if err != nil { return errors.Wrap(err, "failure applying module schema") } + timestamp, _ := moduleFields.GetValue("now") + moduleFields.Delete("now") + event := mb.Event{ + MetricSetFields: metricSetFields, + ModuleFields: moduleFields, + Timestamp: timestamp.(time.Time), + } r.Event(event) return nil } diff --git a/metricbeat/module/nats/stats/data.go b/metricbeat/module/nats/stats/data.go index 3e3ac3274f5..5ac7c13b2a1 100644 --- a/metricbeat/module/nats/stats/data.go +++ b/metricbeat/module/nats/stats/data.go @@ -19,6 +19,7 @@ package stats import ( "encoding/json" + "time" "github.com/pkg/errors" @@ -66,24 +67,24 @@ var ( ) func eventMapping(r mb.ReporterV2, content []byte) error { - var event common.MapStr + var metricsetMetrics common.MapStr var inInterface map[string]interface{} err := json.Unmarshal(content, &inInterface) if err != nil { return errors.Wrap(err, "failure parsing Nats stats API response") } - event, err = statsSchema.Apply(inInterface) + metricsetMetrics, err = statsSchema.Apply(inInterface) if err != nil { return errors.Wrap(err, "failure applying stats schema") } - err = util.UpdateDuration(event, "uptime") + err = util.UpdateDuration(metricsetMetrics, "uptime") if err != nil { return errors.Wrap(err, "failure updating uptime key") } - d, err := event.GetValue("http_req_stats") + d, err := metricsetMetrics.GetValue("http_req_stats") if err != nil { return errors.Wrap(err, "failure retrieving http_req_stats key") } @@ -92,12 +93,12 @@ func eventMapping(r mb.ReporterV2, content []byte) error { return errors.Wrap(err, "failure casting http_req_stats to common.Mapstr") } - err = event.Delete("http_req_stats") + err = metricsetMetrics.Delete("http_req_stats") if err != nil { return errors.Wrap(err, "failure deleting http_req_stats key") } - event["http"] = common.MapStr{ + metricsetMetrics["http"] = common.MapStr{ "req_stats": common.MapStr{ "uri": common.MapStr{ "root": httpStats["root_uri"], @@ -108,7 +109,7 @@ func eventMapping(r mb.ReporterV2, content []byte) error { }, }, } - cpu, err := event.GetValue("cpu") + cpu, err := metricsetMetrics.GetValue("cpu") if err != nil { return errors.Wrap(err, "failure retrieving cpu key") } @@ -116,7 +117,7 @@ func eventMapping(r mb.ReporterV2, content []byte) error { if !ok { return errors.Wrap(err, "failure casting cpu to float64") } - _, err = event.Put("cpu", cpuUtil/100.0) + _, err = metricsetMetrics.Put("cpu", cpuUtil/100.0) if err != nil { return errors.Wrap(err, "failure updating cpu key") } @@ -124,6 +125,13 @@ func eventMapping(r mb.ReporterV2, content []byte) error { if err != nil { return errors.Wrap(err, "failure applying module schema") } - r.Event(mb.Event{MetricSetFields: event, ModuleFields: moduleMetrics}) + timestamp, _ := moduleMetrics.GetValue("now") + moduleMetrics.Delete("now") + evt := mb.Event{ + MetricSetFields: metricsetMetrics, + ModuleFields: moduleMetrics, + Timestamp: timestamp.(time.Time), + } + r.Event(evt) return nil } diff --git a/metricbeat/modules.d/nats.yml.disabled b/metricbeat/modules.d/nats.yml.disabled index 73b542cc507..d398ac0be43 100644 --- a/metricbeat/modules.d/nats.yml.disabled +++ b/metricbeat/modules.d/nats.yml.disabled @@ -3,12 +3,12 @@ - module: nats metricsets: - - "connections" - - "routes" - - "stats" - - "subscriptions" - - "connection" - - "route" + #- "connections" + #- "routes" + #- "stats" + #- "subscriptions" + #- "connection" + #- "route" period: 10s hosts: ["localhost:8222"] #stats.metrics_path: "/varz" diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 85850cf5575..7bdbac1d4e6 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -1032,8 +1032,8 @@ metricbeat.modules: - "routes" - "stats" - "subscriptions" - - "connection" - - "route" + #- "connection" + #- "route" period: 10s hosts: ["localhost:8222"] #stats.metrics_path: "/varz" From 2114fce0f5dd7c8ec5a3150f8250e08bb02b3225 Mon Sep 17 00:00:00 2001 From: chrismark Date: Thu, 12 Nov 2020 18:20:17 +0200 Subject: [PATCH 15/18] fixes Signed-off-by: chrismark --- metricbeat/module/nats/connection/data.go | 10 ++++++---- metricbeat/module/nats/connections/data.go | 11 +++++++---- metricbeat/module/nats/docker-compose.yml | 2 +- metricbeat/module/nats/route/data.go | 14 ++++++++------ metricbeat/module/nats/routes/data.go | 14 ++++++++------ metricbeat/module/nats/stats/data.go | 10 ++++++---- metricbeat/module/nats/util/util.go | 13 +++++++++++++ 7 files changed, 49 insertions(+), 25 deletions(-) diff --git a/metricbeat/module/nats/connection/data.go b/metricbeat/module/nats/connection/data.go index 0eb2f0a39ea..3317ddeb9b9 100644 --- a/metricbeat/module/nats/connection/data.go +++ b/metricbeat/module/nats/connection/data.go @@ -19,7 +19,6 @@ package connection import ( "encoding/json" - "time" "github.com/pkg/errors" @@ -82,12 +81,15 @@ func eventMapping(content map[string]interface{}, fieldsSchema s.Schema) (mb.Eve return mb.Event{}, errors.Wrap(err, "error applying module schema") } - timestamp, _ := moduleFields.GetValue("now") - moduleFields.Delete("now") + timestamp, err := util.GetNatsTimestamp(moduleFields) + moduleFields.Delete("server.time") + if err != nil { + return mb.Event{}, errors.Wrap(err, "failure parsing server timestamp") + } event := mb.Event{ MetricSetFields: fields, ModuleFields: moduleFields, - Timestamp: timestamp.(time.Time), + Timestamp: timestamp, } return event, nil } diff --git a/metricbeat/module/nats/connections/data.go b/metricbeat/module/nats/connections/data.go index 73884c91dec..0be0ee8e05d 100644 --- a/metricbeat/module/nats/connections/data.go +++ b/metricbeat/module/nats/connections/data.go @@ -19,13 +19,13 @@ package connections import ( "encoding/json" - "time" "github.com/pkg/errors" s "github.com/elastic/beats/v7/libbeat/common/schema" c "github.com/elastic/beats/v7/libbeat/common/schema/mapstriface" "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/module/nats/util" ) var ( @@ -57,12 +57,15 @@ func eventMapping(r mb.ReporterV2, content []byte) error { if err != nil { return errors.Wrap(err, "failure applying module schema") } - timestamp, _ := moduleFields.GetValue("now") - moduleFields.Delete("now") + timestamp, err := util.GetNatsTimestamp(moduleFields) + moduleFields.Delete("server.time") + if err != nil { + return errors.Wrap(err, "failure parsing server timestamp") + } event := mb.Event{ MetricSetFields: metricSetFields, ModuleFields: moduleFields, - Timestamp: timestamp.(time.Time), + Timestamp: timestamp, } r.Event(event) return nil diff --git a/metricbeat/module/nats/docker-compose.yml b/metricbeat/module/nats/docker-compose.yml index 54340cf3518..74f8075f9e5 100644 --- a/metricbeat/module/nats/docker-compose.yml +++ b/metricbeat/module/nats/docker-compose.yml @@ -2,7 +2,7 @@ version: '2.3' services: nats: - image: docker.elastic.co/integrations-ci/beats-nats:${NATS_VERSION:-2.0.4}-3 + image: docker.elastic.co/integrations-ci/beats-nats:${NATS_VERSION:-2.0.4}-2 build: context: ./_meta dockerfile: Dockerfile diff --git a/metricbeat/module/nats/route/data.go b/metricbeat/module/nats/route/data.go index 70f10f3474d..3b17bb98f80 100644 --- a/metricbeat/module/nats/route/data.go +++ b/metricbeat/module/nats/route/data.go @@ -19,14 +19,13 @@ package route import ( "encoding/json" - "time" - - "github.com/elastic/beats/v7/metricbeat/mb" "github.com/pkg/errors" s "github.com/elastic/beats/v7/libbeat/common/schema" c "github.com/elastic/beats/v7/libbeat/common/schema/mapstriface" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/module/nats/util" ) var ( @@ -72,12 +71,15 @@ func eventMapping(content map[string]interface{}, fieldsSchema s.Schema) (mb.Eve return mb.Event{}, errors.Wrap(err, "error applying module schema") } - timestamp, _ := moduleFields.GetValue("now") - moduleFields.Delete("now") + timestamp, err := util.GetNatsTimestamp(moduleFields) + moduleFields.Delete("server.time") + if err != nil { + return mb.Event{}, errors.Wrap(err, "failure parsing server timestamp") + } event := mb.Event{ MetricSetFields: fields, ModuleFields: moduleFields, - Timestamp: timestamp.(time.Time), + Timestamp: timestamp, } return event, nil } diff --git a/metricbeat/module/nats/routes/data.go b/metricbeat/module/nats/routes/data.go index fe274873d27..2e7212f94f6 100644 --- a/metricbeat/module/nats/routes/data.go +++ b/metricbeat/module/nats/routes/data.go @@ -19,14 +19,13 @@ package routes import ( "encoding/json" - "time" - - "github.com/elastic/beats/v7/metricbeat/mb" "github.com/pkg/errors" s "github.com/elastic/beats/v7/libbeat/common/schema" c "github.com/elastic/beats/v7/libbeat/common/schema/mapstriface" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/module/nats/util" ) var ( @@ -57,12 +56,15 @@ func eventMapping(r mb.ReporterV2, content []byte) error { if err != nil { return errors.Wrap(err, "failure applying module schema") } - timestamp, _ := moduleFields.GetValue("now") - moduleFields.Delete("now") + timestamp, err := util.GetNatsTimestamp(moduleFields) + moduleFields.Delete("server.time") + if err != nil { + errors.Wrap(err, "failure parsing server timestamp") + } event := mb.Event{ MetricSetFields: metricSetFields, ModuleFields: moduleFields, - Timestamp: timestamp.(time.Time), + Timestamp: timestamp, } r.Event(event) return nil diff --git a/metricbeat/module/nats/stats/data.go b/metricbeat/module/nats/stats/data.go index 5ac7c13b2a1..9226b11df7b 100644 --- a/metricbeat/module/nats/stats/data.go +++ b/metricbeat/module/nats/stats/data.go @@ -19,7 +19,6 @@ package stats import ( "encoding/json" - "time" "github.com/pkg/errors" @@ -125,12 +124,15 @@ func eventMapping(r mb.ReporterV2, content []byte) error { if err != nil { return errors.Wrap(err, "failure applying module schema") } - timestamp, _ := moduleMetrics.GetValue("now") - moduleMetrics.Delete("now") + timestamp, err := util.GetNatsTimestamp(moduleMetrics) + moduleMetrics.Delete("server.time") + if err != nil { + return errors.Wrap(err, "failure parsing server timestamp") + } evt := mb.Event{ MetricSetFields: metricsetMetrics, ModuleFields: moduleMetrics, - Timestamp: timestamp.(time.Time), + Timestamp: timestamp, } r.Event(evt) return nil diff --git a/metricbeat/module/nats/util/util.go b/metricbeat/module/nats/util/util.go index 8fb34ad2928..da95c12391a 100644 --- a/metricbeat/module/nats/util/util.go +++ b/metricbeat/module/nats/util/util.go @@ -21,6 +21,7 @@ import ( "fmt" "strconv" "strings" + "time" "github.com/pkg/errors" @@ -106,3 +107,15 @@ func UpdateDuration(event common.MapStr, key string) error { } return nil } + +// GetNatsTimestamp gets the timestamp of base level metrics NATS server returns +func GetNatsTimestamp(event common.MapStr) (time.Time, error) { + var timeStamp time.Time + timestamp, _ := event.GetValue("server.time") + timestampString := timestamp.(string) + timeStamp, err := time.Parse(time.RFC3339, timestampString) + if err != nil { + return timeStamp, err + } + return timeStamp, nil +} From d6adb08df94f531ca84a4906a8373a3fc5bf5ad0 Mon Sep 17 00:00:00 2001 From: chrismark Date: Mon, 16 Nov 2020 10:52:27 +0200 Subject: [PATCH 16/18] unmarshal time Signed-off-by: chrismark --- metricbeat/module/nats/connection/data.go | 11 ++++------- metricbeat/module/nats/route/data.go | 12 ++++-------- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/metricbeat/module/nats/connection/data.go b/metricbeat/module/nats/connection/data.go index 3317ddeb9b9..cc2b26dfbb5 100644 --- a/metricbeat/module/nats/connection/data.go +++ b/metricbeat/module/nats/connection/data.go @@ -19,6 +19,7 @@ package connection import ( "encoding/json" + "time" "github.com/pkg/errors" @@ -31,8 +32,7 @@ import ( var ( moduleSchema = s.Schema{ "server": s.Object{ - "id": c.Str("server_id"), - "time": c.Str("now"), + "id": c.Str("server_id"), }, } connectionsSchema = s.Schema{ @@ -54,7 +54,7 @@ var ( // Connections stores connections related information type Connections struct { - Now string `json:"now"` + Now time.Time `json:"now"` ServerID string `json:"server_id"` Connections []map[string]interface{} `json:"connections,omitempty"` } @@ -81,15 +81,12 @@ func eventMapping(content map[string]interface{}, fieldsSchema s.Schema) (mb.Eve return mb.Event{}, errors.Wrap(err, "error applying module schema") } - timestamp, err := util.GetNatsTimestamp(moduleFields) - moduleFields.Delete("server.time") if err != nil { return mb.Event{}, errors.Wrap(err, "failure parsing server timestamp") } event := mb.Event{ MetricSetFields: fields, ModuleFields: moduleFields, - Timestamp: timestamp, } return event, nil } @@ -105,12 +102,12 @@ func eventsMapping(r mb.ReporterV2, content []byte) error { for _, con := range connections.Connections { var evt mb.Event con["server_id"] = connections.ServerID - con["now"] = connections.Now evt, err = eventMapping(con, connectionsSchema) if err != nil { r.Error(errors.Wrap(err, "error mapping connection event")) continue } + evt.Timestamp = connections.Now if !r.Event(evt) { return nil } diff --git a/metricbeat/module/nats/route/data.go b/metricbeat/module/nats/route/data.go index 3b17bb98f80..932933f9d9d 100644 --- a/metricbeat/module/nats/route/data.go +++ b/metricbeat/module/nats/route/data.go @@ -19,20 +19,19 @@ package route import ( "encoding/json" + "time" "github.com/pkg/errors" s "github.com/elastic/beats/v7/libbeat/common/schema" c "github.com/elastic/beats/v7/libbeat/common/schema/mapstriface" "github.com/elastic/beats/v7/metricbeat/mb" - "github.com/elastic/beats/v7/metricbeat/module/nats/util" ) var ( moduleSchema = s.Schema{ "server": s.Object{ - "id": c.Str("server_id"), - "time": c.Str("now"), + "id": c.Str("server_id"), }, } routesSchema = s.Schema{ @@ -54,7 +53,7 @@ var ( // Routes stores routes related information type Routes struct { - Now string `json:"now"` + Now time.Time `json:"now"` ServerID string `json:"server_id"` Routes []map[string]interface{} `json:"routes,omitempty"` } @@ -71,15 +70,12 @@ func eventMapping(content map[string]interface{}, fieldsSchema s.Schema) (mb.Eve return mb.Event{}, errors.Wrap(err, "error applying module schema") } - timestamp, err := util.GetNatsTimestamp(moduleFields) - moduleFields.Delete("server.time") if err != nil { return mb.Event{}, errors.Wrap(err, "failure parsing server timestamp") } event := mb.Event{ MetricSetFields: fields, ModuleFields: moduleFields, - Timestamp: timestamp, } return event, nil } @@ -95,12 +91,12 @@ func eventsMapping(r mb.ReporterV2, content []byte) error { for _, con := range connections.Routes { var evt mb.Event con["server_id"] = connections.ServerID - con["now"] = connections.Now evt, err = eventMapping(con, routesSchema) if err != nil { r.Error(errors.Wrap(err, "error mapping connection event")) continue } + evt.Timestamp = connections.Now if !r.Event(evt) { return nil } From 9e290bd882dbaa5d5e0e7bff19a9cd79620407de Mon Sep 17 00:00:00 2001 From: chrismark Date: Mon, 16 Nov 2020 17:32:01 +0200 Subject: [PATCH 17/18] Add deprecate note about server.time Signed-off-by: chrismark --- metricbeat/docs/fields.asciidoc | 13 +++++++++++++ metricbeat/module/nats/_meta/fields.yml | 5 +++++ metricbeat/module/nats/connections/data.go | 1 - metricbeat/module/nats/fields.go | 2 +- metricbeat/module/nats/routes/data.go | 1 - metricbeat/module/nats/stats/data.go | 1 - 6 files changed, 19 insertions(+), 4 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 15ad16931fc..d82cf047af4 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -35474,6 +35474,19 @@ type: keyword -- +*`nats.server.time`*:: ++ +-- + +deprecated:[8.0.0] + +Server time of metric creation + + +type: date + +-- + [float] === connection diff --git a/metricbeat/module/nats/_meta/fields.yml b/metricbeat/module/nats/_meta/fields.yml index 13e3f263d0a..b70dc9ab0a1 100644 --- a/metricbeat/module/nats/_meta/fields.yml +++ b/metricbeat/module/nats/_meta/fields.yml @@ -13,3 +13,8 @@ type: keyword description: > The server ID + - name: server.time + type: date + deprecated: 8.0.0 + description: > + Server time of metric creation diff --git a/metricbeat/module/nats/connections/data.go b/metricbeat/module/nats/connections/data.go index 0be0ee8e05d..241f46f77d2 100644 --- a/metricbeat/module/nats/connections/data.go +++ b/metricbeat/module/nats/connections/data.go @@ -58,7 +58,6 @@ func eventMapping(r mb.ReporterV2, content []byte) error { return errors.Wrap(err, "failure applying module schema") } timestamp, err := util.GetNatsTimestamp(moduleFields) - moduleFields.Delete("server.time") if err != nil { return errors.Wrap(err, "failure parsing server timestamp") } diff --git a/metricbeat/module/nats/fields.go b/metricbeat/module/nats/fields.go index 357a25c578c..bb1be3f6dcd 100644 --- a/metricbeat/module/nats/fields.go +++ b/metricbeat/module/nats/fields.go @@ -32,5 +32,5 @@ func init() { // AssetNats returns asset data. // This is the base64 encoded gzipped contents of module/nats. func AssetNats() string { - return "eJzsms1y4zYMx+95Csye2sPmAXzozE572UNzaNKzlyZhi7MiqQVBZ71P36E+HH1QluzYidOxjpYE/AiQxB+UP8N33C3ACvZ3AKw5xwV8evjy9PjpDkChl6QL1s4u4I87ACifhL+dCjneARDmKDwuYCPuANYac+UX5XOfwQqDe8vx4l0RnyQXivqXhP14fYsvfQPpLAttPXgWrD1r6YEzwfCMhEAoFKzJGXh4cdEmaFN4pC3SvVb7Ow3Od9w9O2r/PgIVr6cMa1Pw9a+BE+msRRnfG3hpD3rCx5/NqMtIv9iMsRaMCgwyaelbL3Wz0Fz9WHSzYrBzYzwcE7hNWKJFcGvgDNOBaCUjrPbmfJJCW8YN0gkUwayQIkfHCWgLnGk/RVagVdpulqsdY5osd3bTu7F2ZAQvIPXSUci198pQFctp5FCwHsnlAVYVSAxMzsItkLRTvTSD9hAK+M2j/D1JqVWOyysBjSyHUPu+0it4JoUwLliOydRWOhPTqwSL/mgTS7XNZNB7sRnMroMRnAF4AHLUY4OUmu2TPIeWyut4hxYbUBf4Mgl1gTfu2hO6h/wgCd3zdi0OS6y/+hrLjkV+qfImAxFazncgJOstgsw1Wh5GjFxgPF+sSnNnDdP1qgFC4xiXuq+GXqmUKrOgFTgLz5mWWVmkqsi+UKECdgdVite/ZtfTk4RIiZTeVQtH6W315GRFi42A7E/aVnXub9m11xN2cl3McHcTAx+ndtzEwESA/mdiYLA9XV1tu6gEINxoz0io+pHYl1YWfMb4bAWdNTrv2LrWhzhTbatBc/+WZwG1qot+He0gxEUaU/3w5ekRCnISfXpfk45GIM8w0XK30VLklZMyfG0eoGA9jEgoWYQklZciR7Vc5070N+QmhAWSRNu/e0QQZRE6EfQHI1gu1WW6v4CzKavSS74DSVguo75qh4H8vFhW29tHwtNNAd0U0E0BHcnzVgqoDepz9xz3LR8M0mV2regC9i5aZw/OlqUgyZUxp3u1E2datNfIjn1NrOr4kbON8MeyL42mEWdgwr7D/xHQc/vDVWqyjGC2UQPp5P0p0JmwPWBQ2jPpVSjPwpwF46xmR3EC/vvP18cRG4cGAn29/mv0sYm1deSwYDCLM81Rp9QY7bGNR7l78vj+7CXFsehbQe9PHiGOBfdh5d+fvKQ4Fp2c69fZtyePEGPgU8evr+wVuweuH6anrg/Tx8+kX2SxRxpUkPOU28o2uAKrTrc8sO4GNNeeR5uG7fx+9ciOIdo+jcsIltmFuFgb9CAqH7GnX7tgVRRjIDp86RZVyAzvR8/Sz9Jp+ZAzeOT60wNWTg/gZJqXJDiNdMnmOTptkgWCEFZYfgmICxe3WP/jZgp/LawLfG/Ez/PG1Iif2gQDlf1K/SlY7eYCiW1/olVAyoVV3g/2nO5niyQ2OMbzXwAAAP//PYJuZw==" + return "eJzsmjuT2zgMx/v9FJhUd0U8KW9c3EzmrklxW9zu1Q5NwhYnIqmAoDfOp7+hHl49KEv22vvIrEpLBn4CSOIP2B/hG+6XYAX7GwDWnOMSPtx+vr/7cAOg0EvSBWtnl/DnDQCUT8I/ToUcbwAIcxQel7AVNwAbjbnyy/K5j2CFwYPlePG+iE+SC0X9ScJ+vL7GL30F6SwLbT14Fqw9a+mBM8HwgIRAKBRsyBm4fXTRJmhTeKQd0kKrw50G5xvuHxy1Px+Bitd9hrUp+PL3mBPWBgdulGDs+CgIpWBUS/hj8WnxaZ7/u8p39ABuAwaZtARJKOLTAyDprEXZuZXKwoTTv5o0lKl/tBmTH9+gxvCtL3WXRXP1k9NdJp2gHcvPBG6Tp2gxBokzTAeilbiwPpjzSQptGbdIZ1AEs0aKHB0noC1wpv0UWYFWabtdrfeMabLc2W3vxsaREbyE1JdOQq69V4aqWE4jh6K3AWaxqkBiYHIWboGkneqlGbSHUMBvHuXvSUqtcly9EtDIcgy17yu9g2dSCOOC5ZhMbaUzMb1KsOi/bWKrtpkMei+2g9V1NIIzAI9AjnpskFKrfZLn2FZ5Gu/QYgPqAl8noS7w1r32hB4g30hCD7xdi8MS6199jWXHIr9WeZOBCC3nexCS9Q5B5hotDyNGLvBQH50dq9LcRcP0etUAoXGMK91XQ09USpVZ0AqchYdMy6wsUlVkH6lQAbujKsXrn7Pr6VlCpERKn6qFo/SxenayosVGQPYXbas694/s2usZJ7kuZrh7FwNvp3a8i4GJAP1iYmBwPL262nZVCUC41Z6RUPUjcSitLPiC8dkJumh0XrB1radKU22rQbN4zllAreqiX0d7CHGTxlTffr6/g4KcRJ8+16SjEcgLLLTcbbUUeeWkDF+bByhYDyMSShYhSeWlyFGtNrkT/QO5CWGBJNH2754QRFmETgT90QiWW3WV7i/gYsqq9JLvqxkiqoFqh4H8vFpW28dHwtO7AnpXQO8K6ESe51JAbVCfu4d4bvlgkK5zakUXcHDRmj04W5aCJFfGnO7Vzlxp0V4jOw41sarjJ642wu+rvjSaRpyBCYcO/3tAz+1f0lKLZQSzjRpIJ+9Pgc6E7QGD0p5Jr0M5C3MWjLOaHcUF+N+/X+5GbBx7Eejr9Z+jj03srRNfCwarONMcdUqN0X638Sh3J48vz15SnIq+E/Ty5BHiVHAf1v7lyUuKU9HJuX6dfX7yCDEGPjV+fWKv2B24vpmeuh6mj8+kH2WxRxpUkMuU28o2uAKrTrccWHcDmmvPo03Dbn6/emLHEG2fx2UEy+xKXKwNehCVj9jTb1ywKooxEB2+dIsqZIaL0Vn6RTotH3IGj1z/9ICV0yM4meYVCU4jXbN5jk6bZIEghDWWvwTEjYs7rP8CNIW/EdYFXhjx47IxNeKHNsFAZb9SfwrW+7lAYtdfaPXfhVxY5/1gz+l+dkhii2M8/wcAAP//ssWVPg==" } diff --git a/metricbeat/module/nats/routes/data.go b/metricbeat/module/nats/routes/data.go index 2e7212f94f6..02eb686e709 100644 --- a/metricbeat/module/nats/routes/data.go +++ b/metricbeat/module/nats/routes/data.go @@ -57,7 +57,6 @@ func eventMapping(r mb.ReporterV2, content []byte) error { return errors.Wrap(err, "failure applying module schema") } timestamp, err := util.GetNatsTimestamp(moduleFields) - moduleFields.Delete("server.time") if err != nil { errors.Wrap(err, "failure parsing server timestamp") } diff --git a/metricbeat/module/nats/stats/data.go b/metricbeat/module/nats/stats/data.go index 9226b11df7b..fb014eea466 100644 --- a/metricbeat/module/nats/stats/data.go +++ b/metricbeat/module/nats/stats/data.go @@ -125,7 +125,6 @@ func eventMapping(r mb.ReporterV2, content []byte) error { return errors.Wrap(err, "failure applying module schema") } timestamp, err := util.GetNatsTimestamp(moduleMetrics) - moduleMetrics.Delete("server.time") if err != nil { return errors.Wrap(err, "failure parsing server timestamp") } From 9943cfebc478b3a12c986597395abeb46dfc94c8 Mon Sep 17 00:00:00 2001 From: chrismark Date: Tue, 17 Nov 2020 10:37:52 +0200 Subject: [PATCH 18/18] fix Signed-off-by: chrismark --- x-pack/packetbeat/packetbeat.reference.yml | 2 +- x-pack/packetbeat/packetbeat.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/packetbeat/packetbeat.reference.yml b/x-pack/packetbeat/packetbeat.reference.yml index e562c23bab0..76ecadf4a2c 100644 --- a/x-pack/packetbeat/packetbeat.reference.yml +++ b/x-pack/packetbeat/packetbeat.reference.yml @@ -1837,7 +1837,7 @@ setup.kibana: #logging.level: info # Enable debug output for selected components. To enable all selectors use ["*"] -# Other available selectors are "beat", "publish", "service" +# Other available selectors are "beat", "publisher", "service" # Multiple selectors can be chained. #logging.selectors: [ ] diff --git a/x-pack/packetbeat/packetbeat.yml b/x-pack/packetbeat/packetbeat.yml index 31c229b1ef7..f7e19b268b8 100644 --- a/x-pack/packetbeat/packetbeat.yml +++ b/x-pack/packetbeat/packetbeat.yml @@ -223,7 +223,7 @@ processors: # At debug level, you can selectively enable logging only for some components. # To enable all selectors use ["*"]. Examples of other selectors are "beat", -# "publish", "service". +# "publisher", "service". #logging.selectors: ["*"] # ============================= X-Pack Monitoring ==============================