From 0e82ebee208a646d73be56429e3c3d9c1f34689a Mon Sep 17 00:00:00 2001 From: sayden Date: Fri, 25 Jan 2019 15:09:36 +0100 Subject: [PATCH 01/10] Atomic commit --- metricbeat/docs/fields.asciidoc | 135 ++++++++++ metricbeat/docs/modules/zookeeper.asciidoc | 8 +- .../docs/modules/zookeeper/server.asciidoc | 21 ++ metricbeat/docs/modules_list.asciidoc | 3 +- metricbeat/include/list.go | 1 + metricbeat/metricbeat.reference.yml | 2 +- .../zookeeper/_meta/config.reference.yml | 2 +- metricbeat/module/zookeeper/_meta/config.yml | 1 + .../module/zookeeper/_meta/docs.asciidoc | 2 +- metricbeat/module/zookeeper/fields.go | 2 +- .../module/zookeeper/server/_meta/data.json | 45 ++++ .../zookeeper/server/_meta/docs.asciidoc | 16 ++ .../module/zookeeper/server/_meta/fields.yml | 56 ++++ .../zookeeper/server/data_integration_test.go | 38 +++ metricbeat/module/zookeeper/server/server.go | 249 ++++++++++++++++++ .../server/server_integration_test.go | 72 +++++ .../module/zookeeper/server/server_test.go | 48 ++++ metricbeat/modules.d/zookeeper.yml.disabled | 1 + metricbeat/tests/system/test_zookeeper.py | 28 ++ 19 files changed, 723 insertions(+), 7 deletions(-) create mode 100644 metricbeat/docs/modules/zookeeper/server.asciidoc create mode 100644 metricbeat/module/zookeeper/server/_meta/data.json create mode 100644 metricbeat/module/zookeeper/server/_meta/docs.asciidoc create mode 100644 metricbeat/module/zookeeper/server/_meta/fields.yml create mode 100644 metricbeat/module/zookeeper/server/data_integration_test.go create mode 100644 metricbeat/module/zookeeper/server/server.go create mode 100644 metricbeat/module/zookeeper/server/server_integration_test.go create mode 100644 metricbeat/module/zookeeper/server/server_test.go diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 979820962de..30faf3f9e99 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -25324,3 +25324,138 @@ Number of znodes reported by the local ZooKeeper process. -- +[float] +== server fields + +server contains the metrics reported by the four-letter `srvr` command. + + +*`zookeeper.server.connections`*:: ++ +-- +type: long + +Connections established by the server + +-- + + +*`zookeeper.server.latency.avg`*:: ++ +-- +type: long + +Average latency of the server + +-- + +*`zookeeper.server.latency.max`*:: ++ +-- +type: long + +Max latency reached by the server + +-- + +*`zookeeper.server.latency.min`*:: ++ +-- +type: long + +Minimum latency that has been reached by the serverreached + +-- + +*`zookeeper.server.mode`*:: ++ +-- +type: keyword + +Server mode + +-- + +*`zookeeper.server.node_count`*:: ++ +-- +type: long + +Total number of nodes + +-- + +*`zookeeper.server.outstanding`*:: ++ +-- +type: long + +Outstanding + +-- + +*`zookeeper.server.received`*:: ++ +-- +type: long + +Received requests to the server + +-- + +*`zookeeper.server.sent`*:: ++ +-- +type: long + +Requests sent by the server + +-- + + +*`zookeeper.server.version.id`*:: ++ +-- +type: string + +Zookeeper version running + +-- + +*`zookeeper.server.version.date`*:: ++ +-- +type: date + +Date of the Zookeeper release in use + +-- + + +*`zookeeper.server.xid.original`*:: ++ +-- +type: keyword + +Original value of the Zookeeper transaction ID + +-- + +*`zookeeper.server.xid.count`*:: ++ +-- +type: long + +Total transactions of the leader in epoch + +-- + +*`zookeeper.server.xid.epoch`*:: ++ +-- +type: long + +Epoch value of the Zookeeper transaction ID + +-- + diff --git a/metricbeat/docs/modules/zookeeper.asciidoc b/metricbeat/docs/modules/zookeeper.asciidoc index 30a7fa3cf48..6840a8c7161 100644 --- a/metricbeat/docs/modules/zookeeper.asciidoc +++ b/metricbeat/docs/modules/zookeeper.asciidoc @@ -6,7 +6,7 @@ This file is generated! See scripts/docs_collector.py == ZooKeeper module The ZooKeeper module fetches statistics from the ZooKeeper service. The default -metricset is `mntr`. +metricset is `mntr`. You can also activate `server` metricset. [float] === Compatibility @@ -26,7 +26,7 @@ in <>. Here is an example configuration: metricbeat.modules: - module: zookeeper enabled: true - metricsets: ["mntr"] + metricsets: ["mntr", "server"] period: 10s hosts: ["localhost:2181"] ---- @@ -38,5 +38,9 @@ The following metricsets are available: * <> +* <> + include::zookeeper/mntr.asciidoc[] +include::zookeeper/server.asciidoc[] + diff --git a/metricbeat/docs/modules/zookeeper/server.asciidoc b/metricbeat/docs/modules/zookeeper/server.asciidoc new file mode 100644 index 00000000000..18097850acf --- /dev/null +++ b/metricbeat/docs/modules/zookeeper/server.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-metricset-zookeeper-server]] +=== ZooKeeper server metricset + +include::../../../module/zookeeper/server/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/zookeeper/server/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 37a11c87765..82b0a2cf43e 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -159,7 +159,8 @@ This file is generated! See scripts/docs_collector.py .2+| .2+| |<> beta[] |<> |<> |image:./images/icon-no.png[No prebuilt dashboards] | -.1+| .1+| |<> +.2+| .2+| |<> +|<> |================================ -- diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go index 0a873f9080f..64c1ffe0be0 100644 --- a/metricbeat/include/list.go +++ b/metricbeat/include/list.go @@ -177,4 +177,5 @@ import ( _ "github.com/elastic/beats/metricbeat/module/windows/service" _ "github.com/elastic/beats/metricbeat/module/zookeeper" _ "github.com/elastic/beats/metricbeat/module/zookeeper/mntr" + _ "github.com/elastic/beats/metricbeat/module/zookeeper/server" ) diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 6c89c863f81..26601043242 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -689,7 +689,7 @@ metricbeat.modules: #------------------------------ ZooKeeper Module ----------------------------- - module: zookeeper enabled: true - metricsets: ["mntr"] + metricsets: ["mntr", "server"] period: 10s hosts: ["localhost:2181"] diff --git a/metricbeat/module/zookeeper/_meta/config.reference.yml b/metricbeat/module/zookeeper/_meta/config.reference.yml index 04742813c55..51b3e0b83bd 100644 --- a/metricbeat/module/zookeeper/_meta/config.reference.yml +++ b/metricbeat/module/zookeeper/_meta/config.reference.yml @@ -1,5 +1,5 @@ - module: zookeeper enabled: true - metricsets: ["mntr"] + metricsets: ["mntr", "server"] period: 10s hosts: ["localhost:2181"] diff --git a/metricbeat/module/zookeeper/_meta/config.yml b/metricbeat/module/zookeeper/_meta/config.yml index 17fb7135973..d6701a8b80f 100644 --- a/metricbeat/module/zookeeper/_meta/config.yml +++ b/metricbeat/module/zookeeper/_meta/config.yml @@ -1,5 +1,6 @@ - module: zookeeper #metricsets: # - mntr + # - server period: 10s hosts: ["localhost:2181"] diff --git a/metricbeat/module/zookeeper/_meta/docs.asciidoc b/metricbeat/module/zookeeper/_meta/docs.asciidoc index 34b3f950010..abd737b4893 100644 --- a/metricbeat/module/zookeeper/_meta/docs.asciidoc +++ b/metricbeat/module/zookeeper/_meta/docs.asciidoc @@ -1,5 +1,5 @@ The ZooKeeper module fetches statistics from the ZooKeeper service. The default -metricset is `mntr`. +metricset is `mntr`. You can also activate `server` metricset. [float] === Compatibility diff --git a/metricbeat/module/zookeeper/fields.go b/metricbeat/module/zookeeper/fields.go index 8292d50c21d..02d0f9c24e0 100644 --- a/metricbeat/module/zookeeper/fields.go +++ b/metricbeat/module/zookeeper/fields.go @@ -32,5 +32,5 @@ func init() { // AssetZookeeper returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/zookeeper. func AssetZookeeper() string { - return "eJy0lsGyozYQRff+iq7ZP3+AF6nKOpVZZJkN0xYXo7JQE6mxzfv6lMBgcOCVJ4O1ciGr7+mrllofdEZ7oE+RM1Aj7IjUqsOBvv0t8kf37duOKEc0wdZqxR/otx0R0ThPFTRYE8mIczCKnI4taQkqpAkfDqrpT+KtSrD+REaqin0e9zuiWErQzIgv7OlABbuIHVGAA0cc6MQ7osLC5fHQqX6Q5wpz4jS0rdPfgzT1/csCcho/xpU/yIhXtj52sEMWAbWEexJjjuPyKXsaU7YpX+X1sWgJ7wvEDjMFeIHw2eZ+3SzUHXk/+Tj3dxjPuUzzKSVq+jWbHPI6o71KyJ/mvshuXj9D7P2iMtd1kJutWJHlrJxF+7mM4cSffo7h90dsSmFJiglXUltmcqzwpt3z5Vnwf5NcEPiEITIdoVfAE3xEdXToTIpkPVXWORthZFKFczrUJSoEdjEz0njdCPF7Ux0RkkWjAH16ybGCUYhzckWIm+uPkSkmj+7HwDQhwGvn1DJRxbessA7ZoCRhU4f+5Jutmor8g9Q60EMtEnfkORUSOuhHtdVBDOKKl0PBVXzbmHUouJcqa8SwfisM638ewzdVxs5ekBqHh0nBt6+ySWxSmeyUlqzEYaw411JHswwrNfxbq+77erUl7eF4vFhp0mhU9rn1pyzgnwZRt7d2IkKDSG+rB/Lk9hED5qPPGddERVjmrtmcoXEfYGAveG5Gv8788M9DrxLOgyQNkl+DRbxhx9ehktwKEPrdja0322/tPTp10dNOGg6hTTs+P0RjYxsv82XaiHBByKKybvj4+Esc0l0zPxgD0wpJ6w3y7H1trReYdLfxoXct4YkpNduZIWQjOXC+diguCNHK8l3NzvJzDjVr2VtuDfbLq19+1t2XE/ucjo11OUUN/Ynv01pmvrKa8k0XZBcbcXJxRyhJXwdODLtXr8nu4fMmyv5R9Z9n/irfvwEAAP//BzvbMw==" + return "eJy0mEuP4zYMx+/5FMRe9rT5ADkUKLo9FMXuAtOe9uJlZCYWRhZdic5jPn0hvx9yJq/xaWBH/P9IkSI1X+CVzht4Y34lKsitAESLoQ18+sn8d/Xu0wogJa+cLkSz3cBvKwCA7jvkJE4rD4qNISWUwvYMkhHsuHRfDImEH7HVwk7bPSjOc7SpX68AfMZOEsV2p/cb2KHxtAJwZAg9bWCPK4CdJpP6TaX6BSzmNCYOj5yL8HPHZdG8iSCH51e38hcotoLa+gq29cJRwa5xovOxWz5kD8+QbciXW+kXxfAuIFaYwcAVhNMw1+tGphrk9eDlOL7tM/Vl6E/GXsJfo4+tX690PrJLJ98ueDfOn9b2OqqMReH4pHMUSlIUTLx+i2MYtvvbGH7vbUMwC7wbcAW1OJNBIavOazxMBe8mOZDDPbWWYUtyJLJA1lO+NVQFyYO2kGtjtCfFgywc01GRUU4OjU8Ul1aehPi9zLfkQog6AXiznNICxo6N4SM5/3T9zjL4EKOmDFTpHFmpIhUnyvGU7LShpFVi99QIfcOTzsscbE+qDUGv5gEr8hR27CroPtsKx4r8QizbhMvx9GTWNuGuyqwOQ9tnYWh7O4Yt8wSNPlBoHJZUMP78LBvYBuHBTkmGAui6jDNnqGjisFyQ/dCs+76cbUG7LY8rM41L8YI21XafOPqvJC/PD+1ABFqROqyWKA3R3lKL2fc5ZUov5OLcBapXEr92pEgfaNqMHmfu42dJjuxeW0loJS+DefqAHV+GCnILQFTvrj9b9fytbaxDZT3spELnzmHHx0XUNbbuMI/TenIHcokXlCcOHy9sKJw148JomRZIzlZRmnxcW6sFBt2tG/SOGVlACM12FBDQHgxhulQUB3Jec/ysRqNx6kOBktUh14rW8dVXj3XNckCbwrbUJgUvrq742q048xFFZR90QFa2yQ8Obk8CXOeBYYXm2mOyGnw+iLIeqmZj/rt845K57fbxuV50x4XDu0N1UakvGZ8fumU82s7/GLRs8oJbo33Wc88CM5tsoqrTwC15MbQ3vxlc9GLmyfQ+wLslD2A84D6k+g1PnaIjVJdjN1KeTYQ3Kk/mwGoYyNDDNkz4UZbmZXzS5/TubvFPXQkzE930+Vjl/8uCZnBFqIr9vVHsLqUfC+tb+w+NSi/N4sH8xu8V2d0j0EurESxcUc2X+t491aynMerN1T3tvfT+2f7TqeuKrrQ2trCVTOfzTi+68HEk+TXMB82x0cs3Z3KYfUofz+/TzNv7A8dO77VFs+hJvB5nzvxo7MABTRlxSxxaj9XJD399XcSJVS3cdE7V1TuQ8y1MPYeFwFLBKltkiH+9heHPYOHKSPwfAAD///RUANo=" } diff --git a/metricbeat/module/zookeeper/server/_meta/data.json b/metricbeat/module/zookeeper/server/_meta/data.json new file mode 100644 index 00000000000..e71e01eb661 --- /dev/null +++ b/metricbeat/module/zookeeper/server/_meta/data.json @@ -0,0 +1,45 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "agent": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "event": { + "dataset": "zookeeper.server", + "duration": 115000, + "module": "zookeeper" + }, + "metricset": { + "name": "server" + }, + "service": { + "address": "localhost:2181", + "type": "zookeeper" + }, + "zookeeper": { + "server": { + "connections": 1, + "latency": { + "avg": 0, + "max": 0, + "min": 0 + }, + "mode": "standalone", + "node_count": 4, + "outstanding": 0, + "received": 48, + "sent": 47, + "version": { + "date": "06/29/2018 04:05 GMT", + "id": "3.4.13-2d71af4dbe22557fda74f9a9b4309b15a7487f03" + }, + "xid": { + "zxid": { + "count": 0, + "epoch": 0, + "original": "0x0" + } + } + } + } +} \ No newline at end of file diff --git a/metricbeat/module/zookeeper/server/_meta/docs.asciidoc b/metricbeat/module/zookeeper/server/_meta/docs.asciidoc new file mode 100644 index 00000000000..c0c3f74a8e4 --- /dev/null +++ b/metricbeat/module/zookeeper/server/_meta/docs.asciidoc @@ -0,0 +1,16 @@ +`server` Metricset fetches the data returned by the `srvr` admin keyword. + +* *connections*: Connections established by the server +* *latency.avg*: Average latency of the server +* *latency.max*: Max latency reached by the server +* *latency.min*: Minimum latency that has been reached by the serverreached +* *mode*: Server mode +* *node_count*: Total number of nodes +* *outstanding*: Outstanding +* *received*: Received requests to the server +* *sent*: Requests sent by the server +* *version.id*: Zookeeper version running +* *version.date*: Date of the Zookeeper release in use +* *xid.original*: Original value of the Zookeeper transaction ID +* *xid.count*: Total transactions of the leader in epoch +* *xid.epoch*: Epoch value of the Zookeeper transaction ID diff --git a/metricbeat/module/zookeeper/server/_meta/fields.yml b/metricbeat/module/zookeeper/server/_meta/fields.yml new file mode 100644 index 00000000000..2c900201bdb --- /dev/null +++ b/metricbeat/module/zookeeper/server/_meta/fields.yml @@ -0,0 +1,56 @@ +- name: server + type: group + description: 'server contains the metrics reported by the four-letter `srvr` command.' + release: ga + fields: + - name: connections + type: long + description: Connections established by the server + - name: latency + type: group + fields: + - name: avg + type: long + description: Average latency of the server + - name: max + type: long + description: Max latency reached by the server + - name: min + type: long + description: Minimum latency that has been reached by the serverreached + - name: mode + type: keyword + description: Server mode + - name: node_count + type: long + description: Total number of nodes + - name: outstanding + type: long + description: Outstanding + - name: received + type: long + description: Received requests to the server + - name: sent + type: long + description: Requests sent by the server + - name: version + type: group + fields: + - name: id + type: string + description: Zookeeper version running + - name: date + type: date + description: Date of the Zookeeper release in use + - name: xid + type: group + fields: + - name: original + type: keyword + description: Original value of the Zookeeper transaction ID + - name: count + type: long + description: Total transactions of the leader in epoch + - name: epoch + type: long + description: Epoch value of the Zookeeper transaction ID diff --git a/metricbeat/module/zookeeper/server/data_integration_test.go b/metricbeat/module/zookeeper/server/data_integration_test.go new file mode 100644 index 00000000000..1dc2c34c95c --- /dev/null +++ b/metricbeat/module/zookeeper/server/data_integration_test.go @@ -0,0 +1,38 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package server + +import ( + "github.com/elastic/beats/metricbeat/module/zookeeper" + "testing" + + _ "github.com/denisenkom/go-mssqldb" + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestData(t *testing.T) { + t.Skip("Skipping `data.json` generation test") + + f := mbtest.NewReportingMetricSetV2(t, getDataConfig()) + events, errs := mbtest.ReportingFetchV2(f) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) + + if err := mbtest.WriteEventsReporterV2(f, t, ""); err != nil { + t.Fatal("write", err) + } +} + +func getDataConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "zookeeper", + "metricsets": []string{"server"}, + "hosts": []string{zookeeper.GetZookeeperEnvHost() + ":" + zookeeper.GetZookeeperEnvPort()}, + } +} diff --git a/metricbeat/module/zookeeper/server/server.go b/metricbeat/module/zookeeper/server/server.go new file mode 100644 index 00000000000..c9ab7049b27 --- /dev/null +++ b/metricbeat/module/zookeeper/server/server.go @@ -0,0 +1,249 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/* +Package server fetches metrics from ZooKeeper by using the srvr command + +See the srvr command documentation at +https://zookeeper.apache.org/doc/current/zookeeperAdmin.html + +ZooKeeper srvr Command Output + + $ echo srvr | nc localhost 2181 + Zookeeper version: 3.4.13-2d71af4dbe22557fda74f9a9b4309b15a7487f03, built on 06/29/2018 04:05 GMT +Latency min/avg/max: 1/2/3 +Received: 46 +Sent: 45 +Connections: 1 +Outstanding: 0 +Zxid: 0x700601132 +Mode: standalone +Node count: 4 +Proposal sizes last/min/max: -3/-999/-1 + + +*/ +package server + +import ( + "bufio" + "encoding/binary" + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/mb/parse" + "github.com/elastic/beats/metricbeat/module/zookeeper" + "github.com/pkg/errors" + "io" + "regexp" + "strconv" + "strings" +) + +var latencyCapturer = regexp.MustCompile(`(\d+)/(\d+)/(\d+)`) +var ipCapturer = regexp.MustCompile(`\d+\.\d+\.\d+\.\d+`) +var thatNumberCapturer = regexp.MustCompile(`\[(\d+)\]`) +var portCapturer = regexp.MustCompile(`:(\d+)\[`) +var dataCapturer = regexp.MustCompile(`(\w+)=(\d+)`) +var fieldsCapturer = regexp.MustCompile(`^([a-zA-Z\s]+):\s(\d+)`) +var versionCapturer = regexp.MustCompile(`:\s(.*),`) +var dateCapturer = regexp.MustCompile(`built on (.*)`) + +func init() { + mb.Registry.MustAddMetricSet("zookeeper", "server", New, + mb.WithHostParser(parse.PassThruHostParser), + mb.DefaultMetricSet(), + ) +} + +// MetricSet for fetching ZooKeeper health metrics. +type MetricSet struct { + mb.BaseMetricSet +} + +// New creates new instance of MetricSet. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + return &MetricSet{ + BaseMetricSet: base, + }, nil +} + +// Fetch fetches metrics from ZooKeeper by making a tcp connection to the +// command port and sending the "srvr" command and parsing the output. +func (m *MetricSet) Fetch(reporter mb.ReporterV2) { + outputReader, err := zookeeper.RunCommand("srvr", m.Host(), m.Module().Config().Timeout) + if err != nil { + reporter.Error(errors.Wrap(err, "srvr command failed")) + return + } + + metricsetFields, err := parseSrvr(outputReader) + if err != nil { + reporter.Error(err) + return + } + + reporter.Event(mb.Event{ + MetricSetFields: metricsetFields, + }) +} + +func parseSrvr(i io.Reader) (common.MapStr, error) { + scanner := bufio.NewScanner(i) + + output := common.MapStr{} + + //Get version + ok := scanner.Scan() + if !ok { + return nil, errors.New("no initial successful scan, aborting") + } + output.Put("version.id", versionCapturer.FindStringSubmatch(scanner.Text())[1]) + output.Put("version.date", dateCapturer.FindStringSubmatch(scanner.Text())[1]) + + for scanner.Scan() { + line := scanner.Text() + + if strings.Contains(line, "Zxid") { + xid, err := parseXid(line) + if err != nil { + return nil, errors.Wrap(err, "error parsing xid line") + } + output.Put("xid", xid) + + continue + } + + if strings.Contains(line, "Latency") { + latency, err := parseLatencyLine(line) + if err != nil { + return nil, errors.Wrap(err, "error parsing latency values") + } + output.Put("latency", latency) + + continue + } + + if strings.Contains(line, "Proposal sizes") { + proposalSizes, err := parseProposalSizes(line) + if err != nil { + return nil, errors.Wrap(err, "error parsing proposal sizes line") + } + output.Put("proposal_sizes", proposalSizes) + + continue + } + + if strings.Contains(line, "Mode") { + output.Put("mode", strings.Split(line, " ")[1]) + continue + } + + // If code reaches here easy to parse lines or blank lines like the following: + // Received: 46 + // + // Sent: 45 + // Connections: 1 + // Outstanding: 0 + results := fieldsCapturer.FindAllStringSubmatch(line, -1) + if len(results) == 0 { + //probably a blank line + continue + } + + for _, result := range results { + val, err := strconv.ParseInt(result[2], 10, 64) + if err != nil { + return nil, err + } + output.Put(strings.ToLower(strings.Replace(result[1], " ", "_", -1)), val) + } + } + return output, nil +} + +func parseXid(line string) (common.MapStr, error){ + output := common.MapStr{} + + zxidString := strings.Split(line, " ")[1] + zxid, err := strconv.ParseInt(zxidString[2:], 16, 64) + if err != nil { + return nil, err + } + + bs := make([]byte, 8) + binary.BigEndian.PutUint64(bs, uint64(zxid)) + + epoch := bs[:4] + count := bs[4:] + + output.Put("zxid.original", zxidString) + output.Put("zxid.epoch", binary.BigEndian.Uint32(epoch)) + output.Put("zxid.count", binary.BigEndian.Uint32(count)) + + return output, nil +} + +func parseProposalSizes(line string) (common.MapStr, error) { + output := common.MapStr{} + + values := strings.Split(strings.Split(line, " ")[3], "/") + last, err := strconv.ParseInt(values[0], 10, 64) + if err != nil { + return nil, err + } + output.Put("last", last) + + min, err := strconv.ParseInt(values[1], 10, 64) + if err != nil { + return nil, err + } + output.Put("min", min) + + max, err := strconv.ParseInt(values[2], 10, 64) + if err != nil { + return nil, err + } + output.Put("max", max) + + return output, nil +} + +func parseLatencyLine(line string) (common.MapStr, error) { + output := common.MapStr{} + + values := latencyCapturer.FindStringSubmatch(line) + + min, err := strconv.ParseInt(values[1], 10, 64) + if err != nil { + return nil, err + } + output.Put("min", min) + + avg, err := strconv.ParseInt(values[2], 10, 64) + if err != nil { + return nil, err + } + output.Put("avg", avg) + + max, err := strconv.ParseInt(values[3], 10, 64) + if err != nil { + return nil, err + } + output.Put("max", max) + + return output, nil +} diff --git a/metricbeat/module/zookeeper/server/server_integration_test.go b/metricbeat/module/zookeeper/server/server_integration_test.go new file mode 100644 index 00000000000..06b68d34e6b --- /dev/null +++ b/metricbeat/module/zookeeper/server/server_integration_test.go @@ -0,0 +1,72 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration + +package server + +import ( + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/metricbeat/module/zookeeper" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/tests/compose" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestFetch(t *testing.T) { + logp.TestingSetup() + + compose.EnsureUp(t, "zookeeper") + + f := mbtest.NewReportingMetricSetV2(t, getConfig()) + events, errs := mbtest.ReportingFetchV2(f) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) + + for _, event := range events { + t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event) + metricsetFields := event.MetricSetFields + + // Check values + version, ok := metricsetFields["version"].(common.MapStr) + if !ok { + t.Fatal("no version field found") + } + assert.Equal(t, "06/29/2018 04:05 GMT", version["date"]) + assert.Equal(t, "3.4.13-2d71af4dbe22557fda74f9a9b4309b15a7487f03", version["id"]) + + received := metricsetFields["received"].(int64) + assert.True(t, received >= 0 ) + + nodeCount := metricsetFields["node_count"].(int64) + assert.True(t, nodeCount >= 1) + } +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "zookeeper", + "metricsets": []string{"server"}, + "hosts": []string{zookeeper.GetZookeeperEnvHost() + ":" + zookeeper.GetZookeeperEnvPort()}, + } +} diff --git a/metricbeat/module/zookeeper/server/server_test.go b/metricbeat/module/zookeeper/server/server_test.go new file mode 100644 index 00000000000..e08de09423c --- /dev/null +++ b/metricbeat/module/zookeeper/server/server_test.go @@ -0,0 +1,48 @@ +package server + +import ( + "bytes" + "github.com/elastic/beats/libbeat/common" + "github.com/stretchr/testify/assert" + "testing" +) + +var srvrTestInput = `Zookeeper version: 3.4.13-2d71af4dbe22557fda74f9a9b4309b15a7487f03, built on 06/29/2018 04:05 GMT +Latency min/avg/max: 1/2/3 +Received: 46 +Sent: 45 +Connections: 1 +Outstanding: 0 +Zxid: 0x700601132 +Mode: standalone +Node count: 4 +Proposal sizes last/min/max: -3/-999/-1 +` + +func TestParser(t *testing.T) { + mapStr, err := parseSrvr(bytes.NewReader([]byte(srvrTestInput))) + if err != nil { + t.Fatal(err) + } + + version := mapStr["version"].(common.MapStr) + assert.Equal(t, "06/29/2018 04:05 GMT", version["date"]) + assert.Equal(t, "3.4.13-2d71af4dbe22557fda74f9a9b4309b15a7487f03", version["id"]) + + latency := mapStr["latency"].(common.MapStr) + assert.Equal(t, int64(1), latency["min"]) + assert.Equal(t, int64(2), latency["avg"]) + assert.Equal(t, int64(3), latency["max"]) + + assert.Equal(t, int64(46), mapStr["received"]) + assert.Equal(t, int64(45), mapStr["sent"]) + assert.Equal(t, int64(1), mapStr["connections"]) + assert.Equal(t, int64(0), mapStr["outstanding"]) + assert.Equal(t, "standalone", mapStr["mode"]) + assert.Equal(t, int64(4), mapStr["node_count"]) + + proposalSizes := mapStr["proposal_sizes"].(common.MapStr) + assert.Equal(t, int64(-3), proposalSizes["last"]) + assert.Equal(t, int64(-999), proposalSizes["min"]) + assert.Equal(t, int64(-1), proposalSizes["max"]) +} diff --git a/metricbeat/modules.d/zookeeper.yml.disabled b/metricbeat/modules.d/zookeeper.yml.disabled index 253de00d3e2..7d44efb938e 100644 --- a/metricbeat/modules.d/zookeeper.yml.disabled +++ b/metricbeat/modules.d/zookeeper.yml.disabled @@ -4,5 +4,6 @@ - module: zookeeper #metricsets: # - mntr + # - server period: 10s hosts: ["localhost:2181"] diff --git a/metricbeat/tests/system/test_zookeeper.py b/metricbeat/tests/system/test_zookeeper.py index ee367201239..6e3baadea63 100644 --- a/metricbeat/tests/system/test_zookeeper.py +++ b/metricbeat/tests/system/test_zookeeper.py @@ -50,6 +50,34 @@ def test_output(self): self.assert_fields_are_documented(evt) + @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_output(self): + """ + ZooKeeper server module outputs an event. + """ + self.render_config_template(modules=[{ + "name": "zookeeper", + "metricsets": ["server"], + "hosts": self.get_hosts(), + "period": "5s" + }]) + proc = self.start_beat() + self.wait_until(lambda: self.output_lines() > 0) + proc.check_kill_and_wait() + self.assert_no_logged_warnings() + + output = self.read_output_json() + self.assertEqual(len(output), 1) + evt = output[0] + + self.assertItemsEqual(self.de_dot(ZK_FIELDS), evt.keys()) + zk_srvr = evt["zookeeper"]["server"] + + assert zk_srvr["connections"] >= 0 + + self.assert_fields_are_documented(evt) + def get_hosts(self): return [os.getenv('ZOOKEEPER_HOST', 'localhost') + ':' + os.getenv('ZOOKEEPER_PORT', '2181')] From be788ed73bdf0d3b8f57f4671f85b8bed31f5d83 Mon Sep 17 00:00:00 2001 From: sayden Date: Fri, 25 Jan 2019 15:55:13 +0100 Subject: [PATCH 02/10] Headers missing added. --- .../zookeeper/server/data_integration_test.go | 19 ++++++++++++++++--- .../module/zookeeper/server/server_test.go | 17 +++++++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/metricbeat/module/zookeeper/server/data_integration_test.go b/metricbeat/module/zookeeper/server/data_integration_test.go index 1dc2c34c95c..a95dbd0f171 100644 --- a/metricbeat/module/zookeeper/server/data_integration_test.go +++ b/metricbeat/module/zookeeper/server/data_integration_test.go @@ -1,6 +1,19 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. package server diff --git a/metricbeat/module/zookeeper/server/server_test.go b/metricbeat/module/zookeeper/server/server_test.go index e08de09423c..290c9c36cea 100644 --- a/metricbeat/module/zookeeper/server/server_test.go +++ b/metricbeat/module/zookeeper/server/server_test.go @@ -1,3 +1,20 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package server import ( From bb8d3fb932549099d54fc4b8fd265c5f0e4a6ebb Mon Sep 17 00:00:00 2001 From: sayden Date: Fri, 25 Jan 2019 20:41:46 +0100 Subject: [PATCH 03/10] Comments addressed --- metricbeat/docs/fields.asciidoc | 7 +- metricbeat/docs/modules/zookeeper.asciidoc | 2 +- .../module/zookeeper/_meta/docs.asciidoc | 2 +- metricbeat/module/zookeeper/fields.go | 2 +- .../module/zookeeper/server/_meta/data.json | 14 +- .../zookeeper/server/_meta/docs.asciidoc | 6 +- .../module/zookeeper/server/_meta/fields.yml | 21 +- metricbeat/module/zookeeper/server/data.go | 201 ++++++++++++++++++ .../zookeeper/server/data_integration_test.go | 7 +- metricbeat/module/zookeeper/server/server.go | 168 +-------------- .../server/server_integration_test.go | 5 +- .../module/zookeeper/server/server_test.go | 10 +- .../performance/data_integration_test.go | 1 + 13 files changed, 246 insertions(+), 200 deletions(-) create mode 100644 metricbeat/module/zookeeper/server/data.go diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 30faf3f9e99..7df1cc5b9ec 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -25431,8 +25431,7 @@ Date of the Zookeeper release in use -- - -*`zookeeper.server.xid.original`*:: +*`zookeeper.server.zxid`*:: + -- type: keyword @@ -25441,7 +25440,7 @@ Original value of the Zookeeper transaction ID -- -*`zookeeper.server.xid.count`*:: +*`zookeeper.server.count`*:: + -- type: long @@ -25450,7 +25449,7 @@ Total transactions of the leader in epoch -- -*`zookeeper.server.xid.epoch`*:: +*`zookeeper.server.epoch`*:: + -- type: long diff --git a/metricbeat/docs/modules/zookeeper.asciidoc b/metricbeat/docs/modules/zookeeper.asciidoc index 6840a8c7161..32eb71cfc56 100644 --- a/metricbeat/docs/modules/zookeeper.asciidoc +++ b/metricbeat/docs/modules/zookeeper.asciidoc @@ -6,7 +6,7 @@ This file is generated! See scripts/docs_collector.py == ZooKeeper module The ZooKeeper module fetches statistics from the ZooKeeper service. The default -metricset is `mntr`. You can also activate `server` metricset. +metricset is `mntr` and `server`. [float] === Compatibility diff --git a/metricbeat/module/zookeeper/_meta/docs.asciidoc b/metricbeat/module/zookeeper/_meta/docs.asciidoc index abd737b4893..67f71737cad 100644 --- a/metricbeat/module/zookeeper/_meta/docs.asciidoc +++ b/metricbeat/module/zookeeper/_meta/docs.asciidoc @@ -1,5 +1,5 @@ The ZooKeeper module fetches statistics from the ZooKeeper service. The default -metricset is `mntr`. You can also activate `server` metricset. +metricset is `mntr` and `server`. [float] === Compatibility diff --git a/metricbeat/module/zookeeper/fields.go b/metricbeat/module/zookeeper/fields.go index 02d0f9c24e0..66273ad0277 100644 --- a/metricbeat/module/zookeeper/fields.go +++ b/metricbeat/module/zookeeper/fields.go @@ -32,5 +32,5 @@ func init() { // AssetZookeeper returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/zookeeper. func AssetZookeeper() string { - return "eJy0mEuP4zYMx+/5FMRe9rT5ADkUKLo9FMXuAtOe9uJlZCYWRhZdic5jPn0hvx9yJq/xaWBH/P9IkSI1X+CVzht4Y34lKsitAESLoQ18+sn8d/Xu0wogJa+cLkSz3cBvKwCA7jvkJE4rD4qNISWUwvYMkhHsuHRfDImEH7HVwk7bPSjOc7SpX68AfMZOEsV2p/cb2KHxtAJwZAg9bWCPK4CdJpP6TaX6BSzmNCYOj5yL8HPHZdG8iSCH51e38hcotoLa+gq29cJRwa5xovOxWz5kD8+QbciXW+kXxfAuIFaYwcAVhNMw1+tGphrk9eDlOL7tM/Vl6E/GXsJfo4+tX690PrJLJ98ueDfOn9b2OqqMReH4pHMUSlIUTLx+i2MYtvvbGH7vbUMwC7wbcAW1OJNBIavOazxMBe8mOZDDPbWWYUtyJLJA1lO+NVQFyYO2kGtjtCfFgywc01GRUU4OjU8Ul1aehPi9zLfkQog6AXiznNICxo6N4SM5/3T9zjL4EKOmDFTpHFmpIhUnyvGU7LShpFVi99QIfcOTzsscbE+qDUGv5gEr8hR27CroPtsKx4r8QizbhMvx9GTWNuGuyqwOQ9tnYWh7O4Yt8wSNPlBoHJZUMP78LBvYBuHBTkmGAui6jDNnqGjisFyQ/dCs+76cbUG7LY8rM41L8YI21XafOPqvJC/PD+1ABFqROqyWKA3R3lKL2fc5ZUov5OLcBapXEr92pEgfaNqMHmfu42dJjuxeW0loJS+DefqAHV+GCnILQFTvrj9b9fytbaxDZT3spELnzmHHx0XUNbbuMI/TenIHcokXlCcOHy9sKJw148JomRZIzlZRmnxcW6sFBt2tG/SOGVlACM12FBDQHgxhulQUB3Jec/ysRqNx6kOBktUh14rW8dVXj3XNckCbwrbUJgUvrq742q048xFFZR90QFa2yQ8Obk8CXOeBYYXm2mOyGnw+iLIeqmZj/rt845K57fbxuV50x4XDu0N1UakvGZ8fumU82s7/GLRs8oJbo33Wc88CM5tsoqrTwC15MbQ3vxlc9GLmyfQ+wLslD2A84D6k+g1PnaIjVJdjN1KeTYQ3Kk/mwGoYyNDDNkz4UZbmZXzS5/TubvFPXQkzE930+Vjl/8uCZnBFqIr9vVHsLqUfC+tb+w+NSi/N4sH8xu8V2d0j0EurESxcUc2X+t491aynMerN1T3tvfT+2f7TqeuKrrQ2trCVTOfzTi+68HEk+TXMB82x0cs3Z3KYfUofz+/TzNv7A8dO77VFs+hJvB5nzvxo7MABTRlxSxxaj9XJD399XcSJVS3cdE7V1TuQ8y1MPYeFwFLBKltkiH+9heHPYOHKSPwfAAD///RUANo=" + return "eJy0mM2O4zYMx+95CmIve5o8QA4Fim4PRbG7wLSnvXgYmYmFyKIr0fmYpy9kW/6Kncnkw6eBPeL/R4oUqbzAjk4reGfeERXkFgCixdAKvvxi/rt692UBkJJXThei2a7gtwUAQPsdchKnlQfFxpASSmF9AskINly6F0Mi4Z/YamGn7RYU5zna1C8XAD5jJ4liu9HbFWzQeFoAODKEnlawxQXARpNJ/apSfQGLOQ2JwyOnIvy747Jo3kwgh+etXfkGiq2gtr6CjV44Ktg1TrQ+tsv77OHps/X5civdoim8C4gVZjBwBeE4zPW6gakGedl7OYxvfMa+9P3J2Ev4a/Ax+rWj04FdOvp2wbth/kTby0llLArHR52jUJKiYOL1+zSGYbv9HMPvnW0IZoE3Pa6gNs1kUMiq0xL3Y8GbSfbkcEvRMqxJDkQWyHrK14aqIHnQFnJtjPakuJeFQzoqMsrJofGJ4tLKgxB/lPmaXAhRKwDvllOawdiwMXwg5x+u31oGH2LUlIEqnSMrVaSmiXI8JhttKIlK7B4aoe941HmZg+1ItSHo1DxgRZ7Chl0F3WVb4ViRn4llTLgcjw9mjQl3VWa1GNo+CkPbz2PYMk/Q6D2FxmFJBeOPz7KebRDu7ZRkKICuzThzgopmGpYLsk/Nuh/z2Ra0Y3lcmWlcihe0qbbbxNF/JXl5fGh7IhBF6rBaojREe00Rs+tzypReyE1zF6h2JH7pSJHe07gZ3c/cxc+SHNjtoiREyctgnp6w4/NQQW4GiOrd9SerHr+1jXWorIedVOjcKez4sIjaxtYe5tO0ntyeXOIF5YHDxysbCmfNsDAi0wzJySpKk+e1tVqg193aQe+QkQWE0GwHAQHtwRCmc0WxJ+c1T5/VaDSOfShQsjrkWtFyevXVY12zHNCmsC61ScGLqyu+dmua+YCisicdkJVt8r2D25MA13lgWKG59pisBp8nUdZD1dmY/yHfsGQ+d/v4Wi+64cLh3b66qNSXjK933TLubed/9Fo2ecG10T7ruM8CczbZTKqOAzfnRd/e+c3gohdnnozvA7yZ8wCGA+5dqt/x2Co6QnU5dgPls4nwk8qjObAaBjL0sA4T/iRL83J60uf05m7xT10JZyba6fO+yv+XBU3vilAV+0ej2E1KP2fWR/t3jUqvzeLe/MYfFdnNI9Br1AgWrqjmS33vlmrW4xh15uqe9lF6/4o/OrVd0ZXWTi2Mkun5vNOJznwcSH4L80FzbHTyzZkcZp/ST+f3+/HM3atr56fTW23RwB5NOSEvDq3H6oSGv77N9ID7Kqsn4SNAPSMFp6lglc38ajL+cq3un2HllR7/HwAA//8U/OsQ" } diff --git a/metricbeat/module/zookeeper/server/_meta/data.json b/metricbeat/module/zookeeper/server/_meta/data.json index e71e01eb661..d08e3885150 100644 --- a/metricbeat/module/zookeeper/server/_meta/data.json +++ b/metricbeat/module/zookeeper/server/_meta/data.json @@ -19,6 +19,8 @@ "zookeeper": { "server": { "connections": 1, + "count": 0, + "epoch": 0, "latency": { "avg": 0, "max": 0, @@ -27,19 +29,13 @@ "mode": "standalone", "node_count": 4, "outstanding": 0, - "received": 48, - "sent": 47, + "received": 16, + "sent": 15, "version": { "date": "06/29/2018 04:05 GMT", "id": "3.4.13-2d71af4dbe22557fda74f9a9b4309b15a7487f03" }, - "xid": { - "zxid": { - "count": 0, - "epoch": 0, - "original": "0x0" - } - } + "zxid": "0x0" } } } \ No newline at end of file diff --git a/metricbeat/module/zookeeper/server/_meta/docs.asciidoc b/metricbeat/module/zookeeper/server/_meta/docs.asciidoc index c0c3f74a8e4..88e40d9ebb6 100644 --- a/metricbeat/module/zookeeper/server/_meta/docs.asciidoc +++ b/metricbeat/module/zookeeper/server/_meta/docs.asciidoc @@ -11,6 +11,6 @@ * *sent*: Requests sent by the server * *version.id*: Zookeeper version running * *version.date*: Date of the Zookeeper release in use -* *xid.original*: Original value of the Zookeeper transaction ID -* *xid.count*: Total transactions of the leader in epoch -* *xid.epoch*: Epoch value of the Zookeeper transaction ID +* *zxid*: Original value of the Zookeeper transaction ID +* *count*: Total transactions of the leader in epoch +* *epoch*: Epoch value of the Zookeeper transaction ID diff --git a/metricbeat/module/zookeeper/server/_meta/fields.yml b/metricbeat/module/zookeeper/server/_meta/fields.yml index 2c900201bdb..3464ca2138f 100644 --- a/metricbeat/module/zookeeper/server/_meta/fields.yml +++ b/metricbeat/module/zookeeper/server/_meta/fields.yml @@ -42,15 +42,12 @@ - name: date type: date description: Date of the Zookeeper release in use - - name: xid - type: group - fields: - - name: original - type: keyword - description: Original value of the Zookeeper transaction ID - - name: count - type: long - description: Total transactions of the leader in epoch - - name: epoch - type: long - description: Epoch value of the Zookeeper transaction ID + - name: zxid + type: keyword + description: Original value of the Zookeeper transaction ID + - name: count + type: long + description: Total transactions of the leader in epoch + - name: epoch + type: long + description: Epoch value of the Zookeeper transaction ID diff --git a/metricbeat/module/zookeeper/server/data.go b/metricbeat/module/zookeeper/server/data.go new file mode 100644 index 00000000000..6d5386b30cf --- /dev/null +++ b/metricbeat/module/zookeeper/server/data.go @@ -0,0 +1,201 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package server + +import ( + "bufio" + "encoding/binary" + "io" + "regexp" + "strconv" + "strings" + + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common" +) + +var latencyCapturer = regexp.MustCompile(`(\d+)/(\d+)/(\d+)`) +var ipCapturer = regexp.MustCompile(`\d+\.\d+\.\d+\.\d+`) +var thatNumberCapturer = regexp.MustCompile(`\[(\d+)\]`) +var portCapturer = regexp.MustCompile(`:(\d+)\[`) +var dataCapturer = regexp.MustCompile(`(\w+)=(\d+)`) +var fieldsCapturer = regexp.MustCompile(`^([a-zA-Z\s]+):\s(\d+)`) +var versionCapturer = regexp.MustCompile(`:\s(.*),`) +var dateCapturer = regexp.MustCompile(`built on (.*)`) + +func parseSrvr(i io.Reader) (common.MapStr, error) { + scanner := bufio.NewScanner(i) + + //Get version + ok := scanner.Scan() + + if !ok { + return nil, errors.New("no initial successful scan, aborting") + } + + output := common.MapStr{} + + output.Put("service.version", versionCapturer.FindStringSubmatch(scanner.Text())[1]) + output.Put("version.date", dateCapturer.FindStringSubmatch(scanner.Text())[1]) + + for scanner.Scan() { + line := scanner.Text() + + if strings.Contains(line, "Zxid") { + xid, err := parseZxid(line) + if err != nil { + err = errors.Wrap(err, "error parsing xid line") + logger.Debug(err.Error()) + continue + } + + output.Update(xid) + + continue + } + + if strings.Contains(line, "Latency") { + latency, err := parseLatencyLine(line) + if err != nil { + err = errors.Wrap(err, "error parsing latency values") + logger.Debug(err.Error()) + continue + } + + output.Put("latency", latency) + + continue + } + + if strings.Contains(line, "Proposal sizes") { + proposalSizes, err := parseProposalSizes(line) + if err != nil { + err = errors.Wrap(err, "error parsing proposal sizes line") + logger.Debug(err.Error()) + continue + } + + output.Put("proposal_sizes", proposalSizes) + + continue + } + + if strings.Contains(line, "Mode") { + output.Put("mode", strings.Split(line, " ")[1]) + continue + } + + // If code reaches here easy to parse lines or blank lines like the following: + // Received: 46 + // + // Sent: 45 + // Connections: 1 + // Outstanding: 0 + results := fieldsCapturer.FindAllStringSubmatch(line, -1) + if len(results) == 0 { + //probably a blank line + continue + } + + for _, result := range results { + val, err := strconv.ParseInt(result[2], 10, 64) + if err != nil { + err = errors.Wrapf(err, "error trying to parse '%s'", result) + logger.Debug(err.Error()) + continue + } + output.Put(strings.ToLower(strings.Replace(result[1], " ", "_", -1)), val) + } + } + + return output, nil +} + +func parseZxid(line string) (common.MapStr, error) { + output := common.MapStr{} + + zxidString := strings.Split(line, " ")[1] + zxid, err := strconv.ParseInt(zxidString[2:], 16, 64) + if err != nil { + return nil, err + } + + bs := make([]byte, 8) + binary.BigEndian.PutUint64(bs, uint64(zxid)) + + epoch := bs[:4] + count := bs[4:] + + output.Put("zxid", zxidString) + output.Put("epoch", binary.BigEndian.Uint32(epoch)) + output.Put("count", binary.BigEndian.Uint32(count)) + + return output, nil +} + +func parseProposalSizes(line string) (common.MapStr, error) { + output := common.MapStr{} + + values := strings.Split(strings.Split(line, " ")[3], "/") + last, err := strconv.ParseInt(values[0], 10, 64) + if err != nil { + return nil, err + } + output.Put("last", last) + + min, err := strconv.ParseInt(values[1], 10, 64) + if err != nil { + return nil, err + } + output.Put("min", min) + + max, err := strconv.ParseInt(values[2], 10, 64) + if err != nil { + return nil, err + } + output.Put("max", max) + + return output, nil +} + +func parseLatencyLine(line string) (common.MapStr, error) { + output := common.MapStr{} + + values := latencyCapturer.FindStringSubmatch(line) + + min, err := strconv.ParseInt(values[1], 10, 64) + if err != nil { + return nil, err + } + output.Put("min", min) + + avg, err := strconv.ParseInt(values[2], 10, 64) + if err != nil { + return nil, err + } + output.Put("avg", avg) + + max, err := strconv.ParseInt(values[3], 10, 64) + if err != nil { + return nil, err + } + output.Put("max", max) + + return output, nil +} diff --git a/metricbeat/module/zookeeper/server/data_integration_test.go b/metricbeat/module/zookeeper/server/data_integration_test.go index a95dbd0f171..2fa7013e238 100644 --- a/metricbeat/module/zookeeper/server/data_integration_test.go +++ b/metricbeat/module/zookeeper/server/data_integration_test.go @@ -15,12 +15,15 @@ // specific language governing permissions and limitations // under the License. +// +build integration + package server import ( - "github.com/elastic/beats/metricbeat/module/zookeeper" "testing" + "github.com/elastic/beats/metricbeat/module/zookeeper" + _ "github.com/denisenkom/go-mssqldb" "github.com/stretchr/testify/assert" @@ -28,7 +31,7 @@ import ( ) func TestData(t *testing.T) { - t.Skip("Skipping `data.json` generation test") + //t.Skip("Skipping `data.json` generation test") f := mbtest.NewReportingMetricSetV2(t, getDataConfig()) events, errs := mbtest.ReportingFetchV2(f) diff --git a/metricbeat/module/zookeeper/server/server.go b/metricbeat/module/zookeeper/server/server.go index c9ab7049b27..06e02577fe6 100644 --- a/metricbeat/module/zookeeper/server/server.go +++ b/metricbeat/module/zookeeper/server/server.go @@ -40,27 +40,16 @@ Proposal sizes last/min/max: -3/-999/-1 package server import ( - "bufio" - "encoding/binary" - "github.com/elastic/beats/libbeat/common" + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/mb/parse" "github.com/elastic/beats/metricbeat/module/zookeeper" - "github.com/pkg/errors" - "io" - "regexp" - "strconv" - "strings" ) -var latencyCapturer = regexp.MustCompile(`(\d+)/(\d+)/(\d+)`) -var ipCapturer = regexp.MustCompile(`\d+\.\d+\.\d+\.\d+`) -var thatNumberCapturer = regexp.MustCompile(`\[(\d+)\]`) -var portCapturer = regexp.MustCompile(`:(\d+)\[`) -var dataCapturer = regexp.MustCompile(`(\w+)=(\d+)`) -var fieldsCapturer = regexp.MustCompile(`^([a-zA-Z\s]+):\s(\d+)`) -var versionCapturer = regexp.MustCompile(`:\s(.*),`) -var dateCapturer = regexp.MustCompile(`built on (.*)`) +var logger = logp.NewLogger("zookeeper.server") func init() { mb.Registry.MustAddMetricSet("zookeeper", "server", New, @@ -100,150 +89,3 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) { MetricSetFields: metricsetFields, }) } - -func parseSrvr(i io.Reader) (common.MapStr, error) { - scanner := bufio.NewScanner(i) - - output := common.MapStr{} - - //Get version - ok := scanner.Scan() - if !ok { - return nil, errors.New("no initial successful scan, aborting") - } - output.Put("version.id", versionCapturer.FindStringSubmatch(scanner.Text())[1]) - output.Put("version.date", dateCapturer.FindStringSubmatch(scanner.Text())[1]) - - for scanner.Scan() { - line := scanner.Text() - - if strings.Contains(line, "Zxid") { - xid, err := parseXid(line) - if err != nil { - return nil, errors.Wrap(err, "error parsing xid line") - } - output.Put("xid", xid) - - continue - } - - if strings.Contains(line, "Latency") { - latency, err := parseLatencyLine(line) - if err != nil { - return nil, errors.Wrap(err, "error parsing latency values") - } - output.Put("latency", latency) - - continue - } - - if strings.Contains(line, "Proposal sizes") { - proposalSizes, err := parseProposalSizes(line) - if err != nil { - return nil, errors.Wrap(err, "error parsing proposal sizes line") - } - output.Put("proposal_sizes", proposalSizes) - - continue - } - - if strings.Contains(line, "Mode") { - output.Put("mode", strings.Split(line, " ")[1]) - continue - } - - // If code reaches here easy to parse lines or blank lines like the following: - // Received: 46 - // - // Sent: 45 - // Connections: 1 - // Outstanding: 0 - results := fieldsCapturer.FindAllStringSubmatch(line, -1) - if len(results) == 0 { - //probably a blank line - continue - } - - for _, result := range results { - val, err := strconv.ParseInt(result[2], 10, 64) - if err != nil { - return nil, err - } - output.Put(strings.ToLower(strings.Replace(result[1], " ", "_", -1)), val) - } - } - return output, nil -} - -func parseXid(line string) (common.MapStr, error){ - output := common.MapStr{} - - zxidString := strings.Split(line, " ")[1] - zxid, err := strconv.ParseInt(zxidString[2:], 16, 64) - if err != nil { - return nil, err - } - - bs := make([]byte, 8) - binary.BigEndian.PutUint64(bs, uint64(zxid)) - - epoch := bs[:4] - count := bs[4:] - - output.Put("zxid.original", zxidString) - output.Put("zxid.epoch", binary.BigEndian.Uint32(epoch)) - output.Put("zxid.count", binary.BigEndian.Uint32(count)) - - return output, nil -} - -func parseProposalSizes(line string) (common.MapStr, error) { - output := common.MapStr{} - - values := strings.Split(strings.Split(line, " ")[3], "/") - last, err := strconv.ParseInt(values[0], 10, 64) - if err != nil { - return nil, err - } - output.Put("last", last) - - min, err := strconv.ParseInt(values[1], 10, 64) - if err != nil { - return nil, err - } - output.Put("min", min) - - max, err := strconv.ParseInt(values[2], 10, 64) - if err != nil { - return nil, err - } - output.Put("max", max) - - return output, nil -} - -func parseLatencyLine(line string) (common.MapStr, error) { - output := common.MapStr{} - - values := latencyCapturer.FindStringSubmatch(line) - - min, err := strconv.ParseInt(values[1], 10, 64) - if err != nil { - return nil, err - } - output.Put("min", min) - - avg, err := strconv.ParseInt(values[2], 10, 64) - if err != nil { - return nil, err - } - output.Put("avg", avg) - - max, err := strconv.ParseInt(values[3], 10, 64) - if err != nil { - return nil, err - } - output.Put("max", max) - - return output, nil -} diff --git a/metricbeat/module/zookeeper/server/server_integration_test.go b/metricbeat/module/zookeeper/server/server_integration_test.go index 06b68d34e6b..2f329fd4102 100644 --- a/metricbeat/module/zookeeper/server/server_integration_test.go +++ b/metricbeat/module/zookeeper/server/server_integration_test.go @@ -20,9 +20,10 @@ package server import ( + "testing" + "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/metricbeat/module/zookeeper" - "testing" "github.com/stretchr/testify/assert" @@ -56,7 +57,7 @@ func TestFetch(t *testing.T) { assert.Equal(t, "3.4.13-2d71af4dbe22557fda74f9a9b4309b15a7487f03", version["id"]) received := metricsetFields["received"].(int64) - assert.True(t, received >= 0 ) + assert.True(t, received >= 0) nodeCount := metricsetFields["node_count"].(int64) assert.True(t, nodeCount >= 1) diff --git a/metricbeat/module/zookeeper/server/server_test.go b/metricbeat/module/zookeeper/server/server_test.go index 290c9c36cea..f715f9174af 100644 --- a/metricbeat/module/zookeeper/server/server_test.go +++ b/metricbeat/module/zookeeper/server/server_test.go @@ -19,9 +19,11 @@ package server import ( "bytes" - "github.com/elastic/beats/libbeat/common" - "github.com/stretchr/testify/assert" "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/libbeat/common" ) var srvrTestInput = `Zookeeper version: 3.4.13-2d71af4dbe22557fda74f9a9b4309b15a7487f03, built on 06/29/2018 04:05 GMT @@ -62,4 +64,8 @@ func TestParser(t *testing.T) { assert.Equal(t, int64(-3), proposalSizes["last"]) assert.Equal(t, int64(-999), proposalSizes["min"]) assert.Equal(t, int64(-1), proposalSizes["max"]) + + assert.Equal(t, "0x700601132", mapStr["zxid"]) + assert.Equal(t, uint32(7), mapStr["epoch"]) + assert.Equal(t, uint32(0x601132), mapStr["count"]) } diff --git a/x-pack/metricbeat/module/mssql/performance/data_integration_test.go b/x-pack/metricbeat/module/mssql/performance/data_integration_test.go index 9c688593783..116a7269825 100644 --- a/x-pack/metricbeat/module/mssql/performance/data_integration_test.go +++ b/x-pack/metricbeat/module/mssql/performance/data_integration_test.go @@ -18,6 +18,7 @@ import ( func TestData(t *testing.T) { t.Skip("Skipping `data.json` generation test") + _, config, err := getHostURI() if err != nil { t.Fatal("error getting config information", err.Error()) From 1f05fff867041770dc515978cd961efa95767735 Mon Sep 17 00:00:00 2001 From: sayden Date: Fri, 25 Jan 2019 20:54:17 +0100 Subject: [PATCH 04/10] data.json updated --- .../module/zookeeper/server/_meta/data.json | 12 +++++------- metricbeat/module/zookeeper/server/data.go | 10 +++++----- metricbeat/module/zookeeper/server/server.go | 15 ++++++++++++--- .../zookeeper/server/server_integration_test.go | 8 +------- metricbeat/module/zookeeper/server/server_test.go | 7 +++---- 5 files changed, 26 insertions(+), 26 deletions(-) diff --git a/metricbeat/module/zookeeper/server/_meta/data.json b/metricbeat/module/zookeeper/server/_meta/data.json index d08e3885150..0cc6c7b0b66 100644 --- a/metricbeat/module/zookeeper/server/_meta/data.json +++ b/metricbeat/module/zookeeper/server/_meta/data.json @@ -14,7 +14,8 @@ }, "service": { "address": "localhost:2181", - "type": "zookeeper" + "type": "zookeeper", + "version": "3.4.13-2d71af4dbe22557fda74f9a9b4309b15a7487f03" }, "zookeeper": { "server": { @@ -29,12 +30,9 @@ "mode": "standalone", "node_count": 4, "outstanding": 0, - "received": 16, - "sent": 15, - "version": { - "date": "06/29/2018 04:05 GMT", - "id": "3.4.13-2d71af4dbe22557fda74f9a9b4309b15a7487f03" - }, + "received": 38, + "sent": 37, + "version_date": "06/29/2018 04:05 GMT", "zxid": "0x0" } } diff --git a/metricbeat/module/zookeeper/server/data.go b/metricbeat/module/zookeeper/server/data.go index 6d5386b30cf..5fcc1ab557d 100644 --- a/metricbeat/module/zookeeper/server/data.go +++ b/metricbeat/module/zookeeper/server/data.go @@ -39,20 +39,20 @@ var fieldsCapturer = regexp.MustCompile(`^([a-zA-Z\s]+):\s(\d+)`) var versionCapturer = regexp.MustCompile(`:\s(.*),`) var dateCapturer = regexp.MustCompile(`built on (.*)`) -func parseSrvr(i io.Reader) (common.MapStr, error) { +func parseSrvr(i io.Reader) (common.MapStr, string, error) { scanner := bufio.NewScanner(i) //Get version ok := scanner.Scan() if !ok { - return nil, errors.New("no initial successful scan, aborting") + return nil, "", errors.New("no initial successful scan, aborting") } output := common.MapStr{} - output.Put("service.version", versionCapturer.FindStringSubmatch(scanner.Text())[1]) - output.Put("version.date", dateCapturer.FindStringSubmatch(scanner.Text())[1]) + version := versionCapturer.FindStringSubmatch(scanner.Text())[1] + output.Put("version_date", dateCapturer.FindStringSubmatch(scanner.Text())[1]) for scanner.Scan() { line := scanner.Text() @@ -124,7 +124,7 @@ func parseSrvr(i io.Reader) (common.MapStr, error) { } } - return output, nil + return output, version, nil } func parseZxid(line string) (common.MapStr, error) { diff --git a/metricbeat/module/zookeeper/server/server.go b/metricbeat/module/zookeeper/server/server.go index 06e02577fe6..f73d6904fcd 100644 --- a/metricbeat/module/zookeeper/server/server.go +++ b/metricbeat/module/zookeeper/server/server.go @@ -42,6 +42,8 @@ package server import ( "github.com/pkg/errors" + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/metricbeat/mb" @@ -79,13 +81,20 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) { return } - metricsetFields, err := parseSrvr(outputReader) + metricsetFields, version, err := parseSrvr(outputReader) if err != nil { reporter.Error(err) return } - reporter.Event(mb.Event{ + event := mb.Event{ MetricSetFields: metricsetFields, - }) + RootFields: common.MapStr{ + "service": common.MapStr{ + "version": version, + }, + }, + } + + reporter.Event(event) } diff --git a/metricbeat/module/zookeeper/server/server_integration_test.go b/metricbeat/module/zookeeper/server/server_integration_test.go index 2f329fd4102..7c5d4abecbc 100644 --- a/metricbeat/module/zookeeper/server/server_integration_test.go +++ b/metricbeat/module/zookeeper/server/server_integration_test.go @@ -27,7 +27,6 @@ import ( "github.com/stretchr/testify/assert" - "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" ) @@ -49,12 +48,7 @@ func TestFetch(t *testing.T) { metricsetFields := event.MetricSetFields // Check values - version, ok := metricsetFields["version"].(common.MapStr) - if !ok { - t.Fatal("no version field found") - } - assert.Equal(t, "06/29/2018 04:05 GMT", version["date"]) - assert.Equal(t, "3.4.13-2d71af4dbe22557fda74f9a9b4309b15a7487f03", version["id"]) + assert.Equal(t, "06/29/2018 04:05 GMT", metricsetFields["version_date"]) received := metricsetFields["received"].(int64) assert.True(t, received >= 0) diff --git a/metricbeat/module/zookeeper/server/server_test.go b/metricbeat/module/zookeeper/server/server_test.go index f715f9174af..eec8e5a2494 100644 --- a/metricbeat/module/zookeeper/server/server_test.go +++ b/metricbeat/module/zookeeper/server/server_test.go @@ -39,14 +39,13 @@ Proposal sizes last/min/max: -3/-999/-1 ` func TestParser(t *testing.T) { - mapStr, err := parseSrvr(bytes.NewReader([]byte(srvrTestInput))) + mapStr, versionID, err := parseSrvr(bytes.NewReader([]byte(srvrTestInput))) if err != nil { t.Fatal(err) } - version := mapStr["version"].(common.MapStr) - assert.Equal(t, "06/29/2018 04:05 GMT", version["date"]) - assert.Equal(t, "3.4.13-2d71af4dbe22557fda74f9a9b4309b15a7487f03", version["id"]) + assert.Equal(t, "06/29/2018 04:05 GMT", mapStr["version_date"]) + assert.Equal(t, "3.4.13-2d71af4dbe22557fda74f9a9b4309b15a7487f03", versionID) latency := mapStr["latency"].(common.MapStr) assert.Equal(t, int64(1), latency["min"]) From d386b864c068c5b16acdccc4cf2e7b5a48ee4c7b Mon Sep 17 00:00:00 2001 From: sayden Date: Fri, 25 Jan 2019 20:57:14 +0100 Subject: [PATCH 05/10] Fields doc needed an update --- metricbeat/docs/fields.asciidoc | 14 ++------------ metricbeat/module/zookeeper/fields.go | 2 +- .../module/zookeeper/server/_meta/docs.asciidoc | 5 ++--- .../module/zookeeper/server/_meta/fields.yml | 14 ++++---------- 4 files changed, 9 insertions(+), 26 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 7df1cc5b9ec..560f6ce525f 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -25363,7 +25363,7 @@ Max latency reached by the server -- type: long -Minimum latency that has been reached by the serverreached +Minimum latency that has been reached by the server -- @@ -25412,17 +25412,7 @@ Requests sent by the server -- - -*`zookeeper.server.version.id`*:: -+ --- -type: string - -Zookeeper version running - --- - -*`zookeeper.server.version.date`*:: +*`zookeeper.server.version_date`*:: + -- type: date diff --git a/metricbeat/module/zookeeper/fields.go b/metricbeat/module/zookeeper/fields.go index 66273ad0277..a122b325be7 100644 --- a/metricbeat/module/zookeeper/fields.go +++ b/metricbeat/module/zookeeper/fields.go @@ -32,5 +32,5 @@ func init() { // AssetZookeeper returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/zookeeper. func AssetZookeeper() string { - return "eJy0mM2O4zYMx+95CmIve5o8QA4Fim4PRbG7wLSnvXgYmYmFyKIr0fmYpy9kW/6Kncnkw6eBPeL/R4oUqbzAjk4reGfeERXkFgCixdAKvvxi/rt692UBkJJXThei2a7gtwUAQPsdchKnlQfFxpASSmF9AskINly6F0Mi4Z/YamGn7RYU5zna1C8XAD5jJ4liu9HbFWzQeFoAODKEnlawxQXARpNJ/apSfQGLOQ2JwyOnIvy747Jo3kwgh+etXfkGiq2gtr6CjV44Ktg1TrQ+tsv77OHps/X5civdoim8C4gVZjBwBeE4zPW6gakGedl7OYxvfMa+9P3J2Ev4a/Ax+rWj04FdOvp2wbth/kTby0llLArHR52jUJKiYOL1+zSGYbv9HMPvnW0IZoE3Pa6gNs1kUMiq0xL3Y8GbSfbkcEvRMqxJDkQWyHrK14aqIHnQFnJtjPakuJeFQzoqMsrJofGJ4tLKgxB/lPmaXAhRKwDvllOawdiwMXwg5x+u31oGH2LUlIEqnSMrVaSmiXI8JhttKIlK7B4aoe941HmZg+1ItSHo1DxgRZ7Chl0F3WVb4ViRn4llTLgcjw9mjQl3VWa1GNo+CkPbz2PYMk/Q6D2FxmFJBeOPz7KebRDu7ZRkKICuzThzgopmGpYLsk/Nuh/z2Ra0Y3lcmWlcihe0qbbbxNF/JXl5fGh7IhBF6rBaojREe00Rs+tzypReyE1zF6h2JH7pSJHe07gZ3c/cxc+SHNjtoiREyctgnp6w4/NQQW4GiOrd9SerHr+1jXWorIedVOjcKez4sIjaxtYe5tO0ntyeXOIF5YHDxysbCmfNsDAi0wzJySpKk+e1tVqg193aQe+QkQWE0GwHAQHtwRCmc0WxJ+c1T5/VaDSOfShQsjrkWtFyevXVY12zHNCmsC61ScGLqyu+dmua+YCisicdkJVt8r2D25MA13lgWKG59pisBp8nUdZD1dmY/yHfsGQ+d/v4Wi+64cLh3b66qNSXjK933TLubed/9Fo2ecG10T7ruM8CczbZTKqOAzfnRd/e+c3gohdnnozvA7yZ8wCGA+5dqt/x2Co6QnU5dgPls4nwk8qjObAaBjL0sA4T/iRL83J60uf05m7xT10JZyba6fO+yv+XBU3vilAV+0ej2E1KP2fWR/t3jUqvzeLe/MYfFdnNI9Br1AgWrqjmS33vlmrW4xh15uqe9lF6/4o/OrVd0ZXWTi2Mkun5vNOJznwcSH4L80FzbHTyzZkcZp/ST+f3+/HM3atr56fTW23RwB5NOSEvDq3H6oSGv77N9ID7Kqsn4SNAPSMFp6lglc38ajL+cq3un2HllR7/HwAA//8U/OsQ" + return "eJy0mM2O4zYMx+95CmIve5o8QA4Fim4PRbG7wLSnvXgYmYmFyKIr0vmYpy9kx47t2JlMPnwa2CP+f6RIkcoLbOiwgHfmDVFBYQagVh0t4Msv5r+rd19mACmJCbZQy34Bv80AANrvkJMGawQMO0dGKYXlATQjWHEZXhypxn9ib5WD9WswnOfoU5nPACTjoIlhv7LrBazQCc0AAjlCoQWscQawsuRSWVSqL+Axpz5xfPRQxH8PXBbHNyPI8XlrV76BYa9ovVSwjReBCg5HJ1of2+Vd9vh02bp8udfTojG8C4gVZjRwBeEwzPW6nqkj8rzzsh/f5hn60vUnY9H4V+9j49eGDjsO6eDbBe/6+dPYno8qY1EE3tsclZIUFROx7+MYjv36cwy/n2xDNAu86nBFtXEmh0reHOa4HQreTLKlgGtqLMOSdEfkgbxQvnRUBUnAesitc1bIcCcL+3RUZJRTQCeJ4dLrgxB/lPmSQgxRKwDvnlOawFixc7yjIA/Xby2DxBgdy8CUIZDXKlLjRDnuk5V1lDRKHB4aoe+4t3mZgz+RWkdwUhPAijyFFYcK+pRtRWBDMhHLJuFy3D+YtUm4qzKrxbD+URjWfx7Dl3mCzm4pNg5PJhp/fJZ1bINyZ6c0QwUMbca5A1Q047BckH9q1v2Yzrao3ZTHlZnGpYqiT61fJ4H+K0n08aHtiEAjUofVE6Ux2ktqME99zrhSlMI4d4FmQyrzQIbslobN6H7mU/w86Y7DppGERvIymNATdnwaKspNAFG9u3Lw5vFbe7QOlfW4kwZDOMQd7xdR29jaw3ycVihsKSSiqA8cPl7ZUTxr+oXRME2QHLyhNHleW6sFOt2tHfR2GXlAiM22FxCwAo4wnSqKLQWxPH5Wo7M49KFAzeqQW0Pz8dVXj3XH5YA+hWVpXQqioa742q1x5h2qyZ50QFa2SToHt5AC13ng2KC79pisBp8nUdZD1dmY/yFfv2Q+d/v4Wi+64cIhYVtdVOpLxte7bhn3tvM/Oi2bRHHprGQn7rPAnE02o6rDwE150bV3fjO46MWZJ8P7AK+mPID+gHuX6nfct4qB0FyOXU/5bCL8pPJgDqyGgQwFlnHC/5il5eD05jbxT10CZybasfO+kv+XFV3nblBV+Ucz2E1KPyfWN/bvmpFej4s7gxt/tCs3zz6vjUa0cMX2H3tOkk5NCyMfeoLfYlc9Ftuv5pei5iSLE0Mp48nxvrfj8bwi8X4Gu7YeHWzRlSPyGtALVuca/PVt4uS8Ly07EtIA1JNFdJoKNtnEbw3DL9fq/hlXXunx/wEAAP//35m7ZA==" } diff --git a/metricbeat/module/zookeeper/server/_meta/docs.asciidoc b/metricbeat/module/zookeeper/server/_meta/docs.asciidoc index 88e40d9ebb6..fd30fbeebe2 100644 --- a/metricbeat/module/zookeeper/server/_meta/docs.asciidoc +++ b/metricbeat/module/zookeeper/server/_meta/docs.asciidoc @@ -3,14 +3,13 @@ * *connections*: Connections established by the server * *latency.avg*: Average latency of the server * *latency.max*: Max latency reached by the server -* *latency.min*: Minimum latency that has been reached by the serverreached +* *latency.min*: Minimum latency that has been reached by the server * *mode*: Server mode * *node_count*: Total number of nodes * *outstanding*: Outstanding * *received*: Received requests to the server * *sent*: Requests sent by the server -* *version.id*: Zookeeper version running -* *version.date*: Date of the Zookeeper release in use +* *version_date*: Date of the Zookeeper release in use * *zxid*: Original value of the Zookeeper transaction ID * *count*: Total transactions of the leader in epoch * *epoch*: Epoch value of the Zookeeper transaction ID diff --git a/metricbeat/module/zookeeper/server/_meta/fields.yml b/metricbeat/module/zookeeper/server/_meta/fields.yml index 3464ca2138f..2e691a459b7 100644 --- a/metricbeat/module/zookeeper/server/_meta/fields.yml +++ b/metricbeat/module/zookeeper/server/_meta/fields.yml @@ -17,7 +17,7 @@ description: Max latency reached by the server - name: min type: long - description: Minimum latency that has been reached by the serverreached + description: Minimum latency that has been reached by the server - name: mode type: keyword description: Server mode @@ -33,15 +33,9 @@ - name: sent type: long description: Requests sent by the server - - name: version - type: group - fields: - - name: id - type: string - description: Zookeeper version running - - name: date - type: date - description: Date of the Zookeeper release in use + - name: version_date + type: date + description: Date of the Zookeeper release in use - name: zxid type: keyword description: Original value of the Zookeeper transaction ID From d2ed29df144cf201ff302adc8126e85b6580c7bb Mon Sep 17 00:00:00 2001 From: sayden Date: Mon, 28 Jan 2019 11:15:22 +0100 Subject: [PATCH 06/10] It seems that X-pack must also be updated manually (yet) --- x-pack/metricbeat/metricbeat.reference.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index c9fd68bc2ba..a54aa989d21 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -707,7 +707,7 @@ metricbeat.modules: #------------------------------ ZooKeeper Module ------------------------------ - module: zookeeper enabled: true - metricsets: ["mntr"] + metricsets: ["mntr", "server"] period: 10s hosts: ["localhost:2181"] From bb62dbebff5c61b8363771d2e0770b258c324ab1 Mon Sep 17 00:00:00 2001 From: sayden Date: Mon, 28 Jan 2019 11:15:41 +0100 Subject: [PATCH 07/10] Improve error handling and messages. --- metricbeat/module/zookeeper/server/data.go | 65 ++++++++++++++++------ 1 file changed, 49 insertions(+), 16 deletions(-) diff --git a/metricbeat/module/zookeeper/server/data.go b/metricbeat/module/zookeeper/server/data.go index 5fcc1ab557d..5a51c708f1e 100644 --- a/metricbeat/module/zookeeper/server/data.go +++ b/metricbeat/module/zookeeper/server/data.go @@ -46,7 +46,7 @@ func parseSrvr(i io.Reader) (common.MapStr, string, error) { ok := scanner.Scan() if !ok { - return nil, "", errors.New("no initial successful scan, aborting") + return nil, "", errors.New("no initial successful text scan, aborting") } output := common.MapStr{} @@ -60,7 +60,7 @@ func parseSrvr(i io.Reader) (common.MapStr, string, error) { if strings.Contains(line, "Zxid") { xid, err := parseZxid(line) if err != nil { - err = errors.Wrap(err, "error parsing xid line") + err = errors.Wrap(err, "error parsing 'zxid' line") logger.Debug(err.Error()) continue } @@ -73,7 +73,7 @@ func parseSrvr(i io.Reader) (common.MapStr, string, error) { if strings.Contains(line, "Latency") { latency, err := parseLatencyLine(line) if err != nil { - err = errors.Wrap(err, "error parsing latency values") + err = errors.Wrap(err, "error parsing 'latency values' line") logger.Debug(err.Error()) continue } @@ -86,7 +86,7 @@ func parseSrvr(i io.Reader) (common.MapStr, string, error) { if strings.Contains(line, "Proposal sizes") { proposalSizes, err := parseProposalSizes(line) if err != nil { - err = errors.Wrap(err, "error parsing proposal sizes line") + err = errors.Wrap(err, "error parsing 'proposal sizes' line") logger.Debug(err.Error()) continue } @@ -97,11 +97,17 @@ func parseSrvr(i io.Reader) (common.MapStr, string, error) { } if strings.Contains(line, "Mode") { - output.Put("mode", strings.Split(line, " ")[1]) + modeSplit := strings.Split(line, " ") + if len(modeSplit) < 1 { + logger.Debugf("less than one token after splitting '%s'", line) + continue + } + + output.Put("mode", modeSplit[1]) continue } - // If code reaches here easy to parse lines or blank lines like the following: + // If code reaches here, just easy to parse lines or blank lines like the following are left: // Received: 46 // // Sent: 45 @@ -114,12 +120,20 @@ func parseSrvr(i io.Reader) (common.MapStr, string, error) { } for _, result := range results { + // When submatching, the method returns the original value and the captured values, as you can see in the + // regexp of fieldsCapturer, they are 2 (so no less than 3, counting original value) + if len(result) < 3 { + logger.Debug("less than 3 tokens when regexp submatching '%s'", line) + continue + } + val, err := strconv.ParseInt(result[2], 10, 64) if err != nil { - err = errors.Wrapf(err, "error trying to parse '%s'", result) + err = errors.Wrapf(err, "error trying to parse value '%s' to int", result[2]) logger.Debug(err.Error()) continue } + output.Put(strings.ToLower(strings.Replace(result[1], " ", "_", -1)), val) } } @@ -130,10 +144,18 @@ func parseSrvr(i io.Reader) (common.MapStr, string, error) { func parseZxid(line string) (common.MapStr, error) { output := common.MapStr{} - zxidString := strings.Split(line, " ")[1] + zxidSplit := strings.Split(line, " ") + if len(zxidSplit) < 2 { + return nil, errors.Errorf("less than 2 tokens when splitting '%s'", line) + } + + zxidString := zxidSplit[1] + if len(zxidString) < 3 { + return nil, errors.Errorf("less than 3 characters on '%s'", line) + } zxid, err := strconv.ParseInt(zxidString[2:], 16, 64) if err != nil { - return nil, err + return nil, errors.Wrapf(err, "error trying to parse value '%s' to int", zxidString[2:]) } bs := make([]byte, 8) @@ -152,22 +174,30 @@ func parseZxid(line string) (common.MapStr, error) { func parseProposalSizes(line string) (common.MapStr, error) { output := common.MapStr{} - values := strings.Split(strings.Split(line, " ")[3], "/") + initialSplit := strings.Split(line, " ") + if len(initialSplit) < 4 { + return nil, errors.Errorf("less than 4 tokens when splitting '%s'", line) + } + + values := strings.Split(initialSplit[3], "/") + if len(values) < 3 { + return nil, errors.Errorf("less than 3 tokens when splitting '%s'", values) + } last, err := strconv.ParseInt(values[0], 10, 64) if err != nil { - return nil, err + return nil, errors.Wrapf(err, "error trying to get 'last' value from '%s'", values[0]) } output.Put("last", last) min, err := strconv.ParseInt(values[1], 10, 64) if err != nil { - return nil, err + return nil, errors.Wrapf(err, "error trying to get 'min' value from '%s'", values[1]) } output.Put("min", min) max, err := strconv.ParseInt(values[2], 10, 64) if err != nil { - return nil, err + return nil, errors.Wrapf(err, "error trying to get 'max' value from '%s'", values[2]) } output.Put("max", max) @@ -178,22 +208,25 @@ func parseLatencyLine(line string) (common.MapStr, error) { output := common.MapStr{} values := latencyCapturer.FindStringSubmatch(line) + if len(values) < 4 { + return nil, errors.Errorf("unexpected number of fields after splitting latency line '%s'", line) + } min, err := strconv.ParseInt(values[1], 10, 64) if err != nil { - return nil, err + return nil, errors.Wrapf(err, "error trying to get latency 'min' value") } output.Put("min", min) avg, err := strconv.ParseInt(values[2], 10, 64) if err != nil { - return nil, err + return nil, errors.Wrapf(err, "error trying to get latency 'avg' value") } output.Put("avg", avg) max, err := strconv.ParseInt(values[3], 10, 64) if err != nil { - return nil, err + return nil, errors.Wrapf(err, "error trying to get latency 'max' value") } output.Put("max", max) From 2bdf989834a05e6cb3b4276fd4804ead59a9b97f Mon Sep 17 00:00:00 2001 From: sayden Date: Mon, 28 Jan 2019 11:16:18 +0100 Subject: [PATCH 08/10] Add changelog entry --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index e6429a8f809..76beea8c75d 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -223,6 +223,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Release use of xpack.enabled: true flag in Elasticsearch and Kibana modules as GA. {pull}10222[10222] - Add support for MySQL 8.0 and tests also for Percona and MariaDB. {pull}10261[10261] - Rename 'db' Metricset to 'transaction_log' in MSSQL Metricbeat module {pull}10109[10109] +- Added 'server' Metricset to Zookeeper Metricbeat module {issue}8938[8938] {pull}10341[10341] *Packetbeat* From b9e63be01e044926539ef891edfdc98aa328abaf Mon Sep 17 00:00:00 2001 From: sayden Date: Mon, 28 Jan 2019 11:18:01 +0100 Subject: [PATCH 09/10] Skip data generation test --- metricbeat/module/zookeeper/server/data_integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/zookeeper/server/data_integration_test.go b/metricbeat/module/zookeeper/server/data_integration_test.go index 2fa7013e238..843db0a7dca 100644 --- a/metricbeat/module/zookeeper/server/data_integration_test.go +++ b/metricbeat/module/zookeeper/server/data_integration_test.go @@ -31,7 +31,7 @@ import ( ) func TestData(t *testing.T) { - //t.Skip("Skipping `data.json` generation test") + t.Skip("Skipping `data.json` generation test") f := mbtest.NewReportingMetricSetV2(t, getDataConfig()) events, errs := mbtest.ReportingFetchV2(f) From b5fab40620dbe28dd9b98e1bd2035ce83ae3b180 Mon Sep 17 00:00:00 2001 From: sayden Date: Mon, 28 Jan 2019 12:56:36 +0100 Subject: [PATCH 10/10] Some improvements on the error handling --- metricbeat/module/zookeeper/server/data.go | 34 +++++++++++----------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/metricbeat/module/zookeeper/server/data.go b/metricbeat/module/zookeeper/server/data.go index 5a51c708f1e..c768b87f1fb 100644 --- a/metricbeat/module/zookeeper/server/data.go +++ b/metricbeat/module/zookeeper/server/data.go @@ -60,7 +60,7 @@ func parseSrvr(i io.Reader) (common.MapStr, string, error) { if strings.Contains(line, "Zxid") { xid, err := parseZxid(line) if err != nil { - err = errors.Wrap(err, "error parsing 'zxid' line") + err = errors.Wrapf(err, "error parsing 'zxid' line '%s'", line) logger.Debug(err.Error()) continue } @@ -73,7 +73,7 @@ func parseSrvr(i io.Reader) (common.MapStr, string, error) { if strings.Contains(line, "Latency") { latency, err := parseLatencyLine(line) if err != nil { - err = errors.Wrap(err, "error parsing 'latency values' line") + err = errors.Wrapf(err, "error parsing 'latency values' line '%s'", line) logger.Debug(err.Error()) continue } @@ -86,7 +86,7 @@ func parseSrvr(i io.Reader) (common.MapStr, string, error) { if strings.Contains(line, "Proposal sizes") { proposalSizes, err := parseProposalSizes(line) if err != nil { - err = errors.Wrap(err, "error parsing 'proposal sizes' line") + err = errors.Wrapf(err, "error parsing 'proposal sizes' line '%s'", line) logger.Debug(err.Error()) continue } @@ -99,7 +99,7 @@ func parseSrvr(i io.Reader) (common.MapStr, string, error) { if strings.Contains(line, "Mode") { modeSplit := strings.Split(line, " ") if len(modeSplit) < 1 { - logger.Debugf("less than one token after splitting '%s'", line) + logger.Debugf("no tokens after splitting line '%s'", line) continue } @@ -123,13 +123,13 @@ func parseSrvr(i io.Reader) (common.MapStr, string, error) { // When submatching, the method returns the original value and the captured values, as you can see in the // regexp of fieldsCapturer, they are 2 (so no less than 3, counting original value) if len(result) < 3 { - logger.Debug("less than 3 tokens when regexp submatching '%s'", line) + logger.Debug("less than 3 tokens (%v) when regexp submatching '%s'", result, line) continue } val, err := strconv.ParseInt(result[2], 10, 64) if err != nil { - err = errors.Wrapf(err, "error trying to parse value '%s' to int", result[2]) + err = errors.Wrapf(err, "error trying to parse value '%s' as int", result[2]) logger.Debug(err.Error()) continue } @@ -146,12 +146,12 @@ func parseZxid(line string) (common.MapStr, error) { zxidSplit := strings.Split(line, " ") if len(zxidSplit) < 2 { - return nil, errors.Errorf("less than 2 tokens when splitting '%s'", line) + return nil, errors.Errorf("less than 2 tokens (%v) after splitting", zxidSplit) } zxidString := zxidSplit[1] if len(zxidString) < 3 { - return nil, errors.Errorf("less than 3 characters on '%s'", line) + return nil, errors.Errorf("less than 3 characters on '%s'", zxidString) } zxid, err := strconv.ParseInt(zxidString[2:], 16, 64) if err != nil { @@ -176,28 +176,28 @@ func parseProposalSizes(line string) (common.MapStr, error) { initialSplit := strings.Split(line, " ") if len(initialSplit) < 4 { - return nil, errors.Errorf("less than 4 tokens when splitting '%s'", line) + return nil, errors.Errorf("less than 4 tokens (%v) after splitting", initialSplit) } values := strings.Split(initialSplit[3], "/") if len(values) < 3 { - return nil, errors.Errorf("less than 3 tokens when splitting '%s'", values) + return nil, errors.Errorf("less than 3 tokens (%v) after splitting", values) } last, err := strconv.ParseInt(values[0], 10, 64) if err != nil { - return nil, errors.Wrapf(err, "error trying to get 'last' value from '%s'", values[0]) + return nil, errors.Wrapf(err, "error trying to parse 'last' value as int from '%s'", values[0]) } output.Put("last", last) min, err := strconv.ParseInt(values[1], 10, 64) if err != nil { - return nil, errors.Wrapf(err, "error trying to get 'min' value from '%s'", values[1]) + return nil, errors.Wrapf(err, "error trying to parse 'min' value as int from '%s'", values[1]) } output.Put("min", min) max, err := strconv.ParseInt(values[2], 10, 64) if err != nil { - return nil, errors.Wrapf(err, "error trying to get 'max' value from '%s'", values[2]) + return nil, errors.Wrapf(err, "error trying to parse 'max' value as int from '%s'", values[2]) } output.Put("max", max) @@ -209,24 +209,24 @@ func parseLatencyLine(line string) (common.MapStr, error) { values := latencyCapturer.FindStringSubmatch(line) if len(values) < 4 { - return nil, errors.Errorf("unexpected number of fields after splitting latency line '%s'", line) + return nil, errors.Errorf("less than 4 fields (%v) after splitting", values) } min, err := strconv.ParseInt(values[1], 10, 64) if err != nil { - return nil, errors.Wrapf(err, "error trying to get latency 'min' value") + return nil, errors.Wrapf(err, "error trying to parse 'min' value '%s' as int", values[1]) } output.Put("min", min) avg, err := strconv.ParseInt(values[2], 10, 64) if err != nil { - return nil, errors.Wrapf(err, "error trying to get latency 'avg' value") + return nil, errors.Wrapf(err, "error trying to parse 'avg' value '%s' as int", values[2]) } output.Put("avg", avg) max, err := strconv.ParseInt(values[3], 10, 64) if err != nil { - return nil, errors.Wrapf(err, "error trying to get latency 'max' value") + return nil, errors.Wrapf(err, "error trying to parse 'max' value '%s' as int", values[3]) } output.Put("max", max)