diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index d0fb43689a4..98a30512188 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -193,6 +193,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add filtering option for prometheus collector. {pull}16420[16420] - Add metricsets based on Ceph Manager Daemon to the `ceph` module. {issue}7723[7723] {pull}16254[16254] - Release `statsd` module as GA. {pull}16447[16447] {issue}14280[14280] +- Add `redisenterprise` module. {pull}16482[16482] {issue}15269[15269] *Packetbeat* diff --git a/libbeat/tests/compose/compose.go b/libbeat/tests/compose/compose.go index 293e57e8784..e9011ca9645 100644 --- a/libbeat/tests/compose/compose.go +++ b/libbeat/tests/compose/compose.go @@ -81,6 +81,13 @@ func EnsureUp(t testing.TB, service string, options ...UpOption) HostInfo { // Wait for health err = compose.Wait(upOptions.Timeout, service) if err != nil { + inspected, inspectErr := compose.Inspect(service) + if inspectErr != nil { + t.Logf("inspection error: %v", err) + } else { + t.Logf("Container state (service: '%s'): %s", service, inspected) + } + t.Fatal(err) } diff --git a/libbeat/tests/compose/project.go b/libbeat/tests/compose/project.go index f4ebfb904e5..602b38b83ae 100644 --- a/libbeat/tests/compose/project.go +++ b/libbeat/tests/compose/project.go @@ -92,6 +92,7 @@ type Driver interface { Kill(ctx context.Context, signal string, service string) error KillOld(ctx context.Context, except []string) error Ps(ctx context.Context, filter ...string) ([]ContainerStatus, error) + Inspect(ctx context.Context, serviceName string) (string, error) LockFile() string @@ -227,6 +228,14 @@ func (c *Project) KillOld(except []string) error { return c.Driver.KillOld(context.TODO(), except) } +// Inspect a container +func (c *Project) Inspect(service string) (string, error) { + c.Lock() + defer c.Unlock() + + return c.Driver.Inspect(context.Background(), service) +} + // Lock acquires the lock (300s) timeout // Normally it should only be seconds that the lock is used, but in some cases it can take longer. // Pid is written to the lock file, and it is used to check if process holding the process is still diff --git a/libbeat/tests/compose/wrapper.go b/libbeat/tests/compose/wrapper.go index 35e37098a38..646900edab7 100644 --- a/libbeat/tests/compose/wrapper.go +++ b/libbeat/tests/compose/wrapper.go @@ -21,6 +21,7 @@ import ( "archive/tar" "bytes" "context" + "encoding/json" "fmt" "net" "os" @@ -371,6 +372,43 @@ func (d *wrapperDriver) serviceNames(ctx context.Context) ([]string, error) { return strings.Fields(stdout.String()), nil } +// Inspect a container. +func (d *wrapperDriver) Inspect(ctx context.Context, serviceName string) (string, error) { + list, err := d.client.ContainerList(ctx, types.ContainerListOptions{All: true}) + if err != nil { + return "", errors.Wrap(err, "listing containers to be inspected") + } + + var found bool + var c types.Container + for _, container := range list { + aServiceName, ok := container.Labels[labelComposeService] + if ok && serviceName == aServiceName { + c = container + found = true + break + } + } + + if !found { + return "", errors.Errorf("container not found for service '%s'", serviceName) + } + + inspect, err := d.client.ContainerInspect(ctx, c.ID) + if err != nil { + return "", errors.Wrap(err, "container failed inspection") + } else if inspect.State == nil { + return "empty container state", nil + } + + state, err := json.Marshal(inspect.State) + if err != nil { + return "", errors.Wrap(err, "container inspection failed") + } + + return string(state), nil +} + func makeFilter(project, service string, projectFilter Filter) filters.Args { f := filters.NewArgs() f.Add("label", fmt.Sprintf("%s=%s", labelComposeProject, project)) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index cec790b06bf..0ccbfb4955e 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -65,6 +65,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -32492,6 +32493,20 @@ type: long -- +[[exported-fields-redisenterprise]] +== Redis Enterprise fields + +Redis metrics collected from Redis Enterprise Server. + + + +[float] +=== redisenterprise + +`redisenterprise` contains the information and statistics from Redis Enterprise Server. + + + [[exported-fields-sql]] == sql fields diff --git a/metricbeat/docs/modules/redis.asciidoc b/metricbeat/docs/modules/redis.asciidoc index b5c7f20364f..84377c51d50 100644 --- a/metricbeat/docs/modules/redis.asciidoc +++ b/metricbeat/docs/modules/redis.asciidoc @@ -33,8 +33,8 @@ redis://HOST[:PORT][?password=PASSWORD[&db=DATABASE]] [float] === Compatibility -The Redis metricsets were tested with Redis 3.2.12, 4.0.11 and 5.0-rc4, and are expected -to work with all versions >= 3.0. +The redis metricsets `info`, `key` and `keyspace` are compatible with all distributions of Redis (OSS and enterprise). +They were tested with Redis 3.2.12, 4.0.11 and 5.0-rc4, and are expected to work with all versions >= 3.0. [float] diff --git a/metricbeat/docs/modules/redisenterprise.asciidoc b/metricbeat/docs/modules/redisenterprise.asciidoc new file mode 100644 index 00000000000..63cc4b39bc5 --- /dev/null +++ b/metricbeat/docs/modules/redisenterprise.asciidoc @@ -0,0 +1,62 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-redisenterprise]] +[role="xpack"] +== Redis Enterprise module + +beta[] + +This module periodically fetches metrics from https://redislabs.com/redis-enterprise/[Redis Enterprise Software]. + +The defaut metricsets are `node` and `proxy`. + +[float] +=== Module-specific configuration notes + +The Redis module has these additional config options: + +*`hosts`*:: URLs that are used to connect to Redis. +URL format: +https://HOST[:PORT] + +[float] +=== Compatibility + +The metricsets `node` and `proxy` are compatible with Redis Enterprise Software (RES). There were tested with RES +5.4.10-22 and are expected to work with all versions >= 5.0.2. + + +[float] +=== Example configuration + +The Redis Enterprise module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: redisenterprise + metricsets: + - node + - proxy + period: 1m + + # Metrics endpoint + hosts: ["https://127.0.0.1:8070/"] +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +* <> + +include::redisenterprise/node.asciidoc[] + +include::redisenterprise/proxy.asciidoc[] + diff --git a/metricbeat/docs/modules/redisenterprise/node.asciidoc b/metricbeat/docs/modules/redisenterprise/node.asciidoc new file mode 100644 index 00000000000..90103d11923 --- /dev/null +++ b/metricbeat/docs/modules/redisenterprise/node.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-redisenterprise-node]] +=== Redis Enterprise node metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/redisenterprise/node/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/redisenterprise/node/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/redisenterprise/proxy.asciidoc b/metricbeat/docs/modules/redisenterprise/proxy.asciidoc new file mode 100644 index 00000000000..cee1e06ebd4 --- /dev/null +++ b/metricbeat/docs/modules/redisenterprise/proxy.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-redisenterprise-proxy]] +=== Redis Enterprise proxy metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/redisenterprise/proxy/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/redisenterprise/proxy/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 5587b923345..137f3bbd7c6 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -200,6 +200,9 @@ This file is generated! See scripts/mage/docs_collector.go .3+| .3+| |<> |<> |<> +|<> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | +.2+| .2+| |<> beta[] +|<> beta[] |<> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | .1+| .1+| |<> beta[] |<> beta[] |image:./images/icon-yes.png[Prebuilt dashboards are available] | @@ -294,6 +297,7 @@ include::modules/postgresql.asciidoc[] include::modules/prometheus.asciidoc[] include::modules/rabbitmq.asciidoc[] include::modules/redis.asciidoc[] +include::modules/redisenterprise.asciidoc[] include::modules/sql.asciidoc[] include::modules/stan.asciidoc[] include::modules/statsd.asciidoc[] diff --git a/metricbeat/mb/testing/data/data_test.go b/metricbeat/mb/testing/data/data_test.go index 3102780e66f..a6d1f468127 100644 --- a/metricbeat/mb/testing/data/data_test.go +++ b/metricbeat/mb/testing/data/data_test.go @@ -24,9 +24,8 @@ import ( "strings" "testing" - mbtest "github.com/elastic/beats/metricbeat/mb/testing" - _ "github.com/elastic/beats/metricbeat/include" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" ) func TestAll(t *testing.T) { diff --git a/metricbeat/module/redis/_meta/docs.asciidoc b/metricbeat/module/redis/_meta/docs.asciidoc index a15de3429fd..8947ff6ec21 100644 --- a/metricbeat/module/redis/_meta/docs.asciidoc +++ b/metricbeat/module/redis/_meta/docs.asciidoc @@ -26,5 +26,5 @@ redis://HOST[:PORT][?password=PASSWORD[&db=DATABASE]] [float] === Compatibility -The Redis metricsets were tested with Redis 3.2.12, 4.0.11 and 5.0-rc4, and are expected -to work with all versions >= 3.0. +The redis metricsets `info`, `key` and `keyspace` are compatible with all distributions of Redis (OSS and enterprise). +They were tested with Redis 3.2.12, 4.0.11 and 5.0-rc4, and are expected to work with all versions >= 3.0. diff --git a/metricbeat/module/redis/docker-compose.yml b/metricbeat/module/redis/docker-compose.yml index a2f9dcc4bdd..3cb5c07c12e 100644 --- a/metricbeat/module/redis/docker-compose.yml +++ b/metricbeat/module/redis/docker-compose.yml @@ -7,5 +7,6 @@ services: context: ./_meta args: REDIS_VERSION: ${REDIS_VERSION:-3.2.12} + privileged: true ports: - 6379 diff --git a/metricbeat/module/redis/module.yml b/metricbeat/module/redis/module.yml index 6a4e4504f9c..f89e64a72b3 100644 --- a/metricbeat/module/redis/module.yml +++ b/metricbeat/module/redis/module.yml @@ -1,3 +1,4 @@ +name: redis dashboards: - - id: AV4YjZ5pux-M-tCAunxK - file: Metricbeat-redis-overview.json + - id: AV4YjZ5pux-M-tCAunxK + file: Metricbeat-redis-overview.json diff --git a/metricbeat/module/redis/test_redis.py b/metricbeat/module/redis/test_redis.py index 9d9967389f9..98ffb982ca3 100644 --- a/metricbeat/module/redis/test_redis.py +++ b/metricbeat/module/redis/test_redis.py @@ -34,6 +34,7 @@ def test_info(self): """ Test redis info metricset """ + self.render_config_template(modules=[{ "name": "redis", "metricsets": ["info"], @@ -137,6 +138,7 @@ def test_module_processors(self): """ Test local processors for Redis info event. """ + fields = ["clients", "cpu"] eventFields = ['beat', 'metricset', 'service', 'event'] eventFields += ['redis.info.' + f for f in fields] diff --git a/x-pack/metricbeat/include/list.go b/x-pack/metricbeat/include/list.go index 590accfd862..5dd93240c1d 100644 --- a/x-pack/metricbeat/include/list.go +++ b/x-pack/metricbeat/include/list.go @@ -41,6 +41,7 @@ import ( _ "github.com/elastic/beats/x-pack/metricbeat/module/oracle" _ "github.com/elastic/beats/x-pack/metricbeat/module/oracle/performance" _ "github.com/elastic/beats/x-pack/metricbeat/module/oracle/tablespace" + _ "github.com/elastic/beats/x-pack/metricbeat/module/redisenterprise" _ "github.com/elastic/beats/x-pack/metricbeat/module/sql" _ "github.com/elastic/beats/x-pack/metricbeat/module/sql/query" _ "github.com/elastic/beats/x-pack/metricbeat/module/stan" diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 7aff7843201..55c591730ca 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -966,6 +966,16 @@ metricbeat.modules: # Redis AUTH password. Empty by default. #password: foobared +#--------------------------- Redis Enterprise Module --------------------------- +- module: redisenterprise + metricsets: + - node + - proxy + period: 1m + + # Metrics endpoint + hosts: ["https://127.0.0.1:8070/"] + #--------------------------------- Sql Module --------------------------------- - module: sql metricsets: diff --git a/x-pack/metricbeat/module/redisenterprise/_meta/Dockerfile b/x-pack/metricbeat/module/redisenterprise/_meta/Dockerfile new file mode 100644 index 00000000000..4081ad4ee31 --- /dev/null +++ b/x-pack/metricbeat/module/redisenterprise/_meta/Dockerfile @@ -0,0 +1,6 @@ +ARG REDISENTERPRISE_VERSION +FROM redislabs/redis:${REDISENTERPRISE_VERSION} + +# Wait for the health endpoint to have monitors information +ADD healthcheck.sh / +HEALTHCHECK --interval=1s --retries=300 CMD /healthcheck.sh diff --git a/x-pack/metricbeat/module/redisenterprise/_meta/config.reference.yml b/x-pack/metricbeat/module/redisenterprise/_meta/config.reference.yml new file mode 100644 index 00000000000..e1f77b45723 --- /dev/null +++ b/x-pack/metricbeat/module/redisenterprise/_meta/config.reference.yml @@ -0,0 +1,8 @@ +- module: redisenterprise + metricsets: + - node + - proxy + period: 1m + + # Metrics endpoint + hosts: ["https://127.0.0.1:8070/"] diff --git a/x-pack/metricbeat/module/redisenterprise/_meta/config.yml b/x-pack/metricbeat/module/redisenterprise/_meta/config.yml new file mode 100644 index 00000000000..e1f77b45723 --- /dev/null +++ b/x-pack/metricbeat/module/redisenterprise/_meta/config.yml @@ -0,0 +1,8 @@ +- module: redisenterprise + metricsets: + - node + - proxy + period: 1m + + # Metrics endpoint + hosts: ["https://127.0.0.1:8070/"] diff --git a/x-pack/metricbeat/module/redisenterprise/_meta/docs.asciidoc b/x-pack/metricbeat/module/redisenterprise/_meta/docs.asciidoc new file mode 100644 index 00000000000..4484f89df91 --- /dev/null +++ b/x-pack/metricbeat/module/redisenterprise/_meta/docs.asciidoc @@ -0,0 +1,18 @@ +This module periodically fetches metrics from https://redislabs.com/redis-enterprise/[Redis Enterprise Software]. + +The defaut metricsets are `node` and `proxy`. + +[float] +=== Module-specific configuration notes + +The Redis module has these additional config options: + +*`hosts`*:: URLs that are used to connect to Redis. +URL format: +https://HOST[:PORT] + +[float] +=== Compatibility + +The metricsets `node` and `proxy` are compatible with Redis Enterprise Software (RES). There were tested with RES +5.4.10-22 and are expected to work with all versions >= 5.0.2. diff --git a/x-pack/metricbeat/module/redisenterprise/_meta/fields.yml b/x-pack/metricbeat/module/redisenterprise/_meta/fields.yml new file mode 100644 index 00000000000..7f697324521 --- /dev/null +++ b/x-pack/metricbeat/module/redisenterprise/_meta/fields.yml @@ -0,0 +1,11 @@ +- key: redisenterprise + title: "Redis Enterprise" + description: > + Redis metrics collected from Redis Enterprise Server. + release: beta + fields: + - name: redisenterprise + type: group + description: > + `redisenterprise` contains the information and statistics from Redis Enterprise Server. + fields: diff --git a/x-pack/metricbeat/module/redisenterprise/_meta/healthcheck.sh b/x-pack/metricbeat/module/redisenterprise/_meta/healthcheck.sh new file mode 100755 index 00000000000..12b298e60c6 --- /dev/null +++ b/x-pack/metricbeat/module/redisenterprise/_meta/healthcheck.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -e + +CHECK_CLUSTER_CREATED=/opt/redislabs/config/check_cluster_created +CHECK_DATABASE_CREATED=/opt/redislabs/config/check_database_created + +if [[ ! -f "${CHECK_CLUSTER_CREATED}" ]]; then + rladmin cluster create name cluster.local username cihan@redislabs.com password redislabs123 + touch ${CHECK_CLUSTER_CREATED} +fi + +if [[ ! -f "${CHECK_DATABASE_CREATED}" ]]; then + curl -s -k -u "cihan@redislabs.com:redislabs123" --request POST \ + --url "https://localhost:9443/v1/bdbs" \ + --header 'content-type: application/json' \ + --data '{"name":"db1","type":"redis","memory_size":102400,"port":12000}' + touch ${CHECK_DATABASE_CREATED} +fi + +curl -s --insecure https://127.0.0.1:8070 >/dev/null diff --git a/x-pack/metricbeat/module/redisenterprise/_meta/supported-versions.yml b/x-pack/metricbeat/module/redisenterprise/_meta/supported-versions.yml new file mode 100644 index 00000000000..b24c80b47a0 --- /dev/null +++ b/x-pack/metricbeat/module/redisenterprise/_meta/supported-versions.yml @@ -0,0 +1,2 @@ +variants: + - REDISENTERPRISE_VERSION: 5.4.10-22 diff --git a/x-pack/metricbeat/module/redisenterprise/docker-compose.yml b/x-pack/metricbeat/module/redisenterprise/docker-compose.yml new file mode 100644 index 00000000000..55ae81cc584 --- /dev/null +++ b/x-pack/metricbeat/module/redisenterprise/docker-compose.yml @@ -0,0 +1,13 @@ +version: '2.3' + +services: + redisenterprise: + image: docker.elastic.co/integrations-ci/beats-redisenterprise:${REDISENTERPRISE_VERSION:-5.4.10-22}-3 + build: + context: ./_meta + args: + REDISENTERPRISE_VERSION: ${REDISENTERPRISE_VERSION:-5.4.10-22} + cap_add: + - SYS_RESOURCE + ports: + - 8070 diff --git a/x-pack/metricbeat/module/redisenterprise/fields.go b/x-pack/metricbeat/module/redisenterprise/fields.go new file mode 100644 index 00000000000..7695ff24329 --- /dev/null +++ b/x-pack/metricbeat/module/redisenterprise/fields.go @@ -0,0 +1,23 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package redisenterprise + +import ( + "github.com/elastic/beats/libbeat/asset" +) + +func init() { + if err := asset.SetFields("metricbeat", "redisenterprise", asset.ModuleFieldsPri, AssetRedisenterprise); err != nil { + panic(err) + } +} + +// AssetRedisenterprise returns asset data. +// This is the base64 encoded gzipped contents of module/redisenterprise. +func AssetRedisenterprise() string { + return "eJykjztOBDEQRHOforT5cgAHZFwALrDGroUWHtvqbhBze2Q+q9VoEAEv7E/p1REvXCOURYzNqUPFGAAXr4w43M8N7i6rQwAKLasMl94ibgMAfJ0tdJVsyL1WZmfBWfuCbQYeqG/UmwAoK5Mx4pGeAnAW1mLxM/OIlhbuyU18HYx40v46vic7WpPT5v+E3JsnaQZ/JqSduy5pfiG1AvPkYj5r/CU/uRa+lm698DLcs51s2//wW+jQ/r7+O/UjAAD//wOWl64=" +} diff --git a/x-pack/metricbeat/module/redisenterprise/module.yml b/x-pack/metricbeat/module/redisenterprise/module.yml new file mode 100644 index 00000000000..85a7bfbae06 --- /dev/null +++ b/x-pack/metricbeat/module/redisenterprise/module.yml @@ -0,0 +1,4 @@ +name: redisenterprise +metricsets: + - node + - proxy diff --git a/x-pack/metricbeat/module/redisenterprise/node/_meta/data.json b/x-pack/metricbeat/module/redisenterprise/node/_meta/data.json new file mode 100644 index 00000000000..2b77561b5f7 --- /dev/null +++ b/x-pack/metricbeat/module/redisenterprise/node/_meta/data.json @@ -0,0 +1,84 @@ +{ + "@timestamp": "2020-02-21T08:25:46.123Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" + }, + "metricset": { + "period": 60000, + "name": "node" + }, + "service": { + "type": "redisenterprise", + "address": "127.0.0.1:8070" + }, + "ecs": { + "version": "1.4.0" + }, + "host": { + "hostname": "host", + "architecture": "x86_64", + "os": { + "family": "darwin", + "name": "Mac OS X", + "kernel": "18.7.0", + "build": "18G95", + "platform": "darwin", + "version": "10.14.6" + }, + "name": "host", + "id": "24F065F8-4274-521D-8DD5-5D27557E15B4", + "ip": [ + "fe80::aede:48ff:fe00:1122", + "fe80::1c05:593c:e271:97ed", + "192.168.0.14", + "fe80::488c:55ff:fe4e:3b46", + "fe80::def:1653:5676:ea4e", + "fe80::f191:956a:99ff:1b98", + "fe80::3128:a84c:7b38:e3a9" + ], + "mac": [ + "ac:de:48:00:11:22", + "a6:83:e7:ae:70:01", + "a4:83:e7:ae:70:01", + "06:83:e7:ae:70:01" + ] + }, + "agent": { + "ephemeral_id": "de3a043f-a71f-441e-b730-ca12a8f7fc49", + "hostname": "MacBook-Elastic.local", + "id": "993db589-f9d8-40f3-94cb-43b2b69fa9f4", + "version": "8.0.0", + "type": "metricbeat" + }, + "prometheus": { + "labels": { + "cluster": "cluster.local", + "node": "1", + "instance": "127.0.0.1:8070", + "job": "redisenterprise" + }, + "metrics": { + "node_cpu_steal_median": 0, + "node_cpu_irqs_max": 0.001, + "node_cpu_user_min": 0.021, + "node_cpu_iowait_median": 0, + "node_cur_aof_rewrites": 0, + "node_ingress_bytes_min": 0, + "node_ingress_bytes_median": 0, + "node_cpu_idle_min": 0.929, + "node_cpu_iowait_max": 0, + "node_cpu_irqs": 0.0003333333333333333, + "node_available_memory": 6.262083584e+09, + "node_provisional_memory_no_overbooking": 4.757468584333e+09, + "node_cpu_idle": 0.9303333333333335, + "node_cpu_steal_max": 0 + } + }, + "event": { + "dataset": "redisenterprise.node", + "module": "redisenterprise", + "duration": 51870343 + } +} diff --git a/x-pack/metricbeat/module/redisenterprise/node/_meta/docs.asciidoc b/x-pack/metricbeat/module/redisenterprise/node/_meta/docs.asciidoc new file mode 100644 index 00000000000..90e351883f4 --- /dev/null +++ b/x-pack/metricbeat/module/redisenterprise/node/_meta/docs.asciidoc @@ -0,0 +1 @@ +This is the `node` metricset of the redisenterprise module. The metricset is compatible with Redis Enterprise Software. diff --git a/x-pack/metricbeat/module/redisenterprise/node/_meta/fields.yml b/x-pack/metricbeat/module/redisenterprise/node/_meta/fields.yml new file mode 100644 index 00000000000..4d523079ac9 --- /dev/null +++ b/x-pack/metricbeat/module/redisenterprise/node/_meta/fields.yml @@ -0,0 +1,4 @@ +- name: node + type: group + release: beta + fields: diff --git a/x-pack/metricbeat/module/redisenterprise/node/_meta/testdata/config.yml b/x-pack/metricbeat/module/redisenterprise/node/_meta/testdata/config.yml new file mode 100644 index 00000000000..361e43c5ccd --- /dev/null +++ b/x-pack/metricbeat/module/redisenterprise/node/_meta/testdata/config.yml @@ -0,0 +1,4 @@ +type: http +url: "/" +suffix: plain +remove_fields_from_comparison: ["prometheus.labels.instance"] diff --git a/x-pack/metricbeat/module/redisenterprise/node/_meta/testdata/redis-node.5.4.10-22.plain b/x-pack/metricbeat/module/redisenterprise/node/_meta/testdata/redis-node.5.4.10-22.plain new file mode 100644 index 00000000000..d1b94f28efd --- /dev/null +++ b/x-pack/metricbeat/module/redisenterprise/node/_meta/testdata/redis-node.5.4.10-22.plain @@ -0,0 +1,144 @@ +# HELP node_cpu_nice_min +# TYPE node_cpu_nice_min gauge +node_cpu_nice_min{cluster="cluster.local",node="1"} 0.0 +# HELP node_cpu_idle_max +# TYPE node_cpu_idle_max gauge +node_cpu_idle_max{cluster="cluster.local",node="1"} 0.914 +# HELP node_egress_bytes +# TYPE node_egress_bytes gauge +node_egress_bytes{cluster="cluster.local",node="1"} 14291.444 +# HELP node_cpu_idle +# TYPE node_cpu_idle gauge +node_cpu_idle{cluster="cluster.local",node="1"} 0.914 +# HELP node_cpu_irqs +# TYPE node_cpu_irqs gauge +node_cpu_irqs{cluster="cluster.local",node="1"} 0.001 +# HELP node_cpu_system_median +# TYPE node_cpu_system_median gauge +node_cpu_system_median{cluster="cluster.local",node="1"} 0.022 +# HELP node_cpu_user +# TYPE node_cpu_user gauge +node_cpu_user{cluster="cluster.local",node="1"} 0.033 +# HELP node_cpu_irqs_min +# TYPE node_cpu_irqs_min gauge +node_cpu_irqs_min{cluster="cluster.local",node="1"} 0.001 +# HELP node_ingress_bytes_min +# TYPE node_ingress_bytes_min gauge +node_ingress_bytes_min{cluster="cluster.local",node="1"} 1006.111 +# HELP node_cpu_system_min +# TYPE node_cpu_system_min gauge +node_cpu_system_min{cluster="cluster.local",node="1"} 0.022 +# HELP node_cpu_irqs_median +# TYPE node_cpu_irqs_median gauge +node_cpu_irqs_median{cluster="cluster.local",node="1"} 0.001 +# HELP node_cpu_steal +# TYPE node_cpu_steal gauge +node_cpu_steal{cluster="cluster.local",node="1"} 0.0 +# HELP node_cpu_user_min +# TYPE node_cpu_user_min gauge +node_cpu_user_min{cluster="cluster.local",node="1"} 0.033 +# HELP node_persistent_storage_free +# TYPE node_persistent_storage_free gauge +node_persistent_storage_free{cluster="cluster.local",node="1"} 55931361507.556 +# HELP node_ephemeral_storage_free +# TYPE node_ephemeral_storage_free gauge +node_ephemeral_storage_free{cluster="cluster.local",node="1"} 55931361507.556 +# HELP node_cpu_idle_median +# TYPE node_cpu_idle_median gauge +node_cpu_idle_median{cluster="cluster.local",node="1"} 0.914 +# HELP node_provisional_memory_no_overbooking +# TYPE node_provisional_memory_no_overbooking gauge +node_provisional_memory_no_overbooking{cluster="cluster.local",node="1"} 5099636117.889 +# HELP node_free_memory +# TYPE node_free_memory gauge +node_free_memory{cluster="cluster.local",node="1"} 6268220757.333 +# HELP node_ingress_bytes +# TYPE node_ingress_bytes gauge +node_ingress_bytes{cluster="cluster.local",node="1"} 1006.111 +# HELP node_cpu_steal_median +# TYPE node_cpu_steal_median gauge +node_cpu_steal_median{cluster="cluster.local",node="1"} 0.0 +# HELP node_conns +# TYPE node_conns gauge +node_conns{cluster="cluster.local",node="1"} 0.0 +# HELP node_ephemeral_storage_avail +# TYPE node_ephemeral_storage_avail gauge +node_ephemeral_storage_avail{cluster="cluster.local",node="1"} 52714645731.556 +# HELP node_cpu_nice +# TYPE node_cpu_nice gauge +node_cpu_nice{cluster="cluster.local",node="1"} 0.0 +# HELP node_ingress_bytes_median +# TYPE node_ingress_bytes_median gauge +node_ingress_bytes_median{cluster="cluster.local",node="1"} 1006.111 +# HELP node_ingress_bytes_max +# TYPE node_ingress_bytes_max gauge +node_ingress_bytes_max{cluster="cluster.local",node="1"} 1006.111 +# HELP node_cpu_iowait +# TYPE node_cpu_iowait gauge +node_cpu_iowait{cluster="cluster.local",node="1"} 0.0 +# HELP node_cpu_user_median +# TYPE node_cpu_user_median gauge +node_cpu_user_median{cluster="cluster.local",node="1"} 0.033 +# HELP node_available_memory_no_overbooking +# TYPE node_available_memory_no_overbooking gauge +node_available_memory_no_overbooking{cluster="cluster.local",node="1"} 6270062136.889 +# HELP node_cpu_nice_median +# TYPE node_cpu_nice_median gauge +node_cpu_nice_median{cluster="cluster.local",node="1"} 0.0 +# HELP node_total_req +# TYPE node_total_req gauge +node_total_req{cluster="cluster.local",node="1"} 0.0 +# HELP node_cur_aof_rewrites +# TYPE node_cur_aof_rewrites gauge +node_cur_aof_rewrites{cluster="cluster.local",node="1"} 0.0 +# HELP node_cpu_iowait_max +# TYPE node_cpu_iowait_max gauge +node_cpu_iowait_max{cluster="cluster.local",node="1"} 0.0 +# HELP node_persistent_storage_avail +# TYPE node_persistent_storage_avail gauge +node_persistent_storage_avail{cluster="cluster.local",node="1"} 52714645731.556 +# HELP node_available_memory +# TYPE node_available_memory gauge +node_available_memory{cluster="cluster.local",node="1"} 6270261475.556 +# HELP node_egress_bytes_min +# TYPE node_egress_bytes_min gauge +node_egress_bytes_min{cluster="cluster.local",node="1"} 14291.444 +# HELP node_cpu_user_max +# TYPE node_cpu_user_max gauge +node_cpu_user_max{cluster="cluster.local",node="1"} 0.033 +# HELP node_cpu_steal_min +# TYPE node_cpu_steal_min gauge +node_cpu_steal_min{cluster="cluster.local",node="1"} 0.0 +# HELP node_cpu_irqs_max +# TYPE node_cpu_irqs_max gauge +node_cpu_irqs_max{cluster="cluster.local",node="1"} 0.001 +# HELP node_cpu_steal_max +# TYPE node_cpu_steal_max gauge +node_cpu_steal_max{cluster="cluster.local",node="1"} 0.0 +# HELP node_cpu_iowait_min +# TYPE node_cpu_iowait_min gauge +node_cpu_iowait_min{cluster="cluster.local",node="1"} 0.0 +# HELP node_cpu_idle_min +# TYPE node_cpu_idle_min gauge +node_cpu_idle_min{cluster="cluster.local",node="1"} 0.914 +# HELP node_cpu_system_max +# TYPE node_cpu_system_max gauge +node_cpu_system_max{cluster="cluster.local",node="1"} 0.022 +# HELP node_egress_bytes_max +# TYPE node_egress_bytes_max gauge +node_egress_bytes_max{cluster="cluster.local",node="1"} 14291.444 +# HELP node_cpu_nice_max +# TYPE node_cpu_nice_max gauge +node_cpu_nice_max{cluster="cluster.local",node="1"} 0.0 +# HELP node_cpu_system +# TYPE node_cpu_system gauge +node_cpu_system{cluster="cluster.local",node="1"} 0.022 +# HELP node_provisional_memory +# TYPE node_provisional_memory gauge +node_provisional_memory{cluster="cluster.local",node="1"} 5099835456.556 +# HELP node_egress_bytes_median +# TYPE node_egress_bytes_median gauge +node_egress_bytes_median{cluster="cluster.local",node="1"} 14291.444 +# HELP node_cpu_iowait_median +# TYPE node_cpu_iowait_median gauge +node_cpu_iowait_median{cluster="cluster.local",node="1"} 0.0 diff --git a/x-pack/metricbeat/module/redisenterprise/node/_meta/testdata/redis-node.5.4.10-22.plain-expected.json b/x-pack/metricbeat/module/redisenterprise/node/_meta/testdata/redis-node.5.4.10-22.plain-expected.json new file mode 100644 index 00000000000..cd9d76d60bb --- /dev/null +++ b/x-pack/metricbeat/module/redisenterprise/node/_meta/testdata/redis-node.5.4.10-22.plain-expected.json @@ -0,0 +1,75 @@ +[ + { + "event": { + "dataset": "redisenterprise.node", + "duration": 115000, + "module": "redisenterprise" + }, + "metricset": { + "name": "node", + "period": 10000 + }, + "prometheus": { + "labels": { + "cluster": "cluster.local", + "instance": "127.0.0.1:55467", + "job": "redisenterprise", + "node": "1" + }, + "metrics": { + "node_available_memory": 6270261475.556, + "node_available_memory_no_overbooking": 6270062136.889, + "node_conns": 0, + "node_cpu_idle": 0.914, + "node_cpu_idle_max": 0.914, + "node_cpu_idle_median": 0.914, + "node_cpu_idle_min": 0.914, + "node_cpu_iowait": 0, + "node_cpu_iowait_max": 0, + "node_cpu_iowait_median": 0, + "node_cpu_iowait_min": 0, + "node_cpu_irqs": 0.001, + "node_cpu_irqs_max": 0.001, + "node_cpu_irqs_median": 0.001, + "node_cpu_irqs_min": 0.001, + "node_cpu_nice": 0, + "node_cpu_nice_max": 0, + "node_cpu_nice_median": 0, + "node_cpu_nice_min": 0, + "node_cpu_steal": 0, + "node_cpu_steal_max": 0, + "node_cpu_steal_median": 0, + "node_cpu_steal_min": 0, + "node_cpu_system": 0.022, + "node_cpu_system_max": 0.022, + "node_cpu_system_median": 0.022, + "node_cpu_system_min": 0.022, + "node_cpu_user": 0.033, + "node_cpu_user_max": 0.033, + "node_cpu_user_median": 0.033, + "node_cpu_user_min": 0.033, + "node_cur_aof_rewrites": 0, + "node_egress_bytes": 14291.444, + "node_egress_bytes_max": 14291.444, + "node_egress_bytes_median": 14291.444, + "node_egress_bytes_min": 14291.444, + "node_ephemeral_storage_avail": 52714645731.556, + "node_ephemeral_storage_free": 55931361507.556, + "node_free_memory": 6268220757.333, + "node_ingress_bytes": 1006.111, + "node_ingress_bytes_max": 1006.111, + "node_ingress_bytes_median": 1006.111, + "node_ingress_bytes_min": 1006.111, + "node_persistent_storage_avail": 52714645731.556, + "node_persistent_storage_free": 55931361507.556, + "node_provisional_memory": 5099835456.556, + "node_provisional_memory_no_overbooking": 5099636117.889, + "node_total_req": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "redisenterprise" + } + } +] diff --git a/x-pack/metricbeat/module/redisenterprise/node/manifest.yml b/x-pack/metricbeat/module/redisenterprise/node/manifest.yml new file mode 100644 index 00000000000..3cf6f512a44 --- /dev/null +++ b/x-pack/metricbeat/module/redisenterprise/node/manifest.yml @@ -0,0 +1,9 @@ +default: true +input: + module: prometheus + metricset: collector + defaults: + metrics_path: / + metrics_filters: + include: ["node_*"] + exclude: ["up"] diff --git a/x-pack/metricbeat/module/redisenterprise/node/node_integration_test.go b/x-pack/metricbeat/module/redisenterprise/node/node_integration_test.go new file mode 100644 index 00000000000..3eda84a1cf1 --- /dev/null +++ b/x-pack/metricbeat/module/redisenterprise/node/node_integration_test.go @@ -0,0 +1,48 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build integration + +package node + +import ( + "os" + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/libbeat/tests/compose" + "github.com/elastic/beats/metricbeat/mb" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + _ "github.com/elastic/beats/metricbeat/module/prometheus" + _ "github.com/elastic/beats/metricbeat/module/prometheus/collector" +) + +func init() { + // To be moved to some kind of helper + os.Setenv("BEAT_STRICT_PERMS", "false") + mb.Registry.SetSecondarySource(mb.NewLightModulesSource("../../../module")) +} + +func TestFetch(t *testing.T) { + service := compose.EnsureUp(t, "redisenterprise", compose.UpWithTimeout(5*time.Minute)) + + f := mbtest.NewFetcher(t, getConfig(service.Host())) + events, errs := f.FetchEvents() + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) + t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events[0]) +} + +func getConfig(host string) map[string]interface{} { + return map[string]interface{}{ + "module": "redisenterprise", + "metricsets": []string{"node"}, + "hosts": []string{host}, + "ssl.verification_mode": "none", + } +} diff --git a/x-pack/metricbeat/module/redisenterprise/node/node_test.go b/x-pack/metricbeat/module/redisenterprise/node/node_test.go new file mode 100644 index 00000000000..153a589d7a6 --- /dev/null +++ b/x-pack/metricbeat/module/redisenterprise/node/node_test.go @@ -0,0 +1,32 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build !integration + +package node + +import ( + "os" + "testing" + + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/metricbeat/mb" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + + // Register input module and metricset + _ "github.com/elastic/beats/metricbeat/module/prometheus" + _ "github.com/elastic/beats/metricbeat/module/prometheus/collector" +) + +func init() { + // To be moved to some kind of helper + os.Setenv("BEAT_STRICT_PERMS", "false") + mb.Registry.SetSecondarySource(mb.NewLightModulesSource("../../../module")) +} + +func TestEventMapping(t *testing.T) { + logp.TestingSetup() + + mbtest.TestDataFiles(t, "redisenterprise", "node") +} diff --git a/x-pack/metricbeat/module/redisenterprise/proxy/_meta/data.json b/x-pack/metricbeat/module/redisenterprise/proxy/_meta/data.json new file mode 100644 index 00000000000..f58ad718622 --- /dev/null +++ b/x-pack/metricbeat/module/redisenterprise/proxy/_meta/data.json @@ -0,0 +1,86 @@ +{ + "@timestamp": "2020-02-21T08:25:46.123Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" + }, + "metricset": { + "period": 60000, + "name": "proxy" + }, + "service": { + "type": "redisenterprise", + "address": "127.0.0.1:8070" + }, + "ecs": { + "version": "1.4.0" + }, + "host": { + "hostname": "host", + "architecture": "x86_64", + "os": { + "family": "darwin", + "name": "Mac OS X", + "kernel": "18.7.0", + "build": "18G95", + "platform": "darwin", + "version": "10.14.6" + }, + "name": "host", + "id": "24F065F8-4274-521D-8DD5-5D27557E15B4", + "ip": [ + "fe80::aede:48ff:fe00:1122", + "fe80::1c05:593c:e271:97ed", + "192.168.0.14", + "fe80::488c:55ff:fe4e:3b46", + "fe80::def:1653:5676:ea4e", + "fe80::f191:956a:99ff:1b98", + "fe80::3128:a84c:7b38:e3a9" + ], + "mac": [ + "ac:de:48:00:11:22", + "a6:83:e7:ae:70:01", + "a4:83:e7:ae:70:01", + "06:83:e7:ae:70:01" + ] + }, + "agent": { + "ephemeral_id": "de3a043f-a71f-441e-b730-ca12a8f7fc49", + "hostname": "MacBook-Elastic.local", + "id": "993db589-f9d8-40f3-94cb-43b2b69fa9f4", + "version": "8.0.0", + "type": "metricbeat" + }, + "prometheus": { + "labels": { + "cluster": "cluster.local", + "proxy": "1", + "instance": "127.0.0.1:8070", + "job": "redisenterprise" + }, + "metrics": { + "listener_acc_latency": 0, + "listener_acc_latency_max": 0, + "listener_acc_other_latency": 0, + "listener_acc_other_latency_max": 0, + "listener_acc_read_latency": 0, + "listener_acc_read_latency_max": 0, + "listener_acc_write_latency": 0, + "listener_acc_write_latency_max": 0, + "listener_auth_cmds": 0, + "listener_auth_cmds_max": 0, + "listener_auth_errors": 0, + "listener_auth_errors_max": 0, + "listener_cmd_flush": 0, + "listener_cmd_flush_max": 0, + "listener_cmd_get": 0, + "listener_cmd_get_max": 0 + } + }, + "event": { + "dataset": "redisenterprise.proxy", + "module": "redisenterprise", + "duration": 51870343 + } +} diff --git a/x-pack/metricbeat/module/redisenterprise/proxy/_meta/docs.asciidoc b/x-pack/metricbeat/module/redisenterprise/proxy/_meta/docs.asciidoc new file mode 100644 index 00000000000..948e0e38ab2 --- /dev/null +++ b/x-pack/metricbeat/module/redisenterprise/proxy/_meta/docs.asciidoc @@ -0,0 +1 @@ +This is the `proxy` metricset of the redisenterprise module. The metricset is compatible with Redis Enterprise Software. diff --git a/x-pack/metricbeat/module/redisenterprise/proxy/_meta/fields.yml b/x-pack/metricbeat/module/redisenterprise/proxy/_meta/fields.yml new file mode 100644 index 00000000000..1c1fa1b036d --- /dev/null +++ b/x-pack/metricbeat/module/redisenterprise/proxy/_meta/fields.yml @@ -0,0 +1,4 @@ +- name: proxy + type: group + release: beta + fields: diff --git a/x-pack/metricbeat/module/redisenterprise/proxy/_meta/testdata/config.yml b/x-pack/metricbeat/module/redisenterprise/proxy/_meta/testdata/config.yml new file mode 100644 index 00000000000..361e43c5ccd --- /dev/null +++ b/x-pack/metricbeat/module/redisenterprise/proxy/_meta/testdata/config.yml @@ -0,0 +1,4 @@ +type: http +url: "/" +suffix: plain +remove_fields_from_comparison: ["prometheus.labels.instance"] diff --git a/x-pack/metricbeat/module/redisenterprise/proxy/_meta/testdata/redis-proxy.5.4.10-22.plain b/x-pack/metricbeat/module/redisenterprise/proxy/_meta/testdata/redis-proxy.5.4.10-22.plain new file mode 100644 index 00000000000..423aaadae08 --- /dev/null +++ b/x-pack/metricbeat/module/redisenterprise/proxy/_meta/testdata/redis-proxy.5.4.10-22.plain @@ -0,0 +1,168 @@ +# HELP listener_acc_other_latency_max +# TYPE listener_acc_other_latency_max counter +listener_acc_other_latency_max{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_cmd_touch_max +# TYPE listener_cmd_touch_max counter +listener_cmd_touch_max{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_egress_bytes +# TYPE listener_egress_bytes counter +listener_egress_bytes{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_read_req_max +# TYPE listener_read_req_max counter +listener_read_req_max{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_monitor_sessions_count +# TYPE listener_monitor_sessions_count counter +listener_monitor_sessions_count{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_write_req_max +# TYPE listener_write_req_max counter +listener_write_req_max{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_write_res_max +# TYPE listener_write_res_max counter +listener_write_res_max{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_write_started_res_max +# TYPE listener_write_started_res_max counter +listener_write_started_res_max{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_total_res_max +# TYPE listener_total_res_max counter +listener_total_res_max{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_acc_other_latency +# TYPE listener_acc_other_latency counter +listener_acc_other_latency{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_total_started_res_max +# TYPE listener_total_started_res_max counter +listener_total_started_res_max{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_total_started_res +# TYPE listener_total_started_res counter +listener_total_started_res{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_other_req_max +# TYPE listener_other_req_max counter +listener_other_req_max{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_write_req +# TYPE listener_write_req counter +listener_write_req{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_write_res +# TYPE listener_write_res counter +listener_write_res{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_cmd_flush_max +# TYPE listener_cmd_flush_max counter +listener_cmd_flush_max{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_cmd_touch +# TYPE listener_cmd_touch counter +listener_cmd_touch{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_auth_cmds +# TYPE listener_auth_cmds counter +listener_auth_cmds{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_ingress_bytes +# TYPE listener_ingress_bytes counter +listener_ingress_bytes{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_acc_write_latency +# TYPE listener_acc_write_latency counter +listener_acc_write_latency{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_conns +# TYPE listener_conns counter +listener_conns{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_write_started_res +# TYPE listener_write_started_res counter +listener_write_started_res{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_acc_latency +# TYPE listener_acc_latency counter +listener_acc_latency{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_acc_read_latency_max +# TYPE listener_acc_read_latency_max counter +listener_acc_read_latency_max{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_ingress_bytes_max +# TYPE listener_ingress_bytes_max counter +listener_ingress_bytes_max{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_total_connections_received +# TYPE listener_total_connections_received counter +listener_total_connections_received{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_last_req_time +# TYPE listener_last_req_time counter +listener_last_req_time{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_other_res +# TYPE listener_other_res counter +listener_other_res{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_other_started_res_max +# TYPE listener_other_started_res_max counter +listener_other_started_res_max{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_other_req +# TYPE listener_other_req counter +listener_other_req{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_max_connections_exceeded +# TYPE listener_max_connections_exceeded counter +listener_max_connections_exceeded{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_cmd_get +# TYPE listener_cmd_get counter +listener_cmd_get{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_total_res +# TYPE listener_total_res counter +listener_total_res{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_total_req +# TYPE listener_total_req counter +listener_total_req{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_read_res_max +# TYPE listener_read_res_max counter +listener_read_res_max{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_acc_latency_max +# TYPE listener_acc_latency_max counter +listener_acc_latency_max{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_cmd_set +# TYPE listener_cmd_set counter +listener_cmd_set{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_total_connections_received_max +# TYPE listener_total_connections_received_max counter +listener_total_connections_received_max{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_auth_cmds_max +# TYPE listener_auth_cmds_max counter +listener_auth_cmds_max{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_acc_write_latency_max +# TYPE listener_acc_write_latency_max counter +listener_acc_write_latency_max{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_auth_errors_max +# TYPE listener_auth_errors_max counter +listener_auth_errors_max{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_total_req_max +# TYPE listener_total_req_max counter +listener_total_req_max{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_max_connections_exceeded_max +# TYPE listener_max_connections_exceeded_max counter +listener_max_connections_exceeded_max{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_last_res_time +# TYPE listener_last_res_time counter +listener_last_res_time{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_cmd_flush +# TYPE listener_cmd_flush counter +listener_cmd_flush{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_acc_read_latency +# TYPE listener_acc_read_latency counter +listener_acc_read_latency{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_read_started_res +# TYPE listener_read_started_res counter +listener_read_started_res{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_egress_bytes_max +# TYPE listener_egress_bytes_max counter +listener_egress_bytes_max{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_cmd_set_max +# TYPE listener_cmd_set_max counter +listener_cmd_set_max{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_auth_errors +# TYPE listener_auth_errors counter +listener_auth_errors{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_other_started_res +# TYPE listener_other_started_res counter +listener_other_started_res{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_read_started_res_max +# TYPE listener_read_started_res_max counter +listener_read_started_res_max{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_cmd_get_max +# TYPE listener_cmd_get_max counter +listener_cmd_get_max{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_other_res_max +# TYPE listener_other_res_max counter +listener_other_res_max{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_read_req +# TYPE listener_read_req counter +listener_read_req{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 +# HELP listener_read_res +# TYPE listener_read_res counter +listener_read_res{bdb="1",cluster="cluster.local",endpoint="1:1",node="1",port="12000",proxy="1:1:1"} 0.0 diff --git a/x-pack/metricbeat/module/redisenterprise/proxy/_meta/testdata/redis-proxy.5.4.10-22.plain-expected.json b/x-pack/metricbeat/module/redisenterprise/proxy/_meta/testdata/redis-proxy.5.4.10-22.plain-expected.json new file mode 100644 index 00000000000..0d155f6664c --- /dev/null +++ b/x-pack/metricbeat/module/redisenterprise/proxy/_meta/testdata/redis-proxy.5.4.10-22.plain-expected.json @@ -0,0 +1,87 @@ +[ + { + "event": { + "dataset": "redisenterprise.proxy", + "duration": 115000, + "module": "redisenterprise" + }, + "metricset": { + "name": "proxy", + "period": 10000 + }, + "prometheus": { + "labels": { + "bdb": "1", + "cluster": "cluster.local", + "endpoint": "1:1", + "instance": "127.0.0.1:58281", + "job": "redisenterprise", + "node": "1", + "port": "12000", + "proxy": "1:1:1" + }, + "metrics": { + "listener_acc_latency": 0, + "listener_acc_latency_max": 0, + "listener_acc_other_latency": 0, + "listener_acc_other_latency_max": 0, + "listener_acc_read_latency": 0, + "listener_acc_read_latency_max": 0, + "listener_acc_write_latency": 0, + "listener_acc_write_latency_max": 0, + "listener_auth_cmds": 0, + "listener_auth_cmds_max": 0, + "listener_auth_errors": 0, + "listener_auth_errors_max": 0, + "listener_cmd_flush": 0, + "listener_cmd_flush_max": 0, + "listener_cmd_get": 0, + "listener_cmd_get_max": 0, + "listener_cmd_set": 0, + "listener_cmd_set_max": 0, + "listener_cmd_touch": 0, + "listener_cmd_touch_max": 0, + "listener_conns": 0, + "listener_egress_bytes": 0, + "listener_egress_bytes_max": 0, + "listener_ingress_bytes": 0, + "listener_ingress_bytes_max": 0, + "listener_last_req_time": 0, + "listener_last_res_time": 0, + "listener_max_connections_exceeded": 0, + "listener_max_connections_exceeded_max": 0, + "listener_monitor_sessions_count": 0, + "listener_other_req": 0, + "listener_other_req_max": 0, + "listener_other_res": 0, + "listener_other_res_max": 0, + "listener_other_started_res": 0, + "listener_other_started_res_max": 0, + "listener_read_req": 0, + "listener_read_req_max": 0, + "listener_read_res": 0, + "listener_read_res_max": 0, + "listener_read_started_res": 0, + "listener_read_started_res_max": 0, + "listener_total_connections_received": 0, + "listener_total_connections_received_max": 0, + "listener_total_req": 0, + "listener_total_req_max": 0, + "listener_total_res": 0, + "listener_total_res_max": 0, + "listener_total_started_res": 0, + "listener_total_started_res_max": 0, + "listener_write_req": 0, + "listener_write_req_max": 0, + "listener_write_res": 0, + "listener_write_res_max": 0, + "listener_write_started_res": 0, + "listener_write_started_res_max": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "redisenterprise" + } + } +] diff --git a/x-pack/metricbeat/module/redisenterprise/proxy/manifest.yml b/x-pack/metricbeat/module/redisenterprise/proxy/manifest.yml new file mode 100644 index 00000000000..20a090d9da3 --- /dev/null +++ b/x-pack/metricbeat/module/redisenterprise/proxy/manifest.yml @@ -0,0 +1,9 @@ +default: true +input: + module: prometheus + metricset: collector + defaults: + metrics_path: / + metrics_filters: + include: ["listener_*"] + exclude: ["up"] diff --git a/x-pack/metricbeat/module/redisenterprise/proxy/proxy_integration_test.go b/x-pack/metricbeat/module/redisenterprise/proxy/proxy_integration_test.go new file mode 100644 index 00000000000..f103b99ee32 --- /dev/null +++ b/x-pack/metricbeat/module/redisenterprise/proxy/proxy_integration_test.go @@ -0,0 +1,48 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build integration + +package proxy + +import ( + "os" + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/libbeat/tests/compose" + "github.com/elastic/beats/metricbeat/mb" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + _ "github.com/elastic/beats/metricbeat/module/prometheus" + _ "github.com/elastic/beats/metricbeat/module/prometheus/collector" +) + +func init() { + // To be moved to some kind of helper + os.Setenv("BEAT_STRICT_PERMS", "false") + mb.Registry.SetSecondarySource(mb.NewLightModulesSource("../../../module")) +} + +func TestFetch(t *testing.T) { + service := compose.EnsureUp(t, "redisenterprise", compose.UpWithTimeout(5*time.Minute)) + + f := mbtest.NewFetcher(t, getConfig(service.Host())) + events, errs := f.FetchEvents() + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) + t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events[0]) +} + +func getConfig(host string) map[string]interface{} { + return map[string]interface{}{ + "module": "redisenterprise", + "metricsets": []string{"proxy"}, + "hosts": []string{host}, + "ssl.verification_mode": "none", + } +} diff --git a/x-pack/metricbeat/module/redisenterprise/proxy/proxy_test.go b/x-pack/metricbeat/module/redisenterprise/proxy/proxy_test.go new file mode 100644 index 00000000000..fb68fdbeb12 --- /dev/null +++ b/x-pack/metricbeat/module/redisenterprise/proxy/proxy_test.go @@ -0,0 +1,32 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build !integration + +package proxy + +import ( + "os" + "testing" + + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/metricbeat/mb" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + + // Register input module and metricset + _ "github.com/elastic/beats/metricbeat/module/prometheus" + _ "github.com/elastic/beats/metricbeat/module/prometheus/collector" +) + +func init() { + // To be moved to some kind of helper + os.Setenv("BEAT_STRICT_PERMS", "false") + mb.Registry.SetSecondarySource(mb.NewLightModulesSource("../../../module")) +} + +func TestEventMapping(t *testing.T) { + logp.TestingSetup() + + mbtest.TestDataFiles(t, "redisenterprise", "proxy") +} diff --git a/x-pack/metricbeat/module/redisenterprise/test_redisenterprise.py b/x-pack/metricbeat/module/redisenterprise/test_redisenterprise.py new file mode 100644 index 00000000000..806f7a37c7a --- /dev/null +++ b/x-pack/metricbeat/module/redisenterprise/test_redisenterprise.py @@ -0,0 +1,50 @@ +import os +from parameterized import parameterized +import redis +import sys +import unittest + +sys.path.append(os.path.join(os.path.dirname(__file__), '../../tests/system')) +from xpack_metricbeat import XPackTest, metricbeat + + +@metricbeat.parameterized_with_supported_versions +class Test(XPackTest): + + COMPOSE_SERVICES = ['redisenterprise'] + + @parameterized.expand([ + ("node", "node"), + ("proxy", "listener") + ]) + @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") + def test_metricset(self, metricset, metric_name_prefix): + """ + Test redis enterprise metricset + """ + + self.render_config_template(modules=[{ + "name": "redisenterprise", + "metricsets": [metricset], + "hosts": ['https://' + self.compose_host(port='8070/tcp')], + "period": "5s", + "extras": { + "ssl.verification_mode": "none" + } + }]) + proc = self.start_beat(home=self.beat_path) + self.wait_until(lambda: self.output_lines() > 0, max_timeout=120) + proc.check_kill_and_wait() + self.assert_no_logged_warnings(replace=['SSL/TLS verifications disabled.']) + + output = self.read_output_json() + self.assertGreater(len(output), 0) + + for evt in output: + self.assert_fields_are_documented(evt) + self.assertIn("prometheus", evt.keys(), evt) + self.assertIn("metrics", evt["prometheus"].keys(), evt) + self.assertGreater(len(evt["prometheus"]["metrics"].keys()), 0) + + for metric in evt["prometheus"]["metrics"].keys(): + assert metric.startswith(metric_name_prefix + "_") diff --git a/x-pack/metricbeat/modules.d/redisenterprise.yml.disabled b/x-pack/metricbeat/modules.d/redisenterprise.yml.disabled new file mode 100644 index 00000000000..c3121d7c2fb --- /dev/null +++ b/x-pack/metricbeat/modules.d/redisenterprise.yml.disabled @@ -0,0 +1,11 @@ +# Module: redisenterprise +# Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-redisenterprise.html + +- module: redisenterprise + metricsets: + - node + - proxy + period: 1m + + # Metrics endpoint + hosts: ["https://127.0.0.1:8070/"]