[Metricbeat] Redis Enterprise: add "node" and "proxy" metricsets

CHANGELOG.next.asciidoc

@@ -193,6 +193,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add filtering option for prometheus collector. {pull}16420[16420]
 - Add metricsets based on Ceph Manager Daemon to the `ceph` module. {issue}7723[7723] {pull}16254[16254]
 - Release `statsd` module as GA. {pull}16447[16447] {issue}14280[14280]
+- Add `redisenterprise` module. {pull}16482[16482] {issue}15269[15269]
 
 *Packetbeat*
 

libbeat/tests/compose/compose.go

@@ -81,6 +81,13 @@ func EnsureUp(t testing.TB, service string, options ...UpOption) HostInfo {
  // Wait for health
  err = compose.Wait(upOptions.Timeout, service)
  if err != nil {
+ inspected, inspectErr := compose.Inspect(service)
+ if inspectErr != nil {
+ t.Logf("inspection error: %v", err)
+ } else {
+ t.Logf("Container state (service: '%s'): %s", service, inspected)
+ }
+
  t.Fatal(err)
  }
 

libbeat/tests/compose/project.go

@@ -92,6 +92,7 @@ type Driver interface {
  Kill(ctx context.Context, signal string, service string) error
  KillOld(ctx context.Context, except []string) error
  Ps(ctx context.Context, filter ...string) ([]ContainerStatus, error)
+ Inspect(ctx context.Context, serviceName string) (string, error)
 
  LockFile() string
 
@@ -227,6 +228,14 @@ func (c *Project) KillOld(except []string) error {
  return c.Driver.KillOld(context.TODO(), except)
 }
 
+// Inspect a container
+func (c *Project) Inspect(service string) (string, error) {
+ c.Lock()
+ defer c.Unlock()
+
+ return c.Driver.Inspect(context.Background(), service)
+}
+
 // Lock acquires the lock (300s) timeout
 // Normally it should only be seconds that the lock is used, but in some cases it can take longer.
 // Pid is written to the lock file, and it is used to check if process holding the process is still

libbeat/tests/compose/wrapper.go

@@ -21,6 +21,7 @@ import (
  "archive/tar"
  "bytes"
  "context"
+ "encoding/json"
  "fmt"
  "net"
  "os"
@@ -371,6 +372,43 @@ func (d *wrapperDriver) serviceNames(ctx context.Context) ([]string, error) {
  return strings.Fields(stdout.String()), nil
 }
 
+// Inspect a container.
+func (d *wrapperDriver) Inspect(ctx context.Context, serviceName string) (string, error) {
+ list, err := d.client.ContainerList(ctx, types.ContainerListOptions{All: true})
+ if err != nil {
+ return "", errors.Wrap(err, "listing containers to be inspected")
+ }
+
+ var found bool
+ var c types.Container
+ for _, container := range list {
+ aServiceName, ok := container.Labels[labelComposeService]
+ if ok && serviceName == aServiceName {
+ c = container
+ found = true
+ break
+ }
+ }
+
+ if !found {
+ return "", errors.Errorf("container not found for service '%s'", serviceName)
+ }
+
+ inspect, err := d.client.ContainerInspect(ctx, c.ID)
+ if err != nil {
+ return "", errors.Wrap(err, "container failed inspection")
+ } else if inspect.State == nil {
+ return "empty container state", nil
+ }
+
+ state, err := json.Marshal(inspect.State)
+ if err != nil {
+ return "", errors.Wrap(err, "container inspection failed")
+ }
+
+ return string(state), nil
+}
+
 func makeFilter(project, service string, projectFilter Filter) filters.Args {
  f := filters.NewArgs()
  f.Add("label", fmt.Sprintf("%s=%s", labelComposeProject, project))

metricbeat/docs/fields.asciidoc

@@ -65,6 +65,7 @@ grouped in the following categories:
 * <<exported-fields-prometheus>>
 * <<exported-fields-rabbitmq>>
 * <<exported-fields-redis>>
+* <<exported-fields-redisenterprise>>
 * <<exported-fields-sql>>
 * <<exported-fields-stan>>
 * <<exported-fields-statsd>>
@@ -32492,6 +32493,20 @@ type: long
 
 --
 
+[[exported-fields-redisenterprise]]
+== Redis Enterprise fields
+
+Redis metrics collected from Redis Enterprise Server.
+
+
+
+[float]
+=== redisenterprise
+
+`redisenterprise` contains the information and statistics from Redis Enterprise Server.
+
+
+
 [[exported-fields-sql]]
 == sql fields
 

metricbeat/docs/modules/redis.asciidoc

@@ -33,8 +33,8 @@ redis://HOST[:PORT][?password=PASSWORD[&db=DATABASE]]
 [float]
 === Compatibility
 
-The Redis metricsets were tested with Redis 3.2.12, 4.0.11 and 5.0-rc4, and are expected
-to work with all versions >= 3.0.
+The redis metricsets `info`, `key` and `keyspace` are compatible with all distributions of Redis (OSS and enterprise).
+They were tested with Redis 3.2.12, 4.0.11 and 5.0-rc4, and are expected to work with all versions >= 3.0.
 
 
 [float]

metricbeat/docs/modules/redisenterprise.asciidoc

@@ -0,0 +1,62 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-module-redisenterprise]]
+[role="xpack"]
+== Redis Enterprise module
+
+beta[]
+
+This module periodically fetches metrics from https://redislabs.com/redis-enterprise/[Redis Enterprise Software].
+
+The defaut metricsets are `node` and `proxy`.
+
+[float]
+=== Module-specific configuration notes
+
+The Redis module has these additional config options:
+
+*`hosts`*:: URLs that are used to connect to Redis.
+URL format:
+https://HOST[:PORT]
+
+[float]
+=== Compatibility
+
+The metricsets `node` and `proxy` are compatible with Redis Enterprise Software (RES). There were tested with RES
+5.4.10-22 and are expected to work with all versions >= 5.0.2.
+
+
+[float]
+=== Example configuration
+
+The Redis Enterprise module supports the standard configuration options that are described
+in <<configuration-metricbeat>>. Here is an example configuration:
+
+[source,yaml]
+----
+metricbeat.modules:
+- module: redisenterprise
+ metricsets:
+ - node
+ - proxy
+ period: 1m
+
+ # Metrics endpoint
+ hosts: ["https://127.0.0.1:8070/"]
+----
+
+[float]
+=== Metricsets
+
+The following metricsets are available:
+
+* <<metricbeat-metricset-redisenterprise-node,node>>
+
+* <<metricbeat-metricset-redisenterprise-proxy,proxy>>
+
+include::redisenterprise/node.asciidoc[]
+
+include::redisenterprise/proxy.asciidoc[]
+

metricbeat/docs/modules/redisenterprise/node.asciidoc

@@ -0,0 +1,24 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-metricset-redisenterprise-node]]
+=== Redis Enterprise node metricset
+
+beta[]
+
+include::../../../../x-pack/metricbeat/module/redisenterprise/node/_meta/docs.asciidoc[]
+
+This is a default metricset. If the host module is unconfigured, this metricset is enabled by default.
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-redisenterprise,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../../x-pack/metricbeat/module/redisenterprise/node/_meta/data.json[]
+----

metricbeat/docs/modules/redisenterprise/proxy.asciidoc

@@ -0,0 +1,24 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-metricset-redisenterprise-proxy]]
+=== Redis Enterprise proxy metricset
+
+beta[]
+
+include::../../../../x-pack/metricbeat/module/redisenterprise/proxy/_meta/docs.asciidoc[]
+
+This is a default metricset. If the host module is unconfigured, this metricset is enabled by default.
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-redisenterprise,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../../x-pack/metricbeat/module/redisenterprise/proxy/_meta/data.json[]
+----

metricbeat/docs/modules_list.asciidoc

@@ -200,6 +200,9 @@ This file is generated! See scripts/mage/docs_collector.go
 .3+| .3+| |<<metricbeat-metricset-redis-info,info>> 
 |<<metricbeat-metricset-redis-key,key>> 
 |<<metricbeat-metricset-redis-keyspace,keyspace>> 
+|<<metricbeat-module-redisenterprise,Redis Enterprise>> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | 
+.2+| .2+| |<<metricbeat-metricset-redisenterprise-node,node>> beta[] 
+|<<metricbeat-metricset-redisenterprise-proxy,proxy>> beta[] 
 |<<metricbeat-module-sql,sql>> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | 
 .1+| .1+| |<<metricbeat-metricset-sql-query,query>> beta[] 
 |<<metricbeat-module-stan,Stan>> beta[] |image:./images/icon-yes.png[Prebuilt dashboards are available] | 
@@ -294,6 +297,7 @@ include::modules/postgresql.asciidoc[]
 include::modules/prometheus.asciidoc[]
 include::modules/rabbitmq.asciidoc[]
 include::modules/redis.asciidoc[]
+include::modules/redisenterprise.asciidoc[]
 include::modules/sql.asciidoc[]
 include::modules/stan.asciidoc[]
 include::modules/statsd.asciidoc[]

metricbeat/mb/testing/data/data_test.go

@@ -24,9 +24,8 @@ import (
  "strings"
  "testing"
 
- mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-
  _ "github.com/elastic/beats/metricbeat/include"
+ mbtest "github.com/elastic/beats/metricbeat/mb/testing"
 )
 
 func TestAll(t *testing.T) {

metricbeat/module/redis/_meta/docs.asciidoc

@@ -26,5 +26,5 @@ redis://HOST[:PORT][?password=PASSWORD[&db=DATABASE]]
 [float]
 === Compatibility
 
-The Redis metricsets were tested with Redis 3.2.12, 4.0.11 and 5.0-rc4, and are expected
-to work with all versions >= 3.0.
+The redis metricsets `info`, `key` and `keyspace` are compatible with all distributions of Redis (OSS and enterprise).
+They were tested with Redis 3.2.12, 4.0.11 and 5.0-rc4, and are expected to work with all versions >= 3.0.

metricbeat/module/redis/docker-compose.yml

@@ -7,5 +7,6 @@ services:
  context: ./_meta
  args:
  REDIS_VERSION: ${REDIS_VERSION:-3.2.12}
+ privileged: true
  ports:
  - 6379

metricbeat/module/redis/module.yml

@@ -1,3 +1,4 @@
+name: redis
 dashboards:
-  - id: AV4YjZ5pux-M-tCAunxK
-  file: Metricbeat-redis-overview.json
+ - id: AV4YjZ5pux-M-tCAunxK
+ file: Metricbeat-redis-overview.json

metricbeat/module/redis/test_redis.py

@@ -34,6 +34,7 @@ def test_info(self):
  """
  Test redis info metricset
  """
+
  self.render_config_template(modules=[{
  "name": "redis",
  "metricsets": ["info"],
@@ -137,6 +138,7 @@ def test_module_processors(self):
  """
  Test local processors for Redis info event.
  """
+
  fields = ["clients", "cpu"]
  eventFields = ['beat', 'metricset', 'service', 'event']
  eventFields += ['redis.info.' + f for f in fields]

x-pack/metricbeat/metricbeat.reference.yml

@@ -966,6 +966,16 @@ metricbeat.modules:
  # Redis AUTH password. Empty by default.
  #password: foobared
 
+#--------------------------- Redis Enterprise Module ---------------------------
+- module: redisenterprise
+ metricsets:
+ - node
+ - proxy
+ period: 1m
+
+ # Metrics endpoint
+ hosts: ["https://127.0.0.1:8070/"]
+
 #--------------------------------- Sql Module ---------------------------------
 - module: sql
  metricsets:

x-pack/metricbeat/module/redisenterprise/_meta/Dockerfile

@@ -0,0 +1,6 @@
+ARG REDISENTERPRISE_VERSION
+FROM redislabs/redis:${REDISENTERPRISE_VERSION}
+
+# Wait for the health endpoint to have monitors information
+ADD healthcheck.sh /
+HEALTHCHECK --interval=1s --retries=300 CMD /healthcheck.sh

x-pack/metricbeat/module/redisenterprise/_meta/config.reference.yml

@@ -0,0 +1,8 @@
+- module: redisenterprise
+ metricsets:
+ - node
+ - proxy
+ period: 1m
+
+ # Metrics endpoint
+ hosts: ["https://127.0.0.1:8070/"]

x-pack/metricbeat/module/redisenterprise/_meta/config.yml

@@ -0,0 +1,8 @@
+- module: redisenterprise
+ metricsets:
+ - node
+ - proxy
+ period: 1m
+
+ # Metrics endpoint
+ hosts: ["https://127.0.0.1:8070/"]

x-pack/metricbeat/module/redisenterprise/_meta/docs.asciidoc

@@ -0,0 +1,18 @@
+This module periodically fetches metrics from https://redislabs.com/redis-enterprise/[Redis Enterprise Software].
+
+The defaut metricsets are `node` and `proxy`.
+
+[float]
+=== Module-specific configuration notes
+
+The Redis module has these additional config options:
+
+*`hosts`*:: URLs that are used to connect to Redis.
+URL format:
+https://HOST[:PORT]
+
+[float]
+=== Compatibility
+
+The metricsets `node` and `proxy` are compatible with Redis Enterprise Software (RES). There were tested with RES
+5.4.10-22 and are expected to work with all versions >= 5.0.2.

x-pack/metricbeat/module/redisenterprise/_meta/fields.yml

@@ -0,0 +1,11 @@
+- key: redisenterprise
+ title: "Redis Enterprise"
+ description: >
+ Redis metrics collected from Redis Enterprise Server.
+ release: beta
+ fields:
+ - name: redisenterprise
+ type: group
+ description: >
+ `redisenterprise` contains the information and statistics from Redis Enterprise Server.
+ fields: