From f67d8d7df70e1bcd705aeda32672b1dbbe014227 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Mon, 13 Aug 2018 17:32:18 +0200
Subject: [PATCH 01/88] Initial implementation of compose test runner

---
 libbeat/tests/compose/project.go              | 12 ---
 libbeat/tests/compose/runner.go               | 75 +++++++++++++++++++
 metricbeat/docker-compose.yml                 | 10 ++-
 .../keyspace/keyspace_integration_test.go     | 55 +++++++++-----
 4 files changed, 122 insertions(+), 30 deletions(-)
 create mode 100644 libbeat/tests/compose/runner.go

diff --git a/libbeat/tests/compose/project.go b/libbeat/tests/compose/project.go
index 6df5d358560..b19b7e75b99 100644
--- a/libbeat/tests/compose/project.go
+++ b/libbeat/tests/compose/project.go
@@ -94,18 +94,6 @@ func NewProject(name string, files []string) (*Project, error) {
 
 // Start the container, unless it's running already
 func (c *Project) Start(service string) error {
-	servicesStatus, err := c.getServices(service)
-	if err != nil {
-		return err
-	}
-
-	if servicesStatus[service] != nil {
-		if servicesStatus[service].Running {
-			// Someone is running it
-			return nil
-		}
-	}
-
 	c.Lock()
 	defer c.Unlock()
 
diff --git a/libbeat/tests/compose/runner.go b/libbeat/tests/compose/runner.go
new file mode 100644
index 00000000000..9641cde9c4a
--- /dev/null
+++ b/libbeat/tests/compose/runner.go
@@ -0,0 +1,75 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package compose
+
+import (
+	"fmt"
+	"os"
+	"sort"
+	"strings"
+	"testing"
+)
+
+type TestRunner struct {
+	Service  string
+	Options  map[string][]string
+	Parallel bool
+}
+
+func (r *TestRunner) scenarios() []map[string]string {
+	n := 1
+	for _, values := range r.Options {
+		n *= len(values)
+	}
+
+	scenarios := make([]map[string]string, n)
+	for variable, values := range r.Options {
+		v := 0
+		for i, s := range scenarios {
+			if s == nil {
+				s = make(map[string]string)
+				scenarios[i] = s
+			}
+			s[variable] = values[v]
+			v = (v + 1) % len(values)
+		}
+	}
+
+	return scenarios
+}
+
+func (r *TestRunner) Run(t *testing.T, tests ...func(t *testing.T)) {
+	for _, s := range r.scenarios() {
+		var vars []string
+		for k, v := range s {
+			os.Setenv(k, v)
+			vars = append(vars, fmt.Sprintf("%s=%s", k, v))
+		}
+		sort.Strings(vars)
+		desc := strings.Join(vars, ",")
+
+		t.Run(desc, func(t *testing.T) {
+			EnsureUp(t, r.Service)
+			for _, test := range tests {
+				t.Run(desc, test)
+			}
+			// Down(t, r.Service)
+		})
+
+	}
+}
diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 0ddc7bf6cd1..5b009c5276e 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -1,4 +1,5 @@
-version: '2'
+version: "2.4"
+
 services:
   beat:
     build: ${PWD}/.
@@ -157,7 +158,14 @@ services:
     build: ./module/rabbitmq/_meta
 
   redis:
+    image: redis:${REDIS_VERSION:-4.0.11}-${IMAGE_OS:-alpine}
     build: ./module/redis/_meta
+    healthcheck:
+      test:
+      - "CMD-SHELL"
+      - "nc -z localhost 6379"
+      interval: 1s
+      retries: 90
 
   traefik:
     build: ./module/traefik/_meta
diff --git a/metricbeat/module/redis/keyspace/keyspace_integration_test.go b/metricbeat/module/redis/keyspace/keyspace_integration_test.go
index 35710d174e2..a0975519363 100644
--- a/metricbeat/module/redis/keyspace/keyspace_integration_test.go
+++ b/metricbeat/module/redis/keyspace/keyspace_integration_test.go
@@ -33,29 +33,50 @@ import (
 
 var host = redis.GetRedisEnvHost() + ":" + redis.GetRedisEnvPort()
 
-func TestFetch(t *testing.T) {
-	compose.EnsureUp(t, "redis")
+var allVersions = []string{
+	"3.2.4",
+	"4.0.11",
+	"5.0-rc",
+}
 
-	addEntry(t)
+var allOS = []string{
+	"alpine",
+	"stretch",
+}
 
-	// Fetch data
-	f := mbtest.NewEventsFetcher(t, getConfig())
-	events, err := f.Fetch()
-	if err != nil {
-		t.Fatal("fetch", err)
-	}
+var runner = compose.TestRunner{
+	Service: "redis",
+	Options: map[string][]string{
+		"REDIS_VERSION": allVersions,
+		"IMAGE_OS":      allOS,
+	},
+	Parallel: true,
+}
+
+func TestFetch(t *testing.T) {
+	runner.Run(t, func(t *testing.T) {
+		addEntry(t)
+
+		// Fetch data
+		f := mbtest.NewEventsFetcher(t, getConfig())
+		events, err := f.Fetch()
+		if err != nil {
+			t.Fatal("fetch", err)
+		}
 
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events)
+		t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events)
 
-	// Make sure at least 1 db keyspace exists
-	assert.True(t, len(events) > 0)
+		// Make sure at least 1 db keyspace exists
+		assert.True(t, len(events) > 0)
 
-	keyspace := events[0]
+		keyspace := events[0]
 
-	assert.True(t, keyspace["avg_ttl"].(int64) >= 0)
-	assert.True(t, keyspace["expires"].(int64) >= 0)
-	assert.True(t, keyspace["keys"].(int64) >= 0)
-	assert.True(t, strings.Contains(keyspace["id"].(string), "db"))
+		assert.True(t, keyspace["avg_ttl"].(int64) >= 0)
+		assert.True(t, keyspace["expires"].(int64) >= 0)
+		assert.True(t, keyspace["keys"].(int64) >= 0)
+		assert.True(t, strings.Contains(keyspace["id"].(string), "db"))
+	},
+	)
 }
 
 func TestData(t *testing.T) {

From 39cb4922c3feb2fb46deca32aa0e498052e16541 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Mon, 13 Aug 2018 19:32:42 +0200
Subject: [PATCH 02/88] Parallel

---
 libbeat/tests/compose/compose.go              | 30 ++++++---
 libbeat/tests/compose/project.go              | 34 ++++++++++
 libbeat/tests/compose/runner.go               | 40 +++++++++--
 libbeat/tests/compose/wrapper.go              | 40 ++++++++++-
 metricbeat/docker-compose.yml                 | 12 +---
 metricbeat/module/redis/_meta/Dockerfile      |  2 -
 metricbeat/module/redis/docker-compose.yml    | 11 +++
 .../keyspace/keyspace_integration_test.go     | 67 ++++++++++---------
 8 files changed, 177 insertions(+), 59 deletions(-)
 delete mode 100644 metricbeat/module/redis/_meta/Dockerfile
 create mode 100644 metricbeat/module/redis/docker-compose.yml

diff --git a/libbeat/tests/compose/compose.go b/libbeat/tests/compose/compose.go
index 9cd6dd205a8..de1b19ebd8f 100644
--- a/libbeat/tests/compose/compose.go
+++ b/libbeat/tests/compose/compose.go
@@ -40,7 +40,8 @@ func EnsureUpWithTimeout(t *testing.T, timeout int, services ...string) {
 		return
 	}
 
-	compose, err := getComposeProject()
+	name := os.Getenv("DOCKER_COMPOSE_PROJECT_NAME")
+	compose, err := getComposeProject(name)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -65,28 +66,37 @@ func EnsureUpWithTimeout(t *testing.T, timeout int, services ...string) {
 	}
 }
 
-func getComposeProject() (*Project, error) {
+func findComposePath() (string, error) {
 	// find docker-compose
 	path, err := os.Getwd()
 	if err != nil {
-		return nil, err
+		return "", err
 	}
 	for {
 		if path == "/" {
-			return nil, errors.New("docker-compose.yml not found")
+			break
 		}
 
-		if _, err = os.Stat(filepath.Join(path, "docker-compose.yml")); err != nil {
-			path = filepath.Dir(path)
-		} else {
-			break
+		composePath := filepath.Join(path, "docker-compose.yml")
+		if _, err = os.Stat(composePath); err == nil {
+			return composePath, nil
 		}
+		path = filepath.Dir(path)
+	}
+
+	return "", errors.New("docker-compose.yml not found")
+}
+
+func getComposeProject(name string) (*Project, error) {
+	path, err := findComposePath()
+	if err != nil {
+		return nil, err
 	}
 
 	return NewProject(
-		os.Getenv("DOCKER_COMPOSE_PROJECT_NAME"),
+		name,
 		[]string{
-			filepath.Join(path, "docker-compose.yml"),
+			path,
 		},
 	)
 }
diff --git a/libbeat/tests/compose/project.go b/libbeat/tests/compose/project.go
index b19b7e75b99..89249f88d08 100644
--- a/libbeat/tests/compose/project.go
+++ b/libbeat/tests/compose/project.go
@@ -56,10 +56,13 @@ const (
 // Driver is the interface of docker compose implementations
 type Driver interface {
 	Up(ctx context.Context, opts UpOptions, service string) error
+	Down(ctx context.Context) error
 	Kill(ctx context.Context, signal string, service string) error
 	Ps(ctx context.Context, filter ...string) ([]ContainerStatus, error)
 	// Containers(ctx context.Context, projectFilter Filter, filter ...string) ([]string, error)
 
+	SetParameters(map[string]string)
+
 	LockFile() string
 }
 
@@ -69,6 +72,7 @@ type ContainerStatus interface {
 	Healthy() bool
 	Running() bool
 	Old() bool
+	Host() string
 }
 
 // Project is a docker-compose project
@@ -137,6 +141,25 @@ func (c *Project) Wait(seconds int, services ...string) error {
 	return nil
 }
 
+// Host gets the host and port of a service
+func (c *Project) Host(service string) (string, error) {
+	servicesStatus, err := c.getServices(service)
+	if err != nil {
+		return "", err
+	}
+
+	if len(servicesStatus) == 0 {
+		return "", errors.New("no container running for service")
+	}
+
+	status, ok := servicesStatus[service]
+	if !ok || status.Host == "" {
+		return "", errors.New("unknown host:port for service")
+	}
+
+	return status.Host, nil
+}
+
 // Kill a container
 func (c *Project) Kill(service string) error {
 	c.Lock()
@@ -178,6 +201,14 @@ func (c *Project) KillOld(except []string) error {
 	return nil
 }
 
+// Down removes all resources of a project
+func (c *Project) Down() error {
+	c.Lock()
+	defer c.Unlock()
+
+	return c.Driver.Down(context.Background())
+}
+
 // Lock acquires the lock (300s) timeout
 // Normally it should only be seconds that the lock is used, but in some cases it can take longer.
 func (c *Project) Lock() {
@@ -214,6 +245,8 @@ type serviceInfo struct {
 	Running bool
 	Healthy bool
 
+	Host string
+
 	// Has been up for too long?:
 	Old bool
 }
@@ -247,6 +280,7 @@ func (c *Project) getServices(filter ...string) (map[string]*serviceInfo, error)
 		if service.Healthy {
 			service.Old = c.Old()
 		}
+		service.Host = c.Host()
 		result[name] = service
 	}
 
diff --git a/libbeat/tests/compose/runner.go b/libbeat/tests/compose/runner.go
index 9641cde9c4a..97da2db478d 100644
--- a/libbeat/tests/compose/runner.go
+++ b/libbeat/tests/compose/runner.go
@@ -18,6 +18,7 @@
 package compose
 
 import (
+	"crypto/sha1"
 	"fmt"
 	"os"
 	"sort"
@@ -29,6 +30,7 @@ type TestRunner struct {
 	Service  string
 	Options  map[string][]string
 	Parallel bool
+	Timeout  int
 }
 
 func (r *TestRunner) scenarios() []map[string]string {
@@ -53,7 +55,12 @@ func (r *TestRunner) scenarios() []map[string]string {
 	return scenarios
 }
 
-func (r *TestRunner) Run(t *testing.T, tests ...func(t *testing.T)) {
+func (r *TestRunner) Run(t *testing.T, tests ...func(t *testing.T, host string)) {
+	timeout := r.Timeout
+	if timeout == 0 {
+		timeout = 300
+	}
+
 	for _, s := range r.scenarios() {
 		var vars []string
 		for k, v := range s {
@@ -63,12 +70,37 @@ func (r *TestRunner) Run(t *testing.T, tests ...func(t *testing.T)) {
 		sort.Strings(vars)
 		desc := strings.Join(vars, ",")
 
+		name := fmt.Sprintf("%s_%x", r.Service, sha1.Sum([]byte(desc)))
+		project, err := getComposeProject(name)
+		if err != nil {
+			t.Fatal(err)
+		}
+		project.SetParameters(s)
+
 		t.Run(desc, func(t *testing.T) {
-			EnsureUp(t, r.Service)
+			if r.Parallel {
+				t.Parallel()
+			}
+
+			err := project.Start(r.Service)
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer project.Down()
+
+			err = project.Wait(timeout, r.Service)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			host, err := project.Host(r.Service)
+			if err != nil {
+				t.Fatal(err)
+			}
+
 			for _, test := range tests {
-				t.Run(desc, test)
+				test(t, host)
 			}
-			// Down(t, r.Service)
 		})
 
 	}
diff --git a/libbeat/tests/compose/wrapper.go b/libbeat/tests/compose/wrapper.go
index f8ae629584a..9657b66d10b 100644
--- a/libbeat/tests/compose/wrapper.go
+++ b/libbeat/tests/compose/wrapper.go
@@ -20,8 +20,10 @@ package compose
 import (
 	"context"
 	"fmt"
+	"net"
 	"os"
 	"os/exec"
+	"strconv"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -39,6 +41,8 @@ const (
 type wrapperDriver struct {
 	Name  string
 	Files []string
+
+	Environment []string
 }
 
 type wrapperContainer struct {
@@ -61,8 +65,27 @@ func (c *wrapperContainer) Old() bool {
 	return strings.Contains(c.info.Status, "minute")
 }
 
+func (c *wrapperContainer) Host() string {
+	// TODO: Support multiple networks/ports
+	var ip string
+	for _, net := range c.info.NetworkSettings.Networks {
+		if len(net.IPAddress) > 0 {
+			ip = net.IPAddress
+			break
+		}
+	}
+	if len(ip) == 0 {
+		return ""
+	}
+
+	for _, port := range c.info.Ports {
+		return net.JoinHostPort(ip, strconv.Itoa(int(port.PrivatePort)))
+	}
+	return ""
+}
+
 func (d *wrapperDriver) LockFile() string {
-	return d.Files[0] + ".lock"
+	return d.Files[0] + "." + d.Name + ".lock"
 }
 
 func (d *wrapperDriver) cmd(ctx context.Context, command string, arg ...string) *exec.Cmd {
@@ -76,9 +99,20 @@ func (d *wrapperDriver) cmd(ctx context.Context, command string, arg ...string)
 	cmd := exec.CommandContext(ctx, "docker-compose", args...)
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
+	if len(d.Environment) > 0 {
+		cmd.Env = d.Environment
+	}
 	return cmd
 }
 
+func (d *wrapperDriver) SetParameters(params map[string]string) {
+	var env []string
+	for k, v := range params {
+		env = append(env, fmt.Sprintf("%s=%s", k, v))
+	}
+	d.Environment = env
+}
+
 func (d *wrapperDriver) Up(ctx context.Context, opts UpOptions, service string) error {
 	var args []string
 
@@ -99,6 +133,10 @@ func (d *wrapperDriver) Up(ctx context.Context, opts UpOptions, service string)
 	return d.cmd(ctx, "up", args...).Run()
 }
 
+func (d *wrapperDriver) Down(ctx context.Context) error {
+	return d.cmd(ctx, "down", "-v").Run()
+}
+
 func (d *wrapperDriver) Kill(ctx context.Context, signal string, service string) error {
 	var args []string
 
diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 5b009c5276e..8f0205eb756 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -1,4 +1,4 @@
-version: "2.4"
+version: "2"
 
 services:
   beat:
@@ -157,16 +157,6 @@ services:
   rabbitmq:
     build: ./module/rabbitmq/_meta
 
-  redis:
-    image: redis:${REDIS_VERSION:-4.0.11}-${IMAGE_OS:-alpine}
-    build: ./module/redis/_meta
-    healthcheck:
-      test:
-      - "CMD-SHELL"
-      - "nc -z localhost 6379"
-      interval: 1s
-      retries: 90
-
   traefik:
     build: ./module/traefik/_meta
 
diff --git a/metricbeat/module/redis/_meta/Dockerfile b/metricbeat/module/redis/_meta/Dockerfile
deleted file mode 100644
index 8f48048cc5f..00000000000
--- a/metricbeat/module/redis/_meta/Dockerfile
+++ /dev/null
@@ -1,2 +0,0 @@
-FROM redis:3.2.4-alpine
-HEALTHCHECK --interval=1s --retries=90 CMD nc -z localhost 6379
diff --git a/metricbeat/module/redis/docker-compose.yml b/metricbeat/module/redis/docker-compose.yml
new file mode 100644
index 00000000000..8f1f92bede5
--- /dev/null
+++ b/metricbeat/module/redis/docker-compose.yml
@@ -0,0 +1,11 @@
+version: "2.4"
+
+services:
+  redis:
+    image: redis:${REDIS_VERSION:-4.0.11}-${IMAGE_OS:-alpine}
+    healthcheck:
+      test:
+      - "CMD-SHELL"
+      - "nc -z localhost 6379"
+      interval: 1s
+      retries: 90
diff --git a/metricbeat/module/redis/keyspace/keyspace_integration_test.go b/metricbeat/module/redis/keyspace/keyspace_integration_test.go
index a0975519363..0f158f60deb 100644
--- a/metricbeat/module/redis/keyspace/keyspace_integration_test.go
+++ b/metricbeat/module/redis/keyspace/keyspace_integration_test.go
@@ -25,40 +25,33 @@ import (
 
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	"github.com/elastic/beats/metricbeat/module/redis"
 
 	rd "github.com/garyburd/redigo/redis"
 	"github.com/stretchr/testify/assert"
 )
 
-var host = redis.GetRedisEnvHost() + ":" + redis.GetRedisEnvPort()
-
-var allVersions = []string{
-	"3.2.4",
-	"4.0.11",
-	"5.0-rc",
-}
-
-var allOS = []string{
-	"alpine",
-	"stretch",
-}
-
 var runner = compose.TestRunner{
 	Service: "redis",
 	Options: map[string][]string{
-		"REDIS_VERSION": allVersions,
-		"IMAGE_OS":      allOS,
+		"REDIS_VERSION": []string{
+			"3.2.4",
+			"4.0.11",
+			"5.0-rc",
+		},
+		"IMAGE_OS": []string{
+			"alpine",
+			"stretch",
+		},
 	},
 	Parallel: true,
 }
 
 func TestFetch(t *testing.T) {
-	runner.Run(t, func(t *testing.T) {
-		addEntry(t)
+	runner.Run(t, func(t *testing.T, host string) {
+		addEntry(t, host)
 
 		// Fetch data
-		f := mbtest.NewEventsFetcher(t, getConfig())
+		f := mbtest.NewEventsFetcher(t, getConfig(host))
 		events, err := f.Fetch()
 		if err != nil {
 			t.Fatal("fetch", err)
@@ -75,25 +68,37 @@ func TestFetch(t *testing.T) {
 		assert.True(t, keyspace["expires"].(int64) >= 0)
 		assert.True(t, keyspace["keys"].(int64) >= 0)
 		assert.True(t, strings.Contains(keyspace["id"].(string), "db"))
-	},
-	)
+	})
 }
 
 func TestData(t *testing.T) {
-	compose.EnsureUp(t, "redis")
-
-	addEntry(t)
+	// TODO: Fix EnsureUp
+	runner := compose.TestRunner{
+		Service: "redis",
+		Options: map[string][]string{
+			"REDIS_VERSION": []string{
+				"4.0.11",
+			},
+			"IMAGE_OS": []string{
+				"alpine",
+			},
+		},
+		Parallel: true,
+	}
+	runner.Run(t, func(t *testing.T, host string) {
+		addEntry(t, host)
 
-	f := mbtest.NewEventsFetcher(t, getConfig())
+		f := mbtest.NewEventsFetcher(t, getConfig(host))
 
-	err := mbtest.WriteEvents(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
+		err := mbtest.WriteEvents(f, t)
+		if err != nil {
+			t.Fatal("write", err)
+		}
+	})
 }
 
 // addEntry adds an entry to redis
-func addEntry(t *testing.T) {
+func addEntry(t *testing.T, host string) {
 	// Insert at least one event to make sure db exists
 	c, err := rd.Dial("tcp", host)
 	if err != nil {
@@ -106,7 +111,7 @@ func addEntry(t *testing.T) {
 	}
 }
 
-func getConfig() map[string]interface{} {
+func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "redis",
 		"metricsets": []string{"keyspace"},

From 1f12a3e6a3f44ffccae2f6a70666c87fe86b8c8b Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Mon, 13 Aug 2018 22:53:19 +0200
Subject: [PATCH 03/88] More parallel

---
 metricbeat/module/redis/keyspace/keyspace_integration_test.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/metricbeat/module/redis/keyspace/keyspace_integration_test.go b/metricbeat/module/redis/keyspace/keyspace_integration_test.go
index 0f158f60deb..736db48b064 100644
--- a/metricbeat/module/redis/keyspace/keyspace_integration_test.go
+++ b/metricbeat/module/redis/keyspace/keyspace_integration_test.go
@@ -47,6 +47,8 @@ var runner = compose.TestRunner{
 }
 
 func TestFetch(t *testing.T) {
+	t.Parallel()
+
 	runner.Run(t, func(t *testing.T, host string) {
 		addEntry(t, host)
 
@@ -72,6 +74,8 @@ func TestFetch(t *testing.T) {
 }
 
 func TestData(t *testing.T) {
+	t.Parallel()
+
 	// TODO: Fix EnsureUp
 	runner := compose.TestRunner{
 		Service: "redis",

From 56729d7eedb669a1dc261834b1335b3145a726d4 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 14 Aug 2018 18:53:25 +0200
Subject: [PATCH 04/88] --no-ansi for compose

---
 libbeat/tests/compose/wrapper.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libbeat/tests/compose/wrapper.go b/libbeat/tests/compose/wrapper.go
index 9657b66d10b..2e67ef0f785 100644
--- a/libbeat/tests/compose/wrapper.go
+++ b/libbeat/tests/compose/wrapper.go
@@ -90,7 +90,7 @@ func (d *wrapperDriver) LockFile() string {
 
 func (d *wrapperDriver) cmd(ctx context.Context, command string, arg ...string) *exec.Cmd {
 	var args []string
-	args = append(args, "--project-name", d.Name)
+	args = append(args, "--no-ansi", "--project-name", d.Name)
 	for _, f := range d.Files {
 		args = append(args, "--file", f)
 	}

From ce4ab513658a95518d2dcd4de43deed15ed879b9 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 14 Aug 2018 19:27:00 +0200
Subject: [PATCH 05/88] Randomize project name

---
 libbeat/tests/compose/runner.go | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/libbeat/tests/compose/runner.go b/libbeat/tests/compose/runner.go
index 97da2db478d..7a4a7e0a146 100644
--- a/libbeat/tests/compose/runner.go
+++ b/libbeat/tests/compose/runner.go
@@ -18,12 +18,14 @@
 package compose
 
 import (
-	"crypto/sha1"
 	"fmt"
+	"math/rand"
 	"os"
 	"sort"
 	"strings"
 	"testing"
+
+	"github.com/pkg/errors"
 )
 
 type TestRunner struct {
@@ -70,7 +72,10 @@ func (r *TestRunner) Run(t *testing.T, tests ...func(t *testing.T, host string))
 		sort.Strings(vars)
 		desc := strings.Join(vars, ",")
 
-		name := fmt.Sprintf("%s_%x", r.Service, sha1.Sum([]byte(desc)))
+		seq := make([]byte, 16)
+		rand.Read(seq)
+		name := fmt.Sprintf("%s_%x", r.Service, seq)
+
 		project, err := getComposeProject(name)
 		if err != nil {
 			t.Fatal(err)
@@ -90,12 +95,12 @@ func (r *TestRunner) Run(t *testing.T, tests ...func(t *testing.T, host string))
 
 			err = project.Wait(timeout, r.Service)
 			if err != nil {
-				t.Fatal(err)
+				t.Fatal(errors.Wrapf(err, "waiting for %s/%s", r.Service, desc))
 			}
 
 			host, err := project.Host(r.Service)
 			if err != nil {
-				t.Fatal(err)
+				t.Fatal(errors.Wrapf(err, "getting host for %s/%s", r.Service, desc))
 			}
 
 			for _, test := range tests {

From c2b8060b879716c13daddba11b2b89018715495d Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 14 Aug 2018 19:51:30 +0200
Subject: [PATCH 06/88] Seed random generator for compose project names

---
 libbeat/tests/compose/runner.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/libbeat/tests/compose/runner.go b/libbeat/tests/compose/runner.go
index 7a4a7e0a146..97cbcac1f79 100644
--- a/libbeat/tests/compose/runner.go
+++ b/libbeat/tests/compose/runner.go
@@ -24,10 +24,15 @@ import (
 	"sort"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/pkg/errors"
 )
 
+func init() {
+	rand.Seed(time.Now().UnixNano())
+}
+
 type TestRunner struct {
 	Service  string
 	Options  map[string][]string

From 166709d165c5875897f2acf741a13ec784bf6626 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 14 Aug 2018 19:51:53 +0200
Subject: [PATCH 07/88] Migrate info metricset

---
 .../redis/info/info_integration_test.go       | 104 +++++++++---------
 .../keyspace/keyspace_integration_test.go     |  36 +-----
 metricbeat/module/redis/mtest/runner.go       |  36 ++++++
 metricbeat/module/redis/testing.go            |  48 --------
 4 files changed, 94 insertions(+), 130 deletions(-)
 create mode 100644 metricbeat/module/redis/mtest/runner.go
 delete mode 100644 metricbeat/module/redis/testing.go

diff --git a/metricbeat/module/redis/info/info_integration_test.go b/metricbeat/module/redis/info/info_integration_test.go
index bbcfd1b8601..1f5ef18796b 100644
--- a/metricbeat/module/redis/info/info_integration_test.go
+++ b/metricbeat/module/redis/info/info_integration_test.go
@@ -23,9 +23,8 @@ import (
 	"testing"
 
 	"github.com/elastic/beats/libbeat/common"
-	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	"github.com/elastic/beats/metricbeat/module/redis"
+	"github.com/elastic/beats/metricbeat/module/redis/mtest"
 
 	rd "github.com/garyburd/redigo/redis"
 	"github.com/stretchr/testify/assert"
@@ -35,70 +34,75 @@ const (
 	password = "foobared"
 )
 
-var redisHost = redis.GetRedisEnvHost() + ":" + redis.GetRedisEnvPort()
-
 func TestFetch(t *testing.T) {
-	compose.EnsureUp(t, "redis")
+	t.Parallel()
 
-	f := mbtest.NewEventFetcher(t, getConfig(""))
-	event, err := f.Fetch()
-	if err != nil {
-		t.Fatal("fetch", err)
-	}
+	mtest.Runner.Run(t, func(t *testing.T, host string) {
+		f := mbtest.NewEventFetcher(t, getConfig("", host))
+		event, err := f.Fetch()
+		if err != nil {
+			t.Fatal("fetch", err)
+		}
 
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+		t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
 
-	// Check fields
-	assert.Equal(t, 9, len(event))
-	server := event["server"].(common.MapStr)
-	assert.Equal(t, "standalone", server["mode"])
+		// Check fields
+		assert.Equal(t, 9, len(event))
+		server := event["server"].(common.MapStr)
+		assert.Equal(t, "standalone", server["mode"])
+	})
 }
 
 func TestData(t *testing.T) {
-	compose.EnsureUp(t, "redis")
+	t.Parallel()
 
-	f := mbtest.NewEventFetcher(t, getConfig(""))
+	// TODO: Fix EnsureUp for this kind of scenarios
+	mtest.DataRunner.Run(t, func(t *testing.T, host string) {
+		f := mbtest.NewEventFetcher(t, getConfig("", host))
 
-	err := mbtest.WriteEvent(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
+		err := mbtest.WriteEvent(f, t)
+		if err != nil {
+			t.Fatal("write", err)
+		}
+	})
 }
 
 func TestPasswords(t *testing.T) {
-	compose.EnsureUp(t, "redis")
-
-	// Add password and ensure it gets reset
-	defer func() {
-		err := resetPassword(redisHost, password)
+	t.Parallel()
+
+	mtest.Runner.Run(t, func(t *testing.T, redisHost string) {
+		// Add password and ensure it gets reset
+		defer func() {
+			err := resetPassword(redisHost, password)
+			if err != nil {
+				t.Fatal("resetting password", err)
+			}
+		}()
+
+		err := addPassword(redisHost, password)
 		if err != nil {
-			t.Fatal("resetting password", err)
+			t.Fatal("adding password", err)
 		}
-	}()
-
-	err := addPassword(redisHost, password)
-	if err != nil {
-		t.Fatal("adding password", err)
-	}
 
-	// Test Fetch metrics with missing password
-	f := mbtest.NewEventFetcher(t, getConfig(""))
-	_, err = f.Fetch()
-	if assert.Error(t, err, "missing password") {
-		assert.Contains(t, err, "NOAUTH Authentication required.")
-	}
+		// Test Fetch metrics with missing password
+		f := mbtest.NewEventFetcher(t, getConfig("", redisHost))
+		_, err = f.Fetch()
+		if assert.Error(t, err, "missing password") {
+			assert.Contains(t, err, "NOAUTH Authentication required.")
+		}
 
-	// Config redis and metricset with an invalid password
-	f = mbtest.NewEventFetcher(t, getConfig("blah"))
-	_, err = f.Fetch()
-	if assert.Error(t, err, "invalid password") {
-		assert.Contains(t, err, "ERR invalid password")
-	}
+		// Config redis and metricset with an invalid password
+		f = mbtest.NewEventFetcher(t, getConfig("blah", redisHost))
+		_, err = f.Fetch()
+		if assert.Error(t, err, "invalid password") {
+			assert.Contains(t, err, "ERR invalid password")
+		}
 
-	// Config redis and metricset with a valid password
-	f = mbtest.NewEventFetcher(t, getConfig(password))
-	_, err = f.Fetch()
-	assert.NoError(t, err, "valid password")
+		// Config redis and metricset with a valid password
+		f = mbtest.NewEventFetcher(t, getConfig(password, redisHost))
+		_, err = f.Fetch()
+		assert.NoError(t, err, "valid password")
+	})
 }
 
 // addPassword will add a password to redis.
@@ -142,7 +146,7 @@ func writeToRedis(host string) error {
 	return err
 }
 
-func getConfig(password string) map[string]interface{} {
+func getConfig(password, redisHost string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "redis",
 		"metricsets": []string{"info"},
diff --git a/metricbeat/module/redis/keyspace/keyspace_integration_test.go b/metricbeat/module/redis/keyspace/keyspace_integration_test.go
index 736db48b064..a9b2d099241 100644
--- a/metricbeat/module/redis/keyspace/keyspace_integration_test.go
+++ b/metricbeat/module/redis/keyspace/keyspace_integration_test.go
@@ -23,33 +23,17 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
+	"github.com/elastic/beats/metricbeat/module/redis/mtest"
 
 	rd "github.com/garyburd/redigo/redis"
 	"github.com/stretchr/testify/assert"
 )
 
-var runner = compose.TestRunner{
-	Service: "redis",
-	Options: map[string][]string{
-		"REDIS_VERSION": []string{
-			"3.2.4",
-			"4.0.11",
-			"5.0-rc",
-		},
-		"IMAGE_OS": []string{
-			"alpine",
-			"stretch",
-		},
-	},
-	Parallel: true,
-}
-
 func TestFetch(t *testing.T) {
 	t.Parallel()
 
-	runner.Run(t, func(t *testing.T, host string) {
+	mtest.Runner.Run(t, func(t *testing.T, host string) {
 		addEntry(t, host)
 
 		// Fetch data
@@ -76,20 +60,8 @@ func TestFetch(t *testing.T) {
 func TestData(t *testing.T) {
 	t.Parallel()
 
-	// TODO: Fix EnsureUp
-	runner := compose.TestRunner{
-		Service: "redis",
-		Options: map[string][]string{
-			"REDIS_VERSION": []string{
-				"4.0.11",
-			},
-			"IMAGE_OS": []string{
-				"alpine",
-			},
-		},
-		Parallel: true,
-	}
-	runner.Run(t, func(t *testing.T, host string) {
+	// TODO: Fix EnsureUp for this kind of scenarios
+	mtest.DataRunner.Run(t, func(t *testing.T, host string) {
 		addEntry(t, host)
 
 		f := mbtest.NewEventsFetcher(t, getConfig(host))
diff --git a/metricbeat/module/redis/mtest/runner.go b/metricbeat/module/redis/mtest/runner.go
new file mode 100644
index 00000000000..20b21e0dc20
--- /dev/null
+++ b/metricbeat/module/redis/mtest/runner.go
@@ -0,0 +1,36 @@
+package mtest
+
+import (
+	"github.com/elastic/beats/libbeat/tests/compose"
+)
+
+var (
+	Runner = compose.TestRunner{
+		Service: "redis",
+		Options: map[string][]string{
+			"REDIS_VERSION": []string{
+				"3.2.4",
+				"4.0.11",
+				"5.0-rc",
+			},
+			"IMAGE_OS": []string{
+				"alpine",
+				"stretch",
+			},
+		},
+		Parallel: true,
+	}
+
+	DataRunner = compose.TestRunner{
+		Service: "redis",
+		Options: map[string][]string{
+			"REDIS_VERSION": []string{
+				"4.0.11",
+			},
+			"IMAGE_OS": []string{
+				"alpine",
+			},
+		},
+		Parallel: true,
+	}
+)
diff --git a/metricbeat/module/redis/testing.go b/metricbeat/module/redis/testing.go
deleted file mode 100644
index 3149a3043b2..00000000000
--- a/metricbeat/module/redis/testing.go
+++ /dev/null
@@ -1,48 +0,0 @@
-// Licensed to Elasticsearch B.V. under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Elasticsearch B.V. licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package redis
-
-import (
-	"os"
-)
-
-// Helper functions for testing used in the redis metricsets
-
-// GetRedisEnvHost returns the hostname of the Redis server to use for testing.
-// It reads the value from the REDIS_HOST environment variable and returns
-// 127.0.0.1 if it is not set.
-func GetRedisEnvHost() string {
-	host := os.Getenv("REDIS_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
-}
-
-// GetRedisEnvPort returns the port of the Redis server to use for testing.
-// It reads the value from the REDIS_PORT environment variable and returns
-// 6379 if it is not set.
-func GetRedisEnvPort() string {
-	port := os.Getenv("REDIS_PORT")
-
-	if len(port) == 0 {
-		port = "6379"
-	}
-	return port
-}

From 0f1a8d6bbb6473efa2d94d47c0dbb5a69deb12bd Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 14 Aug 2018 20:10:18 +0200
Subject: [PATCH 08/88] Run test suites

---
 libbeat/tests/compose/runner.go               |   8 +-
 .../redis/info/info_integration_test.go       | 102 +++++++++---------
 .../keyspace/keyspace_integration_test.go     |  41 +++----
 3 files changed, 77 insertions(+), 74 deletions(-)

diff --git a/libbeat/tests/compose/runner.go b/libbeat/tests/compose/runner.go
index 97cbcac1f79..cb291cbb321 100644
--- a/libbeat/tests/compose/runner.go
+++ b/libbeat/tests/compose/runner.go
@@ -40,6 +40,8 @@ type TestRunner struct {
 	Timeout  int
 }
 
+type Suite map[string]func(t *testing.T, host string)
+
 func (r *TestRunner) scenarios() []map[string]string {
 	n := 1
 	for _, values := range r.Options {
@@ -62,7 +64,7 @@ func (r *TestRunner) scenarios() []map[string]string {
 	return scenarios
 }
 
-func (r *TestRunner) Run(t *testing.T, tests ...func(t *testing.T, host string)) {
+func (r *TestRunner) Run(t *testing.T, tests Suite) {
 	timeout := r.Timeout
 	if timeout == 0 {
 		timeout = 300
@@ -108,8 +110,8 @@ func (r *TestRunner) Run(t *testing.T, tests ...func(t *testing.T, host string))
 				t.Fatal(errors.Wrapf(err, "getting host for %s/%s", r.Service, desc))
 			}
 
-			for _, test := range tests {
-				test(t, host)
+			for name, test := range tests {
+				t.Run(name, func(t *testing.T) { test(t, host) })
 			}
 		})
 
diff --git a/metricbeat/module/redis/info/info_integration_test.go b/metricbeat/module/redis/info/info_integration_test.go
index 1f5ef18796b..356aacc910a 100644
--- a/metricbeat/module/redis/info/info_integration_test.go
+++ b/metricbeat/module/redis/info/info_integration_test.go
@@ -23,6 +23,7 @@ import (
 	"testing"
 
 	"github.com/elastic/beats/libbeat/common"
+	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
 	"github.com/elastic/beats/metricbeat/module/redis/mtest"
 
@@ -34,22 +35,57 @@ const (
 	password = "foobared"
 )
 
-func TestFetch(t *testing.T) {
+func TestInfo(t *testing.T) {
 	t.Parallel()
 
-	mtest.Runner.Run(t, func(t *testing.T, host string) {
-		f := mbtest.NewEventFetcher(t, getConfig("", host))
-		event, err := f.Fetch()
-		if err != nil {
-			t.Fatal("fetch", err)
-		}
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, host string) {
+			f := mbtest.NewEventFetcher(t, getConfig("", host))
+			event, err := f.Fetch()
+			if err != nil {
+				t.Fatal("fetch", err)
+			}
+
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+
+			// Check fields
+			assert.Equal(t, 9, len(event))
+			server := event["server"].(common.MapStr)
+			assert.Equal(t, "standalone", server["mode"])
+		},
+		"Passwords": func(t *testing.T, redisHost string) {
+			// Add password and ensure it gets reset
+			defer func() {
+				err := resetPassword(redisHost, password)
+				if err != nil {
+					t.Fatal("resetting password", err)
+				}
+			}()
+
+			err := addPassword(redisHost, password)
+			if err != nil {
+				t.Fatal("adding password", err)
+			}
+
+			// Test Fetch metrics with missing password
+			f := mbtest.NewEventFetcher(t, getConfig("", redisHost))
+			_, err = f.Fetch()
+			if assert.Error(t, err, "missing password") {
+				assert.Contains(t, err, "NOAUTH Authentication required.")
+			}
 
-		t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+			// Config redis and metricset with an invalid password
+			f = mbtest.NewEventFetcher(t, getConfig("blah", redisHost))
+			_, err = f.Fetch()
+			if assert.Error(t, err, "invalid password") {
+				assert.Contains(t, err, "ERR invalid password")
+			}
 
-		// Check fields
-		assert.Equal(t, 9, len(event))
-		server := event["server"].(common.MapStr)
-		assert.Equal(t, "standalone", server["mode"])
+			// Config redis and metricset with a valid password
+			f = mbtest.NewEventFetcher(t, getConfig(password, redisHost))
+			_, err = f.Fetch()
+			assert.NoError(t, err, "valid password")
+		},
 	})
 }
 
@@ -57,52 +93,14 @@ func TestData(t *testing.T) {
 	t.Parallel()
 
 	// TODO: Fix EnsureUp for this kind of scenarios
-	mtest.DataRunner.Run(t, func(t *testing.T, host string) {
+	mtest.DataRunner.Run(t, compose.Suite{"Data": func(t *testing.T, host string) {
 		f := mbtest.NewEventFetcher(t, getConfig("", host))
 
 		err := mbtest.WriteEvent(f, t)
 		if err != nil {
 			t.Fatal("write", err)
 		}
-	})
-}
-
-func TestPasswords(t *testing.T) {
-	t.Parallel()
-
-	mtest.Runner.Run(t, func(t *testing.T, redisHost string) {
-		// Add password and ensure it gets reset
-		defer func() {
-			err := resetPassword(redisHost, password)
-			if err != nil {
-				t.Fatal("resetting password", err)
-			}
-		}()
-
-		err := addPassword(redisHost, password)
-		if err != nil {
-			t.Fatal("adding password", err)
-		}
-
-		// Test Fetch metrics with missing password
-		f := mbtest.NewEventFetcher(t, getConfig("", redisHost))
-		_, err = f.Fetch()
-		if assert.Error(t, err, "missing password") {
-			assert.Contains(t, err, "NOAUTH Authentication required.")
-		}
-
-		// Config redis and metricset with an invalid password
-		f = mbtest.NewEventFetcher(t, getConfig("blah", redisHost))
-		_, err = f.Fetch()
-		if assert.Error(t, err, "invalid password") {
-			assert.Contains(t, err, "ERR invalid password")
-		}
-
-		// Config redis and metricset with a valid password
-		f = mbtest.NewEventFetcher(t, getConfig(password, redisHost))
-		_, err = f.Fetch()
-		assert.NoError(t, err, "valid password")
-	})
+	}})
 }
 
 // addPassword will add a password to redis.
diff --git a/metricbeat/module/redis/keyspace/keyspace_integration_test.go b/metricbeat/module/redis/keyspace/keyspace_integration_test.go
index a9b2d099241..be4d5cfc417 100644
--- a/metricbeat/module/redis/keyspace/keyspace_integration_test.go
+++ b/metricbeat/module/redis/keyspace/keyspace_integration_test.go
@@ -23,6 +23,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
 	"github.com/elastic/beats/metricbeat/module/redis/mtest"
 
@@ -30,30 +31,32 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func TestFetch(t *testing.T) {
+func TestKeyspace(t *testing.T) {
 	t.Parallel()
 
-	mtest.Runner.Run(t, func(t *testing.T, host string) {
-		addEntry(t, host)
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, host string) {
+			addEntry(t, host)
 
-		// Fetch data
-		f := mbtest.NewEventsFetcher(t, getConfig(host))
-		events, err := f.Fetch()
-		if err != nil {
-			t.Fatal("fetch", err)
-		}
+			// Fetch data
+			f := mbtest.NewEventsFetcher(t, getConfig(host))
+			events, err := f.Fetch()
+			if err != nil {
+				t.Fatal("fetch", err)
+			}
 
-		t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events)
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events)
 
-		// Make sure at least 1 db keyspace exists
-		assert.True(t, len(events) > 0)
+			// Make sure at least 1 db keyspace exists
+			assert.True(t, len(events) > 0)
 
-		keyspace := events[0]
+			keyspace := events[0]
 
-		assert.True(t, keyspace["avg_ttl"].(int64) >= 0)
-		assert.True(t, keyspace["expires"].(int64) >= 0)
-		assert.True(t, keyspace["keys"].(int64) >= 0)
-		assert.True(t, strings.Contains(keyspace["id"].(string), "db"))
+			assert.True(t, keyspace["avg_ttl"].(int64) >= 0)
+			assert.True(t, keyspace["expires"].(int64) >= 0)
+			assert.True(t, keyspace["keys"].(int64) >= 0)
+			assert.True(t, strings.Contains(keyspace["id"].(string), "db"))
+		},
 	})
 }
 
@@ -61,7 +64,7 @@ func TestData(t *testing.T) {
 	t.Parallel()
 
 	// TODO: Fix EnsureUp for this kind of scenarios
-	mtest.DataRunner.Run(t, func(t *testing.T, host string) {
+	mtest.DataRunner.Run(t, compose.Suite{"Data": func(t *testing.T, host string) {
 		addEntry(t, host)
 
 		f := mbtest.NewEventsFetcher(t, getConfig(host))
@@ -70,7 +73,7 @@ func TestData(t *testing.T) {
 		if err != nil {
 			t.Fatal("write", err)
 		}
-	})
+	}})
 }
 
 // addEntry adds an entry to redis

From 89f5f0e94705e392683921f14f2340586037dbec Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Wed, 15 Aug 2018 16:58:06 +0200
Subject: [PATCH 09/88] Allow to override compose runner host and options

Add the possibility to override some options of compose runner using
environment variables:
- If there is a variable with the upercased name of the service + "_HOST"
  (i.e. REDIS_HOST for "redis" service), no scenario is created and the
  value of the variable is injected to the test in the host variable.
- If there is a variable named as one of the options, this single value
  is used instead of the list of values in the options matrix.
---
 libbeat/tests/compose/runner.go | 30 ++++++++++++++++++++++++++++--
 1 file changed, 28 insertions(+), 2 deletions(-)

diff --git a/libbeat/tests/compose/runner.go b/libbeat/tests/compose/runner.go
index cb291cbb321..be8fac9fab0 100644
--- a/libbeat/tests/compose/runner.go
+++ b/libbeat/tests/compose/runner.go
@@ -44,12 +44,19 @@ type Suite map[string]func(t *testing.T, host string)
 
 func (r *TestRunner) scenarios() []map[string]string {
 	n := 1
-	for _, values := range r.Options {
+	options := make(map[string][]string)
+	for env, values := range r.Options {
+		// Allow to ovverride options from environment variables
+		value := os.Getenv(env)
+		if value != "" {
+			values = []string{value}
+		}
+		options[env] = values
 		n *= len(values)
 	}
 
 	scenarios := make([]map[string]string, n)
-	for variable, values := range r.Options {
+	for variable, values := range options {
 		v := 0
 		for i, s := range scenarios {
 			if s == nil {
@@ -64,7 +71,26 @@ func (r *TestRunner) scenarios() []map[string]string {
 	return scenarios
 }
 
+func (r *TestRunner) runHostOverride(t *testing.T, tests Suite) bool {
+	env := strings.ToUpper(r.Service) + "_HOST"
+	host := os.Getenv(env)
+	if host == "" {
+		return false
+	}
+
+	t.Logf("Test host overriden by %s=%s", env, host)
+
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) { test(t, host) })
+	}
+	return true
+}
+
 func (r *TestRunner) Run(t *testing.T, tests Suite) {
+	if r.runHostOverride(t, tests) {
+		return
+	}
+
 	timeout := r.Timeout
 	if timeout == 0 {
 		timeout = 300

From 57e11ea50aad13131a4dcf00e477bd47192f0e85 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Wed, 15 Aug 2018 17:08:59 +0200
Subject: [PATCH 10/88] Typo

---
 libbeat/tests/compose/runner.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libbeat/tests/compose/runner.go b/libbeat/tests/compose/runner.go
index be8fac9fab0..4b8c9912f1d 100644
--- a/libbeat/tests/compose/runner.go
+++ b/libbeat/tests/compose/runner.go
@@ -46,7 +46,7 @@ func (r *TestRunner) scenarios() []map[string]string {
 	n := 1
 	options := make(map[string][]string)
 	for env, values := range r.Options {
-		// Allow to ovverride options from environment variables
+		// Allow to override options from environment variables
 		value := os.Getenv(env)
 		if value != "" {
 			values = []string{value}

From d7fe3436a038981d91ace66f347f535a6546c55c Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Wed, 15 Aug 2018 17:51:25 +0200
Subject: [PATCH 11/88] Use more explicit healthcheck for redis

---
 metricbeat/module/redis/docker-compose.yml | 2 +-
 metricbeat/module/redis/mtest/runner.go    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/metricbeat/module/redis/docker-compose.yml b/metricbeat/module/redis/docker-compose.yml
index 8f1f92bede5..f8e98e46685 100644
--- a/metricbeat/module/redis/docker-compose.yml
+++ b/metricbeat/module/redis/docker-compose.yml
@@ -6,6 +6,6 @@ services:
     healthcheck:
       test:
       - "CMD-SHELL"
-      - "nc -z localhost 6379"
+      - "redis-cli ping"
       interval: 1s
       retries: 90
diff --git a/metricbeat/module/redis/mtest/runner.go b/metricbeat/module/redis/mtest/runner.go
index 20b21e0dc20..343066c8ab0 100644
--- a/metricbeat/module/redis/mtest/runner.go
+++ b/metricbeat/module/redis/mtest/runner.go
@@ -9,7 +9,7 @@ var (
 		Service: "redis",
 		Options: map[string][]string{
 			"REDIS_VERSION": []string{
-				"3.2.4",
+				"3.2.12",
 				"4.0.11",
 				"5.0-rc",
 			},

From d71d2101890f11dd45cd32cf275a47bd89754dc8 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Fri, 17 Aug 2018 18:35:02 +0200
Subject: [PATCH 12/88] Keep a redis service in the main docker-compose file

---
 metricbeat/docker-compose.yml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 8f0205eb756..e8ad49a1527 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -1,4 +1,4 @@
-version: "2"
+version: "2.4"
 
 services:
   beat:
@@ -157,6 +157,11 @@ services:
   rabbitmq:
     build: ./module/rabbitmq/_meta
 
+  redis:
+    extends:
+      file: ./module/redis/docker-compose.yml
+      service: redis
+
   traefik:
     build: ./module/traefik/_meta
 

From 938ba3258f7fa16c7603856066e405ab891a0757 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 21 Aug 2018 08:14:23 +0200
Subject: [PATCH 13/88] Redis test use compose provided hosts

---
 metricbeat/tests/system/test_redis.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/metricbeat/tests/system/test_redis.py b/metricbeat/tests/system/test_redis.py
index 0f2219feecb..94b106ab929 100644
--- a/metricbeat/tests/system/test_redis.py
+++ b/metricbeat/tests/system/test_redis.py
@@ -58,7 +58,10 @@ def test_keyspace(self):
         """
 
         # At least one event must be inserted so db stats exist
-        r = redis.StrictRedis(host=os.getenv('REDIS_HOST', 'localhost'), port=os.getenv('REDIS_PORT', '6379'), db=0)
+        r = redis.StrictRedis(
+            host=os.getenv('REDIS_HOST', self.compose_hosts()[0]),
+            port=os.getenv('REDIS_PORT', '6379'),
+            db=0)
         r.set('foo', 'bar')
 
         self.render_config_template(modules=[{
@@ -116,5 +119,5 @@ def test_module_processors(self):
         self.assertItemsEqual(self.de_dot(CPU_FIELDS), redis_info["cpu"].keys())
 
     def get_hosts(self):
-        return [os.getenv('REDIS_HOST', 'localhost') + ':' +
+        return [os.getenv('REDIS_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('REDIS_PORT', '6379')]

From 4301a8d1ffbbbb291987dd55dc92cc8189295c01 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 21 Aug 2018 08:42:22 +0200
Subject: [PATCH 14/88] Run beat container in host network mode

---
 metricbeat/docker-compose.yml                 | 30 +------------------
 metricbeat/module/aerospike/_meta/env         |  2 --
 metricbeat/module/apache/_meta/env            |  3 --
 metricbeat/module/ceph/_meta/env              |  2 --
 metricbeat/module/couchbase/_meta/env         |  3 --
 metricbeat/module/dropwizard/_meta/env        |  2 --
 metricbeat/module/elasticsearch/_meta/env     |  2 --
 .../elasticsearch/test_elasticsearch.py       |  2 +-
 metricbeat/module/envoyproxy/_meta/env        |  2 --
 metricbeat/module/etcd/_meta/env              |  2 --
 metricbeat/module/haproxy/_meta/env           |  4 ---
 metricbeat/module/http/_meta/env              |  2 --
 metricbeat/module/jolokia/_meta/env           |  2 --
 metricbeat/module/kafka/_meta/env             |  3 --
 metricbeat/module/kibana/_meta/env            |  2 --
 metricbeat/module/kubernetes/_meta/env        |  4 ---
 metricbeat/module/logstash/_meta/env          |  2 --
 metricbeat/module/memcached/_meta/env         |  2 --
 metricbeat/module/mongodb/_meta/env           |  2 --
 metricbeat/module/munin/_meta/env             |  2 --
 metricbeat/module/mysql/_meta/env             |  3 --
 metricbeat/module/nginx/_meta/env             |  2 --
 metricbeat/module/php_fpm/_meta/env           |  2 --
 metricbeat/module/postgresql/_meta/env        |  4 ---
 metricbeat/module/prometheus/_meta/env        |  2 --
 metricbeat/module/rabbitmq/_meta/env          |  4 ---
 metricbeat/module/redis/_meta/env             |  2 --
 metricbeat/module/traefik/_meta/env           |  2 --
 metricbeat/module/traefik/test_traefik.py     |  2 +-
 metricbeat/module/uwsgi/_meta/env             |  2 --
 metricbeat/module/zookeeper/_meta/env         |  2 --
 metricbeat/tests/system/test_aerospike.py     |  2 +-
 metricbeat/tests/system/test_apache.py        |  4 +--
 metricbeat/tests/system/test_ceph.py          |  2 +-
 metricbeat/tests/system/test_config.py        |  2 +-
 metricbeat/tests/system/test_couchbase.py     |  2 +-
 metricbeat/tests/system/test_dropwizard.py    |  2 +-
 metricbeat/tests/system/test_envoyproxy.py    |  2 +-
 metricbeat/tests/system/test_golang.py        |  2 +-
 metricbeat/tests/system/test_http.py          |  3 +-
 metricbeat/tests/system/test_jolokia.py       |  2 +-
 metricbeat/tests/system/test_kibana.py        |  2 +-
 metricbeat/tests/system/test_kubernetes.py    |  4 +--
 metricbeat/tests/system/test_logstash.py      |  2 +-
 metricbeat/tests/system/test_memcached.py     |  2 +-
 metricbeat/tests/system/test_mongodb.py       |  2 +-
 metricbeat/tests/system/test_munin.py         |  2 +-
 metricbeat/tests/system/test_mysql.py         |  4 ++-
 metricbeat/tests/system/test_phpfpm.py        |  2 +-
 metricbeat/tests/system/test_postgresql.py    | 11 +++++--
 metricbeat/tests/system/test_prometheus.py    |  2 +-
 metricbeat/tests/system/test_zookeeper.py     |  2 +-
 52 files changed, 37 insertions(+), 123 deletions(-)
 delete mode 100644 metricbeat/module/aerospike/_meta/env
 delete mode 100644 metricbeat/module/apache/_meta/env
 delete mode 100644 metricbeat/module/ceph/_meta/env
 delete mode 100644 metricbeat/module/couchbase/_meta/env
 delete mode 100644 metricbeat/module/dropwizard/_meta/env
 delete mode 100644 metricbeat/module/elasticsearch/_meta/env
 delete mode 100644 metricbeat/module/envoyproxy/_meta/env
 delete mode 100644 metricbeat/module/etcd/_meta/env
 delete mode 100644 metricbeat/module/haproxy/_meta/env
 delete mode 100644 metricbeat/module/http/_meta/env
 delete mode 100644 metricbeat/module/jolokia/_meta/env
 delete mode 100644 metricbeat/module/kafka/_meta/env
 delete mode 100644 metricbeat/module/kibana/_meta/env
 delete mode 100644 metricbeat/module/kubernetes/_meta/env
 delete mode 100644 metricbeat/module/logstash/_meta/env
 delete mode 100644 metricbeat/module/memcached/_meta/env
 delete mode 100644 metricbeat/module/mongodb/_meta/env
 delete mode 100644 metricbeat/module/munin/_meta/env
 delete mode 100644 metricbeat/module/mysql/_meta/env
 delete mode 100644 metricbeat/module/nginx/_meta/env
 delete mode 100644 metricbeat/module/php_fpm/_meta/env
 delete mode 100644 metricbeat/module/postgresql/_meta/env
 delete mode 100644 metricbeat/module/prometheus/_meta/env
 delete mode 100644 metricbeat/module/rabbitmq/_meta/env
 delete mode 100644 metricbeat/module/redis/_meta/env
 delete mode 100644 metricbeat/module/traefik/_meta/env
 delete mode 100644 metricbeat/module/uwsgi/_meta/env
 delete mode 100644 metricbeat/module/zookeeper/_meta/env

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index e8ad49a1527..30ac09530b7 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -10,36 +10,8 @@ services:
       - ${PWD}/..:/go/src/github.com/elastic/beats/
       # This is required to on-demand launching the rest on containers for tests & also docker module tests:
       - /var/run/docker.sock:/var/run/docker.sock
+    network_mode: host
     command: make
-    env_file:
-      - ./module/aerospike/_meta/env
-      - ./module/apache/_meta/env
-      - ./module/ceph/_meta/env
-      - ./module/couchbase/_meta/env
-      - ./module/dropwizard/_meta/env
-      - ./module/elasticsearch/_meta/env
-      - ./module/envoyproxy/_meta/env
-      - ./module/etcd/_meta/env
-      - ./module/haproxy/_meta/env
-      - ./module/http/_meta/env
-      - ./module/jolokia/_meta/env
-      - ./module/kafka/_meta/env
-      - ./module/kibana/_meta/env
-      - ./module/kubernetes/_meta/env
-      - ./module/logstash/_meta/env
-      - ./module/memcached/_meta/env
-      - ./module/mongodb/_meta/env
-      - ./module/munin/_meta/env
-      - ./module/mysql/_meta/env
-      - ./module/nginx/_meta/env
-      - ./module/php_fpm/_meta/env
-      - ./module/postgresql/_meta/env
-      - ./module/prometheus/_meta/env
-      - ./module/rabbitmq/_meta/env
-      - ./module/redis/_meta/env
-      - ./module/traefik/_meta/env
-      - ./module/uwsgi/_meta/env
-      - ./module/zookeeper/_meta/env
 
   # Modules
   aerospike:
diff --git a/metricbeat/module/aerospike/_meta/env b/metricbeat/module/aerospike/_meta/env
deleted file mode 100644
index 2c95ea957cb..00000000000
--- a/metricbeat/module/aerospike/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-AEROSPIKE_HOST=aerospike
-AEROSPIKE_PORT=3000
diff --git a/metricbeat/module/apache/_meta/env b/metricbeat/module/apache/_meta/env
deleted file mode 100644
index 5339c3407ed..00000000000
--- a/metricbeat/module/apache/_meta/env
+++ /dev/null
@@ -1,3 +0,0 @@
-APACHE_HOST=apache
-APACHE_OLD_HOST=apache_2_4_12
-APACHE_PORT=80
diff --git a/metricbeat/module/ceph/_meta/env b/metricbeat/module/ceph/_meta/env
deleted file mode 100644
index 27b7ce7dbfb..00000000000
--- a/metricbeat/module/ceph/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-CEPH_HOST=ceph
-CEPH_PORT=5000
diff --git a/metricbeat/module/couchbase/_meta/env b/metricbeat/module/couchbase/_meta/env
deleted file mode 100644
index 38aa8e34778..00000000000
--- a/metricbeat/module/couchbase/_meta/env
+++ /dev/null
@@ -1,3 +0,0 @@
-COUCHBASE_HOST=couchbase
-COUCHBASE_PORT=8091
-COUCHBASE_DSN=http://Administrator:password@couchbase:8091
diff --git a/metricbeat/module/dropwizard/_meta/env b/metricbeat/module/dropwizard/_meta/env
deleted file mode 100644
index 771807862fd..00000000000
--- a/metricbeat/module/dropwizard/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-DROPWIZARD_HOST=dropwizard
-DROPWIZARD_PORT=8080
diff --git a/metricbeat/module/elasticsearch/_meta/env b/metricbeat/module/elasticsearch/_meta/env
deleted file mode 100644
index 5856225ceaf..00000000000
--- a/metricbeat/module/elasticsearch/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-ES_HOST=elasticsearch
-ES_PORT=9200
diff --git a/metricbeat/module/elasticsearch/test_elasticsearch.py b/metricbeat/module/elasticsearch/test_elasticsearch.py
index 90906ac641d..49cdb4d4a4e 100644
--- a/metricbeat/module/elasticsearch/test_elasticsearch.py
+++ b/metricbeat/module/elasticsearch/test_elasticsearch.py
@@ -39,7 +39,7 @@ def test_metricsets(self, metricset):
                              ["service.name"], extras={"index_recovery.active_only": "false"})
 
     def get_hosts(self):
-        return [os.getenv('ES_HOST', 'localhost') + ':' +
+        return [os.getenv('ES_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('ES_PORT', '9200')]
 
     def create_ml_job(self):
diff --git a/metricbeat/module/envoyproxy/_meta/env b/metricbeat/module/envoyproxy/_meta/env
deleted file mode 100644
index 75a2386e7f6..00000000000
--- a/metricbeat/module/envoyproxy/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-ENVOYPROXY_HOST=envoyproxy
-ENVOYPROXY_PORT=9901
diff --git a/metricbeat/module/etcd/_meta/env b/metricbeat/module/etcd/_meta/env
deleted file mode 100644
index ea9d5440985..00000000000
--- a/metricbeat/module/etcd/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-ETCD_HOST=etcd
-ETCD_PORT=2379
diff --git a/metricbeat/module/haproxy/_meta/env b/metricbeat/module/haproxy/_meta/env
deleted file mode 100644
index c0762e53c17..00000000000
--- a/metricbeat/module/haproxy/_meta/env
+++ /dev/null
@@ -1,4 +0,0 @@
-HAPROXY_1_6_HOST=haproxy_1_6
-HAPROXY_1_7_HOST=haproxy_1_7
-HAPROXY_HOST=haproxy
-HAPROXY_PORT=14567
diff --git a/metricbeat/module/http/_meta/env b/metricbeat/module/http/_meta/env
deleted file mode 100644
index dfe04431eca..00000000000
--- a/metricbeat/module/http/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-HTTP_HOST=http
-HTTP_PORT=8080
diff --git a/metricbeat/module/jolokia/_meta/env b/metricbeat/module/jolokia/_meta/env
deleted file mode 100644
index 9c0340b6f3c..00000000000
--- a/metricbeat/module/jolokia/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-JOLOKIA_HOST=jolokia
-JOLOKIA_PORT=8778
diff --git a/metricbeat/module/kafka/_meta/env b/metricbeat/module/kafka/_meta/env
deleted file mode 100644
index 227657f404e..00000000000
--- a/metricbeat/module/kafka/_meta/env
+++ /dev/null
@@ -1,3 +0,0 @@
-KAFKA_0_10_2_HOST=kafka_0_10_2
-KAFKA_HOST=kafka
-KAFKA_PORT=9092
diff --git a/metricbeat/module/kibana/_meta/env b/metricbeat/module/kibana/_meta/env
deleted file mode 100644
index a22fc93ec70..00000000000
--- a/metricbeat/module/kibana/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-KIBANA_HOST=kibana
-KIBANA_PORT=5601
\ No newline at end of file
diff --git a/metricbeat/module/kubernetes/_meta/env b/metricbeat/module/kubernetes/_meta/env
deleted file mode 100644
index 03cff050efa..00000000000
--- a/metricbeat/module/kubernetes/_meta/env
+++ /dev/null
@@ -1,4 +0,0 @@
-KUBELET_HOST=172.17.0.1
-KUBELET_PORT=10255
-KUBE_STATE_METRICS_HOST=kubestate
-KUBE_STATE_METRICS_PORT=8080
diff --git a/metricbeat/module/logstash/_meta/env b/metricbeat/module/logstash/_meta/env
deleted file mode 100644
index 57f4dc268f7..00000000000
--- a/metricbeat/module/logstash/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-LOGSTASH_HOST=logstash
-LOGSTASH_PORT=9600
diff --git a/metricbeat/module/memcached/_meta/env b/metricbeat/module/memcached/_meta/env
deleted file mode 100644
index 3efa58cda16..00000000000
--- a/metricbeat/module/memcached/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-MEMCACHED_HOST=memcached
-MEMCACHED_PORT=11211
diff --git a/metricbeat/module/mongodb/_meta/env b/metricbeat/module/mongodb/_meta/env
deleted file mode 100644
index a51807cbe81..00000000000
--- a/metricbeat/module/mongodb/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-MONGODB_HOST=mongodb
-MONGODB_PORT=27017
diff --git a/metricbeat/module/munin/_meta/env b/metricbeat/module/munin/_meta/env
deleted file mode 100644
index b81c5ee8679..00000000000
--- a/metricbeat/module/munin/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-MUNIN_HOST=munin
-MUNIN_PORT=4949
diff --git a/metricbeat/module/mysql/_meta/env b/metricbeat/module/mysql/_meta/env
deleted file mode 100644
index e6aa44eaf4a..00000000000
--- a/metricbeat/module/mysql/_meta/env
+++ /dev/null
@@ -1,3 +0,0 @@
-MYSQL_DSN=root:test@tcp(mysql:3306)/
-MYSQL_HOST=mysql
-MYSQL_PORT=3306
diff --git a/metricbeat/module/nginx/_meta/env b/metricbeat/module/nginx/_meta/env
deleted file mode 100644
index 693e4c78989..00000000000
--- a/metricbeat/module/nginx/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-NGINX_HOST=nginx
-NGINX_PORT=80
diff --git a/metricbeat/module/php_fpm/_meta/env b/metricbeat/module/php_fpm/_meta/env
deleted file mode 100644
index 3437fa5b8c4..00000000000
--- a/metricbeat/module/php_fpm/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-PHPFPM_HOST=phpfpm
-PHPFPM_PORT=81
diff --git a/metricbeat/module/postgresql/_meta/env b/metricbeat/module/postgresql/_meta/env
deleted file mode 100644
index eef1c64138c..00000000000
--- a/metricbeat/module/postgresql/_meta/env
+++ /dev/null
@@ -1,4 +0,0 @@
-POSTGRESQL_DSN=postgres://postgresql:5432?sslmode=disable
-POSTGRESQL_HOST=postgresql
-POSTGRESQL_PORT=5432
-POSTGRESQL_USERNAME=postgres
diff --git a/metricbeat/module/prometheus/_meta/env b/metricbeat/module/prometheus/_meta/env
deleted file mode 100644
index a77775f7c6f..00000000000
--- a/metricbeat/module/prometheus/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-PROMETHEUS_HOST=prometheus
-PROMETHEUS_PORT=9090
diff --git a/metricbeat/module/rabbitmq/_meta/env b/metricbeat/module/rabbitmq/_meta/env
deleted file mode 100644
index a46bb9673d8..00000000000
--- a/metricbeat/module/rabbitmq/_meta/env
+++ /dev/null
@@ -1,4 +0,0 @@
-RABBITMQ_HOST=rabbitmq
-RABBITMQ_PORT=15672
-RABBITMQ_USERNAME=guest
-RABBITMQ_PASSWORD=guest
diff --git a/metricbeat/module/redis/_meta/env b/metricbeat/module/redis/_meta/env
deleted file mode 100644
index 82b68167bc6..00000000000
--- a/metricbeat/module/redis/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-REDIS_HOST=redis
-REDIS_PORT=6379
diff --git a/metricbeat/module/traefik/_meta/env b/metricbeat/module/traefik/_meta/env
deleted file mode 100644
index d8aa826efe6..00000000000
--- a/metricbeat/module/traefik/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-TRAEFIK_HOST=traefik
-TRAEFIK_API_PORT=8080
diff --git a/metricbeat/module/traefik/test_traefik.py b/metricbeat/module/traefik/test_traefik.py
index de8de5f8b84..be42e4d676f 100644
--- a/metricbeat/module/traefik/test_traefik.py
+++ b/metricbeat/module/traefik/test_traefik.py
@@ -25,5 +25,5 @@ def test_health(self, metricset):
         self.check_metricset("traefik", metricset, self.get_hosts(), self.FIELDS + ["service.name"])
 
     def get_hosts(self):
-        return [os.getenv('TRAEFIK_HOST', 'localhost') + ':' +
+        return [os.getenv('TRAEFIK_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('TRAEFIK_API_PORT', '8080')]
diff --git a/metricbeat/module/uwsgi/_meta/env b/metricbeat/module/uwsgi/_meta/env
deleted file mode 100644
index 2f9a17b57d4..00000000000
--- a/metricbeat/module/uwsgi/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-UWSGI_STAT_TCP_SERVER=tcp://uwsgi_tcp:9191
-UWSGI_STAT_HTTP_SERVER=http://uwsgi_http:9192
diff --git a/metricbeat/module/zookeeper/_meta/env b/metricbeat/module/zookeeper/_meta/env
deleted file mode 100644
index 6a063ecec28..00000000000
--- a/metricbeat/module/zookeeper/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-ZOOKEEPER_HOST=zookeeper
-ZOOKEEPER_PORT=2181
diff --git a/metricbeat/tests/system/test_aerospike.py b/metricbeat/tests/system/test_aerospike.py
index 3f957e15885..9d44794603a 100644
--- a/metricbeat/tests/system/test_aerospike.py
+++ b/metricbeat/tests/system/test_aerospike.py
@@ -16,5 +16,5 @@ def test_namespace(self):
         self.check_metricset("aerospike", "namespace", self.get_hosts(), self.FIELDS)
 
     def get_hosts(self):
-        return [os.getenv('AEROSPIKE_HOST', 'localhost') + ':' +
+        return [os.getenv('AEROSPIKE_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('AEROSPIKE_PORT', '3000')]
diff --git a/metricbeat/tests/system/test_apache.py b/metricbeat/tests/system/test_apache.py
index fd164518120..a74f7b98089 100644
--- a/metricbeat/tests/system/test_apache.py
+++ b/metricbeat/tests/system/test_apache.py
@@ -78,7 +78,7 @@ def verify_fields(self, evt):
         # There are more fields that could be checked.
 
     def get_hosts(self):
-        return ['http://' + os.getenv('APACHE_HOST', 'localhost') + ':' +
+        return ['http://' + os.getenv('APACHE_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('APACHE_PORT', '80')]
 
 
@@ -93,5 +93,5 @@ def verify_fields(self, evt):
             self.de_dot(APACHE_OLD_STATUS_FIELDS), apache_status.keys())
 
     def get_hosts(self):
-        return ['http://' + os.getenv('APACHE_OLD_HOST', 'localhost') + ':' +
+        return ['http://' + os.getenv('APACHE_OLD_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('APACHE_PORT', '80')]
diff --git a/metricbeat/tests/system/test_ceph.py b/metricbeat/tests/system/test_ceph.py
index 2e5d91c2e27..fb90acf8a28 100644
--- a/metricbeat/tests/system/test_ceph.py
+++ b/metricbeat/tests/system/test_ceph.py
@@ -24,5 +24,5 @@ def test_ceph(self, metricset):
         self.check_metricset("ceph", metricset, self.get_hosts(), self.FIELDS)
 
     def get_hosts(self):
-        return [os.getenv('CEPH_HOST', 'localhost') + ':' +
+        return [os.getenv('CEPH_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('CEPH_PORT', '5000')]
diff --git a/metricbeat/tests/system/test_config.py b/metricbeat/tests/system/test_config.py
index 9a8ad67cc72..0804e5e0bee 100644
--- a/metricbeat/tests/system/test_config.py
+++ b/metricbeat/tests/system/test_config.py
@@ -27,4 +27,4 @@ def test_invalid_config_with_removed_settings(self):
                                  " has been removed")
 
     def get_host(self):
-        return 'http://' + os.getenv('ES_HOST', 'localhost') + ':' + os.getenv('ES_PORT', '9200')
+        return 'http://' + os.getenv('ES_HOST', self.compose_hosts()[0]) + ':' + os.getenv('ES_PORT', '9200')
diff --git a/metricbeat/tests/system/test_couchbase.py b/metricbeat/tests/system/test_couchbase.py
index f0d75780359..49d11b3f21b 100644
--- a/metricbeat/tests/system/test_couchbase.py
+++ b/metricbeat/tests/system/test_couchbase.py
@@ -22,5 +22,5 @@ def test_couchbase(self, metricset):
         self.check_metricset("couchbase", metricset, self.get_hosts(), self.FIELDS)
 
     def get_hosts(self):
-        return [os.getenv('COUCHBASE_HOST', 'localhost') + ':' +
+        return [os.getenv('COUCHBASE_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('COUCHBASE_PORT', '8091')]
diff --git a/metricbeat/tests/system/test_dropwizard.py b/metricbeat/tests/system/test_dropwizard.py
index a9ab7dbbe8d..651fb4968ec 100644
--- a/metricbeat/tests/system/test_dropwizard.py
+++ b/metricbeat/tests/system/test_dropwizard.py
@@ -30,5 +30,5 @@ def test_dropwizard(self):
         self.assertTrue(len(output) >= 1)
 
     def get_hosts(self):
-        return [os.getenv('DROPWIZARD_HOST', 'localhost') + ':' +
+        return [os.getenv('DROPWIZARD_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('DROPWIZARD_PORT', '8080')]
diff --git a/metricbeat/tests/system/test_envoyproxy.py b/metricbeat/tests/system/test_envoyproxy.py
index 40b7bce1a13..b30bcac1e20 100644
--- a/metricbeat/tests/system/test_envoyproxy.py
+++ b/metricbeat/tests/system/test_envoyproxy.py
@@ -31,5 +31,5 @@ def test_stats(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return [os.getenv('ENVOYPROXY_HOST', 'localhost') + ':' +
+        return [os.getenv('ENVOYPROXY_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('ENVOYPROXY_PORT', '9901')]
diff --git a/metricbeat/tests/system/test_golang.py b/metricbeat/tests/system/test_golang.py
index 19ddf8c9db4..a053a2403fa 100644
--- a/metricbeat/tests/system/test_golang.py
+++ b/metricbeat/tests/system/test_golang.py
@@ -43,5 +43,5 @@ def test_stats(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return ["http://" + os.getenv('GOLANG_HOST', 'localhost') + ':' +
+        return ["http://" + os.getenv('GOLANG_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('GOLANG_PORT', '6060')]
diff --git a/metricbeat/tests/system/test_http.py b/metricbeat/tests/system/test_http.py
index d25eadf6d56..64e6239be92 100644
--- a/metricbeat/tests/system/test_http.py
+++ b/metricbeat/tests/system/test_http.py
@@ -76,4 +76,5 @@ def test_server(self):
         self.assert_fields_are_documented(evt)
 
     def get_host(self):
-        return "http://" + os.getenv('HTTP_HOST', 'localhost') + ':' + os.getenv('HTTP_PORT', '8080')
+        host = self.compose_hosts()[0]
+        return "http://" + os.getenv('HTTP_HOST', host) + ':' + os.getenv('HTTP_PORT', '8080')
diff --git a/metricbeat/tests/system/test_jolokia.py b/metricbeat/tests/system/test_jolokia.py
index 921ee5aef25..3ca67699956 100644
--- a/metricbeat/tests/system/test_jolokia.py
+++ b/metricbeat/tests/system/test_jolokia.py
@@ -50,5 +50,5 @@ def test_jmx(self, mbean):
         assert evt["jolokia"]["test"]["gc"]["collection_count"] >= 0
 
     def get_hosts(self):
-        return [os.getenv('JOLOKIA_HOST', 'localhost') + ':' +
+        return [os.getenv('JOLOKIA_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('JOLOKIA_PORT', '8778')]
diff --git a/metricbeat/tests/system/test_kibana.py b/metricbeat/tests/system/test_kibana.py
index fbb7a5736d2..f3143819884 100644
--- a/metricbeat/tests/system/test_kibana.py
+++ b/metricbeat/tests/system/test_kibana.py
@@ -48,7 +48,7 @@ def test_status(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return [os.getenv('KIBANA_HOST', 'localhost') + ':' +
+        return [os.getenv('KIBANA_HOST', self.compose_hosts()[1]) + ':' +
                 os.getenv('KIBANA_PORT', '5601')]
 
     def get_version(self):
diff --git a/metricbeat/tests/system/test_kubernetes.py b/metricbeat/tests/system/test_kubernetes.py
index 75f94dbd331..1901774a265 100644
--- a/metricbeat/tests/system/test_kubernetes.py
+++ b/metricbeat/tests/system/test_kubernetes.py
@@ -78,7 +78,7 @@ def _test_metricset(self, metricset, expected_events, hosts):
     def get_kubelet_hosts(cls):
         return [
             "http://" +
-            os.getenv('KUBELET_HOST', 'localhost') + ':' +
+            os.getenv('KUBELET_HOST', self.compose_hosts()[0]) + ':' +
             os.getenv('KUBELET_PORT', '10255')
         ]
 
@@ -86,6 +86,6 @@ def get_kubelet_hosts(cls):
     def get_kube_state_hosts(cls):
         return [
             "http://" +
-            os.getenv('KUBE_STATE_METRICS_HOST', 'localhost') + ':' +
+            os.getenv('KUBE_STATE_METRICS_HOST', self.compose_hosts()[1]) + ':' +
             os.getenv('KUBE_STATE_METRICS_PORT', '18080')
         ]
diff --git a/metricbeat/tests/system/test_logstash.py b/metricbeat/tests/system/test_logstash.py
index 8280bfb0223..816aaf11cd7 100644
--- a/metricbeat/tests/system/test_logstash.py
+++ b/metricbeat/tests/system/test_logstash.py
@@ -55,5 +55,5 @@ def test_node_stats(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return [os.getenv('LOGSTASH_HOST', 'localhost') + ':' +
+        return [os.getenv('LOGSTASH_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('LOGSTASH_PORT', '9600')]
diff --git a/metricbeat/tests/system/test_memcached.py b/metricbeat/tests/system/test_memcached.py
index ede980b0581..c41d34b7cc6 100644
--- a/metricbeat/tests/system/test_memcached.py
+++ b/metricbeat/tests/system/test_memcached.py
@@ -31,5 +31,5 @@ def test_stats(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return [os.getenv('MEMCACHED_HOST', 'localhost') + ':' +
+        return [os.getenv('MEMCACHED_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('MEMCACHED_PORT', '11211')]
diff --git a/metricbeat/tests/system/test_mongodb.py b/metricbeat/tests/system/test_mongodb.py
index 3a8493584a4..88327e80a23 100644
--- a/metricbeat/tests/system/test_mongodb.py
+++ b/metricbeat/tests/system/test_mongodb.py
@@ -36,5 +36,5 @@ def test_status(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return [os.getenv('MONGODB_HOST', 'localhost') + ':' +
+        return [os.getenv('MONGODB_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('MONGODB_PORT', '27017')]
diff --git a/metricbeat/tests/system/test_munin.py b/metricbeat/tests/system/test_munin.py
index e1618202fa4..391d8f845d5 100644
--- a/metricbeat/tests/system/test_munin.py
+++ b/metricbeat/tests/system/test_munin.py
@@ -34,5 +34,5 @@ def test_munin_node(self):
         assert evt["munin"][namespace]["cpu"]["user"] > 0
 
     def get_hosts(self):
-        return [os.getenv('MUNIN_HOST', 'localhost') + ':' +
+        return [os.getenv('MUNIN_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('MUNIN_PORT', '4949')]
diff --git a/metricbeat/tests/system/test_mysql.py b/metricbeat/tests/system/test_mysql.py
index 987da6eb88a..4087b141e49 100644
--- a/metricbeat/tests/system/test_mysql.py
+++ b/metricbeat/tests/system/test_mysql.py
@@ -40,4 +40,6 @@ def test_status(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return [os.getenv('MYSQL_DSN', 'root:test@tcp(localhost:3306)/')]
+        host = os.getenv('MYSQL_HOST', self.compose_hosts()[0])
+        port = os.getenv('MYSQL_PORT', '3306')
+        return [os.getenv('MYSQL_DSN', 'root:test@tcp({}:{})/'.format(host, port))]
diff --git a/metricbeat/tests/system/test_phpfpm.py b/metricbeat/tests/system/test_phpfpm.py
index edb5405a404..4049f96748a 100644
--- a/metricbeat/tests/system/test_phpfpm.py
+++ b/metricbeat/tests/system/test_phpfpm.py
@@ -34,5 +34,5 @@ def test_info(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return [os.getenv('PHPFPM_HOST', 'localhost') + ':' +
+        return [os.getenv('PHPFPM_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('PHPFPM_PORT', '81')]
diff --git a/metricbeat/tests/system/test_postgresql.py b/metricbeat/tests/system/test_postgresql.py
index 5150d9ab28b..6f2e1193745 100644
--- a/metricbeat/tests/system/test_postgresql.py
+++ b/metricbeat/tests/system/test_postgresql.py
@@ -19,8 +19,15 @@ def common_checks(self, output):
             self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return [os.getenv("POSTGRESQL_DSN")], os.getenv("POSTGRESQL_USERNAME"), \
-            os.getenv("POSTGRESQL_PASSWORD")
+        username = "postgres"
+        host = self.compose_hosts()[0]
+        port = 5432
+        dsn = "postgres://{}:{}?sslmode=disable".format(host, port)
+        return (
+            [os.getenv("POSTGRESQL_DSN", dsn)],
+            os.getenv("POSTGRESQL_USERNAME", username),
+            os.getenv("POSTGRESQL_PASSWORD"),
+        )
 
     @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test")
     @attr('integration')
diff --git a/metricbeat/tests/system/test_prometheus.py b/metricbeat/tests/system/test_prometheus.py
index 535395460df..ccf6bbf220f 100644
--- a/metricbeat/tests/system/test_prometheus.py
+++ b/metricbeat/tests/system/test_prometheus.py
@@ -34,5 +34,5 @@ def test_stats(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return ["http://" + os.getenv('PROMETHEUS_HOST', 'localhost') + ':' +
+        return ["http://" + os.getenv('PROMETHEUS_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('PROMETHEUS_PORT', '9090')]
diff --git a/metricbeat/tests/system/test_zookeeper.py b/metricbeat/tests/system/test_zookeeper.py
index 7871b2d1797..6d35f9d4d4c 100644
--- a/metricbeat/tests/system/test_zookeeper.py
+++ b/metricbeat/tests/system/test_zookeeper.py
@@ -51,5 +51,5 @@ def test_output(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return [os.getenv('ZOOKEEPER_HOST', 'localhost') + ':' +
+        return [os.getenv('ZOOKEEPER_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('ZOOKEEPER_PORT', '2181')]

From a5722798a16237d5d42e3496e05b5350390cbfd1 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 21 Aug 2018 08:56:17 +0200
Subject: [PATCH 15/88] Run redis python tests with two versions

---
 metricbeat/docker-compose.yml         | 6 ++++++
 metricbeat/tests/system/test_redis.py | 4 ++++
 2 files changed, 10 insertions(+)

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 30ac09530b7..8f6bf83d34c 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -134,6 +134,12 @@ services:
       file: ./module/redis/docker-compose.yml
       service: redis
 
+  redis_3_2_12:
+    extends:
+      file: ./module/redis/docker-compose.yml
+      service: redis
+    image: redis:3.2.12-alpine
+
   traefik:
     build: ./module/traefik/_meta
 
diff --git a/metricbeat/tests/system/test_redis.py b/metricbeat/tests/system/test_redis.py
index 94b106ab929..8fb96e4c8bd 100644
--- a/metricbeat/tests/system/test_redis.py
+++ b/metricbeat/tests/system/test_redis.py
@@ -121,3 +121,7 @@ def test_module_processors(self):
     def get_hosts(self):
         return [os.getenv('REDIS_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('REDIS_PORT', '6379')]
+
+
+class Test_3_2_12(Test):
+    COMPOSE_SERVICES = ['redis_3_2_12']

From 8dd0a98df538d6f017b19bc09283dd1962d68f05 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 21 Aug 2018 10:54:53 +0200
Subject: [PATCH 16/88] Add compose runner for kafka

---
 libbeat/tests/compose/wrapper.go              |   5 +-
 metricbeat/docker-compose.yml                 |  15 +-
 metricbeat/module/kafka/docker-compose.yml    |   8 +
 metricbeat/module/kafka/mtest/runner.go       |  48 ++++++
 .../partition/partition_integration_test.go   | 146 ++++++++----------
 .../redis/info/info_integration_test.go       |  11 +-
 .../keyspace/keyspace_integration_test.go     |   5 +-
 metricbeat/module/redis/mtest/runner.go       |  19 +++
 8 files changed, 159 insertions(+), 98 deletions(-)
 create mode 100644 metricbeat/module/kafka/docker-compose.yml
 create mode 100644 metricbeat/module/kafka/mtest/runner.go

diff --git a/libbeat/tests/compose/wrapper.go b/libbeat/tests/compose/wrapper.go
index 2e67ef0f785..a548a424de2 100644
--- a/libbeat/tests/compose/wrapper.go
+++ b/libbeat/tests/compose/wrapper.go
@@ -20,10 +20,8 @@ package compose
 import (
 	"context"
 	"fmt"
-	"net"
 	"os"
 	"os/exec"
-	"strconv"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -77,11 +75,14 @@ func (c *wrapperContainer) Host() string {
 	if len(ip) == 0 {
 		return ""
 	}
+	return ip
 
+	/* TODO:
 	for _, port := range c.info.Ports {
 		return net.JoinHostPort(ip, strconv.Itoa(int(port.PrivatePort)))
 	}
 	return ""
+	*/
 }
 
 func (d *wrapperDriver) LockFile() string {
diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 6a7230835f0..1fcb07d29c3 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -69,20 +69,23 @@ services:
     build: ./module/jolokia/_meta
 
   kafka:
-    build:
-      context: ./module/kafka/_meta
-      args:
-        KAFKA_VERSION: 2.0.0
+    extends:
+      file: ./module/kafka/docker-compose.yml
+      service: kafka
 
   kafka_1_1_0:
+    extends:
+      file: ./module/kafka/docker-compose.yml
+      service: kafka
     build:
-      context: ./module/kafka/_meta
       args:
         KAFKA_VERSION: 1.1.0
 
   kafka_0_10_2:
+    extends:
+      file: ./module/kafka/docker-compose.yml
+      service: kafka
     build:
-      context: ./module/kafka/_meta
       args:
         KAFKA_VERSION: 0.10.2.1
 
diff --git a/metricbeat/module/kafka/docker-compose.yml b/metricbeat/module/kafka/docker-compose.yml
new file mode 100644
index 00000000000..14041d028ea
--- /dev/null
+++ b/metricbeat/module/kafka/docker-compose.yml
@@ -0,0 +1,8 @@
+version: "2.4"
+
+services:
+  kafka:
+    build:
+      context: ./_meta
+      args:
+        KAFKA_VERSION: 2.0.0
diff --git a/metricbeat/module/kafka/mtest/runner.go b/metricbeat/module/kafka/mtest/runner.go
new file mode 100644
index 00000000000..5aba5e2e4bf
--- /dev/null
+++ b/metricbeat/module/kafka/mtest/runner.go
@@ -0,0 +1,48 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// +build integration
+
+package mtest
+
+import (
+	"github.com/elastic/beats/libbeat/tests/compose"
+)
+
+var (
+	Runner = compose.TestRunner{
+		Service: "kafka",
+		Options: map[string][]string{
+			"KAFKA_VERSION": []string{
+				"0.10.2.1",
+				"1.1.0",
+				"2.0.0",
+			},
+		},
+		Parallel: true,
+	}
+
+	DataRunner = compose.TestRunner{
+		Service: "kafka",
+		Options: map[string][]string{
+			"KAFKA_VERSION": []string{
+				"1.1.0",
+			},
+		},
+		Parallel: true,
+	}
+)
diff --git a/metricbeat/module/kafka/partition/partition_integration_test.go b/metricbeat/module/kafka/partition/partition_integration_test.go
index dc696c26b1d..22b631c74ff 100644
--- a/metricbeat/module/kafka/partition/partition_integration_test.go
+++ b/metricbeat/module/kafka/partition/partition_integration_test.go
@@ -22,7 +22,7 @@ package partition
 import (
 	"fmt"
 	"math/rand"
-	"os"
+	"net"
 	"strconv"
 	"testing"
 	"time"
@@ -34,91 +34,89 @@ import (
 	"github.com/elastic/beats/libbeat/logp"
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-)
-
-const (
-	kafkaDefaultHost = "localhost"
-	kafkaDefaultPort = "9092"
+	"github.com/elastic/beats/metricbeat/module/kafka/mtest"
 )
 
 func TestData(t *testing.T) {
-	compose.EnsureUp(t, "kafka")
+	t.Parallel()
 
-	generateKafkaData(t, "metricbeat-generate-data")
+	mtest.DataRunner.Run(t, compose.Suite{"Data": func(t *testing.T, host string) {
+		generateKafkaData(t, "metricbeat-generate-data", host)
 
-	ms := mbtest.NewReportingMetricSetV2(t, getConfig(""))
-	err := mbtest.WriteEventsReporterV2(ms, t, "")
-	if err != nil {
-		t.Fatal("write", err)
-	}
+		ms := mbtest.NewReportingMetricSetV2(t, getConfig("", host))
+		err := mbtest.WriteEventsReporterV2(ms, t, "")
+		if err != nil {
+			t.Fatal("write", err)
+		}
+	}})
 }
 
 func TestTopic(t *testing.T) {
-
-	compose.EnsureUp(t, "kafka")
+	t.Parallel()
 
 	logp.TestingSetup(logp.WithSelectors("kafka"))
 
-	id := strconv.Itoa(rand.New(rand.NewSource(int64(time.Now().Nanosecond()))).Int())
-	testTopic := fmt.Sprintf("test-metricbeat-%s", id)
+	mtest.DataRunner.Run(t, compose.Suite{"Topic": func(t *testing.T, host string) {
+		id := strconv.Itoa(rand.New(rand.NewSource(int64(time.Now().Nanosecond()))).Int())
+		testTopic := fmt.Sprintf("test-metricbeat-%s", id)
 
-	// Create initial topic
-	generateKafkaData(t, testTopic)
+		// Create initial topic
+		generateKafkaData(t, testTopic, host)
 
-	f := mbtest.NewReportingMetricSetV2(t, getConfig(testTopic))
-	dataBefore, err := mbtest.ReportingFetchV2(f)
-	if err != nil {
-		t.Fatal("write", err)
-	}
-	if len(dataBefore) == 0 {
-		t.Errorf("No offsets fetched from topic (before): %v", testTopic)
-	}
-	t.Logf("before: %v", dataBefore)
+		f := mbtest.NewReportingMetricSetV2(t, getConfig(testTopic, host))
+		dataBefore, err := mbtest.ReportingFetchV2(f)
+		if err != nil {
+			t.Fatal("write", err)
+		}
+		if len(dataBefore) == 0 {
+			t.Errorf("No offsets fetched from topic (before): %v", testTopic)
+		}
+		t.Logf("before: %v", dataBefore)
 
-	var n int64 = 10
-	var i int64 = 0
-	// Create n messages
-	for ; i < n; i++ {
-		generateKafkaData(t, testTopic)
-	}
+		var n int64 = 10
+		// Create n messages
+		for i := int64(0); i < n; i++ {
+			generateKafkaData(t, testTopic, host)
+		}
 
-	dataAfter, err := mbtest.ReportingFetchV2(f)
-	if err != nil {
-		t.Fatal("write", err)
-	}
-	if len(dataAfter) == 0 {
-		t.Errorf("No offsets fetched from topic (after): %v", testTopic)
-	}
-	t.Logf("after: %v", dataAfter)
+		dataAfter, err := mbtest.ReportingFetchV2(f)
+		if err != nil {
+			t.Fatal("write", err)
+		}
+		if len(dataAfter) == 0 {
+			t.Errorf("No offsets fetched from topic (after): %v", testTopic)
+		}
+		t.Logf("after: %v", dataAfter)
 
-	// Checks that no new topics / partitions were added
-	assert.True(t, len(dataBefore) == len(dataAfter))
+		// Checks that no new topics / partitions were added
+		assert.True(t, len(dataBefore) == len(dataAfter))
 
-	var offsetBefore int64 = 0
-	var offsetAfter int64 = 0
+		var offsetBefore int64 = 0
+		var offsetAfter int64 = 0
 
-	// Its possible that other topics exists -> select the right data
-	for _, data := range dataBefore {
-		if data.ModuleFields["topic"].(common.MapStr)["name"] == testTopic {
-			offsetBefore = data.MetricSetFields["offset"].(common.MapStr)["newest"].(int64)
+		// Its possible that other topics exists -> select the right data
+		for _, data := range dataBefore {
+			if data.ModuleFields["topic"].(common.MapStr)["name"] == testTopic {
+				offsetBefore = data.MetricSetFields["offset"].(common.MapStr)["newest"].(int64)
+			}
 		}
-	}
 
-	for _, data := range dataAfter {
-		if data.ModuleFields["topic"].(common.MapStr)["name"] == testTopic {
-			offsetAfter = data.MetricSetFields["offset"].(common.MapStr)["newest"].(int64)
+		for _, data := range dataAfter {
+			if data.ModuleFields["topic"].(common.MapStr)["name"] == testTopic {
+				offsetAfter = data.MetricSetFields["offset"].(common.MapStr)["newest"].(int64)
+			}
 		}
-	}
 
-	// Compares offset before and after
-	if offsetBefore+n != offsetAfter {
-		t.Errorf("Offset before: %v", offsetBefore)
-		t.Errorf("Offset after: %v", offsetAfter)
-	}
-	assert.True(t, offsetBefore+n == offsetAfter)
+		// Compares offset before and after
+		if offsetBefore+n != offsetAfter {
+			t.Errorf("Offset before: %v", offsetBefore)
+			t.Errorf("Offset after: %v", offsetAfter)
+		}
+		assert.True(t, offsetBefore+n == offsetAfter)
+	}})
 }
 
-func generateKafkaData(t *testing.T, topic string) {
+func generateKafkaData(t *testing.T, topic string, host string) {
 	t.Logf("Send Kafka Event to topic: %v", topic)
 
 	config := sarama.NewConfig()
@@ -128,7 +126,7 @@ func generateKafkaData(t *testing.T, topic string) {
 	config.Producer.Retry.Backoff = 500 * time.Millisecond
 	config.Metadata.Retry.Max = 20
 	config.Metadata.Retry.Backoff = 500 * time.Millisecond
-	client, err := sarama.NewClient([]string{getTestKafkaHost()}, config)
+	client, err := sarama.NewClient([]string{net.JoinHostPort(host, "9092")}, config)
 	if err != nil {
 		t.Errorf("%s", err)
 		t.FailNow()
@@ -156,7 +154,7 @@ func generateKafkaData(t *testing.T, topic string) {
 	}
 }
 
-func getConfig(topic string) map[string]interface{} {
+func getConfig(topic string, host string) map[string]interface{} {
 	var topics []string
 	if topic != "" {
 		topics = []string{topic}
@@ -165,25 +163,7 @@ func getConfig(topic string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "kafka",
 		"metricsets": []string{"partition"},
-		"hosts":      []string{getTestKafkaHost()},
+		"hosts":      []string{net.JoinHostPort(host, "9092")},
 		"topics":     topics,
 	}
 }
-
-func getTestKafkaHost() string {
-	return fmt.Sprintf("%v:%v",
-		getenv("KAFKA_HOST", kafkaDefaultHost),
-		getenv("KAFKA_PORT", kafkaDefaultPort),
-	)
-}
-
-func getenv(name, defaultValue string) string {
-	return strDefault(os.Getenv(name), defaultValue)
-}
-
-func strDefault(a, defaults string) string {
-	if len(a) == 0 {
-		return defaults
-	}
-	return a
-}
diff --git a/metricbeat/module/redis/info/info_integration_test.go b/metricbeat/module/redis/info/info_integration_test.go
index 356aacc910a..5152067911f 100644
--- a/metricbeat/module/redis/info/info_integration_test.go
+++ b/metricbeat/module/redis/info/info_integration_test.go
@@ -20,6 +20,7 @@
 package info
 
 import (
+	"net"
 	"testing"
 
 	"github.com/elastic/beats/libbeat/common"
@@ -105,7 +106,7 @@ func TestData(t *testing.T) {
 
 // addPassword will add a password to redis.
 func addPassword(host, pass string) error {
-	c, err := rd.Dial("tcp", host)
+	c, err := rd.Dial("tcp", net.JoinHostPort(host, "6379"))
 	if err != nil {
 		return err
 	}
@@ -117,7 +118,7 @@ func addPassword(host, pass string) error {
 
 // resetPassword changes the password to the redis DB.
 func resetPassword(host, currentPass string) error {
-	c, err := rd.Dial("tcp", host)
+	c, err := rd.Dial("tcp", net.JoinHostPort(host, "6379"))
 	if err != nil {
 		return err
 	}
@@ -134,7 +135,7 @@ func resetPassword(host, currentPass string) error {
 
 // writeToRedis will write to the default DB 0.
 func writeToRedis(host string) error {
-	c, err := rd.Dial("tcp", host)
+	c, err := rd.Dial("tcp", net.JoinHostPort(host, "6379"))
 	if err != nil {
 		return err
 	}
@@ -144,11 +145,11 @@ func writeToRedis(host string) error {
 	return err
 }
 
-func getConfig(password, redisHost string) map[string]interface{} {
+func getConfig(password, host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "redis",
 		"metricsets": []string{"info"},
-		"hosts":      []string{redisHost},
+		"hosts":      []string{net.JoinHostPort(host, "6379")},
 		"password":   password,
 	}
 }
diff --git a/metricbeat/module/redis/keyspace/keyspace_integration_test.go b/metricbeat/module/redis/keyspace/keyspace_integration_test.go
index be4d5cfc417..14a38d7210e 100644
--- a/metricbeat/module/redis/keyspace/keyspace_integration_test.go
+++ b/metricbeat/module/redis/keyspace/keyspace_integration_test.go
@@ -20,6 +20,7 @@
 package keyspace
 
 import (
+	"net"
 	"strings"
 	"testing"
 
@@ -79,7 +80,7 @@ func TestData(t *testing.T) {
 // addEntry adds an entry to redis
 func addEntry(t *testing.T, host string) {
 	// Insert at least one event to make sure db exists
-	c, err := rd.Dial("tcp", host)
+	c, err := rd.Dial("tcp", net.JoinHostPort(host, "6379"))
 	if err != nil {
 		t.Fatal("connect", err)
 	}
@@ -94,6 +95,6 @@ func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "redis",
 		"metricsets": []string{"keyspace"},
-		"hosts":      []string{host},
+		"hosts":      []string{net.JoinHostPort(host, "6379")},
 	}
 }
diff --git a/metricbeat/module/redis/mtest/runner.go b/metricbeat/module/redis/mtest/runner.go
index 343066c8ab0..19001b8e493 100644
--- a/metricbeat/module/redis/mtest/runner.go
+++ b/metricbeat/module/redis/mtest/runner.go
@@ -1,3 +1,22 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// +build integration
+
 package mtest
 
 import (

From 38cc0819bb1bacd4717e082650e232aa1528833a Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 21 Aug 2018 11:18:38 +0200
Subject: [PATCH 17/88] Use docker compose 2.1 version

---
 metricbeat/docker-compose.yml              | 2 +-
 metricbeat/module/kafka/docker-compose.yml | 2 +-
 metricbeat/module/redis/docker-compose.yml | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 1fcb07d29c3..721a8f93318 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -1,4 +1,4 @@
-version: "2.4"
+version: "2.1"
 
 services:
   beat:
diff --git a/metricbeat/module/kafka/docker-compose.yml b/metricbeat/module/kafka/docker-compose.yml
index 14041d028ea..9a6fdade8fb 100644
--- a/metricbeat/module/kafka/docker-compose.yml
+++ b/metricbeat/module/kafka/docker-compose.yml
@@ -1,4 +1,4 @@
-version: "2.4"
+version: "2.1"
 
 services:
   kafka:
diff --git a/metricbeat/module/redis/docker-compose.yml b/metricbeat/module/redis/docker-compose.yml
index f8e98e46685..9fac97dce56 100644
--- a/metricbeat/module/redis/docker-compose.yml
+++ b/metricbeat/module/redis/docker-compose.yml
@@ -1,4 +1,4 @@
-version: "2.4"
+version: "2.1"
 
 services:
   redis:

From b3981f71426747032c671373c0c013639a0d8a79 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 21 Aug 2018 15:30:06 +0200
Subject: [PATCH 18/88] Use kafka runner in consumergroup metricset

---
 .../consumergroup_integration_test.go         | 65 ++++++-------------
 .../partition/partition_integration_test.go   |  2 +-
 2 files changed, 22 insertions(+), 45 deletions(-)

diff --git a/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go b/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go
index df67af8da07..91bdeeb22e0 100644
--- a/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go
+++ b/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go
@@ -20,9 +20,8 @@
 package consumergroup
 
 import (
-	"fmt"
 	"io"
-	"os"
+	"net"
 	"testing"
 	"time"
 
@@ -31,61 +30,39 @@ import (
 
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-)
-
-const (
-	kafkaDefaultHost = "localhost"
-	kafkaDefaultPort = "9092"
+	"github.com/elastic/beats/metricbeat/module/kafka/mtest"
 )
 
 func TestData(t *testing.T) {
-	compose.EnsureUp(t, "kafka")
-
-	c, err := startConsumer(t, "metricbeat-test")
-	if err != nil {
-		t.Fatal(errors.Wrap(err, "starting kafka consumer"))
-	}
-	defer c.Close()
+	mtest.DataRunner.Run(t, compose.Suite{"Data": func(t *testing.T, host string) {
+		c, err := startConsumer(t, "metricbeat-test", host)
+		if err != nil {
+			t.Fatal(errors.Wrap(err, "starting kafka consumer"))
+		}
+		defer c.Close()
 
-	ms := mbtest.NewReportingMetricSetV2(t, getConfig())
-	for retries := 0; retries < 3; retries++ {
-		err = mbtest.WriteEventsReporterV2(ms, t, "")
-		if err == nil {
-			return
+		ms := mbtest.NewReportingMetricSetV2(t, getConfig(host))
+		for retries := 0; retries < 3; retries++ {
+			err = mbtest.WriteEventsReporterV2(ms, t, "")
+			if err == nil {
+				return
+			}
+			time.Sleep(500 * time.Millisecond)
 		}
-		time.Sleep(500 * time.Millisecond)
-	}
-	t.Fatal("write", err)
+		t.Fatal("write", err)
+	}})
 }
 
-func startConsumer(t *testing.T, topic string) (io.Closer, error) {
-	brokers := []string{getTestKafkaHost()}
+func startConsumer(t *testing.T, topic, host string) (io.Closer, error) {
+	brokers := []string{net.JoinHostPort(host, "9092")}
 	topics := []string{topic}
 	return saramacluster.NewConsumer(brokers, "test-group", topics, nil)
 }
 
-func getConfig() map[string]interface{} {
+func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "kafka",
 		"metricsets": []string{"consumergroup"},
-		"hosts":      []string{getTestKafkaHost()},
-	}
-}
-
-func getTestKafkaHost() string {
-	return fmt.Sprintf("%v:%v",
-		getenv("KAFKA_HOST", kafkaDefaultHost),
-		getenv("KAFKA_PORT", kafkaDefaultPort),
-	)
-}
-
-func getenv(name, defaultValue string) string {
-	return strDefault(os.Getenv(name), defaultValue)
-}
-
-func strDefault(a, defaults string) string {
-	if len(a) == 0 {
-		return defaults
+		"hosts":      []string{net.JoinHostPort(host, "9092")},
 	}
-	return a
 }
diff --git a/metricbeat/module/kafka/partition/partition_integration_test.go b/metricbeat/module/kafka/partition/partition_integration_test.go
index 22b631c74ff..cb19fa048d4 100644
--- a/metricbeat/module/kafka/partition/partition_integration_test.go
+++ b/metricbeat/module/kafka/partition/partition_integration_test.go
@@ -56,7 +56,7 @@ func TestTopic(t *testing.T) {
 
 	logp.TestingSetup(logp.WithSelectors("kafka"))
 
-	mtest.DataRunner.Run(t, compose.Suite{"Topic": func(t *testing.T, host string) {
+	mtest.Runner.Run(t, compose.Suite{"Topic": func(t *testing.T, host string) {
 		id := strconv.Itoa(rand.New(rand.NewSource(int64(time.Now().Nanosecond()))).Int())
 		testTopic := fmt.Sprintf("test-metricbeat-%s", id)
 

From b6735ea0f958f5eae835e922bda0e2e8f11b67b0 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Thu, 23 Aug 2018 12:27:23 +0200
Subject: [PATCH 19/88] Retry on docker compose up to avoid failures by system
 limits

---
 libbeat/tests/compose/wrapper.go | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/libbeat/tests/compose/wrapper.go b/libbeat/tests/compose/wrapper.go
index a548a424de2..84911d3a429 100644
--- a/libbeat/tests/compose/wrapper.go
+++ b/libbeat/tests/compose/wrapper.go
@@ -23,6 +23,7 @@ import (
 	"os"
 	"os/exec"
 	"strings"
+	"time"
 
 	"github.com/pkg/errors"
 
@@ -131,7 +132,19 @@ func (d *wrapperDriver) Up(ctx context.Context, opts UpOptions, service string)
 		args = append(args, service)
 	}
 
-	return d.cmd(ctx, "up", args...).Run()
+	for {
+		// It can fail if we have reached some system limit, specially
+		// number of networks, retry while the context is not done
+		err := d.cmd(ctx, "up", args...).Run()
+		if err == nil {
+			return nil
+		}
+		select {
+		case <-time.After(time.Second):
+		case <-ctx.Done():
+			return err
+		}
+	}
 }
 
 func (d *wrapperDriver) Down(ctx context.Context) error {

From f9f0cd1078d754a8c85532b273d96f2b022ab0c0 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Thu, 23 Aug 2018 19:06:29 +0200
Subject: [PATCH 20/88] Use compose runner for aerospike

---
 metricbeat/docker-compose.yml                 |  4 +-
 metricbeat/module/aerospike/_meta/Dockerfile  |  5 +-
 .../module/aerospike/docker-compose.yml       | 14 ++++++
 .../aerospike/{testing.go => mtest/runner.go} | 49 ++++++++++---------
 .../namespace/namespace_integration_test.go   | 21 ++++----
 5 files changed, 57 insertions(+), 36 deletions(-)
 create mode 100644 metricbeat/module/aerospike/docker-compose.yml
 rename metricbeat/module/aerospike/{testing.go => mtest/runner.go} (52%)

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 721a8f93318..3bef2cb9020 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -15,7 +15,9 @@ services:
 
   # Modules
   aerospike:
-    build: ./module/aerospike/_meta
+    extends:
+      file: ./module/aerospike/docker-compose.yml
+      service: aerospike
 
   apache:
     build: ./module/apache/_meta
diff --git a/metricbeat/module/aerospike/_meta/Dockerfile b/metricbeat/module/aerospike/_meta/Dockerfile
index 9693bbde4c2..e5d1c37bbd2 100644
--- a/metricbeat/module/aerospike/_meta/Dockerfile
+++ b/metricbeat/module/aerospike/_meta/Dockerfile
@@ -1,4 +1,5 @@
-FROM aerospike:3.9.0
+ARG AEROSPIKE_VERSION=3.9.0
+
+FROM aerospike:$AEROSPIKE_VERSION
 
 RUN apt-get update && apt-get install -y netcat
-HEALTHCHECK --interval=1s --retries=90 CMD nc -z localhost 3000
diff --git a/metricbeat/module/aerospike/docker-compose.yml b/metricbeat/module/aerospike/docker-compose.yml
new file mode 100644
index 00000000000..6ba1a786ef4
--- /dev/null
+++ b/metricbeat/module/aerospike/docker-compose.yml
@@ -0,0 +1,14 @@
+version: '2.1'
+
+services:
+  aerospike:
+    build:
+      context: ./_meta
+      args:
+        AEROSPIKE_VERSION: ${AEROSPIKE_VERSION:-3.9.0}
+    healthcheck:
+      test:
+      - "CMD-SHELL"
+      - "nc -z localhost 3000"
+      interval: 1s
+      retries: 90
diff --git a/metricbeat/module/aerospike/testing.go b/metricbeat/module/aerospike/mtest/runner.go
similarity index 52%
rename from metricbeat/module/aerospike/testing.go
rename to metricbeat/module/aerospike/mtest/runner.go
index 298634e85c0..ddb50d85c53 100644
--- a/metricbeat/module/aerospike/testing.go
+++ b/metricbeat/module/aerospike/mtest/runner.go
@@ -15,34 +15,35 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package aerospike
+// +build integration
+
+package mtest
 
 import (
-	"os"
+	"github.com/elastic/beats/libbeat/tests/compose"
 )
 
-// Helper functions for testing used in the aerospike MetricSets.
-
-// GetAerospikeEnvHost returns the hostname of the Aerospike server to use for
-// testing. It reads the value from the AEROSPIKE_HOST environment variable and
-// returns localhost if it is not set.
-func GetAerospikeEnvHost() string {
-	host := os.Getenv("AEROSPIKE_HOST")
-
-	if len(host) == 0 {
-		host = "localhost"
+var (
+	Runner = compose.TestRunner{
+		Service: "aerospike",
+		Options: map[string][]string{
+			"AEROSPIKE_VERSION": []string{
+				"3.9.0",
+				"3.13.0.11",
+				"3.16.0.6",
+				"4.3.0.2",
+			},
+		},
+		Parallel: true,
 	}
-	return host
-}
 
-// GetAerospikeEnvPort returns the port of the Aerospike server to use for
-// testing. It reads the value from the AEROSPIKE_PORT environment variable and
-// returns 3000 if it is not set.
-func GetAerospikeEnvPort() string {
-	port := os.Getenv("AEROSPIKE_PORT")
-
-	if len(port) == 0 {
-		port = "3000"
+	DataRunner = compose.TestRunner{
+		Service: "aerospike",
+		Options: map[string][]string{
+			"AEROSPIKE_VERSION": []string{
+				"3.9.0",
+			},
+		},
+		Parallel: true,
 	}
-	return port
-}
+)
diff --git a/metricbeat/module/aerospike/namespace/namespace_integration_test.go b/metricbeat/module/aerospike/namespace/namespace_integration_test.go
index 03fd7ed4397..29f37afdb72 100644
--- a/metricbeat/module/aerospike/namespace/namespace_integration_test.go
+++ b/metricbeat/module/aerospike/namespace/namespace_integration_test.go
@@ -20,27 +20,30 @@
 package namespace
 
 import (
+	"net"
 	"testing"
 
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	"github.com/elastic/beats/metricbeat/module/aerospike"
+	"github.com/elastic/beats/metricbeat/module/aerospike/mtest"
 )
 
 func TestData(t *testing.T) {
-	compose.EnsureUp(t, "aerospike")
+	t.Parallel()
 
-	f := mbtest.NewEventsFetcher(t, getConfig())
-	err := mbtest.WriteEvents(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
+	mtest.DataRunner.Run(t, compose.Suite{"Data": func(t *testing.T, host string) {
+		f := mbtest.NewEventsFetcher(t, getConfig(host))
+		err := mbtest.WriteEvents(f, t)
+		if err != nil {
+			t.Fatal("write", err)
+		}
+	}})
 }
 
-func getConfig() map[string]interface{} {
+func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "aerospike",
 		"metricsets": []string{"namespace"},
-		"hosts":      []string{aerospike.GetAerospikeEnvHost() + ":" + aerospike.GetAerospikeEnvPort()},
+		"hosts":      []string{net.JoinHostPort(host, "3000")},
 	}
 }

From a7382e6537bb2f95aca6bcbfc35358cb8ed69158 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Thu, 23 Aug 2018 20:31:01 +0200
Subject: [PATCH 21/88] Inject a runner controller instead of just the host

---
 libbeat/tests/compose/runner.go               |  35 ++++-
 metricbeat/module/aerospike/mtest/runner.go   |  16 +--
 .../namespace/namespace_integration_test.go   |   6 +-
 .../consumergroup_integration_test.go         |   8 +-
 metricbeat/module/kafka/mtest/runner.go       |  10 --
 .../partition/partition_integration_test.go   | 132 +++++++++---------
 .../redis/info/info_integration_test.go       |  38 +++--
 .../keyspace/keyspace_integration_test.go     |  32 ++---
 metricbeat/module/redis/mtest/runner.go       |  13 --
 9 files changed, 133 insertions(+), 157 deletions(-)

diff --git a/libbeat/tests/compose/runner.go b/libbeat/tests/compose/runner.go
index 4b8c9912f1d..977d9857eed 100644
--- a/libbeat/tests/compose/runner.go
+++ b/libbeat/tests/compose/runner.go
@@ -40,7 +40,7 @@ type TestRunner struct {
 	Timeout  int
 }
 
-type Suite map[string]func(t *testing.T, host string)
+type Suite map[string]func(t *testing.T, r R)
 
 func (r *TestRunner) scenarios() []map[string]string {
 	n := 1
@@ -71,6 +71,12 @@ func (r *TestRunner) scenarios() []map[string]string {
 	return scenarios
 }
 
+func (r *TestRunner) runSuite(t *testing.T, tests Suite, ctl R) {
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) { test(t, ctl) })
+	}
+}
+
 func (r *TestRunner) runHostOverride(t *testing.T, tests Suite) bool {
 	env := strings.ToUpper(r.Service) + "_HOST"
 	host := os.Getenv(env)
@@ -80,9 +86,11 @@ func (r *TestRunner) runHostOverride(t *testing.T, tests Suite) bool {
 
 	t.Logf("Test host overriden by %s=%s", env, host)
 
-	for name, test := range tests {
-		t.Run(name, func(t *testing.T) { test(t, host) })
+	ctl := &runnerControl{
+		host: host,
+		t:    t,
 	}
+	r.runSuite(t, tests, ctl)
 	return true
 }
 
@@ -136,10 +144,27 @@ func (r *TestRunner) Run(t *testing.T, tests Suite) {
 				t.Fatal(errors.Wrapf(err, "getting host for %s/%s", r.Service, desc))
 			}
 
-			for name, test := range tests {
-				t.Run(name, func(t *testing.T) { test(t, host) })
+			ctl := &runnerControl{
+				host:     host,
+				t:        t,
+				scenario: s,
 			}
+			r.runSuite(t, tests, ctl)
 		})
 
 	}
 }
+
+type R interface {
+	Host() string
+}
+
+type runnerControl struct {
+	t        *testing.T
+	host     string
+	scenario map[string]string
+}
+
+func (r *runnerControl) Host() string {
+	return r.host
+}
diff --git a/metricbeat/module/aerospike/mtest/runner.go b/metricbeat/module/aerospike/mtest/runner.go
index ddb50d85c53..24af7a2ab7e 100644
--- a/metricbeat/module/aerospike/mtest/runner.go
+++ b/metricbeat/module/aerospike/mtest/runner.go
@@ -29,19 +29,9 @@ var (
 		Options: map[string][]string{
 			"AEROSPIKE_VERSION": []string{
 				"3.9.0",
-				"3.13.0.11",
-				"3.16.0.6",
-				"4.3.0.2",
-			},
-		},
-		Parallel: true,
-	}
-
-	DataRunner = compose.TestRunner{
-		Service: "aerospike",
-		Options: map[string][]string{
-			"AEROSPIKE_VERSION": []string{
-				"3.9.0",
+				//"3.13.0.11",
+				//"3.16.0.6",
+				//"4.3.0.2",
 			},
 		},
 		Parallel: true,
diff --git a/metricbeat/module/aerospike/namespace/namespace_integration_test.go b/metricbeat/module/aerospike/namespace/namespace_integration_test.go
index 29f37afdb72..9baaebacd8d 100644
--- a/metricbeat/module/aerospike/namespace/namespace_integration_test.go
+++ b/metricbeat/module/aerospike/namespace/namespace_integration_test.go
@@ -28,11 +28,11 @@ import (
 	"github.com/elastic/beats/metricbeat/module/aerospike/mtest"
 )
 
-func TestData(t *testing.T) {
+func TestNamespace(t *testing.T) {
 	t.Parallel()
 
-	mtest.DataRunner.Run(t, compose.Suite{"Data": func(t *testing.T, host string) {
-		f := mbtest.NewEventsFetcher(t, getConfig(host))
+	mtest.Runner.Run(t, compose.Suite{"Data": func(t *testing.T, r compose.R) {
+		f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
 		err := mbtest.WriteEvents(f, t)
 		if err != nil {
 			t.Fatal("write", err)
diff --git a/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go b/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go
index 91bdeeb22e0..4f40288ba51 100644
--- a/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go
+++ b/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go
@@ -33,15 +33,15 @@ import (
 	"github.com/elastic/beats/metricbeat/module/kafka/mtest"
 )
 
-func TestData(t *testing.T) {
-	mtest.DataRunner.Run(t, compose.Suite{"Data": func(t *testing.T, host string) {
-		c, err := startConsumer(t, "metricbeat-test", host)
+func TestConsumerGroup(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{"Data": func(t *testing.T, r compose.R) {
+		c, err := startConsumer(t, "metricbeat-test", r.Host())
 		if err != nil {
 			t.Fatal(errors.Wrap(err, "starting kafka consumer"))
 		}
 		defer c.Close()
 
-		ms := mbtest.NewReportingMetricSetV2(t, getConfig(host))
+		ms := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host()))
 		for retries := 0; retries < 3; retries++ {
 			err = mbtest.WriteEventsReporterV2(ms, t, "")
 			if err == nil {
diff --git a/metricbeat/module/kafka/mtest/runner.go b/metricbeat/module/kafka/mtest/runner.go
index 5aba5e2e4bf..e4d93295a39 100644
--- a/metricbeat/module/kafka/mtest/runner.go
+++ b/metricbeat/module/kafka/mtest/runner.go
@@ -35,14 +35,4 @@ var (
 		},
 		Parallel: true,
 	}
-
-	DataRunner = compose.TestRunner{
-		Service: "kafka",
-		Options: map[string][]string{
-			"KAFKA_VERSION": []string{
-				"1.1.0",
-			},
-		},
-		Parallel: true,
-	}
 )
diff --git a/metricbeat/module/kafka/partition/partition_integration_test.go b/metricbeat/module/kafka/partition/partition_integration_test.go
index cb19fa048d4..ac51227bb7f 100644
--- a/metricbeat/module/kafka/partition/partition_integration_test.go
+++ b/metricbeat/module/kafka/partition/partition_integration_test.go
@@ -37,83 +37,79 @@ import (
 	"github.com/elastic/beats/metricbeat/module/kafka/mtest"
 )
 
-func TestData(t *testing.T) {
+func TestPartition(t *testing.T) {
 	t.Parallel()
 
-	mtest.DataRunner.Run(t, compose.Suite{"Data": func(t *testing.T, host string) {
-		generateKafkaData(t, "metricbeat-generate-data", host)
+	logp.TestingSetup(logp.WithSelectors("kafka"))
 
-		ms := mbtest.NewReportingMetricSetV2(t, getConfig("", host))
-		err := mbtest.WriteEventsReporterV2(ms, t, "")
-		if err != nil {
-			t.Fatal("write", err)
-		}
-	}})
-}
+	mtest.Runner.Run(t, compose.Suite{
+		"Data": func(t *testing.T, r compose.R) {
+			generateKafkaData(t, "metricbeat-generate-data", r.Host())
 
-func TestTopic(t *testing.T) {
-	t.Parallel()
+			ms := mbtest.NewReportingMetricSetV2(t, getConfig("", r.Host()))
+			err := mbtest.WriteEventsReporterV2(ms, t, "")
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+		"Topic": func(t *testing.T, r compose.R) {
+			id := strconv.Itoa(rand.New(rand.NewSource(int64(time.Now().Nanosecond()))).Int())
+			testTopic := fmt.Sprintf("test-metricbeat-%s", id)
+
+			// Create initial topic
+			generateKafkaData(t, testTopic, r.Host())
+
+			f := mbtest.NewReportingMetricSetV2(t, getConfig(testTopic, r.Host()))
+			dataBefore, err := mbtest.ReportingFetchV2(f)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+			if len(dataBefore) == 0 {
+				t.Errorf("No offsets fetched from topic (before): %v", testTopic)
+			}
+			t.Logf("before: %v", dataBefore)
 
-	logp.TestingSetup(logp.WithSelectors("kafka"))
+			var n int64 = 10
+			// Create n messages
+			for i := int64(0); i < n; i++ {
+				generateKafkaData(t, testTopic, r.Host())
+			}
+
+			dataAfter, err := mbtest.ReportingFetchV2(f)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+			if len(dataAfter) == 0 {
+				t.Errorf("No offsets fetched from topic (after): %v", testTopic)
+			}
+			t.Logf("after: %v", dataAfter)
+
+			// Checks that no new topics / partitions were added
+			assert.True(t, len(dataBefore) == len(dataAfter))
+
+			var offsetBefore int64 = 0
+			var offsetAfter int64 = 0
+
+			// Its possible that other topics exists -> select the right data
+			for _, data := range dataBefore {
+				if data.ModuleFields["topic"].(common.MapStr)["name"] == testTopic {
+					offsetBefore = data.MetricSetFields["offset"].(common.MapStr)["newest"].(int64)
+				}
+			}
 
-	mtest.Runner.Run(t, compose.Suite{"Topic": func(t *testing.T, host string) {
-		id := strconv.Itoa(rand.New(rand.NewSource(int64(time.Now().Nanosecond()))).Int())
-		testTopic := fmt.Sprintf("test-metricbeat-%s", id)
-
-		// Create initial topic
-		generateKafkaData(t, testTopic, host)
-
-		f := mbtest.NewReportingMetricSetV2(t, getConfig(testTopic, host))
-		dataBefore, err := mbtest.ReportingFetchV2(f)
-		if err != nil {
-			t.Fatal("write", err)
-		}
-		if len(dataBefore) == 0 {
-			t.Errorf("No offsets fetched from topic (before): %v", testTopic)
-		}
-		t.Logf("before: %v", dataBefore)
-
-		var n int64 = 10
-		// Create n messages
-		for i := int64(0); i < n; i++ {
-			generateKafkaData(t, testTopic, host)
-		}
-
-		dataAfter, err := mbtest.ReportingFetchV2(f)
-		if err != nil {
-			t.Fatal("write", err)
-		}
-		if len(dataAfter) == 0 {
-			t.Errorf("No offsets fetched from topic (after): %v", testTopic)
-		}
-		t.Logf("after: %v", dataAfter)
-
-		// Checks that no new topics / partitions were added
-		assert.True(t, len(dataBefore) == len(dataAfter))
-
-		var offsetBefore int64 = 0
-		var offsetAfter int64 = 0
-
-		// Its possible that other topics exists -> select the right data
-		for _, data := range dataBefore {
-			if data.ModuleFields["topic"].(common.MapStr)["name"] == testTopic {
-				offsetBefore = data.MetricSetFields["offset"].(common.MapStr)["newest"].(int64)
+			for _, data := range dataAfter {
+				if data.ModuleFields["topic"].(common.MapStr)["name"] == testTopic {
+					offsetAfter = data.MetricSetFields["offset"].(common.MapStr)["newest"].(int64)
+				}
 			}
-		}
 
-		for _, data := range dataAfter {
-			if data.ModuleFields["topic"].(common.MapStr)["name"] == testTopic {
-				offsetAfter = data.MetricSetFields["offset"].(common.MapStr)["newest"].(int64)
+			// Compares offset before and after
+			if offsetBefore+n != offsetAfter {
+				t.Errorf("Offset before: %v", offsetBefore)
+				t.Errorf("Offset after: %v", offsetAfter)
 			}
-		}
-
-		// Compares offset before and after
-		if offsetBefore+n != offsetAfter {
-			t.Errorf("Offset before: %v", offsetBefore)
-			t.Errorf("Offset after: %v", offsetAfter)
-		}
-		assert.True(t, offsetBefore+n == offsetAfter)
-	}})
+			assert.True(t, offsetBefore+n == offsetAfter)
+		}})
 }
 
 func generateKafkaData(t *testing.T, topic string, host string) {
diff --git a/metricbeat/module/redis/info/info_integration_test.go b/metricbeat/module/redis/info/info_integration_test.go
index 5152067911f..1c4f946a7b4 100644
--- a/metricbeat/module/redis/info/info_integration_test.go
+++ b/metricbeat/module/redis/info/info_integration_test.go
@@ -40,8 +40,16 @@ func TestInfo(t *testing.T) {
 	t.Parallel()
 
 	mtest.Runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, host string) {
-			f := mbtest.NewEventFetcher(t, getConfig("", host))
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, getConfig("", r.Host()))
+
+			err := mbtest.WriteEvent(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, getConfig("", r.Host()))
 			event, err := f.Fetch()
 			if err != nil {
 				t.Fatal("fetch", err)
@@ -54,56 +62,42 @@ func TestInfo(t *testing.T) {
 			server := event["server"].(common.MapStr)
 			assert.Equal(t, "standalone", server["mode"])
 		},
-		"Passwords": func(t *testing.T, redisHost string) {
+		"Passwords": func(t *testing.T, r compose.R) {
 			// Add password and ensure it gets reset
 			defer func() {
-				err := resetPassword(redisHost, password)
+				err := resetPassword(r.Host(), password)
 				if err != nil {
 					t.Fatal("resetting password", err)
 				}
 			}()
 
-			err := addPassword(redisHost, password)
+			err := addPassword(r.Host(), password)
 			if err != nil {
 				t.Fatal("adding password", err)
 			}
 
 			// Test Fetch metrics with missing password
-			f := mbtest.NewEventFetcher(t, getConfig("", redisHost))
+			f := mbtest.NewEventFetcher(t, getConfig("", r.Host()))
 			_, err = f.Fetch()
 			if assert.Error(t, err, "missing password") {
 				assert.Contains(t, err, "NOAUTH Authentication required.")
 			}
 
 			// Config redis and metricset with an invalid password
-			f = mbtest.NewEventFetcher(t, getConfig("blah", redisHost))
+			f = mbtest.NewEventFetcher(t, getConfig("blah", r.Host()))
 			_, err = f.Fetch()
 			if assert.Error(t, err, "invalid password") {
 				assert.Contains(t, err, "ERR invalid password")
 			}
 
 			// Config redis and metricset with a valid password
-			f = mbtest.NewEventFetcher(t, getConfig(password, redisHost))
+			f = mbtest.NewEventFetcher(t, getConfig(password, r.Host()))
 			_, err = f.Fetch()
 			assert.NoError(t, err, "valid password")
 		},
 	})
 }
 
-func TestData(t *testing.T) {
-	t.Parallel()
-
-	// TODO: Fix EnsureUp for this kind of scenarios
-	mtest.DataRunner.Run(t, compose.Suite{"Data": func(t *testing.T, host string) {
-		f := mbtest.NewEventFetcher(t, getConfig("", host))
-
-		err := mbtest.WriteEvent(f, t)
-		if err != nil {
-			t.Fatal("write", err)
-		}
-	}})
-}
-
 // addPassword will add a password to redis.
 func addPassword(host, pass string) error {
 	c, err := rd.Dial("tcp", net.JoinHostPort(host, "6379"))
diff --git a/metricbeat/module/redis/keyspace/keyspace_integration_test.go b/metricbeat/module/redis/keyspace/keyspace_integration_test.go
index 14a38d7210e..1d47088eaa7 100644
--- a/metricbeat/module/redis/keyspace/keyspace_integration_test.go
+++ b/metricbeat/module/redis/keyspace/keyspace_integration_test.go
@@ -36,11 +36,21 @@ func TestKeyspace(t *testing.T) {
 	t.Parallel()
 
 	mtest.Runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, host string) {
-			addEntry(t, host)
+		"Data": func(t *testing.T, r compose.R) {
+			addEntry(t, r.Host())
+
+			f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
+
+			err := mbtest.WriteEvents(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+		"Fetch": func(t *testing.T, r compose.R) {
+			addEntry(t, r.Host())
 
 			// Fetch data
-			f := mbtest.NewEventsFetcher(t, getConfig(host))
+			f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
 			events, err := f.Fetch()
 			if err != nil {
 				t.Fatal("fetch", err)
@@ -61,22 +71,6 @@ func TestKeyspace(t *testing.T) {
 	})
 }
 
-func TestData(t *testing.T) {
-	t.Parallel()
-
-	// TODO: Fix EnsureUp for this kind of scenarios
-	mtest.DataRunner.Run(t, compose.Suite{"Data": func(t *testing.T, host string) {
-		addEntry(t, host)
-
-		f := mbtest.NewEventsFetcher(t, getConfig(host))
-
-		err := mbtest.WriteEvents(f, t)
-		if err != nil {
-			t.Fatal("write", err)
-		}
-	}})
-}
-
 // addEntry adds an entry to redis
 func addEntry(t *testing.T, host string) {
 	// Insert at least one event to make sure db exists
diff --git a/metricbeat/module/redis/mtest/runner.go b/metricbeat/module/redis/mtest/runner.go
index 19001b8e493..49e0353365c 100644
--- a/metricbeat/module/redis/mtest/runner.go
+++ b/metricbeat/module/redis/mtest/runner.go
@@ -39,17 +39,4 @@ var (
 		},
 		Parallel: true,
 	}
-
-	DataRunner = compose.TestRunner{
-		Service: "redis",
-		Options: map[string][]string{
-			"REDIS_VERSION": []string{
-				"4.0.11",
-			},
-			"IMAGE_OS": []string{
-				"alpine",
-			},
-		},
-		Parallel: true,
-	}
 )

From 760cdc3cc68f92eeca559f2007627489525051fa Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Thu, 23 Aug 2018 20:40:14 +0200
Subject: [PATCH 22/88] Remove local images on down

---
 libbeat/tests/compose/wrapper.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libbeat/tests/compose/wrapper.go b/libbeat/tests/compose/wrapper.go
index 84911d3a429..b5026448d35 100644
--- a/libbeat/tests/compose/wrapper.go
+++ b/libbeat/tests/compose/wrapper.go
@@ -148,7 +148,7 @@ func (d *wrapperDriver) Up(ctx context.Context, opts UpOptions, service string)
 }
 
 func (d *wrapperDriver) Down(ctx context.Context) error {
-	return d.cmd(ctx, "down", "-v").Run()
+	return d.cmd(ctx, "down", "-v", "--rmi=local").Run()
 }
 
 func (d *wrapperDriver) Kill(ctx context.Context, signal string, service string) error {

From 9b9f5be15eee917ba78bbcc2816d9597c512616c Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Thu, 23 Aug 2018 20:46:14 +0200
Subject: [PATCH 23/88] Add more versions to aerospike

---
 metricbeat/module/aerospike/mtest/runner.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/metricbeat/module/aerospike/mtest/runner.go b/metricbeat/module/aerospike/mtest/runner.go
index 24af7a2ab7e..29a0a6a69e0 100644
--- a/metricbeat/module/aerospike/mtest/runner.go
+++ b/metricbeat/module/aerospike/mtest/runner.go
@@ -29,9 +29,9 @@ var (
 		Options: map[string][]string{
 			"AEROSPIKE_VERSION": []string{
 				"3.9.0",
-				//"3.13.0.11",
-				//"3.16.0.6",
-				//"4.3.0.2",
+				"3.13.0.11",
+				"3.16.0.6",
+				"4.3.0.2",
 			},
 		},
 		Parallel: true,

From f45c67e3dc2c88430dbc7bb7a4c42dd652ad3aa3 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Thu, 23 Aug 2018 21:19:34 +0200
Subject: [PATCH 24/88] Use compose runner for apache module

---
 metricbeat/docker-compose.yml                 | 12 +++--
 metricbeat/module/apache/_meta/Dockerfile     |  3 +-
 .../module/apache/_meta/Dockerfile.2.4.12     |  4 --
 metricbeat/module/apache/docker-compose.yml   |  8 +++
 .../apache/{testing.go => mtest/runner.go}    | 29 ++++++-----
 .../module/apache/status/_meta/data.json      | 39 +++++++-------
 .../apache/status/status_integration_test.go  | 51 ++++++++++---------
 7 files changed, 79 insertions(+), 67 deletions(-)
 delete mode 100644 metricbeat/module/apache/_meta/Dockerfile.2.4.12
 create mode 100644 metricbeat/module/apache/docker-compose.yml
 rename metricbeat/module/apache/{testing.go => mtest/runner.go} (67%)

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 3bef2cb9020..b656c6163ed 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -20,12 +20,18 @@ services:
       service: aerospike
 
   apache:
-    build: ./module/apache/_meta
+    extends:
+      file: ./module/apache/docker-compose.yml
+      service: apache
 
   apache_2_4_12:
+    extends:
+      file: ./module/apache/docker-compose.yml
+      service: apache
     build:
-      context: ./module/apache/_meta
-      dockerfile: Dockerfile.2.4.12
+      args:
+        APACHE_VERSION: 2.4.12
+
 
   ceph:
     build: ./module/ceph/_meta
diff --git a/metricbeat/module/apache/_meta/Dockerfile b/metricbeat/module/apache/_meta/Dockerfile
index b6239b871f9..9b5b6ddcc36 100644
--- a/metricbeat/module/apache/_meta/Dockerfile
+++ b/metricbeat/module/apache/_meta/Dockerfile
@@ -1,4 +1,5 @@
-FROM httpd:2.4.20
+ARG APACHE_VERSION
+FROM httpd:$APACHE_VERSION
 RUN apt-get update && apt-get install -y curl
 HEALTHCHECK --interval=1s --retries=90 CMD curl -f http://localhost
 COPY ./httpd.conf /usr/local/apache2/conf/httpd.conf
diff --git a/metricbeat/module/apache/_meta/Dockerfile.2.4.12 b/metricbeat/module/apache/_meta/Dockerfile.2.4.12
deleted file mode 100644
index f35ea2d95c0..00000000000
--- a/metricbeat/module/apache/_meta/Dockerfile.2.4.12
+++ /dev/null
@@ -1,4 +0,0 @@
-FROM httpd:2.4.12
-RUN apt-get update && apt-get install -y curl
-HEALTHCHECK --interval=1s --retries=90 CMD curl -f http://localhost
-COPY ./httpd.conf /usr/local/apache2/conf/httpd.conf
diff --git a/metricbeat/module/apache/docker-compose.yml b/metricbeat/module/apache/docker-compose.yml
new file mode 100644
index 00000000000..c7338df1666
--- /dev/null
+++ b/metricbeat/module/apache/docker-compose.yml
@@ -0,0 +1,8 @@
+version: '2.1'
+
+services:
+  apache:
+    build:
+      context: ./_meta
+      args:
+        APACHE_VERSION: ${APACHE_VERSION:-2.4.20}
diff --git a/metricbeat/module/apache/testing.go b/metricbeat/module/apache/mtest/runner.go
similarity index 67%
rename from metricbeat/module/apache/testing.go
rename to metricbeat/module/apache/mtest/runner.go
index 4b2e0571a35..b069377ee91 100644
--- a/metricbeat/module/apache/testing.go
+++ b/metricbeat/module/apache/mtest/runner.go
@@ -15,22 +15,23 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package apache
+// +build integration
+
+package mtest
 
 import (
-	"os"
+	"github.com/elastic/beats/libbeat/tests/compose"
 )
 
-// Helper functions for testing the Apache module's MetricSets.
-
-// GetApacheEnvHost returns the apache server hostname to use for testing. It
-// reads the value from the APACHE_HOST environment variable and returns
-// 127.0.0.1 if it is not set.
-func GetApacheEnvHost() string {
-	host := os.Getenv("APACHE_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
+var (
+	Runner = compose.TestRunner{
+		Service: "apache",
+		Options: map[string][]string{
+			"APACHE_VERSION": []string{
+				"2.4.12",
+				"2.4.20",
+			},
+		},
+		Parallel: true,
 	}
-	return host
-}
+)
diff --git a/metricbeat/module/apache/status/_meta/data.json b/metricbeat/module/apache/status/_meta/data.json
index e8e3d45845f..171b0c54f95 100644
--- a/metricbeat/module/apache/status/_meta/data.json
+++ b/metricbeat/module/apache/status/_meta/data.json
@@ -2,30 +2,29 @@
     "@timestamp": "2017-10-12T08:05:34.853Z",
     "apache": {
         "status": {
-            "bytes_per_request": 78.7692,
-            "bytes_per_sec": 4.94686,
+            "bytes_per_request": 0,
+            "bytes_per_sec": 0,
             "connections": {
                 "async": {
                     "closing": 0,
-                    "keep_alive": 1,
+                    "keep_alive": 0,
                     "writing": 0
                 },
-                "total": 1
+                "total": 0
             },
             "cpu": {
                 "children_system": 0,
                 "children_user": 0,
-                "load": 3.99758,
-                "system": 16.39,
-                "user": 0.16
+                "system": 0,
+                "user": 0
             },
-            "hostname": "apache",
+            "hostname": "172.26.0.2:80",
             "load": {
-                "1": 170.11,
-                "15": 77.62,
-                "5": 167.51
+                "1": 1.3,
+                "15": 1.56,
+                "5": 1.34
             },
-            "requests_per_sec": 0.0628019,
+            "requests_per_sec": 0.666667,
             "scoreboard": {
                 "closing_connection": 0,
                 "dns_lookup": 0,
@@ -33,22 +32,22 @@
                 "idle_cleanup": 0,
                 "keepalive": 0,
                 "logging": 0,
-                "open_slot": 300,
+                "open_slot": 325,
                 "reading_request": 0,
                 "sending_reply": 1,
                 "starting_up": 0,
                 "total": 400,
-                "waiting_for_connection": 99
+                "waiting_for_connection": 74
             },
-            "total_accesses": 26,
-            "total_kbytes": 2,
+            "total_accesses": 2,
+            "total_kbytes": 0,
             "uptime": {
-                "server_uptime": 414,
-                "uptime": 414
+                "server_uptime": 3,
+                "uptime": 3
             },
             "workers": {
                 "busy": 1,
-                "idle": 99
+                "idle": 74
             }
         }
     },
@@ -57,7 +56,7 @@
         "name": "host.example.com"
     },
     "metricset": {
-        "host": "apache",
+        "host": "172.26.0.2:80",
         "module": "apache",
         "name": "status",
         "rtt": 115
diff --git a/metricbeat/module/apache/status/status_integration_test.go b/metricbeat/module/apache/status/status_integration_test.go
index 01e7f70b53a..986ecb0c19b 100644
--- a/metricbeat/module/apache/status/status_integration_test.go
+++ b/metricbeat/module/apache/status/status_integration_test.go
@@ -20,47 +20,48 @@
 package status
 
 import (
+	"net"
 	"testing"
 
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	"github.com/elastic/beats/metricbeat/module/apache"
+	"github.com/elastic/beats/metricbeat/module/apache/mtest"
 
 	"github.com/stretchr/testify/assert"
 )
 
-func TestFetch(t *testing.T) {
-	compose.EnsureUp(t, "apache")
+func TestStatus(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
 
-	f := mbtest.NewEventFetcher(t, getConfig())
-	event, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-
-	// Check number of fields.
-	if len(event) < 11 {
-		t.Fatal("Too few top-level elements in the event")
-	}
-}
+			err := mbtest.WriteEvent(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
 
-func TestData(t *testing.T) {
-	compose.EnsureUp(t, "apache")
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+			event, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
 
-	f := mbtest.NewEventFetcher(t, getConfig())
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
 
-	err := mbtest.WriteEvent(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
+			// Check number of fields.
+			if len(event) < 11 {
+				t.Fatal("Too few top-level elements in the event")
+			}
+		},
+	})
 }
 
-func getConfig() map[string]interface{} {
+func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "apache",
 		"metricsets": []string{"status"},
-		"hosts":      []string{apache.GetApacheEnvHost()},
+		"hosts":      []string{net.JoinHostPort(host, "80")},
 	}
 }

From 821f71a39a1d48911b7b6e55f4c055b92ace5819 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Thu, 23 Aug 2018 21:28:50 +0200
Subject: [PATCH 25/88] Revert "Remove local images on down"

This reverts commit 760cdc3cc68f92eeca559f2007627489525051fa.
---
 libbeat/tests/compose/wrapper.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libbeat/tests/compose/wrapper.go b/libbeat/tests/compose/wrapper.go
index b5026448d35..84911d3a429 100644
--- a/libbeat/tests/compose/wrapper.go
+++ b/libbeat/tests/compose/wrapper.go
@@ -148,7 +148,7 @@ func (d *wrapperDriver) Up(ctx context.Context, opts UpOptions, service string)
 }
 
 func (d *wrapperDriver) Down(ctx context.Context) error {
-	return d.cmd(ctx, "down", "-v", "--rmi=local").Run()
+	return d.cmd(ctx, "down", "-v").Run()
 }
 
 func (d *wrapperDriver) Kill(ctx context.Context, signal string, service string) error {

From d1321cca894ab29e6250a16ded3eef02a02391d3 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Fri, 24 Aug 2018 17:04:08 +0200
Subject: [PATCH 26/88] Revert "Revert "Remove local images on down""

This reverts commit 821f71a39a1d48911b7b6e55f4c055b92ace5819.
---
 libbeat/tests/compose/wrapper.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libbeat/tests/compose/wrapper.go b/libbeat/tests/compose/wrapper.go
index 84911d3a429..6fddd46bfbf 100644
--- a/libbeat/tests/compose/wrapper.go
+++ b/libbeat/tests/compose/wrapper.go
@@ -87,7 +87,7 @@ func (c *wrapperContainer) Host() string {
 }
 
 func (d *wrapperDriver) LockFile() string {
-	return d.Files[0] + "." + d.Name + ".lock"
+	return d.Files[0] + ".lock"
 }
 
 func (d *wrapperDriver) cmd(ctx context.Context, command string, arg ...string) *exec.Cmd {
@@ -148,7 +148,7 @@ func (d *wrapperDriver) Up(ctx context.Context, opts UpOptions, service string)
 }
 
 func (d *wrapperDriver) Down(ctx context.Context) error {
-	return d.cmd(ctx, "down", "-v").Run()
+	return d.cmd(ctx, "down", "-v", "--rmi=local").Run()
 }
 
 func (d *wrapperDriver) Kill(ctx context.Context, signal string, service string) error {

From 13252a6818546f004c96a2ea7255f4d923fc9f9e Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Thu, 6 Sep 2018 18:10:48 +0200
Subject: [PATCH 27/88] Using public ports as host addresses

---
 libbeat/tests/compose/runner.go               | 11 ++++++++++
 libbeat/tests/compose/wrapper.go              | 20 ++++++++++++++-----
 .../module/aerospike/docker-compose.yml       |  2 ++
 .../namespace/namespace_integration_test.go   |  3 +--
 metricbeat/module/apache/docker-compose.yml   |  2 ++
 .../apache/status/status_integration_test.go  |  3 +--
 metricbeat/module/kafka/_meta/run.sh          |  4 ++--
 metricbeat/module/kafka/broker.go             | 11 +++++++---
 .../consumergroup_integration_test.go         |  5 ++---
 metricbeat/module/kafka/docker-compose.yml    |  2 ++
 .../partition/partition_integration_test.go   |  5 ++---
 metricbeat/module/redis/docker-compose.yml    |  2 ++
 .../redis/info/info_integration_test.go       |  9 ++++-----
 .../keyspace/keyspace_integration_test.go     |  5 ++---
 14 files changed, 56 insertions(+), 28 deletions(-)

diff --git a/libbeat/tests/compose/runner.go b/libbeat/tests/compose/runner.go
index 977d9857eed..1646b5df267 100644
--- a/libbeat/tests/compose/runner.go
+++ b/libbeat/tests/compose/runner.go
@@ -20,6 +20,7 @@ package compose
 import (
 	"fmt"
 	"math/rand"
+	"net"
 	"os"
 	"sort"
 	"strings"
@@ -168,3 +169,13 @@ type runnerControl struct {
 func (r *runnerControl) Host() string {
 	return r.host
 }
+
+func (r *runnerControl) Hostname() string {
+	hostname, _, _ := net.SplitHostPort(r.host)
+	return hostname
+}
+
+func (r *runnerControl) Port() string {
+	_, port, _ := net.SplitHostPort(r.host)
+	return port
+}
diff --git a/libbeat/tests/compose/wrapper.go b/libbeat/tests/compose/wrapper.go
index 6fddd46bfbf..2cac67a0ee5 100644
--- a/libbeat/tests/compose/wrapper.go
+++ b/libbeat/tests/compose/wrapper.go
@@ -20,8 +20,10 @@ package compose
 import (
 	"context"
 	"fmt"
+	"net"
 	"os"
 	"os/exec"
+	"strconv"
 	"strings"
 	"time"
 
@@ -64,8 +66,7 @@ func (c *wrapperContainer) Old() bool {
 	return strings.Contains(c.info.Status, "minute")
 }
 
-func (c *wrapperContainer) Host() string {
-	// TODO: Support multiple networks/ports
+func (c *wrapperContainer) privateHost() string {
 	var ip string
 	for _, net := range c.info.NetworkSettings.Networks {
 		if len(net.IPAddress) > 0 {
@@ -76,14 +77,23 @@ func (c *wrapperContainer) Host() string {
 	if len(ip) == 0 {
 		return ""
 	}
-	return ip
 
-	/* TODO:
 	for _, port := range c.info.Ports {
 		return net.JoinHostPort(ip, strconv.Itoa(int(port.PrivatePort)))
 	}
 	return ""
-	*/
+}
+
+func (c *wrapperContainer) publicHost() string {
+	for _, port := range c.info.Ports {
+		return net.JoinHostPort("localhost", strconv.Itoa(int(port.PublicPort)))
+	}
+	return ""
+}
+
+func (c *wrapperContainer) Host() string {
+	// TODO: Support multiple networks/ports
+	return c.publicHost()
 }
 
 func (d *wrapperDriver) LockFile() string {
diff --git a/metricbeat/module/aerospike/docker-compose.yml b/metricbeat/module/aerospike/docker-compose.yml
index 6ba1a786ef4..ace5433662b 100644
--- a/metricbeat/module/aerospike/docker-compose.yml
+++ b/metricbeat/module/aerospike/docker-compose.yml
@@ -6,6 +6,8 @@ services:
       context: ./_meta
       args:
         AEROSPIKE_VERSION: ${AEROSPIKE_VERSION:-3.9.0}
+    ports:
+      - "3000"
     healthcheck:
       test:
       - "CMD-SHELL"
diff --git a/metricbeat/module/aerospike/namespace/namespace_integration_test.go b/metricbeat/module/aerospike/namespace/namespace_integration_test.go
index 9baaebacd8d..9643548c699 100644
--- a/metricbeat/module/aerospike/namespace/namespace_integration_test.go
+++ b/metricbeat/module/aerospike/namespace/namespace_integration_test.go
@@ -20,7 +20,6 @@
 package namespace
 
 import (
-	"net"
 	"testing"
 
 	"github.com/elastic/beats/libbeat/tests/compose"
@@ -44,6 +43,6 @@ func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "aerospike",
 		"metricsets": []string{"namespace"},
-		"hosts":      []string{net.JoinHostPort(host, "3000")},
+		"hosts":      []string{host},
 	}
 }
diff --git a/metricbeat/module/apache/docker-compose.yml b/metricbeat/module/apache/docker-compose.yml
index c7338df1666..93433d34491 100644
--- a/metricbeat/module/apache/docker-compose.yml
+++ b/metricbeat/module/apache/docker-compose.yml
@@ -6,3 +6,5 @@ services:
       context: ./_meta
       args:
         APACHE_VERSION: ${APACHE_VERSION:-2.4.20}
+    ports:
+    - 80
diff --git a/metricbeat/module/apache/status/status_integration_test.go b/metricbeat/module/apache/status/status_integration_test.go
index 986ecb0c19b..631bd0cc0b1 100644
--- a/metricbeat/module/apache/status/status_integration_test.go
+++ b/metricbeat/module/apache/status/status_integration_test.go
@@ -20,7 +20,6 @@
 package status
 
 import (
-	"net"
 	"testing"
 
 	"github.com/elastic/beats/libbeat/tests/compose"
@@ -62,6 +61,6 @@ func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "apache",
 		"metricsets": []string{"status"},
-		"hosts":      []string{net.JoinHostPort(host, "80")},
+		"hosts":      []string{host},
 	}
 }
diff --git a/metricbeat/module/kafka/_meta/run.sh b/metricbeat/module/kafka/_meta/run.sh
index 36b42a6a0d5..451e183c912 100755
--- a/metricbeat/module/kafka/_meta/run.sh
+++ b/metricbeat/module/kafka/_meta/run.sh
@@ -21,8 +21,8 @@ wait_for_port 2181
 echo "Starting Kafka broker"
 mkdir -p ${KAFKA_LOGS_DIR}
 ${KAFKA_HOME}/bin/kafka-server-start.sh ${KAFKA_HOME}/config/server.properties \
-    --override delete.topic.enable=true --override advertised.host.name=${KAFKA_ADVERTISED_HOST} \
-    --override listeners=PLAINTEXT://0.0.0.0:9092 \
+    --override delete.topic.enable=true \
+    --override listeners=PLAINTEXT://:9092 \
     --override logs.dir=${KAFKA_LOGS_DIR} &
 
 wait_for_port 9092
diff --git a/metricbeat/module/kafka/broker.go b/metricbeat/module/kafka/broker.go
index dd5a15b47a8..2d241c14e88 100644
--- a/metricbeat/module/kafka/broker.go
+++ b/metricbeat/module/kafka/broker.go
@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"github.com/Shopify/sarama"
+	"github.com/pkg/errors"
 
 	"github.com/elastic/beats/libbeat/common"
 )
@@ -102,7 +103,11 @@ func (b *Broker) Close() error {
 // Connect connects the broker to the configured host
 func (b *Broker) Connect() error {
 	if err := b.broker.Open(b.cfg); err != nil {
-		return err
+		return errors.Wrap(err, "opening broker connection")
+	}
+
+	if ok, err := b.broker.Connected(); !ok {
+		return errors.Wrap(err, "broker not connected")
 	}
 
 	if b.id != noID || !b.matchID {
@@ -113,13 +118,13 @@ func (b *Broker) Connect() error {
 	meta, err := queryMetadataWithRetry(b.broker, b.cfg, nil)
 	if err != nil {
 		closeBroker(b.broker)
-		return err
+		return errors.Wrap(err, "querying metadata")
 	}
 
 	other := findMatchingBroker(brokerAddress(b.broker), meta.Brokers)
 	if other == nil { // no broker found
 		closeBroker(b.broker)
-		return fmt.Errorf("No advertised broker with address %v found", b.Addr())
+		return fmt.Errorf("no advertised broker with address %v found", b.Addr())
 	}
 
 	debugf("found matching broker %v with id %v", other.Addr(), other.ID())
diff --git a/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go b/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go
index 4f40288ba51..8a6541057c7 100644
--- a/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go
+++ b/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go
@@ -21,7 +21,6 @@ package consumergroup
 
 import (
 	"io"
-	"net"
 	"testing"
 	"time"
 
@@ -54,7 +53,7 @@ func TestConsumerGroup(t *testing.T) {
 }
 
 func startConsumer(t *testing.T, topic, host string) (io.Closer, error) {
-	brokers := []string{net.JoinHostPort(host, "9092")}
+	brokers := []string{host}
 	topics := []string{topic}
 	return saramacluster.NewConsumer(brokers, "test-group", topics, nil)
 }
@@ -63,6 +62,6 @@ func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "kafka",
 		"metricsets": []string{"consumergroup"},
-		"hosts":      []string{net.JoinHostPort(host, "9092")},
+		"hosts":      []string{host},
 	}
 }
diff --git a/metricbeat/module/kafka/docker-compose.yml b/metricbeat/module/kafka/docker-compose.yml
index 9a6fdade8fb..abd3e8f0aed 100644
--- a/metricbeat/module/kafka/docker-compose.yml
+++ b/metricbeat/module/kafka/docker-compose.yml
@@ -6,3 +6,5 @@ services:
       context: ./_meta
       args:
         KAFKA_VERSION: 2.0.0
+    ports:
+    - 9092
diff --git a/metricbeat/module/kafka/partition/partition_integration_test.go b/metricbeat/module/kafka/partition/partition_integration_test.go
index ac51227bb7f..3b0c8c507fa 100644
--- a/metricbeat/module/kafka/partition/partition_integration_test.go
+++ b/metricbeat/module/kafka/partition/partition_integration_test.go
@@ -22,7 +22,6 @@ package partition
 import (
 	"fmt"
 	"math/rand"
-	"net"
 	"strconv"
 	"testing"
 	"time"
@@ -122,7 +121,7 @@ func generateKafkaData(t *testing.T, topic string, host string) {
 	config.Producer.Retry.Backoff = 500 * time.Millisecond
 	config.Metadata.Retry.Max = 20
 	config.Metadata.Retry.Backoff = 500 * time.Millisecond
-	client, err := sarama.NewClient([]string{net.JoinHostPort(host, "9092")}, config)
+	client, err := sarama.NewClient([]string{host}, config)
 	if err != nil {
 		t.Errorf("%s", err)
 		t.FailNow()
@@ -159,7 +158,7 @@ func getConfig(topic string, host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "kafka",
 		"metricsets": []string{"partition"},
-		"hosts":      []string{net.JoinHostPort(host, "9092")},
+		"hosts":      []string{host},
 		"topics":     topics,
 	}
 }
diff --git a/metricbeat/module/redis/docker-compose.yml b/metricbeat/module/redis/docker-compose.yml
index 9fac97dce56..f0821dde978 100644
--- a/metricbeat/module/redis/docker-compose.yml
+++ b/metricbeat/module/redis/docker-compose.yml
@@ -3,6 +3,8 @@ version: "2.1"
 services:
   redis:
     image: redis:${REDIS_VERSION:-4.0.11}-${IMAGE_OS:-alpine}
+    ports:
+      - 6379
     healthcheck:
       test:
       - "CMD-SHELL"
diff --git a/metricbeat/module/redis/info/info_integration_test.go b/metricbeat/module/redis/info/info_integration_test.go
index 1c4f946a7b4..7d17df5b380 100644
--- a/metricbeat/module/redis/info/info_integration_test.go
+++ b/metricbeat/module/redis/info/info_integration_test.go
@@ -20,7 +20,6 @@
 package info
 
 import (
-	"net"
 	"testing"
 
 	"github.com/elastic/beats/libbeat/common"
@@ -100,7 +99,7 @@ func TestInfo(t *testing.T) {
 
 // addPassword will add a password to redis.
 func addPassword(host, pass string) error {
-	c, err := rd.Dial("tcp", net.JoinHostPort(host, "6379"))
+	c, err := rd.Dial("tcp", host)
 	if err != nil {
 		return err
 	}
@@ -112,7 +111,7 @@ func addPassword(host, pass string) error {
 
 // resetPassword changes the password to the redis DB.
 func resetPassword(host, currentPass string) error {
-	c, err := rd.Dial("tcp", net.JoinHostPort(host, "6379"))
+	c, err := rd.Dial("tcp", host)
 	if err != nil {
 		return err
 	}
@@ -129,7 +128,7 @@ func resetPassword(host, currentPass string) error {
 
 // writeToRedis will write to the default DB 0.
 func writeToRedis(host string) error {
-	c, err := rd.Dial("tcp", net.JoinHostPort(host, "6379"))
+	c, err := rd.Dial("tcp", host)
 	if err != nil {
 		return err
 	}
@@ -143,7 +142,7 @@ func getConfig(password, host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "redis",
 		"metricsets": []string{"info"},
-		"hosts":      []string{net.JoinHostPort(host, "6379")},
+		"hosts":      []string{host},
 		"password":   password,
 	}
 }
diff --git a/metricbeat/module/redis/keyspace/keyspace_integration_test.go b/metricbeat/module/redis/keyspace/keyspace_integration_test.go
index 1d47088eaa7..4109d384568 100644
--- a/metricbeat/module/redis/keyspace/keyspace_integration_test.go
+++ b/metricbeat/module/redis/keyspace/keyspace_integration_test.go
@@ -20,7 +20,6 @@
 package keyspace
 
 import (
-	"net"
 	"strings"
 	"testing"
 
@@ -74,7 +73,7 @@ func TestKeyspace(t *testing.T) {
 // addEntry adds an entry to redis
 func addEntry(t *testing.T, host string) {
 	// Insert at least one event to make sure db exists
-	c, err := rd.Dial("tcp", net.JoinHostPort(host, "6379"))
+	c, err := rd.Dial("tcp", host)
 	if err != nil {
 		t.Fatal("connect", err)
 	}
@@ -89,6 +88,6 @@ func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "redis",
 		"metricsets": []string{"keyspace"},
-		"hosts":      []string{net.JoinHostPort(host, "6379")},
+		"hosts":      []string{host},
 	}
 }

From 01174617b8e414512e0e497e3f0e92586554eb24 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Fri, 7 Sep 2018 18:33:06 +0200
Subject: [PATCH 28/88] Inject context information into the compose containers

Some services need to know the addresses clients are going to use to
connect with them, for example Kafka brokers announce the address that
has to be used to connect with them. This information is not available
in the containers. This change injects some of this information so
services can configure themselves.
---
 libbeat/tests/compose/runner.go      |  4 +-
 libbeat/tests/compose/wrapper.go     | 68 +++++++++++++++++++++++++++-
 metricbeat/module/kafka/_meta/run.sh | 23 +++++++++-
 3 files changed, 90 insertions(+), 5 deletions(-)

diff --git a/libbeat/tests/compose/runner.go b/libbeat/tests/compose/runner.go
index 1646b5df267..58d92255df2 100644
--- a/libbeat/tests/compose/runner.go
+++ b/libbeat/tests/compose/runner.go
@@ -130,10 +130,12 @@ func (r *TestRunner) Run(t *testing.T, tests Suite) {
 			}
 
 			err := project.Start(r.Service)
+			// Down() is "idempotent", Start() has several points where it can fail,
+			// so run Down() even if Start() fails.
+			defer project.Down()
 			if err != nil {
 				t.Fatal(err)
 			}
-			defer project.Down()
 
 			err = project.Wait(timeout, r.Service)
 			if err != nil {
diff --git a/libbeat/tests/compose/wrapper.go b/libbeat/tests/compose/wrapper.go
index 2cac67a0ee5..5e4e3f60269 100644
--- a/libbeat/tests/compose/wrapper.go
+++ b/libbeat/tests/compose/wrapper.go
@@ -18,11 +18,14 @@
 package compose
 
 import (
+	"archive/tar"
+	"bytes"
 	"context"
 	"fmt"
 	"net"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
@@ -86,7 +89,9 @@ func (c *wrapperContainer) privateHost() string {
 
 func (c *wrapperContainer) publicHost() string {
 	for _, port := range c.info.Ports {
-		return net.JoinHostPort("localhost", strconv.Itoa(int(port.PublicPort)))
+		if port.PublicPort != 0 {
+			return net.JoinHostPort("localhost", strconv.Itoa(int(port.PublicPort)))
+		}
 	}
 	return ""
 }
@@ -147,7 +152,7 @@ func (d *wrapperDriver) Up(ctx context.Context, opts UpOptions, service string)
 		// number of networks, retry while the context is not done
 		err := d.cmd(ctx, "up", args...).Run()
 		if err == nil {
-			return nil
+			return d.setupAdvertisedHost(ctx, service)
 		}
 		select {
 		case <-time.After(time.Second):
@@ -157,6 +162,64 @@ func (d *wrapperDriver) Up(ctx context.Context, opts UpOptions, service string)
 	}
 }
 
+func writeToContainer(ctx context.Context, id, filename, content string) error {
+	var buf bytes.Buffer
+	tw := tar.NewWriter(&buf)
+	now := time.Now()
+	err := tw.WriteHeader(&tar.Header{
+		Typeflag:   tar.TypeReg,
+		Name:       filepath.Base(filename),
+		Mode:       0100644,
+		Size:       int64(len(content)),
+		ModTime:    now,
+		AccessTime: now,
+		ChangeTime: now,
+	})
+	if err != nil {
+		return errors.Wrap(err, "failed to write tar header")
+	}
+	if _, err := tw.Write([]byte(content)); err != nil {
+		return errors.Wrap(err, "failed to write tar file")
+	}
+	if err := tw.Close(); err != nil {
+		return errors.Wrap(err, "failed to close tar")
+	}
+
+	cli, err := client.NewEnvClient()
+	if err != nil {
+		return errors.Wrap(err, "failed to start docker client")
+	}
+	defer cli.Close()
+
+	opts := types.CopyToContainerOptions{}
+	err = cli.CopyToContainer(ctx, id, filepath.Dir(filename), bytes.NewReader(buf.Bytes()), opts)
+	if err != nil {
+		return errors.Wrapf(err, "failed to copy environment to container %s", id)
+	}
+	return nil
+}
+
+func (d *wrapperDriver) setupAdvertisedHost(ctx context.Context, service string) error {
+	containers, err := d.containers(ctx, Filter{State: AnyState}, service)
+	if err != nil {
+		return errors.Wrap(err, "setupAdvertisedHost")
+	}
+	if len(containers) == 0 {
+		return errors.Errorf("no containers for service %s", service)
+	}
+
+	for _, c := range containers {
+		w := &wrapperContainer{info: c}
+		content := fmt.Sprintf("SERVICE_HOST=%s", w.Host())
+
+		err := writeToContainer(ctx, c.ID, "/run/compose_env", content)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (d *wrapperDriver) Down(ctx context.Context) error {
 	return d.cmd(ctx, "down", "-v", "--rmi=local").Run()
 }
@@ -206,6 +269,7 @@ func (d *wrapperDriver) containers(ctx context.Context, projectFilter Filter, fi
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to start docker client")
 	}
+	defer cli.Close()
 
 	var serviceFilters []filters.Args
 	if len(filter) == 0 {
diff --git a/metricbeat/module/kafka/_meta/run.sh b/metricbeat/module/kafka/_meta/run.sh
index 451e183c912..696b9507b4b 100755
--- a/metricbeat/module/kafka/_meta/run.sh
+++ b/metricbeat/module/kafka/_meta/run.sh
@@ -1,6 +1,22 @@
 #!/bin/bash
 
-KAFKA_ADVERTISED_HOST=$(dig +short $HOSTNAME)
+# Check if KAFKA_ADVERTISED_HOST is set
+# if not wait to read it from file
+if [ -z "$KAFKA_ADVERTISED_HOST" ]; then
+	echo "KAFKA_ADVERTISED_HOST needed, will wait for it on /var/run/compose_env"
+	while true; do
+		if [ -f /run/compose_env ]; then
+			source /run/compose_env
+			KAFKA_ADVERTISED_HOST=$SERVICE_HOST
+		fi
+		if [ -n "$KAFKA_ADVERTISED_HOST" ]; then
+			# Remove it so it is not reused
+			> /run/compose_env
+			break
+		fi
+		sleep 1
+	done
+fi
 
 wait_for_port() {
     count=20
@@ -22,7 +38,10 @@ echo "Starting Kafka broker"
 mkdir -p ${KAFKA_LOGS_DIR}
 ${KAFKA_HOME}/bin/kafka-server-start.sh ${KAFKA_HOME}/config/server.properties \
     --override delete.topic.enable=true \
-    --override listeners=PLAINTEXT://:9092 \
+    --override listeners=INSIDE://localhost:9091,OUTSIDE://0.0.0.0:9092 \
+    --override advertised.listeners=INSIDE://localhost:9091,OUTSIDE://$KAFKA_ADVERTISED_HOST \
+    --override listener.security.protocol.map=INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT \
+    --override inter.broker.listener.name=INSIDE \
     --override logs.dir=${KAFKA_LOGS_DIR} &
 
 wait_for_port 9092

From 2ca7e84abe0bf488a10c8616496875429ba6d9b6 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Fri, 7 Sep 2018 20:42:46 +0200
Subject: [PATCH 29/88] Keep environment when calling docker compose

---
 libbeat/tests/compose/wrapper.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/libbeat/tests/compose/wrapper.go b/libbeat/tests/compose/wrapper.go
index 5e4e3f60269..119b68280c3 100644
--- a/libbeat/tests/compose/wrapper.go
+++ b/libbeat/tests/compose/wrapper.go
@@ -117,7 +117,7 @@ func (d *wrapperDriver) cmd(ctx context.Context, command string, arg ...string)
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 	if len(d.Environment) > 0 {
-		cmd.Env = d.Environment
+		cmd.Env = append(d.Environment, os.Environ()...)
 	}
 	return cmd
 }
@@ -154,6 +154,7 @@ func (d *wrapperDriver) Up(ctx context.Context, opts UpOptions, service string)
 		if err == nil {
 			return d.setupAdvertisedHost(ctx, service)
 		}
+
 		select {
 		case <-time.After(time.Second):
 		case <-ctx.Done():

From 2246ccbe614d4cf9a6c7bf4b0c804c26f4e642cf Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 11 Sep 2018 12:33:29 +0200
Subject: [PATCH 30/88] pep8

---
 metricbeat/tests/system/test_redis.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/metricbeat/tests/system/test_redis.py b/metricbeat/tests/system/test_redis.py
index 188948238ff..a1404390860 100644
--- a/metricbeat/tests/system/test_redis.py
+++ b/metricbeat/tests/system/test_redis.py
@@ -123,8 +123,10 @@ def get_hosts(self):
         return [os.getenv('REDIS_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('REDIS_PORT', '6379')]
 
+
 class TestRedis4(Test):
     COMPOSE_SERVICES = ['redis_4']
 
+
 class TestRedis5(Test):
     COMPOSE_SERVICES = ['redis_5']

From 5e4335c3574596ba6f3fce1e501dc50631aadc23 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 11 Sep 2018 13:49:25 +0200
Subject: [PATCH 31/88] Add type to simplify config of test runners

---
 libbeat/tests/compose/runner.go             | 12 ++++++++++--
 metricbeat/module/aerospike/mtest/runner.go |  4 ++--
 metricbeat/module/kafka/mtest/runner.go     |  4 ++--
 metricbeat/module/redis/mtest/runner.go     |  6 +++---
 4 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/libbeat/tests/compose/runner.go b/libbeat/tests/compose/runner.go
index 58d92255df2..4484e9b616e 100644
--- a/libbeat/tests/compose/runner.go
+++ b/libbeat/tests/compose/runner.go
@@ -36,13 +36,15 @@ func init() {
 
 type TestRunner struct {
 	Service  string
-	Options  map[string][]string
+	Options  RunnerOptions
 	Parallel bool
 	Timeout  int
 }
 
 type Suite map[string]func(t *testing.T, r R)
 
+type RunnerOptions map[string][]string
+
 func (r *TestRunner) scenarios() []map[string]string {
 	n := 1
 	options := make(map[string][]string)
@@ -96,6 +98,8 @@ func (r *TestRunner) runHostOverride(t *testing.T, tests Suite) bool {
 }
 
 func (r *TestRunner) Run(t *testing.T, tests Suite) {
+	t.Helper()
+
 	if r.runHostOverride(t, tests) {
 		return
 	}
@@ -105,7 +109,11 @@ func (r *TestRunner) Run(t *testing.T, tests Suite) {
 		timeout = 300
 	}
 
-	for _, s := range r.scenarios() {
+	scenarios := r.scenarios()
+	if len(scenarios) == 0 {
+		t.Fatal("Test runner configuration doesn't produce scenarios")
+	}
+	for _, s := range scenarios {
 		var vars []string
 		for k, v := range s {
 			os.Setenv(k, v)
diff --git a/metricbeat/module/aerospike/mtest/runner.go b/metricbeat/module/aerospike/mtest/runner.go
index 29a0a6a69e0..fbd3ef5b56c 100644
--- a/metricbeat/module/aerospike/mtest/runner.go
+++ b/metricbeat/module/aerospike/mtest/runner.go
@@ -26,8 +26,8 @@ import (
 var (
 	Runner = compose.TestRunner{
 		Service: "aerospike",
-		Options: map[string][]string{
-			"AEROSPIKE_VERSION": []string{
+		Options: compose.RunnerOptions{
+			"AEROSPIKE_VERSION": {
 				"3.9.0",
 				"3.13.0.11",
 				"3.16.0.6",
diff --git a/metricbeat/module/kafka/mtest/runner.go b/metricbeat/module/kafka/mtest/runner.go
index e4d93295a39..e3b6a5c2e80 100644
--- a/metricbeat/module/kafka/mtest/runner.go
+++ b/metricbeat/module/kafka/mtest/runner.go
@@ -26,8 +26,8 @@ import (
 var (
 	Runner = compose.TestRunner{
 		Service: "kafka",
-		Options: map[string][]string{
-			"KAFKA_VERSION": []string{
+		Options: compose.RunnerOptions{
+			"KAFKA_VERSION": {
 				"0.10.2.1",
 				"1.1.0",
 				"2.0.0",
diff --git a/metricbeat/module/redis/mtest/runner.go b/metricbeat/module/redis/mtest/runner.go
index 49e0353365c..6d3dbd546e8 100644
--- a/metricbeat/module/redis/mtest/runner.go
+++ b/metricbeat/module/redis/mtest/runner.go
@@ -26,13 +26,13 @@ import (
 var (
 	Runner = compose.TestRunner{
 		Service: "redis",
-		Options: map[string][]string{
-			"REDIS_VERSION": []string{
+		Options: compose.RunnerOptions{
+			"REDIS_VERSION": {
 				"3.2.12",
 				"4.0.11",
 				"5.0-rc",
 			},
-			"IMAGE_OS": []string{
+			"IMAGE_OS": {
 				"alpine",
 				"stretch",
 			},

From 9a21006f0487b9f428d393bdc8acb50983df3438 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 11 Sep 2018 13:49:50 +0200
Subject: [PATCH 32/88] Run dropwizard tests with testrunner

---
 metricbeat/docker-compose.yml                 |   2 +
 .../collector/collector_integration_test.go   | 114 ++++++++----------
 2 files changed, 51 insertions(+), 65 deletions(-)

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index aa4d671695b..417518296b1 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -41,6 +41,8 @@ services:
 
   dropwizard:
     build: ./module/dropwizard/_meta
+    ports:
+      - 8080
 
   elasticsearch:
     build: ./module/elasticsearch/_meta
diff --git a/metricbeat/module/dropwizard/collector/collector_integration_test.go b/metricbeat/module/dropwizard/collector/collector_integration_test.go
index c4b3cf490ab..8f4af491b00 100644
--- a/metricbeat/module/dropwizard/collector/collector_integration_test.go
+++ b/metricbeat/module/dropwizard/collector/collector_integration_test.go
@@ -20,7 +20,6 @@
 package collector
 
 import (
-	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -30,83 +29,68 @@ import (
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
 )
 
-func TestFetch(t *testing.T) {
-	compose.EnsureUp(t, "dropwizard")
+func TestCollector(t *testing.T) {
+	t.Parallel()
 
-	f := mbtest.NewEventsFetcher(t, getConfig())
-	events, err := f.Fetch()
+	runner := compose.TestRunner{Service: "dropwizard"}
 
-	hasTag := false
-	doesntHaveTag := false
-	for _, event := range events {
+	runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
+			events, err := f.Fetch()
 
-		ok, _ := event.HasKey("my_histogram")
-		if ok {
-			_, err := event.GetValue("tags")
-			if err == nil {
-				t.Fatal("write", "my_counter not supposed to have tags")
+			hasTag := false
+			doesntHaveTag := false
+			for _, event := range events {
+
+				ok, _ := event.HasKey("my_histogram")
+				if ok {
+					_, err := event.GetValue("tags")
+					if err == nil {
+						t.Fatal("write", "my_counter not supposed to have tags")
+					}
+					doesntHaveTag = true
+				}
+
+				ok, _ = event.HasKey("my_counter")
+				if ok {
+					tagsRaw, err := event.GetValue("tags")
+					if err != nil {
+						t.Fatal("write", err)
+					} else {
+						tags, ok := tagsRaw.(common.MapStr)
+						if !ok {
+							t.Fatal("write", "unable to cast tags to common.MapStr")
+						} else {
+							assert.Equal(t, len(tags), 1)
+							hasTag = true
+						}
+					}
+				}
+			}
+			assert.Equal(t, hasTag, true)
+			assert.Equal(t, doesntHaveTag, true)
+			if !assert.NoError(t, err) {
+				t.FailNow()
 			}
-			doesntHaveTag = true
-		}
 
-		ok, _ = event.HasKey("my_counter")
-		if ok {
-			tagsRaw, err := event.GetValue("tags")
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events)
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
+			err := mbtest.WriteEvents(f, t)
 			if err != nil {
 				t.Fatal("write", err)
-			} else {
-				tags, ok := tagsRaw.(common.MapStr)
-				if !ok {
-					t.Fatal("write", "unable to cast tags to common.MapStr")
-				} else {
-					assert.Equal(t, len(tags), 1)
-					hasTag = true
-				}
 			}
-		}
-	}
-	assert.Equal(t, hasTag, true)
-	assert.Equal(t, doesntHaveTag, true)
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events)
-}
-
-func TestData(t *testing.T) {
-	compose.EnsureUp(t, "dropwizard")
-
-	f := mbtest.NewEventsFetcher(t, getConfig())
-	err := mbtest.WriteEvents(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
-}
-
-func getEnvHost() string {
-	host := os.Getenv("DROPWIZARD_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
-}
-
-func getEnvPort() string {
-	port := os.Getenv("DROPWIZARD_PORT")
-
-	if len(port) == 0 {
-		port = "8080"
-	}
-	return port
+		},
+	})
 }
 
-func getConfig() map[string]interface{} {
+func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":       "dropwizard",
 		"metricsets":   []string{"collector"},
-		"hosts":        []string{getEnvHost() + ":" + getEnvPort()},
+		"hosts":        []string{host},
 		"namespace":    "testnamespace",
 		"metrics_path": "/test/metrics",
 		"enabled":      true,

From edc7982e5a49cf010ad978b9ebd3fc41c80b1fba Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 11 Sep 2018 14:36:43 +0200
Subject: [PATCH 33/88] Run ES tests with testrunner

---
 metricbeat/docker-compose.yml                 |   2 +
 .../elasticsearch_integration_test.go         | 103 +++++++-----------
 2 files changed, 43 insertions(+), 62 deletions(-)

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 417518296b1..180299e81a7 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -52,6 +52,8 @@ services:
       - "transport.host=127.0.0.1"
       - "http.host=0.0.0.0"
       - "xpack.security.enabled=false"
+    ports:
+      - 9200
 
   envoyproxy:
     build: ./module/envoyproxy/_meta
diff --git a/metricbeat/module/elasticsearch/elasticsearch_integration_test.go b/metricbeat/module/elasticsearch/elasticsearch_integration_test.go
index 5ceff6d7187..2345cd5f07f 100644
--- a/metricbeat/module/elasticsearch/elasticsearch_integration_test.go
+++ b/metricbeat/module/elasticsearch/elasticsearch_integration_test.go
@@ -22,9 +22,7 @@ package elasticsearch_test
 import (
 	"fmt"
 	"io/ioutil"
-	"net"
 	"net/http"
-	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -55,74 +53,55 @@ var metricSets = []string{
 	"shard",
 }
 
-func TestFetch(t *testing.T) {
-	compose.EnsureUp(t, "elasticsearch")
-
-	host := net.JoinHostPort(getEnvHost(), getEnvPort())
-	err := createIndex(host)
-	assert.NoError(t, err)
-
-	err = enableTrialLicense(host)
-	assert.NoError(t, err)
-
-	err = createMLJob(host)
-	assert.NoError(t, err)
-
-	for _, metricSet := range metricSets {
-		t.Run(metricSet, func(t *testing.T) {
-			f := mbtest.NewReportingMetricSetV2(t, getConfig(metricSet))
-			events, errs := mbtest.ReportingFetchV2(f)
-
-			assert.Empty(t, errs)
-			if !assert.NotEmpty(t, events) {
-				t.FailNow()
+func TestElasticsearch(t *testing.T) {
+	runner := compose.TestRunner{Service: "elasticsearch"}
+
+	runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			host := r.Host()
+			err := createIndex(host)
+			assert.NoError(t, err)
+
+			err = enableTrialLicense(host)
+			assert.NoError(t, err)
+
+			err = createMLJob(host)
+			assert.NoError(t, err)
+
+			for _, metricSet := range metricSets {
+				t.Run(metricSet, func(t *testing.T) {
+					f := mbtest.NewReportingMetricSetV2(t, getConfig(metricSet, host))
+					events, errs := mbtest.ReportingFetchV2(f)
+
+					assert.Empty(t, errs)
+					if !assert.NotEmpty(t, events) {
+						t.FailNow()
+					}
+					t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(),
+						events[0].BeatEvent("elasticsearch", metricSet).Fields.StringToPrint())
+				})
 			}
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(),
-				events[0].BeatEvent("elasticsearch", metricSet).Fields.StringToPrint())
-		})
-	}
-}
-
-func TestData(t *testing.T) {
-	compose.EnsureUp(t, "elasticsearch")
-
-	for _, metricSet := range metricSets {
-		t.Run(metricSet, func(t *testing.T) {
-			f := mbtest.NewReportingMetricSetV2(t, getConfig(metricSet))
-			err := mbtest.WriteEventsReporterV2(f, t, metricSet)
-			if err != nil {
-				t.Fatal("write", err)
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			for _, metricSet := range metricSets {
+				t.Run(metricSet, func(t *testing.T) {
+					f := mbtest.NewReportingMetricSetV2(t, getConfig(metricSet, r.Host()))
+					err := mbtest.WriteEventsReporterV2(f, t, metricSet)
+					if err != nil {
+						t.Fatal("write", err)
+					}
+				})
 			}
-		})
-	}
-}
-
-// GetEnvHost returns host for Elasticsearch
-func getEnvHost() string {
-	host := os.Getenv("ES_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
-}
-
-// GetEnvPort returns port for Elasticsearch
-func getEnvPort() string {
-	port := os.Getenv("ES_PORT")
-
-	if len(port) == 0 {
-		port = "9200"
-	}
-	return port
+		},
+	})
 }
 
 // GetConfig returns config for elasticsearch module
-func getConfig(metricset string) map[string]interface{} {
+func getConfig(metricset string, host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "elasticsearch",
 		"metricsets": []string{metricset},
-		"hosts":      []string{getEnvHost() + ":" + getEnvPort()},
+		"hosts":      []string{host},
 		"index_recovery.active_only": false,
 	}
 }

From a322c5f05dfa0a606f47cbeb1a6eb5278987b9af Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 11 Sep 2018 15:24:41 +0200
Subject: [PATCH 34/88] Run envoyproxy tests with testrunner

---
 metricbeat/docker-compose.yml                 |  2 +
 .../server/server_integration_test.go         | 66 +++++++------------
 2 files changed, 26 insertions(+), 42 deletions(-)

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 180299e81a7..71a07c02230 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -57,6 +57,8 @@ services:
 
   envoyproxy:
     build: ./module/envoyproxy/_meta
+    ports:
+      - 9901
 
   etcd:
     build: ./module/etcd/_meta
diff --git a/metricbeat/module/envoyproxy/server/server_integration_test.go b/metricbeat/module/envoyproxy/server/server_integration_test.go
index af001de4594..385732d257c 100644
--- a/metricbeat/module/envoyproxy/server/server_integration_test.go
+++ b/metricbeat/module/envoyproxy/server/server_integration_test.go
@@ -20,7 +20,6 @@
 package server
 
 import (
-	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -29,51 +28,34 @@ import (
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
 )
 
-func TestData(t *testing.T) {
-	compose.EnsureUp(t, "envoyproxy")
-
-	f := mbtest.NewEventFetcher(t, getConfig())
-	err := mbtest.WriteEvent(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
-}
-
-func TestFetch(t *testing.T) {
-	compose.EnsureUp(t, "envoyproxy")
-
-	f := mbtest.NewEventFetcher(t, getConfig())
-	event, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	assert.NotNil(t, event)
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+func TestEnvoyproxy(t *testing.T) {
+	runner := compose.TestRunner{Service: "envoyproxy"}
+
+	runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+			event, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			assert.NotNil(t, event)
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+			err := mbtest.WriteEvent(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
 
-func getConfig() map[string]interface{} {
+func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "envoyproxy",
 		"metricsets": []string{"server"},
-		"hosts":      []string{GetEnvHost() + ":" + GetEnvPort()},
-	}
-}
-
-func GetEnvHost() string {
-	host := os.Getenv("ENVOYPROXY_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
-}
-
-func GetEnvPort() string {
-	port := os.Getenv("ENVOYPROXY_PORT")
-
-	if len(port) == 0 {
-		port = "9901"
+		"hosts":      []string{host},
 	}
-	return port
 }

From 476530aedc83fe711c945c4ee33e8c1a86e79162 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 11 Sep 2018 15:33:27 +0200
Subject: [PATCH 35/88] Run etcd tests with testrunner

---
 metricbeat/docker-compose.yml                 |  2 +
 .../etcd/leader/leader_integration_test.go    | 66 +++++++------------
 .../module/etcd/self/self_integration_test.go | 66 +++++++------------
 .../etcd/store/store_integration_test.go      | 66 +++++++------------
 4 files changed, 74 insertions(+), 126 deletions(-)

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 71a07c02230..594aed0129c 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -62,6 +62,8 @@ services:
 
   etcd:
     build: ./module/etcd/_meta
+    ports:
+      - 2379
 
   haproxy:
     build: ./module/haproxy/_meta
diff --git a/metricbeat/module/etcd/leader/leader_integration_test.go b/metricbeat/module/etcd/leader/leader_integration_test.go
index 8ffee3e5934..bd2dd7bc519 100644
--- a/metricbeat/module/etcd/leader/leader_integration_test.go
+++ b/metricbeat/module/etcd/leader/leader_integration_test.go
@@ -20,7 +20,6 @@
 package leader
 
 import (
-	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -29,51 +28,34 @@ import (
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
 )
 
-func TestData(t *testing.T) {
-	compose.EnsureUp(t, "etcd")
-
-	f := mbtest.NewEventFetcher(t, getConfig())
-	err := mbtest.WriteEvent(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
-}
-
-func TestFetch(t *testing.T) {
-	compose.EnsureUp(t, "etcd")
-
-	f := mbtest.NewEventFetcher(t, getConfig())
-	event, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	assert.NotNil(t, event)
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+func TestLeader(t *testing.T) {
+	runner := compose.TestRunner{Service: "etcd"}
+
+	runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+			event, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			assert.NotNil(t, event)
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+			err := mbtest.WriteEvent(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
 
-func getConfig() map[string]interface{} {
+func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "etcd",
 		"metricsets": []string{"leader"},
-		"hosts":      []string{GetEnvHost() + ":" + GetEnvPort()},
-	}
-}
-
-func GetEnvHost() string {
-	host := os.Getenv("ETCD_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
-}
-
-func GetEnvPort() string {
-	port := os.Getenv("ETCD_PORT")
-
-	if len(port) == 0 {
-		port = "2379"
+		"hosts":      []string{host},
 	}
-	return port
 }
diff --git a/metricbeat/module/etcd/self/self_integration_test.go b/metricbeat/module/etcd/self/self_integration_test.go
index 031d6e92c58..be8b979f7c1 100644
--- a/metricbeat/module/etcd/self/self_integration_test.go
+++ b/metricbeat/module/etcd/self/self_integration_test.go
@@ -20,7 +20,6 @@
 package self
 
 import (
-	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -29,51 +28,34 @@ import (
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
 )
 
-func TestData(t *testing.T) {
-	compose.EnsureUp(t, "etcd")
-
-	f := mbtest.NewEventFetcher(t, getConfig())
-	err := mbtest.WriteEvent(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
-}
-
-func TestFetch(t *testing.T) {
-	compose.EnsureUp(t, "etcd")
-
-	f := mbtest.NewEventFetcher(t, getConfig())
-	event, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	assert.NotNil(t, event)
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+func TestLeader(t *testing.T) {
+	runner := compose.TestRunner{Service: "etcd"}
+
+	runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+			event, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			assert.NotNil(t, event)
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+			err := mbtest.WriteEvent(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
 
-func getConfig() map[string]interface{} {
+func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "etcd",
 		"metricsets": []string{"self"},
-		"hosts":      []string{GetEnvHost() + ":" + GetEnvPort()},
-	}
-}
-
-func GetEnvHost() string {
-	host := os.Getenv("ETCD_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
-}
-
-func GetEnvPort() string {
-	port := os.Getenv("ETCD_PORT")
-
-	if len(port) == 0 {
-		port = "2379"
+		"hosts":      []string{host},
 	}
-	return port
 }
diff --git a/metricbeat/module/etcd/store/store_integration_test.go b/metricbeat/module/etcd/store/store_integration_test.go
index 5b4c49f34c9..22be4797e8c 100644
--- a/metricbeat/module/etcd/store/store_integration_test.go
+++ b/metricbeat/module/etcd/store/store_integration_test.go
@@ -20,7 +20,6 @@
 package store
 
 import (
-	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -29,51 +28,34 @@ import (
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
 )
 
-func TestData(t *testing.T) {
-	compose.EnsureUp(t, "etcd")
-
-	f := mbtest.NewEventFetcher(t, getConfig())
-	err := mbtest.WriteEvent(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
-}
-
-func TestFetch(t *testing.T) {
-	compose.EnsureUp(t, "etcd")
-
-	f := mbtest.NewEventFetcher(t, getConfig())
-	event, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	assert.NotNil(t, event)
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+func TestStore(t *testing.T) {
+	runner := compose.TestRunner{Service: "etcd"}
+
+	runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+			event, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			assert.NotNil(t, event)
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+			err := mbtest.WriteEvent(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
 
-func getConfig() map[string]interface{} {
+func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "etcd",
 		"metricsets": []string{"store"},
-		"hosts":      []string{GetEnvHost() + ":" + GetEnvPort()},
-	}
-}
-
-func GetEnvHost() string {
-	host := os.Getenv("ETCD_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
-}
-
-func GetEnvPort() string {
-	port := os.Getenv("ETCD_PORT")
-
-	if len(port) == 0 {
-		port = "2379"
+		"hosts":      []string{host},
 	}
-	return port
 }

From 46bc0897ce9ee40998c39ba92a39ec914e26dc11 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 11 Sep 2018 15:39:36 +0200
Subject: [PATCH 36/88] Run jolokia tests with testrunner

---
 metricbeat/docker-compose.yml                 |  3 +-
 .../jolokia/jmx/jmx_integration_test.go       | 76 +++++++------------
 2 files changed, 31 insertions(+), 48 deletions(-)

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 594aed0129c..2fe633f09b4 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -32,7 +32,6 @@ services:
       args:
         APACHE_VERSION: 2.4.12
 
-
   ceph:
     build: ./module/ceph/_meta
 
@@ -83,6 +82,8 @@ services:
 
   jolokia:
     build: ./module/jolokia/_meta
+    ports:
+      - 8778
 
   kafka:
     extends:
diff --git a/metricbeat/module/jolokia/jmx/jmx_integration_test.go b/metricbeat/module/jolokia/jmx/jmx_integration_test.go
index e66cd516bda..d28146144e6 100644
--- a/metricbeat/module/jolokia/jmx/jmx_integration_test.go
+++ b/metricbeat/module/jolokia/jmx/jmx_integration_test.go
@@ -20,7 +20,6 @@
 package jmx
 
 import (
-	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -29,40 +28,41 @@ import (
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
 )
 
-func TestFetch(t *testing.T) {
-	compose.EnsureUp(t, "jolokia")
+func TestJMX(t *testing.T) {
+	runner := compose.TestRunner{Service: "jolokia"}
 
-	for _, config := range getConfigs() {
-		f := mbtest.NewEventsFetcher(t, config)
-		events, err := f.Fetch()
-		if !assert.NoError(t, err) {
-			t.FailNow()
-		}
-		t.Logf("%s/%s events: %+v", f.Module().Name(), f.Name(), events)
-		if len(events) == 0 || len(events[0]) <= 1 {
-			t.Fatal("Empty events")
-		}
-	}
-}
-
-func TestData(t *testing.T) {
-	compose.EnsureUp(t, "jolokia")
-
-	for _, config := range getConfigs() {
-		f := mbtest.NewEventsFetcher(t, config)
-		err := mbtest.WriteEvents(f, t)
-		if err != nil {
-			t.Fatal("write", err)
-		}
-	}
+	runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			for _, config := range getConfigs(r.Host()) {
+				f := mbtest.NewEventsFetcher(t, config)
+				events, err := f.Fetch()
+				if !assert.NoError(t, err) {
+					t.FailNow()
+				}
+				t.Logf("%s/%s events: %+v", f.Module().Name(), f.Name(), events)
+				if len(events) == 0 || len(events[0]) <= 1 {
+					t.Fatal("Empty events")
+				}
+			}
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			for _, config := range getConfigs(r.Host()) {
+				f := mbtest.NewEventsFetcher(t, config)
+				err := mbtest.WriteEvents(f, t)
+				if err != nil {
+					t.Fatal("write", err)
+				}
+			}
+		},
+	})
 }
 
-func getConfigs() []map[string]interface{} {
+func getConfigs(host string) []map[string]interface{} {
 	return []map[string]interface{}{
 		{
 			"module":     "jolokia",
 			"metricsets": []string{"jmx"},
-			"hosts":      []string{getEnvHost() + ":" + getEnvPort()},
+			"hosts":      []string{host},
 			"namespace":  "testnamespace",
 			"jmx.mappings": []map[string]interface{}{
 				{
@@ -105,7 +105,7 @@ func getConfigs() []map[string]interface{} {
 		{
 			"module":     "jolokia",
 			"metricsets": []string{"jmx"},
-			"hosts":      []string{getEnvHost() + ":" + getEnvPort()},
+			"hosts":      []string{host},
 			"namespace":  "testnamespace",
 			"jmx.mappings": []map[string]interface{}{
 				{
@@ -153,21 +153,3 @@ func getConfigs() []map[string]interface{} {
 		},
 	}
 }
-
-func getEnvHost() string {
-	host := os.Getenv("JOLOKIA_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
-}
-
-func getEnvPort() string {
-	port := os.Getenv("JOLOKIA_PORT")
-
-	if len(port) == 0 {
-		port = "8778"
-	}
-	return port
-}

From 4a036b72a716956353fcdc5b3ba1440f90b5d807 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 11 Sep 2018 15:51:10 +0200
Subject: [PATCH 37/88] Run http tests with testrunner

---
 metricbeat/docker-compose.yml                 |  2 +
 .../module/http/json/json_integration_test.go | 85 +++++++------------
 2 files changed, 34 insertions(+), 53 deletions(-)

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 2fe633f09b4..be11942971f 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -79,6 +79,8 @@ services:
 
   http:
     build: ./module/http/_meta
+    ports:
+      - 8080
 
   jolokia:
     build: ./module/jolokia/_meta
diff --git a/metricbeat/module/http/json/json_integration_test.go b/metricbeat/module/http/json/json_integration_test.go
index cd215316263..1e91c5ded02 100644
--- a/metricbeat/module/http/json/json_integration_test.go
+++ b/metricbeat/module/http/json/json_integration_test.go
@@ -20,7 +20,6 @@
 package json
 
 import (
-	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -29,41 +28,39 @@ import (
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
 )
 
-func TestFetchObject(t *testing.T) {
-	compose.EnsureUp(t, "http")
-
-	f := mbtest.NewEventsFetcher(t, getConfig("object"))
-	event, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-}
-
-func TestFetchArray(t *testing.T) {
-	compose.EnsureUp(t, "http")
-
-	f := mbtest.NewEventsFetcher(t, getConfig("array"))
-	event, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-}
-func TestData(t *testing.T) {
-	compose.EnsureUp(t, "http")
-
-	f := mbtest.NewEventsFetcher(t, getConfig("object"))
-	err := mbtest.WriteEvents(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
-
+func TestJSON(t *testing.T) {
+	runner := compose.TestRunner{Service: "http"}
+
+	runner.Run(t, compose.Suite{
+		"FetchObject": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, getConfig("object", r.Host()))
+			event, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+		},
+		"FetchArray": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, getConfig("array", r.Host()))
+			event, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, getConfig("object", r.Host()))
+			err := mbtest.WriteEvents(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
 
-func getConfig(jsonType string) map[string]interface{} {
+func getConfig(jsonType string, host string) map[string]interface{} {
 	var path string
 	var responseIsArray bool
 	switch jsonType {
@@ -78,27 +75,9 @@ func getConfig(jsonType string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":        "http",
 		"metricsets":    []string{"json"},
-		"hosts":         []string{getEnvHost() + ":" + getEnvPort()},
+		"hosts":         []string{host},
 		"path":          path,
 		"namespace":     "testnamespace",
 		"json.is_array": responseIsArray,
 	}
 }
-
-func getEnvHost() string {
-	host := os.Getenv("HTTP_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
-}
-
-func getEnvPort() string {
-	port := os.Getenv("HTTP_PORT")
-
-	if len(port) == 0 {
-		port = "8080"
-	}
-	return port
-}

From 084e28ea1444da6ad8d948bb8253315a592fea5f Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 11 Sep 2018 16:11:59 +0200
Subject: [PATCH 38/88] Run kibana tests with testrunner

---
 metricbeat/docker-compose.yml                 |  4 ++
 metricbeat/module/kibana/mtest/testing.go     | 30 ++++---------
 .../kibana/stats/stats_integration_test.go    | 43 ++++++++++---------
 .../kibana/status/status_integration_test.go  | 39 ++++++++---------
 4 files changed, 53 insertions(+), 63 deletions(-)

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index be11942971f..53ec26cc66c 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -110,6 +110,10 @@ services:
 
   kibana:
     build: ./module/kibana/_meta
+    depends_on:
+      - elasticsearch
+    ports:
+      - 5601
 
   #kubernetes:
   #  build: ./module/kubernetes/_meta
diff --git a/metricbeat/module/kibana/mtest/testing.go b/metricbeat/module/kibana/mtest/testing.go
index a6dbc1f2f7e..a8f1d191f80 100644
--- a/metricbeat/module/kibana/mtest/testing.go
+++ b/metricbeat/module/kibana/mtest/testing.go
@@ -18,35 +18,21 @@
 package mtest
 
 import (
-	"net"
-	"os"
+	"github.com/elastic/beats/libbeat/tests/compose"
 )
 
-// GetEnvHost returns host for Kibana
-func GetEnvHost() string {
-	host := os.Getenv("KIBANA_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
+var (
+	Runner = compose.TestRunner{
+		Service:  "kibana",
+		Parallel: true,
 	}
-	return host
-}
-
-// GetEnvPort returns port for Kibana
-func GetEnvPort() string {
-	port := os.Getenv("KIBANA_PORT")
-
-	if len(port) == 0 {
-		port = "5601"
-	}
-	return port
-}
+)
 
 // GetConfig returns config for kibana module
-func GetConfig(metricset string) map[string]interface{} {
+func GetConfig(metricset string, host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "kibana",
 		"metricsets": []string{metricset},
-		"hosts":      []string{net.JoinHostPort(GetEnvHost(), GetEnvPort())},
+		"hosts":      []string{host},
 	}
 }
diff --git a/metricbeat/module/kibana/stats/stats_integration_test.go b/metricbeat/module/kibana/stats/stats_integration_test.go
index eb5eb3566c0..f52ffadca7c 100644
--- a/metricbeat/module/kibana/stats/stats_integration_test.go
+++ b/metricbeat/module/kibana/stats/stats_integration_test.go
@@ -33,30 +33,31 @@ import (
 	"github.com/elastic/beats/metricbeat/module/kibana/mtest"
 )
 
-func TestData(t *testing.T) {
-	compose.EnsureUp(t, "kibana")
+func TestStat(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Data": func(t *testing.T, r compose.R) {
+			config := mtest.GetConfig("stats", r.Host())
+			version, err := getKibanaVersion(r.Host())
+			if err != nil {
+				t.Fatal("getting kibana version", err)
+			}
 
-	config := mtest.GetConfig("stats")
-	host := config["hosts"].([]string)[0]
-	version, err := getKibanaVersion(host)
-	if err != nil {
-		t.Fatal("getting kibana version", err)
-	}
-
-	isStatsAPIAvailable, err := kibana.IsStatsAPIAvailable(version)
-	if err != nil {
-		t.Fatal("checking if kibana stats API is available", err)
-	}
+			isStatsAPIAvailable, err := kibana.IsStatsAPIAvailable(version)
+			if err != nil {
+				t.Fatal("checking if kibana stats API is available", err)
+			}
 
-	if !isStatsAPIAvailable {
-		t.Skip("Kibana stats API is not available until 6.4.0")
-	}
+			if !isStatsAPIAvailable {
+				t.Skip("Kibana stats API is not available until 6.4.0")
+			}
 
-	f := mbtest.NewReportingMetricSetV2(t, config)
-	err = mbtest.WriteEventsReporterV2(f, t, "")
-	if err != nil {
-		t.Fatal("write", err)
-	}
+			f := mbtest.NewReportingMetricSetV2(t, config)
+			err = mbtest.WriteEventsReporterV2(f, t, "")
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
 
 func getKibanaVersion(kibanaHostPort string) (string, error) {
diff --git a/metricbeat/module/kibana/status/status_integration_test.go b/metricbeat/module/kibana/status/status_integration_test.go
index 7bf989118cb..34013a4ad53 100644
--- a/metricbeat/module/kibana/status/status_integration_test.go
+++ b/metricbeat/module/kibana/status/status_integration_test.go
@@ -29,24 +29,23 @@ import (
 	"github.com/elastic/beats/metricbeat/module/kibana/mtest"
 )
 
-func TestFetch(t *testing.T) {
-	compose.EnsureUpWithTimeout(t, 600, "elasticsearch", "kibana")
-
-	f := mbtest.NewEventFetcher(t, mtest.GetConfig("status"))
-	event, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-}
-
-func TestData(t *testing.T) {
-	compose.EnsureUp(t, "elasticsearch", "kibana")
-
-	f := mbtest.NewEventFetcher(t, mtest.GetConfig("status"))
-	err := mbtest.WriteEvent(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
+func TestStatus(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, mtest.GetConfig("status", r.Host()))
+			event, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, mtest.GetConfig("status", r.Host()))
+			err := mbtest.WriteEvent(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }

From 9d93da7d6ae248cc863ff2e4bc547823242dab8f Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 11 Sep 2018 16:30:37 +0200
Subject: [PATCH 39/88] Run logstash tests with testrunner

---
 metricbeat/docker-compose.yml                 |  2 +
 .../logstash/node/node_integration_test.go    | 42 +++++++++----------
 .../node_stats/node_stats_integration_test.go | 42 +++++++++----------
 metricbeat/module/logstash/testing.go         | 31 +++++---------
 4 files changed, 54 insertions(+), 63 deletions(-)

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 53ec26cc66c..d8bf286947a 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -135,6 +135,8 @@ services:
 
   logstash:
     build: ./module/logstash/_meta
+    ports:
+      - 9600
 
   memcached:
     build: ./module/memcached/_meta
diff --git a/metricbeat/module/logstash/node/node_integration_test.go b/metricbeat/module/logstash/node/node_integration_test.go
index 6ec042115e1..891020ae46e 100644
--- a/metricbeat/module/logstash/node/node_integration_test.go
+++ b/metricbeat/module/logstash/node/node_integration_test.go
@@ -29,25 +29,25 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func TestFetch(t *testing.T) {
-	compose.EnsureUp(t, "logstash")
-
-	f := mbtest.NewEventFetcher(t, logstash.GetConfig("node"))
-	event, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	assert.NotNil(t, event)
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-}
-
-func TestData(t *testing.T) {
-	compose.EnsureUp(t, "logstash")
-
-	f := mbtest.NewEventFetcher(t, logstash.GetConfig("node"))
-	err := mbtest.WriteEvent(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
+func TestNode(t *testing.T) {
+	logstash.Runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, logstash.GetConfig("node", r.Host()))
+			event, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			assert.NotNil(t, event)
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, logstash.GetConfig("node", r.Host()))
+			err := mbtest.WriteEvent(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+
+		},
+	})
 }
diff --git a/metricbeat/module/logstash/node_stats/node_stats_integration_test.go b/metricbeat/module/logstash/node_stats/node_stats_integration_test.go
index 0883d629c1f..492694e70f1 100644
--- a/metricbeat/module/logstash/node_stats/node_stats_integration_test.go
+++ b/metricbeat/module/logstash/node_stats/node_stats_integration_test.go
@@ -29,25 +29,25 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func TestFetch(t *testing.T) {
-	compose.EnsureUp(t, "logstash")
-
-	f := mbtest.NewEventFetcher(t, logstash.GetConfig("node_stats"))
-	event, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	assert.NotNil(t, event)
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-}
-
-func TestData(t *testing.T) {
-	compose.EnsureUp(t, "logstash")
-
-	f := mbtest.NewEventFetcher(t, logstash.GetConfig("node_stats"))
-	err := mbtest.WriteEvent(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
+func TestNodeStats(t *testing.T) {
+	logstash.Runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, logstash.GetConfig("node_stats", r.Host()))
+			event, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			assert.NotNil(t, event)
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, logstash.GetConfig("node_stats", r.Host()))
+			err := mbtest.WriteEvent(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+
+		},
+	})
 }
diff --git a/metricbeat/module/logstash/testing.go b/metricbeat/module/logstash/testing.go
index 221b280bda0..f262b02bc9a 100644
--- a/metricbeat/module/logstash/testing.go
+++ b/metricbeat/module/logstash/testing.go
@@ -17,33 +17,22 @@
 
 package logstash
 
-import "os"
+import (
+	"github.com/elastic/beats/libbeat/tests/compose"
+)
 
-// GetEnvHost for Logstash
-func GetEnvHost() string {
-	host := os.Getenv("LOGSTASH_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
-}
-
-// GetEnvPort for Logstash
-func GetEnvPort() string {
-	port := os.Getenv("LOGSTASH_PORT")
-
-	if len(port) == 0 {
-		port = "9600"
+var (
+	Runner = compose.TestRunner{
+		Service:  "logstash",
+		Parallel: true,
 	}
-	return port
-}
+)
 
 // GetConfig for Logstash
-func GetConfig(metricset string) map[string]interface{} {
+func GetConfig(metricset string, host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "logstash",
 		"metricsets": []string{metricset},
-		"hosts":      []string{GetEnvHost() + ":" + GetEnvPort()},
+		"hosts":      []string{host},
 	}
 }

From 9368483ad7fce13769b07952908fdfa01f7a6a3f Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 11 Sep 2018 16:50:39 +0200
Subject: [PATCH 40/88] Skip tests using EnsureUp by now

---
 metricbeat/module/memcached/stats/stats_integration_test.go     | 1 +
 .../module/mongodb/collstats/collstats_integration_test.go      | 2 ++
 metricbeat/module/mongodb/dbstats/dbstats_integration_test.go   | 2 ++
 metricbeat/module/mongodb/metrics/metrics_intergration_test.go  | 2 ++
 .../module/mongodb/replstatus/replstatus_integration_test.go    | 2 ++
 metricbeat/module/mongodb/status/status_integration_test.go     | 2 ++
 metricbeat/module/mysql/mysql_integration_test.go               | 1 +
 metricbeat/module/mysql/status/status_integration_test.go       | 2 ++
 .../module/nginx/stubstatus/stubstatus_integration_test.go      | 1 +
 metricbeat/module/php_fpm/pool/pool_integration_test.go         | 1 +
 .../module/postgresql/activity/activity_integration_test.go     | 2 ++
 .../module/postgresql/bgwriter/bgwriter_integration_test.go     | 2 ++
 .../module/postgresql/database/database_integration_test.go     | 2 ++
 .../module/postgresql/statement/statement_integration_test.go   | 2 ++
 .../module/prometheus/collector/collector_integration_test.go   | 2 ++
 metricbeat/module/prometheus/stats/stats_integration_test.go    | 2 ++
 .../module/rabbitmq/connection/connection_integration_test.go   | 1 +
 .../module/rabbitmq/exchange/exchange_integration_test.go       | 1 +
 metricbeat/module/rabbitmq/node/node_integration_test.go        | 1 +
 metricbeat/module/rabbitmq/queue/queue_integration_test.go      | 1 +
 metricbeat/module/traefik/health/health_integration_test.go     | 2 ++
 metricbeat/module/uwsgi/status/status_integration_test.go       | 2 ++
 metricbeat/module/zookeeper/mntr/mntr_integration_test.go       | 2 ++
 23 files changed, 38 insertions(+)

diff --git a/metricbeat/module/memcached/stats/stats_integration_test.go b/metricbeat/module/memcached/stats/stats_integration_test.go
index a195194815a..112033e08d6 100644
--- a/metricbeat/module/memcached/stats/stats_integration_test.go
+++ b/metricbeat/module/memcached/stats/stats_integration_test.go
@@ -28,6 +28,7 @@ import (
 )
 
 func TestData(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "memcached")
 
 	f := mbtest.NewEventFetcher(t, getConfig())
diff --git a/metricbeat/module/mongodb/collstats/collstats_integration_test.go b/metricbeat/module/mongodb/collstats/collstats_integration_test.go
index 0de42bd027f..4f860e98e8e 100644
--- a/metricbeat/module/mongodb/collstats/collstats_integration_test.go
+++ b/metricbeat/module/mongodb/collstats/collstats_integration_test.go
@@ -30,6 +30,7 @@ import (
 )
 
 func TestFetch(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "mongodb")
 
 	f := mbtest.NewEventsFetcher(t, getConfig())
@@ -51,6 +52,7 @@ func TestFetch(t *testing.T) {
 }
 
 func TestData(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "mongodb")
 
 	f := mbtest.NewEventsFetcher(t, getConfig())
diff --git a/metricbeat/module/mongodb/dbstats/dbstats_integration_test.go b/metricbeat/module/mongodb/dbstats/dbstats_integration_test.go
index 512ef1c2b8e..75fc73790ab 100644
--- a/metricbeat/module/mongodb/dbstats/dbstats_integration_test.go
+++ b/metricbeat/module/mongodb/dbstats/dbstats_integration_test.go
@@ -30,6 +30,7 @@ import (
 )
 
 func TestFetch(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "mongodb")
 
 	f := mbtest.NewEventsFetcher(t, getConfig())
@@ -76,6 +77,7 @@ func TestFetch(t *testing.T) {
 }
 
 func TestData(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "mongodb")
 
 	f := mbtest.NewEventsFetcher(t, getConfig())
diff --git a/metricbeat/module/mongodb/metrics/metrics_intergration_test.go b/metricbeat/module/mongodb/metrics/metrics_intergration_test.go
index 1347cd02874..f39b950846f 100644
--- a/metricbeat/module/mongodb/metrics/metrics_intergration_test.go
+++ b/metricbeat/module/mongodb/metrics/metrics_intergration_test.go
@@ -30,6 +30,7 @@ import (
 )
 
 func TestFetch(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "mongodb")
 
 	f := mbtest.NewEventFetcher(t, getConfig())
@@ -49,6 +50,7 @@ func TestFetch(t *testing.T) {
 }
 
 func TestData(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "mongodb")
 
 	f := mbtest.NewEventFetcher(t, getConfig())
diff --git a/metricbeat/module/mongodb/replstatus/replstatus_integration_test.go b/metricbeat/module/mongodb/replstatus/replstatus_integration_test.go
index 40e60bca51b..526a81aba8f 100644
--- a/metricbeat/module/mongodb/replstatus/replstatus_integration_test.go
+++ b/metricbeat/module/mongodb/replstatus/replstatus_integration_test.go
@@ -34,6 +34,7 @@ import (
 )
 
 func TestFetch(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "mongodb")
 
 	err := initiateReplicaSet(t)
@@ -73,6 +74,7 @@ func TestFetch(t *testing.T) {
 }
 
 func TestData(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "mongodb")
 
 	f := mbtest.NewEventFetcher(t, getConfig())
diff --git a/metricbeat/module/mongodb/status/status_integration_test.go b/metricbeat/module/mongodb/status/status_integration_test.go
index 4b3969979d9..2140b0c181b 100644
--- a/metricbeat/module/mongodb/status/status_integration_test.go
+++ b/metricbeat/module/mongodb/status/status_integration_test.go
@@ -31,6 +31,7 @@ import (
 )
 
 func TestFetch(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "mongodb")
 
 	f := mbtest.NewEventFetcher(t, getConfig())
@@ -53,6 +54,7 @@ func TestFetch(t *testing.T) {
 }
 
 func TestData(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "mongodb")
 
 	f := mbtest.NewEventFetcher(t, getConfig())
diff --git a/metricbeat/module/mysql/mysql_integration_test.go b/metricbeat/module/mysql/mysql_integration_test.go
index 772fd91ae80..32b90a519b6 100644
--- a/metricbeat/module/mysql/mysql_integration_test.go
+++ b/metricbeat/module/mysql/mysql_integration_test.go
@@ -29,6 +29,7 @@ import (
 )
 
 func TestNewDB(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "mysql")
 
 	db, err := NewDB(GetMySQLEnvDSN())
diff --git a/metricbeat/module/mysql/status/status_integration_test.go b/metricbeat/module/mysql/status/status_integration_test.go
index 7ccc0a77ef5..e0a4898de35 100644
--- a/metricbeat/module/mysql/status/status_integration_test.go
+++ b/metricbeat/module/mysql/status/status_integration_test.go
@@ -31,6 +31,7 @@ import (
 )
 
 func TestFetch(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "mysql")
 
 	f := mbtest.NewEventFetcher(t, getConfig(false))
@@ -55,6 +56,7 @@ func TestFetch(t *testing.T) {
 }
 
 func TestFetchRaw(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "mysql")
 
 	f := mbtest.NewEventFetcher(t, getConfig(true))
diff --git a/metricbeat/module/nginx/stubstatus/stubstatus_integration_test.go b/metricbeat/module/nginx/stubstatus/stubstatus_integration_test.go
index d4b12887003..779e400f02b 100644
--- a/metricbeat/module/nginx/stubstatus/stubstatus_integration_test.go
+++ b/metricbeat/module/nginx/stubstatus/stubstatus_integration_test.go
@@ -30,6 +30,7 @@ import (
 )
 
 func TestFetch(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "nginx")
 
 	f := mbtest.NewEventFetcher(t, getConfig())
diff --git a/metricbeat/module/php_fpm/pool/pool_integration_test.go b/metricbeat/module/php_fpm/pool/pool_integration_test.go
index aaac9336929..f779ad41f0c 100644
--- a/metricbeat/module/php_fpm/pool/pool_integration_test.go
+++ b/metricbeat/module/php_fpm/pool/pool_integration_test.go
@@ -28,6 +28,7 @@ import (
 )
 
 func TestData(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "phpfpm")
 
 	f := mbtest.NewEventFetcher(t, getConfig())
diff --git a/metricbeat/module/postgresql/activity/activity_integration_test.go b/metricbeat/module/postgresql/activity/activity_integration_test.go
index d4467d1b546..0bc64f1de54 100644
--- a/metricbeat/module/postgresql/activity/activity_integration_test.go
+++ b/metricbeat/module/postgresql/activity/activity_integration_test.go
@@ -31,6 +31,7 @@ import (
 )
 
 func TestFetch(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "postgresql")
 
 	f := mbtest.NewEventsFetcher(t, getConfig())
@@ -58,6 +59,7 @@ func TestFetch(t *testing.T) {
 }
 
 func TestData(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "postgresql")
 	f := mbtest.NewEventsFetcher(t, getConfig())
 
diff --git a/metricbeat/module/postgresql/bgwriter/bgwriter_integration_test.go b/metricbeat/module/postgresql/bgwriter/bgwriter_integration_test.go
index 72b6bed4f60..04203eee0c5 100644
--- a/metricbeat/module/postgresql/bgwriter/bgwriter_integration_test.go
+++ b/metricbeat/module/postgresql/bgwriter/bgwriter_integration_test.go
@@ -31,6 +31,7 @@ import (
 )
 
 func TestFetch(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "postgresql")
 
 	f := mbtest.NewEventFetcher(t, getConfig())
@@ -60,6 +61,7 @@ func TestFetch(t *testing.T) {
 }
 
 func TestData(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "postgresql")
 
 	f := mbtest.NewEventFetcher(t, getConfig())
diff --git a/metricbeat/module/postgresql/database/database_integration_test.go b/metricbeat/module/postgresql/database/database_integration_test.go
index 50287dce897..e09dd707727 100644
--- a/metricbeat/module/postgresql/database/database_integration_test.go
+++ b/metricbeat/module/postgresql/database/database_integration_test.go
@@ -31,6 +31,7 @@ import (
 )
 
 func TestFetch(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "postgresql")
 
 	f := mbtest.NewEventsFetcher(t, getConfig())
@@ -60,6 +61,7 @@ func TestFetch(t *testing.T) {
 }
 
 func TestData(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "postgresql")
 
 	f := mbtest.NewEventsFetcher(t, getConfig())
diff --git a/metricbeat/module/postgresql/statement/statement_integration_test.go b/metricbeat/module/postgresql/statement/statement_integration_test.go
index 58938df7c6b..b3449ed5c96 100644
--- a/metricbeat/module/postgresql/statement/statement_integration_test.go
+++ b/metricbeat/module/postgresql/statement/statement_integration_test.go
@@ -31,6 +31,7 @@ import (
 )
 
 func TestFetch(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "postgresql")
 
 	f := mbtest.NewEventsFetcher(t, getConfig())
@@ -91,6 +92,7 @@ func TestFetch(t *testing.T) {
 }
 
 func TestData(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "postgresql")
 	f := mbtest.NewEventsFetcher(t, getConfig())
 
diff --git a/metricbeat/module/prometheus/collector/collector_integration_test.go b/metricbeat/module/prometheus/collector/collector_integration_test.go
index c09ac37f27a..d2f186e9d06 100644
--- a/metricbeat/module/prometheus/collector/collector_integration_test.go
+++ b/metricbeat/module/prometheus/collector/collector_integration_test.go
@@ -33,6 +33,7 @@ import (
 // Every prometheus exporter should work here.
 
 func TestFetch(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "prometheus")
 
 	f := mbtest.NewEventsFetcher(t, getConfig())
@@ -45,6 +46,7 @@ func TestFetch(t *testing.T) {
 }
 
 func TestData(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "prometheus")
 
 	f := mbtest.NewEventsFetcher(t, getConfig())
diff --git a/metricbeat/module/prometheus/stats/stats_integration_test.go b/metricbeat/module/prometheus/stats/stats_integration_test.go
index 13154a351b6..45e62466ec0 100644
--- a/metricbeat/module/prometheus/stats/stats_integration_test.go
+++ b/metricbeat/module/prometheus/stats/stats_integration_test.go
@@ -31,6 +31,7 @@ import (
 )
 
 func TestFetch(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "prometheus")
 
 	f := mbtest.NewEventFetcher(t, getConfig())
@@ -47,6 +48,7 @@ func TestFetch(t *testing.T) {
 }
 
 func TestData(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "prometheus")
 
 	f := mbtest.NewEventFetcher(t, getConfig())
diff --git a/metricbeat/module/rabbitmq/connection/connection_integration_test.go b/metricbeat/module/rabbitmq/connection/connection_integration_test.go
index 26597e7c36d..ae0f7a93868 100644
--- a/metricbeat/module/rabbitmq/connection/connection_integration_test.go
+++ b/metricbeat/module/rabbitmq/connection/connection_integration_test.go
@@ -28,6 +28,7 @@ import (
 )
 
 func TestData(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "rabbitmq")
 
 	f := mbtest.NewEventsFetcher(t, getConfig())
diff --git a/metricbeat/module/rabbitmq/exchange/exchange_integration_test.go b/metricbeat/module/rabbitmq/exchange/exchange_integration_test.go
index 0903a7bb035..b8094935b8c 100644
--- a/metricbeat/module/rabbitmq/exchange/exchange_integration_test.go
+++ b/metricbeat/module/rabbitmq/exchange/exchange_integration_test.go
@@ -29,6 +29,7 @@ import (
 )
 
 func TestData(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "rabbitmq")
 
 	f := mbtest.NewEventsFetcher(t, getConfig())
diff --git a/metricbeat/module/rabbitmq/node/node_integration_test.go b/metricbeat/module/rabbitmq/node/node_integration_test.go
index 48c05f1d696..938ac90024e 100644
--- a/metricbeat/module/rabbitmq/node/node_integration_test.go
+++ b/metricbeat/module/rabbitmq/node/node_integration_test.go
@@ -28,6 +28,7 @@ import (
 )
 
 func TestData(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "rabbitmq")
 
 	ms := mbtest.NewReportingMetricSetV2(t, getConfig())
diff --git a/metricbeat/module/rabbitmq/queue/queue_integration_test.go b/metricbeat/module/rabbitmq/queue/queue_integration_test.go
index ccff728cf2f..9b348ac0a37 100644
--- a/metricbeat/module/rabbitmq/queue/queue_integration_test.go
+++ b/metricbeat/module/rabbitmq/queue/queue_integration_test.go
@@ -29,6 +29,7 @@ import (
 )
 
 func TestData(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "rabbitmq")
 
 	f := mbtest.NewEventsFetcher(t, getConfig())
diff --git a/metricbeat/module/traefik/health/health_integration_test.go b/metricbeat/module/traefik/health/health_integration_test.go
index 388921bdd3d..8a49f755fa1 100644
--- a/metricbeat/module/traefik/health/health_integration_test.go
+++ b/metricbeat/module/traefik/health/health_integration_test.go
@@ -42,6 +42,7 @@ func makeBadRequest(config map[string]interface{}) error {
 }
 
 func TestFetch(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "traefik")
 
 	config := mtest.GetConfig("health")
@@ -66,6 +67,7 @@ func TestFetch(t *testing.T) {
 }
 
 func TestData(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "traefik")
 
 	ms := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("health"))
diff --git a/metricbeat/module/uwsgi/status/status_integration_test.go b/metricbeat/module/uwsgi/status/status_integration_test.go
index 5ac48b5e4a2..248b4612f47 100644
--- a/metricbeat/module/uwsgi/status/status_integration_test.go
+++ b/metricbeat/module/uwsgi/status/status_integration_test.go
@@ -30,6 +30,7 @@ import (
 )
 
 func TestFetchTCP(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "uwsgi_tcp")
 
 	f := mbtest.NewEventsFetcher(t, getConfig("tcp"))
@@ -42,6 +43,7 @@ func TestFetchTCP(t *testing.T) {
 }
 
 func TestFetchHTTP(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "uwsgi_http")
 
 	f := mbtest.NewEventsFetcher(t, getConfig("http"))
diff --git a/metricbeat/module/zookeeper/mntr/mntr_integration_test.go b/metricbeat/module/zookeeper/mntr/mntr_integration_test.go
index 7828c4e6e42..2f1624358c0 100644
--- a/metricbeat/module/zookeeper/mntr/mntr_integration_test.go
+++ b/metricbeat/module/zookeeper/mntr/mntr_integration_test.go
@@ -31,6 +31,7 @@ import (
 )
 
 func TestFetch(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "zookeeper")
 
 	f := mbtest.NewEventFetcher(t, getConfig())
@@ -57,6 +58,7 @@ func TestFetch(t *testing.T) {
 }
 
 func TestData(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "zookeeper")
 
 	f := mbtest.NewEventFetcher(t, getConfig())

From 26e33a6bd4fb5759d46102a0e4380d0ac2c00921 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 11 Sep 2018 18:25:02 +0200
Subject: [PATCH 41/88] Fix environment setting

---
 libbeat/tests/compose/runner.go  | 1 -
 libbeat/tests/compose/wrapper.go | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/libbeat/tests/compose/runner.go b/libbeat/tests/compose/runner.go
index 4484e9b616e..39a678043e3 100644
--- a/libbeat/tests/compose/runner.go
+++ b/libbeat/tests/compose/runner.go
@@ -116,7 +116,6 @@ func (r *TestRunner) Run(t *testing.T, tests Suite) {
 	for _, s := range scenarios {
 		var vars []string
 		for k, v := range s {
-			os.Setenv(k, v)
 			vars = append(vars, fmt.Sprintf("%s=%s", k, v))
 		}
 		sort.Strings(vars)
diff --git a/libbeat/tests/compose/wrapper.go b/libbeat/tests/compose/wrapper.go
index 119b68280c3..7a5997f6259 100644
--- a/libbeat/tests/compose/wrapper.go
+++ b/libbeat/tests/compose/wrapper.go
@@ -117,7 +117,7 @@ func (d *wrapperDriver) cmd(ctx context.Context, command string, arg ...string)
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 	if len(d.Environment) > 0 {
-		cmd.Env = append(d.Environment, os.Environ()...)
+		cmd.Env = append(os.Environ(), d.Environment...)
 	}
 	return cmd
 }

From fb972953670bb490e32a3d31d2b69cde87c524b6 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 11 Sep 2018 18:21:47 +0200
Subject: [PATCH 42/88] Test elasticsearch on multiple versions

---
 metricbeat/docker-compose.yml                 | 12 +++---------
 .../module/elasticsearch/_meta/Dockerfile     |  2 --
 .../module/elasticsearch/docker-compose.yml   | 19 +++++++++++++++++++
 .../elasticsearch_integration_test.go         | 13 ++++++++++++-
 4 files changed, 34 insertions(+), 12 deletions(-)
 delete mode 100644 metricbeat/module/elasticsearch/_meta/Dockerfile
 create mode 100644 metricbeat/module/elasticsearch/docker-compose.yml

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index d8bf286947a..ce2e79c05c2 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -44,15 +44,9 @@ services:
       - 8080
 
   elasticsearch:
-    build: ./module/elasticsearch/_meta
-    environment:
-      - "ES_JAVA_OPTS=-Xms90m -Xmx90m"
-      - "network.host="
-      - "transport.host=127.0.0.1"
-      - "http.host=0.0.0.0"
-      - "xpack.security.enabled=false"
-    ports:
-      - 9200
+    extends:
+      file: ./module/elasticsearch/docker-compose.yml
+      service: elasticsearch
 
   envoyproxy:
     build: ./module/envoyproxy/_meta
diff --git a/metricbeat/module/elasticsearch/_meta/Dockerfile b/metricbeat/module/elasticsearch/_meta/Dockerfile
deleted file mode 100644
index 4de23503710..00000000000
--- a/metricbeat/module/elasticsearch/_meta/Dockerfile
+++ /dev/null
@@ -1,2 +0,0 @@
-FROM docker.elastic.co/elasticsearch/elasticsearch:6.3.0
-HEALTHCHECK --interval=1s --retries=300 CMD curl -f http://localhost:9200
diff --git a/metricbeat/module/elasticsearch/docker-compose.yml b/metricbeat/module/elasticsearch/docker-compose.yml
new file mode 100644
index 00000000000..91828821c33
--- /dev/null
+++ b/metricbeat/module/elasticsearch/docker-compose.yml
@@ -0,0 +1,19 @@
+version: "2.1"
+
+services:
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION:-6.3.0}
+    environment:
+      - "ES_JAVA_OPTS=-Xms90m -Xmx90m"
+      - "network.host="
+      - "transport.host=127.0.0.1"
+      - "http.host=0.0.0.0"
+      - "xpack.security.enabled=false"
+    ports:
+      - 9200
+    healthcheck:
+      test:
+      - "CMD-SHELL"
+      - "curl -f http://localhost:9200"
+      interval: 1s
+      retries: 300
diff --git a/metricbeat/module/elasticsearch/elasticsearch_integration_test.go b/metricbeat/module/elasticsearch/elasticsearch_integration_test.go
index 2345cd5f07f..00bc0b3db86 100644
--- a/metricbeat/module/elasticsearch/elasticsearch_integration_test.go
+++ b/metricbeat/module/elasticsearch/elasticsearch_integration_test.go
@@ -54,7 +54,18 @@ var metricSets = []string{
 }
 
 func TestElasticsearch(t *testing.T) {
-	runner := compose.TestRunner{Service: "elasticsearch"}
+	runner := compose.TestRunner{
+		Service: "elasticsearch",
+		Options: compose.RunnerOptions{
+			"ELASTICSEARCH_VERSION": {
+				"6.4.0",
+				"6.3.2",
+				// "6.2.4",
+				"5.6.11",
+			},
+		},
+		Parallel: true,
+	}
 
 	runner.Run(t, compose.Suite{
 		"Fetch": func(t *testing.T, r compose.R) {

From ba2cac40ff14abe37ad4db64b8914ab6c158277a Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 11 Sep 2018 18:37:58 +0200
Subject: [PATCH 43/88] Expose options to tests

---
 libbeat/tests/compose/runner.go                          | 9 +++++++++
 .../elasticsearch/elasticsearch_integration_test.go      | 6 +++++-
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/libbeat/tests/compose/runner.go b/libbeat/tests/compose/runner.go
index 39a678043e3..0ce16f3e185 100644
--- a/libbeat/tests/compose/runner.go
+++ b/libbeat/tests/compose/runner.go
@@ -114,6 +114,7 @@ func (r *TestRunner) Run(t *testing.T, tests Suite) {
 		t.Fatal("Test runner configuration doesn't produce scenarios")
 	}
 	for _, s := range scenarios {
+		s := s
 		var vars []string
 		for k, v := range s {
 			vars = append(vars, fmt.Sprintf("%s=%s", k, v))
@@ -167,6 +168,10 @@ func (r *TestRunner) Run(t *testing.T, tests Suite) {
 
 type R interface {
 	Host() string
+	Hostname() string
+	Port() string
+
+	Option(string) string
 }
 
 type runnerControl struct {
@@ -188,3 +193,7 @@ func (r *runnerControl) Port() string {
 	_, port, _ := net.SplitHostPort(r.host)
 	return port
 }
+
+func (r *runnerControl) Option(key string) string {
+	return r.scenario[key]
+}
diff --git a/metricbeat/module/elasticsearch/elasticsearch_integration_test.go b/metricbeat/module/elasticsearch/elasticsearch_integration_test.go
index 00bc0b3db86..2126db2283a 100644
--- a/metricbeat/module/elasticsearch/elasticsearch_integration_test.go
+++ b/metricbeat/module/elasticsearch/elasticsearch_integration_test.go
@@ -60,7 +60,7 @@ func TestElasticsearch(t *testing.T) {
 			"ELASTICSEARCH_VERSION": {
 				"6.4.0",
 				"6.3.2",
-				// "6.2.4",
+				"6.2.4",
 				"5.6.11",
 			},
 		},
@@ -69,6 +69,10 @@ func TestElasticsearch(t *testing.T) {
 
 	runner.Run(t, compose.Suite{
 		"Fetch": func(t *testing.T, r compose.R) {
+			if r.Option("ELASTICSEARCH_VERSION") == "6.2.4" {
+				t.Skip("This test fails on this version")
+			}
+
 			host := r.Host()
 			err := createIndex(host)
 			assert.NoError(t, err)

From 44a8772a8800bb829065ef5e3fc0036ff225ffdb Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Wed, 12 Sep 2018 11:35:30 +0200
Subject: [PATCH 44/88] Fix kafka python tests

---
 metricbeat/docker-compose.yml            | 3 +++
 metricbeat/module/kafka/_meta/Dockerfile | 1 -
 metricbeat/module/kafka/_meta/run.sh     | 5 +++++
 3 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index ce2e79c05c2..10c4e8d0780 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -85,6 +85,9 @@ services:
     extends:
       file: ./module/kafka/docker-compose.yml
       service: kafka
+    environment:
+      # Needed for python tests
+      KAFKA_ADVERTISED_HOST_AUTO: "yes"
 
   kafka_1_1_0:
     extends:
diff --git a/metricbeat/module/kafka/_meta/Dockerfile b/metricbeat/module/kafka/_meta/Dockerfile
index aab10dbed24..b16e4a4594e 100644
--- a/metricbeat/module/kafka/_meta/Dockerfile
+++ b/metricbeat/module/kafka/_meta/Dockerfile
@@ -3,7 +3,6 @@ FROM debian:stretch
 ARG KAFKA_VERSION=2.0.0
 
 ENV KAFKA_HOME /kafka
-# The advertised host is kafka. This means it will not work if container is started locally and connected from localhost to it
 ENV KAFKA_LOGS_DIR="/kafka-logs"
 ENV _JAVA_OPTIONS "-Djava.net.preferIPv4Stack=true"
 ENV TERM=linux
diff --git a/metricbeat/module/kafka/_meta/run.sh b/metricbeat/module/kafka/_meta/run.sh
index 696b9507b4b..23f085e8c64 100755
--- a/metricbeat/module/kafka/_meta/run.sh
+++ b/metricbeat/module/kafka/_meta/run.sh
@@ -1,5 +1,10 @@
 #!/bin/bash
 
+if [ -n "$KAFKA_ADVERTISED_HOST_AUTO" ]; then
+	ip=$(nslookup $HOSTNAME 2> /dev/null | grep ^Address | cut -d' ' -f3)
+	KAFKA_ADVERTISED_HOST=${ip}:9092
+fi
+
 # Check if KAFKA_ADVERTISED_HOST is set
 # if not wait to read it from file
 if [ -z "$KAFKA_ADVERTISED_HOST" ]; then

From 6881d7bee66834a8a1af37caf233d899bbedb935 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Wed, 12 Sep 2018 12:24:12 +0200
Subject: [PATCH 45/88] Use alpine jre image for kafka

---
 metricbeat/module/kafka/_meta/Dockerfile | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/metricbeat/module/kafka/_meta/Dockerfile b/metricbeat/module/kafka/_meta/Dockerfile
index b16e4a4594e..3be204a8545 100644
--- a/metricbeat/module/kafka/_meta/Dockerfile
+++ b/metricbeat/module/kafka/_meta/Dockerfile
@@ -1,4 +1,4 @@
-FROM debian:stretch
+FROM openjdk:8-jre-alpine3.8
 
 ARG KAFKA_VERSION=2.0.0
 
@@ -7,11 +7,12 @@ ENV KAFKA_LOGS_DIR="/kafka-logs"
 ENV _JAVA_OPTIONS "-Djava.net.preferIPv4Stack=true"
 ENV TERM=linux
 
-RUN apt-get update && apt-get install -y curl openjdk-8-jre-headless netcat dnsutils
+RUN apk add -u bash
 
-RUN mkdir -p ${KAFKA_LOGS_DIR} && mkdir -p ${KAFKA_HOME} && curl -s -o $INSTALL_DIR/kafka.tgz \
-    "http://ftp.wayne.edu/apache/kafka/${KAFKA_VERSION}/kafka_2.11-${KAFKA_VERSION}.tgz" && \
-    tar xzf ${INSTALL_DIR}/kafka.tgz -C ${KAFKA_HOME} --strip-components 1
+RUN mkdir -p ${KAFKA_LOGS_DIR} && mkdir -p ${KAFKA_HOME} && \
+    wget -q -O $INSTALL_DIR/kafka.tgz "http://ftp.wayne.edu/apache/kafka/${KAFKA_VERSION}/kafka_2.11-${KAFKA_VERSION}.tgz" && \
+    tar xzf ${INSTALL_DIR}/kafka.tgz -C ${KAFKA_HOME} --strip-components 1 && \
+    rm -f ${INSTALL_DIR}/kafka.tgz
 
 ADD run.sh /run.sh
 ADD healthcheck.sh /healthcheck.sh

From eb3c9fe5be413fb794cc3fd96d0d5d61dfccb888 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Wed, 12 Sep 2018 12:43:47 +0200
Subject: [PATCH 46/88] Actually test kafka with multiple versions, and reuse
 images

---
 metricbeat/docker-compose.yml              | 8 ++++++++
 metricbeat/module/kafka/docker-compose.yml | 3 ++-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 10c4e8d0780..d38de2ad12a 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -93,6 +93,10 @@ services:
     extends:
       file: ./module/kafka/docker-compose.yml
       service: kafka
+    environment:
+      # Needed for python tests
+      KAFKA_ADVERTISED_HOST_AUTO: "yes"
+    image: metricbeat-kafka:1.1.0
     build:
       args:
         KAFKA_VERSION: 1.1.0
@@ -101,6 +105,10 @@ services:
     extends:
       file: ./module/kafka/docker-compose.yml
       service: kafka
+    environment:
+      # Needed for python tests
+      KAFKA_ADVERTISED_HOST_AUTO: "yes"
+    image: metricbeat-kafka:0.10.2.1
     build:
       args:
         KAFKA_VERSION: 0.10.2.1
diff --git a/metricbeat/module/kafka/docker-compose.yml b/metricbeat/module/kafka/docker-compose.yml
index abd3e8f0aed..b28bf74daee 100644
--- a/metricbeat/module/kafka/docker-compose.yml
+++ b/metricbeat/module/kafka/docker-compose.yml
@@ -2,9 +2,10 @@ version: "2.1"
 
 services:
   kafka:
+    image: metricbeat-kafka:${KAFKA_VERSION:-2.0.0}
     build:
       context: ./_meta
       args:
-        KAFKA_VERSION: 2.0.0
+        KAFKA_VERSION: ${KAFKA_VERSION:-2.0.0}
     ports:
     - 9092

From 22367be99abedcd44be1b4f1339d6c3014934c78 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Wed, 12 Sep 2018 15:46:09 +0200
Subject: [PATCH 47/88] Fix golang python test

---
 metricbeat/tests/system/test_golang.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/metricbeat/tests/system/test_golang.py b/metricbeat/tests/system/test_golang.py
index a053a2403fa..fe07f28f5db 100644
--- a/metricbeat/tests/system/test_golang.py
+++ b/metricbeat/tests/system/test_golang.py
@@ -11,7 +11,7 @@ class Test(metricbeat.BaseTest):
     @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test")
     def test_stats(self):
         """
-        prometheus stats test
+        golang heap test
         """
         self.render_config_template(modules=[{
             "name": "golang",
@@ -43,5 +43,5 @@ def test_stats(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return ["http://" + os.getenv('GOLANG_HOST', self.compose_hosts()[0]) + ':' +
+        return ["http://" + os.getenv('GOLANG_HOST', 'localhost') + ':' +
                 os.getenv('GOLANG_PORT', '6060')]

From b3f09e0451109343d5d0829d83d6759ae554e447 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Wed, 12 Sep 2018 16:11:40 +0200
Subject: [PATCH 48/88] Fix kibana python test

---
 metricbeat/tests/system/test_kibana.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/metricbeat/tests/system/test_kibana.py b/metricbeat/tests/system/test_kibana.py
index f3143819884..456588201ae 100644
--- a/metricbeat/tests/system/test_kibana.py
+++ b/metricbeat/tests/system/test_kibana.py
@@ -48,7 +48,7 @@ def test_status(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return [os.getenv('KIBANA_HOST', self.compose_hosts()[1]) + ':' +
+        return [os.getenv('KIBANA_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('KIBANA_PORT', '5601')]
 
     def get_version(self):

From 3fcd7b82ce874be7be0f2f812604a9a953a84191 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Wed, 12 Sep 2018 16:37:06 +0200
Subject: [PATCH 49/88] Fix test_base python test

---
 metricbeat/tests/system/test_base.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/metricbeat/tests/system/test_base.py b/metricbeat/tests/system/test_base.py
index 361d51a466e..770ea0ef716 100644
--- a/metricbeat/tests/system/test_base.py
+++ b/metricbeat/tests/system/test_base.py
@@ -76,3 +76,18 @@ def test_dashboards(self):
 
         assert exit_code == 0
         assert self.log_contains("Kibana dashboards successfully loaded.")
+
+    def get_elasticsearch_url(self):
+        return "http://{host}:{port}".format(
+            host=os.getenv("ES_HOST", self.compose_hosts()[1]),
+            port=os.getenv("ES_PORT", "9200"),
+        )
+
+    def get_kibana_url(self):
+        """
+        Returns kibana host URL
+        """
+        return "http://{host}:{port}".format(
+            host=os.getenv("KIBANA_HOST", self.compose_hosts()[0]),
+            port=os.getenv("KIBANA_PORT", "5601"),
+        )

From fcfb66354fd2269137d6e038882e166811c9d756 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Wed, 12 Sep 2018 18:45:47 +0200
Subject: [PATCH 50/88] Fix docker python test

---
 metricbeat/tests/system/test_docker.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/metricbeat/tests/system/test_docker.py b/metricbeat/tests/system/test_docker.py
index df0e615e874..616e7627604 100644
--- a/metricbeat/tests/system/test_docker.py
+++ b/metricbeat/tests/system/test_docker.py
@@ -6,6 +6,7 @@
 
 
 class Test(metricbeat.BaseTest):
+    COMPOSE_SERVICES = ["redis"]  # Just to have a container running
 
     @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test")
     def test_container_fields(self):

From f0f995430d965b5485807bdd113f7aa5f1da927c Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Thu, 13 Sep 2018 16:45:03 +0200
Subject: [PATCH 51/88] Generate data in kafka tests before starting consumer

---
 .../consumergroup_integration_test.go         | 12 +---
 metricbeat/module/kafka/mtest/helpers.go      | 72 +++++++++++++++++++
 .../partition/partition_integration_test.go   | 45 +-----------
 3 files changed, 78 insertions(+), 51 deletions(-)
 create mode 100644 metricbeat/module/kafka/mtest/helpers.go

diff --git a/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go b/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go
index 8a6541057c7..e5a3e1f20b7 100644
--- a/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go
+++ b/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go
@@ -20,11 +20,9 @@
 package consumergroup
 
 import (
-	"io"
 	"testing"
 	"time"
 
-	saramacluster "github.com/bsm/sarama-cluster"
 	"github.com/pkg/errors"
 
 	"github.com/elastic/beats/libbeat/tests/compose"
@@ -34,7 +32,9 @@ import (
 
 func TestConsumerGroup(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{"Data": func(t *testing.T, r compose.R) {
-		c, err := startConsumer(t, "metricbeat-test", r.Host())
+		topic := "metricbeat-test"
+		mtest.GenerateKafkaData(t, topic, r.Host())
+		c, err := mtest.StartConsumer(t, topic, r.Host())
 		if err != nil {
 			t.Fatal(errors.Wrap(err, "starting kafka consumer"))
 		}
@@ -52,12 +52,6 @@ func TestConsumerGroup(t *testing.T) {
 	}})
 }
 
-func startConsumer(t *testing.T, topic, host string) (io.Closer, error) {
-	brokers := []string{host}
-	topics := []string{topic}
-	return saramacluster.NewConsumer(brokers, "test-group", topics, nil)
-}
-
 func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "kafka",
diff --git a/metricbeat/module/kafka/mtest/helpers.go b/metricbeat/module/kafka/mtest/helpers.go
new file mode 100644
index 00000000000..024784a210e
--- /dev/null
+++ b/metricbeat/module/kafka/mtest/helpers.go
@@ -0,0 +1,72 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package mtest
+
+import (
+	"io"
+	"testing"
+	"time"
+
+	"github.com/Shopify/sarama"
+	saramacluster "github.com/bsm/sarama-cluster"
+)
+
+func GenerateKafkaData(t *testing.T, topic string, host string) {
+	t.Logf("Send Kafka Event to topic: %v", topic)
+
+	config := sarama.NewConfig()
+	config.Producer.Return.Successes = true
+	// Retry for 10 seconds
+	config.Producer.Retry.Max = 20
+	config.Producer.Retry.Backoff = 500 * time.Millisecond
+	config.Metadata.Retry.Max = 20
+	config.Metadata.Retry.Backoff = 500 * time.Millisecond
+	client, err := sarama.NewClient([]string{host}, config)
+	if err != nil {
+		t.Errorf("%s", err)
+		t.FailNow()
+	}
+
+	producer, err := sarama.NewSyncProducerFromClient(client)
+	if err != nil {
+		t.Error(err)
+	}
+	defer producer.Close()
+
+	msg := &sarama.ProducerMessage{
+		Topic: topic,
+		Value: sarama.StringEncoder("Hello World"),
+	}
+
+	_, _, err = producer.SendMessage(msg)
+	if err != nil {
+		t.Errorf("failed to send message: %s\n", err)
+	}
+
+	err = client.RefreshMetadata(topic)
+	if err != nil {
+		t.Errorf("failed to refresh metadata for topic '%s': %s\n", topic, err)
+	}
+}
+
+func StartConsumer(t *testing.T, topic, host string) (io.Closer, error) {
+	brokers := []string{host}
+	topics := []string{topic}
+
+	return saramacluster.NewConsumer(brokers, "test-group", topics, nil)
+}
diff --git a/metricbeat/module/kafka/partition/partition_integration_test.go b/metricbeat/module/kafka/partition/partition_integration_test.go
index 3b0c8c507fa..1ef103a208f 100644
--- a/metricbeat/module/kafka/partition/partition_integration_test.go
+++ b/metricbeat/module/kafka/partition/partition_integration_test.go
@@ -26,7 +26,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/Shopify/sarama"
 	"github.com/stretchr/testify/assert"
 
 	"github.com/elastic/beats/libbeat/common"
@@ -43,7 +42,7 @@ func TestPartition(t *testing.T) {
 
 	mtest.Runner.Run(t, compose.Suite{
 		"Data": func(t *testing.T, r compose.R) {
-			generateKafkaData(t, "metricbeat-generate-data", r.Host())
+			mtest.GenerateKafkaData(t, "metricbeat-generate-data", r.Host())
 
 			ms := mbtest.NewReportingMetricSetV2(t, getConfig("", r.Host()))
 			err := mbtest.WriteEventsReporterV2(ms, t, "")
@@ -56,7 +55,7 @@ func TestPartition(t *testing.T) {
 			testTopic := fmt.Sprintf("test-metricbeat-%s", id)
 
 			// Create initial topic
-			generateKafkaData(t, testTopic, r.Host())
+			mtest.GenerateKafkaData(t, testTopic, r.Host())
 
 			f := mbtest.NewReportingMetricSetV2(t, getConfig(testTopic, r.Host()))
 			dataBefore, err := mbtest.ReportingFetchV2(f)
@@ -71,7 +70,7 @@ func TestPartition(t *testing.T) {
 			var n int64 = 10
 			// Create n messages
 			for i := int64(0); i < n; i++ {
-				generateKafkaData(t, testTopic, r.Host())
+				mtest.GenerateKafkaData(t, testTopic, r.Host())
 			}
 
 			dataAfter, err := mbtest.ReportingFetchV2(f)
@@ -111,44 +110,6 @@ func TestPartition(t *testing.T) {
 		}})
 }
 
-func generateKafkaData(t *testing.T, topic string, host string) {
-	t.Logf("Send Kafka Event to topic: %v", topic)
-
-	config := sarama.NewConfig()
-	config.Producer.Return.Successes = true
-	// Retry for 10 seconds
-	config.Producer.Retry.Max = 20
-	config.Producer.Retry.Backoff = 500 * time.Millisecond
-	config.Metadata.Retry.Max = 20
-	config.Metadata.Retry.Backoff = 500 * time.Millisecond
-	client, err := sarama.NewClient([]string{host}, config)
-	if err != nil {
-		t.Errorf("%s", err)
-		t.FailNow()
-	}
-
-	producer, err := sarama.NewSyncProducerFromClient(client)
-	if err != nil {
-		t.Error(err)
-	}
-	defer producer.Close()
-
-	msg := &sarama.ProducerMessage{
-		Topic: topic,
-		Value: sarama.StringEncoder("Hello World"),
-	}
-
-	_, _, err = producer.SendMessage(msg)
-	if err != nil {
-		t.Errorf("failed to send message: %s\n", err)
-	}
-
-	err = client.RefreshMetadata(topic)
-	if err != nil {
-		t.Errorf("failed to refresh metadata for topic '%s': %s\n", topic, err)
-	}
-}
-
 func getConfig(topic string, host string) map[string]interface{} {
 	var topics []string
 	if topic != "" {

From 6c0f666db46791b9bfac634b26e02c0cd6a479b5 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Thu, 13 Sep 2018 18:41:57 +0200
Subject: [PATCH 52/88] Reduce number of kafka partitions in tests

---
 metricbeat/module/kafka/_meta/run.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/metricbeat/module/kafka/_meta/run.sh b/metricbeat/module/kafka/_meta/run.sh
index 23f085e8c64..0d845b8cb53 100755
--- a/metricbeat/module/kafka/_meta/run.sh
+++ b/metricbeat/module/kafka/_meta/run.sh
@@ -47,6 +47,8 @@ ${KAFKA_HOME}/bin/kafka-server-start.sh ${KAFKA_HOME}/config/server.properties \
     --override advertised.listeners=INSIDE://localhost:9091,OUTSIDE://$KAFKA_ADVERTISED_HOST \
     --override listener.security.protocol.map=INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT \
     --override inter.broker.listener.name=INSIDE \
+    --override offsets.topic.replication.factor=1 \
+    --override offsets.topic.num.partitions=2 \
     --override logs.dir=${KAFKA_LOGS_DIR} &
 
 wait_for_port 9092

From c14c93eebe1f0cf19ef730546851afd89acc7b3a Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Thu, 13 Sep 2018 20:33:11 +0200
Subject: [PATCH 53/88] Don't prebuild docker images

---
 libbeat/scripts/Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile
index 2c8d370242e..1d307cb38ff 100755
--- a/libbeat/scripts/Makefile
+++ b/libbeat/scripts/Makefile
@@ -371,14 +371,14 @@ import-dashboards: update ${BEAT_NAME}
 # Builds the environment to test beat
 .PHONY: build-image
 build-image: write-environment
-	${DOCKER_COMPOSE} build ${DOCKER_NOCACHE} --pull --force-rm
+	# ${DOCKER_COMPOSE} build ${DOCKER_NOCACHE} --pull --force-rm
 
 # Runs the environment so the redis and elasticsearch can also be used for local development
 # To use it for running the test, set ES_HOST and REDIS_HOST environment variable to the ip of your docker-machine.
 .PHONY: start-environment
 start-environment: stop-environment
 	${DOCKER_COMPOSE} pull --include-deps
-	${DOCKER_COMPOSE} up -d
+	${DOCKER_COMPOSE} up --build -d
 
 .PHONY: stop-environment
 stop-environment:

From 39d3b1bd5db6208795929d4bce89af54c89640a5 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Wed, 12 Sep 2018 15:34:10 +0200
Subject: [PATCH 54/88] Fix uwsgi python test

---
 metricbeat/docker-compose.yml            |  6 +++-
 metricbeat/module/uwsgi/_meta/Dockerfile |  5 +---
 metricbeat/tests/system/test_uwsgi.py    | 36 +++++++++---------------
 3 files changed, 19 insertions(+), 28 deletions(-)

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index d38de2ad12a..d2353db248e 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -194,10 +194,14 @@ services:
   uwsgi_tcp:
     build: ./module/uwsgi/_meta
     command: uwsgi --http :8080 --master --processes 1 --threads 2 --stats 0.0.0.0:9191 --memory-report --wsgi-file app.py
+    ports:
+      - 9191
 
   uwsgi_http:
     build: ./module/uwsgi/_meta
-    command: uwsgi --http :8080 --master --processes 1 --threads 2 --stats 0.0.0.0:9192 --memory-report --stats-http --wsgi-file app.py
+    command: uwsgi --http :8080 --master --processes 1 --threads 2 --stats 0.0.0.0:9191 --memory-report --stats-http --wsgi-file app.py
+    ports:
+      - 9191
 
   zookeeper:
     build: ./module/zookeeper/_meta
diff --git a/metricbeat/module/uwsgi/_meta/Dockerfile b/metricbeat/module/uwsgi/_meta/Dockerfile
index 81d8372ec12..17fc9571bc9 100644
--- a/metricbeat/module/uwsgi/_meta/Dockerfile
+++ b/metricbeat/module/uwsgi/_meta/Dockerfile
@@ -6,7 +6,4 @@ RUN pip install --no-cache-dir --trusted-host pypi.python.org uwsgi
 WORKDIR /app
 COPY testdata/app /app
 
-HEALTHCHECK --interval=1s --retries=60 --timeout=10s CMD curl http://localhost:8080/
-EXPOSE 8080 9191 9192
-
-CMD [""]
+HEALTHCHECK --interval=1s --retries=60 --timeout=10s CMD curl http://localhost:8080
diff --git a/metricbeat/tests/system/test_uwsgi.py b/metricbeat/tests/system/test_uwsgi.py
index 3707d2cab86..7c4bfd79dde 100644
--- a/metricbeat/tests/system/test_uwsgi.py
+++ b/metricbeat/tests/system/test_uwsgi.py
@@ -8,8 +8,7 @@
 
 
 class Test(metricbeat.BaseTest):
-
-    COMPOSE_SERVICES = ['uwsgi_tcp', "uwsgi_http"]
+    COMPOSE_SERVICES = ['uwsgi_tcp']
 
     def common_checks(self, output):
         # Ensure no errors or warnings exist in the log.
@@ -51,15 +50,14 @@ def common_checks(self, output):
 
     @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test")
     @attr('integration')
-    def test_status_tcp(self):
+    def test_status(self):
         """
         uWSGI module outputs an event.
         """
-        hosts = [os.getenv("UWSGI_STAT_TCP_SERVER")]
         self.render_config_template(modules=[{
             "name": "uwsgi",
             "metricsets": ["status"],
-            "hosts": hosts,
+            "hosts": [self.get_host()],
             "period": "5s"
         }])
         proc = self.start_beat()
@@ -69,22 +67,14 @@ def test_status_tcp(self):
         output = self.read_output_json()
         self.common_checks(output)
 
-    @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test")
-    @attr('integration')
-    def test_status_http(self):
-        """
-        uWSGI module outputs an event.
-        """
-        hosts = [os.getenv("UWSGI_STAT_HTTP_SERVER")]
-        self.render_config_template(modules=[{
-            "name": "uwsgi",
-            "metricsets": ["status"],
-            "hosts": hosts,
-            "period": "5s"
-        }])
-        proc = self.start_beat()
-        self.wait_until(lambda: self.output_lines() > 0)
-        proc.check_kill_and_wait()
+    def get_host(self):
+        return os.getenv("UWSGI_STAT_TCP_SERVER",
+                         "tcp://{}:9191".format(self.compose_hosts()[0]))
 
-        output = self.read_output_json()
-        self.common_checks(output)
+
+class TestHTTP(Test):
+    COMPOSE_SERVICES = ['uwsgi_http']
+
+    def get_host(self):
+        return os.getenv("UWSGI_STAT_HTTP_SERVER",
+                         "http://{}:9191".format(self.compose_hosts()[0]))

From ab3a368c898384ac93398e895a2c620b97b75872 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Sun, 28 Oct 2018 17:09:04 +0100
Subject: [PATCH 55/88] Run zookeeper tests with testrunner

---
 metricbeat/docker-compose.yml                 |  2 +
 .../zookeeper/mntr/mntr_integration_test.go   | 65 +++++++++----------
 metricbeat/module/zookeeper/testing.go        | 48 --------------
 3 files changed, 34 insertions(+), 81 deletions(-)
 delete mode 100644 metricbeat/module/zookeeper/testing.go

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index d2353db248e..e25330a02cd 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -205,3 +205,5 @@ services:
 
   zookeeper:
     build: ./module/zookeeper/_meta
+    ports:
+      - 2181
diff --git a/metricbeat/module/zookeeper/mntr/mntr_integration_test.go b/metricbeat/module/zookeeper/mntr/mntr_integration_test.go
index 2f1624358c0..d73aa21b5fb 100644
--- a/metricbeat/module/zookeeper/mntr/mntr_integration_test.go
+++ b/metricbeat/module/zookeeper/mntr/mntr_integration_test.go
@@ -27,52 +27,51 @@ import (
 	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	"github.com/elastic/beats/metricbeat/module/zookeeper"
 )
 
-func TestFetch(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "zookeeper")
+func TestMntr(t *testing.T) {
+	runner := compose.TestRunner{Service: "zookeeper", Parallel: true}
 
-	f := mbtest.NewEventFetcher(t, getConfig())
-	event, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+	runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+			event, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
 
-	// Check values
-	version := event["version"].(string)
-	avgLatency := event["latency"].(common.MapStr)["avg"].(int64)
-	maxLatency := event["latency"].(common.MapStr)["max"].(int64)
-	numAliveConnections := event["num_alive_connections"].(int64)
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
 
-	assert.Equal(t, version, "3.4.8--1, built on 02/06/2016 03:18 GMT")
-	assert.True(t, avgLatency >= 0)
-	assert.True(t, maxLatency >= 0)
-	assert.True(t, numAliveConnections > 0)
+			// Check values
+			version := event["version"].(string)
+			avgLatency := event["latency"].(common.MapStr)["avg"].(int64)
+			maxLatency := event["latency"].(common.MapStr)["max"].(int64)
+			numAliveConnections := event["num_alive_connections"].(int64)
 
-	// Check number of fields. At least 10, depending on environment
-	assert.True(t, len(event) >= 10)
-}
+			assert.Equal(t, version, "3.4.8--1, built on 02/06/2016 03:18 GMT")
+			assert.True(t, avgLatency >= 0)
+			assert.True(t, maxLatency >= 0)
+			assert.True(t, numAliveConnections > 0)
 
-func TestData(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "zookeeper")
+			// Check number of fields. At least 10, depending on environment
+			assert.True(t, len(event) >= 10)
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
 
-	f := mbtest.NewEventFetcher(t, getConfig())
+			err := mbtest.WriteEvent(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 
-	err := mbtest.WriteEvent(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
 }
 
-func getConfig() map[string]interface{} {
+func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "zookeeper",
 		"metricsets": []string{"mntr"},
-		"hosts":      []string{zookeeper.GetZookeeperEnvHost() + ":" + zookeeper.GetZookeeperEnvPort()},
+		"hosts":      []string{host},
 	}
 }
diff --git a/metricbeat/module/zookeeper/testing.go b/metricbeat/module/zookeeper/testing.go
deleted file mode 100644
index afa3cac8398..00000000000
--- a/metricbeat/module/zookeeper/testing.go
+++ /dev/null
@@ -1,48 +0,0 @@
-// Licensed to Elasticsearch B.V. under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Elasticsearch B.V. licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package zookeeper
-
-import (
-	"os"
-)
-
-// Helper functions for testing used in the zookeeper MetricSets.
-
-// GetZookeeperEnvHost returns the hostname of the ZooKeeper server to use for
-// testing. It reads the value from the ZOOKEEPER_HOST environment variable and
-// returns localhost if it is not set.
-func GetZookeeperEnvHost() string {
-	host := os.Getenv("ZOOKEEPER_HOST")
-
-	if len(host) == 0 {
-		host = "localhost"
-	}
-	return host
-}
-
-// GetZookeeperEnvPort returns the port of the ZooKeeper server to use for
-// testing. It reads the value from the ZOOKEEPER_PORT environment variable and
-// returns 2181 if it is not set.
-func GetZookeeperEnvPort() string {
-	port := os.Getenv("ZOOKEEPER_PORT")
-
-	if len(port) == 0 {
-		port = "2181"
-	}
-	return port
-}

From e1d587c36f083034dfa64bb59093add9be743668 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Sun, 28 Oct 2018 17:17:03 +0100
Subject: [PATCH 56/88] Run memcached tests with testrunner

---
 metricbeat/docker-compose.yml                 |  2 +
 .../memcached/stats/stats_integration_test.go | 44 ++++++-------------
 2 files changed, 16 insertions(+), 30 deletions(-)

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index e25330a02cd..c8a5aed7e03 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -145,6 +145,8 @@ services:
 
   memcached:
     build: ./module/memcached/_meta
+    ports:
+      - 11211
 
   mongodb:
     build: ./module/mongodb/_meta
diff --git a/metricbeat/module/memcached/stats/stats_integration_test.go b/metricbeat/module/memcached/stats/stats_integration_test.go
index 112033e08d6..1672d600cb0 100644
--- a/metricbeat/module/memcached/stats/stats_integration_test.go
+++ b/metricbeat/module/memcached/stats/stats_integration_test.go
@@ -20,46 +20,30 @@
 package stats
 
 import (
-	"os"
 	"testing"
 
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
 )
 
-func TestData(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "memcached")
-
-	f := mbtest.NewEventFetcher(t, getConfig())
-	err := mbtest.WriteEvent(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
+func TestStats(t *testing.T) {
+	runner := compose.TestRunner{Service: "memcached"}
+
+	runner.Run(t, compose.Suite{
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+			err := mbtest.WriteEvent(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
 
-func getConfig() map[string]interface{} {
+func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "memcached",
 		"metricsets": []string{"stats"},
-		"hosts":      []string{getEnvHost() + ":" + getEnvPort()},
-	}
-}
-
-func getEnvHost() string {
-	host := os.Getenv("MEMCACHED_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
-}
-
-func getEnvPort() string {
-	port := os.Getenv("MEMCACHED_PORT")
-
-	if len(port) == 0 {
-		port = "11211"
+		"hosts":      []string{host},
 	}
-	return port
 }

From 267a56180f371859cc94d3b95151d3edc105d8d1 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Sun, 28 Oct 2018 17:34:09 +0100
Subject: [PATCH 57/88] Run uwsgi tests with testrunner

---
 .../uwsgi/status/status_integration_test.go   | 58 ++++++++++---------
 metricbeat/module/uwsgi/testing.go            | 38 ------------
 2 files changed, 31 insertions(+), 65 deletions(-)
 delete mode 100644 metricbeat/module/uwsgi/testing.go

diff --git a/metricbeat/module/uwsgi/status/status_integration_test.go b/metricbeat/module/uwsgi/status/status_integration_test.go
index 248b4612f47..b76380c4b95 100644
--- a/metricbeat/module/uwsgi/status/status_integration_test.go
+++ b/metricbeat/module/uwsgi/status/status_integration_test.go
@@ -24,50 +24,54 @@ import (
 
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	"github.com/elastic/beats/metricbeat/module/uwsgi"
 
 	"github.com/stretchr/testify/assert"
 )
 
-func TestFetchTCP(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "uwsgi_tcp")
+func testStatus(t *testing.T, service string) {
+	runner := compose.TestRunner{Service: service}
 
-	f := mbtest.NewEventsFetcher(t, getConfig("tcp"))
-	events, err := f.Fetch()
-	assert.NoError(t, err)
+	runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, getConfig(t, service, r.Host()))
+			events, err := f.Fetch()
+			assert.NoError(t, err)
 
-	assert.True(t, len(events) > 0)
-	totals := findItems(events, "total")
-	assert.Equal(t, 1, len(totals))
+			assert.True(t, len(events) > 0)
+			totals := findItems(events, "total")
+			assert.Equal(t, 1, len(totals))
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, getConfig(t, service, r.Host()))
+			err := mbtest.WriteEvents(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
 
-func TestFetchHTTP(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "uwsgi_http")
-
-	f := mbtest.NewEventsFetcher(t, getConfig("http"))
-	events, err := f.Fetch()
-	assert.NoError(t, err)
+func TestStatusTCP(t *testing.T) {
+	testStatus(t, "uwsgi_tcp")
+}
 
-	assert.True(t, len(events) > 0)
-	totals := findItems(events, "total")
-	assert.Equal(t, 1, len(totals))
+func TestStatusHTTP(t *testing.T) {
+	testStatus(t, "uwsgi_http")
 }
 
-func getConfig(scheme string) map[string]interface{} {
+func getConfig(t *testing.T, service string, host string) map[string]interface{} {
 	conf := map[string]interface{}{
 		"module":     "uwsgi",
 		"metricsets": []string{"status"},
 	}
 
-	switch scheme {
-	case "tcp":
-		conf["hosts"] = []string{uwsgi.GetEnvTCPServer()}
-	case "http", "https":
-		conf["hosts"] = []string{uwsgi.GetEnvHTTPServer()}
+	switch service {
+	case "uwsgi_tcp":
+		conf["hosts"] = []string{"tcp://" + host}
+	case "uwsgi_http":
+		conf["hosts"] = []string{"http://" + host}
 	default:
-		conf["hosts"] = []string{uwsgi.GetEnvTCPServer()}
+		t.Errorf("Unexpected service: %s", service)
 	}
 	return conf
 }
diff --git a/metricbeat/module/uwsgi/testing.go b/metricbeat/module/uwsgi/testing.go
deleted file mode 100644
index b8c465354e6..00000000000
--- a/metricbeat/module/uwsgi/testing.go
+++ /dev/null
@@ -1,38 +0,0 @@
-// Licensed to Elasticsearch B.V. under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Elasticsearch B.V. licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package uwsgi
-
-import "os"
-
-// GetEnvTCPServer returns uwsgi stat server host with tcp mode
-func GetEnvTCPServer() string {
-	env := os.Getenv("UWSGI_STAT_TCP_SERVER")
-	if len(env) == 0 {
-		env = "tcp://127.0.0.1:9191"
-	}
-	return env
-}
-
-// GetEnvHTTPServer returns uwsgi stat server host with http mode
-func GetEnvHTTPServer() string {
-	env := os.Getenv("UWSGI_STAT_HTTP_SERVER")
-	if len(env) == 0 {
-		env = "http://127.0.0.1:9192"
-	}
-	return env
-}

From 513976bc7902b605133349df9d8a77b809ea4a67 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Fri, 9 Nov 2018 22:19:33 +0100
Subject: [PATCH 58/88] Run traefik tests with testrunner

---
 metricbeat/docker-compose.yml                 |  2 +
 .../traefik/health/health_integration_test.go | 69 +++++++++----------
 metricbeat/module/traefik/mtest/runner.go     | 29 ++++++++
 metricbeat/module/traefik/mtest/testing.go    | 26 +------
 4 files changed, 66 insertions(+), 60 deletions(-)
 create mode 100644 metricbeat/module/traefik/mtest/runner.go

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index c8a5aed7e03..0fa903e3826 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -192,6 +192,8 @@ services:
 
   traefik:
     build: ./module/traefik/_meta
+    ports:
+     - 8080
 
   uwsgi_tcp:
     build: ./module/uwsgi/_meta
diff --git a/metricbeat/module/traefik/health/health_integration_test.go b/metricbeat/module/traefik/health/health_integration_test.go
index 8a49f755fa1..c7fcabc4bad 100644
--- a/metricbeat/module/traefik/health/health_integration_test.go
+++ b/metricbeat/module/traefik/health/health_integration_test.go
@@ -23,11 +23,11 @@ import (
 	"net/http"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
 	"github.com/elastic/beats/metricbeat/module/traefik/mtest"
-
-	"github.com/stretchr/testify/assert"
 )
 
 func makeBadRequest(config map[string]interface{}) error {
@@ -41,38 +41,35 @@ func makeBadRequest(config map[string]interface{}) error {
 	return nil
 }
 
-func TestFetch(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "traefik")
-
-	config := mtest.GetConfig("health")
-
-	makeBadRequest(config)
-
-	ms := mbtest.NewReportingMetricSetV2(t, config)
-	reporter := &mbtest.CapturingReporterV2{}
-
-	ms.Fetch(reporter)
-	assert.Nil(t, reporter.GetErrors(), "Errors while fetching metrics")
-
-	event := reporter.GetEvents()[0]
-	assert.NotNil(t, event)
-	t.Logf("%s/%s event: %+v", ms.Module().Name(), ms.Name(), event)
-
-	responseCount, _ := event.MetricSetFields.GetValue("response.count")
-	assert.True(t, responseCount.(int64) >= 1)
-
-	badResponseCount, _ := event.MetricSetFields.GetValue("response.status_codes.404")
-	assert.True(t, badResponseCount.(float64) >= 1)
-}
-
-func TestData(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "traefik")
-
-	ms := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("health"))
-	err := mbtest.WriteEventsReporterV2(ms, t, "")
-	if err != nil {
-		t.Fatal("write", err)
-	}
+func TestHealth(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			config := mtest.GetConfig("health", r.Host())
+
+			makeBadRequest(config)
+
+			ms := mbtest.NewReportingMetricSetV2(t, config)
+			reporter := &mbtest.CapturingReporterV2{}
+
+			ms.Fetch(reporter)
+			assert.Nil(t, reporter.GetErrors(), "Errors while fetching metrics")
+
+			event := reporter.GetEvents()[0]
+			assert.NotNil(t, event)
+			t.Logf("%s/%s event: %+v", ms.Module().Name(), ms.Name(), event)
+
+			responseCount, _ := event.MetricSetFields.GetValue("response.count")
+			assert.True(t, responseCount.(int64) >= 1)
+
+			badResponseCount, _ := event.MetricSetFields.GetValue("response.status_codes.404")
+			assert.True(t, badResponseCount.(float64) >= 1)
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			ms := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("health", r.Host()))
+			err := mbtest.WriteEventsReporterV2(ms, t, "")
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
diff --git a/metricbeat/module/traefik/mtest/runner.go b/metricbeat/module/traefik/mtest/runner.go
new file mode 100644
index 00000000000..10ce0edf162
--- /dev/null
+++ b/metricbeat/module/traefik/mtest/runner.go
@@ -0,0 +1,29 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package mtest
+
+import (
+	"github.com/elastic/beats/libbeat/tests/compose"
+)
+
+var (
+	Runner = compose.TestRunner{
+		Service:  "traefik",
+		Parallel: true,
+	}
+)
diff --git a/metricbeat/module/traefik/mtest/testing.go b/metricbeat/module/traefik/mtest/testing.go
index c71b9accb01..938f2ee7d4e 100644
--- a/metricbeat/module/traefik/mtest/testing.go
+++ b/metricbeat/module/traefik/mtest/testing.go
@@ -17,33 +17,11 @@
 
 package mtest
 
-import "os"
-
-// GetEnvHost for Traefik
-func GetEnvHost() string {
-	host := os.Getenv("TRAEFIK_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
-}
-
-// GetEnvAPIPort for Traefik
-func GetEnvAPIPort() string {
-	port := os.Getenv("TRAEFIK_API_PORT")
-
-	if len(port) == 0 {
-		port = "8080"
-	}
-	return port
-}
-
 // GetConfig for Traefik
-func GetConfig(metricset string) map[string]interface{} {
+func GetConfig(metricset string, host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "traefik",
 		"metricsets": []string{metricset},
-		"hosts":      []string{GetEnvHost() + ":" + GetEnvAPIPort()},
+		"hosts":      []string{host},
 	}
 }

From 8d3ed55185b524135c9e4e2355e41f36595981d2 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Sat, 10 Nov 2018 13:55:53 +0100
Subject: [PATCH 59/88] Run rabbitmq tests with testrunner

---
 metricbeat/docker-compose.yml                 |  2 ++
 .../connection/connection_integration_test.go | 23 ++++++++-------
 .../exchange/exchange_integration_test.go     | 29 ++++++++++---------
 .../module/rabbitmq/mtest/integration.go      | 14 ++-------
 metricbeat/module/rabbitmq/mtest/runner.go    | 29 +++++++++++++++++++
 .../rabbitmq/node/node_integration_test.go    | 23 ++++++++-------
 .../rabbitmq/queue/queue_integration_test.go  | 27 ++++++++---------
 7 files changed, 86 insertions(+), 61 deletions(-)
 create mode 100644 metricbeat/module/rabbitmq/mtest/runner.go

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 0fa903e3826..69c66ee6f52 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -172,6 +172,8 @@ services:
 
   rabbitmq:
     build: ./module/rabbitmq/_meta
+    ports:
+      - 15672
 
   redis:
     extends:
diff --git a/metricbeat/module/rabbitmq/connection/connection_integration_test.go b/metricbeat/module/rabbitmq/connection/connection_integration_test.go
index ae0f7a93868..420e65d5cce 100644
--- a/metricbeat/module/rabbitmq/connection/connection_integration_test.go
+++ b/metricbeat/module/rabbitmq/connection/connection_integration_test.go
@@ -27,19 +27,20 @@ import (
 	"github.com/elastic/beats/metricbeat/module/rabbitmq/mtest"
 )
 
-func TestData(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "rabbitmq")
-
-	f := mbtest.NewEventsFetcher(t, getConfig())
-	err := mbtest.WriteEvents(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
+func TestConnection(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
+			err := mbtest.WriteEvents(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
 
-func getConfig() map[string]interface{} {
-	config := mtest.GetIntegrationConfig()
+func getConfig(host string) map[string]interface{} {
+	config := mtest.GetIntegrationConfig(host)
 	config["metricsets"] = []string{"connection"}
 	return config
 }
diff --git a/metricbeat/module/rabbitmq/exchange/exchange_integration_test.go b/metricbeat/module/rabbitmq/exchange/exchange_integration_test.go
index b8094935b8c..da20d0e7c47 100644
--- a/metricbeat/module/rabbitmq/exchange/exchange_integration_test.go
+++ b/metricbeat/module/rabbitmq/exchange/exchange_integration_test.go
@@ -28,23 +28,24 @@ import (
 	"github.com/elastic/beats/metricbeat/module/rabbitmq/mtest"
 )
 
-func TestData(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "rabbitmq")
-
-	f := mbtest.NewEventsFetcher(t, getConfig())
-	err := mbtest.WriteEventsCond(f, t, func(e common.MapStr) bool {
-		hasIn, _ := e.HasKey("messages.publish_in")
-		hasOut, _ := e.HasKey("messages.publish_out")
-		return hasIn && hasOut
+func TestExchange(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
+			err := mbtest.WriteEventsCond(f, t, func(e common.MapStr) bool {
+				hasIn, _ := e.HasKey("messages.publish_in")
+				hasOut, _ := e.HasKey("messages.publish_out")
+				return hasIn && hasOut
+			})
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
 	})
-	if err != nil {
-		t.Fatal("write", err)
-	}
 }
 
-func getConfig() map[string]interface{} {
-	config := mtest.GetIntegrationConfig()
+func getConfig(host string) map[string]interface{} {
+	config := mtest.GetIntegrationConfig(host)
 	config["metricsets"] = []string{"exchange"}
 	return config
 }
diff --git a/metricbeat/module/rabbitmq/mtest/integration.go b/metricbeat/module/rabbitmq/mtest/integration.go
index 3160c0cf259..cba6493391b 100644
--- a/metricbeat/module/rabbitmq/mtest/integration.go
+++ b/metricbeat/module/rabbitmq/mtest/integration.go
@@ -18,35 +18,25 @@
 package mtest
 
 import (
-	"net"
 	"os"
 )
 
 // GetIntegrationConfig generates a base configuration with common values for
 // integration tests
-func GetIntegrationConfig() map[string]interface{} {
+func GetIntegrationConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":   "rabbitmq",
-		"hosts":    getTestRabbitMQHost(),
+		"hosts":    []string{host},
 		"username": getTestRabbitMQUsername(),
 		"password": getTestRabbitMQPassword(),
 	}
 }
 
 const (
-	rabbitmqDefaultHost     = "localhost"
-	rabbitmqDefaultPort     = "15672"
 	rabbitmqDefaultUsername = "guest"
 	rabbitmqDefaultPassword = "guest"
 )
 
-func getTestRabbitMQHost() string {
-	return net.JoinHostPort(
-		getenv("RABBITMQ_HOST", rabbitmqDefaultHost),
-		getenv("RABBITMQ_PORT", rabbitmqDefaultPort),
-	)
-}
-
 func getTestRabbitMQUsername() string {
 	return getenv("RABBITMQ_USERNAME", rabbitmqDefaultUsername)
 }
diff --git a/metricbeat/module/rabbitmq/mtest/runner.go b/metricbeat/module/rabbitmq/mtest/runner.go
new file mode 100644
index 00000000000..a70257d78b9
--- /dev/null
+++ b/metricbeat/module/rabbitmq/mtest/runner.go
@@ -0,0 +1,29 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package mtest
+
+import (
+	"github.com/elastic/beats/libbeat/tests/compose"
+)
+
+var (
+	Runner = compose.TestRunner{
+		Service:  "rabbitmq",
+		Parallel: true,
+	}
+)
diff --git a/metricbeat/module/rabbitmq/node/node_integration_test.go b/metricbeat/module/rabbitmq/node/node_integration_test.go
index 938ac90024e..ca0ae772b53 100644
--- a/metricbeat/module/rabbitmq/node/node_integration_test.go
+++ b/metricbeat/module/rabbitmq/node/node_integration_test.go
@@ -27,19 +27,20 @@ import (
 	"github.com/elastic/beats/metricbeat/module/rabbitmq/mtest"
 )
 
-func TestData(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "rabbitmq")
-
-	ms := mbtest.NewReportingMetricSetV2(t, getConfig())
-	err := mbtest.WriteEventsReporterV2(ms, t, "")
-	if err != nil {
-		t.Fatal("write", err)
-	}
+func TestNode(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Data": func(t *testing.T, r compose.R) {
+			ms := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host()))
+			err := mbtest.WriteEventsReporterV2(ms, t, "")
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
 
-func getConfig() map[string]interface{} {
-	config := mtest.GetIntegrationConfig()
+func getConfig(host string) map[string]interface{} {
+	config := mtest.GetIntegrationConfig(host)
 	config["metricsets"] = []string{"node"}
 	return config
 }
diff --git a/metricbeat/module/rabbitmq/queue/queue_integration_test.go b/metricbeat/module/rabbitmq/queue/queue_integration_test.go
index 9b348ac0a37..7f84b903fe4 100644
--- a/metricbeat/module/rabbitmq/queue/queue_integration_test.go
+++ b/metricbeat/module/rabbitmq/queue/queue_integration_test.go
@@ -28,22 +28,23 @@ import (
 	"github.com/elastic/beats/metricbeat/module/rabbitmq/mtest"
 )
 
-func TestData(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "rabbitmq")
-
-	f := mbtest.NewEventsFetcher(t, getConfig())
-	err := mbtest.WriteEventsCond(f, t, func(e common.MapStr) bool {
-		hasTotal, _ := e.HasKey("messages.total")
-		return hasTotal
+func TestQueue(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
+			err := mbtest.WriteEventsCond(f, t, func(e common.MapStr) bool {
+				hasTotal, _ := e.HasKey("messages.total")
+				return hasTotal
+			})
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
 	})
-	if err != nil {
-		t.Fatal("write", err)
-	}
 }
 
-func getConfig() map[string]interface{} {
-	config := mtest.GetIntegrationConfig()
+func getConfig(host string) map[string]interface{} {
+	config := mtest.GetIntegrationConfig(host)
 	config["metricsets"] = []string{"queue"}
 	return config
 }

From 23d4e7fb81435cdd3800a6a62e3c043306c0d66c Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Sat, 10 Nov 2018 14:37:45 +0100
Subject: [PATCH 60/88] Run postgresql tests with testrunner

---
 metricbeat/docker-compose.yml                 |   2 +
 .../activity/activity_integration_test.go     |  72 ++++-----
 .../bgwriter/bgwriter_integration_test.go     |  79 ++++-----
 .../database/database_integration_test.go     |  80 ++++------
 .../{testing.go => mtest/runner.go}           |  23 ++-
 metricbeat/module/postgresql/mtest/testing.go |  49 ++++++
 .../statement/statement_integration_test.go   | 150 ++++++++----------
 7 files changed, 226 insertions(+), 229 deletions(-)
 rename metricbeat/module/postgresql/{testing.go => mtest/runner.go} (76%)
 create mode 100644 metricbeat/module/postgresql/mtest/testing.go

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 69c66ee6f52..59a4555679d 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -166,6 +166,8 @@ services:
 
   postgresql:
     build: ./module/postgresql/_meta
+    ports:
+      - 5432
 
   prometheus:
     build: ./module/prometheus/_meta
diff --git a/metricbeat/module/postgresql/activity/activity_integration_test.go b/metricbeat/module/postgresql/activity/activity_integration_test.go
index 0bc64f1de54..c48b399a818 100644
--- a/metricbeat/module/postgresql/activity/activity_integration_test.go
+++ b/metricbeat/module/postgresql/activity/activity_integration_test.go
@@ -25,56 +25,44 @@ import (
 	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	"github.com/elastic/beats/metricbeat/module/postgresql"
+	"github.com/elastic/beats/metricbeat/module/postgresql/mtest"
 
 	"github.com/stretchr/testify/assert"
 )
 
-func TestFetch(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "postgresql")
+func TestActivity(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, mtest.GetConfig("activity", r.Host()))
+			events, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
 
-	f := mbtest.NewEventsFetcher(t, getConfig())
-	events, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
+			assert.True(t, len(events) > 0)
+			event := events[0]
 
-	assert.True(t, len(events) > 0)
-	event := events[0]
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
 
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+			// Check event fields
+			assert.Contains(t, event, "database")
+			db_oid := event["database"].(common.MapStr)["oid"].(int64)
+			assert.True(t, db_oid > 0)
 
-	// Check event fields
-	assert.Contains(t, event, "database")
-	db_oid := event["database"].(common.MapStr)["oid"].(int64)
-	assert.True(t, db_oid > 0)
+			assert.Contains(t, event, "pid")
+			assert.True(t, event["pid"].(int64) > 0)
 
-	assert.Contains(t, event, "pid")
-	assert.True(t, event["pid"].(int64) > 0)
+			assert.Contains(t, event, "user")
+			assert.Contains(t, event["user"].(common.MapStr), "name")
+			assert.Contains(t, event["user"].(common.MapStr), "id")
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, mtest.GetConfig("activity", r.Host()))
 
-	assert.Contains(t, event, "user")
-	assert.Contains(t, event["user"].(common.MapStr), "name")
-	assert.Contains(t, event["user"].(common.MapStr), "id")
-}
-
-func TestData(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "postgresql")
-	f := mbtest.NewEventsFetcher(t, getConfig())
-
-	err := mbtest.WriteEvents(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
-}
-
-func getConfig() map[string]interface{} {
-	return map[string]interface{}{
-		"module":     "postgresql",
-		"metricsets": []string{"activity"},
-		"hosts":      []string{postgresql.GetEnvDSN()},
-		"username":   postgresql.GetEnvUsername(),
-		"password":   postgresql.GetEnvPassword(),
-	}
+			err := mbtest.WriteEvents(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
diff --git a/metricbeat/module/postgresql/bgwriter/bgwriter_integration_test.go b/metricbeat/module/postgresql/bgwriter/bgwriter_integration_test.go
index 04203eee0c5..3562d21d3f0 100644
--- a/metricbeat/module/postgresql/bgwriter/bgwriter_integration_test.go
+++ b/metricbeat/module/postgresql/bgwriter/bgwriter_integration_test.go
@@ -25,59 +25,46 @@ import (
 	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	"github.com/elastic/beats/metricbeat/module/postgresql"
+	"github.com/elastic/beats/metricbeat/module/postgresql/mtest"
 
 	"github.com/stretchr/testify/assert"
 )
 
-func TestFetch(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "postgresql")
+func TestBgwriter(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, mtest.GetConfig("bgwriter", r.Host()))
+			event, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
 
-	f := mbtest.NewEventFetcher(t, getConfig())
-	event, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
 
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+			assert.Contains(t, event, "checkpoints")
+			assert.Contains(t, event, "buffers")
+			assert.Contains(t, event, "stats_reset")
 
-	assert.Contains(t, event, "checkpoints")
-	assert.Contains(t, event, "buffers")
-	assert.Contains(t, event, "stats_reset")
+			checkpoints := event["checkpoints"].(common.MapStr)
+			assert.Contains(t, checkpoints, "scheduled")
+			assert.Contains(t, checkpoints, "requested")
+			assert.Contains(t, checkpoints, "times")
 
-	checkpoints := event["checkpoints"].(common.MapStr)
-	assert.Contains(t, checkpoints, "scheduled")
-	assert.Contains(t, checkpoints, "requested")
-	assert.Contains(t, checkpoints, "times")
+			buffers := event["buffers"].(common.MapStr)
+			assert.Contains(t, buffers, "checkpoints")
+			assert.Contains(t, buffers, "clean")
+			assert.Contains(t, buffers, "clean_full")
+			assert.Contains(t, buffers, "backend")
+			assert.Contains(t, buffers, "backend_fsync")
+			assert.Contains(t, buffers, "allocated")
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, mtest.GetConfig("bgwriter", r.Host()))
 
-	buffers := event["buffers"].(common.MapStr)
-	assert.Contains(t, buffers, "checkpoints")
-	assert.Contains(t, buffers, "clean")
-	assert.Contains(t, buffers, "clean_full")
-	assert.Contains(t, buffers, "backend")
-	assert.Contains(t, buffers, "backend_fsync")
-	assert.Contains(t, buffers, "allocated")
-}
-
-func TestData(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "postgresql")
-
-	f := mbtest.NewEventFetcher(t, getConfig())
-
-	err := mbtest.WriteEvent(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
-}
-
-func getConfig() map[string]interface{} {
-	return map[string]interface{}{
-		"module":     "postgresql",
-		"metricsets": []string{"bgwriter"},
-		"hosts":      []string{postgresql.GetEnvDSN()},
-		"username":   postgresql.GetEnvUsername(),
-		"password":   postgresql.GetEnvPassword(),
-	}
+			err := mbtest.WriteEvent(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
diff --git a/metricbeat/module/postgresql/database/database_integration_test.go b/metricbeat/module/postgresql/database/database_integration_test.go
index e09dd707727..84b2ecfb3fb 100644
--- a/metricbeat/module/postgresql/database/database_integration_test.go
+++ b/metricbeat/module/postgresql/database/database_integration_test.go
@@ -25,59 +25,45 @@ import (
 	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	"github.com/elastic/beats/metricbeat/module/postgresql"
+	"github.com/elastic/beats/metricbeat/module/postgresql/mtest"
 
 	"github.com/stretchr/testify/assert"
 )
 
-func TestFetch(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "postgresql")
+func TestDatabase(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, mtest.GetConfig("database", r.Host()))
+			events, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
 
-	f := mbtest.NewEventsFetcher(t, getConfig())
-	events, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
+			assert.True(t, len(events) > 0)
+			event := events[0]
 
-	assert.True(t, len(events) > 0)
-	event := events[0]
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
 
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+			// Check event fields
+			db_oid := event["oid"].(int64)
+			assert.True(t, db_oid > 0)
+			assert.Contains(t, event, "name")
+			_, ok := event["name"].(string)
+			assert.True(t, ok)
 
-	// Check event fields
-	db_oid := event["oid"].(int64)
-	assert.True(t, db_oid > 0)
-	assert.Contains(t, event, "name")
-	_, ok := event["name"].(string)
-	assert.True(t, ok)
-
-	rows := event["rows"].(common.MapStr)
-	assert.Contains(t, rows, "returned")
-	assert.Contains(t, rows, "fetched")
-	assert.Contains(t, rows, "inserted")
-	assert.Contains(t, rows, "updated")
-	assert.Contains(t, rows, "deleted")
-}
-
-func TestData(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "postgresql")
-
-	f := mbtest.NewEventsFetcher(t, getConfig())
-
-	err := mbtest.WriteEvents(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
-}
-
-func getConfig() map[string]interface{} {
-	return map[string]interface{}{
-		"module":     "postgresql",
-		"metricsets": []string{"database"},
-		"hosts":      []string{postgresql.GetEnvDSN()},
-		"username":   postgresql.GetEnvUsername(),
-		"password":   postgresql.GetEnvPassword(),
-	}
+			rows := event["rows"].(common.MapStr)
+			assert.Contains(t, rows, "returned")
+			assert.Contains(t, rows, "fetched")
+			assert.Contains(t, rows, "inserted")
+			assert.Contains(t, rows, "updated")
+			assert.Contains(t, rows, "deleted")
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, mtest.GetConfig("database", r.Host()))
+			err := mbtest.WriteEvents(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
diff --git a/metricbeat/module/postgresql/testing.go b/metricbeat/module/postgresql/mtest/runner.go
similarity index 76%
rename from metricbeat/module/postgresql/testing.go
rename to metricbeat/module/postgresql/mtest/runner.go
index 13673c8c1d1..af9ac5c0edf 100644
--- a/metricbeat/module/postgresql/testing.go
+++ b/metricbeat/module/postgresql/mtest/runner.go
@@ -15,18 +15,15 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package postgresql
+package mtest
 
-import "os"
+import (
+	"github.com/elastic/beats/libbeat/tests/compose"
+)
 
-func GetEnvDSN() string {
-	return os.Getenv("POSTGRESQL_DSN")
-}
-
-func GetEnvUsername() string {
-	return os.Getenv("POSTGRESQL_USERNAME")
-}
-
-func GetEnvPassword() string {
-	return os.Getenv("POSTGRESQL_PASSWORD")
-}
+var (
+	Runner = compose.TestRunner{
+		Service:  "postgresql",
+		Parallel: true,
+	}
+)
diff --git a/metricbeat/module/postgresql/mtest/testing.go b/metricbeat/module/postgresql/mtest/testing.go
new file mode 100644
index 00000000000..348af73ed28
--- /dev/null
+++ b/metricbeat/module/postgresql/mtest/testing.go
@@ -0,0 +1,49 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package mtest
+
+import (
+	"fmt"
+	"os"
+)
+
+const (
+	defaultUsername = "postgres"
+)
+
+func GetConfig(metricset, host string) map[string]interface{} {
+	dsn := fmt.Sprintf("postgres://%s?sslmode=disable", host)
+	return map[string]interface{}{
+		"module":     "postgresql",
+		"metricsets": []string{metricset},
+		"hosts":      []string{dsn},
+		"username":   getEnvUsername(),
+		"password":   getEnvPassword(),
+	}
+}
+
+func getEnvUsername() string {
+	if username := os.Getenv("POSTGRESQL_USERNAME"); len(username) > 0 {
+		return username
+	}
+	return defaultUsername
+}
+
+func getEnvPassword() string {
+	return os.Getenv("POSTGRESQL_PASSWORD")
+}
diff --git a/metricbeat/module/postgresql/statement/statement_integration_test.go b/metricbeat/module/postgresql/statement/statement_integration_test.go
index b3449ed5c96..5055161c317 100644
--- a/metricbeat/module/postgresql/statement/statement_integration_test.go
+++ b/metricbeat/module/postgresql/statement/statement_integration_test.go
@@ -25,89 +25,77 @@ import (
 	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	"github.com/elastic/beats/metricbeat/module/postgresql"
+	"github.com/elastic/beats/metricbeat/module/postgresql/mtest"
 
 	"github.com/stretchr/testify/assert"
 )
 
-func TestFetch(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "postgresql")
-
-	f := mbtest.NewEventsFetcher(t, getConfig())
-	events, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	assert.True(t, len(events) > 0)
-	event := events[0]
-
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-
-	// Check event fields
-	assert.Contains(t, event, "user")
-	assert.Contains(t, event["user"].(common.MapStr), "id")
-
-	assert.Contains(t, event, "database")
-	db_oid := event["database"].(common.MapStr)["oid"].(int64)
-	assert.True(t, db_oid > 0)
-
-	assert.Contains(t, event, "query")
-	query := event["query"].(common.MapStr)
-	assert.Contains(t, query, "id")
-	assert.Contains(t, query, "text")
-	assert.Contains(t, query, "calls")
-	assert.Contains(t, query, "rows")
-
-	assert.Contains(t, query, "time")
-	time := query["time"].(common.MapStr)
-	assert.Contains(t, time, "total")
-	assert.Contains(t, time, "min")
-	assert.Contains(t, time, "max")
-	assert.Contains(t, time, "mean")
-	assert.Contains(t, time, "stddev")
-
-	assert.Contains(t, query["memory"], "shared")
-	memory := query["memory"].(common.MapStr)
-
-	assert.Contains(t, memory, "shared")
-	shared := memory["shared"].(common.MapStr)
-	assert.Contains(t, shared, "hit")
-	assert.Contains(t, shared, "read")
-	assert.Contains(t, shared, "dirtied")
-	assert.Contains(t, shared, "written")
-
-	assert.Contains(t, memory, "local")
-	local := memory["local"].(common.MapStr)
-	assert.Contains(t, local, "hit")
-	assert.Contains(t, local, "read")
-	assert.Contains(t, local, "dirtied")
-	assert.Contains(t, local, "written")
-
-	assert.Contains(t, memory, "temp")
-	temp := memory["temp"].(common.MapStr)
-	assert.Contains(t, temp, "read")
-	assert.Contains(t, temp, "written")
-}
-
-func TestData(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "postgresql")
-	f := mbtest.NewEventsFetcher(t, getConfig())
-
-	err := mbtest.WriteEvents(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
-}
-
-func getConfig() map[string]interface{} {
-	return map[string]interface{}{
-		"module":     "postgresql",
-		"metricsets": []string{"statement"},
-		"hosts":      []string{postgresql.GetEnvDSN()},
-		"username":   postgresql.GetEnvUsername(),
-		"password":   postgresql.GetEnvPassword(),
-	}
+func TestStatement(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, mtest.GetConfig("statement", r.Host()))
+			events, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			assert.True(t, len(events) > 0)
+			event := events[0]
+
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+
+			// Check event fields
+			assert.Contains(t, event, "user")
+			assert.Contains(t, event["user"].(common.MapStr), "id")
+
+			assert.Contains(t, event, "database")
+			db_oid := event["database"].(common.MapStr)["oid"].(int64)
+			assert.True(t, db_oid > 0)
+
+			assert.Contains(t, event, "query")
+			query := event["query"].(common.MapStr)
+			assert.Contains(t, query, "id")
+			assert.Contains(t, query, "text")
+			assert.Contains(t, query, "calls")
+			assert.Contains(t, query, "rows")
+
+			assert.Contains(t, query, "time")
+			time := query["time"].(common.MapStr)
+			assert.Contains(t, time, "total")
+			assert.Contains(t, time, "min")
+			assert.Contains(t, time, "max")
+			assert.Contains(t, time, "mean")
+			assert.Contains(t, time, "stddev")
+
+			assert.Contains(t, query["memory"], "shared")
+			memory := query["memory"].(common.MapStr)
+
+			assert.Contains(t, memory, "shared")
+			shared := memory["shared"].(common.MapStr)
+			assert.Contains(t, shared, "hit")
+			assert.Contains(t, shared, "read")
+			assert.Contains(t, shared, "dirtied")
+			assert.Contains(t, shared, "written")
+
+			assert.Contains(t, memory, "local")
+			local := memory["local"].(common.MapStr)
+			assert.Contains(t, local, "hit")
+			assert.Contains(t, local, "read")
+			assert.Contains(t, local, "dirtied")
+			assert.Contains(t, local, "written")
+
+			assert.Contains(t, memory, "temp")
+			temp := memory["temp"].(common.MapStr)
+			assert.Contains(t, temp, "read")
+			assert.Contains(t, temp, "written")
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, mtest.GetConfig("statement", r.Host()))
+
+			err := mbtest.WriteEvents(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }

From a337d77594f8364458677cf45e2b4f08850a35b5 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Sun, 11 Nov 2018 19:22:21 +0100
Subject: [PATCH 61/88] Set a default name for compose test scenarios without
 variables

---
 libbeat/tests/compose/runner.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/libbeat/tests/compose/runner.go b/libbeat/tests/compose/runner.go
index 0ce16f3e185..1123f29a5b6 100644
--- a/libbeat/tests/compose/runner.go
+++ b/libbeat/tests/compose/runner.go
@@ -121,6 +121,9 @@ func (r *TestRunner) Run(t *testing.T, tests Suite) {
 		}
 		sort.Strings(vars)
 		desc := strings.Join(vars, ",")
+		if desc == "" {
+			desc = "WithoutVars"
+		}
 
 		seq := make([]byte, 16)
 		rand.Read(seq)

From e4b209faff85f36f1141fcbc3bd9cc39bef31f39 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Sun, 11 Nov 2018 19:47:44 +0100
Subject: [PATCH 62/88] Run prometheus tests with testrunner

---
 metricbeat/docker-compose.yml                 |  2 +
 metricbeat/module/prometheus/_meta/Dockerfile |  1 +
 .../collector/collector_integration_test.go   | 74 ++++++------------
 metricbeat/module/prometheus/mtest/testing.go | 37 +++++++++
 .../stats/stats_integration_test.go           | 78 ++++++-------------
 5 files changed, 90 insertions(+), 102 deletions(-)
 create mode 100644 metricbeat/module/prometheus/mtest/testing.go

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 59a4555679d..4d7be27d0bb 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -171,6 +171,8 @@ services:
 
   prometheus:
     build: ./module/prometheus/_meta
+    ports:
+      - 9090
 
   rabbitmq:
     build: ./module/rabbitmq/_meta
diff --git a/metricbeat/module/prometheus/_meta/Dockerfile b/metricbeat/module/prometheus/_meta/Dockerfile
index 836c8777f62..f9d70de6ba5 100644
--- a/metricbeat/module/prometheus/_meta/Dockerfile
+++ b/metricbeat/module/prometheus/_meta/Dockerfile
@@ -1,3 +1,4 @@
 FROM prom/prometheus:v1.5.1
+RUN mkdir /run # Needed to copy /run/compose_env on startup
 HEALTHCHECK --interval=1s --retries=90 CMD nc -w 1 localhost 9090 </dev/null
 EXPOSE 9090
diff --git a/metricbeat/module/prometheus/collector/collector_integration_test.go b/metricbeat/module/prometheus/collector/collector_integration_test.go
index d2f186e9d06..1669078e643 100644
--- a/metricbeat/module/prometheus/collector/collector_integration_test.go
+++ b/metricbeat/module/prometheus/collector/collector_integration_test.go
@@ -20,11 +20,11 @@
 package collector
 
 import (
-	"os"
 	"testing"
 
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
+	"github.com/elastic/beats/metricbeat/module/prometheus/mtest"
 
 	"github.com/stretchr/testify/assert"
 )
@@ -32,54 +32,30 @@ import (
 // These tests are running with prometheus metrics as an example as this container is already available
 // Every prometheus exporter should work here.
 
-func TestFetch(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "prometheus")
-
-	f := mbtest.NewEventsFetcher(t, getConfig())
-	event, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-}
-
-func TestData(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "prometheus")
-
-	f := mbtest.NewEventsFetcher(t, getConfig())
-
-	err := mbtest.WriteEvents(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
+func TestCollector(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
+			event, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
+
+			err := mbtest.WriteEvents(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
 
-func getConfig() map[string]interface{} {
-	return map[string]interface{}{
-		"module":     "prometheus",
-		"metricsets": []string{"collector"},
-		"hosts":      []string{getPrometheusEnvHost() + ":" + getPrometheusEnvPort()},
-		"namespace":  "collector",
-	}
-}
-
-func getPrometheusEnvHost() string {
-	host := os.Getenv("PROMETHEUS_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
-}
-
-func getPrometheusEnvPort() string {
-	port := os.Getenv("PROMETHEUS_PORT")
-
-	if len(port) == 0 {
-		port = "9090"
-	}
-	return port
+func getConfig(host string) map[string]interface{} {
+	config := mtest.GetConfig("collector", host)
+	config["namespace"] = "collector"
+	return config
 }
diff --git a/metricbeat/module/prometheus/mtest/testing.go b/metricbeat/module/prometheus/mtest/testing.go
new file mode 100644
index 00000000000..f6b7f18106f
--- /dev/null
+++ b/metricbeat/module/prometheus/mtest/testing.go
@@ -0,0 +1,37 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package mtest
+
+import (
+	"github.com/elastic/beats/libbeat/tests/compose"
+)
+
+var (
+	Runner = compose.TestRunner{
+		Service:  "prometheus",
+		Parallel: true,
+	}
+)
+
+func GetConfig(metricset, host string) map[string]interface{} {
+	return map[string]interface{}{
+		"module":     "prometheus",
+		"metricsets": []string{metricset},
+		"hosts":      []string{host},
+	}
+}
diff --git a/metricbeat/module/prometheus/stats/stats_integration_test.go b/metricbeat/module/prometheus/stats/stats_integration_test.go
index 45e62466ec0..0f18a599753 100644
--- a/metricbeat/module/prometheus/stats/stats_integration_test.go
+++ b/metricbeat/module/prometheus/stats/stats_integration_test.go
@@ -20,67 +20,39 @@
 package stats
 
 import (
-	"os"
 	"testing"
 
 	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
+	"github.com/elastic/beats/metricbeat/module/prometheus/mtest"
 
 	"github.com/stretchr/testify/assert"
 )
 
-func TestFetch(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "prometheus")
+func TestStats(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, mtest.GetConfig("stats", r.Host()))
+			event, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+
+			// Check number of fields.
+			assert.Equal(t, 3, len(event))
+			assert.True(t, event["processes"].(common.MapStr)["open_fds"].(int64) > 0)
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, mtest.GetConfig("stats", r.Host()))
+
+			err := mbtest.WriteEvent(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 
-	f := mbtest.NewEventFetcher(t, getConfig())
-	event, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-
-	// Check number of fields.
-	assert.Equal(t, 3, len(event))
-	assert.True(t, event["processes"].(common.MapStr)["open_fds"].(int64) > 0)
-}
-
-func TestData(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "prometheus")
-
-	f := mbtest.NewEventFetcher(t, getConfig())
-
-	err := mbtest.WriteEvent(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
-}
-
-func getConfig() map[string]interface{} {
-	return map[string]interface{}{
-		"module":     "prometheus",
-		"metricsets": []string{"stats"},
-		"hosts":      []string{getPrometheusEnvHost() + ":" + getPrometheusEnvPort()},
-	}
-}
-
-func getPrometheusEnvHost() string {
-	host := os.Getenv("PROMETHEUS_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
-}
-
-func getPrometheusEnvPort() string {
-	port := os.Getenv("PROMETHEUS_PORT")
-
-	if len(port) == 0 {
-		port = "9090"
-	}
-	return port
 }

From 45e195088087fa300a64cfa6975550716f03d4ad Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Mon, 12 Nov 2018 12:06:18 +0100
Subject: [PATCH 63/88] Run mysql tests with testrunner

---
 metricbeat/docker-compose.yml                 |   2 +
 .../module/mysql/{ => mtest}/testing.go       |  41 +++---
 .../module/mysql/mysql_integration_test.go    |  19 +--
 .../mysql/status/status_integration_test.go   | 125 ++++++++----------
 4 files changed, 92 insertions(+), 95 deletions(-)
 rename metricbeat/module/mysql/{ => mtest}/testing.go (57%)

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 4d7be27d0bb..2ff5657a3ae 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -157,6 +157,8 @@ services:
 
   mysql:
     build: ./module/mysql/_meta
+    ports:
+      - 3306
 
   nginx:
     build: ./module/nginx/_meta
diff --git a/metricbeat/module/mysql/testing.go b/metricbeat/module/mysql/mtest/testing.go
similarity index 57%
rename from metricbeat/module/mysql/testing.go
rename to metricbeat/module/mysql/mtest/testing.go
index 6036d717dc2..9a8410ccb8d 100644
--- a/metricbeat/module/mysql/testing.go
+++ b/metricbeat/module/mysql/mtest/testing.go
@@ -15,29 +15,36 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package mysql
+package mtest
 
 import (
-	"os"
-
 	"github.com/go-sql-driver/mysql"
+
+	"github.com/elastic/beats/libbeat/tests/compose"
 )
 
-// Helper functions for testing used in the mysql MetricSets.
+var Runner = compose.TestRunner{
+	Service:  "mysql",
+	Parallel: true,
+}
 
-// GetMySQLEnvDSN returns the MySQL server DSN to use for testing. It
-// reads the value from the MYSQL_DSN environment variable and returns
-// root@tcp(127.0.0.1:3306)/ if it is not set.
-func GetMySQLEnvDSN() string {
-	dsn := os.Getenv("MYSQL_DSN")
+// GetDSN returns the MySQL server DSN to use for testing.
+func GetDSN(host string) string {
+	c := &mysql.Config{
+		Net:    "tcp",
+		Addr:   host,
+		User:   "root",
+		Passwd: "test",
+	}
+	return c.FormatDSN()
+}
 
-	if len(dsn) == 0 {
-		c := &mysql.Config{
-			Net:  "tcp",
-			Addr: "127.0.0.1:3306",
-			User: "root",
-		}
-		dsn = c.FormatDSN()
+// GetConfig returns the configuration for a mysql module
+func GetConfig(metricset, host string, raw bool) map[string]interface{} {
+	return map[string]interface{}{
+		"module":     "mysql",
+		"metricsets": []string{metricset},
+		"hosts":      []string{GetDSN(host)},
+		"raw":        raw,
 	}
-	return dsn
 }
diff --git a/metricbeat/module/mysql/mysql_integration_test.go b/metricbeat/module/mysql/mysql_integration_test.go
index 32b90a519b6..602b617d215 100644
--- a/metricbeat/module/mysql/mysql_integration_test.go
+++ b/metricbeat/module/mysql/mysql_integration_test.go
@@ -25,16 +25,17 @@ import (
 	"github.com/stretchr/testify/assert"
 
 	"github.com/elastic/beats/libbeat/tests/compose"
-	_ "github.com/elastic/beats/metricbeat/mb/testing"
+	"github.com/elastic/beats/metricbeat/module/mysql/mtest"
 )
 
 func TestNewDB(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "mysql")
-
-	db, err := NewDB(GetMySQLEnvDSN())
-	assert.NoError(t, err)
-
-	err = db.Ping()
-	assert.NoError(t, err)
+	mtest.Runner.Run(t, compose.Suite{
+		"NewDB": func(t *testing.T, r compose.R) {
+			db, err := NewDB(mtest.GetDSN(r.Host()))
+			assert.NoError(t, err)
+
+			err = db.Ping()
+			assert.NoError(t, err)
+		},
+	})
 }
diff --git a/metricbeat/module/mysql/status/status_integration_test.go b/metricbeat/module/mysql/status/status_integration_test.go
index e0a4898de35..67f1c874235 100644
--- a/metricbeat/module/mysql/status/status_integration_test.go
+++ b/metricbeat/module/mysql/status/status_integration_test.go
@@ -25,77 +25,64 @@ import (
 	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	"github.com/elastic/beats/metricbeat/module/mysql"
+	"github.com/elastic/beats/metricbeat/module/mysql/mtest"
 
 	"github.com/stretchr/testify/assert"
 )
 
-func TestFetch(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "mysql")
-
-	f := mbtest.NewEventFetcher(t, getConfig(false))
-	event, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-
-	// Check event fields
-	connections := event["connections"].(int64)
-	open := event["open"].(common.MapStr)
-	openTables := open["tables"].(int64)
-	openFiles := open["files"].(int64)
-	openStreams := open["streams"].(int64)
-
-	assert.True(t, connections > 0)
-	assert.True(t, openTables > 0)
-	assert.True(t, openFiles >= 0)
-	assert.True(t, openStreams == 0)
-}
-
-func TestFetchRaw(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "mysql")
-
-	f := mbtest.NewEventFetcher(t, getConfig(true))
-	event, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-
-	// Check event fields
-	cachedThreads := event["threads"].(common.MapStr)["cached"].(int64)
-	assert.True(t, cachedThreads >= 0)
-
-	rawData := event["raw"].(common.MapStr)
-
-	// Make sure field was removed from raw fields as in schema
-	_, exists := rawData["Threads_cached"]
-	assert.False(t, exists)
-
-	// Check a raw field if it is available
-	_, exists = rawData["Slow_launch_threads"]
-	assert.True(t, exists)
-}
-
-func TestData(t *testing.T) {
-	f := mbtest.NewEventFetcher(t, getConfig(false))
-
-	err := mbtest.WriteEvent(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
-}
-
-func getConfig(raw bool) map[string]interface{} {
-	return map[string]interface{}{
-		"module":     "mysql",
-		"metricsets": []string{"status"},
-		"hosts":      []string{mysql.GetMySQLEnvDSN()},
-		"raw":        raw,
-	}
+func TestStatus(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, mtest.GetConfig("status", r.Host(), false))
+			event, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+
+			// Check event fields
+			connections := event["connections"].(int64)
+			open := event["open"].(common.MapStr)
+			openTables := open["tables"].(int64)
+			openFiles := open["files"].(int64)
+			openStreams := open["streams"].(int64)
+
+			assert.True(t, connections > 0)
+			assert.True(t, openTables > 0)
+			assert.True(t, openFiles >= 0)
+			assert.True(t, openStreams == 0)
+		},
+		"FetchRaw": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, mtest.GetConfig("status", r.Host(), true))
+			event, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+
+			// Check event fields
+			cachedThreads := event["threads"].(common.MapStr)["cached"].(int64)
+			assert.True(t, cachedThreads >= 0)
+
+			rawData := event["raw"].(common.MapStr)
+
+			// Make sure field was removed from raw fields as in schema
+			_, exists := rawData["Threads_cached"]
+			assert.False(t, exists)
+
+			// Check a raw field if it is available
+			_, exists = rawData["Slow_launch_threads"]
+			assert.True(t, exists)
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, mtest.GetConfig("status", r.Host(), false))
+
+			err := mbtest.WriteEvent(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }

From 972e304be58bd9ecb1f204e9220f0b166b6c04ed Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Mon, 12 Nov 2018 16:50:40 +0100
Subject: [PATCH 64/88] Run php_fpm tests with testrunner

---
 metricbeat/docker-compose.yml                 |  2 +
 metricbeat/module/php_fpm/mtest/testing.go    | 37 +++++++++++++++
 .../php_fpm/pool/pool_integration_test.go     | 46 +++++--------------
 .../process/process_integration_test.go       | 45 +++++-------------
 4 files changed, 61 insertions(+), 69 deletions(-)
 create mode 100644 metricbeat/module/php_fpm/mtest/testing.go

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 2ff5657a3ae..9498c12ba49 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -165,6 +165,8 @@ services:
 
   phpfpm:
     build: ./module/php_fpm/_meta
+    ports:
+      - 81
 
   postgresql:
     build: ./module/postgresql/_meta
diff --git a/metricbeat/module/php_fpm/mtest/testing.go b/metricbeat/module/php_fpm/mtest/testing.go
new file mode 100644
index 00000000000..7f427379fb4
--- /dev/null
+++ b/metricbeat/module/php_fpm/mtest/testing.go
@@ -0,0 +1,37 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// +build integration
+
+package mtest
+
+import (
+	"github.com/elastic/beats/libbeat/tests/compose"
+)
+
+var Runner = compose.TestRunner{
+	Service:  "phpfpm",
+	Parallel: true,
+}
+
+func GetConfig(metricset, host string) map[string]interface{} {
+	return map[string]interface{}{
+		"module":     "php_fpm",
+		"metricsets": []string{metricset},
+		"hosts":      []string{host},
+	}
+}
diff --git a/metricbeat/module/php_fpm/pool/pool_integration_test.go b/metricbeat/module/php_fpm/pool/pool_integration_test.go
index a2ff0d0f5c6..332bb889a10 100644
--- a/metricbeat/module/php_fpm/pool/pool_integration_test.go
+++ b/metricbeat/module/php_fpm/pool/pool_integration_test.go
@@ -20,45 +20,21 @@
 package pool
 
 import (
-	"os"
 	"testing"
 
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
+	"github.com/elastic/beats/metricbeat/module/php_fpm/mtest"
 )
 
-func TestData(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "phpfpm")
-	f := mbtest.NewReportingMetricSetV2(t, getConfig())
-	err := mbtest.WriteEventsReporterV2(f, t, "")
-	if err != nil {
-		t.Fatal("write", err)
-	}
-}
-
-func getConfig() map[string]interface{} {
-	return map[string]interface{}{
-		"module":     "php_fpm",
-		"metricsets": []string{"pool"},
-		"hosts":      []string{GetEnvHost() + ":" + GetEnvPort()},
-	}
-}
-
-func GetEnvHost() string {
-	host := os.Getenv("PHPFPM_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
-}
-
-func GetEnvPort() string {
-	port := os.Getenv("PHPFPM_PORT")
-
-	if len(port) == 0 {
-		port = "81"
-	}
-	return port
+func TestPool(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("pool", r.Host()))
+			err := mbtest.WriteEventsReporterV2(f, t, "")
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
diff --git a/metricbeat/module/php_fpm/process/process_integration_test.go b/metricbeat/module/php_fpm/process/process_integration_test.go
index 182f4815de4..66aa94e3acd 100644
--- a/metricbeat/module/php_fpm/process/process_integration_test.go
+++ b/metricbeat/module/php_fpm/process/process_integration_test.go
@@ -20,44 +20,21 @@
 package process
 
 import (
-	"os"
 	"testing"
 
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
+	"github.com/elastic/beats/metricbeat/module/php_fpm/mtest"
 )
 
-func TestData(t *testing.T) {
-	compose.EnsureUp(t, "phpfpm")
-	f := mbtest.NewReportingMetricSetV2(t, getConfig())
-	err := mbtest.WriteEventsReporterV2(f, t, "")
-	if err != nil {
-		t.Fatal("write", err)
-	}
-}
-
-func getConfig() map[string]interface{} {
-	return map[string]interface{}{
-		"module":     "php_fpm",
-		"metricsets": []string{"process"},
-		"hosts":      []string{GetEnvHost() + ":" + GetEnvPort()},
-	}
-}
-
-func GetEnvHost() string {
-	host := os.Getenv("PHPFPM_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
-}
-
-func GetEnvPort() string {
-	port := os.Getenv("PHPFPM_PORT")
-
-	if len(port) == 0 {
-		port = "81"
-	}
-	return port
+func TestProcess(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("process", r.Host()))
+			err := mbtest.WriteEventsReporterV2(f, t, "")
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }

From ad8d2bc01316a5f139d6eb6d609735481b64930f Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Mon, 12 Nov 2018 17:03:01 +0100
Subject: [PATCH 65/88] Run nginx tests with testrunner

---
 metricbeat/docker-compose.yml                 |  2 +
 .../stubstatus/stubstatus_integration_test.go | 46 ++++++++++---------
 metricbeat/module/nginx/testing.go            | 35 --------------
 3 files changed, 27 insertions(+), 56 deletions(-)
 delete mode 100644 metricbeat/module/nginx/testing.go

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 9498c12ba49..d9b5adfc920 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -162,6 +162,8 @@ services:
 
   nginx:
     build: ./module/nginx/_meta
+    ports:
+      - 80
 
   phpfpm:
     build: ./module/php_fpm/_meta
diff --git a/metricbeat/module/nginx/stubstatus/stubstatus_integration_test.go b/metricbeat/module/nginx/stubstatus/stubstatus_integration_test.go
index 779e400f02b..7c2a441b97a 100644
--- a/metricbeat/module/nginx/stubstatus/stubstatus_integration_test.go
+++ b/metricbeat/module/nginx/stubstatus/stubstatus_integration_test.go
@@ -24,40 +24,44 @@ import (
 
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	"github.com/elastic/beats/metricbeat/module/nginx"
 
 	"github.com/stretchr/testify/assert"
 )
 
-func TestFetch(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "nginx")
-
-	f := mbtest.NewEventFetcher(t, getConfig())
-	event, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
+func TestStubstatus(t *testing.T) {
+	runner := compose.TestRunner{
+		Service:  "nginx",
+		Parallel: true,
 	}
 
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+	runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+			event, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
 
-	// Check number of fields.
-	assert.Equal(t, 10, len(event))
-}
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
 
-func TestData(t *testing.T) {
-	f := mbtest.NewEventFetcher(t, getConfig())
+			// Check number of fields.
+			assert.Equal(t, 10, len(event))
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
 
-	err := mbtest.WriteEvent(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
+			err := mbtest.WriteEvent(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
 
-func getConfig() map[string]interface{} {
+func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "nginx",
 		"metricsets": []string{"stubstatus"},
-		"hosts":      []string{nginx.GetNginxEnvHost()},
+		"hosts":      []string{host},
 	}
 }
diff --git a/metricbeat/module/nginx/testing.go b/metricbeat/module/nginx/testing.go
deleted file mode 100644
index cee26e1d11a..00000000000
--- a/metricbeat/module/nginx/testing.go
+++ /dev/null
@@ -1,35 +0,0 @@
-// Licensed to Elasticsearch B.V. under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Elasticsearch B.V. licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package nginx
-
-/*
-Helper functions for testing used in the nginx metricsets
-*/
-
-import (
-	"os"
-)
-
-func GetNginxEnvHost() string {
-	host := os.Getenv("NGINX_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
-}

From 47fe92818923567249cc7834d0221f9a8d4ab757 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Mon, 12 Nov 2018 17:26:41 +0100
Subject: [PATCH 66/88] Run mongodb tests with testrunner

---
 metricbeat/docker-compose.yml                 |   2 +
 .../collstats/collstats_integration_test.go   |  69 +++++-----
 .../dbstats/dbstats_integration_test.go       | 119 ++++++++----------
 .../metrics/metrics_intergration_test.go      |  65 ++++------
 .../module/mongodb/{ => mtest}/testing.go     |  35 ++----
 .../replstatus/replstatus_integration_test.go | 108 +++++++---------
 .../mongodb/status/status_integration_test.go |  71 +++++------
 7 files changed, 202 insertions(+), 267 deletions(-)
 rename metricbeat/module/mongodb/{ => mtest}/testing.go (54%)

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index d9b5adfc920..43476b46627 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -151,6 +151,8 @@ services:
   mongodb:
     build: ./module/mongodb/_meta
     command: mongod --replSet beats
+    ports:
+      - 27017
 
   munin:
     build: ./module/munin/_meta
diff --git a/metricbeat/module/mongodb/collstats/collstats_integration_test.go b/metricbeat/module/mongodb/collstats/collstats_integration_test.go
index 4f860e98e8e..98b5b0b5028 100644
--- a/metricbeat/module/mongodb/collstats/collstats_integration_test.go
+++ b/metricbeat/module/mongodb/collstats/collstats_integration_test.go
@@ -26,46 +26,35 @@ import (
 
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	"github.com/elastic/beats/metricbeat/module/mongodb"
+	"github.com/elastic/beats/metricbeat/module/mongodb/mtest"
 )
 
-func TestFetch(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "mongodb")
-
-	f := mbtest.NewEventsFetcher(t, getConfig())
-	events, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	for _, event := range events {
-		t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-
-		// Check a few event Fields
-		db := event["db"].(string)
-		assert.NotEqual(t, db, "")
-
-		collection := event["collection"].(string)
-		assert.NotEqual(t, collection, "")
-	}
-}
-
-func TestData(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "mongodb")
-
-	f := mbtest.NewEventsFetcher(t, getConfig())
-	err := mbtest.WriteEvents(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
-}
-
-func getConfig() map[string]interface{} {
-	return map[string]interface{}{
-		"module":     "mongodb",
-		"metricsets": []string{"collstats"},
-		"hosts":      []string{mongodb.GetEnvHost() + ":" + mongodb.GetEnvPort()},
-	}
+func TestCollstats(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, mtest.GetConfig("collstats", r.Host()))
+			events, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			for _, event := range events {
+				t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+
+				// Check a few event Fields
+				db := event["db"].(string)
+				assert.NotEqual(t, db, "")
+
+				collection := event["collection"].(string)
+				assert.NotEqual(t, collection, "")
+			}
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, mtest.GetConfig("collstats", r.Host()))
+			err := mbtest.WriteEvents(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
diff --git a/metricbeat/module/mongodb/dbstats/dbstats_integration_test.go b/metricbeat/module/mongodb/dbstats/dbstats_integration_test.go
index 75fc73790ab..8cea2158fe1 100644
--- a/metricbeat/module/mongodb/dbstats/dbstats_integration_test.go
+++ b/metricbeat/module/mongodb/dbstats/dbstats_integration_test.go
@@ -26,71 +26,60 @@ import (
 
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	"github.com/elastic/beats/metricbeat/module/mongodb"
+	"github.com/elastic/beats/metricbeat/module/mongodb/mtest"
 )
 
-func TestFetch(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "mongodb")
-
-	f := mbtest.NewEventsFetcher(t, getConfig())
-	events, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	for _, event := range events {
-		t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-
-		// Check a few event Fields
-		db := event["db"].(string)
-		assert.NotEqual(t, db, "")
-
-		collections := event["collections"].(int64)
-		assert.True(t, collections > 0)
-
-		objects := event["objects"].(int64)
-		assert.True(t, objects > 0)
-
-		avgObjSize, err := event.GetValue("avg_obj_size.bytes")
-		assert.NoError(t, err)
-		assert.True(t, avgObjSize.(int64) > 0)
-
-		dataSize, err := event.GetValue("data_size.bytes")
-		assert.NoError(t, err)
-		assert.True(t, dataSize.(int64) > 0)
-
-		storageSize, err := event.GetValue("storage_size.bytes")
-		assert.NoError(t, err)
-		assert.True(t, storageSize.(int64) > 0)
-
-		numExtents := event["num_extents"].(int64)
-		assert.True(t, numExtents >= 0)
-
-		indexes := event["indexes"].(int64)
-		assert.True(t, indexes >= 0)
-
-		indexSize, err := event.GetValue("index_size.bytes")
-		assert.NoError(t, err)
-		assert.True(t, indexSize.(int64) > 0)
-	}
-}
-
-func TestData(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "mongodb")
-
-	f := mbtest.NewEventsFetcher(t, getConfig())
-	err := mbtest.WriteEvents(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
-}
-
-func getConfig() map[string]interface{} {
-	return map[string]interface{}{
-		"module":     "mongodb",
-		"metricsets": []string{"dbstats"},
-		"hosts":      []string{mongodb.GetEnvHost() + ":" + mongodb.GetEnvPort()},
-	}
+func TestDBStats(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, mtest.GetConfig("dbstats", r.Host()))
+			events, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			for _, event := range events {
+				t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+
+				// Check a few event Fields
+				db := event["db"].(string)
+				assert.NotEqual(t, db, "")
+
+				collections := event["collections"].(int64)
+				assert.True(t, collections > 0)
+
+				objects := event["objects"].(int64)
+				assert.True(t, objects > 0)
+
+				avgObjSize, err := event.GetValue("avg_obj_size.bytes")
+				assert.NoError(t, err)
+				assert.True(t, avgObjSize.(int64) > 0)
+
+				dataSize, err := event.GetValue("data_size.bytes")
+				assert.NoError(t, err)
+				assert.True(t, dataSize.(int64) > 0)
+
+				storageSize, err := event.GetValue("storage_size.bytes")
+				assert.NoError(t, err)
+				assert.True(t, storageSize.(int64) > 0)
+
+				numExtents := event["num_extents"].(int64)
+				assert.True(t, numExtents >= 0)
+
+				indexes := event["indexes"].(int64)
+				assert.True(t, indexes >= 0)
+
+				indexSize, err := event.GetValue("index_size.bytes")
+				assert.NoError(t, err)
+				assert.True(t, indexSize.(int64) > 0)
+			}
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, mtest.GetConfig("dbstats", r.Host()))
+			err := mbtest.WriteEvents(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
diff --git a/metricbeat/module/mongodb/metrics/metrics_intergration_test.go b/metricbeat/module/mongodb/metrics/metrics_intergration_test.go
index f39b950846f..0d706716a40 100644
--- a/metricbeat/module/mongodb/metrics/metrics_intergration_test.go
+++ b/metricbeat/module/mongodb/metrics/metrics_intergration_test.go
@@ -26,44 +26,33 @@ import (
 
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	"github.com/elastic/beats/metricbeat/module/mongodb"
+	"github.com/elastic/beats/metricbeat/module/mongodb/mtest"
 )
 
-func TestFetch(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "mongodb")
-
-	f := mbtest.NewEventFetcher(t, getConfig())
-	event, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	// Check a few event Fields
-	findCount, err := event.GetValue("commands.find.total")
-	assert.NoError(t, err)
-	assert.True(t, findCount.(int64) >= 0)
-
-	deletedDocuments, err := event.GetValue("document.deleted")
-	assert.NoError(t, err)
-	assert.True(t, deletedDocuments.(int64) >= 0)
-}
-
-func TestData(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "mongodb")
-
-	f := mbtest.NewEventFetcher(t, getConfig())
-	err := mbtest.WriteEvent(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
-}
-
-func getConfig() map[string]interface{} {
-	return map[string]interface{}{
-		"module":     "mongodb",
-		"metricsets": []string{"metrics"},
-		"hosts":      []string{mongodb.GetEnvHost() + ":" + mongodb.GetEnvPort()},
-	}
+func TestMetrics(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, mtest.GetConfig("metrics", r.Host()))
+			event, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			// Check a few event Fields
+			findCount, err := event.GetValue("commands.find.total")
+			assert.NoError(t, err)
+			assert.True(t, findCount.(int64) >= 0)
+
+			deletedDocuments, err := event.GetValue("document.deleted")
+			assert.NoError(t, err)
+			assert.True(t, deletedDocuments.(int64) >= 0)
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, mtest.GetConfig("metrics", r.Host()))
+			err := mbtest.WriteEvent(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
diff --git a/metricbeat/module/mongodb/testing.go b/metricbeat/module/mongodb/mtest/testing.go
similarity index 54%
rename from metricbeat/module/mongodb/testing.go
rename to metricbeat/module/mongodb/mtest/testing.go
index 31d380b8322..5d4200825a8 100644
--- a/metricbeat/module/mongodb/testing.go
+++ b/metricbeat/module/mongodb/mtest/testing.go
@@ -15,32 +15,21 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package mongodb
+package mtest
 
-import "os"
+import (
+	"github.com/elastic/beats/libbeat/tests/compose"
+)
 
-// Helper functions for testing mongodb metricsets.
-
-// GetEnvHost returns the hostname of the Mongodb server to use for testing.
-// It reads the value from the MONGODB_HOST environment variable and returns
-// 127.0.0.1 if it is not set.
-func GetEnvHost() string {
-	host := os.Getenv("MONGODB_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
+var Runner = compose.TestRunner{
+	Service:  "mongodb",
+	Parallel: true,
 }
 
-// GetEnvPort returns the port of the Mongodb server to use for testing.
-// It reads the value from the MONGODB_PORT environment variable and returns
-// 27017 if it is not set.
-func GetEnvPort() string {
-	port := os.Getenv("MONGODB_PORT")
-
-	if len(port) == 0 {
-		port = "27017"
+func GetConfig(metricset, host string) map[string]interface{} {
+	return map[string]interface{}{
+		"module":     "mongodb",
+		"metricsets": []string{metricset},
+		"hosts":      []string{host},
 	}
-	return port
 }
diff --git a/metricbeat/module/mongodb/replstatus/replstatus_integration_test.go b/metricbeat/module/mongodb/replstatus/replstatus_integration_test.go
index 526a81aba8f..c1a9b721b4d 100644
--- a/metricbeat/module/mongodb/replstatus/replstatus_integration_test.go
+++ b/metricbeat/module/mongodb/replstatus/replstatus_integration_test.go
@@ -31,70 +31,58 @@ import (
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
 	"github.com/elastic/beats/metricbeat/module/mongodb"
+	"github.com/elastic/beats/metricbeat/module/mongodb/mtest"
 )
 
-func TestFetch(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "mongodb")
-
-	err := initiateReplicaSet(t)
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	f := mbtest.NewEventFetcher(t, getConfig())
-	event, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-
-	// Check event fields
-	oplog := event["oplog"].(common.MapStr)
-	allocated := oplog["size"].(common.MapStr)["allocated"].(int64)
-	assert.True(t, allocated >= 0)
-
-	used := oplog["size"].(common.MapStr)["used"].(float64)
-	assert.True(t, used > 0)
-
-	firstTs := oplog["first"].(common.MapStr)["timestamp"].(int64)
-	assert.True(t, firstTs >= 0)
-
-	window := oplog["window"].(int64)
-	assert.True(t, window >= 0)
-
-	members := event["members"].(common.MapStr)
-	primary := members["primary"].(common.MapStr)
-	assert.NotEmpty(t, primary["host"].(string))
-	assert.True(t, primary["optime"].(int64) > 0)
-
-	set := event["set_name"].(string)
-	assert.Equal(t, set, "beats")
-}
-
-func TestData(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "mongodb")
-
-	f := mbtest.NewEventFetcher(t, getConfig())
-	err := mbtest.WriteEvent(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
+func TestReplStatus(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			err := initiateReplicaSet(t, r.Host())
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			f := mbtest.NewEventFetcher(t, mtest.GetConfig("replstatus", r.Host()))
+			event, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+
+			// Check event fields
+			oplog := event["oplog"].(common.MapStr)
+			allocated := oplog["size"].(common.MapStr)["allocated"].(int64)
+			assert.True(t, allocated >= 0)
+
+			used := oplog["size"].(common.MapStr)["used"].(float64)
+			assert.True(t, used > 0)
+
+			firstTs := oplog["first"].(common.MapStr)["timestamp"].(int64)
+			assert.True(t, firstTs >= 0)
+
+			window := oplog["window"].(int64)
+			assert.True(t, window >= 0)
+
+			members := event["members"].(common.MapStr)
+			primary := members["primary"].(common.MapStr)
+			assert.NotEmpty(t, primary["host"].(string))
+			assert.True(t, primary["optime"].(int64) > 0)
+
+			set := event["set_name"].(string)
+			assert.Equal(t, set, "beats")
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, mtest.GetConfig("replstatus", r.Host()))
+			err := mbtest.WriteEvent(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
 
-func getConfig() map[string]interface{} {
-	return map[string]interface{}{
-		"module":     "mongodb",
-		"metricsets": []string{"replstatus"},
-		"hosts":      []string{mongodb.GetEnvHost() + ":" + mongodb.GetEnvPort()},
-	}
-}
-
-func initiateReplicaSet(t *testing.T) error {
-	url := getConfig()["hosts"].([]string)[0]
-
+func initiateReplicaSet(t *testing.T, url string) error {
 	dialInfo, err := mgo.ParseURL(url)
 	if err != nil {
 		return err
diff --git a/metricbeat/module/mongodb/status/status_integration_test.go b/metricbeat/module/mongodb/status/status_integration_test.go
index 2140b0c181b..91eab6daf6c 100644
--- a/metricbeat/module/mongodb/status/status_integration_test.go
+++ b/metricbeat/module/mongodb/status/status_integration_test.go
@@ -27,47 +27,36 @@ import (
 	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	"github.com/elastic/beats/metricbeat/module/mongodb"
+	"github.com/elastic/beats/metricbeat/module/mongodb/mtest"
 )
 
-func TestFetch(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "mongodb")
-
-	f := mbtest.NewEventFetcher(t, getConfig())
-	event, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-
-	// Check event fields
-	current := event["connections"].(common.MapStr)["current"].(int64)
-	assert.True(t, current >= 0)
-
-	available := event["connections"].(common.MapStr)["available"].(int64)
-	assert.True(t, available > 0)
-
-	pageFaults := event["extra_info"].(common.MapStr)["page_faults"].(int64)
-	assert.True(t, pageFaults >= 0)
-}
-
-func TestData(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "mongodb")
-
-	f := mbtest.NewEventFetcher(t, getConfig())
-	err := mbtest.WriteEvent(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
-}
-
-func getConfig() map[string]interface{} {
-	return map[string]interface{}{
-		"module":     "mongodb",
-		"metricsets": []string{"status"},
-		"hosts":      []string{mongodb.GetEnvHost() + ":" + mongodb.GetEnvPort()},
-	}
+func TestStatus(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, mtest.GetConfig("status", r.Host()))
+			event, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+
+			// Check event fields
+			current := event["connections"].(common.MapStr)["current"].(int64)
+			assert.True(t, current >= 0)
+
+			available := event["connections"].(common.MapStr)["available"].(int64)
+			assert.True(t, available > 0)
+
+			pageFaults := event["extra_info"].(common.MapStr)["page_faults"].(int64)
+			assert.True(t, pageFaults >= 0)
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, mtest.GetConfig("status", r.Host()))
+			err := mbtest.WriteEvent(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }

From 19737034d3065f7ba88d78f081e767ca6c768368 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Mon, 12 Nov 2018 17:35:08 +0100
Subject: [PATCH 67/88] Remove unused EnsureUp

---
 libbeat/tests/compose/compose.go | 43 --------------------------------
 1 file changed, 43 deletions(-)

diff --git a/libbeat/tests/compose/compose.go b/libbeat/tests/compose/compose.go
index de1b19ebd8f..0afb9a55833 100644
--- a/libbeat/tests/compose/compose.go
+++ b/libbeat/tests/compose/compose.go
@@ -21,51 +21,8 @@ import (
 	"errors"
 	"os"
 	"path/filepath"
-	"strconv"
-	"testing"
 )
 
-// EnsureUp starts all the requested services (must be defined in docker-compose.yml)
-// with a default timeout of 300 seconds
-func EnsureUp(t *testing.T, services ...string) {
-	EnsureUpWithTimeout(t, 300, services...)
-}
-
-// EnsureUpWithTimeout starts all the requested services (must be defined in docker-compose.yml)
-// Wait for `timeout` seconds for health
-func EnsureUpWithTimeout(t *testing.T, timeout int, services ...string) {
-	// The NO_COMPOSE env variables makes it possible to skip the starting of the environment.
-	// This is useful if the service is already running locally.
-	if noCompose, err := strconv.ParseBool(os.Getenv("NO_COMPOSE")); err == nil && noCompose {
-		return
-	}
-
-	name := os.Getenv("DOCKER_COMPOSE_PROJECT_NAME")
-	compose, err := getComposeProject(name)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Kill no longer used containers
-	err = compose.KillOld(services)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for _, service := range services {
-		err = compose.Start(service)
-		if err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	// Wait for health
-	err = compose.Wait(timeout, services...)
-	if err != nil {
-		t.Fatal(err)
-	}
-}
-
 func findComposePath() (string, error) {
 	// find docker-compose
 	path, err := os.Getwd()

From 7a4a29d0c29d181a0a62e70ef0757e87b3dbebaf Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Mon, 12 Nov 2018 18:11:14 +0100
Subject: [PATCH 68/88] Fix default description

---
 libbeat/tests/compose/runner.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libbeat/tests/compose/runner.go b/libbeat/tests/compose/runner.go
index 1123f29a5b6..51794fdd468 100644
--- a/libbeat/tests/compose/runner.go
+++ b/libbeat/tests/compose/runner.go
@@ -122,7 +122,7 @@ func (r *TestRunner) Run(t *testing.T, tests Suite) {
 		sort.Strings(vars)
 		desc := strings.Join(vars, ",")
 		if desc == "" {
-			desc = "WithoutVars"
+			desc = "WithoutOptions"
 		}
 
 		seq := make([]byte, 16)

From 78a9fe18f24ec0b81ba954e048bb18d8d221ed9a Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Mon, 12 Nov 2018 18:46:55 +0100
Subject: [PATCH 69/88] Allow to use runner ctl as T

---
 libbeat/tests/compose/runner.go | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/libbeat/tests/compose/runner.go b/libbeat/tests/compose/runner.go
index 51794fdd468..d9746c62dc0 100644
--- a/libbeat/tests/compose/runner.go
+++ b/libbeat/tests/compose/runner.go
@@ -41,7 +41,7 @@ type TestRunner struct {
 	Timeout  int
 }
 
-type Suite map[string]func(t *testing.T, r R)
+type Suite map[string]interface{}
 
 type RunnerOptions map[string][]string
 
@@ -76,7 +76,16 @@ func (r *TestRunner) scenarios() []map[string]string {
 
 func (r *TestRunner) runSuite(t *testing.T, tests Suite, ctl R) {
 	for name, test := range tests {
-		t.Run(name, func(t *testing.T) { test(t, ctl) })
+		var testFunc func(t *testing.T)
+		switch f := test.(type) {
+		case func(R):
+			testFunc = func(t *testing.T) { f(ctl) }
+		case func(*testing.T, R):
+			testFunc = func(t *testing.T) { f(t, ctl) }
+		default:
+			t.Fatalf("incorrect test suite function '%s'", name)
+		}
+		t.Run(name, testFunc)
 	}
 }
 
@@ -90,8 +99,8 @@ func (r *TestRunner) runHostOverride(t *testing.T, tests Suite) bool {
 	t.Logf("Test host overriden by %s=%s", env, host)
 
 	ctl := &runnerControl{
+		T:    t,
 		host: host,
-		t:    t,
 	}
 	r.runSuite(t, tests, ctl)
 	return true
@@ -159,8 +168,8 @@ func (r *TestRunner) Run(t *testing.T, tests Suite) {
 			}
 
 			ctl := &runnerControl{
+				T:        t,
 				host:     host,
-				t:        t,
 				scenario: s,
 			}
 			r.runSuite(t, tests, ctl)
@@ -169,7 +178,10 @@ func (r *TestRunner) Run(t *testing.T, tests Suite) {
 	}
 }
 
+// R extends the testing.T interface with methods that expose information about current scenario
 type R interface {
+	testing.TB
+
 	Host() string
 	Hostname() string
 	Port() string
@@ -178,7 +190,8 @@ type R interface {
 }
 
 type runnerControl struct {
-	t        *testing.T
+	*testing.T
+
 	host     string
 	scenario map[string]string
 }

From 605e642a468c7cf64704a619515c732c41d6f1ef Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Mon, 12 Nov 2018 20:04:00 +0100
Subject: [PATCH 70/88] Get compose hosts from docker in python tests

---
 libbeat/tests/system/beat/compose.py          | 28 +++++++++++++------
 metricbeat/docker-compose.yml                 | 18 ++++++++++++
 .../elasticsearch/test_elasticsearch.py       |  3 +-
 metricbeat/tests/system/test_aerospike.py     |  3 +-
 metricbeat/tests/system/test_apache.py        |  7 +----
 metricbeat/tests/system/test_base.py          | 10 ++-----
 metricbeat/tests/system/test_ceph.py          |  3 +-
 metricbeat/tests/system/test_config.py        |  2 +-
 metricbeat/tests/system/test_couchbase.py     |  3 +-
 metricbeat/tests/system/test_dropwizard.py    |  3 +-
 metricbeat/tests/system/test_envoyproxy.py    |  3 +-
 metricbeat/tests/system/test_haproxy.py       |  8 +++---
 metricbeat/tests/system/test_http.py          |  3 +-
 metricbeat/tests/system/test_jolokia.py       |  3 +-
 metricbeat/tests/system/test_kafka.py         |  3 +-
 metricbeat/tests/system/test_kibana.py        |  3 +-
 metricbeat/tests/system/test_kubernetes.py    | 12 ++------
 metricbeat/tests/system/test_logstash.py      |  3 +-
 metricbeat/tests/system/test_memcached.py     |  3 +-
 metricbeat/tests/system/test_mongodb.py       |  3 +-
 metricbeat/tests/system/test_munin.py         |  3 +-
 metricbeat/tests/system/test_mysql.py         |  4 +--
 metricbeat/tests/system/test_phpfpm.py        |  3 +-
 metricbeat/tests/system/test_postgresql.py    |  9 +++---
 metricbeat/tests/system/test_prometheus.py    |  3 +-
 metricbeat/tests/system/test_redis.py         |  8 +++---
 metricbeat/tests/system/test_uwsgi.py         |  6 ++--
 metricbeat/tests/system/test_zookeeper.py     |  3 +-
 28 files changed, 76 insertions(+), 87 deletions(-)

diff --git a/libbeat/tests/system/beat/compose.py b/libbeat/tests/system/beat/compose.py
index 54387f006df..c322c4e34ce 100644
--- a/libbeat/tests/system/beat/compose.py
+++ b/libbeat/tests/system/beat/compose.py
@@ -89,21 +89,33 @@ def compose_down(cls):
         """
         Stop all running containers
         """
+        if os.environ.get('NO_COMPOSE'):
+            return
+
         if INTEGRATION_TESTS and cls.COMPOSE_SERVICES:
             cls.compose_project().kill(service_names=cls.COMPOSE_SERVICES)
 
     @classmethod
-    def compose_hosts(cls):
+    def compose_host(cls, service=None, port=None):
         if not INTEGRATION_TESTS or not cls.COMPOSE_SERVICES:
             return []
 
-        hosts = []
-        for container in cls.compose_project().containers(service_names=cls.COMPOSE_SERVICES):
-            network_settings = container.inspect()['NetworkSettings']
-            for network in network_settings['Networks'].values():
-                if network['IPAddress']:
-                    hosts.append(network['IPAddress'])
-        return hosts
+        if service is None:
+            service = cls.COMPOSE_SERVICES[0]
+
+        host_env = os.environ.get(service.upper() + "_HOST")
+        if host_env:
+            return host_env
+
+        container = cls.compose_project().containers(service_names=[service])[0]
+        info = container.inspect()
+
+        if port is None:
+            portsConfig = info['HostConfig']['PortBindings']
+            port = portsConfig.keys()[0]
+
+        hostPort = info['NetworkSettings']['Ports'][port][0]['HostPort']
+        return "localhost:%s" % hostPort
 
     @classmethod
     def compose_project(cls):
diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 43476b46627..6bfcafb656b 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -34,9 +34,13 @@ services:
 
   ceph:
     build: ./module/ceph/_meta
+    ports:
+      - 5000
 
   couchbase:
     build: ./module/couchbase/_meta
+    ports:
+      - 8091
 
   dropwizard:
     build: ./module/dropwizard/_meta
@@ -60,16 +64,28 @@ services:
 
   haproxy:
     build: ./module/haproxy/_meta
+    ports:
+      - 14567
+      - 14568
+      - 14569
 
   haproxy_1_6:
     build:
       context: ./module/haproxy/_meta
       dockerfile: Dockerfile.1.6
+    ports:
+      - 14567
+      - 14568
+      - 14569
 
   haproxy_1_7:
     build:
       context: ./module/haproxy/_meta
       dockerfile: Dockerfile.1.7
+    ports:
+      - 14567
+      - 14568
+      - 14569
 
   http:
     build: ./module/http/_meta
@@ -156,6 +172,8 @@ services:
 
   munin:
     build: ./module/munin/_meta
+    ports:
+      - 4949
 
   mysql:
     build: ./module/mysql/_meta
diff --git a/metricbeat/module/elasticsearch/test_elasticsearch.py b/metricbeat/module/elasticsearch/test_elasticsearch.py
index fbc24822c51..308f77d1c7a 100644
--- a/metricbeat/module/elasticsearch/test_elasticsearch.py
+++ b/metricbeat/module/elasticsearch/test_elasticsearch.py
@@ -45,8 +45,7 @@ def test_metricsets(self, metricset):
                              ["service.name"], extras={"index_recovery.active_only": "false"})
 
     def get_hosts(self):
-        return [os.getenv('ES_HOST', self.compose_hosts()[0]) + ':' +
-                os.getenv('ES_PORT', '9200')]
+        return [self.compose_host()]
 
     def create_ml_job(self):
         es = Elasticsearch(self.get_hosts())
diff --git a/metricbeat/tests/system/test_aerospike.py b/metricbeat/tests/system/test_aerospike.py
index 9d44794603a..6f118fda0d8 100644
--- a/metricbeat/tests/system/test_aerospike.py
+++ b/metricbeat/tests/system/test_aerospike.py
@@ -16,5 +16,4 @@ def test_namespace(self):
         self.check_metricset("aerospike", "namespace", self.get_hosts(), self.FIELDS)
 
     def get_hosts(self):
-        return [os.getenv('AEROSPIKE_HOST', self.compose_hosts()[0]) + ':' +
-                os.getenv('AEROSPIKE_PORT', '3000')]
+        return [self.compose_host()]
diff --git a/metricbeat/tests/system/test_apache.py b/metricbeat/tests/system/test_apache.py
index a74f7b98089..41fa7b31300 100644
--- a/metricbeat/tests/system/test_apache.py
+++ b/metricbeat/tests/system/test_apache.py
@@ -78,8 +78,7 @@ def verify_fields(self, evt):
         # There are more fields that could be checked.
 
     def get_hosts(self):
-        return ['http://' + os.getenv('APACHE_HOST', self.compose_hosts()[0]) + ':' +
-                os.getenv('APACHE_PORT', '80')]
+        return ['http://' + self.compose_host()]
 
 
 class ApacheOldStatusTest(ApacheStatusTest):
@@ -91,7 +90,3 @@ def verify_fields(self, evt):
         apache_status = evt["apache"]["status"]
         self.assertItemsEqual(
             self.de_dot(APACHE_OLD_STATUS_FIELDS), apache_status.keys())
-
-    def get_hosts(self):
-        return ['http://' + os.getenv('APACHE_OLD_HOST', self.compose_hosts()[0]) + ':' +
-                os.getenv('APACHE_PORT', '80')]
diff --git a/metricbeat/tests/system/test_base.py b/metricbeat/tests/system/test_base.py
index 770ea0ef716..cb50d0cb835 100644
--- a/metricbeat/tests/system/test_base.py
+++ b/metricbeat/tests/system/test_base.py
@@ -78,16 +78,10 @@ def test_dashboards(self):
         assert self.log_contains("Kibana dashboards successfully loaded.")
 
     def get_elasticsearch_url(self):
-        return "http://{host}:{port}".format(
-            host=os.getenv("ES_HOST", self.compose_hosts()[1]),
-            port=os.getenv("ES_PORT", "9200"),
-        )
+        return "http://" + self.compose_host("elasticsearch")
 
     def get_kibana_url(self):
         """
         Returns kibana host URL
         """
-        return "http://{host}:{port}".format(
-            host=os.getenv("KIBANA_HOST", self.compose_hosts()[0]),
-            port=os.getenv("KIBANA_PORT", "5601"),
-        )
+        return "http://" + self.compose_host("kibana")
diff --git a/metricbeat/tests/system/test_ceph.py b/metricbeat/tests/system/test_ceph.py
index fb90acf8a28..0cd10b29a22 100644
--- a/metricbeat/tests/system/test_ceph.py
+++ b/metricbeat/tests/system/test_ceph.py
@@ -24,5 +24,4 @@ def test_ceph(self, metricset):
         self.check_metricset("ceph", metricset, self.get_hosts(), self.FIELDS)
 
     def get_hosts(self):
-        return [os.getenv('CEPH_HOST', self.compose_hosts()[0]) + ':' +
-                os.getenv('CEPH_PORT', '5000')]
+        return [self.compose_host()]
diff --git a/metricbeat/tests/system/test_config.py b/metricbeat/tests/system/test_config.py
index 0804e5e0bee..d4ac45a9da4 100644
--- a/metricbeat/tests/system/test_config.py
+++ b/metricbeat/tests/system/test_config.py
@@ -27,4 +27,4 @@ def test_invalid_config_with_removed_settings(self):
                                  " has been removed")
 
     def get_host(self):
-        return 'http://' + os.getenv('ES_HOST', self.compose_hosts()[0]) + ':' + os.getenv('ES_PORT', '9200')
+        return 'http://' + self.compose_host()
diff --git a/metricbeat/tests/system/test_couchbase.py b/metricbeat/tests/system/test_couchbase.py
index 49d11b3f21b..f8b2f619e09 100644
--- a/metricbeat/tests/system/test_couchbase.py
+++ b/metricbeat/tests/system/test_couchbase.py
@@ -22,5 +22,4 @@ def test_couchbase(self, metricset):
         self.check_metricset("couchbase", metricset, self.get_hosts(), self.FIELDS)
 
     def get_hosts(self):
-        return [os.getenv('COUCHBASE_HOST', self.compose_hosts()[0]) + ':' +
-                os.getenv('COUCHBASE_PORT', '8091')]
+        return [self.compose_host()]
diff --git a/metricbeat/tests/system/test_dropwizard.py b/metricbeat/tests/system/test_dropwizard.py
index 651fb4968ec..34297426f2d 100644
--- a/metricbeat/tests/system/test_dropwizard.py
+++ b/metricbeat/tests/system/test_dropwizard.py
@@ -30,5 +30,4 @@ def test_dropwizard(self):
         self.assertTrue(len(output) >= 1)
 
     def get_hosts(self):
-        return [os.getenv('DROPWIZARD_HOST', self.compose_hosts()[0]) + ':' +
-                os.getenv('DROPWIZARD_PORT', '8080')]
+        return [self.compose_host()]
diff --git a/metricbeat/tests/system/test_envoyproxy.py b/metricbeat/tests/system/test_envoyproxy.py
index b30bcac1e20..013cbb13aa4 100644
--- a/metricbeat/tests/system/test_envoyproxy.py
+++ b/metricbeat/tests/system/test_envoyproxy.py
@@ -31,5 +31,4 @@ def test_stats(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return [os.getenv('ENVOYPROXY_HOST', self.compose_hosts()[0]) + ':' +
-                os.getenv('ENVOYPROXY_PORT', '9901')]
+        return [self.compose_host()]
diff --git a/metricbeat/tests/system/test_haproxy.py b/metricbeat/tests/system/test_haproxy.py
index 8b7dd6e976f..9a3c0f192f8 100644
--- a/metricbeat/tests/system/test_haproxy.py
+++ b/metricbeat/tests/system/test_haproxy.py
@@ -32,7 +32,7 @@ def test_info_socket(self):
         self.render_config_template(modules=[{
             "name": "haproxy",
             "metricsets": ["info"],
-            "hosts": ["tcp://%s:%d" % (self.compose_hosts()[0], 14567)],
+            "hosts": ["tcp://%s" % (self.compose_host(port="14567/tcp"))],
             "period": "5s"
         }])
         self._test_info()
@@ -59,7 +59,7 @@ def test_stat_socket(self):
         self.render_config_template(modules=[{
             "name": "haproxy",
             "metricsets": ["stat"],
-            "hosts": ["tcp://%s:%d" % (self.compose_hosts()[0], 14567)],
+            "hosts": ["tcp://%s" % (self.compose_host(port="14567/tcp"))],
             "period": "5s"
         }])
         self._test_stat()
@@ -72,7 +72,7 @@ def test_stat_http(self):
         self.render_config_template(modules=[{
             "name": "haproxy",
             "metricsets": ["stat"],
-            "hosts": ["http://%s:%d/stats" % (self.compose_hosts()[0], 14568)],
+            "hosts": ["http://%s/stats" % (self.compose_host(port="14568/tcp"))],
             "period": "5s"
         }])
         self._test_stat()
@@ -87,7 +87,7 @@ def test_stat_http_auth(self):
             "metricsets": ["stat"],
             "username": "admin",
             "password": "admin",
-            "hosts": ["http://%s:%d/stats" % (self.compose_hosts()[0], 14569)],
+            "hosts": ["http://%s/stats" % (self.compose_host(port="14569/tcp"))],
             "period": "5s"
         }])
         self._test_stat()
diff --git a/metricbeat/tests/system/test_http.py b/metricbeat/tests/system/test_http.py
index 64e6239be92..c4b343572a2 100644
--- a/metricbeat/tests/system/test_http.py
+++ b/metricbeat/tests/system/test_http.py
@@ -76,5 +76,4 @@ def test_server(self):
         self.assert_fields_are_documented(evt)
 
     def get_host(self):
-        host = self.compose_hosts()[0]
-        return "http://" + os.getenv('HTTP_HOST', host) + ':' + os.getenv('HTTP_PORT', '8080')
+        return "http://" + self.compose_host()
diff --git a/metricbeat/tests/system/test_jolokia.py b/metricbeat/tests/system/test_jolokia.py
index 3ca67699956..b44f03985b6 100644
--- a/metricbeat/tests/system/test_jolokia.py
+++ b/metricbeat/tests/system/test_jolokia.py
@@ -50,5 +50,4 @@ def test_jmx(self, mbean):
         assert evt["jolokia"]["test"]["gc"]["collection_count"] >= 0
 
     def get_hosts(self):
-        return [os.getenv('JOLOKIA_HOST', self.compose_hosts()[0]) + ':' +
-                os.getenv('JOLOKIA_PORT', '8778')]
+        return [self.compose_host()]
diff --git a/metricbeat/tests/system/test_kafka.py b/metricbeat/tests/system/test_kafka.py
index a7ba2d26e28..4f030515dea 100644
--- a/metricbeat/tests/system/test_kafka.py
+++ b/metricbeat/tests/system/test_kafka.py
@@ -43,8 +43,7 @@ def create_topic(self):
         producer.send('foobar', b'some_message_bytes')
 
     def get_hosts(self):
-        return [self.compose_hosts()[0] + ':' +
-                os.getenv('KAFKA_PORT', '9092')]
+        return [self.compose_host()]
 
 
 class Kafka_1_1_0_Test(KafkaTest):
diff --git a/metricbeat/tests/system/test_kibana.py b/metricbeat/tests/system/test_kibana.py
index 456588201ae..33f6f376c1b 100644
--- a/metricbeat/tests/system/test_kibana.py
+++ b/metricbeat/tests/system/test_kibana.py
@@ -48,8 +48,7 @@ def test_status(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return [os.getenv('KIBANA_HOST', self.compose_hosts()[0]) + ':' +
-                os.getenv('KIBANA_PORT', '5601')]
+        return [self.compose_host("kibana")]
 
     def get_version(self):
         host = self.get_hosts()[0]
diff --git a/metricbeat/tests/system/test_kubernetes.py b/metricbeat/tests/system/test_kubernetes.py
index abb14161474..5c469bc3902 100644
--- a/metricbeat/tests/system/test_kubernetes.py
+++ b/metricbeat/tests/system/test_kubernetes.py
@@ -77,16 +77,8 @@ def _test_metricset(self, metricset, expected_events, hosts):
 
     @classmethod
     def get_kubelet_hosts(cls):
-        return [
-            "http://" +
-            os.getenv('KUBELET_HOST', self.compose_hosts()[0]) + ':' +
-            os.getenv('KUBELET_PORT', '10255')
-        ]
+        return [self.compose_host("kubernetes")]
 
     @classmethod
     def get_kube_state_hosts(cls):
-        return [
-            "http://" +
-            os.getenv('KUBE_STATE_METRICS_HOST', self.compose_hosts()[1]) + ':' +
-            os.getenv('KUBE_STATE_METRICS_PORT', '18080')
-        ]
+        return [self.compose_host("kubestate")]
diff --git a/metricbeat/tests/system/test_logstash.py b/metricbeat/tests/system/test_logstash.py
index 816aaf11cd7..2f6c5b2c38d 100644
--- a/metricbeat/tests/system/test_logstash.py
+++ b/metricbeat/tests/system/test_logstash.py
@@ -55,5 +55,4 @@ def test_node_stats(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return [os.getenv('LOGSTASH_HOST', self.compose_hosts()[0]) + ':' +
-                os.getenv('LOGSTASH_PORT', '9600')]
+        return [self.compose_host()]
diff --git a/metricbeat/tests/system/test_memcached.py b/metricbeat/tests/system/test_memcached.py
index c41d34b7cc6..38c325a29f0 100644
--- a/metricbeat/tests/system/test_memcached.py
+++ b/metricbeat/tests/system/test_memcached.py
@@ -31,5 +31,4 @@ def test_stats(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return [os.getenv('MEMCACHED_HOST', self.compose_hosts()[0]) + ':' +
-                os.getenv('MEMCACHED_PORT', '11211')]
+        return [self.compose_host()]
diff --git a/metricbeat/tests/system/test_mongodb.py b/metricbeat/tests/system/test_mongodb.py
index 88327e80a23..257721b3bc5 100644
--- a/metricbeat/tests/system/test_mongodb.py
+++ b/metricbeat/tests/system/test_mongodb.py
@@ -36,5 +36,4 @@ def test_status(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return [os.getenv('MONGODB_HOST', self.compose_hosts()[0]) + ':' +
-                os.getenv('MONGODB_PORT', '27017')]
+        return [self.compose_host()]
diff --git a/metricbeat/tests/system/test_munin.py b/metricbeat/tests/system/test_munin.py
index 391d8f845d5..049dc9c63fd 100644
--- a/metricbeat/tests/system/test_munin.py
+++ b/metricbeat/tests/system/test_munin.py
@@ -34,5 +34,4 @@ def test_munin_node(self):
         assert evt["munin"][namespace]["cpu"]["user"] > 0
 
     def get_hosts(self):
-        return [os.getenv('MUNIN_HOST', self.compose_hosts()[0]) + ':' +
-                os.getenv('MUNIN_PORT', '4949')]
+        return [self.compose_host()]
diff --git a/metricbeat/tests/system/test_mysql.py b/metricbeat/tests/system/test_mysql.py
index 4087b141e49..244d5aefa7b 100644
--- a/metricbeat/tests/system/test_mysql.py
+++ b/metricbeat/tests/system/test_mysql.py
@@ -40,6 +40,4 @@ def test_status(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        host = os.getenv('MYSQL_HOST', self.compose_hosts()[0])
-        port = os.getenv('MYSQL_PORT', '3306')
-        return [os.getenv('MYSQL_DSN', 'root:test@tcp({}:{})/'.format(host, port))]
+        return ['root:test@tcp({})/'.format(self.compose_host())]
diff --git a/metricbeat/tests/system/test_phpfpm.py b/metricbeat/tests/system/test_phpfpm.py
index 4049f96748a..df20358e9d7 100644
--- a/metricbeat/tests/system/test_phpfpm.py
+++ b/metricbeat/tests/system/test_phpfpm.py
@@ -34,5 +34,4 @@ def test_info(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return [os.getenv('PHPFPM_HOST', self.compose_hosts()[0]) + ':' +
-                os.getenv('PHPFPM_PORT', '81')]
+        return [self.compose_host()]
diff --git a/metricbeat/tests/system/test_postgresql.py b/metricbeat/tests/system/test_postgresql.py
index 6f2e1193745..43a8db4a385 100644
--- a/metricbeat/tests/system/test_postgresql.py
+++ b/metricbeat/tests/system/test_postgresql.py
@@ -20,12 +20,11 @@ def common_checks(self, output):
 
     def get_hosts(self):
         username = "postgres"
-        host = self.compose_hosts()[0]
-        port = 5432
-        dsn = "postgres://{}:{}?sslmode=disable".format(host, port)
+        host = self.compose_host()
+        dsn = "postgres://{}?sslmode=disable".format(host)
         return (
-            [os.getenv("POSTGRESQL_DSN", dsn)],
-            os.getenv("POSTGRESQL_USERNAME", username),
+            [dsn],
+            username,
             os.getenv("POSTGRESQL_PASSWORD"),
         )
 
diff --git a/metricbeat/tests/system/test_prometheus.py b/metricbeat/tests/system/test_prometheus.py
index ccf6bbf220f..dc94b4b9144 100644
--- a/metricbeat/tests/system/test_prometheus.py
+++ b/metricbeat/tests/system/test_prometheus.py
@@ -34,5 +34,4 @@ def test_stats(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return ["http://" + os.getenv('PROMETHEUS_HOST', self.compose_hosts()[0]) + ':' +
-                os.getenv('PROMETHEUS_PORT', '9090')]
+        return [self.compose_host()]
diff --git a/metricbeat/tests/system/test_redis.py b/metricbeat/tests/system/test_redis.py
index a1404390860..728ab5b58a6 100644
--- a/metricbeat/tests/system/test_redis.py
+++ b/metricbeat/tests/system/test_redis.py
@@ -59,9 +59,10 @@ def test_keyspace(self):
         """
 
         # At least one event must be inserted so db stats exist
+        host, port = self.compose_host().split(":")
         r = redis.StrictRedis(
-            host=os.getenv('REDIS_HOST', self.compose_hosts()[0]),
-            port=os.getenv('REDIS_PORT', '6379'),
+            host=host,
+            port=port,
             db=0)
         r.set('foo', 'bar')
 
@@ -120,8 +121,7 @@ def test_module_processors(self):
         self.assertItemsEqual(self.de_dot(CPU_FIELDS), redis_info["cpu"].keys())
 
     def get_hosts(self):
-        return [os.getenv('REDIS_HOST', self.compose_hosts()[0]) + ':' +
-                os.getenv('REDIS_PORT', '6379')]
+        return [self.compose_host()]
 
 
 class TestRedis4(Test):
diff --git a/metricbeat/tests/system/test_uwsgi.py b/metricbeat/tests/system/test_uwsgi.py
index 7c4bfd79dde..a35576fb85e 100644
--- a/metricbeat/tests/system/test_uwsgi.py
+++ b/metricbeat/tests/system/test_uwsgi.py
@@ -68,13 +68,11 @@ def test_status(self):
         self.common_checks(output)
 
     def get_host(self):
-        return os.getenv("UWSGI_STAT_TCP_SERVER",
-                         "tcp://{}:9191".format(self.compose_hosts()[0]))
+        return "tcp://" + self.compose_host()
 
 
 class TestHTTP(Test):
     COMPOSE_SERVICES = ['uwsgi_http']
 
     def get_host(self):
-        return os.getenv("UWSGI_STAT_HTTP_SERVER",
-                         "http://{}:9191".format(self.compose_hosts()[0]))
+        return "http://" + self.compose_host()
diff --git a/metricbeat/tests/system/test_zookeeper.py b/metricbeat/tests/system/test_zookeeper.py
index 6d35f9d4d4c..23260a3eb2d 100644
--- a/metricbeat/tests/system/test_zookeeper.py
+++ b/metricbeat/tests/system/test_zookeeper.py
@@ -51,5 +51,4 @@ def test_output(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return [os.getenv('ZOOKEEPER_HOST', self.compose_hosts()[0]) + ':' +
-                os.getenv('ZOOKEEPER_PORT', '2181')]
+        return [self.compose_host()]

From 3fc2715c3f09225eb859ad64a300b7bed6c585df Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Wed, 14 Nov 2018 16:47:46 +0100
Subject: [PATCH 71/88] Fix traefik test

---
 metricbeat/module/traefik/test_traefik.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/metricbeat/module/traefik/test_traefik.py b/metricbeat/module/traefik/test_traefik.py
index be42e4d676f..50a5df27670 100644
--- a/metricbeat/module/traefik/test_traefik.py
+++ b/metricbeat/module/traefik/test_traefik.py
@@ -25,5 +25,4 @@ def test_health(self, metricset):
         self.check_metricset("traefik", metricset, self.get_hosts(), self.FIELDS + ["service.name"])
 
     def get_hosts(self):
-        return [os.getenv('TRAEFIK_HOST', self.compose_hosts()[0]) + ':' +
-                os.getenv('TRAEFIK_API_PORT', '8080')]
+        return [self.compose_host()]

From 58c4e28e6f716689a656d45d7979c8e14a32cb52 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Thu, 15 Nov 2018 20:11:20 +0100
Subject: [PATCH 72/88] Add also implementation for private port to python

---
 libbeat/tests/system/beat/compose.py | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/libbeat/tests/system/beat/compose.py b/libbeat/tests/system/beat/compose.py
index c322c4e34ce..ad6b2e2f6f0 100644
--- a/libbeat/tests/system/beat/compose.py
+++ b/libbeat/tests/system/beat/compose.py
@@ -95,6 +95,20 @@ def compose_down(cls):
         if INTEGRATION_TESTS and cls.COMPOSE_SERVICES:
             cls.compose_project().kill(service_names=cls.COMPOSE_SERVICES)
 
+    @classmethod
+    def _private_host(cls, info, port):
+        networks = info['NetworkSettings']['Networks'].values()
+        port = port.split("/")[0]
+        for network in networks:
+            ip = network['IPAddress']
+            if ip:
+                return "%s:%s" % (ip, port)
+
+    @classmethod
+    def _public_host(cls, info, port):
+        hostPort = info['NetworkSettings']['Ports'][port][0]['HostPort']
+        return "localhost:%s" % hostPort
+
     @classmethod
     def compose_host(cls, service=None, port=None):
         if not INTEGRATION_TESTS or not cls.COMPOSE_SERVICES:
@@ -109,13 +123,11 @@ def compose_host(cls, service=None, port=None):
 
         container = cls.compose_project().containers(service_names=[service])[0]
         info = container.inspect()
-
         if port is None:
             portsConfig = info['HostConfig']['PortBindings']
             port = portsConfig.keys()[0]
 
-        hostPort = info['NetworkSettings']['Ports'][port][0]['HostPort']
-        return "localhost:%s" % hostPort
+        return cls._public_host(info, port)
 
     @classmethod
     def compose_project(cls):

From 58981e54b610d4cde3e6197813a9e84b495618c1 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 20 Nov 2018 10:49:37 +0100
Subject: [PATCH 73/88] Better error message if no port is configured

---
 libbeat/tests/system/beat/compose.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/libbeat/tests/system/beat/compose.py b/libbeat/tests/system/beat/compose.py
index ad6b2e2f6f0..cbbca55947c 100644
--- a/libbeat/tests/system/beat/compose.py
+++ b/libbeat/tests/system/beat/compose.py
@@ -123,8 +123,10 @@ def compose_host(cls, service=None, port=None):
 
         container = cls.compose_project().containers(service_names=[service])[0]
         info = container.inspect()
+        portsConfig = info['HostConfig']['PortBindings']
+        if len(portsConfig) == 0:
+            raise Exception("No exposed ports for service %s" % service)
         if port is None:
-            portsConfig = info['HostConfig']['PortBindings']
             port = portsConfig.keys()[0]
 
         return cls._public_host(info, port)

From a0d8c3ecdd7bd686a483a2f31f91e21eed923261 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 20 Nov 2018 10:51:05 +0100
Subject: [PATCH 74/88] Fix etcd system tests

---
 metricbeat/docker-compose.yml       | 2 ++
 metricbeat/module/etcd/test_etcd.py | 3 +--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 7e3f66d62fa..8364e7eeec2 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -67,6 +67,8 @@ services:
       context: ./module/etcd/_meta
       args:
         ETCD_VERSION: v3.2.25
+    ports:
+      - 2379
 
   haproxy:
     build: ./module/haproxy/_meta
diff --git a/metricbeat/module/etcd/test_etcd.py b/metricbeat/module/etcd/test_etcd.py
index 9441c6e6d75..98118594767 100644
--- a/metricbeat/module/etcd/test_etcd.py
+++ b/metricbeat/module/etcd/test_etcd.py
@@ -25,8 +25,7 @@ def test_metricset(self, metricset):
         self.check_metricset("etcd", metricset, self.get_hosts(), ['etcd.' + metricset])
 
     def get_hosts(self):
-        return [self.compose_hosts()[0] + ':' +
-                os.getenv('ETCD_PORT', '2379')]
+        return [self.compose_host()]
 
 
 class Test_3_2(Test):

From 95696e5f8662bbb3d8511db97c2f5387e326a11d Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Thu, 22 Nov 2018 16:22:48 +0100
Subject: [PATCH 75/88] Properly wrap testing.T into runner control

---
 libbeat/tests/compose/runner.go | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/libbeat/tests/compose/runner.go b/libbeat/tests/compose/runner.go
index d9746c62dc0..233caf675cb 100644
--- a/libbeat/tests/compose/runner.go
+++ b/libbeat/tests/compose/runner.go
@@ -79,9 +79,11 @@ func (r *TestRunner) runSuite(t *testing.T, tests Suite, ctl R) {
 		var testFunc func(t *testing.T)
 		switch f := test.(type) {
 		case func(R):
-			testFunc = func(t *testing.T) { f(ctl) }
+			testFunc = func(t *testing.T) { f(ctl.WithT(t)) }
 		case func(*testing.T, R):
-			testFunc = func(t *testing.T) { f(t, ctl) }
+			testFunc = func(t *testing.T) { f(t, ctl.WithT(t)) }
+		case func(*testing.T):
+			testFunc = func(t *testing.T) { f(t) }
 		default:
 			t.Fatalf("incorrect test suite function '%s'", name)
 		}
@@ -99,7 +101,6 @@ func (r *TestRunner) runHostOverride(t *testing.T, tests Suite) bool {
 	t.Logf("Test host overriden by %s=%s", env, host)
 
 	ctl := &runnerControl{
-		T:    t,
 		host: host,
 	}
 	r.runSuite(t, tests, ctl)
@@ -168,7 +169,6 @@ func (r *TestRunner) Run(t *testing.T, tests Suite) {
 			}
 
 			ctl := &runnerControl{
-				T:        t,
 				host:     host,
 				scenario: s,
 			}
@@ -182,6 +182,8 @@ func (r *TestRunner) Run(t *testing.T, tests Suite) {
 type R interface {
 	testing.TB
 
+	WithT(t *testing.T) R
+
 	Host() string
 	Hostname() string
 	Port() string
@@ -196,6 +198,12 @@ type runnerControl struct {
 	scenario map[string]string
 }
 
+func (r *runnerControl) WithT(t *testing.T) R {
+	ctl := *r
+	ctl.T = t
+	return &ctl
+}
+
 func (r *runnerControl) Host() string {
 	return r.host
 }

From 6ba4e9d683eeb60e53dd5247d06d083c68967279 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Fri, 23 Nov 2018 12:56:05 +0100
Subject: [PATCH 76/88] Move compose test functions to top-level

---
 .../namespace/namespace_integration_test.go   |  18 ++-
 .../apache/status/status_integration_test.go  |  43 +++---
 .../collector/collector_integration_test.go   |  86 +++++------
 .../elasticsearch_integration_test.go         |  84 ++++++-----
 .../server/server_integration_test.go         |  38 ++---
 .../etcd/leader/leader_integration_test.go    |  38 ++---
 .../module/etcd/self/self_integration_test.go |  38 ++---
 .../etcd/store/store_integration_test.go      |  38 ++---
 .../module/http/json/json_integration_test.go |  54 ++++---
 .../jolokia/jmx/jmx_integration_test.go       |  48 +++---
 .../consumergroup_integration_test.go         |  40 ++---
 .../partition/partition_integration_test.go   | 139 +++++++++---------
 .../kibana/stats/stats_integration_test.go    | 102 +++++++------
 .../kibana/status/status_integration_test.go  |  44 +++---
 .../logstash/logstash_integration_test.go     |  50 ++++---
 .../memcached/stats/stats_integration_test.go |  16 +-
 .../collstats/collstats_integration_test.go   |  54 +++----
 .../dbstats/dbstats_integration_test.go       | 104 ++++++-------
 .../metrics/metrics_intergration_test.go      |  50 ++++---
 .../replstatus/replstatus_integration_test.go |  89 +++++------
 .../mongodb/status/status_integration_test.go |  56 +++----
 .../module/mysql/mysql_integration_test.go    |  16 +-
 .../mysql/status/status_integration_test.go   | 110 +++++++-------
 .../stubstatus/stubstatus_integration_test.go |  40 ++---
 .../php_fpm/pool/pool_integration_test.go     |  16 +-
 .../process/process_integration_test.go       |  16 +-
 .../activity/activity_integration_test.go     |  58 ++++----
 .../bgwriter/bgwriter_integration_test.go     |  64 ++++----
 .../database/database_integration_test.go     |  64 ++++----
 .../statement/statement_integration_test.go   | 134 +++++++++--------
 .../collector/collector_integration_test.go   |  38 ++---
 .../stats/stats_integration_test.go           |  46 +++---
 .../connection/connection_integration_test.go |  16 +-
 .../exchange/exchange_integration_test.go     |  24 +--
 .../rabbitmq/node/node_integration_test.go    |  16 +-
 .../rabbitmq/queue/queue_integration_test.go  |  22 +--
 .../redis/info/info_integration_test.go       | 118 ++++++++-------
 .../keyspace/keyspace_integration_test.go     |  70 ++++-----
 .../traefik/health/health_integration_test.go |  60 ++++----
 .../uwsgi/status/status_integration_test.go   |  44 +++---
 .../zookeeper/mntr/mntr_integration_test.go   |  57 +++----
 41 files changed, 1202 insertions(+), 1056 deletions(-)

diff --git a/metricbeat/module/aerospike/namespace/namespace_integration_test.go b/metricbeat/module/aerospike/namespace/namespace_integration_test.go
index 9643548c699..843af5450da 100644
--- a/metricbeat/module/aerospike/namespace/namespace_integration_test.go
+++ b/metricbeat/module/aerospike/namespace/namespace_integration_test.go
@@ -28,15 +28,17 @@ import (
 )
 
 func TestNamespace(t *testing.T) {
-	t.Parallel()
+	mtest.Runner.Run(t, compose.Suite{
+		"Data": testData,
+	})
+}
 
-	mtest.Runner.Run(t, compose.Suite{"Data": func(t *testing.T, r compose.R) {
-		f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
-		err := mbtest.WriteEvents(f, t)
-		if err != nil {
-			t.Fatal("write", err)
-		}
-	}})
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
+	err := mbtest.WriteEvents(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
 }
 
 func getConfig(host string) map[string]interface{} {
diff --git a/metricbeat/module/apache/status/status_integration_test.go b/metricbeat/module/apache/status/status_integration_test.go
index 631bd0cc0b1..0fbdd22a410 100644
--- a/metricbeat/module/apache/status/status_integration_test.go
+++ b/metricbeat/module/apache/status/status_integration_test.go
@@ -31,30 +31,33 @@ import (
 
 func TestStatus(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+		"Data":  testData,
+		"Fetch": testFetch,
+	})
+}
 
-			err := mbtest.WriteEvent(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+
+	err := mbtest.WriteEvent(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
 
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
-			event, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+	event, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
 
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
 
-			// Check number of fields.
-			if len(event) < 11 {
-				t.Fatal("Too few top-level elements in the event")
-			}
-		},
-	})
+	// Check number of fields.
+	if len(event) < 11 {
+		t.Fatal("Too few top-level elements in the event")
+	}
 }
 
 func getConfig(host string) map[string]interface{} {
diff --git a/metricbeat/module/dropwizard/collector/collector_integration_test.go b/metricbeat/module/dropwizard/collector/collector_integration_test.go
index 8f4af491b00..f864cabe210 100644
--- a/metricbeat/module/dropwizard/collector/collector_integration_test.go
+++ b/metricbeat/module/dropwizard/collector/collector_integration_test.go
@@ -35,55 +35,59 @@ func TestCollector(t *testing.T) {
 	runner := compose.TestRunner{Service: "dropwizard"}
 
 	runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
-			events, err := f.Fetch()
+		"Fetch": testFetch,
+		"Data":  testData,
+	})
+}
 
-			hasTag := false
-			doesntHaveTag := false
-			for _, event := range events {
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
+	events, err := f.Fetch()
 
-				ok, _ := event.HasKey("my_histogram")
-				if ok {
-					_, err := event.GetValue("tags")
-					if err == nil {
-						t.Fatal("write", "my_counter not supposed to have tags")
-					}
-					doesntHaveTag = true
-				}
+	hasTag := false
+	doesntHaveTag := false
+	for _, event := range events {
 
-				ok, _ = event.HasKey("my_counter")
-				if ok {
-					tagsRaw, err := event.GetValue("tags")
-					if err != nil {
-						t.Fatal("write", err)
-					} else {
-						tags, ok := tagsRaw.(common.MapStr)
-						if !ok {
-							t.Fatal("write", "unable to cast tags to common.MapStr")
-						} else {
-							assert.Equal(t, len(tags), 1)
-							hasTag = true
-						}
-					}
-				}
-			}
-			assert.Equal(t, hasTag, true)
-			assert.Equal(t, doesntHaveTag, true)
-			if !assert.NoError(t, err) {
-				t.FailNow()
+		ok, _ := event.HasKey("my_histogram")
+		if ok {
+			_, err := event.GetValue("tags")
+			if err == nil {
+				t.Fatal("write", "my_counter not supposed to have tags")
 			}
+			doesntHaveTag = true
+		}
 
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events)
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
-			err := mbtest.WriteEvents(f, t)
+		ok, _ = event.HasKey("my_counter")
+		if ok {
+			tagsRaw, err := event.GetValue("tags")
 			if err != nil {
 				t.Fatal("write", err)
+			} else {
+				tags, ok := tagsRaw.(common.MapStr)
+				if !ok {
+					t.Fatal("write", "unable to cast tags to common.MapStr")
+				} else {
+					assert.Equal(t, len(tags), 1)
+					hasTag = true
+				}
 			}
-		},
-	})
+		}
+	}
+	assert.Equal(t, hasTag, true)
+	assert.Equal(t, doesntHaveTag, true)
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events)
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
+	err := mbtest.WriteEvents(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
 }
 
 func getConfig(host string) map[string]interface{} {
diff --git a/metricbeat/module/elasticsearch/elasticsearch_integration_test.go b/metricbeat/module/elasticsearch/elasticsearch_integration_test.go
index ec4cd3a3a96..311d964c320 100644
--- a/metricbeat/module/elasticsearch/elasticsearch_integration_test.go
+++ b/metricbeat/module/elasticsearch/elasticsearch_integration_test.go
@@ -72,49 +72,53 @@ func TestElasticsearch(t *testing.T) {
 	}
 
 	runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			if r.Option("ELASTICSEARCH_VERSION") == "6.2.4" {
-				t.Skip("This test fails on this version")
-			}
+		"Fetch": testFetch,
+		"Data":  testData,
+	})
+}
+
+func testFetch(t *testing.T, r compose.R) {
+	if r.Option("ELASTICSEARCH_VERSION") == "6.2.4" {
+		t.Skip("This test fails on this version")
+	}
+
+	host := r.Host()
+	err := createIndex(host)
+	assert.NoError(t, err)
+
+	err = enableTrialLicense(host)
+	assert.NoError(t, err)
+
+	err = createMLJob(host)
+	assert.NoError(t, err)
 
-			host := r.Host()
-			err := createIndex(host)
-			assert.NoError(t, err)
-
-			err = enableTrialLicense(host)
-			assert.NoError(t, err)
-
-			err = createMLJob(host)
-			assert.NoError(t, err)
-
-			for _, metricSet := range metricSets {
-				t.Run(metricSet, func(t *testing.T) {
-					checkSkip(t, metricSet, host)
-					f := mbtest.NewReportingMetricSetV2(t, getConfig(metricSet, host))
-					events, errs := mbtest.ReportingFetchV2(f)
-
-					assert.Empty(t, errs)
-					if !assert.NotEmpty(t, events) {
-						t.FailNow()
-					}
-					t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(),
-						events[0].BeatEvent("elasticsearch", metricSet).Fields.StringToPrint())
-				})
+	for _, metricSet := range metricSets {
+		t.Run(metricSet, func(t *testing.T) {
+			checkSkip(t, metricSet, host)
+			f := mbtest.NewReportingMetricSetV2(t, getConfig(metricSet, host))
+			events, errs := mbtest.ReportingFetchV2(f)
+
+			assert.Empty(t, errs)
+			if !assert.NotEmpty(t, events) {
+				t.FailNow()
 			}
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			for _, metricSet := range metricSets {
-				t.Run(metricSet, func(t *testing.T) {
-					checkSkip(t, metricSet, r.Host())
-					f := mbtest.NewReportingMetricSetV2(t, getConfig(metricSet, r.Host()))
-					err := mbtest.WriteEventsReporterV2(f, t, metricSet)
-					if err != nil {
-						t.Fatal("write", err)
-					}
-				})
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(),
+				events[0].BeatEvent("elasticsearch", metricSet).Fields.StringToPrint())
+		})
+	}
+}
+
+func testData(t *testing.T, r compose.R) {
+	for _, metricSet := range metricSets {
+		t.Run(metricSet, func(t *testing.T) {
+			checkSkip(t, metricSet, r.Host())
+			f := mbtest.NewReportingMetricSetV2(t, getConfig(metricSet, r.Host()))
+			err := mbtest.WriteEventsReporterV2(f, t, metricSet)
+			if err != nil {
+				t.Fatal("write", err)
 			}
-		},
-	})
+		})
+	}
 }
 
 // GetConfig returns config for elasticsearch module
diff --git a/metricbeat/module/envoyproxy/server/server_integration_test.go b/metricbeat/module/envoyproxy/server/server_integration_test.go
index 385732d257c..00e8b79a9c3 100644
--- a/metricbeat/module/envoyproxy/server/server_integration_test.go
+++ b/metricbeat/module/envoyproxy/server/server_integration_test.go
@@ -32,26 +32,30 @@ func TestEnvoyproxy(t *testing.T) {
 	runner := compose.TestRunner{Service: "envoyproxy"}
 
 	runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
-			event, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
-
-			assert.NotNil(t, event)
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
-			err := mbtest.WriteEvent(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Fetch": testFetch,
+		"Data":  testData,
 	})
 }
 
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+	event, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	assert.NotNil(t, event)
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+	err := mbtest.WriteEvent(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
+
 func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "envoyproxy",
diff --git a/metricbeat/module/etcd/leader/leader_integration_test.go b/metricbeat/module/etcd/leader/leader_integration_test.go
index bd2dd7bc519..82583abc450 100644
--- a/metricbeat/module/etcd/leader/leader_integration_test.go
+++ b/metricbeat/module/etcd/leader/leader_integration_test.go
@@ -32,26 +32,30 @@ func TestLeader(t *testing.T) {
 	runner := compose.TestRunner{Service: "etcd"}
 
 	runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
-			event, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
-
-			assert.NotNil(t, event)
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
-			err := mbtest.WriteEvent(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Fetch": testFetch,
+		"Data":  testData,
 	})
 }
 
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+	event, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	assert.NotNil(t, event)
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+	err := mbtest.WriteEvent(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
+
 func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "etcd",
diff --git a/metricbeat/module/etcd/self/self_integration_test.go b/metricbeat/module/etcd/self/self_integration_test.go
index be8b979f7c1..c8ef9f1726b 100644
--- a/metricbeat/module/etcd/self/self_integration_test.go
+++ b/metricbeat/module/etcd/self/self_integration_test.go
@@ -32,26 +32,30 @@ func TestLeader(t *testing.T) {
 	runner := compose.TestRunner{Service: "etcd"}
 
 	runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
-			event, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
-
-			assert.NotNil(t, event)
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
-			err := mbtest.WriteEvent(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Fetch": testFetch,
+		"Data":  testData,
 	})
 }
 
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+	event, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	assert.NotNil(t, event)
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+	err := mbtest.WriteEvent(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
+
 func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "etcd",
diff --git a/metricbeat/module/etcd/store/store_integration_test.go b/metricbeat/module/etcd/store/store_integration_test.go
index 22be4797e8c..53dc6a41afd 100644
--- a/metricbeat/module/etcd/store/store_integration_test.go
+++ b/metricbeat/module/etcd/store/store_integration_test.go
@@ -32,26 +32,30 @@ func TestStore(t *testing.T) {
 	runner := compose.TestRunner{Service: "etcd"}
 
 	runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
-			event, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
-
-			assert.NotNil(t, event)
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
-			err := mbtest.WriteEvent(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Fetch": testFetch,
+		"Data":  testData,
 	})
 }
 
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+	event, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	assert.NotNil(t, event)
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+	err := mbtest.WriteEvent(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
+
 func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "etcd",
diff --git a/metricbeat/module/http/json/json_integration_test.go b/metricbeat/module/http/json/json_integration_test.go
index 1e91c5ded02..85748e0a843 100644
--- a/metricbeat/module/http/json/json_integration_test.go
+++ b/metricbeat/module/http/json/json_integration_test.go
@@ -32,32 +32,38 @@ func TestJSON(t *testing.T) {
 	runner := compose.TestRunner{Service: "http"}
 
 	runner.Run(t, compose.Suite{
-		"FetchObject": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, getConfig("object", r.Host()))
-			event, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
+		"FetchObject": testFetchObject,
+		"FetchArray":  testFetchArray,
+		"Data":        testData,
+	})
+}
+
+func testFetchObject(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, getConfig("object", r.Host()))
+	event, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
 
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-		},
-		"FetchArray": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, getConfig("array", r.Host()))
-			event, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+}
 
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, getConfig("object", r.Host()))
-			err := mbtest.WriteEvents(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
-	})
+func testFetchArray(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, getConfig("array", r.Host()))
+	event, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, getConfig("object", r.Host()))
+	err := mbtest.WriteEvents(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
 }
 
 func getConfig(jsonType string, host string) map[string]interface{} {
diff --git a/metricbeat/module/jolokia/jmx/jmx_integration_test.go b/metricbeat/module/jolokia/jmx/jmx_integration_test.go
index d28146144e6..b7ba62f3742 100644
--- a/metricbeat/module/jolokia/jmx/jmx_integration_test.go
+++ b/metricbeat/module/jolokia/jmx/jmx_integration_test.go
@@ -32,31 +32,35 @@ func TestJMX(t *testing.T) {
 	runner := compose.TestRunner{Service: "jolokia"}
 
 	runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			for _, config := range getConfigs(r.Host()) {
-				f := mbtest.NewEventsFetcher(t, config)
-				events, err := f.Fetch()
-				if !assert.NoError(t, err) {
-					t.FailNow()
-				}
-				t.Logf("%s/%s events: %+v", f.Module().Name(), f.Name(), events)
-				if len(events) == 0 || len(events[0]) <= 1 {
-					t.Fatal("Empty events")
-				}
-			}
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			for _, config := range getConfigs(r.Host()) {
-				f := mbtest.NewEventsFetcher(t, config)
-				err := mbtest.WriteEvents(f, t)
-				if err != nil {
-					t.Fatal("write", err)
-				}
-			}
-		},
+		"Fetch": testFetch,
+		"Data":  testData,
 	})
 }
 
+func testFetch(t *testing.T, r compose.R) {
+	for _, config := range getConfigs(r.Host()) {
+		f := mbtest.NewEventsFetcher(t, config)
+		events, err := f.Fetch()
+		if !assert.NoError(t, err) {
+			t.FailNow()
+		}
+		t.Logf("%s/%s events: %+v", f.Module().Name(), f.Name(), events)
+		if len(events) == 0 || len(events[0]) <= 1 {
+			t.Fatal("Empty events")
+		}
+	}
+}
+
+func testData(t *testing.T, r compose.R) {
+	for _, config := range getConfigs(r.Host()) {
+		f := mbtest.NewEventsFetcher(t, config)
+		err := mbtest.WriteEvents(f, t)
+		if err != nil {
+			t.Fatal("write", err)
+		}
+	}
+}
+
 func getConfigs(host string) []map[string]interface{} {
 	return []map[string]interface{}{
 		{
diff --git a/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go b/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go
index e5a3e1f20b7..e2da5be0658 100644
--- a/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go
+++ b/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go
@@ -31,25 +31,29 @@ import (
 )
 
 func TestConsumerGroup(t *testing.T) {
-	mtest.Runner.Run(t, compose.Suite{"Data": func(t *testing.T, r compose.R) {
-		topic := "metricbeat-test"
-		mtest.GenerateKafkaData(t, topic, r.Host())
-		c, err := mtest.StartConsumer(t, topic, r.Host())
-		if err != nil {
-			t.Fatal(errors.Wrap(err, "starting kafka consumer"))
-		}
-		defer c.Close()
-
-		ms := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host()))
-		for retries := 0; retries < 3; retries++ {
-			err = mbtest.WriteEventsReporterV2(ms, t, "")
-			if err == nil {
-				return
-			}
-			time.Sleep(500 * time.Millisecond)
+	mtest.Runner.Run(t, compose.Suite{
+		"Data": testData,
+	})
+}
+
+func testData(t *testing.T, r compose.R) {
+	topic := "metricbeat-test"
+	mtest.GenerateKafkaData(t, topic, r.Host())
+	c, err := mtest.StartConsumer(t, topic, r.Host())
+	if err != nil {
+		t.Fatal(errors.Wrap(err, "starting kafka consumer"))
+	}
+	defer c.Close()
+
+	ms := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host()))
+	for retries := 0; retries < 3; retries++ {
+		err = mbtest.WriteEventsReporterV2(ms, t, "")
+		if err == nil {
+			return
 		}
-		t.Fatal("write", err)
-	}})
+		time.Sleep(500 * time.Millisecond)
+	}
+	t.Fatal("write", err)
 }
 
 func getConfig(host string) map[string]interface{} {
diff --git a/metricbeat/module/kafka/partition/partition_integration_test.go b/metricbeat/module/kafka/partition/partition_integration_test.go
index 1ef103a208f..ace9f55e7cb 100644
--- a/metricbeat/module/kafka/partition/partition_integration_test.go
+++ b/metricbeat/module/kafka/partition/partition_integration_test.go
@@ -41,73 +41,78 @@ func TestPartition(t *testing.T) {
 	logp.TestingSetup(logp.WithSelectors("kafka"))
 
 	mtest.Runner.Run(t, compose.Suite{
-		"Data": func(t *testing.T, r compose.R) {
-			mtest.GenerateKafkaData(t, "metricbeat-generate-data", r.Host())
-
-			ms := mbtest.NewReportingMetricSetV2(t, getConfig("", r.Host()))
-			err := mbtest.WriteEventsReporterV2(ms, t, "")
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
-		"Topic": func(t *testing.T, r compose.R) {
-			id := strconv.Itoa(rand.New(rand.NewSource(int64(time.Now().Nanosecond()))).Int())
-			testTopic := fmt.Sprintf("test-metricbeat-%s", id)
-
-			// Create initial topic
-			mtest.GenerateKafkaData(t, testTopic, r.Host())
-
-			f := mbtest.NewReportingMetricSetV2(t, getConfig(testTopic, r.Host()))
-			dataBefore, err := mbtest.ReportingFetchV2(f)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-			if len(dataBefore) == 0 {
-				t.Errorf("No offsets fetched from topic (before): %v", testTopic)
-			}
-			t.Logf("before: %v", dataBefore)
-
-			var n int64 = 10
-			// Create n messages
-			for i := int64(0); i < n; i++ {
-				mtest.GenerateKafkaData(t, testTopic, r.Host())
-			}
-
-			dataAfter, err := mbtest.ReportingFetchV2(f)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-			if len(dataAfter) == 0 {
-				t.Errorf("No offsets fetched from topic (after): %v", testTopic)
-			}
-			t.Logf("after: %v", dataAfter)
-
-			// Checks that no new topics / partitions were added
-			assert.True(t, len(dataBefore) == len(dataAfter))
-
-			var offsetBefore int64 = 0
-			var offsetAfter int64 = 0
-
-			// Its possible that other topics exists -> select the right data
-			for _, data := range dataBefore {
-				if data.ModuleFields["topic"].(common.MapStr)["name"] == testTopic {
-					offsetBefore = data.MetricSetFields["offset"].(common.MapStr)["newest"].(int64)
-				}
-			}
-
-			for _, data := range dataAfter {
-				if data.ModuleFields["topic"].(common.MapStr)["name"] == testTopic {
-					offsetAfter = data.MetricSetFields["offset"].(common.MapStr)["newest"].(int64)
-				}
-			}
-
-			// Compares offset before and after
-			if offsetBefore+n != offsetAfter {
-				t.Errorf("Offset before: %v", offsetBefore)
-				t.Errorf("Offset after: %v", offsetAfter)
-			}
-			assert.True(t, offsetBefore+n == offsetAfter)
-		}})
+		"Data":  testData,
+		"Topic": testTopic,
+	})
+}
+
+func testData(t *testing.T, r compose.R) {
+	mtest.GenerateKafkaData(t, "metricbeat-generate-data", r.Host())
+
+	ms := mbtest.NewReportingMetricSetV2(t, getConfig("", r.Host()))
+	err := mbtest.WriteEventsReporterV2(ms, t, "")
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
+
+func testTopic(t *testing.T, r compose.R) {
+	id := strconv.Itoa(rand.New(rand.NewSource(int64(time.Now().Nanosecond()))).Int())
+	testTopic := fmt.Sprintf("test-metricbeat-%s", id)
+
+	// Create initial topic
+	mtest.GenerateKafkaData(t, testTopic, r.Host())
+
+	f := mbtest.NewReportingMetricSetV2(t, getConfig(testTopic, r.Host()))
+	dataBefore, err := mbtest.ReportingFetchV2(f)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+	if len(dataBefore) == 0 {
+		t.Errorf("No offsets fetched from topic (before): %v", testTopic)
+	}
+	t.Logf("before: %v", dataBefore)
+
+	var n int64 = 10
+	// Create n messages
+	for i := int64(0); i < n; i++ {
+		mtest.GenerateKafkaData(t, testTopic, r.Host())
+	}
+
+	dataAfter, err := mbtest.ReportingFetchV2(f)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+	if len(dataAfter) == 0 {
+		t.Errorf("No offsets fetched from topic (after): %v", testTopic)
+	}
+	t.Logf("after: %v", dataAfter)
+
+	// Checks that no new topics / partitions were added
+	assert.True(t, len(dataBefore) == len(dataAfter))
+
+	var offsetBefore int64 = 0
+	var offsetAfter int64 = 0
+
+	// Its possible that other topics exists -> select the right data
+	for _, data := range dataBefore {
+		if data.ModuleFields["topic"].(common.MapStr)["name"] == testTopic {
+			offsetBefore = data.MetricSetFields["offset"].(common.MapStr)["newest"].(int64)
+		}
+	}
+
+	for _, data := range dataAfter {
+		if data.ModuleFields["topic"].(common.MapStr)["name"] == testTopic {
+			offsetAfter = data.MetricSetFields["offset"].(common.MapStr)["newest"].(int64)
+		}
+	}
+
+	// Compares offset before and after
+	if offsetBefore+n != offsetAfter {
+		t.Errorf("Offset before: %v", offsetBefore)
+		t.Errorf("Offset after: %v", offsetAfter)
+	}
+	assert.True(t, offsetBefore+n == offsetAfter)
 }
 
 func getConfig(topic string, host string) map[string]interface{} {
diff --git a/metricbeat/module/kibana/stats/stats_integration_test.go b/metricbeat/module/kibana/stats/stats_integration_test.go
index 0dd68e44369..8872e975c8b 100644
--- a/metricbeat/module/kibana/stats/stats_integration_test.go
+++ b/metricbeat/module/kibana/stats/stats_integration_test.go
@@ -37,58 +37,62 @@ import (
 
 func TestStat(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			config := mtest.GetConfig("stats", r.Host())
-			version, err := getKibanaVersion(r.Host())
-			if err != nil {
-				t.Fatal("getting kibana version", err)
-			}
-
-			isStatsAPIAvailable, err := kibana.IsStatsAPIAvailable(version)
-			if err != nil {
-				t.Fatal("checking if kibana stats API is available", err)
-			}
-
-			if !isStatsAPIAvailable {
-				t.Skip("Kibana stats API is not available until 6.4.0")
-			}
-
-			f := mbtest.NewReportingMetricSetV2(t, config)
-			events, errs := mbtest.ReportingFetchV2(f)
-
-			assert.Empty(t, errs)
-			if !assert.NotEmpty(t, events) {
-				t.FailNow()
-			}
-
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(),
-				events[0].BeatEvent("kibana", "stats").Fields.StringToPrint())
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			config := mtest.GetConfig("stats", r.Host())
-			version, err := getKibanaVersion(r.Host())
-			if err != nil {
-				t.Fatal("getting kibana version", err)
-			}
-
-			isStatsAPIAvailable, err := kibana.IsStatsAPIAvailable(version)
-			if err != nil {
-				t.Fatal("checking if kibana stats API is available", err)
-			}
-
-			if !isStatsAPIAvailable {
-				t.Skip("Kibana stats API is not available until 6.4.0")
-			}
-
-			f := mbtest.NewReportingMetricSetV2(t, config)
-			err = mbtest.WriteEventsReporterV2(f, t, "")
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Fetch": testFetch,
+		"Data":  testData,
 	})
 }
 
+func testFetch(t *testing.T, r compose.R) {
+	config := mtest.GetConfig("stats", r.Host())
+	version, err := getKibanaVersion(r.Host())
+	if err != nil {
+		t.Fatal("getting kibana version", err)
+	}
+
+	isStatsAPIAvailable, err := kibana.IsStatsAPIAvailable(version)
+	if err != nil {
+		t.Fatal("checking if kibana stats API is available", err)
+	}
+
+	if !isStatsAPIAvailable {
+		t.Skip("Kibana stats API is not available until 6.4.0")
+	}
+
+	f := mbtest.NewReportingMetricSetV2(t, config)
+	events, errs := mbtest.ReportingFetchV2(f)
+
+	assert.Empty(t, errs)
+	if !assert.NotEmpty(t, events) {
+		t.FailNow()
+	}
+
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(),
+		events[0].BeatEvent("kibana", "stats").Fields.StringToPrint())
+}
+
+func testData(t *testing.T, r compose.R) {
+	config := mtest.GetConfig("stats", r.Host())
+	version, err := getKibanaVersion(r.Host())
+	if err != nil {
+		t.Fatal("getting kibana version", err)
+	}
+
+	isStatsAPIAvailable, err := kibana.IsStatsAPIAvailable(version)
+	if err != nil {
+		t.Fatal("checking if kibana stats API is available", err)
+	}
+
+	if !isStatsAPIAvailable {
+		t.Skip("Kibana stats API is not available until 6.4.0")
+	}
+
+	f := mbtest.NewReportingMetricSetV2(t, config)
+	err = mbtest.WriteEventsReporterV2(f, t, "")
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
+
 func getKibanaVersion(kibanaHostPort string) (string, error) {
 	resp, err := http.Get("http://" + kibanaHostPort + "/api/status")
 	if err != nil {
diff --git a/metricbeat/module/kibana/status/status_integration_test.go b/metricbeat/module/kibana/status/status_integration_test.go
index d86857b91e9..f8dcb36fef8 100644
--- a/metricbeat/module/kibana/status/status_integration_test.go
+++ b/metricbeat/module/kibana/status/status_integration_test.go
@@ -31,25 +31,29 @@ import (
 
 func TestStatus(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("status", r.Host()))
-			events, errs := mbtest.ReportingFetchV2(f)
-
-			assert.Empty(t, errs)
-			if !assert.NotEmpty(t, events) {
-				t.FailNow()
-			}
-
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(),
-				events[0].BeatEvent("kibana", "status").Fields.StringToPrint())
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			config := mtest.GetConfig("status", r.Host())
-			f := mbtest.NewReportingMetricSetV2(t, config)
-			err := mbtest.WriteEventsReporterV2(f, t, "")
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Fetch": testFetch,
+		"Data":  testData,
 	})
 }
+
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("status", r.Host()))
+	events, errs := mbtest.ReportingFetchV2(f)
+
+	assert.Empty(t, errs)
+	if !assert.NotEmpty(t, events) {
+		t.FailNow()
+	}
+
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(),
+		events[0].BeatEvent("kibana", "status").Fields.StringToPrint())
+}
+
+func testData(t *testing.T, r compose.R) {
+	config := mtest.GetConfig("status", r.Host())
+	f := mbtest.NewReportingMetricSetV2(t, config)
+	err := mbtest.WriteEventsReporterV2(f, t, "")
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
diff --git a/metricbeat/module/logstash/logstash_integration_test.go b/metricbeat/module/logstash/logstash_integration_test.go
index 8997f86709b..1e0e8a9fe8a 100644
--- a/metricbeat/module/logstash/logstash_integration_test.go
+++ b/metricbeat/module/logstash/logstash_integration_test.go
@@ -38,29 +38,33 @@ var metricSets = []string{
 
 func TestLogstash(t *testing.T) {
 	logstash.Runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			for _, metricSet := range metricSets {
-				f := mbtest.NewReportingMetricSetV2(t, logstash.GetConfig(metricSet, r.Host()))
-				events, errs := mbtest.ReportingFetchV2(f)
+		"Fetch": testFetch,
+		"Data":  testData,
+	})
+}
 
-				assert.Empty(t, errs)
-				if !assert.NotEmpty(t, events) {
-					t.FailNow()
-				}
+func testFetch(t *testing.T, r compose.R) {
+	for _, metricSet := range metricSets {
+		f := mbtest.NewReportingMetricSetV2(t, logstash.GetConfig(metricSet, r.Host()))
+		events, errs := mbtest.ReportingFetchV2(f)
 
-				t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(),
-					events[0].BeatEvent("logstash", metricSet).Fields.StringToPrint())
-			}
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			for _, metricSet := range metricSets {
-				config := logstash.GetConfig(metricSet, r.Host())
-				f := mbtest.NewReportingMetricSetV2(t, config)
-				err := mbtest.WriteEventsReporterV2(f, t, "")
-				if err != nil {
-					t.Fatal("write", err)
-				}
-			}
-		},
-	})
+		assert.Empty(t, errs)
+		if !assert.NotEmpty(t, events) {
+			t.FailNow()
+		}
+
+		t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(),
+			events[0].BeatEvent("logstash", metricSet).Fields.StringToPrint())
+	}
+}
+
+func testData(t *testing.T, r compose.R) {
+	for _, metricSet := range metricSets {
+		config := logstash.GetConfig(metricSet, r.Host())
+		f := mbtest.NewReportingMetricSetV2(t, config)
+		err := mbtest.WriteEventsReporterV2(f, t, "")
+		if err != nil {
+			t.Fatal("write", err)
+		}
+	}
 }
diff --git a/metricbeat/module/memcached/stats/stats_integration_test.go b/metricbeat/module/memcached/stats/stats_integration_test.go
index 1672d600cb0..29a513189bf 100644
--- a/metricbeat/module/memcached/stats/stats_integration_test.go
+++ b/metricbeat/module/memcached/stats/stats_integration_test.go
@@ -30,16 +30,18 @@ func TestStats(t *testing.T) {
 	runner := compose.TestRunner{Service: "memcached"}
 
 	runner.Run(t, compose.Suite{
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
-			err := mbtest.WriteEvent(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Data": testData,
 	})
 }
 
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+	err := mbtest.WriteEvent(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
+
 func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "memcached",
diff --git a/metricbeat/module/mongodb/collstats/collstats_integration_test.go b/metricbeat/module/mongodb/collstats/collstats_integration_test.go
index 98b5b0b5028..53aca599748 100644
--- a/metricbeat/module/mongodb/collstats/collstats_integration_test.go
+++ b/metricbeat/module/mongodb/collstats/collstats_integration_test.go
@@ -31,30 +31,34 @@ import (
 
 func TestCollstats(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, mtest.GetConfig("collstats", r.Host()))
-			events, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
-
-			for _, event := range events {
-				t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-
-				// Check a few event Fields
-				db := event["db"].(string)
-				assert.NotEqual(t, db, "")
-
-				collection := event["collection"].(string)
-				assert.NotEqual(t, collection, "")
-			}
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, mtest.GetConfig("collstats", r.Host()))
-			err := mbtest.WriteEvents(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Fetch": testFetch,
+		"Data":  testData,
 	})
 }
+
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, mtest.GetConfig("collstats", r.Host()))
+	events, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	for _, event := range events {
+		t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+
+		// Check a few event Fields
+		db := event["db"].(string)
+		assert.NotEqual(t, db, "")
+
+		collection := event["collection"].(string)
+		assert.NotEqual(t, collection, "")
+	}
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, mtest.GetConfig("collstats", r.Host()))
+	err := mbtest.WriteEvents(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
diff --git a/metricbeat/module/mongodb/dbstats/dbstats_integration_test.go b/metricbeat/module/mongodb/dbstats/dbstats_integration_test.go
index 8cea2158fe1..e794495c057 100644
--- a/metricbeat/module/mongodb/dbstats/dbstats_integration_test.go
+++ b/metricbeat/module/mongodb/dbstats/dbstats_integration_test.go
@@ -31,55 +31,59 @@ import (
 
 func TestDBStats(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, mtest.GetConfig("dbstats", r.Host()))
-			events, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
-
-			for _, event := range events {
-				t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-
-				// Check a few event Fields
-				db := event["db"].(string)
-				assert.NotEqual(t, db, "")
-
-				collections := event["collections"].(int64)
-				assert.True(t, collections > 0)
-
-				objects := event["objects"].(int64)
-				assert.True(t, objects > 0)
-
-				avgObjSize, err := event.GetValue("avg_obj_size.bytes")
-				assert.NoError(t, err)
-				assert.True(t, avgObjSize.(int64) > 0)
-
-				dataSize, err := event.GetValue("data_size.bytes")
-				assert.NoError(t, err)
-				assert.True(t, dataSize.(int64) > 0)
-
-				storageSize, err := event.GetValue("storage_size.bytes")
-				assert.NoError(t, err)
-				assert.True(t, storageSize.(int64) > 0)
-
-				numExtents := event["num_extents"].(int64)
-				assert.True(t, numExtents >= 0)
-
-				indexes := event["indexes"].(int64)
-				assert.True(t, indexes >= 0)
-
-				indexSize, err := event.GetValue("index_size.bytes")
-				assert.NoError(t, err)
-				assert.True(t, indexSize.(int64) > 0)
-			}
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, mtest.GetConfig("dbstats", r.Host()))
-			err := mbtest.WriteEvents(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Fetch": testFetch,
+		"Data":  testData,
 	})
 }
+
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, mtest.GetConfig("dbstats", r.Host()))
+	events, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	for _, event := range events {
+		t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+
+		// Check a few event Fields
+		db := event["db"].(string)
+		assert.NotEqual(t, db, "")
+
+		collections := event["collections"].(int64)
+		assert.True(t, collections > 0)
+
+		objects := event["objects"].(int64)
+		assert.True(t, objects > 0)
+
+		avgObjSize, err := event.GetValue("avg_obj_size.bytes")
+		assert.NoError(t, err)
+		assert.True(t, avgObjSize.(int64) > 0)
+
+		dataSize, err := event.GetValue("data_size.bytes")
+		assert.NoError(t, err)
+		assert.True(t, dataSize.(int64) > 0)
+
+		storageSize, err := event.GetValue("storage_size.bytes")
+		assert.NoError(t, err)
+		assert.True(t, storageSize.(int64) > 0)
+
+		numExtents := event["num_extents"].(int64)
+		assert.True(t, numExtents >= 0)
+
+		indexes := event["indexes"].(int64)
+		assert.True(t, indexes >= 0)
+
+		indexSize, err := event.GetValue("index_size.bytes")
+		assert.NoError(t, err)
+		assert.True(t, indexSize.(int64) > 0)
+	}
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, mtest.GetConfig("dbstats", r.Host()))
+	err := mbtest.WriteEvents(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
diff --git a/metricbeat/module/mongodb/metrics/metrics_intergration_test.go b/metricbeat/module/mongodb/metrics/metrics_intergration_test.go
index 0d706716a40..5126810e1ab 100644
--- a/metricbeat/module/mongodb/metrics/metrics_intergration_test.go
+++ b/metricbeat/module/mongodb/metrics/metrics_intergration_test.go
@@ -31,28 +31,32 @@ import (
 
 func TestMetrics(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, mtest.GetConfig("metrics", r.Host()))
-			event, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
-
-			// Check a few event Fields
-			findCount, err := event.GetValue("commands.find.total")
-			assert.NoError(t, err)
-			assert.True(t, findCount.(int64) >= 0)
-
-			deletedDocuments, err := event.GetValue("document.deleted")
-			assert.NoError(t, err)
-			assert.True(t, deletedDocuments.(int64) >= 0)
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, mtest.GetConfig("metrics", r.Host()))
-			err := mbtest.WriteEvent(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Fetch": testFetch,
+		"Data":  testData,
 	})
 }
+
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, mtest.GetConfig("metrics", r.Host()))
+	event, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	// Check a few event Fields
+	findCount, err := event.GetValue("commands.find.total")
+	assert.NoError(t, err)
+	assert.True(t, findCount.(int64) >= 0)
+
+	deletedDocuments, err := event.GetValue("document.deleted")
+	assert.NoError(t, err)
+	assert.True(t, deletedDocuments.(int64) >= 0)
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, mtest.GetConfig("metrics", r.Host()))
+	err := mbtest.WriteEvent(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
diff --git a/metricbeat/module/mongodb/replstatus/replstatus_integration_test.go b/metricbeat/module/mongodb/replstatus/replstatus_integration_test.go
index c1a9b721b4d..6db3179fc80 100644
--- a/metricbeat/module/mongodb/replstatus/replstatus_integration_test.go
+++ b/metricbeat/module/mongodb/replstatus/replstatus_integration_test.go
@@ -36,52 +36,55 @@ import (
 
 func TestReplStatus(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			err := initiateReplicaSet(t, r.Host())
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
-
-			f := mbtest.NewEventFetcher(t, mtest.GetConfig("replstatus", r.Host()))
-			event, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
-
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-
-			// Check event fields
-			oplog := event["oplog"].(common.MapStr)
-			allocated := oplog["size"].(common.MapStr)["allocated"].(int64)
-			assert.True(t, allocated >= 0)
-
-			used := oplog["size"].(common.MapStr)["used"].(float64)
-			assert.True(t, used > 0)
-
-			firstTs := oplog["first"].(common.MapStr)["timestamp"].(int64)
-			assert.True(t, firstTs >= 0)
-
-			window := oplog["window"].(int64)
-			assert.True(t, window >= 0)
-
-			members := event["members"].(common.MapStr)
-			primary := members["primary"].(common.MapStr)
-			assert.NotEmpty(t, primary["host"].(string))
-			assert.True(t, primary["optime"].(int64) > 0)
-
-			set := event["set_name"].(string)
-			assert.Equal(t, set, "beats")
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, mtest.GetConfig("replstatus", r.Host()))
-			err := mbtest.WriteEvent(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Fetch": testFetch,
+		"Data":  testData,
 	})
 }
 
+func testFetch(t *testing.T, r compose.R) {
+	err := initiateReplicaSet(t, r.Host())
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	f := mbtest.NewEventFetcher(t, mtest.GetConfig("replstatus", r.Host()))
+	event, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+
+	// Check event fields
+	oplog := event["oplog"].(common.MapStr)
+	allocated := oplog["size"].(common.MapStr)["allocated"].(int64)
+	assert.True(t, allocated >= 0)
+
+	used := oplog["size"].(common.MapStr)["used"].(float64)
+	assert.True(t, used > 0)
+
+	firstTs := oplog["first"].(common.MapStr)["timestamp"].(int64)
+	assert.True(t, firstTs >= 0)
+
+	window := oplog["window"].(int64)
+	assert.True(t, window >= 0)
+
+	members := event["members"].(common.MapStr)
+	primary := members["primary"].(common.MapStr)
+	assert.NotEmpty(t, primary["host"].(string))
+	assert.True(t, primary["optime"].(int64) > 0)
+
+	set := event["set_name"].(string)
+	assert.Equal(t, set, "beats")
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, mtest.GetConfig("replstatus", r.Host()))
+	err := mbtest.WriteEvent(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
 func initiateReplicaSet(t *testing.T, url string) error {
 	dialInfo, err := mgo.ParseURL(url)
 	if err != nil {
diff --git a/metricbeat/module/mongodb/status/status_integration_test.go b/metricbeat/module/mongodb/status/status_integration_test.go
index 91eab6daf6c..cb19832f489 100644
--- a/metricbeat/module/mongodb/status/status_integration_test.go
+++ b/metricbeat/module/mongodb/status/status_integration_test.go
@@ -32,31 +32,35 @@ import (
 
 func TestStatus(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, mtest.GetConfig("status", r.Host()))
-			event, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
-
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-
-			// Check event fields
-			current := event["connections"].(common.MapStr)["current"].(int64)
-			assert.True(t, current >= 0)
-
-			available := event["connections"].(common.MapStr)["available"].(int64)
-			assert.True(t, available > 0)
-
-			pageFaults := event["extra_info"].(common.MapStr)["page_faults"].(int64)
-			assert.True(t, pageFaults >= 0)
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, mtest.GetConfig("status", r.Host()))
-			err := mbtest.WriteEvent(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Fetch": testFetch,
+		"Data":  testData,
 	})
 }
+
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, mtest.GetConfig("status", r.Host()))
+	event, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+
+	// Check event fields
+	current := event["connections"].(common.MapStr)["current"].(int64)
+	assert.True(t, current >= 0)
+
+	available := event["connections"].(common.MapStr)["available"].(int64)
+	assert.True(t, available > 0)
+
+	pageFaults := event["extra_info"].(common.MapStr)["page_faults"].(int64)
+	assert.True(t, pageFaults >= 0)
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, mtest.GetConfig("status", r.Host()))
+	err := mbtest.WriteEvent(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
diff --git a/metricbeat/module/mysql/mysql_integration_test.go b/metricbeat/module/mysql/mysql_integration_test.go
index 602b617d215..47217fae02e 100644
--- a/metricbeat/module/mysql/mysql_integration_test.go
+++ b/metricbeat/module/mysql/mysql_integration_test.go
@@ -30,12 +30,14 @@ import (
 
 func TestNewDB(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"NewDB": func(t *testing.T, r compose.R) {
-			db, err := NewDB(mtest.GetDSN(r.Host()))
-			assert.NoError(t, err)
-
-			err = db.Ping()
-			assert.NoError(t, err)
-		},
+		"NewDB": testNewDB,
 	})
 }
+
+func testNewDB(t *testing.T, r compose.R) {
+	db, err := NewDB(mtest.GetDSN(r.Host()))
+	assert.NoError(t, err)
+
+	err = db.Ping()
+	assert.NoError(t, err)
+}
diff --git a/metricbeat/module/mysql/status/status_integration_test.go b/metricbeat/module/mysql/status/status_integration_test.go
index 67f1c874235..a6b72b3488d 100644
--- a/metricbeat/module/mysql/status/status_integration_test.go
+++ b/metricbeat/module/mysql/status/status_integration_test.go
@@ -32,57 +32,63 @@ import (
 
 func TestStatus(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, mtest.GetConfig("status", r.Host(), false))
-			event, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
-
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-
-			// Check event fields
-			connections := event["connections"].(int64)
-			open := event["open"].(common.MapStr)
-			openTables := open["tables"].(int64)
-			openFiles := open["files"].(int64)
-			openStreams := open["streams"].(int64)
-
-			assert.True(t, connections > 0)
-			assert.True(t, openTables > 0)
-			assert.True(t, openFiles >= 0)
-			assert.True(t, openStreams == 0)
-		},
-		"FetchRaw": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, mtest.GetConfig("status", r.Host(), true))
-			event, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
-
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-
-			// Check event fields
-			cachedThreads := event["threads"].(common.MapStr)["cached"].(int64)
-			assert.True(t, cachedThreads >= 0)
-
-			rawData := event["raw"].(common.MapStr)
-
-			// Make sure field was removed from raw fields as in schema
-			_, exists := rawData["Threads_cached"]
-			assert.False(t, exists)
-
-			// Check a raw field if it is available
-			_, exists = rawData["Slow_launch_threads"]
-			assert.True(t, exists)
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, mtest.GetConfig("status", r.Host(), false))
-
-			err := mbtest.WriteEvent(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Fetch":    testFetch,
+		"FetchRaw": testFetchRaw,
+		"Data":     testData,
 	})
 }
+
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, mtest.GetConfig("status", r.Host(), false))
+	event, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+
+	// Check event fields
+	connections := event["connections"].(int64)
+	open := event["open"].(common.MapStr)
+	openTables := open["tables"].(int64)
+	openFiles := open["files"].(int64)
+	openStreams := open["streams"].(int64)
+
+	assert.True(t, connections > 0)
+	assert.True(t, openTables > 0)
+	assert.True(t, openFiles >= 0)
+	assert.True(t, openStreams == 0)
+}
+
+func testFetchRaw(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, mtest.GetConfig("status", r.Host(), true))
+	event, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+
+	// Check event fields
+	cachedThreads := event["threads"].(common.MapStr)["cached"].(int64)
+	assert.True(t, cachedThreads >= 0)
+
+	rawData := event["raw"].(common.MapStr)
+
+	// Make sure field was removed from raw fields as in schema
+	_, exists := rawData["Threads_cached"]
+	assert.False(t, exists)
+
+	// Check a raw field if it is available
+	_, exists = rawData["Slow_launch_threads"]
+	assert.True(t, exists)
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, mtest.GetConfig("status", r.Host(), false))
+
+	err := mbtest.WriteEvent(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
diff --git a/metricbeat/module/nginx/stubstatus/stubstatus_integration_test.go b/metricbeat/module/nginx/stubstatus/stubstatus_integration_test.go
index 7c2a441b97a..68d998bcde6 100644
--- a/metricbeat/module/nginx/stubstatus/stubstatus_integration_test.go
+++ b/metricbeat/module/nginx/stubstatus/stubstatus_integration_test.go
@@ -35,27 +35,31 @@ func TestStubstatus(t *testing.T) {
 	}
 
 	runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
-			event, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
+		"Fetch": tesFetch,
+		"Data":  testData,
+	})
+}
 
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+	event, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
 
-			// Check number of fields.
-			assert.Equal(t, 10, len(event))
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
 
-			err := mbtest.WriteEvent(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
-	})
+	// Check number of fields.
+	assert.Equal(t, 10, len(event))
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+
+	err := mbtest.WriteEvent(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
 }
 
 func getConfig(host string) map[string]interface{} {
diff --git a/metricbeat/module/php_fpm/pool/pool_integration_test.go b/metricbeat/module/php_fpm/pool/pool_integration_test.go
index 332bb889a10..5c8489f4de5 100644
--- a/metricbeat/module/php_fpm/pool/pool_integration_test.go
+++ b/metricbeat/module/php_fpm/pool/pool_integration_test.go
@@ -29,12 +29,14 @@ import (
 
 func TestPool(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("pool", r.Host()))
-			err := mbtest.WriteEventsReporterV2(f, t, "")
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Data": testData,
 	})
 }
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("pool", r.Host()))
+	err := mbtest.WriteEventsReporterV2(f, t, "")
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
diff --git a/metricbeat/module/php_fpm/process/process_integration_test.go b/metricbeat/module/php_fpm/process/process_integration_test.go
index 66aa94e3acd..55dcd22c877 100644
--- a/metricbeat/module/php_fpm/process/process_integration_test.go
+++ b/metricbeat/module/php_fpm/process/process_integration_test.go
@@ -29,12 +29,14 @@ import (
 
 func TestProcess(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("process", r.Host()))
-			err := mbtest.WriteEventsReporterV2(f, t, "")
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Data": testData,
 	})
 }
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("process", r.Host()))
+	err := mbtest.WriteEventsReporterV2(f, t, "")
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
diff --git a/metricbeat/module/postgresql/activity/activity_integration_test.go b/metricbeat/module/postgresql/activity/activity_integration_test.go
index c48b399a818..7533986cd22 100644
--- a/metricbeat/module/postgresql/activity/activity_integration_test.go
+++ b/metricbeat/module/postgresql/activity/activity_integration_test.go
@@ -32,37 +32,41 @@ import (
 
 func TestActivity(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, mtest.GetConfig("activity", r.Host()))
-			events, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
+		"Fetch": testFetch,
+		"Data":  testData,
+	})
+}
 
-			assert.True(t, len(events) > 0)
-			event := events[0]
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, mtest.GetConfig("activity", r.Host()))
+	events, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
 
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+	assert.True(t, len(events) > 0)
+	event := events[0]
 
-			// Check event fields
-			assert.Contains(t, event, "database")
-			db_oid := event["database"].(common.MapStr)["oid"].(int64)
-			assert.True(t, db_oid > 0)
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
 
-			assert.Contains(t, event, "pid")
-			assert.True(t, event["pid"].(int64) > 0)
+	// Check event fields
+	assert.Contains(t, event, "database")
+	db_oid := event["database"].(common.MapStr)["oid"].(int64)
+	assert.True(t, db_oid > 0)
 
-			assert.Contains(t, event, "user")
-			assert.Contains(t, event["user"].(common.MapStr), "name")
-			assert.Contains(t, event["user"].(common.MapStr), "id")
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, mtest.GetConfig("activity", r.Host()))
+	assert.Contains(t, event, "pid")
+	assert.True(t, event["pid"].(int64) > 0)
 
-			err := mbtest.WriteEvents(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
-	})
+	assert.Contains(t, event, "user")
+	assert.Contains(t, event["user"].(common.MapStr), "name")
+	assert.Contains(t, event["user"].(common.MapStr), "id")
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, mtest.GetConfig("activity", r.Host()))
+
+	err := mbtest.WriteEvents(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
 }
diff --git a/metricbeat/module/postgresql/bgwriter/bgwriter_integration_test.go b/metricbeat/module/postgresql/bgwriter/bgwriter_integration_test.go
index 3562d21d3f0..0a307d165f8 100644
--- a/metricbeat/module/postgresql/bgwriter/bgwriter_integration_test.go
+++ b/metricbeat/module/postgresql/bgwriter/bgwriter_integration_test.go
@@ -32,39 +32,43 @@ import (
 
 func TestBgwriter(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, mtest.GetConfig("bgwriter", r.Host()))
-			event, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
+		"Fetch": testFetch,
+		"Data":  testData,
+	})
+}
 
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, mtest.GetConfig("bgwriter", r.Host()))
+	event, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
 
-			assert.Contains(t, event, "checkpoints")
-			assert.Contains(t, event, "buffers")
-			assert.Contains(t, event, "stats_reset")
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
 
-			checkpoints := event["checkpoints"].(common.MapStr)
-			assert.Contains(t, checkpoints, "scheduled")
-			assert.Contains(t, checkpoints, "requested")
-			assert.Contains(t, checkpoints, "times")
+	assert.Contains(t, event, "checkpoints")
+	assert.Contains(t, event, "buffers")
+	assert.Contains(t, event, "stats_reset")
 
-			buffers := event["buffers"].(common.MapStr)
-			assert.Contains(t, buffers, "checkpoints")
-			assert.Contains(t, buffers, "clean")
-			assert.Contains(t, buffers, "clean_full")
-			assert.Contains(t, buffers, "backend")
-			assert.Contains(t, buffers, "backend_fsync")
-			assert.Contains(t, buffers, "allocated")
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, mtest.GetConfig("bgwriter", r.Host()))
+	checkpoints := event["checkpoints"].(common.MapStr)
+	assert.Contains(t, checkpoints, "scheduled")
+	assert.Contains(t, checkpoints, "requested")
+	assert.Contains(t, checkpoints, "times")
 
-			err := mbtest.WriteEvent(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
-	})
+	buffers := event["buffers"].(common.MapStr)
+	assert.Contains(t, buffers, "checkpoints")
+	assert.Contains(t, buffers, "clean")
+	assert.Contains(t, buffers, "clean_full")
+	assert.Contains(t, buffers, "backend")
+	assert.Contains(t, buffers, "backend_fsync")
+	assert.Contains(t, buffers, "allocated")
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, mtest.GetConfig("bgwriter", r.Host()))
+
+	err := mbtest.WriteEvent(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
 }
diff --git a/metricbeat/module/postgresql/database/database_integration_test.go b/metricbeat/module/postgresql/database/database_integration_test.go
index 84b2ecfb3fb..f7685b65c8a 100644
--- a/metricbeat/module/postgresql/database/database_integration_test.go
+++ b/metricbeat/module/postgresql/database/database_integration_test.go
@@ -32,38 +32,42 @@ import (
 
 func TestDatabase(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, mtest.GetConfig("database", r.Host()))
-			events, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
+		"Fetch": testFetch,
+		"Data":  testData,
+	})
+}
 
-			assert.True(t, len(events) > 0)
-			event := events[0]
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, mtest.GetConfig("database", r.Host()))
+	events, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
 
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+	assert.True(t, len(events) > 0)
+	event := events[0]
 
-			// Check event fields
-			db_oid := event["oid"].(int64)
-			assert.True(t, db_oid > 0)
-			assert.Contains(t, event, "name")
-			_, ok := event["name"].(string)
-			assert.True(t, ok)
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
 
-			rows := event["rows"].(common.MapStr)
-			assert.Contains(t, rows, "returned")
-			assert.Contains(t, rows, "fetched")
-			assert.Contains(t, rows, "inserted")
-			assert.Contains(t, rows, "updated")
-			assert.Contains(t, rows, "deleted")
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, mtest.GetConfig("database", r.Host()))
-			err := mbtest.WriteEvents(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
-	})
+	// Check event fields
+	db_oid := event["oid"].(int64)
+	assert.True(t, db_oid > 0)
+	assert.Contains(t, event, "name")
+	_, ok := event["name"].(string)
+	assert.True(t, ok)
+
+	rows := event["rows"].(common.MapStr)
+	assert.Contains(t, rows, "returned")
+	assert.Contains(t, rows, "fetched")
+	assert.Contains(t, rows, "inserted")
+	assert.Contains(t, rows, "updated")
+	assert.Contains(t, rows, "deleted")
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, mtest.GetConfig("database", r.Host()))
+	err := mbtest.WriteEvents(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
 }
diff --git a/metricbeat/module/postgresql/statement/statement_integration_test.go b/metricbeat/module/postgresql/statement/statement_integration_test.go
index 5055161c317..6a1acb967d1 100644
--- a/metricbeat/module/postgresql/statement/statement_integration_test.go
+++ b/metricbeat/module/postgresql/statement/statement_integration_test.go
@@ -32,70 +32,74 @@ import (
 
 func TestStatement(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, mtest.GetConfig("statement", r.Host()))
-			events, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
-
-			assert.True(t, len(events) > 0)
-			event := events[0]
-
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-
-			// Check event fields
-			assert.Contains(t, event, "user")
-			assert.Contains(t, event["user"].(common.MapStr), "id")
-
-			assert.Contains(t, event, "database")
-			db_oid := event["database"].(common.MapStr)["oid"].(int64)
-			assert.True(t, db_oid > 0)
-
-			assert.Contains(t, event, "query")
-			query := event["query"].(common.MapStr)
-			assert.Contains(t, query, "id")
-			assert.Contains(t, query, "text")
-			assert.Contains(t, query, "calls")
-			assert.Contains(t, query, "rows")
-
-			assert.Contains(t, query, "time")
-			time := query["time"].(common.MapStr)
-			assert.Contains(t, time, "total")
-			assert.Contains(t, time, "min")
-			assert.Contains(t, time, "max")
-			assert.Contains(t, time, "mean")
-			assert.Contains(t, time, "stddev")
-
-			assert.Contains(t, query["memory"], "shared")
-			memory := query["memory"].(common.MapStr)
-
-			assert.Contains(t, memory, "shared")
-			shared := memory["shared"].(common.MapStr)
-			assert.Contains(t, shared, "hit")
-			assert.Contains(t, shared, "read")
-			assert.Contains(t, shared, "dirtied")
-			assert.Contains(t, shared, "written")
-
-			assert.Contains(t, memory, "local")
-			local := memory["local"].(common.MapStr)
-			assert.Contains(t, local, "hit")
-			assert.Contains(t, local, "read")
-			assert.Contains(t, local, "dirtied")
-			assert.Contains(t, local, "written")
-
-			assert.Contains(t, memory, "temp")
-			temp := memory["temp"].(common.MapStr)
-			assert.Contains(t, temp, "read")
-			assert.Contains(t, temp, "written")
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, mtest.GetConfig("statement", r.Host()))
-
-			err := mbtest.WriteEvents(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Fetch": testFetch,
+		"Data":  testData,
 	})
 }
+
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, mtest.GetConfig("statement", r.Host()))
+	events, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	assert.True(t, len(events) > 0)
+	event := events[0]
+
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+
+	// Check event fields
+	assert.Contains(t, event, "user")
+	assert.Contains(t, event["user"].(common.MapStr), "id")
+
+	assert.Contains(t, event, "database")
+	db_oid := event["database"].(common.MapStr)["oid"].(int64)
+	assert.True(t, db_oid > 0)
+
+	assert.Contains(t, event, "query")
+	query := event["query"].(common.MapStr)
+	assert.Contains(t, query, "id")
+	assert.Contains(t, query, "text")
+	assert.Contains(t, query, "calls")
+	assert.Contains(t, query, "rows")
+
+	assert.Contains(t, query, "time")
+	time := query["time"].(common.MapStr)
+	assert.Contains(t, time, "total")
+	assert.Contains(t, time, "min")
+	assert.Contains(t, time, "max")
+	assert.Contains(t, time, "mean")
+	assert.Contains(t, time, "stddev")
+
+	assert.Contains(t, query["memory"], "shared")
+	memory := query["memory"].(common.MapStr)
+
+	assert.Contains(t, memory, "shared")
+	shared := memory["shared"].(common.MapStr)
+	assert.Contains(t, shared, "hit")
+	assert.Contains(t, shared, "read")
+	assert.Contains(t, shared, "dirtied")
+	assert.Contains(t, shared, "written")
+
+	assert.Contains(t, memory, "local")
+	local := memory["local"].(common.MapStr)
+	assert.Contains(t, local, "hit")
+	assert.Contains(t, local, "read")
+	assert.Contains(t, local, "dirtied")
+	assert.Contains(t, local, "written")
+
+	assert.Contains(t, memory, "temp")
+	temp := memory["temp"].(common.MapStr)
+	assert.Contains(t, temp, "read")
+	assert.Contains(t, temp, "written")
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, mtest.GetConfig("statement", r.Host()))
+
+	err := mbtest.WriteEvents(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
diff --git a/metricbeat/module/prometheus/collector/collector_integration_test.go b/metricbeat/module/prometheus/collector/collector_integration_test.go
index 1669078e643..cfa610cb898 100644
--- a/metricbeat/module/prometheus/collector/collector_integration_test.go
+++ b/metricbeat/module/prometheus/collector/collector_integration_test.go
@@ -34,26 +34,30 @@ import (
 
 func TestCollector(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
-			event, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
-
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
-
-			err := mbtest.WriteEvents(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Fetch": testFetch,
+		"Data":  testData,
 	})
 }
 
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
+	event, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
+
+	err := mbtest.WriteEvents(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
+
 func getConfig(host string) map[string]interface{} {
 	config := mtest.GetConfig("collector", host)
 	config["namespace"] = "collector"
diff --git a/metricbeat/module/prometheus/stats/stats_integration_test.go b/metricbeat/module/prometheus/stats/stats_integration_test.go
index 0f18a599753..6cc37b4d6c8 100644
--- a/metricbeat/module/prometheus/stats/stats_integration_test.go
+++ b/metricbeat/module/prometheus/stats/stats_integration_test.go
@@ -32,27 +32,31 @@ import (
 
 func TestStats(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, mtest.GetConfig("stats", r.Host()))
-			event, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
-
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-
-			// Check number of fields.
-			assert.Equal(t, 3, len(event))
-			assert.True(t, event["processes"].(common.MapStr)["open_fds"].(int64) > 0)
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, mtest.GetConfig("stats", r.Host()))
-
-			err := mbtest.WriteEvent(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Fetch": testFetch,
+		"Data":  testData,
 	})
 
 }
+
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, mtest.GetConfig("stats", r.Host()))
+	event, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+
+	// Check number of fields.
+	assert.Equal(t, 3, len(event))
+	assert.True(t, event["processes"].(common.MapStr)["open_fds"].(int64) > 0)
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, mtest.GetConfig("stats", r.Host()))
+
+	err := mbtest.WriteEvent(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
diff --git a/metricbeat/module/rabbitmq/connection/connection_integration_test.go b/metricbeat/module/rabbitmq/connection/connection_integration_test.go
index 420e65d5cce..e669f99dc5f 100644
--- a/metricbeat/module/rabbitmq/connection/connection_integration_test.go
+++ b/metricbeat/module/rabbitmq/connection/connection_integration_test.go
@@ -29,16 +29,18 @@ import (
 
 func TestConnection(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
-			err := mbtest.WriteEvents(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Data": testData,
 	})
 }
 
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
+	err := mbtest.WriteEvents(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
+
 func getConfig(host string) map[string]interface{} {
 	config := mtest.GetIntegrationConfig(host)
 	config["metricsets"] = []string{"connection"}
diff --git a/metricbeat/module/rabbitmq/exchange/exchange_integration_test.go b/metricbeat/module/rabbitmq/exchange/exchange_integration_test.go
index da20d0e7c47..c28c510d020 100644
--- a/metricbeat/module/rabbitmq/exchange/exchange_integration_test.go
+++ b/metricbeat/module/rabbitmq/exchange/exchange_integration_test.go
@@ -30,20 +30,22 @@ import (
 
 func TestExchange(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
-			err := mbtest.WriteEventsCond(f, t, func(e common.MapStr) bool {
-				hasIn, _ := e.HasKey("messages.publish_in")
-				hasOut, _ := e.HasKey("messages.publish_out")
-				return hasIn && hasOut
-			})
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Data": testData,
 	})
 }
 
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
+	err := mbtest.WriteEventsCond(f, t, func(e common.MapStr) bool {
+		hasIn, _ := e.HasKey("messages.publish_in")
+		hasOut, _ := e.HasKey("messages.publish_out")
+		return hasIn && hasOut
+	})
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
+
 func getConfig(host string) map[string]interface{} {
 	config := mtest.GetIntegrationConfig(host)
 	config["metricsets"] = []string{"exchange"}
diff --git a/metricbeat/module/rabbitmq/node/node_integration_test.go b/metricbeat/module/rabbitmq/node/node_integration_test.go
index ca0ae772b53..26b97ea245e 100644
--- a/metricbeat/module/rabbitmq/node/node_integration_test.go
+++ b/metricbeat/module/rabbitmq/node/node_integration_test.go
@@ -29,16 +29,18 @@ import (
 
 func TestNode(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Data": func(t *testing.T, r compose.R) {
-			ms := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host()))
-			err := mbtest.WriteEventsReporterV2(ms, t, "")
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Data": testData,
 	})
 }
 
+func testData(t *testing.T, r compose.R) {
+	ms := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host()))
+	err := mbtest.WriteEventsReporterV2(ms, t, "")
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
+
 func getConfig(host string) map[string]interface{} {
 	config := mtest.GetIntegrationConfig(host)
 	config["metricsets"] = []string{"node"}
diff --git a/metricbeat/module/rabbitmq/queue/queue_integration_test.go b/metricbeat/module/rabbitmq/queue/queue_integration_test.go
index 7f84b903fe4..026fc2c9163 100644
--- a/metricbeat/module/rabbitmq/queue/queue_integration_test.go
+++ b/metricbeat/module/rabbitmq/queue/queue_integration_test.go
@@ -30,19 +30,21 @@ import (
 
 func TestQueue(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
-			err := mbtest.WriteEventsCond(f, t, func(e common.MapStr) bool {
-				hasTotal, _ := e.HasKey("messages.total")
-				return hasTotal
-			})
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Data": testData,
 	})
 }
 
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
+	err := mbtest.WriteEventsCond(f, t, func(e common.MapStr) bool {
+		hasTotal, _ := e.HasKey("messages.total")
+		return hasTotal
+	})
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
+
 func getConfig(host string) map[string]interface{} {
 	config := mtest.GetIntegrationConfig(host)
 	config["metricsets"] = []string{"queue"}
diff --git a/metricbeat/module/redis/info/info_integration_test.go b/metricbeat/module/redis/info/info_integration_test.go
index 7d17df5b380..c3151929085 100644
--- a/metricbeat/module/redis/info/info_integration_test.go
+++ b/metricbeat/module/redis/info/info_integration_test.go
@@ -36,67 +36,71 @@ const (
 )
 
 func TestInfo(t *testing.T) {
-	t.Parallel()
-
 	mtest.Runner.Run(t, compose.Suite{
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, getConfig("", r.Host()))
-
-			err := mbtest.WriteEvent(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, getConfig("", r.Host()))
-			event, err := f.Fetch()
-			if err != nil {
-				t.Fatal("fetch", err)
-			}
-
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-
-			// Check fields
-			assert.Equal(t, 9, len(event))
-			server := event["server"].(common.MapStr)
-			assert.Equal(t, "standalone", server["mode"])
-		},
-		"Passwords": func(t *testing.T, r compose.R) {
-			// Add password and ensure it gets reset
-			defer func() {
-				err := resetPassword(r.Host(), password)
-				if err != nil {
-					t.Fatal("resetting password", err)
-				}
-			}()
-
-			err := addPassword(r.Host(), password)
-			if err != nil {
-				t.Fatal("adding password", err)
-			}
-
-			// Test Fetch metrics with missing password
-			f := mbtest.NewEventFetcher(t, getConfig("", r.Host()))
-			_, err = f.Fetch()
-			if assert.Error(t, err, "missing password") {
-				assert.Contains(t, err, "NOAUTH Authentication required.")
-			}
-
-			// Config redis and metricset with an invalid password
-			f = mbtest.NewEventFetcher(t, getConfig("blah", r.Host()))
-			_, err = f.Fetch()
-			if assert.Error(t, err, "invalid password") {
-				assert.Contains(t, err, "ERR invalid password")
-			}
-
-			// Config redis and metricset with a valid password
-			f = mbtest.NewEventFetcher(t, getConfig(password, r.Host()))
-			_, err = f.Fetch()
-			assert.NoError(t, err, "valid password")
-		},
+		"Fetch":     testFetch,
+		"Data":      testData,
+		"Passwords": testPasswords,
 	})
 }
 
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, getConfig("", r.Host()))
+	event, err := f.Fetch()
+	if err != nil {
+		t.Fatal("fetch", err)
+	}
+
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+
+	// Check fields
+	assert.Equal(t, 9, len(event))
+	server := event["server"].(common.MapStr)
+	assert.Equal(t, "standalone", server["mode"])
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, getConfig("", r.Host()))
+
+	err := mbtest.WriteEvent(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
+
+func testPasswords(t *testing.T, r compose.R) {
+	// Add password and ensure it gets reset
+	defer func() {
+		err := resetPassword(r.Host(), password)
+		if err != nil {
+			t.Fatal("resetting password", err)
+		}
+	}()
+
+	err := addPassword(r.Host(), password)
+	if err != nil {
+		t.Fatal("adding password", err)
+	}
+
+	// Test Fetch metrics with missing password
+	f := mbtest.NewEventFetcher(t, getConfig("", r.Host()))
+	_, err = f.Fetch()
+	if assert.Error(t, err, "missing password") {
+		assert.Contains(t, err, "NOAUTH Authentication required.")
+	}
+
+	// Config redis and metricset with an invalid password
+	f = mbtest.NewEventFetcher(t, getConfig("blah", r.Host()))
+	_, err = f.Fetch()
+	if assert.Error(t, err, "invalid password") {
+		assert.Contains(t, err, "ERR invalid password")
+	}
+
+	// Config redis and metricset with a valid password
+	f = mbtest.NewEventFetcher(t, getConfig(password, r.Host()))
+	_, err = f.Fetch()
+	assert.NoError(t, err, "valid password")
+}
+
 // addPassword will add a password to redis.
 func addPassword(host, pass string) error {
 	c, err := rd.Dial("tcp", host)
diff --git a/metricbeat/module/redis/keyspace/keyspace_integration_test.go b/metricbeat/module/redis/keyspace/keyspace_integration_test.go
index 4109d384568..08925d011f7 100644
--- a/metricbeat/module/redis/keyspace/keyspace_integration_test.go
+++ b/metricbeat/module/redis/keyspace/keyspace_integration_test.go
@@ -32,44 +32,46 @@ import (
 )
 
 func TestKeyspace(t *testing.T) {
-	t.Parallel()
-
 	mtest.Runner.Run(t, compose.Suite{
-		"Data": func(t *testing.T, r compose.R) {
-			addEntry(t, r.Host())
-
-			f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
-
-			err := mbtest.WriteEvents(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
-		"Fetch": func(t *testing.T, r compose.R) {
-			addEntry(t, r.Host())
-
-			// Fetch data
-			f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
-			events, err := f.Fetch()
-			if err != nil {
-				t.Fatal("fetch", err)
-			}
-
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events)
-
-			// Make sure at least 1 db keyspace exists
-			assert.True(t, len(events) > 0)
-
-			keyspace := events[0]
-
-			assert.True(t, keyspace["avg_ttl"].(int64) >= 0)
-			assert.True(t, keyspace["expires"].(int64) >= 0)
-			assert.True(t, keyspace["keys"].(int64) >= 0)
-			assert.True(t, strings.Contains(keyspace["id"].(string), "db"))
-		},
+		"Fetch": testFetch,
+		"Data":  testData,
 	})
 }
 
+func testFetch(t *testing.T, r compose.R) {
+	addEntry(t, r.Host())
+
+	// Fetch data
+	f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
+	events, err := f.Fetch()
+	if err != nil {
+		t.Fatal("fetch", err)
+	}
+
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events)
+
+	// Make sure at least 1 db keyspace exists
+	assert.True(t, len(events) > 0)
+
+	keyspace := events[0]
+
+	assert.True(t, keyspace["avg_ttl"].(int64) >= 0)
+	assert.True(t, keyspace["expires"].(int64) >= 0)
+	assert.True(t, keyspace["keys"].(int64) >= 0)
+	assert.True(t, strings.Contains(keyspace["id"].(string), "db"))
+}
+
+func testData(t *testing.T, r compose.R) {
+	addEntry(t, r.Host())
+
+	f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
+
+	err := mbtest.WriteEvents(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
+
 // addEntry adds an entry to redis
 func addEntry(t *testing.T, host string) {
 	// Insert at least one event to make sure db exists
diff --git a/metricbeat/module/traefik/health/health_integration_test.go b/metricbeat/module/traefik/health/health_integration_test.go
index c7fcabc4bad..48f3d4eef96 100644
--- a/metricbeat/module/traefik/health/health_integration_test.go
+++ b/metricbeat/module/traefik/health/health_integration_test.go
@@ -43,33 +43,37 @@ func makeBadRequest(config map[string]interface{}) error {
 
 func TestHealth(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			config := mtest.GetConfig("health", r.Host())
-
-			makeBadRequest(config)
-
-			ms := mbtest.NewReportingMetricSetV2(t, config)
-			reporter := &mbtest.CapturingReporterV2{}
-
-			ms.Fetch(reporter)
-			assert.Nil(t, reporter.GetErrors(), "Errors while fetching metrics")
-
-			event := reporter.GetEvents()[0]
-			assert.NotNil(t, event)
-			t.Logf("%s/%s event: %+v", ms.Module().Name(), ms.Name(), event)
-
-			responseCount, _ := event.MetricSetFields.GetValue("response.count")
-			assert.True(t, responseCount.(int64) >= 1)
-
-			badResponseCount, _ := event.MetricSetFields.GetValue("response.status_codes.404")
-			assert.True(t, badResponseCount.(float64) >= 1)
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			ms := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("health", r.Host()))
-			err := mbtest.WriteEventsReporterV2(ms, t, "")
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Fetch": testFetch,
+		"Data":  testData,
 	})
 }
+
+func testFetch(t *testing.T, r compose.R) {
+	config := mtest.GetConfig("health", r.Host())
+
+	makeBadRequest(config)
+
+	ms := mbtest.NewReportingMetricSetV2(t, config)
+	reporter := &mbtest.CapturingReporterV2{}
+
+	ms.Fetch(reporter)
+	assert.Nil(t, reporter.GetErrors(), "Errors while fetching metrics")
+
+	event := reporter.GetEvents()[0]
+	assert.NotNil(t, event)
+	t.Logf("%s/%s event: %+v", ms.Module().Name(), ms.Name(), event)
+
+	responseCount, _ := event.MetricSetFields.GetValue("response.count")
+	assert.True(t, responseCount.(int64) >= 1)
+
+	badResponseCount, _ := event.MetricSetFields.GetValue("response.status_codes.404")
+	assert.True(t, badResponseCount.(float64) >= 1)
+}
+
+func testData(t *testing.T, r compose.R) {
+	ms := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("health", r.Host()))
+	err := mbtest.WriteEventsReporterV2(ms, t, "")
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
diff --git a/metricbeat/module/uwsgi/status/status_integration_test.go b/metricbeat/module/uwsgi/status/status_integration_test.go
index b76380c4b95..67726728d57 100644
--- a/metricbeat/module/uwsgi/status/status_integration_test.go
+++ b/metricbeat/module/uwsgi/status/status_integration_test.go
@@ -28,35 +28,39 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+func TestStatusTCP(t *testing.T) {
+	testStatus(t, "uwsgi_tcp")
+}
+
+func TestStatusHTTP(t *testing.T) {
+	testStatus(t, "uwsgi_http")
+}
+
 func testStatus(t *testing.T, service string) {
 	runner := compose.TestRunner{Service: service}
 
 	runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, getConfig(t, service, r.Host()))
-			events, err := f.Fetch()
-			assert.NoError(t, err)
-
-			assert.True(t, len(events) > 0)
-			totals := findItems(events, "total")
-			assert.Equal(t, 1, len(totals))
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, getConfig(t, service, r.Host()))
-			err := mbtest.WriteEvents(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Fetch": testFetch,
+		"Data":  testData,
 	})
 }
 
-func TestStatusTCP(t *testing.T) {
-	testStatus(t, "uwsgi_tcp")
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, getConfig(t, service, r.Host()))
+	events, err := f.Fetch()
+	assert.NoError(t, err)
+
+	assert.True(t, len(events) > 0)
+	totals := findItems(events, "total")
+	assert.Equal(t, 1, len(totals))
 }
 
-func TestStatusHTTP(t *testing.T) {
-	testStatus(t, "uwsgi_http")
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, getConfig(t, service, r.Host()))
+	err := mbtest.WriteEvents(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
 }
 
 func getConfig(t *testing.T, service string, host string) map[string]interface{} {
diff --git a/metricbeat/module/zookeeper/mntr/mntr_integration_test.go b/metricbeat/module/zookeeper/mntr/mntr_integration_test.go
index d73aa21b5fb..8c3578ef0db 100644
--- a/metricbeat/module/zookeeper/mntr/mntr_integration_test.go
+++ b/metricbeat/module/zookeeper/mntr/mntr_integration_test.go
@@ -33,39 +33,42 @@ func TestMntr(t *testing.T) {
 	runner := compose.TestRunner{Service: "zookeeper", Parallel: true}
 
 	runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
-			event, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
+		"Fetch": testFetch,
+		"Data":  testData,
+	})
 
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+}
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+	event, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
 
-			// Check values
-			version := event["version"].(string)
-			avgLatency := event["latency"].(common.MapStr)["avg"].(int64)
-			maxLatency := event["latency"].(common.MapStr)["max"].(int64)
-			numAliveConnections := event["num_alive_connections"].(int64)
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
 
-			assert.Equal(t, version, "3.4.8--1, built on 02/06/2016 03:18 GMT")
-			assert.True(t, avgLatency >= 0)
-			assert.True(t, maxLatency >= 0)
-			assert.True(t, numAliveConnections > 0)
+	// Check values
+	version := event["version"].(string)
+	avgLatency := event["latency"].(common.MapStr)["avg"].(int64)
+	maxLatency := event["latency"].(common.MapStr)["max"].(int64)
+	numAliveConnections := event["num_alive_connections"].(int64)
 
-			// Check number of fields. At least 10, depending on environment
-			assert.True(t, len(event) >= 10)
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+	assert.Equal(t, version, "3.4.8--1, built on 02/06/2016 03:18 GMT")
+	assert.True(t, avgLatency >= 0)
+	assert.True(t, maxLatency >= 0)
+	assert.True(t, numAliveConnections > 0)
 
-			err := mbtest.WriteEvent(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
-	})
+	// Check number of fields. At least 10, depending on environment
+	assert.True(t, len(event) >= 10)
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
 
+	err := mbtest.WriteEvent(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
 }
 
 func getConfig(host string) map[string]interface{} {

From 494aa299904102e0a7b7be6718aca17433ee5e7c Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Fri, 23 Nov 2018 17:11:46 +0100
Subject: [PATCH 77/88] Fix tests after refactor

---
 .../stubstatus/stubstatus_integration_test.go |  2 +-
 .../uwsgi/status/status_integration_test.go   | 19 +++++++++++++------
 2 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/metricbeat/module/nginx/stubstatus/stubstatus_integration_test.go b/metricbeat/module/nginx/stubstatus/stubstatus_integration_test.go
index 68d998bcde6..7560df4af2a 100644
--- a/metricbeat/module/nginx/stubstatus/stubstatus_integration_test.go
+++ b/metricbeat/module/nginx/stubstatus/stubstatus_integration_test.go
@@ -35,7 +35,7 @@ func TestStubstatus(t *testing.T) {
 	}
 
 	runner.Run(t, compose.Suite{
-		"Fetch": tesFetch,
+		"Fetch": testFetch,
 		"Data":  testData,
 	})
 }
diff --git a/metricbeat/module/uwsgi/status/status_integration_test.go b/metricbeat/module/uwsgi/status/status_integration_test.go
index 67726728d57..b6ad484d784 100644
--- a/metricbeat/module/uwsgi/status/status_integration_test.go
+++ b/metricbeat/module/uwsgi/status/status_integration_test.go
@@ -29,24 +29,31 @@ import (
 )
 
 func TestStatusTCP(t *testing.T) {
+	t.Parallel()
 	testStatus(t, "uwsgi_tcp")
 }
 
 func TestStatusHTTP(t *testing.T) {
+	t.Parallel()
 	testStatus(t, "uwsgi_http")
 }
 
 func testStatus(t *testing.T, service string) {
 	runner := compose.TestRunner{Service: service}
 
+	test := uwsgiTest{Service: service}
 	runner.Run(t, compose.Suite{
-		"Fetch": testFetch,
-		"Data":  testData,
+		"Fetch": test.Fetch,
+		"Data":  test.Data,
 	})
 }
 
-func testFetch(t *testing.T, r compose.R) {
-	f := mbtest.NewEventsFetcher(t, getConfig(t, service, r.Host()))
+type uwsgiTest struct {
+	Service string
+}
+
+func (u *uwsgiTest) Fetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, getConfig(t, u.Service, r.Host()))
 	events, err := f.Fetch()
 	assert.NoError(t, err)
 
@@ -55,8 +62,8 @@ func testFetch(t *testing.T, r compose.R) {
 	assert.Equal(t, 1, len(totals))
 }
 
-func testData(t *testing.T, r compose.R) {
-	f := mbtest.NewEventsFetcher(t, getConfig(t, service, r.Host()))
+func (u *uwsgiTest) Data(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, getConfig(t, u.Service, r.Host()))
 	err := mbtest.WriteEvents(f, t)
 	if err != nil {
 		t.Fatal("write", err)

From 1f0bb6f3e643ac6774b84ea6f7efe6f6561ede6f Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 27 Nov 2018 18:17:14 +0100
Subject: [PATCH 78/88] Remove leftover comment

---
 libbeat/tests/compose/project.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/libbeat/tests/compose/project.go b/libbeat/tests/compose/project.go
index 89249f88d08..776f3583d7e 100644
--- a/libbeat/tests/compose/project.go
+++ b/libbeat/tests/compose/project.go
@@ -59,7 +59,6 @@ type Driver interface {
 	Down(ctx context.Context) error
 	Kill(ctx context.Context, signal string, service string) error
 	Ps(ctx context.Context, filter ...string) ([]ContainerStatus, error)
-	// Containers(ctx context.Context, projectFilter Filter, filter ...string) ([]string, error)
 
 	SetParameters(map[string]string)
 

From 77e24a7ad65bdfccee6176970b0be41fbb928c3a Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 27 Nov 2018 18:58:12 +0100
Subject: [PATCH 79/88] Try to make hound CI happy

---
 libbeat/tests/compose/runner.go               | 27 ++++++++++++++++---
 metricbeat/module/aerospike/mtest/runner.go   |  1 +
 metricbeat/module/apache/mtest/runner.go      |  1 +
 metricbeat/module/kafka/mtest/helpers.go      |  2 ++
 metricbeat/module/kafka/mtest/runner.go       |  1 +
 metricbeat/module/kibana/mtest/testing.go     |  1 +
 metricbeat/module/logstash/testing.go         |  1 +
 metricbeat/module/mongodb/mtest/testing.go    |  2 ++
 metricbeat/module/mysql/mtest/testing.go      |  1 +
 metricbeat/module/php_fpm/mtest/testing.go    |  2 ++
 metricbeat/module/prometheus/mtest/testing.go |  2 ++
 metricbeat/module/rabbitmq/mtest/runner.go    |  1 +
 metricbeat/module/redis/mtest/runner.go       |  1 +
 metricbeat/module/traefik/mtest/runner.go     |  1 +
 14 files changed, 41 insertions(+), 3 deletions(-)

diff --git a/libbeat/tests/compose/runner.go b/libbeat/tests/compose/runner.go
index 233caf675cb..12206ef5bdf 100644
--- a/libbeat/tests/compose/runner.go
+++ b/libbeat/tests/compose/runner.go
@@ -34,15 +34,30 @@ func init() {
 	rand.Seed(time.Now().UnixNano())
 }
 
+// TestRunner starts a service with different combinations of options and
+// runs tests on each one of these combinations
 type TestRunner struct {
-	Service  string
-	Options  RunnerOptions
+	// Name of the service managed by this runner
+	Service string
+
+	// Map of options with the list of possible values
+	Options RunnerOptions
+
+	// Set to true if this runner can run in parallel with other runners
 	Parallel bool
-	Timeout  int
+
+	// Timeout to start the managed service
+	Timeout int
 }
 
+// Suite is a set of tests to be run with a TestRunner
+// Each test must be one of:
+// - func(R)
+// - func(*testing.T, R)
+// - func(*testing.T)
 type Suite map[string]interface{}
 
+// RunnerOptions are the possible options of a runner scenario
 type RunnerOptions map[string][]string
 
 func (r *TestRunner) scenarios() []map[string]string {
@@ -107,6 +122,7 @@ func (r *TestRunner) runHostOverride(t *testing.T, tests Suite) bool {
 	return true
 }
 
+// Run runs a tests suite
 func (r *TestRunner) Run(t *testing.T, tests Suite) {
 	t.Helper()
 
@@ -198,26 +214,31 @@ type runnerControl struct {
 	scenario map[string]string
 }
 
+// WithT creates a copy of R with the given T
 func (r *runnerControl) WithT(t *testing.T) R {
 	ctl := *r
 	ctl.T = t
 	return &ctl
 }
 
+// Host returns the host:port the test should use to connect to the service
 func (r *runnerControl) Host() string {
 	return r.host
 }
 
+// Hostname is the address of the host
 func (r *runnerControl) Hostname() string {
 	hostname, _, _ := net.SplitHostPort(r.host)
 	return hostname
 }
 
+// Port is the port of the host
 func (r *runnerControl) Port() string {
 	_, port, _ := net.SplitHostPort(r.host)
 	return port
 }
 
+// Option returns the value of an option for the current scenario
 func (r *runnerControl) Option(key string) string {
 	return r.scenario[key]
 }
diff --git a/metricbeat/module/aerospike/mtest/runner.go b/metricbeat/module/aerospike/mtest/runner.go
index fbd3ef5b56c..2463ad13c88 100644
--- a/metricbeat/module/aerospike/mtest/runner.go
+++ b/metricbeat/module/aerospike/mtest/runner.go
@@ -24,6 +24,7 @@ import (
 )
 
 var (
+	// Runner is the compose test runner for aerospike
 	Runner = compose.TestRunner{
 		Service: "aerospike",
 		Options: compose.RunnerOptions{
diff --git a/metricbeat/module/apache/mtest/runner.go b/metricbeat/module/apache/mtest/runner.go
index b069377ee91..2b86eedb79c 100644
--- a/metricbeat/module/apache/mtest/runner.go
+++ b/metricbeat/module/apache/mtest/runner.go
@@ -24,6 +24,7 @@ import (
 )
 
 var (
+	// Runner is the compose test runner for apache
 	Runner = compose.TestRunner{
 		Service: "apache",
 		Options: map[string][]string{
diff --git a/metricbeat/module/kafka/mtest/helpers.go b/metricbeat/module/kafka/mtest/helpers.go
index 024784a210e..4f3c309260d 100644
--- a/metricbeat/module/kafka/mtest/helpers.go
+++ b/metricbeat/module/kafka/mtest/helpers.go
@@ -26,6 +26,7 @@ import (
 	saramacluster "github.com/bsm/sarama-cluster"
 )
 
+// GenerateKafkaData generates kafka data for tests
 func GenerateKafkaData(t *testing.T, topic string, host string) {
 	t.Logf("Send Kafka Event to topic: %v", topic)
 
@@ -64,6 +65,7 @@ func GenerateKafkaData(t *testing.T, topic string, host string) {
 	}
 }
 
+// StartConsumer starts a kafka consumer for tests
 func StartConsumer(t *testing.T, topic, host string) (io.Closer, error) {
 	brokers := []string{host}
 	topics := []string{topic}
diff --git a/metricbeat/module/kafka/mtest/runner.go b/metricbeat/module/kafka/mtest/runner.go
index e3b6a5c2e80..56ce7b056b6 100644
--- a/metricbeat/module/kafka/mtest/runner.go
+++ b/metricbeat/module/kafka/mtest/runner.go
@@ -24,6 +24,7 @@ import (
 )
 
 var (
+	// Runner is the compose test runner for kafka tests
 	Runner = compose.TestRunner{
 		Service: "kafka",
 		Options: compose.RunnerOptions{
diff --git a/metricbeat/module/kibana/mtest/testing.go b/metricbeat/module/kibana/mtest/testing.go
index a8f1d191f80..968a78d6074 100644
--- a/metricbeat/module/kibana/mtest/testing.go
+++ b/metricbeat/module/kibana/mtest/testing.go
@@ -22,6 +22,7 @@ import (
 )
 
 var (
+	// Runner is the compose test runner for kibana tests
 	Runner = compose.TestRunner{
 		Service:  "kibana",
 		Parallel: true,
diff --git a/metricbeat/module/logstash/testing.go b/metricbeat/module/logstash/testing.go
index 3a7137e8987..01055e93181 100644
--- a/metricbeat/module/logstash/testing.go
+++ b/metricbeat/module/logstash/testing.go
@@ -22,6 +22,7 @@ import (
 )
 
 var (
+	// Runner is the compose test runner for logstash
 	Runner = compose.TestRunner{
 		Service:  "logstash",
 		Parallel: true,
diff --git a/metricbeat/module/mongodb/mtest/testing.go b/metricbeat/module/mongodb/mtest/testing.go
index 5d4200825a8..f1657d1f161 100644
--- a/metricbeat/module/mongodb/mtest/testing.go
+++ b/metricbeat/module/mongodb/mtest/testing.go
@@ -21,11 +21,13 @@ import (
 	"github.com/elastic/beats/libbeat/tests/compose"
 )
 
+// Runner is the compose test runner for mongodb
 var Runner = compose.TestRunner{
 	Service:  "mongodb",
 	Parallel: true,
 }
 
+// GetConfig creates a config for a metricset and host
 func GetConfig(metricset, host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "mongodb",
diff --git a/metricbeat/module/mysql/mtest/testing.go b/metricbeat/module/mysql/mtest/testing.go
index 9a8410ccb8d..62477a029e4 100644
--- a/metricbeat/module/mysql/mtest/testing.go
+++ b/metricbeat/module/mysql/mtest/testing.go
@@ -23,6 +23,7 @@ import (
 	"github.com/elastic/beats/libbeat/tests/compose"
 )
 
+// Runner is a compose test runner for mysql
 var Runner = compose.TestRunner{
 	Service:  "mysql",
 	Parallel: true,
diff --git a/metricbeat/module/php_fpm/mtest/testing.go b/metricbeat/module/php_fpm/mtest/testing.go
index 7f427379fb4..bf5cce996d3 100644
--- a/metricbeat/module/php_fpm/mtest/testing.go
+++ b/metricbeat/module/php_fpm/mtest/testing.go
@@ -23,11 +23,13 @@ import (
 	"github.com/elastic/beats/libbeat/tests/compose"
 )
 
+// Runner is a compose test runner for php fpm
 var Runner = compose.TestRunner{
 	Service:  "phpfpm",
 	Parallel: true,
 }
 
+// GetConfig creates a configuration for a metricset and host
 func GetConfig(metricset, host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "php_fpm",
diff --git a/metricbeat/module/prometheus/mtest/testing.go b/metricbeat/module/prometheus/mtest/testing.go
index f6b7f18106f..fb32c5e7a38 100644
--- a/metricbeat/module/prometheus/mtest/testing.go
+++ b/metricbeat/module/prometheus/mtest/testing.go
@@ -22,12 +22,14 @@ import (
 )
 
 var (
+	// Runner is a compose test runner for prometheus
 	Runner = compose.TestRunner{
 		Service:  "prometheus",
 		Parallel: true,
 	}
 )
 
+// GetConfig creates a configuration for a metricset and host
 func GetConfig(metricset, host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "prometheus",
diff --git a/metricbeat/module/rabbitmq/mtest/runner.go b/metricbeat/module/rabbitmq/mtest/runner.go
index a70257d78b9..f8b8a0c0a27 100644
--- a/metricbeat/module/rabbitmq/mtest/runner.go
+++ b/metricbeat/module/rabbitmq/mtest/runner.go
@@ -22,6 +22,7 @@ import (
 )
 
 var (
+	// Runner is a compose test runner for RabbitMQ tests
 	Runner = compose.TestRunner{
 		Service:  "rabbitmq",
 		Parallel: true,
diff --git a/metricbeat/module/redis/mtest/runner.go b/metricbeat/module/redis/mtest/runner.go
index 6d3dbd546e8..4ffb857ffc7 100644
--- a/metricbeat/module/redis/mtest/runner.go
+++ b/metricbeat/module/redis/mtest/runner.go
@@ -24,6 +24,7 @@ import (
 )
 
 var (
+	// Runner is a compose test runner for Redis tests
 	Runner = compose.TestRunner{
 		Service: "redis",
 		Options: compose.RunnerOptions{
diff --git a/metricbeat/module/traefik/mtest/runner.go b/metricbeat/module/traefik/mtest/runner.go
index 10ce0edf162..896aca2eab7 100644
--- a/metricbeat/module/traefik/mtest/runner.go
+++ b/metricbeat/module/traefik/mtest/runner.go
@@ -22,6 +22,7 @@ import (
 )
 
 var (
+	// Runner is a compose test runner for traefik
 	Runner = compose.TestRunner{
 		Service:  "traefik",
 		Parallel: true,

From d80c4d4f5dfa00e50ba81e18f51e8b6767420dbd Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 27 Nov 2018 19:01:23 +0100
Subject: [PATCH 80/88] Remove commented out line

---
 libbeat/scripts/Makefile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile
index 23e08ce032a..b07f2f04f37 100755
--- a/libbeat/scripts/Makefile
+++ b/libbeat/scripts/Makefile
@@ -375,7 +375,6 @@ import-dashboards: update ${BEAT_NAME}
 # Builds the environment to test beat
 .PHONY: build-image
 build-image: write-environment
-	# ${DOCKER_COMPOSE} build ${DOCKER_NOCACHE} --pull --force-rm
 
 # Runs the environment so the redis and elasticsearch can also be used for local development
 # To use it for running the test, set ES_HOST and REDIS_HOST environment variable to the ip of your docker-machine.

From fc39b364271925210afb11486923ff1e2f3159c9 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 27 Nov 2018 19:17:43 +0100
Subject: [PATCH 81/88] Add tests for scenarios

---
 libbeat/tests/compose/runner_test.go | 80 ++++++++++++++++++++++++++++
 1 file changed, 80 insertions(+)
 create mode 100644 libbeat/tests/compose/runner_test.go

diff --git a/libbeat/tests/compose/runner_test.go b/libbeat/tests/compose/runner_test.go
new file mode 100644
index 00000000000..30b5d188460
--- /dev/null
+++ b/libbeat/tests/compose/runner_test.go
@@ -0,0 +1,80 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package compose
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRunnerScenarios(t *testing.T) {
+	cases := []struct {
+		Title    string
+		Options  RunnerOptions
+		Expected []map[string]string
+	}{
+		{
+			Title:    "Nil options",
+			Options:  nil,
+			Expected: []map[string]string{nil},
+		},
+		{
+			Title:    "Empty options",
+			Options:  RunnerOptions{},
+			Expected: []map[string]string{nil},
+		},
+		{
+			Title: "One option, two values",
+			Options: RunnerOptions{
+				"FOO": {"bar", "baz"},
+			},
+			Expected: []map[string]string{
+				{"FOO": "bar"},
+				{"FOO": "baz"},
+			},
+		},
+		{
+			Title: "Multiple options",
+			Options: RunnerOptions{
+				"FOO": {"bar", "baz"},
+				"BAZ": {"stuff"},
+			},
+			Expected: []map[string]string{
+				{"FOO": "bar", "BAZ": "stuff"},
+				{"FOO": "baz", "BAZ": "stuff"},
+			},
+		},
+		{
+			Title: "Multiple options, single values",
+			Options: RunnerOptions{
+				"FOO": {"bar"},
+				"BAZ": {"stuff"},
+			},
+			Expected: []map[string]string{
+				{"FOO": "bar", "BAZ": "stuff"},
+			},
+		},
+	}
+
+	for _, c := range cases {
+		r := TestRunner{Options: c.Options}
+		found := r.scenarios()
+		assert.Equal(t, c.Expected, found, c.Title)
+	}
+}

From 0de565e734648d36a33ff228aa50cff432063aba Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Sat, 19 Jan 2019 20:59:26 +0100
Subject: [PATCH 82/88] Fix nats tests

---
 metricbeat/docker-compose.yml                 |  2 +
 metricbeat/module/nats/_meta/env              |  2 -
 .../connections_integration_test.go           | 41 ++++++-------------
 metricbeat/module/nats/mtest/runner.go        | 31 ++++++++++++++
 .../nats/routes/routes_integration_test.go    | 41 ++++++-------------
 .../nats/stats/stats_integration_test.go      | 41 ++++++-------------
 metricbeat/tests/system/test_nats.py          |  5 +--
 7 files changed, 73 insertions(+), 90 deletions(-)
 delete mode 100644 metricbeat/module/nats/_meta/env
 create mode 100644 metricbeat/module/nats/mtest/runner.go

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index e8515ffd281..4e596f55674 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -190,6 +190,8 @@ services:
 
   nats:
     build: ./module/nats/_meta
+    ports:
+      - 8222
 
   nginx:
     build: ./module/nginx/_meta
diff --git a/metricbeat/module/nats/_meta/env b/metricbeat/module/nats/_meta/env
deleted file mode 100644
index 69e439d0a4f..00000000000
--- a/metricbeat/module/nats/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-NATS_HOST=nats
-NATS_PORT=8222
diff --git a/metricbeat/module/nats/connections/connections_integration_test.go b/metricbeat/module/nats/connections/connections_integration_test.go
index c5bc1bbd095..c15c3603808 100644
--- a/metricbeat/module/nats/connections/connections_integration_test.go
+++ b/metricbeat/module/nats/connections/connections_integration_test.go
@@ -20,57 +20,42 @@
 package connections
 
 import (
-	"os"
 	"testing"
 
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
+	"github.com/elastic/beats/metricbeat/module/nats/mtest"
 )
 
-func TestData(t *testing.T) {
-	compose.EnsureUp(t, "nats")
+func TestConnections(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": testFetch,
+		"Data":  testData,
+	})
+}
 
-	metricSet := mbtest.NewReportingMetricSetV2(t, getConfig())
+func testData(t *testing.T, r compose.R) {
+	metricSet := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host()))
 	err := mbtest.WriteEventsReporterV2(metricSet, t, "./test_data.json")
 	if err != nil {
 		t.Fatal("write", err)
 	}
 }
 
-func TestFetch(t *testing.T) {
-	compose.EnsureUp(t, "nats")
-
+func testFetch(t *testing.T, r compose.R) {
 	reporter := &mbtest.CapturingReporterV2{}
 
-	metricSet := mbtest.NewReportingMetricSetV2(t, getConfig())
+	metricSet := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host()))
 	metricSet.Fetch(reporter)
 
 	e := mbtest.StandardizeEvent(metricSet, reporter.GetEvents()[0])
 	t.Logf("%s/%s event: %+v", metricSet.Module().Name(), metricSet.Name(), e.Fields.StringToPrint())
 }
 
-func getConfig() map[string]interface{} {
+func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "nats",
 		"metricsets": []string{"connections"},
-		"hosts":      []string{GetEnvHost() + ":" + GetEnvPort()},
-	}
-}
-
-func GetEnvHost() string {
-	host := os.Getenv("NATS_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
-}
-
-func GetEnvPort() string {
-	port := os.Getenv("NATS_PORT")
-
-	if len(port) == 0 {
-		port = "8222"
+		"hosts":      []string{host},
 	}
-	return port
 }
diff --git a/metricbeat/module/nats/mtest/runner.go b/metricbeat/module/nats/mtest/runner.go
new file mode 100644
index 00000000000..25174c34361
--- /dev/null
+++ b/metricbeat/module/nats/mtest/runner.go
@@ -0,0 +1,31 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// +build integration
+
+package mtest
+
+import (
+	"github.com/elastic/beats/libbeat/tests/compose"
+)
+
+var (
+	// Runner is a compose test runner for Nats tests
+	Runner = compose.TestRunner{
+		Service: "nats",
+	}
+)
diff --git a/metricbeat/module/nats/routes/routes_integration_test.go b/metricbeat/module/nats/routes/routes_integration_test.go
index 8481fc3a49c..57e1f8213c5 100644
--- a/metricbeat/module/nats/routes/routes_integration_test.go
+++ b/metricbeat/module/nats/routes/routes_integration_test.go
@@ -20,57 +20,42 @@
 package routes
 
 import (
-	"os"
 	"testing"
 
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
+	mtest "github.com/elastic/beats/metricbeat/module/nats/mtest"
 )
 
-func TestData(t *testing.T) {
-	compose.EnsureUp(t, "nats")
+func TestRoutes(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": testFetch,
+		"Data":  testData,
+	})
+}
 
-	metricSet := mbtest.NewReportingMetricSetV2(t, getConfig())
+func testData(t *testing.T, r compose.R) {
+	metricSet := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host()))
 	err := mbtest.WriteEventsReporterV2(metricSet, t, "./test_data.json")
 	if err != nil {
 		t.Fatal("write", err)
 	}
 }
 
-func TestFetch(t *testing.T) {
-	compose.EnsureUp(t, "nats")
-
+func testFetch(t *testing.T, r compose.R) {
 	reporter := &mbtest.CapturingReporterV2{}
 
-	metricSet := mbtest.NewReportingMetricSetV2(t, getConfig())
+	metricSet := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host()))
 	metricSet.Fetch(reporter)
 
 	e := mbtest.StandardizeEvent(metricSet, reporter.GetEvents()[0])
 	t.Logf("%s/%s event: %+v", metricSet.Module().Name(), metricSet.Name(), e.Fields.StringToPrint())
 }
 
-func getConfig() map[string]interface{} {
+func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "nats",
 		"metricsets": []string{"routes"},
-		"hosts":      []string{GetEnvHost() + ":" + GetEnvPort()},
-	}
-}
-
-func GetEnvHost() string {
-	host := os.Getenv("NATS_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
-}
-
-func GetEnvPort() string {
-	port := os.Getenv("NATS_PORT")
-
-	if len(port) == 0 {
-		port = "8222"
+		"hosts":      []string{host},
 	}
-	return port
 }
diff --git a/metricbeat/module/nats/stats/stats_integration_test.go b/metricbeat/module/nats/stats/stats_integration_test.go
index 171f7b7904f..db82d381f19 100644
--- a/metricbeat/module/nats/stats/stats_integration_test.go
+++ b/metricbeat/module/nats/stats/stats_integration_test.go
@@ -20,57 +20,42 @@
 package stats
 
 import (
-	"os"
 	"testing"
 
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
+	"github.com/elastic/beats/metricbeat/module/nats/mtest"
 )
 
-func TestData(t *testing.T) {
-	compose.EnsureUp(t, "nats")
+func TestStats(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": testFetch,
+		"Data":  testData,
+	})
+}
 
-	metricSet := mbtest.NewReportingMetricSetV2(t, getConfig())
+func testData(t *testing.T, r compose.R) {
+	metricSet := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host()))
 	err := mbtest.WriteEventsReporterV2(metricSet, t, "./test_data.json")
 	if err != nil {
 		t.Fatal("write", err)
 	}
 }
 
-func TestFetch(t *testing.T) {
-	compose.EnsureUp(t, "nats")
-
+func testFetch(t *testing.T, r compose.R) {
 	reporter := &mbtest.CapturingReporterV2{}
 
-	metricSet := mbtest.NewReportingMetricSetV2(t, getConfig())
+	metricSet := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host()))
 	metricSet.Fetch(reporter)
 
 	e := mbtest.StandardizeEvent(metricSet, reporter.GetEvents()[0])
 	t.Logf("%s/%s event: %+v", metricSet.Module().Name(), metricSet.Name(), e.Fields.StringToPrint())
 }
 
-func getConfig() map[string]interface{} {
+func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "nats",
 		"metricsets": []string{"stats"},
-		"hosts":      []string{GetEnvHost() + ":" + GetEnvPort()},
-	}
-}
-
-func GetEnvHost() string {
-	host := os.Getenv("NATS_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
-}
-
-func GetEnvPort() string {
-	port := os.Getenv("NATS_PORT")
-
-	if len(port) == 0 {
-		port = "8222"
+		"hosts":      []string{host},
 	}
-	return port
 }
diff --git a/metricbeat/tests/system/test_nats.py b/metricbeat/tests/system/test_nats.py
index c40badd42a6..352f927b05d 100644
--- a/metricbeat/tests/system/test_nats.py
+++ b/metricbeat/tests/system/test_nats.py
@@ -85,7 +85,4 @@ def test_routes(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return ["{}:{}".format(
-            os.getenv('NATS_HOST', 'localhost'),
-            os.getenv('NATS_PORT', '8222')
-        )]
+        return [self.compose_host()]

From a200df2bfb7d59304f6c8322e278a0d98d166ffa Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Sat, 19 Jan 2019 21:13:53 +0100
Subject: [PATCH 83/88] Add ES versions

---
 .../module/elasticsearch/elasticsearch_integration_test.go  | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/metricbeat/module/elasticsearch/elasticsearch_integration_test.go b/metricbeat/module/elasticsearch/elasticsearch_integration_test.go
index 1ae6a710ab1..1c30f0c3814 100644
--- a/metricbeat/module/elasticsearch/elasticsearch_integration_test.go
+++ b/metricbeat/module/elasticsearch/elasticsearch_integration_test.go
@@ -62,10 +62,12 @@ func TestElasticsearch(t *testing.T) {
 		Service: "elasticsearch",
 		Options: compose.RunnerOptions{
 			"ELASTICSEARCH_VERSION": {
+				// "7.0.0-alpha2",
+				"6.5.4",
 				"6.4.3",
 				"6.3.2",
 				"6.2.4",
-				"5.6.11",
+				"5.6.14",
 			},
 		},
 		Parallel: true,
@@ -78,7 +80,7 @@ func TestElasticsearch(t *testing.T) {
 }
 
 func testFetch(t *testing.T, r compose.R) {
-	if r.Option("ELASTICSEARCH_VERSION") == "6.2.4" {
+	if v := r.Option("ELASTICSEARCH_VERSION"); v == "6.2.4" || v == "5.6.14" {
 		t.Skip("This test fails on this version")
 	}
 

From 76a402c936a77c0b974bc9e7d79d563efd2ee16f Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Sat, 19 Jan 2019 21:45:11 +0100
Subject: [PATCH 84/88] Fix mssql tests

---
 x-pack/metricbeat/docker-compose.yml           |  5 +++--
 x-pack/metricbeat/module/mssql/_meta/env       |  4 ----
 .../module/mssql/db/data_integration_test.go   |  7 ++++---
 .../module/mssql/db/db_integration_test.go     | 14 ++++++++++----
 .../module/mssql/{testing => mtest}/mssql.go   |  8 ++++----
 x-pack/metricbeat/module/mssql/mtest/runner.go | 18 ++++++++++++++++++
 .../mssql/performance/data_integration_test.go | 18 +++++++-----------
 .../performance_integration_test.go            | 17 +++++++++++------
 x-pack/metricbeat/tests/system/test_mssql.py   |  2 +-
 9 files changed, 58 insertions(+), 35 deletions(-)
 delete mode 100644 x-pack/metricbeat/module/mssql/_meta/env
 rename x-pack/metricbeat/module/mssql/{testing => mtest}/mssql.go (79%)
 create mode 100644 x-pack/metricbeat/module/mssql/mtest/runner.go

diff --git a/x-pack/metricbeat/docker-compose.yml b/x-pack/metricbeat/docker-compose.yml
index 1a67babb75d..5d3bb202f23 100644
--- a/x-pack/metricbeat/docker-compose.yml
+++ b/x-pack/metricbeat/docker-compose.yml
@@ -8,10 +8,11 @@ services:
     volumes:
       - ${PWD}/../..:/go/src/github.com/elastic/beats/
       - /var/run/docker.sock:/var/run/docker.sock
+    network_mode: host
     command: make
-    env_file:
-      - ./module/mssql/_meta/env
 
   # Modules
   mssql:
     build: ./module/mssql/_meta
+    ports:
+      - 1433
diff --git a/x-pack/metricbeat/module/mssql/_meta/env b/x-pack/metricbeat/module/mssql/_meta/env
deleted file mode 100644
index 145a143e51f..00000000000
--- a/x-pack/metricbeat/module/mssql/_meta/env
+++ /dev/null
@@ -1,4 +0,0 @@
-MSSQL_HOST=mssql
-MSSQL_PORT=1433
-MSSQL_USER=SA
-MSSQL_PASSWORD=1234_asdf
diff --git a/x-pack/metricbeat/module/mssql/db/data_integration_test.go b/x-pack/metricbeat/module/mssql/db/data_integration_test.go
index 7a8fc8bafc5..a64dcb41f65 100644
--- a/x-pack/metricbeat/module/mssql/db/data_integration_test.go
+++ b/x-pack/metricbeat/module/mssql/db/data_integration_test.go
@@ -7,14 +7,15 @@ package db
 import (
 	"testing"
 
+	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	mtest "github.com/elastic/beats/x-pack/metricbeat/module/mssql/testing"
+	"github.com/elastic/beats/x-pack/metricbeat/module/mssql/mtest"
 )
 
-func TestData(t *testing.T) {
+func testData(t *testing.T, r compose.R) {
 	t.Skip("Skipping `data.json` generation test")
 
-	f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("db"))
+	f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig(r.Host(), "db"))
 
 	err := mbtest.WriteEventsReporterV2(f, t, "")
 	if err != nil {
diff --git a/x-pack/metricbeat/module/mssql/db/db_integration_test.go b/x-pack/metricbeat/module/mssql/db/db_integration_test.go
index 4fc9f594b7d..da531e5ac59 100644
--- a/x-pack/metricbeat/module/mssql/db/db_integration_test.go
+++ b/x-pack/metricbeat/module/mssql/db/db_integration_test.go
@@ -14,14 +14,20 @@ import (
 	"github.com/elastic/beats/libbeat/logp"
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	mtest "github.com/elastic/beats/x-pack/metricbeat/module/mssql/testing"
+	"github.com/elastic/beats/x-pack/metricbeat/module/mssql/mtest"
 )
 
-func TestFetch(t *testing.T) {
+func TestDb(t *testing.T) {
 	logp.TestingSetup()
-	compose.EnsureUp(t, "mssql")
 
-	f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("db"))
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": testFetch,
+		"Data":  testData,
+	})
+}
+
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig(r.Host(), "db"))
 	events, errs := mbtest.ReportingFetchV2(f)
 	if len(errs) > 0 {
 		t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs)
diff --git a/x-pack/metricbeat/module/mssql/testing/mssql.go b/x-pack/metricbeat/module/mssql/mtest/mssql.go
similarity index 79%
rename from x-pack/metricbeat/module/mssql/testing/mssql.go
rename to x-pack/metricbeat/module/mssql/mtest/mssql.go
index 971b06badb5..7d405af6ed0 100644
--- a/x-pack/metricbeat/module/mssql/testing/mssql.go
+++ b/x-pack/metricbeat/module/mssql/mtest/mssql.go
@@ -2,19 +2,19 @@
 // or more contributor license agreements. Licensed under the Elastic License;
 // you may not use this file except in compliance with the Elastic License.
 
-package testing
+package mtest
 
 import "os"
 
 // GetConfig returns the required configuration options for testing a MSSQL
 // metricset.
-func GetConfig(metricSets ...string) map[string]interface{} {
+func GetConfig(host string, metricSets ...string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "mssql",
 		"metricsets": metricSets,
-		"hosts":      []string{EnvOr("MSSQL_HOST", "localhost")},
+		"hosts":      []string{host},
 		"username":   EnvOr("MSSQL_USER", "SA"),
-		"password":   EnvOr("MSSQL_PASSWORD", ""),
+		"password":   EnvOr("MSSQL_PASSWORD", "1234_asdf"),
 	}
 }
 
diff --git a/x-pack/metricbeat/module/mssql/mtest/runner.go b/x-pack/metricbeat/module/mssql/mtest/runner.go
new file mode 100644
index 00000000000..1a8c2b8033f
--- /dev/null
+++ b/x-pack/metricbeat/module/mssql/mtest/runner.go
@@ -0,0 +1,18 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+// +build integration
+
+package mtest
+
+import (
+	"github.com/elastic/beats/libbeat/tests/compose"
+)
+
+var (
+	// Runner is a compose test runner for Redis tests
+	Runner = compose.TestRunner{
+		Service: "mssql",
+	}
+)
diff --git a/x-pack/metricbeat/module/mssql/performance/data_integration_test.go b/x-pack/metricbeat/module/mssql/performance/data_integration_test.go
index 9c688593783..f93e051cadf 100644
--- a/x-pack/metricbeat/module/mssql/performance/data_integration_test.go
+++ b/x-pack/metricbeat/module/mssql/performance/data_integration_test.go
@@ -12,13 +12,14 @@ import (
 	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
 
+	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	mtest "github.com/elastic/beats/x-pack/metricbeat/module/mssql/testing"
+	"github.com/elastic/beats/x-pack/metricbeat/module/mssql/mtest"
 )
 
-func TestData(t *testing.T) {
+func testData(t *testing.T, r compose.R) {
 	t.Skip("Skipping `data.json` generation test")
-	_, config, err := getHostURI()
+	_, config, err := getHostURI(r.Host())
 	if err != nil {
 		t.Fatal("error getting config information", err.Error())
 	}
@@ -35,13 +36,8 @@ func TestData(t *testing.T) {
 	}
 }
 
-func getHostURI() (string, map[string]interface{}, error) {
-	config := mtest.GetConfig("performance")
-
-	host, ok := config["hosts"].([]string)
-	if !ok {
-		return "", nil, errors.New("error getting host name information")
-	}
+func getHostURI(host string) (string, map[string]interface{}, error) {
+	config := mtest.GetConfig(host, "performance")
 
 	username, ok := config["username"].(string)
 	if !ok {
@@ -56,7 +52,7 @@ func getHostURI() (string, map[string]interface{}, error) {
 	u := &url.URL{
 		Scheme: "sqlserver",
 		User:   url.UserPassword(username, password),
-		Host:   host[0],
+		Host:   host,
 	}
 
 	return u.String(), config, nil
diff --git a/x-pack/metricbeat/module/mssql/performance/performance_integration_test.go b/x-pack/metricbeat/module/mssql/performance/performance_integration_test.go
index 6aae33523ee..a6cb11a9ec3 100644
--- a/x-pack/metricbeat/module/mssql/performance/performance_integration_test.go
+++ b/x-pack/metricbeat/module/mssql/performance/performance_integration_test.go
@@ -15,19 +15,24 @@ import (
 
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	mtest "github.com/elastic/beats/x-pack/metricbeat/module/mssql/testing"
+	"github.com/elastic/beats/x-pack/metricbeat/module/mssql/mtest"
 )
 
+func TestPerformance(t *testing.T) {
+	logp.TestingSetup()
+
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": testFetch,
+	})
+}
+
 type keyAssertion struct {
 	key       string
 	assertion func(v interface{}, key string)
 }
 
-func TestFetch(t *testing.T) {
-	logp.TestingSetup()
-	compose.EnsureUp(t, "mssql")
-
-	f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("performance"))
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig(r.Host(), "performance"))
 	events, errs := mbtest.ReportingFetchV2(f)
 	if len(errs) > 0 {
 		t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs)
diff --git a/x-pack/metricbeat/tests/system/test_mssql.py b/x-pack/metricbeat/tests/system/test_mssql.py
index 252c8a1cb06..885f2e5e384 100644
--- a/x-pack/metricbeat/tests/system/test_mssql.py
+++ b/x-pack/metricbeat/tests/system/test_mssql.py
@@ -75,7 +75,7 @@ def test_performance(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return [os.getenv('MSSQL_HOST', 'mssql')]
+        return [self.compose_host()]
 
     def get_username(self):
         return os.getenv('MSSQL_USERNAME', 'SA')

From fae8cda20efac3eac4476d5ecb36ad09bf1a827f Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Sun, 20 Jan 2019 02:15:55 +0100
Subject: [PATCH 85/88] Update versions of the stack

---
 metricbeat/module/elasticsearch/docker-compose.yml | 2 +-
 metricbeat/module/kibana/_meta/Dockerfile          | 2 +-
 metricbeat/module/logstash/_meta/Dockerfile        | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/metricbeat/module/elasticsearch/docker-compose.yml b/metricbeat/module/elasticsearch/docker-compose.yml
index e046f6eefed..518bb6bf853 100644
--- a/metricbeat/module/elasticsearch/docker-compose.yml
+++ b/metricbeat/module/elasticsearch/docker-compose.yml
@@ -2,7 +2,7 @@ version: "2.1"
 
 services:
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION:-6.4.3}
+    image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION:-6.5.4}
     environment:
       - "ES_JAVA_OPTS=-Xms90m -Xmx90m"
       - "network.host="
diff --git a/metricbeat/module/kibana/_meta/Dockerfile b/metricbeat/module/kibana/_meta/Dockerfile
index ce77bb67674..a6cef013f57 100644
--- a/metricbeat/module/kibana/_meta/Dockerfile
+++ b/metricbeat/module/kibana/_meta/Dockerfile
@@ -1,2 +1,2 @@
-FROM docker.elastic.co/kibana/kibana:6.5.1
+FROM docker.elastic.co/kibana/kibana:6.5.4
 HEALTHCHECK --interval=1s --retries=300 CMD curl -f http://localhost:5601/api/status | grep '"disconnects"'
diff --git a/metricbeat/module/logstash/_meta/Dockerfile b/metricbeat/module/logstash/_meta/Dockerfile
index 4c47a068ed1..7ebabe0dc44 100644
--- a/metricbeat/module/logstash/_meta/Dockerfile
+++ b/metricbeat/module/logstash/_meta/Dockerfile
@@ -1,4 +1,4 @@
-FROM docker.elastic.co/logstash/logstash:6.3.0
+FROM docker.elastic.co/logstash/logstash:6.5.4
 
 COPY healthcheck.sh /
 ENV XPACK_MONITORING_ENABLED=FALSE

From ffe2062475145f8a381cc9daaa6ef1cde7daf160 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Sun, 20 Jan 2019 12:46:41 +0100
Subject: [PATCH 86/88] Fix redis key test

---
 metricbeat/tests/system/test_redis.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/metricbeat/tests/system/test_redis.py b/metricbeat/tests/system/test_redis.py
index c4793906a67..a3ca518e167 100644
--- a/metricbeat/tests/system/test_redis.py
+++ b/metricbeat/tests/system/test_redis.py
@@ -95,9 +95,10 @@ def test_key(self):
         """
 
         # At least one event must be inserted so db stats exist
+        host, port = self.compose_host().split(":")
         r = redis.StrictRedis(
-            host=self.compose_hosts()[0],
-            port=os.getenv('REDIS_PORT', '6379'),
+            host=host,
+            port=port,
             db=0)
         r.flushall()
         r.rpush('list-key', 'one', 'two', 'three')

From 64dfd65b9f987edc58e29fff9c2a125251486879 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Wed, 6 Mar 2019 18:34:56 +0100
Subject: [PATCH 87/88] Move couchbase DSN helper to mtest

---
 .../module/couchbase/bucket/bucket_integration_test.go       | 4 ++--
 .../module/couchbase/cluster/cluster_integration_test.go     | 4 ++--
 metricbeat/module/couchbase/{testing.go => mtest/dsn.go}     | 5 +++--
 metricbeat/module/couchbase/node/node_integration_test.go    | 4 ++--
 4 files changed, 9 insertions(+), 8 deletions(-)
 rename metricbeat/module/couchbase/{testing.go => mtest/dsn.go} (90%)

diff --git a/metricbeat/module/couchbase/bucket/bucket_integration_test.go b/metricbeat/module/couchbase/bucket/bucket_integration_test.go
index 302d1327daf..4cbe93210e9 100644
--- a/metricbeat/module/couchbase/bucket/bucket_integration_test.go
+++ b/metricbeat/module/couchbase/bucket/bucket_integration_test.go
@@ -24,7 +24,7 @@ import (
 
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	"github.com/elastic/beats/metricbeat/module/couchbase"
+	"github.com/elastic/beats/metricbeat/module/couchbase/mtest"
 )
 
 func TestBucket(t *testing.T) {
@@ -46,6 +46,6 @@ func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "couchbase",
 		"metricsets": []string{"bucket"},
-		"hosts":      []string{couchbase.GetEnvDSN(host)},
+		"hosts":      []string{mtest.GetDSN(host)},
 	}
 }
diff --git a/metricbeat/module/couchbase/cluster/cluster_integration_test.go b/metricbeat/module/couchbase/cluster/cluster_integration_test.go
index c67e3ff5145..949dc3b4f05 100644
--- a/metricbeat/module/couchbase/cluster/cluster_integration_test.go
+++ b/metricbeat/module/couchbase/cluster/cluster_integration_test.go
@@ -26,7 +26,7 @@ import (
 
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	"github.com/elastic/beats/metricbeat/module/couchbase"
+	"github.com/elastic/beats/metricbeat/module/couchbase/mtest"
 )
 
 func TestNode(t *testing.T) {
@@ -59,6 +59,6 @@ func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "couchbase",
 		"metricsets": []string{"cluster"},
-		"hosts":      []string{couchbase.GetEnvDSN(host)},
+		"hosts":      []string{mtest.GetDSN(host)},
 	}
 }
diff --git a/metricbeat/module/couchbase/testing.go b/metricbeat/module/couchbase/mtest/dsn.go
similarity index 90%
rename from metricbeat/module/couchbase/testing.go
rename to metricbeat/module/couchbase/mtest/dsn.go
index bb52baee925..993d17c8bd8 100644
--- a/metricbeat/module/couchbase/testing.go
+++ b/metricbeat/module/couchbase/mtest/dsn.go
@@ -15,8 +15,9 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package couchbase
+package mtest
 
-func GetEnvDSN(host string) string {
+// GetDSN gets the test DSN for a given host
+func GetDSN(host string) string {
 	return "http://Administrator:password@" + host
 }
diff --git a/metricbeat/module/couchbase/node/node_integration_test.go b/metricbeat/module/couchbase/node/node_integration_test.go
index 82b587f5320..8072131a24c 100644
--- a/metricbeat/module/couchbase/node/node_integration_test.go
+++ b/metricbeat/module/couchbase/node/node_integration_test.go
@@ -26,7 +26,7 @@ import (
 
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	"github.com/elastic/beats/metricbeat/module/couchbase"
+	"github.com/elastic/beats/metricbeat/module/couchbase/mtest"
 )
 
 func TestNode(t *testing.T) {
@@ -59,6 +59,6 @@ func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "couchbase",
 		"metricsets": []string{"node"},
-		"hosts":      []string{couchbase.GetEnvDSN(host)},
+		"hosts":      []string{mtest.GetDSN(host)},
 	}
 }

From 2c9cd9757f8f7a001cbf8b43b437cbb436539edc Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Wed, 6 Mar 2019 18:56:32 +0100
Subject: [PATCH 88/88] Fix mysql tests

---
 metricbeat/module/mysql/mtest/testing.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/metricbeat/module/mysql/mtest/testing.go b/metricbeat/module/mysql/mtest/testing.go
index 62477a029e4..1e6f585b1de 100644
--- a/metricbeat/module/mysql/mtest/testing.go
+++ b/metricbeat/module/mysql/mtest/testing.go
@@ -36,6 +36,10 @@ func GetDSN(host string) string {
 		Addr:   host,
 		User:   "root",
 		Passwd: "test",
+
+		// Required if password is set and FormatDSN() is used
+		// since clients for MySQL 8.0
+		AllowNativePasswords: true,
 	}
 	return c.FormatDSN()
 }