diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile index a3c905d1b18..c1d0b35036e 100755 --- a/libbeat/scripts/Makefile +++ b/libbeat/scripts/Makefile @@ -388,14 +388,13 @@ import-dashboards: update ${BEAT_NAME} # Builds the environment to test beat .PHONY: build-image build-image: write-environment - ${DOCKER_COMPOSE} build ${DOCKER_NOCACHE} --pull --force-rm # Runs the environment so the redis and elasticsearch can also be used for local development # To use it for running the test, set ES_HOST and REDIS_HOST environment variable to the ip of your docker-machine. .PHONY: start-environment start-environment: stop-environment ${DOCKER_COMPOSE} pull --include-deps - ${DOCKER_COMPOSE} up -d + ${DOCKER_COMPOSE} up --build -d .PHONY: stop-environment stop-environment: diff --git a/libbeat/tests/compose/compose.go b/libbeat/tests/compose/compose.go index 9cd6dd205a8..0afb9a55833 100644 --- a/libbeat/tests/compose/compose.go +++ b/libbeat/tests/compose/compose.go @@ -21,72 +21,39 @@ import ( "errors" "os" "path/filepath" - "strconv" - "testing" ) -// EnsureUp starts all the requested services (must be defined in docker-compose.yml) -// with a default timeout of 300 seconds -func EnsureUp(t *testing.T, services ...string) { - EnsureUpWithTimeout(t, 300, services...) -} - -// EnsureUpWithTimeout starts all the requested services (must be defined in docker-compose.yml) -// Wait for `timeout` seconds for health -func EnsureUpWithTimeout(t *testing.T, timeout int, services ...string) { - // The NO_COMPOSE env variables makes it possible to skip the starting of the environment. - // This is useful if the service is already running locally. - if noCompose, err := strconv.ParseBool(os.Getenv("NO_COMPOSE")); err == nil && noCompose { - return - } - - compose, err := getComposeProject() - if err != nil { - t.Fatal(err) - } - - // Kill no longer used containers - err = compose.KillOld(services) +func findComposePath() (string, error) { + // find docker-compose + path, err := os.Getwd() if err != nil { - t.Fatal(err) + return "", err } + for { + if path == "/" { + break + } - for _, service := range services { - err = compose.Start(service) - if err != nil { - t.Fatal(err) + composePath := filepath.Join(path, "docker-compose.yml") + if _, err = os.Stat(composePath); err == nil { + return composePath, nil } + path = filepath.Dir(path) } - // Wait for health - err = compose.Wait(timeout, services...) - if err != nil { - t.Fatal(err) - } + return "", errors.New("docker-compose.yml not found") } -func getComposeProject() (*Project, error) { - // find docker-compose - path, err := os.Getwd() +func getComposeProject(name string) (*Project, error) { + path, err := findComposePath() if err != nil { return nil, err } - for { - if path == "/" { - return nil, errors.New("docker-compose.yml not found") - } - - if _, err = os.Stat(filepath.Join(path, "docker-compose.yml")); err != nil { - path = filepath.Dir(path) - } else { - break - } - } return NewProject( - os.Getenv("DOCKER_COMPOSE_PROJECT_NAME"), + name, []string{ - filepath.Join(path, "docker-compose.yml"), + path, }, ) } diff --git a/libbeat/tests/compose/project.go b/libbeat/tests/compose/project.go index 6df5d358560..776f3583d7e 100644 --- a/libbeat/tests/compose/project.go +++ b/libbeat/tests/compose/project.go @@ -56,9 +56,11 @@ const ( // Driver is the interface of docker compose implementations type Driver interface { Up(ctx context.Context, opts UpOptions, service string) error + Down(ctx context.Context) error Kill(ctx context.Context, signal string, service string) error Ps(ctx context.Context, filter ...string) ([]ContainerStatus, error) - // Containers(ctx context.Context, projectFilter Filter, filter ...string) ([]string, error) + + SetParameters(map[string]string) LockFile() string } @@ -69,6 +71,7 @@ type ContainerStatus interface { Healthy() bool Running() bool Old() bool + Host() string } // Project is a docker-compose project @@ -94,18 +97,6 @@ func NewProject(name string, files []string) (*Project, error) { // Start the container, unless it's running already func (c *Project) Start(service string) error { - servicesStatus, err := c.getServices(service) - if err != nil { - return err - } - - if servicesStatus[service] != nil { - if servicesStatus[service].Running { - // Someone is running it - return nil - } - } - c.Lock() defer c.Unlock() @@ -149,6 +140,25 @@ func (c *Project) Wait(seconds int, services ...string) error { return nil } +// Host gets the host and port of a service +func (c *Project) Host(service string) (string, error) { + servicesStatus, err := c.getServices(service) + if err != nil { + return "", err + } + + if len(servicesStatus) == 0 { + return "", errors.New("no container running for service") + } + + status, ok := servicesStatus[service] + if !ok || status.Host == "" { + return "", errors.New("unknown host:port for service") + } + + return status.Host, nil +} + // Kill a container func (c *Project) Kill(service string) error { c.Lock() @@ -190,6 +200,14 @@ func (c *Project) KillOld(except []string) error { return nil } +// Down removes all resources of a project +func (c *Project) Down() error { + c.Lock() + defer c.Unlock() + + return c.Driver.Down(context.Background()) +} + // Lock acquires the lock (300s) timeout // Normally it should only be seconds that the lock is used, but in some cases it can take longer. func (c *Project) Lock() { @@ -226,6 +244,8 @@ type serviceInfo struct { Running bool Healthy bool + Host string + // Has been up for too long?: Old bool } @@ -259,6 +279,7 @@ func (c *Project) getServices(filter ...string) (map[string]*serviceInfo, error) if service.Healthy { service.Old = c.Old() } + service.Host = c.Host() result[name] = service } diff --git a/libbeat/tests/compose/runner.go b/libbeat/tests/compose/runner.go new file mode 100644 index 00000000000..12206ef5bdf --- /dev/null +++ b/libbeat/tests/compose/runner.go @@ -0,0 +1,244 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package compose + +import ( + "fmt" + "math/rand" + "net" + "os" + "sort" + "strings" + "testing" + "time" + + "github.com/pkg/errors" +) + +func init() { + rand.Seed(time.Now().UnixNano()) +} + +// TestRunner starts a service with different combinations of options and +// runs tests on each one of these combinations +type TestRunner struct { + // Name of the service managed by this runner + Service string + + // Map of options with the list of possible values + Options RunnerOptions + + // Set to true if this runner can run in parallel with other runners + Parallel bool + + // Timeout to start the managed service + Timeout int +} + +// Suite is a set of tests to be run with a TestRunner +// Each test must be one of: +// - func(R) +// - func(*testing.T, R) +// - func(*testing.T) +type Suite map[string]interface{} + +// RunnerOptions are the possible options of a runner scenario +type RunnerOptions map[string][]string + +func (r *TestRunner) scenarios() []map[string]string { + n := 1 + options := make(map[string][]string) + for env, values := range r.Options { + // Allow to override options from environment variables + value := os.Getenv(env) + if value != "" { + values = []string{value} + } + options[env] = values + n *= len(values) + } + + scenarios := make([]map[string]string, n) + for variable, values := range options { + v := 0 + for i, s := range scenarios { + if s == nil { + s = make(map[string]string) + scenarios[i] = s + } + s[variable] = values[v] + v = (v + 1) % len(values) + } + } + + return scenarios +} + +func (r *TestRunner) runSuite(t *testing.T, tests Suite, ctl R) { + for name, test := range tests { + var testFunc func(t *testing.T) + switch f := test.(type) { + case func(R): + testFunc = func(t *testing.T) { f(ctl.WithT(t)) } + case func(*testing.T, R): + testFunc = func(t *testing.T) { f(t, ctl.WithT(t)) } + case func(*testing.T): + testFunc = func(t *testing.T) { f(t) } + default: + t.Fatalf("incorrect test suite function '%s'", name) + } + t.Run(name, testFunc) + } +} + +func (r *TestRunner) runHostOverride(t *testing.T, tests Suite) bool { + env := strings.ToUpper(r.Service) + "_HOST" + host := os.Getenv(env) + if host == "" { + return false + } + + t.Logf("Test host overriden by %s=%s", env, host) + + ctl := &runnerControl{ + host: host, + } + r.runSuite(t, tests, ctl) + return true +} + +// Run runs a tests suite +func (r *TestRunner) Run(t *testing.T, tests Suite) { + t.Helper() + + if r.runHostOverride(t, tests) { + return + } + + timeout := r.Timeout + if timeout == 0 { + timeout = 300 + } + + scenarios := r.scenarios() + if len(scenarios) == 0 { + t.Fatal("Test runner configuration doesn't produce scenarios") + } + for _, s := range scenarios { + s := s + var vars []string + for k, v := range s { + vars = append(vars, fmt.Sprintf("%s=%s", k, v)) + } + sort.Strings(vars) + desc := strings.Join(vars, ",") + if desc == "" { + desc = "WithoutOptions" + } + + seq := make([]byte, 16) + rand.Read(seq) + name := fmt.Sprintf("%s_%x", r.Service, seq) + + project, err := getComposeProject(name) + if err != nil { + t.Fatal(err) + } + project.SetParameters(s) + + t.Run(desc, func(t *testing.T) { + if r.Parallel { + t.Parallel() + } + + err := project.Start(r.Service) + // Down() is "idempotent", Start() has several points where it can fail, + // so run Down() even if Start() fails. + defer project.Down() + if err != nil { + t.Fatal(err) + } + + err = project.Wait(timeout, r.Service) + if err != nil { + t.Fatal(errors.Wrapf(err, "waiting for %s/%s", r.Service, desc)) + } + + host, err := project.Host(r.Service) + if err != nil { + t.Fatal(errors.Wrapf(err, "getting host for %s/%s", r.Service, desc)) + } + + ctl := &runnerControl{ + host: host, + scenario: s, + } + r.runSuite(t, tests, ctl) + }) + + } +} + +// R extends the testing.T interface with methods that expose information about current scenario +type R interface { + testing.TB + + WithT(t *testing.T) R + + Host() string + Hostname() string + Port() string + + Option(string) string +} + +type runnerControl struct { + *testing.T + + host string + scenario map[string]string +} + +// WithT creates a copy of R with the given T +func (r *runnerControl) WithT(t *testing.T) R { + ctl := *r + ctl.T = t + return &ctl +} + +// Host returns the host:port the test should use to connect to the service +func (r *runnerControl) Host() string { + return r.host +} + +// Hostname is the address of the host +func (r *runnerControl) Hostname() string { + hostname, _, _ := net.SplitHostPort(r.host) + return hostname +} + +// Port is the port of the host +func (r *runnerControl) Port() string { + _, port, _ := net.SplitHostPort(r.host) + return port +} + +// Option returns the value of an option for the current scenario +func (r *runnerControl) Option(key string) string { + return r.scenario[key] +} diff --git a/libbeat/tests/compose/runner_test.go b/libbeat/tests/compose/runner_test.go new file mode 100644 index 00000000000..30b5d188460 --- /dev/null +++ b/libbeat/tests/compose/runner_test.go @@ -0,0 +1,80 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package compose + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestRunnerScenarios(t *testing.T) { + cases := []struct { + Title string + Options RunnerOptions + Expected []map[string]string + }{ + { + Title: "Nil options", + Options: nil, + Expected: []map[string]string{nil}, + }, + { + Title: "Empty options", + Options: RunnerOptions{}, + Expected: []map[string]string{nil}, + }, + { + Title: "One option, two values", + Options: RunnerOptions{ + "FOO": {"bar", "baz"}, + }, + Expected: []map[string]string{ + {"FOO": "bar"}, + {"FOO": "baz"}, + }, + }, + { + Title: "Multiple options", + Options: RunnerOptions{ + "FOO": {"bar", "baz"}, + "BAZ": {"stuff"}, + }, + Expected: []map[string]string{ + {"FOO": "bar", "BAZ": "stuff"}, + {"FOO": "baz", "BAZ": "stuff"}, + }, + }, + { + Title: "Multiple options, single values", + Options: RunnerOptions{ + "FOO": {"bar"}, + "BAZ": {"stuff"}, + }, + Expected: []map[string]string{ + {"FOO": "bar", "BAZ": "stuff"}, + }, + }, + } + + for _, c := range cases { + r := TestRunner{Options: c.Options} + found := r.scenarios() + assert.Equal(t, c.Expected, found, c.Title) + } +} diff --git a/libbeat/tests/compose/wrapper.go b/libbeat/tests/compose/wrapper.go index 4a10ef5135b..b363774587d 100644 --- a/libbeat/tests/compose/wrapper.go +++ b/libbeat/tests/compose/wrapper.go @@ -18,11 +18,17 @@ package compose import ( + "archive/tar" + "bytes" "context" "fmt" + "net" "os" "os/exec" + "path/filepath" + "strconv" "strings" + "time" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" @@ -38,6 +44,8 @@ const ( type wrapperDriver struct { Name string Files []string + + Environment []string } type wrapperContainer struct { @@ -60,13 +68,45 @@ func (c *wrapperContainer) Old() bool { return strings.Contains(c.info.Status, "minute") } +func (c *wrapperContainer) privateHost() string { + var ip string + for _, net := range c.info.NetworkSettings.Networks { + if len(net.IPAddress) > 0 { + ip = net.IPAddress + break + } + } + if len(ip) == 0 { + return "" + } + + for _, port := range c.info.Ports { + return net.JoinHostPort(ip, strconv.Itoa(int(port.PrivatePort))) + } + return "" +} + +func (c *wrapperContainer) publicHost() string { + for _, port := range c.info.Ports { + if port.PublicPort != 0 { + return net.JoinHostPort("localhost", strconv.Itoa(int(port.PublicPort))) + } + } + return "" +} + +func (c *wrapperContainer) Host() string { + // TODO: Support multiple networks/ports + return c.publicHost() +} + func (d *wrapperDriver) LockFile() string { return d.Files[0] + ".lock" } func (d *wrapperDriver) cmd(ctx context.Context, command string, arg ...string) *exec.Cmd { var args []string - args = append(args, "--project-name", d.Name) + args = append(args, "--no-ansi", "--project-name", d.Name) for _, f := range d.Files { args = append(args, "--file", f) } @@ -75,9 +115,20 @@ func (d *wrapperDriver) cmd(ctx context.Context, command string, arg ...string) cmd := exec.CommandContext(ctx, "docker-compose", args...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr + if len(d.Environment) > 0 { + cmd.Env = append(os.Environ(), d.Environment...) + } return cmd } +func (d *wrapperDriver) SetParameters(params map[string]string) { + var env []string + for k, v := range params { + env = append(env, fmt.Sprintf("%s=%s", k, v)) + } + d.Environment = env +} + func (d *wrapperDriver) Up(ctx context.Context, opts UpOptions, service string) error { var args []string @@ -95,7 +146,82 @@ func (d *wrapperDriver) Up(ctx context.Context, opts UpOptions, service string) args = append(args, service) } - return d.cmd(ctx, "up", args...).Run() + for { + // It can fail if we have reached some system limit, specially + // number of networks, retry while the context is not done + err := d.cmd(ctx, "up", args...).Run() + if err == nil { + return d.setupAdvertisedHost(ctx, service) + } + + select { + case <-time.After(time.Second): + case <-ctx.Done(): + return err + } + } +} + +func writeToContainer(ctx context.Context, id, filename, content string) error { + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + now := time.Now() + err := tw.WriteHeader(&tar.Header{ + Typeflag: tar.TypeReg, + Name: filepath.Base(filename), + Mode: 0100644, + Size: int64(len(content)), + ModTime: now, + AccessTime: now, + ChangeTime: now, + }) + if err != nil { + return errors.Wrap(err, "failed to write tar header") + } + if _, err := tw.Write([]byte(content)); err != nil { + return errors.Wrap(err, "failed to write tar file") + } + if err := tw.Close(); err != nil { + return errors.Wrap(err, "failed to close tar") + } + + cli, err := client.NewEnvClient() + if err != nil { + return errors.Wrap(err, "failed to start docker client") + } + defer cli.Close() + + opts := types.CopyToContainerOptions{} + err = cli.CopyToContainer(ctx, id, filepath.Dir(filename), bytes.NewReader(buf.Bytes()), opts) + if err != nil { + return errors.Wrapf(err, "failed to copy environment to container %s", id) + } + return nil +} + +func (d *wrapperDriver) setupAdvertisedHost(ctx context.Context, service string) error { + containers, err := d.containers(ctx, Filter{State: AnyState}, service) + if err != nil { + return errors.Wrap(err, "setupAdvertisedHost") + } + if len(containers) == 0 { + return errors.Errorf("no containers for service %s", service) + } + + for _, c := range containers { + w := &wrapperContainer{info: c} + content := fmt.Sprintf("SERVICE_HOST=%s", w.Host()) + + err := writeToContainer(ctx, c.ID, "/run/compose_env", content) + if err != nil { + return err + } + } + return nil +} + +func (d *wrapperDriver) Down(ctx context.Context) error { + return d.cmd(ctx, "down", "-v", "--rmi=local").Run() } func (d *wrapperDriver) Kill(ctx context.Context, signal string, service string) error { @@ -143,6 +269,7 @@ func (d *wrapperDriver) containers(ctx context.Context, projectFilter Filter, fi if err != nil { return nil, errors.Wrap(err, "failed to start docker client") } + defer cli.Close() var serviceFilters []filters.Args if len(filter) == 0 { diff --git a/libbeat/tests/system/beat/compose.py b/libbeat/tests/system/beat/compose.py index 54387f006df..cbbca55947c 100644 --- a/libbeat/tests/system/beat/compose.py +++ b/libbeat/tests/system/beat/compose.py @@ -89,21 +89,47 @@ def compose_down(cls): """ Stop all running containers """ + if os.environ.get('NO_COMPOSE'): + return + if INTEGRATION_TESTS and cls.COMPOSE_SERVICES: cls.compose_project().kill(service_names=cls.COMPOSE_SERVICES) @classmethod - def compose_hosts(cls): + def _private_host(cls, info, port): + networks = info['NetworkSettings']['Networks'].values() + port = port.split("/")[0] + for network in networks: + ip = network['IPAddress'] + if ip: + return "%s:%s" % (ip, port) + + @classmethod + def _public_host(cls, info, port): + hostPort = info['NetworkSettings']['Ports'][port][0]['HostPort'] + return "localhost:%s" % hostPort + + @classmethod + def compose_host(cls, service=None, port=None): if not INTEGRATION_TESTS or not cls.COMPOSE_SERVICES: return [] - hosts = [] - for container in cls.compose_project().containers(service_names=cls.COMPOSE_SERVICES): - network_settings = container.inspect()['NetworkSettings'] - for network in network_settings['Networks'].values(): - if network['IPAddress']: - hosts.append(network['IPAddress']) - return hosts + if service is None: + service = cls.COMPOSE_SERVICES[0] + + host_env = os.environ.get(service.upper() + "_HOST") + if host_env: + return host_env + + container = cls.compose_project().containers(service_names=[service])[0] + info = container.inspect() + portsConfig = info['HostConfig']['PortBindings'] + if len(portsConfig) == 0: + raise Exception("No exposed ports for service %s" % service) + if port is None: + port = portsConfig.keys()[0] + + return cls._public_host(info, port) @classmethod def compose_project(cls): diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml index c987f374560..9c216c49566 100644 --- a/metricbeat/docker-compose.yml +++ b/metricbeat/docker-compose.yml @@ -1,4 +1,4 @@ -version: '2.3' +version: "2.3" services: beat: @@ -10,64 +10,47 @@ services: - ${PWD}/..:/go/src/github.com/elastic/beats/ # This is required to on-demand launching the rest on containers for tests & also docker module tests: - /var/run/docker.sock:/var/run/docker.sock + network_mode: host command: make - env_file: - - ./module/aerospike/_meta/env - - ./module/apache/_meta/env - - ./module/ceph/_meta/env - - ./module/consul/_meta/env - - ./module/couchbase/_meta/env - - ./module/couchdb/_meta/env - - ./module/dropwizard/_meta/env - - ./module/elasticsearch/_meta/env - - ./module/envoyproxy/_meta/env - - ./module/etcd/_meta/env - - ./module/golang/_meta/env - - ./module/haproxy/_meta/env - - ./module/http/_meta/env - - ./module/jolokia/_meta/env - - ./module/kafka/_meta/env - - ./module/kibana/_meta/env - #- ./module/kubernetes/_meta/env - - ./module/logstash/_meta/env - - ./module/memcached/_meta/env - - ./module/mongodb/_meta/env - - ./module/munin/_meta/env - - ./module/mysql/_meta/env - - ./module/nats/_meta/env - - ./module/nginx/_meta/env - - ./module/php_fpm/_meta/env - - ./module/postgresql/_meta/env - - ./module/prometheus/_meta/env - - ./module/rabbitmq/_meta/env - - ./module/redis/_meta/env - - ./module/traefik/_meta/env - - ./module/uwsgi/_meta/env - - ./module/zookeeper/_meta/env # Modules aerospike: - build: ./module/aerospike/_meta + extends: + file: ./module/aerospike/docker-compose.yml + service: aerospike apache: - build: ./module/apache/_meta + extends: + file: ./module/apache/docker-compose.yml + service: apache apache_2_4_12: + extends: + file: ./module/apache/docker-compose.yml + service: apache build: - context: ./module/apache/_meta - dockerfile: Dockerfile.2.4.12 + args: + APACHE_VERSION: 2.4.12 ceph: build: ./module/ceph/_meta + ports: + - 5000 consul: build: ./module/consul/_meta + ports: + - 8500 couchbase: build: ./module/couchbase/_meta + ports: + - 8091 couchdb: build: ./module/couchdb/_meta + ports: + - 5984 # Dummy container for docker tests, so there is a container with # healthcheck generating events @@ -80,70 +63,110 @@ services: dropwizard: build: ./module/dropwizard/_meta + ports: + - 8080 elasticsearch: - build: ./module/elasticsearch/_meta - environment: - - "ES_JAVA_OPTS=-Xms90m -Xmx90m" - - "network.host=" - - "transport.host=127.0.0.1" - - "http.host=0.0.0.0" - - "xpack.security.enabled=false" + extends: + file: ./module/elasticsearch/docker-compose.yml + service: elasticsearch envoyproxy: build: ./module/envoyproxy/_meta + ports: + - 9901 etcd: build: ./module/etcd/_meta + ports: + - 2379 etcd_3_2: build: context: ./module/etcd/_meta args: ETCD_VERSION: v3.2.25 + ports: + - 2379 golang: build: ./module/golang/_meta + ports: + - 6060 haproxy: build: ./module/haproxy/_meta + ports: + - 14567 + - 14568 + - 14569 haproxy_1_6: build: context: ./module/haproxy/_meta dockerfile: Dockerfile.1.6 + ports: + - 14567 + - 14568 + - 14569 haproxy_1_7: build: context: ./module/haproxy/_meta dockerfile: Dockerfile.1.7 + ports: + - 14567 + - 14568 + - 14569 http: build: ./module/http/_meta + ports: + - 8080 jolokia: build: ./module/jolokia/_meta + ports: + - 8778 kafka: - build: - context: ./module/kafka/_meta - args: - KAFKA_VERSION: 2.1.0 + extends: + file: ./module/kafka/docker-compose.yml + service: kafka + environment: + # Needed for python tests + KAFKA_ADVERTISED_HOST_AUTO: "yes" kafka_1_1_0: + extends: + file: ./module/kafka/docker-compose.yml + service: kafka + environment: + # Needed for python tests + KAFKA_ADVERTISED_HOST_AUTO: "yes" + image: metricbeat-kafka:1.1.0 build: - context: ./module/kafka/_meta args: KAFKA_VERSION: 1.1.0 kafka_0_10_2: + extends: + file: ./module/kafka/docker-compose.yml + service: kafka + environment: + # Needed for python tests + KAFKA_ADVERTISED_HOST_AUTO: "yes" + image: metricbeat-kafka:0.10.2.3 build: - context: ./module/kafka/_meta args: - KAFKA_VERSION: 0.10.2.1 + KAFKA_VERSION: 0.10.2.3 kibana: build: ./module/kibana/_meta + depends_on: + - elasticsearch + ports: + - 5601 #kubernetes: # build: ./module/kubernetes/_meta @@ -165,96 +188,138 @@ services: logstash: build: ./module/logstash/_meta + ports: + - 9600 memcached: build: ./module/memcached/_meta + ports: + - 11211 mongodb: build: ./module/mongodb/_meta command: mongod --replSet beats + ports: + - 27017 munin: build: ./module/munin/_meta + ports: + - 4949 mysql: build: context: ./module/mysql/_meta args: MYSQL_IMAGE: mysql:5.7.24 + ports: + - 3306 mysql_8_0: build: context: ./module/mysql/_meta args: MYSQL_IMAGE: mysql:8.0.13 + ports: + - 3306 percona_5_7: build: context: ./module/mysql/_meta args: MYSQL_IMAGE: percona:5.7.24 + ports: + - 3306 percona_8_0: build: context: ./module/mysql/_meta args: MYSQL_IMAGE: percona:8.0.13-4 + ports: + - 3306 mariadb_10_2: build: context: ./module/mysql/_meta args: - MYSQL_IMAGE: mariadb:10.2.17 + MYSQL_IMAGE: mariadb:10.2.37 + ports: + - 3306 mariadb_10_3: build: context: ./module/mysql/_meta args: MYSQL_IMAGE: mariadb:10.3.12 + ports: + - 3306 nats: build: ./module/nats/_meta + ports: + - 8222 nginx: build: ./module/nginx/_meta + ports: + - 80 phpfpm: build: ./module/php_fpm/_meta + ports: + - 81 postgresql: build: ./module/postgresql/_meta + ports: + - 5432 prometheus: build: ./module/prometheus/_meta + ports: + - 9090 rabbitmq: build: ./module/rabbitmq/_meta + ports: + - 15672 redis: - build: ./module/redis/_meta + extends: + file: ./module/redis/docker-compose.yml + service: redis redis_4: - build: - context: ./module/redis/_meta - args: - REDIS_VERSION: 4.0.11 + extends: + file: ./module/redis/docker-compose.yml + service: redis + image: redis:4.0.11-alpine redis_5: - build: - context: ./module/redis/_meta - args: - REDIS_VERSION: 5.0-rc4 + extends: + file: ./module/redis/docker-compose.yml + service: redis + image: redis:5.0-rc4-alpine traefik: build: ./module/traefik/_meta + ports: + - 8080 uwsgi_tcp: build: ./module/uwsgi/_meta command: uwsgi --http :8080 --master --processes 1 --threads 2 --stats 0.0.0.0:9191 --memory-report --wsgi-file app.py + ports: + - 9191 uwsgi_http: build: ./module/uwsgi/_meta - command: uwsgi --http :8080 --master --processes 1 --threads 2 --stats 0.0.0.0:9192 --memory-report --stats-http --wsgi-file app.py + command: uwsgi --http :8080 --master --processes 1 --threads 2 --stats 0.0.0.0:9191 --memory-report --stats-http --wsgi-file app.py + ports: + - 9191 zookeeper: build: ./module/zookeeper/_meta + ports: + - 2181 diff --git a/metricbeat/module/aerospike/_meta/Dockerfile b/metricbeat/module/aerospike/_meta/Dockerfile index 9693bbde4c2..e5d1c37bbd2 100644 --- a/metricbeat/module/aerospike/_meta/Dockerfile +++ b/metricbeat/module/aerospike/_meta/Dockerfile @@ -1,4 +1,5 @@ -FROM aerospike:3.9.0 +ARG AEROSPIKE_VERSION=3.9.0 + +FROM aerospike:$AEROSPIKE_VERSION RUN apt-get update && apt-get install -y netcat -HEALTHCHECK --interval=1s --retries=90 CMD nc -z localhost 3000 diff --git a/metricbeat/module/aerospike/_meta/env b/metricbeat/module/aerospike/_meta/env deleted file mode 100644 index 2c95ea957cb..00000000000 --- a/metricbeat/module/aerospike/_meta/env +++ /dev/null @@ -1,2 +0,0 @@ -AEROSPIKE_HOST=aerospike -AEROSPIKE_PORT=3000 diff --git a/metricbeat/module/aerospike/docker-compose.yml b/metricbeat/module/aerospike/docker-compose.yml new file mode 100644 index 00000000000..e23613e5441 --- /dev/null +++ b/metricbeat/module/aerospike/docker-compose.yml @@ -0,0 +1,16 @@ +version: '2.3' + +services: + aerospike: + build: + context: ./_meta + args: + AEROSPIKE_VERSION: ${AEROSPIKE_VERSION:-3.9.0} + ports: + - "3000" + healthcheck: + test: + - "CMD-SHELL" + - "nc -z localhost 3000" + interval: 1s + retries: 90 diff --git a/metricbeat/module/apache/testing.go b/metricbeat/module/aerospike/mtest/runner.go similarity index 67% rename from metricbeat/module/apache/testing.go rename to metricbeat/module/aerospike/mtest/runner.go index 4b2e0571a35..2463ad13c88 100644 --- a/metricbeat/module/apache/testing.go +++ b/metricbeat/module/aerospike/mtest/runner.go @@ -15,22 +15,26 @@ // specific language governing permissions and limitations // under the License. -package apache +// +build integration + +package mtest import ( - "os" + "github.com/elastic/beats/libbeat/tests/compose" ) -// Helper functions for testing the Apache module's MetricSets. - -// GetApacheEnvHost returns the apache server hostname to use for testing. It -// reads the value from the APACHE_HOST environment variable and returns -// 127.0.0.1 if it is not set. -func GetApacheEnvHost() string { - host := os.Getenv("APACHE_HOST") - - if len(host) == 0 { - host = "127.0.0.1" +var ( + // Runner is the compose test runner for aerospike + Runner = compose.TestRunner{ + Service: "aerospike", + Options: compose.RunnerOptions{ + "AEROSPIKE_VERSION": { + "3.9.0", + "3.13.0.11", + "3.16.0.6", + "4.3.0.2", + }, + }, + Parallel: true, } - return host -} +) diff --git a/metricbeat/module/aerospike/namespace/namespace_integration_test.go b/metricbeat/module/aerospike/namespace/namespace_integration_test.go index 03fd7ed4397..843af5450da 100644 --- a/metricbeat/module/aerospike/namespace/namespace_integration_test.go +++ b/metricbeat/module/aerospike/namespace/namespace_integration_test.go @@ -24,23 +24,27 @@ import ( "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" - "github.com/elastic/beats/metricbeat/module/aerospike" + "github.com/elastic/beats/metricbeat/module/aerospike/mtest" ) -func TestData(t *testing.T) { - compose.EnsureUp(t, "aerospike") +func TestNamespace(t *testing.T) { + mtest.Runner.Run(t, compose.Suite{ + "Data": testData, + }) +} - f := mbtest.NewEventsFetcher(t, getConfig()) +func testData(t *testing.T, r compose.R) { + f := mbtest.NewEventsFetcher(t, getConfig(r.Host())) err := mbtest.WriteEvents(f, t) if err != nil { t.Fatal("write", err) } } -func getConfig() map[string]interface{} { +func getConfig(host string) map[string]interface{} { return map[string]interface{}{ "module": "aerospike", "metricsets": []string{"namespace"}, - "hosts": []string{aerospike.GetAerospikeEnvHost() + ":" + aerospike.GetAerospikeEnvPort()}, + "hosts": []string{host}, } } diff --git a/metricbeat/module/aerospike/testing.go b/metricbeat/module/aerospike/testing.go deleted file mode 100644 index 298634e85c0..00000000000 --- a/metricbeat/module/aerospike/testing.go +++ /dev/null @@ -1,48 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package aerospike - -import ( - "os" -) - -// Helper functions for testing used in the aerospike MetricSets. - -// GetAerospikeEnvHost returns the hostname of the Aerospike server to use for -// testing. It reads the value from the AEROSPIKE_HOST environment variable and -// returns localhost if it is not set. -func GetAerospikeEnvHost() string { - host := os.Getenv("AEROSPIKE_HOST") - - if len(host) == 0 { - host = "localhost" - } - return host -} - -// GetAerospikeEnvPort returns the port of the Aerospike server to use for -// testing. It reads the value from the AEROSPIKE_PORT environment variable and -// returns 3000 if it is not set. -func GetAerospikeEnvPort() string { - port := os.Getenv("AEROSPIKE_PORT") - - if len(port) == 0 { - port = "3000" - } - return port -} diff --git a/metricbeat/module/apache/_meta/Dockerfile b/metricbeat/module/apache/_meta/Dockerfile index b6239b871f9..9b5b6ddcc36 100644 --- a/metricbeat/module/apache/_meta/Dockerfile +++ b/metricbeat/module/apache/_meta/Dockerfile @@ -1,4 +1,5 @@ -FROM httpd:2.4.20 +ARG APACHE_VERSION +FROM httpd:$APACHE_VERSION RUN apt-get update && apt-get install -y curl HEALTHCHECK --interval=1s --retries=90 CMD curl -f http://localhost COPY ./httpd.conf /usr/local/apache2/conf/httpd.conf diff --git a/metricbeat/module/apache/_meta/Dockerfile.2.4.12 b/metricbeat/module/apache/_meta/Dockerfile.2.4.12 deleted file mode 100644 index f35ea2d95c0..00000000000 --- a/metricbeat/module/apache/_meta/Dockerfile.2.4.12 +++ /dev/null @@ -1,4 +0,0 @@ -FROM httpd:2.4.12 -RUN apt-get update && apt-get install -y curl -HEALTHCHECK --interval=1s --retries=90 CMD curl -f http://localhost -COPY ./httpd.conf /usr/local/apache2/conf/httpd.conf diff --git a/metricbeat/module/apache/_meta/env b/metricbeat/module/apache/_meta/env deleted file mode 100644 index 5339c3407ed..00000000000 --- a/metricbeat/module/apache/_meta/env +++ /dev/null @@ -1,3 +0,0 @@ -APACHE_HOST=apache -APACHE_OLD_HOST=apache_2_4_12 -APACHE_PORT=80 diff --git a/metricbeat/module/apache/docker-compose.yml b/metricbeat/module/apache/docker-compose.yml new file mode 100644 index 00000000000..20cfa1a6396 --- /dev/null +++ b/metricbeat/module/apache/docker-compose.yml @@ -0,0 +1,10 @@ +version: '2.3' + +services: + apache: + build: + context: ./_meta + args: + APACHE_VERSION: ${APACHE_VERSION:-2.4.20} + ports: + - 80 diff --git a/metricbeat/module/apache/mtest/runner.go b/metricbeat/module/apache/mtest/runner.go new file mode 100644 index 00000000000..2b86eedb79c --- /dev/null +++ b/metricbeat/module/apache/mtest/runner.go @@ -0,0 +1,38 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration + +package mtest + +import ( + "github.com/elastic/beats/libbeat/tests/compose" +) + +var ( + // Runner is the compose test runner for apache + Runner = compose.TestRunner{ + Service: "apache", + Options: map[string][]string{ + "APACHE_VERSION": []string{ + "2.4.12", + "2.4.20", + }, + }, + Parallel: true, + } +) diff --git a/metricbeat/module/apache/status/_meta/data.json b/metricbeat/module/apache/status/_meta/data.json index e8e3d45845f..171b0c54f95 100644 --- a/metricbeat/module/apache/status/_meta/data.json +++ b/metricbeat/module/apache/status/_meta/data.json @@ -2,30 +2,29 @@ "@timestamp": "2017-10-12T08:05:34.853Z", "apache": { "status": { - "bytes_per_request": 78.7692, - "bytes_per_sec": 4.94686, + "bytes_per_request": 0, + "bytes_per_sec": 0, "connections": { "async": { "closing": 0, - "keep_alive": 1, + "keep_alive": 0, "writing": 0 }, - "total": 1 + "total": 0 }, "cpu": { "children_system": 0, "children_user": 0, - "load": 3.99758, - "system": 16.39, - "user": 0.16 + "system": 0, + "user": 0 }, - "hostname": "apache", + "hostname": "172.26.0.2:80", "load": { - "1": 170.11, - "15": 77.62, - "5": 167.51 + "1": 1.3, + "15": 1.56, + "5": 1.34 }, - "requests_per_sec": 0.0628019, + "requests_per_sec": 0.666667, "scoreboard": { "closing_connection": 0, "dns_lookup": 0, @@ -33,22 +32,22 @@ "idle_cleanup": 0, "keepalive": 0, "logging": 0, - "open_slot": 300, + "open_slot": 325, "reading_request": 0, "sending_reply": 1, "starting_up": 0, "total": 400, - "waiting_for_connection": 99 + "waiting_for_connection": 74 }, - "total_accesses": 26, - "total_kbytes": 2, + "total_accesses": 2, + "total_kbytes": 0, "uptime": { - "server_uptime": 414, - "uptime": 414 + "server_uptime": 3, + "uptime": 3 }, "workers": { "busy": 1, - "idle": 99 + "idle": 74 } } }, @@ -57,7 +56,7 @@ "name": "host.example.com" }, "metricset": { - "host": "apache", + "host": "172.26.0.2:80", "module": "apache", "name": "status", "rtt": 115 diff --git a/metricbeat/module/apache/status/status_integration_test.go b/metricbeat/module/apache/status/status_integration_test.go index 01e7f70b53a..0fbdd22a410 100644 --- a/metricbeat/module/apache/status/status_integration_test.go +++ b/metricbeat/module/apache/status/status_integration_test.go @@ -24,15 +24,29 @@ import ( "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" - "github.com/elastic/beats/metricbeat/module/apache" + "github.com/elastic/beats/metricbeat/module/apache/mtest" "github.com/stretchr/testify/assert" ) -func TestFetch(t *testing.T) { - compose.EnsureUp(t, "apache") +func TestStatus(t *testing.T) { + mtest.Runner.Run(t, compose.Suite{ + "Data": testData, + "Fetch": testFetch, + }) +} + +func testData(t *testing.T, r compose.R) { + f := mbtest.NewEventFetcher(t, getConfig(r.Host())) + + err := mbtest.WriteEvent(f, t) + if err != nil { + t.Fatal("write", err) + } +} - f := mbtest.NewEventFetcher(t, getConfig()) +func testFetch(t *testing.T, r compose.R) { + f := mbtest.NewEventFetcher(t, getConfig(r.Host())) event, err := f.Fetch() if !assert.NoError(t, err) { t.FailNow() @@ -46,21 +60,10 @@ func TestFetch(t *testing.T) { } } -func TestData(t *testing.T) { - compose.EnsureUp(t, "apache") - - f := mbtest.NewEventFetcher(t, getConfig()) - - err := mbtest.WriteEvent(f, t) - if err != nil { - t.Fatal("write", err) - } -} - -func getConfig() map[string]interface{} { +func getConfig(host string) map[string]interface{} { return map[string]interface{}{ "module": "apache", "metricsets": []string{"status"}, - "hosts": []string{apache.GetApacheEnvHost()}, + "hosts": []string{host}, } } diff --git a/metricbeat/module/ceph/_meta/env b/metricbeat/module/ceph/_meta/env deleted file mode 100644 index 27b7ce7dbfb..00000000000 --- a/metricbeat/module/ceph/_meta/env +++ /dev/null @@ -1,2 +0,0 @@ -CEPH_HOST=ceph -CEPH_PORT=5000 diff --git a/metricbeat/module/consul/_meta/env b/metricbeat/module/consul/_meta/env deleted file mode 100644 index 211d9aa8769..00000000000 --- a/metricbeat/module/consul/_meta/env +++ /dev/null @@ -1,2 +0,0 @@ -CONSUL_HOST=consul -CONSUL_PORT=8500 diff --git a/metricbeat/module/consul/agent/agent_integration_test.go b/metricbeat/module/consul/agent/agent_integration_test.go index 970db3b00a8..7de8805147d 100644 --- a/metricbeat/module/consul/agent/agent_integration_test.go +++ b/metricbeat/module/consul/agent/agent_integration_test.go @@ -22,24 +22,29 @@ package agent import ( "testing" - "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/libbeat/tests/compose" - "github.com/elastic/beats/metricbeat/module/consul" - - "github.com/elastic/beats/libbeat/logp" - "github.com/stretchr/testify/assert" + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/elastic/beats/metricbeat/module/consul" ) -func TestFetch(t *testing.T) { - t.Skip("Skip flaky test on Consul Agent") +func TestConnection(t *testing.T) { logp.TestingSetup() - compose.EnsureUp(t, "consul") + runner := compose.TestRunner{Service: "consul"} + runner.Run(t, compose.Suite{ + "Fetch": testFetch, + "Data": testData, + }) +} + +func testFetch(t *testing.T, r compose.R) { + t.Skip("Skip flaky test on Consul Agent") - f := mbtest.NewReportingMetricSetV2(t, consul.GetConfig([]string{"agent"})) + f := mbtest.NewReportingMetricSetV2(t, consul.GetConfig(r.Host(), []string{"agent"})) events, errs := mbtest.ReportingFetchV2(f) if len(errs) > 0 { t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) diff --git a/metricbeat/module/consul/agent/data_integration_test.go b/metricbeat/module/consul/agent/data_integration_test.go index bffe36098be..1e319c41e21 100644 --- a/metricbeat/module/consul/agent/data_integration_test.go +++ b/metricbeat/module/consul/agent/data_integration_test.go @@ -22,24 +22,13 @@ package agent import ( "testing" - "github.com/elastic/beats/metricbeat/module/consul" - - _ "github.com/denisenkom/go-mssqldb" - "github.com/stretchr/testify/assert" - + "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/elastic/beats/metricbeat/module/consul" ) -func TestData(t *testing.T) { - t.Skip("Skipping `data.json` generation test") - - f := mbtest.NewReportingMetricSetV2(t, consul.GetConfig([]string{"agent"})) - events, errs := mbtest.ReportingFetchV2(f) - if len(errs) > 0 { - t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) - } - assert.NotEmpty(t, events) - +func testData(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, consul.GetConfig(r.Host(), []string{"agent"})) if err := mbtest.WriteEventsReporterV2(f, t, ""); err != nil { t.Fatal("write", err) } diff --git a/metricbeat/module/consul/testing.go b/metricbeat/module/consul/testing.go index 135d0292129..eb3ed89eaec 100644 --- a/metricbeat/module/consul/testing.go +++ b/metricbeat/module/consul/testing.go @@ -17,26 +17,11 @@ package consul -import ( - "fmt" - "os" -) - //GetConfig returns a config object specific for a Consul module and a provided Metricset in 'ms' -func GetConfig(ms []string) map[string]interface{} { +func GetConfig(host string, ms []string) map[string]interface{} { return map[string]interface{}{ "module": "consul", "metricsets": ms, - "hosts": []string{fmt.Sprintf("%s:%s", EnvOr("CONSUL_HOST", "localhost"), EnvOr("CONSUL_PORT", "8500"))}, - } -} - -// EnvOr returns the value of the specified environment variable if it is -// non-empty. Otherwise it return def. -func EnvOr(name, def string) string { - s := os.Getenv(name) - if s == "" { - return def + "hosts": []string{host}, } - return s } diff --git a/metricbeat/module/couchbase/_meta/env b/metricbeat/module/couchbase/_meta/env deleted file mode 100644 index 38aa8e34778..00000000000 --- a/metricbeat/module/couchbase/_meta/env +++ /dev/null @@ -1,3 +0,0 @@ -COUCHBASE_HOST=couchbase -COUCHBASE_PORT=8091 -COUCHBASE_DSN=http://Administrator:password@couchbase:8091 diff --git a/metricbeat/module/couchbase/bucket/bucket_integration_test.go b/metricbeat/module/couchbase/bucket/bucket_integration_test.go index 147f0e764df..4cbe93210e9 100644 --- a/metricbeat/module/couchbase/bucket/bucket_integration_test.go +++ b/metricbeat/module/couchbase/bucket/bucket_integration_test.go @@ -22,23 +22,30 @@ package bucket import ( "testing" + "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" - "github.com/elastic/beats/metricbeat/module/couchbase" + "github.com/elastic/beats/metricbeat/module/couchbase/mtest" ) -func TestData(t *testing.T) { - f := mbtest.NewEventsFetcher(t, getConfig()) +func TestBucket(t *testing.T) { + runner := compose.TestRunner{Service: "couchbase"} + runner.Run(t, compose.Suite{ + "Data": testData, + }) +} +func testData(t *testing.T, r compose.R) { + f := mbtest.NewEventsFetcher(t, getConfig(r.Host())) err := mbtest.WriteEvents(f, t) if err != nil { t.Fatal("write", err) } } -func getConfig() map[string]interface{} { +func getConfig(host string) map[string]interface{} { return map[string]interface{}{ "module": "couchbase", "metricsets": []string{"bucket"}, - "hosts": []string{couchbase.GetEnvDSN()}, + "hosts": []string{mtest.GetDSN(host)}, } } diff --git a/metricbeat/module/couchbase/cluster/cluster_integration_test.go b/metricbeat/module/couchbase/cluster/cluster_integration_test.go index 72a94a530d3..949dc3b4f05 100644 --- a/metricbeat/module/couchbase/cluster/cluster_integration_test.go +++ b/metricbeat/module/couchbase/cluster/cluster_integration_test.go @@ -26,13 +26,19 @@ import ( "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" - "github.com/elastic/beats/metricbeat/module/couchbase" + "github.com/elastic/beats/metricbeat/module/couchbase/mtest" ) -func TestFetch(t *testing.T) { - compose.EnsureUp(t, "couchbase") +func TestNode(t *testing.T) { + runner := compose.TestRunner{Service: "couchbase"} + runner.Run(t, compose.Suite{ + "Fetch": testFetch, + "Data": testData, + }) +} - f := mbtest.NewReportingMetricSetV2(t, getConfig()) +func testFetch(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host())) events, errs := mbtest.ReportingFetchV2(f) if len(errs) > 0 { t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) @@ -42,25 +48,17 @@ func TestFetch(t *testing.T) { t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events[0]) } -func TestData(t *testing.T) { - compose.EnsureUp(t, "couchbase") - - f := mbtest.NewReportingMetricSetV2(t, getConfig()) - events, errs := mbtest.ReportingFetchV2(f) - if len(errs) > 0 { - t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) - } - - assert.NotEmpty(t, events) +func testData(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host())) if err := mbtest.WriteEventsReporterV2(f, t, ""); err != nil { t.Fatal("write", err) } } -func getConfig() map[string]interface{} { +func getConfig(host string) map[string]interface{} { return map[string]interface{}{ "module": "couchbase", "metricsets": []string{"cluster"}, - "hosts": []string{couchbase.GetEnvDSN()}, + "hosts": []string{mtest.GetDSN(host)}, } } diff --git a/metricbeat/module/postgresql/testing.go b/metricbeat/module/couchbase/mtest/dsn.go similarity index 76% rename from metricbeat/module/postgresql/testing.go rename to metricbeat/module/couchbase/mtest/dsn.go index 13673c8c1d1..993d17c8bd8 100644 --- a/metricbeat/module/postgresql/testing.go +++ b/metricbeat/module/couchbase/mtest/dsn.go @@ -15,18 +15,9 @@ // specific language governing permissions and limitations // under the License. -package postgresql +package mtest -import "os" - -func GetEnvDSN() string { - return os.Getenv("POSTGRESQL_DSN") -} - -func GetEnvUsername() string { - return os.Getenv("POSTGRESQL_USERNAME") -} - -func GetEnvPassword() string { - return os.Getenv("POSTGRESQL_PASSWORD") +// GetDSN gets the test DSN for a given host +func GetDSN(host string) string { + return "http://Administrator:password@" + host } diff --git a/metricbeat/module/couchbase/node/node_integration_test.go b/metricbeat/module/couchbase/node/node_integration_test.go index 9513e91cf7a..8072131a24c 100644 --- a/metricbeat/module/couchbase/node/node_integration_test.go +++ b/metricbeat/module/couchbase/node/node_integration_test.go @@ -26,13 +26,19 @@ import ( "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" - "github.com/elastic/beats/metricbeat/module/couchbase" + "github.com/elastic/beats/metricbeat/module/couchbase/mtest" ) -func TestFetch(t *testing.T) { - compose.EnsureUp(t, "couchbase") +func TestNode(t *testing.T) { + runner := compose.TestRunner{Service: "couchbase"} + runner.Run(t, compose.Suite{ + "Fetch": testFetch, + "Data": testData, + }) +} - f := mbtest.NewReportingMetricSetV2(t, getConfig()) +func testFetch(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host())) events, errs := mbtest.ReportingFetchV2(f) if len(errs) > 0 { t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) @@ -42,25 +48,17 @@ func TestFetch(t *testing.T) { t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events[0]) } -func TestData(t *testing.T) { - compose.EnsureUp(t, "couchbase") - - f := mbtest.NewReportingMetricSetV2(t, getConfig()) - events, errs := mbtest.ReportingFetchV2(f) - if len(errs) > 0 { - t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) - } - - assert.NotEmpty(t, events) +func testData(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host())) if err := mbtest.WriteEventsReporterV2(f, t, ""); err != nil { t.Fatal("write", err) } } -func getConfig() map[string]interface{} { +func getConfig(host string) map[string]interface{} { return map[string]interface{}{ "module": "couchbase", "metricsets": []string{"node"}, - "hosts": []string{couchbase.GetEnvDSN()}, + "hosts": []string{mtest.GetDSN(host)}, } } diff --git a/metricbeat/module/couchdb/_meta/env b/metricbeat/module/couchdb/_meta/env deleted file mode 100644 index d790ad1f99b..00000000000 --- a/metricbeat/module/couchdb/_meta/env +++ /dev/null @@ -1,2 +0,0 @@ -COUCHDB_HOST=couchdb -COUCHDB_PORT=5984 diff --git a/metricbeat/module/couchdb/server/server_integration_test.go b/metricbeat/module/couchdb/server/server_integration_test.go index d1b806ea17f..d0f5a3e1494 100644 --- a/metricbeat/module/couchdb/server/server_integration_test.go +++ b/metricbeat/module/couchdb/server/server_integration_test.go @@ -20,7 +20,6 @@ package server import ( - "os" "testing" "github.com/stretchr/testify/assert" @@ -29,10 +28,16 @@ import ( mbtest "github.com/elastic/beats/metricbeat/mb/testing" ) -func TestFetch(t *testing.T) { - compose.EnsureUp(t, "couchdb") +func TestServer(t *testing.T) { + runner := compose.TestRunner{Service: "couchdb"} + runner.Run(t, compose.Suite{ + "Fetch": testFetch, + "Data": testData, + }) +} - f := mbtest.NewReportingMetricSetV2(t, getConfig()) +func testFetch(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host())) events, errs := mbtest.ReportingFetchV2(f) if len(errs) > 0 { t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) @@ -42,43 +47,17 @@ func TestFetch(t *testing.T) { t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events[0]) } -func TestData(t *testing.T) { - compose.EnsureUp(t, "couchdb") - - f := mbtest.NewReportingMetricSetV2(t, getConfig()) - events, errs := mbtest.ReportingFetchV2(f) - if len(errs) > 0 { - t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) - } - assert.NotEmpty(t, events) - +func testData(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host())) if err := mbtest.WriteEventsReporterV2(f, t, ""); err != nil { t.Fatal("write", err) } } -func getConfig() map[string]interface{} { +func getConfig(host string) map[string]interface{} { return map[string]interface{}{ "module": "couchdb", "metricsets": []string{"server"}, - "hosts": []string{GetEnvHost() + ":" + GetEnvPort()}, - } -} - -func GetEnvHost() string { - host := os.Getenv("COUCHDB_HOST") - - if len(host) == 0 { - host = "127.0.0.1" - } - return host -} - -func GetEnvPort() string { - port := os.Getenv("COUCHDB_PORT") - - if len(port) == 0 { - port = "5984" + "hosts": []string{host}, } - return port } diff --git a/metricbeat/module/dropwizard/_meta/env b/metricbeat/module/dropwizard/_meta/env deleted file mode 100644 index 771807862fd..00000000000 --- a/metricbeat/module/dropwizard/_meta/env +++ /dev/null @@ -1,2 +0,0 @@ -DROPWIZARD_HOST=dropwizard -DROPWIZARD_PORT=8080 diff --git a/metricbeat/module/dropwizard/collector/collector_integration_test.go b/metricbeat/module/dropwizard/collector/collector_integration_test.go index c4b3cf490ab..f864cabe210 100644 --- a/metricbeat/module/dropwizard/collector/collector_integration_test.go +++ b/metricbeat/module/dropwizard/collector/collector_integration_test.go @@ -20,7 +20,6 @@ package collector import ( - "os" "testing" "github.com/stretchr/testify/assert" @@ -30,10 +29,19 @@ import ( mbtest "github.com/elastic/beats/metricbeat/mb/testing" ) -func TestFetch(t *testing.T) { - compose.EnsureUp(t, "dropwizard") +func TestCollector(t *testing.T) { + t.Parallel() - f := mbtest.NewEventsFetcher(t, getConfig()) + runner := compose.TestRunner{Service: "dropwizard"} + + runner.Run(t, compose.Suite{ + "Fetch": testFetch, + "Data": testData, + }) +} + +func testFetch(t *testing.T, r compose.R) { + f := mbtest.NewEventsFetcher(t, getConfig(r.Host())) events, err := f.Fetch() hasTag := false @@ -74,39 +82,19 @@ func TestFetch(t *testing.T) { t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events) } -func TestData(t *testing.T) { - compose.EnsureUp(t, "dropwizard") - - f := mbtest.NewEventsFetcher(t, getConfig()) +func testData(t *testing.T, r compose.R) { + f := mbtest.NewEventsFetcher(t, getConfig(r.Host())) err := mbtest.WriteEvents(f, t) if err != nil { t.Fatal("write", err) } } -func getEnvHost() string { - host := os.Getenv("DROPWIZARD_HOST") - - if len(host) == 0 { - host = "127.0.0.1" - } - return host -} - -func getEnvPort() string { - port := os.Getenv("DROPWIZARD_PORT") - - if len(port) == 0 { - port = "8080" - } - return port -} - -func getConfig() map[string]interface{} { +func getConfig(host string) map[string]interface{} { return map[string]interface{}{ "module": "dropwizard", "metricsets": []string{"collector"}, - "hosts": []string{getEnvHost() + ":" + getEnvPort()}, + "hosts": []string{host}, "namespace": "testnamespace", "metrics_path": "/test/metrics", "enabled": true, diff --git a/metricbeat/module/elasticsearch/_meta/Dockerfile b/metricbeat/module/elasticsearch/_meta/Dockerfile deleted file mode 100644 index ec61f1c3160..00000000000 --- a/metricbeat/module/elasticsearch/_meta/Dockerfile +++ /dev/null @@ -1,2 +0,0 @@ -FROM docker.elastic.co/elasticsearch/elasticsearch:6.6.0 -HEALTHCHECK --interval=1s --retries=300 CMD curl -f http://localhost:9200/_xpack/license diff --git a/metricbeat/module/elasticsearch/_meta/env b/metricbeat/module/elasticsearch/_meta/env deleted file mode 100644 index 5856225ceaf..00000000000 --- a/metricbeat/module/elasticsearch/_meta/env +++ /dev/null @@ -1,2 +0,0 @@ -ES_HOST=elasticsearch -ES_PORT=9200 diff --git a/metricbeat/module/elasticsearch/docker-compose.yml b/metricbeat/module/elasticsearch/docker-compose.yml new file mode 100644 index 00000000000..47d6e33cb6b --- /dev/null +++ b/metricbeat/module/elasticsearch/docker-compose.yml @@ -0,0 +1,19 @@ +version: "2.3" + +services: + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION:-6.5.4} + environment: + - "ES_JAVA_OPTS=-Xms90m -Xmx90m" + - "network.host=" + - "transport.host=127.0.0.1" + - "http.host=0.0.0.0" + - "xpack.security.enabled=false" + ports: + - 9200 + healthcheck: + test: + - "CMD-SHELL" + - "curl -f http://localhost:9200/_xpack/license" + interval: 1s + retries: 300 diff --git a/metricbeat/module/elasticsearch/elasticsearch_integration_test.go b/metricbeat/module/elasticsearch/elasticsearch_integration_test.go index 6e173a95a4c..0cced75df36 100644 --- a/metricbeat/module/elasticsearch/elasticsearch_integration_test.go +++ b/metricbeat/module/elasticsearch/elasticsearch_integration_test.go @@ -24,9 +24,7 @@ import ( "encoding/json" "fmt" "io/ioutil" - "net" "net/http" - "os" "testing" "github.com/pkg/errors" @@ -61,10 +59,34 @@ var metricSets = []string{ "shard", } -func TestFetch(t *testing.T) { - compose.EnsureUp(t, "elasticsearch") +func TestElasticsearch(t *testing.T) { + runner := compose.TestRunner{ + Service: "elasticsearch", + Options: compose.RunnerOptions{ + "ELASTICSEARCH_VERSION": { + // "7.0.0-alpha2", + "6.5.4", + "6.4.3", + "6.3.2", + "6.2.4", + "5.6.14", + }, + }, + Parallel: true, + } + + runner.Run(t, compose.Suite{ + "Fetch": testFetch, + "Data": testData, + }) +} + +func testFetch(t *testing.T, r compose.R) { + if v := r.Option("ELASTICSEARCH_VERSION"); v == "6.2.4" || v == "5.6.14" { + t.Skip("This test fails on this version") + } - host := net.JoinHostPort(getEnvHost(), getEnvPort()) + host := r.Host() err := createIndex(host) assert.NoError(t, err) @@ -83,9 +105,9 @@ func TestFetch(t *testing.T) { assert.NoError(t, err) for _, metricSet := range metricSets { - checkSkip(t, metricSet, version) t.Run(metricSet, func(t *testing.T) { - f := mbtest.NewReportingMetricSetV2(t, getConfig(metricSet)) + checkSkip(t, metricSet, version, host) + f := mbtest.NewReportingMetricSetV2(t, getConfig(metricSet, host)) events, errs := mbtest.ReportingFetchV2(f) assert.Empty(t, errs) @@ -98,20 +120,16 @@ func TestFetch(t *testing.T) { } } -func TestData(t *testing.T) { - compose.EnsureUp(t, "elasticsearch") - - host := net.JoinHostPort(getEnvHost(), getEnvPort()) - - version, err := getElasticsearchVersion(host) +func testData(t *testing.T, r compose.R) { + version, err := getElasticsearchVersion(r.Host()) if err != nil { t.Fatal("getting elasticsearch version", err) } for _, metricSet := range metricSets { - checkSkip(t, metricSet, version) t.Run(metricSet, func(t *testing.T) { - f := mbtest.NewReportingMetricSetV2(t, getConfig(metricSet)) + checkSkip(t, metricSet, version, r.Host()) + f := mbtest.NewReportingMetricSetV2(t, getConfig(metricSet, r.Host())) err := mbtest.WriteEventsReporterV2(f, t, metricSet) if err != nil { t.Fatal("write", err) @@ -120,32 +138,12 @@ func TestData(t *testing.T) { } } -// GetEnvHost returns host for Elasticsearch -func getEnvHost() string { - host := os.Getenv("ES_HOST") - - if len(host) == 0 { - host = "127.0.0.1" - } - return host -} - -// GetEnvPort returns port for Elasticsearch -func getEnvPort() string { - port := os.Getenv("ES_PORT") - - if len(port) == 0 { - port = "9200" - } - return port -} - // GetConfig returns config for elasticsearch module -func getConfig(metricset string) map[string]interface{} { +func getConfig(metricset string, host string) map[string]interface{} { return map[string]interface{}{ "module": elasticsearch.ModuleName, "metricsets": []string{metricset}, - "hosts": []string{getEnvHost() + ":" + getEnvPort()}, + "hosts": []string{host}, "index_recovery.active_only": false, } } @@ -305,7 +303,7 @@ func checkExists(url string) bool { return false } -func checkSkip(t *testing.T, metricset string, version *common.Version) { +func checkSkip(t *testing.T, metricset string, version *common.Version, host string) { if metricset != "ccr" { return } diff --git a/metricbeat/module/elasticsearch/test_elasticsearch.py b/metricbeat/module/elasticsearch/test_elasticsearch.py index 2a4ea48195b..4c597d1215e 100644 --- a/metricbeat/module/elasticsearch/test_elasticsearch.py +++ b/metricbeat/module/elasticsearch/test_elasticsearch.py @@ -48,8 +48,7 @@ def test_metricsets(self, metricset): ["service"], extras={"index_recovery.active_only": "false"}) def get_hosts(self): - return [os.getenv('ES_HOST', 'localhost') + ':' + - os.getenv('ES_PORT', '9200')] + return [self.compose_host()] def create_ml_job(self, es): es_version = self.get_version(es) diff --git a/metricbeat/module/envoyproxy/_meta/env b/metricbeat/module/envoyproxy/_meta/env deleted file mode 100644 index 75a2386e7f6..00000000000 --- a/metricbeat/module/envoyproxy/_meta/env +++ /dev/null @@ -1,2 +0,0 @@ -ENVOYPROXY_HOST=envoyproxy -ENVOYPROXY_PORT=9901 diff --git a/metricbeat/module/envoyproxy/server/server_integration_test.go b/metricbeat/module/envoyproxy/server/server_integration_test.go index af001de4594..00e8b79a9c3 100644 --- a/metricbeat/module/envoyproxy/server/server_integration_test.go +++ b/metricbeat/module/envoyproxy/server/server_integration_test.go @@ -20,7 +20,6 @@ package server import ( - "os" "testing" "github.com/stretchr/testify/assert" @@ -29,20 +28,17 @@ import ( mbtest "github.com/elastic/beats/metricbeat/mb/testing" ) -func TestData(t *testing.T) { - compose.EnsureUp(t, "envoyproxy") +func TestEnvoyproxy(t *testing.T) { + runner := compose.TestRunner{Service: "envoyproxy"} - f := mbtest.NewEventFetcher(t, getConfig()) - err := mbtest.WriteEvent(f, t) - if err != nil { - t.Fatal("write", err) - } + runner.Run(t, compose.Suite{ + "Fetch": testFetch, + "Data": testData, + }) } -func TestFetch(t *testing.T) { - compose.EnsureUp(t, "envoyproxy") - - f := mbtest.NewEventFetcher(t, getConfig()) +func testFetch(t *testing.T, r compose.R) { + f := mbtest.NewEventFetcher(t, getConfig(r.Host())) event, err := f.Fetch() if !assert.NoError(t, err) { t.FailNow() @@ -52,28 +48,18 @@ func TestFetch(t *testing.T) { t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event) } -func getConfig() map[string]interface{} { - return map[string]interface{}{ - "module": "envoyproxy", - "metricsets": []string{"server"}, - "hosts": []string{GetEnvHost() + ":" + GetEnvPort()}, - } -} - -func GetEnvHost() string { - host := os.Getenv("ENVOYPROXY_HOST") - - if len(host) == 0 { - host = "127.0.0.1" +func testData(t *testing.T, r compose.R) { + f := mbtest.NewEventFetcher(t, getConfig(r.Host())) + err := mbtest.WriteEvent(f, t) + if err != nil { + t.Fatal("write", err) } - return host } -func GetEnvPort() string { - port := os.Getenv("ENVOYPROXY_PORT") - - if len(port) == 0 { - port = "9901" +func getConfig(host string) map[string]interface{} { + return map[string]interface{}{ + "module": "envoyproxy", + "metricsets": []string{"server"}, + "hosts": []string{host}, } - return port } diff --git a/metricbeat/module/etcd/_meta/env b/metricbeat/module/etcd/_meta/env deleted file mode 100644 index ea9d5440985..00000000000 --- a/metricbeat/module/etcd/_meta/env +++ /dev/null @@ -1,2 +0,0 @@ -ETCD_HOST=etcd -ETCD_PORT=2379 diff --git a/metricbeat/module/etcd/leader/leader_integration_test.go b/metricbeat/module/etcd/leader/leader_integration_test.go index f9cfe680796..e2f70afb3ac 100644 --- a/metricbeat/module/etcd/leader/leader_integration_test.go +++ b/metricbeat/module/etcd/leader/leader_integration_test.go @@ -20,23 +20,27 @@ package leader import ( - "os" "testing" - "github.com/stretchr/testify/assert" - "github.com/elastic/beats/libbeat/logp" + "github.com/stretchr/testify/assert" "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" ) -func TestFetch(t *testing.T) { +func TestLeader(t *testing.T) { logp.TestingSetup() - compose.EnsureUp(t, "etcd") + runner := compose.TestRunner{Service: "etcd"} + runner.Run(t, compose.Suite{ + "Fetch": testFetch, + "Data": testData, + }) +} - f := mbtest.NewReportingMetricSetV2(t, getConfig()) +func testFetch(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host())) events, errs := mbtest.ReportingFetchV2(f) if len(errs) > 0 { t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) @@ -45,43 +49,17 @@ func TestFetch(t *testing.T) { t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events[0]) } -func TestData(t *testing.T) { - compose.EnsureUp(t, "etcd") - - f := mbtest.NewReportingMetricSetV2(t, getConfig()) - events, errs := mbtest.ReportingFetchV2(f) - if len(errs) > 0 { - t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) - } - assert.NotEmpty(t, events) - +func testData(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host())) if err := mbtest.WriteEventsReporterV2(f, t, ""); err != nil { t.Fatal("write", err) } } -func getConfig() map[string]interface{} { +func getConfig(host string) map[string]interface{} { return map[string]interface{}{ "module": "etcd", "metricsets": []string{"leader"}, - "hosts": []string{GetEnvHost() + ":" + GetEnvPort()}, - } -} - -func GetEnvHost() string { - host := os.Getenv("ETCD_HOST") - - if len(host) == 0 { - host = "127.0.0.1" - } - return host -} - -func GetEnvPort() string { - port := os.Getenv("ETCD_PORT") - - if len(port) == 0 { - port = "2379" + "hosts": []string{host}, } - return port } diff --git a/metricbeat/module/etcd/self/self_integration_test.go b/metricbeat/module/etcd/self/self_integration_test.go index 90e0b77b41e..0b9ef3b5524 100644 --- a/metricbeat/module/etcd/self/self_integration_test.go +++ b/metricbeat/module/etcd/self/self_integration_test.go @@ -20,7 +20,6 @@ package self import ( - "os" "testing" "github.com/stretchr/testify/assert" @@ -29,10 +28,17 @@ import ( mbtest "github.com/elastic/beats/metricbeat/mb/testing" ) -func TestFetch(t *testing.T) { - compose.EnsureUp(t, "etcd") +func TestLeader(t *testing.T) { + runner := compose.TestRunner{Service: "etcd"} - f := mbtest.NewReportingMetricSetV2(t, getConfig()) + runner.Run(t, compose.Suite{ + "Fetch": testFetch, + "Data": testData, + }) +} + +func testFetch(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host())) events, errs := mbtest.ReportingFetchV2(f) if len(errs) > 0 { t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) @@ -42,43 +48,17 @@ func TestFetch(t *testing.T) { t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events[0]) } -func TestData(t *testing.T) { - compose.EnsureUp(t, "etcd") - - f := mbtest.NewReportingMetricSetV2(t, getConfig()) - events, errs := mbtest.ReportingFetchV2(f) - if len(errs) > 0 { - t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) - } - assert.NotEmpty(t, events) - +func testData(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host())) if err := mbtest.WriteEventsReporterV2(f, t, ""); err != nil { t.Fatal("write", err) } } -func getConfig() map[string]interface{} { +func getConfig(host string) map[string]interface{} { return map[string]interface{}{ "module": "etcd", "metricsets": []string{"self"}, - "hosts": []string{GetEnvHost() + ":" + GetEnvPort()}, - } -} - -func GetEnvHost() string { - host := os.Getenv("ETCD_HOST") - - if len(host) == 0 { - host = "127.0.0.1" - } - return host -} - -func GetEnvPort() string { - port := os.Getenv("ETCD_PORT") - - if len(port) == 0 { - port = "2379" + "hosts": []string{host}, } - return port } diff --git a/metricbeat/module/etcd/store/store_integration_test.go b/metricbeat/module/etcd/store/store_integration_test.go index 932ed5d4925..18425e8617c 100644 --- a/metricbeat/module/etcd/store/store_integration_test.go +++ b/metricbeat/module/etcd/store/store_integration_test.go @@ -20,7 +20,6 @@ package store import ( - "os" "testing" "github.com/stretchr/testify/assert" @@ -30,55 +29,36 @@ import ( mbtest "github.com/elastic/beats/metricbeat/mb/testing" ) -func TestFetch(t *testing.T) { +func TestStore(t *testing.T) { logp.TestingSetup() - compose.EnsureUp(t, "etcd") - f := mbtest.NewReportingMetricSetV2(t, getConfig()) - events, errs := mbtest.ReportingFetchV2(f) - if len(errs) > 0 { - t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) - } - assert.NotEmpty(t, events) + runner := compose.TestRunner{Service: "etcd"} + runner.Run(t, compose.Suite{ + "Fetch": testFetch, + "Data": testData, + }) } -func TestData(t *testing.T) { - compose.EnsureUp(t, "etcd") - - f := mbtest.NewReportingMetricSetV2(t, getConfig()) +func testFetch(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host())) events, errs := mbtest.ReportingFetchV2(f) if len(errs) > 0 { t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) } assert.NotEmpty(t, events) +} +func testData(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host())) if err := mbtest.WriteEventsReporterV2(f, t, ""); err != nil { t.Fatal("write", err) } } -func getConfig() map[string]interface{} { +func getConfig(host string) map[string]interface{} { return map[string]interface{}{ "module": "etcd", "metricsets": []string{"store"}, - "hosts": []string{GetEnvHost() + ":" + GetEnvPort()}, - } -} - -func GetEnvHost() string { - host := os.Getenv("ETCD_HOST") - - if len(host) == 0 { - host = "127.0.0.1" - } - return host -} - -func GetEnvPort() string { - port := os.Getenv("ETCD_PORT") - - if len(port) == 0 { - port = "2379" + "hosts": []string{host}, } - return port } diff --git a/metricbeat/module/etcd/test_etcd.py b/metricbeat/module/etcd/test_etcd.py index 9441c6e6d75..98118594767 100644 --- a/metricbeat/module/etcd/test_etcd.py +++ b/metricbeat/module/etcd/test_etcd.py @@ -25,8 +25,7 @@ def test_metricset(self, metricset): self.check_metricset("etcd", metricset, self.get_hosts(), ['etcd.' + metricset]) def get_hosts(self): - return [self.compose_hosts()[0] + ':' + - os.getenv('ETCD_PORT', '2379')] + return [self.compose_host()] class Test_3_2(Test): diff --git a/metricbeat/module/golang/_meta/env b/metricbeat/module/golang/_meta/env deleted file mode 100644 index f030c0adc85..00000000000 --- a/metricbeat/module/golang/_meta/env +++ /dev/null @@ -1,2 +0,0 @@ -GOLANG_HOST=golang -GOLANG_PORT=6060 diff --git a/metricbeat/module/golang/expvar/expvar_integration_test.go b/metricbeat/module/golang/expvar/expvar_integration_test.go index 8ee3cceaf69..10b9a8bb342 100644 --- a/metricbeat/module/golang/expvar/expvar_integration_test.go +++ b/metricbeat/module/golang/expvar/expvar_integration_test.go @@ -20,30 +20,35 @@ package expvar import ( - "os" "testing" "github.com/stretchr/testify/assert" + "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" ) -func TestData(t *testing.T) { - compose.EnsureUp(t, "golang") +func TestExpvar(t *testing.T) { + logp.TestingSetup() - f := mbtest.NewReportingMetricSetV2(t, getConfig()) + runner := compose.TestRunner{Service: "golang"} + runner.Run(t, compose.Suite{ + "Data": testData, + "Fetch": testFetch, + }) +} +func testData(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host())) err := mbtest.WriteEventsReporterV2(f, t, "") if !assert.NoError(t, err) { t.FailNow() } } -func TestFetch(t *testing.T) { - compose.EnsureUp(t, "golang") - - f := mbtest.NewReportingMetricSetV2(t, getConfig()) +func testFetch(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host())) events, errs := mbtest.ReportingFetchV2(f) if len(errs) > 0 { @@ -55,29 +60,11 @@ func TestFetch(t *testing.T) { t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events[0]) } -func getConfig() map[string]interface{} { +func getConfig(host string) map[string]interface{} { return map[string]interface{}{ "module": "golang", "metricsets": []string{"expvar"}, "expvar.namespace": "metricbeat", - "hosts": []string{GetEnvHost() + ":" + GetEnvPort()}, - } -} - -func GetEnvHost() string { - host := os.Getenv("GOLANG_HOST") - - if len(host) == 0 { - host = "127.0.0.1" - } - return host -} - -func GetEnvPort() string { - port := os.Getenv("GOLANG_PORT") - - if len(port) == 0 { - port = "6060" + "hosts": []string{host}, } - return port } diff --git a/metricbeat/module/golang/heap/heap_integration_test.go b/metricbeat/module/golang/heap/heap_integration_test.go index 7ec2a5e6f2c..d1f28362f75 100644 --- a/metricbeat/module/golang/heap/heap_integration_test.go +++ b/metricbeat/module/golang/heap/heap_integration_test.go @@ -20,30 +20,35 @@ package heap import ( - "os" "testing" + "github.com/elastic/beats/libbeat/logp" "github.com/stretchr/testify/assert" "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" ) -func TestData(t *testing.T) { - compose.EnsureUp(t, "golang") +func TestHeap(t *testing.T) { + logp.TestingSetup() - f := mbtest.NewReportingMetricSetV2(t, getConfig()) + runner := compose.TestRunner{Service: "golang"} + runner.Run(t, compose.Suite{ + "Data": testData, + "Fetch": testFetch, + }) +} +func testData(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host())) err := mbtest.WriteEventsReporterV2(f, t, "") if !assert.NoError(t, err) { t.FailNow() } } -func TestFetch(t *testing.T) { - compose.EnsureUp(t, "golang") - - f := mbtest.NewReportingMetricSetV2(t, getConfig()) +func testFetch(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host())) events, errs := mbtest.ReportingFetchV2(f) if len(errs) > 0 { @@ -55,28 +60,10 @@ func TestFetch(t *testing.T) { t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events[0]) } -func getConfig() map[string]interface{} { +func getConfig(host string) map[string]interface{} { return map[string]interface{}{ "module": "golang", "metricsets": []string{"heap"}, - "hosts": []string{GetEnvHost() + ":" + GetEnvPort()}, - } -} - -func GetEnvHost() string { - host := os.Getenv("GOLANG_HOST") - - if len(host) == 0 { - host = "127.0.0.1" - } - return host -} - -func GetEnvPort() string { - port := os.Getenv("GOLANG_PORT") - - if len(port) == 0 { - port = "6060" + "hosts": []string{host}, } - return port } diff --git a/metricbeat/module/haproxy/_meta/env b/metricbeat/module/haproxy/_meta/env deleted file mode 100644 index c0762e53c17..00000000000 --- a/metricbeat/module/haproxy/_meta/env +++ /dev/null @@ -1,4 +0,0 @@ -HAPROXY_1_6_HOST=haproxy_1_6 -HAPROXY_1_7_HOST=haproxy_1_7 -HAPROXY_HOST=haproxy -HAPROXY_PORT=14567 diff --git a/metricbeat/module/haproxy/info/info_integration_test.go b/metricbeat/module/haproxy/info/info_integration_test.go index b085ea13096..2798f342fb6 100644 --- a/metricbeat/module/haproxy/info/info_integration_test.go +++ b/metricbeat/module/haproxy/info/info_integration_test.go @@ -26,13 +26,18 @@ import ( "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" - "github.com/elastic/beats/metricbeat/module/haproxy" ) -func TestFetch(t *testing.T) { - compose.EnsureUp(t, "haproxy") +func TestInfo(t *testing.T) { + runner := compose.TestRunner{Service: "haproxy"} + runner.Run(t, compose.Suite{ + "Fetch": testFetch, + "Data": testData, + }) +} - f := mbtest.NewReportingMetricSetV2(t, getConfig()) +func testFetch(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host())) events, errs := mbtest.ReportingFetchV2(f) assert.Empty(t, errs) @@ -45,10 +50,8 @@ func TestFetch(t *testing.T) { } -func TestData(t *testing.T) { - compose.EnsureUp(t, "haproxy") - - config := getConfig() +func testData(t *testing.T, r compose.R) { + config := getConfig(r.Host()) f := mbtest.NewReportingMetricSetV2(t, config) err := mbtest.WriteEventsReporterV2(f, t, ".") if err != nil { @@ -57,10 +60,10 @@ func TestData(t *testing.T) { } -func getConfig() map[string]interface{} { +func getConfig(host string) map[string]interface{} { return map[string]interface{}{ "module": "haproxy", "metricsets": []string{"info"}, - "hosts": []string{"tcp://" + haproxy.GetEnvHost() + ":" + haproxy.GetEnvPort()}, + "hosts": []string{"tcp://" + host}, } } diff --git a/metricbeat/module/haproxy/stat/stat_integration_test.go b/metricbeat/module/haproxy/stat/stat_integration_test.go index bc4f3258b53..758f221b73c 100644 --- a/metricbeat/module/haproxy/stat/stat_integration_test.go +++ b/metricbeat/module/haproxy/stat/stat_integration_test.go @@ -26,13 +26,18 @@ import ( "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" - "github.com/elastic/beats/metricbeat/module/haproxy" ) -func TestFetch(t *testing.T) { - compose.EnsureUp(t, "haproxy") +func TestStat(t *testing.T) { + runner := compose.TestRunner{Service: "haproxy"} + runner.Run(t, compose.Suite{ + "Fetch": testFetch, + "Data": testData, + }) +} - f := mbtest.NewReportingMetricSetV2(t, getConfig()) +func testFetch(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host())) events, errs := mbtest.ReportingFetchV2(f) assert.Empty(t, errs) @@ -45,10 +50,8 @@ func TestFetch(t *testing.T) { } -func TestData(t *testing.T) { - compose.EnsureUp(t, "haproxy") - - config := getConfig() +func testData(t *testing.T, r compose.R) { + config := getConfig(r.Host()) f := mbtest.NewReportingMetricSetV2(t, config) err := mbtest.WriteEventsReporterV2(f, t, ".") if err != nil { @@ -57,10 +60,10 @@ func TestData(t *testing.T) { } -func getConfig() map[string]interface{} { +func getConfig(host string) map[string]interface{} { return map[string]interface{}{ "module": "haproxy", "metricsets": []string{"stat"}, - "hosts": []string{"tcp://" + haproxy.GetEnvHost() + ":" + haproxy.GetEnvPort()}, + "hosts": []string{"tcp://" + host}, } } diff --git a/metricbeat/module/haproxy/testing.go b/metricbeat/module/haproxy/testing.go deleted file mode 100644 index f5bf9913413..00000000000 --- a/metricbeat/module/haproxy/testing.go +++ /dev/null @@ -1,46 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package haproxy - -import "os" - -// Helper functions for testing used in the haproxy metricsets - -// GetEnvHost returns the hostname of the HAProxy server to use for testing. -// It reads the value from the HAPROXY_HOST environment variable and returns -// 127.0.0.1 if it is not set. -func GetEnvHost() string { - host := os.Getenv("HAPROXY_HOST") - - if len(host) == 0 { - host = "127.0.0.1" - } - return host -} - -// GetRedisEnvPort returns the port of the HAProxy server to use for testing. -// It reads the value from the HAPROXY_PORT environment variable and returns -// 14567 if it is not set. -func GetEnvPort() string { - port := os.Getenv("HAPROXY_PORT") - - if len(port) == 0 { - port = "14567" - } - return port -} diff --git a/metricbeat/module/http/_meta/env b/metricbeat/module/http/_meta/env deleted file mode 100644 index dfe04431eca..00000000000 --- a/metricbeat/module/http/_meta/env +++ /dev/null @@ -1,2 +0,0 @@ -HTTP_HOST=http -HTTP_PORT=8080 diff --git a/metricbeat/module/http/json/json_integration_test.go b/metricbeat/module/http/json/json_integration_test.go index cd215316263..85748e0a843 100644 --- a/metricbeat/module/http/json/json_integration_test.go +++ b/metricbeat/module/http/json/json_integration_test.go @@ -20,7 +20,6 @@ package json import ( - "os" "testing" "github.com/stretchr/testify/assert" @@ -29,10 +28,18 @@ import ( mbtest "github.com/elastic/beats/metricbeat/mb/testing" ) -func TestFetchObject(t *testing.T) { - compose.EnsureUp(t, "http") +func TestJSON(t *testing.T) { + runner := compose.TestRunner{Service: "http"} - f := mbtest.NewEventsFetcher(t, getConfig("object")) + runner.Run(t, compose.Suite{ + "FetchObject": testFetchObject, + "FetchArray": testFetchArray, + "Data": testData, + }) +} + +func testFetchObject(t *testing.T, r compose.R) { + f := mbtest.NewEventsFetcher(t, getConfig("object", r.Host())) event, err := f.Fetch() if !assert.NoError(t, err) { t.FailNow() @@ -41,10 +48,8 @@ func TestFetchObject(t *testing.T) { t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event) } -func TestFetchArray(t *testing.T) { - compose.EnsureUp(t, "http") - - f := mbtest.NewEventsFetcher(t, getConfig("array")) +func testFetchArray(t *testing.T, r compose.R) { + f := mbtest.NewEventsFetcher(t, getConfig("array", r.Host())) event, err := f.Fetch() if !assert.NoError(t, err) { t.FailNow() @@ -52,18 +57,16 @@ func TestFetchArray(t *testing.T) { t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event) } -func TestData(t *testing.T) { - compose.EnsureUp(t, "http") - f := mbtest.NewEventsFetcher(t, getConfig("object")) +func testData(t *testing.T, r compose.R) { + f := mbtest.NewEventsFetcher(t, getConfig("object", r.Host())) err := mbtest.WriteEvents(f, t) if err != nil { t.Fatal("write", err) } - } -func getConfig(jsonType string) map[string]interface{} { +func getConfig(jsonType string, host string) map[string]interface{} { var path string var responseIsArray bool switch jsonType { @@ -78,27 +81,9 @@ func getConfig(jsonType string) map[string]interface{} { return map[string]interface{}{ "module": "http", "metricsets": []string{"json"}, - "hosts": []string{getEnvHost() + ":" + getEnvPort()}, + "hosts": []string{host}, "path": path, "namespace": "testnamespace", "json.is_array": responseIsArray, } } - -func getEnvHost() string { - host := os.Getenv("HTTP_HOST") - - if len(host) == 0 { - host = "127.0.0.1" - } - return host -} - -func getEnvPort() string { - port := os.Getenv("HTTP_PORT") - - if len(port) == 0 { - port = "8080" - } - return port -} diff --git a/metricbeat/module/jolokia/_meta/env b/metricbeat/module/jolokia/_meta/env deleted file mode 100644 index 9c0340b6f3c..00000000000 --- a/metricbeat/module/jolokia/_meta/env +++ /dev/null @@ -1,2 +0,0 @@ -JOLOKIA_HOST=jolokia -JOLOKIA_PORT=8778 diff --git a/metricbeat/module/jolokia/jmx/jmx_integration_test.go b/metricbeat/module/jolokia/jmx/jmx_integration_test.go index e8cff3b62cf..e3cfaded03c 100644 --- a/metricbeat/module/jolokia/jmx/jmx_integration_test.go +++ b/metricbeat/module/jolokia/jmx/jmx_integration_test.go @@ -20,7 +20,6 @@ package jmx import ( - "os" "testing" "github.com/stretchr/testify/assert" @@ -29,10 +28,17 @@ import ( mbtest "github.com/elastic/beats/metricbeat/mb/testing" ) -func TestFetch(t *testing.T) { - compose.EnsureUp(t, "jolokia") +func TestJMX(t *testing.T) { + runner := compose.TestRunner{Service: "jolokia"} - for _, config := range getConfigs() { + runner.Run(t, compose.Suite{ + "Fetch": testFetch, + "Data": testData, + }) +} + +func testFetch(t *testing.T, r compose.R) { + for _, config := range getConfigs(r.Host()) { f := mbtest.NewEventsFetcher(t, config) events, err := f.Fetch() if !assert.NoError(t, err) { @@ -45,10 +51,8 @@ func TestFetch(t *testing.T) { } } -func TestData(t *testing.T) { - compose.EnsureUp(t, "jolokia") - - for _, config := range getConfigs() { +func testData(t *testing.T, r compose.R) { + for _, config := range getConfigs(r.Host()) { f := mbtest.NewEventsFetcher(t, config) err := mbtest.WriteEvents(f, t) if err != nil { @@ -57,12 +61,12 @@ func TestData(t *testing.T) { } } -func getConfigs() []map[string]interface{} { +func getConfigs(host string) []map[string]interface{} { return []map[string]interface{}{ { "module": "jolokia", "metricsets": []string{"jmx"}, - "hosts": []string{getEnvHost() + ":" + getEnvPort()}, + "hosts": []string{host}, "namespace": "testnamespace", "jmx.mappings": []map[string]interface{}{ { @@ -105,7 +109,7 @@ func getConfigs() []map[string]interface{} { { "module": "jolokia", "metricsets": []string{"jmx"}, - "hosts": []string{getEnvHost() + ":" + getEnvPort()}, + "hosts": []string{host}, "namespace": "testnamespace", "jmx.mappings": []map[string]interface{}{ { @@ -154,7 +158,7 @@ func getConfigs() []map[string]interface{} { { "module": "jolokia", "metricsets": []string{"jmx"}, - "hosts": []string{getEnvHost() + ":" + getEnvPort()}, + "hosts": []string{host}, "namespace": "testnamespace", "http_method": "GET", "jmx.mappings": []map[string]interface{}{ @@ -188,21 +192,3 @@ func getConfigs() []map[string]interface{} { }, } } - -func getEnvHost() string { - host := os.Getenv("JOLOKIA_HOST") - - if len(host) == 0 { - host = "127.0.0.1" - } - return host -} - -func getEnvPort() string { - port := os.Getenv("JOLOKIA_PORT") - - if len(port) == 0 { - port = "8778" - } - return port -} diff --git a/metricbeat/module/kafka/_meta/Dockerfile b/metricbeat/module/kafka/_meta/Dockerfile index 3813e4c65d2..a312fb25e23 100644 --- a/metricbeat/module/kafka/_meta/Dockerfile +++ b/metricbeat/module/kafka/_meta/Dockerfile @@ -1,18 +1,18 @@ -FROM debian:stretch +FROM openjdk:8-jre-alpine3.8 ARG KAFKA_VERSION=2.1.0 ENV KAFKA_HOME /kafka -# The advertised host is kafka. This means it will not work if container is started locally and connected from localhost to it ENV KAFKA_LOGS_DIR="/kafka-logs" ENV _JAVA_OPTIONS "-Djava.net.preferIPv4Stack=true" ENV TERM=linux -RUN apt-get update && apt-get install -y curl openjdk-8-jre-headless netcat dnsutils +RUN apk add -u bash -RUN mkdir -p ${KAFKA_LOGS_DIR} && mkdir -p ${KAFKA_HOME} && curl -s -o $INSTALL_DIR/kafka.tgz \ - "https://archive.apache.org/dist/kafka/${KAFKA_VERSION}/kafka_2.11-${KAFKA_VERSION}.tgz" && \ - tar xzf ${INSTALL_DIR}/kafka.tgz -C ${KAFKA_HOME} --strip-components 1 +RUN mkdir -p ${KAFKA_LOGS_DIR} && mkdir -p ${KAFKA_HOME} && \ + wget -q -O $INSTALL_DIR/kafka.tgz "https://archive.apache.org/dist/kafka/${KAFKA_VERSION}/kafka_2.11-${KAFKA_VERSION}.tgz" && \ + tar xzf ${INSTALL_DIR}/kafka.tgz -C ${KAFKA_HOME} --strip-components 1 && \ + rm -f ${INSTALL_DIR}/kafka.tgz ADD run.sh /run.sh ADD healthcheck.sh /healthcheck.sh diff --git a/metricbeat/module/kafka/_meta/env b/metricbeat/module/kafka/_meta/env deleted file mode 100644 index 227657f404e..00000000000 --- a/metricbeat/module/kafka/_meta/env +++ /dev/null @@ -1,3 +0,0 @@ -KAFKA_0_10_2_HOST=kafka_0_10_2 -KAFKA_HOST=kafka -KAFKA_PORT=9092 diff --git a/metricbeat/module/kafka/_meta/run.sh b/metricbeat/module/kafka/_meta/run.sh index 36b42a6a0d5..0d845b8cb53 100755 --- a/metricbeat/module/kafka/_meta/run.sh +++ b/metricbeat/module/kafka/_meta/run.sh @@ -1,6 +1,27 @@ #!/bin/bash -KAFKA_ADVERTISED_HOST=$(dig +short $HOSTNAME) +if [ -n "$KAFKA_ADVERTISED_HOST_AUTO" ]; then + ip=$(nslookup $HOSTNAME 2> /dev/null | grep ^Address | cut -d' ' -f3) + KAFKA_ADVERTISED_HOST=${ip}:9092 +fi + +# Check if KAFKA_ADVERTISED_HOST is set +# if not wait to read it from file +if [ -z "$KAFKA_ADVERTISED_HOST" ]; then + echo "KAFKA_ADVERTISED_HOST needed, will wait for it on /var/run/compose_env" + while true; do + if [ -f /run/compose_env ]; then + source /run/compose_env + KAFKA_ADVERTISED_HOST=$SERVICE_HOST + fi + if [ -n "$KAFKA_ADVERTISED_HOST" ]; then + # Remove it so it is not reused + > /run/compose_env + break + fi + sleep 1 + done +fi wait_for_port() { count=20 @@ -21,8 +42,13 @@ wait_for_port 2181 echo "Starting Kafka broker" mkdir -p ${KAFKA_LOGS_DIR} ${KAFKA_HOME}/bin/kafka-server-start.sh ${KAFKA_HOME}/config/server.properties \ - --override delete.topic.enable=true --override advertised.host.name=${KAFKA_ADVERTISED_HOST} \ - --override listeners=PLAINTEXT://0.0.0.0:9092 \ + --override delete.topic.enable=true \ + --override listeners=INSIDE://localhost:9091,OUTSIDE://0.0.0.0:9092 \ + --override advertised.listeners=INSIDE://localhost:9091,OUTSIDE://$KAFKA_ADVERTISED_HOST \ + --override listener.security.protocol.map=INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT \ + --override inter.broker.listener.name=INSIDE \ + --override offsets.topic.replication.factor=1 \ + --override offsets.topic.num.partitions=2 \ --override logs.dir=${KAFKA_LOGS_DIR} & wait_for_port 9092 diff --git a/metricbeat/module/kafka/broker.go b/metricbeat/module/kafka/broker.go index da3dfacb3c8..481ecd1d2c0 100644 --- a/metricbeat/module/kafka/broker.go +++ b/metricbeat/module/kafka/broker.go @@ -27,6 +27,7 @@ import ( "time" "github.com/Shopify/sarama" + "github.com/pkg/errors" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/common/kafka" @@ -108,7 +109,11 @@ func (b *Broker) Close() error { // Connect connects the broker to the configured host func (b *Broker) Connect() error { if err := b.broker.Open(b.cfg); err != nil { - return err + return errors.Wrap(err, "opening broker connection") + } + + if ok, err := b.broker.Connected(); !ok { + return errors.Wrap(err, "broker not connected") } if b.id != noID || !b.matchID { @@ -119,14 +124,14 @@ func (b *Broker) Connect() error { meta, err := queryMetadataWithRetry(b.broker, b.cfg, nil) if err != nil { closeBroker(b.broker) - return err + return errors.Wrap(err, "querying metadata") } finder := brokerFinder{Net: &defaultNet{}} other := finder.findBroker(brokerAddress(b.broker), meta.Brokers) if other == nil { // no broker found closeBroker(b.broker) - return fmt.Errorf("No advertised broker with address %v found", b.Addr()) + return fmt.Errorf("no advertised broker with address %v found", b.Addr()) } debugf("found matching broker %v with id %v", other.Addr(), other.ID()) diff --git a/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go b/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go index df67af8da07..e2da5be0658 100644 --- a/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go +++ b/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go @@ -20,34 +20,32 @@ package consumergroup import ( - "fmt" - "io" - "os" "testing" "time" - saramacluster "github.com/bsm/sarama-cluster" "github.com/pkg/errors" "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/elastic/beats/metricbeat/module/kafka/mtest" ) -const ( - kafkaDefaultHost = "localhost" - kafkaDefaultPort = "9092" -) - -func TestData(t *testing.T) { - compose.EnsureUp(t, "kafka") +func TestConsumerGroup(t *testing.T) { + mtest.Runner.Run(t, compose.Suite{ + "Data": testData, + }) +} - c, err := startConsumer(t, "metricbeat-test") +func testData(t *testing.T, r compose.R) { + topic := "metricbeat-test" + mtest.GenerateKafkaData(t, topic, r.Host()) + c, err := mtest.StartConsumer(t, topic, r.Host()) if err != nil { t.Fatal(errors.Wrap(err, "starting kafka consumer")) } defer c.Close() - ms := mbtest.NewReportingMetricSetV2(t, getConfig()) + ms := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host())) for retries := 0; retries < 3; retries++ { err = mbtest.WriteEventsReporterV2(ms, t, "") if err == nil { @@ -58,34 +56,10 @@ func TestData(t *testing.T) { t.Fatal("write", err) } -func startConsumer(t *testing.T, topic string) (io.Closer, error) { - brokers := []string{getTestKafkaHost()} - topics := []string{topic} - return saramacluster.NewConsumer(brokers, "test-group", topics, nil) -} - -func getConfig() map[string]interface{} { +func getConfig(host string) map[string]interface{} { return map[string]interface{}{ "module": "kafka", "metricsets": []string{"consumergroup"}, - "hosts": []string{getTestKafkaHost()}, - } -} - -func getTestKafkaHost() string { - return fmt.Sprintf("%v:%v", - getenv("KAFKA_HOST", kafkaDefaultHost), - getenv("KAFKA_PORT", kafkaDefaultPort), - ) -} - -func getenv(name, defaultValue string) string { - return strDefault(os.Getenv(name), defaultValue) -} - -func strDefault(a, defaults string) string { - if len(a) == 0 { - return defaults + "hosts": []string{host}, } - return a } diff --git a/metricbeat/module/kafka/docker-compose.yml b/metricbeat/module/kafka/docker-compose.yml new file mode 100644 index 00000000000..f626bfba077 --- /dev/null +++ b/metricbeat/module/kafka/docker-compose.yml @@ -0,0 +1,11 @@ +version: "2.3" + +services: + kafka: + image: metricbeat-kafka:${KAFKA_VERSION:-2.0.0} + build: + context: ./_meta + args: + KAFKA_VERSION: ${KAFKA_VERSION:-2.0.0} + ports: + - 9092 diff --git a/metricbeat/module/kafka/mtest/helpers.go b/metricbeat/module/kafka/mtest/helpers.go new file mode 100644 index 00000000000..4f3c309260d --- /dev/null +++ b/metricbeat/module/kafka/mtest/helpers.go @@ -0,0 +1,74 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mtest + +import ( + "io" + "testing" + "time" + + "github.com/Shopify/sarama" + saramacluster "github.com/bsm/sarama-cluster" +) + +// GenerateKafkaData generates kafka data for tests +func GenerateKafkaData(t *testing.T, topic string, host string) { + t.Logf("Send Kafka Event to topic: %v", topic) + + config := sarama.NewConfig() + config.Producer.Return.Successes = true + // Retry for 10 seconds + config.Producer.Retry.Max = 20 + config.Producer.Retry.Backoff = 500 * time.Millisecond + config.Metadata.Retry.Max = 20 + config.Metadata.Retry.Backoff = 500 * time.Millisecond + client, err := sarama.NewClient([]string{host}, config) + if err != nil { + t.Errorf("%s", err) + t.FailNow() + } + + producer, err := sarama.NewSyncProducerFromClient(client) + if err != nil { + t.Error(err) + } + defer producer.Close() + + msg := &sarama.ProducerMessage{ + Topic: topic, + Value: sarama.StringEncoder("Hello World"), + } + + _, _, err = producer.SendMessage(msg) + if err != nil { + t.Errorf("failed to send message: %s\n", err) + } + + err = client.RefreshMetadata(topic) + if err != nil { + t.Errorf("failed to refresh metadata for topic '%s': %s\n", topic, err) + } +} + +// StartConsumer starts a kafka consumer for tests +func StartConsumer(t *testing.T, topic, host string) (io.Closer, error) { + brokers := []string{host} + topics := []string{topic} + + return saramacluster.NewConsumer(brokers, "test-group", topics, nil) +} diff --git a/metricbeat/module/kafka/mtest/runner.go b/metricbeat/module/kafka/mtest/runner.go new file mode 100644 index 00000000000..56ce7b056b6 --- /dev/null +++ b/metricbeat/module/kafka/mtest/runner.go @@ -0,0 +1,39 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration + +package mtest + +import ( + "github.com/elastic/beats/libbeat/tests/compose" +) + +var ( + // Runner is the compose test runner for kafka tests + Runner = compose.TestRunner{ + Service: "kafka", + Options: compose.RunnerOptions{ + "KAFKA_VERSION": { + "0.10.2.1", + "1.1.0", + "2.0.0", + }, + }, + Parallel: true, + } +) diff --git a/metricbeat/module/kafka/partition/partition_integration_test.go b/metricbeat/module/kafka/partition/partition_integration_test.go index dc696c26b1d..ace9f55e7cb 100644 --- a/metricbeat/module/kafka/partition/partition_integration_test.go +++ b/metricbeat/module/kafka/partition/partition_integration_test.go @@ -22,50 +22,48 @@ package partition import ( "fmt" "math/rand" - "os" "strconv" "testing" "time" - "github.com/Shopify/sarama" "github.com/stretchr/testify/assert" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/elastic/beats/metricbeat/module/kafka/mtest" ) -const ( - kafkaDefaultHost = "localhost" - kafkaDefaultPort = "9092" -) +func TestPartition(t *testing.T) { + t.Parallel() + + logp.TestingSetup(logp.WithSelectors("kafka")) -func TestData(t *testing.T) { - compose.EnsureUp(t, "kafka") + mtest.Runner.Run(t, compose.Suite{ + "Data": testData, + "Topic": testTopic, + }) +} - generateKafkaData(t, "metricbeat-generate-data") +func testData(t *testing.T, r compose.R) { + mtest.GenerateKafkaData(t, "metricbeat-generate-data", r.Host()) - ms := mbtest.NewReportingMetricSetV2(t, getConfig("")) + ms := mbtest.NewReportingMetricSetV2(t, getConfig("", r.Host())) err := mbtest.WriteEventsReporterV2(ms, t, "") if err != nil { t.Fatal("write", err) } } -func TestTopic(t *testing.T) { - - compose.EnsureUp(t, "kafka") - - logp.TestingSetup(logp.WithSelectors("kafka")) - +func testTopic(t *testing.T, r compose.R) { id := strconv.Itoa(rand.New(rand.NewSource(int64(time.Now().Nanosecond()))).Int()) testTopic := fmt.Sprintf("test-metricbeat-%s", id) // Create initial topic - generateKafkaData(t, testTopic) + mtest.GenerateKafkaData(t, testTopic, r.Host()) - f := mbtest.NewReportingMetricSetV2(t, getConfig(testTopic)) + f := mbtest.NewReportingMetricSetV2(t, getConfig(testTopic, r.Host())) dataBefore, err := mbtest.ReportingFetchV2(f) if err != nil { t.Fatal("write", err) @@ -76,10 +74,9 @@ func TestTopic(t *testing.T) { t.Logf("before: %v", dataBefore) var n int64 = 10 - var i int64 = 0 // Create n messages - for ; i < n; i++ { - generateKafkaData(t, testTopic) + for i := int64(0); i < n; i++ { + mtest.GenerateKafkaData(t, testTopic, r.Host()) } dataAfter, err := mbtest.ReportingFetchV2(f) @@ -118,45 +115,7 @@ func TestTopic(t *testing.T) { assert.True(t, offsetBefore+n == offsetAfter) } -func generateKafkaData(t *testing.T, topic string) { - t.Logf("Send Kafka Event to topic: %v", topic) - - config := sarama.NewConfig() - config.Producer.Return.Successes = true - // Retry for 10 seconds - config.Producer.Retry.Max = 20 - config.Producer.Retry.Backoff = 500 * time.Millisecond - config.Metadata.Retry.Max = 20 - config.Metadata.Retry.Backoff = 500 * time.Millisecond - client, err := sarama.NewClient([]string{getTestKafkaHost()}, config) - if err != nil { - t.Errorf("%s", err) - t.FailNow() - } - - producer, err := sarama.NewSyncProducerFromClient(client) - if err != nil { - t.Error(err) - } - defer producer.Close() - - msg := &sarama.ProducerMessage{ - Topic: topic, - Value: sarama.StringEncoder("Hello World"), - } - - _, _, err = producer.SendMessage(msg) - if err != nil { - t.Errorf("failed to send message: %s\n", err) - } - - err = client.RefreshMetadata(topic) - if err != nil { - t.Errorf("failed to refresh metadata for topic '%s': %s\n", topic, err) - } -} - -func getConfig(topic string) map[string]interface{} { +func getConfig(topic string, host string) map[string]interface{} { var topics []string if topic != "" { topics = []string{topic} @@ -165,25 +124,7 @@ func getConfig(topic string) map[string]interface{} { return map[string]interface{}{ "module": "kafka", "metricsets": []string{"partition"}, - "hosts": []string{getTestKafkaHost()}, + "hosts": []string{host}, "topics": topics, } } - -func getTestKafkaHost() string { - return fmt.Sprintf("%v:%v", - getenv("KAFKA_HOST", kafkaDefaultHost), - getenv("KAFKA_PORT", kafkaDefaultPort), - ) -} - -func getenv(name, defaultValue string) string { - return strDefault(os.Getenv(name), defaultValue) -} - -func strDefault(a, defaults string) string { - if len(a) == 0 { - return defaults - } - return a -} diff --git a/metricbeat/module/kibana/_meta/env b/metricbeat/module/kibana/_meta/env deleted file mode 100644 index a22fc93ec70..00000000000 --- a/metricbeat/module/kibana/_meta/env +++ /dev/null @@ -1,2 +0,0 @@ -KIBANA_HOST=kibana -KIBANA_PORT=5601 \ No newline at end of file diff --git a/metricbeat/module/kibana/mtest/testing.go b/metricbeat/module/kibana/mtest/testing.go index a6dbc1f2f7e..968a78d6074 100644 --- a/metricbeat/module/kibana/mtest/testing.go +++ b/metricbeat/module/kibana/mtest/testing.go @@ -18,35 +18,22 @@ package mtest import ( - "net" - "os" + "github.com/elastic/beats/libbeat/tests/compose" ) -// GetEnvHost returns host for Kibana -func GetEnvHost() string { - host := os.Getenv("KIBANA_HOST") - - if len(host) == 0 { - host = "127.0.0.1" +var ( + // Runner is the compose test runner for kibana tests + Runner = compose.TestRunner{ + Service: "kibana", + Parallel: true, } - return host -} - -// GetEnvPort returns port for Kibana -func GetEnvPort() string { - port := os.Getenv("KIBANA_PORT") - - if len(port) == 0 { - port = "5601" - } - return port -} +) // GetConfig returns config for kibana module -func GetConfig(metricset string) map[string]interface{} { +func GetConfig(metricset string, host string) map[string]interface{} { return map[string]interface{}{ "module": "kibana", "metricsets": []string{metricset}, - "hosts": []string{net.JoinHostPort(GetEnvHost(), GetEnvPort())}, + "hosts": []string{host}, } } diff --git a/metricbeat/module/kibana/stats/stats_integration_test.go b/metricbeat/module/kibana/stats/stats_integration_test.go index f8cd7b9f528..eac22f8fc3b 100644 --- a/metricbeat/module/kibana/stats/stats_integration_test.go +++ b/metricbeat/module/kibana/stats/stats_integration_test.go @@ -35,12 +35,16 @@ import ( "github.com/elastic/beats/metricbeat/module/kibana/mtest" ) -func TestFetch(t *testing.T) { - compose.EnsureUpWithTimeout(t, 600, "elasticsearch", "kibana") +func TestStat(t *testing.T) { + mtest.Runner.Run(t, compose.Suite{ + "Fetch": testFetch, + "Data": testData, + }) +} - config := mtest.GetConfig("stats") - host := config["hosts"].([]string)[0] - version, err := getKibanaVersion(host) +func testFetch(t *testing.T, r compose.R) { + config := mtest.GetConfig("stats", r.Host()) + version, err := getKibanaVersion(r.Host()) if err != nil { t.Fatal("getting kibana version", err) } @@ -66,12 +70,9 @@ func TestFetch(t *testing.T) { events[0].BeatEvent("kibana", "stats").Fields.StringToPrint()) } -func TestData(t *testing.T) { - compose.EnsureUp(t, "kibana") - - config := mtest.GetConfig("stats") - host := config["hosts"].([]string)[0] - version, err := getKibanaVersion(host) +func testData(t *testing.T, r compose.R) { + config := mtest.GetConfig("stats", r.Host()) + version, err := getKibanaVersion(r.Host()) if err != nil { t.Fatal("getting kibana version", err) } diff --git a/metricbeat/module/kibana/status/status_integration_test.go b/metricbeat/module/kibana/status/status_integration_test.go index a358b27b3ed..f8dcb36fef8 100644 --- a/metricbeat/module/kibana/status/status_integration_test.go +++ b/metricbeat/module/kibana/status/status_integration_test.go @@ -29,10 +29,15 @@ import ( "github.com/elastic/beats/metricbeat/module/kibana/mtest" ) -func TestFetch(t *testing.T) { - compose.EnsureUpWithTimeout(t, 600, "elasticsearch", "kibana") +func TestStatus(t *testing.T) { + mtest.Runner.Run(t, compose.Suite{ + "Fetch": testFetch, + "Data": testData, + }) +} - f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("status")) +func testFetch(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("status", r.Host())) events, errs := mbtest.ReportingFetchV2(f) assert.Empty(t, errs) @@ -43,3 +48,12 @@ func TestFetch(t *testing.T) { t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events[0].BeatEvent("kibana", "status").Fields.StringToPrint()) } + +func testData(t *testing.T, r compose.R) { + config := mtest.GetConfig("status", r.Host()) + f := mbtest.NewReportingMetricSetV2(t, config) + err := mbtest.WriteEventsReporterV2(f, t, "") + if err != nil { + t.Fatal("write", err) + } +} diff --git a/metricbeat/module/kubernetes/_meta/env b/metricbeat/module/kubernetes/_meta/env deleted file mode 100644 index 03cff050efa..00000000000 --- a/metricbeat/module/kubernetes/_meta/env +++ /dev/null @@ -1,4 +0,0 @@ -KUBELET_HOST=172.17.0.1 -KUBELET_PORT=10255 -KUBE_STATE_METRICS_HOST=kubestate -KUBE_STATE_METRICS_PORT=8080 diff --git a/metricbeat/module/logstash/_meta/env b/metricbeat/module/logstash/_meta/env deleted file mode 100644 index 57f4dc268f7..00000000000 --- a/metricbeat/module/logstash/_meta/env +++ /dev/null @@ -1,2 +0,0 @@ -LOGSTASH_HOST=logstash -LOGSTASH_PORT=9600 diff --git a/metricbeat/module/logstash/logstash_integration_test.go b/metricbeat/module/logstash/logstash_integration_test.go index 6987b44535c..1e0e8a9fe8a 100644 --- a/metricbeat/module/logstash/logstash_integration_test.go +++ b/metricbeat/module/logstash/logstash_integration_test.go @@ -36,11 +36,16 @@ var metricSets = []string{ "node_stats", } -func TestFetch(t *testing.T) { - compose.EnsureUp(t, "logstash") +func TestLogstash(t *testing.T) { + logstash.Runner.Run(t, compose.Suite{ + "Fetch": testFetch, + "Data": testData, + }) +} +func testFetch(t *testing.T, r compose.R) { for _, metricSet := range metricSets { - f := mbtest.NewReportingMetricSetV2(t, logstash.GetConfig(metricSet)) + f := mbtest.NewReportingMetricSetV2(t, logstash.GetConfig(metricSet, r.Host())) events, errs := mbtest.ReportingFetchV2(f) assert.Empty(t, errs) @@ -53,13 +58,11 @@ func TestFetch(t *testing.T) { } } -func TestData(t *testing.T) { - compose.EnsureUp(t, "logstash") - +func testData(t *testing.T, r compose.R) { for _, metricSet := range metricSets { - config := logstash.GetConfig(metricSet) + config := logstash.GetConfig(metricSet, r.Host()) f := mbtest.NewReportingMetricSetV2(t, config) - err := mbtest.WriteEventsReporterV2(f, t, metricSet) + err := mbtest.WriteEventsReporterV2(f, t, "") if err != nil { t.Fatal("write", err) } diff --git a/metricbeat/module/logstash/testing.go b/metricbeat/module/logstash/testing.go index 0f76fada202..01055e93181 100644 --- a/metricbeat/module/logstash/testing.go +++ b/metricbeat/module/logstash/testing.go @@ -17,33 +17,23 @@ package logstash -import "os" +import ( + "github.com/elastic/beats/libbeat/tests/compose" +) -// GetEnvHost for Logstash -func GetEnvHost() string { - host := os.Getenv("LOGSTASH_HOST") - - if len(host) == 0 { - host = "127.0.0.1" - } - return host -} - -// GetEnvPort for Logstash -func GetEnvPort() string { - port := os.Getenv("LOGSTASH_PORT") - - if len(port) == 0 { - port = "9600" +var ( + // Runner is the compose test runner for logstash + Runner = compose.TestRunner{ + Service: "logstash", + Parallel: true, } - return port -} +) // GetConfig for Logstash -func GetConfig(metricset string) map[string]interface{} { +func GetConfig(metricset string, host string) map[string]interface{} { return map[string]interface{}{ "module": ModuleName, "metricsets": []string{metricset}, - "hosts": []string{GetEnvHost() + ":" + GetEnvPort()}, + "hosts": []string{host}, } } diff --git a/metricbeat/module/memcached/_meta/env b/metricbeat/module/memcached/_meta/env deleted file mode 100644 index 3efa58cda16..00000000000 --- a/metricbeat/module/memcached/_meta/env +++ /dev/null @@ -1,2 +0,0 @@ -MEMCACHED_HOST=memcached -MEMCACHED_PORT=11211 diff --git a/metricbeat/module/memcached/stats/stats_integration_test.go b/metricbeat/module/memcached/stats/stats_integration_test.go index 40a0a1e6ffc..d6545d5316d 100644 --- a/metricbeat/module/memcached/stats/stats_integration_test.go +++ b/metricbeat/module/memcached/stats/stats_integration_test.go @@ -20,7 +20,6 @@ package stats import ( - "os" "testing" "github.com/stretchr/testify/assert" @@ -29,25 +28,23 @@ import ( mbtest "github.com/elastic/beats/metricbeat/mb/testing" ) -func TestData(t *testing.T) { - compose.EnsureUp(t, "memcached") - - f := mbtest.NewReportingMetricSetV2(t, getConfig()) - events, errs := mbtest.ReportingFetchV2(f) - if len(errs) > 0 { - t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) - } - assert.NotEmpty(t, events) +func TestStats(t *testing.T) { + runner := compose.TestRunner{Service: "memcached"} + runner.Run(t, compose.Suite{ + "Data": testData, + "Fetch": testFetch, + }) +} +func testData(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host())) if err := mbtest.WriteEventsReporterV2(f, t, ""); err != nil { t.Fatal("write", err) } } -func TestFetch(t *testing.T) { - compose.EnsureUp(t, "memcached") - - f := mbtest.NewReportingMetricSetV2(t, getConfig()) +func testFetch(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host())) events, errs := mbtest.ReportingFetchV2(f) if len(errs) > 0 { t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) @@ -58,28 +55,10 @@ func TestFetch(t *testing.T) { t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event) } -func getConfig() map[string]interface{} { +func getConfig(host string) map[string]interface{} { return map[string]interface{}{ "module": "memcached", "metricsets": []string{"stats"}, - "hosts": []string{getEnvHost() + ":" + getEnvPort()}, - } -} - -func getEnvHost() string { - host := os.Getenv("MEMCACHED_HOST") - - if len(host) == 0 { - host = "127.0.0.1" - } - return host -} - -func getEnvPort() string { - port := os.Getenv("MEMCACHED_PORT") - - if len(port) == 0 { - port = "11211" + "hosts": []string{host}, } - return port } diff --git a/metricbeat/module/mongodb/_meta/env b/metricbeat/module/mongodb/_meta/env deleted file mode 100644 index a51807cbe81..00000000000 --- a/metricbeat/module/mongodb/_meta/env +++ /dev/null @@ -1,2 +0,0 @@ -MONGODB_HOST=mongodb -MONGODB_PORT=27017 diff --git a/metricbeat/module/mongodb/collstats/collstats_integration_test.go b/metricbeat/module/mongodb/collstats/collstats_integration_test.go index dc828330f5a..15010fc9ce7 100644 --- a/metricbeat/module/mongodb/collstats/collstats_integration_test.go +++ b/metricbeat/module/mongodb/collstats/collstats_integration_test.go @@ -26,13 +26,18 @@ import ( "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" - "github.com/elastic/beats/metricbeat/module/mongodb" + "github.com/elastic/beats/metricbeat/module/mongodb/mtest" ) -func TestFetch(t *testing.T) { - compose.EnsureUp(t, "mongodb") +func TestCollstats(t *testing.T) { + mtest.Runner.Run(t, compose.Suite{ + "Fetch": testFetch, + "Data": testData, + }) +} - f := mbtest.NewReportingMetricSetV2(t, getConfig()) +func testFetch(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("collstats", r.Host())) events, errs := mbtest.ReportingFetchV2(f) if len(errs) > 0 { t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) @@ -52,25 +57,9 @@ func TestFetch(t *testing.T) { } } -func TestData(t *testing.T) { - compose.EnsureUp(t, "mongodb") - - f := mbtest.NewReportingMetricSetV2(t, getConfig()) - events, errs := mbtest.ReportingFetchV2(f) - if len(errs) > 0 { - t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) - } - assert.NotEmpty(t, events) - +func testData(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("collstats", r.Host())) if err := mbtest.WriteEventsReporterV2(f, t, ""); err != nil { t.Fatal("write", err) } } - -func getConfig() map[string]interface{} { - return map[string]interface{}{ - "module": "mongodb", - "metricsets": []string{"collstats"}, - "hosts": []string{mongodb.GetEnvHost() + ":" + mongodb.GetEnvPort()}, - } -} diff --git a/metricbeat/module/mongodb/dbstats/dbstats_integration_test.go b/metricbeat/module/mongodb/dbstats/dbstats_integration_test.go index 9ef22d6a700..2c35f3ba038 100644 --- a/metricbeat/module/mongodb/dbstats/dbstats_integration_test.go +++ b/metricbeat/module/mongodb/dbstats/dbstats_integration_test.go @@ -26,13 +26,18 @@ import ( "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" - "github.com/elastic/beats/metricbeat/module/mongodb" + "github.com/elastic/beats/metricbeat/module/mongodb/mtest" ) -func TestFetch(t *testing.T) { - compose.EnsureUp(t, "mongodb") +func TestDBStats(t *testing.T) { + mtest.Runner.Run(t, compose.Suite{ + "Fetch": testFetch, + "Data": testData, + }) +} - f := mbtest.NewReportingMetricSetV2(t, getConfig()) +func testFetch(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("dbstats", r.Host())) events, errs := mbtest.ReportingFetchV2(f) if len(errs) > 0 { t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) @@ -77,25 +82,9 @@ func TestFetch(t *testing.T) { } } -func TestData(t *testing.T) { - compose.EnsureUp(t, "mongodb") - - f := mbtest.NewReportingMetricSetV2(t, getConfig()) - events, errs := mbtest.ReportingFetchV2(f) - if len(errs) > 0 { - t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) - } - assert.NotEmpty(t, events) - +func testData(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("dbstats", r.Host())) if err := mbtest.WriteEventsReporterV2(f, t, ""); err != nil { t.Fatal("write", err) } } - -func getConfig() map[string]interface{} { - return map[string]interface{}{ - "module": "mongodb", - "metricsets": []string{"dbstats"}, - "hosts": []string{mongodb.GetEnvHost() + ":" + mongodb.GetEnvPort()}, - } -} diff --git a/metricbeat/module/mongodb/metrics/metrics_intergration_test.go b/metricbeat/module/mongodb/metrics/metrics_intergration_test.go index f59cb703b28..bffda71fef3 100644 --- a/metricbeat/module/mongodb/metrics/metrics_intergration_test.go +++ b/metricbeat/module/mongodb/metrics/metrics_intergration_test.go @@ -26,13 +26,18 @@ import ( "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" - "github.com/elastic/beats/metricbeat/module/mongodb" + "github.com/elastic/beats/metricbeat/module/mongodb/mtest" ) -func TestFetch(t *testing.T) { - compose.EnsureUp(t, "mongodb") +func TestMetrics(t *testing.T) { + mtest.Runner.Run(t, compose.Suite{ + "Fetch": testFetch, + "Data": testData, + }) +} - f := mbtest.NewReportingMetricSetV2(t, getConfig()) +func testFetch(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("metrics", r.Host())) events, errs := mbtest.ReportingFetchV2(f) if len(errs) > 0 { t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) @@ -50,25 +55,9 @@ func TestFetch(t *testing.T) { assert.True(t, deletedDocuments.(int64) >= 0) } -func TestData(t *testing.T) { - compose.EnsureUp(t, "mongodb") - - f := mbtest.NewReportingMetricSetV2(t, getConfig()) - events, errs := mbtest.ReportingFetchV2(f) - if len(errs) > 0 { - t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) - } - assert.NotEmpty(t, events) - +func testData(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("metrics", r.Host())) if err := mbtest.WriteEventsReporterV2(f, t, ""); err != nil { t.Fatal("write", err) } } - -func getConfig() map[string]interface{} { - return map[string]interface{}{ - "module": "mongodb", - "metricsets": []string{"metrics"}, - "hosts": []string{mongodb.GetEnvHost() + ":" + mongodb.GetEnvPort()}, - } -} diff --git a/metricbeat/module/uwsgi/testing.go b/metricbeat/module/mongodb/mtest/testing.go similarity index 64% rename from metricbeat/module/uwsgi/testing.go rename to metricbeat/module/mongodb/mtest/testing.go index b8c465354e6..f1657d1f161 100644 --- a/metricbeat/module/uwsgi/testing.go +++ b/metricbeat/module/mongodb/mtest/testing.go @@ -15,24 +15,23 @@ // specific language governing permissions and limitations // under the License. -package uwsgi +package mtest -import "os" +import ( + "github.com/elastic/beats/libbeat/tests/compose" +) -// GetEnvTCPServer returns uwsgi stat server host with tcp mode -func GetEnvTCPServer() string { - env := os.Getenv("UWSGI_STAT_TCP_SERVER") - if len(env) == 0 { - env = "tcp://127.0.0.1:9191" - } - return env +// Runner is the compose test runner for mongodb +var Runner = compose.TestRunner{ + Service: "mongodb", + Parallel: true, } -// GetEnvHTTPServer returns uwsgi stat server host with http mode -func GetEnvHTTPServer() string { - env := os.Getenv("UWSGI_STAT_HTTP_SERVER") - if len(env) == 0 { - env = "http://127.0.0.1:9192" +// GetConfig creates a config for a metricset and host +func GetConfig(metricset, host string) map[string]interface{} { + return map[string]interface{}{ + "module": "mongodb", + "metricsets": []string{metricset}, + "hosts": []string{host}, } - return env } diff --git a/metricbeat/module/mongodb/replstatus/replstatus_integration_test.go b/metricbeat/module/mongodb/replstatus/replstatus_integration_test.go index 9af662c8dde..145eb06b4c6 100644 --- a/metricbeat/module/mongodb/replstatus/replstatus_integration_test.go +++ b/metricbeat/module/mongodb/replstatus/replstatus_integration_test.go @@ -31,17 +31,23 @@ import ( "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" "github.com/elastic/beats/metricbeat/module/mongodb" + "github.com/elastic/beats/metricbeat/module/mongodb/mtest" ) -func TestFetch(t *testing.T) { - compose.EnsureUp(t, "mongodb") +func TestReplStatus(t *testing.T) { + mtest.Runner.Run(t, compose.Suite{ + "Fetch": testFetch, + "Data": testData, + }) +} - err := initiateReplicaSet(t) +func testFetch(t *testing.T, r compose.R) { + err := initiateReplicaSet(t, r.Host()) if !assert.NoError(t, err) { t.FailNow() } - f := mbtest.NewReportingMetricSetV2(t, getConfig()) + f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("replstatus", r.Host())) events, errs := mbtest.ReportingFetchV2(f) if len(errs) > 0 { t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) @@ -74,32 +80,13 @@ func TestFetch(t *testing.T) { assert.Equal(t, set, "beats") } -func TestData(t *testing.T) { - compose.EnsureUp(t, "mongodb") - - f := mbtest.NewReportingMetricSetV2(t, getConfig()) - events, errs := mbtest.ReportingFetchV2(f) - if len(errs) > 0 { - t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) - } - assert.NotEmpty(t, events) - +func testData(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("replstatus", r.Host())) if err := mbtest.WriteEventsReporterV2(f, t, ""); err != nil { t.Fatal("write", err) } } - -func getConfig() map[string]interface{} { - return map[string]interface{}{ - "module": "mongodb", - "metricsets": []string{"replstatus"}, - "hosts": []string{mongodb.GetEnvHost() + ":" + mongodb.GetEnvPort()}, - } -} - -func initiateReplicaSet(t *testing.T) error { - url := getConfig()["hosts"].([]string)[0] - +func initiateReplicaSet(t *testing.T, url string) error { dialInfo, err := mgo.ParseURL(url) if err != nil { return err diff --git a/metricbeat/module/mongodb/status/status_integration_test.go b/metricbeat/module/mongodb/status/status_integration_test.go index 83ce4046829..f7457e54609 100644 --- a/metricbeat/module/mongodb/status/status_integration_test.go +++ b/metricbeat/module/mongodb/status/status_integration_test.go @@ -26,13 +26,18 @@ import ( "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" - "github.com/elastic/beats/metricbeat/module/mongodb" + "github.com/elastic/beats/metricbeat/module/mongodb/mtest" ) -func TestFetch(t *testing.T) { - compose.EnsureUp(t, "mongodb") +func TestStatus(t *testing.T) { + mtest.Runner.Run(t, compose.Suite{ + "Fetch": testFetch, + "Data": testData, + }) +} - f := mbtest.NewReportingMetricSetV2(t, getConfig()) +func testFetch(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("status", r.Host())) events, errs := mbtest.ReportingFetchV2(f) assert.Empty(t, errs) @@ -56,22 +61,11 @@ func TestFetch(t *testing.T) { assert.True(t, pageFaults.(int64) >= 0) } -func TestData(t *testing.T) { - compose.EnsureUp(t, "mongodb") - - config := getConfig() - f := mbtest.NewReportingMetricSetV2(t, config) +func testData(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("status", r.Host())) err := mbtest.WriteEventsReporterV2(f, t, ".") if err != nil { t.Fatal("write", err) } } - -func getConfig() map[string]interface{} { - return map[string]interface{}{ - "module": "mongodb", - "metricsets": []string{"status"}, - "hosts": []string{mongodb.GetEnvHost() + ":" + mongodb.GetEnvPort()}, - } -} diff --git a/metricbeat/module/mongodb/testing.go b/metricbeat/module/mongodb/testing.go deleted file mode 100644 index 31d380b8322..00000000000 --- a/metricbeat/module/mongodb/testing.go +++ /dev/null @@ -1,46 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package mongodb - -import "os" - -// Helper functions for testing mongodb metricsets. - -// GetEnvHost returns the hostname of the Mongodb server to use for testing. -// It reads the value from the MONGODB_HOST environment variable and returns -// 127.0.0.1 if it is not set. -func GetEnvHost() string { - host := os.Getenv("MONGODB_HOST") - - if len(host) == 0 { - host = "127.0.0.1" - } - return host -} - -// GetEnvPort returns the port of the Mongodb server to use for testing. -// It reads the value from the MONGODB_PORT environment variable and returns -// 27017 if it is not set. -func GetEnvPort() string { - port := os.Getenv("MONGODB_PORT") - - if len(port) == 0 { - port = "27017" - } - return port -} diff --git a/metricbeat/module/munin/_meta/env b/metricbeat/module/munin/_meta/env deleted file mode 100644 index b81c5ee8679..00000000000 --- a/metricbeat/module/munin/_meta/env +++ /dev/null @@ -1,2 +0,0 @@ -MUNIN_HOST=munin -MUNIN_PORT=4949 diff --git a/metricbeat/module/munin/node/node_integration_test.go b/metricbeat/module/munin/node/node_integration_test.go index 1e8fbbe2fd6..3377266c921 100644 --- a/metricbeat/module/munin/node/node_integration_test.go +++ b/metricbeat/module/munin/node/node_integration_test.go @@ -20,7 +20,6 @@ package node import ( - "os" "testing" "github.com/stretchr/testify/assert" @@ -29,10 +28,16 @@ import ( mbtest "github.com/elastic/beats/metricbeat/mb/testing" ) -func TestFetch(t *testing.T) { - compose.EnsureUp(t, "munin") +func TestNode(t *testing.T) { + runner := compose.TestRunner{Service: "munin"} + runner.Run(t, compose.Suite{ + "Fetch": testFetch, + "Data": testData, + }) +} - f := mbtest.NewReportingMetricSetV2(t, getConfig()) +func testFetch(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host())) events, errs := mbtest.ReportingFetchV2(f) assert.Empty(t, errs) @@ -44,10 +49,8 @@ func TestFetch(t *testing.T) { events[0].BeatEvent("munin", "node").Fields.StringToPrint()) } -func TestData(t *testing.T) { - compose.EnsureUp(t, "munin") - - config := getConfig() +func testData(t *testing.T, r compose.R) { + config := getConfig(r.Host()) f := mbtest.NewReportingMetricSetV2(t, config) err := mbtest.WriteEventsReporterV2(f, t, ".") if err != nil { @@ -55,34 +58,10 @@ func TestData(t *testing.T) { } } -func getConfig() map[string]interface{} { +func getConfig(host string) map[string]interface{} { return map[string]interface{}{ "module": "munin", "metricsets": []string{"node"}, - "hosts": []string{GetEnvHost() + ":" + GetEnvPort()}, - } -} - -// GetEnvHost returns the hostname of the Mongodb server to use for testing. -// It reads the value from the MONGODB_HOST environment variable and returns -// 127.0.0.1 if it is not set. -func GetEnvHost() string { - host := os.Getenv("MUNIN_HOST") - - if len(host) == 0 { - host = "127.0.0.1" - } - return host -} - -// GetEnvPort returns the port of the Mongodb server to use for testing. -// It reads the value from the MONGODB_PORT environment variable and returns -// 27017 if it is not set. -func GetEnvPort() string { - port := os.Getenv("MUNIN_PORT") - - if len(port) == 0 { - port = "4949" + "hosts": []string{host}, } - return port } diff --git a/metricbeat/module/mysql/_meta/env b/metricbeat/module/mysql/_meta/env deleted file mode 100644 index e6aa44eaf4a..00000000000 --- a/metricbeat/module/mysql/_meta/env +++ /dev/null @@ -1,3 +0,0 @@ -MYSQL_DSN=root:test@tcp(mysql:3306)/ -MYSQL_HOST=mysql -MYSQL_PORT=3306 diff --git a/metricbeat/module/mysql/mtest/testing.go b/metricbeat/module/mysql/mtest/testing.go new file mode 100644 index 00000000000..1e6f585b1de --- /dev/null +++ b/metricbeat/module/mysql/mtest/testing.go @@ -0,0 +1,55 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mtest + +import ( + "github.com/go-sql-driver/mysql" + + "github.com/elastic/beats/libbeat/tests/compose" +) + +// Runner is a compose test runner for mysql +var Runner = compose.TestRunner{ + Service: "mysql", + Parallel: true, +} + +// GetDSN returns the MySQL server DSN to use for testing. +func GetDSN(host string) string { + c := &mysql.Config{ + Net: "tcp", + Addr: host, + User: "root", + Passwd: "test", + + // Required if password is set and FormatDSN() is used + // since clients for MySQL 8.0 + AllowNativePasswords: true, + } + return c.FormatDSN() +} + +// GetConfig returns the configuration for a mysql module +func GetConfig(metricset, host string, raw bool) map[string]interface{} { + return map[string]interface{}{ + "module": "mysql", + "metricsets": []string{metricset}, + "hosts": []string{GetDSN(host)}, + "raw": raw, + } +} diff --git a/metricbeat/module/mysql/mysql_integration_test.go b/metricbeat/module/mysql/mysql_integration_test.go index 772fd91ae80..47217fae02e 100644 --- a/metricbeat/module/mysql/mysql_integration_test.go +++ b/metricbeat/module/mysql/mysql_integration_test.go @@ -25,13 +25,17 @@ import ( "github.com/stretchr/testify/assert" "github.com/elastic/beats/libbeat/tests/compose" - _ "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/elastic/beats/metricbeat/module/mysql/mtest" ) func TestNewDB(t *testing.T) { - compose.EnsureUp(t, "mysql") + mtest.Runner.Run(t, compose.Suite{ + "NewDB": testNewDB, + }) +} - db, err := NewDB(GetMySQLEnvDSN()) +func testNewDB(t *testing.T, r compose.R) { + db, err := NewDB(mtest.GetDSN(r.Host())) assert.NoError(t, err) err = db.Ping() diff --git a/metricbeat/module/mysql/status/status_integration_test.go b/metricbeat/module/mysql/status/status_integration_test.go index 7ccc0a77ef5..a6b72b3488d 100644 --- a/metricbeat/module/mysql/status/status_integration_test.go +++ b/metricbeat/module/mysql/status/status_integration_test.go @@ -25,15 +25,21 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" - "github.com/elastic/beats/metricbeat/module/mysql" + "github.com/elastic/beats/metricbeat/module/mysql/mtest" "github.com/stretchr/testify/assert" ) -func TestFetch(t *testing.T) { - compose.EnsureUp(t, "mysql") +func TestStatus(t *testing.T) { + mtest.Runner.Run(t, compose.Suite{ + "Fetch": testFetch, + "FetchRaw": testFetchRaw, + "Data": testData, + }) +} - f := mbtest.NewEventFetcher(t, getConfig(false)) +func testFetch(t *testing.T, r compose.R) { + f := mbtest.NewEventFetcher(t, mtest.GetConfig("status", r.Host(), false)) event, err := f.Fetch() if !assert.NoError(t, err) { t.FailNow() @@ -54,10 +60,8 @@ func TestFetch(t *testing.T) { assert.True(t, openStreams == 0) } -func TestFetchRaw(t *testing.T) { - compose.EnsureUp(t, "mysql") - - f := mbtest.NewEventFetcher(t, getConfig(true)) +func testFetchRaw(t *testing.T, r compose.R) { + f := mbtest.NewEventFetcher(t, mtest.GetConfig("status", r.Host(), true)) event, err := f.Fetch() if !assert.NoError(t, err) { t.FailNow() @@ -80,20 +84,11 @@ func TestFetchRaw(t *testing.T) { assert.True(t, exists) } -func TestData(t *testing.T) { - f := mbtest.NewEventFetcher(t, getConfig(false)) +func testData(t *testing.T, r compose.R) { + f := mbtest.NewEventFetcher(t, mtest.GetConfig("status", r.Host(), false)) err := mbtest.WriteEvent(f, t) if err != nil { t.Fatal("write", err) } } - -func getConfig(raw bool) map[string]interface{} { - return map[string]interface{}{ - "module": "mysql", - "metricsets": []string{"status"}, - "hosts": []string{mysql.GetMySQLEnvDSN()}, - "raw": raw, - } -} diff --git a/metricbeat/module/nats/_meta/env b/metricbeat/module/nats/_meta/env deleted file mode 100644 index 69e439d0a4f..00000000000 --- a/metricbeat/module/nats/_meta/env +++ /dev/null @@ -1,2 +0,0 @@ -NATS_HOST=nats -NATS_PORT=8222 diff --git a/metricbeat/module/nats/connections/connections_integration_test.go b/metricbeat/module/nats/connections/connections_integration_test.go index c5bc1bbd095..c15c3603808 100644 --- a/metricbeat/module/nats/connections/connections_integration_test.go +++ b/metricbeat/module/nats/connections/connections_integration_test.go @@ -20,57 +20,42 @@ package connections import ( - "os" "testing" "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/elastic/beats/metricbeat/module/nats/mtest" ) -func TestData(t *testing.T) { - compose.EnsureUp(t, "nats") +func TestConnections(t *testing.T) { + mtest.Runner.Run(t, compose.Suite{ + "Fetch": testFetch, + "Data": testData, + }) +} - metricSet := mbtest.NewReportingMetricSetV2(t, getConfig()) +func testData(t *testing.T, r compose.R) { + metricSet := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host())) err := mbtest.WriteEventsReporterV2(metricSet, t, "./test_data.json") if err != nil { t.Fatal("write", err) } } -func TestFetch(t *testing.T) { - compose.EnsureUp(t, "nats") - +func testFetch(t *testing.T, r compose.R) { reporter := &mbtest.CapturingReporterV2{} - metricSet := mbtest.NewReportingMetricSetV2(t, getConfig()) + metricSet := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host())) metricSet.Fetch(reporter) e := mbtest.StandardizeEvent(metricSet, reporter.GetEvents()[0]) t.Logf("%s/%s event: %+v", metricSet.Module().Name(), metricSet.Name(), e.Fields.StringToPrint()) } -func getConfig() map[string]interface{} { +func getConfig(host string) map[string]interface{} { return map[string]interface{}{ "module": "nats", "metricsets": []string{"connections"}, - "hosts": []string{GetEnvHost() + ":" + GetEnvPort()}, - } -} - -func GetEnvHost() string { - host := os.Getenv("NATS_HOST") - - if len(host) == 0 { - host = "127.0.0.1" - } - return host -} - -func GetEnvPort() string { - port := os.Getenv("NATS_PORT") - - if len(port) == 0 { - port = "8222" + "hosts": []string{host}, } - return port } diff --git a/metricbeat/module/nginx/testing.go b/metricbeat/module/nats/mtest/runner.go similarity index 79% rename from metricbeat/module/nginx/testing.go rename to metricbeat/module/nats/mtest/runner.go index cee26e1d11a..25174c34361 100644 --- a/metricbeat/module/nginx/testing.go +++ b/metricbeat/module/nats/mtest/runner.go @@ -15,21 +15,17 @@ // specific language governing permissions and limitations // under the License. -package nginx +// +build integration -/* -Helper functions for testing used in the nginx metricsets -*/ +package mtest import ( - "os" + "github.com/elastic/beats/libbeat/tests/compose" ) -func GetNginxEnvHost() string { - host := os.Getenv("NGINX_HOST") - - if len(host) == 0 { - host = "127.0.0.1" +var ( + // Runner is a compose test runner for Nats tests + Runner = compose.TestRunner{ + Service: "nats", } - return host -} +) diff --git a/metricbeat/module/nats/routes/routes_integration_test.go b/metricbeat/module/nats/routes/routes_integration_test.go index 8481fc3a49c..57e1f8213c5 100644 --- a/metricbeat/module/nats/routes/routes_integration_test.go +++ b/metricbeat/module/nats/routes/routes_integration_test.go @@ -20,57 +20,42 @@ package routes import ( - "os" "testing" "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" + mtest "github.com/elastic/beats/metricbeat/module/nats/mtest" ) -func TestData(t *testing.T) { - compose.EnsureUp(t, "nats") +func TestRoutes(t *testing.T) { + mtest.Runner.Run(t, compose.Suite{ + "Fetch": testFetch, + "Data": testData, + }) +} - metricSet := mbtest.NewReportingMetricSetV2(t, getConfig()) +func testData(t *testing.T, r compose.R) { + metricSet := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host())) err := mbtest.WriteEventsReporterV2(metricSet, t, "./test_data.json") if err != nil { t.Fatal("write", err) } } -func TestFetch(t *testing.T) { - compose.EnsureUp(t, "nats") - +func testFetch(t *testing.T, r compose.R) { reporter := &mbtest.CapturingReporterV2{} - metricSet := mbtest.NewReportingMetricSetV2(t, getConfig()) + metricSet := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host())) metricSet.Fetch(reporter) e := mbtest.StandardizeEvent(metricSet, reporter.GetEvents()[0]) t.Logf("%s/%s event: %+v", metricSet.Module().Name(), metricSet.Name(), e.Fields.StringToPrint()) } -func getConfig() map[string]interface{} { +func getConfig(host string) map[string]interface{} { return map[string]interface{}{ "module": "nats", "metricsets": []string{"routes"}, - "hosts": []string{GetEnvHost() + ":" + GetEnvPort()}, - } -} - -func GetEnvHost() string { - host := os.Getenv("NATS_HOST") - - if len(host) == 0 { - host = "127.0.0.1" - } - return host -} - -func GetEnvPort() string { - port := os.Getenv("NATS_PORT") - - if len(port) == 0 { - port = "8222" + "hosts": []string{host}, } - return port } diff --git a/metricbeat/module/nats/stats/stats_integration_test.go b/metricbeat/module/nats/stats/stats_integration_test.go index 171f7b7904f..db82d381f19 100644 --- a/metricbeat/module/nats/stats/stats_integration_test.go +++ b/metricbeat/module/nats/stats/stats_integration_test.go @@ -20,57 +20,42 @@ package stats import ( - "os" "testing" "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/elastic/beats/metricbeat/module/nats/mtest" ) -func TestData(t *testing.T) { - compose.EnsureUp(t, "nats") +func TestStats(t *testing.T) { + mtest.Runner.Run(t, compose.Suite{ + "Fetch": testFetch, + "Data": testData, + }) +} - metricSet := mbtest.NewReportingMetricSetV2(t, getConfig()) +func testData(t *testing.T, r compose.R) { + metricSet := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host())) err := mbtest.WriteEventsReporterV2(metricSet, t, "./test_data.json") if err != nil { t.Fatal("write", err) } } -func TestFetch(t *testing.T) { - compose.EnsureUp(t, "nats") - +func testFetch(t *testing.T, r compose.R) { reporter := &mbtest.CapturingReporterV2{} - metricSet := mbtest.NewReportingMetricSetV2(t, getConfig()) + metricSet := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host())) metricSet.Fetch(reporter) e := mbtest.StandardizeEvent(metricSet, reporter.GetEvents()[0]) t.Logf("%s/%s event: %+v", metricSet.Module().Name(), metricSet.Name(), e.Fields.StringToPrint()) } -func getConfig() map[string]interface{} { +func getConfig(host string) map[string]interface{} { return map[string]interface{}{ "module": "nats", "metricsets": []string{"stats"}, - "hosts": []string{GetEnvHost() + ":" + GetEnvPort()}, - } -} - -func GetEnvHost() string { - host := os.Getenv("NATS_HOST") - - if len(host) == 0 { - host = "127.0.0.1" - } - return host -} - -func GetEnvPort() string { - port := os.Getenv("NATS_PORT") - - if len(port) == 0 { - port = "8222" + "hosts": []string{host}, } - return port } diff --git a/metricbeat/module/nats/subscriptions/subscriptions_integration_test.go b/metricbeat/module/nats/subscriptions/subscriptions_integration_test.go index 6d8c6f1d4cf..dfe0ab2155e 100644 --- a/metricbeat/module/nats/subscriptions/subscriptions_integration_test.go +++ b/metricbeat/module/nats/subscriptions/subscriptions_integration_test.go @@ -20,57 +20,42 @@ package subscriptions import ( - "os" "testing" "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" ) -func TestData(t *testing.T) { - compose.EnsureUp(t, "nats") +func TestSubscriptions(t *testing.T) { + runner := compose.TestRunner{Service: "nats"} + runner.Run(t, compose.Suite{ + "Data": testData, + "Fetch": testFetch, + }) +} - metricSet := mbtest.NewReportingMetricSetV2(t, getConfig()) +func testData(t *testing.T, r compose.R) { + metricSet := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host())) err := mbtest.WriteEventsReporterV2(metricSet, t, "./test_data.json") if err != nil { t.Fatal("write", err) } } -func TestFetch(t *testing.T) { - compose.EnsureUp(t, "nats") - +func testFetch(t *testing.T, r compose.R) { reporter := &mbtest.CapturingReporterV2{} - metricSet := mbtest.NewReportingMetricSetV2(t, getConfig()) + metricSet := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host())) metricSet.Fetch(reporter) e := mbtest.StandardizeEvent(metricSet, reporter.GetEvents()[0]) t.Logf("%s/%s event: %+v", metricSet.Module().Name(), metricSet.Name(), e.Fields.StringToPrint()) } -func getConfig() map[string]interface{} { +func getConfig(host string) map[string]interface{} { return map[string]interface{}{ "module": "nats", "metricsets": []string{"subscriptions"}, - "hosts": []string{GetEnvHost() + ":" + GetEnvPort()}, - } -} - -func GetEnvHost() string { - host := os.Getenv("NATS_HOST") - - if len(host) == 0 { - host = "127.0.0.1" - } - return host -} - -func GetEnvPort() string { - port := os.Getenv("NATS_PORT") - - if len(port) == 0 { - port = "8222" + "hosts": []string{host}, } - return port } diff --git a/metricbeat/module/nginx/_meta/env b/metricbeat/module/nginx/_meta/env deleted file mode 100644 index 693e4c78989..00000000000 --- a/metricbeat/module/nginx/_meta/env +++ /dev/null @@ -1,2 +0,0 @@ -NGINX_HOST=nginx -NGINX_PORT=80 diff --git a/metricbeat/module/nginx/stubstatus/stubstatus_integration_test.go b/metricbeat/module/nginx/stubstatus/stubstatus_integration_test.go index d4b12887003..7560df4af2a 100644 --- a/metricbeat/module/nginx/stubstatus/stubstatus_integration_test.go +++ b/metricbeat/module/nginx/stubstatus/stubstatus_integration_test.go @@ -24,15 +24,24 @@ import ( "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" - "github.com/elastic/beats/metricbeat/module/nginx" "github.com/stretchr/testify/assert" ) -func TestFetch(t *testing.T) { - compose.EnsureUp(t, "nginx") +func TestStubstatus(t *testing.T) { + runner := compose.TestRunner{ + Service: "nginx", + Parallel: true, + } + + runner.Run(t, compose.Suite{ + "Fetch": testFetch, + "Data": testData, + }) +} - f := mbtest.NewEventFetcher(t, getConfig()) +func testFetch(t *testing.T, r compose.R) { + f := mbtest.NewEventFetcher(t, getConfig(r.Host())) event, err := f.Fetch() if !assert.NoError(t, err) { t.FailNow() @@ -44,8 +53,8 @@ func TestFetch(t *testing.T) { assert.Equal(t, 10, len(event)) } -func TestData(t *testing.T) { - f := mbtest.NewEventFetcher(t, getConfig()) +func testData(t *testing.T, r compose.R) { + f := mbtest.NewEventFetcher(t, getConfig(r.Host())) err := mbtest.WriteEvent(f, t) if err != nil { @@ -53,10 +62,10 @@ func TestData(t *testing.T) { } } -func getConfig() map[string]interface{} { +func getConfig(host string) map[string]interface{} { return map[string]interface{}{ "module": "nginx", "metricsets": []string{"stubstatus"}, - "hosts": []string{nginx.GetNginxEnvHost()}, + "hosts": []string{host}, } } diff --git a/metricbeat/module/php_fpm/_meta/env b/metricbeat/module/php_fpm/_meta/env deleted file mode 100644 index 3437fa5b8c4..00000000000 --- a/metricbeat/module/php_fpm/_meta/env +++ /dev/null @@ -1,2 +0,0 @@ -PHPFPM_HOST=phpfpm -PHPFPM_PORT=81 diff --git a/metricbeat/module/php_fpm/mtest/testing.go b/metricbeat/module/php_fpm/mtest/testing.go new file mode 100644 index 00000000000..bf5cce996d3 --- /dev/null +++ b/metricbeat/module/php_fpm/mtest/testing.go @@ -0,0 +1,39 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration + +package mtest + +import ( + "github.com/elastic/beats/libbeat/tests/compose" +) + +// Runner is a compose test runner for php fpm +var Runner = compose.TestRunner{ + Service: "phpfpm", + Parallel: true, +} + +// GetConfig creates a configuration for a metricset and host +func GetConfig(metricset, host string) map[string]interface{} { + return map[string]interface{}{ + "module": "php_fpm", + "metricsets": []string{metricset}, + "hosts": []string{host}, + } +} diff --git a/metricbeat/module/php_fpm/pool/pool_integration_test.go b/metricbeat/module/php_fpm/pool/pool_integration_test.go index 9ace6ed2391..0b942fc334e 100644 --- a/metricbeat/module/php_fpm/pool/pool_integration_test.go +++ b/metricbeat/module/php_fpm/pool/pool_integration_test.go @@ -20,19 +20,24 @@ package pool import ( - "os" "testing" "github.com/stretchr/testify/assert" "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/elastic/beats/metricbeat/module/php_fpm/mtest" ) -func TestFetch(t *testing.T) { - compose.EnsureUp(t, "phpfpm") +func TestPool(t *testing.T) { + mtest.Runner.Run(t, compose.Suite{ + "Fetch": testFetch, + "Data": testData, + }) +} - f := mbtest.NewReportingMetricSetV2(t, getConfig()) +func testFetch(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("pool", r.Host())) events, errs := mbtest.ReportingFetchV2(f) assert.Empty(t, errs) @@ -42,31 +47,12 @@ func TestFetch(t *testing.T) { t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events[0].BeatEvent("haproxy", "info").Fields.StringToPrint()) - -} - -func getConfig() map[string]interface{} { - return map[string]interface{}{ - "module": "php_fpm", - "metricsets": []string{"pool"}, - "hosts": []string{GetEnvHost() + ":" + GetEnvPort()}, - } } -func GetEnvHost() string { - host := os.Getenv("PHPFPM_HOST") - - if len(host) == 0 { - host = "127.0.0.1" - } - return host -} - -func GetEnvPort() string { - port := os.Getenv("PHPFPM_PORT") - - if len(port) == 0 { - port = "81" +func testData(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("pool", r.Host())) + err := mbtest.WriteEventsReporterV2(f, t, "") + if err != nil { + t.Fatal("write", err) } - return port } diff --git a/metricbeat/module/php_fpm/process/process_integration_test.go b/metricbeat/module/php_fpm/process/process_integration_test.go index a5f7a1d8d64..d9c082d1a9c 100644 --- a/metricbeat/module/php_fpm/process/process_integration_test.go +++ b/metricbeat/module/php_fpm/process/process_integration_test.go @@ -20,19 +20,23 @@ package process import ( - "os" "testing" "github.com/stretchr/testify/assert" "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/elastic/beats/metricbeat/module/php_fpm/mtest" ) -func TestFetch(t *testing.T) { - compose.EnsureUp(t, "phpfpm") +func TestProcess(t *testing.T) { + mtest.Runner.Run(t, compose.Suite{ + "Fetch": testFetch, + }) +} - f := mbtest.NewReportingMetricSetV2Error(t, getConfig()) +func testFetch(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2Error(t, mtest.GetConfig("process", r.Host())) events, errs := mbtest.ReportingFetchV2Error(f) assert.Empty(t, errs) @@ -44,29 +48,3 @@ func TestFetch(t *testing.T) { events[0].BeatEvent("php_fpm", "process").Fields.StringToPrint()) } - -func getConfig() map[string]interface{} { - return map[string]interface{}{ - "module": "php_fpm", - "metricsets": []string{"process"}, - "hosts": []string{GetEnvHost() + ":" + GetEnvPort()}, - } -} - -func GetEnvHost() string { - host := os.Getenv("PHPFPM_HOST") - - if len(host) == 0 { - host = "127.0.0.1" - } - return host -} - -func GetEnvPort() string { - port := os.Getenv("PHPFPM_PORT") - - if len(port) == 0 { - port = "81" - } - return port -} diff --git a/metricbeat/module/postgresql/_meta/env b/metricbeat/module/postgresql/_meta/env deleted file mode 100644 index eef1c64138c..00000000000 --- a/metricbeat/module/postgresql/_meta/env +++ /dev/null @@ -1,4 +0,0 @@ -POSTGRESQL_DSN=postgres://postgresql:5432?sslmode=disable -POSTGRESQL_HOST=postgresql -POSTGRESQL_PORT=5432 -POSTGRESQL_USERNAME=postgres diff --git a/metricbeat/module/postgresql/activity/activity_integration_test.go b/metricbeat/module/postgresql/activity/activity_integration_test.go index daab8378874..ae92afcae67 100644 --- a/metricbeat/module/postgresql/activity/activity_integration_test.go +++ b/metricbeat/module/postgresql/activity/activity_integration_test.go @@ -25,15 +25,20 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" - "github.com/elastic/beats/metricbeat/module/postgresql" + "github.com/elastic/beats/metricbeat/module/postgresql/mtest" "github.com/stretchr/testify/assert" ) -func TestFetch(t *testing.T) { - compose.EnsureUp(t, "postgresql") +func TestActivity(t *testing.T) { + mtest.Runner.Run(t, compose.Suite{ + "Fetch": testFetch, + "Data": testData, + }) +} - f := mbtest.NewReportingMetricSetV2(t, getConfig()) +func testFetch(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("activity", r.Host())) events, errs := mbtest.ReportingFetchV2(f) if len(errs) > 0 { t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) @@ -56,27 +61,9 @@ func TestFetch(t *testing.T) { assert.Contains(t, event["user"].(common.MapStr), "id") } -func TestData(t *testing.T) { - compose.EnsureUp(t, "postgresql") - - f := mbtest.NewReportingMetricSetV2(t, getConfig()) - events, errs := mbtest.ReportingFetchV2(f) - if len(errs) > 0 { - t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) - } - assert.NotEmpty(t, events) - +func testData(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("activity", r.Host())) if err := mbtest.WriteEventsReporterV2(f, t, ""); err != nil { t.Fatal("write", err) } } - -func getConfig() map[string]interface{} { - return map[string]interface{}{ - "module": "postgresql", - "metricsets": []string{"activity"}, - "hosts": []string{postgresql.GetEnvDSN()}, - "username": postgresql.GetEnvUsername(), - "password": postgresql.GetEnvPassword(), - } -} diff --git a/metricbeat/module/postgresql/bgwriter/bgwriter_integration_test.go b/metricbeat/module/postgresql/bgwriter/bgwriter_integration_test.go index bcd69a800cc..b70d2fc823f 100644 --- a/metricbeat/module/postgresql/bgwriter/bgwriter_integration_test.go +++ b/metricbeat/module/postgresql/bgwriter/bgwriter_integration_test.go @@ -25,15 +25,20 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" - "github.com/elastic/beats/metricbeat/module/postgresql" + "github.com/elastic/beats/metricbeat/module/postgresql/mtest" "github.com/stretchr/testify/assert" ) -func TestFetch(t *testing.T) { - compose.EnsureUp(t, "postgresql") +func TestBgwriter(t *testing.T) { + mtest.Runner.Run(t, compose.Suite{ + "Fetch": testFetch, + "Data": testData, + }) +} - f := mbtest.NewReportingMetricSetV2(t, getConfig()) +func testFetch(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("bgwriter", r.Host())) events, errs := mbtest.ReportingFetchV2(f) if len(errs) > 0 { t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) @@ -61,27 +66,9 @@ func TestFetch(t *testing.T) { assert.Contains(t, buffers, "allocated") } -func TestData(t *testing.T) { - compose.EnsureUp(t, "postgresql") - - f := mbtest.NewReportingMetricSetV2(t, getConfig()) - events, errs := mbtest.ReportingFetchV2(f) - if len(errs) > 0 { - t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) - } - assert.NotEmpty(t, events) - +func testData(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("bgwriter", r.Host())) if err := mbtest.WriteEventsReporterV2(f, t, ""); err != nil { t.Fatal("write", err) } } - -func getConfig() map[string]interface{} { - return map[string]interface{}{ - "module": "postgresql", - "metricsets": []string{"bgwriter"}, - "hosts": []string{postgresql.GetEnvDSN()}, - "username": postgresql.GetEnvUsername(), - "password": postgresql.GetEnvPassword(), - } -} diff --git a/metricbeat/module/postgresql/database/database_integration_test.go b/metricbeat/module/postgresql/database/database_integration_test.go index 907b357062d..bf55800ea65 100644 --- a/metricbeat/module/postgresql/database/database_integration_test.go +++ b/metricbeat/module/postgresql/database/database_integration_test.go @@ -25,15 +25,20 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" - "github.com/elastic/beats/metricbeat/module/postgresql" + "github.com/elastic/beats/metricbeat/module/postgresql/mtest" "github.com/stretchr/testify/assert" ) -func TestFetch(t *testing.T) { - compose.EnsureUp(t, "postgresql") +func TestDatabase(t *testing.T) { + mtest.Runner.Run(t, compose.Suite{ + "Fetch": testFetch, + "Data": testData, + }) +} - f := mbtest.NewReportingMetricSetV2(t, getConfig()) +func testFetch(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("database", r.Host())) events, errs := mbtest.ReportingFetchV2(f) if len(errs) > 0 { t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) @@ -58,10 +63,8 @@ func TestFetch(t *testing.T) { assert.Contains(t, rows, "deleted") } -func TestData(t *testing.T) { - compose.EnsureUp(t, "postgresql") - - f := mbtest.NewReportingMetricSetV2(t, getConfig()) +func testData(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("database", r.Host())) events, errs := mbtest.ReportingFetchV2(f) if len(errs) > 0 { t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) @@ -72,13 +75,3 @@ func TestData(t *testing.T) { t.Fatal("write", err) } } - -func getConfig() map[string]interface{} { - return map[string]interface{}{ - "module": "postgresql", - "metricsets": []string{"database"}, - "hosts": []string{postgresql.GetEnvDSN()}, - "username": postgresql.GetEnvUsername(), - "password": postgresql.GetEnvPassword(), - } -} diff --git a/metricbeat/module/couchbase/testing.go b/metricbeat/module/postgresql/mtest/runner.go similarity index 81% rename from metricbeat/module/couchbase/testing.go rename to metricbeat/module/postgresql/mtest/runner.go index 7fe665308f3..af9ac5c0edf 100644 --- a/metricbeat/module/couchbase/testing.go +++ b/metricbeat/module/postgresql/mtest/runner.go @@ -15,15 +15,15 @@ // specific language governing permissions and limitations // under the License. -package couchbase +package mtest -import "os" +import ( + "github.com/elastic/beats/libbeat/tests/compose" +) -func GetEnvDSN() string { - dsn := os.Getenv("COUCHBASE_DSN") - - if len(dsn) == 0 { - dsn = "http://Administrator:password@localhost:8091" +var ( + Runner = compose.TestRunner{ + Service: "postgresql", + Parallel: true, } - return dsn -} +) diff --git a/metricbeat/module/mysql/testing.go b/metricbeat/module/postgresql/mtest/testing.go similarity index 57% rename from metricbeat/module/mysql/testing.go rename to metricbeat/module/postgresql/mtest/testing.go index 6036d717dc2..348af73ed28 100644 --- a/metricbeat/module/mysql/testing.go +++ b/metricbeat/module/postgresql/mtest/testing.go @@ -15,29 +15,35 @@ // specific language governing permissions and limitations // under the License. -package mysql +package mtest import ( + "fmt" "os" - - "github.com/go-sql-driver/mysql" ) -// Helper functions for testing used in the mysql MetricSets. +const ( + defaultUsername = "postgres" +) -// GetMySQLEnvDSN returns the MySQL server DSN to use for testing. It -// reads the value from the MYSQL_DSN environment variable and returns -// root@tcp(127.0.0.1:3306)/ if it is not set. -func GetMySQLEnvDSN() string { - dsn := os.Getenv("MYSQL_DSN") +func GetConfig(metricset, host string) map[string]interface{} { + dsn := fmt.Sprintf("postgres://%s?sslmode=disable", host) + return map[string]interface{}{ + "module": "postgresql", + "metricsets": []string{metricset}, + "hosts": []string{dsn}, + "username": getEnvUsername(), + "password": getEnvPassword(), + } +} - if len(dsn) == 0 { - c := &mysql.Config{ - Net: "tcp", - Addr: "127.0.0.1:3306", - User: "root", - } - dsn = c.FormatDSN() +func getEnvUsername() string { + if username := os.Getenv("POSTGRESQL_USERNAME"); len(username) > 0 { + return username } - return dsn + return defaultUsername +} + +func getEnvPassword() string { + return os.Getenv("POSTGRESQL_PASSWORD") } diff --git a/metricbeat/module/postgresql/statement/statement_integration_test.go b/metricbeat/module/postgresql/statement/statement_integration_test.go index 4b326594192..aac3f0433cf 100644 --- a/metricbeat/module/postgresql/statement/statement_integration_test.go +++ b/metricbeat/module/postgresql/statement/statement_integration_test.go @@ -25,15 +25,20 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" - "github.com/elastic/beats/metricbeat/module/postgresql" + "github.com/elastic/beats/metricbeat/module/postgresql/mtest" "github.com/stretchr/testify/assert" ) -func TestFetch(t *testing.T) { - compose.EnsureUp(t, "postgresql") +func TestStatement(t *testing.T) { + mtest.Runner.Run(t, compose.Suite{ + "Fetch": testFetch, + "Data": testData, + }) +} - f := mbtest.NewReportingMetricSetV2(t, getConfig()) +func testFetch(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("statement", r.Host())) events, errs := mbtest.ReportingFetchV2(f) if len(errs) > 0 { t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) @@ -89,27 +94,9 @@ func TestFetch(t *testing.T) { assert.Contains(t, temp, "written") } -func TestData(t *testing.T) { - compose.EnsureUp(t, "postgresql") - - f := mbtest.NewReportingMetricSetV2(t, getConfig()) - events, errs := mbtest.ReportingFetchV2(f) - if len(errs) > 0 { - t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) - } - assert.NotEmpty(t, events) - +func testData(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("statement", r.Host())) if err := mbtest.WriteEventsReporterV2(f, t, ""); err != nil { t.Fatal("write", err) } } - -func getConfig() map[string]interface{} { - return map[string]interface{}{ - "module": "postgresql", - "metricsets": []string{"statement"}, - "hosts": []string{postgresql.GetEnvDSN()}, - "username": postgresql.GetEnvUsername(), - "password": postgresql.GetEnvPassword(), - } -} diff --git a/metricbeat/module/prometheus/_meta/Dockerfile b/metricbeat/module/prometheus/_meta/Dockerfile index 55b29cbbb27..213cb1685ca 100644 --- a/metricbeat/module/prometheus/_meta/Dockerfile +++ b/metricbeat/module/prometheus/_meta/Dockerfile @@ -1,3 +1,8 @@ FROM prom/prometheus:v2.6.0 + +USER root +RUN mkdir /run # Needed to copy /run/compose_env on startup +USER nobody + HEALTHCHECK --interval=1s --retries=90 CMD nc -w 1 localhost 9090 = 1) } + +func testData(t *testing.T, r compose.R) { + ms := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("health", r.Host())) + err := mbtest.WriteEventsReporterV2(ms, t, "") + if err != nil { + t.Fatal("write", err) + } +} diff --git a/metricbeat/module/traefik/mtest/runner.go b/metricbeat/module/traefik/mtest/runner.go new file mode 100644 index 00000000000..896aca2eab7 --- /dev/null +++ b/metricbeat/module/traefik/mtest/runner.go @@ -0,0 +1,30 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mtest + +import ( + "github.com/elastic/beats/libbeat/tests/compose" +) + +var ( + // Runner is a compose test runner for traefik + Runner = compose.TestRunner{ + Service: "traefik", + Parallel: true, + } +) diff --git a/metricbeat/module/traefik/mtest/testing.go b/metricbeat/module/traefik/mtest/testing.go index c71b9accb01..938f2ee7d4e 100644 --- a/metricbeat/module/traefik/mtest/testing.go +++ b/metricbeat/module/traefik/mtest/testing.go @@ -17,33 +17,11 @@ package mtest -import "os" - -// GetEnvHost for Traefik -func GetEnvHost() string { - host := os.Getenv("TRAEFIK_HOST") - - if len(host) == 0 { - host = "127.0.0.1" - } - return host -} - -// GetEnvAPIPort for Traefik -func GetEnvAPIPort() string { - port := os.Getenv("TRAEFIK_API_PORT") - - if len(port) == 0 { - port = "8080" - } - return port -} - // GetConfig for Traefik -func GetConfig(metricset string) map[string]interface{} { +func GetConfig(metricset string, host string) map[string]interface{} { return map[string]interface{}{ "module": "traefik", "metricsets": []string{metricset}, - "hosts": []string{GetEnvHost() + ":" + GetEnvAPIPort()}, + "hosts": []string{host}, } } diff --git a/metricbeat/module/traefik/test_traefik.py b/metricbeat/module/traefik/test_traefik.py index de8de5f8b84..50a5df27670 100644 --- a/metricbeat/module/traefik/test_traefik.py +++ b/metricbeat/module/traefik/test_traefik.py @@ -25,5 +25,4 @@ def test_health(self, metricset): self.check_metricset("traefik", metricset, self.get_hosts(), self.FIELDS + ["service.name"]) def get_hosts(self): - return [os.getenv('TRAEFIK_HOST', 'localhost') + ':' + - os.getenv('TRAEFIK_API_PORT', '8080')] + return [self.compose_host()] diff --git a/metricbeat/module/uwsgi/_meta/Dockerfile b/metricbeat/module/uwsgi/_meta/Dockerfile index 81d8372ec12..17fc9571bc9 100644 --- a/metricbeat/module/uwsgi/_meta/Dockerfile +++ b/metricbeat/module/uwsgi/_meta/Dockerfile @@ -6,7 +6,4 @@ RUN pip install --no-cache-dir --trusted-host pypi.python.org uwsgi WORKDIR /app COPY testdata/app /app -HEALTHCHECK --interval=1s --retries=60 --timeout=10s CMD curl http://localhost:8080/ -EXPOSE 8080 9191 9192 - -CMD [""] +HEALTHCHECK --interval=1s --retries=60 --timeout=10s CMD curl http://localhost:8080 diff --git a/metricbeat/module/uwsgi/_meta/env b/metricbeat/module/uwsgi/_meta/env deleted file mode 100644 index 2f9a17b57d4..00000000000 --- a/metricbeat/module/uwsgi/_meta/env +++ /dev/null @@ -1,2 +0,0 @@ -UWSGI_STAT_TCP_SERVER=tcp://uwsgi_tcp:9191 -UWSGI_STAT_HTTP_SERVER=http://uwsgi_http:9192 diff --git a/metricbeat/module/uwsgi/status/status_integration_test.go b/metricbeat/module/uwsgi/status/status_integration_test.go index 5ac48b5e4a2..b6ad484d784 100644 --- a/metricbeat/module/uwsgi/status/status_integration_test.go +++ b/metricbeat/module/uwsgi/status/status_integration_test.go @@ -24,27 +24,36 @@ import ( "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" - "github.com/elastic/beats/metricbeat/module/uwsgi" "github.com/stretchr/testify/assert" ) -func TestFetchTCP(t *testing.T) { - compose.EnsureUp(t, "uwsgi_tcp") +func TestStatusTCP(t *testing.T) { + t.Parallel() + testStatus(t, "uwsgi_tcp") +} - f := mbtest.NewEventsFetcher(t, getConfig("tcp")) - events, err := f.Fetch() - assert.NoError(t, err) +func TestStatusHTTP(t *testing.T) { + t.Parallel() + testStatus(t, "uwsgi_http") +} - assert.True(t, len(events) > 0) - totals := findItems(events, "total") - assert.Equal(t, 1, len(totals)) +func testStatus(t *testing.T, service string) { + runner := compose.TestRunner{Service: service} + + test := uwsgiTest{Service: service} + runner.Run(t, compose.Suite{ + "Fetch": test.Fetch, + "Data": test.Data, + }) } -func TestFetchHTTP(t *testing.T) { - compose.EnsureUp(t, "uwsgi_http") +type uwsgiTest struct { + Service string +} - f := mbtest.NewEventsFetcher(t, getConfig("http")) +func (u *uwsgiTest) Fetch(t *testing.T, r compose.R) { + f := mbtest.NewEventsFetcher(t, getConfig(t, u.Service, r.Host())) events, err := f.Fetch() assert.NoError(t, err) @@ -53,19 +62,27 @@ func TestFetchHTTP(t *testing.T) { assert.Equal(t, 1, len(totals)) } -func getConfig(scheme string) map[string]interface{} { +func (u *uwsgiTest) Data(t *testing.T, r compose.R) { + f := mbtest.NewEventsFetcher(t, getConfig(t, u.Service, r.Host())) + err := mbtest.WriteEvents(f, t) + if err != nil { + t.Fatal("write", err) + } +} + +func getConfig(t *testing.T, service string, host string) map[string]interface{} { conf := map[string]interface{}{ "module": "uwsgi", "metricsets": []string{"status"}, } - switch scheme { - case "tcp": - conf["hosts"] = []string{uwsgi.GetEnvTCPServer()} - case "http", "https": - conf["hosts"] = []string{uwsgi.GetEnvHTTPServer()} + switch service { + case "uwsgi_tcp": + conf["hosts"] = []string{"tcp://" + host} + case "uwsgi_http": + conf["hosts"] = []string{"http://" + host} default: - conf["hosts"] = []string{uwsgi.GetEnvTCPServer()} + t.Errorf("Unexpected service: %s", service) } return conf } diff --git a/metricbeat/module/zookeeper/_meta/env b/metricbeat/module/zookeeper/_meta/env deleted file mode 100644 index 6a063ecec28..00000000000 --- a/metricbeat/module/zookeeper/_meta/env +++ /dev/null @@ -1,2 +0,0 @@ -ZOOKEEPER_HOST=zookeeper -ZOOKEEPER_PORT=2181 diff --git a/metricbeat/module/zookeeper/mntr/mntr_integration_test.go b/metricbeat/module/zookeeper/mntr/mntr_integration_test.go index f5d1035c84f..2e11a5bb369 100644 --- a/metricbeat/module/zookeeper/mntr/mntr_integration_test.go +++ b/metricbeat/module/zookeeper/mntr/mntr_integration_test.go @@ -27,13 +27,20 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" - "github.com/elastic/beats/metricbeat/module/zookeeper" ) -func TestFetch(t *testing.T) { - compose.EnsureUp(t, "zookeeper") +func TestMntr(t *testing.T) { + runner := compose.TestRunner{Service: "zookeeper", Parallel: true} - f := mbtest.NewReportingMetricSetV2(t, getConfig()) + runner.Run(t, compose.Suite{ + "Fetch": testFetch, + "Data": testData, + }) + +} + +func testFetch(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host())) events, errs := mbtest.ReportingFetchV2(f) assert.Empty(t, errs) @@ -58,20 +65,18 @@ func TestFetch(t *testing.T) { assert.True(t, len(event) >= 10) } -func TestData(t *testing.T) { - compose.EnsureUp(t, "zookeeper") - - f := mbtest.NewReportingMetricSetV2(t, getConfig()) +func testData(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host())) err := mbtest.WriteEventsReporterV2(f, t, ".") if err != nil { t.Fatal("write", err) } } -func getConfig() map[string]interface{} { +func getConfig(host string) map[string]interface{} { return map[string]interface{}{ "module": "zookeeper", "metricsets": []string{"mntr"}, - "hosts": []string{zookeeper.GetZookeeperEnvHost() + ":" + zookeeper.GetZookeeperEnvPort()}, + "hosts": []string{host}, } } diff --git a/metricbeat/module/zookeeper/server/data_integration_test.go b/metricbeat/module/zookeeper/server/data_integration_test.go index 843db0a7dca..36706584fd4 100644 --- a/metricbeat/module/zookeeper/server/data_integration_test.go +++ b/metricbeat/module/zookeeper/server/data_integration_test.go @@ -22,33 +22,13 @@ package server import ( "testing" - "github.com/elastic/beats/metricbeat/module/zookeeper" - - _ "github.com/denisenkom/go-mssqldb" - "github.com/stretchr/testify/assert" - + "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" ) -func TestData(t *testing.T) { - t.Skip("Skipping `data.json` generation test") - - f := mbtest.NewReportingMetricSetV2(t, getDataConfig()) - events, errs := mbtest.ReportingFetchV2(f) - if len(errs) > 0 { - t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) - } - assert.NotEmpty(t, events) - +func testData(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host())) if err := mbtest.WriteEventsReporterV2(f, t, ""); err != nil { t.Fatal("write", err) } } - -func getDataConfig() map[string]interface{} { - return map[string]interface{}{ - "module": "zookeeper", - "metricsets": []string{"server"}, - "hosts": []string{zookeeper.GetZookeeperEnvHost() + ":" + zookeeper.GetZookeeperEnvPort()}, - } -} diff --git a/metricbeat/module/zookeeper/server/server_integration_test.go b/metricbeat/module/zookeeper/server/server_integration_test.go index 75b8beb9940..c4a45b21fb4 100644 --- a/metricbeat/module/zookeeper/server/server_integration_test.go +++ b/metricbeat/module/zookeeper/server/server_integration_test.go @@ -22,21 +22,26 @@ package server import ( "testing" - "github.com/elastic/beats/libbeat/logp" - "github.com/elastic/beats/metricbeat/module/zookeeper" - "github.com/stretchr/testify/assert" + "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" ) -func TestFetch(t *testing.T) { +func TestServer(t *testing.T) { logp.TestingSetup() - compose.EnsureUp(t, "zookeeper") + runner := compose.TestRunner{Service: "zookeeper", Parallel: true} + runner.Run(t, compose.Suite{ + "Fetch": testFetch, + "Data": testData, + }) +} + +func testFetch(t *testing.T, r compose.R) { - f := mbtest.NewReportingMetricSetV2(t, getConfig()) + f := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host())) events, errs := mbtest.ReportingFetchV2(f) if len(errs) > 0 { t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) @@ -58,10 +63,10 @@ func TestFetch(t *testing.T) { } } -func getConfig() map[string]interface{} { +func getConfig(host string) map[string]interface{} { return map[string]interface{}{ "module": "zookeeper", "metricsets": []string{"server"}, - "hosts": []string{zookeeper.GetZookeeperEnvHost() + ":" + zookeeper.GetZookeeperEnvPort()}, + "hosts": []string{host}, } } diff --git a/metricbeat/module/zookeeper/testing.go b/metricbeat/module/zookeeper/testing.go deleted file mode 100644 index afa3cac8398..00000000000 --- a/metricbeat/module/zookeeper/testing.go +++ /dev/null @@ -1,48 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package zookeeper - -import ( - "os" -) - -// Helper functions for testing used in the zookeeper MetricSets. - -// GetZookeeperEnvHost returns the hostname of the ZooKeeper server to use for -// testing. It reads the value from the ZOOKEEPER_HOST environment variable and -// returns localhost if it is not set. -func GetZookeeperEnvHost() string { - host := os.Getenv("ZOOKEEPER_HOST") - - if len(host) == 0 { - host = "localhost" - } - return host -} - -// GetZookeeperEnvPort returns the port of the ZooKeeper server to use for -// testing. It reads the value from the ZOOKEEPER_PORT environment variable and -// returns 2181 if it is not set. -func GetZookeeperEnvPort() string { - port := os.Getenv("ZOOKEEPER_PORT") - - if len(port) == 0 { - port = "2181" - } - return port -} diff --git a/metricbeat/tests/system/test_aerospike.py b/metricbeat/tests/system/test_aerospike.py index 3f957e15885..6f118fda0d8 100644 --- a/metricbeat/tests/system/test_aerospike.py +++ b/metricbeat/tests/system/test_aerospike.py @@ -16,5 +16,4 @@ def test_namespace(self): self.check_metricset("aerospike", "namespace", self.get_hosts(), self.FIELDS) def get_hosts(self): - return [os.getenv('AEROSPIKE_HOST', 'localhost') + ':' + - os.getenv('AEROSPIKE_PORT', '3000')] + return [self.compose_host()] diff --git a/metricbeat/tests/system/test_apache.py b/metricbeat/tests/system/test_apache.py index fd164518120..41fa7b31300 100644 --- a/metricbeat/tests/system/test_apache.py +++ b/metricbeat/tests/system/test_apache.py @@ -78,8 +78,7 @@ def verify_fields(self, evt): # There are more fields that could be checked. def get_hosts(self): - return ['http://' + os.getenv('APACHE_HOST', 'localhost') + ':' + - os.getenv('APACHE_PORT', '80')] + return ['http://' + self.compose_host()] class ApacheOldStatusTest(ApacheStatusTest): @@ -91,7 +90,3 @@ def verify_fields(self, evt): apache_status = evt["apache"]["status"] self.assertItemsEqual( self.de_dot(APACHE_OLD_STATUS_FIELDS), apache_status.keys()) - - def get_hosts(self): - return ['http://' + os.getenv('APACHE_OLD_HOST', 'localhost') + ':' + - os.getenv('APACHE_PORT', '80')] diff --git a/metricbeat/tests/system/test_base.py b/metricbeat/tests/system/test_base.py index e5d58e0284d..499a8f38d7c 100644 --- a/metricbeat/tests/system/test_base.py +++ b/metricbeat/tests/system/test_base.py @@ -75,3 +75,12 @@ def test_dashboards(self): assert exit_code == 0 assert self.log_contains("Kibana dashboards successfully loaded.") + + def get_elasticsearch_url(self): + return "http://" + self.compose_host("elasticsearch") + + def get_kibana_url(self): + """ + Returns kibana host URL + """ + return "http://" + self.compose_host("kibana") diff --git a/metricbeat/tests/system/test_ceph.py b/metricbeat/tests/system/test_ceph.py index 2e5d91c2e27..0cd10b29a22 100644 --- a/metricbeat/tests/system/test_ceph.py +++ b/metricbeat/tests/system/test_ceph.py @@ -24,5 +24,4 @@ def test_ceph(self, metricset): self.check_metricset("ceph", metricset, self.get_hosts(), self.FIELDS) def get_hosts(self): - return [os.getenv('CEPH_HOST', 'localhost') + ':' + - os.getenv('CEPH_PORT', '5000')] + return [self.compose_host()] diff --git a/metricbeat/tests/system/test_couchbase.py b/metricbeat/tests/system/test_couchbase.py index f0d75780359..f8b2f619e09 100644 --- a/metricbeat/tests/system/test_couchbase.py +++ b/metricbeat/tests/system/test_couchbase.py @@ -22,5 +22,4 @@ def test_couchbase(self, metricset): self.check_metricset("couchbase", metricset, self.get_hosts(), self.FIELDS) def get_hosts(self): - return [os.getenv('COUCHBASE_HOST', 'localhost') + ':' + - os.getenv('COUCHBASE_PORT', '8091')] + return [self.compose_host()] diff --git a/metricbeat/tests/system/test_docker.py b/metricbeat/tests/system/test_docker.py index 2d266d59dd8..6fc8e8a36d8 100644 --- a/metricbeat/tests/system/test_docker.py +++ b/metricbeat/tests/system/test_docker.py @@ -6,6 +6,7 @@ class Test(metricbeat.BaseTest): + COMPOSE_SERVICES = ["redis"] # Just to have a container running COMPOSE_SERVICES = ['docker'] diff --git a/metricbeat/tests/system/test_dropwizard.py b/metricbeat/tests/system/test_dropwizard.py index a9ab7dbbe8d..34297426f2d 100644 --- a/metricbeat/tests/system/test_dropwizard.py +++ b/metricbeat/tests/system/test_dropwizard.py @@ -30,5 +30,4 @@ def test_dropwizard(self): self.assertTrue(len(output) >= 1) def get_hosts(self): - return [os.getenv('DROPWIZARD_HOST', 'localhost') + ':' + - os.getenv('DROPWIZARD_PORT', '8080')] + return [self.compose_host()] diff --git a/metricbeat/tests/system/test_envoyproxy.py b/metricbeat/tests/system/test_envoyproxy.py index 40b7bce1a13..013cbb13aa4 100644 --- a/metricbeat/tests/system/test_envoyproxy.py +++ b/metricbeat/tests/system/test_envoyproxy.py @@ -31,5 +31,4 @@ def test_stats(self): self.assert_fields_are_documented(evt) def get_hosts(self): - return [os.getenv('ENVOYPROXY_HOST', 'localhost') + ':' + - os.getenv('ENVOYPROXY_PORT', '9901')] + return [self.compose_host()] diff --git a/metricbeat/tests/system/test_haproxy.py b/metricbeat/tests/system/test_haproxy.py index cf3510db71c..1618982959a 100644 --- a/metricbeat/tests/system/test_haproxy.py +++ b/metricbeat/tests/system/test_haproxy.py @@ -32,7 +32,7 @@ def test_info_socket(self): self.render_config_template(modules=[{ "name": "haproxy", "metricsets": ["info"], - "hosts": ["tcp://%s:%d" % (self.compose_hosts()[0], 14567)], + "hosts": ["tcp://%s" % (self.compose_host(port="14567/tcp"))], "period": "5s" }]) self._test_info() @@ -59,7 +59,7 @@ def test_stat_socket(self): self.render_config_template(modules=[{ "name": "haproxy", "metricsets": ["stat"], - "hosts": ["tcp://%s:%d" % (self.compose_hosts()[0], 14567)], + "hosts": ["tcp://%s" % (self.compose_host(port="14567/tcp"))], "period": "5s" }]) self._test_stat() @@ -72,7 +72,7 @@ def test_stat_http(self): self.render_config_template(modules=[{ "name": "haproxy", "metricsets": ["stat"], - "hosts": ["http://%s:%d/stats" % (self.compose_hosts()[0], 14568)], + "hosts": ["http://%s/stats" % (self.compose_host(port="14568/tcp"))], "period": "5s" }]) self._test_stat() @@ -87,7 +87,7 @@ def test_stat_http_auth(self): "metricsets": ["stat"], "username": "admin", "password": "admin", - "hosts": ["http://%s:%d/stats" % (self.compose_hosts()[0], 14569)], + "hosts": ["http://%s/stats" % (self.compose_host(port="14569/tcp"))], "period": "5s" }]) self._test_stat() diff --git a/metricbeat/tests/system/test_http.py b/metricbeat/tests/system/test_http.py index d25eadf6d56..c4b343572a2 100644 --- a/metricbeat/tests/system/test_http.py +++ b/metricbeat/tests/system/test_http.py @@ -76,4 +76,4 @@ def test_server(self): self.assert_fields_are_documented(evt) def get_host(self): - return "http://" + os.getenv('HTTP_HOST', 'localhost') + ':' + os.getenv('HTTP_PORT', '8080') + return "http://" + self.compose_host() diff --git a/metricbeat/tests/system/test_jolokia.py b/metricbeat/tests/system/test_jolokia.py index 921ee5aef25..b44f03985b6 100644 --- a/metricbeat/tests/system/test_jolokia.py +++ b/metricbeat/tests/system/test_jolokia.py @@ -50,5 +50,4 @@ def test_jmx(self, mbean): assert evt["jolokia"]["test"]["gc"]["collection_count"] >= 0 def get_hosts(self): - return [os.getenv('JOLOKIA_HOST', 'localhost') + ':' + - os.getenv('JOLOKIA_PORT', '8778')] + return [self.compose_host()] diff --git a/metricbeat/tests/system/test_kafka.py b/metricbeat/tests/system/test_kafka.py index a7ba2d26e28..4f030515dea 100644 --- a/metricbeat/tests/system/test_kafka.py +++ b/metricbeat/tests/system/test_kafka.py @@ -43,8 +43,7 @@ def create_topic(self): producer.send('foobar', b'some_message_bytes') def get_hosts(self): - return [self.compose_hosts()[0] + ':' + - os.getenv('KAFKA_PORT', '9092')] + return [self.compose_host()] class Kafka_1_1_0_Test(KafkaTest): diff --git a/metricbeat/tests/system/test_kibana.py b/metricbeat/tests/system/test_kibana.py index fbb7a5736d2..33f6f376c1b 100644 --- a/metricbeat/tests/system/test_kibana.py +++ b/metricbeat/tests/system/test_kibana.py @@ -48,8 +48,7 @@ def test_status(self): self.assert_fields_are_documented(evt) def get_hosts(self): - return [os.getenv('KIBANA_HOST', 'localhost') + ':' + - os.getenv('KIBANA_PORT', '5601')] + return [self.compose_host("kibana")] def get_version(self): host = self.get_hosts()[0] diff --git a/metricbeat/tests/system/test_kubernetes.py b/metricbeat/tests/system/test_kubernetes.py index 6ede3cfe30c..5c469bc3902 100644 --- a/metricbeat/tests/system/test_kubernetes.py +++ b/metricbeat/tests/system/test_kubernetes.py @@ -77,16 +77,8 @@ def _test_metricset(self, metricset, expected_events, hosts): @classmethod def get_kubelet_hosts(cls): - return [ - "http://" + - os.getenv('KUBELET_HOST', 'localhost') + ':' + - os.getenv('KUBELET_PORT', '10255') - ] + return [self.compose_host("kubernetes")] @classmethod def get_kube_state_hosts(cls): - return [ - "http://" + - os.getenv('KUBE_STATE_METRICS_HOST', 'localhost') + ':' + - os.getenv('KUBE_STATE_METRICS_PORT', '18080') - ] + return [self.compose_host("kubestate")] diff --git a/metricbeat/tests/system/test_logstash.py b/metricbeat/tests/system/test_logstash.py index 8280bfb0223..2f6c5b2c38d 100644 --- a/metricbeat/tests/system/test_logstash.py +++ b/metricbeat/tests/system/test_logstash.py @@ -55,5 +55,4 @@ def test_node_stats(self): self.assert_fields_are_documented(evt) def get_hosts(self): - return [os.getenv('LOGSTASH_HOST', 'localhost') + ':' + - os.getenv('LOGSTASH_PORT', '9600')] + return [self.compose_host()] diff --git a/metricbeat/tests/system/test_memcached.py b/metricbeat/tests/system/test_memcached.py index ede980b0581..38c325a29f0 100644 --- a/metricbeat/tests/system/test_memcached.py +++ b/metricbeat/tests/system/test_memcached.py @@ -31,5 +31,4 @@ def test_stats(self): self.assert_fields_are_documented(evt) def get_hosts(self): - return [os.getenv('MEMCACHED_HOST', 'localhost') + ':' + - os.getenv('MEMCACHED_PORT', '11211')] + return [self.compose_host()] diff --git a/metricbeat/tests/system/test_mongodb.py b/metricbeat/tests/system/test_mongodb.py index 4400d1c95b8..db361fbd720 100644 --- a/metricbeat/tests/system/test_mongodb.py +++ b/metricbeat/tests/system/test_mongodb.py @@ -36,5 +36,4 @@ def test_status(self): self.assert_fields_are_documented(evt) def get_hosts(self): - return [os.getenv('MONGODB_HOST', 'localhost') + ':' + - os.getenv('MONGODB_PORT', '27017')] + return [self.compose_host()] diff --git a/metricbeat/tests/system/test_munin.py b/metricbeat/tests/system/test_munin.py index 63cc80a94ff..727d811d351 100644 --- a/metricbeat/tests/system/test_munin.py +++ b/metricbeat/tests/system/test_munin.py @@ -38,5 +38,4 @@ def test_munin_node(self): self.assert_fields_are_documented(evt) def get_hosts(self): - return [self.compose_hosts()[0] + ':' + - os.getenv('MUNIN_PORT', '4949')] + return [self.compose_host()] diff --git a/metricbeat/tests/system/test_mysql.py b/metricbeat/tests/system/test_mysql.py index 0241079d980..d0c9bb0e81a 100644 --- a/metricbeat/tests/system/test_mysql.py +++ b/metricbeat/tests/system/test_mysql.py @@ -43,7 +43,7 @@ def test_status(self): self.assert_fields_are_documented(evt) def get_hosts(self): - return ['root:test@tcp({}:3306)/'.format(self.compose_hosts()[0])] + return ['root:test@tcp({})/'.format(self.compose_host())] class TestMysql80(Test): diff --git a/metricbeat/tests/system/test_nats.py b/metricbeat/tests/system/test_nats.py index d2f5bb6d8ef..6375ec45a5f 100644 --- a/metricbeat/tests/system/test_nats.py +++ b/metricbeat/tests/system/test_nats.py @@ -110,7 +110,4 @@ def test_subscriptions(self): self.assert_fields_are_documented(evt) def get_hosts(self): - return ["{}:{}".format( - os.getenv('NATS_HOST', 'localhost'), - os.getenv('NATS_PORT', '8222') - )] + return [self.compose_host()] diff --git a/metricbeat/tests/system/test_phpfpm.py b/metricbeat/tests/system/test_phpfpm.py index edb5405a404..df20358e9d7 100644 --- a/metricbeat/tests/system/test_phpfpm.py +++ b/metricbeat/tests/system/test_phpfpm.py @@ -34,5 +34,4 @@ def test_info(self): self.assert_fields_are_documented(evt) def get_hosts(self): - return [os.getenv('PHPFPM_HOST', 'localhost') + ':' + - os.getenv('PHPFPM_PORT', '81')] + return [self.compose_host()] diff --git a/metricbeat/tests/system/test_postgresql.py b/metricbeat/tests/system/test_postgresql.py index 5150d9ab28b..43a8db4a385 100644 --- a/metricbeat/tests/system/test_postgresql.py +++ b/metricbeat/tests/system/test_postgresql.py @@ -19,8 +19,14 @@ def common_checks(self, output): self.assert_fields_are_documented(evt) def get_hosts(self): - return [os.getenv("POSTGRESQL_DSN")], os.getenv("POSTGRESQL_USERNAME"), \ - os.getenv("POSTGRESQL_PASSWORD") + username = "postgres" + host = self.compose_host() + dsn = "postgres://{}?sslmode=disable".format(host) + return ( + [dsn], + username, + os.getenv("POSTGRESQL_PASSWORD"), + ) @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") @attr('integration') diff --git a/metricbeat/tests/system/test_prometheus.py b/metricbeat/tests/system/test_prometheus.py index 07fec23705e..b3bb18b7e06 100644 --- a/metricbeat/tests/system/test_prometheus.py +++ b/metricbeat/tests/system/test_prometheus.py @@ -33,5 +33,4 @@ def test_stats(self): self.assert_fields_are_documented(evt) def get_hosts(self): - return ["http://" + os.getenv('PROMETHEUS_HOST', 'localhost') + ':' + - os.getenv('PROMETHEUS_PORT', '9090')] + return [self.compose_host()] diff --git a/metricbeat/tests/system/test_redis.py b/metricbeat/tests/system/test_redis.py index 61e9fd1e56b..4ecf57587b4 100644 --- a/metricbeat/tests/system/test_redis.py +++ b/metricbeat/tests/system/test_redis.py @@ -60,9 +60,10 @@ def test_keyspace(self): """ # At least one event must be inserted so db stats exist + host, port = self.compose_host().split(":") r = redis.StrictRedis( - host=self.get_host(), - port=os.getenv('REDIS_PORT', '6379'), + host=host, + port=port, db=0) r.flushall() r.set('foo', 'bar') @@ -95,9 +96,10 @@ def test_key(self): """ # At least one event must be inserted so db stats exist + host, port = self.compose_host().split(":") r = redis.StrictRedis( - host=self.get_host(), - port=os.getenv('REDIS_PORT', '6379'), + host=host, + port=port, db=0) r.flushall() r.rpush('list-key', 'one', 'two', 'three') @@ -159,8 +161,7 @@ def test_module_processors(self): self.assertItemsEqual(self.de_dot(CPU_FIELDS), redis_info["cpu"].keys()) def get_hosts(self): - return [self.get_host() + ':' + - os.getenv('REDIS_PORT', '6379')] + return [self.compose_host()] def get_host(self): if len(self.compose_hosts()) > 0: diff --git a/metricbeat/tests/system/test_uwsgi.py b/metricbeat/tests/system/test_uwsgi.py index 3707d2cab86..a35576fb85e 100644 --- a/metricbeat/tests/system/test_uwsgi.py +++ b/metricbeat/tests/system/test_uwsgi.py @@ -8,8 +8,7 @@ class Test(metricbeat.BaseTest): - - COMPOSE_SERVICES = ['uwsgi_tcp', "uwsgi_http"] + COMPOSE_SERVICES = ['uwsgi_tcp'] def common_checks(self, output): # Ensure no errors or warnings exist in the log. @@ -51,15 +50,14 @@ def common_checks(self, output): @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") @attr('integration') - def test_status_tcp(self): + def test_status(self): """ uWSGI module outputs an event. """ - hosts = [os.getenv("UWSGI_STAT_TCP_SERVER")] self.render_config_template(modules=[{ "name": "uwsgi", "metricsets": ["status"], - "hosts": hosts, + "hosts": [self.get_host()], "period": "5s" }]) proc = self.start_beat() @@ -69,22 +67,12 @@ def test_status_tcp(self): output = self.read_output_json() self.common_checks(output) - @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") - @attr('integration') - def test_status_http(self): - """ - uWSGI module outputs an event. - """ - hosts = [os.getenv("UWSGI_STAT_HTTP_SERVER")] - self.render_config_template(modules=[{ - "name": "uwsgi", - "metricsets": ["status"], - "hosts": hosts, - "period": "5s" - }]) - proc = self.start_beat() - self.wait_until(lambda: self.output_lines() > 0) - proc.check_kill_and_wait() + def get_host(self): + return "tcp://" + self.compose_host() - output = self.read_output_json() - self.common_checks(output) + +class TestHTTP(Test): + COMPOSE_SERVICES = ['uwsgi_http'] + + def get_host(self): + return "http://" + self.compose_host() diff --git a/metricbeat/tests/system/test_zookeeper.py b/metricbeat/tests/system/test_zookeeper.py index 6e3baadea63..eb2ff892aac 100644 --- a/metricbeat/tests/system/test_zookeeper.py +++ b/metricbeat/tests/system/test_zookeeper.py @@ -79,5 +79,4 @@ def test_output(self): self.assert_fields_are_documented(evt) def get_hosts(self): - return [os.getenv('ZOOKEEPER_HOST', 'localhost') + ':' + - os.getenv('ZOOKEEPER_PORT', '2181')] + return [self.compose_host()] diff --git a/x-pack/metricbeat/docker-compose.yml b/x-pack/metricbeat/docker-compose.yml index bec53d09468..82535cc8a6d 100644 --- a/x-pack/metricbeat/docker-compose.yml +++ b/x-pack/metricbeat/docker-compose.yml @@ -8,10 +8,11 @@ services: volumes: - ${PWD}/../..:/go/src/github.com/elastic/beats/ - /var/run/docker.sock:/var/run/docker.sock + network_mode: host command: make - env_file: - - ./module/mssql/_meta/env # Modules mssql: build: ./module/mssql/_meta + ports: + - 1433 diff --git a/x-pack/metricbeat/module/mssql/_meta/env b/x-pack/metricbeat/module/mssql/_meta/env deleted file mode 100644 index 145a143e51f..00000000000 --- a/x-pack/metricbeat/module/mssql/_meta/env +++ /dev/null @@ -1,4 +0,0 @@ -MSSQL_HOST=mssql -MSSQL_PORT=1433 -MSSQL_USER=SA -MSSQL_PASSWORD=1234_asdf diff --git a/x-pack/metricbeat/module/mssql/testing/mssql.go b/x-pack/metricbeat/module/mssql/mtest/mssql.go similarity index 79% rename from x-pack/metricbeat/module/mssql/testing/mssql.go rename to x-pack/metricbeat/module/mssql/mtest/mssql.go index 971b06badb5..7d405af6ed0 100644 --- a/x-pack/metricbeat/module/mssql/testing/mssql.go +++ b/x-pack/metricbeat/module/mssql/mtest/mssql.go @@ -2,19 +2,19 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package testing +package mtest import "os" // GetConfig returns the required configuration options for testing a MSSQL // metricset. -func GetConfig(metricSets ...string) map[string]interface{} { +func GetConfig(host string, metricSets ...string) map[string]interface{} { return map[string]interface{}{ "module": "mssql", "metricsets": metricSets, - "hosts": []string{EnvOr("MSSQL_HOST", "localhost")}, + "hosts": []string{host}, "username": EnvOr("MSSQL_USER", "SA"), - "password": EnvOr("MSSQL_PASSWORD", ""), + "password": EnvOr("MSSQL_PASSWORD", "1234_asdf"), } } diff --git a/x-pack/metricbeat/module/mssql/mtest/runner.go b/x-pack/metricbeat/module/mssql/mtest/runner.go new file mode 100644 index 00000000000..1a8c2b8033f --- /dev/null +++ b/x-pack/metricbeat/module/mssql/mtest/runner.go @@ -0,0 +1,18 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build integration + +package mtest + +import ( + "github.com/elastic/beats/libbeat/tests/compose" +) + +var ( + // Runner is a compose test runner for Redis tests + Runner = compose.TestRunner{ + Service: "mssql", + } +) diff --git a/x-pack/metricbeat/module/mssql/performance/data_integration_test.go b/x-pack/metricbeat/module/mssql/performance/data_integration_test.go index 116a7269825..02a352c3f82 100644 --- a/x-pack/metricbeat/module/mssql/performance/data_integration_test.go +++ b/x-pack/metricbeat/module/mssql/performance/data_integration_test.go @@ -12,14 +12,15 @@ import ( "github.com/pkg/errors" "github.com/stretchr/testify/assert" + "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" - mtest "github.com/elastic/beats/x-pack/metricbeat/module/mssql/testing" + "github.com/elastic/beats/x-pack/metricbeat/module/mssql/mtest" ) -func TestData(t *testing.T) { +func testData(t *testing.T, r compose.R) { t.Skip("Skipping `data.json` generation test") + _, config, err := getHostURI(r.Host()) - _, config, err := getHostURI() if err != nil { t.Fatal("error getting config information", err.Error()) } @@ -36,13 +37,8 @@ func TestData(t *testing.T) { } } -func getHostURI() (string, map[string]interface{}, error) { - config := mtest.GetConfig("performance") - - host, ok := config["hosts"].([]string) - if !ok { - return "", nil, errors.New("error getting host name information") - } +func getHostURI(host string) (string, map[string]interface{}, error) { + config := mtest.GetConfig(host, "performance") username, ok := config["username"].(string) if !ok { @@ -57,7 +53,7 @@ func getHostURI() (string, map[string]interface{}, error) { u := &url.URL{ Scheme: "sqlserver", User: url.UserPassword(username, password), - Host: host[0], + Host: host, } return u.String(), config, nil diff --git a/x-pack/metricbeat/module/mssql/performance/performance_integration_test.go b/x-pack/metricbeat/module/mssql/performance/performance_integration_test.go index 1c17350731a..ec7b72302a9 100644 --- a/x-pack/metricbeat/module/mssql/performance/performance_integration_test.go +++ b/x-pack/metricbeat/module/mssql/performance/performance_integration_test.go @@ -15,19 +15,24 @@ import ( "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" - mtest "github.com/elastic/beats/x-pack/metricbeat/module/mssql/testing" + "github.com/elastic/beats/x-pack/metricbeat/module/mssql/mtest" ) +func TestPerformance(t *testing.T) { + logp.TestingSetup() + + mtest.Runner.Run(t, compose.Suite{ + "Fetch": testFetch, + }) +} + type keyAssertion struct { key string assertion func(v interface{}, key string) } -func TestFetch(t *testing.T) { - logp.TestingSetup() - compose.EnsureUp(t, "mssql") - - f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("performance")) +func testFetch(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig(r.Host(), "performance")) events, errs := mbtest.ReportingFetchV2(f) if len(errs) > 0 { t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) diff --git a/x-pack/metricbeat/module/mssql/transaction_log/data_integration_test.go b/x-pack/metricbeat/module/mssql/transaction_log/data_integration_test.go index 8616e427c0b..3882bfa52b0 100644 --- a/x-pack/metricbeat/module/mssql/transaction_log/data_integration_test.go +++ b/x-pack/metricbeat/module/mssql/transaction_log/data_integration_test.go @@ -7,15 +7,15 @@ package transaction_log import ( "testing" + "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" - mtest "github.com/elastic/beats/x-pack/metricbeat/module/mssql/testing" + "github.com/elastic/beats/x-pack/metricbeat/module/mssql/mtest" ) -func TestData(t *testing.T) { +func testData(t *testing.T, r compose.R) { t.Skip("Skipping `data.json` generation test") - f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("transaction_log")) - + f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig(r.Host(), "transaction_log")) err := mbtest.WriteEventsReporterV2(f, t, "") if err != nil { t.Fatal("write", err) diff --git a/x-pack/metricbeat/module/mssql/transaction_log/transaction_log_integration_test.go b/x-pack/metricbeat/module/mssql/transaction_log/transaction_log_integration_test.go index f899ec26141..d44be5a80ac 100644 --- a/x-pack/metricbeat/module/mssql/transaction_log/transaction_log_integration_test.go +++ b/x-pack/metricbeat/module/mssql/transaction_log/transaction_log_integration_test.go @@ -14,14 +14,20 @@ import ( "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" - mtest "github.com/elastic/beats/x-pack/metricbeat/module/mssql/testing" + "github.com/elastic/beats/x-pack/metricbeat/module/mssql/mtest" ) -func TestFetch(t *testing.T) { +func TestDb(t *testing.T) { logp.TestingSetup() - compose.EnsureUp(t, "mssql") - f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("transaction_log")) + mtest.Runner.Run(t, compose.Suite{ + "Fetch": testFetch, + "Data": testData, + }) +} + +func testFetch(t *testing.T, r compose.R) { + f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig(r.Host(), "transaction_log")) events, errs := mbtest.ReportingFetchV2(f) if len(errs) > 0 { t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) diff --git a/x-pack/metricbeat/tests/system/test_mssql.py b/x-pack/metricbeat/tests/system/test_mssql.py index 963177b718c..c3b9eac61ef 100644 --- a/x-pack/metricbeat/tests/system/test_mssql.py +++ b/x-pack/metricbeat/tests/system/test_mssql.py @@ -75,7 +75,7 @@ def test_performance(self): self.assert_fields_are_documented(evt) def get_hosts(self): - return [os.getenv('MSSQL_HOST', 'mssql')] + return [self.compose_host()] def get_username(self): return os.getenv('MSSQL_USERNAME', 'SA')