First implementation of Metricbeat

This is the first basic implementation of MetricBeat. For more details on
what it contains, check below.

= MetricSets
The following basic MetricSet are part of this first version.

* Apache module with Apache status
* Redis module with Redis info
* MySQL module with status

Be aware that only very few metrics are collected per MetricSet, but more
could be added easily

= Data Model

The current data model of each metricset can be found in the file
beater/metricbeat.go in the header docs.

= Error Handling

It is assumed, that a module or metricset can panic. This panic is catched
to not kill all other modules. This error is reported in the log files. The failing
module / metricset will not be restarted

= Configuration

Each module and metricSet can have its own configuration. The following fields
are generic configurations per module and are always available:

* Period
* Hosts

More details can be found in etc/beat.yml

= Testing

* Golang test framework for unit tests
* Golang test framework combined with docker-compose for integration tests
* Python libbeat system test framework combined with docker-compose for system tests

All tests are intended to be run inside the docker environment.

= Vendoring

The prevent dependency conflicts between modules, each module can have its own vendor
environment. This is implemented as an example in the MySQL module.

.travis.yml

@@ -20,6 +20,7 @@ env:
     - TARGETS="-C filebeat testsuite"
     - TARGETS="-C winlogbeat testsuite"
     - TARGETS="-C packetbeat testsuite"
+    - TARGETS="-C metricbeat testsuite"
     - TARGETS="-C libbeat crosscompile"
     - TARGETS="-C winlogbeat crosscompile"
   global:
@@ -41,6 +42,8 @@ matrix:
       env: TARGETS="-C winlogbeat crosscompile"
     - os: osx
       env: TARGETS="-C libbeat testsuite"
+    - os: osx
+      env: TARGETS="-C metricbeat testsuite"
   fast_finish: true
   allow_failures:
     - env: TARGETS="-C libbeat crosscompile"

Makefile

@@ -1,6 +1,6 @@
 BUILD_DIR=build
 COVERAGE_DIR=${BUILD_DIR}/coverage
-BEATS=packetbeat topbeat filebeat winlogbeat
+BEATS=packetbeat topbeat filebeat winlogbeat metricbeat
 PROJECTS=libbeat ${BEATS}
 
 # Runs complete testsuites (unit, system, integration) for all beats,

glide.yaml

@@ -20,6 +20,7 @@ import:
 - package: github.com/samuel/go-parser
 - package: github.com/samuel/go-thrift
   version: 2187045faa54fce7f5028706ffeb2f2fc342aa7e
+  # Alternative name? gopkg.in/mgo.v2 https://github.com/Masterminds/glide/issues/221#issuecomment-179817757
 - package: labix.org/v2/mgo
 - package: github.com/tsg/gopacket
 - package: github.com/joeshaw/multierror

libbeat/scripts/Makefile

@@ -28,7 +28,7 @@ ARCH?=$(shell uname -m)
 export PATH := ./bin:$(PATH)
 export GO15VENDOREXPERIMENT=1
 GOFILES = $(shell find . -type f -name '*.go')
-GOFILES_NOVENDOR = $(shell find . -type f -name '*.go' -not -path "./vendor/*")
+GOFILES_NOVENDOR = $(shell find . -type f -name '*.go' -not -path "*/vendor/*")
 SHELL=bash
 ES_HOST?="elasticsearch"
 BUILD_DIR?=$(shell pwd)/build
@@ -44,7 +44,7 @@ PYTHON_ENV?=${BUILD_DIR}/python-env
 
 # Conditionally enable the race detector when RACE_DETECTOR=1.
 ifeq ($(RACE_DETECTOR),1)
-    RACE=-race
+	RACE=-race
 endif
 
 
@@ -181,7 +181,11 @@ testsuite:
 
 	# Runs system tests if SYSTEM_TESTS is set to true
 	if [ $(SYSTEM_TESTS) = true ]; then \
-		 $(MAKE) system-tests; \
+		if [ $(TEST_ENVIRONMENT) = true ]; then \
+			${DOCKER_COMPOSE} run beat make system-tests; \
+		else \
+			$(MAKE) system-tests; \
+		fi \
 	fi
 
 	$(MAKE) benchmark-tests
@@ -214,7 +218,7 @@ build-image: write-environment
 # To use it for running the test, set ES_HOST and REDIS_HOST environment variable to the ip of your docker-machine.
 .PHONY: start-environment
 start-environment: stop-environment
-	${DOCKER_COMPOSE} up -d redis elasticsearch logstash
+	${DOCKER_COMPOSE} up -d
 
 .PHONY: stop-environment
 stop-environment:

metricbeat/.gitignore

@@ -0,0 +1,4 @@
+build
+
+/metricbeat
+/metricbeat.test

metricbeat/CHANGELOG.md

@@ -0,0 +1 @@
+This is only a temporary changelog to keep track of changes until the first release for contributors.

metricbeat/Dockerfile

@@ -0,0 +1,23 @@
+FROM golang:1.5.3
+MAINTAINER Nicolas Ruflin <ruflin@elastic.co>
+
+RUN set -x && \
+    apt-get update && \
+    apt-get install -y netcat python-virtualenv python-pip && \
+    apt-get clean
+
+
+## Install go package dependencies
+RUN set -x \
+  go get \
+	github.com/pierrre/gotestcover \
+	github.com/tsg/goautotest \
+	golang.org/x/tools/cmd/cover \
+	golang.org/x/tools/cmd/vet
+
+# Setup work environment
+ENV METRICBEAT_PATH /go/src/github.com/elastic/beats/metricbeat
+ENV GO15VENDOREXPERIMENT=1
+
+RUN mkdir -p $METRICBEAT_PATH/build/coverage
+WORKDIR $METRICBEAT_PATH

metricbeat/Makefile

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+BEATNAME=metricbeat
+SYSTEM_TESTS=true
+TEST_ENVIRONMENT?=true
+
+include ../libbeat/scripts/Makefile

metricbeat/README.md

@@ -0,0 +1,23 @@
+# Metricbeat
+
+Metricbeat takes metrics and statistics from your systems and ships them to elasticsearch or logstash.
+
+**WARNING: Metricbeat is currently still in an experimental phase and under heavy development.**
+
+## Usage
+
+Metricbeat should be installed as local as possible so it can fetch metrics directly from the intended systems. For example if there are multiple MySQL servers, Metricbeat should be installed on each machine if possible instead of a centralised installation.
+
+## Contributions
+
+Contributions of new modules and metricsets to Metricbeat are highly welcome. To guarantee the quality of all metricsets we defined the following requirements for each Metricset:
+
+* Unit tests
+* Integration tests
+* Kibana Dashboards
+* Template
+
+Best is to start your own module as its own beat first (see below use it as a library) so you can test it and then start a discussion with our team if it would fit into Metricbeat.
+
+## Use it as library
+Metricbeat can also be used as a library so you can implement your own module on top of metricbeat and building your own beat based on it, withouth getting your module into the main repository. This allows to make use of the schedule and interfaces of Metricbeat. A developer guide and how to do this will follow soon.

metricbeat/TODO.md

@@ -0,0 +1,13 @@
+List of things to keep track on:
+
+* Create dashboard for each implemented metricset
+* validate schema with topbeat -> will it work as expected
+* send also document if remote system is not reachable -> error document
+  * What happens on error of fetch?
+* disable _all for all fields
+* disable _source? for all fields
+* Filters on metricset level
+  * Idea: Basic, Medium, All data to have predefined sets / levels
+  * Or is this done via filter implementation
+* Add fields per module / metricsets
+* Improve apache docker image

metricbeat/beater/config.go

@@ -0,0 +1,30 @@
+package beater
+
+import (
+	"github.com/elastic/beats/metricbeat/helper"
+)
+
+type MetricbeatConfig struct {
+	Metricbeat helper.ModulesConfig
+}
+
+// Raw module config to be processed later by the module
+type RawModulesConfig struct {
+	Metricbeat struct {
+		Modules map[string]interface{}
+	}
+}
+
+// Raw metric config to be processed later by the metric
+type RawMetricsConfig struct {
+	Metricbeat struct {
+		Modules map[string]struct {
+			MetricSets map[string]interface{} `yaml:"metricsets"`
+		}
+	}
+}
+
+// getModuleConfig returns config for the specified module
+func (config *MetricbeatConfig) getModuleConfig(moduleName string) helper.ModuleConfig {
+	return config.Metricbeat.Modules[moduleName]
+}

metricbeat/beater/metricbeat.go

@@ -0,0 +1,120 @@
+/*
+
+Metricbeat collects metric sets from different modules.
+
+
+Each event created has the following format:
+
+	curl -XPUT http://localhost:9200/metricbeat/metricsets -d
+	{
+		"metriset": metricsetName,
+		"module": moduleName,
+		"moduleName-metricSetName": {
+			"metric1": "value",
+			"metric2": "value",
+			"metric3": "value",
+			"nestedmetric": {
+				"metric4": "value"
+			}
+		},
+		"@timestamp": timestamp
+	}
+
+All documents are currently stored in one index called metricbeat. It is important to use an independent namespace
+for each MetricSet to prevent type conflicts. Also all values are stored under the same type "metricsets".
+
+*/
+package beater
+
+import (
+	"fmt"
+
+	"github.com/elastic/beats/libbeat/beat"
+	"github.com/elastic/beats/libbeat/cfgfile"
+	"github.com/elastic/beats/libbeat/logp"
+	"github.com/elastic/beats/metricbeat/helper"
+)
+
+type Metricbeat struct {
+	done          chan struct{}
+	MbConfig      *MetricbeatConfig
+	ModulesConfig *RawModulesConfig
+	MetricsConfig *RawMetricsConfig
+}
+
+// New creates a new Metricbeat instance
+func New() *Metricbeat {
+	return &Metricbeat{}
+}
+
+func (mb *Metricbeat) Config(b *beat.Beat) error {
+
+	mb.MbConfig = &MetricbeatConfig{}
+	err := cfgfile.Read(mb.MbConfig, "")
+	if err != nil {
+		fmt.Println(err)
+		logp.Err("Error reading configuration file: %v", err)
+		return err
+	}
+
+	mb.ModulesConfig = &RawModulesConfig{}
+	err = cfgfile.Read(mb.ModulesConfig, "")
+	if err != nil {
+		fmt.Println(err)
+		logp.Err("Error reading configuration file: %v", err)
+		return err
+	}
+
+	mb.MetricsConfig = &RawMetricsConfig{}
+	err = cfgfile.Read(mb.MetricsConfig, "")
+	if err != nil {
+		fmt.Println(err)
+		logp.Err("Error reading configuration file: %v", err)
+		return err
+	}
+
+	logp.Info("Setup base and raw configuration for Modules and Metrics")
+	// Apply the base configuration to each module and metric
+	for moduleName, module := range helper.Registry {
+		// Check if config for module exist. Only configured modules are loaded
+		if _, ok := mb.MbConfig.Metricbeat.Modules[moduleName]; !ok {
+			continue
+		}
+		module.BaseConfig = mb.MbConfig.getModuleConfig(moduleName)
+		module.RawConfig = mb.ModulesConfig.Metricbeat.Modules[moduleName]
+		module.Enabled = true
+
+		for metricSetName, metricSet := range module.MetricSets {
+			// Check if config for metricset exist. Only configured metricset are loaded
+			if _, ok := mb.MbConfig.getModuleConfig(moduleName).MetricSets[metricSetName]; !ok {
+				continue
+			}
+			metricSet.BaseConfig = mb.MbConfig.getModuleConfig(moduleName).MetricSets[metricSetName]
+			metricSet.RawConfig = mb.MetricsConfig.Metricbeat.Modules[moduleName].MetricSets[metricSetName]
+			metricSet.Enabled = true
+		}
+	}
+
+	return nil
+}
+
+func (mb *Metricbeat) Setup(b *beat.Beat) error {
+	mb.done = make(chan struct{})
+	return nil
+}
+
+func (mb *Metricbeat) Run(b *beat.Beat) error {
+
+	helper.StartModules(b)
+	<-mb.done
+
+	return nil
+}
+
+func (mb *Metricbeat) Cleanup(b *beat.Beat) error {
+	return nil
+}
+
+func (mb *Metricbeat) Stop() {
+	close(mb.done)
+}

metricbeat/docker-compose.yml

@@ -0,0 +1,36 @@
+beat:
+  build: .
+  links:
+    - apache
+    - elasticsearch
+    - mysql
+    - redis
+  environment:
+    - APACHE_HOST=apache
+    - APACHE_PORT=80
+    - REDIS_HOST=redis
+    - REDIS_PORT=6379
+    - MYSQL_DSN=root:test@tcp(mysql:3306)/
+    - MYSQL_HOST=mysql
+    - MYSQL_PORT=3306
+    - TEST_ENVIRONMENT=false
+  working_dir: /go/src/github.com/elastic/beats/metricbeat
+  volumes:
+    - ..:/go/src/github.com/elastic/beats/
+  command: make
+  entrypoint: /go/src/github.com/elastic/beats/metricbeat/docker-entrypoint.sh
+elasticsearch:
+  image: elasticsearch:2.1.0
+  command: elasticsearch -Des.network.host=0.0.0.0
+
+# Modules
+apache:
+  build: tests/environments/apache
+
+mysql:
+  image: mysql:5.7.10
+  environment:
+    - MYSQL_ROOT_PASSWORD=test
+
+redis:
+  image: redis:3.0.6