Backport jolokia module to 5.4

* Add Http helper changes
* Add changes to handle empty fields in python scripts

CHANGELOG.asciidoc

@@ -60,6 +60,7 @@ https://github.com/elastic/beats/compare/v5.3.0...master[Check the HEAD diff]
 *Heartbeat*
 
 *Metricbeat*
+- Add beta Jolokia module. {pull}3844[3844]
 
 *Packetbeat*
 

libbeat/scripts/generate_index_pattern.py

@@ -19,6 +19,9 @@
 
 def fields_to_json(section, path, output):
 
+    if not section["fields"]:
+        return
+
     for field in section["fields"]:
         if path == "":
             newpath = field["name"]

libbeat/scripts/generate_template.py

@@ -135,6 +135,10 @@ def dedot(group):
     fields = []
     dedotted = {}
 
+    if not group["fields"]:
+        return
+
+
     for field in group["fields"]:
         if "." in field["name"]:
             # dedot
@@ -169,8 +173,10 @@ def fill_section_properties(args, section, defaults, path):
     properties = {}
     dynamic_templates = []
 
-    if "fields" in section:
+    if "fields" in section and section["fields"]:
         for field in section["fields"]:
+            if not field:
+                continue
             prop, dynamic = fill_field_properties(args, field, defaults, path)
             properties.update(prop)
             dynamic_templates.extend(dynamic)
@@ -186,6 +192,9 @@ def fill_field_properties(args, field, defaults, path):
     properties = {}
     dynamic_templates = []
 
+    if not field:
+        return
+
     for key in defaults.keys():
         if key not in field:
             field[key] = defaults[key]

libbeat/tests/system/beat/beat.py

@@ -390,6 +390,10 @@ def load_fields(self, fields_doc="../../_meta/fields.generated.yml"):
         def extract_fields(doc_list, name):
             fields = []
             dictfields = []
+
+            if not doc_list:
+                return fields, dictfields
+
             for field in doc_list:
 
                 # Chain together names

metricbeat/docker-compose.yml

@@ -18,6 +18,7 @@ services:
       - ceph
       - couchbase
       - haproxy
+      - jolokia
       - kafka
       - mongodb
       - mysql
@@ -34,6 +35,7 @@ services:
       - ${PWD}/module/couchbase/_meta/env
       - ${PWD}/module/haproxy/_meta/env
       - ${PWD}/module/kafka/_meta/env
+      - ${PWD}/module/jolokia/_meta/env
       - ${PWD}/module/mongodb/_meta/env
       - ${PWD}/module/mysql/_meta/env
       - ${PWD}/module/nginx/_meta/env
@@ -56,6 +58,9 @@ services:
   haproxy:
     build: ${PWD}/module/haproxy/_meta
 
+  jolokia:
+    build: ${PWD}/module/jolokia/_meta
+
   kafka:
     build: ${PWD}/module/kafka/_meta
 

metricbeat/docs/fields.asciidoc

@@ -20,6 +20,7 @@ grouped in the following categories:
 * <<exported-fields-couchbase>>
 * <<exported-fields-docker>>
 * <<exported-fields-haproxy>>
+* <<exported-fields-jolokia>>
 * <<exported-fields-kafka>>
 * <<exported-fields-mongodb>>
 * <<exported-fields-mysql>>
@@ -2804,6 +2805,20 @@ type: integer
 The average queue time in ms over the last 1024 requests.
 
 
+[[exported-fields-jolokia]]
+== Jolokia Fields
+
+[]beta
+Jolokia Module
+
+
+
+[float]
+== jolokia Fields
+
+jolokia contains metrics exposed via jolokia agent
+
+
 [[exported-fields-kafka]]
 == kafka Fields
 

metricbeat/docs/modules/jolokia.asciidoc

@@ -0,0 +1,43 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[metricbeat-module-jolokia]]
+== Jolokia Module
+
+beta[]
+
+This is the Jolokia Module.
+
+
+
+[float]
+=== Example Configuration
+
+The Jolokia module supports the standard configuration options that are described
+in <<configuration-metricbeat>>. Here is an example configuration:
+
+[source,yaml]
+----
+metricbeat.modules:
+#- module: jolokia
+#  metricsets: ["jmx"]
+#  enabled: true
+#  period: 10s
+#  hosts: ["localhost"]
+#  namespace: "metrics"
+#  path: "/jolokia/?ignoreErrors=true&canonicalNaming=false"
+#  jmx.mapping:
+#  jmx.application:
+#  jmx.instance:
+----
+
+[float]
+=== Metricsets
+
+The following metricsets are available:
+
+* <<metricbeat-metricset-jolokia-jmx,jmx>>
+
+include::jolokia/jmx.asciidoc[]
+

metricbeat/docs/modules/jolokia/jmx.asciidoc

@@ -0,0 +1,19 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[metricbeat-metricset-jolokia-jmx]]
+include::../../../module/jolokia/jmx/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-jolokia,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/jolokia/jmx/_meta/data.json[]
+----

metricbeat/docs/modules_list.asciidoc

@@ -7,6 +7,7 @@ This file is generated! See scripts/docs_collector.py
   * <<metricbeat-module-couchbase,Couchbase>>
   * <<metricbeat-module-docker,Docker>>
   * <<metricbeat-module-haproxy,HAProxy>>
+  * <<metricbeat-module-jolokia,Jolokia>>
   * <<metricbeat-module-kafka,kafka>>
   * <<metricbeat-module-mongodb,MongoDB>>
   * <<metricbeat-module-mysql,MySQL>>
@@ -26,6 +27,7 @@ include::modules/ceph.asciidoc[]
 include::modules/couchbase.asciidoc[]
 include::modules/docker.asciidoc[]
 include::modules/haproxy.asciidoc[]
+include::modules/jolokia.asciidoc[]
 include::modules/kafka.asciidoc[]
 include::modules/mongodb.asciidoc[]
 include::modules/mysql.asciidoc[]

metricbeat/include/list.go

@@ -31,6 +31,8 @@ import (
 	_ "github.com/elastic/beats/metricbeat/module/haproxy"
 	_ "github.com/elastic/beats/metricbeat/module/haproxy/info"
 	_ "github.com/elastic/beats/metricbeat/module/haproxy/stat"
+	_ "github.com/elastic/beats/metricbeat/module/jolokia"
+	_ "github.com/elastic/beats/metricbeat/module/jolokia/jmx"
 	_ "github.com/elastic/beats/metricbeat/module/kafka"
 	_ "github.com/elastic/beats/metricbeat/module/kafka/consumergroup"
 	_ "github.com/elastic/beats/metricbeat/module/kafka/partition"

metricbeat/metricbeat.full.yml

@@ -128,6 +128,18 @@ metricbeat.modules:
   #period: 10s
   #hosts: ["tcp://127.0.0.1:14567"]
 
+#------------------------------- Jolokia Module ------------------------------
+#- module: jolokia
+#  metricsets: ["jmx"]
+#  enabled: true
+#  period: 10s
+#  hosts: ["localhost"]
+#  namespace: "metrics"
+#  path: "/jolokia/?ignoreErrors=true&canonicalNaming=false"
+#  jmx.mapping:
+#  jmx.application:
+#  jmx.instance:
+
 #-------------------------------- kafka Module -------------------------------
 #- module: kafka
   #metricsets: ["partition"]

metricbeat/module/jolokia/_meta/Dockerfile

@@ -0,0 +1,11 @@
+# Tomcat is started to fetch Jolokia metrics from it
+FROM jolokia/java-jolokia:7
+ENV TOMCAT_VERSION 7.0.55
+ENV TC apache-tomcat-${TOMCAT_VERSION}
+
+HEALTHCHECK CMD curl -f curl localhost:8778/jolokia/
+EXPOSE 8778
+RUN wget http://archive.apache.org/dist/tomcat/tomcat-7/v${TOMCAT_VERSION}/bin/${TC}.tar.gz
+RUN tar xzf ${TC}.tar.gz -C /opt
+
+CMD env CATALINA_OPTS=$(jolokia_opts) /opt/${TC}/bin/catalina.sh run

metricbeat/module/jolokia/_meta/config.yml

@@ -0,0 +1,10 @@
+#- module: jolokia
+#  metricsets: ["jmx"]
+#  enabled: true
+#  period: 10s
+#  hosts: ["localhost"]
+#  namespace: "metrics"
+#  path: "/jolokia/?ignoreErrors=true&canonicalNaming=false"
+#  jmx.mapping:
+#  jmx.application:
+#  jmx.instance:

metricbeat/module/jolokia/_meta/docs.asciidoc

@@ -0,0 +1,6 @@
+== Jolokia Module
+
+beta[]
+
+This is the Jolokia Module.
+

metricbeat/module/jolokia/_meta/env

@@ -0,0 +1,2 @@
+JOLOKIA_HOST=jolokia
+JOLOKIA_PORT=8778

metricbeat/module/jolokia/_meta/fields.yml

@@ -0,0 +1,13 @@
+- key: jolokia
+  title: "Jolokia"
+  description: >
+    []beta
+
+    Jolokia Module
+  short_config: false
+  fields:
+    - name: jolokia
+      type: group
+      description: >
+          jolokia contains metrics exposed via jolokia agent
+      fields:

metricbeat/module/jolokia/doc.go

@@ -0,0 +1,4 @@
+/*
+Package jolokia is a Metricbeat module that contains MetricSets.
+*/
+package jolokia

metricbeat/module/jolokia/jmx/_meta/data.json

@@ -0,0 +1,34 @@
+{
+    "@timestamp": "2016-05-23T08:05:34.853Z",
+    "beat": {
+        "hostname": "host.example.com",
+        "name": "host.example.com"
+    },
+    "jolokia": {
+        "testnamespace": {
+            "memory": {
+                "heap_usage": {
+                    "committed": 1.09051904e+08,
+                    "init": 3.2753408e+07,
+                    "max": 6.20756992e+08,
+                    "used": 5.8796168e+07
+                },
+                "non_heap_usage": {
+                    "committed": 3.244032e+07,
+                    "init": 2.4576e+07,
+                    "max": 2.24395264e+08,
+                    "used": 1.7975176e+07
+                }
+            },
+            "uptime": 6.1802139e+07
+        }
+    },
+    "metricset": {
+        "host": "127.0.0.1:8778",
+        "module": "jolokia",
+        "name": "jmx",
+        "namespace": "testnamespace",
+        "rtt": 115
+    },
+    "type": "metricsets"
+}

metricbeat/module/jolokia/jmx/_meta/docs.asciidoc

@@ -0,0 +1,49 @@
+=== jolokia jmx MetricSet
+
+This is the jmx metricset of the module jolokia.
+
+[float]
+=== Features and configuration
+Tested with Jolokia 1.3.4.
+
+Metrics to be collected from each Jolokia instance are defined in the mapping section with an MBean ObjectName and
+an array of Attributes to be requested with Elastic field names under which the return values should be saved.
+
+For example: to get the "Uptime" attribute from the "java.lang:type=Runtime" MBean and map it to something like
+"uptime" (actually "jolokia.jmx.uptime", the prexif is added by beats framework) you have to configure following
+mapping:
+
+```
+- module: jolokia
+  metricsets: ["jmx"]
+  hosts: ["localhost:8778"]
+  namespace: "metrics"
+  jmx.mappings:
+    - mbean: 'java.lang:type=Runtime'
+      attributes:
+        - attr: Uptime
+          field: uptime
+```
+
+In case the underlying attribute is an object (e.g. see HeapMemoryUsage attribute in java.lang:type=Memory) it`s
+structure will be published to Elastic "as is".
+
+It is possible to configure nested metric aliases by using dots in the mapping name (e.g. gc.cms_collection_time). For examples please refer to the
+/jolokia/jmx/test/config.yml.
+
+All metrics from a single mapping will be POSTed to the defined host/port and sent to Elastic as a single event.
+To make it possible to differentiate between metrics from multiple similar applications running on the same host,
+please configure multiple modules.
+
+It is required to set a namespace in the general module config section.
+
+[float]
+=== Limitations
+No authentication against Jolokia is supported yet. No wildcards in Jolokia requests supported yet.
+All Jolokia requests have canonicalNaming set to false (details see here: https://jolokia.org/reference/html/protocol.html).
+
+
+[float]
+=== Exposed fields, Dashboards, Indexes, etc.
+Since this is a very general module that can be tailored for any application that exposes it's metrics over Jolokia, it
+comes with no exposed fields description, dashboards or index patterns.