From 5443c8ac37ca333342611605690df13a214b128f Mon Sep 17 00:00:00 2001
From: Nicolas Ruflin <spam@ruflin.com>
Date: Tue, 12 Jun 2018 11:20:12 +0200
Subject: [PATCH] Add field docs check to heartbeat (#7276)

---
 filebeat/tests/system/filebeat.py               | 12 ------------
 heartbeat/_meta/fields.common.yml               |  2 ++
 heartbeat/docs/fields.asciidoc                  | 16 ++++++++++++++++
 heartbeat/include/fields.go                     |  2 +-
 heartbeat/monitors/active/http/_meta/fields.yml |  5 +++++
 heartbeat/tests/system/test_monitor.py          | 11 +++++++++++
 libbeat/tests/system/beat/beat.py               | 14 ++++++++++++++
 metricbeat/tests/system/metricbeat.py           | 14 --------------
 8 files changed, 49 insertions(+), 27 deletions(-)

diff --git a/filebeat/tests/system/filebeat.py b/filebeat/tests/system/filebeat.py
index 4d34de9ccca..6b217c64f4b 100644
--- a/filebeat/tests/system/filebeat.py
+++ b/filebeat/tests/system/filebeat.py
@@ -45,15 +45,3 @@ def get_registry_entry_by_path(self, path):
                         tmp_entry = entry
 
         return tmp_entry
-
-    def assert_fields_are_documented(self, evt):
-        """
-        Assert that all keys present in evt are documented in fields.yml.
-        This reads from the global fields.yml, means `make collect` has to be run before the check.
-        """
-        expected_fields, dict_fields = self.load_fields()
-        flat = self.flatten_object(evt, dict_fields)
-
-        for key in flat.keys():
-            if key not in expected_fields:
-                raise Exception("Key '{}' found in event is not documented!".format(key))
diff --git a/heartbeat/_meta/fields.common.yml b/heartbeat/_meta/fields.common.yml
index e5e5dc6e153..90688a14909 100644
--- a/heartbeat/_meta/fields.common.yml
+++ b/heartbeat/_meta/fields.common.yml
@@ -2,6 +2,8 @@
   title: "Common heartbeat monitor"
   description:
   fields:
+    - name: type
+      type: keyword
     - name: monitor
       type: group
       description: >
diff --git a/heartbeat/docs/fields.asciidoc b/heartbeat/docs/fields.asciidoc
index 0bc6cb664e7..7b9ac87393c 100644
--- a/heartbeat/docs/fields.asciidoc
+++ b/heartbeat/docs/fields.asciidoc
@@ -213,6 +213,13 @@ Region in which this host is running.
 None
 
 
+*`type`*::
++
+--
+type: keyword
+
+--
+
 [float]
 == monitor fields
 
@@ -573,6 +580,15 @@ Duration in microseconds
 
 --
 
+*`http.rtt.content.us`*::
++
+--
+type: long
+
+Time required to retrieved the content in micro seconds.
+
+--
+
 [float]
 == total fields
 
diff --git a/heartbeat/include/fields.go b/heartbeat/include/fields.go
index ae6fcca823f..f3adc029cd7 100644
--- a/heartbeat/include/fields.go
+++ b/heartbeat/include/fields.go
@@ -14,5 +14,5 @@ func init() {
 
 // Asset returns asset data
 func Asset() string {
-	return "eJzsW19z27gRf/en2MlTOyNrmqTJdPzQaeqkPc1d7jy2+yxDwErEGQQYALSsm374Dv6RoEjqj6Vc3ZnqJSYJ7C52f/jtLshcXMIjbq5ggcReAFhuBV7B38MVQ0M1ryxX8gr+egEAcK2kJVwaoKoslfTzYMlRMAPkiXBBFgKBSyBCAD6htGA3FZrpBcRhVxde0CVIUmJQPHV/+ruDOt3vvkA/AdQSbIHeQjAoGZcrf0OoFZRoDFmhmcIsG+WncdOIMmidge45VXLJV7UmTh0sucCJu+8eEgtPRNRuJtQGmZfJrbuUyubC/BQolLFRUxx/r7yqjh0T98zfenCXD40c5Vc8bte077Skcb/jGtuIAY221hIZLDZelarQqZErMBtjsQQlYV1wWrSGZ77TtZRcrgassbzE35Q8wJo08nta84TacCX3GxMHJlh5OPvgr1A6U5CBLbgJUJ52ofvmb24pxpKyehOFOqxfASM2+UHjt5prZFdgdZ1uLpUuie2Mw2dSVm7rfapXtbHw7qMt4N2f3n6cwNt3V+8/XH14P33//t1h3vUmwToAGeM2dBtEI1WawZqYdn1bi7JkZXZr+aQX3GqiN35s8BYljgo83ivUIVBEMn9hNZGGUNvGI/hpS3Fgh44f1eJXpGmvhYt5ePKIm7XSbLehDVfVBnW7pxxBBWVbFqDWSncMWGlVV7uVfHGTEgPSoNHhlzDG3VgigMulcjubEuP5y+sx0wSGyIpJYLImkllzP9lk8dlmN0fMak2LcqY9BVSxvnSh5OoY6U5IX7ST1RPdjdlB0gNMYoqiQtWszVHX7hIqrZ44Q7dMSxixZDhtfY1PYalVGSQ1U42LVUtBhLG5HzBPIt1IisYoPZrF3NCpnzVNYrc3NtI9u/fnLL11LZzCjTKGO+D6nGSAaHQCJ7CiOAGlgfEVt0QoikROR23j0lgiKc75nq0ziwNh9jmZ5JIIlIQWXG5v3SEN+zNToyPP64dpiQPmGc4aP9t30xIZr8vd2r8GER5ixymPZQ4X3G7mWcprLKjNJRJjL9/SPUSaCQKfEXmb7bgJ5nDTprkdkPPc2ES1MSU+uXw+HHpxirPln0qtBIadNq5d42pvqr31Y/atL250puij3z9xp39O1wPCwzMwllhHv0IgdTnbb/PwzO1ZUyht5yEDXMGSCOOCRiQtlE76LptdftEl5bTkxiwYzA9jPB5zAuopZ6dx4r8k/1ZjKxA4G2L1Rl05lD6O0pjjwotL1Wk0wBUSi5oLC0ruMiUjgxdact3odLJ26RJkgcL0tHVqCdhdT+yxZeY9EfQ0oHVgbiH7Q7gaEDJzxUAGVJfletTTYtPd34vMqPs4XJ4ekx9iW9GPxpmQHghiAORE04JbpLbWZ1hDRxz8AaerKTz/5eP8458nQHQ5gaqiEyh5Zf7YN0WZaSWIdSX9aZb8cgdJULSBorTKTKBe1NLWE1hzydR6xIhux/NyG6KcQR1LUnKxOVlFEBMXqZEVxE6A4YITOYGlRlwYtmu1vOqZ0Lm1Q/tP3FhHaLObS8KYRmPQ9BWUhJ62yKSmIJqticZW2QRqUxMhNvD103VuQ+KRx3qBWqJF07LJj/m9AbXt86YM7ta0rVDIuWR3Wmwn7SWgjtFwFA1Vip0hPWQeqBQL3Daoqj6VmrY0OXmD1GoqQs+3qFZiX5nrwM7qQSdxxIWHJtfDFAVpUJKqr4lIqaw//zqbukzksM5zFiyZXtqpXXapPUPJNqg3yE19tD+5bdnlzXU4yi2QaOsPwEoluVX6zRbbjGz+OHp054+eznitcXb3RGacLk4/WbgvsFGaHUPBWUujTIlpTp5wjJhO5aSOtmUtBPyqFq51J+Es2qWBJroD62XxpLlnRR7Gng33yhKR9PpjeDR2SNZ2LHPVtencHjmG6un+nM7GuYSSU60MUiWZ6a/N0AJPjeankKah1iLKm8I/lE5tNjxYWj1M4MEK4/4prHWXRLLwt3kY8HlWs7/QqlR+u0LDoH7iFGGBLhAxJsimcB0OZktuDJerCfB2LO+6vpnk0DK7GTD5hLprdrPTylluVdeS9O5i0pHn36Dw6iFgKzGd8fc1GiWekAGvIBZYTZtFa61RWi91YIXGEttB5ODx/QvjNZOMU+Johy8bBqKqFgyeiOCM2NBeJ09Y5SLXvFRrS8S4wIzBfRcjlHqsqwNJu5UBx5B2pqhh7PBkjLB/H5yfG6wtbmrZNukr/oRyDDva9te5kz8bDksgcxGPcQEinRn+GKvotdm/M6VG2BlFH82HDHV3v1z/ePfBdRTPmwNh18g4CnW5ItAo/Gu5rgvG4Hd0VLYy6093IMgGNWiPBKt5FV6xHRoNqqTsFqvjhuwxxhvES+wABo0lC8FNASTpckF84iS5zQ2SrFJcblsBsCCuPFAye3efCbGq4/rp1vShZcMuIMIuMPYWPw7IhEgrsib5jYsVSqo34VW6D9uBsLRivHEde9faIqMDyH18+F8HZEEkMwV5xO8GySWXDo/O1EZZhjShkbBNhjiJdq30Y09wi8RXg7x02Gttlaff+/ubI5umKGHY8WPJ16k5Dm21Fj20Hf4C+S5mW1f3pj4iLjOPSFkLy+f9mDSQJ+uLfiD6NcAepN1ntdGQRXBfcOPqRwJSyUsiidj8ljwVPlMI3+Ysa7GNJ6WBrFYaV+F4YCi9o6mUNP0a8IjNm/yZZEFFNCnRoj5494YKdb717r61hkuLq+Yt1UF+BbhN9gTpIy/1T2Quj97TqCuVyS9jrn/31t3s8wXaNaKEJdfGwmJjfbEZ99u32vW1oeJca24tStfb9aQ1UQ1D45lpwGi03KG0VbrFiD2BPYbsMGJv+M/KOgAsW2XxYzVw4p1JC8U2oDQoKTZAoNK45M8T/zJ2gBLdT9blAjUwhUHSshbClWCVRuO/7Stcn2KJ8IEEiciw75kYJuUNCd9ZKfZ6yokhZQlqc2fp2fGm8iB1y3/C/MFBpyMc4J7wC278PxLgOyLBbXmcRxp4ERJ24iD/hpWqshJoscM8A4zRZ4qxmuo11lBDyhLC5wUS1ktfL3RztzBNHJ873Fii7XYUnPMHyD2kAbc3syzhG/QYrQ77u3i0kdvRd/2vR87v+bMTZM6JMZE2sck/KJ3CnQuhgTW3RU+c/2xFcsuJgPvrm7y1JdZiWdkpfJEszAaytKhbyuxJY5wBLZA+djj5NdPvawFO7Jo4LfOuaXb99ebAbinOhGO6pdkNVM7XB7blYX/33zX2C+pd30uFKPEluMXBF1qo2yjYU8w5jg0byXCbcdItVg4P3cL6wLL63AeG6XSG5tF2+++oIxl6dMSdisSeLzmaqZTux+Ko+KcGz0mKW/YcId86Aro+tY8685nkIGvn55Jb3HtE49Mexb8WLvsOjekOj7aNgrsyFqvWe/jMjf+vMl33vhZH/ScAAP//+HFU/w=="
+	return "eJzsW99z47YRfvdfsXNP7YzMae56Nx0/dHp10kaTXOK5c59lCFiJiEGAAUDLyvSP7+AXCYqkfli61J2pX3Iiwd0P2A/f7oLM1TU84vYGlkjsFYDlVuAN/D38Ymio5rXlSt7AX68AAG6VtIRLA1RVlZL+OVhxFMwAeSJckKVA4BKIEIBPKC3YbY2muII47ObKG7oGSSoMjgv3T3911Kf7uy/RPwBqBbZEjxAMSsbl2l8Qag0VGkPWaAqYZ6P8Y9y0pgxaB9Ddp0qu+LrRxLmDFRc4c9fdTWLhiYjGPQmNQeZtcut+SmVzY/4RKJWx0VMcf6+8qx6OmbvnLz24nw+tHeVnPI2rGC5a8nh44VpsxIBG22iJDJZb70rV6NzINZitsViBkrApOS074Nna6UZKLtcjaCyv8Dclj0CTRn5NNE+oDVfyMJg4MNHK09kHf43SQUEGtuQmULnoU/fN39xUjCVV/SYadVy/AUZsWgeNvzZcI7sBq5t0caV0RWxvHD6TqnZb72OzboyFtx9sCW//9M2HGXzz9ubd+5v374p3794et7oeEmwCkTFuQ7dBNFKlGWyI6ea3MylL1ma/l496ya0meuvHhtWixEmB53uNOgSKSOZ/WE2kIdR28QjrtOM4qENvHdXyF6Rpr4Ufi3DnEbcbpdl+oK1WNQZ1t6ecQAVnOwhQa6V7ANZaNfV+J9+5h5IC0uDR8Zcwxt1YIoDLlXI7mxLj9cv7MUUiQ1TFZDChiWLWXk+YLD7b7OIErA5atFMMHFDFhtaFkutTrDsjQ9PO1sB0P2ZHWQ80iSmKCtWwLkfdup9Qa/XEGbppWsKIJeNp61O8CyutqmCpfdS4WHUSRBhb+AGLZNKNpGiM0pNZzA0t/FNFMru7sZEe2L0/Zemtj7CAO2UMd8T1OckA0egMzmBNcQZKA+NrbolQFIksJrFxaSyRFBf8wNaZx4Ew/zZBckkEKkJLLne37piHw5mp9ZHn9eO8xAGLjGftOtu3RYWMN9V+75+CCU+x05zHMocLbreLLOW1CBpzjcTY62/oASHNDIHPiLzLdtwEONx0aW4P5bw2tlFtocQ718/HUy8+4rD8U6m1wLDTpr1rXB9MtZ/9mEPzixudKfro90/c6d+m3yPGwz0wllgnv0IgdTnbb/Nwz+1ZUyptFyED3MCKCOOCRiQtlU7+rttdftUX5TTlFhaM5ocpHY85AXXB2Xma+C/Jf22wMwicjal6664aSx8necx54c2l6jQCcIXEsuHCgpL7oGRi8EIkt61PZ2ufL0GWKMzAW6+WgP31xAEsc78SwU9LWkfmjrLfh18jRuauGMiI6rLcQHo6brrrB5kZfZ/Gy/Nj8n1sK4bRuBDTg0CMkJxoWnKL1Db6AnPomYM/YLEu4PkvHxYf/jwDoqsZ1DWdQcVr88chFGWKWhDrSvrzkPz8BZKhiIGitMrMoFk20jYz2HDJ1GYCRL/jeTmGaGfUx4pUXGzPdhHMxElqZCWxM2C45ETOYKURl4btmy2vBxB6l/Z4/5Eb6wRtfndNGNNoDJqhg4rQ8yaZ3JREsw3R2DmbQWMaIsQWPn28zTEkHXlslqglWjSdmvyQXxtx291vy+B+TdsZhVxL9qfF7qGDAtQDDSfJUK3YBdJDtgK1YkHbRl0150rTjidnb1RaTU3o5SbVWRw6cx3YRVfQWZxYwmOT63GOgjWoSD30RKRU1p9/XcxdZnLc5yULlswv7dUu+9xeoGQb9Rvspj7an9x26vLmNhzllki09QdglZLcKv1mR20mNn/WhA3xtpIaTE7Kw+QRjocWn+4f20xryvnHD/cltk6zsyq4aP2UOTHt8RROqde5wtXztmqEgF/U0vX3JBxYu1zRUmBkviweRw9Q5GEcYLhXlojk15/Vo7FjtnZjmbtuTO/yxFnVwPe36QCdS6g41cogVZKZ4dwMLfHcaH4MuRwaLaK9Av6hdOrF4cHS+mEGD1YY95/SWveTSBb+bR5G1jwr7F+IKtXorhoxqJ84RViiC0SMCbICbsPpbcWN4XI9A96N5f2lbx9ybJnfjUA+ozib3+1FOc9R9ZGkFxyznj3/moXXD4FbSQ6Nv67RKPGEDHgNsQprezHaaI3SeqsjMzSW2B4jR8/4XxivuWScEic7fNUqEFWNYPBEBGfEhh48rYRVLnLtm7eujowTzGTetzpCqcemPlLZOxtwimhnjlrFDnemBPv34fmlydrxppFdJ7/mTyinuKPtcJ579bPVsEQyF/EYFyDSwfBnXeWgF/+dJTXSzij6aN5nrPvy8+0PX967tuN5eyTtWhsnsS53BBqFf3fXX4Ip+p0clZ3M+uMXEGSLGrRngtW8Du/hjo0GVVL2K9ppIAfAeEC8wh5h0FiyFNyUQJIvF8QnTtKyuUGS1YrLXRQAS+LKAyWzF/yZEat6S1/sPD42bdhHRNhHxsHkpwmZGGlF1km/cbFCSfU2vG/3YTu2zhXT3e3UC9mOGT1CHtLD/zohSyKZKckjfjVKrrh0fHRQW2cZ04RGwrYZ4yTajdKPA8MdE18N89KJsLV1nn7v7+9O7KyihfGFn0q+zs1pbGu0GLDt+LfMX2K2dXVv6iPiNPOIVI2wfDGMSUt5srkaBmJYAxxg2n1WG40hgvuSG1c/EpBKXhNJxPa3tFLhW4bwAc+qEbt8UhrIeq1xHc4QxtI7mlpJM6wBT9i8aT2TLaiJJhVa1Efv3lChLnZe8HdouLS4bl9lHbWuAJ8TnmB94s3/mcrl2XuedKUy+WXK9e/BvNt9vkS7QZSw4tpYWG6tLzbjfvu1cX1tqDg3mluL0vV2A2ttVMPQeLAaOBqRO5Z2TncUcWBwoJA9RRwM/0lZR4BV5yx+0QbOvIO0VGwLSoOSYgsEao0r/jzzb2xHJNH9yaZaogamMFhaNUK4EqzWaPwHgKXrUywRPpAgERkOVyaGSXkg4WMsxV5POTHmLFFt4ZBenG8qD1K//CfMHxz0OsIR7Ql/YRn/zwT4ikxwWx4XUQZexIS9PMg/dKWqqgVa7CnPiGIMlWKqpnqNNdSYs8TwRYmEDdLXC5e5X5gmjc8X3Fii7W4U3OKPiHtIA25vZlnCN+gxWj31d/HoIren7/pfjxxV0qK0xctOHQ43Exqt5viErP0+xalNggYRWzEOzgvSxdU7hxezfEuc/JPYAr44fhnYcFsOzPkPbyS3nAi4v73L+25iLVa1LeA7ycLTQFYWdafnA2uMM6Al0sdewnjNueG1sDq2dJxWeUs3v/10d2QrF5+EU1q5+R3Ubq2PPDMI4jN8Wzqs9vd98RWixFfgJgff0VJ9joa9/l3iTLO1DJ8zwfyMteNDv+o/sua/9GlmOjqiebTd/jvpvIieHHHnIkn7S86NaqWHsTgp/qn7dJbilr1EyHfOp27PbfIufGA6qtr5oemO9p7QlXXvCV6Lln2FrnnPinZdjPtlLNbd6uEzN/5/9ukv72tZqP8EAAD//wjRipo="
 }
diff --git a/heartbeat/monitors/active/http/_meta/fields.yml b/heartbeat/monitors/active/http/_meta/fields.yml
index 082c1f2e54d..6cb7f0aae32 100644
--- a/heartbeat/monitors/active/http/_meta/fields.yml
+++ b/heartbeat/monitors/active/http/_meta/fields.yml
@@ -82,6 +82,11 @@
                   type: long
                   description: Duration in microseconds
 
+            - name: content.us
+              type: long
+              description:
+                Time required to retrieved the content in micro seconds.
+
             - name: total
               type: group
               description: |
diff --git a/heartbeat/tests/system/test_monitor.py b/heartbeat/tests/system/test_monitor.py
index 6f4ff2a8e51..423eee1b6d3 100644
--- a/heartbeat/tests/system/test_monitor.py
+++ b/heartbeat/tests/system/test_monitor.py
@@ -2,6 +2,8 @@
 import BaseHTTPServer
 import threading
 from parameterized import parameterized
+import os
+from nose.plugins.skip import SkipTest
 
 
 class Test(BaseTest):
@@ -35,6 +37,11 @@ def test_http(self, status_code):
         output = self.read_output()
         assert status_code == output[0]["http.response.status_code"]
 
+        if os.name == "nt":
+            # Currently skipped on Windows as fields.yml not generated
+            raise SkipTest
+        self.assert_fields_are_documented(output[0])
+
     @parameterized.expand([
         ("8181", "up"),
         ("8182", "down"),
@@ -63,6 +70,10 @@ def test_tcp(self, port, status):
 
         output = self.read_output()
         assert status == output[0]["monitor.status"]
+        if os.name == "nt":
+            # Currently skipped on Windows as fields.yml not generated
+            raise SkipTest
+        self.assert_fields_are_documented(output[0])
 
     def start_server(self, content, status_code):
         class HTTPHandler(BaseHTTPServer.BaseHTTPRequestHandler):
diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py
index fe0ad480509..d54077c4618 100644
--- a/libbeat/tests/system/beat/beat.py
+++ b/libbeat/tests/system/beat/beat.py
@@ -579,3 +579,17 @@ def get_kibana_url(self):
             host=os.getenv("KIBANA_HOST", "localhost"),
             port=os.getenv("KIBANA_PORT", "5601"),
         )
+
+    def assert_fields_are_documented(self, evt):
+        """
+        Assert that all keys present in evt are documented in fields.yml.
+        This reads from the global fields.yml, means `make collect` has to be run before the check.
+        """
+        expected_fields, dict_fields = self.load_fields()
+        flat = self.flatten_object(evt, dict_fields)
+
+        for key in flat.keys():
+            documented = key in expected_fields
+            metaKey = key.startswith('@metadata.')
+            if not(documented or metaKey):
+                raise Exception("Key '{}' found in event is not documented!".format(key))
diff --git a/metricbeat/tests/system/metricbeat.py b/metricbeat/tests/system/metricbeat.py
index 74fe07c7ab7..7deb99a3200 100644
--- a/metricbeat/tests/system/metricbeat.py
+++ b/metricbeat/tests/system/metricbeat.py
@@ -23,20 +23,6 @@ def setUpClass(self):
         self.beat_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../"))
         super(BaseTest, self).setUpClass()
 
-    def assert_fields_are_documented(self, evt):
-        """
-        Assert that all keys present in evt are documented in fields.yml.
-        This reads from the global fields.yml, means `make collect` has to be run before the check.
-        """
-        expected_fields, _ = self.load_fields()
-        flat = self.flatten_object(evt, [])
-
-        for key in flat.keys():
-            documented = key in expected_fields
-            metaKey = key.startswith('@metadata.')
-            if not(documented or metaKey):
-                raise Exception("Key '{}' found in event is not documented!".format(key))
-
     def de_dot(self, existing_fields):
         fields = {}