Cherry-pick to 6.6: Ignore timestamp in redis, haproxy and system fil…

…ebeat module (elastic#9855) (elastic#9858)

* Ignore timestamp in redis, haproxy and system filebeat module (elastic#9855)

* Ignore timestamp in redis, haproxy and system filebeat module

* Fixing formatting

(cherry picked from commit a5f0323)

* Use event.dataset in if statement

* Update golden files for icinga/startup test to exclude timestamp (elastic#9506)

The icing/startup log file does not contain a timestamp. Because of this the timestamp from filebeat is taken. During the generation of golden files still a timestamp was added but every skipped on comparison. Instead now the timestamp is not added to the generated file anymore to now show a diff each time GENERATE is run.

* Regenerate expectations for logs without timestamps (elastic#9862)

* Rerun GENERATE=1 to update expected json files

filebeat/module/haproxy/log/test/default.log-expected.json

@@ -1,6 +1,5 @@
 [
     {
-        "@timestamp": "2018-09-20T15:42:59.000Z", 
         "event.dataset": "haproxy.log", 
         "fileset.module": "haproxy", 
         "fileset.name": "log", 

filebeat/module/haproxy/log/test/haproxy.log-expected.json

@@ -1,6 +1,5 @@
 [
     {
-        "@timestamp": "2018-07-30T09:03:52.726Z", 
         "event.dataset": "haproxy.log", 
         "fileset.module": "haproxy", 
         "fileset.name": "log", 

filebeat/module/haproxy/log/test/tcplog.log-expected.json

@@ -1,6 +1,5 @@
 [
     {
-        "@timestamp": "2018-09-20T15:44:23.285Z", 
         "event.dataset": "haproxy.log", 
         "fileset.module": "haproxy", 
         "fileset.name": "log", 

filebeat/module/icinga/startup/test/test.log-expected.json

@@ -1,6 +1,5 @@
 [
     {
-        "@timestamp": "2018-12-12T11:22:05.182Z", 
         "event.dataset": "icinga.startup", 
         "fileset.module": "icinga", 
         "fileset.name": "startup", 
@@ -12,7 +11,6 @@
         "prospector.type": "log"
     }, 
     {
-        "@timestamp": "2018-12-12T11:22:05.182Z", 
         "event.dataset": "icinga.startup", 
         "fileset.module": "icinga", 
         "fileset.name": "startup", 

filebeat/module/redis/log/test/test.log-expected.json

@@ -1,6 +1,5 @@
 [
     {
-        "@timestamp": "2018-05-30T12:23:52.442Z", 
         "event.dataset": "redis.log", 
         "fileset.module": "redis", 
         "fileset.name": "log", 
@@ -13,7 +12,6 @@
         "redis.log.role": "master"
     }, 
     {
-        "@timestamp": "2018-05-30T10:05:20.000Z", 
         "event.dataset": "redis.log", 
         "fileset.module": "redis", 
         "fileset.name": "log", 
@@ -24,7 +22,6 @@
         "redis.log.message": "0 clients connected (0 slaves), 618932 bytes in use, 0 shared objects."
     }, 
     {
-        "@timestamp": "2018-05-31T04:32:08.000Z", 
         "event.dataset": "redis.log", 
         "fileset.module": "redis", 
         "fileset.name": "log", 
@@ -35,7 +32,6 @@
         "redis.log.message": "The server is now ready to accept connections on port 6379\""
     }, 
     {
-        "@timestamp": "2017-05-30T10:57:24.000Z", 
         "event.dataset": "redis.log", 
         "fileset.module": "redis", 
         "fileset.name": "log", 

filebeat/module/system/auth/test/test.log-expected.json

@@ -1,6 +1,5 @@
 [
     {
-        "@timestamp": "2018-02-21T21:54:44.000Z", 
         "event.dataset": "system.auth", 
         "fileset.module": "system", 
         "fileset.name": "auth", 
@@ -18,7 +17,6 @@
         "system.auth.user": "vagrant"
     }, 
     {
-        "@timestamp": "2018-02-23T00:13:35.000Z", 
         "event.dataset": "system.auth", 
         "fileset.module": "system", 
         "fileset.name": "auth", 
@@ -35,7 +33,6 @@
         "system.auth.user": "vagrant"
     }, 
     {
-        "@timestamp": "2018-02-21T21:56:12.000Z", 
         "event.dataset": "system.auth", 
         "fileset.module": "system", 
         "fileset.name": "auth", 
@@ -50,7 +47,6 @@
         "system.auth.user": "test"
     }, 
     {
-        "@timestamp": "2018-02-20T08:35:22.000Z", 
         "event.dataset": "system.auth", 
         "fileset.module": "system", 
         "fileset.name": "auth", 
@@ -73,7 +69,6 @@
         "system.auth.user": "root"
     }, 
     {
-        "@timestamp": "2018-02-21T23:35:33.000Z", 
         "event.dataset": "system.auth", 
         "fileset.module": "system", 
         "fileset.name": "auth", 
@@ -89,7 +84,6 @@
         "system.auth.user": "vagrant"
     }, 
     {
-        "@timestamp": "2018-02-19T15:30:04.000Z", 
         "event.dataset": "system.auth", 
         "fileset.module": "system", 
         "fileset.name": "auth", 
@@ -102,7 +96,6 @@
         "system.auth.timestamp": "Feb 19 15:30:04"
     }, 
     {
-        "@timestamp": "2018-02-23T00:08:48.000Z", 
         "event.dataset": "system.auth", 
         "fileset.module": "system", 
         "fileset.name": "auth", 
@@ -118,7 +111,6 @@
         "system.auth.user": "vagrant"
     }, 
     {
-        "@timestamp": "2018-02-24T00:13:02.000Z", 
         "event.dataset": "system.auth", 
         "fileset.module": "system", 
         "fileset.name": "auth", 
@@ -135,7 +127,6 @@
         "system.auth.user": "tsg"
     }, 
     {
-        "@timestamp": "2018-02-22T11:47:05.000Z", 
         "event.dataset": "system.auth", 
         "fileset.module": "system", 
         "fileset.name": "auth", 
@@ -149,7 +140,6 @@
         "system.auth.timestamp": "Feb 22 11:47:05"
     }, 
     {
-        "@timestamp": "2018-02-22T11:47:05.000Z", 
         "event.dataset": "system.auth", 
         "fileset.module": "system", 
         "fileset.name": "auth", 

filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json

@@ -1,6 +1,5 @@
 [
     {
-        "@timestamp": "2018-12-13T11:35:28.000Z", 
         "event.dataset": "system.syslog", 
         "fileset.module": "system", 
         "fileset.name": "syslog", 
@@ -17,7 +16,6 @@
         "system.syslog.timestamp": "Dec 13 11:35:28"
     }, 
     {
-        "@timestamp": "2018-12-13T11:35:28.000Z", 
         "event.dataset": "system.syslog", 
         "fileset.module": "system", 
         "fileset.name": "syslog", 
@@ -31,7 +29,6 @@
         "system.syslog.timestamp": "Dec 13 11:35:28"
     }, 
     {
-        "@timestamp": "2018-04-04T03:39:57.000Z", 
         "event.dataset": "system.syslog", 
         "fileset.module": "system", 
         "fileset.name": "syslog", 

filebeat/tests/system/test_modules.py

@@ -159,6 +159,7 @@ def _test_expected_events(self, test_file, objects):
                 for k, obj in enumerate(objects):
                     objects[k] = self.flatten_object(obj, {}, "")
                     clean_keys(objects[k])
+
                 json.dump(objects, f, indent=4, sort_keys=True)
 
         with open(test_file + "-expected.json", "r") as f:
@@ -175,11 +176,6 @@ def _test_expected_events(self, test_file, objects):
                 obj = self.flatten_object(obj, {}, "")
                 clean_keys(obj)
 
-                # Remove timestamp for comparison where timestamp is not part of the log line
-                if obj["fileset.module"] == "icinga" and obj["fileset.name"] == "startup":
-                    delete_key(obj, "@timestamp")
-                    delete_key(ev, "@timestamp")
-
                 if ev == obj:
                     found = True
                     break
@@ -199,6 +195,10 @@ def clean_keys(obj):
     for key in host_keys + time_keys + other_keys:
         delete_key(obj, key)
 
+    # Remove timestamp for comparison where timestamp is not part of the log line
+    if obj["event.dataset"] in ["icinga.startup", "redis.log", "haproxy.log", "system.auth", "system.syslog"]:
+        delete_key(obj, "@timestamp")
+
 
 def delete_key(obj, key):
     if key in obj: