diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index b493a4501b0..a8f723253f7 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -101,6 +101,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - netflow: Fix bytes/packets counters on some devices (NSEL and Netstream). {pull}15449[15449] - netflow: Fix compatibility with some Cisco devices by changing the field `class_id` from short to long. {pull}15449[15449] - Fixed dashboard for Cisco ASA Firewall. {issue}15420[15420] {pull}15553[15553] +- Ensure all zeek timestamps include millisecond precision. {issue}14599[14599] {pull}16766[16766] - Fix s3 input hanging with GetObjectRequest API call by adding context_timeout config. {issue}15502[15502] {pull}15590[15590] - Add shared_credential_file to cloudtrail config {issue}15652[15652] {pull}15656[15656] - Fix typos in zeek notice fileset config file. {issue}15764[15764] {pull}15765[15765] diff --git a/x-pack/filebeat/module/zeek/files/ingest/pipeline.json b/x-pack/filebeat/module/zeek/files/ingest/pipeline.json index 7d3617a095e..1c47b4d0b42 100644 --- a/x-pack/filebeat/module/zeek/files/ingest/pipeline.json +++ b/x-pack/filebeat/module/zeek/files/ingest/pipeline.json @@ -2,9 +2,20 @@ "description": "Pipeline for normalizing Zeek files.log", "processors": [ { - "script": { - "lang": "painless", - "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['files']['ts'] * 1000; ctx.zeek.files.remove('ts');" + "set": { + "field": "event.created", + "value": "{{_ingest.timestamp}}" + } + }, + { + "date": { + "field": "zeek.files.ts", + "formats": ["UNIX"] + } + }, + { + "remove": { + "field": "zeek.files.ts" } }, { @@ -37,10 +48,10 @@ } } ], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } + "on_failure": [{ + "set": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } }] -} \ No newline at end of file +} diff --git a/x-pack/filebeat/module/zeek/files/test/files-json.log-expected.json b/x-pack/filebeat/module/zeek/files/test/files-json.log-expected.json index 210574f7a96..4cc0e2d38e0 100644 --- a/x-pack/filebeat/module/zeek/files/test/files-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/files/test/files-json.log-expected.json @@ -1,6 +1,6 @@ [ { - "@timestamp": 1547688796000, + "@timestamp": "2019-01-17T01:33:16.636Z", "event.dataset": "zeek.files", "event.id": "C8I0zn3r9EPbfLgta6", "event.module": "zeek", @@ -37,7 +37,7 @@ "zeek.session_id": "C8I0zn3r9EPbfLgta6" }, { - "@timestamp": 1547688801000, + "@timestamp": "2019-01-17T01:33:21.566Z", "event.dataset": "zeek.files", "event.id": "C6sjVo23iNApLnlAt6", "event.module": "zeek", diff --git a/x-pack/filebeat/module/zeek/http/ingest/pipeline.json b/x-pack/filebeat/module/zeek/http/ingest/pipeline.json index f3a94b7eb97..af771f8c745 100644 --- a/x-pack/filebeat/module/zeek/http/ingest/pipeline.json +++ b/x-pack/filebeat/module/zeek/http/ingest/pipeline.json @@ -2,9 +2,20 @@ "description": "Pipeline for normalizing Zeek http.log", "processors": [ { - "script": { - "lang": "painless", - "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['http']['ts'] * 1000; ctx.zeek.http.remove('ts');" + "set": { + "field": "event.created", + "value": "{{_ingest.timestamp}}" + } + }, + { + "date": { + "field": "zeek.http.ts", + "formats": ["UNIX"] + } + }, + { + "remove": { + "field": "zeek.http.ts" } }, { @@ -103,10 +114,10 @@ } } ], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } + "on_failure": [{ + "set": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } }] } diff --git a/x-pack/filebeat/module/zeek/http/test/http-json.log-expected.json b/x-pack/filebeat/module/zeek/http/test/http-json.log-expected.json index 3b43b05efda..20d3fedb1c7 100644 --- a/x-pack/filebeat/module/zeek/http/test/http-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/http/test/http-json.log-expected.json @@ -1,6 +1,6 @@ [ { - "@timestamp": 1547687130000, + "@timestamp": "2019-01-17T01:05:30.172Z", "destination.address": "17.253.5.203", "destination.as.number": 6185, "destination.as.organization.name": "Apple Inc.", diff --git a/x-pack/filebeat/module/zeek/notice/ingest/pipeline.json b/x-pack/filebeat/module/zeek/notice/ingest/pipeline.json index a5baa7d7dfe..b343068d6c6 100644 --- a/x-pack/filebeat/module/zeek/notice/ingest/pipeline.json +++ b/x-pack/filebeat/module/zeek/notice/ingest/pipeline.json @@ -2,9 +2,20 @@ "description": "Pipeline for normalizing Zeek notice.log", "processors": [ { - "script": { - "lang": "painless", - "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['notice']['ts'] * 1000; ctx.zeek.notice.remove('ts');" + "set": { + "field": "event.created", + "value": "{{_ingest.timestamp}}" + } + }, + { + "date": { + "field": "zeek.notice.ts", + "formats": ["UNIX"] + } + }, + { + "remove": { + "field": "zeek.notice.ts" } }, { @@ -95,10 +106,10 @@ } } ], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } + "on_failure": [{ + "set": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } }] } diff --git a/x-pack/filebeat/module/zeek/notice/test/notice-json.log-expected.json b/x-pack/filebeat/module/zeek/notice/test/notice-json.log-expected.json index 2c519aa8c54..58a59ab4d7b 100644 --- a/x-pack/filebeat/module/zeek/notice/test/notice-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/notice/test/notice-json.log-expected.json @@ -1,6 +1,6 @@ [ { - "@timestamp": 1320435875000, + "@timestamp": "2011-11-04T19:44:35.879Z", "event.dataset": "zeek.notice", "event.module": "zeek", "fileset.name": "notice", @@ -20,7 +20,7 @@ "zeek.notice.suppress_for": 3600 }, { - "@timestamp": 1551393388000, + "@timestamp": "2019-02-28T22:36:28.426Z", "destination.address": "207.154.238.205", "destination.as.number": 14061, "destination.as.organization.name": "DigitalOcean, LLC",