Fix zeek pipelines to include millisecond precision (#16766) (#16769)

* Fix zeek pipelines to include millisecond precision * Add changelog entry (cherry picked from commit 2c310c0)
elastic · Mar 3, 2020 · 42353ad · 42353ad
1 parent fff389b
commit 42353ad
Show file tree

Hide file tree

Showing 7 changed files with 64 additions and 30 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -101,6 +101,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - netflow: Fix bytes/packets counters on some devices (NSEL and Netstream). {pull}15449[15449]
 - netflow: Fix compatibility with some Cisco devices by changing the field `class_id` from short to long. {pull}15449[15449]
 - Fixed dashboard for Cisco ASA Firewall. {issue}15420[15420] {pull}15553[15553]
+- Ensure all zeek timestamps include millisecond precision. {issue}14599[14599] {pull}16766[16766]
 - Fix s3 input hanging with GetObjectRequest API call by adding context_timeout config. {issue}15502[15502] {pull}15590[15590]
 - Add shared_credential_file to cloudtrail config {issue}15652[15652] {pull}15656[15656]
 - Fix typos in zeek notice fileset config file. {issue}15764[15764] {pull}15765[15765]

diff --git a/x-pack/filebeat/module/zeek/files/ingest/pipeline.json b/x-pack/filebeat/module/zeek/files/ingest/pipeline.json
@@ -2,9 +2,20 @@
  "description": "Pipeline for normalizing Zeek files.log",
  "processors": [
  {
- "script": {
- "lang": "painless",
- "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['files']['ts'] * 1000; ctx.zeek.files.remove('ts');"
+ "set": {
+ "field": "event.created",
+ "value": "{{_ingest.timestamp}}"
+ }
+ },
+ {
+ "date": {
+ "field": "zeek.files.ts",
+ "formats": ["UNIX"]
+ }
+ },
+ {
+ "remove": {
+ "field": "zeek.files.ts"
  }
  },
  {
@@ -37,10 +48,10 @@
  }
  }
  ],
- "on_failure" : [{ 
- "set" : { 
- "field" : "error.message", 
- "value" : "{{ _ingest.on_failure_message }}" 
- } 
+ "on_failure": [{
+ "set": {
+ "field": "error.message",
+ "value": "{{ _ingest.on_failure_message }}"
+ }
  }]
-}
+}
diff --git a/x-pack/filebeat/module/zeek/files/test/files-json.log-expected.json b/x-pack/filebeat/module/zeek/files/test/files-json.log-expected.json
@@ -1,6 +1,6 @@
 [
  {
- "@timestamp": 1547688796000,
+ "@timestamp": "2019-01-17T01:33:16.636Z",
  "event.dataset": "zeek.files",
  "event.id": "C8I0zn3r9EPbfLgta6",
  "event.module": "zeek",
@@ -37,7 +37,7 @@
  "zeek.session_id": "C8I0zn3r9EPbfLgta6"
  },
  {
- "@timestamp": 1547688801000,
+ "@timestamp": "2019-01-17T01:33:21.566Z",
  "event.dataset": "zeek.files",
  "event.id": "C6sjVo23iNApLnlAt6",
  "event.module": "zeek",

diff --git a/x-pack/filebeat/module/zeek/http/ingest/pipeline.json b/x-pack/filebeat/module/zeek/http/ingest/pipeline.json
@@ -2,9 +2,20 @@
  "description": "Pipeline for normalizing Zeek http.log",
  "processors": [
  {
- "script": {
- "lang": "painless",
- "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['http']['ts'] * 1000; ctx.zeek.http.remove('ts');"
+ "set": {
+ "field": "event.created",
+ "value": "{{_ingest.timestamp}}"
+ }
+ },
+ {
+ "date": {
+ "field": "zeek.http.ts",
+ "formats": ["UNIX"]
+ }
+ },
+ {
+ "remove": {
+ "field": "zeek.http.ts"
  }
  },
  {
@@ -103,10 +114,10 @@
  }
  }
  ],
- "on_failure" : [{ 
- "set" : { 
- "field" : "error.message", 
- "value" : "{{ _ingest.on_failure_message }}" 
- } 
+ "on_failure": [{
+ "set": {
+ "field": "error.message",
+ "value": "{{ _ingest.on_failure_message }}"
+ }
  }]
 }
diff --git a/x-pack/filebeat/module/zeek/http/test/http-json.log-expected.json b/x-pack/filebeat/module/zeek/http/test/http-json.log-expected.json
@@ -1,6 +1,6 @@
 [
  {
- "@timestamp": 1547687130000,
+ "@timestamp": "2019-01-17T01:05:30.172Z",
  "destination.address": "17.253.5.203",
  "destination.as.number": 6185,
  "destination.as.organization.name": "Apple Inc.",

diff --git a/x-pack/filebeat/module/zeek/notice/ingest/pipeline.json b/x-pack/filebeat/module/zeek/notice/ingest/pipeline.json
@@ -2,9 +2,20 @@
  "description": "Pipeline for normalizing Zeek notice.log",
  "processors": [
  {
- "script": {
- "lang": "painless",
- "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = (long)ctx['zeek']['notice']['ts'] * 1000; ctx.zeek.notice.remove('ts');"
+ "set": {
+ "field": "event.created",
+ "value": "{{_ingest.timestamp}}"
+ }
+ },
+ {
+ "date": {
+ "field": "zeek.notice.ts",
+ "formats": ["UNIX"]
+ }
+ },
+ {
+ "remove": {
+ "field": "zeek.notice.ts"
  }
  },
  {
@@ -95,10 +106,10 @@
  }
  }
  ],
- "on_failure" : [{ 
- "set" : { 
- "field" : "error.message", 
- "value" : "{{ _ingest.on_failure_message }}" 
- } 
+ "on_failure": [{
+ "set": {
+ "field": "error.message",
+ "value": "{{ _ingest.on_failure_message }}"
+ }
  }]
 }
diff --git a/x-pack/filebeat/module/zeek/notice/test/notice-json.log-expected.json b/x-pack/filebeat/module/zeek/notice/test/notice-json.log-expected.json
@@ -1,6 +1,6 @@
 [
  {
- "@timestamp": 1320435875000,
+ "@timestamp": "2011-11-04T19:44:35.879Z",
  "event.dataset": "zeek.notice",
  "event.module": "zeek",
  "fileset.name": "notice",
@@ -20,7 +20,7 @@
  "zeek.notice.suppress_for": 3600
  },
  {
- "@timestamp": 1551393388000,
+ "@timestamp": "2019-02-28T22:36:28.426Z",
  "destination.address": "207.154.238.205",
  "destination.as.number": 14061,
  "destination.as.organization.name": "DigitalOcean, LLC",