x-pack/filebeat/threatintel: make modules agree more with others in t…

…he beat (#30570) * x-pack/filebeat/threatintel: make ECS version format agree across modules * x-pack/filebeat/threatintel: add event.timezone field * update changelog * Update CHANGELOG.next.asciidoc Co-authored-by: Andrew Kroh <andrew.kroh@elastic.co> Co-authored-by: Andrew Kroh <andrew.kroh@elastic.co>
elastic · Feb 24, 2022 · 9cf957b · 9cf957b
1 parent eb0d928
commit 9cf957b
Show file tree

Hide file tree

Showing 26 changed files with 524 additions and 24 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -53,6 +53,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...main[Check the HEAD dif
 - Report the starting offset of the line in `log.offset` when using `filestream` instead of the end to be ECS compliant. {pull}30445[30445]
 - auditd: Prevent mapping explosion when truncated EXECVE records are ingested. {pull}30382[30382]
 - elasticsearch: fix duplicate ingest when using a common appender configuration {issue}30428[30428] {pull}30440[30440]
+- Fix ECS version string in threatintel to be consistent with other modules and add event.timezone. {issue}30499[30499] {pull}30570[30570]
 
 *Heartbeat*
 

diff --git a/x-pack/filebeat/module/threatintel/abusemalware/config/config.yml b/x-pack/filebeat/module/threatintel/abusemalware/config/config.yml
@@ -38,3 +38,10 @@ tags:
 {{end}}
 
 publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
+
+processors:
+  - add_locale: ~
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.12.0
diff --git a/x-pack/filebeat/module/threatintel/abusemalware/ingest/pipeline.yml b/x-pack/filebeat/module/threatintel/abusemalware/ingest/pipeline.yml
@@ -7,9 +7,6 @@ processors:
   - set:
       field: event.ingested
       value: "{{_ingest.timestamp}}"
-  - set:
-      field: ecs.version
-      value: "1.12"
   - set:
       field: event.kind
       value: enrichment

diff --git a/x-pack/filebeat/module/threatintel/abusemalware/test/abusechmalware.ndjson.log-expected.json b/x-pack/filebeat/module/threatintel/abusemalware/test/abusechmalware.ndjson.log-expected.json
@@ -4,6 +4,7 @@
         "event.dataset": "threatintel.abusemalware",
         "event.kind": "enrichment",
         "event.module": "threatintel",
+        "event.timezone": "-02:00",
         "event.type": "indicator",
         "fileset.name": "abusemalware",
         "input.type": "log",
@@ -37,6 +38,7 @@
         "event.dataset": "threatintel.abusemalware",
         "event.kind": "enrichment",
         "event.module": "threatintel",
+        "event.timezone": "-02:00",
         "event.type": "indicator",
         "fileset.name": "abusemalware",
         "input.type": "log",
@@ -73,6 +75,7 @@
         "event.dataset": "threatintel.abusemalware",
         "event.kind": "enrichment",
         "event.module": "threatintel",
+        "event.timezone": "-02:00",
         "event.type": "indicator",
         "fileset.name": "abusemalware",
         "input.type": "log",
@@ -106,6 +109,7 @@
         "event.dataset": "threatintel.abusemalware",
         "event.kind": "enrichment",
         "event.module": "threatintel",
+        "event.timezone": "-02:00",
         "event.type": "indicator",
         "fileset.name": "abusemalware",
         "input.type": "log",
@@ -139,6 +143,7 @@
         "event.dataset": "threatintel.abusemalware",
         "event.kind": "enrichment",
         "event.module": "threatintel",
+        "event.timezone": "-02:00",
         "event.type": "indicator",
         "fileset.name": "abusemalware",
         "input.type": "log",
@@ -173,6 +178,7 @@
         "event.dataset": "threatintel.abusemalware",
         "event.kind": "enrichment",
         "event.module": "threatintel",
+        "event.timezone": "-02:00",
         "event.type": "indicator",
         "fileset.name": "abusemalware",
         "input.type": "log",
@@ -206,6 +212,7 @@
         "event.dataset": "threatintel.abusemalware",
         "event.kind": "enrichment",
         "event.module": "threatintel",
+        "event.timezone": "-02:00",
         "event.type": "indicator",
         "fileset.name": "abusemalware",
         "input.type": "log",
@@ -239,6 +246,7 @@
         "event.dataset": "threatintel.abusemalware",
         "event.kind": "enrichment",
         "event.module": "threatintel",
+        "event.timezone": "-02:00",
         "event.type": "indicator",
         "fileset.name": "abusemalware",
         "input.type": "log",
@@ -273,6 +281,7 @@
         "event.dataset": "threatintel.abusemalware",
         "event.kind": "enrichment",
         "event.module": "threatintel",
+        "event.timezone": "-02:00",
         "event.type": "indicator",
         "fileset.name": "abusemalware",
         "input.type": "log",
@@ -306,6 +315,7 @@
         "event.dataset": "threatintel.abusemalware",
         "event.kind": "enrichment",
         "event.module": "threatintel",
+        "event.timezone": "-02:00",
         "event.type": "indicator",
         "fileset.name": "abusemalware",
         "input.type": "log",
@@ -339,6 +349,7 @@
         "event.dataset": "threatintel.abusemalware",
         "event.kind": "enrichment",
         "event.module": "threatintel",
+        "event.timezone": "-02:00",
         "event.type": "indicator",
         "fileset.name": "abusemalware",
         "input.type": "log",
@@ -372,6 +383,7 @@
         "event.dataset": "threatintel.abusemalware",
         "event.kind": "enrichment",
         "event.module": "threatintel",
+        "event.timezone": "-02:00",
         "event.type": "indicator",
         "fileset.name": "abusemalware",
         "input.type": "log",
@@ -405,6 +417,7 @@
         "event.dataset": "threatintel.abusemalware",
         "event.kind": "enrichment",
         "event.module": "threatintel",
+        "event.timezone": "-02:00",
         "event.type": "indicator",
         "fileset.name": "abusemalware",
         "input.type": "log",
@@ -438,6 +451,7 @@
         "event.dataset": "threatintel.abusemalware",
         "event.kind": "enrichment",
         "event.module": "threatintel",
+        "event.timezone": "-02:00",
         "event.type": "indicator",
         "fileset.name": "abusemalware",
         "input.type": "log",
@@ -472,6 +486,7 @@
         "event.dataset": "threatintel.abusemalware",
         "event.kind": "enrichment",
         "event.module": "threatintel",
+        "event.timezone": "-02:00",
         "event.type": "indicator",
         "fileset.name": "abusemalware",
         "input.type": "log",
@@ -505,6 +520,7 @@
         "event.dataset": "threatintel.abusemalware",
         "event.kind": "enrichment",
         "event.module": "threatintel",
+        "event.timezone": "-02:00",
         "event.type": "indicator",
         "fileset.name": "abusemalware",
         "input.type": "log",
@@ -538,6 +554,7 @@
         "event.dataset": "threatintel.abusemalware",
         "event.kind": "enrichment",
         "event.module": "threatintel",
+        "event.timezone": "-02:00",
         "event.type": "indicator",
         "fileset.name": "abusemalware",
         "input.type": "log",
@@ -572,6 +589,7 @@
         "event.dataset": "threatintel.abusemalware",
         "event.kind": "enrichment",
         "event.module": "threatintel",
+        "event.timezone": "-02:00",
         "event.type": "indicator",
         "fileset.name": "abusemalware",
         "input.type": "log",
@@ -605,6 +623,7 @@
         "event.dataset": "threatintel.abusemalware",
         "event.kind": "enrichment",
         "event.module": "threatintel",
+        "event.timezone": "-02:00",
         "event.type": "indicator",
         "fileset.name": "abusemalware",
         "input.type": "log",
@@ -638,6 +657,7 @@
         "event.dataset": "threatintel.abusemalware",
         "event.kind": "enrichment",
         "event.module": "threatintel",
+        "event.timezone": "-02:00",
         "event.type": "indicator",
         "fileset.name": "abusemalware",
         "input.type": "log",
@@ -672,6 +692,7 @@
         "event.dataset": "threatintel.abusemalware",
         "event.kind": "enrichment",
         "event.module": "threatintel",
+        "event.timezone": "-02:00",
         "event.type": "indicator",
         "fileset.name": "abusemalware",
         "input.type": "log",
@@ -705,6 +726,7 @@
         "event.dataset": "threatintel.abusemalware",
         "event.kind": "enrichment",
         "event.module": "threatintel",
+        "event.timezone": "-02:00",
         "event.type": "indicator",
         "fileset.name": "abusemalware",
         "input.type": "log",
@@ -738,6 +760,7 @@
         "event.dataset": "threatintel.abusemalware",
         "event.kind": "enrichment",
         "event.module": "threatintel",
+        "event.timezone": "-02:00",
         "event.type": "indicator",
         "fileset.name": "abusemalware",
         "input.type": "log",
@@ -771,6 +794,7 @@
         "event.dataset": "threatintel.abusemalware",
         "event.kind": "enrichment",
         "event.module": "threatintel",
+        "event.timezone": "-02:00",
         "event.type": "indicator",
         "fileset.name": "abusemalware",
         "input.type": "log",
@@ -804,6 +828,7 @@
         "event.dataset": "threatintel.abusemalware",
         "event.kind": "enrichment",
         "event.module": "threatintel",
+        "event.timezone": "-02:00",
         "event.type": "indicator",
         "fileset.name": "abusemalware",
         "input.type": "log",

diff --git a/x-pack/filebeat/module/threatintel/abuseurl/config/config.yml b/x-pack/filebeat/module/threatintel/abuseurl/config/config.yml
@@ -35,3 +35,10 @@ tags:
 {{end}}
 
 publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
+
+processors:
+  - add_locale: ~
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.12.0
diff --git a/x-pack/filebeat/module/threatintel/abuseurl/ingest/pipeline.yml b/x-pack/filebeat/module/threatintel/abuseurl/ingest/pipeline.yml
@@ -7,9 +7,6 @@ processors:
   - set:
       field: event.ingested
       value: "{{_ingest.timestamp}}"
-  - set:
-      field: ecs.version
-      value: "1.12"
   - set:
       field: event.kind
       value: enrichment