[Filebeat] Improve ECS categorization field mappings for mssql module (…

…elastic#17376) Improve ECS categorization field mappings for mssql module. - event.kind - event.category - event.type Closes elastic#16171 (cherry picked from commit e3c72b3)
leehinman · Apr 1, 2020 · 4074784 · 4074784
1 parent a83ff8c
commit 4074784
Show file tree

Hide file tree

Showing 5 changed files with 157 additions and 59 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -289,6 +289,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add source field in k8s events {pull}17209[17209]
 - Improve AWS cloudtrail field mappings {issue}16086[16086] {issue}16110[16110] {pull}17155[17155]
 - Move azure-eventhub input to GA. {issue}15671[15671] {pull}17313[17313]
+- Improve ECS categorization field mappings for mssql module. {issue}16171[16171] {pull}17376[17376]
 
 *Heartbeat*
 

diff --git a/x-pack/filebeat/module/mssql/log/ingest/pipeline.json b/x-pack/filebeat/module/mssql/log/ingest/pipeline.json
diff --git a/x-pack/filebeat/module/mssql/log/ingest/pipeline.yml b/x-pack/filebeat/module/mssql/log/ingest/pipeline.yml
@@ -0,0 +1,50 @@
+description: Pipeline to parse MSSQL logs
+processors:
+- grok:
+    field: message
+    patterns:
+    - '%{MSSQL_DATE:date} %{DATA:mssql.log.origin} [ ]*%{GREEDYDATA:msg_temp}'
+    pattern_definitions:
+      MSSQL_DATE: '%{DATA} %{DATA}'
+- date:
+    if: ctx.event.timezone == null
+    field: date
+    formats:
+    - yyyy-MM-dd HH:mm:ss.SS
+    on_failure:
+    - append:
+        field: error.message
+        value: '{{ _ingest.on_failure_message }}'
+- date:
+    if: ctx.event.timezone != null
+    field: date
+    formats:
+    - yyyy-MM-dd HH:mm:ss.SS
+    timezone: '{{ event.timezone }}'
+    on_failure:
+    - append:
+        field: error.message
+        value: '{{ _ingest.on_failure_message }}'
+- remove:
+    field: date
+    ignore_missing: true
+- rename:
+    field: message
+    target_field: log.original
+- rename:
+    field: msg_temp
+    target_field: message
+    ignore_missing: true
+- set:
+    field: event.kind
+    value: event
+- append:
+    field: event.category
+    value: database
+- append:
+    field: event.type
+    value: info
+on_failure:
+- set:
+    field: error.message
+    value: '{{ _ingest.on_failure_message }}'
diff --git a/x-pack/filebeat/module/mssql/log/manifest.yml b/x-pack/filebeat/module/mssql/log/manifest.yml
@@ -11,5 +11,5 @@ var:
     os.linux:
       - /var/opt/mssql/log/error*
 
-ingest_pipeline: ingest/pipeline.json
+ingest_pipeline: ingest/pipeline.yml
 input: config/config.yml
diff --git a/x-pack/filebeat/module/mssql/log/test/test.log-expected.json b/x-pack/filebeat/module/mssql/log/test/test.log-expected.json
@@ -1,9 +1,16 @@
 [
     {
         "@timestamp": "2019-05-03T09:01:09.990-02:00",
+        "event.category": [
+            "database"
+        ],
         "event.dataset": "mssql.log",
+        "event.kind": "event",
         "event.module": "mssql",
         "event.timezone": "-02:00",
+        "event.type": [
+            "info"
+        ],
         "fileset.name": "log",
         "input.type": "log",
         "log.flags": [
@@ -17,9 +24,16 @@
     },
     {
         "@timestamp": "2019-05-03T09:01:09.990-02:00",
+        "event.category": [
+            "database"
+        ],
         "event.dataset": "mssql.log",
+        "event.kind": "event",
         "event.module": "mssql",
         "event.timezone": "-02:00",
+        "event.type": [
+            "info"
+        ],
         "fileset.name": "log",
         "input.type": "log",
         "log.offset": 226,
@@ -30,9 +44,16 @@
     },
     {
         "@timestamp": "2019-05-03T09:01:09.990-02:00",
+        "event.category": [
+            "database"
+        ],
         "event.dataset": "mssql.log",
+        "event.kind": "event",
         "event.module": "mssql",
         "event.timezone": "-02:00",
+        "event.type": [
+            "info"
+        ],
         "fileset.name": "log",
         "input.type": "log",
         "log.offset": 282,
@@ -43,9 +64,16 @@
     },
     {
         "@timestamp": "2019-05-03T09:01:09.990-02:00",
+        "event.category": [
+            "database"
+        ],
         "event.dataset": "mssql.log",
+        "event.kind": "event",
         "event.module": "mssql",
         "event.timezone": "-02:00",
+        "event.type": [
+            "info"
+        ],
         "fileset.name": "log",
         "input.type": "log",
         "log.offset": 344,
@@ -56,9 +84,16 @@
     },
     {
         "@timestamp": "2019-05-03T09:01:10.000-02:00",
+        "event.category": [
+            "database"
+        ],
         "event.dataset": "mssql.log",
+        "event.kind": "event",
         "event.module": "mssql",
         "event.timezone": "-02:00",
+        "event.type": [
+            "info"
+        ],
         "fileset.name": "log",
         "input.type": "log",
         "log.offset": 400,
@@ -69,9 +104,16 @@
     },
     {
         "@timestamp": "2019-05-03T09:01:10.000-02:00",
+        "event.category": [
+            "database"
+        ],
         "event.dataset": "mssql.log",
+        "event.kind": "event",
         "event.module": "mssql",
         "event.timezone": "-02:00",
+        "event.type": [
+            "info"
+        ],
         "fileset.name": "log",
         "input.type": "log",
         "log.offset": 462,
@@ -82,9 +124,16 @@
     },
     {
         "@timestamp": "2019-05-03T09:01:10.000-02:00",
+        "event.category": [
+            "database"
+        ],
         "event.dataset": "mssql.log",
+        "event.kind": "event",
         "event.module": "mssql",
         "event.timezone": "-02:00",
+        "event.type": [
+            "info"
+        ],
         "fileset.name": "log",
         "input.type": "log",
         "log.flags": [
@@ -98,9 +147,16 @@
     },
     {
         "@timestamp": "2019-05-03T09:01:10.000-02:00",
+        "event.category": [
+            "database"
+        ],
         "event.dataset": "mssql.log",
+        "event.kind": "event",
         "event.module": "mssql",
         "event.timezone": "-02:00",
+        "event.type": [
+            "info"
+        ],
         "fileset.name": "log",
         "input.type": "log",
         "log.offset": 734,
@@ -111,9 +167,16 @@
     },
     {
         "@timestamp": "2019-05-03T09:01:10.000-02:00",
+        "event.category": [
+            "database"
+        ],
         "event.dataset": "mssql.log",
+        "event.kind": "event",
         "event.module": "mssql",
         "event.timezone": "-02:00",
+        "event.type": [
+            "info"
+        ],
         "fileset.name": "log",
         "input.type": "log",
         "log.offset": 1011,
@@ -124,9 +187,16 @@
     },
     {
         "@timestamp": "2019-05-03T09:01:10.000-02:00",
+        "event.category": [
+            "database"
+        ],
         "event.dataset": "mssql.log",
+        "event.kind": "event",
         "event.module": "mssql",
         "event.timezone": "-02:00",
+        "event.type": [
+            "info"
+        ],
         "fileset.name": "log",
         "input.type": "log",
         "log.offset": 1166,
@@ -137,9 +207,16 @@
     },
     {
         "@timestamp": "2019-05-03T09:01:10.000-02:00",
+        "event.category": [
+            "database"
+        ],
         "event.dataset": "mssql.log",
+        "event.kind": "event",
         "event.module": "mssql",
         "event.timezone": "-02:00",
+        "event.type": [
+            "info"
+        ],
         "fileset.name": "log",
         "input.type": "log",
         "log.offset": 1289,
@@ -150,9 +227,16 @@
     },
     {
         "@timestamp": "2019-05-03T09:01:10.010-02:00",
+        "event.category": [
+            "database"
+        ],
         "event.dataset": "mssql.log",
+        "event.kind": "event",
         "event.module": "mssql",
         "event.timezone": "-02:00",
+        "event.type": [
+            "info"
+        ],
         "fileset.name": "log",
         "input.type": "log",
         "log.offset": 1373,
@@ -163,9 +247,16 @@
     },
     {
         "@timestamp": "2019-05-03T09:01:10.200-02:00",
+        "event.category": [
+            "database"
+        ],
         "event.dataset": "mssql.log",
+        "event.kind": "event",
         "event.module": "mssql",
         "event.timezone": "-02:00",
+        "event.type": [
+            "info"
+        ],
         "fileset.name": "log",
         "input.type": "log",
         "log.offset": 1435,
@@ -176,9 +267,16 @@
     },
     {
         "@timestamp": "2019-05-03T09:01:11.930-02:00",
+        "event.category": [
+            "database"
+        ],
         "event.dataset": "mssql.log",
+        "event.kind": "event",
         "event.module": "mssql",
         "event.timezone": "-02:00",
+        "event.type": [
+            "info"
+        ],
         "fileset.name": "log",
         "input.type": "log",
         "log.offset": 1528,
@@ -189,9 +287,16 @@
     },
     {
         "@timestamp": "2019-05-03T09:01:12.030-02:00",
+        "event.category": [
+            "database"
+        ],
         "event.dataset": "mssql.log",
+        "event.kind": "event",
         "event.module": "mssql",
         "event.timezone": "-02:00",
+        "event.type": [
+            "info"
+        ],
         "fileset.name": "log",
         "input.type": "log",
         "log.offset": 1599,