[Filebeat] Improve ECS categorization field mappings for azure module (…

…#19376) * Improve ECS categorization field mappings in azure module - activitylogs + convert pipeline to yml + add azure.activitylogs.result_type + set default_field: false + populate event.outcome with allowed values + set event.action + populate event.category with allowed values + set event.kind + set event.type + add support tickets example + add geoip for source.ip + add AS info for source.ip + add user.name + add user.full_name + add user.domain + update dashboards - auditlogs + convert pipeline to yml + set default_field: false + add azure.auditlogs.category + populate event.outcome with allowed values + set event.action + set event.kind + update dashboards - signinlogs + convert pipeline to yml + set default_field: false + set event.action + populate event.category with allowed values + set event.type + populate event.outcome with allowed values + add azure.signinlogs.category + add azure.signinlogs.result_type + set user.name + set user.domain + set user.full_name + set user.id + add geoip for source.ip + add AS info for source.ip + update dashboards Closes #16155
elastic · Jul 8, 2020 · 00a274e · 00a274e
1 parent 03bedd7
commit 00a274e
Show file tree

Hide file tree

Showing 25 changed files with 939 additions and 986 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -437,6 +437,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Changed the panw module to pass through (rather than drop) message types other than threat and traffic. {issue}16815[16815] {pull}19375[19375]
 - Add support for timezone offsets and `Z` to decode_cef timestamp parser. {pull}19346[19346]
 - Improve ECS categorization field mappings in traefik module. {issue}16183[16183] {pull}19379[19379]
+- Improve ECS categorization field mappings in azure module. {issue}16155[16155] {pull}19376[19376]
 
 *Heartbeat*
 

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -2378,6 +2378,16 @@ type: keyword
 Operation name
 
 
+type: keyword
+
+--
+
+*`azure.activitylogs.result_type`*::
++
+--
+Result type
+
+
 type: keyword
 
 --
@@ -2398,6 +2408,16 @@ type: keyword
 Category
 
 
+type: keyword
+
+--
+
+*`azure.activitylogs.event_category`*::
++
+--
+Event Category
+
+
 type: keyword
 
 --
@@ -2436,6 +2456,16 @@ Fields for Azure audit logs.
 
 
 
+*`azure.auditlogs.category`*::
++
+--
+The category of the operation.  Currently, Audit is the only supported value.
+
+
+type: keyword
+
+--
+
 *`azure.auditlogs.operation_name`*::
 +
 --
@@ -2831,6 +2861,16 @@ type: keyword
 Result description
 
 
+type: keyword
+
+--
+
+*`azure.signinlogs.result_type`*::
++
+--
+Result type
+
+
 type: keyword
 
 --
@@ -2841,6 +2881,16 @@ type: keyword
 Identity
 
 
+type: keyword
+
+--
+
+*`azure.signinlogs.category`*::
++
+--
+Category
+
+
 type: keyword
 
 --

diff --git a/x-pack/filebeat/module/azure/_meta/kibana/7/dashboard/Filebeat-azure-alerts-overview.json b/x-pack/filebeat/module/azure/_meta/kibana/7/dashboard/Filebeat-azure-alerts-overview.json
@@ -245,7 +245,7 @@
             "default_timefield": "@timestamp",
             "filter": {
               "language": "kuery",
-              "query": "event.dataset :\"azure.activitylogs\" and event.category : \"Alert\""
+              "query": "event.dataset :\"azure.activitylogs\" and azure.activitylogs.event_category : \"Alert\""
             },
             "id": "61ca57f0-469d-11e7-af02-69e470af7417",
             "index_pattern": "filebeat-*",
@@ -259,7 +259,7 @@
                 "fill": 0.5,
                 "filter": {
                   "language": "kuery",
-                  "query": "event.outcome: \"Activated\""
+                  "query": "azure.activitylogs.result_type: \"Activated\""
                 },
                 "formatter": "number",
                 "hide_in_legend": 0,
@@ -283,7 +283,7 @@
                 "fill": 0.5,
                 "filter": {
                   "language": "kuery",
-                  "query": "event.outcome: \"Resolved\" or event.outcome: \"Succeeded\""
+                  "query": "azure.activitylogs.result_type: \"Resolved\" or azure.activitylogs.result_type: \"Succeeded\""
                 },
                 "formatter": "number",
                 "hide_in_legend": 0,
@@ -328,7 +328,7 @@
             "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index",
             "query": {
               "language": "kuery",
-              "query": "event.dataset :\"azure.activitylogs\" and event.category : \"Alert\" "
+              "query": "event.dataset :\"azure.activitylogs\" and azure.activitylogs.event_category : \"Alert\" "
             }
           }
         },
@@ -354,21 +354,21 @@
                   {
                     "input": {
                       "language": "kuery",
-                      "query": "event.outcome : \"Activated\""
+                      "query": "azure.activitylogs.result_type : \"Activated\""
                     },
                     "label": "Activated"
                   },
                   {
                     "input": {
                       "language": "kuery",
-                      "query": "event.outcome : \"Resolved\""
+                      "query": "azure.activitylogs.result_type : \"Resolved\""
                     },
                     "label": "Resolved"
                   },
                   {
                     "input": {
                       "language": "kuery",
-                      "query": "event.outcome : \"Succeeded\""
+                      "query": "azure.activitylogs.result_type : \"Succeeded\""
                     },
                     "label": "Succeeded"
                   }
@@ -455,7 +455,7 @@
             "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index",
             "query": {
               "language": "kuery",
-              "query": "event.dataset :\"azure.activitylogs\" and event.category : \"Alert\" "
+              "query": "event.dataset :\"azure.activitylogs\" and azure.activitylogs.event_category : \"Alert\" "
             }
           }
         },

diff --git a/x-pack/filebeat/module/azure/_meta/kibana/7/dashboard/Filebeat-azure-overview.json b/x-pack/filebeat/module/azure/_meta/kibana/7/dashboard/Filebeat-azure-overview.json
@@ -448,7 +448,7 @@
                         "default_timefield":"@timestamp",
                         "filter":{
                             "language":"kuery",
-                            "query":"event.dataset :\"azure.activitylogs\" and event.category :\"Administrative\" "
+                            "query":"event.dataset :\"azure.activitylogs\" and azure.activitylogs.event_category :\"Administrative\" "
                         },
                         "id":"61ca57f0-469d-11e7-af02-69e470af7417",
                         "index_pattern":"filebeat-*",
@@ -874,7 +874,7 @@
                         "default_timefield":"@timestamp",
                         "filter":{
                             "language":"kuery",
-                            "query":"event.dataset :\"azure.activitylogs\" and event.category : \"Alert\""
+                            "query":"event.dataset :\"azure.activitylogs\" and azure.activitylogs.event_category : \"Alert\""
                         },
                         "id":"61ca57f0-469d-11e7-af02-69e470af7417",
                         "index_pattern":"filebeat-*",
@@ -888,7 +888,7 @@
                                 "fill":0.5,
                                 "filter":{
                                     "language":"kuery",
-                                    "query":"event.outcome: \"Activated\""
+                                    "query":"azure.activitylogs.result_type: \"Activated\""
                                 },
                                 "formatter":"number",
                                 "hide_in_legend":0,
@@ -912,7 +912,7 @@
                                 "fill":0.5,
                                 "filter":{
                                     "language":"kuery",
-                                    "query":"event.outcome: \"Resolved\" or event.outcome: \"Succeeded\""
+                                    "query":"azure.activitylogs.result_type: \"Resolved\" or azure.activitylogs.result_type: \"Succeeded\""
                                 },
                                 "formatter":"number",
                                 "hide_in_legend":0,
@@ -981,7 +981,7 @@
                         "default_timefield":"@timestamp",
                         "filter":{
                             "language":"kuery",
-                            "query":"event.dataset :\"azure.activitylogs\" and event.category : \"ServiceHealth\""
+                            "query":"event.dataset :\"azure.activitylogs\" and azure.activitylogs.event_category : \"ServiceHealth\""
                         },
                         "id":"61ca57f0-469d-11e7-af02-69e470af7417",
                         "index_pattern":"filebeat-*",
@@ -995,7 +995,7 @@
                                 "fill":0.5,
                                 "filter":{
                                     "language":"kuery",
-                                    "query":"event.outcome: \"Active\""
+                                    "query":"azure.activitylogs.result_type: \"Active\""
                                 },
                                 "formatter":"number",
                                 "hide_in_legend":0,
@@ -1019,7 +1019,7 @@
                                 "fill":0.5,
                                 "filter":{
                                     "language":"kuery",
-                                    "query":"event.outcome: \"Resolved\" "
+                                    "query":"azure.activitylogs.result_type: \"Resolved\" "
                                 },
                                 "formatter":"number",
                                 "hide_in_legend":0,
@@ -1243,7 +1243,7 @@
                         "indexRefName":"kibanaSavedObjectMeta.searchSourceJSON.index",
                         "query":{
                             "language":"kuery",
-                            "query":"event.dataset :\"azure.activitylogs\" and event.category : \"Alert\" "
+                            "query":"event.dataset :\"azure.activitylogs\" and azure.activitylogs.event_category : \"Alert\" "
                         }
                     }
                 },
@@ -1271,21 +1271,21 @@
                                     {
                                         "input":{
                                             "language":"kuery",
-                                            "query":"event.outcome : \"Activated\""
+                                            "query":"azure.activitylogs.result_type : \"Activated\""
                                         },
                                         "label":"Activated"
                                     },
                                     {
                                         "input":{
                                             "language":"kuery",
-                                            "query":"event.outcome : \"Resolved\""
+                                            "query":"azure.activitylogs.result_type : \"Resolved\""
                                         },
                                         "label":"Resolved"
                                     },
                                     {
                                         "input":{
                                             "language":"kuery",
-                                            "query":"event.outcome : \"Succeeded\""
+                                            "query":"azure.activitylogs.result_type : \"Succeeded\""
                                         },
                                         "label":"Succeeded"
                                     }
@@ -1378,7 +1378,7 @@
                         "indexRefName":"kibanaSavedObjectMeta.searchSourceJSON.index",
                         "query":{
                             "language":"kuery",
-                            "query":"event.dataset :\"azure.activitylogs\" and event.category : \"ServiceHealth\" "
+                            "query":"event.dataset :\"azure.activitylogs\" and azure.activitylogs.event_category : \"ServiceHealth\" "
                         }
                     }
                 },
@@ -1406,14 +1406,14 @@
                                     {
                                         "input":{
                                             "language":"kuery",
-                                            "query":"event.outcome : \"Active\""
+                                            "query":"azure.activitylogs.result_type : \"Active\""
                                         },
                                         "label":"Active"
                                     },
                                     {
                                         "input":{
                                             "language":"kuery",
-                                            "query":"event.outcome : \"Resolved\""
+                                            "query":"azure.activitylogs.result_type : \"Resolved\""
                                         },
                                         "label":"Resolved"
                                     }

diff --git a/x-pack/filebeat/module/azure/_meta/kibana/7/dashboard/Filebeat-azure-user-activity.json b/x-pack/filebeat/module/azure/_meta/kibana/7/dashboard/Filebeat-azure-user-activity.json
@@ -425,7 +425,7 @@
                         "default_timefield":"@timestamp",
                         "filter":{
                             "language":"kuery",
-                            "query":"event.dataset :\"azure.activitylogs\" and event.category :\"Administrative\" and azure.activitylogs.identity.claims_initiated_by_user.fullname :*"
+                            "query":"event.dataset :\"azure.activitylogs\" and azure.activitylogs.event_category :\"Administrative\" and azure.activitylogs.identity.claims_initiated_by_user.fullname :*"
                         },
                         "id":"61ca57f0-469d-11e7-af02-69e470af7417",
                         "index_pattern":"filebeat-*",
@@ -880,7 +880,7 @@
                                 "fill":0.5,
                                 "filter":{
                                     "language":"kuery",
-                                    "query":"event.outcome : \"Success\" "
+                                    "query":"azure.activitylogs.result_type : \"Success\" "
                                 },
                                 "formatter":"number",
                                 "id":"61ca57f1-469d-11e7-af02-69e470af7417",
@@ -896,7 +896,7 @@
                                 "separate_axis":0,
                                 "split_mode":"filter",
                                 "stacked":"none",
-                                "terms_field":"event.outcome"
+                                "terms_field":"azure.activitylogs.result_type"
                             },
                             {
                                 "axis_position":"right",
@@ -905,7 +905,7 @@
                                 "fill":0.5,
                                 "filter":{
                                     "language":"kuery",
-                                    "query":"event.outcome : \"Fail\" "
+                                    "query":"azure.activitylogs.result_type : \"Fail\" "
                                 },
                                 "formatter":"number",
                                 "id":"78e85470-f0cb-11e9-bf79-0db2fc8554f1",

diff --git a/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml b/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml
@@ -1,6 +1,7 @@
 - name: activitylogs
   type: group
   release: beta
+  default_field: false
   description: >
     Fields for Azure activity logs.
   fields:
@@ -86,6 +87,10 @@
       type: keyword
       description: >
         Operation name
+    - name: result_type
+      type: keyword
+      description: >
+        Result type
     - name: result_signature
       type: keyword
       description: >
@@ -94,6 +99,10 @@
       type: keyword
       description: >
         Category
+    - name: event_category
+      type: keyword
+      description: >
+        Event Category
     - name: properties
       type: group
       description: >