[Filebeat] Improve ECS categorization field mappings for azure module (…

…elastic#19376) * Improve ECS categorization field mappings in azure module - activitylogs + convert pipeline to yml + add azure.activitylogs.result_type + set default_field: false + populate event.outcome with allowed values + set event.action + populate event.category with allowed values + set event.kind + set event.type + add support tickets example + add geoip for source.ip + add AS info for source.ip + add user.name + add user.full_name + add user.domain + update dashboards - auditlogs + convert pipeline to yml + set default_field: false + add azure.auditlogs.category + populate event.outcome with allowed values + set event.action + set event.kind + update dashboards - signinlogs + convert pipeline to yml + set default_field: false + set event.action + populate event.category with allowed values + set event.type + populate event.outcome with allowed values + add azure.signinlogs.category + add azure.signinlogs.result_type + set user.name + set user.domain + set user.full_name + set user.id + add geoip for source.ip + add AS info for source.ip + update dashboards Closes elastic#16155
melchiormoulin · Oct 14, 2020 · 19f9ce6 · 19f9ce6
1 parent 4a00034
commit 19f9ce6
Show file tree

Hide file tree

Showing 25 changed files with 939 additions and 986 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -437,6 +437,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Changed the panw module to pass through (rather than drop) message types other than threat and traffic. {issue}16815[16815] {pull}19375[19375]
 - Add support for timezone offsets and `Z` to decode_cef timestamp parser. {pull}19346[19346]
 - Improve ECS categorization field mappings in traefik module. {issue}16183[16183] {pull}19379[19379]
+- Improve ECS categorization field mappings in azure module. {issue}16155[16155] {pull}19376[19376]
 
 *Heartbeat*
 

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -2378,6 +2378,16 @@ type: keyword
 Operation name
 
 
+type: keyword
+
+--
+
+*`azure.activitylogs.result_type`*::
++
+--
+Result type
+
+
 type: keyword
 
 --
@@ -2398,6 +2408,16 @@ type: keyword
 Category
 
 
+type: keyword
+
+--
+
+*`azure.activitylogs.event_category`*::
++
+--
+Event Category
+
+
 type: keyword
 
 --
@@ -2436,6 +2456,16 @@ Fields for Azure audit logs.
 
 
 
+*`azure.auditlogs.category`*::
++
+--
+The category of the operation. Currently, Audit is the only supported value.
+
+
+type: keyword
+
+--
+
 *`azure.auditlogs.operation_name`*::
 +
 --
@@ -2831,6 +2861,16 @@ type: keyword
 Result description
 
 
+type: keyword
+
+--
+
+*`azure.signinlogs.result_type`*::
++
+--
+Result type
+
+
 type: keyword
 
 --
@@ -2841,6 +2881,16 @@ type: keyword
 Identity
 
 
+type: keyword
+
+--
+
+*`azure.signinlogs.category`*::
++
+--
+Category
+
+
 type: keyword
 
 --

diff --git a/x-pack/filebeat/module/azure/_meta/kibana/7/dashboard/Filebeat-azure-alerts-overview.json b/x-pack/filebeat/module/azure/_meta/kibana/7/dashboard/Filebeat-azure-alerts-overview.json
@@ -245,7 +245,7 @@
  "default_timefield": "@timestamp",
  "filter": {
  "language": "kuery",
- "query": "event.dataset :\"azure.activitylogs\" and event.category : \"Alert\""
+ "query": "event.dataset :\"azure.activitylogs\" and azure.activitylogs.event_category : \"Alert\""
  },
  "id": "61ca57f0-469d-11e7-af02-69e470af7417",
  "index_pattern": "filebeat-*",
@@ -259,7 +259,7 @@
  "fill": 0.5,
  "filter": {
  "language": "kuery",
- "query": "event.outcome: \"Activated\""
+ "query": "azure.activitylogs.result_type: \"Activated\""
  },
  "formatter": "number",
  "hide_in_legend": 0,
@@ -283,7 +283,7 @@
  "fill": 0.5,
  "filter": {
  "language": "kuery",
- "query": "event.outcome: \"Resolved\" or event.outcome: \"Succeeded\""
+ "query": "azure.activitylogs.result_type: \"Resolved\" or azure.activitylogs.result_type: \"Succeeded\""
  },
  "formatter": "number",
  "hide_in_legend": 0,
@@ -328,7 +328,7 @@
  "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index",
  "query": {
  "language": "kuery",
- "query": "event.dataset :\"azure.activitylogs\" and event.category : \"Alert\" "
+ "query": "event.dataset :\"azure.activitylogs\" and azure.activitylogs.event_category : \"Alert\" "
  }
  }
  },
@@ -354,21 +354,21 @@
  {
  "input": {
  "language": "kuery",
- "query": "event.outcome : \"Activated\""
+ "query": "azure.activitylogs.result_type : \"Activated\""
  },
  "label": "Activated"
  },
  {
  "input": {
  "language": "kuery",
- "query": "event.outcome : \"Resolved\""
+ "query": "azure.activitylogs.result_type : \"Resolved\""
  },
  "label": "Resolved"
  },
  {
  "input": {
  "language": "kuery",
- "query": "event.outcome : \"Succeeded\""
+ "query": "azure.activitylogs.result_type : \"Succeeded\""
  },
  "label": "Succeeded"
  }
@@ -455,7 +455,7 @@
  "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index",
  "query": {
  "language": "kuery",
- "query": "event.dataset :\"azure.activitylogs\" and event.category : \"Alert\" "
+ "query": "event.dataset :\"azure.activitylogs\" and azure.activitylogs.event_category : \"Alert\" "
  }
  }
  },

diff --git a/x-pack/filebeat/module/azure/_meta/kibana/7/dashboard/Filebeat-azure-overview.json b/x-pack/filebeat/module/azure/_meta/kibana/7/dashboard/Filebeat-azure-overview.json
@@ -448,7 +448,7 @@
  "default_timefield":"@timestamp",
  "filter":{
  "language":"kuery",
- "query":"event.dataset :\"azure.activitylogs\" and event.category :\"Administrative\" "
+ "query":"event.dataset :\"azure.activitylogs\" and azure.activitylogs.event_category :\"Administrative\" "
  },
  "id":"61ca57f0-469d-11e7-af02-69e470af7417",
  "index_pattern":"filebeat-*",
@@ -874,7 +874,7 @@
  "default_timefield":"@timestamp",
  "filter":{
  "language":"kuery",
- "query":"event.dataset :\"azure.activitylogs\" and event.category : \"Alert\""
+ "query":"event.dataset :\"azure.activitylogs\" and azure.activitylogs.event_category : \"Alert\""
  },
  "id":"61ca57f0-469d-11e7-af02-69e470af7417",
  "index_pattern":"filebeat-*",
@@ -888,7 +888,7 @@
  "fill":0.5,
  "filter":{
  "language":"kuery",
- "query":"event.outcome: \"Activated\""
+ "query":"azure.activitylogs.result_type: \"Activated\""
  },
  "formatter":"number",
  "hide_in_legend":0,
@@ -912,7 +912,7 @@
  "fill":0.5,
  "filter":{
  "language":"kuery",
- "query":"event.outcome: \"Resolved\" or event.outcome: \"Succeeded\""
+ "query":"azure.activitylogs.result_type: \"Resolved\" or azure.activitylogs.result_type: \"Succeeded\""
  },
  "formatter":"number",
  "hide_in_legend":0,
@@ -981,7 +981,7 @@
  "default_timefield":"@timestamp",
  "filter":{
  "language":"kuery",
- "query":"event.dataset :\"azure.activitylogs\" and event.category : \"ServiceHealth\""
+ "query":"event.dataset :\"azure.activitylogs\" and azure.activitylogs.event_category : \"ServiceHealth\""
  },
  "id":"61ca57f0-469d-11e7-af02-69e470af7417",
  "index_pattern":"filebeat-*",
@@ -995,7 +995,7 @@
  "fill":0.5,
  "filter":{
  "language":"kuery",
- "query":"event.outcome: \"Active\""
+ "query":"azure.activitylogs.result_type: \"Active\""
  },
  "formatter":"number",
  "hide_in_legend":0,
@@ -1019,7 +1019,7 @@
  "fill":0.5,
  "filter":{
  "language":"kuery",
- "query":"event.outcome: \"Resolved\" "
+ "query":"azure.activitylogs.result_type: \"Resolved\" "
  },
  "formatter":"number",
  "hide_in_legend":0,
@@ -1243,7 +1243,7 @@
  "indexRefName":"kibanaSavedObjectMeta.searchSourceJSON.index",
  "query":{
  "language":"kuery",
- "query":"event.dataset :\"azure.activitylogs\" and event.category : \"Alert\" "
+ "query":"event.dataset :\"azure.activitylogs\" and azure.activitylogs.event_category : \"Alert\" "
  }
  }
  },
@@ -1271,21 +1271,21 @@
  {
  "input":{
  "language":"kuery",
- "query":"event.outcome : \"Activated\""
+ "query":"azure.activitylogs.result_type : \"Activated\""
  },
  "label":"Activated"
  },
  {
  "input":{
  "language":"kuery",
- "query":"event.outcome : \"Resolved\""
+ "query":"azure.activitylogs.result_type : \"Resolved\""
  },
  "label":"Resolved"
  },
  {
  "input":{
  "language":"kuery",
- "query":"event.outcome : \"Succeeded\""
+ "query":"azure.activitylogs.result_type : \"Succeeded\""
  },
  "label":"Succeeded"
  }
@@ -1378,7 +1378,7 @@
  "indexRefName":"kibanaSavedObjectMeta.searchSourceJSON.index",
  "query":{
  "language":"kuery",
- "query":"event.dataset :\"azure.activitylogs\" and event.category : \"ServiceHealth\" "
+ "query":"event.dataset :\"azure.activitylogs\" and azure.activitylogs.event_category : \"ServiceHealth\" "
  }
  }
  },
@@ -1406,14 +1406,14 @@
  {
  "input":{
  "language":"kuery",
- "query":"event.outcome : \"Active\""
+ "query":"azure.activitylogs.result_type : \"Active\""
  },
  "label":"Active"
  },
  {
  "input":{
  "language":"kuery",
- "query":"event.outcome : \"Resolved\""
+ "query":"azure.activitylogs.result_type : \"Resolved\""
  },
  "label":"Resolved"
  }

diff --git a/x-pack/filebeat/module/azure/_meta/kibana/7/dashboard/Filebeat-azure-user-activity.json b/x-pack/filebeat/module/azure/_meta/kibana/7/dashboard/Filebeat-azure-user-activity.json
@@ -425,7 +425,7 @@
  "default_timefield":"@timestamp",
  "filter":{
  "language":"kuery",
- "query":"event.dataset :\"azure.activitylogs\" and event.category :\"Administrative\" and azure.activitylogs.identity.claims_initiated_by_user.fullname :*"
+ "query":"event.dataset :\"azure.activitylogs\" and azure.activitylogs.event_category :\"Administrative\" and azure.activitylogs.identity.claims_initiated_by_user.fullname :*"
  },
  "id":"61ca57f0-469d-11e7-af02-69e470af7417",
  "index_pattern":"filebeat-*",
@@ -880,7 +880,7 @@
  "fill":0.5,
  "filter":{
  "language":"kuery",
- "query":"event.outcome : \"Success\" "
+ "query":"azure.activitylogs.result_type : \"Success\" "
  },
  "formatter":"number",
  "id":"61ca57f1-469d-11e7-af02-69e470af7417",
@@ -896,7 +896,7 @@
  "separate_axis":0,
  "split_mode":"filter",
  "stacked":"none",
- "terms_field":"event.outcome"
+ "terms_field":"azure.activitylogs.result_type"
  },
  {
  "axis_position":"right",
@@ -905,7 +905,7 @@
  "fill":0.5,
  "filter":{
  "language":"kuery",
- "query":"event.outcome : \"Fail\" "
+ "query":"azure.activitylogs.result_type : \"Fail\" "
  },
  "formatter":"number",
  "id":"78e85470-f0cb-11e9-bf79-0db2fc8554f1",

diff --git a/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml b/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml
@@ -1,6 +1,7 @@
 - name: activitylogs
  type: group
  release: beta
+ default_field: false
  description: >
  Fields for Azure activity logs.
  fields:
@@ -86,6 +87,10 @@
  type: keyword
  description: >
  Operation name
+ - name: result_type
+ type: keyword
+ description: >
+ Result type
  - name: result_signature
  type: keyword
  description: >
@@ -94,6 +99,10 @@
  type: keyword
  description: >
  Category
+ - name: event_category
+ type: keyword
+ description: >
+ Event Category
  - name: properties
  type: group
  description: >