From 19f9ce6522094c958a1b1a880b95281878370b3b Mon Sep 17 00:00:00 2001 From: Lee Hinman <57081003+leehinman@users.noreply.github.com> Date: Wed, 8 Jul 2020 08:22:59 -0500 Subject: [PATCH] [Filebeat] Improve ECS categorization field mappings for azure module (#19376) * Improve ECS categorization field mappings in azure module - activitylogs + convert pipeline to yml + add azure.activitylogs.result_type + set default_field: false + populate event.outcome with allowed values + set event.action + populate event.category with allowed values + set event.kind + set event.type + add support tickets example + add geoip for source.ip + add AS info for source.ip + add user.name + add user.full_name + add user.domain + update dashboards - auditlogs + convert pipeline to yml + set default_field: false + add azure.auditlogs.category + populate event.outcome with allowed values + set event.action + set event.kind + update dashboards - signinlogs + convert pipeline to yml + set default_field: false + set event.action + populate event.category with allowed values + set event.type + populate event.outcome with allowed values + add azure.signinlogs.category + add azure.signinlogs.result_type + set user.name + set user.domain + set user.full_name + set user.id + add geoip for source.ip + add AS info for source.ip + update dashboards Closes #16155 --- CHANGELOG.next.asciidoc | 1 + filebeat/docs/fields.asciidoc | 50 ++ .../Filebeat-azure-alerts-overview.json | 16 +- .../7/dashboard/Filebeat-azure-overview.json | 28 +- .../Filebeat-azure-user-activity.json | 8 +- .../azure/activitylogs/_meta/fields.yml | 9 + .../azure/activitylogs/ingest/pipeline.json | 249 ---------- .../azure/activitylogs/ingest/pipeline.yml | 230 ++++++++++ .../module/azure/activitylogs/manifest.yml | 4 +- .../test/activitylogs.log-expected.json | 15 +- .../test/supporttickets_write.log | 1 + .../supporttickets_write.log-expected.json | 74 +++ .../module/azure/auditlogs/_meta/fields.yml | 5 + .../azure/auditlogs/ingest/pipeline.json | 194 -------- .../azure/auditlogs/ingest/pipeline.yml | 141 ++++++ .../module/azure/auditlogs/manifest.yml | 4 +- .../test/auditlogs.log-expected.json | 8 +- .../module/azure/azure-shared-pipeline.json | 69 --- .../module/azure/azure-shared-pipeline.yml | 44 ++ x-pack/filebeat/module/azure/fields.go | 2 +- .../module/azure/signinlogs/_meta/fields.yml | 9 + .../azure/signinlogs/ingest/pipeline.json | 431 ------------------ .../azure/signinlogs/ingest/pipeline.yml | 299 ++++++++++++ .../module/azure/signinlogs/manifest.yml | 4 +- .../test/signinlogs.log-expected.json | 30 +- 25 files changed, 939 insertions(+), 986 deletions(-) delete mode 100644 x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json create mode 100644 x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml create mode 100644 x-pack/filebeat/module/azure/activitylogs/test/supporttickets_write.log create mode 100644 x-pack/filebeat/module/azure/activitylogs/test/supporttickets_write.log-expected.json delete mode 100644 x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.json create mode 100644 x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.yml delete mode 100644 x-pack/filebeat/module/azure/azure-shared-pipeline.json create mode 100644 x-pack/filebeat/module/azure/azure-shared-pipeline.yml delete mode 100644 x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.json create mode 100644 x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.yml diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 177a9069a38..843fa980ed1 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -437,6 +437,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Changed the panw module to pass through (rather than drop) message types other than threat and traffic. {issue}16815[16815] {pull}19375[19375] - Add support for timezone offsets and `Z` to decode_cef timestamp parser. {pull}19346[19346] - Improve ECS categorization field mappings in traefik module. {issue}16183[16183] {pull}19379[19379] +- Improve ECS categorization field mappings in azure module. {issue}16155[16155] {pull}19376[19376] *Heartbeat* diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index e63dff8db52..dd7c67f9894 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -2378,6 +2378,16 @@ type: keyword Operation name +type: keyword + +-- + +*`azure.activitylogs.result_type`*:: ++ +-- +Result type + + type: keyword -- @@ -2398,6 +2408,16 @@ type: keyword Category +type: keyword + +-- + +*`azure.activitylogs.event_category`*:: ++ +-- +Event Category + + type: keyword -- @@ -2436,6 +2456,16 @@ Fields for Azure audit logs. +*`azure.auditlogs.category`*:: ++ +-- +The category of the operation. Currently, Audit is the only supported value. + + +type: keyword + +-- + *`azure.auditlogs.operation_name`*:: + -- @@ -2831,6 +2861,16 @@ type: keyword Result description +type: keyword + +-- + +*`azure.signinlogs.result_type`*:: ++ +-- +Result type + + type: keyword -- @@ -2841,6 +2881,16 @@ type: keyword Identity +type: keyword + +-- + +*`azure.signinlogs.category`*:: ++ +-- +Category + + type: keyword -- diff --git a/x-pack/filebeat/module/azure/_meta/kibana/7/dashboard/Filebeat-azure-alerts-overview.json b/x-pack/filebeat/module/azure/_meta/kibana/7/dashboard/Filebeat-azure-alerts-overview.json index 8674e2f3db3..5ef32ba0d11 100644 --- a/x-pack/filebeat/module/azure/_meta/kibana/7/dashboard/Filebeat-azure-alerts-overview.json +++ b/x-pack/filebeat/module/azure/_meta/kibana/7/dashboard/Filebeat-azure-alerts-overview.json @@ -245,7 +245,7 @@ "default_timefield": "@timestamp", "filter": { "language": "kuery", - "query": "event.dataset :\"azure.activitylogs\" and event.category : \"Alert\"" + "query": "event.dataset :\"azure.activitylogs\" and azure.activitylogs.event_category : \"Alert\"" }, "id": "61ca57f0-469d-11e7-af02-69e470af7417", "index_pattern": "filebeat-*", @@ -259,7 +259,7 @@ "fill": 0.5, "filter": { "language": "kuery", - "query": "event.outcome: \"Activated\"" + "query": "azure.activitylogs.result_type: \"Activated\"" }, "formatter": "number", "hide_in_legend": 0, @@ -283,7 +283,7 @@ "fill": 0.5, "filter": { "language": "kuery", - "query": "event.outcome: \"Resolved\" or event.outcome: \"Succeeded\"" + "query": "azure.activitylogs.result_type: \"Resolved\" or azure.activitylogs.result_type: \"Succeeded\"" }, "formatter": "number", "hide_in_legend": 0, @@ -328,7 +328,7 @@ "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", "query": { "language": "kuery", - "query": "event.dataset :\"azure.activitylogs\" and event.category : \"Alert\" " + "query": "event.dataset :\"azure.activitylogs\" and azure.activitylogs.event_category : \"Alert\" " } } }, @@ -354,21 +354,21 @@ { "input": { "language": "kuery", - "query": "event.outcome : \"Activated\"" + "query": "azure.activitylogs.result_type : \"Activated\"" }, "label": "Activated" }, { "input": { "language": "kuery", - "query": "event.outcome : \"Resolved\"" + "query": "azure.activitylogs.result_type : \"Resolved\"" }, "label": "Resolved" }, { "input": { "language": "kuery", - "query": "event.outcome : \"Succeeded\"" + "query": "azure.activitylogs.result_type : \"Succeeded\"" }, "label": "Succeeded" } @@ -455,7 +455,7 @@ "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", "query": { "language": "kuery", - "query": "event.dataset :\"azure.activitylogs\" and event.category : \"Alert\" " + "query": "event.dataset :\"azure.activitylogs\" and azure.activitylogs.event_category : \"Alert\" " } } }, diff --git a/x-pack/filebeat/module/azure/_meta/kibana/7/dashboard/Filebeat-azure-overview.json b/x-pack/filebeat/module/azure/_meta/kibana/7/dashboard/Filebeat-azure-overview.json index e15c8e0c363..fa66d908ea4 100644 --- a/x-pack/filebeat/module/azure/_meta/kibana/7/dashboard/Filebeat-azure-overview.json +++ b/x-pack/filebeat/module/azure/_meta/kibana/7/dashboard/Filebeat-azure-overview.json @@ -448,7 +448,7 @@ "default_timefield":"@timestamp", "filter":{ "language":"kuery", - "query":"event.dataset :\"azure.activitylogs\" and event.category :\"Administrative\" " + "query":"event.dataset :\"azure.activitylogs\" and azure.activitylogs.event_category :\"Administrative\" " }, "id":"61ca57f0-469d-11e7-af02-69e470af7417", "index_pattern":"filebeat-*", @@ -874,7 +874,7 @@ "default_timefield":"@timestamp", "filter":{ "language":"kuery", - "query":"event.dataset :\"azure.activitylogs\" and event.category : \"Alert\"" + "query":"event.dataset :\"azure.activitylogs\" and azure.activitylogs.event_category : \"Alert\"" }, "id":"61ca57f0-469d-11e7-af02-69e470af7417", "index_pattern":"filebeat-*", @@ -888,7 +888,7 @@ "fill":0.5, "filter":{ "language":"kuery", - "query":"event.outcome: \"Activated\"" + "query":"azure.activitylogs.result_type: \"Activated\"" }, "formatter":"number", "hide_in_legend":0, @@ -912,7 +912,7 @@ "fill":0.5, "filter":{ "language":"kuery", - "query":"event.outcome: \"Resolved\" or event.outcome: \"Succeeded\"" + "query":"azure.activitylogs.result_type: \"Resolved\" or azure.activitylogs.result_type: \"Succeeded\"" }, "formatter":"number", "hide_in_legend":0, @@ -981,7 +981,7 @@ "default_timefield":"@timestamp", "filter":{ "language":"kuery", - "query":"event.dataset :\"azure.activitylogs\" and event.category : \"ServiceHealth\"" + "query":"event.dataset :\"azure.activitylogs\" and azure.activitylogs.event_category : \"ServiceHealth\"" }, "id":"61ca57f0-469d-11e7-af02-69e470af7417", "index_pattern":"filebeat-*", @@ -995,7 +995,7 @@ "fill":0.5, "filter":{ "language":"kuery", - "query":"event.outcome: \"Active\"" + "query":"azure.activitylogs.result_type: \"Active\"" }, "formatter":"number", "hide_in_legend":0, @@ -1019,7 +1019,7 @@ "fill":0.5, "filter":{ "language":"kuery", - "query":"event.outcome: \"Resolved\" " + "query":"azure.activitylogs.result_type: \"Resolved\" " }, "formatter":"number", "hide_in_legend":0, @@ -1243,7 +1243,7 @@ "indexRefName":"kibanaSavedObjectMeta.searchSourceJSON.index", "query":{ "language":"kuery", - "query":"event.dataset :\"azure.activitylogs\" and event.category : \"Alert\" " + "query":"event.dataset :\"azure.activitylogs\" and azure.activitylogs.event_category : \"Alert\" " } } }, @@ -1271,21 +1271,21 @@ { "input":{ "language":"kuery", - "query":"event.outcome : \"Activated\"" + "query":"azure.activitylogs.result_type : \"Activated\"" }, "label":"Activated" }, { "input":{ "language":"kuery", - "query":"event.outcome : \"Resolved\"" + "query":"azure.activitylogs.result_type : \"Resolved\"" }, "label":"Resolved" }, { "input":{ "language":"kuery", - "query":"event.outcome : \"Succeeded\"" + "query":"azure.activitylogs.result_type : \"Succeeded\"" }, "label":"Succeeded" } @@ -1378,7 +1378,7 @@ "indexRefName":"kibanaSavedObjectMeta.searchSourceJSON.index", "query":{ "language":"kuery", - "query":"event.dataset :\"azure.activitylogs\" and event.category : \"ServiceHealth\" " + "query":"event.dataset :\"azure.activitylogs\" and azure.activitylogs.event_category : \"ServiceHealth\" " } } }, @@ -1406,14 +1406,14 @@ { "input":{ "language":"kuery", - "query":"event.outcome : \"Active\"" + "query":"azure.activitylogs.result_type : \"Active\"" }, "label":"Active" }, { "input":{ "language":"kuery", - "query":"event.outcome : \"Resolved\"" + "query":"azure.activitylogs.result_type : \"Resolved\"" }, "label":"Resolved" } diff --git a/x-pack/filebeat/module/azure/_meta/kibana/7/dashboard/Filebeat-azure-user-activity.json b/x-pack/filebeat/module/azure/_meta/kibana/7/dashboard/Filebeat-azure-user-activity.json index 33fec90f0c6..fd2ee9c225e 100644 --- a/x-pack/filebeat/module/azure/_meta/kibana/7/dashboard/Filebeat-azure-user-activity.json +++ b/x-pack/filebeat/module/azure/_meta/kibana/7/dashboard/Filebeat-azure-user-activity.json @@ -425,7 +425,7 @@ "default_timefield":"@timestamp", "filter":{ "language":"kuery", - "query":"event.dataset :\"azure.activitylogs\" and event.category :\"Administrative\" and azure.activitylogs.identity.claims_initiated_by_user.fullname :*" + "query":"event.dataset :\"azure.activitylogs\" and azure.activitylogs.event_category :\"Administrative\" and azure.activitylogs.identity.claims_initiated_by_user.fullname :*" }, "id":"61ca57f0-469d-11e7-af02-69e470af7417", "index_pattern":"filebeat-*", @@ -880,7 +880,7 @@ "fill":0.5, "filter":{ "language":"kuery", - "query":"event.outcome : \"Success\" " + "query":"azure.activitylogs.result_type : \"Success\" " }, "formatter":"number", "id":"61ca57f1-469d-11e7-af02-69e470af7417", @@ -896,7 +896,7 @@ "separate_axis":0, "split_mode":"filter", "stacked":"none", - "terms_field":"event.outcome" + "terms_field":"azure.activitylogs.result_type" }, { "axis_position":"right", @@ -905,7 +905,7 @@ "fill":0.5, "filter":{ "language":"kuery", - "query":"event.outcome : \"Fail\" " + "query":"azure.activitylogs.result_type : \"Fail\" " }, "formatter":"number", "id":"78e85470-f0cb-11e9-bf79-0db2fc8554f1", diff --git a/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml b/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml index c562d987d24..28ff5a06fd3 100644 --- a/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml +++ b/x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml @@ -1,6 +1,7 @@ - name: activitylogs type: group release: beta + default_field: false description: > Fields for Azure activity logs. fields: @@ -86,6 +87,10 @@ type: keyword description: > Operation name + - name: result_type + type: keyword + description: > + Result type - name: result_signature type: keyword description: > @@ -94,6 +99,10 @@ type: keyword description: > Category + - name: event_category + type: keyword + description: > + Event Category - name: properties type: group description: > diff --git a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json deleted file mode 100644 index cb6dbf66270..00000000000 --- a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.json +++ /dev/null @@ -1,249 +0,0 @@ -{ - "description": "Pipeline for parsing azure activity logs.", - "processors": [ - { - "rename" : { - "field" : "azure", - "target_field" : "azure-eventhub", - "ignore_missing": true - } - }, - { - "script": { - "source": "ctx.message = ctx.message.replace(params.empty_field_name, '')", - "params": { - "empty_field_name": "\"\":\"\"," - }, - "ignore_failure": true - } - }, - { - "json" : { - "field" : "message", - "target_field" : "azure.activitylogs" - } - }, - { - "date": { - "field": "azure.activitylogs.time", - "target_field": "@timestamp", - "ignore_failure": true, - "formats": [ - "ISO8601" - ] - } - }, - { - "remove": { - "field": ["message", "azure.activitylogs.time"], - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.activitylogs.resourceId", - "target_field": "azure.resource_id", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.activitylogs.callerIpAddress", - "target_field": "source.ip", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.activitylogs.level", - "target_field": "log.level", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.activitylogs.durationMs", - "target_field": "event.duration", - "ignore_missing": true - } - }, - { - "script": { - "lang": "painless", - "source": "if (ctx.event.duration!= null) {ctx.event.duration = ctx.event.duration * params.param_nano;}", - "params": { - "param_nano": 1000000 - }, - "ignore_failure": true - } - }, - { - "rename": { - "field": "azure.activitylogs.location", - "target_field": "geo.name", - "ignore_missing": true - } - }, - { - "script": { - "lang": "painless", - "source": "if (ctx.azure.activitylogs.properties != null && ctx.azure.activitylogs.properties.eventCategory != null) {ctx.eventCategory = ctx.azure.activitylogs.properties.eventCategory} if (ctx.azure.activitylogs.properties != null && ctx.azure.activitylogs.properties.policies != null) { ctx.eventCategory = 'Policy'} if (ctx.eventCategory == null) {ctx.eventCategory='Administrative'}", - "ignore_failure": true - } - }, - { - "rename": { - "field": "eventCategory", - "target_field": "event.category", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.activitylogs.resultType", - "target_field": "event.outcome", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.activitylogs.operationName", - "target_field": "azure.activitylogs.operation_name", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.activitylogs.resultSignature", - "target_field": "azure.activitylogs.result_signature", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.activitylogs.identity.authorization.evidence.roleAssignmentScope", - "target_field": "azure.activitylogs.identity.authorization.evidence.role_assignment_scope", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.activitylogs.identity.authorization.evidence.roleDefinitionId", - "target_field": "azure.activitylogs.identity.authorization.evidence.role_definition_id", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.activitylogs.identity.authorization.evidence.roleAssignmentId", - "target_field": "azure.activitylogs.identity.authorization.evidence.role_assignment_id", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.activitylogs.identity.authorization.evidence.principalId", - "target_field": "azure.activitylogs.identity.authorization.evidence.principal_id", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.activitylogs.identity.authorization.evidence.principalType", - "target_field": "azure.activitylogs.identity.authorization.evidence.principal_type", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.activitylogs.correlationId", - "target_field": "azure.correlation_id", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.activitylogs.properties.serviceRequestId", - "target_field": "azure.activitylogs.properties.service_request_id", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.activitylogs.properties.statusMessage", - "target_field": "message", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.activitylogs.properties.statusCode", - "target_field": "azure.activitylogs.properties.status_code", - "ignore_missing": true - } - }, - { - "geoip" : { - "field" : "source.ip", - "target_field" : "geo", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.activitylogs.identity.claims.name", - "target_field": "azure.activitylogs.identity.claims_initiated_by_user.fullname", - "ignore_missing": true - } - }, - { - "script": { - "lang": "painless", - "source": "if (ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/surname'] != null) { ctx.azure.activitylogs.identity.claims_initiated_by_user.surname = ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/surname'];}", - "ignore_failure": true - } - }, - { - "script": { - "lang": "painless", - "source": "if (ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name'] != null) { ctx.azure.activitylogs.identity.claims_initiated_by_user.name = ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name'];}", - "ignore_failure": true - } - }, - { - "script": { - "lang": "painless", - "source": "if (ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname'] != null) { ctx.azure.activitylogs.identity.claims_initiated_by_user.givenname = ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname'];}", - "ignore_failure": true - } - }, - { - "set": { - "if" : "ctx.azure.activitylogs.identity!= null && ctx.azure.activitylogs.identity.claims_initiated_by_user != null && ctx.azure.activitylogs.identity.claims_initiated_by_user.name != null", - "field": "azure.activitylogs.identity.claims_initiated_by_user.schema", - "value": "http://schemas.xmlsoap.org/ws/2005/05/identity/claims", - "ignore_failure": true - } - }, - { - "script": { - "lang": "painless", - "source": "if (ctx.azure.activitylogs.identity.claims != null) { ctx.temp_claims = new HashMap(); for (String key : ctx.azure.activitylogs.identity.claims.keySet()) { ctx.temp_claims[key.replace('.', '_')] = ctx.azure.activitylogs.identity.claims.get(key) ;}ctx.azure.activitylogs.identity.claims = ctx.temp_claims; ctx.remove('temp_claims');}", - "ignore_failure": true - } - }, - { - "pipeline": { - "name": "{< IngestPipeline "azure-shared-pipeline" >}" - } - } - ], - "on_failure": [ - { - "set": { - "field": "error.message", - "value": "{{ _ingest.on_failure_message }}" - } - } - ] -} diff --git a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml new file mode 100644 index 00000000000..dac11495608 --- /dev/null +++ b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml @@ -0,0 +1,230 @@ +description: Pipeline for parsing azure activity logs. +processors: +- rename: + field: azure + target_field: azure-eventhub + ignore_missing: true +- script: + source: ctx.message = ctx.message.replace(params.empty_field_name, '') + params: + empty_field_name: '"":"",' + ignore_failure: true +- json: + field: message + target_field: azure.activitylogs +- date: + field: azure.activitylogs.time + target_field: '@timestamp' + ignore_failure: true + formats: + - ISO8601 +- remove: + field: + - message + - azure.activitylogs.time + ignore_missing: true +- rename: + field: azure.activitylogs.resourceId + target_field: azure.resource_id + ignore_missing: true +- rename: + field: azure.activitylogs.callerIpAddress + target_field: source.ip + ignore_missing: true +- rename: + field: azure.activitylogs.level + target_field: log.level + ignore_missing: true +- rename: + field: azure.activitylogs.durationMs + target_field: event.duration + ignore_missing: true +- script: + lang: painless + source: if (ctx.event.duration!= null) {ctx.event.duration = ctx.event.duration + * params.param_nano;} + params: + param_nano: 1000000 + ignore_failure: true +- rename: + field: azure.activitylogs.location + target_field: geo.name + ignore_missing: true +- script: + lang: painless + source: >- + if (ctx?.azure?.activitylogs?.properties?.eventCategory != null) { + ctx.azure.activitylogs.event_category = ctx.azure.activitylogs.properties.eventCategory; + } + else if (ctx?.azure?.activitylogs?.properties?.policies != null) { + ctx.azure.activitylogs.event_category = 'Policy'; + } + else { + ctx.azure.activitylogs.event_category = 'Administrative'; + } + ignore_failure: true +- rename: + field: azure.activitylogs.resultType + target_field: azure.activitylogs.result_type + ignore_missing: true +- convert: + field: azure.activitylogs.result_type + target_field: event.outcome + type: string + if: "ctx?.azure?.activitylogs?.resultType != null && ctx.azure.activitylogs.resultType instanceof String && (ctx.azure.activitylogs.resultType.toLowerCase() == 'success' || ctx.azure.activitylogs.resultType.toLowerCase() == 'failure')" +- rename: + field: azure.activitylogs.operationName + target_field: azure.activitylogs.operation_name + ignore_missing: true +- convert: + field: azure.activitylogs.operation_name + target_field: event.action + type: string + ignore_missing: true +- rename: + field: azure.activitylogs.resultSignature + target_field: azure.activitylogs.result_signature + ignore_missing: true +- rename: + field: azure.activitylogs.identity.authorization.evidence.roleAssignmentScope + target_field: azure.activitylogs.identity.authorization.evidence.role_assignment_scope + ignore_missing: true +- rename: + field: azure.activitylogs.identity.authorization.evidence.roleDefinitionId + target_field: azure.activitylogs.identity.authorization.evidence.role_definition_id + ignore_missing: true +- rename: + field: azure.activitylogs.identity.authorization.evidence.roleAssignmentId + target_field: azure.activitylogs.identity.authorization.evidence.role_assignment_id + ignore_missing: true +- rename: + field: azure.activitylogs.identity.authorization.evidence.principalId + target_field: azure.activitylogs.identity.authorization.evidence.principal_id + ignore_missing: true +- rename: + field: azure.activitylogs.identity.authorization.evidence.principalType + target_field: azure.activitylogs.identity.authorization.evidence.principal_type + ignore_missing: true +- rename: + field: azure.activitylogs.correlationId + target_field: azure.correlation_id + ignore_missing: true +- rename: + field: azure.activitylogs.properties.serviceRequestId + target_field: azure.activitylogs.properties.service_request_id + ignore_missing: true +- rename: + field: azure.activitylogs.properties.statusMessage + target_field: message + ignore_missing: true +- rename: + field: azure.activitylogs.properties.statusCode + target_field: azure.activitylogs.properties.status_code + ignore_missing: true +- geoip: + field: source.ip + target_field: geo + ignore_missing: true +- rename: + field: azure.activitylogs.identity.claims.name + target_field: azure.activitylogs.identity.claims_initiated_by_user.fullname + ignore_missing: true +- script: + lang: painless + source: >- + if (ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/surname'] != null) { + ctx.azure.activitylogs.identity.claims_initiated_by_user.surname = ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/surname']; + } + ignore_failure: true +- script: + lang: painless + source: >- + if (ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name'] != null) { + ctx.azure.activitylogs.identity.claims_initiated_by_user.name = ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name']; + } + ignore_failure: true +- script: + lang: painless + source: >- + if (ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname'] != null) { + ctx.azure.activitylogs.identity.claims_initiated_by_user.givenname = ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname']; + } + ignore_failure: true +- set: + if: ctx.azure.activitylogs.identity!= null && ctx.azure.activitylogs.identity.claims_initiated_by_user + != null && ctx.azure.activitylogs.identity.claims_initiated_by_user.name != + null + field: azure.activitylogs.identity.claims_initiated_by_user.schema + value: http://schemas.xmlsoap.org/ws/2005/05/identity/claims + ignore_failure: true +- script: + lang: painless + source: >- + if (ctx.azure.activitylogs.identity.claims != null) { + ctx.temp_claims = new HashMap(); + for (String key : ctx.azure.activitylogs.identity.claims.keySet()) { + ctx.temp_claims[key.replace('.', '_')] = ctx.azure.activitylogs.identity.claims.get(key); + } + ctx.azure.activitylogs.identity.claims = ctx.temp_claims; ctx.remove('temp_claims'); + } + ignore_failure: true +- script: + lang: painless + ignore_failure: true + params: + "write": + type: + - change + "read": + type: + - access + "delete": + type: + - deletion + "action": + type: + - change + source: >- + if (ctx?.azure?.activitylogs?.category == null) { + return; + } + def hm = new HashMap(params.get(ctx.azure.activitylogs.category.toLowerCase())); + hm.forEach((k, v) -> ctx.event[k] = v); +- geoip: + field: source.ip + target_field: source.geo +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- grok: + field: azure.activitylogs.identity.claims_initiated_by_user.name + patterns: + - '%{USERNAME:user.name}@%{HOSTNAME:user.domain}' + ignore_missing: true +- convert: + field: azure.activitylogs.identity.claims_initiated_by_user.fullname + target_field: user.full_name + type: string + ignore_missing: true +- set: + field: event.kind + value: event +- pipeline: + name: '{< IngestPipeline "azure-shared-pipeline" >}' +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/azure/activitylogs/manifest.yml b/x-pack/filebeat/module/azure/activitylogs/manifest.yml index c83f17ce1a0..92c9682ebaa 100644 --- a/x-pack/filebeat/module/azure/activitylogs/manifest.yml +++ b/x-pack/filebeat/module/azure/activitylogs/manifest.yml @@ -15,6 +15,6 @@ var: default: [forwarded] ingest_pipeline: - - ingest/pipeline.json - - ../azure-shared-pipeline.json + - ingest/pipeline.yml + - ../azure-shared-pipeline.yml input: config/{{.input}}.yml diff --git a/x-pack/filebeat/module/azure/activitylogs/test/activitylogs.log-expected.json b/x-pack/filebeat/module/azure/activitylogs/test/activitylogs.log-expected.json index 258a04d0aab..4c0e8d4701a 100644 --- a/x-pack/filebeat/module/azure/activitylogs/test/activitylogs.log-expected.json +++ b/x-pack/filebeat/module/azure/activitylogs/test/activitylogs.log-expected.json @@ -2,6 +2,7 @@ { "@timestamp": "2019-10-24T00:13:46.355Z", "azure.activitylogs.category": "Action", + "azure.activitylogs.event_category": "Administrative", "azure.activitylogs.identity.authorization.action": "Microsoft.EventHub/namespaces/authorizationRules/listKeys/action", "azure.activitylogs.identity.authorization.evidence.principal_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", "azure.activitylogs.identity.authorization.evidence.principal_type": "ServicePrincipal", @@ -26,6 +27,7 @@ "azure.activitylogs.identity.claims.ver": "1.0", "azure.activitylogs.operation_name": "MICROSOFT.EVENTHUB/NAMESPACES/AUTHORIZATIONRULES/LISTKEYS/ACTION", "azure.activitylogs.result_signature": "Started.", + "azure.activitylogs.result_type": "Start", "azure.correlation_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", "azure.resource.authorization_rule": "ROOTMANAGESHAREDACCESSKEY", "azure.resource.group": "SA-HEMA", @@ -34,11 +36,14 @@ "azure.resource.provider": "MICROSOFT.EVENTHUB", "azure.subscription_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", "cloud.provider": "azure", - "event.category": "Administrative", + "event.action": "MICROSOFT.EVENTHUB/NAMESPACES/AUTHORIZATIONRULES/LISTKEYS/ACTION", "event.dataset": "azure.activitylogs", - "event.duration": 0.0, + "event.duration": 0, + "event.kind": "event", "event.module": "azure", - "event.outcome": "Start", + "event.type": [ + "change" + ], "fileset.name": "activitylogs", "geo.continent_name": "Europe", "geo.country_iso_code": "GB", @@ -48,6 +53,10 @@ "log.level": "Information", "log.offset": 0, "service.type": "azure", + "source.geo.continent_name": "Europe", + "source.geo.country_iso_code": "GB", + "source.geo.location.lat": 51.4964, + "source.geo.location.lon": -0.1224, "source.ip": "51.251.141.41", "tags": [ "forwarded" diff --git a/x-pack/filebeat/module/azure/activitylogs/test/supporttickets_write.log b/x-pack/filebeat/module/azure/activitylogs/test/supporttickets_write.log new file mode 100644 index 00000000000..d1f15fa5d1d --- /dev/null +++ b/x-pack/filebeat/module/azure/activitylogs/test/supporttickets_write.log @@ -0,0 +1 @@ +{"time":"2015-01-21T22:14:26.9792776Z","resourceId":"/subscriptions/s1/resourceGroups/MSSupportGroup/providers/microsoft.support/supporttickets/115012112305841","operationName":"microsoft.support/supporttickets/write","category":"Write","resultType":"Success","resultSignature":"Succeeded.Created","durationMs":2826,"callerIpAddress":"111.111.111.11","correlationId":"c776f9f4-36e5-4e0e-809b-c9b3c3fb62a8","identity":{"authorization":{"scope":"/subscriptions/s1/resourceGroups/MSSupportGroup/providers/microsoft.support/supporttickets/115012112305841","action":"microsoft.support/supporttickets/write","evidence":{"role":"Subscription Admin"}},"claims":{"aud":"https://management.core.windows.net/","iss":"https://sts.windows.net/72f988bf-86f1-41af-91ab-2d7cd011db47/","iat":"1421876371","nbf":"1421876371","exp":"1421880271","ver":"1.0","http://schemas.microsoft.com/identity/claims/tenantid":"1e8d8218-c5e7-4578-9acc-9abbd5d23315 ","http://schemas.microsoft.com/claims/authnmethodsreferences":"pwd","http://schemas.microsoft.com/identity/claims/objectidentifier":"2468adf0-8211-44e3-95xq-85137af64708","http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn":"admin@contoso.com","puid":"20030000801A118C","http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier":"9vckmEGF7zDKk1YzIY8k0t1_EAPaXoeHyPRn6f413zM","http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname":"John","http://schemas.xmlsoap.org/ws/2005/05/identity/claims/surname":"Smith","name":"John Smith","groups":"cacfe77c-e058-4712-83qw-f9b08849fd60,7f71d11d-4c41-4b23-99d2-d32ce7aa621c,31522864-0578-4ea0-9gdc-e66cc564d18c","http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name":" admin@contoso.com","appid":"c44b4083-3bq0-49c1-b47d-974e53cbdf3c","appidacr":"2","http://schemas.microsoft.com/identity/claims/scope":"user_impersonation","http://schemas.microsoft.com/claims/authnclassreference":"1"}},"level":"Information","location":"global","properties":{"statusCode":"Created","serviceRequestId":"50d5cddb-8ca0-47ad-9b80-6cde2207f97c"}} diff --git a/x-pack/filebeat/module/azure/activitylogs/test/supporttickets_write.log-expected.json b/x-pack/filebeat/module/azure/activitylogs/test/supporttickets_write.log-expected.json new file mode 100644 index 00000000000..7ba307ee669 --- /dev/null +++ b/x-pack/filebeat/module/azure/activitylogs/test/supporttickets_write.log-expected.json @@ -0,0 +1,74 @@ +[ + { + "@timestamp": "2015-01-21T22:14:26.979Z", + "azure.activitylogs.category": "Write", + "azure.activitylogs.event_category": "Administrative", + "azure.activitylogs.identity.authorization.action": "microsoft.support/supporttickets/write", + "azure.activitylogs.identity.authorization.evidence.role": "Subscription Admin", + "azure.activitylogs.identity.authorization.scope": "/subscriptions/s1/resourceGroups/MSSupportGroup/providers/microsoft.support/supporttickets/115012112305841", + "azure.activitylogs.identity.claims.appid": "c44b4083-3bq0-49c1-b47d-974e53cbdf3c", + "azure.activitylogs.identity.claims.appidacr": "2", + "azure.activitylogs.identity.claims.aud": "https://management.core.windows.net/", + "azure.activitylogs.identity.claims.exp": "1421880271", + "azure.activitylogs.identity.claims.groups": "cacfe77c-e058-4712-83qw-f9b08849fd60,7f71d11d-4c41-4b23-99d2-d32ce7aa621c,31522864-0578-4ea0-9gdc-e66cc564d18c", + "azure.activitylogs.identity.claims.http://schemas_microsoft_com/claims/authnclassreference": "1", + "azure.activitylogs.identity.claims.http://schemas_microsoft_com/claims/authnmethodsreferences": "pwd", + "azure.activitylogs.identity.claims.http://schemas_microsoft_com/identity/claims/objectidentifier": "2468adf0-8211-44e3-95xq-85137af64708", + "azure.activitylogs.identity.claims.http://schemas_microsoft_com/identity/claims/scope": "user_impersonation", + "azure.activitylogs.identity.claims.http://schemas_microsoft_com/identity/claims/tenantid": "1e8d8218-c5e7-4578-9acc-9abbd5d23315 ", + "azure.activitylogs.identity.claims.http://schemas_xmlsoap_org/ws/2005/05/identity/claims/givenname": "John", + "azure.activitylogs.identity.claims.http://schemas_xmlsoap_org/ws/2005/05/identity/claims/name": " admin@contoso.com", + "azure.activitylogs.identity.claims.http://schemas_xmlsoap_org/ws/2005/05/identity/claims/nameidentifier": "9vckmEGF7zDKk1YzIY8k0t1_EAPaXoeHyPRn6f413zM", + "azure.activitylogs.identity.claims.http://schemas_xmlsoap_org/ws/2005/05/identity/claims/surname": "Smith", + "azure.activitylogs.identity.claims.http://schemas_xmlsoap_org/ws/2005/05/identity/claims/upn": "admin@contoso.com", + "azure.activitylogs.identity.claims.iat": "1421876371", + "azure.activitylogs.identity.claims.iss": "https://sts.windows.net/72f988bf-86f1-41af-91ab-2d7cd011db47/", + "azure.activitylogs.identity.claims.nbf": "1421876371", + "azure.activitylogs.identity.claims.puid": "20030000801A118C", + "azure.activitylogs.identity.claims.ver": "1.0", + "azure.activitylogs.identity.claims_initiated_by_user.fullname": "John Smith", + "azure.activitylogs.identity.claims_initiated_by_user.givenname": "John", + "azure.activitylogs.identity.claims_initiated_by_user.name": " admin@contoso.com", + "azure.activitylogs.identity.claims_initiated_by_user.schema": "http://schemas.xmlsoap.org/ws/2005/05/identity/claims", + "azure.activitylogs.identity.claims_initiated_by_user.surname": "Smith", + "azure.activitylogs.operation_name": "microsoft.support/supporttickets/write", + "azure.activitylogs.properties.service_request_id": "50d5cddb-8ca0-47ad-9b80-6cde2207f97c", + "azure.activitylogs.properties.status_code": "Created", + "azure.activitylogs.result_signature": "Succeeded.Created", + "azure.activitylogs.result_type": "Success", + "azure.correlation_id": "c776f9f4-36e5-4e0e-809b-c9b3c3fb62a8", + "azure.resource.id": "/subscriptions/s1/resourceGroups/MSSupportGroup/providers/microsoft.support/supporttickets/115012112305841", + "azure.resource.provider": "microsoft.support/supporttickets/115012112305841", + "cloud.provider": "azure", + "event.action": "microsoft.support/supporttickets/write", + "event.dataset": "azure.activitylogs", + "event.duration": -1468967296, + "event.kind": "event", + "event.module": "azure", + "event.type": [ + "change" + ], + "fileset.name": "activitylogs", + "geo.continent_name": "Asia", + "geo.country_iso_code": "JP", + "geo.location.lat": 35.69, + "geo.location.lon": 139.69, + "input.type": "log", + "log.level": "Information", + "log.offset": 0, + "service.type": "azure", + "source.as.number": 2516, + "source.as.organization.name": "KDDI CORPORATION", + "source.geo.continent_name": "Asia", + "source.geo.country_iso_code": "JP", + "source.geo.location.lat": 35.69, + "source.geo.location.lon": 139.69, + "source.ip": "111.111.111.11", + "tags": [ + "forwarded" + ], + "user.domain": "contoso.com", + "user.full_name": "John Smith", + "user.name": "admin" + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/azure/auditlogs/_meta/fields.yml b/x-pack/filebeat/module/azure/auditlogs/_meta/fields.yml index d65ca163d47..e46892f11ec 100644 --- a/x-pack/filebeat/module/azure/auditlogs/_meta/fields.yml +++ b/x-pack/filebeat/module/azure/auditlogs/_meta/fields.yml @@ -2,7 +2,12 @@ type: group description: > Fields for Azure audit logs. + default_field: false fields: + - name: category + type: keyword + description: > + The category of the operation. Currently, Audit is the only supported value. - name: operation_name type: keyword description: > diff --git a/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.json b/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.json deleted file mode 100644 index c8908d45bc2..00000000000 --- a/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.json +++ /dev/null @@ -1,194 +0,0 @@ -{ - "description": "Pipeline for parsing azure activity logs.", - "processors": [ - { - "rename" : { - "field" : "azure", - "target_field" : "azure-eventhub", - "ignore_missing": true - } - }, - { - "json" : { - "field" : "message", - "target_field" : "azure.auditlogs" - } - }, - { - "drop": { - "if" : "ctx.azure.auditlogs.category != 'AuditLogs'" - } - }, - { - "date": { - "field": "azure.auditlogs.time", - "target_field": "@timestamp", - "ignore_failure": true, - "formats": [ - "ISO8601" - ] - } - }, - { - "rename": { - "field": "azure.auditlogs.resourceId", - "target_field": "azure.resource_id", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.auditlogs.durationMs", - "target_field": "event.duration", - "ignore_missing": true - } - }, - { - "script": { - "lang": "painless", - "source": "ctx.event.duration = ctx.event.duration * params.param_nano", - "params": { - "param_nano": 1000000 - } - } - }, - { - "rename": { - "field": "azure.auditlogs.properties.result", - "target_field": "event.outcome", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.auditlogs.level", - "target_field": "log.level", - "ignore_missing": true - } - }, - { - "remove": { - "field": ["message", "azure.auditlogs.time"], - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.auditlogs.category", - "target_field": "event.category", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.auditlogs.operationName", - "target_field": "azure.auditlogs.operation_name", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.auditlogs.resultSignature", - "target_field": "azure.auditlogs.result_signature", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.auditlogs.operationVersion", - "target_field": "azure.auditlogs.operation_version", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.auditlogs.tenantId", - "target_field": "azure.tenant_id", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.auditlogs.correlationId", - "target_field": "azure.correlation_id", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.auditlogs.properties.activityDisplayName", - "target_field": "azure.auditlogs.properties.activity_display_name", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.auditlogs.properties.activityDateTime", - "target_field": "azure.auditlogs.properties.activity_datetime", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.auditlogs.properties.additionalDetails", - "target_field": "azure.auditlogs.properties.additional_details", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.auditlogs.properties.resultReason", - "target_field": "azure.auditlogs.properties.result_reason", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.auditlogs.properties.correlationId", - "target_field": "azure.auditlogs.properties.correlation_id", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.auditlogs.properties.loggedByService", - "target_field": "azure.auditlogs.properties.logged_by_service", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.auditlogs.properties.operationType", - "target_field": "azure.auditlogs.properties.operation_type", - "ignore_missing": true - } - }, - { - "script": { - "lang": "painless", - "source": "if (ctx.azure.auditlogs.properties.targetResources != null) {ctx.azure.auditlogs.properties.target_resources = new HashMap(); for (def i = 0; i < ctx.azure.auditlogs.properties.targetResources.length; i++) { String index = String.valueOf(i); ctx.azure.auditlogs.properties.target_resources[index]= new HashMap(); if(ctx.azure.auditlogs.properties.targetResources[i].displayName!= null) {ctx.azure.auditlogs.properties.target_resources[index].display_name=ctx.azure.auditlogs.properties.targetResources[i].displayName;}ctx.azure.auditlogs.properties.target_resources[index].id=ctx.azure.auditlogs.properties.targetResources[i].id;ctx.azure.auditlogs.properties.target_resources[index].type=ctx.azure.auditlogs.properties.targetResources[i].type; if(ctx.azure.auditlogs.properties.targetResources[i].ipAddress!= null) {ctx.azure.auditlogs.properties.target_resources[index].ip_address=ctx.azure.auditlogs.properties.targetResources[i].ipAddress;} if (ctx.azure.auditlogs.properties.targetResources[i].userPrincipalName!=null) {ctx.azure.auditlogs.properties.target_resources[index].user_principal_name=ctx.azure.auditlogs.properties.targetResources[i].userPrincipalName;}ctx.azure.auditlogs.properties.target_resources[index].modified_properties= new HashMap(); for (def j = 0; j < ctx.azure.auditlogs.properties.targetResources[i].modifiedProperties.length; j++) { String n = String.valueOf(j);ctx.azure.auditlogs.properties.target_resources[index].modified_properties[n]= new HashMap();ctx.azure.auditlogs.properties.target_resources[index].modified_properties[n].display_name=ctx.azure.auditlogs.properties.targetResources[i].modifiedProperties[j].displayName;ctx.azure.auditlogs.properties.target_resources[index].modified_properties[n].new_value=ctx.azure.auditlogs.properties.targetResources[i].modifiedProperties[j].newValue;ctx.azure.auditlogs.properties.target_resources[index].modified_properties[n].old_value=ctx.azure.auditlogs.properties.targetResources[i].modifiedProperties[j].oldValue; }} ctx.azure.auditlogs.properties.remove('targetResources');}", - "ignore_failure": true - } - }, - { - "rename": { - "field": "azure.auditlogs.properties.initiatedBy", - "target_field": "azure.auditlogs.properties.initiated_by", - "ignore_missing": true - } - }, - { - "pipeline": { - "name": "{< IngestPipeline "azure-shared-pipeline" >}" - } - } - ], - "on_failure": [ - { - "set": { - "field": "error.message", - "value": "{{ _ingest.on_failure_message }}" - } - } - ] -} diff --git a/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.yml b/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.yml new file mode 100644 index 00000000000..2bf26322faf --- /dev/null +++ b/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.yml @@ -0,0 +1,141 @@ +description: Pipeline for parsing azure activity logs. +processors: +- rename: + field: azure + target_field: azure-eventhub + ignore_missing: true +- json: + field: message + target_field: azure.auditlogs +- drop: + if: ctx.azure.auditlogs.category != 'AuditLogs' +- date: + field: azure.auditlogs.time + target_field: '@timestamp' + ignore_failure: true + formats: + - ISO8601 +- rename: + field: azure.auditlogs.resourceId + target_field: azure.resource_id + ignore_missing: true +- rename: + field: azure.auditlogs.durationMs + target_field: event.duration + ignore_missing: true +- script: + lang: painless + source: ctx.event.duration = ctx.event.duration * params.param_nano + params: + param_nano: 1000000 +- rename: + field: azure.auditlogs.properties.result + target_field: event.outcome + if: "ctx?.azure?.auditlogs?.properties?.result != null && ctx.azure.auditlogs.properties.result instanceof String && (ctx.azure.auditlogs.properties.result.toLowerCase() == 'success' || ctx.azure.auditlogs.properties.result.toLowerCase() == 'failure')" +- rename: + field: azure.auditlogs.level + target_field: log.level + ignore_missing: true +- remove: + field: + - message + - azure.auditlogs.time + ignore_missing: true +- convert: + field: azure.auditlogs.operationName + target_field: event.action + type: string + ignore_missing: true + ignore_failure: true +- rename: + field: azure.auditlogs.operationName + target_field: azure.auditlogs.operation_name + ignore_missing: true +- rename: + field: azure.auditlogs.resultSignature + target_field: azure.auditlogs.result_signature + ignore_missing: true +- rename: + field: azure.auditlogs.operationVersion + target_field: azure.auditlogs.operation_version + ignore_missing: true +- rename: + field: azure.auditlogs.tenantId + target_field: azure.tenant_id + ignore_missing: true +- rename: + field: azure.auditlogs.correlationId + target_field: azure.correlation_id + ignore_missing: true +- rename: + field: azure.auditlogs.properties.activityDisplayName + target_field: azure.auditlogs.properties.activity_display_name + ignore_missing: true +- rename: + field: azure.auditlogs.properties.activityDateTime + target_field: azure.auditlogs.properties.activity_datetime + ignore_missing: true +- rename: + field: azure.auditlogs.properties.additionalDetails + target_field: azure.auditlogs.properties.additional_details + ignore_missing: true +- rename: + field: azure.auditlogs.properties.resultReason + target_field: azure.auditlogs.properties.result_reason + ignore_missing: true +- rename: + field: azure.auditlogs.properties.correlationId + target_field: azure.auditlogs.properties.correlation_id + ignore_missing: true +- rename: + field: azure.auditlogs.properties.loggedByService + target_field: azure.auditlogs.properties.logged_by_service + ignore_missing: true +- rename: + field: azure.auditlogs.properties.operationType + target_field: azure.auditlogs.properties.operation_type + ignore_missing: true +- script: + lang: painless + source: >- + if (ctx.azure.auditlogs.properties.targetResources != null) { + ctx.azure.auditlogs.properties.target_resources = new HashMap(); + for (def i = 0; i < ctx.azure.auditlogs.properties.targetResources.length; i++) { + String index = String.valueOf(i); + ctx.azure.auditlogs.properties.target_resources[index] = new HashMap(); + if(ctx.azure.auditlogs.properties.targetResources[i].displayName != null) { + ctx.azure.auditlogs.properties.target_resources[index].display_name = ctx.azure.auditlogs.properties.targetResources[i].displayName; + } + ctx.azure.auditlogs.properties.target_resources[index].id = ctx.azure.auditlogs.properties.targetResources[i].id; + ctx.azure.auditlogs.properties.target_resources[index].type = ctx.azure.auditlogs.properties.targetResources[i].type; + if (ctx.azure.auditlogs.properties.targetResources[i].ipAddress != null) { + ctx.azure.auditlogs.properties.target_resources[index].ip_address = ctx.azure.auditlogs.properties.targetResources[i].ipAddress; + } + if (ctx.azure.auditlogs.properties.targetResources[i].userPrincipalName != null) { + ctx.azure.auditlogs.properties.target_resources[index].user_principal_name = ctx.azure.auditlogs.properties.targetResources[i].userPrincipalName; + } + ctx.azure.auditlogs.properties.target_resources[index].modified_properties = new HashMap(); + for (def j = 0; j < ctx.azure.auditlogs.properties.targetResources[i].modifiedProperties.length; j++) { + String n = String.valueOf(j); + ctx.azure.auditlogs.properties.target_resources[index].modified_properties[n] = new HashMap(); + ctx.azure.auditlogs.properties.target_resources[index].modified_properties[n].display_name = ctx.azure.auditlogs.properties.targetResources[i].modifiedProperties[j].displayName; + ctx.azure.auditlogs.properties.target_resources[index].modified_properties[n].new_value = ctx.azure.auditlogs.properties.targetResources[i].modifiedProperties[j].newValue; + ctx.azure.auditlogs.properties.target_resources[index].modified_properties[n].old_value = ctx.azure.auditlogs.properties.targetResources[i].modifiedProperties[j].oldValue; + } + } + ctx.azure.auditlogs.properties.remove('targetResources'); + } + ignore_failure: true +- rename: + field: azure.auditlogs.properties.initiatedBy + target_field: azure.auditlogs.properties.initiated_by + ignore_missing: true +- set: + field: event.kind + value: event +- pipeline: + name: '{< IngestPipeline "azure-shared-pipeline" >}' +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/azure/auditlogs/manifest.yml b/x-pack/filebeat/module/azure/auditlogs/manifest.yml index 85029fc97a9..f8afbc44401 100644 --- a/x-pack/filebeat/module/azure/auditlogs/manifest.yml +++ b/x-pack/filebeat/module/azure/auditlogs/manifest.yml @@ -15,6 +15,6 @@ var: default: [forwarded] ingest_pipeline: - - ingest/pipeline.json - - ../azure-shared-pipeline.json + - ingest/pipeline.yml + - ../azure-shared-pipeline.yml input: config/{{.input}}.yml diff --git a/x-pack/filebeat/module/azure/auditlogs/test/auditlogs.log-expected.json b/x-pack/filebeat/module/azure/auditlogs/test/auditlogs.log-expected.json index 9e3a37a4352..7d18285024a 100644 --- a/x-pack/filebeat/module/azure/auditlogs/test/auditlogs.log-expected.json +++ b/x-pack/filebeat/module/azure/auditlogs/test/auditlogs.log-expected.json @@ -1,6 +1,7 @@ [ { "@timestamp": "2019-10-18T15:30:51.027Z", + "azure.auditlogs.category": "AuditLogs", "azure.auditlogs.identity": "Device Registration Service", "azure.auditlogs.operation_name": "Update device", "azure.auditlogs.operation_version": "1.0", @@ -28,11 +29,12 @@ "azure.resource.provider": "Microsoft.aadiam", "azure.tenant_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", "cloud.provider": "azure", - "event.category": "AuditLogs", + "event.action": "Update device", "event.dataset": "azure.auditlogs", - "event.duration": 0.0, + "event.duration": 0, + "event.kind": "event", "event.module": "azure", - "event.outcome": "Success", + "event.outcome": "success", "fileset.name": "auditlogs", "input.type": "log", "log.level": "Informational", diff --git a/x-pack/filebeat/module/azure/azure-shared-pipeline.json b/x-pack/filebeat/module/azure/azure-shared-pipeline.json deleted file mode 100644 index 9bfad9cf1bb..00000000000 --- a/x-pack/filebeat/module/azure/azure-shared-pipeline.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "description": "Pipeline for parsing azure activity logs.", - "processors": [ - { - "set": { - "field": "cloud.provider", - "value": "azure" - } - }, - { - "grok": { - "field": "azure.resource_id", - "patterns": ["/SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/NAMESPACES/%{NAMESPACE:azure.resource.namespace}/AUTHORIZATIONRULES/%{RULE:azure.resource.authorization_rule}"], - "pattern_definitions" : { - "SUBID" : "(\\{){0,1}[0-9a-fA-F]{8}\\-[0-9a-fA-F]{4}\\-[0-9a-fA-F]{4}\\-[0-9a-fA-F]{4}\\-[0-9a-fA-F]{12}(\\}){0,1}", - "GROUPID" : ".+", - "PROVIDERNAME" : ".+", - "NAMESPACE": ".+", - "RULE": ".+" - }, - "ignore_failure": true - } - }, - { - "grok": { - "field": "azure.resource_id", - "patterns": ["/SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name}"], - "pattern_definitions" : { - "SUBID" : "(\\{){0,1}[0-9a-fA-F]{8}\\-[0-9a-fA-F]{4}\\-[0-9a-fA-F]{4}\\-[0-9a-fA-F]{4}\\-[0-9a-fA-F]{12}(\\}){0,1}", - "GROUPID" : ".+", - "PROVIDERNAME" : "([A-Z])\\w+.([A-Z])\\w+/([A-Z])\\w+.", - "NAME": "((?!AUTHORIZATIONRULES).)*$" - }, - "ignore_failure": true - } - }, - { - "grok": { - "field": "azure.resource_id", - "patterns": ["/providers/%{PROVIDER:azure.resource.provider}"], - "pattern_definitions" : { - "PROVIDER" : ".+" - }, - "ignore_failure": true - } - }, - { - "rename": { - "field": "azure.resource_id", - "target_field": "azure.resource.id", - "ignore_missing": true - } - }, - { - "script": { - "source": "if (ctx.event.outcome !=null) {ctx.event.outcome = ctx.event.outcome.substring(0,1).toUpperCase() + ctx.event.outcome.substring(1,ctx.event.outcome.length()).toLowerCase();}", - "ignore_failure": true - } - } - ], - "on_failure": [ - { - "set": { - "field": "error.message", - "value": "{{ _ingest.on_failure_message }}" - } - } - ] -} diff --git a/x-pack/filebeat/module/azure/azure-shared-pipeline.yml b/x-pack/filebeat/module/azure/azure-shared-pipeline.yml new file mode 100644 index 00000000000..e849758d3ad --- /dev/null +++ b/x-pack/filebeat/module/azure/azure-shared-pipeline.yml @@ -0,0 +1,44 @@ +description: Pipeline for parsing azure activity logs. +processors: +- set: + field: cloud.provider + value: azure +- grok: + field: azure.resource_id + patterns: + - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/NAMESPACES/%{NAMESPACE:azure.resource.namespace}/AUTHORIZATIONRULES/%{RULE:azure.resource.authorization_rule} + pattern_definitions: + SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1} + GROUPID: .+ + PROVIDERNAME: .+ + NAMESPACE: .+ + RULE: .+ + ignore_failure: true +- grok: + field: azure.resource_id + patterns: + - /SUBSCRIPTIONS/%{SUBID:azure.subscription_id}/RESOURCEGROUPS/%{GROUPID:azure.resource.group}/PROVIDERS/%{PROVIDERNAME:azure.resource.provider}/%{NAME:azure.resource.name} + pattern_definitions: + SUBID: (\{){0,1}[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}(\}){0,1} + GROUPID: .+ + PROVIDERNAME: ([A-Z])\w+.([A-Z])\w+/([A-Z])\w+. + NAME: ((?!AUTHORIZATIONRULES).)*$ + ignore_failure: true +- grok: + field: azure.resource_id + patterns: + - /providers/%{PROVIDER:azure.resource.provider} + pattern_definitions: + PROVIDER: .+ + ignore_failure: true +- rename: + field: azure.resource_id + target_field: azure.resource.id + ignore_missing: true +- lowercase: + field: event.outcome + ignore_missing: true +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/azure/fields.go b/x-pack/filebeat/module/azure/fields.go index 846cbb25232..d358caa2edd 100644 --- a/x-pack/filebeat/module/azure/fields.go +++ b/x-pack/filebeat/module/azure/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAzure returns asset data. // This is the base64 encoded gzipped contents of module/azure. func AssetAzure() string { - return "eJzsW09v27gTvedTDHIs0P7uOfwAb9ouAmybIk3PAiONFW5kkUtSDtxPvyBl/ScpOqbkLnZ9aiPnvSdyZjh8ZN7DCx5ugPysBF4BKKoKvIHrjf7/9RWAwAKJxBt4QkWuADKUqaBcUVbewP+vAADMd+ELy6pCQ2wpFpm8MY/eQ0l22MHrjzpwvIFcsIoff2LBHML0oWT11H47oVn7vAF+wcMrE/2fW+HrTy29Dwl3HyeUKRMCCxKF8bbDslEpLEmpzmapYWwEAiWrRIoT/P6EzKA/TDHGs9WnHLyM74VmaPvU/Vfrc43fIhbdFLdh5ILtaYZiAVKN8T/NIjkZjHbHbn+6Kn085q9jtLZ8VOqZCfqzTkFR15lIpJs+NgywW/pU0T1Vh4LlcjZvxgUzQMZnkzywZeJYkBpC0IwfgtMMS0XVwTo0tryYGZg7O55NQl9GWhC6kwktqaJEYZY8HZJKTvLDLy1Anv7cGi5oueDpAA4ul2zwR/NQqj3AThAMtigfy8jpHst1tPzupeqWXLGOnO8eokbMtiqKddR89jG1Y5M+446sMDR2nmHWfXjnzDL29CemyvK4fpDMSe19LdkRzmmZH3/n+t31OdnrfKVB0V+iemxmCEJKhkwZXyMxnDT9Ncr6FpGVbNw8jRTU/VA5aUmGYlzzdoKUT34e3/T15QpWYEKkpHm5w1IlvimFwME84S3054EVCJ0Eb1SNhGe4NcvfeGuyluqOf9ySuySvL/LkELjISPbmP2AkuaBlSjkpVhf7rWE+TaZWcimhE+5GHuMo6k1F3J3MfYM77Sl7G/GqUImec6I6dyQC+YNBBjty2zAQhTkT9t3Cm2hvbYi9XTJHoSiO1/s3b0++uRDnNigSxZ6mmAj8q0LpSPa5cAwIw+81DzzUPHA3xWklKaIqmaQss6VIDC2GAAYE021ulVEVtMc9aS+rUU/ayC6VlI/P2GG7E7Oj36OQ02YnmgIbfNBm/k3c1u28z3Y8910nFiT8OtUvfjnSU9uGuht/rjjV47JMEfiD5S78sdOVZFTyghxsGRhJzabxuI5UdvdlFC8CifRsBs8S9GCwTeVSz2imcWa0nOcCEVU5zgvGUgqW57XXdlzeFguhvLbZXDTTKupo/CKo6bosK4fz/CESvWc6ujwiChX15JD+wpkJpPGlIrvpdnqm2Ys0Dk70dnUhIkedvPXBgs+mctkCC9hPj0ZVeyA2NaLCHKCZSglx7ZePvmIJc0EfWYxj/9dOu3vHF1HEo4ulHQmekCwTKG0zHFkMUA4bD1mjqZIokm5vvE7s/JAoug35fAjtWEa3FLOk62asmQtBpl6gzwxvSPYTxuDL8Z3cHVrzCbUPS3xN9qSoVjY3vuIr+GlPKFALaWyKVZhOVmSXGMj7InMIbAtI7yjz5IUrpI8ot0zsjufPmBOR0TI3PeiRmb31QJNwVzZGM+E3TorQBDp2kq1b5zwf7SuPGgKNUxNQG2GaWOvrnW0CYBgDFudpUX0bzkO84fHEry1zOu0zDY3jagHEzKcfbo7QhNIyL5hNoW0G/ONSae2TloAcotzXbi6mzNLmhvyjzXyal7SMbjVr2Pe0/M9stijwmc3/Nvf3SN+Dii7AhX0hf38Zv7tO4/MM7wuYc6lA080Tt8/+dlfutgY3CGB1/gYewPI+u1mOgzz2X8jODvNHYo3OTLMyELXUwBghPkeZ88W4Hf16n3qF8yDOw8LU6+fFqBwcXPjdbUuKpUr0wFQSl0oVQ6JHH6wkXdqWmbmARYqEpClKmdT3CZbK4JYOajpw0LU9jqA51eqWvnFxfyQC0Vy5cAc1lQktFQpzUrNQSN9J8HG07Rd7wTKhUlYoFkywR00DNY0/wwaClju+GwjynuBxwXSgGR+Y7jCx3BpuFG0LZl3XA/R8a1nMwg20hB0tCipRp5g7vgWVL0mGitBimYF6oPIFHAQDEQXusUhIngvMdReyoBxDBR4qi7CsEnoK67ZxcW01m2lS76ZsA3m6gC0U5EaQHX/8J3krLLLt33YFrbQzy8gZlvd3F3KIq41CMOG6LAdxz8w+aS6wcrXeFZrrhDMF4FKDdVS3xnHwR0M1Z6IeLQldBw5S4W55XfcNI3gYG3lPgr3O2bxRVP3mIfrF7xYoUUnlu9Ad83hfc/m7g+ZC78yV+GjXaNtHV38HAAD//3MzpKM=" + return "eJzsW01z2zYQvftX7PiYady7Dp1RnaTjmSbOOM6ZA5MrGjVEsAAoj/LrO+D3BwBCFkil0+rUmsq+R2B3sfsWeg8veNwA+VEIvAJQVDHcwPVW///1FYBAhkTiBp5QkSuABGUsaK4ozzbw2xUAQPld+MyTgmkTO4oskZvy0XvIyB478/qjjjluIBW8yOu/GGwOzfRNyeKp/XZEk/Z5Y/gFj69c9P9uNF99Kup9k3D3YQIZcyGQkSCIt50tE5TCjGTqbJTKjAlAoOSFiHFiv78hM9YfpjbGu9WHHLyM64VmYPvQ/VfrY43fIhTc1G6DmAt+oAmKBUC1jV81iszJYLU7dPPTVeHDIX8ZW2vTR6GeuaA/qhAUVZ4JBLrt24aB7RY+VvRA1ZHxVM7GzThhdjR2pGAqKiNlAzvCJPqF26cytmDHRZ2vGj6gCd14RyFmiqqjceVMYTOzbndmeyYKfRoxI3QvI5pRRYnCJHo6RoWchI+bmgc9/bktsaDFgqcjWLBstMHt7EOqZv87gTCYgmBMI6UHzNbh8ocTqjuRxTp0vjmAGjK7grF12HxyIbVrEz/jnqywNGacYdTdvLNGGX/6C2NleFw9iOao9r4W7Ume0yyt/831u+tzotf6SoMzYYnssZ0B8EkZMub5GoFhhekfYca3CMxka8dpqKAul7JJxTIkY9u3E6h8dOO4tq9PV3CGEZGSptkeMxW5thQ8F/OEt9CfB84QOgpOrxoRT3BXHn/jzmUt1h3+uGK3UV6f5MkucJGV7O2/x0rmgmYxzQlbnezXBvk0mprJpYhOsBt6PEdR9RxhG537xu60puz16bpbMCzLea1dwZT9fWtQ7WhEdYpNOGSz5bZKIQpTLswtyptgb00Wu4NIB3N40I/arhs6F9q1FMVxffPmduyrzeJcQyZRHGiMkcC/C5SW5DYXfh5h963CgYcKB+6mdlpKiqhCRjFPTCkhBJcSAAYA066/SKjyavlP6t211UnjPiMOuPr68P77+IytVeA7UM/YZcIbgNtCCMwUO/4C2/JtqKy+k7EjyCLPudDN9oGwAm9WzauPfab23NrBH1DIab0ajIHJvJce8yZsoyLjEpbPfdeJyAw/z1kSPsPqrW2j125/Lt9W67JMXvuTpzb7Yy0zSqjMGTmaIjAQm20jU9ZQZgFt5C8CiXT082cReihtl8lY5ys2v1rWyU9AVpaJ0JgK42layaX1ib2YC6WVUmqDmWZRS+0egE1XKBsxrBOmQPCO7ejiiChU1BFD+gtnBpC2LxXZTxWRmSog0DpYrbenCxEp6uCtRkcupdGm7CygID6WrNqR51RL9BPxZjIlhFXQPriSJcw5fWAylha+3XZ70x6QxKMNpV2JPCJJIlCadjgwGaA5bB1gDadCoog6eWMd3/kuUXSayrwL7XlCdxSTqKtmjJELXrqs56gA3hDsJ6zB5/qd7BVa8/FVgDN8jcpmxu4US+hTX/AV3LAnJKiFODbJyo8nZ8klFvKeJRaCbQLpTaNPPrh86ohsx8W+vmGAKREJzdKyBq2R+Vtn0iS3RWOwOcrWCuEbQHUl2Qqu1hF3n3lQF2jEJ4/cCNPAWp/vbBEAQx8wiGmL8tvmuY+8P974tWlOt32moLHcDoGQ8fTdjuEbUJrmBaPJt8yAf10orT0s84ghmrvKzcWYGcpcn/9oI5+mGc2Cq+fa7HuahdTP/9ei/3vicA3fMxWcgM32Ree7K88cVp7nLiP5V5nsPM3/AvpkLLBsaIh91PB2YfK2Ml5aAKP4OZBBlh81lBWJ15jhJ1L0/SSiUKszU68NSC21MCURl6ie54thW1qWPvQKI7E893NTp6QZInPkYLPf3RmmmKlIL0whcalQKUH06oMRpAvbLCmvERIWkThGKaPqlshSEdzCQQUHFri2jhM0pZrd0vdo7msgEM1FGrtTUxnRTKEoh1ULufSdBBdGW2LyF8wiKmWBYsEAe9QwUMG4I2xAaLkJ5oCQc4iZC64drZTC6R4jw933htGOceO57sHna4tSHtxAM9hTxqhEHWJ2/xZUvkQJKkLZMgv1QOULWAAGJBgekEUkTQWmugpZkE4JBQ4oA7GkEHoLq7JxcW4VWlmk3k3RBvR0AlvIyUtCZvvj352ucMi2P2D0OmlnjpEzVP9vNss+wj4KwYXtCiSEHRt+1FhgxGrlOywvic4kgEstVs1ujYn4hxJqTkeuZRedB45S4X55XvcNIjgQG3pPgr/OKd1BWP3uAPrJr1coUUijPrMAqUeN5a4OmmvaMz/sCHY5un109U8AAAD//4AUZiI=" } diff --git a/x-pack/filebeat/module/azure/signinlogs/_meta/fields.yml b/x-pack/filebeat/module/azure/signinlogs/_meta/fields.yml index 25f1b30d121..9cb2ebbe9ce 100644 --- a/x-pack/filebeat/module/azure/signinlogs/_meta/fields.yml +++ b/x-pack/filebeat/module/azure/signinlogs/_meta/fields.yml @@ -2,6 +2,7 @@ type: group description: > Fields for Azure sign-in logs. + default_field: false fields: - name: operation_name type: keyword @@ -23,10 +24,18 @@ type: keyword description: > Result description + - name: result_type + type: keyword + description: > + Result type - name: identity type: keyword description: > Identity + - name: category + type: keyword + description: > + Category - name: properties type: group description: > diff --git a/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.json b/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.json deleted file mode 100644 index f802bf2e4bc..00000000000 --- a/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.json +++ /dev/null @@ -1,431 +0,0 @@ -{ - "description": "Pipeline for parsing azure signin logs.", - "processors": [ - { - "rename" : { - "field" : "azure", - "target_field" : "azure-eventhub", - "ignore_missing": true - } - }, - { - "json" : { - "field" : "message", - "target_field" : "azure.signinlogs" - } - }, - { - "drop": { - "if" : "ctx.azure.signinlogs.category != 'SignInLogs'" - } - }, - { - "date": { - "field": "azure.signinlogs.time", - "target_field": "@timestamp", - "ignore_failure": false, - "formats": [ - "ISO8601" - ] - } - }, - { - "remove": { - "field": ["message", "azure.signinlogs.time"], - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.resourceId", - "target_field": "azure.resource_id", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.callerIpAddress", - "target_field": "source.ip", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.Level", - "target_field": "log.level", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.durationMs", - "target_field": "event.duration", - "ignore_missing": true - } - }, - { - "script": { - "lang": "painless", - "source": "ctx.event.duration = ctx.event.duration * params.param_nano", - "params": { - "param_nano": 1000000 - } - } - }, - { - "rename": { - "field": "azure.signinlogs.location", - "target_field": "geo.country_iso_code", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.resultType", - "target_field": "event.outcome", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.category", - "target_field": "event.category", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.operationName", - "target_field": "azure.signinlogs.operation_name", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.resultSignature", - "target_field": "azure.signinlogs.result_signature", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.resultDescription", - "target_field": "azure.signinlogs.result_description", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.operationVersion", - "target_field": "azure.signinlogs.operation_version", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.tenantId", - "target_field": "azure.tenant_id", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.correlationId", - "target_field": "azure.correlation_id", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.properties.networkLocationDetails", - "target_field": "azure.signinlogs.properties.network_location_details", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.properties.resourceId", - "target_field": "azure.signinlogs.properties.resource_id", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.properties.appliedConditionalAccessPolicies", - "target_field": "azure.signinlogs.properties.applied_conditional_access_policies", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.properties.authenticationDetails", - "target_field": "azure.signinlogs.properties.authentication_details", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.properties.authenticationRequirementPolicies", - "target_field": "azure.signinlogs.properties.authentication_requirement_policies", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.properties.authenticationProcessingDetails", - "target_field": "azure.signinlogs.properties.authentication_processing_details", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.properties.deviceDetail", - "target_field": "azure.signinlogs.properties.device_detail", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.properties.device_detail.deviceId", - "target_field": "azure.signinlogs.properties.device_detail.device_id", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.properties.device_detail.operatingSystem", - "target_field": "azure.signinlogs.properties.device_detail.operating_system", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.properties.device_detail.displayName", - "target_field": "azure.signinlogs.properties.device_detail.display_name", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.properties.device_detail.trustType", - "target_field": "azure.signinlogs.properties.device_detail.trust_type", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.properties.createdDateTime", - "target_field": "azure.signinlogs.properties.created_at", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.properties.userDisplayName", - "target_field": "azure.signinlogs.properties.user_display_name", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.properties.correlationId", - "target_field": "azure.signinlogs.properties.correlation_id", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.properties.userPrincipalName", - "target_field": "azure.signinlogs.properties.user_principal_name", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.properties.userId", - "target_field": "azure.signinlogs.properties.user_id", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.properties.appId", - "target_field": "azure.signinlogs.properties.app_id", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.properties.appDisplayName", - "target_field": "azure.signinlogs.properties.app_display_name", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.properties.ipAddress", - "target_field": "azure.signinlogs.properties.ip_address", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.properties.clientAppUsed", - "target_field": "azure.signinlogs.properties.client_app_used", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.properties.conditionalAccessStatus", - "target_field": "azure.signinlogs.properties.conditional_access_status", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.properties.originalRequestId", - "target_field": "azure.signinlogs.properties.original_request_id", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.properties.isInteractive", - "target_field": "azure.signinlogs.properties.is_interactive", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.properties.tokenIssuerName", - "target_field": "azure.signinlogs.properties.token_issuer_name", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.properties.tokenIssuerType", - "target_field": "azure.signinlogs.properties.token_issuer_type", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.properties.processingTimeInMilliseconds", - "target_field": "azure.signinlogs.properties.processing_time_ms", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.properties.riskDetail", - "target_field": "azure.signinlogs.properties.risk_detail", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.properties.riskLevelAggregated", - "target_field": "azure.signinlogs.properties.risk_level_aggregated", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.properties.riskLevelDuringSignIn", - "target_field": "azure.signinlogs.properties.risk_level_during_signin", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.properties.riskState", - "target_field": "azure.signinlogs.properties.risk_state", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.properties.resourceDisplayName", - "target_field": "azure.signinlogs.properties.resource_display_name", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.properties.status.errorCode", - "target_field": "azure.signinlogs.properties.status.error_code", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.properties.status.failureReason", - "target_field": "message", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.properties.status.additionalDetails", - "target_field": "message", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.properties.location.city", - "target_field": "geo.city_name", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.properties.location.state", - "target_field": "geo.country_name", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.properties.location.geoCoordinates.latitude", - "target_field": "geo.location.lat", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.properties.location.geoCoordinates.longitude", - "target_field": "geo.location.lon", - "ignore_missing": true - } - }, - { - "rename": { - "field": "azure.signinlogs.properties.servicePrincipalId", - "target_field": "azure.signinlogs.properties.service_principal_id", - "ignore_missing": true - } - }, - { - "remove": { - "field": ["azure.signinlogs.properties.location"], - "ignore_missing": true - } - }, - { - "pipeline": { - "name": "{< IngestPipeline "azure-shared-pipeline" >}" - } - } - ], - "on_failure": [ - { - "set": { - "field": "error.message", - "value": "{{ _ingest.on_failure_message }}" - } - } - ] -} diff --git a/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.yml b/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.yml new file mode 100644 index 00000000000..9d5351bf36a --- /dev/null +++ b/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.yml @@ -0,0 +1,299 @@ +description: Pipeline for parsing azure signin logs. +processors: +- rename: + field: azure + target_field: azure-eventhub + ignore_missing: true +- json: + field: message + target_field: azure.signinlogs +- drop: + if: ctx.azure.signinlogs.category != 'SignInLogs' +- date: + field: azure.signinlogs.time + target_field: '@timestamp' + ignore_failure: false + formats: + - ISO8601 +- remove: + field: + - message + - azure.signinlogs.time + ignore_missing: true +- rename: + field: azure.signinlogs.resourceId + target_field: azure.resource_id + ignore_missing: true +- rename: + field: azure.signinlogs.callerIpAddress + target_field: source.ip + ignore_missing: true +- rename: + field: azure.signinlogs.Level + target_field: log.level + ignore_missing: true +- rename: + field: azure.signinlogs.durationMs + target_field: event.duration + ignore_missing: true +- script: + lang: painless + source: ctx.event.duration = ctx.event.duration * params.param_nano + params: + param_nano: 1000000 +- rename: + field: azure.signinlogs.location + target_field: geo.country_iso_code + ignore_missing: true +- rename: + field: azure.signinlogs.resultType + target_field: azure.signinlogs.result_type + ignore_missing: true +- rename: + field: azure.signinlogs.operationName + target_field: azure.signinlogs.operation_name + ignore_missing: true +- convert: + field: azure.signinlogs.operation_name + target_field: event.action + type: string + ignore_missing: true +- rename: + field: azure.signinlogs.resultSignature + target_field: azure.signinlogs.result_signature + ignore_missing: true +- rename: + field: azure.signinlogs.resultDescription + target_field: azure.signinlogs.result_description + ignore_missing: true +- rename: + field: azure.signinlogs.operationVersion + target_field: azure.signinlogs.operation_version + ignore_missing: true +- rename: + field: azure.signinlogs.tenantId + target_field: azure.tenant_id + ignore_missing: true +- rename: + field: azure.signinlogs.correlationId + target_field: azure.correlation_id + ignore_missing: true +- rename: + field: azure.signinlogs.properties.networkLocationDetails + target_field: azure.signinlogs.properties.network_location_details + ignore_missing: true +- rename: + field: azure.signinlogs.properties.resourceId + target_field: azure.signinlogs.properties.resource_id + ignore_missing: true +- rename: + field: azure.signinlogs.properties.appliedConditionalAccessPolicies + target_field: azure.signinlogs.properties.applied_conditional_access_policies + ignore_missing: true +- rename: + field: azure.signinlogs.properties.authenticationDetails + target_field: azure.signinlogs.properties.authentication_details + ignore_missing: true +- rename: + field: azure.signinlogs.properties.authenticationRequirementPolicies + target_field: azure.signinlogs.properties.authentication_requirement_policies + ignore_missing: true +- rename: + field: azure.signinlogs.properties.authenticationProcessingDetails + target_field: azure.signinlogs.properties.authentication_processing_details + ignore_missing: true +- rename: + field: azure.signinlogs.properties.deviceDetail + target_field: azure.signinlogs.properties.device_detail + ignore_missing: true +- rename: + field: azure.signinlogs.properties.device_detail.deviceId + target_field: azure.signinlogs.properties.device_detail.device_id + ignore_missing: true +- rename: + field: azure.signinlogs.properties.device_detail.operatingSystem + target_field: azure.signinlogs.properties.device_detail.operating_system + ignore_missing: true +- rename: + field: azure.signinlogs.properties.device_detail.displayName + target_field: azure.signinlogs.properties.device_detail.display_name + ignore_missing: true +- rename: + field: azure.signinlogs.properties.device_detail.trustType + target_field: azure.signinlogs.properties.device_detail.trust_type + ignore_missing: true +- rename: + field: azure.signinlogs.properties.createdDateTime + target_field: azure.signinlogs.properties.created_at + ignore_missing: true +- rename: + field: azure.signinlogs.properties.userDisplayName + target_field: azure.signinlogs.properties.user_display_name + ignore_missing: true +- rename: + field: azure.signinlogs.properties.correlationId + target_field: azure.signinlogs.properties.correlation_id + ignore_missing: true +- rename: + field: azure.signinlogs.properties.userPrincipalName + target_field: azure.signinlogs.properties.user_principal_name + ignore_missing: true +- rename: + field: azure.signinlogs.properties.userId + target_field: azure.signinlogs.properties.user_id + ignore_missing: true +- rename: + field: azure.signinlogs.properties.appId + target_field: azure.signinlogs.properties.app_id + ignore_missing: true +- rename: + field: azure.signinlogs.properties.appDisplayName + target_field: azure.signinlogs.properties.app_display_name + ignore_missing: true +- rename: + field: azure.signinlogs.properties.ipAddress + target_field: azure.signinlogs.properties.ip_address + ignore_missing: true +- rename: + field: azure.signinlogs.properties.clientAppUsed + target_field: azure.signinlogs.properties.client_app_used + ignore_missing: true +- rename: + field: azure.signinlogs.properties.conditionalAccessStatus + target_field: azure.signinlogs.properties.conditional_access_status + ignore_missing: true +- rename: + field: azure.signinlogs.properties.originalRequestId + target_field: azure.signinlogs.properties.original_request_id + ignore_missing: true +- rename: + field: azure.signinlogs.properties.isInteractive + target_field: azure.signinlogs.properties.is_interactive + ignore_missing: true +- rename: + field: azure.signinlogs.properties.tokenIssuerName + target_field: azure.signinlogs.properties.token_issuer_name + ignore_missing: true +- rename: + field: azure.signinlogs.properties.tokenIssuerType + target_field: azure.signinlogs.properties.token_issuer_type + ignore_missing: true +- rename: + field: azure.signinlogs.properties.processingTimeInMilliseconds + target_field: azure.signinlogs.properties.processing_time_ms + ignore_missing: true +- rename: + field: azure.signinlogs.properties.riskDetail + target_field: azure.signinlogs.properties.risk_detail + ignore_missing: true +- rename: + field: azure.signinlogs.properties.riskLevelAggregated + target_field: azure.signinlogs.properties.risk_level_aggregated + ignore_missing: true +- rename: + field: azure.signinlogs.properties.riskLevelDuringSignIn + target_field: azure.signinlogs.properties.risk_level_during_signin + ignore_missing: true +- rename: + field: azure.signinlogs.properties.riskState + target_field: azure.signinlogs.properties.risk_state + ignore_missing: true +- rename: + field: azure.signinlogs.properties.resourceDisplayName + target_field: azure.signinlogs.properties.resource_display_name + ignore_missing: true +- rename: + field: azure.signinlogs.properties.status.errorCode + target_field: azure.signinlogs.properties.status.error_code + ignore_missing: true +- rename: + field: azure.signinlogs.properties.status.failureReason + target_field: message + ignore_missing: true +- rename: + field: azure.signinlogs.properties.status.additionalDetails + target_field: message + ignore_missing: true +- rename: + field: azure.signinlogs.properties.location.city + target_field: geo.city_name + ignore_missing: true +- rename: + field: azure.signinlogs.properties.location.state + target_field: geo.country_name + ignore_missing: true +- rename: + field: azure.signinlogs.properties.location.geoCoordinates.latitude + target_field: geo.location.lat + ignore_missing: true +- rename: + field: azure.signinlogs.properties.location.geoCoordinates.longitude + target_field: geo.location.lon + ignore_missing: true +- rename: + field: azure.signinlogs.properties.servicePrincipalId + target_field: azure.signinlogs.properties.service_principal_id + ignore_missing: true +- remove: + field: + - azure.signinlogs.properties.location + ignore_missing: true +- set: + field: event.kind + value: event +- set: + field: event.category + value: + - authentication +- set: + field: event.type + value: + - info +- set: + field: event.outcome + value: success + if: "ctx?.azure?.signinlogs?.properties?.status?.error_code == null || ctx.azure.signinlogs.properties.status.error_code == 0" +- set: + field: event.outcome + value: failure + if: "ctx?.azure?.signinlogs?.properties?.status?.error_code != null || ctx.azure.signinlogs.properties.status.error_code > 0" +- grok: + field: azure.signinlogs.properties.user_principal_name + patterns: + - '%{USERNAME:user.name}@%{HOSTNAME:user.domain}' + ignore_missing: true +- convert: + field: azure.signinlogs.properties.user_display_name + target_field: user.full_name + ignore_missing: true + type: string +- convert: + field: azure.signinlogs.properties.user_id + target_field: user.id + ignore_missing: true + type: string +- geoip: + field: source.ip + target_field: source.geo +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- pipeline: + name: '{< IngestPipeline "azure-shared-pipeline" >}' +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/azure/signinlogs/manifest.yml b/x-pack/filebeat/module/azure/signinlogs/manifest.yml index c08e0eaeb87..85770053bd5 100644 --- a/x-pack/filebeat/module/azure/signinlogs/manifest.yml +++ b/x-pack/filebeat/module/azure/signinlogs/manifest.yml @@ -15,6 +15,6 @@ var: default: [forwarded] ingest_pipeline: - - ingest/pipeline.json - - ../azure-shared-pipeline.json + - ingest/pipeline.yml + - ../azure-shared-pipeline.yml input: config/{{.input}}.yml diff --git a/x-pack/filebeat/module/azure/signinlogs/test/signinlogs.log-expected.json b/x-pack/filebeat/module/azure/signinlogs/test/signinlogs.log-expected.json index 8bc3778fe07..b7e28171a54 100644 --- a/x-pack/filebeat/module/azure/signinlogs/test/signinlogs.log-expected.json +++ b/x-pack/filebeat/module/azure/signinlogs/test/signinlogs.log-expected.json @@ -4,6 +4,7 @@ "azure.correlation_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", "azure.resource.id": "/tenants/8a4de8b5-095c-47d0-a96f-a75130c61d53/providers/Microsoft.aadiam", "azure.resource.provider": "Microsoft.aadiam", + "azure.signinlogs.category": "SignInLogs", "azure.signinlogs.identity": "Test LTest", "azure.signinlogs.operation_name": "Sign-in activity", "azure.signinlogs.operation_version": "1.0", @@ -34,13 +35,21 @@ "azure.signinlogs.properties.user_principal_name": "test@elastic.co", "azure.signinlogs.result_description": "This error occurred due to 'Keep me signed in' interrupt when the user was signing-in.", "azure.signinlogs.result_signature": "None", + "azure.signinlogs.result_type": "50140", "azure.tenant_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", "cloud.provider": "azure", - "event.category": "SignInLogs", + "event.action": "Sign-in activity", + "event.category": [ + "authentication" + ], "event.dataset": "azure.signinlogs", - "event.duration": 0.0, + "event.duration": 0, + "event.kind": "event", "event.module": "azure", - "event.outcome": "50140", + "event.outcome": "failure", + "event.type": [ + "info" + ], "fileset.name": "signinlogs", "geo.city_name": "Champs-Sur-Marne", "geo.country_iso_code": "FR", @@ -52,9 +61,22 @@ "log.offset": 0, "message": "This error occurred due to 'Keep me signed in' interrupt when the user was signing-in.", "service.type": "azure", + "source.as.number": 8426, + "source.as.organization.name": "Claranet Ltd", + "source.geo.city_name": "Farnham Royal", + "source.geo.continent_name": "Europe", + "source.geo.country_iso_code": "GB", + "source.geo.location.lat": 51.5333, + "source.geo.location.lon": -0.6167, + "source.geo.region_iso_code": "GB-BKM", + "source.geo.region_name": "Buckinghamshire", "source.ip": "81.171.241.231", "tags": [ "forwarded" - ] + ], + "user.domain": "elastic.co", + "user.full_name": "Test LTest", + "user.id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", + "user.name": "test" } ] \ No newline at end of file