initial commit for adding Microsoft Threat Protection package

packages/microsoft/data_stream/m365_defender/agent/stream/httpjson.yml.hbs

@@ -0,0 +1,29 @@
+http_method: GET
+http_headers: {"Content-Type": "application/json"}
+interval: {{interval}}
+json_objects_array: value
+split_events_by: alerts..entities
+url: {{url}}
+oauth2.client.id: {{oauth2_client_id}}
+oauth2.client.secret: {{oauth2_client_secret}}
+oauth2.token_url: {{oauth2_client_secret}}
+oauth2.provider: azure
+oauth2.azure.resource: https://api.security.microsoft.com
+http_headers: {{http_headers}}
+date_cursor.field: lastUpdateTime
+date_cursor.url_field: '$filter'
+date_cursor.value_template: {{date_cursor.value_template}}
+date_cursor.initial_interval: 55m
+date_cursor.date_format: '2006-01-02T15:04:05.9999999Z'
+tags: {{tags}}
+{{#contains tags "forwarded"}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+processors:
+  - decode_json_fields:
+      fields: [message]
+      target: json
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.6.0

packages/microsoft/data_stream/m365_defender/agent/stream/log.yml.hbs

@@ -0,0 +1,17 @@
+paths:
+{{#each paths as |path i|}}
+  - {{path}}
+{{/each}}
+exclude_files: [".gz$"]
+tags: {{tags}}
+{{#contains tags "forwarded"}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+processors:
+  - decode_json_fields:
+      fields: [message]
+      target: json
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.6.0

packages/microsoft/data_stream/m365_defender/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,281 @@
+---
+description: Pipeline for parsing microsoft atp logs
+processors:
+- set:
+    field: event.ingested
+    value: '{{_ingest.timestamp}}'
+- remove:
+    field:
+      - message
+      - json.comments
+      - host
+    ignore_missing: true
+
+#########################
+## ECS General Mapping ##
+#########################
+- script:
+    lang: painless
+    if: ctx?.json != null
+    source: |
+      void handleMap(Map map) {
+        for (def x : map.values()) {
+          if (x instanceof Map) {
+              handleMap(x);
+          } else if (x instanceof List) {
+              handleList(x);
+          }
+        }
+      map.values().removeIf(v -> v == null);
+      }
+      void handleList(List list) {
+        for (def x : list) {
+            if (x instanceof Map) {
+                handleMap(x);
+            } else if (x instanceof List) {
+                handleList(x);
+            }
+        }
+      }
+      handleMap(ctx);
+
+- set:
+    field: cloud.provider
+    value: azure
+- set:
+    field: '@timestamp'
+    value: '{{json.lastUpdateTime}}'
+    if: ctx.json?.lastUpdateTime != null
+- rename:
+    field: json.alerts.title
+    target_field: message
+    ignore_missing: true
+
+#######################
+## ECS Event Mapping ##
+#######################
+- set:
+    field: event.kind
+    value: alert
+# Events returned from the API is always in UTC, so should never use anything else
+- set:
+    field: event.timezone
+    value: UTC
+- set:
+    field: event.action
+    value: '{{json.alerts.category}}'
+    if: ctx.json?.alerts?.category != null
+- set:
+    field: event.provider
+    value: '{{json.alerts.serviceSource}}'
+    if: ctx.json?.alerts?.serviceSource != null
+- set:
+    field: event.created
+    value: '{{json.createdTime}}'
+    if: ctx.json?.createdTime != null
+- append:
+    field: event.category
+    value: host
+- append:
+    field: event.category
+    value: malware
+    if: ctx.json?.determination == 'Malware'
+- append:
+    field: event.category
+    value: process
+    if: ctx.json?.entities?.entityType == 'Process'
+- append:
+    field: event.type
+    value: user
+    if: ctx.json?.entities?.entityType == 'User'
+- append:
+    field: event.type
+    value:
+      - creation
+      - start
+    if: ctx.json?.status == 'New'
+- append:
+    field: event.type
+    value: end
+    if: ctx.json?.status == 'Resolved'
+- rename:
+    field: json.alerts.alertId
+    target_field: event.id
+    ignore_missing: true
+- rename:
+    field: json.alerts.firstActivity
+    target_field: event.start
+    ignore_missing: true
+- rename:
+    field: json.alerts.lastActivity
+    target_field: event.end
+    ignore_missing: true
+- set:
+    field: event.severity
+    value: 0
+    if: ctx.json?.severity == 'Unspecified'
+- set:
+    field: event.severity
+    value: 1
+    if: ctx.json?.severity == 'Informational'
+- set:
+    field: event.severity
+    value: 2
+    if: ctx.json?.severity == 'Low'
+- set:
+    field: event.severity
+    value: 3
+    if: ctx.json?.severity == 'Medium'
+- set:
+    field: event.severity
+    value: 4
+    if: ctx.json?.severity == 'High'
+- script:
+    lang: painless
+    if: "ctx?.event?.start != null && ctx?.event?.end != null"
+    source: >
+      Instant eventstart = ZonedDateTime.parse(ctx?.event?.start).toInstant();
+      Instant eventend = ZonedDateTime.parse(ctx?.event?.end).toInstant();
+      ctx.event['duration'] = ChronoUnit.NANOS.between(eventstart, eventend);
+
+########################
+## ECS Threat Mapping ##
+########################
+- set:
+    field: threat.framework
+    value: MITRE ATT&CK
+    if: ctx.json?.alerts?.category != null
+- rename:
+    field: json.alerts.category
+    target_field: threat.technique.name
+    ignore_missing: true
+- rename:
+    field: json.alerts.description
+    target_field: rule.description
+    ignore_missing: true
+    if: ctx.json?.alerts?.description.length() < 1020
+
+######################
+## ECS File Mapping ##
+######################
+- rename:
+    field: json.alerts.entities.fileName
+    target_field: file.name
+    ignore_missing: true
+- rename:
+    field: json.alerts.entities.sha256
+    target_field: file.hash.sha256
+    ignore_missing: true
+- rename:
+    field: json.alerts.entities.sha1
+    target_field: file.hash.sha1
+    ignore_missing: true
+- rename:
+    field: json.alerts.entities.filePath
+    target_field: file.path
+    ignore_missing: true
+
+#########################
+## ECS Process Mapping ##
+#########################
+- rename:
+    field: json.alerts.entities.processId
+    target_field: process.pid
+    ignore_missing: true
+- rename:
+    field: json.alerts.entities.processCommandLine
+    target_field: process.command_line
+    ignore_missing: true
+- rename:
+    field: json.alerts.entities.processCreationTime
+    target_field: process.start
+    ignore_missing: true
+- rename:
+    field: json.alerts.entities.parentProcessId
+    target_field: process.parent.pid
+    ignore_missing: true
+- rename:
+    field: json.alerts.entities.parentProcessCreationTime
+    target_field: process.parent.start
+    ignore_missing: true
+
+##########################
+## ECS Observer Mapping ##
+##########################
+- set:
+    field: observer.product
+    value: 365 Defender
+- set:
+    field: observer.vendor
+    value: Microsoft
+- rename:
+    field: json.alerts.serviceSource
+    target_field: observer.name
+    ignore_missing: true
+
+#####################
+## ECS URL Mapping ##
+#####################
+- rename:
+    field: json.alerts.entities.url
+    target_field: url.full
+    ignore_missing: true
+    if: ctx?.json?.entities?.url != null
+
+######################
+## ECS User Mapping ##
+######################
+- rename:
+    field: json.alerts.entities.userPrincipalName
+    target_field: host.user.name
+    ignore_missing: true
+- rename:
+    field: json.alerts.entities.domainName
+    target_field: host.user.domain
+    ignore_missing: true
+- rename:
+    field: json.alerts.entities.aadUserId
+    target_field: host.user.id
+    ignore_missing: true
+
+#########################
+## ECS Related Mapping ##
+#########################
+- append:
+    field: related.ip
+    value: '{{json.alerts.entities.ipAddress}}'
+    if: ctx.json?.entities?.ipAddress != null
+- append:
+    field: related.user
+    value: '{{host.user.name}}'
+    if: ctx.host?.user?.name != null
+- append:
+    field: related.hash
+    value: '{{file.hash.sha1}}'
+    if: ctx.file?.hash?.sha1 != null
+- append:
+    field: related.hash
+    value: '{{file.hash.sha256}}'
+    if: ctx.file?.hash?.sha256 != null
+- append:
+    field: related.hosts
+    value: '{{host.hostname}}'
+    if: ctx.host?.hostname != null
+
+#############
+## Cleanup ##
+#############
+- remove:
+    ignore_missing: true
+    field:
+      - json.createdTime
+      - json.severity
+      - json.lastUpdateTime
+- rename:
+    field: json
+    target_field: microsoft.m365_defender
+    ignore_missing: true
+on_failure:
+- set:
+    field: error.message
+    value: '{{_ingest.on_failure_message}}'

packages/microsoft/data_stream/m365_defender/fields/base-fields.yml

@@ -0,0 +1,21 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset name.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: dataset.type
+  type: constant_keyword
+  description: Dataset type.
+- name: dataset.name
+  type: constant_keyword
+  description: Dataset name.
+- name: dataset.namespace
+  type: constant_keyword
+  description: Dataset namespace.
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.