[Filebeat] Improve ECS field mappings in santa module (elastic#17982) (…

…elastic#18114)

* Improve ECS field mappings in santa module

- move certificate.common_name to
  santa.certificate.common_name (breaking change)
- move certificate.sha256 to
  santa.certificate.sha256 (breaking change)
- move hash.sha256 to process.hash.sha256 (breaking change)
- event.action
- event.category
- event.kind
- event.type
- event.outcome
- log.level
- add full path to executable to process.args
- related.hash
- related.user
- Add new default file path

Closes elastic#16180

(cherry picked from commit 81dfe61)

CHANGELOG.next.asciidoc

@@ -35,6 +35,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - CEF extensions are now mapped to the data types defined in the CEF guide. {pull}14342[14342]
 - Improve ECS field mappings in panw module. event.outcome now only contains success/failure per ECS specification. {issue}16025[16025] {pull}17910[17910]
 - Improve ECS categorization field mappings for nginx module. http.request.referrer is now lowercase & http.request.referrer only populated when nginx sets a value {issue}16174[16174] {pull}17844[17844]
+- Improve ECS field mappings in santa module. move hash.sha256 to process.hash.sha256 & move certificate fields to santa.certificate . {issue}16180[16180] {pull}17982[17982]
 
 *Heartbeat*
 

filebeat/docs/fields.asciidoc

@@ -28702,7 +28702,7 @@ The disk volume path.
 
 --
 
-*`certificate.common_name`*::
+*`santa.certificate.common_name`*::
 +
 --
 Common name from code signing certificate.
@@ -28711,7 +28711,7 @@ type: keyword
 
 --
 
-*`certificate.sha256`*::
+*`santa.certificate.sha256`*::
 +
 --
 SHA256 hash of code signing certificate.

filebeat/module/santa/_meta/fields.yml

@@ -56,10 +56,10 @@
  - name: mount
  description: The disk volume path.
 
- - name: certificate.common_name
- type: keyword
- description: Common name from code signing certificate.
+  - name: certificate.common_name
+  type: keyword
+  description: Common name from code signing certificate.
 
- - name: certificate.sha256
- type: keyword
- description: SHA256 hash of code signing certificate.
+  - name: certificate.sha256
+  type: keyword
+  description: SHA256 hash of code signing certificate.

filebeat/module/santa/log/ingest/pipeline.yml

@@ -0,0 +1,91 @@
+description: Pipeline for parsing Google Santa logs.
+processors:
+- grok:
+ field: message
+ patterns:
+ - '\[%{TIMESTAMP_ISO8601:process.start}\] %{NOT_SEPARATOR:log.level} santad: action=%{NOT_SEPARATOR:santa.action}\|decision=%{NOT_SEPARATOR:santa.decision}\|reason=%{NOT_SEPARATOR:santa.reason}\|sha256=%{NOT_SEPARATOR:process.hash.sha256}\|path=%{NOT_SEPARATOR:process.executable}(\|args=%{NOT_SEPARATOR:santa.args})?(\|cert_sha256=%{NOT_SEPARATOR:santa.certificate.sha256})?(\|cert_cn=%{NOT_SEPARATOR:santa.certificate.common_name})?\|pid=%{NUMBER:process.pid:long}\|ppid=%{NUMBER:process.ppid:long}\|uid=%{NUMBER:user.id}\|user=%{NOT_SEPARATOR:user.name}\|gid=%{NUMBER:group.id}\|group=%{NOT_SEPARATOR:group.name}\|mode=%{WORD:santa.mode}'
+ - '\[%{TIMESTAMP_ISO8601:timestamp}\] %{NOT_SEPARATOR:log.level} santad: action=%{NOT_SEPARATOR:santa.action}\|mount=%{NOT_SEPARATOR:santa.disk.mount}\|volume=%{NOT_SEPARATOR:santa.disk.volume}\|bsdname=%{NOT_SEPARATOR:santa.disk.bsdname}\|fs=%{NOT_SEPARATOR:santa.disk.fs}\|model=%{NOT_SEPARATOR:santa.disk.model}\|serial=%{NOT_SEPARATOR:santa.disk.serial}\|bus=%{NOT_SEPARATOR:santa.disk.bus}\|dmgpath=%{NOT_SEPARATOR:santa.disk.dmgpath}?'
+ pattern_definitions:
+ NOT_SEPARATOR: '[^\|]+'
+- rename:
+ field: message
+ target_field: log.original
+- date:
+ field: process.start
+ target_field: process.start
+ formats:
+ - ISO8601
+ ignore_failure: true
+- set:
+ field: '@timestamp'
+ value: '{{ process.start }}'
+ ignore_failure: true
+- split:
+ field: santa.args
+ separator: ' '
+ ignore_failure: true
+- date:
+ field: timestamp
+ target_field: '@timestamp'
+ formats:
+ - ISO8601
+ ignore_failure: true
+- remove:
+ field: timestamp
+ ignore_missing: true
+- append:
+ field: process.args
+ value: "{{process.executable}}"
+ if: "ctx?.process?.executable != null"
+- foreach:
+ field: santa.args
+ processor:
+ append:
+ field: process.args
+ value: "{{_ingest._value}}"
+ ignore_missing: true
+- remove:
+ field: santa.args
+ ignore_missing: true
+- set:
+ field: event.kind
+ value: event
+- append:
+ field: event.category
+ value: process
+ if: "ctx?.santa?.action == 'EXEC'"
+- append:
+ field: event.type
+ value: start
+ if: "ctx?.santa?.action == 'EXEC'"
+- set:
+ field: event.outcome
+ value: success
+ if: "ctx?.santa?.decision == 'ALLOW'"
+- set:
+ field: event.outcome
+ value: failure
+ if: "ctx?.santa?.decision == 'DENY'"
+- set:
+ field: event.action
+ value: "{{santa.action}}"
+ if: "ctx?.santa?.action != null"
+- lowercase:
+ field: event.action
+ ignore_missing: true
+- append:
+ field: related.user
+ value: "{{user.name}}"
+ if: "ctx?.user?.name != null"
+- append:
+ field: related.hash
+ value: "{{santa.certificate.sha256}}"
+ if: "ctx?.santa?.certificate?.sha256 != null"
+- append:
+ field: related.hash
+ value: "{{process.hash.sha256}}"
+ if: "ctx?.process?.hash != null"
+on_failure:
+- set:
+ field: error.message
+ value: '{{ _ingest.on_failure_message }}'

filebeat/module/santa/log/manifest.yml

@@ -4,8 +4,9 @@ var:
  - name: paths
  default:
  - /var/log/santa.log
+ - /var/db/santa/santa.log
  - name: input
  default: file
 
-ingest_pipeline: ingest/pipeline.json
+ingest_pipeline: ingest/pipeline.yml
 input: config/{{.input}}.yml