Skip to content

Commit

Permalink
Cherry-pick elastic#10135 to 6.6: Elasticsearch/audit fileset should …
Browse files Browse the repository at this point in the history
…be more lenient in parsing node name (elastic#10465)

Cherry-pick of PR elastic#10135 to 6.6 branch. Original message: 

Resolves elastic#10035.

This PR:

* Uses `DATA` instead of `WORD` in the grok pattern for parsing out `elasticsearch.node.name`,
* Breaks out the grok pattern into pattern definitions to increase readability
* Removes a redundant `?` after a `*` in the grok pattern (between `elasticsearch.audit.action` and `elasticsearch.audit.uri`), and
* Properly reindents the pipeline JSON (so you might want to view the diff with `?w=1` appended to the URL)
  • Loading branch information
ycombinator authored Feb 1, 2019
1 parent 4198c69 commit 1f7a366
Show file tree
Hide file tree
Showing 4 changed files with 72 additions and 38 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ https://github.com/elastic/beats/compare/1035569addc4a3b29ffa14f8a08c27c1ace16ef
- Fix bad bytes count in `docker` input when filtering by stream. {pull}10211[10211]
- Add `convert_timezone` option to Logstash module to convert dates to UTC. {issue}9756[9756] {pull}9797[9797]
- Add `convert_timezone` option to Elasticsearch module to convert dates to UTC. {issue}9756[9756] {pull}9761[9761]
- Make elasticsearch/audit fileset be more lenient in parsing node name. {issue}10035[10035] {pull}10135[10135]

*Heartbeat*

Expand Down
91 changes: 53 additions & 38 deletions filebeat/module/elasticsearch/audit/ingest/pipeline.json
Original file line number Diff line number Diff line change
@@ -1,41 +1,56 @@
{
"description": "Pipeline for parsing elasticsearch audit logs",
"processors": [
{
"rename": {
"field": "@timestamp",
"target_field": "event.created"
}
},
{
"grok": {
"field": "message",
"patterns": [
"\\[%{TIMESTAMP_ISO8601:elasticsearch.audit.timestamp}\\]\\s*(\\[%{WORD:elasticsearch.node.name}\\])?\\s*\\[%{WORD:elasticsearch.audit.layer}\\]\\s*\\[%{WORD:elasticsearch.audit.event_type}\\]\\s*(origin_type\\=\\[%{WORD:elasticsearch.audit.origin_type}\\])?,?\\s*(origin_address\\=\\[%{IPORHOST:elasticsearch.audit.origin_address}\\])?,?\\s*(principal\\=\\[%{WORD:elasticsearch.audit.principal}\\])?,?\\s*(action\\=\\[%{DATA:elasticsearch.audit.action}\\])?,?\\s*?(uri=\\[%{DATA:elasticsearch.audit.uri}\\])?,?\\s*(request\\=\\[%{WORD:elasticsearch.audit.request}\\])?,?\\s*(request_body\\=\\[%{DATA:elasticsearch.audit.request_body}\\])?,?"
]
}
},
{
"date": {
"field": "elasticsearch.audit.timestamp",
"target_field": "@timestamp",
"formats": [
"ISO8601"
],
{< if .convert_timezone >}"timezone": "{{ event.timezone }}",{< end >}
"ignore_failure": true
}
},
{
"remove": {
"field": "elasticsearch.audit.timestamp"
}
}
"description": "Pipeline for parsing elasticsearch audit logs",
"processors": [
{
"rename": {
"field": "@timestamp",
"target_field": "event.created"
}
},
{
"grok": {
"field": "message",
"pattern_definitions": {
"ES_TIMESTAMP": "\\[%{TIMESTAMP_ISO8601:elasticsearch.audit.timestamp}\\]",
"ES_NODE_NAME": "(\\[%{DATA:elasticsearch.node.name}\\])?",
"ES_AUDIT_LAYER": "\\[%{WORD:elasticsearch.audit.layer}\\]",
"ES_AUDIT_EVENT_TYPE": "\\[%{WORD:elasticsearch.audit.event_type}\\]",
"ES_AUDIT_ORIGIN_TYPE": "(origin_type\\=\\[%{WORD:elasticsearch.audit.origin_type}\\])?",
"ES_AUDIT_ORIGIN_ADDRESS": "(origin_address\\=\\[%{IPORHOST:elasticsearch.audit.origin_address}\\])?",
"ES_AUDIT_PRINCIPAL": "(principal\\=\\[%{WORD:elasticsearch.audit.principal}\\])?",
"ES_AUDIT_ACTION": "(action\\=\\[%{DATA:elasticsearch.audit.action}\\])?",
"ES_AUDIT_URI": "(uri=\\[%{DATA:elasticsearch.audit.uri}\\])?",
"ES_AUDIT_REQUEST": "(request\\=\\[%{WORD:elasticsearch.audit.request}\\])?",
"ES_AUDIT_REQUEST_BODY": "(request_body\\=\\[%{DATA:elasticsearch.audit.request_body}\\])?"
},
"patterns": [
"%{ES_TIMESTAMP}\\s*%{ES_NODE_NAME}\\s*%{ES_AUDIT_LAYER}\\s*%{ES_AUDIT_EVENT_TYPE}\\s*%{ES_AUDIT_ORIGIN_TYPE},?\\s*%{ES_AUDIT_ORIGIN_ADDRESS},?\\s*%{ES_AUDIT_PRINCIPAL},?\\s*%{ES_AUDIT_ACTION},?\\s*%{ES_AUDIT_URI},?\\s*%{ES_AUDIT_REQUEST},?\\s*%{ES_AUDIT_REQUEST_BODY},?"
]
}
},
{
"date": {
"field": "elasticsearch.audit.timestamp",
"target_field": "@timestamp",
"formats": [
"ISO8601"
],
{< if .convert_timezone >}"timezone": "{{ event.timezone }}",{< end >}
"ignore_failure": true
}
},
{
"remove": {
"field": "elasticsearch.audit.timestamp"
}
}
],
"on_failure" : [{
"set" : {
"field" : "error.message",
"value" : "{{ _ingest.on_failure_message }}"
}
}]
"on_failure": [
{
"set": {
"field": "error.message",
"value": "{{ _ingest.on_failure_message }}"
}
}
]
}
1 change: 1 addition & 0 deletions filebeat/module/elasticsearch/audit/test/test.log
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,4 @@
[2018-06-19T05:26:27,268] [rest] [authentication_failed] origin_address=[147.107.128.77], principal=[N078801], uri=[/_xpack/security/_authenticate]
[2018-06-19T05:55:26,898] [transport] [access_denied] origin_type=[rest], origin_address=[147.107.128.77], principal=[_anonymous], action=[cluster:monitor/main], request=[MainRequest]
[2018-06-19T05:24:15,190] [v_VJhjV] [rest] [authentication_failed] origin_address=[172.18.0.3], principal=[elastic], uri=[/_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip], request_body=[body]
[2019-01-08T14:15:02,011] [NodeName-0] [transport] [access_granted] origin_type=[transport], origin_address=[192.168.2.1], principal=[username], realm=[active_directory], roles=[kibana_user,my_custom_role_1,foo_reader], action=[indices:data/read/search[free_context]], indices=[foo-2019.01.04,foo-2019.01.03,foo-2019.01.06,foo-2019.01.05,foo-2019.01.08,servicelog-2019.01.07], request=[SearchFreeContextRequest]
17 changes: 17 additions & 0 deletions filebeat/module/elasticsearch/audit/test/test.log-expected.json
Original file line number Diff line number Diff line change
Expand Up @@ -117,5 +117,22 @@
"offset": 986,
"prospector.type": "log",
"service.name": "elasticsearch"
},
{
"@timestamp": "2019-01-08T14:15:02.011Z",
"elasticsearch.audit.event_type": "access_granted",
"elasticsearch.audit.layer": "transport",
"elasticsearch.audit.origin_address": "192.168.2.1",
"elasticsearch.audit.origin_type": "transport",
"elasticsearch.audit.principal": "username",
"elasticsearch.node.name": "NodeName-0",
"event.dataset": "elasticsearch.audit",
"fileset.module": "elasticsearch",
"fileset.name": "audit",
"input.type": "log",
"message": "[2019-01-08T14:15:02,011] [NodeName-0] [transport] [access_granted] origin_type=[transport], origin_address=[192.168.2.1], principal=[username], realm=[active_directory], roles=[kibana_user,my_custom_role_1,foo_reader], action=[indices:data/read/search[free_context]], indices=[foo-2019.01.04,foo-2019.01.03,foo-2019.01.06,foo-2019.01.05,foo-2019.01.08,servicelog-2019.01.07], request=[SearchFreeContextRequest]",
"offset": 1210,
"prospector.type": "log",
"service.name": "elasticsearch"
}
]

0 comments on commit 1f7a366

Please sign in to comment.