diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 973b0aa4db3..7da6fc0f29e 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -177,6 +177,7 @@ https://github.com/elastic/beats/compare/v6.2.3...master[Check the HEAD diff] - Support MySQL 5.7.19 by mysql/slowlog {pull}6969[6969] - Correctly join partial log lines when using `docker` input. {pull}6967[6967] - Add support for TLS with client authentication to the TCP input {pull}7056[7056] +- Converted part of pipeline from treafik/access metricSet to dissect to improve efficeny. {pull}7209[7209] *Heartbeat* diff --git a/filebeat/module/traefik/access/config/traefik-access.yml b/filebeat/module/traefik/access/config/traefik-access.yml index 0afd17317d4..2df383f52fd 100644 --- a/filebeat/module/traefik/access/config/traefik-access.yml +++ b/filebeat/module/traefik/access/config/traefik-access.yml @@ -4,3 +4,12 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] + +processors: +- dissect: + tokenizer: '%{traefik.access.remote_ip} - %{traefik.access.user_name} [%{traefik.access.time}] + "%{traefik.access.method} %{traefik.access.url} HTTP/%{traefik.access.http_version}" + %{traefik.access.response_code} %{traefik.access.message}' + + field: "message" + target_prefix: "" diff --git a/filebeat/module/traefik/access/ingest/pipeline.json b/filebeat/module/traefik/access/ingest/pipeline.json index ec9209abe5c..8e95385c14b 100644 --- a/filebeat/module/traefik/access/ingest/pipeline.json +++ b/filebeat/module/traefik/access/ingest/pipeline.json @@ -1,52 +1,70 @@ { "description": "Pipeline for parsing Traefik access logs. Requires the geoip and user_agent plugins.", - "processors": [{ - "grok": { - "field": "message", - "patterns":[ - "%{IPORHOST:traefik.access.remote_ip} - %{DATA:traefik.access.user_name} \\[%{HTTPDATE:traefik.access.time}\\] \"%{WORD:traefik.access.method} %{DATA:traefik.access.url} HTTP/%{NUMBER:traefik.access.http_version}\" %{NUMBER:traefik.access.response_code} (?:%{NUMBER:traefik.access.body_sent.bytes}|-)( \"%{DATA:traefik.access.referrer}\")?( \"%{DATA:traefik.access.agent}\")?(?:%{NUMBER:traefik.access.request_count}|-)?( \"%{DATA:traefik.access.frontend_name}\")?( \"%{DATA:traefik.access.backend_url}\")?" - ], - "ignore_missing": true + "processors": [ + { + "grok": { + "field": "traefik.access.message", + "patterns": [ + "(?:%{NUMBER:traefik.access.body_sent.bytes}|-)( \"%{DATA:traefik.access.referrer}\")?( \"%{DATA:traefik.access.agent}\")?(?:%{NUMBER:traefik.access.request_count}|-)?( \"%{DATA:traefik.access.frontend_name}\")?( \"%{DATA:traefik.access.backend_url}\")?" + ], + "ignore_missing": true + } + }, + { + "remove": { + "field": "message" + } + }, + { + "remove": { + "field": "traefik.access.message" + } + }, + { + "rename": { + "field": "@timestamp", + "target_field": "read_timestamp" + } + }, + { + "date": { + "field": "traefik.access.time", + "target_field": "@timestamp", + "formats": [ + "dd/MMM/YYYY:H:m:s Z" + ] + } + }, + { + "remove": { + "field": "traefik.access.time" + } + }, + { + "user_agent": { + "field": "traefik.access.agent", + "target_field": "traefik.access.user_agent", + "ignore_failure": true + } + }, + { + "remove": { + "field": "traefik.access.agent" + } + }, + { + "geoip": { + "field": "traefik.access.remote_ip", + "target_field": "traefik.access.geoip" + } } - },{ - "remove":{ - "field": "message" + ], + "on_failure": [ + { + "set": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } } - }, { - "rename": { - "field": "@timestamp", - "target_field": "read_timestamp" - } - }, { - "date": { - "field": "traefik.access.time", - "target_field": "@timestamp", - "formats": ["dd/MMM/YYYY:H:m:s Z"] - } - }, { - "remove": { - "field": "traefik.access.time" - } - }, { - "user_agent": { - "field": "traefik.access.agent", - "target_field": "traefik.access.user_agent", - "ignore_failure": true - } - }, { - "remove": { - "field": "traefik.access.agent" - } - }, { - "geoip": { - "field": "traefik.access.remote_ip", - "target_field": "traefik.access.geoip" - } - }], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] + ] }