Add dissect to traefik/access metricset for first stage of processing

This should provide an example use case on how grok and dissect can be combined. The resulting outcome should be identical. The main difference is that now some of the fields can be used on the Beats side for filtering out events.
elastic · Jun 1, 2018 · 06c4c76 · 06c4c76
1 parent aa61a38
commit 06c4c76
Show file tree

Hide file tree

Showing 3 changed files with 75 additions and 47 deletions.
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -177,6 +177,7 @@ https://github.com/elastic/beats/compare/v6.2.3...master[Check the HEAD diff]
 - Support MySQL 5.7.19 by mysql/slowlog {pull}6969[6969]
 - Correctly join partial log lines when using `docker` input. {pull}6967[6967]
 - Add support for TLS with client authentication to the TCP input {pull}7056[7056]
+- Converted part of pipeline from treafik/access metricSet to dissect to improve efficeny. {pull}7209[7209]
 
 *Heartbeat*
 

diff --git a/filebeat/module/traefik/access/config/traefik-access.yml b/filebeat/module/traefik/access/config/traefik-access.yml
@@ -4,3 +4,12 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
+
+processors:
+- dissect:
+    tokenizer: '%{traefik.access.remote_ip} - %{traefik.access.user_name} [%{traefik.access.time}]
+    "%{traefik.access.method} %{traefik.access.url} HTTP/%{traefik.access.http_version}"
+    %{traefik.access.response_code} %{traefik.access.message}'
+
+    field: "message"
+    target_prefix: ""
diff --git a/filebeat/module/traefik/access/ingest/pipeline.json b/filebeat/module/traefik/access/ingest/pipeline.json
@@ -1,52 +1,70 @@
 {
     "description": "Pipeline for parsing Traefik access logs. Requires the geoip and user_agent plugins.",
-    "processors": [{
-        "grok": {
-            "field": "message",
-            "patterns":[
-                "%{IPORHOST:traefik.access.remote_ip} - %{DATA:traefik.access.user_name} \\[%{HTTPDATE:traefik.access.time}\\] \"%{WORD:traefik.access.method} %{DATA:traefik.access.url} HTTP/%{NUMBER:traefik.access.http_version}\" %{NUMBER:traefik.access.response_code} (?:%{NUMBER:traefik.access.body_sent.bytes}|-)( \"%{DATA:traefik.access.referrer}\")?( \"%{DATA:traefik.access.agent}\")?(?:%{NUMBER:traefik.access.request_count}|-)?( \"%{DATA:traefik.access.frontend_name}\")?( \"%{DATA:traefik.access.backend_url}\")?"
-            ],
-            "ignore_missing": true
+    "processors": [
+        {
+            "grok": {
+                "field": "traefik.access.message",
+                "patterns": [
+                    "(?:%{NUMBER:traefik.access.body_sent.bytes}|-)( \"%{DATA:traefik.access.referrer}\")?( \"%{DATA:traefik.access.agent}\")?(?:%{NUMBER:traefik.access.request_count}|-)?( \"%{DATA:traefik.access.frontend_name}\")?( \"%{DATA:traefik.access.backend_url}\")?"
+                ],
+                "ignore_missing": true
+            }
+        },
+        {
+            "remove": {
+                "field": "message"
+            }
+        },
+        {
+            "remove": {
+                "field": "traefik.access.message"
+            }
+        },
+        {
+            "rename": {
+                "field": "@timestamp",
+                "target_field": "read_timestamp"
+            }
+        },
+        {
+            "date": {
+                "field": "traefik.access.time",
+                "target_field": "@timestamp",
+                "formats": [
+                    "dd/MMM/YYYY:H:m:s Z"
+                ]
+            }
+        },
+        {
+            "remove": {
+                "field": "traefik.access.time"
+            }
+        },
+        {
+            "user_agent": {
+                "field": "traefik.access.agent",
+                "target_field": "traefik.access.user_agent",
+                "ignore_failure": true
+            }
+        },
+        {
+            "remove": {
+                "field": "traefik.access.agent"
+            }
+        },
+        {
+            "geoip": {
+                "field": "traefik.access.remote_ip",
+                "target_field": "traefik.access.geoip"
+            }
         }
-    },{
-        "remove":{
-            "field": "message"
+    ],
+    "on_failure": [
+        {
+            "set": {
+                "field": "error.message",
+                "value": "{{ _ingest.on_failure_message }}"
+            }
         }
-    }, {
-        "rename": {
-            "field": "@timestamp",
-            "target_field": "read_timestamp"
-        }
-    }, {
-        "date": {
-            "field": "traefik.access.time",
-            "target_field": "@timestamp",
-            "formats": ["dd/MMM/YYYY:H:m:s Z"]
-        }
-    }, {
-        "remove": {
-            "field": "traefik.access.time"
-        }
-    }, {
-        "user_agent": {
-            "field": "traefik.access.agent",
-            "target_field": "traefik.access.user_agent",
-            "ignore_failure": true
-        }
-    }, {
-        "remove": {
-            "field": "traefik.access.agent"
-        }
-    }, {
-        "geoip": {
-            "field": "traefik.access.remote_ip",
-            "target_field": "traefik.access.geoip"
-        }
-    }],
-    "on_failure" : [{
-        "set" : {
-            "field" : "error.message",
-            "value" : "{{ _ingest.on_failure_message }}"
-        }
-    }]
+    ]
 }