remove url_decode for http.request.referrer

elastic · Apr 26, 2021 · aabbfa3 · aabbfa3
1 parent e4fe9b8
commit aabbfa3
Show file tree

Hide file tree

Showing 37 changed files with 1,097 additions and 1,152 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -849,8 +849,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Added `http.request.id` to `nginx/ingress_controller` and `elasticsearch/audit`. {pull}24994[24994]
 - Add `awsfargate` module to collect container logs from Amazon ECS on Fargate. {pull}25041[25041]
 - New module `cyberarkpas` for CyberArk Privileged Access Security audit logs. {pull}24803[24803]
-- Update URI decoding and parsing across multiple modules. {issue}19088[19088] {pull}24699[24699]
-- Add `uri_parts` processor to multiple modules ingest pipelines. {issue}19088[19088] {pull}24699[24699]
+- Add `uri_parts` processor to Apache, Nginx, IIS, Traefik, S3Access, Cisco, F5, Fortinet, Google Workspace, Imperva, Microsoft, Netscout, O365, Sophos, Squid, Suricata, Zeek, Zia, Zoom, and ZScaler modules ingest pipelines. {issue}19088[19088] {pull}24699[24699]
 
 *Heartbeat*
 

diff --git a/filebeat/module/apache/access/ingest/pipeline.yml b/filebeat/module/apache/access/ingest/pipeline.yml
@@ -27,10 +27,6 @@ processors:
 - uri_parts:
     field: _tmp.url_orig
     ignore_failure: true
-- urldecode:
-    field: http.request.referrer
-    ignore_missing: true
-    ignore_failure: true
 - set:
     field: url.domain
     value: "{{destination.domain}}"

diff --git a/filebeat/module/apache/access/test/test.log-expected.json b/filebeat/module/apache/access/test/test.log-expected.json
@@ -175,7 +175,6 @@
         "url.path": "/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4",
         "user.name": "-",
         "user_agent.device.name": "Other",
-        "user_agent.device.type": "Desktop",
         "user_agent.name": "Firefox Alpha",
         "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2",
         "user_agent.os.full": "Windows 7",

diff --git a/filebeat/module/apache/error/ingest/pipeline.yml b/filebeat/module/apache/error/ingest/pipeline.yml
@@ -24,10 +24,6 @@ processors:
     - "File does not exist: %{URIPATH:file.path}"
     ignore_missing: true
     ignore_failure: true
-- urldecode:
-    field: http.request.referrer
-    ignore_missing: true
-    ignore_failure: true
 - date:
     if: ctx.event.timezone == null
     field: apache.error.timestamp

diff --git a/filebeat/module/iis/access/ingest/pipeline.yml b/filebeat/module/iis/access/ingest/pipeline.yml
@@ -49,10 +49,6 @@ processors:
     value: "{{_tmp.url_path}}"
     ignore_failure: true
     if: ctx?._tmp?.url_path != null && ctx?.url?.original == null
-- urldecode:
-    field: http.request.referrer
-    ignore_missing: true
-    ignore_failure: true
 - urldecode:
     field: _tmp.url_query
     target_field: url.query

diff --git a/filebeat/module/nginx/access/ingest/pipeline.yml b/filebeat/module/nginx/access/ingest/pipeline.yml
@@ -28,10 +28,6 @@ processors:
 - uri_parts:
     field: _tmp.url_orig
     ignore_failure: true
-- urldecode:
-    field: http.request.referrer
-    ignore_missing: true
-    ignore_failure: true
 - set:
     field: url.domain
     value: "{{destination.domain}}"

diff --git a/filebeat/module/nginx/access/test/access.log-expected.json b/filebeat/module/nginx/access/test/access.log-expected.json
@@ -580,7 +580,7 @@
         ],
         "fileset.name": "access",
         "http.request.method": "GET",
-        "http.request.referrer": "http://lessons.example.com/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4",
+        "http.request.referrer": "http://lessons.example.com/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4",
         "http.response.body.bytes": 7648063,
         "http.response.status_code": 206,
         "http.version": "1.1",
@@ -600,7 +600,6 @@
         "url.original": "/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4",
         "url.path": "/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4",
         "user_agent.device.name": "Kindle",
-        "user_agent.device.type": "Tablet",
         "user_agent.name": "Amazon Silk",
         "user_agent.original": "Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/81.2.16 like Chrome/81.0.4044.138 Safari/537.36",
         "user_agent.os.full": "Android 5.1.1",
@@ -625,7 +624,7 @@
         ],
         "fileset.name": "access",
         "http.request.method": "GET",
-        "http.request.referrer": "http://lessons.example.com/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4",
+        "http.request.referrer": "http://lessons.example.com/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4",
         "http.response.body.bytes": 7648063,
         "http.response.status_code": 206,
         "http.version": "1.1",
@@ -644,7 +643,6 @@
         "url.original": "/%D0%A0%D1%83%D1%81%D1%81%D0%BA%D0%B0%D1%8F%20%D1%88%D0%BA%D0%BE%D0%BB%D0%B0%20-%20InternetUrok%201%D0%BA%D0%BB%D0%B0%D1%81%D1%81/",
         "url.path": "/\u0420\u0443\u0441\u0441\u043a\u0430\u044f \u0448\u043a\u043e\u043b\u0430 - InternetUrok 1\u043a\u043b\u0430\u0441\u0441/",
         "user_agent.device.name": "Kindle",
-        "user_agent.device.type": "Tablet",
         "user_agent.name": "Amazon Silk",
         "user_agent.original": "Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/81.2.16 like Chrome/81.0.4044.138 Safari/537.36",
         "user_agent.os.full": "Android 5.1.1",

diff --git a/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml
@@ -37,10 +37,6 @@ processors:
   - uri_parts:
       field: url.original
       ignore_failure: true
-  - urldecode:
-      field: http.request.referrer
-      ignore_missing: true
-      ignore_failure: true
   - set:
       field: url.domain
       value: "{{destination.domain}}"

diff --git a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json
@@ -1391,7 +1391,6 @@
         "url.original": "/v2/some",
         "url.path": "/v2/some",
         "user_agent.device.name": "Mac",
-        "user_agent.device.type": "Desktop",
         "user_agent.name": "Firefox",
         "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0",
         "user_agent.os.full": "Mac OS X 10.14",

diff --git a/filebeat/module/traefik/access/ingest/pipeline.yml b/filebeat/module/traefik/access/ingest/pipeline.yml
@@ -23,10 +23,6 @@ processors:
 - uri_parts:
     field: temp.url_orig
     ignore_failure: true
-- urldecode:
-    field: http.request.referrer
-    ignore_missing: true
-    ignore_failure: true
 - rename:
     field: '@timestamp'
     target_field: event.created

diff --git a/x-pack/filebeat/module/cisco/asa/test/additional_messages.log-expected.json b/x-pack/filebeat/module/cisco/asa/test/additional_messages.log-expected.json
@@ -2546,7 +2546,10 @@
             "cisco-asa",
             "forwarded"
         ],
-        "url.original": "http://10.20.30.40/IOFUHSIU98[0]"
+        "url.domain": "10.20.30.40",
+        "url.original": "http://10.20.30.40/IOFUHSIU98[0]",
+        "url.path": "/IOFUHSIU98[0]",
+        "url.scheme": "http"
     },
     {
         "cisco.asa.message_id": "304001",
@@ -2590,7 +2593,10 @@
             "cisco-asa",
             "forwarded"
         ],
-        "url.original": "http://10.20.30.40/some/longer/url-asd-er9789870[0]_=23"
+        "url.domain": "10.20.30.40",
+        "url.original": "http://10.20.30.40/some/longer/url-asd-er9789870[0]_=23",
+        "url.path": "/some/longer/url-asd-er9789870[0]_=23",
+        "url.scheme": "http"
     },
     {
         "cisco.asa.message_id": "304001",

diff --git a/x-pack/filebeat/module/cisco/meraki/ingest/pipeline.yml b/x-pack/filebeat/module/cisco/meraki/ingest/pipeline.yml
@@ -29,10 +29,6 @@ processors:
   - remove:
       field: _temp_
       ignore_missing: true
-  - urldecode:
-      field: http.request.referrer
-      ignore_missing: true
-      ignore_failure: true
   # IP Geolocation Lookup
   - geoip:
         field: source.ip

diff --git a/x-pack/filebeat/module/cisco/meraki/test/generated.log-expected.json b/x-pack/filebeat/module/cisco/meraki/test/generated.log-expected.json
@@ -137,8 +137,8 @@
             "appliance"
         ],
         "related.ip": [
-            "10.112.46.169",
-            "10.155.236.240"
+            "10.155.236.240",
+            "10.112.46.169"
         ],
         "rsa.internal.messageid": "flows",
         "rsa.misc.action": [
@@ -587,8 +587,8 @@
         "observer.type": "Wireless",
         "observer.vendor": "Cisco",
         "related.ip": [
-            "10.187.77.245",
-            "10.88.231.224"
+            "10.88.231.224",
+            "10.187.77.245"
         ],
         "rsa.internal.messageid": "ids-alerts",
         "rsa.misc.event_type": "ids-alerts",
@@ -662,8 +662,8 @@
             "appliance"
         ],
         "related.ip": [
-            "10.205.47.51",
-            "10.219.84.37"
+            "10.219.84.37",
+            "10.205.47.51"
         ],
         "rsa.internal.messageid": "events",
         "rsa.misc.event_source": "appliance",
@@ -734,8 +734,8 @@
             "appliance"
         ],
         "related.ip": [
-            "10.182.178.217",
-            "10.63.194.87"
+            "10.63.194.87",
+            "10.182.178.217"
         ],
         "rsa.counters.dclass_r1": "fdeFi",
         "rsa.internal.messageid": "events",
@@ -1094,8 +1094,8 @@
         "observer.type": "Wireless",
         "observer.vendor": "Cisco",
         "related.ip": [
-            "10.173.136.186",
-            "10.221.102.245"
+            "10.221.102.245",
+            "10.173.136.186"
         ],
         "rsa.internal.event_desc": "idestlab",
         "rsa.internal.messageid": "security_event",
@@ -1133,8 +1133,8 @@
         "observer.type": "Wireless",
         "observer.vendor": "Cisco",
         "related.ip": [
-            "10.54.37.86",
-            "10.58.64.108"
+            "10.58.64.108",
+            "10.54.37.86"
         ],
         "rsa.internal.messageid": "ids-alerts",
         "rsa.misc.event_type": "ids-alerts",
@@ -1214,8 +1214,8 @@
         "observer.type": "Wireless",
         "observer.vendor": "Cisco",
         "related.ip": [
-            "10.0.200.27",
-            "10.183.44.198"
+            "10.183.44.198",
+            "10.0.200.27"
         ],
         "rsa.internal.event_desc": "uradi security_event tot",
         "rsa.internal.messageid": "security_event",
@@ -1402,8 +1402,8 @@
         "observer.type": "Wireless",
         "observer.vendor": "Cisco",
         "related.ip": [
-            "10.242.77.170",
-            "10.150.245.88"
+            "10.150.245.88",
+            "10.242.77.170"
         ],
         "rsa.internal.messageid": "ids-alerts",
         "rsa.misc.event_type": "ids-alerts",
@@ -1501,8 +1501,8 @@
         "observer.type": "Wireless",
         "observer.vendor": "Cisco",
         "related.ip": [
-            "10.94.6.140",
-            "10.147.15.213"
+            "10.147.15.213",
+            "10.94.6.140"
         ],
         "rsa.internal.messageid": "ids-alerts",
         "rsa.misc.event_type": "ids-alerts",
@@ -1540,8 +1540,8 @@
             "appliance"
         ],
         "related.ip": [
-            "10.230.6.127",
-            "10.111.157.56"
+            "10.111.157.56",
+            "10.230.6.127"
         ],
         "rsa.internal.messageid": "flows",
         "rsa.misc.action": [
@@ -1740,8 +1740,8 @@
         "observer.type": "Wireless",
         "observer.vendor": "Cisco",
         "related.ip": [
-            "10.124.63.4",
-            "10.90.99.245"
+            "10.90.99.245",
+            "10.124.63.4"
         ],
         "rsa.internal.event_desc": "etconsec",
         "rsa.internal.messageid": "security_event",
@@ -2018,8 +2018,8 @@
             "appliance"
         ],
         "related.ip": [
-            "10.201.168.116",
-            "10.86.188.179"
+            "10.86.188.179",
+            "10.201.168.116"
         ],
         "rsa.internal.messageid": "events",
         "rsa.misc.event_source": "appliance",
@@ -2209,8 +2209,8 @@
             "uames4985.mail.localdomain"
         ],
         "related.ip": [
-            "10.150.163.151",
-            "10.144.57.239"
+            "10.144.57.239",
+            "10.150.163.151"
         ],
         "rsa.internal.messageid": "events",
         "rsa.misc.event_source": "appliance",
@@ -2383,8 +2383,8 @@
             "appliance"
         ],
         "related.ip": [
-            "10.103.49.129",
-            "10.2.110.73"
+            "10.2.110.73",
+            "10.103.49.129"
         ],
         "rsa.counters.dclass_r1": "orumS",
         "rsa.internal.messageid": "events",
@@ -2480,8 +2480,8 @@
             "lors2232.api.example"
         ],
         "related.ip": [
-            "10.105.136.146",
-            "10.46.217.155"
+            "10.46.217.155",
+            "10.105.136.146"
         ],
         "rsa.internal.messageid": "events",
         "rsa.misc.event_source": "appliance",
@@ -2524,8 +2524,8 @@
             "appliance"
         ],
         "related.ip": [
-            "10.245.199.23",
-            "10.123.62.215"
+            "10.123.62.215",
+            "10.245.199.23"
         ],
         "rsa.db.index": "iusmodt",
         "rsa.internal.messageid": "flows",
@@ -2755,8 +2755,8 @@
         "observer.type": "Wireless",
         "observer.vendor": "Cisco",
         "related.ip": [
-            "10.121.9.5",
-            "10.244.32.189"
+            "10.244.32.189",
+            "10.121.9.5"
         ],
         "rsa.internal.messageid": "ids-alerts",
         "rsa.misc.event_type": "ids-alerts",
@@ -2929,8 +2929,8 @@
             "appliance"
         ],
         "related.ip": [
-            "10.17.111.91",
-            "10.65.0.157"
+            "10.65.0.157",
+            "10.17.111.91"
         ],
         "rsa.db.index": "nostrum",
         "rsa.internal.messageid": "flows",
@@ -3344,8 +3344,8 @@
         "observer.type": "Wireless",
         "observer.vendor": "Cisco",
         "related.ip": [
-            "10.147.165.30",
-            "10.195.90.73"
+            "10.195.90.73",
+            "10.147.165.30"
         ],
         "rsa.internal.messageid": "ids-alerts",
         "rsa.misc.event_type": "ids-alerts",

diff --git a/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml b/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml
@@ -1662,10 +1662,6 @@ processors:
       ignore_failure: true
       allow_duplicates: false
       if: ctx?._temp_?.url_domain != null
-  - urldecode:
-      field: http.request.referrer
-      ignore_missing: true
-      ignore_failure: true
 
   #
   # Populate ECS event.code

diff --git a/x-pack/filebeat/module/f5/bigipafm/ingest/pipeline.yml b/x-pack/filebeat/module/f5/bigipafm/ingest/pipeline.yml
@@ -34,9 +34,6 @@ processors:
   - remove:
       field: _temp_
       ignore_missing: true
-  - urldecode:
-      field: http.request.referrer
-      ignore_missing: true
   # IP Geolocation Lookup
   - geoip:
         field: source.ip