From 500fb47fbe9b94932b65c35a0b16365321581edb Mon Sep 17 00:00:00 2001 From: "Lee E. Hinman" Date: Mon, 20 Apr 2020 14:08:43 -0500 Subject: [PATCH] Improve ECS categorization field mappings for nginx module - access + event.kind + event.category + event.type + event.outcome + lowercase http.request.method + improve grok to not populate empty fields + related.ip + related.users - error + event.kind + event.category + event.outcome - ingress_controller + event.kind + event.category + event.type + event.outcome + lowercase http.request.method + improve grok to not populate empty fields + related.ip + related.users Closes #16174 --- CHANGELOG.next.asciidoc | 1 + .../module/nginx/access/ingest/default.json | 150 ---------- .../module/nginx/access/ingest/pipeline.yml | 167 +++++++++++ filebeat/module/nginx/access/manifest.yml | 2 +- .../access/test/access.log-expected.json | 178 +++++++++--- .../test/test-with-host.log-expected.json | 152 +++++++--- .../nginx/access/test/test.log-expected.json | 147 +++++++--- .../module/nginx/error/ingest/pipeline.json | 47 --- .../module/nginx/error/ingest/pipeline.yml | 51 ++++ filebeat/module/nginx/error/manifest.yml | 2 +- .../nginx/error/test/error.log-expected.json | 28 ++ .../ingress_controller/ingest/pipeline.json | 151 ---------- .../ingress_controller/ingest/pipeline.yml | 172 +++++++++++ .../nginx/ingress_controller/manifest.yml | 2 +- .../test/test.log-expected.json | 267 +++++++++++++----- 15 files changed, 992 insertions(+), 525 deletions(-) delete mode 100644 filebeat/module/nginx/access/ingest/default.json create mode 100644 filebeat/module/nginx/access/ingest/pipeline.yml delete mode 100644 filebeat/module/nginx/error/ingest/pipeline.json create mode 100644 filebeat/module/nginx/error/ingest/pipeline.yml delete mode 100644 filebeat/module/nginx/ingress_controller/ingest/pipeline.json create mode 100644 filebeat/module/nginx/ingress_controller/ingest/pipeline.yml diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 8e6611da1dd..1e252cc7c05 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -260,6 +260,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add support for Google Application Default Credentials to the Google Pub/Sub input and Google Cloud modules. {pull}15668[15668] - Enhance `elasticsearch/deprecation` fileset to handle ECS-compatible logs emitted by Elasticsearch. {issue}17715[17715] {pull}17728[17728] - Enhance `elasticsearch/slowlog` fileset to handle ECS-compatible logs emitted by Elasticsearch. {issue}17715[17715] {pull}17729[17729] +- Improve ECS categorization field mappings for nginx module. {issue}16174[16174] {pull}17844[17844] *Heartbeat* diff --git a/filebeat/module/nginx/access/ingest/default.json b/filebeat/module/nginx/access/ingest/default.json deleted file mode 100644 index 04efd885e69..00000000000 --- a/filebeat/module/nginx/access/ingest/default.json +++ /dev/null @@ -1,150 +0,0 @@ -{ - "description": "Pipeline for parsing Nginx access logs. Requires the geoip and user_agent plugins.", - "processors": [ - { - "grok": { - "field": "message", - "patterns": [ - "(%{NGINX_HOST} )?\"?(?:%{NGINX_ADDRESS_LIST:nginx.access.remote_ip_list}|%{NOTSPACE:source.address}) - %{DATA:user.name} \\[%{HTTPDATE:nginx.access.time}\\] \"%{DATA:nginx.access.info}\" %{NUMBER:http.response.status_code:long} %{NUMBER:http.response.body.bytes:long} \"%{DATA:http.request.referrer}\" \"%{DATA:user_agent.original}\"" - ], - "pattern_definitions": { - "NGINX_HOST": "(?:%{IP:destination.ip}|%{NGINX_NOTSEPARATOR:destination.domain})(:%{NUMBER:destination.port})?", - "NGINX_NOTSEPARATOR": "[^\t ,:]+", - "NGINX_ADDRESS_LIST": "(?:%{IP}|%{WORD})(\"?,?\\s*(?:%{IP}|%{WORD}))*" - }, - "ignore_missing": true - } - }, - { - "grok": { - "field": "nginx.access.info", - "patterns": [ - "%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}", - "" - ], - "ignore_missing": true - } - }, - { - "remove": { - "field": "nginx.access.info" - } - }, - { - "split": { - "field": "nginx.access.remote_ip_list", - "separator": "\"?,?\\s+", - "ignore_missing": true - } - }, - { - "split": { - "field": "nginx.access.origin", - "separator": "\"?,?\\s+", - "ignore_missing": true - } - }, - { - "set": { - "field": "source.address", - "if": "ctx.source?.address == null", - "value": "" - } - }, - { - "script": { - "if": "ctx.nginx?.access?.remote_ip_list != null && ctx.nginx.access.remote_ip_list.length > 0", - "lang": "painless", - "source": "boolean isPrivate(def dot, def ip) { try { StringTokenizer tok = new StringTokenizer(ip, dot); int firstByte = Integer.parseInt(tok.nextToken()); int secondByte = Integer.parseInt(tok.nextToken()); if (firstByte == 10) { return true; } if (firstByte == 192 && secondByte == 168) { return true; } if (firstByte == 172 && secondByte >= 16 && secondByte <= 31) { return true; } if (firstByte == 127) { return true; } return false; } catch (Exception e) { return false; } } try { ctx.source.address = null; if (ctx.nginx.access.remote_ip_list == null) { return; } def found = false; for (def item : ctx.nginx.access.remote_ip_list) { if (!isPrivate(params.dot, item)) { ctx.source.address = item; found = true; break; } } if (!found) { ctx.source.address = ctx.nginx.access.remote_ip_list[0]; }} catch (Exception e) { ctx.source.address = null; }", - "params": { - "dot": "." - } - } - }, - { - "remove": { - "field": "source.address", - "if": "ctx.source.address == null" - } - }, - { - "grok": { - "field": "source.address", - "patterns": ["^%{IP:source.ip}$"], - "ignore_failure": true - } - }, - { - "remove": { - "field": "message" - } - }, - { - "rename": { - "field": "@timestamp", - "target_field": "event.created" - } - }, - { - "date": { - "field": "nginx.access.time", - "target_field": "@timestamp", - "formats": [ - "dd/MMM/yyyy:H:m:s Z" - ], - "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}] - } - }, - { - "remove": { - "field": "nginx.access.time" - } - }, - { - "user_agent": { - "field": "user_agent.original" - } - }, - { - "geoip": { - "field": "source.ip", - "target_field": "source.geo", - "ignore_missing": true - } - }, - { - "geoip": { - "database_file": "GeoLite2-ASN.mmdb", - "field": "source.ip", - "target_field": "source.as", - "properties": [ - "asn", - "organization_name" - ], - "ignore_missing": true - } - }, - { - "rename": { - "field": "source.as.asn", - "target_field": "source.as.number", - "ignore_missing": true - } - }, - { - "rename": { - "field": "source.as.organization_name", - "target_field": "source.as.organization.name", - "ignore_missing": true - } - } - ], - "on_failure": [ - { - "set": { - "field": "error.message", - "value": "{{ _ingest.on_failure_message }}" - } - } - ] -} diff --git a/filebeat/module/nginx/access/ingest/pipeline.yml b/filebeat/module/nginx/access/ingest/pipeline.yml new file mode 100644 index 00000000000..3a41265875b --- /dev/null +++ b/filebeat/module/nginx/access/ingest/pipeline.yml @@ -0,0 +1,167 @@ +description: Pipeline for parsing Nginx access logs. Requires the geoip and user_agent + plugins. +processors: +- grok: + field: message + patterns: + - (%{NGINX_HOST} )?"?(?:%{NGINX_ADDRESS_LIST:nginx.access.remote_ip_list}|%{NOTSPACE:source.address}) + - (-|%{DATA:user.name}) \[%{HTTPDATE:nginx.access.time}\] "%{DATA:nginx.access.info}" + %{NUMBER:http.response.status_code:long} %{NUMBER:http.response.body.bytes:long} + "(-|%{DATA:http.request.referrer})" "(-|%{DATA:user_agent.original})" + pattern_definitions: + NGINX_HOST: (?:%{IP:destination.ip}|%{NGINX_NOTSEPARATOR:destination.domain})(:%{NUMBER:destination.port})? + NGINX_NOTSEPARATOR: "[^\t ,:]+" + NGINX_ADDRESS_LIST: (?:%{IP}|%{WORD})("?,?\s*(?:%{IP}|%{WORD}))* + ignore_missing: true +- grok: + field: nginx.access.info + patterns: + - '%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}' + - "" + ignore_missing: true +- remove: + field: nginx.access.info +- split: + field: nginx.access.remote_ip_list + separator: '"?,?\s+' + ignore_missing: true +- split: + field: nginx.access.origin + separator: '"?,?\s+' + ignore_missing: true +- set: + field: source.address + if: ctx.source?.address == null + value: "" +- script: + if: ctx.nginx?.access?.remote_ip_list != null && ctx.nginx.access.remote_ip_list.length > 0 + lang: painless + source: >- + boolean isPrivate(def dot, def ip) { + try { + StringTokenizer tok = new StringTokenizer(ip, dot); + int firstByte = Integer.parseInt(tok.nextToken()); + int secondByte = Integer.parseInt(tok.nextToken()); + if (firstByte == 10) { + return true; + } + if (firstByte == 192 && secondByte == 168) { + return true; + } + if (firstByte == 172 && secondByte >= 16 && secondByte <= 31) { + return true; + } + if (firstByte == 127) { + return true; + } + return false; + } + catch (Exception e) { + return false; + } + } + try { + ctx.source.address = null; + if (ctx.nginx.access.remote_ip_list == null) { + return; + } + def found = false; + for (def item : ctx.nginx.access.remote_ip_list) { + if (!isPrivate(params.dot, item)) { + ctx.source.address = item; + found = true; + break; + } + } + if (!found) { + ctx.source.address = ctx.nginx.access.remote_ip_list[0]; + } + } + catch (Exception e) { + ctx.source.address = null; + } + params: + dot: . +- remove: + field: source.address + if: ctx.source.address == null +- grok: + field: source.address + patterns: + - ^%{IP:source.ip}$ + ignore_failure: true +- remove: + field: message +- rename: + field: '@timestamp' + target_field: event.created +- date: + field: nginx.access.time + target_field: '@timestamp' + formats: + - dd/MMM/yyyy:H:m:s Z + on_failure: + - append: + field: error.message + value: '{{ _ingest.on_failure_message }}' +- remove: + field: nginx.access.time +- user_agent: + field: user_agent.original + ignore_missing: true +- geoip: + field: source.ip + target_field: source.geo + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- set: + field: event.kind + value: event +- append: + field: event.category + value: web +- append: + field: event.type + value: access +- set: + field: event.outcome + value: success + if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400" +- set: + field: event.outcome + value: failure + if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400" +- lowercase: + field: http.request.method + ignore_missing: true +- append: + field: related.ip + value: "{{source.ip}}" + if: "ctx?.source?.ip != null" +- append: + field: related.ip + value: "{{destination.ip}}" + if: "ctx?.destination?.ip != null" +- append: + field: related.user + value: "{{user.name}}" + if: "ctx?.user?.name != null" +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/filebeat/module/nginx/access/manifest.yml b/filebeat/module/nginx/access/manifest.yml index d3d6211dccc..8d04dd32e5c 100644 --- a/filebeat/module/nginx/access/manifest.yml +++ b/filebeat/module/nginx/access/manifest.yml @@ -9,7 +9,7 @@ var: os.windows: - c:/programdata/nginx/logs/*access.log* -ingest_pipeline: ingest/default.json +ingest_pipeline: ingest/pipeline.yml input: config/nginx-access.yml requires.processors: diff --git a/filebeat/module/nginx/access/test/access.log-expected.json b/filebeat/module/nginx/access/test/access.log-expected.json index a121dd67613..12c94f2996d 100644 --- a/filebeat/module/nginx/access/test/access.log-expected.json +++ b/filebeat/module/nginx/access/test/access.log-expected.json @@ -1,12 +1,19 @@ [ { "@timestamp": "2016-10-25T12:49:33.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.access", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "success", "event.timezone": "-02:00", + "event.type": [ + "access" + ], "fileset.name": "access", - "http.request.method": "GET", - "http.request.referrer": "-", + "http.request.method": "get", "http.response.body.bytes": 612, "http.response.status_code": 200, "http.version": "1.1", @@ -15,6 +22,9 @@ "nginx.access.remote_ip_list": [ "77.179.66.156" ], + "related.ip": [ + "77.179.66.156" + ], "service.type": "nginx", "source.address": "77.179.66.156", "source.as.number": 6805, @@ -28,7 +38,6 @@ "source.geo.region_name": "Rheinland-Pfalz", "source.ip": "77.179.66.156", "url.original": "/", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36", @@ -39,11 +48,19 @@ }, { "@timestamp": "2016-10-25T12:49:34.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.access", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "failure", "event.timezone": "-02:00", + "event.type": [ + "access" + ], "fileset.name": "access", - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "http://localhost:8080/", "http.response.body.bytes": 571, "http.response.status_code": 404, @@ -53,6 +70,9 @@ "nginx.access.remote_ip_list": [ "77.179.66.156" ], + "related.ip": [ + "77.179.66.156" + ], "service.type": "nginx", "source.address": "77.179.66.156", "source.as.number": 6805, @@ -66,7 +86,6 @@ "source.geo.region_name": "Rheinland-Pfalz", "source.ip": "77.179.66.156", "url.original": "/favicon.ico", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36", @@ -77,12 +96,19 @@ }, { "@timestamp": "2016-10-25T12:50:44.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.access", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "failure", "event.timezone": "-02:00", + "event.type": [ + "access" + ], "fileset.name": "access", - "http.request.method": "GET", - "http.request.referrer": "-", + "http.request.method": "get", "http.response.body.bytes": 571, "http.response.status_code": 404, "http.version": "1.1", @@ -91,6 +117,9 @@ "nginx.access.remote_ip_list": [ "77.179.66.156" ], + "related.ip": [ + "77.179.66.156" + ], "service.type": "nginx", "source.address": "77.179.66.156", "source.as.number": 6805, @@ -104,7 +133,6 @@ "source.geo.region_name": "Rheinland-Pfalz", "source.ip": "77.179.66.156", "url.original": "/adsasd", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36", @@ -115,12 +143,19 @@ }, { "@timestamp": "2016-12-07T09:34:43.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.access", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "success", "event.timezone": "-02:00", + "event.type": [ + "access" + ], "fileset.name": "access", - "http.request.method": "GET", - "http.request.referrer": "-", + "http.request.method": "get", "http.response.body.bytes": 612, "http.response.status_code": 200, "http.version": "1.1", @@ -129,6 +164,9 @@ "nginx.access.remote_ip_list": [ "77.179.66.156" ], + "related.ip": [ + "77.179.66.156" + ], "service.type": "nginx", "source.address": "77.179.66.156", "source.as.number": 6805, @@ -142,7 +180,6 @@ "source.geo.region_name": "Rheinland-Pfalz", "source.ip": "77.179.66.156", "url.original": "/", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36", @@ -153,11 +190,19 @@ }, { "@timestamp": "2016-12-07T09:34:43.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.access", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "failure", "event.timezone": "-02:00", + "event.type": [ + "access" + ], "fileset.name": "access", - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "http://localhost:8080/", "http.response.body.bytes": 571, "http.response.status_code": 404, @@ -167,6 +212,9 @@ "nginx.access.remote_ip_list": [ "77.179.66.156" ], + "related.ip": [ + "77.179.66.156" + ], "service.type": "nginx", "source.address": "77.179.66.156", "source.as.number": 6805, @@ -180,7 +228,6 @@ "source.geo.region_name": "Rheinland-Pfalz", "source.ip": "77.179.66.156", "url.original": "/favicon.ico", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36", @@ -191,12 +238,19 @@ }, { "@timestamp": "2016-12-07T09:43:18.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.access", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "failure", "event.timezone": "-02:00", + "event.type": [ + "access" + ], "fileset.name": "access", - "http.request.method": "GET", - "http.request.referrer": "-", + "http.request.method": "get", "http.response.body.bytes": 571, "http.response.status_code": 404, "http.version": "1.1", @@ -205,6 +259,9 @@ "nginx.access.remote_ip_list": [ "77.179.66.156" ], + "related.ip": [ + "77.179.66.156" + ], "service.type": "nginx", "source.address": "77.179.66.156", "source.as.number": 6805, @@ -218,7 +275,6 @@ "source.geo.region_name": "Rheinland-Pfalz", "source.ip": "77.179.66.156", "url.original": "/test", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36", @@ -229,12 +285,19 @@ }, { "@timestamp": "2016-12-07T09:43:21.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.access", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "failure", "event.timezone": "-02:00", + "event.type": [ + "access" + ], "fileset.name": "access", - "http.request.method": "GET", - "http.request.referrer": "-", + "http.request.method": "get", "http.response.body.bytes": 571, "http.response.status_code": 404, "http.version": "1.1", @@ -243,6 +306,9 @@ "nginx.access.remote_ip_list": [ "77.179.66.156" ], + "related.ip": [ + "77.179.66.156" + ], "service.type": "nginx", "source.address": "77.179.66.156", "source.as.number": 6805, @@ -256,7 +322,6 @@ "source.geo.region_name": "Rheinland-Pfalz", "source.ip": "77.179.66.156", "url.original": "/test", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36", @@ -267,12 +332,19 @@ }, { "@timestamp": "2016-12-07T09:43:23.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.access", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "failure", "event.timezone": "-02:00", + "event.type": [ + "access" + ], "fileset.name": "access", - "http.request.method": "GET", - "http.request.referrer": "-", + "http.request.method": "get", "http.response.body.bytes": 571, "http.response.status_code": 404, "http.version": "1.1", @@ -281,6 +353,9 @@ "nginx.access.remote_ip_list": [ "77.179.66.156" ], + "related.ip": [ + "77.179.66.156" + ], "service.type": "nginx", "source.address": "77.179.66.156", "source.as.number": 6805, @@ -294,7 +369,6 @@ "source.geo.region_name": "Rheinland-Pfalz", "source.ip": "77.179.66.156", "url.original": "/test1", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36", @@ -305,12 +379,19 @@ }, { "@timestamp": "2016-12-07T10:04:37.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.access", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "failure", "event.timezone": "-02:00", + "event.type": [ + "access" + ], "fileset.name": "access", - "http.request.method": "GET", - "http.request.referrer": "-", + "http.request.method": "get", "http.response.body.bytes": 571, "http.response.status_code": 404, "http.version": "1.1", @@ -319,11 +400,13 @@ "nginx.access.remote_ip_list": [ "127.0.0.1" ], + "related.ip": [ + "127.0.0.1" + ], "service.type": "nginx", "source.address": "127.0.0.1", "source.ip": "127.0.0.1", "url.original": "/test1", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36", @@ -334,12 +417,19 @@ }, { "@timestamp": "2016-12-07T10:04:58.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.access", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "success", "event.timezone": "-02:00", + "event.type": [ + "access" + ], "fileset.name": "access", - "http.request.method": "GET", - "http.request.referrer": "-", + "http.request.method": "get", "http.response.body.bytes": 0, "http.response.status_code": 304, "http.version": "1.1", @@ -348,11 +438,13 @@ "nginx.access.remote_ip_list": [ "127.0.0.1" ], + "related.ip": [ + "127.0.0.1" + ], "service.type": "nginx", "source.address": "127.0.0.1", "source.ip": "127.0.0.1", "url.original": "/", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", @@ -363,12 +455,19 @@ }, { "@timestamp": "2016-12-07T10:04:59.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.access", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "success", "event.timezone": "-02:00", + "event.type": [ + "access" + ], "fileset.name": "access", - "http.request.method": "GET", - "http.request.referrer": "-", + "http.request.method": "get", "http.response.body.bytes": 0, "http.response.status_code": 304, "http.version": "1.1", @@ -377,11 +476,13 @@ "nginx.access.remote_ip_list": [ "127.0.0.1" ], + "related.ip": [ + "127.0.0.1" + ], "service.type": "nginx", "source.address": "127.0.0.1", "source.ip": "127.0.0.1", "url.original": "/", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", @@ -392,12 +493,19 @@ }, { "@timestamp": "2016-12-07T10:05:07.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.access", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "failure", "event.timezone": "-02:00", + "event.type": [ + "access" + ], "fileset.name": "access", - "http.request.method": "GET", - "http.request.referrer": "-", + "http.request.method": "get", "http.response.body.bytes": 169, "http.response.status_code": 404, "http.version": "1.1", @@ -406,11 +514,13 @@ "nginx.access.remote_ip_list": [ "127.0.0.1" ], + "related.ip": [ + "127.0.0.1" + ], "service.type": "nginx", "source.address": "127.0.0.1", "source.ip": "127.0.0.1", "url.original": "/taga", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", diff --git a/filebeat/module/nginx/access/test/test-with-host.log-expected.json b/filebeat/module/nginx/access/test/test-with-host.log-expected.json index 38695946ca5..a641922d139 100644 --- a/filebeat/module/nginx/access/test/test-with-host.log-expected.json +++ b/filebeat/module/nginx/access/test/test-with-host.log-expected.json @@ -2,12 +2,19 @@ { "@timestamp": "2016-12-07T10:05:07.000Z", "destination.domain": "example.com", + "event.category": [ + "web" + ], "event.dataset": "nginx.access", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "success", "event.timezone": "-02:00", + "event.type": [ + "access" + ], "fileset.name": "access", - "http.request.method": "GET", - "http.request.referrer": "-", + "http.request.method": "get", "http.response.body.bytes": 571, "http.response.status_code": 200, "http.version": "1.1", @@ -18,11 +25,13 @@ "10.0.0.1", "127.0.0.1" ], + "related.ip": [ + "10.0.0.2" + ], "service.type": "nginx", "source.address": "10.0.0.2", "source.ip": "10.0.0.2", "url.original": "/ocelot", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", @@ -34,12 +43,19 @@ { "@timestamp": "2017-05-29T19:02:48.000Z", "destination.domain": "example.com", + "event.category": [ + "web" + ], "event.dataset": "nginx.access", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "failure", "event.timezone": "-02:00", + "event.type": [ + "access" + ], "fileset.name": "access", - "http.request.method": "GET", - "http.request.referrer": "-", + "http.request.method": "get", "http.response.body.bytes": 612, "http.response.status_code": 404, "http.version": "1.1", @@ -48,11 +64,13 @@ "nginx.access.remote_ip_list": [ "172.17.0.1" ], + "related.ip": [ + "172.17.0.1" + ], "service.type": "nginx", "source.address": "172.17.0.1", "source.ip": "172.17.0.1", "url.original": "/stringpatch", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Firefox Alpha", "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", @@ -64,12 +82,19 @@ { "@timestamp": "2016-12-07T10:05:07.000Z", "destination.domain": "example.com", + "event.category": [ + "web" + ], "event.dataset": "nginx.access", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "success", "event.timezone": "-02:00", + "event.type": [ + "access" + ], "fileset.name": "access", - "http.request.method": "GET", - "http.request.referrer": "-", + "http.request.method": "get", "http.response.body.bytes": 571, "http.response.status_code": 200, "http.version": "1.1", @@ -80,6 +105,9 @@ "10.0.0.1", "85.181.35.98" ], + "related.ip": [ + "85.181.35.98" + ], "service.type": "nginx", "source.address": "85.181.35.98", "source.as.number": 6805, @@ -93,7 +121,6 @@ "source.geo.region_name": "Land Berlin", "source.ip": "85.181.35.98", "url.original": "/ocelot", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", @@ -106,12 +133,19 @@ "@timestamp": "2016-12-07T10:05:07.000Z", "destination.domain": "example.com", "destination.port": "80", + "event.category": [ + "web" + ], "event.dataset": "nginx.access", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "success", "event.timezone": "-02:00", + "event.type": [ + "access" + ], "fileset.name": "access", - "http.request.method": "GET", - "http.request.referrer": "-", + "http.request.method": "get", "http.response.body.bytes": 571, "http.response.status_code": 200, "http.version": "1.1", @@ -120,6 +154,9 @@ "nginx.access.remote_ip_list": [ "85.181.35.98" ], + "related.ip": [ + "85.181.35.98" + ], "service.type": "nginx", "source.address": "85.181.35.98", "source.as.number": 6805, @@ -133,7 +170,6 @@ "source.geo.region_name": "Land Berlin", "source.ip": "85.181.35.98", "url.original": "/ocelot", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36", @@ -146,12 +182,19 @@ "@timestamp": "2016-01-22T13:18:29.000Z", "destination.domain": "example.com", "destination.port": "80", + "event.category": [ + "web" + ], "event.dataset": "nginx.access", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "success", "event.timezone": "-02:00", + "event.type": [ + "access" + ], "fileset.name": "access", - "http.request.method": "GET", - "http.request.referrer": "-", + "http.request.method": "get", "http.response.body.bytes": 25507, "http.response.status_code": 200, "http.version": "1.1", @@ -163,6 +206,9 @@ "204.246.1.1", "10.2.1.185" ], + "related.ip": [ + "199.96.1.1" + ], "service.type": "nginx", "source.address": "199.96.1.1", "source.as.number": 19065, @@ -176,7 +222,6 @@ "source.geo.region_name": "Illinois", "source.ip": "199.96.1.1", "url.original": "/assets/xxxx?q=100", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Other", "user_agent.original": "Amazon CloudFront" @@ -184,12 +229,19 @@ { "@timestamp": "2016-12-30T06:47:09.000Z", "destination.ip": "1.2.3.4", + "event.category": [ + "web" + ], "event.dataset": "nginx.access", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "failure", "event.timezone": "-02:00", + "event.type": [ + "access" + ], "fileset.name": "access", - "http.request.method": "GET", - "http.request.referrer": "-", + "http.request.method": "get", "http.response.body.bytes": 8571, "http.response.status_code": 404, "http.version": "1.1", @@ -200,6 +252,10 @@ "10.225.192.17", "10.2.2.121" ], + "related.ip": [ + "2a03:0000:10ff:f00f:0000:0000:0:8000", + "1.2.3.4" + ], "service.type": "nginx", "source.address": "2a03:0000:10ff:f00f:0000:0000:0:8000", "source.geo.continent_name": "Europe", @@ -208,7 +264,6 @@ "source.geo.location.lon": -8.0, "source.ip": "2a03:0000:10ff:f00f:0000:0000:0:8000", "url.original": "/test.html", - "user.name": "-", "user_agent.device.name": "Spider", "user_agent.name": "Facebot", "user_agent.original": "Mozilla/5.0 (compatible; Facebot 1.0; https://developers.facebook.com/docs/sharing/webmasters/crawler)", @@ -218,11 +273,18 @@ "@timestamp": "2018-04-12T07:48:40.000Z", "destination.ip": "1.2.3.4", "destination.port": "80", + "event.category": [ + "web" + ], "event.dataset": "nginx.access", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "failure", "event.timezone": "-02:00", + "event.type": [ + "access" + ], "fileset.name": "access", - "http.request.referrer": "-", "http.response.body.bytes": 0, "http.response.status_code": 400, "input.type": "log", @@ -230,43 +292,53 @@ "nginx.access.remote_ip_list": [ "127.0.0.1" ], + "related.ip": [ + "127.0.0.1", + "1.2.3.4" + ], "service.type": "nginx", "source.address": "127.0.0.1", - "source.ip": "127.0.0.1", - "user.name": "-", - "user_agent.device.name": "Other", - "user_agent.name": "Other", - "user_agent.original": "-" + "source.ip": "127.0.0.1" }, { "@timestamp": "2019-02-26T14:39:42.000Z", "destination.domain": "example.com", "destination.port": "80", + "event.category": [ + "web" + ], "event.dataset": "nginx.access", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "failure", "event.timezone": "-02:00", + "event.type": [ + "access" + ], "fileset.name": "access", - "http.request.referrer": "-", "http.response.body.bytes": 173, "http.response.status_code": 400, "input.type": "log", "log.offset": 1269, "service.type": "nginx", - "source.address": "unix:", - "user.name": "-", - "user_agent.device.name": "Other", - "user_agent.name": "Other", - "user_agent.original": "-" + "source.address": "unix:" }, { "@timestamp": "2017-05-29T19:02:48.000Z", "destination.ip": "1.2.3.4", + "event.category": [ + "web" + ], "event.dataset": "nginx.access", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "success", "event.timezone": "-02:00", + "event.type": [ + "access" + ], "fileset.name": "access", - "http.request.method": "GET", - "http.request.referrer": "-", + "http.request.method": "get", "http.response.body.bytes": 612, "http.response.status_code": 200, "http.version": "1.1", @@ -275,10 +347,12 @@ "nginx.access.remote_ip_list": [ "localhost" ], + "related.ip": [ + "1.2.3.4" + ], "service.type": "nginx", "source.address": "localhost", "url.original": "/test2", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Firefox Alpha", "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", @@ -290,12 +364,19 @@ { "@timestamp": "2017-05-29T19:02:48.000Z", "destination.domain": "example.com", + "event.category": [ + "web" + ], "event.dataset": "nginx.access", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "success", "event.timezone": "-02:00", + "event.type": [ + "access" + ], "fileset.name": "access", - "http.request.method": "GET", - "http.request.referrer": "-", + "http.request.method": "get", "http.response.body.bytes": 612, "http.response.status_code": 200, "http.version": "1.1", @@ -308,7 +389,6 @@ "service.type": "nginx", "source.address": "localhost", "url.original": "/test2", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Firefox Alpha", "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", diff --git a/filebeat/module/nginx/access/test/test.log-expected.json b/filebeat/module/nginx/access/test/test.log-expected.json index 247b7a12e21..22959d1a8be 100644 --- a/filebeat/module/nginx/access/test/test.log-expected.json +++ b/filebeat/module/nginx/access/test/test.log-expected.json @@ -1,12 +1,19 @@ [ { "@timestamp": "2016-12-07T10:05:07.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.access", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "success", "event.timezone": "-02:00", + "event.type": [ + "access" + ], "fileset.name": "access", - "http.request.method": "GET", - "http.request.referrer": "-", + "http.request.method": "get", "http.response.body.bytes": 571, "http.response.status_code": 200, "http.version": "1.1", @@ -17,11 +24,13 @@ "10.0.0.1", "127.0.0.1" ], + "related.ip": [ + "10.0.0.2" + ], "service.type": "nginx", "source.address": "10.0.0.2", "source.ip": "10.0.0.2", "url.original": "/ocelot", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", @@ -32,12 +41,19 @@ }, { "@timestamp": "2017-05-29T19:02:48.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.access", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "failure", "event.timezone": "-02:00", + "event.type": [ + "access" + ], "fileset.name": "access", - "http.request.method": "GET", - "http.request.referrer": "-", + "http.request.method": "get", "http.response.body.bytes": 612, "http.response.status_code": 404, "http.version": "1.1", @@ -46,11 +62,13 @@ "nginx.access.remote_ip_list": [ "172.17.0.1" ], + "related.ip": [ + "172.17.0.1" + ], "service.type": "nginx", "source.address": "172.17.0.1", "source.ip": "172.17.0.1", "url.original": "/stringpatch", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Firefox Alpha", "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", @@ -61,12 +79,19 @@ }, { "@timestamp": "2016-12-07T10:05:07.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.access", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "success", "event.timezone": "-02:00", + "event.type": [ + "access" + ], "fileset.name": "access", - "http.request.method": "GET", - "http.request.referrer": "-", + "http.request.method": "get", "http.response.body.bytes": 571, "http.response.status_code": 200, "http.version": "1.1", @@ -77,6 +102,9 @@ "10.0.0.1", "85.181.35.98" ], + "related.ip": [ + "85.181.35.98" + ], "service.type": "nginx", "source.address": "85.181.35.98", "source.as.number": 6805, @@ -90,7 +118,6 @@ "source.geo.region_name": "Land Berlin", "source.ip": "85.181.35.98", "url.original": "/ocelot", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", @@ -101,12 +128,19 @@ }, { "@timestamp": "2016-12-07T10:05:07.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.access", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "success", "event.timezone": "-02:00", + "event.type": [ + "access" + ], "fileset.name": "access", - "http.request.method": "GET", - "http.request.referrer": "-", + "http.request.method": "get", "http.response.body.bytes": 571, "http.response.status_code": 200, "http.version": "1.1", @@ -115,6 +149,9 @@ "nginx.access.remote_ip_list": [ "85.181.35.98" ], + "related.ip": [ + "85.181.35.98" + ], "service.type": "nginx", "source.address": "85.181.35.98", "source.as.number": 6805, @@ -128,7 +165,6 @@ "source.geo.region_name": "Land Berlin", "source.ip": "85.181.35.98", "url.original": "/ocelot", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36", @@ -139,12 +175,19 @@ }, { "@timestamp": "2016-01-22T13:18:29.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.access", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "success", "event.timezone": "-02:00", + "event.type": [ + "access" + ], "fileset.name": "access", - "http.request.method": "GET", - "http.request.referrer": "-", + "http.request.method": "get", "http.response.body.bytes": 25507, "http.response.status_code": 200, "http.version": "1.1", @@ -156,6 +199,9 @@ "204.246.1.1", "10.2.1.185" ], + "related.ip": [ + "199.96.1.1" + ], "service.type": "nginx", "source.address": "199.96.1.1", "source.as.number": 19065, @@ -169,19 +215,25 @@ "source.geo.region_name": "Illinois", "source.ip": "199.96.1.1", "url.original": "/assets/xxxx?q=100", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Other", "user_agent.original": "Amazon CloudFront" }, { "@timestamp": "2016-12-30T06:47:09.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.access", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "failure", "event.timezone": "-02:00", + "event.type": [ + "access" + ], "fileset.name": "access", - "http.request.method": "GET", - "http.request.referrer": "-", + "http.request.method": "get", "http.response.body.bytes": 8571, "http.response.status_code": 404, "http.version": "1.1", @@ -192,6 +244,9 @@ "10.225.192.17", "10.2.2.121" ], + "related.ip": [ + "2a03:0000:10ff:f00f:0000:0000:0:8000" + ], "service.type": "nginx", "source.address": "2a03:0000:10ff:f00f:0000:0000:0:8000", "source.geo.continent_name": "Europe", @@ -200,7 +255,6 @@ "source.geo.location.lon": -8.0, "source.ip": "2a03:0000:10ff:f00f:0000:0000:0:8000", "url.original": "/test.html", - "user.name": "-", "user_agent.device.name": "Spider", "user_agent.name": "Facebot", "user_agent.original": "Mozilla/5.0 (compatible; Facebot 1.0; https://developers.facebook.com/docs/sharing/webmasters/crawler)", @@ -208,11 +262,18 @@ }, { "@timestamp": "2018-04-12T07:48:40.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.access", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "failure", "event.timezone": "-02:00", + "event.type": [ + "access" + ], "fileset.name": "access", - "http.request.referrer": "-", "http.response.body.bytes": 0, "http.response.status_code": 400, "input.type": "log", @@ -220,40 +281,49 @@ "nginx.access.remote_ip_list": [ "127.0.0.1" ], + "related.ip": [ + "127.0.0.1" + ], "service.type": "nginx", "source.address": "127.0.0.1", - "source.ip": "127.0.0.1", - "user.name": "-", - "user_agent.device.name": "Other", - "user_agent.name": "Other", - "user_agent.original": "-" + "source.ip": "127.0.0.1" }, { "@timestamp": "2019-02-26T14:39:42.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.access", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "failure", "event.timezone": "-02:00", + "event.type": [ + "access" + ], "fileset.name": "access", - "http.request.referrer": "-", "http.response.body.bytes": 173, "http.response.status_code": 400, "input.type": "log", "log.offset": 1184, "service.type": "nginx", - "source.address": "unix:", - "user.name": "-", - "user_agent.device.name": "Other", - "user_agent.name": "Other", - "user_agent.original": "-" + "source.address": "unix:" }, { "@timestamp": "2017-05-29T19:02:48.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.access", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "success", "event.timezone": "-02:00", + "event.type": [ + "access" + ], "fileset.name": "access", - "http.request.method": "GET", - "http.request.referrer": "-", + "http.request.method": "get", "http.response.body.bytes": 612, "http.response.status_code": 200, "http.version": "1.1", @@ -265,7 +335,6 @@ "service.type": "nginx", "source.address": "localhost", "url.original": "/test2", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Firefox Alpha", "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", @@ -276,12 +345,19 @@ }, { "@timestamp": "2017-05-29T19:02:48.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.access", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "success", "event.timezone": "-02:00", + "event.type": [ + "access" + ], "fileset.name": "access", - "http.request.method": "GET", - "http.request.referrer": "-", + "http.request.method": "get", "http.response.body.bytes": 612, "http.response.status_code": 200, "http.version": "1.1", @@ -294,7 +370,6 @@ "service.type": "nginx", "source.address": "localhost", "url.original": "/test2", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Firefox Alpha", "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", diff --git a/filebeat/module/nginx/error/ingest/pipeline.json b/filebeat/module/nginx/error/ingest/pipeline.json deleted file mode 100644 index 473fa087922..00000000000 --- a/filebeat/module/nginx/error/ingest/pipeline.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "description": "Pipeline for parsing the Nginx error logs", - "processors": [{ - "grok": { - "field": "message", - "patterns": [ - "%{DATA:nginx.error.time} \\[%{DATA:log.level}\\] %{NUMBER:process.pid:long}#%{NUMBER:process.thread.id:long}: (\\*%{NUMBER:nginx.error.connection_id:long} )?%{GREEDYMULTILINE:message}" - ], - "pattern_definitions": { - "GREEDYMULTILINE":"(.|\n|\t)*" - }, - "ignore_missing": true - } - }, { - "rename": { - "field": "@timestamp", - "target_field": "event.created" - } - }, { - "date": { - "if": "ctx.event.timezone == null", - "field": "nginx.error.time", - "target_field": "@timestamp", - "formats": ["yyyy/MM/dd H:m:s"], - "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}] - } - }, { - "date": { - "if": "ctx.event.timezone != null", - "field": "nginx.error.time", - "target_field": "@timestamp", - "formats": ["yyyy/MM/dd H:m:s"], - "timezone": "{{ event.timezone }}", - "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}] - } - }, { - "remove": { - "field": "nginx.error.time" - } - }], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/filebeat/module/nginx/error/ingest/pipeline.yml b/filebeat/module/nginx/error/ingest/pipeline.yml new file mode 100644 index 00000000000..5a33c34710c --- /dev/null +++ b/filebeat/module/nginx/error/ingest/pipeline.yml @@ -0,0 +1,51 @@ +description: Pipeline for parsing the Nginx error logs +processors: +- grok: + field: message + patterns: + - '%{DATA:nginx.error.time} \[%{DATA:log.level}\] %{NUMBER:process.pid:long}#%{NUMBER:process.thread.id:long}: + (\*%{NUMBER:nginx.error.connection_id:long} )?%{GREEDYMULTILINE:message}' + pattern_definitions: + GREEDYMULTILINE: |- + (.| + | )* + ignore_missing: true +- rename: + field: '@timestamp' + target_field: event.created +- date: + if: ctx.event.timezone == null + field: nginx.error.time + target_field: '@timestamp' + formats: + - yyyy/MM/dd H:m:s + on_failure: + - append: + field: error.message + value: '{{ _ingest.on_failure_message }}' +- date: + if: ctx.event.timezone != null + field: nginx.error.time + target_field: '@timestamp' + formats: + - yyyy/MM/dd H:m:s + timezone: '{{ event.timezone }}' + on_failure: + - append: + field: error.message + value: '{{ _ingest.on_failure_message }}' +- remove: + field: nginx.error.time +- set: + field: event.kind + value: event +- append: + field: event.category + value: web +- append: + field: event.type + value: error +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/filebeat/module/nginx/error/manifest.yml b/filebeat/module/nginx/error/manifest.yml index 641ec771bbb..b83c154693d 100644 --- a/filebeat/module/nginx/error/manifest.yml +++ b/filebeat/module/nginx/error/manifest.yml @@ -9,5 +9,5 @@ var: os.windows: - c:/programdata/nginx/logs/error.log* -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/nginx-error.yml diff --git a/filebeat/module/nginx/error/test/error.log-expected.json b/filebeat/module/nginx/error/test/error.log-expected.json index 6252e87d66b..8896a490705 100644 --- a/filebeat/module/nginx/error/test/error.log-expected.json +++ b/filebeat/module/nginx/error/test/error.log-expected.json @@ -1,9 +1,16 @@ [ { "@timestamp": "2016-10-25T14:49:34.000-02:00", + "event.category": [ + "web" + ], "event.dataset": "nginx.error", + "event.kind": "event", "event.module": "nginx", "event.timezone": "-02:00", + "event.type": [ + "error" + ], "fileset.name": "error", "input.type": "log", "log.level": "error", @@ -16,9 +23,16 @@ }, { "@timestamp": "2016-10-25T14:50:44.000-02:00", + "event.category": [ + "web" + ], "event.dataset": "nginx.error", + "event.kind": "event", "event.module": "nginx", "event.timezone": "-02:00", + "event.type": [ + "error" + ], "fileset.name": "error", "input.type": "log", "log.level": "error", @@ -31,9 +45,16 @@ }, { "@timestamp": "2019-10-30T23:26:34.000-02:00", + "event.category": [ + "web" + ], "event.dataset": "nginx.error", + "event.kind": "event", "event.module": "nginx", "event.timezone": "-02:00", + "event.type": [ + "error" + ], "fileset.name": "error", "input.type": "log", "log.flags": [ @@ -49,9 +70,16 @@ }, { "@timestamp": "2019-11-05T14:50:44.000-02:00", + "event.category": [ + "web" + ], "event.dataset": "nginx.error", + "event.kind": "event", "event.module": "nginx", "event.timezone": "-02:00", + "event.type": [ + "error" + ], "fileset.name": "error", "input.type": "log", "log.level": "error", diff --git a/filebeat/module/nginx/ingress_controller/ingest/pipeline.json b/filebeat/module/nginx/ingress_controller/ingest/pipeline.json deleted file mode 100644 index e660f22f022..00000000000 --- a/filebeat/module/nginx/ingress_controller/ingest/pipeline.json +++ /dev/null @@ -1,151 +0,0 @@ -{ - "description": "Pipeline for parsing Nginx ingress controller access logs. Requires the geoip and user_agent plugins.", - "processors": [ - { - "grok": { - "field": "message", - "patterns": [ - "(%{NGINX_HOST} )?\"?(?:%{NGINX_ADDRESS_LIST:nginx.ingress_controller.remote_ip_list}|%{NOTSPACE:source.address}) - %{DATA:user.name} \\[%{HTTPDATE:nginx.ingress_controller.time}\\] \"%{DATA:nginx.ingress_controller.info}\" %{NUMBER:http.response.status_code:long} %{NUMBER:http.response.body.bytes:long} \"%{DATA:http.request.referrer}\" \"%{DATA:user_agent.original}\" %{NUMBER:nginx.ingress_controller.http.request.length:long} %{NUMBER:nginx.ingress_controller.http.request.time:double} \\[%{DATA:nginx.ingress_controller.upstream.name}\\] \\[%{DATA:nginx.ingress_controller.upstream.alternative_name}\\] (%{UPSTREAM_ADDRESS}|-) (%{NUMBER:nginx.ingress_controller.upstream.response.length:long}|-) (%{NUMBER:nginx.ingress_controller.upstream.response.time:double}|-) (%{NUMBER:nginx.ingress_controller.upstream.response.status_code:long}|-) %{GREEDYDATA:nginx.ingress_controller.http.request.id}" - ], - "pattern_definitions": { - "NGINX_HOST": "(?:%{IP:destination.ip}|%{NGINX_NOTSEPARATOR:destination.domain})(:%{NUMBER:destination.port})?", - "NGINX_NOTSEPARATOR": "[^\t ,:]+", - "NGINX_ADDRESS_LIST": "(?:%{IP}|%{WORD})(\"?,?\\s*(?:%{IP}|%{WORD}))*", - "UPSTREAM_ADDRESS": "%{IP:nginx.ingress_controller.upstream.ip}(:%{NUMBER:nginx.ingress_controller.upstream.port})?" - }, - "ignore_missing": true - } - }, - { - "grok": { - "field": "nginx.ingress_controller.info", - "patterns": [ - "%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}", - "" - ], - "ignore_missing": true - } - }, - { - "remove": { - "field": "nginx.ingress_controller.info" - } - }, - { - "split": { - "field": "nginx.ingress_controller.remote_ip_list", - "separator": "\"?,?\\s+", - "ignore_missing": true - } - }, - { - "split": { - "field": "nginx.ingress_controller.origin", - "separator": "\"?,?\\s+", - "ignore_missing": true - } - }, - { - "set": { - "field": "source.address", - "if": "ctx.source?.address == null", - "value": "" - } - }, - { - "script": { - "if": "ctx.nginx?.access?.remote_ip_list != null && ctx.nginx.ingress_controller.remote_ip_list.length > 0", - "lang": "painless", - "source": "boolean isPrivate(def dot, def ip) { try { StringTokenizer tok = new StringTokenizer(ip, dot); int firstByte = Integer.parseInt(tok.nextToken()); int secondByte = Integer.parseInt(tok.nextToken()); if (firstByte == 10) { return true; } if (firstByte == 192 && secondByte == 168) { return true; } if (firstByte == 172 && secondByte >= 16 && secondByte <= 31) { return true; } if (firstByte == 127) { return true; } return false; } catch (Exception e) { return false; } } try { ctx.source.address = null; if (ctx.nginx.ingress_controller.remote_ip_list == null) { return; } def found = false; for (def item : ctx.nginx.ingress_controller.remote_ip_list) { if (!isPrivate(params.dot, item)) { ctx.source.address = item; found = true; break; } } if (!found) { ctx.source.address = ctx.nginx.ingress_controller.remote_ip_list[0]; }} catch (Exception e) { ctx.source.address = null; }", - "params": { - "dot": "." - } - } - }, - { - "remove": { - "field": "source.address", - "if": "ctx.source.address == null" - } - }, - { - "grok": { - "field": "source.address", - "patterns": ["^%{IP:source.ip}$"], - "ignore_failure": true - } - }, - { - "remove": { - "field": "message" - } - }, - { - "rename": { - "field": "@timestamp", - "target_field": "event.created" - } - }, - { - "date": { - "field": "nginx.ingress_controller.time", - "target_field": "@timestamp", - "formats": [ - "dd/MMM/yyyy:H:m:s Z" - ], - "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}] - } - }, - { - "remove": { - "field": "nginx.ingress_controller.time" - } - }, - { - "user_agent": { - "field": "user_agent.original" - } - }, - { - "geoip": { - "field": "source.ip", - "target_field": "source.geo", - "ignore_missing": true - } - }, - { - "geoip": { - "database_file": "GeoLite2-ASN.mmdb", - "field": "source.ip", - "target_field": "source.as", - "properties": [ - "asn", - "organization_name" - ], - "ignore_missing": true - } - }, - { - "rename": { - "field": "source.as.asn", - "target_field": "source.as.number", - "ignore_missing": true - } - }, - { - "rename": { - "field": "source.as.organization_name", - "target_field": "source.as.organization.name", - "ignore_missing": true - } - } - ], - "on_failure": [ - { - "set": { - "field": "error.message", - "value": "{{ _ingest.on_failure_message }}" - } - } - ] -} diff --git a/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml new file mode 100644 index 00000000000..9721be136e3 --- /dev/null +++ b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml @@ -0,0 +1,172 @@ +description: Pipeline for parsing Nginx ingress controller access logs. Requires the + geoip and user_agent plugins. +processors: +- grok: + field: message + patterns: + - (%{NGINX_HOST} )?"?(?:%{NGINX_ADDRESS_LIST:nginx.ingress_controller.remote_ip_list}|%{NOTSPACE:source.address}) + - (-|%{DATA:user.name}) \[%{HTTPDATE:nginx.ingress_controller.time}\] "%{DATA:nginx.ingress_controller.info}" + %{NUMBER:http.response.status_code:long} %{NUMBER:http.response.body.bytes:long} + "(-|%{DATA:http.request.referrer})" "(-|%{DATA:user_agent.original})" %{NUMBER:nginx.ingress_controller.http.request.length:long} + %{NUMBER:nginx.ingress_controller.http.request.time:double} \[%{DATA:nginx.ingress_controller.upstream.name}\] + \[%{DATA:nginx.ingress_controller.upstream.alternative_name}\] (%{UPSTREAM_ADDRESS}|-) + (%{NUMBER:nginx.ingress_controller.upstream.response.length:long}|-) (%{NUMBER:nginx.ingress_controller.upstream.response.time:double}|-) + (%{NUMBER:nginx.ingress_controller.upstream.response.status_code:long}|-) %{GREEDYDATA:nginx.ingress_controller.http.request.id} + pattern_definitions: + NGINX_HOST: (?:%{IP:destination.ip}|%{NGINX_NOTSEPARATOR:destination.domain})(:%{NUMBER:destination.port})? + NGINX_NOTSEPARATOR: "[^\t ,:]+" + NGINX_ADDRESS_LIST: (?:%{IP}|%{WORD})("?,?\s*(?:%{IP}|%{WORD}))* + UPSTREAM_ADDRESS: '%{IP:nginx.ingress_controller.upstream.ip}(:%{NUMBER:nginx.ingress_controller.upstream.port})?' + ignore_missing: true +- grok: + field: nginx.ingress_controller.info + patterns: + - '%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}' + - "" + ignore_missing: true +- remove: + field: nginx.ingress_controller.info +- split: + field: nginx.ingress_controller.remote_ip_list + separator: '"?,?\s+' + ignore_missing: true +- split: + field: nginx.ingress_controller.origin + separator: '"?,?\s+' + ignore_missing: true +- set: + field: source.address + if: ctx.source?.address == null + value: "" +- script: + if: ctx.nginx?.access?.remote_ip_list != null && ctx.nginx.ingress_controller.remote_ip_list.length > 0 + lang: painless + source: >- + boolean isPrivate(def dot, def ip) { + try { + StringTokenizer tok = new StringTokenizer(ip, dot); + int firstByte = Integer.parseInt(tok.nextToken()); + int secondByte = Integer.parseInt(tok.nextToken()); + if (firstByte == 10) { + return true; + } + if (firstByte == 192 && secondByte == 168) { + return true; + } + if (firstByte == 172 && secondByte >= 16 && secondByte <= 31) { + return true; + } + if (firstByte == 127) { + return true; + } + return false; + } + catch (Exception e) { + return false; + } + } + try { + ctx.source.address = null; + if (ctx.nginx.ingress_controller.remote_ip_list == null) { + return; + } + def found = false; + for (def item : ctx.nginx.ingress_controller.remote_ip_list) { + if (!isPrivate(params.dot, item)) { + ctx.source.address = item; + found = true; + break; + } + } + if (!found) { + ctx.source.address = ctx.nginx.ingress_controller.remote_ip_list[0]; + } + } + catch (Exception e) { + ctx.source.address = null; + } + params: + dot: . +- remove: + field: source.address + if: ctx.source.address == null +- grok: + field: source.address + patterns: + - ^%{IP:source.ip}$ + ignore_failure: true +- remove: + field: message +- rename: + field: '@timestamp' + target_field: event.created +- date: + field: nginx.ingress_controller.time + target_field: '@timestamp' + formats: + - dd/MMM/yyyy:H:m:s Z + on_failure: + - append: + field: error.message + value: '{{ _ingest.on_failure_message }}' +- remove: + field: nginx.ingress_controller.time +- user_agent: + field: user_agent.original + ignore_missing: true +- geoip: + field: source.ip + target_field: source.geo + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- set: + field: event.kind + value: event +- append: + field: event.category + value: web +- append: + field: event.type + value: info +- set: + field: event.outcome + value: success + if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400" +- set: + field: event.outcome + value: failure + if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400" +- lowercase: + field: http.request.method + ignore_missing: true +- append: + field: related.ip + value: "{{source.ip}}" + if: "ctx?.source?.ip != null" +- append: + field: related.ip + value: "{{destination.ip}}" + if: "ctx?.destination?.ip != null" +- append: + field: related.user + value: "{{user.name}}" + if: "ctx?.user?.name != null" +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/filebeat/module/nginx/ingress_controller/manifest.yml b/filebeat/module/nginx/ingress_controller/manifest.yml index 0f51e4d5c04..326beb11461 100644 --- a/filebeat/module/nginx/ingress_controller/manifest.yml +++ b/filebeat/module/nginx/ingress_controller/manifest.yml @@ -9,7 +9,7 @@ var: os.windows: - c:/programdata/nginx/logs/*access.log* -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/ingress_controller.yml requires.processors: diff --git a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json index 2dc9d1afbce..a2bf0f6c6e0 100644 --- a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json +++ b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json @@ -1,12 +1,19 @@ [ { "@timestamp": "2020-02-07T11:48:51.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.ingress_controller", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "success", "event.timezone": "-02:00", + "event.type": [ + "info" + ], "fileset.name": "ingress_controller", - "http.request.method": "POST", - "http.request.referrer": "-", + "http.request.method": "post", "http.response.body.bytes": 59, "http.response.status_code": 200, "http.version": "1.1", @@ -28,7 +35,6 @@ "service.type": "nginx", "source.address": "", "url.original": "/products", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "curl", "user_agent.original": "curl/7.54.0", @@ -36,12 +42,19 @@ }, { "@timestamp": "2020-02-07T11:49:15.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.ingress_controller", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "success", "event.timezone": "-02:00", + "event.type": [ + "info" + ], "fileset.name": "ingress_controller", - "http.request.method": "GET", - "http.request.referrer": "-", + "http.request.method": "get", "http.response.body.bytes": 59, "http.response.status_code": 200, "http.version": "1.1", @@ -63,7 +76,6 @@ "service.type": "nginx", "source.address": "", "url.original": "/products/42", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "curl", "user_agent.original": "curl/7.54.0", @@ -71,12 +83,19 @@ }, { "@timestamp": "2020-02-07T11:49:30.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.ingress_controller", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "success", "event.timezone": "-02:00", + "event.type": [ + "info" + ], "fileset.name": "ingress_controller", - "http.request.method": "DELETE", - "http.request.referrer": "-", + "http.request.method": "delete", "http.response.body.bytes": 59, "http.response.status_code": 200, "http.version": "1.1", @@ -98,7 +117,6 @@ "service.type": "nginx", "source.address": "", "url.original": "/products/42", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "curl", "user_agent.original": "curl/7.54.0", @@ -106,12 +124,19 @@ }, { "@timestamp": "2020-02-07T11:49:43.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.ingress_controller", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "success", "event.timezone": "-02:00", + "event.type": [ + "info" + ], "fileset.name": "ingress_controller", - "http.request.method": "PATCH", - "http.request.referrer": "-", + "http.request.method": "patch", "http.response.body.bytes": 59, "http.response.status_code": 200, "http.version": "1.1", @@ -133,7 +158,6 @@ "service.type": "nginx", "source.address": "", "url.original": "/products/42", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "curl", "user_agent.original": "curl/7.54.0", @@ -141,12 +165,19 @@ }, { "@timestamp": "2020-02-07T11:49:50.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.ingress_controller", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "failure", "event.timezone": "-02:00", + "event.type": [ + "info" + ], "fileset.name": "ingress_controller", - "http.request.method": "PATCHp", - "http.request.referrer": "-", + "http.request.method": "patchp", "http.response.body.bytes": 163, "http.response.status_code": 400, "http.version": "1.1", @@ -162,20 +193,23 @@ "nginx.ingress_controller.upstream.name": "", "service.type": "nginx", "source.address": "", - "url.original": "/products/42", - "user.name": "-", - "user_agent.device.name": "Other", - "user_agent.name": "Other", - "user_agent.original": "-" + "url.original": "/products/42" }, { "@timestamp": "2020-02-07T11:50:09.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.ingress_controller", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "failure", "event.timezone": "-02:00", + "event.type": [ + "info" + ], "fileset.name": "ingress_controller", "http.request.method": "geti", - "http.request.referrer": "-", "http.response.body.bytes": 163, "http.response.status_code": 400, "http.version": "1.1", @@ -191,20 +225,23 @@ "nginx.ingress_controller.upstream.name": "", "service.type": "nginx", "source.address": "", - "url.original": "/products/42", - "user.name": "-", - "user_agent.device.name": "Other", - "user_agent.name": "Other", - "user_agent.original": "-" + "url.original": "/products/42" }, { "@timestamp": "2020-02-07T11:55:05.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.ingress_controller", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "success", "event.timezone": "-02:00", + "event.type": [ + "info" + ], "fileset.name": "ingress_controller", - "http.request.method": "GET", - "http.request.referrer": "-", + "http.request.method": "get", "http.response.body.bytes": 59, "http.response.status_code": 200, "http.version": "1.1", @@ -226,7 +263,6 @@ "service.type": "nginx", "source.address": "", "url.original": "/products/42", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Wget", "user_agent.original": "Wget/1.20.3 (darwin18.6.0)", @@ -234,12 +270,19 @@ }, { "@timestamp": "2020-02-07T11:55:57.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.ingress_controller", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "success", "event.timezone": "-02:00", + "event.type": [ + "info" + ], "fileset.name": "ingress_controller", - "http.request.method": "GET", - "http.request.referrer": "-", + "http.request.method": "get", "http.response.body.bytes": 59, "http.response.status_code": 200, "http.version": "1.1", @@ -261,7 +304,6 @@ "service.type": "nginx", "source.address": "", "url.original": "/products/42", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36", @@ -272,11 +314,19 @@ }, { "@timestamp": "2020-02-07T11:55:57.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.ingress_controller", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "success", "event.timezone": "-02:00", + "event.type": [ + "info" + ], "fileset.name": "ingress_controller", - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "http://hello-world.info/products/42", "http.response.body.bytes": 59, "http.response.status_code": 200, @@ -299,7 +349,6 @@ "service.type": "nginx", "source.address": "", "url.original": "/favicon.ico", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36", @@ -310,12 +359,19 @@ }, { "@timestamp": "2020-02-07T11:56:24.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.ingress_controller", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "success", "event.timezone": "-02:00", + "event.type": [ + "info" + ], "fileset.name": "ingress_controller", - "http.request.method": "GET", - "http.request.referrer": "-", + "http.request.method": "get", "http.response.body.bytes": 61, "http.response.status_code": 200, "http.version": "1.1", @@ -337,7 +393,6 @@ "service.type": "nginx", "source.address": "", "url.original": "/v2", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36", @@ -348,11 +403,19 @@ }, { "@timestamp": "2020-02-07T11:56:24.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.ingress_controller", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "success", "event.timezone": "-02:00", + "event.type": [ + "info" + ], "fileset.name": "ingress_controller", - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "http://hello-world.info/v2", "http.response.body.bytes": 59, "http.response.status_code": 200, @@ -375,7 +438,6 @@ "service.type": "nginx", "source.address": "", "url.original": "/favicon.ico", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36", @@ -386,12 +448,19 @@ }, { "@timestamp": "2020-02-07T11:56:36.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.ingress_controller", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "success", "event.timezone": "-02:00", + "event.type": [ + "info" + ], "fileset.name": "ingress_controller", - "http.request.method": "GET", - "http.request.referrer": "-", + "http.request.method": "get", "http.response.body.bytes": 59, "http.response.status_code": 200, "http.version": "1.1", @@ -413,7 +482,6 @@ "service.type": "nginx", "source.address": "", "url.original": "/products/42", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", @@ -424,11 +492,19 @@ }, { "@timestamp": "2020-02-07T11:56:36.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.ingress_controller", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "success", "event.timezone": "-02:00", + "event.type": [ + "info" + ], "fileset.name": "ingress_controller", - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "http://hello-world.info/products/42", "http.response.body.bytes": 59, "http.response.status_code": 200, @@ -451,7 +527,6 @@ "service.type": "nginx", "source.address": "", "url.original": "/favicon.ico", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", @@ -462,12 +537,19 @@ }, { "@timestamp": "2020-02-07T11:56:54.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.ingress_controller", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "success", "event.timezone": "-02:00", + "event.type": [ + "info" + ], "fileset.name": "ingress_controller", - "http.request.method": "GET", - "http.request.referrer": "-", + "http.request.method": "get", "http.response.body.bytes": 59, "http.response.status_code": 200, "http.version": "1.1", @@ -489,7 +571,6 @@ "service.type": "nginx", "source.address": "", "url.original": "/products/42", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", @@ -500,12 +581,19 @@ }, { "@timestamp": "2020-02-07T11:56:54.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.ingress_controller", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "success", "event.timezone": "-02:00", + "event.type": [ + "info" + ], "fileset.name": "ingress_controller", - "http.request.method": "GET", - "http.request.referrer": "-", + "http.request.method": "get", "http.response.body.bytes": 59, "http.response.status_code": 200, "http.version": "1.1", @@ -527,7 +615,6 @@ "service.type": "nginx", "source.address": "", "url.original": "/", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", @@ -538,11 +625,19 @@ }, { "@timestamp": "2020-02-07T11:56:54.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.ingress_controller", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "success", "event.timezone": "-02:00", + "event.type": [ + "info" + ], "fileset.name": "ingress_controller", - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "http://hello-world.info/", "http.response.body.bytes": 59, "http.response.status_code": 200, @@ -565,7 +660,6 @@ "service.type": "nginx", "source.address": "", "url.original": "/favicon.ico", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", @@ -576,12 +670,19 @@ }, { "@timestamp": "2020-02-07T11:56:56.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.ingress_controller", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "success", "event.timezone": "-02:00", + "event.type": [ + "info" + ], "fileset.name": "ingress_controller", - "http.request.method": "GET", - "http.request.referrer": "-", + "http.request.method": "get", "http.response.body.bytes": 61, "http.response.status_code": 200, "http.version": "1.1", @@ -603,7 +704,6 @@ "service.type": "nginx", "source.address": "", "url.original": "/v2", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", @@ -614,11 +714,19 @@ }, { "@timestamp": "2020-02-07T11:56:56.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.ingress_controller", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "success", "event.timezone": "-02:00", + "event.type": [ + "info" + ], "fileset.name": "ingress_controller", - "http.request.method": "GET", + "http.request.method": "get", "http.request.referrer": "http://hello-world.info/v2", "http.response.body.bytes": 59, "http.response.status_code": 200, @@ -641,7 +749,6 @@ "service.type": "nginx", "source.address": "", "url.original": "/favicon.ico", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", @@ -652,12 +759,19 @@ }, { "@timestamp": "2020-02-07T12:00:28.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.ingress_controller", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "success", "event.timezone": "-02:00", + "event.type": [ + "info" + ], "fileset.name": "ingress_controller", - "http.request.method": "GET", - "http.request.referrer": "-", + "http.request.method": "get", "http.response.body.bytes": 59, "http.response.status_code": 200, "http.version": "1.1", @@ -679,7 +793,6 @@ "service.type": "nginx", "source.address": "", "url.original": "/products/42?address=delhi+technological+university", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Python Requests", "user_agent.original": "python-requests/2.22.0", @@ -687,12 +800,19 @@ }, { "@timestamp": "2020-02-07T12:02:38.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.ingress_controller", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "success", "event.timezone": "-02:00", + "event.type": [ + "info" + ], "fileset.name": "ingress_controller", - "http.request.method": "GET", - "http.request.referrer": "-", + "http.request.method": "get", "http.response.body.bytes": 61, "http.response.status_code": 200, "http.version": "1.1", @@ -714,7 +834,6 @@ "service.type": "nginx", "source.address": "", "url.original": "/v2", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0", @@ -725,12 +844,19 @@ }, { "@timestamp": "2020-02-07T12:02:38.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.ingress_controller", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "success", "event.timezone": "-02:00", + "event.type": [ + "info" + ], "fileset.name": "ingress_controller", - "http.request.method": "GET", - "http.request.referrer": "-", + "http.request.method": "get", "http.response.body.bytes": 59, "http.response.status_code": 200, "http.version": "1.1", @@ -752,7 +878,6 @@ "service.type": "nginx", "source.address": "", "url.original": "/favicon.ico", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0", @@ -763,12 +888,19 @@ }, { "@timestamp": "2020-02-07T12:02:42.000Z", + "event.category": [ + "web" + ], "event.dataset": "nginx.ingress_controller", + "event.kind": "event", "event.module": "nginx", + "event.outcome": "success", "event.timezone": "-02:00", + "event.type": [ + "info" + ], "fileset.name": "ingress_controller", - "http.request.method": "GET", - "http.request.referrer": "-", + "http.request.method": "get", "http.response.body.bytes": 61, "http.response.status_code": 200, "http.version": "1.1", @@ -790,7 +922,6 @@ "service.type": "nginx", "source.address": "", "url.original": "/v2/some", - "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0",