diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 7ab6da89c2d..b15df9f2e45 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -41,6 +41,7 @@ https://github.com/elastic/beats/compare/v5.3.0...master[Check the HEAD diff] - Fix issue that new prospector was not reloaded on conflict {pull}4128[4128] - Fix grok pattern in filebeat module system/auth without hostname. {pull}4224[4224] - Fix the Mysql slowlog parsing of IP addresses. {pull}4183[4183] +- Allow string characters in user agent patch version (NGINX and Apache) {pull}4415[4415] *Heartbeat* diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 49487198796..461655b7e42 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -153,7 +153,7 @@ The minor version of the user agent. [float] === apache2.access.user_agent.patch -type: long +type: keyword The patch version of the user agent. @@ -875,7 +875,7 @@ The minor version of the user agent. [float] === nginx.access.user_agent.patch -type: long +type: keyword The patch version of the user agent. diff --git a/filebeat/filebeat.template-es2x.json b/filebeat/filebeat.template-es2x.json index 87ae3fa25fe..853cd011844 100644 --- a/filebeat/filebeat.template-es2x.json +++ b/filebeat/filebeat.template-es2x.json @@ -124,7 +124,9 @@ "type": "string" }, "patch": { - "type": "long" + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" } } }, @@ -534,7 +536,9 @@ "type": "string" }, "patch": { - "type": "long" + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" } } }, diff --git a/filebeat/filebeat.template-es6x.json b/filebeat/filebeat.template-es6x.json index 428543abfe1..f6206aec20f 100644 --- a/filebeat/filebeat.template-es6x.json +++ b/filebeat/filebeat.template-es6x.json @@ -104,7 +104,8 @@ "type": "keyword" }, "patch": { - "type": "long" + "ignore_above": 1024, + "type": "keyword" } } }, @@ -452,7 +453,8 @@ "type": "keyword" }, "patch": { - "type": "long" + "ignore_above": 1024, + "type": "keyword" } } }, diff --git a/filebeat/filebeat.template.json b/filebeat/filebeat.template.json index d701ae47311..ea406c7ee25 100644 --- a/filebeat/filebeat.template.json +++ b/filebeat/filebeat.template.json @@ -107,7 +107,8 @@ "type": "keyword" }, "patch": { - "type": "long" + "ignore_above": 1024, + "type": "keyword" } } }, @@ -455,7 +456,8 @@ "type": "keyword" }, "patch": { - "type": "long" + "ignore_above": 1024, + "type": "keyword" } } }, diff --git a/filebeat/module/apache2/access/_meta/fields.yml b/filebeat/module/apache2/access/_meta/fields.yml index c245407dbb4..97fabdc5cab 100644 --- a/filebeat/module/apache2/access/_meta/fields.yml +++ b/filebeat/module/apache2/access/_meta/fields.yml @@ -61,7 +61,7 @@ description: > The minor version of the user agent. - name: patch - type: long + type: keyword description: > The patch version of the user agent. - name: name diff --git a/filebeat/module/apache2/access/test/test.log b/filebeat/module/apache2/access/test/test.log index a19899c8ce6..c04f25846ba 100644 --- a/filebeat/module/apache2/access/test/test.log +++ b/filebeat/module/apache2/access/test/test.log @@ -1,3 +1,4 @@ ::1 - - [26/Dec/2016:16:16:29 +0200] "GET /favicon.ico HTTP/1.1" 404 209 192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] "GET /hello HTTP/1.1" 404 499 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0" ::1 - - [26/Dec/2016:16:16:48 +0200] "-" 408 - +172.17.0.1 - - [29/May/2017:19:02:48 +0000] "GET /stringpatch HTTP/1.1" 404 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-" diff --git a/filebeat/module/apache2/access/test/test.log-expected.json b/filebeat/module/apache2/access/test/test.log-expected.json index 0997759acc7..9acaf955fc3 100644 --- a/filebeat/module/apache2/access/test/test.log-expected.json +++ b/filebeat/module/apache2/access/test/test.log-expected.json @@ -1,140 +1,193 @@ [ -{ - "_index": "filebeat-2016.12.27", - "_type": "log", - "_id": "AVlBCaYsqYg9cc5KQfcT", - "_score": null, - "_source": { - "@timestamp": "2016-12-26T14:16:29.000Z", - "offset": 73, - "apache2": { - "access": { - "response_code": "404", - "remote_ip": "::1", - "method": "GET", - "user_name": "-", - "http_version": "1.1", - "body_sent": { - "bytes": "209" + { + "_id": "AVlBCaYsqYg9cc5KQfcT", + "_index": "filebeat-2016.12.27", + "_score": null, + "_source": { + "@timestamp": "2016-12-26T14:16:29.000Z", + "apache2": { + "access": { + "body_sent": { + "bytes": "209" + }, + "http_version": "1.1", + "method": "GET", + "remote_ip": "::1", + "response_code": "404", + "url": "/favicon.ico", + "user_name": "-" + } + }, + "beat": { + "hostname": "192-168-0-7.rdsnet.ro", + "name": "192-168-0-7.rdsnet.ro", + "version": "6.0.0-alpha1" + }, + "fields": { + "pipeline_id": "apache2-access-with_plugins", + "source_type": "apache2-access" + }, + "offset": 73, + "prospector": { + "type": "log" + }, + "read_timestamp": "2016-12-27T15:52:23.304Z", + "source": "module/apache2/access/test/test.log" }, - "url": "/favicon.ico" - } - }, - "beat": { - "hostname": "192-168-0-7.rdsnet.ro", - "name": "192-168-0-7.rdsnet.ro", - "version": "6.0.0-alpha1" - }, - "input_type": "log", - "read_timestamp": "2016-12-27T15:52:23.304Z", - "source": "module/apache2/access/test/test.log", - "fields": { - "pipeline_id": "apache2-access-with_plugins", - "source_type": "apache2-access" - }, - "type": "log" - }, - "fields": { - "@timestamp": [ - 1482761789000 - ] - }, - "sort": [ - 1482761789000 - ] -}, -{ - "_index": "filebeat-2016.12.27", - "_type": "log", - "_id": "AVlBCaYsqYg9cc5KQfcU", - "_score": null, - "_source": { - "@timestamp": "2016-12-26T16:22:13.000Z", - "offset": 238, - "apache2": { - "access": { - "referrer": "-", - "response_code": "404", - "remote_ip": "192.168.33.1", - "method": "GET", - "user_name": "-", - "http_version": "1.1", - "body_sent": { - "bytes": "499" + "_type": "log", + "fields": { + "@timestamp": [ + 1482761789000 + ] }, - "url": "/hello", - "user_agent": { - "major": "50", - "minor": "0", - "os": "Mac OS X 10.12", - "os_minor": "12", - "os_major": "10", - "name": "Firefox", - "os_name": "Mac OS X", - "device": "Other" - } - } - }, - "beat": { - "hostname": "192-168-0-7.rdsnet.ro", - "name": "192-168-0-7.rdsnet.ro", - "version": "6.0.0-alpha1" - }, - "input_type": "log", - "read_timestamp": "2016-12-27T15:52:23.304Z", - "source": "module/apache2/access/test/test.log", - "fields": { - "pipeline_id": "apache2-access-with_plugins", - "source_type": "apache2-access" - }, - "type": "log" - }, - "fields": { - "@timestamp": [ - 1482769333000 - ] - }, - "sort": [ - 1482769333000 - ] -}, -{ - "_index": "filebeat-2016.12.27", - "_type": "log", - "_id": "AVlBCaYsqYg9cc5KQfc-", - "_score": null, - "_source": { - "@timestamp": "2016-12-26T14:16:48.000Z", - "offset": 285, - "apache2": { - "access": { - "response_code": "408", - "remote_ip": "::1", - "user_name": "-" - } + "sort": [ + 1482761789000 + ] }, - "beat": { - "hostname": "192-168-0-7.rdsnet.ro", - "name": "192-168-0-7.rdsnet.ro", - "version": "6.0.0-alpha1" + { + "_id": "AVlBCaYsqYg9cc5KQfcU", + "_index": "filebeat-2016.12.27", + "_score": null, + "_source": { + "@timestamp": "2016-12-26T16:22:13.000Z", + "apache2": { + "access": { + "body_sent": { + "bytes": "499" + }, + "http_version": "1.1", + "method": "GET", + "referrer": "-", + "remote_ip": "192.168.33.1", + "response_code": "404", + "url": "/hello", + "user_agent": { + "device": "Other", + "major": "50", + "minor": "0", + "name": "Firefox", + "os": "Mac OS X 10.12", + "os_major": "10", + "os_minor": "12", + "os_name": "Mac OS X" + }, + "user_name": "-" + } + }, + "beat": { + "hostname": "192-168-0-7.rdsnet.ro", + "name": "192-168-0-7.rdsnet.ro", + "version": "6.0.0-alpha1" + }, + "fields": { + "pipeline_id": "apache2-access-with_plugins", + "source_type": "apache2-access" + }, + "offset": 238, + "prospector": { + "type": "log" + }, + "read_timestamp": "2016-12-27T15:52:23.304Z", + "source": "module/apache2/access/test/test.log" + }, + "_type": "log", + "fields": { + "@timestamp": [ + 1482769333000 + ] + }, + "sort": [ + 1482769333000 + ] }, - "input_type": "log", - "read_timestamp": "2016-12-27T16:04:58.319Z", - "source": "module/apache2/access/test/test.log", - "fields": { - "pipeline_id": "apache2-access-with_plugins", - "source_type": "apache2-access" + { + "_id": "AVlBCaYsqYg9cc5KQfc-", + "_index": "filebeat-2016.12.27", + "_score": null, + "_source": { + "@timestamp": "2016-12-26T14:16:48.000Z", + "apache2": { + "access": { + "remote_ip": "::1", + "response_code": "408", + "user_name": "-" + } + }, + "beat": { + "hostname": "192-168-0-7.rdsnet.ro", + "name": "192-168-0-7.rdsnet.ro", + "version": "6.0.0-alpha1" + }, + "fields": { + "pipeline_id": "apache2-access-with_plugins", + "source_type": "apache2-access" + }, + "offset": 285, + "prospector": { + "type": "log" + }, + "read_timestamp": "2016-12-27T16:04:58.319Z", + "source": "module/apache2/access/test/test.log" + }, + "_type": "log", + "fields": { + "@timestamp": [ + 1482761808000 + ] + }, + "sort": [ + 1482761808000 + ] }, - "type": "log" - }, - "fields": { - "@timestamp": [ - 1482761808000 - ] - }, - "sort": [ - 1482761808000 - ] -} + { + "_id": "AVxVs7QZsqw9BQCgtCgc", + "_index": "filebeat-6.0.0-alpha2-2017.05.29", + "_score": null, + "_source": { + "@timestamp": "2017-05-29T19:02:48.000Z", + "apache2": { + "access": { + "body_sent": { + "bytes": "612" + }, + "http_version": "1.1", + "method": "GET", + "referrer": "-", + "remote_ip": "172.17.0.1", + "response_code": "404", + "url": "/stringpatch", + "user_agent": { + "device": "Other", + "major": "15", + "minor": "0", + "name": "Firefox Alpha", + "os": "Windows 7", + "os_name": "Windows 7", + "patch": "a2" + }, + "user_name": "-" + } + }, + "beat": { + "hostname": "X1", + "name": "X1", + "version": "6.0.0-alpha2" + }, + "offset": 443, + "prospector": { + "type": "log" + }, + "read_timestamp": "2017-05-29T19:34:14.378Z", + "source": "/home/exekias/go/src/github.com/elastic/beats/filebeat/apache2.log" + }, + "_type": "doc", + "fields": { + "@timestamp": [ + 1496084568000 + ] + }, + "sort": [ + 1496084568000 + ] + } ] - diff --git a/filebeat/module/nginx/access/_meta/fields.yml b/filebeat/module/nginx/access/_meta/fields.yml index ca8e9282461..0b5f1eb275e 100644 --- a/filebeat/module/nginx/access/_meta/fields.yml +++ b/filebeat/module/nginx/access/_meta/fields.yml @@ -61,7 +61,7 @@ description: > The minor version of the user agent. - name: patch - type: long + type: keyword description: > The patch version of the user agent. - name: name