Merge remote-tracking branch 'origin/master' into stop-close-on-signa…

…l-goroutine
elastic · Mar 18, 2019 · 5c01cc0 · 5c01cc0
2 parents 581df59 + 258c1c8
commit 5c01cc0
Show file tree

Hide file tree

Showing 53 changed files with 1,317 additions and 541 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -141,6 +141,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Reconnections of Kubernetes watchers are now logged at debug level when they are harmless. {pull}10988[10988]
 - Include ip and boolean type when generating index pattern. {pull}10995[10995]
 - Cancelling enrollment of a beat will not enroll the beat. {issue}10150[10150]
+- Add missing fields and test cases for libbeat add_kubernetes_metadata processor. {issue}11133[11133], {pull}11134[11134]
 
 *Auditbeat*
 
@@ -168,6 +169,11 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix a bug when converting NetFlow fields to snake_case. {pull}10950[10950]
 - Add on_failure handler for Zeek ingest pipelines. Fix one field name error for notice and add an additional test case. {issue}11004[11004] {pull}11105[11105]
 - Fix goroutine leak happening when harvesters are dynamically stopped. {pull}11263[11263]
+- Fix issue preventing docker container events to be stored if the container has a network interface without ip address. {issue}11225[11225] {pull}11247[11247]
+- Add on_failure handler for Zeek ingest pipelines. Fix one field name error for notice and add an additional test 
+  case. {issue}11004[11004] {pull}11105[11105]
+- Change URLPATH grok pattern to support brackets. {issue}11135[11135] {pull}11252[11252]
+- Add support for iis log with different address format. {issue}11255[11255] {pull}11256[11256]
 
 *Heartbeat*
 

diff --git a/auditbeat/docs/fields.asciidoc b/auditbeat/docs/fields.asciidoc
@@ -6336,6 +6336,36 @@ type: object
 Kubernetes annotations map
 
 
+--
+
+*`kubernetes.replicaset.name`*::
++
+--
+type: keyword
+
+Kubernetes replicaset name
+
+
+--
+
+*`kubernetes.deployment.name`*::
++
+--
+type: keyword
+
+Kubernetes deployment name
+
+
+--
+
+*`kubernetes.statefulset.name`*::
++
+--
+type: keyword
+
+Kubernetes statefulset name
+
+
 --
 
 *`kubernetes.container.name`*::

diff --git a/auditbeat/include/fields.go b/auditbeat/include/fields.go
diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -6654,6 +6654,36 @@ type: object
 Kubernetes annotations map
 
 
+--
+
+*`kubernetes.replicaset.name`*::
++
+--
+type: keyword
+
+Kubernetes replicaset name
+
+
+--
+
+*`kubernetes.deployment.name`*::
++
+--
+type: keyword
+
+Kubernetes deployment name
+
+
+--
+
+*`kubernetes.statefulset.name`*::
++
+--
+type: keyword
+
+Kubernetes statefulset name
+
+
 --
 
 *`kubernetes.container.name`*::

diff --git a/filebeat/include/fields.go b/filebeat/include/fields.go
diff --git a/filebeat/module/iis/access/ingest/default.json b/filebeat/module/iis/access/ingest/default.json
@@ -4,11 +4,15 @@
     "grok": {
       "field": "message",
       "patterns":[
-        "%{TIMESTAMP_ISO8601:iis.access.time} %{IPORHOST:destination.address} %{WORD:http.request.method} %{URIPATH:url.path} %{NOTSPACE:url.query} %{NUMBER:destination.port:long} %{NOTSPACE:user.name} %{IPORHOST:source.address} %{NOTSPACE:user_agent.original} %{NOTSPACE:http.request.referrer} %{NUMBER:http.response.status_code:long} %{NUMBER:iis.access.sub_status:long} %{NUMBER:iis.access.win32_status:long} %{NUMBER:temp.duration:long}",
+        "%{TIMESTAMP_ISO8601:iis.access.time} %{IPORHOST:destination.address} %{WORD:http.request.method} %{URIPATHWITHBRACKET:url.path} %{NOTSPACE:url.query} %{NUMBER:destination.port:long} %{NOTSPACE:user.name} %{IPORHOST:source.address} %{NOTSPACE:user_agent.original} %{NOTSPACE:http.request.referrer} %{NUMBER:http.response.status_code:long} %{NUMBER:iis.access.sub_status:long} %{NUMBER:iis.access.win32_status:long} %{NUMBER:temp.duration:long}",
         "%{TIMESTAMP_ISO8601:iis.access.time} %{NOTSPACE:iis.access.site_name} %{WORD:http.request.method} %{URIPATH:url.path} %{NOTSPACE:url.query} %{NUMBER:destination.port:long} %{NOTSPACE:user.name} %{IPORHOST:source.address} %{NOTSPACE:user_agent.original} %{NOTSPACE:iis.access.cookie} %{NOTSPACE:http.request.referrer} %{NOTSPACE:destination.domain} %{NUMBER:http.response.status_code:long} %{NUMBER:iis.access.sub_status:long} %{NUMBER:iis.access.win32_status:long} %{NUMBER:http.response.body.bytes:long} %{NUMBER:http.request.body.bytes:long} %{NUMBER:temp.duration:long}",
         "%{TIMESTAMP_ISO8601:iis.access.time} %{NOTSPACE:iis.access.site_name} %{NOTSPACE:iis.access.server_name} %{IPORHOST:destination.address} %{WORD:http.request.method} %{URIPATH:url.path} %{NOTSPACE:url.query} %{NUMBER:destination.port:long} %{NOTSPACE:user.name} %{IPORHOST:source.address} HTTP/%{NUMBER:http.version} %{NOTSPACE:user_agent.original} %{NOTSPACE:iis.access.cookie} %{NOTSPACE:http.request.referrer} %{NOTSPACE:destination.domain} %{NUMBER:http.response.status_code:long} %{NUMBER:iis.access.sub_status:long} %{NUMBER:iis.access.win32_status:long} %{NUMBER:http.response.body.bytes:long} %{NUMBER:http.request.body.bytes:long} %{NUMBER:temp.duration:long}",
-        "%{TIMESTAMP_ISO8601:iis.access.time} \\[%{IPORHOST:destination.address}\\]\\(http://%{IPORHOST:destination.address}\\) %{WORD:http.request.method} %{URIPATH:url.path} %{NOTSPACE:url.query} %{NUMBER:destination.port:long} %{NOTSPACE:user.name} \\[%{IPORHOST:source.address}\\]\\(http://%{IPORHOST:source.address}\\) %{NOTSPACE:user_agent.original} %{NUMBER:http.response.status_code:long} %{NUMBER:iis.access.sub_status:long} %{NUMBER:iis.access.win32_status:long} %{NUMBER:temp.duration:long}"
-        ],
+        "%{TIMESTAMP_ISO8601:iis.access.time} \\[%{IPORHOST:destination.address}\\]\\(http://%{IPORHOST:destination.address}\\) %{WORD:http.request.method} %{URIPATH:url.path} %{NOTSPACE:url.query} %{NUMBER:destination.port:long} %{NOTSPACE:user.name} \\[%{IPORHOST:source.address}\\]\\(http://%{IPORHOST:source.address}\\) %{NOTSPACE:user_agent.original} %{NUMBER:http.response.status_code:long} %{NUMBER:iis.access.sub_status:long} %{NUMBER:iis.access.win32_status:long} %{NUMBER:temp.duration:long}",
+        "%{TIMESTAMP_ISO8601:iis.access.time} %{IPORHOST:destination.address} %{WORD:http.request.method} %{URIPATH:url.path} %{NOTSPACE:url.query} %{NUMBER:destination.port:long} %{NOTSPACE:user.name} %{IPORHOST:source.address} %{NOTSPACE:user_agent.original} %{NUMBER:http.response.status_code:long} %{NUMBER:iis.access.sub_status:long} %{NUMBER:iis.access.win32_status:long} %{NUMBER:temp.duration:long}"
+      ],
+      "pattern_definitions": {
+          "URIPATHWITHBRACKET": "(?:/[A-Za-z0-9$.+!*'(){},~:;=@#%&_\\-\\[\\]]*)+"
+      },
       "ignore_missing": true
     }
   }, {

diff --git a/filebeat/module/iis/access/test/test-iis-7.5.log b/filebeat/module/iis/access/test/test-iis-7.5.log
@@ -3,3 +3,6 @@
 #Date: 2018-08-28 18:24:25
 #Fields: date time s-ip cs-method cs-uri-stem cs-uri-query s-port cs-username c-ip cs(User-Agent) sc-status sc-substatus sc-win32-status time-taken
 2018-08-28 18:24:25 [10.100.220.70](http://10.100.220.70) GET / - 80 - [10.100.118.31](http://10.100.118.31) Mozilla/4.0+(compatible;+MSIE+7.0;+Windows+NT+6.3;+WOW64;+Trident/7.0;+.NET4.0E;+.NET4.0C;+.NET+CLR+3.5.30729;+.NET+CLR[+2.0.50727](tel:+2050727);+.NET+CLR+3.0.30729) 404 4 2 792
+2019-03-06 18:43:17 10.0.140.107 GET /health-monitoring - 80 - 10.0.140.2 - 200 0 0 15
+2019-03-06 18:43:17 10.0.140.107 GET /health-monitoring - 80 - 10.0.140.2 - 200 0 0 15
+2019-03-06 18:43:17 2001:cdba:0000:0000:0000:0000:3257:9652 GET /health-monitoring - 80 - 2001:cdba:0000:0000:0000:0000:3257:9652 - 200 0 0 15
diff --git a/filebeat/module/iis/access/test/test-iis-7.5.log-expected.json b/filebeat/module/iis/access/test/test-iis-7.5.log-expected.json
@@ -26,5 +26,83 @@
         "user_agent.original": "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.3; WOW64; Trident/7.0; .NET4.0E; .NET4.0C; .NET CLR 3.5.30729; .NET CLR[ 2.0.50727](tel: 2050727); .NET CLR 3.0.30729)",
         "user_agent.os.name": "Windows 8.1",
         "user_agent.version": "7.0"
+    },
+    {
+        "@timestamp": "2019-03-06T18:43:17.000Z",
+        "destination.address": "10.0.140.107",
+        "destination.ip": "10.0.140.107",
+        "destination.port": 80,
+        "ecs.version": "1.0.0",
+        "event.dataset": "iis.access",
+        "event.duration": 15000000,
+        "event.module": "iis",
+        "fileset.name": "access",
+        "http.request.method": "GET",
+        "http.response.status_code": 200,
+        "iis.access.sub_status": 0,
+        "iis.access.win32_status": 0,
+        "input.type": "log",
+        "log.offset": 532,
+        "service.type": "iis",
+        "source.address": "10.0.140.2",
+        "source.ip": "10.0.140.2",
+        "url.path": "/health-monitoring",
+        "url.query": "-",
+        "user.name": "-",
+        "user_agent.device.name": "Other",
+        "user_agent.name": "Other",
+        "user_agent.original": "-"
+    },
+    {
+        "@timestamp": "2019-03-06T18:43:17.000Z",
+        "destination.address": "10.0.140.107",
+        "destination.ip": "10.0.140.107",
+        "destination.port": 80,
+        "ecs.version": "1.0.0",
+        "event.dataset": "iis.access",
+        "event.duration": 15000000,
+        "event.module": "iis",
+        "fileset.name": "access",
+        "http.request.method": "GET",
+        "http.response.status_code": 200,
+        "iis.access.sub_status": 0,
+        "iis.access.win32_status": 0,
+        "input.type": "log",
+        "log.offset": 619,
+        "service.type": "iis",
+        "source.address": "10.0.140.2",
+        "source.ip": "10.0.140.2",
+        "url.path": "/health-monitoring",
+        "url.query": "-",
+        "user.name": "-",
+        "user_agent.device.name": "Other",
+        "user_agent.name": "Other",
+        "user_agent.original": "-"
+    },
+    {
+        "@timestamp": "2019-03-06T18:43:17.000Z",
+        "destination.address": "2001:cdba:0000:0000:0000:0000:3257:9652",
+        "destination.ip": "2001:cdba:0000:0000:0000:0000:3257:9652",
+        "destination.port": 80,
+        "ecs.version": "1.0.0",
+        "event.dataset": "iis.access",
+        "event.duration": 15000000,
+        "event.module": "iis",
+        "fileset.name": "access",
+        "http.request.method": "GET",
+        "http.response.status_code": 200,
+        "iis.access.sub_status": 0,
+        "iis.access.win32_status": 0,
+        "input.type": "log",
+        "log.offset": 706,
+        "service.type": "iis",
+        "source.address": "2001:cdba:0000:0000:0000:0000:3257:9652",
+        "source.ip": "2001:cdba:0000:0000:0000:0000:3257:9652",
+        "url.path": "/health-monitoring",
+        "url.query": "-",
+        "user.name": "-",
+        "user_agent.device.name": "Other",
+        "user_agent.name": "Other",
+        "user_agent.original": "-"
     }
 ]
diff --git a/filebeat/module/iis/access/test/test.log b/filebeat/module/iis/access/test/test.log
@@ -13,3 +13,5 @@
 #Date: 2018-01-01 10:11:12
 #Fields: date time s-sitename s-computername s-ip cs-method cs-uri-stem cs-uri-query s-port cs-username c-ip cs-version cs(User-Agent) cs(Cookie) cs(Referer) cs-host sc-status sc-substatus sc-win32-status sc-bytes cs-bytes time-taken
 2018-01-01 10:11:12 W3SVC1 MACHINE-NAME 127.0.0.1 GET / - 80 - 85.181.35.98 HTTP/1.1 Mozilla/5.0+(Macintosh;+Intel+Mac+OS+X+10_14_0)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/70.0.3538.102+Safari/537.36 - - example.com 200 0 0 123 456 789
+2018-12-31 12:52:33 10.44.0.136 GET / redirect:${%23req%3d%23context.get('com.opensymphony.xwork2.dispatcher.HttpServletRequest'),%23webroot%3d%23req.getSession().getServletContext().getRealPath('/'),%23resp.println(%23webroot),%23resp.flush(),%23resp.close()} 443 - 10.50.6.188 Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0) - 401 0 0 0
+2018-12-31 12:52:33 10.44.0.136 GET /${#context['xwork.MethodAccessor.denyMethodExecution']=!(#_memberAccess['allowStaticMethodAccess']=true),(@java.lang.Runtime@getRuntime()).exec('ipconfig').waitFor()}.action - 443 - 10.50.6.188 Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0) - 404 0 2 0
diff --git a/filebeat/module/iis/access/test/test.log-expected.json b/filebeat/module/iis/access/test/test.log-expected.json
@@ -111,5 +111,63 @@
         "user_agent.os.name": "Mac OS X",
         "user_agent.os.version": "10.14.0",
         "user_agent.version": "70.0.3538"
+    },
+    {
+        "@timestamp": "2018-12-31T12:52:33.000Z",
+        "destination.address": "10.44.0.136",
+        "destination.ip": "10.44.0.136",
+        "destination.port": 443,
+        "ecs.version": "1.0.0",
+        "event.dataset": "iis.access",
+        "event.duration": 0,
+        "event.module": "iis",
+        "fileset.name": "access",
+        "http.request.method": "GET",
+        "http.request.referrer": "-",
+        "http.response.status_code": 401,
+        "iis.access.sub_status": 0,
+        "iis.access.win32_status": 0,
+        "input.type": "log",
+        "log.offset": 1447,
+        "service.type": "iis",
+        "source.address": "10.50.6.188",
+        "source.ip": "10.50.6.188",
+        "url.path": "/",
+        "url.query": "redirect:${%23req%3d%23context.get('com.opensymphony.xwork2.dispatcher.HttpServletRequest'),%23webroot%3d%23req.getSession().getServletContext().getRealPath('/'),%23resp.println(%23webroot),%23resp.flush(),%23resp.close()}",
+        "user.name": "-",
+        "user_agent.device.name": "Other",
+        "user_agent.name": "IE",
+        "user_agent.original": "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)",
+        "user_agent.os.name": "Windows XP",
+        "user_agent.version": "8.0"
+    },
+    {
+        "@timestamp": "2018-12-31T12:52:33.000Z",
+        "destination.address": "10.44.0.136",
+        "destination.ip": "10.44.0.136",
+        "destination.port": 443,
+        "ecs.version": "1.0.0",
+        "event.dataset": "iis.access",
+        "event.duration": 0,
+        "event.module": "iis",
+        "fileset.name": "access",
+        "http.request.method": "GET",
+        "http.request.referrer": "-",
+        "http.response.status_code": 404,
+        "iis.access.sub_status": 0,
+        "iis.access.win32_status": 2,
+        "input.type": "log",
+        "log.offset": 1802,
+        "service.type": "iis",
+        "source.address": "10.50.6.188",
+        "source.ip": "10.50.6.188",
+        "url.path": "/${#context['xwork.MethodAccessor.denyMethodExecution']=!(#_memberAccess['allowStaticMethodAccess']=true),(@java.lang.Runtime@getRuntime()).exec('ipconfig').waitFor()}.action",
+        "url.query": "-",
+        "user.name": "-",
+        "user_agent.device.name": "Other",
+        "user_agent.name": "IE",
+        "user_agent.original": "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)",
+        "user_agent.os.name": "Windows XP",
+        "user_agent.version": "8.0"
     }
 ]
diff --git a/filebeat/module/system/auth/ingest/pipeline.json b/filebeat/module/system/auth/ingest/pipeline.json
@@ -9,12 +9,12 @@
                     "GREEDYMULTILINE" : "(.|\n)*"
                 },
                 "patterns": [
-                    "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} sshd(?:\\[%{POSINT:process.pid:long}\\])?: %{DATA:event.action} %{DATA:system.auth.ssh.method} for (invalid user )?%{DATA:user.name} from %{IPORHOST:source.ip} port %{NUMBER:source.port:long} ssh2(: %{GREEDYDATA:system.auth.ssh.signature})?",
-                    "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} sshd(?:\\[%{POSINT:process.pid:long}\\])?: %{DATA:event.action} user %{DATA:user.name} from %{IPORHOST:source.ip}",
-                    "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} sshd(?:\\[%{POSINT:process.pid:long}\\])?: Did not receive identification string from %{IPORHOST:system.auth.ssh.dropped_ip}",
-                    "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} sudo(?:\\[%{POSINT:process.pid:long}\\])?: \\s*%{DATA:user.name} :( %{DATA:system.auth.sudo.error} ;)? TTY=%{DATA:system.auth.sudo.tty} ; PWD=%{DATA:system.auth.sudo.pwd} ; USER=%{DATA:system.auth.sudo.user} ; COMMAND=%{GREEDYDATA:system.auth.sudo.command}",
-                    "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} groupadd(?:\\[%{POSINT:process.pid:long}\\])?: new group: name=%{DATA:group.name}, GID=%{NUMBER:group.id}",
-                    "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} useradd(?:\\[%{POSINT:process.pid:long}\\])?: new user: name=%{DATA:user.name}, UID=%{NUMBER:user.id}, GID=%{NUMBER:group.id}, home=%{DATA:system.auth.useradd.home}, shell=%{DATA:system.auth.useradd.shell}$",
+                    "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: %{DATA:event.outcome} %{DATA:system.auth.ssh.method} for (invalid user )?%{DATA:user.name} from %{IPORHOST:source.ip} port %{NUMBER:source.port:long} ssh2(: %{GREEDYDATA:system.auth.ssh.signature})?",
+                    "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: %{DATA:event.outcome} user %{DATA:user.name} from %{IPORHOST:source.ip}",
+                    "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: Did not receive identification string from %{IPORHOST:system.auth.ssh.dropped_ip}",
+                    "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: \\s*%{DATA:user.name} :( %{DATA:system.auth.sudo.error} ;)? TTY=%{DATA:system.auth.sudo.tty} ; PWD=%{DATA:system.auth.sudo.pwd} ; USER=%{DATA:system.auth.sudo.user} ; COMMAND=%{GREEDYDATA:system.auth.sudo.command}",
+                    "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: new group: name=%{DATA:group.name}, GID=%{NUMBER:group.id}",
+                    "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: new user: name=%{DATA:user.name}, UID=%{NUMBER:user.id}, GID=%{NUMBER:group.id}, home=%{DATA:system.auth.useradd.home}, shell=%{DATA:system.auth.useradd.shell}$",
                     "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname}? %{DATA:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: %{GREEDYMULTILINE:system.auth.message}"
                 ]
             }