WIP POC Event categorization on Linux auth logs

filebeat/docs/fields.asciidoc

@@ -12004,6 +12004,14 @@ alias to: user.name
 --
 
 
+*`system.auth.ssh.action`*::
++
+--
+The SSH action. Can be one of "Accepted", "Failed", "Invalid".
+
+
+--
+
 *`system.auth.ssh.method`*::
 +
 --

filebeat/module/system/auth/_meta/fields.yml

@@ -31,6 +31,9 @@
  - name: ssh
  type: group
  fields:
+ - name: action
+ description: >
+ The SSH action. Can be one of "Accepted", "Failed", "Invalid".
  - name: method
  description: >
  The SSH authentication method. Can be one of "password" or "publickey".

filebeat/module/system/auth/ingest/pipeline.json

@@ -1,6 +1,8 @@
 {
  "description": "Pipeline for parsing system authorisation/secure logs",
  "processors": [
+ { "set": { "field": "event.kind", "value": "event" } },
+
  {
  "grok": {
  "field": "message",
@@ -9,33 +11,72 @@
  "GREEDYMULTILINE" : "(.|\n)*"
  },
  "patterns": [
- "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} sshd(?:\\[%{POSINT:process.pid:long}\\])?: %{DATA:event.action} %{DATA:system.auth.ssh.method} for (invalid user )?%{DATA:user.name} from %{IPORHOST:source.ip} port %{NUMBER:source.port:long} ssh2(: %{GREEDYDATA:system.auth.ssh.signature})?",
- "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} sshd(?:\\[%{POSINT:process.pid:long}\\])?: %{DATA:event.action} user %{DATA:user.name} from %{IPORHOST:source.ip}",
- "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} sshd(?:\\[%{POSINT:process.pid:long}\\])?: Did not receive identification string from %{IPORHOST:system.auth.ssh.dropped_ip}",
- "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} sudo(?:\\[%{POSINT:process.pid:long}\\])?: \\s*%{DATA:user.name} :( %{DATA:system.auth.sudo.error} ;)? TTY=%{DATA:system.auth.sudo.tty} ; PWD=%{DATA:system.auth.sudo.pwd} ; USER=%{DATA:system.auth.sudo.user} ; COMMAND=%{GREEDYDATA:system.auth.sudo.command}",
- "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} groupadd(?:\\[%{POSINT:process.pid:long}\\])?: new group: name=%{DATA:group.name}, GID=%{NUMBER:group.id:long}",
- "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} useradd(?:\\[%{POSINT:process.pid:long}\\])?: new user: name=%{DATA:user.name}, UID=%{NUMBER:user.id:long}, GID=%{NUMBER:group.id:long}, home=%{DATA:system.auth.useradd.home}, shell=%{DATA:system.auth.useradd.shell}$",
- "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname}? %{DATA:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: %{GREEDYMULTILINE:system.auth.message}"
+ "^%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: +%{GREEDYMULTILINE:message}",
+ "^%{SYSLOGTIMESTAMP:system.auth.timestamp} %{DATA:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: +%{GREEDYMULTILINE:message}"
  ]
  }
  },
+
  {
- "remove": {
- "field": "message"
+ "grok": {
+ "field": "message",
+ "ignore_missing": true,
+ "ignore_failure": true,
+ "patterns": [
+ "^%{DATA:system.auth.ssh.action} %{DATA:system.auth.ssh.method} for (invalid user )?%{DATA:user.name} from %{IPORHOST:source.ip} port %{NUMBER:source.port:long} ssh2(: %{GREEDYDATA:system.auth.ssh.signature})?",
+ "^%{DATA:system.auth.ssh.action} user %{DATA:user.name} from %{IPORHOST:source.ip}",
+ "^Did not receive identification string from %{IPORHOST:system.auth.ssh.dropped_ip}"
+ ]
+ }
+ },
+
+ {
+ "grok": {
+ "field": "message",
+ "ignore_missing": true,
+ "ignore_failure": true,
+ "patterns": [
+ "^%{DATA:user.name} :( %{DATA:system.auth.sudo.error} ;)? TTY=%{DATA:system.auth.sudo.tty} ; PWD=%{DATA:system.auth.sudo.pwd} ; USER=%{DATA:system.auth.sudo.user} ; COMMAND=%{GREEDYDATA:system.auth.sudo.command}",
+ "^new group: name=%{DATA:group.name}, GID=%{NUMBER:group.id:long}",
+ "^new user: name=%{DATA:user.name}, UID=%{NUMBER:user.id:long}, GID=%{NUMBER:group.id:long}, home=%{DATA:system.auth.useradd.home}, shell=%{DATA:system.auth.useradd.shell}$"
+ ]
  }
  },
+
  {
- "rename": {
- "field": "system.auth.message",
- "target_field": "message",
- "ignore_missing": true
+ "set": {
+ "field": "event.category",
+ "value": "authentication",
+ "if": "ctx.containsKey('process') && ctx.process.containsKey('name') && ctx.process.name == 'sshd'"
+ }
+ },
+ {
+ "set": {
+ "field": "event.action",
+ "value": "ssh_login",
+ "if": "ctx.event.containsKey('category') && ctx.event.category == 'authentication'"
+ }
+ },
+ {
+ "set": {
+ "field": "event.outcome",
+ "value": "success",
+ "if": "ctx.event.containsKey('action') && ctx.event.action == 'ssh_login' && ctx.system.auth.containsKey('ssh') && ctx.system.auth.ssh.containsKey('action') && ctx.system.auth.ssh.action == 'Accepted'"
+ }
+ },
+ {
+ "set": {
+ "field": "event.outcome",
+ "value": "failure",
+ "if": "ctx.event.containsKey('action') && ctx.event.action == 'ssh_login' && ctx.system.auth.containsKey('ssh') && ((ctx.system.auth.ssh.containsKey('action') && ctx.system.auth.ssh.action == 'Failed') || (ctx.system.auth.ssh.containsKey('dropped_ip')))"
  }
  },
+
  {
  "set": {
  "field": "source.ip",
  "value": "{{system.auth.ssh.dropped_ip}}",
- "if": "ctx.containsKey('system') && ctx.system.containsKey('auth') && ctx.system.auth.containsKey('ssh') && ctx.system.auth.ssh.containsKey('dropped_ip')"
+ "if": "ctx.system.auth.containsKey('ssh') && ctx.system.auth.ssh.containsKey('dropped_ip')"
  }
  },
  {

filebeat/module/system/auth/test/test.log

@@ -1,10 +1,31 @@
 Feb 21 21:54:44 localhost sshd[3402]: Accepted publickey for vagrant from 10.0.2.2 port 63673 ssh2: RSA 39:33:99:e9:a0:dc:f2:33:a3:e5:72:3b:7c:3a:56:84
 Feb 23 00:13:35 localhost sshd[7483]: Accepted password for vagrant from 192.168.33.1 port 58803 ssh2
-Feb 21 21:56:12 localhost sshd[3430]: Invalid user test from 10.0.2.2
 Feb 20 08:35:22 slave22 sshd[5774]: Failed password for root from 116.31.116.24 port 29160 ssh2
-Feb 21 23:35:33 localhost sudo: vagrant : TTY=pts/0 ; PWD=/home/vagrant ; USER=root ; COMMAND=/bin/ls
+Jan 4 19:02:26 ubuntu-xenial sudo: vagrant : TTY=pts/0 ; PWD=/home/vagrant ; USER=root ; COMMAND=/bin/ls
+Jan 4 19:02:26 ubuntu-xenial sudo: pam_unix(sudo:session): session opened for user root by vagrant(uid=0)
+Jan 4 19:02:26 ubuntu-xenial sudo: pam_unix(sudo:session): session closed for user root
 Feb 19 15:30:04 slave22 sshd[18406]: Did not receive identification string from 123.57.245.163
 Feb 23 00:08:48 localhost sudo: vagrant : TTY=pts/1 ; PWD=/home/vagrant ; USER=root ; COMMAND=/bin/cat /var/log/secure
 Feb 24 00:13:02 precise32 sudo: tsg : user NOT in sudoers ; TTY=pts/1 ; PWD=/home/vagrant ; USER=root ; COMMAND=/bin/ls
 Feb 22 11:47:05 localhost groupadd[6991]: new group: name=apache, GID=48
 Feb 22 11:47:05 localhost useradd[6995]: new user: name=apache, UID=48, GID=48, home=/usr/share/httpd, shell=/sbin/nologin
+Jan 4 15:43:53 ubuntu-xenial sshd[2124]: Accepted publickey for vagrant from 10.0.2.2 port 64484 ssh2: RSA SHA256:sKpJLT3xGfw7fCcOslGbKVMyIm+2MbYVCqVkV5a1zeQ
+Jan 4 15:43:53 ubuntu-xenial sshd[2124]: pam_unix(sshd:session): session opened for user vagrant by (uid=0)
+Jan 4 15:43:53 ubuntu-xenial systemd: pam_unix(systemd-user:session): session opened for user vagrant by (uid=0)
+Jan 4 15:43:53 ubuntu-xenial systemd-logind[1055]: New session 2 of user vagrant.
+Jan 4 16:20:27 ubuntu-xenial sshd[9448]: Received disconnect from 10.0.2.2 port 50729:11: disconnected by user
+Jan 4 16:20:27 ubuntu-xenial sshd[9448]: Disconnected from 10.0.2.2 port 50729
+Jan 4 16:20:27 ubuntu-xenial sshd[9415]: pam_unix(sshd:session): session closed for user vagrant
+Jan 4 16:20:27 ubuntu-xenial systemd-logind[1055]: Removed session 14.
+Jan 4 15:44:27 ubuntu-xenial sudo: pam_unix(sudo:session): session opened for user root by vagrant(uid=0)
+Jan 4 15:44:27 ubuntu-xenial su[8849]: Successful su for root by root
+Jan 4 15:44:27 ubuntu-xenial su[8849]: + /dev/pts/1 root:root
+Jan 4 15:44:27 ubuntu-xenial su[8849]: pam_unix(su:session): session opened for user root by vagrant(uid=0)
+Jan 4 15:44:27 ubuntu-xenial su[8849]: pam_systemd(su:session): Cannot create session: Already running in a session
+Jan 4 16:04:14 ubuntu-xenial sshd[1061]: Received signal 15; terminating.
+Jan 4 16:04:14 ubuntu-xenial sshd[9230]: Server listening on 0.0.0.0 port 22.
+Jan 4 16:04:14 ubuntu-xenial sshd[9230]: Server listening on :: port 22.
+Jan 4 16:04:14 ubuntu-xenial sudo: pam_unix(sudo:session): session closed for user root
+Jan 4 16:09:22 ubuntu-xenial sshd[9356]: Invalid user mat from 10.0.2.2
+Jan 4 16:09:22 ubuntu-xenial sshd[9356]: input_userauth_request: invalid user mat [preauth]
+Jan 4 16:09:22 ubuntu-xenial sshd[9356]: Connection closed by 10.0.2.2 port 49945 [preauth]