Adding categorization fields for the system/auth module (elastic#11334)

* Adding categorization fields for the system/auth module

This PR adds the following fields for the SSH login events:

* `event.category: authentication`
* `event.action: ssh_login`
* `event.type` either `authentication_success` or `authentication_failure`

The `event.outcome` is currently not quite ECS compliant, but I didn't touch it to
avoid a breaking change.

The PR doesn't attempt to categorize other logs besides the SSH login attempts,
so it's a subset of elastic#9905, but it's what we need for the UI.

* Normalized event.outcome and brought back `system.auth.ssh.event`.

* changelog

CHANGELOG.next.asciidoc

@@ -174,7 +174,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add on_failure handler for Zeek ingest pipelines. Fix one field name error for notice and add an additional test case. {issue}11004[11004] {pull}11105[11105]
 - Fix goroutine leak happening when harvesters are dynamically stopped. {pull}11263[11263]
 - Fix issue preventing docker container events to be stored if the container has a network interface without ip address. {issue}11225[11225] {pull}11247[11247]
-- Add on_failure handler for Zeek ingest pipelines. Fix one field name error for notice and add an additional test 
+- Add on_failure handler for Zeek ingest pipelines. Fix one field name error for notice and add an additional test
  case. {issue}11004[11004] {pull}11105[11105]
 - Change URLPATH grok pattern to support brackets. {issue}11135[11135] {pull}11252[11252]
 - Add support for iis log with different address format. {issue}11255[11255] {pull}11256[11256]
@@ -300,6 +300,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Change Suricata module pipeline to handle `destination.domain` being set if a reverse DNS processor is used. {issue}10510[10510]
 - Add the `network.community_id` flow identifier to field to the IPTables, Suricata, and Zeek modules. {pull}11005[11005]
 - Add support for loading custom NetFlow and IPFIX field definitions to netflow input. {pull}10945[10945]
+- Added categorization fields for SSH login events in the system/auth fileset. {pull}11334[11334]
 
 *Heartbeat*
 

dev-tools/ecs-migration.yml

@@ -486,11 +486,6 @@
  alias: true
  beat: filebeat
 
-- from: system.auth.ssh.event
- to: event.action
- alias: true
- beat: filebeat
-
 - from: system.auth.program
  to: process.name
  alias: true

filebeat/docs/fields.asciidoc

@@ -13261,9 +13261,10 @@ The client IP from SSH connections that are open and immediately dropped.
 *`system.auth.ssh.event`*::
 +
 --
-type: alias
+example: Accepted
+
+The SSH event as found in the logs (Accepted, Invalid, Failed, etc.)
 
-alias to: event.action
 
 --
 

filebeat/module/system/auth/_meta/fields.yml

@@ -43,9 +43,10 @@
  The client IP from SSH connections that are open and immediately dropped.
 
  - name: event
- type: alias
- path: event.action
- migration: true
+ example: Accepted
+ description: >
+ The SSH event as found in the logs (Accepted, Invalid, Failed, etc.)
+
  - name: ip
  type: alias
  path: source.ip

filebeat/module/system/auth/ingest/pipeline.json

@@ -9,8 +9,8 @@
  "GREEDYMULTILINE" : "(.|\n)*"
  },
  "patterns": [
- "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: %{DATA:event.outcome} %{DATA:system.auth.ssh.method} for (invalid user )?%{DATA:user.name} from %{IPORHOST:source.ip} port %{NUMBER:source.port:long} ssh2(: %{GREEDYDATA:system.auth.ssh.signature})?",
- "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: %{DATA:event.outcome} user %{DATA:user.name} from %{IPORHOST:source.ip}",
+ "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: %{DATA:system.auth.ssh.event} %{DATA:system.auth.ssh.method} for (invalid user )?%{DATA:user.name} from %{IPORHOST:source.ip} port %{NUMBER:source.port:long} ssh2(: %{GREEDYDATA:system.auth.ssh.signature})?",
+ "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: %{DATA:system.auth.ssh.event} user %{DATA:user.name} from %{IPORHOST:source.ip}",
  "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: Did not receive identification string from %{IPORHOST:system.auth.ssh.dropped_ip}",
  "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: \\s*%{DATA:user.name} :( %{DATA:system.auth.sudo.error} ;)? TTY=%{DATA:system.auth.sudo.tty} ; PWD=%{DATA:system.auth.sudo.pwd} ; USER=%{DATA:system.auth.sudo.user} ; COMMAND=%{GREEDYDATA:system.auth.sudo.command}",
  "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: new group: name=%{DATA:group.name}, GID=%{NUMBER:group.id}",
@@ -61,6 +61,13 @@
  "target_field": "source.geo",
  "ignore_failure": true
  }
+ },
+ {
+ "script": {
+ "lang": "painless",
+ "ignore_failure": true,
+ "source": "if (ctx.system.auth.ssh.event == \"Accepted\") { if (!ctx.containsKey(\"event\")) { ctx.event = [:]; } ctx.event.type = \"authentication_success\"; ctx.event.category = \"authentication\"; ctx.event.action = \"ssh_login\"; ctx.event.outcome = \"success\"; } else if (ctx.system.auth.ssh.event == \"Invalid\" || ctx.system.auth.ssh.event == \"Failed\") { if (!ctx.containsKey(\"event\")) { ctx.event = [:]; } ctx.event.type = \"authentication_failure\"; ctx.event.category = \"authentication\"; ctx.event.action = \"ssh_login\"; ctx.event.outcome = \"failure\"; }"
+ }
  }
  ],
  "on_failure" : [{

filebeat/module/system/auth/test/test.log-expected.json

@@ -3,7 +3,11 @@
  "ecs.version": "1.0.0",
  "event.dataset": "system.auth",
  "event.module": "system",
- "event.outcome": "Accepted",
+ "system.auth.ssh.event": "Accepted",
+ "event.outcome": "success",
+ "event.category": "authentication",
+ "event.action": "ssh_login",
+ "event.type": "authentication_success",
  "fileset.name": "auth",
  "host.hostname": "localhost",
  "input.type": "log",
@@ -21,7 +25,11 @@
  "ecs.version": "1.0.0",
  "event.dataset": "system.auth",
  "event.module": "system",
- "event.outcome": "Accepted",
+ "system.auth.ssh.event": "Accepted",
+ "event.outcome": "success",
+ "event.category": "authentication",
+ "event.action": "ssh_login",
+ "event.type": "authentication_success",
  "fileset.name": "auth",
  "host.hostname": "localhost",
  "input.type": "log",
@@ -38,7 +46,11 @@
  "ecs.version": "1.0.0",
  "event.dataset": "system.auth",
  "event.module": "system",
- "event.outcome": "Invalid",
+ "system.auth.ssh.event": "Invalid",
+ "event.outcome": "failure",
+ "event.category": "authentication",
+ "event.action": "ssh_login",
+ "event.type": "authentication_failure",
  "fileset.name": "auth",
  "host.hostname": "localhost",
  "input.type": "log",
@@ -53,7 +65,11 @@
  "ecs.version": "1.0.0",
  "event.dataset": "system.auth",
  "event.module": "system",
- "event.outcome": "Failed",
+ "system.auth.ssh.event": "Failed",
+ "event.outcome": "failure",
+ "event.category": "authentication",
+ "event.action": "ssh_login",
+ "event.type": "authentication_failure",
  "fileset.name": "auth",
  "host.hostname": "slave22",
  "input.type": "log",
@@ -170,4 +186,4 @@
  "user.id": "48",
  "user.name": "apache"
  }
-]
+]