[Filebeat] Add ECS tls & categorization fields to apache module (#16121…

…) (#16200) * Add ECS tls & categorization fields to apache module - tls.cipher (access) - tls.protocol (access) - tls.protocol_version (access) - event.kind (access) - event.category (access) - event.outcome (access) - lowercase http.request.method for ECS compliance (access) - event.kind (error) - event.category (error) - event.type (error) Closes #16032 (cherry picked from commit 990a5f7)
elastic · Feb 10, 2020 · a8a3208 · a8a3208
1 parent 3a0fc0b
commit a8a3208
Show file tree

Hide file tree

Showing 15 changed files with 327 additions and 236 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -186,6 +186,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add dashboard for AWS vpcflow fileset. {pull}16007[16007]
 - Add ECS tls fields to zeek:smtp,rdp,ssl and aws:s3access,elb {issue}15757[15757] {pull}15935[15936]
 - move create-[module,fileset,fields] to mage and enable in x-pack/filebeat {pull}15836[15836]
+- Add ECS tls and categorization fields to apache module. {issue}16032[16032] {pull}16121[16121]
 
 *Heartbeat*
 

diff --git a/filebeat/module/apache/access/ingest/default.json b/filebeat/module/apache/access/ingest/default.json
diff --git a/filebeat/module/apache/access/ingest/pipeline.yml b/filebeat/module/apache/access/ingest/pipeline.yml
@@ -0,0 +1,103 @@
+description: "Pipeline for parsing Apache HTTP Server access logs. Requires the geoip and user_agent plugins."
+
+processors:
+- grok:
+    field: message
+    patterns:
+    - '%{IPORHOST:destination.domain} %{IPORHOST:source.ip} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\]
+      "(?:%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}|-)?"
+      %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)(
+      "%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?'
+    - '%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\]
+      "(?:%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}|-)?"
+      %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)(
+      "%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?'
+    - '%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\]
+      "-" %{NUMBER:http.response.status_code:long} -'
+    - \[%{HTTPDATE:apache.access.time}\] %{IPORHOST:source.address} %{DATA:apache.access.ssl.protocol}
+      %{DATA:apache.access.ssl.cipher} "%{WORD:http.request.method} %{DATA:url.original}
+      HTTP/%{NUMBER:http.version}" (-|%{NUMBER:http.response.body.bytes:long})
+    ignore_missing: true
+- remove:
+    field: message
+- set:
+    field: event.kind
+    value: event
+- set:
+    field: event.category
+    value: web
+- set:
+    field: event.outcome
+    value: success
+    if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400"
+- set:
+    field: event.outcome
+    value: failure
+    if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code > 399"
+- lowercase:
+    field: http.request.method
+    ignore_missing: true
+- grok:
+    field: source.address
+    ignore_missing: true
+    patterns:
+    - ^(%{IP:source.ip}|%{HOSTNAME:source.domain})$
+- rename:
+    field: '@timestamp'
+    target_field: event.created
+- date:
+    field: apache.access.time
+    target_field: '@timestamp'
+    formats:
+    - dd/MMM/yyyy:H:m:s Z
+    ignore_failure: true
+- remove:
+    field: apache.access.time
+    ignore_failure: true
+- user_agent:
+    field: user_agent.original
+    ignore_failure: true
+- geoip:
+    field: source.ip
+    target_field: source.geo
+    ignore_missing: true
+- geoip:
+    database_file: GeoLite2-ASN.mmdb
+    field: source.ip
+    target_field: source.as
+    properties:
+    - asn
+    - organization_name
+    ignore_missing: true
+- rename:
+    field: source.as.asn
+    target_field: source.as.number
+    ignore_missing: true
+- rename:
+    field: source.as.organization_name
+    target_field: source.as.organization.name
+    ignore_missing: true
+- set:
+    field: tls.cipher
+    value: '{{apache.access.ssl.cipher}}'
+    if: ctx?.apache?.access?.ssl?.cipher != null
+
+- script:
+    lang: painless
+    if: ctx?.apache?.access?.ssl?.protocol != null
+    source: >-
+      def parts = ctx.apache.access.ssl.protocol.toLowerCase().splitOnToken("v");
+      if (parts.length != 2) {
+        return;
+      }
+      if (parts[1].contains(".")) {
+        ctx.tls.version = parts[1];
+      } else {
+        ctx.tls.version = parts[1] + ".0";
+      }
+      ctx.tls.version_protocol = parts[0];
+
+on_failure:
+- set:
+    field: error.message
+    value: '{{ _ingest.on_failure_message }}'
diff --git a/filebeat/module/apache/access/manifest.yml b/filebeat/module/apache/access/manifest.yml
@@ -12,7 +12,7 @@ var:
       - "C:/tools/Apache/httpd-2.*/Apache24/logs/access.log*"
       - "C:/Program Files/Apache Software Foundation/Apache2.*/logs/access.log*"
 
-ingest_pipeline: ingest/default.json
+ingest_pipeline: ingest/pipeline.yml
 input: config/access.yml
 
 requires.processors:

diff --git a/filebeat/module/apache/access/test/darwin-2.4.23.log-expected.json b/filebeat/module/apache/access/test/darwin-2.4.23.log-expected.json
@@ -1,10 +1,13 @@
 [
     {
         "@timestamp": "2016-12-26T14:16:28.000Z",
+        "event.category": "web",
         "event.dataset": "apache.access",
+        "event.kind": "event",
         "event.module": "apache",
+        "event.outcome": "success",
         "fileset.name": "access",
-        "http.request.method": "GET",
+        "http.request.method": "get",
         "http.response.body.bytes": 45,
         "http.response.status_code": 200,
         "http.version": "1.1",
@@ -18,10 +21,13 @@
     },
     {
         "@timestamp": "2016-12-26T14:16:29.000Z",
+        "event.category": "web",
         "event.dataset": "apache.access",
+        "event.kind": "event",
         "event.module": "apache",
+        "event.outcome": "failure",
         "fileset.name": "access",
-        "http.request.method": "GET",
+        "http.request.method": "get",
         "http.response.body.bytes": 209,
         "http.response.status_code": 404,
         "http.version": "1.1",
@@ -35,8 +41,11 @@
     },
     {
         "@timestamp": "2016-12-26T14:16:48.000Z",
+        "event.category": "web",
         "event.dataset": "apache.access",
+        "event.kind": "event",
         "event.module": "apache",
+        "event.outcome": "failure",
         "fileset.name": "access",
         "http.response.status_code": 408,
         "input.type": "log",
@@ -48,10 +57,13 @@
     },
     {
         "@timestamp": "2016-12-26T16:23:35.000Z",
+        "event.category": "web",
         "event.dataset": "apache.access",
+        "event.kind": "event",
         "event.module": "apache",
+        "event.outcome": "success",
         "fileset.name": "access",
-        "http.request.method": "GET",
+        "http.request.method": "get",
         "http.response.body.bytes": 45,
         "http.response.status_code": 200,
         "http.version": "1.1",
@@ -74,10 +86,13 @@
     },
     {
         "@timestamp": "2016-12-26T16:23:41.000Z",
+        "event.category": "web",
         "event.dataset": "apache.access",
+        "event.kind": "event",
         "event.module": "apache",
+        "event.outcome": "failure",
         "fileset.name": "access",
-        "http.request.method": "GET",
+        "http.request.method": "get",
         "http.response.body.bytes": 206,
         "http.response.status_code": 404,
         "http.version": "1.1",
@@ -100,10 +115,13 @@
     },
     {
         "@timestamp": "2016-12-26T16:23:45.000Z",
+        "event.category": "web",
         "event.dataset": "apache.access",
+        "event.kind": "event",
         "event.module": "apache",
+        "event.outcome": "failure",
         "fileset.name": "access",
-        "http.request.method": "GET",
+        "http.request.method": "get",
         "http.response.body.bytes": 201,
         "http.response.status_code": 404,
         "http.version": "1.1",

diff --git a/filebeat/module/apache/access/test/ssl-request.log-expected.json b/filebeat/module/apache/access/test/ssl-request.log-expected.json
@@ -3,27 +3,34 @@
         "@timestamp": "2018-08-10T07:45:56.000Z",
         "apache.access.ssl.cipher": "ECDHE-RSA-AES128-GCM-SHA256",
         "apache.access.ssl.protocol": "TLSv1.2",
+        "event.category": "web",
         "event.dataset": "apache.access",
+        "event.kind": "event",
         "event.module": "apache",
         "fileset.name": "access",
-        "http.request.method": "GET",
+        "http.request.method": "get",
         "http.response.body.bytes": 1375,
         "http.version": "1.1",
         "input.type": "log",
         "log.offset": 0,
         "service.type": "apache",
         "source.address": "172.30.0.119",
         "source.ip": "172.30.0.119",
+        "tls.cipher": "ECDHE-RSA-AES128-GCM-SHA256",
+        "tls.version": "1.2",
+        "tls.version_protocol": "tls",
         "url.original": "/nagiosxi/ajaxhelper.php?cmd=getxicoreajax&amp;opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D&amp;nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21"
     },
     {
         "@timestamp": "2019-10-16T09:53:47.000Z",
         "apache.access.ssl.cipher": "ECDHE-RSA-AES128-GCM-SHA256",
         "apache.access.ssl.protocol": "TLSv1.2",
+        "event.category": "web",
         "event.dataset": "apache.access",
+        "event.kind": "event",
         "event.module": "apache",
         "fileset.name": "access",
-        "http.request.method": "GET",
+        "http.request.method": "get",
         "http.version": "1.1",
         "input.type": "log",
         "log.offset": 276,
@@ -34,6 +41,9 @@
         "source.geo.location.lat": 37.751,
         "source.geo.location.lon": -97.822,
         "source.ip": "11.19.0.217",
+        "tls.cipher": "ECDHE-RSA-AES128-GCM-SHA256",
+        "tls.version": "1.2",
+        "tls.version_protocol": "tls",
         "url.original": "/appl/ajaxhelper.php?cmd=getxicoreajax&opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D&nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d"
     }
 ]
diff --git a/filebeat/module/apache/access/test/test-vhost.log-expected.json b/filebeat/module/apache/access/test/test-vhost.log-expected.json
@@ -2,10 +2,13 @@
     {
         "@timestamp": "2016-12-26T16:22:14.000Z",
         "destination.domain": "vhost1.domaine.fr",
+        "event.category": "web",
         "event.dataset": "apache.access",
+        "event.kind": "event",
         "event.module": "apache",
+        "event.outcome": "failure",
         "fileset.name": "access",
-        "http.request.method": "GET",
+        "http.request.method": "get",
         "http.request.referrer": "-",
         "http.response.body.bytes": 499,
         "http.response.status_code": 404,