Convert Filebeat nginx.error to ECS

CHANGELOG.next.asciidoc

@@ -27,6 +27,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Rename many `traefik.access.*` fields to map to ECS. {pull}9005[9005]
 - Fix parsing of GC entries in elasticsearch server log. {issue}9513[9513] {pull}9810[9810]
 - Rename a few `logstash.*` fields to map to ECS, remove logstash.slowlog.message. {pull}9935[9935]
+- Rename a few `nginx.error.*` fields to map to ECS. {pull}10007[10007]
 
 *Heartbeat*
 

dev-tools/ecs-migration.yml

@@ -510,6 +510,22 @@
   to: user_agent.original
   alias: true
 
+- from: nginx.error.level
+  to: log.level
+  alias: true
+
+- from: nginx.error.pid
+  to: process.pid
+  alias: true
+
+- from: nginx.error.tid
+  to: process.thread.id
+  alias: true
+
+- from: nginx.error.message
+  to: message
+  alias: true
+
 ## PostgreSQL module
 
 - from: postgresql.log.timezone

filebeat/docs/fields.asciidoc

@@ -9937,53 +9937,49 @@ Contains fields for the Nginx error logs.
 
 
 
-*`nginx.error.level`*::
+*`nginx.error.connection_id`*::
 +
 --
-type: keyword
+type: long
 
-Error level (e.g. error, critical).
+Connection identifier.
 
 
 --
 
-*`nginx.error.pid`*::
+*`nginx.error.level`*::
 +
 --
-type: long
-
-Process identifier (PID).
+type: alias
 
+alias to: log.level
 
 --
 
-*`nginx.error.tid`*::
+*`nginx.error.pid`*::
 +
 --
-type: long
-
-Thread identifier.
+type: alias
 
+alias to: process.pid
 
 --
 
-*`nginx.error.connection_id`*::
+*`nginx.error.tid`*::
 +
 --
-type: long
-
-Connection identifier.
+type: alias
 
+alias to: process.thread.id
 
 --
 
 *`nginx.error.message`*::
 +
 --
-type: text
-
-The error message
+type: alias
 
+alias to: message
 
 --
 

filebeat/module/nginx/error/_meta/fields.yml

@@ -3,23 +3,24 @@
   description: >
     Contains fields for the Nginx error logs.
   fields:
-    - name: level
-      type: keyword
-      description: >
-        Error level (e.g. error, critical).
-    - name: pid
-      type: long
-      description: >
-        Process identifier (PID).
-    - name: tid
-      type: long
-      description: >
-        Thread identifier.
     - name: connection_id
       type: long
       description: >
         Connection identifier.
+
+    - name: level
+      type: alias
+      path: log.level
+      migration: true
+    - name: pid
+      type: alias
+      path: process.pid
+      migration: true
+    - name: tid
+      type: alias
+      path: process.thread.id
+      migration: true
     - name: message
-      type: text
-      description: >
-        The error message
+      type: alias
+      path: message
+      migration: true

filebeat/module/nginx/error/ingest/pipeline.json

@@ -4,18 +4,14 @@
     "grok": {
       "field": "message",
       "patterns": [
-        "%{DATA:nginx.error.time} \\[%{DATA:nginx.error.level}\\] %{NUMBER:nginx.error.pid}#%{NUMBER:nginx.error.tid}: (\\*%{NUMBER:nginx.error.connection_id} )?%{GREEDYDATA:nginx.error.message}"
+        "%{DATA:nginx.error.time} \\[%{DATA:log.level}\\] %{NUMBER:process.pid:long}#%{NUMBER:process.thread.id:long}: (\\*%{NUMBER:nginx.error.connection_id:long} )?%{GREEDYDATA:message}"
       ],
       "ignore_missing": true
     }
-  },{
-    "remove":{
-      "field": "message"
-    }
   }, {
     "rename": {
       "field": "@timestamp",
-      "target_field": "read_timestamp"
+      "target_field": "event.created"
     }
   }, {
     "date": {

filebeat/module/nginx/error/test/error.log-expected.json

@@ -6,12 +6,12 @@
         "event.module": "nginx",
         "fileset.name": "error",
         "input.type": "log",
+        "log.level": "error",
         "log.offset": 0,
-        "nginx.error.connection_id": "1",
-        "nginx.error.level": "error",
-        "nginx.error.message": "open() \"/usr/local/Cellar/nginx/1.10.2_1/html/favicon.ico\" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: \"GET /favicon.ico HTTP/1.1\", host: \"localhost:8080\", referrer: \"http://localhost:8080/\"",
-        "nginx.error.pid": "54053",
-        "nginx.error.tid": "0"
+        "message": "open() \"/usr/local/Cellar/nginx/1.10.2_1/html/favicon.ico\" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: \"GET /favicon.ico HTTP/1.1\", host: \"localhost:8080\", referrer: \"http://localhost:8080/\"",
+        "nginx.error.connection_id": 1,
+        "process.pid": 54053,
+        "process.thread.id": 0
     },
     {
         "@timestamp": "2016-10-25T14:50:44.000Z",
@@ -20,11 +20,11 @@
         "event.module": "nginx",
         "fileset.name": "error",
         "input.type": "log",
+        "log.level": "error",
         "log.offset": 273,
-        "nginx.error.connection_id": "3",
-        "nginx.error.level": "error",
-        "nginx.error.message": "open() \"/usr/local/Cellar/nginx/1.10.2_1/html/adsasd\" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: \"GET /adsasd HTTP/1.1\", host: \"localhost:8080\"",
-        "nginx.error.pid": "54053",
-        "nginx.error.tid": "0"
+        "message": "open() \"/usr/local/Cellar/nginx/1.10.2_1/html/adsasd\" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: \"GET /adsasd HTTP/1.1\", host: \"localhost:8080\"",
+        "nginx.error.connection_id": 3,
+        "process.pid": 54053,
+        "process.thread.id": 0
     }
 ]