Convert Filebeat postgresql.log to ECS (#9308)

- Map Postgresql module fields to ECS:
  - postgresql.log.level => log.level
  - postgresql.log.user => user.name
  - postgresql.log.thread_id => process.pid
  - postgresql.log.timezone => event.timezone
  - postgresql.log.message => message
- Coerce to int: pid, core_id
- Coerce duration to float
- Populate `event.duration` (ns) based on `postgresql.log.duration` (ms)

CHANGELOG.asciidoc

@@ -181,6 +181,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha1...v7.0.0-alpha2[Check the
 - Use `log.source.address` instead of `log.source.ip` for network input sources. {pull}9487[9487]
 - Rename many `redis.log.*` fields to map to ECS. {pull}9315[9315]
 - Rename many `icinga.*` fields to map to ECS. {pull}9294[9294]
+- Rename many `postgresql.log.*` fields to map to ECS. {pull}9303[9303]
 
 *Metricbeat*
 

dev-tools/ecs-migration.yml

@@ -434,6 +434,28 @@
   to: user_agent.original
   alias: true
 
+## PostgreSQL module
+
+- from: postgresql.log.timezone
+  to: event.timezone
+  alias: true
+
+- from: postgresql.log.thread_id
+  to: process.pid
+  alias: true
+
+- from: postgresql.log.user
+  to: user.name
+  alias: true
+
+- from: postgresql.log.level
+  to: log.level
+  alias: true
+
+- from: postgresql.log.message
+  to: message
+  alias: true
+
 ## Redis module
 
 - from: redis.log.pid

filebeat/docs/fields.asciidoc

@@ -9892,88 +9892,87 @@ The timestamp from the log line.
 
 --
 
-*`postgresql.log.timezone`*::
+*`postgresql.log.core_id`*::
 +
 --
-The timezone of timestamp.
+type: long
+
+Core id
 
 
 --
 
-*`postgresql.log.thread_id`*::
+*`postgresql.log.database`*::
 +
 --
-type: long
-
-Process id
+example: mydb
 
+Name of database
 
 --
 
-*`postgresql.log.core_id`*::
+*`postgresql.log.duration`*::
 +
 --
-type: long
+type: float
 
-Core id
+example: 30.0
 
+Duration of a query.
 
 --
 
-*`postgresql.log.user`*::
+*`postgresql.log.query`*::
 +
 --
-example: admin
+example: SELECT * FROM users;
 
-Name of user
+Query statement.
 
 --
 
-*`postgresql.log.database`*::
+*`postgresql.log.timezone`*::
 +
 --
-example: mydb
+type: alias
 
-Name of database
+alias to: event.timezone
 
 --
 
-*`postgresql.log.level`*::
+*`postgresql.log.thread_id`*::
 +
 --
-example: FATAL
+type: alias
 
-The log level.
+alias to: process.pid
 
 --
 
-*`postgresql.log.duration`*::
+*`postgresql.log.user`*::
 +
 --
-type: float
-
-example: 30.0
+type: alias
 
-Duration of a query.
+alias to: user.name
 
 --
 
-*`postgresql.log.query`*::
+*`postgresql.log.level`*::
 +
 --
-example: SELECT * FROM users;
+type: alias
 
-Query statement.
+alias to: log.level
 
 --
 
 *`postgresql.log.message`*::
 +
 --
-type: text
-
-The logged message.
+type: alias
 
+alias to: message
 
 --
 

filebeat/module/postgresql/log/_meta/fields.yml

@@ -6,29 +6,14 @@
     - name: timestamp
       description: >
         The timestamp from the log line.
-    - name: timezone
-      description: >
-        The timezone of timestamp.
-    - name: thread_id
-      type: long
-      description: >
-        Process id
     - name: core_id
       type: long
       description: >
         Core id
-    - name: user
-      example: "admin"
-      description:
-        Name of user
     - name: database
       example: "mydb"
       description:
         Name of database
-    - name: level
-      example: "FATAL"
-      description:
-        The log level.
     - name: duration
       type: float
       example: "30.0"
@@ -38,7 +23,24 @@
       example: "SELECT * FROM users;"
       description:
         Query statement.
+
+    - name: timezone
+      type: alias
+      path: event.timezone
+      migration: true
+    - name: thread_id
+      type: alias
+      path: process.pid
+      migration: true
+    - name: user
+      type: alias
+      path: user.name
+      migration: true
+    - name: level
+      type: alias
+      path: log.level
+      migration: true
     - name: message
-      type: text
-      description: >
-        The logged message.
+      type: alias
+      path: message
+      migration: true

filebeat/module/postgresql/log/ingest/pipeline.json

@@ -6,7 +6,7 @@
         "field": "message",
         "ignore_missing": true,
         "patterns": [
-            "^%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:postgresql.log.timezone} \\[%{NUMBER:postgresql.log.thread_id}(-%{BASE16FLOAT:postgresql.log.core_id})?\\] ((\\[%{USERNAME:postgresql.log.user}\\]@\\[%{POSTGRESQL_DB_NAME:postgresql.log.database}\\]|%{USERNAME:postgresql.log.user}@%{POSTGRESQL_DB_NAME:postgresql.log.database}) )?%{WORD:postgresql.log.level}:  (duration: %{NUMBER:postgresql.log.duration} ms  statement: %{GREEDYDATA:postgresql.log.query}|%{GREEDYDATA:postgresql.log.message})"
+            "^%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:event.timezone} \\[%{NUMBER:process.pid:long}(-%{BASE16FLOAT:postgresql.log.core_id:long})?\\] ((\\[%{USERNAME:user.name}\\]@\\[%{POSTGRESQL_DB_NAME:postgresql.log.database}\\]|%{USERNAME:user.name}@%{POSTGRESQL_DB_NAME:postgresql.log.database}) )?%{WORD:log.level}:  (duration: %{NUMBER:postgresql.log.duration:float} ms  statement: %{GREEDYDATA:postgresql.log.query}|%{GREEDYDATA:message})"
         ],
         "pattern_definitions": {
           "LOCALDATETIME": "[-0-9]+ %{TIME}",
@@ -24,6 +24,14 @@
         ],
         "ignore_failure": true
       }
+    },
+    {
+      "script": {
+          "lang": "painless",
+          "source": "ctx.event.duration = Math.round(ctx.postgresql.log.duration * params.scale)",
+          "params": { "scale": 1000000 },
+          "if": "ctx.postgresql.log.containsKey('duration')"
+      }
     }
   ],
   "on_failure": [