Update mysqlenterprise module to ECS 1.8 (#23978)

Updates mysqlenterprise:

- Improve related.* field mapping.
- Populate event.original.
- Generate iam user creation and deletion from CREATE USER and DROP USER.

CHANGELOG.next.asciidoc

@@ -842,6 +842,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Upgrade panw module to ecs 1.8 {issue}23118[23118] {pull}23931[23931]
 - Updated aws/cloudtrail fileset to ECS 1.8. {issue}23118[23118] {pull}23911[23911]
 - Upgrade juniper/srx to ecs 1.8.0. {issue}23118[23118] {pull}23936[23936]
+- Update mysqlenterprise module to ECS 1.8. {issue}23118[23118] {pull}23978[23978]
 
 *Heartbeat*
 

x-pack/filebeat/module/mysqlenterprise/audit/config/config.yml

@@ -13,4 +13,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.6.0
+        ecs.version: 1.8.0

x-pack/filebeat/module/mysqlenterprise/audit/ingest/pipeline.yml

@@ -23,7 +23,7 @@ processors:
 - append:
     field: event.category
     value: iam
-    if: '["create_user", "delete_user", "grant", "flush_privileges"].contains(ctx.mysqlenterprise.audit?.general_data?.sql_command)'
+    if: '["create_user", "delete_user", "drop_user", "grant", "flush_privileges"].contains(ctx.mysqlenterprise.audit?.general_data?.sql_command)'
 - append:
     field: event.type
     value: access
@@ -128,6 +128,38 @@ processors:
       if (ctx.process.args.length > 0) {
         ctx.process.executable = ctx.process.args[0];
       }
+# Query parsing
+- grok:
+    field: mysqlenterprise.audit.general_data.query
+    if: '["create_user", "delete_user", "drop_user"].contains(ctx.mysqlenterprise?.audit?.general_data?.sql_command)'
+    ignore_failure: true
+    patterns:
+      - '(?i)(?:CREATE|DROP)\s+USER(?:\s+IF\s+(?:NOT\s+)?EXISTS)?\s+(?:%{START_QUOTE}%{QUOTED:user.target.name}%{END_QUOTE}|%{UNQUOTED:user.target.name})(?:@(?:%{START_QUOTE}%{QUOTED:user.target.domain}%{END_QUOTE}|%{UNQUOTED:user.target.domain}))?'
+    pattern_definitions:
+      START_QUOTE: (?<__quote>['"`])
+      QUOTED: (?~\k<__quote>)
+      END_QUOTE: (?:\k<__quote>)
+      UNQUOTED: (?:[^\s@;]*+)
+- remove:
+    field: __quote
+    ignore_missing: true
+- set:
+    field: user.name
+    value: '{{server.user.name}}'
+    ignore_empty_value: true
+    if: 'ctx.user?.target != null'
+- append:
+    field: event.type
+    value:
+      - user
+      - creation
+    if: 'ctx.mysqlenterprise?.audit?.general_data?.sql_command == "create_user"'
+- append:
+    field: event.type
+    value:
+      - user
+      - deletion
+    if: 'ctx.mysqlenterprise?.audit?.general_data?.sql_command == "drop_user" || ctx.mysqlenterprise?.audit?.general_data?.sql_command == "delete_user"'
 
 # Attributes starting with _ is only supported by MySQL 8.0.19 and above.
 - convert:
@@ -138,23 +170,39 @@ processors:
 - append:
     field: related.user
     value: '{{server.user.name}}'
+    allow_duplicates: false
     if: ctx?.server?.user?.name != null
 - append:
     field: related.user
     value: '{{client.user.name}}'
+    allow_duplicates: false
     if: ctx?.client?.user?.name != null
+- append:
+    field: related.user
+    value: '{{user.target.name}}'
+    allow_duplicates: false
+    if: ctx?.user?.target?.name != null
 - append:
     field: related.ip
     value: '{{client.ip}}'
+    allow_duplicates: false
     if: ctx?.client?.ip != null
+- append:
+    field: related.hosts
+    value: '{{client.domain}}'
+    allow_duplicates: false
+    if: ctx?.client?.domain != null
 - date:
     field: mysqlenterprise.audit.timestamp
     formats:
     - yyyy-MM-dd HH:mm:ss
     if: ctx?.mysqlenterprise?.audit?.timestamp != null
+- rename:
+    field: message
+    target_field: event.original
+    ignore_missing: true
 - remove:
     field:
-    - message
     - mysqlenterprise.audit.event
     - mysqlenterprise.audit.timestamp
     - mysqlenterprise.audit.connection_data.connection_attributes._pid

x-pack/filebeat/module/mysqlenterprise/audit/test/mysql_audit_test.log

@@ -9,7 +9,7 @@
 { "timestamp": "2020-10-19 19:28:27", "id": 0, "class": "general", "event": "status", "connection_id": 15, "account": { "user": "root", "host": "localhost" }, "login": { "user": "root", "os": "", "ip": "", "proxy": "" }, "general_data": { "command": "Query", "sql_command": "grant", "query": "GRANT ALL PRIVILEGES ON *.* TO 'root'@'hades.home' IDENTIFIED BY 'password'", "status": 1064 } },
 { "timestamp": "2020-10-19 19:28:54", "id": 0, "class": "general", "event": "status", "connection_id": 15, "account": { "user": "root", "host": "localhost" }, "login": { "user": "root", "os": "", "ip": "", "proxy": "" }, "general_data": { "command": "Query", "sql_command": "grant", "query": "GRANT ALL PRIVILEGES ON *.* TO 'root'@'%'", "status": 1410 } },
 { "timestamp": "2020-10-19 19:29:36", "id": 0, "class": "general", "event": "status", "connection_id": 15, "account": { "user": "root", "host": "localhost" }, "login": { "user": "root", "os": "", "ip": "", "proxy": "" }, "general_data": { "command": "Query", "sql_command": "create_user", "query": "CREATE USER 'audit_test_user'@'localhost' IDENTIFIED BY <secret>", "status": 1396 } },
-{ "timestamp": "2020-10-19 19:30:00", "id": 0, "class": "general", "event": "status", "connection_id": 15, "account": { "user": "root", "host": "localhost" }, "login": { "user": "root", "os": "", "ip": "", "proxy": "" }, "general_data": { "command": "Query", "sql_command": "create_user", "query": "CREATE USER 'audit_test_user2'@'hades.home' IDENTIFIED BY <secret>", "status": 0 } },
+{ "timestamp": "2020-10-19 19:30:00", "id": 0, "class": "general", "event": "status", "connection_id": 15, "account": { "user": "root", "host": "localhost" }, "login": { "user": "root", "os": "", "ip": "", "proxy": "" }, "general_data": { "command": "Query", "sql_command": "create_user", "query": "CREATE USER IF NOT EXISTS 'audit_test_user2'@'hades.home' IDENTIFIED BY <secret>", "status": 0 } },
 { "timestamp": "2020-10-19 19:30:18", "id": 0, "class": "general", "event": "status", "connection_id": 15, "account": { "user": "root", "host": "localhost" }, "login": { "user": "root", "os": "", "ip": "", "proxy": "" }, "general_data": { "command": "Query", "sql_command": "grant", "query": "GRANT ALL PRIVILEGES ON *.* TO ‘audit_test_user2’@’hades.home’", "status": 1410 } },
 { "timestamp": "2020-10-19 19:30:32", "id": 0, "class": "general", "event": "status", "connection_id": 15, "account": { "user": "root", "host": "localhost" }, "login": { "user": "root", "os": "", "ip": "", "proxy": "" }, "general_data": { "command": "Query", "sql_command": "grant", "query": "GRANT ALL PRIVILEGES ON *.* TO 'audit_test_user'@'hades.home'", "status": 1410 } },
 { "timestamp": "2020-10-19 19:30:49", "id": 0, "class": "general", "event": "status", "connection_id": 15, "account": { "user": "root", "host": "localhost" }, "login": { "user": "root", "os": "", "ip": "", "proxy": "" }, "general_data": { "command": "Query", "sql_command": "grant", "query": "GRANT ALL PRIVILEGES ON *.* TO 'audit_test_user'@'hades.home'", "status": 1410 } },
@@ -29,3 +29,6 @@
 { "timestamp": "2020-10-19 19:32:10", "id": 0, "class": "connection", "event": "disconnect", "connection_id": 16, "account": { "user": "audit_test_user2", "host": "hades.home" }, "login": { "user": "audit_test_user2", "os": "", "ip": "192.168.2.5", "proxy": "" }, "connection_data": { "connection_type": "ssl" } },
 { "timestamp": "2020-10-19 19:32:12", "id": 0, "class": "connection", "event": "disconnect", "connection_id": 15, "account": { "user": "root", "host": "localhost" }, "login": { "user": "root", "os": "", "ip": "", "proxy": "" }, "connection_data": { "connection_type": "socket" } },
 { "timestamp": "2020-10-19 19:32:16", "id": 0, "class": "audit", "event": "shutdown", "connection_id": 0, "shutdown_data": { "server_id": 1 } }
+{ "timestamp": "2021-02-10 19:05:42", "id": 2, "class": "audit", "event": "status", "connection_id": 42, "account": { "user": "adrian", "host": "elastic" }, "login": { "user": "adrian", "os": "", "ip": "192.168.7.76", "proxy": "" }, "general_data": { "command": "Query", "sql_command": "create_user", "query": "crEAtE  uSeR  'evil user'@elastic IDENTIFIED BY <secret>", "status": 1396 } },
+{ "timestamp": "2021-02-10 19:05:42", "id": 2, "class": "audit", "event": "status", "connection_id": 42, "account": { "user": "adrian", "host": "elastic" }, "login": { "user": "evil user", "os": "", "ip": "192.168.7.76", "proxy": "" }, "general_data": { "command": "Query", "sql_command": "drop_db", "query": "DROP DATABASE prod", "status": 1396 } },
+{ "timestamp": "2021-02-10 19:05:42", "id": 2, "class": "audit", "event": "status", "connection_id": 42, "account": { "user": "adrian", "host": "elastic" }, "login": { "user": "evil user", "os": "", "ip": "192.168.7.76", "proxy": "" }, "general_data": { "command": "Query", "sql_command": "drop_user", "query": "DrOp usEr IF EXISTS 'evil user'@%", "status": 1396 } },