Improve ECS field mappings in Sysmon module. (elastic#18381)

- related.hash, related.ip, and related.user are now populated. - hashes are now also populated to the corresponding process.hash, process.pe.imphash, file.hash or file.pe.imphash - file.name, file.directory, and file.extension are now populated. - rule.name is populated for all events when present. Closes elastic#18364
marc-gr · May 14, 2020 · 096b88e · 096b88e
1 parent 6cbd8cb
commit 096b88e
Show file tree

Hide file tree

Showing 5 changed files with 456 additions and 12 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -46,6 +46,10 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 *Winlogbeat*
 
 - Add support to Sysmon file delete events (event ID 23). {issue}18094[18094]
+- Improve ECS field mappings in Sysmon module. `related.hash`, `related.ip`, and `related.user` are now populated. {issue}18364[18364]
+- Improve ECS field mappings in Sysmon module. Hashes are now also populated to the corresponding `process.hash`, `process.pe.imphash`, `file.hash`, or `file.pe.imphash`. {issue}18364[18364]
+- Improve ECS field mappings in Sysmon module. `file.name`, `file.directory`, and `file.extension` are now populated. {issue}18364[18364]
+- Improve ECS field mappings in Sysmon module. `rule.name` is populated for all events when present. {issue}18364[18364]
 
 *Functionbeat*
 

diff --git a/x-pack/winlogbeat/module/sysmon/config/winlogbeat-sysmon.js b/x-pack/winlogbeat/module/sysmon/config/winlogbeat-sysmon.js
@@ -332,10 +332,21 @@ var sysmon = (function () {
             evt.Delete("user");
             evt.Put("user.domain", userParts[0]);
             evt.Put("user.name", userParts[1]);
+            evt.AppendTo("related.user", userParts[1]);
             evt.Delete("winlog.event_data.User");
         }
     };
 
+    var setRuleName = function (evt) {
+        var ruleName = evt.Get("winlog.event_data.RuleName");
+        if (!ruleName || ruleName === "-") {
+            return;
+        }
+
+        evt.Put("rule.name", ruleName);
+        evt.Delete("winlog.event_data.RuleName");
+    };
+
     var addNetworkDirection = function (evt) {
         switch (evt.Get("winlog.event_data.Initiated")) {
             case "true":
@@ -361,7 +372,39 @@ var sysmon = (function () {
         evt.Delete("winlog.event_data.DestinationIsIpv6");
     };
 
-    var addHashes = function (evt, hashField) {
+    var setRelatedIP = function (evt) {
+        var sourceIP = evt.Get("source.ip");
+        if (sourceIP) {
+            evt.AppendTo("related.ip", sourceIP);
+        }
+
+        var destIP = evt.Get("destination.ip");
+        if (destIP) {
+            evt.AppendTo("related.ip", destIP);
+        }
+    };
+
+    var getHashPath = function (namespace, hashKey) {
+        if (hashKey === "imphash") {
+            return namespace + ".pe.imphash";
+        }
+
+        return namespace + ".hash." + hashKey;
+    };
+
+    var emptyHashRegex = /^0*$/;
+
+    var hashIsEmpty = function (value) {
+        if (!value) {
+            return true;
+        }
+
+        return emptyHashRegex.test(value);
+    }
+
+    // Adds hashes from the given hashField in the event to the 'hash' key
+    // in the specified namespace. It also adds all the hashes to 'related.hash'.
+    var addHashes = function (evt, namespace, hashField) {
         var hashes = evt.Get(hashField);
         evt.Delete(hashField);
         hashes.split(",").forEach(function (hash) {
@@ -372,16 +415,31 @@ var sysmon = (function () {
 
             var key = parts[0].toLowerCase();
             var value = parts[1].toLowerCase();
+
+            if (hashIsEmpty(value)) {
+                return;
+            }
+
+            var path = getHashPath(namespace, key);
+
+            evt.Put(path, value);
+            evt.AppendTo("related.hash", value);
+
+            // TODO: remove in 8.0, see (https://github.com/elastic/beats/issues/18364).
             evt.Put("hash." + key, value);
         });
     };
 
-    var splitHashes = function (evt) {
-        addHashes(evt, "winlog.event_data.Hashes");
+    var splitFileHashes = function (evt) {
+        addHashes(evt, "file", "winlog.event_data.Hashes");
     };
 
-    var splitHash = function (evt) {
-        addHashes(evt, "winlog.event_data.Hash");
+    var splitFileHash = function (evt) {
+        addHashes(evt, "file", "winlog.event_data.Hash");
+    };
+
+    var splitProcessHashes = function (evt) {
+        addHashes(evt, "process", "winlog.event_data.Hashes");
     };
 
     var removeEmptyEventData = function (evt) {
@@ -477,6 +535,28 @@ var sysmon = (function () {
         evt.Put("file.code_signature.valid", signatureStatus === "Valid");
     };
 
+    var setAdditionalFileFieldsFromPath = function (evt) {
+        var filePath = evt.Get("file.path");
+        if (!filePath) {
+            return;
+        }
+
+        evt.Put("file.name", path.basename(filePath));
+        evt.Put("file.directory", path.dirname(filePath));
+
+        // path returns extensions with a preceding ., e.g.: .tmp, .png
+        // according to ecs the expected format is without it, so we need to remove it.
+        var ext = path.extname(filePath);
+        if (!ext) {
+            return;
+        }
+
+        if (ext.charAt(0) === ".") {
+            ext = ext.substr(1);
+        }
+        evt.Put("file.extension", ext);
+    };
+
     // https://docs.microsoft.com/en-us/windows/win32/sysinfo/registry-hives
     var commonRegistryHives = {
         HKEY_CLASSES_ROOT: "HKCR",
@@ -606,10 +686,11 @@ var sysmon = (function () {
             ignore_missing: true,
             fail_on_error: false,
         })
+        .Add(setRuleName)
         .Add(setProcessNameUsingExe)
         .Add(splitProcessArgs)
         .Add(addUser)
-        .Add(splitHashes)
+        .Add(splitProcessHashes)
         .Add(setParentProcessNameUsingExe)
         .Add(splitParentProcessArgs)
         .Add(removeEmptyEventData)
@@ -652,6 +733,8 @@ var sysmon = (function () {
             ignore_missing: true,
             fail_on_error: false,
         })
+        .Add(setRuleName)
+        .Add(setAdditionalFileFieldsFromPath)
         .Add(setProcessNameUsingExe)
         .Add(removeEmptyEventData)
         .Build();
@@ -727,6 +810,8 @@ var sysmon = (function () {
             ignore_missing: true,
             fail_on_error: false,
         })
+        .Add(setRuleName)
+        .Add(setRelatedIP)
         .Add(setProcessNameUsingExe)
         .Add(addUser)
         .Add(addNetworkDirection)
@@ -792,6 +877,7 @@ var sysmon = (function () {
             ignore_missing: true,
             fail_on_error: false,
         })
+        .Add(setRuleName)
         .Add(setProcessNameUsingExe)
         .Add(removeEmptyEventData)
         .Build();
@@ -833,8 +919,10 @@ var sysmon = (function () {
             ],
             fail_on_error: false,
         })
+        .Add(setRuleName)
+        .Add(setAdditionalFileFieldsFromPath)
         .Add(setAdditionalSignatureFields)
-        .Add(splitHashes)
+        .Add(splitFileHashes)
         .Add(removeEmptyEventData)
         .Build();
 
@@ -888,9 +976,11 @@ var sysmon = (function () {
             ],
             fail_on_error: false,
         })
+        .Add(setRuleName)
+        .Add(setAdditionalFileFieldsFromPath)
         .Add(setAdditionalSignatureFields)
         .Add(setProcessNameUsingExe)
-        .Add(splitHashes)
+        .Add(splitFileHashes)
         .Add(removeEmptyEventData)
         .Build();
 
@@ -921,6 +1011,7 @@ var sysmon = (function () {
             ignore_missing: true,
             fail_on_error: false,
         })
+        .Add(setRuleName)
         .Add(setProcessNameUsingExe)
         .Add(removeEmptyEventData)
         .Build();
@@ -956,6 +1047,8 @@ var sysmon = (function () {
             ignore_missing: true,
             fail_on_error: false,
         })
+        .Add(setRuleName)
+        .Add(setAdditionalFileFieldsFromPath)
         .Add(setProcessNameUsingExe)
         .Add(removeEmptyEventData)
         .Build();
@@ -998,6 +1091,7 @@ var sysmon = (function () {
             ignore_missing: true,
             fail_on_error: false,
         })
+        .Add(setRuleName)
         .Add(setProcessNameUsingExe)
         .Add(removeEmptyEventData)
         .Build();
@@ -1039,6 +1133,8 @@ var sysmon = (function () {
             ignore_missing: true,
             fail_on_error: false,
         })
+        .Add(setRuleName)
+        .Add(setAdditionalFileFieldsFromPath)
         .Add(setProcessNameUsingExe)
         .Add(removeEmptyEventData)
         .Build();
@@ -1070,6 +1166,7 @@ var sysmon = (function () {
             ignore_missing: true,
             fail_on_error: false,
         })
+        .Add(setRuleName)
         .Add(setRegistryFields)
         .Add(setProcessNameUsingExe)
         .Add(removeEmptyEventData)
@@ -1102,6 +1199,7 @@ var sysmon = (function () {
             ignore_missing: true,
             fail_on_error: false,
         })
+        .Add(setRuleName)
         .Add(setRegistryFields)
         .Add(setProcessNameUsingExe)
         .Add(removeEmptyEventData)
@@ -1134,6 +1232,7 @@ var sysmon = (function () {
             ignore_missing: true,
             fail_on_error: false,
         })
+        .Add(setRuleName)
         .Add(setRegistryFields)
         .Add(setProcessNameUsingExe)
         .Add(removeEmptyEventData)
@@ -1176,8 +1275,10 @@ var sysmon = (function () {
             ignore_missing: true,
             fail_on_error: false,
         })
+        .Add(setRuleName)
+        .Add(setAdditionalFileFieldsFromPath)
         .Add(setProcessNameUsingExe)
-        .Add(splitHash)
+        .Add(splitFileHash)
         .Add(removeEmptyEventData)
         .Build();
 
@@ -1235,6 +1336,7 @@ var sysmon = (function () {
             ignore_missing: true,
             fail_on_error: false,
         })
+        .Add(setRuleName)
         .Add(setProcessNameUsingExe)
         .Add(removeEmptyEventData)
         .Build();
@@ -1276,6 +1378,7 @@ var sysmon = (function () {
             ignore_missing: true,
             fail_on_error: false,
         })
+        .Add(setRuleName)
         .Add(setProcessNameUsingExe)
         .Add(removeEmptyEventData)
         .Build();
@@ -1294,6 +1397,7 @@ var sysmon = (function () {
             ignore_missing: true,
             fail_on_error: false,
         })
+        .Add(setRuleName)
         .Add(addUser)
         .Add(removeEmptyEventData)
         .Build();
@@ -1316,6 +1420,7 @@ var sysmon = (function () {
             ignore_missing: true,
             fail_on_error: false,
         })
+        .Add(setRuleName)
         .Add(addUser)
         .Add(setProcessNameUsingExe)
         .Add(removeEmptyEventData)
@@ -1335,6 +1440,7 @@ var sysmon = (function () {
             ignore_missing: true,
             fail_on_error: false,
         })
+        .Add(setRuleName)
         .Add(addUser)
         .Add(removeEmptyEventData)
         .Build();
@@ -1389,6 +1495,7 @@ var sysmon = (function () {
             field: "dns.question.name",
             target_field: "dns.question.registered_domain",
         })
+        .Add(setRuleName)
         .Add(translateDnsQueryStatus)
         .Add(splitDnsQueryResults)
         .Add(setProcessNameUsingExe)
@@ -1425,7 +1532,7 @@ var sysmon = (function () {
                 },
                 {
                     from: "winlog.event_data.TargetFilename",
-                    to: "file.name",
+                    to: "file.path",
                 },
                 {
                     from: "winlog.event_data.Image",
@@ -1446,9 +1553,11 @@ var sysmon = (function () {
             ignore_missing: true,
             fail_on_error: false,
         })
+        .Add(setRuleName)
         .Add(addUser)
-        .Add(splitHashes)
+        .Add(splitProcessHashes)
         .Add(setProcessNameUsingExe)
+        .Add(setAdditionalFileFieldsFromPath)
         .Add(removeEmptyEventData)
         .Build();
 

diff --git a/x-pack/winlogbeat/module/sysmon/test/testdata/sysmon-11-filedelete.evtx b/x-pack/winlogbeat/module/sysmon/test/testdata/sysmon-11-filedelete.evtx