Improve ECS field mappings in Sysmon module. (#18381) (#18519)

- related.hash, related.ip, and related.user are now populated. - hashes are now also populated to the corresponding process.hash, process.pe.imphash, file.hash or file.pe.imphash - file.name, file.directory, and file.extension are now populated. - rule.name is populated for all events when present. Closes #18364 (cherry picked from commit 096b88e)
elastic · May 15, 2020 · b68cde3 · b68cde3
1 parent fb9a22e
commit b68cde3
Show file tree

Hide file tree

Showing 5 changed files with 456 additions and 12 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -63,6 +63,10 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 *Winlogbeat*
 
 - Add support to Sysmon file delete events (event ID 23). {issue}18094[18094]
+- Improve ECS field mappings in Sysmon module. `related.hash`, `related.ip`, and `related.user` are now populated. {issue}18364[18364]
+- Improve ECS field mappings in Sysmon module. Hashes are now also populated to the corresponding `process.hash`, `process.pe.imphash`, `file.hash`, or `file.pe.imphash`. {issue}18364[18364]
+- Improve ECS field mappings in Sysmon module. `file.name`, `file.directory`, and `file.extension` are now populated. {issue}18364[18364]
+- Improve ECS field mappings in Sysmon module. `rule.name` is populated for all events when present. {issue}18364[18364]
 
 *Functionbeat*
 

diff --git a/x-pack/winlogbeat/module/sysmon/config/winlogbeat-sysmon.js b/x-pack/winlogbeat/module/sysmon/config/winlogbeat-sysmon.js
@@ -332,10 +332,21 @@ var sysmon = (function () {
  evt.Delete("user");
  evt.Put("user.domain", userParts[0]);
  evt.Put("user.name", userParts[1]);
+ evt.AppendTo("related.user", userParts[1]);
  evt.Delete("winlog.event_data.User");
  }
  };
 
+ var setRuleName = function (evt) {
+ var ruleName = evt.Get("winlog.event_data.RuleName");
+ if (!ruleName || ruleName === "-") {
+ return;
+ }
+
+ evt.Put("rule.name", ruleName);
+ evt.Delete("winlog.event_data.RuleName");
+ };
+
  var addNetworkDirection = function (evt) {
  switch (evt.Get("winlog.event_data.Initiated")) {
  case "true":
@@ -361,7 +372,39 @@ var sysmon = (function () {
  evt.Delete("winlog.event_data.DestinationIsIpv6");
  };
 
- var addHashes = function (evt, hashField) {
+ var setRelatedIP = function (evt) {
+ var sourceIP = evt.Get("source.ip");
+ if (sourceIP) {
+ evt.AppendTo("related.ip", sourceIP);
+ }
+
+ var destIP = evt.Get("destination.ip");
+ if (destIP) {
+ evt.AppendTo("related.ip", destIP);
+ }
+ };
+
+ var getHashPath = function (namespace, hashKey) {
+ if (hashKey === "imphash") {
+ return namespace + ".pe.imphash";
+ }
+
+ return namespace + ".hash." + hashKey;
+ };
+
+ var emptyHashRegex = /^0*$/;
+
+ var hashIsEmpty = function (value) {
+ if (!value) {
+ return true;
+ }
+
+ return emptyHashRegex.test(value);
+ }
+
+ // Adds hashes from the given hashField in the event to the 'hash' key
+ // in the specified namespace. It also adds all the hashes to 'related.hash'.
+ var addHashes = function (evt, namespace, hashField) {
  var hashes = evt.Get(hashField);
  evt.Delete(hashField);
  hashes.split(",").forEach(function (hash) {
@@ -372,16 +415,31 @@ var sysmon = (function () {
 
  var key = parts[0].toLowerCase();
  var value = parts[1].toLowerCase();
+
+ if (hashIsEmpty(value)) {
+ return;
+ }
+
+ var path = getHashPath(namespace, key);
+
+ evt.Put(path, value);
+ evt.AppendTo("related.hash", value);
+
+ // TODO: remove in 8.0, see (https://github.com/elastic/beats/issues/18364).
  evt.Put("hash." + key, value);
  });
  };
 
- var splitHashes = function (evt) {
- addHashes(evt, "winlog.event_data.Hashes");
+ var splitFileHashes = function (evt) {
+ addHashes(evt, "file", "winlog.event_data.Hashes");
  };
 
- var splitHash = function (evt) {
- addHashes(evt, "winlog.event_data.Hash");
+ var splitFileHash = function (evt) {
+ addHashes(evt, "file", "winlog.event_data.Hash");
+ };
+
+ var splitProcessHashes = function (evt) {
+ addHashes(evt, "process", "winlog.event_data.Hashes");
  };
 
  var removeEmptyEventData = function (evt) {
@@ -477,6 +535,28 @@ var sysmon = (function () {
  evt.Put("file.code_signature.valid", signatureStatus === "Valid");
  };
 
+ var setAdditionalFileFieldsFromPath = function (evt) {
+ var filePath = evt.Get("file.path");
+ if (!filePath) {
+ return;
+ }
+
+ evt.Put("file.name", path.basename(filePath));
+ evt.Put("file.directory", path.dirname(filePath));
+
+ // path returns extensions with a preceding ., e.g.: .tmp, .png
+ // according to ecs the expected format is without it, so we need to remove it.
+ var ext = path.extname(filePath);
+ if (!ext) {
+ return;
+ }
+
+ if (ext.charAt(0) === ".") {
+ ext = ext.substr(1);
+ }
+ evt.Put("file.extension", ext);
+ };
+
  // https://docs.microsoft.com/en-us/windows/win32/sysinfo/registry-hives
  var commonRegistryHives = {
  HKEY_CLASSES_ROOT: "HKCR",
@@ -606,10 +686,11 @@ var sysmon = (function () {
  ignore_missing: true,
  fail_on_error: false,
  })
+ .Add(setRuleName)
  .Add(setProcessNameUsingExe)
  .Add(splitProcessArgs)
  .Add(addUser)
- .Add(splitHashes)
+ .Add(splitProcessHashes)
  .Add(setParentProcessNameUsingExe)
  .Add(splitParentProcessArgs)
  .Add(removeEmptyEventData)
@@ -652,6 +733,8 @@ var sysmon = (function () {
  ignore_missing: true,
  fail_on_error: false,
  })
+ .Add(setRuleName)
+ .Add(setAdditionalFileFieldsFromPath)
  .Add(setProcessNameUsingExe)
  .Add(removeEmptyEventData)
  .Build();
@@ -727,6 +810,8 @@ var sysmon = (function () {
  ignore_missing: true,
  fail_on_error: false,
  })
+ .Add(setRuleName)
+ .Add(setRelatedIP)
  .Add(setProcessNameUsingExe)
  .Add(addUser)
  .Add(addNetworkDirection)
@@ -792,6 +877,7 @@ var sysmon = (function () {
  ignore_missing: true,
  fail_on_error: false,
  })
+ .Add(setRuleName)
  .Add(setProcessNameUsingExe)
  .Add(removeEmptyEventData)
  .Build();
@@ -833,8 +919,10 @@ var sysmon = (function () {
  ],
  fail_on_error: false,
  })
+ .Add(setRuleName)
+ .Add(setAdditionalFileFieldsFromPath)
  .Add(setAdditionalSignatureFields)
- .Add(splitHashes)
+ .Add(splitFileHashes)
  .Add(removeEmptyEventData)
  .Build();
 
@@ -888,9 +976,11 @@ var sysmon = (function () {
  ],
  fail_on_error: false,
  })
+ .Add(setRuleName)
+ .Add(setAdditionalFileFieldsFromPath)
  .Add(setAdditionalSignatureFields)
  .Add(setProcessNameUsingExe)
- .Add(splitHashes)
+ .Add(splitFileHashes)
  .Add(removeEmptyEventData)
  .Build();
 
@@ -921,6 +1011,7 @@ var sysmon = (function () {
  ignore_missing: true,
  fail_on_error: false,
  })
+ .Add(setRuleName)
  .Add(setProcessNameUsingExe)
  .Add(removeEmptyEventData)
  .Build();
@@ -956,6 +1047,8 @@ var sysmon = (function () {
  ignore_missing: true,
  fail_on_error: false,
  })
+ .Add(setRuleName)
+ .Add(setAdditionalFileFieldsFromPath)
  .Add(setProcessNameUsingExe)
  .Add(removeEmptyEventData)
  .Build();
@@ -998,6 +1091,7 @@ var sysmon = (function () {
  ignore_missing: true,
  fail_on_error: false,
  })
+ .Add(setRuleName)
  .Add(setProcessNameUsingExe)
  .Add(removeEmptyEventData)
  .Build();
@@ -1039,6 +1133,8 @@ var sysmon = (function () {
  ignore_missing: true,
  fail_on_error: false,
  })
+ .Add(setRuleName)
+ .Add(setAdditionalFileFieldsFromPath)
  .Add(setProcessNameUsingExe)
  .Add(removeEmptyEventData)
  .Build();
@@ -1070,6 +1166,7 @@ var sysmon = (function () {
  ignore_missing: true,
  fail_on_error: false,
  })
+ .Add(setRuleName)
  .Add(setRegistryFields)
  .Add(setProcessNameUsingExe)
  .Add(removeEmptyEventData)
@@ -1102,6 +1199,7 @@ var sysmon = (function () {
  ignore_missing: true,
  fail_on_error: false,
  })
+ .Add(setRuleName)
  .Add(setRegistryFields)
  .Add(setProcessNameUsingExe)
  .Add(removeEmptyEventData)
@@ -1134,6 +1232,7 @@ var sysmon = (function () {
  ignore_missing: true,
  fail_on_error: false,
  })
+ .Add(setRuleName)
  .Add(setRegistryFields)
  .Add(setProcessNameUsingExe)
  .Add(removeEmptyEventData)
@@ -1176,8 +1275,10 @@ var sysmon = (function () {
  ignore_missing: true,
  fail_on_error: false,
  })
+ .Add(setRuleName)
+ .Add(setAdditionalFileFieldsFromPath)
  .Add(setProcessNameUsingExe)
- .Add(splitHash)
+ .Add(splitFileHash)
  .Add(removeEmptyEventData)
  .Build();
 
@@ -1235,6 +1336,7 @@ var sysmon = (function () {
  ignore_missing: true,
  fail_on_error: false,
  })
+ .Add(setRuleName)
  .Add(setProcessNameUsingExe)
  .Add(removeEmptyEventData)
  .Build();
@@ -1276,6 +1378,7 @@ var sysmon = (function () {
  ignore_missing: true,
  fail_on_error: false,
  })
+ .Add(setRuleName)
  .Add(setProcessNameUsingExe)
  .Add(removeEmptyEventData)
  .Build();
@@ -1294,6 +1397,7 @@ var sysmon = (function () {
  ignore_missing: true,
  fail_on_error: false,
  })
+ .Add(setRuleName)
  .Add(addUser)
  .Add(removeEmptyEventData)
  .Build();
@@ -1316,6 +1420,7 @@ var sysmon = (function () {
  ignore_missing: true,
  fail_on_error: false,
  })
+ .Add(setRuleName)
  .Add(addUser)
  .Add(setProcessNameUsingExe)
  .Add(removeEmptyEventData)
@@ -1335,6 +1440,7 @@ var sysmon = (function () {
  ignore_missing: true,
  fail_on_error: false,
  })
+ .Add(setRuleName)
  .Add(addUser)
  .Add(removeEmptyEventData)
  .Build();
@@ -1389,6 +1495,7 @@ var sysmon = (function () {
  field: "dns.question.name",
  target_field: "dns.question.registered_domain",
  })
+ .Add(setRuleName)
  .Add(translateDnsQueryStatus)
  .Add(splitDnsQueryResults)
  .Add(setProcessNameUsingExe)
@@ -1425,7 +1532,7 @@ var sysmon = (function () {
  },
  {
  from: "winlog.event_data.TargetFilename",
- to: "file.name",
+ to: "file.path",
  },
  {
  from: "winlog.event_data.Image",
@@ -1446,9 +1553,11 @@ var sysmon = (function () {
  ignore_missing: true,
  fail_on_error: false,
  })
+ .Add(setRuleName)
  .Add(addUser)
- .Add(splitHashes)
+ .Add(splitProcessHashes)
  .Add(setProcessNameUsingExe)
+ .Add(setAdditionalFileFieldsFromPath)
  .Add(removeEmptyEventData)
  .Build();
 

diff --git a/x-pack/winlogbeat/module/sysmon/test/testdata/sysmon-11-filedelete.evtx b/x-pack/winlogbeat/module/sysmon/test/testdata/sysmon-11-filedelete.evtx