Merge branch 'main' into feat/improve-mimecast-parsing

SEKOIA-IO · Jan 9, 2025 · 6021049 · 6021049
2 parents 977c77a + e3d059a
commit 6021049
Show file tree

Hide file tree

Showing 15 changed files with 557 additions and 20 deletions.
diff --git a/Mimecast/mimecast-email-security/_meta/fields.yml b/Mimecast/mimecast-email-security/_meta/fields.yml
@@ -40,6 +40,11 @@ mimecast.siem.rejection.type:
   name: mimecast.siem.rejection.type
   type: keyword
 
+mimecast.siem.scan_results:
+  description: The reason that the click was blocked.
+  name: mimecast.siem.scan_results
+  type: keyword
+
 mimecast.siem.virus_found:
   description: The name of the virus found on the email, if applicable.
   name: mimecast.siem.virus_found

diff --git a/Mimecast/mimecast-email-security/_meta/smart-descriptions.json b/Mimecast/mimecast-email-security/_meta/smart-descriptions.json
@@ -1,7 +1,19 @@
 [
   {
     "value": "{mimecast.siem.rejection.info} for email from {email.from.address}",
-    "conditions": [{ "field": "mimecast.siem.rejection.info" }]
+    "conditions": [
+      { "field": "mimecast.siem.rejection.info" },
+      { "field": "email.from.address" }
+    ]
+  },
+  {
+    "value": "Protected {email.to.address} by {event.action} URL {url.original}",
+    "conditions": [
+      { "field": "event.dataset", "value": "url protect" },
+      { "field": "event.action" },
+      { "field": "email.to.address" },
+      { "field": "url.original" }
+    ]
   },
   {
     "value": "{event.action} {email.direction} email from {email.from.address} to {email.to.address}",
@@ -26,5 +38,34 @@
       { "field": "event.action" },
       { "field": "email.from.address" }
     ]
+  },
+  {
+    "value": "Spam detected in email from {email.from.address} to {email.to.address}",
+    "conditions": [
+      { "field": "event.dataset", "value": "spam" },
+      { "field": "email.from.address" },
+      { "field": "email.to.address" }
+    ]
+  },
+  {
+    "value": "Attachment Protect triggered on file {email.attachments.file.name}",
+    "conditions": [
+      { "field": "event.dataset", "value": "attachment protect" },
+      { "field": "email.attachments.file.name" }
+    ]
+  },
+  {
+    "value": "AV detected {mimecast.siem.virus_found} in file {email.attachments.file.name} from {email.from.address}",
+    "conditions": [
+      { "field": "event.dataset", "value": "av" },
+      { "field": "mimecast.siem.virus_found" }
+    ]
+  },
+  {
+    "value": "{mimecast.siem.scan_results} on email from {email.from.address}",
+    "conditions": [
+      { "field": "mimecast.siem.scan_results" },
+      { "field": "email.from.address" }
+    ]
   }
 ]
diff --git a/Mimecast/mimecast-email-security/ingest/parser.yml b/Mimecast/mimecast-email-security/ingest/parser.yml
@@ -45,7 +45,6 @@ stages:
 
           email.message_id: "{{parse_event.message.messageId.lstrip('<').rstrip('>')}}"
           email.subject: "{{parse_event.message.subject}}"
-          email.to.address: ["{{parse_event.message.recipients}}"]
           email.from.address:
             [
               "{{parse_event.message.senderEnvelope or parse_event.message.senderHeader}}",
@@ -60,7 +59,21 @@ stages:
           mimecast.siem.delivered: "{{parse_event.message.delivered}}"
           mimecast.siem.delivery_errors: "{{parse_event.message.deliveryErrors}}"
 
+          mimecast.siem.scan_results: "{{parse_event.message.scanResults}}"
+
           destination.ip: "{{parse_event.message.destinationIp}}"
+          url.original: "{{parse_event.message.url}}"
+
+      - set:
+          email.to.address: ["{{parse_event.message.recipients}}"]
+        filter: "{{parse_event.message.get('recipients') != None}}"
+
+      - set:
+          email.from.address:
+            [
+              "{{parse_event.message.senderEnvelope or parse_event.message.senderHeader}}",
+            ]
+        filter: "{{parse_event.message.get('senderEnvelope') != None or parse_event.message.get('senderHeader') != None}}"
 
       - set:
           email.attachments: >
@@ -69,6 +82,14 @@ stages:
                 {"file": {"name": "{{ item.strip('" ') }}"}},
               {%- endfor -%}
             ]
+        filter: "{{parse_event.message.get('attachments') != None}}"
+
+      - set:
+          email.attachments: >
+            [
+                {"file": {"name": "{{ parse_event.message.fileName.strip('" ') }}"}},
+            ]
+        filter: "{{parse_event.message.get('fileName') != None}}"
 
       - set:
           url.original: "{{ parse_rejectioninfo.message.Url }}"

diff --git a/Mimecast/mimecast-email-security/tests/test_attachment_protect.json b/Mimecast/mimecast-email-security/tests/test_attachment_protect.json
@@ -0,0 +1,39 @@
+{
+  "input": {
+    "message": "{\"processingId\": \"processingId\", \"aggregateId\": \"aggregateId\", \"sha1\": \"816b013c8be6e5708690645964b5d442c085041e\", \"accountId\": \"C0A0\", \"fileName\": \"tpsreport.docx\", \"sha256\": \"efe51c2453821310c7a34dca3054021d0f6d453b7133c381d75e3140901efd12\", \"fileExtension\": \"xlsm\", \"subType\": null, \"eventType\": \"attachment protect\", \"timestamp\": 1689692409135, \"md5\": \"4dbe9dbfb53438d9ce410535355cd973\"}"
+  },
+  "expected": {
+    "message": "{\"processingId\": \"processingId\", \"aggregateId\": \"aggregateId\", \"sha1\": \"816b013c8be6e5708690645964b5d442c085041e\", \"accountId\": \"C0A0\", \"fileName\": \"tpsreport.docx\", \"sha256\": \"efe51c2453821310c7a34dca3054021d0f6d453b7133c381d75e3140901efd12\", \"fileExtension\": \"xlsm\", \"subType\": null, \"eventType\": \"attachment protect\", \"timestamp\": 1689692409135, \"md5\": \"4dbe9dbfb53438d9ce410535355cd973\"}",
+    "event": {
+      "category": [
+        "email"
+      ],
+      "dataset": "attachment protect",
+      "provider": "Mimecast",
+      "type": [
+        "info"
+      ]
+    },
+    "@timestamp": "2023-07-18T15:00:09.135000Z",
+    "email": {
+      "attachments": [
+        {
+          "file": {
+            "name": "tpsreport.docx"
+          }
+        }
+      ],
+      "from": {
+        "address": [
+          "null"
+        ]
+      }
+    },
+    "mimecast": {
+      "siem": {
+        "aggregate_id": "aggregateId",
+        "processing_id": "processingId"
+      }
+    }
+  }
+}
diff --git a/Mimecast/mimecast-email-security/tests/test_av_logs.json b/Mimecast/mimecast-email-security/tests/test_av_logs.json
@@ -0,0 +1,56 @@
+{
+  "input": {
+    "message": "{\"processingId\": \"processingId\", \"aggregateId\": \"aggregateId\", \"fileName\": \"tpsreport.docx\", \"sha256\": \"efe51c2453821310c7a34dca3054021d0f6d453b7133c381d75e3140901efd12\", \"subject\": \"siem_av - email subject line\", \"senderEnvelope\": \"auser@mimecast.com\", \"messageId\": \"messageId\", \"senderDomainInternal\": \"true\", \"eventType\": \"av\", \"sha1\": \"816b013c8be6e5708690645964b5d442c085041e\", \"accountId\": \"C0A0\", \"virusFound\": \"bad.virus.found\", \"route\": \"Inbound\", \"recipients\": \"auser@mimecast.com\", \"fileExtension\": \"docx\", \"subType\": null, \"senderIp\": \"123.123.123.123\", \"senderDomain\": \"mimecast.com\", \"timestamp\": 1689685338586, \"emailSize\": \"1648832\", \"md5\": \"4dbe9dbfb53438d9ce410535355cd973\"}"
+  },
+  "expected": {
+    "message": "{\"processingId\": \"processingId\", \"aggregateId\": \"aggregateId\", \"fileName\": \"tpsreport.docx\", \"sha256\": \"efe51c2453821310c7a34dca3054021d0f6d453b7133c381d75e3140901efd12\", \"subject\": \"siem_av - email subject line\", \"senderEnvelope\": \"auser@mimecast.com\", \"messageId\": \"messageId\", \"senderDomainInternal\": \"true\", \"eventType\": \"av\", \"sha1\": \"816b013c8be6e5708690645964b5d442c085041e\", \"accountId\": \"C0A0\", \"virusFound\": \"bad.virus.found\", \"route\": \"Inbound\", \"recipients\": \"auser@mimecast.com\", \"fileExtension\": \"docx\", \"subType\": null, \"senderIp\": \"123.123.123.123\", \"senderDomain\": \"mimecast.com\", \"timestamp\": 1689685338586, \"emailSize\": \"1648832\", \"md5\": \"4dbe9dbfb53438d9ce410535355cd973\"}",
+    "event": {
+      "category": [
+        "email"
+      ],
+      "dataset": "av",
+      "provider": "Mimecast",
+      "type": [
+        "info"
+      ]
+    },
+    "@timestamp": "2023-07-18T13:02:18.586000Z",
+    "email": {
+      "attachments": [
+        {
+          "file": {
+            "name": "tpsreport.docx"
+          }
+        }
+      ],
+      "from": {
+        "address": [
+          "auser@mimecast.com"
+        ]
+      },
+      "message_id": "messageId",
+      "subject": "siem_av - email subject line",
+      "to": {
+        "address": [
+          "auser@mimecast.com"
+        ]
+      }
+    },
+    "mimecast": {
+      "siem": {
+        "aggregate_id": "aggregateId",
+        "processing_id": "processingId",
+        "virus_found": "bad.virus.found"
+      }
+    },
+    "related": {
+      "ip": [
+        "123.123.123.123"
+      ]
+    },
+    "source": {
+      "address": "123.123.123.123",
+      "ip": "123.123.123.123"
+    }
+  }
+}
diff --git a/Mimecast/mimecast-email-security/tests/test_delivery.json b/Mimecast/mimecast-email-security/tests/test_delivery.json
@@ -0,0 +1,50 @@
+{
+  "input": {
+    "message": "{\"processingId\": \"processingId\", \"aggregateId\": \"aggregateId\", \"accountId\": \"C0A0\", \"timestamp\": 1731187649343, \"messageId\": \"<11111111111111111111111111111111111111@mail.gmail.com>\", \"senderEnvelope\": \"john.doe@example.org\", \"subject\": \"My little subject\", \"recipients\": \"jane.doe@example.com\", \"delivered\": \"true\", \"destinationIp\": \"5.6.7.8\", \"Hostname\": \"mail-111111111.inbound.protection.outlook.com\", \"numberAttachments\": \"0\", \"direction\": \"Inbound\", \"totalSizeAttachments\": \"0\", \"deliveryAttempts\": \"1\", \"tlsVersion\": \"TLSv1.3\", \"tlsCipher\": \"TLS_AES_256_GCM_SHA384\", \"emailSize\": \"30126\", \"tlsUsed\": \"Yes\", \"route\": \"Office 365 Inbound Routing Policy Definition\", \"deliveryErrors\": null, \"rejectionType\": null, \"rejectionCode\": null, \"rejectionInfo\": null, \"deliveryTime\": \"5333\", \"type\": \"delivery\", \"subtype\": \"true\", \"_offset\": 1069263, \"_partition\": 66}"
+  },
+  "expected": {
+    "message": "{\"processingId\": \"processingId\", \"aggregateId\": \"aggregateId\", \"accountId\": \"C0A0\", \"timestamp\": 1731187649343, \"messageId\": \"<11111111111111111111111111111111111111@mail.gmail.com>\", \"senderEnvelope\": \"john.doe@example.org\", \"subject\": \"My little subject\", \"recipients\": \"jane.doe@example.com\", \"delivered\": \"true\", \"destinationIp\": \"5.6.7.8\", \"Hostname\": \"mail-111111111.inbound.protection.outlook.com\", \"numberAttachments\": \"0\", \"direction\": \"Inbound\", \"totalSizeAttachments\": \"0\", \"deliveryAttempts\": \"1\", \"tlsVersion\": \"TLSv1.3\", \"tlsCipher\": \"TLS_AES_256_GCM_SHA384\", \"emailSize\": \"30126\", \"tlsUsed\": \"Yes\", \"route\": \"Office 365 Inbound Routing Policy Definition\", \"deliveryErrors\": null, \"rejectionType\": null, \"rejectionCode\": null, \"rejectionInfo\": null, \"deliveryTime\": \"5333\", \"type\": \"delivery\", \"subtype\": \"true\", \"_offset\": 1069263, \"_partition\": 66}",
+    "event": {
+      "category": [
+        "email"
+      ],
+      "dataset": "delivery",
+      "provider": "Mimecast",
+      "type": [
+        "info"
+      ]
+    },
+    "@timestamp": "2024-11-09T21:27:29.343000Z",
+    "destination": {
+      "address": "5.6.7.8",
+      "ip": "5.6.7.8"
+    },
+    "email": {
+      "direction": "Inbound",
+      "from": {
+        "address": [
+          "john.doe@example.org"
+        ]
+      },
+      "message_id": "11111111111111111111111111111111111111@mail.gmail.com",
+      "subject": "My little subject",
+      "to": {
+        "address": [
+          "jane.doe@example.com"
+        ]
+      }
+    },
+    "mimecast": {
+      "siem": {
+        "aggregate_id": "aggregateId",
+        "delivered": true,
+        "processing_id": "processingId"
+      }
+    },
+    "related": {
+      "ip": [
+        "5.6.7.8"
+      ]
+    }
+  }
+}
diff --git a/Mimecast/mimecast-email-security/tests/test_impersonation_protect.json b/Mimecast/mimecast-email-security/tests/test_impersonation_protect.json
@@ -0,0 +1,48 @@
+{
+  "input": {
+    "message": "{\"processingId\": \"processingId\", \"aggregateId\": \"aggregateId\", \"taggedMalicious\": \"false\", \"subject\": \"siem_impersonation - email subject line\", \"internalUserName\": \"false\", \"senderEnvelope\": \"auser@mimecast.com\", \"policyDefinition\": \"Default Impersonation Definition\", \"newDomain\": \"false\", \"customThreatDictionary\": \"false\", \"action\": \"Hold\", \"senderIp\": \"123.123.123.123\", \"timestamp\": 1689685338545, \"similarInternalDomain\": \"false\", \"messageId\": \"\", \"eventType\": \"impersonation protect\", \"itemsDetected\": \"1\", \"mimecastThreatDictionary\": \"false\", \"accountId\": \"C0A0\", \"customNameMatch\": \"false\", \"route\": \"Inbound\", \"similarMimecastExternalDomain\": \"false\", \"recipients\": \"auser@mimecast.com\", \"similarCustomExternalDomain\": \"false\", \"subType\": \"Hold\", \"taggedExternal\": \"false\", \"replyMismatch\": \"false\"}"
+  },
+  "expected": {
+    "message": "{\"processingId\": \"processingId\", \"aggregateId\": \"aggregateId\", \"taggedMalicious\": \"false\", \"subject\": \"siem_impersonation - email subject line\", \"internalUserName\": \"false\", \"senderEnvelope\": \"auser@mimecast.com\", \"policyDefinition\": \"Default Impersonation Definition\", \"newDomain\": \"false\", \"customThreatDictionary\": \"false\", \"action\": \"Hold\", \"senderIp\": \"123.123.123.123\", \"timestamp\": 1689685338545, \"similarInternalDomain\": \"false\", \"messageId\": \"\", \"eventType\": \"impersonation protect\", \"itemsDetected\": \"1\", \"mimecastThreatDictionary\": \"false\", \"accountId\": \"C0A0\", \"customNameMatch\": \"false\", \"route\": \"Inbound\", \"similarMimecastExternalDomain\": \"false\", \"recipients\": \"auser@mimecast.com\", \"similarCustomExternalDomain\": \"false\", \"subType\": \"Hold\", \"taggedExternal\": \"false\", \"replyMismatch\": \"false\"}",
+    "event": {
+      "action": "Hold",
+      "category": [
+        "email"
+      ],
+      "dataset": "impersonation protect",
+      "provider": "Mimecast",
+      "type": [
+        "info"
+      ]
+    },
+    "@timestamp": "2023-07-18T13:02:18.545000Z",
+    "email": {
+      "from": {
+        "address": [
+          "auser@mimecast.com"
+        ]
+      },
+      "subject": "siem_impersonation - email subject line",
+      "to": {
+        "address": [
+          "auser@mimecast.com"
+        ]
+      }
+    },
+    "mimecast": {
+      "siem": {
+        "aggregate_id": "aggregateId",
+        "processing_id": "processingId"
+      }
+    },
+    "related": {
+      "ip": [
+        "123.123.123.123"
+      ]
+    },
+    "source": {
+      "address": "123.123.123.123",
+      "ip": "123.123.123.123"
+    }
+  }
+}
diff --git a/Mimecast/mimecast-email-security/tests/test_internal_email_project.json b/Mimecast/mimecast-email-security/tests/test_internal_email_project.json
@@ -0,0 +1,40 @@
+{
+  "input": {
+    "message": "{\"processingId\": \"processingId\", \"aggregateId\": \"aggregateId\", \"accountId\": \"C0A0\", \"timestamp\": 1730905847558, \"messageId\": \"<11111111111111111111111111111111111111@mail.gmail.com>\", \"senderEnvelope\": \"john.doe@example.org\", \"subject\": \"My dangerous email\", \"recipients\": \"jane.doe@example.com\", \"urlCategory\": \"Dangerous file extension\", \"scanResults\": \"Restricted File Type - Found executable extension: dll\", \"route\": \"Internal\", \"monitoredDomainSource\": null, \"similarDomain\": null, \"type\": \"internal email protect\", \"subtype\": null, \"_offset\": 994904, \"_partition\": 66}"
+  },
+  "expected": {
+    "message": "{\"processingId\": \"processingId\", \"aggregateId\": \"aggregateId\", \"accountId\": \"C0A0\", \"timestamp\": 1730905847558, \"messageId\": \"<11111111111111111111111111111111111111@mail.gmail.com>\", \"senderEnvelope\": \"john.doe@example.org\", \"subject\": \"My dangerous email\", \"recipients\": \"jane.doe@example.com\", \"urlCategory\": \"Dangerous file extension\", \"scanResults\": \"Restricted File Type - Found executable extension: dll\", \"route\": \"Internal\", \"monitoredDomainSource\": null, \"similarDomain\": null, \"type\": \"internal email protect\", \"subtype\": null, \"_offset\": 994904, \"_partition\": 66}",
+    "event": {
+      "category": [
+        "email"
+      ],
+      "dataset": "internal email protect",
+      "provider": "Mimecast",
+      "type": [
+        "info"
+      ]
+    },
+    "@timestamp": "2024-11-06T15:10:47.558000Z",
+    "email": {
+      "from": {
+        "address": [
+          "john.doe@example.org"
+        ]
+      },
+      "message_id": "11111111111111111111111111111111111111@mail.gmail.com",
+      "subject": "My dangerous email",
+      "to": {
+        "address": [
+          "jane.doe@example.com"
+        ]
+      }
+    },
+    "mimecast": {
+      "siem": {
+        "aggregate_id": "aggregateId",
+        "processing_id": "processingId",
+        "scan_results": "Restricted File Type - Found executable extension: dll"
+      }
+    }
+  }
+}
diff --git a/Mimecast/mimecast-email-security/tests/test_process.json b/Mimecast/mimecast-email-security/tests/test_process.json
@@ -23,12 +23,7 @@
         ]
       },
       "message_id": "CAF7=BmDb+6qHo+J5EB9oH+S4ncJOfEMsUYAEirX4MRZRJX+esw@mail.gmail.com",
-      "subject": "Moderate",
-      "to": {
-        "address": [
-          "null"
-        ]
-      }
+      "subject": "Moderate"
     },
     "mimecast": {
       "siem": {

diff --git a/Mimecast/mimecast-email-security/tests/test_process_with_attachment.json b/Mimecast/mimecast-email-security/tests/test_process_with_attachment.json
@@ -30,12 +30,7 @@
         ]
       },
       "message_id": "messageId",
-      "subject": "siem_process - email subject line",
-      "to": {
-        "address": [
-          "null"
-        ]
-      }
+      "subject": "siem_process - email subject line"
     },
     "mimecast": {
       "siem": {

diff --git a/Mimecast/mimecast-email-security/tests/test_process_with_multiple_attachments.json b/Mimecast/mimecast-email-security/tests/test_process_with_multiple_attachments.json
@@ -100,12 +100,7 @@
         ]
       },
       "message_id": "1@mail.gmail.com",
-      "subject": "TEST SEKOIA",
-      "to": {
-        "address": [
-          "null"
-        ]
-      }
+      "subject": "TEST SEKOIA"
     },
     "mimecast": {
       "siem": {