[EWSO365] Handle corrupt Message-ID header (#32776)

* Add handling of Message-ID header coming from attachment.item.headers
Add debug logs

* Update malformed Message-ID handling to also consider escape characters

* Update handle_incorrect_message_id to consider escape characters
Update RN; Update docker image

* Add UT; Update docker image ref

* Update release notes

* Replace string.find with regex search

* Remove XSUP-32660 debug logs

* Add test use cases

* Apply suggestions from code review

Co-authored-by: dorschw <81086590+dorschw@users.noreply.github.com>

* Remove redundant IndexError handling

* Update docker image

* Update release notes

---------

Co-authored-by: dorschw <81086590+dorschw@users.noreply.github.com>

Packs/MicrosoftExchangeOnline/Integrations/EWSO365/EWSO365.py

@@ -2037,7 +2037,7 @@ def get_item_as_eml(client: EWSClient, item_id, target_mailbox=None):      # pra
     return None
 
 
-def handle_attached_email_with_incorrect_id(attached_email: Message):
+def handle_attached_email_with_incorrect_message_id(attached_email: Message):
     """This function handles a malformed Message-ID value which can be returned in the header of certain email objects.
     This issue happens due to a current bug in "email" library and further explained in XSUP-32074.
     Public issue link: https://github.com/python/cpython/issues/105802
@@ -2053,12 +2053,14 @@ def handle_attached_email_with_incorrect_id(attached_email: Message):
     for i in range(len(attached_email._headers)):
         if attached_email._headers[i][0] == "Message-ID":
             message_id = attached_email._headers[i][1]
+            demisto.debug(f'Handling Message-ID header, {message_id=}.')
             try:
-                if message_id.endswith("]>") and message_id.startswith("<["):
-                    demisto.debug(f"Fixing invalid {message_id=} attachment header by removing its square bracket \
-                        wrapper (see XSUP-32074 for further information)")
+                message_id_value = handle_incorrect_message_id(message_id)
+                if message_id_value != message_id:
+                    # If the Message-ID header was fixed in the context of this function
+                    # the header will be replaced in _headers list
                     attached_email._headers.pop(i)
-                    message_id_value = f"<{message_id[2:-2]}>"
+                    attached_email._headers.append(("Message-ID", message_id_value))
 
             except Exception as e:
                 # The function is designed to handle a specific format error for the Message-ID header
@@ -2069,12 +2071,24 @@ def handle_attached_email_with_incorrect_id(attached_email: Message):
                 demisto.debug(f"Invalid {message_id=}, Error: {e}")
                 break
             break
-    if message_id_value:
-        # If the Message-ID header was fixed in the context of this function, it will be inserted again to the _headers list
-        attached_email._headers.append(("Message-ID", message_id_value))
     return attached_email
 
 
+def handle_incorrect_message_id(message_id: str) -> str:
+    """
+    Use regex to identify and correct one of the following invalid message_id formats:
+    1. '<[message_id]>' --> '<message_id>'
+    2. '\r\n\t<[message_id]>' --> '\r\n\t<message_id>'
+    If no necessary changes identified the original 'message_id' argument value is returned.
+    """
+    if re.search("\<\[.*\]\>", message_id):
+        # find and replace "<[" with "<" and "]>" with ">"
+        fixed_message_id = re.sub(r'<\[(.*?)\]>', r'<\1>', message_id)
+        demisto.debug('Fixed message id {message_id} to {fixed_message_id}')
+        return fixed_message_id
+    return message_id
+
+
 def parse_incident_from_item(item):     # pragma: no cover
     """
     Parses an incident from an item
@@ -2184,7 +2198,7 @@ def parse_incident_from_item(item):     # pragma: no cover
                     if attachment.item.headers:
                         # compare header keys case-insensitive
                         attached_email_headers = []
-                        attached_email = handle_attached_email_with_incorrect_id(attached_email)
+                        attached_email = handle_attached_email_with_incorrect_message_id(attached_email)
                         for h, v in attached_email.items():
                             if not isinstance(v, str):
                                 try:
@@ -2203,7 +2217,16 @@ def parse_incident_from_item(item):     # pragma: no cover
                                     and header.name.lower() != "content-type"
                             ):
                                 try:
-                                    attached_email.add_header(header.name, header.value)
+                                    if header.name.lower() == "message-id":
+                                        """ Handle a case where a Message-ID header was NOT already in attached_email,
+                                        and instead is coming from attachment.item.headers.
+                                        Meaning it wasn't handled in handle_attached_email_with_incorrect_message_id function
+                                        and instead it is handled here using handle_incorrect_message_id function."""
+                                        correct_message_id = handle_incorrect_message_id(header.value)
+                                        if (header.name.lower(), correct_message_id) not in attached_email_headers:
+                                            attached_email.add_header(header.name, correct_message_id)
+                                    else:
+                                        attached_email.add_header(header.name, header.value)
                                 except ValueError as err:
                                     if "There may be at most" not in str(err):
                                         raise err

Packs/MicrosoftExchangeOnline/Integrations/EWSO365/EWSO365.yml

@@ -959,7 +959,7 @@ script:
   - description: Run this command if for some reason you need to rerun the authentication process.
     name: ews-auth-reset
     arguments: []
-  dockerimage: demisto/py3ews:1.0.0.86480
+  dockerimage: demisto/py3ews:1.0.0.88266
   isfetch: true
   script: ''
   subtype: python3

Packs/MicrosoftExchangeOnline/Integrations/EWSO365/EWSO365_test.py

@@ -18,8 +18,9 @@
     get_expanded_group,
     get_item_as_eml,
     get_searchable_mailboxes,
-    handle_attached_email_with_incorrect_id,
+    handle_attached_email_with_incorrect_message_id,
     handle_html,
+    handle_incorrect_message_id,
     handle_transient_files,
     parse_incident_from_item,
     parse_item_as_dict,
@@ -864,4 +865,29 @@ def test_handle_attached_email_with_incorrect_id(mocker, headers, expected_forma
     email_policy = SMTP
     attached_email = email.message_from_bytes(mime_content, policy=email_policy)
     attached_email._headers = headers
-    assert handle_attached_email_with_incorrect_id(attached_email)._headers == expected_formatted_headers
+    assert handle_attached_email_with_incorrect_message_id(attached_email)._headers == expected_formatted_headers
+
+
+@pytest.mark.parametrize("message_id, expected_message_id_output", [
+    pytest.param('<message_id>', '<message_id>', id="valid message_id 1"),
+    pytest.param('<mess<[age_id>', '<mess<[age_id>', id="valid message_id 2"),
+    pytest.param('<>]message_id>', '<>]message_id>', id="valid message_id 3"),
+    pytest.param('<[message_id]>', '<message_id>', id="invalid message_id"),
+    pytest.param('\r\n\t<message_id>', '\r\n\t<message_id>', id="valid message_id with escape chars"),
+    pytest.param('\r\n\t<[message_id]>', '\r\n\t<message_id>', id="invalid message_id with escape chars"),
+])
+def test_handle_incorrect_message_id(message_id, expected_message_id_output):
+    """
+    Given:
+        - case 1: valid Message-ID header value in attached email object
+        - case 1: invalid Message-ID header value in attached email object
+        - case 3: a Message-ID header value format which is not tested in the context of handle_attached_email_with_incorrect_id
+    When:
+        - fetching email which have an attached email with Message-ID header
+    Then:
+        - case 1: verify the header in the correct format
+        - case 2: correct the invalid Message-ID header value
+        - case 3: return the header value without without further handling
+
+    """
+    assert handle_incorrect_message_id(message_id) == expected_message_id_output

Packs/MicrosoftExchangeOnline/ReleaseNotes/1_2_40.md

@@ -0,0 +1,8 @@
+
+#### Integrations
+
+##### EWS O365
+- Updated the Docker image to: *demisto/py3ews:1.0.0.88266*.
+
+- Fixed an issue where fetching failed when email attachments had headers with an invalid format containing escape characters (`\r\n\t<[invalid_value]>` instead of `\r\n\t<valid_value>`), by removing the square brackets.
+

Packs/MicrosoftExchangeOnline/pack_metadata.json

@@ -2,7 +2,7 @@
     "name": "Microsoft Exchange Online",
     "description": "Exchange Online and Office 365 (mail)",
     "support": "xsoar",
-    "currentVersion": "1.2.39",
+    "currentVersion": "1.2.40",
     "author": "Cortex XSOAR",
     "url": "https://www.paloaltonetworks.com/cortex",
     "email": "",