[transformers] Enhance to be more durable #2 (demisto#31063)

* ExtractDomainAndFQDNFromUrlAndEmail * ExtractEmailV2 * ruff fix * FormatURL * Bump pack from version CommonScripts to 1.12.47. * Bump pack from version CommonScripts to 1.12.48. * Bump pack from version CommonScripts to 1.12.49. * Added unit tests * Update the docker image * Bump pack from version CommonScripts to 1.12.50. * Update Packs/CommonScripts/ReleaseNotes/1_12_50.md Co-authored-by: Dean Arbel <darbel@paloaltonetworks.com> * Update Packs/CommonScripts/ReleaseNotes/1_12_50.md Co-authored-by: Dean Arbel <darbel@paloaltonetworks.com> * Update the RN * Update the docker image * empty * Skip "FormattingPerformance - Test" TPB in XSOAR 8 --------- Co-authored-by: Content Bot <bot@demisto.com> Co-authored-by: Dean Arbel <darbel@paloaltonetworks.com>
riskive · Nov 28, 2023 · f65c749 · f65c749
1 parent 4aff1a6
commit f65c749
Show file tree

Hide file tree

Showing 11 changed files with 142 additions and 70 deletions.
diff --git a/Packs/CommonScripts/ReleaseNotes/1_12_50.md b/Packs/CommonScripts/ReleaseNotes/1_12_50.md
@@ -0,0 +1,18 @@
+
+#### Scripts
+
+##### FormatURL
+
+- Improved error handling for better performance when the script fails.
+- Updated the Docker image to: *demisto/python3:3.10.13.80593*.
+
+##### ExtractDomainAndFQDNFromUrlAndEmail
+
+- Updated the Docker image to: *demisto/py3-tools:1.0.0.81280*.
+- Improved error handling for better performance when the script fails.
+
+
+##### ExtractEmailV2
+
+- Updated the Docker image to: *demisto/python3:3.10.13.80593*.
+- Improved error handling for better performance when the script fails.
diff --git a/...cripts/Scripts/ExtractDomainAndFQDNFromUrlAndEmail/ExtractDomainAndFQDNFromUrlAndEmail.py b/...cripts/Scripts/ExtractDomainAndFQDNFromUrlAndEmail/ExtractDomainAndFQDNFromUrlAndEmail.py
@@ -7,7 +7,7 @@
 PROOFPOINT_PREFIXES = ['https://urldefense.proofpoint.com/',
                        "https://urldefense.com/"]
 ATP_LINK_REG = r'(https:\/\/\w*|\w*)\.safelinks\.protection\.outlook\.com/'
-DOMAIN_REGEX = r"(?i)(?P<scheme>(?:http|ftp|hxxp)s?(?:://|-3A__|%3A%2F%2F))?(?P<domain>(?:[\w\-–_]+(?:\.|\[\.\]))+[^\W\d_]{2,})(?:[_/\s\"',)\]}>]|[.]\s?|%2F|.?$)"  # noqa: E501
+DOMAIN_REGEX = r"(?i)(?P<scheme>(?:http|ftp|hxxp)s?(?:://|-3A__|%3A%2F%2F))?(?P<domain>(?:[\w\-–_]+(?:\.|\[\.\]))+[^\W\d_]{2,})(?:[_/\s\"',)\]}>]|[.]\s?|%2F|.?$)"  # noqa: E501, RUF001
 
 
 def atp_get_original_url(safe_url):  # pragma: no cover
@@ -60,7 +60,7 @@ def get_fqdn(the_input):
         # get the subdomain using tld.subdomain
         subdomain = domain.subdomain
         if (subdomain):
-            fqdn = "{}.{}".format(subdomain, domain.fld)
+            fqdn = f"{subdomain}.{domain.fld}"
         else:
             fqdn = domain.fld
 
@@ -109,27 +109,33 @@ def extract_fqdn(the_input):
 
 
 def main():
-    the_input = demisto.args().get('input')
-
-    # argToList returns the argument as is if it's already a list so no need to check here
-    the_input = argToList(the_input)
-    entries_list = []
-    # Otherwise assumes it's already an array
-    for item in the_input:
-        input_entry = {
-            "Type": entryTypes["note"],
-            "ContentsFormat": formats["json"],
-            "Contents": [extract_fqdn(item)],
-            "EntryContext": {"Domain": item}
-        }
-        if input_entry.get("Contents") == ['']:
-            input_entry['Contents'] = []
-        entries_list.append(input_entry)
-    if entries_list:
-        demisto.results(entries_list)
-    else:
-        # Return empty string so it wouldn't create an empty domain indicator.
-        demisto.results('')
+    try:
+        the_input = demisto.args().get('input')
+
+        # argToList returns the argument as is if it's already a list so no need to check here
+        the_input = argToList(the_input)
+        entries_list = []
+        # Otherwise assumes it's already an array
+        for item in the_input:
+            input_entry = {
+                "Type": entryTypes["note"],
+                "ContentsFormat": formats["json"],
+                "Contents": [extract_fqdn(item)],
+                "EntryContext": {"Domain": item}
+            }
+            if input_entry.get("Contents") == ['']:
+                input_entry['Contents'] = []
+            entries_list.append(input_entry)
+        if entries_list:
+            demisto.results(entries_list)
+        else:
+            # Return empty string so it wouldn't create an empty domain indicator.
+            demisto.results('')
+
+    except Exception as e:
+        return_error(
+            f'Failed to execute the automation. Error: \n{str(e)}'
+        )
 
 
 # python2 uses __builtin__ python3 uses builtins

diff --git a/...ripts/Scripts/ExtractDomainAndFQDNFromUrlAndEmail/ExtractDomainAndFQDNFromUrlAndEmail.yml b/...ripts/Scripts/ExtractDomainAndFQDNFromUrlAndEmail/ExtractDomainAndFQDNFromUrlAndEmail.yml
@@ -15,7 +15,7 @@ tags:
 - indicator-format
 timeout: '0'
 type: python
-dockerimage: demisto/py3-tools:1.0.0.63856
+dockerimage: demisto/py3-tools:1.0.0.81280
 runas: DBotWeakRole
 tests:
 - ExtractDomainAndFQDNFromUrlAndEmail-Test

diff --git a/...s/Scripts/ExtractDomainAndFQDNFromUrlAndEmail/ExtractDomainAndFQDNFromUrlAndEmail_test.py b/...s/Scripts/ExtractDomainAndFQDNFromUrlAndEmail/ExtractDomainAndFQDNFromUrlAndEmail_test.py
@@ -1,3 +1,4 @@
+from pytest_mock import MockFixture
 import demistomock as demisto
 from ExtractDomainAndFQDNFromUrlAndEmail import extract_fqdn, main
 import pytest
@@ -72,3 +73,21 @@ def test_extract_fqdn_or_domain_empty_indicators(mocker):
     results = demisto.results.call_args[0]
 
     assert results[0] == [{'Contents': [], 'ContentsFormat': 'json', 'Type': 1, 'EntryContext': {'Domain': '1Ab.Vt'}}]
+
+
+def test_main_raise_error(mocker: MockFixture):
+    """
+    Given:
+        - Exception during the automation
+    When:
+        - Running the automation
+    Then:
+        - Ensure the return_error is called with the correct error message.
+    """
+    return_error_mock = mocker.patch('ExtractDomainAndFQDNFromUrlAndEmail.return_error')
+    mocker.patch('ExtractDomainAndFQDNFromUrlAndEmail.argToList', side_effect=Exception('Test Exception'))
+
+    main()
+
+    assert return_error_mock.call_count == 1
+    assert 'Error: \nTest Exception' in return_error_mock.call_args[0][0]
diff --git a/Packs/CommonScripts/Scripts/ExtractEmailFormatting/ExtractEmailFormatting.py b/Packs/CommonScripts/Scripts/ExtractEmailFormatting/ExtractEmailFormatting.py
@@ -48,10 +48,7 @@ def check_tld(email_address: str) -> bool:
         Boolean: True if it's not a common file extension
 
     """
-    if email_address.split(".")[-1] not in COMMON_FILE_EXT:
-        return True
-    else:
-        return False
+    return email_address.split(".")[-1] not in COMMON_FILE_EXT
 
 
 def refang_email(email_address: str) -> str:
@@ -91,24 +88,30 @@ def extract_email_from_url_query(email_address: str) -> str:
 def main():
     list_results = []
 
-    emails = argToList(demisto.args().get('input'))
+    try:
+        emails = argToList(demisto.args().get('input'))
 
-    clean_emails = [extract_email(address) for address in emails]
+        clean_emails = [extract_email(address) for address in emails]
 
-    list_results = [refang_email(email_address) for email_address in clean_emails]
+        list_results = [refang_email(email_address) for email_address in clean_emails]
 
-    output = [
-        {
-            'Type': entryTypes['note'],
-            'ContentsFormat': formats['json'],
-            'Contents': [email_address] if email_address else [],
-            'EntryContext': {'Email': email_address} if email_address else {},
-        } for email_address in list_results]
+        output = [
+            {
+                'Type': entryTypes['note'],
+                'ContentsFormat': formats['json'],
+                'Contents': [email_address] if email_address else [],
+                'EntryContext': {'Email': email_address} if email_address else {},
+            } for email_address in list_results]
 
-    if output:
-        return_results(output)
-    else:
-        return_results('')
+        if output:
+            return_results(output)
+        else:
+            return_results('')
+
+    except Exception as e:
+        return_error(
+            f'Failed to execute the automation. Error: \n{str(e)}'
+        )
 
 
 if __name__ in ('__main__', 'builtin', 'builtins'):

diff --git a/Packs/CommonScripts/Scripts/ExtractEmailFormatting/ExtractEmailFormatting.yml b/Packs/CommonScripts/Scripts/ExtractEmailFormatting/ExtractEmailFormatting.yml
@@ -15,7 +15,7 @@ args:
   isArray: true
 scripttarget: 0
 subtype: python3
-dockerimage: demisto/python3:3.10.12.63474
+dockerimage: demisto/python3:3.10.13.80593
 fromversion: 5.5.0
 tests:
 - ExtractEmailV2-Test
diff --git a/Packs/CommonScripts/Scripts/ExtractEmailFormatting/ExtractEmailFormatting_test.py b/Packs/CommonScripts/Scripts/ExtractEmailFormatting/ExtractEmailFormatting_test.py
@@ -1,3 +1,4 @@
+from pytest_mock import MockFixture
 from ExtractEmailFormatting import main, check_tld, extract_email, refang_email, extract_email_from_url_query
 import pytest
 import demistomock as demisto
@@ -79,7 +80,7 @@ def test_main(mocker):
     mocker.patch.object(demisto, 'results')
     main()
     results = [email_address['Contents'] for email_address in demisto.results.call_args[0][0]]
-    assert EXPECTED_RESULTS == results
+    assert results == EXPECTED_RESULTS
 
 
 def test_main_invalid_emails(mocker):
@@ -94,4 +95,22 @@ def test_main_invalid_emails(mocker):
     mocker.patch.object(demisto, 'args', return_value={"input": ''})
     mocker.patch.object(demisto, 'results')
     main()
-    assert '' == demisto.results.call_args[0][0]
+    assert demisto.results.call_args[0][0] == ''
+
+
+def test_main_raise_error(mocker: MockFixture):
+    """
+    Given:
+        - Exception during the automation
+    When:
+        - Running the automation
+    Then:
+        - Ensure the return_error is called with the correct error message.
+    """
+    return_error_mock = mocker.patch('ExtractEmailFormatting.return_error')
+    mocker.patch('ExtractEmailFormatting.argToList', side_effect=Exception('Test Exception'))
+
+    main()
+
+    assert return_error_mock.call_count == 1
+    assert 'Error: \nTest Exception' in return_error_mock.call_args[0][0]
diff --git a/Packs/CommonScripts/Scripts/FormatURL/FormatURL.py b/Packs/CommonScripts/Scripts/FormatURL/FormatURL.py
@@ -742,35 +742,41 @@ def main():
 
     formatted_urls: List[str] = []
 
-    for url in raw_urls:
-        formatted_url = ''
+    try:
+        for url in raw_urls:
+            formatted_url = ''
 
-        if _is_valid_cidr(url):
-            # If input is a valid CIDR formatter will ignore it to let it become a CIDR
-            formatted_urls.append('')
-            continue
+            if _is_valid_cidr(url):
+                # If input is a valid CIDR formatter will ignore it to let it become a CIDR
+                formatted_urls.append('')
+                continue
 
-        try:
-            formatted_url = URLFormatter(url).output
+            try:
+                formatted_url = URLFormatter(url).output
 
-        except URLError:
-            demisto.debug(traceback.format_exc())
+            except URLError:
+                demisto.debug(traceback.format_exc())
+
+            except Exception:
+                demisto.debug(traceback.format_exc())
 
-        except Exception:
-            demisto.debug(traceback.format_exc())
+            finally:
+                formatted_urls.append(formatted_url)
 
-        finally:
-            formatted_urls.append(formatted_url)
+        output = [{
+            'Type': entryTypes['note'],
+            'ContentsFormat': formats['json'],
+            'Contents': [urls],
+            'EntryContext': {'URL': urls},
+        } for urls in formatted_urls]
 
-    output = [{
-        'Type': entryTypes['note'],
-        'ContentsFormat': formats['json'],
-        'Contents': [urls],
-        'EntryContext': {'URL': urls},
-    } for urls in formatted_urls]
+        for url in output:
+            demisto.results(url)
 
-    for url in output:
-        demisto.results(url)
+    except Exception as e:
+        return_error(
+            f'Failed to execute the automation. Error: \n{str(e)}'
+        )
 
 
 if __name__ in ('__main__', '__builtin__', 'builtins'):

diff --git a/Packs/CommonScripts/Scripts/FormatURL/FormatURL.yml b/Packs/CommonScripts/Scripts/FormatURL/FormatURL.yml
@@ -18,7 +18,7 @@ tags:
 timeout: '0'
 type: python
 subtype: python3
-dockerimage: demisto/python3:3.10.12.63474
+dockerimage: demisto/python3:3.10.13.80593
 fromversion: 5.5.0
 tests:
 - FormatURL-Test

diff --git a/Packs/CommonScripts/pack_metadata.json b/Packs/CommonScripts/pack_metadata.json
@@ -2,7 +2,7 @@
     "name": "Common Scripts",
     "description": "Frequently used scripts pack.",
     "support": "xsoar",
-    "currentVersion": "1.12.49",
+    "currentVersion": "1.12.50",
     "author": "Cortex XSOAR",
     "url": "https://www.paloaltonetworks.com/cortex",
     "email": "",

diff --git a/Tests/conf.json b/Tests/conf.json
@@ -2976,7 +2976,8 @@
         },
         {
             "playbookID": "FormattingPerformance - Test",
-            "fromversion": "5.0.0"
+            "fromversion": "5.0.0",
+            "toversion": "7.9.9"
         },
         {
             "integrations": "AWS - EC2",