From f65c7490e3b80f45bf766b93b02054cf2fb2d766 Mon Sep 17 00:00:00 2001 From: Menachem Weinfeld <90556466+mmhw@users.noreply.github.com> Date: Tue, 28 Nov 2023 12:57:28 +0200 Subject: [PATCH] [transformers] Enhance to be more durable #2 (#31063) * ExtractDomainAndFQDNFromUrlAndEmail * ExtractEmailV2 * ruff fix * FormatURL * Bump pack from version CommonScripts to 1.12.47. * Bump pack from version CommonScripts to 1.12.48. * Bump pack from version CommonScripts to 1.12.49. * Added unit tests * Update the docker image * Bump pack from version CommonScripts to 1.12.50. * Update Packs/CommonScripts/ReleaseNotes/1_12_50.md Co-authored-by: Dean Arbel * Update Packs/CommonScripts/ReleaseNotes/1_12_50.md Co-authored-by: Dean Arbel * Update the RN * Update the docker image * empty * Skip "FormattingPerformance - Test" TPB in XSOAR 8 --------- Co-authored-by: Content Bot Co-authored-by: Dean Arbel --- Packs/CommonScripts/ReleaseNotes/1_12_50.md | 18 +++++++ .../ExtractDomainAndFQDNFromUrlAndEmail.py | 52 +++++++++++-------- .../ExtractDomainAndFQDNFromUrlAndEmail.yml | 2 +- ...xtractDomainAndFQDNFromUrlAndEmail_test.py | 19 +++++++ .../ExtractEmailFormatting.py | 39 +++++++------- .../ExtractEmailFormatting.yml | 2 +- .../ExtractEmailFormatting_test.py | 23 +++++++- .../Scripts/FormatURL/FormatURL.py | 50 ++++++++++-------- .../Scripts/FormatURL/FormatURL.yml | 2 +- Packs/CommonScripts/pack_metadata.json | 2 +- Tests/conf.json | 3 +- 11 files changed, 142 insertions(+), 70 deletions(-) create mode 100644 Packs/CommonScripts/ReleaseNotes/1_12_50.md diff --git a/Packs/CommonScripts/ReleaseNotes/1_12_50.md b/Packs/CommonScripts/ReleaseNotes/1_12_50.md new file mode 100644 index 000000000000..0e74608adc15 --- /dev/null +++ b/Packs/CommonScripts/ReleaseNotes/1_12_50.md @@ -0,0 +1,18 @@ + +#### Scripts + +##### FormatURL + +- Improved error handling for better performance when the script fails. +- Updated the Docker image to: *demisto/python3:3.10.13.80593*. + +##### ExtractDomainAndFQDNFromUrlAndEmail + +- Updated the Docker image to: *demisto/py3-tools:1.0.0.81280*. +- Improved error handling for better performance when the script fails. + + +##### ExtractEmailV2 + +- Updated the Docker image to: *demisto/python3:3.10.13.80593*. +- Improved error handling for better performance when the script fails. diff --git a/Packs/CommonScripts/Scripts/ExtractDomainAndFQDNFromUrlAndEmail/ExtractDomainAndFQDNFromUrlAndEmail.py b/Packs/CommonScripts/Scripts/ExtractDomainAndFQDNFromUrlAndEmail/ExtractDomainAndFQDNFromUrlAndEmail.py index 1aab01312b21..c7befaf4e89a 100644 --- a/Packs/CommonScripts/Scripts/ExtractDomainAndFQDNFromUrlAndEmail/ExtractDomainAndFQDNFromUrlAndEmail.py +++ b/Packs/CommonScripts/Scripts/ExtractDomainAndFQDNFromUrlAndEmail/ExtractDomainAndFQDNFromUrlAndEmail.py @@ -7,7 +7,7 @@ PROOFPOINT_PREFIXES = ['https://urldefense.proofpoint.com/', "https://urldefense.com/"] ATP_LINK_REG = r'(https:\/\/\w*|\w*)\.safelinks\.protection\.outlook\.com/' -DOMAIN_REGEX = r"(?i)(?P(?:http|ftp|hxxp)s?(?:://|-3A__|%3A%2F%2F))?(?P(?:[\w\-–_]+(?:\.|\[\.\]))+[^\W\d_]{2,})(?:[_/\s\"',)\]}>]|[.]\s?|%2F|.?$)" # noqa: E501 +DOMAIN_REGEX = r"(?i)(?P(?:http|ftp|hxxp)s?(?:://|-3A__|%3A%2F%2F))?(?P(?:[\w\-–_]+(?:\.|\[\.\]))+[^\W\d_]{2,})(?:[_/\s\"',)\]}>]|[.]\s?|%2F|.?$)" # noqa: E501, RUF001 def atp_get_original_url(safe_url): # pragma: no cover @@ -60,7 +60,7 @@ def get_fqdn(the_input): # get the subdomain using tld.subdomain subdomain = domain.subdomain if (subdomain): - fqdn = "{}.{}".format(subdomain, domain.fld) + fqdn = f"{subdomain}.{domain.fld}" else: fqdn = domain.fld @@ -109,27 +109,33 @@ def extract_fqdn(the_input): def main(): - the_input = demisto.args().get('input') - - # argToList returns the argument as is if it's already a list so no need to check here - the_input = argToList(the_input) - entries_list = [] - # Otherwise assumes it's already an array - for item in the_input: - input_entry = { - "Type": entryTypes["note"], - "ContentsFormat": formats["json"], - "Contents": [extract_fqdn(item)], - "EntryContext": {"Domain": item} - } - if input_entry.get("Contents") == ['']: - input_entry['Contents'] = [] - entries_list.append(input_entry) - if entries_list: - demisto.results(entries_list) - else: - # Return empty string so it wouldn't create an empty domain indicator. - demisto.results('') + try: + the_input = demisto.args().get('input') + + # argToList returns the argument as is if it's already a list so no need to check here + the_input = argToList(the_input) + entries_list = [] + # Otherwise assumes it's already an array + for item in the_input: + input_entry = { + "Type": entryTypes["note"], + "ContentsFormat": formats["json"], + "Contents": [extract_fqdn(item)], + "EntryContext": {"Domain": item} + } + if input_entry.get("Contents") == ['']: + input_entry['Contents'] = [] + entries_list.append(input_entry) + if entries_list: + demisto.results(entries_list) + else: + # Return empty string so it wouldn't create an empty domain indicator. + demisto.results('') + + except Exception as e: + return_error( + f'Failed to execute the automation. Error: \n{str(e)}' + ) # python2 uses __builtin__ python3 uses builtins diff --git a/Packs/CommonScripts/Scripts/ExtractDomainAndFQDNFromUrlAndEmail/ExtractDomainAndFQDNFromUrlAndEmail.yml b/Packs/CommonScripts/Scripts/ExtractDomainAndFQDNFromUrlAndEmail/ExtractDomainAndFQDNFromUrlAndEmail.yml index 5714add273a2..158aa6010ece 100644 --- a/Packs/CommonScripts/Scripts/ExtractDomainAndFQDNFromUrlAndEmail/ExtractDomainAndFQDNFromUrlAndEmail.yml +++ b/Packs/CommonScripts/Scripts/ExtractDomainAndFQDNFromUrlAndEmail/ExtractDomainAndFQDNFromUrlAndEmail.yml @@ -15,7 +15,7 @@ tags: - indicator-format timeout: '0' type: python -dockerimage: demisto/py3-tools:1.0.0.63856 +dockerimage: demisto/py3-tools:1.0.0.81280 runas: DBotWeakRole tests: - ExtractDomainAndFQDNFromUrlAndEmail-Test diff --git a/Packs/CommonScripts/Scripts/ExtractDomainAndFQDNFromUrlAndEmail/ExtractDomainAndFQDNFromUrlAndEmail_test.py b/Packs/CommonScripts/Scripts/ExtractDomainAndFQDNFromUrlAndEmail/ExtractDomainAndFQDNFromUrlAndEmail_test.py index 8a4f481c72e3..40f0c5147e22 100644 --- a/Packs/CommonScripts/Scripts/ExtractDomainAndFQDNFromUrlAndEmail/ExtractDomainAndFQDNFromUrlAndEmail_test.py +++ b/Packs/CommonScripts/Scripts/ExtractDomainAndFQDNFromUrlAndEmail/ExtractDomainAndFQDNFromUrlAndEmail_test.py @@ -1,3 +1,4 @@ +from pytest_mock import MockFixture import demistomock as demisto from ExtractDomainAndFQDNFromUrlAndEmail import extract_fqdn, main import pytest @@ -72,3 +73,21 @@ def test_extract_fqdn_or_domain_empty_indicators(mocker): results = demisto.results.call_args[0] assert results[0] == [{'Contents': [], 'ContentsFormat': 'json', 'Type': 1, 'EntryContext': {'Domain': '1Ab.Vt'}}] + + +def test_main_raise_error(mocker: MockFixture): + """ + Given: + - Exception during the automation + When: + - Running the automation + Then: + - Ensure the return_error is called with the correct error message. + """ + return_error_mock = mocker.patch('ExtractDomainAndFQDNFromUrlAndEmail.return_error') + mocker.patch('ExtractDomainAndFQDNFromUrlAndEmail.argToList', side_effect=Exception('Test Exception')) + + main() + + assert return_error_mock.call_count == 1 + assert 'Error: \nTest Exception' in return_error_mock.call_args[0][0] diff --git a/Packs/CommonScripts/Scripts/ExtractEmailFormatting/ExtractEmailFormatting.py b/Packs/CommonScripts/Scripts/ExtractEmailFormatting/ExtractEmailFormatting.py index 2c540d2d637a..05c58783e627 100644 --- a/Packs/CommonScripts/Scripts/ExtractEmailFormatting/ExtractEmailFormatting.py +++ b/Packs/CommonScripts/Scripts/ExtractEmailFormatting/ExtractEmailFormatting.py @@ -48,10 +48,7 @@ def check_tld(email_address: str) -> bool: Boolean: True if it's not a common file extension """ - if email_address.split(".")[-1] not in COMMON_FILE_EXT: - return True - else: - return False + return email_address.split(".")[-1] not in COMMON_FILE_EXT def refang_email(email_address: str) -> str: @@ -91,24 +88,30 @@ def extract_email_from_url_query(email_address: str) -> str: def main(): list_results = [] - emails = argToList(demisto.args().get('input')) + try: + emails = argToList(demisto.args().get('input')) - clean_emails = [extract_email(address) for address in emails] + clean_emails = [extract_email(address) for address in emails] - list_results = [refang_email(email_address) for email_address in clean_emails] + list_results = [refang_email(email_address) for email_address in clean_emails] - output = [ - { - 'Type': entryTypes['note'], - 'ContentsFormat': formats['json'], - 'Contents': [email_address] if email_address else [], - 'EntryContext': {'Email': email_address} if email_address else {}, - } for email_address in list_results] + output = [ + { + 'Type': entryTypes['note'], + 'ContentsFormat': formats['json'], + 'Contents': [email_address] if email_address else [], + 'EntryContext': {'Email': email_address} if email_address else {}, + } for email_address in list_results] - if output: - return_results(output) - else: - return_results('') + if output: + return_results(output) + else: + return_results('') + + except Exception as e: + return_error( + f'Failed to execute the automation. Error: \n{str(e)}' + ) if __name__ in ('__main__', 'builtin', 'builtins'): diff --git a/Packs/CommonScripts/Scripts/ExtractEmailFormatting/ExtractEmailFormatting.yml b/Packs/CommonScripts/Scripts/ExtractEmailFormatting/ExtractEmailFormatting.yml index a9c468cd8cb7..cf575fa0a393 100644 --- a/Packs/CommonScripts/Scripts/ExtractEmailFormatting/ExtractEmailFormatting.yml +++ b/Packs/CommonScripts/Scripts/ExtractEmailFormatting/ExtractEmailFormatting.yml @@ -15,7 +15,7 @@ args: isArray: true scripttarget: 0 subtype: python3 -dockerimage: demisto/python3:3.10.12.63474 +dockerimage: demisto/python3:3.10.13.80593 fromversion: 5.5.0 tests: - ExtractEmailV2-Test diff --git a/Packs/CommonScripts/Scripts/ExtractEmailFormatting/ExtractEmailFormatting_test.py b/Packs/CommonScripts/Scripts/ExtractEmailFormatting/ExtractEmailFormatting_test.py index a9653a40480a..a30e1524415a 100644 --- a/Packs/CommonScripts/Scripts/ExtractEmailFormatting/ExtractEmailFormatting_test.py +++ b/Packs/CommonScripts/Scripts/ExtractEmailFormatting/ExtractEmailFormatting_test.py @@ -1,3 +1,4 @@ +from pytest_mock import MockFixture from ExtractEmailFormatting import main, check_tld, extract_email, refang_email, extract_email_from_url_query import pytest import demistomock as demisto @@ -79,7 +80,7 @@ def test_main(mocker): mocker.patch.object(demisto, 'results') main() results = [email_address['Contents'] for email_address in demisto.results.call_args[0][0]] - assert EXPECTED_RESULTS == results + assert results == EXPECTED_RESULTS def test_main_invalid_emails(mocker): @@ -94,4 +95,22 @@ def test_main_invalid_emails(mocker): mocker.patch.object(demisto, 'args', return_value={"input": ''}) mocker.patch.object(demisto, 'results') main() - assert '' == demisto.results.call_args[0][0] + assert demisto.results.call_args[0][0] == '' + + +def test_main_raise_error(mocker: MockFixture): + """ + Given: + - Exception during the automation + When: + - Running the automation + Then: + - Ensure the return_error is called with the correct error message. + """ + return_error_mock = mocker.patch('ExtractEmailFormatting.return_error') + mocker.patch('ExtractEmailFormatting.argToList', side_effect=Exception('Test Exception')) + + main() + + assert return_error_mock.call_count == 1 + assert 'Error: \nTest Exception' in return_error_mock.call_args[0][0] diff --git a/Packs/CommonScripts/Scripts/FormatURL/FormatURL.py b/Packs/CommonScripts/Scripts/FormatURL/FormatURL.py index 8d394f768fe5..782b9a74ce4d 100644 --- a/Packs/CommonScripts/Scripts/FormatURL/FormatURL.py +++ b/Packs/CommonScripts/Scripts/FormatURL/FormatURL.py @@ -742,35 +742,41 @@ def main(): formatted_urls: List[str] = [] - for url in raw_urls: - formatted_url = '' + try: + for url in raw_urls: + formatted_url = '' - if _is_valid_cidr(url): - # If input is a valid CIDR formatter will ignore it to let it become a CIDR - formatted_urls.append('') - continue + if _is_valid_cidr(url): + # If input is a valid CIDR formatter will ignore it to let it become a CIDR + formatted_urls.append('') + continue - try: - formatted_url = URLFormatter(url).output + try: + formatted_url = URLFormatter(url).output - except URLError: - demisto.debug(traceback.format_exc()) + except URLError: + demisto.debug(traceback.format_exc()) + + except Exception: + demisto.debug(traceback.format_exc()) - except Exception: - demisto.debug(traceback.format_exc()) + finally: + formatted_urls.append(formatted_url) - finally: - formatted_urls.append(formatted_url) + output = [{ + 'Type': entryTypes['note'], + 'ContentsFormat': formats['json'], + 'Contents': [urls], + 'EntryContext': {'URL': urls}, + } for urls in formatted_urls] - output = [{ - 'Type': entryTypes['note'], - 'ContentsFormat': formats['json'], - 'Contents': [urls], - 'EntryContext': {'URL': urls}, - } for urls in formatted_urls] + for url in output: + demisto.results(url) - for url in output: - demisto.results(url) + except Exception as e: + return_error( + f'Failed to execute the automation. Error: \n{str(e)}' + ) if __name__ in ('__main__', '__builtin__', 'builtins'): diff --git a/Packs/CommonScripts/Scripts/FormatURL/FormatURL.yml b/Packs/CommonScripts/Scripts/FormatURL/FormatURL.yml index 8a0b30ebadb7..5a9faeee848f 100644 --- a/Packs/CommonScripts/Scripts/FormatURL/FormatURL.yml +++ b/Packs/CommonScripts/Scripts/FormatURL/FormatURL.yml @@ -18,7 +18,7 @@ tags: timeout: '0' type: python subtype: python3 -dockerimage: demisto/python3:3.10.12.63474 +dockerimage: demisto/python3:3.10.13.80593 fromversion: 5.5.0 tests: - FormatURL-Test diff --git a/Packs/CommonScripts/pack_metadata.json b/Packs/CommonScripts/pack_metadata.json index 66b425c390ec..382955569dc3 100644 --- a/Packs/CommonScripts/pack_metadata.json +++ b/Packs/CommonScripts/pack_metadata.json @@ -2,7 +2,7 @@ "name": "Common Scripts", "description": "Frequently used scripts pack.", "support": "xsoar", - "currentVersion": "1.12.49", + "currentVersion": "1.12.50", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "", diff --git a/Tests/conf.json b/Tests/conf.json index a14245a84f5a..80e6c0af856a 100644 --- a/Tests/conf.json +++ b/Tests/conf.json @@ -2976,7 +2976,8 @@ }, { "playbookID": "FormattingPerformance - Test", - "fromversion": "5.0.0" + "fromversion": "5.0.0", + "toversion": "7.9.9" }, { "integrations": "AWS - EC2",