From bb29b744a60413d66e1b5fac14dee094296e2656 Mon Sep 17 00:00:00 2001 From: Shahaf Ben Yakir <44666568+ShahafBenYakir@users.noreply.github.com> Date: Fri, 23 Jun 2023 12:17:31 +0300 Subject: [PATCH] Threat connect outputs (#27257) * refactored code to align with standard * Fixed owner argument * Fixed unittests * Mypy fixes * reverted name change * reverted argument deprecation * Added RN and version Fixed argument description * Added RN and version Fixed argument description * bumped docker * fxied docs * fixed docs * adjusted readme * removed commented code --- .../Integrations/ThreatConnectV3/README.md | 135 +++++---- .../ThreatConnectV3/ThreatConnectV3.py | 281 ++++++++---------- .../ThreatConnectV3/ThreatConnectV3.yml | 156 ++++------ .../ThreatConnectV3/ThreatConnectV3_test.py | 2 + Packs/ThreatConnect/ReleaseNotes/3_0_5.md | 9 + Packs/ThreatConnect/pack_metadata.json | 2 +- 6 files changed, 259 insertions(+), 326 deletions(-) create mode 100644 Packs/ThreatConnect/ReleaseNotes/3_0_5.md diff --git a/Packs/ThreatConnect/Integrations/ThreatConnectV3/README.md b/Packs/ThreatConnect/Integrations/ThreatConnectV3/README.md index 5c939aed0c2d..5d71a3733a09 100644 --- a/Packs/ThreatConnect/Integrations/ThreatConnectV3/README.md +++ b/Packs/ThreatConnect/Integrations/ThreatConnectV3/README.md @@ -33,7 +33,7 @@ For more information - click [here](https://training.threatconnect.com/learn/art | First fetch timestamp (<number> <time unit>, for example, 12 hours, 7 days, 3 months, 1 year) | | True | | Incident Metadata | The metadata to collect. | False | | Source Reliability | Reliability of the source providing the intelligence data. | True | - | Rating Threshold for Malicious Indicators (needed for reputation calculation) | Rating Threshold for Malicious Indicators, it is necessary to calculate reputation. | False | + | Rating Threshold for Malicious Indicators (needed for reputation calculation) | Rating Threshold for Malicious Indicators. This is necessary to calculate reputation. | False | | Confidence Threshold for Malicious Indicators (needed for reputation calculation) | Confidence Threshold for Malicious Indicators. This is necessary to calculate reputation. | False | | Indicator Reputation Freshness in days (needed for reputation calculation) | Indicator Reputation Freshness.This is necessary to calculate reputation. | False | | Trust any certificate (not secure) | Whether or not to trust any certificate| False | @@ -41,17 +41,21 @@ For more information - click [here](https://training.threatconnect.com/learn/art | Maximum number of incidents to fetch | The maximum amount of incident to fetch per run | 200 | 4. Click **Test** to validate the URLs, token, and connection. + ## Commands + You can execute these commands from the Cortex XSOAR CLI, as part of an automation, or in a playbook. After you successfully execute a command, a DBot message appears in the War Room with the command details. + ### ip + *** Searches for an indicator of type IP address. - #### Base Command `ip` + #### Input | **Argument Name** | **Description** | **Required** | @@ -61,7 +65,6 @@ Searches for an indicator of type IP address. | ratingThreshold | A comma-separated list of results filtered by indicators whose threat rating is greater than the specified value. Can be "0" - "Unknown", "1" - "Suspicious", "2" - "Low", "3" - Moderate, "4" - High, or "5" - "Critical". | Optional | | confidenceThreshold | A comma-separated list of results filtered by indicators whose confidence rating is greater than the specified value. Can be "0%" - "Unknown," "1% " - "Discredited", "2-29%" - "Improbable," "30-49%" - "Doubtful," "50-69%" - "Possible", "70-89%" - "Probable," or "90-100%" - "Confirmed". | Optional | - #### Context Output | **Path** | **Type** | **Description** | @@ -86,13 +89,14 @@ Searches for an indicator of type IP address. | TC.Indicator.WebLink | string | The web link of the indicator. | ### url + *** Searches for an indicator of type URL. - #### Base Command `url` + #### Input | **Argument Name** | **Description** | **Required** | @@ -102,7 +106,6 @@ Searches for an indicator of type URL. | ratingThreshold | A comma-separated list of results filtered by indicators whose threat rating is greater than the specified value. Can be "0" - "Unknown", "1" - "Suspicious", "2" - "Low", "3" - Moderate, "4" - High, or "5" - "Critical". | Optional | | confidenceThreshold | A comma-separated list of results filtered by indicators whose confidence rating is greater than the specified value. Can be "0%" - "Unknown," "1% " - "Discredited", "2-29%" - "Improbable," "30-49%" - "Doubtful," "50-69%" - "Possible", "70-89%" - "Probable," or "90-100%" - "Confirmed". | Optional | - #### Context Output | **Path** | **Type** | **Description** | @@ -127,13 +130,14 @@ Searches for an indicator of type URL. | TC.Indicator.WebLink | string | The web link of the indicator. | ### file + *** Searches for an indicator of type file. - #### Base Command `file` + #### Input | **Argument Name** | **Description** | **Required** | @@ -143,7 +147,6 @@ Searches for an indicator of type file. | ratingThreshold | A comma-separated list of results filtered by indicators whose threat rating is greater than the specified value. Can be "0" - "Unknown", "1" - "Suspicious", "2" - "Low", "3" - Moderate, "4" - High, or "5" - "Critical". | Optional | | confidenceThreshold | A comma-separated list of results filtered by indicators whose confidence rating is greater than the specified value. Can be "0%" - "Unknown," "1% " - "Discredited", "2-29%" - "Improbable," "30-49%" - "Doubtful," "50-69%" - "Possible", "70-89%" - "Probable," or "90-100%" - "Confirmed". | Optional | - #### Context Output | **Path** | **Type** | **Description** | @@ -173,19 +176,19 @@ Searches for an indicator of type file. | TC.Indicator.WebLink | string | The web link of the indicator. | ### tc-owners + *** Retrieves all owners for the current account. - #### Base Command `tc-owners` + #### Input | **Argument Name** | **Description** | **Required** | | --- | --- | --- | - #### Context Output | **Path** | **Type** | **Description** | @@ -195,13 +198,14 @@ Retrieves all owners for the current account. | TC.Owner.Type | string | The type of the owner. | ### tc-indicators + *** Retrieves a list of all indicators. - #### Base Command `tc-indicators` + #### Input | **Argument Name** | **Description** | **Required** | @@ -209,7 +213,7 @@ Retrieves a list of all indicators. | owner | A comma-separated list of results filtered by the owner of the indicator. | Optional | | page | The page to take the results from. | Optional | | limit | The maximum number of results that can be returned. The default is 500. | Optional | - +| fields_to_return | Comma separated list of additional fields to return as part of the result indicator metadata. Possible values are: associatedGroups, associatedIndicators, observations, tags, and attributes. | Optional | #### Context Output @@ -250,13 +254,14 @@ Retrieves a list of all indicators. | TC.Indicator.WebLink | string | The web link of the indicator. | ### tc-get-tags + *** Returns a list of all ThreatConnect tags. - #### Base Command `tc-get-tags` + #### Input | **Argument Name** | **Description** | **Required** | @@ -265,7 +270,6 @@ Returns a list of all ThreatConnect tags. | limit | The maximum number of results that can be returned. The default is 500. | Optional | | name | The name of the tag to get. | Optional | - #### Context Output | **Path** | **Type** | **Description** | @@ -273,13 +277,14 @@ Returns a list of all ThreatConnect tags. | TC.Tags | Unknown | A list of tags. | ### tc-tag-indicator + *** Adds a tag to an existing indicator. - #### Base Command `tc-tag-indicator` + #### Input | **Argument Name** | **Description** | **Required** | @@ -287,18 +292,18 @@ Adds a tag to an existing indicator. | tag | The name of the tag. | Required | | indicator | The indicator to tag. For example, for an IP indicator, "8.8.8.8". | Required | - #### Context Output There is no context output for this command. ### tc-get-indicator + *** Retrieves information about an indicator. - #### Base Command `tc-get-indicator` + #### Input | **Argument Name** | **Description** | **Required** | @@ -351,13 +356,14 @@ Retrieves information about an indicator. | TC.Indicator.WebLink | string | The web link of the indicator. | ### tc-get-indicators-by-tag + *** Fetches all indicators that have a tag. - #### Base Command `tc-get-indicators-by-tag` + #### Input | **Argument Name** | **Description** | **Required** | @@ -407,13 +413,14 @@ Fetches all indicators that have a tag. | TC.Indicator.WebLink | string | The web link of the indicator. | ### tc-add-indicator + *** Adds a new indicator to ThreatConnect. - #### Base Command `tc-add-indicator` + #### Input | **Argument Name** | **Description** | **Required** | @@ -426,7 +433,6 @@ Adds a new indicator to ThreatConnect. | description | The description of the indicator. | Optional | | tags | A comma-separated list of the tags to apply to the campaign. | Optional | - #### Context Output | **Path** | **Type** | **Description** | @@ -461,13 +467,14 @@ Adds a new indicator to ThreatConnect. | TC.Indicator.WebLink | string | The web link of the indicator. | ### tc-create-incident + *** Creates a new incident group. - #### Base Command `tc-create-incident` + #### Input | **Argument Name** | **Description** | **Required** | @@ -478,7 +485,6 @@ Creates a new incident group. | securityLabel | The security label applied to the incident. Possible values are: TLP:RED, TLP:GREEN, TLP:AMBER, TLP:WHITE. | Optional | | description | The description of the incident. | Optional | - #### Context Output | **Path** | **Type** | **Description** | @@ -491,13 +497,14 @@ Creates a new incident group. | TC.Incident.ID | Unknown | The ID of the new incident. | ### tc-incident-associate-indicator + *** Associates an indicator with an existing incident. The indicator must exist before running this command. To add an indicator, run the tc-add-indicator command. - #### Base Command `tc-incident-associate-indicator` + #### Input | **Argument Name** | **Description** | **Required** | @@ -505,7 +512,6 @@ Associates an indicator with an existing incident. The indicator must exist befo | incidentId | The ID of the incident to which the indicator is associated. | Required | | indicator | The ID of the indicator. | Required | - #### Context Output | **Path** | **Type** | **Description** | @@ -540,13 +546,14 @@ Associates an indicator with an existing incident. The indicator must exist befo | TC.Indicator.WebLink | string | The web link of the indicator. | ### domain + *** Searches for an indicator of type domain. - #### Base Command `domain` + #### Input | **Argument Name** | **Description** | **Required** | @@ -556,7 +563,6 @@ Searches for an indicator of type domain. | ratingThreshold | A comma-separated list of results filtered by indicators whose threat rating is greater than the specified value. Can be "0" - "Unknown", "1" - "Suspicious", "2" - "Low", "3" - Moderate, "4" - High, or "5" - "Critical". | Optional | | confidenceThreshold | A comma-separated list of results filtered by indicators whose confidence rating is greater than the specified value. Can be "0%" - "Unknown," "1% " - "Discredited", "2-29%" - "Improbable," "30-49%" - "Doubtful," "50-69%" - "Possible", "70-89%" - "Probable," or "90-100%" - "Confirmed". | Optional | - #### Context Output | **Path** | **Type** | **Description** | @@ -582,20 +588,20 @@ Searches for an indicator of type domain. | TC.Indicator.WebLink | string | The web link of the indicator. | ### tc-get-incident-associate-indicators + *** Returns indicators that are related to a specific incident. - #### Base Command `tc-get-incident-associate-indicators` + #### Input | **Argument Name** | **Description** | **Required** | | --- | --- | --- | | incidentId | The ID of the incident. | Required | - #### Context Output | **Path** | **Type** | **Description** | @@ -634,13 +640,14 @@ Returns indicators that are related to a specific incident. | File.Malicious.Description | string | For malicious files, the full description. | ### tc-update-indicator + *** Updates the indicator in ThreatConnect. - #### Base Command `tc-update-indicator` + #### Input | **Argument Name** | **Description** | **Required** | @@ -655,7 +662,6 @@ Updates the indicator in ThreatConnect. | securityLabel | The security label applied to the incident. Possible values are: TLP:RED, TLP:GREEN, TLP:AMBER, TLP:WHITE. | Optional | | tags | A comma-separated list of tags. | Optional | - #### Context Output | **Path** | **Type** | **Description** | @@ -690,13 +696,14 @@ Updates the indicator in ThreatConnect. | TC.Indicator.WebLink | string | The web link of the indicator. | ### tc-delete-indicator-tag + *** Removes a tag from a specified indicator. - #### Base Command `tc-delete-indicator-tag` + #### Input | **Argument Name** | **Description** | **Required** | @@ -704,7 +711,6 @@ Removes a tag from a specified indicator. | indicator | The ID of the indicator from which to remove a tag. | Required | | tag | The name of the tag to remove from the indicator. | Required | - #### Context Output | **Path** | **Type** | **Description** | @@ -739,31 +745,32 @@ Removes a tag from a specified indicator. | TC.Indicator.WebLink | string | The web link of the indicator. | ### tc-delete-indicator + *** Deletes an indicator from ThreatConnect. - #### Base Command `tc-delete-indicator` + #### Input | **Argument Name** | **Description** | **Required** | | --- | --- | --- | | indicator | The ID of the indicator to delete. | Required | - #### Context Output There is no context output for this command. ### tc-create-campaign + *** Creates a group based on the Campaign type. - #### Base Command `tc-create-campaign` + #### Input | **Argument Name** | **Description** | **Required** | @@ -774,7 +781,6 @@ Creates a group based on the Campaign type. | tag | Comma-separated list of the tags to apply to the campaign. | Optional | | securityLabel | The security label applied to the incident. Possible values are: TLP:RED, TLP:GREEN, TLP:AMBER, TLP:WHITE. | Optional | - #### Context Output | **Path** | **Type** | **Description** | @@ -787,13 +793,14 @@ Creates a group based on the Campaign type. | TC.Campaign.ID | string | The ID of the campaign. | ### tc-create-event + *** Creates a group based on the Event type. - #### Base Command `tc-create-event` + #### Input | **Argument Name** | **Description** | **Required** | @@ -819,13 +826,14 @@ Creates a group based on the Event type. | TC.Event.Type | string | The type of the event. | ### tc-create-threat + *** Creates a group based on the "Threats" type. - #### Base Command `tc-create-threat` + #### Input | **Argument Name** | **Description** | **Required** | @@ -836,7 +844,6 @@ Creates a group based on the "Threats" type. | securityLabel | The security label applied to the threat. Possible values are: TLP:RED, TLP:GREEN, TLP:AMBER, TLP:WHITE. | Optional | | description | The description of the threat. | Optional | - #### Context Output | **Path** | **Type** | **Description** | @@ -845,42 +852,42 @@ Creates a group based on the "Threats" type. | TC.Threat.ID | string | The ID of the threat. | ### tc-delete-group + *** Deletes a group. - #### Base Command `tc-delete-group` + #### Input | **Argument Name** | **Description** | **Required** | | --- | --- | --- | | groupID | A comma-separated list of the IDs of the groups to delete. | Required | - #### Context Output There is no context output for this command. ### tc-get-events + *** Returns a list of events. - #### Base Command `tc-get-events` + #### Input | **Argument Name** | **Description** | **Required** | | --- | --- | --- | -| fromDate | The date to retrieve groups from in the format yyyy-mm-dd, e.g., 1111-11-11. | Optional | +| fromDate | The date to retrieve groups from in the yyyy-mm-dd format, e.g., 1111-11-11. | Optional | | tag | The tag to retrieve groups by. | Optional | | page | The page to take the results from. | Optional | | limit | The maximum number of results that can be returned. The default is 500. | Optional | | id | A comma-separated list of IDs to filter the groups by. | Optional | -| filter | A free text TQL filter. (Refer [here](https://knowledge.threatconnect.com/docs/threatconnect-query-language-tql) for a basic TQL guide). | Optional | - +| filter | A free text TQL filter. Refer to https://knowledge.threatconnect.com/docs/threatconnect-query-language-tql for a basic TQL guide. | Optional | #### Context Output @@ -896,13 +903,14 @@ Returns a list of events. | TC.Event.Tags | String | The tags of the event. | ### tc-list-groups + *** Returns all groups. - #### Base Command `tc-list-groups` + #### Input | **Argument Name** | **Description** | **Required** | @@ -913,7 +921,7 @@ Returns all groups. | page | The page to take the results from. | Optional | | limit | The maximum number of results that can be returned. The default is 500. | Optional | | id | A comma-separated list of IDs to filter the groups by. | Optional | -| filter | A free text TQL filter. (Refer [here](https://knowledge.threatconnect.com/docs/threatconnect-query-language-tql) for a basic TQL guide). | Optional | +| filter | A free text TQL filter. Refer to https://knowledge.threatconnect.com/docs/threatconnect-query-language-tql for a basic TQL guide. | Optional | | include_tags | Add group tags metadata to the results. | Optional | | include_security_labels | Add group security labels metadata to the results. | Optional | | include_attributes | Add group attributes metadata to the results. | Optional | @@ -921,7 +929,6 @@ Returns all groups. | include_associated_indicators | Add group associated indicators metadata to the results. | Optional | | include_all_metaData | Add all group metadata to the results. | Optional | - #### Context Output | **Path** | **Type** | **Description** | @@ -934,13 +941,14 @@ Returns all groups. | TC.Group.ID | Number | The ID of the group. | ### tc-add-group-tag + *** Adds tags to a specified group. - #### Base Command `tc-add-group-tag` + #### Input | **Argument Name** | **Description** | **Required** | @@ -948,24 +956,23 @@ Adds tags to a specified group. | group_id | The ID of the group to which to add the tag. To get the ID, run the tc-list-groups command. | Required | | tag_name | The name of the tag to add to the group. | Required | - #### Context Output There is no context output for this command. ### tc-get-indicator-types + *** Returns all indicator types available. - #### Base Command `tc-get-indicator-types` + #### Input | **Argument Name** | **Description** | **Required** | | --- | --- | --- | - #### Context Output | **Path** | **Type** | **Description** | @@ -979,13 +986,14 @@ Returns all indicator types available. | TC.IndicatorType.Value1Label | String | The value label of the indicator. | ### tc-create-document-group + *** Creates a document group. - #### Base Command `tc-create-document-group` + #### Input | **Argument Name** | **Description** | **Required** | @@ -998,7 +1006,6 @@ Creates a document group. | description | A description of the group. | Optional | | entry_id | The ID of the entry, as displayed in the War Room. | Required | - #### Context Output | **Path** | **Type** | **Description** | @@ -1011,20 +1018,20 @@ Creates a document group. | TC.Group.ID | Number | The ID of the group to which the attribute was added. | ### tc-download-document + *** Downloads the contents of a document. - #### Base Command `tc-download-document` + #### Input | **Argument Name** | **Description** | **Required** | | --- | --- | --- | | document_id | The ID of the document. | Required | - #### Context Output | **Path** | **Type** | **Description** | @@ -1041,20 +1048,20 @@ Downloads the contents of a document. | File.Extension | String | The extension of the file. | ### tc-get-associated-groups + *** Returns groups associated with a specified group. - #### Base Command `tc-get-associated-groups` + #### Input | **Argument Name** | **Description** | **Required** | | --- | --- | --- | | group_id | The ID of the group. To get the ID, run the tc-list-groups command. | Required | - #### Context Output | **Path** | **Type** | **Description** | @@ -1066,38 +1073,38 @@ Returns groups associated with a specified group. | TC.Group.AssociatedGroup.Type | String | The type of the group. | ### tc-get-indicator-owners + *** Get the owner for an indicator. - #### Base Command `tc-get-indicator-owners` + #### Input | **Argument Name** | **Description** | **Required** | | --- | --- | --- | | indicator | Indicator ID. | Required | - #### Context Output There is no context output for this command. ### tc-download-report + *** The group report to download in PDF format. - #### Base Command `tc-download-report` + #### Input | **Argument Name** | **Description** | **Required** | | --- | --- | --- | | group_id | The ID of the group. | Required | - #### Context Output | **Path** | **Type** | **Description** | @@ -1114,13 +1121,14 @@ The group report to download in PDF format. | File.Extension | String | The extension of the file. | ### tc-update-group + *** Updates a group. - #### Base Command `tc-update-group` + #### Input | **Argument Name** | **Description** | **Required** | @@ -1131,11 +1139,10 @@ Updates a group. | security_label | The security label applied to the threat. Possible values are: TLP:RED, TLP:GREEN, TLP:AMBER, TLP:WHITE. | Optional | | associated_group_id | An ID to associate a group by. | Optional | | associated_indicator_id | An ID to associate an indicator by. | Optional | -| security_label | The type of update to the group metadata (associated indicators, attributes,tags, etc.). Possible values are: append, delete, replace. | Optional | +| mode | The type of update to the group metadata(associated indicators, attributes,tags etc.). Possible values are: append, delete, replace. | Optional | | attribute_value | The value of the attribute to associate. | Optional | | attribute_type | The type of the attribute to associate. | Optional | - #### Context Output | **Path** | **Type** | **Description** | diff --git a/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3.py b/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3.py index 7add900b7f6e..937cbb717808 100644 --- a/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3.py +++ b/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3.py @@ -49,7 +49,7 @@ def create_header(self, url_suffix: str, method: Method) -> dict: 'Content-Type': 'application/json'} -def create_context(indicators, include_dbot_score=False): +def create_context(indicators, include_dbot_score=False, fields_to_return: list = None): indicators_dbot_score = {} # type: dict params = demisto.params() rating_threshold = int(params.get('rating', '3')) @@ -74,7 +74,7 @@ def create_context(indicators, include_dbot_score=False): 'Host': 'hostName', 'File': 'md5' } - + human_readable = [] for ind in indicators: indicator_type = tc_type_to_demisto_type.get(ind['type'], ind['type']) value_field = type_to_value_field.get(ind['type'], 'summary') @@ -153,7 +153,7 @@ def create_context(indicators, include_dbot_score=False): dbot_object['Indicator'] = k indicators_dbot_score[k] = dbot_object - context[TC_INDICATOR_PATH].append({ + new_indicator = { 'ID': ind['id'], 'Name': value, 'Type': ind['type'], @@ -164,42 +164,30 @@ def create_context(indicators, include_dbot_score=False): 'Rating': rating, 'Confidence': confidence, 'WebLink': ind.get('webLink'), + } + # relevant for domain + if indicator_type == 'domain': + new_indicator['Active'] = ind.get('whoisActive') - # relevant for domain - 'Active': ind.get('whoisActive'), - - # relevant for file - 'File.MD5': md5, - 'File.SHA1': sha1, - 'File.SHA256': sha256, - }) - - if 'group_associations' in ind: - if ind['group_associations']: - context[TC_INDICATOR_PATH][0]['IndicatorGroups'] = ind['group_associations'] - - if 'indicator_associations' in ind: - if ind['indicator_associations']: - context[TC_INDICATOR_PATH][0]['IndicatorAssociations'] = ind[ - 'indicator_associations'] + # relevant for file + elif indicator_type == 'file': + new_indicator['File.MD5'] = md5 + new_indicator['File.SHA1'] = sha1 + new_indicator['File.SHA256'] = sha256 - if 'indicator_tags' in ind: - if ind['indicator_tags']: - context[TC_INDICATOR_PATH][0]['IndicatorTags'] = ind['indicator_tags'] + human_readable.append(new_indicator) - if 'indicator_observations' in ind: - if ind['indicator_observations']: - context[TC_INDICATOR_PATH][0]['IndicatorsObservations'] = ind[ - 'indicator_observations'] + # The context should not contain the additional fields (there are added a separate HR message after) + context_indicator = copy.deepcopy(new_indicator) + if fields_to_return: + for field in fields_to_return: + context_indicator[field.capitalize()] = ind.get(field) - if 'indicator_attributes' in ind: - if ind['indicator_attributes']: - context[TC_INDICATOR_PATH][0]['IndicatorAttributes'] = ind[ - 'indicator_attributes'] + context[TC_INDICATOR_PATH].append(context_indicator) context['DBotScore'] = list(indicators_dbot_score.values()) - context = {k: createContext(v, removeNull=True)[:MAX_CONTEXT] for k, v in context.items() if v} - return context, context.get(TC_INDICATOR_PATH, []) + context = {k: createContext(v)[:MAX_CONTEXT] for k, v in context.items() if v} + return context, human_readable def detection_to_incident(threatconnect_data: dict, threatconnect_date: str) -> dict: @@ -215,7 +203,7 @@ def detection_to_incident(threatconnect_data: dict, threatconnect_date: str) -> return incident -def get_indicators(client: Client, args_type: str, type_name: str, args: dict) -> None: # pragma: no cover +def get_indicator_reputation(client: Client, args_type: str, type_name: str, args: dict) -> None: # pragma: no cover owners_query = create_or_query(args.get('owners', demisto.params().get('defaultOrg')), 'ownerName') query = create_or_query(args.get(args_type), 'summary') # type: ignore rating_threshold = args.get('ratingThreshold', '') @@ -235,14 +223,15 @@ def get_indicators(client: Client, args_type: str, type_name: str, args: dict) - indicators = response.get('data') - ec, indicators = create_context(indicators, include_dbot_score=True) + ec, human_readable = create_context(indicators, include_dbot_score=True) return_results({ 'Type': entryTypes['note'], 'ContentsFormat': formats['json'], 'Contents': indicators, 'ReadableContentsFormat': formats['markdown'], - 'HumanReadable': tableToMarkdown('ThreatConnect URL Reputation for: {}'.format(args.get(args_type)), indicators, - headerTransform=pascalToSpace), + 'HumanReadable': tableToMarkdown('ThreatConnect URL Reputation for: {}'.format(args.get(args_type)), + human_readable, + headerTransform=pascalToSpace, removeNull=True), 'EntryContext': ec }) @@ -258,19 +247,19 @@ def create_or_query(delimiter_str: str, param_name: str, wrapper: str = '"') -> def get_ip_indicators(client: Client, args: dict): # pragma: no cover - return get_indicators(client, 'ip', 'Address', args) + return get_indicator_reputation(client, 'ip', 'Address', args) def get_url_indicators(client: Client, args: dict): # pragma: no cover - return get_indicators(client, 'url', 'URL', args) + return get_indicator_reputation(client, 'url', 'URL', args) def get_domain_indicators(client: Client, args: dict): # pragma: no cover - return get_indicators(client, 'domain', 'Host', args) + return get_indicator_reputation(client, 'domain', 'Host', args) def get_file_indicators(client: Client, args: dict): - return get_indicators(client, 'file', 'File', args) + return get_indicator_reputation(client, 'file', 'File', args) def tc_delete_group_command(client: Client, args: dict) -> Any: # pragma: no cover @@ -297,72 +286,34 @@ def tc_delete_group_command(client: Client, args: dict) -> Any: # pragma: no co }) -def tc_get_indicators_command(client: Client, args: dict, confidence_threshold: str = '', rating_threshold: str = '', - tag: str = '', owners: str = '', indicator_id: str = '', indicator_type: str = '', - return_raw=False, group_associations: str = 'false', summary: str = '', - indicator_associations: str = 'false', - indicator_observations: str = 'false', indicator_tags: str = 'false', - indicator_attributes: str = 'false') -> Any: # pragma: no cover - owners = args.get('owner', owners) - limit = args.get('limit', '500') - page = args.get('page', '0') - tag = args.get('tag', tag) - indicator_type = args.get('type', indicator_type) - indicator_id = args.get('id', indicator_id) - rating_threshold = args.get('ratingThreshold', rating_threshold) - confidence_threshold = args.get('confidenceThreshold', confidence_threshold) - - indicator_attributes = args.get('indicator_attributes', indicator_attributes) - indicator_tags = args.get('indicator_tags', indicator_tags) - indicator_observations = args.get('indicator_observations', indicator_observations) - indicator_associations = args.get('indicator_associations', indicator_associations) - group_associations = args.get('group_associations', group_associations) +def tc_get_indicators(client: Client, tag: str = '', page: str = '0', limit: str = '500', owners: str = '', + indicator_id: str = '', summary: str = '', + fields_to_return: list = None) -> Any: # pragma: no cover tql_prefix = '' if summary: summary = f' AND summary EQ "{summary}"' tql_prefix = '?tql=' - if rating_threshold: - rating_threshold = f'AND (rating > {rating_threshold}) ' - tql_prefix = '?tql=' - if confidence_threshold: - confidence_threshold = f'AND (confidence > {confidence_threshold}) ' - tql_prefix = '?tql=' + if tag: tag = f' AND tag LIKE "%{tag}%"' - indicator_tags = 'true' - tql_prefix = '?tql=' - if indicator_type: - indicator_type = f' AND typeName EQ "{indicator_type}"' tql_prefix = '?tql=' + if owners: - owners = ' AND ' + create_or_query(args.get('owner', owners), 'ownerName') + owners = ' AND ' + create_or_query(owners, 'ownerName') tql_prefix = '?tql=' if indicator_id: - indicator_id = ' AND ' + create_or_query(args.get('id', indicator_id), 'id').replace('"', '') + indicator_id = ' AND ' + create_or_query(indicator_id, 'id').replace('"', '') tql_prefix = '?tql=' - fields = set_fields({'associatedGroups': group_associations, 'associatedIndicators': indicator_associations, - 'observations': indicator_observations, 'tags': indicator_tags, - 'attributes': indicator_attributes}) - tql = f'{indicator_id}{summary}{indicator_type}{owners}{tag}{confidence_threshold}' \ - f'{rating_threshold}'.replace(' AND ', '', 1) + fields = set_fields(fields_to_return) + + tql = f'{indicator_id}{summary}{owners}{tag}'.replace(' AND ', '', 1) tql = urllib.parse.quote(tql.encode('utf8')) url = f'/api/v3/indicators{tql_prefix}{tql}{fields}&resultStart={page}&resultLimit={limit}' if not tql_prefix: url = url.replace('&', '?', 1) response = client.make_request(Method.GET, url) - if return_raw: - return response.get('data') - indicators = response.get('data') - ec, indicators = create_context(indicators, include_dbot_score=True) - return_results({ - 'Type': entryTypes['note'], - 'ContentsFormat': formats['json'], - 'Contents': response.get('data'), - 'ReadableContentsFormat': formats['markdown'], - 'HumanReadable': tableToMarkdown('ThreatConnect Indicators:', indicators, headerTransform=pascalToSpace), - 'EntryContext': ec - }) + return response.get('data') def tc_get_owners_command(client: Client, args: dict) -> Any: # pragma: no cover # type: ignore # noqa @@ -388,6 +339,24 @@ def tc_get_owners_command(client: Client, args: dict) -> Any: # pragma: no cove }) +def tc_get_indicators_command(client: Client, args: dict): + owners = args.get('owner', '') + limit = args.get('limit', '500') + page = args.get('page', '0') + fields_to_return = argToList(args.get('fields_to_return') or []) + indicators = tc_get_indicators(client=client, owners=owners, limit=limit, page=page) + ec, human_readable = create_context(indicators, include_dbot_score=True, fields_to_return=fields_to_return) + return_results({ + 'Type': entryTypes['note'], + 'ContentsFormat': formats['json'], + 'Contents': indicators, + 'ReadableContentsFormat': formats['markdown'], + 'HumanReadable': tableToMarkdown('ThreatConnect Indicators:', human_readable, headerTransform=pascalToSpace, + removeNull=True), + 'EntryContext': ec + }) + + def tc_get_indicator_owners(client: Client, args: dict) -> Any: # pragma: no cover indicator = args.get('indicator') url = f'/api/v3/indicators/{indicator}' @@ -454,7 +423,7 @@ def get_last_run_time(groups: list) -> str: def convert_to_dict(arr: list): new_dict = {} for item in arr: - new_dict[item] = 'true' + new_dict[item] = True return new_dict @@ -464,7 +433,7 @@ def fetch_incidents(client: Client, args: dict) -> None: # pragma: no cover if tags == 'None': tags = '' status = params.get('status', '') - fields = set_fields(convert_to_dict(params.get('fields'))) + fields = set_fields(argToList(params.get('fields'))) max_fetch = params.get('max_fetch', '200') group_type = params.get('group_type', ['Incident']) last_run = demisto.getLastRun() @@ -513,14 +482,15 @@ def tc_get_incident_associate_indicators_command(client: Client, args: dict) -> if not response: return_error('No incident groups were found for the given arguments') - ec, indicators = create_context(response[0].get('associatedIndicators', {}).get('data', []), - include_dbot_score=True) + ec, human_readable = create_context(response[0].get('associatedIndicators', {}).get('data', []), + include_dbot_score=True) return_results({ 'Type': entryTypes['note'], 'ContentsFormat': formats['json'], 'Contents': response, 'ReadableContentsFormat': formats['markdown'], - 'HumanReadable': tableToMarkdown('Incident Associated Indicators:', indicators, headerTransform=pascalToSpace), + 'HumanReadable': tableToMarkdown('Incident Associated Indicators:', human_readable, + headerTransform=pascalToSpace, removeNull=True), 'EntryContext': ec }) @@ -604,18 +574,14 @@ def tc_create_event_command(client: Client, args: dict) -> None: # pragma: no c }) -def set_fields(fields) -> str: # pragma: no cover +def set_fields(fields: Optional[list]) -> str: # pragma: no cover fields_str = '' - if fields.get('include_all_metadata'): - return '&fields=tags&fields=associatedIndicators&fields=associatedGroups&fields=securityLabels' \ - '&fields=attributes&fields=associatedVictimAssets' - try: - del fields['include_all_metadata'] - except KeyError: - pass - for arg in fields: - if fields[arg] and fields[arg] != 'false': - fields_str += f'&fields={arg}' + if fields: + if 'include_all_metadata' in fields: + return '&fields=tags&fields=associatedIndicators&fields=associatedGroups&fields=securityLabels' \ + '&fields=attributes&fields=associatedVictimAssets' + for field in fields: + fields_str += f'&fields={field}' return fields_str @@ -678,11 +644,16 @@ def list_groups(client: Client, args: dict, group_id: str = '', from_date: str = include_attributes = args.get('include_attributes', include_attributes) include_security_labels = args.get('include_security_labels', include_security_labels) include_tags = args.get('include_tags', include_tags) - fields = set_fields({'tags': include_tags, 'securityLabels': include_security_labels, - 'attributes': include_attributes, - 'associatedGroups': include_associated_groups, - 'associatedIndicators': include_associated_indicators, - 'include_all_metadata': include_all_metadata}) + + # we create a list of fields to return based on the given arguments + list_of_fields = [field for field, should_include in + {'tags': include_tags, 'securityLabels': include_security_labels, + 'attributes': include_attributes, + 'associatedGroups': include_associated_groups, + 'associatedIndicators': include_associated_indicators, + 'include_all_metadata': include_all_metadata}.items() if + (should_include and should_include != 'false')] + fields = set_fields(list_of_fields) if tql_prefix: tql = f'{tql_filter}{group_id}{group_type}{from_date}{tag}{security_label}'.replace(' AND ', '', 1) tql = urllib.parse.quote(tql.encode('utf8')) @@ -782,8 +753,14 @@ def tc_get_indicator_types(client: Client, args: dict) -> None: # pragma: no co def tc_get_indicators_by_tag_command(client: Client, args: dict) -> None: # pragma: no cover - response = tc_get_indicators_command(client, args, return_raw=True) - ec, indicators = create_context(response, include_dbot_score=True) + owners: str = args.get('owner', '') + limit: str = args.get('limit', '500') + page: str = args.get('page', '0') + tag: str = args.get('tag') or '' + fields_to_return = argToList(args.get('fields_to_return') or []).append(['tags']) + indicators = tc_get_indicators(client, owners=owners, limit=limit, page=page, tag=tag, + fields_to_return=fields_to_return) + ec, human_readable = create_context(indicators, include_dbot_score=True) return_results({ 'Type': entryTypes['note'], @@ -791,28 +768,29 @@ def tc_get_indicators_by_tag_command(client: Client, args: dict) -> None: # pra 'Contents': indicators, 'ReadableContentsFormat': formats['markdown'], 'HumanReadable': tableToMarkdown('ThreatConnect Indicators with tag: {}'.format(args.get('tag')), - indicators, - headerTransform=pascalToSpace), + human_readable, + headerTransform=pascalToSpace, removeNull=True), 'EntryContext': ec }) def tc_get_indicator_command(client: Client, args: dict) -> None: # pragma: no cover - indicator = args.get('indicator') + indicator = args.get('indicator', '') + fields_to_return = argToList(args.get('fields_to_return') or []) indicator_id = '' summary = '' # We do this to check if the given indicator is an ID or a summary - try: + if indicator.isdigit(): # If it's an int it means that it's an ID - int(indicator) # type: ignore - indicator_id = indicator # type: ignore - except ValueError: + indicator_id = indicator + else: # If not we'll treat it as a summary summary = indicator # type: ignore - response = tc_get_indicators_command(client, args, return_raw=True, indicator_id=indicator_id, - summary=summary) # type: ignore - ec, indicators = create_context(response, include_dbot_score=True) - if not indicators: + + response = tc_get_indicators(client, indicator_id=indicator_id, summary=summary, + fields_to_return=fields_to_return) # type: ignore + ec, human_readable = create_context(response, include_dbot_score=True, fields_to_return=fields_to_return) + if not ec: return_results({ 'Type': entryTypes['note'], 'ContentsFormat': formats['text'], @@ -825,20 +803,13 @@ def tc_get_indicator_command(client: Client, args: dict) -> None: # pragma: no associated_indicators = response[0].get('associatedIndicators') associated_groups = response[0].get('associatedGroups') - if ec == []: - ec = {} - if ec: - indicators = copy.deepcopy(ec) - indicators = indicators['TC.Indicator(val.ID && val.ID === obj.ID)'] - return_results({ 'Type': entryTypes['note'], 'ContentsFormat': formats['json'], 'Contents': response, 'ReadableContentsFormat': formats['markdown'], - 'HumanReadable': tableToMarkdown('ThreatConnect indicator for: {}'.format(args.get('id', '')), - indicators, - headerTransform=pascalToSpace), + 'HumanReadable': tableToMarkdown('ThreatConnect indicator for: {}'.format(args.get('indicator', '')), + human_readable, headerTransform=pascalToSpace, removeNull=True), 'EntryContext': ec }) @@ -846,10 +817,9 @@ def tc_get_indicator_command(client: Client, args: dict) -> None: # pragma: no return_results({ 'Type': entryTypes['note'], 'ContentsFormat': formats['json'], - 'Contents': associated_groups.get('data', []), 'ReadableContentsFormat': formats['markdown'], 'HumanReadable': tableToMarkdown( - 'ThreatConnect Associated Groups for indicator: {}'.format(args.get('id', '')), + 'ThreatConnect Associated Groups for indicator: {}'.format(args.get('indicator', '')), associated_groups.get('data', []), headerTransform=pascalToSpace) }) @@ -858,10 +828,9 @@ def tc_get_indicator_command(client: Client, args: dict) -> None: # pragma: no return_results({ 'Type': entryTypes['note'], 'ContentsFormat': formats['json'], - 'Contents': associated_indicators.get('data', []), 'ReadableContentsFormat': formats['markdown'], 'HumanReadable': tableToMarkdown( - 'ThreatConnect Associated Indicators for indicator: {}'.format(args.get('id', '')), + 'ThreatConnect Associated Indicators for indicator: {}'.format(args.get('indicator', '')), associated_indicators.get('data', []), headerTransform=pascalToSpace) }) @@ -870,10 +839,9 @@ def tc_get_indicator_command(client: Client, args: dict) -> None: # pragma: no return_results({ 'Type': entryTypes['note'], 'ContentsFormat': formats['json'], - 'Contents': include_tags.get('data', []), 'ReadableContentsFormat': formats['markdown'], 'HumanReadable': tableToMarkdown( - 'ThreatConnect Tags for indicator: {}'.format(args.get('id', '')), + 'ThreatConnect Tags for indicator: {}'.format(args.get('indicator', '')), include_tags.get('data', []), headerTransform=pascalToSpace) }) @@ -882,19 +850,17 @@ def tc_get_indicator_command(client: Client, args: dict) -> None: # pragma: no return_results({ 'Type': entryTypes['note'], 'ContentsFormat': formats['json'], - 'Contents': include_attributes.get('data', []), 'ReadableContentsFormat': formats['markdown'], 'HumanReadable': tableToMarkdown( - 'ThreatConnect Attributes for indicator: {}'.format(args.get('id', '')), + 'ThreatConnect Attributes for indicator: {}'.format(args.get('indicator', '')), include_attributes.get('data', []), headerTransform=pascalToSpace) }) - if include_observations is not None: + if include_observations: return_results({ 'Type': entryTypes['note'], 'ContentsFormat': formats['json'], - 'Contents': include_observations, 'ReadableContentsFormat': formats['markdown'], 'HumanReadable': tableToMarkdown( 'ThreatConnect Observations for indicator: {}'.format(args.get('id', '')), @@ -1117,14 +1083,14 @@ def tc_add_indicator_command(client: Client, args: dict, rating: str = '0', indi url = '/api/v3/indicators' response = client.make_request(Method.POST, url, payload=json.dumps(payload)) # type: ignore - ec, indicators = create_context([response.get('data')]) + ec, human_readable = create_context([response.get('data')]) return_results({ 'Type': entryTypes['note'], 'ContentsFormat': formats['json'], 'Contents': response.get('data'), 'ReadableContentsFormat': formats['markdown'], - 'HumanReadable': tableToMarkdown('Created new indicator successfully:', indicators, - headerTransform=pascalToSpace), + 'HumanReadable': tableToMarkdown('Created new indicator successfully:', human_readable, + headerTransform=pascalToSpace, removeNull=True), 'EntryContext': ec }) @@ -1159,15 +1125,15 @@ def tc_update_indicator_command(client: Client, args: dict, rating: str = None, if return_raw: return response.get('data'), - ec, indicators = create_context([response.get('data')]) + ec, human_readable = create_context([response.get('data')]) return_results({ 'Type': entryTypes['note'], 'ContentsFormat': formats['json'], 'Contents': response.get('data'), 'ReadableContentsFormat': formats['markdown'], - 'HumanReadable': tableToMarkdown('Updated indicator successfully:', indicators, - headerTransform=pascalToSpace), + 'HumanReadable': tableToMarkdown('Updated indicator successfully:', human_readable, + headerTransform=pascalToSpace, removeNull=True), 'EntryContext': ec }) @@ -1175,7 +1141,7 @@ def tc_update_indicator_command(client: Client, args: dict, rating: str = None, def tc_tag_indicator_command(client: Client, args: dict) -> None: # pragma: no cover tags = args.get('tag') response = tc_update_indicator_command(client, args, mode='append', return_raw=True, tags=tags) - ec, indicators = create_context([response]) + ec, human_readable = create_context([response], fields_to_return=['tags']) return_results({ 'Type': entryTypes['note'], @@ -1184,8 +1150,7 @@ def tc_tag_indicator_command(client: Client, args: dict) -> None: # pragma: no 'ReadableContentsFormat': formats['markdown'], 'HumanReadable': tableToMarkdown( f'Added the tag {args.get("tags")} to indicator {args.get("indicator")} successfully', - indicators, - headerTransform=pascalToSpace), + human_readable, headerTransform=pascalToSpace, removeNull=True), 'EntryContext': ec }) @@ -1195,7 +1160,7 @@ def tc_delete_indicator_tag_command(client: Client, args: dict) -> None: # prag indicator_id = args.get('indicator') response = tc_update_indicator_command(client, args, mode='delete', return_raw=True, tags=tag, indicator=indicator_id) - ec, indicators = create_context([response]) + ec, human_readable = create_context([response], fields_to_return=['tags']) return_results({ 'Type': entryTypes['note'], @@ -1205,8 +1170,8 @@ def tc_delete_indicator_tag_command(client: Client, args: dict) -> None: # prag 'HumanReadable': tableToMarkdown( f'removed the tag {tag} from indicator {indicator_id} successfully', # type: ignore # noqa - indicators, - headerTransform=pascalToSpace), + human_readable, + headerTransform=pascalToSpace, removeNull=True), 'EntryContext': ec }) @@ -1216,7 +1181,7 @@ def tc_incident_associate_indicator_command(client: Client, args: dict) -> None: indicator = args.get('indicator') response = tc_update_group(client, args, mode='append', raw_data=True, group_id=group_id, associated_indicator_id=indicator) - ec, indicators = create_context([response.get('data')]) + ec, human_readable = create_context([response.get('data')]) return_results({ 'Type': entryTypes['note'], @@ -1225,8 +1190,8 @@ def tc_incident_associate_indicator_command(client: Client, args: dict) -> None: 'ReadableContentsFormat': formats['markdown'], 'HumanReadable': tableToMarkdown( f'Associated the incident {group_id} to indicator {indicator} successfully', - indicators, - headerTransform=pascalToSpace), + human_readable, + headerTransform=pascalToSpace, removeNull=True), 'EntryContext': ec }) diff --git a/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3.yml b/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3.yml index dfff7fbbdba2..80a2cdd10dd5 100644 --- a/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3.yml +++ b/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3.yml @@ -56,8 +56,7 @@ configuration: required: false type: 16 - defaultvalue: 3 days - display: First fetch timestamp (