From e3d3c042ea5d4459bf3e1adb5a45cfacad0779d5 Mon Sep 17 00:00:00 2001 From: sbenyakir Date: Sun, 4 Jun 2023 18:23:52 +0300 Subject: [PATCH 01/13] refactored code to align with standard --- .../ThreatConnectV3/ThreatConnectV3.py | 339 ++++++++++-------- .../ThreatConnectV3/ThreatConnectV3.yml | 32 +- 2 files changed, 204 insertions(+), 167 deletions(-) diff --git a/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3.py b/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3.py index 7add900b7f6..2a4c3698ceb 100644 --- a/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3.py +++ b/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3.py @@ -49,7 +49,7 @@ def create_header(self, url_suffix: str, method: Method) -> dict: 'Content-Type': 'application/json'} -def create_context(indicators, include_dbot_score=False): +def create_context(indicators, include_dbot_score=False, fields_to_return: list = None): indicators_dbot_score = {} # type: dict params = demisto.params() rating_threshold = int(params.get('rating', '3')) @@ -74,7 +74,7 @@ def create_context(indicators, include_dbot_score=False): 'Host': 'hostName', 'File': 'md5' } - + human_readable = [] for ind in indicators: indicator_type = tc_type_to_demisto_type.get(ind['type'], ind['type']) value_field = type_to_value_field.get(ind['type'], 'summary') @@ -153,7 +153,7 @@ def create_context(indicators, include_dbot_score=False): dbot_object['Indicator'] = k indicators_dbot_score[k] = dbot_object - context[TC_INDICATOR_PATH].append({ + new_indicator = { 'ID': ind['id'], 'Name': value, 'Type': ind['type'], @@ -164,42 +164,53 @@ def create_context(indicators, include_dbot_score=False): 'Rating': rating, 'Confidence': confidence, 'WebLink': ind.get('webLink'), - - # relevant for domain - 'Active': ind.get('whoisActive'), - - # relevant for file - 'File.MD5': md5, - 'File.SHA1': sha1, - 'File.SHA256': sha256, - }) - - if 'group_associations' in ind: - if ind['group_associations']: - context[TC_INDICATOR_PATH][0]['IndicatorGroups'] = ind['group_associations'] - - if 'indicator_associations' in ind: - if ind['indicator_associations']: - context[TC_INDICATOR_PATH][0]['IndicatorAssociations'] = ind[ - 'indicator_associations'] - - if 'indicator_tags' in ind: - if ind['indicator_tags']: - context[TC_INDICATOR_PATH][0]['IndicatorTags'] = ind['indicator_tags'] - - if 'indicator_observations' in ind: - if ind['indicator_observations']: - context[TC_INDICATOR_PATH][0]['IndicatorsObservations'] = ind[ - 'indicator_observations'] - - if 'indicator_attributes' in ind: - if ind['indicator_attributes']: - context[TC_INDICATOR_PATH][0]['IndicatorAttributes'] = ind[ - 'indicator_attributes'] + } + # relevant for domain + if indicator_type == 'domain': + new_indicator['Active'] = ind.get('whoisActive') + + # relevant for file + elif indicator_type == 'file': + new_indicator['File.MD5'] = md5 + new_indicator['File.SHA1'] = sha1 + new_indicator['File.SHA256'] = sha256 + + human_readable.append(new_indicator) + + # The context should not contain the additional fields (there are added a separate HR message after) + context_indicator = copy.deepcopy(new_indicator) + if fields_to_return: + for field in fields_to_return: + context_indicator[field.capitalize()] = ind.get(field) + + context[TC_INDICATOR_PATH].append(context_indicator) + + # if 'group_associations' in ind: + # if ind['group_associations']: + # context[TC_INDICATOR_PATH][0]['IndicatorGroups'] = ind['group_associations'] + # + # if 'indicator_associations' in ind: + # if ind['indicator_associations']: + # context[TC_INDICATOR_PATH][0]['IndicatorAssociations'] = ind[ + # 'indicator_associations'] + # + # if 'indicator_tags' in ind: + # if ind['indicator_tags']: + # context[TC_INDICATOR_PATH][0]['IndicatorTags'] = ind['indicator_tags'] + # + # if 'indicator_observations' in ind: + # if ind['indicator_observations']: + # context[TC_INDICATOR_PATH][0]['IndicatorsObservations'] = ind[ + # 'indicator_observations'] + # + # if 'indicator_attributes' in ind: + # if ind['indicator_attributes']: + # context[TC_INDICATOR_PATH][0]['IndicatorAttributes'] = ind[ + # 'indicator_attributes'] context['DBotScore'] = list(indicators_dbot_score.values()) - context = {k: createContext(v, removeNull=True)[:MAX_CONTEXT] for k, v in context.items() if v} - return context, context.get(TC_INDICATOR_PATH, []) + context = {k: createContext(v)[:MAX_CONTEXT] for k, v in context.items() if v} + return context, human_readable def detection_to_incident(threatconnect_data: dict, threatconnect_date: str) -> dict: @@ -215,7 +226,7 @@ def detection_to_incident(threatconnect_data: dict, threatconnect_date: str) -> return incident -def get_indicators(client: Client, args_type: str, type_name: str, args: dict) -> None: # pragma: no cover +def get_indicator_reputation(client: Client, args_type: str, type_name: str, args: dict) -> None: # pragma: no cover owners_query = create_or_query(args.get('owners', demisto.params().get('defaultOrg')), 'ownerName') query = create_or_query(args.get(args_type), 'summary') # type: ignore rating_threshold = args.get('ratingThreshold', '') @@ -235,14 +246,15 @@ def get_indicators(client: Client, args_type: str, type_name: str, args: dict) - indicators = response.get('data') - ec, indicators = create_context(indicators, include_dbot_score=True) + ec, human_readable = create_context(indicators, include_dbot_score=True) return_results({ 'Type': entryTypes['note'], 'ContentsFormat': formats['json'], 'Contents': indicators, 'ReadableContentsFormat': formats['markdown'], - 'HumanReadable': tableToMarkdown('ThreatConnect URL Reputation for: {}'.format(args.get(args_type)), indicators, - headerTransform=pascalToSpace), + 'HumanReadable': tableToMarkdown('ThreatConnect URL Reputation for: {}'.format(args.get(args_type)), + human_readable, + headerTransform=pascalToSpace, removeNull=True), 'EntryContext': ec }) @@ -258,19 +270,19 @@ def create_or_query(delimiter_str: str, param_name: str, wrapper: str = '"') -> def get_ip_indicators(client: Client, args: dict): # pragma: no cover - return get_indicators(client, 'ip', 'Address', args) + return get_indicator_reputation(client, 'ip', 'Address', args) def get_url_indicators(client: Client, args: dict): # pragma: no cover - return get_indicators(client, 'url', 'URL', args) + return get_indicator_reputation(client, 'url', 'URL', args) def get_domain_indicators(client: Client, args: dict): # pragma: no cover - return get_indicators(client, 'domain', 'Host', args) + return get_indicator_reputation(client, 'domain', 'Host', args) def get_file_indicators(client: Client, args: dict): - return get_indicators(client, 'file', 'File', args) + return get_indicator_reputation(client, 'file', 'File', args) def tc_delete_group_command(client: Client, args: dict) -> Any: # pragma: no cover @@ -297,72 +309,57 @@ def tc_delete_group_command(client: Client, args: dict) -> Any: # pragma: no co }) -def tc_get_indicators_command(client: Client, args: dict, confidence_threshold: str = '', rating_threshold: str = '', - tag: str = '', owners: str = '', indicator_id: str = '', indicator_type: str = '', - return_raw=False, group_associations: str = 'false', summary: str = '', - indicator_associations: str = 'false', - indicator_observations: str = 'false', indicator_tags: str = 'false', - indicator_attributes: str = 'false') -> Any: # pragma: no cover - owners = args.get('owner', owners) - limit = args.get('limit', '500') - page = args.get('page', '0') - tag = args.get('tag', tag) - indicator_type = args.get('type', indicator_type) - indicator_id = args.get('id', indicator_id) - rating_threshold = args.get('ratingThreshold', rating_threshold) - confidence_threshold = args.get('confidenceThreshold', confidence_threshold) - - indicator_attributes = args.get('indicator_attributes', indicator_attributes) - indicator_tags = args.get('indicator_tags', indicator_tags) - indicator_observations = args.get('indicator_observations', indicator_observations) - indicator_associations = args.get('indicator_associations', indicator_associations) - group_associations = args.get('group_associations', group_associations) +def tc_get_indicators(client: Client, tag: str = '', page: str = '0', limit: str = '500', owners: str = '', + indicator_id: str = '', summary: str = '', + fields_to_return: list = None) -> Any: # pragma: no cover + # owners = args.get('owner', owners) + # limit = args.get('limit', '500') + # page = args.get('page', '0') + # tag = args.get('tag', tag) + # indicator_type = args.get('type', indicator_type) + # indicator_id = args.get('id', indicator_id) + # rating_threshold = args.get('ratingThreshold', rating_threshold) + # confidence_threshold = args.get('confidenceThreshold', confidence_threshold) + # fields_to_return = fields_to_return or argToList(args.get('fields_to_return')) or [] + + # indicator_attributes = args.get('indicator_attributes', indicator_attributes) + # indicator_tags = args.get('indicator_tags', indicator_tags) or args.get('tags') + # indicator_observations = args.get('indicator_observations', indicator_observations) + # indicator_associations = args.get('indicator_associations', indicator_associations) + # group_associations = args.get('group_associations', group_associations) tql_prefix = '' if summary: summary = f' AND summary EQ "{summary}"' tql_prefix = '?tql=' - if rating_threshold: - rating_threshold = f'AND (rating > {rating_threshold}) ' - tql_prefix = '?tql=' - if confidence_threshold: - confidence_threshold = f'AND (confidence > {confidence_threshold}) ' - tql_prefix = '?tql=' + # if rating_threshold: + # rating_threshold = f'AND (rating > {rating_threshold}) ' + # tql_prefix = '?tql=' + # if confidence_threshold: + # confidence_threshold = f'AND (confidence > {confidence_threshold}) ' + # tql_prefix = '?tql=' if tag: tag = f' AND tag LIKE "%{tag}%"' - indicator_tags = 'true' - tql_prefix = '?tql=' - if indicator_type: - indicator_type = f' AND typeName EQ "{indicator_type}"' tql_prefix = '?tql=' + # if indicator_type: + # indicator_type = f' AND typeName EQ "{indicator_type}"' + # tql_prefix = '?tql=' if owners: - owners = ' AND ' + create_or_query(args.get('owner', owners), 'ownerName') + owners = ' AND ' + create_or_query(owners, 'ownerName') tql_prefix = '?tql=' if indicator_id: - indicator_id = ' AND ' + create_or_query(args.get('id', indicator_id), 'id').replace('"', '') + indicator_id = ' AND ' + create_or_query(indicator_id, 'id').replace('"', '') tql_prefix = '?tql=' - fields = set_fields({'associatedGroups': group_associations, 'associatedIndicators': indicator_associations, - 'observations': indicator_observations, 'tags': indicator_tags, - 'attributes': indicator_attributes}) - tql = f'{indicator_id}{summary}{indicator_type}{owners}{tag}{confidence_threshold}' \ - f'{rating_threshold}'.replace(' AND ', '', 1) + fields = set_fields(fields_to_return) + # tql = f'{indicator_id}{summary}{indicator_type}{owners}{tag}{confidence_threshold}' \ + # f'{rating_threshold}'.replace(' AND ', '', 1) + tql = f'{indicator_id}{summary}{owners}{tag}'.replace(' AND ', '', 1) tql = urllib.parse.quote(tql.encode('utf8')) url = f'/api/v3/indicators{tql_prefix}{tql}{fields}&resultStart={page}&resultLimit={limit}' if not tql_prefix: url = url.replace('&', '?', 1) response = client.make_request(Method.GET, url) - if return_raw: - return response.get('data') - indicators = response.get('data') - ec, indicators = create_context(indicators, include_dbot_score=True) - return_results({ - 'Type': entryTypes['note'], - 'ContentsFormat': formats['json'], - 'Contents': response.get('data'), - 'ReadableContentsFormat': formats['markdown'], - 'HumanReadable': tableToMarkdown('ThreatConnect Indicators:', indicators, headerTransform=pascalToSpace), - 'EntryContext': ec - }) + return response.get('data') def tc_get_owners_command(client: Client, args: dict) -> Any: # pragma: no cover # type: ignore # noqa @@ -388,6 +385,24 @@ def tc_get_owners_command(client: Client, args: dict) -> Any: # pragma: no cove }) +def tc_get_indicators_command(client: Client, args: dict): + owners = args.get('owner') + limit = args.get('limit', '500') + page = args.get('page', '0') + fields_to_return = argToList(args.get('fields_to_return') or []) + indicators = tc_get_indicators(client=client, owners=owners, limit=limit, page=page) + ec, human_readable = create_context(indicators, include_dbot_score=True, fields_to_return=fields_to_return) + return_results({ + 'Type': entryTypes['note'], + 'ContentsFormat': formats['json'], + 'Contents': indicators, + 'ReadableContentsFormat': formats['markdown'], + 'HumanReadable': tableToMarkdown('ThreatConnect Indicators:', human_readable, headerTransform=pascalToSpace, + removeNull=True), + 'EntryContext': ec + }) + + def tc_get_indicator_owners(client: Client, args: dict) -> Any: # pragma: no cover indicator = args.get('indicator') url = f'/api/v3/indicators/{indicator}' @@ -454,17 +469,23 @@ def get_last_run_time(groups: list) -> str: def convert_to_dict(arr: list): new_dict = {} for item in arr: - new_dict[item] = 'true' + new_dict[item] = True return new_dict +# def convert_to_list(dictionary: dict) -> list: +# +# for key, val in dictionary: +# if val and val != 'false' + + def fetch_incidents(client: Client, args: dict) -> None: # pragma: no cover params = demisto.params() tags = params.get('tags', '') if tags == 'None': tags = '' status = params.get('status', '') - fields = set_fields(convert_to_dict(params.get('fields'))) + fields = set_fields(argToList(params.get('fields'))) max_fetch = params.get('max_fetch', '200') group_type = params.get('group_type', ['Incident']) last_run = demisto.getLastRun() @@ -513,14 +534,15 @@ def tc_get_incident_associate_indicators_command(client: Client, args: dict) -> if not response: return_error('No incident groups were found for the given arguments') - ec, indicators = create_context(response[0].get('associatedIndicators', {}).get('data', []), - include_dbot_score=True) + ec, human_readable = create_context(response[0].get('associatedIndicators', {}).get('data', []), + include_dbot_score=True) return_results({ 'Type': entryTypes['note'], 'ContentsFormat': formats['json'], 'Contents': response, 'ReadableContentsFormat': formats['markdown'], - 'HumanReadable': tableToMarkdown('Incident Associated Indicators:', indicators, headerTransform=pascalToSpace), + 'HumanReadable': tableToMarkdown('Incident Associated Indicators:', human_readable, + headerTransform=pascalToSpace, removeNull=True), 'EntryContext': ec }) @@ -604,18 +626,13 @@ def tc_create_event_command(client: Client, args: dict) -> None: # pragma: no c }) -def set_fields(fields) -> str: # pragma: no cover +def set_fields(fields: list) -> str: # pragma: no cover fields_str = '' - if fields.get('include_all_metadata'): + if 'include_all_metadata' in fields: return '&fields=tags&fields=associatedIndicators&fields=associatedGroups&fields=securityLabels' \ '&fields=attributes&fields=associatedVictimAssets' - try: - del fields['include_all_metadata'] - except KeyError: - pass - for arg in fields: - if fields[arg] and fields[arg] != 'false': - fields_str += f'&fields={arg}' + for field in fields: + fields_str += f'&fields={field}' return fields_str @@ -678,11 +695,16 @@ def list_groups(client: Client, args: dict, group_id: str = '', from_date: str = include_attributes = args.get('include_attributes', include_attributes) include_security_labels = args.get('include_security_labels', include_security_labels) include_tags = args.get('include_tags', include_tags) - fields = set_fields({'tags': include_tags, 'securityLabels': include_security_labels, - 'attributes': include_attributes, - 'associatedGroups': include_associated_groups, - 'associatedIndicators': include_associated_indicators, - 'include_all_metadata': include_all_metadata}) + + # we create a list of fields to return based on the given arguments + list_of_fields = [field for field, should_include in + {'tags': include_tags, 'securityLabels': include_security_labels, + 'attributes': include_attributes, + 'associatedGroups': include_associated_groups, + 'associatedIndicators': include_associated_indicators, + 'include_all_metadata': include_all_metadata}.items() if + (should_include and should_include != 'false')] + fields = set_fields(list_of_fields) if tql_prefix: tql = f'{tql_filter}{group_id}{group_type}{from_date}{tag}{security_label}'.replace(' AND ', '', 1) tql = urllib.parse.quote(tql.encode('utf8')) @@ -782,8 +804,14 @@ def tc_get_indicator_types(client: Client, args: dict) -> None: # pragma: no co def tc_get_indicators_by_tag_command(client: Client, args: dict) -> None: # pragma: no cover - response = tc_get_indicators_command(client, args, return_raw=True) - ec, indicators = create_context(response, include_dbot_score=True) + owners = args.get('owner') + limit = args.get('limit', '500') + page = args.get('page', '0') + tag = args.get('tag') + fields_to_return = argToList(args.get('fields_to_return') or []).append(['tags']) + indicators = tc_get_indicators(client, owners=owners, limit=limit, page=page, tag=tag, + fields_to_return=fields_to_return) + ec, human_readable = create_context(indicators, include_dbot_score=True) return_results({ 'Type': entryTypes['note'], @@ -791,28 +819,29 @@ def tc_get_indicators_by_tag_command(client: Client, args: dict) -> None: # pra 'Contents': indicators, 'ReadableContentsFormat': formats['markdown'], 'HumanReadable': tableToMarkdown('ThreatConnect Indicators with tag: {}'.format(args.get('tag')), - indicators, - headerTransform=pascalToSpace), + human_readable, + headerTransform=pascalToSpace, removeNull=True), 'EntryContext': ec }) def tc_get_indicator_command(client: Client, args: dict) -> None: # pragma: no cover indicator = args.get('indicator') + fields_to_return = argToList(args.get('fields_to_return') or []) indicator_id = '' summary = '' # We do this to check if the given indicator is an ID or a summary - try: + if indicator.isdigit(): # If it's an int it means that it's an ID - int(indicator) # type: ignore - indicator_id = indicator # type: ignore - except ValueError: + indicator_id = indicator + else: # If not we'll treat it as a summary summary = indicator # type: ignore - response = tc_get_indicators_command(client, args, return_raw=True, indicator_id=indicator_id, - summary=summary) # type: ignore - ec, indicators = create_context(response, include_dbot_score=True) - if not indicators: + + response = tc_get_indicators(client, indicator_id=indicator_id, summary=summary, + fields_to_return=fields_to_return) # type: ignore + ec, human_readable = create_context(response, include_dbot_score=True, fields_to_return=fields_to_return) + if not ec: return_results({ 'Type': entryTypes['note'], 'ContentsFormat': formats['text'], @@ -825,20 +854,18 @@ def tc_get_indicator_command(client: Client, args: dict) -> None: # pragma: no associated_indicators = response[0].get('associatedIndicators') associated_groups = response[0].get('associatedGroups') - if ec == []: - ec = {} - if ec: - indicators = copy.deepcopy(ec) - indicators = indicators['TC.Indicator(val.ID && val.ID === obj.ID)'] - + # if ec == []: + # ec = {} + # if ec: + # indicators = copy.deepcopy(ec) + # indicators = indicators['TC.Indicator(val.ID && val.ID === obj.ID)'] return_results({ 'Type': entryTypes['note'], 'ContentsFormat': formats['json'], 'Contents': response, 'ReadableContentsFormat': formats['markdown'], - 'HumanReadable': tableToMarkdown('ThreatConnect indicator for: {}'.format(args.get('id', '')), - indicators, - headerTransform=pascalToSpace), + 'HumanReadable': tableToMarkdown('ThreatConnect indicator for: {}'.format(args.get('indicator', '')), + human_readable, headerTransform=pascalToSpace, removeNull=True), 'EntryContext': ec }) @@ -846,10 +873,9 @@ def tc_get_indicator_command(client: Client, args: dict) -> None: # pragma: no return_results({ 'Type': entryTypes['note'], 'ContentsFormat': formats['json'], - 'Contents': associated_groups.get('data', []), 'ReadableContentsFormat': formats['markdown'], 'HumanReadable': tableToMarkdown( - 'ThreatConnect Associated Groups for indicator: {}'.format(args.get('id', '')), + 'ThreatConnect Associated Groups for indicator: {}'.format(args.get('indicator', '')), associated_groups.get('data', []), headerTransform=pascalToSpace) }) @@ -858,10 +884,9 @@ def tc_get_indicator_command(client: Client, args: dict) -> None: # pragma: no return_results({ 'Type': entryTypes['note'], 'ContentsFormat': formats['json'], - 'Contents': associated_indicators.get('data', []), 'ReadableContentsFormat': formats['markdown'], 'HumanReadable': tableToMarkdown( - 'ThreatConnect Associated Indicators for indicator: {}'.format(args.get('id', '')), + 'ThreatConnect Associated Indicators for indicator: {}'.format(args.get('indicator', '')), associated_indicators.get('data', []), headerTransform=pascalToSpace) }) @@ -870,10 +895,9 @@ def tc_get_indicator_command(client: Client, args: dict) -> None: # pragma: no return_results({ 'Type': entryTypes['note'], 'ContentsFormat': formats['json'], - 'Contents': include_tags.get('data', []), 'ReadableContentsFormat': formats['markdown'], 'HumanReadable': tableToMarkdown( - 'ThreatConnect Tags for indicator: {}'.format(args.get('id', '')), + 'ThreatConnect Tags for indicator: {}'.format(args.get('indicator', '')), include_tags.get('data', []), headerTransform=pascalToSpace) }) @@ -882,19 +906,17 @@ def tc_get_indicator_command(client: Client, args: dict) -> None: # pragma: no return_results({ 'Type': entryTypes['note'], 'ContentsFormat': formats['json'], - 'Contents': include_attributes.get('data', []), 'ReadableContentsFormat': formats['markdown'], 'HumanReadable': tableToMarkdown( - 'ThreatConnect Attributes for indicator: {}'.format(args.get('id', '')), + 'ThreatConnect Attributes for indicator: {}'.format(args.get('indicator', '')), include_attributes.get('data', []), headerTransform=pascalToSpace) }) - if include_observations is not None: + if include_observations: return_results({ 'Type': entryTypes['note'], 'ContentsFormat': formats['json'], - 'Contents': include_observations, 'ReadableContentsFormat': formats['markdown'], 'HumanReadable': tableToMarkdown( 'ThreatConnect Observations for indicator: {}'.format(args.get('id', '')), @@ -1117,14 +1139,14 @@ def tc_add_indicator_command(client: Client, args: dict, rating: str = '0', indi url = '/api/v3/indicators' response = client.make_request(Method.POST, url, payload=json.dumps(payload)) # type: ignore - ec, indicators = create_context([response.get('data')]) + ec, human_readable = create_context([response.get('data')]) return_results({ 'Type': entryTypes['note'], 'ContentsFormat': formats['json'], 'Contents': response.get('data'), 'ReadableContentsFormat': formats['markdown'], - 'HumanReadable': tableToMarkdown('Created new indicator successfully:', indicators, - headerTransform=pascalToSpace), + 'HumanReadable': tableToMarkdown('Created new indicator successfully:', human_readable, + headerTransform=pascalToSpace, removeNull=True), 'EntryContext': ec }) @@ -1159,15 +1181,15 @@ def tc_update_indicator_command(client: Client, args: dict, rating: str = None, if return_raw: return response.get('data'), - ec, indicators = create_context([response.get('data')]) + ec, human_readable = create_context([response.get('data')]) return_results({ 'Type': entryTypes['note'], 'ContentsFormat': formats['json'], 'Contents': response.get('data'), 'ReadableContentsFormat': formats['markdown'], - 'HumanReadable': tableToMarkdown('Updated indicator successfully:', indicators, - headerTransform=pascalToSpace), + 'HumanReadable': tableToMarkdown('Updated indicator successfully:', human_readable, + headerTransform=pascalToSpace, removeNull=True), 'EntryContext': ec }) @@ -1175,7 +1197,7 @@ def tc_update_indicator_command(client: Client, args: dict, rating: str = None, def tc_tag_indicator_command(client: Client, args: dict) -> None: # pragma: no cover tags = args.get('tag') response = tc_update_indicator_command(client, args, mode='append', return_raw=True, tags=tags) - ec, indicators = create_context([response]) + ec, human_readable = create_context([response], fields_to_return='tags') return_results({ 'Type': entryTypes['note'], @@ -1184,8 +1206,7 @@ def tc_tag_indicator_command(client: Client, args: dict) -> None: # pragma: no 'ReadableContentsFormat': formats['markdown'], 'HumanReadable': tableToMarkdown( f'Added the tag {args.get("tags")} to indicator {args.get("indicator")} successfully', - indicators, - headerTransform=pascalToSpace), + human_readable, headerTransform=pascalToSpace, removeNull=True), 'EntryContext': ec }) @@ -1195,7 +1216,7 @@ def tc_delete_indicator_tag_command(client: Client, args: dict) -> None: # prag indicator_id = args.get('indicator') response = tc_update_indicator_command(client, args, mode='delete', return_raw=True, tags=tag, indicator=indicator_id) - ec, indicators = create_context([response]) + ec, human_readable = create_context([response], fields_to_return=True) return_results({ 'Type': entryTypes['note'], @@ -1205,8 +1226,8 @@ def tc_delete_indicator_tag_command(client: Client, args: dict) -> None: # prag 'HumanReadable': tableToMarkdown( f'removed the tag {tag} from indicator {indicator_id} successfully', # type: ignore # noqa - indicators, - headerTransform=pascalToSpace), + human_readable, + headerTransform=pascalToSpace, removeNull=True), 'EntryContext': ec }) @@ -1216,7 +1237,7 @@ def tc_incident_associate_indicator_command(client: Client, args: dict) -> None: indicator = args.get('indicator') response = tc_update_group(client, args, mode='append', raw_data=True, group_id=group_id, associated_indicator_id=indicator) - ec, indicators = create_context([response.get('data')]) + ec, human_readable = create_context([response.get('data')]) return_results({ 'Type': entryTypes['note'], @@ -1225,8 +1246,8 @@ def tc_incident_associate_indicator_command(client: Client, args: dict) -> None: 'ReadableContentsFormat': formats['markdown'], 'HumanReadable': tableToMarkdown( f'Associated the incident {group_id} to indicator {indicator} successfully', - indicators, - headerTransform=pascalToSpace), + human_readable, + headerTransform=pascalToSpace, removeNull=True), 'EntryContext': ec }) diff --git a/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3.yml b/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3.yml index dfff7fbbdba..212cdf039e9 100644 --- a/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3.yml +++ b/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3.yml @@ -1,6 +1,6 @@ category: Data Enrichment & Threat Intelligence commonfields: - id: ThreatConnect v3 + id: ThreatConnect v3 dev version: -1 configuration: - defaultvalue: https://api.threatconnect.com @@ -128,8 +128,8 @@ configuration: required: false description: ThreatConnect's integration is a intelligence-driven security operations solution with intelligence, automation, analytics, and workflows. -display: ThreatConnect v3 -name: ThreatConnect v3 +display: ThreatConnect v3 dev +name: ThreatConnect v3 dev script: commands: - arguments: @@ -396,6 +396,11 @@ script: - description: The maximum number of results that can be returned. The default is 500. name: limit + - name: fields_to_return + description: 'List of comma separated values representing additional fields to return as part of the result + indicator metadata. Possible values are: associatedGroups, associatedIndicators, observations, tags, and + attributes.' + isArray: true description: Retrieves a list of all indicators. name: tc-indicators outputs: @@ -525,6 +530,11 @@ script: description: The ID of the indicator by which to search. name: indicator required: true + - name: fields_to_return + description: 'List of comma separated values representing additional fields to return as part of the result + indicator metadata. Possible values are: associatedGroups, associatedIndicators, observations, tags, and + attributes.' + isArray: true description: Retrieves information about an indicator. name: tc-get-indicator outputs: @@ -656,6 +666,11 @@ script: name: page - description: The maximum number of results that can be returned. The default is 500. name: limit + - name: fields_to_return + description: 'List of comma separated values representing additional fields to return as part of the result + indicator metadata. Possible values are: associatedGroups, associatedIndicators, observations, tags, and + attributes.' + isArray: true description: Fetches all indicators that have a tag. name: tc-get-indicators-by-tag outputs: @@ -1736,21 +1751,22 @@ script: type: string - description: Add group tags metadata to the results. name: include_tags - type: bool + deprecated: true - description: Add group security labels metadata to the results. name: include_security_labels - type: bool + deprecated: true - description: Add group attributes metadata to the results. name: include_attributes - type: bool + deprecated: true - description: Add group associated groups metadata to the results. name: include_associated_groups - type: bool + deprecated: true - description: Add group associated indicators metadata to the results. name: include_associated_indicators - type: bool + deprecated: true - description: Add all group metadata to the results. name: include_all_metaData + deprecated: true type: bool description: Returns all groups. name: tc-list-groups From 49a2c020aad16f2e8db4df538f6c937bf200cbbb Mon Sep 17 00:00:00 2001 From: sbenyakir Date: Mon, 5 Jun 2023 18:02:24 +0300 Subject: [PATCH 02/13] Fixed owner argument --- .../ThreatConnectV3/ThreatConnectV3.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3.py b/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3.py index 2a4c3698ceb..71d3ec473e6 100644 --- a/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3.py +++ b/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3.py @@ -386,7 +386,7 @@ def tc_get_owners_command(client: Client, args: dict) -> Any: # pragma: no cove def tc_get_indicators_command(client: Client, args: dict): - owners = args.get('owner') + owners = args.get('owner', '') limit = args.get('limit', '500') page = args.get('page', '0') fields_to_return = argToList(args.get('fields_to_return') or []) @@ -628,11 +628,12 @@ def tc_create_event_command(client: Client, args: dict) -> None: # pragma: no c def set_fields(fields: list) -> str: # pragma: no cover fields_str = '' - if 'include_all_metadata' in fields: - return '&fields=tags&fields=associatedIndicators&fields=associatedGroups&fields=securityLabels' \ - '&fields=attributes&fields=associatedVictimAssets' - for field in fields: - fields_str += f'&fields={field}' + if fields: + if 'include_all_metadata' in fields: + return '&fields=tags&fields=associatedIndicators&fields=associatedGroups&fields=securityLabels' \ + '&fields=attributes&fields=associatedVictimAssets' + for field in fields: + fields_str += f'&fields={field}' return fields_str @@ -804,7 +805,7 @@ def tc_get_indicator_types(client: Client, args: dict) -> None: # pragma: no co def tc_get_indicators_by_tag_command(client: Client, args: dict) -> None: # pragma: no cover - owners = args.get('owner') + owners = args.get('owner', '') limit = args.get('limit', '500') page = args.get('page', '0') tag = args.get('tag') From 8cfbb5ac0113d793dd09541b7d9cbfa4faa64c87 Mon Sep 17 00:00:00 2001 From: sbenyakir Date: Tue, 6 Jun 2023 15:20:48 +0300 Subject: [PATCH 03/13] Fixed unittests --- .../Integrations/ThreatConnectV3/ThreatConnectV3_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3_test.py b/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3_test.py index a5c1a65e698..9390f9f5784 100644 --- a/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3_test.py +++ b/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3_test.py @@ -39,6 +39,7 @@ def test_create_context(): # type: ignore # noqa }] res = ({'TC.Indicator(val.ID && val.ID === obj.ID)': [{'Confidence': 32, 'CreateDate': '2021-12-09T12:57:18Z', + 'Description': None, 'ID': 40435508, 'LastModified': '2022-07-26T13:51:49Z', 'Name': 'http://yourwebsite.com/opcache.php', @@ -53,6 +54,7 @@ def test_create_context(): # type: ignore # noqa 'Vendor': 'ThreatConnect'}}]}, [{'Confidence': 32, 'CreateDate': '2021-12-09T12:57:18Z', + 'Description': None, 'ID': 40435508, 'LastModified': '2022-07-26T13:51:49Z', 'Name': 'http://yourwebsite.com/opcache.php', From c73c8a16ef542a3abb2824ad0b09db554809b6b7 Mon Sep 17 00:00:00 2001 From: sbenyakir Date: Tue, 6 Jun 2023 16:22:05 +0300 Subject: [PATCH 04/13] Mypy fixes --- .../ThreatConnectV3/ThreatConnectV3.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3.py b/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3.py index 71d3ec473e6..4917a800475 100644 --- a/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3.py +++ b/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3.py @@ -626,7 +626,7 @@ def tc_create_event_command(client: Client, args: dict) -> None: # pragma: no c }) -def set_fields(fields: list) -> str: # pragma: no cover +def set_fields(fields: Optional[list]) -> str: # pragma: no cover fields_str = '' if fields: if 'include_all_metadata' in fields: @@ -805,10 +805,10 @@ def tc_get_indicator_types(client: Client, args: dict) -> None: # pragma: no co def tc_get_indicators_by_tag_command(client: Client, args: dict) -> None: # pragma: no cover - owners = args.get('owner', '') - limit = args.get('limit', '500') - page = args.get('page', '0') - tag = args.get('tag') + owners: str = args.get('owner', '') + limit: str = args.get('limit', '500') + page: str = args.get('page', '0') + tag: str = args.get('tag') or '' fields_to_return = argToList(args.get('fields_to_return') or []).append(['tags']) indicators = tc_get_indicators(client, owners=owners, limit=limit, page=page, tag=tag, fields_to_return=fields_to_return) @@ -827,7 +827,7 @@ def tc_get_indicators_by_tag_command(client: Client, args: dict) -> None: # pra def tc_get_indicator_command(client: Client, args: dict) -> None: # pragma: no cover - indicator = args.get('indicator') + indicator = args.get('indicator', '') fields_to_return = argToList(args.get('fields_to_return') or []) indicator_id = '' summary = '' @@ -1198,7 +1198,7 @@ def tc_update_indicator_command(client: Client, args: dict, rating: str = None, def tc_tag_indicator_command(client: Client, args: dict) -> None: # pragma: no cover tags = args.get('tag') response = tc_update_indicator_command(client, args, mode='append', return_raw=True, tags=tags) - ec, human_readable = create_context([response], fields_to_return='tags') + ec, human_readable = create_context([response], fields_to_return=['tags']) return_results({ 'Type': entryTypes['note'], @@ -1217,7 +1217,7 @@ def tc_delete_indicator_tag_command(client: Client, args: dict) -> None: # prag indicator_id = args.get('indicator') response = tc_update_indicator_command(client, args, mode='delete', return_raw=True, tags=tag, indicator=indicator_id) - ec, human_readable = create_context([response], fields_to_return=True) + ec, human_readable = create_context([response], fields_to_return=['tags']) return_results({ 'Type': entryTypes['note'], From e2b8be3b83dda295fa9bc1a162e6e6e27022a8cf Mon Sep 17 00:00:00 2001 From: sbenyakir Date: Tue, 6 Jun 2023 16:24:13 +0300 Subject: [PATCH 05/13] reverted name change --- .../Integrations/ThreatConnectV3/ThreatConnectV3.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3.yml b/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3.yml index 212cdf039e9..5b1aba65091 100644 --- a/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3.yml +++ b/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3.yml @@ -1,6 +1,6 @@ category: Data Enrichment & Threat Intelligence commonfields: - id: ThreatConnect v3 dev + id: ThreatConnect v3 version: -1 configuration: - defaultvalue: https://api.threatconnect.com @@ -128,8 +128,8 @@ configuration: required: false description: ThreatConnect's integration is a intelligence-driven security operations solution with intelligence, automation, analytics, and workflows. -display: ThreatConnect v3 dev -name: ThreatConnect v3 dev +display: ThreatConnect v3 +name: ThreatConnect v3 script: commands: - arguments: From 8bb5d46af23b6940a433b95e12e44caaf941bce2 Mon Sep 17 00:00:00 2001 From: sbenyakir Date: Tue, 6 Jun 2023 16:27:43 +0300 Subject: [PATCH 06/13] reverted argument deprecation --- .../Integrations/ThreatConnectV3/ThreatConnectV3.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3.yml b/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3.yml index 5b1aba65091..9f756421823 100644 --- a/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3.yml +++ b/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3.yml @@ -1751,23 +1751,16 @@ script: type: string - description: Add group tags metadata to the results. name: include_tags - deprecated: true - description: Add group security labels metadata to the results. name: include_security_labels - deprecated: true - description: Add group attributes metadata to the results. name: include_attributes - deprecated: true - description: Add group associated groups metadata to the results. name: include_associated_groups - deprecated: true - description: Add group associated indicators metadata to the results. name: include_associated_indicators - deprecated: true - description: Add all group metadata to the results. name: include_all_metaData - deprecated: true - type: bool description: Returns all groups. name: tc-list-groups outputs: From 9e17144f2ff391f00222889809600c8030687f15 Mon Sep 17 00:00:00 2001 From: sbenyakir Date: Tue, 6 Jun 2023 17:58:54 +0300 Subject: [PATCH 07/13] Added RN and version Fixed argument description --- .../Integrations/ThreatConnectV3/ThreatConnectV3.yml | 6 +++--- Packs/ThreatConnect/ReleaseNotes/3_0_5.md | 8 ++++++++ Packs/ThreatConnect/pack_metadata.json | 2 +- 3 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 Packs/ThreatConnect/ReleaseNotes/3_0_5.md diff --git a/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3.yml b/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3.yml index 9f756421823..db1ff3f4f20 100644 --- a/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3.yml +++ b/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3.yml @@ -397,7 +397,7 @@ script: is 500. name: limit - name: fields_to_return - description: 'List of comma separated values representing additional fields to return as part of the result + description: 'Comma separated list of additional fields to return as part of the result indicator metadata. Possible values are: associatedGroups, associatedIndicators, observations, tags, and attributes.' isArray: true @@ -531,7 +531,7 @@ script: name: indicator required: true - name: fields_to_return - description: 'List of comma separated values representing additional fields to return as part of the result + description: 'Comma separated list of additional fields to return as part of the result indicator metadata. Possible values are: associatedGroups, associatedIndicators, observations, tags, and attributes.' isArray: true @@ -667,7 +667,7 @@ script: - description: The maximum number of results that can be returned. The default is 500. name: limit - name: fields_to_return - description: 'List of comma separated values representing additional fields to return as part of the result + description: 'Comma separated list of additional fields to return as part of the result indicator metadata. Possible values are: associatedGroups, associatedIndicators, observations, tags, and attributes.' isArray: true diff --git a/Packs/ThreatConnect/ReleaseNotes/3_0_5.md b/Packs/ThreatConnect/ReleaseNotes/3_0_5.md new file mode 100644 index 00000000000..1d93aeccb79 --- /dev/null +++ b/Packs/ThreatConnect/ReleaseNotes/3_0_5.md @@ -0,0 +1,8 @@ + +#### Integrations + +##### ThreatConnect v3 + +- Added the *fields_to_return* argument to the **tc-get-indicators-by-tag**, **tc-get-indicator** and **tc-indicators** commands. +This will allow specifying additional fields to return as part of the result. +- Cleaned the integration code from unused functionalities. \ No newline at end of file diff --git a/Packs/ThreatConnect/pack_metadata.json b/Packs/ThreatConnect/pack_metadata.json index c123d64f744..fcfec70d11b 100644 --- a/Packs/ThreatConnect/pack_metadata.json +++ b/Packs/ThreatConnect/pack_metadata.json @@ -2,7 +2,7 @@ "name": "ThreatConnect", "description": "Threat intelligence platform.", "support": "xsoar", - "currentVersion": "3.0.4", + "currentVersion": "3.0.5", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "", From 1f2a557e6a9f82e78df138a256f6026ba289aca1 Mon Sep 17 00:00:00 2001 From: sbenyakir Date: Tue, 6 Jun 2023 18:02:08 +0300 Subject: [PATCH 08/13] Added RN and version Fixed argument description --- .../ThreatConnectV3/ThreatConnectV3.yml | 153 ++++++------------ Packs/ThreatConnect/ReleaseNotes/3_0_5.md | 4 +- 2 files changed, 49 insertions(+), 108 deletions(-) diff --git a/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3.yml b/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3.yml index db1ff3f4f20..f417de9684f 100644 --- a/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3.yml +++ b/Packs/ThreatConnect/Integrations/ThreatConnectV3/ThreatConnectV3.yml @@ -56,8 +56,7 @@ configuration: required: false type: 16 - defaultvalue: 3 days - display: First fetch timestamp (