diff --git a/Packs/FeedMISP/Integrations/FeedMISP/FeedMISP.py b/Packs/FeedMISP/Integrations/FeedMISP/FeedMISP.py index e9ff6a435187..96e0ec7a3816 100644 --- a/Packs/FeedMISP/Integrations/FeedMISP/FeedMISP.py +++ b/Packs/FeedMISP/Integrations/FeedMISP/FeedMISP.py @@ -113,6 +113,8 @@ 'misp-galaxy:mitre-course-of-action': ThreatIntel.ObjectsNames.COURSE_OF_ACTION, } +LIMIT: int = 2000 + class Client(BaseClient): @@ -210,7 +212,8 @@ def handle_file_type_fields(raw_type: str, indicator_obj: Dict[str, Any]) -> Non indicator_obj['fields'][raw_type.upper()] = hash_value -def build_params_dict(tags: List[str], attribute_type: List[str], limit: int, page: int) -> Dict[str, Any]: +def build_params_dict(tags: List[str], attribute_type: List[str], limit: int, page: int, from_timestamp: str | None = None + ) -> Dict[str, Any]: """ Creates a dictionary in the format required by MISP to be used as a query. Args: @@ -229,20 +232,28 @@ def build_params_dict(tags: List[str], attribute_type: List[str], limit: int, pa 'limit': limit, 'page': page } + if from_timestamp: + params['from'] = from_timestamp return params -def clean_user_query(query: str) -> Dict[str, Any]: +def parsing_user_query(query: str, limit: int, page: int = 1, from_timestamp: str | None = None) -> Dict[str, Any]: """ - Takes the query string created by the user, adds necessary argument and removes unnecessary arguments + Parsing the query string created by the user by adding necessary argument and removing unnecessary arguments Args: query: User's query string Returns: Dict which has only needed arguments to be sent to MISP """ + global LIMIT try: params = json.loads(query) params["returnFormat"] = "json" params.pop("timestamp", None) + if 'page' not in params: + params["page"] = page + params["limit"] = params.get("limit") or LIMIT + if from_timestamp: + params['from'] = from_timestamp except Exception as err: demisto.debug(str(err)) raise DemistoException(f'Could not parse user query. \nError massage: {err}') @@ -472,8 +483,8 @@ def get_attributes_command(client: Client, args: Dict[str, str], params: Dict[st query = args.get('query', None) attribute_type = argToList(args.get('attribute_type', '')) page = arg_to_number(args.get('page')) or 1 - params_dict = clean_user_query(query) if query else build_params_dict(tags=tags, attribute_type=attribute_type, limit=limit, - page=page) + params_dict = parsing_user_query(query, limit, page) if query else build_params_dict(tags=tags, attribute_type=attribute_type, + limit=limit, page=page) response = client.search_query(params_dict) if error_message := response.get('Error'): raise DemistoException(error_message) @@ -512,19 +523,23 @@ def fetch_attributes_command(client: Client, params: Dict[str, str]): feed_tags = argToList(params.get("feedTags", [])) attribute_types = argToList(params.get('attribute_types', '')) query = params.get('query', None) - params_dict = clean_user_query(query) if query else build_params_dict(tags=tags, attribute_type=attribute_types, limit=2000, - page=1) + last_run = demisto.getLastRun().get('timestamp') or "" + params_dict = parsing_user_query(query, LIMIT, from_timestamp=last_run) if query else\ + build_params_dict(tags=tags, attribute_type=attribute_types, limit=LIMIT, page=1, from_timestamp=last_run) search_query_per_page = client.search_query(params_dict) + demisto.debug(f'params_dict: {params_dict}') while len(search_query_per_page.get("response", {}).get("Attribute", [])): demisto.debug(f'search_query_per_page number of attributes:\ - {len(search_query_per_page.get("response", {}).get("Attribute", []))}\ - page: {params_dict["page"]}') + {len(search_query_per_page.get("response", {}).get("Attribute", []))} page: {params_dict["page"]}') indicators = build_indicators(search_query_per_page, attribute_types, tlp_color, params.get('url'), reputation, feed_tags) - demisto.createIndicators(indicators) + for iter_ in batch(indicators, batch_size=2000): + demisto.createIndicators(iter_) params_dict['page'] += 1 + last_run = search_query_per_page['response']['Attribute'][-1]['timestamp'] search_query_per_page = client.search_query(params_dict) if error_message := search_query_per_page.get('Error'): raise DemistoException(f"Error in API call - check the input parameters and the API Key. Error: {error_message}") + demisto.setLastRun({'timestamp': last_run}) def main(): diff --git a/Packs/FeedMISP/Integrations/FeedMISP/FeedMISP.yml b/Packs/FeedMISP/Integrations/FeedMISP/FeedMISP.yml index 75eee6171477..6a7331d06bfe 100644 --- a/Packs/FeedMISP/Integrations/FeedMISP/FeedMISP.yml +++ b/Packs/FeedMISP/Integrations/FeedMISP/FeedMISP.yml @@ -142,7 +142,7 @@ script: script: '-' type: python subtype: python3 - dockerimage: demisto/python3:3.10.13.86272 + dockerimage: demisto/python3:3.10.13.89009 fromversion: 5.5.0 tests: - MISPfeed Test diff --git a/Packs/FeedMISP/Integrations/FeedMISP/FeedMISP_test.py b/Packs/FeedMISP/Integrations/FeedMISP/FeedMISP_test.py index 3c3592f51b0c..e6d41ad058cc 100644 --- a/Packs/FeedMISP/Integrations/FeedMISP/FeedMISP_test.py +++ b/Packs/FeedMISP/Integrations/FeedMISP/FeedMISP_test.py @@ -3,7 +3,7 @@ import demistomock as demisto from CommonServerPython import DemistoException, ThreatIntel, FeedIndicatorType -from FeedMISP import clean_user_query, build_indicators_iterator, \ +from FeedMISP import parsing_user_query, build_indicators_iterator, \ handle_file_type_fields, get_galaxy_indicator_type, build_indicators_from_galaxies, \ update_indicator_fields, get_ip_type, Client, fetch_attributes_command @@ -90,7 +90,7 @@ def test_handle_file_type_fields_hash_and_filename(): assert indicator_obj['value'] == 'somehashvalue' -def test_clean_user_query_success(): +def test_parsing_user_query_success(): """ Given - A json string query @@ -99,12 +99,12 @@ def test_clean_user_query_success(): Then - create a dict from json string """ - querystr = '{"returnFormat": "json", "type": {"OR": ["ip-src"]}, "tags": {"OR": ["tlp:%"]}}' - params = clean_user_query(querystr) - assert len(params) == 3 + querystr = '{"returnFormat": "json","limit": "3", "type": {"OR": ["ip-src"]}, "tags": {"OR": ["tlp:%"]}}' + params = parsing_user_query(querystr, limit=40000) + assert len(params) == 5 -def test_clean_user_query_bad_query(): +def test_parsing_user_query_bad_query(): """ Given - A json string query @@ -115,10 +115,10 @@ def test_clean_user_query_bad_query(): """ with pytest.raises(DemistoException): querystr = '{"returnFormat": "json", "type": {"OR": ["md5"]}, "tags": {"OR": ["tlp:%"]' - clean_user_query(querystr) + parsing_user_query(querystr, limit=4) -def test_clean_user_query_change_format(): +def test_parsing_user_query_change_format(): """ Given - A json parsed result from qualys @@ -128,11 +128,11 @@ def test_clean_user_query_change_format(): - change return format to json """ querystr = '{"returnFormat": "xml", "type": {"OR": ["md5"]}, "tags": {"OR": ["tlp:%"]}}' - params = clean_user_query(querystr) + params = parsing_user_query(querystr, limit=4) assert params["returnFormat"] == "json" -def test_clean_user_query_remove_timestamp(): +def test_parsing_user_query_remove_timestamp(): """ Given - A json parsed result from qualys @@ -141,9 +141,9 @@ def test_clean_user_query_remove_timestamp(): Then - Return query without the timestamp parameter """ - good_query = '{"returnFormat": "json", "type": {"OR": ["md5"]}, "tags": {"OR": ["tlp:%"]}}' + good_query = '{"returnFormat": "json", "type": {"OR": ["md5"]}, "tags": {"OR": ["tlp:%"]}, "page": 1, "limit": 2000}' querystr = '{"returnFormat": "json", "timestamp": "1617875568", "type": {"OR": ["md5"]}, "tags": {"OR": ["tlp:%"]}}' - params = clean_user_query(querystr) + params = parsing_user_query(querystr, limit=2) assert good_query == json.dumps(params) @@ -317,6 +317,7 @@ def test_search_query_indicators_pagination(mocker): 'type': 'attribute', 'filters': {'category': ['Payload delivery']}, } + mocker.patch("FeedMISP.LIMIT", new=2000) mocker.patch.object(demisto, 'setLastRun') mocker.patch.object(demisto, 'createIndicators') fetch_attributes_command(client, params_dict) diff --git a/Packs/FeedMISP/ReleaseNotes/1_0_32.md b/Packs/FeedMISP/ReleaseNotes/1_0_32.md new file mode 100644 index 000000000000..6e860eaad010 --- /dev/null +++ b/Packs/FeedMISP/ReleaseNotes/1_0_32.md @@ -0,0 +1,7 @@ + +#### Integrations + +##### MISP Feed + +- Fixed an issue where ***fetch-indicators*** retrieves all feed indicators. +- Updated the Docker image to: *demisto/python3:3.10.13.89009*. diff --git a/Packs/FeedMISP/pack_metadata.json b/Packs/FeedMISP/pack_metadata.json index 3b4056d8a51a..6fd9b9652430 100644 --- a/Packs/FeedMISP/pack_metadata.json +++ b/Packs/FeedMISP/pack_metadata.json @@ -2,7 +2,7 @@ "name": "MISP Feed", "description": "Indicators feed from MISP", "support": "xsoar", - "currentVersion": "1.0.31", + "currentVersion": "1.0.32", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "",