diff --git a/Packs/ContentManagement/.pack-ignore b/Packs/ContentManagement/.pack-ignore index 2c9be3267b85..738f17a5bfa2 100644 --- a/Packs/ContentManagement/.pack-ignore +++ b/Packs/ContentManagement/.pack-ignore @@ -34,6 +34,9 @@ ignore=RM106 [file:playbook-Configuration_Setup.yml] ignore=PB106 +[file:playbook-Delete_Custom_Content.yml] +ignore=PB105 + [known_words] cliname cicd diff --git a/Packs/ContentManagement/Playbooks/playbook-Delete_Custom_Content.yml b/Packs/ContentManagement/Playbooks/playbook-Delete_Custom_Content.yml new file mode 100644 index 000000000000..9a43feb8e902 --- /dev/null +++ b/Packs/ContentManagement/Playbooks/playbook-Delete_Custom_Content.yml @@ -0,0 +1,408 @@ +id: Delete Custom Content +version: -1 +contentitemexportablefields: + contentitemfields: {} +name: Delete Custom Content +description: This playbook deletes custom content from the system. It deletes Playbooks, Scripts, Layouts, Classifiers, Mappers, Incident Types and Incident Fields. +starttaskid: "0" +tasks: + "0": + id: "0" + taskid: b91d4d6a-5343-49d1-8854-c778204ef5c6 + type: start + task: + id: b91d4d6a-5343-49d1-8854-c778204ef5c6 + version: -1 + name: "" + iscommand: false + brand: "" + description: '' + nexttasks: + '#none#': + - "9" + separatecontext: false + continueonerrortype: "" + view: |- + { + "position": { + "x": 265, + "y": 50 + } + } + note: false + timertriggers: [] + ignoreworker: false + skipunavailable: false + quietmode: 0 + isoversize: false + isautoswitchedtoquietmode: false + "1": + id: "1" + taskid: fab4e101-1e2b-45b0-847e-e69efca67d62 + type: regular + task: + id: fab4e101-1e2b-45b0-847e-e69efca67d62 + version: -1 + name: Download custom content + description: Download files from Core server + script: '|||core-api-download' + type: regular + iscommand: true + brand: "" + nexttasks: + '#none#': + - "2" + scriptarguments: + uri: + simple: /content/bundle + separatecontext: false + continueonerrortype: "" + view: |- + { + "position": { + "x": 265, + "y": 370 + } + } + note: false + timertriggers: [] + ignoreworker: false + skipunavailable: false + quietmode: 0 + isoversize: false + isautoswitchedtoquietmode: false + "2": + id: "2" + taskid: e20d6675-4dc5-4b6b-842b-4db3257ccfeb + type: regular + task: + id: e20d6675-4dc5-4b6b-842b-4db3257ccfeb + version: -1 + name: GetIdsFromCustomContent + description: Extract custom content ids from custom content bundle file and exclude ids as specified. + scriptName: GetIdsFromCustomContent + type: regular + iscommand: false + brand: "" + nexttasks: + '#none#': + - "13" + scriptarguments: + file_entry_id: + simple: ${File.EntryID} + separatecontext: false + continueonerrortype: "" + view: |- + { + "position": { + "x": 265, + "y": 545 + } + } + note: false + timertriggers: [] + ignoreworker: false + skipunavailable: false + quietmode: 0 + isoversize: false + isautoswitchedtoquietmode: false + "3": + id: "3" + taskid: b376e0ed-14a7-4a90-8e29-ec279cd50067 + type: regular + task: + id: b376e0ed-14a7-4a90-8e29-ec279cd50067 + version: -1 + name: Delete Content + description: Delete content to keep XSOAR tidy. + scriptName: DeleteContent + type: regular + iscommand: false + brand: "" + nexttasks: + '#none#': + - "4" + scriptarguments: + dry_run: + simple: ${inputs.dry_run} + include_ids_dict: + simple: ${GetIdsFromCustomContent.included_ids} + separatecontext: false + continueonerrortype: "" + view: |- + { + "position": { + "x": 265, + "y": 895 + } + } + note: false + timertriggers: [] + ignoreworker: false + skipunavailable: false + quietmode: 0 + isoversize: false + isautoswitchedtoquietmode: false + "4": + id: "4" + taskid: 7c757eec-7fe1-46c4-86b0-673004f532ba + type: condition + task: + id: 7c757eec-7fe1-46c4-86b0-673004f532ba + version: -1 + description: "" + name: Check Deletion Status + type: condition + iscommand: false + brand: "" + nexttasks: + '#default#': + - "6" + "yes": + - "5" + separatecontext: false + conditions: + - label: "yes" + condition: + - - operator: isEqualString + left: + value: + simple: ConfigurationSetup.Deletion.status + iscontext: true + right: + value: + simple: Completed + - operator: isEqualString + left: + value: + simple: ConfigurationSetup.Deletion.status + iscontext: true + right: + value: + simple: Dry run, nothing really deleted. + continueonerrortype: "" + view: |- + { + "position": { + "x": 265, + "y": 1070 + } + } + note: false + timertriggers: [] + ignoreworker: false + skipunavailable: false + quietmode: 0 + isoversize: false + isautoswitchedtoquietmode: false + "5": + id: "5" + taskid: 1e75bc98-ff87-4cf7-856c-b3d7f5b670c5 + type: regular + task: + id: 1e75bc98-ff87-4cf7-856c-b3d7f5b670c5 + version: -1 + name: Print Status + description: Prints text to war room (Markdown supported) + scriptName: Print + type: regular + iscommand: false + brand: "" + nexttasks: + '#none#': + - "7" + scriptarguments: + value: + simple: ${ConfigurationSetup.Deletion.status} + separatecontext: false + continueonerrortype: "" + view: |- + { + "position": { + "x": 50, + "y": 1245 + } + } + note: false + timertriggers: [] + ignoreworker: false + skipunavailable: false + quietmode: 0 + isoversize: false + isautoswitchedtoquietmode: false + "6": + id: "6" + taskid: 9b1ed2f4-0c81-4675-8f22-1bb51118e29e + type: regular + task: + id: 9b1ed2f4-0c81-4675-8f22-1bb51118e29e + version: -1 + name: Print Deletion Error + description: Prints an error entry with a given message + scriptName: PrintErrorEntry + type: regular + iscommand: false + brand: "" + nexttasks: + '#none#': + - "7" + scriptarguments: + message: + simple: |- + The deletion was NOT successfully completed. + The status is "${ConfigurationSetup.Deletion.status}" + The deleted ids are: ${ConfigurationSetup.Deletion.successfully_deleted}. + separatecontext: false + continueonerrortype: "" + view: |- + { + "position": { + "x": 480, + "y": 1245 + } + } + note: false + timertriggers: [] + ignoreworker: false + skipunavailable: false + quietmode: 0 + isoversize: false + isautoswitchedtoquietmode: false + "7": + id: "7" + taskid: acfcc1bc-7423-4d74-841b-9c97c340cd1f + type: title + task: + id: acfcc1bc-7423-4d74-841b-9c97c340cd1f + version: -1 + name: Done + type: title + iscommand: false + brand: "" + description: '' + separatecontext: false + continueonerrortype: "" + view: |- + { + "position": { + "x": 265, + "y": 1420 + } + } + note: false + timertriggers: [] + ignoreworker: false + skipunavailable: false + quietmode: 0 + isoversize: false + isautoswitchedtoquietmode: false + "9": + id: "9" + taskid: 24047bff-051e-4673-84aa-6f61da6bc4ce + type: regular + task: + id: 24047bff-051e-4673-84aa-6f61da6bc4ce + version: -1 + name: Delete Context + description: |- + Delete field from context. + + This automation runs using the default Limited User role, unless you explicitly change the permissions. + For more information, see the section about permissions here: + https://docs-cortex.paloaltonetworks.com/r/Cortex-XSOAR/6.10/Cortex-XSOAR-Administrator-Guide/Automations + scriptName: DeleteContext + type: regular + iscommand: false + brand: "" + nexttasks: + '#none#': + - "1" + scriptarguments: + all: + simple: "yes" + separatecontext: false + continueonerrortype: "" + view: |- + { + "position": { + "x": 265, + "y": 195 + } + } + note: false + timertriggers: [] + ignoreworker: false + skipunavailable: false + quietmode: 0 + isoversize: false + isautoswitchedtoquietmode: false + "13": + id: "13" + taskid: eb100a21-52ca-46e6-87b7-9189ae6dd46a + type: regular + task: + id: eb100a21-52ca-46e6-87b7-9189ae6dd46a + version: -1 + name: Delete From Context Entities Not Relevant + description: |- + Delete field from context. + + This automation runs using the default Limited User role, unless you explicitly change the permissions. + For more information, see the section about permissions here: + https://docs-cortex.paloaltonetworks.com/r/Cortex-XSOAR/6.10/Cortex-XSOAR-Administrator-Guide/Automations + scriptName: DeleteContext + type: regular + iscommand: false + brand: "" + nexttasks: + '#none#': + - "3" + scriptarguments: + all: + simple: "yes" + keysToKeep: + simple: File,GetIdsFromCustomContent.included_ids.playbook,GetIdsFromCustomContent.included_ids.script,GetIdsFromCustomContent.included_ids.layoutscontainer,GetIdsFromCustomContent.included_ids.incidenttype,GetIdsFromCustomContent.included_ids.incidentfield,GetIdsFromCustomContent.included_ids.classifier,GetIdsFromCustomContent.included_ids.mapper + separatecontext: false + continueonerrortype: "" + view: |- + { + "position": { + "x": 265, + "y": 720 + } + } + note: false + timertriggers: [] + ignoreworker: false + skipunavailable: false + quietmode: 0 + isoversize: false + isautoswitchedtoquietmode: false +system: true +view: |- + { + "linkLabelsPosition": { + "4_5_yes": 0.71 + }, + "paper": { + "dimensions": { + "height": 1435, + "width": 810, + "x": 50, + "y": 50 + } + } + } +inputs: +- key: dry_run + value: + simple: "true" + required: true + description: If true, will not actually delete any content entities. + playbookInputQuery: +outputs: [] +tests: +- No tests (auto formatted) +fromversion: 6.8.0 +marketplaces: +- xsoar diff --git a/Packs/ContentManagement/Playbooks/playbook-Delete_Custom_Content_README.md b/Packs/ContentManagement/Playbooks/playbook-Delete_Custom_Content_README.md new file mode 100644 index 000000000000..fc8424ab15cd --- /dev/null +++ b/Packs/ContentManagement/Playbooks/playbook-Delete_Custom_Content_README.md @@ -0,0 +1,44 @@ +This playbook deletes custom content from the system. It deletes Playbooks, Scripts, Layouts, Classifiers, Mappers, Incident Types and Incident Fields. + +## Dependencies + +This playbook uses the following sub-playbooks, integrations, and scripts. + +### Sub-playbooks + +This playbook does not use any sub-playbooks. + +### Integrations + +This playbook does not use any integrations. + +### Scripts + +* DeleteContent +* DeleteContext +* PrintErrorEntry +* Print +* GetIdsFromCustomContent + +### Commands + +core-api-download + +## Playbook Inputs + +--- + +| **Name** | **Description** | **Default Value** | **Required** | +| --- | --- | --- | --- | +| dry_run | If true, will not actually delete any content entities. | true | Required | + +## Playbook Outputs + +--- +There are no outputs for this playbook. + +## Playbook Image + +--- + +![Delete Custom Content](../doc_files/Delete_Custom_Content.png) diff --git a/Packs/ContentManagement/ReleaseNotes/1_2_9.md b/Packs/ContentManagement/ReleaseNotes/1_2_9.md new file mode 100644 index 000000000000..c9643da67791 --- /dev/null +++ b/Packs/ContentManagement/ReleaseNotes/1_2_9.md @@ -0,0 +1,16 @@ + +#### Playbooks + +##### New: Delete Custom Content + +New: This playbook deletes custom content from the system. It deletes Playbooks, Scripts, Layouts, Classifiers, Mappers, Incident Types and Incident Fields. (Available from Cortex XSOAR 6.8.0). + +#### Scripts + +##### New: GetIdsFromCustomContent + +New: Extract custom content IDs from custom content bundle file and exclude IDs as specified. (Available from Cortex XSOAR 6.8.0). +##### DeleteContent + +- Improved the *readable output* to include the deleted content name. +- Updated the Docker image to: *demisto/python3:3.10.12.62631*. diff --git a/Packs/ContentManagement/Scripts/DeleteContent/DeleteContent.py b/Packs/ContentManagement/Scripts/DeleteContent/DeleteContent.py index f636dfcb6f43..e94af6c57dfc 100644 --- a/Packs/ContentManagement/Scripts/DeleteContent/DeleteContent.py +++ b/Packs/ContentManagement/Scripts/DeleteContent/DeleteContent.py @@ -3,7 +3,6 @@ from abc import ABC, abstractmethod from typing import Tuple -from urllib.parse import quote import requests import json @@ -12,15 +11,33 @@ CORE_PACKS_LIST_URL = "https://raw.githubusercontent.com/demisto/content/master/Tests/Marketplace/core_packs_list.json" -def verify_search_response_in_list(response: Any, name: str): - ids = [entity.get('id', '') for entity in response] if type(response) is list else [] - return False if name not in ids else name +def verify_search_response_in_list(response: Any, id: str) -> str: + """ + Return: + The id if it is in the response, else return empty string + """ + ids = [entity.get('id', '') for entity in response] if isinstance(response, list) else [] + return '' if id not in ids else id -def verify_search_response_in_dict(response: Union[dict, str, list]): - if type(response) is dict and response.get("id"): - return response.get("id") - return False +def verify_search_response_in_dict(response: dict | str | list) -> str: + """ + Return: + The id if it is in the response, else return empty string + """ + if isinstance(response, dict) and response.get("id"): + return response.get("id", "") + return '' + + +def get_the_name_of_specific_id(response: dict | str | list, id: str) -> str: + if isinstance(response, dict): + response = [response] + if isinstance(response, list): + for entity in response: + if entity.get("id") == id: + return entity.get("name", id) + return id class EntityAPI(ABC): @@ -28,387 +45,438 @@ class EntityAPI(ABC): name = '' @abstractmethod - def search_specific_id(self, specific_id: str): + def search_specific_id(self, specific_id: str) -> Tuple[bool, dict | str]: pass @abstractmethod - def search_all(self): + def search_all(self) -> Tuple[bool, dict | str]: pass @abstractmethod - def delete_specific_id(self, specific_id: str): + def delete_specific_id(self, specific_id: str) -> Tuple[bool, dict | str]: pass @abstractmethod - def verify_specific_search_response(self, response: Union[dict, str], name: str): + def verify_specific_search_response(self, response: dict | str, id: str) -> str: pass - def parse_all_entities_response(self, response: Union[dict, str, list]): - return [entity.get('id', '') for entity in response] if type(response) is list else [] + @abstractmethod + def get_name_by_id(self, response: dict | str, id: str) -> str: + pass + + def parse_all_entities_response(self, response) -> list: + return [entity.get('id', '') for entity in response] if isinstance(response, list) else [] -class PlaybookAPI(EntityAPI): # works +class PlaybookAPI(EntityAPI): name = 'playbook' - def search_specific_id(self, specific_id: str): + def search_specific_id(self, specific_id: str) -> Tuple[bool, dict | str]: return execute_command('demisto-api-get', {'uri': f'/playbook/{specific_id}'}, fail_on_error=False) - def search_all(self): + def search_all(self) -> Tuple[bool, dict | str]: return execute_command('demisto-api-post', {'uri': '/playbook/search', 'body': {'page': 0, 'size': 100}}, fail_on_error=False) - def delete_specific_id(self, specific_id: str): + def delete_specific_id(self, specific_id: str) -> Tuple[bool, dict | str]: return execute_command('demisto-api-post', {'uri': '/playbook/delete', 'body': {'id': specific_id}}, fail_on_error=False) - def verify_specific_search_response(self, response: Union[dict, str], name: str): + def verify_specific_search_response(self, response: dict | str, id: str) -> str: return verify_search_response_in_dict(response) - def parse_all_entities_response(self, response: Union[dict, str, list]): + def get_name_by_id(self, response: dict | str, id: str) -> str: + return get_the_name_of_specific_id(response, id) + + def parse_all_entities_response(self, response: dict | str | list) -> list: return [entity.get('id', '') for entity in response.get('playbooks', [])] if type(response) is dict else [] -class IntegrationAPI(EntityAPI): # works +class IntegrationAPI(EntityAPI): name = 'integration' - def search_specific_id(self, specific_id: str): + def search_specific_id(self, specific_id: str) -> Tuple[bool, dict | str]: return execute_command('demisto-api-post', {'uri': '/settings/integration/search', 'body': {'page': 0, 'size': 100, 'query': f'name:"{specific_id}"'}}, fail_on_error=False) - def search_all(self): + def search_all(self) -> Tuple[bool, dict | str]: return execute_command('demisto-api-post', {'uri': '/settings/integration/search', 'body': {'page': 0, 'size': 100}}, fail_on_error=False) - def delete_specific_id(self, specific_id: str): + def delete_specific_id(self, specific_id: str) -> Tuple[bool, dict | str]: return execute_command('demisto-api-post', {'uri': '/settings/integration-conf/delete', - 'body': {'id': quote(specific_id)}}, + 'body': {'id': specific_id}}, fail_on_error=False) - def verify_specific_search_response(self, response: Union[dict, str, list], name: str): - integrations = response.get('configurations', []) if type(response) is dict else response - return verify_search_response_in_list(integrations, name) + def verify_specific_search_response(self, response: dict | str | list, id: str) -> str: + integrations = response.get('configurations', []) if isinstance(response, dict) else response + return verify_search_response_in_list(integrations, id) + + def get_name_by_id(self, response: dict | str | list, id: str) -> str: + integrations = response.get('configurations', []) if isinstance(response, dict) else response + return get_the_name_of_specific_id(integrations, id) - def parse_all_entities_response(self, response: Union[dict, str, list]): - integrations = response.get('configurations', []) if type(response) is dict else response + def parse_all_entities_response(self, response: dict | str | list) -> list: + integrations = response.get('configurations', []) if isinstance(response, dict) else response return [entity.get('id') for entity in integrations] if type(integrations) is list else [] -class ScriptAPI(EntityAPI): # works :) +class ScriptAPI(EntityAPI): name = 'script' always_excluded = ['CommonServerUserPowerShell', 'CommonServerUserPython', 'CommonUserServer', SCRIPT_NAME] - def search_specific_id(self, specific_id: str): + def search_specific_id(self, specific_id: str) -> Tuple[bool, dict | str]: return execute_command('demisto-api-post', {'uri': '/automation/search', - 'body': {'page': 0, 'size': 1, 'query': f'name:"{specific_id}"'}}, + 'body': {'page': 0, 'size': 1, 'query': f'id:"{specific_id}"'}}, fail_on_error=False) - def search_all(self): + def search_all(self) -> Tuple[bool, dict | str]: return execute_command('demisto-api-post', {'uri': '/automation/search', 'body': {'page': 0, 'size': 100}}, fail_on_error=False) - def delete_specific_id(self, specific_id: str): + def delete_specific_id(self, specific_id: str) -> Tuple[bool, dict | str]: return execute_command('demisto-api-post', {'uri': '/automation/delete', 'body': {'script': {'id': specific_id}}}, fail_on_error=False) - def verify_specific_search_response(self, response: Union[dict, str, list], name: str): - scripts = response.get('scripts') if type(response) is dict else response - return verify_search_response_in_list(scripts, name) + def verify_specific_search_response(self, response: dict | str | list, id: str) -> str: + scripts = response.get('scripts') if isinstance(response, dict) else response + return verify_search_response_in_list(scripts, id) + + def get_name_by_id(self, response: dict | str | list, id: str) -> str: + scripts = response.get('scripts', []) if isinstance(response, dict) else response + return get_the_name_of_specific_id(scripts, id) - def parse_all_entities_response(self, response: Union[dict, str, list]): + def parse_all_entities_response(self, response: dict | str | list) -> list: return [entity.get('id', '') for entity in response.get('scripts', [])] if type(response) is dict else [] -class IncidentFieldAPI(EntityAPI): # checked and works +class IncidentFieldAPI(EntityAPI): name = 'incidentfield' - def search_specific_id(self, specific_id: str): + def search_specific_id(self, specific_id: str) -> Tuple[bool, dict | str]: return execute_command('demisto-api-get', {'uri': '/incidentfields'}, fail_on_error=False) - def search_all(self): + def search_all(self) -> Tuple[bool, dict | str]: return execute_command('demisto-api-get', {'uri': '/incidentfields'}, fail_on_error=False) - def delete_specific_id(self, specific_id: str): + def delete_specific_id(self, specific_id: str) -> Tuple[bool, dict | str]: return execute_command('demisto-api-delete', {'uri': f'/incidentfield/{specific_id}'}, fail_on_error=False) - def verify_specific_search_response(self, response: Union[dict, str], name: str): - return verify_search_response_in_list(response, name) + def verify_specific_search_response(self, response: dict | str, id: str) -> str: + return verify_search_response_in_list(response, id) + def get_name_by_id(self, response: dict | str, id: str) -> str: + return get_the_name_of_specific_id(response, id) -class PreProcessingRuleAPI(EntityAPI): # checked and works + +class PreProcessingRuleAPI(EntityAPI): name = 'pre-process-rule' - def search_specific_id(self, specific_id: str): + def search_specific_id(self, specific_id: str) -> Tuple[bool, dict | str]: return execute_command('demisto-api-get', {'uri': '/preprocess/rules'}, fail_on_error=False) - def search_all(self): + def search_all(self) -> Tuple[bool, dict | str]: return execute_command('demisto-api-get', {'uri': '/preprocess/rules'}, fail_on_error=False) - def delete_specific_id(self, specific_id: str): + def delete_specific_id(self, specific_id: str) -> Tuple[bool, dict | str]: return execute_command('demisto-api-delete', {'uri': f'/preprocess/rule/{specific_id}'}, fail_on_error=False) - def verify_specific_search_response(self, response: Union[dict, str, list], name: str): - return verify_search_response_in_list(response, name) + def verify_specific_search_response(self, response: dict | str | list, id: str) -> str: + return verify_search_response_in_list(response, id) + + def get_name_by_id(self, response: dict | str | list, id: str) -> str: + return get_the_name_of_specific_id(response, id) -class WidgetAPI(EntityAPI): # works +class WidgetAPI(EntityAPI): name = 'widget' - def search_specific_id(self, specific_id: str): + def search_specific_id(self, specific_id: str) -> Tuple[bool, dict | str]: return execute_command('demisto-api-get', {'uri': f'/widgets/{specific_id}'}, fail_on_error=False) - def search_all(self): + def search_all(self) -> Tuple[bool, dict | str]: return execute_command('demisto-api-get', {'uri': '/widgets'}, fail_on_error=False) - def delete_specific_id(self, specific_id: str): + def delete_specific_id(self, specific_id: str) -> Tuple[bool, dict | str]: return execute_command('demisto-api-delete', {'uri': f'/widgets/{specific_id}'}, fail_on_error=False) - def verify_specific_search_response(self, response: Union[dict, str], name: str): + def verify_specific_search_response(self, response: dict | str, id: str) -> str: return verify_search_response_in_dict(response) - def parse_all_entities_response(self, response: Union[dict, str, list]): + def get_name_by_id(self, response: dict | str, id: str) -> str: + return get_the_name_of_specific_id(response, id) + + def parse_all_entities_response(self, response: dict | str | list) -> list: if type(response) is dict: return list(response.keys()) return [entity.get('id', '') for entity in response] if type(response) is list else [] -class DashboardAPI(EntityAPI): # works +class DashboardAPI(EntityAPI): name = 'dashboard' - def search_specific_id(self, specific_id: str): + def search_specific_id(self, specific_id: str) -> Tuple[bool, dict | str]: return execute_command('demisto-api-get', {'uri': f'/dashboards/{specific_id}'}, fail_on_error=False) - def search_all(self): + def search_all(self) -> Tuple[bool, dict | str]: return execute_command('demisto-api-get', {'uri': '/dashboards'}, fail_on_error=False) - def delete_specific_id(self, specific_id: str): + def delete_specific_id(self, specific_id: str) -> Tuple[bool, dict | str]: return execute_command('demisto-api-delete', {'uri': f'/dashboards/{specific_id}'}, fail_on_error=False) - def verify_specific_search_response(self, response: Union[dict, str], name: str): + def verify_specific_search_response(self, response: dict | str, id: str) -> str: return verify_search_response_in_dict(response) - def parse_all_entities_response(self, response: Union[dict, str, list]): + def get_name_by_id(self, response: dict | str, id: str) -> str: + return get_the_name_of_specific_id(response, id) + + def parse_all_entities_response(self, response: dict | str | list) -> list: if type(response) is dict: return list(response.keys()) return [entity.get('id', '') for entity in response] if type(response) is list else [] -class ReportAPI(EntityAPI): # works +class ReportAPI(EntityAPI): name = 'report' - def search_specific_id(self, specific_id: str): + def search_specific_id(self, specific_id: str) -> Tuple[bool, dict | str]: return execute_command('demisto-api-get', {'uri': f'/reports/{specific_id}'}, fail_on_error=False) - def search_all(self): + def search_all(self) -> Tuple[bool, dict | str]: return execute_command('demisto-api-get', {'uri': '/reports'}, fail_on_error=False) - def delete_specific_id(self, specific_id: str): + def delete_specific_id(self, specific_id: str) -> Tuple[bool, dict | str]: return execute_command('demisto-api-delete', {'uri': f'/report/{specific_id}'}, fail_on_error=False) - def verify_specific_search_response(self, response: Union[dict, str], name: str): + def verify_specific_search_response(self, response: dict | str, id: str) -> str: return verify_search_response_in_dict(response) + def get_name_by_id(self, response: dict | str, id: str) -> str: + return get_the_name_of_specific_id(response, id) + -class IncidentTypeAPI(EntityAPI): # checked and works +class IncidentTypeAPI(EntityAPI): name = 'incidenttype' - def search_specific_id(self, specific_id: str): + def search_specific_id(self, specific_id: str) -> Tuple[bool, dict | str]: return execute_command('demisto-api-get', {'uri': '/incidenttypes/export'}, fail_on_error=False) - def search_all(self): + def search_all(self) -> Tuple[bool, dict | str]: return execute_command('demisto-api-get', {'uri': '/incidenttypes/export'}, fail_on_error=False) - def delete_specific_id(self, specific_id: str): + def delete_specific_id(self, specific_id: str) -> Tuple[bool, dict | str]: return execute_command('demisto-api-post', {'uri': '/incidenttype/delete', 'body': {'id': specific_id}}, fail_on_error=False) - def verify_specific_search_response(self, response: Union[dict, str, list], name: str): - return verify_search_response_in_list(response, name) + def verify_specific_search_response(self, response: dict | str | list, id: str) -> str: + return verify_search_response_in_list(response, id) + def get_name_by_id(self, response: dict | str | list, id: str) -> str: + return get_the_name_of_specific_id(response, id) -class ClassifierAPI(EntityAPI): # works + +class ClassifierAPI(EntityAPI): name = 'classifier' - def search_specific_id(self, specific_id: str): + def search_specific_id(self, specific_id: str) -> Tuple[bool, dict | str]: return execute_command('demisto-api-get', {'uri': f'/classifier/{specific_id}'}, fail_on_error=False) - def search_all(self): + def search_all(self) -> Tuple[bool, dict | str]: return execute_command('demisto-api-post', {'uri': '/classifier/search', 'body': {'page': 0, 'size': 100}}, fail_on_error=False) - def delete_specific_id(self, specific_id: str): + def delete_specific_id(self, specific_id: str) -> Tuple[bool, dict | str]: return execute_command('demisto-api-delete', {'uri': f'/classifier/{specific_id}'}, fail_on_error=False) - def verify_specific_search_response(self, response: Union[dict, str, list], name: str): + def verify_specific_search_response(self, response: dict | str | list, id: str) -> str: return verify_search_response_in_dict(response) - def parse_all_entities_response(self, response: Union[dict, str, list]): + def get_name_by_id(self, response: dict | str | list, id: str) -> str: + return get_the_name_of_specific_id(response, id) + + def parse_all_entities_response(self, response: dict | str | list) -> list: classifiers = response.get('classifiers', []) if type(response) is dict else [] return [entity.get('id', '') for entity in classifiers] if type(classifiers) is list else [] -class ReputationAPI(EntityAPI): # works +class MapperAPI(ClassifierAPI): + name = 'mapper' + + +class ReputationAPI(EntityAPI): name = 'reputation' - def search_specific_id(self, specific_id: str): + def search_specific_id(self, specific_id: str) -> Tuple[bool, dict | str]: return execute_command('demisto-api-get', {'uri': '/reputation/export'}, fail_on_error=False) - def search_all(self): + def search_all(self) -> Tuple[bool, dict | str]: return execute_command('demisto-api-get', {'uri': '/reputation/export'}, fail_on_error=False) - def delete_specific_id(self, specific_id: str): + def delete_specific_id(self, specific_id: str) -> Tuple[bool, dict | str]: return execute_command('demisto-api-delete', {'uri': f'/reputation/{specific_id}'}, fail_on_error=False) - def verify_specific_search_response(self, response: Union[dict, str, list], name: str): - return verify_search_response_in_list(response, name) + def verify_specific_search_response(self, response: dict | str | list, id: str) -> str: + return verify_search_response_in_list(response, id) + def get_name_by_id(self, response: dict | str | list, id: str) -> str: + return get_the_name_of_specific_id(response, id) -class LayoutAPI(EntityAPI): # works - name = 'layout' - def search_specific_id(self, specific_id: str): +class LayoutAPI(EntityAPI): + name = 'layoutscontainer' + + def search_specific_id(self, specific_id: str) -> Tuple[bool, dict | str]: return execute_command('demisto-api-get', {'uri': f'/layout/{specific_id}'}, fail_on_error=False) - def search_all(self): + def search_all(self) -> Tuple[bool, dict | str]: return execute_command('demisto-api-get', {'uri': '/layouts'}, fail_on_error=False) - def delete_specific_id(self, specific_id: str): + def delete_specific_id(self, specific_id: str) -> Tuple[bool, dict | str]: return execute_command('demisto-api-post', {'uri': f'/layout/{specific_id}/remove', 'body': {}}, fail_on_error=False) - def verify_specific_search_response(self, response: Union[dict, str, list], name: str): + def verify_specific_search_response(self, response: dict | str | list, id: str) -> str: return verify_search_response_in_dict(response) + def get_name_by_id(self, response: dict | str | list, id: str) -> str: + return get_the_name_of_specific_id(response, id) + class JobAPI(EntityAPI): name = 'job' - def search_specific_id(self, specific_id: str): + def search_specific_id(self, specific_id: str) -> Tuple[bool, dict | str]: return execute_command('demisto-api-post', {'uri': '/jobs/search', 'body': {'page': 0, 'size': 1, 'query': f'name:"{specific_id}"'}}, fail_on_error=False) - def search_all(self): + def search_all(self) -> Tuple[bool, dict | str]: return execute_command('demisto-api-post', {'uri': '/jobs/search', 'body': {'page': 0, 'size': 100}}, fail_on_error=False) - def delete_specific_id(self, specific_id: str): + def delete_specific_id(self, specific_id: str) -> Tuple[bool, dict | str]: return execute_command('demisto-api-delete', {'uri': f'jobs/{specific_id}'}, fail_on_error=False) - def verify_specific_search_response(self, response: Union[dict, str], name: str): + def verify_specific_search_response(self, response: dict | str, id: str) -> str: job_params = {} - if type(response) is dict: - search_results = response.get('data') - if search_results: + if isinstance(response, dict): + if search_results := response.get('data'): job_params = search_results[0] - if not job_params or not job_params.get("id"): - return False - return job_params.get("id") + return job_params.get("id", '') if job_params and job_params.get("id") else '' + + def get_name_by_id(self, response: dict | str, id: str) -> str: + job_params = {} + if isinstance(response, dict): + if search_results := response.get('data'): + job_params = search_results[0] + return get_the_name_of_specific_id(job_params, id) - def parse_all_entities_response(self, response: Union[dict, str, list]): + def parse_all_entities_response(self, response: dict | str | list) -> list: return [entity.get('name', '') for entity in response.get('data', [])] if type(response) is dict else [] class ListAPI(EntityAPI): name = 'list' - def search_specific_id(self, specific_id: str): + def search_specific_id(self, specific_id: str) -> Tuple[bool, dict | str]: return execute_command('demisto-api-get', {'uri': f'/lists/download/{specific_id}'}, fail_on_error=False) - def search_all(self): + def search_all(self) -> Tuple[bool, dict | str]: return execute_command('demisto-api-get', {'uri': '/lists/names'}, fail_on_error=False) - def delete_specific_id(self, specific_id: str): + def delete_specific_id(self, specific_id: str) -> Tuple[bool, dict | str]: return execute_command('demisto-api-post', {'uri': '/lists/delete', 'body': {'id': specific_id}}, fail_on_error=False) - def verify_specific_search_response(self, response: Union[dict, str], name: str): - if response: - return name - return False + def verify_specific_search_response(self, response: Union[dict, str], id: str) -> str: + return id if response else '' - def parse_all_entities_response(self, response: Union[dict, str, list]): + def get_name_by_id(self, response: dict | str, id: str) -> str: + return id + + def parse_all_entities_response(self, response: list) -> list: return response @@ -422,58 +490,62 @@ def __init__(self, proxy_skip=True, verify=True): core_packs_response = requests.get(CORE_PACKS_LIST_URL, verify=verify) self.always_excluded = json.loads(core_packs_response.text).get("core_packs_list") + self.always_excluded - def search_specific_id(self, specific_id: str): + def search_specific_id(self, specific_id: str) -> Tuple[bool, dict | str]: return execute_command('demisto-api-get', {'uri': f'/contentpacks/installed/{specific_id}'}, fail_on_error=False) - def search_all(self): + def search_all(self) -> Tuple[bool, dict | str]: return execute_command('demisto-api-get', {'uri': '/contentpacks/installed-expired'}, fail_on_error=False) - def delete_specific_id(self, specific_id: str): + def delete_specific_id(self, specific_id: str) -> Tuple[bool, dict | str]: return execute_command('demisto-api-delete', {'uri': f'/contentpacks/installed/{specific_id}'}, fail_on_error=False) - def verify_specific_search_response(self, response: Union[dict, str], name: str): + def verify_specific_search_response(self, response: Union[dict, str], id: str) -> str: return verify_search_response_in_dict(response) + def get_name_by_id(self, response: dict | str, id: str) -> str: + return get_the_name_of_specific_id(response, id) + -def search_and_delete_existing_entity(name: str, entity_api: EntityAPI, dry_run: bool = True) -> bool: - """Searches the machine for previously configured entity_types with the given name. +def search_and_delete_existing_entity(id: str, entity_api: EntityAPI, dry_run: bool = True) -> Tuple[bool, str]: + """Searches the machine for previously configured entity_types with the given id. Args: - name (str): The name of the entity to update it's past configurations. + id (str): The id of the entity to update it's past configurations. Returns: True if deleted, False otherwise. + The name of the entity if it exists, otherwise the given id. """ - status, res = entity_api.search_specific_id(specific_id=name) + status, res = entity_api.search_specific_id(specific_id=id) if not status: - demisto.debug(f'Could not find {entity_api.name} with id {name} - Response:\n{res}') - return False + demisto.debug(f'Could not find {entity_api.name} with id {id} - Response:\n{res}') + return False, id - specific_id = entity_api.verify_specific_search_response(res.get('response'), name) + specific_id = entity_api.verify_specific_search_response(res.get('response', {}), id) # type: ignore[union-attr] + specific_name = entity_api.get_name_by_id(res.get('response', {}), id) # type: ignore[union-attr] if not specific_id: - return False + return False, id if not dry_run: status, res = entity_api.delete_specific_id(specific_id=specific_id) else: - demisto.debug(f'DRY RUN - Not deleting {entity_api.name} with id {name}.') + demisto.debug(f'DRY RUN - Not deleting {entity_api.name} with id "{id}" and name "{specific_name}".') status = True - res = True if not status: - demisto.debug(f'Could not delete {entity_api.name} with id {name} - Response:\n{res}') - return False + demisto.debug(f'Could not delete {entity_api.name} with id "{id}" and name "{specific_name}" - Response:\n{res}') + return False, specific_name - return True + return True, specific_name def search_for_all_entities(entity_api: EntityAPI) -> list: @@ -492,13 +564,11 @@ def search_for_all_entities(entity_api: EntityAPI) -> list: demisto.debug(error_message) raise Exception(error_message) - entity_ids = entity_api.parse_all_entities_response(res.get('response', {})) + return entity_api.parse_all_entities_response(res.get('response', {})) # type: ignore[union-attr] - return entity_ids - -def get_and_delete_entities(entity_api: EntityAPI, excluded_ids: list = [], included_ids: list = [], - dry_run: bool = True) -> Tuple[list, list, list]: +def get_and_delete_entities(entity_api: EntityAPI, excluded_ids: list = [], included_ids: list = [], dry_run: bool = True + ) -> Tuple[list[dict], list[dict], list]: """Search and delete entities with provided EntityAPI. Args: @@ -508,11 +578,11 @@ def get_and_delete_entities(entity_api: EntityAPI, excluded_ids: list = [], incl dry_run (bool): If true, will not really delete anything. Returns: - (list) successfully deleted ids, (list) not deleted ids + (list) successfully deleted ids, (list) not deleted ids, (list) extended excluded ids. """ demisto.debug(f'Starting handling {entity_api.name} entities.') - succesfully_deleted = [] - not_deleted = [] + successfully_deleted: list[dict] = [] + not_deleted: list[dict] = [] extended_excluded_ids = excluded_ids.copy() if not included_ids and not excluded_ids: @@ -527,12 +597,14 @@ def get_and_delete_entities(entity_api: EntityAPI, excluded_ids: list = [], incl if included_ids: for included_id in included_ids: if included_id in new_included_ids: - if search_and_delete_existing_entity(included_id, entity_api=entity_api, dry_run=dry_run): - succesfully_deleted.append(included_id) + status, name = search_and_delete_existing_entity(included_id, entity_api=entity_api, dry_run=dry_run) + id_and_name = {'id': included_id, 'name': name} + if status: + successfully_deleted.append(id_and_name) else: - not_deleted.append(included_id) + not_deleted.append(id_and_name) else: - not_deleted.append(included_id) + not_deleted.append({'id': included_id, 'name': included_id}) else: all_entities = search_for_all_entities(entity_api=entity_api) @@ -541,41 +613,45 @@ def get_and_delete_entities(entity_api: EntityAPI, excluded_ids: list = [], incl for entity_id in all_entities: if entity_id not in extended_excluded_ids: - if search_and_delete_existing_entity(entity_id, entity_api=entity_api, dry_run=dry_run): - succesfully_deleted.append(entity_id) + status, name = search_and_delete_existing_entity(entity_id, entity_api=entity_api, dry_run=dry_run) + id_and_name = {'id': entity_id, 'name': name} + if status: + successfully_deleted.append(id_and_name) else: demisto.debug(f'Did not find or could not delete {entity_api.name} with ' f'id {entity_id} in xsoar.') - not_deleted.append(entity_id) + not_deleted.append(id_and_name) else: - not_deleted.append(entity_id) + not_deleted.append({'id': entity_id, 'name': entity_id}) - return succesfully_deleted, not_deleted, extended_excluded_ids + return successfully_deleted, not_deleted, extended_excluded_ids def get_deletion_status(excluded: list, included: list, deleted: list, undeleted: list) -> bool: + deleted_ids = [entity.get('id') for entity in deleted] + undeleted_ids = [entity.get('id') for entity in undeleted] if excluded: - if undeleted == excluded: + if undeleted_ids == excluded: return True else: for excluded_id in excluded: - if excluded_id in deleted: + if excluded_id in deleted_ids: return False return True elif included: - if set(deleted) == set(included): + if set(deleted_ids) == set(included): return True # Nothing excluded - elif not undeleted: + elif not undeleted_ids: return True return False -def handle_content_enitity(entity_api: EntityAPI, - included_ids_dict: Optional[dict], - excluded_ids_dict: Optional[dict], - dry_run: bool) -> Tuple[bool, dict, dict]: +def handle_content_entity(entity_api: EntityAPI, + included_ids_dict: Optional[dict], + excluded_ids_dict: Optional[dict], + dry_run: bool) -> Tuple[bool, dict, dict]: excluded_ids = excluded_ids_dict.get(entity_api.name, []) if excluded_ids_dict else [] included_ids = included_ids_dict.get(entity_api.name, []) if included_ids_dict else [] @@ -592,9 +668,7 @@ def handle_content_enitity(entity_api: EntityAPI, def handle_input_json(input_dict: Any) -> Any: - if type(input_dict) == str: - return json.loads(input_dict) - return input_dict + return json.loads(input_dict) if isinstance(input_dict, str) else input_dict def get_and_delete_needed_ids(args: dict) -> CommandResults: @@ -625,35 +699,40 @@ def get_and_delete_needed_ids(args: dict) -> CommandResults: verify_cert = argToBoolean(args.get('verify_cert', 'true')) entities_to_delete = [InstalledPackAPI(proxy_skip=skip_proxy, verify=verify_cert), IntegrationAPI(), ScriptAPI(), - PlaybookAPI(), IncidentFieldAPI(), + IncidentTypeAPI(), PlaybookAPI(), IncidentFieldAPI(), PreProcessingRuleAPI(), WidgetAPI(), DashboardAPI(), ReportAPI(), JobAPI(), ListAPI(), - IncidentTypeAPI(), ClassifierAPI(), ReputationAPI(), LayoutAPI()] + ClassifierAPI(), MapperAPI(), ReputationAPI(), LayoutAPI()] - all_deleted: dict = dict() - all_not_deleted: dict = dict() + all_deleted: dict = {} + all_not_deleted: dict = {} all_deletion_statuses: list = [] for entity in entities_to_delete: - entity_deletion_status, deleted, undeleted = handle_content_enitity(entity, include_ids, exclude_ids, dry_run) - all_deleted.update(deleted) - all_not_deleted.update(undeleted) + entity_deletion_status, deleted, undeleted = handle_content_entity(entity, include_ids, exclude_ids, dry_run) + all_deleted |= deleted + all_not_deleted |= undeleted all_deletion_statuses.append(entity_deletion_status) deletion_status = 'Failed' if dry_run: deletion_status = 'Dry run, nothing really deleted.' - else: - if all(all_deletion_statuses): - deletion_status = 'Completed' + elif all(all_deletion_statuses): + deletion_status = 'Completed' + successfully_deleted_ids = {key: [value['id'] for value in lst] for key, lst in all_deleted.items() if lst} + successfully_deleted_names = {key: [value['name'] for value in lst] for key, lst in all_deleted.items() if lst} + not_deleted_ids = {key: [value['id'] for value in lst] for key, lst in all_not_deleted.items() if lst} + not_deleted_names = {key: [value['name'] for value in lst] for key, lst in all_not_deleted.items() if lst} return CommandResults( outputs_prefix='ConfigurationSetup.Deletion', outputs_key_field='name', outputs={ # Only show keys with values. - 'successfully_deleted': {key: value for key, value in all_deleted.items() if value}, - 'not_deleted': {key: value for key, value in all_not_deleted.items() if value}, + 'successfully_deleted': successfully_deleted_ids, + 'not_deleted': not_deleted_ids, 'status': deletion_status, }, + readable_output=f'### Deletion status: {deletion_status}\n' + tableToMarkdown( + 'Successfully deleted', successfully_deleted_names) + tableToMarkdown('Not deleted', not_deleted_names) ) diff --git a/Packs/ContentManagement/Scripts/DeleteContent/DeleteContent.yml b/Packs/ContentManagement/Scripts/DeleteContent/DeleteContent.yml index 24fbd830ed52..e5c73d05fd05 100644 --- a/Packs/ContentManagement/Scripts/DeleteContent/DeleteContent.yml +++ b/Packs/ContentManagement/Scripts/DeleteContent/DeleteContent.yml @@ -13,7 +13,7 @@ args: secret: false - auto: PREDEFINED default: false - description: If set to true, the flow will work as usuall except that no content items will be deleted from the system. + description: If set to true, the flow will work as usual except that no content items will be deleted from the system. isArray: false name: dry_run required: true @@ -66,7 +66,7 @@ tags: timeout: 3600 type: python subtype: python3 -dockerimage: demisto/python3:3.10.10.48392 +dockerimage: demisto/python3:3.10.12.62631 tests: - No tests (auto formatted) fromversion: 6.0.0 diff --git a/Packs/ContentManagement/Scripts/DeleteContent/DeleteContent_test.py b/Packs/ContentManagement/Scripts/DeleteContent/DeleteContent_test.py index 3e50f3a9918e..414c479673c5 100644 --- a/Packs/ContentManagement/Scripts/DeleteContent/DeleteContent_test.py +++ b/Packs/ContentManagement/Scripts/DeleteContent/DeleteContent_test.py @@ -20,7 +20,7 @@ 'incidenttype': ['incidenttype1', 'incidenttype2'], 'classifier': ['classifier1', 'classifier2'], 'reputation': ['reputation1', 'reputation2'], - 'layout': ['layout1', 'layout2'] + 'layoutscontainer': ['layout1', 'layout2'] } @@ -123,7 +123,7 @@ def search_response(self, command_name, command_args): command_uri = command_args.get('uri') if command_uri == '/automation/search': if command_args.get('body', {}).get('size') == 1: - script_name = command_args.get('body', {}).get('query').split('name:"')[1].split('"')[0] + script_name = command_args.get('body', {}).get('query').split('id:"')[1].split('"')[0] if script_name in self.xsoar_state_ids: # if search and found return True, {'scripts': [{'id': script_name}]} @@ -373,7 +373,7 @@ def delete_response(self, command_name, command_args): class MockLayoutResponses(MockEntityResponses): - entity_name = 'layout' + entity_name = 'layoutscontainer' def search_response(self, command_name, command_args): command_uri = command_args.get('uri') @@ -449,7 +449,7 @@ def mock_demisto_responses(command_name, command_args, xsoar_ids_state): 'incidenttype': ['incidenttype1'], 'classifier': ['classifier1'], 'reputation': ['reputation1'], - 'layout': ['layout1']}, + 'layoutscontainer': ['layout1']}, 'delete_unspecified': 'false'}, XSOAR_IDS_FULL_STATE, { 'not_deleted': {}, 'successfully_deleted': {'job': ['job1'], 'list': ['list1'], 'pack': ['installed_pack_id1'], @@ -457,7 +457,7 @@ def mock_demisto_responses(command_name, command_args, xsoar_ids_state): 'incidentfield': ['incidentfield1'], 'pre-process-rule': ['pre-process-rule1'], 'widget': ['widget1'], 'dashboard': ['dashboard1'], 'report': ['report1'], 'incidenttype': ['incidenttype1'], 'classifier': ['classifier1'], - 'reputation': ['reputation1'], 'layout': ['layout1']}, + 'reputation': ['reputation1'], 'layoutscontainer': ['layout1']}, 'status': 'Completed'}, id='delete only included ids'), pytest.param( {'dry_run': 'false', 'exclude_ids_dict': {'job': ['job1'], @@ -474,20 +474,20 @@ def mock_demisto_responses(command_name, command_args, xsoar_ids_state): 'incidenttype': ['incidenttype1'], 'classifier': ['classifier1'], 'reputation': ['reputation1'], - 'layout': ['layout1']}}, XSOAR_IDS_FULL_STATE, { + 'layoutscontainer': ['layout1']}}, XSOAR_IDS_FULL_STATE, { 'not_deleted': {'pack': ['installed_pack_id1', 'Base'], 'job': ['job1'], 'list': ['list1'], 'script': ['script1', 'CommonUserServer'], 'playbook': ['playbook1'], 'integration': ['integration1'], 'incidentfield': ['incidentfield1'], 'pre-process-rule': ['pre-process-rule1'], 'widget': ['widget1'], 'dashboard': ['dashboard1'], 'report': ['report1'], 'incidenttype': ['incidenttype1'], 'classifier': ['classifier1'], - 'reputation': ['reputation1'], 'layout': ['layout1']}, + 'reputation': ['reputation1'], 'layoutscontainer': ['layout1']}, 'successfully_deleted': { # packs can only be deleted when included. 'job': ['job2'], 'list': ['list2'], 'playbook': ['playbook2'], 'script': ['script2'], 'integration': ['integration2'], 'incidentfield': ['incidentfield2'], 'pre-process-rule': ['pre-process-rule2'], 'widget': ['widget2'], 'dashboard': ['dashboard2'], 'report': ['report2'], 'incidenttype': ['incidenttype2'], - 'classifier': ['classifier2'], 'reputation': ['reputation2'], 'layout': ['layout2'], + 'classifier': ['classifier2'], 'reputation': ['reputation2'], 'layoutscontainer': ['layout2'], 'pack': ['installed_pack_id2'], }, 'status': 'Completed'}, id='dont delete excluded ids'), @@ -506,7 +506,7 @@ def mock_demisto_responses(command_name, command_args, xsoar_ids_state): 'incidenttype': ['incidenttype3'], 'classifier': ['classifier3'], 'reputation': ['reputation3'], - 'layout': ['layout3']}}, XSOAR_IDS_FULL_STATE, { + 'layoutscontainer': ['layout3']}}, XSOAR_IDS_FULL_STATE, { 'not_deleted': {'pack': ['Base'], 'script': ['CommonUserServer']}, 'successfully_deleted': {'job': ['job1', 'job2'], 'list': ['list1', 'list2'], 'script': ['script1', 'script2'], 'playbook': ['playbook1', 'playbook2'], @@ -518,7 +518,7 @@ def mock_demisto_responses(command_name, command_args, xsoar_ids_state): 'incidenttype': ['incidenttype1', 'incidenttype2'], 'classifier': ['classifier1', 'classifier2'], 'reputation': ['reputation1', 'reputation2'], - 'layout': ['layout1', 'layout2'], + 'layoutscontainer': ['layout1', 'layout2'], 'pack': ['installed_pack_id1', 'installed_pack_id2']}, 'status': 'Completed'}, id='exclude unfound id'), pytest.param( @@ -536,14 +536,14 @@ def mock_demisto_responses(command_name, command_args, xsoar_ids_state): 'incidenttype': ['incidenttype3'], 'classifier': ['classifier3'], 'reputation': ['reputation3'], - 'layout': ['layout3']}}, XSOAR_IDS_FULL_STATE, { + 'layoutscontainer': ['layout3']}}, XSOAR_IDS_FULL_STATE, { 'not_deleted': {'job': ['job3'], 'pack': ['installed_pack3'], 'list': ['list3'], 'script': ['script3'], 'playbook': ['playbook3'], 'integration': ['integration3'], 'incidentfield': ['incidentfield3'], 'pre-process-rule': ['pre-process-rule3'], 'widget': ['widget3'], 'dashboard': ['dashboard3'], 'report': ['report3'], 'incidenttype': ['incidenttype3'], 'classifier': ['classifier3'], 'reputation': ['reputation3'], - 'layout': ['layout3']}, + 'layoutscontainer': ['layout3']}, 'successfully_deleted': {}, 'status': 'Failed'}, id='include unfound id'), pytest.param( diff --git a/Packs/ContentManagement/Scripts/GetIdsFromCustomContent/GetIdsFromCustomContent.py b/Packs/ContentManagement/Scripts/GetIdsFromCustomContent/GetIdsFromCustomContent.py new file mode 100644 index 000000000000..3449892724b2 --- /dev/null +++ b/Packs/ContentManagement/Scripts/GetIdsFromCustomContent/GetIdsFromCustomContent.py @@ -0,0 +1,165 @@ +import demistomock as demisto +from CommonServerPython import * +from CommonServerUserPython import * + +from typing import Dict, Any, Tuple +from demisto_sdk.commands.common.tools import _get_file_id, get_file_displayed_name, find_type, get_file +from pathlib import Path +from collections import defaultdict + +import traceback +import tempfile +import tarfile +import json +import os + + +def update_file_prefix(file_name: str) -> str: + """ + Custom content scripts are prefixed with automation instead of script. + Removing the "playbook-" prefix from files name. + """ + if file_name.startswith('playbook-'): + return file_name[len('playbook-'):] + if file_name.startswith('automation-'): + return file_name.replace('automation-', 'script-') + return file_name + + +def get_content_details(tar_file_handler: Any, member_file: Any) -> Tuple[str, dict]: + """Get content id from tar member file. + + Args: + tar_file_handler: Tarfile open handler that contains the member file to inspect. + member_file: The member file in the tar file to inspect. + + Return: + (entity, file_id_name) of the member file. + """ + file_type_str = '' + file_id = None + with tempfile.TemporaryDirectory() as tmp_dir_name: + file_name = update_file_prefix(member_file.name.strip('/')) + file_path = os.path.join(tmp_dir_name, file_name) + with open(file_path, 'w') as file_desc: + if extracted_file := tar_file_handler.extractfile(member_file): + file_desc.write(extracted_file.read().decode('utf-8')) + else: + raise Exception(f'Could not extract file {file_name} from tar: {file_path}') + + if not os.path.isfile(file_path): + raise Exception(f"Could not create file {file_path}") + + file_type = find_type(path=file_path) + file_type_str = file_type.value if file_type else '' + if file_type_str == 'automation': + file_type_str = 'script' + + file_type_str = 'list' if not file_type_str and file_name.startswith('list-') else file_type_str + file_dict = get_file(file_path, Path(file_name).suffix[1:]) + file_id = _get_file_id(file_type_str, file_dict) + file_id = file_id if file_id else file_dict.get('id') + file_name = get_file_displayed_name(file_path) + + file_id_name = {"id": file_id, "name": file_name} + return file_type_str, file_id_name + + +def get_custom_content_ids(file_entry_id: Any) -> dict: + """Get custom content ids from custom content bundle. + + Args: + file_entry_id (str): The entry id of the custom content zip file. + + Return: + A dict of custom content ids. + """ + custom_content_ids: defaultdict[Any, list] = defaultdict(list) + get_file_path_res = demisto.getFilePath(file_entry_id) + custom_content_file_path = get_file_path_res.get('path') + if not custom_content_file_path: + raise ValueError(f"Could not find file path for entry id {file_entry_id}") + custom_content_tar_file = tarfile.open(custom_content_file_path) + custom_content_members = custom_content_tar_file.getmembers() + + for custom_content_member in custom_content_members: + entity, entity_id_name = get_content_details(custom_content_tar_file, custom_content_member) + if entity and entity_id_name.get('id'): + custom_content_ids[entity].append(entity_id_name) + else: + raise Exception(f"Could not parse content type and id from file name {custom_content_member.name}") + + return custom_content_ids + + +''' COMMAND FUNCTION ''' + + +def filter_lists(include: list, exclude: list) -> list: + return [item for item in include if item.get('id') not in exclude] + + +def get_included_ids_command(args: Dict[str, Any]) -> CommandResults: + """Get included ids from installed custom content unless id is excluded. + + Args: + exclude_ids_list (List[Dict[str:List[str]]]): A list of dicts, each specifies entity ids to exclude. + (example: [{'integration': ['HelloWorld', 'MyIntegration']}, {'script': ['say_hello']}] + file_entry_id (str): The entry id of the custom content zip file. + + Return: + CommandResults Outputs with included ids dict and excluded ids dict, ready to pass to DeleteContent script. + """ + if excluded_ids_dicts := args.get('exclude_ids_list', []): + if not isinstance(excluded_ids_dicts, list): + try: + excluded_ids_dicts = json.loads(str(args.get('exclude_ids_list'))) + except json.JSONDecodeError as err: + raise ValueError(f'Failed decoding excluded_ids_list as json: {str(err)}') + + custom_content_ids = get_custom_content_ids(file_entry_id=args.get('file_entry_id')) + + included_custom_ids_names = {} + excluded_ids: defaultdict[Any, list] = defaultdict(list) + if excluded_ids_dicts: + # Merge exclusion dicts + for excluded_ids_dict in excluded_ids_dicts: + for excluded_entity in excluded_ids_dict.keys(): + excluded_ids[excluded_entity] += excluded_ids_dict.get(excluded_entity, []) + + # Exclude what is relevant + for custom_entity in custom_content_ids.keys(): + included_custom_ids_names[custom_entity] = filter_lists(include=custom_content_ids.get(custom_entity, []), + exclude=excluded_ids.get(custom_entity, [])) + + # Remove included entities from excluded dict + for entity in included_custom_ids_names: + if entity in excluded_ids.keys(): + excluded_ids.pop(entity) + + else: + included_custom_ids_names = custom_content_ids + + included_custom_ids = {key: [value['id'] for value in lst] for key, lst in included_custom_ids_names.items() if lst} + included_custom_name = {key: [value['name'] for value in lst] for key, lst in included_custom_ids_names.items() if lst} + return CommandResults( + outputs_prefix='GetIdsFromCustomContent', + outputs_key_field='', + outputs={'included_ids': included_custom_ids, + 'excluded_ids': {key: value for key, value in excluded_ids.items() if value}}, + readable_output=tableToMarkdown('Included ids', included_custom_name) + tableToMarkdown('Excluded ids', excluded_ids) + ) + + +def main(): # pragma: no cover + try: + return_results(get_included_ids_command(demisto.args())) + except Exception as ex: + return_error(f'Failed to execute GetIdsFromCustomContent. Error: {str(ex)}\n Traceback: {traceback.format_exc()}') + + +''' ENTRY POINT ''' + + +if __name__ in ('__main__', '__builtin__', 'builtins'): # pragma: no cover + main() diff --git a/Packs/ContentManagement/Scripts/GetIdsFromCustomContent/GetIdsFromCustomContent.yml b/Packs/ContentManagement/Scripts/GetIdsFromCustomContent/GetIdsFromCustomContent.yml new file mode 100644 index 000000000000..09c1198fcf91 --- /dev/null +++ b/Packs/ContentManagement/Scripts/GetIdsFromCustomContent/GetIdsFromCustomContent.yml @@ -0,0 +1,35 @@ +args: +- default: false + description: 'List of dictionaries of IDs to exclude in a JSON format (e.g., [{"job": ["job1", "job2"], "pack": ["pack1"]}, {"job": ["job3"]}]' + isArray: false + name: exclude_ids_list + required: false + secret: false +- default: false + description: The entry ID of the custom content tar file. + isArray: false + name: file_entry_id + required: false + secret: false +comment: Extract custom content IDs from custom content bundle file and exclude IDs as specified. +commonfields: + id: GetIdsFromCustomContent + version: -1 +enabled: false +name: GetIdsFromCustomContent +outputs: +- contextPath: GetIdsFromCustomContent.included_ids + description: Dictionary of IDs of custom content excluding the ones specified. + type: Unknown +- contextPath: GetIdsFromCustomContent.excluded_ids + description: Dictionary of IDs of custom content excluding the ones specified. + type: Unknown +script: '-' +system: false +timeout: '0' +type: python +subtype: python3 +dockerimage: demisto/xsoar-tools:1.0.0.62936 +fromversion: 6.8.0 +tests: +- No tests (auto formatted) diff --git a/Packs/ContentManagement/Scripts/GetIdsFromCustomContent/GetIdsFromCustomContent_test.py b/Packs/ContentManagement/Scripts/GetIdsFromCustomContent/GetIdsFromCustomContent_test.py new file mode 100644 index 000000000000..c67abcfd8f8e --- /dev/null +++ b/Packs/ContentManagement/Scripts/GetIdsFromCustomContent/GetIdsFromCustomContent_test.py @@ -0,0 +1,142 @@ +import demistomock as demisto +import pytest + +from GetIdsFromCustomContent import get_included_ids_command + + +EXAMPLE_CUSTOM_CONTENT_PATH = 'test_data/content-bundle-for-test.tar.gz' +EXAMPLE_CUSTOM_CONTENT_NAME = 'content-bundle-for-test.tar.gz' +IDS_IN_EXAMPLE_CONFIG = { + 'included_ids': { + 'dashboard': ['e499e9c3-0383-46ce-831d-98c3d501641d'], + 'incidentfield': ['incident_xdrfilename'], + 'incidenttype': ['TOPdesk Incident'], + 'indicatorfield': ['indicator_xdrstatus'], + 'integration': ['pff'], + 'layoutscontainer': ['Carbon Black EDR Incidents'], + 'list': ['list1'], + 'mapper': ['TOPdesk-incoming-mapper'], + 'playbook': ['UnzipFile-Test'], + 'pre-process-rule': ['1e61a15a-1c1e-481c-8d78-211c99099c23'], + 'report': ['4a62cafd-03f1-4a02-85b2-d6b58ec8184f'], + 'reputation': ['7c7f69e3-56d4-4d13-8285-8bf10d4949b4'], + 'script': ['ZipStrings'], + 'widget': ['0b674563-66ca-4c41-8eac-134722296026'], + }, + 'excluded_ids': {} +} + + +@pytest.mark.parametrize('exclude_ids_list, expected_outputs', [ + pytest.param([], IDS_IN_EXAMPLE_CONFIG, id='exclude none'), + pytest.param([{'dashboard': ['e499e9c3-0383-46ce-831d-98c3d501641d'], + 'incidentfield': ['incident_xdrfilename'], + 'incidenttype': ['TOPdesk Incident'], + 'indicatorfield': ['indicator_xdrstatus'], + 'integration': ['pff'], + 'layoutscontainer': ['Carbon Black EDR Incidents'], + 'list': ['list1'], + 'mapper': ['TOPdesk-incoming-mapper'], + 'playbook': ['UnzipFile-Test'], + 'pre-process-rule': ['1e61a15a-1c1e-481c-8d78-211c99099c23'], + 'report': ['4a62cafd-03f1-4a02-85b2-d6b58ec8184f'], + 'reputation': ['7c7f69e3-56d4-4d13-8285-8bf10d4949b4'], + 'script': ['ZipStrings'], + 'widget': ['0b674563-66ca-4c41-8eac-134722296026']}], + {'excluded_ids': {}, 'included_ids': {}}, id='exclude all'), + pytest.param([{'dashboard': ['e499e9c3-0383-46ce-831d-98c3d501641d']}], + {'included_ids': { + 'incidentfield': ['incident_xdrfilename'], + 'incidenttype': ['TOPdesk Incident'], + 'indicatorfield': ['indicator_xdrstatus'], + 'integration': ['pff'], + 'layoutscontainer': ['Carbon Black EDR Incidents'], + 'list': ['list1'], + 'mapper': ['TOPdesk-incoming-mapper'], + 'playbook': ['UnzipFile-Test'], + 'pre-process-rule': ['1e61a15a-1c1e-481c-8d78-211c99099c23'], + 'report': ['4a62cafd-03f1-4a02-85b2-d6b58ec8184f'], + 'reputation': ['7c7f69e3-56d4-4d13-8285-8bf10d4949b4'], + 'script': ['ZipStrings'], + 'widget': ['0b674563-66ca-4c41-8eac-134722296026']}, + 'excluded_ids': {}}, id='exclude 1') +]) +def test_get_included_ids_command(mocker, exclude_ids_list, expected_outputs): + """ + Given: + An example custom content file. + + When: + Running GetIdsFromCustomContent. + + Then: + Assert the right ids are returned. + """ + mocker.patch.object(demisto, "getFilePath", return_value={ + "path": EXAMPLE_CUSTOM_CONTENT_PATH, + "name": EXAMPLE_CUSTOM_CONTENT_NAME + }) + + args = { + 'file_entry_id': 'some_id', + 'exclude_ids_list': exclude_ids_list + } + response = get_included_ids_command(args) + assert response.outputs == expected_outputs + + +@pytest.mark.parametrize('custom_content_ids, exclude_ids_list, expected_outputs', [ + pytest.param({'dashboard': [{'id': 'dashboard1', 'name': 'dashboard1'}, + {'id': 'dashboard2', 'name': 'dashboard2'}, + {'id': 'dashboard3', 'name': 'dashboard3'}]}, + [{'dashboard': ['dashboard1']}, {'dashboard': ['dashboard2']}], + {'included_ids': {'dashboard': ['dashboard3']}, 'excluded_ids': {}}, + id='exclude dashboard1, dashboard2, include dashboard3'), + pytest.param({'dashboard': [{'id': 'dashboard1', 'name': 'dashboard1'}]}, + [{'report': ['report1']}], + {'included_ids': {'dashboard': ['dashboard1']}, 'excluded_ids': {'report': ['report1']}}, + id='include dashboard1, exclude report1'), + pytest.param({}, [{'report': ['report1']}, {'report': ['report2', 'report3']}], + {'included_ids': {}, 'excluded_ids': {'report': ['report1', 'report2', 'report3']}}, + id='include dashboard1, exclude report1') +]) +def test_get_included_ids_with_excluded(mocker, custom_content_ids, exclude_ids_list, expected_outputs): + """ + Given: + An example custom content file. + An excluded_ids_list. + + When: + Running GetIdsFromCustomContent. + + Then: + Assert the right ids are returned. + """ + mocker.patch("GetIdsFromCustomContent.get_custom_content_ids", return_value=custom_content_ids) + + args = { + 'file_entry_id': 'some_id', + 'exclude_ids_list': exclude_ids_list + } + response = get_included_ids_command(args) + assert response.outputs == expected_outputs + + +def test_get_included_ids_with_bad_excluded_ids(): + """ + Given: + A bad excluded_ids_list. + + When: + Running GetIdsFromCustomContent. + + Then: + Assert exception is raised and a relevant error message is printed. + """ + args = { + 'file_entry_id': 'some_id', + 'exclude_ids_list': 'not at all a json ::' + } + with pytest.raises(ValueError) as err: + get_included_ids_command(args) + assert 'Failed decoding excluded_ids_list as json' in str(err.value) diff --git a/Packs/ContentManagement/Scripts/GetIdsFromCustomContent/README.md b/Packs/ContentManagement/Scripts/GetIdsFromCustomContent/README.md new file mode 100644 index 000000000000..70530bd7cd2f --- /dev/null +++ b/Packs/ContentManagement/Scripts/GetIdsFromCustomContent/README.md @@ -0,0 +1,28 @@ +Extract custom content IDs from custom content bundle file and exclude IDs as specified. + +## Script Data + +--- + +| **Name** | **Description** | +| --- | --- | +| Script Type | python3 | +| Cortex XSOAR Version | 6.8.0 | + +## Inputs + +--- + +| **Argument Name** | **Description** | +| --- | --- | +| exclude_ids_list | List of dictionaries of IDs to exclude in a JSON format \(e.g., \[\{"job": \["job1", "job2"\], "pack": \["pack1"\]\}, \{"job": \["job3"\]\}\] | +| file_entry_id | The entry ID of the custom content tar file. | + +## Outputs + +--- + +| **Path** | **Description** | **Type** | +| --- | --- | --- | +| GetIdsFromCustomContent.included_ids | Dictionary of IDs of custom content excluding the ones specified. | Unknown | +| GetIdsFromCustomContent.excluded_ids | Dictionary of IDs of custom content excluding the ones specified. | Unknown | diff --git a/Packs/ContentManagement/Scripts/GetIdsFromCustomContent/test_data/content-bundle-for-test.tar.gz b/Packs/ContentManagement/Scripts/GetIdsFromCustomContent/test_data/content-bundle-for-test.tar.gz new file mode 100644 index 000000000000..7738b1b3747c Binary files /dev/null and b/Packs/ContentManagement/Scripts/GetIdsFromCustomContent/test_data/content-bundle-for-test.tar.gz differ diff --git a/Packs/ContentManagement/doc_files/Delete_Custom_Content.png b/Packs/ContentManagement/doc_files/Delete_Custom_Content.png new file mode 100644 index 000000000000..596961333dd8 Binary files /dev/null and b/Packs/ContentManagement/doc_files/Delete_Custom_Content.png differ diff --git a/Packs/ContentManagement/pack_metadata.json b/Packs/ContentManagement/pack_metadata.json index e83dad9e6b86..78cba52a0a33 100644 --- a/Packs/ContentManagement/pack_metadata.json +++ b/Packs/ContentManagement/pack_metadata.json @@ -2,7 +2,7 @@ "name": "XSOAR CI/CD", "description": "This pack enables you to orchestrate your XSOAR system configuration.", "support": "xsoar", - "currentVersion": "1.2.8", + "currentVersion": "1.2.9", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "",