diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4489ffb --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +# Python +__pycache__/ +/venv + +# IntelliJ +/.idea +*.iml diff --git a/README.md b/README.md index 9f0271f..3aed75b 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,21 @@ # admin-tools Gridsuite admin tools + +# Clean orphan elements +This python script will send requests to gridsuite services configured in "constant.py" in order to clean orphan elements + +Developed with Python version 3.8.10 + +## Script modes + +Two executions modes are available : +- **exec** : this mode will actually remove orphan elements by executing DELETE requests to services +- **test** (default) : this mode will not modify nor remove any element. It will only display which element will be delete if script is ran with "exec" mode + +## Execution + +Command line to run script with "exec" mode : +
+    python clean_oprhans_element.py --mode=exec
+
+ diff --git a/clean_orphans_elements.py b/clean_orphans_elements.py new file mode 100644 index 0000000..c3e070b --- /dev/null +++ b/clean_orphans_elements.py @@ -0,0 +1,26 @@ +import argparse + +from functions.clean_orphan_elements.clean_orphan_contingency_lists import delete_orphan_contingency_lists +from functions.clean_orphan_elements.clean_orphan_filters import delete_orphan_filters +from functions.clean_orphan_elements.clean_orphan_networks import delete_orphan_network + +parser = argparse.ArgumentParser(description='Send requests to the gridsuite services to remove orphan elements', ) + +parser.add_argument("--dry-run", help="test mode (default) will not execute any modification request", + action="store_true") + +args = parser.parse_args() +dry_run = args.dry_run + +if dry_run: + print("Orphans deletion script will run without deleting anything (test mode)") +else: + print("Orphans deletion script (exec mode)") + +print("\n\n") + +delete_orphan_network(dry_run) + +delete_orphan_contingency_lists(dry_run) + +delete_orphan_filters(dry_run) diff --git a/constant.py b/constant.py new file mode 100644 index 0000000..847ea67 --- /dev/null +++ b/constant.py @@ -0,0 +1,32 @@ +# ENV FOLDERS +DEV_FOLDER = "sql/dev/" +PROD_FOLDER = "sql/prod/" + +# SQL FILES +GET_EXISTING_NETWORKS_SQL = "get_existing_networks.sql" +GET_ORPHAN_NETWORKS_SQL = "get_orphan_networks.sql" +DELETE_ORPHAN_NETWORKS_SQL = "delete_orphan_networks.sql" + +# SERVICES URL +HTTP_PROTOCOL = "http://" +API_VERSION = "/v1" + +STUDY_SERVER_URL = HTTP_PROTOCOL + "localhost:5001" + API_VERSION +NETWORK_STORE_SERVER_URL = HTTP_PROTOCOL + "localhost:8080" + API_VERSION +DIRECTORY_SERVER_URL = HTTP_PROTOCOL + "localhost:5026" + API_VERSION +ACTIONS_SERVER_URL = HTTP_PROTOCOL + "localhost:5022" + API_VERSION +FILTER_SERVER_URL = HTTP_PROTOCOL + "localhost:5027" + API_VERSION + +# PATHS +GET_STUDIES = STUDY_SERVER_URL + "/supervision/studies" + +GET_NETWORKS = NETWORK_STORE_SERVER_URL + "/networks" +DELETE_NETWORKS = NETWORK_STORE_SERVER_URL + "/networks" + +GET_DIRECTORY_ELEMENTS = DIRECTORY_SERVER_URL + "/supervision/elements" + +GET_CONTINGENCY_LISTS = ACTIONS_SERVER_URL + "/contingency-lists" +DELETE_CONTINGENCY_LISTS = ACTIONS_SERVER_URL + "/contingency-lists" + +GET_FILTERS = FILTER_SERVER_URL + "/filters" +DELETE_FILTERS = FILTER_SERVER_URL + "/filters" diff --git a/functions/__init__.py b/functions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/functions/clean_orphan_elements/__init__.py b/functions/clean_orphan_elements/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/functions/clean_orphan_elements/clean_orphan_contingency_lists.py b/functions/clean_orphan_elements/clean_orphan_contingency_lists.py new file mode 100644 index 0000000..53ef948 --- /dev/null +++ b/functions/clean_orphan_elements/clean_orphan_contingency_lists.py @@ -0,0 +1,63 @@ +import requests +import constant + + +def get_directory_element_uuid(element): + return element["elementUuid"] + + +def get_actions_element_uuid(element): + return element["id"] + + +def delete_contingency_lists(contingency_list_uuids, dry_run): + if dry_run: + for orphan_cl in contingency_list_uuids: + print("DELETE " + constant.DELETE_CONTINGENCY_LISTS + "/" + orphan_cl) + else: + for orphan_cl in contingency_list_uuids: + requests.delete(constant.DELETE_CONTINGENCY_LISTS + "/" + orphan_cl) + + +def delete_orphan_contingency_lists(dry_run): + # DELETING ORPHAN ACTIONS IN ACTIONS SERVER + print("/// Orphan actions deletion ///") + # GET EXISTING ACTIONS FROM DIRECTORY SERVER + print("Getting existing contingency lists from directory-server") + get_directory_contingency_lists_response = requests.get(constant.GET_DIRECTORY_ELEMENTS, + params={"elementType": "CONTINGENCY_LIST"}) + get_directory_contingency_lists_response_json = get_directory_contingency_lists_response.json() + get_directory_contingency_lists_response_json_uuid = map(get_directory_element_uuid, + get_directory_contingency_lists_response_json) + existing_contingency_lists_uuid = list(get_directory_contingency_lists_response_json_uuid) + + print("Done") + + # GET CONTINGENCY LISTS FROM ACTIONS SERVER + print("Getting all contingency lists from actions-server") + get_actions_contingency_lists_response = requests.get(constant.GET_CONTINGENCY_LISTS) + get_actions_contingency_lists_json = get_actions_contingency_lists_response.json() + get_actions_contingency_lists_uuid = map(get_actions_element_uuid, get_actions_contingency_lists_json) + all_contingency_lists_uuid = list(get_actions_contingency_lists_uuid) + + print("Done") + + # GET ORPHANS CONTINGENCY LISTS - CONTINGENCY LISTS IN ACTIONS SERVER WHICH ARE NOT KNOWN IN DIRECTORY SERVER + print("Computing orphan contingency lists") + orphan_contingency_lists = [] + for element_uuid in all_contingency_lists_uuid: + if element_uuid not in existing_contingency_lists_uuid: + orphan_contingency_lists.append(element_uuid) + + print("Done") + + # DELETING OPRHANS + print("Deleting the following orphan contingency lists : ") + for orphan_cl in orphan_contingency_lists: + print(" - ", orphan_cl) + + delete_contingency_lists(orphan_contingency_lists, dry_run) + + print("Done") + + print("\n\n") diff --git a/functions/clean_orphan_elements/clean_orphan_filters.py b/functions/clean_orphan_elements/clean_orphan_filters.py new file mode 100644 index 0000000..1c120b4 --- /dev/null +++ b/functions/clean_orphan_elements/clean_orphan_filters.py @@ -0,0 +1,61 @@ +import constant +import requests + + +def get_directory_element_uuid(element): + return element["elementUuid"] + + +def delete_filters(filter_uuids, dry_run): + if dry_run: + for orphan_f in filter_uuids: + print("DELETE " + constant.DELETE_FILTERS + "/" + orphan_f) + else: + for orphan_f in filter_uuids: + requests.delete(constant.DELETE_FILTERS + "/" + orphan_f) + + +def get_element_id(element): + return element["id"] + + +def delete_orphan_filters(dry_run): + # DELETING ORPHAN FILTERS IN FILTER SERVER + print("/// Orphan filters deletion ///") + # GET EXISTING FILTERS FROM DIRECTORY SERVER + print("Getting existing filters from directory-server") + get_directory_filters_response = requests.get(constant.GET_DIRECTORY_ELEMENTS, params={"elementType": "FILTER"}) + get_directory_filters_response_json = get_directory_filters_response.json() + get_directory_filters_response_json_uuid = map(get_directory_element_uuid, get_directory_filters_response_json) + existing_filters_uuid = list(get_directory_filters_response_json_uuid) + + print("Done") + + # GET CONTINGENCY LISTS FROM ACTIONS SERVER + print("Getting all filters from filter-server") + get_actions_filters_response = requests.get(constant.GET_FILTERS) + get_actions_filters_json = get_actions_filters_response.json() + get_actions_filters_uuid = map(get_element_id, get_actions_filters_json) + all_filters_uuid = list(get_actions_filters_uuid) + + print("Done") + + # GET ORPHANS CONTINGENCY LISTS - CONTINGENCY LISTS IN ACTIONS SERVER WHICH ARE NOT KNOWN IN DIRECTORY SERVER + print("Computing orphan filters") + orphan_filters = [] + for element_uuid in all_filters_uuid: + if element_uuid not in existing_filters_uuid: + orphan_filters.append(element_uuid) + + print("Done") + + # DELETING OPRHANS + print("Deleting the following orphan filters : ") + for orphan_cl in orphan_filters: + print(" - ", orphan_cl) + + delete_filters(orphan_filters, dry_run) + + print("Done") + + print("\n\n") diff --git a/functions/clean_orphan_elements/clean_orphan_networks.py b/functions/clean_orphan_elements/clean_orphan_networks.py new file mode 100644 index 0000000..0aa8ffd --- /dev/null +++ b/functions/clean_orphan_elements/clean_orphan_networks.py @@ -0,0 +1,60 @@ +import requests +import constant + + +def get_network_uuid_from_study(study): + return study["networkUuid"] + + +def get_network_uuid_from_network(network): + return network["uuid"] + + +def delete_networks(network_uuids, dry_run): + if dry_run: + for orphan_n in network_uuids: + print("DELETE " + constant.DELETE_NETWORKS + "/" + orphan_n) + else: + for orphan_n in network_uuids: + requests.delete(constant.DELETE_NETWORKS + "/" + orphan_n) + + +def delete_orphan_network(dry_run): + # DELETING ORPHAN NETWORKS IN NETWORK STORE SERVER + print("/// Orphan networks deletion ///") + # GET EXISTING NETWORKS AMONG EXISTING STUDIES + print("Getting existing networks from existing studies") + get_studies_response = requests.get(constant.GET_STUDIES) + + get_studies_response_json = get_studies_response.json() + get_studies_response_json_network_uuid = map(get_network_uuid_from_study, get_studies_response_json) + existing_networks_uuid = list(get_studies_response_json_network_uuid) + + print("Done") + # GET NETWORKS SAVED IN NETWORK STORE SERVER + print("Getting networks from network store server") + get_networks_response = requests.get(constant.GET_NETWORKS) + + get_networks_response_json = get_networks_response.json() + get_networks_response_json_uuid = map(get_network_uuid_from_network, get_networks_response_json) + all_networks_uuid = list(get_networks_response_json_uuid) + + print("Done") + # GET ORPHANS NETWORKS - NETWORKS IN NETWORK STORE SERVER WHICH ARE NOT KNOWN IN STUDY SERVER + print("Computing orphan networks") + orphan_networks = [] + for network_uuid in all_networks_uuid: + if network_uuid not in existing_networks_uuid: + orphan_networks.append(network_uuid) + + print("Done") + # DELETING OPRHANS + print("Deleting the following orphan networks : ") + for orphan_n in orphan_networks: + print(" - ", orphan_n) + + delete_networks(orphan_networks, dry_run) + + print("Done") + + print("\n\n")