[WIP] Clean orphan elements script

.gitignore

@@ -0,0 +1,7 @@
+# Python
+__pycache__/
+/venv
+
+# IntelliJ
+/.idea
+*.iml

README.md

@@ -1,2 +1,21 @@
 # admin-tools
 Gridsuite admin tools
+
+# Clean orphan elements
+This python script will send requests to gridsuite services configured in "constant.py" in order to clean orphan elements
+
+Developed with Python version 3.8.10
+
+## Script modes
+
+Two executions modes are available :
+- **exec** : this mode will actually remove orphan elements by executing DELETE requests to services
+- **test** (default) : this mode will not modify nor remove any element. It will only display which element will be delete if script is ran with "exec" mode
+
+## Execution
+
+Command line to run script with "exec" mode :
+<pre>
+    python clean_oprhans_element.py --mode=exec
+</pre>
+

clean_orphans_elements.py

@@ -0,0 +1,26 @@
+import argparse
+
+from functions.clean_orphan_elements.clean_orphan_contingency_lists import delete_orphan_contingency_lists
+from functions.clean_orphan_elements.clean_orphan_filters import delete_orphan_filters
+from functions.clean_orphan_elements.clean_orphan_networks import delete_orphan_network
+
+parser = argparse.ArgumentParser(description='Send requests to the gridsuite services to remove orphan elements', )
+
+parser.add_argument("--dry-run", help="test mode (default) will not execute any modification request",
+                    action="store_true")
+
+args = parser.parse_args()
+dry_run = args.dry_run
+
+if dry_run:
+    print("Orphans deletion script will run without deleting anything (test mode)")
+else:
+    print("Orphans deletion script (exec mode)")
+
+print("\n\n")
+
+delete_orphan_network(dry_run)
+
+delete_orphan_contingency_lists(dry_run)
+
+delete_orphan_filters(dry_run)

constant.py

@@ -0,0 +1,32 @@
+# ENV FOLDERS
+DEV_FOLDER = "sql/dev/"
+PROD_FOLDER = "sql/prod/"
+
+# SQL FILES
+GET_EXISTING_NETWORKS_SQL = "get_existing_networks.sql"
+GET_ORPHAN_NETWORKS_SQL = "get_orphan_networks.sql"
+DELETE_ORPHAN_NETWORKS_SQL = "delete_orphan_networks.sql"
+
+# SERVICES URL
+HTTP_PROTOCOL = "http://"
+API_VERSION = "/v1"
+
+STUDY_SERVER_URL = HTTP_PROTOCOL + "localhost:5001" + API_VERSION
+NETWORK_STORE_SERVER_URL = HTTP_PROTOCOL + "localhost:8080" + API_VERSION
+DIRECTORY_SERVER_URL = HTTP_PROTOCOL + "localhost:5026" + API_VERSION
+ACTIONS_SERVER_URL = HTTP_PROTOCOL + "localhost:5022" + API_VERSION
+FILTER_SERVER_URL = HTTP_PROTOCOL + "localhost:5027" + API_VERSION
+
+# PATHS
+GET_STUDIES = STUDY_SERVER_URL + "/supervision/studies"
+
+GET_NETWORKS = NETWORK_STORE_SERVER_URL + "/networks"
+DELETE_NETWORKS = NETWORK_STORE_SERVER_URL + "/networks"
+
+GET_DIRECTORY_ELEMENTS = DIRECTORY_SERVER_URL + "/supervision/elements"
+
+GET_CONTINGENCY_LISTS = ACTIONS_SERVER_URL + "/contingency-lists"
+DELETE_CONTINGENCY_LISTS = ACTIONS_SERVER_URL + "/contingency-lists"
+
+GET_FILTERS = FILTER_SERVER_URL + "/filters"
+DELETE_FILTERS = FILTER_SERVER_URL + "/filters"

functions/clean_orphan_elements/clean_orphan_contingency_lists.py

@@ -0,0 +1,63 @@
+import requests
+import constant
+
+
+def get_directory_element_uuid(element):
+    return element["elementUuid"]
+
+
+def get_actions_element_uuid(element):
+    return element["id"]
+
+
+def delete_contingency_lists(contingency_list_uuids, dry_run):
+    if dry_run:
+        for orphan_cl in contingency_list_uuids:
+            print("DELETE " + constant.DELETE_CONTINGENCY_LISTS + "/" + orphan_cl)
+    else:
+        for orphan_cl in contingency_list_uuids:
+            requests.delete(constant.DELETE_CONTINGENCY_LISTS + "/" + orphan_cl)
+
+
+def delete_orphan_contingency_lists(dry_run):
+    # DELETING ORPHAN ACTIONS IN ACTIONS SERVER
+    print("/// Orphan actions deletion ///")
+    # GET EXISTING ACTIONS FROM DIRECTORY SERVER
+    print("Getting existing contingency lists from directory-server")
+    get_directory_contingency_lists_response = requests.get(constant.GET_DIRECTORY_ELEMENTS,
+                                                            params={"elementType": "CONTINGENCY_LIST"})
+    get_directory_contingency_lists_response_json = get_directory_contingency_lists_response.json()
+    get_directory_contingency_lists_response_json_uuid = map(get_directory_element_uuid,
+                                                             get_directory_contingency_lists_response_json)
+    existing_contingency_lists_uuid = list(get_directory_contingency_lists_response_json_uuid)
+
+    print("Done")
+
+    # GET CONTINGENCY LISTS FROM ACTIONS SERVER
+    print("Getting all contingency lists from actions-server")
+    get_actions_contingency_lists_response = requests.get(constant.GET_CONTINGENCY_LISTS)
+    get_actions_contingency_lists_json = get_actions_contingency_lists_response.json()
+    get_actions_contingency_lists_uuid = map(get_actions_element_uuid, get_actions_contingency_lists_json)
+    all_contingency_lists_uuid = list(get_actions_contingency_lists_uuid)
+
+    print("Done")
+
+    # GET ORPHANS CONTINGENCY LISTS - CONTINGENCY LISTS IN ACTIONS SERVER WHICH ARE NOT KNOWN IN DIRECTORY SERVER
+    print("Computing orphan contingency lists")
+    orphan_contingency_lists = []
+    for element_uuid in all_contingency_lists_uuid:
+        if element_uuid not in existing_contingency_lists_uuid:
+            orphan_contingency_lists.append(element_uuid)
+
+    print("Done")
+
+    # DELETING OPRHANS
+    print("Deleting the following orphan contingency lists : ")
+    for orphan_cl in orphan_contingency_lists:
+        print(" - ", orphan_cl)
+
+    delete_contingency_lists(orphan_contingency_lists, dry_run)
+
+    print("Done")
+
+    print("\n\n")

functions/clean_orphan_elements/clean_orphan_filters.py

@@ -0,0 +1,61 @@
+import constant
+import requests
+
+
+def get_directory_element_uuid(element):
+    return element["elementUuid"]
+
+
+def delete_filters(filter_uuids, dry_run):
+    if dry_run:
+        for orphan_f in filter_uuids:
+            print("DELETE " + constant.DELETE_FILTERS + "/" + orphan_f)
+    else:
+        for orphan_f in filter_uuids:
+            requests.delete(constant.DELETE_FILTERS + "/" + orphan_f)
+
+
+def get_element_id(element):
+    return element["id"]
+
+
+def delete_orphan_filters(dry_run):
+    # DELETING ORPHAN FILTERS IN FILTER SERVER
+    print("/// Orphan filters deletion ///")
+    # GET EXISTING FILTERS FROM DIRECTORY SERVER
+    print("Getting existing filters from directory-server")
+    get_directory_filters_response = requests.get(constant.GET_DIRECTORY_ELEMENTS, params={"elementType": "FILTER"})
+    get_directory_filters_response_json = get_directory_filters_response.json()
+    get_directory_filters_response_json_uuid = map(get_directory_element_uuid, get_directory_filters_response_json)
+    existing_filters_uuid = list(get_directory_filters_response_json_uuid)
+
+    print("Done")
+
+    # GET CONTINGENCY LISTS FROM ACTIONS SERVER
+    print("Getting all filters from filter-server")
+    get_actions_filters_response = requests.get(constant.GET_FILTERS)
+    get_actions_filters_json = get_actions_filters_response.json()
+    get_actions_filters_uuid = map(get_element_id, get_actions_filters_json)
+    all_filters_uuid = list(get_actions_filters_uuid)
+
+    print("Done")
+
+    # GET ORPHANS CONTINGENCY LISTS - CONTINGENCY LISTS IN ACTIONS SERVER WHICH ARE NOT KNOWN IN DIRECTORY SERVER
+    print("Computing orphan filters")
+    orphan_filters = []
+    for element_uuid in all_filters_uuid:
+        if element_uuid not in existing_filters_uuid:
+            orphan_filters.append(element_uuid)
+
+    print("Done")
+
+    # DELETING OPRHANS
+    print("Deleting the following orphan filters : ")
+    for orphan_cl in orphan_filters:
+        print(" - ", orphan_cl)
+
+    delete_filters(orphan_filters, dry_run)
+
+    print("Done")
+
+    print("\n\n")

functions/clean_orphan_elements/clean_orphan_networks.py

@@ -0,0 +1,60 @@
+import requests
+import constant
+
+
+def get_network_uuid_from_study(study):
+    return study["networkUuid"]
+
+
+def get_network_uuid_from_network(network):
+    return network["uuid"]
+
+
+def delete_networks(network_uuids, dry_run):
+    if dry_run:
+        for orphan_n in network_uuids:
+            print("DELETE " + constant.DELETE_NETWORKS + "/" + orphan_n)
+    else:
+        for orphan_n in network_uuids:
+            requests.delete(constant.DELETE_NETWORKS + "/" + orphan_n)
+
+
+def delete_orphan_network(dry_run):
+    # DELETING ORPHAN NETWORKS IN NETWORK STORE SERVER
+    print("/// Orphan networks deletion ///")
+    # GET EXISTING NETWORKS AMONG EXISTING STUDIES
+    print("Getting existing networks from existing studies")
+    get_studies_response = requests.get(constant.GET_STUDIES)
+
+    get_studies_response_json = get_studies_response.json()
+    get_studies_response_json_network_uuid = map(get_network_uuid_from_study, get_studies_response_json)
+    existing_networks_uuid = list(get_studies_response_json_network_uuid)
+
+    print("Done")
+    # GET NETWORKS SAVED IN NETWORK STORE SERVER
+    print("Getting networks from network store server")
+    get_networks_response = requests.get(constant.GET_NETWORKS)
+
+    get_networks_response_json = get_networks_response.json()
+    get_networks_response_json_uuid = map(get_network_uuid_from_network, get_networks_response_json)
+    all_networks_uuid = list(get_networks_response_json_uuid)
+
+    print("Done")
+    # GET ORPHANS NETWORKS - NETWORKS IN NETWORK STORE SERVER WHICH ARE NOT KNOWN IN STUDY SERVER
+    print("Computing orphan networks")
+    orphan_networks = []
+    for network_uuid in all_networks_uuid:
+        if network_uuid not in existing_networks_uuid:
+            orphan_networks.append(network_uuid)
+
+    print("Done")
+    # DELETING OPRHANS
+    print("Deleting the following orphan networks : ")
+    for orphan_n in orphan_networks:
+        print(" - ", orphan_n)
+
+    delete_networks(orphan_networks, dry_run)
+
+    print("Done")
+
+    print("\n\n")