Introduce accumulating statistic in time-series DB

ocw/lib/azure.py

@@ -10,7 +10,8 @@
 from dateutil.parser import parse
 from webui.PCWConfig import PCWConfig
 from .provider import Provider
-from ..models import Instance
+from ..models import Instance, ProviderChoice
+from .influx import Influx
 
 
 class Azure(Provider):
@@ -82,9 +83,11 @@ def get_storage_key(self, storage_account: str) -> str:
         return storage_keys[0]
 
     def list_instances(self) -> list:
-        return list(self.compute_mgmt_client().virtual_machines.list_all())
+        all_vms = list(self.compute_mgmt_client().virtual_machines.list_all())
+        Influx().write(ProviderChoice.AZURE.value ,Influx.VMS_QUANTITY, len(all_vms))
+        return all_vms
 
-    def get_vm_types_in_resource_group(self, resource_group: str) -> str:
+    def get_vm_types_in_resource_group(self, resource_group: str) -> str | None:
         self.log_dbg(f"Listing VMs for {resource_group}")
         type_set = set()
         try:
@@ -94,9 +97,7 @@ def get_vm_types_in_resource_group(self, resource_group: str) -> str:
         except ResourceNotFoundError:
             self.log_dbg(f"{resource_group} already deleted")
             return None
-        if len(type_set) > 0:
-            return ', '.join(type_set)
-        return "N/A"
+        return ', '.join(type_set) if type_set else "N/A"
 
     def get_resource_properties(self, resource_id):
         return self.resource_mgmt_client().resources.get_by_id(resource_id, api_version="2023-07-03").properties

ocw/management/commands/influx.py

@@ -0,0 +1,10 @@
+from django.core.management.base import BaseCommand
+from ocw.lib.influx import Influx
+
+
+class Command(BaseCommand):
+    help = 'Delete all leftovers in all providers (according to pcw.ini)'
+
+    def handle(self, *args, **options):
+        influx=Influx()
+        influx.write("test", "test", 22)

requirements.txt

@@ -21,3 +21,4 @@ openstacksdk~=1.5.0
 python-dateutil
 apscheduler
 kubernetes
+influxdb-client

templates/pcw.ini

@@ -57,3 +57,11 @@ vpc_cleanup = true
 [updaterun]
 # if openqa_ttl tag is not defined this TTL will be set to the instance
 default_ttl = 44100 # value is in seconds
+
+# used to store statistic about amount of entities tracked in the cloud
+[influxdb]
+# defines standard influxdb connection params - organization , bucket
+# for more details please refer to official documentation https://docs.influxdata.com/influxdb/v2/api-guide/client-libraries/python/#Copyright
+org=pcw
+bucket=cloud_stat
+url=http://localhost:8086

webui/PCWConfig.py

@@ -53,8 +53,8 @@ def getList(self, config_path: str, default: list = None) -> list:
 class PCWConfig():
 
     @staticmethod
-    def get_feature_property(feature: str, property: str, namespace: str = None):
-        default_values = {
+    def get_feature_property(feature: str, feature_property: str, namespace: str | None = None) -> str | int:
+        default_values : dict[str, dict[str, int | type[int] | str | type[str] | type[str] | None ]] = {
             'cleanup/max-age-hours': {'default': 24 * 7, 'return_type': int},
             'cleanup/azure-gallery-name': {'default': 'test_image_gallery', 'return_type': str},
             'cleanup/azure-storage-resourcegroup': {'default': 'openqa-upload', 'return_type': str},
@@ -70,16 +70,19 @@ def get_feature_property(feature: str, property: str, namespace: str = None):
             'notify/smtp': {'default': None, 'return_type': str},
             'notify/smtp-port': {'default': 25, 'return_type': int},
             'notify/from': {'default': 'pcw@publiccloud.qa.suse.de', 'return_type': str},
+            'influxdb/org' : {'default': 'pcw', 'return_type': str},
+            'influxdb/bucket' : {'default': 'cloud_stat', 'return_type': str},
+            'influxdb/url' : {'default': None, 'return_type': str},
         }
-        key = '/'.join([feature, property])
+        key = '/'.join([feature, feature_property])
         if key not in default_values:
             raise LookupError(f"Missing {key} in default_values list")
         if namespace:
-            setting = f'{feature}.namespace.{namespace}/{property}'
+            setting = f'{feature}.namespace.{namespace}/{feature_property}'
             if PCWConfig.has(setting):
-                return default_values[key]['return_type'](ConfigFile().get(setting))
+                return default_values[key]['return_type'](ConfigFile().get(setting)) # type: ignore
         return default_values[key]['return_type'](
-            ConfigFile().get(key, default_values[key]['default']))
+            ConfigFile().get(key, default_values[key]['default'])) # type: ignore
 
     @staticmethod
     def get_namespaces_for(feature: str) -> list:
@@ -114,10 +117,10 @@ def has(setting: str) -> bool:
             return False
 
     @staticmethod
-    def getBoolean(config_path: str, namespace: str = None, default=False) -> bool:
+    def getBoolean(config_path: str, namespace: str | None = None, default=False) -> bool:
         if namespace:
-            feature, property = config_path.split('/')
-            setting = f'{feature}.namespace.{namespace}/{property}'
+            feature, feature_property = config_path.split('/')
+            setting = f'{feature}.namespace.{namespace}/{feature_property}'
             if PCWConfig.has(setting):
                 value = ConfigFile().get(setting)
             else: