feat: added support for Tags encoding (#6)

internal/database/helpers.py

@@ -1,3 +1,24 @@
+
+# Go From this
+# 'CostCenter': '1234'; 'org': 'trey';
+# To JSON
+# [{"key": "CostCenter", "value": "1234"}, {"key": "org", "value": "trey"}]
+def format_tags_for_db(data : str, logger) -> str:
+    splitted = str.split(data, ';')
+    output = []
+    if len(splitted) > 0:
+        for split in splitted:
+            partial = '{'
+            values = str.split(split, ':')
+            if len(values) == 2:
+                partial += '"key": "' + values[0] + '", "value": "' + values[1] + '"'
+            else:
+                partial += '"key": "", "value": ""'
+            partial += '}'
+            output.append(partial)
+    logger.debug(output)
+    return output
+
 def get_focus_create() -> str:
     return '''
 AvailabilityZone STRING,
@@ -49,11 +70,10 @@ def get_focus_create() -> str:
 SkuPriceId STRING,
 SubAccountId STRING,
 SubAccountName STRING,
-Tags STRING,
+Tags ARRAY(OBJECT AS (key STRING, value STRING)),
 PRIMARY KEY (ResourceId, ResourceName, BillingPeriodEnd, BillingPeriodStart, ChargePeriodEnd, ChargePeriodStart)
 '''
 
-
 def get_notebook_create() -> str:
     return '''
 NOTEBOOK_NAME string,

main.py

@@ -9,7 +9,7 @@
 
 from flask import Flask, request, jsonify
 from waitress import serve
-from internal.database.helpers import get_focus_create, get_resource_create
+from internal.database.helpers import get_focus_create, get_resource_create, format_tags_for_db
 
 """ Error strings for the webservice """
 url_IncorrectError = 'URL composition incorrect'
@@ -61,14 +61,19 @@ def upload_data():
         if not data:
             return jsonify({'result': 'No data to process'}), 200
 
-
         if not db.does_table_exist(table_name, username, password):
             if metric_type == 'cost':
                 db.create_table(table_name, username, password, get_focus_create)
             elif metric_type == 'resource':
                 db.create_table(table_name, username, password, get_resource_create)
-
         # Process the bulk insert
+
+        if metric_type == 'cost':
+            # Modify Tags column to match DB format
+            for i in range(len(data)):
+                if 'Tags' in data[i]['labels'].keys():
+                    data[i]['labels']['Tags'] = format_tags_for_db(data[i]['labels']['Tags'], app.logger)
+
         records_inserted = db.bulk_insert(table_name, data, username, password)
         app.logger.info(f"records inserted: {records_inserted}")
         total_time = time.time_ns() / (10 ** 9) - time_start

readme.md

@@ -51,6 +51,19 @@ Compute endpoint list:
 curl -u <db-user>:<db-password> http://finops-database-handler.finops:8088/compute/list
 ```
 
+### Querying the data
+The Tags in the CSV data read by the exporters need to be in the following format::
+```
+{"CostCenter": "1234","Cost department": "Marketing","env": "prod","org": "trey","Project": "Foo"}
+```
+If this formatting is not used, the finops-database-handler will not insert the data into the database.
+To query the data using the information present inside the tags, you can use a query like this:
+```sql
+SELECT resourceid, tags['value']
+FROM "doc"."focus_table"
+WHERE 'CostAllocationTest' like any(tags['key']) or 'Sameer' like any(tags['value'])
+```
+
 ## Installation
 The webservice can be installed through the [HELM chart](https://github.com/krateoplatformops/finops-database-handler-chart):
 ```sh