Raise alerts for volume and brick status change

tendrl-bug-id: Tendrl#286
Signed-off-by: nnDarshan <darshan.n.2024@gmail.com>

tendrl/gluster_integration/sds_sync/__init__.py

@@ -3,6 +3,7 @@
 import json
 import os
 import re
+import socket
 import subprocess
 
 from tendrl.gluster_integration.sds_sync import brick_utilization
@@ -12,6 +13,8 @@
 
 from tendrl.commons import sds_sync
 from tendrl.gluster_integration import ini2json
+from tendrl.commons.utils.time_utils import now as tendrl_now
+
 
 
 class GlusterIntegrationSdsSyncStateThread(sds_sync.SdsSyncThread):
@@ -20,6 +23,38 @@ def __init__(self):
         super(GlusterIntegrationSdsSyncStateThread, self).__init__()
         self._complete = gevent.event.Event()
 
+    def _emit_event(self, resource, curr_value, msg):
+        alert = {}
+        import pdb;pdb.set_trace()
+        alert['source'] = 'tendrl-gluster-integration'
+        alert['pid'] = os.getpid()
+        alert['timestamp'] = tendrl_now().isoformat()
+        alert['alert_type'] = 'status'
+        severity = "info"
+        if curr_value == "Stopped":
+            severity = "critical"
+        alert['severity'] = severity
+        alert['resource'] = resource
+        alert['current_value'] = curr_value
+        alert['tags'] = dict(
+            message=msg,
+            cluster_id=NS.tendrl_context.cluster_id,
+            cluster_name=NS.tendrl_context.cluster_name,
+            sds_name=NS.tendrl_context.sds_name,
+            fqdn=socket.getfqdn()
+        )
+        alert['node_id'] = NS.node_context.node_id
+        if not NS.node_context.node_id:
+            return
+        Event(
+            Message(
+                "notice",
+                "alerting",
+                {'message': json.dumps(alert)},
+                node_id=NS.node_context.node_id
+            )
+        )
+
     def _run(self):
         Event(
             Message(
@@ -82,8 +117,35 @@ def _run(self):
                 if "Volumes" in raw_data:
                     index = 1
                     volumes = raw_data['Volumes']
+                    import pdb;pdb.set_trace()
+                    node_context = NS.node_context.load()
+                    tag_list = list(node_context.tags)
                     while True:
                         try:
+                            # Raise alerts for volume state change.
+                            if "provisioner/gluster" in tag_list:
+                                try:
+                                    stored_volume_status = NS._int.client.read(
+                                        "clusters/%s/Volumes/%s/status" % (
+                                            NS.tendrl_context.integration_id,
+                                            volumes['volume%s.id' % index]
+                                        )
+                                    ).value
+                                    current_status = volumes['volume%s.status' % index]
+                                    if current_status != stored_volume_status:
+                                        msg = "Status of volume: %s changed from %s to %s" % (
+                                            volumes['volume%s.name' % index],
+                                            stored_volume_status,
+                                            current_status
+                                        )
+                                        self._emit_event(
+                                            volumes['volume%s.name' % index],
+                                            current_status,
+                                            msg)                                        
+
+                                except etcd.EtcdKeyNotFound:
+                                    pass
+
                             volume = NS.gluster.objects.Volume(
                                 vol_id=volumes[
                                     'volume%s.id' % index
@@ -187,6 +249,7 @@ def _run(self):
                                         hostname not in network_ip):
                                         b_index += 1
                                         continue
+
                                     sub_vol_size = (int(volumes['volume%s.brickcount' % index])) / int(
                                         volumes[
                                             'volume%s.subvol_count' % index
@@ -198,6 +261,41 @@ def _run(self):
                                         )
                                     ].split(":")[-1].replace("/","_")
 
+                                    import pdb;pdb.set_trace()
+                                    # Raise alerts if the brick path changes
+                                    try:
+                                        stored_brick_status = NS._int.client.read(
+                                            "clusters/%s/Bricks/all/%s/status" % (
+                                                NS.tendrl_context.integration_id,
+                                                brick_name
+                                            )
+                                        ).value
+                                        current_status = volumes.get(
+                                            'volume%s.brick%s.status' % (
+                                                index,
+                                                b_index
+                                            )
+                                        )
+                                        if current_status != stored_brick_status:
+                                            msg = "Status of brick: %s under volume %s changed from %s to %s" % (
+                                                volumes[
+                                                    'volume%s.brick%s.path' % (
+                                                        index, b_index
+                                                    )
+                                                ],
+                                                volumes['volume%s.name' % index],
+                                                stored_brick_status,
+                                                current_status
+                                            )
+                                            self._emit_event(
+                                                volumes['volume%s.name' % index],
+                                                current_status,
+                                                msg
+                                            )                                        
+
+                                    except etcd.EtcdKeyNotFound:
+                                        pass
+
                                     vol_brick_path = "clusters/%s/Volumes/%s/Bricks/subvolume%s/%s" % (
                                         NS.tendrl_context.integration_id,
                                         volumes['volume%s.id' % index],