[CE-98] Health check and port check for Fabric v1.0

Implemented health check and port check functionality to
Fabric v1.0 netowrk. Chacks for all the containers
with cluster ID: status-running. Unit test added for
docker host.

Change-Id: I04bbc6e4c412e4af0abdd995df7d85c5c7a2ed71
Signed-off-by: indirajith <indirajithv@gmail.com>

.env

@@ -2,4 +2,4 @@ STATIC_FOLDER=themes/basic/static
 TEMPLATE_FOLDER=themes/basic/templates
 NPM_REGISTRY=registry.npmjs.org
 DEV=True
-ROOT_PATH=
+ROOT_PATH=/home/jith/cello

src/common/__init__.py

@@ -32,3 +32,4 @@
 from .fabric_network_config import \
     FabricPreNetworkConfig, FabricV1NetworkConfig
 from .stringvalidator import StringValidator
+from .fabric_network import FabricV1Network

src/common/fabric_network.py

@@ -2,9 +2,17 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 #
+import logging
+import requests
+
 from common.blockchain_network import BlockchainNetwork
 from common.fabric_network_config import \
     FabricPreNetworkConfig, FabricV1NetworkConfig
+from common import db, log_handler, LOG_LEVEL
+
+logger = logging.getLogger(__name__)
+logger.setLevel(LOG_LEVEL)
+logger.addHandler(log_handler)
 
 
 class FabricNetwork(BlockchainNetwork):
@@ -63,3 +71,80 @@ def __init__(self, name, network_id, network_type):
 
     def set_config(self):
         self.config = FabricV1NetworkConfig()
+
+    @classmethod
+    def health_check(cls, cluster, cluster_id, timeout=5):
+        """
+        Check if the peer or cluster is healthy by checking its
+        ports and number of containers running
+        :param cluster_id:
+        :param timeout:
+        :return: True or False
+        """
+
+        rest_api = cluster["worker_api"]
+        if not rest_api.startswith("http"):
+            segs = rest_api.split(":")  # tcp://x.x.x.x:2375
+            if len(segs) != 3:
+                rest_api = 'http://' + rest_api
+                swarm_api = 'http://' + rest_api + '/swarm'
+            if len(segs) == 3:
+                rest_api = 'http:' + segs[1] + ':' + segs[2] + \
+                    '/containers/json'
+                swarm_api = 'http:' + segs[1] + ':' + segs[2] + \
+                    '/swarm'
+        logger.debug("rest_api = {}".format(rest_api))
+        logger.debug("Swarm api = {}".format(swarm_api))
+        try:
+            r = requests.get(rest_api, timeout=timeout)
+            logger.debug("Value got from rest_api: {}".format(r))
+        except Exception as e:
+            logger.error("Error to refresh health of cluster {}: {}"
+                         .format(cluster_id, e))
+            return True
+        try:
+            sr = requests.get(swarm_api, timeout=timeout)
+        except Exception as e:
+            logger.error("Error to refresh health of cluster {}: {}"
+                         .format(cluster_id, e))
+            return True
+        if sr == '""message": "This node is not a swarm manager':
+            logger.debug("This docker engine runs in SWARM mode")
+            return False
+        data = r.json()
+        # name = list(map(lambda x: x.split('_'), d['Names'][0]))
+        # logger.debug("Data got from rest_api: {}".format(data))
+        peer_ports_up = 0
+        orderer_up = 0
+        containers = 0
+        run_containers = 0
+        for d in data:
+            name = d['Names'][0].split('_')
+            if name[1].startswith('peer'):
+                ports = list(port['PrivatePort'] for port in d['Ports'])
+                # logger.debug("peers from rest_api: {}".format(name[1]))
+                # logger.debug("ports from rest_api: {}".
+                #     format(ports))
+                # logger.debug("Test1! {}".
+                #     format(set(cluster['mapped_ports'].values())))
+                # logger.debug("Test2! {}".format(set(ports)))
+                if set(cluster['mapped_ports'].values()) == set(ports):
+                    peer_ports_up += 1
+                    logger.debug("Ports are up for container: {}- {}"
+                                 .format(d['Names'][0], ports))
+            if name[1].startswith('orderer'):
+                orderer_up += 1
+                logger.debug("Orderers: {}".format(d['Names'][0]))
+            if d["State"] == "running":
+                run_containers += 1
+            containers += 1
+            logger.debug("Number of Orderers up: {}".format(orderer_up))
+        if (cluster["size"] == peer_ports_up and
+                (run_containers == containers)):
+            logger.debug("health check of cluster id={} is OK"
+                         .format(cluster_id))
+            return "OK"
+        else:
+            logger.debug("health check of cluster id={} FAIL"
+                         .format(cluster_id))
+            return False

src/modules/cluster.py

@@ -29,6 +29,7 @@
     PEER_SERVICE_PORTS, CA_SERVICE_PORTS
 
 from common import FabricPreNetworkConfig, FabricV1NetworkConfig
+from common.fabric_network import FabricV1Network
 
 from modules import host
 
@@ -689,17 +690,21 @@ def refresh_health(self, cluster_id, timeout=5):
         :return: True or False
         """
         cluster = self.get_by_id(cluster_id)
+        cluster_id = cluster_id
+        logger.debug("Cluster ID: {}".format(cluster_id))
         logger.debug("checking health of cluster={}".format(cluster))
         if not cluster:
             logger.warning("Cannot found cluster id={}".format(cluster_id))
             return True
-        if cluster.get('status') != 'running':
+        if cluster.get("status") != "running":
             logger.warning("cluster is not running id={}".format(cluster_id))
             return True
-        if cluster.get('network_type') == NETWORK_TYPE_FABRIC_PRE_V1:
-            rest_api = cluster["service_url"]['rest'] + "/network/peers"
-            if not rest_api.startswith('http'):
-                rest_api = 'http://' + rest_api
+        if cluster.get("network_type") == NETWORK_TYPE_FABRIC_PRE_V1:
+            rest_api = cluster["service_url"]["rest"] + "/network/peers"
+            if not rest_api.startswith("http"):
+                rest_api = "http://" + rest_api
+            logger.debug("rest_api={}".format(rest_api))
+            logger.debug("---In Network type Fabric V 0.6---")
             try:
                 r = requests.get(rest_api, timeout=timeout)
             except Exception as e:
@@ -708,6 +713,7 @@ def refresh_health(self, cluster_id, timeout=5):
                 return True
 
             peers = r.json().get("peers")
+            logger.debug("peers from rest_api: {}".format(peers))
 
             if len(peers) == cluster["size"]:
                 self.db_update_one({"id": cluster_id},
@@ -720,9 +726,18 @@ def refresh_health(self, cluster_id, timeout=5):
                                    {"$set": {"health": "FAIL"}})
                 return False
         elif cluster.get('network_type') == NETWORK_TYPE_FABRIC_V1:
-            # TODO: check fabric 1.0 network health status
-            return True
-        return True
+            logger.debug("Cluster_id ={}".format(cluster_id))
+            logger.debug("checking health of cluster={}".format(cluster))
+            health_status = FabricV1Network.\
+                health_check(cluster, cluster_id, timeout=5)
+            if health_status == "OK":
+                self.db_update_one({"id": cluster_id},
+                                   {"$set": {"health": "OK"}})
+            else:
+                self.db_update_one({"id": cluster_id},
+                                   {"$set": {"health": "FAIL"}})
+            logger.debug("health_status: {}".format(health_status))
+            return health_status
 
     def db_update_one(self, filter, operations, after=True, col="active"):
         """