Add daemon which periodically pushes process and docker stats to State DB

files/image_config/procdockerstats/procdockerstats

@@ -0,0 +1,211 @@
+# !/usr/bin/env python
+'''
+This code is for a specific daemon, process-docker.
+Which sends process and docker CPU/memory utilization data to DB every 2 mins.
+'''
+
+import sys
+import os
+import time
+import syslog
+import subprocess
+import re
+import swsssdk 
+from datetime import datetime
+
+VERSION = '1.0'
+
+SYSLOG_IDENTIFIER = "procdockerstats"
+
+REDIS_HOSTIP = "127.0.0.1"
+
+# ========================== Syslog wrappers ==========================
+def log_info(msg, also_print_to_console=False):
+ syslog.openlog(SYSLOG_IDENTIFIER)
+ syslog.syslog(syslog.LOG_INFO, msg)
+ syslog.closelog()
+
+def log_warning(msg, also_print_to_console=False):
+ syslog.openlog(SYSLOG_IDENTIFIER)
+ syslog.syslog(syslog.LOG_WARNING, msg)
+ syslog.closelog()
+
+def log_error(msg, also_print_to_console=False):
+ syslog.openlog(SYSLOG_IDENTIFIER)
+ syslog.syslog(syslog.LOG_ERR, msg)
+ syslog.closelog()
+
+# ========================== ProcessDocker class ==========================
+class ProcDockerStats:
+
+ def __init__(self):
+ self.state_db = swsssdk.SonicV2Connector(host=REDIS_HOSTIP)
+ self.state_db.connect("STATE_DB") 
+
+ def run_command(self, cmd):
+ proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
+ (stdout, stderr) = proc.communicate()
+ if proc.returncode != 0:
+ log_error("Error running command '{}'".format(cmd))
+ else:
+ return stdout
+
+ def format_docker_cmd_output(self, cmdout):
+ lines = re.split("\n", cmdout)
+ keys = re.split(" +", lines[0])
+ dict1 = dict()
+ dict_list = []
+ for item in lines[1:]:
+ values1 = re.split(" +", item)
+ dict1 = dict(zip(keys, values1))
+ dict_list.append(dict1)
+ formatted_dict = self.create_docker_dict(dict_list)
+ return formatted_dict
+
+ def format_process_cmd_output(self, cmdout):
+ lines = re.split("\n", cmdout)
+ keys = re.split(" +", lines[0])
+ keylist = list(filter(None, keys))
+ dict1 = dict()
+ dict_list = []
+ for item in lines[1:]:
+ values1 = re.split(" +", str(item))
+ # To remove extra space before UID
+ val = list(filter(None, values1))
+ # Merging extra columns created due to space in cmd ouput
+ val[8:] = [''.join(val[8:])] 
+ dict1 = dict(zip(keylist, val))
+ dict_list.append(dict1)
+ return dict_list
+
+ def convert_to_bytes(self, value):
+ u = re.search('[a-zA-Z]+', value)
+ v = float(filter(str.isdigit, value))
+ unit = u.group(0)
+ b = 'B'
+ kb = 'KB'
+ mb = 'MB'
+ mib = 'MiB'
+ gib = 'GiB' 
+ if unit.lower() == b.lower():
+ return v
+ elif unit.lower() == kb.lower():
+ v1 = v * 1000
+ return v1
+ elif unit.lower() == mb.lower():
+ v1 = v * 1000000
+ return v1
+ elif unit.lower() == mib.lower():
+ v1 = v * 1048576
+ return v1
+ elif unit.lower() == gib.lower():
+ v1 = v * 1073741824
+ return v1
+
+ def create_docker_dict(self, dict_list):
+ dockerdict = {}
+ for row in dict_list[0:]: 
+ cid = row.get('CONTAINER ID')
+ if cid:
+ key = 'Docker_Stats|' + str(cid) 
+ dockerdict[key] = {}
+ dockerdict[key]['NAME'] = row.get('NAME')
+
+ splitcol = row.get('CPU %')
+ cpu = re.split("%", str(splitcol))
+ dockerdict[key]['CPU%'] = str(cpu[0])
+
+ splitcol = row.get('MEM USAGE / LIMIT')
+ memuse = re.split(" / ", str(splitcol))
+ # converting MiB and GiB to bytes
+ dockerdict[key]['MEM'] = str(self.convert_to_bytes(memuse[0]))
+ dockerdict[key]['MEM_LIMIT'] = str(self.convert_to_bytes(memuse[1]))
+
+ splitcol = row.get('MEM %')
+ mem = re.split("%", str(splitcol))
+ dockerdict[key]['MEM%'] = str(mem[0])
+
+ splitcol = row.get('NET I/O')
+ netio = re.split(" / ", str(splitcol))
+ dockerdict[key]['NET_IN'] = str(self.convert_to_bytes(netio[0]))
+ dockerdict[key]['NET_OUT'] = str(self.convert_to_bytes(netio[1]))
+
+ splitcol = row.get('BLOCK I/O')
+ blockio = re.split(" / ", str(splitcol))
+ dockerdict[key]['BLOCK_IN'] = str(self.convert_to_bytes(blockio[0]))
+ dockerdict[key]['BLOCK_OUT'] = str(self.convert_to_bytes(blockio[1]))
+
+ dockerdict[key]['PIDS'] = row.get('PIDS')
+ return dockerdict
+
+ def update_dockerstats_command(self):
+ cmd = "docker stats --no-stream -a"
+ data = self.run_command(cmd)
+ if not data:
+ log_error("'{}' returned null output".format(cmd))
+ return False
+ dockerdata = self.format_docker_cmd_output(data)
+ if not dockerdata:
+ log_error("formatting for docker output failed")
+ return False
+ # wipe out all data from state_db before updating
+ self.state_db.delete_all_by_pattern('STATE_DB', 'Docker_Stats|*')
+ for k1,v1 in dockerdata.iteritems():
+ for k2,v2 in v1.iteritems():
+ self.update_state_db(k1, k2, v2)
+
+ def update_processstats_command(self):
+
+ data = self.run_command("ps -eo uid,pid,ppid,%mem,%cpu,stime,tty,time,cmd --sort -%cpu | head -5")
+ processdata = self.format_process_cmd_output(data)
+ value = ""
+ # wipe out all data before updating with new values
+ self.state_db.delete_all_by_pattern('STATE_DB', 'Process_Stats|*')
+ for row in processdata[0:]:
+ cid = row.get('PID')
+ if cid:
+ value = 'Process_Stats|' + str(cid)
+ uid = row.get('UID')
+ self.update_state_db(value, 'UID', uid)
+ ppid = row.get('PPID')
+ self.update_state_db(value, 'PPID', ppid)
+ cpu = row.get('%CPU')
+ self.update_state_db(value, '%CPU', str(cpu))
+ mem = row.get('%MEM')
+ self.update_state_db(value, '%MEM', str(mem))
+ stime = row.get('STIME')
+ self.update_state_db(value, 'STIME', stime)
+ tty = row.get('TT')
+ self.update_state_db(value, 'TT', tty)
+ time = row.get('TIME')
+ self.update_state_db(value, 'TIME', time)
+ cmd = row.get('CMD')
+ self.update_state_db(value, 'CMD', cmd)
+
+ def update_state_db(self, key1, key2, value2):
+ self.state_db.set('STATE_DB', key1, key2, value2)
+
+ def run(self):
+ self.update_dockerstats_command()
+ datetimeobj = datetime.now()
+ # Adding key to store latest update time.
+ self.update_state_db('Docker_Stats|LastUpdateTime', 'lastupdate', datetimeobj)
+ self.update_processstats_command()
+ self.update_state_db('Process_Stats|LastUpdateTime', 'lastupdate', datetimeobj)
+
+# main start
+def main():
+ log_info("process-docker stats daemon starting up..")
+ if not os.getuid() == 0:
+ log_error("Must be root to run process-docker daemon")
+ print "Error: Must be root to run process-docker daemon"
+ sys.exit(1)
+ pd = ProcDockerStats()
+ # Data need to be updated every 2 mins. hence adding delay of 120 seconds
+ while True:
+ pd.run()
+ time.sleep(120)
+ log_info("process-docker stats daemon exited")
+
+if __name__ == '__main__':
+ main()

files/image_config/procdockerstats/procdockerstats.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=Control Plane ACL configuration daemon
+Requires=updategraph.service
+After=updategraph.service
+
+[Service]
+Type=simple
+ExecStart=/usr/bin/procdockerstats
+
+[Install]
+WantedBy=multi-user.target
+