[system-health] Add support for monitoring system health (sonic-net#4835

)

* system health first commit

* system health daemon first commit

* Finish healthd

* Changes due to lower layer logic change

* Get ASIC temperature from TEMPERATURE_INFO table

* Add system health make rule and service files

* fix bugs found during manual test

* Change make file to install system-health library to host

* Set system LED to blink on bootup time

* Caught exceptions in system health checker to make it more robust

* fix issue that fan/psu presence will always be true

* fix issue for external checker

* move system-health service to right after rc-local service

* Set system-health service start after database service

* Get system up time via /proc/uptime

* Provide more information in stat for CLI to use

* fix typo

* Set default category to External for external checker

* If external checker reported OK, save it to stat too

* Trim string for external checker output

* fix issue: PSU voltage check always return OK

* Add unit test cases for system health library

* Fix LGTM warnings

* fix demo comments: 1. get boot up timeout from monit configuration file; 2. set system led in library instead of daemon

* Remove boot_timeout configuration because it will get from monit config file

* Fix argument miss

* fix unit test failure

* fix issue: summary status is not correct

* Fix format issues found in code review

* rename th to threshold to make it clearer

* Fix review comment: 1. add a .dep file for system health; 2. deprecated daemon_base and uses sonic-py-common instead

* Fix unit test failure

* Fix LGTM alert

* Fix LGTM alert

* Fix review comments

* Fix review comment

* 1. Add relevant comments for system health; 2. rename external_checker to user_define_checker

* Ignore check for unknown service type

* Fix unit test issue

* Rename user define checker to user defined checker

* Rename user_define_checkers to user_defined_checkers for configuration file

* Renmae file user_define_checker.py -> user_defined_checker.py

* Fix typo

* Adjust import order for config.py

Co-authored-by: Joe LeVeque <jleveque@users.noreply.github.com>

* Adjust import order for src/system-health/health_checker/hardware_checker.py

Co-authored-by: Joe LeVeque <jleveque@users.noreply.github.com>

* Adjust import order for src/system-health/scripts/healthd

Co-authored-by: Joe LeVeque <jleveque@users.noreply.github.com>

* Adjust import orders in src/system-health/tests/test_system_health.py

* Fix typo

* Add new line after import

* If system health configuration file not exist, healthd should exit

* Fix indent and enable pytest coverage

* Fix typo

* Fix typo

* Remove global logger and use log functions inherited from super class

* Change info level logger to notice level

Co-authored-by: Joe LeVeque <jleveque@users.noreply.github.com>

device/mellanox/x86_64-mlnx_msn2010-r0/system_health_monitoring_config.json

@@ -1,11 +1,11 @@
 {
  "services_to_ignore": [],
  "devices_to_ignore": ["psu.voltage", "psu.temperature"],
- "external_checkers": [],
+ "user_defined_checkers": [],
  "polling_interval": 60,
  "led_color": {
- "fault": "orange",
- "normal": "green",
- "booting": "orange_blink"
+  "fault": "orange",
+  "normal": "green",
+  "booting": "orange_blink"
  }
 }

device/mellanox/x86_64-mlnx_msn2700-r0/system_health_monitoring_config.json

@@ -1,11 +1,11 @@
 {
  "services_to_ignore": [],
  "devices_to_ignore": ["psu.voltage"],
- "external_checkers": [],
+ "user_defined_checkers": [],
  "polling_interval": 60,
  "led_color": {
- "fault": "orange",
- "normal": "green",
- "booting": "orange_blink"
+  "fault": "orange",
+  "normal": "green",
+  "booting": "orange_blink"
  }
 }

device/mellanox/x86_64-mlnx_msn2700_simx-r0/system_health_monitoring_config.json

@@ -1,11 +1,11 @@
 {
  "services_to_ignore": [],
  "devices_to_ignore": ["psu","asic","fan"],
- "external_checkers": [],
+ "user_defined_checkers": [],
  "polling_interval": 60,
  "led_color": {
- "fault": "orange",
- "normal": "green",
- "booting": "orange_blink"
+  "fault": "orange",
+  "normal": "green",
+  "booting": "orange_blink"
  }
 }

files/build_templates/sonic_debian_extension.j2

@@ -172,6 +172,12 @@ sudo cp {{platform_common_py2_wheel_path}} $FILESYSTEM_ROOT/$PLATFORM_COMMON_PY2
 sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install $PLATFORM_COMMON_PY2_WHEEL_NAME
 sudo rm -rf $FILESYSTEM_ROOT/$PLATFORM_COMMON_PY2_WHEEL_NAME
 
+# Install system-health Python 2 package
+SYSTEM_HEALTH_PY2_WHEEL_NAME=$(basename {{system_health_py2_wheel_path}})
+sudo cp {{system_health_py2_wheel_path}} $FILESYSTEM_ROOT/$SYSTEM_HEALTH_PY2_WHEEL_NAME
+sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install $SYSTEM_HEALTH_PY2_WHEEL_NAME
+sudo rm -rf $FILESYSTEM_ROOT/$SYSTEM_HEALTH_PY2_WHEEL_NAME
+
 # Install sonic-platform-common Python 3 package
 PLATFORM_COMMON_PY3_WHEEL_NAME=$(basename {{platform_common_py3_wheel_path}})
 sudo cp {{platform_common_py3_wheel_path}} $FILESYSTEM_ROOT/$PLATFORM_COMMON_PY3_WHEEL_NAME
@@ -283,6 +289,10 @@ sudo mkdir -p $FILESYSTEM_ROOT/etc/systemd/system/syslog.socket.d
 sudo cp $IMAGE_CONFIGS/syslog/override.conf $FILESYSTEM_ROOT/etc/systemd/system/syslog.socket.d/override.conf
 sudo cp $IMAGE_CONFIGS/syslog/host_umount.sh $FILESYSTEM_ROOT/usr/bin/
 
+# Copy system-health files
+sudo LANG=C cp $IMAGE_CONFIGS/system-health/system-health.service $FILESYSTEM_ROOT_USR_LIB_SYSTEMD_SYSTEM
+echo "system-health.service" | sudo tee -a $GENERATED_SERVICE_FILE
+
 # Copy logrotate.d configuration files
 sudo cp -f $IMAGE_CONFIGS/logrotate/logrotate.d/* $FILESYSTEM_ROOT/etc/logrotate.d/
 

files/image_config/system-health/system-health.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=SONiC system health monitor
+Requires=database.service updategraph.service
+After=database.service updategraph.service
+
+[Service]
+ExecStart=/usr/local/bin/healthd
+Restart=always
+
+[Install]
+WantedBy=multi-user.target

rules/system-health.dep

@@ -0,0 +1,8 @@
+SPATH := $($(SYSTEM_HEALTH)_SRC_PATH)
+DEP_FILES := $(SONIC_COMMON_FILES_LIST) rules/system-health.mk rules/system-health.dep
+DEP_FILES += $(SONIC_COMMON_BASE_FILES_LIST)
+DEP_FILES += $(shell git ls-files $(SPATH))
+
+$(SYSTEM_HEALTH)_CACHE_MODE := GIT_CONTENT_SHA 
+$(SYSTEM_HEALTH)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST)
+$(SYSTEM_HEALTH)_DEP_FILES := $(DEP_FILES)

rules/system-health.mk

@@ -0,0 +1,9 @@
+# system health python2 wheel
+
+SYSTEM_HEALTH = system_health-1.0-py2-none-any.whl
+$(SYSTEM_HEALTH)_SRC_PATH = $(SRC_PATH)/system-health
+$(SYSTEM_HEALTH)_PYTHON_VERSION = 2
+$(SYSTEM_HEALTH)_DEPENDS = $(SONIC_PY_COMMON_PY2) $(SWSSSDK_PY2) $(SONIC_CONFIG_ENGINE)
+SONIC_PYTHON_WHEELS += $(SYSTEM_HEALTH)
+
+export system_health_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SYSTEM_HEALTH))"

slave.mk

@@ -819,7 +819,8 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
  $(addprefix $(PYTHON_WHEELS_PATH)/,$(REDIS_DUMP_LOAD_PY2)) \
  $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PLATFORM_API_PY2)) \
  $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MODELS_PY3)) \
- $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MGMT_PY))
+ $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MGMT_PY)) \
+ $(addprefix $(PYTHON_WHEELS_PATH)/,$(SYSTEM_HEALTH))
  $(HEADER)
  # Pass initramfs and linux kernel explicitly. They are used for all platforms
  export debs_path="$(IMAGE_DISTRO_DEBS_PATH)"

src/system-health/.gitignore

@@ -0,0 +1,8 @@
+*/deb_dist/
+*/dist/
+*/build/
+*/*.tar.gz
+*/*.egg-info
+*/.cache/
+*.pyc
+*/__pycache__/

src/system-health/health_checker/__init__.py

@@ -0,0 +1,2 @@
+from . import hardware_checker
+from . import service_checker

src/system-health/health_checker/config.py

@@ -0,0 +1,144 @@
+import json
+import os
+
+from sonic_py_common import device_info
+
+
+class Config(object):
+ """
+ Manage configuration of system health.
+ """
+
+ # Default system health check interval
+ DEFAULT_INTERVAL = 60
+
+ # Default boot up timeout. When reboot system, system health will wait a few seconds before starting to work.
+ DEFAULT_BOOTUP_TIMEOUT = 300
+
+ # Default LED configuration. Different platform has different LED capability. This configuration allow vendor to
+ # override the default behavior.
+ DEFAULT_LED_CONFIG = {
+ 'fault': 'red',
+ 'normal': 'green',
+ 'booting': 'orange_blink'
+ }
+
+ # System health configuration file name
+ CONFIG_FILE = 'system_health_monitoring_config.json'
+
+ # Monit service configuration file path
+ MONIT_CONFIG_FILE = '/etc/monit/monitrc'
+
+ # Monit service start delay configuration entry
+ MONIT_START_DELAY_CONFIG = 'with start delay'
+
+ def __init__(self):
+ """
+ Constructor. Initialize all configuration entry to default value in case there is no configuration file.
+ """
+ self.platform_name = device_info.get_platform()
+ self._config_file = os.path.join('/usr/share/sonic/device/', self.platform_name, Config.CONFIG_FILE)
+ self._last_mtime = None
+ self.config_data = None
+ self.interval = Config.DEFAULT_INTERVAL
+ self.ignore_services = None
+ self.ignore_devices = None
+ self.user_defined_checkers = None
+
+ def config_file_exists(self):
+ return os.path.exists(self._config_file)
+
+ def load_config(self):
+ """
+ Load the configuration file from disk.
+ 1. If there is no configuration file, current config entries will reset to default value
+ 2. Only read the configuration file is last_mtime changes for better performance
+ 3. If there is any format issues in configuration file, current config entries will reset to default value
+ :return:
+ """
+ if not self.config_file_exists():
+ if self._last_mtime is not None:
+ self._reset()
+ return
+
+ mtime = os.stat(self._config_file)
+ if mtime != self._last_mtime:
+ try:
+ self._last_mtime = mtime
+ with open(self._config_file, 'r') as f:
+ self.config_data = json.load(f)
+
+ self.interval = self.config_data.get('polling_interval', Config.DEFAULT_INTERVAL)
+ self.ignore_services = self._get_list_data('services_to_ignore')
+ self.ignore_devices = self._get_list_data('devices_to_ignore')
+ self.user_defined_checkers = self._get_list_data('user_defined_checkers')
+ except Exception as e:
+ self._reset()
+
+ def _reset(self):
+ """
+ Reset current configuration entry to default value
+ :return:
+ """
+ self._last_mtime = None
+ self.config_data = None
+ self.interval = Config.DEFAULT_INTERVAL
+ self.ignore_services = None
+ self.ignore_devices = None
+ self.user_defined_checkers = None
+
+ def get_led_color(self, status):
+ """
+ Get desired LED color according to the input status
+ :param status: System health status
+ :return: StringLED color
+ """
+ if self.config_data and 'led_color' in self.config_data:
+ if status in self.config_data['led_color']:
+ return self.config_data['led_color'][status]
+
+ return self.DEFAULT_LED_CONFIG[status]
+
+ def get_bootup_timeout(self):
+ """
+ Get boot up timeout from monit configuration file.
+ 1. If monit configuration file does not exist, return default value
+ 2. If there is any exception while parsing monit config, return default value
+ :return: Integer timeout value
+ """
+ if not os.path.exists(Config.MONIT_CONFIG_FILE):
+ return self.DEFAULT_BOOTUP_TIMEOUT
+
+ try:
+ with open(Config.MONIT_CONFIG_FILE) as f:
+ lines = f.readlines()
+ for line in lines:
+ if not line:
+ continue
+
+ line = line.strip()
+ if not line:
+ continue
+
+ pos = line.find('#')
+ if pos == 0:
+ continue
+
+ line = line[:pos]
+ pos = line.find(Config.MONIT_START_DELAY_CONFIG)
+ if pos != -1:
+ return int(line[pos + len(Config.MONIT_START_DELAY_CONFIG):].strip())
+ except Exception:
+ return self.DEFAULT_BOOTUP_TIMEOUT
+
+ def _get_list_data(self, key):
+ """
+ Get list type configuration data by key and remove duplicate element.
+ :param key: Key of the configuration entry
+ :return: A set of configuration data if key exists
+ """
+ if key in self.config_data:
+ data = self.config_data[key]
+ if isinstance(data, list):
+ return set(data)
+ return None