[sonic-host-service]: Add SONiC Host Services infrastructure (#4840)

- Why I did it

When SONiC is configured with the management framework and/or telemetry services, the applications running inside those containers need to access some functionality on the host system. The following is a non-exhaustive list of such functionality:

Image management
Configuration save and load
ZTP enable/disable and status
Show tech support
- How I did it

The host service is a Python process that listens for requests via D-Bus. It will then service those requests and send a response back to the requestor.

This PR only introduces the host service infrastructure. Applications that need access to the host services must add applets that will register on D-Bus endpoints to service the appropriate functionality.

- How to verify it

- Description for the changelog

Add SONiC Host Service for container to execute select commands in host

Signed-off-by: Nirenjan Krishnan <Nirenjan.Krishnan@dell.com>

files/build_templates/sonic_debian_extension.j2

@@ -196,6 +196,12 @@ sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/sonic-ztp_*.deb || \
  sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f
 {% endif %}
 
+{% if include_host_service == "y" %}
+# Install SONiC Host Service (and its dependencies via 'apt-get -y install -f')
+sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/sonic-host-service_*.deb || \
+ sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f
+{% endif %}
+
 # SONiC utilities installs bash-completion as a dependency. However, it is disabled by default
 # in bash.bashrc, so we copy a version of the file with it enabled here.
 sudo cp -f $IMAGE_CONFIGS/bash/bash.bashrc $FILESYSTEM_ROOT/etc/

rules/config

@@ -125,6 +125,10 @@ INCLUDE_SFLOW = y
 # INCLUDE_MGMT_FRAMEWORK - build docker-sonic-mgmt-framework for CLI and REST server support
 INCLUDE_MGMT_FRAMEWORK = y
 
+# INCLUDE_HOST_SERVICE - build sonic-host-services for mgmt-framework and/or
+# telemetry containers to access host functionality
+INCLUDE_HOST_SERVICE = n
+
 # INCLUDE_RESTAPI - build docker-sonic-restapi for configuring the switch using REST APIs
 INCLUDE_RESTAPI = n
 

rules/docker-sonic-mgmt-framework.mk

@@ -28,6 +28,7 @@ $(DOCKER_MGMT_FRAMEWORK)_CONTAINER_NAME = mgmt-framework
 $(DOCKER_MGMT_FRAMEWORK)_RUN_OPT += --privileged -t
 $(DOCKER_MGMT_FRAMEWORK)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro
 $(DOCKER_MGMT_FRAMEWORK)_RUN_OPT += -v /etc:/host_etc:ro
+$(DOCKER_MGMT_FRAMEWORK)_RUN_OPT += -v /var/run/dbus:/var/run/dbus:rw
 $(DOCKER_MGMT_FRAMEWORK)_RUN_OPT += --mount type=bind,source="/var/platform/",target="/mnt/platform/"
 
 $(DOCKER_MGMT_FRAMEWORK)_BASE_IMAGE_FILES += sonic-cli:/usr/bin/sonic-cli

rules/docker-telemetry.mk

@@ -26,6 +26,7 @@ endif
 $(DOCKER_TELEMETRY)_CONTAINER_NAME = telemetry
 $(DOCKER_TELEMETRY)_RUN_OPT += --privileged -t
 $(DOCKER_TELEMETRY)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro
+$(DOCKER_TELEMETRY)_RUN_OPT += -v /var/run/dbus:/var/run/dbus:rw
 $(DOCKER_TELEMETRY)_RUN_OPT += --mount type=bind,source="/var/platform/",target="/mnt/platform/"
 
 $(DOCKER_TELEMETRY)_FILES += $(SUPERVISOR_PROC_EXIT_LISTENER_SCRIPT)

rules/sonic-host-service.dep

@@ -0,0 +1,11 @@
+
+SPATH := $($(SONIC_HOST_SERVICE)_SRC_PATH)
+DEP_FILES := $(SONIC_COMMON_FILES_LIST) rules/sonic-host-service.mk rules/sonic-host-service.dep
+SMDEP_FILES := $(addprefix $(SPATH)/,$(shell cd $(SPATH) && git ls-files))
+
+$(SONIC_HOST_SERVICE)_CACHE_MODE := GIT_CONTENT_SHA
+$(SONIC_HOST_SERVICE)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST)
+$(SONIC_HOST_SERVICE)_DEP_FILES := $(DEP_FILES)
+$(SONIC_HOST_SERVICE)_SMDEP_FILES := $(SMDEP_FILES)
+$(SONIC_HOST_SERVICE)_SMDEP_PATHS := $(SPATH)
+

rules/sonic-host-service.mk

@@ -0,0 +1,9 @@
+# SONiC host service package
+
+ifeq ($(INCLUDE_HOST_SERVICE), y)
+
+SONIC_HOST_SERVICE = sonic-host-service_1.0.0_all.deb
+$(SONIC_HOST_SERVICE)_SRC_PATH = $(SRC_PATH)/sonic-host-service
+SONIC_MAKE_DEBS += $(SONIC_HOST_SERVICE)
+
+endif

slave.mk

@@ -224,6 +224,7 @@ $(info "VS_PREPARE_MEM" : "$(VS_PREPARE_MEM)")
 $(info "INCLUDE_MGMT_FRAMEWORK" : "$(INCLUDE_MGMT_FRAMEWORK)")
 $(info "INCLUDE_ICCPD" : "$(INCLUDE_ICCPD)")
 $(info "INCLUDE_SYSTEM_TELEMETRY" : "$(INCLUDE_SYSTEM_TELEMETRY)")
+$(info "INCLUDE_HOST_SERVICE" : "$(INCLUDE_HOST_SERVICE)")
 $(info "INCLUDE_RESTAPI" : "$(INCLUDE_RESTAPI)")
 $(info "INCLUDE_SFLOW" : "$(INCLUDE_SFLOW)")
 $(info "INCLUDE_NAT" : "$(INCLUDE_NAT)")
@@ -794,7 +795,8 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
  $(PYTHON_SWSSCOMMON)) \
  $$(addprefix $(TARGET_PATH)/,$$($$*_DOCKERS)) \
  $$(addprefix $(FILES_PATH)/,$$($$*_FILES)) \
- $(if $(findstring y,$(ENABLE_ZTP)),$(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(SONIC_ZTP))) \
+ $(if $(findstring y,$(ENABLE_ZTP)),$(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(SONIC_ZTP))) \
+ $(if $(findstring y,$(INCLUDE_HOST_SERVICE)),$(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(SONIC_HOST_SERVICE))) \
  $(addprefix $(PYTHON_DEBS_PATH)/,$(SONIC_UTILS)) \
  $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PY_COMMON_PY2)) \
  $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PY_COMMON_PY3)) \
@@ -821,6 +823,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
  export enable_ztp="$(ENABLE_ZTP)"
  export include_system_telemetry="$(INCLUDE_SYSTEM_TELEMETRY)"
  export include_restapi="$(INCLUDE_RESTAPI)"
+ export include_host_service="$(INCLUDE_HOST_SERVICE)"
  export include_nat="$(INCLUDE_NAT)"
  export include_sflow="$(INCLUDE_SFLOW)"
  export include_mgmt_framework="$(INCLUDE_MGMT_FRAMEWORK)"

src/sonic-host-service/Makefile

@@ -0,0 +1,41 @@
+################################################################################
+# #
+# Copyright 2020 Dell Inc. #
+# #
+# Licensed under the Apache License, Version 2.0 (the "License"); #
+# you may not use this file except in compliance with the License. #
+# You may obtain a copy of the License at #
+# #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+# #
+# Unless required by applicable law or agreed to in writing, software #
+# distributed under the License is distributed on an "AS IS" BASIS, #
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
+# See the License for the specific language governing permissions and #
+# limitations under the License. #
+# #
+################################################################################
+
+TOPDIR := $(abspath .)
+MAIN_TARGET = sonic-host-service_1.0.0_all.deb
+INSTALL := /usr/bin/install
+
+$(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
+ dpkg-buildpackage -us -uc -b
+ mv ../$(MAIN_TARGET) $(DEST)/
+
+SOURCES := $(wildcard '*.*') $(wildcard 'host_modules/*.py')
+install: $(SOURCES)
+ # Scripts for host service
+ $(INSTALL) -d $(DESTDIR)/usr/lib/sonic_host_service/host_modules
+ $(INSTALL) -D $(TOPDIR)/sonic_host_server.py $(DESTDIR)/usr/lib/sonic_host_service
+ $(INSTALL) -D $(TOPDIR)/host_modules/*.py $(DESTDIR)/usr/lib/sonic_host_service/host_modules
+
+ # D-Bus permissions configuration
+ $(INSTALL) -d $(DESTDIR)/etc/dbus-1/system.d
+ $(INSTALL) -D $(TOPDIR)/org.sonic.hostservice.conf $(DESTDIR)/etc/dbus-1/system.d
+
+ # systemd unit
+ $(INSTALL) -d $(DESTDIR)/lib/systemd/system
+ $(INSTALL) -D $(TOPDIR)/sonic-hostservice.service $(DESTDIR)/lib/systemd/system
+

src/sonic-host-service/debian/.gitignore

@@ -0,0 +1,4 @@
+.debhelper
+sonic-host-service.*
+sonic-host-service/
+files

src/sonic-host-service/debian/changelog

@@ -0,0 +1,5 @@
+sonic-host-service (1.0.0) UNRELEASED; urgency=low
+
+ * Initial release.
+
+ -- Nirenjan Krishnan <Nirenjan.Krishnan@dell.com> Mon, 22 Jun 2020 00:00:00 +0000

src/sonic-host-service/debian/compat

@@ -0,0 +1 @@
+9

src/sonic-host-service/debian/control

@@ -0,0 +1,15 @@
+Source: sonic-host-service
+Maintainer: Nirenjan Krishnan <Nirenjan.Krishnan@dell.com>
+Build-Depends: debhelper (>= 8.0.0),
+ dh-systemd
+Vcs-Git: https://github.com/Azure/sonic-buildimage
+Homepage: https://github.com/Azure/SONiC/
+Standards-Version: 3.9.3
+Section: net
+
+Package: sonic-host-service
+Priority: extra
+Architecture: all
+Depends: python3-systemd, python3-dbus, python3-gi, ${misc:Depends}
+Description: SONiC Host Service
+

src/sonic-host-service/debian/rules

@@ -0,0 +1,9 @@
+#!/usr/bin/make -f
+%:
+ dh $@ --with systemd --parallel
+
+override_dh_auto_clean:
+override_dh_auto_test:
+override_dh_auto_build:
+override_dh_auto_install:
+ make install DESTDIR=debian/sonic-host-service

src/sonic-host-service/host_modules/host_service.py

@@ -0,0 +1,34 @@
+"""Base class for host modules"""
+
+import dbus.service
+import dbus
+
+BUS_NAME_BASE = 'org.SONiC.HostService'
+BUS_PATH = '/org/SONiC/HostService'
+
+def bus_name(mod_name):
+ """Return the bus name for the service"""
+ return BUS_NAME_BASE + '.' + mod_name
+
+def bus_path(mod_name):
+ """Return the bus path for the service"""
+ return BUS_PATH + '/' + mod_name
+
+method = dbus.service.method
+
+class HostService(dbus.service.Object):
+ """Service class for top level DBus endpoint"""
+ def __init__(self, mod_name):
+ self.bus = dbus.SystemBus()
+ self.bus_name = dbus.service.BusName(BUS_NAME_BASE, self.bus)
+ super(HostService, self).__init__(self.bus_name, BUS_PATH)
+
+class HostModule(dbus.service.Object):
+ """Base class for all host modules"""
+ def __init__(self, mod_name):
+ self.bus = dbus.SystemBus()
+ self.bus_name = dbus.service.BusName(bus_name(mod_name), self.bus)
+ super(HostModule, self).__init__(self.bus_name, bus_path(mod_name))
+
+def register():
+ return HostService, "host_service"

src/sonic-host-service/org.sonic.hostservice.conf

@@ -0,0 +1,18 @@
+<!DOCTYPE busconfig PUBLIC
+ "-//freedesktop//DTD D-BUS Bus Configuration 1.0//EN"
+ "http://www.freedesktop.org/standards/dbus/1.0/busconfig.dtd">
+<busconfig>
+
+ <!-- Only root can own the bus -->
+
+ <policy user="root">
+ <allow own_prefix="org.SONiC.HostService"/>
+ </policy>
+
+ <!-- Allow user "root" to invoke methods on the bus -->
+ <policy user="root">
+ <allow send_destination="org.SONiC.HostService"/>
+ <allow receive_sender="org.SONiC.HostService"/>
+ </policy>
+
+</busconfig>

src/sonic-host-service/sonic-hostservice.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=SONiC Host Service
+
+[Service]
+Type=dbus
+BusName=org.SONiC.HostService
+
+ExecStart=/usr/bin/python3 -u /usr/lib/sonic_host_service/sonic_host_server.py
+
+Restart=on-failure
+RestartSec=10
+TimeoutStopSec=3
+
+[Install]
+WantedBy=multi-user.target

src/sonic-host-service/sonic_host_server.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+"""Host Service to handle docker-to-host communication"""
+
+import os
+import os.path
+import glob
+import importlib
+import sys
+
+import dbus
+import dbus.service
+import dbus.mainloop.glib
+
+from gi.repository import GObject
+
+def register_modules():
+ """Register all host modules"""
+ mod_path = os.path.join(os.path.dirname(__file__), 'host_modules')
+ sys.path.append(mod_path)
+ for mod_file in glob.glob(os.path.join(mod_path, '*.py')):
+ if os.path.isfile(mod_file) and not mod_file.endswith('__init__.py'):
+ mod_name = os.path.basename(mod_file)[:-3]
+ module = importlib.import_module(mod_name)
+
+ register_cb = getattr(module, 'register', None)
+ if not register_cb:
+ raise Exception('Missing register function for ' + mod_name)
+
+ register_dbus(register_cb)
+
+def register_dbus(register_cb):
+ """Register DBus handlers for individual modules"""
+ handler_class, mod_name = register_cb()
+ handlers[mod_name] = handler_class(mod_name)
+
+# Create a main loop reactor
+GObject.threads_init()
+dbus.mainloop.glib.threads_init()
+dbus.mainloop.glib.DBusGMainLoop(set_as_default=True)
+loop = GObject.MainLoop()
+handlers = {}
+
+class SignalManager(object):
+ ''' This is used to manage signals received (e.g. SIGINT).
+ When stopping a process (systemctl stop [service]), systemd sends
+ a SIGTERM signal.
+ '''
+ shutdown = False
+ def __init__(self):
+ ''' Install signal handlers.
+
+ SIGTERM is invoked when systemd wants to stop the daemon.
+ For example, "systemctl stop mydaemon.service"
+ or, "systemctl restart mydaemon.service"
+
+ '''
+ import signal
+ signal.signal(signal.SIGTERM, self.sigterm_hdlr)
+
+ def sigterm_hdlr(self, _signum, _frame):
+ self.shutdown = True
+ loop.quit()
+
+sigmgr = SignalManager()
+register_modules()
+
+# Only run if we actually have some handlers
+if handlers:
+ import systemd.daemon
+ systemd.daemon.notify("READY=1")
+
+ while not sigmgr.shutdown:
+ loop.run()
+ if sigmgr.shutdown:
+ break
+
+ systemd.daemon.notify("STOPPING=1")
+else:
+ print("No handlers to register, quitting...")