[sonic-package-manager] support sonic-cli-gen and packages with YANG model

config/config_mgmt.py

@@ -2,8 +2,11 @@
 config_mgmt.py provides classes for configuration validation and for Dynamic
 Port Breakout.
 '''
+
+import os
 import re
 import syslog
+import yang as ly
 from json import load
 from sys import flags
 from time import sleep as tsleep
@@ -46,34 +49,38 @@ def __init__(self, source="configDB", debug=False, allowTablesWithoutYang=True):
         try:
             self.configdbJsonIn = None
             self.configdbJsonOut = None
+            self.source = source
             self.allowTablesWithoutYang = allowTablesWithoutYang
 
             # logging vars
             self.SYSLOG_IDENTIFIER = "ConfigMgmt"
             self.DEBUG = debug
 
-            self.sy = sonic_yang.SonicYang(YANG_DIR, debug=debug)
-            # load yang models
-            self.sy.loadYangModel()
-            # load jIn from config DB or from config DB json file.
-            if source.lower() == 'configdb':
-                self.readConfigDB()
-            # treat any other source as file input
-            else:
-                self.readConfigDBJson(source)
-            # this will crop config, xlate and load.
-            self.sy.loadData(self.configdbJsonIn)
-
-            # Raise if tables without YANG models are not allowed but exist.
-            if not allowTablesWithoutYang and len(self.sy.tablesWithOutYang):
-                raise Exception('Config has tables without YANG models')
+            self.__init_sonic_yang()
 
         except Exception as e:
             self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, msg=str(e))
             raise Exception('ConfigMgmt Class creation failed')
 
         return
 
+    def __init_sonic_yang(self):
+        self.sy = sonic_yang.SonicYang(YANG_DIR, debug=self.DEBUG)
+        # load yang models
+        self.sy.loadYangModel()
+        # load jIn from config DB or from config DB json file.
+        if self.source.lower() == 'configdb':
+            self.readConfigDB()
+        # treat any other source as file input
+        else:
+            self.readConfigDBJson(self.source)
+        # this will crop config, xlate and load.
+        self.sy.loadData(self.configdbJsonIn)
+
+        # Raise if tables without YANG models are not allowed but exist.
+        if not self.allowTablesWithoutYang and len(self.sy.tablesWithOutYang):
+            raise Exception('Config has tables without YANG models')
+
     def __del__(self):
         pass
 
@@ -213,6 +220,70 @@ def writeConfigDB(self, jDiff):
 
         return
 
+    def add_module(self, yang_module_str, replace_if_exists=False):
+        """
+        Validate and add new YANG module to the system.
+
+        Parameters:
+            yang_module_str (str): YANG module in string representation.
+
+        Returns:
+            None
+        """
+
+        module_name = self.get_module_name(yang_module_str)
+        module_path = os.path.join(YANG_DIR, '{}.yang'.format(module_name))
+        if os.path.exists(module_path) and not replace_if_exists:
+            raise Exception('{} already exists'.format(module_name))
+        with open(module_path, 'w') as module_file:
+            module_file.write(yang_module_str)
+        try:
+            self.__init_sonic_yang()
+        except Exception:
+            os.remove(module_path)
+            raise
+
+    def remove_module(self, module_name):
+        """
+        Remove YANG module on the system and validate.
+
+        Parameters:
+            module_name (str): YANG module name.
+
+        Returns:
+            None
+        """
+
+        module_path = os.path.join(YANG_DIR, '{}.yang'.format(module_name))
+        if not os.path.exists(module_path):
+            return
+        with open(module_path, 'r') as module_file:
+            yang_module_str = module_file.read()
+        try:
+            os.remove(module_path)
+            self.__init_sonic_yang()
+        except Exception:
+            self.add_module(yang_module_str)
+            raise
+
+    @staticmethod
+    def get_module_name(yang_module_str):
+        """
+        Read yangs module name from yang_module_str
+
+        Parameters:
+            yang_module_str(str): YANG module string.
+
+        Returns:
+            str: Module name
+        """
+
+        # Instantiate new context since parse_module_mem() loads the module into context.
+        sy = sonic_yang.SonicYang(YANG_DIR)
+        module = sy.ctx.parse_module_mem(yang_module_str, ly.LYS_IN_YANG)
+        return module.name()
+
+
 # End of Class ConfigMgmt
 
 class ConfigMgmtDPB(ConfigMgmt):
@@ -417,8 +488,8 @@ def _deletePorts(self, ports=list(), force=False):
                     deps.extend(dep)
 
             # No further action with no force and deps exist
-            if force == False and deps:
-                return configToLoad, deps, False;
+            if not force and deps:
+                return configToLoad, deps, False
 
             # delets all deps, No topological sort is needed as of now, if deletion
             # of deps fails, return immediately
@@ -436,8 +507,8 @@ def _deletePorts(self, ports=list(), force=False):
                 self.sy.deleteNode(str(xPathPort))
 
             # Let`s Validate the tree now
-            if self.validateConfigData()==False:
-                return configToLoad, deps, False;
+            if not self.validateConfigData():
+                return configToLoad, deps, False
 
             # All great if we are here, Lets get the diff
             self.configdbJsonOut = self.sy.getData()

config/main.py

@@ -1993,20 +1993,28 @@ def warm_restart(ctx, redis_unix_socket_path):
     ctx.obj = {'db': config_db, 'state_db': state_db, 'prefix': prefix}
 
 @warm_restart.command('enable')
-@click.argument('module', metavar='<module>', default='system', required=False, type=click.Choice(["system", "swss", "bgp", "teamd"]))
+@click.argument('module', metavar='<module>', default='system', required=False)
 @click.pass_context
 def warm_restart_enable(ctx, module):
     state_db = ctx.obj['state_db']
+    config_db = ctx.obj['db']
+    feature_table = config_db.get_table('FEATURE')
+    if module != 'system' and module not in feature_table:
+        exit('Feature {} is unknown'.format(module))
     prefix = ctx.obj['prefix']
     _hash = '{}{}'.format(prefix, module)
     state_db.set(state_db.STATE_DB, _hash, 'enable', 'true')
     state_db.close(state_db.STATE_DB)
 
 @warm_restart.command('disable')
-@click.argument('module', metavar='<module>', default='system', required=False, type=click.Choice(["system", "swss", "bgp", "teamd"]))
+@click.argument('module', metavar='<module>', default='system', required=False)
 @click.pass_context
 def warm_restart_enable(ctx, module):
     state_db = ctx.obj['state_db']
+    config_db = ctx.obj['db']
+    feature_table = config_db.get_table('FEATURE')
+    if module != 'system' and module not in feature_table:
+        exit('Feature {} is unknown'.format(module))
     prefix = ctx.obj['prefix']
     _hash = '{}{}'.format(prefix, module)
     state_db.set(state_db.STATE_DB, _hash, 'enable', 'false')

scripts/fast-reboot

@@ -7,6 +7,7 @@ WARM_DIR=/host/warmboot
 REDIS_FILE=dump.rdb
 REBOOT_SCRIPT_NAME=$(basename $0)
 REBOOT_TYPE="${REBOOT_SCRIPT_NAME}"
+SHUTDOWN_ORDER_FILE="/etc/sonic/${REBOOT_TYPE}_order"
 VERBOSE=no
 FORCE=no
 IGNORE_ASIC=no
@@ -567,82 +568,72 @@ if [ -x ${LOG_SSD_HEALTH} ]; then
 fi
 
 
-# Kill nat docker after saving the conntrack table
-debug "Stopping nat ..."
-/usr/local/bin/dump_nat_entries.py
-docker kill nat > /dev/null || true
-systemctl stop nat
-debug "Stopped nat ..."
-
-# Kill radv before stopping BGP service to prevent announcing our departure.
-debug "Stopping radv service..."
-systemctl stop radv
-debug "Stopped radv service..."
-
-# Kill bgpd to start the bgp graceful restart procedure
-debug "Stopping bgp ..."
-systemctl stop bgp
-debug "Stopped  bgp ..."
-
-# Kill sflow docker	
-debug "Stopping sflow ..."	
-container kill sflow &> /dev/null || debug "Docker sflow is not running ($?) ..."
-systemctl stop sflow
-debug "Stopped sflow ..."
-
-# Kill lldp, otherwise it sends informotion about reboot.
-# We call `docker kill lldp` to ensure the container stops as quickly as possible,
-# then immediately call `systemctl stop lldp` to prevent the service from
-# restarting the container automatically.
-container kill lldp &> /dev/null || debug "Docker lldp is not running ($?) ..."
-systemctl stop lldp
-
-if [[ "$REBOOT_TYPE" = "fast-reboot" ]]; then
-    debug "Stopping teamd ..."
-    systemctl stop teamd
-    debug "Stopped teamd ..."
+if [[ -f ${SHUTDOWN_ORDER_FILE} ]]; then
+    SERVICES_TO_STOP="$(cat ${SHUTDOWN_ORDER_FILE})"
+else
+    # TODO: to be removed once sonic-buildimage change is in
+    if [[ "${REBOOT_TYPE}" == "fast-reboot" ]]; then
+        SERVICES_TO_STOP="nat radv bgp sflow lldp swss teamd syncd"
+    elif [[ "${REBOOT_TYPE}" == "fastfast-reboot" || "${REBOOT_TYPE}" == "warm-reboot" ]]; then
+        SERVICES_TO_STOP="nat radv bgp sflow lldp teamd swss syncd"
+    else
+        error "Unexpected reboot type ${REBOOT_TYPE}"
+        exit $EXIT_FAILURE
+    fi
 fi
 
-debug "Stopping swss service ..."
-systemctl stop swss
-debug "Stopped swss service ..."
+for service in ${SERVICES_TO_STOP}; do
+    debug "Stopping ${service} ..."
 
-if [[ "$REBOOT_TYPE" = "warm-reboot" || "$REBOOT_TYPE" = "fastfast-reboot" ]]; then
-    # Pre-shutdown syncd
-    initialize_pre_shutdown
+    # TODO: These exceptions for nat, sflow, lldp
+    # have to be coded in corresponding service scripts
 
-    if [[ "x$sonic_asic_type" == x"mellanox" ]]; then
-        check_issu_bank_file
+    if [[ "${service}" = "nat" ]]; then
+        /usr/local/bin/dump_nat_entries.py
     fi
 
-    request_pre_shutdown
-
-    wait_for_pre_shutdown_complete_or_fail
-
-    if [[ "x$sonic_asic_type" == x"mellanox" ]]; then
-        check_issu_bank_file
+    if [[ "${service}" = "nat" || "${service}" = "sflow" || "${service}" = "lldp" ]]; then
+        container kill "${service}"  &> /dev/null || debug "Docker ${service} is not running ($?) ..."
     fi
 
-    # Warm reboot: dump state to host disk
-    if [[ "$REBOOT_TYPE" = "fastfast-reboot" ]]; then
-        sonic-db-cli ASIC_DB FLUSHDB > /dev/null
-        sonic-db-cli COUNTERS_DB FLUSHDB > /dev/null
-        sonic-db-cli FLEX_COUNTER_DB FLUSHDB > /dev/null
+    if [[ "${service}" = "syncd" ]]; then
+        systemctl stop ${service} || debug "Ignore stopping ${service} service error $?"
+    else
+        systemctl stop ${service}
     fi
 
-    # TODO: backup_database preserves FDB_TABLE
-    # need to cleanup as well for fastfast boot case
-    backup_database
+    debug "Stopped ${service}"
 
-    # Stop teamd gracefully
-    debug "Stopping teamd ..."
-    systemctl stop teamd
-    debug "Stopped  teamd ..."
-fi
+    if [[ "${service}" = "swss" ]]; then
+        if [[ "$REBOOT_TYPE" = "warm-reboot" || "$REBOOT_TYPE" = "fastfast-reboot" ]]; then
+            # Pre-shutdown syncd
+            initialize_pre_shutdown
+
+            if [[ "x$sonic_asic_type" == x"mellanox" ]]; then
+                check_issu_bank_file
+            fi
 
-debug "Stopping syncd ..."
-systemctl stop syncd || debug "Ignore stopping syncd service error $?"
-debug "Stopped  syncd ..."
+            request_pre_shutdown
+
+            wait_for_pre_shutdown_complete_or_fail
+
+            if [[ "x$sonic_asic_type" == x"mellanox" ]]; then
+                check_issu_bank_file
+            fi
+
+            # Warm reboot: dump state to host disk
+            if [[ "$REBOOT_TYPE" = "fastfast-reboot" ]]; then
+                sonic-db-cli ASIC_DB FLUSHDB > /dev/null
+                sonic-db-cli COUNTERS_DB FLUSHDB > /dev/null
+                sonic-db-cli FLEX_COUNTER_DB FLUSHDB > /dev/null
+            fi
+
+            # TODO: backup_database preserves FDB_TABLE
+            # need to cleanup as well for fastfast boot case
+            backup_database
+        fi
+    fi
+done
 
 # Kill other containers to make the reboot faster
 # We call `docker kill ...` to ensure the container stops as quickly as possible,

scripts/generate_shutdown_order.py

@@ -0,0 +1,15 @@
+#!/usr/bin/python3
+
+''' This script is used to generate initial warm/fast shutdown order file '''
+
+from sonic_package_manager import PackageManager
+
+def main():
+    manager = PackageManager.get_manager()
+    installed_packages = manager.get_installed_packages()
+    print('installed packages {}'.format(installed_packages))
+    manager.service_creator.generate_shutdown_sequence_files(installed_packages)
+    print('Done.')
+
+if __name__ == '__main__':
+    main()