[sonic-package-manager] support sonic-cli-gen and packages with YANG model

config/config_mgmt.py

@@ -2,8 +2,13 @@
 config_mgmt.py provides classes for configuration validation and for Dynamic
 Port Breakout.
 '''
+
+import os
 import re
+import shutil
 import syslog
+import tempfile
+import yang as ly
 from json import load
 from sys import flags
 from time import sleep as tsleep
@@ -46,34 +51,38 @@ def __init__(self, source="configDB", debug=False, allowTablesWithoutYang=True):
  try:
  self.configdbJsonIn = None
  self.configdbJsonOut = None
+ self.source = source
  self.allowTablesWithoutYang = allowTablesWithoutYang
 
  # logging vars
  self.SYSLOG_IDENTIFIER = "ConfigMgmt"
  self.DEBUG = debug
 
- self.sy = sonic_yang.SonicYang(YANG_DIR, debug=debug)
- # load yang models
- self.sy.loadYangModel()
- # load jIn from config DB or from config DB json file.
- if source.lower() == 'configdb':
- self.readConfigDB()
- # treat any other source as file input
- else:
- self.readConfigDBJson(source)
- # this will crop config, xlate and load.
- self.sy.loadData(self.configdbJsonIn)
-
- # Raise if tables without YANG models are not allowed but exist.
- if not allowTablesWithoutYang and len(self.sy.tablesWithOutYang):
- raise Exception('Config has tables without YANG models')
+ self.__init_sonic_yang()
 
  except Exception as e:
  self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, msg=str(e))
  raise Exception('ConfigMgmt Class creation failed')
 
  return
 
+ def __init_sonic_yang(self):
+ self.sy = sonic_yang.SonicYang(YANG_DIR, debug=self.DEBUG)
+ # load yang models
+ self.sy.loadYangModel()
+ # load jIn from config DB or from config DB json file.
+ if self.source.lower() == 'configdb':
+ self.readConfigDB()
+ # treat any other source as file input
+ else:
+ self.readConfigDBJson(self.source)
+ # this will crop config, xlate and load.
+ self.sy.loadData(self.configdbJsonIn)
+
+ # Raise if tables without YANG models are not allowed but exist.
+ if not self.allowTablesWithoutYang and len(self.sy.tablesWithOutYang):
+ raise Exception('Config has tables without YANG models')
+
  def __del__(self):
  pass
 
@@ -213,6 +222,69 @@ def writeConfigDB(self, jDiff):
 
  return
 
+ def add_module(self, yang_module_str):
+ """
+ Validate and add new YANG module to the system.
+
+ Parameters:
+ yang_module_str (str): YANG module in string representation.
+
+ Returns:
+ None
+ """
+
+ module_name = self.get_module_name(yang_module_str)
+ module_path = os.path.join(YANG_DIR, '{}.yang'.format(module_name))
+ if os.path.exists(module_path):
+ raise Exception('{} already exists'.format(module_name))
+ with open(module_path, 'w') as module_file:
+ module_file.write(yang_module_str)
+ try:
+ self.__init_sonic_yang()
+ except Exception:
+ os.remove(module_path)
+ raise
+
+ def remove_module(self, module_name):
+ """
+ Remove YANG module from the system and validate.
+
+ Parameters:
+ module_name (str): YANG module name.
+
+ Returns:
+ None
+ """
+
+ module_path = os.path.join(YANG_DIR, '{}.yang'.format(module_name))
+ if not os.path.exists(module_path):
+ return
+ temp = tempfile.NamedTemporaryFile(delete=False)
+ try:
+ shutil.move(module_path, temp.name)
+ self.__init_sonic_yang()
+ except Exception:
+ shutil.move(temp.name, module_path)
+ raise
+
+ @staticmethod
+ def get_module_name(yang_module_str):
+ """
+ Read yangs module name from yang_module_str
+
+ Parameters:
+ yang_module_str(str): YANG module string.
+
+ Returns:
+ str: Module name
+ """
+
+ # Instantiate new context since parse_module_mem() loads the module into context.
+ sy = sonic_yang.SonicYang(YANG_DIR)
+ module = sy.ctx.parse_module_mem(yang_module_str, ly.LYS_IN_YANG)
+ return module.name()
+
+
 # End of Class ConfigMgmt
 
 class ConfigMgmtDPB(ConfigMgmt):
@@ -417,8 +489,8 @@ def _deletePorts(self, ports=list(), force=False):
  deps.extend(dep)
 
  # No further action with no force and deps exist
- if force == False and deps:
- return configToLoad, deps, False;
+ if not force and deps:
+ return configToLoad, deps, False
 
  # delets all deps, No topological sort is needed as of now, if deletion
  # of deps fails, return immediately
@@ -436,8 +508,8 @@ def _deletePorts(self, ports=list(), force=False):
  self.sy.deleteNode(str(xPathPort))
 
  # Let`s Validate the tree now
- if self.validateConfigData()==False:
- return configToLoad, deps, False;
+ if not self.validateConfigData():
+ return configToLoad, deps, False
 
  # All great if we are here, Lets get the diff
  self.configdbJsonOut = self.sy.getData()

sonic_package_manager/main.py

@@ -414,10 +414,11 @@ def reset(ctx, name, force, yes, skip_host_plugins):
 
 @cli.command()
 @add_options(PACKAGE_COMMON_OPERATION_OPTIONS)
+@click.option('--keep-config', is_flag=True, help='Keep features configuration in CONFIG DB.')
 @click.argument('name')
 @click.pass_context
 @root_privileges_required
-def uninstall(ctx, name, force, yes):
+def uninstall(ctx, name, force, yes, keep_config):
  """ Uninstall package. """
 
  manager: PackageManager = ctx.obj
@@ -428,6 +429,7 @@ def uninstall(ctx, name, force, yes):
 
  uninstall_opts = {
  'force': force,
+ 'keep_config': keep_config,
  }
 
  try:

sonic_package_manager/manager.py

@@ -10,8 +10,11 @@
 
 import docker
 import filelock
+from config import config_mgmt
 from sonic_py_common import device_info
 
+from sonic_cli_gen.generator import CliGenerator
+
 from sonic_package_manager import utils
 from sonic_package_manager.constraint import (
  VersionConstraint,
@@ -45,7 +48,10 @@
  run_command
 )
 from sonic_package_manager.service_creator.feature import FeatureRegistry
-from sonic_package_manager.service_creator.sonic_db import SonicDB
+from sonic_package_manager.service_creator.sonic_db import (
+ INIT_CFG_JSON,
+ SonicDB
+)
 from sonic_package_manager.service_creator.utils import in_chroot
 from sonic_package_manager.source import (
  PackageSource,
@@ -435,13 +441,16 @@ def install_from_source(self,
 
  @under_lock
  @opt_check
- def uninstall(self, name: str, force=False):
+ def uninstall(self, name: str,
+ force: bool = False,
+ keep_config: bool = False):
  """ Uninstall SONiC Package referenced by name. The uninstallation
  can be forced if force argument is True.
 
  Args:
  name: SONiC Package name.
  force: Force the installation.
+ keep_config: Keep feature configuration in databases.
  Raises:
  PackageManagerError
  """
@@ -482,7 +491,7 @@ def uninstall(self, name: str, force=False):
  self._systemctl_action(package, 'stop')
  self._systemctl_action(package, 'disable')
  self._uninstall_cli_plugins(package)
- self.service_creator.remove(package)
+ self.service_creator.remove(package, keep_config=keep_config)
  self.service_creator.generate_shutdown_sequence_files(
  self._get_installed_packages_except(package)
  )
@@ -1000,9 +1009,13 @@ def get_manager() -> 'PackageManager':
  docker_api = DockerApi(docker.from_env(), ProgressManager())
  registry_resolver = RegistryResolver()
  metadata_resolver = MetadataResolver(docker_api, registry_resolver)
+ cfg_mgmt = config_mgmt.ConfigMgmt(source=INIT_CFG_JSON)
+ cli_generator = CliGenerator(log)
  feature_registry = FeatureRegistry(SonicDB)
  service_creator = ServiceCreator(feature_registry,
- SonicDB)
+ SonicDB,
+ cli_generator,
+ cfg_mgmt)
 
  return PackageManager(docker_api,
  registry_resolver,

sonic_package_manager/manifest.py

@@ -205,7 +205,9 @@ def unmarshal(self, value):
  ManifestField('mandatory', DefaultMarshaller(bool), False),
  ManifestField('show', DefaultMarshaller(str), ''),
  ManifestField('config', DefaultMarshaller(str), ''),
- ManifestField('clear', DefaultMarshaller(str), '')
+ ManifestField('clear', DefaultMarshaller(str), ''),
+ ManifestField('auto-generate-show', DefaultMarshaller(bool), False),
+ ManifestField('auto-generate-config', DefaultMarshaller(bool), False),
  ])
  ])
 

sonic_package_manager/metadata.py

@@ -4,7 +4,7 @@
 
 import json
 import tarfile
-from typing import Dict
+from typing import Dict, Optional
 
 from sonic_package_manager import utils
 from sonic_package_manager.errors import MetadataError
@@ -54,6 +54,7 @@ class Metadata:
 
  manifest: Manifest
  components: Dict[str, Version] = field(default_factory=dict)
+ yang_module_str: Optional[str] = None
 
 
 class MetadataResolver:
@@ -163,5 +164,6 @@ def from_labels(cls, labels: Dict[str, str]) -> Metadata:
  except ValueError as err:
  raise MetadataError(f'Failed to parse component version: {err}')
 
- return Metadata(Manifest.marshal(manifest_dict), components)
+ yang_module_str = sonic_metadata.get('yang-module')
 
+ return Metadata(Manifest.marshal(manifest_dict), components, yang_module_str)