Merge remote-tracking branch 'origin/master' into dev-logrotate-config

config/main.py

@@ -20,13 +20,15 @@
 from jsonpointer import JsonPointerException
 from collections import OrderedDict
 from generic_config_updater.generic_updater import GenericUpdater, ConfigFormat, extract_scope
+from generic_config_updater.gu_common import HOST_NAMESPACE, GenericConfigUpdaterError
 from minigraph import parse_device_desc_xml, minigraph_encoder
 from natsort import natsorted
 from portconfig import get_child_ports
 from socket import AF_INET, AF_INET6
 from sonic_py_common import device_info, multi_asic
 from sonic_py_common.general import getstatusoutput_noshell
 from sonic_py_common.interface import get_interface_table_name, get_port_table_name, get_intf_longname
+from sonic_yang_cfg_generator import SonicYangCfgDbGenerator
 from utilities_common import util_base
 from swsscommon import swsscommon
 from swsscommon.swsscommon import SonicV2Connector, ConfigDBConnector
@@ -1155,25 +1157,75 @@ def validate_gre_type(ctx, _, value):
  return gre_type_value
  except ValueError:
  raise click.UsageError("{} is not a valid GRE type".format(value))
-
+
+
+def multi_asic_save_config(db, filename):
+ """A function to save all asic's config to single file
+ """
+ all_current_config = {}
+ cfgdb_clients = db.cfgdb_clients
+
+ for ns, config_db in cfgdb_clients.items():
+ current_config = config_db.get_config()
+ sonic_cfggen.FormatConverter.to_serialized(current_config)
+ asic_name = "localhost" if ns == DEFAULT_NAMESPACE else ns
+ all_current_config[asic_name] = sort_dict(current_config)
+ click.echo("Integrate each ASIC's config into a single JSON file {}.".format(filename))
+ with open(filename, 'w') as file:
+ json.dump(all_current_config, file, indent=4)
+
 # Function to apply patch for a single ASIC.
 def apply_patch_for_scope(scope_changes, results, config_format, verbose, dry_run, ignore_non_yang_tables, ignore_path):
  scope, changes = scope_changes
  # Replace localhost to DEFAULT_NAMESPACE which is db definition of Host
- if scope.lower() == "localhost" or scope == "":
+ if scope.lower() == HOST_NAMESPACE or scope == "":
  scope = multi_asic.DEFAULT_NAMESPACE
- 
- scope_for_log = scope if scope else "localhost"
+
+ scope_for_log = scope if scope else HOST_NAMESPACE
  try:
  # Call apply_patch with the ASIC-specific changes and predefined parameters
- GenericUpdater(namespace=scope).apply_patch(jsonpatch.JsonPatch(changes), config_format, verbose, dry_run, ignore_non_yang_tables, ignore_path)
+ GenericUpdater(namespace=scope).apply_patch(jsonpatch.JsonPatch(changes),
+ config_format,
+ verbose,
+ dry_run,
+ ignore_non_yang_tables,
+ ignore_path)
  results[scope_for_log] = {"success": True, "message": "Success"}
  log.log_notice(f"'apply-patch' executed successfully for {scope_for_log} by {changes}")
  except Exception as e:
  results[scope_for_log] = {"success": False, "message": str(e)}
  log.log_error(f"'apply-patch' executed failed for {scope_for_log} by {changes} due to {str(e)}")
 
 
+def validate_patch(patch):
+ try:
+ command = ["show", "runningconfiguration", "all"]
+ proc = subprocess.Popen(command, text=True, stdout=subprocess.PIPE)
+ all_running_config, returncode = proc.communicate()
+ if returncode:
+ log.log_notice(f"Fetch all runningconfiguration failed as output:{all_running_config}")
+ return False
+
+ # Structure validation and simulate apply patch.
+ all_target_config = patch.apply(json.loads(all_running_config))
+
+ # Verify target config by YANG models
+ target_config = all_target_config.pop(HOST_NAMESPACE) if multi_asic.is_multi_asic() else all_target_config
+ target_config.pop("bgpraw", None)
+ if not SonicYangCfgDbGenerator().validate_config_db_json(target_config):
+ return False
+
+ if multi_asic.is_multi_asic():
+ for asic in multi_asic.get_namespace_list():
+ target_config = all_target_config.pop(asic)
+ target_config.pop("bgpraw", None)
+ if not SonicYangCfgDbGenerator().validate_config_db_json(target_config):
+ return False
+
+ return True
+ except Exception as e:
+ raise GenericConfigUpdaterError(f"Validate json patch: {patch} failed due to:{e}")
+
 # This is our main entrypoint - the main 'config' command
 @click.group(cls=clicommon.AbbreviationGroup, context_settings=CONTEXT_SETTINGS)
 @click.pass_context
@@ -1241,7 +1293,8 @@ def config(ctx):
 @click.option('-y', '--yes', is_flag=True, callback=_abort_if_false,
  expose_value=False, prompt='Existing files will be overwritten, continue?')
 @click.argument('filename', required=False)
-def save(filename):
+@clicommon.pass_db
+def save(db, filename):
  """Export current config DB to a file on disk.\n
  <filename> : Names of configuration file(s) to save, separated by comma with no spaces in between
  """
@@ -1256,7 +1309,13 @@ def save(filename):
  if filename is not None:
  cfg_files = filename.split(',')
 
- if len(cfg_files) != num_cfg_file:
+ # If only one filename is provided in multi-ASIC mode,
+ # save all ASIC configurations to that single file.
+ if len(cfg_files) == 1 and multi_asic.is_multi_asic():
+ filename = cfg_files[0]
+ multi_asic_save_config(db, filename)
+ return
+ elif len(cfg_files) != num_cfg_file:
  click.echo("Input {} config file(s) separated by comma for multiple files ".format(num_cfg_file))
  return
 
@@ -1381,6 +1440,9 @@ def apply_patch(ctx, patch_file_path, format, dry_run, ignore_non_yang_tables, i
  patch_as_json = json.loads(text)
  patch = jsonpatch.JsonPatch(patch_as_json)
 
+ if not validate_patch(patch):
+ raise GenericConfigUpdaterError(f"Failed validating patch:{patch}")
+
  results = {}
  config_format = ConfigFormat[format.upper()]
  # Initialize a dictionary to hold changes categorized by scope
@@ -1403,7 +1465,8 @@ def apply_patch(ctx, patch_file_path, format, dry_run, ignore_non_yang_tables, i
  # Empty case to force validate YANG model.
  if not changes_by_scope:
  asic_list = [multi_asic.DEFAULT_NAMESPACE]
- asic_list.extend(multi_asic.get_namespace_list())
+ if multi_asic.is_multi_asic():
+ asic_list.extend(multi_asic.get_namespace_list())
  for asic in asic_list:
  changes_by_scope[asic] = []
 
@@ -1416,7 +1479,7 @@ def apply_patch(ctx, patch_file_path, format, dry_run, ignore_non_yang_tables, i
 
  if failures:
  failure_messages = '\n'.join([f"- {failed_scope}: {results[failed_scope]['message']}" for failed_scope in failures])
- raise Exception(f"Failed to apply patch on the following scopes:\n{failure_messages}")
+ raise GenericConfigUpdaterError(f"Failed to apply patch on the following scopes:\n{failure_messages}")
 
  log.log_notice(f"Patch applied successfully for {patch}.")
  click.secho("Patch applied successfully.", fg="cyan", underline=True)
@@ -1620,9 +1683,9 @@ def reload(db, filename, yes, load_sysinfo, no_service_restart, force, file_form
  file_input = read_json_file(file)
 
  platform = file_input.get("DEVICE_METADATA", {}).\
- get("localhost", {}).get("platform")
+ get(HOST_NAMESPACE, {}).get("platform")
  mac = file_input.get("DEVICE_METADATA", {}).\
- get("localhost", {}).get("mac")
+ get(HOST_NAMESPACE, {}).get("mac")
 
  if not platform or not mac:
  log.log_warning("Input file does't have platform or mac. platform: {}, mac: {}"
@@ -1995,8 +2058,8 @@ def override_config_table(db, input_config_db, dry_run):
  if multi_asic.is_multi_asic() and len(config_input):
  # Golden Config will use "localhost" to represent host name
  if ns == DEFAULT_NAMESPACE:
- if "localhost" in config_input.keys():
- ns_config_input = config_input["localhost"]
+ if HOST_NAMESPACE in config_input.keys():
+ ns_config_input = config_input[HOST_NAMESPACE]
  else:
  click.secho("Wrong config format! 'localhost' not found in host config! cannot override.. abort")
  sys.exit(1)

generic_config_updater/generic_updater.py

@@ -2,7 +2,7 @@
 import jsonpointer
 import os
 from enum import Enum
-from .gu_common import GenericConfigUpdaterError, EmptyTableError, ConfigWrapper, \
+from .gu_common import HOST_NAMESPACE, GenericConfigUpdaterError, EmptyTableError, ConfigWrapper, \
  DryRunConfigWrapper, PatchWrapper, genericUpdaterLogging
 from .patch_sorter import StrictPatchSorter, NonStrictPatchSorter, ConfigSplitter, \
  TablesWithoutYangConfigSplitter, IgnorePathsFromYangConfigSplitter
@@ -16,28 +16,26 @@ def extract_scope(path):
  if not path:
  raise Exception("Wrong patch with empty path.")
 
- try:
- pointer = jsonpointer.JsonPointer(path)
- parts = pointer.parts
- except Exception as e:
- raise Exception(f"Error resolving path: '{path}' due to {e}")
+ pointer = jsonpointer.JsonPointer(path)
+ parts = pointer.parts
 
  if not parts:
- raise Exception("Wrong patch with empty path.")
+ raise GenericConfigUpdaterError("Wrong patch with empty path.")
  if parts[0].startswith("asic"):
  if not parts[0][len("asic"):].isnumeric():
- raise Exception(f"Error resolving path: '{path}' due to incorrect ASIC number.")
+ raise GenericConfigUpdaterError(f"Error resolving path: '{path}' due to incorrect ASIC number.")
  scope = parts[0]
  remainder = "/" + "/".join(parts[1:])
- elif parts[0] == "localhost":
- scope = "localhost"
+ elif parts[0] == HOST_NAMESPACE:
+ scope = HOST_NAMESPACE
  remainder = "/" + "/".join(parts[1:])
  else:
  scope = ""
  remainder = path
 
  return scope, remainder
 
+
 class ConfigLock:
  def acquire_lock(self):
  # TODO: Implement ConfigLock
@@ -67,7 +65,7 @@ def __init__(self,
  self.changeapplier = changeapplier if changeapplier is not None else ChangeApplier(namespace=self.namespace)
 
  def apply(self, patch, sort=True):
- scope = self.namespace if self.namespace else 'localhost'
+ scope = self.namespace if self.namespace else HOST_NAMESPACE
  self.logger.log_notice(f"{scope}: Patch application starting.")
  self.logger.log_notice(f"{scope}: Patch: {patch}")
 
@@ -84,10 +82,10 @@ def apply(self, patch, sort=True):
  self.config_wrapper.validate_field_operation(old_config, target_config)
 
  # Validate target config does not have empty tables since they do not show up in ConfigDb
- self.logger.log_notice(f"{scope}: alidating target config does not have empty tables, " \
- "since they do not show up in ConfigDb.")
+ self.logger.log_notice(f"""{scope}: validating target config does not have empty tables,
+ since they do not show up in ConfigDb.""")
  empty_tables = self.config_wrapper.get_empty_tables(target_config)
- if empty_tables: # if there are empty tables
+ if empty_tables:  # if there are empty tables
  empty_tables_txt = ", ".join(empty_tables)
  raise EmptyTableError(f"{scope}: given patch is not valid because it will result in empty tables " \
  "which is not allowed in ConfigDb. " \
@@ -105,9 +103,6 @@ def apply(self, patch, sort=True):
  self.logger.log_notice(f"The {scope} patch was converted into {changes_len} " \
  f"change{'s' if changes_len != 1 else ''}{':' if changes_len > 0 else '.'}")
 
- for change in changes:
- self.logger.log_notice(f" * {change}")
-
  # Apply changes in order
  self.logger.log_notice(f"{scope}: applying {changes_len} change{'s' if changes_len != 1 else ''} " \
  f"in order{':' if changes_len > 0 else '.'}")

generic_config_updater/gu_common.py

@@ -16,6 +16,7 @@
 SYSLOG_IDENTIFIER = "GenericConfigUpdater"
 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
 GCU_FIELD_OP_CONF_FILE = f"{SCRIPT_DIR}/gcu_field_operation_validators.conf.json"
+HOST_NAMESPACE = "localhost"
 
 class GenericConfigUpdaterError(Exception):
  pass

scripts/fabricstat

@@ -399,6 +399,49 @@ class FabricIsolation(FabricStat):
  print(tabulate(body, header, tablefmt='simple', stralign='right'))
  return
 
+class FabricRate(FabricStat):
+ def rate_print(self):
+ # Connect to database
+ self.db = multi_asic.connect_to_all_dbs_for_ns(self.namespace)
+ # Get the set of all fabric ports
+ port_keys = self.db.keys(self.db.STATE_DB, FABRIC_PORT_STATUS_TABLE_PREFIX + '*')
+ # Create a new dictionary. The keys are the local port values in integer format.
+ # Only fabric ports that have remote port data are added.
+ port_dict = {}
+ for port_key in port_keys:
+ port_data = self.db.get_all(self.db.STATE_DB, port_key)
+ port_number = int(port_key.replace("FABRIC_PORT_TABLE|PORT", ""))
+ port_dict.update({port_number: port_data})
+ # Create ordered table of fabric ports.
+ rxRate = 0
+ rxData = 0
+ txRate = 0
+ txData = 0
+ time = 0
+ local_time = ""
+ # RX data , Tx data , Time are for testing
+ asic = "asic0"
+ if self.namespace:
+ asic = self.namespace
+ header = ["ASIC", "Link ID", "Rx Data Mbps", "Tx Data Mbps"]
+ body = []
+ for port_number in sorted(port_dict.keys()):
+ port_data = port_dict[port_number]
+ if "OLD_RX_RATE_AVG" in port_data:
+ rxRate = port_data["OLD_RX_RATE_AVG"]
+ if "OLD_RX_DATA" in port_data:
+ rxData = port_data["OLD_RX_DATA"]
+ if "OLD_TX_RATE_AVG" in port_data:
+ txRate = port_data["OLD_TX_RATE_AVG"]
+ if "OLD_TX_DATA" in port_data:
+ txData = port_data["OLD_TX_DATA"]
+ if "LAST_TIME" in port_data:
+ time = int(port_data["LAST_TIME"])
+ local_time = datetime.fromtimestamp(time)
+ body.append((asic, port_number, rxRate, txRate));
+ click.echo()
+ click.echo(tabulate(body, header, tablefmt='simple', stralign='right'))
+
 def main():
  global cnstat_dir
  global cnstat_fqn_file_port
@@ -415,6 +458,8 @@ Examples:
  fabricstat -q -n asic0
  fabricstat -c
  fabricstat -c -n asic0
+ fabricstat -s
+ fabricstat -s -n asic0
  fabricstat -C
  fabricstat -D
 """)
@@ -425,6 +470,7 @@ Examples:
  parser.add_argument('-e', '--errors', action='store_true', help='Display errors')
  parser.add_argument('-c','--capacity',action='store_true', help='Display fabric capacity')
  parser.add_argument('-i','--isolation', action='store_true', help='Display fabric ports isolation status')
+ parser.add_argument('-s','--rate', action='store_true', help='Display fabric counters rate')
  parser.add_argument('-C','--clear', action='store_true', help='Copy & clear fabric counters')
  parser.add_argument('-D','--delete', action='store_true', help='Delete saved stats')
 
@@ -433,6 +479,7 @@ Examples:
  reachability = args.reachability
  capacity_status = args.capacity
  isolation_status = args.isolation
+ rate = args.rate
  namespace = args.namespace
  errors_only = args.errors
 
@@ -455,17 +502,21 @@ Examples:
 
  def nsStat(ns, errors_only):
  if queue:
- stat = FabricQueueStat(ns)
+  stat = FabricQueueStat(ns)
  elif reachability:
- stat = FabricReachability(ns)
- stat.reachability_print()
- return
+  stat = FabricReachability(ns)
+  stat.reachability_print()
+  return
  elif isolation_status:
- stat = FabricIsolation(ns)
- stat.isolation_print()
- return
+ stat = FabricIsolation(ns)
+ stat.isolation_print()
+ return
+ elif rate:
+ stat = FabricRate(ns)
+ stat.rate_print()
+ return
  else:
- stat = FabricPortStat(ns)
+  stat = FabricPortStat(ns)
  cnstat_dict = stat.get_cnstat_dict()
  if save_fresh_stats:
  stat.save_fresh_stats()

show/fabric.py

@@ -74,3 +74,13 @@ def queue(namespace):
  if namespace is not None:
  cmd += ['-n', str(namespace)]
  clicommon.run_command(cmd)
+
+
+@counters.command()
+@multi_asic_util.multi_asic_click_option_namespace
+def rate(namespace):
+ """Show fabric counters rate"""
+ cmd = ['fabricstat', '-s']
+ if namespace is not None:
+ cmd += ['-n', str(namespace)]
+ clicommon.run_command(cmd)

show/main.py

@@ -1438,11 +1438,11 @@ def all(verbose):
  for ns in ns_list:
  ns_config = get_config_json_by_namespace(ns)
  if bgp_util.is_bgp_feature_state_enabled(ns):
- ns_config['bgpraw'] = bgp_util.run_bgp_show_command(bgpraw_cmd, ns)
+ ns_config['bgpraw'] = bgp_util.run_bgp_show_command(bgpraw_cmd, ns, exit_on_fail=False)
  output[ns] = ns_config
  click.echo(json.dumps(output, indent=4))
  else:
- host_config['bgpraw'] = bgp_util.run_bgp_show_command(bgpraw_cmd)
+ host_config['bgpraw'] = bgp_util.run_bgp_show_command(bgpraw_cmd, exit_on_fail=False)
  click.echo(json.dumps(output['localhost'], indent=4))
 
 

tests/config_save_output/all_config_db.json

@@ -0,0 +1,5 @@
+{
+ "localhost": {},
+ "asic0": {},
+ "asic1": {}
+}