From 042254e1f9f08c9b4abe22c5ceab8148166033a8 Mon Sep 17 00:00:00 2001 From: rajendra-dendukuri <47423477+rajendra-dendukuri@users.noreply.github.com> Date: Fri, 4 Dec 2020 12:10:43 -0500 Subject: [PATCH] Kdump improvements (#1284) - Added support for reporting kdump reboot cause to detect and report a kernel crash - Added additional commands to sonic-kdump-config wrapper script to dump Kdump configuration and status information in JSON format - Automatically save Kdump configuration in startup configuration file. This is an added convenience to the user as Kdump configuration changes require a system reboot and user is expected to save the configuration so that they are activated post reboot. - Additional bug fixes and improvements to allow hostcfgd to process Kdump configuration changes in ConfigDB. sonic-kdump-config script will write to ConfigDB. - Moved kdump CLI commands to a dedicated file Signed-off-by: Rajendra Dendukuri --- config/kdump.py | 44 ++++++ config/main.py | 46 +----- scripts/reboot | 21 ++- scripts/sonic-kdump-config | 287 +++++++++++++++++++++++++++++++++---- show/kdump.py | 10 +- show/main.py | 1 + 6 files changed, 320 insertions(+), 89 deletions(-) create mode 100644 config/kdump.py diff --git a/config/kdump.py b/config/kdump.py new file mode 100644 index 000000000000..c99a6485b623 --- /dev/null +++ b/config/kdump.py @@ -0,0 +1,44 @@ +import os +import click +import utilities_common.cli as clicommon +from swsssdk import ConfigDBConnector + +@click.group(cls=clicommon.AbbreviationGroup, name="kdump") +def kdump(): + """ Configure kdump """ + if os.geteuid() != 0: + exit("Root privileges are required for this operation") + +@kdump.command() +def disable(): + """Disable kdump operation""" + config_db = ConfigDBConnector() + if config_db is not None: + config_db.connect() + config_db.mod_entry("KDUMP", "config", {"enabled": "false"}) + +@kdump.command() +def enable(): + """Enable kdump operation""" + config_db = ConfigDBConnector() + if config_db is not None: + config_db.connect() + config_db.mod_entry("KDUMP", "config", {"enabled": "true"}) + +@kdump.command() +@click.argument('kdump_memory', metavar='', required=True) +def memory(kdump_memory): + """Set memory allocated for kdump capture kernel""" + config_db = ConfigDBConnector() + if config_db is not None: + config_db.connect() + config_db.mod_entry("KDUMP", "config", {"memory": kdump_memory}) + +@kdump.command('num-dumps') +@click.argument('kdump_num_dumps', metavar='', required=True, type=int) +def num_dumps(kdump_num_dumps): + """Set max number of dump files for kdump""" + config_db = ConfigDBConnector() + if config_db is not None: + config_db.connect() + config_db.mod_entry("KDUMP", "config", {"num_dumps": kdump_num_dumps}) diff --git a/config/main.py b/config/main.py index b832c45cb965..2316d7e38c31 100755 --- a/config/main.py +++ b/config/main.py @@ -28,6 +28,7 @@ from . import chassis_modules from . import console from . import feature +from . import kdump from . import kube from . import mlnx from . import muxcable @@ -878,6 +879,7 @@ def config(ctx): config.add_command(chassis_modules.chassis_modules) config.add_command(console.console) config.add_command(feature.feature) +config.add_command(kdump.kdump) config.add_command(kube.kubernetes) config.add_command(muxcable.muxcable) config.add_command(nat.nat) @@ -1901,50 +1903,6 @@ def shutdown(): """Shut down BGP session(s)""" pass -@config.group(cls=clicommon.AbbreviationGroup) -def kdump(): - """ Configure kdump """ - if os.geteuid() != 0: - exit("Root privileges are required for this operation") - -@kdump.command() -def disable(): - """Disable kdump operation""" - config_db = ConfigDBConnector() - if config_db is not None: - config_db.connect() - config_db.mod_entry("KDUMP", "config", {"enabled": "false"}) - clicommon.run_command("sonic-kdump-config --disable") - -@kdump.command() -def enable(): - """Enable kdump operation""" - config_db = ConfigDBConnector() - if config_db is not None: - config_db.connect() - config_db.mod_entry("KDUMP", "config", {"enabled": "true"}) - clicommon.run_command("sonic-kdump-config --enable") - -@kdump.command() -@click.argument('kdump_memory', metavar='', required=True) -def memory(kdump_memory): - """Set memory allocated for kdump capture kernel""" - config_db = ConfigDBConnector() - if config_db is not None: - config_db.connect() - config_db.mod_entry("KDUMP", "config", {"memory": kdump_memory}) - clicommon.run_command("sonic-kdump-config --memory %s" % kdump_memory) - -@kdump.command('num-dumps') -@click.argument('kdump_num_dumps', metavar='', required=True, type=int) -def num_dumps(kdump_num_dumps): - """Set max number of dump files for kdump""" - config_db = ConfigDBConnector() - if config_db is not None: - config_db.connect() - config_db.mod_entry("KDUMP", "config", {"num_dumps": kdump_num_dumps}) - clicommon.run_command("sonic-kdump-config --num_dumps %d" % kdump_num_dumps) - # 'all' subcommand @shutdown.command() @click.option('-v', '--verbose', is_flag=True, help="Enable verbose output") diff --git a/scripts/reboot b/scripts/reboot index 0d7d400c57d4..23092a56b62f 100755 --- a/scripts/reboot +++ b/scripts/reboot @@ -1,19 +1,28 @@ #!/bin/bash +DEVPATH="/usr/share/sonic/device" +PLAT_REBOOT="platform_reboot" +REBOOT_CAUSE_FILE="/host/reboot-cause/reboot-cause.txt" +REBOOT_TIME=$(date) # Reboot immediately if we run the kdump capture kernel VMCORE_FILE=/proc/vmcore if [ -e $VMCORE_FILE -a -s $VMCORE_FILE ]; then - echo "We have a /proc/vmcore, then we just kdump'ed" - /sbin/reboot + echo "We have a /proc/vmcore, then we just kdump'ed" + echo "User issued 'kdump' command [User: kdump, Time: ${REBOOT_TIME}]" > ${REBOOT_CAUSE_FILE} + sync + PLATFORM=$(grep -oP 'platform=\K\S+' /proc/cmdline) + if [ ! -z "${PLATFORM}" -a -x ${DEVPATH}/${PLATFORM}/${PLAT_REBOOT} ]; then + exec ${DEVPATH}/${PLATFORM}/${PLAT_REBOOT} + fi + # If no platform-specific reboot tool, just run /sbin/reboot + /sbin/reboot + echo 1 > /proc/sys/kernel/sysrq + echo b > /proc/sysrq-trigger fi REBOOT_USER=$(logname) -REBOOT_TIME=$(date) PLATFORM=$(sonic-cfggen -H -v DEVICE_METADATA.localhost.platform) ASIC_TYPE=$(sonic-cfggen -y /etc/sonic/sonic_version.yml -v asic_type) -DEVPATH="/usr/share/sonic/device" -PLAT_REBOOT="platform_reboot" -REBOOT_CAUSE_FILE="/host/reboot-cause/reboot-cause.txt" VERBOSE=no EXIT_NEXT_IMAGE_NOT_EXISTS=4 EXIT_SONIC_INSTALLER_VERIFY_REBOOT=21 diff --git a/scripts/sonic-kdump-config b/scripts/sonic-kdump-config index 0aef37b803bd..7dbde1d7083c 100755 --- a/scripts/sonic-kdump-config +++ b/scripts/sonic-kdump-config @@ -22,11 +22,13 @@ import argparse import shlex import os import subprocess +import json from swsssdk import ConfigDBConnector aboot_cfg_template ="/host/image-%s/kernel-cmdline" grub_cfg = "/host/grub/grub.cfg" kdump_cfg = "/etc/default/kdump-tools" +kdump_mem_file = "/sys/kernel/kexec_crash_size" machine_cfg = "/host/machine.conf" ## Same as print(), but output to stderr instead of stdout @@ -80,6 +82,14 @@ def get_current_image(): print_err("Unable to locate current SONiC image") sys.exit(1) +## Read allocated memory size +def get_crash_kernel_size(): + try: + with open(kdump_mem_file, 'r') as fp: + return fp.read().rstrip('\n') + except Exception as e: + return "0" + ## Search which SONiC image is the Next image def get_next_image(): (rc, img, err_str) = run_command("sonic-installer list | grep 'Next: ' | cut -d '-' -f 3-", use_shell=True); @@ -129,7 +139,7 @@ def search_for_crash_kernel(where): ## Search for "crashkernel=X" in /proc/cmdline # -# @return Return the X from "crashkernel=X" in /proc/cmdline +# @return Return the X from "crashkernel=X" in /proc/cmdline # None in case "crashkernel=" is not found def search_for_crash_kernel_in_cmdline(): try: @@ -139,13 +149,88 @@ def search_for_crash_kernel_in_cmdline(): sys.exit(1) return search_for_crash_kernel(cmdline[0]) +def cmd_dump_db(): + print("Read Kdump configuration from db:") + config_db = ConfigDBConnector(use_unix_socket_path=True) + if config_db is not None: + config_db.connect() + table_data = config_db.get_table('KDUMP') + if table_data is not None: + config_data = table_data.get('config') + if config_data is not None: + print(config_data) + else: + print("empty") + +def cmd_dump_config_json(): + kdump_enabled = get_kdump_administrative_mode() + kdump_memory = get_kdump_memory() + kdump_num_dumps = get_kdump_num_dumps() + data = { "enable" : kdump_enabled, \ + "memory" : kdump_memory, \ + "max-dumps" : int(kdump_num_dumps) } + print(json.dumps(data, indent=4)) + +def cmd_dump_kdump_records_json(): + data = dict() + log_files = dict() + (rc_logs, crash_log_filenames, err_str) = run_command("find /var/crash/ -name 'dmesg.*'", use_shell=False); + if rc_logs == 0: + crash_log_filenames.sort(reverse=True) + for f in crash_log_filenames: + log_files[f[11:23]] = f + + core_files = dict() + (rc_cores, crash_vmcore_filenames, err_str) = run_command("find /var/crash/ -name 'kdump.*'", use_shell=False); + if rc_cores == 0: + crash_vmcore_filenames.sort(reverse=True) + for f in crash_vmcore_filenames: + core_files[f[11:23]] = f + + kdump_records = dict() + for k in sorted(log_files.keys(), reverse=True): + try: + f = open(log_files[k], "r") + log_content = f.read() + f.close() + except Exception as e: + log_content = "" + log_lines = log_content.split("\n")[-100:] + kdump_records[k] = { "id" : k, \ + "vmcore-diagnostic-message" : "\n".join(log_lines), \ + "vmcore-diagnostic-message-file" : log_files[k], \ + "vmcore" : "Kernel vmcore not found" } + + for k in sorted(core_files.keys(), reverse=True): + if kdump_records.get(k): + kdump_records[k]["vmcore"] = core_files[k] + else: + kdump_records[k] = { "id" : k, \ + "vmcore-dmesg-file" : "Kernel crash log not found", \ + "vmcore-dmesg" : "", \ + "vmcore" : core_files[k] } + data["kdump-record"] = kdump_records + print(json.dumps(data, indent=4)) + +def cmd_dump_status_json(): + kdump_enabled = get_kdump_administrative_mode() + kdump_oper_state = get_kdump_oper_mode(kdump_enabled) + kdump_memory = get_kdump_memory() + kdump_num_dumps = get_kdump_num_dumps() + data = { "enable" : kdump_enabled, \ + "current-state" : kdump_oper_state, \ + "memory" : kdump_memory, \ + "allocated-memory" : get_crash_kernel_size(), \ + "max-dumps" : int(kdump_num_dumps) } + print(json.dumps(data, indent=4)) + ## Query current configuration to check if kdump is enabled or disabled # # @return True if kdump is enable, False if kdump is not enabled # We read the running configuration to check if kdump is enabled or not def get_kdump_administrative_mode(): kdump_is_enabled = False - config_db = ConfigDBConnector() + config_db = ConfigDBConnector(use_unix_socket_path=True) if config_db is not None: config_db.connect() table_data = config_db.get_table('KDUMP') @@ -160,37 +245,53 @@ def get_kdump_administrative_mode(): else: return False +def get_kdump_oper_mode(kdump_enabled): + (rc, lines, err_str) = run_command("/usr/sbin/kdump-config status", use_shell=False); + if len(lines) >= 1 and ": ready to kdump" in lines[0]: + use_kdump_in_cfg = read_use_kdump() + if use_kdump_in_cfg: + return('Ready') + else: + return('Not Ready') + elif not kdump_enabled: + return('Disabled') + else: + return('Ready after Reboot') + ## Query current configuration for kdump memory # # @return The current memory string used for kdump (read from running configuration) def get_kdump_memory(): - (rc, lines, err_str) = run_command("show kdump memory", use_shell=False) - try: - if rc == 0 and len(lines) == 1: - p = lines[0].find(': ') - if p != -1: - #print('XXX') - #print(lines[0][p+2:]) - #print('XXX') - return lines[0][p+2:] - except Exception: - pass - return "0M-2G:256M,2G-4G:320M,4G-8G:384M,8G-:448M" + memory = "0M-2G:256M,2G-4G:320M,4G-8G:384M,8G-:448M" + config_db = ConfigDBConnector(use_unix_socket_path=True) + if config_db is not None: + config_db.connect() + table_data = config_db.get_table('KDUMP') + if table_data is not None: + config_data = table_data.get('config') + if config_data is not None: + mem = config_data.get('memory') + if mem: + memory = mem + return memory ## Query current configuration for kdump num_dumps # # @return The maximum number of kernel dump files stored locally # (read from running configuration) def get_kdump_num_dumps(): - (rc, lines, err_str) = run_command("show kdump num_dumps", use_shell=False) - try: - if rc == 0 and len(lines) == 1: - p = lines[0].find(': ') - if p != -1: - return int(lines[0][p+2:]) - except Exception: - pass - return 3 + num_dumps = 3 + config_db = ConfigDBConnector(use_unix_socket_path=True) + if config_db is not None: + config_db.connect() + table_data = config_db.get_table('KDUMP') + if table_data is not None: + config_data = table_data.get('config') + if config_data is not None: + num = config_data.get('num_dumps') + if num: + num_dumps = num + return num_dumps ## Read current value for USE_KDUMP in kdump config file # @@ -214,6 +315,11 @@ def write_use_kdump(use_kdump): (rc, lines, err_str) = run_command("/bin/sed -i -e 's/USE_KDUMP=.*/USE_KDUMP=%s/' %s" % (use_kdump, kdump_cfg), use_shell=False); if rc == 0 and type(lines) == list and len(lines) == 0: use_kdump_in_cfg = read_use_kdump() + if use_kdump == 0: + (rc, lines, err_str) = run_command("/usr/sbin/kdump-config unload", use_shell=False) + if rc != 0: + print_err("Error Unable to unload the Kdump kernel '%s'", err_str) + sys.exit(1) if use_kdump_in_cfg != use_kdump: print_err("Unable to write USE_KDUMP into %s" % kdump_cfg) sys.exit(1) @@ -250,6 +356,39 @@ def write_num_dumps(num_dumps): print_err("Error while writing KDUMP_NUM_DUMPS into %s" % kdump_cfg) sys.exit(1) +## Save kdump configuration into the startup configuration +# @kdump_enabled Administrative mode (False/True) +# @memory Amount of memory allocated for the capture kernel +# @num_dumps Max number of core files saved locally +def save_config(kdump_enabled, memory, num_dumps): + + configdb_fname = '/etc/sonic/config_db.json' + + # Read current configuration + if not os.path.exists(configdb_fname): + print_err("Startup configuration not found, Kdump configuration is not saved") + return + else: + try: + with open(configdb_fname) as json_file: + data = json.load(json_file) + except Exception as e: + print_err("Error [%s] while reading startup configuration" % e) + return + + # Rewrite configuration + try: + kdump_data = {'config': {'enabled': '', 'num_dumps': '', 'memory': ''}} + (kdump_data['config'])['enabled'] = str(kdump_enabled).lower() + (kdump_data['config'])['num_dumps'] = str(num_dumps) + (kdump_data['config'])['memory'] = memory + data['KDUMP'] = kdump_data + with open(configdb_fname, 'w') as fp: + json.dump(data, fp, indent=4, sort_keys=False) + print("Kdump configuration has been updated in the startup configuration") + except Exception as e: + print_err("Error [%s] while saving Kdump configuration to startup configuration" % e) + ## Enable kdump # # @param verbose If True, the function will display a few additinal information @@ -293,11 +432,42 @@ def kdump_enable(verbose, kdump_enabled, memory, num_dumps, image, cmdline_file) if changed: rewrite_cfg(lines, cmdline_file) + save_config(kdump_enabled, memory, num_dumps) + else: + save_config(kdump_enabled, memory, num_dumps) write_use_kdump(1) + if crash_kernel_in_cmdline is not None: + (rc, lines, err_str) = run_command("/usr/sbin/kdump-config load", use_shell=False) + if rc != 0: + print_err("Error Unable to unload Kdump the kernel '%s'", err_str) + sys.exit(1) return changed +## Read kdump configuration saved in the startup configuration file +# +# @param config_param If True, the function will display a few additional information +# @return Value of the configuration parameter saved in the startup configuration file +# @return None if the startup configuration file does not exist or the kdump +# configuration parameter is not present in the file. +def get_kdump_config_json(config_param): + configdb_fname = '/etc/sonic/config_db.json' + + # Read the startup configuration file + if not os.path.exists(configdb_fname): + return None + else: + try: + with open(configdb_fname) as json_file: + data = json.load(json_file) + if data.get("KDUMP") is not None and \ + data.get("KDUMP").get("config") is not None: + return data.get("KDUMP").get("config").get(config_param) + except Exception as e: + print_err("Error [%s] while reading startup configuration" % e) + return None + ## Command: Enable kdump # # @param verbose If True, the function will display a few additinal information @@ -353,6 +523,30 @@ def kdump_disable(verbose, kdump_enabled, memory, num_dumps, image, cmdline_file if changed: rewrite_cfg(lines, cmdline_file) + if not os.path.exists('/etc/sonic/config_db.json'): + print_err("Startup configuration not found, Kdump configuration is not saved") + return False + + current_img = get_current_image(); + if verbose: + print("Current image=[%s]\n" % current_img) + lines = [line.rstrip('\n') for line in open(grub_cfg)] + current_img_index = locate_image(lines, "loop=image-"+current_img) + + changed = False + curr_crash_kernel_mem = search_for_crash_kernel(lines[current_img_index]) + if curr_crash_kernel_mem is None: + print("Kdump is already disabled") + else: + lines[current_img_index] = lines[current_img_index].replace("crashkernel="+curr_crash_kernel_mem, "") + changed = True + if verbose: + print("Removed [%s] in grub.cfg" % ("crashkernel="+curr_crash_kernel_mem)) + + if changed: + rewrite_grub_cfg(lines, grub_cfg) + save_config(kdump_enabled, memory, num_dumps) + return changed ## Command: Disable kdump @@ -379,7 +573,7 @@ def cmd_kdump_disable(verbose, image=get_current_image()): ## Command: Set / Get memory # # @param verbose If True, the function will display a few additional information -# @param memory If not None, new value to set. +# @param memory If not None, new value to set. # If None, display current value read from running configuration def cmd_kdump_memory(verbose, memory): if memory is None: @@ -389,14 +583,19 @@ def cmd_kdump_memory(verbose, memory): use_kdump_in_cfg = read_use_kdump() if use_kdump_in_cfg: crash_kernel_in_cmdline = search_for_crash_kernel_in_cmdline() - if memory != crash_kernel_in_cmdline: - cmd_kdump_enable(verbose, False) - print("kdump updated memory will be only operational after the system reboots") + memory_in_db = get_kdump_memory() + memory_in_json = get_kdump_config_json("memory") + if memory != crash_kernel_in_cmdline or memory != memory_in_db or memory != memory_in_json: + cmd_kdump_enable(verbose) + print("Kdump updated memory will be only operational after the system reboots") + else: + num_dumps = get_kdump_num_dumps() + save_config(False, memory, num_dumps) ## Command: Set / Get num_dumps # # @param verbose If True, the function will display a few additional information -# @param memory If not None, new value to set. +# @param memory If not None, new value to set. # If None, display current value read from running configuration def cmd_kdump_num_dumps(verbose, num_dumps): if num_dumps is None: @@ -404,6 +603,9 @@ def cmd_kdump_num_dumps(verbose, num_dumps): print('\n'.join(lines)) else: write_num_dumps(num_dumps) + kdump_enabled = get_kdump_administrative_mode() + kdump_memory = get_kdump_memory() + save_config(kdump_enabled, kdump_memory, num_dumps) ## Command: Display kdump status def cmd_kdump_status(): @@ -459,7 +661,7 @@ def cmd_kdump_file(num_lines, filename): fname = None nb_dumps = get_nb_dumps_in_var_crash() if nb_dumps == 0: - print("There is no kernel core file stored") + print("Kernel crash log not found") else: (rc, lines, err_str) = run_command("find /var/crash/ -name 'dmesg.*'", use_shell=False); if rc == 0 and nb_dumps == len(lines): @@ -471,7 +673,7 @@ def cmd_kdump_file(num_lines, filename): else: print("Invalid record number - Should be between 1 and %d" % nb_dumps) sys.exit(1) - fname = lines[num-1] + fname = sorted(lines, reverse=True)[num-1] else: lines.sort(reverse=True) for x in lines: @@ -496,6 +698,10 @@ def main(): parser = argparse.ArgumentParser(description="kdump configuration and status tool", formatter_class=argparse.RawTextHelpFormatter) + # Dump kdump db configuration + parser.add_argument('--dump-db', action='store_true', + help='Dump kdump db configuration') + # Enable kdump on Current image parser.add_argument('--enable', action='store_true', help='Enable kdump (Current image)') @@ -512,6 +718,19 @@ def main(): parser.add_argument('--status', action='store_true', help='Show kdump status') + # kdump status on Current Image + parser.add_argument('--status-json', action='store_true', + help='Show kdump status in json format') + + # kdump status on Current Image + parser.add_argument('--kdump-records-json', action='store_true', + help='Show kdump records in json format') + + # kdump config on Current Image + # kdump config on Current Image + parser.add_argument('--config-json', action='store_true', + help='Show kdump config in json format') + # Maximum number of kernel core dumps parser.add_argument('--num_dumps', nargs='?', type=int, action='store', default=False, help='Maximum number of kernel dump files stored') @@ -563,6 +782,14 @@ def main(): cmd_kdump_files() elif options.file: cmd_kdump_file(options.lines, options.file[0]) + elif options.dump_db: + cmd_dump_db() + elif options.status_json: + cmd_dump_status_json() + elif options.config_json: + cmd_dump_config_json() + elif options.kdump_records_json: + cmd_dump_kdump_records_json() else: parser.print_help() sys.exit(1) diff --git a/show/kdump.py b/show/kdump.py index 871073689116..90391628365b 100644 --- a/show/kdump.py +++ b/show/kdump.py @@ -2,16 +2,14 @@ import utilities_common.cli as clicommon from swsssdk import ConfigDBConnector - # # 'kdump command ("show kdump ...") # -@click.group(cls=clicommon.AliasedGroup) +@click.group(cls=clicommon.AliasedGroup, name="kdump") def kdump(): """Show kdump configuration, status and information """ pass - @kdump.command('enabled') def enabled(): """Show if kdump is enabled or disabled""" @@ -30,7 +28,6 @@ def enabled(): else: click.echo("kdump is disabled") - @kdump.command('status') def status(): """Show kdump status""" @@ -39,7 +36,6 @@ def status(): clicommon.run_command("sonic-kdump-config --num_dumps") clicommon.run_command("sonic-kdump-config --files") - @kdump.command('memory') def memory(): """Show kdump memory information""" @@ -56,7 +52,6 @@ def memory(): kdump_memory = kdump_memory_from_db click.echo("Memory Reserved: {}".format(kdump_memory)) - @kdump.command('num_dumps') def num_dumps(): """Show kdump max number of dump files""" @@ -73,20 +68,17 @@ def num_dumps(): kdump_num_dumps = kdump_num_dumps_from_db click.echo("Maximum number of Kernel Core files Stored: {}".format(kdump_num_dumps)) - @kdump.command('files') def files(): """Show kdump kernel core dump files""" clicommon.run_command("sonic-kdump-config --files") - @kdump.command() @click.argument('record', required=True) @click.argument('lines', metavar='', required=False) def log(record, lines): """Show kdump kernel core dump file kernel log""" cmd = "sonic-kdump-config --file {}".format(record) - if lines is not None: cmd += " --lines {}".format(lines) diff --git a/show/main.py b/show/main.py index 60a3e6b683c0..d163ac930a59 100755 --- a/show/main.py +++ b/show/main.py @@ -143,6 +143,7 @@ def cli(ctx): cli.add_command(dropcounters.dropcounters) cli.add_command(feature.feature) cli.add_command(fgnhg.fgnhg) +cli.add_command(kdump.kdump) cli.add_command(interfaces.interfaces) cli.add_command(kdump.kdump) cli.add_command(kube.kubernetes)