From 548896a38dd083b29c23d0522f12b950b4c7677d Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Thu, 22 Apr 2021 10:52:55 +0000 Subject: [PATCH 01/76] Worked version Signed-off-by: Vadym Hlushko --- sonic_cli_gen/__init__.py | 0 sonic_cli_gen/generator.py | 31 +++++++++++++++++++++++++++++++ sonic_cli_gen/main.py | 27 +++++++++++++++++++++++++++ sonic_cli_gen/yang_parser.py | 36 ++++++++++++++++++++++++++++++++++++ 4 files changed, 94 insertions(+) create mode 100644 sonic_cli_gen/__init__.py create mode 100644 sonic_cli_gen/generator.py create mode 100644 sonic_cli_gen/main.py create mode 100644 sonic_cli_gen/yang_parser.py diff --git a/sonic_cli_gen/__init__.py b/sonic_cli_gen/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py new file mode 100644 index 0000000000..400ba41a47 --- /dev/null +++ b/sonic_cli_gen/generator.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python + +try: + from sonic_cli_gen.yang_parser import YangParser +except ImportError as e: + raise ImportError("%s - required module not found" % str(e)) + +class CliGenerator: + """ SONiC CLI generator. This class provides public API + for sonic-cli-gen python library. It can generate config, + show, sonic-clear commands """ + def __init__(self, + yang_model): + """ Initialize PackageManager. """ + + self.yang_model_name = yang_model + + def generate_config_plugin(self): + parser = YangParser(self.yang_model_name) + parser.yang_to_dict() + pass + + #TODO + def generate_show_plugin(self): + print ("show") + pass + + # to be implemented in the next Phases + def generate_sonic_clear_plugin(self): + print ("sonic-clear") + pass diff --git a/sonic_cli_gen/main.py b/sonic_cli_gen/main.py new file mode 100644 index 0000000000..542ad91867 --- /dev/null +++ b/sonic_cli_gen/main.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python + +try: + import sys + import os + import click + from sonic_cli_gen.generator import CliGenerator +except ImportError as e: + raise ImportError("%s - required module not found" % str(e)) + +@click.group() +@click.pass_context +def cli(ctx): + """ SONiC CLI generator """ + print ("cli") + pass + +@cli.command() +@click.pass_context +def generate_config(ctx): + """ List available packages """ + gen = CliGenerator('sonic-vlan.yang') + gen.generate_config_plugin() + pass + +if __name__ == '__main__': + cli() \ No newline at end of file diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py new file mode 100644 index 0000000000..50740b1f10 --- /dev/null +++ b/sonic_cli_gen/yang_parser.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python + +try: + import os + import sys + import glob + import yang as ly +except ImportError as e: + raise ImportError("%s - required module not found" % str(e)) + + +YANG_DIR = "/usr/local/yang-models/" + +class YangParser: + """ YANG model parser """ + def __init__(self, + yang_model): + self.yang_model = yang_model + self.ly_ctx = None + + try: + self.ly_ctx = ly.Context(YANG_DIR) + except Exception as e: + self.fail(e) + + def fail(self, e): + print(e) + raise e + + def yang_to_dict(self): + print ("YANG TO DICT") + data = { + 'yang_dir': YANG_DIR. + 'yang_files': glob + } + pass \ No newline at end of file From 3350f27807b6ff2dcd609a2b444bd2be2b865f0f Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Fri, 23 Apr 2021 13:55:06 +0000 Subject: [PATCH 02/76] added stub for function to determine static or dynamic YANG Signed-off-by: Vadym Hlushko --- setup.py | 2 ++ sonic_cli_gen/main.py | 2 +- sonic_cli_gen/yang_parser.py | 31 ++++++++++++++++++------------- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/setup.py b/setup.py index cd706eb433..9aff1ac9c2 100644 --- a/setup.py +++ b/setup.py @@ -52,6 +52,7 @@ 'undebug', 'utilities_common', 'watchdogutil', + 'sonic_cli_gen', ], package_data={ 'show': ['aliases.ini'], @@ -153,6 +154,7 @@ 'sonic_installer = sonic_installer.main:sonic_installer', # Deprecated 'undebug = undebug.main:cli', 'watchdogutil = watchdogutil.main:watchdogutil', + 'sonic-cli-gen = sonic_cli_gen.main:cli', ] }, install_requires=[ diff --git a/sonic_cli_gen/main.py b/sonic_cli_gen/main.py index 542ad91867..a8b01a7034 100644 --- a/sonic_cli_gen/main.py +++ b/sonic_cli_gen/main.py @@ -19,7 +19,7 @@ def cli(ctx): @click.pass_context def generate_config(ctx): """ List available packages """ - gen = CliGenerator('sonic-vlan.yang') + gen = CliGenerator('sonic-vlan') gen.generate_config_plugin() pass diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index 50740b1f10..dfaec45e43 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -3,34 +3,39 @@ try: import os import sys - import glob - import yang as ly + from config.config_mgmt import ConfigMgmt except ImportError as e: raise ImportError("%s - required module not found" % str(e)) -YANG_DIR = "/usr/local/yang-models/" - class YangParser: """ YANG model parser """ def __init__(self, yang_model): self.yang_model = yang_model - self.ly_ctx = None + self.conf_mgmt = None try: - self.ly_ctx = ly.Context(YANG_DIR) + self.conf_mgmt = ConfigMgmt() except Exception as e: - self.fail(e) + raise Exception("Failed to load the {} class".format(str(e))) def fail(self, e): print(e) raise e def yang_to_dict(self): - print ("YANG TO DICT") - data = { - 'yang_dir': YANG_DIR. - 'yang_files': glob - } - pass \ No newline at end of file + yang_model_type = self._determine_yang_model_type() + + if (yang_model_type == 'static'): + print('static') + pass + else: + pass + + def _determine_yang_model_type(self): + cond = True + if cond: + return 'static' + else: + return 'dynamic' \ No newline at end of file From 1d1643ac298bbd515e88d0e44d4bdce9ee6bcfde Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Mon, 26 Apr 2021 09:26:09 +0000 Subject: [PATCH 03/76] _find_index_of_yang_model() Signed-off-by: Vadym Hlushko --- sonic_cli_gen/yang_parser.py | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index dfaec45e43..ef58a8f0f6 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -3,23 +3,27 @@ try: import os import sys + import pprint from config.config_mgmt import ConfigMgmt except ImportError as e: raise ImportError("%s - required module not found" % str(e)) +# Config DB schema view +STATIC_TABLE = 'static' +LIST_TABLE = 'list' class YangParser: """ YANG model parser """ def __init__(self, - yang_model): - self.yang_model = yang_model + yang_model_name): + self.yang_model_name = yang_model_name self.conf_mgmt = None try: self.conf_mgmt = ConfigMgmt() except Exception as e: raise Exception("Failed to load the {} class".format(str(e))) - + def fail(self, e): print(e) raise e @@ -27,15 +31,16 @@ def fail(self, e): def yang_to_dict(self): yang_model_type = self._determine_yang_model_type() - if (yang_model_type == 'static'): - print('static') - pass - else: - pass - def _determine_yang_model_type(self): - cond = True - if cond: - return 'static' - else: - return 'dynamic' \ No newline at end of file + y_index = self._find_index_of_yang_model() + print("INDEX {}".format(y_index)) + + def _find_index_of_yang_model(self): + for i in range(len(self.conf_mgmt.sy.yJson)): + if (self.conf_mgmt.sy.yJson[i]['module']['@name'] == self.yang_model_name): + return i + + + + + \ No newline at end of file From cd31f0d307b28aa8a8548a1a55d1050b8a3a6de1 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Mon, 26 Apr 2021 14:05:52 +0000 Subject: [PATCH 04/76] added function to init list's for module, top, tables Signed-off-by: Vadym Hlushko --- sonic_cli_gen/generator.py | 2 +- sonic_cli_gen/yang_parser.py | 29 +++++++++++++++++++++++------ 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py index 400ba41a47..154d82409b 100644 --- a/sonic_cli_gen/generator.py +++ b/sonic_cli_gen/generator.py @@ -17,7 +17,7 @@ def __init__(self, def generate_config_plugin(self): parser = YangParser(self.yang_model_name) - parser.yang_to_dict() + parser.parse_yang_model() pass #TODO diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index ef58a8f0f6..c211f9e948 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -18,6 +18,11 @@ def __init__(self, yang_model_name): self.yang_model_name = yang_model_name self.conf_mgmt = None + #index of yang model inside conf_mgmt.sy.yJson object + self.idx_yJson = None + self.y_module = None + self.y_top_level_container = None + self.y_tables = None try: self.conf_mgmt = ConfigMgmt() @@ -28,17 +33,29 @@ def fail(self, e): print(e) raise e - def yang_to_dict(self): - yang_model_type = self._determine_yang_model_type() + def parse_yang_model(self): + self._init_yang_module_and_containers() - def _determine_yang_model_type(self): - y_index = self._find_index_of_yang_model() - print("INDEX {}".format(y_index)) + def _determine_tables_type(self): + #for table in y_top_level_container['container']: + pass + def _init_yang_module_and_containers(self): + self._find_index_of_yang_model() + + self.y_module = self.conf_mgmt.sy.yJson[self.idx_yJson]['module'] + if self.y_module.get('container') is not None: + self.y_top_level_container = self.y_module['container'] + self.y_tables = self.y_top_level_container['container'] + import pdb; pdb.set_trace() + else: + raise KeyError('YANG model {} does NOT have "container" element'.format(self.yang_model_name)) + + # find index of yang_model inside yJson object def _find_index_of_yang_model(self): for i in range(len(self.conf_mgmt.sy.yJson)): if (self.conf_mgmt.sy.yJson[i]['module']['@name'] == self.yang_model_name): - return i + self.idx_yJson = i From 81b803de3f144105d46057a92a9bdb81596109dd Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Mon, 26 Apr 2021 15:32:05 +0000 Subject: [PATCH 05/76] Added func to determine - static or list, +comments Signed-off-by: Vadym Hlushko --- sonic_cli_gen/yang_parser.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index c211f9e948..77f9ded716 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -4,25 +4,27 @@ import os import sys import pprint + from collections import OrderedDict from config.config_mgmt import ConfigMgmt except ImportError as e: raise ImportError("%s - required module not found" % str(e)) -# Config DB schema view -STATIC_TABLE = 'static' -LIST_TABLE = 'list' - class YangParser: """ YANG model parser """ def __init__(self, yang_model_name): self.yang_model_name = yang_model_name self.conf_mgmt = None - #index of yang model inside conf_mgmt.sy.yJson object + # index of yang model inside conf_mgmt.sy.yJson object self.idx_yJson = None - self.y_module = None - self.y_top_level_container = None - self.y_tables = None + # 'module' entity from .yang file + self.y_module = OrderedDict() + # top level 'container' entity from .yang file + self.y_top_level_container = OrderedDict() + # 'container' entities from .yang file + self.y_tables = list() + # dictionary that represent Config DB schema + self.yang_2_dict = OrderedDict() try: self.conf_mgmt = ConfigMgmt() @@ -36,9 +38,14 @@ def fail(self, e): def parse_yang_model(self): self._init_yang_module_and_containers() + self._determine_tables_type() + def _determine_tables_type(self): - #for table in y_top_level_container['container']: - pass + for table in self.y_tables: + if table.get('list') is None: + self.yang_2_dict[table.get('@name')] = {'type': 'static'} + else: + self.yang_2_dict[table.get('@name')] = {'type': 'list'} def _init_yang_module_and_containers(self): self._find_index_of_yang_model() @@ -47,7 +54,6 @@ def _init_yang_module_and_containers(self): if self.y_module.get('container') is not None: self.y_top_level_container = self.y_module['container'] self.y_tables = self.y_top_level_container['container'] - import pdb; pdb.set_trace() else: raise KeyError('YANG model {} does NOT have "container" element'.format(self.yang_model_name)) From ecdf8b6a58a9f1b44a3076d3a1a2a3a9300f8c54 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Wed, 28 Apr 2021 10:07:08 +0000 Subject: [PATCH 06/76] Added auto compleation file Signed-off-by: Vadym Hlushko --- sonic-utilities-data/bash_completion.d/sonic-cli-gen | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 sonic-utilities-data/bash_completion.d/sonic-cli-gen diff --git a/sonic-utilities-data/bash_completion.d/sonic-cli-gen b/sonic-utilities-data/bash_completion.d/sonic-cli-gen new file mode 100644 index 0000000000..3327f9c513 --- /dev/null +++ b/sonic-utilities-data/bash_completion.d/sonic-cli-gen @@ -0,0 +1,8 @@ +_sonic_cli_gen_completion() { + COMPREPLY=( $( env COMP_WORDS="${COMP_WORDS[*]}" \ + COMP_CWORD=$COMP_CWORD \ + _SONIC_CLI_GEN_COMPLETE=complete $1 ) ) + return 0 +} + +complete -F _sonic_cli_gen_completion -o default sonic-cli-gen; From 42779c7d3ef336988483c64688b224dae6017fb2 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Wed, 28 Apr 2021 18:12:36 +0300 Subject: [PATCH 07/76] Add CLI Jinja templates Signed-off-by: Stepan Blyschak --- sonic-utilities-data/debian/install | 5 +- .../templates/sonic-cli-gen/common.j2 | 11 + .../templates/sonic-cli-gen/config.py.j2 | 352 ++++++++++++++++++ .../templates/sonic-cli-gen/show.py.j2 | 93 +++++ 4 files changed, 459 insertions(+), 2 deletions(-) create mode 100644 sonic-utilities-data/templates/sonic-cli-gen/common.j2 create mode 100644 sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 create mode 100644 sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 diff --git a/sonic-utilities-data/debian/install b/sonic-utilities-data/debian/install index 82d087d54d..1f67b78c20 100644 --- a/sonic-utilities-data/debian/install +++ b/sonic-utilities-data/debian/install @@ -1,2 +1,3 @@ -bash_completion.d/ /etc/ -templates/*.j2 /usr/share/sonic/templates/ +bash_completion.d/ /etc/ +templates/*.j2 /usr/share/sonic/templates/ +templates/sonic-cli-gen/*.j2 /usr/share/sonic/templates/sonic-cli-gen/ diff --git a/sonic-utilities-data/templates/sonic-cli-gen/common.j2 b/sonic-utilities-data/templates/sonic-cli-gen/common.j2 new file mode 100644 index 0000000000..51ebaa004c --- /dev/null +++ b/sonic-utilities-data/templates/sonic-cli-gen/common.j2 @@ -0,0 +1,11 @@ +{% macro make_cli_name(name) -%} +{{ name|lower|replace("_", "-") }} +{%- endmacro %} + +{% macro key_converter(keys) %} +{%- if keys|length > 1 %} + ({{ keys|map(attribute="name")|join(",") }}) +{%- else %} + {{ keys|map(attribute="name")|first }} +{%- endif %} +{% endmacro %} diff --git a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 new file mode 100644 index 0000000000..8536d33303 --- /dev/null +++ b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 @@ -0,0 +1,352 @@ +{%- from "common.j2" import make_cli_name, key_converter -%} +""" Autogenerated config CLI plugin """ + +import click +from config import config_mgmt +import utilities_common.cli as clicommon + + +def exit_cli(*args, **kwargs): + """ Print a message and abort CLI. """ + + click.secho(*args, **kwargs) + raise click.Abort() + +def validate_config_or_raise(cfg): + """ Validate config db data using ConfigMgmt """ + + try: + config_mgmt.ConfigMgmt().loadData(cfg) + except Exception as err: + raise Exception('Failed to validate configuration: {}'.format(err)) + + +def mod_entry_validated(db, table, key, data): + """ Modify existing entry and validate configuration """ + + cfg = db.get_config() + cfg.setdefault(table, {}) + cfg[table].setdefault(key, {}) + cfg[table][key].update(data) + + validate_config_or_raise(cfg) + db.mod_entry(table, key, data) + + +def add_entry_validated(db, table, key, data): + """ Add new entry in table and validate configuration""" + + cfg = db.get_config() + cfg.setdefault(table, {}) + if key in cfg[table]: + raise Exception(f"{key} already exists") + cfg[table][key] = data + + validate_config_or_raise(cfg) + db.set_entry(table, key, data) + + +def update_entry_validated(db, table, key, data): + """ Update entry in table and validate configuration""" + + cfg = db.get_config() + cfg.setdefault(table, {}) + if key not in cfg[table]: + raise Exception(f"{key} does not exist") + cfg[table][key].update(data) + + validate_config_or_raise(cfg) + db.mod_entry(table, key, data) + + +def del_entry_validated(db, table, key): + """ Delete entry in table and validate configuration """ + + cfg = db.get_config() + cfg.setdefault(table, {}) + if key not in cfg[table]: + raise Exception(f"{key} does not exist") + cfg[table].pop(key) + + validate_config_or_raise(cfg) + db.set_entry(table, key, None) + + +def add_list_entry_validated(db, table, key, attr, data): + """ Add new entry into list in table and validate configuration""" + + cfg = db.get_config() + cfg.setdefault(table, {}) + if key not in cfg[table]: + raise Exception(f"{key} does not exist") + cfg[table][key].setdefault(attr, []) + for entry in data: + if entry in cfg[table][key][attr]: + raise Exception(f"{entry} already exists") + cfg[table][key][attr].append(entry) + + validate_config_or_raise(cfg) + db.set_entry(table, key, {attr: cfg[table][key][attr]}) + + +def del_list_entry_validated(db, table, key, attr, data): + """ Delete entry from list in table and validate configuration""" + + cfg = db.get_config() + cfg.setdefault(table, {}) + if key not in cfg[table]: + raise Exception(f"{key} does not exist") + cfg[table][key].setdefault(attr, []) + for entry in data: + if entry not in cfg[table][key][attr]: + raise Exception(f"{entry} does not exist") + cfg[table][key][attr].remove(entry) + + validate_config_or_raise(cfg) + db.set_entry(table, key, {attr: cfg[table][key][attr]}) + + +{% macro config_object_list_update(table, object, list) %} +@{{ table.name }}.group(name="{{ make_cli_name(list.name) }}") +def {{ table.name }}_{{ list.name }}(): + """ Add/Remove {{ list.name }} in {{ table.name }} """ + + pass + +@{{ table.name }}_{{ list.name }}.command(name="add") +{%- for key in object["keys"] %} +@click.argument("{{ make_cli_name(key.name) }}") +{%- endfor %} +@click.argument("{{ make_cli_name(list.name) }}", nargs=-1) +@clicommon.pass_db +def {{ table.name }}_{{ list.name }}_add(db, + {{ object["keys"]|map(attribute="name")|join(",") }}, + {{ list.name|lower }}): + """ Add {{ list.name }} in {{ table.name }} """ + + table = "{{ table.name }}" + key = {{ key_converter(object["keys"]) }} + attr = "{{ list.name }}" + data = {{ list.name|lower }} + + try: + add_list_entry_validated(db.cfgdb, table, key, attr, data) + except Exception as err: + exit_cli(f"Error: {err}", fg="red") + +@{{ table.name }}_{{ list.name }}.command(name="remove") +{%- for key in object["keys"] %} +@click.argument("{{ make_cli_name(key.name) }}") +{%- endfor %} +@click.argument("{{ make_cli_name(list.name) }}", nargs=-1) +@clicommon.pass_db +def {{ table.name }}_{{ list.name }}_remove(db, + {{ object["keys"]|map(attribute="name")|join(",") }}, + {{ list.name|lower }}): + """ Remove {{ list.name }} in {{ table.name }} """ + + table = "{{ table.name }}" + key = {{ key_converter(object["keys"]) }} + attr = "{{ list.name }}" + data = {{ list.name|lower }} + + try: + del_list_entry_validated(db.cfgdb, table, key, attr, data) + except Exception as err: + exit_cli(f"Error: {err}", fg="red") +{% endmacro %} + + +{% macro config_object_list_update_all(table, object) %} +{% for list in object.lists %} +{{ config_object_list_update(table, object, list) }} +{% endfor %} +{% endmacro %} + + +{% macro config_static_object_attr(table, object, attr) %} +@{{ table.name }}_{{ object.name }}.command(name="{{ make_cli_name(attr.name) }}") +@click.argument("{{ make_cli_name(attr.name) }}") +@clicommon.pass_db +def {{ table.name }}_{{ object.name }}_{{ attr.name }}(db, {{ attr.name|lower }}): + """ {{ attr.description|default("") }} """ + + table = "{{ table.name }}" + key = "{{ object.name }}" + data = { + "{{ attr.name }}": {{ attr.name|lower }}, + } + try: + mod_entry_validated(db.cfgdb, table, key, data) + except Exception as err: + exit_cli(f"Error: {err}", fg="red") +{% endmacro %} + + +{# Static objects config CLI generation +E.g: + @TABLE.group(name="object") + def TABLE_object(db): +#} +{% macro config_static_object(table, object) %} +@{{ table.name }}.group(name="{{ make_cli_name(object.name) }}") +@clicommon.pass_db +def {{ table.name }}_{{ object.name }}(db): + """ {{ object.description|default("") }} """ + + pass + +{# Static objects attributes config CLI generation +E.g: + @TABLE_object.command(name="attribute") + def TABLE_object_attribute(db, attribute): +#} +{% for attr in object.attrs %} +{{ config_static_object_attr(table, object, attr) }} +{% endfor %} + +{{ config_object_list_update_all(table, object) }} +{% endmacro %} + +{# Dynamic objects config CLI generation #} + +{# Dynamic objects add command +E.g: + @TABLE.command(name="add") + @click.argument("key1") + @click.argument("key2") + @click.option("--attr1") + @click.option("--attr2") + @click.option("--attr3") + def TABLE_object_add(db, key1, key2, attr1, attr2, attr3): +#} +{% macro config_dynamic_object_add(table, object) %} +@{{ table.name }}.command(name="add") +{%- for key in object["keys"] %} +@click.argument("{{ make_cli_name(key.name) }}") +{%- endfor %} +{%- for attr in object.attrs + object.lists %} +@click.option("--{{ make_cli_name(attr.name) }}") +{%- endfor %} +@clicommon.pass_db +def {{ table.name }}_add(db, + {{ object["keys"]|map(attribute="name")|join(",") }}, + {{ (object.attrs + object.lists)|map(attribute="name")|join(",") }}): + """ Add object in {{ table.name }}. """ + + table = "{{ table.name }}" + key = {{ key_converter(object["keys"]) }} + data = {} +{%- for attr in object.attrs %} + if {{ attr.name|lower }} is not None: + data["{{ attr.name }}"] = {{ attr.name|lower }} +{%- endfor %} +{%- for list in object.lists %} + if {{ list.name|lower }} is not None: + data["{{ list.name }}"] = {{ list.name|lower }}.split(",") +{%- endfor %} + + try: + add_entry_validated(db.cfgdb, table, key, data) + except Exception as err: + exit_cli(f"Error: {err}", fg="red") +{% endmacro %} + +{# Dynamic objects update command +E.g: + @TABLE.command(name="update") + @click.argument("key1") + @click.argument("key2") + @click.option("--attr1") + @click.option("--attr2") + @click.option("--attr3") + def TABLE_object_update(db, key1, key2, attr1, attr2, attr3): +#} +{% macro config_dynamic_object_update(table, object) %} +@{{ table.name }}.command(name="update") +{%- for key in object["keys"] %} +@click.argument("{{ make_cli_name(key.name) }}") +{%- endfor %} +{%- for attr in object.attrs + object.lists %} +@click.option("--{{ make_cli_name(attr.name) }}") +{%- endfor %} +@clicommon.pass_db +def {{ table.name }}_update(db, + {{ object["keys"]|map(attribute="name")|join(",") }}, + {{ (object.attrs + object.lists)|map(attribute="name")|join(",") }}): + """ Add object in {{ table.name }}. """ + + table = "{{ table.name }}" + key = {{ key_converter(object["keys"]) }} + data = {} +{%- for attr in object.attrs %} + if {{ attr.name|lower }} is not None: + data["{{ attr.name }}"] = {{ attr.name|lower }} +{%- endfor %} +{%- for list in object.lists %} + if {{ list.name|lower }} is not None: + data["{{ list.name }}"] = {{ list.name|lower }}.split(",") +{%- endfor %} + + try: + update_entry_validated(db.cfgdb, table, key, data) + except Exception as err: + exit_cli(f"Error: {err}", fg="red") +{% endmacro %} + +{# Dynamic objects delete command +E.g: + @TABLE.command(name="delete") + @click.argument("key1") + @click.argument("key2") + def TABLE_object_add(db, key1, key2): +#} +{% macro config_dynamic_object_delete(table, object) %} +@{{ table.name }}.command(name="delete") +{%- for key in object["keys"] %} +@click.argument("{{ make_cli_name(key.name) }}") +{%- endfor %} +@clicommon.pass_db +def {{ table.name }}_add(db, + {{ object["keys"]|map(attribute="name")|join(",") }}): + """ Delete object in {{ table.name }}. """ + + table = "{{ table.name }}" + key = {{ key_converter(object["keys"]) }} + try: + del_entry_validated(db.cfg, table, key) + except Exception as err: + exit_cli(f"Error: {err}", fg="red") +{% endmacro %} + +{% macro config_dynamic_object(table, object) %} +{{ config_dynamic_object_add(table, object) }} +{{ config_dynamic_object_update(table, object) }} +{{ config_dynamic_object_delete(table, object) }} +{{ config_object_list_update_all(table, object) }} +{% endmacro %} + + +{% for table in tables %} +@click.group(name="{{ make_cli_name(table.name) }}", + cls=clicommon.AliasedGroup) +def {{ table.name }}(): + """ {{ table.description|default("") }} """ + + pass + +{% if "static-objects" in table %} +{% for object in table["static-objects"] %} +{{ config_static_object(table, object) }} +{% endfor %} +{% elif "dynamic-objects" in table %} +{% for object in table["dynamic-objects"] %} +{{ config_dynamic_object(table, object) }} +{% endfor %} +{% endif %} +{% endfor %} + +def register(cli): +{%- for table in tables %} + cli.add_command({{ table.name }}) +{%- endfor %} diff --git a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 new file mode 100644 index 0000000000..84f76fea2f --- /dev/null +++ b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 @@ -0,0 +1,93 @@ +import click +import tabulate +from utilities_common.db import Db +import utilities_common.cli as clicommon + +{% from "common.j2" import make_cli_name %} + +{% for table in tables %} +{% if "static-objects" in table %} +@click.group(name="{{ make_cli_name(table.name) }}", cls=clicommon.AliasedGroup) +def {{ table.name }}(): + """ {{ table.description|default('') }} """ + + pass + +{% for object in table["static-objects"] %} +@{{ table.name }}.command(name="{{ make_cli_name(object.name) }}") +@clicommon.pass_db +def {{ table.name }}_{{ object.name }}(db): + """ {{ object.description|default('') }} """ + + header = [ +{%- for attr in object.attrs %} + "{{ attr.name|upper() }}", +{%- endfor %} +{%- for list in object.lists %} + "{{ list.name|upper() }}", +{%- endfor %} + ] + body = [] + + table = db.cfgdb.get_table("{{ table.name }}") + entry = table.get("{{ object.name }}") + body.append( + [ +{%- for attr in object.attrs -%} + entry.get("{{ attr.name }}", "N/A"), +{%- endfor %} +{%- for list in object.lists -%} + "\n".join(entry.get("{{ list.name }}", [])), +{%- endfor %} + ] + ) + click.echo(tabulate.tabulate(body, header)) + +{% endfor %} +{% elif "dynamic-objects" in table %} +{% for object in table["dynamic-objects"] %} +@click.group(name="{{ make_cli_name(table.name) }}", + cls=clicommon.AliasedGroup, + invoke_without_command=True) +@clicommon.pass_db +def {{ table.name }}(db): + """ {{ object.description|default('') }} """ + + header = [ +{%- for key in object["keys"] %} + "{{ key.name|upper() }}", +{%- endfor %} +{%- for attr in object.attrs %} + "{{ attr.name|upper() }}", +{%- endfor %} +{%- for list in object.lists %} + "{{ list.name|upper() }}", +{%- endfor %} + ] + body = [] + + table = db.cfgdb.get_table("{{ table.name }}") + for key, entry in table.items(): + if not isinstance(key, tuple ): + key = (key,) + body.append( + [ + *key, +{%- for attr in object.attrs -%} + entry.get("{{ attr.name }}", "N/A"), +{%- endfor %} +{%- for list in object.lists -%} + "\n".join(entry.get("{{ list.name }}", [])), +{%- endfor %} + ] + ) + + click.echo(tabulate.tabulate(body, header)) +{% endfor %} +{% endif %} +{% endfor %} + +def register(cli): +{%- for table in tables %} + cli.add_command({{ table.name }}) +{%- endfor %} From 12bf0b27687d42c968356865301f9daa82b1c932 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Wed, 28 Apr 2021 18:17:43 +0300 Subject: [PATCH 08/76] Fix the generated function name is incorrect Signed-off-by: Stepan Blyschak --- sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 index 8536d33303..013fec2517 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 @@ -307,7 +307,7 @@ E.g: @click.argument("{{ make_cli_name(key.name) }}") {%- endfor %} @clicommon.pass_db -def {{ table.name }}_add(db, +def {{ table.name }}_delete(db, {{ object["keys"]|map(attribute="name")|join(",") }}): """ Delete object in {{ table.name }}. """ From c8eaa66120d6ee13974db530064cd3e4127fc0ff Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Thu, 29 Apr 2021 18:18:04 +0300 Subject: [PATCH 09/76] update and enhance templates with multiple lists in container Signed-off-by: Stepan Blyschak --- .../templates/sonic-cli-gen/common.j2 | 10 +- .../templates/sonic-cli-gen/config.py.j2 | 234 ++++++++++-------- .../templates/sonic-cli-gen/show.py.j2 | 66 +++-- 3 files changed, 179 insertions(+), 131 deletions(-) diff --git a/sonic-utilities-data/templates/sonic-cli-gen/common.j2 b/sonic-utilities-data/templates/sonic-cli-gen/common.j2 index 51ebaa004c..3b83ee5635 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/common.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/common.j2 @@ -1,11 +1,3 @@ -{% macro make_cli_name(name) -%} +{% macro cli_name(name) -%} {{ name|lower|replace("_", "-") }} {%- endmacro %} - -{% macro key_converter(keys) %} -{%- if keys|length > 1 %} - ({{ keys|map(attribute="name")|join(",") }}) -{%- else %} - {{ keys|map(attribute="name")|first }} -{%- endif %} -{% endmacro %} diff --git a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 index 013fec2517..541c083d2e 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 @@ -1,4 +1,4 @@ -{%- from "common.j2" import make_cli_name, key_converter -%} +{%- from "common.j2" import cli_name -%} """ Autogenerated config CLI plugin """ import click @@ -12,6 +12,7 @@ def exit_cli(*args, **kwargs): click.secho(*args, **kwargs) raise click.Abort() + def validate_config_or_raise(cfg): """ Validate config db data using ConfigMgmt """ @@ -101,54 +102,91 @@ def del_list_entry_validated(db, table, key, attr, data): if entry not in cfg[table][key][attr]: raise Exception(f"{entry} does not exist") cfg[table][key][attr].remove(entry) + if not cfg[table][key][attr]: + cfg[table][key].pop(attr) validate_config_or_raise(cfg) db.set_entry(table, key, {attr: cfg[table][key][attr]}) +{%- macro gen_click_arguments(args) -%} +{%- for arg in args %} +@click.argument( + "{{ cli_name(arg.name) }}", + nargs={% if arg.is_list %}-1{% else %}1{% endif %}, +) +{%- endfor %} +{%- endmacro %} + +{%- macro gen_click_options(opts) -%} +{%- for opt in opts %} +@click.option( + "--{{ cli_name(opt.name) }}", +) +{%- endfor %} +{%- endmacro %} -{% macro config_object_list_update(table, object, list) %} -@{{ table.name }}.group(name="{{ make_cli_name(list.name) }}") -def {{ table.name }}_{{ list.name }}(): - """ Add/Remove {{ list.name }} in {{ table.name }} """ +{% macro pythonize(attrs) -%} +{{ attrs|map(attribute="name")|map("lower")|map("replace", "-", "_")|join(", ") }} +{%- endmacro %} + +{% macro config_object_list_update(group, table, object, attr) %} +{% set list_update_group = group + "_" + attr.name %} + +@{{ group }}.group(name="{{ cli_name(attr.name) }}") +def {{ list_update_group }}(): + """ Add/Delete {{ attr.name }} in {{ table.name }} """ pass -@{{ table.name }}_{{ list.name }}.command(name="add") -{%- for key in object["keys"] %} -@click.argument("{{ make_cli_name(key.name) }}") -{%- endfor %} -@click.argument("{{ make_cli_name(list.name) }}", nargs=-1) +{# Add entries to list attribute config CLI generation +E.g: + @TABLE_object.command(name="add") + @click.argument("key1", nargs=1) + @click.argument("key2", nargs=1) + @click.argument("attribute", nargs=-1) + def TABLE_object_attribute_add(db, key1, key2, attribute): +#} +@{{ list_update_group }}.command(name="add") +{{ gen_click_arguments(object["keys"] + [attr]) }} @clicommon.pass_db -def {{ table.name }}_{{ list.name }}_add(db, - {{ object["keys"]|map(attribute="name")|join(",") }}, - {{ list.name|lower }}): - """ Add {{ list.name }} in {{ table.name }} """ +def {{ list_update_group }}_add( + db, + {{ pythonize(object["keys"] + [attr]) }} +): + """ Add {{ attr.name }} in {{ table.name }} """ table = "{{ table.name }}" - key = {{ key_converter(object["keys"]) }} - attr = "{{ list.name }}" - data = {{ list.name|lower }} + key = {{ pythonize(object["keys"]) }} + attr = "{{ attr.name }}" + data = {{ pythonize([attr]) }} try: add_list_entry_validated(db.cfgdb, table, key, attr, data) except Exception as err: exit_cli(f"Error: {err}", fg="red") -@{{ table.name }}_{{ list.name }}.command(name="remove") -{%- for key in object["keys"] %} -@click.argument("{{ make_cli_name(key.name) }}") -{%- endfor %} -@click.argument("{{ make_cli_name(list.name) }}", nargs=-1) + +{# Delete entries from list attribute config CLI generation +E.g: + @TABLE_object.command(name="delete") + @click.argument("key1", nargs=1) + @click.argument("key2", nargs=1) + @click.argument("attribute", nargs=-1) + def TABLE_object_attribute_delete(db, key1, key2, attribute): +#} +@{{ list_update_group }}.command(name="delete") +{{ gen_click_arguments(object["keys"] + [attr]) }} @clicommon.pass_db -def {{ table.name }}_{{ list.name }}_remove(db, - {{ object["keys"]|map(attribute="name")|join(",") }}, - {{ list.name|lower }}): - """ Remove {{ list.name }} in {{ table.name }} """ +def {{ list_update_group }}_delete( + db, + {{ pythonize(object["keys"] + [attr]) }} +): + """ Delete {{ attr.name }} in {{ table.name }} """ table = "{{ table.name }}" - key = {{ key_converter(object["keys"]) }} - attr = "{{ list.name }}" - data = {{ list.name|lower }} + key = {{ pythonize(object["keys"]) }} + attr = "{{ attr.name }}" + data = {{ pythonize([attr]) }} try: del_list_entry_validated(db.cfgdb, table, key, attr, data) @@ -157,24 +195,26 @@ def {{ table.name }}_{{ list.name }}_remove(db, {% endmacro %} -{% macro config_object_list_update_all(table, object) %} -{% for list in object.lists %} -{{ config_object_list_update(table, object, list) }} +{% macro config_object_list_update_all(group, table, object) %} +{% for attr in object.attrs %} +{% if attr.is_list %} +{{ config_object_list_update(group, table, object, attr) }} +{% endif %} {% endfor %} {% endmacro %} {% macro config_static_object_attr(table, object, attr) %} -@{{ table.name }}_{{ object.name }}.command(name="{{ make_cli_name(attr.name) }}") -@click.argument("{{ make_cli_name(attr.name) }}") +@{{ table.name }}_{{ object.name }}.command(name="{{ cli_name(attr.name) }}") +{{ gen_click_arguments([attr]) }} @clicommon.pass_db -def {{ table.name }}_{{ object.name }}_{{ attr.name }}(db, {{ attr.name|lower }}): - """ {{ attr.description|default("") }} """ +def {{ table.name }}_{{ object.name }}_{{ attr.name }}(db, {{ pythonize([attr]) }}): + """ {{ attr.description }} """ table = "{{ table.name }}" key = "{{ object.name }}" data = { - "{{ attr.name }}": {{ attr.name|lower }}, + "{{ attr.name }}": {{ pythonize([attr]) }}, } try: mod_entry_validated(db.cfgdb, table, key, data) @@ -189,10 +229,10 @@ E.g: def TABLE_object(db): #} {% macro config_static_object(table, object) %} -@{{ table.name }}.group(name="{{ make_cli_name(object.name) }}") +@{{ table.name }}.group(name="{{ cli_name(object.name) }}") @clicommon.pass_db def {{ table.name }}_{{ object.name }}(db): - """ {{ object.description|default("") }} """ + """ {{ object.description }} """ pass @@ -205,7 +245,7 @@ E.g: {{ config_static_object_attr(table, object, attr) }} {% endfor %} -{{ config_object_list_update_all(table, object) }} +{{ config_object_list_update_all(table.name + "_" + object.name, table, object) }} {% endmacro %} {# Dynamic objects config CLI generation #} @@ -218,32 +258,26 @@ E.g: @click.option("--attr1") @click.option("--attr2") @click.option("--attr3") - def TABLE_object_add(db, key1, key2, attr1, attr2, attr3): + def TABLE_TABLE_LIST_add(db, key1, key2, attr1, attr2, attr3): #} -{% macro config_dynamic_object_add(table, object) %} -@{{ table.name }}.command(name="add") -{%- for key in object["keys"] %} -@click.argument("{{ make_cli_name(key.name) }}") -{%- endfor %} -{%- for attr in object.attrs + object.lists %} -@click.option("--{{ make_cli_name(attr.name) }}") -{%- endfor %} +{% macro config_dynamic_object_add(group, table, object) %} +@{{ group }}.command(name="add") +{{ gen_click_arguments(object["keys"]) }} +{{ gen_click_options(object.attrs) }} @clicommon.pass_db -def {{ table.name }}_add(db, - {{ object["keys"]|map(attribute="name")|join(",") }}, - {{ (object.attrs + object.lists)|map(attribute="name")|join(",") }}): +def {{ group }}_add(db, {{ pythonize(object["keys"] + object.attrs) }}): """ Add object in {{ table.name }}. """ table = "{{ table.name }}" - key = {{ key_converter(object["keys"]) }} + key = {{ pythonize(object["keys"]) }} data = {} {%- for attr in object.attrs %} - if {{ attr.name|lower }} is not None: - data["{{ attr.name }}"] = {{ attr.name|lower }} -{%- endfor %} -{%- for list in object.lists %} - if {{ list.name|lower }} is not None: - data["{{ list.name }}"] = {{ list.name|lower }}.split(",") + if {{ pythonize([attr]) }} is not None: +{%- if not attr.is_list %} + data["{{ attr.name }}"] = {{ pythonize([attr]) }} +{%- else %} + data["{{ attr.name }}"] = {{ pythonize([attr]) }}.split(",") +{%- endif %} {%- endfor %} try: @@ -260,32 +294,26 @@ E.g: @click.option("--attr1") @click.option("--attr2") @click.option("--attr3") - def TABLE_object_update(db, key1, key2, attr1, attr2, attr3): + def TABLE_TABLE_LIST_update(db, key1, key2, attr1, attr2, attr3): #} -{% macro config_dynamic_object_update(table, object) %} -@{{ table.name }}.command(name="update") -{%- for key in object["keys"] %} -@click.argument("{{ make_cli_name(key.name) }}") -{%- endfor %} -{%- for attr in object.attrs + object.lists %} -@click.option("--{{ make_cli_name(attr.name) }}") -{%- endfor %} +{% macro config_dynamic_object_update(group, table, object) %} +@{{ group }}.command(name="update") +{{ gen_click_arguments(object["keys"]) }} +{{ gen_click_options(object.attrs) }} @clicommon.pass_db -def {{ table.name }}_update(db, - {{ object["keys"]|map(attribute="name")|join(",") }}, - {{ (object.attrs + object.lists)|map(attribute="name")|join(",") }}): +def {{ group }}_update(db, {{ pythonize(object["keys"] + object.attrs) }}): """ Add object in {{ table.name }}. """ table = "{{ table.name }}" - key = {{ key_converter(object["keys"]) }} + key = {{ pythonize(object["keys"]) }} data = {} {%- for attr in object.attrs %} - if {{ attr.name|lower }} is not None: - data["{{ attr.name }}"] = {{ attr.name|lower }} -{%- endfor %} -{%- for list in object.lists %} - if {{ list.name|lower }} is not None: - data["{{ list.name }}"] = {{ list.name|lower }}.split(",") + if {{ pythonize([attr]) }} is not None: +{%- if not attr.is_list %} + data["{{ attr.name }}"] = {{ pythonize([attr]) }} +{%- else %} + data["{{ attr.name }}"] = {{ pythonize([attr]) }}.split(",") +{%- endif %} {%- endfor %} try: @@ -299,20 +327,17 @@ E.g: @TABLE.command(name="delete") @click.argument("key1") @click.argument("key2") - def TABLE_object_add(db, key1, key2): + def TABLE_TABLE_LIST_delete(db, key1, key2): #} -{% macro config_dynamic_object_delete(table, object) %} -@{{ table.name }}.command(name="delete") -{%- for key in object["keys"] %} -@click.argument("{{ make_cli_name(key.name) }}") -{%- endfor %} +{% macro config_dynamic_object_delete(group, table, object) %} +@{{ group }}.command(name="delete") +{{ gen_click_arguments(object["keys"]) }} @clicommon.pass_db -def {{ table.name }}_delete(db, - {{ object["keys"]|map(attribute="name")|join(",") }}): +def {{ group }}_delete(db, {{ pythonize(object["keys"]) }}): """ Delete object in {{ table.name }}. """ table = "{{ table.name }}" - key = {{ key_converter(object["keys"]) }} + key = {{ pythonize(object["keys"]) }} try: del_entry_validated(db.cfg, table, key) except Exception as err: @@ -320,27 +345,40 @@ def {{ table.name }}_delete(db, {% endmacro %} {% macro config_dynamic_object(table, object) %} -{{ config_dynamic_object_add(table, object) }} -{{ config_dynamic_object_update(table, object) }} -{{ config_dynamic_object_delete(table, object) }} -{{ config_object_list_update_all(table, object) }} +{# Generate another nesting group in case table holds two types of objects #} +{% if table.dynamic_objects|length > 1 %} +{% set group = table.name + "_" + object.name %} +@{{ table.name }}.group(name="{{ cli_name(object.name) }}", + cls=clicommon.AliasedGroup) +def {{ group }}(): + """ {{ object.description }} """ + + pass +{% else %} +{% set group = table.name %} +{% endif %} + +{{ config_dynamic_object_add(group, table, object) }} +{{ config_dynamic_object_update(group, table, object) }} +{{ config_dynamic_object_delete(group, table, object) }} +{{ config_object_list_update_all(group, table, object) }} {% endmacro %} {% for table in tables %} -@click.group(name="{{ make_cli_name(table.name) }}", +@click.group(name="{{ cli_name(table.name) }}", cls=clicommon.AliasedGroup) def {{ table.name }}(): - """ {{ table.description|default("") }} """ + """ {{ table.description }} """ pass -{% if "static-objects" in table %} -{% for object in table["static-objects"] %} +{% if "static_objects" in table %} +{% for object in table.static_objects %} {{ config_static_object(table, object) }} {% endfor %} -{% elif "dynamic-objects" in table %} -{% for object in table["dynamic-objects"] %} +{% elif "dynamic_objects" in table %} +{% for object in table.dynamic_objects %} {{ config_dynamic_object(table, object) }} {% endfor %} {% endif %} @@ -350,3 +388,5 @@ def register(cli): {%- for table in tables %} cli.add_command({{ table.name }}) {%- endfor %} + +{{ tables|map(attribute="name")|first }}() \ No newline at end of file diff --git a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 index 84f76fea2f..880393f66e 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 @@ -1,30 +1,28 @@ +{% from "common.j2" import cli_name -%} +""" Autogenerated show CLI plugin """ + import click import tabulate -from utilities_common.db import Db import utilities_common.cli as clicommon -{% from "common.j2" import make_cli_name %} {% for table in tables %} -{% if "static-objects" in table %} -@click.group(name="{{ make_cli_name(table.name) }}", cls=clicommon.AliasedGroup) +{% if "static_objects" in table %} +@click.group(name="{{ cli_name(table.name) }}", cls=clicommon.AliasedGroup) def {{ table.name }}(): - """ {{ table.description|default('') }} """ + """ {{ table.description }}""" pass -{% for object in table["static-objects"] %} -@{{ table.name }}.command(name="{{ make_cli_name(object.name) }}") +{% for object in table.static_objects %} +@{{ table.name }}.command(name="{{ cli_name(object.name) }}") @clicommon.pass_db def {{ table.name }}_{{ object.name }}(db): - """ {{ object.description|default('') }} """ + """ {{ object.description }} """ header = [ {%- for attr in object.attrs %} "{{ attr.name|upper() }}", -{%- endfor %} -{%- for list in object.lists %} - "{{ list.name|upper() }}", {%- endfor %} ] body = [] @@ -34,24 +32,42 @@ def {{ table.name }}_{{ object.name }}(db): body.append( [ {%- for attr in object.attrs -%} +{%- if not attr.is_list %} entry.get("{{ attr.name }}", "N/A"), -{%- endfor %} -{%- for list in object.lists -%} - "\n".join(entry.get("{{ list.name }}", [])), +{%- else %} + "\n".join(entry.get("{{ attr.name }}", [])), +{%- endif %} {%- endfor %} ] ) click.echo(tabulate.tabulate(body, header)) {% endfor %} -{% elif "dynamic-objects" in table %} -{% for object in table["dynamic-objects"] %} -@click.group(name="{{ make_cli_name(table.name) }}", +{% elif "dynamic_objects" in table %} +{% if table.dynamic_objects|length > 1 %} +@click.group(name="{{ cli_name(table.name) }}", + cls=clicommon.AliasedGroup) +def {{ table.name }}(): + """ {{ table.description }} """ + + pass +{% endif %} +{% for object in table.dynamic_objects %} +{# Generate another nesting group in case table holds two types of objects #} +{% if table.dynamic_objects|length > 1 %} +{% set group = table.name %} +{% set name = object.name %} +{% else %} +{% set group = "click" %} +{% set name = table.name %} +{% endif %} + +@{{ group }}.group(name="{{ cli_name(name) }}", cls=clicommon.AliasedGroup, invoke_without_command=True) @clicommon.pass_db -def {{ table.name }}(db): - """ {{ object.description|default('') }} """ +def {{ name }}(db): + """ {{ object.description }} """ header = [ {%- for key in object["keys"] %} @@ -59,9 +75,6 @@ def {{ table.name }}(db): {%- endfor %} {%- for attr in object.attrs %} "{{ attr.name|upper() }}", -{%- endfor %} -{%- for list in object.lists %} - "{{ list.name|upper() }}", {%- endfor %} ] body = [] @@ -74,10 +87,11 @@ def {{ table.name }}(db): [ *key, {%- for attr in object.attrs -%} +{%- if not attr.is_list %} entry.get("{{ attr.name }}", "N/A"), -{%- endfor %} -{%- for list in object.lists -%} - "\n".join(entry.get("{{ list.name }}", [])), +{%- else %} + "\n".join(entry.get("{{ attr.name }}", [])), +{%- endif %} {%- endfor %} ] ) @@ -91,3 +105,5 @@ def register(cli): {%- for table in tables %} cli.add_command({{ table.name }}) {%- endfor %} + +{{ tables|map(attribute="name")|first }}() \ No newline at end of file From dd29e33a0150d0c9e1323f4d565b0e0acf1ff0a9 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Thu, 29 Apr 2021 19:37:10 +0300 Subject: [PATCH 10/76] fix cli templates Signed-off-by: Stepan Blyschak --- .../templates/sonic-cli-gen/config.py.j2 | 15 ++++++++------- .../templates/sonic-cli-gen/show.py.j2 | 2 -- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 index 541c083d2e..e7ff755a37 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 @@ -87,7 +87,7 @@ def add_list_entry_validated(db, table, key, attr, data): cfg[table][key][attr].append(entry) validate_config_or_raise(cfg) - db.set_entry(table, key, {attr: cfg[table][key][attr]}) + db.set_entry(table, key, cfg[table][key]) def del_list_entry_validated(db, table, key, attr, data): @@ -106,13 +106,14 @@ def del_list_entry_validated(db, table, key, attr, data): cfg[table][key].pop(attr) validate_config_or_raise(cfg) - db.set_entry(table, key, {attr: cfg[table][key][attr]}) + db.set_entry(table, key, cfg[table][key]) {%- macro gen_click_arguments(args) -%} {%- for arg in args %} @click.argument( "{{ cli_name(arg.name) }}", nargs={% if arg.is_list %}-1{% else %}1{% endif %}, + required=True, ) {%- endfor %} {%- endmacro %} @@ -132,7 +133,8 @@ def del_list_entry_validated(db, table, key, attr, data): {% macro config_object_list_update(group, table, object, attr) %} {% set list_update_group = group + "_" + attr.name %} -@{{ group }}.group(name="{{ cli_name(attr.name) }}") +@{{ group }}.group(name="{{ cli_name(attr.name) }}", + cls=clicommon.AliasedGroup) def {{ list_update_group }}(): """ Add/Delete {{ attr.name }} in {{ table.name }} """ @@ -229,7 +231,8 @@ E.g: def TABLE_object(db): #} {% macro config_static_object(table, object) %} -@{{ table.name }}.group(name="{{ cli_name(object.name) }}") +@{{ table.name }}.group(name="{{ cli_name(object.name) }}", + cls=clicommon.AliasedGroup) @clicommon.pass_db def {{ table.name }}_{{ object.name }}(db): """ {{ object.description }} """ @@ -339,7 +342,7 @@ def {{ group }}_delete(db, {{ pythonize(object["keys"]) }}): table = "{{ table.name }}" key = {{ pythonize(object["keys"]) }} try: - del_entry_validated(db.cfg, table, key) + del_entry_validated(db.cfgdb, table, key) except Exception as err: exit_cli(f"Error: {err}", fg="red") {% endmacro %} @@ -388,5 +391,3 @@ def register(cli): {%- for table in tables %} cli.add_command({{ table.name }}) {%- endfor %} - -{{ tables|map(attribute="name")|first }}() \ No newline at end of file diff --git a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 index 880393f66e..a0c51502b3 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 @@ -105,5 +105,3 @@ def register(cli): {%- for table in tables %} cli.add_command({{ table.name }}) {%- endfor %} - -{{ tables|map(attribute="name")|first }}() \ No newline at end of file From 16857c4e60823745872840c0c10ea8955707500f Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Fri, 30 Apr 2021 02:43:50 +0300 Subject: [PATCH 11/76] add clear list command Signed-off-by: Stepan Blyschak --- .../templates/sonic-cli-gen/config.py.j2 | 41 +++++++++++ .../templates/sonic-cli-gen/show.py.j2 | 70 ++++++++----------- 2 files changed, 72 insertions(+), 39 deletions(-) diff --git a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 index e7ff755a37..4d70f9d1e8 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 @@ -41,6 +41,7 @@ def add_entry_validated(db, table, key, data): cfg.setdefault(table, {}) if key in cfg[table]: raise Exception(f"{key} already exists") + cfg[table][key] = data validate_config_or_raise(cfg) @@ -54,6 +55,7 @@ def update_entry_validated(db, table, key, data): cfg.setdefault(table, {}) if key not in cfg[table]: raise Exception(f"{key} does not exist") + cfg[table][key].update(data) validate_config_or_raise(cfg) @@ -67,6 +69,7 @@ def del_entry_validated(db, table, key): cfg.setdefault(table, {}) if key not in cfg[table]: raise Exception(f"{key} does not exist") + cfg[table].pop(key) validate_config_or_raise(cfg) @@ -108,6 +111,13 @@ def del_list_entry_validated(db, table, key, attr, data): validate_config_or_raise(cfg) db.set_entry(table, key, cfg[table][key]) + +def clear_list_entry_validated(db, table, key, attr): + """ Clear list in object and validate configuration""" + + update_entry_validated(db, table, key, {attr: []}) + + {%- macro gen_click_arguments(args) -%} {%- for arg in args %} @click.argument( @@ -194,6 +204,34 @@ def {{ list_update_group }}_delete( del_list_entry_validated(db.cfgdb, table, key, attr, data) except Exception as err: exit_cli(f"Error: {err}", fg="red") + + +{# Clear entries from list attribute config CLI generation +E.g: + @TABLE_object.command(name="delete") + @click.argument("key1", nargs=1) + @click.argument("key2", nargs=1) + def TABLE_object_attribute_clear(db, key1, key2): +#} +@{{ list_update_group }}.command(name="clear") +{{ gen_click_arguments(object["keys"]) }} +@clicommon.pass_db +def {{ list_update_group }}_delete( + db, + {{ pythonize(object["keys"]) }} +): + """ Clear {{ attr.name }} in {{ table.name }} """ + + table = "{{ table.name }}" + key = {{ pythonize(object["keys"]) }} + attr = "{{ attr.name }}" + data = {{ pythonize([attr]) }} + + try: + clear_list_entry_validated(db.cfgdb, table, key, attr) + except Exception as err: + exit_cli(f"Error: {err}", fg="red") + {% endmacro %} @@ -389,5 +427,8 @@ def {{ table.name }}(): def register(cli): {%- for table in tables %} + cli_node = {{ table.name }} + if cli_node.name in cli.commands: + raise Exception(f"{cli_node.name} already exists in CLI") cli.add_command({{ table.name }}) {%- endfor %} diff --git a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 index a0c51502b3..191a466852 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 @@ -3,12 +3,29 @@ import click import tabulate +import natsort import utilities_common.cli as clicommon +{% macro print_attr(attr) %} +{%- if not attr.is_list %} +entry.get("{{ attr.name }}", "N/A") +{%- else %} +"\n".join(entry.get("{{ attr.name }}", [])) +{%- endif %} +{% endmacro %} + + +{% macro gen_header(attrs) %} +{% for attr in attrs %} +"{{ attr.name|upper|replace("_", " ")|replace("-", " ") }}", +{% endfor %} +{% endmacro %} + {% for table in tables %} {% if "static_objects" in table %} -@click.group(name="{{ cli_name(table.name) }}", cls=clicommon.AliasedGroup) +@click.group(name="{{ cli_name(table.name) }}", + cls=clicommon.AliasedGroup) def {{ table.name }}(): """ {{ table.description }}""" @@ -20,26 +37,13 @@ def {{ table.name }}(): def {{ table.name }}_{{ object.name }}(db): """ {{ object.description }} """ - header = [ -{%- for attr in object.attrs %} - "{{ attr.name|upper() }}", -{%- endfor %} - ] + header = [{{ gen_header(object.attrs) }}] body = [] table = db.cfgdb.get_table("{{ table.name }}") entry = table.get("{{ object.name }}") - body.append( - [ -{%- for attr in object.attrs -%} -{%- if not attr.is_list %} - entry.get("{{ attr.name }}", "N/A"), -{%- else %} - "\n".join(entry.get("{{ attr.name }}", [])), -{%- endif %} -{%- endfor %} - ] - ) + row = [{%- for attr in object.attrs -%} {{ print_attr(attr) }}, {%- endfor %}] + body.append(row) click.echo(tabulate.tabulate(body, header)) {% endfor %} @@ -69,32 +73,17 @@ def {{ table.name }}(): def {{ name }}(db): """ {{ object.description }} """ - header = [ -{%- for key in object["keys"] %} - "{{ key.name|upper() }}", -{%- endfor %} -{%- for attr in object.attrs %} - "{{ attr.name|upper() }}", -{%- endfor %} - ] + header = [{{ gen_header(object["keys"] + object.attrs) }}] body = [] table = db.cfgdb.get_table("{{ table.name }}") - for key, entry in table.items(): - if not isinstance(key, tuple ): + for key in natsort.natsorted(table): + entry = table[key] + if not isinstance(key, tuple): key = (key,) - body.append( - [ - *key, -{%- for attr in object.attrs -%} -{%- if not attr.is_list %} - entry.get("{{ attr.name }}", "N/A"), -{%- else %} - "\n".join(entry.get("{{ attr.name }}", [])), -{%- endif %} -{%- endfor %} - ] - ) + + row = [*key, {%- for attr in object.attrs -%} {{ print_attr(attr) }}, {%- endfor %}] + body.append(row) click.echo(tabulate.tabulate(body, header)) {% endfor %} @@ -103,5 +92,8 @@ def {{ name }}(db): def register(cli): {%- for table in tables %} + cli_node = {{ table.name }} + if cli_node.name in cli.commands: + raise Exception(f"{cli_node.name} already exists in CLI") cli.add_command({{ table.name }}) {%- endfor %} From dfa3cccdea80965e08628d52305da946b4138c42 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Fri, 30 Apr 2021 11:01:08 +0000 Subject: [PATCH 12/76] Parsing for static YANG models - sonic-flex_counter.yang, sonic-device_metadata.yang DONE Signed-off-by: Vadym Hlushko --- sonic_cli_gen/main.py | 2 +- sonic_cli_gen/yang_parser.py | 93 +++++++++++++++++++++++++++++------- 2 files changed, 78 insertions(+), 17 deletions(-) diff --git a/sonic_cli_gen/main.py b/sonic_cli_gen/main.py index a8b01a7034..d6826a6e7c 100644 --- a/sonic_cli_gen/main.py +++ b/sonic_cli_gen/main.py @@ -19,7 +19,7 @@ def cli(ctx): @click.pass_context def generate_config(ctx): """ List available packages """ - gen = CliGenerator('sonic-vlan') + gen = CliGenerator('sonic-device_metadata') gen.generate_config_plugin() pass diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index 77f9ded716..ba9026adec 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -1,6 +1,7 @@ #!/usr/bin/env python try: + import pdb import os import sys import pprint @@ -18,34 +19,94 @@ def __init__(self, # index of yang model inside conf_mgmt.sy.yJson object self.idx_yJson = None # 'module' entity from .yang file - self.y_module = OrderedDict() + self.y_module = None # top level 'container' entity from .yang file - self.y_top_level_container = OrderedDict() - # 'container' entities from .yang file - self.y_tables = list() + self.y_top_level_container = None + # 'container' entities from .yang file that represent Config DB table + self.y_table_containers = None # dictionary that represent Config DB schema - self.yang_2_dict = OrderedDict() + self.yang_2_dict = dict() try: self.conf_mgmt = ConfigMgmt() except Exception as e: raise Exception("Failed to load the {} class".format(str(e))) - def fail(self, e): - print(e) - raise e - def parse_yang_model(self): self._init_yang_module_and_containers() - self._determine_tables_type() + # determine how many (1 or couple) containers yang model have after 'top level container' + if isinstance(self.y_table_containers, list): + print('LIST') + for tbl_cont in self.y_table_containers: + self._fill_yang_2_dict(tbl_cont) + else: + print('NOT LIST') + self._fill_yang_2_dict(self.y_table_containers) + - def _determine_tables_type(self): - for table in self.y_tables: - if table.get('list') is None: - self.yang_2_dict[table.get('@name')] = {'type': 'static'} + def _fill_yang_2_dict(self, tbl_cont): + self.yang_2_dict['tables'] = list() + # element for self.yang_2_dict list + y2d_elem = dict() + + y2d_elem['name'] = tbl_cont.get('@name') + y2d_elem['description'] = '' + if tbl_cont.get('description') is not None: + y2d_elem['description'] = tbl_cont.get('description').get('text') + y2d_elem['dynamic-objects'] = list() + y2d_elem['static-objects'] = list() + + # determine if 'container' is a 'list' or 'static' + # 'static' means that yang model 'container' entity does NOT have a 'list' entity + if tbl_cont.get('list') is None: + # TODO write comment about objects containers inside table containers + obj_cont = tbl_cont.get('container') + if isinstance(obj_cont, list): + # flex counter + print ("FLEX") + for cont in obj_cont: + self._on_static_container(cont, y2d_elem) else: - self.yang_2_dict[table.get('@name')] = {'type': 'list'} + print ("METADATA") + # device metadata + self._on_static_container(obj_cont, y2d_elem) + else: + self._on_list_container(tbl_cont, y2d_elem) + + self.yang_2_dict['tables'].append(y2d_elem) + pdb.set_trace() + + def _on_static_container(self, cont, y2d_elem): + # element for y2d_elem['static-objects'] + static_obj_elem = dict() + static_obj_elem['name'] = cont.get('@name') + static_obj_elem['description'] = '' + if cont.get('description') is not None: + static_obj_elem['description'] = cont.get('description').get('text') + + self._parse_yang_leafs(cont.get('leaf'), static_obj_elem, y2d_elem) + + def _parse_yang_leafs(self, y_leafs, static_obj_elem, y2d_elem): + static_obj_elem['attrs'] = list() + # The YANG 'container entity may have only 1 'leaf' element or list of 'leaf' elements + if isinstance(y_leafs, list): + for leaf in y_leafs: + attr = dict() + attr['name'] = leaf.get('@name') + attr['is-leaf-list'] = leaf.get('__isleafList') + static_obj_elem['attrs'].append(attr) + + y2d_elem['static-objects'].append(static_obj_elem) + else: + attr = dict() + attr['name'] = y_leafs.get('@name') + attr['is-leaf-list'] = y_leafs.get('__isleafList') + static_obj_elem['attrs'].append(attr) + y2d_elem['static-objects'].append(static_obj_elem) + + def _on_list_container(self, cont, y2d_elem): + pass def _init_yang_module_and_containers(self): self._find_index_of_yang_model() @@ -53,7 +114,7 @@ def _init_yang_module_and_containers(self): self.y_module = self.conf_mgmt.sy.yJson[self.idx_yJson]['module'] if self.y_module.get('container') is not None: self.y_top_level_container = self.y_module['container'] - self.y_tables = self.y_top_level_container['container'] + self.y_table_containers = self.y_top_level_container['container'] else: raise KeyError('YANG model {} does NOT have "container" element'.format(self.yang_model_name)) From 6b75f4ba1bc9ba66a4ee359b61108a8ec1e45539 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Wed, 5 May 2021 14:35:40 +0000 Subject: [PATCH 13/76] Done refactoring for parsing 'static' YANG models Signed-off-by: Vadym Hlushko --- sonic_cli_gen/__init__.py | 5 + sonic_cli_gen/generator.py | 4 +- sonic_cli_gen/main.py | 8 +- sonic_cli_gen/yang_parser.py | 243 ++++++++++++++++++++++------------- 4 files changed, 165 insertions(+), 95 deletions(-) diff --git a/sonic_cli_gen/__init__.py b/sonic_cli_gen/__init__.py index e69de29bb2..7e49cacd56 100644 --- a/sonic_cli_gen/__init__.py +++ b/sonic_cli_gen/__init__.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python + +from sonic_cli_gen.generator import CliGenerator + +__all__ = ['CliGenerator'] \ No newline at end of file diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py index 154d82409b..96d87f776c 100644 --- a/sonic_cli_gen/generator.py +++ b/sonic_cli_gen/generator.py @@ -8,7 +8,9 @@ class CliGenerator: """ SONiC CLI generator. This class provides public API for sonic-cli-gen python library. It can generate config, - show, sonic-clear commands """ + show, sonic-clear commands + """ + def __init__(self, yang_model): """ Initialize PackageManager. """ diff --git a/sonic_cli_gen/main.py b/sonic_cli_gen/main.py index d6826a6e7c..f5ba5a49b5 100644 --- a/sonic_cli_gen/main.py +++ b/sonic_cli_gen/main.py @@ -12,14 +12,14 @@ @click.pass_context def cli(ctx): """ SONiC CLI generator """ - print ("cli") pass @cli.command() +@click.argument('yang_model_name') @click.pass_context -def generate_config(ctx): - """ List available packages """ - gen = CliGenerator('sonic-device_metadata') +def generate_config(ctx, yang_model_name): + """ Generate config plugin """ + gen = CliGenerator(yang_model_name) gen.generate_config_plugin() pass diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index ba9026adec..2c5b481b98 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -11,120 +11,183 @@ raise ImportError("%s - required module not found" % str(e)) class YangParser: - """ YANG model parser """ + """ YANG model parser + + Attributes: + yang_model_name: Name of the YANG model file + conf_mgmt: Instance of Config Mgmt class to help parse YANG models + idx_yJson: Index of YANG model file (1 attr) inside conf_mgmt.sy.yJson object + y_module: Reference to 'module' entity from YANG model file + y_top_level_container: Reference to top level 'container' entity from YANG model file + y_table_containers: Reference to 'container' entities from YANG model file + that represent Config DB tables + yang_2_dict: dictionary created from YANG model file that represent Config DB schema + """ def __init__(self, yang_model_name): self.yang_model_name = yang_model_name self.conf_mgmt = None - # index of yang model inside conf_mgmt.sy.yJson object self.idx_yJson = None - # 'module' entity from .yang file self.y_module = None - # top level 'container' entity from .yang file self.y_top_level_container = None - # 'container' entities from .yang file that represent Config DB table self.y_table_containers = None - # dictionary that represent Config DB schema self.yang_2_dict = dict() try: self.conf_mgmt = ConfigMgmt() except Exception as e: raise Exception("Failed to load the {} class".format(str(e))) + + def _init_yang_module_and_containers(self): + """ Initialize inner class variables: + self.y_module + self.y_top_level_container + self.y_table_containers - def parse_yang_model(self): - self._init_yang_module_and_containers() - - # determine how many (1 or couple) containers yang model have after 'top level container' - if isinstance(self.y_table_containers, list): - print('LIST') - for tbl_cont in self.y_table_containers: - self._fill_yang_2_dict(tbl_cont) - else: - print('NOT LIST') - self._fill_yang_2_dict(self.y_table_containers) + Raises: + KeyError: if invalid YANG model provided + KeyError: if YANG models is NOT exist + """ + self._find_index_of_yang_model() - def _fill_yang_2_dict(self, tbl_cont): - self.yang_2_dict['tables'] = list() - # element for self.yang_2_dict list - y2d_elem = dict() - - y2d_elem['name'] = tbl_cont.get('@name') - y2d_elem['description'] = '' - if tbl_cont.get('description') is not None: - y2d_elem['description'] = tbl_cont.get('description').get('text') - y2d_elem['dynamic-objects'] = list() - y2d_elem['static-objects'] = list() - - # determine if 'container' is a 'list' or 'static' - # 'static' means that yang model 'container' entity does NOT have a 'list' entity - if tbl_cont.get('list') is None: - # TODO write comment about objects containers inside table containers - obj_cont = tbl_cont.get('container') - if isinstance(obj_cont, list): - # flex counter - print ("FLEX") - for cont in obj_cont: - self._on_static_container(cont, y2d_elem) + if self.idx_yJson is not None: + self.y_module = self.conf_mgmt.sy.yJson[self.idx_yJson]['module'] + if self.y_module.get('container') is not None: + self.y_top_level_container = self.y_module['container'] + self.y_table_containers = self.y_top_level_container['container'] else: - print ("METADATA") - # device metadata - self._on_static_container(obj_cont, y2d_elem) + raise KeyError('YANG model {} does NOT have "container" element'.format(self.yang_model_name)) else: - self._on_list_container(tbl_cont, y2d_elem) - - self.yang_2_dict['tables'].append(y2d_elem) - pdb.set_trace() + raise KeyError('YANG model {} is NOT exist'.format(self.yang_model_name)) - def _on_static_container(self, cont, y2d_elem): - # element for y2d_elem['static-objects'] - static_obj_elem = dict() - static_obj_elem['name'] = cont.get('@name') - static_obj_elem['description'] = '' - if cont.get('description') is not None: - static_obj_elem['description'] = cont.get('description').get('text') - - self._parse_yang_leafs(cont.get('leaf'), static_obj_elem, y2d_elem) - - def _parse_yang_leafs(self, y_leafs, static_obj_elem, y2d_elem): - static_obj_elem['attrs'] = list() - # The YANG 'container entity may have only 1 'leaf' element or list of 'leaf' elements - if isinstance(y_leafs, list): - for leaf in y_leafs: - attr = dict() - attr['name'] = leaf.get('@name') - attr['is-leaf-list'] = leaf.get('__isleafList') - static_obj_elem['attrs'].append(attr) + def _find_index_of_yang_model(self): + """ Find index of provided YANG model inside yJson object + and save it to self.idx_yJson variable + """ - y2d_elem['static-objects'].append(static_obj_elem) - else: - attr = dict() - attr['name'] = y_leafs.get('@name') - attr['is-leaf-list'] = y_leafs.get('__isleafList') - static_obj_elem['attrs'].append(attr) - y2d_elem['static-objects'].append(static_obj_elem) + for i in range(len(self.conf_mgmt.sy.yJson)): + if (self.conf_mgmt.sy.yJson[i]['module']['@name'] == self.yang_model_name): + self.idx_yJson = i - def _on_list_container(self, cont, y2d_elem): - pass + def parse_yang_model(self): + """ Parse proviced YANG model + and save output to self.yang_2_dict obj + """ - def _init_yang_module_and_containers(self): - self._find_index_of_yang_model() + self._init_yang_module_and_containers() + self.yang_2_dict['tables'] = list() - self.y_module = self.conf_mgmt.sy.yJson[self.idx_yJson]['module'] - if self.y_module.get('container') is not None: - self.y_top_level_container = self.y_module['container'] - self.y_table_containers = self.y_top_level_container['container'] + # determine how many (1 or couple) containers yang model have after 'top level container' + if isinstance(self.y_table_containers, list): + for tbl_cont in self.y_table_containers: + y2d_elem = on_table_container(tbl_cont) + self.yang_2_dict['tables'].append(y2d_elem) else: - raise KeyError('YANG model {} does NOT have "container" element'.format(self.yang_model_name)) - - # find index of yang_model inside yJson object - def _find_index_of_yang_model(self): - for i in range(len(self.conf_mgmt.sy.yJson)): - if (self.conf_mgmt.sy.yJson[i]['module']['@name'] == self.yang_model_name): - self.idx_yJson = i + y2d_elem = on_table_container(self.y_table_containers) + self.yang_2_dict['tables'].append(y2d_elem) + pdb.set_trace() - - - \ No newline at end of file +def on_table_container(tbl_cont: OrderedDict) -> dict: + """ Parse 'table' container, + 'table' container goes after 'top level' container + + Args: + tbl_cont: reference to 'table' container + Returns: + dictionary - element for self.yang_2_dict['tables'] + """ + + if tbl_cont.get('description') is not None: + description = tbl_cont.get('description').get('text') + else: + description = '' + + y2d_elem = { + 'name': tbl_cont.get('@name'), + 'description': description, + 'dynamic-objects': list(), + 'static-objects': list() + } + + # determine if 'container' is a 'list' or 'static' + # 'static' means that yang model 'container' entity does NOT have a 'list' entity + if tbl_cont.get('list') is None: + # 'object' container goes after 'table' container + # 'object' container have 2 types - list (like sonic-flex_counter.yang) and NOT list (like sonic-device_metadata.yang) + obj_cont = tbl_cont.get('container') + if isinstance(obj_cont, list): + for cont in obj_cont: + static_obj_elem = on_static_container(cont) + y2d_elem['static-objects'].append(static_obj_elem) + else: + static_obj_elem = on_static_container(obj_cont) + y2d_elem['static-objects'].append(static_obj_elem) + else: + on_list_container(tbl_cont) + + return y2d_elem + +def on_static_container(cont: OrderedDict) -> dict: + """ Parse container that does NOT have a 'list' entity ('static') + + Args: + cont: reference to 'static' container + Returns: + dictionary - element for y2d_elem['static-objects'] + """ + + if cont.get('description') is not None: + description = cont.get('description').get('text') + else: + description = '' + + static_obj_elem = { + 'name': cont.get('@name'), + 'description': description, + 'attrs': list() + } + static_obj_elem['attrs'] = parse_yang_leafs(cont.get('leaf')) + + return static_obj_elem + +def parse_yang_leafs(y_leafs) -> list: + """ Parse all the 'leafs' + + Args: + y_leafs: reference to all 'leaf' elements + Returns: + list - list of parsed 'leafs' + """ + ret_attrs_list = list() + # The YANG 'container' entity may have only 1 'leaf' element OR a list of 'leaf' elements + if isinstance(y_leafs, list): + for leaf in y_leafs: + attr = on_leaf(leaf) + ret_attrs_list.append(attr) + else: + attr = on_leaf(y_leafs) + ret_attrs_list.append(attr) + + return ret_attrs_list + +def on_leaf(leaf: OrderedDict) -> dict: + """ Parse a single 'leaf' element + + Args: + leaf: reference to a 'leaf' entity + Returns: + dictionary - parsed 'leaf' element + """ + mandatory = False + if leaf.get('mandatory') is not None: + mandatory = leaf.get('mandatory').get('@value') + + attr = { 'name': leaf.get('@name'), + 'is-leaf-list': leaf.get('__isleafList'), + 'mandatory': mandatory } + return attr + +def on_list_container(cont): + pass \ No newline at end of file From d2be52b6b24180eb870b57b60ea02d8a9a9a324f Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Thu, 6 May 2021 14:17:24 +0000 Subject: [PATCH 14/76] Added parsing for 'dynamic' YANG models, done refactoring of whole code Signed-off-by: Vadym Hlushko --- sonic_cli_gen/generator.py | 14 ++- sonic_cli_gen/main.py | 11 ++- sonic_cli_gen/yang_parser.py | 179 +++++++++++++++++++++++++---------- 3 files changed, 150 insertions(+), 54 deletions(-) diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py index 96d87f776c..c328fb9995 100644 --- a/sonic_cli_gen/generator.py +++ b/sonic_cli_gen/generator.py @@ -8,7 +8,7 @@ class CliGenerator: """ SONiC CLI generator. This class provides public API for sonic-cli-gen python library. It can generate config, - show, sonic-clear commands + show, sonic-clear CLI plugins """ def __init__(self, @@ -18,16 +18,20 @@ def __init__(self, self.yang_model_name = yang_model def generate_config_plugin(self): + """ Generate CLI plugin for 'config' CLI group. """ parser = YangParser(self.yang_model_name) - parser.parse_yang_model() + yang_dict = parser.parse_yang_model() pass - #TODO def generate_show_plugin(self): - print ("show") + """ Generate CLI plugin for 'show' CLI group. """ + parser = YangParser(self.yang_model_name) + yang_dict = parser.parse_yang_model() pass # to be implemented in the next Phases def generate_sonic_clear_plugin(self): - print ("sonic-clear") + """ Generate CLI plugin for 'sonic-clear' CLI group. """ + parser = YangParser(self.yang_model_name) + yang_dict = parser.parse_yang_model() pass diff --git a/sonic_cli_gen/main.py b/sonic_cli_gen/main.py index f5ba5a49b5..bf783e8541 100644 --- a/sonic_cli_gen/main.py +++ b/sonic_cli_gen/main.py @@ -18,10 +18,19 @@ def cli(ctx): @click.argument('yang_model_name') @click.pass_context def generate_config(ctx, yang_model_name): - """ Generate config plugin """ + """ Generate CLI plugin (click) for 'config' CLI group. """ gen = CliGenerator(yang_model_name) gen.generate_config_plugin() pass +@cli.command() +@click.argument('yang_model_name') +@click.pass_context +def generate_show(ctx, yang_model_name): + """ Generate CLI plugin (click) for 'show' CLI group. """ + gen = CliGenerator(yang_model_name) + gen.generate_show_plugin() + pass + if __name__ == '__main__': cli() \ No newline at end of file diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index 2c5b481b98..5e1229e103 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -1,10 +1,6 @@ #!/usr/bin/env python try: - import pdb - import os - import sys - import pprint from collections import OrderedDict from config.config_mgmt import ConfigMgmt except ImportError as e: @@ -70,15 +66,19 @@ def _find_index_of_yang_model(self): if (self.conf_mgmt.sy.yJson[i]['module']['@name'] == self.yang_model_name): self.idx_yJson = i - def parse_yang_model(self): + def parse_yang_model(self) -> dict: """ Parse proviced YANG model and save output to self.yang_2_dict obj + + Returns: + dictionary - parsed YANG model in dictionary format """ self._init_yang_module_and_containers() self.yang_2_dict['tables'] = list() - # determine how many (1 or couple) containers yang model have after 'top level container' + # determine how many (1 or couple) containers YANG model have after 'top level' container + # 'table' container it is a container that goes after 'top level' container if isinstance(self.y_table_containers, list): for tbl_cont in self.y_table_containers: y2d_elem = on_table_container(tbl_cont) @@ -86,8 +86,22 @@ def parse_yang_model(self): else: y2d_elem = on_table_container(self.y_table_containers) self.yang_2_dict['tables'].append(y2d_elem) + + return self.yang_2_dict + +def get_description(y_entity: OrderedDict) -> str: + """ Parse 'description' entity from any YANG element - pdb.set_trace() + Args: + y_entity: reference to YANG 'container' OR 'list' OR 'leaf' ... + Returns: + str - text of the 'description' + """ + + if y_entity.get('description') is not None: + return y_entity.get('description').get('text') + else: + return '' def on_table_container(tbl_cont: OrderedDict) -> dict: """ Parse 'table' container, @@ -99,80 +113,150 @@ def on_table_container(tbl_cont: OrderedDict) -> dict: dictionary - element for self.yang_2_dict['tables'] """ - if tbl_cont.get('description') is not None: - description = tbl_cont.get('description').get('text') - else: - description = '' - y2d_elem = { 'name': tbl_cont.get('@name'), - 'description': description, + 'description': get_description(tbl_cont), 'dynamic-objects': list(), 'static-objects': list() } - # determine if 'container' is a 'list' or 'static' - # 'static' means that yang model 'container' entity does NOT have a 'list' entity - if tbl_cont.get('list') is None: + # determine if 'container' have a 'list' entity + tbl_cont_lists = tbl_cont.get('list') + + if tbl_cont_lists is None: + is_list = False # 'object' container goes after 'table' container # 'object' container have 2 types - list (like sonic-flex_counter.yang) and NOT list (like sonic-device_metadata.yang) obj_cont = tbl_cont.get('container') if isinstance(obj_cont, list): for cont in obj_cont: - static_obj_elem = on_static_container(cont) + static_obj_elem = on_container(cont, is_list) y2d_elem['static-objects'].append(static_obj_elem) else: - static_obj_elem = on_static_container(obj_cont) + static_obj_elem = on_container(obj_cont, is_list) y2d_elem['static-objects'].append(static_obj_elem) else: - on_list_container(tbl_cont) + is_list = True + # 'container' can have more than 1 'list' + if isinstance(tbl_cont_lists, list): + for _list in tbl_cont_lists: + dynamic_obj_elem = on_container(_list, is_list) + y2d_elem['dynamic-objects'].append(dynamic_obj_elem) + else: + dynamic_obj_elem = on_container(tbl_cont_lists, is_list) + y2d_elem['dynamic-objects'].append(dynamic_obj_elem) return y2d_elem -def on_static_container(cont: OrderedDict) -> dict: - """ Parse container that does NOT have a 'list' entity ('static') +def on_container(cont: OrderedDict, is_list: bool) -> dict: + """ Parse a 'container' that have only 'leafs' or 'list' with 'leafs' Args: - cont: reference to 'static' container + cont: reference to 'container' Returns: - dictionary - element for y2d_elem['static-objects'] + dictionary - element for y2d_elem['static-objects'] OR y2d_elem['dynamic-objects'] """ - if cont.get('description') is not None: - description = cont.get('description').get('text') - else: - description = '' - - static_obj_elem = { + obj_elem = { 'name': cont.get('@name'), - 'description': description, + 'description': get_description(cont), 'attrs': list() } - static_obj_elem['attrs'] = parse_yang_leafs(cont.get('leaf')) - return static_obj_elem + if is_list: + obj_elem['key'] = cont.get('key').get('@value') + + attrs_list = list() + + if cont.get('leaf') is not None: + is_leaf_list = False + ret_leafs = on_leafs(cont.get('leaf'), is_leaf_list) + attrs_list.extend(ret_leafs) + + if cont.get('leaf-list') is not None: + is_leaf_list = True + ret_leaf_lists = on_leafs(cont.get('leaf-list'), is_leaf_list) + attrs_list.extend(ret_leaf_lists) + + if cont.get('choice') is not None: + y_choices = cont.get('choice') + ret_choice_leafs = on_choices(y_choices) + attrs_list.extend(ret_choice_leafs) + + obj_elem['attrs'] = attrs_list + + return obj_elem + +def on_choices(y_choices) -> list: + """ Parse a YANG 'choice' entities -def parse_yang_leafs(y_leafs) -> list: - """ Parse all the 'leafs' + Args: + cont: reference to 'choice' + Returns: + dictionary - element for obj_elem['attrs'], 'attrs' contain a parsed 'leafs' + """ + + ret_attrs = list() + + # the YANG model can have multiple 'choice' entities inside 'container' or 'list' + if isinstance(y_choices, list): + for choice in y_choices: + attrs = on_choice_cases(choice.get('case')) + ret_attrs.extend(attrs) + else: + ret_attrs = on_choice_cases(y_choices.get('case')) + + return ret_attrs + +def on_choice_cases(y_cases: list) -> list: + """ Parse a single YANG 'case' entity from 'choice' entity + + Args: + cont: reference to 'case' + Returns: + dictionary - element for obj_elem['attrs'], 'attrs' contain a parsed 'leafs' + """ + + ret_attrs = list() + + if isinstance(y_cases, list): + for case in y_cases: + if case.get('leaf') is not None: + is_leaf_list = False + ret_leafs = on_leafs(case.get('leaf'), is_leaf_list) + ret_attrs.extend(ret_leafs) + + if case.get('leaf-list') is not None: + is_leaf_list = True + ret_leaf_lists = on_leafs(case.get('leaf-list'), is_leaf_list) + ret_attrs.extend(ret_leaf_lists) + else: + raise Exception('It has no sense to using a single "case" element inside "choice" element') + + return ret_attrs + +def on_leafs(y_leafs, is_leaf_list: bool) -> list: + """ Parse all the 'leaf' or 'leaf-list' elements Args: y_leafs: reference to all 'leaf' elements Returns: - list - list of parsed 'leafs' + list - list of parsed 'leaf' elements """ - ret_attrs_list = list() + + ret_attrs = list() # The YANG 'container' entity may have only 1 'leaf' element OR a list of 'leaf' elements if isinstance(y_leafs, list): for leaf in y_leafs: - attr = on_leaf(leaf) - ret_attrs_list.append(attr) + attr = on_leaf(leaf, is_leaf_list) + ret_attrs.append(attr) else: - attr = on_leaf(y_leafs) - ret_attrs_list.append(attr) + attr = on_leaf(y_leafs, is_leaf_list) + ret_attrs.append(attr) - return ret_attrs_list + return ret_attrs -def on_leaf(leaf: OrderedDict) -> dict: +def on_leaf(leaf: OrderedDict, is_leaf_list: bool) -> dict: """ Parse a single 'leaf' element Args: @@ -180,14 +264,13 @@ def on_leaf(leaf: OrderedDict) -> dict: Returns: dictionary - parsed 'leaf' element """ + mandatory = False if leaf.get('mandatory') is not None: - mandatory = leaf.get('mandatory').get('@value') + mandatory = True attr = { 'name': leaf.get('@name'), - 'is-leaf-list': leaf.get('__isleafList'), + 'description': get_description(leaf), + 'is-leaf-list': is_leaf_list, 'mandatory': mandatory } - return attr - -def on_list_container(cont): - pass \ No newline at end of file + return attr \ No newline at end of file From 73a3b121a428f83245d1c236bdf4b38e0ef1c504 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Wed, 12 May 2021 11:31:47 +0000 Subject: [PATCH 15/76] Done parser for 'grouping' BUT need to deeply test it, added functions - get_leafs(), get_leaf_lists(), get_choices(), get_uses_grouping() Signed-off-by: Vadym Hlushko --- sonic_cli_gen/generator.py | 1 + sonic_cli_gen/yang_parser.py | 198 ++++++++++++++++++++++++----------- 2 files changed, 137 insertions(+), 62 deletions(-) diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py index c328fb9995..6a3b6f2283 100644 --- a/sonic_cli_gen/generator.py +++ b/sonic_cli_gen/generator.py @@ -21,6 +21,7 @@ def generate_config_plugin(self): """ Generate CLI plugin for 'config' CLI group. """ parser = YangParser(self.yang_model_name) yang_dict = parser.parse_yang_model() + import pprint; pprint.pprint(yang_dict) pass def generate_show_plugin(self): diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index 5e1229e103..a7de4885d8 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -12,7 +12,7 @@ class YangParser: Attributes: yang_model_name: Name of the YANG model file conf_mgmt: Instance of Config Mgmt class to help parse YANG models - idx_yJson: Index of YANG model file (1 attr) inside conf_mgmt.sy.yJson object + idx_yJson: Index of yang_model_file (1 attr) inside conf_mgmt.sy.yJson object y_module: Reference to 'module' entity from YANG model file y_top_level_container: Reference to top level 'container' entity from YANG model file y_table_containers: Reference to 'container' entities from YANG model file @@ -81,33 +81,22 @@ def parse_yang_model(self) -> dict: # 'table' container it is a container that goes after 'top level' container if isinstance(self.y_table_containers, list): for tbl_cont in self.y_table_containers: - y2d_elem = on_table_container(tbl_cont) + y2d_elem = on_table_container(self.y_module, tbl_cont) self.yang_2_dict['tables'].append(y2d_elem) else: - y2d_elem = on_table_container(self.y_table_containers) + y2d_elem = on_table_container(self.y_module, self.y_table_containers) self.yang_2_dict['tables'].append(y2d_elem) - - return self.yang_2_dict -def get_description(y_entity: OrderedDict) -> str: - """ Parse 'description' entity from any YANG element - - Args: - y_entity: reference to YANG 'container' OR 'list' OR 'leaf' ... - Returns: - str - text of the 'description' - """ + return self.yang_2_dict - if y_entity.get('description') is not None: - return y_entity.get('description').get('text') - else: - return '' +#------------------------------HANDLERS--------------------------------# -def on_table_container(tbl_cont: OrderedDict) -> dict: +def on_table_container(y_module: OrderedDict, tbl_cont: OrderedDict) -> dict: """ Parse 'table' container, 'table' container goes after 'top level' container Args: + y_module: reference to 'module' tbl_cont: reference to 'table' container Returns: dictionary - element for self.yang_2_dict['tables'] @@ -120,39 +109,45 @@ def on_table_container(tbl_cont: OrderedDict) -> dict: 'static-objects': list() } - # determine if 'container' have a 'list' entity - tbl_cont_lists = tbl_cont.get('list') - - if tbl_cont_lists is None: - is_list = False + # determine if 'table container' have a 'list' entity + if tbl_cont.get('list') is None: # 'object' container goes after 'table' container # 'object' container have 2 types - list (like sonic-flex_counter.yang) and NOT list (like sonic-device_metadata.yang) obj_cont = tbl_cont.get('container') if isinstance(obj_cont, list): for cont in obj_cont: - static_obj_elem = on_container(cont, is_list) + static_obj_elem = on_object_container(y_module, cont, is_list=False) y2d_elem['static-objects'].append(static_obj_elem) else: - static_obj_elem = on_container(obj_cont, is_list) + static_obj_elem = on_object_container(y_module, obj_cont, is_list=False) y2d_elem['static-objects'].append(static_obj_elem) else: - is_list = True + tbl_cont_lists = tbl_cont.get('list') # 'container' can have more than 1 'list' if isinstance(tbl_cont_lists, list): for _list in tbl_cont_lists: - dynamic_obj_elem = on_container(_list, is_list) + dynamic_obj_elem = on_object_container(y_module, _list, is_list=True) y2d_elem['dynamic-objects'].append(dynamic_obj_elem) else: - dynamic_obj_elem = on_container(tbl_cont_lists, is_list) + dynamic_obj_elem = on_object_container(y_module, tbl_cont_lists, is_list=True) y2d_elem['dynamic-objects'].append(dynamic_obj_elem) return y2d_elem -def on_container(cont: OrderedDict, is_list: bool) -> dict: - """ Parse a 'container' that have only 'leafs' or 'list' with 'leafs' +def on_object_container(y_module: OrderedDict, cont: OrderedDict, is_list: bool) -> dict: + """ Parse a 'object container'. + 'Object container' represent OBJECT inside Config DB schema: + { + "TABLE": { + "OBJECT": { + "attr": "value" + } + } + } Args: - cont: reference to 'container' + y_module: reference to 'module' + cont: reference to 'object container' Returns: dictionary - element for y2d_elem['static-objects'] OR y2d_elem['dynamic-objects'] """ @@ -164,30 +159,63 @@ def on_container(cont: OrderedDict, is_list: bool) -> dict: } if is_list: - obj_elem['key'] = cont.get('key').get('@value') + obj_elem['keys'] = get_list_keys(cont) attrs_list = list() + attrs_list.extend(get_leafs(cont)) + attrs_list.extend(get_leaf_lists(cont)) + attrs_list.extend(get_choices(y_module, cont)) + # TODO: need to test 'grouping' + #attrs_list.extend(get_uses_grouping(y_module, cont)) - if cont.get('leaf') is not None: - is_leaf_list = False - ret_leafs = on_leafs(cont.get('leaf'), is_leaf_list) - attrs_list.extend(ret_leafs) + obj_elem['attrs'] = attrs_list - if cont.get('leaf-list') is not None: - is_leaf_list = True - ret_leaf_lists = on_leafs(cont.get('leaf-list'), is_leaf_list) - attrs_list.extend(ret_leaf_lists) + return obj_elem - if cont.get('choice') is not None: - y_choices = cont.get('choice') - ret_choice_leafs = on_choices(y_choices) - attrs_list.extend(ret_choice_leafs) +def on_grouping(y_module: OrderedDict, y_grouping, y_uses) -> list: + """ Parse a YANG 'grouping' and 'uses' entities + 'grouping' element can have - 'leaf', 'leaf-list', 'choice' - obj_elem['attrs'] = attrs_list + Args: + y_module: reference to 'module' + y_grouping: reference to 'grouping' + y_uses: reference to 'uses' + Returns: + dictionary - element for obj_elem['attrs'], 'attrs' contain a parsed 'leafs' + """ - return obj_elem + ret_attrs = list() + + if isinstance(y_uses, list): + if isinstance(y_grouping, list): + for use in y_uses: + for group in y_grouping: + if use.get('@name') == group.get('@name'): + ret_attrs.extend(get_leafs(group)) + ret_attrs.extend(get_leaf_lists(group)) + ret_attrs.extend(get_choices(y_module, group)) + else: + for use in y_uses: + if use.get('@name') == y_grouping.get('@name'): + ret_attrs.extend(get_leafs(y_grouping)) + ret_attrs.extend(get_leaf_lists(y_grouping)) + ret_attrs.extend(get_choices(y_module, y_grouping)) + else: + if isinstance(y_grouping, list): + for group in y_grouping: + if y_uses.get('@name') == group.get('@name'): + ret_attrs.extend(get_leafs(group)) + ret_attrs.extend(get_leaf_lists(group)) + ret_attrs.extend(get_choices(y_module, group)) + else: + if y_uses.get('@name') == y_grouping.get('@name'): + ret_attrs.extend(get_leafs(y_grouping)) + ret_attrs.extend(get_leaf_lists(y_grouping)) + ret_attrs.extend(get_choices(y_module, y_grouping)) + + return ret_attrs -def on_choices(y_choices) -> list: +def on_choices(y_module: OrderedDict, y_choices) -> list: """ Parse a YANG 'choice' entities Args: @@ -201,18 +229,20 @@ def on_choices(y_choices) -> list: # the YANG model can have multiple 'choice' entities inside 'container' or 'list' if isinstance(y_choices, list): for choice in y_choices: - attrs = on_choice_cases(choice.get('case')) + attrs = on_choice_cases(y_module, choice.get('case')) ret_attrs.extend(attrs) else: - ret_attrs = on_choice_cases(y_choices.get('case')) + ret_attrs = on_choice_cases(y_module, y_choices.get('case')) return ret_attrs -def on_choice_cases(y_cases: list) -> list: +def on_choice_cases(y_module: OrderedDict, y_cases: list) -> list: """ Parse a single YANG 'case' entity from 'choice' entity + 'case' element can have inside - 'leaf', 'leaf-list', 'uses' Args: - cont: reference to 'case' + y_module: reference to 'module' + y_cases: reference to 'case' Returns: dictionary - element for obj_elem['attrs'], 'attrs' contain a parsed 'leafs' """ @@ -221,15 +251,10 @@ def on_choice_cases(y_cases: list) -> list: if isinstance(y_cases, list): for case in y_cases: - if case.get('leaf') is not None: - is_leaf_list = False - ret_leafs = on_leafs(case.get('leaf'), is_leaf_list) - ret_attrs.extend(ret_leafs) - - if case.get('leaf-list') is not None: - is_leaf_list = True - ret_leaf_lists = on_leafs(case.get('leaf-list'), is_leaf_list) - ret_attrs.extend(ret_leaf_lists) + ret_attrs.extend(get_leafs(case)) + ret_attrs.extend(get_leaf_lists(case)) + # TODO: need to deeply test it + #ret_attrs.extend(get_uses_grouping(y_module, case)) else: raise Exception('It has no sense to using a single "case" element inside "choice" element') @@ -272,5 +297,54 @@ def on_leaf(leaf: OrderedDict, is_leaf_list: bool) -> dict: attr = { 'name': leaf.get('@name'), 'description': get_description(leaf), 'is-leaf-list': is_leaf_list, - 'mandatory': mandatory } - return attr \ No newline at end of file + 'is-mandatory': mandatory } + return attr + +#----------------------GETERS-------------------------# + +def get_description(y_entity: OrderedDict) -> str: + """ Parse 'description' entity from any YANG element + + Args: + y_entity: reference to YANG 'container' OR 'list' OR 'leaf' ... + Returns: + str - text of the 'description' + """ + + if y_entity.get('description') is not None: + return y_entity.get('description').get('text') + else: + return '' + +def get_leafs(y_entity: OrderedDict) -> list: + if y_entity.get('leaf') is not None: + return on_leafs(y_entity.get('leaf'), is_leaf_list=False) + + return [] + +def get_leaf_lists(y_entity: OrderedDict) -> list: + if y_entity.get('leaf-list') is not None: + return on_leafs(y_entity.get('leaf-list'), is_leaf_list=True) + + return [] + +def get_choices(y_module: OrderedDict, y_entity: OrderedDict) -> list: + if y_entity.get('choice') is not None: + return on_choices(y_module, y_entity.get('choice')) + + return [] + +def get_uses_grouping(y_module: OrderedDict, y_entity: OrderedDict) -> list: + if y_entity.get('uses') is not None and y_module.get('grouping') is not None: + return on_grouping(y_module, y_module.get('grouping'), y_entity.get('uses')) + + return [] + +def get_list_keys(y_list: OrderedDict) -> list: + ret_list = list() + keys = y_list.get('key').get('@value').split() + for k in keys: + key = { 'name': k } + ret_list.append(key) + + return ret_list \ No newline at end of file From 718f8abbb5611556534403e00ed9766d2bdb4fe6 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Wed, 12 May 2021 14:37:59 +0300 Subject: [PATCH 16/76] align with implementation Signed-off-by: Stepan Blyschak --- .../templates/sonic-cli-gen/config.py.j2 | 19 ++++++++++--------- .../templates/sonic-cli-gen/show.py.j2 | 14 +++++++------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 index 4d70f9d1e8..10fca2512f 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 @@ -122,7 +122,7 @@ def clear_list_entry_validated(db, table, key, attr): {%- for arg in args %} @click.argument( "{{ cli_name(arg.name) }}", - nargs={% if arg.is_list %}-1{% else %}1{% endif %}, + nargs={% if arg["is-leaf-list"] %}-1{% else %}1{% endif %}, required=True, ) {%- endfor %} @@ -132,6 +132,7 @@ def clear_list_entry_validated(db, table, key, attr): {%- for opt in opts %} @click.option( "--{{ cli_name(opt.name) }}", + help="{{ opt.description }}{% if opt.['is-mandatory'] %}[mandatory]{% endif %}, ) {%- endfor %} {%- endmacro %} @@ -237,7 +238,7 @@ def {{ list_update_group }}_delete( {% macro config_object_list_update_all(group, table, object) %} {% for attr in object.attrs %} -{% if attr.is_list %} +{% if attr["is-leaf-list"] %} {{ config_object_list_update(group, table, object, attr) }} {% endif %} {% endfor %} @@ -314,7 +315,7 @@ def {{ group }}_add(db, {{ pythonize(object["keys"] + object.attrs) }}): data = {} {%- for attr in object.attrs %} if {{ pythonize([attr]) }} is not None: -{%- if not attr.is_list %} +{%- if not attr["is-leaf-list"] %} data["{{ attr.name }}"] = {{ pythonize([attr]) }} {%- else %} data["{{ attr.name }}"] = {{ pythonize([attr]) }}.split(",") @@ -350,7 +351,7 @@ def {{ group }}_update(db, {{ pythonize(object["keys"] + object.attrs) }}): data = {} {%- for attr in object.attrs %} if {{ pythonize([attr]) }} is not None: -{%- if not attr.is_list %} +{%- if not attr["is-leaf-list"] %} data["{{ attr.name }}"] = {{ pythonize([attr]) }} {%- else %} data["{{ attr.name }}"] = {{ pythonize([attr]) }}.split(",") @@ -387,7 +388,7 @@ def {{ group }}_delete(db, {{ pythonize(object["keys"]) }}): {% macro config_dynamic_object(table, object) %} {# Generate another nesting group in case table holds two types of objects #} -{% if table.dynamic_objects|length > 1 %} +{% if table["dynamic-objects"]|length > 1 %} {% set group = table.name + "_" + object.name %} @{{ table.name }}.group(name="{{ cli_name(object.name) }}", cls=clicommon.AliasedGroup) @@ -414,12 +415,12 @@ def {{ table.name }}(): pass -{% if "static_objects" in table %} -{% for object in table.static_objects %} +{% if "static-objects" in table %} +{% for object in table["static-objects"] %} {{ config_static_object(table, object) }} {% endfor %} -{% elif "dynamic_objects" in table %} -{% for object in table.dynamic_objects %} +{% elif "dynamic-objects" in table %} +{% for object in table["dynamic-objects"] %} {{ config_dynamic_object(table, object) }} {% endfor %} {% endif %} diff --git a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 index 191a466852..98b06e1e86 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 @@ -7,7 +7,7 @@ import natsort import utilities_common.cli as clicommon {% macro print_attr(attr) %} -{%- if not attr.is_list %} +{%- if not attr["is-leaf-list"] %} entry.get("{{ attr.name }}", "N/A") {%- else %} "\n".join(entry.get("{{ attr.name }}", [])) @@ -23,7 +23,7 @@ entry.get("{{ attr.name }}", "N/A") {% for table in tables %} -{% if "static_objects" in table %} +{% if "static-objects" in table %} @click.group(name="{{ cli_name(table.name) }}", cls=clicommon.AliasedGroup) def {{ table.name }}(): @@ -31,7 +31,7 @@ def {{ table.name }}(): pass -{% for object in table.static_objects %} +{% for object in table["static-objects"] %} @{{ table.name }}.command(name="{{ cli_name(object.name) }}") @clicommon.pass_db def {{ table.name }}_{{ object.name }}(db): @@ -47,8 +47,8 @@ def {{ table.name }}_{{ object.name }}(db): click.echo(tabulate.tabulate(body, header)) {% endfor %} -{% elif "dynamic_objects" in table %} -{% if table.dynamic_objects|length > 1 %} +{% elif "dynamic-objects" in table %} +{% if table["dynamic-objects"]|length > 1 %} @click.group(name="{{ cli_name(table.name) }}", cls=clicommon.AliasedGroup) def {{ table.name }}(): @@ -56,9 +56,9 @@ def {{ table.name }}(): pass {% endif %} -{% for object in table.dynamic_objects %} +{% for object in table["dynamic-objects"] %} {# Generate another nesting group in case table holds two types of objects #} -{% if table.dynamic_objects|length > 1 %} +{% if table["dynamic-objects"]|length > 1 %} {% set group = table.name %} {% set name = object.name %} {% else %} From fa6299c55044a272a415308233b7633a88552412 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Wed, 12 May 2021 14:46:00 +0300 Subject: [PATCH 17/76] generate cli from templates Signed-off-by: Stepan Blyschak --- sonic_cli_gen/generator.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py index 6a3b6f2283..373fd55774 100644 --- a/sonic_cli_gen/generator.py +++ b/sonic_cli_gen/generator.py @@ -1,9 +1,8 @@ #!/usr/bin/env python -try: - from sonic_cli_gen.yang_parser import YangParser -except ImportError as e: - raise ImportError("%s - required module not found" % str(e)) +import jinja2 + +from sonic_cli_gen.yang_parser import YangParser class CliGenerator: """ SONiC CLI generator. This class provides public API @@ -16,23 +15,30 @@ def __init__(self, """ Initialize PackageManager. """ self.yang_model_name = yang_model + self.loader = jinja2.FileSystemLoader(['/usr/share/sonic/templates/sonic-cli-gen/']) + self.env = jinja2.Environment(loader=self.loader) def generate_config_plugin(self): """ Generate CLI plugin for 'config' CLI group. """ parser = YangParser(self.yang_model_name) yang_dict = parser.parse_yang_model() - import pprint; pprint.pprint(yang_dict) - pass + template = self.env.get_template('config.py.j2') + with open('config.py', 'w') as config_py: + config_py.write(template.render(yang_dict)) + def generate_show_plugin(self): """ Generate CLI plugin for 'show' CLI group. """ parser = YangParser(self.yang_model_name) yang_dict = parser.parse_yang_model() - pass + template = self.env.get_template('show.py.j2') + with open('show.py', 'w') as show_py: + show_py.write(template.render(yang_dict)) # to be implemented in the next Phases def generate_sonic_clear_plugin(self): """ Generate CLI plugin for 'sonic-clear' CLI group. """ parser = YangParser(self.yang_model_name) yang_dict = parser.parse_yang_model() - pass + raise NotImplementedError + From 9d9f66ba5cb7aa35468754d49e6fb3ea00d405fa Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Wed, 12 May 2021 12:14:59 +0000 Subject: [PATCH 18/76] Table can only have 'static-objects' OR 'dynamic-objects' Signed-off-by: Vadym Hlushko --- sonic_cli_gen/yang_parser.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index a7de4885d8..5556727924 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -104,13 +104,14 @@ def on_table_container(y_module: OrderedDict, tbl_cont: OrderedDict) -> dict: y2d_elem = { 'name': tbl_cont.get('@name'), - 'description': get_description(tbl_cont), - 'dynamic-objects': list(), - 'static-objects': list() + 'description': get_description(tbl_cont) } # determine if 'table container' have a 'list' entity if tbl_cont.get('list') is None: + # TODO: comment aboit it + y2d_elem['static-objects'] = list() + # 'object' container goes after 'table' container # 'object' container have 2 types - list (like sonic-flex_counter.yang) and NOT list (like sonic-device_metadata.yang) obj_cont = tbl_cont.get('container') @@ -122,6 +123,10 @@ def on_table_container(y_module: OrderedDict, tbl_cont: OrderedDict) -> dict: static_obj_elem = on_object_container(y_module, obj_cont, is_list=False) y2d_elem['static-objects'].append(static_obj_elem) else: + + # TODO: comment aboit it + y2d_elem['dynamic-objects'] = list() + tbl_cont_lists = tbl_cont.get('list') # 'container' can have more than 1 'list' if isinstance(tbl_cont_lists, list): From 38f4365881d2e4be523051aa8c837160566042a4 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Wed, 12 May 2021 15:33:14 +0300 Subject: [PATCH 19/76] add missing " in help for option Signed-off-by: Stepan Blyschak --- sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 index 10fca2512f..1ad0341656 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 @@ -132,7 +132,7 @@ def clear_list_entry_validated(db, table, key, attr): {%- for opt in opts %} @click.option( "--{{ cli_name(opt.name) }}", - help="{{ opt.description }}{% if opt.['is-mandatory'] %}[mandatory]{% endif %}, + help="{{ opt.description }}{% if opt['is-mandatory'] %}[mandatory]{% endif %}", ) {%- endfor %} {%- endmacro %} @@ -419,7 +419,8 @@ def {{ table.name }}(): {% for object in table["static-objects"] %} {{ config_static_object(table, object) }} {% endfor %} -{% elif "dynamic-objects" in table %} + +{% if "dynamic-objects" in table %} {% for object in table["dynamic-objects"] %} {{ config_dynamic_object(table, object) }} {% endfor %} From 0826ac5bd835e74842e1310f1bf56584350f757d Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Wed, 12 May 2021 12:56:32 +0000 Subject: [PATCH 20/76] Added remove_keys(), get_mandatory() Signed-off-by: Vadym Hlushko --- sonic_cli_gen/yang_parser.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index 5556727924..89a9822689 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -137,8 +137,21 @@ def on_table_container(y_module: OrderedDict, tbl_cont: OrderedDict) -> dict: dynamic_obj_elem = on_object_container(y_module, tbl_cont_lists, is_list=True) y2d_elem['dynamic-objects'].append(dynamic_obj_elem) + # remove 'keys' elementsfrom 'attrs' list + remove_keys(y2d_elem['dynamic-objects']) + return y2d_elem +# TODO: think about name +def remove_keys(dynamic_objects: OrderedDict): + for obj in dynamic_objects: + for key in obj.get('keys'): + for attr in obj.get('attrs'): + if key.get('name') == attr.get('name'): + key['description'] = attr.get('description') + obj['attrs'].remove(attr) + break + def on_object_container(y_module: OrderedDict, cont: OrderedDict, is_list: bool) -> dict: """ Parse a 'object container'. 'Object container' represent OBJECT inside Config DB schema: @@ -295,18 +308,21 @@ def on_leaf(leaf: OrderedDict, is_leaf_list: bool) -> dict: dictionary - parsed 'leaf' element """ - mandatory = False - if leaf.get('mandatory') is not None: - mandatory = True - attr = { 'name': leaf.get('@name'), 'description': get_description(leaf), 'is-leaf-list': is_leaf_list, - 'is-mandatory': mandatory } + 'is-mandatory': get_mandatory(leaf) } + return attr #----------------------GETERS-------------------------# +def get_mandatory(y_leaf: OrderedDict) -> bool: + if y_leaf.get('mandatory') is not None: + return True + + return False + def get_description(y_entity: OrderedDict) -> str: """ Parse 'description' entity from any YANG element From 069000f84a2a5b3cac8df0d013e9510ebbf42b62 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Wed, 12 May 2021 16:56:39 +0300 Subject: [PATCH 21/76] fix issues Signed-off-by: Stepan Blyschak --- .../templates/sonic-cli-gen/config.py.j2 | 10 +++++++++- .../templates/sonic-cli-gen/show.py.j2 | 5 ++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 index 1ad0341656..ac19835cd1 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 @@ -2,8 +2,13 @@ """ Autogenerated config CLI plugin """ import click -from config import config_mgmt import utilities_common.cli as clicommon +import utilities_common.general as general +from config import config_mgmt + + +# Load sonic-cfggen from source since /usr/local/bin/sonic-cfggen does not have .py extension. +sonic_cfggen = general.load_module_from_source('sonic_cfggen', '/usr/local/bin/sonic-cfggen') def exit_cli(*args, **kwargs): @@ -17,6 +22,7 @@ def validate_config_or_raise(cfg): """ Validate config db data using ConfigMgmt """ try: + cfg = sonic_cfggen.FormatConverter.to_serialized(cfg) config_mgmt.ConfigMgmt().loadData(cfg) except Exception as err: raise Exception('Failed to validate configuration: {}'.format(err)) @@ -419,12 +425,14 @@ def {{ table.name }}(): {% for object in table["static-objects"] %} {{ config_static_object(table, object) }} {% endfor %} +{% endif %} {% if "dynamic-objects" in table %} {% for object in table["dynamic-objects"] %} {{ config_dynamic_object(table, object) }} {% endfor %} {% endif %} + {% endfor %} def register(cli): diff --git a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 index 98b06e1e86..58e7dfd2e0 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 @@ -41,7 +41,7 @@ def {{ table.name }}_{{ object.name }}(db): body = [] table = db.cfgdb.get_table("{{ table.name }}") - entry = table.get("{{ object.name }}") + entry = table.get("{{ object.name }}", {}) row = [{%- for attr in object.attrs -%} {{ print_attr(attr) }}, {%- endfor %}] body.append(row) click.echo(tabulate.tabulate(body, header)) @@ -77,8 +77,7 @@ def {{ name }}(db): body = [] table = db.cfgdb.get_table("{{ table.name }}") - for key in natsort.natsorted(table): - entry = table[key] + for key, entry in natsort.natsorted(table).items(): if not isinstance(key, tuple): key = (key,) From 1723f0f29050a6ba4a8be4ce4ec9f6887dd60c8a Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Thu, 13 May 2021 12:33:15 +0000 Subject: [PATCH 22/76] Refactored generator.py: generate_cli_plugin(), get_cli_plugin_path() Signed-off-by: Vadym Hlushko --- sonic_cli_gen/generator.py | 38 +++++++++++++++--------------------- sonic_cli_gen/main.py | 15 ++++---------- sonic_cli_gen/yang_parser.py | 7 ++----- 3 files changed, 22 insertions(+), 38 deletions(-) diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py index 373fd55774..0a02a104bc 100644 --- a/sonic_cli_gen/generator.py +++ b/sonic_cli_gen/generator.py @@ -1,6 +1,8 @@ #!/usr/bin/env python import jinja2 +import os +import pkgutil from sonic_cli_gen.yang_parser import YangParser @@ -18,27 +20,19 @@ def __init__(self, self.loader = jinja2.FileSystemLoader(['/usr/share/sonic/templates/sonic-cli-gen/']) self.env = jinja2.Environment(loader=self.loader) - def generate_config_plugin(self): - """ Generate CLI plugin for 'config' CLI group. """ + def generate_cli_plugin(self, cli_group, plugin_name): + """ Generate CLI plugin. """ parser = YangParser(self.yang_model_name) yang_dict = parser.parse_yang_model() - template = self.env.get_template('config.py.j2') - with open('config.py', 'w') as config_py: - config_py.write(template.render(yang_dict)) - - - def generate_show_plugin(self): - """ Generate CLI plugin for 'show' CLI group. """ - parser = YangParser(self.yang_model_name) - yang_dict = parser.parse_yang_model() - template = self.env.get_template('show.py.j2') - with open('show.py', 'w') as show_py: - show_py.write(template.render(yang_dict)) - - # to be implemented in the next Phases - def generate_sonic_clear_plugin(self): - """ Generate CLI plugin for 'sonic-clear' CLI group. """ - parser = YangParser(self.yang_model_name) - yang_dict = parser.parse_yang_model() - raise NotImplementedError - + plugin_path = get_cli_plugin_path(cli_group, plugin_name + '_yang.py') + template = self.env.get_template(cli_group + '.py.j2') + with open(plugin_path, 'w') as plugin_py: + plugin_py.write(template.render(yang_dict)) + +def get_cli_plugin_path(command, plugin_name): + pkg_loader = pkgutil.get_loader(f'{command}.plugins') + if pkg_loader is None: + raise PackageManagerError(f'Failed to get plugins path for {command} CLI') + plugins_pkg_path = os.path.dirname(pkg_loader.path) + + return os.path.join(plugins_pkg_path, plugin_name) \ No newline at end of file diff --git a/sonic_cli_gen/main.py b/sonic_cli_gen/main.py index bf783e8541..8cf748e394 100644 --- a/sonic_cli_gen/main.py +++ b/sonic_cli_gen/main.py @@ -1,12 +1,7 @@ #!/usr/bin/env python -try: - import sys - import os - import click - from sonic_cli_gen.generator import CliGenerator -except ImportError as e: - raise ImportError("%s - required module not found" % str(e)) +import click +from sonic_cli_gen.generator import CliGenerator @click.group() @click.pass_context @@ -20,8 +15,7 @@ def cli(ctx): def generate_config(ctx, yang_model_name): """ Generate CLI plugin (click) for 'config' CLI group. """ gen = CliGenerator(yang_model_name) - gen.generate_config_plugin() - pass + gen.generate_cli_plugin(cli_group='config', plugin_name=yang_model_name) @cli.command() @click.argument('yang_model_name') @@ -29,8 +23,7 @@ def generate_config(ctx, yang_model_name): def generate_show(ctx, yang_model_name): """ Generate CLI plugin (click) for 'show' CLI group. """ gen = CliGenerator(yang_model_name) - gen.generate_show_plugin() - pass + gen.generate_cli_plugin(cli_group='show', plugin_name=yang_model_name) if __name__ == '__main__': cli() \ No newline at end of file diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index 89a9822689..a67a3d06d5 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -1,10 +1,7 @@ #!/usr/bin/env python -try: - from collections import OrderedDict - from config.config_mgmt import ConfigMgmt -except ImportError as e: - raise ImportError("%s - required module not found" % str(e)) +from collections import OrderedDict +from config.config_mgmt import ConfigMgmt class YangParser: """ YANG model parser From 0a78e19ea3ea863905ed49e9d9c9c867ebec94af Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Thu, 13 May 2021 16:36:33 +0000 Subject: [PATCH 23/76] Removed arguments from constructor of class CliGenerator, added function comments Signed-off-by: Vadym Hlushko --- sonic_cli_gen/generator.py | 7 ++-- sonic_cli_gen/main.py | 4 +-- sonic_cli_gen/yang_parser.py | 70 ++++++++++++++++++++++++++---------- 3 files changed, 56 insertions(+), 25 deletions(-) diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py index 0a02a104bc..6214bbe9c5 100644 --- a/sonic_cli_gen/generator.py +++ b/sonic_cli_gen/generator.py @@ -12,18 +12,17 @@ class CliGenerator: show, sonic-clear CLI plugins """ - def __init__(self, - yang_model): + def __init__(self): """ Initialize PackageManager. """ - self.yang_model_name = yang_model self.loader = jinja2.FileSystemLoader(['/usr/share/sonic/templates/sonic-cli-gen/']) self.env = jinja2.Environment(loader=self.loader) def generate_cli_plugin(self, cli_group, plugin_name): """ Generate CLI plugin. """ - parser = YangParser(self.yang_model_name) + parser = YangParser(plugin_name) yang_dict = parser.parse_yang_model() + #import pprint; pprint.pprint(yang_dict) plugin_path = get_cli_plugin_path(cli_group, plugin_name + '_yang.py') template = self.env.get_template(cli_group + '.py.j2') with open(plugin_path, 'w') as plugin_py: diff --git a/sonic_cli_gen/main.py b/sonic_cli_gen/main.py index 8cf748e394..0ee0b030f5 100644 --- a/sonic_cli_gen/main.py +++ b/sonic_cli_gen/main.py @@ -14,7 +14,7 @@ def cli(ctx): @click.pass_context def generate_config(ctx, yang_model_name): """ Generate CLI plugin (click) for 'config' CLI group. """ - gen = CliGenerator(yang_model_name) + gen = CliGenerator() gen.generate_cli_plugin(cli_group='config', plugin_name=yang_model_name) @cli.command() @@ -22,7 +22,7 @@ def generate_config(ctx, yang_model_name): @click.pass_context def generate_show(ctx, yang_model_name): """ Generate CLI plugin (click) for 'show' CLI group. """ - gen = CliGenerator(yang_model_name) + gen = CliGenerator() gen.generate_cli_plugin(cli_group='show', plugin_name=yang_model_name) if __name__ == '__main__': diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index a67a3d06d5..46bb147be6 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -14,7 +14,35 @@ class YangParser: y_top_level_container: Reference to top level 'container' entity from YANG model file y_table_containers: Reference to 'container' entities from YANG model file that represent Config DB tables - yang_2_dict: dictionary created from YANG model file that represent Config DB schema + yang_2_dict: dictionary created from YANG model file that represent Config DB schema. + In case if YANG model has a 'list' entity: + { + 'tables': [{ + 'name': 'value', + 'description': 'value', + 'dynamic-objects': [ + 'name': 'value', + 'description': 'value, + 'attrs': [ + { + 'name': 'value', + 'descruption': 'value', + 'is-leaf-list': False, + 'is-mandatory': False + } + ... + ], + 'keys': [ + { + 'name': 'ACL_TABLE_NAME', + 'description': 'value' + } + ... + ] + ], + }] + } + In case if YANG model does NOT have a 'list' entity, it has the same structure as above, but 'dynamic-objects' changed to 'static-objects' and have no 'keys' """ def __init__(self, yang_model_name): @@ -106,7 +134,6 @@ def on_table_container(y_module: OrderedDict, tbl_cont: OrderedDict) -> dict: # determine if 'table container' have a 'list' entity if tbl_cont.get('list') is None: - # TODO: comment aboit it y2d_elem['static-objects'] = list() # 'object' container goes after 'table' container @@ -120,12 +147,9 @@ def on_table_container(y_module: OrderedDict, tbl_cont: OrderedDict) -> dict: static_obj_elem = on_object_container(y_module, obj_cont, is_list=False) y2d_elem['static-objects'].append(static_obj_elem) else: - - # TODO: comment aboit it y2d_elem['dynamic-objects'] = list() - tbl_cont_lists = tbl_cont.get('list') - # 'container' can have more than 1 'list' + # 'container' can have more than 1 'list' entity if isinstance(tbl_cont_lists, list): for _list in tbl_cont_lists: dynamic_obj_elem = on_object_container(y_module, _list, is_list=True) @@ -134,21 +158,11 @@ def on_table_container(y_module: OrderedDict, tbl_cont: OrderedDict) -> dict: dynamic_obj_elem = on_object_container(y_module, tbl_cont_lists, is_list=True) y2d_elem['dynamic-objects'].append(dynamic_obj_elem) - # remove 'keys' elementsfrom 'attrs' list - remove_keys(y2d_elem['dynamic-objects']) + # move 'keys' elements from 'attrs' to 'keys' + change_dyn_obj_struct(y2d_elem['dynamic-objects']) return y2d_elem -# TODO: think about name -def remove_keys(dynamic_objects: OrderedDict): - for obj in dynamic_objects: - for key in obj.get('keys'): - for attr in obj.get('attrs'): - if key.get('name') == attr.get('name'): - key['description'] = attr.get('description') - obj['attrs'].remove(attr) - break - def on_object_container(y_module: OrderedDict, cont: OrderedDict, is_list: bool) -> dict: """ Parse a 'object container'. 'Object container' represent OBJECT inside Config DB schema: @@ -365,4 +379,22 @@ def get_list_keys(y_list: OrderedDict) -> list: key = { 'name': k } ret_list.append(key) - return ret_list \ No newline at end of file + return ret_list + +def change_dyn_obj_struct(dynamic_objects: OrderedDict): + """ Rearrange self.yang_2_dict['dynamic_objects'] structure. + If YANG model have a 'list' entity - inside the 'list' it has 'key' entity. + 'key' entity it is whitespace-separeted list of 'leafs', those 'leafs' was + parsed by 'on_leaf()' function and placed under 'attrs' in self.yang_2_dict['dynamic_objects'] + need to move 'leafs' from 'attrs' and put them to 'keys' section of elf.yang_2_dict['dynamic_objects'] + + Args: + dynamic_objects: reference to self.yang_2_dict['dynamic_objects'] + """ + for obj in dynamic_objects: + for key in obj.get('keys'): + for attr in obj.get('attrs'): + if key.get('name') == attr.get('name'): + key['description'] = attr.get('description') + obj['attrs'].remove(attr) + break \ No newline at end of file From 30d2cb9c774db6df42121a0f486362f07602c5d8 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Fri, 14 May 2021 15:17:36 +0000 Subject: [PATCH 24/76] Added sceleton for UT, added additional check for yang_parser.py, changed constructor for YangParser Signed-off-by: Vadym Hlushko --- sonic_cli_gen/generator.py | 7 +- sonic_cli_gen/yang_parser.py | 12 +- tests/cli_autogen_input/config_db.json | 544 ++++++++++++++++++ .../sonic-one-table-container.yang | 19 + tests/cli_autogen_input/sonic-vlan.yang | 190 ++++++ tests/cli_autogen_yang_parser_test.py | 56 ++ 6 files changed, 824 insertions(+), 4 deletions(-) create mode 100644 tests/cli_autogen_input/config_db.json create mode 100644 tests/cli_autogen_input/sonic-one-table-container.yang create mode 100644 tests/cli_autogen_input/sonic-vlan.yang create mode 100644 tests/cli_autogen_yang_parser_test.py diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py index 6214bbe9c5..f3fcf0e62b 100644 --- a/sonic_cli_gen/generator.py +++ b/sonic_cli_gen/generator.py @@ -20,9 +20,12 @@ def __init__(self): def generate_cli_plugin(self, cli_group, plugin_name): """ Generate CLI plugin. """ - parser = YangParser(plugin_name) + + parser = YangParser(yang_model_name=plugin_name, + config_db_path='configDB', + allow_tbl_without_yang=True, + debug=False) yang_dict = parser.parse_yang_model() - #import pprint; pprint.pprint(yang_dict) plugin_path = get_cli_plugin_path(cli_group, plugin_name + '_yang.py') template = self.env.get_template(cli_group + '.py.j2') with open(plugin_path, 'w') as plugin_py: diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index 46bb147be6..b36d862ee8 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -45,7 +45,10 @@ class YangParser: In case if YANG model does NOT have a 'list' entity, it has the same structure as above, but 'dynamic-objects' changed to 'static-objects' and have no 'keys' """ def __init__(self, - yang_model_name): + yang_model_name, + config_db_path, + allow_tbl_without_yang, + debug): self.yang_model_name = yang_model_name self.conf_mgmt = None self.idx_yJson = None @@ -55,7 +58,9 @@ def __init__(self, self.yang_2_dict = dict() try: - self.conf_mgmt = ConfigMgmt() + self.conf_mgmt = ConfigMgmt(source=config_db_path, + debug=debug, + allowTablesWithoutYang=allow_tbl_without_yang) except Exception as e: raise Exception("Failed to load the {} class".format(str(e))) @@ -181,6 +186,9 @@ def on_object_container(y_module: OrderedDict, cont: OrderedDict, is_list: bool) dictionary - element for y2d_elem['static-objects'] OR y2d_elem['dynamic-objects'] """ + if cont is None: + return {} + obj_elem = { 'name': cont.get('@name'), 'description': get_description(cont), diff --git a/tests/cli_autogen_input/config_db.json b/tests/cli_autogen_input/config_db.json new file mode 100644 index 0000000000..5473d6158a --- /dev/null +++ b/tests/cli_autogen_input/config_db.json @@ -0,0 +1,544 @@ +{ + "COPP_GROUP": { + "default": { + "cbs": "600", + "cir": "600", + "meter_type": "packets", + "mode": "sr_tcm", + "queue": "0", + "red_action": "drop" + }, + "queue1_group1": { + "cbs": "6000", + "cir": "6000", + "meter_type": "packets", + "mode": "sr_tcm", + "queue": "1", + "red_action": "drop", + "trap_action": "trap", + "trap_priority": "1" + }, + "queue1_group2": { + "cbs": "600", + "cir": "600", + "meter_type": "packets", + "mode": "sr_tcm", + "queue": "1", + "red_action": "drop", + "trap_action": "trap", + "trap_priority": "1" + }, + "queue2_group1": { + "cbs": "1000", + "cir": "1000", + "genetlink_mcgrp_name": "packets", + "genetlink_name": "psample", + "meter_type": "packets", + "mode": "sr_tcm", + "queue": "2", + "red_action": "drop", + "trap_action": "trap", + "trap_priority": "1" + }, + "queue4_group1": { + "cbs": "600", + "cir": "600", + "color": "blind", + "meter_type": "packets", + "mode": "sr_tcm", + "queue": "4", + "red_action": "drop", + "trap_action": "trap", + "trap_priority": "4" + }, + "queue4_group2": { + "cbs": "600", + "cir": "600", + "meter_type": "packets", + "mode": "sr_tcm", + "queue": "4", + "red_action": "drop", + "trap_action": "copy", + "trap_priority": "4" + }, + "queue4_group3": { + "cbs": "600", + "cir": "600", + "color": "blind", + "meter_type": "packets", + "mode": "sr_tcm", + "queue": "4", + "red_action": "drop", + "trap_action": "trap", + "trap_priority": "4" + } + }, + "COPP_TRAP": { + "arp": { + "trap_group": "queue4_group2", + "trap_ids": "arp_req,arp_resp,neigh_discovery" + }, + "bgp": { + "trap_group": "queue4_group1", + "trap_ids": "bgp,bgpv6" + }, + "dhcp": { + "trap_group": "queue4_group3", + "trap_ids": "dhcp,dhcpv6" + }, + "ip2me": { + "trap_group": "queue1_group1", + "trap_ids": "ip2me" + }, + "lacp": { + "trap_group": "queue4_group1", + "trap_ids": "lacp" + }, + "lldp": { + "trap_group": "queue4_group3", + "trap_ids": "lldp" + }, + "nat": { + "trap_group": "queue1_group2", + "trap_ids": "src_nat_miss,dest_nat_miss" + }, + "sflow": { + "trap_group": "queue2_group1", + "trap_ids": "sample_packet" + }, + "ssh": { + "trap_group": "queue4_group2", + "trap_ids": "ssh" + }, + "udld": { + "trap_group": "queue4_group3", + "trap_ids": "udld" + } + }, + "CRM": { + "Config": { + "acl_counter_high_threshold": "85", + "acl_counter_low_threshold": "70", + "acl_counter_threshold_type": "percentage", + "acl_entry_high_threshold": "85", + "acl_entry_low_threshold": "70", + "acl_entry_threshold_type": "percentage", + "acl_group_high_threshold": "85", + "acl_group_low_threshold": "70", + "acl_group_threshold_type": "percentage", + "acl_table_high_threshold": "85", + "acl_table_low_threshold": "70", + "acl_table_threshold_type": "percentage", + "dnat_entry_high_threshold": "85", + "dnat_entry_low_threshold": "70", + "dnat_entry_threshold_type": "percentage", + "fdb_entry_high_threshold": "85", + "fdb_entry_low_threshold": "70", + "fdb_entry_threshold_type": "percentage", + "ipmc_entry_high_threshold": "85", + "ipmc_entry_low_threshold": "70", + "ipmc_entry_threshold_type": "percentage", + "ipv4_neighbor_high_threshold": "85", + "ipv4_neighbor_low_threshold": "70", + "ipv4_neighbor_threshold_type": "percentage", + "ipv4_nexthop_high_threshold": "85", + "ipv4_nexthop_low_threshold": "70", + "ipv4_nexthop_threshold_type": "percentage", + "ipv4_route_high_threshold": "85", + "ipv4_route_low_threshold": "70", + "ipv4_route_threshold_type": "percentage", + "ipv6_neighbor_high_threshold": "85", + "ipv6_neighbor_low_threshold": "70", + "ipv6_neighbor_threshold_type": "percentage", + "ipv6_nexthop_high_threshold": "85", + "ipv6_nexthop_low_threshold": "70", + "ipv6_nexthop_threshold_type": "percentage", + "ipv6_route_high_threshold": "85", + "ipv6_route_low_threshold": "70", + "ipv6_route_threshold_type": "percentage", + "nexthop_group_high_threshold": "85", + "nexthop_group_low_threshold": "70", + "nexthop_group_member_high_threshold": "85", + "nexthop_group_member_low_threshold": "70", + "nexthop_group_member_threshold_type": "percentage", + "nexthop_group_threshold_type": "percentage", + "polling_interval": "300", + "snat_entry_high_threshold": "85", + "snat_entry_low_threshold": "70", + "snat_entry_threshold_type": "percentage" + } + }, + "DEVICE_METADATA": { + "localhost": { + "buffer_model": "traditional", + "default_bgp_status": "up", + "default_pfcwd_status": "disable", + "hostname": "r-bulldog-02", + "hwsku": "ACS-MSN2100", + "mac": "98:03:9b:f8:e7:c0", + "platform": "x86_64-mlnx_msn2100-r0", + "type": "ToRRouter" + } + }, + "FEATURE": { + "bgp": { + "auto_restart": "enabled", + "has_global_scope": "False", + "has_per_asic_scope": "True", + "has_timer": "False", + "high_mem_alert": "disabled", + "state": "enabled" + }, + "database": { + "auto_restart": "disabled", + "has_global_scope": "True", + "has_per_asic_scope": "True", + "has_timer": "False", + "high_mem_alert": "disabled", + "state": "enabled" + }, + "dhcp_relay": { + "auto_restart": "enabled", + "has_global_scope": "True", + "has_per_asic_scope": "False", + "has_timer": "False", + "high_mem_alert": "disabled", + "state": "enabled" + }, + "lldp": { + "auto_restart": "enabled", + "has_global_scope": "True", + "has_per_asic_scope": "True", + "has_timer": "False", + "high_mem_alert": "disabled", + "state": "enabled", + "status": "enabled" + }, + "mgmt-framework": { + "auto_restart": "enabled", + "has_global_scope": "True", + "has_per_asic_scope": "False", + "has_timer": "True", + "high_mem_alert": "disabled", + "state": "enabled" + }, + "nat": { + "auto_restart": "enabled", + "has_global_scope": "True", + "has_per_asic_scope": "False", + "has_timer": "False", + "high_mem_alert": "disabled", + "state": "disabled" + }, + "pmon": { + "auto_restart": "enabled", + "has_global_scope": "True", + "has_per_asic_scope": "False", + "has_timer": "False", + "high_mem_alert": "disabled", + "state": "enabled" + }, + "radv": { + "auto_restart": "enabled", + "has_global_scope": "True", + "has_per_asic_scope": "False", + "has_timer": "False", + "high_mem_alert": "disabled", + "state": "enabled" + }, + "sflow": { + "auto_restart": "enabled", + "has_global_scope": "True", + "has_per_asic_scope": "False", + "has_timer": "False", + "high_mem_alert": "disabled", + "state": "disabled" + }, + "snmp": { + "auto_restart": "enabled", + "has_global_scope": "True", + "has_per_asic_scope": "False", + "has_timer": "True", + "high_mem_alert": "disabled", + "state": "enabled" + }, + "swss": { + "auto_restart": "enabled", + "has_global_scope": "False", + "has_per_asic_scope": "True", + "has_timer": "False", + "high_mem_alert": "disabled", + "state": "enabled" + }, + "syncd": { + "auto_restart": "enabled", + "has_global_scope": "False", + "has_per_asic_scope": "True", + "has_timer": "False", + "high_mem_alert": "disabled", + "state": "enabled" + }, + "teamd": { + "auto_restart": "enabled", + "has_global_scope": "False", + "has_per_asic_scope": "True", + "has_timer": "False", + "high_mem_alert": "disabled", + "state": "enabled" + }, + "telemetry": { + "auto_restart": "enabled", + "has_global_scope": "True", + "has_per_asic_scope": "False", + "has_timer": "True", + "high_mem_alert": "disabled", + "state": "enabled" + }, + "what-just-happened": { + "auto_restart": "disabled", + "has_timer": "True", + "high_mem_alert": "disabled", + "state": "enabled" + } + }, + "FLEX_COUNTER_TABLE": { + "BUFFER_POOL_WATERMARK": { + "FLEX_COUNTER_STATUS": "enable" + }, + "PFCWD": { + "FLEX_COUNTER_STATUS": "enable" + }, + "PG_WATERMARK": { + "FLEX_COUNTER_STATUS": "enable" + }, + "PORT": { + "FLEX_COUNTER_STATUS": "enable" + }, + "PORT_BUFFER_DROP": { + "FLEX_COUNTER_STATUS": "enable" + }, + "QUEUE": { + "FLEX_COUNTER_STATUS": "enable" + }, + "QUEUE_WATERMARK": { + "FLEX_COUNTER_STATUS": "enable" + }, + "RIF": { + "FLEX_COUNTER_STATUS": "enable" + } + }, + "KDUMP": { + "config": { + "enabled": "false", + "num_dumps": "3", + "memory": "0M-2G:256M,2G-4G:320M,4G-8G:384M,8G-:448M" + } + }, + "MGMT_INTERFACE": { + "eth0|10.210.25.44/22": { + "gwaddr": "10.210.24.1" + } + }, + "PORT": { + "Ethernet0": { + "admin_status": "up", + "alias": "etp1", + "index": "1", + "lanes": "0,1,2,3", + "speed": "100000" + }, + "Ethernet12": { + "admin_status": "up", + "alias": "etp4a", + "index": "4", + "lanes": "12,13", + "speed": "50000" + }, + "Ethernet14": { + "admin_status": "up", + "alias": "etp4b", + "index": "4", + "lanes": "14,15", + "speed": "50000" + }, + "Ethernet16": { + "admin_status": "up", + "alias": "etp5a", + "index": "5", + "lanes": "16,17", + "speed": "50000" + }, + "Ethernet18": { + "admin_status": "up", + "alias": "etp5b", + "index": "5", + "lanes": "18,19", + "speed": "50000" + }, + "Ethernet20": { + "admin_status": "up", + "alias": "etp6a", + "index": "6", + "lanes": "20", + "speed": "25000" + }, + "Ethernet21": { + "admin_status": "up", + "alias": "etp6b", + "index": "6", + "lanes": "21", + "speed": "25000" + }, + "Ethernet22": { + "admin_status": "up", + "alias": "etp6c", + "index": "6", + "lanes": "22", + "speed": "25000" + }, + "Ethernet23": { + "admin_status": "up", + "alias": "etp6d", + "index": "6", + "lanes": "23", + "speed": "25000" + }, + "Ethernet24": { + "admin_status": "up", + "alias": "etp7a", + "index": "7", + "lanes": "24", + "speed": "25000" + }, + "Ethernet25": { + "admin_status": "up", + "alias": "etp7b", + "index": "7", + "lanes": "25", + "speed": "25000" + }, + "Ethernet26": { + "admin_status": "up", + "alias": "etp7c", + "index": "7", + "lanes": "26", + "speed": "25000" + }, + "Ethernet27": { + "admin_status": "up", + "alias": "etp7d", + "index": "7", + "lanes": "27", + "speed": "25000" + }, + "Ethernet28": { + "admin_status": "up", + "alias": "etp8", + "index": "8", + "lanes": "28,29,30,31", + "speed": "100000" + }, + "Ethernet32": { + "admin_status": "up", + "alias": "etp9", + "index": "9", + "lanes": "32,33,34,35", + "speed": "100000" + }, + "Ethernet36": { + "admin_status": "up", + "alias": "etp10", + "index": "10", + "lanes": "36,37,38,39", + "speed": "100000" + }, + "Ethernet4": { + "admin_status": "up", + "alias": "etp2", + "index": "2", + "lanes": "4,5,6,7", + "speed": "100000" + }, + "Ethernet40": { + "admin_status": "up", + "alias": "etp11", + "index": "11", + "lanes": "40,41,42,43", + "speed": "100000" + }, + "Ethernet44": { + "admin_status": "up", + "alias": "etp12", + "index": "12", + "lanes": "44,45,46,47", + "speed": "100000" + }, + "Ethernet48": { + "admin_status": "up", + "alias": "etp13", + "index": "13", + "lanes": "48,49,50,51", + "speed": "100000" + }, + "Ethernet52": { + "admin_status": "up", + "alias": "etp14", + "index": "14", + "lanes": "52,53,54,55", + "speed": "100000" + }, + "Ethernet56": { + "admin_status": "up", + "alias": "etp15", + "index": "15", + "lanes": "56,57,58,59", + "speed": "100000" + }, + "Ethernet60": { + "admin_status": "up", + "alias": "etp16", + "index": "16", + "lanes": "60,61,62,63", + "speed": "100000" + }, + "Ethernet8": { + "admin_status": "up", + "alias": "etp3", + "index": "3", + "lanes": "8,9,10,11", + "speed": "100000" + } + }, + "SNMP": { + "LOCATION": { + "Location": "public" + } + }, + "SNMP_COMMUNITY": { + "public": { + "TYPE": "RO" + } + }, + "VERSIONS": { + "DATABASE": { + "VERSION": "version_2_0_0" + } + }, + "WJH": { + "global": { + "mode": "debug", + "nice_level": "1", + "pci_bandwidth": "50" + } + }, + "WJH_CHANNEL": { + "forwarding": { + "drop_category_list": "L2,L3,Tunnel", + "type": "raw_and_aggregated" + }, + "layer-1": { + "drop_category_list": "L1", + "type": "raw_and_aggregated" + } + } +} diff --git a/tests/cli_autogen_input/sonic-one-table-container.yang b/tests/cli_autogen_input/sonic-one-table-container.yang new file mode 100644 index 0000000000..ca5d248ece --- /dev/null +++ b/tests/cli_autogen_input/sonic-one-table-container.yang @@ -0,0 +1,19 @@ +module sonic-one-table-container { + + yang-version 1.1; + + namespace "http://github.com/Azure/sonic-one"; + prefix one; + + container sonic-one-table-container { + + container ONE_TABLE { + + description "ONE_TABLE description"; + + leaf random { + type string; + } + } + } +} \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-vlan.yang b/tests/cli_autogen_input/sonic-vlan.yang new file mode 100644 index 0000000000..2962161ef0 --- /dev/null +++ b/tests/cli_autogen_input/sonic-vlan.yang @@ -0,0 +1,190 @@ +module sonic-vlan { + + yang-version 1.1; + + namespace "http://github.com/Azure/sonic-vlan"; + prefix vlan; + + import ietf-inet-types { + prefix inet; + } + + import sonic-types { + prefix stypes; + revision-date 2019-07-01; + } + + import sonic-extension { + prefix ext; + revision-date 2019-07-01; + } + + import sonic-port { + prefix port; + revision-date 2019-07-01; + } + + import sonic-vrf { + prefix vrf; + } + + description "VLAN yang Module for SONiC OS"; + + revision 2021-03-30 { + description "Modify the type of vrf name"; + } + + revision 2019-07-01 { + description "First Revision"; + } + + container sonic-vlan { + + container VLAN_INTERFACE { + + description "VLAN_INTERFACE part of config_db.json"; + + list VLAN_INTERFACE_LIST { + + description "VLAN INTERFACE part of config_db.json with vrf"; + + key "name"; + + leaf name { + type leafref { + path /vlan:sonic-vlan/vlan:VLAN/vlan:VLAN_LIST/vlan:name; + } + } + + leaf vrf_name { + type leafref{ + path "/vrf:sonic-vrf/vrf:VRF/vrf:VRF_LIST/vrf:name"; + } + } + } + /* end of VLAN_INTERFACE_LIST */ + + list VLAN_INTERFACE_IPPREFIX_LIST { + + key "name ip-prefix"; + + leaf name { + /* This node must be present in VLAN_INTERFACE_LIST */ + must "(current() = ../../VLAN_INTERFACE_LIST[name=current()]/name)" + { + error-message "Must condition not satisfied, Try adding Vlan: {}, Example: 'Vlan100': {}"; + } + + type leafref { + path "/vlan:sonic-vlan/vlan:VLAN/vlan:VLAN_LIST/vlan:name"; + } + } + + leaf ip-prefix { + type union { + type stypes:sonic-ip4-prefix; + type stypes:sonic-ip6-prefix; + } + } + + leaf scope { + type enumeration { + enum global; + enum local; + } + } + + leaf family { + + /* family leaf needed for backward compatibility + Both ip4 and ip6 address are string in IETF RFC 6021, + so must statement can check based on : or ., family + should be IPv4 or IPv6 according. + */ + + must "(contains(../ip-prefix, ':') and current()='IPv6') or + (contains(../ip-prefix, '.') and current()='IPv4')"; + type stypes:ip-family; + } + } + /* end of VLAN_INTERFACE_LIST */ + } + /* end of VLAN_INTERFACE container */ + + container VLAN { + + description "VLAN part of config_db.json"; + + list VLAN_LIST { + + key "name"; + + leaf name { + type string { + pattern 'Vlan([0-9]{1,3}|[1-3][0-9]{3}|[4][0][0-8][0-9]|[4][0][9][0-4])'; + } + } + + leaf vlanid { + type uint16 { + range 1..4094; + } + } + + leaf description { + type string { + length 1..255; + } + } + + leaf-list dhcp_servers { + type inet:ip-address; + } + + leaf mtu { + type uint16 { + range 1..9216; + } + } + + leaf admin_status { + type stypes:admin_status; + } + } + /* end of VLAN_LIST */ + } + /* end of container VLAN */ + + container VLAN_MEMBER { + + description "VLAN_MEMBER part of config_db.json"; + + list VLAN_MEMBER_LIST { + + key "name port"; + + leaf name { + type leafref { + path "/vlan:sonic-vlan/vlan:VLAN/vlan:VLAN_LIST/vlan:name"; + } + } + + leaf port { + /* key elements are mandatory by default */ + type leafref { + path /port:sonic-port/port:PORT/port:PORT_LIST/port:name; + } + } + + leaf tagging_mode { + mandatory true; + type stypes:vlan_tagging_mode; + } + } + /* end of list VLAN_MEMBER_LIST */ + } + /* end of container VLAN_MEMBER */ + } + /* end of container sonic-vlan */ +} +/* end of module sonic-vlan */ diff --git a/tests/cli_autogen_yang_parser_test.py b/tests/cli_autogen_yang_parser_test.py new file mode 100644 index 0000000000..fb5141aff1 --- /dev/null +++ b/tests/cli_autogen_yang_parser_test.py @@ -0,0 +1,56 @@ +import sys +import os +import pytest +import logging +# debug +import pprint + +from sonic_cli_gen.yang_parser import YangParser + +logger = logging.getLogger(__name__) + +test_path = os.path.dirname(os.path.abspath(__file__)) +config_db_path = os.path.join(test_path, 'cli_autogen_input/config_db.json') +yang_models_path = '/usr/local/yang-models' + + +class TestYangParser: + def test_one_table_container(self): + yang_model_name = 'sonic-one-table-container' + + + move_yang_model(yang_model_name) + parser = YangParser(yang_model_name = yang_model_name, + config_db_path = config_db_path, + allow_tbl_without_yang = True, + debug = False) + yang_dict = parser.parse_yang_model() + pretty_log(yang_dict) + + pass + +def move_yang_model(yang_model_name): + """ Move provided YANG model to known location for YangParser class + + Args: + yang_model_name: name of provided YANG model + """ + src_path = os.path.join(test_path, 'cli_autogen_input', yang_model_name + '.yang') + cmd = 'sudo cp {} {}'.format(src_path, yang_models_path) + os.system(cmd) + +def remove_yang_model(yang_model_name): + """ Remove YANG model from well known system location + + Args: + yang_model_name: name of provided YANG model + """ + yang_model_path = os.path.join(yang_models_path, yang_model_name + '.yang') + cmd = 'sudo rm {}'.format(yang_model_path) + os.system(cmd) + +# DEBUG function +def pretty_log(dictionary): + for line in pprint.pformat(dictionary).split('\n'): + logging.warning(line) + From 0d371de158fbac9487fbae8688a8cc1180652e69 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Fri, 14 May 2021 19:37:23 +0000 Subject: [PATCH 25/76] Refactored _init_yang_module_and_containers() Signed-off-by: Vadym Hlushko --- sonic_cli_gen/yang_parser.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index b36d862ee8..a114a68165 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -71,21 +71,22 @@ def _init_yang_module_and_containers(self): self.y_table_containers Raises: - KeyError: if invalid YANG model provided - KeyError: if YANG models is NOT exist + KeyError: if YANG model is invalid or NOT exist """ self._find_index_of_yang_model() - if self.idx_yJson is not None: - self.y_module = self.conf_mgmt.sy.yJson[self.idx_yJson]['module'] - if self.y_module.get('container') is not None: - self.y_top_level_container = self.y_module['container'] - self.y_table_containers = self.y_top_level_container['container'] - else: - raise KeyError('YANG model {} does NOT have "container" element'.format(self.yang_model_name)) - else: + if self.idx_yJson is None: raise KeyError('YANG model {} is NOT exist'.format(self.yang_model_name)) + self.y_module = self.conf_mgmt.sy.yJson[self.idx_yJson]['module'] + + if self.y_module.get('container') is None: + raise KeyError('YANG model {} does NOT have "top level container" element'.format(self.yang_model_name)) + self.y_top_level_container = self.y_module.get('container') + + if self.y_top_level_container.get('container') is None: + raise KeyError('YANG model {} does NOT have "container" element after "top level container"'.format(self.yang_model_name)) + self.y_table_containers = self.y_top_level_container.get('container') def _find_index_of_yang_model(self): """ Find index of provided YANG model inside yJson object From 1593039dadf59f1df8c57c1b153bbc778f11d205 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Fri, 14 May 2021 20:32:16 +0000 Subject: [PATCH 26/76] added 2 test cases Signed-off-by: Vadym Hlushko --- sonic_cli_gen/yang_parser.py | 15 ++++--- .../cli_autogen_input/assert_dictionaries.py | 40 +++++++++++++++++++ .../sonic-many-table-containers.yang | 20 ++++++++++ .../sonic-one-table-container.yang | 12 ++---- tests/cli_autogen_yang_parser_test.py | 30 +++++++++----- 5 files changed, 95 insertions(+), 22 deletions(-) create mode 100644 tests/cli_autogen_input/assert_dictionaries.py create mode 100644 tests/cli_autogen_input/sonic-many-table-containers.yang diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index a114a68165..32e2b1a7f0 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -81,16 +81,21 @@ def _init_yang_module_and_containers(self): self.y_module = self.conf_mgmt.sy.yJson[self.idx_yJson]['module'] if self.y_module.get('container') is None: - raise KeyError('YANG model {} does NOT have "top level container" element'.format(self.yang_model_name)) + raise KeyError('YANG model {} does NOT have "top level container" element \ + Please follow the SONiC YANG model guidelines: \ + https://github.com/Azure/SONiC/blob/master/doc/mgmt/SONiC_YANG_Model_Guidelines.md'.format(self.yang_model_name)) self.y_top_level_container = self.y_module.get('container') if self.y_top_level_container.get('container') is None: - raise KeyError('YANG model {} does NOT have "container" element after "top level container"'.format(self.yang_model_name)) + raise KeyError('YANG model {} does NOT have "container" element after "top level container" \ + Please follow the SONiC YANG model guidelines: \ + https://github.com/Azure/SONiC/blob/master/doc/mgmt/SONiC_YANG_Model_Guidelines.md'.format(self.yang_model_name)) self.y_table_containers = self.y_top_level_container.get('container') def _find_index_of_yang_model(self): - """ Find index of provided YANG model inside yJson object + """ Find index of provided YANG model inside yJson object, and save it to self.idx_yJson variable + yJson object contain all yang-models parsed from directory - /usr/local/yang-models """ for i in range(len(self.conf_mgmt.sy.yJson)): @@ -99,7 +104,7 @@ def _find_index_of_yang_model(self): def parse_yang_model(self) -> dict: """ Parse proviced YANG model - and save output to self.yang_2_dict obj + and save output to self.yang_2_dict object Returns: dictionary - parsed YANG model in dictionary format @@ -108,7 +113,7 @@ def parse_yang_model(self) -> dict: self._init_yang_module_and_containers() self.yang_2_dict['tables'] = list() - # determine how many (1 or couple) containers YANG model have after 'top level' container + # determine how many (1 or couple) containers a YANG model have after 'top level' container # 'table' container it is a container that goes after 'top level' container if isinstance(self.y_table_containers, list): for tbl_cont in self.y_table_containers: diff --git a/tests/cli_autogen_input/assert_dictionaries.py b/tests/cli_autogen_input/assert_dictionaries.py new file mode 100644 index 0000000000..623233e7ee --- /dev/null +++ b/tests/cli_autogen_input/assert_dictionaries.py @@ -0,0 +1,40 @@ +""" +Module holding correct dictionaries for test YANG models +""" + +one_table_container = { + "tables":[ + { + "description":"FIRST_TABLE description", + "name":"FIRST_TABLE", + "static-objects":[ + { + + } + ] + } + ] +} + +many_table_containers = { + "tables":[ + { + "description":"FIRST_TABLE description", + "name":"FIRST_TABLE", + "static-objects":[ + { + + } + ] + }, + { + "description":"SECOND_TABLE description", + "name":"SECOND_TABLE", + "static-objects":[ + { + + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-many-table-containers.yang b/tests/cli_autogen_input/sonic-many-table-containers.yang new file mode 100644 index 0000000000..41a7d3c288 --- /dev/null +++ b/tests/cli_autogen_input/sonic-many-table-containers.yang @@ -0,0 +1,20 @@ +module sonic-many-table-containers { + + yang-version 1.1; + + namespace "http://github.com/Azure/sonic-many-table-cont"; + prefix many-table-cont; + + container sonic-many-table-containers { + + container FIRST_TABLE { + + description "FIRST_TABLE description"; + } + + container SECOND_TABLE { + + description "SECOND_TABLE description"; + } + } +} diff --git a/tests/cli_autogen_input/sonic-one-table-container.yang b/tests/cli_autogen_input/sonic-one-table-container.yang index ca5d248ece..b04afa7ed9 100644 --- a/tests/cli_autogen_input/sonic-one-table-container.yang +++ b/tests/cli_autogen_input/sonic-one-table-container.yang @@ -2,18 +2,14 @@ module sonic-one-table-container { yang-version 1.1; - namespace "http://github.com/Azure/sonic-one"; - prefix one; + namespace "http://github.com/Azure/sonic-one-table"; + prefix one-table; container sonic-one-table-container { - container ONE_TABLE { + container FIRST_TABLE { - description "ONE_TABLE description"; - - leaf random { - type string; - } + description "FIRST_TABLE description"; } } } \ No newline at end of file diff --git a/tests/cli_autogen_yang_parser_test.py b/tests/cli_autogen_yang_parser_test.py index fb5141aff1..e9010c4877 100644 --- a/tests/cli_autogen_yang_parser_test.py +++ b/tests/cli_autogen_yang_parser_test.py @@ -6,28 +6,40 @@ import pprint from sonic_cli_gen.yang_parser import YangParser +from .cli_autogen_input import assert_dictionaries + logger = logging.getLogger(__name__) test_path = os.path.dirname(os.path.abspath(__file__)) -config_db_path = os.path.join(test_path, 'cli_autogen_input/config_db.json') yang_models_path = '/usr/local/yang-models' class TestYangParser: + + def test_one_table_container(self): yang_model_name = 'sonic-one-table-container' + template('sonic-one-table-container', assert_dictionaries.one_table_container) + def test_many_table_containers(self): + yang_model_name = 'sonic-many-table-containers' + template('sonic-many-table-containers', assert_dictionaries.many_table_containers) + +def template(yang_model_name, correct_dict): + config_db_path = os.path.join(test_path, 'cli_autogen_input/config_db.json') + move_yang_model(yang_model_name) + parser = YangParser(yang_model_name = yang_model_name, + config_db_path = config_db_path, + allow_tbl_without_yang = True, + debug = False) + yang_dict = parser.parse_yang_model() + # debug + pretty_log(yang_dict) - move_yang_model(yang_model_name) - parser = YangParser(yang_model_name = yang_model_name, - config_db_path = config_db_path, - allow_tbl_without_yang = True, - debug = False) - yang_dict = parser.parse_yang_model() - pretty_log(yang_dict) + assert yang_dict == correct_dict - pass + remove_yang_model(yang_model_name) def move_yang_model(yang_model_name): """ Move provided YANG model to known location for YangParser class From 9004c1cb55c70451efef83c472cbb38dee0a7c60 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Mon, 17 May 2021 13:48:32 +0300 Subject: [PATCH 27/76] [sonic-cli-gen] put autogenerated plugins into plugins.auto Signed-off-by: Stepan Blyschak --- clear/plugins/auto/__init__.py | 0 config/plugins/auto/__init__.py | 0 setup.py | 3 +++ show/plugins/auto/__init__.py | 0 sonic_cli_gen/generator.py | 2 +- utilities_common/util_base.py | 1 + 6 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 clear/plugins/auto/__init__.py create mode 100644 config/plugins/auto/__init__.py create mode 100644 show/plugins/auto/__init__.py diff --git a/clear/plugins/auto/__init__.py b/clear/plugins/auto/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/config/plugins/auto/__init__.py b/config/plugins/auto/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/setup.py b/setup.py index 8639a52412..9e3f30e3ad 100644 --- a/setup.py +++ b/setup.py @@ -23,8 +23,10 @@ 'acl_loader', 'clear', 'clear.plugins', + 'clear.plugins.auto', 'config', 'config.plugins', + 'config.plugins.auto', 'connect', 'consutil', 'counterpoll', @@ -46,6 +48,7 @@ 'show', 'show.interfaces', 'show.plugins', + 'show.plugins.auto', 'sonic_installer', 'sonic_installer.bootloader', 'sonic_package_manager', diff --git a/show/plugins/auto/__init__.py b/show/plugins/auto/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py index 6214bbe9c5..f64395c18b 100644 --- a/sonic_cli_gen/generator.py +++ b/sonic_cli_gen/generator.py @@ -29,7 +29,7 @@ def generate_cli_plugin(self, cli_group, plugin_name): plugin_py.write(template.render(yang_dict)) def get_cli_plugin_path(command, plugin_name): - pkg_loader = pkgutil.get_loader(f'{command}.plugins') + pkg_loader = pkgutil.get_loader(f'{command}.plugins.auto') if pkg_loader is None: raise PackageManagerError(f'Failed to get plugins path for {command} CLI') plugins_pkg_path = os.path.dirname(pkg_loader.path) diff --git a/utilities_common/util_base.py b/utilities_common/util_base.py index ff5570735c..9bea158b59 100644 --- a/utilities_common/util_base.py +++ b/utilities_common/util_base.py @@ -24,6 +24,7 @@ def iter_namespace(ns_pkg): for _, module_name, ispkg in iter_namespace(plugins_namespace): if ispkg: + yield from self.load_plugins(importlib.import_module(module_name)) continue log.log_debug('importing plugin: {}'.format(module_name)) try: From 32e27a216e71428c5f6d2fa7a5c68314dea66825 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Mon, 17 May 2021 15:45:21 +0300 Subject: [PATCH 28/76] [sonic-cli-gen] fix show.py.j2 template Signed-off-by: Stepan Blyschak --- sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 index 58e7dfd2e0..ea82b9761f 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 @@ -77,7 +77,8 @@ def {{ name }}(db): body = [] table = db.cfgdb.get_table("{{ table.name }}") - for key, entry in natsort.natsorted(table).items(): + for key in natsort.natsorted(table): + entry = table[key] if not isinstance(key, tuple): key = (key,) From 02e6d27a4d56903ff8c91f27ce05d907c3ac694e Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Mon, 17 May 2021 13:29:20 +0300 Subject: [PATCH 29/76] [sonic-cli-gen] add remove plugin method Signed-off-by: Stepan Blyschak --- sonic_cli_gen/generator.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py index f64395c18b..adb6ac5a96 100644 --- a/sonic_cli_gen/generator.py +++ b/sonic_cli_gen/generator.py @@ -27,6 +27,12 @@ def generate_cli_plugin(self, cli_group, plugin_name): template = self.env.get_template(cli_group + '.py.j2') with open(plugin_path, 'w') as plugin_py: plugin_py.write(template.render(yang_dict)) + + def remove_cli_plugin(self, cli_group, plugin_name): + plugin_path = get_cli_plugin_path(cli_group, plugin_name + '_yang.py') + if os.path.exists(plugin_path): + os.remove(plugin_path) + def get_cli_plugin_path(command, plugin_name): pkg_loader = pkgutil.get_loader(f'{command}.plugins.auto') From cb7fa0696034f7c550ba33fff6681db876f8c376 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Mon, 17 May 2021 15:00:12 +0000 Subject: [PATCH 30/76] refactored past test cases, added new Signed-off-by: Vadym Hlushko --- sonic_cli_gen/yang_parser.py | 3 +- .../cli_autogen_input/assert_dictionaries.py | 219 +++++++++++++++++- tests/cli_autogen_input/sonic-1-list.yang | 26 +++ .../sonic-1-object-container.yang | 20 ++ .../sonic-1-table-container.yang | 15 ++ tests/cli_autogen_input/sonic-2-lists.yang | 37 +++ .../sonic-2-object-containers.yang | 25 ++ .../sonic-2-table-containers.yang | 20 ++ .../sonic-many-table-containers.yang | 20 -- .../sonic-one-table-container.yang | 15 -- .../sonic-static-object-complex-1.yang | 49 ++++ .../sonic-static-object-complex-2.yang | 71 ++++++ .../cli_autogen_input/sonic-test-complex.yang | 115 +++++++++ tests/cli_autogen_input/sonic-vlan.yang | 190 --------------- tests/cli_autogen_yang_parser_test.py | 51 +++- 15 files changed, 630 insertions(+), 246 deletions(-) create mode 100644 tests/cli_autogen_input/sonic-1-list.yang create mode 100644 tests/cli_autogen_input/sonic-1-object-container.yang create mode 100644 tests/cli_autogen_input/sonic-1-table-container.yang create mode 100644 tests/cli_autogen_input/sonic-2-lists.yang create mode 100644 tests/cli_autogen_input/sonic-2-object-containers.yang create mode 100644 tests/cli_autogen_input/sonic-2-table-containers.yang delete mode 100644 tests/cli_autogen_input/sonic-many-table-containers.yang delete mode 100644 tests/cli_autogen_input/sonic-one-table-container.yang create mode 100644 tests/cli_autogen_input/sonic-static-object-complex-1.yang create mode 100644 tests/cli_autogen_input/sonic-static-object-complex-2.yang create mode 100644 tests/cli_autogen_input/sonic-test-complex.yang delete mode 100644 tests/cli_autogen_input/sonic-vlan.yang diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index 32e2b1a7f0..c77250f066 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -26,7 +26,7 @@ class YangParser: 'attrs': [ { 'name': 'value', - 'descruption': 'value', + 'description': 'value', 'is-leaf-list': False, 'is-mandatory': False } @@ -44,6 +44,7 @@ class YangParser: } In case if YANG model does NOT have a 'list' entity, it has the same structure as above, but 'dynamic-objects' changed to 'static-objects' and have no 'keys' """ + def __init__(self, yang_model_name, config_db_path, diff --git a/tests/cli_autogen_input/assert_dictionaries.py b/tests/cli_autogen_input/assert_dictionaries.py index 623233e7ee..9a2989d500 100644 --- a/tests/cli_autogen_input/assert_dictionaries.py +++ b/tests/cli_autogen_input/assert_dictionaries.py @@ -5,22 +5,21 @@ one_table_container = { "tables":[ { - "description":"FIRST_TABLE description", - "name":"FIRST_TABLE", + "description":"TABLE_1 description", + "name":"TABLE_1", "static-objects":[ { - } ] } ] } -many_table_containers = { +two_table_containers = { "tables":[ { - "description":"FIRST_TABLE description", - "name":"FIRST_TABLE", + "description":"TABLE_1 description", + "name":"TABLE_1", "static-objects":[ { @@ -28,8 +27,8 @@ ] }, { - "description":"SECOND_TABLE description", - "name":"SECOND_TABLE", + "description":"TABLE_2 description", + "name":"TABLE_2", "static-objects":[ { @@ -37,4 +36,208 @@ ] } ] +} + +one_object_container = { + "tables":[ + { + "description":"TABLE_1 description", + "name":"TABLE_1", + "static-objects":[ + { + "name":"OBJECT_1", + "description":"OBJECT_1 description", + "attrs":[ + ] + } + ] + } + ] +} + +two_object_containers = { + "tables":[ + { + "description":"FIRST_TABLE description", + "name":"TABLE_1", + "static-objects":[ + { + "name":"OBJECT_1", + "description":"OBJECT_1 description", + "attrs":[ + ] + }, + { + "name":"OBJECT_2", + "description":"OBJECT_2 description", + "attrs":[ + ] + } + ] + } + ] +} + +one_list = { + "tables":[ + { + "description":"TABLE_1 description", + "name":"TABLE_1", + "dynamic-objects":[ + { + "name":"TABLE_1_LIST", + "description":"TABLE_1_LIST description", + "keys":[ + { + "name": "key_name", + "description": "", + } + ], + "attrs":[ + ] + } + ] + } + ] +} + +two_lists = { + "tables":[ + { + "description":"TABLE_1 description", + "name":"TABLE_1", + "dynamic-objects":[ + { + "name":"TABLE_1_LIST_1", + "description":"TABLE_1_LIST_1 description", + "keys":[ + { + "name": "key_name1", + "description": "", + } + ], + "attrs":[ + ] + }, + { + "name":"TABLE_1_LIST_2", + "description":"TABLE_1_LIST_2 description", + "keys":[ + { + "name": "key_name2", + "description": "", + } + ], + "attrs":[ + ] + } + ] + } + ] +} + +static_object_complex_1 = { + "tables":[ + { + "description":"TABLE_1 description", + "name":"TABLE_1", + "static-objects":[ + { + "name":"OBJECT_1", + "description":"OBJECT_1 description", + "attrs":[ + { + "name":"OBJ_1_LEAF_1", + "description": "OBJ_1_LEAF_1 description", + "is-mandatory": False, + "is-leaf-list": False, + }, + { + "name":"OBJ_1_LEAF_LIST_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": True, + }, + { + "name":"OBJ_1_CHOICE_1_LEAF_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + }, + { + "name":"OBJ_1_CHOICE_1_LEAF_2", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + } + ] + } + ] + } + ] +} + +static_object_complex_2 = { + "tables":[ + { + "description":"TABLE_1 description", + "name":"TABLE_1", + "static-objects":[ + { + "name":"OBJECT_1", + "description":"OBJECT_1 description", + "attrs":[ + { + "name":"OBJ_1_LEAF_1", + "description": "OBJ_1_LEAF_1 description", + "is-mandatory": False, + "is-leaf-list": False, + }, + { + "name":"OBJ_1_LEAF_2", + "description": "OBJ_1_LEAF_2 description", + "is-mandatory": False, + "is-leaf-list": False, + }, + { + "name":"OBJ_1_LEAF_LIST_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": True, + }, + { + "name":"OBJ_1_LEAF_LIST_2", + "description": "", + "is-mandatory": False, + "is-leaf-list": True, + }, + { + "name":"OBJ_1_CHOICE_1_LEAF_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + }, + { + "name":"OBJ_1_CHOICE_1_LEAF_2", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + }, + { + "name":"OBJ_1_CHOICE_2_LEAF_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + }, + { + "name":"OBJ_1_CHOICE_2_LEAF_2", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + }, + ] + } + ] + } + ] } \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-1-list.yang b/tests/cli_autogen_input/sonic-1-list.yang new file mode 100644 index 0000000000..c7fc4ee824 --- /dev/null +++ b/tests/cli_autogen_input/sonic-1-list.yang @@ -0,0 +1,26 @@ +module sonic-1-list { + + yang-version 1.1; + + namespace "http://github.com/Azure/s-1-list"; + prefix s-1-list; + + container sonic-1-list { + + container TABLE_1 { + + description "TABLE_1 description"; + + list TABLE_1_LIST { + + description "TABLE_1_LIST description"; + + key "key_name"; + + leaf key_name { + type string; + } + } + } + } +} \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-1-object-container.yang b/tests/cli_autogen_input/sonic-1-object-container.yang new file mode 100644 index 0000000000..d52b2a8caf --- /dev/null +++ b/tests/cli_autogen_input/sonic-1-object-container.yang @@ -0,0 +1,20 @@ +module sonic-1-object-container { + + yang-version 1.1; + + namespace "http://github.com/Azure/s-1-object"; + prefix s-1-object; + + container sonic-1-object-container { + + container TABLE_1 { + + description "TABLE_1 description"; + + container OBJECT_1 { + + description "OBJECT_1 description"; + } + } + } +} \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-1-table-container.yang b/tests/cli_autogen_input/sonic-1-table-container.yang new file mode 100644 index 0000000000..8963148158 --- /dev/null +++ b/tests/cli_autogen_input/sonic-1-table-container.yang @@ -0,0 +1,15 @@ +module sonic-1-table-container { + + yang-version 1.1; + + namespace "http://github.com/Azure/s-1-table"; + prefix s-1-table; + + container sonic-1-table-container { + + container TABLE_1 { + + description "TABLE_1 description"; + } + } +} \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-2-lists.yang b/tests/cli_autogen_input/sonic-2-lists.yang new file mode 100644 index 0000000000..2a4cd42fd9 --- /dev/null +++ b/tests/cli_autogen_input/sonic-2-lists.yang @@ -0,0 +1,37 @@ +module sonic-2-lists { + + yang-version 1.1; + + namespace "http://github.com/Azure/s-2-lists"; + prefix s-2-lists; + + container sonic-2-lists { + + container TABLE_1 { + + description "TABLE_1 description"; + + list TABLE_1_LIST_1 { + + description "TABLE_1_LIST_1 description"; + + key "key_name1"; + + leaf key_name1 { + type string; + } + } + + list TABLE_1_LIST_2 { + + description "TABLE_1_LIST_2 description"; + + key "key_name2"; + + leaf key_name2 { + type string; + } + } + } + } +} \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-2-object-containers.yang b/tests/cli_autogen_input/sonic-2-object-containers.yang new file mode 100644 index 0000000000..1aaaeb1a19 --- /dev/null +++ b/tests/cli_autogen_input/sonic-2-object-containers.yang @@ -0,0 +1,25 @@ +module sonic-2-object-containers { + + yang-version 1.1; + + namespace "http://github.com/Azure/s-2-object"; + prefix s-2-object; + + container sonic-2-object-containers { + + container TABLE_1 { + + description "FIRST_TABLE description"; + + container OBJECT_1 { + + description "OBJECT_1 description"; + } + + container OBJECT_2 { + + description "OBJECT_2 description"; + } + } + } +} \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-2-table-containers.yang b/tests/cli_autogen_input/sonic-2-table-containers.yang new file mode 100644 index 0000000000..a3f13474b5 --- /dev/null +++ b/tests/cli_autogen_input/sonic-2-table-containers.yang @@ -0,0 +1,20 @@ +module sonic-2-table-containers { + + yang-version 1.1; + + namespace "http://github.com/Azure/s-2-table"; + prefix s-2-table; + + container sonic-2-table-containers { + + container TABLE_1 { + + description "TABLE_1 description"; + } + + container TABLE_2 { + + description "TABLE_2 description"; + } + } +} diff --git a/tests/cli_autogen_input/sonic-many-table-containers.yang b/tests/cli_autogen_input/sonic-many-table-containers.yang deleted file mode 100644 index 41a7d3c288..0000000000 --- a/tests/cli_autogen_input/sonic-many-table-containers.yang +++ /dev/null @@ -1,20 +0,0 @@ -module sonic-many-table-containers { - - yang-version 1.1; - - namespace "http://github.com/Azure/sonic-many-table-cont"; - prefix many-table-cont; - - container sonic-many-table-containers { - - container FIRST_TABLE { - - description "FIRST_TABLE description"; - } - - container SECOND_TABLE { - - description "SECOND_TABLE description"; - } - } -} diff --git a/tests/cli_autogen_input/sonic-one-table-container.yang b/tests/cli_autogen_input/sonic-one-table-container.yang deleted file mode 100644 index b04afa7ed9..0000000000 --- a/tests/cli_autogen_input/sonic-one-table-container.yang +++ /dev/null @@ -1,15 +0,0 @@ -module sonic-one-table-container { - - yang-version 1.1; - - namespace "http://github.com/Azure/sonic-one-table"; - prefix one-table; - - container sonic-one-table-container { - - container FIRST_TABLE { - - description "FIRST_TABLE description"; - } - } -} \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-static-object-complex-1.yang b/tests/cli_autogen_input/sonic-static-object-complex-1.yang new file mode 100644 index 0000000000..a7dfee86ab --- /dev/null +++ b/tests/cli_autogen_input/sonic-static-object-complex-1.yang @@ -0,0 +1,49 @@ +module sonic-static-object-complex-1 { + + yang-version 1.1; + + namespace "http://github.com/Azure/static-complex-1"; + prefix static-complex-1; + + container sonic-static-object-complex-1 { + /* sonic-static-object-complex-1 - top level container */ + + container TABLE_1 { + /* TABLE_1 - table container */ + + description "TABLE_1 description"; + + container OBJECT_1 { + /* OBJECT_1 - object container, it have: + * 1 leaf, + * 1 leaf-list + * 1 choice + */ + + description "OBJECT_1 description"; + + leaf OBJ_1_LEAF_1 { + description "OBJ_1_LEAF_1 description"; + type string; + } + + leaf-list OBJ_1_LEAF_LIST_1 { + type string; + } + + choice OBJ_1_CHOICE_1 { + case OBJ_1_CHOICE_1_CASE_1 { + leaf OBJ_1_CHOICE_1_LEAF_1 { + type uint16; + } + } + case OBJ_1_CHOICE_1_CASE_2 { + leaf OBJ_1_CHOICE_1_LEAF_2 { + type string; + } + } + } + } + } + } +} \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-static-object-complex-2.yang b/tests/cli_autogen_input/sonic-static-object-complex-2.yang new file mode 100644 index 0000000000..451a445ce6 --- /dev/null +++ b/tests/cli_autogen_input/sonic-static-object-complex-2.yang @@ -0,0 +1,71 @@ +module sonic-static-object-complex-2 { + + yang-version 1.1; + + namespace "http://github.com/Azure/static-complex-2"; + prefix static-complex-2; + + container sonic-static-object-complex-2 { + /* sonic-static-object-complex-2 - top level container */ + + container TABLE_1 { + /* TABLE_1 - table container */ + + description "TABLE_1 description"; + + container OBJECT_1 { + /* OBJECT_1 - object container, it have: + * 2 leafs, + * 2 leaf-lists, + * 2 choices + */ + + description "OBJECT_1 description"; + + leaf OBJ_1_LEAF_1 { + description "OBJ_1_LEAF_1 description"; + type string; + } + + leaf OBJ_1_LEAF_2 { + description "OBJ_1_LEAF_2 description"; + type string; + } + + leaf-list OBJ_1_LEAF_LIST_1 { + type string; + } + + leaf-list OBJ_1_LEAF_LIST_2 { + type string; + } + + choice OBJ_1_CHOICE_1 { + case OBJ_1_CHOICE_1_CASE_1 { + leaf OBJ_1_CHOICE_1_LEAF_1 { + type uint16; + } + } + case OBJ_1_CHOICE_1_CASE_2 { + leaf OBJ_1_CHOICE_1_LEAF_2 { + type string; + } + } + } + + choice OBJ_1_CHOICE_2 { + case OBJ_1_CHOICE_2_CASE_1 { + leaf OBJ_1_CHOICE_2_LEAF_1 { + type uint16; + } + } + case OBJ_1_CHOICE_2_CASE_2 { + leaf OBJ_1_CHOICE_2_LEAF_2 { + type string; + } + } + } + } + } + } +} \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-test-complex.yang b/tests/cli_autogen_input/sonic-test-complex.yang new file mode 100644 index 0000000000..e4485c9697 --- /dev/null +++ b/tests/cli_autogen_input/sonic-test-complex.yang @@ -0,0 +1,115 @@ +module sonic-test-1 { + + yang-version 1.1; + + namespace "http://github.com/Azure/sonic-test-1"; + prefix many-test-1; + + container sonic-test-1 { + /* sonic-test-1 - top level container, it have: + * 2 table containers. Table container - represent Config DB table name. + */ + + container TABLE_1 { + /* TABLE_1 - table container, it have: + * 2 object containers, Object container - represent Confg DB object name. + */ + description "TABLE_1 description"; + + container OBJECT_1 { + /* OBJECT_1 - object container, it have: + * 1 leaf, + * 1 leaf-list + * 1 choice + */ + description "OBJECT_1 description"; + + leaf OBJ_1_LEAF_1 { + description "OBJ_1_LEAF_1 description"; + type string; + } + + leaf-list OBJ_1_LEAF_LIST_1 { + mandatory true; + type string; + } + + choice OBJ_1_CH_1 { + case OBJ_1_CH_1_CASE_1 { + leaf OBJ_1_CH_1_LEAF_1 { + type uint16; + } + } + case OBJ_1_CH_1_CASE_2 { + leaf OBJ_1_CH_1_LEAF_2 { + type string; + } + } + } + } + + container OBJECT_2 { + /* OBJECT_2 - table container, it have: + * 2 leaf, + * 2 leaf-list + * 2 choice + */ + description "OBJECT_2 description"; + + leaf OBJ_2_LEAF_1 { + description "OBJ_2_LEAF_1 description"; + type string; + } + + leaf OBJ_2_LEAF_2 { + mandatory true; + type string; + } + + leaf-list OBJ_2_LEAF_LIST_1 { + description "OBJ_2_LEAF_LIST_1 description"; + mandatory true; + type string; + } + + leaf-list OBJ_2_LEAF_LIST_2 { + type string; + } + + choice OBJ_2_CH_1 { + case OBJ_2_CH_1_CASE_1 { + leaf OBJ_2_CH_1_LEAF_1 { + type uint16; + } + } + case OBJ_2_CH_1_CASE_2 { + leaf OBJ_2_CH_1_LEAF_2 { + type string { + type string; + } + } + } + } + + choice OBJ_2_CH_2 { + case OBJ_2_CH_2_CASE_1 { + leaf OBJ_2_CH_2_LEAF_1 { + type uint16; + } + } + case OBJ_2_CH_2_CASE_2 { + leaf OBJ_2_CH_2_LEAF_2 { + type string { + type string; + } + } + } + } + } + } + + container TABLE_2 { + + } + } +} \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-vlan.yang b/tests/cli_autogen_input/sonic-vlan.yang deleted file mode 100644 index 2962161ef0..0000000000 --- a/tests/cli_autogen_input/sonic-vlan.yang +++ /dev/null @@ -1,190 +0,0 @@ -module sonic-vlan { - - yang-version 1.1; - - namespace "http://github.com/Azure/sonic-vlan"; - prefix vlan; - - import ietf-inet-types { - prefix inet; - } - - import sonic-types { - prefix stypes; - revision-date 2019-07-01; - } - - import sonic-extension { - prefix ext; - revision-date 2019-07-01; - } - - import sonic-port { - prefix port; - revision-date 2019-07-01; - } - - import sonic-vrf { - prefix vrf; - } - - description "VLAN yang Module for SONiC OS"; - - revision 2021-03-30 { - description "Modify the type of vrf name"; - } - - revision 2019-07-01 { - description "First Revision"; - } - - container sonic-vlan { - - container VLAN_INTERFACE { - - description "VLAN_INTERFACE part of config_db.json"; - - list VLAN_INTERFACE_LIST { - - description "VLAN INTERFACE part of config_db.json with vrf"; - - key "name"; - - leaf name { - type leafref { - path /vlan:sonic-vlan/vlan:VLAN/vlan:VLAN_LIST/vlan:name; - } - } - - leaf vrf_name { - type leafref{ - path "/vrf:sonic-vrf/vrf:VRF/vrf:VRF_LIST/vrf:name"; - } - } - } - /* end of VLAN_INTERFACE_LIST */ - - list VLAN_INTERFACE_IPPREFIX_LIST { - - key "name ip-prefix"; - - leaf name { - /* This node must be present in VLAN_INTERFACE_LIST */ - must "(current() = ../../VLAN_INTERFACE_LIST[name=current()]/name)" - { - error-message "Must condition not satisfied, Try adding Vlan: {}, Example: 'Vlan100': {}"; - } - - type leafref { - path "/vlan:sonic-vlan/vlan:VLAN/vlan:VLAN_LIST/vlan:name"; - } - } - - leaf ip-prefix { - type union { - type stypes:sonic-ip4-prefix; - type stypes:sonic-ip6-prefix; - } - } - - leaf scope { - type enumeration { - enum global; - enum local; - } - } - - leaf family { - - /* family leaf needed for backward compatibility - Both ip4 and ip6 address are string in IETF RFC 6021, - so must statement can check based on : or ., family - should be IPv4 or IPv6 according. - */ - - must "(contains(../ip-prefix, ':') and current()='IPv6') or - (contains(../ip-prefix, '.') and current()='IPv4')"; - type stypes:ip-family; - } - } - /* end of VLAN_INTERFACE_LIST */ - } - /* end of VLAN_INTERFACE container */ - - container VLAN { - - description "VLAN part of config_db.json"; - - list VLAN_LIST { - - key "name"; - - leaf name { - type string { - pattern 'Vlan([0-9]{1,3}|[1-3][0-9]{3}|[4][0][0-8][0-9]|[4][0][9][0-4])'; - } - } - - leaf vlanid { - type uint16 { - range 1..4094; - } - } - - leaf description { - type string { - length 1..255; - } - } - - leaf-list dhcp_servers { - type inet:ip-address; - } - - leaf mtu { - type uint16 { - range 1..9216; - } - } - - leaf admin_status { - type stypes:admin_status; - } - } - /* end of VLAN_LIST */ - } - /* end of container VLAN */ - - container VLAN_MEMBER { - - description "VLAN_MEMBER part of config_db.json"; - - list VLAN_MEMBER_LIST { - - key "name port"; - - leaf name { - type leafref { - path "/vlan:sonic-vlan/vlan:VLAN/vlan:VLAN_LIST/vlan:name"; - } - } - - leaf port { - /* key elements are mandatory by default */ - type leafref { - path /port:sonic-port/port:PORT/port:PORT_LIST/port:name; - } - } - - leaf tagging_mode { - mandatory true; - type stypes:vlan_tagging_mode; - } - } - /* end of list VLAN_MEMBER_LIST */ - } - /* end of container VLAN_MEMBER */ - } - /* end of container sonic-vlan */ -} -/* end of module sonic-vlan */ diff --git a/tests/cli_autogen_yang_parser_test.py b/tests/cli_autogen_yang_parser_test.py index e9010c4877..76375f0fa3 100644 --- a/tests/cli_autogen_yang_parser_test.py +++ b/tests/cli_autogen_yang_parser_test.py @@ -17,14 +17,41 @@ class TestYangParser: - - def test_one_table_container(self): - yang_model_name = 'sonic-one-table-container' - template('sonic-one-table-container', assert_dictionaries.one_table_container) - - def test_many_table_containers(self): - yang_model_name = 'sonic-many-table-containers' - template('sonic-many-table-containers', assert_dictionaries.many_table_containers) + #def test_1_table_container(self): + # yang_model_name = 'sonic-1-table-container' + # template(yang_model_name, assert_dictionaries.one_table_container) + # + #def test_2_table_containers(self): + # yang_model_name = 'sonic-2-table-containers' + # template(yang_model_name, assert_dictionaries.two_table_containers) + + #def test_1_object_container(self): + # yang_model_name = 'sonic-1-object-container' + # template(yang_model_name, assert_dictionaries.one_object_container) + + #def test_2_object_containers(self): + # yang_model_name = 'sonic-2-object-containers' + # template(yang_model_name, assert_dictionaries.two_object_containers) + + #def test_1_list(self): + # yang_model_name = 'sonic-1-list' + # template(yang_model_name, assert_dictionaries.one_list) + + #def test_2_lists(self): + # yang_model_name = 'sonic-2-lists' + # template(yang_model_name, assert_dictionaries.two_lists) + + #def test_static_object_complex_1(self): + # """ Test object container with: 1 leaf, 1 leaf-list, 1 choice. + # """ + # yang_model_name = 'sonic-static-object-complex-1' + # template(yang_model_name, assert_dictionaries.static_object_complex_1) + + #def test_static_object_complex_2(self): + # """ Test object container with: 2 leafs, 2 leaf-lists, 2 choices. + # """ + # yang_model_name = 'sonic-static-object-complex-2' + # template(yang_model_name, assert_dictionaries.static_object_complex_2) def template(yang_model_name, correct_dict): config_db_path = os.path.join(test_path, 'cli_autogen_input/config_db.json') @@ -34,8 +61,8 @@ def template(yang_model_name, correct_dict): allow_tbl_without_yang = True, debug = False) yang_dict = parser.parse_yang_model() - # debug - pretty_log(yang_dict) + + pretty_log_debug(yang_dict) assert yang_dict == correct_dict @@ -62,7 +89,7 @@ def remove_yang_model(yang_model_name): os.system(cmd) # DEBUG function -def pretty_log(dictionary): +def pretty_log_debug(dictionary): for line in pprint.pformat(dictionary).split('\n'): - logging.warning(line) + logging.debug(line) From 5737f5f03b23547beca498b8cdd31f4380246a96 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Mon, 17 May 2021 15:23:50 +0000 Subject: [PATCH 31/76] added test_dynamic_object_complex Signed-off-by: Vadym Hlushko --- .../cli_autogen_input/assert_dictionaries.py | 122 ++++++++++++++++++ .../sonic-dynamic-object-complex-1.yang | 57 ++++++++ .../sonic-dynamic-object-complex-2.yang | 84 ++++++++++++ tests/cli_autogen_yang_parser_test.py | 12 ++ 4 files changed, 275 insertions(+) create mode 100644 tests/cli_autogen_input/sonic-dynamic-object-complex-1.yang create mode 100644 tests/cli_autogen_input/sonic-dynamic-object-complex-2.yang diff --git a/tests/cli_autogen_input/assert_dictionaries.py b/tests/cli_autogen_input/assert_dictionaries.py index 9a2989d500..3aac3c2f6f 100644 --- a/tests/cli_autogen_input/assert_dictionaries.py +++ b/tests/cli_autogen_input/assert_dictionaries.py @@ -240,4 +240,126 @@ ] } ] +} + +dynamic_object_complex_1 = { + "tables":[ + { + "description":"TABLE_1 description", + "name":"TABLE_1", + "dynamic-objects":[ + { + "name":"OBJECT_1_LIST", + "description":"OBJECT_1_LIST description", + "attrs":[ + { + "name":"OBJ_1_LEAF_1", + "description": "OBJ_1_LEAF_1 description", + "is-mandatory": False, + "is-leaf-list": False, + }, + { + "name":"OBJ_1_LEAF_LIST_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": True, + }, + { + "name":"OBJ_1_CHOICE_1_LEAF_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + }, + { + "name":"OBJ_1_CHOICE_1_LEAF_2", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + } + ], + "keys":[ + { + "name": "KEY_LEAF_1", + "description": "KEY_LEAF_1 description", + } + ] + } + ] + } + ] +} + +dynamic_object_complex_2 = { + "tables":[ + { + "description":"TABLE_1 description", + "name":"TABLE_1", + "dynamic-objects":[ + { + "name":"OBJECT_1_LIST", + "description":"OBJECT_1_LIST description", + "attrs":[ + { + "name":"OBJ_1_LEAF_1", + "description": "OBJ_1_LEAF_1 description", + "is-mandatory": False, + "is-leaf-list": False, + }, + { + "name":"OBJ_1_LEAF_2", + "description": "OBJ_1_LEAF_2 description", + "is-mandatory": False, + "is-leaf-list": False, + }, + { + "name":"OBJ_1_LEAF_LIST_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": True, + }, + { + "name":"OBJ_1_LEAF_LIST_2", + "description": "", + "is-mandatory": False, + "is-leaf-list": True, + }, + { + "name":"OBJ_1_CHOICE_1_LEAF_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + }, + { + "name":"OBJ_1_CHOICE_1_LEAF_2", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + }, + { + "name":"OBJ_1_CHOICE_2_LEAF_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + }, + { + "name":"OBJ_1_CHOICE_2_LEAF_2", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + } + ], + "keys":[ + { + "name": "KEY_LEAF_1", + "description": "KEY_LEAF_1 description", + }, + { + "name": "KEY_LEAF_2", + "description": "KEY_LEAF_2 description", + } + ] + } + ] + } + ] } \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-dynamic-object-complex-1.yang b/tests/cli_autogen_input/sonic-dynamic-object-complex-1.yang new file mode 100644 index 0000000000..9beb98549d --- /dev/null +++ b/tests/cli_autogen_input/sonic-dynamic-object-complex-1.yang @@ -0,0 +1,57 @@ +module sonic-dynamic-object-complex-1 { + + yang-version 1.1; + + namespace "http://github.com/Azure/dynamic-complex-1"; + prefix dynamic-complex-1; + + container sonic-dynamic-object-complex-1 { + /* sonic-dynamic-object-complex-1 - top level container */ + + container TABLE_1 { + /* TABLE_1 - table container */ + + description "TABLE_1 description"; + + list OBJECT_1_LIST { + /* OBJECT_1_LIST - dynamic object container, it have: + * 1 key, + * 1 leaf, + * 1 leaf-list + * 1 choice + */ + + description "OBJECT_1_LIST description"; + + key "KEY_LEAF_1"; + + leaf KEY_LEAF_1 { + description "KEY_LEAF_1 description"; + type string; + } + + leaf OBJ_1_LEAF_1 { + description "OBJ_1_LEAF_1 description"; + type string; + } + + leaf-list OBJ_1_LEAF_LIST_1 { + type string; + } + + choice OBJ_1_CHOICE_1 { + case OBJ_1_CHOICE_1_CASE_1 { + leaf OBJ_1_CHOICE_1_LEAF_1 { + type uint16; + } + } + case OBJ_1_CHOICE_1_CASE_2 { + leaf OBJ_1_CHOICE_1_LEAF_2 { + type string; + } + } + } + } + } + } +} \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-dynamic-object-complex-2.yang b/tests/cli_autogen_input/sonic-dynamic-object-complex-2.yang new file mode 100644 index 0000000000..00e25c8135 --- /dev/null +++ b/tests/cli_autogen_input/sonic-dynamic-object-complex-2.yang @@ -0,0 +1,84 @@ +module sonic-dynamic-object-complex-2 { + + yang-version 1.1; + + namespace "http://github.com/Azure/dynamic-complex-2"; + prefix dynamic-complex-2; + + container sonic-dynamic-object-complex-2 { + /* sonic-dynamic-object-complex-2 - top level container */ + + container TABLE_1 { + /* TABLE_1 - table container */ + + description "TABLE_1 description"; + + list OBJECT_1_LIST { + /* OBJECT_1_LIST - dynamic object container, it have: + * 2 keys + * 2 leaf, + * 2 leaf-list + * 2 choice + */ + + description "OBJECT_1_LIST description"; + + key "KEY_LEAF_1 KEY_LEAF_2"; + + leaf KEY_LEAF_1 { + description "KEY_LEAF_1 description"; + type string; + } + + leaf KEY_LEAF_2 { + description "KEY_LEAF_2 description"; + type string; + } + + leaf OBJ_1_LEAF_1 { + description "OBJ_1_LEAF_1 description"; + type string; + } + + leaf OBJ_1_LEAF_2 { + description "OBJ_1_LEAF_2 description"; + type string; + } + + leaf-list OBJ_1_LEAF_LIST_1 { + type string; + } + + leaf-list OBJ_1_LEAF_LIST_2 { + type string; + } + + choice OBJ_1_CHOICE_1 { + case OBJ_1_CHOICE_1_CASE_1 { + leaf OBJ_1_CHOICE_1_LEAF_1 { + type uint16; + } + } + case OBJ_1_CHOICE_1_CASE_2 { + leaf OBJ_1_CHOICE_1_LEAF_2 { + type string; + } + } + } + + choice OBJ_1_CHOICE_2 { + case OBJ_1_CHOICE_2_CASE_1 { + leaf OBJ_1_CHOICE_2_LEAF_1 { + type uint16; + } + } + case OBJ_1_CHOICE_2_CASE_2 { + leaf OBJ_1_CHOICE_2_LEAF_2 { + type string; + } + } + } + } + } + } +} \ No newline at end of file diff --git a/tests/cli_autogen_yang_parser_test.py b/tests/cli_autogen_yang_parser_test.py index 76375f0fa3..acabfe5082 100644 --- a/tests/cli_autogen_yang_parser_test.py +++ b/tests/cli_autogen_yang_parser_test.py @@ -53,6 +53,18 @@ class TestYangParser: # yang_model_name = 'sonic-static-object-complex-2' # template(yang_model_name, assert_dictionaries.static_object_complex_2) + #def test_dynamic_object_complex_1(self): + # """ Test object container with: 1 key, 1 leaf, 1 leaf-list, 1 choice. + # """ + # yang_model_name = 'sonic-dynamic-object-complex-1' + # template(yang_model_name, assert_dictionaries.dynamic_object_complex_1) + + def test_dynamic_object_complex_2(self): + """ Test object container with: 2 keys, 2 leafs, 2 leaf-list, 2 choice. + """ + yang_model_name = 'sonic-dynamic-object-complex-2' + template(yang_model_name, assert_dictionaries.dynamic_object_complex_2) + def template(yang_model_name, correct_dict): config_db_path = os.path.join(test_path, 'cli_autogen_input/config_db.json') move_yang_model(yang_model_name) From 7b167a6b5ad0c1a5e89b47c7cc0daada4e24f417 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Tue, 18 May 2021 14:54:28 +0300 Subject: [PATCH 32/76] [sonic-cli-gen] implement grouping in show template Signed-off-by: Stepan Blyschak --- .../templates/sonic-cli-gen/show.py.j2 | 55 ++++++++++++++----- 1 file changed, 41 insertions(+), 14 deletions(-) diff --git a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 index ea82b9761f..3859573d72 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 @@ -6,19 +6,46 @@ import tabulate import natsort import utilities_common.cli as clicommon -{% macro print_attr(attr) %} -{%- if not attr["is-leaf-list"] %} -entry.get("{{ attr.name }}", "N/A") -{%- else %} -"\n".join(entry.get("{{ attr.name }}", [])) -{%- endif %} -{% endmacro %} + +{% macro column_name(name) -%} +{{ name|upper|replace("_", " ")|replace("-", " ") }} +{%- endmacro %} + + +def print_attr_helper(entry, attr): + if attr["is-leaf-list"]: + return "\n".join(entry.get(attr["name"], [])) + return entry.get(attr["name"], "N/A") + + +def print_group_helper(entry, attrs): + data = [] + for attr in attrs: + if entry.get(attr["name"]): + data.append((attr["name"] + ":", print_attr_helper(entry, attr))) + return tabulate.tabulate(data, tablefmt="plain") -{% macro gen_header(attrs) %} -{% for attr in attrs %} -"{{ attr.name|upper|replace("_", " ")|replace("-", " ") }}", +{% macro gen_row(entry, attrs) -%} +[ +{%- for attr in attrs|rejectattr("group", "defined") %} +print_attr_helper({{ entry }}, {{ attr }}), +{%- endfor %} +{%- for group, attrs in attrs|selectattr("group", "defined")|groupby("group") %} +print_group_helper({{ entry }}, {{ attrs }}), +{%- endfor %} +] +{% endmacro %} + +{% macro gen_header(attrs) -%} +[ +{% for attr in attrs|rejectattr("group", "defined") %} +"{{ column_name(attr.name) }}", +{% endfor %} +{% for group, attrs in attrs|selectattr("group", "defined")|groupby("group") %} +"{{ column_name(group) }}", {% endfor %} +] {% endmacro %} @@ -37,12 +64,12 @@ def {{ table.name }}(): def {{ table.name }}_{{ object.name }}(db): """ {{ object.description }} """ - header = [{{ gen_header(object.attrs) }}] + header = {{ gen_header(object.attrs) }} body = [] table = db.cfgdb.get_table("{{ table.name }}") entry = table.get("{{ object.name }}", {}) - row = [{%- for attr in object.attrs -%} {{ print_attr(attr) }}, {%- endfor %}] + row = {{ gen_row("entry", object.attrs) }} body.append(row) click.echo(tabulate.tabulate(body, header)) @@ -73,7 +100,7 @@ def {{ table.name }}(): def {{ name }}(db): """ {{ object.description }} """ - header = [{{ gen_header(object["keys"] + object.attrs) }}] + header = {{ gen_header(object["keys"] + object.attrs) }} body = [] table = db.cfgdb.get_table("{{ table.name }}") @@ -82,7 +109,7 @@ def {{ name }}(db): if not isinstance(key, tuple): key = (key,) - row = [*key, {%- for attr in object.attrs -%} {{ print_attr(attr) }}, {%- endfor %}] + row = [*key] + {{ gen_row("entry", object.attrs) }} body.append(row) click.echo(tabulate.tabulate(body, header)) From 16e0ccac525a0d5e0257fe71b509681cd7c24596 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Tue, 18 May 2021 15:01:36 +0000 Subject: [PATCH 33/76] Reworked 'grouping' parser Signed-off-by: Vadym Hlushko --- sonic_cli_gen/generator.py | 9 +- sonic_cli_gen/yang_parser.py | 166 ++++++++++++++++++-------- tests/cli_autogen_yang_parser_test.py | 86 ++++++------- 3 files changed, 168 insertions(+), 93 deletions(-) diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py index f3fcf0e62b..37924b1ce5 100644 --- a/sonic_cli_gen/generator.py +++ b/sonic_cli_gen/generator.py @@ -26,10 +26,11 @@ def generate_cli_plugin(self, cli_group, plugin_name): allow_tbl_without_yang=True, debug=False) yang_dict = parser.parse_yang_model() - plugin_path = get_cli_plugin_path(cli_group, plugin_name + '_yang.py') - template = self.env.get_template(cli_group + '.py.j2') - with open(plugin_path, 'w') as plugin_py: - plugin_py.write(template.render(yang_dict)) + import pprint; pprint.pprint(yang_dict) + #plugin_path = get_cli_plugin_path(cli_group, plugin_name + '_yang.py') + #template = self.env.get_template(cli_group + '.py.j2') + #with open(plugin_path, 'w') as plugin_py: + # plugin_py.write(template.render(yang_dict)) def get_cli_plugin_path(command, plugin_name): pkg_loader = pkgutil.get_loader(f'{command}.plugins') diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index c77250f066..bb05d5a21a 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -118,17 +118,17 @@ def parse_yang_model(self) -> dict: # 'table' container it is a container that goes after 'top level' container if isinstance(self.y_table_containers, list): for tbl_cont in self.y_table_containers: - y2d_elem = on_table_container(self.y_module, tbl_cont) + y2d_elem = on_table_container(self.y_module, tbl_cont, self.conf_mgmt) self.yang_2_dict['tables'].append(y2d_elem) else: - y2d_elem = on_table_container(self.y_module, self.y_table_containers) + y2d_elem = on_table_container(self.y_module, self.y_table_containers, self.conf_mgmt) self.yang_2_dict['tables'].append(y2d_elem) return self.yang_2_dict #------------------------------HANDLERS--------------------------------# -def on_table_container(y_module: OrderedDict, tbl_cont: OrderedDict) -> dict: +def on_table_container(y_module: OrderedDict, tbl_cont: OrderedDict, conf_mgmt) -> dict: """ Parse 'table' container, 'table' container goes after 'top level' container @@ -153,10 +153,10 @@ def on_table_container(y_module: OrderedDict, tbl_cont: OrderedDict) -> dict: obj_cont = tbl_cont.get('container') if isinstance(obj_cont, list): for cont in obj_cont: - static_obj_elem = on_object_container(y_module, cont, is_list=False) + static_obj_elem = on_object_container(y_module, cont, conf_mgmt, is_list=False) y2d_elem['static-objects'].append(static_obj_elem) else: - static_obj_elem = on_object_container(y_module, obj_cont, is_list=False) + static_obj_elem = on_object_container(y_module, obj_cont, conf_mgmt, is_list=False) y2d_elem['static-objects'].append(static_obj_elem) else: y2d_elem['dynamic-objects'] = list() @@ -164,10 +164,10 @@ def on_table_container(y_module: OrderedDict, tbl_cont: OrderedDict) -> dict: # 'container' can have more than 1 'list' entity if isinstance(tbl_cont_lists, list): for _list in tbl_cont_lists: - dynamic_obj_elem = on_object_container(y_module, _list, is_list=True) + dynamic_obj_elem = on_object_container(y_module, _list, conf_mgmt, is_list=True) y2d_elem['dynamic-objects'].append(dynamic_obj_elem) else: - dynamic_obj_elem = on_object_container(y_module, tbl_cont_lists, is_list=True) + dynamic_obj_elem = on_object_container(y_module, tbl_cont_lists, conf_mgmt, is_list=True) y2d_elem['dynamic-objects'].append(dynamic_obj_elem) # move 'keys' elements from 'attrs' to 'keys' @@ -175,7 +175,7 @@ def on_table_container(y_module: OrderedDict, tbl_cont: OrderedDict) -> dict: return y2d_elem -def on_object_container(y_module: OrderedDict, cont: OrderedDict, is_list: bool) -> dict: +def on_object_container(y_module: OrderedDict, cont: OrderedDict, conf_mgmt, is_list: bool) -> dict: """ Parse a 'object container'. 'Object container' represent OBJECT inside Config DB schema: { @@ -208,58 +208,47 @@ def on_object_container(y_module: OrderedDict, cont: OrderedDict, is_list: bool) attrs_list = list() attrs_list.extend(get_leafs(cont)) attrs_list.extend(get_leaf_lists(cont)) - attrs_list.extend(get_choices(y_module, cont)) + attrs_list.extend(get_choices(y_module, cont, conf_mgmt)) # TODO: need to test 'grouping' - #attrs_list.extend(get_uses_grouping(y_module, cont)) + attrs_list.extend(get_uses(y_module, cont, conf_mgmt)) obj_elem['attrs'] = attrs_list return obj_elem -def on_grouping(y_module: OrderedDict, y_grouping, y_uses) -> list: - """ Parse a YANG 'grouping' and 'uses' entities - 'grouping' element can have - 'leaf', 'leaf-list', 'choice' +def on_uses(y_module: OrderedDict, y_uses, conf_mgmt) -> list: + """ Parse a YANG 'uses' entities + 'uses' refearing to 'grouping' YANG entity Args: y_module: reference to 'module' - y_grouping: reference to 'grouping' y_uses: reference to 'uses' Returns: dictionary - element for obj_elem['attrs'], 'attrs' contain a parsed 'leafs' """ - ret_attrs = list() + y_grouping = get_all_grouping(y_module, y_uses, conf_mgmt) - if isinstance(y_uses, list): - if isinstance(y_grouping, list): - for use in y_uses: - for group in y_grouping: - if use.get('@name') == group.get('@name'): - ret_attrs.extend(get_leafs(group)) - ret_attrs.extend(get_leaf_lists(group)) - ret_attrs.extend(get_choices(y_module, group)) - else: + if y_grouping == []: + # not sure if it can happend + raise Exception('EMPTY') + + for group in y_grouping: + if isinstance(y_uses, list): for use in y_uses: - if use.get('@name') == y_grouping.get('@name'): - ret_attrs.extend(get_leafs(y_grouping)) - ret_attrs.extend(get_leaf_lists(y_grouping)) - ret_attrs.extend(get_choices(y_module, y_grouping)) - else: - if isinstance(y_grouping, list): - for group in y_grouping: - if y_uses.get('@name') == group.get('@name'): + if use.get('@name') == group.get('@name'): ret_attrs.extend(get_leafs(group)) ret_attrs.extend(get_leaf_lists(group)) - ret_attrs.extend(get_choices(y_module, group)) + ret_attrs.extend(get_choices(y_module, group, conf_mgmt)) else: - if y_uses.get('@name') == y_grouping.get('@name'): - ret_attrs.extend(get_leafs(y_grouping)) - ret_attrs.extend(get_leaf_lists(y_grouping)) - ret_attrs.extend(get_choices(y_module, y_grouping)) + if y_uses.get('@name') == group.get('@name'): + ret_attrs.extend(get_leafs(group)) + ret_attrs.extend(get_leaf_lists(group)) + ret_attrs.extend(get_choices(y_module, group, conf_mgmt)) return ret_attrs -def on_choices(y_module: OrderedDict, y_choices) -> list: +def on_choices(y_module: OrderedDict, y_choices, conf_mgmt) -> list: """ Parse a YANG 'choice' entities Args: @@ -270,17 +259,17 @@ def on_choices(y_module: OrderedDict, y_choices) -> list: ret_attrs = list() - # the YANG model can have multiple 'choice' entities inside 'container' or 'list' + # the YANG model can have multiple 'choice' entities inside a 'container' or 'list' if isinstance(y_choices, list): for choice in y_choices: - attrs = on_choice_cases(y_module, choice.get('case')) + attrs = on_choice_cases(y_module, choice.get('case'), conf_mgmt) ret_attrs.extend(attrs) else: - ret_attrs = on_choice_cases(y_module, y_choices.get('case')) + ret_attrs = on_choice_cases(y_module, y_choices.get('case'), conf_mgmt) return ret_attrs -def on_choice_cases(y_module: OrderedDict, y_cases: list) -> list: +def on_choice_cases(y_module: OrderedDict, y_cases: list, conf_mgmt) -> list: """ Parse a single YANG 'case' entity from 'choice' entity 'case' element can have inside - 'leaf', 'leaf-list', 'uses' @@ -298,7 +287,7 @@ def on_choice_cases(y_module: OrderedDict, y_cases: list) -> list: ret_attrs.extend(get_leafs(case)) ret_attrs.extend(get_leaf_lists(case)) # TODO: need to deeply test it - #ret_attrs.extend(get_uses_grouping(y_module, case)) + ret_attrs.extend(get_uses(y_module, case, conf_mgmt)) else: raise Exception('It has no sense to using a single "case" element inside "choice" element') @@ -375,18 +364,99 @@ def get_leaf_lists(y_entity: OrderedDict) -> list: return [] -def get_choices(y_module: OrderedDict, y_entity: OrderedDict) -> list: +def get_choices(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt) -> list: if y_entity.get('choice') is not None: - return on_choices(y_module, y_entity.get('choice')) + return on_choices(y_module, y_entity.get('choice'), conf_mgmt) return [] -def get_uses_grouping(y_module: OrderedDict, y_entity: OrderedDict) -> list: - if y_entity.get('uses') is not None and y_module.get('grouping') is not None: - return on_grouping(y_module, y_module.get('grouping'), y_entity.get('uses')) +def get_uses(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt) -> list: + if y_entity.get('uses') is not None: + return on_uses(y_module, y_entity.get('uses'), conf_mgmt) return [] +def get_all_grouping(y_module: OrderedDict, y_uses: OrderedDict, conf_mgmt) -> list: + # WARNING + # TODO add to the design statement that grouping should be defined under the 'module' and NOT in nested containers + ret_grouping = list() + prefix_list = get_import_prefixes(y_uses) + + # in case if 'grouping' located in the same YANG model + local_grouping = y_module.get('grouping') + if local_grouping is not None: + if isinstance(local_grouping, list): + for group in local_grouping: + ret_grouping.append(group) + else: + ret_grouping.append(local_grouping) + + # if prefix_list is NOT empty it means that 'grouping' was imported from another YANG model + if prefix_list != []: + for prefix in prefix_list: + y_import = y_module.get('import') + if isinstance(y_import, list): + for _import in y_import: + if _import.get('prefix').get('@value') == prefix: + ret_grouping.extend(get_grouping_from_another_yang_model(_import.get('@module'), conf_mgmt)) + else: + if y_import.get('prefix').get('@value') == prefix: + ret_grouping.extend(get_grouping_from_another_yang_model(y_import.get('@module'), conf_mgmt)) + + return ret_grouping + +def get_grouping_from_another_yang_model(yang_model_name: str, conf_mgmt) -> list: + """ Get the YANG 'grouping' entity + + Args: + yang_model_name - YANG model to search + conf_mgmt - reference to ConfigMgmt class instance, + it have yJson object which contain all parsed YANG models + + Returns: + list - list 'grouping' entities + """ + ret_grouping = list() + + for i in range(len(conf_mgmt.sy.yJson)): + if (conf_mgmt.sy.yJson[i].get('module').get('@name') == yang_model_name): + grouping = conf_mgmt.sy.yJson[i].get('module').get('grouping') + if isinstance(grouping, list): + for group in grouping: + ret_grouping.append(group) + else: + ret_grouping.append(grouping) + + return ret_grouping + +def get_import_prefixes(y_uses: OrderedDict) -> list: + """ Parse 'import prefix' of YANG 'uses' entity + Example: + { + uses stypes:endpoint; + } + 'stypes' - prefix of imported YANG module. + 'endpoint' - YANG 'grouping' entity name + + Args: + y_uses: refrence to YANG 'uses' + Returns: + list - of parsed prefixes + """ + ret_prefixes = list() + + if isinstance(y_uses, list): + for use in y_uses: + prefix = use.get('@name').split(':')[0] + if prefix != use.get('@name'): + ret_prefixes.append(prefix) + else: + prefix = y_uses.get('@name').split(':')[0] + if prefix != y_uses.get('@name'): + ret_prefixes.append(prefix) + + return ret_prefixes + def get_list_keys(y_list: OrderedDict) -> list: ret_list = list() keys = y_list.get('key').get('@value').split() diff --git a/tests/cli_autogen_yang_parser_test.py b/tests/cli_autogen_yang_parser_test.py index acabfe5082..1e61b00fba 100644 --- a/tests/cli_autogen_yang_parser_test.py +++ b/tests/cli_autogen_yang_parser_test.py @@ -17,47 +17,47 @@ class TestYangParser: - #def test_1_table_container(self): - # yang_model_name = 'sonic-1-table-container' - # template(yang_model_name, assert_dictionaries.one_table_container) - # - #def test_2_table_containers(self): - # yang_model_name = 'sonic-2-table-containers' - # template(yang_model_name, assert_dictionaries.two_table_containers) - - #def test_1_object_container(self): - # yang_model_name = 'sonic-1-object-container' - # template(yang_model_name, assert_dictionaries.one_object_container) - - #def test_2_object_containers(self): - # yang_model_name = 'sonic-2-object-containers' - # template(yang_model_name, assert_dictionaries.two_object_containers) - - #def test_1_list(self): - # yang_model_name = 'sonic-1-list' - # template(yang_model_name, assert_dictionaries.one_list) - - #def test_2_lists(self): - # yang_model_name = 'sonic-2-lists' - # template(yang_model_name, assert_dictionaries.two_lists) - - #def test_static_object_complex_1(self): - # """ Test object container with: 1 leaf, 1 leaf-list, 1 choice. - # """ - # yang_model_name = 'sonic-static-object-complex-1' - # template(yang_model_name, assert_dictionaries.static_object_complex_1) - - #def test_static_object_complex_2(self): - # """ Test object container with: 2 leafs, 2 leaf-lists, 2 choices. - # """ - # yang_model_name = 'sonic-static-object-complex-2' - # template(yang_model_name, assert_dictionaries.static_object_complex_2) - - #def test_dynamic_object_complex_1(self): - # """ Test object container with: 1 key, 1 leaf, 1 leaf-list, 1 choice. - # """ - # yang_model_name = 'sonic-dynamic-object-complex-1' - # template(yang_model_name, assert_dictionaries.dynamic_object_complex_1) + def test_1_table_container(self): + yang_model_name = 'sonic-1-table-container' + template(yang_model_name, assert_dictionaries.one_table_container) + + def test_2_table_containers(self): + yang_model_name = 'sonic-2-table-containers' + template(yang_model_name, assert_dictionaries.two_table_containers) + + def test_1_object_container(self): + yang_model_name = 'sonic-1-object-container' + template(yang_model_name, assert_dictionaries.one_object_container) + + def test_2_object_containers(self): + yang_model_name = 'sonic-2-object-containers' + template(yang_model_name, assert_dictionaries.two_object_containers) + + def test_1_list(self): + yang_model_name = 'sonic-1-list' + template(yang_model_name, assert_dictionaries.one_list) + + def test_2_lists(self): + yang_model_name = 'sonic-2-lists' + template(yang_model_name, assert_dictionaries.two_lists) + + def test_static_object_complex_1(self): + """ Test object container with: 1 leaf, 1 leaf-list, 1 choice. + """ + yang_model_name = 'sonic-static-object-complex-1' + template(yang_model_name, assert_dictionaries.static_object_complex_1) + + def test_static_object_complex_2(self): + """ Test object container with: 2 leafs, 2 leaf-lists, 2 choices. + """ + yang_model_name = 'sonic-static-object-complex-2' + template(yang_model_name, assert_dictionaries.static_object_complex_2) + + def test_dynamic_object_complex_1(self): + """ Test object container with: 1 key, 1 leaf, 1 leaf-list, 1 choice. + """ + yang_model_name = 'sonic-dynamic-object-complex-1' + template(yang_model_name, assert_dictionaries.dynamic_object_complex_1) def test_dynamic_object_complex_2(self): """ Test object container with: 2 keys, 2 leafs, 2 leaf-list, 2 choice. @@ -65,6 +65,10 @@ def test_dynamic_object_complex_2(self): yang_model_name = 'sonic-dynamic-object-complex-2' template(yang_model_name, assert_dictionaries.dynamic_object_complex_2) + # TODO: UT for choice + # TODO: UT for grouping + + def template(yang_model_name, correct_dict): config_db_path = os.path.join(test_path, 'cli_autogen_input/config_db.json') move_yang_model(yang_model_name) From b0fb3a96ff9cc087f89fe49421604a036d34e0b7 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Tue, 18 May 2021 15:39:55 +0000 Subject: [PATCH 34/76] Added 'group' to 'attrs' Signed-off-by: Vadym Hlushko --- sonic_cli_gen/yang_parser.py | 35 +++++++------- .../cli_autogen_input/assert_dictionaries.py | 24 ++++++++++ .../sonic-choice-complex-1.yang | 47 +++++++++++++++++++ tests/cli_autogen_input/sonic-grouping.yang | 18 +++++++ tests/cli_autogen_yang_parser_test.py | 8 ++++ 5 files changed, 115 insertions(+), 17 deletions(-) create mode 100644 tests/cli_autogen_input/sonic-choice-complex-1.yang create mode 100644 tests/cli_autogen_input/sonic-grouping.yang diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index bb05d5a21a..4a8664b78e 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -206,8 +206,8 @@ def on_object_container(y_module: OrderedDict, cont: OrderedDict, conf_mgmt, is_ obj_elem['keys'] = get_list_keys(cont) attrs_list = list() - attrs_list.extend(get_leafs(cont)) - attrs_list.extend(get_leaf_lists(cont)) + attrs_list.extend(get_leafs(cont, grouping_name = '')) + attrs_list.extend(get_leaf_lists(cont, grouping_name = '')) attrs_list.extend(get_choices(y_module, cont, conf_mgmt)) # TODO: need to test 'grouping' attrs_list.extend(get_uses(y_module, cont, conf_mgmt)) @@ -237,13 +237,13 @@ def on_uses(y_module: OrderedDict, y_uses, conf_mgmt) -> list: if isinstance(y_uses, list): for use in y_uses: if use.get('@name') == group.get('@name'): - ret_attrs.extend(get_leafs(group)) - ret_attrs.extend(get_leaf_lists(group)) + ret_attrs.extend(get_leafs(group, group.get('@name'))) + ret_attrs.extend(get_leaf_lists(group, grouping_name = '')) ret_attrs.extend(get_choices(y_module, group, conf_mgmt)) else: if y_uses.get('@name') == group.get('@name'): - ret_attrs.extend(get_leafs(group)) - ret_attrs.extend(get_leaf_lists(group)) + ret_attrs.extend(get_leafs(group, group.get('@name'))) + ret_attrs.extend(get_leaf_lists(group, grouping_name = '')) ret_attrs.extend(get_choices(y_module, group, conf_mgmt)) return ret_attrs @@ -284,8 +284,8 @@ def on_choice_cases(y_module: OrderedDict, y_cases: list, conf_mgmt) -> list: if isinstance(y_cases, list): for case in y_cases: - ret_attrs.extend(get_leafs(case)) - ret_attrs.extend(get_leaf_lists(case)) + ret_attrs.extend(get_leafs(case, grouping_name = '')) + ret_attrs.extend(get_leaf_lists(case, grouping_name = '')) # TODO: need to deeply test it ret_attrs.extend(get_uses(y_module, case, conf_mgmt)) else: @@ -293,7 +293,7 @@ def on_choice_cases(y_module: OrderedDict, y_cases: list, conf_mgmt) -> list: return ret_attrs -def on_leafs(y_leafs, is_leaf_list: bool) -> list: +def on_leafs(y_leafs, grouping_name, is_leaf_list: bool) -> list: """ Parse all the 'leaf' or 'leaf-list' elements Args: @@ -306,15 +306,15 @@ def on_leafs(y_leafs, is_leaf_list: bool) -> list: # The YANG 'container' entity may have only 1 'leaf' element OR a list of 'leaf' elements if isinstance(y_leafs, list): for leaf in y_leafs: - attr = on_leaf(leaf, is_leaf_list) + attr = on_leaf(leaf, is_leaf_list, grouping_name) ret_attrs.append(attr) else: - attr = on_leaf(y_leafs, is_leaf_list) + attr = on_leaf(y_leafs, is_leaf_list, grouping_name) ret_attrs.append(attr) return ret_attrs -def on_leaf(leaf: OrderedDict, is_leaf_list: bool) -> dict: +def on_leaf(leaf: OrderedDict, is_leaf_list: bool, grouping_name: str) -> dict: """ Parse a single 'leaf' element Args: @@ -326,7 +326,8 @@ def on_leaf(leaf: OrderedDict, is_leaf_list: bool) -> dict: attr = { 'name': leaf.get('@name'), 'description': get_description(leaf), 'is-leaf-list': is_leaf_list, - 'is-mandatory': get_mandatory(leaf) } + 'is-mandatory': get_mandatory(leaf), + 'group': grouping_name} return attr @@ -352,15 +353,15 @@ def get_description(y_entity: OrderedDict) -> str: else: return '' -def get_leafs(y_entity: OrderedDict) -> list: +def get_leafs(y_entity: OrderedDict, grouping_name) -> list: if y_entity.get('leaf') is not None: - return on_leafs(y_entity.get('leaf'), is_leaf_list=False) + return on_leafs(y_entity.get('leaf'), grouping_name, is_leaf_list=False) return [] -def get_leaf_lists(y_entity: OrderedDict) -> list: +def get_leaf_lists(y_entity: OrderedDict, grouping_name) -> list: if y_entity.get('leaf-list') is not None: - return on_leafs(y_entity.get('leaf-list'), is_leaf_list=True) + return on_leafs(y_entity.get('leaf-list'), grouping_name, is_leaf_list=True) return [] diff --git a/tests/cli_autogen_input/assert_dictionaries.py b/tests/cli_autogen_input/assert_dictionaries.py index 3aac3c2f6f..751784dae9 100644 --- a/tests/cli_autogen_input/assert_dictionaries.py +++ b/tests/cli_autogen_input/assert_dictionaries.py @@ -151,24 +151,28 @@ "description": "OBJ_1_LEAF_1 description", "is-mandatory": False, "is-leaf-list": False, + "group": '', }, { "name":"OBJ_1_LEAF_LIST_1", "description": "", "is-mandatory": False, "is-leaf-list": True, + "group": '', }, { "name":"OBJ_1_CHOICE_1_LEAF_1", "description": "", "is-mandatory": False, "is-leaf-list": False, + "group": '', }, { "name":"OBJ_1_CHOICE_1_LEAF_2", "description": "", "is-mandatory": False, "is-leaf-list": False, + "group": '', } ] } @@ -192,48 +196,56 @@ "description": "OBJ_1_LEAF_1 description", "is-mandatory": False, "is-leaf-list": False, + "group": '', }, { "name":"OBJ_1_LEAF_2", "description": "OBJ_1_LEAF_2 description", "is-mandatory": False, "is-leaf-list": False, + "group": '', }, { "name":"OBJ_1_LEAF_LIST_1", "description": "", "is-mandatory": False, "is-leaf-list": True, + "group": '', }, { "name":"OBJ_1_LEAF_LIST_2", "description": "", "is-mandatory": False, "is-leaf-list": True, + "group": '', }, { "name":"OBJ_1_CHOICE_1_LEAF_1", "description": "", "is-mandatory": False, "is-leaf-list": False, + "group": '', }, { "name":"OBJ_1_CHOICE_1_LEAF_2", "description": "", "is-mandatory": False, "is-leaf-list": False, + "group": '', }, { "name":"OBJ_1_CHOICE_2_LEAF_1", "description": "", "is-mandatory": False, "is-leaf-list": False, + "group": '', }, { "name":"OBJ_1_CHOICE_2_LEAF_2", "description": "", "is-mandatory": False, "is-leaf-list": False, + "group": '', }, ] } @@ -257,24 +269,28 @@ "description": "OBJ_1_LEAF_1 description", "is-mandatory": False, "is-leaf-list": False, + "group": '', }, { "name":"OBJ_1_LEAF_LIST_1", "description": "", "is-mandatory": False, "is-leaf-list": True, + "group": '', }, { "name":"OBJ_1_CHOICE_1_LEAF_1", "description": "", "is-mandatory": False, "is-leaf-list": False, + "group": '', }, { "name":"OBJ_1_CHOICE_1_LEAF_2", "description": "", "is-mandatory": False, "is-leaf-list": False, + "group": '', } ], "keys":[ @@ -304,48 +320,56 @@ "description": "OBJ_1_LEAF_1 description", "is-mandatory": False, "is-leaf-list": False, + "group": '', }, { "name":"OBJ_1_LEAF_2", "description": "OBJ_1_LEAF_2 description", "is-mandatory": False, "is-leaf-list": False, + "group": '', }, { "name":"OBJ_1_LEAF_LIST_1", "description": "", "is-mandatory": False, "is-leaf-list": True, + "group": '', }, { "name":"OBJ_1_LEAF_LIST_2", "description": "", "is-mandatory": False, "is-leaf-list": True, + "group": '', }, { "name":"OBJ_1_CHOICE_1_LEAF_1", "description": "", "is-mandatory": False, "is-leaf-list": False, + "group": '', }, { "name":"OBJ_1_CHOICE_1_LEAF_2", "description": "", "is-mandatory": False, "is-leaf-list": False, + "group": '', }, { "name":"OBJ_1_CHOICE_2_LEAF_1", "description": "", "is-mandatory": False, "is-leaf-list": False, + "group": '', }, { "name":"OBJ_1_CHOICE_2_LEAF_2", "description": "", "is-mandatory": False, "is-leaf-list": False, + "group": '', } ], "keys":[ diff --git a/tests/cli_autogen_input/sonic-choice-complex-1.yang b/tests/cli_autogen_input/sonic-choice-complex-1.yang new file mode 100644 index 0000000000..4e45717493 --- /dev/null +++ b/tests/cli_autogen_input/sonic-choice-complex-1.yang @@ -0,0 +1,47 @@ +module sonic-static-object-complex-1 { + + yang-version 1.1; + + namespace "http://github.com/Azure/static-complex-1"; + prefix static-complex-1; + + import sonic-grouping { + prefix sgrop; + } + + container sonic-static-object-complex-1 { + /* sonic-static-object-complex-1 - top level container */ + + container TABLE_1 { + /* TABLE_1 - table container */ + + description "TABLE_1 description"; + + container OBJECT_1 { + /* OBJECT_1 - object container, it have: + * 1 choice + */ + + description "OBJECT_1 description"; + + choice CHOICE_1 { + case CHOICE_1_CASE_1 { + leaf LEAF_1 { + type uint16; + } + + leaf-list LEAF_LIST_1 { + type string; + } + } + + case CHOICE_1_CASE_2 { + leaf LEAF_2 { + type string; + } + } + } + } + } + } +} \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-grouping.yang b/tests/cli_autogen_input/sonic-grouping.yang new file mode 100644 index 0000000000..59aa0b5948 --- /dev/null +++ b/tests/cli_autogen_input/sonic-grouping.yang @@ -0,0 +1,18 @@ +module sonic-grouping{ + + yang-version 1.1; + + namespace "http://github.com/Azure/s-grouping"; + prefix s-grouping; + + grouping target { + leaf t_address { + type inet:ip-address; + description "Target IP address."; + } + leaf t_port { + type inet:port-number; + description "Target port number."; + } + } +} \ No newline at end of file diff --git a/tests/cli_autogen_yang_parser_test.py b/tests/cli_autogen_yang_parser_test.py index 1e61b00fba..647f9e1907 100644 --- a/tests/cli_autogen_yang_parser_test.py +++ b/tests/cli_autogen_yang_parser_test.py @@ -65,6 +65,14 @@ def test_dynamic_object_complex_2(self): yang_model_name = 'sonic-dynamic-object-complex-2' template(yang_model_name, assert_dictionaries.dynamic_object_complex_2) + #def test_choice_complex_1(self): + # """ Test object container with choice that have: 1 leaf, 1 leaf-list, 1 uses + # """ + # yang_model_name = 'sonic-choice-complex' + # grouping_yang = 'sonic-grouping' + # move_yang_model(grouping_yang) + # template(yang_model_name, assert_dictionaries.dynamic_object_complex_2) + # TODO: UT for choice # TODO: UT for grouping From e2acf73e3ee09e4c70c0bd163b02c77cc102395d Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Wed, 19 May 2021 11:06:32 +0000 Subject: [PATCH 35/76] Fixed 'grouping' parsing Signed-off-by: Vadym Hlushko --- sonic_cli_gen/yang_parser.py | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index 4a8664b78e..91f65c0da3 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -228,26 +228,30 @@ def on_uses(y_module: OrderedDict, y_uses, conf_mgmt) -> list: """ ret_attrs = list() y_grouping = get_all_grouping(y_module, y_uses, conf_mgmt) + # trim prefixes in order to the next checks + trim_uses_prefixes(y_uses) if y_grouping == []: # not sure if it can happend raise Exception('EMPTY') + # TODO: 'refine' support for group in y_grouping: if isinstance(y_uses, list): for use in y_uses: - if use.get('@name') == group.get('@name'): + if group.get('@name') == use.get('@name'): ret_attrs.extend(get_leafs(group, group.get('@name'))) ret_attrs.extend(get_leaf_lists(group, grouping_name = '')) ret_attrs.extend(get_choices(y_module, group, conf_mgmt)) else: - if y_uses.get('@name') == group.get('@name'): + if group.get('@name') == y_uses.get('@name'): ret_attrs.extend(get_leafs(group, group.get('@name'))) ret_attrs.extend(get_leaf_lists(group, grouping_name = '')) ret_attrs.extend(get_choices(y_module, group, conf_mgmt)) return ret_attrs + def on_choices(y_module: OrderedDict, y_choices, conf_mgmt) -> list: """ Parse a YANG 'choice' entities @@ -378,6 +382,8 @@ def get_uses(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt) -> list: return [] def get_all_grouping(y_module: OrderedDict, y_uses: OrderedDict, conf_mgmt) -> list: + """ Get all 'grouping' entities that is 'uses' in current YANG + """ # WARNING # TODO add to the design statement that grouping should be defined under the 'module' and NOT in nested containers ret_grouping = list() @@ -458,6 +464,28 @@ def get_import_prefixes(y_uses: OrderedDict) -> list: return ret_prefixes +def trim_uses_prefixes(y_uses) -> list: + """ Trim prefixes from 'uses' YANG entities. + If YANG 'grouping' was imported from another YANG file, it use 'prefix' before 'grouping' name: + { + uses sgrop:endpoint; + } + Where 'sgrop' = 'prefix'; 'endpoint' = 'grouping' name. + + Args: + y_uses - reference to 'uses' + """ + prefixes = get_import_prefixes(y_uses) + + for prefix in prefixes: + if isinstance(y_uses, list): + for use in y_uses: + if prefix in use.get('@name'): + use['@name'] = use.get('@name').split(':')[1] + else: + if prefix in y_uses.get('@name'): + y_uses['@name'] = y_uses.get('@name').split(':')[1] + def get_list_keys(y_list: OrderedDict) -> list: ret_list = list() keys = y_list.get('key').get('@value').split() From 3305faa0e84cce01d5a518981caf51201961e2f6 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Wed, 19 May 2021 16:52:39 +0000 Subject: [PATCH 36/76] DONE. ALL UT PASSED Signed-off-by: Vadym Hlushko --- sonic_cli_gen/yang_parser.py | 26 +- .../cli_autogen_input/assert_dictionaries.py | 236 ++++++++++++++++++ .../sonic-choice-complex-1.yang | 47 ---- .../sonic-choice-complex.yang | 91 +++++++ tests/cli_autogen_input/sonic-grouping-1.yang | 25 ++ tests/cli_autogen_input/sonic-grouping-2.yang | 25 ++ .../sonic-grouping-complex.yang | 96 +++++++ tests/cli_autogen_input/sonic-grouping.yang | 18 -- tests/cli_autogen_yang_parser_test.py | 40 ++- 9 files changed, 515 insertions(+), 89 deletions(-) delete mode 100644 tests/cli_autogen_input/sonic-choice-complex-1.yang create mode 100644 tests/cli_autogen_input/sonic-choice-complex.yang create mode 100644 tests/cli_autogen_input/sonic-grouping-1.yang create mode 100644 tests/cli_autogen_input/sonic-grouping-2.yang create mode 100644 tests/cli_autogen_input/sonic-grouping-complex.yang delete mode 100644 tests/cli_autogen_input/sonic-grouping.yang diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index 91f65c0da3..428aa6a23e 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -208,7 +208,7 @@ def on_object_container(y_module: OrderedDict, cont: OrderedDict, conf_mgmt, is_ attrs_list = list() attrs_list.extend(get_leafs(cont, grouping_name = '')) attrs_list.extend(get_leaf_lists(cont, grouping_name = '')) - attrs_list.extend(get_choices(y_module, cont, conf_mgmt)) + attrs_list.extend(get_choices(y_module, cont, conf_mgmt, grouping_name = '')) # TODO: need to test 'grouping' attrs_list.extend(get_uses(y_module, cont, conf_mgmt)) @@ -241,18 +241,18 @@ def on_uses(y_module: OrderedDict, y_uses, conf_mgmt) -> list: for use in y_uses: if group.get('@name') == use.get('@name'): ret_attrs.extend(get_leafs(group, group.get('@name'))) - ret_attrs.extend(get_leaf_lists(group, grouping_name = '')) - ret_attrs.extend(get_choices(y_module, group, conf_mgmt)) + ret_attrs.extend(get_leaf_lists(group, group.get('@name'))) + ret_attrs.extend(get_choices(y_module, group, conf_mgmt, group.get('@name'))) else: if group.get('@name') == y_uses.get('@name'): ret_attrs.extend(get_leafs(group, group.get('@name'))) - ret_attrs.extend(get_leaf_lists(group, grouping_name = '')) - ret_attrs.extend(get_choices(y_module, group, conf_mgmt)) + ret_attrs.extend(get_leaf_lists(group, group.get('@name'))) + ret_attrs.extend(get_choices(y_module, group, conf_mgmt, group.get('@name'))) return ret_attrs -def on_choices(y_module: OrderedDict, y_choices, conf_mgmt) -> list: +def on_choices(y_module: OrderedDict, y_choices, conf_mgmt: ConfigMgmt, grouping_name: str) -> list: """ Parse a YANG 'choice' entities Args: @@ -266,14 +266,14 @@ def on_choices(y_module: OrderedDict, y_choices, conf_mgmt) -> list: # the YANG model can have multiple 'choice' entities inside a 'container' or 'list' if isinstance(y_choices, list): for choice in y_choices: - attrs = on_choice_cases(y_module, choice.get('case'), conf_mgmt) + attrs = on_choice_cases(y_module, choice.get('case'), conf_mgmt, grouping_name) ret_attrs.extend(attrs) else: - ret_attrs = on_choice_cases(y_module, y_choices.get('case'), conf_mgmt) + ret_attrs = on_choice_cases(y_module, y_choices.get('case'), conf_mgmt, grouping_name) return ret_attrs -def on_choice_cases(y_module: OrderedDict, y_cases: list, conf_mgmt) -> list: +def on_choice_cases(y_module: OrderedDict, y_cases: list, conf_mgmt: ConfigMgmt, grouping_name: str) -> list: """ Parse a single YANG 'case' entity from 'choice' entity 'case' element can have inside - 'leaf', 'leaf-list', 'uses' @@ -288,8 +288,8 @@ def on_choice_cases(y_module: OrderedDict, y_cases: list, conf_mgmt) -> list: if isinstance(y_cases, list): for case in y_cases: - ret_attrs.extend(get_leafs(case, grouping_name = '')) - ret_attrs.extend(get_leaf_lists(case, grouping_name = '')) + ret_attrs.extend(get_leafs(case, grouping_name)) + ret_attrs.extend(get_leaf_lists(case, grouping_name)) # TODO: need to deeply test it ret_attrs.extend(get_uses(y_module, case, conf_mgmt)) else: @@ -369,9 +369,9 @@ def get_leaf_lists(y_entity: OrderedDict, grouping_name) -> list: return [] -def get_choices(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt) -> list: +def get_choices(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: ConfigMgmt, grouping_name: str) -> list: if y_entity.get('choice') is not None: - return on_choices(y_module, y_entity.get('choice'), conf_mgmt) + return on_choices(y_module, y_entity.get('choice'), conf_mgmt, grouping_name) return [] diff --git a/tests/cli_autogen_input/assert_dictionaries.py b/tests/cli_autogen_input/assert_dictionaries.py index 751784dae9..263e48366d 100644 --- a/tests/cli_autogen_input/assert_dictionaries.py +++ b/tests/cli_autogen_input/assert_dictionaries.py @@ -386,4 +386,240 @@ ] } ] +} + +choice_complex = { + "tables":[ + { + "description":"TABLE_1 description", + "name":"TABLE_1", + "static-objects":[ + { + "name":"OBJECT_1", + "description":"OBJECT_1 description", + "attrs":[ + { + "name":"LEAF_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": '', + }, + { + "name":"LEAF_LIST_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": True, + "group": '', + }, + { + "name":"GR_1_LEAF_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": "GR_1", + }, + { + "name":"GR_1_LEAF_2", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": 'GR_1', + }, + { + "name":"LEAF_2", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": '', + }, + { + "name":"LEAF_3", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": '', + }, + { + "name":"LEAF_LIST_2", + "description": "", + "is-mandatory": False, + "is-leaf-list": True, + "group": '', + }, + { + "name":"LEAF_LIST_3", + "description": "", + "is-mandatory": False, + "is-leaf-list": True, + "group": '', + }, + { + "name":"GR_5_LEAF_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": 'GR_5', + }, + { + "name":"GR_5_LEAF_2", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": 'GR_5', + }, + { + "name":"GR_2_LEAF_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": 'GR_2', + }, + { + "name":"GR_2_LEAF_2", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": 'GR_2', + }, + { + "name":"GR_3_LEAF_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": 'GR_3', + }, + { + "name":"GR_3_LEAF_2", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": 'GR_3', + }, + ] + } + ] + } + ] +} + +grouping_complex = { + "tables":[ + { + "description":"TABLE_1 description", + "name":"TABLE_1", + "static-objects":[ + { + "name":"OBJECT_1", + "description":"OBJECT_1 description", + "attrs":[ + { + "name":"GR_1_LEAF_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": "GR_1", + }, + { + "name":"GR_1_LEAF_2", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": 'GR_1', + }, + ] + }, + { + "name":"OBJECT_2", + "description":"OBJECT_2 description", + "attrs":[ + { + "name":"GR_5_LEAF_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": "GR_5", + }, + { + "name":"GR_5_LEAF_LIST_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": True, + "group": "GR_5", + }, + { + "name":"GR_6_LEAF_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": "GR_6", + }, + { + "name":"GR_6_LEAF_2", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": "GR_6", + }, + { + "name":"GR_6_CASE_1_LEAF_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": "GR_6", + }, + { + "name":"GR_6_CASE_1_LEAF_LIST_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": True, + "group": "GR_6", + }, + { + "name":"GR_6_CASE_2_LEAF_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": "GR_6", + }, + { + "name":"GR_6_CASE_2_LEAF_2", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": "GR_6", + }, + { + "name":"GR_6_CASE_2_LEAF_LIST_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": True, + "group": "GR_6", + }, + { + "name":"GR_6_CASE_2_LEAF_LIST_2", + "description": "", + "is-mandatory": False, + "is-leaf-list": True, + "group": "GR_6", + }, + { + "name":"GR_4_LEAF_1", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": "GR_4", + }, + { + "name":"GR_4_LEAF_2", + "description": "", + "is-mandatory": False, + "is-leaf-list": False, + "group": "GR_4", + }, + ] + } + ] + } + ] } \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-choice-complex-1.yang b/tests/cli_autogen_input/sonic-choice-complex-1.yang deleted file mode 100644 index 4e45717493..0000000000 --- a/tests/cli_autogen_input/sonic-choice-complex-1.yang +++ /dev/null @@ -1,47 +0,0 @@ -module sonic-static-object-complex-1 { - - yang-version 1.1; - - namespace "http://github.com/Azure/static-complex-1"; - prefix static-complex-1; - - import sonic-grouping { - prefix sgrop; - } - - container sonic-static-object-complex-1 { - /* sonic-static-object-complex-1 - top level container */ - - container TABLE_1 { - /* TABLE_1 - table container */ - - description "TABLE_1 description"; - - container OBJECT_1 { - /* OBJECT_1 - object container, it have: - * 1 choice - */ - - description "OBJECT_1 description"; - - choice CHOICE_1 { - case CHOICE_1_CASE_1 { - leaf LEAF_1 { - type uint16; - } - - leaf-list LEAF_LIST_1 { - type string; - } - } - - case CHOICE_1_CASE_2 { - leaf LEAF_2 { - type string; - } - } - } - } - } - } -} \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-choice-complex.yang b/tests/cli_autogen_input/sonic-choice-complex.yang new file mode 100644 index 0000000000..7d6a66d89f --- /dev/null +++ b/tests/cli_autogen_input/sonic-choice-complex.yang @@ -0,0 +1,91 @@ +module sonic-choice-complex { + + yang-version 1.1; + + namespace "http://github.com/Azure/choice-complex"; + prefix choice-complex; + + import sonic-grouping-1 { + prefix sgroup1; + } + + import sonic-grouping-2 { + prefix sgroup2; + } + + grouping GR_5 { + leaf GR_5_LEAF_1 { + type string; + } + + leaf GR_5_LEAF_2 { + type string; + } + } + + grouping GR_6 { + leaf GR_6_LEAF_1 { + type string; + } + + leaf GR_6_LEAF_2 { + type string; + } + } + + container sonic-choice-complex { + /* sonic-choice-complex - top level container */ + + container TABLE_1 { + /* TABLE_1 - table container */ + + description "TABLE_1 description"; + + container OBJECT_1 { + /* OBJECT_1 - object container, it have + * 1 choice, which have 2 cases. + * first case have: 1 leaf, 1 leaf-list, 1 uses + * second case have: 2 leafs, 2 leaf-lists, 2 uses + */ + + description "OBJECT_1 description"; + + choice CHOICE_1 { + case CHOICE_1_CASE_1 { + leaf LEAF_1 { + type uint16; + } + + leaf-list LEAF_LIST_1 { + type string; + } + + uses sgroup1:GR_1; + } + + case CHOICE_1_CASE_2 { + leaf LEAF_2 { + type string; + } + + leaf LEAF_3 { + type string; + } + + leaf-list LEAF_LIST_2 { + type string; + } + + leaf-list LEAF_LIST_3 { + type string; + } + + uses GR_5; + uses sgroup1:GR_2; + uses sgroup2:GR_3; + } + } + } + } + } +} \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-grouping-1.yang b/tests/cli_autogen_input/sonic-grouping-1.yang new file mode 100644 index 0000000000..831c3a4ad8 --- /dev/null +++ b/tests/cli_autogen_input/sonic-grouping-1.yang @@ -0,0 +1,25 @@ +module sonic-grouping-1{ + + yang-version 1.1; + + namespace "http://github.com/Azure/s-grouping-1"; + prefix s-grouping-1; + + grouping GR_1 { + leaf GR_1_LEAF_1 { + type string; + } + leaf GR_1_LEAF_2 { + type string; + } + } + + grouping GR_2 { + leaf GR_2_LEAF_1 { + type string; + } + leaf GR_2_LEAF_2 { + type string; + } + } +} \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-grouping-2.yang b/tests/cli_autogen_input/sonic-grouping-2.yang new file mode 100644 index 0000000000..bfaa13db15 --- /dev/null +++ b/tests/cli_autogen_input/sonic-grouping-2.yang @@ -0,0 +1,25 @@ +module sonic-grouping-2 { + + yang-version 1.1; + + namespace "http://github.com/Azure/s-grouping-2"; + prefix s-grouping-2; + + grouping GR_3 { + leaf GR_3_LEAF_1 { + type string; + } + leaf GR_3_LEAF_2 { + type string; + } + } + + grouping GR_4 { + leaf GR_4_LEAF_1 { + type string; + } + leaf GR_4_LEAF_2 { + type string; + } + } +} \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-grouping-complex.yang b/tests/cli_autogen_input/sonic-grouping-complex.yang new file mode 100644 index 0000000000..d6ed68563a --- /dev/null +++ b/tests/cli_autogen_input/sonic-grouping-complex.yang @@ -0,0 +1,96 @@ +module sonic-grouping-complex { + + yang-version 1.1; + + namespace "http://github.com/Azure/grouping-complex"; + prefix grouping-complex; + + import sonic-grouping-1 { + prefix sgroup1; + } + + import sonic-grouping-2 { + prefix sgroup2; + } + + grouping GR_5 { + leaf GR_5_LEAF_1 { + type string; + } + + leaf-list GR_5_LEAF_LIST_1 { + type string; + } + } + + grouping GR_6 { + leaf GR_6_LEAF_1 { + type string; + } + + leaf GR_6_LEAF_2 { + type string; + } + + choice GR_6_CHOICE_1 { + case CHOICE_1_CASE_1 { + leaf GR_6_CASE_1_LEAF_1 { + type uint16; + } + + leaf-list GR_6_CASE_1_LEAF_LIST_1 { + type string; + } + } + + case CHOICE_1_CASE_2 { + leaf GR_6_CASE_2_LEAF_1 { + type uint16; + } + + leaf GR_6_CASE_2_LEAF_2 { + type uint16; + } + + leaf-list GR_6_CASE_2_LEAF_LIST_1 { + type string; + } + + leaf-list GR_6_CASE_2_LEAF_LIST_2 { + type string; + } + } + } + } + + container sonic-grouping-complex { + /* sonic-grouping-complex - top level container */ + + container TABLE_1 { + /* TABLE_1 - table container */ + + description "TABLE_1 description"; + + container OBJECT_1 { + /* OBJECT_1 - object container, it have + * 1 choice, which have 2 cases. + * first case have: 1 leaf, 1 leaf-list, 1 uses + * second case have: 2 leafs, 2 leaf-lists, 2 uses + */ + + description "OBJECT_1 description"; + + uses sgroup1:GR_1; + } + + container OBJECT_2 { + + description "OBJECT_2 description"; + + uses GR_5; + uses GR_6; + uses sgroup2:GR_4; + } + } + } +} \ No newline at end of file diff --git a/tests/cli_autogen_input/sonic-grouping.yang b/tests/cli_autogen_input/sonic-grouping.yang deleted file mode 100644 index 59aa0b5948..0000000000 --- a/tests/cli_autogen_input/sonic-grouping.yang +++ /dev/null @@ -1,18 +0,0 @@ -module sonic-grouping{ - - yang-version 1.1; - - namespace "http://github.com/Azure/s-grouping"; - prefix s-grouping; - - grouping target { - leaf t_address { - type inet:ip-address; - description "Target IP address."; - } - leaf t_port { - type inet:port-number; - description "Target port number."; - } - } -} \ No newline at end of file diff --git a/tests/cli_autogen_yang_parser_test.py b/tests/cli_autogen_yang_parser_test.py index 647f9e1907..bb4f634b3b 100644 --- a/tests/cli_autogen_yang_parser_test.py +++ b/tests/cli_autogen_yang_parser_test.py @@ -17,6 +17,9 @@ class TestYangParser: + #create function like 'start' which copy all YANG to location + #create function teardown + def test_1_table_container(self): yang_model_name = 'sonic-1-table-container' template(yang_model_name, assert_dictionaries.one_table_container) @@ -65,17 +68,32 @@ def test_dynamic_object_complex_2(self): yang_model_name = 'sonic-dynamic-object-complex-2' template(yang_model_name, assert_dictionaries.dynamic_object_complex_2) - #def test_choice_complex_1(self): - # """ Test object container with choice that have: 1 leaf, 1 leaf-list, 1 uses - # """ - # yang_model_name = 'sonic-choice-complex' - # grouping_yang = 'sonic-grouping' - # move_yang_model(grouping_yang) - # template(yang_model_name, assert_dictionaries.dynamic_object_complex_2) - - # TODO: UT for choice - # TODO: UT for grouping - + def test_choice_complex(self): + """ Test object container with choice that have complex strucutre: + leafs, leaf-lists, multiple 'uses' from different files + """ + yang_model_name = 'sonic-choice-complex' + grouping_yang_1 = 'sonic-grouping-1' + grouping_yang_2 = 'sonic-grouping-2' + move_yang_model(grouping_yang_1) + move_yang_model(grouping_yang_2) + template(yang_model_name, assert_dictionaries.choice_complex) + remove_yang_model(grouping_yang_1) + remove_yang_model(grouping_yang_2) + + def test_choice_complex(self): + """ Test object container with muplitple 'uses' that using 'grouping' + from different files. The used 'grouping' have a complex strucutre: + leafs, leaf-lists, choices + """ + yang_model_name = 'sonic-grouping-complex' + grouping_yang_1 = 'sonic-grouping-1' + grouping_yang_2 = 'sonic-grouping-2' + move_yang_model(grouping_yang_1) + move_yang_model(grouping_yang_2) + template(yang_model_name, assert_dictionaries.grouping_complex) + remove_yang_model(grouping_yang_1) + remove_yang_model(grouping_yang_2) def template(yang_model_name, correct_dict): config_db_path = os.path.join(test_path, 'cli_autogen_input/config_db.json') From 30e8ac9b9decaa494b2ce403569d0abe602d8dec Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Wed, 19 May 2021 17:53:02 +0000 Subject: [PATCH 37/76] Refactored UT Signed-off-by: Vadym Hlushko --- tests/cli_autogen_yang_parser_test.py | 124 ++++++++++++-------------- 1 file changed, 56 insertions(+), 68 deletions(-) diff --git a/tests/cli_autogen_yang_parser_test.py b/tests/cli_autogen_yang_parser_test.py index bb4f634b3b..c6a81bef50 100644 --- a/tests/cli_autogen_yang_parser_test.py +++ b/tests/cli_autogen_yang_parser_test.py @@ -1,137 +1,125 @@ -import sys import os -import pytest import logging -# debug import pprint from sonic_cli_gen.yang_parser import YangParser from .cli_autogen_input import assert_dictionaries - logger = logging.getLogger(__name__) test_path = os.path.dirname(os.path.abspath(__file__)) yang_models_path = '/usr/local/yang-models' - +test_yang_models = [ + 'sonic-1-table-container', + 'sonic-2-table-containers', + 'sonic-1-object-container', + 'sonic-2-object-containers', + 'sonic-1-list', + 'sonic-2-lists', + 'sonic-static-object-complex-1', + 'sonic-static-object-complex-2', + 'sonic-dynamic-object-complex-1', + 'sonic-dynamic-object-complex-2', + 'sonic-choice-complex', + 'sonic-grouping-complex', + 'sonic-grouping-1', + 'sonic-grouping-2', +] class TestYangParser: - - #create function like 'start' which copy all YANG to location - #create function teardown + @classmethod + def setup_class(cls): + logger.info("SETUP") + os.environ['UTILITIES_UNIT_TESTING'] = "1" + move_yang_models_to_well_know_location() + + @classmethod + def teardown_class(cls): + logger.info("TEARDOWN") + os.environ['UTILITIES_UNIT_TESTING'] = "0" + remove_yang_models_to_well_know_location() def test_1_table_container(self): - yang_model_name = 'sonic-1-table-container' - template(yang_model_name, assert_dictionaries.one_table_container) + template('sonic-1-table-container', assert_dictionaries.one_table_container) def test_2_table_containers(self): - yang_model_name = 'sonic-2-table-containers' - template(yang_model_name, assert_dictionaries.two_table_containers) + template('sonic-2-table-containers', assert_dictionaries.two_table_containers) def test_1_object_container(self): - yang_model_name = 'sonic-1-object-container' - template(yang_model_name, assert_dictionaries.one_object_container) + template('sonic-1-object-container', assert_dictionaries.one_object_container) def test_2_object_containers(self): - yang_model_name = 'sonic-2-object-containers' - template(yang_model_name, assert_dictionaries.two_object_containers) + template('sonic-2-object-containers', assert_dictionaries.two_object_containers) def test_1_list(self): - yang_model_name = 'sonic-1-list' - template(yang_model_name, assert_dictionaries.one_list) + template('sonic-1-list', assert_dictionaries.one_list) def test_2_lists(self): - yang_model_name = 'sonic-2-lists' - template(yang_model_name, assert_dictionaries.two_lists) + template('sonic-2-lists', assert_dictionaries.two_lists) def test_static_object_complex_1(self): """ Test object container with: 1 leaf, 1 leaf-list, 1 choice. """ - yang_model_name = 'sonic-static-object-complex-1' - template(yang_model_name, assert_dictionaries.static_object_complex_1) + template('sonic-static-object-complex-1', assert_dictionaries.static_object_complex_1) def test_static_object_complex_2(self): """ Test object container with: 2 leafs, 2 leaf-lists, 2 choices. """ - yang_model_name = 'sonic-static-object-complex-2' - template(yang_model_name, assert_dictionaries.static_object_complex_2) + template('sonic-static-object-complex-2', assert_dictionaries.static_object_complex_2) def test_dynamic_object_complex_1(self): """ Test object container with: 1 key, 1 leaf, 1 leaf-list, 1 choice. """ - yang_model_name = 'sonic-dynamic-object-complex-1' - template(yang_model_name, assert_dictionaries.dynamic_object_complex_1) + template('sonic-dynamic-object-complex-1', assert_dictionaries.dynamic_object_complex_1) def test_dynamic_object_complex_2(self): """ Test object container with: 2 keys, 2 leafs, 2 leaf-list, 2 choice. """ - yang_model_name = 'sonic-dynamic-object-complex-2' - template(yang_model_name, assert_dictionaries.dynamic_object_complex_2) + template('sonic-dynamic-object-complex-2', assert_dictionaries.dynamic_object_complex_2) def test_choice_complex(self): """ Test object container with choice that have complex strucutre: leafs, leaf-lists, multiple 'uses' from different files """ - yang_model_name = 'sonic-choice-complex' - grouping_yang_1 = 'sonic-grouping-1' - grouping_yang_2 = 'sonic-grouping-2' - move_yang_model(grouping_yang_1) - move_yang_model(grouping_yang_2) - template(yang_model_name, assert_dictionaries.choice_complex) - remove_yang_model(grouping_yang_1) - remove_yang_model(grouping_yang_2) + template('sonic-choice-complex', assert_dictionaries.choice_complex) def test_choice_complex(self): """ Test object container with muplitple 'uses' that using 'grouping' from different files. The used 'grouping' have a complex strucutre: leafs, leaf-lists, choices """ - yang_model_name = 'sonic-grouping-complex' - grouping_yang_1 = 'sonic-grouping-1' - grouping_yang_2 = 'sonic-grouping-2' - move_yang_model(grouping_yang_1) - move_yang_model(grouping_yang_2) - template(yang_model_name, assert_dictionaries.grouping_complex) - remove_yang_model(grouping_yang_1) - remove_yang_model(grouping_yang_2) + template('sonic-grouping-complex', assert_dictionaries.grouping_complex) def template(yang_model_name, correct_dict): config_db_path = os.path.join(test_path, 'cli_autogen_input/config_db.json') - move_yang_model(yang_model_name) parser = YangParser(yang_model_name = yang_model_name, config_db_path = config_db_path, allow_tbl_without_yang = True, debug = False) yang_dict = parser.parse_yang_model() - pretty_log_debug(yang_dict) - assert yang_dict == correct_dict - remove_yang_model(yang_model_name) - -def move_yang_model(yang_model_name): - """ Move provided YANG model to known location for YangParser class - - Args: - yang_model_name: name of provided YANG model +def move_yang_models_to_well_know_location(): + """ Move a test YANG models to known location + in order to be parsed by YangParser class """ - src_path = os.path.join(test_path, 'cli_autogen_input', yang_model_name + '.yang') - cmd = 'sudo cp {} {}'.format(src_path, yang_models_path) - os.system(cmd) - -def remove_yang_model(yang_model_name): - """ Remove YANG model from well known system location - - Args: - yang_model_name: name of provided YANG model + for yang_model in test_yang_models: + src_path = os.path.join(test_path, 'cli_autogen_input', yang_model + '.yang') + cmd = 'sudo cp {} {}'.format(src_path, yang_models_path) + os.system(cmd) + +def remove_yang_models_to_well_know_location(): + """ Remove a test YANG models to known location + in order to be parsed by YangParser class """ - yang_model_path = os.path.join(yang_models_path, yang_model_name + '.yang') - cmd = 'sudo rm {}'.format(yang_model_path) - os.system(cmd) + for yang_model in test_yang_models: + yang_model_path = os.path.join(yang_models_path, yang_model + '.yang') + cmd = 'sudo rm {}'.format(yang_model_path) + os.system(cmd) -# DEBUG function def pretty_log_debug(dictionary): + """ Pretty print of parsed dictionary + """ for line in pprint.pformat(dictionary).split('\n'): logging.debug(line) - From 15aeb8c545f1dd66ee7bead402c37059965cbdac Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Wed, 19 May 2021 18:39:51 +0000 Subject: [PATCH 38/76] Added docstrings, fixed names of variables Signed-off-by: Vadym Hlushko --- sonic_cli_gen/yang_parser.py | 176 ++++++++++++++++++++++++++--------- 1 file changed, 131 insertions(+), 45 deletions(-) diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index 428aa6a23e..a0bfe3faac 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -117,8 +117,8 @@ def parse_yang_model(self) -> dict: # determine how many (1 or couple) containers a YANG model have after 'top level' container # 'table' container it is a container that goes after 'top level' container if isinstance(self.y_table_containers, list): - for tbl_cont in self.y_table_containers: - y2d_elem = on_table_container(self.y_module, tbl_cont, self.conf_mgmt) + for tbl_container in self.y_table_containers: + y2d_elem = on_table_container(self.y_module, tbl_container, self.conf_mgmt) self.yang_2_dict['tables'].append(y2d_elem) else: y2d_elem = on_table_container(self.y_module, self.y_table_containers, self.conf_mgmt) @@ -128,46 +128,48 @@ def parse_yang_model(self) -> dict: #------------------------------HANDLERS--------------------------------# -def on_table_container(y_module: OrderedDict, tbl_cont: OrderedDict, conf_mgmt) -> dict: +def on_table_container(y_module: OrderedDict, tbl_container: OrderedDict, conf_mgmt: ConfigMgmt) -> dict: """ Parse 'table' container, 'table' container goes after 'top level' container Args: y_module: reference to 'module' - tbl_cont: reference to 'table' container + tbl_container: reference to 'table' container + conf_mgmt: reference to ConfigMgmt class instance, + it have yJson object which contain all parsed YANG models Returns: - dictionary - element for self.yang_2_dict['tables'] + dictionary: element for self.yang_2_dict['tables'] """ y2d_elem = { - 'name': tbl_cont.get('@name'), - 'description': get_description(tbl_cont) + 'name': tbl_container.get('@name'), + 'description': get_description(tbl_container) } # determine if 'table container' have a 'list' entity - if tbl_cont.get('list') is None: + if tbl_container.get('list') is None: y2d_elem['static-objects'] = list() # 'object' container goes after 'table' container # 'object' container have 2 types - list (like sonic-flex_counter.yang) and NOT list (like sonic-device_metadata.yang) - obj_cont = tbl_cont.get('container') - if isinstance(obj_cont, list): - for cont in obj_cont: - static_obj_elem = on_object_container(y_module, cont, conf_mgmt, is_list=False) + obj_container = tbl_container.get('container') + if isinstance(obj_container, list): + for y_container in obj_container: + static_obj_elem = on_object_container(y_module, y_container, conf_mgmt, is_list=False) y2d_elem['static-objects'].append(static_obj_elem) else: - static_obj_elem = on_object_container(y_module, obj_cont, conf_mgmt, is_list=False) + static_obj_elem = on_object_container(y_module, obj_container, conf_mgmt, is_list=False) y2d_elem['static-objects'].append(static_obj_elem) else: y2d_elem['dynamic-objects'] = list() - tbl_cont_lists = tbl_cont.get('list') + tbl_container_lists = tbl_container.get('list') # 'container' can have more than 1 'list' entity - if isinstance(tbl_cont_lists, list): - for _list in tbl_cont_lists: + if isinstance(tbl_container_lists, list): + for _list in tbl_container_lists: dynamic_obj_elem = on_object_container(y_module, _list, conf_mgmt, is_list=True) y2d_elem['dynamic-objects'].append(dynamic_obj_elem) else: - dynamic_obj_elem = on_object_container(y_module, tbl_cont_lists, conf_mgmt, is_list=True) + dynamic_obj_elem = on_object_container(y_module, tbl_container_lists, conf_mgmt, is_list=True) y2d_elem['dynamic-objects'].append(dynamic_obj_elem) # move 'keys' elements from 'attrs' to 'keys' @@ -175,7 +177,7 @@ def on_table_container(y_module: OrderedDict, tbl_cont: OrderedDict, conf_mgmt) return y2d_elem -def on_object_container(y_module: OrderedDict, cont: OrderedDict, conf_mgmt, is_list: bool) -> dict: +def on_object_container(y_module: OrderedDict, y_container: OrderedDict, conf_mgmt, is_list: bool) -> dict: """ Parse a 'object container'. 'Object container' represent OBJECT inside Config DB schema: { @@ -188,52 +190,58 @@ def on_object_container(y_module: OrderedDict, cont: OrderedDict, conf_mgmt, is_ Args: y_module: reference to 'module' - cont: reference to 'object container' + y_container: reference to 'object container' + conf_mgmt: reference to ConfigMgmt class instance, + it have yJson object which contain all parsed YANG models + is_list: boolean flag to determine if container has 'list' Returns: - dictionary - element for y2d_elem['static-objects'] OR y2d_elem['dynamic-objects'] + dictionary: element for y2d_elem['static-objects'] OR y2d_elem['dynamic-objects'] """ - if cont is None: + if y_container is None: return {} obj_elem = { - 'name': cont.get('@name'), - 'description': get_description(cont), + 'name': y_container.get('@name'), + 'description': get_description(y_container), 'attrs': list() } if is_list: - obj_elem['keys'] = get_list_keys(cont) + obj_elem['keys'] = get_list_keys(y_container) attrs_list = list() - attrs_list.extend(get_leafs(cont, grouping_name = '')) - attrs_list.extend(get_leaf_lists(cont, grouping_name = '')) - attrs_list.extend(get_choices(y_module, cont, conf_mgmt, grouping_name = '')) - # TODO: need to test 'grouping' - attrs_list.extend(get_uses(y_module, cont, conf_mgmt)) + # grouping_name is empty because 'grouping' is not used so far + attrs_list.extend(get_leafs(y_container, grouping_name = '')) + attrs_list.extend(get_leaf_lists(y_container, grouping_name = '')) + attrs_list.extend(get_choices(y_module, y_container, conf_mgmt, grouping_name = '')) + attrs_list.extend(get_uses(y_module, y_container, conf_mgmt)) obj_elem['attrs'] = attrs_list return obj_elem -def on_uses(y_module: OrderedDict, y_uses, conf_mgmt) -> list: +def on_uses(y_module: OrderedDict, y_uses, conf_mgmt: ConfigMgmt) -> list: """ Parse a YANG 'uses' entities 'uses' refearing to 'grouping' YANG entity Args: y_module: reference to 'module' y_uses: reference to 'uses' + conf_mgmt: reference to ConfigMgmt class instance, + it have yJson object which contain all parsed YANG model Returns: - dictionary - element for obj_elem['attrs'], 'attrs' contain a parsed 'leafs' + list: element for obj_elem['attrs'], 'attrs' contain a parsed 'leafs' """ + ret_attrs = list() y_grouping = get_all_grouping(y_module, y_uses, conf_mgmt) # trim prefixes in order to the next checks trim_uses_prefixes(y_uses) + # not sure if it can happend if y_grouping == []: - # not sure if it can happend - raise Exception('EMPTY') + raise Exception('Grouping NOT found') # TODO: 'refine' support for group in y_grouping: @@ -251,14 +259,17 @@ def on_uses(y_module: OrderedDict, y_uses, conf_mgmt) -> list: return ret_attrs - def on_choices(y_module: OrderedDict, y_choices, conf_mgmt: ConfigMgmt, grouping_name: str) -> list: """ Parse a YANG 'choice' entities Args: - cont: reference to 'choice' + y_module: reference to 'module' + y_choices: reference to 'choice' element + conf_mgmt: reference to ConfigMgmt class instance, + it have yJson object which contain all parsed YANG model + grouping_name: if YANG entity contain 'uses', this arg represent 'grouping' name Returns: - dictionary - element for obj_elem['attrs'], 'attrs' contain a parsed 'leafs' + list: element for obj_elem['attrs'], 'attrs' contain a parsed 'leafs' """ ret_attrs = list() @@ -280,8 +291,11 @@ def on_choice_cases(y_module: OrderedDict, y_cases: list, conf_mgmt: ConfigMgmt, Args: y_module: reference to 'module' y_cases: reference to 'case' + conf_mgmt: reference to ConfigMgmt class instance, + it have yJson object which contain all parsed YANG model + grouping_name: if YANG entity contain 'uses', this arg represent 'grouping' name Returns: - dictionary - element for obj_elem['attrs'], 'attrs' contain a parsed 'leafs' + list: element for obj_elem['attrs'], 'attrs' contain a parsed 'leafs' """ ret_attrs = list() @@ -302,8 +316,10 @@ def on_leafs(y_leafs, grouping_name, is_leaf_list: bool) -> list: Args: y_leafs: reference to all 'leaf' elements + grouping_name: if YANG entity contain 'uses', this arg represent 'grouping' name + is_leaf_list: boolean to determine if 'leaf-list' was passed in 'y_leafs' arg Returns: - list - list of parsed 'leaf' elements + list: list of parsed 'leaf' elements """ ret_attrs = list() @@ -323,8 +339,10 @@ def on_leaf(leaf: OrderedDict, is_leaf_list: bool, grouping_name: str) -> dict: Args: leaf: reference to a 'leaf' entity + grouping_name: if YANG entity contain 'uses', this arg represent 'grouping' name + is_leaf_list: boolean to determine if 'leaf-list' was passed in 'y_leafs' arg Returns: - dictionary - parsed 'leaf' element + dictionary: parsed 'leaf' element """ attr = { 'name': leaf.get('@name'), @@ -338,6 +356,14 @@ def on_leaf(leaf: OrderedDict, is_leaf_list: bool, grouping_name: str) -> dict: #----------------------GETERS-------------------------# def get_mandatory(y_leaf: OrderedDict) -> bool: + """ Parse 'mandatory' statement for 'leaf' + + Args: + y_leaf: reference to a 'leaf' entity + Returns: + bool: 'leaf' 'mandatory' value + """ + if y_leaf.get('mandatory') is not None: return True @@ -357,34 +383,83 @@ def get_description(y_entity: OrderedDict) -> str: else: return '' -def get_leafs(y_entity: OrderedDict, grouping_name) -> list: +def get_leafs(y_entity: OrderedDict, grouping_name: str) -> list: + """ Check if YANG entity have 'leafs', if so call handler + + Args: + y_entity: reference YANG 'container' or 'list' or 'choice' or 'uses' + grouping_name: if YANG entity contain 'uses', this arg represent 'grouping' name + Returns: + list: list of parsed 'leaf' elements + """ + if y_entity.get('leaf') is not None: return on_leafs(y_entity.get('leaf'), grouping_name, is_leaf_list=False) return [] -def get_leaf_lists(y_entity: OrderedDict, grouping_name) -> list: +def get_leaf_lists(y_entity: OrderedDict, grouping_name: str) -> list: + """ Check if YANG entity have 'leaf-list', if so call handler + + Args: + y_entity: reference YANG 'container' or 'list' or 'choice' or 'uses' + grouping_name: if YANG entity contain 'uses', this arg represent 'grouping' name + Returns: + list: list of parsed 'leaf-list' elements + """ + if y_entity.get('leaf-list') is not None: return on_leafs(y_entity.get('leaf-list'), grouping_name, is_leaf_list=True) return [] def get_choices(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: ConfigMgmt, grouping_name: str) -> list: + """ Check if YANG entity have 'choice', if so call handler + + Args: + y_module: reference to 'module' + y_entity: reference YANG 'container' or 'list' or 'choice' or 'uses' + conf_mgmt: reference to ConfigMgmt class instance, + it have yJson object which contain all parsed YANG model + grouping_name: if YANG entity contain 'uses', this arg represent 'grouping' name + Returns: + list: list of parsed elements inside 'choice' + """ + if y_entity.get('choice') is not None: return on_choices(y_module, y_entity.get('choice'), conf_mgmt, grouping_name) return [] -def get_uses(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt) -> list: +def get_uses(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: ConfigMgmt) -> list: + """ Check if YANG entity have 'uses', if so call handler + + Args: + y_module: reference to 'module' + y_entity: reference YANG 'container' or 'list' or 'choice' or 'uses' + conf_mgmt: reference to ConfigMgmt class instance, + it have yJson object which contain all parsed YANG model + Returns: + list: list of parsed elements inside 'grouping' that referenced by 'uses' + """ + if y_entity.get('uses') is not None: return on_uses(y_module, y_entity.get('uses'), conf_mgmt) return [] -def get_all_grouping(y_module: OrderedDict, y_uses: OrderedDict, conf_mgmt) -> list: - """ Get all 'grouping' entities that is 'uses' in current YANG +def get_all_grouping(y_module: OrderedDict, y_uses: OrderedDict, conf_mgmt: ConfigMgmt) -> list: + """ Get all 'grouping' entities that is referenced by 'uses' in current YANG model + + Args: + y_module: reference to 'module' + y_entity: reference to 'uses' + conf_mgmt: reference to ConfigMgmt class instance, + it have yJson object which contain all parsed YANG model + Returns: + list: list of 'grouping' elements """ - # WARNING + # TODO add to the design statement that grouping should be defined under the 'module' and NOT in nested containers ret_grouping = list() prefix_list = get_import_prefixes(y_uses) @@ -475,6 +550,7 @@ def trim_uses_prefixes(y_uses) -> list: Args: y_uses - reference to 'uses' """ + prefixes = get_import_prefixes(y_uses) for prefix in prefixes: @@ -487,6 +563,15 @@ def trim_uses_prefixes(y_uses) -> list: y_uses['@name'] = y_uses.get('@name').split(':')[1] def get_list_keys(y_list: OrderedDict) -> list: + """ Parse YANG 'keys' + If YANG have 'list', inside the list exist 'keys' + + Args: + y_list: reference to 'list' + Returns: + list: parsed keys + """ + ret_list = list() keys = y_list.get('key').get('@value').split() for k in keys: @@ -505,6 +590,7 @@ def change_dyn_obj_struct(dynamic_objects: OrderedDict): Args: dynamic_objects: reference to self.yang_2_dict['dynamic_objects'] """ + for obj in dynamic_objects: for key in obj.get('keys'): for attr in obj.get('attrs'): From ea2c838266170abe0ef4a41a07fc6f04400dff09 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Wed, 19 May 2021 18:45:47 +0000 Subject: [PATCH 39/76] Added test_grouping_complex to the execution pipeline Signed-off-by: Vadym Hlushko --- tests/cli_autogen_yang_parser_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cli_autogen_yang_parser_test.py b/tests/cli_autogen_yang_parser_test.py index c6a81bef50..67898503af 100644 --- a/tests/cli_autogen_yang_parser_test.py +++ b/tests/cli_autogen_yang_parser_test.py @@ -83,7 +83,7 @@ def test_choice_complex(self): """ template('sonic-choice-complex', assert_dictionaries.choice_complex) - def test_choice_complex(self): + def test_grouping_complex(self): """ Test object container with muplitple 'uses' that using 'grouping' from different files. The used 'grouping' have a complex strucutre: leafs, leaf-lists, choices From e18c888ea5e6ed574b6e73a4c99c195184fe0423 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Thu, 20 May 2021 14:28:33 +0000 Subject: [PATCH 40/76] Removed unused test YANG model Signed-off-by: Vadym Hlushko --- .../cli_autogen_input/sonic-test-complex.yang | 115 ------------------ 1 file changed, 115 deletions(-) delete mode 100644 tests/cli_autogen_input/sonic-test-complex.yang diff --git a/tests/cli_autogen_input/sonic-test-complex.yang b/tests/cli_autogen_input/sonic-test-complex.yang deleted file mode 100644 index e4485c9697..0000000000 --- a/tests/cli_autogen_input/sonic-test-complex.yang +++ /dev/null @@ -1,115 +0,0 @@ -module sonic-test-1 { - - yang-version 1.1; - - namespace "http://github.com/Azure/sonic-test-1"; - prefix many-test-1; - - container sonic-test-1 { - /* sonic-test-1 - top level container, it have: - * 2 table containers. Table container - represent Config DB table name. - */ - - container TABLE_1 { - /* TABLE_1 - table container, it have: - * 2 object containers, Object container - represent Confg DB object name. - */ - description "TABLE_1 description"; - - container OBJECT_1 { - /* OBJECT_1 - object container, it have: - * 1 leaf, - * 1 leaf-list - * 1 choice - */ - description "OBJECT_1 description"; - - leaf OBJ_1_LEAF_1 { - description "OBJ_1_LEAF_1 description"; - type string; - } - - leaf-list OBJ_1_LEAF_LIST_1 { - mandatory true; - type string; - } - - choice OBJ_1_CH_1 { - case OBJ_1_CH_1_CASE_1 { - leaf OBJ_1_CH_1_LEAF_1 { - type uint16; - } - } - case OBJ_1_CH_1_CASE_2 { - leaf OBJ_1_CH_1_LEAF_2 { - type string; - } - } - } - } - - container OBJECT_2 { - /* OBJECT_2 - table container, it have: - * 2 leaf, - * 2 leaf-list - * 2 choice - */ - description "OBJECT_2 description"; - - leaf OBJ_2_LEAF_1 { - description "OBJ_2_LEAF_1 description"; - type string; - } - - leaf OBJ_2_LEAF_2 { - mandatory true; - type string; - } - - leaf-list OBJ_2_LEAF_LIST_1 { - description "OBJ_2_LEAF_LIST_1 description"; - mandatory true; - type string; - } - - leaf-list OBJ_2_LEAF_LIST_2 { - type string; - } - - choice OBJ_2_CH_1 { - case OBJ_2_CH_1_CASE_1 { - leaf OBJ_2_CH_1_LEAF_1 { - type uint16; - } - } - case OBJ_2_CH_1_CASE_2 { - leaf OBJ_2_CH_1_LEAF_2 { - type string { - type string; - } - } - } - } - - choice OBJ_2_CH_2 { - case OBJ_2_CH_2_CASE_1 { - leaf OBJ_2_CH_2_LEAF_1 { - type uint16; - } - } - case OBJ_2_CH_2_CASE_2 { - leaf OBJ_2_CH_2_LEAF_2 { - type string { - type string; - } - } - } - } - } - } - - container TABLE_2 { - - } - } -} \ No newline at end of file From 7bd3ec90627378659d2c5516a935037a20d702ee Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Fri, 21 May 2021 13:59:07 +0000 Subject: [PATCH 41/76] Fixed indentation Signed-off-by: Vadym Hlushko --- setup.py | 4 ++-- sonic_cli_gen/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 9e3f30e3ad..0240125036 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ 'undebug', 'utilities_common', 'watchdogutil', - 'sonic_cli_gen', + 'sonic_cli_gen', ], package_data={ 'show': ['aliases.ini'], @@ -161,7 +161,7 @@ 'spm = sonic_package_manager.main:cli', 'undebug = undebug.main:cli', 'watchdogutil = watchdogutil.main:watchdogutil', - 'sonic-cli-gen = sonic_cli_gen.main:cli', + 'sonic-cli-gen = sonic_cli_gen.main:cli', ] }, install_requires=[ diff --git a/sonic_cli_gen/__init__.py b/sonic_cli_gen/__init__.py index 7e49cacd56..57da8bc819 100644 --- a/sonic_cli_gen/__init__.py +++ b/sonic_cli_gen/__init__.py @@ -2,4 +2,4 @@ from sonic_cli_gen.generator import CliGenerator -__all__ = ['CliGenerator'] \ No newline at end of file +__all__ = ['CliGenerator'] From d5dea7f8b1b76a74c2329e99eacfe3be2a1b77ce Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Fri, 21 May 2021 15:32:43 +0000 Subject: [PATCH 42/76] Added sonic-cli-gen remove cmd, reworked private initializer functions Signed-off-by: Vadym Hlushko --- sonic_cli_gen/generator.py | 17 +++++++++++---- sonic_cli_gen/main.py | 41 ++++++++++++++++++++++++------------ sonic_cli_gen/yang_parser.py | 41 ++++++++++++++++++------------------ 3 files changed, 62 insertions(+), 37 deletions(-) diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py index d4a9c81ac0..85191c6ce3 100644 --- a/sonic_cli_gen/generator.py +++ b/sonic_cli_gen/generator.py @@ -1,15 +1,15 @@ #!/usr/bin/env python -import jinja2 import os import pkgutil +import jinja2 from sonic_cli_gen.yang_parser import YangParser class CliGenerator: """ SONiC CLI generator. This class provides public API for sonic-cli-gen python library. It can generate config, - show, sonic-clear CLI plugins + show CLI plugins """ def __init__(self): @@ -18,28 +18,37 @@ def __init__(self): self.loader = jinja2.FileSystemLoader(['/usr/share/sonic/templates/sonic-cli-gen/']) self.env = jinja2.Environment(loader=self.loader) + def generate_cli_plugin(self, cli_group, plugin_name): - """ Generate CLI plugin. """ + """ Generate click CLI plugin. """ parser = YangParser(yang_model_name=plugin_name, config_db_path='configDB', allow_tbl_without_yang=True, debug=False) + # yang_dict will be used as an input for templates located in - /usr/share/sonic/templates/sonic-cli-gen/ yang_dict = parser.parse_yang_model() plugin_path = get_cli_plugin_path(cli_group, plugin_name + '_yang.py') template = self.env.get_template(cli_group + '.py.j2') with open(plugin_path, 'w') as plugin_py: plugin_py.write(template.render(yang_dict)) + print('\nAuto-generation successful!\nLocation: {}'.format(plugin_path)) + def remove_cli_plugin(self, cli_group, plugin_name): plugin_path = get_cli_plugin_path(cli_group, plugin_name + '_yang.py') if os.path.exists(plugin_path): os.remove(plugin_path) + print('{} was removed.'.format(plugin_path)) + else: + print('Path {} doest NOT exist!'.format(plugin_path)) + def get_cli_plugin_path(command, plugin_name): pkg_loader = pkgutil.get_loader(f'{command}.plugins.auto') if pkg_loader is None: - raise PackageManagerError(f'Failed to get plugins path for {command} CLI') + raise Exception(f'Failed to get plugins path for {command} CLI') plugins_pkg_path = os.path.dirname(pkg_loader.path) return os.path.join(plugins_pkg_path, plugin_name) + diff --git a/sonic_cli_gen/main.py b/sonic_cli_gen/main.py index 0ee0b030f5..602ff7305d 100644 --- a/sonic_cli_gen/main.py +++ b/sonic_cli_gen/main.py @@ -6,24 +6,39 @@ @click.group() @click.pass_context def cli(ctx): - """ SONiC CLI generator """ - pass + """ SONiC CLI Auto-generator tool.\r + Generate click CLI plugin for 'config' or 'show' CLI groups.\r + CLI plugin will be generated from the YANG model, which should be in:\r\n + /usr/local/yang-models/ \n + Generated CLI plugin will be placed in: \r\n + /usr/local/lib/python3.7/dist-packages//plugins/auto/ + """ + + context = { + 'gen': CliGenerator() + } + ctx.obj = context + @cli.command() -@click.argument('yang_model_name') +@click.argument('cli_group', type = click.Choice(['config', 'show'])) +@click.argument('yang_model_name', type = click.STRING) @click.pass_context -def generate_config(ctx, yang_model_name): - """ Generate CLI plugin (click) for 'config' CLI group. """ - gen = CliGenerator() - gen.generate_cli_plugin(cli_group='config', plugin_name=yang_model_name) +def generate(ctx, cli_group, yang_model_name): + """ Generate click CLI plugin. """ + + ctx.obj['gen'].generate_cli_plugin(cli_group = cli_group, plugin_name = yang_model_name) + @cli.command() -@click.argument('yang_model_name') +@click.argument('cli_group', type = click.Choice(['config', 'show'])) +@click.argument('yang_model_name', type = click.STRING) @click.pass_context -def generate_show(ctx, yang_model_name): - """ Generate CLI plugin (click) for 'show' CLI group. """ - gen = CliGenerator() - gen.generate_cli_plugin(cli_group='show', plugin_name=yang_model_name) +def remove(ctx, cli_group, yang_model_name): + """ Remove generated click CLI plugin from. """ + + ctx.obj['gen'].remove_cli_plugin(cli_group = cli_group, plugin_name = yang_model_name) + if __name__ == '__main__': - cli() \ No newline at end of file + cli() diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index a0bfe3faac..f8e030f465 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -3,13 +3,14 @@ from collections import OrderedDict from config.config_mgmt import ConfigMgmt +yang_guidelines_link = 'https://github.com/Azure/SONiC/blob/master/doc/mgmt/SONiC_YANG_Model_Guidelines.md' + class YangParser: """ YANG model parser Attributes: yang_model_name: Name of the YANG model file conf_mgmt: Instance of Config Mgmt class to help parse YANG models - idx_yJson: Index of yang_model_file (1 attr) inside conf_mgmt.sy.yJson object y_module: Reference to 'module' entity from YANG model file y_top_level_container: Reference to top level 'container' entity from YANG model file y_table_containers: Reference to 'container' entities from YANG model file @@ -52,7 +53,6 @@ def __init__(self, debug): self.yang_model_name = yang_model_name self.conf_mgmt = None - self.idx_yJson = None self.y_module = None self.y_top_level_container = None self.y_table_containers = None @@ -72,36 +72,36 @@ def _init_yang_module_and_containers(self): self.y_table_containers Raises: - KeyError: if YANG model is invalid or NOT exist + Exception: if YANG model is invalid or NOT exist """ - self._find_index_of_yang_model() + self.y_module = self._find_yang_model_in_yjson_obj() - if self.idx_yJson is None: - raise KeyError('YANG model {} is NOT exist'.format(self.yang_model_name)) - self.y_module = self.conf_mgmt.sy.yJson[self.idx_yJson]['module'] + if self.y_module is None: + raise Exception('The YANG model {} is NOT exist'.format(self.yang_model_name)) if self.y_module.get('container') is None: - raise KeyError('YANG model {} does NOT have "top level container" element \ - Please follow the SONiC YANG model guidelines: \ - https://github.com/Azure/SONiC/blob/master/doc/mgmt/SONiC_YANG_Model_Guidelines.md'.format(self.yang_model_name)) + raise Exception('The YANG model {} does NOT have "top level container" element\ + Please follow the SONiC YANG model guidelines:\n{}'.format(self.yang_model_name, yang_guidelines_link)) self.y_top_level_container = self.y_module.get('container') if self.y_top_level_container.get('container') is None: - raise KeyError('YANG model {} does NOT have "container" element after "top level container" \ - Please follow the SONiC YANG model guidelines: \ - https://github.com/Azure/SONiC/blob/master/doc/mgmt/SONiC_YANG_Model_Guidelines.md'.format(self.yang_model_name)) + raise Exception('The YANG model {} does NOT have "container" element after "top level container"\ + Please follow the SONiC YANG model guidelines:\n{}'.format(self.yang_model_name, yang_guidelines_link)) self.y_table_containers = self.y_top_level_container.get('container') - def _find_index_of_yang_model(self): - """ Find index of provided YANG model inside yJson object, - and save it to self.idx_yJson variable + def _find_yang_model_in_yjson_obj(self) -> OrderedDict: + """ Find provided YANG model inside yJson object, yJson object contain all yang-models parsed from directory - /usr/local/yang-models + + Returns: + reference to yang_model_name """ - for i in range(len(self.conf_mgmt.sy.yJson)): - if (self.conf_mgmt.sy.yJson[i]['module']['@name'] == self.yang_model_name): - self.idx_yJson = i + # TODO: consider to check yJson type + for yang_model in self.conf_mgmt.sy.yJson: + if yang_model.get('module').get('@name') == self.yang_model_name: + return yang_model.get('module') def parse_yang_model(self) -> dict: """ Parse proviced YANG model @@ -597,4 +597,5 @@ def change_dyn_obj_struct(dynamic_objects: OrderedDict): if key.get('name') == attr.get('name'): key['description'] = attr.get('description') obj['attrs'].remove(attr) - break \ No newline at end of file + break + From aa4af1cf05615e794792ee15559d61051d3c9a8f Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Mon, 24 May 2021 07:54:34 +0000 Subject: [PATCH 43/76] Added new lines, fixed some docstrings Signed-off-by: Vadym Hlushko --- sonic_cli_gen/__init__.py | 1 + sonic_cli_gen/generator.py | 9 +++++++-- sonic_cli_gen/main.py | 1 + sonic_cli_gen/yang_parser.py | 26 +++++++++++++++++++++++--- 4 files changed, 32 insertions(+), 5 deletions(-) diff --git a/sonic_cli_gen/__init__.py b/sonic_cli_gen/__init__.py index 57da8bc819..e7e775c0fb 100644 --- a/sonic_cli_gen/__init__.py +++ b/sonic_cli_gen/__init__.py @@ -3,3 +3,4 @@ from sonic_cli_gen.generator import CliGenerator __all__ = ['CliGenerator'] + diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py index 85191c6ce3..48142a9e7b 100644 --- a/sonic_cli_gen/generator.py +++ b/sonic_cli_gen/generator.py @@ -13,14 +13,16 @@ class CliGenerator: """ def __init__(self): - """ Initialize PackageManager. """ + """ Initialize CliGenerator. """ self.loader = jinja2.FileSystemLoader(['/usr/share/sonic/templates/sonic-cli-gen/']) self.env = jinja2.Environment(loader=self.loader) def generate_cli_plugin(self, cli_group, plugin_name): - """ Generate click CLI plugin. """ + """ Generate click CLI plugin and put it to: + /usr/local/lib/python3.7/dist-packages//plugins/auto/ + """ parser = YangParser(yang_model_name=plugin_name, config_db_path='configDB', @@ -36,6 +38,9 @@ def generate_cli_plugin(self, cli_group, plugin_name): def remove_cli_plugin(self, cli_group, plugin_name): + """ Remove CLI plugin from directory: + /usr/local/lib/python3.7/dist-packages//plugins/auto/ + """ plugin_path = get_cli_plugin_path(cli_group, plugin_name + '_yang.py') if os.path.exists(plugin_path): os.remove(plugin_path) diff --git a/sonic_cli_gen/main.py b/sonic_cli_gen/main.py index 602ff7305d..f23249d354 100644 --- a/sonic_cli_gen/main.py +++ b/sonic_cli_gen/main.py @@ -42,3 +42,4 @@ def remove(ctx, cli_group, yang_model_name): if __name__ == '__main__': cli() + diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index f8e030f465..952a2e60a2 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -29,7 +29,8 @@ class YangParser: 'name': 'value', 'description': 'value', 'is-leaf-list': False, - 'is-mandatory': False + 'is-mandatory': False, + 'group': 'value' } ... ], @@ -126,6 +127,7 @@ def parse_yang_model(self) -> dict: return self.yang_2_dict + #------------------------------HANDLERS--------------------------------# def on_table_container(y_module: OrderedDict, tbl_container: OrderedDict, conf_mgmt: ConfigMgmt) -> dict: @@ -177,6 +179,7 @@ def on_table_container(y_module: OrderedDict, tbl_container: OrderedDict, conf_m return y2d_elem + def on_object_container(y_module: OrderedDict, y_container: OrderedDict, conf_mgmt, is_list: bool) -> dict: """ Parse a 'object container'. 'Object container' represent OBJECT inside Config DB schema: @@ -221,6 +224,7 @@ def on_object_container(y_module: OrderedDict, y_container: OrderedDict, conf_mg return obj_elem + def on_uses(y_module: OrderedDict, y_uses, conf_mgmt: ConfigMgmt) -> list: """ Parse a YANG 'uses' entities 'uses' refearing to 'grouping' YANG entity @@ -259,6 +263,7 @@ def on_uses(y_module: OrderedDict, y_uses, conf_mgmt: ConfigMgmt) -> list: return ret_attrs + def on_choices(y_module: OrderedDict, y_choices, conf_mgmt: ConfigMgmt, grouping_name: str) -> list: """ Parse a YANG 'choice' entities @@ -284,6 +289,7 @@ def on_choices(y_module: OrderedDict, y_choices, conf_mgmt: ConfigMgmt, grouping return ret_attrs + def on_choice_cases(y_module: OrderedDict, y_cases: list, conf_mgmt: ConfigMgmt, grouping_name: str) -> list: """ Parse a single YANG 'case' entity from 'choice' entity 'case' element can have inside - 'leaf', 'leaf-list', 'uses' @@ -304,13 +310,13 @@ def on_choice_cases(y_module: OrderedDict, y_cases: list, conf_mgmt: ConfigMgmt, for case in y_cases: ret_attrs.extend(get_leafs(case, grouping_name)) ret_attrs.extend(get_leaf_lists(case, grouping_name)) - # TODO: need to deeply test it ret_attrs.extend(get_uses(y_module, case, conf_mgmt)) else: raise Exception('It has no sense to using a single "case" element inside "choice" element') return ret_attrs + def on_leafs(y_leafs, grouping_name, is_leaf_list: bool) -> list: """ Parse all the 'leaf' or 'leaf-list' elements @@ -334,6 +340,7 @@ def on_leafs(y_leafs, grouping_name, is_leaf_list: bool) -> list: return ret_attrs + def on_leaf(leaf: OrderedDict, is_leaf_list: bool, grouping_name: str) -> dict: """ Parse a single 'leaf' element @@ -353,6 +360,7 @@ def on_leaf(leaf: OrderedDict, is_leaf_list: bool, grouping_name: str) -> dict: return attr + #----------------------GETERS-------------------------# def get_mandatory(y_leaf: OrderedDict) -> bool: @@ -369,6 +377,7 @@ def get_mandatory(y_leaf: OrderedDict) -> bool: return False + def get_description(y_entity: OrderedDict) -> str: """ Parse 'description' entity from any YANG element @@ -383,6 +392,7 @@ def get_description(y_entity: OrderedDict) -> str: else: return '' + def get_leafs(y_entity: OrderedDict, grouping_name: str) -> list: """ Check if YANG entity have 'leafs', if so call handler @@ -398,6 +408,7 @@ def get_leafs(y_entity: OrderedDict, grouping_name: str) -> list: return [] + def get_leaf_lists(y_entity: OrderedDict, grouping_name: str) -> list: """ Check if YANG entity have 'leaf-list', if so call handler @@ -413,6 +424,7 @@ def get_leaf_lists(y_entity: OrderedDict, grouping_name: str) -> list: return [] + def get_choices(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: ConfigMgmt, grouping_name: str) -> list: """ Check if YANG entity have 'choice', if so call handler @@ -431,6 +443,7 @@ def get_choices(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: ConfigM return [] + def get_uses(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: ConfigMgmt) -> list: """ Check if YANG entity have 'uses', if so call handler @@ -448,6 +461,7 @@ def get_uses(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: ConfigMgmt return [] + def get_all_grouping(y_module: OrderedDict, y_uses: OrderedDict, conf_mgmt: ConfigMgmt) -> list: """ Get all 'grouping' entities that is referenced by 'uses' in current YANG model @@ -460,7 +474,6 @@ def get_all_grouping(y_module: OrderedDict, y_uses: OrderedDict, conf_mgmt: Conf list: list of 'grouping' elements """ - # TODO add to the design statement that grouping should be defined under the 'module' and NOT in nested containers ret_grouping = list() prefix_list = get_import_prefixes(y_uses) @@ -487,6 +500,7 @@ def get_all_grouping(y_module: OrderedDict, y_uses: OrderedDict, conf_mgmt: Conf return ret_grouping + def get_grouping_from_another_yang_model(yang_model_name: str, conf_mgmt) -> list: """ Get the YANG 'grouping' entity @@ -498,6 +512,7 @@ def get_grouping_from_another_yang_model(yang_model_name: str, conf_mgmt) -> lis Returns: list - list 'grouping' entities """ + ret_grouping = list() for i in range(len(conf_mgmt.sy.yJson)): @@ -511,6 +526,7 @@ def get_grouping_from_another_yang_model(yang_model_name: str, conf_mgmt) -> lis return ret_grouping + def get_import_prefixes(y_uses: OrderedDict) -> list: """ Parse 'import prefix' of YANG 'uses' entity Example: @@ -525,6 +541,7 @@ def get_import_prefixes(y_uses: OrderedDict) -> list: Returns: list - of parsed prefixes """ + ret_prefixes = list() if isinstance(y_uses, list): @@ -539,6 +556,7 @@ def get_import_prefixes(y_uses: OrderedDict) -> list: return ret_prefixes + def trim_uses_prefixes(y_uses) -> list: """ Trim prefixes from 'uses' YANG entities. If YANG 'grouping' was imported from another YANG file, it use 'prefix' before 'grouping' name: @@ -562,6 +580,7 @@ def trim_uses_prefixes(y_uses) -> list: if prefix in y_uses.get('@name'): y_uses['@name'] = y_uses.get('@name').split(':')[1] + def get_list_keys(y_list: OrderedDict) -> list: """ Parse YANG 'keys' If YANG have 'list', inside the list exist 'keys' @@ -580,6 +599,7 @@ def get_list_keys(y_list: OrderedDict) -> list: return ret_list + def change_dyn_obj_struct(dynamic_objects: OrderedDict): """ Rearrange self.yang_2_dict['dynamic_objects'] structure. If YANG model have a 'list' entity - inside the 'list' it has 'key' entity. From 718a431831589f50ce6ee89f5d3c083fe01956b4 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Mon, 24 May 2021 08:00:00 +0000 Subject: [PATCH 44/76] removed type from docstring Signed-off-by: Vadym Hlushko --- sonic_cli_gen/yang_parser.py | 42 ++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index 952a2e60a2..a48b7b707a 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -105,17 +105,17 @@ def _find_yang_model_in_yjson_obj(self) -> OrderedDict: return yang_model.get('module') def parse_yang_model(self) -> dict: - """ Parse proviced YANG model + """ Parse provided YANG model and save output to self.yang_2_dict object Returns: - dictionary - parsed YANG model in dictionary format + parsed YANG model in dictionary format """ self._init_yang_module_and_containers() self.yang_2_dict['tables'] = list() - # determine how many (1 or couple) containers a YANG model have after 'top level' container + # determine how many (1 or more) containers a YANG model have after 'top level' container # 'table' container it is a container that goes after 'top level' container if isinstance(self.y_table_containers, list): for tbl_container in self.y_table_containers: @@ -140,7 +140,7 @@ def on_table_container(y_module: OrderedDict, tbl_container: OrderedDict, conf_m conf_mgmt: reference to ConfigMgmt class instance, it have yJson object which contain all parsed YANG models Returns: - dictionary: element for self.yang_2_dict['tables'] + element for self.yang_2_dict['tables'] """ y2d_elem = { @@ -198,7 +198,7 @@ def on_object_container(y_module: OrderedDict, y_container: OrderedDict, conf_mg it have yJson object which contain all parsed YANG models is_list: boolean flag to determine if container has 'list' Returns: - dictionary: element for y2d_elem['static-objects'] OR y2d_elem['dynamic-objects'] + element for y2d_elem['static-objects'] OR y2d_elem['dynamic-objects'] """ if y_container is None: @@ -235,7 +235,7 @@ def on_uses(y_module: OrderedDict, y_uses, conf_mgmt: ConfigMgmt) -> list: conf_mgmt: reference to ConfigMgmt class instance, it have yJson object which contain all parsed YANG model Returns: - list: element for obj_elem['attrs'], 'attrs' contain a parsed 'leafs' + element for obj_elem['attrs'], 'attrs' contain a parsed 'leafs' """ ret_attrs = list() @@ -274,7 +274,7 @@ def on_choices(y_module: OrderedDict, y_choices, conf_mgmt: ConfigMgmt, grouping it have yJson object which contain all parsed YANG model grouping_name: if YANG entity contain 'uses', this arg represent 'grouping' name Returns: - list: element for obj_elem['attrs'], 'attrs' contain a parsed 'leafs' + element for obj_elem['attrs'], 'attrs' contain a parsed 'leafs' """ ret_attrs = list() @@ -301,7 +301,7 @@ def on_choice_cases(y_module: OrderedDict, y_cases: list, conf_mgmt: ConfigMgmt, it have yJson object which contain all parsed YANG model grouping_name: if YANG entity contain 'uses', this arg represent 'grouping' name Returns: - list: element for obj_elem['attrs'], 'attrs' contain a parsed 'leafs' + element for obj_elem['attrs'], 'attrs' contain a parsed 'leafs' """ ret_attrs = list() @@ -325,7 +325,7 @@ def on_leafs(y_leafs, grouping_name, is_leaf_list: bool) -> list: grouping_name: if YANG entity contain 'uses', this arg represent 'grouping' name is_leaf_list: boolean to determine if 'leaf-list' was passed in 'y_leafs' arg Returns: - list: list of parsed 'leaf' elements + list of parsed 'leaf' elements """ ret_attrs = list() @@ -349,7 +349,7 @@ def on_leaf(leaf: OrderedDict, is_leaf_list: bool, grouping_name: str) -> dict: grouping_name: if YANG entity contain 'uses', this arg represent 'grouping' name is_leaf_list: boolean to determine if 'leaf-list' was passed in 'y_leafs' arg Returns: - dictionary: parsed 'leaf' element + parsed 'leaf' element """ attr = { 'name': leaf.get('@name'), @@ -369,7 +369,7 @@ def get_mandatory(y_leaf: OrderedDict) -> bool: Args: y_leaf: reference to a 'leaf' entity Returns: - bool: 'leaf' 'mandatory' value + 'leaf' 'mandatory' value """ if y_leaf.get('mandatory') is not None: @@ -384,7 +384,7 @@ def get_description(y_entity: OrderedDict) -> str: Args: y_entity: reference to YANG 'container' OR 'list' OR 'leaf' ... Returns: - str - text of the 'description' + text of the 'description' """ if y_entity.get('description') is not None: @@ -400,7 +400,7 @@ def get_leafs(y_entity: OrderedDict, grouping_name: str) -> list: y_entity: reference YANG 'container' or 'list' or 'choice' or 'uses' grouping_name: if YANG entity contain 'uses', this arg represent 'grouping' name Returns: - list: list of parsed 'leaf' elements + list of parsed 'leaf' elements """ if y_entity.get('leaf') is not None: @@ -416,7 +416,7 @@ def get_leaf_lists(y_entity: OrderedDict, grouping_name: str) -> list: y_entity: reference YANG 'container' or 'list' or 'choice' or 'uses' grouping_name: if YANG entity contain 'uses', this arg represent 'grouping' name Returns: - list: list of parsed 'leaf-list' elements + list of parsed 'leaf-list' elements """ if y_entity.get('leaf-list') is not None: @@ -435,7 +435,7 @@ def get_choices(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: ConfigM it have yJson object which contain all parsed YANG model grouping_name: if YANG entity contain 'uses', this arg represent 'grouping' name Returns: - list: list of parsed elements inside 'choice' + list of parsed elements inside 'choice' """ if y_entity.get('choice') is not None: @@ -453,7 +453,7 @@ def get_uses(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: ConfigMgmt conf_mgmt: reference to ConfigMgmt class instance, it have yJson object which contain all parsed YANG model Returns: - list: list of parsed elements inside 'grouping' that referenced by 'uses' + list of parsed elements inside 'grouping' that referenced by 'uses' """ if y_entity.get('uses') is not None: @@ -471,7 +471,7 @@ def get_all_grouping(y_module: OrderedDict, y_uses: OrderedDict, conf_mgmt: Conf conf_mgmt: reference to ConfigMgmt class instance, it have yJson object which contain all parsed YANG model Returns: - list: list of 'grouping' elements + list of 'grouping' elements """ ret_grouping = list() @@ -510,7 +510,7 @@ def get_grouping_from_another_yang_model(yang_model_name: str, conf_mgmt) -> lis it have yJson object which contain all parsed YANG models Returns: - list - list 'grouping' entities + list 'grouping' entities """ ret_grouping = list() @@ -539,7 +539,7 @@ def get_import_prefixes(y_uses: OrderedDict) -> list: Args: y_uses: refrence to YANG 'uses' Returns: - list - of parsed prefixes + list of parsed prefixes """ ret_prefixes = list() @@ -588,7 +588,7 @@ def get_list_keys(y_list: OrderedDict) -> list: Args: y_list: reference to 'list' Returns: - list: parsed keys + liss of parsed keys """ ret_list = list() @@ -600,7 +600,7 @@ def get_list_keys(y_list: OrderedDict) -> list: return ret_list -def change_dyn_obj_struct(dynamic_objects: OrderedDict): +def change_dyn_obj_struct(dynamic_objects: list): """ Rearrange self.yang_2_dict['dynamic_objects'] structure. If YANG model have a 'list' entity - inside the 'list' it has 'key' entity. 'key' entity it is whitespace-separeted list of 'leafs', those 'leafs' was From 290752358c30c8c9718b90346825d65f923347c1 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Mon, 24 May 2021 13:52:54 +0000 Subject: [PATCH 45/76] Added list_handler() Signed-off-by: Vadym Hlushko --- sonic_cli_gen/yang_parser.py | 42 +++++++++++++++--------------------- 1 file changed, 17 insertions(+), 25 deletions(-) diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index a48b7b707a..177fa5478e 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -3,6 +3,8 @@ from collections import OrderedDict from config.config_mgmt import ConfigMgmt +from typing import List, Dict + yang_guidelines_link = 'https://github.com/Azure/SONiC/blob/master/doc/mgmt/SONiC_YANG_Model_Guidelines.md' class YangParser: @@ -117,19 +119,21 @@ def parse_yang_model(self) -> dict: # determine how many (1 or more) containers a YANG model have after 'top level' container # 'table' container it is a container that goes after 'top level' container - if isinstance(self.y_table_containers, list): - for tbl_container in self.y_table_containers: - y2d_elem = on_table_container(self.y_module, tbl_container, self.conf_mgmt) - self.yang_2_dict['tables'].append(y2d_elem) - else: - y2d_elem = on_table_container(self.y_module, self.y_table_containers, self.conf_mgmt) - self.yang_2_dict['tables'].append(y2d_elem) + self.yang_2_dict['tables'] += list_handler(self.y_table_containers, lambda e: on_table_container(self.y_module, e, self.conf_mgmt)) return self.yang_2_dict #------------------------------HANDLERS--------------------------------# +def list_handler(y_entity, callback) -> List[Dict]: + """ Determine if the type of entity is a list, if so - call the callback for every list element """ + if isinstance(y_entity, list): + return [callback(e) for e in y_entity] + else: + return [callback(y_entity)] + + def on_table_container(y_module: OrderedDict, tbl_container: OrderedDict, conf_mgmt: ConfigMgmt) -> dict: """ Parse 'table' container, 'table' container goes after 'top level' container @@ -142,7 +146,6 @@ def on_table_container(y_module: OrderedDict, tbl_container: OrderedDict, conf_m Returns: element for self.yang_2_dict['tables'] """ - y2d_elem = { 'name': tbl_container.get('@name'), 'description': get_description(tbl_container) @@ -153,26 +156,15 @@ def on_table_container(y_module: OrderedDict, tbl_container: OrderedDict, conf_m y2d_elem['static-objects'] = list() # 'object' container goes after 'table' container - # 'object' container have 2 types - list (like sonic-flex_counter.yang) and NOT list (like sonic-device_metadata.yang) obj_container = tbl_container.get('container') - if isinstance(obj_container, list): - for y_container in obj_container: - static_obj_elem = on_object_container(y_module, y_container, conf_mgmt, is_list=False) - y2d_elem['static-objects'].append(static_obj_elem) - else: - static_obj_elem = on_object_container(y_module, obj_container, conf_mgmt, is_list=False) - y2d_elem['static-objects'].append(static_obj_elem) + # 'object' container have 2 types - list (like sonic-flex_counter.yang) and NOT list (like sonic-device_metadata.yang) + y2d_elem['static-objects'] += list_handler(obj_container, lambda e: on_object_container(y_module, e, conf_mgmt, is_list = False)) else: y2d_elem['dynamic-objects'] = list() tbl_container_lists = tbl_container.get('list') + # 'container' can have more than 1 'list' entity - if isinstance(tbl_container_lists, list): - for _list in tbl_container_lists: - dynamic_obj_elem = on_object_container(y_module, _list, conf_mgmt, is_list=True) - y2d_elem['dynamic-objects'].append(dynamic_obj_elem) - else: - dynamic_obj_elem = on_object_container(y_module, tbl_container_lists, conf_mgmt, is_list=True) - y2d_elem['dynamic-objects'].append(dynamic_obj_elem) + y2d_elem['dynamic-objects'] += list_handler(tbl_container_lists, lambda e: on_object_container(y_module, e, conf_mgmt, is_list = True)) # move 'keys' elements from 'attrs' to 'keys' change_dyn_obj_struct(y2d_elem['dynamic-objects']) @@ -566,7 +558,7 @@ def trim_uses_prefixes(y_uses) -> list: Where 'sgrop' = 'prefix'; 'endpoint' = 'grouping' name. Args: - y_uses - reference to 'uses' + y_uses: reference to 'uses' """ prefixes = get_import_prefixes(y_uses) @@ -588,7 +580,7 @@ def get_list_keys(y_list: OrderedDict) -> list: Args: y_list: reference to 'list' Returns: - liss of parsed keys + list of parsed keys """ ret_list = list() From a005d6d6120370833259806169c3e0babad665c1 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Fri, 21 May 2021 19:29:09 +0300 Subject: [PATCH 46/76] [sonic-cli-gen] address review comments Signed-off-by: Stepan Blyschak --- .../templates/sonic-cli-gen/show.py.j2 | 127 ++++++++++++++++-- 1 file changed, 115 insertions(+), 12 deletions(-) diff --git a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 index 3859573d72..aad5fc539c 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 @@ -1,5 +1,13 @@ {% from "common.j2" import cli_name -%} -""" Autogenerated show CLI plugin """ +""" +Auto-generated show CLI plugin. +{% if source_template is defined %} +Source template: {{ source_template }} +{% endif %} +{% if source_yang_module is defined %} +Source YANG module: {{ source_yang_module }} +{% endif %} +""" import click import tabulate @@ -12,45 +20,131 @@ import utilities_common.cli as clicommon {%- endmacro %} -def print_attr_helper(entry, attr): +def format_attr_value(entry, attr): + """ Helper that formats attribute to be presented in the table output. + + Args: + entry (Dict[str, str]): CONFIG DB entry configuration. + attr (Dict): Attribute metadata. + + Returns: + str: fomatted attribute value. + """ + if attr["is-leaf-list"]: return "\n".join(entry.get(attr["name"], [])) return entry.get(attr["name"], "N/A") -def print_group_helper(entry, attrs): +def format_group_value(entry, attrs): + """ Helper that formats grouped attribute to be presented in the table output. + + Args: + entry (Dict[str, str]): CONFIG DB entry configuration. + attrs (List[Dict]): Attributes metadata that belongs to the same group. + + Returns: + str: fomatted group attributes. + """ + data = [] for attr in attrs: if entry.get(attr["name"]): - data.append((attr["name"] + ":", print_attr_helper(entry, attr))) + data.append((attr["name"] + ":", format_attr_value(entry, attr))) return tabulate.tabulate(data, tablefmt="plain") +{# Generates a python list that represents a row in the table view. +E.g: +Jinja2: +{{ + gen_row("entry", [ + {"name": "leaf1"}, + {"name": "leaf_1"}, + {"name": "leaf_2"}, + {"name": "leaf_3", "group": "group_0"} + ]) +}} +Result: +[ + format_attr_value( + entry, + {'name': 'leaf1'} + ), + format_attr_value( + entry, + {'name': 'leaf_1'} + ), + format_attr_value( + entry, + {'name': 'leaf_2'} + ), + format_group_value( + entry, + [{'name': 'leaf_3', 'group': 'group_0'}] + ), +] +#} {% macro gen_row(entry, attrs) -%} [ {%- for attr in attrs|rejectattr("group", "defined") %} -print_attr_helper({{ entry }}, {{ attr }}), + format_attr_value( + {{ entry }}, + {{ attr }} + ), {%- endfor %} {%- for group, attrs in attrs|selectattr("group", "defined")|groupby("group") %} -print_group_helper({{ entry }}, {{ attrs }}), + format_group_value( + {{ entry }}, + {{ attrs }} + ), {%- endfor %} ] {% endmacro %} +{# Generates a list that represents a header in table view. +E.g: +Jinja2: {{ + gen_header([ + {"name": "key"}, + {"name": "leaf_1"}, + {"name": "leaf_2"}, + {"name": "leaf_3", "group": "group_0"} + ]) + }} + +Result: +[ + "KEY", + "LEAF 1", + "LEAF 2", + "GROUP 0", +] + +#} {% macro gen_header(attrs) -%} [ -{% for attr in attrs|rejectattr("group", "defined") %} -"{{ column_name(attr.name) }}", -{% endfor %} -{% for group, attrs in attrs|selectattr("group", "defined")|groupby("group") %} -"{{ column_name(group) }}", -{% endfor %} +{% for attr in attrs|rejectattr("group", "defined") -%} + "{{ column_name(attr.name) }}", +{% endfor -%} +{% for group, attrs in attrs|selectattr("group", "defined")|groupby("group") -%} + "{{ column_name(group) }}", +{% endfor -%} ] {% endmacro %} {% for table in tables %} {% if "static-objects" in table %} +{# For static objects generate a command group called against table name. +E.g: +@click.group(name="table-name", + cls=clicommon.AliasedGroup) +def TABLE_NAME(): + """ TABLE DESCRIPTION """ + + pass +#} @click.group(name="{{ cli_name(table.name) }}", cls=clicommon.AliasedGroup) def {{ table.name }}(): @@ -59,6 +153,15 @@ def {{ table.name }}(): pass {% for object in table["static-objects"] %} +{# For every object in static table generate a command +in the group to show individual object configuration. +CLI command is named against the object key in DB. +E.g: +@TABLE_NAME.command(name="object-name") +@clicommon.pass_db +def TABLE_NAME_object_name(db): + ... +#} @{{ table.name }}.command(name="{{ cli_name(object.name) }}") @clicommon.pass_db def {{ table.name }}_{{ object.name }}(db): From aa00b616f7851caf3cba8f861d7201d9bd52724f Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Mon, 24 May 2021 18:37:26 +0300 Subject: [PATCH 47/76] [sonic-cli-gen] fix review comments Signed-off-by: Stepan Blyschak --- .../templates/sonic-cli-gen/config.py.j2 | 149 +++++++++++------- .../templates/sonic-cli-gen/show.py.j2 | 20 ++- 2 files changed, 111 insertions(+), 58 deletions(-) diff --git a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 index ac19835cd1..402b7e3dd2 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 @@ -1,5 +1,13 @@ {%- from "common.j2" import cli_name -%} -""" Autogenerated config CLI plugin """ +""" +Autogenerated config CLI plugin. +{% if source_template is defined %} +Source template: {{ source_template }} +{% endif %} +{% if source_yang_module is defined %} +Source YANG module: {{ source_yang_module }} +{% endif %} +""" import click import utilities_common.cli as clicommon @@ -11,7 +19,7 @@ from config import config_mgmt sonic_cfggen = general.load_module_from_source('sonic_cfggen', '/usr/local/bin/sonic-cfggen') -def exit_cli(*args, **kwargs): +def exit_with_error(*args, **kwargs): """ Print a message and abort CLI. """ click.secho(*args, **kwargs) @@ -28,20 +36,8 @@ def validate_config_or_raise(cfg): raise Exception('Failed to validate configuration: {}'.format(err)) -def mod_entry_validated(db, table, key, data): - """ Modify existing entry and validate configuration """ - - cfg = db.get_config() - cfg.setdefault(table, {}) - cfg[table].setdefault(key, {}) - cfg[table][key].update(data) - - validate_config_or_raise(cfg) - db.mod_entry(table, key, data) - - def add_entry_validated(db, table, key, data): - """ Add new entry in table and validate configuration""" + """ Add new entry in table and validate configuration """ cfg = db.get_config() cfg.setdefault(table, {}) @@ -54,18 +50,28 @@ def add_entry_validated(db, table, key, data): db.set_entry(table, key, data) -def update_entry_validated(db, table, key, data): - """ Update entry in table and validate configuration""" +def update_entry_validated(db, table, key, data, create_if_not_exists=False): + """ Update entry in table and validate configuration. + If attribute value in data is None, the attribute is deleted. + """ cfg = db.get_config() cfg.setdefault(table, {}) + + if create_if_not_exists: + cfg[table].setdefault(key, {}) + if key not in cfg[table]: raise Exception(f"{key} does not exist") - cfg[table][key].update(data) + for attr, value in data.items(): + if value is None and attr in cfg[table][key]: + cfg[table][key].pop(attr) + else: + cfg[table][key][attr] = value validate_config_or_raise(cfg) - db.mod_entry(table, key, data) + db.set_entry(table, key, cfg[table][key]) def del_entry_validated(db, table, key): @@ -121,33 +127,65 @@ def del_list_entry_validated(db, table, key, attr, data): def clear_list_entry_validated(db, table, key, attr): """ Clear list in object and validate configuration""" - update_entry_validated(db, table, key, {attr: []}) + update_entry_validated(db, table, key, {attr: None}) -{%- macro gen_click_arguments(args) -%} -{%- for arg in args %} +{# Generate click arguments macro +Jinja2 Call: + {{ gen_click_arguments([{"name": "leaf1", "is-leaf-list": False}, + {"name": "leaf2", "is-leaf-list": Talse}) }} +Result: @click.argument( - "{{ cli_name(arg.name) }}", - nargs={% if arg["is-leaf-list"] %}-1{% else %}1{% endif %}, + "leaf1", + nargs=1, + required=True, +) +@click.argument( + "leaf2", + nargs=-1, + required=True, +) +#} +{%- macro gen_click_arguments(attrs) -%} +{%- for attr in attrs %} +@click.argument( + "{{ cli_name(attr.name) }}", + nargs={% if attr["is-leaf-list"] %}-1{% else %}1{% endif %}, required=True, ) {%- endfor %} {%- endmacro %} -{%- macro gen_click_options(opts) -%} -{%- for opt in opts %} + +{# Generate click options macro +Jinja2 Call: + {{ gen_click_arguments([{"name": "leaf1", "is-mandatory": True, "description": "leaf1-desc"}, + {"name": "leaf2", "is-mandatory": False, "description": "leaf2-desc"}) }} +Result: @click.option( - "--{{ cli_name(opt.name) }}", - help="{{ opt.description }}{% if opt['is-mandatory'] %}[mandatory]{% endif %}", + "--leaf1", + help="leaf1-desc [mandatory]", +) +@click.option( + "--leaf2", + help="leaf2-desc", +) +#} +{%- macro gen_click_options(attrs) -%} +{%- for attr in attrs %} +@click.option( + "--{{ cli_name(attr.name) }}", + help="{{ attr.description }}{% if attr['is-mandatory'] %}[mandatory]{% endif %}", ) {%- endfor %} {%- endmacro %} +{# Generate valid python identifier from input names #} {% macro pythonize(attrs) -%} {{ attrs|map(attribute="name")|map("lower")|map("replace", "-", "_")|join(", ") }} {%- endmacro %} -{% macro config_object_list_update(group, table, object, attr) %} +{% macro gen_cfg_obj_list_update(group, table, object, attr) %} {% set list_update_group = group + "_" + attr.name %} @{{ group }}.group(name="{{ cli_name(attr.name) }}", @@ -182,7 +220,7 @@ def {{ list_update_group }}_add( try: add_list_entry_validated(db.cfgdb, table, key, attr, data) except Exception as err: - exit_cli(f"Error: {err}", fg="red") + exit_with_error(f"Error: {err}", fg="red") {# Delete entries from list attribute config CLI generation @@ -210,7 +248,7 @@ def {{ list_update_group }}_delete( try: del_list_entry_validated(db.cfgdb, table, key, attr, data) except Exception as err: - exit_cli(f"Error: {err}", fg="red") + exit_with_error(f"Error: {err}", fg="red") {# Clear entries from list attribute config CLI generation @@ -223,7 +261,7 @@ E.g: @{{ list_update_group }}.command(name="clear") {{ gen_click_arguments(object["keys"]) }} @clicommon.pass_db -def {{ list_update_group }}_delete( +def {{ list_update_group }}_clear( db, {{ pythonize(object["keys"]) }} ): @@ -232,26 +270,25 @@ def {{ list_update_group }}_delete( table = "{{ table.name }}" key = {{ pythonize(object["keys"]) }} attr = "{{ attr.name }}" - data = {{ pythonize([attr]) }} try: clear_list_entry_validated(db.cfgdb, table, key, attr) except Exception as err: - exit_cli(f"Error: {err}", fg="red") + exit_with_error(f"Error: {err}", fg="red") {% endmacro %} -{% macro config_object_list_update_all(group, table, object) %} +{% macro gen_cfg_obj_list_update_all(group, table, object) %} {% for attr in object.attrs %} {% if attr["is-leaf-list"] %} -{{ config_object_list_update(group, table, object, attr) }} +{{ gen_cfg_obj_list_update(group, table, object, attr) }} {% endif %} {% endfor %} {% endmacro %} -{% macro config_static_object_attr(table, object, attr) %} +{% macro gen_cfg_static_obj_attr(table, object, attr) %} @{{ table.name }}_{{ object.name }}.command(name="{{ cli_name(attr.name) }}") {{ gen_click_arguments([attr]) }} @clicommon.pass_db @@ -264,9 +301,9 @@ def {{ table.name }}_{{ object.name }}_{{ attr.name }}(db, {{ pythonize([attr]) "{{ attr.name }}": {{ pythonize([attr]) }}, } try: - mod_entry_validated(db.cfgdb, table, key, data) + update_entry_validated(db.cfgdb, table, key, data, create_if_not_exists=True) except Exception as err: - exit_cli(f"Error: {err}", fg="red") + exit_with_error(f"Error: {err}", fg="red") {% endmacro %} @@ -275,7 +312,7 @@ E.g: @TABLE.group(name="object") def TABLE_object(db): #} -{% macro config_static_object(table, object) %} +{% macro gen_cfg_static_obj(table, object) %} @{{ table.name }}.group(name="{{ cli_name(object.name) }}", cls=clicommon.AliasedGroup) @clicommon.pass_db @@ -290,10 +327,10 @@ E.g: def TABLE_object_attribute(db, attribute): #} {% for attr in object.attrs %} -{{ config_static_object_attr(table, object, attr) }} +{{ gen_cfg_static_obj_attr(table, object, attr) }} {% endfor %} -{{ config_object_list_update_all(table.name + "_" + object.name, table, object) }} +{{ gen_cfg_obj_list_update_all(table.name + "_" + object.name, table, object) }} {% endmacro %} {# Dynamic objects config CLI generation #} @@ -308,7 +345,7 @@ E.g: @click.option("--attr3") def TABLE_TABLE_LIST_add(db, key1, key2, attr1, attr2, attr3): #} -{% macro config_dynamic_object_add(group, table, object) %} +{% macro gen_cfg_dyn_obj_add(group, table, object) %} @{{ group }}.command(name="add") {{ gen_click_arguments(object["keys"]) }} {{ gen_click_options(object.attrs) }} @@ -331,7 +368,7 @@ def {{ group }}_add(db, {{ pythonize(object["keys"] + object.attrs) }}): try: add_entry_validated(db.cfgdb, table, key, data) except Exception as err: - exit_cli(f"Error: {err}", fg="red") + exit_with_error(f"Error: {err}", fg="red") {% endmacro %} {# Dynamic objects update command @@ -344,7 +381,7 @@ E.g: @click.option("--attr3") def TABLE_TABLE_LIST_update(db, key1, key2, attr1, attr2, attr3): #} -{% macro config_dynamic_object_update(group, table, object) %} +{% macro gen_cfg_dyn_obj_update(group, table, object) %} @{{ group }}.command(name="update") {{ gen_click_arguments(object["keys"]) }} {{ gen_click_options(object.attrs) }} @@ -367,7 +404,7 @@ def {{ group }}_update(db, {{ pythonize(object["keys"] + object.attrs) }}): try: update_entry_validated(db.cfgdb, table, key, data) except Exception as err: - exit_cli(f"Error: {err}", fg="red") + exit_with_error(f"Error: {err}", fg="red") {% endmacro %} {# Dynamic objects delete command @@ -377,7 +414,7 @@ E.g: @click.argument("key2") def TABLE_TABLE_LIST_delete(db, key1, key2): #} -{% macro config_dynamic_object_delete(group, table, object) %} +{% macro gen_cfg_dyn_obj_delete(group, table, object) %} @{{ group }}.command(name="delete") {{ gen_click_arguments(object["keys"]) }} @clicommon.pass_db @@ -389,11 +426,11 @@ def {{ group }}_delete(db, {{ pythonize(object["keys"]) }}): try: del_entry_validated(db.cfgdb, table, key) except Exception as err: - exit_cli(f"Error: {err}", fg="red") + exit_with_error(f"Error: {err}", fg="red") {% endmacro %} -{% macro config_dynamic_object(table, object) %} -{# Generate another nesting group in case table holds two types of objects #} +{% macro gen_cfg_dyn_obj(table, object) %} +{# Generate another nested group in case table holds two types of objects #} {% if table["dynamic-objects"]|length > 1 %} {% set group = table.name + "_" + object.name %} @{{ table.name }}.group(name="{{ cli_name(object.name) }}", @@ -406,10 +443,10 @@ def {{ group }}(): {% set group = table.name %} {% endif %} -{{ config_dynamic_object_add(group, table, object) }} -{{ config_dynamic_object_update(group, table, object) }} -{{ config_dynamic_object_delete(group, table, object) }} -{{ config_object_list_update_all(group, table, object) }} +{{ gen_cfg_dyn_obj_add(group, table, object) }} +{{ gen_cfg_dyn_obj_update(group, table, object) }} +{{ gen_cfg_dyn_obj_delete(group, table, object) }} +{{ gen_cfg_obj_list_update_all(group, table, object) }} {% endmacro %} @@ -423,13 +460,13 @@ def {{ table.name }}(): {% if "static-objects" in table %} {% for object in table["static-objects"] %} -{{ config_static_object(table, object) }} +{{ gen_cfg_static_obj(table, object) }} {% endfor %} {% endif %} {% if "dynamic-objects" in table %} {% for object in table["dynamic-objects"] %} -{{ config_dynamic_object(table, object) }} +{{ gen_cfg_dyn_obj(table, object) }} {% endfor %} {% endif %} diff --git a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 index aad5fc539c..6ee27f2013 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 @@ -94,10 +94,19 @@ Result: ), {%- endfor %} {%- for group, attrs in attrs|selectattr("group", "defined")|groupby("group") %} +{%- if group == "" %} +{%- for attr in attrs %} + format_attr_value( + {{ entry }}, + {{ attr }} + ), +{%- endfor %} +{%- else %} format_group_value( {{ entry }}, {{ attrs }} ), +{%- endif %} {%- endfor %} ] {% endmacro %} @@ -128,7 +137,13 @@ Result: "{{ column_name(attr.name) }}", {% endfor -%} {% for group, attrs in attrs|selectattr("group", "defined")|groupby("group") -%} +{%- if group == "" %} +{% for attr in attrs -%} + "{{ column_name(attr.name) }}", +{% endfor -%} +{%- else %} "{{ column_name(group) }}", +{%- endif %} {% endfor -%} ] {% endmacro %} @@ -148,7 +163,7 @@ def TABLE_NAME(): @click.group(name="{{ cli_name(table.name) }}", cls=clicommon.AliasedGroup) def {{ table.name }}(): - """ {{ table.description }}""" + """ {{ table.description }} """ pass @@ -196,12 +211,13 @@ def {{ table.name }}(): {% set name = table.name %} {% endif %} +{# Generate an implementation to display table. #} @{{ group }}.group(name="{{ cli_name(name) }}", cls=clicommon.AliasedGroup, invoke_without_command=True) @clicommon.pass_db def {{ name }}(db): - """ {{ object.description }} """ + """ {{ object.description }} [Callable command group] """ header = {{ gen_header(object["keys"] + object.attrs) }} body = [] From 5c4e219107eb2ac13553fb825e1da02631418754 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Mon, 24 May 2021 16:14:19 +0000 Subject: [PATCH 48/76] Fixed comments, added list_handler() where needed Signed-off-by: Vadym Hlushko --- sonic_cli_gen/yang_parser.py | 134 +++++++++++++++++------------------ 1 file changed, 66 insertions(+), 68 deletions(-) diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index 177fa5478e..0fdbb7fceb 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -94,21 +94,19 @@ def _init_yang_module_and_containers(self): self.y_table_containers = self.y_top_level_container.get('container') def _find_yang_model_in_yjson_obj(self) -> OrderedDict: - """ Find provided YANG model inside yJson object, - yJson object contain all yang-models parsed from directory - /usr/local/yang-models + """ Find provided YANG model inside the yJson object, + the yJson object contain all yang-models parsed from directory - /usr/local/yang-models Returns: reference to yang_model_name """ - # TODO: consider to check yJson type for yang_model in self.conf_mgmt.sy.yJson: if yang_model.get('module').get('@name') == self.yang_model_name: return yang_model.get('module') def parse_yang_model(self) -> dict: - """ Parse provided YANG model - and save output to self.yang_2_dict object + """ Parse provided YANG model and save the output to self.yang_2_dict object Returns: parsed YANG model in dictionary format @@ -117,9 +115,10 @@ def parse_yang_model(self) -> dict: self._init_yang_module_and_containers() self.yang_2_dict['tables'] = list() - # determine how many (1 or more) containers a YANG model have after 'top level' container - # 'table' container it is a container that goes after 'top level' container - self.yang_2_dict['tables'] += list_handler(self.y_table_containers, lambda e: on_table_container(self.y_module, e, self.conf_mgmt)) + # determine how many (1 or more) containers a YANG model have after the 'top level' container + # 'table' container goes after the 'top level' container + self.yang_2_dict['tables'] += list_handler(self.y_table_containers, + lambda e: on_table_container(self.y_module, e, self.conf_mgmt)) return self.yang_2_dict @@ -128,6 +127,7 @@ def parse_yang_model(self) -> dict: def list_handler(y_entity, callback) -> List[Dict]: """ Determine if the type of entity is a list, if so - call the callback for every list element """ + if isinstance(y_entity, list): return [callback(e) for e in y_entity] else: @@ -151,20 +151,20 @@ def on_table_container(y_module: OrderedDict, tbl_container: OrderedDict, conf_m 'description': get_description(tbl_container) } - # determine if 'table container' have a 'list' entity + # determine if 'table container' has a 'list' entity if tbl_container.get('list') is None: y2d_elem['static-objects'] = list() - # 'object' container goes after 'table' container - obj_container = tbl_container.get('container') + # 'object' container goes after the 'table' container # 'object' container have 2 types - list (like sonic-flex_counter.yang) and NOT list (like sonic-device_metadata.yang) - y2d_elem['static-objects'] += list_handler(obj_container, lambda e: on_object_container(y_module, e, conf_mgmt, is_list = False)) + y2d_elem['static-objects'] += list_handler(tbl_container.get('container'), + lambda e: on_object_entity(y_module, e, conf_mgmt, is_list = False)) else: y2d_elem['dynamic-objects'] = list() - tbl_container_lists = tbl_container.get('list') # 'container' can have more than 1 'list' entity - y2d_elem['dynamic-objects'] += list_handler(tbl_container_lists, lambda e: on_object_container(y_module, e, conf_mgmt, is_list = True)) + y2d_elem['dynamic-objects'] += list_handler(tbl_container.get('list'), + lambda e: on_object_entity(y_module, e, conf_mgmt, is_list = True)) # move 'keys' elements from 'attrs' to 'keys' change_dyn_obj_struct(y2d_elem['dynamic-objects']) @@ -172,9 +172,9 @@ def on_table_container(y_module: OrderedDict, tbl_container: OrderedDict, conf_m return y2d_elem -def on_object_container(y_module: OrderedDict, y_container: OrderedDict, conf_mgmt, is_list: bool) -> dict: - """ Parse a 'object container'. - 'Object container' represent OBJECT inside Config DB schema: +def on_object_entity(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: ConfigMgmt, is_list: bool) -> dict: + """ Parse a 'object' entity, it could be a 'container' or a 'list' + 'Object' entity represent OBJECT inside Config DB schema: { "TABLE": { "OBJECT": { @@ -185,32 +185,32 @@ def on_object_container(y_module: OrderedDict, y_container: OrderedDict, conf_mg Args: y_module: reference to 'module' - y_container: reference to 'object container' + y_entity: reference to 'object' entity conf_mgmt: reference to ConfigMgmt class instance, it have yJson object which contain all parsed YANG models - is_list: boolean flag to determine if container has 'list' + is_list: boolean flag to determine if a 'list' was passed Returns: element for y2d_elem['static-objects'] OR y2d_elem['dynamic-objects'] """ - if y_container is None: + if y_entity is None: return {} obj_elem = { - 'name': y_container.get('@name'), - 'description': get_description(y_container), + 'name': y_entity.get('@name'), + 'description': get_description(y_entity), 'attrs': list() } if is_list: - obj_elem['keys'] = get_list_keys(y_container) + obj_elem['keys'] = get_list_keys(y_entity) attrs_list = list() # grouping_name is empty because 'grouping' is not used so far - attrs_list.extend(get_leafs(y_container, grouping_name = '')) - attrs_list.extend(get_leaf_lists(y_container, grouping_name = '')) - attrs_list.extend(get_choices(y_module, y_container, conf_mgmt, grouping_name = '')) - attrs_list.extend(get_uses(y_module, y_container, conf_mgmt)) + attrs_list.extend(get_leafs(y_entity, grouping_name = '')) + attrs_list.extend(get_leaf_lists(y_entity, grouping_name = '')) + attrs_list.extend(get_choices(y_module, y_entity, conf_mgmt, grouping_name = '')) + attrs_list.extend(get_uses(y_module, y_entity, conf_mgmt)) obj_elem['attrs'] = attrs_list @@ -219,7 +219,7 @@ def on_object_container(y_module: OrderedDict, y_container: OrderedDict, conf_mg def on_uses(y_module: OrderedDict, y_uses, conf_mgmt: ConfigMgmt) -> list: """ Parse a YANG 'uses' entities - 'uses' refearing to 'grouping' YANG entity + 'uses' referring to 'grouping' YANG entity Args: y_module: reference to 'module' @@ -282,18 +282,18 @@ def on_choices(y_module: OrderedDict, y_choices, conf_mgmt: ConfigMgmt, grouping return ret_attrs -def on_choice_cases(y_module: OrderedDict, y_cases: list, conf_mgmt: ConfigMgmt, grouping_name: str) -> list: - """ Parse a single YANG 'case' entity from 'choice' entity - 'case' element can have inside - 'leaf', 'leaf-list', 'uses' +def on_choice_cases(y_module: OrderedDict, y_cases, conf_mgmt: ConfigMgmt, grouping_name: str) -> list: + """ Parse a single YANG 'case' entity from the 'choice' entity. + The 'case' element can has inside - 'leaf', 'leaf-list', 'uses' Args: y_module: reference to 'module' y_cases: reference to 'case' conf_mgmt: reference to ConfigMgmt class instance, it have yJson object which contain all parsed YANG model - grouping_name: if YANG entity contain 'uses', this arg represent 'grouping' name + grouping_name: if YANG entity contain 'uses', this argument represent 'grouping' name Returns: - element for obj_elem['attrs'], 'attrs' contain a parsed 'leafs' + element for the obj_elem['attrs'], the 'attrs' contain a parsed 'leafs' """ ret_attrs = list() @@ -314,32 +314,26 @@ def on_leafs(y_leafs, grouping_name, is_leaf_list: bool) -> list: Args: y_leafs: reference to all 'leaf' elements - grouping_name: if YANG entity contain 'uses', this arg represent 'grouping' name - is_leaf_list: boolean to determine if 'leaf-list' was passed in 'y_leafs' arg + grouping_name: if YANG entity contain 'uses', this argument represent the 'grouping' name + is_leaf_list: boolean to determine if a 'leaf-list' was passed as 'y_leafs' argument Returns: list of parsed 'leaf' elements """ ret_attrs = list() # The YANG 'container' entity may have only 1 'leaf' element OR a list of 'leaf' elements - if isinstance(y_leafs, list): - for leaf in y_leafs: - attr = on_leaf(leaf, is_leaf_list, grouping_name) - ret_attrs.append(attr) - else: - attr = on_leaf(y_leafs, is_leaf_list, grouping_name) - ret_attrs.append(attr) + ret_attrs += list_handler(y_leafs, lambda e: on_leaf(e, grouping_name, is_leaf_list)) return ret_attrs -def on_leaf(leaf: OrderedDict, is_leaf_list: bool, grouping_name: str) -> dict: +def on_leaf(leaf: OrderedDict, grouping_name: str, is_leaf_list: bool) -> dict: """ Parse a single 'leaf' element Args: leaf: reference to a 'leaf' entity - grouping_name: if YANG entity contain 'uses', this arg represent 'grouping' name - is_leaf_list: boolean to determine if 'leaf-list' was passed in 'y_leafs' arg + grouping_name: if YANG entity contain 'uses', this argument represent 'grouping' name + is_leaf_list: boolean to determine if 'leaf-list' was passed in 'y_leafs' argument Returns: parsed 'leaf' element """ @@ -356,7 +350,7 @@ def on_leaf(leaf: OrderedDict, is_leaf_list: bool, grouping_name: str) -> dict: #----------------------GETERS-------------------------# def get_mandatory(y_leaf: OrderedDict) -> bool: - """ Parse 'mandatory' statement for 'leaf' + """ Parse the 'mandatory' statement for a 'leaf' Args: y_leaf: reference to a 'leaf' entity @@ -371,7 +365,7 @@ def get_mandatory(y_leaf: OrderedDict) -> bool: def get_description(y_entity: OrderedDict) -> str: - """ Parse 'description' entity from any YANG element + """ Parse the 'description' entity from any YANG element Args: y_entity: reference to YANG 'container' OR 'list' OR 'leaf' ... @@ -386,11 +380,11 @@ def get_description(y_entity: OrderedDict) -> str: def get_leafs(y_entity: OrderedDict, grouping_name: str) -> list: - """ Check if YANG entity have 'leafs', if so call handler + """ Check if the YANG entity have 'leafs', if so call handler Args: y_entity: reference YANG 'container' or 'list' or 'choice' or 'uses' - grouping_name: if YANG entity contain 'uses', this arg represent 'grouping' name + grouping_name: if YANG entity contain 'uses', this argument represent 'grouping' name Returns: list of parsed 'leaf' elements """ @@ -402,11 +396,11 @@ def get_leafs(y_entity: OrderedDict, grouping_name: str) -> list: def get_leaf_lists(y_entity: OrderedDict, grouping_name: str) -> list: - """ Check if YANG entity have 'leaf-list', if so call handler + """ Check if the YANG entity have 'leaf-list', if so call handler Args: y_entity: reference YANG 'container' or 'list' or 'choice' or 'uses' - grouping_name: if YANG entity contain 'uses', this arg represent 'grouping' name + grouping_name: if YANG entity contain 'uses', this argument represent 'grouping' name Returns: list of parsed 'leaf-list' elements """ @@ -418,14 +412,14 @@ def get_leaf_lists(y_entity: OrderedDict, grouping_name: str) -> list: def get_choices(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: ConfigMgmt, grouping_name: str) -> list: - """ Check if YANG entity have 'choice', if so call handler + """ Check if the YANG entity have 'choice', if so call handler Args: y_module: reference to 'module' y_entity: reference YANG 'container' or 'list' or 'choice' or 'uses' conf_mgmt: reference to ConfigMgmt class instance, it have yJson object which contain all parsed YANG model - grouping_name: if YANG entity contain 'uses', this arg represent 'grouping' name + grouping_name: if YANG entity contain 'uses', this argument represent 'grouping' name Returns: list of parsed elements inside 'choice' """ @@ -437,7 +431,7 @@ def get_choices(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: ConfigM def get_uses(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: ConfigMgmt) -> list: - """ Check if YANG entity have 'uses', if so call handler + """ Check if the YANG entity have 'uses', if so call handler Args: y_module: reference to 'module' @@ -455,7 +449,7 @@ def get_uses(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: ConfigMgmt def get_all_grouping(y_module: OrderedDict, y_uses: OrderedDict, conf_mgmt: ConfigMgmt) -> list: - """ Get all 'grouping' entities that is referenced by 'uses' in current YANG model + """ Get all the 'grouping' entities that was referenced by 'uses' in current YANG model Args: y_module: reference to 'module' @@ -467,14 +461,14 @@ def get_all_grouping(y_module: OrderedDict, y_uses: OrderedDict, conf_mgmt: Conf """ ret_grouping = list() + # prefix_list needed to find what YANG model was imported prefix_list = get_import_prefixes(y_uses) # in case if 'grouping' located in the same YANG model local_grouping = y_module.get('grouping') if local_grouping is not None: if isinstance(local_grouping, list): - for group in local_grouping: - ret_grouping.append(group) + ret_grouping.extend(local_grouping) else: ret_grouping.append(local_grouping) @@ -502,17 +496,16 @@ def get_grouping_from_another_yang_model(yang_model_name: str, conf_mgmt) -> lis it have yJson object which contain all parsed YANG models Returns: - list 'grouping' entities + list of 'grouping' entities """ ret_grouping = list() - for i in range(len(conf_mgmt.sy.yJson)): - if (conf_mgmt.sy.yJson[i].get('module').get('@name') == yang_model_name): - grouping = conf_mgmt.sy.yJson[i].get('module').get('grouping') + for yang_model in conf_mgmt.sy.yJson: + if (yang_model.get('module').get('@name') == yang_model_name): + grouping = yang_model.get('module').get('grouping') if isinstance(grouping, list): - for group in grouping: - ret_grouping.append(group) + ret_grouping.extend(grouping) else: ret_grouping.append(grouping) @@ -550,8 +543,8 @@ def get_import_prefixes(y_uses: OrderedDict) -> list: def trim_uses_prefixes(y_uses) -> list: - """ Trim prefixes from 'uses' YANG entities. - If YANG 'grouping' was imported from another YANG file, it use 'prefix' before 'grouping' name: + """ Trim prefixes from the 'uses' YANG entities. + If the YANG 'grouping' was imported from another YANG file, it use the 'prefix' before the 'grouping' name: { uses sgrop:endpoint; } @@ -559,6 +552,9 @@ def trim_uses_prefixes(y_uses) -> list: Args: y_uses: reference to 'uses' + + Returns: + list of 'uses' without 'prefixes' """ prefixes = get_import_prefixes(y_uses) @@ -574,16 +570,18 @@ def trim_uses_prefixes(y_uses) -> list: def get_list_keys(y_list: OrderedDict) -> list: - """ Parse YANG 'keys' - If YANG have 'list', inside the list exist 'keys' + """ Parse YANG the 'key' entity. + If YANG model has a 'list' entity, inside the 'list' there is 'key' entity. + 'key' - whitespace separeted list of 'leafs' Args: - y_list: reference to 'list' + y_list: reference to the 'list' Returns: list of parsed keys """ ret_list = list() + keys = y_list.get('key').get('@value').split() for k in keys: key = { 'name': k } From 6de672c5096fe0f2ed66d185f3a3dd80ca1ad61d Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Mon, 24 May 2021 16:35:21 +0000 Subject: [PATCH 49/76] Added logger to CliGenerator class constructor Signed-off-by: Vadym Hlushko --- sonic_cli_gen/generator.py | 20 +++++++++++++------- sonic_cli_gen/main.py | 7 ++++++- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py index 48142a9e7b..5a4f1be93d 100644 --- a/sonic_cli_gen/generator.py +++ b/sonic_cli_gen/generator.py @@ -9,19 +9,25 @@ class CliGenerator: """ SONiC CLI generator. This class provides public API for sonic-cli-gen python library. It can generate config, - show CLI plugins + show CLI plugins. + + Attributes: + loader: the loaded j2 templates + env: j2 central object + logger: logger """ - def __init__(self): + def __init__(self, logger): """ Initialize CliGenerator. """ self.loader = jinja2.FileSystemLoader(['/usr/share/sonic/templates/sonic-cli-gen/']) self.env = jinja2.Environment(loader=self.loader) + self.logger = logger def generate_cli_plugin(self, cli_group, plugin_name): """ Generate click CLI plugin and put it to: - /usr/local/lib/python3.7/dist-packages//plugins/auto/ + /usr/local/lib//dist-packages//plugins/auto/ """ parser = YangParser(yang_model_name=plugin_name, @@ -34,19 +40,19 @@ def generate_cli_plugin(self, cli_group, plugin_name): template = self.env.get_template(cli_group + '.py.j2') with open(plugin_path, 'w') as plugin_py: plugin_py.write(template.render(yang_dict)) - print('\nAuto-generation successful!\nLocation: {}'.format(plugin_path)) + self.logger.info(' Auto-generation successful! Location: {}'.format(plugin_path)) def remove_cli_plugin(self, cli_group, plugin_name): """ Remove CLI plugin from directory: - /usr/local/lib/python3.7/dist-packages//plugins/auto/ + /usr/local/lib//dist-packages//plugins/auto/ """ plugin_path = get_cli_plugin_path(cli_group, plugin_name + '_yang.py') if os.path.exists(plugin_path): os.remove(plugin_path) - print('{} was removed.'.format(plugin_path)) + self.logger.info(' {} was removed.'.format(plugin_path)) else: - print('Path {} doest NOT exist!'.format(plugin_path)) + self.logger.warning(' Path {} doest NOT exist!'.format(plugin_path)) def get_cli_plugin_path(command, plugin_name): diff --git a/sonic_cli_gen/main.py b/sonic_cli_gen/main.py index f23249d354..8f4de9a213 100644 --- a/sonic_cli_gen/main.py +++ b/sonic_cli_gen/main.py @@ -1,8 +1,13 @@ #!/usr/bin/env python +import sys import click +import logging from sonic_cli_gen.generator import CliGenerator +logger = logging.getLogger('sonic-cli-gen') +logging.basicConfig(stream=sys.stdout, level=logging.INFO) + @click.group() @click.pass_context def cli(ctx): @@ -15,7 +20,7 @@ def cli(ctx): """ context = { - 'gen': CliGenerator() + 'gen': CliGenerator(logger) } ctx.obj = context From 1584fe3d767d00d9158d735f97dd2e277c241d77 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Tue, 25 May 2021 08:16:10 +0000 Subject: [PATCH 50/76] pep8 for generator.py Signed-off-by: Vadym Hlushko --- sonic_cli_gen/generator.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py index 5a4f1be93d..4f48b0201a 100644 --- a/sonic_cli_gen/generator.py +++ b/sonic_cli_gen/generator.py @@ -6,6 +6,9 @@ from sonic_cli_gen.yang_parser import YangParser +templates_path = '/usr/share/sonic/templates/sonic-cli-gen/' + + class CliGenerator: """ SONiC CLI generator. This class provides public API for sonic-cli-gen python library. It can generate config, @@ -20,11 +23,10 @@ class CliGenerator: def __init__(self, logger): """ Initialize CliGenerator. """ - self.loader = jinja2.FileSystemLoader(['/usr/share/sonic/templates/sonic-cli-gen/']) + self.loader = jinja2.FileSystemLoader(templates_path) self.env = jinja2.Environment(loader=self.loader) self.logger = logger - def generate_cli_plugin(self, cli_group, plugin_name): """ Generate click CLI plugin and put it to: /usr/local/lib//dist-packages//plugins/auto/ @@ -34,7 +36,8 @@ def generate_cli_plugin(self, cli_group, plugin_name): config_db_path='configDB', allow_tbl_without_yang=True, debug=False) - # yang_dict will be used as an input for templates located in - /usr/share/sonic/templates/sonic-cli-gen/ + # yang_dict will be used as an input for templates located in + # /usr/share/sonic/templates/sonic-cli-gen/ yang_dict = parser.parse_yang_model() plugin_path = get_cli_plugin_path(cli_group, plugin_name + '_yang.py') template = self.env.get_template(cli_group + '.py.j2') @@ -42,7 +45,6 @@ def generate_cli_plugin(self, cli_group, plugin_name): plugin_py.write(template.render(yang_dict)) self.logger.info(' Auto-generation successful! Location: {}'.format(plugin_path)) - def remove_cli_plugin(self, cli_group, plugin_name): """ Remove CLI plugin from directory: /usr/local/lib//dist-packages//plugins/auto/ @@ -52,7 +54,7 @@ def remove_cli_plugin(self, cli_group, plugin_name): os.remove(plugin_path) self.logger.info(' {} was removed.'.format(plugin_path)) else: - self.logger.warning(' Path {} doest NOT exist!'.format(plugin_path)) + self.logger.info(' Path {} doest NOT exist!'.format(plugin_path)) def get_cli_plugin_path(command, plugin_name): From 8f27d03c92bd44573c98313f0e99a9d6c7a38ed9 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Tue, 25 May 2021 08:57:25 +0000 Subject: [PATCH 51/76] pep8 for the rest of files Signed-off-by: Vadym Hlushko --- sonic_cli_gen/main.py | 13 +- sonic_cli_gen/yang_parser.py | 244 ++++++++++++++++++++++------------- 2 files changed, 164 insertions(+), 93 deletions(-) diff --git a/sonic_cli_gen/main.py b/sonic_cli_gen/main.py index 8f4de9a213..bfcd301aed 100644 --- a/sonic_cli_gen/main.py +++ b/sonic_cli_gen/main.py @@ -8,6 +8,7 @@ logger = logging.getLogger('sonic-cli-gen') logging.basicConfig(stream=sys.stdout, level=logging.INFO) + @click.group() @click.pass_context def cli(ctx): @@ -26,23 +27,23 @@ def cli(ctx): @cli.command() -@click.argument('cli_group', type = click.Choice(['config', 'show'])) -@click.argument('yang_model_name', type = click.STRING) +@click.argument('cli_group', type=click.Choice(['config', 'show'])) +@click.argument('yang_model_name', type=click.STRING) @click.pass_context def generate(ctx, cli_group, yang_model_name): """ Generate click CLI plugin. """ - ctx.obj['gen'].generate_cli_plugin(cli_group = cli_group, plugin_name = yang_model_name) + ctx.obj['gen'].generate_cli_plugin(cli_group, yang_model_name) @cli.command() -@click.argument('cli_group', type = click.Choice(['config', 'show'])) -@click.argument('yang_model_name', type = click.STRING) +@click.argument('cli_group', type=click.Choice(['config', 'show'])) +@click.argument('yang_model_name', type=click.STRING) @click.pass_context def remove(ctx, cli_group, yang_model_name): """ Remove generated click CLI plugin from. """ - ctx.obj['gen'].remove_cli_plugin(cli_group = cli_group, plugin_name = yang_model_name) + ctx.obj['gen'].remove_cli_plugin(cli_group, yang_model_name) if __name__ == '__main__': diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index 0fdbb7fceb..23e420987c 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -2,23 +2,28 @@ from collections import OrderedDict from config.config_mgmt import ConfigMgmt - from typing import List, Dict yang_guidelines_link = 'https://github.com/Azure/SONiC/blob/master/doc/mgmt/SONiC_YANG_Model_Guidelines.md' + class YangParser: """ YANG model parser Attributes: yang_model_name: Name of the YANG model file - conf_mgmt: Instance of Config Mgmt class to help parse YANG models - y_module: Reference to 'module' entity from YANG model file - y_top_level_container: Reference to top level 'container' entity from YANG model file - y_table_containers: Reference to 'container' entities from YANG model file - that represent Config DB tables - yang_2_dict: dictionary created from YANG model file that represent Config DB schema. - In case if YANG model has a 'list' entity: + conf_mgmt: Instance of Config Mgmt class to + help parse YANG models + y_module: Reference to 'module' entity + from YANG model file + y_top_level_container: Reference to top level 'container' + entity from YANG model file + y_table_containers: Reference to 'container' entities + from YANG model file that represent Config DB tables + yang_2_dict: dictionary created from YANG model file that + represent Config DB schema. + + Below the 'yang_2_dict' obj in case if YANG model has a 'list' entity: { 'tables': [{ 'name': 'value', @@ -46,7 +51,9 @@ class YangParser: ], }] } - In case if YANG model does NOT have a 'list' entity, it has the same structure as above, but 'dynamic-objects' changed to 'static-objects' and have no 'keys' + In case if YANG model does NOT have a 'list' entity, + it has the same structure as above, but 'dynamic-objects' + changed to 'static-objects' and have no 'keys' """ def __init__(self, @@ -62,12 +69,12 @@ def __init__(self, self.yang_2_dict = dict() try: - self.conf_mgmt = ConfigMgmt(source=config_db_path, - debug=debug, - allowTablesWithoutYang=allow_tbl_without_yang) + self.conf_mgmt = ConfigMgmt(config_db_path, + debug, + allow_tbl_without_yang) except Exception as e: raise Exception("Failed to load the {} class".format(str(e))) - + def _init_yang_module_and_containers(self): """ Initialize inner class variables: self.y_module @@ -84,18 +91,23 @@ def _init_yang_module_and_containers(self): raise Exception('The YANG model {} is NOT exist'.format(self.yang_model_name)) if self.y_module.get('container') is None: - raise Exception('The YANG model {} does NOT have "top level container" element\ - Please follow the SONiC YANG model guidelines:\n{}'.format(self.yang_model_name, yang_guidelines_link)) + raise Exception('The YANG model {} does NOT have\ + "top level container" element\ + Please follow the SONiC YANG model guidelines:\ + \n{}'.format(self.yang_model_name, yang_guidelines_link)) self.y_top_level_container = self.y_module.get('container') if self.y_top_level_container.get('container') is None: - raise Exception('The YANG model {} does NOT have "container" element after "top level container"\ - Please follow the SONiC YANG model guidelines:\n{}'.format(self.yang_model_name, yang_guidelines_link)) + raise Exception('The YANG model {} does NOT have "container"\ + element after "top level container"\ + Please follow the SONiC YANG model guidelines:\ + \n{}'.format(self.yang_model_name, yang_guidelines_link)) self.y_table_containers = self.y_top_level_container.get('container') def _find_yang_model_in_yjson_obj(self) -> OrderedDict: """ Find provided YANG model inside the yJson object, - the yJson object contain all yang-models parsed from directory - /usr/local/yang-models + the yJson object contain all yang-models + parsed from directory - /usr/local/yang-models Returns: reference to yang_model_name @@ -106,7 +118,8 @@ def _find_yang_model_in_yjson_obj(self) -> OrderedDict: return yang_model.get('module') def parse_yang_model(self) -> dict: - """ Parse provided YANG model and save the output to self.yang_2_dict object + """ Parse provided YANG model and save + the output to self.yang_2_dict object Returns: parsed YANG model in dictionary format @@ -115,18 +128,21 @@ def parse_yang_model(self) -> dict: self._init_yang_module_and_containers() self.yang_2_dict['tables'] = list() - # determine how many (1 or more) containers a YANG model have after the 'top level' container + # determine how many (1 or more) containers a YANG model + # has after the 'top level' container # 'table' container goes after the 'top level' container self.yang_2_dict['tables'] += list_handler(self.y_table_containers, - lambda e: on_table_container(self.y_module, e, self.conf_mgmt)) + lambda e: on_table_container(self.y_module, e, self.conf_mgmt)) return self.yang_2_dict -#------------------------------HANDLERS--------------------------------# +# ------------------------------HANDLERS-------------------------------- # def list_handler(y_entity, callback) -> List[Dict]: - """ Determine if the type of entity is a list, if so - call the callback for every list element """ + """ Determine if the type of entity is a list, + if so - call the callback for every list element + """ if isinstance(y_entity, list): return [callback(e) for e in y_entity] @@ -134,7 +150,9 @@ def list_handler(y_entity, callback) -> List[Dict]: return [callback(y_entity)] -def on_table_container(y_module: OrderedDict, tbl_container: OrderedDict, conf_mgmt: ConfigMgmt) -> dict: +def on_table_container(y_module: OrderedDict, + tbl_container: OrderedDict, + conf_mgmt: ConfigMgmt) -> dict: """ Parse 'table' container, 'table' container goes after 'top level' container @@ -144,7 +162,7 @@ def on_table_container(y_module: OrderedDict, tbl_container: OrderedDict, conf_m conf_mgmt: reference to ConfigMgmt class instance, it have yJson object which contain all parsed YANG models Returns: - element for self.yang_2_dict['tables'] + element for self.yang_2_dict['tables'] """ y2d_elem = { 'name': tbl_container.get('@name'), @@ -156,15 +174,16 @@ def on_table_container(y_module: OrderedDict, tbl_container: OrderedDict, conf_m y2d_elem['static-objects'] = list() # 'object' container goes after the 'table' container - # 'object' container have 2 types - list (like sonic-flex_counter.yang) and NOT list (like sonic-device_metadata.yang) + # 'object' container have 2 types - list (like sonic-flex_counter.yang) + # and NOT list (like sonic-device_metadata.yang) y2d_elem['static-objects'] += list_handler(tbl_container.get('container'), - lambda e: on_object_entity(y_module, e, conf_mgmt, is_list = False)) + lambda e: on_object_entity(y_module, e, conf_mgmt, is_list=False)) else: y2d_elem['dynamic-objects'] = list() # 'container' can have more than 1 'list' entity y2d_elem['dynamic-objects'] += list_handler(tbl_container.get('list'), - lambda e: on_object_entity(y_module, e, conf_mgmt, is_list = True)) + lambda e: on_object_entity(y_module, e, conf_mgmt, is_list=True)) # move 'keys' elements from 'attrs' to 'keys' change_dyn_obj_struct(y2d_elem['dynamic-objects']) @@ -172,7 +191,10 @@ def on_table_container(y_module: OrderedDict, tbl_container: OrderedDict, conf_m return y2d_elem -def on_object_entity(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: ConfigMgmt, is_list: bool) -> dict: +def on_object_entity(y_module: OrderedDict, + y_entity: OrderedDict, + conf_mgmt: ConfigMgmt, + is_list: bool) -> dict: """ Parse a 'object' entity, it could be a 'container' or a 'list' 'Object' entity represent OBJECT inside Config DB schema: { @@ -207,9 +229,9 @@ def on_object_entity(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: Co attrs_list = list() # grouping_name is empty because 'grouping' is not used so far - attrs_list.extend(get_leafs(y_entity, grouping_name = '')) - attrs_list.extend(get_leaf_lists(y_entity, grouping_name = '')) - attrs_list.extend(get_choices(y_module, y_entity, conf_mgmt, grouping_name = '')) + attrs_list.extend(get_leafs(y_entity, grouping_name='')) + attrs_list.extend(get_leaf_lists(y_entity, grouping_name='')) + attrs_list.extend(get_choices(y_module, y_entity, conf_mgmt, grouping_name='')) attrs_list.extend(get_uses(y_module, y_entity, conf_mgmt)) obj_elem['attrs'] = attrs_list @@ -217,7 +239,9 @@ def on_object_entity(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: Co return obj_elem -def on_uses(y_module: OrderedDict, y_uses, conf_mgmt: ConfigMgmt) -> list: +def on_uses(y_module: OrderedDict, + y_uses, + conf_mgmt: ConfigMgmt) -> list: """ Parse a YANG 'uses' entities 'uses' referring to 'grouping' YANG entity @@ -256,7 +280,10 @@ def on_uses(y_module: OrderedDict, y_uses, conf_mgmt: ConfigMgmt) -> list: return ret_attrs -def on_choices(y_module: OrderedDict, y_choices, conf_mgmt: ConfigMgmt, grouping_name: str) -> list: +def on_choices(y_module: OrderedDict, + y_choices, + conf_mgmt: ConfigMgmt, + grouping_name: str) -> list: """ Parse a YANG 'choice' entities Args: @@ -271,18 +298,24 @@ def on_choices(y_module: OrderedDict, y_choices, conf_mgmt: ConfigMgmt, grouping ret_attrs = list() - # the YANG model can have multiple 'choice' entities inside a 'container' or 'list' + # the YANG model can have multiple 'choice' entities + # inside a 'container' or 'list' if isinstance(y_choices, list): for choice in y_choices: - attrs = on_choice_cases(y_module, choice.get('case'), conf_mgmt, grouping_name) + attrs = on_choice_cases(y_module, choice.get('case'), + conf_mgmt, grouping_name) ret_attrs.extend(attrs) else: - ret_attrs = on_choice_cases(y_module, y_choices.get('case'), conf_mgmt, grouping_name) + ret_attrs = on_choice_cases(y_module, y_choices.get('case'), + conf_mgmt, grouping_name) return ret_attrs -def on_choice_cases(y_module: OrderedDict, y_cases, conf_mgmt: ConfigMgmt, grouping_name: str) -> list: +def on_choice_cases(y_module: OrderedDict, + y_cases, + conf_mgmt: ConfigMgmt, + grouping_name: str) -> list: """ Parse a single YANG 'case' entity from the 'choice' entity. The 'case' element can has inside - 'leaf', 'leaf-list', 'uses' @@ -290,10 +323,13 @@ def on_choice_cases(y_module: OrderedDict, y_cases, conf_mgmt: ConfigMgmt, group y_module: reference to 'module' y_cases: reference to 'case' conf_mgmt: reference to ConfigMgmt class instance, - it have yJson object which contain all parsed YANG model - grouping_name: if YANG entity contain 'uses', this argument represent 'grouping' name + it have yJson object which contain all + parsed YANG model + grouping_name: if YANG entity contain 'uses', + this argument represent 'grouping' name Returns: - element for the obj_elem['attrs'], the 'attrs' contain a parsed 'leafs' + element for the obj_elem['attrs'], the 'attrs' + contain a parsed 'leafs' """ ret_attrs = list() @@ -305,49 +341,58 @@ def on_choice_cases(y_module: OrderedDict, y_cases, conf_mgmt: ConfigMgmt, group ret_attrs.extend(get_uses(y_module, case, conf_mgmt)) else: raise Exception('It has no sense to using a single "case" element inside "choice" element') - + return ret_attrs -def on_leafs(y_leafs, grouping_name, is_leaf_list: bool) -> list: +def on_leafs(y_leafs, + grouping_name: str, + is_leaf_list: bool) -> list: """ Parse all the 'leaf' or 'leaf-list' elements Args: y_leafs: reference to all 'leaf' elements - grouping_name: if YANG entity contain 'uses', this argument represent the 'grouping' name - is_leaf_list: boolean to determine if a 'leaf-list' was passed as 'y_leafs' argument + grouping_name: if YANG entity contain 'uses', + this argument represent the 'grouping' name + is_leaf_list: boolean to determine if a 'leaf-list' + was passed as 'y_leafs' argument Returns: - list of parsed 'leaf' elements + list of parsed 'leaf' elements """ ret_attrs = list() - # The YANG 'container' entity may have only 1 'leaf' element OR a list of 'leaf' elements + # The YANG 'container' entity may have only 1 'leaf' + # element OR a list of 'leaf' elements ret_attrs += list_handler(y_leafs, lambda e: on_leaf(e, grouping_name, is_leaf_list)) return ret_attrs -def on_leaf(leaf: OrderedDict, grouping_name: str, is_leaf_list: bool) -> dict: +def on_leaf(leaf: OrderedDict, + grouping_name: str, + is_leaf_list: bool) -> dict: """ Parse a single 'leaf' element Args: leaf: reference to a 'leaf' entity - grouping_name: if YANG entity contain 'uses', this argument represent 'grouping' name - is_leaf_list: boolean to determine if 'leaf-list' was passed in 'y_leafs' argument + grouping_name: if YANG entity contain 'uses', + this argument represent 'grouping' name + is_leaf_list: boolean to determine if 'leaf-list' + was passed in 'y_leafs' argument Returns: parsed 'leaf' element """ - attr = { 'name': leaf.get('@name'), - 'description': get_description(leaf), - 'is-leaf-list': is_leaf_list, - 'is-mandatory': get_mandatory(leaf), - 'group': grouping_name} + attr = {'name': leaf.get('@name'), + 'description': get_description(leaf), + 'is-leaf-list': is_leaf_list, + 'is-mandatory': get_mandatory(leaf), + 'group': grouping_name} return attr -#----------------------GETERS-------------------------# +# ----------------------GETERS------------------------- # def get_mandatory(y_leaf: OrderedDict) -> bool: """ Parse the 'mandatory' statement for a 'leaf' @@ -379,14 +424,17 @@ def get_description(y_entity: OrderedDict) -> str: return '' -def get_leafs(y_entity: OrderedDict, grouping_name: str) -> list: +def get_leafs(y_entity: OrderedDict, + grouping_name: str) -> list: """ Check if the YANG entity have 'leafs', if so call handler Args: - y_entity: reference YANG 'container' or 'list' or 'choice' or 'uses' - grouping_name: if YANG entity contain 'uses', this argument represent 'grouping' name + y_entity: reference YANG 'container' or 'list' + or 'choice' or 'uses' + grouping_name: if YANG entity contain 'uses', + this argument represent 'grouping' name Returns: - list of parsed 'leaf' elements + list of parsed 'leaf' elements """ if y_entity.get('leaf') is not None: @@ -395,14 +443,17 @@ def get_leafs(y_entity: OrderedDict, grouping_name: str) -> list: return [] -def get_leaf_lists(y_entity: OrderedDict, grouping_name: str) -> list: +def get_leaf_lists(y_entity: OrderedDict, + grouping_name: str) -> list: """ Check if the YANG entity have 'leaf-list', if so call handler Args: - y_entity: reference YANG 'container' or 'list' or 'choice' or 'uses' - grouping_name: if YANG entity contain 'uses', this argument represent 'grouping' name + y_entity: reference YANG 'container' or 'list' + or 'choice' or 'uses' + grouping_name: if YANG entity contain 'uses', + this argument represent 'grouping' name Returns: - list of parsed 'leaf-list' elements + list of parsed 'leaf-list' elements """ if y_entity.get('leaf-list') is not None: @@ -411,15 +462,20 @@ def get_leaf_lists(y_entity: OrderedDict, grouping_name: str) -> list: return [] -def get_choices(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: ConfigMgmt, grouping_name: str) -> list: +def get_choices(y_module: OrderedDict, + y_entity: OrderedDict, + conf_mgmt: ConfigMgmt, + grouping_name: str) -> list: """ Check if the YANG entity have 'choice', if so call handler Args: y_module: reference to 'module' - y_entity: reference YANG 'container' or 'list' or 'choice' or 'uses' + y_entity: reference YANG 'container' or 'list' + or 'choice' or 'uses' conf_mgmt: reference to ConfigMgmt class instance, - it have yJson object which contain all parsed YANG model - grouping_name: if YANG entity contain 'uses', this argument represent 'grouping' name + it have yJson object which contain all parsed YANG model + grouping_name: if YANG entity contain 'uses', + this argument represent 'grouping' name Returns: list of parsed elements inside 'choice' """ @@ -430,16 +486,20 @@ def get_choices(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: ConfigM return [] -def get_uses(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: ConfigMgmt) -> list: +def get_uses(y_module: OrderedDict, + y_entity: OrderedDict, + conf_mgmt: ConfigMgmt) -> list: """ Check if the YANG entity have 'uses', if so call handler Args: y_module: reference to 'module' - y_entity: reference YANG 'container' or 'list' or 'choice' or 'uses' + y_entity: reference YANG 'container' or 'list' + or 'choice' or 'uses' conf_mgmt: reference to ConfigMgmt class instance, - it have yJson object which contain all parsed YANG model + it have yJson object which contain all parsed YANG model Returns: - list of parsed elements inside 'grouping' that referenced by 'uses' + list of parsed elements inside 'grouping' + that referenced by 'uses' """ if y_entity.get('uses') is not None: @@ -448,14 +508,17 @@ def get_uses(y_module: OrderedDict, y_entity: OrderedDict, conf_mgmt: ConfigMgmt return [] -def get_all_grouping(y_module: OrderedDict, y_uses: OrderedDict, conf_mgmt: ConfigMgmt) -> list: - """ Get all the 'grouping' entities that was referenced by 'uses' in current YANG model +def get_all_grouping(y_module: OrderedDict, + y_uses: OrderedDict, + conf_mgmt: ConfigMgmt) -> list: + """ Get all the 'grouping' entities that was referenced + by 'uses' in current YANG model Args: y_module: reference to 'module' y_entity: reference to 'uses' conf_mgmt: reference to ConfigMgmt class instance, - it have yJson object which contain all parsed YANG model + it have yJson object which contain all parsed YANG model Returns: list of 'grouping' elements """ @@ -472,7 +535,8 @@ def get_all_grouping(y_module: OrderedDict, y_uses: OrderedDict, conf_mgmt: Conf else: ret_grouping.append(local_grouping) - # if prefix_list is NOT empty it means that 'grouping' was imported from another YANG model + # if prefix_list is NOT empty it means that 'grouping' + # was imported from another YANG model if prefix_list != []: for prefix in prefix_list: y_import = y_module.get('import') @@ -487,13 +551,14 @@ def get_all_grouping(y_module: OrderedDict, y_uses: OrderedDict, conf_mgmt: Conf return ret_grouping -def get_grouping_from_another_yang_model(yang_model_name: str, conf_mgmt) -> list: +def get_grouping_from_another_yang_model(yang_model_name: str, + conf_mgmt) -> list: """ Get the YANG 'grouping' entity Args: yang_model_name - YANG model to search conf_mgmt - reference to ConfigMgmt class instance, - it have yJson object which contain all parsed YANG models + it have yJson object which contain all parsed YANG models Returns: list of 'grouping' entities @@ -516,7 +581,7 @@ def get_import_prefixes(y_uses: OrderedDict) -> list: """ Parse 'import prefix' of YANG 'uses' entity Example: { - uses stypes:endpoint; + uses stypes:endpoint; } 'stypes' - prefix of imported YANG module. 'endpoint' - YANG 'grouping' entity name @@ -544,7 +609,8 @@ def get_import_prefixes(y_uses: OrderedDict) -> list: def trim_uses_prefixes(y_uses) -> list: """ Trim prefixes from the 'uses' YANG entities. - If the YANG 'grouping' was imported from another YANG file, it use the 'prefix' before the 'grouping' name: + If the YANG 'grouping' was imported from another + YANG file, it use the 'prefix' before the 'grouping' name: { uses sgrop:endpoint; } @@ -571,8 +637,9 @@ def trim_uses_prefixes(y_uses) -> list: def get_list_keys(y_list: OrderedDict) -> list: """ Parse YANG the 'key' entity. - If YANG model has a 'list' entity, inside the 'list' there is 'key' entity. - 'key' - whitespace separeted list of 'leafs' + If YANG model has a 'list' entity, inside the 'list' + there is 'key' entity. The 'key' - whitespace + separeted list of 'leafs' Args: y_list: reference to the 'list' @@ -584,7 +651,7 @@ def get_list_keys(y_list: OrderedDict) -> list: keys = y_list.get('key').get('@value').split() for k in keys: - key = { 'name': k } + key = {'name': k} ret_list.append(key) return ret_list @@ -592,10 +659,13 @@ def get_list_keys(y_list: OrderedDict) -> list: def change_dyn_obj_struct(dynamic_objects: list): """ Rearrange self.yang_2_dict['dynamic_objects'] structure. - If YANG model have a 'list' entity - inside the 'list' it has 'key' entity. - 'key' entity it is whitespace-separeted list of 'leafs', those 'leafs' was - parsed by 'on_leaf()' function and placed under 'attrs' in self.yang_2_dict['dynamic_objects'] - need to move 'leafs' from 'attrs' and put them to 'keys' section of elf.yang_2_dict['dynamic_objects'] + If YANG model have a 'list' entity - inside the 'list' + it has 'key' entity. The 'key' entity it is whitespace + separeted list of 'leafs', those 'leafs' was parsed by + 'on_leaf()' function and placed under 'attrs' in + self.yang_2_dict['dynamic_objects'] need to move 'leafs' + from 'attrs' and put them into 'keys' section of + self.yang_2_dict['dynamic_objects'] Args: dynamic_objects: reference to self.yang_2_dict['dynamic_objects'] From aa6f806f7294f7de46584a4a49a8466fbdda43df Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Tue, 25 May 2021 09:24:09 +0000 Subject: [PATCH 52/76] pep8 for tests files, added comments to YANG models Signed-off-by: Vadym Hlushko --- sonic_cli_gen/yang_parser.py | 2 +- tests/cli_autogen_input/sonic-1-list.yang | 3 + .../sonic-1-object-container.yang | 3 + .../sonic-1-table-container.yang | 2 + tests/cli_autogen_input/sonic-2-lists.yang | 5 + .../sonic-2-object-containers.yang | 4 + .../sonic-2-table-containers.yang | 3 + tests/cli_autogen_yang_parser_test.py | 114 ++++++++++++++---- 8 files changed, 111 insertions(+), 25 deletions(-) diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index 23e420987c..f0c737802b 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -196,7 +196,7 @@ def on_object_entity(y_module: OrderedDict, conf_mgmt: ConfigMgmt, is_list: bool) -> dict: """ Parse a 'object' entity, it could be a 'container' or a 'list' - 'Object' entity represent OBJECT inside Config DB schema: + 'Object' entity represent OBJECT in Config DB schema: { "TABLE": { "OBJECT": { diff --git a/tests/cli_autogen_input/sonic-1-list.yang b/tests/cli_autogen_input/sonic-1-list.yang index c7fc4ee824..79a6529b3d 100644 --- a/tests/cli_autogen_input/sonic-1-list.yang +++ b/tests/cli_autogen_input/sonic-1-list.yang @@ -6,12 +6,15 @@ module sonic-1-list { prefix s-1-list; container sonic-1-list { + /* sonic-1-list - top level container */ container TABLE_1 { + /* TABLE_1 - table container */ description "TABLE_1 description"; list TABLE_1_LIST { + /* TABLE_1 - object container */ description "TABLE_1_LIST description"; diff --git a/tests/cli_autogen_input/sonic-1-object-container.yang b/tests/cli_autogen_input/sonic-1-object-container.yang index d52b2a8caf..e28ef7f90a 100644 --- a/tests/cli_autogen_input/sonic-1-object-container.yang +++ b/tests/cli_autogen_input/sonic-1-object-container.yang @@ -6,12 +6,15 @@ module sonic-1-object-container { prefix s-1-object; container sonic-1-object-container { + /* sonic-1-object-container - top level container */ container TABLE_1 { + /* TABLE_1 - table container */ description "TABLE_1 description"; container OBJECT_1 { + /* OBJECT_1 - object container */ description "OBJECT_1 description"; } diff --git a/tests/cli_autogen_input/sonic-1-table-container.yang b/tests/cli_autogen_input/sonic-1-table-container.yang index 8963148158..58e7293c0d 100644 --- a/tests/cli_autogen_input/sonic-1-table-container.yang +++ b/tests/cli_autogen_input/sonic-1-table-container.yang @@ -6,8 +6,10 @@ module sonic-1-table-container { prefix s-1-table; container sonic-1-table-container { + /* sonic-1-table-container - top level container */ container TABLE_1 { + /* TABLE_1 - table container */ description "TABLE_1 description"; } diff --git a/tests/cli_autogen_input/sonic-2-lists.yang b/tests/cli_autogen_input/sonic-2-lists.yang index 2a4cd42fd9..b20200415b 100644 --- a/tests/cli_autogen_input/sonic-2-lists.yang +++ b/tests/cli_autogen_input/sonic-2-lists.yang @@ -6,12 +6,16 @@ module sonic-2-lists { prefix s-2-lists; container sonic-2-lists { + /* sonic-2-lists - top level container */ container TABLE_1 { + /* TALBE_1 - table container */ + description "TABLE_1 description"; list TABLE_1_LIST_1 { + /* TALBE_1_LIST_1 - object container */ description "TABLE_1_LIST_1 description"; @@ -23,6 +27,7 @@ module sonic-2-lists { } list TABLE_1_LIST_2 { + /* TALBE_1_LIST_2 - object container */ description "TABLE_1_LIST_2 description"; diff --git a/tests/cli_autogen_input/sonic-2-object-containers.yang b/tests/cli_autogen_input/sonic-2-object-containers.yang index 1aaaeb1a19..249faf4c89 100644 --- a/tests/cli_autogen_input/sonic-2-object-containers.yang +++ b/tests/cli_autogen_input/sonic-2-object-containers.yang @@ -6,17 +6,21 @@ module sonic-2-object-containers { prefix s-2-object; container sonic-2-object-containers { + /* sonic-2-object-containers - top level container */ container TABLE_1 { + /* TABLE_1 - table container */ description "FIRST_TABLE description"; container OBJECT_1 { + /* OBJECT_1 - object container */ description "OBJECT_1 description"; } container OBJECT_2 { + /* OBJECT_2 - object container */ description "OBJECT_2 description"; } diff --git a/tests/cli_autogen_input/sonic-2-table-containers.yang b/tests/cli_autogen_input/sonic-2-table-containers.yang index a3f13474b5..393512a313 100644 --- a/tests/cli_autogen_input/sonic-2-table-containers.yang +++ b/tests/cli_autogen_input/sonic-2-table-containers.yang @@ -6,13 +6,16 @@ module sonic-2-table-containers { prefix s-2-table; container sonic-2-table-containers { + /* sonic-2-table-containers - top level container */ container TABLE_1 { + /* TABLE_1 - table container */ description "TABLE_1 description"; } container TABLE_2 { + /* TABLE_2 - table container */ description "TABLE_2 description"; } diff --git a/tests/cli_autogen_yang_parser_test.py b/tests/cli_autogen_yang_parser_test.py index 67898503af..023b9fa6e1 100644 --- a/tests/cli_autogen_yang_parser_test.py +++ b/tests/cli_autogen_yang_parser_test.py @@ -26,6 +26,7 @@ 'sonic-grouping-2', ] + class TestYangParser: @classmethod def setup_class(cls): @@ -37,89 +38,154 @@ def setup_class(cls): def teardown_class(cls): logger.info("TEARDOWN") os.environ['UTILITIES_UNIT_TESTING'] = "0" - remove_yang_models_to_well_know_location() + remove_yang_models_from_well_know_location() def test_1_table_container(self): - template('sonic-1-table-container', assert_dictionaries.one_table_container) - + """ Test for 1 'table' container + 'table' container represent TABLE in Config DB schema: + { + "TABLE": { + "OBJECT": { + "attr": "value" + ... + } + } + } + """ + + template('sonic-1-table-container', + assert_dictionaries.one_table_container) + def test_2_table_containers(self): - template('sonic-2-table-containers', assert_dictionaries.two_table_containers) + """ Test for 2 'table' containers """ + + template('sonic-2-table-containers', + assert_dictionaries.two_table_containers) def test_1_object_container(self): - template('sonic-1-object-container', assert_dictionaries.one_object_container) + """ Test for 1 'object' container + 'object' container represent OBJECT in Config DB schema: + { + "TABLE": { + "OBJECT": { + "attr": "value" + ... + } + } + } + """ + + template('sonic-1-object-container', + assert_dictionaries.one_object_container) def test_2_object_containers(self): - template('sonic-2-object-containers', assert_dictionaries.two_object_containers) + """ Test for 2 'object' containers """ + + template('sonic-2-object-containers', + assert_dictionaries.two_object_containers) def test_1_list(self): + """ Test for 1 container that has inside + the YANG 'list' entity + """ + template('sonic-1-list', assert_dictionaries.one_list) def test_2_lists(self): + """ Test for 2 containers that have inside + the YANG 'list' entity + """ + template('sonic-2-lists', assert_dictionaries.two_lists) def test_static_object_complex_1(self): - """ Test object container with: 1 leaf, 1 leaf-list, 1 choice. + """ Test for the object container with: + 1 leaf, 1 leaf-list, 1 choice. """ - template('sonic-static-object-complex-1', assert_dictionaries.static_object_complex_1) + + template('sonic-static-object-complex-1', + assert_dictionaries.static_object_complex_1) def test_static_object_complex_2(self): - """ Test object container with: 2 leafs, 2 leaf-lists, 2 choices. + """ Test for object container with: + 2 leafs, 2 leaf-lists, 2 choices. """ - template('sonic-static-object-complex-2', assert_dictionaries.static_object_complex_2) + + template('sonic-static-object-complex-2', + assert_dictionaries.static_object_complex_2) def test_dynamic_object_complex_1(self): - """ Test object container with: 1 key, 1 leaf, 1 leaf-list, 1 choice. + """ Test object container with: + 1 key, 1 leaf, 1 leaf-list, 1 choice. """ - template('sonic-dynamic-object-complex-1', assert_dictionaries.dynamic_object_complex_1) + + template('sonic-dynamic-object-complex-1', + assert_dictionaries.dynamic_object_complex_1) def test_dynamic_object_complex_2(self): - """ Test object container with: 2 keys, 2 leafs, 2 leaf-list, 2 choice. + """ Test object container with: + 2 keys, 2 leafs, 2 leaf-list, 2 choice. """ - template('sonic-dynamic-object-complex-2', assert_dictionaries.dynamic_object_complex_2) + + template('sonic-dynamic-object-complex-2', + assert_dictionaries.dynamic_object_complex_2) def test_choice_complex(self): """ Test object container with choice that have complex strucutre: leafs, leaf-lists, multiple 'uses' from different files """ - template('sonic-choice-complex', assert_dictionaries.choice_complex) + + template('sonic-choice-complex', + assert_dictionaries.choice_complex) def test_grouping_complex(self): """ Test object container with muplitple 'uses' that using 'grouping' from different files. The used 'grouping' have a complex strucutre: leafs, leaf-lists, choices """ - template('sonic-grouping-complex', assert_dictionaries.grouping_complex) + + template('sonic-grouping-complex', + assert_dictionaries.grouping_complex) + def template(yang_model_name, correct_dict): - config_db_path = os.path.join(test_path, 'cli_autogen_input/config_db.json') - parser = YangParser(yang_model_name = yang_model_name, - config_db_path = config_db_path, - allow_tbl_without_yang = True, - debug = False) + config_db_path = os.path.join(test_path, + 'cli_autogen_input/config_db.json') + parser = YangParser(yang_model_name=yang_model_name, + config_db_path=config_db_path, + allow_tbl_without_yang=True, + debug=False) yang_dict = parser.parse_yang_model() pretty_log_debug(yang_dict) assert yang_dict == correct_dict + def move_yang_models_to_well_know_location(): """ Move a test YANG models to known location in order to be parsed by YangParser class """ for yang_model in test_yang_models: - src_path = os.path.join(test_path, 'cli_autogen_input', yang_model + '.yang') + src_path = os.path.join(test_path, + 'cli_autogen_input', + yang_model + '.yang') cmd = 'sudo cp {} {}'.format(src_path, yang_models_path) os.system(cmd) -def remove_yang_models_to_well_know_location(): + +def remove_yang_models_from_well_know_location(): """ Remove a test YANG models to known location in order to be parsed by YangParser class """ for yang_model in test_yang_models: - yang_model_path = os.path.join(yang_models_path, yang_model + '.yang') + yang_model_path = os.path.join(yang_models_path, + yang_model + '.yang') cmd = 'sudo rm {}'.format(yang_model_path) os.system(cmd) + def pretty_log_debug(dictionary): """ Pretty print of parsed dictionary """ for line in pprint.pformat(dictionary).split('\n'): logging.debug(line) + From 83ce9ee534a92713ee11a93e7ecf6f08d90f850d Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Tue, 25 May 2021 11:41:48 +0000 Subject: [PATCH 53/76] Added handler for 1 'choice' 'case' Signed-off-by: Vadym Hlushko --- sonic_cli_gen/yang_parser.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/sonic_cli_gen/yang_parser.py b/sonic_cli_gen/yang_parser.py index f0c737802b..df0382536f 100644 --- a/sonic_cli_gen/yang_parser.py +++ b/sonic_cli_gen/yang_parser.py @@ -259,10 +259,6 @@ def on_uses(y_module: OrderedDict, # trim prefixes in order to the next checks trim_uses_prefixes(y_uses) - # not sure if it can happend - if y_grouping == []: - raise Exception('Grouping NOT found') - # TODO: 'refine' support for group in y_grouping: if isinstance(y_uses, list): @@ -340,7 +336,9 @@ def on_choice_cases(y_module: OrderedDict, ret_attrs.extend(get_leaf_lists(case, grouping_name)) ret_attrs.extend(get_uses(y_module, case, conf_mgmt)) else: - raise Exception('It has no sense to using a single "case" element inside "choice" element') + ret_attrs.extend(get_leafs(y_cases, grouping_name)) + ret_attrs.extend(get_leaf_lists(y_cases, grouping_name)) + ret_attrs.extend(get_uses(y_module, y_cases, conf_mgmt)) return ret_attrs From 702be7427a732a9deaa8a7e6e0eb3748737c3730 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Tue, 25 May 2021 11:47:20 +0000 Subject: [PATCH 54/76] Code style Signed-off-by: Vadym Hlushko --- tests/cli_autogen_yang_parser_test.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/cli_autogen_yang_parser_test.py b/tests/cli_autogen_yang_parser_test.py index 023b9fa6e1..61c2dc98d9 100644 --- a/tests/cli_autogen_yang_parser_test.py +++ b/tests/cli_autogen_yang_parser_test.py @@ -115,7 +115,7 @@ def test_static_object_complex_2(self): assert_dictionaries.static_object_complex_2) def test_dynamic_object_complex_1(self): - """ Test object container with: + """ Test for object container with: 1 key, 1 leaf, 1 leaf-list, 1 choice. """ @@ -123,7 +123,7 @@ def test_dynamic_object_complex_1(self): assert_dictionaries.dynamic_object_complex_1) def test_dynamic_object_complex_2(self): - """ Test object container with: + """ Test for object container with: 2 keys, 2 leafs, 2 leaf-list, 2 choice. """ @@ -131,7 +131,8 @@ def test_dynamic_object_complex_2(self): assert_dictionaries.dynamic_object_complex_2) def test_choice_complex(self): - """ Test object container with choice that have complex strucutre: + """ Test for object container with the 'choice' + that have complex strucutre: leafs, leaf-lists, multiple 'uses' from different files """ @@ -139,8 +140,8 @@ def test_choice_complex(self): assert_dictionaries.choice_complex) def test_grouping_complex(self): - """ Test object container with muplitple 'uses' that using 'grouping' - from different files. The used 'grouping' have a complex strucutre: + """ Test for object container with multitple 'uses' that using 'grouping' + from different files. The used 'grouping' have a complex structure: leafs, leaf-lists, choices """ @@ -149,6 +150,8 @@ def test_grouping_complex(self): def template(yang_model_name, correct_dict): + """ General template for every test case """ + config_db_path = os.path.join(test_path, 'cli_autogen_input/config_db.json') parser = YangParser(yang_model_name=yang_model_name, @@ -164,6 +167,7 @@ def move_yang_models_to_well_know_location(): """ Move a test YANG models to known location in order to be parsed by YangParser class """ + for yang_model in test_yang_models: src_path = os.path.join(test_path, 'cli_autogen_input', @@ -176,6 +180,7 @@ def remove_yang_models_from_well_know_location(): """ Remove a test YANG models to known location in order to be parsed by YangParser class """ + for yang_model in test_yang_models: yang_model_path = os.path.join(yang_models_path, yang_model + '.yang') @@ -184,8 +189,8 @@ def remove_yang_models_from_well_know_location(): def pretty_log_debug(dictionary): - """ Pretty print of parsed dictionary - """ + """ Pretty print of parsed dictionary """ + for line in pprint.pformat(dictionary).split('\n'): logging.debug(line) From c433a4c7a231ad0486407daf5aad56be16c091df Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Wed, 26 May 2021 15:36:58 +0000 Subject: [PATCH 55/76] Fixed review comments Signed-off-by: Vadym Hlushko --- tests/cli_autogen_yang_parser_test.py | 36 +++++++++++++-------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/cli_autogen_yang_parser_test.py b/tests/cli_autogen_yang_parser_test.py index 61c2dc98d9..9ed915c69b 100644 --- a/tests/cli_autogen_yang_parser_test.py +++ b/tests/cli_autogen_yang_parser_test.py @@ -32,13 +32,13 @@ class TestYangParser: def setup_class(cls): logger.info("SETUP") os.environ['UTILITIES_UNIT_TESTING'] = "1" - move_yang_models_to_well_know_location() + move_yang_models() @classmethod def teardown_class(cls): logger.info("TEARDOWN") os.environ['UTILITIES_UNIT_TESTING'] = "0" - remove_yang_models_from_well_know_location() + remove_yang_models() def test_1_table_container(self): """ Test for 1 'table' container @@ -53,13 +53,13 @@ def test_1_table_container(self): } """ - template('sonic-1-table-container', + base_test('sonic-1-table-container', assert_dictionaries.one_table_container) def test_2_table_containers(self): """ Test for 2 'table' containers """ - template('sonic-2-table-containers', + base_test('sonic-2-table-containers', assert_dictionaries.two_table_containers) def test_1_object_container(self): @@ -75,13 +75,13 @@ def test_1_object_container(self): } """ - template('sonic-1-object-container', + base_test('sonic-1-object-container', assert_dictionaries.one_object_container) def test_2_object_containers(self): """ Test for 2 'object' containers """ - template('sonic-2-object-containers', + base_test('sonic-2-object-containers', assert_dictionaries.two_object_containers) def test_1_list(self): @@ -89,21 +89,21 @@ def test_1_list(self): the YANG 'list' entity """ - template('sonic-1-list', assert_dictionaries.one_list) + base_test('sonic-1-list', assert_dictionaries.one_list) def test_2_lists(self): """ Test for 2 containers that have inside the YANG 'list' entity """ - template('sonic-2-lists', assert_dictionaries.two_lists) + base_test('sonic-2-lists', assert_dictionaries.two_lists) def test_static_object_complex_1(self): """ Test for the object container with: 1 leaf, 1 leaf-list, 1 choice. """ - template('sonic-static-object-complex-1', + base_test('sonic-static-object-complex-1', assert_dictionaries.static_object_complex_1) def test_static_object_complex_2(self): @@ -111,7 +111,7 @@ def test_static_object_complex_2(self): 2 leafs, 2 leaf-lists, 2 choices. """ - template('sonic-static-object-complex-2', + base_test('sonic-static-object-complex-2', assert_dictionaries.static_object_complex_2) def test_dynamic_object_complex_1(self): @@ -119,7 +119,7 @@ def test_dynamic_object_complex_1(self): 1 key, 1 leaf, 1 leaf-list, 1 choice. """ - template('sonic-dynamic-object-complex-1', + base_test('sonic-dynamic-object-complex-1', assert_dictionaries.dynamic_object_complex_1) def test_dynamic_object_complex_2(self): @@ -127,7 +127,7 @@ def test_dynamic_object_complex_2(self): 2 keys, 2 leafs, 2 leaf-list, 2 choice. """ - template('sonic-dynamic-object-complex-2', + base_test('sonic-dynamic-object-complex-2', assert_dictionaries.dynamic_object_complex_2) def test_choice_complex(self): @@ -136,7 +136,7 @@ def test_choice_complex(self): leafs, leaf-lists, multiple 'uses' from different files """ - template('sonic-choice-complex', + base_test('sonic-choice-complex', assert_dictionaries.choice_complex) def test_grouping_complex(self): @@ -145,12 +145,12 @@ def test_grouping_complex(self): leafs, leaf-lists, choices """ - template('sonic-grouping-complex', + base_test('sonic-grouping-complex', assert_dictionaries.grouping_complex) -def template(yang_model_name, correct_dict): - """ General template for every test case """ +def base_test(yang_model_name, correct_dict): + """ General logic for each test case """ config_db_path = os.path.join(test_path, 'cli_autogen_input/config_db.json') @@ -163,7 +163,7 @@ def template(yang_model_name, correct_dict): assert yang_dict == correct_dict -def move_yang_models_to_well_know_location(): +def move_yang_models(): """ Move a test YANG models to known location in order to be parsed by YangParser class """ @@ -176,7 +176,7 @@ def move_yang_models_to_well_know_location(): os.system(cmd) -def remove_yang_models_from_well_know_location(): +def remove_yang_models(): """ Remove a test YANG models to known location in order to be parsed by YangParser class """ From 94cde8e8117f5830ed20ae04aaa4a9b5098bd2f1 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Mon, 30 Aug 2021 11:36:19 +0000 Subject: [PATCH 56/76] Changed tabs to spaces Signed-off-by: Vadym Hlushko --- .../cli_autogen_input/assert_dictionaries.py | 3 +- tests/cli_autogen_input/sonic-1-list.yang | 36 ++-- .../sonic-1-object-container.yang | 28 +-- .../sonic-1-table-container.yang | 20 +- tests/cli_autogen_input/sonic-2-lists.yang | 52 ++--- .../sonic-2-object-containers.yang | 36 ++-- .../sonic-2-table-containers.yang | 26 +-- .../sonic-choice-complex.yang | 174 ++++++++--------- .../sonic-dynamic-object-complex-1.yang | 106 +++++----- .../sonic-dynamic-object-complex-2.yang | 160 +++++++-------- tests/cli_autogen_input/sonic-grouping-1.yang | 32 +-- tests/cli_autogen_input/sonic-grouping-2.yang | 32 +-- .../sonic-grouping-complex.yang | 184 +++++++++--------- .../sonic-static-object-complex-1.yang | 90 ++++----- .../sonic-static-object-complex-2.yang | 114 +++++------ 15 files changed, 547 insertions(+), 546 deletions(-) diff --git a/tests/cli_autogen_input/assert_dictionaries.py b/tests/cli_autogen_input/assert_dictionaries.py index 263e48366d..bed2a4a06a 100644 --- a/tests/cli_autogen_input/assert_dictionaries.py +++ b/tests/cli_autogen_input/assert_dictionaries.py @@ -622,4 +622,5 @@ ] } ] -} \ No newline at end of file +} + diff --git a/tests/cli_autogen_input/sonic-1-list.yang b/tests/cli_autogen_input/sonic-1-list.yang index 79a6529b3d..bc8603add4 100644 --- a/tests/cli_autogen_input/sonic-1-list.yang +++ b/tests/cli_autogen_input/sonic-1-list.yang @@ -2,28 +2,28 @@ module sonic-1-list { yang-version 1.1; - namespace "http://github.com/Azure/s-1-list"; - prefix s-1-list; + namespace "http://github.com/Azure/s-1-list"; + prefix s-1-list; - container sonic-1-list { - /* sonic-1-list - top level container */ + container sonic-1-list { + /* sonic-1-list - top level container */ - container TABLE_1 { - /* TABLE_1 - table container */ + container TABLE_1 { + /* TABLE_1 - table container */ - description "TABLE_1 description"; + description "TABLE_1 description"; - list TABLE_1_LIST { - /* TABLE_1 - object container */ + list TABLE_1_LIST { + /* TABLE_1 - object container */ - description "TABLE_1_LIST description"; + description "TABLE_1_LIST description"; - key "key_name"; + key "key_name"; - leaf key_name { - type string; - } - } - } - } -} \ No newline at end of file + leaf key_name { + type string; + } + } + } + } +} diff --git a/tests/cli_autogen_input/sonic-1-object-container.yang b/tests/cli_autogen_input/sonic-1-object-container.yang index e28ef7f90a..8d19979157 100644 --- a/tests/cli_autogen_input/sonic-1-object-container.yang +++ b/tests/cli_autogen_input/sonic-1-object-container.yang @@ -2,22 +2,22 @@ module sonic-1-object-container { yang-version 1.1; - namespace "http://github.com/Azure/s-1-object"; - prefix s-1-object; + namespace "http://github.com/Azure/s-1-object"; + prefix s-1-object; - container sonic-1-object-container { - /* sonic-1-object-container - top level container */ + container sonic-1-object-container { + /* sonic-1-object-container - top level container */ - container TABLE_1 { - /* TABLE_1 - table container */ + container TABLE_1 { + /* TABLE_1 - table container */ - description "TABLE_1 description"; + description "TABLE_1 description"; - container OBJECT_1 { - /* OBJECT_1 - object container */ + container OBJECT_1 { + /* OBJECT_1 - object container */ - description "OBJECT_1 description"; - } - } - } -} \ No newline at end of file + description "OBJECT_1 description"; + } + } + } +} diff --git a/tests/cli_autogen_input/sonic-1-table-container.yang b/tests/cli_autogen_input/sonic-1-table-container.yang index 58e7293c0d..36b98415e5 100644 --- a/tests/cli_autogen_input/sonic-1-table-container.yang +++ b/tests/cli_autogen_input/sonic-1-table-container.yang @@ -2,16 +2,16 @@ module sonic-1-table-container { yang-version 1.1; - namespace "http://github.com/Azure/s-1-table"; - prefix s-1-table; + namespace "http://github.com/Azure/s-1-table"; + prefix s-1-table; - container sonic-1-table-container { - /* sonic-1-table-container - top level container */ + container sonic-1-table-container { + /* sonic-1-table-container - top level container */ - container TABLE_1 { - /* TABLE_1 - table container */ + container TABLE_1 { + /* TABLE_1 - table container */ - description "TABLE_1 description"; - } - } -} \ No newline at end of file + description "TABLE_1 description"; + } + } +} diff --git a/tests/cli_autogen_input/sonic-2-lists.yang b/tests/cli_autogen_input/sonic-2-lists.yang index b20200415b..fce9704f00 100644 --- a/tests/cli_autogen_input/sonic-2-lists.yang +++ b/tests/cli_autogen_input/sonic-2-lists.yang @@ -2,41 +2,41 @@ module sonic-2-lists { yang-version 1.1; - namespace "http://github.com/Azure/s-2-lists"; - prefix s-2-lists; + namespace "http://github.com/Azure/s-2-lists"; + prefix s-2-lists; - container sonic-2-lists { - /* sonic-2-lists - top level container */ + container sonic-2-lists { + /* sonic-2-lists - top level container */ - container TABLE_1 { - /* TALBE_1 - table container */ + container TABLE_1 { + /* TALBE_1 - table container */ - description "TABLE_1 description"; + description "TABLE_1 description"; - list TABLE_1_LIST_1 { - /* TALBE_1_LIST_1 - object container */ + list TABLE_1_LIST_1 { + /* TALBE_1_LIST_1 - object container */ - description "TABLE_1_LIST_1 description"; + description "TABLE_1_LIST_1 description"; - key "key_name1"; + key "key_name1"; - leaf key_name1 { - type string; - } - } + leaf key_name1 { + type string; + } + } - list TABLE_1_LIST_2 { - /* TALBE_1_LIST_2 - object container */ + list TABLE_1_LIST_2 { + /* TALBE_1_LIST_2 - object container */ - description "TABLE_1_LIST_2 description"; + description "TABLE_1_LIST_2 description"; - key "key_name2"; + key "key_name2"; - leaf key_name2 { - type string; - } - } - } - } -} \ No newline at end of file + leaf key_name2 { + type string; + } + } + } + } +} diff --git a/tests/cli_autogen_input/sonic-2-object-containers.yang b/tests/cli_autogen_input/sonic-2-object-containers.yang index 249faf4c89..e633b66246 100644 --- a/tests/cli_autogen_input/sonic-2-object-containers.yang +++ b/tests/cli_autogen_input/sonic-2-object-containers.yang @@ -2,28 +2,28 @@ module sonic-2-object-containers { yang-version 1.1; - namespace "http://github.com/Azure/s-2-object"; - prefix s-2-object; + namespace "http://github.com/Azure/s-2-object"; + prefix s-2-object; - container sonic-2-object-containers { - /* sonic-2-object-containers - top level container */ + container sonic-2-object-containers { + /* sonic-2-object-containers - top level container */ - container TABLE_1 { - /* TABLE_1 - table container */ + container TABLE_1 { + /* TABLE_1 - table container */ - description "FIRST_TABLE description"; + description "FIRST_TABLE description"; - container OBJECT_1 { - /* OBJECT_1 - object container */ + container OBJECT_1 { + /* OBJECT_1 - object container */ - description "OBJECT_1 description"; - } + description "OBJECT_1 description"; + } - container OBJECT_2 { - /* OBJECT_2 - object container */ + container OBJECT_2 { + /* OBJECT_2 - object container */ - description "OBJECT_2 description"; - } - } - } -} \ No newline at end of file + description "OBJECT_2 description"; + } + } + } +} diff --git a/tests/cli_autogen_input/sonic-2-table-containers.yang b/tests/cli_autogen_input/sonic-2-table-containers.yang index 393512a313..f5284c67ee 100644 --- a/tests/cli_autogen_input/sonic-2-table-containers.yang +++ b/tests/cli_autogen_input/sonic-2-table-containers.yang @@ -2,22 +2,22 @@ module sonic-2-table-containers { yang-version 1.1; - namespace "http://github.com/Azure/s-2-table"; - prefix s-2-table; + namespace "http://github.com/Azure/s-2-table"; + prefix s-2-table; - container sonic-2-table-containers { - /* sonic-2-table-containers - top level container */ + container sonic-2-table-containers { + /* sonic-2-table-containers - top level container */ - container TABLE_1 { - /* TABLE_1 - table container */ + container TABLE_1 { + /* TABLE_1 - table container */ - description "TABLE_1 description"; - } + description "TABLE_1 description"; + } - container TABLE_2 { - /* TABLE_2 - table container */ + container TABLE_2 { + /* TABLE_2 - table container */ - description "TABLE_2 description"; - } - } + description "TABLE_2 description"; + } + } } diff --git a/tests/cli_autogen_input/sonic-choice-complex.yang b/tests/cli_autogen_input/sonic-choice-complex.yang index 7d6a66d89f..9d6e0de9ee 100644 --- a/tests/cli_autogen_input/sonic-choice-complex.yang +++ b/tests/cli_autogen_input/sonic-choice-complex.yang @@ -2,90 +2,90 @@ module sonic-choice-complex { yang-version 1.1; - namespace "http://github.com/Azure/choice-complex"; - prefix choice-complex; - - import sonic-grouping-1 { - prefix sgroup1; - } - - import sonic-grouping-2 { - prefix sgroup2; - } - - grouping GR_5 { - leaf GR_5_LEAF_1 { - type string; - } - - leaf GR_5_LEAF_2 { - type string; - } - } - - grouping GR_6 { - leaf GR_6_LEAF_1 { - type string; - } - - leaf GR_6_LEAF_2 { - type string; - } - } - - container sonic-choice-complex { - /* sonic-choice-complex - top level container */ - - container TABLE_1 { - /* TABLE_1 - table container */ - - description "TABLE_1 description"; - - container OBJECT_1 { - /* OBJECT_1 - object container, it have - * 1 choice, which have 2 cases. - * first case have: 1 leaf, 1 leaf-list, 1 uses - * second case have: 2 leafs, 2 leaf-lists, 2 uses - */ - - description "OBJECT_1 description"; - - choice CHOICE_1 { - case CHOICE_1_CASE_1 { - leaf LEAF_1 { - type uint16; - } - - leaf-list LEAF_LIST_1 { - type string; - } - - uses sgroup1:GR_1; - } - - case CHOICE_1_CASE_2 { - leaf LEAF_2 { - type string; - } - - leaf LEAF_3 { - type string; - } - - leaf-list LEAF_LIST_2 { - type string; - } - - leaf-list LEAF_LIST_3 { - type string; - } - - uses GR_5; - uses sgroup1:GR_2; - uses sgroup2:GR_3; - } - } - } - } - } -} \ No newline at end of file + namespace "http://github.com/Azure/choice-complex"; + prefix choice-complex; + + import sonic-grouping-1 { + prefix sgroup1; + } + + import sonic-grouping-2 { + prefix sgroup2; + } + + grouping GR_5 { + leaf GR_5_LEAF_1 { + type string; + } + + leaf GR_5_LEAF_2 { + type string; + } + } + + grouping GR_6 { + leaf GR_6_LEAF_1 { + type string; + } + + leaf GR_6_LEAF_2 { + type string; + } + } + + container sonic-choice-complex { + /* sonic-choice-complex - top level container */ + + container TABLE_1 { + /* TABLE_1 - table container */ + + description "TABLE_1 description"; + + container OBJECT_1 { + /* OBJECT_1 - object container, it have + * 1 choice, which have 2 cases. + * first case have: 1 leaf, 1 leaf-list, 1 uses + * second case have: 2 leafs, 2 leaf-lists, 2 uses + */ + + description "OBJECT_1 description"; + + choice CHOICE_1 { + case CHOICE_1_CASE_1 { + leaf LEAF_1 { + type uint16; + } + + leaf-list LEAF_LIST_1 { + type string; + } + + uses sgroup1:GR_1; + } + + case CHOICE_1_CASE_2 { + leaf LEAF_2 { + type string; + } + + leaf LEAF_3 { + type string; + } + + leaf-list LEAF_LIST_2 { + type string; + } + + leaf-list LEAF_LIST_3 { + type string; + } + + uses GR_5; + uses sgroup1:GR_2; + uses sgroup2:GR_3; + } + } + } + } + } +} diff --git a/tests/cli_autogen_input/sonic-dynamic-object-complex-1.yang b/tests/cli_autogen_input/sonic-dynamic-object-complex-1.yang index 9beb98549d..383e94fb43 100644 --- a/tests/cli_autogen_input/sonic-dynamic-object-complex-1.yang +++ b/tests/cli_autogen_input/sonic-dynamic-object-complex-1.yang @@ -2,56 +2,56 @@ module sonic-dynamic-object-complex-1 { yang-version 1.1; - namespace "http://github.com/Azure/dynamic-complex-1"; - prefix dynamic-complex-1; - - container sonic-dynamic-object-complex-1 { - /* sonic-dynamic-object-complex-1 - top level container */ - - container TABLE_1 { - /* TABLE_1 - table container */ - - description "TABLE_1 description"; - - list OBJECT_1_LIST { - /* OBJECT_1_LIST - dynamic object container, it have: - * 1 key, - * 1 leaf, - * 1 leaf-list - * 1 choice - */ - - description "OBJECT_1_LIST description"; - - key "KEY_LEAF_1"; - - leaf KEY_LEAF_1 { - description "KEY_LEAF_1 description"; - type string; - } - - leaf OBJ_1_LEAF_1 { - description "OBJ_1_LEAF_1 description"; - type string; - } - - leaf-list OBJ_1_LEAF_LIST_1 { - type string; - } - - choice OBJ_1_CHOICE_1 { - case OBJ_1_CHOICE_1_CASE_1 { - leaf OBJ_1_CHOICE_1_LEAF_1 { - type uint16; - } - } - case OBJ_1_CHOICE_1_CASE_2 { - leaf OBJ_1_CHOICE_1_LEAF_2 { - type string; - } - } - } - } - } - } -} \ No newline at end of file + namespace "http://github.com/Azure/dynamic-complex-1"; + prefix dynamic-complex-1; + + container sonic-dynamic-object-complex-1 { + /* sonic-dynamic-object-complex-1 - top level container */ + + container TABLE_1 { + /* TABLE_1 - table container */ + + description "TABLE_1 description"; + + list OBJECT_1_LIST { + /* OBJECT_1_LIST - dynamic object container, it have: + * 1 key, + * 1 leaf, + * 1 leaf-list + * 1 choice + */ + + description "OBJECT_1_LIST description"; + + key "KEY_LEAF_1"; + + leaf KEY_LEAF_1 { + description "KEY_LEAF_1 description"; + type string; + } + + leaf OBJ_1_LEAF_1 { + description "OBJ_1_LEAF_1 description"; + type string; + } + + leaf-list OBJ_1_LEAF_LIST_1 { + type string; + } + + choice OBJ_1_CHOICE_1 { + case OBJ_1_CHOICE_1_CASE_1 { + leaf OBJ_1_CHOICE_1_LEAF_1 { + type uint16; + } + } + case OBJ_1_CHOICE_1_CASE_2 { + leaf OBJ_1_CHOICE_1_LEAF_2 { + type string; + } + } + } + } + } + } +} diff --git a/tests/cli_autogen_input/sonic-dynamic-object-complex-2.yang b/tests/cli_autogen_input/sonic-dynamic-object-complex-2.yang index 00e25c8135..a365b014ad 100644 --- a/tests/cli_autogen_input/sonic-dynamic-object-complex-2.yang +++ b/tests/cli_autogen_input/sonic-dynamic-object-complex-2.yang @@ -2,83 +2,83 @@ module sonic-dynamic-object-complex-2 { yang-version 1.1; - namespace "http://github.com/Azure/dynamic-complex-2"; - prefix dynamic-complex-2; - - container sonic-dynamic-object-complex-2 { - /* sonic-dynamic-object-complex-2 - top level container */ - - container TABLE_1 { - /* TABLE_1 - table container */ - - description "TABLE_1 description"; - - list OBJECT_1_LIST { - /* OBJECT_1_LIST - dynamic object container, it have: - * 2 keys - * 2 leaf, - * 2 leaf-list - * 2 choice - */ - - description "OBJECT_1_LIST description"; - - key "KEY_LEAF_1 KEY_LEAF_2"; - - leaf KEY_LEAF_1 { - description "KEY_LEAF_1 description"; - type string; - } - - leaf KEY_LEAF_2 { - description "KEY_LEAF_2 description"; - type string; - } - - leaf OBJ_1_LEAF_1 { - description "OBJ_1_LEAF_1 description"; - type string; - } - - leaf OBJ_1_LEAF_2 { - description "OBJ_1_LEAF_2 description"; - type string; - } - - leaf-list OBJ_1_LEAF_LIST_1 { - type string; - } - - leaf-list OBJ_1_LEAF_LIST_2 { - type string; - } - - choice OBJ_1_CHOICE_1 { - case OBJ_1_CHOICE_1_CASE_1 { - leaf OBJ_1_CHOICE_1_LEAF_1 { - type uint16; - } - } - case OBJ_1_CHOICE_1_CASE_2 { - leaf OBJ_1_CHOICE_1_LEAF_2 { - type string; - } - } - } - - choice OBJ_1_CHOICE_2 { - case OBJ_1_CHOICE_2_CASE_1 { - leaf OBJ_1_CHOICE_2_LEAF_1 { - type uint16; - } - } - case OBJ_1_CHOICE_2_CASE_2 { - leaf OBJ_1_CHOICE_2_LEAF_2 { - type string; - } - } - } - } - } - } -} \ No newline at end of file + namespace "http://github.com/Azure/dynamic-complex-2"; + prefix dynamic-complex-2; + + container sonic-dynamic-object-complex-2 { + /* sonic-dynamic-object-complex-2 - top level container */ + + container TABLE_1 { + /* TABLE_1 - table container */ + + description "TABLE_1 description"; + + list OBJECT_1_LIST { + /* OBJECT_1_LIST - dynamic object container, it have: + * 2 keys + * 2 leaf, + * 2 leaf-list + * 2 choice + */ + + description "OBJECT_1_LIST description"; + + key "KEY_LEAF_1 KEY_LEAF_2"; + + leaf KEY_LEAF_1 { + description "KEY_LEAF_1 description"; + type string; + } + + leaf KEY_LEAF_2 { + description "KEY_LEAF_2 description"; + type string; + } + + leaf OBJ_1_LEAF_1 { + description "OBJ_1_LEAF_1 description"; + type string; + } + + leaf OBJ_1_LEAF_2 { + description "OBJ_1_LEAF_2 description"; + type string; + } + + leaf-list OBJ_1_LEAF_LIST_1 { + type string; + } + + leaf-list OBJ_1_LEAF_LIST_2 { + type string; + } + + choice OBJ_1_CHOICE_1 { + case OBJ_1_CHOICE_1_CASE_1 { + leaf OBJ_1_CHOICE_1_LEAF_1 { + type uint16; + } + } + case OBJ_1_CHOICE_1_CASE_2 { + leaf OBJ_1_CHOICE_1_LEAF_2 { + type string; + } + } + } + + choice OBJ_1_CHOICE_2 { + case OBJ_1_CHOICE_2_CASE_1 { + leaf OBJ_1_CHOICE_2_LEAF_1 { + type uint16; + } + } + case OBJ_1_CHOICE_2_CASE_2 { + leaf OBJ_1_CHOICE_2_LEAF_2 { + type string; + } + } + } + } + } + } +} diff --git a/tests/cli_autogen_input/sonic-grouping-1.yang b/tests/cli_autogen_input/sonic-grouping-1.yang index 831c3a4ad8..bf0be792f5 100644 --- a/tests/cli_autogen_input/sonic-grouping-1.yang +++ b/tests/cli_autogen_input/sonic-grouping-1.yang @@ -2,24 +2,24 @@ module sonic-grouping-1{ yang-version 1.1; - namespace "http://github.com/Azure/s-grouping-1"; - prefix s-grouping-1; + namespace "http://github.com/Azure/s-grouping-1"; + prefix s-grouping-1; - grouping GR_1 { - leaf GR_1_LEAF_1 { - type string; - } - leaf GR_1_LEAF_2 { - type string; - } + grouping GR_1 { + leaf GR_1_LEAF_1 { + type string; + } + leaf GR_1_LEAF_2 { + type string; + } } - grouping GR_2 { - leaf GR_2_LEAF_1 { - type string; - } - leaf GR_2_LEAF_2 { - type string; + grouping GR_2 { + leaf GR_2_LEAF_1 { + type string; + } + leaf GR_2_LEAF_2 { + type string; } } -} \ No newline at end of file +} diff --git a/tests/cli_autogen_input/sonic-grouping-2.yang b/tests/cli_autogen_input/sonic-grouping-2.yang index bfaa13db15..58e9df6621 100644 --- a/tests/cli_autogen_input/sonic-grouping-2.yang +++ b/tests/cli_autogen_input/sonic-grouping-2.yang @@ -2,24 +2,24 @@ module sonic-grouping-2 { yang-version 1.1; - namespace "http://github.com/Azure/s-grouping-2"; - prefix s-grouping-2; + namespace "http://github.com/Azure/s-grouping-2"; + prefix s-grouping-2; - grouping GR_3 { - leaf GR_3_LEAF_1 { - type string; - } - leaf GR_3_LEAF_2 { - type string; - } + grouping GR_3 { + leaf GR_3_LEAF_1 { + type string; + } + leaf GR_3_LEAF_2 { + type string; + } } - grouping GR_4 { - leaf GR_4_LEAF_1 { - type string; - } - leaf GR_4_LEAF_2 { - type string; + grouping GR_4 { + leaf GR_4_LEAF_1 { + type string; + } + leaf GR_4_LEAF_2 { + type string; } } -} \ No newline at end of file +} diff --git a/tests/cli_autogen_input/sonic-grouping-complex.yang b/tests/cli_autogen_input/sonic-grouping-complex.yang index d6ed68563a..22956789b0 100644 --- a/tests/cli_autogen_input/sonic-grouping-complex.yang +++ b/tests/cli_autogen_input/sonic-grouping-complex.yang @@ -2,95 +2,95 @@ module sonic-grouping-complex { yang-version 1.1; - namespace "http://github.com/Azure/grouping-complex"; - prefix grouping-complex; - - import sonic-grouping-1 { - prefix sgroup1; - } - - import sonic-grouping-2 { - prefix sgroup2; - } - - grouping GR_5 { - leaf GR_5_LEAF_1 { - type string; - } - - leaf-list GR_5_LEAF_LIST_1 { - type string; - } - } - - grouping GR_6 { - leaf GR_6_LEAF_1 { - type string; - } - - leaf GR_6_LEAF_2 { - type string; - } - - choice GR_6_CHOICE_1 { - case CHOICE_1_CASE_1 { - leaf GR_6_CASE_1_LEAF_1 { - type uint16; - } - - leaf-list GR_6_CASE_1_LEAF_LIST_1 { - type string; - } - } - - case CHOICE_1_CASE_2 { - leaf GR_6_CASE_2_LEAF_1 { - type uint16; - } - - leaf GR_6_CASE_2_LEAF_2 { - type uint16; - } - - leaf-list GR_6_CASE_2_LEAF_LIST_1 { - type string; - } - - leaf-list GR_6_CASE_2_LEAF_LIST_2 { - type string; - } - } - } - } - - container sonic-grouping-complex { - /* sonic-grouping-complex - top level container */ - - container TABLE_1 { - /* TABLE_1 - table container */ - - description "TABLE_1 description"; - - container OBJECT_1 { - /* OBJECT_1 - object container, it have - * 1 choice, which have 2 cases. - * first case have: 1 leaf, 1 leaf-list, 1 uses - * second case have: 2 leafs, 2 leaf-lists, 2 uses - */ - - description "OBJECT_1 description"; - - uses sgroup1:GR_1; - } - - container OBJECT_2 { - - description "OBJECT_2 description"; - - uses GR_5; - uses GR_6; - uses sgroup2:GR_4; - } - } - } -} \ No newline at end of file + namespace "http://github.com/Azure/grouping-complex"; + prefix grouping-complex; + + import sonic-grouping-1 { + prefix sgroup1; + } + + import sonic-grouping-2 { + prefix sgroup2; + } + + grouping GR_5 { + leaf GR_5_LEAF_1 { + type string; + } + + leaf-list GR_5_LEAF_LIST_1 { + type string; + } + } + + grouping GR_6 { + leaf GR_6_LEAF_1 { + type string; + } + + leaf GR_6_LEAF_2 { + type string; + } + + choice GR_6_CHOICE_1 { + case CHOICE_1_CASE_1 { + leaf GR_6_CASE_1_LEAF_1 { + type uint16; + } + + leaf-list GR_6_CASE_1_LEAF_LIST_1 { + type string; + } + } + + case CHOICE_1_CASE_2 { + leaf GR_6_CASE_2_LEAF_1 { + type uint16; + } + + leaf GR_6_CASE_2_LEAF_2 { + type uint16; + } + + leaf-list GR_6_CASE_2_LEAF_LIST_1 { + type string; + } + + leaf-list GR_6_CASE_2_LEAF_LIST_2 { + type string; + } + } + } + } + + container sonic-grouping-complex { + /* sonic-grouping-complex - top level container */ + + container TABLE_1 { + /* TABLE_1 - table container */ + + description "TABLE_1 description"; + + container OBJECT_1 { + /* OBJECT_1 - object container, it have + * 1 choice, which have 2 cases. + * first case have: 1 leaf, 1 leaf-list, 1 uses + * second case have: 2 leafs, 2 leaf-lists, 2 uses + */ + + description "OBJECT_1 description"; + + uses sgroup1:GR_1; + } + + container OBJECT_2 { + + description "OBJECT_2 description"; + + uses GR_5; + uses GR_6; + uses sgroup2:GR_4; + } + } + } +} diff --git a/tests/cli_autogen_input/sonic-static-object-complex-1.yang b/tests/cli_autogen_input/sonic-static-object-complex-1.yang index a7dfee86ab..fa082d3b25 100644 --- a/tests/cli_autogen_input/sonic-static-object-complex-1.yang +++ b/tests/cli_autogen_input/sonic-static-object-complex-1.yang @@ -2,48 +2,48 @@ module sonic-static-object-complex-1 { yang-version 1.1; - namespace "http://github.com/Azure/static-complex-1"; - prefix static-complex-1; - - container sonic-static-object-complex-1 { - /* sonic-static-object-complex-1 - top level container */ - - container TABLE_1 { - /* TABLE_1 - table container */ - - description "TABLE_1 description"; - - container OBJECT_1 { - /* OBJECT_1 - object container, it have: - * 1 leaf, - * 1 leaf-list - * 1 choice - */ - - description "OBJECT_1 description"; - - leaf OBJ_1_LEAF_1 { - description "OBJ_1_LEAF_1 description"; - type string; - } - - leaf-list OBJ_1_LEAF_LIST_1 { - type string; - } - - choice OBJ_1_CHOICE_1 { - case OBJ_1_CHOICE_1_CASE_1 { - leaf OBJ_1_CHOICE_1_LEAF_1 { - type uint16; - } - } - case OBJ_1_CHOICE_1_CASE_2 { - leaf OBJ_1_CHOICE_1_LEAF_2 { - type string; - } - } - } - } - } - } -} \ No newline at end of file + namespace "http://github.com/Azure/static-complex-1"; + prefix static-complex-1; + + container sonic-static-object-complex-1 { + /* sonic-static-object-complex-1 - top level container */ + + container TABLE_1 { + /* TABLE_1 - table container */ + + description "TABLE_1 description"; + + container OBJECT_1 { + /* OBJECT_1 - object container, it have: + * 1 leaf, + * 1 leaf-list + * 1 choice + */ + + description "OBJECT_1 description"; + + leaf OBJ_1_LEAF_1 { + description "OBJ_1_LEAF_1 description"; + type string; + } + + leaf-list OBJ_1_LEAF_LIST_1 { + type string; + } + + choice OBJ_1_CHOICE_1 { + case OBJ_1_CHOICE_1_CASE_1 { + leaf OBJ_1_CHOICE_1_LEAF_1 { + type uint16; + } + } + case OBJ_1_CHOICE_1_CASE_2 { + leaf OBJ_1_CHOICE_1_LEAF_2 { + type string; + } + } + } + } + } + } +} diff --git a/tests/cli_autogen_input/sonic-static-object-complex-2.yang b/tests/cli_autogen_input/sonic-static-object-complex-2.yang index 451a445ce6..4e53b2e1b1 100644 --- a/tests/cli_autogen_input/sonic-static-object-complex-2.yang +++ b/tests/cli_autogen_input/sonic-static-object-complex-2.yang @@ -2,70 +2,70 @@ module sonic-static-object-complex-2 { yang-version 1.1; - namespace "http://github.com/Azure/static-complex-2"; - prefix static-complex-2; + namespace "http://github.com/Azure/static-complex-2"; + prefix static-complex-2; - container sonic-static-object-complex-2 { - /* sonic-static-object-complex-2 - top level container */ + container sonic-static-object-complex-2 { + /* sonic-static-object-complex-2 - top level container */ - container TABLE_1 { - /* TABLE_1 - table container */ + container TABLE_1 { + /* TABLE_1 - table container */ - description "TABLE_1 description"; + description "TABLE_1 description"; - container OBJECT_1 { - /* OBJECT_1 - object container, it have: - * 2 leafs, - * 2 leaf-lists, - * 2 choices - */ + container OBJECT_1 { + /* OBJECT_1 - object container, it have: + * 2 leafs, + * 2 leaf-lists, + * 2 choices + */ - description "OBJECT_1 description"; + description "OBJECT_1 description"; - leaf OBJ_1_LEAF_1 { - description "OBJ_1_LEAF_1 description"; - type string; - } - - leaf OBJ_1_LEAF_2 { - description "OBJ_1_LEAF_2 description"; - type string; - } + leaf OBJ_1_LEAF_1 { + description "OBJ_1_LEAF_1 description"; + type string; + } + + leaf OBJ_1_LEAF_2 { + description "OBJ_1_LEAF_2 description"; + type string; + } - leaf-list OBJ_1_LEAF_LIST_1 { - type string; - } + leaf-list OBJ_1_LEAF_LIST_1 { + type string; + } - leaf-list OBJ_1_LEAF_LIST_2 { - type string; - } + leaf-list OBJ_1_LEAF_LIST_2 { + type string; + } - choice OBJ_1_CHOICE_1 { - case OBJ_1_CHOICE_1_CASE_1 { - leaf OBJ_1_CHOICE_1_LEAF_1 { - type uint16; - } - } - case OBJ_1_CHOICE_1_CASE_2 { - leaf OBJ_1_CHOICE_1_LEAF_2 { - type string; - } - } - } + choice OBJ_1_CHOICE_1 { + case OBJ_1_CHOICE_1_CASE_1 { + leaf OBJ_1_CHOICE_1_LEAF_1 { + type uint16; + } + } + case OBJ_1_CHOICE_1_CASE_2 { + leaf OBJ_1_CHOICE_1_LEAF_2 { + type string; + } + } + } - choice OBJ_1_CHOICE_2 { - case OBJ_1_CHOICE_2_CASE_1 { - leaf OBJ_1_CHOICE_2_LEAF_1 { - type uint16; - } - } - case OBJ_1_CHOICE_2_CASE_2 { - leaf OBJ_1_CHOICE_2_LEAF_2 { - type string; - } - } - } - } - } - } -} \ No newline at end of file + choice OBJ_1_CHOICE_2 { + case OBJ_1_CHOICE_2_CASE_1 { + leaf OBJ_1_CHOICE_2_LEAF_1 { + type uint16; + } + } + case OBJ_1_CHOICE_2_CASE_2 { + leaf OBJ_1_CHOICE_2_LEAF_2 { + type string; + } + } + } + } + } + } +} From 5fec4a7b3004c22fe413c211a5bc83c42ba07cd5 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Tue, 31 Aug 2021 09:07:45 +0000 Subject: [PATCH 57/76] Removed unnecessary config_db.json file Signed-off-by: Vadym Hlushko --- tests/cli_autogen_input/config_db.json | 544 ------------------------- tests/cli_autogen_yang_parser_test.py | 2 +- 2 files changed, 1 insertion(+), 545 deletions(-) delete mode 100644 tests/cli_autogen_input/config_db.json diff --git a/tests/cli_autogen_input/config_db.json b/tests/cli_autogen_input/config_db.json deleted file mode 100644 index 5473d6158a..0000000000 --- a/tests/cli_autogen_input/config_db.json +++ /dev/null @@ -1,544 +0,0 @@ -{ - "COPP_GROUP": { - "default": { - "cbs": "600", - "cir": "600", - "meter_type": "packets", - "mode": "sr_tcm", - "queue": "0", - "red_action": "drop" - }, - "queue1_group1": { - "cbs": "6000", - "cir": "6000", - "meter_type": "packets", - "mode": "sr_tcm", - "queue": "1", - "red_action": "drop", - "trap_action": "trap", - "trap_priority": "1" - }, - "queue1_group2": { - "cbs": "600", - "cir": "600", - "meter_type": "packets", - "mode": "sr_tcm", - "queue": "1", - "red_action": "drop", - "trap_action": "trap", - "trap_priority": "1" - }, - "queue2_group1": { - "cbs": "1000", - "cir": "1000", - "genetlink_mcgrp_name": "packets", - "genetlink_name": "psample", - "meter_type": "packets", - "mode": "sr_tcm", - "queue": "2", - "red_action": "drop", - "trap_action": "trap", - "trap_priority": "1" - }, - "queue4_group1": { - "cbs": "600", - "cir": "600", - "color": "blind", - "meter_type": "packets", - "mode": "sr_tcm", - "queue": "4", - "red_action": "drop", - "trap_action": "trap", - "trap_priority": "4" - }, - "queue4_group2": { - "cbs": "600", - "cir": "600", - "meter_type": "packets", - "mode": "sr_tcm", - "queue": "4", - "red_action": "drop", - "trap_action": "copy", - "trap_priority": "4" - }, - "queue4_group3": { - "cbs": "600", - "cir": "600", - "color": "blind", - "meter_type": "packets", - "mode": "sr_tcm", - "queue": "4", - "red_action": "drop", - "trap_action": "trap", - "trap_priority": "4" - } - }, - "COPP_TRAP": { - "arp": { - "trap_group": "queue4_group2", - "trap_ids": "arp_req,arp_resp,neigh_discovery" - }, - "bgp": { - "trap_group": "queue4_group1", - "trap_ids": "bgp,bgpv6" - }, - "dhcp": { - "trap_group": "queue4_group3", - "trap_ids": "dhcp,dhcpv6" - }, - "ip2me": { - "trap_group": "queue1_group1", - "trap_ids": "ip2me" - }, - "lacp": { - "trap_group": "queue4_group1", - "trap_ids": "lacp" - }, - "lldp": { - "trap_group": "queue4_group3", - "trap_ids": "lldp" - }, - "nat": { - "trap_group": "queue1_group2", - "trap_ids": "src_nat_miss,dest_nat_miss" - }, - "sflow": { - "trap_group": "queue2_group1", - "trap_ids": "sample_packet" - }, - "ssh": { - "trap_group": "queue4_group2", - "trap_ids": "ssh" - }, - "udld": { - "trap_group": "queue4_group3", - "trap_ids": "udld" - } - }, - "CRM": { - "Config": { - "acl_counter_high_threshold": "85", - "acl_counter_low_threshold": "70", - "acl_counter_threshold_type": "percentage", - "acl_entry_high_threshold": "85", - "acl_entry_low_threshold": "70", - "acl_entry_threshold_type": "percentage", - "acl_group_high_threshold": "85", - "acl_group_low_threshold": "70", - "acl_group_threshold_type": "percentage", - "acl_table_high_threshold": "85", - "acl_table_low_threshold": "70", - "acl_table_threshold_type": "percentage", - "dnat_entry_high_threshold": "85", - "dnat_entry_low_threshold": "70", - "dnat_entry_threshold_type": "percentage", - "fdb_entry_high_threshold": "85", - "fdb_entry_low_threshold": "70", - "fdb_entry_threshold_type": "percentage", - "ipmc_entry_high_threshold": "85", - "ipmc_entry_low_threshold": "70", - "ipmc_entry_threshold_type": "percentage", - "ipv4_neighbor_high_threshold": "85", - "ipv4_neighbor_low_threshold": "70", - "ipv4_neighbor_threshold_type": "percentage", - "ipv4_nexthop_high_threshold": "85", - "ipv4_nexthop_low_threshold": "70", - "ipv4_nexthop_threshold_type": "percentage", - "ipv4_route_high_threshold": "85", - "ipv4_route_low_threshold": "70", - "ipv4_route_threshold_type": "percentage", - "ipv6_neighbor_high_threshold": "85", - "ipv6_neighbor_low_threshold": "70", - "ipv6_neighbor_threshold_type": "percentage", - "ipv6_nexthop_high_threshold": "85", - "ipv6_nexthop_low_threshold": "70", - "ipv6_nexthop_threshold_type": "percentage", - "ipv6_route_high_threshold": "85", - "ipv6_route_low_threshold": "70", - "ipv6_route_threshold_type": "percentage", - "nexthop_group_high_threshold": "85", - "nexthop_group_low_threshold": "70", - "nexthop_group_member_high_threshold": "85", - "nexthop_group_member_low_threshold": "70", - "nexthop_group_member_threshold_type": "percentage", - "nexthop_group_threshold_type": "percentage", - "polling_interval": "300", - "snat_entry_high_threshold": "85", - "snat_entry_low_threshold": "70", - "snat_entry_threshold_type": "percentage" - } - }, - "DEVICE_METADATA": { - "localhost": { - "buffer_model": "traditional", - "default_bgp_status": "up", - "default_pfcwd_status": "disable", - "hostname": "r-bulldog-02", - "hwsku": "ACS-MSN2100", - "mac": "98:03:9b:f8:e7:c0", - "platform": "x86_64-mlnx_msn2100-r0", - "type": "ToRRouter" - } - }, - "FEATURE": { - "bgp": { - "auto_restart": "enabled", - "has_global_scope": "False", - "has_per_asic_scope": "True", - "has_timer": "False", - "high_mem_alert": "disabled", - "state": "enabled" - }, - "database": { - "auto_restart": "disabled", - "has_global_scope": "True", - "has_per_asic_scope": "True", - "has_timer": "False", - "high_mem_alert": "disabled", - "state": "enabled" - }, - "dhcp_relay": { - "auto_restart": "enabled", - "has_global_scope": "True", - "has_per_asic_scope": "False", - "has_timer": "False", - "high_mem_alert": "disabled", - "state": "enabled" - }, - "lldp": { - "auto_restart": "enabled", - "has_global_scope": "True", - "has_per_asic_scope": "True", - "has_timer": "False", - "high_mem_alert": "disabled", - "state": "enabled", - "status": "enabled" - }, - "mgmt-framework": { - "auto_restart": "enabled", - "has_global_scope": "True", - "has_per_asic_scope": "False", - "has_timer": "True", - "high_mem_alert": "disabled", - "state": "enabled" - }, - "nat": { - "auto_restart": "enabled", - "has_global_scope": "True", - "has_per_asic_scope": "False", - "has_timer": "False", - "high_mem_alert": "disabled", - "state": "disabled" - }, - "pmon": { - "auto_restart": "enabled", - "has_global_scope": "True", - "has_per_asic_scope": "False", - "has_timer": "False", - "high_mem_alert": "disabled", - "state": "enabled" - }, - "radv": { - "auto_restart": "enabled", - "has_global_scope": "True", - "has_per_asic_scope": "False", - "has_timer": "False", - "high_mem_alert": "disabled", - "state": "enabled" - }, - "sflow": { - "auto_restart": "enabled", - "has_global_scope": "True", - "has_per_asic_scope": "False", - "has_timer": "False", - "high_mem_alert": "disabled", - "state": "disabled" - }, - "snmp": { - "auto_restart": "enabled", - "has_global_scope": "True", - "has_per_asic_scope": "False", - "has_timer": "True", - "high_mem_alert": "disabled", - "state": "enabled" - }, - "swss": { - "auto_restart": "enabled", - "has_global_scope": "False", - "has_per_asic_scope": "True", - "has_timer": "False", - "high_mem_alert": "disabled", - "state": "enabled" - }, - "syncd": { - "auto_restart": "enabled", - "has_global_scope": "False", - "has_per_asic_scope": "True", - "has_timer": "False", - "high_mem_alert": "disabled", - "state": "enabled" - }, - "teamd": { - "auto_restart": "enabled", - "has_global_scope": "False", - "has_per_asic_scope": "True", - "has_timer": "False", - "high_mem_alert": "disabled", - "state": "enabled" - }, - "telemetry": { - "auto_restart": "enabled", - "has_global_scope": "True", - "has_per_asic_scope": "False", - "has_timer": "True", - "high_mem_alert": "disabled", - "state": "enabled" - }, - "what-just-happened": { - "auto_restart": "disabled", - "has_timer": "True", - "high_mem_alert": "disabled", - "state": "enabled" - } - }, - "FLEX_COUNTER_TABLE": { - "BUFFER_POOL_WATERMARK": { - "FLEX_COUNTER_STATUS": "enable" - }, - "PFCWD": { - "FLEX_COUNTER_STATUS": "enable" - }, - "PG_WATERMARK": { - "FLEX_COUNTER_STATUS": "enable" - }, - "PORT": { - "FLEX_COUNTER_STATUS": "enable" - }, - "PORT_BUFFER_DROP": { - "FLEX_COUNTER_STATUS": "enable" - }, - "QUEUE": { - "FLEX_COUNTER_STATUS": "enable" - }, - "QUEUE_WATERMARK": { - "FLEX_COUNTER_STATUS": "enable" - }, - "RIF": { - "FLEX_COUNTER_STATUS": "enable" - } - }, - "KDUMP": { - "config": { - "enabled": "false", - "num_dumps": "3", - "memory": "0M-2G:256M,2G-4G:320M,4G-8G:384M,8G-:448M" - } - }, - "MGMT_INTERFACE": { - "eth0|10.210.25.44/22": { - "gwaddr": "10.210.24.1" - } - }, - "PORT": { - "Ethernet0": { - "admin_status": "up", - "alias": "etp1", - "index": "1", - "lanes": "0,1,2,3", - "speed": "100000" - }, - "Ethernet12": { - "admin_status": "up", - "alias": "etp4a", - "index": "4", - "lanes": "12,13", - "speed": "50000" - }, - "Ethernet14": { - "admin_status": "up", - "alias": "etp4b", - "index": "4", - "lanes": "14,15", - "speed": "50000" - }, - "Ethernet16": { - "admin_status": "up", - "alias": "etp5a", - "index": "5", - "lanes": "16,17", - "speed": "50000" - }, - "Ethernet18": { - "admin_status": "up", - "alias": "etp5b", - "index": "5", - "lanes": "18,19", - "speed": "50000" - }, - "Ethernet20": { - "admin_status": "up", - "alias": "etp6a", - "index": "6", - "lanes": "20", - "speed": "25000" - }, - "Ethernet21": { - "admin_status": "up", - "alias": "etp6b", - "index": "6", - "lanes": "21", - "speed": "25000" - }, - "Ethernet22": { - "admin_status": "up", - "alias": "etp6c", - "index": "6", - "lanes": "22", - "speed": "25000" - }, - "Ethernet23": { - "admin_status": "up", - "alias": "etp6d", - "index": "6", - "lanes": "23", - "speed": "25000" - }, - "Ethernet24": { - "admin_status": "up", - "alias": "etp7a", - "index": "7", - "lanes": "24", - "speed": "25000" - }, - "Ethernet25": { - "admin_status": "up", - "alias": "etp7b", - "index": "7", - "lanes": "25", - "speed": "25000" - }, - "Ethernet26": { - "admin_status": "up", - "alias": "etp7c", - "index": "7", - "lanes": "26", - "speed": "25000" - }, - "Ethernet27": { - "admin_status": "up", - "alias": "etp7d", - "index": "7", - "lanes": "27", - "speed": "25000" - }, - "Ethernet28": { - "admin_status": "up", - "alias": "etp8", - "index": "8", - "lanes": "28,29,30,31", - "speed": "100000" - }, - "Ethernet32": { - "admin_status": "up", - "alias": "etp9", - "index": "9", - "lanes": "32,33,34,35", - "speed": "100000" - }, - "Ethernet36": { - "admin_status": "up", - "alias": "etp10", - "index": "10", - "lanes": "36,37,38,39", - "speed": "100000" - }, - "Ethernet4": { - "admin_status": "up", - "alias": "etp2", - "index": "2", - "lanes": "4,5,6,7", - "speed": "100000" - }, - "Ethernet40": { - "admin_status": "up", - "alias": "etp11", - "index": "11", - "lanes": "40,41,42,43", - "speed": "100000" - }, - "Ethernet44": { - "admin_status": "up", - "alias": "etp12", - "index": "12", - "lanes": "44,45,46,47", - "speed": "100000" - }, - "Ethernet48": { - "admin_status": "up", - "alias": "etp13", - "index": "13", - "lanes": "48,49,50,51", - "speed": "100000" - }, - "Ethernet52": { - "admin_status": "up", - "alias": "etp14", - "index": "14", - "lanes": "52,53,54,55", - "speed": "100000" - }, - "Ethernet56": { - "admin_status": "up", - "alias": "etp15", - "index": "15", - "lanes": "56,57,58,59", - "speed": "100000" - }, - "Ethernet60": { - "admin_status": "up", - "alias": "etp16", - "index": "16", - "lanes": "60,61,62,63", - "speed": "100000" - }, - "Ethernet8": { - "admin_status": "up", - "alias": "etp3", - "index": "3", - "lanes": "8,9,10,11", - "speed": "100000" - } - }, - "SNMP": { - "LOCATION": { - "Location": "public" - } - }, - "SNMP_COMMUNITY": { - "public": { - "TYPE": "RO" - } - }, - "VERSIONS": { - "DATABASE": { - "VERSION": "version_2_0_0" - } - }, - "WJH": { - "global": { - "mode": "debug", - "nice_level": "1", - "pci_bandwidth": "50" - } - }, - "WJH_CHANNEL": { - "forwarding": { - "drop_category_list": "L2,L3,Tunnel", - "type": "raw_and_aggregated" - }, - "layer-1": { - "drop_category_list": "L1", - "type": "raw_and_aggregated" - } - } -} diff --git a/tests/cli_autogen_yang_parser_test.py b/tests/cli_autogen_yang_parser_test.py index 9ed915c69b..2e0eee6c20 100644 --- a/tests/cli_autogen_yang_parser_test.py +++ b/tests/cli_autogen_yang_parser_test.py @@ -153,7 +153,7 @@ def base_test(yang_model_name, correct_dict): """ General logic for each test case """ config_db_path = os.path.join(test_path, - 'cli_autogen_input/config_db.json') + 'mock_tables/config_db.json') parser = YangParser(yang_model_name=yang_model_name, config_db_path=config_db_path, allow_tbl_without_yang=True, From 3cd8ad9a2b5cb927c3a15dfe9f765237f1d73286 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Tue, 7 Sep 2021 09:52:44 +0000 Subject: [PATCH 58/76] Added test case, added support for UT for sonic-cli-gen implementation Signed-off-by: Vadym Hlushko --- sonic_cli_gen/generator.py | 36 +++++++++++++++++++------- tests/cli_autogen_input/config_db.json | 14 ++++++++++ tests/cli_autogen_test.py | 35 +++++++++++++++++++++++++ tests/cli_autogen_yang_parser_test.py | 2 +- 4 files changed, 76 insertions(+), 11 deletions(-) create mode 100644 tests/cli_autogen_input/config_db.json create mode 100644 tests/cli_autogen_test.py diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py index 4f48b0201a..c4a6ddf64e 100644 --- a/sonic_cli_gen/generator.py +++ b/sonic_cli_gen/generator.py @@ -6,7 +6,10 @@ from sonic_cli_gen.yang_parser import YangParser -templates_path = '/usr/share/sonic/templates/sonic-cli-gen/' +templates_path_switch = '/usr/share/sonic/templates/sonic-cli-gen/' + +config_db_path_ut = '/sonic/src/sonic-utilities/tests/cli_autogen_input/config_db.json' +templates_path_ut = '/sonic/src/sonic-utilities/sonic-utilities-data/templates/sonic-cli-gen/' class CliGenerator: @@ -15,41 +18,54 @@ class CliGenerator: show CLI plugins. Attributes: - loader: the loaded j2 templates - env: j2 central object logger: logger """ def __init__(self, logger): """ Initialize CliGenerator. """ - self.loader = jinja2.FileSystemLoader(templates_path) - self.env = jinja2.Environment(loader=self.loader) self.logger = logger + def generate_cli_plugin(self, cli_group, plugin_name): """ Generate click CLI plugin and put it to: /usr/local/lib//dist-packages//plugins/auto/ """ - parser = YangParser(yang_model_name=plugin_name, - config_db_path='configDB', - allow_tbl_without_yang=True, - debug=False) + if os.environ["UTILITIES_UNIT_TESTING"] == "2": + config_db_path = config_db_path_ut + loader = jinja2.FileSystemLoader(templates_path_ut) + else: + config_db_path = 'configDB' + loader = jinja2.FileSystemLoader(templates_path_switch) + + parser = YangParser( + yang_model_name=plugin_name, + config_db_path=config_db_path, + allow_tbl_without_yang=True, + debug=False + ) # yang_dict will be used as an input for templates located in # /usr/share/sonic/templates/sonic-cli-gen/ yang_dict = parser.parse_yang_model() + + j2_env = jinja2.Environment(loader=loader) + template = j2_env.get_template(cli_group + '.py.j2') + plugin_path = get_cli_plugin_path(cli_group, plugin_name + '_yang.py') - template = self.env.get_template(cli_group + '.py.j2') + with open(plugin_path, 'w') as plugin_py: plugin_py.write(template.render(yang_dict)) self.logger.info(' Auto-generation successful! Location: {}'.format(plugin_path)) + def remove_cli_plugin(self, cli_group, plugin_name): """ Remove CLI plugin from directory: /usr/local/lib//dist-packages//plugins/auto/ """ + plugin_path = get_cli_plugin_path(cli_group, plugin_name + '_yang.py') + if os.path.exists(plugin_path): os.remove(plugin_path) self.logger.info(' {} was removed.'.format(plugin_path)) diff --git a/tests/cli_autogen_input/config_db.json b/tests/cli_autogen_input/config_db.json new file mode 100644 index 0000000000..d8d2efe021 --- /dev/null +++ b/tests/cli_autogen_input/config_db.json @@ -0,0 +1,14 @@ +{ + "DEVICE_METADATA": { + "localhost": { + "buffer_model": "traditional", + "default_bgp_status": "up", + "default_pfcwd_status": "disable", + "hostname": "r-sonic-01", + "hwsku": "ACS-MSN2100", + "mac": "ff:ff:ff:ff:ff:00", + "platform": "x86_64-mlnx_msn2100-r0", + "type": "ToRRouter" + } + } +} \ No newline at end of file diff --git a/tests/cli_autogen_test.py b/tests/cli_autogen_test.py new file mode 100644 index 0000000000..624de34fc2 --- /dev/null +++ b/tests/cli_autogen_test.py @@ -0,0 +1,35 @@ +import os +import logging +import pprint + +import sonic_cli_gen.main as cli_gen + +from sonic_cli_gen.generator import CliGenerator +from click.testing import CliRunner +from utilities_common.db import Db + +logger = logging.getLogger(__name__) + +test_path = os.path.dirname(os.path.abspath(__file__)) +yang_models_path = '/usr/local/yang-models' + + +class TestCliAutogen: + @classmethod + def setup_class(cls): + logger.info("SETUP") + os.environ['UTILITIES_UNIT_TESTING'] = "2" + + @classmethod + def teardown_class(cls): + logger.info("TEARDOWN") + os.environ['UTILITIES_UNIT_TESTING'] = "0" + + def test_one(self): + runner = CliRunner() + db = Db() + obj = {'config_db': db.cfgdb} + + gen = CliGenerator(logger) + res = gen.generate_cli_plugin('config', 'sonic-device_metadata') + diff --git a/tests/cli_autogen_yang_parser_test.py b/tests/cli_autogen_yang_parser_test.py index 2e0eee6c20..1a49d9a441 100644 --- a/tests/cli_autogen_yang_parser_test.py +++ b/tests/cli_autogen_yang_parser_test.py @@ -31,7 +31,7 @@ class TestYangParser: @classmethod def setup_class(cls): logger.info("SETUP") - os.environ['UTILITIES_UNIT_TESTING'] = "1" + os.environ['UTILITIES_UNIT_TESTING'] = "2" move_yang_models() @classmethod From 607b254c7d631fa65a1a92acbefea06c23463a58 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Tue, 7 Sep 2021 15:42:03 +0000 Subject: [PATCH 59/76] Added registering of plugins to UT Signed-off-by: Vadym Hlushko --- tests/cli_autogen_test.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/tests/cli_autogen_test.py b/tests/cli_autogen_test.py index 624de34fc2..a9662a56d4 100644 --- a/tests/cli_autogen_test.py +++ b/tests/cli_autogen_test.py @@ -2,7 +2,12 @@ import logging import pprint -import sonic_cli_gen.main as cli_gen +import show.plugins as show_plugins + +import show.main as show +import config.main as config + +from utilities_common import util_base from sonic_cli_gen.generator import CliGenerator from click.testing import CliRunner @@ -13,23 +18,35 @@ test_path = os.path.dirname(os.path.abspath(__file__)) yang_models_path = '/usr/local/yang-models' +show_device_metadata_localhost="""\ +HWSKU DEFAULT BGP STATUS DOCKER ROUTING CONFIG MODE HOSTNAME PLATFORM MAC DEFAULT PFCWD STATUS BGP ASN DEPLOYMENT ID TYPE BUFFER MODEL FRR MGMT FRAMEWORK CONFIG +---------------------- -------------------- ---------------------------- ------------ ---------------------- ----------------- ---------------------- --------- --------------- --------- -------------- --------------------------- +Mellanox-SN3800-D112C8 down separated sonic-switch x86_64-mlnx_msn3800-r0 1d:34:db:16:a6:00 enable N/A 1 ToRRouter N/A N/A +""" + +gen = CliGenerator(logger) class TestCliAutogen: @classmethod def setup_class(cls): logger.info("SETUP") os.environ['UTILITIES_UNIT_TESTING'] = "2" + gen.generate_cli_plugin('show', 'sonic-device_metadata') + helper = util_base.UtilHelper() + for plugin in helper.load_plugins(show_plugins): + helper.register_plugin(plugin, show.cli) @classmethod def teardown_class(cls): logger.info("TEARDOWN") + gen.remove_cli_plugin('show', 'sonic-device_metadata') os.environ['UTILITIES_UNIT_TESTING'] = "0" def test_one(self): runner = CliRunner() - db = Db() - obj = {'config_db': db.cfgdb} - gen = CliGenerator(logger) - res = gen.generate_cli_plugin('config', 'sonic-device_metadata') + result = runner.invoke(show.cli.commands['device-metadata'].commands['localhost'], []) + logger.debug(result.output) + assert result.output == show_device_metadata_localhost + From 6816903420e04cd9af3a491e88f699267959838c Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Tue, 7 Sep 2021 15:45:03 +0000 Subject: [PATCH 60/76] Fixed name Signed-off-by: Vadym Hlushko --- tests/cli_autogen_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/cli_autogen_test.py b/tests/cli_autogen_test.py index a9662a56d4..7d35ff014d 100644 --- a/tests/cli_autogen_test.py +++ b/tests/cli_autogen_test.py @@ -4,8 +4,8 @@ import show.plugins as show_plugins -import show.main as show -import config.main as config +import show.main as show_main +import config.main as config_main from utilities_common import util_base @@ -34,7 +34,7 @@ def setup_class(cls): gen.generate_cli_plugin('show', 'sonic-device_metadata') helper = util_base.UtilHelper() for plugin in helper.load_plugins(show_plugins): - helper.register_plugin(plugin, show.cli) + helper.register_plugin(plugin, show_main.cli) @classmethod def teardown_class(cls): @@ -45,7 +45,7 @@ def teardown_class(cls): def test_one(self): runner = CliRunner() - result = runner.invoke(show.cli.commands['device-metadata'].commands['localhost'], []) + result = runner.invoke(show_main.cli.commands['device-metadata'].commands['localhost'], []) logger.debug(result.output) assert result.output == show_device_metadata_localhost From 2aee8837223d3f709ca685681d4ce205d57c883f Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Wed, 8 Sep 2021 08:47:40 +0000 Subject: [PATCH 61/76] Added helper.load_and_register_plugins() Signed-off-by: Vadym Hlushko --- clear/main.py | 3 +-- config/main.py | 3 +-- show/main.py | 3 +-- tests/cli_autogen_test.py | 25 ++++++++----------------- utilities_common/util_base.py | 6 ++++++ 5 files changed, 17 insertions(+), 23 deletions(-) diff --git a/clear/main.py b/clear/main.py index 4302ae00aa..bb7e475630 100755 --- a/clear/main.py +++ b/clear/main.py @@ -452,8 +452,7 @@ def translations(): # Load plugins and register them helper = util_base.UtilHelper() -for plugin in helper.load_plugins(plugins): - helper.register_plugin(plugin, cli) +helper.load_and_register_plugins(plugins, cli) if __name__ == '__main__': diff --git a/config/main.py b/config/main.py index e9bab3172d..a728c7bd32 100644 --- a/config/main.py +++ b/config/main.py @@ -4562,8 +4562,7 @@ def delete(ctx): # Load plugins and register them helper = util_base.UtilHelper() -for plugin in helper.load_plugins(plugins): - helper.register_plugin(plugin, config) +helper.load_and_register_plugins(plugins, config) if __name__ == '__main__': diff --git a/show/main.py b/show/main.py index b0b2986a78..9afb0217e9 100755 --- a/show/main.py +++ b/show/main.py @@ -1473,8 +1473,7 @@ def ztp(status, verbose): # Load plugins and register them helper = util_base.UtilHelper() -for plugin in helper.load_plugins(plugins): - helper.register_plugin(plugin, cli) +helper.load_and_register_plugins(plugins, cli) if __name__ == '__main__': diff --git a/tests/cli_autogen_test.py b/tests/cli_autogen_test.py index 7d35ff014d..7265ef4df1 100644 --- a/tests/cli_autogen_test.py +++ b/tests/cli_autogen_test.py @@ -3,29 +3,22 @@ import pprint import show.plugins as show_plugins - import show.main as show_main import config.main as config_main from utilities_common import util_base - from sonic_cli_gen.generator import CliGenerator from click.testing import CliRunner -from utilities_common.db import Db logger = logging.getLogger(__name__) - -test_path = os.path.dirname(os.path.abspath(__file__)) -yang_models_path = '/usr/local/yang-models' - -show_device_metadata_localhost="""\ -HWSKU DEFAULT BGP STATUS DOCKER ROUTING CONFIG MODE HOSTNAME PLATFORM MAC DEFAULT PFCWD STATUS BGP ASN DEPLOYMENT ID TYPE BUFFER MODEL FRR MGMT FRAMEWORK CONFIG ----------------------- -------------------- ---------------------------- ------------ ---------------------- ----------------- ---------------------- --------- --------------- --------- -------------- --------------------------- -Mellanox-SN3800-D112C8 down separated sonic-switch x86_64-mlnx_msn3800-r0 1d:34:db:16:a6:00 enable N/A 1 ToRRouter N/A N/A -""" - gen = CliGenerator(logger) +#show_device_metadata_localhost="""\ +#HWSKU DEFAULT BGP STATUS DOCKER ROUTING CONFIG MODE HOSTNAME PLATFORM MAC DEFAULT PFCWD STATUS BGP ASN DEPLOYMENT ID TYPE BUFFER MODEL FRR MGMT FRAMEWORK CONFIG +#---------------------- -------------------- ---------------------------- ------------ ---------------------- ----------------- ---------------------- --------- --------------- --------- -------------- --------------------------- +#Mellanox-SN3800-D112C8 down separated sonic-switch x86_64-mlnx_msn3800-r0 1d:34:db:16:a6:00 enable N/A 1 ToRRouter N/A N/A +#""" + class TestCliAutogen: @classmethod def setup_class(cls): @@ -33,8 +26,7 @@ def setup_class(cls): os.environ['UTILITIES_UNIT_TESTING'] = "2" gen.generate_cli_plugin('show', 'sonic-device_metadata') helper = util_base.UtilHelper() - for plugin in helper.load_plugins(show_plugins): - helper.register_plugin(plugin, show_main.cli) + helper.load_and_register_plugins(show_plugins, show_main.cli) @classmethod def teardown_class(cls): @@ -47,6 +39,5 @@ def test_one(self): result = runner.invoke(show_main.cli.commands['device-metadata'].commands['localhost'], []) logger.debug(result.output) - assert result.output == show_device_metadata_localhost - + #assert result.output == show_device_metadata_localhost diff --git a/utilities_common/util_base.py b/utilities_common/util_base.py index 9bea158b59..98fc230629 100644 --- a/utilities_common/util_base.py +++ b/utilities_common/util_base.py @@ -83,3 +83,9 @@ def check_pddf_mode(self): return True else: return False + + def load_and_register_plugins(self, plugins, cli): + """ Load plugins and register them """ + + for plugin in self.load_plugins(plugins): + self.register_plugin(plugin, cli) \ No newline at end of file From 9fea6dbd4d8c32f6feca493c841e3c3e2248ea60 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Thu, 9 Sep 2021 11:23:07 +0000 Subject: [PATCH 62/76] Added test cases for sonic-device_metadata Signed-off-by: Vadym Hlushko --- .../autogen_test/show_cmd_output.py | 16 +++ .../autogen_test/sonic-device_metadata.yang | 123 ++++++++++++++++++ tests/cli_autogen_input/config_db.json | 20 ++- tests/cli_autogen_test.py | 91 +++++++++++-- 4 files changed, 227 insertions(+), 23 deletions(-) create mode 100644 tests/cli_autogen_input/autogen_test/show_cmd_output.py create mode 100644 tests/cli_autogen_input/autogen_test/sonic-device_metadata.yang diff --git a/tests/cli_autogen_input/autogen_test/show_cmd_output.py b/tests/cli_autogen_input/autogen_test/show_cmd_output.py new file mode 100644 index 0000000000..cb0632f9ed --- /dev/null +++ b/tests/cli_autogen_input/autogen_test/show_cmd_output.py @@ -0,0 +1,16 @@ +""" +Module holding correct output for the show command for cli_autogen_test.py +""" + + +show_device_metadata_localhost="""\ +HWSKU DEFAULT BGP STATUS DOCKER ROUTING CONFIG MODE HOSTNAME PLATFORM MAC DEFAULT PFCWD STATUS BGP ASN DEPLOYMENT ID TYPE BUFFER MODEL FRR MGMT FRAMEWORK CONFIG +----------- -------------------- ---------------------------- ---------- ---------------------- ----------------- ---------------------- --------- --------------- --------- -------------- --------------------------- +ACS-MSN2100 up N/A r-sonic-01 x86_64-mlnx_msn2100-r0 ff:ff:ff:ff:ff:00 disable N/A N/A ToRRouter traditional N/A +""" + +show_device_metadata_localhost_changed_buffer_model="""\ +HWSKU DEFAULT BGP STATUS DOCKER ROUTING CONFIG MODE HOSTNAME PLATFORM MAC DEFAULT PFCWD STATUS BGP ASN DEPLOYMENT ID TYPE BUFFER MODEL FRR MGMT FRAMEWORK CONFIG +----------- -------------------- ---------------------------- ---------- ---------------------- ----------------- ---------------------- --------- --------------- --------- -------------- --------------------------- +ACS-MSN2100 up N/A r-sonic-01 x86_64-mlnx_msn2100-r0 ff:ff:ff:ff:ff:00 disable N/A N/A ToRRouter dynamic N/A +""" \ No newline at end of file diff --git a/tests/cli_autogen_input/autogen_test/sonic-device_metadata.yang b/tests/cli_autogen_input/autogen_test/sonic-device_metadata.yang new file mode 100644 index 0000000000..400cbf3bcd --- /dev/null +++ b/tests/cli_autogen_input/autogen_test/sonic-device_metadata.yang @@ -0,0 +1,123 @@ +module sonic-device_metadata { + + yang-version 1.1; + + namespace "http://github.com/Azure/sonic-device_metadata"; + prefix device_metadata; + + import ietf-yang-types { + prefix yang; + } + + import ietf-inet-types { + prefix inet; + } + + import sonic-types { + prefix stypes; + revision-date 2019-07-01; + } + + description "DEVICE_METADATA YANG Module for SONiC OS"; + + revision 2021-02-27 { + description "Added frr_mgmt_framework_config field to handle BGP + config DB schema events to configure FRR protocols."; + } + + revision 2020-04-10 { + description "First Revision"; + } + + container sonic-device_metadata { + + container DEVICE_METADATA { + + description "DEVICE_METADATA part of config_db.json"; + + container localhost{ + + leaf hwsku { + type stypes:hwsku; + } + + leaf default_bgp_status { + type enumeration { + enum up; + enum down; + } + default up; + } + + leaf docker_routing_config_mode { + type string { + pattern "unified|split|separated"; + } + default "unified"; + } + + leaf hostname { + type string { + length 1..255; + } + } + + leaf platform { + type string { + length 1..255; + } + } + + leaf mac { + type yang:mac-address; + } + + leaf default_pfcwd_status { + type enumeration { + enum disable; + enum enable; + } + default disable; + } + + leaf bgp_asn { + type inet:as-number; + } + + leaf deployment_id { + type uint32; + } + + leaf type { + type string { + length 1..255; + pattern "ToRRouter|LeafRouter|SpineChassisFrontendRouter|ChassisBackendRouter|ASIC"; + } + } + + leaf buffer_model { + description "This leaf is added for dynamic buffer calculation. + The dynamic model represents the model in which the buffer configurations, + like the headroom sizes and buffer pool sizes, are dynamically calculated based + on the ports' speed, cable length, and MTU. This model is used by Mellanox so far. + The traditional model represents the model in which all the buffer configurations + are statically configured in CONFIG_DB tables. This is the default model used by all other vendors"; + type string { + pattern "dynamic|traditional"; + } + } + + leaf frr_mgmt_framework_config { + type boolean; + description "FRR configurations are handled by sonic-frr-mgmt-framework module when set to true, + otherwise, sonic-bgpcfgd handles the FRR configurations based on the predefined templates."; + default "false"; + } + } + /* end of container localhost */ + } + /* end of container DEVICE_METADATA */ + } + /* end of top level container */ +} +/* end of module sonic-device_metadata */ diff --git a/tests/cli_autogen_input/config_db.json b/tests/cli_autogen_input/config_db.json index d8d2efe021..7b2f72b815 100644 --- a/tests/cli_autogen_input/config_db.json +++ b/tests/cli_autogen_input/config_db.json @@ -1,14 +1,12 @@ { - "DEVICE_METADATA": { - "localhost": { - "buffer_model": "traditional", - "default_bgp_status": "up", - "default_pfcwd_status": "disable", - "hostname": "r-sonic-01", - "hwsku": "ACS-MSN2100", - "mac": "ff:ff:ff:ff:ff:00", - "platform": "x86_64-mlnx_msn2100-r0", - "type": "ToRRouter" - } + "DEVICE_METADATA|localhost": { + "buffer_model": "traditional", + "default_bgp_status": "up", + "default_pfcwd_status": "disable", + "hostname": "r-sonic-01", + "hwsku": "ACS-MSN2100", + "mac": "ff:ff:ff:ff:ff:00", + "platform": "x86_64-mlnx_msn2100-r0", + "type": "ToRRouter" } } \ No newline at end of file diff --git a/tests/cli_autogen_test.py b/tests/cli_autogen_test.py index 7265ef4df1..5d9958454e 100644 --- a/tests/cli_autogen_test.py +++ b/tests/cli_autogen_test.py @@ -1,43 +1,110 @@ import os import logging -import pprint +import pytest import show.plugins as show_plugins import show.main as show_main +import config.plugins as config_plugins import config.main as config_main +from .cli_autogen_input.autogen_test import show_cmd_output from utilities_common import util_base from sonic_cli_gen.generator import CliGenerator +from .mock_tables import dbconnector +from utilities_common.db import Db from click.testing import CliRunner logger = logging.getLogger(__name__) gen = CliGenerator(logger) -#show_device_metadata_localhost="""\ -#HWSKU DEFAULT BGP STATUS DOCKER ROUTING CONFIG MODE HOSTNAME PLATFORM MAC DEFAULT PFCWD STATUS BGP ASN DEPLOYMENT ID TYPE BUFFER MODEL FRR MGMT FRAMEWORK CONFIG -#---------------------- -------------------- ---------------------------- ------------ ---------------------- ----------------- ---------------------- --------- --------------- --------- -------------- --------------------------- -#Mellanox-SN3800-D112C8 down separated sonic-switch x86_64-mlnx_msn3800-r0 1d:34:db:16:a6:00 enable N/A 1 ToRRouter N/A N/A -#""" +test_path = os.path.dirname(os.path.abspath(__file__)) +mock_db_path = os.path.join(test_path, 'cli_autogen_input') + +SUCCESS = 0 +ERROR = 1 +INVALID_VALUE = 'INVALID' + class TestCliAutogen: @classmethod def setup_class(cls): - logger.info("SETUP") + logger.info('SETUP') os.environ['UTILITIES_UNIT_TESTING'] = "2" + gen.generate_cli_plugin('show', 'sonic-device_metadata') + gen.generate_cli_plugin('config', 'sonic-device_metadata') + helper = util_base.UtilHelper() helper.load_and_register_plugins(show_plugins, show_main.cli) + helper.load_and_register_plugins(config_plugins, config_main.config) + @classmethod def teardown_class(cls): - logger.info("TEARDOWN") + logger.info('TEARDOWN') + gen.remove_cli_plugin('show', 'sonic-device_metadata') + gen.remove_cli_plugin('config', 'sonic-device_metadata') + + dbconnector.dedicated_dbs['CONFIG_DB'] = None + os.environ['UTILITIES_UNIT_TESTING'] = "0" - def test_one(self): + + def test_show_device_metadata(self): + dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db') + db = Db() runner = CliRunner() - result = runner.invoke(show_main.cli.commands['device-metadata'].commands['localhost'], []) - logger.debug(result.output) - #assert result.output == show_device_metadata_localhost + result = runner.invoke( + show_main.cli.commands['device-metadata'].commands['localhost'], [], obj=db + ) + + logger.debug("\n" + result.output) + logger.debug(result.exit_code) + assert result.exit_code == SUCCESS + assert result.output == show_cmd_output.show_device_metadata_localhost + + + def test_config_device_metadata(self): + dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db') + db = Db() + runner = CliRunner() + + result = runner.invoke( + config_main.config.commands['device-metadata'].commands['localhost'].commands['buffer-model'], ['dynamic'], obj=db + ) + + result = runner.invoke( + show_main.cli.commands['device-metadata'].commands['localhost'], [], obj=db + ) + + logger.debug("\n" + result.output) + logger.debug(result.exit_code) + assert result.exit_code == SUCCESS + assert result.output == show_cmd_output.show_device_metadata_localhost_changed_buffer_model + + + @pytest.mark.parametrize("parameter,value", [ + ('default-bgp-status', INVALID_VALUE), + ('docker-routing-config-mode', INVALID_VALUE), + ('mac', INVALID_VALUE), + ('default-pfcwd-status', INVALID_VALUE), + ('bgp-asn', INVALID_VALUE), + ('type', INVALID_VALUE), + ('buffer-model', INVALID_VALUE), + ('frr-mgmt-framework-config', INVALID_VALUE) + ]) + def test_config_device_metadata_invalid(self, parameter, value): + dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db') + db = Db() + runner = CliRunner() + + result = runner.invoke( + config_main.config.commands['device-metadata'].commands['localhost'].commands[parameter], [value], obj=db + ) + + logger.debug("\n" + result.output) + logger.debug(result.exit_code) + assert result.exit_code == ERROR From 69926ed309b0254437eed5df3c4be96b7dfce3bd Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Fri, 10 Sep 2021 14:02:48 +0000 Subject: [PATCH 63/76] Added cli_autogen_common.py, added separate dir for each cli_autogen test Signed-off-by: Vadym Hlushko --- tests/cli_autogen_input/cli_autogen_common.py | 24 +++++++ .../assert_dictionaries.py | 0 .../{ => yang_parser_test}/sonic-1-list.yang | 0 .../sonic-1-object-container.yang | 0 .../sonic-1-table-container.yang | 0 .../{ => yang_parser_test}/sonic-2-lists.yang | 0 .../sonic-2-object-containers.yang | 0 .../sonic-2-table-containers.yang | 0 .../sonic-choice-complex.yang | 0 .../sonic-dynamic-object-complex-1.yang | 0 .../sonic-dynamic-object-complex-2.yang | 0 .../sonic-grouping-1.yang | 0 .../sonic-grouping-2.yang | 0 .../sonic-grouping-complex.yang | 0 .../sonic-static-object-complex-1.yang | 0 .../sonic-static-object-complex-2.yang | 0 tests/cli_autogen_yang_parser_test.py | 62 ++++++------------- 17 files changed, 43 insertions(+), 43 deletions(-) create mode 100644 tests/cli_autogen_input/cli_autogen_common.py rename tests/cli_autogen_input/{ => yang_parser_test}/assert_dictionaries.py (100%) rename tests/cli_autogen_input/{ => yang_parser_test}/sonic-1-list.yang (100%) rename tests/cli_autogen_input/{ => yang_parser_test}/sonic-1-object-container.yang (100%) rename tests/cli_autogen_input/{ => yang_parser_test}/sonic-1-table-container.yang (100%) rename tests/cli_autogen_input/{ => yang_parser_test}/sonic-2-lists.yang (100%) rename tests/cli_autogen_input/{ => yang_parser_test}/sonic-2-object-containers.yang (100%) rename tests/cli_autogen_input/{ => yang_parser_test}/sonic-2-table-containers.yang (100%) rename tests/cli_autogen_input/{ => yang_parser_test}/sonic-choice-complex.yang (100%) rename tests/cli_autogen_input/{ => yang_parser_test}/sonic-dynamic-object-complex-1.yang (100%) rename tests/cli_autogen_input/{ => yang_parser_test}/sonic-dynamic-object-complex-2.yang (100%) rename tests/cli_autogen_input/{ => yang_parser_test}/sonic-grouping-1.yang (100%) rename tests/cli_autogen_input/{ => yang_parser_test}/sonic-grouping-2.yang (100%) rename tests/cli_autogen_input/{ => yang_parser_test}/sonic-grouping-complex.yang (100%) rename tests/cli_autogen_input/{ => yang_parser_test}/sonic-static-object-complex-1.yang (100%) rename tests/cli_autogen_input/{ => yang_parser_test}/sonic-static-object-complex-2.yang (100%) diff --git a/tests/cli_autogen_input/cli_autogen_common.py b/tests/cli_autogen_input/cli_autogen_common.py new file mode 100644 index 0000000000..bdc44bfa4f --- /dev/null +++ b/tests/cli_autogen_input/cli_autogen_common.py @@ -0,0 +1,24 @@ +import os + +yang_models_path = '/usr/local/yang-models' + + +def move_yang_models(test_path, test_name, test_yang_models): + """ Move a test YANG models to known location """ + + for yang_model in test_yang_models: + src_path = os.path.join(test_path, + 'cli_autogen_input', + test_name, + yang_model) + cmd = 'sudo cp {} {}'.format(src_path, yang_models_path) + os.system(cmd) + + +def remove_yang_models(test_yang_models): + """ Remove a test YANG models to known location """ + + for yang_model in test_yang_models: + yang_model_path = os.path.join(yang_models_path, yang_model) + cmd = 'sudo rm {}'.format(yang_model_path) + os.system(cmd) diff --git a/tests/cli_autogen_input/assert_dictionaries.py b/tests/cli_autogen_input/yang_parser_test/assert_dictionaries.py similarity index 100% rename from tests/cli_autogen_input/assert_dictionaries.py rename to tests/cli_autogen_input/yang_parser_test/assert_dictionaries.py diff --git a/tests/cli_autogen_input/sonic-1-list.yang b/tests/cli_autogen_input/yang_parser_test/sonic-1-list.yang similarity index 100% rename from tests/cli_autogen_input/sonic-1-list.yang rename to tests/cli_autogen_input/yang_parser_test/sonic-1-list.yang diff --git a/tests/cli_autogen_input/sonic-1-object-container.yang b/tests/cli_autogen_input/yang_parser_test/sonic-1-object-container.yang similarity index 100% rename from tests/cli_autogen_input/sonic-1-object-container.yang rename to tests/cli_autogen_input/yang_parser_test/sonic-1-object-container.yang diff --git a/tests/cli_autogen_input/sonic-1-table-container.yang b/tests/cli_autogen_input/yang_parser_test/sonic-1-table-container.yang similarity index 100% rename from tests/cli_autogen_input/sonic-1-table-container.yang rename to tests/cli_autogen_input/yang_parser_test/sonic-1-table-container.yang diff --git a/tests/cli_autogen_input/sonic-2-lists.yang b/tests/cli_autogen_input/yang_parser_test/sonic-2-lists.yang similarity index 100% rename from tests/cli_autogen_input/sonic-2-lists.yang rename to tests/cli_autogen_input/yang_parser_test/sonic-2-lists.yang diff --git a/tests/cli_autogen_input/sonic-2-object-containers.yang b/tests/cli_autogen_input/yang_parser_test/sonic-2-object-containers.yang similarity index 100% rename from tests/cli_autogen_input/sonic-2-object-containers.yang rename to tests/cli_autogen_input/yang_parser_test/sonic-2-object-containers.yang diff --git a/tests/cli_autogen_input/sonic-2-table-containers.yang b/tests/cli_autogen_input/yang_parser_test/sonic-2-table-containers.yang similarity index 100% rename from tests/cli_autogen_input/sonic-2-table-containers.yang rename to tests/cli_autogen_input/yang_parser_test/sonic-2-table-containers.yang diff --git a/tests/cli_autogen_input/sonic-choice-complex.yang b/tests/cli_autogen_input/yang_parser_test/sonic-choice-complex.yang similarity index 100% rename from tests/cli_autogen_input/sonic-choice-complex.yang rename to tests/cli_autogen_input/yang_parser_test/sonic-choice-complex.yang diff --git a/tests/cli_autogen_input/sonic-dynamic-object-complex-1.yang b/tests/cli_autogen_input/yang_parser_test/sonic-dynamic-object-complex-1.yang similarity index 100% rename from tests/cli_autogen_input/sonic-dynamic-object-complex-1.yang rename to tests/cli_autogen_input/yang_parser_test/sonic-dynamic-object-complex-1.yang diff --git a/tests/cli_autogen_input/sonic-dynamic-object-complex-2.yang b/tests/cli_autogen_input/yang_parser_test/sonic-dynamic-object-complex-2.yang similarity index 100% rename from tests/cli_autogen_input/sonic-dynamic-object-complex-2.yang rename to tests/cli_autogen_input/yang_parser_test/sonic-dynamic-object-complex-2.yang diff --git a/tests/cli_autogen_input/sonic-grouping-1.yang b/tests/cli_autogen_input/yang_parser_test/sonic-grouping-1.yang similarity index 100% rename from tests/cli_autogen_input/sonic-grouping-1.yang rename to tests/cli_autogen_input/yang_parser_test/sonic-grouping-1.yang diff --git a/tests/cli_autogen_input/sonic-grouping-2.yang b/tests/cli_autogen_input/yang_parser_test/sonic-grouping-2.yang similarity index 100% rename from tests/cli_autogen_input/sonic-grouping-2.yang rename to tests/cli_autogen_input/yang_parser_test/sonic-grouping-2.yang diff --git a/tests/cli_autogen_input/sonic-grouping-complex.yang b/tests/cli_autogen_input/yang_parser_test/sonic-grouping-complex.yang similarity index 100% rename from tests/cli_autogen_input/sonic-grouping-complex.yang rename to tests/cli_autogen_input/yang_parser_test/sonic-grouping-complex.yang diff --git a/tests/cli_autogen_input/sonic-static-object-complex-1.yang b/tests/cli_autogen_input/yang_parser_test/sonic-static-object-complex-1.yang similarity index 100% rename from tests/cli_autogen_input/sonic-static-object-complex-1.yang rename to tests/cli_autogen_input/yang_parser_test/sonic-static-object-complex-1.yang diff --git a/tests/cli_autogen_input/sonic-static-object-complex-2.yang b/tests/cli_autogen_input/yang_parser_test/sonic-static-object-complex-2.yang similarity index 100% rename from tests/cli_autogen_input/sonic-static-object-complex-2.yang rename to tests/cli_autogen_input/yang_parser_test/sonic-static-object-complex-2.yang diff --git a/tests/cli_autogen_yang_parser_test.py b/tests/cli_autogen_yang_parser_test.py index 1a49d9a441..ed82693e91 100644 --- a/tests/cli_autogen_yang_parser_test.py +++ b/tests/cli_autogen_yang_parser_test.py @@ -3,27 +3,28 @@ import pprint from sonic_cli_gen.yang_parser import YangParser -from .cli_autogen_input import assert_dictionaries +from .cli_autogen_input.yang_parser_test import assert_dictionaries +from .cli_autogen_input.cli_autogen_common import move_yang_models, remove_yang_models logger = logging.getLogger(__name__) test_path = os.path.dirname(os.path.abspath(__file__)) -yang_models_path = '/usr/local/yang-models' + test_yang_models = [ - 'sonic-1-table-container', - 'sonic-2-table-containers', - 'sonic-1-object-container', - 'sonic-2-object-containers', - 'sonic-1-list', - 'sonic-2-lists', - 'sonic-static-object-complex-1', - 'sonic-static-object-complex-2', - 'sonic-dynamic-object-complex-1', - 'sonic-dynamic-object-complex-2', - 'sonic-choice-complex', - 'sonic-grouping-complex', - 'sonic-grouping-1', - 'sonic-grouping-2', + 'sonic-1-table-container.yang', + 'sonic-2-table-containers.yang', + 'sonic-1-object-container.yang', + 'sonic-2-object-containers.yang', + 'sonic-1-list.yang', + 'sonic-2-lists.yang', + 'sonic-static-object-complex-1.yang', + 'sonic-static-object-complex-2.yang', + 'sonic-dynamic-object-complex-1.yang', + 'sonic-dynamic-object-complex-2.yang', + 'sonic-choice-complex.yang', + 'sonic-grouping-complex.yang', + 'sonic-grouping-1.yang', + 'sonic-grouping-2.yang', ] @@ -32,13 +33,13 @@ class TestYangParser: def setup_class(cls): logger.info("SETUP") os.environ['UTILITIES_UNIT_TESTING'] = "2" - move_yang_models() + move_yang_models(test_path, 'yang_parser_test', test_yang_models) @classmethod def teardown_class(cls): logger.info("TEARDOWN") os.environ['UTILITIES_UNIT_TESTING'] = "0" - remove_yang_models() + remove_yang_models(test_yang_models) def test_1_table_container(self): """ Test for 1 'table' container @@ -163,31 +164,6 @@ def base_test(yang_model_name, correct_dict): assert yang_dict == correct_dict -def move_yang_models(): - """ Move a test YANG models to known location - in order to be parsed by YangParser class - """ - - for yang_model in test_yang_models: - src_path = os.path.join(test_path, - 'cli_autogen_input', - yang_model + '.yang') - cmd = 'sudo cp {} {}'.format(src_path, yang_models_path) - os.system(cmd) - - -def remove_yang_models(): - """ Remove a test YANG models to known location - in order to be parsed by YangParser class - """ - - for yang_model in test_yang_models: - yang_model_path = os.path.join(yang_models_path, - yang_model + '.yang') - cmd = 'sudo rm {}'.format(yang_model_path) - os.system(cmd) - - def pretty_log_debug(dictionary): """ Pretty print of parsed dictionary """ From dd7ec5ea20630a24b8afdd5684529bf5bf1e7745 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Fri, 10 Sep 2021 14:40:53 +0000 Subject: [PATCH 64/76] Added backup and restore func for yang models Signed-off-by: Vadym Hlushko --- tests/cli_autogen_input/cli_autogen_common.py | 15 +++++++++++++++ tests/cli_autogen_test.py | 4 ++++ 2 files changed, 19 insertions(+) diff --git a/tests/cli_autogen_input/cli_autogen_common.py b/tests/cli_autogen_input/cli_autogen_common.py index bdc44bfa4f..31895d62ad 100644 --- a/tests/cli_autogen_input/cli_autogen_common.py +++ b/tests/cli_autogen_input/cli_autogen_common.py @@ -22,3 +22,18 @@ def remove_yang_models(test_yang_models): yang_model_path = os.path.join(yang_models_path, yang_model) cmd = 'sudo rm {}'.format(yang_model_path) os.system(cmd) + + +def backup_yang_models(): + """ Make a copy of existing YANG models """ + + cmd = 'sudo cp -R {} {}'.format(yang_models_path, yang_models_path + '_backup') + os.system(cmd) + + +def restore_backup_yang_models(): + """ Restore existing YANG models from backup """ + + cmd = 'sudo mv {} {}'.format(yang_models_path + '_backup/', '/usr/local/yang-models') + os.system(cmd) + \ No newline at end of file diff --git a/tests/cli_autogen_test.py b/tests/cli_autogen_test.py index 5d9958454e..cd5933249f 100644 --- a/tests/cli_autogen_test.py +++ b/tests/cli_autogen_test.py @@ -7,6 +7,7 @@ import config.plugins as config_plugins import config.main as config_main from .cli_autogen_input.autogen_test import show_cmd_output +from .cli_autogen_input.cli_autogen_common import backup_yang_models, restore_backup_yang_models from utilities_common import util_base from sonic_cli_gen.generator import CliGenerator @@ -108,3 +109,6 @@ def test_config_device_metadata_invalid(self, parameter, value): logger.debug(result.exit_code) assert result.exit_code == ERROR + def test_one(self): + backup_yang_models() + restore_backup_yang_models() From eb852009d876525d5e91c7f1cfe5fba7538e69fa Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Fri, 10 Sep 2021 14:51:34 +0000 Subject: [PATCH 65/76] Removed if-else UT login from generator.py Signed-off-by: Vadym Hlushko --- sonic_cli_gen/generator.py | 21 ++++++++++----------- tests/cli_autogen_test.py | 24 +++++++++++++++++++----- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py index c4a6ddf64e..8f23daa6ac 100644 --- a/sonic_cli_gen/generator.py +++ b/sonic_cli_gen/generator.py @@ -8,8 +8,7 @@ templates_path_switch = '/usr/share/sonic/templates/sonic-cli-gen/' -config_db_path_ut = '/sonic/src/sonic-utilities/tests/cli_autogen_input/config_db.json' -templates_path_ut = '/sonic/src/sonic-utilities/sonic-utilities-data/templates/sonic-cli-gen/' + class CliGenerator: @@ -27,28 +26,28 @@ def __init__(self, logger): self.logger = logger - def generate_cli_plugin(self, cli_group, plugin_name): + def generate_cli_plugin( + self, + cli_group, + plugin_name, + config_db_path='configDB', + templates_path='/usr/share/sonic/templates/sonic-cli-gen/' + ): """ Generate click CLI plugin and put it to: /usr/local/lib//dist-packages//plugins/auto/ """ - if os.environ["UTILITIES_UNIT_TESTING"] == "2": - config_db_path = config_db_path_ut - loader = jinja2.FileSystemLoader(templates_path_ut) - else: - config_db_path = 'configDB' - loader = jinja2.FileSystemLoader(templates_path_switch) - parser = YangParser( yang_model_name=plugin_name, config_db_path=config_db_path, allow_tbl_without_yang=True, debug=False - ) + ) # yang_dict will be used as an input for templates located in # /usr/share/sonic/templates/sonic-cli-gen/ yang_dict = parser.parse_yang_model() + loader = jinja2.FileSystemLoader(templates_path) j2_env = jinja2.Environment(loader=loader) template = j2_env.get_template(cli_group + '.py.j2') diff --git a/tests/cli_autogen_test.py b/tests/cli_autogen_test.py index cd5933249f..846a289a8e 100644 --- a/tests/cli_autogen_test.py +++ b/tests/cli_autogen_test.py @@ -25,6 +25,9 @@ ERROR = 1 INVALID_VALUE = 'INVALID' +config_db_path = '/sonic/src/sonic-utilities/tests/cli_autogen_input/config_db.json' +templates_path = '/sonic/src/sonic-utilities/sonic-utilities-data/templates/sonic-cli-gen/' + class TestCliAutogen: @classmethod @@ -32,8 +35,20 @@ def setup_class(cls): logger.info('SETUP') os.environ['UTILITIES_UNIT_TESTING'] = "2" - gen.generate_cli_plugin('show', 'sonic-device_metadata') - gen.generate_cli_plugin('config', 'sonic-device_metadata') + backup_yang_models() + + gen.generate_cli_plugin( + cli_group='show', + plugin_name='sonic-device_metadata', + config_db_path=config_db_path, + templates_path=templates_path + ) + gen.generate_cli_plugin( + cli_group='config', + plugin_name='sonic-device_metadata', + config_db_path=config_db_path, + templates_path=templates_path + ) helper = util_base.UtilHelper() helper.load_and_register_plugins(show_plugins, show_main.cli) @@ -47,6 +62,8 @@ def teardown_class(cls): gen.remove_cli_plugin('show', 'sonic-device_metadata') gen.remove_cli_plugin('config', 'sonic-device_metadata') + restore_backup_yang_models() + dbconnector.dedicated_dbs['CONFIG_DB'] = None os.environ['UTILITIES_UNIT_TESTING'] = "0" @@ -109,6 +126,3 @@ def test_config_device_metadata_invalid(self, parameter, value): logger.debug(result.exit_code) assert result.exit_code == ERROR - def test_one(self): - backup_yang_models() - restore_backup_yang_models() From 5ccefa615a85ad55381ad8092d3a31604fe28254 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Mon, 13 Sep 2021 14:50:43 +0000 Subject: [PATCH 66/76] Added couple UT for device-neighbor Signed-off-by: Vadym Hlushko --- .../autogen_test/show_cmd_output.py | 22 ++++ .../autogen_test/sonic-device_neighbor.yang | 78 +++++++++++++ tests/cli_autogen_input/cli_autogen_common.py | 3 +- tests/cli_autogen_input/config_db.json | 53 +++++++++ tests/cli_autogen_test.py | 105 +++++++++++++++--- 5 files changed, 242 insertions(+), 19 deletions(-) create mode 100644 tests/cli_autogen_input/autogen_test/sonic-device_neighbor.yang diff --git a/tests/cli_autogen_input/autogen_test/show_cmd_output.py b/tests/cli_autogen_input/autogen_test/show_cmd_output.py index cb0632f9ed..1ed66b9be1 100644 --- a/tests/cli_autogen_input/autogen_test/show_cmd_output.py +++ b/tests/cli_autogen_input/autogen_test/show_cmd_output.py @@ -13,4 +13,26 @@ HWSKU DEFAULT BGP STATUS DOCKER ROUTING CONFIG MODE HOSTNAME PLATFORM MAC DEFAULT PFCWD STATUS BGP ASN DEPLOYMENT ID TYPE BUFFER MODEL FRR MGMT FRAMEWORK CONFIG ----------- -------------------- ---------------------------- ---------- ---------------------- ----------------- ---------------------- --------- --------------- --------- -------------- --------------------------- ACS-MSN2100 up N/A r-sonic-01 x86_64-mlnx_msn2100-r0 ff:ff:ff:ff:ff:00 disable N/A N/A ToRRouter dynamic N/A +""" + +show_device_neighbor="""\ +PEER NAME NAME MGMT ADDR LOCAL PORT PORT TYPE +----------- -------- ----------- ------------ ------ ------ +Ethernet0 Servers 10.217.0.1 Ethernet0 eth0 type +Ethernet4 Servers0 10.217.0.2 Ethernet4 eth1 type +""" + +show_device_neighbor_added="""\ +PEER NAME NAME MGMT ADDR LOCAL PORT PORT TYPE +----------- -------- ----------- ------------ ------ ------ +Ethernet0 Servers 10.217.0.1 Ethernet0 eth0 type +Ethernet4 Servers0 10.217.0.2 Ethernet4 eth1 type +Ethernet8 Servers1 10.217.0.3 Ethernet8 eth2 type +""" + + +show_device_neighbor_deleted="""\ +PEER NAME NAME MGMT ADDR LOCAL PORT PORT TYPE +----------- -------- ----------- ------------ ------ ------ +Ethernet4 Servers0 10.217.0.2 Ethernet4 eth1 type """ \ No newline at end of file diff --git a/tests/cli_autogen_input/autogen_test/sonic-device_neighbor.yang b/tests/cli_autogen_input/autogen_test/sonic-device_neighbor.yang new file mode 100644 index 0000000000..e1c745dd9a --- /dev/null +++ b/tests/cli_autogen_input/autogen_test/sonic-device_neighbor.yang @@ -0,0 +1,78 @@ +module sonic-device_neighbor { + + yang-version 1.1; + + namespace "http://github.com/Azure/sonic-device_neighbor"; + prefix device_neighbor; + + import ietf-inet-types { + prefix inet; + } + + import sonic-extension { + prefix ext; + revision-date 2019-07-01; + } + + import sonic-port { + prefix port; + revision-date 2019-07-01; + } + + description "DEVICE_NEIGHBOR YANG Module for SONiC OS"; + + revision 2020-04-10 { + description "First Revision"; + } + + container sonic-device_neighbor { + + container DEVICE_NEIGHBOR { + + description "DEVICE_NEIGHBOR part of config_db.json"; + + list DEVICE_NEIGHBOR_LIST { + + key "peer_name"; + + leaf peer_name { + type string { + length 1..255; + } + } + + leaf name { + type string { + length 1..255; + } + } + + leaf mgmt_addr { + type inet:ip-address; + } + + leaf local_port { + type leafref { + path /port:sonic-port/port:PORT/port:PORT_LIST/port:name; + } + } + + leaf port { + type string { + length 1..255; + } + } + + leaf type { + type string { + length 1..255; + } + } + } + /* end of list DEVICE_NEIGHBOR_LIST */ + } + /* end of container DEVICE_NEIGHBOR */ + } + /* end of top level container */ +} +/* end of module sonic-device_neighbor */ diff --git a/tests/cli_autogen_input/cli_autogen_common.py b/tests/cli_autogen_input/cli_autogen_common.py index 31895d62ad..a39602ea22 100644 --- a/tests/cli_autogen_input/cli_autogen_common.py +++ b/tests/cli_autogen_input/cli_autogen_common.py @@ -34,6 +34,7 @@ def backup_yang_models(): def restore_backup_yang_models(): """ Restore existing YANG models from backup """ - cmd = 'sudo mv {} {}'.format(yang_models_path + '_backup/', '/usr/local/yang-models') + cmd = 'sudo cp {} {}'.format(yang_models_path + '_backup/*', yang_models_path) os.system(cmd) + os.system('sudo rm -rf {}'.format(yang_models_path + '_backup')) \ No newline at end of file diff --git a/tests/cli_autogen_input/config_db.json b/tests/cli_autogen_input/config_db.json index 7b2f72b815..5d8c863cec 100644 --- a/tests/cli_autogen_input/config_db.json +++ b/tests/cli_autogen_input/config_db.json @@ -8,5 +8,58 @@ "mac": "ff:ff:ff:ff:ff:00", "platform": "x86_64-mlnx_msn2100-r0", "type": "ToRRouter" + }, + "PORT|Ethernet0": { + "alias": "etp1", + "description": "etp1", + "index": "0", + "lanes": "0, 1, 2, 3", + "mtu": "9100", + "pfc_asym": "off", + "speed": "100000" + }, + "PORT|Ethernet4": { + "admin_status": "up", + "alias": "etp2", + "description": "Servers0:eth0", + "index": "1", + "lanes": "4, 5, 6, 7", + "mtu": "9100", + "pfc_asym": "off", + "speed": "100000" + }, + "PORT|Ethernet8": { + "admin_status": "up", + "alias": "etp3", + "description": "Servers0:eth2", + "index": "2", + "lanes": "8, 9, 10, 11", + "mtu": "9100", + "pfc_asym": "off", + "speed": "100000" + }, + "PORT|Ethernet12": { + "admin_status": "up", + "alias": "etp4", + "description": "Servers0:eth4", + "index": "3", + "lanes": "12, 13, 14, 15", + "mtu": "9100", + "pfc_asym": "off", + "speed": "100000" + }, + "DEVICE_NEIGHBOR|Ethernet0": { + "name": "Servers", + "port": "eth0", + "mgmt_addr": "10.217.0.1", + "local_port": "Ethernet0", + "type": "type" + }, + "DEVICE_NEIGHBOR|Ethernet4": { + "name": "Servers0", + "port": "eth1", + "mgmt_addr": "10.217.0.2", + "local_port": "Ethernet4", + "type": "type" } } \ No newline at end of file diff --git a/tests/cli_autogen_test.py b/tests/cli_autogen_test.py index 846a289a8e..3edbfb7159 100644 --- a/tests/cli_autogen_test.py +++ b/tests/cli_autogen_test.py @@ -7,7 +7,7 @@ import config.plugins as config_plugins import config.main as config_main from .cli_autogen_input.autogen_test import show_cmd_output -from .cli_autogen_input.cli_autogen_common import backup_yang_models, restore_backup_yang_models +from .cli_autogen_input.cli_autogen_common import backup_yang_models, restore_backup_yang_models, move_yang_models, remove_yang_models from utilities_common import util_base from sonic_cli_gen.generator import CliGenerator @@ -28,27 +28,34 @@ config_db_path = '/sonic/src/sonic-utilities/tests/cli_autogen_input/config_db.json' templates_path = '/sonic/src/sonic-utilities/sonic-utilities-data/templates/sonic-cli-gen/' +test_yang_models = [ + 'sonic-device_metadata.yang', + 'sonic-device_neighbor.yang', +] + class TestCliAutogen: @classmethod def setup_class(cls): logger.info('SETUP') - os.environ['UTILITIES_UNIT_TESTING'] = "2" + os.environ['UTILITIES_UNIT_TESTING'] = '2' backup_yang_models() - - gen.generate_cli_plugin( - cli_group='show', - plugin_name='sonic-device_metadata', - config_db_path=config_db_path, - templates_path=templates_path - ) - gen.generate_cli_plugin( - cli_group='config', - plugin_name='sonic-device_metadata', - config_db_path=config_db_path, - templates_path=templates_path - ) + move_yang_models(test_path, 'autogen_test', test_yang_models) + + for yang_model in test_yang_models: + gen.generate_cli_plugin( + cli_group='show', + plugin_name=yang_model.split('.')[0], + config_db_path=config_db_path, + templates_path=templates_path + ) + gen.generate_cli_plugin( + cli_group='config', + plugin_name=yang_model.split('.')[0], + config_db_path=config_db_path, + templates_path=templates_path + ) helper = util_base.UtilHelper() helper.load_and_register_plugins(show_plugins, show_main.cli) @@ -59,14 +66,15 @@ def setup_class(cls): def teardown_class(cls): logger.info('TEARDOWN') - gen.remove_cli_plugin('show', 'sonic-device_metadata') - gen.remove_cli_plugin('config', 'sonic-device_metadata') + for yang_model in test_yang_models: + gen.remove_cli_plugin('show', yang_model.split('.')[0]) + gen.remove_cli_plugin('config', yang_model.split('.')[0]) restore_backup_yang_models() dbconnector.dedicated_dbs['CONFIG_DB'] = None - os.environ['UTILITIES_UNIT_TESTING'] = "0" + os.environ['UTILITIES_UNIT_TESTING'] = '0' def test_show_device_metadata(self): @@ -126,3 +134,64 @@ def test_config_device_metadata_invalid(self, parameter, value): logger.debug(result.exit_code) assert result.exit_code == ERROR + + def test_show_device_neighbor(self): + dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db') + db = Db() + runner = CliRunner() + + result = runner.invoke( + show_main.cli.commands['device-neighbor'], [], obj=db + ) + + logger.debug("\n" + result.output) + logger.debug(result.exit_code) + assert show_cmd_output.show_device_neighbor + assert result.exit_code == SUCCESS + + + def test_config_device_neighbor_add(self): + dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db') + db = Db() + runner = CliRunner() + + result = runner.invoke( + config_main.config.commands['device-neighbor'].commands['add'], + ['Ethernet8', '--name', 'Servers1', '--mgmt-addr', '10.217.0.3', + '--local-port', 'Ethernet8', '--port', 'eth2', '--type', 'type'], + obj=db + ) + logger.debug("\n" + result.output) + logger.debug(result.exit_code) + + result = runner.invoke( + show_main.cli.commands['device-neighbor'], [], obj=db + ) + + logger.debug("\n" + result.output) + logger.debug(result.exit_code) + assert result.exit_code == SUCCESS + assert result.output == show_cmd_output.show_device_neighbor_added + + + def test_config_device_neighbor_delete(self): + dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db') + db = Db() + runner = CliRunner() + + result = runner.invoke( + config_main.config.commands['device-neighbor'].commands['delete'], + ['Ethernet0'], obj=db + ) + logger.debug("\n" + result.output) + logger.debug(result.exit_code) + + result = runner.invoke( + show_main.cli.commands['device-neighbor'], [], obj=db + ) + + logger.debug("\n" + result.output) + logger.debug(result.exit_code) + assert result.exit_code == SUCCESS + assert result.output == show_cmd_output.show_device_neighbor_deleted + From 37ac0b7effdde5f89b4c6bda794a16b1576bd37f Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Tue, 14 Sep 2021 06:37:33 +0000 Subject: [PATCH 67/76] Added UT for update flow Signed-off-by: Vadym Hlushko --- .../autogen_test/show_cmd_output.py | 47 ++++++++++++++++++- tests/cli_autogen_test.py | 47 +++++++++++++++++++ 2 files changed, 92 insertions(+), 2 deletions(-) diff --git a/tests/cli_autogen_input/autogen_test/show_cmd_output.py b/tests/cli_autogen_input/autogen_test/show_cmd_output.py index 1ed66b9be1..19c02c7783 100644 --- a/tests/cli_autogen_input/autogen_test/show_cmd_output.py +++ b/tests/cli_autogen_input/autogen_test/show_cmd_output.py @@ -1,5 +1,5 @@ """ -Module holding correct output for the show command for cli_autogen_test.py +This module are holding correct output for the show command for cli_autogen_test.py """ @@ -9,12 +9,14 @@ ACS-MSN2100 up N/A r-sonic-01 x86_64-mlnx_msn2100-r0 ff:ff:ff:ff:ff:00 disable N/A N/A ToRRouter traditional N/A """ + show_device_metadata_localhost_changed_buffer_model="""\ HWSKU DEFAULT BGP STATUS DOCKER ROUTING CONFIG MODE HOSTNAME PLATFORM MAC DEFAULT PFCWD STATUS BGP ASN DEPLOYMENT ID TYPE BUFFER MODEL FRR MGMT FRAMEWORK CONFIG ----------- -------------------- ---------------------------- ---------- ---------------------- ----------------- ---------------------- --------- --------------- --------- -------------- --------------------------- ACS-MSN2100 up N/A r-sonic-01 x86_64-mlnx_msn2100-r0 ff:ff:ff:ff:ff:00 disable N/A N/A ToRRouter dynamic N/A """ + show_device_neighbor="""\ PEER NAME NAME MGMT ADDR LOCAL PORT PORT TYPE ----------- -------- ----------- ------------ ------ ------ @@ -22,6 +24,7 @@ Ethernet4 Servers0 10.217.0.2 Ethernet4 eth1 type """ + show_device_neighbor_added="""\ PEER NAME NAME MGMT ADDR LOCAL PORT PORT TYPE ----------- -------- ----------- ------------ ------ ------ @@ -35,4 +38,44 @@ PEER NAME NAME MGMT ADDR LOCAL PORT PORT TYPE ----------- -------- ----------- ------------ ------ ------ Ethernet4 Servers0 10.217.0.2 Ethernet4 eth1 type -""" \ No newline at end of file +""" + + +show_device_neighbor_updated_mgmt_addr="""\ +PEER NAME NAME MGMT ADDR LOCAL PORT PORT TYPE +----------- -------- ----------- ------------ ------ ------ +Ethernet0 Servers 10.217.0.5 Ethernet0 eth0 type +Ethernet4 Servers0 10.217.0.2 Ethernet4 eth1 type +""" + + +show_device_neighbor_updated_name="""\ +PEER NAME NAME MGMT ADDR LOCAL PORT PORT TYPE +----------- -------- ----------- ------------ ------ ------ +Ethernet0 Servers1 10.217.0.1 Ethernet0 eth0 type +Ethernet4 Servers0 10.217.0.2 Ethernet4 eth1 type +""" + + +show_device_neighbor_updated_local_port="""\ +PEER NAME NAME MGMT ADDR LOCAL PORT PORT TYPE +----------- -------- ----------- ------------ ------ ------ +Ethernet0 Servers 10.217.0.1 Ethernet12 eth0 type +Ethernet4 Servers0 10.217.0.2 Ethernet4 eth1 type +""" + + +show_device_neighbor_updated_port="""\ +PEER NAME NAME MGMT ADDR LOCAL PORT PORT TYPE +----------- -------- ----------- ------------ ------ ------ +Ethernet0 Servers 10.217.0.1 Ethernet0 eth2 type +Ethernet4 Servers0 10.217.0.2 Ethernet4 eth1 type +""" + + +show_device_neighbor_updated_type="""\ +PEER NAME NAME MGMT ADDR LOCAL PORT PORT TYPE +----------- -------- ----------- ------------ ------ ------ +Ethernet0 Servers 10.217.0.1 Ethernet0 eth0 type2 +Ethernet4 Servers0 10.217.0.2 Ethernet4 eth1 type +""" diff --git a/tests/cli_autogen_test.py b/tests/cli_autogen_test.py index 3edbfb7159..442b18ef2c 100644 --- a/tests/cli_autogen_test.py +++ b/tests/cli_autogen_test.py @@ -195,3 +195,50 @@ def test_config_device_neighbor_delete(self): assert result.exit_code == SUCCESS assert result.output == show_cmd_output.show_device_neighbor_deleted + + @pytest.mark.parametrize("parameter,value,output", [ + ('--mgmt-addr', '10.217.0.5', show_cmd_output.show_device_neighbor_updated_mgmt_addr), + ('--name', 'Servers1', show_cmd_output.show_device_neighbor_updated_name), + ('--local-port', 'Ethernet12', show_cmd_output.show_device_neighbor_updated_local_port), + ('--port', 'eth2', show_cmd_output.show_device_neighbor_updated_port), + ('--type', 'type2', show_cmd_output.show_device_neighbor_updated_type), + ]) + def test_config_device_neighbor_update(self, parameter, value, output): + dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db') + db = Db() + runner = CliRunner() + + result = runner.invoke( + config_main.config.commands['device-neighbor'].commands['update'], + ['Ethernet0', parameter, value], obj=db + ) + logger.debug("\n" + result.output) + logger.debug(result.exit_code) + + result = runner.invoke( + show_main.cli.commands['device-neighbor'], [], obj=db + ) + + logger.debug("\n" + result.output) + logger.debug(result.exit_code) + assert result.exit_code == SUCCESS + assert result.output == output + + + @pytest.mark.parametrize("parameter,value", [ + ('--mgmt-addr', INVALID_VALUE), + ('--local-port', INVALID_VALUE) + ]) + def test_config_device_neighbor_update_invalid(self, parameter, value): + dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db') + db = Db() + runner = CliRunner() + + result = runner.invoke( + config_main.config.commands['device-neighbor'].commands['update'], + ['Ethernet0', parameter, value], obj=db + ) + logger.debug("\n" + result.output) + logger.debug(result.exit_code) + assert result.exit_code == ERROR + From 440220375ac619d4b7bff54507b7fa5c2454ca10 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Tue, 14 Sep 2021 06:54:33 +0000 Subject: [PATCH 68/76] Fixed codestyle Signed-off-by: Vadym Hlushko --- tests/cli_autogen_input/cli_autogen_common.py | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/tests/cli_autogen_input/cli_autogen_common.py b/tests/cli_autogen_input/cli_autogen_common.py index a39602ea22..141bceed9c 100644 --- a/tests/cli_autogen_input/cli_autogen_common.py +++ b/tests/cli_autogen_input/cli_autogen_common.py @@ -7,12 +7,13 @@ def move_yang_models(test_path, test_name, test_yang_models): """ Move a test YANG models to known location """ for yang_model in test_yang_models: - src_path = os.path.join(test_path, - 'cli_autogen_input', - test_name, - yang_model) - cmd = 'sudo cp {} {}'.format(src_path, yang_models_path) - os.system(cmd) + src_path = os.path.join( + test_path, + 'cli_autogen_input', + test_name, + yang_model + ) + os.system('sudo cp {} {}'.format(src_path, yang_models_path)) def remove_yang_models(test_yang_models): @@ -20,21 +21,18 @@ def remove_yang_models(test_yang_models): for yang_model in test_yang_models: yang_model_path = os.path.join(yang_models_path, yang_model) - cmd = 'sudo rm {}'.format(yang_model_path) - os.system(cmd) + os.system('sudo rm {}'.format(yang_model_path)) def backup_yang_models(): """ Make a copy of existing YANG models """ - cmd = 'sudo cp -R {} {}'.format(yang_models_path, yang_models_path + '_backup') - os.system(cmd) + os.system('sudo cp -R {} {}'.format(yang_models_path, yang_models_path + '_backup')) def restore_backup_yang_models(): """ Restore existing YANG models from backup """ - cmd = 'sudo cp {} {}'.format(yang_models_path + '_backup/*', yang_models_path) - os.system(cmd) + os.system('sudo cp {} {}'.format(yang_models_path + '_backup/*', yang_models_path)) os.system('sudo rm -rf {}'.format(yang_models_path + '_backup')) \ No newline at end of file From 9d07092d257e10b84f65898bf7e105caa5f68c56 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Tue, 14 Sep 2021 06:58:43 +0000 Subject: [PATCH 69/76] Fixed codestyle Signed-off-by: Vadym Hlushko --- sonic_cli_gen/generator.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py index 8f23daa6ac..faf3c7d694 100644 --- a/sonic_cli_gen/generator.py +++ b/sonic_cli_gen/generator.py @@ -9,8 +9,6 @@ templates_path_switch = '/usr/share/sonic/templates/sonic-cli-gen/' - - class CliGenerator: """ SONiC CLI generator. This class provides public API for sonic-cli-gen python library. It can generate config, From 5b6a8aab5ab6fd0ce5fe9be74c0e7a88c6dc5e98 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Tue, 14 Sep 2021 08:12:36 +0000 Subject: [PATCH 70/76] Fixed test path variables Signed-off-by: Vadym Hlushko --- tests/cli_autogen_test.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/tests/cli_autogen_test.py b/tests/cli_autogen_test.py index 442b18ef2c..13407d1c13 100644 --- a/tests/cli_autogen_test.py +++ b/tests/cli_autogen_test.py @@ -19,15 +19,14 @@ gen = CliGenerator(logger) test_path = os.path.dirname(os.path.abspath(__file__)) -mock_db_path = os.path.join(test_path, 'cli_autogen_input') +mock_db_path = os.path.join(test_path, 'cli_autogen_input', 'config_db') +config_db_path = os.path.join(test_path, 'cli_autogen_input', 'config_db.json') +templates_path = os.path.join(test_path, '../', 'sonic-utilities-data', 'templates', 'sonic-cli-gen') SUCCESS = 0 ERROR = 1 INVALID_VALUE = 'INVALID' -config_db_path = '/sonic/src/sonic-utilities/tests/cli_autogen_input/config_db.json' -templates_path = '/sonic/src/sonic-utilities/sonic-utilities-data/templates/sonic-cli-gen/' - test_yang_models = [ 'sonic-device_metadata.yang', 'sonic-device_neighbor.yang', @@ -78,7 +77,7 @@ def teardown_class(cls): def test_show_device_metadata(self): - dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db') + dbconnector.dedicated_dbs['CONFIG_DB'] = mock_db_path db = Db() runner = CliRunner() @@ -93,7 +92,7 @@ def test_show_device_metadata(self): def test_config_device_metadata(self): - dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db') + dbconnector.dedicated_dbs['CONFIG_DB'] = mock_db_path db = Db() runner = CliRunner() @@ -122,7 +121,7 @@ def test_config_device_metadata(self): ('frr-mgmt-framework-config', INVALID_VALUE) ]) def test_config_device_metadata_invalid(self, parameter, value): - dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db') + dbconnector.dedicated_dbs['CONFIG_DB'] = mock_db_path db = Db() runner = CliRunner() @@ -136,7 +135,7 @@ def test_config_device_metadata_invalid(self, parameter, value): def test_show_device_neighbor(self): - dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db') + dbconnector.dedicated_dbs['CONFIG_DB'] = mock_db_path db = Db() runner = CliRunner() @@ -151,7 +150,7 @@ def test_show_device_neighbor(self): def test_config_device_neighbor_add(self): - dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db') + dbconnector.dedicated_dbs['CONFIG_DB'] = mock_db_path db = Db() runner = CliRunner() @@ -175,7 +174,7 @@ def test_config_device_neighbor_add(self): def test_config_device_neighbor_delete(self): - dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db') + dbconnector.dedicated_dbs['CONFIG_DB'] = mock_db_path db = Db() runner = CliRunner() @@ -204,7 +203,7 @@ def test_config_device_neighbor_delete(self): ('--type', 'type2', show_cmd_output.show_device_neighbor_updated_type), ]) def test_config_device_neighbor_update(self, parameter, value, output): - dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db') + dbconnector.dedicated_dbs['CONFIG_DB'] = mock_db_path db = Db() runner = CliRunner() @@ -230,7 +229,7 @@ def test_config_device_neighbor_update(self, parameter, value, output): ('--local-port', INVALID_VALUE) ]) def test_config_device_neighbor_update_invalid(self, parameter, value): - dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db') + dbconnector.dedicated_dbs['CONFIG_DB'] = mock_db_path db = Db() runner = CliRunner() From df1d329d7b8ddddcac5a3965c5bb990ef4cf1cf4 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Thu, 21 Oct 2021 11:30:00 +0000 Subject: [PATCH 71/76] [sonic-cli-gen] add docstrings Signed-off-by: Stepan Blyschak --- .../templates/sonic-cli-gen/config.py.j2 | 96 +++++++++++++++++-- .../templates/sonic-cli-gen/show.py.j2 | 17 +++- 2 files changed, 100 insertions(+), 13 deletions(-) diff --git a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 index 402b7e3dd2..53a85c3c41 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 @@ -20,14 +20,25 @@ sonic_cfggen = general.load_module_from_source('sonic_cfggen', '/usr/local/bin/s def exit_with_error(*args, **kwargs): - """ Print a message and abort CLI. """ + """ Print a message with click.secho and abort CLI. + + Args: + args: Positional arguments to pass to click.secho + kwargs: Keyword arguments to pass to click.secho + """ click.secho(*args, **kwargs) raise click.Abort() def validate_config_or_raise(cfg): - """ Validate config db data using ConfigMgmt """ + """ Validate config db data using ConfigMgmt. + + Args: + cfg (Dict): Config DB data to validate. + Raises: + Exception: when cfg does not satisfy YANG schema. + """ try: cfg = sonic_cfggen.FormatConverter.to_serialized(cfg) @@ -37,7 +48,16 @@ def validate_config_or_raise(cfg): def add_entry_validated(db, table, key, data): - """ Add new entry in table and validate configuration """ + """ Add new entry in table and validate configuration. + + Args: + db (swsscommon.ConfigDBConnector): Config DB connector obect. + table (str): Table name to add new entry to. + key (Union[str, Tuple]): Key name in the table. + data (Dict): Entry data. + Raises: + Exception: when cfg does not satisfy YANG schema. + """ cfg = db.get_config() cfg.setdefault(table, {}) @@ -53,6 +73,18 @@ def add_entry_validated(db, table, key, data): def update_entry_validated(db, table, key, data, create_if_not_exists=False): """ Update entry in table and validate configuration. If attribute value in data is None, the attribute is deleted. + + Args: + db (swsscommon.ConfigDBConnector): Config DB connector obect. + table (str): Table name to add new entry to. + key (Union[str, Tuple]): Key name in the table. + data (Dict): Entry data. + create_if_not_exists (bool): + In case entry does not exists already a new entry + is not created if this flag is set to False and + creates a new entry if flag is set to True. + Raises: + Exception: when cfg does not satisfy YANG schema. """ cfg = db.get_config() @@ -75,7 +107,15 @@ def update_entry_validated(db, table, key, data, create_if_not_exists=False): def del_entry_validated(db, table, key): - """ Delete entry in table and validate configuration """ + """ Delete entry in table and validate configuration. + + Args: + db (swsscommon.ConfigDBConnector): Config DB connector obect. + table (str): Table name to add new entry to. + key (Union[str, Tuple]): Key name in the table. + Raises: + Exception: when cfg does not satisfy YANG schema. + """ cfg = db.get_config() cfg.setdefault(table, {}) @@ -89,7 +129,17 @@ def del_entry_validated(db, table, key): def add_list_entry_validated(db, table, key, attr, data): - """ Add new entry into list in table and validate configuration""" + """ Add new entry into list in table and validate configuration. + + Args: + db (swsscommon.ConfigDBConnector): Config DB connector obect. + table (str): Table name to add data to. + key (Union[str, Tuple]): Key name in the table. + attr (str): Attribute name which represents a list the data needs to be added to. + data (List): Data list to add to config DB. + Raises: + Exception: when cfg does not satisfy YANG schema. + """ cfg = db.get_config() cfg.setdefault(table, {}) @@ -106,7 +156,17 @@ def add_list_entry_validated(db, table, key, attr, data): def del_list_entry_validated(db, table, key, attr, data): - """ Delete entry from list in table and validate configuration""" + """ Delete entry from list in table and validate configuration. + + Args: + db (swsscommon.ConfigDBConnector): Config DB connector obect. + table (str): Table name to remove data from. + key (Union[str, Tuple]): Key name in the table. + attr (str): Attribute name which represents a list the data needs to be removed from. + data (Dict): Data list to remove from config DB. + Raises: + Exception: when cfg does not satisfy YANG schema. + """ cfg = db.get_config() cfg.setdefault(table, {}) @@ -125,7 +185,16 @@ def del_list_entry_validated(db, table, key, attr, data): def clear_list_entry_validated(db, table, key, attr): - """ Clear list in object and validate configuration""" + """ Clear list in object and validate configuration. + + Args: + db (swsscommon.ConfigDBConnector): Config DB connector obect. + table (str): Table name to remove the list attribute from. + key (Union[str, Tuple]): Key name in the table. + attr (str): Attribute name which represents a list that needs to be removed. + Raises: + Exception: when cfg does not satisfy YANG schema. + """ update_entry_validated(db, table, key, {attr: None}) @@ -235,7 +304,7 @@ E.g: {{ gen_click_arguments(object["keys"] + [attr]) }} @clicommon.pass_db def {{ list_update_group }}_delete( - db, + db, {{ pythonize(object["keys"] + [attr]) }} ): """ Delete {{ attr.name }} in {{ table.name }} """ @@ -262,7 +331,7 @@ E.g: {{ gen_click_arguments(object["keys"]) }} @clicommon.pass_db def {{ list_update_group }}_clear( - db, + db, {{ pythonize(object["keys"]) }} ): """ Clear {{ attr.name }} in {{ table.name }} """ @@ -473,6 +542,15 @@ def {{ table.name }}(): {% endfor %} def register(cli): + """ Register new CLI nodes in root CLI. + + Args: + cli: Root CLI node. + Raises: + Exception: when root CLI already has a command + we are trying to register. + """ + {%- for table in tables %} cli_node = {{ table.name }} if cli_node.name in cli.commands: diff --git a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 index 6ee27f2013..2a3d065fdf 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/show.py.j2 @@ -63,7 +63,7 @@ Jinja2: {"name": "leaf_1"}, {"name": "leaf_2"}, {"name": "leaf_3", "group": "group_0"} - ]) + ]) }} Result: [ @@ -111,15 +111,15 @@ Result: ] {% endmacro %} -{# Generates a list that represents a header in table view. +{# Generates a list that represents a header in table view. E.g: -Jinja2: {{ +Jinja2: {{ gen_header([ {"name": "key"}, {"name": "leaf_1"}, {"name": "leaf_2"}, {"name": "leaf_3", "group": "group_0"} - ]) + ]) }} Result: @@ -237,6 +237,15 @@ def {{ name }}(db): {% endfor %} def register(cli): + """ Register new CLI nodes in root CLI. + + Args: + cli (click.core.Command): Root CLI node. + Raises: + Exception: when root CLI already has a command + we are trying to register. + """ + {%- for table in tables %} cli_node = {{ table.name }} if cli_node.name in cli.commands: From 9e1854f4c592c6abc7e6d3efaf78e31d9021607f Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak Date: Mon, 1 Nov 2021 22:43:32 +0200 Subject: [PATCH 72/76] use dict.pop() with default value Signed-off-by: Stepan Blyshchak --- sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 index 53a85c3c41..8c9bbbd184 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 @@ -97,8 +97,8 @@ def update_entry_validated(db, table, key, data, create_if_not_exists=False): raise Exception(f"{key} does not exist") for attr, value in data.items(): - if value is None and attr in cfg[table][key]: - cfg[table][key].pop(attr) + if value is None: + cfg[table][key].pop(attr, None) else: cfg[table][key][attr] = value From f2677a173f6006e9e5742b92b57fc24dd79ab0fb Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak Date: Mon, 1 Nov 2021 22:58:30 +0200 Subject: [PATCH 73/76] if no data ion update_entry_validated() abort Signed-off-by: Stepan Blyshchak --- sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 index 8c9bbbd184..6b5d213427 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 @@ -90,6 +90,9 @@ def update_entry_validated(db, table, key, data, create_if_not_exists=False): cfg = db.get_config() cfg.setdefault(table, {}) + if not data: + raise Exception(f"No field/values to update {key}") + if create_if_not_exists: cfg[table].setdefault(key, {}) From 4e0ac3e0dd3d3fb42babb002dcd7a9e03d4f20ac Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak Date: Tue, 2 Nov 2021 18:44:06 +0200 Subject: [PATCH 74/76] if entry config did not change avoid calling set_entry() Signed-off-by: Stepan Blyshchak --- sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 index 6b5d213427..37b3cf6a19 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 @@ -99,12 +99,19 @@ def update_entry_validated(db, table, key, data, create_if_not_exists=False): if key not in cfg[table]: raise Exception(f"{key} does not exist") + entry_changed = False for attr, value in data.items(): + if value == cfg[table][key][attr]: + continue + entry_changed = True if value is None: cfg[table][key].pop(attr, None) else: cfg[table][key][attr] = value + if not entry_changed: + return + validate_config_or_raise(cfg) db.set_entry(table, key, cfg[table][key]) From 6a7d7fa447e437cbba50f6dcbe07f2811bcae3c8 Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak Date: Thu, 4 Nov 2021 16:34:26 +0200 Subject: [PATCH 75/76] pass a copy of cfg to FormatConverter.to_serialized() as it modifies original cfg Signed-off-by: Stepan Blyshchak --- sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 index 37b3cf6a19..7706ae3940 100644 --- a/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 +++ b/sonic-utilities-data/templates/sonic-cli-gen/config.py.j2 @@ -9,6 +9,7 @@ Source YANG module: {{ source_yang_module }} {% endif %} """ +import copy import click import utilities_common.cli as clicommon import utilities_common.general as general @@ -41,7 +42,7 @@ def validate_config_or_raise(cfg): """ try: - cfg = sonic_cfggen.FormatConverter.to_serialized(cfg) + cfg = sonic_cfggen.FormatConverter.to_serialized(copy.deepcopy(cfg)) config_mgmt.ConfigMgmt().loadData(cfg) except Exception as err: raise Exception('Failed to validate configuration: {}'.format(err)) From afc4afb8ad2a564c042b7b9985a3c129d52470b8 Mon Sep 17 00:00:00 2001 From: Vadym Hlushko Date: Tue, 16 Nov 2021 12:04:24 +0000 Subject: [PATCH 76/76] Added exception handling for j2 templates Signed-off-by: Vadym Hlushko --- sonic_cli_gen/generator.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sonic_cli_gen/generator.py b/sonic_cli_gen/generator.py index faf3c7d694..9d5bac6008 100644 --- a/sonic_cli_gen/generator.py +++ b/sonic_cli_gen/generator.py @@ -47,7 +47,10 @@ def generate_cli_plugin( loader = jinja2.FileSystemLoader(templates_path) j2_env = jinja2.Environment(loader=loader) - template = j2_env.get_template(cli_group + '.py.j2') + try: + template = j2_env.get_template(cli_group + '.py.j2') + except jinja2.exceptions.TemplateNotFound: + self.logger.error(' Templates for auto-generation does NOT exist in folder {}'.format(templates_path)) plugin_path = get_cli_plugin_path(cli_group, plugin_name + '_yang.py') @@ -67,7 +70,7 @@ def remove_cli_plugin(self, cli_group, plugin_name): os.remove(plugin_path) self.logger.info(' {} was removed.'.format(plugin_path)) else: - self.logger.info(' Path {} doest NOT exist!'.format(plugin_path)) + self.logger.warning(' Path {} doest NOT exist!'.format(plugin_path)) def get_cli_plugin_path(command, plugin_name):