From 7f475163a85a5e60275e9655531956a7e4c2c326 Mon Sep 17 00:00:00 2001 From: ganglyu Date: Tue, 24 Oct 2023 14:08:53 +0800 Subject: [PATCH 1/3] Run yang validation in unit test --- scripts/db_migrator.py | 18 ++++++++++++++++++ ...cross_branch_upgrade_to_4_0_3_expected.json | 6 +++--- .../cross_branch_upgrade_to_4_0_3_input.json | 6 +++--- .../config_db/portchannel-expected.json | 4 ---- .../config_db/portchannel-input.json | 4 ---- .../config_db/qos_map_table_expected.json | 10 +++++++++- .../config_db/qos_map_table_input.json | 10 +++++++++- ...g-buffer-dynamic-double-pools-expected.json | 12 +++++++++++- ...ming-buffer-dynamic-double-pools-input.json | 12 +++++++++++- ...ng-buffer-dynamic-single-pool-expected.json | 12 +++++++++++- ...iming-buffer-dynamic-single-pool-input.json | 12 +++++++++++- 11 files changed, 86 insertions(+), 20 deletions(-) diff --git a/scripts/db_migrator.py b/scripts/db_migrator.py index a4240eac4d..0653b4f3c8 100755 --- a/scripts/db_migrator.py +++ b/scripts/db_migrator.py @@ -6,6 +6,7 @@ import sys import traceback import re +import sonic_yang from sonic_py_common import device_info, logger from swsscommon.swsscommon import SonicV2Connector, ConfigDBConnector, SonicDBConfig @@ -13,6 +14,7 @@ INIT_CFG_FILE = '/etc/sonic/init_cfg.json' MINIGRAPH_FILE = '/etc/sonic/minigraph.xml' +YANG_MODELS_DIR = "/usr/local/yang-models" # mock the redis for unit test purposes # try: @@ -1122,6 +1124,22 @@ def migrate(self): version = next_version # Perform common migration ops self.common_migration_ops() + if os.environ["UTILITIES_UNIT_TESTING"] == "2": + config = self.configDB.get_config() + # Fix table key in tuple + for table_name, table in config.items(): + new_table = {} + hit = False + for table_key, table_val in table.items(): + if isinstance(table_key, tuple): + new_key = "|".join(table_key) + new_table[new_key] = table_val + hit = True + else: + new_table[table_key] = table_val + if hit: + config[table_name] = new_table + def main(): try: diff --git a/tests/db_migrator_input/config_db/cross_branch_upgrade_to_4_0_3_expected.json b/tests/db_migrator_input/config_db/cross_branch_upgrade_to_4_0_3_expected.json index 1ebfbc6afe..a64d38bc51 100644 --- a/tests/db_migrator_input/config_db/cross_branch_upgrade_to_4_0_3_expected.json +++ b/tests/db_migrator_input/config_db/cross_branch_upgrade_to_4_0_3_expected.json @@ -3,17 +3,17 @@ "VERSION": "version_4_0_3" }, "FLEX_COUNTER_TABLE|ACL": { - "FLEX_COUNTER_STATUS": "true", + "FLEX_COUNTER_STATUS": "enable", "FLEX_COUNTER_DELAY_STATUS": "true", "POLL_INTERVAL": "10000" }, "FLEX_COUNTER_TABLE|QUEUE": { - "FLEX_COUNTER_STATUS": "true", + "FLEX_COUNTER_STATUS": "enable", "FLEX_COUNTER_DELAY_STATUS": "true", "POLL_INTERVAL": "10000" }, "FLEX_COUNTER_TABLE|PG_WATERMARK": { - "FLEX_COUNTER_STATUS": "false", + "FLEX_COUNTER_STATUS": "disable", "FLEX_COUNTER_DELAY_STATUS": "true" } } diff --git a/tests/db_migrator_input/config_db/cross_branch_upgrade_to_4_0_3_input.json b/tests/db_migrator_input/config_db/cross_branch_upgrade_to_4_0_3_input.json index 07ce763683..e2d8d04588 100644 --- a/tests/db_migrator_input/config_db/cross_branch_upgrade_to_4_0_3_input.json +++ b/tests/db_migrator_input/config_db/cross_branch_upgrade_to_4_0_3_input.json @@ -3,16 +3,16 @@ "VERSION": "version_1_0_1" }, "FLEX_COUNTER_TABLE|ACL": { - "FLEX_COUNTER_STATUS": "true", + "FLEX_COUNTER_STATUS": "enable", "FLEX_COUNTER_DELAY_STATUS": "true", "POLL_INTERVAL": "10000" }, "FLEX_COUNTER_TABLE|QUEUE": { - "FLEX_COUNTER_STATUS": "true", + "FLEX_COUNTER_STATUS": "enable", "FLEX_COUNTER_DELAY_STATUS": "false", "POLL_INTERVAL": "10000" }, "FLEX_COUNTER_TABLE|PG_WATERMARK": { - "FLEX_COUNTER_STATUS": "false" + "FLEX_COUNTER_STATUS": "disable" } } diff --git a/tests/db_migrator_input/config_db/portchannel-expected.json b/tests/db_migrator_input/config_db/portchannel-expected.json index 2644e5f4e9..f380c75363 100644 --- a/tests/db_migrator_input/config_db/portchannel-expected.json +++ b/tests/db_migrator_input/config_db/portchannel-expected.json @@ -1,28 +1,24 @@ { "PORTCHANNEL|PortChannel0": { "admin_status": "up", - "members@": "Ethernet0,Ethernet4", "min_links": "2", "mtu": "9100", "lacp_key": "auto" }, "PORTCHANNEL|PortChannel1": { "admin_status": "up", - "members@": "Ethernet8,Ethernet12", "min_links": "2", "mtu": "9100", "lacp_key": "auto" }, "PORTCHANNEL|PortChannel0123": { "admin_status": "up", - "members@": "Ethernet16", "min_links": "1", "mtu": "9100", "lacp_key": "auto" }, "PORTCHANNEL|PortChannel0011": { "admin_status": "up", - "members@": "Ethernet20,Ethernet24", "min_links": "2", "mtu": "9100", "lacp_key": "auto" diff --git a/tests/db_migrator_input/config_db/portchannel-input.json b/tests/db_migrator_input/config_db/portchannel-input.json index 753a88601d..43a9fabdb5 100644 --- a/tests/db_migrator_input/config_db/portchannel-input.json +++ b/tests/db_migrator_input/config_db/portchannel-input.json @@ -1,25 +1,21 @@ { "PORTCHANNEL|PortChannel0": { "admin_status": "up", - "members@": "Ethernet0,Ethernet4", "min_links": "2", "mtu": "9100" }, "PORTCHANNEL|PortChannel1": { "admin_status": "up", - "members@": "Ethernet8,Ethernet12", "min_links": "2", "mtu": "9100" }, "PORTCHANNEL|PortChannel0123": { "admin_status": "up", - "members@": "Ethernet16", "min_links": "1", "mtu": "9100" }, "PORTCHANNEL|PortChannel0011": { "admin_status": "up", - "members@": "Ethernet20,Ethernet24", "min_links": "2", "mtu": "9100" }, diff --git a/tests/db_migrator_input/config_db/qos_map_table_expected.json b/tests/db_migrator_input/config_db/qos_map_table_expected.json index 47381ec550..f84c1a900b 100644 --- a/tests/db_migrator_input/config_db/qos_map_table_expected.json +++ b/tests/db_migrator_input/config_db/qos_map_table_expected.json @@ -29,6 +29,14 @@ "pfc_to_queue_map": "AZURE", "tc_to_pg_map": "AZURE", "tc_to_queue_map": "AZURE" - } + }, + "TC_TO_QUEUE_MAP|AZURE": {"0": "0"}, + "TC_TO_PRIORITY_GROUP_MAP|AZURE": {"0": "0"}, + "MAP_PFC_PRIORITY_TO_QUEUE|AZURE": {"0": "0"}, + "DSCP_TO_TC_MAP|AZURE": {"0": "0"}, + "PORT|Ethernet0": {"lanes": "0", "speed": "1000"}, + "PORT|Ethernet92": {"lanes": "92", "speed": "1000"}, + "PORT|Ethernet96": {"lanes": "96", "speed": "1000"}, + "PORT|Ethernet100": {"lanes": "100", "speed": "1000"} } diff --git a/tests/db_migrator_input/config_db/qos_map_table_input.json b/tests/db_migrator_input/config_db/qos_map_table_input.json index c62e293daf..3c288b9534 100644 --- a/tests/db_migrator_input/config_db/qos_map_table_input.json +++ b/tests/db_migrator_input/config_db/qos_map_table_input.json @@ -27,5 +27,13 @@ "pfc_to_queue_map": "AZURE", "tc_to_pg_map": "AZURE", "tc_to_queue_map": "AZURE" - } + }, + "TC_TO_QUEUE_MAP|AZURE": {"0": "0"}, + "TC_TO_PRIORITY_GROUP_MAP|AZURE": {"0": "0"}, + "MAP_PFC_PRIORITY_TO_QUEUE|AZURE": {"0": "0"}, + "DSCP_TO_TC_MAP|AZURE": {"0": "0"}, + "PORT|Ethernet0": {"lanes": "0", "speed": "1000"}, + "PORT|Ethernet92": {"lanes": "92", "speed": "1000"}, + "PORT|Ethernet96": {"lanes": "96", "speed": "1000"}, + "PORT|Ethernet100": {"lanes": "100", "speed": "1000"} } diff --git a/tests/db_migrator_input/config_db/reclaiming-buffer-dynamic-double-pools-expected.json b/tests/db_migrator_input/config_db/reclaiming-buffer-dynamic-double-pools-expected.json index 5181daa057..decf997d67 100644 --- a/tests/db_migrator_input/config_db/reclaiming-buffer-dynamic-double-pools-expected.json +++ b/tests/db_migrator_input/config_db/reclaiming-buffer-dynamic-double-pools-expected.json @@ -12,7 +12,7 @@ "profile": "NULL" }, "BUFFER_PG|Ethernet8|3-4": { - "profile": "customized_lossless_profile" + "profile": "customized_ingress_lossless_profile" }, "BUFFER_PG|Ethernet12|0": { "profile": "ingress_lossy_profile" @@ -103,6 +103,11 @@ "BUFFER_PORT_INGRESS_PROFILE_LIST|Ethernet24": { "profile_list": "ingress_lossless_profile,ingress_lossy_profile" }, + "BUFFER_PROFILE|customized_egress_lossless_profile": { + "dynamic_th": "7", + "pool": "egress_lossless_pool", + "size": "0" + }, "BUFFER_PROFILE|egress_lossless_profile": { "dynamic_th": "7", "pool": "egress_lossless_pool", @@ -113,6 +118,11 @@ "pool": "egress_lossy_pool", "size": "9216" }, + "BUFFER_PROFILE|customized_ingress_lossless_profile": { + "dynamic_th": "7", + "pool": "ingress_lossless_pool", + "size": "0" + }, "BUFFER_PROFILE|ingress_lossless_profile": { "dynamic_th": "7", "pool": "ingress_lossless_pool", diff --git a/tests/db_migrator_input/config_db/reclaiming-buffer-dynamic-double-pools-input.json b/tests/db_migrator_input/config_db/reclaiming-buffer-dynamic-double-pools-input.json index d8deef194f..d3337ccadb 100644 --- a/tests/db_migrator_input/config_db/reclaiming-buffer-dynamic-double-pools-input.json +++ b/tests/db_migrator_input/config_db/reclaiming-buffer-dynamic-double-pools-input.json @@ -3,7 +3,7 @@ "profile": "NULL" }, "BUFFER_PG|Ethernet8|3-4": { - "profile": "customized_lossless_profile" + "profile": "customized_ingress_lossless_profile" }, "BUFFER_PG|Ethernet12|0": { "profile": "ingress_lossy_profile" @@ -55,6 +55,11 @@ "BUFFER_PORT_INGRESS_PROFILE_LIST|Ethernet24": { "profile_list": "ingress_lossless_profile,ingress_lossy_profile" }, + "BUFFER_PROFILE|customized_egress_lossless_profile": { + "dynamic_th": "7", + "pool": "egress_lossless_pool", + "size": "0" + }, "BUFFER_PROFILE|egress_lossless_profile": { "dynamic_th": "7", "pool": "egress_lossless_pool", @@ -65,6 +70,11 @@ "pool": "egress_lossy_pool", "size": "9216" }, + "BUFFER_PROFILE|customized_ingress_lossless_profile": { + "dynamic_th": "7", + "pool": "ingress_lossless_pool", + "size": "0" + }, "BUFFER_PROFILE|ingress_lossless_profile": { "dynamic_th": "7", "pool": "ingress_lossless_pool", diff --git a/tests/db_migrator_input/config_db/reclaiming-buffer-dynamic-single-pool-expected.json b/tests/db_migrator_input/config_db/reclaiming-buffer-dynamic-single-pool-expected.json index 278a40bc0a..3572be8b69 100644 --- a/tests/db_migrator_input/config_db/reclaiming-buffer-dynamic-single-pool-expected.json +++ b/tests/db_migrator_input/config_db/reclaiming-buffer-dynamic-single-pool-expected.json @@ -12,7 +12,7 @@ "profile": "NULL" }, "BUFFER_PG|Ethernet8|3-4": { - "profile": "customized_lossless_profile" + "profile": "customized_ingress_lossless_profile" }, "BUFFER_PG|Ethernet12|0": { "profile": "ingress_lossy_profile" @@ -99,6 +99,11 @@ "BUFFER_PORT_INGRESS_PROFILE_LIST|Ethernet24": { "profile_list": "ingress_lossless_profile" }, + "BUFFER_PROFILE|customized_egress_lossless_profile": { + "dynamic_th": "7", + "pool": "egress_lossless_pool", + "size": "0" + }, "BUFFER_PROFILE|egress_lossless_profile": { "dynamic_th": "7", "pool": "egress_lossless_pool", @@ -109,6 +114,11 @@ "pool": "egress_lossy_pool", "size": "9216" }, + "BUFFER_PROFILE|customized_ingress_lossless_profile": { + "dynamic_th": "7", + "pool": "ingress_lossless_pool", + "size": "0" + }, "BUFFER_PROFILE|ingress_lossless_profile": { "dynamic_th": "7", "pool": "ingress_lossless_pool", diff --git a/tests/db_migrator_input/config_db/reclaiming-buffer-dynamic-single-pool-input.json b/tests/db_migrator_input/config_db/reclaiming-buffer-dynamic-single-pool-input.json index b3bda32f23..60f4455cad 100644 --- a/tests/db_migrator_input/config_db/reclaiming-buffer-dynamic-single-pool-input.json +++ b/tests/db_migrator_input/config_db/reclaiming-buffer-dynamic-single-pool-input.json @@ -3,7 +3,7 @@ "profile": "NULL" }, "BUFFER_PG|Ethernet8|3-4": { - "profile": "customized_lossless_profile" + "profile": "customized_ingress_lossless_profile" }, "BUFFER_PG|Ethernet12|0": { "profile": "ingress_lossy_profile" @@ -51,6 +51,11 @@ "BUFFER_PORT_INGRESS_PROFILE_LIST|Ethernet24": { "profile_list": "ingress_lossless_profile" }, + "BUFFER_PROFILE|customized_egress_lossless_profile": { + "dynamic_th": "7", + "pool": "egress_lossless_pool", + "size": "0" + }, "BUFFER_PROFILE|egress_lossless_profile": { "dynamic_th": "7", "pool": "egress_lossless_pool", @@ -61,6 +66,11 @@ "pool": "egress_lossy_pool", "size": "9216" }, + "BUFFER_PROFILE|customized_ingress_lossless_profile": { + "dynamic_th": "7", + "pool": "ingress_lossless_pool", + "size": "0" + }, "BUFFER_PROFILE|ingress_lossless_profile": { "dynamic_th": "7", "pool": "ingress_lossless_pool", From d6ac80e24cd6a46bf780dc7c834fae16264773d1 Mon Sep 17 00:00:00 2001 From: ganglyu Date: Tue, 24 Oct 2023 14:19:09 +0800 Subject: [PATCH 2/3] Fix yang validation --- scripts/db_migrator.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/db_migrator.py b/scripts/db_migrator.py index 0653b4f3c8..432d39f8af 100755 --- a/scripts/db_migrator.py +++ b/scripts/db_migrator.py @@ -1139,6 +1139,11 @@ def migrate(self): new_table[table_key] = table_val if hit: config[table_name] = new_table + # Run yang validation + yang_parser = sonic_yang.SonicYang(YANG_MODELS_DIR) + yang_parser.loadYangModel() + yang_parser.loadData(configdbJson=config) + yang_parser.validate_data_tree() def main(): From ec1d5b9c7da5b2cbdb97c1d826f59a74b069bf11 Mon Sep 17 00:00:00 2001 From: ganglyu Date: Sat, 28 Oct 2023 17:02:24 +0800 Subject: [PATCH 3/3] Yang validation in production runtime as background task --- scripts/db_migrator.py | 41 +++++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/scripts/db_migrator.py b/scripts/db_migrator.py index 432d39f8af..f15f27bae8 100755 --- a/scripts/db_migrator.py +++ b/scripts/db_migrator.py @@ -7,6 +7,7 @@ import traceback import re import sonic_yang +import syslog from sonic_py_common import device_info, logger from swsscommon.swsscommon import SonicV2Connector, ConfigDBConnector, SonicDBConfig @@ -1124,26 +1125,30 @@ def migrate(self): version = next_version # Perform common migration ops self.common_migration_ops() - if os.environ["UTILITIES_UNIT_TESTING"] == "2": - config = self.configDB.get_config() - # Fix table key in tuple - for table_name, table in config.items(): - new_table = {} - hit = False - for table_key, table_val in table.items(): - if isinstance(table_key, tuple): - new_key = "|".join(table_key) - new_table[new_key] = table_val - hit = True - else: - new_table[table_key] = table_val - if hit: - config[table_name] = new_table - # Run yang validation - yang_parser = sonic_yang.SonicYang(YANG_MODELS_DIR) - yang_parser.loadYangModel() + config = self.configDB.get_config() + # Fix table key in tuple + for table_name, table in config.items(): + new_table = {} + hit = False + for table_key, table_val in table.items(): + if isinstance(table_key, tuple): + new_key = "|".join(table_key) + new_table[new_key] = table_val + hit = True + else: + new_table[table_key] = table_val + if hit: + config[table_name] = new_table + # Run yang validation + yang_parser = sonic_yang.SonicYang(YANG_MODELS_DIR) + yang_parser.loadYangModel() + try: yang_parser.loadData(configdbJson=config) yang_parser.validate_data_tree() + except sonic_yang.SonicYangException as e: + syslog.syslog(syslog.LOG_CRIT, "Yang validation failed: " + str(e)) + if os.environ["UTILITIES_UNIT_TESTING"] == "2": + raise def main():