Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Run yang validation in unit test #3025

Merged
merged 3 commits into from
Nov 1, 2023
Merged

Run yang validation in unit test #3025

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7f47516
Run yang validation in unit test
ganglyu Oct 24, 2023
d6ac80e
Fix yang validation
ganglyu Oct 24, 2023
ec1d5b9
Yang validation in production runtime as background task
ganglyu Oct 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions scripts/db_migrator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,15 @@
import sys
import traceback
import re
import sonic_yang

from sonic_py_common import device_info, logger
from swsscommon.swsscommon import SonicV2Connector, ConfigDBConnector, SonicDBConfig
from minigraph import parse_xml

INIT_CFG_FILE = '/etc/sonic/init_cfg.json'
MINIGRAPH_FILE = '/etc/sonic/minigraph.xml'
YANG_MODELS_DIR = "/usr/local/yang-models"

# mock the redis for unit test purposes #
try:
Expand Down Expand Up @@ -1122,6 +1124,27 @@ def migrate(self):
version = next_version
# Perform common migration ops
self.common_migration_ops()
if os.environ["UTILITIES_UNIT_TESTING"] == "2":
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

UTILITIES_UNIT_TESTING

I am thinking this detection is useful in production runtime as background task, not just unit test. It will help detect bugs like:

  1. db_migrator generates a new config but not yang validated
  2. load_minigraph generates a new config but not yang validated

Not sure if there are other places the yang validation could help in runtime. But let's only detect and generate CRIT syslog, not to block the existing workflow.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Updated

config = self.configDB.get_config()
# Fix table key in tuple
for table_name, table in config.items():
new_table = {}
hit = False
for table_key, table_val in table.items():
if isinstance(table_key, tuple):
new_key = "|".join(table_key)
new_table[new_key] = table_val
hit = True
else:
new_table[table_key] = table_val
if hit:
config[table_name] = new_table
# Run yang validation
yang_parser = sonic_yang.SonicYang(YANG_MODELS_DIR)
yang_parser.loadYangModel()
yang_parser.loadData(configdbJson=config)
yang_parser.validate_data_tree()


def main():
try:
Expand Down
6 changes: 3 additions & 3 deletions tests/db_migrator_input/config_db/cross_branch_upgrade_to_4_0_3_expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,17 @@
"VERSION": "version_4_0_3"
},
"FLEX_COUNTER_TABLE|ACL": {
"FLEX_COUNTER_STATUS": "true",
"FLEX_COUNTER_STATUS": "enable",
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

enable

Just wondering, should we fix yang model, or fix the test data?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I will check with owner.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@vadymhlushko-mlnx has confirmed this is a mistake.

"FLEX_COUNTER_DELAY_STATUS": "true",
"POLL_INTERVAL": "10000"
},
"FLEX_COUNTER_TABLE|QUEUE": {
"FLEX_COUNTER_STATUS": "true",
"FLEX_COUNTER_STATUS": "enable",
"FLEX_COUNTER_DELAY_STATUS": "true",
"POLL_INTERVAL": "10000"
},
"FLEX_COUNTER_TABLE|PG_WATERMARK": {
"FLEX_COUNTER_STATUS": "false",
"FLEX_COUNTER_STATUS": "disable",
"FLEX_COUNTER_DELAY_STATUS": "true"
}
}
6 changes: 3 additions & 3 deletions tests/db_migrator_input/config_db/cross_branch_upgrade_to_4_0_3_input.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,16 @@
"VERSION": "version_1_0_1"
},
"FLEX_COUNTER_TABLE|ACL": {
"FLEX_COUNTER_STATUS": "true",
"FLEX_COUNTER_STATUS": "enable",
"FLEX_COUNTER_DELAY_STATUS": "true",
"POLL_INTERVAL": "10000"
},
"FLEX_COUNTER_TABLE|QUEUE": {
"FLEX_COUNTER_STATUS": "true",
"FLEX_COUNTER_STATUS": "enable",
"FLEX_COUNTER_DELAY_STATUS": "false",
"POLL_INTERVAL": "10000"
},
"FLEX_COUNTER_TABLE|PG_WATERMARK": {
"FLEX_COUNTER_STATUS": "false"
"FLEX_COUNTER_STATUS": "disable"
}
}
4 changes: 0 additions & 4 deletions tests/db_migrator_input/config_db/portchannel-expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,28 +1,24 @@
{
"PORTCHANNEL|PortChannel0": {
"admin_status": "up",
"members@": "Ethernet0,Ethernet4",
"min_links": "2",
"mtu": "9100",
"lacp_key": "auto"
},
"PORTCHANNEL|PortChannel1": {
"admin_status": "up",
"members@": "Ethernet8,Ethernet12",
"min_links": "2",
"mtu": "9100",
"lacp_key": "auto"
},
"PORTCHANNEL|PortChannel0123": {
"admin_status": "up",
"members@": "Ethernet16",
"min_links": "1",
"mtu": "9100",
"lacp_key": "auto"
},
"PORTCHANNEL|PortChannel0011": {
"admin_status": "up",
"members@": "Ethernet20,Ethernet24",
"min_links": "2",
"mtu": "9100",
"lacp_key": "auto"
Expand Down
4 changes: 0 additions & 4 deletions tests/db_migrator_input/config_db/portchannel-input.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,25 +1,21 @@
{
"PORTCHANNEL|PortChannel0": {
"admin_status": "up",
"members@": "Ethernet0,Ethernet4",
"min_links": "2",
"mtu": "9100"
},
"PORTCHANNEL|PortChannel1": {
"admin_status": "up",
"members@": "Ethernet8,Ethernet12",
"min_links": "2",
"mtu": "9100"
},
"PORTCHANNEL|PortChannel0123": {
"admin_status": "up",
"members@": "Ethernet16",
"min_links": "1",
"mtu": "9100"
},
"PORTCHANNEL|PortChannel0011": {
"admin_status": "up",
"members@": "Ethernet20,Ethernet24",
"min_links": "2",
"mtu": "9100"
},
Expand Down
10 changes: 9 additions & 1 deletion tests/db_migrator_input/config_db/qos_map_table_expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,14 @@
"pfc_to_queue_map": "AZURE",
"tc_to_pg_map": "AZURE",
"tc_to_queue_map": "AZURE"
}
},
"TC_TO_QUEUE_MAP|AZURE": {"0": "0"},
"TC_TO_PRIORITY_GROUP_MAP|AZURE": {"0": "0"},
"MAP_PFC_PRIORITY_TO_QUEUE|AZURE": {"0": "0"},
"DSCP_TO_TC_MAP|AZURE": {"0": "0"},
"PORT|Ethernet0": {"lanes": "0", "speed": "1000"},
"PORT|Ethernet92": {"lanes": "92", "speed": "1000"},
"PORT|Ethernet96": {"lanes": "96", "speed": "1000"},
"PORT|Ethernet100": {"lanes": "100", "speed": "1000"}
}

10 changes: 9 additions & 1 deletion tests/db_migrator_input/config_db/qos_map_table_input.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,5 +27,13 @@
"pfc_to_queue_map": "AZURE",
"tc_to_pg_map": "AZURE",
"tc_to_queue_map": "AZURE"
}
},
"TC_TO_QUEUE_MAP|AZURE": {"0": "0"},
"TC_TO_PRIORITY_GROUP_MAP|AZURE": {"0": "0"},
"MAP_PFC_PRIORITY_TO_QUEUE|AZURE": {"0": "0"},
"DSCP_TO_TC_MAP|AZURE": {"0": "0"},
"PORT|Ethernet0": {"lanes": "0", "speed": "1000"},
"PORT|Ethernet92": {"lanes": "92", "speed": "1000"},
"PORT|Ethernet96": {"lanes": "96", "speed": "1000"},
"PORT|Ethernet100": {"lanes": "100", "speed": "1000"}
}
12 changes: 11 additions & 1 deletion tests/db_migrator_input/config_db/reclaiming-buffer-dynamic-double-pools-expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
"profile": "NULL"
},
"BUFFER_PG|Ethernet8|3-4": {
"profile": "customized_lossless_profile"
"profile": "customized_ingress_lossless_profile"
},
"BUFFER_PG|Ethernet12|0": {
"profile": "ingress_lossy_profile"
Expand Down Expand Up @@ -103,6 +103,11 @@
"BUFFER_PORT_INGRESS_PROFILE_LIST|Ethernet24": {
"profile_list": "ingress_lossless_profile,ingress_lossy_profile"
},
"BUFFER_PROFILE|customized_egress_lossless_profile": {
"dynamic_th": "7",
"pool": "egress_lossless_pool",
"size": "0"
},
"BUFFER_PROFILE|egress_lossless_profile": {
"dynamic_th": "7",
"pool": "egress_lossless_pool",
Expand All @@ -113,6 +118,11 @@
"pool": "egress_lossy_pool",
"size": "9216"
},
"BUFFER_PROFILE|customized_ingress_lossless_profile": {
"dynamic_th": "7",
"pool": "ingress_lossless_pool",
"size": "0"
},
"BUFFER_PROFILE|ingress_lossless_profile": {
"dynamic_th": "7",
"pool": "ingress_lossless_pool",
Expand Down
12 changes: 11 additions & 1 deletion tests/db_migrator_input/config_db/reclaiming-buffer-dynamic-double-pools-input.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"profile": "NULL"
},
"BUFFER_PG|Ethernet8|3-4": {
"profile": "customized_lossless_profile"
"profile": "customized_ingress_lossless_profile"
},
"BUFFER_PG|Ethernet12|0": {
"profile": "ingress_lossy_profile"
Expand Down Expand Up @@ -55,6 +55,11 @@
"BUFFER_PORT_INGRESS_PROFILE_LIST|Ethernet24": {
"profile_list": "ingress_lossless_profile,ingress_lossy_profile"
},
"BUFFER_PROFILE|customized_egress_lossless_profile": {
"dynamic_th": "7",
"pool": "egress_lossless_pool",
"size": "0"
},
"BUFFER_PROFILE|egress_lossless_profile": {
"dynamic_th": "7",
"pool": "egress_lossless_pool",
Expand All @@ -65,6 +70,11 @@
"pool": "egress_lossy_pool",
"size": "9216"
},
"BUFFER_PROFILE|customized_ingress_lossless_profile": {
"dynamic_th": "7",
"pool": "ingress_lossless_pool",
"size": "0"
},
"BUFFER_PROFILE|ingress_lossless_profile": {
"dynamic_th": "7",
"pool": "ingress_lossless_pool",
Expand Down
12 changes: 11 additions & 1 deletion tests/db_migrator_input/config_db/reclaiming-buffer-dynamic-single-pool-expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
"profile": "NULL"
},
"BUFFER_PG|Ethernet8|3-4": {
"profile": "customized_lossless_profile"
"profile": "customized_ingress_lossless_profile"
},
"BUFFER_PG|Ethernet12|0": {
"profile": "ingress_lossy_profile"
Expand Down Expand Up @@ -99,6 +99,11 @@
"BUFFER_PORT_INGRESS_PROFILE_LIST|Ethernet24": {
"profile_list": "ingress_lossless_profile"
},
"BUFFER_PROFILE|customized_egress_lossless_profile": {
"dynamic_th": "7",
"pool": "egress_lossless_pool",
"size": "0"
},
"BUFFER_PROFILE|egress_lossless_profile": {
"dynamic_th": "7",
"pool": "egress_lossless_pool",
Expand All @@ -109,6 +114,11 @@
"pool": "egress_lossy_pool",
"size": "9216"
},
"BUFFER_PROFILE|customized_ingress_lossless_profile": {
"dynamic_th": "7",
"pool": "ingress_lossless_pool",
"size": "0"
},
"BUFFER_PROFILE|ingress_lossless_profile": {
"dynamic_th": "7",
"pool": "ingress_lossless_pool",
Expand Down
12 changes: 11 additions & 1 deletion tests/db_migrator_input/config_db/reclaiming-buffer-dynamic-single-pool-input.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"profile": "NULL"
},
"BUFFER_PG|Ethernet8|3-4": {
"profile": "customized_lossless_profile"
"profile": "customized_ingress_lossless_profile"
},
"BUFFER_PG|Ethernet12|0": {
"profile": "ingress_lossy_profile"
Expand Down Expand Up @@ -51,6 +51,11 @@
"BUFFER_PORT_INGRESS_PROFILE_LIST|Ethernet24": {
"profile_list": "ingress_lossless_profile"
},
"BUFFER_PROFILE|customized_egress_lossless_profile": {
"dynamic_th": "7",
"pool": "egress_lossless_pool",
"size": "0"
},
"BUFFER_PROFILE|egress_lossless_profile": {
"dynamic_th": "7",
"pool": "egress_lossless_pool",
Expand All @@ -61,6 +66,11 @@
"pool": "egress_lossy_pool",
"size": "9216"
},
"BUFFER_PROFILE|customized_ingress_lossless_profile": {
"dynamic_th": "7",
"pool": "ingress_lossless_pool",
"size": "0"
},
"BUFFER_PROFILE|ingress_lossless_profile": {
"dynamic_th": "7",
"pool": "ingress_lossless_pool",
Expand Down
Loading