Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[_sonic_yang_ext.py]: Parse multilist in YANG Container. #38

Merged
merged 3 commits into from
Feb 13, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
135 changes: 83 additions & 52 deletions src/sonic-yang-mgmt/_sonic_yang_ext.py
Original file line number Diff line number Diff line change
Expand Up @@ -129,31 +129,28 @@ def cropConfigDB(self, croppedFile=None, allowExtraTables=True):

"""
Extract keys from table entry in Config DB and return in a dict
For Example: regex = <vlan_name>| and tableKey = "Vlan111|2a04:5555:45:6709::1/64"

1.) first code will extract key list from regex, i.e. vlan_name and ip_prefix.
2.) then will create another regex(regexV) to extract Values from tableKey by
replacing " --> extractor i.e. (.*?)" in regex.
3.) Then will extract values from tableKey with regexV.
4.) Resulting Dict will be:
Input:
tableKey: Config DB Primary Key, Example tableKey = "Vlan111|2a04:5555:45:6709::1/64"
keys: key string from YANG list, i.e. 'vlan_name ip-prefix'.
regex: A regex to extract keys from tableKeys, good to have it as accurate as possible.

Return:
KeyDict = {"vlan_name": "Vlan111", "ip-prefix": "2a04:5555:45:6709::1/64"}
"""
def extractKey(self, tableKey, regex):
def extractKey(self, tableKey, keys, regex):

# get the keys from regex of key extractor
keyList = re.findall(r'<(.*?)>', regex)
# create a regex to get values from tableKey
# and change separator to text in regexV
regexV = re.sub('<.*?>', '(.*?)', regex)
regexV = re.sub('\|', '\\|', regexV)
keyList = keys.split()
#self.logInFile("extractKey {}".format(keyList))
# get the value groups
value = re.match(r'^'+regexV+'$', tableKey)
value = re.match(regex, tableKey)
# create the keyDict
i = 1
keyDict = dict()
for k in keyList:
if value.group(i):
keyDict[k] = value.group(i)
# self.logInFile("extractKey {} {}".format(k, keyDict[k]))
else:
raise Exception("Value not found for {} in {}".format(k, tableKey))
i = i + 1
Expand Down Expand Up @@ -254,37 +251,49 @@ def xlateList(self, model, yang, config, table):
# TODO: define a keyExt dict as of now, but we should be able to extract
# this from YANG model extentions.
keyExt = {
"VLAN_INTERFACE": "<vlan_name>|<ip-prefix>",
"ACL_RULE": "<ACL_TABLE_NAME>|<RULE_NAME>",
"VLAN": "<vlan_name>",
"VLAN_MEMBER": "<vlan_name>|<port>",
"ACL_TABLE": "<ACL_TABLE_NAME>",
"INTERFACE": "<interface>|<ip-prefix>",
"PORT": "<port_name>"
"VLAN_INTERFACE_LIST": "^(Vlan[a-zA-Z0-9_-]+)$",
"VLAN_LIST": "^(Vlan[a-zA-Z0-9_-]+)$",
"VLAN_INTERFACE_IPPREFIX_LIST": "^(Vlan[a-zA-Z0-9_-]+)\|([a-fA-F0-9:./]+$)",
"VLAN_MEMBER_LIST": "^(Vlan[a-zA-Z0-9-_]+)\|(Ethernet[0-9]+)$",
"ACL_RULE_LIST": "^([a-zA-Z0-9_-]+)\|([a-zA-Z0-9_-]+)$",
"ACL_TABLE_LIST": "^([a-zA-Z0-9-_]+)$",
"INTERFACE_LIST": "^(Ethernet[0-9]+)$",
"INTERFACE_IPPREFIX_LIST": "^(Ethernet[0-9]+)\|([a-fA-F0-9:./]+)$",
"PORT_LIST": "^(Ethernet[0-9]+)$",
"LOOPBACK_INTERFACE_LIST": "^([a-zA-Z0-9-_]+)$",
"LOOPBACK_INTERFACE_IPPREFIX_LIST": "^([a-zA-Z0-9-_]+)\|([a-fA-F0-9:./]+)$",
}
#create a dict to map each key under primary key with a dict yang model.
#This is done to improve performance of mapping from values of TABLEs in
#config DB to leaf in YANG LIST.

leafDict = self.createLeafDict(model)

self.logInFile("Xlate {}".format(table))
# Find and extracts key from each dict in config
for pkey in config:
keyRegEx = keyExt[model['@name']]
# get keys from YANG model list itself
listKeys = model['key']['@value']

for pkey in config.keys():
try:
vKey = None
self.logInFile("xlate Extract pkey {} {}".format(pkey,keyExt[table]))
keyDict = self.extractKey(pkey, keyExt[table])
self.logInFile("xlateList Extract pkey {}".format(pkey))
# Find and extracts key from each dict in config
keyDict = self.extractKey(pkey, listKeys, keyRegEx)
# fill rest of the values in keyDict
for vKey in config[pkey]:
self.logInFile("xlate vkey {}".format(vKey), keyExt[table])
self.logInFile("xlateList vkey {}".format(vKey))
keyDict[vKey] = self.findYangTypedValue(vKey, \
config[pkey][vKey], leafDict)
yang.append(keyDict)
# delete pkey from config, done to match one key with one list
del config[pkey]

except Exception as e:
print("Exception while Config DB --> YANG: pkey:{}, "\
self.logInFile("xlateList Exception {}".format(e))
self.logInFile("Exception while Config DB --> YANG: pkey:{}, "\
"vKey:{}, value: {}".format(pkey, vKey, config[pkey].get(vKey)))
raise e
# with multilist, we continue matching other keys.
continue

return

Expand All @@ -295,19 +304,36 @@ def xlateList(self, model, yang, config, table):
"""
def xlateContainer(self, model, yang, config, table):

# if container contains single list with containerName_LIST and
# config is not empty then xLate the list
# To Handle multiple List, Make a copy of config, because we delete keys
# from config after each match. This is done to match one pkey with one list.
configC = config.copy()

clist = model.get('list')
# If single list exists in container,
if clist and isinstance(clist, dict) and \
clist['@name'] == model['@name']+"_LIST" and bool(config):
clist['@name'] == model['@name']+"_LIST" and bool(configC):
#print(clist['@name'])
yang[clist['@name']] = list()
self.xlateList(model['list'], yang[clist['@name']], \
config, table)
self.logInFile("xlateContainer listD {}".format(clist['@name']))
self.xlateList(clist, yang[clist['@name']], \
configC, table)
# clean empty lists
if len(yang[clist['@name']]) == 0:
del yang[clist['@name']]
#print(yang[clist['@name']])

# TODO: Handle mupltiple list and rest of the field in Container.
# We do not have any such instance in Yang model today.
# If multi-list exists in container,
elif clist and isinstance(clist, list) and bool(configC):
for modelList in clist:
yang[modelList['@name']] = list()
self.logInFile("xlateContainer listL {}".format(modelList['@name']))
self.xlateList(modelList, yang[modelList['@name']], configC, table)
# clean empty lists
if len(yang[modelList['@name']]) == 0:
del yang[modelList['@name']]

if len(configC):
raise(Exception("All Keys are not parsed in {}".format(table)))

return

Expand All @@ -325,6 +351,7 @@ def xlateConfigDBtoYang(self, jIn, yangJ):
# Add new top level container for first table in this container
yangJ[key] = dict() if yangJ.get(key) is None else yangJ[key]
yangJ[key][subkey] = dict()
self.logInFile("xlateConfigDBtoYang {}:{}".format(key, subkey))
self.xlateContainer(cmap['container'], yangJ[key][subkey], \
jIn[table], table)

Expand Down Expand Up @@ -389,7 +416,6 @@ def revYangConvert(val):

return vValue


"""
Rev xlate from <TABLE>_LIST to table in config DB
"""
Expand All @@ -398,26 +424,31 @@ def revXlateList(self, model, yang, config, table):
# TODO: define a keyExt dict as of now, but we should be able to
# extract this from YANG model extentions.
keyExt = {
"VLAN_INTERFACE": "<vlan_name>|<ip-prefix>",
"ACL_RULE": "<ACL_TABLE_NAME>|<RULE_NAME>",
"VLAN": "<vlan_name>",
"VLAN_MEMBER": "<vlan_name>|<port>",
"ACL_TABLE": "<ACL_TABLE_NAME>",
"INTERFACE": "<interface>|<ip-prefix>",
"PORT": "<port_name>"
"VLAN_INTERFACE_IPPREFIX_LIST": "<vlan_name>|<ip-prefix>",
"VLAN_INTERFACE_LIST": "<vlan_name>",
"VLAN_MEMBER_LIST": "<vlan_name>|<port>",
"VLAN_LIST": "<vlan_name>",
"ACL_RULE_LIST": "<ACL_TABLE_NAME>|<RULE_NAME>",
"ACL_TABLE_LIST": "<ACL_TABLE_NAME>",
"INTERFACE_LIST": "<port_name>",
"INTERFACE_IPPREFIX_LIST": "<port_name>|<ip-prefix>",
"LOOPBACK_INTERFACE_LIST": "<loopback_interface_name>",
"LOOPBACK_INTERFACE_IPPREFIX_LIST": "<loopback_interface_name>|<ip-prefix>",
"PORT_LIST": "<port_name>"

}

keyRegEx = keyExt[model['@name']]
# create a dict to map each key under primary key with a dict yang model.
# This is done to improve performance of mapping from values of TABLEs in
# config DB to leaf in YANG LIST.
leafDict = self.createLeafDict(model)

# list with name <TABLE>_LIST should be removed,
# right now we have only this instance of LIST
if model['@name'] == table + "_LIST":
# list with name <NAME>_LIST should be removed,
if "_LIST" in model['@name']:
for entry in yang:
# create key of config DB table
pkey, pkeydict = self.createKey(entry, keyExt[table])
pkey, pkeydict = self.createKey(entry, keyRegEx)
config[pkey]= dict()
# fill rest of the entries
for key in entry:
Expand All @@ -438,9 +469,8 @@ def revXlateContainer(self, model, yang, config, table):
modelList = model['list']
# Pass matching list from Yang Json
self.revXlateList(modelList, yang[modelList['@name']], config, table)
else:
# TODO: Container[TABLE] contains multiple lists. [Test Pending]
# No instance now.

elif isinstance(model['list'], list):
for modelList in model['list']:
self.revXlateList(modelList, yang[modelList['@name']], config, table)

Expand Down Expand Up @@ -575,10 +605,11 @@ def load_data(self, configdbJson, allowExtraTables=True):
# reset xlate
self.xlateJson = dict()
# self.jIn will be cropped
self.cropConfigDB("cropped.json", allowExtraTables)
self.cropConfigDB(allowExtraTables=allowExtraTables)
# xlated result will be in self.xlateJson
self.xlateConfigDB()
#print(self.xlateJson)
self.logInFile("Try to load Data in the tree")
self.root = self.ctx.parse_data_mem(dumps(self.xlateJson), \
ly.LYD_JSON, ly.LYD_OPT_CONFIG|ly.LYD_OPT_STRICT)

Expand Down
7 changes: 7 additions & 0 deletions src/sonic-yang-mgmt/tests/yang-model-tests/yangTest.json
Original file line number Diff line number Diff line change
Expand Up @@ -627,6 +627,8 @@

"SAMPLE_CONFIG_DB_JSON": {
"VLAN_INTERFACE": {
"Vlan111": {},
"Vlan777": {},
"Vlan111|2a04:5555:45:6709::1/64": {
"scope": "global",
"family": "IPv6"
Expand Down Expand Up @@ -1178,6 +1180,10 @@
}
},
"INTERFACE": {
"Ethernet112": {},
"Ethernet14": {},
"Ethernet16": {},
"Ethernet18": {},
"Ethernet112|2a04:5555:40:a709::2/126": {
"scope": "global",
"family": "IPv6"
Expand Down Expand Up @@ -1259,6 +1265,7 @@
}
},
"LOOPBACK_INTERFACE": {
"Loopback0": {},
"Loopback0|2a04:5555:40:4::4e9/128": {
"scope": "global",
"family": "IPv6"
Expand Down
2 changes: 2 additions & 0 deletions src/sonic-yang-mgmt/yang-models/sonic-loopback-interface.yang
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,11 @@ module sonic-loopback-interface {
/* end of LOOPBACK_INTERFACE_LIST */

list LOOPBACK_INTERFACE_IPPREFIX_LIST {

key "loopback_interface_name ip-prefix";

leaf loopback_interface_name{

/* This node must be present in LOOPBACK_INTERFACE_LIST */
must "(current() = ../../LOOPBACK_INTERFACE_LIST[loopback_interface_name=current()]/loopback_interface_name)"
{
Expand Down
2 changes: 0 additions & 2 deletions src/sonic-yang-mgmt/yang-models/sonic-vlan.yang
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,6 @@ module sonic-vlan {
}

leaf admin_status {
mandatory true;
type head:admin_status;
}

Expand Down Expand Up @@ -172,7 +171,6 @@ module sonic-vlan {

leaf port {
/* key elements are mandatory by default */
mandatory true;
type leafref {
path /port:sonic-port/port:PORT/port:PORT_LIST/port:port_name;
}
Expand Down