[_sonic_yang_ext.py]: Parse multilist in YANG Container. (#38)

* [_sonic_yang_ext.py]: Parse multilist in YANG Container.

This is needed to support VRF feature in SONiC.

Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com

* [sonic-vlan.yang]: Change Vlan yang models to remove admin-status as mandatory attribute.

Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com

* [_sonic_yang_ext.py]: Minor fix in _sonic_yang_ext.py

Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com

src/sonic-yang-mgmt/_sonic_yang_ext.py

@@ -129,31 +129,28 @@ def cropConfigDB(self, croppedFile=None, allowExtraTables=True):
 
 """
 Extract keys from table entry in Config DB and return in a dict
-For Example: regex = <vlan_name>| and tableKey = "Vlan111|2a04:5555:45:6709::1/64"
 
-1.) first code will extract key list from regex, i.e. vlan_name and ip_prefix.
-2.) then will create another regex(regexV) to extract Values from tableKey by
- replacing " --> extractor i.e. (.*?)" in regex.
-3.) Then will extract values from tableKey with regexV.
-4.) Resulting Dict will be:
+Input:
+tableKey: Config DB Primary Key, Example tableKey = "Vlan111|2a04:5555:45:6709::1/64"
+keys: key string from YANG list, i.e. 'vlan_name ip-prefix'.
+regex: A regex to extract keys from tableKeys, good to have it as accurate as possible.
+
+Return:
 KeyDict = {"vlan_name": "Vlan111", "ip-prefix": "2a04:5555:45:6709::1/64"}
 """
-def extractKey(self, tableKey, regex):
+def extractKey(self, tableKey, keys, regex):
 
- # get the keys from regex of key extractor
- keyList = re.findall(r'<(.*?)>', regex)
- # create a regex to get values from tableKey
- # and change separator to text in regexV
- regexV = re.sub('<.*?>', '(.*?)', regex)
- regexV = re.sub('\|', '\\|', regexV)
+ keyList = keys.split()
+ #self.logInFile("extractKey {}".format(keyList))
  # get the value groups
- value = re.match(r'^'+regexV+'$', tableKey)
+ value = re.match(regex, tableKey)
  # create the keyDict
  i = 1
  keyDict = dict()
  for k in keyList:
  if value.group(i):
  keyDict[k] = value.group(i)
+ # self.logInFile("extractKey {} {}".format(k, keyDict[k]))
  else:
  raise Exception("Value not found for {} in {}".format(k, tableKey))
  i = i + 1
@@ -254,37 +251,49 @@ def xlateList(self, model, yang, config, table):
  # TODO: define a keyExt dict as of now, but we should be able to extract
  # this from YANG model extentions.
  keyExt = {
- "VLAN_INTERFACE": "<vlan_name>|<ip-prefix>",
- "ACL_RULE": "<ACL_TABLE_NAME>|<RULE_NAME>",
- "VLAN": "<vlan_name>",
- "VLAN_MEMBER": "<vlan_name>|<port>",
- "ACL_TABLE": "<ACL_TABLE_NAME>",
- "INTERFACE": "<interface>|<ip-prefix>",
- "PORT": "<port_name>"
+ "VLAN_INTERFACE_LIST": "^(Vlan[a-zA-Z0-9_-]+)$",
+ "VLAN_LIST": "^(Vlan[a-zA-Z0-9_-]+)$",
+ "VLAN_INTERFACE_IPPREFIX_LIST": "^(Vlan[a-zA-Z0-9_-]+)\|([a-fA-F0-9:./]+$)",
+ "VLAN_MEMBER_LIST": "^(Vlan[a-zA-Z0-9-_]+)\|(Ethernet[0-9]+)$",
+ "ACL_RULE_LIST": "^([a-zA-Z0-9_-]+)\|([a-zA-Z0-9_-]+)$",
+ "ACL_TABLE_LIST": "^([a-zA-Z0-9-_]+)$",
+ "INTERFACE_LIST": "^(Ethernet[0-9]+)$",
+ "INTERFACE_IPPREFIX_LIST": "^(Ethernet[0-9]+)\|([a-fA-F0-9:./]+)$",
+ "PORT_LIST": "^(Ethernet[0-9]+)$",
+ "LOOPBACK_INTERFACE_LIST": "^([a-zA-Z0-9-_]+)$",
+ "LOOPBACK_INTERFACE_IPPREFIX_LIST": "^([a-zA-Z0-9-_]+)\|([a-fA-F0-9:./]+)$",
  }
  #create a dict to map each key under primary key with a dict yang model.
  #This is done to improve performance of mapping from values of TABLEs in
  #config DB to leaf in YANG LIST.
 
  leafDict = self.createLeafDict(model)
 
- self.logInFile("Xlate {}".format(table))
- # Find and extracts key from each dict in config
- for pkey in config:
+ keyRegEx = keyExt[model['@name']]
+ # get keys from YANG model list itself
+ listKeys = model['key']['@value']
+
+ for pkey in config.keys():
  try:
  vKey = None
- self.logInFile("xlate Extract pkey {} {}".format(pkey,keyExt[table]))
- keyDict = self.extractKey(pkey, keyExt[table])
+ self.logInFile("xlateList Extract pkey {}".format(pkey))
+ # Find and extracts key from each dict in config
+ keyDict = self.extractKey(pkey, listKeys, keyRegEx)
  # fill rest of the values in keyDict
  for vKey in config[pkey]:
- self.logInFile("xlate vkey {}".format(vKey), keyExt[table])
+ self.logInFile("xlateList vkey {}".format(vKey))
  keyDict[vKey] = self.findYangTypedValue(vKey, \
  config[pkey][vKey], leafDict)
  yang.append(keyDict)
+ # delete pkey from config, done to match one key with one list
+ del config[pkey]
+
  except Exception as e:
- print("Exception while Config DB --> YANG: pkey:{}, "\
+ self.logInFile("xlateList Exception {}".format(e))
+ self.logInFile("Exception while Config DB --> YANG: pkey:{}, "\
  "vKey:{}, value: {}".format(pkey, vKey, config[pkey].get(vKey)))
- raise e
+ # with multilist, we continue matching other keys.
+ continue
 
  return
 
@@ -295,19 +304,36 @@ def xlateList(self, model, yang, config, table):
 """
 def xlateContainer(self, model, yang, config, table):
 
- # if container contains single list with containerName_LIST and
- # config is not empty then xLate the list
+ # To Handle multiple List, Make a copy of config, because we delete keys
+ # from config after each match. This is done to match one pkey with one list.
+ configC = config.copy()
+
  clist = model.get('list')
+ # If single list exists in container,
  if clist and isinstance(clist, dict) and \
- clist['@name'] == model['@name']+"_LIST" and bool(config):
+ clist['@name'] == model['@name']+"_LIST" and bool(configC):
  #print(clist['@name'])
  yang[clist['@name']] = list()
- self.xlateList(model['list'], yang[clist['@name']], \
- config, table)
+ self.logInFile("xlateContainer listD {}".format(clist['@name']))
+ self.xlateList(clist, yang[clist['@name']], \
+ configC, table)
+ # clean empty lists
+ if len(yang[clist['@name']]) == 0:
+ del yang[clist['@name']]
  #print(yang[clist['@name']])
 
- # TODO: Handle mupltiple list and rest of the field in Container.
- # We do not have any such instance in Yang model today.
+ # If multi-list exists in container,
+ elif clist and isinstance(clist, list) and bool(configC):
+ for modelList in clist:
+ yang[modelList['@name']] = list()
+ self.logInFile("xlateContainer listL {}".format(modelList['@name']))
+ self.xlateList(modelList, yang[modelList['@name']], configC, table)
+ # clean empty lists
+ if len(yang[modelList['@name']]) == 0:
+ del yang[modelList['@name']]
+
+ if len(configC):
+ raise(Exception("All Keys are not parsed in {}".format(table)))
 
  return
 
@@ -325,6 +351,7 @@ def xlateConfigDBtoYang(self, jIn, yangJ):
  # Add new top level container for first table in this container
  yangJ[key] = dict() if yangJ.get(key) is None else yangJ[key]
  yangJ[key][subkey] = dict()
+ self.logInFile("xlateConfigDBtoYang {}:{}".format(key, subkey))
  self.xlateContainer(cmap['container'], yangJ[key][subkey], \
  jIn[table], table)
 
@@ -389,7 +416,6 @@ def revYangConvert(val):
 
  return vValue
 
-
 """
 Rev xlate from <TABLE>_LIST to table in config DB
 """
@@ -398,26 +424,31 @@ def revXlateList(self, model, yang, config, table):
  # TODO: define a keyExt dict as of now, but we should be able to
  # extract this from YANG model extentions.
  keyExt = {
- "VLAN_INTERFACE": "<vlan_name>|<ip-prefix>",
- "ACL_RULE": "<ACL_TABLE_NAME>|<RULE_NAME>",
- "VLAN": "<vlan_name>",
- "VLAN_MEMBER": "<vlan_name>|<port>",
- "ACL_TABLE": "<ACL_TABLE_NAME>",
- "INTERFACE": "<interface>|<ip-prefix>",
- "PORT": "<port_name>"
+ "VLAN_INTERFACE_IPPREFIX_LIST": "<vlan_name>|<ip-prefix>",
+ "VLAN_INTERFACE_LIST": "<vlan_name>",
+ "VLAN_MEMBER_LIST": "<vlan_name>|<port>",
+ "VLAN_LIST": "<vlan_name>",
+ "ACL_RULE_LIST": "<ACL_TABLE_NAME>|<RULE_NAME>",
+ "ACL_TABLE_LIST": "<ACL_TABLE_NAME>",
+ "INTERFACE_LIST": "<port_name>",
+ "INTERFACE_IPPREFIX_LIST": "<port_name>|<ip-prefix>",
+ "LOOPBACK_INTERFACE_LIST": "<loopback_interface_name>",
+ "LOOPBACK_INTERFACE_IPPREFIX_LIST": "<loopback_interface_name>|<ip-prefix>",
+ "PORT_LIST": "<port_name>"
+
  }
 
+ keyRegEx = keyExt[model['@name']]
  # create a dict to map each key under primary key with a dict yang model.
  # This is done to improve performance of mapping from values of TABLEs in
  # config DB to leaf in YANG LIST.
  leafDict = self.createLeafDict(model)
 
- # list with name <TABLE>_LIST should be removed,
- # right now we have only this instance of LIST
- if model['@name'] == table + "_LIST":
+ # list with name <NAME>_LIST should be removed,
+ if "_LIST" in model['@name']:
  for entry in yang:
  # create key of config DB table
- pkey, pkeydict = self.createKey(entry, keyExt[table])
+ pkey, pkeydict = self.createKey(entry, keyRegEx)
  config[pkey]= dict()
  # fill rest of the entries
  for key in entry:
@@ -438,9 +469,8 @@ def revXlateContainer(self, model, yang, config, table):
  modelList = model['list']
  # Pass matching list from Yang Json
  self.revXlateList(modelList, yang[modelList['@name']], config, table)
- else:
- # TODO: Container[TABLE] contains multiple lists. [Test Pending]
- # No instance now.
+
+ elif isinstance(model['list'], list):
  for modelList in model['list']:
  self.revXlateList(modelList, yang[modelList['@name']], config, table)
 
@@ -575,10 +605,11 @@ def load_data(self, configdbJson, allowExtraTables=True):
  # reset xlate
  self.xlateJson = dict()
  # self.jIn will be cropped
- self.cropConfigDB("cropped.json", allowExtraTables)
+ self.cropConfigDB(allowExtraTables=allowExtraTables)
  # xlated result will be in self.xlateJson
  self.xlateConfigDB()
  #print(self.xlateJson)
+ self.logInFile("Try to load Data in the tree")
  self.root = self.ctx.parse_data_mem(dumps(self.xlateJson), \
  ly.LYD_JSON, ly.LYD_OPT_CONFIG|ly.LYD_OPT_STRICT)
 

src/sonic-yang-mgmt/tests/yang-model-tests/yangTest.json

@@ -627,6 +627,8 @@
 
  "SAMPLE_CONFIG_DB_JSON": {
  "VLAN_INTERFACE": {
+ "Vlan111": {},
+ "Vlan777": {},
  "Vlan111|2a04:5555:45:6709::1/64": {
  "scope": "global",
  "family": "IPv6"
@@ -1178,6 +1180,10 @@
  }
  },
  "INTERFACE": {
+ "Ethernet112": {},
+ "Ethernet14": {},
+ "Ethernet16": {},
+ "Ethernet18": {},
  "Ethernet112|2a04:5555:40:a709::2/126": {
  "scope": "global",
  "family": "IPv6"
@@ -1259,6 +1265,7 @@
  }
  },
  "LOOPBACK_INTERFACE": {
+ "Loopback0": {},
  "Loopback0|2a04:5555:40:4::4e9/128": {
  "scope": "global",
  "family": "IPv6"

src/sonic-yang-mgmt/yang-models/sonic-loopback-interface.yang

@@ -44,9 +44,11 @@ module sonic-loopback-interface {
  /* end of LOOPBACK_INTERFACE_LIST */
 
  list LOOPBACK_INTERFACE_IPPREFIX_LIST {
+
  key "loopback_interface_name ip-prefix";
 
  leaf loopback_interface_name{
+
  /* This node must be present in LOOPBACK_INTERFACE_LIST */
  must "(current() = ../../LOOPBACK_INTERFACE_LIST[loopback_interface_name=current()]/loopback_interface_name)"
  {

src/sonic-yang-mgmt/yang-models/sonic-vlan.yang

@@ -140,7 +140,6 @@ module sonic-vlan {
  }
 
  leaf admin_status {
- mandatory true;
  type head:admin_status;
  }
 
@@ -172,7 +171,6 @@ module sonic-vlan {
 
  leaf port {
  /* key elements are mandatory by default */
- mandatory true;
  type leafref {
  path /port:sonic-port/port:PORT/port:PORT_LIST/port:port_name;
  }