[buildImage] Add support of platform.json parsing to portconfig.py fi…

…le (#3909)

**- What I did**
Add support of **platform.json** parsing to **portconfig.py** file which is being used by **_sonic-cfggen_** and ***minigraph.py*** file under ***src/sonic-config-engine*** folder to get port config via get_port_config  function.

**- How I did it**

1.  **portconfig.py** file will first check whether the **platform.json** file is there or not. if not then whether port_config.ini file is there or not. Modified **get_port_config_file_name** for this purpose.
2. Added two separate functions i.e. **parse_platform_json_file**  to get port attributes from **platform.json** and **gen_port_config** to generate port attributes.
3. Added another two functions i.e  get_breakout_mode parse_breakout_mode to get breakout mode and parse breakout mode from platform.json respectively.

**- How to verify it**

rebuilt "sonic_config_engine-1.0" wheel package with all the test cases.All the below-mentioned test cases passed.

```
# Check whether all interfaces present or not as per platform.json
def test_platform_json_interfaces_keys(self):
       
# Check specific Interface with it's proper configuration as per platform.json
def test_platform_json_specific_ethernet_interfaces(self):

# Check all Interface with it's proper configuration as per platform.json
def test_platform_json_all_ethernet_interfaces(self):
```

Signed-off-by: Sangita Maity <sangitamaity0211@gmail.com>

src/sonic-config-engine/minigraph.py

@@ -772,7 +772,7 @@ def enable_internal_bgp_session(bgp_sessions, filename, asic_name):
 # Main functions
 #
 ###############################################################################
-def parse_xml(filename, platform=None, port_config_file=None, asic_name=None):
+def parse_xml(filename, platform=None, port_config_file=None, asic_name=None, hwsku_config_file=None):
  """ Parse minigraph xml file.
 
  Keyword arguments:
@@ -782,6 +782,7 @@ def parse_xml(filename, platform=None, port_config_file=None, asic_name=None):
  asic_name -- asic name; to parse multi-asic device minigraph to 
  generate asic specific configuration.
  """
+
  root = ET.parse(filename).getroot()
 
  u_neighbors = None
@@ -836,7 +837,7 @@ def parse_xml(filename, platform=None, port_config_file=None, asic_name=None):
  if child.tag == str(docker_routing_config_mode_qn):
  docker_routing_config_mode = child.text
 
- (ports, alias_map, alias_asic_map) = get_port_config(hwsku=hwsku, platform=platform, port_config_file=port_config_file, asic=asic_id)
+ (ports, alias_map, alias_asic_map) = get_port_config(hwsku=hwsku, platform=platform, port_config_file=port_config_file, asic=asic_id, hwsku_config_file=hwsku_config_file)
  port_alias_map.update(alias_map)
  port_alias_asic_map.update(alias_asic_map)
 

src/sonic-config-engine/portconfig.py

@@ -1,33 +1,144 @@
 #!/usr/bin/env python
-import os
-import sys
+try:
+ import os
+ import sys
+ import json
+ import ast
+ import re
+ from collections import OrderedDict
+ from swsssdk import ConfigDBConnector
+except ImportError as e:
+ raise ImportError("%s - required module not found" % str(e))
 
+# Global Variable
+PLATFORM_ROOT_PATH = '/usr/share/sonic/device'
+PLATFORM_ROOT_PATH_DOCKER = '/usr/share/sonic/platform'
+SONIC_ROOT_PATH = '/usr/share/sonic'
+HWSKU_ROOT_PATH = '/usr/share/sonic/hwsku'
+
+PLATFORM_JSON = 'platform.json'
+PORT_CONFIG_INI = 'port_config.ini'
+HWSKU_JSON = 'hwsku.json'
+
+PORT_STR = "Ethernet"
+BRKOUT_MODE = "default_brkout_mode"
+CUR_BRKOUT_MODE = "brkout_mode"
+INTF_KEY = "interfaces"
+
+BRKOUT_PATTERN = r'(\d{1,3})x(\d{1,3}G)(\[\d{1,3}G\])?(\((\d{1,3})\))?'
+
+
+#
+# Helper Functions
+#
+def readJson(filename):
+ # Read 'platform.json' or 'hwsku.json' file
+ try:
+ with open(filename) as fp:
+ try:
+ data = json.load(fp)
+ except json.JSONDecodeError:
+ print("Json file does not exist")
+ data_dict = ast.literal_eval(json.dumps(data))
+ return data_dict
+ except Exception as e:
+ print("error occurred while parsing json:", sys.exc_info()[1])
+ return None
+
+def db_connect_configdb():
+ """
+ Connect to configdb
+ """
+ config_db = ConfigDBConnector()
+ if config_db is None:
+ return None
+ try:
+ """
+ This could be blocking during the config load_minigraph phase,
+ as the CONFIG_DB_INITIALIZED is not yet set in the configDB.
+ We can ignore the check by using config_db.db_connect('CONFIG_DB') instead
+ """
+ # Connect only if available & initialized
+ config_db.connect(wait_for_init=False)
+ except Exception as e:
+ config_db = None
+ return config_db
 
 def get_port_config_file_name(hwsku=None, platform=None, asic=None):
+
+ # check 'platform.json' file presence
+ port_config_candidates_Json = []
+ port_config_candidates_Json.append(os.path.join(PLATFORM_ROOT_PATH_DOCKER, PLATFORM_JSON))
+ if platform:
+ port_config_candidates_Json.append(os.path.join(PLATFORM_ROOT_PATH, platform, PLATFORM_JSON))
+
+ # check 'portconfig.ini' file presence
  port_config_candidates = []
- port_config_candidates.append('/usr/share/sonic/hwsku/port_config.ini')
+ port_config_candidates.append(os.path.join(HWSKU_ROOT_PATH, PORT_CONFIG_INI))
  if hwsku:
  if platform:
  if asic:
- port_config_candidates.append(os.path.join('/usr/share/sonic/device', platform, hwsku, asic,'port_config.ini'))
- port_config_candidates.append(os.path.join('/usr/share/sonic/device', platform, hwsku, 'port_config.ini'))
- port_config_candidates.append(os.path.join('/usr/share/sonic/platform', hwsku, 'port_config.ini'))
- port_config_candidates.append(os.path.join('/usr/share/sonic', hwsku, 'port_config.ini'))
- for candidate in port_config_candidates:
+ port_config_candidates.append(os.path.join(PLATFORM_ROOT_PATH, platform, hwsku, asic, PORT_CONFIG_INI))
+ port_config_candidates.append(os.path.join(PLATFORM_ROOT_PATH, platform, hwsku, PORT_CONFIG_INI))
+ port_config_candidates.append(os.path.join(PLATFORM_ROOT_PATH_DOCKER, hwsku, PORT_CONFIG_INI))
+ port_config_candidates.append(os.path.join(SONIC_ROOT_PATH, hwsku, PORT_CONFIG_INI))
+
+ elif platform and not hwsku:
+ port_config_candidates.append(os.path.join(PLATFORM_ROOT_PATH, platform, PORT_CONFIG_INI))
+
+ for candidate in port_config_candidates_Json + port_config_candidates:
+ if os.path.isfile(candidate):
+ return candidate
+ return None
+
+def get_hwsku_file_name(hwsku=None, platform=None):
+ hwsku_candidates_Json = []
+ hwsku_candidates_Json.append(os.path.join(HWSKU_ROOT_PATH, HWSKU_JSON))
+ if hwsku:
+ if platform:
+ hwsku_candidates_Json.append(os.path.join(PLATFORM_ROOT_PATH, platform, hwsku, HWSKU_JSON))
+ hwsku_candidates_Json.append(os.path.join(PLATFORM_ROOT_PATH_DOCKER, hwsku, HWSKU_JSON))
+ hwsku_candidates_Json.append(os.path.join(SONIC_ROOT_PATH, hwsku, HWSKU_JSON))
+ for candidate in hwsku_candidates_Json:
  if os.path.isfile(candidate):
  return candidate
  return None
-
 
+def get_port_config(hwsku=None, platform=None, port_config_file=None, hwsku_config_file=None, asic=None):
+ config_db = db_connect_configdb()
+ # If available, Read from CONFIG DB first
+ if config_db is not None and port_config_file is None:
+
+ port_data = config_db.get_table("PORT")
+ if bool(port_data):
+ ports = ast.literal_eval(json.dumps(port_data))
+ port_alias_map = {}
+ port_alias_asic_map = {}
+ for intf_name in ports.keys():
+ port_alias_map[ports[intf_name]["alias"]] = intf_name
+ return (ports, port_alias_map, port_alias_asic_map)
 
-def get_port_config(hwsku=None, platform=None, port_config_file=None, asic=None):
  if not port_config_file:
  port_config_file = get_port_config_file_name(hwsku, platform, asic)
  if not port_config_file:
  return ({}, {}, {})
- return parse_port_config_file(port_config_file)
 
 
+ # Read from 'platform.json' file
+ if port_config_file.endswith('.json'):
+ if not hwsku_config_file:
+ hwsku_json_file = get_hwsku_file_name(hwsku, platform)
+ if not hwsku_json_file:
+ return ({}, {}, {})
+ else:
+ hwsku_json_file = hwsku_config_file
+
+ return parse_platform_json_file(hwsku_json_file, port_config_file)
+
+ # If 'platform.json' file is not available, read from 'port_config.ini'
+ else:
+ return parse_port_config_file(port_config_file)
+
 def parse_port_config_file(port_config_file):
  ports = {}
  port_alias_map = {}
@@ -63,4 +174,142 @@ def parse_port_config_file(port_config_file):
  port_alias_asic_map[data['alias']] = data['asic_port_name'].strip()
  return (ports, port_alias_map, port_alias_asic_map)
 
+# Generate configs (i.e. alias, lanes, speed, index) for port
+def gen_port_config(ports, parent_intf_id, index, alias_at_lanes, lanes, k, offset):
+ if k is not None:
+ num_lane_used, speed, alt_speed, _ , assigned_lane = k[0], k[1], k[2], k[3], k[4]
+
+ # In case of symmetric mode
+ if assigned_lane is None:
+ assigned_lane = len(lanes.split(","))
+
+ parent_intf_id = int(offset)+int(parent_intf_id)
+ alias_start = 0 + offset
+
+ step = int(assigned_lane)//int(num_lane_used)
+ for i in range(0,int(assigned_lane), step):
+ intf_name = PORT_STR + str(parent_intf_id)
+ ports[intf_name] = {}
+ ports[intf_name]['alias'] = alias_at_lanes.split(",")[alias_start]
+ ports[intf_name]['lanes'] = ','.join(lanes.split(",")[alias_start:alias_start+step])
+ if speed:
+ speed_pat = re.search("^((\d+)G|\d+)$", speed.upper())
+ if speed_pat is None:
+ raise Exception('{} speed is not Supported...'.format(speed))
+ speed_G, speed_orig = speed_pat.group(2), speed_pat.group(1)
+ if speed_G:
+ conv_speed = int(speed_G)*1000
+ else:
+ conv_speed = int(speed_orig)
+ ports[intf_name]['speed'] = str(conv_speed)
+ else:
+ raise Exception('Regex return for speed is None...')
+
+ ports[intf_name]['index'] = index.split(",")[alias_start]
+ ports[intf_name]['admin_status'] = "up"
+
+ parent_intf_id += step
+ alias_start += step
+
+ offset = int(assigned_lane) + int(offset)
+ return offset
+ else:
+ raise Exception('Regex return for k is None...')
+
+"""
+Given a port and breakout mode, this method returns
+the list of child ports using platform_json file
+"""
+def get_child_ports(interface, breakout_mode, platform_json_file):
+ child_ports = {}
+
+ port_dict = readJson(platform_json_file)
+
+ index = port_dict[INTF_KEY][interface]['index']
+ alias_at_lanes = port_dict[INTF_KEY][interface]['alias_at_lanes']
+ lanes = port_dict[INTF_KEY][interface]['lanes']
+
+ """
+ Example of match_list for some breakout_mode using regex
+ Breakout Mode -------> Match_list
+ -----------------------------
+ 2x25G(2)+1x50G(2) ---> [('2', '25G', None, '(2)', '2'), ('1', '50G', None, '(2)', '2')]
+ 1x50G(2)+2x25G(2) ---> [('1', '50G', None, '(2)', '2'), ('2', '25G', None, '(2)', '2')]
+ 1x100G[40G] ---------> [('1', '100G', '[40G]', None, None)]
+ 2x50G ---------------> [('2', '50G', None, None, None)]
+ """
+ # Asymmetric breakout mode
+ if re.search("\+",breakout_mode) is not None:
+ breakout_parts = breakout_mode.split("+")
+ match_list = [re.match(BRKOUT_PATTERN, i).groups() for i in breakout_parts]
+
+ # Symmetric breakout mode
+ else:
+ match_list = [re.match(BRKOUT_PATTERN, breakout_mode).groups()]
+
+ offset = 0
+ parent_intf_id = int(re.search("Ethernet(\d+)", interface).group(1))
+ for k in match_list:
+ offset = gen_port_config(child_ports, parent_intf_id, index, alias_at_lanes, lanes, k, offset)
+ return child_ports
+
+def parse_platform_json_file(hwsku_json_file, platform_json_file):
+ ports = {}
+ port_alias_map = {}
+ port_alias_asic_map = {}
+
+ port_dict = readJson(platform_json_file)
+ hwsku_dict = readJson(hwsku_json_file)
+
+ if not port_dict:
+ raise Exception("port_dict is none")
+ if not hwsku_dict:
+ raise Exception("hwsku_dict is none")
+
+ if INTF_KEY not in port_dict or INTF_KEY not in hwsku_dict:
+ raise Exception("INTF_KEY is not present in appropriate file")
+
+ for intf in port_dict[INTF_KEY]:
+ if intf not in hwsku_dict[INTF_KEY]:
+ raise Exception("{} is not available in hwsku_dict".format(intf))
+
+ # take default_brkout_mode from hwsku.json
+ brkout_mode = hwsku_dict[INTF_KEY][intf][BRKOUT_MODE]
+
+ child_ports = get_child_ports(intf, brkout_mode, platform_json_file)
+ ports.update(child_ports)
+
+ if not ports:
+ raise Exception("Ports dictionary is empty")
+
+ for i in ports.keys():
+ port_alias_map[ports[i]["alias"]]= i
+ return (ports, port_alias_map, port_alias_asic_map)
+
+
+def get_breakout_mode(hwsku=None, platform=None, port_config_file=None):
+ if not port_config_file:
+ port_config_file = get_port_config_file_name(hwsku, platform)
+ if not port_config_file:
+ return None
+ if port_config_file.endswith('.json'):
+ hwsku_json_file = get_hwsku_file_name(hwsku, platform)
+ if not hwsku_json_file:
+ raise Exception("'hwsku_json' file does not exist!!! This file is necessary to proceed forward.")
+
+ return parse_breakout_mode(hwsku_json_file)
+ else:
+ return None
+
+def parse_breakout_mode(hwsku_json_file):
+ brkout_table = {}
+ hwsku_dict = readJson(hwsku_json_file)
+ if not hwsku_dict:
+ raise Exception("hwsku_dict is empty")
+ if INTF_KEY not in hwsku_dict:
+ raise Exception("INTF_KEY is not present in hwsku_dict")
 
+ for intf in hwsku_dict[INTF_KEY]:
+ brkout_table[intf] = {}
+ brkout_table[intf][CUR_BRKOUT_MODE] = hwsku_dict[INTF_KEY][intf][BRKOUT_MODE]
+ return brkout_table

src/sonic-config-engine/sonic-cfggen

@@ -38,7 +38,7 @@ from minigraph import minigraph_encoder
 from minigraph import parse_xml
 from minigraph import parse_device_desc_xml
 from minigraph import parse_asic_sub_role
-from portconfig import get_port_config
+from portconfig import get_port_config, get_port_config_file_name, get_breakout_mode
 from sonic_device_util import get_machine_info
 from sonic_device_util import get_platform_info
 from sonic_device_util import get_system_mac
@@ -199,6 +199,7 @@ def main():
  group.add_argument("-k", "--hwsku", help="HwSKU")
  parser.add_argument("-n", "--namespace", help="namespace name", nargs='?', const=None, default=None)
  parser.add_argument("-p", "--port-config", help="port config file, used with -m or -k", nargs='?', const=None)
+ parser.add_argument("-S", "--hwsku-config", help="hwsku config file, used with -p and -m or -k", nargs='?', const=None)
  parser.add_argument("-y", "--yaml", help="yaml file that contains additional variables", action='append', default=[])
  parser.add_argument("-j", "--json", help="json file that contains additional variables", action='append', default=[])
  parser.add_argument("-a", "--additional-data", help="addition data, in json string")
@@ -240,12 +241,18 @@ def main():
  'hwsku': hwsku
  }}}
  deep_update(data, hardware_data)
+ if args.port_config is None:
+ args.port_config = get_port_config_file_name(hwsku, platform)
  (ports, _, _) = get_port_config(hwsku, platform, args.port_config, asic_id)
  if not ports:
  print('Failed to get port config', file=sys.stderr)
  sys.exit(1)
  deep_update(data, {'PORT': ports})
 
+ brkout_table = get_breakout_mode(hwsku, platform, args.port_config)
+ if brkout_table is not None:
+ deep_update(data, {'BREAKOUT_CFG': brkout_table})
+
  for json_file in args.json:
  with open(json_file, 'r') as stream:
  deep_update(data, FormatConverter.to_deserialized(json.load(stream)))
@@ -254,11 +261,11 @@ def main():
  minigraph = args.minigraph
  if platform:
  if args.port_config != None:
- deep_update(data, parse_xml(minigraph, platform, args.port_config, asic_name=asic_name))
+ deep_update(data, parse_xml(minigraph, platform, args.port_config, asic_name=asic_name, hwsku_config_file=args.hwsku_config))
  else:
  deep_update(data, parse_xml(minigraph, platform, asic_name=asic_name))
  else:
- deep_update(data, parse_xml(minigraph, port_config_file=args.port_config, asic_name=asic_name))
+ deep_update(data, parse_xml(minigraph, port_config_file=args.port_config, asic_name=asic_name, hwsku_config_file=args.hwsku_config))
 
  if args.device_description != None:
  deep_update(data, parse_device_desc_xml(args.device_description))