[Multi-asic]: Support multi-asic platform (sonic-net#126)

* [sonic-ax-impl]: Add SNMP support for multi-asic platform:
- Modify interface, lag, queue related functions to get data from
  all databases.
- Update implementation of the following MIBS to get data from all
  databases:
  - InterfaceMIB
  - InterfaceMIBObjects
  - CiscoSystemExtMIB
  - csqIfQosGroupStatsTable
* Fix as per review comments.
Test case for init_namespace_dbs
* Remove whitespace.
* Add namespace support to IpCidrRouteTable MIB.
* Add testcases for multiple namesapce for below MIBS:
InterfaceMIB
InterfaceMIBObjects
cpfcIfPriorityTable
* Remove debug print messages.
* Modified as per review comments.
Added a new class for Namespace related functions.
Updated single namespace testcase to load database_config.json.
* Changes made as per LGTM comments and review comments.
* Minor fix to remove import of module not being used.
* Fix PortChannel name in asic1 mock table.
* Remove import of libraries that are not required.
* Corrections in namespace database json files.
* Add namespace test case for csqIfQosGroupStatsTable.
* Fix SonicV2Connector mock __init__ function to match changes
in swsssdk.
* Change function name as per review comment.
Add mock connect function to get mock db filenames.

src/sonic_ax_impl/mibs/__init__.py

@@ -1,7 +1,9 @@
 import pprint
 import re
+import os
 
 from swsssdk import SonicV2Connector
+from swsssdk import SonicDBConfig
 from swsssdk import port_util
 from swsssdk.port_util import get_index, get_index_from_str
 from ax_interface.mib import MIBUpdater
@@ -148,7 +150,7 @@ def init_db():
     """
     # SyncD database connector. THIS MUST BE INITIALIZED ON A PER-THREAD BASIS.
     # Redis PubSub objects (such as those within swsssdk) are NOT thread-safe.
-    db_conn = SonicV2Connector(**redis_kwargs)
+    db_conn = SonicV2Connector(**redis_kwargs) 
 
     return db_conn
 
@@ -182,8 +184,6 @@ def init_mgmt_interface_tables(db_conn):
 
     return oid_name_map, if_alias_map
 
-
-# TODO: the function name include interface, but only return port by design. Fix the design or the name
 def init_sync_d_interface_tables(db_conn):
     """
     Initializes interface maps for SyncD-connected MIB(s).
@@ -196,9 +196,12 @@ def init_sync_d_interface_tables(db_conn):
     # { if_name (SONiC) -> sai_id }
     # ex: { "Ethernet76" : "1000000000023" }
     if_name_map, if_id_map = port_util.get_interface_oid_map(db_conn)
-    if_name_map = {if_name: sai_id for if_name, sai_id in if_name_map.items() if re.match(port_util.SONIC_ETHERNET_RE_PATTERN, if_name.decode())}
-    if_id_map = {sai_id: if_name for sai_id, if_name in if_id_map.items() if re.match(port_util.SONIC_ETHERNET_RE_PATTERN, if_name.decode())}
-
+    if_name_map = {if_name: sai_id for if_name, sai_id in if_name_map.items() if \
+                   (re.match(port_util.SONIC_ETHERNET_RE_PATTERN, if_name.decode()) or \
+                    re.match(port_util.SONIC_ETHERNET_BP_RE_PATTERN, if_name.decode()))}
+    if_id_map = {sai_id: if_name for sai_id, if_name in if_id_map.items() if \
+                 (re.match(port_util.SONIC_ETHERNET_RE_PATTERN, if_name.decode()) or \
+                  re.match(port_util.SONIC_ETHERNET_BP_RE_PATTERN, if_name.decode()))}
     logger.debug("Port name map:\n" + pprint.pformat(if_name_map, indent=2))
     logger.debug("Interface name map:\n" + pprint.pformat(if_id_map, indent=2))
 
@@ -241,7 +244,6 @@ def init_sync_d_interface_tables(db_conn):
 
     return if_name_map, if_alias_map, if_id_map, oid_sai_map, oid_name_map
 
-
 def init_sync_d_lag_tables(db_conn):
     """
     Helper method. Connects to and initializes LAG interface maps for SyncD-connected MIB(s).
@@ -391,7 +393,7 @@ class RedisOidTreeUpdater(MIBUpdater):
     def __init__(self, prefix_str):
         super().__init__()
 
-        self.db_conn = init_db()
+        self.db_conn = Namespace.init_namespace_dbs() 
         if prefix_str.startswith('.'):
             prefix_str = prefix_str[1:]
         self.prefix_str = prefix_str
@@ -417,8 +419,7 @@ def update_data(self):
         self.oid_list = []
         self.oid_map = {}
 
-        self.db_conn.connect(SNMP_OVERLAY_DB)
-        keys = self.db_conn.keys(SNMP_OVERLAY_DB, self.prefix_str + '*')
+        keys = Namespace.dbs_keys(self.db_conn, SNMP_OVERLAY_DB, self.prefix_str + '*') 
         # TODO: fix db_conn.keys to return empty list instead of None if there is no match
         if keys is None:
             keys = []
@@ -427,7 +428,7 @@ def update_data(self):
             key = key.decode()
             oid = oid2tuple(key, dot_prefix=False)
             self.oid_list.append(oid)
-            value = self.db_conn.get_all(SNMP_OVERLAY_DB, key)
+            value = Namespace.dbs_get_all(self.db_conn, SNMP_OVERLAY_DB, key) 
             if value[b'type'] in [b'COUNTER_32', b'COUNTER_64']:
                 self.oid_map[oid] = int(value[b'data'])
             else:
@@ -439,3 +440,134 @@ def get_oidvalue(self, oid):
         if oid not in self.oid_map:
             return None
         return self.oid_map[oid]
+
+class Namespace:
+    @staticmethod
+    def init_namespace_dbs():
+        db_conn= []
+        SonicDBConfig.load_sonic_global_db_config()
+        for namespace in SonicDBConfig.get_ns_list():
+            db = SonicV2Connector(use_unix_socket_path=True, namespace=namespace)
+            db_conn.append(db)
+
+        return db_conn
+
+    @staticmethod
+    def connect_all_dbs(dbs, db_name):
+        for db_conn in dbs:
+            db_conn.connect(db_name)
+
+    @staticmethod
+    def dbs_keys(dbs, db_name, pattern='*'):
+        """
+        db keys function execute on global and all namespace DBs.
+        """
+        result_keys=[]
+        for db_conn in dbs:
+            db_conn.connect(db_name)
+            keys = db_conn.keys(db_name, pattern)
+            if keys is not None:
+                result_keys.extend(keys)
+        return result_keys
+
+    @staticmethod
+    def dbs_get_all(dbs, db_name, _hash, *args, **kwargs):
+        """
+        db get_all function executed on global and all namespace DBs.
+        """
+        result = {}
+        for db_conn in dbs:
+            db_conn.connect(db_name)
+            if(db_conn.exists(db_name, _hash)):
+                ns_result = db_conn.get_all(db_name, _hash, *args, **kwargs)
+                if ns_result is not None:
+                    result.update(ns_result)
+        return result
+
+    @staticmethod
+    def get_non_host_dbs(dbs):
+        """
+        From the list of all dbs, return the list of dbs
+        which will have interface related tables.
+        For single namespace db, return the single db.
+        For multiple namespace dbs, return all dbs except the
+        host namespace db which is the first db in the list.
+        """
+        if len(dbs) == 1:
+            return dbs
+        else:
+            return dbs[1:]
+
+
+    @staticmethod
+    def init_namespace_sync_d_interface_tables(dbs):
+        if_name_map = {}
+        if_alias_map = {}
+        if_id_map = {}
+        oid_sai_map = {}
+        oid_name_map = {}
+
+        """
+        all_ns_db - will have db_conn to all namespace DBs and
+        global db. First db in the list is global db.
+        Ignore first global db to get interface tables if there
+        are multiple namespaces.
+        """
+        for db_conn in Namespace.get_non_host_dbs(dbs):
+            if_name_map_ns, \
+            if_alias_map_ns, \
+            if_id_map_ns, \
+            oid_sai_map_ns, \
+            oid_name_map_ns = init_sync_d_interface_tables(db_conn)
+            if_name_map.update(if_name_map_ns)
+            if_alias_map.update(if_alias_map_ns)
+            if_id_map.update(if_id_map_ns)
+            oid_sai_map.update(oid_sai_map_ns)
+            oid_name_map.update(oid_name_map_ns)
+
+        return if_name_map, if_alias_map, if_id_map, oid_sai_map, oid_name_map
+
+    @staticmethod
+    def init_namespace_sync_d_lag_tables(dbs):
+
+        lag_name_if_name_map = {}
+        if_name_lag_name_map = {}
+        oid_lag_name_map = {}
+
+        """
+        all_ns_db - will have db_conn to all namespace DBs and
+        global db. First db in the list is global db.
+        Ignore first global db to get lag tables if
+        there are multiple namespaces.
+        """
+        for db_conn in Namespace.get_non_host_dbs(dbs):
+            lag_name_if_name_map_ns, \
+            if_name_lag_name_map_ns, \
+            oid_lag_name_map_ns = init_sync_d_lag_tables(db_conn)
+            lag_name_if_name_map.update(lag_name_if_name_map_ns)
+            if_name_lag_name_map.update(if_name_lag_name_map_ns)
+            oid_lag_name_map.update(oid_lag_name_map_ns)
+
+        return lag_name_if_name_map, if_name_lag_name_map, oid_lag_name_map
+
+    @staticmethod
+    def init_namespace_sync_d_queue_tables(dbs):
+        port_queues_map = {}
+        queue_stat_map = {}
+        port_queue_list_map = {}
+
+        """
+        all_ns_db - will have db_conn to all namespace DBs and
+        global db. First db in the list is global db.
+        Ignore first global db to get queue tables if there
+        are multiple namespaces.
+        """
+        for db_conn in Namespace.get_non_host_dbs(dbs):
+            port_queues_map_ns, \
+            queue_stat_map_ns, \
+            port_queue_list_map_ns = init_sync_d_queue_tables(db_conn)
+            port_queues_map.update(port_queues_map_ns)
+            queue_stat_map.update(queue_stat_map_ns)
+            port_queue_list_map.update(port_queue_list_map_ns)
+
+        return port_queues_map, queue_stat_map, port_queue_list_map

src/sonic_ax_impl/mibs/ietf/rfc1213.py

@@ -4,6 +4,7 @@
 from bisect import bisect_right
 
 from sonic_ax_impl import mibs
+from sonic_ax_impl.mibs import Namespace
 from ax_interface.mib import MIBMeta, ValueType, MIBUpdater, MIBEntry, SubtreeMIBEntry, OverlayAdpaterMIBEntry, OidMIBEntry
 from ax_interface.encodings import ObjectIdentifier
 from ax_interface.util import mac_decimals, ip2tuple_v4
@@ -93,7 +94,7 @@ def get_next(self, sub_id):
 class NextHopUpdater(MIBUpdater):
     def __init__(self):
         super().__init__()
-        self.db_conn = mibs.init_db()
+        self.db_conn = Namespace.init_namespace_dbs()
         self.nexthop_map = {}
         self.route_list = []
 
@@ -105,8 +106,7 @@ def update_data(self):
         self.nexthop_map = {}
         self.route_list = []
 
-        self.db_conn.connect(mibs.APPL_DB)
-        route_entries = self.db_conn.keys(mibs.APPL_DB, "ROUTE_TABLE:*")
+        route_entries = Namespace.dbs_keys(self.db_conn, mibs.APPL_DB, "ROUTE_TABLE:*")
         if not route_entries:
             return
 
@@ -115,7 +115,7 @@ def update_data(self):
             ipnstr = routestr[len("ROUTE_TABLE:"):]
             if ipnstr == "0.0.0.0/0":
                 ipn = ipaddress.ip_network(ipnstr)
-                ent = self.db_conn.get_all(mibs.APPL_DB, routestr, blocking=True)
+                ent = Namespace.dbs_get_all(self.db_conn, mibs.APPL_DB, routestr, blocking=True)
                 nexthops = ent[b"nexthop"].decode()
                 for nh in nexthops.split(','):
                     # TODO: if ipn contains IP range, create more sub_id here
@@ -152,7 +152,7 @@ class InterfacesUpdater(MIBUpdater):
 
     def __init__(self):
         super().__init__()
-        self.db_conn = mibs.init_db()
+        self.db_conn = Namespace.init_namespace_dbs() 
 
         self.lag_name_if_name_map = {}
         self.if_name_lag_name_map = {}
@@ -177,23 +177,27 @@ def reinit_data(self):
         self.if_alias_map, \
         self.if_id_map, \
         self.oid_sai_map, \
-        self.oid_name_map = mibs.init_sync_d_interface_tables(self.db_conn)
-
+        self.oid_name_map = Namespace.init_namespace_sync_d_interface_tables(self.db_conn)
+        """
+        db_conn - will have db_conn to all namespace DBs and
+        global db. First db in the list is global db.
+        Use first global db to get management interface table.
+        """
         self.mgmt_oid_name_map, \
-        self.mgmt_alias_map = mibs.init_mgmt_interface_tables(self.db_conn)
+        self.mgmt_alias_map = mibs.init_mgmt_interface_tables(self.db_conn[0])
 
     def update_data(self):
         """
         Update redis (caches config)
         Pulls the table references for each interface.
         """
         self.if_counters = \
-            {sai_id: self.db_conn.get_all(mibs.COUNTERS_DB, mibs.counter_table(sai_id), blocking=True)
-             for sai_id in self.if_id_map}
+            {sai_id: Namespace.dbs_get_all(self.db_conn, mibs.COUNTERS_DB, mibs.counter_table(sai_id), blocking=True)
+            for sai_id in self.if_id_map}
 
         self.lag_name_if_name_map, \
         self.if_name_lag_name_map, \
-        self.oid_lag_name_map = mibs.init_sync_d_lag_tables(self.db_conn)
+        self.oid_lag_name_map = Namespace.init_namespace_sync_d_lag_tables(self.db_conn)
 
         self.if_range = sorted(list(self.oid_sai_map.keys()) +
                                list(self.oid_lag_name_map.keys()) +
@@ -318,7 +322,7 @@ def _get_if_entry(self, sub_id):
         else:
             return None
 
-        return self.db_conn.get_all(db, if_table, blocking=True)
+        return Namespace.dbs_get_all(self.db_conn, db, if_table, blocking=True)
 
     def _get_if_entry_state_db(self, sub_id):
         """
@@ -337,7 +341,7 @@ def _get_if_entry_state_db(self, sub_id):
         else:
             return None
 
-        return self.db_conn.get_all(db, if_table, blocking=False)
+        return Namespace.dbs_get_all(self.db_conn, db, if_table, blocking=False)
 
     def _get_status(self, sub_id, key):
         """

src/sonic_ax_impl/mibs/ietf/rfc2863.py

@@ -3,6 +3,7 @@
 
 from sonic_ax_impl import mibs
 from ax_interface.mib import MIBMeta, MIBUpdater, ValueType, SubtreeMIBEntry, OverlayAdpaterMIBEntry, OidMIBEntry
+from sonic_ax_impl.mibs import Namespace
 
 @unique
 class DbTables32(int, Enum):
@@ -47,7 +48,7 @@ class InterfaceMIBUpdater(MIBUpdater):
     def __init__(self):
         super().__init__()
 
-        self.db_conn = mibs.init_db()
+        self.db_conn = Namespace.init_namespace_dbs()
 
         self.lag_name_if_name_map = {}
         self.if_name_lag_name_map = {}
@@ -74,14 +75,18 @@ def reinit_data(self):
         self.if_alias_map, \
         self.if_id_map, \
         self.oid_sai_map, \
-        self.oid_name_map = mibs.init_sync_d_interface_tables(self.db_conn)
+        self.oid_name_map = Namespace.init_namespace_sync_d_interface_tables(self.db_conn)
 
         self.lag_name_if_name_map, \
         self.if_name_lag_name_map, \
-        self.oid_lag_name_map = mibs.init_sync_d_lag_tables(self.db_conn)
-
+        self.oid_lag_name_map = Namespace.init_namespace_sync_d_lag_tables(self.db_conn)
+        """
+        db_conn - will have db_conn to all namespace DBs and
+        global db. First db in the list is global db.
+        Use first global db to get management interface table.
+        """
         self.mgmt_oid_name_map, \
-        self.mgmt_alias_map = mibs.init_mgmt_interface_tables(self.db_conn)
+        self.mgmt_alias_map = mibs.init_mgmt_interface_tables(self.db_conn[0])
 
         self.if_range = sorted(list(self.oid_sai_map.keys()) +
                                list(self.oid_lag_name_map.keys()) +
@@ -94,7 +99,7 @@ def update_data(self):
         Pulls the table references for each interface.
         """
         self.if_counters = {
-            sai_id: self.db_conn.get_all(mibs.COUNTERS_DB, mibs.counter_table(sai_id), blocking=True)
+            sai_id: Namespace.dbs_get_all(self.db_conn, mibs.COUNTERS_DB, mibs.counter_table(sai_id), blocking=True)
             for sai_id in self.if_id_map}
 
 
@@ -217,7 +222,7 @@ def _get_if_entry(self, sub_id):
         else:
             return None
 
-        return self.db_conn.get_all(db, if_table, blocking=True)
+        return Namespace.dbs_get_all(self.db_conn, db, if_table, blocking=True)
 
     def get_high_speed(self, sub_id):
         """