Enable overriding interface counters OIDs (sonic-net#98)

* Add test for interface IN_OCTETS
* Add OverlayAdpaterMIBEntry class
* Refine
* InterfacesMIB supports counter override
* (reformat)
* Add missing test data
* Remvoe unused code
* Refactor
* (reformat)
* InterfaceMIBObjects supports counter override

src/ax_interface/mib.py

@@ -86,22 +86,16 @@ def __new__(mcs, name, bases, attributes, prefix=None):
  for me in vars(cls).values():
  if isinstance(me, MIBEntry):
  setattr(me, MIBEntry.PREFIXLEN, _prefix_len + len(me.subtree))
+ setattr(me, MIBEntry.PREFIX, _prefix + me.subtree)
 
  sub_ids = {}
 
- # gather all static MIB entries.
- static_entries = (v for v in vars(cls).values() if type(v) is MIBEntry)
- for me in static_entries:
+ # gather all MIB entries.
+ mib_entries = (v for v in vars(cls).values() if isinstance(v, MIBEntry))
+ for me in mib_entries:
  sub_ids.update({_prefix + me.subtree: me})
  prefixes.append(_prefix + me.subtree)
 
- # gather all subtree IDs
- # to support dynamic sub_id in the subtree, not to pour leaves into dictionary
- subtree_entries = (v for v in vars(cls).values() if type(v) is SubtreeMIBEntry)
- for sme in subtree_entries:
- sub_ids.update({_prefix + sme.subtree: sme})
- prefixes.append(_prefix + sme.subtree)
-
  # gather all updater instances
  updaters = set(v for k, v in vars(cls).items() if isinstance(v, MIBUpdater))
 
@@ -134,6 +128,7 @@ def __init__(cls, name, bases, attributes, prefix=None):
 
 class MIBEntry:
  PREFIXLEN = '__prefixlen__'
+ PREFIX = '__prefix__'
 
  def __init__(self, subtree, value_type, callable_, *args):
  """
@@ -155,7 +150,7 @@ def __init__(self, subtree, value_type, callable_, *args):
  raise ValueError("Third argument must be a callable object--got literal instead.")
  self._callable_ = callable_
  self._callable_args = args
- self.subtree = subtree
+ self.subtree_str = subtree
  self.value_type = value_type
  self.subtree = util.oid2tuple(subtree, dot_prefix=False)
 
@@ -174,8 +169,11 @@ def replace_sub_id(self, oid_key, sub_id):
  def get_next(self, sub_id):
  return None
 
+ def get_prefix(self):
+ return getattr(self, MIBEntry.PREFIX)
+
 class SubtreeMIBEntry(MIBEntry):
- def __init__(self, subtree, iterator, value_type, callable_, *args, updater=None):
+ def __init__(self, subtree, iterator, value_type, callable_, *args):
  super().__init__(subtree, value_type, callable_, *args)
  self.iterator = iterator
 
@@ -209,6 +207,46 @@ def get_next(self, sub_id):
  logger.exception("SubtreeMIBEntry.get_next() caught an unexpected exception during iterator.get_next()")
  return None
 
+# Define MIB entry (subtree) with a callable, which accepts a starndard OID tuple as a paramter
+class OidMIBEntry(MIBEntry):
+ def __init__(self, subtree, value_type, callable_):
+ super().__init__(subtree, value_type, callable_)
+
+ def __iter__(self):
+ raise NotImplementedError
+
+ def __call__(self, sub_id):
+ return self._callable_.__call__(self.get_prefix() + sub_id)
+
+class OverlayAdpaterMIBEntry(MIBEntry):
+ def __init__(self, underlay_mibentry, overlay_mibentry):
+ assert underlay_mibentry.value_type == overlay_mibentry.value_type
+ assert underlay_mibentry.subtree == overlay_mibentry.subtree
+
+ super().__init__(underlay_mibentry.subtree_str, underlay_mibentry.value_type, underlay_mibentry._callable_)
+ self.underlay_mibentry = underlay_mibentry
+ self.overlay_mibentry = overlay_mibentry
+
+ def __setattr__(self, name, value):
+ super().__setattr__(name, value)
+ if name.startswith('__') and name.endswith('__'):
+ setattr(self.underlay_mibentry, name, value)
+ setattr(self.overlay_mibentry, name, value)
+
+ def __iter__(self):
+ return self.underlay_mibentry.__iter__()
+
+ def __call__(self, sub_id=None):
+ overlay_val = self.overlay_mibentry(sub_id)
+ if overlay_val is not None:
+ return overlay_val
+
+ underlay_val = self.underlay_mibentry(sub_id)
+ return underlay_val
+
+ def get_next(self, sub_id):
+ return self.underlay_mibentry.get_next(sub_id)
+
 class MIBTable(dict):
  """
  Simplistic LUT for Get/GetNext OID. Interprets iterables as keys and implements the same interfaces as dict's.

src/sonic_ax_impl/mibs/__init__.py

@@ -4,6 +4,8 @@
 from swsssdk import SonicV2Connector
 from swsssdk import port_util
 from swsssdk.port_util import get_index, get_index_from_str
+from ax_interface.mib import MIBUpdater
+from ax_interface.util import oid2tuple
 from sonic_ax_impl import logger
 
 COUNTERS_PORT_NAME_MAP = b'COUNTERS_PORT_NAME_MAP'
@@ -16,6 +18,7 @@
 COUNTERS_DB = 'COUNTERS_DB'
 CONFIG_DB = 'CONFIG_DB'
 STATE_DB = 'STATE_DB'
+SNMP_OVERLAY_DB = 'SNMP_OVERLAY_DB'
 
 TABLE_NAME_SEPARATOR_COLON = ':'
 TABLE_NAME_SEPARATOR_VBAR = '|'
@@ -363,3 +366,56 @@ def get_transceiver_sensor_sub_id(ifindex, sensor):
 
  transceiver_oid, = get_transceiver_sub_id(ifindex)
  return (transceiver_oid + SENSOR_PART_ID_MAP[sensor], )
+
+class RedisOidTreeUpdater(MIBUpdater):
+ def __init__(self, prefix_str):
+ super().__init__()
+
+ self.db_conn = init_db()
+ if prefix_str.startswith('.'):
+ prefix_str = prefix_str[1:]
+ self.prefix_str = prefix_str
+
+ def get_next(self, sub_id):
+ """
+ :param sub_id: The 1-based sub-identifier query.
+ :return: the next sub id.
+ """
+ raise NotImplementedError
+
+ def reinit_data(self):
+ """
+ Subclass update loopback information
+ """
+ pass
+
+ def update_data(self):
+ """
+ Update redis (caches config)
+ Pulls the table references for each interface.
+ """
+ self.oid_list = []
+ self.oid_map = {}
+
+ self.db_conn.connect(SNMP_OVERLAY_DB)
+ keys = self.db_conn.keys(SNMP_OVERLAY_DB, self.prefix_str + '*')
+ # TODO: fix db_conn.keys to return empty list instead of None if there is no match
+ if keys is None:
+ keys = []
+
+ for key in keys:
+ key = key.decode()
+ oid = oid2tuple(key, dot_prefix=False)
+ self.oid_list.append(oid)
+ value = self.db_conn.get_all(SNMP_OVERLAY_DB, key)
+ if value[b'type'] in [b'COUNTER_32', b'COUNTER_64']:
+ self.oid_map[oid] = int(value[b'data'])
+ else:
+ raise ValueError("Invalid value type")
+
+ self.oid_list.sort()
+
+ def get_oidvalue(self, oid):
+ if oid not in self.oid_map:
+ return None
+ return self.oid_map[oid]

src/sonic_ax_impl/mibs/ietf/rfc1213.py

@@ -4,11 +4,10 @@
 from bisect import bisect_right
 
 from sonic_ax_impl import mibs
-from ax_interface import MIBMeta, ValueType, MIBUpdater, MIBEntry, SubtreeMIBEntry
+from ax_interface.mib import MIBMeta, ValueType, MIBUpdater, MIBEntry, SubtreeMIBEntry, OverlayAdpaterMIBEntry, OidMIBEntry
 from ax_interface.encodings import ObjectIdentifier
 from ax_interface.util import mac_decimals, ip2tuple_v4
 
-
 @unique
 class DbTables(int, Enum):
  """
@@ -271,6 +270,7 @@ def get_counter(self, sub_id, table_name):
  :param table_name: the redis table (either IntEnum or string literal) to query.
  :return: the counter for the respective sub_id/table.
  """
+
  oid = self.get_oid(sub_id)
  if not oid:
  return
@@ -423,14 +423,15 @@ def get_if_type(self, sub_id):
  else:
  return IfTypes.ethernetCsmacd
 
-
 class InterfacesMIB(metaclass=MIBMeta, prefix='.1.3.6.1.2.1.2'):
  """
  'interfaces' https://tools.ietf.org/html/rfc1213#section-3.5
  """
 
  if_updater = InterfacesUpdater()
 
+ oidtree_updater = mibs.RedisOidTreeUpdater(prefix_str='1.3.6.1.2.1.2')
+
  # (subtree, value_type, callable_, *args, handler=None)
  ifNumber = MIBEntry('1', ValueType.INTEGER, if_updater.get_if_number)
 
@@ -467,52 +468,88 @@ class InterfacesMIB(metaclass=MIBMeta, prefix='.1.3.6.1.2.1.2'):
  SubtreeMIBEntry('2.1.9', if_updater, ValueType.TIME_TICKS, lambda sub_id: 0)
 
  ifInOctets = \
- SubtreeMIBEntry('2.1.10', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
- DbTables(10))
+ OverlayAdpaterMIBEntry(
+ SubtreeMIBEntry('2.1.10', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
+ DbTables(10)),
+ OidMIBEntry('2.1.10', ValueType.COUNTER_32, oidtree_updater.get_oidvalue)
+ )
 
  ifInUcastPkts = \
- SubtreeMIBEntry('2.1.11', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
- DbTables(11))
+ OverlayAdpaterMIBEntry(
+ SubtreeMIBEntry('2.1.11', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
+ DbTables(11)),
+ OidMIBEntry('2.1.11', ValueType.COUNTER_32, oidtree_updater.get_oidvalue)
+ )
 
  ifInNUcastPkts = \
- SubtreeMIBEntry('2.1.12', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
- DbTables(12))
+ OverlayAdpaterMIBEntry(
+ SubtreeMIBEntry('2.1.12', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
+ DbTables(12)),
+ OidMIBEntry('2.1.12', ValueType.COUNTER_32, oidtree_updater.get_oidvalue)
+ )
 
  ifInDiscards = \
- SubtreeMIBEntry('2.1.13', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
- DbTables(13))
+ OverlayAdpaterMIBEntry(
+ SubtreeMIBEntry('2.1.13', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
+ DbTables(13)),
+ OidMIBEntry('2.1.13', ValueType.COUNTER_32, oidtree_updater.get_oidvalue)
+ )
 
  ifInErrors = \
- SubtreeMIBEntry('2.1.14', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
- DbTables(14))
+ OverlayAdpaterMIBEntry(
+ SubtreeMIBEntry('2.1.14', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
+ DbTables(14)),
+ OidMIBEntry('2.1.14', ValueType.COUNTER_32, oidtree_updater.get_oidvalue)
+ )
 
  ifInUnknownProtos = \
- SubtreeMIBEntry('2.1.15', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
- DbTables(15))
+ OverlayAdpaterMIBEntry(
+ SubtreeMIBEntry('2.1.15', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
+ DbTables(15)),
+ OidMIBEntry('2.1.15', ValueType.COUNTER_32, oidtree_updater.get_oidvalue)
+ )
 
  ifOutOctets = \
- SubtreeMIBEntry('2.1.16', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
- DbTables(16))
+ OverlayAdpaterMIBEntry(
+ SubtreeMIBEntry('2.1.16', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
+ DbTables(16)),
+ OidMIBEntry('2.1.16', ValueType.COUNTER_32, oidtree_updater.get_oidvalue)
+ )
 
  ifOutUcastPkts = \
- SubtreeMIBEntry('2.1.17', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
- DbTables(17))
+ OverlayAdpaterMIBEntry(
+ SubtreeMIBEntry('2.1.17', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
+ DbTables(17)),
+ OidMIBEntry('2.1.17', ValueType.COUNTER_32, oidtree_updater.get_oidvalue)
+ )
 
  ifOutNUcastPkts = \
- SubtreeMIBEntry('2.1.18', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
- DbTables(18))
+ OverlayAdpaterMIBEntry(
+ SubtreeMIBEntry('2.1.18', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
+ DbTables(18)),
+ OidMIBEntry('2.1.18', ValueType.COUNTER_32, oidtree_updater.get_oidvalue)
+ )
 
  ifOutDiscards = \
- SubtreeMIBEntry('2.1.19', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
- DbTables(19))
+ OverlayAdpaterMIBEntry(
+ SubtreeMIBEntry('2.1.19', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
+ DbTables(19)),
+ OidMIBEntry('2.1.19', ValueType.COUNTER_32, oidtree_updater.get_oidvalue)
+ )
 
  ifOutErrors = \
- SubtreeMIBEntry('2.1.20', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
- DbTables(20))
+ OverlayAdpaterMIBEntry(
+ SubtreeMIBEntry('2.1.20', if_updater, ValueType.COUNTER_32, if_updater.get_counter,
+ DbTables(20)),
+ OidMIBEntry('2.1.20', ValueType.COUNTER_32, oidtree_updater.get_oidvalue)
+ )
 
  ifOutQLen = \
- SubtreeMIBEntry('2.1.21', if_updater, ValueType.GAUGE_32, if_updater.get_counter,
- DbTables(21))
+ OverlayAdpaterMIBEntry(
+ SubtreeMIBEntry('2.1.21', if_updater, ValueType.GAUGE_32, if_updater.get_counter,
+ DbTables(21)),
+ OidMIBEntry('2.1.21', ValueType.GAUGE_32, oidtree_updater.get_oidvalue)
+ )
 
  # FIXME Placeholder
  ifSpecific = \