Handle error seen when interface counter is not available in COUNTERS_DB (#341)

SuvarnaMeenakshi · web-flow · commit 25f9e4fa16e8 · 2025-03-03T18:08:00.000-08:00
**- What I did** Fix to handle error seen: ``` snmp-subagent [ax_interface] ERROR: SubtreeMIBEntry.__call__() caught an unexpected exception during _callable_.__call__()#012Traceback (most recent call last):#12 File "/usr/local/lib/python3.9/dist-packages/ax_interface/mib.py", line 194, in __call__#012 return self._callable_.__call__(sub_id, *self._callable_args)#12 File "/usr/local/lib/python3.9/dist-packages/sonic_ax_impl/mibs/vendor/cisco/ciscoPfcExtMIB.py", line 248, in indications_per_priority#012 counter_value += self._get_counter(mibs.get_index_from_str(lag_member), counter_name)#012TypeError: unsupported operand type(s) for += ``` Above error log occurs when the services restarted, swss/syncd due to some crash/reboot/config reload, this will also cause snmp service to restart. During this time, it can happen that all interface COUNTERS are not yet available in COUNTERS_DB for a short period. At this point, if a SNMP query is made to retrieve interface/PFC counters, then this error syslog will show up until the COUNTERS_DB data is populated with counters for all interfaces. MSFT ADO 26506804 **- How I did it** Avoid adding up counters if _get_counter returns None. **- How to verify it** Before FIX: send a continuous query to get pfc counters for any of the configured port-channel interface: watch -n 1 snmpwalk -v2c -c <comm> <ip> 1.3.6.1.4.1.9.9.813.1.2.1.3.1103 on the device, execute config reload and we should see the error log on the device: After fix, perform same snmpwalk as above, SNMP ERR log will not be seen.
diff --git a/src/sonic_ax_impl/mibs/ietf/rfc2863.py b/src/sonic_ax_impl/mibs/ietf/rfc2863.py
@@ -250,7 +250,11 @@ def _get_counter(self, oid, table_name, mask):
         if oid in self.oid_lag_name_map:
             counter_value = 0
             for lag_member in self.lag_name_if_name_map[self.oid_lag_name_map[oid]]:
-                counter_value += self._get_counter(mibs.get_index_from_str(lag_member), table_name, mask)
+                member_counter = self._get_counter(mibs.get_index_from_str(lag_member), table_name, mask)
+                if member_counter is not None:
+                    counter_value += member_counter
+                else:
+                    return None
 
             return counter_value & mask
 
diff --git a/src/sonic_ax_impl/mibs/vendor/cisco/ciscoPfcExtMIB.py b/src/sonic_ax_impl/mibs/vendor/cisco/ciscoPfcExtMIB.py
@@ -121,7 +121,12 @@ def cpfc_if_requests(self, sub_id):
         if oid in self.oid_lag_name_map:
             counter_value = 0
             for lag_member in self.lag_name_if_name_map[self.oid_lag_name_map[oid]]:
-                counter_value += self._get_counter(mibs.get_index_from_str(lag_member), counter_name)
+                member_counter = self._get_counter(mibs.get_index_from_str(lag_member), counter_name)
+                if member_counter is not None:
+                    counter_value += member_counter
+                else:
+                    mibs.logger.warning("SyncD 'COUNTERS_DB' missing attribute '{}' for lag member '{}'.".format(counter_name, lag_member))
+                    return None
 
             return counter_value
         else:
@@ -143,7 +148,12 @@ def cpfc_if_indications(self, sub_id):
         if oid in self.oid_lag_name_map:
             counter_value = 0
             for lag_member in self.lag_name_if_name_map[self.oid_lag_name_map[oid]]:
-                counter_value += self._get_counter(mibs.get_index_from_str(lag_member), counter_name)
+                member_counter = self._get_counter(mibs.get_index_from_str(lag_member), counter_name)
+                if member_counter is not None:
+                    counter_value += member_counter
+                else:
+                    mibs.logger.warning("SyncD 'COUNTERS_DB' missing attribute '{}' for lag member '{}'.".format(counter_name, lag_member))
+                    return None
 
             return counter_value
         else:
@@ -215,7 +225,12 @@ def requests_per_priority(self, sub_id):
         if port_oid in self.oid_lag_name_map:
             counter_value = 0
             for lag_member in self.lag_name_if_name_map[self.oid_lag_name_map[port_oid]]:
-                counter_value += self._get_counter(mibs.get_index_from_str(lag_member), counter_name)
+                member_counter = self._get_counter(mibs.get_index_from_str(lag_member), counter_name)
+                if member_counter is not None:
+                    counter_value += member_counter
+                else:
+                    mibs.logger.warning("SyncD 'COUNTERS_DB' missing attribute '{}' for lag member '{}'.".format(counter_name, lag_member))
+                    return None
 
             return counter_value
         else:
@@ -245,7 +260,12 @@ def indications_per_priority(self, sub_id):
         if port_oid in self.oid_lag_name_map:
             counter_value = 0
             for lag_member in self.lag_name_if_name_map[self.oid_lag_name_map[port_oid]]:
-                counter_value += self._get_counter(mibs.get_index_from_str(lag_member), counter_name)
+                member_counter = self._get_counter(mibs.get_index_from_str(lag_member), counter_name)
+                if member_counter is not None:
+                    counter_value += member_counter
+                else:
+                    mibs.logger.warning("SyncD 'COUNTERS_DB' missing attribute '{}' for lag member '{}'.".format(counter_name, lag_member))
+                    return None
 
             return counter_value
         else:
diff --git a/tests/mock_tables/counters_db.json b/tests/mock_tables/counters_db.json
@@ -2794,108 +2794,6 @@
     "SAI_PORT_STAT_PFC_7_RX_PKTS": "8",
     "SAI_PORT_STAT_PFC_7_TX_PKTS": "8"
   },
-  "COUNTERS:oid:0x1000000000003": {
-    "SAI_PORT_STAT_ETHER_STATS_TX_NO_ERRORS": "0",
-    "SAI_PORT_STAT_ETHER_STATS_OVERSIZE_PKTS": "0",
-    "SAI_PORT_STAT_IF_OUT_ERRORS": "0",
-    "SAI_PORT_STAT_ETHER_TX_OVERSIZE_PKTS": "0",
-    "SAI_PORT_STAT_ETHER_STATS_PKTS_1519_TO_2047_OCTETS": "0",
-    "SAI_PORT_STAT_IP_IN_RECEIVES": "0",
-    "SAI_PORT_STAT_ETHER_IN_PKTS_64_OCTETS": "0",
-    "SAI_PORT_STAT_IPV6_OUT_UCAST_PKTS": "0",
-    "SAI_PORT_STAT_ETHER_OUT_PKTS_4096_TO_9216_OCTETS": "0",
-    "SAI_PORT_STAT_IF_IN_ERRORS": "0",
-    "SAI_PORT_STAT_ETHER_STATS_PKTS": "0",
-    "SAI_PORT_STAT_ETHER_STATS_BROADCAST_PKTS": "0",
-    "SAI_PORT_STAT_IF_IN_DISCARDS": "0",
-    "SAI_PORT_STAT_IP_OUT_DISCARDS": "0",
-    "SAI_PORT_STAT_IF_IN_UNKNOWN_PROTOS": "0",
-    "SAI_PORT_STAT_IPV6_IN_DISCARDS": "0",
-    "SAI_PORT_STAT_IPV6_OUT_DISCARDS": "0",
-    "SAI_PORT_STAT_IPV6_IN_OCTETS": "0",
-    "SAI_PORT_STAT_ETHER_OUT_PKTS_65_TO_127_OCTETS": "0",
-    "SAI_PORT_STAT_IF_IN_BROADCAST_PKTS": "0",
-    "SAI_PORT_STAT_ETHER_IN_PKTS_1519_TO_2047_OCTETS": "0",
-    "SAI_PORT_STAT_IF_OUT_MULTICAST_PKTS": "0",
-    "SAI_PORT_STAT_ETHER_OUT_PKTS_512_TO_1023_OCTETS": "0",
-    "SAI_PORT_STAT_ETHER_STATS_PKTS_256_TO_511_OCTETS": "0",
-    "SAI_PORT_STAT_ETHER_STATS_PKTS_9217_TO_16383_OCTETS": "0",
-    "SAI_PORT_STAT_ETHER_IN_PKTS_512_TO_1023_OCTETS": "0",
-    "SAI_PORT_STAT_IPV6_IN_NON_UCAST_PKTS": "0",
-    "SAI_PORT_STAT_ETHER_IN_PKTS_4096_TO_9216_OCTETS": "0",
-    "SAI_PORT_STAT_IF_OUT_BROADCAST_PKTS": "0",
-    "SAI_PORT_STAT_IPV6_OUT_NON_UCAST_PKTS": "0",
-    "SAI_PORT_STAT_IF_IN_VLAN_DISCARDS": "0",
-    "SAI_PORT_STAT_ETHER_IN_PKTS_65_TO_127_OCTETS": "0",
-    "SAI_PORT_STAT_IP_IN_NON_UCAST_PKTS": "0",
-    "SAI_PORT_STAT_ETHER_STATS_FRAGMENTS": "0",
-    "SAI_PORT_STAT_IPV6_IN_UCAST_PKTS": "0",
-    "SAI_PORT_STAT_IPV6_IN_RECEIVES": "0",
-    "SAI_PORT_STAT_ETHER_STATS_PKTS_4096_TO_9216_OCTETS": "0",
-    "SAI_PORT_STAT_IF_OUT_DISCARDS": "0",
-    "SAI_PORT_STAT_ETHER_STATS_DROP_EVENTS": "0",
-    "SAI_PORT_STAT_IPV6_OUT_MCAST_PKTS": "0",
-    "SAI_PORT_STAT_ETHER_RX_OVERSIZE_PKTS": "0",
-    "SAI_PORT_STAT_IF_OUT_OCTETS": "0",
-    "SAI_PORT_STAT_IF_IN_NON_UCAST_PKTS": "0",
-    "SAI_PORT_STAT_ETHER_IN_PKTS_9217_TO_16383_OCTETS": "0",
-    "SAI_PORT_STAT_ETHER_OUT_PKTS_1024_TO_1518_OCTETS": "0",
-    "SAI_PORT_STAT_ETHER_STATS_PKTS_2048_TO_4095_OCTETS": "0",
-    "SAI_PORT_STAT_ETHER_STATS_PKTS_512_TO_1023_OCTETS": "0",
-    "SAI_PORT_STAT_ETHER_OUT_PKTS_1519_TO_2047_OCTETS": "0",
-    "SAI_PORT_STAT_ETHER_STATS_RX_NO_ERRORS": "0",
-    "SAI_PORT_STAT_ETHER_OUT_PKTS_64_OCTETS": "0",
-    "SAI_PORT_STAT_ETHER_STATS_COLLISIONS": "0",
-    "SAI_PORT_STAT_ETHER_IN_PKTS_1024_TO_1518_OCTETS": "0",
-    "SAI_PORT_STAT_ETHER_STATS_PKTS_1024_TO_1518_OCTETS": "0",
-    "SAI_PORT_STAT_ETHER_IN_PKTS_256_TO_511_OCTETS": "0",
-    "SAI_PORT_STAT_ETHER_OUT_PKTS_2048_TO_4095_OCTETS": "0",
-    "SAI_PORT_STAT_ETHER_STATS_OCTETS": "0",
-    "SAI_PORT_STAT_IF_OUT_UCAST_PKTS": "0",
-    "SAI_PORT_STAT_ETHER_STATS_UNDERSIZE_PKTS": "0",
-    "SAI_PORT_STAT_ETHER_OUT_PKTS_128_TO_255_OCTETS": "0",
-    "SAI_PORT_STAT_ETHER_STATS_PKTS_64_OCTETS": "0",
-    "SAI_PORT_STAT_IP_OUT_OCTETS": "0",
-    "SAI_PORT_STAT_IF_IN_UCAST_PKTS": "0",
-    "SAI_PORT_STAT_ETHER_OUT_PKTS_9217_TO_16383_OCTETS": "0",
-    "SAI_PORT_STAT_ETHER_IN_PKTS_2048_TO_4095_OCTETS": "0",
-    "SAI_PORT_STAT_IP_OUT_NON_UCAST_PKTS": "0",
-    "SAI_PORT_STAT_ETHER_STATS_JABBERS": "0",
-    "SAI_PORT_STAT_IF_IN_OCTETS": "0",
-    "SAI_PORT_STAT_IPV6_IN_MCAST_PKTS": "0",
-    "SAI_PORT_STAT_ETHER_STATS_PKTS_65_TO_127_OCTETS": "0",
-    "SAI_PORT_STAT_IF_OUT_QLEN": "0",
-    "SAI_PORT_STAT_ETHER_STATS_PKTS_128_TO_255_OCTETS": "0",
-    "SAI_PORT_STAT_IP_IN_DISCARDS": "0",
-    "SAI_PORT_STAT_IPV6_OUT_OCTETS": "0",
-    "SAI_PORT_STAT_IF_OUT_NON_UCAST_PKTS": "0",
-    "SAI_PORT_STAT_IP_IN_OCTETS": "0",
-    "SAI_PORT_STAT_ETHER_OUT_PKTS_256_TO_511_OCTETS": "0",
-    "SAI_PORT_STAT_ETHER_STATS_CRC_ALIGN_ERRORS": "0",
-    "SAI_PORT_STAT_IP_OUT_UCAST_PKTS": "0",
-    "SAI_PORT_STAT_IP_IN_UCAST_PKTS": "0",
-    "SAI_PORT_STAT_ETHER_STATS_MULTICAST_PKTS": "0",
-    "SAI_PORT_STAT_ETHER_IN_PKTS_128_TO_255_OCTETS": "0",
-    "SAI_PORT_STAT_IF_IN_MULTICAST_PKTS": "0",
-    "SAI_PORT_STAT_PAUSE_RX_PKTS": "0",
-    "SAI_PORT_STAT_PAUSE_TX_PKTS": "0",
-    "SAI_PORT_STAT_PFC_0_RX_PKTS": "1",
-    "SAI_PORT_STAT_PFC_0_TX_PKTS": "1",
-    "SAI_PORT_STAT_PFC_1_RX_PKTS": "2",
-    "SAI_PORT_STAT_PFC_1_TX_PKTS": "2",
-    "SAI_PORT_STAT_PFC_2_RX_PKTS": "3",
-    "SAI_PORT_STAT_PFC_2_TX_PKTS": "3",
-    "SAI_PORT_STAT_PFC_3_RX_PKTS": "4",
-    "SAI_PORT_STAT_PFC_3_TX_PKTS": "4",
-    "SAI_PORT_STAT_PFC_4_RX_PKTS": "5",
-    "SAI_PORT_STAT_PFC_4_TX_PKTS": "5",
-    "SAI_PORT_STAT_PFC_5_RX_PKTS": "6",
-    "SAI_PORT_STAT_PFC_5_TX_PKTS": "6",
-    "SAI_PORT_STAT_PFC_6_RX_PKTS": "7",
-    "SAI_PORT_STAT_PFC_6_TX_PKTS": "7",
-    "SAI_PORT_STAT_PFC_7_RX_PKTS": "8",
-    "SAI_PORT_STAT_PFC_7_TX_PKTS": "8"
-  },
   "COUNTERS:oid:0x1000000000013": {
     "SAI_PORT_STAT_ETHER_STATS_TX_NO_ERRORS": "0",
     "SAI_PORT_STAT_ETHER_STATS_OVERSIZE_PKTS": "0",
diff --git a/tests/test_pfc.py b/tests/test_pfc.py
@@ -1,6 +1,12 @@
 import os
 import sys
 import importlib
+import pytest
+
+if sys.version_info.major == 3:
+    from unittest import mock
+else:
+    import mock
 
 # noinspection PyUnresolvedReferences
 import tests.mock_tables.dbconnector
@@ -155,6 +161,126 @@ def test_getPduIndicationForPriority(self):
         self.assertEqual(str(value0.name), str(oid))
         self.assertEqual(value0.data, 1)
 
+    def test_getPduPrioIndicationLagMemberCounterNotAvailable(self):
+        oid = ObjectIdentifier(8, 0, 0, 0, (1, 3, 6, 1, 4, 1, 9, 9, 813, 1, 2, 1, 3, 1004, 0))
+        get_pdu = GetPDU(
+            header=PDUHeader(1, PduTypes.GET, 16, 0, 42, 0, 0, 0),
+            oids=[oid]
+        )
+
+        with mock.patch('ax_interface.logger.exception') as mock_logger:
+            encoded = get_pdu.encode()
+            response = get_pdu.make_response(self.lut_prio)
+            mock_logger.assert_not_called()
+        print(response)
+        value0 = response.values[0]
+        self.assertEqual(value0.data, None)
+
+    def test_getPduPrioIndicationLag(self):
+        oid = ObjectIdentifier(8, 0, 0, 0, (1, 3, 6, 1, 4, 1, 9, 9, 813, 1, 2, 1, 3, 1003, 0))
+        get_pdu = GetPDU(
+            header=PDUHeader(1, PduTypes.GET, 16, 0, 42, 0, 0, 0),
+            oids=[oid]
+        )
+
+        with mock.patch('ax_interface.logger.exception') as mock_logger:
+            encoded = get_pdu.encode()
+            response = get_pdu.make_response(self.lut_prio)
+            mock_logger.assert_not_called()
+        print(response)
+        value0 = response.values[0]
+        self.assertEqual(value0.data, 1)
+
+    def test_getPduPrioRequestsLagMemberCounterNotAvailable(self):
+        oid = ObjectIdentifier(8, 0, 0, 0, (1, 3, 6, 1, 4, 1, 9, 9, 813, 1, 2, 1, 2, 1004, 0))
+        get_pdu = GetPDU(
+            header=PDUHeader(1, PduTypes.GET, 16, 0, 42, 0, 0, 0),
+            oids=[oid]
+        )
+
+        with mock.patch('ax_interface.logger.exception') as mock_logger:
+            encoded = get_pdu.encode()
+            response = get_pdu.make_response(self.lut_prio)
+            mock_logger.assert_not_called()
+        print(response)
+        value0 = response.values[0]
+        self.assertEqual(value0.data, None)
+
+    def test_getPduPrioRequestsLag(self):
+        oid = ObjectIdentifier(8, 0, 0, 0, (1, 3, 6, 1, 4, 1, 9, 9, 813, 1, 2, 1, 2, 1003, 1))
+        get_pdu = GetPDU(
+            header=PDUHeader(1, PduTypes.GET, 16, 0, 42, 0, 0, 0),
+            oids=[oid]
+        )
+
+        with mock.patch('ax_interface.logger.exception') as mock_logger:
+            encoded = get_pdu.encode()
+            response = get_pdu.make_response(self.lut_prio)
+            mock_logger.assert_not_called()
+        print(response)
+        value0 = response.values[0]
+        self.assertEqual(value0.data, 2)
+
+    def test_getPduifIndicationLagMemberCounterNotAvailable(self):
+        oid = ObjectIdentifier(8, 0, 0, 0, (1, 3, 6, 1, 4, 1, 9, 9, 813, 1, 1, 1, 2, 1004))
+        get_pdu = GetPDU(
+            header=PDUHeader(1, PduTypes.GET, 16, 0, 42, 0, 0, 0),
+            oids=[oid]
+        )
+
+        with mock.patch('ax_interface.logger.exception') as mock_logger:
+            encoded = get_pdu.encode()
+            response = get_pdu.make_response(self.lut_port)
+            mock_logger.assert_not_called()
+        print(response)
+        value0 = response.values[0]
+        self.assertEqual(value0.data, None)
+
+    def test_getPduifIndicationLag(self):
+        oid = ObjectIdentifier(8, 0, 0, 0, (1, 3, 6, 1, 4, 1, 9, 9, 813, 1, 1, 1, 2, 1003))
+        get_pdu = GetPDU(
+            header=PDUHeader(1, PduTypes.GET, 16, 0, 42, 0, 0, 0),
+            oids=[oid]
+        )
+
+        with mock.patch('ax_interface.logger.exception') as mock_logger:
+            encoded = get_pdu.encode()
+            response = get_pdu.make_response(self.lut_port)
+            mock_logger.assert_not_called()
+        print(response)
+        value0 = response.values[0]
+        self.assertEqual(value0.data, 4)
+
+    def test_getPduifRequestLagMemberCounterNotAvailable(self):
+        oid = ObjectIdentifier(8, 0, 0, 0, (1, 3, 6, 1, 4, 1, 9, 9, 813, 1, 1, 1, 1, 1004))
+        get_pdu = GetPDU(
+            header=PDUHeader(1, PduTypes.GET, 16, 0, 42, 0, 0, 0),
+            oids=[oid]
+        )
+
+        with mock.patch('ax_interface.logger.exception') as mock_logger:
+            encoded = get_pdu.encode()
+            response = get_pdu.make_response(self.lut_port)
+            mock_logger.assert_not_called()
+        print(response)
+        value0 = response.values[0]
+        self.assertEqual(value0.data, None)
+
+    def test_getPduifRequestLag(self):
+        oid = ObjectIdentifier(8, 0, 0, 0, (1, 3, 6, 1, 4, 1, 9, 9, 813, 1, 1, 1, 1, 1003))
+        get_pdu = GetPDU(
+            header=PDUHeader(1, PduTypes.GET, 16, 0, 42, 0, 0, 0),
+            oids=[oid]
+        )
+
+        with mock.patch('ax_interface.logger.exception') as mock_logger:
+            encoded = get_pdu.encode()
+            response = get_pdu.make_response(self.lut_port)
+            mock_logger.assert_not_called()
+        print(response)
+        value0 = response.values[0]
+        self.assertEqual(value0.data, 4)
+
     def test_getNextPduindicationForPriority(self):
         oid = ObjectIdentifier(8, 0, 0, 0, (1, 3, 6, 1, 4, 1, 9, 9, 813, 1, 2, 1, 3, 1, 0))
         expected_oid = ObjectIdentifier(8, 0, 0, 0, (1, 3, 6, 1, 4, 1, 9, 9, 813, 1, 2, 1, 3, 1, 1))
diff --git a/tests/test_rfc2863.py b/tests/test_rfc2863.py
@@ -12,6 +12,7 @@
 sys.path.insert(0, os.path.join(modules_path, 'src'))
 
 from sonic_ax_impl.mibs.ietf.rfc2863 import InterfaceMIBUpdater
+from sonic_ax_impl.mibs.ietf.rfc2863 import DbTables64
 
 class TestInterfaceMIBUpdater(TestCase):
 
@@ -66,3 +67,26 @@ def test_InterfaceMIBUpdater_get_high_speed(self):
             speed = updater.get_high_speed((1999,))
             print("999 speed: {}".format(speed))
             self.assertTrue(speed == 0)
+
+
+    @mock.patch('sonic_ax_impl.mibs.Namespace.get_sync_d_from_all_namespace', mock_get_sync_d_from_all_namespace)
+    @mock.patch('sonic_ax_impl.mibs.Namespace.dbs_get_all', mock_dbs_get_all)
+    @mock.patch('sonic_ax_impl.mibs.lag_entry_table', mock_lag_entry_table)
+    @mock.patch('sonic_ax_impl.mibs.init_mgmt_interface_tables', mock_init_mgmt_interface_tables)
+    def test_InterfaceMIBUpdater_get_counters(self):
+        updater = InterfaceMIBUpdater()
+
+        updater.reinit_data()
+        updater.update_data()
+        def mock_get_counter(oid, table_name, mask):
+            if oid == 121:
+                return None
+            else:
+                return updater._get_counter(oid, table_name, mask)
+
+        try:
+            counter = updater.get_counter64((1103,), DbTables64(6))
+        except TypeError:
+            self.fail("Caught Type error")
+        self.assertTrue(counter == None)
+
