feat: add multi-asic in test_cont_link_flap (#14673)

Description of PR
Currently, test_cont_link_flap is not compatible with multi-asic. It's failing on vtysh command

    raise RunAnsibleModuleFail("run module {} failed".format(self.module_name), res)
tests.common.errors.RunAnsibleModuleFail: run module shell failed, Ansible Results =>
{"changed": true, "cmd": "vtysh -c \"show memory bgpd\"", "delta": "0:00:00.006507", "end": "2024-09-17 00:06:50.914882", "failed": true, "msg": "non-zero return code", "rc": 1, "start": "2024-09-17 00:06:50.908375", "stderr": "Usage: /usr/bin/vtysh -n [0 to 2] [OPTION]... ", "stderr_lines": ["Usage: /usr/bin/vtysh -n [0 to 2] [OPTION]... "], "stdout": "", "stdout_lines": []}
This is because for multi-asic platform, we need to specify -n.

Summary: add multi-asic support for test_cont_link_flap
Fixes # (issue) 2954952

Approach
What is the motivation for this PR?
Support multi-asic for test_cont_link_flap by calculating the memory based of each individual asic

How did you do it?
Since SonicAsic.run_asic has already provided a back-ward compatible to multi-asic and single-asic (https://github.com/sonic-net/sonic-mgmt/blob/master/tests/common/devices/sonic_asic.py#L404) we can leverage this API to execute the test.

The test is also working with single-asic since in our system, single-asic chassis will have 1 asic with namespace=None and asic-index=0

How did you verify/test it?
Verified on T2 device and T0 kms

Signed-off-by: Austin Pham <austinpham@microsoft.com>

tests/platform_tests/link_flap/test_cont_link_flap.py

@@ -11,6 +11,8 @@
 import time
 import math
 
+from collections import defaultdict
+
 from tests.common.helpers.assertions import pytest_assert, pytest_require
 from tests.common import port_toggle
 from tests.platform_tests.link_flap.link_flap_utils import build_test_candidates,\
@@ -32,11 +34,23 @@ class TestContLinkFlap(object):
     """
 
     def get_frr_daemon_memory_usage(self, duthost, daemon):
-        frr_daemon_memory_output = duthost.shell(f'vtysh -c "show memory {daemon}"')["stdout"]
-        logging.info(f"{daemon} memory status: \n%s", frr_daemon_memory_output)
-        frr_daemon_memory = duthost.shell(
-            f'vtysh -c "show memory {daemon}" | grep "Used ordinary blocks"')["stdout"].split()[-2]
-        return frr_daemon_memory
+        frr_daemon_memory_per_asics = {}
+
+        for asic in duthost.asics:
+            frr_daemon_memory_output = asic.run_vtysh(f'-c "show memory {daemon}"')["stdout"]
+
+            logging.info(
+                f"{daemon}{('-' + asic.namespace) if asic.namespace else ''} memory status: \n%s",
+                frr_daemon_memory_output
+            )
+
+            frr_daemon_memory = asic.run_vtysh(
+                f'-c "show memory {daemon}" | grep "Used ordinary blocks"'
+            )["stdout"].split()[-2]
+
+            frr_daemon_memory_per_asics[asic.asic_index] = frr_daemon_memory
+
+        return frr_daemon_memory_per_asics
 
     def test_cont_link_flap(self, request, duthosts, nbrhosts, enum_rand_one_per_hwsku_frontend_hostname,
                             fanouthosts, bring_up_dut_interfaces, tbinfo):
@@ -150,32 +164,41 @@ def test_cont_link_flap(self, request, duthosts, nbrhosts, enum_rand_one_per_hws
 
         # Check the FRR daemons memory usage at end
         end_time_frr_daemon_memory = {}
-        incr_frr_daemon_memory_threshold = {}
+        incr_frr_daemon_memory_threshold = defaultdict(lambda: {})
+
         for daemon in frr_demons_to_check:
-            incr_frr_daemon_memory_threshold[daemon] = 10 if tbinfo["topo"]["type"] in ["m0", "mx"] else 5
-            min_threshold_percent = 1 / float(start_time_frr_daemon_memory[daemon]) * 100
-            if min_threshold_percent > incr_frr_daemon_memory_threshold[daemon]:
-                incr_frr_daemon_memory_threshold[daemon] = math.ceil(min_threshold_percent)
-            logging.info(f"The memory increment threshold for frr daemon {daemon} "
-                         f"is {incr_frr_daemon_memory_threshold[daemon]}%")
+            for asic_index, asic_frr_memory in start_time_frr_daemon_memory[daemon].items():
+                incr_frr_daemon_memory_threshold[daemon][asic_index] = 10 if tbinfo["topo"]["type"] in ["m0", "mx"]\
+                                                                       else 5
+
+                min_threshold_percent = 1 / float(asic_frr_memory) * 100
+
+                if min_threshold_percent > incr_frr_daemon_memory_threshold[daemon][asic_index]:
+                    incr_frr_daemon_memory_threshold[daemon][asic_index] = math.ceil(min_threshold_percent)
+
+                logging.info(f"The memory increment threshold for frr daemon {daemon}-asic{asic_index} "
+                             f"is {incr_frr_daemon_memory_threshold[daemon][asic_index]}%")
+
         for daemon in frr_demons_to_check:
             # Record FRR daemon memory status at end
             end_time_frr_daemon_memory[daemon] = self.get_frr_daemon_memory_usage(duthost, daemon)
             logging.info(f"{daemon} memory usage at end: \n%s", end_time_frr_daemon_memory[daemon])
 
             # Calculate diff in FRR daemon memory
-            incr_frr_daemon_memory = \
-                float(end_time_frr_daemon_memory[daemon]) - float(start_time_frr_daemon_memory[daemon])
-            logging.info(f"{daemon} absolute difference: %d", incr_frr_daemon_memory)
-
-            # Check FRR daemon memory only if it is increased else default to pass
-            if incr_frr_daemon_memory > 0:
-                percent_incr_frr_daemon_memory = \
-                    (incr_frr_daemon_memory / float(start_time_frr_daemon_memory[daemon])) * 100
-                logging.info(f"{daemon} memory percentage increase: %d", percent_incr_frr_daemon_memory)
-                pytest_assert(percent_incr_frr_daemon_memory < incr_frr_daemon_memory_threshold[daemon],
-                              f"{daemon} memory increase more than expected: "
-                              f"{incr_frr_daemon_memory_threshold[daemon]}%")
+            for asic_index, end_frr_memory in end_time_frr_daemon_memory[daemon].items():
+                incr_frr_daemon_memory = float(end_frr_memory) - float(start_time_frr_daemon_memory[daemon][asic_index])
+
+                daemon_name = daemon if not duthost.is_multi_asic else f"{daemon}-asic{asic_index}"
+                logging.info(f"{daemon_name} absolute difference: %d", incr_frr_daemon_memory)
+
+                # Check FRR daemon memory only if it is increased else default to pass
+                if incr_frr_daemon_memory > 0:
+                    percent_incr_frr_daemon_memory = \
+                        (incr_frr_daemon_memory / float(start_time_frr_daemon_memory[daemon][asic_index])) * 100
+                    logging.info(f"{daemon_name} memory percentage increase: %d", percent_incr_frr_daemon_memory)
+                    pytest_assert(percent_incr_frr_daemon_memory < incr_frr_daemon_memory_threshold[daemon][asic_index],
+                                  f"{daemon_name} memory increase more than expected: "
+                                  f"{incr_frr_daemon_memory_threshold[daemon][asic_index]}%")
 
         # Record orchagent CPU utilization at end
         orch_cpu = duthost.shell(