From c17c5cef098538a7a45fe1aae0c47ae06ab7437e Mon Sep 17 00:00:00 2001 From: "Austin (Thang Pham)" Date: Wed, 25 Sep 2024 13:24:54 +1000 Subject: [PATCH] feat: add multi-asic in test_cont_link_flap (#14673) Description of PR Currently, test_cont_link_flap is not compatible with multi-asic. It's failing on vtysh command raise RunAnsibleModuleFail("run module {} failed".format(self.module_name), res) tests.common.errors.RunAnsibleModuleFail: run module shell failed, Ansible Results => {"changed": true, "cmd": "vtysh -c \"show memory bgpd\"", "delta": "0:00:00.006507", "end": "2024-09-17 00:06:50.914882", "failed": true, "msg": "non-zero return code", "rc": 1, "start": "2024-09-17 00:06:50.908375", "stderr": "Usage: /usr/bin/vtysh -n [0 to 2] [OPTION]... ", "stderr_lines": ["Usage: /usr/bin/vtysh -n [0 to 2] [OPTION]... "], "stdout": "", "stdout_lines": []} This is because for multi-asic platform, we need to specify -n. Summary: add multi-asic support for test_cont_link_flap Fixes # (issue) 2954952 Approach What is the motivation for this PR? Support multi-asic for test_cont_link_flap by calculating the memory based of each individual asic How did you do it? Since SonicAsic.run_asic has already provided a back-ward compatible to multi-asic and single-asic (https://github.com/sonic-net/sonic-mgmt/blob/master/tests/common/devices/sonic_asic.py#L404) we can leverage this API to execute the test. The test is also working with single-asic since in our system, single-asic chassis will have 1 asic with namespace=None and asic-index=0 How did you verify/test it? Verified on T2 device and T0 kms Signed-off-by: Austin Pham --- .../link_flap/test_cont_link_flap.py | 71 ++++++++++++------- 1 file changed, 47 insertions(+), 24 deletions(-) diff --git a/tests/platform_tests/link_flap/test_cont_link_flap.py b/tests/platform_tests/link_flap/test_cont_link_flap.py index af38c0ad81..0a2c8717ac 100644 --- a/tests/platform_tests/link_flap/test_cont_link_flap.py +++ b/tests/platform_tests/link_flap/test_cont_link_flap.py @@ -11,6 +11,8 @@ import time import math +from collections import defaultdict + from tests.common.helpers.assertions import pytest_assert, pytest_require from tests.common import port_toggle from tests.platform_tests.link_flap.link_flap_utils import build_test_candidates, toggle_one_link,\ @@ -31,11 +33,23 @@ class TestContLinkFlap(object): """ def get_frr_daemon_memory_usage(self, duthost, daemon): - frr_daemon_memory_output = duthost.shell(f'vtysh -c "show memory {daemon}"')["stdout"] - logging.info(f"{daemon} memory status: \n%s", frr_daemon_memory_output) - frr_daemon_memory = duthost.shell( - f'vtysh -c "show memory {daemon}" | grep "Used ordinary blocks"')["stdout"].split()[-2] - return frr_daemon_memory + frr_daemon_memory_per_asics = {} + + for asic in duthost.asics: + frr_daemon_memory_output = asic.run_vtysh(f'-c "show memory {daemon}"')["stdout"] + + logging.info( + f"{daemon}{('-' + asic.namespace) if asic.namespace else ''} memory status: \n%s", + frr_daemon_memory_output + ) + + frr_daemon_memory = asic.run_vtysh( + f'-c "show memory {daemon}" | grep "Used ordinary blocks"' + )["stdout"].split()[-2] + + frr_daemon_memory_per_asics[asic.asic_index] = frr_daemon_memory + + return frr_daemon_memory_per_asics def test_cont_link_flap(self, request, duthosts, nbrhosts, enum_rand_one_per_hwsku_frontend_hostname, fanouthosts, bring_up_dut_interfaces, tbinfo): @@ -149,32 +163,41 @@ def test_cont_link_flap(self, request, duthosts, nbrhosts, enum_rand_one_per_hws # Check the FRR daemons memory usage at end end_time_frr_daemon_memory = {} - incr_frr_daemon_memory_threshold = {} + incr_frr_daemon_memory_threshold = defaultdict(lambda: {}) + for daemon in frr_demons_to_check: - incr_frr_daemon_memory_threshold[daemon] = 10 if tbinfo["topo"]["type"] in ["m0", "mx"] else 5 - min_threshold_percent = 1 / float(start_time_frr_daemon_memory[daemon]) * 100 - if min_threshold_percent > incr_frr_daemon_memory_threshold[daemon]: - incr_frr_daemon_memory_threshold[daemon] = math.ceil(min_threshold_percent) - logging.info(f"The memory increment threshold for frr daemon {daemon} " - f"is {incr_frr_daemon_memory_threshold[daemon]}%") + for asic_index, asic_frr_memory in start_time_frr_daemon_memory[daemon].items(): + incr_frr_daemon_memory_threshold[daemon][asic_index] = 10 if tbinfo["topo"]["type"] in ["m0", "mx"]\ + else 5 + + min_threshold_percent = 1 / float(asic_frr_memory) * 100 + + if min_threshold_percent > incr_frr_daemon_memory_threshold[daemon][asic_index]: + incr_frr_daemon_memory_threshold[daemon][asic_index] = math.ceil(min_threshold_percent) + + logging.info(f"The memory increment threshold for frr daemon {daemon}-asic{asic_index} " + f"is {incr_frr_daemon_memory_threshold[daemon][asic_index]}%") + for daemon in frr_demons_to_check: # Record FRR daemon memory status at end end_time_frr_daemon_memory[daemon] = self.get_frr_daemon_memory_usage(duthost, daemon) logging.info(f"{daemon} memory usage at end: \n%s", end_time_frr_daemon_memory[daemon]) # Calculate diff in FRR daemon memory - incr_frr_daemon_memory = \ - float(end_time_frr_daemon_memory[daemon]) - float(start_time_frr_daemon_memory[daemon]) - logging.info(f"{daemon} absolute difference: %d", incr_frr_daemon_memory) - - # Check FRR daemon memory only if it is increased else default to pass - if incr_frr_daemon_memory > 0: - percent_incr_frr_daemon_memory = \ - (incr_frr_daemon_memory / float(start_time_frr_daemon_memory[daemon])) * 100 - logging.info(f"{daemon} memory percentage increase: %d", percent_incr_frr_daemon_memory) - pytest_assert(percent_incr_frr_daemon_memory < incr_frr_daemon_memory_threshold[daemon], - f"{daemon} memory increase more than expected: " - f"{incr_frr_daemon_memory_threshold[daemon]}%") + for asic_index, end_frr_memory in end_time_frr_daemon_memory[daemon].items(): + incr_frr_daemon_memory = float(end_frr_memory) - float(start_time_frr_daemon_memory[daemon][asic_index]) + + daemon_name = daemon if not duthost.is_multi_asic else f"{daemon}-asic{asic_index}" + logging.info(f"{daemon_name} absolute difference: %d", incr_frr_daemon_memory) + + # Check FRR daemon memory only if it is increased else default to pass + if incr_frr_daemon_memory > 0: + percent_incr_frr_daemon_memory = \ + (incr_frr_daemon_memory / float(start_time_frr_daemon_memory[daemon][asic_index])) * 100 + logging.info(f"{daemon_name} memory percentage increase: %d", percent_incr_frr_daemon_memory) + pytest_assert(percent_incr_frr_daemon_memory < incr_frr_daemon_memory_threshold[daemon][asic_index], + f"{daemon_name} memory increase more than expected: " + f"{incr_frr_daemon_memory_threshold[daemon][asic_index]}%") # Record orchagent CPU utilization at end orch_cpu = duthost.shell(