diff --git a/tests/bfd/bfd_base.py b/tests/bfd/bfd_base.py new file mode 100644 index 00000000000..0a7f44d4b95 --- /dev/null +++ b/tests/bfd/bfd_base.py @@ -0,0 +1,124 @@ + +import random +import pytest, re +import logging +logger = logging.getLogger(__name__) + + +class BfdBase: + def list_to_dict(self, sample_list): + header = sample_list[1].split() + data_rows = sample_list[3:] + for data in data_rows: + data_dict = {} + data = data.encode("utf-8").split() + data_dict['Peer Addr'] = data[0] + data_dict['Interface'] = data[1] + data_dict['Vrf'] = data[2] + data_dict['State'] = data[3] + data_dict['Type'] = data[4] + data_dict['Local Addr'] = data[5] + data_dict['TX Interval'] = data[6] + data_dict['RX Interval'] = data[7] + data_dict['Multiplier'] = data[8] + data_dict['Multihop'] = data[9] + data_dict['Local Discriminator'] = data[10] + return data_dict + + def selecting_route_to_delete(self, asic_routes, nexthops): + for asic in asic_routes: + for prefix in asic_routes[asic]: + nexthops_in_static_route_output = asic_routes[asic][prefix] + #If nexthops on source dut are same destination dut's interfaces, we are picking that static route + if sorted(nexthops_in_static_route_output) == sorted(nexthops): + return prefix + + def delete_bfd(self, asic_number, prefix, dut): + command = 'sonic-db-cli -n asic{} CONFIG_DB HSET "STATIC_ROUTE|{}" bfd \'false\''.format(asic_number, prefix).replace('\\', '') + dut.shell(command) + + def add_bfd(self, asic_number, prefix, dut): + command = 'sonic-db-cli -n asic{} CONFIG_DB HSET "STATIC_ROUTE|{}" bfd \'true\''.format(asic_number, prefix).replace('\\', '') + dut.shell(command) + + def extract_current_bfd_state(self, list_of_nexthops, asic_number, dut): + + for nexthop in list_of_nexthops: + bfd_peer_command = "ip netns exec asic{} show bfd peer {}".format(asic_number, nexthop) + logger.info("Verifying BFD status on {}".format(dut)) + logger.info(bfd_peer_command) + bfd_peer_output = dut.shell(bfd_peer_command, module_ignore_errors=True)["stdout"].encode("utf-8").strip().split("\n") + if "No BFD sessions found" in bfd_peer_output[0]: + return "No BFD sessions found" + else: + entry = self.list_to_dict(bfd_peer_output) + return entry['State'] + + def extract_routes(self, static_route_output): + asic_routes = {} + asic = None + for line in static_route_output: + if line.startswith("asic"): + asic = line.split(':')[0] + asic_routes[asic] = {} + elif line.startswith("S>*") or line.startswith(" *"): + parts = line.split(',') + if line.startswith("S>*"): + prefix = re.search(r"(\d+\.\d+\.\d+\.\d+/\d+)", parts[0]).group(1) + next_hop = re.search(r"via\s+(\d+\.\d+\.\d+\.\d+)", parts[0]).group(1) + asic_routes[asic].setdefault(prefix, []).append(next_hop) + return asic_routes + + @pytest.fixture(scope='class', name="select_src_dst_dut_and_asic", + params=(["multi_dut"])) + def select_src_dst_dut_and_asic(self, duthosts, request, tbinfo): + src_dut_index = 0 + dst_dut_index = 0 + src_asic_index = 0 + dst_asic_index = 0 + if (len(duthosts.frontend_nodes)) < 2: + pytest.skip("Don't have 2 frontend nodes - so can't run multi_dut tests") + dut_indices = random.sample(list(range(len(duthosts.frontend_nodes))), 2) + src_dut_index = dut_indices[0] + dst_dut_index = dut_indices[1] + asic_indices = random.sample(duthosts[src_dut_index].get_asic_namespace_list(), 2) + src_asic_index = asic_indices[0].split("asic")[1] + dst_asic_index = asic_indices[1].split("asic")[1] + + yield { + "src_dut_index": src_dut_index, + "dst_dut_index": dst_dut_index, + "src_asic_index": int(src_asic_index), + "dst_asic_index": int(dst_asic_index) + } + + @pytest.fixture(scope='class') + def get_src_dst_asic_and_duts(self, duthosts, select_src_dst_dut_and_asic): + logger.info("Printing select_src_dst_dut_and_asic") + logger.info(select_src_dst_dut_and_asic) + + logger.info("Printing duthosts.frontend_nodes") + logger.info(duthosts.frontend_nodes) + src_dut = duthosts.frontend_nodes[select_src_dst_dut_and_asic["src_dut_index"]] + dst_dut = duthosts.frontend_nodes[select_src_dst_dut_and_asic["dst_dut_index"]] + + logger.info("Printing source dut asics") + logger.info(src_dut.asics) + logger.info("Printing destination dut asics") + logger.info(dst_dut.asics) + src_asic = src_dut.asics[select_src_dst_dut_and_asic["src_asic_index"]] + dst_asic = dst_dut.asics[select_src_dst_dut_and_asic["dst_asic_index"]] + + all_asics = [src_asic, dst_asic] + all_duts = [src_dut, dst_dut] + + rtn_dict = { + "src_asic": src_asic, + "dst_asic": dst_asic, + "src_dut": src_dut, + "dst_dut": dst_dut, + "all_asics": all_asics, + "all_duts": all_duts + } + rtn_dict.update(select_src_dst_dut_and_asic) + yield rtn_dict diff --git a/tests/bfd/conftest.py b/tests/bfd/conftest.py index 6e49ab13bb7..c1648e090dc 100644 --- a/tests/bfd/conftest.py +++ b/tests/bfd/conftest.py @@ -1,3 +1,19 @@ +import pytest +from bfd_base import BfdBase + +@pytest.fixture(scope='class') +def bfd_base_instance(): + return BfdBase() + def pytest_addoption(parser): parser.addoption("--num_sessions", action="store", default=5) parser.addoption("--num_sessions_scale", action="store", default=128) + +@pytest.fixture(scope='function') +def bfd_cleanup_db(request, autouse=True): + yield + command = 'sonic-db-cli -n asic{} CONFIG_DB HSET "STATIC_ROUTE|{}" bfd \'true\''.format(request.config.src_asic.asic_index, request.config.src_prefix).replace('\\', '') + #1 - for new entry , 0 for modification of existing entry + request.config.src_dut.shell(command) + command = 'sonic-db-cli -n asic{} CONFIG_DB HSET "STATIC_ROUTE|{}" bfd \'true\''.format(request.config.dst_asic.asic_index, request.config.dst_prefix).replace('\\', '') + request.config.dst_dut.shell(command) diff --git a/tests/bfd/test_bfd_static_route.py b/tests/bfd/test_bfd_static_route.py new file mode 100644 index 00000000000..e5a5fe86c31 --- /dev/null +++ b/tests/bfd/test_bfd_static_route.py @@ -0,0 +1,178 @@ +import pytest +from bfd_base import BfdBase +import logging +import time + +pytestmark = [ + pytest.mark.topology('t2') +] +logger = logging.getLogger(__name__) + +class TestBfdStaticRoute(BfdBase): + test_case_status = True + total_iterations = 100 + + def test_bfd_deletion(self, duthost, request, duthosts, tbinfo, get_src_dst_asic_and_duts, bfd_base_instance, bfd_cleanup_db): + """ + Test case #1 - To verify deletion of BFD session between two line cards. + Test Steps: + 1. Delete BFD on Source dut + 2. Verify that on Source dut BFD gets cleaned up and static route exists. + 3. Verify that on Destination dut BFD goes down and static route will be removed. + 4. Delete BFD on Destination dut. + 5. Verify that on Destination dut BFD gets cleaned up and static route will be added back. + """ + logger.info("Selecting Source dut, destination dut, source asic, destination asic, source prefix, destination prefix") + src_asic, dst_asic, src_dut, dst_dut, src_dut_nexthops, dst_dut_nexthops, src_prefix, dst_prefix = self.select_src_dst_dut_with_asic(request, get_src_dst_asic_and_duts, bfd_base_instance) + + logger.info("BFD deletion on source dut") + bfd_base_instance.delete_bfd(src_asic.asic_index, src_prefix, src_dut) + + logger.info("BFD & Static route verifications") + self.verify_bfd_static_route(dst_asic, dst_prefix, dst_dut, dst_dut_nexthops, "Route Removal", "Down", bfd_base_instance) + self.verify_bfd_static_route(src_asic, src_prefix, src_dut, src_dut_nexthops, "Route Addition", "No BFD sessions found", bfd_base_instance) + + logger.info("BFD deletion on destination dut") + bfd_base_instance.delete_bfd(dst_asic.asic_index, dst_prefix, dst_dut) + + logger.info("BFD & Static route verifications") + self.verify_bfd_static_route(dst_asic, dst_prefix, dst_dut, dst_dut_nexthops, "Route Addition", "No BFD sessions found", bfd_base_instance) + self.verify_bfd_static_route(src_asic, src_prefix, src_dut, src_dut_nexthops, "Route Addition", "No BFD sessions found", bfd_base_instance) + + assert self.test_case_status, "BFD deletion did not influence static routes" + logger.info("test_bfd_static_route_deletion completed") + + def verify_bfd_static_route(self, asic, prefix, dut, dut_nexthops, expected_prefix_state, expected_bfd_state, bfd_base_instance): + #Verification of BFD session + timeout = 180 # Timeout in seconds (3 minutes) + start_time = time.time() + while True: + bfd_state = bfd_base_instance.extract_current_bfd_state(dut_nexthops.values(), asic.asic_index, dut) + if bfd_state == expected_bfd_state: + break + if time.time() - start_time >= timeout: + self.test_case_status = False + assert False, "Expected BFD state '{}' was not reached within {} seconds".format(expected_state, timeout) + time.sleep(1) + + #Verification of static route + static_route_output = dut.shell("show ip route static", module_ignore_errors=True)["stdout"].encode("utf-8").strip().split("\n") + asic_routes = bfd_base_instance.extract_routes(static_route_output) + + if expected_prefix_state == "Route Removal": + if prefix in asic_routes["asic{}".format(asic.asic_index)].keys(): + self.test_case_status = False + logger.info("Prefix being validated: ", prefix) + logger.info("List of available prefixes now:", asic_routes["asic{}".format(asic.asic_index)].keys()) + assert False, "Prefix removal is not successful" + elif expected_prefix_state == "Route Addition": + if prefix not in asic_routes["asic{}".format(asic.asic_index)].keys(): + self.test_case_status = False + logger.info("Prefix being validated: ", prefix) + logger.info("List of available prefixes now:", asic_routes["asic{}".format(asic.asic_index)].keys()) + assert False, "Prefix has been removed even though BFD doesnt exist" + + def select_src_dst_dut_with_asic(self, request, get_src_dst_asic_and_duts, bfd_base_instance): + logger.debug("Selecting source and destination DUTs with ASICs...") + #Random selection of dut & asic. + src_asic = get_src_dst_asic_and_duts['src_asic'] + dst_asic = get_src_dst_asic_and_duts['dst_asic'] + src_dut = get_src_dst_asic_and_duts['src_dut'] + dst_dut = get_src_dst_asic_and_duts['dst_dut'] + + logger.info("Source Asic: %s", src_asic) + logger.info("Destination Asic: %s", dst_asic) + logger.info("Source dut: %s", src_dut) + logger.info("Destination dut: %s", dst_dut) + + request.config.src_asic = src_asic + request.config.dst_asic = dst_asic + request.config.src_dut = src_dut + request.config.dst_dut = dst_dut + + #Extracting static routes and corresponding nexthops + src_dut_static_route_output = src_dut.shell("show ip route static", module_ignore_errors=True)["stdout"].encode("utf-8").strip().split("\n") + src_asic_routes = bfd_base_instance.extract_routes(src_dut_static_route_output) + + dst_dut_static_route_output = dst_dut.shell("show ip route static", module_ignore_errors=True)["stdout"].encode("utf-8").strip().split("\n") + dst_asic_routes = bfd_base_instance.extract_routes(dst_dut_static_route_output) + + #Extracting nexthops + try: + dst_dut_nexthops = {intf['interface']:intf['ipv4 address/mask'].split("/24")[0] for intf in src_dut.show_and_parse("show ip int -d all -n asic{}".format(src_asic.asic_index)) if "PortChannel" in intf['interface'] and intf['bgp neighbor'] == "N/A" } + src_dut_nexthops = {intf['interface']:intf['ipv4 address/mask'].split("/24")[0] for intf in dst_dut.show_and_parse("show ip int -d all -n asic{}".format(dst_asic.asic_index)) if "PortChannel" in intf['interface'] and intf['bgp neighbor'] == "N/A" } + except Exception: + pytest.fail("Possibily containers are down on {} and {}".format(src_dut, dst_dut)) + + #Picking a static route to delete correspinding BFD session + src_prefix = bfd_base_instance.selecting_route_to_delete(src_asic_routes, src_dut_nexthops.values()) + dst_prefix = bfd_base_instance.selecting_route_to_delete(dst_asic_routes, dst_dut_nexthops.values()) + request.config.src_prefix = src_prefix + request.config.dst_prefix = dst_prefix + logger.info("Source prefix: %s", src_prefix) + logger.info("Destination prefix: %s", dst_prefix) + + #Verification of BFD sessions before deleting them. + dst_bfd_state = bfd_base_instance.extract_current_bfd_state(dst_dut_nexthops.values(), dst_asic.asic_index, dst_dut) + if dst_bfd_state != "Up": + self.test_case_status = False + assert False, "BFD sessions are expected to stay up at the beginning of the test case but it's down on {}".format(dst_dut) + src_bfd_state = bfd_base_instance.extract_current_bfd_state(src_dut_nexthops.values(), src_asic.asic_index, src_dut) + if src_bfd_state != "Up": + self.test_case_status = False + assert False, "BFD sessions are expected to stay up at the beginning of the test case but it's down on {}".format(src_dut) + logger.debug("Source and destination DUTs selection completed") + + return src_asic, dst_asic, src_dut, dst_dut, src_dut_nexthops, dst_dut_nexthops, src_prefix, dst_prefix + + def test_bfd_flap(self, duthost, request, duthosts, tbinfo, get_src_dst_asic_and_duts, bfd_base_instance): + """ + Test case #2 - To flap the BFD session ( Up <--> Down <---> Up) between linecards for 100 times. + Test Steps: + 1. Delete BFD on Source dut + 2. Verify that on Source dut BFD gets cleaned up and static route exists. + 3. Verify that on Destination dut BFD goes down and static route will be removed. + 4. Add BFD on Source dut. + 5. Verify that on Source dut BFD is up + 6. Verify that on destination dut BFD is up and static route is added back. + 7. Repeat above steps 100 times. + """ + logger.info("Selecting Source dut, destination dut, source asic, destination asic, source prefix, destination prefix") + src_asic, dst_asic, src_dut, dst_dut, src_dut_nexthops, dst_dut_nexthops, src_prefix, dst_prefix = self.select_src_dst_dut_with_asic(request, get_src_dst_asic_and_duts, bfd_base_instance) + + successful_iterations = 0 # Counter for successful iterations + + for i in range(self.total_iterations): + logger.info("Iteration {}".format(i)) + + logger.info("BFD deletion on source dut") + bfd_base_instance.delete_bfd(src_asic.asic_index, src_prefix, src_dut) + + logger.info("Waiting for 5s post BFD shutdown") + time.sleep(5) + + logger.info("BFD & Static route verifications") + self.verify_bfd_static_route(dst_asic, dst_prefix, dst_dut, dst_dut_nexthops, "Route Removal", "Down", bfd_base_instance) + self.verify_bfd_static_route(src_asic, src_prefix, src_dut, src_dut_nexthops, "Route Addition", "No BFD sessions found", bfd_base_instance) + + logger.info("BFD addition on source dut") + bfd_base_instance.add_bfd(src_asic.asic_index, src_prefix, src_dut) + + logger.info("BFD & Static route verifications") + self.verify_bfd_static_route(dst_asic, dst_prefix, dst_dut, dst_dut_nexthops, "Route Addition", "Up", bfd_base_instance) + self.verify_bfd_static_route(src_asic, src_prefix, src_dut, src_dut_nexthops, "Route Addition", "Up", bfd_base_instance) + + # Check if both iterations were successful and increment the counter + if self.test_case_status: + successful_iterations += 1 + + # Determine the success rate + logger.info("successful_iterations: %d", successful_iterations) + success_rate = (successful_iterations / self.total_iterations) * 100 + + logger.info("Current success rate: %.2f%%", success_rate) + # Check if the success rate is above the threshold (e.g., 98%) + assert success_rate >= 98, "BFD flap verification success rate is below 98% ({}%)".format(success_rate) + + logger.info("test_bfd_flap completed") +