[rdma] Test PFC watchdog's impact on runtime traffic (#2979)

Signed-off-by: Wei Bai webai@microsoft.com

What is the motivation for this PR?
Test if enabling PFC watchdog will impact runtime traffic

How did you do it?
Start data traffic first then enable PFC watchdog at the SONiC DUT

How did you verify/test it?
I did test using Mellanox SN2700 and IXIA chassis

tests/ixia/pfcwd/files/pfcwd_runtime_traffic_helper.py

@@ -0,0 +1,223 @@
+import time
+
+from tests.common.helpers.assertions import pytest_assert
+from tests.common.ixia.ixia_fixtures import ixia_api_serv_ip, ixia_api_serv_port,\
+ ixia_api_serv_user, ixia_api_serv_passwd, ixia_api
+from tests.common.ixia.ixia_helpers import get_dut_port_id
+from tests.common.ixia.common_helpers import start_pfcwd, stop_pfcwd
+
+from abstract_open_traffic_generator.flow import DeviceTxRx, TxRx, Flow, Header,\
+ Size, Rate, Duration, FixedSeconds
+from abstract_open_traffic_generator.flow_ipv4 import Priority, Dscp
+from abstract_open_traffic_generator.flow import Pattern as FieldPattern
+from abstract_open_traffic_generator.flow import Ipv4 as Ipv4Header
+from abstract_open_traffic_generator.flow import Ethernet as EthernetHeader
+from abstract_open_traffic_generator.control import State, ConfigState,\
+ FlowTransmitState
+from abstract_open_traffic_generator.result import FlowRequest
+
+DATA_FLOW_NAME = "Data Flow"
+DATA_PKT_SIZE = 1024
+DATA_FLOW_DURATION_SEC = 15
+PFCWD_START_DELAY_SEC = 3
+IXIA_POLL_DELAY_SEC = 2
+TOLERANCE_THRESHOLD = 0.05
+
+def run_pfcwd_runtime_traffic_test(api,
+ testbed_config,
+ conn_data,
+ fanout_data,
+ duthost,
+ dut_port,
+ prio_list,
+ prio_dscp_map):
+ """
+ Test PFC watchdog's impact on runtime traffic
+
+ Args:
+ api (obj): IXIA session
+ testbed_config (obj): L2/L3 config of a T0 testbed
+ conn_data (dict): the dictionary returned by conn_graph_fact.
+ fanout_data (dict): the dictionary returned by fanout_graph_fact.
+ duthost (Ansible host instance): device under test
+ dut_port (str): DUT port to test
+ prio_list (list): priorities of data flows
+ prio_dscp_map (dict): Priority vs. DSCP map (key = priority).
+
+ Returns:
+ N/A
+ """
+ pytest_assert(testbed_config is not None, 'Fail to get L2/3 testbed config')
+
+ stop_pfcwd(duthost)
+
+ """ Get the ID of the port to test """
+ port_id = get_dut_port_id(dut_hostname=duthost.hostname,
+ dut_port=dut_port,
+ conn_data=conn_data,
+ fanout_data=fanout_data)
+
+ pytest_assert(port_id is not None,
+ 'Fail to get ID for port {}'.format(dut_port))
+
+ flows = __gen_traffic(testbed_config=testbed_config,
+ port_id=port_id,
+ data_flow_name=DATA_FLOW_NAME,
+ data_flow_dur_sec=DATA_FLOW_DURATION_SEC,
+ data_pkt_size=DATA_PKT_SIZE,
+ prio_list=prio_list,
+ prio_dscp_map=prio_dscp_map)
+
+ """ Tgen config = testbed config + flow config """
+ config = testbed_config
+ config.flows = flows
+
+ all_flow_names = [flow.name for flow in flows]
+
+ flow_stats = __run_traffic(api=api,
+ config=config,
+ duthost=duthost,
+ all_flow_names=all_flow_names,
+ pfcwd_start_delay_sec=PFCWD_START_DELAY_SEC,
+ exp_dur_sec=DATA_FLOW_DURATION_SEC)
+
+ speed_str = config.layer1[0].speed
+ speed_gbps = int(speed_str.split('_')[1])
+
+ __verify_results(rows=flow_stats,
+ speed_gbps=speed_gbps,
+ data_flow_dur_sec=DATA_FLOW_DURATION_SEC,
+ data_pkt_size=DATA_PKT_SIZE,
+ tolerance=TOLERANCE_THRESHOLD)
+
+def __gen_traffic(testbed_config,
+ port_id,
+ data_flow_name,
+ data_flow_dur_sec,
+ data_pkt_size,
+ prio_list,
+ prio_dscp_map):
+ """
+ Generate configurations of flows
+
+ Args:
+ testbed_config (obj): L2/L3 config of a T0 testbed
+ port_id (int): ID of DUT port to test.
+ data_flow_name (str): data flow name
+ data_flow_dur_sec (int): duration of data flows in second
+ data_pkt_size (int): size of data packets in byte
+ prio_list (list): priorities of data flows
+ prio_dscp_map (dict): Priority vs. DSCP map (key = priority).
+
+ Returns:
+ flows configurations (list): the list should have configurations of
+ len(prio_list) data flows
+ """
+ rx_port_id = port_id
+ tx_port_id = (port_id + 1) % len(testbed_config.devices)
+
+ data_endpoint = DeviceTxRx(
+ tx_device_names=[testbed_config.devices[tx_port_id].name],
+ rx_device_names=[testbed_config.devices[rx_port_id].name],
+ )
+
+ result = list()
+ data_flow_rate_percent = int(100 / len(prio_list))
+
+ """ For each priority """
+ for prio in prio_list:
+ ip_prio = Priority(Dscp(phb=FieldPattern(choice=prio_dscp_map[prio]),
+ ecn=FieldPattern(choice=Dscp.ECN_CAPABLE_TRANSPORT_1)))
+ pfc_queue = FieldPattern([prio])
+
+ data_flow = Flow(
+ name='{} Prio {}'.format(data_flow_name, prio),
+ tx_rx=TxRx(data_endpoint),
+ packet=[
+ Header(choice=EthernetHeader(pfc_queue=pfc_queue)),
+ Header(choice=Ipv4Header(priority=ip_prio))
+ ],
+ size=Size(data_pkt_size),
+ rate=Rate('line', data_flow_rate_percent),
+ duration=Duration(FixedSeconds(seconds=data_flow_dur_sec))
+ )
+
+ result.append(data_flow)
+
+ return result
+
+def __run_traffic(api, config, duthost, all_flow_names, pfcwd_start_delay_sec, exp_dur_sec):
+ """
+ Start traffic at time 0 and enable PFC watchdog at pfcwd_start_delay_sec
+
+ Args:
+ api (obj): IXIA session
+ config (obj): experiment config (testbed config + flow config)
+ duthost (Ansible host instance): device under test
+ all_flow_names (list): list of names of all the flows
+ pfcwd_start_delay_sec (int): PFC watchdog start delay in second
+ exp_dur_sec (int): experiment duration in second
+
+ Returns:
+ per-flow statistics (list)
+ """
+
+ api.set_state(State(ConfigState(config=config, state='set')))
+ api.set_state(State(FlowTransmitState(state='start')))
+
+ time.sleep(pfcwd_start_delay_sec)
+ start_pfcwd(duthost)
+ time.sleep(exp_dur_sec - pfcwd_start_delay_sec)
+
+ attempts = 0
+ max_attempts = 20
+
+ while attempts < max_attempts:
+ rows = api.get_flow_results(FlowRequest(flow_names=all_flow_names))
+ """ If all the flows have stopped """
+ transmit_states = [row['transmit'] for row in rows]
+ if len(rows) == len(all_flow_names) and\
+ list(set(transmit_states)) == ['stopped']:
+ time.sleep(IXIA_POLL_DELAY_SEC)
+ break
+ else:
+ time.sleep(1)
+ attempts += 1
+
+ """ Dump per-flow statistics """
+ rows = api.get_flow_results(FlowRequest(flow_names=all_flow_names))
+ api.set_state(State(FlowTransmitState(state='stop')))
+
+ return rows
+
+def __verify_results(rows, speed_gbps, data_flow_dur_sec, data_pkt_size, tolerance):
+ """
+ Verify if we get expected experiment results
+
+ Args:
+ rows (list): per-flow statistics
+ speed_gbps (int): link speed in Gbps
+ data_flow_dur_sec (int): duration of data flows in second
+ data_pkt_size (int): size of data packets in byte
+ tolerance (float): maximum allowable deviation
+
+ Returns:
+ N/A
+ """
+ data_flow_rate_percent = int(100 / len(rows))
+
+ for row in rows:
+ flow_name = row['name']
+ tx_frames = row['frames_tx']
+ rx_frames = row['frames_rx']
+
+ pytest_assert(tx_frames == rx_frames, "{} packets of {} are dropped".\
+ format(tx_frames-rx_frames, flow_name))
+
+ exp_rx_pkts = data_flow_rate_percent / 100.0 * speed_gbps \
+ * 1e9 * data_flow_dur_sec / 8.0 / data_pkt_size
+
+ deviation = (rx_frames - exp_rx_pkts) / float(exp_rx_pkts)
+ pytest_assert(abs(deviation) < tolerance,
+ "{} should receive {} packets (actual {})".\
+ format(flow_name, exp_rx_pkts, rx_frames))

tests/ixia/pfcwd/test_pfcwd_runtime_traffic.py

@@ -0,0 +1,53 @@
+import pytest
+
+from tests.common.helpers.assertions import pytest_require, pytest_assert
+from tests.common.fixtures.conn_graph_facts import conn_graph_facts,\
+ fanout_graph_facts
+from tests.common.ixia.ixia_fixtures import ixia_api_serv_ip, ixia_api_serv_port,\
+ ixia_api_serv_user, ixia_api_serv_passwd, ixia_api, ixia_testbed
+from tests.common.ixia.qos_fixtures import prio_dscp_map, all_prio_list
+
+from files.pfcwd_runtime_traffic_helper import run_pfcwd_runtime_traffic_test
+
+@pytest.mark.topology("tgen")
+
+def test_pfcwd_runtime_traffic(ixia_api,
+ ixia_testbed,
+ conn_graph_facts,
+ fanout_graph_facts,
+ duthosts,
+ rand_one_dut_hostname,
+ rand_one_dut_portname_oper_up,
+ all_prio_list,
+ prio_dscp_map):
+ """
+ Test PFC watchdog's impact on runtime traffic
+
+ Args:
+ ixia_api (pytest fixture): IXIA session
+ ixia_testbed (pytest fixture): L2/L3 config of a T0 testbed
+ conn_graph_facts (pytest fixture): connection graph
+ fanout_graph_facts (pytest fixture): fanout graph
+ duthosts (pytest fixture): list of DUTs
+ rand_one_dut_hostname (str): hostname of DUT
+ rand_one_dut_portname_oper_up (str): port to test, e.g., 's6100-1|Ethernet0'
+ all_prio_list (pytest fixture): list of all the priorities
+ prio_dscp_map (pytest fixture): priority vs. DSCP map (key = priority)
+
+ Returns:
+ N/A
+ """
+ dut_hostname, dut_port = rand_one_dut_portname_oper_up.split('|')
+ pytest_require(rand_one_dut_hostname == dut_hostname,
+ "Port is not mapped to the expected DUT")
+
+ duthost = duthosts[rand_one_dut_hostname]
+
+ run_pfcwd_runtime_traffic_test(api=ixia_api,
+ testbed_config=ixia_testbed,
+ conn_data=conn_graph_facts,
+ fanout_data=fanout_graph_facts,
+ duthost=duthost,
+ dut_port=dut_port,
+ prio_list=all_prio_list,
+ prio_dscp_map=prio_dscp_map)