[dualtor] test_server_down fails and experiences teardown failure

[theasianpianist]

The test is unable to toggle the mux simulator

duthosts = <tests.common.devices.duthosts.DutHosts object at 0x7fcfa983eb90>
tbinfo = {'auto_recover': 'True', 'comment': 'hellogemini', 'conf-name': 'vms17-dual-t0-7050-1', 'duts': ['str2-7050cx3-acs-06', 'str2-7050cx3-acs-07'], ...}
rand_selected_interface = ('Ethernet60', {'server_ipv4': '192.168.0.16/32', 'server_ipv6': 'fc02:1000::10/128', 'state': 'auto'})
simulator_flap_counter = <function _simulator_flap_counter at 0x7fcf99ae5c50>
simulator_server_down = <function _drop_helper at 0x7fcf99ae5f50>
toggle_simulator_port_to_upper_tor = <function _toggle_simulator_port_to_upper_tor at 0x7fcfaae04750>
loganalyzer = {'str2-7050cx3-acs-06': <tests.common.plugins.loganalyzer.loganalyzer.LogAnalyzer instance at 0x7fcf98cc9dc0>, 'str2-7050cx3-acs-07': <tests.common.plugins.loganalyzer.loganalyzer.LogAnalyzer instance at 0x7fcf98d2feb0>}

    def test_server_down(duthosts, tbinfo, rand_selected_interface, simulator_flap_counter, simulator_server_down, toggle_simulator_port_to_upper_tor, loganalyzer):
        """
        Verify that mux cable is not toggled excessively.
        """
    
        for analyzer in list(loganalyzer.values()):
            analyzer.ignore_regex.append(r".*ERR swss#orchagent: :- setState: State transition from active to active is not-handled")
    
        upper_tor = duthosts[tbinfo['duts'][0]]
        lower_tor = duthosts[tbinfo['duts'][1]]
        PAUSE_TIME = 5
    
        itfs, _ = rand_selected_interface
        # Set upper_tor as active
>       toggle_simulator_port_to_upper_tor(itfs)

PAUSE_TIME = 5
_          = {'server_ipv4': '192.168.0.16/32', 'server_ipv6': 'fc02:1000::10/128', 'state': 'auto'}
analyzer   = <tests.common.plugins.loganalyzer.loganalyzer.LogAnalyzer instance at 0x7fcf98d2feb0>
duthosts   = <tests.common.devices.duthosts.DutHosts object at 0x7fcfa983eb90>
itfs       = 'Ethernet60'
loganalyzer = {'str2-7050cx3-acs-06': <tests.common.plugins.loganalyzer.loganalyzer.LogAnalyzer instance at 0x7fcf98cc9dc0>, 'str2-7050cx3-acs-07': <tests.common.plugins.loganalyzer.loganalyzer.LogAnalyzer instance at 0x7fcf98d2feb0>}
lower_tor  = <MultiAsicSonicHost> str2-7050cx3-acs-07
rand_selected_interface = ('Ethernet60', {'server_ipv4': '192.168.0.16/32', 'server_ipv6': 'fc02:1000::10/128', 'state': 'auto'})
simulator_flap_counter = <function _simulator_flap_counter at 0x7fcf99ae5c50>
simulator_server_down = <function _drop_helper at 0x7fcf99ae5f50>
tbinfo     = {'auto_recover': 'True', 'comment': 'hellogemini', 'conf-name': 'vms17-dual-t0-7050-1', 'duts': ['str2-7050cx3-acs-06', 'str2-7050cx3-acs-07'], ...}
toggle_simulator_port_to_upper_tor = <function _toggle_simulator_port_to_upper_tor at 0x7fcfaae04750>
upper_tor  = <MultiAsicSonicHost> str2-7050cx3-acs-06

dualtor/test_server_failure.py:31: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

interface_name = 'Ethernet60'

    def _toggle_simulator_port_to_upper_tor(interface_name):
        """
        A helper function to toggle y_cable simulator ports
        Args:
            interface_name: a str, the name of interface
            target: "upper_tor" or "lower_tor"
        """
        server_url = url(interface_name)
        data = {"active_side": UPPER_TOR}
>       pytest_assert(_post(server_url, data), "Failed to toggle to upper_tor on interface {}".format(interface_name))
E       Failed: Failed to toggle to upper_tor on interface Ethernet60

data       = {'active_side': 'upper_tor'}
interface_name = 'Ethernet60'
server_url = 'http://10.64.246.96:8080/mux/vms17-7/15'
url        = <function _url at 0x7fcfaad8d350>

common/dualtor/mux_simulator_control.py:183: Failed

During teardown, the test also experiences a failure since the majority of the test body never ran

set_drop = <function _set_drop at 0x7fcf99ae5a50>
set_output = <function _set_output at 0x7fcf99ae5d50>

    @pytest.fixture
    def simulator_server_down(set_drop, set_output):
        """
        A fixture to set drop on a given mux cable
        """
        tmp_list = []
        def _drop_helper(interface_name):
            tmp_list.append(interface_name)
            set_drop(interface_name, [UPPER_TOR, LOWER_TOR])
    
        yield _drop_helper
>       set_output(tmp_list[0], [UPPER_TOR, LOWER_TOR])
E       IndexError: list index out of range

_drop_helper = <function _drop_helper at 0x7fcf99ae5f50>
set_drop   = <function _set_drop at 0x7fcf99ae5a50>
set_output = <function _set_output at 0x7fcf99ae5d50>
tmp_list   = []

common/dualtor/mux_simulator_control.py:341: IndexError