Making loganalyzer fixture multi-dut aware (#2809)

What is the motivation for this PR?
In a multi-dut testbed, the existing loganalyzer plugin takes 'rand_one_dut_hostname'
and thus analyzes only a single DUT.

However, when testing multiple DUTs (instead of rand-one-dut-hostname), the log analyzer
would run on one randomly chosen DUT, while the test is repeated on multiple DUTs.

Moreover, in a multi-dut testbet for a T2 chassis, we need to analyze the logs on all the DUTS,
where affects of action on one linecard (one DUT in the testbed) on another linecard
(another DUT in the testbed) could be ignored.

How did you do it?
This PR makes the logAnalyzer run on all the DUTs in a multi-dut testbed.
Even if the test is parameterized with a hostname of one of the DUTS in duthosts,
we would analyze all the DUTs in duthosts (all DUTs in the testbed).

Changed the return value of loganalyzer fixture to be a dictionary keyed
with the hostname of the DUTs, and the value being the analyzer object.

Modified tests that are using loganalyzer (to add more exceptions etc.) to use
the new dictionary structure. If the test is running on a single DUT, then
we modify the loganalyzer object associated with just that DUT.

How did you verify/test it?
Ran test_features (enumerated using enum_dut_hostname) and validated that the analyzer is run on all the DUTs as the one that the test is ran.

tests/autorestart/test_container_autorestart.py

@@ -28,7 +28,7 @@ def config_reload_after_tests(duthost):
     config_reload(duthost)
 
 @pytest.fixture(autouse=True)
-def ignore_expected_loganalyzer_exception(loganalyzer, enum_dut_feature):
+def ignore_expected_loganalyzer_exception(duthost, loganalyzer, enum_dut_feature):
     """
         Ignore expected failure/error messages during testing the autorestart feature.
 
@@ -95,9 +95,9 @@ def ignore_expected_loganalyzer_exception(loganalyzer, enum_dut_feature):
     _, feature = decode_dut_port_name(enum_dut_feature)
 
     if loganalyzer:
-        loganalyzer.ignore_regex.extend(ignore_regex_dict['common'])
+        loganalyzer[duthost.hostname].ignore_regex.extend(ignore_regex_dict['common'])
         if feature in ignore_regex_dict:
-            loganalyzer.ignore_regex.extend(ignore_regex_dict[feature])
+            loganalyzer[duthost.hostname].ignore_regex.extend(ignore_regex_dict[feature])
 
 
 def get_group_program_info(duthost, container_name, group_name):

tests/common/plugins/loganalyzer/__init__.py

@@ -3,6 +3,7 @@
 
 from loganalyzer import LogAnalyzer
 from tests.common.errors import RunAnsibleModuleFail
+import re
 
 
 def pytest_addoption(parser):
@@ -11,33 +12,40 @@ def pytest_addoption(parser):
 
 
 @pytest.fixture(autouse=True)
-def loganalyzer(duthosts, rand_one_dut_hostname, request):
-    duthost = duthosts[rand_one_dut_hostname]
+def loganalyzer(duthosts, request):
     if request.config.getoption("--disable_loganalyzer") or "disable_loganalyzer" in request.keywords:
         logging.info("Log analyzer is disabled")
         yield
         return
 
-    # Force rotate logs
-    try:
-        duthost.shell(
-            "/usr/sbin/logrotate -f /etc/logrotate.conf > /dev/null 2>&1"
-            )
-    except RunAnsibleModuleFail as e:
-        logging.warning("logrotate is failed. Command returned:\n"
-                        "Stdout: {}\n"
-                        "Stderr: {}\n"
-                        "Return code: {}".format(e.results["stdout"], e.results["stderr"], e.results["rc"]))
-
-    loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix=request.node.name)
-    logging.info("Add start marker into DUT syslog")
-    marker = loganalyzer.init()
-    logging.info("Load config and analyze log")
-    # Read existed common regular expressions located with legacy loganalyzer module
-    loganalyzer.load_common_config()
-
-    yield loganalyzer
+    analyzers = {}
+    markers = {}
+    # Analyze all the duts
+    for duthost in duthosts:
+        # Force rotate logs
+        try:
+            duthost.shell(
+                "/usr/sbin/logrotate -f /etc/logrotate.conf > /dev/null 2>&1"
+                )
+        except RunAnsibleModuleFail as e:
+            logging.warning("logrotate is failed. Command returned:\n"
+                            "Stdout: {}\n"
+                            "Stderr: {}\n"
+                            "Return code: {}".format(e.results["stdout"], e.results["stderr"], e.results["rc"]))
+
+        loganalyzer = LogAnalyzer(ansible_host=duthost, marker_prefix=request.node.name)
+        logging.info("Add start marker into DUT syslog")
+        marker = loganalyzer.init()
+        logging.info("Load config and analyze log")
+        # Read existed common regular expressions located with legacy loganalyzer module
+        loganalyzer.load_common_config()
+        analyzers[duthost.hostname] = loganalyzer
+        markers[duthost.hostname] = marker
+
+    yield analyzers
+
     # Skip LogAnalyzer if case is skipped
     if "rep_call" in request.node.__dict__ and request.node.rep_call.skipped:
         return
-    loganalyzer.analyze(marker)
+    for dut_hostname, dut_analyzer in analyzers.items():
+        dut_analyzer.analyze(markers[dut_hostname])

tests/copp/test_copp.py

@@ -131,7 +131,7 @@ def copp_testbed(
         _teardown_testbed(duthost, creds, ptfhost, test_params)
 
 @pytest.fixture(autouse=True)
-def ignore_expected_loganalyzer_exceptions(loganalyzer):
+def ignore_expected_loganalyzer_exceptions(rand_one_dut_hostname, loganalyzer):
     """
         Ignore expected failures logs during test execution.
 
@@ -152,7 +152,7 @@ def ignore_expected_loganalyzer_exceptions(loganalyzer):
     ]
 
     if loganalyzer:  # Skip if loganalyzer is disabled
-        loganalyzer.ignore_regex.extend(ignoreRegex)
+        loganalyzer[rand_one_dut_hostname].ignore_regex.extend(ignoreRegex)
 
 def _copp_runner(dut, ptf, protocol, test_params, dut_type):
     """

tests/dhcp_relay/test_dhcp_relay.py

@@ -16,13 +16,13 @@
 DEFAULT_DHCP_CLIENT_PORT = 68
 
 @pytest.fixture(autouse=True)
-def ignore_expected_loganalyzer_exceptions(loganalyzer):
+def ignore_expected_loganalyzer_exceptions(rand_one_dut_hostname, loganalyzer):
     """Ignore expected failures logs during test execution."""
     if loganalyzer:
         ignoreRegex = [
             ".*ERR snmp#snmp-subagent.*",
         ]
-        loganalyzer.ignore_regex.extend(ignoreRegex)
+        loganalyzer[rand_one_dut_hostname].ignore_regex.extend(ignoreRegex)
 
     yield
 

tests/drop_packets/drop_packets.py

@@ -170,16 +170,16 @@ def rif_port_down(duthosts, rand_one_dut_hostname, setup, fanouthosts, loganalyz
 
     fanout_neighbor, fanout_intf = fanout_switch_port_lookup(fanouthosts, duthost.hostname, rif_member_iface)
 
-    loganalyzer.expect_regex = [LOG_EXPECT_PORT_OPER_DOWN_RE.format(rif_member_iface)]
-    with loganalyzer as _:
+    loganalyzer[rand_one_dut_hostname].expect_regex = [LOG_EXPECT_PORT_OPER_DOWN_RE.format(rif_member_iface)]
+    with loganalyzer[rand_one_dut_hostname] as _:
         fanout_neighbor.shutdown(fanout_intf)
 
     time.sleep(1)
 
     yield ip_dst
 
-    loganalyzer.expect_regex = [LOG_EXPECT_PORT_OPER_UP_RE.format(rif_member_iface)]
-    with loganalyzer as _:
+    loganalyzer[rand_one_dut_hostname].expect_regex = [LOG_EXPECT_PORT_OPER_UP_RE.format(rif_member_iface)]
+    with loganalyzer[rand_one_dut_hostname] as _:
         fanout_neighbor.no_shutdown(fanout_intf)
         time.sleep(wait_after_ports_up)
 

tests/drop_packets/test_drop_counters.py

@@ -74,14 +74,14 @@ def acl_setup(duthosts, rand_one_dut_hostname, loganalyzer):
 
     logger.info("Applying {}".format(dut_conf_file_path))
 
-    loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_CREATE_RE]
-    with loganalyzer as analyzer:
+    loganalyzer[rand_one_dut_hostname].expect_regex = [LOG_EXPECT_ACL_RULE_CREATE_RE]
+    with loganalyzer[rand_one_dut_hostname] as analyzer:
         duthost.command("config acl update full {}".format(dut_conf_file_path))
 
     yield
 
-    loganalyzer.expect_regex = [LOG_EXPECT_ACL_RULE_REMOVE_RE]
-    with loganalyzer as analyzer:
+    loganalyzer[rand_one_dut_hostname].expect_regex = [LOG_EXPECT_ACL_RULE_REMOVE_RE]
+    with loganalyzer[rand_one_dut_hostname] as analyzer:
         logger.info("Applying {}".format(dut_clear_conf_file_path))
         duthost.command("config acl update full {}".format(dut_clear_conf_file_path))
         logger.info("Removing {}".format(dut_tmp_dir))

tests/k8s/conftest.py

@@ -19,7 +19,7 @@ def check_k8s_vms(k8scluster):
 
 
 @pytest.fixture(autouse=True)
-def ignore_expected_loganalyzer_exceptions(loganalyzer):
+def ignore_expected_loganalyzer_exceptions(duthost, loganalyzer):
     """
        Ignore expected failures logs during test execution
 
@@ -35,5 +35,5 @@ def ignore_expected_loganalyzer_exceptions(loganalyzer):
              ".*for troubleshooting tips.*",
              ".*kubeproxy.*",
          ]
-         loganalyzer.ignore_regex.extend(ignoreRegex)
+         loganalyzer[duthost.hostname].ignore_regex.extend(ignoreRegex)
     yield

tests/lldp/test_lldp.py

@@ -29,15 +29,15 @@ def test_lldp(duthosts, rand_one_dut_hostname, localhost, collect_techsupport):
         assert v['chassis']['name'] == config_facts['DEVICE_NEIGHBOR'][k]['name']
         # Compare the LLDP neighbor interface with minigraph neigbhor interface (exclude the management port)
         assert v['port']['ifname'] == config_facts['DEVICE_NEIGHBOR'][k]['port']
-   
+
 
 def test_lldp_neighbor(duthosts, rand_one_dut_hostname, localhost, eos,
                        collect_techsupport, loganalyzer):
     """ verify LLDP information on neighbors """
     duthost = duthosts[rand_one_dut_hostname]
 
     if loganalyzer:
-        loganalyzer.ignore_regex.extend([
+        loganalyzer[rand_one_dut_hostname].ignore_regex.extend([
             ".*ERR syncd#syncd: :- check_fdb_event_notification_data.*",
             ".*ERR syncd#syncd: :- process_on_fdb_event: invalid OIDs in fdb \
                 notifications, NOT translating and NOT storing in ASIC DB.*",

tests/pc/test_po_cleanup.py

@@ -9,7 +9,7 @@
 ]
 
 @pytest.fixture(autouse=True)
-def ignore_expected_loganalyzer_exceptions(loganalyzer):
+def ignore_expected_loganalyzer_exceptions(rand_one_dut_hostname, loganalyzer):
     """
         Ignore expected failures logs during test execution.
 
@@ -25,12 +25,12 @@ def ignore_expected_loganalyzer_exceptions(loganalyzer):
         ignoreRegex = [
             ".*",
         ]
-        loganalyzer.ignore_regex.extend(ignoreRegex)
+        loganalyzer[rand_one_dut_hostname].ignore_regex.extend(ignoreRegex)
         expectRegex = [
             ".*teamd#teammgrd: :- cleanTeamProcesses.*",
             ".*teamd#teamsyncd: :- cleanTeamSync.*"
         ]
-        loganalyzer.expect_regex.extend(expectRegex)
+        loganalyzer[rand_one_dut_hostname].expect_regex.extend(expectRegex)
 
 
 def check_kernel_po_interface_cleaned(duthost):

tests/pc/test_po_update.py

@@ -11,7 +11,7 @@
 ]
 
 @pytest.fixture(autouse=True)
-def ignore_expected_loganalyzer_exceptions(loganalyzer):
+def ignore_expected_loganalyzer_exceptions(rand_one_dut_hostname, loganalyzer):
     """
         Ignore expected failures logs during test execution.
 
@@ -29,7 +29,7 @@ def ignore_expected_loganalyzer_exceptions(loganalyzer):
             ".*ERR syncd#syncd: :- process_on_fdb_event: FDB notification was not sent since it contain invalid OIDs, bug.*",
             ".*ERR syncd#syncd: :- translate_vid_to_rid: unable to get RID for VID.*",
         ]
-        loganalyzer.ignore_regex.extend(ignoreRegex)
+        loganalyzer[rand_one_dut_hostname].ignore_regex.extend(ignoreRegex)
 
     yield
 

tests/pfcwd/test_pfcwd_timer_accuracy.py

@@ -39,7 +39,7 @@ def ignore_loganalyzer_exceptions(rand_one_dut_hostname, loganalyzer):
             ".*ERR syncd#syncd: :- process_on_fdb_event: invalid OIDs in fdb notifications, NOT translating and NOT storing in ASIC DB.*",
             ".*ERR syncd#syncd: :- process_on_fdb_event: FDB notification was not sent since it contain invalid OIDs, bug.*"
         ]
-        loganalyzer.ignore_regex.extend(ignoreRegex)
+        loganalyzer[rand_one_dut_hostname].ignore_regex.extend(ignoreRegex)
 
     yield
 

tests/qos/qos_sai_base.py

@@ -500,7 +500,7 @@ def updateDockerService(host, docker="", action="", service=""):
             updateDockerService(duthost, action="start", **service)
 
     @pytest.fixture(autouse=True)
-    def updateLoganalyzerExceptions(self, loganalyzer):
+    def updateLoganalyzerExceptions(self, rand_one_dut_hostname, loganalyzer):
         """
             Update loganalyzer ignore regex list
 
@@ -530,7 +530,7 @@ def updateLoganalyzerExceptions(self, loganalyzer):
                 ".*ERR monit.*'bgp\|bgpmon' status failed.*'/usr/bin/python.* /usr/local/bin/bgpmon' is not running.*",
                 ".*ERR monit.*bgp\|fpmsyncd.*status failed.*NoSuchProcess process no longer exists.*",
             ]
-            loganalyzer.ignore_regex.extend(ignoreRegex)
+            loganalyzer[rand_one_dut_hostname].ignore_regex.extend(ignoreRegex)
 
         yield
 

tests/route/test_route_perf.py

@@ -20,7 +20,7 @@
 ROUTE_TABLE_NAME = 'ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY'
 
 @pytest.fixture(autouse=True)
-def ignore_expected_loganalyzer_exceptions(loganalyzer):
+def ignore_expected_loganalyzer_exceptions(rand_one_dut_hostname, loganalyzer):
     """
         Ignore expected failures logs during test execution.
 
@@ -38,7 +38,7 @@ def ignore_expected_loganalyzer_exceptions(loganalyzer):
     ]
     if loganalyzer:
         # Skip if loganalyzer is disabled
-        loganalyzer.ignore_regex.extend(ignoreRegex)
+        loganalyzer[rand_one_dut_hostname].ignore_regex.extend(ignoreRegex)
 
 @pytest.fixture(params=[4, 6])
 def ip_versions(request):