[vs tests] Activate flaky plugin for virtual switch pytests (#1186)

Signed-off-by: Danny Allen <daall@microsoft.com>

tests/README.md

@@ -8,8 +8,9 @@ SWSS Integration tests runs on docker-sonic-vs which runs on top of SAI virtual
 
 - Install docker and pytest on your dev machine
     ```
-    sudo pip install --system docker==3.5.0
-    sudo pip install --system pytest==3.3.0 docker redis
+    sudo pip install docker==3.5.0
+    sudo pip install pytest==3.3.0
+    sudo pip install flaky docker redis
     ```
 - Compile and install swss common library. Follow instructions [here](https://github.com/Azure/sonic-swss-common/blob/master/README.md) to first install prerequisites to build swss common library. 
     ```

tests/test_acl.py

@@ -1,7 +1,10 @@
-from swsscommon import swsscommon
 import time
 import re
 import json
+import pytest
+
+from swsscommon import swsscommon
+from flaky import flaky
 
 
 class BaseTestAcl(object):
@@ -185,6 +188,7 @@ def verify_acl_rule(self, dvs, field, value):
                 assert False
 
 
+@pytest.mark.flaky
 class TestAcl(BaseTestAcl):
     def test_AclTableCreation(self, dvs, testlog):
         self.setup_db(dvs)
@@ -1370,6 +1374,8 @@ def test_AclRuleRedirectToNexthop(self, dvs, testlog):
         # bring down interface
         dvs.set_interface_status("Ethernet4", "down")
 
+
+@pytest.mark.flaky
 class TestAclRuleValidation(BaseTestAcl):
     """ Test class for cases that check if orchagent corectly validates
     ACL rules input

tests/test_acl_ctrl.py

@@ -1,7 +1,11 @@
+import time
+import pytest
+
 from swsscommon import swsscommon
+from flaky import flaky
 
-import time
 
+@pytest.mark.flaky
 class TestPortChannelAcl(object):
     def setup_db(self, dvs):
         self.pdb = swsscommon.DBConnector(0, dvs.redis_sock, 0)

tests/test_acl_egress_table.py

@@ -1,7 +1,11 @@
+import time
+import pytest
+
 from swsscommon import swsscommon
+from flaky import flaky
 
-import time
 
+@pytest.mark.flaky
 class TestEgressAclTable(object):
     def setup_db(self, dvs):
         self.pdb = swsscommon.DBConnector(0, dvs.redis_sock, 0)

tests/test_acl_portchannel.py

@@ -1,7 +1,11 @@
+import time
+import pytest
+
 from swsscommon import swsscommon
+from flaky import flaky
 
-import time
 
+@pytest.mark.flaky
 class TestPortChannelAcl(object):
     def setup_db(self, dvs):
         self.pdb = swsscommon.DBConnector(0, dvs.redis_sock, 0)

tests/test_admin_status.py

@@ -1,6 +1,11 @@
-from swsscommon import swsscommon
 import time
+import pytest
+
+from swsscommon import swsscommon
+from flaky import flaky
+
 
+@pytest.mark.flaky
 class TestAdminStatus(object):
     def setup_db(self, dvs):
         self.pdb = swsscommon.DBConnector(0, dvs.redis_sock, 0)

tests/test_crm.py

@@ -1,10 +1,12 @@
-from swsscommon import swsscommon
 import os
 import re
 import time
 import json
 import redis
+import pytest
 
+from swsscommon import swsscommon
+from flaky import flaky
 
 def getCrmCounterValue(dvs, key, counter):
 
@@ -62,6 +64,8 @@ def check_syslog(dvs, marker, err_log, expected_cnt):
     (exitcode, num) = dvs.runcmd(['sh', '-c', "awk \'/%s/,ENDFILE {print;}\' /var/log/syslog | grep \"%s\" | wc -l" % (marker, err_log)])
     assert num.strip() >= str(expected_cnt)
 
+
+@pytest.mark.flaky
 class TestCrm(object):
     def test_CrmFdbEntry(self, dvs, testlog):
 

tests/test_dirbcast.py

@@ -1,8 +1,13 @@
-from swsscommon import swsscommon
 import time
 import re
 import json
+import pytest
+
+from swsscommon import swsscommon
+from flaky import flaky
+
 
+@pytest.mark.flaky
 class TestDirectedBroadcast(object):
     def test_DirectedBroadcast(self, dvs, testlog):
 

tests/test_drop_counters.py

@@ -1,6 +1,8 @@
-from swsscommon import swsscommon
-
 import time
+import pytest
+
+from swsscommon import swsscommon
+from flaky import flaky
 
 # Supported drop counters
 PORT_INGRESS_DROPS = 'PORT_INGRESS_DROPS'
@@ -53,9 +55,11 @@
 EXPECTED_ASIC_FIELDS = [ASIC_COUNTER_TYPE_FIELD, ASIC_COUNTER_INGRESS_REASON_LIST_FIELD, ASIC_COUNTER_EGRESS_REASON_LIST_FIELD]
 EXPECTED_NUM_ASIC_FIELDS = 2
 
+
 # FIXME: It is really annoying to have to re-run tests due to inconsistent timing, should
 # implement some sort of polling interface for checking ASIC/flex counter tables after
 # applying changes to config DB
+@pytest.mark.flaky
 class TestDropCounters(object):
     def setup_db(self, dvs):
         self.asic_db = swsscommon.DBConnector(1, dvs.redis_sock, 0)

tests/test_dtel.py

@@ -1,16 +1,19 @@
-from swsscommon import swsscommon
-
-
 import time
 import re
 import json
+import pytest
+
+from swsscommon import swsscommon
+from flaky import flaky
 
+
+@pytest.mark.flaky
 class TestDtel(object):
     def test_DtelGlobalAttribs(self, dvs, testlog):
-    
+
         db = swsscommon.DBConnector(4, dvs.redis_sock, 0)
         adb = swsscommon.DBConnector(1, dvs.redis_sock, 0)
-    
+
         # create DTel global attributes in config db
         tbl = swsscommon.Table(db, "DTEL")
 

tests/test_fdb.py

@@ -1,8 +1,11 @@
-from swsscommon import swsscommon
 import os
 import sys
 import time
 import json
+import pytest
+
+from swsscommon import swsscommon
+from flaky import flaky
 from distutils.version import StrictVersion
 
 def create_entry(tbl, key, pairs):
@@ -24,6 +27,8 @@ def how_many_entries_exist(db, table):
     tbl =  swsscommon.Table(db, table)
     return len(tbl.getKeys())
 
+
+@pytest.mark.flaky
 class TestFdb(object):
     def test_FdbWarmRestartNotifications(self, dvs, testlog):
         dvs.setup_db()

tests/test_fdb_update.py

@@ -1,11 +1,15 @@
-from swsscommon import swsscommon
 import os
 import sys
 import time
 import json
+import pytest
+
+from swsscommon import swsscommon
+from flaky import flaky
 from distutils.version import StrictVersion
 
 
+@pytest.mark.flaky
 class TestFdbUpdate(object):
     def create_entry(self, tbl, key, pairs):
         fvs = swsscommon.FieldValuePairs(pairs)

tests/test_interface.py

@@ -1,8 +1,12 @@
-from swsscommon import swsscommon
-
 import time
 import json
+import pytest
+
+from swsscommon import swsscommon
+from flaky import flaky
+
 
+@pytest.mark.flaky
 class TestRouterInterface(object):
     def setup_db(self, dvs):
         self.pdb = swsscommon.DBConnector(0, dvs.redis_sock, 0)

tests/test_mirror.py

@@ -1,13 +1,14 @@
 # This test suite covers the functionality of mirror feature in SwSS
-
 import platform
 import pytest
 import time
-from distutils.version import StrictVersion
 
 from swsscommon import swsscommon
+from flaky import flaky
+from distutils.version import StrictVersion
 
 
+@pytest.mark.flaky
 class TestMirror(object):
     def setup_db(self, dvs):
         self.pdb = swsscommon.DBConnector(0, dvs.redis_sock, 0)

tests/test_mirror_ipv6_combined.py

@@ -1,15 +1,16 @@
 # This test suite covers the functionality of mirror feature in SwSS
-
 import platform
 import pytest
 import time
-from distutils.version import StrictVersion
 
 from swsscommon import swsscommon
+from flaky import flaky
+from distutils.version import StrictVersion
 
 DVS_FAKE_PLATFORM = "broadcom"
 
 
+@pytest.mark.flaky
 class TestMirror(object):
     def setup_db(self, dvs):
         self.pdb = swsscommon.DBConnector(0, dvs.redis_sock, 0)

tests/test_mirror_ipv6_separate.py

@@ -1,15 +1,16 @@
 # This test suite covers the functionality of mirror feature in SwSS
-
 import platform
 import pytest
 import time
 from distutils.version import StrictVersion
 
 from swsscommon import swsscommon
+from flaky import flaky
 
 DVS_FAKE_PLATFORM = "mellanox"
 
 
+@pytest.mark.flaky
 class TestMirror(object):
     def setup_db(self, dvs):
         self.pdb = swsscommon.DBConnector(0, dvs.redis_sock, 0)

tests/test_mirror_policer.py

@@ -1,13 +1,14 @@
 # This test suite covers the functionality of mirror feature in SwSS
-
 import platform
 import pytest
 import time
-from distutils.version import StrictVersion
 
 from swsscommon import swsscommon
+from flaky import flaky
+from distutils.version import StrictVersion
 
 
+@pytest.mark.flaky
 class TestMirror(object):
     def setup_db(self, dvs):
         self.pdb = swsscommon.DBConnector(0, dvs.redis_sock, 0)

tests/test_nat.py

@@ -1,11 +1,14 @@
-from swsscommon import swsscommon
 import time
 import re
 import json
 import pytest
 import pdb
 import os
 
+from swsscommon import swsscommon
+from flaky import flaky
+
+
 # FIXME: These tests depend on changes in sonic-buildimage, we need to reenable
 # them once those changes are merged.
 @pytest.mark.skip(reason="Depends on changes in sonic-buildimage")

tests/test_neighbor.py

@@ -1,8 +1,12 @@
-from swsscommon import swsscommon
-
 import time
 import json
+import pytest
+
+from swsscommon import swsscommon
+from flaky import flaky
+
 
+@pytest.mark.flaky
 class TestNeighbor(object):
     def setup_db(self, dvs):
         self.pdb = swsscommon.DBConnector(0, dvs.redis_sock, 0)

tests/test_nhg.py

@@ -1,9 +1,14 @@
-from swsscommon import swsscommon
 import os
 import re
 import time
 import json
+import pytest
+
+from swsscommon import swsscommon
+from flaky import flaky
+
 
+@pytest.mark.flaky
 class TestNextHopGroup(object):
     def test_route_nhg(self, dvs, testlog):
         config_db = swsscommon.DBConnector(swsscommon.CONFIG_DB, dvs.redis_sock, 0)

tests/test_pfc.py

@@ -1,6 +1,8 @@
 import time
-from swsscommon import swsscommon
+import pytest
 
+from swsscommon import swsscommon
+from flaky import flaky
 
 def getBitMaskStr(bits):
 
@@ -57,6 +59,8 @@ def getPortAttr(dvs, port_oid, port_attr):
 
     return ''
 
+
+@pytest.mark.flaky
 class TestPfc(object):
     def test_PfcAsymmetric(self, dvs, testlog):
 

tests/test_policer.py

@@ -3,7 +3,10 @@
 import time
 
 from swsscommon import swsscommon
+from flaky import flaky
 
+
+@pytest.mark.flaky
 class TestPolicer(object):
     def test_PolicerBasic(self, dvs, testlog):
         dvs.setup_db()

tests/test_port.py

@@ -1,8 +1,12 @@
-from swsscommon import swsscommon
-
 import time
 import os
+import pytest
+
+from swsscommon import swsscommon
+from flaky import flaky
+
 
+@pytest.mark.flaky
 class TestPort(object):
     def test_PortMtu(self, dvs, testlog):
         pdb = swsscommon.DBConnector(0, dvs.redis_sock, 0)