Fix test_warm_reboot issues blocking PR merge (#2309)

* Two fixes: sleep after stop and check values in routes

tests/conftest.py

@@ -658,6 +658,7 @@ def stop_swss(self):
         for pname in self.swssd:
             cmd += "supervisorctl stop {}; ".format(pname)
         self.runcmd(['sh', '-c', cmd])
+        time.sleep(5)
 
     # deps: warm_reboot
     def start_zebra(self):
@@ -669,7 +670,7 @@ def start_zebra(self):
     # deps: warm_reboot
     def stop_zebra(self):
         self.runcmd(['sh', '-c', 'pkill -9 zebra'])
-        time.sleep(1)
+        time.sleep(5)
 
     # deps: warm_reboot
     def start_fpmsyncd(self):

tests/test_warm_reboot.py

@@ -118,12 +118,14 @@ def how_many_entries_exist(db, table):
 
 def stop_neighsyncd(dvs):
     dvs.runcmd(['sh', '-c', 'pkill -x neighsyncd'])
+    time.sleep(1)
 
 def start_neighsyncd(dvs):
     dvs.runcmd(['sh', '-c', 'supervisorctl start neighsyncd'])
 
 def stop_restore_neighbors(dvs):
     dvs.runcmd(['sh', '-c', 'pkill -x restore_neighbors'])
+    time.sleep(1)
 
 def start_restore_neighbors(dvs):
     dvs.runcmd(['sh', '-c', 'supervisorctl start restore_neighbors'])
@@ -307,6 +309,7 @@ def test_PortSyncdWarmRestart(self, dvs, testlog):
 
         # restart portsyncd
         dvs.runcmd(['sh', '-c', 'pkill -x portsyncd'])
+        time.sleep(1)
 
         pubsub = dvs.SubscribeAsicDbObject("SAI_OBJECT_TYPE")
         dvs.runcmd(['sh', '-c', 'supervisorctl start portsyncd'])
@@ -343,7 +346,6 @@ def test_PortSyncdWarmRestart(self, dvs, testlog):
         intf_tbl._del("Ethernet20")
         time.sleep(2)
 
-
     def test_VlanMgrdWarmRestart(self, dvs, testlog):
 
         conf_db = swsscommon.DBConnector(swsscommon.CONFIG_DB, dvs.redis_sock, 0)
@@ -427,6 +429,7 @@ def test_VlanMgrdWarmRestart(self, dvs, testlog):
         restore_count = swss_get_RestoreCount(dvs, state_db)
 
         dvs.runcmd(['sh', '-c', 'pkill -x vlanmgrd'])
+        time.sleep(1)
 
         pubsub = dvs.SubscribeAsicDbObject("SAI_OBJECT_TYPE")
 
@@ -1075,7 +1078,6 @@ def test_swss_port_state_syncup(self, dvs, testlog):
     #
     ################################################################################
 
-
     def test_routing_WarmRestart(self, dvs, testlog):
 
         appl_db = swsscommon.DBConnector(swsscommon.APPL_DB, dvs.redis_sock, 0)
@@ -1261,7 +1263,8 @@ def test_routing_WarmRestart(self, dvs, testlog):
         rt_key = json.loads(addobjs[0]['key'])
         rt_val = json.loads(addobjs[0]['vals'])
         assert rt_key == "192.168.100.0/24"
-        assert rt_val == {"ifname": "Ethernet0", "nexthop": "111.0.0.2"}
+        assert rt_val.get("ifname") == "Ethernet0"
+        assert rt_val.get("nexthop") == "111.0.0.2"
 
         # Verify the changed prefix is seen in sairedis
         (addobjs, delobjs) = dvs.GetSubscribedAsicDbObjects(pubsubAsicDB)
@@ -1333,7 +1336,8 @@ def test_routing_WarmRestart(self, dvs, testlog):
         rt_key = json.loads(addobjs[0]['key'])
         rt_val = json.loads(addobjs[0]['vals'])
         assert rt_key == "192.168.200.0/24"
-        assert rt_val == {"ifname": "Ethernet0,Ethernet4,Ethernet8", "nexthop": "111.0.0.2,122.0.0.2,133.0.0.2"}
+        assert rt_val.get("ifname") == "Ethernet0,Ethernet4,Ethernet8"
+        assert rt_val.get("nexthop") == "111.0.0.2,122.0.0.2,133.0.0.2"
 
         # Verify the changed prefix is seen in sairedis
         (addobjs, delobjs) = dvs.GetSubscribedAsicDbObjects(pubsubAsicDB)
@@ -1406,7 +1410,8 @@ def test_routing_WarmRestart(self, dvs, testlog):
         rt_key = json.loads(addobjs[0]['key'])
         rt_val = json.loads(addobjs[0]['vals'])
         assert rt_key == "192.168.1.3"
-        assert rt_val == {"ifname": "Ethernet0,Ethernet4,Ethernet8", "nexthop": "111.0.0.2,122.0.0.2,133.0.0.2"}
+        assert rt_val.get("ifname") == "Ethernet0,Ethernet4,Ethernet8"
+        assert rt_val.get("nexthop") == "111.0.0.2,122.0.0.2,133.0.0.2"
 
         # Verify the changed prefix is seen in sairedis
         (addobjs, delobjs) = dvs.GetSubscribedAsicDbObjects(pubsubAsicDB)
@@ -1444,7 +1449,8 @@ def test_routing_WarmRestart(self, dvs, testlog):
         rt_key = json.loads(addobjs[0]['key'])
         rt_val = json.loads(addobjs[0]['vals'])
         assert rt_key == "192.168.1.3"
-        assert rt_val == {"ifname": "Ethernet0,Ethernet4", "nexthop": "111.0.0.2,122.0.0.2"}
+        assert rt_val.get("ifname") == "Ethernet0,Ethernet4"
+        assert rt_val.get("nexthop") == "111.0.0.2,122.0.0.2"
 
         # Verify the changed prefix is seen in sairedis
         (addobjs, delobjs) = dvs.GetSubscribedAsicDbObjects(pubsubAsicDB)
@@ -1481,7 +1487,8 @@ def test_routing_WarmRestart(self, dvs, testlog):
         rt_key = json.loads(addobjs[0]['key'])
         rt_val = json.loads(addobjs[0]['vals'])
         assert rt_key == "fc00:4:4::1"
-        assert rt_val == {"ifname": "Ethernet0", "nexthop": "1110::2"}
+        assert rt_val.get("ifname") == "Ethernet0"
+        assert rt_val.get("nexthop") == "1110::2"
 
         # Verify the changed prefix is seen in sairedis
         (addobjs, delobjs) = dvs.GetSubscribedAsicDbObjects(pubsubAsicDB)
@@ -1579,7 +1586,8 @@ def test_routing_WarmRestart(self, dvs, testlog):
         rt_key = json.loads(addobjs[0]['key'])
         rt_val = json.loads(addobjs[0]['vals'])
         assert rt_key == "192.168.100.0/24"
-        assert rt_val == {"ifname": "Ethernet0", "nexthop": "111.0.0.2"}
+        assert rt_val.get("ifname") == "Ethernet0"
+        assert rt_val.get("nexthop") == "111.0.0.2"
 
         # Verify the changed prefix is seen in sairedis
         (addobjs, delobjs) = dvs.GetSubscribedAsicDbObjects(pubsubAsicDB)
@@ -1691,7 +1699,8 @@ def test_routing_WarmRestart(self, dvs, testlog):
         rt_key = json.loads(addobjs[0]['key'])
         rt_val = json.loads(addobjs[0]['vals'])
         assert rt_key == "192.168.100.0/24"
-        assert rt_val == {"ifname": "Ethernet4", "nexthop": "122.0.0.2"}
+        assert rt_val.get("ifname") == "Ethernet4"
+        assert rt_val.get("nexthop") == "122.0.0.2"
 
         # Verify the changed prefix is seen in sairedis
         (addobjs, delobjs) = dvs.GetSubscribedAsicDbObjects(pubsubAsicDB)
@@ -2172,6 +2181,7 @@ def test_VrfMgrdWarmRestart(self, dvs, testlog):
         (exitcode, vrf_before) = dvs.runcmd(['sh', '-c', "ip link show | grep Vrf"])
 
         dvs.runcmd(['sh', '-c', 'pkill -x vrfmgrd'])
+        time.sleep(1)
 
         pubsub = dvs.SubscribeAsicDbObject("SAI_OBJECT_TYPE")