[vstest]: stablize fgnhg test (#1491)

use standard wait API to wait for asic db and state db updates

Signed-off-by: Guohan Lu <lguohan@gmail.com>

tests/test_fgnhg.py

@@ -111,7 +111,7 @@ def _access_function():
  elif key == "SAI_NEXT_HOP_GROUP_MEMBER_ATTR_NEXT_HOP_GROUP_ID":
  if nhgid != val:
  return false_ret
- if (index == -1 or 
+ if (index == -1 or
  nh_oid == "0" or
  nh_oid_map.get(nh_oid,"NULL") == "NULL" or
  nh_oid_map.get(nh_oid) not in nh_memb_exp_count):
@@ -149,18 +149,18 @@ def verify_programmed_fg_state_db_entry(state_db,nh_memb_exp_count):
  memb_dict = nh_memb_exp_count
  keys = state_db.get_keys("FG_ROUTE_TABLE")
  assert len(keys) != 0
- for key in keys: 
+ for key in keys:
  fvs = state_db.get_entry("FG_ROUTE_TABLE", key)
  assert fvs != {}
  for key, value in fvs.items():
  assert value in nh_memb_exp_count
- memb_dict[value] = memb_dict[value] - 1 
+ memb_dict[value] = memb_dict[value] - 1
 
  for idx,memb in memb_dict.items():
- assert memb == 0 
+ assert memb == 0
 
 
-def validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map, 
+def validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
  nh_memb_exp_count, nh_oid_map, nhgid, bucket_size):
  state_db_entry_memb_exp_count = {}
 
@@ -227,11 +227,11 @@ def test_route_fgnhg(self, dvs, testlog):
 
  asic_routes_count = len(asic_db.get_keys(ASIC_ROUTE_TB))
  ps = swsscommon.ProducerStateTable(app_db.db_connection, ROUTE_TB)
- fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"), 
+ fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"),
  ("ifname", "Ethernet12,Ethernet16,Ethernet20")])
 
  ps.set(fg_nhg_prefix, fvs)
- # No ASIC_DB entry we can wait for since ARP is not resolved yet, 
+ # No ASIC_DB entry we can wait for since ARP is not resolved yet,
  # We just use sleep so that the sw receives this entry
  time.sleep(1)
 
@@ -261,7 +261,7 @@ def test_route_fgnhg(self, dvs, testlog):
 
  validate_asic_nhg(asic_db, nhgid, bucket_size)
 
- nh_oid_map = get_nh_oid_map(asic_db) 
+ nh_oid_map = get_nh_oid_map(asic_db)
 
  ### Test scenarios with bank 0 having 0 members up
  # ARP is not resolved for 10.0.0.7, so fg nhg should be created without 10.0.0.7
@@ -272,7 +272,7 @@ def test_route_fgnhg(self, dvs, testlog):
  # Resolve ARP for 10.0.0.7
  asic_nh_count = len(asic_db.get_keys("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP"))
  dvs.runcmd("arp -s 10.0.0.7 00:00:00:00:00:04")
- asic_db.wait_for_n_keys("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP", asic_nh_count + 1) 
+ asic_db.wait_for_n_keys("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP", asic_nh_count + 1)
  nh_oid_map = get_nh_oid_map(asic_db)
  # Now that ARP was resolved, 10.0.0.7 should be added as a valid fg nhg member
  nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
@@ -359,13 +359,13 @@ def test_route_fgnhg(self, dvs, testlog):
  nh_memb_exp_count = {"10.0.0.11":60}
  validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
  nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
- 
+
  # Bring down last link, there shouldn't be a crash or other bad orchagent state because of this
  shutdown_link(dvs, app_db, 5)
  # Nothing to check for in this case, sleep 1s for the shutdown to reach sw
  time.sleep(1)
 
- # bring all links up one by one
+ # bring all links up one by one
  startup_link(dvs, app_db, 3)
  startup_link(dvs, app_db, 4)
  startup_link(dvs, app_db, 5)
@@ -399,14 +399,12 @@ def test_route_fgnhg(self, dvs, testlog):
  # Remove route
  asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix)
  ps._del(fg_nhg_prefix)
- asic_db.wait_for_deleted_entry(ASIC_ROUTE_TB, asic_rt_key)
 
- keys = asic_db.get_keys(ASIC_NHG_MEMB)
- assert len(keys) == 0
+ # validate routes and nhg member in asic db, route entry in state db are removed
+ asic_db.wait_for_deleted_entry(ASIC_ROUTE_TB, asic_rt_key)
+ asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 0)
+ state_db.wait_for_n_keys("FG_ROUTE_TABLE", 0)
 
- keys = state_db.get_keys("FG_ROUTE_TABLE")
- assert len(keys) == 0
-
  remove_entry(config_db, "FG_NHG_PREFIX", fg_nhg_prefix)
  # Nothing we can wait for in terms of db entries, we sleep here
  # to give the sw enough time to delete the entry
@@ -415,17 +413,16 @@ def test_route_fgnhg(self, dvs, testlog):
  # Add an ECMP route, since we deleted the FG_NHG_PREFIX it should see
  # standard(non-Fine grained) ECMP behavior
  asic_routes_count = len(asic_db.get_keys(ASIC_ROUTE_TB))
- fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"), 
+ fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.7,10.0.0.9,10.0.0.11"),
  ("ifname", "Ethernet12,Ethernet16,Ethernet20")])
  ps.set(fg_nhg_prefix, fvs)
 
  keys = asic_db.wait_for_n_keys(ASIC_ROUTE_TB, asic_routes_count + 1)
  nhgid = asic_route_exists_and_is_nhg(asic_db, keys, fg_nhg_prefix)
  assert nhgid is not None
 
- keys = asic_db.get_keys(ASIC_NHG_MEMB)
- assert len(keys) == 3
-
+ asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 3)
+
  # add fgnhg prefix: The regular route should transition to fine grained ECMP
  fvs = {"FG_NHG": fg_nhg_name}
  create_entry(config_db, FG_NHG_PREFIX, fg_nhg_prefix, fvs)
@@ -435,10 +432,10 @@ def test_route_fgnhg(self, dvs, testlog):
  keys = asic_db.get_keys(ASIC_ROUTE_TB)
  nhgid = asic_route_exists_and_is_nhg(asic_db, keys, fg_nhg_prefix)
  validate_asic_nhg(asic_db, nhgid, bucket_size)
- 
+
  nh_oid_map = {}
- nh_oid_map = get_nh_oid_map(asic_db) 
- 
+ nh_oid_map = get_nh_oid_map(asic_db)
+
  nh_memb_exp_count = {"10.0.0.7":20,"10.0.0.9":20,"10.0.0.11":20}
  validate_fine_grained_asic_n_state_db_entries(asic_db, state_db, ip_to_if_map,
  nh_memb_exp_count, nh_oid_map, nhgid, bucket_size)
@@ -449,7 +446,7 @@ def test_route_fgnhg(self, dvs, testlog):
  # Validate regular ECMP
  asic_db.wait_for_n_keys(ASIC_NHG_MEMB, 3)
  asic_route_exists_and_is_nhg(asic_db, keys, fg_nhg_prefix)
- 
+
  # remove prefix entry
  asic_rt_key = get_asic_route_key(asic_db, fg_nhg_prefix)
  ps._del(fg_nhg_prefix)
@@ -458,7 +455,7 @@ def test_route_fgnhg(self, dvs, testlog):
 
  # Cleanup all FG, arp and interface
  remove_entry(config_db, "FG_NHG", fg_nhg_name)
- 
+
  for i in range(0,NUM_NHs):
  if_name_key = "Ethernet" + str(i*4)
  ip_pref_key = "Ethernet" + str(i*4) + "|10.0.0." + str(i*2) + "/31"
@@ -469,8 +466,8 @@ def test_route_fgnhg(self, dvs, testlog):
  remove_entry(config_db, "FG_NHG_MEMBER", "10.0.0." + str(1 + i*2))
 
 
- ### Create new set of entries with a greater number of FG members and 
- ### bigger bucket size such that the # of nhs are not divisible by 
+ ### Create new set of entries with a greater number of FG members and
+ ### bigger bucket size such that the # of nhs are not divisible by
  ### bucket size.
  fg_nhg_name = "new_fgnhg_v4"
  fg_nhg_prefix = "3.3.3.0/24"
@@ -503,10 +500,10 @@ def test_route_fgnhg(self, dvs, testlog):
  create_entry(config_db, FG_NHG_MEMBER, "10.0.0." + str(1 + i*2), fvs)
  ip_to_if_map["10.0.0." + str(1 + i*2)] = if_name_key
  dvs.runcmd("arp -s 10.0.0." + str(1 + i*2) + " 00:00:00:00:00:" + str(1 + i*2))
- 
+
  # Program the route
  asic_routes_count = len(asic_db.get_keys(ASIC_ROUTE_TB))
- fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.1,10.0.0.11"), 
+ fvs = swsscommon.FieldValuePairs([("nexthop","10.0.0.1,10.0.0.11"),
  ("ifname", "Ethernet0,Ethernet20")])
  ps.set(fg_nhg_prefix, fvs)
 
@@ -517,7 +514,7 @@ def test_route_fgnhg(self, dvs, testlog):
 
  validate_asic_nhg(asic_db, nhgid, bucket_size)
 
- nh_oid_map = get_nh_oid_map(asic_db) 
+ nh_oid_map = get_nh_oid_map(asic_db)
 
  # Test addition of route with 0 members in bank
  nh_memb_exp_count = {"10.0.0.1":64,"10.0.0.11":64}