check pod readiness in ingress scale test (#5774)

tests/conftest.py

@@ -16,7 +16,7 @@
     NS_COUNT,
     NUM_REPLICAS,
 )
-from suite.utils.resources_utils import get_first_pod_name
+from suite.utils.resources_utils import are_all_pods_in_ready_state, get_first_pod_name, wait_before_test
 
 
 def pytest_addoption(parser) -> None:
@@ -200,6 +200,10 @@ def pytest_runtest_makereport(item) -> None:
         pod_namespace = item.funcargs["ingress_controller_prerequisites"].namespace
         pod_name = get_first_pod_name(item.funcargs["kube_apis"].v1, pod_namespace)
         print("\n===================== IC Logs Start =====================")
+        count = 0
+        while (not are_all_pods_in_ready_state(item.funcargs["kube_apis"].v1, pod_namespace)) and count < 10:
+            count += 1
+            wait_before_test()
         print(item.funcargs["kube_apis"].v1.read_namespaced_pod_log(pod_name, pod_namespace))
         print("\n===================== IC Logs End =====================")
 

tests/suite/test_rl_ingress.py

@@ -8,6 +8,7 @@
 from suite.fixtures.fixtures import PublicEndpoint
 from suite.utils.custom_assertions import assert_event_count_increased
 from suite.utils.resources_utils import (
+    are_all_pods_in_ready_state,
     create_example_app,
     create_items_from_yaml,
     delete_common_app,
@@ -129,6 +130,11 @@ def test_ingress_rate_limit_sscaled(
         """
         ns = ingress_controller_prerequisites.namespace
         scale_deployment(kube_apis.v1, kube_apis.apps_v1_api, "nginx-ingress", ns, 4)
+        count = 0
+        while (not are_all_pods_in_ready_state(kube_apis.v1, ns)) and count < 10:
+            count += 1
+            wait_before_test()
+
         ic_pods = get_pod_list(kube_apis.v1, ns)
         for i in range(len(ic_pods)):
             conf = get_ingress_nginx_template_conf(
@@ -140,3 +146,4 @@ def test_ingress_rate_limit_sscaled(
             )
             flag = ("rate=10r/s" in conf) or ("rate=13r/s" in conf)
             assert flag
+        scale_deployment(kube_apis.v1, kube_apis.apps_v1_api, "nginx-ingress", ns, 1)