add functional tests for rate-limit scaling (#5758)

tests/data/rate-limit/ingress/mergeable-scaled/annotations-rl-ingress.yaml

@@ -0,0 +1,34 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.org/mergeable-ingress-type: "master"
+  name: annotations-rl-ingress-master
+spec:
+  ingressClassName: nginx
+  rules:
+  - host: ingress-rl.example.com
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: annotations-rl-ingress-minion
+  annotations:
+    nginx.org/limit-req-rate: 40r/s
+    nginx.org/limit-req-key: ${binary_remote_addr}
+    nginx.org/limit-req-zone-size: 10M
+    nginx.org/mergeable-ingress-type: "minion"
+    nginx.org/limit-req-scale: "true"
+spec:
+  ingressClassName: nginx
+  rules:
+  - host: ingress-rl.example.com
+    http:
+      paths:
+      - path: /backend1
+        pathType: Prefix
+        backend:
+          service:
+            name: backend1-svc
+            port:
+              number: 80

tests/data/rate-limit/ingress/standard-scaled/annotations-rl-ingress.yaml

@@ -0,0 +1,29 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.org/limit-req-rate: 40r/s
+    nginx.org/limit-req-key: ${binary_remote_addr}
+    nginx.org/limit-req-zone-size: 10M
+    nginx.org/limit-req-scale: "true"
+  name: annotations-rl-ingress
+spec:
+  ingressClassName: nginx
+  rules:
+  - host: ingress-rl.example.com
+    http:
+      paths:
+      - path: /backend2
+        pathType: Prefix
+        backend:
+          service:
+            name: backend2-svc
+            port:
+              number: 80
+      - path: /backend1
+        pathType: Prefix
+        backend:
+          service:
+            name: backend1-svc
+            port:
+              number: 80

tests/data/rate-limit/policies/rate-limit-primary-scaled.yaml

@@ -0,0 +1,10 @@
+apiVersion: k8s.nginx.org/v1
+kind: Policy
+metadata:
+  name: rate-limit-primary-scaled
+spec:
+  rateLimit:
+    rate: 40r/s
+    key: ${binary_remote_addr}
+    zoneSize: 10M
+    scale: true

tests/data/rate-limit/route-subroute/virtual-server-route-pri-subroute-scaled.yaml

@@ -0,0 +1,22 @@
+apiVersion: k8s.nginx.org/v1
+kind: VirtualServerRoute
+metadata:
+  name: backends
+spec:
+  host: virtual-server-route.example.com
+  upstreams:
+  - name: backend1
+    service: backend1-svc
+    port: 80
+  - name: backend3
+    service: backend3-svc
+    port: 80
+  subroutes:
+  - path: "/backends/backend1"
+    policies:
+    - name: rate-limit-primary-scaled
+    action:
+      pass: backend1
+  - path: "/backends/backend3"
+    action:
+      pass: backend3

tests/data/rate-limit/spec/virtual-server-primary-scaled.yaml

@@ -0,0 +1,22 @@
+apiVersion: k8s.nginx.org/v1
+kind: VirtualServer
+metadata:
+  name: virtual-server
+spec:
+  host: virtual-server.example.com
+  policies:
+  - name: rate-limit-primary-scaled
+  upstreams:
+  - name: backend2
+    service: backend2-svc
+    port: 80
+  - name: backend1
+    service: backend1-svc
+    port: 80
+  routes:
+  - path: "/backend1"
+    action:
+      pass: backend1
+  - path: "/backend2"
+    action:
+      pass: backend2

tests/suite/test_rl_ingress.py

@@ -14,11 +14,10 @@
     delete_items_from_yaml,
     ensure_connection_to_public_endpoint,
     ensure_response_from_backend,
-    get_events,
     get_first_pod_name,
     get_ingress_nginx_template_conf,
-    replace_configmap_from_yaml,
-    replace_ingress,
+    get_pod_list,
+    scale_deployment,
     wait_before_test,
     wait_until_all_pods_are_ready,
 )
@@ -74,6 +73,7 @@ def annotations_setup(
 
     create_example_app(kube_apis, "simple", test_namespace)
     wait_until_all_pods_are_ready(kube_apis.v1, test_namespace)
+
     ensure_connection_to_public_endpoint(
         ingress_controller_endpoint.public_ip, ingress_controller_endpoint.port, ingress_controller_endpoint.port_ssl
     )
@@ -98,7 +98,6 @@ def fin():
     )
 
 
-@pytest.mark.ingresses
 @pytest.mark.annotations
 @pytest.mark.parametrize("annotations_setup", ["standard", "mergeable"], indirect=True)
 class TestRateLimitIngress:
@@ -117,3 +116,27 @@ def test_ingress_rate_limit(self, kube_apis, annotations_setup, ingress_controll
             )
             counter.append(resp.status_code)
         assert (counter.count(200)) <= 2 and (429 in counter)  # check for only 2 200s in the list
+
+
+@pytest.mark.annotations
+@pytest.mark.parametrize("annotations_setup", ["standard-scaled", "mergeable-scaled"], indirect=True)
+class TestRateLimitIngressScaled:
+    def test_ingress_rate_limit_sscaled(
+        self, kube_apis, annotations_setup, ingress_controller_prerequisites, test_namespace
+    ):
+        """
+        Test if rate-limit scaling works with standard and mergeable ingresses
+        """
+        ns = ingress_controller_prerequisites.namespace
+        scale_deployment(kube_apis.v1, kube_apis.apps_v1_api, "nginx-ingress", ns, 4)
+        ic_pods = get_pod_list(kube_apis.v1, ns)
+        for i in range(len(ic_pods)):
+            conf = get_ingress_nginx_template_conf(
+                kube_apis.v1,
+                annotations_setup.namespace,
+                annotations_setup.ingress_name,
+                ic_pods[i].metadata.name,
+                ingress_controller_prerequisites.namespace,
+            )
+            flag = ("rate=10r/s" in conf) or ("rate=13r/s" in conf)
+            assert flag

tests/suite/test_rl_policies.py

@@ -5,7 +5,13 @@
 from settings import TEST_DATA
 from suite.utils.custom_resources_utils import read_custom_resource
 from suite.utils.policy_resources_utils import create_policy_from_yaml, delete_policy
-from suite.utils.resources_utils import wait_before_test
+from suite.utils.resources_utils import (
+    get_first_pod_name,
+    get_pod_list,
+    get_vs_nginx_template_conf,
+    scale_deployment,
+    wait_before_test,
+)
 from suite.utils.vs_vsr_resources_utils import (
     create_virtual_server_from_yaml,
     delete_virtual_server,
@@ -14,7 +20,9 @@
 
 std_vs_src = f"{TEST_DATA}/rate-limit/standard/virtual-server.yaml"
 rl_pol_pri_src = f"{TEST_DATA}/rate-limit/policies/rate-limit-primary.yaml"
+rl_pol_pri_sca_src = f"{TEST_DATA}/rate-limit/policies/rate-limit-primary-scaled.yaml"
 rl_vs_pri_src = f"{TEST_DATA}/rate-limit/spec/virtual-server-primary.yaml"
+rl_vs_pri_sca_src = f"{TEST_DATA}/rate-limit/spec/virtual-server-primary-scaled.yaml"
 rl_pol_sec_src = f"{TEST_DATA}/rate-limit/policies/rate-limit-secondary.yaml"
 rl_vs_sec_src = f"{TEST_DATA}/rate-limit/spec/virtual-server-secondary.yaml"
 rl_pol_invalid = f"{TEST_DATA}/rate-limit/policies/rate-limit-invalid.yaml"
@@ -307,3 +315,51 @@ def test_rl_override_spec_route(
         delete_policy(kube_apis.custom_objects, pol_name_sec, test_namespace)
         self.restore_default_vs(kube_apis, virtual_server_setup)
         assert rate_sec >= occur.count(200) >= (rate_sec - 2)
+
+    @pytest.mark.parametrize("src", [rl_vs_pri_sca_src])
+    def test_rl_policy_scaled(
+        self,
+        kube_apis,
+        ingress_controller_prerequisites,
+        crd_ingress_controller,
+        virtual_server_setup,
+        test_namespace,
+        src,
+    ):
+        """
+        Test if rate-limit scaling is being calculated correctly
+        """
+        ns = ingress_controller_prerequisites.namespace
+        scale_deployment(kube_apis.v1, kube_apis.apps_v1_api, "nginx-ingress", ns, 4)
+
+        print(f"Create rl policy")
+        pol_name = create_policy_from_yaml(kube_apis.custom_objects, rl_pol_pri_sca_src, test_namespace)
+        print(f"Patch vs with policy: {src}")
+        patch_virtual_server_from_yaml(
+            kube_apis.custom_objects,
+            virtual_server_setup.vs_name,
+            src,
+            virtual_server_setup.namespace,
+        )
+        wait_before_test()
+
+        policy_info = read_custom_resource(kube_apis.custom_objects, test_namespace, "policies", pol_name)
+        ic_pods = get_pod_list(kube_apis.v1, ns)
+        for i in range(len(ic_pods)):
+            conf = get_vs_nginx_template_conf(
+                kube_apis.v1,
+                virtual_server_setup.namespace,
+                virtual_server_setup.vs_name,
+                ic_pods[i].metadata.name,
+                ingress_controller_prerequisites.namespace,
+            )
+            assert "rate=10r/s" in conf
+        # restore replicas, policy and vs
+        scale_deployment(kube_apis.v1, kube_apis.apps_v1_api, "nginx-ingress", ns, 1)
+        delete_policy(kube_apis.custom_objects, pol_name, test_namespace)
+        self.restore_default_vs(kube_apis, virtual_server_setup)
+        assert (
+            policy_info["status"]
+            and policy_info["status"]["reason"] == "AddedOrUpdated"
+            and policy_info["status"]["state"] == "Valid"
+        )

tests/suite/test_rl_policies_vsr.py

@@ -5,12 +5,18 @@
 from settings import TEST_DATA
 from suite.utils.custom_resources_utils import read_custom_resource
 from suite.utils.policy_resources_utils import create_policy_from_yaml, delete_policy
-from suite.utils.resources_utils import wait_before_test
-from suite.utils.vs_vsr_resources_utils import patch_v_s_route_from_yaml, patch_virtual_server_from_yaml
+from suite.utils.resources_utils import get_pod_list, scale_deployment, wait_before_test
+from suite.utils.vs_vsr_resources_utils import (
+    get_vs_nginx_template_conf,
+    patch_v_s_route_from_yaml,
+    patch_virtual_server_from_yaml,
+)
 
 std_vs_src = f"{TEST_DATA}/virtual-server-route/standard/virtual-server.yaml"
 rl_pol_pri_src = f"{TEST_DATA}/rate-limit/policies/rate-limit-primary.yaml"
+rl_pol_pri_sca_src = f"{TEST_DATA}/rate-limit/policies/rate-limit-primary-scaled.yaml"
 rl_vsr_pri_src = f"{TEST_DATA}/rate-limit/route-subroute/virtual-server-route-pri-subroute.yaml"
+rl_vsr_pri_sca_src = f"{TEST_DATA}/rate-limit/route-subroute/virtual-server-route-pri-subroute-scaled.yaml"
 rl_pol_sec_src = f"{TEST_DATA}/rate-limit/policies/rate-limit-secondary.yaml"
 rl_vsr_sec_src = f"{TEST_DATA}/rate-limit/route-subroute/virtual-server-route-sec-subroute.yaml"
 rl_pol_invalid_src = f"{TEST_DATA}/rate-limit/policies/rate-limit-invalid.yaml"
@@ -355,3 +361,58 @@ def test_override_vs_vsr(
             kube_apis.custom_objects, v_s_route_setup.vs_name, std_vs_src, v_s_route_setup.namespace
         )
         assert rate_sec >= occur.count(200) >= (rate_sec - 2)
+
+    @pytest.mark.parametrize("src", [rl_vsr_pri_sca_src])
+    def test_rl_policy_scaled_vsr(
+        self,
+        kube_apis,
+        ingress_controller_prerequisites,
+        crd_ingress_controller,
+        v_s_route_app_setup,
+        v_s_route_setup,
+        test_namespace,
+        src,
+    ):
+        """
+        Test if rate-limiting policy is working with ~1 rps in vsr:subroute
+        """
+
+        ns = ingress_controller_prerequisites.namespace
+        scale_deployment(kube_apis.v1, kube_apis.apps_v1_api, "nginx-ingress", ns, 4)
+
+        print(f"Create rl policy")
+        pol_name = create_policy_from_yaml(
+            kube_apis.custom_objects, rl_pol_pri_sca_src, v_s_route_setup.route_m.namespace
+        )
+        print(f"Patch vsr with policy: {src}")
+        patch_v_s_route_from_yaml(
+            kube_apis.custom_objects,
+            v_s_route_setup.route_m.name,
+            src,
+            v_s_route_setup.route_m.namespace,
+        )
+
+        wait_before_test()
+        policy_info = read_custom_resource(
+            kube_apis.custom_objects, v_s_route_setup.route_m.namespace, "policies", pol_name
+        )
+
+        ic_pods = get_pod_list(kube_apis.v1, ns)
+        for i in range(len(ic_pods)):
+            conf = get_vs_nginx_template_conf(
+                kube_apis.v1,
+                v_s_route_setup.route_m.namespace,
+                v_s_route_setup.vs_name,
+                ic_pods[i].metadata.name,
+                ingress_controller_prerequisites.namespace,
+            )
+            assert "rate=10r/s" in conf
+        # restore replicas, policy and vsr
+        scale_deployment(kube_apis.v1, kube_apis.apps_v1_api, "nginx-ingress", ns, 1)
+        delete_policy(kube_apis.custom_objects, pol_name, v_s_route_setup.route_m.namespace)
+        self.restore_default_vsr(kube_apis, v_s_route_setup)
+        assert (
+            policy_info["status"]
+            and policy_info["status"]["reason"] == "AddedOrUpdated"
+            and policy_info["status"]["state"] == "Valid"
+        )

tests/suite/utils/resources_utils.py

@@ -293,13 +293,24 @@ def wait_until_all_pods_are_ready(v1: CoreV1Api, namespace) -> None:
     while not are_all_pods_in_ready_state(v1, namespace) and counter < 200:
         # remove counter based condition from line #264 and #269 if --batch-start="True"
         print("There are pods that are not Ready. Wait for 1 sec...")
-        time.sleep(1)
+        wait_before_test()
         counter = counter + 1
     if counter >= 300:
         raise PodNotReadyException()
     print("All pods are Ready")
 
 
+def get_pod_list(v1: CoreV1Api, namespace) -> []:
+    """
+    Get a list of pods in a namespace.
+
+    :param v1: CoreV1Api
+    :param namespace: namespace
+    :return: []
+    """
+    return v1.list_namespaced_pod(namespace).items
+
+
 def get_first_pod_name(v1: CoreV1Api, namespace) -> str:
     """
     Return 1st pod_name in a list of pods in a namespace.
@@ -901,16 +912,26 @@ def get_file_contents(v1: CoreV1Api, file_path, pod_name, pod_namespace, print_l
     :return: str
     """
     command = ["cat", file_path]
-    resp = stream(
-        v1.connect_get_namespaced_pod_exec,
-        pod_name,
-        pod_namespace,
-        command=command,
-        stderr=True,
-        stdin=False,
-        stdout=True,
-        tty=False,
-    )
+    retries = 0
+    while retries <= 3:
+        wait_before_test()
+        try:
+            resp = stream(
+                v1.connect_get_namespaced_pod_exec,
+                pod_name,
+                pod_namespace,
+                command=command,
+                stderr=True,
+                stdin=False,
+                stdout=True,
+                tty=False,
+            )
+            break
+        except Exception as e:
+            print(f"Error: {e}")
+            retries += 1
+            if retries == 3:
+                raise e
     result_conf = str(resp)
     if print_log:
         print("\nFile contents:\n" + result_conf)