feat: add support for EIP QoS

dist/images/cleanup.sh

@@ -113,7 +113,8 @@ kubectl delete --ignore-not-found crd htbqoses.kubeovn.io security-groups.kubeov
                                       vpc-nat-gateways.kubeovn.io vpcs.kubeovn.io vlans.kubeovn.io provider-networks.kubeovn.io \
                                       iptables-dnat-rules.kubeovn.io  iptables-eips.kubeovn.io  iptables-fip-rules.kubeovn.io \
                                       iptables-snat-rules.kubeovn.io vips.kubeovn.io switch-lb-rules.kubeovn.io vpc-dnses.kubeovn.io \
-                                      ovn-eips.kubeovn.io ovn-fips.kubeovn.io ovn-snat-rules.kubeovn.io ovn-dnat-rules.kubeovn.io
+                                      ovn-eips.kubeovn.io ovn-fips.kubeovn.io ovn-snat-rules.kubeovn.io ovn-dnat-rules.kubeovn.io \
+                                      qos-policies.kubeovn.io
 
 # Remove annotations/labels in namespaces and nodes
 kubectl annotate no --all ovn.kubernetes.io/cidr-

dist/images/install.sh

@@ -468,6 +468,8 @@ spec:
                   type: string
                 redo:
                   type: string
+                qosPolicy:
+                  type: string
                 conditions:
                   type: array
                   items:
@@ -496,6 +498,8 @@ spec:
                   type: string
                 natGwDp:
                   type: string
+                qosPolicy:
+                  type: string
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
@@ -1898,6 +1902,75 @@ spec:
         status: {}
   conversion:
     strategy: None
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: qos-policies.kubeovn.io
+spec:
+  group: kubeovn.io
+  names:
+    plural: qos-policies
+    singular: qos-policy
+    shortNames:
+      - qos
+    kind: QoSPolicy
+    listKind: QoSPolicyList
+  scope: Cluster
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      subresources:
+        status: {}
+      additionalPrinterColumns:
+      - jsonPath: .spec.bandwidthLimitRule.ingressMax
+        name: IngressMax
+        type: string
+      - jsonPath: .spec.bandwidthLimitRule.egressMax
+        name: EgressMax
+        type: string
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            status:
+              type: object
+              properties:
+                bandwidthLimitRule:
+                  type: object
+                  properties:
+                    ingressMax:
+                      type: string
+                    egressMax:
+                      type: string
+                conditions:
+                  type: array
+                  items:
+                    type: object
+                    properties:
+                      type:
+                        type: string
+                      status:
+                        type: string
+                      reason:
+                        type: string
+                      message:
+                        type: string
+                      lastUpdateTime:
+                        type: string
+                      lastTransitionTime:
+                        type: string
+            spec:
+              type: object
+              properties:
+                bandwidthLimitRule:
+                  type: object
+                  properties:
+                    ingressMax:
+                      type: string
+                    egressMax:
+                      type: string
 EOF
 
 if $DPDK; then
@@ -1954,6 +2027,8 @@ rules:
       - switch-lb-rules/status
       - vpc-dnses
       - vpc-dnses/status
+      - qos-policies
+      - qos-policies/status
     verbs:
       - "*"
   - apiGroups:
@@ -2465,6 +2540,8 @@ rules:
       - vpc-dnses/status
       - switch-lb-rules
       - switch-lb-rules/status
+      - qos-policies
+      - qos-policies/status
     verbs:
       - "*"
   - apiGroups:

dist/images/vpcnatgateway/nat-gateway.sh

@@ -240,6 +240,97 @@ function del_dnat() {
     done
 }
 
+
+# example usage:
+# delete_tc_filter "1:0" "192.168.1.1" "src"
+function delete_tc_filter() {
+    qdisc_id=$1
+    v4ip=$2
+    direction=$3
+
+    # tc -p -s -d filter show dev net1 parent $qdisc_id prio 1 
+    # output like this:
+    # filter protocol ip u32 chain 0
+    # filter protocol ip u32 chain 0 fh 800: ht divisor 1
+    # filter protocol ip u32 chain 0 fh 800::800 order 2048 key ht 800 bkt 0 *flowid :1 not_in_hw
+    # match IP dst x.x.x.1/32
+    # police 0x1 rate 1Mbit burst 1Mb mtu 2Kb action drop overhead 0b linklayer ethernet
+    #         ref 1 bind 1  installed 392 sec used 392 sec firstused 18818153 sec
+    # Sent 0 bytes 0 pkts (dropped 0, overlimits 0)
+
+    # get the corresponding filterID by the EIP, and use the filterID to delete the corresponding filtering rule.
+    ipList=$(tc -p -s -d filter show dev net1 parent $qdisc_id prio 1 | grep "match IP $direction" | awk '{print $4}')
+    i=0
+    for line in $ipList; do
+        i=$((i+1))
+        if echo "$line" | grep $v4ip; then
+            filterID=$(tc -p -s -d filter show dev net1 parent $qdisc_id prio 1 | grep "filter protocol ip u32 \(fh\|chain [0-9]\+ fh\) \(\w\+::\w\+\) *" | awk '{print $8}' | sed -n $i"p")
+            exec_cmd "tc filter del dev net1 parent $qdisc_id protocol ip prio 1 handle $filterID u32"
+            break
+        fi
+    done
+}
+
+function eip_ingress_qos_add() {
+    for rule in $@
+    do
+        arr=(${rule//,/ })
+        v4ip=(${arr[0]//\// })
+        rate=${arr[1]}
+        direction="dst"
+        tc qdisc add dev net1 ingress 2>/dev/nul || true
+        # get qdisc id
+        qdisc_id=$(tc qdisc show dev net1 ingress | awk '{print $3}')
+        # del old filter
+        tc -p -s -d filter show dev net1 parent $qdisc_id prio 1 | grep -w $v4ip
+        if [ "$?" -eq 0 ];then
+            delete_tc_filter $qdisc_id $v4ip $direction
+        fi
+        exec_cmd "tc filter add dev net1 parent $qdisc_id protocol ip prio 1 u32 match ip $direction $v4ip police rate "$rate"Mbit burst "$rate"Mb drop flowid :1"
+    done
+}
+
+function eip_egress_qos_add() {
+    for rule in $@
+    do
+        arr=(${rule//,/ })
+        v4ip=(${arr[0]//\// })
+        rate=${arr[1]}
+        qdisc_id="1:0"
+        direction="src"
+        tc qdisc add dev net1 root handle $qdisc_id htb 2>/dev/nul || true
+        # del old filter
+        tc -p -s -d filter show dev net1 parent $qdisc_id prio 1 | grep -w $v4ip
+        if [ "$?" -eq 0 ];then
+            delete_tc_filter $qdisc_id $v4ip $direction
+        fi
+        exec_cmd "tc filter add dev net1 parent $qdisc_id protocol ip prio 1 u32 match ip $direction $v4ip police rate "$rate"Mbit burst "$rate"Mb drop flowid :1"
+    done
+}
+
+function eip_ingress_qos_del() {
+    for rule in $@
+    do
+        arr=(${rule//,/ })
+        v4ip=(${arr[0]//\// })
+        direction="dst"
+        qdisc_id=$(tc qdisc show dev net1 ingress | awk '{print $3}')
+        delete_tc_filter $qdisc_id $v4ip $direction
+    done
+}
+
+function eip_egress_qos_del() {
+    for rule in $@
+    do
+        arr=(${rule//,/ })
+        v4ip=(${arr[0]//\// })
+        direction="src"
+        qdisc_id="1:0"
+        delete_tc_filter $qdisc_id $v4ip $direction
+    done
+}
+
+
 rules=${@:2:${#}}
 opt=$1
 case $opt in
@@ -299,6 +390,22 @@ case $opt in
         echo "get-iptables-version $rules"
         get_iptables_version $rules
         ;;
+ eip-ingress-qos-add)
+        echo "eip-ingress-qos-add $rules"
+        eip_ingress_qos_add $rules
+        ;;
+ eip-egress-qos-add)
+        echo "eip-egress-qos-add $rules"
+        eip_egress_qos_add $rules
+        ;;
+ eip-ingress-qos-del)
+        echo "eip-ingress-qos-del $rules"
+        eip_ingress_qos_del $rules
+        ;;
+ eip-egress-qos-del)
+        echo "eip-egress-qos-del $rules"
+        eip_egress_qos_del $rules
+        ;;
  *)
         echo "Usage: $0 [init|subnet-route-add|subnet-route-del|eip-add|eip-del|floating-ip-add|floating-ip-del|dnat-add|dnat-del|snat-add|snat-del] ..."
         exit 1

docs/eip-qos.md

@@ -0,0 +1,50 @@
+# EIP QoS
+
+EIP QoS is a feature in Kube-OVN that allows for dynamic configuration of rate limits for both Ingress and Egress traffic on custom VPC EIPs.
+
+## Creating a QoS Policy
+
+To create a QoS Policy, use the following YAML configuration:
+
+```
+apiVersion: kubeovn.io/v1
+kind: QoSPolicy
+metadata:
+  name: qos-example
+spec:
+  bandwidthLimitRule:
+    ingressMax: "1" # Mbps
+    egressMax: "1" # Mbps
+```
+
+It is allowed to limit only one direction, just like this:
+
+```
+apiVersion: kubeovn.io/v1
+kind: QoSPolicy
+metadata:
+  name: qos-example
+spec:
+  bandwidthLimitRule:
+    ingressMax: "1" # Mbps
+```
+
+## Enabling EIP QoS
+To enable EIP QoS, use the following YAML configuration:
+
+```
+kind: IptablesEIP
+apiVersion: kubeovn.io/v1
+metadata:
+  name: eip-random
+spec:
+  natGwDp: gw1
+  qosPolicy: qos-example
+```
+
+You can also add or update the `.spec.qosPolicy` field to an existing EIP.
+
+## Limitations
+
+* After creating a QoS Policy, the bandwidth limit rules cannot be changed. If you need to set new rate limit rules for an EIP, you can update a new QoS Policy to the `IptablesEIP.spec.qosPolicy` field.
+* You can only delete a QoS Policy if it is not currently in use. Therefore, before deleting a QoS Policy, you must first remove the `IptablesEIP.spec.qosPolicy` field from any associated IptablesEIP.

kubeovn-helm/templates/ovn-CR.yaml

@@ -42,6 +42,8 @@ rules:
       - vpc-dnses/status
       - switch-lb-rules
       - switch-lb-rules/status
+      - qos-policies
+      - qos-policies/status
     verbs:
       - "*"
   - apiGroups:

pkg/apis/kubeovn/v1/register.go

@@ -67,6 +67,8 @@ func addKnownTypes(scheme *runtime.Scheme) error {
 		&SwitchLBRuleList{},
 		&VpcDns{},
 		&VpcDnsList{},
+		&QoSPolicy{},
+		&QoSPolicyList{},
 	)
 	metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
 	return nil

pkg/apis/kubeovn/v1/status.go

@@ -127,3 +127,13 @@ func (odrs *OvnDnatRuleStatus) Bytes() ([]byte, error) {
 	klog.V(5).Info("status body", newStr)
 	return []byte(newStr), nil
 }
+
+func (qoss *QoSPolicyStatus) Bytes() ([]byte, error) {
+	bytes, err := json.Marshal(qoss)
+	if err != nil {
+		return nil, err
+	}
+	newStr := fmt.Sprintf(`{"status": %s}`, string(bytes))
+	klog.V(5).Info("status body", newStr)
+	return []byte(newStr), nil
+}