feat: natgw qos

kubeovn · May 4, 2023 · c2af2bf · c2af2bf
1 parent bdd201b
commit c2af2bf
Show file tree

Hide file tree

Showing 15 changed files with 444 additions and 51 deletions.
diff --git a/charts/templates/ovn-CR.yaml b/charts/templates/ovn-CR.yaml
@@ -11,6 +11,7 @@ rules:
       - vpcs
       - vpcs/status
       - vpc-nat-gateways
+      - vpc-nat-gateways/status
       - subnets
       - subnets/status
       - ips

diff --git a/dist/images/install.sh b/dist/images/install.sh
@@ -385,10 +385,17 @@ spec:
       name: v1
       served: true
       storage: true
+      subresources:
+        status: {}
       schema:
         openAPIV3Schema:
           type: object
           properties:
+            status:
+              type: object
+              properties:
+                qosPolicy:
+                  type: string
             spec:
               type: object
               properties:
@@ -406,6 +413,8 @@ spec:
                   type: array
                   items:
                     type: string
+                qosPolicy:
+                  type: string
                 tolerations:
                   type: array
                   items:
@@ -2322,6 +2331,7 @@ rules:
       - vpcs
       - vpcs/status
       - vpc-nat-gateways
+      - vpc-nat-gateways/status
       - subnets
       - subnets/status
       - ips
@@ -2835,6 +2845,7 @@ rules:
       - vpcs
       - vpcs/status
       - vpc-nat-gateways
+      - vpc-nat-gateways/status
       - subnets
       - subnets/status
       - ips

diff --git a/dist/images/vpcnatgateway/nat-gateway.sh b/dist/images/vpcnatgateway/nat-gateway.sh
@@ -242,11 +242,13 @@ function del_dnat() {
 
 
 # example usage:
-# delete_tc_filter "1:0" "192.168.1.1" "src"
-function delete_tc_filter() {
-    qdisc_id=$1
-    v4ip=$2
-    direction=$3
+# delete_tc_u32_filter "net1" "1:0" "192.168.1.1" "src"
+function delete_tc_u32_filter() {
+    dev=$1
+    qdisc_id=$2
+    cidr=$3
+    matchDirection=$4
+
 
     # tc -p -s -d filter show dev net1 parent $qdisc_id
     # filter protocol ip pref 10 u32 chain 0
@@ -259,16 +261,16 @@ function delete_tc_filter() {
     #  Sent 0 bytes 0 pkts (dropped 0, overlimits 0)
 
     # get the corresponding filterID by the EIP, and use the filterID to delete the corresponding filtering rule.
-    ipList=$(tc -p -s -d filter show dev net1 parent $qdisc_id | grep "match IP " | awk '{print $4}')
+    ipList=$(tc -p -s -d filter show dev $dev parent $qdisc_id | grep -E "match IP src|dst ([0-9]{1,3}\.){3}[0-9]{1,3}/[0-9]{1,2}")
     i=0
-    for line in $ipList; do
+    echo "$ipList" | while read line; do
         i=$((i+1))
-        if echo "$line" | grep $v4ip; then
-            result=$(tc -p -s -d filter show dev net1 parent $qdisc_id | grep "filter protocol ip pref [0-9]\+ u32 \(fh\|chain [0-9]\+ fh\) \(\w\+::\w\+\) *" | awk '{print $5,$10}' | sed -n $i"p")
+        if echo "$line" | grep "$matchDirection $cidr"; then
+            result=$(tc -p -s -d filter show dev $dev parent $qdisc_id | grep "filter protocol ip pref [0-9]\+ u32 \(fh\|chain [0-9]\+ fh\) \(\w\+::\w\+\) *" | awk '{print $5,$10}' | sed -n $i"p")
             arr=($result)
             pref=${arr[0]}
             filterID=${arr[1]}
-            exec_cmd "tc filter del dev net1 parent $qdisc_id protocol ip prio $pref handle $filterID u32"
+            exec_cmd "tc filter del dev $dev parent $qdisc_id protocol ip prio $pref handle $filterID u32"
             break
         fi
     done
@@ -285,16 +287,17 @@ function eip_ingress_qos_add() {
         priority=${arr[1]}
         rate=${arr[2]}
         burst=${arr[3]}
-        direction="dst"
-        tc qdisc add dev net1 ingress 2>/dev/nul || true
+        dev="net1"
+        matchDirection="dst"
+        tc qdisc add dev $dev ingress 2>/dev/nul || true
         # get qdisc id
-        qdisc_id=$(tc qdisc show dev net1 ingress | awk '{print $3}')
+        qdisc_id=$(tc qdisc show dev $dev ingress | awk '{print $3}')
         # del old filter
-        tc -p -s -d filter show dev net1 parent $qdisc_id | grep -w $v4ip
+        tc -p -s -d filter show dev $dev parent $qdisc_id | grep -w $v4ip
         if [ "$?" -eq 0 ];then
-            delete_tc_filter $qdisc_id $v4ip $direction
+            delete_tc_u32_filter $dev $qdisc_id $v4ip $matchDirection
         fi
-        exec_cmd "tc filter add dev net1 parent $qdisc_id protocol ip prio $priority u32 match ip $direction $v4ip police rate "$rate"Mbit burst "$burst"Mb drop flowid :1"
+        exec_cmd "tc filter add dev $dev parent $qdisc_id protocol ip prio $priority u32 match ip $matchDirection $v4ip police rate "$rate"Mbit burst "$burst"Mb drop flowid :1"
     done
 }
 
@@ -310,27 +313,104 @@ function eip_egress_qos_add() {
         rate=${arr[2]}
         burst=${arr[3]}
         qdisc_id="1:0"
-        direction="src"
-        tc qdisc add dev net1 root handle $qdisc_id htb 2>/dev/nul || true
+        matchDirection="src"
+        dev="net1"
+        tc qdisc add dev $dev root handle $qdisc_id htb 2>/dev/nul || true
         # del old filter
-        tc -p -s -d filter show dev net1 parent $qdisc_id | grep -w $v4ip
+        tc -p -s -d filter show dev $dev parent $qdisc_id | grep -w $v4ip
         if [ "$?" -eq 0 ];then
-            delete_tc_filter $qdisc_id $v4ip $direction
+            delete_tc_u32_filter $dev $qdisc_id $v4ip $matchDirection
+        fi
+        exec_cmd "tc filter add dev $dev parent $qdisc_id protocol ip prio $priority u32 match ip $matchDirection $v4ip police rate "$rate"Mbit burst "$burst"Mb drop flowid :1"
+    done
+}
+
+function qos_add() {
+    for rule in $@
+    do
+        IFS=',' read -r -a arr <<< "$rule"
+        local qdiscType=(${arr[0]})
+        local dev=${arr[1]}
+        local priority=${arr[2]}
+        local classifierType=${arr[3]}
+        local matchType=${arr[4]}
+        local matchDirection=${arr[5]}
+        local cidr=${arr[6]}
+        local rate=${arr[7]}
+        local burst=${arr[8]}
+
+        if [ "$qdiscType" == "ingress" ];then
+            tc qdisc add dev $dev ingress 2>/dev/null || true
+            # get qdisc id
+            qdisc_id=$(tc qdisc show dev $dev ingress | awk '{print $3}')
+        elif [ "$qdiscType" == "egress" ];then
+            qdisc_id="1:0"
+            tc qdisc add dev $dev root handle $qdisc_id htb 2>/dev/null || true
+        fi
+
+        if [ "$classifierType" == "u32" ];then
+            # del old filter
+            tc -p -s -d filter show dev $dev parent $qdisc_id | grep -w $cidr
+            if [ "$?" -eq 0 ];then
+                delete_tc_u32_filter $dev $qdisc_id $cidr $matchDirection
+            fi
+            exec_cmd "tc filter add dev $dev parent $qdisc_id protocol ip prio $priority u32 match $matchType $matchDirection $cidr police rate "$rate"Mbit burst "$burst"Mb drop flowid :1"
+        elif [ "$classifierType" == "matchall" ];then
+            # del old filter
+            tc -p -s -d filter show dev $dev parent $qdisc_id | grep -w matchall
+            if [ "$?" -eq 0 ];then
+                exec_cmd "tc filter del dev $dev parent $qdisc_id protocol ip prio $priority matchall"
+            fi
+            exec_cmd "tc filter add dev $dev parent $qdisc_id protocol ip prio $priority matchall action police rate "$rate"Mbit burst "$burst"Mb drop flowid :1"
+        fi
+    done
+}
+
+function qos_del() {
+    for rule in $@
+    do
+        IFS=',' read -r -a arr <<< "$rule"
+        local qdiscType=(${arr[0]})
+        local dev=${arr[1]}
+        local priority=${arr[2]}
+        local classifierType=${arr[3]}
+        local matchType=${arr[4]}
+        local matchDirection=${arr[5]}
+        local cidr=${arr[6]}
+        local rate=${arr[7]}
+        local burst=${arr[8]}
+
+        if [ "$qdiscType" == "ingress" ];then
+            qdisc_id=$(tc qdisc show dev $dev ingress | awk '{print $3}')
+            if [ -z "$qdisc_id" ]; then
+                exit 0
+            fi
+        elif [ "$qdiscType" == "egress" ];then
+            qdisc_id="1:0"
+        fi
+        # if qdisc_id is empty, this means ingress qdisc is not added, so we don't need to delete filter.
+        if [ "$classifierType" == "u32" ];then
+            delete_tc_u32_filter $dev $qdisc_id $cidr $matchDirection
+        elif [ "$classifierType" == "matchall" ];then
+            tc -p -s -d filter show dev $dev parent $qdisc_id | grep -w matchall
+            if [ "$?" -eq 0 ];then
+                exec_cmd "tc filter del dev $dev parent $qdisc_id protocol ip prio $priority matchall"
+            fi
         fi
-        exec_cmd "tc filter add dev net1 parent $qdisc_id protocol ip prio $priority u32 match ip $direction $v4ip police rate "$rate"Mbit burst "$burst"Mb drop flowid :1"
     done
 }
 
 function eip_ingress_qos_del() {
     for rule in $@
     do
         arr=(${rule//,/ })
-        v4ip=(${arr[0]//\// })
-        direction="dst"
-        qdisc_id=$(tc qdisc show dev net1 ingress | awk '{print $3}')
+        cidr=(${arr[0]//\// })
+        matchDirection="dst"
+        dev="net1"
+        qdisc_id=$(tc qdisc show dev $dev ingress | awk '{print $3}')
         # if qdisc_id is empty, this means ingress qdisc is not added, so we don't need to delete filter.
         if [ -n "$qdisc_id" ]; then
-            delete_tc_filter $qdisc_id $v4ip $direction
+            delete_tc_u32_filter $dev $qdisc_id $cidr $matchDirection
         fi
     done
 }
@@ -339,10 +419,11 @@ function eip_egress_qos_del() {
     for rule in $@
     do
         arr=(${rule//,/ })
-        v4ip=(${arr[0]//\// })
-        direction="src"
+        cidr=(${arr[0]//\// })
+        matchDirection="src"
         qdisc_id="1:0"
-        delete_tc_filter $qdisc_id $v4ip $direction
+        dev="net1"
+        delete_tc_u32_filter $dev $qdisc_id $cidr $matchDirection
     done
 }
 
@@ -422,6 +503,14 @@ case $opt in
         echo "eip-egress-qos-del $rules"
         eip_egress_qos_del $rules
         ;;
+ qos-add)
+        echo "qos-add $rules"
+        qos_add $rules
+        ;;
+ qos-del)
+        echo "qos-del $rules"
+        qos_del $rules
+        ;;
  *)
         echo "Usage: $0 [init|subnet-route-add|subnet-route-del|eip-add|eip-del|floating-ip-add|floating-ip-del|dnat-add|dnat-del|snat-add|snat-del] ..."
         exit 1

diff --git a/pkg/apis/kubeovn/v1/status.go b/pkg/apis/kubeovn/v1/status.go
@@ -137,3 +137,13 @@ func (qoss *QoSPolicyStatus) Bytes() ([]byte, error) {
 	klog.V(5).Info("status body", newStr)
 	return []byte(newStr), nil
 }
+
+func (vns *VpcNatStatus) Bytes() ([]byte, error) {
+	bytes, err := json.Marshal(vns)
+	if err != nil {
+		return nil, err
+	}
+	newStr := fmt.Sprintf(`{"status": %s}`, string(bytes))
+	klog.V(5).Info("status body", newStr)
+	return []byte(newStr), nil
+}
diff --git a/pkg/apis/kubeovn/v1/types.go b/pkg/apis/kubeovn/v1/types.go
@@ -44,7 +44,8 @@ var (
 type QoSPolicyBindingType string
 
 const (
-	QoSBindingTypeEIP QoSPolicyBindingType = "EIP"
+	QoSBindingTypeEIP   QoSPolicyBindingType = "EIP"
+	QoSBindingTypeNatGw QoSPolicyBindingType = "NATGW"
 )
 
 type QoSPolicyRuleDirection string
@@ -479,7 +480,8 @@ type VpcNatGateway struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`
 
-	Spec VpcNatSpec `json:"spec"`
+	Spec   VpcNatSpec   `json:"spec"`
+	Status VpcNatStatus `json:"status,omitempty"`
 }
 
 type VpcNatSpec struct {
@@ -490,6 +492,11 @@ type VpcNatSpec struct {
 	Selector        []string            `json:"selector"`
 	Tolerations     []corev1.Toleration `json:"tolerations"`
 	Affinity        corev1.Affinity     `json:"affinity"`
+	QoSPolicy       string              `json:"qosPolicy"`
+}
+
+type VpcNatStatus struct {
+	QoSPolicy string `json:"qosPolicy" patchStrategy:"merge"`
 }
 
 // +genclient

diff --git a/pkg/apis/kubeovn/v1/zz_generated.deepcopy.go b/pkg/apis/kubeovn/v1/zz_generated.deepcopy.go
diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_vpcnatgateway.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_vpcnatgateway.go
diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/vpcnatgateway.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/vpcnatgateway.go
diff --git a/pkg/controller/init.go b/pkg/controller/init.go
@@ -707,7 +707,7 @@ func (c *Controller) initSyncCrdVpcNatGw() error {
 		return err
 	}
 	for _, gw := range gws {
-		if err := c.updateCrdNatGw(gw.Name); err != nil {
+		if err := c.updateCrdNatGw(gw.Name, ""); err != nil {
 			klog.Errorf("failed to update nat gw: %v", gw.Name, err)
 			return err
 		}