From d4413fceb97cf35e30b349671b81c28d868bc17f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E9=A9=AC=E6=B4=AA=E8=B4=9E?= <hzma@alauda.io>
Date: Tue, 9 Jan 2024 11:18:36 +0800
Subject: [PATCH] update policy route when subnet cidr changed

---
 pkg/controller/subnet.go    | 74 ++++++++++++++++++++++++++++++++++++-
 pkg/ovs/ovn-nbctl-legacy.go | 18 +++++++++
 2 files changed, 91 insertions(+), 1 deletion(-)

diff --git a/pkg/controller/subnet.go b/pkg/controller/subnet.go
index cf3b4934742..fb99529a785 100644
--- a/pkg/controller/subnet.go
+++ b/pkg/controller/subnet.go
@@ -1159,7 +1159,13 @@ func (c *Controller) reconcileOvnRoute(subnet *kubeovnv1.Subnet) error {
 			}
 		}
 	} else {
-		if err = c.addCommonRoutesForSubnet(subnet); err != nil {
+		// It's diffcult to update policy route when subnet cidr is changed, add check for cidr changed situation
+		if err := c.reconcilePolicyRouteForCidrChangedSubnet(subnet, true); err != nil {
+			klog.Error(err)
+			return err
+		}
+
+		if err := c.addCommonRoutesForSubnet(subnet); err != nil {
 			klog.Error(err)
 			return err
 		}
@@ -1326,6 +1332,11 @@ func (c *Controller) reconcileOvnRoute(subnet *kubeovnv1.Subnet) error {
 				return fmt.Errorf("failed to add ecmp policy route, no gateway node exists")
 			}
 
+			if err := c.reconcilePolicyRouteForCidrChangedSubnet(subnet, false); err != nil {
+				klog.Error(err)
+				return err
+			}
+
 			if c.config.EnableEcmp {
 				// centralized subnet, enable ecmp, add ecmp policy route
 				gatewayNodes := strings.Split(subnet.Spec.GatewayNode, ",")
@@ -2353,3 +2364,64 @@ func (c *Controller) clearOldU2OResource(subnet *kubeovnv1.Subnet) error {
 	}
 	return nil
 }
+
+func (c *Controller) reconcilePolicyRouteForCidrChangedSubnet(subnet *kubeovnv1.Subnet, isCommonRoute bool) error {
+	var match string
+	var priority int32
+
+	if isCommonRoute {
+		priority = util.SubnetRouterPolicyPriority
+	} else {
+		priority = util.GatewayRouterPolicyPriority
+	}
+
+	policyMatches, err := c.ovnLegacyClient.FilterSubnetPolicyRoutes(priority, subnet.Name)
+	if err != nil {
+		klog.Errorf("failed to list logical router policies: %v", err)
+		return err
+	}
+	if len(policyMatches) == 0 {
+		return nil
+	}
+
+	for _, policyMatch := range policyMatches {
+		policyProtocol := kubeovnv1.ProtocolIPv4
+		if strings.Contains(policyMatch, "ip6") {
+			policyProtocol = kubeovnv1.ProtocolIPv6
+		}
+
+		for _, cidr := range strings.Split(subnet.Spec.CIDRBlock, ",") {
+			if cidr == "" {
+				continue
+			}
+			if policyProtocol != util.CheckProtocol(cidr) {
+				continue
+			}
+
+			af := 4
+			if util.CheckProtocol(cidr) == kubeovnv1.ProtocolIPv6 {
+				af = 6
+			}
+
+			if isCommonRoute {
+				match = fmt.Sprintf("ip%d.dst == %s", af, cidr)
+			} else {
+				if subnet.Spec.GatewayType == kubeovnv1.GWCentralizedType {
+					match = fmt.Sprintf("ip%d.src == %s", af, cidr)
+				} else {
+					// distributed subnet does not need process gateway route policy
+					continue
+				}
+			}
+
+			if policyMatch != match {
+				klog.Infof("delete old policy route for subnet %s with match %s priority %d, new match %v", subnet.Name, policyMatch, priority, match)
+				if err = c.ovnLegacyClient.DeletePolicyRoute(subnet.Spec.Vpc, priority, policyMatch); err != nil {
+					klog.Errorf("failed to delete policy route for subnet %s: %v", subnet.Name, err)
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}
diff --git a/pkg/ovs/ovn-nbctl-legacy.go b/pkg/ovs/ovn-nbctl-legacy.go
index de0ee178661..37919881d13 100644
--- a/pkg/ovs/ovn-nbctl-legacy.go
+++ b/pkg/ovs/ovn-nbctl-legacy.go
@@ -2652,6 +2652,24 @@ func (c *LegacyClient) GetPolicyRouteParas(priority int32, match string) ([]stri
 	return result[0]["nexthops"], nameIpMap, nil
 }
 
+func (c *LegacyClient) FilterSubnetPolicyRoutes(priority int32, subnetName string) ([]string, error) {
+	results, err := c.CustomFindEntity("Logical_Router_Policy", []string{"match"}, fmt.Sprintf("priority=%d", priority), fmt.Sprintf("external_ids:subnet=\"%s\"", subnetName), fmt.Sprintf("external_ids:vendor=\"%s\"", util.CniTypeName))
+	if err != nil {
+		klog.Errorf("customFindEntity failed, %v", err)
+		return nil, err
+	}
+	if len(results) == 0 {
+		return nil, nil
+	}
+
+	var policyMatches []string
+	for _, result := range results {
+		policyMatches = append(policyMatches, strings.Join(result["match"], " "))
+	}
+
+	return policyMatches, nil
+}
+
 func (c LegacyClient) SetPolicyRouteExternalIds(priority int32, match string, nameIpMaps map[string]string) error {
 	result, err := c.CustomFindEntity("Logical_Router_Policy", []string{"_uuid"}, fmt.Sprintf("priority=%d", priority), fmt.Sprintf("match=\"%s\"", match))
 	if err != nil {