From 2ded787af86caf1cd60b9250b55681eabc50087a Mon Sep 17 00:00:00 2001 From: oilbeater Date: Wed, 17 Jul 2024 07:22:45 +0000 Subject: [PATCH] use route policy to reimplement northGateway Signed-off-by: oilbeater --- pkg/controller/gc.go | 41 +++++++++++++++++++++++++++++++ pkg/controller/pod.go | 53 ++++++++++++++++++++++++++-------------- pkg/controller/subnet.go | 21 +++------------- pkg/util/const.go | 13 +++++----- 4 files changed, 86 insertions(+), 42 deletions(-) diff --git a/pkg/controller/gc.go b/pkg/controller/gc.go index 1ebcab59dc6..6890eb37505 100644 --- a/pkg/controller/gc.go +++ b/pkg/controller/gc.go @@ -3,6 +3,7 @@ package controller import ( "context" "fmt" + "slices" "strings" "unicode" @@ -32,6 +33,7 @@ func (c *Controller) gc() error { c.gcLogicalSwitchPort, c.gcLoadBalancer, c.gcPortGroup, + c.gcRoutePolicy, c.gcStaticRoute, c.gcVpcNatGateway, c.gcLogicalRouterPort, @@ -671,6 +673,45 @@ func (c *Controller) gcPortGroup() error { return nil } +func (c *Controller) gcRoutePolicy() error { + klog.Infof("start to gc route policy") + + policies, err := c.OVNNbClient.ListLogicalRouterPolicies(c.config.ClusterRouter, util.NorthGatewayRoutePolicyPriority, nil, true) + if err != nil { + klog.Errorf("failed to list route policy, %v", err) + return err + } + + podIPs := []string{} + pods, err := c.podsLister.List(labels.Everything()) + if err != nil { + klog.Errorf("failed to list pods, %v", err) + return err + } + for _, pod := range pods { + if pod.Annotations != nil && pod.Annotations[util.NorthGatewayAnnotation] != "" { + podIPs = append(podIPs, strings.Split(pod.Annotations[util.IPAddressAnnotation], ",")...) + } + } + + for _, policy := range policies { + parts := strings.Split(policy.Match, "==") + if len(parts) != 2 { + continue + } + srcIP := strings.TrimSpace(parts[1]) + if !slices.Contains(podIPs, srcIP) { + klog.Infof("gc route policy %s", policy.Match) + if err := c.OVNNbClient.DeleteLogicalRouterPolicy(c.config.ClusterRouter, policy.Priority, policy.Match); err != nil { + klog.Errorf("failed to delete route policy %s: %v", policy.Match, err) + return err + } + } + } + + return nil +} + func (c *Controller) gcStaticRoute() error { klog.Infof("start to gc static routes") routes, err := c.OVNNbClient.ListLogicalRouterStaticRoutes(c.config.ClusterRouter, nil, nil, "", nil) diff --git a/pkg/controller/pod.go b/pkg/controller/pod.go index aff50f20110..50387ad0ca4 100644 --- a/pkg/controller/pod.go +++ b/pkg/controller/pod.go @@ -821,18 +821,32 @@ func (c *Controller) reconcileRouteSubnets(cachedPod, pod *v1.Pod, needRoutePodN } } - if pod.Annotations[util.NorthGatewayAnnotation] != "" { - if err := c.addStaticRouteToVpc( - subnet.Spec.Vpc, - &kubeovnv1.StaticRoute{ - Policy: kubeovnv1.PolicySrc, - CIDR: podIP, - NextHopIP: pod.Annotations[util.NorthGatewayAnnotation], - RouteTable: subnet.Spec.RouteTable, - }, - ); err != nil { - klog.Errorf("failed to add static route, %v", err) - return err + if pod.Annotations[util.NorthGatewayAnnotation] != "" && pod.Annotations[util.IPAddressAnnotation] != "" { + for _, podAddr := range strings.Split(pod.Annotations[util.IPAddressAnnotation], ",") { + if util.CheckProtocol(podAddr) != util.CheckProtocol(pod.Annotations[util.NorthGatewayAnnotation]) { + continue + } + ipSuffix := "ip4" + if util.CheckProtocol(podAddr) == kubeovnv1.ProtocolIPv6 { + ipSuffix = "ip6" + } + + if err := c.addPolicyRouteToVpc( + subnet.Spec.Vpc, + &kubeovnv1.PolicyRoute{ + Priority: util.NorthGatewayRoutePolicyPriority, + Match: fmt.Sprintf("%s.src == %s", ipSuffix, podAddr), + Action: kubeovnv1.PolicyRouteActionReroute, + NextHopIP: pod.Annotations[util.NorthGatewayAnnotation], + }, + map[string]string{ + "vendor": util.CniTypeName, + "subnet": subnet.Name, + }, + ); err != nil { + klog.Errorf("failed to add policy route, %v", err) + return err + } } } else if c.config.EnableEipSnat { if err = c.deleteStaticRouteFromVpc( @@ -971,19 +985,22 @@ func (c *Controller) handleDeletePod(key string) error { klog.Error(err) return err } - // If pod has snat or eip, also need delete staticRoute when delete pod + if vpc.Name == c.config.ClusterRouter { - if err = c.deleteStaticRouteFromVpc( + ipSuffix := "ip4" + if util.CheckProtocol(address.IP) == kubeovnv1.ProtocolIPv6 { + ipSuffix = "ip6" + } + if err = c.deletePolicyRouteFromVpc( vpc.Name, - subnet.Spec.RouteTable, - address.IP, - "", - kubeovnv1.PolicyDst, + util.NorthGatewayRoutePolicyPriority, + fmt.Sprintf("%s.src == %s", ipSuffix, address.IP), ); err != nil { klog.Errorf("failed to delete static route, %v", err) return err } } + if c.config.EnableEipSnat { if pod.Annotations[util.EipAnnotation] != "" { if err = c.OVNNbClient.DeleteNat(c.config.ClusterRouter, ovnnb.NATTypeDNATAndSNAT, pod.Annotations[util.EipAnnotation], address.IP); err != nil { diff --git a/pkg/controller/subnet.go b/pkg/controller/subnet.go index 89f14916061..1b15a4e2ffc 100644 --- a/pkg/controller/subnet.go +++ b/pkg/controller/subnet.go @@ -1546,24 +1546,9 @@ func (c *Controller) reconcileDistributedSubnetRouteInDefaultVpc(subnet *kubeovn continue } - if pod.Annotations[util.NorthGatewayAnnotation] != "" { - if err := c.addStaticRouteToVpc( - subnet.Spec.Vpc, - &kubeovnv1.StaticRoute{ - Policy: kubeovnv1.PolicySrc, - CIDR: pod.Annotations[fmt.Sprintf(util.IPAddressAnnotationTemplate, podNet.ProviderName)], - NextHopIP: pod.Annotations[util.NorthGatewayAnnotation], - RouteTable: util.MainRouteTable, - }, - ); err != nil { - klog.Errorf("add static route failed, %v", err) - return err - } - } else { - podName := c.getNameByPod(pod) - portName := ovs.PodNameToPortName(podName, pod.Namespace, podNet.ProviderName) - podPorts = append(podPorts, portName) - } + podName := c.getNameByPod(pod) + portName := ovs.PodNameToPortName(podName, pod.Namespace, podNet.ProviderName) + podPorts = append(podPorts, portName) } if pod.Annotations[util.NorthGatewayAnnotation] != "" { diff --git a/pkg/util/const.go b/pkg/util/const.go index f9516dd5b8a..1db2044f81f 100644 --- a/pkg/util/const.go +++ b/pkg/util/const.go @@ -190,12 +190,13 @@ const ( OvnFip = "ovn" IptablesFip = "iptables" - U2OSubnetPolicyPriority = 29400 - GatewayRouterPolicyPriority = 29000 - OvnICPolicyPriority = 29500 - NodeRouterPolicyPriority = 30000 - NodeLocalDNSPolicyPriority = 30100 - SubnetRouterPolicyPriority = 31000 + U2OSubnetPolicyPriority = 29400 + GatewayRouterPolicyPriority = 29000 + NorthGatewayRoutePolicyPriority = 29250 + OvnICPolicyPriority = 29500 + NodeRouterPolicyPriority = 30000 + NodeLocalDNSPolicyPriority = 30100 + SubnetRouterPolicyPriority = 31000 OffloadType = "offload-port" InternalType = "internal-port"