use route policy to reimplement northGateway

Signed-off-by: oilbeater <liumengxinfly@gmail.com>

pkg/controller/gc.go

@@ -3,6 +3,7 @@ package controller
 import (
  "context"
  "fmt"
+ "slices"
  "strings"
  "unicode"
 
@@ -32,6 +33,7 @@ func (c *Controller) gc() error {
  c.gcLogicalSwitchPort,
  c.gcLoadBalancer,
  c.gcPortGroup,
+ c.gcRoutePolicy,
  c.gcStaticRoute,
  c.gcVpcNatGateway,
  c.gcLogicalRouterPort,
@@ -671,6 +673,45 @@ func (c *Controller) gcPortGroup() error {
  return nil
 }
 
+func (c *Controller) gcRoutePolicy() error {
+ klog.Infof("start to gc route policy")
+
+ policies, err := c.OVNNbClient.ListLogicalRouterPolicies(c.config.ClusterRouter, util.NorthGatewayRoutePolicyPriority, nil, true)
+ if err != nil {
+ klog.Errorf("failed to list route policy, %v", err)
+ return err
+ }
+
+ podIPs := []string{}
+ pods, err := c.podsLister.List(labels.Everything())
+ if err != nil {
+ klog.Errorf("failed to list pods, %v", err)
+ return err
+ }
+ for _, pod := range pods {
+ if pod.Annotations != nil && pod.Annotations[util.NorthGatewayAnnotation] != "" {
+ podIPs = append(podIPs, strings.Split(pod.Annotations[util.IPAddressAnnotation], ",")...)
+ }
+ }
+
+ for _, policy := range policies {
+ parts := strings.Split(policy.Match, "==")
+ if len(parts) != 2 {
+ continue
+ }
+ srcIP := strings.TrimSpace(parts[1])
+ if !slices.Contains(podIPs, srcIP) {
+ klog.Infof("gc route policy %s", policy.Match)
+ if err := c.OVNNbClient.DeleteLogicalRouterPolicy(c.config.ClusterRouter, policy.Priority, policy.Match); err != nil {
+ klog.Errorf("failed to delete route policy %s: %v", policy.Match, err)
+ return err
+ }
+ }
+ }
+
+ return nil
+}
+
 func (c *Controller) gcStaticRoute() error {
  klog.Infof("start to gc static routes")
  routes, err := c.OVNNbClient.ListLogicalRouterStaticRoutes(c.config.ClusterRouter, nil, nil, "", nil)

pkg/controller/pod.go

@@ -821,18 +821,32 @@ func (c *Controller) reconcileRouteSubnets(cachedPod, pod *v1.Pod, needRoutePodN
  }
  }
 
- if pod.Annotations[util.NorthGatewayAnnotation] != "" {
- if err := c.addStaticRouteToVpc(
- subnet.Spec.Vpc,
- &kubeovnv1.StaticRoute{
- Policy: kubeovnv1.PolicySrc,
- CIDR: podIP,
- NextHopIP: pod.Annotations[util.NorthGatewayAnnotation],
- RouteTable: subnet.Spec.RouteTable,
- },
- ); err != nil {
- klog.Errorf("failed to add static route, %v", err)
- return err
+ if pod.Annotations[util.NorthGatewayAnnotation] != "" && pod.Annotations[util.IPAddressAnnotation] != "" {
+ for _, podAddr := range strings.Split(pod.Annotations[util.IPAddressAnnotation], ",") {
+ if util.CheckProtocol(podAddr) != util.CheckProtocol(pod.Annotations[util.NorthGatewayAnnotation]) {
+ continue
+ }
+ ipSuffix := "ip4"
+ if util.CheckProtocol(podAddr) == kubeovnv1.ProtocolIPv6 {
+ ipSuffix = "ip6"
+ }
+
+ if err := c.addPolicyRouteToVpc(
+ subnet.Spec.Vpc,
+ &kubeovnv1.PolicyRoute{
+ Priority: util.NorthGatewayRoutePolicyPriority,
+ Match: fmt.Sprintf("%s.src == %s", ipSuffix, podAddr),
+ Action: kubeovnv1.PolicyRouteActionReroute,
+ NextHopIP: pod.Annotations[util.NorthGatewayAnnotation],
+ },
+ map[string]string{
+ "vendor": util.CniTypeName,
+ "subnet": subnet.Name,
+ },
+ ); err != nil {
+ klog.Errorf("failed to add policy route, %v", err)
+ return err
+ }
  }
  } else if c.config.EnableEipSnat {
  if err = c.deleteStaticRouteFromVpc(
@@ -971,19 +985,22 @@ func (c *Controller) handleDeletePod(key string) error {
  klog.Error(err)
  return err
  }
- // If pod has snat or eip, also need delete staticRoute when delete pod
+
  if vpc.Name == c.config.ClusterRouter {
- if err = c.deleteStaticRouteFromVpc(
+ ipSuffix := "ip4"
+ if util.CheckProtocol(address.IP) == kubeovnv1.ProtocolIPv6 {
+ ipSuffix = "ip6"
+ }
+ if err = c.deletePolicyRouteFromVpc(
  vpc.Name,
- subnet.Spec.RouteTable,
- address.IP,
- "",
- kubeovnv1.PolicyDst,
+ util.NorthGatewayRoutePolicyPriority,
+ fmt.Sprintf("%s.src == %s", ipSuffix, address.IP),
  ); err != nil {
  klog.Errorf("failed to delete static route, %v", err)
  return err
  }
  }
+
  if c.config.EnableEipSnat {
  if pod.Annotations[util.EipAnnotation] != "" {
  if err = c.OVNNbClient.DeleteNat(c.config.ClusterRouter, ovnnb.NATTypeDNATAndSNAT, pod.Annotations[util.EipAnnotation], address.IP); err != nil {

pkg/controller/subnet.go

@@ -1546,24 +1546,9 @@ func (c *Controller) reconcileDistributedSubnetRouteInDefaultVpc(subnet *kubeovn
  continue
  }
 
- if pod.Annotations[util.NorthGatewayAnnotation] != "" {
- if err := c.addStaticRouteToVpc(
- subnet.Spec.Vpc,
- &kubeovnv1.StaticRoute{
- Policy: kubeovnv1.PolicySrc,
- CIDR: pod.Annotations[fmt.Sprintf(util.IPAddressAnnotationTemplate, podNet.ProviderName)],
- NextHopIP: pod.Annotations[util.NorthGatewayAnnotation],
- RouteTable: util.MainRouteTable,
- },
- ); err != nil {
- klog.Errorf("add static route failed, %v", err)
- return err
- }
- } else {
- podName := c.getNameByPod(pod)
- portName := ovs.PodNameToPortName(podName, pod.Namespace, podNet.ProviderName)
- podPorts = append(podPorts, portName)
- }
+ podName := c.getNameByPod(pod)
+ portName := ovs.PodNameToPortName(podName, pod.Namespace, podNet.ProviderName)
+ podPorts = append(podPorts, portName)
  }
 
  if pod.Annotations[util.NorthGatewayAnnotation] != "" {

pkg/util/const.go

@@ -190,12 +190,13 @@ const (
  OvnFip = "ovn"
  IptablesFip = "iptables"
 
- U2OSubnetPolicyPriority = 29400
- GatewayRouterPolicyPriority = 29000
- OvnICPolicyPriority = 29500
- NodeRouterPolicyPriority = 30000
- NodeLocalDNSPolicyPriority = 30100
- SubnetRouterPolicyPriority = 31000
+ U2OSubnetPolicyPriority = 29400
+ GatewayRouterPolicyPriority = 29000
+ NorthGatewayRoutePolicyPriority = 29250
+ OvnICPolicyPriority = 29500
+ NodeRouterPolicyPriority = 30000
+ NodeLocalDNSPolicyPriority = 30100
+ SubnetRouterPolicyPriority = 31000
 
  OffloadType = "offload-port"
  InternalType = "internal-port"