From 435d566da6678d116250c7830f6a2bd18d581673 Mon Sep 17 00:00:00 2001 From: changluyi Date: Mon, 4 Dec 2023 10:12:13 +0800 Subject: [PATCH 01/13] ovn support ic ecmp --- Makefile | 10 +- dist/images/start-ic-db.sh | 7 - pkg/controller/{ovn-ic.go => ovn_ic.go} | 377 +++++++++++++----------- pkg/ovs/ovn-nb-logical_router_route.go | 14 +- pkg/ovs/ovn-nbctl-legacy.go | 45 ++- pkg/util/const.go | 1 + yamls/kind.yaml.j2 | 6 + 7 files changed, 266 insertions(+), 194 deletions(-) rename pkg/controller/{ovn-ic.go => ovn_ic.go} (58%) diff --git a/Makefile b/Makefile index d4fd6f9c2d3..cec0a1e5ef1 100644 --- a/Makefile +++ b/Makefile @@ -252,8 +252,9 @@ kind-init-ipv4: kind-clean @$(MAKE) kind-create .PHONY: kind-init-ovn-ic -kind-init-ovn-ic: kind-clean-ovn-ic kind-init - @$(MAKE) kind-generate-config +kind-init-ovn-ic: kind-clean-ovn-ic + @ovn_ic=true $(MAKE) kind-init + @ovn_ic=true $(MAKE) kind-generate-config $(call kind_create_cluster,yamls/kind.yaml,kube-ovn1) .PHONY: kind-init-iptables @@ -360,6 +361,7 @@ kind-install-overlay-ipv4: kind-install kind-install-ovn-ic: kind-install $(call kind_load_image,kube-ovn1,$(REGISTRY)/kube-ovn:$(VERSION)) kubectl config use-context kind-kube-ovn1 + $(MAKE) kind-untaint-control-plane sed -e 's/10.16.0/10.18.0/g' \ -e 's/10.96.0/10.98.0/g' \ -e 's/100.64.0/100.68.0/g' \ @@ -370,8 +372,8 @@ kind-install-ovn-ic: kind-install docker run -d --name ovn-ic-db --network kind $(REGISTRY)/kube-ovn:$(VERSION) bash start-ic-db.sh @set -e; \ ic_db_host=$$(docker inspect ovn-ic-db -f "{{.NetworkSettings.Networks.kind.IPAddress}}"); \ - zone=az0 ic_db_host=$$ic_db_host gateway_node_name=kube-ovn-worker j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-0.yaml; \ - zone=az1 ic_db_host=$$ic_db_host gateway_node_name=kube-ovn1-worker j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-1.yaml + zone=az0 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn-worker,kube-ovn-worker2;kube-ovn-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-0.yaml; \ + zone=az1 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn1-worker,kube-ovn1-worker2;kube-ovn1-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-1.yaml kubectl config use-context kind-kube-ovn kubectl apply -f ovn-ic-0.yaml kubectl config use-context kind-kube-ovn1 diff --git a/dist/images/start-ic-db.sh b/dist/images/start-ic-db.sh index 8dd16fd1eeb..40c7801e0b7 100644 --- a/dist/images/start-ic-db.sh +++ b/dist/images/start-ic-db.sh @@ -1,9 +1,6 @@ #!/bin/bash set -eo pipefail -TS_NAME=${TS_NAME:-ts} -TS_CIDR=${TS_CIDR:-169.254.100.0/24} - function quit { /usr/share/ovn/scripts/ovn-ctl stop_ic_ovsdb exit 0 @@ -19,8 +16,6 @@ trap quit EXIT if [[ -z "$NODE_IPS" && -z "$LOCAL_IP" ]]; then /usr/share/ovn/scripts/ovn-ctl --db-ic-nb-create-insecure-remote=yes --db-ic-sb-create-insecure-remote=yes start_ic_ovsdb /usr/share/ovn/scripts/ovn-ctl status_ic_ovsdb - ovn-ic-nbctl --may-exist ts-add "$TS_NAME" - ovn-ic-nbctl set Transit_Switch ts external_ids:subnet="$TS_CIDR" tail --follow=name --retry /var/log/ovn/ovsdb-server-ic-nb.log else if [[ -z "$LEADER_IP" ]]; then @@ -33,8 +28,6 @@ else --ovn-ic-sb-db="$(gen_conn_str 6648)" \ start_ic_ovsdb /usr/share/ovn/scripts/ovn-ctl status_ic_ovsdb - ovn-ic-nbctl --may-exist ts-add "$TS_NAME" - ovn-ic-nbctl set Transit_Switch ts external_ids:subnet="$TS_CIDR" tail --follow=name --retry /var/log/ovn/ovsdb-server-ic-nb.log else echo "follower start with local ${LOCAL_IP}, leader ${LEADER_IP} and cluster $(gen_conn_str 6647)" diff --git a/pkg/controller/ovn-ic.go b/pkg/controller/ovn_ic.go similarity index 58% rename from pkg/controller/ovn-ic.go rename to pkg/controller/ovn_ic.go index 12fcede35f0..9466fc48d64 100644 --- a/pkg/controller/ovn-ic.go +++ b/pkg/controller/ovn_ic.go @@ -4,19 +4,22 @@ import ( "context" "encoding/json" "fmt" - "github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb" "os" "os/exec" "reflect" "strings" "time" + "github.com/scylladb/go-set/strset" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "k8s.io/klog/v2" + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + "github.com/kubeovn/kube-ovn/pkg/ovs" + "github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb" "github.com/kubeovn/kube-ovn/pkg/util" ) @@ -56,84 +59,82 @@ func (c *Controller) resyncInterConnection() { klog.Info("finish removing ovn-ic") return - } else { - blackList := []string{} - autoRoute := false - if cm.Data["auto-route"] == "true" { - autoRoute = true - } - subnets, err := c.subnetsLister.List(labels.Everything()) - if err != nil { - klog.Errorf("failed to list subnets, %v", err) - return - } - for _, subnet := range subnets { - if subnet.Spec.DisableInterConnection || subnet.Name == c.config.NodeSwitch { - blackList = append(blackList, subnet.Spec.CIDRBlock) - } - } - nodes, err := c.nodesLister.List(labels.Everything()) - if err != nil { - klog.Errorf("failed to list node, %v", err) - return + } + blackList := []string{} + autoRoute := false + if cm.Data["auto-route"] == "true" { + autoRoute = true + } + subnets, err := c.subnetsLister.List(labels.Everything()) + if err != nil { + klog.Errorf("failed to list subnets, %v", err) + return + } + for _, subnet := range subnets { + if subnet.Spec.DisableInterConnection || subnet.Name == c.config.NodeSwitch { + blackList = append(blackList, subnet.Spec.CIDRBlock) } - for _, node := range nodes { - ipv4, ipv6 := util.GetNodeInternalIP(*node) - if ipv4 != "" { - blackList = append(blackList, ipv4) - } - if ipv6 != "" { - blackList = append(blackList, ipv6) - } + } + nodes, err := c.nodesLister.List(labels.Everything()) + if err != nil { + klog.Errorf("failed to list node, %v", err) + return + } + for _, node := range nodes { + ipv4, ipv6 := util.GetNodeInternalIP(*node) + if ipv4 != "" { + blackList = append(blackList, ipv4) } - if err := c.ovnLegacyClient.SetICAutoRoute(autoRoute, blackList); err != nil { - klog.Errorf("failed to config auto route, %v", err) - return + if ipv6 != "" { + blackList = append(blackList, ipv6) } + } + if err := c.ovnLegacyClient.SetICAutoRoute(autoRoute, blackList); err != nil { + klog.Errorf("failed to config auto route, %v", err) + return + } - isCMEqual := reflect.DeepEqual(cm.Data, lastIcCm) - if icEnabled == "true" && lastIcCm != nil && isCMEqual { + isCMEqual := reflect.DeepEqual(cm.Data, lastIcCm) + if icEnabled == "true" && lastIcCm != nil && isCMEqual { + return + } + if icEnabled == "true" && lastIcCm != nil && !isCMEqual { + if err := c.removeInterConnection(lastIcCm["az-name"]); err != nil { + klog.Errorf("failed to remove ovn-ic, %v", err) return } - if icEnabled == "true" && lastIcCm != nil && !isCMEqual { - if err := c.removeInterConnection(lastIcCm["az-name"]); err != nil { - klog.Errorf("failed to remove ovn-ic, %v", err) - return - } - if err := c.delLearnedRoute(); err != nil { - klog.Errorf("failed to remove learned static routes, %v", err) - return - } - c.ovnLegacyClient.OvnICSbAddress = genHostAddress(cm.Data["ic-db-host"], cm.Data["ic-sb-port"]) - - c.ovnLegacyClient.OvnICNbAddress = genHostAddress(cm.Data["ic-db-host"], cm.Data["ic-nb-port"]) - klog.Info("start to reestablish ovn-ic") - if err := c.establishInterConnection(cm.Data); err != nil { - klog.Errorf("failed to reestablish ovn-ic, %v", err) - return - } - - if err := c.RemoveOldChassisInSbDB(lastIcCm["az-name"]); err != nil { - klog.Errorf("failed to remove remote chassis: %v", err) - } - - icEnabled = "true" - lastIcCm = cm.Data - klog.Info("finish reestablishing ovn-ic") + if err := c.delLearnedRoute(); err != nil { + klog.Errorf("failed to remove learned static routes, %v", err) return } + c.ovnLegacyClient.OvnICSbAddress = genHostAddress(cm.Data["ic-db-host"], cm.Data["ic-sb-port"]) c.ovnLegacyClient.OvnICNbAddress = genHostAddress(cm.Data["ic-db-host"], cm.Data["ic-nb-port"]) - klog.Info("start to establish ovn-ic") + klog.Info("start to reestablish ovn-ic") if err := c.establishInterConnection(cm.Data); err != nil { - klog.Errorf("failed to establish ovn-ic, %v", err) + klog.Errorf("failed to reestablish ovn-ic, %v", err) return } + + if err := c.RemoveOldChassisInSbDB(lastIcCm["az-name"]); err != nil { + klog.Errorf("failed to remove remote chassis: %v", err) + } + icEnabled = "true" lastIcCm = cm.Data - klog.Info("finish establishing ovn-ic") + klog.Info("finish reestablishing ovn-ic") return } + + c.ovnLegacyClient.OvnICNbAddress = genHostAddress(cm.Data["ic-db-host"], cm.Data["ic-nb-port"]) + klog.Info("start to establish ovn-ic") + if err := c.establishInterConnection(cm.Data); err != nil { + klog.Errorf("failed to establish ovn-ic, %v", err) + return + } + icEnabled = "true" + lastIcCm = cm.Data + klog.Info("finish establishing ovn-ic") } func (c *Controller) removeInterConnection(azName string) error { @@ -145,8 +146,7 @@ func (c *Controller) removeInterConnection(azName string) error { } for _, cachedNode := range nodes { no := cachedNode.DeepCopy() - patchPayloadTemplate := - `[{ + patchPayloadTemplate := `[{ "op": "%s", "path": "/metadata/labels", "value": %s @@ -186,78 +186,85 @@ func (c *Controller) establishInterConnection(config map[string]string) error { return err } - tsPort := fmt.Sprintf("ts-%s", config["az-name"]) - exist, err := c.ovnLegacyClient.LogicalSwitchPortExists(tsPort) - if err != nil { - klog.Errorf("failed to list logical switch ports, %v", err) - return err - } - if exist { - klog.Infof("ts port %s already exists", tsPort) - return nil - } - if err := c.ovnLegacyClient.SetAzName(config["az-name"]); err != nil { klog.Errorf("failed to set az name. %v", err) return err } - chassises := []string{} - gwNodes := strings.Split(config["gw-nodes"], ",") - for _, gw := range gwNodes { - gw = strings.TrimSpace(gw) - cachedNode, err := c.nodesLister.Get(gw) + groups := strings.Split(config["gw-nodes"], ";") + for i, group := range groups { + gwNodes := strings.Split(group, ",") + chassises := make([]string, len(gwNodes)) + for j, gw := range gwNodes { + gw = strings.TrimSpace(gw) + chassisID, err := c.ovnLegacyClient.GetChassis(gw) + if err != nil { + klog.Errorf("failed to get gw %s chassisID: %v", gw, err) + return err + } + if chassisID == "" { + return fmt.Errorf("no chassisID for gw %s", gw) + } + chassises[j] = chassisID + + cachedNode, err := c.nodesLister.Get(gw) + if err != nil { + klog.Errorf("failed to get gw node %s, %v", gw, err) + return err + } + node := cachedNode.DeepCopy() + patchPayloadTemplate := `[{ + "op": "%s", + "path": "/metadata/labels", + "value": %s + }]` + op := "replace" + if len(node.Labels) == 0 { + op = "add" + } + node.Labels[util.ICGatewayLabel] = "true" + raw, _ := json.Marshal(node.Labels) + patchPayload := fmt.Sprintf(patchPayloadTemplate, op, raw) + _, err = c.config.KubeClient.CoreV1().Nodes().Patch(context.Background(), gw, types.JSONPatchType, []byte(patchPayload), metav1.PatchOptions{}, "") + if err != nil { + klog.Errorf("patch gw node %s failed %v", gw, err) + return err + } + } + + tsName := c.getTSName(i) + cidr, err := c.getTSCidr(i) if err != nil { - klog.Errorf("failed to get gw node %s, %v", gw, err) + klog.Errorf("failed to get ts cidr index %d err: %v", i, err) return err } - node := cachedNode.DeepCopy() - patchPayloadTemplate := - `[{ - "op": "%s", - "path": "/metadata/labels", - "value": %s - }]` - op := "replace" - if len(node.Labels) == 0 { - op = "add" - } - node.Labels[util.ICGatewayLabel] = "true" - raw, _ := json.Marshal(node.Labels) - patchPayload := fmt.Sprintf(patchPayloadTemplate, op, raw) - _, err = c.config.KubeClient.CoreV1().Nodes().Patch(context.Background(), gw, types.JSONPatchType, []byte(patchPayload), metav1.PatchOptions{}, "") - if err != nil { - klog.Errorf("patch gw node %s failed %v", gw, err) + + if err := c.createTS(genHostAddress(config["ic-db-host"], config["ic-nb-port"]), tsName, cidr); err != nil { + klog.Errorf("failed to create ts %q: %v", tsName, err) return err } - chassisID, err := c.ovnLegacyClient.GetChassis(gw) + + tsPort := fmt.Sprintf("%s-%s", tsName, config["az-name"]) + exist, err := c.ovnLegacyClient.LogicalSwitchPortExists(tsPort) if err != nil { - klog.Errorf("failed to get gw %s chassisID, %v", gw, err) + klog.Errorf("failed to list logical switch ports, %v", err) return err } - if chassisID == "" { - return fmt.Errorf("no chassisID for gw %s", gw) + if exist { + klog.Infof("ts port %s already exists", tsPort) + continue } - chassises = append(chassises, chassisID) - } - if len(chassises) == 0 { - klog.Error("no available ic gw") - return fmt.Errorf("no available ic gw") - } - if err := c.waitTsReady(); err != nil { - klog.Errorf("failed to wait ts ready, %v", err) - return err - } - subnet, err := c.acquireLrpAddress(util.InterconnectionSwitch) - if err != nil { - klog.Errorf("failed to acquire lrp address, %v", err) - return err - } + lrpAddr, err := c.acquireLrpAddress(tsName) + if err != nil { + klog.Errorf("failed to acquire lrp address, %v", err) + return err + } - if err := c.ovnLegacyClient.CreateICLogicalRouterPort(config["az-name"], util.GenerateMac(), subnet, chassises); err != nil { - klog.Errorf("failed to create ovn-ic lrp %v", err) - return err + if err = c.ovnLegacyClient.CreateICLogicalRouterPort(config["az-name"], tsName, util.GenerateMac(), lrpAddr, chassises); err != nil { + klog.Errorf("failed to create ovn-ic lrp %v", err) + return err + } } return nil @@ -266,7 +273,7 @@ func (c *Controller) establishInterConnection(config map[string]string) error { func (c *Controller) acquireLrpAddress(ts string) (string, error) { cidr, err := c.ovnLegacyClient.GetTsSubnet(ts) if err != nil { - klog.Errorf("failed to get ts subnet, %v", err) + klog.Errorf("failed to get ts subnet %s: %v", ts, err) return "", err } existAddress, err := c.ovnLegacyClient.ListRemoteLogicalSwitchPortAddress() @@ -279,6 +286,7 @@ func (c *Controller) acquireLrpAddress(ts string) (string, error) { if !util.ContainsString(existAddress, random) { return random, nil } + klog.Infof("random ip %s already exists", random) time.Sleep(1 * time.Second) } @@ -304,7 +312,7 @@ func (c *Controller) startOVNIC(icHost, icNbPort, icSbPort string) error { } output, err := cmd.CombinedOutput() if err != nil { - return fmt.Errorf("%s", output) + return fmt.Errorf("output: %s, err: %v", output, err) } return nil } @@ -313,27 +321,30 @@ func (c *Controller) stopOVNIC() error { cmd := exec.Command("/usr/share/ovn/scripts/ovn-ctl", "stop_ic") output, err := cmd.CombinedOutput() if err != nil { - return fmt.Errorf("%s", output) + return fmt.Errorf("output: %s, err: %v", output, err) } return nil } -func (c *Controller) waitTsReady() error { - retry := 6 - for retry > 0 { - exists, err := c.ovnLegacyClient.LogicalSwitchExists(util.InterconnectionSwitch, false) - if err != nil { - klog.Errorf("failed to list logical switch, %v", err) - return err - } - if exists { - return nil - } - klog.Info("wait for ts logical switch ready") - time.Sleep(5 * time.Second) - retry = retry - 1 +func (c *Controller) createTS(icNBAddr, tsName, subnet string) error { + cmd := exec.Command("ovn-ic-nbctl", + fmt.Sprintf("--db=%s", icNBAddr), + ovs.MayExist, "ts-add", tsName, + "--", "set", "Transit_Switch", tsName, fmt.Sprintf(`external_ids:subnet="%s"`, subnet)) + if os.Getenv("ENABLE_SSL") == "true" { + cmd = exec.Command("ovn-ic-nbctl", + fmt.Sprintf("--db=%s", icNBAddr), + "--private-key=/var/run/tls/key", + "--certificate=/var/run/tls/cert", + "--ca-cert=/var/run/tls/cacert", + ovs.MayExist, "ts-add", tsName, + "--", "set", "Transit_Switch", tsName, fmt.Sprintf(`external_ids:subnet="%s"`, subnet)) + } + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("output: %s, err: %v", output, err) } - return fmt.Errorf("timeout to wait ts ready") + return nil } func (c *Controller) delLearnedRoute() error { @@ -365,9 +376,6 @@ func (c *Controller) delLearnedRoute() error { if err != nil { klog.Errorf("failed to list logical router of static route %s, %v", aLdPort["_uuid"][0], err) return err - } else if len(itsRouter) != 1 { - klog.Errorf("number wrong of logical router for static route %s, %v", aLdPort["_uuid"][0], itsRouter) - return nil } if err := c.ovnLegacyClient.DeleteStaticRoute(aLdPort["ip_prefix"][0], itsRouter[0]["name"][0]); err != nil { klog.Errorf("failed to delete stale route %s, %v", aLdPort["ip_prefix"][0], err) @@ -379,7 +387,7 @@ func (c *Controller) delLearnedRoute() error { return nil } -func genHostAddress(host string, port string) (hostAddress string) { +func genHostAddress(host, port string) (hostAddress string) { hostList := strings.Split(host, ",") if len(hostList) == 1 { hostAddress = fmt.Sprintf("tcp:[%s]:%s", hostList[0], port) @@ -388,7 +396,7 @@ func genHostAddress(host string, port string) (hostAddress string) { i := 0 for i < len(hostList)-1 { builder.WriteString(fmt.Sprintf("tcp:[%s]:%s,", hostList[i], port)) - i += 1 + i++ } builder.WriteString(fmt.Sprintf("tcp:[%s]:%s", hostList[i], port)) hostAddress = builder.String() @@ -399,30 +407,32 @@ func genHostAddress(host string, port string) (hostAddress string) { func (c *Controller) SynRouteToPolicy() { c.syncOneRouteToPolicy(util.OvnICKey, util.OvnICConnected) c.syncOneRouteToPolicy(util.OvnICKey, util.OvnICStatic) + // To support the version before kube-ovn v1.9, in which version the option tag is origin="" + c.syncOneRouteToPolicy(util.OvnICKey, util.OvnICNone) } func (c *Controller) RemoveOldChassisInSbDB(azName string) error { azUUID, err := c.ovnLegacyClient.GetAzUUID(azName) if err != nil { klog.Errorf("failed to get UUID of AZ %s: %v", lastIcCm["az-name"], err) + return err } gateways, err := c.ovnLegacyClient.GetGatewayUUIDsInOneAZ(azUUID) if err != nil { klog.Errorf("failed to get gateway UUIDs in AZ %s: %v", azUUID, err) + return err } routes, err := c.ovnLegacyClient.GetRouteUUIDsInOneAZ(azUUID) if err != nil { klog.Errorf("failed to get route UUIDs in AZ %s: %v", azUUID, err) + return err } c.ovnLegacyClient.DestroyGateways(gateways) c.ovnLegacyClient.DestroyRoutes(routes) - if err := c.ovnLegacyClient.DestroyChassis(azUUID); err != nil { - return err - } - return nil + return c.ovnLegacyClient.DestroyChassis(azUUID) } func stripPrefix(policyMatch string) (string, error) { @@ -445,13 +455,15 @@ func (c *Controller) syncOneRouteToPolicy(key, value string) { klog.Errorf("failed to list lr ovn-ic route %v", err) return } + + lrPolicyList, err := c.ovnClient.GetLogicalRouterPoliciesByExtID(key, value) + if err != nil { + klog.Errorf("failed to list ovn-ic lr policy: %v", err) + return + } + if len(lrRouteList) == 0 { klog.V(5).Info(" lr ovn-ic route does not exist") - lrPolicyList, err := c.ovnClient.GetLogicalRouterPoliciesByExtID(key, value) - if err != nil { - klog.Errorf("failed to list ovn-ic lr policy ", err) - return - } for _, lrPolicy := range lrPolicyList { if err := c.ovnClient.DeleteRouterPolicy(lr, lrPolicy.UUID); err != nil { klog.Errorf("deleting router policy failed %v", err) @@ -461,35 +473,62 @@ func (c *Controller) syncOneRouteToPolicy(key, value string) { } policyMap := map[string]string{} - lrPolicyList, err := c.ovnClient.GetLogicalRouterPoliciesByExtID(key, value) - if err != nil { - klog.Errorf("failed to list ovn-ic lr policy ", err) - return - } + for _, lrPolicy := range lrPolicyList { match, err := stripPrefix(lrPolicy.Match) if err != nil { - klog.Errorf("policy match abnormal ", err) + klog.Errorf("policy match abnormal: %v", err) continue } policyMap[match] = lrPolicy.UUID } + networks := strset.NewWithSize(len(lrRouteList)) for _, lrRoute := range lrRouteList { - if _, ok := policyMap[lrRoute.IPPrefix]; ok { - delete(policyMap, lrRoute.IPPrefix) - } else { - matchFiled := util.MatchV4Dst + " == " + lrRoute.IPPrefix - if err := c.ovnClient.AddRouterPolicy(lr, matchFiled, ovnnb.LogicalRouterPolicyActionAllow, - map[string]string{}, - map[string]string{key: value, "vendor": util.CniTypeName}, - util.OvnICPolicyPriority); err != nil { - klog.Errorf("adding router policy failed %v", err) - } - } + networks.Add(lrRoute.IPPrefix) } + + networks.Each(func(prefix string) bool { + if _, ok := policyMap[prefix]; ok { + delete(policyMap, prefix) + return true + } + match := util.MatchV4Dst + " == " + prefix + if err := c.ovnClient.AddRouterPolicy(lr, match, ovnnb.LogicalRouterPolicyActionAllow, + nil, map[string]string{key: value, "vendor": util.CniTypeName}, + util.OvnICPolicyPriority); err != nil { + klog.Errorf("failed to add router policy: %v", err) + } + + return true + }) for _, uuid := range policyMap { if err := c.ovnClient.DeleteRouterPolicy(lr, uuid); err != nil { klog.Errorf("deleting router policy failed %v", err) } } } + +func (c *Controller) getTSName(index int) string { + if index == 0 { + return util.InterconnectionSwitch + } + return fmt.Sprintf("%s%d", util.InterconnectionSwitch, index) +} + +func (c *Controller) getTSCidr(index int) (string, error) { + defaultSubnet, err := c.subnetsLister.Get(c.config.DefaultLogicalSwitch) + if err != nil { + return "", err + } + + var cidr string + switch defaultSubnet.Spec.Protocol { + case kubeovnv1.ProtocolIPv4: + cidr = fmt.Sprintf("169.254.%d.0/24", 100+index) + case kubeovnv1.ProtocolIPv6: + cidr = fmt.Sprintf("fe80:a9fe:%02x::/112", 100+index) + case kubeovnv1.ProtocolDual: + cidr = fmt.Sprintf("169.254.%d.0/24,fe80:a9fe:%02x::/112", 100+index, 100+index) + } + return cidr, nil +} diff --git a/pkg/ovs/ovn-nb-logical_router_route.go b/pkg/ovs/ovn-nb-logical_router_route.go index c4b43771c3e..1140a66ef3f 100644 --- a/pkg/ovs/ovn-nb-logical_router_route.go +++ b/pkg/ovs/ovn-nb-logical_router_route.go @@ -13,7 +13,12 @@ func (c OvnClient) GetLogicalRouterRouteByOpts(key, value string) ([]ovnnb.Logic defer cancel() api, err := c.ovnNbClient.WherePredict(ctx, func(r *ovnnb.LogicalRouterStaticRoute) bool { - return r.Options[key] == value + if len(r.Options) > 0 { + if v, ok := r.Options[key]; ok { + return v == value + } + } + return false }) if err != nil { return nil, err @@ -32,7 +37,12 @@ func (c OvnClient) GetLogicalRouterPoliciesByExtID(key, value string) ([]ovnnb.L defer cancel() api, err := c.ovnNbClient.WherePredict(ctx, func(p *ovnnb.LogicalRouterPolicy) bool { - return p.ExternalIDs[key] == value + if len(p.ExternalIDs) > 0 { + if v, ok := p.ExternalIDs[key]; ok { + return v == value + } + } + return false }) if err != nil { return nil, err diff --git a/pkg/ovs/ovn-nbctl-legacy.go b/pkg/ovs/ovn-nbctl-legacy.go index de0ee178661..0268a4453a4 100644 --- a/pkg/ovs/ovn-nbctl-legacy.go +++ b/pkg/ovs/ovn-nbctl-legacy.go @@ -128,21 +128,22 @@ func (c LegacyClient) DeleteLogicalRouterPort(port string) error { return nil } -func (c LegacyClient) CreateICLogicalRouterPort(az, mac, subnet string, chassises []string) error { - lrpName := fmt.Sprintf("%s-ts", az) +func (c LegacyClient) CreateICLogicalRouterPort(az, ts, mac, subnet string, chassises []string) error { + lspName := fmt.Sprintf("%s-%s", ts, az) + lrpName := fmt.Sprintf("%s-%s", az, ts) klog.Infof("add vpc lrp %s", lrpName) if _, err := c.ovnNbCommand(MayExist, "lrp-add", c.ClusterRouter, lrpName, mac, subnet); err != nil { return fmt.Errorf("failed to create ovn-ic lrp, %v", err) } - if _, err := c.ovnNbCommand(MayExist, "lsp-add", util.InterconnectionSwitch, fmt.Sprintf("ts-%s", az), "--", - "lsp-set-addresses", fmt.Sprintf("ts-%s", az), "router", "--", - "lsp-set-type", fmt.Sprintf("ts-%s", az), "router", "--", - "lsp-set-options", fmt.Sprintf("ts-%s", az), fmt.Sprintf("router-port=%s-ts", az), "--", - "set", "logical_switch_port", fmt.Sprintf("ts-%s", az), fmt.Sprintf("external_ids:vendor=%s", util.CniTypeName)); err != nil { + if _, err := c.ovnNbCommand(MayExist, "lsp-add", ts, lspName, "--", + "lsp-set-addresses", lspName, "router", "--", + "lsp-set-type", lspName, "router", "--", + "lsp-set-options", lspName, fmt.Sprintf("router-port=%s", lrpName), "--", + "set", "logical_switch_port", lspName, fmt.Sprintf("external_ids:vendor=%s", util.CniTypeName)); err != nil { return fmt.Errorf("failed to create ovn-ic lsp, %v", err) } for index, chassis := range chassises { - if _, err := c.ovnNbCommand("lrp-set-gateway-chassis", fmt.Sprintf("%s-ts", az), chassis, fmt.Sprintf("%d", 100-index)); err != nil { + if _, err := c.ovnNbCommand("lrp-set-gateway-chassis", lrpName, chassis, strconv.Itoa(100-index)); err != nil { return fmt.Errorf("failed to set gateway chassis, %v", err) } } @@ -150,12 +151,32 @@ func (c LegacyClient) CreateICLogicalRouterPort(az, mac, subnet string, chassise } func (c LegacyClient) DeleteICLogicalRouterPort(az string) error { - if err := c.DeleteLogicalRouterPort(fmt.Sprintf("%s-ts", az)); err != nil { - return fmt.Errorf("failed to delete ovn-ic logical router port: %v", err) + lsps, err := c.ListLogicalSwitchPort(true) + if err != nil { + klog.Errorf("failed to list logical switch port, %v", err) + return err } - if err := c.DeleteLogicalSwitchPort(fmt.Sprintf("ts-%s", az)); err != nil { - return fmt.Errorf("failed to delete ovn-ic logical switch port: %v", err) + + var icLsps []string + for _, lsp := range lsps { + names := strings.Split(lsp, "-") + if strings.HasPrefix(names[0], "ts") && names[1] == az { + icLsps = append(icLsps, lsp) + } } + + for _, icLsp := range icLsps { + names := strings.Split(icLsp, "-") + lrpName := fmt.Sprintf("%s-%s", names[1], names[0]) + + if err := c.DeleteLogicalRouterPort(lrpName); err != nil { + return fmt.Errorf("failed to delete ovn-ic logical router port: %v", err) + } + if err := c.DeleteLogicalSwitchPort(icLsp); err != nil { + return fmt.Errorf("failed to delete ovn-ic logical switch port: %v", err) + } + } + return nil } diff --git a/pkg/util/const.go b/pkg/util/const.go index 9620d7abea3..cd52d9d74ba 100644 --- a/pkg/util/const.go +++ b/pkg/util/const.go @@ -199,6 +199,7 @@ const ( OvnICKey = "origin" OvnICConnected = "connected" OvnICStatic = "static" + OvnICNone = "" MatchV4Src = "ip4.src" MatchV4Dst = "ip4.dst" diff --git a/yamls/kind.yaml.j2 b/yamls/kind.yaml.j2 index b93426fbfc5..0ec7041c7a8 100644 --- a/yamls/kind.yaml.j2 +++ b/yamls/kind.yaml.j2 @@ -10,6 +10,9 @@ {%- if ha is not defined -%} {%- set ha = "false" -%} {%- endif -%} +{%- if ovn_ic is not defined -%} + {%- set ovn_ic = "false" -%} +{%- endif -%} {%- if single is not defined -%} {%- set single = "false" -%} {%- endif -%} @@ -59,5 +62,8 @@ nodes: image: kindest/node:{{ k8s_version }} labels: kube-ovn/role: master + {%- elif ovn_ic is equalto "true" %} + - role: worker + image: kindest/node:{{ k8s_version }} {%- endif %} {%- endif %} From b6a0e8aee87b962280c83c60c9ba66d84eff4c14 Mon Sep 17 00:00:00 2001 From: changluyi Date: Mon, 8 Jan 2024 09:30:22 +0800 Subject: [PATCH 02/13] refactor all ovn ic ecmp Signed-off-by: changluyi --- Makefile | 16 +- cmd/cmdmain.go | 4 + cmd/ovn_ic_client/ovn_ic_client.go | 24 ++ dist/images/Dockerfile | 3 +- dist/images/clean-ic-az-db.sh | 47 +++ dist/images/install-ic-server.sh | 148 +++++++++ dist/images/install.sh | 98 ++++++ dist/images/kubectl-ko | 28 ++ dist/images/ovn-ic-healthcheck.sh | 28 ++ dist/images/start-ic-client.sh | 60 ++++ dist/images/start-ic-db.sh | 143 ++++++++- kubeovn-helm/templates/ic-client-deploy.yaml | 97 ++++++ pkg/controller/controller.go | 8 - pkg/daemon/gateway_linux.go | 1 - pkg/ovn_ic_client/config.go | 142 +++++++++ pkg/ovn_ic_client/controller.go | 105 +++++++ .../ovn_ic_client.go} | 283 ++++++++++-------- pkg/ovn_leader_checker/ovn.go | 144 +++++++-- pkg/ovs/ovn-ic-nbctl.go | 18 ++ pkg/ovs/ovn-ic-sbctl.go | 17 ++ pkg/ovs/ovn-nbctl-legacy.go | 54 +++- test/e2e/ovn-ic/e2e_test.go | 176 ++++++++++- yamls/kind.yaml.j2 | 2 + 23 files changed, 1456 insertions(+), 190 deletions(-) create mode 100644 cmd/ovn_ic_client/ovn_ic_client.go create mode 100644 dist/images/clean-ic-az-db.sh create mode 100644 dist/images/install-ic-server.sh create mode 100644 dist/images/ovn-ic-healthcheck.sh create mode 100644 dist/images/start-ic-client.sh create mode 100644 kubeovn-helm/templates/ic-client-deploy.yaml create mode 100644 pkg/ovn_ic_client/config.go create mode 100644 pkg/ovn_ic_client/controller.go rename pkg/{controller/ovn_ic.go => ovn_ic_client/ovn_ic_client.go} (76%) diff --git a/Makefile b/Makefile index cec0a1e5ef1..44bed8d35ae 100644 --- a/Makefile +++ b/Makefile @@ -253,7 +253,7 @@ kind-init-ipv4: kind-clean .PHONY: kind-init-ovn-ic kind-init-ovn-ic: kind-clean-ovn-ic - @ovn_ic=true $(MAKE) kind-init + @ha=true $(MAKE) kind-init @ovn_ic=true $(MAKE) kind-generate-config $(call kind_create_cluster,yamls/kind.yaml,kube-ovn1) @@ -369,17 +369,17 @@ kind-install-ovn-ic: kind-install dist/images/install.sh | bash kubectl describe no - docker run -d --name ovn-ic-db --network kind $(REGISTRY)/kube-ovn:$(VERSION) bash start-ic-db.sh - @set -e; \ - ic_db_host=$$(docker inspect ovn-ic-db -f "{{.NetworkSettings.Networks.kind.IPAddress}}"); \ - zone=az0 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn-worker,kube-ovn-worker2;kube-ovn-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-0.yaml; \ - zone=az1 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn1-worker,kube-ovn1-worker2;kube-ovn1-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-1.yaml kubectl config use-context kind-kube-ovn + bash dist/images/install-ic-server.sh + + @set -e; \ + ic_db_host=$$(kubectl get deployment ovn-ic-server -n kube-system -o jsonpath='{range .spec.template.spec.containers[0].env[?(@.name=="NODE_IPS")]}{.value}{end}'); \ + ic_db_host=$${ic_db_host%?}; \ + zone=az0 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn-worker,kube-ovn-worker2,kube-ovn-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-0.yaml; \ + zone=az1 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn1-worker,kube-ovn1-worker2,kube-ovn1-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-1.yaml kubectl apply -f ovn-ic-0.yaml kubectl config use-context kind-kube-ovn1 kubectl apply -f ovn-ic-1.yaml - sleep 6 - docker exec ovn-ic-db ovn-ic-sbctl show .PHONY: kind-install-underlay kind-install-underlay: kind-install-underlay-ipv4 diff --git a/cmd/cmdmain.go b/cmd/cmdmain.go index 0b6514daffc..da68c5dc0cc 100644 --- a/cmd/cmdmain.go +++ b/cmd/cmdmain.go @@ -8,6 +8,7 @@ import ( "github.com/kubeovn/kube-ovn/cmd/controller" "github.com/kubeovn/kube-ovn/cmd/controller_health_check" "github.com/kubeovn/kube-ovn/cmd/daemon" + "github.com/kubeovn/kube-ovn/cmd/ovn_ic_client" "github.com/kubeovn/kube-ovn/cmd/ovn_leader_checker" "github.com/kubeovn/kube-ovn/cmd/ovn_monitor" "github.com/kubeovn/kube-ovn/cmd/pinger" @@ -24,6 +25,7 @@ const ( CmdSpeaker = "kube-ovn-speaker" CmdControllerHealthCheck = "kube-ovn-controller-healthcheck" CmdOvnLeaderChecker = "kube-ovn-leader-checker" + CmdOvnICClient = "kube-ovn-ic-client" ) func main() { @@ -45,6 +47,8 @@ func main() { controller_health_check.CmdMain() case CmdOvnLeaderChecker: ovn_leader_checker.CmdMain() + case CmdOvnICClient: + ovn_ic_client.CmdMain() default: util.LogFatalAndExit(nil, "%s is an unknown command", cmd) } diff --git a/cmd/ovn_ic_client/ovn_ic_client.go b/cmd/ovn_ic_client/ovn_ic_client.go new file mode 100644 index 00000000000..db489dbbecb --- /dev/null +++ b/cmd/ovn_ic_client/ovn_ic_client.go @@ -0,0 +1,24 @@ +package ovn_ic_client + +import ( + "k8s.io/klog/v2" + "k8s.io/sample-controller/pkg/signals" + + "github.com/kubeovn/kube-ovn/pkg/ovn_ic_client" + "github.com/kubeovn/kube-ovn/pkg/util" + "github.com/kubeovn/kube-ovn/versions" +) + +func CmdMain() { + defer klog.Flush() + + klog.Infof(versions.String()) + config, err := ovn_ic_client.ParseFlags() + if err != nil { + util.LogFatalAndExit(err, "failed to parse config") + } + + stopCh := signals.SetupSignalHandler() + ctl := ovn_ic_client.NewController(config) + ctl.Run(stopCh) +} diff --git a/dist/images/Dockerfile b/dist/images/Dockerfile index 22c0e18abf2..423e9dcb4ba 100644 --- a/dist/images/Dockerfile +++ b/dist/images/Dockerfile @@ -28,4 +28,5 @@ RUN ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn && \ ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-pinger && \ ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-speaker && \ ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-controller-healthcheck && \ - ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-leader-checker + ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-leader-checker && \ + ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-ic-client diff --git a/dist/images/clean-ic-az-db.sh b/dist/images/clean-ic-az-db.sh new file mode 100644 index 00000000000..8e5e7b7c64f --- /dev/null +++ b/dist/images/clean-ic-az-db.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +if [ "$#" -ne 2 ]; then + echo "use method $0 {az|node} {azName|nodeName}" + echo " eg: ./clean-ic-az-db.sh az az1" + echo " eg: ./clean-ic-az-db.sh node kube-ovn-worker; it will delete all resource of az that the node belong to" + exit 1 +fi + +filter_type=$1 +filter_value=$2 +availability_zone_uuid= + +if [ "$filter_type" != "az" ] && [ "$filter_type" != "node" ]; then + echo "filter_type should be az or node." + exit 1 +fi + +if [ "$filter_type" == "az" ]; then + availability_zone_uuid=$(ovn-ic-sbctl --columns=_uuid find availability_zone name=$filter_value | awk '{print $3}') +fi + +echo $availability_zone_uuid + +if [ "$filter_type" == "node" ]; then + availability_zone_uuid=$(ovn-ic-sbctl --columns=availability_zone find gateway hostname=$filter_value | awk '{print $3}') +fi + +if ! ovn-ic-sbctl get availability_zone $availability_zone_uuid name >/dev/null 2>&1; then + echo "Availability zone $availability_zone_uuid not found." + exit 1 +fi + +resource_types=("Gateway" "Route" "Port_Binding") + +for resource_type in "${resource_types[@]}"; do + uuid_array=($(ovn-ic-sbctl --columns=_uuid find $resource_type availability_zone=$availability_zone_uuid | awk '{print $3}')) + + for uuid in "${uuid_array[@]}"; do + ovn-ic-sbctl destroy $resource_type $uuid + echo "Destroyed $resource_type: $uuid" + done +done + +ovn-ic-sbctl destroy availability_zone $availability_zone_uuid +echo "Destroyed availability_zone: $availability_zone_uuid" + diff --git a/dist/images/install-ic-server.sh b/dist/images/install-ic-server.sh new file mode 100644 index 00000000000..1df4c6411b4 --- /dev/null +++ b/dist/images/install-ic-server.sh @@ -0,0 +1,148 @@ +#!/usr/bin/env bash +set -euo pipefail + +REGISTRY="kubeovn" +VERSION="v1.11.13" +TS_NUM=${TS_NUM:-3} +IMAGE_PULL_POLICY="IfNotPresent" +addresses=$(kubectl get no -lkube-ovn/role=master --no-headers -o wide | awk '{print $6}' | tr \\n ',') +count=$(kubectl get no -lkube-ovn/role=master --no-headers | wc -l) +OVN_LEADER_PROBE_INTERVAL=${OVN_LEADER_PROBE_INTERVAL:-5} + +cat < ovn-ic-server.yaml +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: ovn-ic-server + namespace: kube-system + annotations: + kubernetes.io/description: | + OVN IC Server +spec: + replicas: $count + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate + selector: + matchLabels: + app: ovn-ic-server + template: + metadata: + labels: + app: ovn-ic-server + component: network + type: infra + spec: + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + - key: CriticalAddonsOnly + operator: Exists + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app: ovn-ic-server + topologyKey: kubernetes.io/hostname + priorityClassName: system-cluster-critical + serviceAccountName: ovn + hostNetwork: true + containers: + - name: ovn-ic-server + image: "$REGISTRY/kube-ovn:$VERSION" + imagePullPolicy: $IMAGE_PULL_POLICY + command: ["/kube-ovn/start-ic-db.sh"] + securityContext: + capabilities: + add: ["SYS_NICE"] + env: + - name: ENABLE_SSL + value: "false" + - name: TS_NUM + value: "$TS_NUM" + - name: NODE_IPS + value: $addresses + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: OVN_LEADER_PROBE_INTERVAL + value: "$OVN_LEADER_PROBE_INTERVAL" + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IPS + valueFrom: + fieldRef: + fieldPath: status.podIPs + resources: + requests: + cpu: 300m + memory: 200Mi + limits: + cpu: 3 + memory: 1Gi + volumeMounts: + - mountPath: /var/run/ovn + name: host-run-ovn + - mountPath: /etc/ovn + name: host-config-ovn + - mountPath: /var/log/ovn + name: host-log-ovn + - mountPath: /etc/localtime + name: localtime + - mountPath: /var/run/tls + name: kube-ovn-tls + readinessProbe: + exec: + command: + - bash + - /kube-ovn/ovn-ic-healthcheck.sh + periodSeconds: 15 + timeoutSeconds: 45 + livenessProbe: + exec: + command: + - bash + - /kube-ovn/ovn-ic-healthcheck.sh + initialDelaySeconds: 30 + periodSeconds: 15 + failureThreshold: 5 + timeoutSeconds: 4 + nodeSelector: + kubernetes.io/os: "linux" + kube-ovn/role: "master" + volumes: + - name: host-run-ovn + hostPath: + path: /run/ovn + - name: host-config-ovn + hostPath: + path: /etc/origin/ovn + - name: host-log-ovn + hostPath: + path: /var/log/ovn + - name: localtime + hostPath: + path: /etc/localtime + - name: kube-ovn-tls + secret: + optional: true + secretName: kube-ovn-tls +EOF + +kubectl apply -f ovn-ic-server.yaml +kubectl rollout status deployment/ovn-ic-server -n kube-system --timeout 600s + +echo "OVN IC Server installed Successfully" diff --git a/dist/images/install.sh b/dist/images/install.sh index 62010851109..61d4ea7d2b6 100644 --- a/dist/images/install.sh +++ b/dist/images/install.sh @@ -3559,6 +3559,104 @@ spec: optional: true secretName: kube-ovn-tls --- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: ovn-ic-client + namespace: kube-system + annotations: + kubernetes.io/description: | + OVN IC Client +spec: + replicas: 1 + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate + selector: + matchLabels: + app: ovn-ic-client + template: + metadata: + labels: + app: ovn-ic-client + component: network + type: infra + spec: + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + - key: CriticalAddonsOnly + operator: Exists + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app: ovn-ic-client + topologyKey: kubernetes.io/hostname + priorityClassName: system-cluster-critical + serviceAccountName: ovn + hostNetwork: true + containers: + - name: ovn-ic-client + image: "$REGISTRY/kube-ovn:$VERSION" + imagePullPolicy: $IMAGE_PULL_POLICY + command: ["/kube-ovn/start-ic-client.sh"] + securityContext: + capabilities: + add: ["SYS_NICE"] + env: + - name: ENABLE_SSL + value: "$ENABLE_SSL" + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OVN_DB_IPS + value: $addresses + resources: + requests: + cpu: 300m + memory: 200Mi + limits: + cpu: 3 + memory: 1Gi + volumeMounts: + - mountPath: /var/run/ovn + name: host-run-ovn + - mountPath: /etc/ovn + name: host-config-ovn + - mountPath: /var/log/ovn + name: host-log-ovn + - mountPath: /etc/localtime + name: localtime + - mountPath: /var/run/tls + name: kube-ovn-tls + nodeSelector: + kubernetes.io/os: "linux" + kube-ovn/role: "master" + volumes: + - name: host-run-ovn + hostPath: + path: /run/ovn + - name: host-config-ovn + hostPath: + path: /etc/origin/ovn + - name: host-log-ovn + hostPath: + path: /var/log/ovn + - name: localtime + hostPath: + path: /etc/localtime + - name: kube-ovn-tls + secret: + optional: true + secretName: kube-ovn-tls +--- kind: Service apiVersion: v1 metadata: diff --git a/dist/images/kubectl-ko b/dist/images/kubectl-ko index dcacc6b85d2..4005121faa6 100755 --- a/dist/images/kubectl-ko +++ b/dist/images/kubectl-ko @@ -5,6 +5,8 @@ KUBE_OVN_NS=kube-system WITHOUT_KUBE_PROXY=${WITHOUT_KUBE_PROXY:-false} OVN_NB_POD= OVN_SB_POD= +OVN_IC_NB_POD= +OVN_IC_SB_POD= KUBE_OVN_VERSION= REGISTRY="kubeovn" OVN_NORTHD_POD= @@ -34,6 +36,8 @@ showHelp(){ echo " tuning {install-fastpath|local-install-fastpath|remove-fastpath|install-stt|local-install-stt|remove-stt} {centos7|centos8}} [kernel-devel-version] deploy kernel optimisation components to the system" echo " reload restart all kube-ovn components" echo " perf [image] performance test default image is kubeovn/test:v1.12.0" + echo " icnbctl [ovn-nbctl options ...] invoke ovn-ic-nbctl" + echo " icsbctl [ovn-sbctl options ...] invoke ovn-ic-sbctl" } # usage: ipv4_to_hex 192.168.0.1 @@ -756,6 +760,22 @@ getOvnCentralPod(){ KUBE_OVN_VERSION=$(basename $image | awk -F ':' '{print $2}') } +getOVNICNBPod(){ + OVN_IC_NB_POD=$(kubectl get pod -n $KUBE_OVN_NS -l ovn-ic-nb-leader=true | grep ovn-ic-server | head -n 1 | awk '{print $1}') + if [ -z "$OVN_IC_NB_POD" ]; then + echo "ic nb leader not exists" + exit 1 + fi +} + +getOVNICSBPod(){ + OVN_IC_SB_POD=$(kubectl get pod -n $KUBE_OVN_NS -l ovn-ic-sb-leader=true | grep ovn-ic-server | head -n 1 | awk '{print $1}') + if [ -z "$OVN_IC_SB_POD" ]; then + echo "ic sb leader not exists" + exit 1 + fi +} + getOvnCentralDbStatus(){ NB_PODS=$(kubectl get pod -n $KUBE_OVN_NS | grep ovn-central | awk '{print $1}') for pod in $NB_PODS @@ -1534,6 +1554,14 @@ case $subcommand in sbctl) kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-sbctl "$@" ;; + icnbctl) + getOVNICNBPod + kubectl exec "$OVN_IC_NB_POD" -n $KUBE_OVN_NS -- ovn-ic-nbctl "$@" + ;; + icsbctl) + getOVNICSBPod + kubectl exec "$OVN_IC_SB_POD" -n $KUBE_OVN_NS -- ovn-ic-sbctl "$@" + ;; vsctl|ofctl|dpctl|appctl) xxctl "$subcommand" "$@" ;; diff --git a/dist/images/ovn-ic-healthcheck.sh b/dist/images/ovn-ic-healthcheck.sh new file mode 100644 index 00000000000..ef0b95f22b6 --- /dev/null +++ b/dist/images/ovn-ic-healthcheck.sh @@ -0,0 +1,28 @@ +#!/bin/bash +set -euo pipefail +shopt -s expand_aliases + +alias ovn-ctl='/usr/share/ovn/scripts/ovn-ctl' + +ic_nb_status=$(ovn-appctl -t /var/run/ovn/ovn_ic_nb_db.ctl cluster/status OVN_IC_Northbound | grep Status) +ic_sb_status=$(ovn-appctl -t /var/run/ovn/ovn_ic_sb_db.ctl cluster/status OVN_IC_Southbound | grep Status) +ic_nb_role=$(ovn-appctl -t /var/run/ovn/ovn_ic_nb_db.ctl cluster/status OVN_IC_Northbound | grep Role) +ic_sb_role=$(ovn-appctl -t /var/run/ovn/ovn_ic_sb_db.ctl cluster/status OVN_IC_Southbound | grep Role) + +if ! echo ${ic_nb_status} | grep -v "failed"; then + echo "nb health check failed" + exit 1 +fi +if ! echo ${ic_sb_status} | grep -v "failed"; then + echo "sb health check failed" + exit 1 +fi + +if echo ${ic_nb_status} | grep "disconnected" && echo ${ic_nb_role} | grep "candidate"; then + echo "nb health check failed" + exit 1 +fi +if echo ${ic_sb_status} | grep "disconnected" && echo ${ic_sb_role} | grep "candidate"; then + echo "sb health check failed" + exit 1 +fi diff --git a/dist/images/start-ic-client.sh b/dist/images/start-ic-client.sh new file mode 100644 index 00000000000..0bfc6b1954b --- /dev/null +++ b/dist/images/start-ic-client.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +set -euo pipefail +ENABLE_SSL=${ENABLE_SSL:-false} +OVN_DB_IPS=${OVN_DB_IPS:-} + +function gen_conn_str { + if [[ -z "${OVN_DB_IPS}" ]]; then + if [[ "$1" == "6641" ]]; then + if [[ "$ENABLE_SSL" == "false" ]]; then + x="tcp:[${OVN_NB_SERVICE_HOST}]:${OVN_NB_SERVICE_PORT}" + else + x="ssl:[${OVN_NB_SERVICE_HOST}]:${OVN_NB_SERVICE_PORT}" + fi + else + if [[ "$ENABLE_SSL" == "false" ]]; then + x="tcp:[${OVN_SB_SERVICE_HOST}]:${OVN_SB_SERVICE_PORT}" + else + x="ssl:[${OVN_SB_SERVICE_HOST}]:${OVN_SB_SERVICE_PORT}" + fi + fi + else + t=$(echo -n "${OVN_DB_IPS}" | sed 's/[[:space:]]//g' | sed 's/,/ /g') + if [[ "$ENABLE_SSL" == "false" ]]; then + x=$(for i in ${t}; do echo -n "tcp:[$i]:$1",; done| sed 's/,$//') + else + x=$(for i in ${t}; do echo -n "ssl:[$i]:$1",; done| sed 's/,$//') + fi + fi + echo "$x" +} + +nb_addr="$(gen_conn_str 6641)" +sb_addr="$(gen_conn_str 6642)" + +for ((i=0; i<3; i++)); do + if [[ "$ENABLE_SSL" == "false" ]]; then + OVN_NB_DAEMON=$(ovn-nbctl --db="$nb_addr" --pidfile --detach --overwrite-pidfile) + else + OVN_NB_DAEMON=$(ovn-nbctl -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert --db="$nb_addr" --pidfile --detach --overwrite-pidfile) + fi + if echo -n "${OVN_NB_DAEMON}" | grep -qE '^/var/run/ovn/ovn-nbctl\.[0-9]+\.ctl$'; then + export OVN_NB_DAEMON + break + fi + if [ $(echo ${OVN_NB_DAEMON} | wc -c) -gt 64 ]; then + OVN_NB_DAEMON="$(echo ${OVN_NB_DAEMON} | cut -c1-64)..." + fi + echo "invalid ovn-nbctl daemon socket: \"${OVN_NB_DAEMON}\"" + unset OVN_NB_DAEMON + pkill -f ovn-nbctl +done + +if [ -z "${OVN_NB_DAEMON}" ]; then + echo "failed to start ovn-nbctl daemon" + exit 1 +fi + +exec ./kube-ovn-ic-client --ovn-nb-addr="$nb_addr" \ + --ovn-sb-addr="$sb_addr" \ + $@ diff --git a/dist/images/start-ic-db.sh b/dist/images/start-ic-db.sh index 40c7801e0b7..72b616c3b1f 100644 --- a/dist/images/start-ic-db.sh +++ b/dist/images/start-ic-db.sh @@ -1,6 +1,15 @@ #!/bin/bash set -eo pipefail +TS_NAME=${TS_NAME:-ts} +LOCAL_IP=${LOCAL_IP:-$POD_IP} +TS_NUM=${TS_NUM:-ts} + +function get_leader_ip { + t=$(echo -n "${NODE_IPS}" | sed 's/[[:space:]]//g' | sed 's/,/ /g') + echo -n "${t}" | cut -f 1 -d " " +} + function quit { /usr/share/ovn/scripts/ovn-ctl stop_ic_ovsdb exit 0 @@ -12,13 +21,108 @@ function gen_conn_str { echo "$x" } +function ovn_db_pre_start() { + local db="" + local port="" + case $1 in + ic_nb) + db=OVN_IC_Northbound + port=6645 + ;; + ic_sb) + db=OVN_IC_Southbound + port=6646 + ;; + *) + echo "invalid database: $1" + exit 1 + ;; + esac + + local db_file="/etc/ovn/ovn${1}_db.db" + [ ! -e "$db_file" ] && return + ! ovsdb-tool db-is-clustered "$db_file" && return + ovsdb-tool check-cluster "$db_file" && return + + local db_bak="$db_file.backup-$(date +%s)-$(random_str)" + echo "backup $db_file to $db_bak" + cp "$db_file" "$db_bak" || return 1 + + echo "detected database corruption for file $db_file, try to fix it." + ovsdb-tool fix-cluster "$db_file" && return + + echo "failed to fix database file $db_file, rebuild it." + local sid=$(ovsdb-tool db-sid "$db_file") + if ! echo -n "$sid" | grep -qE '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'; then + echo "failed to get sid from $1 db file $db_file" + return 1 + fi + echo "get local server id $sid" + + local local_addr="$(gen_conn_addr $POD_IP $port)" + echo "local address: $local_addr" + + local remote_addr=() + local ips=$(echo -n "${NODE_IPS}" | sed 's/,/ /g') + for ip in ${ips[*]}; do + if [ ! "$POD_IP" = "$ip" ]; then + remote_addr=(${remote_addr[*]} "$(gen_conn_addr $ip $port)") + fi + done + echo "remote addresses: ${remote_addr[*]}" + + local db_new="$db_file.init-$(random_str)" + echo "generating new database file $db_new" + ovsdb-tool --sid $sid join-cluster "$db_new" $db $local_addr ${remote_addr[*]} || return 1 + + echo "use new database file $db_new" + mv "$db_new" "$db_file" +} + + +function is_clustered { + t=$(echo -n "${NODE_IPS}" | sed 's/,/ /g') + if [[ "$ENABLE_SSL" == "false" ]]; then + x=$(for i in ${t}; do echo -n "tcp:[${i}]:6645,"; done | sed 's/,/ /g') + for i in ${x}; + do + nb_leader=$(timeout 10 ovsdb-client query ${i} "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_IC_Northbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]") + if [[ $nb_leader =~ "true" ]] + then + return 0 + fi + done + else + x=$(for i in ${t}; do echo -n "ssl:[${i}]:6645,"; done| sed 's/,/ /g') + for i in ${x}; + do + nb_leader=$(timeout 10 ovsdb-client -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert query ${i} "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_IC_Northbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]") + if [[ $nb_leader =~ "true" ]] + then + return 0 + fi + done + fi + return 1 +} + trap quit EXIT +/usr/share/ovn/scripts/ovn-ctl stop_ic_ovsdb +ovn_db_pre_start ic_nb +ovn_db_pre_start ic_sb + if [[ -z "$NODE_IPS" && -z "$LOCAL_IP" ]]; then /usr/share/ovn/scripts/ovn-ctl --db-ic-nb-create-insecure-remote=yes --db-ic-sb-create-insecure-remote=yes start_ic_ovsdb /usr/share/ovn/scripts/ovn-ctl status_ic_ovsdb - tail --follow=name --retry /var/log/ovn/ovsdb-server-ic-nb.log else - if [[ -z "$LEADER_IP" ]]; then + ic_nb_leader_ip=$(get_leader_ip nb) + ic_sb_leader_ip=$(get_leader_ip sb) + set +eo pipefail + is_clustered + result=$? + set -eo pipefail + # leader up only when no cluster and on first node + if [[ ${result} -eq 1 && "$ic_nb_leader_ip" == "${POD_IP}" ]]; then echo "leader start with local ${LOCAL_IP} and cluster $(gen_conn_str 6647)" /usr/share/ovn/scripts/ovn-ctl --db-ic-nb-create-insecure-remote=yes \ --db-ic-sb-create-insecure-remote=yes \ @@ -28,18 +132,43 @@ else --ovn-ic-sb-db="$(gen_conn_str 6648)" \ start_ic_ovsdb /usr/share/ovn/scripts/ovn-ctl status_ic_ovsdb - tail --follow=name --retry /var/log/ovn/ovsdb-server-ic-nb.log else - echo "follower start with local ${LOCAL_IP}, leader ${LEADER_IP} and cluster $(gen_conn_str 6647)" + # known leader always first + set +eo pipefail + if [ ${result} -eq 0 ]; then + t=$(echo -n "${NODE_IPS}" | sed 's/,/ /g') + for i in ${t}; + do + ic_nb_leader=$(timeout 10 ovsdb-client query "tcp:${i}:6645" "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_IC_Northbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]") + if [[ $ic_nb_leader =~ "true" ]] + then + ic_nb_leader_ip=${i} + break + fi + done + for i in ${t}; + do + ic_sb_leader=$(timeout 10 ovsdb-client query "tcp:${i}:6646" "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_IC_Southbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]") + if [[ $ic_sb_leader =~ "true" ]] + then + ic_sb_leader_ip=${i} + break + fi + done + fi + set -eo pipefail + echo "follower start with local ${LOCAL_IP}, ovn-ic-nb leader ${ic_nb_leader_ip} ovn-ic-sb leader ${ic_sb_leader_ip}" /usr/share/ovn/scripts/ovn-ctl --db-ic-nb-create-insecure-remote=yes \ --db-ic-sb-create-insecure-remote=yes \ --db-ic-sb-cluster-local-addr="${LOCAL_IP}" \ --db-ic-nb-cluster-local-addr="${LOCAL_IP}" \ - --db-ic-nb-cluster-remote-addr="${LEADER_IP}" \ - --db-ic-sb-cluster-remote-addr="${LEADER_IP}" \ + --db-ic-nb-cluster-remote-addr="${ic_nb_leader_ip}" \ + --db-ic-sb-cluster-remote-addr="${ic_sb_leader_ip}" \ --ovn-ic-nb-db="$(gen_conn_str 6647)" \ --ovn-ic-sb-db="$(gen_conn_str 6648)" \ start_ic_ovsdb - tail --follow=name --retry /var/log/ovn/ovsdb-server-ic-nb.log fi fi + +chmod 600 /etc/ovn/* +/kube-ovn/kube-ovn-leader-checker --probeInterval=${OVN_LEADER_PROBE_INTERVAL} --isICDBServer=true diff --git a/kubeovn-helm/templates/ic-client-deploy.yaml b/kubeovn-helm/templates/ic-client-deploy.yaml new file mode 100644 index 00000000000..4d75c814682 --- /dev/null +++ b/kubeovn-helm/templates/ic-client-deploy.yaml @@ -0,0 +1,97 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: ovn-ic-client + namespace: kube-system + annotations: + kubernetes.io/description: | + OVN IC Client +spec: + replicas: 1 + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate + selector: + matchLabels: + app: ovn-ic-client + template: + metadata: + labels: + app: ovn-ic-client + component: network + type: infra + spec: + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + - key: CriticalAddonsOnly + operator: Exists + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app: ovn-ic-client + topologyKey: kubernetes.io/hostname + priorityClassName: system-cluster-critical + serviceAccountName: ovn + hostNetwork: true + containers: + - name: ovn-ic-client + image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: ["/kube-ovn/start-ic-client.sh"] + securityContext: + capabilities: + add: ["SYS_NICE"] + env: + - name: ENABLE_SSL + value: "{{ .Values.networking.ENABLE_SSL }}" + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OVN_DB_IPS + value: "{{ .Values.MASTER_NODES }}" + resources: + requests: + cpu: 300m + memory: 200Mi + limits: + cpu: 3 + memory: 1Gi + volumeMounts: + - mountPath: /var/run/ovn + name: host-run-ovn + - mountPath: /etc/ovn + name: host-config-ovn + - mountPath: /var/log/ovn + name: host-log-ovn + - mountPath: /etc/localtime + name: localtime + - mountPath: /var/run/tls + name: kube-ovn-tls + nodeSelector: + kubernetes.io/os: "linux" + kube-ovn/role: "master" + volumes: + - name: host-run-ovn + hostPath: + path: /run/ovn + - name: host-config-ovn + hostPath: + path: /etc/origin/ovn + - name: host-log-ovn + hostPath: + path: /var/log/ovn + - name: localtime + hostPath: + path: /etc/localtime + - name: kube-ovn-tls + secret: + optional: true + secretName: kube-ovn-tls \ No newline at end of file diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 8229668bccc..b76743bbe4e 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -928,14 +928,6 @@ func (c *Controller) startWorkers(ctx context.Context) { go wait.Until(c.runUpdateVlanWorker, time.Second, ctx.Done()) } - go wait.Until(func() { - c.resyncInterConnection() - }, time.Second, ctx.Done()) - - go wait.Until(func() { - c.SynRouteToPolicy() - }, 5*time.Second, ctx.Done()) - go wait.Until(func() { c.resyncExternalGateway() }, time.Second, ctx.Done()) diff --git a/pkg/daemon/gateway_linux.go b/pkg/daemon/gateway_linux.go index 41464f77023..3cfe8ca42c5 100644 --- a/pkg/daemon/gateway_linux.go +++ b/pkg/daemon/gateway_linux.go @@ -539,7 +539,6 @@ func (c *Controller) setIptables() error { ) } } - var natPreroutingRules, natPostroutingRules, ovnMasqueradeRules, manglePostroutingRules []util.IPTableRule for _, rule := range iptablesRules { if rule.Table == NAT { diff --git a/pkg/ovn_ic_client/config.go b/pkg/ovn_ic_client/config.go new file mode 100644 index 00000000000..55f55820389 --- /dev/null +++ b/pkg/ovn_ic_client/config.go @@ -0,0 +1,142 @@ +package ovn_ic_client + +import ( + "flag" + "fmt" + "os" + + "github.com/spf13/pflag" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog/v2" + + clientset "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned" + "github.com/kubeovn/kube-ovn/pkg/util" +) + +// Configuration is the controller conf +type Configuration struct { + KubeConfigFile string + KubeClient kubernetes.Interface + KubeOvnClient clientset.Interface + + PodNamespace string + OvnNbAddr string + OvnSbAddr string + OvnTimeout int + + NodeSwitch string + ClusterRouter string + NodeSwitchCIDR string + + ClusterTcpLoadBalancer string + ClusterUdpLoadBalancer string + ClusterTcpSessionLoadBalancer string + ClusterUdpSessionLoadBalancer string +} + +func ParseFlags() (*Configuration, error) { + var ( + argKubeConfigFile = pflag.String("kubeconfig", "", "Path to kubeconfig file with authorization and master location information. If not set use the inCluster token.") + + argOvnNbAddr = pflag.String("ovn-nb-addr", "", "ovn-nb address") + argOvnSbAddr = pflag.String("ovn-sb-addr", "", "ovn-sb address") + argOvnTimeout = pflag.Int("ovn-timeout", 60, "") + + argClusterRouter = pflag.String("cluster-router", util.DefaultVpc, "The router name for cluster router") + argNodeSwitch = pflag.String("node-switch", "join", "The name of node gateway switch which help node to access pod network") + argNodeSwitchCIDR = pflag.String("node-switch-cidr", "100.64.0.0/16", "The cidr for node switch") + + argClusterTcpLoadBalancer = pflag.String("cluster-tcp-loadbalancer", "cluster-tcp-loadbalancer", "The name for cluster tcp loadbalancer") + argClusterUdpLoadBalancer = pflag.String("cluster-udp-loadbalancer", "cluster-udp-loadbalancer", "The name for cluster udp loadbalancer") + argClusterTcpSessionLoadBalancer = pflag.String("cluster-tcp-session-loadbalancer", "cluster-tcp-session-loadbalancer", "The name for cluster tcp session loadbalancer") + argClusterUdpSessionLoadBalancer = pflag.String("cluster-udp-session-loadbalancer", "cluster-udp-session-loadbalancer", "The name for cluster udp session loadbalancer") + ) + + klogFlags := flag.NewFlagSet("klog", flag.ContinueOnError) + klog.InitFlags(klogFlags) + + // Sync the glog and klog flags. + pflag.CommandLine.VisitAll(func(f1 *pflag.Flag) { + f2 := klogFlags.Lookup(f1.Name) + if f2 != nil { + value := f1.Value.String() + if err := f2.Value.Set(value); err != nil { + klog.Errorf("failed to set flag %v", err) + } + } + }) + + // change the behavior of cmdline + // not exit. not good + pflag.CommandLine.Init(os.Args[0], pflag.ContinueOnError) + pflag.CommandLine.AddGoFlagSet(klogFlags) + pflag.CommandLine.AddGoFlagSet(flag.CommandLine) + + if err := pflag.CommandLine.Parse(os.Args[1:]); err != nil { + return nil, err + } + + config := &Configuration{ + KubeConfigFile: *argKubeConfigFile, + + PodNamespace: os.Getenv("POD_NAMESPACE"), + OvnNbAddr: *argOvnNbAddr, + OvnSbAddr: *argOvnSbAddr, + OvnTimeout: *argOvnTimeout, + + ClusterRouter: *argClusterRouter, + NodeSwitch: *argNodeSwitch, + NodeSwitchCIDR: *argNodeSwitchCIDR, + + ClusterTcpLoadBalancer: *argClusterTcpLoadBalancer, + ClusterUdpLoadBalancer: *argClusterUdpLoadBalancer, + ClusterTcpSessionLoadBalancer: *argClusterTcpSessionLoadBalancer, + ClusterUdpSessionLoadBalancer: *argClusterUdpSessionLoadBalancer, + } + + if err := config.initKubeClient(); err != nil { + return nil, fmt.Errorf("failed to init kube client, %v", err) + } + + return config, nil +} + +func (config *Configuration) initKubeClient() error { + var cfg *rest.Config + var err error + if config.KubeConfigFile == "" { + klog.Infof("no --kubeconfig, use in-cluster kubernetes config") + cfg, err = rest.InClusterConfig() + if err != nil { + klog.Errorf("use in cluster config failed %v", err) + return err + } + } else { + cfg, err = clientcmd.BuildConfigFromFlags("", config.KubeConfigFile) + if err != nil { + klog.Errorf("use --kubeconfig %s failed %v", config.KubeConfigFile, err) + return err + } + } + cfg.QPS = 1000 + cfg.Burst = 2000 + + kubeOvnClient, err := clientset.NewForConfig(cfg) + if err != nil { + klog.Errorf("init kubeovn client failed %v", err) + return err + } + config.KubeOvnClient = kubeOvnClient + + cfg.ContentType = "application/vnd.kubernetes.protobuf" + cfg.AcceptContentTypes = "application/vnd.kubernetes.protobuf,application/json" + kubeClient, err := kubernetes.NewForConfig(cfg) + if err != nil { + klog.Errorf("init kubernetes client failed %v", err) + return err + } + config.KubeClient = kubeClient + return nil +} diff --git a/pkg/ovn_ic_client/controller.go b/pkg/ovn_ic_client/controller.go new file mode 100644 index 00000000000..04686f0c36b --- /dev/null +++ b/pkg/ovn_ic_client/controller.go @@ -0,0 +1,105 @@ +package ovn_ic_client + +import ( + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + kubeinformers "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/scheme" + typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" + listerv1 "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/record" + "k8s.io/klog/v2" + + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + kubeovninformer "github.com/kubeovn/kube-ovn/pkg/client/informers/externalversions" + kubeovnlister "github.com/kubeovn/kube-ovn/pkg/client/listers/kubeovn/v1" + "github.com/kubeovn/kube-ovn/pkg/ovs" + "github.com/kubeovn/kube-ovn/pkg/util" +) + +const controllerAgentName = "ovn-ic-client" + +type Controller struct { + config *Configuration + + subnetsLister kubeovnlister.SubnetLister + subnetSynced cache.InformerSynced + nodesLister listerv1.NodeLister + nodesSynced cache.InformerSynced + configMapsLister listerv1.ConfigMapLister + configMapsSynced cache.InformerSynced + + informerFactory kubeinformers.SharedInformerFactory + kubeovnInformerFactory kubeovninformer.SharedInformerFactory + recorder record.EventRecorder + + ovnClient *ovs.OvnClient + ovnLegacyClient *ovs.LegacyClient +} + +func NewController(config *Configuration) *Controller { + utilruntime.Must(kubeovnv1.AddToScheme(scheme.Scheme)) + klog.V(4).Info("Creating event broadcaster") + eventBroadcaster := record.NewBroadcaster() + eventBroadcaster.StartLogging(klog.Infof) + eventBroadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: config.KubeClient.CoreV1().Events("")}) + recorder := eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: controllerAgentName}) + + informerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(config.KubeClient, 0, + kubeinformers.WithTweakListOptions(func(listOption *metav1.ListOptions) { + listOption.AllowWatchBookmarks = true + })) + kubeovnInformerFactory := kubeovninformer.NewSharedInformerFactoryWithOptions(config.KubeOvnClient, 0, + kubeovninformer.WithTweakListOptions(func(listOption *metav1.ListOptions) { + listOption.AllowWatchBookmarks = true + })) + + nodeInformer := informerFactory.Core().V1().Nodes() + subnetInformer := kubeovnInformerFactory.Kubeovn().V1().Subnets() + configMapInformer := informerFactory.Core().V1().ConfigMaps() + + controller := &Controller{ + config: config, + ovnLegacyClient: ovs.NewLegacyClient(config.OvnNbAddr, config.OvnTimeout, config.OvnSbAddr, config.ClusterRouter, config.ClusterTcpLoadBalancer, config.ClusterUdpLoadBalancer, config.ClusterTcpSessionLoadBalancer, config.ClusterUdpSessionLoadBalancer, config.NodeSwitch, config.NodeSwitchCIDR), + + subnetsLister: subnetInformer.Lister(), + subnetSynced: subnetInformer.Informer().HasSynced, + nodesLister: nodeInformer.Lister(), + nodesSynced: nodeInformer.Informer().HasSynced, + configMapsLister: configMapInformer.Lister(), + configMapsSynced: configMapInformer.Informer().HasSynced, + + informerFactory: informerFactory, + kubeovnInformerFactory: kubeovnInformerFactory, + recorder: recorder, + } + + var err error + if controller.ovnClient, err = ovs.NewOvnClient(config.OvnNbAddr, config.OvnTimeout); err != nil { + util.LogFatalAndExit(err, "failed to create ovn client") + } + + return controller +} + +func (c *Controller) Run(stopCh <-chan struct{}) { + defer utilruntime.HandleCrash() + c.informerFactory.Start(stopCh) + c.kubeovnInformerFactory.Start(stopCh) + + if !cache.WaitForCacheSync(stopCh, c.subnetSynced, c.nodesSynced) { + util.LogFatalAndExit(nil, "failed to wait for caches to sync") + return + } + + klog.Info("Started workers") + go wait.Until(c.resyncInterConnection, time.Second, stopCh) + go wait.Until(c.SynRouteToPolicy, 5*time.Second, stopCh) + <-stopCh + klog.Info("Shutting down workers") +} diff --git a/pkg/controller/ovn_ic.go b/pkg/ovn_ic_client/ovn_ic_client.go similarity index 76% rename from pkg/controller/ovn_ic.go rename to pkg/ovn_ic_client/ovn_ic_client.go index 9466fc48d64..523b8f39278 100644 --- a/pkg/controller/ovn_ic.go +++ b/pkg/ovn_ic_client/ovn_ic_client.go @@ -1,4 +1,4 @@ -package controller +package ovn_ic_client import ( "context" @@ -7,6 +7,7 @@ import ( "os" "os/exec" "reflect" + "sort" "strings" "time" @@ -17,8 +18,6 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/klog/v2" - kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" - "github.com/kubeovn/kube-ovn/pkg/ovs" "github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb" "github.com/kubeovn/kube-ovn/pkg/util" ) @@ -26,45 +25,34 @@ import ( var ( icEnabled = "unknown" lastIcCm map[string]string + lastTSs []string + curTSs []string ) -func (c *Controller) resyncInterConnection() { - cm, err := c.configMapsLister.ConfigMaps(c.config.PodNamespace).Get(util.InterconnectionConfig) - if err != nil && !k8serrors.IsNotFound(err) { - klog.Errorf("failed to get ovn-ic-config, %v", err) +const ( + icNoAction = iota + icFirstEstablish + icConfigChange +) + +func (c *Controller) disableOVNIC(azName string) { + if err := c.removeInterConnection(azName); err != nil { + klog.Errorf("failed to remove ovn-ic, %v", err) return } - - if k8serrors.IsNotFound(err) || cm.Data["enable-ic"] == "false" { - if icEnabled == "false" { - return - } - klog.Info("start to remove ovn-ic") - azName := "" - if cm != nil { - azName = cm.Data["az-name"] - } else if lastIcCm != nil { - azName = lastIcCm["az-name"] - } - if err := c.removeInterConnection(azName); err != nil { - klog.Errorf("failed to remove ovn-ic, %v", err) - return - } - if err := c.delLearnedRoute(); err != nil { - klog.Errorf("failed to remove learned static routes, %v", err) - return - } - icEnabled = "false" - lastIcCm = nil - - klog.Info("finish removing ovn-ic") + if err := c.delLearnedRoute(); err != nil { + klog.Errorf("failed to remove learned static routes, %v", err) return } - blackList := []string{} - autoRoute := false - if cm.Data["auto-route"] == "true" { - autoRoute = true + + if err := c.RemoveOldChassisInSbDB(azName); err != nil { + klog.Errorf("failed to remove remote chassis: %v", err) } +} + +func (c *Controller) setAutoRoute(autoRoute bool) { + + var blackList []string subnets, err := c.subnetsLister.List(labels.Everything()) if err != nil { klog.Errorf("failed to list subnets, %v", err) @@ -93,48 +81,108 @@ func (c *Controller) resyncInterConnection() { klog.Errorf("failed to config auto route, %v", err) return } +} - isCMEqual := reflect.DeepEqual(cm.Data, lastIcCm) - if icEnabled == "true" && lastIcCm != nil && isCMEqual { +func (c *Controller) getICState(cmData, lastcmData map[string]string) int { + isCMEqual := reflect.DeepEqual(cmData, lastcmData) + isTsEqual := true + if icEnabled != "true" && len(lastcmData) == 0 && cmData["enable-ic"] == "true" { + return icFirstEstablish + } + + if icEnabled == "true" && lastcmData != nil && isCMEqual { + var err error + c.ovnLegacyClient.OvnICNbAddress = genHostAddress(cmData["ic-db-host"], cmData["ic-nb-port"]) + curTSs, err = c.ovnLegacyClient.GetTs() + if err != nil { + klog.Errorf("failed to get Transit_Switch, %v", err) + return icNoAction + } + isTsEqual = reflect.DeepEqual(lastTSs, curTSs) + if isTsEqual { + return icNoAction + } + } + return icConfigChange +} + +func (c *Controller) resyncInterConnection() { + cm, err := c.configMapsLister.ConfigMaps(c.config.PodNamespace).Get(util.InterconnectionConfig) + if err != nil && !k8serrors.IsNotFound(err) { + klog.Errorf("failed to get ovn-ic-config, %v", err) return } - if icEnabled == "true" && lastIcCm != nil && !isCMEqual { - if err := c.removeInterConnection(lastIcCm["az-name"]); err != nil { - klog.Errorf("failed to remove ovn-ic, %v", err) + + if k8serrors.IsNotFound(err) || cm.Data["enable-ic"] == "false" { + if icEnabled == "false" { return } - if err := c.delLearnedRoute(); err != nil { - klog.Errorf("failed to remove learned static routes, %v", err) - return + klog.Info("start to remove ovn-ic") + azName := "" + icDBHost := "" + if cm != nil { + azName = cm.Data["az-name"] + icDBHost = cm.Data["ic-db-host"] + } else if lastIcCm != nil { + azName = lastIcCm["az-name"] + icDBHost = lastIcCm["ic-db-host"] + } + + if icDBHost != "" { + c.ovnLegacyClient.OvnICSbAddress = genHostAddress(icDBHost, cm.Data["ic-sb-port"]) + c.ovnLegacyClient.OvnICNbAddress = genHostAddress(icDBHost, cm.Data["ic-nb-port"]) } - c.ovnLegacyClient.OvnICSbAddress = genHostAddress(cm.Data["ic-db-host"], cm.Data["ic-sb-port"]) + c.disableOVNIC(azName) + icEnabled = "false" + lastIcCm = nil + + klog.Info("finish removing ovn-ic") + return + } + + autoRoute := false + if cm.Data["auto-route"] == "true" { + autoRoute = true + } + c.setAutoRoute(autoRoute) + + switch c.getICState(cm.Data, lastIcCm) { + case icNoAction: + return + case icFirstEstablish: c.ovnLegacyClient.OvnICNbAddress = genHostAddress(cm.Data["ic-db-host"], cm.Data["ic-nb-port"]) + klog.Info("start to establish ovn-ic") + if err := c.establishInterConnection(cm.Data); err != nil { + klog.Errorf("failed to establish ovn-ic, %v", err) + return + } + curTSs, err := c.ovnLegacyClient.GetTs() + if err != nil { + klog.Errorf("failed to get Transit_Switch, %v", err) + return + } + icEnabled = "true" + lastIcCm = cm.Data + lastTSs = curTSs + klog.Info("finish establishing ovn-ic") + return + case icConfigChange: + c.ovnLegacyClient.OvnICSbAddress = genHostAddress(lastIcCm["ic-db-host"], cm.Data["ic-sb-port"]) + c.ovnLegacyClient.OvnICNbAddress = genHostAddress(lastIcCm["ic-db-host"], cm.Data["ic-nb-port"]) + c.disableOVNIC(lastIcCm["az-name"]) klog.Info("start to reestablish ovn-ic") if err := c.establishInterConnection(cm.Data); err != nil { klog.Errorf("failed to reestablish ovn-ic, %v", err) return } - if err := c.RemoveOldChassisInSbDB(lastIcCm["az-name"]); err != nil { - klog.Errorf("failed to remove remote chassis: %v", err) - } - icEnabled = "true" lastIcCm = cm.Data + lastTSs = curTSs klog.Info("finish reestablishing ovn-ic") return } - - c.ovnLegacyClient.OvnICNbAddress = genHostAddress(cm.Data["ic-db-host"], cm.Data["ic-nb-port"]) - klog.Info("start to establish ovn-ic") - if err := c.establishInterConnection(cm.Data); err != nil { - klog.Errorf("failed to establish ovn-ic, %v", err) - return - } - icEnabled = "true" - lastIcCm = cm.Data - klog.Info("finish establishing ovn-ic") } func (c *Controller) removeInterConnection(azName string) error { @@ -165,18 +213,18 @@ func (c *Controller) removeInterConnection(azName string) error { } } - if azName != "" { - if err := c.ovnLegacyClient.DeleteICLogicalRouterPort(azName); err != nil { - klog.Errorf("failed to delete ovn-ic lrp, %v", err) - return err - } - } - if err := c.stopOVNIC(); err != nil { klog.Errorf("failed to stop ovn-ic, %v", err) return err } + if azName != "" { + if err := c.ovnLegacyClient.DeleteICResources(azName); err != nil { + klog.Errorf("failed to delete ovn-ic resource on az %s , %v", azName, err) + return err + } + } + return nil } @@ -191,11 +239,20 @@ func (c *Controller) establishInterConnection(config map[string]string) error { return err } - groups := strings.Split(config["gw-nodes"], ";") - for i, group := range groups { - gwNodes := strings.Split(group, ",") - chassises := make([]string, len(gwNodes)) - for j, gw := range gwNodes { + tsNames, err := c.ovnLegacyClient.GetTs() + if err != nil { + klog.Errorf("failed to list ic logical switch. %v ", err) + return err + } + + sort.Strings(tsNames) + + gwNodes := strings.Split(config["gw-nodes"], ",") + chassises := make([]string, len(gwNodes)) + + for i, tsName := range tsNames { + gwNodesOrdered := moveElements(gwNodes, i) + for j, gw := range gwNodesOrdered { gw = strings.TrimSpace(gw) chassisID, err := c.ovnLegacyClient.GetChassis(gw) if err != nil { @@ -232,18 +289,6 @@ func (c *Controller) establishInterConnection(config map[string]string) error { } } - tsName := c.getTSName(i) - cidr, err := c.getTSCidr(i) - if err != nil { - klog.Errorf("failed to get ts cidr index %d err: %v", i, err) - return err - } - - if err := c.createTS(genHostAddress(config["ic-db-host"], config["ic-nb-port"]), tsName, cidr); err != nil { - klog.Errorf("failed to create ts %q: %v", tsName, err) - return err - } - tsPort := fmt.Sprintf("%s-%s", tsName, config["az-name"]) exist, err := c.ovnLegacyClient.LogicalSwitchPortExists(tsPort) if err != nil { @@ -286,7 +331,6 @@ func (c *Controller) acquireLrpAddress(ts string) (string, error) { if !util.ContainsString(existAddress, random) { return random, nil } - klog.Infof("random ip %s already exists", random) time.Sleep(1 * time.Second) } @@ -326,27 +370,6 @@ func (c *Controller) stopOVNIC() error { return nil } -func (c *Controller) createTS(icNBAddr, tsName, subnet string) error { - cmd := exec.Command("ovn-ic-nbctl", - fmt.Sprintf("--db=%s", icNBAddr), - ovs.MayExist, "ts-add", tsName, - "--", "set", "Transit_Switch", tsName, fmt.Sprintf(`external_ids:subnet="%s"`, subnet)) - if os.Getenv("ENABLE_SSL") == "true" { - cmd = exec.Command("ovn-ic-nbctl", - fmt.Sprintf("--db=%s", icNBAddr), - "--private-key=/var/run/tls/key", - "--certificate=/var/run/tls/cert", - "--ca-cert=/var/run/tls/cacert", - ovs.MayExist, "ts-add", tsName, - "--", "set", "Transit_Switch", tsName, fmt.Sprintf(`external_ids:subnet="%s"`, subnet)) - } - output, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("output: %s, err: %v", output, err) - } - return nil -} - func (c *Controller) delLearnedRoute() error { originalPorts, err := c.ovnLegacyClient.CustomFindEntity("Logical_Router_Static_Route", []string{"_uuid", "ip_prefix"}) if err != nil { @@ -377,9 +400,12 @@ func (c *Controller) delLearnedRoute() error { klog.Errorf("failed to list logical router of static route %s, %v", aLdPort["_uuid"][0], err) return err } - if err := c.ovnLegacyClient.DeleteStaticRoute(aLdPort["ip_prefix"][0], itsRouter[0]["name"][0]); err != nil { - klog.Errorf("failed to delete stale route %s, %v", aLdPort["ip_prefix"][0], err) - return err + + if len(aLdPort["ip_prefix"]) != 0 && len(itsRouter) != 0 && len(itsRouter[0]["name"]) != 0 { + if err := c.ovnLegacyClient.DeleteStaticRoute(aLdPort["ip_prefix"][0], itsRouter[0]["name"][0]); err != nil { + klog.Errorf("failed to delete stale route %s, %v", aLdPort["ip_prefix"][0], err) + return err + } } } klog.V(5).Infof("finish removing learned routes") @@ -412,12 +438,21 @@ func (c *Controller) SynRouteToPolicy() { } func (c *Controller) RemoveOldChassisInSbDB(azName string) error { + if azName == "" { + return nil + } + azUUID, err := c.ovnLegacyClient.GetAzUUID(azName) if err != nil { klog.Errorf("failed to get UUID of AZ %s: %v", lastIcCm["az-name"], err) return err } + if azUUID == "" { + klog.Infof("%s have already been deleted", azName) + return nil + } + gateways, err := c.ovnLegacyClient.GetGatewayUUIDsInOneAZ(azUUID) if err != nil { klog.Errorf("failed to get gateway UUIDs in AZ %s: %v", azUUID, err) @@ -430,6 +465,13 @@ func (c *Controller) RemoveOldChassisInSbDB(azName string) error { return err } + portBindings, err := c.ovnLegacyClient.GetPortBindingUUIDsInOneAZ(azUUID) + if err != nil { + klog.Errorf("failed to get Port_Binding UUIDs in AZ %s: %v", azUUID, err) + return err + } + + c.ovnLegacyClient.DestroyPortBindings(portBindings) c.ovnLegacyClient.DestroyGateways(gateways) c.ovnLegacyClient.DestroyRoutes(routes) return c.ovnLegacyClient.DestroyChassis(azUUID) @@ -447,7 +489,7 @@ func stripPrefix(policyMatch string) (string, error) { func (c *Controller) syncOneRouteToPolicy(key, value string) { lr, err := c.ovnClient.GetLogicalRouter(util.DefaultVpc, false) if err != nil { - klog.Errorf("logical router does not exist %v at %v", err, time.Now()) + klog.Infof("logical router %s is not ready at %v", util.DefaultVpc, time.Now()) return } lrRouteList, err := c.ovnClient.GetLogicalRouterRouteByOpts(key, value) @@ -508,27 +550,10 @@ func (c *Controller) syncOneRouteToPolicy(key, value string) { } } -func (c *Controller) getTSName(index int) string { - if index == 0 { - return util.InterconnectionSwitch - } - return fmt.Sprintf("%s%d", util.InterconnectionSwitch, index) -} - -func (c *Controller) getTSCidr(index int) (string, error) { - defaultSubnet, err := c.subnetsLister.Get(c.config.DefaultLogicalSwitch) - if err != nil { - return "", err +func moveElements(arr []string, order int) []string { + if order >= len(arr) { + order = order % len(arr) } - var cidr string - switch defaultSubnet.Spec.Protocol { - case kubeovnv1.ProtocolIPv4: - cidr = fmt.Sprintf("169.254.%d.0/24", 100+index) - case kubeovnv1.ProtocolIPv6: - cidr = fmt.Sprintf("fe80:a9fe:%02x::/112", 100+index) - case kubeovnv1.ProtocolDual: - cidr = fmt.Sprintf("169.254.%d.0/24,fe80:a9fe:%02x::/112", 100+index, 100+index) - } - return cidr, nil + return append(arr[order:], arr[:order]...) } diff --git a/pkg/ovn_leader_checker/ovn.go b/pkg/ovn_leader_checker/ovn.go index 078bc2bb5db..28177c2e245 100755 --- a/pkg/ovn_leader_checker/ovn.go +++ b/pkg/ovn_leader_checker/ovn.go @@ -23,6 +23,7 @@ import ( "k8s.io/client-go/tools/clientcmd" "k8s.io/klog/v2" + "github.com/kubeovn/kube-ovn/pkg/ovs" "github.com/kubeovn/kube-ovn/pkg/util" ) @@ -44,6 +45,7 @@ type Configuration struct { KubeClient kubernetes.Interface ProbeInterval int EnableCompact bool + ISICDBServer bool } // ParseFlags parses cmd args then init kubeclient and conf @@ -53,6 +55,7 @@ func ParseFlags() (*Configuration, error) { argKubeConfigFile = pflag.String("kubeconfig", "", "Path to kubeconfig file with authorization and master location information. If not set use the inCluster token.") argProbeInterval = pflag.Int("probeInterval", DefaultProbeInterval, "interval of probing leader in seconds") argEnableCompact = pflag.Bool("enableCompact", true, "is enable compact") + argIsICDBServer = pflag.Bool("isICDBServer", false, "is ic db server ") ) klogFlags := flag.NewFlagSet("klog", flag.ContinueOnError) @@ -83,6 +86,7 @@ func ParseFlags() (*Configuration, error) { KubeConfigFile: *argKubeConfigFile, ProbeInterval: *argProbeInterval, EnableCompact: *argEnableCompact, + ISICDBServer: *argIsICDBServer, } return config, nil } @@ -351,7 +355,7 @@ func doOvnLeaderCheck(cfg *Configuration, podName string, podNamespace string) { util.LogFatalAndExit(nil, "preValidChkCfg: invalid cfg") } - if !checkOvnIsAlive() { + if !cfg.ISICDBServer && !checkOvnIsAlive() { klog.Errorf("ovn is not alive") return } @@ -366,27 +370,47 @@ func doOvnLeaderCheck(cfg *Configuration, podName string, podNamespace string) { for k, v := range cachedPod.Labels { labels[k] = v } - nbLeader := isDBLeader("OVN_Northbound", 6641) - sbLeader := isDBLeader("OVN_Southbound", 6642) - northdLeader := checkNorthdActive() - updatePodLabels(labels, "ovn-nb-leader", nbLeader) - updatePodLabels(labels, "ovn-sb-leader", sbLeader) - updatePodLabels(labels, "ovn-northd-leader", northdLeader) - if err = patchPodLabels(cfg, cachedPod, labels); err != nil { - klog.Errorf("patch label error %v", err) - return - } - if sbLeader && checkNorthdSvcExist(cfg, podNamespace, "ovn-northd") { - if !checkNorthdEpAlive(cfg, podNamespace, "ovn-northd") { - klog.Warning("no available northd leader, try to release the lock") - stealLock() + + if !cfg.ISICDBServer { + nbLeader := isDBLeader("OVN_Northbound", 6641) + sbLeader := isDBLeader("OVN_Southbound", 6642) + northdLeader := checkNorthdActive() + updatePodLabels(labels, "ovn-nb-leader", nbLeader) + updatePodLabels(labels, "ovn-sb-leader", sbLeader) + updatePodLabels(labels, "ovn-northd-leader", northdLeader) + if err = patchPodLabels(cfg, cachedPod, labels); err != nil { + klog.Errorf("patch label error %v", err) + return + } + if sbLeader && checkNorthdSvcExist(cfg, podNamespace, "ovn-northd") { + if !checkNorthdEpAlive(cfg, podNamespace, "ovn-northd") { + klog.Warning("no available northd leader, try to release the lock") + stealLock() + } } - } - if cfg.EnableCompact { - compactOvnDatabase("nb") - compactOvnDatabase("sb") + if cfg.EnableCompact { + compactOvnDatabase("nb") + compactOvnDatabase("sb") + } + } else { + icNbLeader := isDBLeader("OVN_IC_Northbound", 6645) + icSbLeader := isDBLeader("OVN_IC_Southbound", 6646) + updatePodLabels(labels, "ovn-ic-nb-leader", icNbLeader) + updatePodLabels(labels, "ovn-ic-sb-leader", icSbLeader) + if err = patchPodLabels(cfg, cachedPod, labels); err != nil { + klog.Errorf("patch label error %v", err) + return + } + + if icNbLeader { + if err := updateTS(); err != nil { + klog.Errorf("update ts num failed err: %v ", err) + return + } + } } + } func StartOvnLeaderCheck(cfg *Configuration) { @@ -398,3 +422,85 @@ func StartOvnLeaderCheck(cfg *Configuration) { time.Sleep(interval) } } + +func getTSName(index int) string { + if index == 0 { + return util.InterconnectionSwitch + } + return fmt.Sprintf("%s%d", util.InterconnectionSwitch, index) +} + +func getTSCidr(index int) string { + cidr := fmt.Sprintf("169.254.%d.0/24", 100+index) + return cidr +} + +func updateTS() error { + cmdstr := "ovn-ic-nbctl show | wc -l" + cmd := exec.Command("sh", "-c", cmdstr) + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("ovn-ic-nbctl show output: %s, err: %v", output, err) + } + lines := strings.Split(string(output), "\n") + if len(lines) < 1 { + return fmt.Errorf("ovsn-ic-nbctl show count line: err") + } + + existTSCount, err := strconv.Atoi(lines[0]) + + if err != nil { + return fmt.Errorf("existTSCount atoi failed output: %s, err: %v", output, err) + } + + expectTSCount, err := strconv.Atoi(os.Getenv("TS_NUM")) + + if err != nil { + return fmt.Errorf("expectTSCount atoi failed output: %s, err: %v", output, err) + } + + if expectTSCount == existTSCount { + return nil + } + + if expectTSCount > existTSCount { + for i := expectTSCount - 1; i > existTSCount-1; i-- { + tsName := getTSName(i) + subnet := getTSCidr(i) + cmd := exec.Command("ovn-ic-nbctl", + ovs.MayExist, "ts-add", tsName, + "--", "set", "Transit_Switch", tsName, fmt.Sprintf(`external_ids:subnet="%s"`, subnet)) + if os.Getenv("ENABLE_SSL") == "true" { + cmd = exec.Command("ovn-ic-nbctl", + "--private-key=/var/run/tls/key", + "--certificate=/var/run/tls/cert", + "--ca-cert=/var/run/tls/cacert", + ovs.MayExist, "ts-add", tsName, + "--", "set", "Transit_Switch", tsName, fmt.Sprintf(`external_ids:subnet="%s"`, subnet)) + } + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("output: %s, err: %v", output, err) + } + } + } else if expectTSCount < existTSCount { + for i := existTSCount - 1; i >= expectTSCount; i-- { + tsName := getTSName(i) + cmd := exec.Command("ovn-ic-nbctl", + "ts-del", tsName) + if os.Getenv("ENABLE_SSL") == "true" { + cmd = exec.Command("ovn-ic-nbctl", + "--private-key=/var/run/tls/key", + "--certificate=/var/run/tls/cert", + "--ca-cert=/var/run/tls/cacert", + "ts-del", tsName) + } + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("output: %s, err: %v", output, err) + } + } + } + + return nil +} diff --git a/pkg/ovs/ovn-ic-nbctl.go b/pkg/ovs/ovn-ic-nbctl.go index 2b4ebd51563..c6c58007803 100644 --- a/pkg/ovs/ovn-ic-nbctl.go +++ b/pkg/ovs/ovn-ic-nbctl.go @@ -44,3 +44,21 @@ func (c LegacyClient) GetTsSubnet(ts string) (string, error) { } return subnet, nil } + +func (c LegacyClient) GetTs() ([]string, error) { + cmd := []string{"--format=csv", "--data=bare", "--no-heading", "--columns=name", "find", "Transit_Switch"} + output, err := c.ovnIcNbCommand(cmd...) + if err != nil { + klog.Errorf("failed to list logical switch port, %v", err) + return nil, err + } + lines := strings.Split(output, "\n") + result := make([]string, 0, len(lines)) + for _, l := range lines { + if len(strings.TrimSpace(l)) == 0 { + continue + } + result = append(result, strings.TrimSpace(l)) + } + return result, nil +} diff --git a/pkg/ovs/ovn-ic-sbctl.go b/pkg/ovs/ovn-ic-sbctl.go index 63db80223c1..c0f3dffb3e7 100644 --- a/pkg/ovs/ovn-ic-sbctl.go +++ b/pkg/ovs/ovn-ic-sbctl.go @@ -91,6 +91,14 @@ func (c LegacyClient) GetRouteUUIDsInOneAZ(uuid string) ([]string, error) { return routes, nil } +func (c LegacyClient) GetPortBindingUUIDsInOneAZ(uuid string) ([]string, error) { + portBindings, err := c.FindUUIDWithAttrInTable("availability_zone", uuid, "Port_Binding") + if err != nil { + return nil, fmt.Errorf("failed to get ovn-ic-sb Port_Binding with uuid %v: %v", uuid, err) + } + return portBindings, nil +} + func (c LegacyClient) DestroyGateways(uuids []string) { for _, uuid := range uuids { if err := c.DestroyTableWithUUID(uuid, "gateway"); err != nil { @@ -109,6 +117,15 @@ func (c LegacyClient) DestroyRoutes(uuids []string) { } } +func (c LegacyClient) DestroyPortBindings(uuids []string) { + for _, uuid := range uuids { + if err := c.DestroyTableWithUUID(uuid, "Port_Binding"); err != nil { + klog.Errorf("failed to delete Port_Binding %v: %v", uuid, err) + } + continue + } +} + func (c LegacyClient) DestroyChassis(uuid string) error { if err := c.DestroyTableWithUUID(uuid, "availability_zone"); err != nil { return fmt.Errorf("failed to delete chassis %v: %v", uuid, err) diff --git a/pkg/ovs/ovn-nbctl-legacy.go b/pkg/ovs/ovn-nbctl-legacy.go index 0268a4453a4..525d075c15a 100644 --- a/pkg/ovs/ovn-nbctl-legacy.go +++ b/pkg/ovs/ovn-nbctl-legacy.go @@ -150,30 +150,33 @@ func (c LegacyClient) CreateICLogicalRouterPort(az, ts, mac, subnet string, chas return nil } -func (c LegacyClient) DeleteICLogicalRouterPort(az string) error { - lsps, err := c.ListLogicalSwitchPort(true) +func (c LegacyClient) DeleteICResources(az string) error { + lrps, err := c.ListLogicalRouterPort() if err != nil { klog.Errorf("failed to list logical switch port, %v", err) return err } - var icLsps []string - for _, lsp := range lsps { - names := strings.Split(lsp, "-") - if strings.HasPrefix(names[0], "ts") && names[1] == az { - icLsps = append(icLsps, lsp) + icTSs := make([]string, 0) + for _, lrp := range lrps { + lastIndex := strings.LastIndex(lrp, "-") + firstPart := lrp[:lastIndex] + secondPart := lrp[lastIndex+1:] + if firstPart == az && strings.HasPrefix(secondPart, util.InterconnectionSwitch) { + lsp := fmt.Sprintf("%s-%s", secondPart, firstPart) + if err := c.DeleteLogicalRouterPort(lrp); err != nil { + return fmt.Errorf("failed to delete ovn-ic logical router port: %v", err) + } + if err := c.DeleteLogicalSwitchPort(lsp); err != nil { + return fmt.Errorf("failed to delete ovn-ic logical switch port: %v", err) + } + icTSs = append(icTSs, secondPart) } } - for _, icLsp := range icLsps { - names := strings.Split(icLsp, "-") - lrpName := fmt.Sprintf("%s-%s", names[1], names[0]) - - if err := c.DeleteLogicalRouterPort(lrpName); err != nil { - return fmt.Errorf("failed to delete ovn-ic logical router port: %v", err) - } - if err := c.DeleteLogicalSwitchPort(icLsp); err != nil { - return fmt.Errorf("failed to delete ovn-ic logical switch port: %v", err) + for _, icTS := range icTSs { + if err := c.DeleteLogicalSwitch(icTS); err != nil { + return err } } @@ -792,6 +795,25 @@ func (c LegacyClient) ListLogicalSwitchPort(needVendorFilter bool) ([]string, er return result, nil } +func (c LegacyClient) ListLogicalRouterPort() ([]string, error) { + cmdArg := []string{"--format=csv", "--data=bare", "--no-heading", "--columns=name", "find", "logical_router_port"} + + output, err := c.ovnNbCommand(cmdArg...) + if err != nil { + klog.Errorf("failed to list logical router port, %v", err) + return nil, err + } + lines := strings.Split(output, "\n") + result := make([]string, 0, len(lines)) + for _, l := range lines { + if len(strings.TrimSpace(l)) == 0 { + continue + } + result = append(result, strings.TrimSpace(l)) + } + return result, nil +} + func (c LegacyClient) LogicalSwitchPortExists(port string) (bool, error) { output, err := c.ovnNbCommand("--format=csv", "--data=bare", "--no-heading", "--columns=name", "find", "logical_switch_port", fmt.Sprintf("name=%s", port)) if err != nil { diff --git a/test/e2e/ovn-ic/e2e_test.go b/test/e2e/ovn-ic/e2e_test.go index c38609f7a8c..b0c68b2f776 100644 --- a/test/e2e/ovn-ic/e2e_test.go +++ b/test/e2e/ovn-ic/e2e_test.go @@ -8,6 +8,7 @@ import ( "math/rand" "net" "os" + "os/exec" "path/filepath" "strconv" "strings" @@ -77,7 +78,7 @@ func execPodOrDie(kubeContext, namespace, pod, cmd string) string { return e2epodoutput.RunHostCmdOrDie(namespace, pod, cmd) } -var _ = framework.OrderedDescribe("[group:ovn-ic]", func() { +var _ = framework.SerialDescribe("[group:ovn-ic]", func() { frameworks := make([]*framework.Framework, len(clusters)) for i := range clusters { frameworks[i] = framework.NewFrameworkWithContext("ovn-ic", "kind-"+clusters[i]) @@ -210,4 +211,177 @@ var _ = framework.OrderedDescribe("[group:ovn-ic]", func() { fnCheckPodHTTP() }) + + framework.ConformanceIt("should be able to update az name", func() { + azNames := make([]string, len(clusters)) + for i := range clusters { + ginkgo.By("fetching the ConfigMap in cluster " + clusters[i]) + cm, err := clientSets[i].CoreV1().ConfigMaps(framework.KubeOvnNamespace).Get(context.TODO(), util.InterconnectionConfig, metav1.GetOptions{}) + framework.ExpectNoError(err, "failed to get ConfigMap") + azNames[i] = cm.Data["az-name"] + } + + azNames[0] = fmt.Sprintf("az%04d", rand.Intn(10000)) + configMapPatchPayload, err := json.Marshal(corev1.ConfigMap{ + Data: map[string]string{ + "az-name": azNames[0], + }, + }) + framework.ExpectNoError(err, "failed to marshal patch data") + + ginkgo.By("patching the ConfigMap in cluster " + clusters[0]) + _, err = clientSets[0].CoreV1().ConfigMaps(framework.KubeOvnNamespace).Patch(context.TODO(), util.InterconnectionConfig, k8stypes.StrategicMergePatchType, []byte(configMapPatchPayload), metav1.PatchOptions{}) + framework.ExpectNoError(err, "failed to patch ConfigMap") + + ginkgo.By("Waiting for new az names to be applied") + time.Sleep(10 * time.Second) + + pods, err := clientSets[0].CoreV1().Pods("kube-system").List(context.TODO(), metav1.ListOptions{LabelSelector: "app=ovs"}) + framework.ExpectNoError(err, "failed to get ovs-ovn pods") + cmd := "ovn-appctl -t ovn-controller inc-engine/recompute" + for _, pod := range pods.Items { + execPodOrDie(frameworks[0].KubeContext, "kube-system", pod.Name, cmd) + } + time.Sleep(2 * time.Second) + + ginkgo.By("Ensuring logical switch ts exists in cluster " + clusters[0]) + output := execOrDie(frameworks[0].KubeContext, "ko nbctl show ts") + for _, az := range azNames { + lsp := "ts-" + az + framework.ExpectTrue(strings.Contains(output, lsp), "should have lsp "+lsp) + framework.ExpectTrue(strings.Contains(output, lsp), "should have lsp "+lsp) + } + + fnCheckPodHTTP() + }) + + framework.ConformanceIt("Should Support ECMP OVN Interconnection", func() { + + ginkgo.By("case 1: ecmp gateway network test") + checkECMPCount(3) + fnCheckPodHTTP() + + ginkgo.By("case 2: reduce two clusters from 3 gateway to 1 gateway") + oldGatewayStr := make([]string, len(clusters)) + gwNodes := make([]string, len(clusters)) + for i := range clusters { + ginkgo.By("fetching the ConfigMap in cluster " + clusters[i]) + cm, err := clientSets[i].CoreV1().ConfigMaps(framework.KubeOvnNamespace).Get(context.TODO(), util.InterconnectionConfig, metav1.GetOptions{}) + framework.ExpectNoError(err, "failed to get ConfigMap") + gwNodes[i] = cm.Data["gw-nodes"] + oldGatewayStr[i] = cm.Data["gw-nodes"] + gws := strings.Split(gwNodes[i], ",") + newGatewayStr := strings.Join(gws[0:len(gws)-2], ",") + + configMapPatchPayload, err := json.Marshal(corev1.ConfigMap{ + Data: map[string]string{ + "gw-nodes": newGatewayStr, + }, + }) + _, err = clientSets[i].CoreV1().ConfigMaps(framework.KubeOvnNamespace).Patch(context.TODO(), util.InterconnectionConfig, k8stypes.StrategicMergePatchType, []byte(configMapPatchPayload), metav1.PatchOptions{}) + framework.ExpectNoError(err, "patch ovn-ic-config failed") + } + fnCheckPodHTTP() + + ginkgo.By("case 3: recover two clusters from 1 gateway to 3 gateway") + gwNodes = make([]string, len(clusters)) + for i := range clusters { + ginkgo.By("fetching the ConfigMap in cluster " + clusters[i]) + + configMapPatchPayload, err := json.Marshal(corev1.ConfigMap{ + Data: map[string]string{ + "gw-nodes": oldGatewayStr[i], + }, + }) + + _, err = clientSets[i].CoreV1().ConfigMaps(framework.KubeOvnNamespace).Patch(context.TODO(), util.InterconnectionConfig, k8stypes.StrategicMergePatchType, []byte(configMapPatchPayload), metav1.PatchOptions{}) + framework.ExpectNoError(err, "patch ovn-ic-config failed") + } + fnCheckPodHTTP() + + ginkgo.By("case 4: scale ecmp path from 3 to 5 ") + switchCmd := "kubectl config use-context kind-kube-ovn" + _, err := exec.Command("bash", "-c", switchCmd).CombinedOutput() + framework.ExpectNoError(err, "switch to kube-ovn cluster failed") + + patchCmd := "kubectl patch deployment ovn-ic-server -n kube-system --type='json' -p=\"[{'op': 'replace', 'path': '/spec/template/spec/containers/0/env/1/value', 'value': '5'}]\"" + _, err = exec.Command("bash", "-c", patchCmd).CombinedOutput() + checkECMPCount(5) + fnCheckPodHTTP() + + ginkgo.By("case 5: reduce ecmp path from 5 to 3 ") + patchCmd = "kubectl patch deployment ovn-ic-server -n kube-system --type='json' -p=\"[{'op': 'replace', 'path': '/spec/template/spec/containers/0/env/1/value', 'value': '3'}]\"" + _, err = exec.Command("bash", "-c", patchCmd).CombinedOutput() + checkECMPCount(3) + fnCheckPodHTTP() + + ginkgo.By("case 6: disable gateway kube-ovn1-worker gateway") + switchCmd = "kubectl config use-context kind-kube-ovn1" + _, err = exec.Command("bash", "-c", switchCmd).CombinedOutput() + framework.ExpectNoError(err, "switch to kube-ovn1 cluster failed") + + disableNetworkCmd := "docker exec kube-ovn1-worker iptables -I INPUT -p udp --dport 6081 -j DROP" + _, err = exec.Command("bash", "-c", disableNetworkCmd).CombinedOutput() + framework.ExpectNoError(err, "disable kube-ovn1-worker gateway failed") + + taintCmd := "kubectl taint nodes kube-ovn1-worker e2e=test:NoSchedule" + _, err = exec.Command("bash", "-c", taintCmd).CombinedOutput() + fnCheckPodHTTP() + + ginkgo.By("case 7: disable gateway kube-ovn1-worker2 gateway") + switchCmd = "kubectl config use-context kind-kube-ovn1" + _, err = exec.Command("bash", "-c", switchCmd).CombinedOutput() + framework.ExpectNoError(err, "switch to kube-ovn1 cluster failed") + + disableNetworkCmd = "docker exec kube-ovn1-worker2 iptables -I INPUT -p udp --dport 6081 -j DROP" + _, err = exec.Command("bash", "-c", disableNetworkCmd).CombinedOutput() + framework.ExpectNoError(err, "disable kube-ovn1-worker2 gateway failed") + + taintCmd = "kubectl taint nodes kube-ovn1-worker2 e2e=test:NoSchedule" + _, err = exec.Command("bash", "-c", taintCmd).CombinedOutput() + fnCheckPodHTTP() + + ginkgo.By("case 8: enable gateway kube-ovn1-worker gateway") + switchCmd = "kubectl config use-context kind-kube-ovn1" + _, err = exec.Command("bash", "-c", switchCmd).CombinedOutput() + framework.ExpectNoError(err, "switch to kube-ovn1 cluster failed") + + disableNetworkCmd = "docker exec kube-ovn1-worker iptables -D INPUT -p udp --dport 6081 -j DROP" + _, err = exec.Command("bash", "-c", disableNetworkCmd).CombinedOutput() + framework.ExpectNoError(err, "enable kube-ovn1-worker gateway failed") + + taintCmd = "kubectl taint nodes kube-ovn1-worker e2e=test:NoSchedule-" + _, err = exec.Command("bash", "-c", taintCmd).CombinedOutput() + fnCheckPodHTTP() + + ginkgo.By("case 9: enable gateway kube-ovn1-worker2 gateway") + switchCmd = "kubectl config use-context kind-kube-ovn1" + _, err = exec.Command("bash", "-c", switchCmd).CombinedOutput() + framework.ExpectNoError(err, "switch to kube-ovn1 cluster failed") + + disableNetworkCmd = "docker exec kube-ovn1-worker2 iptables -D INPUT -p udp --dport 6081 -j DROP" + _, err = exec.Command("bash", "-c", disableNetworkCmd).CombinedOutput() + framework.ExpectNoError(err, "enable kube-ovn1-worker2 gateway failed") + + taintCmd = "kubectl taint nodes kube-ovn1-worker2 e2e=test:NoSchedule-" + _, err = exec.Command("bash", "-c", taintCmd).CombinedOutput() + fnCheckPodHTTP() + }) }) + +func checkECMPCount(expectCount int) { + ecmpCount := 0 + maxRetryTimes := 10 + for i := 0; i < maxRetryTimes; i++ { + time.Sleep(3 * time.Second) + execCmd := "kubectl ko nbctl lr-route-list ovn-cluster " + output, err := exec.Command("bash", "-c", execCmd).CombinedOutput() + framework.ExpectNoError(err) + ecmpCount = strings.Count(string(output), "ecmp") + if ecmpCount == expectCount { + break + } + } + + framework.ExpectEqual(ecmpCount, expectCount) +} diff --git a/yamls/kind.yaml.j2 b/yamls/kind.yaml.j2 index 0ec7041c7a8..0359a0a3c4a 100644 --- a/yamls/kind.yaml.j2 +++ b/yamls/kind.yaml.j2 @@ -63,6 +63,8 @@ nodes: labels: kube-ovn/role: master {%- elif ovn_ic is equalto "true" %} + - role: worker + image: kindest/node:{{ k8s_version }} - role: worker image: kindest/node:{{ k8s_version }} {%- endif %} From 69de7f98a1b6fa92cbdbe92668261c0f9ff3dc47 Mon Sep 17 00:00:00 2001 From: changluyi Date: Mon, 8 Jan 2024 09:31:18 +0800 Subject: [PATCH 03/13] conflict merge Signed-off-by: changluyi --- kubeovn-helm/templates/ovsovn-ds.yaml | 2 +- kubeovn-helm/values.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kubeovn-helm/templates/ovsovn-ds.yaml b/kubeovn-helm/templates/ovsovn-ds.yaml index 46616bcd17f..2d9aa59240c 100644 --- a/kubeovn-helm/templates/ovsovn-ds.yaml +++ b/kubeovn-helm/templates/ovsovn-ds.yaml @@ -120,7 +120,7 @@ spec: cpu: 200m memory: 200Mi limits: - cpu: "2" + cpu: 1000m memory: 1000Mi nodeSelector: kubernetes.io/os: "linux" diff --git a/kubeovn-helm/values.yaml b/kubeovn-helm/values.yaml index 70a55893354..db268b7231c 100644 --- a/kubeovn-helm/values.yaml +++ b/kubeovn-helm/values.yaml @@ -8,7 +8,7 @@ global: images: kubeovn: repository: kube-ovn - tag: v1.11.13 + tag: v1.11.12 support_arm: true thirdparty: true From 21fb49b0c392855d8c3364cf32841f823ffbc7d1 Mon Sep 17 00:00:00 2001 From: changluyi Date: Mon, 8 Jan 2024 09:42:54 +0800 Subject: [PATCH 04/13] conflict merge Signed-off-by: changluyi --- kubeovn-helm/templates/ovsovn-ds.yaml | 2 +- kubeovn-helm/values.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kubeovn-helm/templates/ovsovn-ds.yaml b/kubeovn-helm/templates/ovsovn-ds.yaml index 2d9aa59240c..46616bcd17f 100644 --- a/kubeovn-helm/templates/ovsovn-ds.yaml +++ b/kubeovn-helm/templates/ovsovn-ds.yaml @@ -120,7 +120,7 @@ spec: cpu: 200m memory: 200Mi limits: - cpu: 1000m + cpu: "2" memory: 1000Mi nodeSelector: kubernetes.io/os: "linux" diff --git a/kubeovn-helm/values.yaml b/kubeovn-helm/values.yaml index db268b7231c..70a55893354 100644 --- a/kubeovn-helm/values.yaml +++ b/kubeovn-helm/values.yaml @@ -8,7 +8,7 @@ global: images: kubeovn: repository: kube-ovn - tag: v1.11.12 + tag: v1.11.13 support_arm: true thirdparty: true From f1f54b4d8926167671bb4c27f3afa9519036bf42 Mon Sep 17 00:00:00 2001 From: changluyi Date: Mon, 8 Jan 2024 11:32:05 +0800 Subject: [PATCH 05/13] golang lint Signed-off-by: changluyi --- pkg/ovn_ic_client/ovn_ic_client.go | 3 +-- test/e2e/ovn-ic/e2e_test.go | 19 +++++++++---------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/pkg/ovn_ic_client/ovn_ic_client.go b/pkg/ovn_ic_client/ovn_ic_client.go index 523b8f39278..6451642d1d6 100644 --- a/pkg/ovn_ic_client/ovn_ic_client.go +++ b/pkg/ovn_ic_client/ovn_ic_client.go @@ -85,7 +85,6 @@ func (c *Controller) setAutoRoute(autoRoute bool) { func (c *Controller) getICState(cmData, lastcmData map[string]string) int { isCMEqual := reflect.DeepEqual(cmData, lastcmData) - isTsEqual := true if icEnabled != "true" && len(lastcmData) == 0 && cmData["enable-ic"] == "true" { return icFirstEstablish } @@ -98,7 +97,7 @@ func (c *Controller) getICState(cmData, lastcmData map[string]string) int { klog.Errorf("failed to get Transit_Switch, %v", err) return icNoAction } - isTsEqual = reflect.DeepEqual(lastTSs, curTSs) + isTsEqual := reflect.DeepEqual(lastTSs, curTSs) if isTsEqual { return icNoAction } diff --git a/test/e2e/ovn-ic/e2e_test.go b/test/e2e/ovn-ic/e2e_test.go index b0c68b2f776..cf40e6ef7e5 100644 --- a/test/e2e/ovn-ic/e2e_test.go +++ b/test/e2e/ovn-ic/e2e_test.go @@ -273,7 +273,7 @@ var _ = framework.SerialDescribe("[group:ovn-ic]", func() { gws := strings.Split(gwNodes[i], ",") newGatewayStr := strings.Join(gws[0:len(gws)-2], ",") - configMapPatchPayload, err := json.Marshal(corev1.ConfigMap{ + configMapPatchPayload, _ := json.Marshal(corev1.ConfigMap{ Data: map[string]string{ "gw-nodes": newGatewayStr, }, @@ -284,17 +284,16 @@ var _ = framework.SerialDescribe("[group:ovn-ic]", func() { fnCheckPodHTTP() ginkgo.By("case 3: recover two clusters from 1 gateway to 3 gateway") - gwNodes = make([]string, len(clusters)) for i := range clusters { ginkgo.By("fetching the ConfigMap in cluster " + clusters[i]) - configMapPatchPayload, err := json.Marshal(corev1.ConfigMap{ + configMapPatchPayload, _ := json.Marshal(corev1.ConfigMap{ Data: map[string]string{ "gw-nodes": oldGatewayStr[i], }, }) - _, err = clientSets[i].CoreV1().ConfigMaps(framework.KubeOvnNamespace).Patch(context.TODO(), util.InterconnectionConfig, k8stypes.StrategicMergePatchType, []byte(configMapPatchPayload), metav1.PatchOptions{}) + _, err := clientSets[i].CoreV1().ConfigMaps(framework.KubeOvnNamespace).Patch(context.TODO(), util.InterconnectionConfig, k8stypes.StrategicMergePatchType, []byte(configMapPatchPayload), metav1.PatchOptions{}) framework.ExpectNoError(err, "patch ovn-ic-config failed") } fnCheckPodHTTP() @@ -305,13 +304,13 @@ var _ = framework.SerialDescribe("[group:ovn-ic]", func() { framework.ExpectNoError(err, "switch to kube-ovn cluster failed") patchCmd := "kubectl patch deployment ovn-ic-server -n kube-system --type='json' -p=\"[{'op': 'replace', 'path': '/spec/template/spec/containers/0/env/1/value', 'value': '5'}]\"" - _, err = exec.Command("bash", "-c", patchCmd).CombinedOutput() + _, _ = exec.Command("bash", "-c", patchCmd).CombinedOutput() checkECMPCount(5) fnCheckPodHTTP() ginkgo.By("case 5: reduce ecmp path from 5 to 3 ") patchCmd = "kubectl patch deployment ovn-ic-server -n kube-system --type='json' -p=\"[{'op': 'replace', 'path': '/spec/template/spec/containers/0/env/1/value', 'value': '3'}]\"" - _, err = exec.Command("bash", "-c", patchCmd).CombinedOutput() + _, _ = exec.Command("bash", "-c", patchCmd).CombinedOutput() checkECMPCount(3) fnCheckPodHTTP() @@ -325,7 +324,7 @@ var _ = framework.SerialDescribe("[group:ovn-ic]", func() { framework.ExpectNoError(err, "disable kube-ovn1-worker gateway failed") taintCmd := "kubectl taint nodes kube-ovn1-worker e2e=test:NoSchedule" - _, err = exec.Command("bash", "-c", taintCmd).CombinedOutput() + _, _ = exec.Command("bash", "-c", taintCmd).CombinedOutput() fnCheckPodHTTP() ginkgo.By("case 7: disable gateway kube-ovn1-worker2 gateway") @@ -338,7 +337,7 @@ var _ = framework.SerialDescribe("[group:ovn-ic]", func() { framework.ExpectNoError(err, "disable kube-ovn1-worker2 gateway failed") taintCmd = "kubectl taint nodes kube-ovn1-worker2 e2e=test:NoSchedule" - _, err = exec.Command("bash", "-c", taintCmd).CombinedOutput() + _, _ = exec.Command("bash", "-c", taintCmd).CombinedOutput() fnCheckPodHTTP() ginkgo.By("case 8: enable gateway kube-ovn1-worker gateway") @@ -351,7 +350,7 @@ var _ = framework.SerialDescribe("[group:ovn-ic]", func() { framework.ExpectNoError(err, "enable kube-ovn1-worker gateway failed") taintCmd = "kubectl taint nodes kube-ovn1-worker e2e=test:NoSchedule-" - _, err = exec.Command("bash", "-c", taintCmd).CombinedOutput() + _, _ = exec.Command("bash", "-c", taintCmd).CombinedOutput() fnCheckPodHTTP() ginkgo.By("case 9: enable gateway kube-ovn1-worker2 gateway") @@ -364,7 +363,7 @@ var _ = framework.SerialDescribe("[group:ovn-ic]", func() { framework.ExpectNoError(err, "enable kube-ovn1-worker2 gateway failed") taintCmd = "kubectl taint nodes kube-ovn1-worker2 e2e=test:NoSchedule-" - _, err = exec.Command("bash", "-c", taintCmd).CombinedOutput() + _, _ = exec.Command("bash", "-c", taintCmd).CombinedOutput() fnCheckPodHTTP() }) }) From acc90889537ff09cf1424a0cf41e3ca99faa66da Mon Sep 17 00:00:00 2001 From: Changlu Yi Date: Mon, 8 Jan 2024 13:36:50 +0800 Subject: [PATCH 06/13] change chmod Signed-off-by: Changlu Yi --- dist/images/clean-ic-az-db.sh | 0 dist/images/install-ic-server.sh | 0 dist/images/ovn-ic-healthcheck.sh | 0 dist/images/start-ic-client.sh | 0 dist/images/start-ic-db.sh | 0 5 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 dist/images/clean-ic-az-db.sh mode change 100644 => 100755 dist/images/install-ic-server.sh mode change 100644 => 100755 dist/images/ovn-ic-healthcheck.sh mode change 100644 => 100755 dist/images/start-ic-client.sh mode change 100644 => 100755 dist/images/start-ic-db.sh diff --git a/dist/images/clean-ic-az-db.sh b/dist/images/clean-ic-az-db.sh old mode 100644 new mode 100755 diff --git a/dist/images/install-ic-server.sh b/dist/images/install-ic-server.sh old mode 100644 new mode 100755 diff --git a/dist/images/ovn-ic-healthcheck.sh b/dist/images/ovn-ic-healthcheck.sh old mode 100644 new mode 100755 diff --git a/dist/images/start-ic-client.sh b/dist/images/start-ic-client.sh old mode 100644 new mode 100755 diff --git a/dist/images/start-ic-db.sh b/dist/images/start-ic-db.sh old mode 100644 new mode 100755 From 343e2599bd4a4bad0599d721a13748b192b5d633 Mon Sep 17 00:00:00 2001 From: changluyi Date: Mon, 8 Jan 2024 14:00:44 +0800 Subject: [PATCH 07/13] fix Signed-off-by: changluyi --- Makefile | 2 +- dist/images/install-ic-server.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 44bed8d35ae..0ebfa05744f 100644 --- a/Makefile +++ b/Makefile @@ -370,7 +370,7 @@ kind-install-ovn-ic: kind-install kubectl describe no kubectl config use-context kind-kube-ovn - bash dist/images/install-ic-server.sh + sed 's/VERSION=.*/VERSION=$(VERSION)/' dist/images/install-ic-server.sh | bash @set -e; \ ic_db_host=$$(kubectl get deployment ovn-ic-server -n kube-system -o jsonpath='{range .spec.template.spec.containers[0].env[?(@.name=="NODE_IPS")]}{.value}{end}'); \ diff --git a/dist/images/install-ic-server.sh b/dist/images/install-ic-server.sh index 1df4c6411b4..8a8b8910a38 100755 --- a/dist/images/install-ic-server.sh +++ b/dist/images/install-ic-server.sh @@ -2,7 +2,7 @@ set -euo pipefail REGISTRY="kubeovn" -VERSION="v1.11.13" +VERSION="v1.11.14" TS_NUM=${TS_NUM:-3} IMAGE_PULL_POLICY="IfNotPresent" addresses=$(kubectl get no -lkube-ovn/role=master --no-headers -o wide | awk '{print $6}' | tr \\n ',') From c1f14461c51eb636552ed2cea4f1b85a289d5ecf Mon Sep 17 00:00:00 2001 From: changluyi Date: Mon, 8 Jan 2024 14:56:53 +0800 Subject: [PATCH 08/13] add clean up Signed-off-by: changluyi --- dist/images/cleanup.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dist/images/cleanup.sh b/dist/images/cleanup.sh index 14183c3e79d..0fa6534b759 100644 --- a/dist/images/cleanup.sh +++ b/dist/images/cleanup.sh @@ -82,6 +82,8 @@ kubectl delete --ignore-not-found cm ovn-config ovn-ic-config ovn-external-gw-co kubectl delete --ignore-not-found svc kube-ovn-pinger kube-ovn-controller kube-ovn-cni kube-ovn-monitor -n kube-system kubectl delete --ignore-not-found ds kube-ovn-cni -n kube-system kubectl delete --ignore-not-found deploy kube-ovn-controller -n kube-system +kubectl delete --ignore-not-found deploy ovn-ic-client -n kube-system +kubectl delete --ignore-not-found deploy ovn-ic-server -n kube-system # ensure kube-ovn-cni has been deleted while :; do From dbd9a017e6b95df42b902942cd8a1f5837129c71 Mon Sep 17 00:00:00 2001 From: Changlu Yi Date: Tue, 9 Jan 2024 15:16:34 +0800 Subject: [PATCH 09/13] add switch for enable_ic Signed-off-by: Changlu Yi --- Makefile | 5 +- dist/images/install.sh | 138 ++++++++++--------- kubeovn-helm/templates/ic-client-deploy.yaml | 4 +- kubeovn-helm/values.yaml | 1 + test/e2e/ovn-ic/e2e_test.go | 43 ------ 5 files changed, 80 insertions(+), 111 deletions(-) diff --git a/Makefile b/Makefile index 0ebfa05744f..ba042ff6b67 100644 --- a/Makefile +++ b/Makefile @@ -358,7 +358,8 @@ kind-install-ipv4: kind-install-overlay-ipv4 kind-install-overlay-ipv4: kind-install .PHONY: kind-install-ovn-ic -kind-install-ovn-ic: kind-install +kind-install-ovn-ic: + @ENABLE_IC=true $(MAKE) kind-install $(call kind_load_image,kube-ovn1,$(REGISTRY)/kube-ovn:$(VERSION)) kubectl config use-context kind-kube-ovn1 $(MAKE) kind-untaint-control-plane @@ -366,7 +367,7 @@ kind-install-ovn-ic: kind-install -e 's/10.96.0/10.98.0/g' \ -e 's/100.64.0/100.68.0/g' \ -e 's/VERSION=.*/VERSION=$(VERSION)/' \ - dist/images/install.sh | bash + dist/images/install.sh | ENABLE_IC=true bash kubectl describe no kubectl config use-context kind-kube-ovn diff --git a/dist/images/install.sh b/dist/images/install.sh index 61d4ea7d2b6..a8b897db6a1 100644 --- a/dist/images/install.sh +++ b/dist/images/install.sh @@ -19,6 +19,7 @@ ENABLE_EXTERNAL_VPC=${ENABLE_EXTERNAL_VPC:-true} CNI_CONFIG_PRIORITY=${CNI_CONFIG_PRIORITY:-01} ENABLE_LB_SVC=${ENABLE_LB_SVC:-false} ENABLE_KEEP_VM_IP=${ENABLE_KEEP_VM_IP:-true} +ENABLE_IC=${ENABLE_IC:-false} # exchange link names of OVS bridge and the provider nic # in the default provider-network @@ -3559,6 +3560,76 @@ spec: optional: true secretName: kube-ovn-tls --- +kind: Service +apiVersion: v1 +metadata: + name: kube-ovn-monitor + namespace: kube-system + labels: + app: kube-ovn-monitor +spec: + ports: + - name: metrics + port: 10661 + type: ClusterIP + ${SVC_YAML_IPFAMILYPOLICY} + selector: + app: kube-ovn-monitor + sessionAffinity: None +--- +kind: Service +apiVersion: v1 +metadata: + name: kube-ovn-pinger + namespace: kube-system + labels: + app: kube-ovn-pinger +spec: + ${SVC_YAML_IPFAMILYPOLICY} + selector: + app: kube-ovn-pinger + ports: + - port: 8080 + name: metrics +--- +kind: Service +apiVersion: v1 +metadata: + name: kube-ovn-controller + namespace: kube-system + labels: + app: kube-ovn-controller +spec: + ${SVC_YAML_IPFAMILYPOLICY} + selector: + app: kube-ovn-controller + ports: + - port: 10660 + name: metrics +--- +kind: Service +apiVersion: v1 +metadata: + name: kube-ovn-cni + namespace: kube-system + labels: + app: kube-ovn-cni +spec: + ${SVC_YAML_IPFAMILYPOLICY} + selector: + app: kube-ovn-cni + ports: + - port: 10665 + name: metrics +EOF + +kubectl apply -f kube-ovn.yaml +kubectl rollout status deployment/kube-ovn-controller -n kube-system --timeout 300s +kubectl rollout status daemonset/kube-ovn-cni -n kube-system --timeout 300s + +if $ENABLE_IC; then + +cat < ovn-ic-client.yaml kind: Deployment apiVersion: apps/v1 metadata: @@ -3656,73 +3727,10 @@ spec: secret: optional: true secretName: kube-ovn-tls ---- -kind: Service -apiVersion: v1 -metadata: - name: kube-ovn-monitor - namespace: kube-system - labels: - app: kube-ovn-monitor -spec: - ports: - - name: metrics - port: 10661 - type: ClusterIP - ${SVC_YAML_IPFAMILYPOLICY} - selector: - app: kube-ovn-monitor - sessionAffinity: None ---- -kind: Service -apiVersion: v1 -metadata: - name: kube-ovn-pinger - namespace: kube-system - labels: - app: kube-ovn-pinger -spec: - ${SVC_YAML_IPFAMILYPOLICY} - selector: - app: kube-ovn-pinger - ports: - - port: 8080 - name: metrics ---- -kind: Service -apiVersion: v1 -metadata: - name: kube-ovn-controller - namespace: kube-system - labels: - app: kube-ovn-controller -spec: - ${SVC_YAML_IPFAMILYPOLICY} - selector: - app: kube-ovn-controller - ports: - - port: 10660 - name: metrics ---- -kind: Service -apiVersion: v1 -metadata: - name: kube-ovn-cni - namespace: kube-system - labels: - app: kube-ovn-cni -spec: - ${SVC_YAML_IPFAMILYPOLICY} - selector: - app: kube-ovn-cni - ports: - - port: 10665 - name: metrics EOF +kubectl apply -f ovn-ic-client.yaml +fi -kubectl apply -f kube-ovn.yaml -kubectl rollout status deployment/kube-ovn-controller -n kube-system --timeout 300s -kubectl rollout status daemonset/kube-ovn-cni -n kube-system --timeout 300s echo "-------------------------------" echo "" diff --git a/kubeovn-helm/templates/ic-client-deploy.yaml b/kubeovn-helm/templates/ic-client-deploy.yaml index 4d75c814682..cea6a699c36 100644 --- a/kubeovn-helm/templates/ic-client-deploy.yaml +++ b/kubeovn-helm/templates/ic-client-deploy.yaml @@ -1,3 +1,4 @@ +{{- if eq .Values.func.ENABLE_IC true }} kind: Deployment apiVersion: apps/v1 metadata: @@ -94,4 +95,5 @@ spec: - name: kube-ovn-tls secret: optional: true - secretName: kube-ovn-tls \ No newline at end of file + secretName: kube-ovn-tls +{{- end }} \ No newline at end of file diff --git a/kubeovn-helm/values.yaml b/kubeovn-helm/values.yaml index 70a55893354..5aaca31cce2 100644 --- a/kubeovn-helm/values.yaml +++ b/kubeovn-helm/values.yaml @@ -52,6 +52,7 @@ func: CHECK_GATEWAY: true LOGICAL_GATEWAY: false ENABLE_BIND_LOCAL_IP: true + ENABLE_IC: true ipv4: POD_CIDR: "10.16.0.0/16" diff --git a/test/e2e/ovn-ic/e2e_test.go b/test/e2e/ovn-ic/e2e_test.go index cf40e6ef7e5..ca9e5645f59 100644 --- a/test/e2e/ovn-ic/e2e_test.go +++ b/test/e2e/ovn-ic/e2e_test.go @@ -212,49 +212,6 @@ var _ = framework.SerialDescribe("[group:ovn-ic]", func() { fnCheckPodHTTP() }) - framework.ConformanceIt("should be able to update az name", func() { - azNames := make([]string, len(clusters)) - for i := range clusters { - ginkgo.By("fetching the ConfigMap in cluster " + clusters[i]) - cm, err := clientSets[i].CoreV1().ConfigMaps(framework.KubeOvnNamespace).Get(context.TODO(), util.InterconnectionConfig, metav1.GetOptions{}) - framework.ExpectNoError(err, "failed to get ConfigMap") - azNames[i] = cm.Data["az-name"] - } - - azNames[0] = fmt.Sprintf("az%04d", rand.Intn(10000)) - configMapPatchPayload, err := json.Marshal(corev1.ConfigMap{ - Data: map[string]string{ - "az-name": azNames[0], - }, - }) - framework.ExpectNoError(err, "failed to marshal patch data") - - ginkgo.By("patching the ConfigMap in cluster " + clusters[0]) - _, err = clientSets[0].CoreV1().ConfigMaps(framework.KubeOvnNamespace).Patch(context.TODO(), util.InterconnectionConfig, k8stypes.StrategicMergePatchType, []byte(configMapPatchPayload), metav1.PatchOptions{}) - framework.ExpectNoError(err, "failed to patch ConfigMap") - - ginkgo.By("Waiting for new az names to be applied") - time.Sleep(10 * time.Second) - - pods, err := clientSets[0].CoreV1().Pods("kube-system").List(context.TODO(), metav1.ListOptions{LabelSelector: "app=ovs"}) - framework.ExpectNoError(err, "failed to get ovs-ovn pods") - cmd := "ovn-appctl -t ovn-controller inc-engine/recompute" - for _, pod := range pods.Items { - execPodOrDie(frameworks[0].KubeContext, "kube-system", pod.Name, cmd) - } - time.Sleep(2 * time.Second) - - ginkgo.By("Ensuring logical switch ts exists in cluster " + clusters[0]) - output := execOrDie(frameworks[0].KubeContext, "ko nbctl show ts") - for _, az := range azNames { - lsp := "ts-" + az - framework.ExpectTrue(strings.Contains(output, lsp), "should have lsp "+lsp) - framework.ExpectTrue(strings.Contains(output, lsp), "should have lsp "+lsp) - } - - fnCheckPodHTTP() - }) - framework.ConformanceIt("Should Support ECMP OVN Interconnection", func() { ginkgo.By("case 1: ecmp gateway network test") From a87d9b95ccbfe7eb3e4f572923fd7b0c9f0b8299 Mon Sep 17 00:00:00 2001 From: Changlu Yi Date: Thu, 11 Jan 2024 19:19:38 +0800 Subject: [PATCH 10/13] refactor Signed-off-by: Changlu Yi --- cmd/cmdmain.go | 8 +- cmd/ovn_ic_controller/ovn_ic_controller.go | 24 + dist/images/Dockerfile | 2 +- dist/images/cleanup.sh | 2 +- dist/images/install.sh | 20 +- dist/images/start-ic-controller.sh | 60 ++ .../templates/ic-controller-deploy.yaml | 99 ++++ pkg/ovn_ic_controller/config.go | 142 +++++ pkg/ovn_ic_controller/controller.go | 105 ++++ pkg/ovn_ic_controller/ovn_ic_controller.go | 558 ++++++++++++++++++ pkg/ovn_leader_checker/ovn.go | 22 +- 11 files changed, 1010 insertions(+), 32 deletions(-) create mode 100644 cmd/ovn_ic_controller/ovn_ic_controller.go create mode 100755 dist/images/start-ic-controller.sh create mode 100644 kubeovn-helm/templates/ic-controller-deploy.yaml create mode 100644 pkg/ovn_ic_controller/config.go create mode 100644 pkg/ovn_ic_controller/controller.go create mode 100644 pkg/ovn_ic_controller/ovn_ic_controller.go diff --git a/cmd/cmdmain.go b/cmd/cmdmain.go index da68c5dc0cc..5ab13cf3a43 100644 --- a/cmd/cmdmain.go +++ b/cmd/cmdmain.go @@ -8,7 +8,7 @@ import ( "github.com/kubeovn/kube-ovn/cmd/controller" "github.com/kubeovn/kube-ovn/cmd/controller_health_check" "github.com/kubeovn/kube-ovn/cmd/daemon" - "github.com/kubeovn/kube-ovn/cmd/ovn_ic_client" + "github.com/kubeovn/kube-ovn/cmd/ovn_ic_controller" "github.com/kubeovn/kube-ovn/cmd/ovn_leader_checker" "github.com/kubeovn/kube-ovn/cmd/ovn_monitor" "github.com/kubeovn/kube-ovn/cmd/pinger" @@ -25,7 +25,7 @@ const ( CmdSpeaker = "kube-ovn-speaker" CmdControllerHealthCheck = "kube-ovn-controller-healthcheck" CmdOvnLeaderChecker = "kube-ovn-leader-checker" - CmdOvnICClient = "kube-ovn-ic-client" + CmdOvnICController = "kube-ovn-ic-controller" ) func main() { @@ -47,8 +47,8 @@ func main() { controller_health_check.CmdMain() case CmdOvnLeaderChecker: ovn_leader_checker.CmdMain() - case CmdOvnICClient: - ovn_ic_client.CmdMain() + case CmdOvnICController: + ovn_ic_controller.CmdMain() default: util.LogFatalAndExit(nil, "%s is an unknown command", cmd) } diff --git a/cmd/ovn_ic_controller/ovn_ic_controller.go b/cmd/ovn_ic_controller/ovn_ic_controller.go new file mode 100644 index 00000000000..a3dcfa419d5 --- /dev/null +++ b/cmd/ovn_ic_controller/ovn_ic_controller.go @@ -0,0 +1,24 @@ +package ovn_ic_controller + +import ( + "k8s.io/klog/v2" + "k8s.io/sample-controller/pkg/signals" + + "github.com/kubeovn/kube-ovn/pkg/ovn_ic_controller" + "github.com/kubeovn/kube-ovn/pkg/util" + "github.com/kubeovn/kube-ovn/versions" +) + +func CmdMain() { + defer klog.Flush() + + klog.Infof(versions.String()) + config, err := ovn_ic_controller.ParseFlags() + if err != nil { + util.LogFatalAndExit(err, "failed to parse config") + } + + stopCh := signals.SetupSignalHandler() + ctl := ovn_ic_controller.NewController(config) + ctl.Run(stopCh) +} diff --git a/dist/images/Dockerfile b/dist/images/Dockerfile index 423e9dcb4ba..5c51efc6ab9 100644 --- a/dist/images/Dockerfile +++ b/dist/images/Dockerfile @@ -29,4 +29,4 @@ RUN ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn && \ ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-speaker && \ ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-controller-healthcheck && \ ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-leader-checker && \ - ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-ic-client + ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-ic-controller diff --git a/dist/images/cleanup.sh b/dist/images/cleanup.sh index 0fa6534b759..d06b599dc0b 100644 --- a/dist/images/cleanup.sh +++ b/dist/images/cleanup.sh @@ -82,7 +82,7 @@ kubectl delete --ignore-not-found cm ovn-config ovn-ic-config ovn-external-gw-co kubectl delete --ignore-not-found svc kube-ovn-pinger kube-ovn-controller kube-ovn-cni kube-ovn-monitor -n kube-system kubectl delete --ignore-not-found ds kube-ovn-cni -n kube-system kubectl delete --ignore-not-found deploy kube-ovn-controller -n kube-system -kubectl delete --ignore-not-found deploy ovn-ic-client -n kube-system +kubectl delete --ignore-not-found deploy ovn-ic-controller -n kube-system kubectl delete --ignore-not-found deploy ovn-ic-server -n kube-system # ensure kube-ovn-cni has been deleted diff --git a/dist/images/install.sh b/dist/images/install.sh index a8b897db6a1..357c6a33f3f 100644 --- a/dist/images/install.sh +++ b/dist/images/install.sh @@ -19,7 +19,7 @@ ENABLE_EXTERNAL_VPC=${ENABLE_EXTERNAL_VPC:-true} CNI_CONFIG_PRIORITY=${CNI_CONFIG_PRIORITY:-01} ENABLE_LB_SVC=${ENABLE_LB_SVC:-false} ENABLE_KEEP_VM_IP=${ENABLE_KEEP_VM_IP:-true} -ENABLE_IC=${ENABLE_IC:-false} +ENABLE_IC=${ENABLE_IC:-true} # exchange link names of OVS bridge and the provider nic # in the default provider-network @@ -3629,15 +3629,15 @@ kubectl rollout status daemonset/kube-ovn-cni -n kube-system --timeout 300s if $ENABLE_IC; then -cat < ovn-ic-client.yaml +cat < ovn-ic-controller.yaml kind: Deployment apiVersion: apps/v1 metadata: - name: ovn-ic-client + name: ovn-ic-controller namespace: kube-system annotations: kubernetes.io/description: | - OVN IC Client + OVN IC Controller spec: replicas: 1 strategy: @@ -3647,11 +3647,11 @@ spec: type: RollingUpdate selector: matchLabels: - app: ovn-ic-client + app: ovn-ic-controller template: metadata: labels: - app: ovn-ic-client + app: ovn-ic-controller component: network type: infra spec: @@ -3667,16 +3667,16 @@ spec: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchLabels: - app: ovn-ic-client + app: ovn-ic-controller topologyKey: kubernetes.io/hostname priorityClassName: system-cluster-critical serviceAccountName: ovn hostNetwork: true containers: - - name: ovn-ic-client + - name: ovn-ic-controller image: "$REGISTRY/kube-ovn:$VERSION" imagePullPolicy: $IMAGE_PULL_POLICY - command: ["/kube-ovn/start-ic-client.sh"] + command: ["/kube-ovn/start-ic-controller.sh"] securityContext: capabilities: add: ["SYS_NICE"] @@ -3728,7 +3728,7 @@ spec: optional: true secretName: kube-ovn-tls EOF -kubectl apply -f ovn-ic-client.yaml +kubectl apply -f ovn-ic-controller.yaml fi echo "-------------------------------" diff --git a/dist/images/start-ic-controller.sh b/dist/images/start-ic-controller.sh new file mode 100755 index 00000000000..764032656a7 --- /dev/null +++ b/dist/images/start-ic-controller.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +set -euo pipefail +ENABLE_SSL=${ENABLE_SSL:-false} +OVN_DB_IPS=${OVN_DB_IPS:-} + +function gen_conn_str { + if [[ -z "${OVN_DB_IPS}" ]]; then + if [[ "$1" == "6641" ]]; then + if [[ "$ENABLE_SSL" == "false" ]]; then + x="tcp:[${OVN_NB_SERVICE_HOST}]:${OVN_NB_SERVICE_PORT}" + else + x="ssl:[${OVN_NB_SERVICE_HOST}]:${OVN_NB_SERVICE_PORT}" + fi + else + if [[ "$ENABLE_SSL" == "false" ]]; then + x="tcp:[${OVN_SB_SERVICE_HOST}]:${OVN_SB_SERVICE_PORT}" + else + x="ssl:[${OVN_SB_SERVICE_HOST}]:${OVN_SB_SERVICE_PORT}" + fi + fi + else + t=$(echo -n "${OVN_DB_IPS}" | sed 's/[[:space:]]//g' | sed 's/,/ /g') + if [[ "$ENABLE_SSL" == "false" ]]; then + x=$(for i in ${t}; do echo -n "tcp:[$i]:$1",; done| sed 's/,$//') + else + x=$(for i in ${t}; do echo -n "ssl:[$i]:$1",; done| sed 's/,$//') + fi + fi + echo "$x" +} + +nb_addr="$(gen_conn_str 6641)" +sb_addr="$(gen_conn_str 6642)" + +for ((i=0; i<3; i++)); do + if [[ "$ENABLE_SSL" == "false" ]]; then + OVN_NB_DAEMON=$(ovn-nbctl --db="$nb_addr" --pidfile --detach --overwrite-pidfile) + else + OVN_NB_DAEMON=$(ovn-nbctl -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert --db="$nb_addr" --pidfile --detach --overwrite-pidfile) + fi + if echo -n "${OVN_NB_DAEMON}" | grep -qE '^/var/run/ovn/ovn-nbctl\.[0-9]+\.ctl$'; then + export OVN_NB_DAEMON + break + fi + if [ $(echo ${OVN_NB_DAEMON} | wc -c) -gt 64 ]; then + OVN_NB_DAEMON="$(echo ${OVN_NB_DAEMON} | cut -c1-64)..." + fi + echo "invalid ovn-nbctl daemon socket: \"${OVN_NB_DAEMON}\"" + unset OVN_NB_DAEMON + pkill -f ovn-nbctl +done + +if [ -z "${OVN_NB_DAEMON}" ]; then + echo "failed to start ovn-nbctl daemon" + exit 1 +fi + +exec ./kube-ovn-ic-controller --ovn-nb-addr="$nb_addr" \ + --ovn-sb-addr="$sb_addr" \ + $@ diff --git a/kubeovn-helm/templates/ic-controller-deploy.yaml b/kubeovn-helm/templates/ic-controller-deploy.yaml new file mode 100644 index 00000000000..ec60974b73a --- /dev/null +++ b/kubeovn-helm/templates/ic-controller-deploy.yaml @@ -0,0 +1,99 @@ +{{- if eq .Values.func.ENABLE_IC true }} +kind: Deployment +apiVersion: apps/v1 +metadata: + name: ovn-ic-controller + namespace: kube-system + annotations: + kubernetes.io/description: | + OVN IC Client +spec: + replicas: 1 + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate + selector: + matchLabels: + app: ovn-ic-controller + template: + metadata: + labels: + app: ovn-ic-controller + component: network + type: infra + spec: + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + - key: CriticalAddonsOnly + operator: Exists + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app: ovn-ic-controller + topologyKey: kubernetes.io/hostname + priorityClassName: system-cluster-critical + serviceAccountName: ovn + hostNetwork: true + containers: + - name: ovn-ic-controller + image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: ["/kube-ovn/start-ic-controller.sh"] + securityContext: + capabilities: + add: ["SYS_NICE"] + env: + - name: ENABLE_SSL + value: "{{ .Values.networking.ENABLE_SSL }}" + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OVN_DB_IPS + value: "{{ .Values.MASTER_NODES }}" + resources: + requests: + cpu: 300m + memory: 200Mi + limits: + cpu: 3 + memory: 1Gi + volumeMounts: + - mountPath: /var/run/ovn + name: host-run-ovn + - mountPath: /etc/ovn + name: host-config-ovn + - mountPath: /var/log/ovn + name: host-log-ovn + - mountPath: /etc/localtime + name: localtime + - mountPath: /var/run/tls + name: kube-ovn-tls + nodeSelector: + kubernetes.io/os: "linux" + kube-ovn/role: "master" + volumes: + - name: host-run-ovn + hostPath: + path: /run/ovn + - name: host-config-ovn + hostPath: + path: /etc/origin/ovn + - name: host-log-ovn + hostPath: + path: /var/log/ovn + - name: localtime + hostPath: + path: /etc/localtime + - name: kube-ovn-tls + secret: + optional: true + secretName: kube-ovn-tls +{{- end }} \ No newline at end of file diff --git a/pkg/ovn_ic_controller/config.go b/pkg/ovn_ic_controller/config.go new file mode 100644 index 00000000000..103ea327fe9 --- /dev/null +++ b/pkg/ovn_ic_controller/config.go @@ -0,0 +1,142 @@ +package ovn_ic_controller + +import ( + "flag" + "fmt" + "os" + + "github.com/spf13/pflag" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog/v2" + + clientset "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned" + "github.com/kubeovn/kube-ovn/pkg/util" +) + +// Configuration is the controller conf +type Configuration struct { + KubeConfigFile string + KubeClient kubernetes.Interface + KubeOvnClient clientset.Interface + + PodNamespace string + OvnNbAddr string + OvnSbAddr string + OvnTimeout int + + NodeSwitch string + ClusterRouter string + NodeSwitchCIDR string + + ClusterTcpLoadBalancer string + ClusterUdpLoadBalancer string + ClusterTcpSessionLoadBalancer string + ClusterUdpSessionLoadBalancer string +} + +func ParseFlags() (*Configuration, error) { + var ( + argKubeConfigFile = pflag.String("kubeconfig", "", "Path to kubeconfig file with authorization and master location information. If not set use the inCluster token.") + + argOvnNbAddr = pflag.String("ovn-nb-addr", "", "ovn-nb address") + argOvnSbAddr = pflag.String("ovn-sb-addr", "", "ovn-sb address") + argOvnTimeout = pflag.Int("ovn-timeout", 60, "") + + argClusterRouter = pflag.String("cluster-router", util.DefaultVpc, "The router name for cluster router") + argNodeSwitch = pflag.String("node-switch", "join", "The name of node gateway switch which help node to access pod network") + argNodeSwitchCIDR = pflag.String("node-switch-cidr", "100.64.0.0/16", "The cidr for node switch") + + argClusterTcpLoadBalancer = pflag.String("cluster-tcp-loadbalancer", "cluster-tcp-loadbalancer", "The name for cluster tcp loadbalancer") + argClusterUdpLoadBalancer = pflag.String("cluster-udp-loadbalancer", "cluster-udp-loadbalancer", "The name for cluster udp loadbalancer") + argClusterTcpSessionLoadBalancer = pflag.String("cluster-tcp-session-loadbalancer", "cluster-tcp-session-loadbalancer", "The name for cluster tcp session loadbalancer") + argClusterUdpSessionLoadBalancer = pflag.String("cluster-udp-session-loadbalancer", "cluster-udp-session-loadbalancer", "The name for cluster udp session loadbalancer") + ) + + klogFlags := flag.NewFlagSet("klog", flag.ContinueOnError) + klog.InitFlags(klogFlags) + + // Sync the glog and klog flags. + pflag.CommandLine.VisitAll(func(f1 *pflag.Flag) { + f2 := klogFlags.Lookup(f1.Name) + if f2 != nil { + value := f1.Value.String() + if err := f2.Value.Set(value); err != nil { + klog.Errorf("failed to set flag %v", err) + } + } + }) + + // change the behavior of cmdline + // not exit. not good + pflag.CommandLine.Init(os.Args[0], pflag.ContinueOnError) + pflag.CommandLine.AddGoFlagSet(klogFlags) + pflag.CommandLine.AddGoFlagSet(flag.CommandLine) + + if err := pflag.CommandLine.Parse(os.Args[1:]); err != nil { + return nil, err + } + + config := &Configuration{ + KubeConfigFile: *argKubeConfigFile, + + PodNamespace: os.Getenv("POD_NAMESPACE"), + OvnNbAddr: *argOvnNbAddr, + OvnSbAddr: *argOvnSbAddr, + OvnTimeout: *argOvnTimeout, + + ClusterRouter: *argClusterRouter, + NodeSwitch: *argNodeSwitch, + NodeSwitchCIDR: *argNodeSwitchCIDR, + + ClusterTcpLoadBalancer: *argClusterTcpLoadBalancer, + ClusterUdpLoadBalancer: *argClusterUdpLoadBalancer, + ClusterTcpSessionLoadBalancer: *argClusterTcpSessionLoadBalancer, + ClusterUdpSessionLoadBalancer: *argClusterUdpSessionLoadBalancer, + } + + if err := config.initKubeClient(); err != nil { + return nil, fmt.Errorf("failed to init kube client, %v", err) + } + + return config, nil +} + +func (config *Configuration) initKubeClient() error { + var cfg *rest.Config + var err error + if config.KubeConfigFile == "" { + klog.Infof("no --kubeconfig, use in-cluster kubernetes config") + cfg, err = rest.InClusterConfig() + if err != nil { + klog.Errorf("use in cluster config failed %v", err) + return err + } + } else { + cfg, err = clientcmd.BuildConfigFromFlags("", config.KubeConfigFile) + if err != nil { + klog.Errorf("use --kubeconfig %s failed %v", config.KubeConfigFile, err) + return err + } + } + cfg.QPS = 1000 + cfg.Burst = 2000 + + kubeOvnClient, err := clientset.NewForConfig(cfg) + if err != nil { + klog.Errorf("init kubeovn client failed %v", err) + return err + } + config.KubeOvnClient = kubeOvnClient + + cfg.ContentType = "application/vnd.kubernetes.protobuf" + cfg.AcceptContentTypes = "application/vnd.kubernetes.protobuf,application/json" + kubeClient, err := kubernetes.NewForConfig(cfg) + if err != nil { + klog.Errorf("init kubernetes client failed %v", err) + return err + } + config.KubeClient = kubeClient + return nil +} diff --git a/pkg/ovn_ic_controller/controller.go b/pkg/ovn_ic_controller/controller.go new file mode 100644 index 00000000000..e23b8445c54 --- /dev/null +++ b/pkg/ovn_ic_controller/controller.go @@ -0,0 +1,105 @@ +package ovn_ic_controller + +import ( + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + kubeinformers "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/scheme" + typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" + listerv1 "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/record" + "k8s.io/klog/v2" + + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + kubeovninformer "github.com/kubeovn/kube-ovn/pkg/client/informers/externalversions" + kubeovnlister "github.com/kubeovn/kube-ovn/pkg/client/listers/kubeovn/v1" + "github.com/kubeovn/kube-ovn/pkg/ovs" + "github.com/kubeovn/kube-ovn/pkg/util" +) + +const controllerAgentName = "ovn-ic-controller" + +type Controller struct { + config *Configuration + + subnetsLister kubeovnlister.SubnetLister + subnetSynced cache.InformerSynced + nodesLister listerv1.NodeLister + nodesSynced cache.InformerSynced + configMapsLister listerv1.ConfigMapLister + configMapsSynced cache.InformerSynced + + informerFactory kubeinformers.SharedInformerFactory + kubeovnInformerFactory kubeovninformer.SharedInformerFactory + recorder record.EventRecorder + + ovnClient *ovs.OvnClient + ovnLegacyClient *ovs.LegacyClient +} + +func NewController(config *Configuration) *Controller { + utilruntime.Must(kubeovnv1.AddToScheme(scheme.Scheme)) + klog.V(4).Info("Creating event broadcaster") + eventBroadcaster := record.NewBroadcaster() + eventBroadcaster.StartLogging(klog.Infof) + eventBroadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: config.KubeClient.CoreV1().Events("")}) + recorder := eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: controllerAgentName}) + + informerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(config.KubeClient, 0, + kubeinformers.WithTweakListOptions(func(listOption *metav1.ListOptions) { + listOption.AllowWatchBookmarks = true + })) + kubeovnInformerFactory := kubeovninformer.NewSharedInformerFactoryWithOptions(config.KubeOvnClient, 0, + kubeovninformer.WithTweakListOptions(func(listOption *metav1.ListOptions) { + listOption.AllowWatchBookmarks = true + })) + + nodeInformer := informerFactory.Core().V1().Nodes() + subnetInformer := kubeovnInformerFactory.Kubeovn().V1().Subnets() + configMapInformer := informerFactory.Core().V1().ConfigMaps() + + controller := &Controller{ + config: config, + ovnLegacyClient: ovs.NewLegacyClient(config.OvnNbAddr, config.OvnTimeout, config.OvnSbAddr, config.ClusterRouter, config.ClusterTcpLoadBalancer, config.ClusterUdpLoadBalancer, config.ClusterTcpSessionLoadBalancer, config.ClusterUdpSessionLoadBalancer, config.NodeSwitch, config.NodeSwitchCIDR), + + subnetsLister: subnetInformer.Lister(), + subnetSynced: subnetInformer.Informer().HasSynced, + nodesLister: nodeInformer.Lister(), + nodesSynced: nodeInformer.Informer().HasSynced, + configMapsLister: configMapInformer.Lister(), + configMapsSynced: configMapInformer.Informer().HasSynced, + + informerFactory: informerFactory, + kubeovnInformerFactory: kubeovnInformerFactory, + recorder: recorder, + } + + var err error + if controller.ovnClient, err = ovs.NewOvnClient(config.OvnNbAddr, config.OvnTimeout); err != nil { + util.LogFatalAndExit(err, "failed to create ovn client") + } + + return controller +} + +func (c *Controller) Run(stopCh <-chan struct{}) { + defer utilruntime.HandleCrash() + c.informerFactory.Start(stopCh) + c.kubeovnInformerFactory.Start(stopCh) + + if !cache.WaitForCacheSync(stopCh, c.subnetSynced, c.nodesSynced) { + util.LogFatalAndExit(nil, "failed to wait for caches to sync") + return + } + + klog.Info("Started workers") + go wait.Until(c.resyncInterConnection, time.Second, stopCh) + go wait.Until(c.SynRouteToPolicy, 5*time.Second, stopCh) + <-stopCh + klog.Info("Shutting down workers") +} diff --git a/pkg/ovn_ic_controller/ovn_ic_controller.go b/pkg/ovn_ic_controller/ovn_ic_controller.go new file mode 100644 index 00000000000..f3128869bf1 --- /dev/null +++ b/pkg/ovn_ic_controller/ovn_ic_controller.go @@ -0,0 +1,558 @@ +package ovn_ic_controller + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "reflect" + "sort" + "strings" + "time" + + "github.com/scylladb/go-set/strset" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" + + "github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb" + "github.com/kubeovn/kube-ovn/pkg/util" +) + +var ( + icEnabled = "unknown" + lastIcCm map[string]string + lastTSs []string + curTSs []string +) + +const ( + icNoAction = iota + icFirstEstablish + icConfigChange +) + +func (c *Controller) disableOVNIC(azName string) { + if err := c.removeInterConnection(azName); err != nil { + klog.Errorf("failed to remove ovn-ic, %v", err) + return + } + if err := c.delLearnedRoute(); err != nil { + klog.Errorf("failed to remove learned static routes, %v", err) + return + } + + if err := c.RemoveOldChassisInSbDB(azName); err != nil { + klog.Errorf("failed to remove remote chassis: %v", err) + } +} + +func (c *Controller) setAutoRoute(autoRoute bool) { + + var blackList []string + subnets, err := c.subnetsLister.List(labels.Everything()) + if err != nil { + klog.Errorf("failed to list subnets, %v", err) + return + } + for _, subnet := range subnets { + if subnet.Spec.DisableInterConnection || subnet.Name == c.config.NodeSwitch { + blackList = append(blackList, subnet.Spec.CIDRBlock) + } + } + nodes, err := c.nodesLister.List(labels.Everything()) + if err != nil { + klog.Errorf("failed to list node, %v", err) + return + } + for _, node := range nodes { + ipv4, ipv6 := util.GetNodeInternalIP(*node) + if ipv4 != "" { + blackList = append(blackList, ipv4) + } + if ipv6 != "" { + blackList = append(blackList, ipv6) + } + } + if err := c.ovnLegacyClient.SetICAutoRoute(autoRoute, blackList); err != nil { + klog.Errorf("failed to config auto route, %v", err) + return + } +} + +func (c *Controller) getICState(cmData, lastcmData map[string]string) int { + isCMEqual := reflect.DeepEqual(cmData, lastcmData) + if icEnabled != "true" && len(lastcmData) == 0 && cmData["enable-ic"] == "true" { + return icFirstEstablish + } + + if icEnabled == "true" && lastcmData != nil && isCMEqual { + var err error + c.ovnLegacyClient.OvnICNbAddress = genHostAddress(cmData["ic-db-host"], cmData["ic-nb-port"]) + curTSs, err = c.ovnLegacyClient.GetTs() + if err != nil { + klog.Errorf("failed to get Transit_Switch, %v", err) + return icNoAction + } + isTsEqual := reflect.DeepEqual(lastTSs, curTSs) + if isTsEqual { + return icNoAction + } + } + return icConfigChange +} + +func (c *Controller) resyncInterConnection() { + cm, err := c.configMapsLister.ConfigMaps(c.config.PodNamespace).Get(util.InterconnectionConfig) + if err != nil && !k8serrors.IsNotFound(err) { + klog.Errorf("failed to get ovn-ic-config, %v", err) + return + } + + if k8serrors.IsNotFound(err) || cm.Data["enable-ic"] == "false" { + if icEnabled == "false" { + return + } + klog.Info("start to remove ovn-ic") + azName := "" + icDBHost := "" + if cm != nil { + azName = cm.Data["az-name"] + icDBHost = cm.Data["ic-db-host"] + } else if lastIcCm != nil { + azName = lastIcCm["az-name"] + icDBHost = lastIcCm["ic-db-host"] + } + + if icDBHost != "" { + c.ovnLegacyClient.OvnICSbAddress = genHostAddress(icDBHost, cm.Data["ic-sb-port"]) + c.ovnLegacyClient.OvnICNbAddress = genHostAddress(icDBHost, cm.Data["ic-nb-port"]) + } + + c.disableOVNIC(azName) + icEnabled = "false" + lastIcCm = nil + + klog.Info("finish removing ovn-ic") + return + } + + autoRoute := false + if cm.Data["auto-route"] == "true" { + autoRoute = true + } + c.setAutoRoute(autoRoute) + + switch c.getICState(cm.Data, lastIcCm) { + case icNoAction: + return + case icFirstEstablish: + c.ovnLegacyClient.OvnICNbAddress = genHostAddress(cm.Data["ic-db-host"], cm.Data["ic-nb-port"]) + klog.Info("start to establish ovn-ic") + if err := c.establishInterConnection(cm.Data); err != nil { + klog.Errorf("failed to establish ovn-ic, %v", err) + return + } + curTSs, err := c.ovnLegacyClient.GetTs() + if err != nil { + klog.Errorf("failed to get Transit_Switch, %v", err) + return + } + icEnabled = "true" + lastIcCm = cm.Data + lastTSs = curTSs + klog.Info("finish establishing ovn-ic") + return + case icConfigChange: + c.ovnLegacyClient.OvnICSbAddress = genHostAddress(lastIcCm["ic-db-host"], cm.Data["ic-sb-port"]) + c.ovnLegacyClient.OvnICNbAddress = genHostAddress(lastIcCm["ic-db-host"], cm.Data["ic-nb-port"]) + c.disableOVNIC(lastIcCm["az-name"]) + klog.Info("start to reestablish ovn-ic") + if err := c.establishInterConnection(cm.Data); err != nil { + klog.Errorf("failed to reestablish ovn-ic, %v", err) + return + } + + icEnabled = "true" + lastIcCm = cm.Data + lastTSs = curTSs + klog.Info("finish reestablishing ovn-ic") + return + } +} + +func (c *Controller) removeInterConnection(azName string) error { + sel, _ := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{MatchLabels: map[string]string{util.ICGatewayLabel: "true"}}) + nodes, err := c.nodesLister.List(sel) + if err != nil { + klog.Errorf("failed to list nodes, %v", err) + return err + } + for _, cachedNode := range nodes { + no := cachedNode.DeepCopy() + patchPayloadTemplate := `[{ + "op": "%s", + "path": "/metadata/labels", + "value": %s + }]` + op := "replace" + if len(no.Labels) == 0 { + op = "add" + } + no.Labels[util.ICGatewayLabel] = "false" + raw, _ := json.Marshal(no.Labels) + patchPayload := fmt.Sprintf(patchPayloadTemplate, op, raw) + _, err = c.config.KubeClient.CoreV1().Nodes().Patch(context.Background(), no.Name, types.JSONPatchType, []byte(patchPayload), metav1.PatchOptions{}, "") + if err != nil { + klog.Errorf("patch ic gw node %s failed %v", no.Name, err) + return err + } + } + + if err := c.stopOVNIC(); err != nil { + klog.Errorf("failed to stop ovn-ic, %v", err) + return err + } + + if azName != "" { + if err := c.ovnLegacyClient.DeleteICResources(azName); err != nil { + klog.Errorf("failed to delete ovn-ic resource on az %s , %v", azName, err) + return err + } + } + + return nil +} + +func (c *Controller) establishInterConnection(config map[string]string) error { + if err := c.startOVNIC(config["ic-db-host"], config["ic-nb-port"], config["ic-sb-port"]); err != nil { + klog.Errorf("failed to start ovn-ic, %v", err) + return err + } + + if err := c.ovnLegacyClient.SetAzName(config["az-name"]); err != nil { + klog.Errorf("failed to set az name. %v", err) + return err + } + + tsNames, err := c.ovnLegacyClient.GetTs() + if err != nil { + klog.Errorf("failed to list ic logical switch. %v ", err) + return err + } + + sort.Strings(tsNames) + + gwNodes := strings.Split(config["gw-nodes"], ",") + chassises := make([]string, len(gwNodes)) + + for i, tsName := range tsNames { + gwNodesOrdered := moveElements(gwNodes, i) + for j, gw := range gwNodesOrdered { + gw = strings.TrimSpace(gw) + chassisID, err := c.ovnLegacyClient.GetChassis(gw) + if err != nil { + klog.Errorf("failed to get gw %s chassisID: %v", gw, err) + return err + } + if chassisID == "" { + return fmt.Errorf("no chassisID for gw %s", gw) + } + chassises[j] = chassisID + + cachedNode, err := c.nodesLister.Get(gw) + if err != nil { + klog.Errorf("failed to get gw node %s, %v", gw, err) + return err + } + node := cachedNode.DeepCopy() + patchPayloadTemplate := `[{ + "op": "%s", + "path": "/metadata/labels", + "value": %s + }]` + op := "replace" + if len(node.Labels) == 0 { + op = "add" + } + node.Labels[util.ICGatewayLabel] = "true" + raw, _ := json.Marshal(node.Labels) + patchPayload := fmt.Sprintf(patchPayloadTemplate, op, raw) + _, err = c.config.KubeClient.CoreV1().Nodes().Patch(context.Background(), gw, types.JSONPatchType, []byte(patchPayload), metav1.PatchOptions{}, "") + if err != nil { + klog.Errorf("patch gw node %s failed %v", gw, err) + return err + } + } + + tsPort := fmt.Sprintf("%s-%s", tsName, config["az-name"]) + exist, err := c.ovnLegacyClient.LogicalSwitchPortExists(tsPort) + if err != nil { + klog.Errorf("failed to list logical switch ports, %v", err) + return err + } + if exist { + klog.Infof("ts port %s already exists", tsPort) + continue + } + + lrpAddr, err := c.acquireLrpAddress(tsName) + if err != nil { + klog.Errorf("failed to acquire lrp address, %v", err) + return err + } + + if err = c.ovnLegacyClient.CreateICLogicalRouterPort(config["az-name"], tsName, util.GenerateMac(), lrpAddr, chassises); err != nil { + klog.Errorf("failed to create ovn-ic lrp %v", err) + return err + } + } + + return nil +} + +func (c *Controller) acquireLrpAddress(ts string) (string, error) { + cidr, err := c.ovnLegacyClient.GetTsSubnet(ts) + if err != nil { + klog.Errorf("failed to get ts subnet %s: %v", ts, err) + return "", err + } + existAddress, err := c.ovnLegacyClient.ListRemoteLogicalSwitchPortAddress() + if err != nil { + klog.Errorf("failed to list remote port address, %v", err) + return "", err + } + for { + random := util.GenerateRandomV4IP(cidr) + if !util.ContainsString(existAddress, random) { + return random, nil + } + klog.Infof("random ip %s already exists", random) + time.Sleep(1 * time.Second) + } +} + +func (c *Controller) startOVNIC(icHost, icNbPort, icSbPort string) error { + cmd := exec.Command("/usr/share/ovn/scripts/ovn-ctl", + fmt.Sprintf("--ovn-ic-nb-db=%s", genHostAddress(icHost, icNbPort)), + fmt.Sprintf("--ovn-ic-sb-db=%s", genHostAddress(icHost, icSbPort)), + fmt.Sprintf("--ovn-northd-nb-db=%s", c.config.OvnNbAddr), + fmt.Sprintf("--ovn-northd-sb-db=%s", c.config.OvnSbAddr), + "start_ic") + if os.Getenv("ENABLE_SSL") == "true" { + cmd = exec.Command("/usr/share/ovn/scripts/ovn-ctl", + fmt.Sprintf("--ovn-ic-nb-db=%s", genHostAddress(icHost, icNbPort)), + fmt.Sprintf("--ovn-ic-sb-db=%s", genHostAddress(icHost, icSbPort)), + fmt.Sprintf("--ovn-northd-nb-db=%s", c.config.OvnNbAddr), + fmt.Sprintf("--ovn-northd-sb-db=%s", c.config.OvnSbAddr), + "--ovn-ic-ssl-key=/var/run/tls/key", + "--ovn-ic-ssl-cert=/var/run/tls/cert", + "--ovn-ic-ssl-ca-cert=/var/run/tls/cacert", + "start_ic") + } + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("output: %s, err: %v", output, err) + } + return nil +} + +func (c *Controller) stopOVNIC() error { + cmd := exec.Command("/usr/share/ovn/scripts/ovn-ctl", "stop_ic") + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("output: %s, err: %v", output, err) + } + return nil +} + +func (c *Controller) delLearnedRoute() error { + originalPorts, err := c.ovnLegacyClient.CustomFindEntity("Logical_Router_Static_Route", []string{"_uuid", "ip_prefix"}) + if err != nil { + klog.Errorf("failed to list static routes of logical router, %v", err) + return err + } + filteredPorts, err := c.ovnLegacyClient.CustomFindEntity("Logical_Router_Static_Route", []string{"_uuid", "ip_prefix"}, "external_ids:ic-learned-route{<=}1") + if err != nil { + klog.Errorf("failed to filter static routes of logical router, %v", err) + return err + } + learnedPorts := []map[string][]string{} + for _, aOriPort := range originalPorts { + isFiltered := false + for _, aFtPort := range filteredPorts { + if aFtPort["_uuid"][0] == aOriPort["_uuid"][0] { + isFiltered = true + } + } + if !isFiltered { + learnedPorts = append(learnedPorts, aOriPort) + } + } + if len(learnedPorts) != 0 { + for _, aLdPort := range learnedPorts { + itsRouter, err := c.ovnLegacyClient.CustomFindEntity("Logical_Router", []string{"name"}, fmt.Sprintf("static_routes{>}%s", aLdPort["_uuid"][0])) + if err != nil { + klog.Errorf("failed to list logical router of static route %s, %v", aLdPort["_uuid"][0], err) + return err + } + + if len(aLdPort["ip_prefix"]) != 0 && len(itsRouter) != 0 && len(itsRouter[0]["name"]) != 0 { + if err := c.ovnLegacyClient.DeleteStaticRoute(aLdPort["ip_prefix"][0], itsRouter[0]["name"][0]); err != nil { + klog.Errorf("failed to delete stale route %s, %v", aLdPort["ip_prefix"][0], err) + return err + } + } + } + klog.V(5).Infof("finish removing learned routes") + } + return nil +} + +func genHostAddress(host, port string) (hostAddress string) { + hostList := strings.Split(host, ",") + if len(hostList) == 1 { + hostAddress = fmt.Sprintf("tcp:[%s]:%s", hostList[0], port) + } else { + var builder strings.Builder + i := 0 + for i < len(hostList)-1 { + builder.WriteString(fmt.Sprintf("tcp:[%s]:%s,", hostList[i], port)) + i++ + } + builder.WriteString(fmt.Sprintf("tcp:[%s]:%s", hostList[i], port)) + hostAddress = builder.String() + } + return hostAddress +} + +func (c *Controller) SynRouteToPolicy() { + c.syncOneRouteToPolicy(util.OvnICKey, util.OvnICConnected) + c.syncOneRouteToPolicy(util.OvnICKey, util.OvnICStatic) + // To support the version before kube-ovn v1.9, in which version the option tag is origin="" + c.syncOneRouteToPolicy(util.OvnICKey, util.OvnICNone) +} + +func (c *Controller) RemoveOldChassisInSbDB(azName string) error { + if azName == "" { + return nil + } + + azUUID, err := c.ovnLegacyClient.GetAzUUID(azName) + if err != nil { + klog.Errorf("failed to get UUID of AZ %s: %v", lastIcCm["az-name"], err) + return err + } + + if azUUID == "" { + klog.Infof("%s have already been deleted", azName) + return nil + } + + gateways, err := c.ovnLegacyClient.GetGatewayUUIDsInOneAZ(azUUID) + if err != nil { + klog.Errorf("failed to get gateway UUIDs in AZ %s: %v", azUUID, err) + return err + } + + routes, err := c.ovnLegacyClient.GetRouteUUIDsInOneAZ(azUUID) + if err != nil { + klog.Errorf("failed to get route UUIDs in AZ %s: %v", azUUID, err) + return err + } + + portBindings, err := c.ovnLegacyClient.GetPortBindingUUIDsInOneAZ(azUUID) + if err != nil { + klog.Errorf("failed to get Port_Binding UUIDs in AZ %s: %v", azUUID, err) + return err + } + + c.ovnLegacyClient.DestroyPortBindings(portBindings) + c.ovnLegacyClient.DestroyGateways(gateways) + c.ovnLegacyClient.DestroyRoutes(routes) + return c.ovnLegacyClient.DestroyChassis(azUUID) +} + +func stripPrefix(policyMatch string) (string, error) { + matches := strings.Split(policyMatch, "==") + if strings.Trim(matches[0], " ") == util.MatchV4Dst { + return strings.Trim(matches[1], " "), nil + } else { + return "", fmt.Errorf("policy %s is mismatched", policyMatch) + } +} + +func (c *Controller) syncOneRouteToPolicy(key, value string) { + lr, err := c.ovnClient.GetLogicalRouter(util.DefaultVpc, false) + if err != nil { + klog.Infof("logical router %s is not ready at %v", util.DefaultVpc, time.Now()) + return + } + lrRouteList, err := c.ovnClient.GetLogicalRouterRouteByOpts(key, value) + if err != nil { + klog.Errorf("failed to list lr ovn-ic route %v", err) + return + } + + lrPolicyList, err := c.ovnClient.GetLogicalRouterPoliciesByExtID(key, value) + if err != nil { + klog.Errorf("failed to list ovn-ic lr policy: %v", err) + return + } + + if len(lrRouteList) == 0 { + klog.V(5).Info(" lr ovn-ic route does not exist") + for _, lrPolicy := range lrPolicyList { + if err := c.ovnClient.DeleteRouterPolicy(lr, lrPolicy.UUID); err != nil { + klog.Errorf("deleting router policy failed %v", err) + } + } + return + } + + policyMap := map[string]string{} + + for _, lrPolicy := range lrPolicyList { + match, err := stripPrefix(lrPolicy.Match) + if err != nil { + klog.Errorf("policy match abnormal: %v", err) + continue + } + policyMap[match] = lrPolicy.UUID + } + networks := strset.NewWithSize(len(lrRouteList)) + for _, lrRoute := range lrRouteList { + networks.Add(lrRoute.IPPrefix) + } + + networks.Each(func(prefix string) bool { + if _, ok := policyMap[prefix]; ok { + delete(policyMap, prefix) + return true + } + match := util.MatchV4Dst + " == " + prefix + if err := c.ovnClient.AddRouterPolicy(lr, match, ovnnb.LogicalRouterPolicyActionAllow, + nil, map[string]string{key: value, "vendor": util.CniTypeName}, + util.OvnICPolicyPriority); err != nil { + klog.Errorf("failed to add router policy: %v", err) + } + + return true + }) + for _, uuid := range policyMap { + if err := c.ovnClient.DeleteRouterPolicy(lr, uuid); err != nil { + klog.Errorf("deleting router policy failed %v", err) + } + } +} + +func moveElements(arr []string, order int) []string { + if order >= len(arr) { + order = order % len(arr) + } + + return append(arr[order:], arr[:order]...) +} diff --git a/pkg/ovn_leader_checker/ovn.go b/pkg/ovn_leader_checker/ovn.go index 28177c2e245..b3d9f0b71e5 100755 --- a/pkg/ovn_leader_checker/ovn.go +++ b/pkg/ovn_leader_checker/ovn.go @@ -410,7 +410,6 @@ func doOvnLeaderCheck(cfg *Configuration, podName string, podNamespace string) { } } } - } func StartOvnLeaderCheck(cfg *Configuration) { @@ -436,30 +435,21 @@ func getTSCidr(index int) string { } func updateTS() error { - cmdstr := "ovn-ic-nbctl show | wc -l" - cmd := exec.Command("sh", "-c", cmdstr) + cmd := exec.Command("ovn-ic-nbctl", "show") output, err := cmd.CombinedOutput() if err != nil { return fmt.Errorf("ovn-ic-nbctl show output: %s, err: %v", output, err) } - lines := strings.Split(string(output), "\n") - if len(lines) < 1 { - return fmt.Errorf("ovsn-ic-nbctl show count line: err") - } - - existTSCount, err := strconv.Atoi(lines[0]) - - if err != nil { - return fmt.Errorf("existTSCount atoi failed output: %s, err: %v", output, err) + var existTSCount int + if lines := strings.TrimSpace(string(output)); lines != "" { + existTSCount = len(strings.Split(lines, "\n")) } - expectTSCount, err := strconv.Atoi(os.Getenv("TS_NUM")) - if err != nil { return fmt.Errorf("expectTSCount atoi failed output: %s, err: %v", output, err) } - if expectTSCount == existTSCount { + klog.Info("%d TS found, no chenges required.") return nil } @@ -483,7 +473,7 @@ func updateTS() error { return fmt.Errorf("output: %s, err: %v", output, err) } } - } else if expectTSCount < existTSCount { + } else { for i := existTSCount - 1; i >= expectTSCount; i-- { tsName := getTSName(i) cmd := exec.Command("ovn-ic-nbctl", From ac69495cd1df0b0c74119c530de773673ee0aac1 Mon Sep 17 00:00:00 2001 From: Changlu Yi Date: Fri, 12 Jan 2024 13:46:33 +0800 Subject: [PATCH 11/13] remove no need code --- cmd/ovn_ic_client/ovn_ic_client.go | 24 - dist/images/start-ic-client.sh | 60 -- kubeovn-helm/templates/ic-client-deploy.yaml | 99 ---- pkg/ovn_ic_client/config.go | 142 ----- pkg/ovn_ic_client/controller.go | 105 ---- pkg/ovn_ic_client/ovn_ic_client.go | 558 ------------------- 6 files changed, 988 deletions(-) delete mode 100644 cmd/ovn_ic_client/ovn_ic_client.go delete mode 100755 dist/images/start-ic-client.sh delete mode 100644 kubeovn-helm/templates/ic-client-deploy.yaml delete mode 100644 pkg/ovn_ic_client/config.go delete mode 100644 pkg/ovn_ic_client/controller.go delete mode 100644 pkg/ovn_ic_client/ovn_ic_client.go diff --git a/cmd/ovn_ic_client/ovn_ic_client.go b/cmd/ovn_ic_client/ovn_ic_client.go deleted file mode 100644 index db489dbbecb..00000000000 --- a/cmd/ovn_ic_client/ovn_ic_client.go +++ /dev/null @@ -1,24 +0,0 @@ -package ovn_ic_client - -import ( - "k8s.io/klog/v2" - "k8s.io/sample-controller/pkg/signals" - - "github.com/kubeovn/kube-ovn/pkg/ovn_ic_client" - "github.com/kubeovn/kube-ovn/pkg/util" - "github.com/kubeovn/kube-ovn/versions" -) - -func CmdMain() { - defer klog.Flush() - - klog.Infof(versions.String()) - config, err := ovn_ic_client.ParseFlags() - if err != nil { - util.LogFatalAndExit(err, "failed to parse config") - } - - stopCh := signals.SetupSignalHandler() - ctl := ovn_ic_client.NewController(config) - ctl.Run(stopCh) -} diff --git a/dist/images/start-ic-client.sh b/dist/images/start-ic-client.sh deleted file mode 100755 index 0bfc6b1954b..00000000000 --- a/dist/images/start-ic-client.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -ENABLE_SSL=${ENABLE_SSL:-false} -OVN_DB_IPS=${OVN_DB_IPS:-} - -function gen_conn_str { - if [[ -z "${OVN_DB_IPS}" ]]; then - if [[ "$1" == "6641" ]]; then - if [[ "$ENABLE_SSL" == "false" ]]; then - x="tcp:[${OVN_NB_SERVICE_HOST}]:${OVN_NB_SERVICE_PORT}" - else - x="ssl:[${OVN_NB_SERVICE_HOST}]:${OVN_NB_SERVICE_PORT}" - fi - else - if [[ "$ENABLE_SSL" == "false" ]]; then - x="tcp:[${OVN_SB_SERVICE_HOST}]:${OVN_SB_SERVICE_PORT}" - else - x="ssl:[${OVN_SB_SERVICE_HOST}]:${OVN_SB_SERVICE_PORT}" - fi - fi - else - t=$(echo -n "${OVN_DB_IPS}" | sed 's/[[:space:]]//g' | sed 's/,/ /g') - if [[ "$ENABLE_SSL" == "false" ]]; then - x=$(for i in ${t}; do echo -n "tcp:[$i]:$1",; done| sed 's/,$//') - else - x=$(for i in ${t}; do echo -n "ssl:[$i]:$1",; done| sed 's/,$//') - fi - fi - echo "$x" -} - -nb_addr="$(gen_conn_str 6641)" -sb_addr="$(gen_conn_str 6642)" - -for ((i=0; i<3; i++)); do - if [[ "$ENABLE_SSL" == "false" ]]; then - OVN_NB_DAEMON=$(ovn-nbctl --db="$nb_addr" --pidfile --detach --overwrite-pidfile) - else - OVN_NB_DAEMON=$(ovn-nbctl -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert --db="$nb_addr" --pidfile --detach --overwrite-pidfile) - fi - if echo -n "${OVN_NB_DAEMON}" | grep -qE '^/var/run/ovn/ovn-nbctl\.[0-9]+\.ctl$'; then - export OVN_NB_DAEMON - break - fi - if [ $(echo ${OVN_NB_DAEMON} | wc -c) -gt 64 ]; then - OVN_NB_DAEMON="$(echo ${OVN_NB_DAEMON} | cut -c1-64)..." - fi - echo "invalid ovn-nbctl daemon socket: \"${OVN_NB_DAEMON}\"" - unset OVN_NB_DAEMON - pkill -f ovn-nbctl -done - -if [ -z "${OVN_NB_DAEMON}" ]; then - echo "failed to start ovn-nbctl daemon" - exit 1 -fi - -exec ./kube-ovn-ic-client --ovn-nb-addr="$nb_addr" \ - --ovn-sb-addr="$sb_addr" \ - $@ diff --git a/kubeovn-helm/templates/ic-client-deploy.yaml b/kubeovn-helm/templates/ic-client-deploy.yaml deleted file mode 100644 index cea6a699c36..00000000000 --- a/kubeovn-helm/templates/ic-client-deploy.yaml +++ /dev/null @@ -1,99 +0,0 @@ -{{- if eq .Values.func.ENABLE_IC true }} -kind: Deployment -apiVersion: apps/v1 -metadata: - name: ovn-ic-client - namespace: kube-system - annotations: - kubernetes.io/description: | - OVN IC Client -spec: - replicas: 1 - strategy: - rollingUpdate: - maxSurge: 0 - maxUnavailable: 1 - type: RollingUpdate - selector: - matchLabels: - app: ovn-ic-client - template: - metadata: - labels: - app: ovn-ic-client - component: network - type: infra - spec: - tolerations: - - effect: NoSchedule - operator: Exists - - effect: NoExecute - operator: Exists - - key: CriticalAddonsOnly - operator: Exists - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - app: ovn-ic-client - topologyKey: kubernetes.io/hostname - priorityClassName: system-cluster-critical - serviceAccountName: ovn - hostNetwork: true - containers: - - name: ovn-ic-client - image: {{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }} - imagePullPolicy: {{ .Values.image.pullPolicy }} - command: ["/kube-ovn/start-ic-client.sh"] - securityContext: - capabilities: - add: ["SYS_NICE"] - env: - - name: ENABLE_SSL - value: "{{ .Values.networking.ENABLE_SSL }}" - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: OVN_DB_IPS - value: "{{ .Values.MASTER_NODES }}" - resources: - requests: - cpu: 300m - memory: 200Mi - limits: - cpu: 3 - memory: 1Gi - volumeMounts: - - mountPath: /var/run/ovn - name: host-run-ovn - - mountPath: /etc/ovn - name: host-config-ovn - - mountPath: /var/log/ovn - name: host-log-ovn - - mountPath: /etc/localtime - name: localtime - - mountPath: /var/run/tls - name: kube-ovn-tls - nodeSelector: - kubernetes.io/os: "linux" - kube-ovn/role: "master" - volumes: - - name: host-run-ovn - hostPath: - path: /run/ovn - - name: host-config-ovn - hostPath: - path: /etc/origin/ovn - - name: host-log-ovn - hostPath: - path: /var/log/ovn - - name: localtime - hostPath: - path: /etc/localtime - - name: kube-ovn-tls - secret: - optional: true - secretName: kube-ovn-tls -{{- end }} \ No newline at end of file diff --git a/pkg/ovn_ic_client/config.go b/pkg/ovn_ic_client/config.go deleted file mode 100644 index 55f55820389..00000000000 --- a/pkg/ovn_ic_client/config.go +++ /dev/null @@ -1,142 +0,0 @@ -package ovn_ic_client - -import ( - "flag" - "fmt" - "os" - - "github.com/spf13/pflag" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/klog/v2" - - clientset "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned" - "github.com/kubeovn/kube-ovn/pkg/util" -) - -// Configuration is the controller conf -type Configuration struct { - KubeConfigFile string - KubeClient kubernetes.Interface - KubeOvnClient clientset.Interface - - PodNamespace string - OvnNbAddr string - OvnSbAddr string - OvnTimeout int - - NodeSwitch string - ClusterRouter string - NodeSwitchCIDR string - - ClusterTcpLoadBalancer string - ClusterUdpLoadBalancer string - ClusterTcpSessionLoadBalancer string - ClusterUdpSessionLoadBalancer string -} - -func ParseFlags() (*Configuration, error) { - var ( - argKubeConfigFile = pflag.String("kubeconfig", "", "Path to kubeconfig file with authorization and master location information. If not set use the inCluster token.") - - argOvnNbAddr = pflag.String("ovn-nb-addr", "", "ovn-nb address") - argOvnSbAddr = pflag.String("ovn-sb-addr", "", "ovn-sb address") - argOvnTimeout = pflag.Int("ovn-timeout", 60, "") - - argClusterRouter = pflag.String("cluster-router", util.DefaultVpc, "The router name for cluster router") - argNodeSwitch = pflag.String("node-switch", "join", "The name of node gateway switch which help node to access pod network") - argNodeSwitchCIDR = pflag.String("node-switch-cidr", "100.64.0.0/16", "The cidr for node switch") - - argClusterTcpLoadBalancer = pflag.String("cluster-tcp-loadbalancer", "cluster-tcp-loadbalancer", "The name for cluster tcp loadbalancer") - argClusterUdpLoadBalancer = pflag.String("cluster-udp-loadbalancer", "cluster-udp-loadbalancer", "The name for cluster udp loadbalancer") - argClusterTcpSessionLoadBalancer = pflag.String("cluster-tcp-session-loadbalancer", "cluster-tcp-session-loadbalancer", "The name for cluster tcp session loadbalancer") - argClusterUdpSessionLoadBalancer = pflag.String("cluster-udp-session-loadbalancer", "cluster-udp-session-loadbalancer", "The name for cluster udp session loadbalancer") - ) - - klogFlags := flag.NewFlagSet("klog", flag.ContinueOnError) - klog.InitFlags(klogFlags) - - // Sync the glog and klog flags. - pflag.CommandLine.VisitAll(func(f1 *pflag.Flag) { - f2 := klogFlags.Lookup(f1.Name) - if f2 != nil { - value := f1.Value.String() - if err := f2.Value.Set(value); err != nil { - klog.Errorf("failed to set flag %v", err) - } - } - }) - - // change the behavior of cmdline - // not exit. not good - pflag.CommandLine.Init(os.Args[0], pflag.ContinueOnError) - pflag.CommandLine.AddGoFlagSet(klogFlags) - pflag.CommandLine.AddGoFlagSet(flag.CommandLine) - - if err := pflag.CommandLine.Parse(os.Args[1:]); err != nil { - return nil, err - } - - config := &Configuration{ - KubeConfigFile: *argKubeConfigFile, - - PodNamespace: os.Getenv("POD_NAMESPACE"), - OvnNbAddr: *argOvnNbAddr, - OvnSbAddr: *argOvnSbAddr, - OvnTimeout: *argOvnTimeout, - - ClusterRouter: *argClusterRouter, - NodeSwitch: *argNodeSwitch, - NodeSwitchCIDR: *argNodeSwitchCIDR, - - ClusterTcpLoadBalancer: *argClusterTcpLoadBalancer, - ClusterUdpLoadBalancer: *argClusterUdpLoadBalancer, - ClusterTcpSessionLoadBalancer: *argClusterTcpSessionLoadBalancer, - ClusterUdpSessionLoadBalancer: *argClusterUdpSessionLoadBalancer, - } - - if err := config.initKubeClient(); err != nil { - return nil, fmt.Errorf("failed to init kube client, %v", err) - } - - return config, nil -} - -func (config *Configuration) initKubeClient() error { - var cfg *rest.Config - var err error - if config.KubeConfigFile == "" { - klog.Infof("no --kubeconfig, use in-cluster kubernetes config") - cfg, err = rest.InClusterConfig() - if err != nil { - klog.Errorf("use in cluster config failed %v", err) - return err - } - } else { - cfg, err = clientcmd.BuildConfigFromFlags("", config.KubeConfigFile) - if err != nil { - klog.Errorf("use --kubeconfig %s failed %v", config.KubeConfigFile, err) - return err - } - } - cfg.QPS = 1000 - cfg.Burst = 2000 - - kubeOvnClient, err := clientset.NewForConfig(cfg) - if err != nil { - klog.Errorf("init kubeovn client failed %v", err) - return err - } - config.KubeOvnClient = kubeOvnClient - - cfg.ContentType = "application/vnd.kubernetes.protobuf" - cfg.AcceptContentTypes = "application/vnd.kubernetes.protobuf,application/json" - kubeClient, err := kubernetes.NewForConfig(cfg) - if err != nil { - klog.Errorf("init kubernetes client failed %v", err) - return err - } - config.KubeClient = kubeClient - return nil -} diff --git a/pkg/ovn_ic_client/controller.go b/pkg/ovn_ic_client/controller.go deleted file mode 100644 index 04686f0c36b..00000000000 --- a/pkg/ovn_ic_client/controller.go +++ /dev/null @@ -1,105 +0,0 @@ -package ovn_ic_client - -import ( - "time" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/apimachinery/pkg/util/wait" - kubeinformers "k8s.io/client-go/informers" - "k8s.io/client-go/kubernetes/scheme" - typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" - listerv1 "k8s.io/client-go/listers/core/v1" - "k8s.io/client-go/tools/cache" - "k8s.io/client-go/tools/record" - "k8s.io/klog/v2" - - kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" - kubeovninformer "github.com/kubeovn/kube-ovn/pkg/client/informers/externalversions" - kubeovnlister "github.com/kubeovn/kube-ovn/pkg/client/listers/kubeovn/v1" - "github.com/kubeovn/kube-ovn/pkg/ovs" - "github.com/kubeovn/kube-ovn/pkg/util" -) - -const controllerAgentName = "ovn-ic-client" - -type Controller struct { - config *Configuration - - subnetsLister kubeovnlister.SubnetLister - subnetSynced cache.InformerSynced - nodesLister listerv1.NodeLister - nodesSynced cache.InformerSynced - configMapsLister listerv1.ConfigMapLister - configMapsSynced cache.InformerSynced - - informerFactory kubeinformers.SharedInformerFactory - kubeovnInformerFactory kubeovninformer.SharedInformerFactory - recorder record.EventRecorder - - ovnClient *ovs.OvnClient - ovnLegacyClient *ovs.LegacyClient -} - -func NewController(config *Configuration) *Controller { - utilruntime.Must(kubeovnv1.AddToScheme(scheme.Scheme)) - klog.V(4).Info("Creating event broadcaster") - eventBroadcaster := record.NewBroadcaster() - eventBroadcaster.StartLogging(klog.Infof) - eventBroadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: config.KubeClient.CoreV1().Events("")}) - recorder := eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: controllerAgentName}) - - informerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(config.KubeClient, 0, - kubeinformers.WithTweakListOptions(func(listOption *metav1.ListOptions) { - listOption.AllowWatchBookmarks = true - })) - kubeovnInformerFactory := kubeovninformer.NewSharedInformerFactoryWithOptions(config.KubeOvnClient, 0, - kubeovninformer.WithTweakListOptions(func(listOption *metav1.ListOptions) { - listOption.AllowWatchBookmarks = true - })) - - nodeInformer := informerFactory.Core().V1().Nodes() - subnetInformer := kubeovnInformerFactory.Kubeovn().V1().Subnets() - configMapInformer := informerFactory.Core().V1().ConfigMaps() - - controller := &Controller{ - config: config, - ovnLegacyClient: ovs.NewLegacyClient(config.OvnNbAddr, config.OvnTimeout, config.OvnSbAddr, config.ClusterRouter, config.ClusterTcpLoadBalancer, config.ClusterUdpLoadBalancer, config.ClusterTcpSessionLoadBalancer, config.ClusterUdpSessionLoadBalancer, config.NodeSwitch, config.NodeSwitchCIDR), - - subnetsLister: subnetInformer.Lister(), - subnetSynced: subnetInformer.Informer().HasSynced, - nodesLister: nodeInformer.Lister(), - nodesSynced: nodeInformer.Informer().HasSynced, - configMapsLister: configMapInformer.Lister(), - configMapsSynced: configMapInformer.Informer().HasSynced, - - informerFactory: informerFactory, - kubeovnInformerFactory: kubeovnInformerFactory, - recorder: recorder, - } - - var err error - if controller.ovnClient, err = ovs.NewOvnClient(config.OvnNbAddr, config.OvnTimeout); err != nil { - util.LogFatalAndExit(err, "failed to create ovn client") - } - - return controller -} - -func (c *Controller) Run(stopCh <-chan struct{}) { - defer utilruntime.HandleCrash() - c.informerFactory.Start(stopCh) - c.kubeovnInformerFactory.Start(stopCh) - - if !cache.WaitForCacheSync(stopCh, c.subnetSynced, c.nodesSynced) { - util.LogFatalAndExit(nil, "failed to wait for caches to sync") - return - } - - klog.Info("Started workers") - go wait.Until(c.resyncInterConnection, time.Second, stopCh) - go wait.Until(c.SynRouteToPolicy, 5*time.Second, stopCh) - <-stopCh - klog.Info("Shutting down workers") -} diff --git a/pkg/ovn_ic_client/ovn_ic_client.go b/pkg/ovn_ic_client/ovn_ic_client.go deleted file mode 100644 index 6451642d1d6..00000000000 --- a/pkg/ovn_ic_client/ovn_ic_client.go +++ /dev/null @@ -1,558 +0,0 @@ -package ovn_ic_client - -import ( - "context" - "encoding/json" - "fmt" - "os" - "os/exec" - "reflect" - "sort" - "strings" - "time" - - "github.com/scylladb/go-set/strset" - k8serrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/types" - "k8s.io/klog/v2" - - "github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb" - "github.com/kubeovn/kube-ovn/pkg/util" -) - -var ( - icEnabled = "unknown" - lastIcCm map[string]string - lastTSs []string - curTSs []string -) - -const ( - icNoAction = iota - icFirstEstablish - icConfigChange -) - -func (c *Controller) disableOVNIC(azName string) { - if err := c.removeInterConnection(azName); err != nil { - klog.Errorf("failed to remove ovn-ic, %v", err) - return - } - if err := c.delLearnedRoute(); err != nil { - klog.Errorf("failed to remove learned static routes, %v", err) - return - } - - if err := c.RemoveOldChassisInSbDB(azName); err != nil { - klog.Errorf("failed to remove remote chassis: %v", err) - } -} - -func (c *Controller) setAutoRoute(autoRoute bool) { - - var blackList []string - subnets, err := c.subnetsLister.List(labels.Everything()) - if err != nil { - klog.Errorf("failed to list subnets, %v", err) - return - } - for _, subnet := range subnets { - if subnet.Spec.DisableInterConnection || subnet.Name == c.config.NodeSwitch { - blackList = append(blackList, subnet.Spec.CIDRBlock) - } - } - nodes, err := c.nodesLister.List(labels.Everything()) - if err != nil { - klog.Errorf("failed to list node, %v", err) - return - } - for _, node := range nodes { - ipv4, ipv6 := util.GetNodeInternalIP(*node) - if ipv4 != "" { - blackList = append(blackList, ipv4) - } - if ipv6 != "" { - blackList = append(blackList, ipv6) - } - } - if err := c.ovnLegacyClient.SetICAutoRoute(autoRoute, blackList); err != nil { - klog.Errorf("failed to config auto route, %v", err) - return - } -} - -func (c *Controller) getICState(cmData, lastcmData map[string]string) int { - isCMEqual := reflect.DeepEqual(cmData, lastcmData) - if icEnabled != "true" && len(lastcmData) == 0 && cmData["enable-ic"] == "true" { - return icFirstEstablish - } - - if icEnabled == "true" && lastcmData != nil && isCMEqual { - var err error - c.ovnLegacyClient.OvnICNbAddress = genHostAddress(cmData["ic-db-host"], cmData["ic-nb-port"]) - curTSs, err = c.ovnLegacyClient.GetTs() - if err != nil { - klog.Errorf("failed to get Transit_Switch, %v", err) - return icNoAction - } - isTsEqual := reflect.DeepEqual(lastTSs, curTSs) - if isTsEqual { - return icNoAction - } - } - return icConfigChange -} - -func (c *Controller) resyncInterConnection() { - cm, err := c.configMapsLister.ConfigMaps(c.config.PodNamespace).Get(util.InterconnectionConfig) - if err != nil && !k8serrors.IsNotFound(err) { - klog.Errorf("failed to get ovn-ic-config, %v", err) - return - } - - if k8serrors.IsNotFound(err) || cm.Data["enable-ic"] == "false" { - if icEnabled == "false" { - return - } - klog.Info("start to remove ovn-ic") - azName := "" - icDBHost := "" - if cm != nil { - azName = cm.Data["az-name"] - icDBHost = cm.Data["ic-db-host"] - } else if lastIcCm != nil { - azName = lastIcCm["az-name"] - icDBHost = lastIcCm["ic-db-host"] - } - - if icDBHost != "" { - c.ovnLegacyClient.OvnICSbAddress = genHostAddress(icDBHost, cm.Data["ic-sb-port"]) - c.ovnLegacyClient.OvnICNbAddress = genHostAddress(icDBHost, cm.Data["ic-nb-port"]) - } - - c.disableOVNIC(azName) - icEnabled = "false" - lastIcCm = nil - - klog.Info("finish removing ovn-ic") - return - } - - autoRoute := false - if cm.Data["auto-route"] == "true" { - autoRoute = true - } - c.setAutoRoute(autoRoute) - - switch c.getICState(cm.Data, lastIcCm) { - case icNoAction: - return - case icFirstEstablish: - c.ovnLegacyClient.OvnICNbAddress = genHostAddress(cm.Data["ic-db-host"], cm.Data["ic-nb-port"]) - klog.Info("start to establish ovn-ic") - if err := c.establishInterConnection(cm.Data); err != nil { - klog.Errorf("failed to establish ovn-ic, %v", err) - return - } - curTSs, err := c.ovnLegacyClient.GetTs() - if err != nil { - klog.Errorf("failed to get Transit_Switch, %v", err) - return - } - icEnabled = "true" - lastIcCm = cm.Data - lastTSs = curTSs - klog.Info("finish establishing ovn-ic") - return - case icConfigChange: - c.ovnLegacyClient.OvnICSbAddress = genHostAddress(lastIcCm["ic-db-host"], cm.Data["ic-sb-port"]) - c.ovnLegacyClient.OvnICNbAddress = genHostAddress(lastIcCm["ic-db-host"], cm.Data["ic-nb-port"]) - c.disableOVNIC(lastIcCm["az-name"]) - klog.Info("start to reestablish ovn-ic") - if err := c.establishInterConnection(cm.Data); err != nil { - klog.Errorf("failed to reestablish ovn-ic, %v", err) - return - } - - icEnabled = "true" - lastIcCm = cm.Data - lastTSs = curTSs - klog.Info("finish reestablishing ovn-ic") - return - } -} - -func (c *Controller) removeInterConnection(azName string) error { - sel, _ := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{MatchLabels: map[string]string{util.ICGatewayLabel: "true"}}) - nodes, err := c.nodesLister.List(sel) - if err != nil { - klog.Errorf("failed to list nodes, %v", err) - return err - } - for _, cachedNode := range nodes { - no := cachedNode.DeepCopy() - patchPayloadTemplate := `[{ - "op": "%s", - "path": "/metadata/labels", - "value": %s - }]` - op := "replace" - if len(no.Labels) == 0 { - op = "add" - } - no.Labels[util.ICGatewayLabel] = "false" - raw, _ := json.Marshal(no.Labels) - patchPayload := fmt.Sprintf(patchPayloadTemplate, op, raw) - _, err = c.config.KubeClient.CoreV1().Nodes().Patch(context.Background(), no.Name, types.JSONPatchType, []byte(patchPayload), metav1.PatchOptions{}, "") - if err != nil { - klog.Errorf("patch ic gw node %s failed %v", no.Name, err) - return err - } - } - - if err := c.stopOVNIC(); err != nil { - klog.Errorf("failed to stop ovn-ic, %v", err) - return err - } - - if azName != "" { - if err := c.ovnLegacyClient.DeleteICResources(azName); err != nil { - klog.Errorf("failed to delete ovn-ic resource on az %s , %v", azName, err) - return err - } - } - - return nil -} - -func (c *Controller) establishInterConnection(config map[string]string) error { - if err := c.startOVNIC(config["ic-db-host"], config["ic-nb-port"], config["ic-sb-port"]); err != nil { - klog.Errorf("failed to start ovn-ic, %v", err) - return err - } - - if err := c.ovnLegacyClient.SetAzName(config["az-name"]); err != nil { - klog.Errorf("failed to set az name. %v", err) - return err - } - - tsNames, err := c.ovnLegacyClient.GetTs() - if err != nil { - klog.Errorf("failed to list ic logical switch. %v ", err) - return err - } - - sort.Strings(tsNames) - - gwNodes := strings.Split(config["gw-nodes"], ",") - chassises := make([]string, len(gwNodes)) - - for i, tsName := range tsNames { - gwNodesOrdered := moveElements(gwNodes, i) - for j, gw := range gwNodesOrdered { - gw = strings.TrimSpace(gw) - chassisID, err := c.ovnLegacyClient.GetChassis(gw) - if err != nil { - klog.Errorf("failed to get gw %s chassisID: %v", gw, err) - return err - } - if chassisID == "" { - return fmt.Errorf("no chassisID for gw %s", gw) - } - chassises[j] = chassisID - - cachedNode, err := c.nodesLister.Get(gw) - if err != nil { - klog.Errorf("failed to get gw node %s, %v", gw, err) - return err - } - node := cachedNode.DeepCopy() - patchPayloadTemplate := `[{ - "op": "%s", - "path": "/metadata/labels", - "value": %s - }]` - op := "replace" - if len(node.Labels) == 0 { - op = "add" - } - node.Labels[util.ICGatewayLabel] = "true" - raw, _ := json.Marshal(node.Labels) - patchPayload := fmt.Sprintf(patchPayloadTemplate, op, raw) - _, err = c.config.KubeClient.CoreV1().Nodes().Patch(context.Background(), gw, types.JSONPatchType, []byte(patchPayload), metav1.PatchOptions{}, "") - if err != nil { - klog.Errorf("patch gw node %s failed %v", gw, err) - return err - } - } - - tsPort := fmt.Sprintf("%s-%s", tsName, config["az-name"]) - exist, err := c.ovnLegacyClient.LogicalSwitchPortExists(tsPort) - if err != nil { - klog.Errorf("failed to list logical switch ports, %v", err) - return err - } - if exist { - klog.Infof("ts port %s already exists", tsPort) - continue - } - - lrpAddr, err := c.acquireLrpAddress(tsName) - if err != nil { - klog.Errorf("failed to acquire lrp address, %v", err) - return err - } - - if err = c.ovnLegacyClient.CreateICLogicalRouterPort(config["az-name"], tsName, util.GenerateMac(), lrpAddr, chassises); err != nil { - klog.Errorf("failed to create ovn-ic lrp %v", err) - return err - } - } - - return nil -} - -func (c *Controller) acquireLrpAddress(ts string) (string, error) { - cidr, err := c.ovnLegacyClient.GetTsSubnet(ts) - if err != nil { - klog.Errorf("failed to get ts subnet %s: %v", ts, err) - return "", err - } - existAddress, err := c.ovnLegacyClient.ListRemoteLogicalSwitchPortAddress() - if err != nil { - klog.Errorf("failed to list remote port address, %v", err) - return "", err - } - for { - random := util.GenerateRandomV4IP(cidr) - if !util.ContainsString(existAddress, random) { - return random, nil - } - klog.Infof("random ip %s already exists", random) - time.Sleep(1 * time.Second) - } -} - -func (c *Controller) startOVNIC(icHost, icNbPort, icSbPort string) error { - cmd := exec.Command("/usr/share/ovn/scripts/ovn-ctl", - fmt.Sprintf("--ovn-ic-nb-db=%s", genHostAddress(icHost, icNbPort)), - fmt.Sprintf("--ovn-ic-sb-db=%s", genHostAddress(icHost, icSbPort)), - fmt.Sprintf("--ovn-northd-nb-db=%s", c.config.OvnNbAddr), - fmt.Sprintf("--ovn-northd-sb-db=%s", c.config.OvnSbAddr), - "start_ic") - if os.Getenv("ENABLE_SSL") == "true" { - cmd = exec.Command("/usr/share/ovn/scripts/ovn-ctl", - fmt.Sprintf("--ovn-ic-nb-db=%s", genHostAddress(icHost, icNbPort)), - fmt.Sprintf("--ovn-ic-sb-db=%s", genHostAddress(icHost, icSbPort)), - fmt.Sprintf("--ovn-northd-nb-db=%s", c.config.OvnNbAddr), - fmt.Sprintf("--ovn-northd-sb-db=%s", c.config.OvnSbAddr), - "--ovn-ic-ssl-key=/var/run/tls/key", - "--ovn-ic-ssl-cert=/var/run/tls/cert", - "--ovn-ic-ssl-ca-cert=/var/run/tls/cacert", - "start_ic") - } - output, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("output: %s, err: %v", output, err) - } - return nil -} - -func (c *Controller) stopOVNIC() error { - cmd := exec.Command("/usr/share/ovn/scripts/ovn-ctl", "stop_ic") - output, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("output: %s, err: %v", output, err) - } - return nil -} - -func (c *Controller) delLearnedRoute() error { - originalPorts, err := c.ovnLegacyClient.CustomFindEntity("Logical_Router_Static_Route", []string{"_uuid", "ip_prefix"}) - if err != nil { - klog.Errorf("failed to list static routes of logical router, %v", err) - return err - } - filteredPorts, err := c.ovnLegacyClient.CustomFindEntity("Logical_Router_Static_Route", []string{"_uuid", "ip_prefix"}, "external_ids:ic-learned-route{<=}1") - if err != nil { - klog.Errorf("failed to filter static routes of logical router, %v", err) - return err - } - learnedPorts := []map[string][]string{} - for _, aOriPort := range originalPorts { - isFiltered := false - for _, aFtPort := range filteredPorts { - if aFtPort["_uuid"][0] == aOriPort["_uuid"][0] { - isFiltered = true - } - } - if !isFiltered { - learnedPorts = append(learnedPorts, aOriPort) - } - } - if len(learnedPorts) != 0 { - for _, aLdPort := range learnedPorts { - itsRouter, err := c.ovnLegacyClient.CustomFindEntity("Logical_Router", []string{"name"}, fmt.Sprintf("static_routes{>}%s", aLdPort["_uuid"][0])) - if err != nil { - klog.Errorf("failed to list logical router of static route %s, %v", aLdPort["_uuid"][0], err) - return err - } - - if len(aLdPort["ip_prefix"]) != 0 && len(itsRouter) != 0 && len(itsRouter[0]["name"]) != 0 { - if err := c.ovnLegacyClient.DeleteStaticRoute(aLdPort["ip_prefix"][0], itsRouter[0]["name"][0]); err != nil { - klog.Errorf("failed to delete stale route %s, %v", aLdPort["ip_prefix"][0], err) - return err - } - } - } - klog.V(5).Infof("finish removing learned routes") - } - return nil -} - -func genHostAddress(host, port string) (hostAddress string) { - hostList := strings.Split(host, ",") - if len(hostList) == 1 { - hostAddress = fmt.Sprintf("tcp:[%s]:%s", hostList[0], port) - } else { - var builder strings.Builder - i := 0 - for i < len(hostList)-1 { - builder.WriteString(fmt.Sprintf("tcp:[%s]:%s,", hostList[i], port)) - i++ - } - builder.WriteString(fmt.Sprintf("tcp:[%s]:%s", hostList[i], port)) - hostAddress = builder.String() - } - return hostAddress -} - -func (c *Controller) SynRouteToPolicy() { - c.syncOneRouteToPolicy(util.OvnICKey, util.OvnICConnected) - c.syncOneRouteToPolicy(util.OvnICKey, util.OvnICStatic) - // To support the version before kube-ovn v1.9, in which version the option tag is origin="" - c.syncOneRouteToPolicy(util.OvnICKey, util.OvnICNone) -} - -func (c *Controller) RemoveOldChassisInSbDB(azName string) error { - if azName == "" { - return nil - } - - azUUID, err := c.ovnLegacyClient.GetAzUUID(azName) - if err != nil { - klog.Errorf("failed to get UUID of AZ %s: %v", lastIcCm["az-name"], err) - return err - } - - if azUUID == "" { - klog.Infof("%s have already been deleted", azName) - return nil - } - - gateways, err := c.ovnLegacyClient.GetGatewayUUIDsInOneAZ(azUUID) - if err != nil { - klog.Errorf("failed to get gateway UUIDs in AZ %s: %v", azUUID, err) - return err - } - - routes, err := c.ovnLegacyClient.GetRouteUUIDsInOneAZ(azUUID) - if err != nil { - klog.Errorf("failed to get route UUIDs in AZ %s: %v", azUUID, err) - return err - } - - portBindings, err := c.ovnLegacyClient.GetPortBindingUUIDsInOneAZ(azUUID) - if err != nil { - klog.Errorf("failed to get Port_Binding UUIDs in AZ %s: %v", azUUID, err) - return err - } - - c.ovnLegacyClient.DestroyPortBindings(portBindings) - c.ovnLegacyClient.DestroyGateways(gateways) - c.ovnLegacyClient.DestroyRoutes(routes) - return c.ovnLegacyClient.DestroyChassis(azUUID) -} - -func stripPrefix(policyMatch string) (string, error) { - matches := strings.Split(policyMatch, "==") - if strings.Trim(matches[0], " ") == util.MatchV4Dst { - return strings.Trim(matches[1], " "), nil - } else { - return "", fmt.Errorf("policy %s is mismatched", policyMatch) - } -} - -func (c *Controller) syncOneRouteToPolicy(key, value string) { - lr, err := c.ovnClient.GetLogicalRouter(util.DefaultVpc, false) - if err != nil { - klog.Infof("logical router %s is not ready at %v", util.DefaultVpc, time.Now()) - return - } - lrRouteList, err := c.ovnClient.GetLogicalRouterRouteByOpts(key, value) - if err != nil { - klog.Errorf("failed to list lr ovn-ic route %v", err) - return - } - - lrPolicyList, err := c.ovnClient.GetLogicalRouterPoliciesByExtID(key, value) - if err != nil { - klog.Errorf("failed to list ovn-ic lr policy: %v", err) - return - } - - if len(lrRouteList) == 0 { - klog.V(5).Info(" lr ovn-ic route does not exist") - for _, lrPolicy := range lrPolicyList { - if err := c.ovnClient.DeleteRouterPolicy(lr, lrPolicy.UUID); err != nil { - klog.Errorf("deleting router policy failed %v", err) - } - } - return - } - - policyMap := map[string]string{} - - for _, lrPolicy := range lrPolicyList { - match, err := stripPrefix(lrPolicy.Match) - if err != nil { - klog.Errorf("policy match abnormal: %v", err) - continue - } - policyMap[match] = lrPolicy.UUID - } - networks := strset.NewWithSize(len(lrRouteList)) - for _, lrRoute := range lrRouteList { - networks.Add(lrRoute.IPPrefix) - } - - networks.Each(func(prefix string) bool { - if _, ok := policyMap[prefix]; ok { - delete(policyMap, prefix) - return true - } - match := util.MatchV4Dst + " == " + prefix - if err := c.ovnClient.AddRouterPolicy(lr, match, ovnnb.LogicalRouterPolicyActionAllow, - nil, map[string]string{key: value, "vendor": util.CniTypeName}, - util.OvnICPolicyPriority); err != nil { - klog.Errorf("failed to add router policy: %v", err) - } - - return true - }) - for _, uuid := range policyMap { - if err := c.ovnClient.DeleteRouterPolicy(lr, uuid); err != nil { - klog.Errorf("deleting router policy failed %v", err) - } - } -} - -func moveElements(arr []string, order int) []string { - if order >= len(arr) { - order = order % len(arr) - } - - return append(arr[order:], arr[:order]...) -} From a6fee934103846b734dacb11af273a6c9cc3ca2a Mon Sep 17 00:00:00 2001 From: Changlu Yi Date: Mon, 15 Jan 2024 13:53:03 +0800 Subject: [PATCH 12/13] enable ic set default to false, and when upgrade need to check whether cluster enable ic. Signed-off-by: Changlu Yi --- Makefile | 6 ++++-- dist/images/install.sh | 2 +- kubeovn-helm/values.yaml | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index ba042ff6b67..61baa8b83d6 100644 --- a/Makefile +++ b/Makefile @@ -312,7 +312,8 @@ kind-install-chart: kind-load-image kind-untaint-control-plane helm install kubeovn ./kubeovn-helm \ --set global.images.kubeovn.tag=$(VERSION) \ --set replicaCount=$$(echo $$ips | awk -F ',' '{print NF}') \ - --set MASTER_NODES="$$(echo $$ips | sed 's/,/\\,/g')" + --set MASTER_NODES="$$(echo $$ips | sed 's/,/\\,/g')" \ + --set func.ENABLE_IC=$$(kubectl get node --show-labels | grep -q "ovn.kubernetes.io/ic-gw" && echo true || echo false) sleep 60 kubectl -n kube-system rollout status --timeout=1s deployment/ovn-central kubectl -n kube-system rollout status --timeout=1s deployment/kube-ovn-controller @@ -325,7 +326,8 @@ kind-upgrade-chart: kind-load-image helm upgrade kubeovn ./kubeovn-helm \ --set global.images.kubeovn.tag=$(VERSION) \ --set replicaCount=$$(echo $(OVN_DB_IPS) | awk -F ',' '{print NF}') \ - --set MASTER_NODES='$(OVN_DB_IPS)' + --set MASTER_NODES='$(OVN_DB_IPS)' \ + --set func.ENABLE_IC=$$(kubectl get node --show-labels | grep -q "ovn.kubernetes.io/ic-gw" && echo true || echo false) sleep 90 kubectl -n kube-system rollout status --timeout=1s deployment/ovn-central kubectl -n kube-system rollout status --timeout=1s deployment/kube-ovn-controller diff --git a/dist/images/install.sh b/dist/images/install.sh index 357c6a33f3f..60d13c1aafa 100644 --- a/dist/images/install.sh +++ b/dist/images/install.sh @@ -19,7 +19,7 @@ ENABLE_EXTERNAL_VPC=${ENABLE_EXTERNAL_VPC:-true} CNI_CONFIG_PRIORITY=${CNI_CONFIG_PRIORITY:-01} ENABLE_LB_SVC=${ENABLE_LB_SVC:-false} ENABLE_KEEP_VM_IP=${ENABLE_KEEP_VM_IP:-true} -ENABLE_IC=${ENABLE_IC:-true} +ENABLE_IC=$(kubectl get node --show-labels | grep -q "ovn.kubernetes.io/ic-gw" && echo true || echo false) # exchange link names of OVS bridge and the provider nic # in the default provider-network diff --git a/kubeovn-helm/values.yaml b/kubeovn-helm/values.yaml index 5aaca31cce2..45945685ef2 100644 --- a/kubeovn-helm/values.yaml +++ b/kubeovn-helm/values.yaml @@ -52,7 +52,7 @@ func: CHECK_GATEWAY: true LOGICAL_GATEWAY: false ENABLE_BIND_LOCAL_IP: true - ENABLE_IC: true + ENABLE_IC: false ipv4: POD_CIDR: "10.16.0.0/16" From c7174044f6eb0a35709d3247c8375210079f0aae Mon Sep 17 00:00:00 2001 From: Changlu Yi Date: Mon, 15 Jan 2024 14:49:38 +0800 Subject: [PATCH 13/13] fix Signed-off-by: Changlu Yi --- dist/images/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dist/images/install.sh b/dist/images/install.sh index 60d13c1aafa..63eac07026b 100644 --- a/dist/images/install.sh +++ b/dist/images/install.sh @@ -19,7 +19,7 @@ ENABLE_EXTERNAL_VPC=${ENABLE_EXTERNAL_VPC:-true} CNI_CONFIG_PRIORITY=${CNI_CONFIG_PRIORITY:-01} ENABLE_LB_SVC=${ENABLE_LB_SVC:-false} ENABLE_KEEP_VM_IP=${ENABLE_KEEP_VM_IP:-true} -ENABLE_IC=$(kubectl get node --show-labels | grep -q "ovn.kubernetes.io/ic-gw" && echo true || echo false) +ENABLE_IC=${ENABLE_IC:-$(kubectl get node --show-labels | grep -q "ovn.kubernetes.io/ic-gw" && echo true || echo false)} # exchange link names of OVS bridge and the provider nic # in the default provider-network