Ovn ic ecmp enhance 1.11

Makefile

@@ -252,8 +252,9 @@ kind-init-ipv4: kind-clean
  @$(MAKE) kind-create
 
 .PHONY: kind-init-ovn-ic
-kind-init-ovn-ic: kind-clean-ovn-ic kind-init
- @$(MAKE) kind-generate-config
+kind-init-ovn-ic: kind-clean-ovn-ic
+ @ha=true $(MAKE) kind-init
+ @ovn_ic=true $(MAKE) kind-generate-config
  $(call kind_create_cluster,yamls/kind.yaml,kube-ovn1)
 
 .PHONY: kind-init-iptables
@@ -311,7 +312,8 @@ kind-install-chart: kind-load-image kind-untaint-control-plane
  helm install kubeovn ./kubeovn-helm \
  --set global.images.kubeovn.tag=$(VERSION) \
  --set replicaCount=$$(echo $$ips | awk -F ',' '{print NF}') \
- --set MASTER_NODES="$$(echo $$ips | sed 's/,/\\,/g')"
+ --set MASTER_NODES="$$(echo $$ips | sed 's/,/\\,/g')" \
+ --set func.ENABLE_IC=$$(kubectl get node --show-labels | grep -q "ovn.kubernetes.io/ic-gw" && echo true || echo false)
  sleep 60
  kubectl -n kube-system rollout status --timeout=1s deployment/ovn-central
  kubectl -n kube-system rollout status --timeout=1s deployment/kube-ovn-controller
@@ -324,7 +326,8 @@ kind-upgrade-chart: kind-load-image
  helm upgrade kubeovn ./kubeovn-helm \
  --set global.images.kubeovn.tag=$(VERSION) \
  --set replicaCount=$$(echo $(OVN_DB_IPS) | awk -F ',' '{print NF}') \
- --set MASTER_NODES='$(OVN_DB_IPS)'
+ --set MASTER_NODES='$(OVN_DB_IPS)' \
+ --set func.ENABLE_IC=$$(kubectl get node --show-labels | grep -q "ovn.kubernetes.io/ic-gw" && echo true || echo false)
  sleep 90
  kubectl -n kube-system rollout status --timeout=1s deployment/ovn-central
  kubectl -n kube-system rollout status --timeout=1s deployment/kube-ovn-controller
@@ -357,27 +360,29 @@ kind-install-ipv4: kind-install-overlay-ipv4
 kind-install-overlay-ipv4: kind-install
 
 .PHONY: kind-install-ovn-ic
-kind-install-ovn-ic: kind-install
+kind-install-ovn-ic:
+ @ENABLE_IC=true $(MAKE) kind-install
  $(call kind_load_image,kube-ovn1,$(REGISTRY)/kube-ovn:$(VERSION))
  kubectl config use-context kind-kube-ovn1
+ $(MAKE) kind-untaint-control-plane
  sed -e 's/10.16.0/10.18.0/g' \
  -e 's/10.96.0/10.98.0/g' \
  -e 's/100.64.0/100.68.0/g' \
  -e 's/VERSION=.*/VERSION=$(VERSION)/' \
- dist/images/install.sh | bash
+ dist/images/install.sh | ENABLE_IC=true bash
  kubectl describe no
 
- docker run -d --name ovn-ic-db --network kind $(REGISTRY)/kube-ovn:$(VERSION) bash start-ic-db.sh
- @set -e; \
- ic_db_host=$$(docker inspect ovn-ic-db -f "{{.NetworkSettings.Networks.kind.IPAddress}}"); \
- zone=az0 ic_db_host=$$ic_db_host gateway_node_name=kube-ovn-worker j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-0.yaml; \
- zone=az1 ic_db_host=$$ic_db_host gateway_node_name=kube-ovn1-worker j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-1.yaml
  kubectl config use-context kind-kube-ovn
+ sed 's/VERSION=.*/VERSION=$(VERSION)/' dist/images/install-ic-server.sh | bash
+
+ @set -e; \
+ ic_db_host=$$(kubectl get deployment ovn-ic-server -n kube-system -o jsonpath='{range .spec.template.spec.containers[0].env[?(@.name=="NODE_IPS")]}{.value}{end}'); \
+ ic_db_host=$${ic_db_host%?}; \
+ zone=az0 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn-worker,kube-ovn-worker2,kube-ovn-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-0.yaml; \
+ zone=az1 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn1-worker,kube-ovn1-worker2,kube-ovn1-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-1.yaml
  kubectl apply -f ovn-ic-0.yaml
  kubectl config use-context kind-kube-ovn1
  kubectl apply -f ovn-ic-1.yaml
- sleep 6
- docker exec ovn-ic-db ovn-ic-sbctl show
 
 .PHONY: kind-install-underlay
 kind-install-underlay: kind-install-underlay-ipv4

cmd/cmdmain.go

@@ -8,6 +8,7 @@ import (
  "github.com/kubeovn/kube-ovn/cmd/controller"
  "github.com/kubeovn/kube-ovn/cmd/controller_health_check"
  "github.com/kubeovn/kube-ovn/cmd/daemon"
+ "github.com/kubeovn/kube-ovn/cmd/ovn_ic_controller"
  "github.com/kubeovn/kube-ovn/cmd/ovn_leader_checker"
  "github.com/kubeovn/kube-ovn/cmd/ovn_monitor"
  "github.com/kubeovn/kube-ovn/cmd/pinger"
@@ -24,6 +25,7 @@ const (
  CmdSpeaker = "kube-ovn-speaker"
  CmdControllerHealthCheck = "kube-ovn-controller-healthcheck"
  CmdOvnLeaderChecker = "kube-ovn-leader-checker"
+ CmdOvnICController = "kube-ovn-ic-controller"
 )
 
 func main() {
@@ -45,6 +47,8 @@ func main() {
  controller_health_check.CmdMain()
  case CmdOvnLeaderChecker:
  ovn_leader_checker.CmdMain()
+ case CmdOvnICController:
+ ovn_ic_controller.CmdMain()
  default:
  util.LogFatalAndExit(nil, "%s is an unknown command", cmd)
  }

cmd/ovn_ic_controller/ovn_ic_controller.go

@@ -0,0 +1,24 @@
+package ovn_ic_controller
+
+import (
+ "k8s.io/klog/v2"
+ "k8s.io/sample-controller/pkg/signals"
+
+ "github.com/kubeovn/kube-ovn/pkg/ovn_ic_controller"
+ "github.com/kubeovn/kube-ovn/pkg/util"
+ "github.com/kubeovn/kube-ovn/versions"
+)
+
+func CmdMain() {
+ defer klog.Flush()
+
+ klog.Infof(versions.String())
+ config, err := ovn_ic_controller.ParseFlags()
+ if err != nil {
+ util.LogFatalAndExit(err, "failed to parse config")
+ }
+
+ stopCh := signals.SetupSignalHandler()
+ ctl := ovn_ic_controller.NewController(config)
+ ctl.Run(stopCh)
+}

dist/images/Dockerfile

@@ -28,4 +28,5 @@ RUN ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn && \
  ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-pinger && \
  ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-speaker && \
  ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-controller-healthcheck && \
- ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-leader-checker
+ ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-leader-checker && \
+ ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-ic-controller

dist/images/clean-ic-az-db.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+if [ "$#" -ne 2 ]; then
+ echo "use method $0 {az|node} {azName|nodeName}"
+ echo " eg: ./clean-ic-az-db.sh az az1"
+ echo " eg: ./clean-ic-az-db.sh node kube-ovn-worker; it will delete all resource of az that the node belong to"
+ exit 1
+fi
+
+filter_type=$1
+filter_value=$2
+availability_zone_uuid=
+
+if [ "$filter_type" != "az" ] && [ "$filter_type" != "node" ]; then
+ echo "filter_type should be az or node."
+ exit 1
+fi
+
+if [ "$filter_type" == "az" ]; then
+ availability_zone_uuid=$(ovn-ic-sbctl --columns=_uuid find availability_zone name=$filter_value | awk '{print $3}')
+fi
+
+echo $availability_zone_uuid
+
+if [ "$filter_type" == "node" ]; then
+ availability_zone_uuid=$(ovn-ic-sbctl --columns=availability_zone find gateway hostname=$filter_value | awk '{print $3}')
+fi
+
+if ! ovn-ic-sbctl get availability_zone $availability_zone_uuid name >/dev/null 2>&1; then
+ echo "Availability zone $availability_zone_uuid not found."
+ exit 1
+fi
+
+resource_types=("Gateway" "Route" "Port_Binding")
+
+for resource_type in "${resource_types[@]}"; do
+ uuid_array=($(ovn-ic-sbctl --columns=_uuid find $resource_type availability_zone=$availability_zone_uuid | awk '{print $3}'))
+
+ for uuid in "${uuid_array[@]}"; do
+ ovn-ic-sbctl destroy $resource_type $uuid
+ echo "Destroyed $resource_type: $uuid"
+ done
+done
+
+ovn-ic-sbctl destroy availability_zone $availability_zone_uuid
+echo "Destroyed availability_zone: $availability_zone_uuid"
+

dist/images/cleanup.sh

@@ -82,6 +82,8 @@ kubectl delete --ignore-not-found cm ovn-config ovn-ic-config ovn-external-gw-co
 kubectl delete --ignore-not-found svc kube-ovn-pinger kube-ovn-controller kube-ovn-cni kube-ovn-monitor -n kube-system
 kubectl delete --ignore-not-found ds kube-ovn-cni -n kube-system
 kubectl delete --ignore-not-found deploy kube-ovn-controller -n kube-system
+kubectl delete --ignore-not-found deploy ovn-ic-controller -n kube-system
+kubectl delete --ignore-not-found deploy ovn-ic-server -n kube-system
 
 # ensure kube-ovn-cni has been deleted
 while :; do

dist/images/install-ic-server.sh

@@ -0,0 +1,148 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+REGISTRY="kubeovn"
+VERSION="v1.11.14"
+TS_NUM=${TS_NUM:-3}
+IMAGE_PULL_POLICY="IfNotPresent"
+addresses=$(kubectl get no -lkube-ovn/role=master --no-headers -o wide | awk '{print $6}' | tr \\n ',')
+count=$(kubectl get no -lkube-ovn/role=master --no-headers | wc -l)
+OVN_LEADER_PROBE_INTERVAL=${OVN_LEADER_PROBE_INTERVAL:-5}
+
+cat <<EOF > ovn-ic-server.yaml
+---
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+ name: ovn-ic-server
+ namespace: kube-system
+ annotations:
+ kubernetes.io/description: |
+ OVN IC Server
+spec:
+ replicas: $count
+ strategy:
+ rollingUpdate:
+ maxSurge: 0
+ maxUnavailable: 1
+ type: RollingUpdate
+ selector:
+ matchLabels:
+ app: ovn-ic-server
+ template:
+ metadata:
+ labels:
+ app: ovn-ic-server
+ component: network
+ type: infra
+ spec:
+ tolerations:
+ - effect: NoSchedule
+ operator: Exists
+ - effect: NoExecute
+ operator: Exists
+ - key: CriticalAddonsOnly
+ operator: Exists
+ affinity:
+ podAntiAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ - labelSelector:
+ matchLabels:
+ app: ovn-ic-server
+ topologyKey: kubernetes.io/hostname
+ priorityClassName: system-cluster-critical
+ serviceAccountName: ovn
+ hostNetwork: true
+ containers:
+ - name: ovn-ic-server
+ image: "$REGISTRY/kube-ovn:$VERSION"
+ imagePullPolicy: $IMAGE_PULL_POLICY
+ command: ["/kube-ovn/start-ic-db.sh"]
+ securityContext:
+ capabilities:
+ add: ["SYS_NICE"]
+ env:
+ - name: ENABLE_SSL
+ value: "false"
+ - name: TS_NUM
+ value: "$TS_NUM"
+ - name: NODE_IPS
+ value: $addresses
+ - name: POD_IP
+ valueFrom:
+ fieldRef:
+ fieldPath: status.podIP
+ - name: OVN_LEADER_PROBE_INTERVAL
+ value: "$OVN_LEADER_PROBE_INTERVAL"
+ - name: POD_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.name
+ - name: POD_NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+ - name: POD_IPS
+ valueFrom:
+ fieldRef:
+ fieldPath: status.podIPs
+ resources:
+ requests:
+ cpu: 300m
+ memory: 200Mi
+ limits:
+ cpu: 3
+ memory: 1Gi
+ volumeMounts:
+ - mountPath: /var/run/ovn
+ name: host-run-ovn
+ - mountPath: /etc/ovn
+ name: host-config-ovn
+ - mountPath: /var/log/ovn
+ name: host-log-ovn
+ - mountPath: /etc/localtime
+ name: localtime
+ - mountPath: /var/run/tls
+ name: kube-ovn-tls
+ readinessProbe:
+ exec:
+ command:
+ - bash
+ - /kube-ovn/ovn-ic-healthcheck.sh
+ periodSeconds: 15
+ timeoutSeconds: 45
+ livenessProbe:
+ exec:
+ command:
+ - bash
+ - /kube-ovn/ovn-ic-healthcheck.sh
+ initialDelaySeconds: 30
+ periodSeconds: 15
+ failureThreshold: 5
+ timeoutSeconds: 4
+ nodeSelector:
+ kubernetes.io/os: "linux"
+ kube-ovn/role: "master"
+ volumes:
+ - name: host-run-ovn
+ hostPath:
+ path: /run/ovn
+ - name: host-config-ovn
+ hostPath:
+ path: /etc/origin/ovn
+ - name: host-log-ovn
+ hostPath:
+ path: /var/log/ovn
+ - name: localtime
+ hostPath:
+ path: /etc/localtime
+ - name: kube-ovn-tls
+ secret:
+ optional: true
+ secretName: kube-ovn-tls
+EOF
+
+kubectl apply -f ovn-ic-server.yaml
+kubectl rollout status deployment/ovn-ic-server -n kube-system --timeout 600s
+
+echo "OVN IC Server installed Successfully"