diff --git a/.github/workflows/build-x86-image.yaml b/.github/workflows/build-x86-image.yaml index 727eb287abf..e4e46999d2e 100644 --- a/.github/workflows/build-x86-image.yaml +++ b/.github/workflows/build-x86-image.yaml @@ -23,7 +23,7 @@ concurrency: env: GOSEC_VERSION: '2.15.0' HELM_VERSION: v3.11.1 - SUBMARINER_VERSION: v0.13.4 + SUBMARINER_VERSION: '0.14.3' jobs: build-kube-ovn-base: @@ -1593,7 +1593,7 @@ jobs: - name: Install submariner subctl env: - VERSION: ${{ env.SUBMARINER_VERSION }} + VERSION: v${{ env.SUBMARINER_VERSION }} DESTDIR: /usr/local/bin run: curl -Ls https://get.submariner.io | bash diff --git a/.github/workflows/scheduled-e2e.yaml b/.github/workflows/scheduled-e2e.yaml index a1f1db57b89..c49f7970ff6 100644 --- a/.github/workflows/scheduled-e2e.yaml +++ b/.github/workflows/scheduled-e2e.yaml @@ -11,7 +11,7 @@ concurrency: env: HELM_VERSION: v3.11.1 - SUBMARINER_VERSION: v0.13.4 + SUBMARINER_VERSION: '0.14.3' jobs: k8s-conformance-e2e: @@ -1249,7 +1249,7 @@ jobs: - name: Install submariner subctl env: - VERSION: ${{ env.SUBMARINER_VERSION }} + VERSION: v${{ env.SUBMARINER_VERSION }} DESTDIR: /usr/local/bin run: curl -Ls https://get.submariner.io | bash diff --git a/.gitignore b/.gitignore index 00cfb1f8674..7e6ad2d7242 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,9 @@ ovn.yaml ovn-ic-0.yaml ovn-ic-1.yaml kind.yaml +broker-info.subm +broker-info.subm.* +broker-info-internal.subm kube-ovn.tar vpc-nat-gateway.tar image-amd64.tar diff --git a/Makefile b/Makefile index 96734f2f0e2..5d0b66dca69 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,13 @@ COMMIT = git-$(shell git rev-parse --short HEAD) DATE = $(shell date +"%Y-%m-%d_%H:%M:%S") GOLDFLAGS = "-w -s -extldflags '-z now' -X github.com/kubeovn/kube-ovn/versions.COMMIT=$(COMMIT) -X github.com/kubeovn/kube-ovn/versions.VERSION=$(RELEASE_TAG) -X github.com/kubeovn/kube-ovn/versions.BUILDDATE=$(DATE)" +OS_LINUX = 0 +ifneq ($(OS),Windows_NT) +ifeq ($(shell uname -s),Linux) +OS_LINUX = 1 +endif +endif + CONTROL_PLANE_TAINTS = node-role.kubernetes.io/master node-role.kubernetes.io/control-plane CHART_UPGRADE_RESTART_OVS=$(shell echo $${CHART_UPGRADE_RESTART_OVS:-false}) @@ -37,6 +44,14 @@ CERT_MANAGER_CAINJECTOR = quay.io/jetstack/cert-manager-cainjector:$(CERT_MANAGE CERT_MANAGER_WEBHOOK = quay.io/jetstack/cert-manager-webhook:$(CERT_MANAGER_VERSION) CERT_MANAGER_YAML = https://github.com/cert-manager/cert-manager/releases/download/$(CERT_MANAGER_VERSION)/cert-manager.yaml +SUBMARINER_VERSION = $(shell echo $${SUBMARINER_VERSION:-0.14.3}) +SUBMARINER_OPERATOR = quay.io/submariner/submariner-operator:$(SUBMARINER_VERSION) +SUBMARINER_GATEWAY = quay.io/submariner/submariner-gateway:$(SUBMARINER_VERSION) +SUBMARINER_LIGHTHOUSE_AGENT = quay.io/submariner/lighthouse-agent:$(SUBMARINER_VERSION) +SUBMARINER_LIGHTHOUSE_COREDNS = quay.io/submariner/lighthouse-coredns:$(SUBMARINER_VERSION) +SUBMARINER_ROUTE_AGENT = quay.io/submariner/submariner-route-agent:$(SUBMARINER_VERSION) +SUBMARINER_NETTEST = quay.io/submariner/nettest:$(SUBMARINER_VERSION) + VPC_NAT_GW_IMG = $(REGISTRY)/vpc-nat-gateway:$(VERSION) E2E_NETWORK = bridge @@ -239,6 +254,52 @@ define kind_load_image kind load docker-image --name $(1) $(2) endef +define kind_load_submariner_images + $(call kind_load_image,$(1),$(SUBMARINER_OPERATOR),1) + $(call kind_load_image,$(1),$(SUBMARINER_GATEWAY),1) + $(call kind_load_image,$(1),$(SUBMARINER_LIGHTHOUSE_AGENT),1) + $(call kind_load_image,$(1),$(SUBMARINER_LIGHTHOUSE_COREDNS),1) + $(call kind_load_image,$(1),$(SUBMARINER_ROUTE_AGENT),1) + $(call kind_load_image,$(1),$(SUBMARINER_NETTEST),1) +endef + +define kubectl_wait_exist_and_ready + @echo "Waiting for $(2) $(1)/$(3) to exist..." + @n=0; while ! kubectl -n $(1) get $(2) -o name | awk -F / '{print $$2}' | grep -q ^$(3)$$; do \ + test $$n -eq 60 && exit 1; \ + sleep 1; \ + n=$$(($$n+1)); \ + done + kubectl -n $(1) rollout status --timeout=60s $(2) $(3) +endef + +define kubectl_wait_submariner_ready + $(call kubectl_wait_exist_and_ready,submariner-operator,deployment,submariner-operator) + $(call kubectl_wait_exist_and_ready,submariner-operator,deployment,submariner-lighthouse-agent) + $(call kubectl_wait_exist_and_ready,submariner-operator,deployment,submariner-lighthouse-coredns) + $(call kubectl_wait_exist_and_ready,submariner-operator,daemonset,submariner-gateway) + $(call kubectl_wait_exist_and_ready,submariner-operator,daemonset,submariner-metrics-proxy) + $(call kubectl_wait_exist_and_ready,submariner-operator,daemonset,submariner-routeagent) +endef + +define subctl_join + @if [ $(OS_LINUX) -ne 1 ]; then \ + set -e; \ + docker exec $(1)-control-plane bash -c "command -v xz >/dev/null || (apt update && apt install -y xz-utils)"; \ + docker exec -e VERSION=v$(SUBMARINER_VERSION) -e DESTDIR=/usr/local/bin $(1)-control-plane bash -c "command -v subctl >/dev/null || curl -Ls https://get.submariner.io | bash"; \ + docker cp broker-info-internal.subm $(1)-control-plane:/broker-info-internal.subm; \ + fi + + kubectl config use-context kind-$(1) + kubectl label --overwrite node $(1)-worker submariner.io/gateway=true + @if [ $(OS_LINUX) -eq 1 ]; then \ + subctl join broker-info-internal.subm --clusterid $(2) --clustercidr $$(echo '$(3)' | tr ';' ',') --natt=false --cable-driver vxlan --health-check=false --context=kind-$(1); \ + else \ + docker exec $(1)-control-plane sh -c "subctl join /broker-info-internal.subm --clusterid $(2) --clustercidr $$(echo '$(3)' | tr ';' ',') --natt=false --cable-driver vxlan --health-check=false"; \ + fi + $(call kubectl_wait_submariner_ready) +endef + .PHONY: kind-generate-config kind-generate-config: j2 yamls/kind.yaml.j2 -o yamls/kind.yaml @@ -391,7 +452,10 @@ kind-install-ovn-ic: kind-install .PHONY: kind-install-ovn-submariner kind-install-ovn-submariner: kind-install + $(call kind_load_submariner_images,kube-ovn) + $(call kind_load_submariner_images,kube-ovn1) $(call kind_load_image,kube-ovn1,$(REGISTRY)/kube-ovn:$(VERSION)) + kubectl config use-context kind-kube-ovn1 @$(MAKE) kind-untaint-control-plane sed -e 's/10.16.0/10.18.0/g' \ @@ -401,28 +465,14 @@ kind-install-ovn-submariner: kind-install dist/images/install.sh | bash kubectl describe no - kubectl config use-context kind-kube-ovn - kubectl config set-cluster kind-kube-ovn --server=https://$(shell docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' kube-ovn-control-plane):6443 - - kubectl config use-context kind-kube-ovn1 - kubectl config set-cluster kind-kube-ovn1 --server=https://$(shell docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' kube-ovn1-control-plane):6443 - kubectl config use-context kind-kube-ovn subctl deploy-broker - kubectl label nodes kube-ovn-worker submariner.io/gateway=true - subctl join broker-info.subm --clusterid cluster0 --clustercidr 10.16.0.0/16 --natt=false --cable-driver vxlan --health-check=false --kubecontext=kind-kube-ovn - kubectl patch clusterrole submariner-operator --type merge --patch-file yamls/subopeRules.yaml - sleep 10 - kubectl -n submariner-operator delete pod --selector=name=submariner-operator - kubectl patch subnet ovn-default --type='merge' --patch '{"spec": {"gatewayNode": "kube-ovn-worker","gatewayType": "centralized"}}' + cat broker-info.subm | base64 -d | \ + jq '.brokerURL = "https://$(shell docker inspect --format='{{.NetworkSettings.Networks.kind.IPAddress}}' kube-ovn-control-plane):6443"' | \ + base64 > broker-info-internal.subm - kubectl config use-context kind-kube-ovn1 - kubectl label nodes kube-ovn1-worker submariner.io/gateway=true - subctl join broker-info.subm --clusterid cluster1 --clustercidr 10.18.0.0/16 --natt=false --cable-driver vxlan --health-check=false --kubecontext=kind-kube-ovn1 - kubectl patch clusterrole submariner-operator --type merge --patch-file yamls/subopeRules.yaml - sleep 10 - kubectl -n submariner-operator delete pod --selector=name=submariner-operator - kubectl patch subnet ovn-default --type='merge' --patch '{"spec": {"gatewayNode": "kube-ovn1-worker","gatewayType": "centralized"}}' + $(call subctl_join,kube-ovn,cluster0,100.64.0.0/16;10.16.0.0/16) + $(call subctl_join,kube-ovn1,cluster1,100.68.0.0/16;10.18.0.0/16) .PHONY: kind-install-underlay kind-install-underlay: kind-install-underlay-ipv4 diff --git a/Makefile.e2e b/Makefile.e2e index c03475a2316..cc61a3943ec 100644 --- a/Makefile.e2e +++ b/Makefile.e2e @@ -1,3 +1,5 @@ +KUBECONFIG = $(shell echo $${KUBECONFIG:-$(HOME)/.kube/config}) + E2E_BRANCH := $(shell echo $${E2E_BRANCH:-master}) E2E_IP_FAMILY := $(shell echo $${E2E_IP_FAMILY:-ipv4}) E2E_NETWORK_MODE := $(shell echo $${E2E_NETWORK_MODE:-overlay}) @@ -118,22 +120,9 @@ kube-ovn-ic-conformance-e2e: .PHONY: kube-ovn-submariner-conformance-e2e kube-ovn-submariner-conformance-e2e: - while ! kubectl wait pod --for=condition=Ready -l app=submariner-routeagent -n submariner-operator && \ - ! kubectl wait pod --for=condition=Ready -l app=submariner-gateway -n submariner-operator && \ - ! kubectl wait pod --for=condition=Ready -l app=submariner-lighthouse-agent -n submariner-operator && \ - ! kubectl wait pod --for=condition=Ready -l app=submariner-lighthouse-coredns -n submariner-operator ; do \ - sleep 3; \ - done - subctl verify --kubecontexts kind-kube-ovn,kind-kube-ovn1 --verbose --disruptive-tests --kubeconfig ~/.kube/config 2>&1 | { \ - output=$$(cat); \ - if echo "$$output" | grep -q "1 Failed"; then \ - echo "$$output"; \ - exit 0; \ - else \ - echo "$$output"; \ - exit 1; \ - fi \ - } + KUBECONFIG=$(KUBECONFIG) subctl verify \ + --context kind-kube-ovn --tocontext kind-kube-ovn1 \ + --verbose --disruptive-tests .PHONY: kube-ovn-lb-svc-conformance-e2e kube-ovn-lb-svc-conformance-e2e: diff --git a/yamls/subopeRules.yaml b/yamls/subopeRules.yaml deleted file mode 100644 index c9dcb1a56a7..00000000000 --- a/yamls/subopeRules.yaml +++ /dev/null @@ -1,77 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: submariner-operator -rules: - - apiGroups: - - "" - resources: - - configmaps - verbs: - - create - - get - - list - - watch - - update - - apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - get - - list - - create - - update - - delete - - watch - - apiGroups: - - "" - resources: - - pods - - services - - nodes - verbs: - - get - - list - - watch - - apiGroups: - - operator.openshift.io - resources: - - dnses - verbs: - - get - - list - - watch - - update - - apiGroups: - - config.openshift.io - resources: - - networks - verbs: - - get - - list - - apiGroups: - - "" - resources: - - namespaces - verbs: - - get - - list - - watch - - apiGroups: - - monitoring.coreos.com - resources: - - servicemonitors - verbs: - - get - - create - - apiGroups: - - "apps" - resources: - - daemonsets - verbs: - - create - - get - - list - - watch - - update