Skip to content

Commit

Permalink
Prevent host net pods from getting kube-vip addr (vmware-tanzu#2103)
Browse files Browse the repository at this point in the history 
On IPv6 only clusters or dual stack clusters with IPv6 as the primary IP
family, configure `--node-ip` for kubelet on the control plane nodes to
the first detected IP address prior to launching kube-vip.

Otherwise, prior to cloud-provider-vsphere taking over, kubelet will use
the kube-vip address as the node IP and host network pods that start up
early will get this address set as their pod IP.

See: vmware-tanzu#2098

Co-authored-by: Christian Ang <angc@vmware.com>
  • Loading branch information
@mcwumbly @christianang
2 people authored and Chandra Pamuluri committed Apr 28, 2022
1 parent 7f36ef2 commit 5b68cc9
Show file tree
Hide file tree
Showing 2 changed files with 80 additions and 15 deletions.
28 changes: 16 additions & 12 deletions pkg/v1/providers/infrastructure-vsphere/v1.1.0/ytt/overlay.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -235,12 +235,6 @@ spec:
#@ end
bindPort: #@ data.values.CLUSTER_API_SERVER_PORT
#@ end
#@ if data.values.TKG_IP_FAMILY == "ipv6,ipv4":
nodeRegistration:
kubeletExtraArgs:
#@overlay/match missing_ok=True
node-ip: "::"
#@ end
joinConfiguration:
#@ if (not data.values.AVI_CONTROL_PLANE_HA_PROVIDER) and data.values.CLUSTER_API_SERVER_PORT:
#@overlay/match missing_ok=True
Expand All @@ -253,12 +247,6 @@ spec:
#@ end
bindPort: #@ data.values.CLUSTER_API_SERVER_PORT
#@ end
#@ if data.values.TKG_IP_FAMILY == "ipv6,ipv4":
nodeRegistration:
kubeletExtraArgs:
#@overlay/match missing_ok=True
node-ip: "::"
#@ end
clusterConfiguration:
imageRepository: #@ kubeadm_image_repo(bomDataForK8sVersion.kubeadmConfigSpec.imageRepository)
etcd:
Expand Down Expand Up @@ -302,6 +290,13 @@ spec:
#@overlay/remove
- content:
#@ end
#@ if data.values.TKG_IP_FAMILY in ["ipv6", "ipv6,ipv4"]:
#@overlay/append
- content: ""
owner: root:root
path: /etc/sysconfig/kubelet
permissions: "0640"
#@ end
users:
#@overlay/match by=overlay.index(0)
#@overlay/replace
Expand All @@ -316,6 +311,15 @@ spec:
#@overlay/append
- sed -i '/listen-client-urls/ s/$/,https:\/\/127.0.0.1:2379/' /etc/kubernetes/manifests/etcd.yaml
#@ end
#! When using kube-vip as the control plane endpoint provider with IPv6 as
#! the primary IP family, set --node-ip on kubelet so that host network pods
#! do not get the kube-vip address as their pod IP.
#! See: https://github.com/vmware-tanzu/tanzu-framework/issues/2098
#@ if not data.values.AVI_CONTROL_PLANE_HA_PROVIDER and data.values.TKG_IP_FAMILY in ["ipv6", "ipv6,ipv4"]:
preKubeadmCommands:
#@overlay/append
- echo "KUBELET_EXTRA_ARGS=--node-ip=$(ip -6 -json addr show dev eth0 scope global | jq -r .[0].addr_info[0].local)" >> /etc/sysconfig/kubelet
#@ end
replicas: #@ data.values.CONTROL_PLANE_MACHINE_COUNT
version: #@ data.values.KUBERNETES_VERSION

Expand Down
67 changes: 64 additions & 3 deletions pkg/v1/providers/tests/unit/ip_family_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -357,7 +357,19 @@ var _ = Describe("TKG_IP_FAMILY Ytt Templating", func() {
Expect(kubeadmControlPlaneDocs[0]).To(HaveYAMLPathWithValue("$.spec.kubeadmConfigSpec.joinConfiguration.controlPlane.localAPIEndpoint.advertiseAddress", "0.0.0.0"))
Expect(kubeadmControlPlaneDocs[0]).To(HaveYAMLPathWithValue("$.spec.kubeadmConfigSpec.joinConfiguration.controlPlane.localAPIEndpoint.bindPort", "443"))
})
It("does not configure node ip in KUBELET_EXTRA_ARGS in /etc/sysconfig/kubelet", func() {
output, err := ytt.RenderYTTTemplate(ytt.CommandOptions{}, paths, strings.NewReader(values))
Expect(err).NotTo(HaveOccurred())

kubeadmControlPlaneDocs, err := FindDocsMatchingYAMLPath(output, map[string]string{
"$.kind": "KubeadmControlPlane",
})
Expect(err).NotTo(HaveOccurred())

Expect(kubeadmControlPlaneDocs).To(HaveLen(1))
Expect(kubeadmControlPlaneDocs[0]).NotTo(HaveYAMLPath("$.spec.kubeadmConfigSpec.files[1]"))
Expect(kubeadmControlPlaneDocs[0]).NotTo(HaveYAMLPath("$.spec.kubeadmConfigSpec.preKubeadmCommands[5]"))
})
})

When("data values are set to single stack IPv6 settings", func() {
Expand Down Expand Up @@ -403,6 +415,22 @@ var _ = Describe("TKG_IP_FAMILY Ytt Templating", func() {
Expect(kubeadmControlPlaneDocs[0]).To(HaveYAMLPathWithValue("$.spec.kubeadmConfigSpec.joinConfiguration.controlPlane.localAPIEndpoint.advertiseAddress", "::/0"))
Expect(kubeadmControlPlaneDocs[0]).To(HaveYAMLPathWithValue("$.spec.kubeadmConfigSpec.joinConfiguration.controlPlane.localAPIEndpoint.bindPort", "443"))
})
It("configures node-ip on the control plane nodes by echoing the detected node ip into KUBELET_EXTRA_ARGS in /etc/sysconfig/kubelet", func() {
output, err := ytt.RenderYTTTemplate(ytt.CommandOptions{}, paths, strings.NewReader(values))
Expect(err).NotTo(HaveOccurred())

kubeadmControlPlaneDocs, err := FindDocsMatchingYAMLPath(output, map[string]string{
"$.kind": "KubeadmControlPlane",
})
Expect(err).NotTo(HaveOccurred())

Expect(kubeadmControlPlaneDocs).To(HaveLen(1))
Expect(kubeadmControlPlaneDocs[0]).To(HaveYAMLPathWithValue("$.spec.kubeadmConfigSpec.files[1].content", ""))
Expect(kubeadmControlPlaneDocs[0]).To(HaveYAMLPathWithValue("$.spec.kubeadmConfigSpec.files[1].owner", "root:root"))
Expect(kubeadmControlPlaneDocs[0]).To(HaveYAMLPathWithValue("$.spec.kubeadmConfigSpec.files[1].path", "/etc/sysconfig/kubelet"))
Expect(kubeadmControlPlaneDocs[0]).To(HaveYAMLPathWithValue("$.spec.kubeadmConfigSpec.files[1].permissions", "0640"))
Expect(kubeadmControlPlaneDocs[0]).To(HaveYAMLPathWithValue("$.spec.kubeadmConfigSpec.preKubeadmCommands[5]", "echo \"KUBELET_EXTRA_ARGS=--node-ip=$(ip -6 -json addr show dev eth0 scope global | jq -r .[0].addr_info[0].local)\" >> /etc/sysconfig/kubelet"))
})
})

When("data values are set to ipv4,ipv6 dual stack settings", func() {
Expand Down Expand Up @@ -495,6 +523,19 @@ var _ = Describe("TKG_IP_FAMILY Ytt Templating", func() {
Expect(kubeadmConfigTemplateDocs).To(HaveLen(1))
Expect(kubeadmConfigTemplateDocs[0]).NotTo(HaveYAMLPath("$.spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.node-ip"))
})
It("does not configure node ip in KUBELET_EXTRA_ARGS in /etc/sysconfig/kubelet", func() {
output, err := ytt.RenderYTTTemplate(ytt.CommandOptions{}, paths, strings.NewReader(values))
Expect(err).NotTo(HaveOccurred())

kubeadmControlPlaneDocs, err := FindDocsMatchingYAMLPath(output, map[string]string{
"$.kind": "KubeadmControlPlane",
})
Expect(err).NotTo(HaveOccurred())

Expect(kubeadmControlPlaneDocs).To(HaveLen(1))
Expect(kubeadmControlPlaneDocs[0]).NotTo(HaveYAMLPath("$.spec.kubeadmConfigSpec.files[1]"))
Expect(kubeadmControlPlaneDocs[0]).NotTo(HaveYAMLPath("$.spec.kubeadmConfigSpec.preKubeadmCommands[5]"))
})
})

When("data values are set to ipv6,ipv4 dual stack settings", func() {
Expand Down Expand Up @@ -567,7 +608,7 @@ var _ = Describe("TKG_IP_FAMILY Ytt Templating", func() {
Expect(kubeadmControlPlaneDocs).To(HaveLen(1))
Expect(kubeadmControlPlaneDocs[0]).To(HaveYAMLPathWithValue("$.spec.kubeadmConfigSpec.clusterConfiguration.apiServer.extraArgs.advertise-address", "2001:db8::2"))
})
It("renders control plane template to configure kubelet with '::' as the node IP", func() {
It("does not render node-ip field for control plane nodes", func() {
output, err := ytt.RenderYTTTemplate(ytt.CommandOptions{}, paths, strings.NewReader(values))
Expect(err).NotTo(HaveOccurred())

Expand All @@ -577,8 +618,12 @@ var _ = Describe("TKG_IP_FAMILY Ytt Templating", func() {
Expect(err).NotTo(HaveOccurred())

Expect(kubeadmControlPlaneDocs).To(HaveLen(1))
Expect(kubeadmControlPlaneDocs[0]).To(HaveYAMLPathWithValue("$.spec.kubeadmConfigSpec.initConfiguration.nodeRegistration.kubeletExtraArgs.node-ip", "::"))
Expect(kubeadmControlPlaneDocs[0]).To(HaveYAMLPathWithValue("$.spec.kubeadmConfigSpec.joinConfiguration.nodeRegistration.kubeletExtraArgs.node-ip", "::"))
Expect(kubeadmControlPlaneDocs[0]).NotTo(HaveYAMLPath("$.spec.kubeadmConfigSpec.initConfiguration.nodeRegistration.kubeletExtraArgs.node-ip"))
Expect(kubeadmControlPlaneDocs[0]).NotTo(HaveYAMLPath("$.spec.kubeadmConfigSpec.joinConfiguration.nodeRegistration.kubeletExtraArgs.node-ip"))
})
It("configures kubelet on the worker nodes with '::' as the node IP", func() {
output, err := ytt.RenderYTTTemplate(ytt.CommandOptions{}, paths, strings.NewReader(values))
Expect(err).NotTo(HaveOccurred())

kubeadmConfigTemplateDocs, err := FindDocsMatchingYAMLPath(output, map[string]string{
"$.kind": "KubeadmConfigTemplate",
Expand All @@ -588,6 +633,22 @@ var _ = Describe("TKG_IP_FAMILY Ytt Templating", func() {
Expect(kubeadmConfigTemplateDocs).To(HaveLen(1))
Expect(kubeadmConfigTemplateDocs[0]).To(HaveYAMLPathWithValue("$.spec.template.spec.joinConfiguration.nodeRegistration.kubeletExtraArgs.node-ip", "::"))
})
It("configures node-ip on the control plane nodes by echoing the detected node ip into KUBELET_EXTRA_ARGS in /etc/sysconfig/kubelet", func() {
output, err := ytt.RenderYTTTemplate(ytt.CommandOptions{}, paths, strings.NewReader(values))
Expect(err).NotTo(HaveOccurred())

kubeadmControlPlaneDocs, err := FindDocsMatchingYAMLPath(output, map[string]string{
"$.kind": "KubeadmControlPlane",
})
Expect(err).NotTo(HaveOccurred())

Expect(kubeadmControlPlaneDocs).To(HaveLen(1))
Expect(kubeadmControlPlaneDocs[0]).To(HaveYAMLPathWithValue("$.spec.kubeadmConfigSpec.files[1].content", ""))
Expect(kubeadmControlPlaneDocs[0]).To(HaveYAMLPathWithValue("$.spec.kubeadmConfigSpec.files[1].owner", "root:root"))
Expect(kubeadmControlPlaneDocs[0]).To(HaveYAMLPathWithValue("$.spec.kubeadmConfigSpec.files[1].path", "/etc/sysconfig/kubelet"))
Expect(kubeadmControlPlaneDocs[0]).To(HaveYAMLPathWithValue("$.spec.kubeadmConfigSpec.files[1].permissions", "0640"))
Expect(kubeadmControlPlaneDocs[0]).To(HaveYAMLPathWithValue("$.spec.kubeadmConfigSpec.preKubeadmCommands[5]", "echo \"KUBELET_EXTRA_ARGS=--node-ip=$(ip -6 -json addr show dev eth0 scope global | jq -r .[0].addr_info[0].local)\" >> /etc/sysconfig/kubelet"))
})
})
})

Expand Down

0 comments on commit 5b68cc9

Please sign in to comment.