Merge pull request #1335 from jsturtevant/fix-packet-fragmentation

Fix issue for Windows Flannel deployments when packet size is over 1400 MTU
kubernetes-sigs · Apr 30, 2021 · 2203c18 · 2203c18
2 parents 5fbcdd8 + c626e1d
commit 2203c18
Show file tree

Hide file tree

Showing 8 changed files with 208 additions and 5 deletions.
diff --git a/docs/book/src/topics/windows.md b/docs/book/src/topics/windows.md
@@ -99,13 +99,13 @@ ssh -t -i .sshkey -o 'ProxyCommand ssh -i .sshkey -W %h:%p capi@<api-server-ip>'
 
 > There is also a [CAPZ kubectl plugin](https://github.com/kubernetes-sigs/cluster-api-provider-azure/blob/master/hack/debugging/Readme.md) that automates the ssh connection using the Management cluster
 
-To RDP:
+To RDP you can proxy through the api server:
 
 ```
 ssh -L 5555:10.1.0.4:3389 capi@20.69.66.232
 ```
 
-And then open an RDP client to `localhost:5555`
+And then open an RDP client on your local machine to `localhost:5555`
 
 ### Image creation
 The images are built using [image-builder](https://github.com/kubernetes-sigs/image-builder) and published the the Azure Market place. They use [Cloudbase-init](https://cloudbase-init.readthedocs.io/en/latest/) to bootstrap the machines via Kubeadm.  
@@ -125,9 +125,13 @@ If you would like customize your images please refer to the documentation on bui
 
 ### Kube-proxy and CNIs
 
-Kube-proxy and Windows CNIs are deployed via Cluster Resource Sets.  Windows doesn't not have a kube-proxy image due 
+Kube-proxy and Windows CNIs are deployed via Cluster Resource Sets.  Windows does not have a kube-proxy image due 
 to not having Privileged containers which would provide access to the host.  The current solution is using wins.exe as 
 demonstrated in the [Kubeadm support for Windows](https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/adding-windows-nodes/).    
 
-Windows Privileged Container support is in [KEP](https://github.com/kubernetes/enhancements/pull/2037) form with plans to 
-implement in 1.21.  Kube-proxy and other CNI will then be replaced with the Privileged containers. 
+Windows HostProcess Container support is in [KEP](https://github.com/kubernetes/enhancements/pull/2037) form with plans to 
+[implement in 1.22](https://github.com/kubernetes/kubernetes/pull/99576).  Kube-proxy and other CNI's will then be replaced with the HostProcess containers.
+
+Flannel is being used as the default CNI.  An important note for Flannel vxlan deployments is that the MTU for the linux nodes must be set to 1400.  
+This is because [Azure's VNET MTU is 1400](https://docs.microsoft.com/en-us/azure/virtual-network/virtual-network-tcpip-performance-tuning#azure-and-vm-mtu) which can cause fragmentation on packets sent from the Linux node to Windows node resulting in dropped packets. 
+To mitigate this we set the Linux eth0 port match 1400 and Flannel will automatically pick this up and [subtract 50](https://github.com/flannel-io/flannel/issues/1011) for the flannel network created.
diff --git a/templates/cluster-template-machinepool-windows.yaml b/templates/cluster-template-machinepool-windows.yaml
@@ -87,6 +87,18 @@ spec:
         overwrite: false
         tableType: gpt
     files:
+    - content: |
+        network:
+          version: 2
+          ethernets:
+            eth0:
+              mtu: 1400
+              match:
+                macaddress: MACADDRESS
+              set-name: eth0
+      owner: root:root
+      path: /etc/netplan/60-eth0.yaml
+      permissions: "0644"
     - contentFrom:
         secret:
           key: control-plane-azure.json
@@ -109,6 +121,10 @@ spec:
     mounts:
     - - LABEL=etcd_disk
       - /var/lib/etcddisk
+    postKubeadmCommands:
+    - mac=$(ip -o link | grep eth0 | grep ether | awk '{ print $17 }')
+    - sed -i -e "s/MACADDRESS/$${mac}/g" /etc/netplan/60-eth0.yaml
+    - netplan apply
   replicas: ${CONTROL_PLANE_MACHINE_COUNT}
   version: ${KUBERNETES_VERSION}
 ---
@@ -182,12 +198,28 @@ spec:
     owner: root:root
     path: /etc/kubernetes/azure.json
     permissions: "0644"
+  - content: |
+      network:
+        version: 2
+        ethernets:
+          eth0:
+            mtu: 1400
+            match:
+              macaddress: MACADDRESS
+            set-name: eth0
+    owner: root:root
+    path: /etc/netplan/60-eth0.yaml
+    permissions: "0644"
   joinConfiguration:
     nodeRegistration:
       kubeletExtraArgs:
         cloud-config: /etc/kubernetes/azure.json
         cloud-provider: azure
       name: '{{ ds.meta_data["local_hostname"] }}'
+  postKubeadmCommands:
+  - mac=$(ip -o link | grep eth0 | grep ether | awk '{ print $17 }')
+  - sed -i -e "s/MACADDRESS/$${mac}/g" /etc/netplan/60-eth0.yaml
+  - netplan apply
   useExperimentalRetryJoin: true
 ---
 apiVersion: exp.cluster.x-k8s.io/v1alpha4

diff --git a/templates/cluster-template-windows.yaml b/templates/cluster-template-windows.yaml
@@ -87,6 +87,18 @@ spec:
         overwrite: false
         tableType: gpt
     files:
+    - content: |
+        network:
+          version: 2
+          ethernets:
+            eth0:
+              mtu: 1400
+              match:
+                macaddress: MACADDRESS
+              set-name: eth0
+      owner: root:root
+      path: /etc/netplan/60-eth0.yaml
+      permissions: "0644"
     - contentFrom:
         secret:
           key: control-plane-azure.json
@@ -109,6 +121,10 @@ spec:
     mounts:
     - - LABEL=etcd_disk
       - /var/lib/etcddisk
+    postKubeadmCommands:
+    - mac=$(ip -o link | grep eth0 | grep ether | awk '{ print $17 }')
+    - sed -i -e "s/MACADDRESS/$${mac}/g" /etc/netplan/60-eth0.yaml
+    - netplan apply
   replicas: ${CONTROL_PLANE_MACHINE_COUNT}
   version: ${KUBERNETES_VERSION}
 ---
@@ -186,12 +202,28 @@ spec:
         owner: root:root
         path: /etc/kubernetes/azure.json
         permissions: "0644"
+      - content: |
+          network:
+            version: 2
+            ethernets:
+              eth0:
+                mtu: 1400
+                match:
+                  macaddress: MACADDRESS
+                set-name: eth0
+        owner: root:root
+        path: /etc/netplan/60-eth0.yaml
+        permissions: "0644"
       joinConfiguration:
         nodeRegistration:
           kubeletExtraArgs:
             cloud-config: /etc/kubernetes/azure.json
             cloud-provider: azure
           name: '{{ ds.meta_data["local_hostname"] }}'
+      postKubeadmCommands:
+      - mac=$(ip -o link | grep eth0 | grep ether | awk '{ print $17 }')
+      - sed -i -e "s/MACADDRESS/$${mac}/g" /etc/netplan/60-eth0.yaml
+      - netplan apply
       useExperimentalRetryJoin: true
 ---
 apiVersion: cluster.x-k8s.io/v1alpha4

diff --git a/templates/flavors/base-windows/patches/kubeadm-control-plane.yaml b/templates/flavors/base-windows/patches/kubeadm-control-plane.yaml
@@ -4,8 +4,38 @@ metadata:
   name: "${CLUSTER_NAME}-control-plane"
 spec:
   kubeadmConfigSpec:
+    postKubeadmCommands:
+      # Azures vnet MTU is 1400. 
+      # When using Flannel VXLAN to avoid packet fragmentation 
+      # that results dropped packets on Windows we need to match.
+      # Flannel will automatically choose eth0 - 50
+      # a bug in netplan requires matching on macaddress
+      # https://bugs.launchpad.net/netplan/+bug/1807273
+      - mac=$(ip -o link | grep eth0 | grep ether | awk '{ print $17 }')
+      - sed -i -e "s/MACADDRESS/$${mac}/g" /etc/netplan/60-eth0.yaml
+      - netplan apply 
     clusterConfiguration:
       controllerManager:
         extraArgs:
           allocate-node-cidrs: "true"
           configure-cloud-routes: "false"
+    files:
+    - path: /etc/netplan/60-eth0.yaml
+      permissions: "0644"
+      owner: root:root
+      content: |
+        network:
+          version: 2
+          ethernets:
+            eth0:
+              mtu: 1400
+              match:
+                macaddress: MACADDRESS
+              set-name: eth0
+    - contentFrom:
+        secret:
+          name: ${CLUSTER_NAME}-control-plane-azure-json
+          key: control-plane-azure.json
+      owner: root:root
+      path: /etc/kubernetes/azure.json
+      permissions: "0644"
diff --git a/templates/flavors/machinepool-windows/machine-pool-deployment.yaml b/templates/flavors/machinepool-windows/machine-pool-deployment.yaml
@@ -40,6 +40,14 @@ kind: KubeadmConfig
 metadata:
   name: "${CLUSTER_NAME}-mp-0"
 spec:
+  postKubeadmCommands:
+    # Azures vnet MTU is 1400. 
+    # When using Flannel VXLAN to avoid packet fragmentation 
+    # that results dropped packets on Windows we need to match.
+    # Flannel will automatically choose eth0 - 50
+    - mac=$(ip -o link | grep eth0 | grep ether | awk '{ print $17 }')
+    - sed -i -e "s/MACADDRESS/$${mac}/g" /etc/netplan/60-eth0.yaml
+    - netplan apply 
   useExperimentalRetryJoin: true
   joinConfiguration:
     nodeRegistration:
@@ -55,3 +63,15 @@ spec:
     owner: root:root
     path: /etc/kubernetes/azure.json
     permissions: "0644"
+  - path: /etc/netplan/60-eth0.yaml
+    permissions: "0644"
+    owner: root:root
+    content: |
+      network:
+        version: 2
+        ethernets:
+          eth0:
+            mtu: 1400
+            match:
+              macaddress: MACADDRESS
+            set-name: eth0
diff --git a/templates/flavors/windows/machine-deployment.yaml b/templates/flavors/windows/machine-deployment.yaml
@@ -44,6 +44,14 @@ metadata:
 spec:
   template:
     spec:
+      postKubeadmCommands:
+        # Azures vnet MTU is 1400. 
+        # When using Flannel VXLAN to avoid packet fragmentation 
+        # that results dropped packets on Windows we need to match.
+        # Flannel will automatically choose eth0 - 50
+        - mac=$(ip -o link | grep eth0 | grep ether | awk '{ print $17 }')
+        - sed -i -e "s/MACADDRESS/$${mac}/g" /etc/netplan/60-eth0.yaml
+        - netplan apply 
       useExperimentalRetryJoin: true
       joinConfiguration:
         nodeRegistration:
@@ -59,3 +67,16 @@ spec:
         owner: root:root
         path: /etc/kubernetes/azure.json
         permissions: "0644"
+      - path: /etc/netplan/60-eth0.yaml
+        permissions: "0644"
+        owner: root:root
+        content: |
+          network:
+            version: 2
+            ethernets:
+              eth0:
+                mtu: 1400
+                match:
+                  macaddress: MACADDRESS
+                set-name: eth0
+      
diff --git a/templates/test/ci/cluster-template-prow-machine-pool-windows.yaml b/templates/test/ci/cluster-template-prow-machine-pool-windows.yaml
@@ -92,6 +92,18 @@ spec:
         overwrite: false
         tableType: gpt
     files:
+    - content: |
+        network:
+          version: 2
+          ethernets:
+            eth0:
+              mtu: 1400
+              match:
+                macaddress: MACADDRESS
+              set-name: eth0
+      owner: root:root
+      path: /etc/netplan/60-eth0.yaml
+      permissions: "0644"
     - contentFrom:
         secret:
           key: control-plane-azure.json
@@ -114,6 +126,10 @@ spec:
     mounts:
     - - LABEL=etcd_disk
       - /var/lib/etcddisk
+    postKubeadmCommands:
+    - mac=$(ip -o link | grep eth0 | grep ether | awk '{ print $17 }')
+    - sed -i -e "s/MACADDRESS/$${mac}/g" /etc/netplan/60-eth0.yaml
+    - netplan apply
   replicas: ${CONTROL_PLANE_MACHINE_COUNT}
   version: ${KUBERNETES_VERSION}
 ---
@@ -187,12 +203,28 @@ spec:
     owner: root:root
     path: /etc/kubernetes/azure.json
     permissions: "0644"
+  - content: |
+      network:
+        version: 2
+        ethernets:
+          eth0:
+            mtu: 1400
+            match:
+              macaddress: MACADDRESS
+            set-name: eth0
+    owner: root:root
+    path: /etc/netplan/60-eth0.yaml
+    permissions: "0644"
   joinConfiguration:
     nodeRegistration:
       kubeletExtraArgs:
         cloud-config: /etc/kubernetes/azure.json
         cloud-provider: azure
       name: '{{ ds.meta_data["local_hostname"] }}'
+  postKubeadmCommands:
+  - mac=$(ip -o link | grep eth0 | grep ether | awk '{ print $17 }')
+  - sed -i -e "s/MACADDRESS/$${mac}/g" /etc/netplan/60-eth0.yaml
+  - netplan apply
   useExperimentalRetryJoin: true
 ---
 apiVersion: exp.cluster.x-k8s.io/v1alpha4

diff --git a/templates/test/ci/cluster-template-prow-windows.yaml b/templates/test/ci/cluster-template-prow-windows.yaml
@@ -92,6 +92,18 @@ spec:
         overwrite: false
         tableType: gpt
     files:
+    - content: |
+        network:
+          version: 2
+          ethernets:
+            eth0:
+              mtu: 1400
+              match:
+                macaddress: MACADDRESS
+              set-name: eth0
+      owner: root:root
+      path: /etc/netplan/60-eth0.yaml
+      permissions: "0644"
     - contentFrom:
         secret:
           key: control-plane-azure.json
@@ -114,6 +126,10 @@ spec:
     mounts:
     - - LABEL=etcd_disk
       - /var/lib/etcddisk
+    postKubeadmCommands:
+    - mac=$(ip -o link | grep eth0 | grep ether | awk '{ print $17 }')
+    - sed -i -e "s/MACADDRESS/$${mac}/g" /etc/netplan/60-eth0.yaml
+    - netplan apply
   replicas: ${CONTROL_PLANE_MACHINE_COUNT}
   version: ${KUBERNETES_VERSION}
 ---
@@ -191,12 +207,28 @@ spec:
         owner: root:root
         path: /etc/kubernetes/azure.json
         permissions: "0644"
+      - content: |
+          network:
+            version: 2
+            ethernets:
+              eth0:
+                mtu: 1400
+                match:
+                  macaddress: MACADDRESS
+                set-name: eth0
+        owner: root:root
+        path: /etc/netplan/60-eth0.yaml
+        permissions: "0644"
       joinConfiguration:
         nodeRegistration:
           kubeletExtraArgs:
             cloud-config: /etc/kubernetes/azure.json
             cloud-provider: azure
           name: '{{ ds.meta_data["local_hostname"] }}'
+      postKubeadmCommands:
+      - mac=$(ip -o link | grep eth0 | grep ether | awk '{ print $17 }')
+      - sed -i -e "s/MACADDRESS/$${mac}/g" /etc/netplan/60-eth0.yaml
+      - netplan apply
       useExperimentalRetryJoin: true
 ---
 apiVersion: cluster.x-k8s.io/v1alpha4