From c0ef5f560658292e11bea3d30877f0027b820865 Mon Sep 17 00:00:00 2001 From: Varun Venkatesh Date: Mon, 18 Jan 2021 18:12:13 +0530 Subject: [PATCH 1/4] feat: allow creation of dualstack Windows clusters --- examples/dualstack/README.md | 6 +-- examples/dualstack/kubernetes-windows.json | 60 ++++++++++++++++++++++ pkg/api/vlabs/validate.go | 4 +- pkg/api/vlabs/validate_test.go | 5 +- 4 files changed, 67 insertions(+), 8 deletions(-) create mode 100644 examples/dualstack/kubernetes-windows.json diff --git a/examples/dualstack/README.md b/examples/dualstack/README.md index 2d98c7ca69..b530755d96 100644 --- a/examples/dualstack/README.md +++ b/examples/dualstack/README.md @@ -117,10 +117,8 @@ nginx-ipv6 LoadBalancer fd00::6283 2603:1030:805:3::3 80:31140/TCP - Dual stack clusters are supported only with kubenet and azurecni. - Dual stack cluster with azurecni are only supported with `bridge` network mode. -- Dual stack clusters are supported only with Linux. -- Dual stack clusters with Windows is not supported at this time because it requires - - Kubernetes version 1.19+ and - - [backport to 2004 to support dualstack containers](https://github.com/Azure/aks-engine/issues/3568). +- Dual stack clusters are supported on Windows from version 2004 (kernel version 10.0.19041.610) and Kubernetes version 1.19 + - https://kubernetes.io/docs/setup/production-environment/windows/intro-windows-in-kubernetes/#ipv4-ipv6-dual-stack - Dual stack clusters are supported with - ipvs kube-proxy mode (Kubernetes version 1.16+) - iptables kube-proxy mode (Kubernetes version 1.18+). diff --git a/examples/dualstack/kubernetes-windows.json b/examples/dualstack/kubernetes-windows.json new file mode 100644 index 0000000000..d8402ce9d0 --- /dev/null +++ b/examples/dualstack/kubernetes-windows.json @@ -0,0 +1,60 @@ +{ + "apiVersion": "vlabs", + "properties": { + "featureFlags": { + "enableIPv6DualStack": true + }, + "orchestratorProfile": { + "orchestratorRelease": "1.19", + "kubernetesConfig": { + "apiServerConfig": { + "--feature-gates": "IPv6DualStack=true" + }, + "kubeletConfig": { + "--feature-gates": "IPv6DualStack=true" + }, + "controllerManagerConfig": { + "--feature-gates": "IPv6DualStack=true" + }, + "kubeProxyMode": "ipvs", + "networkPlugin": "azure", + "networkMode": "bridge", + "networkPolicy": "", + "useManagedIdentity": false + } + }, + "masterProfile": { + "count": 1, + "dnsPrefix": "", + "vmSize": "Standard_D2_v3" + }, + "agentPoolProfiles": [ + { + "name": "windowspool2", + "count": 1, + "vmSize": "Standard_D2_v3", + "availabilityProfile": "VirtualMachineScaleSets", + "osType": "Windows", + "osDiskSizeGB": 128 + } + ], + "windowsProfile": { + "windowsPublisher": "MicrosoftWindowsServer", + "windowsOffer": "WindowsServer", + "windowsSku": "Datacenter-Core-2004-with-Containers-smalldisk", + "imageVersion": "latest", + "adminUsername": "azureuser", + "adminPassword": "replacepassword1234$" + }, + "linuxProfile": { + "adminUsername": "azureuser", + "ssh": { + "publicKeys": [ + { + "keyData": "" + } + ] + } + } + } +} diff --git a/pkg/api/vlabs/validate.go b/pkg/api/vlabs/validate.go index 232383c696..5595854f0f 100644 --- a/pkg/api/vlabs/validate.go +++ b/pkg/api/vlabs/validate.go @@ -469,8 +469,8 @@ func (a *Properties) validateAgentPoolProfiles(isUpdate bool) error { // validate os type is linux if dual stack feature is enabled if a.FeatureFlags.IsIPv6DualStackEnabled() || a.FeatureFlags.IsIPv6OnlyEnabled() { - if agentPoolProfile.OSType == Windows { - return errors.Errorf("Dual stack and single stack IPv6 feature is supported only with Linux, but agent pool '%s' is of os type %s", agentPoolProfile.Name, agentPoolProfile.OSType) + if agentPoolProfile.OSType == Windows && !common.IsKubernetesVersionGe(a.OrchestratorProfile.OrchestratorVersion, "1.19.0") { + return errors.Errorf("Dual stack and single stack IPv6 feature is supported on Windows only from Kubernetes version 1.19, but OrchestratorProfile.OrchestratorVersion is '%s'", a.OrchestratorProfile.OrchestratorVersion) } if agentPoolProfile.Distro == Flatcar { return errors.Errorf("Dual stack and single stack IPv6 feature is currently supported only with Ubuntu, but agent pool '%s' is of distro type %s", agentPoolProfile.Name, agentPoolProfile.Distro) diff --git a/pkg/api/vlabs/validate_test.go b/pkg/api/vlabs/validate_test.go index 8dafeb8fd6..ecfae1c919 100644 --- a/pkg/api/vlabs/validate_test.go +++ b/pkg/api/vlabs/validate_test.go @@ -4139,7 +4139,7 @@ func TestValidateProperties_OrchestratorSpecificProperties(t *testing.T) { } }) - t.Run("Should not support os type other than linux for single stack ipv6 and dual stack feature", func(t *testing.T) { + t.Run("Should not support os type other than linux along with kubernetes version less than 1.19 for single stack ipv6 and dual stack feature", func(t *testing.T) { t.Parallel() cs := getK8sDefaultContainerService(true) for _, featureFlags := range []FeatureFlags{{EnableIPv6DualStack: true}, {EnableIPv6Only: true}} { @@ -4149,7 +4149,8 @@ func TestValidateProperties_OrchestratorSpecificProperties(t *testing.T) { masterProfile.Distro = Ubuntu agentPoolProfiles := cs.Properties.AgentPoolProfiles agentPoolProfiles[0].OSType = Windows - expectedMsg := fmt.Sprintf("Dual stack and single stack IPv6 feature is supported only with Linux, but agent pool '%s' is of os type %s", agentPoolProfiles[0].Name, agentPoolProfiles[0].OSType) + cs.Properties.OrchestratorProfile.OrchestratorVersion = "1.17" + expectedMsg := fmt.Sprintf("Dual stack and single stack IPv6 feature is supported on Windows only from Kubernetes version 1.19, but OrchestratorProfile.OrchestratorVersion is '%s'", cs.Properties.OrchestratorProfile.OrchestratorVersion) if err := cs.Properties.validateAgentPoolProfiles(false); err.Error() != expectedMsg { t.Errorf("expected error with message : %s, but got %s", expectedMsg, err.Error()) } From fe1ae004ca01b82a5cdd23be93af5c438ff01d7e Mon Sep 17 00:00:00 2001 From: Varun Venkatesh <67044367+vavenk-ms@users.noreply.github.com> Date: Tue, 19 Jan 2021 23:20:06 +0530 Subject: [PATCH 2/4] Apply suggestions from code review Co-authored-by: James Sturtevant --- pkg/api/vlabs/validate_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/api/vlabs/validate_test.go b/pkg/api/vlabs/validate_test.go index ecfae1c919..be49d8aa8b 100644 --- a/pkg/api/vlabs/validate_test.go +++ b/pkg/api/vlabs/validate_test.go @@ -4139,7 +4139,7 @@ func TestValidateProperties_OrchestratorSpecificProperties(t *testing.T) { } }) - t.Run("Should not support os type other than linux along with kubernetes version less than 1.19 for single stack ipv6 and dual stack feature", func(t *testing.T) { + t.Run("Should not support os type other than linux for versions less than 1.19 for single stack ipv6 and dual stack feature", func(t *testing.T) { t.Parallel() cs := getK8sDefaultContainerService(true) for _, featureFlags := range []FeatureFlags{{EnableIPv6DualStack: true}, {EnableIPv6Only: true}} { @@ -4149,7 +4149,7 @@ func TestValidateProperties_OrchestratorSpecificProperties(t *testing.T) { masterProfile.Distro = Ubuntu agentPoolProfiles := cs.Properties.AgentPoolProfiles agentPoolProfiles[0].OSType = Windows - cs.Properties.OrchestratorProfile.OrchestratorVersion = "1.17" + cs.Properties.OrchestratorProfile.OrchestratorVersion = "1.18" expectedMsg := fmt.Sprintf("Dual stack and single stack IPv6 feature is supported on Windows only from Kubernetes version 1.19, but OrchestratorProfile.OrchestratorVersion is '%s'", cs.Properties.OrchestratorProfile.OrchestratorVersion) if err := cs.Properties.validateAgentPoolProfiles(false); err.Error() != expectedMsg { t.Errorf("expected error with message : %s, but got %s", expectedMsg, err.Error()) From d991f32033a4a72811cf95ad502014775e4ba45d Mon Sep 17 00:00:00 2001 From: Varun Venkatesh Date: Wed, 3 Feb 2021 21:00:14 +0530 Subject: [PATCH 3/4] Disable IPv6-only on Windows --- pkg/api/vlabs/validate.go | 9 +++++++-- pkg/api/vlabs/validate_test.go | 32 ++++++++++++++++++++------------ 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/pkg/api/vlabs/validate.go b/pkg/api/vlabs/validate.go index 5595854f0f..85062c882a 100644 --- a/pkg/api/vlabs/validate.go +++ b/pkg/api/vlabs/validate.go @@ -469,8 +469,13 @@ func (a *Properties) validateAgentPoolProfiles(isUpdate bool) error { // validate os type is linux if dual stack feature is enabled if a.FeatureFlags.IsIPv6DualStackEnabled() || a.FeatureFlags.IsIPv6OnlyEnabled() { - if agentPoolProfile.OSType == Windows && !common.IsKubernetesVersionGe(a.OrchestratorProfile.OrchestratorVersion, "1.19.0") { - return errors.Errorf("Dual stack and single stack IPv6 feature is supported on Windows only from Kubernetes version 1.19, but OrchestratorProfile.OrchestratorVersion is '%s'", a.OrchestratorProfile.OrchestratorVersion) + if agentPoolProfile.OSType == Windows { + if a.FeatureFlags.IsIPv6DualStackEnabled() && !common.IsKubernetesVersionGe(a.OrchestratorProfile.OrchestratorVersion, "1.19.0") { + return errors.Errorf("Dual stack IPv6 feature is supported on Windows only from Kubernetes version 1.19, but OrchestratorProfile.OrchestratorVersion is '%s'", a.OrchestratorProfile.OrchestratorVersion) + } + if a.FeatureFlags.IsIPv6OnlyEnabled() { + return errors.Errorf("Single stack IPv6 feature is supported only with Linux, but agent pool '%s' is of os type %s", agentPoolProfile.Name, agentPoolProfile.OSType) + } } if agentPoolProfile.Distro == Flatcar { return errors.Errorf("Dual stack and single stack IPv6 feature is currently supported only with Ubuntu, but agent pool '%s' is of distro type %s", agentPoolProfile.Name, agentPoolProfile.Distro) diff --git a/pkg/api/vlabs/validate_test.go b/pkg/api/vlabs/validate_test.go index 8c8cdcc507..4d2d7b836d 100644 --- a/pkg/api/vlabs/validate_test.go +++ b/pkg/api/vlabs/validate_test.go @@ -4139,21 +4139,30 @@ func TestValidateProperties_OrchestratorSpecificProperties(t *testing.T) { } }) - t.Run("Should not support os type other than linux for versions less than 1.19 for single stack ipv6 and dual stack feature", func(t *testing.T) { + t.Run("Should not support os type other than linux for single stack ipv6 and versions less than 1.19 for dual stack feature", func(t *testing.T) { t.Parallel() cs := getK8sDefaultContainerService(true) + + masterProfile := cs.Properties.MasterProfile + masterProfile.Distro = Ubuntu + agentPoolProfiles := cs.Properties.AgentPoolProfiles + + agentPoolProfiles[0].OSType = Windows + cs.Properties.FeatureFlags = &FeatureFlags{EnableIPv6DualStack: true} + cs.Properties.OrchestratorProfile.OrchestratorVersion = "1.18" + expectedMsg := fmt.Sprintf("Dual stack IPv6 feature is supported on Windows only from Kubernetes version 1.19, but OrchestratorProfile.OrchestratorVersion is '%s'", cs.Properties.OrchestratorProfile.OrchestratorVersion) + if err := cs.Properties.validateAgentPoolProfiles(false); err.Error() != expectedMsg { + t.Errorf("expected error with message : %s, but got %s", expectedMsg, err.Error()) + } + + cs.Properties.FeatureFlags = &FeatureFlags{EnableIPv6Only: true} + expectedMsg = fmt.Sprintf("Single stack IPv6 feature is supported only with Linux, but agent pool '%s' is of os type %s", agentPoolProfiles[0].Name, agentPoolProfiles[0].OSType) + if err := cs.Properties.validateAgentPoolProfiles(false); err.Error() != expectedMsg { + t.Errorf("expected error with message : %s, but got %s", expectedMsg, err.Error()) + } + for _, featureFlags := range []FeatureFlags{{EnableIPv6DualStack: true}, {EnableIPv6Only: true}} { cs.Properties.FeatureFlags = &featureFlags - masterProfile := cs.Properties.MasterProfile - - masterProfile.Distro = Ubuntu - agentPoolProfiles := cs.Properties.AgentPoolProfiles - agentPoolProfiles[0].OSType = Windows - cs.Properties.OrchestratorProfile.OrchestratorVersion = "1.18" - expectedMsg := fmt.Sprintf("Dual stack and single stack IPv6 feature is supported on Windows only from Kubernetes version 1.19, but OrchestratorProfile.OrchestratorVersion is '%s'", cs.Properties.OrchestratorProfile.OrchestratorVersion) - if err := cs.Properties.validateAgentPoolProfiles(false); err.Error() != expectedMsg { - t.Errorf("expected error with message : %s, but got %s", expectedMsg, err.Error()) - } agentPoolProfiles[0].OSType = Linux agentPoolProfiles[0].Distro = Flatcar @@ -4162,7 +4171,6 @@ func TestValidateProperties_OrchestratorSpecificProperties(t *testing.T) { t.Errorf("expected error with message : %s, but got %s", expectedMsg, err.Error()) } } - }) } From 93a0d8b5678ea975f42f127cb795b90de1a47877 Mon Sep 17 00:00:00 2001 From: Varun Venkatesh Date: Wed, 17 Feb 2021 23:44:49 +0530 Subject: [PATCH 4/4] fix: restart hns for Windows dual-stack --- .../provisioning/windows/windowsnodereset.ps1 | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/staging/provisioning/windows/windowsnodereset.ps1 b/staging/provisioning/windows/windowsnodereset.ps1 index 6df9d17289..98e89c7f50 100644 --- a/staging/provisioning/windows/windowsnodereset.ps1 +++ b/staging/provisioning/windows/windowsnodereset.ps1 @@ -15,6 +15,7 @@ $global:NetworkMode = "L2Bridge" $global:NetworkPlugin = $Global:ClusterConfiguration.Cni.Name $global:ContainerRuntime = $Global:ClusterConfiguration.Cri.Name $UseContainerD = ($global:ContainerRuntime -eq "containerd") +$IsDualStackEnabled = $Global:ClusterConfiguration.Kubernetes.Kubeproxy.FeatureGates -contains "IPv6DualStack=true" filter Timestamp { "$(Get-Date -Format o): $_" } @@ -45,6 +46,35 @@ if ($global:EnableHostsConfigAgent) { Stop-Service hosts-config-agent } +# Due to a bug in hns there is a race where it picks up the incorrect IPv6 address from the node in some cases. +# Hns service has to be restarted after the node internal IPv6 address is available when dual-stack is enabled. +# TODO Remove this once the bug is fixed in hns. +function Restart-HnsService { + do { + Start-Sleep -Seconds 1 + $nodeInternalIPv6Address = (Get-NetIPAddress | Where-Object {$_.PrefixOrigin -eq "Dhcp" -and $_.AddressFamily -eq "IPv6"}).IPAddress + } while ($nodeInternalIPv6Address -eq $null) + Write-Log "Got node internal IPv6 address: $nodeInternalIPv6Address" + + $hnsManagementIPv6Address = (Get-HnsNetwork | Where-Object {$_.IPv6 -eq $true}).ManagementIPv6 + Write-Log "Got hns ManagementIPv6: $hnsManagementIPv6Address" + + if ($hnsManagementIPv6Address -ne $nodeInternalIPv6Address) { + Restart-Service hns + Write-Log "Restarted hns service" + + $hnsManagementIPv6Address = (Get-HnsNetwork | Where-Object {$_.IPv6 -eq $true}).ManagementIPv6 + Write-Log "Got hns ManagementIPv6: $hnsManagementIPv6Address after restart" + } + else { + Write-Log "Hns network has correct IPv6 address, not restarting" + } +} + +if ($IsDualStackEnabled) { + Restart-HnsService +} + # # Perform cleanup #