Skip to content
This repository has been archived by the owner on Oct 24, 2023. It is now read-only.

Commit

Permalink
feat: allow creation of dualstack Windows clusters (#4176)
Browse files Browse the repository at this point in the history 
* feat: allow creation of dualstack Windows clusters

* Apply suggestions from code review

Co-authored-by: James Sturtevant <jsturtevant@gmail.com>

* Disable IPv6-only on Windows

* fix: restart hns for Windows dual-stack

Co-authored-by: Varun Venkatesh <vavenk@microsoft.com>
Co-authored-by: James Sturtevant <jsturtevant@gmail.com>
  • Loading branch information
@vavenk-ms @vavenk @jsturtevant
3 people authored Mar 2, 2021
1 parent 30bb338 commit 9047a45
Show file tree
Hide file tree
Showing 5 changed files with 118 additions and 16 deletions.
6 changes: 2 additions & 4 deletions examples/dualstack/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,10 +113,8 @@ nginx-ipv6 LoadBalancer fd00::6283 2603:1030:805:3::3 80:31140/TCP

- Dual-stack clusters are supported only with kubenet and azurecni.
- Dual-stack cluster with azurecni are only supported with `bridge` network mode.
- Dual-stack clusters are supported only with Linux.
- Dual-stack clusters with Windows is not supported at this time because it requires
- Kubernetes version 1.19+ and
- [backport to 2004 to support dual-stack containers](https://github.com/Azure/aks-engine/issues/3568).
- Dual-stack clusters are supported on Windows from version 2004 (kernel version 10.0.19041.610) and Kubernetes version 1.19
- https://kubernetes.io/docs/setup/production-environment/windows/intro-windows-in-kubernetes/#ipv4-ipv6-dual-stack
- Dual-stack clusters are supported with
- `ipvs` kube-proxy mode (Kubernetes version 1.16+)
- `iptables` kube-proxy mode (Kubernetes version 1.18+).
Expand Down
60 changes: 60 additions & 0 deletions examples/dualstack/kubernetes-windows.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
{
"apiVersion": "vlabs",
"properties": {
"featureFlags": {
"enableIPv6DualStack": true
},
"orchestratorProfile": {
"orchestratorRelease": "1.19",
"kubernetesConfig": {
"apiServerConfig": {
"--feature-gates": "IPv6DualStack=true"
},
"kubeletConfig": {
"--feature-gates": "IPv6DualStack=true"
},
"controllerManagerConfig": {
"--feature-gates": "IPv6DualStack=true"
},
"kubeProxyMode": "ipvs",
"networkPlugin": "azure",
"networkMode": "bridge",
"networkPolicy": "",
"useManagedIdentity": false
}
},
"masterProfile": {
"count": 1,
"dnsPrefix": "",
"vmSize": "Standard_D2_v3"
},
"agentPoolProfiles": [
{
"name": "windowspool2",
"count": 1,
"vmSize": "Standard_D2_v3",
"availabilityProfile": "VirtualMachineScaleSets",
"osType": "Windows",
"osDiskSizeGB": 128
}
],
"windowsProfile": {
"windowsPublisher": "MicrosoftWindowsServer",
"windowsOffer": "WindowsServer",
"windowsSku": "Datacenter-Core-2004-with-Containers-smalldisk",
"imageVersion": "latest",
"adminUsername": "azureuser",
"adminPassword": "replacepassword1234$"
},
"linuxProfile": {
"adminUsername": "azureuser",
"ssh": {
"publicKeys": [
{
"keyData": ""
}
]
}
}
}
}
7 changes: 6 additions & 1 deletion pkg/api/vlabs/validate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -435,7 +435,12 @@ func (a *Properties) validateAgentPoolProfiles(isUpdate bool) error {
// validate os type is linux if dual stack feature is enabled
if a.FeatureFlags.IsIPv6DualStackEnabled() || a.FeatureFlags.IsIPv6OnlyEnabled() {
if agentPoolProfile.OSType == Windows {
return errors.Errorf("Dual stack and single stack IPv6 feature is supported only with Linux, but agent pool '%s' is of os type %s", agentPoolProfile.Name, agentPoolProfile.OSType)
if a.FeatureFlags.IsIPv6DualStackEnabled() && !common.IsKubernetesVersionGe(a.OrchestratorProfile.OrchestratorVersion, "1.19.0") {
return errors.Errorf("Dual stack IPv6 feature is supported on Windows only from Kubernetes version 1.19, but OrchestratorProfile.OrchestratorVersion is '%s'", a.OrchestratorProfile.OrchestratorVersion)
}
if a.FeatureFlags.IsIPv6OnlyEnabled() {
return errors.Errorf("Single stack IPv6 feature is supported only with Linux, but agent pool '%s' is of os type %s", agentPoolProfile.Name, agentPoolProfile.OSType)
}
}
if agentPoolProfile.Distro == Flatcar {
return errors.Errorf("Dual stack and single stack IPv6 feature is currently supported only with Ubuntu, but agent pool '%s' is of distro type %s", agentPoolProfile.Name, agentPoolProfile.Distro)
Expand Down
31 changes: 20 additions & 11 deletions pkg/api/vlabs/validate_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4146,20 +4146,30 @@ func TestValidateProperties_OrchestratorSpecificProperties(t *testing.T) {
}
})

t.Run("Should not support os type other than linux for single stack ipv6 and dual stack feature", func(t *testing.T) {
t.Run("Should not support os type other than linux for single stack ipv6 and versions less than 1.19 for dual stack feature", func(t *testing.T) {
t.Parallel()
cs := getK8sDefaultContainerService(true)

masterProfile := cs.Properties.MasterProfile
masterProfile.Distro = Ubuntu
agentPoolProfiles := cs.Properties.AgentPoolProfiles

agentPoolProfiles[0].OSType = Windows
cs.Properties.FeatureFlags = &FeatureFlags{EnableIPv6DualStack: true}
cs.Properties.OrchestratorProfile.OrchestratorVersion = "1.18"
expectedMsg := fmt.Sprintf("Dual stack IPv6 feature is supported on Windows only from Kubernetes version 1.19, but OrchestratorProfile.OrchestratorVersion is '%s'", cs.Properties.OrchestratorProfile.OrchestratorVersion)
if err := cs.Properties.validateAgentPoolProfiles(false); err.Error() != expectedMsg {
t.Errorf("expected error with message : %s, but got %s", expectedMsg, err.Error())
}

cs.Properties.FeatureFlags = &FeatureFlags{EnableIPv6Only: true}
expectedMsg = fmt.Sprintf("Single stack IPv6 feature is supported only with Linux, but agent pool '%s' is of os type %s", agentPoolProfiles[0].Name, agentPoolProfiles[0].OSType)
if err := cs.Properties.validateAgentPoolProfiles(false); err.Error() != expectedMsg {
t.Errorf("expected error with message : %s, but got %s", expectedMsg, err.Error())
}

for _, featureFlags := range []FeatureFlags{{EnableIPv6DualStack: true}, {EnableIPv6Only: true}} {
cs.Properties.FeatureFlags = &featureFlags
masterProfile := cs.Properties.MasterProfile

masterProfile.Distro = Ubuntu
agentPoolProfiles := cs.Properties.AgentPoolProfiles
agentPoolProfiles[0].OSType = Windows
expectedMsg := fmt.Sprintf("Dual stack and single stack IPv6 feature is supported only with Linux, but agent pool '%s' is of os type %s", agentPoolProfiles[0].Name, agentPoolProfiles[0].OSType)
if err := cs.Properties.validateAgentPoolProfiles(false); err.Error() != expectedMsg {
t.Errorf("expected error with message : %s, but got %s", expectedMsg, err.Error())
}

agentPoolProfiles[0].OSType = Linux
agentPoolProfiles[0].Distro = Flatcar
Expand All @@ -4168,7 +4178,6 @@ func TestValidateProperties_OrchestratorSpecificProperties(t *testing.T) {
t.Errorf("expected error with message : %s, but got %s", expectedMsg, err.Error())
}
}

})
}

Expand Down
30 changes: 30 additions & 0 deletions staging/provisioning/windows/windowsnodereset.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ $global:NetworkMode = "L2Bridge"
$global:NetworkPlugin = $Global:ClusterConfiguration.Cni.Name
$global:ContainerRuntime = $Global:ClusterConfiguration.Cri.Name
$UseContainerD = ($global:ContainerRuntime -eq "containerd")
$IsDualStackEnabled = $Global:ClusterConfiguration.Kubernetes.Kubeproxy.FeatureGates -contains "IPv6DualStack=true"

filter Timestamp { "$(Get-Date -Format o): $_" }

Expand Down Expand Up @@ -45,6 +46,35 @@ if ($global:EnableHostsConfigAgent) {
Stop-Service hosts-config-agent
}

# Due to a bug in hns there is a race where it picks up the incorrect IPv6 address from the node in some cases.
# Hns service has to be restarted after the node internal IPv6 address is available when dual-stack is enabled.
# TODO Remove this once the bug is fixed in hns.
function Restart-HnsService {
do {
Start-Sleep -Seconds 1
$nodeInternalIPv6Address = (Get-NetIPAddress | Where-Object {$_.PrefixOrigin -eq "Dhcp" -and $_.AddressFamily -eq "IPv6"}).IPAddress
} while ($nodeInternalIPv6Address -eq $null)
Write-Log "Got node internal IPv6 address: $nodeInternalIPv6Address"

$hnsManagementIPv6Address = (Get-HnsNetwork | Where-Object {$_.IPv6 -eq $true}).ManagementIPv6
Write-Log "Got hns ManagementIPv6: $hnsManagementIPv6Address"

if ($hnsManagementIPv6Address -ne $nodeInternalIPv6Address) {
Restart-Service hns
Write-Log "Restarted hns service"

$hnsManagementIPv6Address = (Get-HnsNetwork | Where-Object {$_.IPv6 -eq $true}).ManagementIPv6
Write-Log "Got hns ManagementIPv6: $hnsManagementIPv6Address after restart"
}
else {
Write-Log "Hns network has correct IPv6 address, not restarting"
}
}

if ($IsDualStackEnabled) {
Restart-HnsService
}

#
# Perform cleanup
#
Expand Down

0 comments on commit 9047a45

Please sign in to comment.