From ce3db55b924b0ba6516ee885de205d74a4b59c88 Mon Sep 17 00:00:00 2001 From: samaea <32162601+samaea@users.noreply.github.com> Date: Sat, 15 Apr 2023 01:18:05 +0000 Subject: [PATCH 1/3] Working Bicep templates. --- bicep/aksagentpool.bicep | 4 ++++ bicep/aksnetcontrib.bicep | 17 +++++++++++++++++ bicep/main.bicep | 9 +++++++++ bicep/network.bicep | 27 +++++++++++++++++++++++++++ 4 files changed, 57 insertions(+) diff --git a/bicep/aksagentpool.bicep b/bicep/aksagentpool.bicep index fae38e2f1..9a94fe620 100644 --- a/bicep/aksagentpool.bicep +++ b/bicep/aksagentpool.bicep @@ -33,6 +33,9 @@ param nodeLabels object = {} @description('The subnet the node pool will use') param subnetId string +@description('The subnet the pods will use') +param podSubnetID string + @description('OS Type for the node pool') @allowed(['Linux','Windows']) param osType string @@ -70,6 +73,7 @@ resource userNodepool 'Microsoft.ContainerService/managedClusters/agentPools@202 maxPods: maxPods type: 'VirtualMachineScaleSets' vnetSubnetID: !empty(subnetId) ? subnetId : null + podSubnetID: !empty(podSubnetID) ? podSubnetID : null upgradeSettings: { maxSurge: '33%' } diff --git a/bicep/aksnetcontrib.bicep b/bicep/aksnetcontrib.bicep index 488318c24..04426dd40 100644 --- a/bicep/aksnetcontrib.bicep +++ b/bicep/aksnetcontrib.bicep @@ -2,6 +2,7 @@ //name/rg required to new up an existing reference and form a dependency //principalid required as it needs to be used to establish a unique roleassignment name param byoAKSSubnetId string +param byoAKSPodSubnetId string param user_identity_principalId string @allowed([ @@ -12,6 +13,7 @@ param rbacAssignmentScope string = 'Subnet' var networkContributorRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '4d97b98b-1d4f-4787-a291-c67834d212e7') +var existingAksPodSubnetName = !empty(byoAKSPodSubnetId) ? split(byoAKSPodSubnetId, '/')[10] : '' var existingAksSubnetName = !empty(byoAKSSubnetId) ? split(byoAKSSubnetId, '/')[10] : '' var existingAksVnetName = !empty(byoAKSSubnetId) ? split(byoAKSSubnetId, '/')[8] : '' @@ -23,6 +25,11 @@ resource existingAksSubnet 'Microsoft.Network/virtualNetworks/subnets@2022-07-01 name: existingAksSubnetName } +resource existingAksPodSubnet 'Microsoft.Network/virtualNetworks/subnets@2022-07-01' existing = { + parent: existingvnet + name: existingAksPodSubnetName +} + resource subnetRbac 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (rbacAssignmentScope == 'subnet') { name: guid(user_identity_principalId, networkContributorRole, existingAksSubnetName) scope: existingAksSubnet @@ -33,6 +40,16 @@ resource subnetRbac 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (r } } +resource podSubnetRbac 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (rbacAssignmentScope == 'subnet') { + name: guid(user_identity_principalId, networkContributorRole, existingAksPodSubnetName) + scope: existingAksPodSubnet + properties: { + roleDefinitionId: networkContributorRole + principalId: user_identity_principalId + principalType: 'ServicePrincipal' + } +} + resource existingVnetRbac 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (rbacAssignmentScope != 'subnet') { name: guid(user_identity_principalId, networkContributorRole, existingAksVnetName) scope: existingvnet diff --git a/bicep/main.bicep b/bicep/main.bicep index ffcccc796..cdbdf0c05 100644 --- a/bicep/main.bicep +++ b/bicep/main.bicep @@ -35,6 +35,9 @@ param custom_vnet bool = false @description('Full resource id path of an existing subnet to use for AKS') param byoAKSSubnetId string = '' +@description('Full resource id path of an existing pod subnet to use for AKS') +param byoAKSPodSubnetId string = '' + @description('Full resource id path of an existing subnet to use for Application Gateway') param byoAGWSubnetId string = '' @@ -62,6 +65,7 @@ module aksnetcontrib './aksnetcontrib.bicep' = if (!empty(byoAKSSubnetId) && cre scope: resourceGroup(existingAksVnetRG) params: { byoAKSSubnetId: byoAKSSubnetId + byoAKSPodSubnetId: byoAKSPodSubnetId user_identity_principalId: createAksUai ? aksUai.properties.principalId : '' rbacAssignmentScope: uaiNetworkScopeRbac } @@ -130,6 +134,8 @@ module network './network.bicep' = if (custom_vnet) { location: location networkPluginIsKubenet: networkPlugin=='kubenet' vnetAddressPrefix: vnetAddressPrefix + vnetPodAddressPrefix: cniDynamicIpAllocation ? podCidr : '' + cniDynamicIpAllocation: cniDynamicIpAllocation aksPrincipleId: aksPrincipalId vnetAksSubnetAddressPrefix: vnetAksSubnetAddressPrefix ingressApplicationGateway: ingressApplicationGateway @@ -161,6 +167,7 @@ output CustomVnetId string = custom_vnet ? network.outputs.vnetId : '' output CustomVnetPrivateLinkSubnetId string = custom_vnet ? network.outputs.privateLinkSubnetId : '' var aksSubnetId = custom_vnet ? network.outputs.aksSubnetId : byoAKSSubnetId +var aksPodSubnetId = custom_vnet ? network.outputs.aksPodSubnetId : byoAKSPodSubnetId var appGwSubnetId = ingressApplicationGateway ? (custom_vnet ? network.outputs.appGwSubnetId : byoAGWSubnetId) : '' @@ -1141,6 +1148,7 @@ var systemPoolBase = { maxPods: 30 type: 'VirtualMachineScaleSets' vnetSubnetID: !empty(aksSubnetId) ? aksSubnetId : null + podSubnetID: !empty(aksPodSubnetId) ? aksPodSubnetId : null upgradeSettings: { maxSurge: '33%' } @@ -1352,6 +1360,7 @@ module userNodePool '../bicep/aksagentpool.bicep' = if (!JustUseSystemPool){ AksName: aks.name PoolName: poolName subnetId: aksSubnetId + podSubnetID: !empty(aksPodSubnetId) ? aksPodSubnetId : '' agentCount: agentCount agentCountMax: agentCountMax agentVMSize: agentVMSize diff --git a/bicep/network.bicep b/bicep/network.bicep index f3af62aa9..e166567d3 100644 --- a/bicep/network.bicep +++ b/bicep/network.bicep @@ -6,6 +6,9 @@ param aksPrincipleId string = '' param vnetAddressPrefix string param vnetAksSubnetAddressPrefix string +param vnetPodAddressPrefix string + +param cniDynamicIpAllocation bool = false //Nsg param workspaceName string = '' @@ -160,10 +163,33 @@ var aks_baseSubnet = { }: {}) } +var aks_podSubnet_name = 'aks-sn-pods' +var aks_podSubnet = { + name: aks_podSubnet_name + properties: union({ + addressPrefix: vnetPodAddressPrefix + }, privateLinks ? { + privateEndpointNetworkPolicies: 'Disabled' + privateLinkServiceNetworkPolicies: 'Enabled' + } : {}, natGateway ? { + natGateway: { + id: natGw.id + } + } : {}, azureFirewalls ? { + routeTable: { + id: vnet_udr.id //resourceId('Microsoft.Network/routeTables', routeFwTableName) + } + }: {}) +} + var aks_subnet = networkSecurityGroups ? union(aks_baseSubnet, nsgAks.outputs.nsgSubnetObj) : aks_baseSubnet +var aks_podsubnet = networkSecurityGroups ? union(aks_podSubnet, nsgAks.outputs.nsgSubnetObj) : aks_podSubnet + + var subnets = union( array(aks_subnet), + cniDynamicIpAllocation ? array(aks_podsubnet) : [], azureFirewalls ? array(fw_subnet) : [], privateLinks ? array(private_link_subnet) : [], acrPrivatePool ? array(acrpool_subnet) : [], @@ -189,6 +215,7 @@ resource vnet 'Microsoft.Network/virtualNetworks@2022-07-01' = { output vnetId string = vnet.id output vnetName string = vnet.name output aksSubnetId string = resourceId('Microsoft.Network/virtualNetworks/subnets', vnet.name, aks_subnet_name) +output aksPodSubnetId string = resourceId('Microsoft.Network/virtualNetworks/subnets', vnet.name, aks_podSubnet_name) output fwSubnetId string = azureFirewalls ? '${vnet.id}/subnets/${fw_subnet_name}' : '' output fwMgmtSubnetId string = azureFirewallsManagementSeperation ? '${vnet.id}/subnets/${fwmgmt_subnet_name}' : '' output acrPoolSubnetId string = acrPrivatePool ? '${vnet.id}/subnets/${acrpool_subnet_name}' : '' From 90bf3e076c635793e960cd65efb40a7225ae64b4 Mon Sep 17 00:00:00 2001 From: samaea <32162601+samaea@users.noreply.github.com> Date: Sat, 15 Apr 2023 01:48:34 +0000 Subject: [PATCH 2/3] UI implementation completed. --- helper/src/components/deployTab.js | 1 + helper/src/components/networkTab.js | 3 ++- helper/src/components/portalnav.js | 1 + helper/src/config.json | 1 + 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/helper/src/components/deployTab.js b/helper/src/components/deployTab.js index 3ef8bcf43..9df71d899 100644 --- a/helper/src/components/deployTab.js +++ b/helper/src/components/deployTab.js @@ -51,6 +51,7 @@ export default function DeployTab({ defaults, updateFn, tabValues, invalidArray, ...(net.bastion && defaults.net.bastionSubnetAddressPrefix !== net.bastionSubnetAddressPrefix && {bastionSubnetAddressPrefix: net.bastionSubnetAddressPrefix}) }), ...(net.vnet_opt === "byo" && { byoAKSSubnetId: net.byoAKSSubnetId, ...serviceparams }), + ...(net.vnet_opt === "byo" && net.cniDynamicIpAllocation && { byoAKSPodSubnetId: net.byoAKSPodSubnetId}), ...(net.vnet_opt === "byo" && addons.ingress === 'appgw' && { byoAGWSubnetId: net.byoAGWSubnetId }), ...(cluster.enable_aad && { enable_aad: true, ...(cluster.enableAzureRBAC === false && cluster.aad_tenant_id && { aad_tenant_id: cluster.aad_tenant_id }) }), ...(cluster.enable_aad && cluster.AksDisableLocalAccounts !== defaults.cluster.AksDisableLocalAccounts && { AksDisableLocalAccounts: cluster.AksDisableLocalAccounts }), diff --git a/helper/src/components/networkTab.js b/helper/src/components/networkTab.js index 44ba0f5d9..9c2fe6432 100644 --- a/helper/src/components/networkTab.js +++ b/helper/src/components/networkTab.js @@ -404,7 +404,8 @@ function BYOVNET({ net, addons, updateFn, invalidArray }) { updateFn("byoAKSSubnetId", v)} errorMessage={getError(invalidArray, 'byoAKSSubnetId')} required placeholder="Resource Id" label={Enter your existing AKS Nodes subnet ResourceId} /> - + updateFn("byoAKSPodSubnetId", v)} errorMessage={getError(invalidArray, 'byoAKSPodSubnetId')} required placeholder="Resource Id" label={Enter your existing AKS Pods subnet ResourceId} /> + updateFn("byoAGWSubnetId", v)} errorMessage={getError(invalidArray, 'byoAGWSubnetId')} required placeholder="Resource Id" label={Enter your existing Application Gateway subnet ResourceId} /> Ensure your Application Gateway subnet meets these requirements here diff --git a/helper/src/components/portalnav.js b/helper/src/components/portalnav.js index ec7286bb3..a1471ef05 100644 --- a/helper/src/components/portalnav.js +++ b/helper/src/components/portalnav.js @@ -380,6 +380,7 @@ export default function PortalNav({ config }) { invalidFn('addons', 'appgwKVIntegration', addons.ingress === "appgw" && addons.appgwKVIntegration && addons.csisecret !== 'akvNew', 'KeyVault integration requires the \'CSI Secrets\' \'Yes, Provision a new KeyVault\' option to be selected') invalidFn('addons', 'ingress', cluster.osType === "Windows" && addons.ingress !== "appgw" && addons.ingress !== "none", 'Neither the Windows nodepool or the system pool will be able to run your selected Ingress Controller. To support this Ingress Controller, add another linux nodepool post cluster creation.') invalidFn('net', 'byoAKSSubnetId', net.vnet_opt === 'byo' && !net.byoAKSSubnetId.match('^/subscriptions/[^/ ]+/resourceGroups/[^/ ]+/providers/Microsoft.Network/virtualNetworks/[^/ ]+/subnets/[^/ ]+$'), 'Enter a valid Subnet Id where AKS nodes will be installed') + invalidFn('net', 'byoAKSPodSubnetId', net.vnet_opt === 'byo' && net.cniDynamicIpAllocation && !net.byoAKSPodSubnetId.match('^/subscriptions/[^/ ]+/resourceGroups/[^/ ]+/providers/Microsoft.Network/virtualNetworks/[^/ ]+/subnets/[^/ ]+$'), 'Enter a valid Subnet Id where AKS pods will be installed') invalidFn('net', 'byoAGWSubnetId', net.vnet_opt === 'byo' && addons.ingress === 'appgw' && !net.byoAGWSubnetId.match('^/subscriptions/[^/ ]+/resourceGroups/[^/ ]+/providers/Microsoft.Network/virtualNetworks/[^/ ]+/subnets/[^/ ]+$'), 'Enter a valid Subnet Id where Application Gateway is installed') invalidFn('net', 'vnet_opt', net.vnet_opt === "default" && (net.afw || net.vnetprivateend), 'Cannot use default networking of you select Firewall or Private Link') invalidFn('net', 'afw', diff --git a/helper/src/config.json b/helper/src/config.json index 1c89b8e44..3abb85efa 100644 --- a/helper/src/config.json +++ b/helper/src/config.json @@ -126,6 +126,7 @@ "enableNodePublicIP": false, "vnet_opt": "default", "byoAKSSubnetId": "", + "byoAKSPodSubnetId": "", "byoAGWSubnetId": "", "vnetAddressPrefix": "10.240.0.0/16", "vnetAksSubnetAddressPrefix": "10.240.0.0/22", From ee49393b10ed165c8e5bf914af3a04c8f0dc96e2 Mon Sep 17 00:00:00 2001 From: samaea <32162601+samaea@users.noreply.github.com> Date: Fri, 5 May 2023 10:35:04 +0000 Subject: [PATCH 3/3] Amended podsubnet prefix for consistency. --- bicep/network.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bicep/network.bicep b/bicep/network.bicep index 44cc6dde9..1392f0471 100644 --- a/bicep/network.bicep +++ b/bicep/network.bicep @@ -163,7 +163,7 @@ var aks_baseSubnet = { }: {}) } -var aks_podSubnet_name = 'aks-sn-pods' +var aks_podSubnet_name = 'aks-pods-sn' var aks_podSubnet = { name: aks_podSubnet_name properties: union({