From a5cd164d3cbde8cd1cd335d20fd4e79437010bf9 Mon Sep 17 00:00:00 2001 From: Elena Neroslavskaya Date: Thu, 16 Mar 2023 19:55:50 -0400 Subject: [PATCH 1/7] ad managed Node RG name parameter and retrictionLevel --- bicep/main.bicep | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/bicep/main.bicep b/bicep/main.bicep index c2fcbda65..442821195 100644 --- a/bicep/main.bicep +++ b/bicep/main.bicep @@ -1073,6 +1073,18 @@ param enableNodePublicIP bool = false param warIngressNginx bool = false +@description('The name of the resource group to create the AKS cluster managed resources in') +param managedNodeResourceGroup string = '' + + +// Preview feature requires: az feature register --namespace "Microsoft.ContainerService" --name "NRGLockdownPreview" +@allowed([ + 'ReadOnly' + 'Unrestricted' +]) +@description('The restriction level applied to the cluster node resource group') +param restrictionLevelNodeResourceGroup string = 'Unrestricted' + @description('System Pool presets are derived from the recommended system pool specs') var systemPoolPresets = { CostOptimised : { @@ -1278,10 +1290,14 @@ var aksProperties = union({ enabled: fileCSIDriver } } + nodeResourceGroupProfile: { + restrictionLevel: restrictionLevelNodeResourceGroup + } }, aksOutboundTrafficType == 'managedNATGateway' ? managedNATGatewayProfile : {}, defenderForContainers && createLaw ? azureDefenderSecurityProfile : {}, -keyVaultKmsCreateAndPrereqs || !empty(keyVaultKmsByoKeyId) ? azureKeyVaultKms : {} +keyVaultKmsCreateAndPrereqs || !empty(keyVaultKmsByoKeyId) ? azureKeyVaultKms : {}, +!empty(managedNodeResourceGroup) ? { nodeResourceGroup: managedNodeResourceGroup} : {} ) resource aks 'Microsoft.ContainerService/managedClusters@2022-11-02-preview' = { From aa3e38f7b2b7916b689a066c2c923910713c850a Mon Sep 17 00:00:00 2001 From: Elena Neroslavskaya Date: Thu, 16 Mar 2023 22:50:14 -0400 Subject: [PATCH 2/7] add userDefinedRouting aks egress --- bicep/main.bicep | 1 + helper/src/components/networkTab.js | 3 ++- helper/src/components/portalnav.js | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/bicep/main.bicep b/bicep/main.bicep index 442821195..13af345a5 100644 --- a/bicep/main.bicep +++ b/bicep/main.bicep @@ -1045,6 +1045,7 @@ param AutoscaleProfile object = { 'loadBalancer' 'managedNATGateway' 'userAssignedNATGateway' + 'userDefinedRouting' ]) @description('Outbound traffic type for the egress traffic of your cluster') param aksOutboundTrafficType string = 'loadBalancer' diff --git a/helper/src/components/networkTab.js b/helper/src/components/networkTab.js index 392ad588b..4dcfaa141 100644 --- a/helper/src/components/networkTab.js +++ b/helper/src/components/networkTab.js @@ -169,7 +169,8 @@ export default function NetworkTab ({ defaults, tabValues, updateFn, invalidArra options={[ { key: 'loadBalancer', text: 'Load Balancer' }, { key: 'managedNATGateway', text: 'Managed NAT Gateway' }, - { key: 'userAssignedNATGateway', text: 'Assigned NAT Gateway'} + { key: 'userAssignedNATGateway', text: 'Assigned NAT Gateway'}, + { key: 'userDefinedRouting', text: 'User Defined Routing'} ]} onChange={(ev, { key }) => updateFn("aksOutboundTrafficType", key)} /> diff --git a/helper/src/components/portalnav.js b/helper/src/components/portalnav.js index a464258e8..476c36c64 100644 --- a/helper/src/components/portalnav.js +++ b/helper/src/components/portalnav.js @@ -388,6 +388,7 @@ export default function PortalNav({ config }) { : 'This template can only deploy Azure Firewall in single VNET with Custom Networking') invalidFn('net', 'aksOutboundTrafficType', (net.aksOutboundTrafficType === 'managedNATGateway' && net.vnet_opt !== "default") || (net.aksOutboundTrafficType === 'userAssignedNATGateway' && net.vnet_opt === "default"), 'When using Managed Nat Gateway, only default networking is supported. For other networking options, use Assigned NAT Gateway') + invalidFn('net', 'aksOutboundTrafficType', (net.aksOutboundTrafficType === 'userDeinedRouting' && net.vnet_opt !== "byo"), 'When using User Defined Routing, only Bring your Own networking is supported.') invalidFn('net', 'serviceCidr', net.vnet_opt === "custom" && !isCidrValid(net.serviceCidr), invalidCidrMessage) invalidFn('net', 'podCidr', !isCidrValid(net.podCidr), invalidCidrMessage) invalidFn('net', 'dnsServiceIP', !isIPValid(net.dnsServiceIP), 'Enter a valid IP') From 3d344a8902b6b0b50a22d3694f3e92265b5580b7 Mon Sep 17 00:00:00 2001 From: Elena Neroslavskaya Date: Fri, 17 Mar 2023 00:13:40 -0400 Subject: [PATCH 3/7] add userDefinedNetorking checks --- helper/src/components/networkTab.js | 8 ++++---- helper/src/components/portalnav.js | 7 +++++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/helper/src/components/networkTab.js b/helper/src/components/networkTab.js index 4dcfaa141..844290e26 100644 --- a/helper/src/components/networkTab.js +++ b/helper/src/components/networkTab.js @@ -183,7 +183,7 @@ export default function NetworkTab ({ defaults, tabValues, updateFn, invalidArra label="Create NAT Gateway for AKS Subnet (Custom VNet Only)" /> updateFn("afw", v)} label="Implement Azure Firewall & UDR next hop" /> diff --git a/helper/src/components/portalnav.js b/helper/src/components/portalnav.js index 476c36c64..10827b7e9 100644 --- a/helper/src/components/portalnav.js +++ b/helper/src/components/portalnav.js @@ -387,8 +387,11 @@ export default function PortalNav({ config }) { 'Please de-select, when using Bring your own VNET, configure a firewall as part of your own VNET setup, (in a subnet or peered network)' : 'This template can only deploy Azure Firewall in single VNET with Custom Networking') - invalidFn('net', 'aksOutboundTrafficType', (net.aksOutboundTrafficType === 'managedNATGateway' && net.vnet_opt !== "default") || (net.aksOutboundTrafficType === 'userAssignedNATGateway' && net.vnet_opt === "default"), 'When using Managed Nat Gateway, only default networking is supported. For other networking options, use Assigned NAT Gateway') - invalidFn('net', 'aksOutboundTrafficType', (net.aksOutboundTrafficType === 'userDeinedRouting' && net.vnet_opt !== "byo"), 'When using User Defined Routing, only Bring your Own networking is supported.') + invalidFn('net', 'aksOutboundTrafficType', (net.aksOutboundTrafficType === 'managedNATGateway' && net.vnet_opt !== "default") || (net.aksOutboundTrafficType === 'userAssignedNATGateway' && net.vnet_opt === "default") || (net.aksOutboundTrafficType === 'userDefinedRouting' && net.vnet_opt === "defult"), + net.aksOutboundTrafficType === 'userDefinedRouting' ? + 'When using User Defined Routing, only custom and BYON networking is supported.' + : + 'When using Managed Nat Gateway, only default networking is supported. For other networking options, use Assigned NAT Gateway') invalidFn('net', 'serviceCidr', net.vnet_opt === "custom" && !isCidrValid(net.serviceCidr), invalidCidrMessage) invalidFn('net', 'podCidr', !isCidrValid(net.podCidr), invalidCidrMessage) invalidFn('net', 'dnsServiceIP', !isIPValid(net.dnsServiceIP), 'Enter a valid IP') From f5c8cf770701dfd579f67ba5bfc207eb20be82c8 Mon Sep 17 00:00:00 2001 From: Elena Neroslavskaya Date: Fri, 17 Mar 2023 00:19:52 -0400 Subject: [PATCH 4/7] spell BYON --- helper/src/components/portalnav.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helper/src/components/portalnav.js b/helper/src/components/portalnav.js index 10827b7e9..288df7147 100644 --- a/helper/src/components/portalnav.js +++ b/helper/src/components/portalnav.js @@ -389,7 +389,7 @@ export default function PortalNav({ config }) { 'This template can only deploy Azure Firewall in single VNET with Custom Networking') invalidFn('net', 'aksOutboundTrafficType', (net.aksOutboundTrafficType === 'managedNATGateway' && net.vnet_opt !== "default") || (net.aksOutboundTrafficType === 'userAssignedNATGateway' && net.vnet_opt === "default") || (net.aksOutboundTrafficType === 'userDefinedRouting' && net.vnet_opt === "defult"), net.aksOutboundTrafficType === 'userDefinedRouting' ? - 'When using User Defined Routing, only custom and BYON networking is supported.' + 'When using User Defined Routing, only custom and Bring your Own networking is supported.' : 'When using Managed Nat Gateway, only default networking is supported. For other networking options, use Assigned NAT Gateway') invalidFn('net', 'serviceCidr', net.vnet_opt === "custom" && !isCidrValid(net.serviceCidr), invalidCidrMessage) From 095bdeca697bd6102e97e8511791f221faf36fcb Mon Sep 17 00:00:00 2001 From: Elena Neroslavskaya Date: Fri, 17 Mar 2023 00:36:25 -0400 Subject: [PATCH 5/7] similar to PR 546 bug in bicep install --- .github/workflows/ghpagesTest.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ghpagesTest.yml b/.github/workflows/ghpagesTest.yml index 43e844bda..8c3f06a08 100644 --- a/.github/workflows/ghpagesTest.yml +++ b/.github/workflows/ghpagesTest.yml @@ -121,6 +121,7 @@ jobs: if: ${{ github.event.pull_request.head.repo.fork }} shell: pwsh run: | + az config set bicep.use_binary_from_path=False az bicep install - name: Bicep build From 1607ddb82a1b3deb965fd4344b76c0e537966490 Mon Sep 17 00:00:00 2001 From: Elena Neroslavskaya Date: Fri, 17 Mar 2023 13:16:02 -0400 Subject: [PATCH 6/7] update description --- bicep/main.bicep | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bicep/main.bicep b/bicep/main.bicep index 442821195..299a065f1 100644 --- a/bicep/main.bicep +++ b/bicep/main.bicep @@ -1073,7 +1073,7 @@ param enableNodePublicIP bool = false param warIngressNginx bool = false -@description('The name of the resource group to create the AKS cluster managed resources in') +@description('The name of the NEW resource group to create the AKS cluster managed resources in') param managedNodeResourceGroup string = '' From 54631fd90ca4710b0506491c2e9c3afdf326ac26 Mon Sep 17 00:00:00 2001 From: Elena Neroslavskaya Date: Tue, 28 Mar 2023 20:32:44 -0400 Subject: [PATCH 7/7] add udr ui changes --- bicep/network.bicep | 2 +- helper/src/components/deployTab.js | 2 +- helper/src/components/networkTab.js | 7 +++++-- helper/src/components/portalnav.js | 25 ++++++++++++++++--------- 4 files changed, 23 insertions(+), 13 deletions(-) diff --git a/bicep/network.bicep b/bicep/network.bicep index a4e06554a..f3af62aa9 100644 --- a/bicep/network.bicep +++ b/bicep/network.bicep @@ -119,7 +119,7 @@ resource vnet_udr 'Microsoft.Network/routeTables@2022-07-01' = if (azureFirewall { name: 'AKSNodesEgress' properties: { - addressPrefix: '0.0.0.0/1' + addressPrefix: '0.0.0.0/0' nextHopType: 'VirtualAppliance' nextHopIpAddress: azureFirewalls ? calcAzFwIp.outputs.FirewallPrivateIp : null } diff --git a/helper/src/components/deployTab.js b/helper/src/components/deployTab.js index ea2f9f800..d9583e323 100644 --- a/helper/src/components/deployTab.js +++ b/helper/src/components/deployTab.js @@ -107,7 +107,7 @@ export default function DeployTab({ defaults, updateFn, tabValues, invalidArray, ...(addons.appgwKVIntegration && addons.csisecret === 'akvNew' && { appgwKVIntegration: true }) }) }), - ...(net.vnet_opt === "byo" && { + ...(net.vnet_opt !== "default" && { ...(net.aksOutboundTrafficType !== defaults.net.aksOutboundTrafficType && {aksOutboundTrafficType: net.aksOutboundTrafficType}) }), ...(cluster.keyVaultKms !== defaults.cluster.keyVaultKms && { diff --git a/helper/src/components/networkTab.js b/helper/src/components/networkTab.js index 844290e26..44ba0f5d9 100644 --- a/helper/src/components/networkTab.js +++ b/helper/src/components/networkTab.js @@ -158,6 +158,9 @@ export default function NetworkTab ({ defaults, tabValues, updateFn, invalidArra NAT Gateway allows more traffic flows than a Load Balancer.docs + {net.aksOutboundTrafficType==='userDefinedRouting' && net.vnet_opt === 'byo' && + Ensure that the AKS Subnet is configured with a UDR and that your Virtual Network Appliance is properly configured to allow necessary traffic + } {hasError(invalidArray, 'aksOutboundTrafficType') && {getError(invalidArray, 'aksOutboundTrafficType')} } @@ -211,10 +214,10 @@ export default function NetworkTab ({ defaults, tabValues, updateFn, invalidArra {getError(invalidArray, 'afw')} } updateFn("afw", v)} label="Implement Azure Firewall & UDR next hop" /> diff --git a/helper/src/components/portalnav.js b/helper/src/components/portalnav.js index 288df7147..ec7286bb3 100644 --- a/helper/src/components/portalnav.js +++ b/helper/src/components/portalnav.js @@ -382,16 +382,23 @@ export default function PortalNav({ config }) { invalidFn('net', 'byoAKSSubnetId', net.vnet_opt === 'byo' && !net.byoAKSSubnetId.match('^/subscriptions/[^/ ]+/resourceGroups/[^/ ]+/providers/Microsoft.Network/virtualNetworks/[^/ ]+/subnets/[^/ ]+$'), 'Enter a valid Subnet Id where AKS nodes will be installed') invalidFn('net', 'byoAGWSubnetId', net.vnet_opt === 'byo' && addons.ingress === 'appgw' && !net.byoAGWSubnetId.match('^/subscriptions/[^/ ]+/resourceGroups/[^/ ]+/providers/Microsoft.Network/virtualNetworks/[^/ ]+/subnets/[^/ ]+$'), 'Enter a valid Subnet Id where Application Gateway is installed') invalidFn('net', 'vnet_opt', net.vnet_opt === "default" && (net.afw || net.vnetprivateend), 'Cannot use default networking of you select Firewall or Private Link') - invalidFn('net', 'afw', net.afw && net.vnet_opt !== "custom", - net.vnet_opt === "byo" ? - 'Please de-select, when using Bring your own VNET, configure a firewall as part of your own VNET setup, (in a subnet or peered network)' - : - 'This template can only deploy Azure Firewall in single VNET with Custom Networking') - invalidFn('net', 'aksOutboundTrafficType', (net.aksOutboundTrafficType === 'managedNATGateway' && net.vnet_opt !== "default") || (net.aksOutboundTrafficType === 'userAssignedNATGateway' && net.vnet_opt === "default") || (net.aksOutboundTrafficType === 'userDefinedRouting' && net.vnet_opt === "defult"), + invalidFn('net', 'afw', + (net.afw && net.vnet_opt !== "custom") || + (!net.afw && net.aksOutboundTrafficType === 'userDefinedRouting' && net.vnet_opt === "custom"), + net.afw && net.vnet_opt === "byo" ? + 'Please de-select, when using Bring your own VNET, configure a firewall as part of your own VNET setup, (in a subnet or peered network)' + : net.afw && net.vnet_opt === "default" ? + 'This template can only deploy Azure Firewall in single VNET with Custom Networking' + :'Ensure to select Azure Firewall when using Custom Networking and User Defined Routing for Outbound Traffic Type') + + invalidFn('net', 'aksOutboundTrafficType', + (net.aksOutboundTrafficType === 'managedNATGateway' && net.vnet_opt !== "default") || + (net.aksOutboundTrafficType === 'userAssignedNATGateway' && net.vnet_opt === "default") || + (net.aksOutboundTrafficType === 'userDefinedRouting' && net.vnet_opt === "default"), net.aksOutboundTrafficType === 'userDefinedRouting' ? - 'When using User Defined Routing, only custom and Bring your Own networking is supported.' - : - 'When using Managed Nat Gateway, only default networking is supported. For other networking options, use Assigned NAT Gateway') + 'When using User Defined Routing, only custom and Bring your Own networking is supported.' + : + 'When using Managed Nat Gateway, only default networking is supported. For other networking options, use Assigned NAT Gateway') invalidFn('net', 'serviceCidr', net.vnet_opt === "custom" && !isCidrValid(net.serviceCidr), invalidCidrMessage) invalidFn('net', 'podCidr', !isCidrValid(net.podCidr), invalidCidrMessage) invalidFn('net', 'dnsServiceIP', !isIPValid(net.dnsServiceIP), 'Enter a valid IP')