From b420117251686a78e688ed1bf10ed849f6837f20 Mon Sep 17 00:00:00 2001 From: Alexander Sehr Date: Mon, 14 Oct 2024 12:13:30 +0200 Subject: [PATCH] fix: Aligned AKS interface to AVM specs & added UDT (#3506) ## Description - Aligned AKS interface to AVM specs - Added UDT & mapping for primary agent pool Depending on #3228 ## Pipeline Reference | Pipeline | | -------- | | [![avm.res.container-service.managed-cluster](https://github.com/Azure/bicep-registry-modules/actions/workflows/avm.res.container-service.managed-cluster.yml/badge.svg?branch=users%2Falsehr%2FcontainerServiceInterfaceFix&event=workflow_dispatch)](https://github.com/Azure/bicep-registry-modules/actions/workflows/avm.res.container-service.managed-cluster.yml) | ## Type of Change - [ ] Update to CI Environment or utilities (Non-module affecting changes) - [ ] Azure Verified Module updates: - [x] Bugfix containing backwards-compatible bug fixes, and I have NOT bumped the MAJOR or MINOR version in `version.json`: - [ ] Someone has opened a bug report issue, and I have included "Closes #{bug_report_issue_number}" in the PR description. - [ ] The bug was found by the module author, and no one has opened an issue to report it yet. - [ ] Feature update backwards compatible feature updates, and I have bumped the MINOR version in `version.json`. - [ ] Breaking changes and I have bumped the MAJOR version in `version.json`. - [ ] Update to documentation --- .../managed-cluster/README.md | 577 ++++++++++--- .../managed-cluster/agent-pool/README.md | 22 +- .../managed-cluster/agent-pool/main.bicep | 18 +- .../managed-cluster/agent-pool/main.json | 29 +- .../managed-cluster/main.bicep | 140 +++- .../managed-cluster/main.json | 776 +++++++++--------- .../maintenance-configurations/main.json | 4 +- .../tests/e2e/automatic/main.test.bicep | 3 +- .../tests/e2e/azure/main.test.bicep | 16 +- .../tests/e2e/defaults/main.test.bicep | 2 +- .../tests/e2e/kubenet/main.test.bicep | 8 +- .../tests/e2e/priv/main.test.bicep | 12 +- .../tests/e2e/waf-aligned/main.test.bicep | 14 +- 13 files changed, 1045 insertions(+), 576 deletions(-) diff --git a/avm/res/container-service/managed-cluster/README.md b/avm/res/container-service/managed-cluster/README.md index b5cbcf20e7..27d80860c1 100644 --- a/avm/res/container-service/managed-cluster/README.md +++ b/avm/res/container-service/managed-cluster/README.md @@ -54,7 +54,7 @@ module managedCluster 'br/public:avm/res/container-service/managed-cluster:' // Required parameters param name = 'csauto001' -param primaryAgentPoolProfile = [ +param primaryAgentPoolProfiles = [ { count: 3 mode: 'System' @@ -217,10 +217,10 @@ module managedCluster 'br/public:avm/res/container-service/managed-cluster:' + vnetSubnetResourceId: '' } ] // Non-required parameters agentPools: [ { availabilityZones: [ - '3' + 3 ] count: 2 enableAutoScaling: true @@ -261,11 +261,11 @@ module managedCluster 'br/public:avm/res/container-service/managed-cluster:' + vnetSubnetResourceId: '' } { availabilityZones: [ - '3' + 3 ] count: 2 enableAutoScaling: true @@ -282,7 +282,7 @@ module managedCluster 'br/public:avm/res/container-service/managed-cluster:' + vnetSubnetResourceId: '' } ] autoNodeOsUpgradeProfileUpgradeChannel: 'Unmanaged' @@ -422,7 +422,7 @@ module managedCluster 'br/public:avm/res/container-service/managed-cluster:' ] } - monitoringWorkspaceId: '' + monitoringWorkspaceResourceId: '' networkDataplane: 'azure' networkPlugin: 'azure' networkPluginMode: 'overlay' @@ -472,11 +472,11 @@ module managedCluster 'br/public:avm/res/container-service/managed-cluster:" + "vnetSubnetResourceId": "" } ] }, @@ -501,7 +501,7 @@ module managedCluster 'br/public:avm/res/container-service/managed-cluster:" + "vnetSubnetResourceId": "" }, { "availabilityZones": [ - "3" + 3 ], "count": 2, "enableAutoScaling": true, @@ -540,7 +540,7 @@ module managedCluster 'br/public:avm/res/container-service/managed-cluster:" + "vnetSubnetResourceId": "" } ] }, @@ -723,8 +723,8 @@ module managedCluster 'br/public:avm/res/container-service/managed-cluster:" + "monitoringWorkspaceResourceId": { + "value": "" }, "networkDataplane": { "value": "azure" @@ -785,10 +785,10 @@ using 'br/public:avm/res/container-service/managed-cluster:' // Required parameters param name = 'csmaz001' -param primaryAgentPoolProfile = [ +param primaryAgentPoolProfiles = [ { availabilityZones: [ - '3' + 3 ] count: 1 enableAutoScaling: true @@ -804,14 +804,14 @@ param primaryAgentPoolProfile = [ osType: 'Linux' type: 'VirtualMachineScaleSets' vmSize: 'Standard_DS2_v2' - vnetSubnetID: '' + vnetSubnetResourceId: '' } ] // Non-required parameters param agentPools = [ { availabilityZones: [ - '3' + 3 ] count: 2 enableAutoScaling: true @@ -829,11 +829,11 @@ param agentPools = [ scaleSetPriority: 'Regular' type: 'VirtualMachineScaleSets' vmSize: 'Standard_DS2_v2' - vnetSubnetID: '' + vnetSubnetResourceId: '' } { availabilityZones: [ - '3' + 3 ] count: 2 enableAutoScaling: true @@ -850,7 +850,7 @@ param agentPools = [ scaleSetPriority: 'Regular' type: 'VirtualMachineScaleSets' vmSize: 'Standard_DS2_v2' - vnetSubnetID: '' + vnetSubnetResourceId: '' } ] param autoNodeOsUpgradeProfileUpgradeChannel = 'Unmanaged' @@ -990,7 +990,7 @@ param managedIdentities = { '' ] } -param monitoringWorkspaceId = '' +param monitoringWorkspaceResourceId = '' param networkDataplane = 'azure' param networkPlugin = 'azure' param networkPluginMode = 'overlay' @@ -1040,7 +1040,7 @@ module managedCluster 'br/public:avm/res/container-service/managed-cluster:' // Required parameters param name = 'csmin001' -param primaryAgentPoolProfile = [ +param primaryAgentPoolProfiles = [ { count: 3 mode: 'System' @@ -1141,10 +1141,10 @@ module managedCluster 'br/public:avm/res/container-service/managed-cluster:' // Required parameters param name = 'csmkube001' -param primaryAgentPoolProfile = [ +param primaryAgentPoolProfiles = [ { availabilityZones: [ - '3' + 3 ] count: 1 enableAutoScaling: true @@ -1436,7 +1436,7 @@ param primaryAgentPoolProfile = [ param agentPools = [ { availabilityZones: [ - '3' + 3 ] count: 2 enableAutoScaling: true @@ -1456,7 +1456,7 @@ param agentPools = [ } { availabilityZones: [ - '3' + 3 ] count: 2 enableAutoScaling: true @@ -1540,10 +1540,10 @@ module managedCluster 'br/public:avm/res/container-service/managed-cluster:' + vnetSubnetResourceId: '' } ] // Non-required parameters agentPools: [ { availabilityZones: [ - '3' + 3 ] count: 2 enableAutoScaling: true @@ -1583,11 +1583,11 @@ module managedCluster 'br/public:avm/res/container-service/managed-cluster:' + vnetSubnetResourceId: '' } { availabilityZones: [ - '3' + 3 ] count: 2 enableAutoScaling: true @@ -1638,11 +1638,11 @@ module managedCluster 'br/public:avm/res/container-service/managed-cluster:" + "vnetSubnetResourceId": "" } ] }, @@ -1667,7 +1667,7 @@ module managedCluster 'br/public:avm/res/container-service/managed-cluster:" + "vnetSubnetResourceId": "" }, { "availabilityZones": [ - "3" + 3 ], "count": 2, "enableAutoScaling": true, @@ -1752,10 +1752,10 @@ using 'br/public:avm/res/container-service/managed-cluster:' // Required parameters param name = 'csmpriv001' -param primaryAgentPoolProfile = [ +param primaryAgentPoolProfiles = [ { availabilityZones: [ - '3' + 3 ] count: 1 enableAutoScaling: true @@ -1771,14 +1771,14 @@ param primaryAgentPoolProfile = [ osType: 'Linux' type: 'VirtualMachineScaleSets' vmSize: 'Standard_DS2_v2' - vnetSubnetID: '' + vnetSubnetResourceId: '' } ] // Non-required parameters param agentPools = [ { availabilityZones: [ - '3' + 3 ] count: 2 enableAutoScaling: true @@ -1795,11 +1795,11 @@ param agentPools = [ scaleSetPriority: 'Regular' type: 'VirtualMachineScaleSets' vmSize: 'Standard_DS2_v2' - vnetSubnetID: '' + vnetSubnetResourceId: '' } { availabilityZones: [ - '3' + 3 ] count: 2 enableAutoScaling: true @@ -1850,10 +1850,10 @@ module managedCluster 'br/public:avm/res/container-service/managed-cluster:' + vnetSubnetResourceId: '' } ] // Non-required parameters agentPools: [ { availabilityZones: [ - '3' + 3 ] count: 3 enableAutoScaling: true @@ -1894,11 +1894,11 @@ module managedCluster 'br/public:avm/res/container-service/managed-cluster:' + vnetSubnetResourceId: '' } { availabilityZones: [ - '3' + 3 ] count: 3 enableAutoScaling: true @@ -1990,7 +1990,7 @@ module managedCluster 'br/public:avm/res/container-service/managed-cluster:' ] } - monitoringWorkspaceId: '' + monitoringWorkspaceResourceId: '' networkPlugin: 'azure' networkPolicy: 'azure' omsAgentEnabled: true @@ -2022,11 +2022,11 @@ module managedCluster 'br/public:avm/res/container-service/managed-cluster:" + "vnetSubnetResourceId": "" } ] }, @@ -2051,7 +2051,7 @@ module managedCluster 'br/public:avm/res/container-service/managed-cluster:" + "vnetSubnetResourceId": "" }, { "availabilityZones": [ - "3" + 3 ], "count": 3, "enableAutoScaling": true, @@ -2186,8 +2186,8 @@ module managedCluster 'br/public:avm/res/container-service/managed-cluster:" + "monitoringWorkspaceResourceId": { + "value": "" }, "networkPlugin": { "value": "azure" @@ -2230,10 +2230,10 @@ using 'br/public:avm/res/container-service/managed-cluster:' // Required parameters param name = 'cswaf001' -param primaryAgentPoolProfile = [ +param primaryAgentPoolProfiles = [ { availabilityZones: [ - '3' + 3 ] count: 3 enableAutoScaling: true @@ -2249,14 +2249,14 @@ param primaryAgentPoolProfile = [ osType: 'Linux' type: 'VirtualMachineScaleSets' vmSize: 'Standard_DS2_v2' - vnetSubnetID: '' + vnetSubnetResourceId: '' } ] // Non-required parameters param agentPools = [ { availabilityZones: [ - '3' + 3 ] count: 3 enableAutoScaling: true @@ -2274,11 +2274,11 @@ param agentPools = [ scaleSetPriority: 'Regular' type: 'VirtualMachineScaleSets' vmSize: 'Standard_DS2_v2' - vnetSubnetID: '' + vnetSubnetResourceId: '' } { availabilityZones: [ - '3' + 3 ] count: 3 enableAutoScaling: true @@ -2370,7 +2370,7 @@ param managedIdentities = { '' ] } -param monitoringWorkspaceId = '' +param monitoringWorkspaceResourceId = '' param networkPlugin = 'azure' param networkPolicy = 'azure' param omsAgentEnabled = true @@ -2394,7 +2394,7 @@ param tags = { | Parameter | Type | Description | | :-- | :-- | :-- | | [`name`](#parameter-name) | string | Specifies the name of the AKS cluster. | -| [`primaryAgentPoolProfile`](#parameter-primaryagentpoolprofile) | array | Properties of the primary agent pool. | +| [`primaryAgentPoolProfiles`](#parameter-primaryagentpoolprofiles) | array | Properties of the primary agent pool. | **Conditional parameters** @@ -2421,12 +2421,12 @@ param tags = { | [`autoNodeOsUpgradeProfileUpgradeChannel`](#parameter-autonodeosupgradeprofileupgradechannel) | string | Auto-upgrade channel on the Node Os. | | [`autoScalerProfileBalanceSimilarNodeGroups`](#parameter-autoscalerprofilebalancesimilarnodegroups) | bool | Specifies the balance of similar node groups for the auto-scaler of the AKS cluster. | | [`autoScalerProfileExpander`](#parameter-autoscalerprofileexpander) | string | Specifies the expand strategy for the auto-scaler of the AKS cluster. | -| [`autoScalerProfileMaxEmptyBulkDelete`](#parameter-autoscalerprofilemaxemptybulkdelete) | string | Specifies the maximum empty bulk delete for the auto-scaler of the AKS cluster. | -| [`autoScalerProfileMaxGracefulTerminationSec`](#parameter-autoscalerprofilemaxgracefulterminationsec) | string | Specifies the max graceful termination time interval in seconds for the auto-scaler of the AKS cluster. | +| [`autoScalerProfileMaxEmptyBulkDelete`](#parameter-autoscalerprofilemaxemptybulkdelete) | int | Specifies the maximum empty bulk delete for the auto-scaler of the AKS cluster. | +| [`autoScalerProfileMaxGracefulTerminationSec`](#parameter-autoscalerprofilemaxgracefulterminationsec) | int | Specifies the max graceful termination time interval in seconds for the auto-scaler of the AKS cluster. | | [`autoScalerProfileMaxNodeProvisionTime`](#parameter-autoscalerprofilemaxnodeprovisiontime) | string | Specifies the maximum node provisioning time for the auto-scaler of the AKS cluster. Values must be an integer followed by an "m". No unit of time other than minutes (m) is supported. | -| [`autoScalerProfileMaxTotalUnreadyPercentage`](#parameter-autoscalerprofilemaxtotalunreadypercentage) | string | Specifies the mximum total unready percentage for the auto-scaler of the AKS cluster. The maximum is 100 and the minimum is 0. | +| [`autoScalerProfileMaxTotalUnreadyPercentage`](#parameter-autoscalerprofilemaxtotalunreadypercentage) | int | Specifies the mximum total unready percentage for the auto-scaler of the AKS cluster. The maximum is 100 and the minimum is 0. | | [`autoScalerProfileNewPodScaleUpDelay`](#parameter-autoscalerprofilenewpodscaleupdelay) | string | For scenarios like burst/batch scale where you do not want CA to act before the kubernetes scheduler could schedule all the pods, you can tell CA to ignore unscheduled pods before they are a certain age. Values must be an integer followed by a unit ("s" for seconds, "m" for minutes, "h" for hours, etc). | -| [`autoScalerProfileOkTotalUnreadyCount`](#parameter-autoscalerprofileoktotalunreadycount) | string | Specifies the OK total unready count for the auto-scaler of the AKS cluster. | +| [`autoScalerProfileOkTotalUnreadyCount`](#parameter-autoscalerprofileoktotalunreadycount) | int | Specifies the OK total unready count for the auto-scaler of the AKS cluster. | | [`autoScalerProfileScaleDownDelayAfterAdd`](#parameter-autoscalerprofilescaledowndelayafteradd) | string | Specifies the scale down delay after add of the auto-scaler of the AKS cluster. | | [`autoScalerProfileScaleDownDelayAfterDelete`](#parameter-autoscalerprofilescaledowndelayafterdelete) | string | Specifies the scale down delay after delete of the auto-scaler of the AKS cluster. | | [`autoScalerProfileScaleDownDelayAfterFailure`](#parameter-autoscalerprofilescaledowndelayafterfailure) | string | Specifies scale down delay after failure of the auto-scaler of the AKS cluster. | @@ -2481,12 +2481,12 @@ param tags = { | [`loadBalancerSku`](#parameter-loadbalancersku) | string | Specifies the sku of the load balancer used by the virtual machine scale sets used by nodepools. | | [`location`](#parameter-location) | string | Specifies the location of AKS cluster. It picks up Resource Group's location by default. | | [`lock`](#parameter-lock) | object | The lock settings of the service. | -| [`maintenanceConfigurations`](#parameter-maintenanceconfigurations) | array | Maintenance Window for Cluster auto upgrade and node OS upgrade. | +| [`maintenanceConfigurations`](#parameter-maintenanceconfigurations) | array | Whether or not to use AKS Automatic mode. | | [`managedIdentities`](#parameter-managedidentities) | object | The managed identity definition for this resource. Only one type of identity is supported: system-assigned or user-assigned, but not both. | | [`managedOutboundIPCount`](#parameter-managedoutboundipcount) | int | Outbound IP Count for the Load balancer. | | [`metricAnnotationsAllowList`](#parameter-metricannotationsallowlist) | string | A comma-separated list of Kubernetes cluster metrics annotations. | | [`metricLabelsAllowlist`](#parameter-metriclabelsallowlist) | string | A comma-separated list of kubernetes cluster metrics labels. | -| [`monitoringWorkspaceId`](#parameter-monitoringworkspaceid) | string | Resource ID of the monitoring log analytics workspace. | +| [`monitoringWorkspaceResourceId`](#parameter-monitoringworkspaceresourceid) | string | Resource ID of the monitoring log analytics workspace. | | [`networkDataplane`](#parameter-networkdataplane) | string | Network dataplane used in the Kubernetes cluster. Not compatible with kubenet network plugin. | | [`networkPlugin`](#parameter-networkplugin) | string | Specifies the network plugin used for building Kubernetes network. | | [`networkPluginMode`](#parameter-networkpluginmode) | string | Network plugin mode used for building the Kubernetes network. Not compatible with kubenet network plugin. | @@ -2519,13 +2519,372 @@ Specifies the name of the AKS cluster. - Required: Yes - Type: string -### Parameter: `primaryAgentPoolProfile` +### Parameter: `primaryAgentPoolProfiles` Properties of the primary agent pool. - Required: Yes - Type: array +**Required parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`name`](#parameter-primaryagentpoolprofilesname) | string | The name of the agent pool. | + +**Optional parameters** + +| Parameter | Type | Description | +| :-- | :-- | :-- | +| [`availabilityZones`](#parameter-primaryagentpoolprofilesavailabilityzones) | array | The availability zones of the agent pool. | +| [`count`](#parameter-primaryagentpoolprofilescount) | int | The number of agents (VMs) to host docker containers. Allowed values must be in the range of 1 to 100 (inclusive). | +| [`enableAutoScaling`](#parameter-primaryagentpoolprofilesenableautoscaling) | bool | Whether to enable auto-scaling for the agent pool. | +| [`enableDefaultTelemetry`](#parameter-primaryagentpoolprofilesenabledefaulttelemetry) | bool | The enable default telemetry of the agent pool. | +| [`enableEncryptionAtHost`](#parameter-primaryagentpoolprofilesenableencryptionathost) | bool | Whether to enable encryption at host for the agent pool. | +| [`enableFIPS`](#parameter-primaryagentpoolprofilesenablefips) | bool | Whether to enable FIPS for the agent pool. | +| [`enableNodePublicIP`](#parameter-primaryagentpoolprofilesenablenodepublicip) | bool | Whether to enable node public IP for the agent pool. | +| [`enableUltraSSD`](#parameter-primaryagentpoolprofilesenableultrassd) | bool | Whether to enable Ultra SSD for the agent pool. | +| [`gpuInstanceProfile`](#parameter-primaryagentpoolprofilesgpuinstanceprofile) | string | The GPU instance profile of the agent pool. | +| [`kubeletDiskType`](#parameter-primaryagentpoolprofileskubeletdisktype) | string | The kubelet disk type of the agent pool. | +| [`maxCount`](#parameter-primaryagentpoolprofilesmaxcount) | int | The maximum number of agents (VMs) to host docker containers. Allowed values must be in the range of 1 to 100 (inclusive). | +| [`maxPods`](#parameter-primaryagentpoolprofilesmaxpods) | int | The maximum number of pods that can run on a node. | +| [`maxSurge`](#parameter-primaryagentpoolprofilesmaxsurge) | string | The maximum number of nodes that can be created during an upgrade. | +| [`minCount`](#parameter-primaryagentpoolprofilesmincount) | int | The minimum number of agents (VMs) to host docker containers. Allowed values must be in the range of 1 to 100 (inclusive). | +| [`minPods`](#parameter-primaryagentpoolprofilesminpods) | int | The minimum number of pods that can run on a node. | +| [`mode`](#parameter-primaryagentpoolprofilesmode) | string | The mode of the agent pool. | +| [`nodeLabels`](#parameter-primaryagentpoolprofilesnodelabels) | object | The node labels of the agent pool. | +| [`nodePublicIpPrefixResourceId`](#parameter-primaryagentpoolprofilesnodepublicipprefixresourceid) | string | The node public IP prefix ID of the agent pool. | +| [`nodeTaints`](#parameter-primaryagentpoolprofilesnodetaints) | array | The node taints of the agent pool. | +| [`orchestratorVersion`](#parameter-primaryagentpoolprofilesorchestratorversion) | string | The Kubernetes version of the agent pool. | +| [`osDiskSizeGB`](#parameter-primaryagentpoolprofilesosdisksizegb) | int | The OS disk size in GB of the agent pool. | +| [`osDiskType`](#parameter-primaryagentpoolprofilesosdisktype) | string | The OS disk type of the agent pool. | +| [`osSku`](#parameter-primaryagentpoolprofilesossku) | string | The OS SKU of the agent pool. | +| [`osType`](#parameter-primaryagentpoolprofilesostype) | string | The OS type of the agent pool. | +| [`podSubnetResourceId`](#parameter-primaryagentpoolprofilespodsubnetresourceid) | string | The pod subnet ID of the agent pool. | +| [`proximityPlacementGroupResourceId`](#parameter-primaryagentpoolprofilesproximityplacementgroupresourceid) | string | The proximity placement group resource ID of the agent pool. | +| [`scaleDownMode`](#parameter-primaryagentpoolprofilesscaledownmode) | string | The scale down mode of the agent pool. | +| [`scaleSetEvictionPolicy`](#parameter-primaryagentpoolprofilesscalesetevictionpolicy) | string | The scale set eviction policy of the agent pool. | +| [`scaleSetPriority`](#parameter-primaryagentpoolprofilesscalesetpriority) | string | The scale set priority of the agent pool. | +| [`sourceResourceId`](#parameter-primaryagentpoolprofilessourceresourceid) | string | The source resource ID to create the agent pool from. | +| [`spotMaxPrice`](#parameter-primaryagentpoolprofilesspotmaxprice) | int | The spot max price of the agent pool. | +| [`tags`](#parameter-primaryagentpoolprofilestags) | object | The tags of the agent pool. | +| [`type`](#parameter-primaryagentpoolprofilestype) | string | The type of the agent pool. | +| [`vmSize`](#parameter-primaryagentpoolprofilesvmsize) | string | The VM size of the agent pool. | +| [`vnetSubnetResourceId`](#parameter-primaryagentpoolprofilesvnetsubnetresourceid) | string | The VNet subnet ID of the agent pool. | +| [`workloadRuntime`](#parameter-primaryagentpoolprofilesworkloadruntime) | string | The workload runtime of the agent pool. | + +### Parameter: `primaryAgentPoolProfiles.name` + +The name of the agent pool. + +- Required: Yes +- Type: string + +### Parameter: `primaryAgentPoolProfiles.availabilityZones` + +The availability zones of the agent pool. + +- Required: No +- Type: array + +### Parameter: `primaryAgentPoolProfiles.count` + +The number of agents (VMs) to host docker containers. Allowed values must be in the range of 1 to 100 (inclusive). + +- Required: No +- Type: int + +### Parameter: `primaryAgentPoolProfiles.enableAutoScaling` + +Whether to enable auto-scaling for the agent pool. + +- Required: No +- Type: bool + +### Parameter: `primaryAgentPoolProfiles.enableDefaultTelemetry` + +The enable default telemetry of the agent pool. + +- Required: No +- Type: bool + +### Parameter: `primaryAgentPoolProfiles.enableEncryptionAtHost` + +Whether to enable encryption at host for the agent pool. + +- Required: No +- Type: bool + +### Parameter: `primaryAgentPoolProfiles.enableFIPS` + +Whether to enable FIPS for the agent pool. + +- Required: No +- Type: bool + +### Parameter: `primaryAgentPoolProfiles.enableNodePublicIP` + +Whether to enable node public IP for the agent pool. + +- Required: No +- Type: bool + +### Parameter: `primaryAgentPoolProfiles.enableUltraSSD` + +Whether to enable Ultra SSD for the agent pool. + +- Required: No +- Type: bool + +### Parameter: `primaryAgentPoolProfiles.gpuInstanceProfile` + +The GPU instance profile of the agent pool. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'MIG1g' + 'MIG2g' + 'MIG3g' + 'MIG4g' + 'MIG7g' + ] + ``` + +### Parameter: `primaryAgentPoolProfiles.kubeletDiskType` + +The kubelet disk type of the agent pool. + +- Required: No +- Type: string + +### Parameter: `primaryAgentPoolProfiles.maxCount` + +The maximum number of agents (VMs) to host docker containers. Allowed values must be in the range of 1 to 100 (inclusive). + +- Required: No +- Type: int + +### Parameter: `primaryAgentPoolProfiles.maxPods` + +The maximum number of pods that can run on a node. + +- Required: No +- Type: int + +### Parameter: `primaryAgentPoolProfiles.maxSurge` + +The maximum number of nodes that can be created during an upgrade. + +- Required: No +- Type: string + +### Parameter: `primaryAgentPoolProfiles.minCount` + +The minimum number of agents (VMs) to host docker containers. Allowed values must be in the range of 1 to 100 (inclusive). + +- Required: No +- Type: int + +### Parameter: `primaryAgentPoolProfiles.minPods` + +The minimum number of pods that can run on a node. + +- Required: No +- Type: int + +### Parameter: `primaryAgentPoolProfiles.mode` + +The mode of the agent pool. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'System' + 'User' + ] + ``` + +### Parameter: `primaryAgentPoolProfiles.nodeLabels` + +The node labels of the agent pool. + +- Required: No +- Type: object + +### Parameter: `primaryAgentPoolProfiles.nodePublicIpPrefixResourceId` + +The node public IP prefix ID of the agent pool. + +- Required: No +- Type: string + +### Parameter: `primaryAgentPoolProfiles.nodeTaints` + +The node taints of the agent pool. + +- Required: No +- Type: array + +### Parameter: `primaryAgentPoolProfiles.orchestratorVersion` + +The Kubernetes version of the agent pool. + +- Required: No +- Type: string + +### Parameter: `primaryAgentPoolProfiles.osDiskSizeGB` + +The OS disk size in GB of the agent pool. + +- Required: No +- Type: int + +### Parameter: `primaryAgentPoolProfiles.osDiskType` + +The OS disk type of the agent pool. + +- Required: No +- Type: string + +### Parameter: `primaryAgentPoolProfiles.osSku` + +The OS SKU of the agent pool. + +- Required: No +- Type: string + +### Parameter: `primaryAgentPoolProfiles.osType` + +The OS type of the agent pool. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'Linux' + 'Windows' + ] + ``` + +### Parameter: `primaryAgentPoolProfiles.podSubnetResourceId` + +The pod subnet ID of the agent pool. + +- Required: No +- Type: string + +### Parameter: `primaryAgentPoolProfiles.proximityPlacementGroupResourceId` + +The proximity placement group resource ID of the agent pool. + +- Required: No +- Type: string + +### Parameter: `primaryAgentPoolProfiles.scaleDownMode` + +The scale down mode of the agent pool. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'Deallocate' + 'Delete' + ] + ``` + +### Parameter: `primaryAgentPoolProfiles.scaleSetEvictionPolicy` + +The scale set eviction policy of the agent pool. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'Deallocate' + 'Delete' + ] + ``` + +### Parameter: `primaryAgentPoolProfiles.scaleSetPriority` + +The scale set priority of the agent pool. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'Low' + 'Regular' + 'Spot' + ] + ``` + +### Parameter: `primaryAgentPoolProfiles.sourceResourceId` + +The source resource ID to create the agent pool from. + +- Required: No +- Type: string + +### Parameter: `primaryAgentPoolProfiles.spotMaxPrice` + +The spot max price of the agent pool. + +- Required: No +- Type: int + +### Parameter: `primaryAgentPoolProfiles.tags` + +The tags of the agent pool. + +- Required: No +- Type: object + +### Parameter: `primaryAgentPoolProfiles.type` + +The type of the agent pool. + +- Required: No +- Type: string +- Allowed: + ```Bicep + [ + 'AvailabilitySet' + 'VirtualMachineScaleSets' + ] + ``` + +### Parameter: `primaryAgentPoolProfiles.vmSize` + +The VM size of the agent pool. + +- Required: No +- Type: string + +### Parameter: `primaryAgentPoolProfiles.vnetSubnetResourceId` + +The VNet subnet ID of the agent pool. + +- Required: No +- Type: string + +### Parameter: `primaryAgentPoolProfiles.workloadRuntime` + +The workload runtime of the agent pool. + +- Required: No +- Type: string + ### Parameter: `aksServicePrincipalProfile` Information about a service principal identity for the cluster to use for manipulating Azure APIs. Required if no managed identities are assigned to the cluster. @@ -2642,14 +3001,14 @@ Define one or more secondary/additional agent pools. | [`minPods`](#parameter-agentpoolsminpods) | int | The minimum number of pods that can run on a node. | | [`mode`](#parameter-agentpoolsmode) | string | The mode of the agent pool. | | [`nodeLabels`](#parameter-agentpoolsnodelabels) | object | The node labels of the agent pool. | -| [`nodePublicIpPrefixId`](#parameter-agentpoolsnodepublicipprefixid) | string | The node public IP prefix ID of the agent pool. | +| [`nodePublicIpPrefixResourceId`](#parameter-agentpoolsnodepublicipprefixresourceid) | string | The node public IP prefix ID of the agent pool. | | [`nodeTaints`](#parameter-agentpoolsnodetaints) | array | The node taints of the agent pool. | | [`orchestratorVersion`](#parameter-agentpoolsorchestratorversion) | string | The Kubernetes version of the agent pool. | | [`osDiskSizeGB`](#parameter-agentpoolsosdisksizegb) | int | The OS disk size in GB of the agent pool. | | [`osDiskType`](#parameter-agentpoolsosdisktype) | string | The OS disk type of the agent pool. | | [`osSku`](#parameter-agentpoolsossku) | string | The OS SKU of the agent pool. | | [`osType`](#parameter-agentpoolsostype) | string | The OS type of the agent pool. | -| [`podSubnetId`](#parameter-agentpoolspodsubnetid) | string | The pod subnet ID of the agent pool. | +| [`podSubnetResourceId`](#parameter-agentpoolspodsubnetresourceid) | string | The pod subnet ID of the agent pool. | | [`proximityPlacementGroupResourceId`](#parameter-agentpoolsproximityplacementgroupresourceid) | string | The proximity placement group resource ID of the agent pool. | | [`scaleDownMode`](#parameter-agentpoolsscaledownmode) | string | The scale down mode of the agent pool. | | [`scaleSetEvictionPolicy`](#parameter-agentpoolsscalesetevictionpolicy) | string | The scale set eviction policy of the agent pool. | @@ -2659,14 +3018,14 @@ Define one or more secondary/additional agent pools. | [`tags`](#parameter-agentpoolstags) | object | The tags of the agent pool. | | [`type`](#parameter-agentpoolstype) | string | The type of the agent pool. | | [`vmSize`](#parameter-agentpoolsvmsize) | string | The VM size of the agent pool. | -| [`vnetSubnetID`](#parameter-agentpoolsvnetsubnetid) | string | The VNet subnet ID of the agent pool. | +| [`vnetSubnetResourceId`](#parameter-agentpoolsvnetsubnetresourceid) | string | The VNet subnet ID of the agent pool. | | [`workloadRuntime`](#parameter-agentpoolsworkloadruntime) | string | The workload runtime of the agent pool. | ### Parameter: `agentPools.name` The name of the agent pool. -- Required: No +- Required: Yes - Type: string ### Parameter: `agentPools.availabilityZones` @@ -2805,7 +3164,7 @@ The node labels of the agent pool. - Required: No - Type: object -### Parameter: `agentPools.nodePublicIpPrefixId` +### Parameter: `agentPools.nodePublicIpPrefixResourceId` The node public IP prefix ID of the agent pool. @@ -2861,7 +3220,7 @@ The OS type of the agent pool. ] ``` -### Parameter: `agentPools.podSubnetId` +### Parameter: `agentPools.podSubnetResourceId` The pod subnet ID of the agent pool. @@ -2960,7 +3319,7 @@ The VM size of the agent pool. - Required: No - Type: string -### Parameter: `agentPools.vnetSubnetID` +### Parameter: `agentPools.vnetSubnetResourceId` The VNet subnet ID of the agent pool. @@ -3028,16 +3387,16 @@ Specifies the expand strategy for the auto-scaler of the AKS cluster. Specifies the maximum empty bulk delete for the auto-scaler of the AKS cluster. - Required: No -- Type: string -- Default: `'10'` +- Type: int +- Default: `10` ### Parameter: `autoScalerProfileMaxGracefulTerminationSec` Specifies the max graceful termination time interval in seconds for the auto-scaler of the AKS cluster. - Required: No -- Type: string -- Default: `'600'` +- Type: int +- Default: `600` ### Parameter: `autoScalerProfileMaxNodeProvisionTime` @@ -3052,8 +3411,8 @@ Specifies the maximum node provisioning time for the auto-scaler of the AKS clus Specifies the mximum total unready percentage for the auto-scaler of the AKS cluster. The maximum is 100 and the minimum is 0. - Required: No -- Type: string -- Default: `'45'` +- Type: int +- Default: `45` ### Parameter: `autoScalerProfileNewPodScaleUpDelay` @@ -3068,8 +3427,8 @@ For scenarios like burst/batch scale where you do not want CA to act before the Specifies the OK total unready count for the auto-scaler of the AKS cluster. - Required: No -- Type: string -- Default: `'3'` +- Type: int +- Default: `3` ### Parameter: `autoScalerProfileScaleDownDelayAfterAdd` @@ -3416,7 +3775,7 @@ If set to true, getting static credentials will be disabled for this cluster. Th - Required: No - Type: bool -- Default: `False` +- Default: `True` ### Parameter: `disablePrometheusMetricsScraping` @@ -3823,7 +4182,7 @@ Specify the name of lock. ### Parameter: `maintenanceConfigurations` -Maintenance Window for Cluster auto upgrade and node OS upgrade. +Whether or not to use AKS Automatic mode. - Required: No - Type: array @@ -3908,7 +4267,7 @@ A comma-separated list of kubernetes cluster metrics labels. - Type: string - Default: `''` -### Parameter: `monitoringWorkspaceId` +### Parameter: `monitoringWorkspaceResourceId` Resource ID of the monitoring log analytics workspace. diff --git a/avm/res/container-service/managed-cluster/agent-pool/README.md b/avm/res/container-service/managed-cluster/agent-pool/README.md index b83fb64af6..67ac41ab37 100644 --- a/avm/res/container-service/managed-cluster/agent-pool/README.md +++ b/avm/res/container-service/managed-cluster/agent-pool/README.md @@ -47,14 +47,14 @@ This module deploys an Azure Kubernetes Service (AKS) Managed Cluster Agent Pool | [`minCount`](#parameter-mincount) | int | The minimum number of nodes for auto-scaling. | | [`mode`](#parameter-mode) | string | A cluster must have at least one "System" Agent Pool at all times. For additional information on agent pool restrictions and best practices, see: /azure/aks/use-system-pools. | | [`nodeLabels`](#parameter-nodelabels) | object | The node labels to be persisted across all nodes in agent pool. | -| [`nodePublicIpPrefixId`](#parameter-nodepublicipprefixid) | string | ResourceId of the node PublicIPPrefix. | +| [`nodePublicIpPrefixResourceId`](#parameter-nodepublicipprefixresourceid) | string | ResourceId of the node PublicIPPrefix. | | [`nodeTaints`](#parameter-nodetaints) | array | The taints added to new nodes during node pool create and scale. For example, key=value:NoSchedule. | | [`orchestratorVersion`](#parameter-orchestratorversion) | string | As a best practice, you should upgrade all node pools in an AKS cluster to the same Kubernetes version. The node pool version must have the same major version as the control plane. The node pool minor version must be within two minor versions of the control plane version. The node pool version cannot be greater than the control plane version. For more information see upgrading a node pool (https://learn.microsoft.com/en-us/azure/aks/use-multiple-node-pools#upgrade-a-node-pool). | | [`osDiskSizeGB`](#parameter-osdisksizegb) | int | OS Disk Size in GB to be used to specify the disk size for every machine in the master/agent pool. If you specify 0, it will apply the default osDisk size according to the vmSize specified. | | [`osDiskType`](#parameter-osdisktype) | string | The default is "Ephemeral" if the VM supports it and has a cache disk larger than the requested OSDiskSizeGB. Otherwise, defaults to "Managed". May not be changed after creation. For more information see Ephemeral OS (https://learn.microsoft.com/en-us/azure/aks/cluster-configuration#ephemeral-os). | | [`osSku`](#parameter-ossku) | string | Specifies the OS SKU used by the agent pool. The default is Ubuntu if OSType is Linux. The default is Windows2019 when Kubernetes <= 1.24 or Windows2022 when Kubernetes >= 1.25 if OSType is Windows. | | [`osType`](#parameter-ostype) | string | The operating system type. The default is Linux. | -| [`podSubnetId`](#parameter-podsubnetid) | string | Subnet ID for the pod IPs. If omitted, pod IPs are statically assigned on the node subnet (see vnetSubnetID for more details). This is of the form: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName}. | +| [`podSubnetResourceId`](#parameter-podsubnetresourceid) | string | Subnet resource ID for the pod IPs. If omitted, pod IPs are statically assigned on the node subnet (see vnetSubnetID for more details). This is of the form: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName}. | | [`proximityPlacementGroupResourceId`](#parameter-proximityplacementgroupresourceid) | string | The ID for the Proximity Placement Group. | | [`scaleDownMode`](#parameter-scaledownmode) | string | Describes how VMs are added to or removed from Agent Pools. See [billing states](https://learn.microsoft.com/en-us/azure/virtual-machines/states-billing). | | [`scaleSetEvictionPolicy`](#parameter-scalesetevictionpolicy) | string | The eviction policy specifies what to do with the VM when it is evicted. The default is Delete. For more information about eviction see spot VMs. | @@ -64,7 +64,7 @@ This module deploys an Azure Kubernetes Service (AKS) Managed Cluster Agent Pool | [`tags`](#parameter-tags) | object | Tags of the resource. | | [`type`](#parameter-type) | string | The type of Agent Pool. | | [`vmSize`](#parameter-vmsize) | string | VM size. VM size availability varies by region. If a node contains insufficient compute resources (memory, cpu, etc) pods might fail to run correctly. For more details on restricted VM sizes, see: /azure/aks/quotas-skus-regions. | -| [`vnetSubnetId`](#parameter-vnetsubnetid) | string | Node Subnet ID. If this is not specified, a VNET and subnet will be generated and used. If no podSubnetID is specified, this applies to nodes and pods, otherwise it applies to just nodes. This is of the form: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName}. | +| [`vnetSubnetResourceId`](#parameter-vnetsubnetresourceid) | string | Node Subnet ID. If this is not specified, a VNET and subnet will be generated and used. If no podSubnetID is specified, this applies to nodes and pods, otherwise it applies to just nodes. This is of the form: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName}. | | [`workloadRuntime`](#parameter-workloadruntime) | string | Determines the type of workload a node can run. | ### Parameter: `name` @@ -87,6 +87,14 @@ The list of Availability zones to use for nodes. This can only be specified if t - Required: No - Type: array +- Default: + ```Bicep + [ + 1 + 2 + 3 + ] + ``` ### Parameter: `count` @@ -202,7 +210,7 @@ The node labels to be persisted across all nodes in agent pool. - Required: No - Type: object -### Parameter: `nodePublicIpPrefixId` +### Parameter: `nodePublicIpPrefixResourceId` ResourceId of the node PublicIPPrefix. @@ -276,9 +284,9 @@ The operating system type. The default is Linux. ] ``` -### Parameter: `podSubnetId` +### Parameter: `podSubnetResourceId` -Subnet ID for the pod IPs. If omitted, pod IPs are statically assigned on the node subnet (see vnetSubnetID for more details). This is of the form: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName}. +Subnet resource ID for the pod IPs. If omitted, pod IPs are statically assigned on the node subnet (see vnetSubnetID for more details). This is of the form: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName}. - Required: No - Type: string @@ -370,7 +378,7 @@ VM size. VM size availability varies by region. If a node contains insufficient - Type: string - Default: `'Standard_D2s_v3'` -### Parameter: `vnetSubnetId` +### Parameter: `vnetSubnetResourceId` Node Subnet ID. If this is not specified, a VNET and subnet will be generated and used. If no podSubnetID is specified, this applies to nodes and pods, otherwise it applies to just nodes. This is of the form: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName}. diff --git a/avm/res/container-service/managed-cluster/agent-pool/main.bicep b/avm/res/container-service/managed-cluster/agent-pool/main.bicep index 0a79a28c61..e5ae30e13d 100644 --- a/avm/res/container-service/managed-cluster/agent-pool/main.bicep +++ b/avm/res/container-service/managed-cluster/agent-pool/main.bicep @@ -9,7 +9,7 @@ param managedClusterName string param name string @description('Optional. The list of Availability zones to use for nodes. This can only be specified if the AgentPoolType property is "VirtualMachineScaleSets".') -param availabilityZones array? +param availabilityZones int[] = [1, 2, 3] @description('Optional. Desired Number of agents (VMs) specified to host docker containers. Allowed values must be in the range of 0 to 1000 (inclusive) for user pools and in the range of 1 to 1000 (inclusive) for system pools. The default value is 1.') @minValue(0) @@ -63,7 +63,7 @@ param mode string? param nodeLabels object? @description('Optional. ResourceId of the node PublicIPPrefix.') -param nodePublicIpPrefixId string? +param nodePublicIpPrefixResourceId string? @description('Optional. The taints added to new nodes during node pool create and scale. For example, key=value:NoSchedule.') param nodeTaints array? @@ -98,8 +98,8 @@ param osSku string? ]) param osType string = 'Linux' -@description('Optional. Subnet ID for the pod IPs. If omitted, pod IPs are statically assigned on the node subnet (see vnetSubnetID for more details). This is of the form: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName}.') -param podSubnetId string? +@description('Optional. Subnet resource ID for the pod IPs. If omitted, pod IPs are statically assigned on the node subnet (see vnetSubnetID for more details). This is of the form: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName}.') +param podSubnetResourceId string? @description('Optional. The ID for the Proximity Placement Group.') param proximityPlacementGroupResourceId string? @@ -141,7 +141,7 @@ param maxSurge string? param vmSize string = 'Standard_D2s_v3' @description('Optional. Node Subnet ID. If this is not specified, a VNET and subnet will be generated and used. If no podSubnetID is specified, this applies to nodes and pods, otherwise it applies to just nodes. This is of the form: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName}.') -param vnetSubnetId string? +param vnetSubnetResourceId string? @description('Optional. Determines the type of workload a node can run.') param workloadRuntime string? @@ -154,7 +154,7 @@ resource agentPool 'Microsoft.ContainerService/managedClusters/agentPools@2023-0 name: name parent: managedCluster properties: { - availabilityZones: availabilityZones + availabilityZones: map(availabilityZones ?? [], zone => '${zone}') count: count creationData: !empty(sourceResourceId) ? { @@ -173,14 +173,14 @@ resource agentPool 'Microsoft.ContainerService/managedClusters/agentPools@2023-0 minCount: minCount mode: mode nodeLabels: nodeLabels - nodePublicIPPrefixID: nodePublicIpPrefixId + nodePublicIPPrefixID: nodePublicIpPrefixResourceId nodeTaints: nodeTaints orchestratorVersion: orchestratorVersion osDiskSizeGB: osDiskSizeGB osDiskType: osDiskType osSKU: osSku osType: osType - podSubnetID: podSubnetId + podSubnetID: podSubnetResourceId proximityPlacementGroupID: proximityPlacementGroupResourceId scaleDownMode: scaleDownMode scaleSetEvictionPolicy: scaleSetEvictionPolicy @@ -192,7 +192,7 @@ resource agentPool 'Microsoft.ContainerService/managedClusters/agentPools@2023-0 maxSurge: maxSurge } vmSize: vmSize - vnetSubnetID: vnetSubnetId + vnetSubnetID: vnetSubnetResourceId workloadRuntime: workloadRuntime } } diff --git a/avm/res/container-service/managed-cluster/agent-pool/main.json b/avm/res/container-service/managed-cluster/agent-pool/main.json index d8141c2a61..65a21588ad 100644 --- a/avm/res/container-service/managed-cluster/agent-pool/main.json +++ b/avm/res/container-service/managed-cluster/agent-pool/main.json @@ -5,8 +5,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.29.47.4906", - "templateHash": "2004205618690542488" + "version": "0.30.23.60470", + "templateHash": "13856766172443517827" }, "name": "Azure Kubernetes Service (AKS) Managed Cluster Agent Pools", "description": "This module deploys an Azure Kubernetes Service (AKS) Managed Cluster Agent Pool.", @@ -27,7 +27,14 @@ }, "availabilityZones": { "type": "array", - "nullable": true, + "items": { + "type": "int" + }, + "defaultValue": [ + 1, + 2, + 3 + ], "metadata": { "description": "Optional. The list of Availability zones to use for nodes. This can only be specified if the AgentPoolType property is \"VirtualMachineScaleSets\"." } @@ -139,7 +146,7 @@ "description": "Optional. The node labels to be persisted across all nodes in agent pool." } }, - "nodePublicIpPrefixId": { + "nodePublicIpPrefixResourceId": { "type": "string", "nullable": true, "metadata": { @@ -203,11 +210,11 @@ "description": "Optional. The operating system type. The default is Linux." } }, - "podSubnetId": { + "podSubnetResourceId": { "type": "string", "nullable": true, "metadata": { - "description": "Optional. Subnet ID for the pod IPs. If omitted, pod IPs are statically assigned on the node subnet (see vnetSubnetID for more details). This is of the form: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName}." + "description": "Optional. Subnet resource ID for the pod IPs. If omitted, pod IPs are statically assigned on the node subnet (see vnetSubnetID for more details). This is of the form: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName}." } }, "proximityPlacementGroupResourceId": { @@ -285,7 +292,7 @@ "description": "Optional. VM size. VM size availability varies by region. If a node contains insufficient compute resources (memory, cpu, etc) pods might fail to run correctly. For more details on restricted VM sizes, see: /azure/aks/quotas-skus-regions." } }, - "vnetSubnetId": { + "vnetSubnetResourceId": { "type": "string", "nullable": true, "metadata": { @@ -312,7 +319,7 @@ "apiVersion": "2023-07-02-preview", "name": "[format('{0}/{1}', parameters('managedClusterName'), parameters('name'))]", "properties": { - "availabilityZones": "[parameters('availabilityZones')]", + "availabilityZones": "[map(coalesce(parameters('availabilityZones'), createArray()), lambda('zone', format('{0}', lambdaVariables('zone'))))]", "count": "[parameters('count')]", "creationData": "[if(not(empty(parameters('sourceResourceId'))), createObject('sourceResourceId', parameters('sourceResourceId')), null())]", "enableAutoScaling": "[parameters('enableAutoScaling')]", @@ -327,14 +334,14 @@ "minCount": "[parameters('minCount')]", "mode": "[parameters('mode')]", "nodeLabels": "[parameters('nodeLabels')]", - "nodePublicIPPrefixID": "[parameters('nodePublicIpPrefixId')]", + "nodePublicIPPrefixID": "[parameters('nodePublicIpPrefixResourceId')]", "nodeTaints": "[parameters('nodeTaints')]", "orchestratorVersion": "[parameters('orchestratorVersion')]", "osDiskSizeGB": "[parameters('osDiskSizeGB')]", "osDiskType": "[parameters('osDiskType')]", "osSKU": "[parameters('osSku')]", "osType": "[parameters('osType')]", - "podSubnetID": "[parameters('podSubnetId')]", + "podSubnetID": "[parameters('podSubnetResourceId')]", "proximityPlacementGroupID": "[parameters('proximityPlacementGroupResourceId')]", "scaleDownMode": "[parameters('scaleDownMode')]", "scaleSetEvictionPolicy": "[parameters('scaleSetEvictionPolicy')]", @@ -346,7 +353,7 @@ "maxSurge": "[parameters('maxSurge')]" }, "vmSize": "[parameters('vmSize')]", - "vnetSubnetID": "[parameters('vnetSubnetId')]", + "vnetSubnetID": "[parameters('vnetSubnetResourceId')]", "workloadRuntime": "[parameters('workloadRuntime')]" }, "dependsOn": [ diff --git a/avm/res/container-service/managed-cluster/main.bicep b/avm/res/container-service/managed-cluster/main.bicep index 555b07cab8..88a8d6a89b 100644 --- a/avm/res/container-service/managed-cluster/main.bicep +++ b/avm/res/container-service/managed-cluster/main.bicep @@ -12,7 +12,7 @@ param location string = resourceGroup().location param dnsPrefix string = name @description('Optional. The managed identity definition for this resource. Only one type of identity is supported: system-assigned or user-assigned, but not both.') -param managedIdentities managedIdentitiesType +param managedIdentities managedIdentitiesType? @description('Optional. Network dataplane used in the Kubernetes cluster. Not compatible with kubenet network plugin.') @allowed([ @@ -110,7 +110,7 @@ param aadProfileServerAppSecret string? param aadProfileTenantId string = subscription().tenantId @description('Optional. Specifies the AAD group object IDs that will have admin role of the cluster.') -param aadProfileAdminGroupObjectIDs array? +param aadProfileAdminGroupObjectIDs string[]? @description('Optional. Specifies whether to enable managed AAD integration.') param aadProfileManaged bool = true @@ -122,13 +122,13 @@ param enableRBAC bool = true param aadProfileEnableAzureRBAC bool = enableRBAC @description('Optional. If set to true, getting static credentials will be disabled for this cluster. This must only be used on Managed Clusters that are AAD enabled.') -param disableLocalAccounts bool = false +param disableLocalAccounts bool = true @description('Optional. Name of the resource group containing agent pool nodes.') param nodeResourceGroup string = '${resourceGroup().name}_aks_${name}_nodes' @description('Optional. IP ranges are specified in CIDR format, e.g. 137.117.106.88/29. This feature is not compatible with clusters that use Public IP Per Node, or clusters that are using a Basic Load Balancer.') -param authorizedIPRanges array? +param authorizedIPRanges string[]? @description('Optional. Whether to disable run command for the cluster or not.') param disableRunCommand bool = false @@ -151,13 +151,13 @@ param enablePrivateClusterPublicFQDN bool = false param privateDNSZone string? @description('Required. Properties of the primary agent pool.') -param primaryAgentPoolProfile array +param primaryAgentPoolProfiles agentPoolType[] @description('Optional. Define one or more secondary/additional agent pools.') -param agentPools agentPoolType +param agentPools agentPoolType[]? -@description('Optional. Maintenance Window for Cluster auto upgrade and node OS upgrade.') -param maintenanceConfigurations maintenanceConfigurationType +@description('Optional. Whether or not to use AKS Automatic mode.') +param maintenanceConfigurations maintenanceConfigurationType[]? @description('Optional. Specifies whether the cost analysis add-on is enabled or not. If Enabled `enableStorageProfileDiskCSIDriver` is set to true as it is needed.') param costAnalysisEnabled bool = false @@ -225,7 +225,7 @@ param autoScalerProfileScaleDownUnreadyTime string = '20m' param autoScalerProfileUtilizationThreshold string = '0.5' @description('Optional. Specifies the max graceful termination time interval in seconds for the auto-scaler of the AKS cluster.') -param autoScalerProfileMaxGracefulTerminationSec string = '600' +param autoScalerProfileMaxGracefulTerminationSec int = 600 @description('Optional. Specifies the balance of similar node groups for the auto-scaler of the AKS cluster.') param autoScalerProfileBalanceSimilarNodeGroups bool = false @@ -240,19 +240,19 @@ param autoScalerProfileBalanceSimilarNodeGroups bool = false param autoScalerProfileExpander string = 'random' @description('Optional. Specifies the maximum empty bulk delete for the auto-scaler of the AKS cluster.') -param autoScalerProfileMaxEmptyBulkDelete string = '10' +param autoScalerProfileMaxEmptyBulkDelete int = 10 @description('Optional. Specifies the maximum node provisioning time for the auto-scaler of the AKS cluster. Values must be an integer followed by an "m". No unit of time other than minutes (m) is supported.') param autoScalerProfileMaxNodeProvisionTime string = '15m' @description('Optional. Specifies the mximum total unready percentage for the auto-scaler of the AKS cluster. The maximum is 100 and the minimum is 0.') -param autoScalerProfileMaxTotalUnreadyPercentage string = '45' +param autoScalerProfileMaxTotalUnreadyPercentage int = 45 @description('Optional. For scenarios like burst/batch scale where you do not want CA to act before the kubernetes scheduler could schedule all the pods, you can tell CA to ignore unscheduled pods before they are a certain age. Values must be an integer followed by a unit ("s" for seconds, "m" for minutes, "h" for hours, etc).') param autoScalerProfileNewPodScaleUpDelay string = '0s' @description('Optional. Specifies the OK total unready count for the auto-scaler of the AKS cluster.') -param autoScalerProfileOkTotalUnreadyCount string = '3' +param autoScalerProfileOkTotalUnreadyCount int = 3 @description('Optional. Specifies if nodes with local storage should be skipped for the auto-scaler of the AKS cluster.') param autoScalerProfileSkipNodesWithLocalStorage bool = true @@ -336,7 +336,7 @@ param diagnosticSettings diagnosticSettingType param omsAgentEnabled bool = true @description('Optional. Resource ID of the monitoring log analytics workspace.') -param monitoringWorkspaceId string? +param monitoringWorkspaceResourceId string? @description('Optional. Enable/Disable usage telemetry for module.') param enableTelemetry bool = true @@ -354,7 +354,7 @@ param tags object? param diskEncryptionSetResourceId string? @description('Optional. Settings and configurations for the flux extension.') -param fluxExtension extensionType +param fluxExtension extensionType? @description('Optional. Configurations for provisioning the cluster with HTTP proxy servers.') param httpProxyConfig object? @@ -369,7 +369,7 @@ param kedaAddon bool = false param vpaAddon bool = false @description('Optional. The customer managed key definition.') -param customerManagedKey customerManagedKeyType +param customerManagedKey customerManagedKeyType? @description('Optional. Whether the metric state of the kubenetes cluster is enabled.') param enableAzureMonitorProfileMetrics bool = false @@ -541,12 +541,58 @@ resource managedCluster 'Microsoft.ContainerService/managedClusters@2024-03-02-p tier: skuTier } properties: { + agentPoolProfiles: map(primaryAgentPoolProfiles, profile => { + name: profile.name + count: profile.count ?? 1 + availabilityZones: map(profile.?availabilityZones ?? [1, 2, 3], zone => '${zone}') + creationData: !empty(profile.?sourceResourceId) + ? { + #disable-next-line use-resource-id-functions // Not possible to reference as nested + sourceResourceId: profile.sourceResourceId + } + : null + enableAutoScaling: profile.?enableAutoScaling ?? false + enableEncryptionAtHost: profile.?enableEncryptionAtHost ?? false + enableFIPS: profile.?enableFIPS ?? false + enableNodePublicIP: profile.?enableNodePublicIP ?? false + enableUltraSSD: profile.?enableUltraSSD ?? false + gpuInstanceProfile: profile.?gpuInstanceProfile + kubeletDiskType: profile.?kubeletDiskType + maxCount: profile.?maxCount + maxPods: profile.?maxPods + minCount: profile.?minCount + mode: profile.?mode + nodeLabels: profile.?nodeLabels + #disable-next-line use-resource-id-functions // Not possible to reference as nested + nodePublicIPPrefixID: profile.?nodePublicIpPrefixResourceId + nodeTaints: profile.?nodeTaints + orchestratorVersion: profile.?orchestratorVersion + osDiskSizeGB: profile.?osDiskSizeGB + osDiskType: profile.?osDiskType + osType: profile.?osType ?? 'Linux' + #disable-next-line use-resource-id-functions // Not possible to reference as nested + podSubnetID: profile.?podSubnetResourceId + #disable-next-line use-resource-id-functions // Not possible to reference as nested + proximityPlacementGroupID: profile.?proximityPlacementGroupResourceId + scaleDownMode: profile.?scaleDownMode ?? 'Delete' + scaleSetEvictionPolicy: profile.?scaleSetEvictionPolicy ?? 'Delete' + scaleSetPriority: profile.?scaleSetPriority + spotMaxPrice: profile.?spotMaxPrice + tags: profile.?tags + type: profile.?type + upgradeSettings: { + maxSurge: profile.?maxSurge + } + vmSize: profile.?vmSize ?? 'Standard_D2s_v3' + #disable-next-line use-resource-id-functions // Not possible to reference as nested + vnetSubnetID: profile.?vnetSubnetResourceId + workloadRuntime: profile.?workloadRuntime + }) httpProxyConfig: httpProxyConfig identityProfile: identityProfile diskEncryptionSetID: diskEncryptionSetResourceId kubernetesVersion: kubernetesVersion dnsPrefix: dnsPrefix - agentPoolProfiles: primaryAgentPoolProfile linuxProfile: !empty(sshPublicKey) ? { adminUsername: adminUsername @@ -590,11 +636,10 @@ resource managedCluster 'Microsoft.ContainerService/managedClusters@2024-03-02-p : null } omsagent: { - enabled: omsAgentEnabled && !empty(monitoringWorkspaceId) - #disable-next-line BCP321 // Value will not be used if null or empty - config: omsAgentEnabled && !empty(monitoringWorkspaceId) + enabled: omsAgentEnabled && !empty(monitoringWorkspaceResourceId) + config: omsAgentEnabled && !empty(monitoringWorkspaceResourceId) ? { - logAnalyticsWorkspaceResourceID: monitoringWorkspaceId + logAnalyticsWorkspaceResourceID: monitoringWorkspaceResourceId! } : null } @@ -675,12 +720,12 @@ resource managedCluster 'Microsoft.ContainerService/managedClusters@2024-03-02-p autoScalerProfile: { 'balance-similar-node-groups': toLower(string(autoScalerProfileBalanceSimilarNodeGroups)) expander: autoScalerProfileExpander - 'max-empty-bulk-delete': autoScalerProfileMaxEmptyBulkDelete - 'max-graceful-termination-sec': autoScalerProfileMaxGracefulTerminationSec + 'max-empty-bulk-delete': '${autoScalerProfileMaxEmptyBulkDelete}' + 'max-graceful-termination-sec': '${autoScalerProfileMaxGracefulTerminationSec}' 'max-node-provision-time': autoScalerProfileMaxNodeProvisionTime - 'max-total-unready-percentage': autoScalerProfileMaxTotalUnreadyPercentage + 'max-total-unready-percentage': '${autoScalerProfileMaxTotalUnreadyPercentage}' 'new-pod-scale-up-delay': autoScalerProfileNewPodScaleUpDelay - 'ok-total-unready-count': autoScalerProfileOkTotalUnreadyCount + 'ok-total-unready-count': '${autoScalerProfileOkTotalUnreadyCount}' 'scale-down-delay-after-add': autoScalerProfileScaleDownDelayAfterAdd 'scale-down-delay-after-delete': autoScalerProfileScaleDownDelayAfterDelete 'scale-down-delay-after-failure': autoScalerProfileScaleDownDelayAfterFailure @@ -706,7 +751,9 @@ resource managedCluster 'Microsoft.ContainerService/managedClusters@2024-03-02-p containerInsights: enableContainerInsights ? { enabled: enableContainerInsights - logAnalyticsWorkspaceResourceId: !empty(monitoringWorkspaceId) ? monitoringWorkspaceId : null + logAnalyticsWorkspaceResourceId: !empty(monitoringWorkspaceResourceId) + ? monitoringWorkspaceResourceId + : null disableCustomMetrics: disableCustomMetrics disablePrometheusMetricsScraping: disablePrometheusMetricsScraping syslogPort: syslogPort @@ -734,7 +781,7 @@ resource managedCluster 'Microsoft.ContainerService/managedClusters@2024-03-02-p securityMonitoring: { enabled: enableAzureDefender } - logAnalyticsWorkspaceResourceId: monitoringWorkspaceId + logAnalyticsWorkspaceResourceId: monitoringWorkspaceResourceId } : null workloadIdentity: enableWorkloadIdentity @@ -768,8 +815,8 @@ resource managedCluster 'Microsoft.ContainerService/managedClusters@2024-03-02-p } module managedCluster_maintenanceConfigurations 'maintenance-configurations/main.bicep' = [ - for (maintenanceConfiguration, index) in (maintenanceConfigurations ?? []): if (!empty(maintenanceConfiguration)) { - name: '${uniqueString(deployment().name, location)}-ManagedCluster-MaintenanceConfigurations-${index}' + for (maintenanceConfiguration, index) in (maintenanceConfigurations ?? []): { + name: '${uniqueString(deployment().name, location)}-ManagedCluster-MaintenanceConfiguration-${index}' params: { name: maintenanceConfiguration!.name maintenanceWindow: maintenanceConfiguration!.maintenanceWindow @@ -799,14 +846,14 @@ module managedCluster_agentPools 'agent-pool/main.bicep' = [ minCount: agentPool.?minCount mode: agentPool.?mode nodeLabels: agentPool.?nodeLabels - nodePublicIpPrefixId: agentPool.?nodePublicIpPrefixId + nodePublicIpPrefixResourceId: agentPool.?nodePublicIpPrefixResourceId nodeTaints: agentPool.?nodeTaints orchestratorVersion: agentPool.?orchestratorVersion ?? kubernetesVersion osDiskSizeGB: agentPool.?osDiskSizeGB osDiskType: agentPool.?osDiskType osSku: agentPool.?osSku osType: agentPool.?osType - podSubnetId: agentPool.?podSubnetId + podSubnetResourceId: agentPool.?podSubnetResourceId proximityPlacementGroupResourceId: agentPool.?proximityPlacementGroupResourceId scaleDownMode: agentPool.?scaleDownMode scaleSetEvictionPolicy: agentPool.?scaleSetEvictionPolicy @@ -816,7 +863,7 @@ module managedCluster_agentPools 'agent-pool/main.bicep' = [ type: agentPool.?type maxSurge: agentPool.?maxSurge vmSize: agentPool.?vmSize - vnetSubnetId: agentPool.?vnetSubnetId + vnetSubnetResourceId: agentPool.?vnetSubnetResourceId workloadRuntime: agentPool.?workloadRuntime } } @@ -970,12 +1017,13 @@ output webAppRoutingIdentityObjectId string = managedCluster.properties.?ingress // Definitions // // =============== // +@export() type agentPoolType = { @description('Required. The name of the agent pool.') - name: string? + name: string @description('Optional. The availability zones of the agent pool.') - availabilityZones: string[]? + availabilityZones: int[]? @description('Optional. The number of agents (VMs) to host docker containers. Allowed values must be in the range of 1 to 100 (inclusive).') count: int? @@ -1023,7 +1071,7 @@ type agentPoolType = { nodeLabels: object? @description('Optional. The node public IP prefix ID of the agent pool.') - nodePublicIpPrefixId: string? + nodePublicIpPrefixResourceId: string? @description('Optional. The node taints of the agent pool.') nodeTaints: string[]? @@ -1044,7 +1092,7 @@ type agentPoolType = { osType: ('Linux' | 'Windows')? @description('Optional. The pod subnet ID of the agent pool.') - podSubnetId: string? + podSubnetResourceId: string? @description('Optional. The proximity placement group resource ID of the agent pool.') proximityPlacementGroupResourceId: string? @@ -1074,23 +1122,25 @@ type agentPoolType = { vmSize: string? @description('Optional. The VNet subnet ID of the agent pool.') - vnetSubnetID: string? + vnetSubnetResourceId: string? @description('Optional. The workload runtime of the agent pool.') workloadRuntime: string? @description('Optional. The enable default telemetry of the agent pool.') enableDefaultTelemetry: bool? -}[]? +} +@export() type managedIdentitiesType = { @description('Optional. Enables system assigned managed identity on the resource.') systemAssigned: bool? @description('Optional. The resource ID(s) to assign to the resource.') userAssignedResourcesIds: string[]? -}? +} +@export() type lockType = { @description('Optional. Specify the name of lock.') name: string? @@ -1099,6 +1149,7 @@ type lockType = { kind: ('CanNotDelete' | 'ReadOnly' | 'None')? }? +@export() type roleAssignmentType = { @description('Optional. The name (as GUID) of the role assignment. If not provided, a GUID will be generated.') name: string? @@ -1125,6 +1176,7 @@ type roleAssignmentType = { delegatedManagedIdentityResourceId: string? }[]? +@export() type diagnosticSettingType = { @description('Optional. The name of diagnostic setting.') name: string? @@ -1169,11 +1221,13 @@ type diagnosticSettingType = { marketplacePartnerResourceId: string? }[]? +@export() type fluxConfigurationProtectedSettingsType = { @description('Optional. The SSH private key to use for Git authentication.') sshPrivateKey: string? -}? +} +@export() type extensionType = { @description('Required. The name of the extension.') name: string? @@ -1198,8 +1252,9 @@ type extensionType = { @description('Optional. The flux configurations of the extension.') configurations: array? -}? +} +@export() type customerManagedKeyType = { @description('Required. The resource ID of a key vault to reference a customer managed key for encryption from.') keyVaultResourceId: string @@ -1212,12 +1267,13 @@ type customerManagedKeyType = { @description('Required. Network access of key vault. The possible values are Public and Private. Public means the key vault allows public access from all networks. Private means the key vault disables public access and enables private link. The default value is Public.') keyVaultNetworkAccess: ('Private' | 'Public') -}? +} +@export() type maintenanceConfigurationType = { @description('Required. Name of maintenance window.') name: ('aksManagedAutoUpgradeSchedule' | 'aksManagedNodeOSUpgradeSchedule') @description('Required. Maintenance window for the maintenance configuration.') maintenanceWindow: object -}[]? +} diff --git a/avm/res/container-service/managed-cluster/main.json b/avm/res/container-service/managed-cluster/main.json index 142cc567ba..c0c9f95d89 100644 --- a/avm/res/container-service/managed-cluster/main.json +++ b/avm/res/container-service/managed-cluster/main.json @@ -6,7 +6,7 @@ "_generator": { "name": "bicep", "version": "0.30.23.60470", - "templateHash": "13565722664676041295" + "templateHash": "14675004991337717729" }, "name": "Azure Kubernetes Service (AKS) Managed Clusters", "description": "This module deploys an Azure Kubernetes Service (AKS) Managed Cluster.", @@ -14,310 +14,308 @@ }, "definitions": { "agentPoolType": { - "type": "array", - "items": { - "type": "object", - "properties": { - "name": { - "type": "string", - "nullable": true, - "metadata": { - "description": "Required. The name of the agent pool." - } - }, - "availabilityZones": { - "type": "array", - "items": { - "type": "string" - }, - "nullable": true, - "metadata": { - "description": "Optional. The availability zones of the agent pool." - } - }, - "count": { - "type": "int", - "nullable": true, - "metadata": { - "description": "Optional. The number of agents (VMs) to host docker containers. Allowed values must be in the range of 1 to 100 (inclusive)." - } - }, - "sourceResourceId": { - "type": "string", - "nullable": true, - "metadata": { - "description": "Optional. The source resource ID to create the agent pool from." - } - }, - "enableAutoScaling": { - "type": "bool", - "nullable": true, - "metadata": { - "description": "Optional. Whether to enable auto-scaling for the agent pool." - } - }, - "enableEncryptionAtHost": { - "type": "bool", - "nullable": true, - "metadata": { - "description": "Optional. Whether to enable encryption at host for the agent pool." - } - }, - "enableFIPS": { - "type": "bool", - "nullable": true, - "metadata": { - "description": "Optional. Whether to enable FIPS for the agent pool." - } - }, - "enableNodePublicIP": { - "type": "bool", - "nullable": true, - "metadata": { - "description": "Optional. Whether to enable node public IP for the agent pool." - } - }, - "enableUltraSSD": { - "type": "bool", - "nullable": true, - "metadata": { - "description": "Optional. Whether to enable Ultra SSD for the agent pool." - } - }, - "gpuInstanceProfile": { - "type": "string", - "allowedValues": [ - "MIG1g", - "MIG2g", - "MIG3g", - "MIG4g", - "MIG7g" - ], - "nullable": true, - "metadata": { - "description": "Optional. The GPU instance profile of the agent pool." - } - }, - "kubeletDiskType": { - "type": "string", - "nullable": true, - "metadata": { - "description": "Optional. The kubelet disk type of the agent pool." - } - }, - "maxCount": { - "type": "int", - "nullable": true, - "metadata": { - "description": "Optional. The maximum number of agents (VMs) to host docker containers. Allowed values must be in the range of 1 to 100 (inclusive)." - } - }, - "minCount": { - "type": "int", - "nullable": true, - "metadata": { - "description": "Optional. The minimum number of agents (VMs) to host docker containers. Allowed values must be in the range of 1 to 100 (inclusive)." - } - }, - "maxPods": { - "type": "int", - "nullable": true, - "metadata": { - "description": "Optional. The maximum number of pods that can run on a node." - } - }, - "minPods": { - "type": "int", - "nullable": true, - "metadata": { - "description": "Optional. The minimum number of pods that can run on a node." - } - }, - "mode": { - "type": "string", - "allowedValues": [ - "System", - "User" - ], - "nullable": true, - "metadata": { - "description": "Optional. The mode of the agent pool." - } - }, - "nodeLabels": { - "type": "object", - "nullable": true, - "metadata": { - "description": "Optional. The node labels of the agent pool." - } - }, - "nodePublicIpPrefixId": { - "type": "string", - "nullable": true, - "metadata": { - "description": "Optional. The node public IP prefix ID of the agent pool." - } - }, - "nodeTaints": { - "type": "array", - "items": { - "type": "string" - }, - "nullable": true, - "metadata": { - "description": "Optional. The node taints of the agent pool." - } - }, - "orchestratorVersion": { - "type": "string", - "nullable": true, - "metadata": { - "description": "Optional. The Kubernetes version of the agent pool." - } - }, - "osDiskSizeGB": { - "type": "int", - "nullable": true, - "metadata": { - "description": "Optional. The OS disk size in GB of the agent pool." - } - }, - "osDiskType": { - "type": "string", - "nullable": true, - "metadata": { - "description": "Optional. The OS disk type of the agent pool." - } - }, - "osSku": { - "type": "string", - "nullable": true, - "metadata": { - "description": "Optional. The OS SKU of the agent pool." - } - }, - "osType": { - "type": "string", - "allowedValues": [ - "Linux", - "Windows" - ], - "nullable": true, - "metadata": { - "description": "Optional. The OS type of the agent pool." - } - }, - "podSubnetId": { - "type": "string", - "nullable": true, - "metadata": { - "description": "Optional. The pod subnet ID of the agent pool." - } - }, - "proximityPlacementGroupResourceId": { - "type": "string", - "nullable": true, - "metadata": { - "description": "Optional. The proximity placement group resource ID of the agent pool." - } - }, - "scaleDownMode": { - "type": "string", - "allowedValues": [ - "Deallocate", - "Delete" - ], - "nullable": true, - "metadata": { - "description": "Optional. The scale down mode of the agent pool." - } - }, - "scaleSetEvictionPolicy": { - "type": "string", - "allowedValues": [ - "Deallocate", - "Delete" - ], - "nullable": true, - "metadata": { - "description": "Optional. The scale set eviction policy of the agent pool." - } - }, - "scaleSetPriority": { - "type": "string", - "allowedValues": [ - "Low", - "Regular", - "Spot" - ], - "nullable": true, - "metadata": { - "description": "Optional. The scale set priority of the agent pool." - } - }, - "spotMaxPrice": { - "type": "int", - "nullable": true, - "metadata": { - "description": "Optional. The spot max price of the agent pool." - } - }, - "tags": { - "type": "object", - "nullable": true, - "metadata": { - "description": "Optional. The tags of the agent pool." - } - }, - "type": { - "type": "string", - "allowedValues": [ - "AvailabilitySet", - "VirtualMachineScaleSets" - ], - "nullable": true, - "metadata": { - "description": "Optional. The type of the agent pool." - } - }, - "maxSurge": { - "type": "string", - "nullable": true, - "metadata": { - "description": "Optional. The maximum number of nodes that can be created during an upgrade." - } - }, - "vmSize": { - "type": "string", - "nullable": true, - "metadata": { - "description": "Optional. The VM size of the agent pool." - } - }, - "vnetSubnetID": { - "type": "string", - "nullable": true, - "metadata": { - "description": "Optional. The VNet subnet ID of the agent pool." - } + "type": "object", + "properties": { + "name": { + "type": "string", + "metadata": { + "description": "Required. The name of the agent pool." + } + }, + "availabilityZones": { + "type": "array", + "items": { + "type": "int" }, - "workloadRuntime": { - "type": "string", - "nullable": true, - "metadata": { - "description": "Optional. The workload runtime of the agent pool." - } + "nullable": true, + "metadata": { + "description": "Optional. The availability zones of the agent pool." + } + }, + "count": { + "type": "int", + "nullable": true, + "metadata": { + "description": "Optional. The number of agents (VMs) to host docker containers. Allowed values must be in the range of 1 to 100 (inclusive)." + } + }, + "sourceResourceId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The source resource ID to create the agent pool from." + } + }, + "enableAutoScaling": { + "type": "bool", + "nullable": true, + "metadata": { + "description": "Optional. Whether to enable auto-scaling for the agent pool." + } + }, + "enableEncryptionAtHost": { + "type": "bool", + "nullable": true, + "metadata": { + "description": "Optional. Whether to enable encryption at host for the agent pool." + } + }, + "enableFIPS": { + "type": "bool", + "nullable": true, + "metadata": { + "description": "Optional. Whether to enable FIPS for the agent pool." + } + }, + "enableNodePublicIP": { + "type": "bool", + "nullable": true, + "metadata": { + "description": "Optional. Whether to enable node public IP for the agent pool." + } + }, + "enableUltraSSD": { + "type": "bool", + "nullable": true, + "metadata": { + "description": "Optional. Whether to enable Ultra SSD for the agent pool." + } + }, + "gpuInstanceProfile": { + "type": "string", + "allowedValues": [ + "MIG1g", + "MIG2g", + "MIG3g", + "MIG4g", + "MIG7g" + ], + "nullable": true, + "metadata": { + "description": "Optional. The GPU instance profile of the agent pool." + } + }, + "kubeletDiskType": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The kubelet disk type of the agent pool." + } + }, + "maxCount": { + "type": "int", + "nullable": true, + "metadata": { + "description": "Optional. The maximum number of agents (VMs) to host docker containers. Allowed values must be in the range of 1 to 100 (inclusive)." + } + }, + "minCount": { + "type": "int", + "nullable": true, + "metadata": { + "description": "Optional. The minimum number of agents (VMs) to host docker containers. Allowed values must be in the range of 1 to 100 (inclusive)." + } + }, + "maxPods": { + "type": "int", + "nullable": true, + "metadata": { + "description": "Optional. The maximum number of pods that can run on a node." + } + }, + "minPods": { + "type": "int", + "nullable": true, + "metadata": { + "description": "Optional. The minimum number of pods that can run on a node." + } + }, + "mode": { + "type": "string", + "allowedValues": [ + "System", + "User" + ], + "nullable": true, + "metadata": { + "description": "Optional. The mode of the agent pool." + } + }, + "nodeLabels": { + "type": "object", + "nullable": true, + "metadata": { + "description": "Optional. The node labels of the agent pool." + } + }, + "nodePublicIpPrefixResourceId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The node public IP prefix ID of the agent pool." + } + }, + "nodeTaints": { + "type": "array", + "items": { + "type": "string" }, - "enableDefaultTelemetry": { - "type": "bool", - "nullable": true, - "metadata": { - "description": "Optional. The enable default telemetry of the agent pool." - } + "nullable": true, + "metadata": { + "description": "Optional. The node taints of the agent pool." + } + }, + "orchestratorVersion": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The Kubernetes version of the agent pool." + } + }, + "osDiskSizeGB": { + "type": "int", + "nullable": true, + "metadata": { + "description": "Optional. The OS disk size in GB of the agent pool." + } + }, + "osDiskType": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The OS disk type of the agent pool." + } + }, + "osSku": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The OS SKU of the agent pool." + } + }, + "osType": { + "type": "string", + "allowedValues": [ + "Linux", + "Windows" + ], + "nullable": true, + "metadata": { + "description": "Optional. The OS type of the agent pool." + } + }, + "podSubnetResourceId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The pod subnet ID of the agent pool." + } + }, + "proximityPlacementGroupResourceId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The proximity placement group resource ID of the agent pool." + } + }, + "scaleDownMode": { + "type": "string", + "allowedValues": [ + "Deallocate", + "Delete" + ], + "nullable": true, + "metadata": { + "description": "Optional. The scale down mode of the agent pool." + } + }, + "scaleSetEvictionPolicy": { + "type": "string", + "allowedValues": [ + "Deallocate", + "Delete" + ], + "nullable": true, + "metadata": { + "description": "Optional. The scale set eviction policy of the agent pool." + } + }, + "scaleSetPriority": { + "type": "string", + "allowedValues": [ + "Low", + "Regular", + "Spot" + ], + "nullable": true, + "metadata": { + "description": "Optional. The scale set priority of the agent pool." + } + }, + "spotMaxPrice": { + "type": "int", + "nullable": true, + "metadata": { + "description": "Optional. The spot max price of the agent pool." + } + }, + "tags": { + "type": "object", + "nullable": true, + "metadata": { + "description": "Optional. The tags of the agent pool." + } + }, + "type": { + "type": "string", + "allowedValues": [ + "AvailabilitySet", + "VirtualMachineScaleSets" + ], + "nullable": true, + "metadata": { + "description": "Optional. The type of the agent pool." + } + }, + "maxSurge": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The maximum number of nodes that can be created during an upgrade." + } + }, + "vmSize": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The VM size of the agent pool." + } + }, + "vnetSubnetResourceId": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The VNet subnet ID of the agent pool." + } + }, + "workloadRuntime": { + "type": "string", + "nullable": true, + "metadata": { + "description": "Optional. The workload runtime of the agent pool." + } + }, + "enableDefaultTelemetry": { + "type": "bool", + "nullable": true, + "metadata": { + "description": "Optional. The enable default telemetry of the agent pool." } } }, - "nullable": true + "metadata": { + "__bicep_export!": true + } }, "managedIdentitiesType": { "type": "object", @@ -340,7 +338,9 @@ } } }, - "nullable": true + "metadata": { + "__bicep_export!": true + } }, "lockType": { "type": "object", @@ -365,7 +365,10 @@ } } }, - "nullable": true + "nullable": true, + "metadata": { + "__bicep_export!": true + } }, "roleAssignmentType": { "type": "array", @@ -438,7 +441,10 @@ } } }, - "nullable": true + "nullable": true, + "metadata": { + "__bicep_export!": true + } }, "diagnosticSettingType": { "type": "array", @@ -558,7 +564,10 @@ } } }, - "nullable": true + "nullable": true, + "metadata": { + "__bicep_export!": true + } }, "fluxConfigurationProtectedSettingsType": { "type": "object", @@ -571,7 +580,9 @@ } } }, - "nullable": true + "metadata": { + "__bicep_export!": true + } }, "extensionType": { "type": "object", @@ -633,7 +644,9 @@ } } }, - "nullable": true + "metadata": { + "__bicep_export!": true + } }, "customerManagedKeyType": { "type": "object", @@ -668,32 +681,33 @@ } } }, - "nullable": true + "metadata": { + "__bicep_export!": true + } }, "maintenanceConfigurationType": { - "type": "array", - "items": { - "type": "object", - "properties": { - "name": { - "type": "string", - "allowedValues": [ - "aksManagedAutoUpgradeSchedule", - "aksManagedNodeOSUpgradeSchedule" - ], - "metadata": { - "description": "Required. Name of maintenance window." - } - }, - "maintenanceWindow": { - "type": "object", - "metadata": { - "description": "Required. Maintenance window for the maintenance configuration." - } + "type": "object", + "properties": { + "name": { + "type": "string", + "allowedValues": [ + "aksManagedAutoUpgradeSchedule", + "aksManagedNodeOSUpgradeSchedule" + ], + "metadata": { + "description": "Required. Name of maintenance window." + } + }, + "maintenanceWindow": { + "type": "object", + "metadata": { + "description": "Required. Maintenance window for the maintenance configuration." } } }, - "nullable": true + "metadata": { + "__bicep_export!": true + } } }, "parameters": { @@ -719,6 +733,7 @@ }, "managedIdentities": { "$ref": "#/definitions/managedIdentitiesType", + "nullable": true, "metadata": { "description": "Optional. The managed identity definition for this resource. Only one type of identity is supported: system-assigned or user-assigned, but not both." } @@ -899,6 +914,9 @@ }, "aadProfileAdminGroupObjectIDs": { "type": "array", + "items": { + "type": "string" + }, "nullable": true, "metadata": { "description": "Optional. Specifies the AAD group object IDs that will have admin role of the cluster." @@ -927,7 +945,7 @@ }, "disableLocalAccounts": { "type": "bool", - "defaultValue": false, + "defaultValue": true, "metadata": { "description": "Optional. If set to true, getting static credentials will be disabled for this cluster. This must only be used on Managed Clusters that are AAD enabled." } @@ -941,6 +959,9 @@ }, "authorizedIPRanges": { "type": "array", + "items": { + "type": "string" + }, "nullable": true, "metadata": { "description": "Optional. IP ranges are specified in CIDR format, e.g. 137.117.106.88/29. This feature is not compatible with clusters that use Public IP Per Node, or clusters that are using a Basic Load Balancer." @@ -986,22 +1007,33 @@ "description": "Optional. Private DNS Zone configuration. Set to 'system' and AKS will create a private DNS zone in the node resource group. Set to '' to disable private DNS Zone creation and use public DNS. Supply the resource ID here of an existing Private DNS zone to use an existing zone." } }, - "primaryAgentPoolProfile": { + "primaryAgentPoolProfiles": { "type": "array", + "items": { + "$ref": "#/definitions/agentPoolType" + }, "metadata": { "description": "Required. Properties of the primary agent pool." } }, "agentPools": { - "$ref": "#/definitions/agentPoolType", + "type": "array", + "items": { + "$ref": "#/definitions/agentPoolType" + }, + "nullable": true, "metadata": { "description": "Optional. Define one or more secondary/additional agent pools." } }, "maintenanceConfigurations": { - "$ref": "#/definitions/maintenanceConfigurationType", + "type": "array", + "items": { + "$ref": "#/definitions/maintenanceConfigurationType" + }, + "nullable": true, "metadata": { - "description": "Optional. Maintenance Window for Cluster auto upgrade and node OS upgrade." + "description": "Optional. Whether or not to use AKS Automatic mode." } }, "costAnalysisEnabled": { @@ -1152,8 +1184,8 @@ } }, "autoScalerProfileMaxGracefulTerminationSec": { - "type": "string", - "defaultValue": "600", + "type": "int", + "defaultValue": 600, "metadata": { "description": "Optional. Specifies the max graceful termination time interval in seconds for the auto-scaler of the AKS cluster." } @@ -1179,8 +1211,8 @@ } }, "autoScalerProfileMaxEmptyBulkDelete": { - "type": "string", - "defaultValue": "10", + "type": "int", + "defaultValue": 10, "metadata": { "description": "Optional. Specifies the maximum empty bulk delete for the auto-scaler of the AKS cluster." } @@ -1193,8 +1225,8 @@ } }, "autoScalerProfileMaxTotalUnreadyPercentage": { - "type": "string", - "defaultValue": "45", + "type": "int", + "defaultValue": 45, "metadata": { "description": "Optional. Specifies the mximum total unready percentage for the auto-scaler of the AKS cluster. The maximum is 100 and the minimum is 0." } @@ -1207,8 +1239,8 @@ } }, "autoScalerProfileOkTotalUnreadyCount": { - "type": "string", - "defaultValue": "3", + "type": "int", + "defaultValue": 3, "metadata": { "description": "Optional. Specifies the OK total unready count for the auto-scaler of the AKS cluster." } @@ -1377,7 +1409,7 @@ "description": "Optional. Specifies whether the OMS agent is enabled." } }, - "monitoringWorkspaceId": { + "monitoringWorkspaceResourceId": { "type": "string", "nullable": true, "metadata": { @@ -1419,6 +1451,7 @@ }, "fluxExtension": { "$ref": "#/definitions/extensionType", + "nullable": true, "metadata": { "description": "Optional. Settings and configurations for the flux extension." } @@ -1453,6 +1486,7 @@ }, "customerManagedKey": { "$ref": "#/definitions/customerManagedKeyType", + "nullable": true, "metadata": { "description": "Optional. The customer managed key definition." } @@ -1593,12 +1627,12 @@ "tier": "[parameters('skuTier')]" }, "properties": { + "agentPoolProfiles": "[map(parameters('primaryAgentPoolProfiles'), lambda('profile', createObject('name', lambdaVariables('profile').name, 'count', coalesce(lambdaVariables('profile').count, 1), 'availabilityZones', map(coalesce(tryGet(lambdaVariables('profile'), 'availabilityZones'), createArray(1, 2, 3)), lambda('zone', format('{0}', lambdaVariables('zone')))), 'creationData', if(not(empty(tryGet(lambdaVariables('profile'), 'sourceResourceId'))), createObject('sourceResourceId', lambdaVariables('profile').sourceResourceId), null()), 'enableAutoScaling', coalesce(tryGet(lambdaVariables('profile'), 'enableAutoScaling'), false()), 'enableEncryptionAtHost', coalesce(tryGet(lambdaVariables('profile'), 'enableEncryptionAtHost'), false()), 'enableFIPS', coalesce(tryGet(lambdaVariables('profile'), 'enableFIPS'), false()), 'enableNodePublicIP', coalesce(tryGet(lambdaVariables('profile'), 'enableNodePublicIP'), false()), 'enableUltraSSD', coalesce(tryGet(lambdaVariables('profile'), 'enableUltraSSD'), false()), 'gpuInstanceProfile', tryGet(lambdaVariables('profile'), 'gpuInstanceProfile'), 'kubeletDiskType', tryGet(lambdaVariables('profile'), 'kubeletDiskType'), 'maxCount', tryGet(lambdaVariables('profile'), 'maxCount'), 'maxPods', tryGet(lambdaVariables('profile'), 'maxPods'), 'minCount', tryGet(lambdaVariables('profile'), 'minCount'), 'mode', tryGet(lambdaVariables('profile'), 'mode'), 'nodeLabels', tryGet(lambdaVariables('profile'), 'nodeLabels'), 'nodePublicIPPrefixID', tryGet(lambdaVariables('profile'), 'nodePublicIpPrefixResourceId'), 'nodeTaints', tryGet(lambdaVariables('profile'), 'nodeTaints'), 'orchestratorVersion', tryGet(lambdaVariables('profile'), 'orchestratorVersion'), 'osDiskSizeGB', tryGet(lambdaVariables('profile'), 'osDiskSizeGB'), 'osDiskType', tryGet(lambdaVariables('profile'), 'osDiskType'), 'osType', coalesce(tryGet(lambdaVariables('profile'), 'osType'), 'Linux'), 'podSubnetID', tryGet(lambdaVariables('profile'), 'podSubnetResourceId'), 'proximityPlacementGroupID', tryGet(lambdaVariables('profile'), 'proximityPlacementGroupResourceId'), 'scaleDownMode', coalesce(tryGet(lambdaVariables('profile'), 'scaleDownMode'), 'Delete'), 'scaleSetEvictionPolicy', coalesce(tryGet(lambdaVariables('profile'), 'scaleSetEvictionPolicy'), 'Delete'), 'scaleSetPriority', tryGet(lambdaVariables('profile'), 'scaleSetPriority'), 'spotMaxPrice', tryGet(lambdaVariables('profile'), 'spotMaxPrice'), 'tags', tryGet(lambdaVariables('profile'), 'tags'), 'type', tryGet(lambdaVariables('profile'), 'type'), 'upgradeSettings', createObject('maxSurge', tryGet(lambdaVariables('profile'), 'maxSurge')), 'vmSize', coalesce(tryGet(lambdaVariables('profile'), 'vmSize'), 'Standard_D2s_v3'), 'vnetSubnetID', tryGet(lambdaVariables('profile'), 'vnetSubnetResourceId'), 'workloadRuntime', tryGet(lambdaVariables('profile'), 'workloadRuntime'))))]", "httpProxyConfig": "[parameters('httpProxyConfig')]", "identityProfile": "[parameters('identityProfile')]", "diskEncryptionSetID": "[parameters('diskEncryptionSetResourceId')]", "kubernetesVersion": "[parameters('kubernetesVersion')]", "dnsPrefix": "[parameters('dnsPrefix')]", - "agentPoolProfiles": "[parameters('primaryAgentPoolProfile')]", "linuxProfile": "[if(not(empty(parameters('sshPublicKey'))), createObject('adminUsername', parameters('adminUsername'), 'ssh', createObject('publicKeys', createArray(createObject('keyData', coalesce(parameters('sshPublicKey'), ''))))), null())]", "servicePrincipalProfile": "[parameters('aksServicePrincipalProfile')]", "metricsProfile": { @@ -1621,8 +1655,8 @@ "config": "[if(and(parameters('ingressApplicationGatewayEnabled'), not(empty(parameters('appGatewayResourceId')))), createObject('applicationGatewayId', parameters('appGatewayResourceId'), 'effectiveApplicationGatewayId', parameters('appGatewayResourceId')), null())]" }, "omsagent": { - "enabled": "[and(parameters('omsAgentEnabled'), not(empty(parameters('monitoringWorkspaceId'))))]", - "config": "[if(and(parameters('omsAgentEnabled'), not(empty(parameters('monitoringWorkspaceId')))), createObject('logAnalyticsWorkspaceResourceID', parameters('monitoringWorkspaceId')), null())]" + "enabled": "[and(parameters('omsAgentEnabled'), not(empty(parameters('monitoringWorkspaceResourceId'))))]", + "config": "[if(and(parameters('omsAgentEnabled'), not(empty(parameters('monitoringWorkspaceResourceId')))), createObject('logAnalyticsWorkspaceResourceID', parameters('monitoringWorkspaceResourceId')), null())]" }, "aciConnectorLinux": { "enabled": "[parameters('aciConnectorLinuxEnabled')]" @@ -1685,12 +1719,12 @@ "autoScalerProfile": { "balance-similar-node-groups": "[toLower(string(parameters('autoScalerProfileBalanceSimilarNodeGroups')))]", "expander": "[parameters('autoScalerProfileExpander')]", - "max-empty-bulk-delete": "[parameters('autoScalerProfileMaxEmptyBulkDelete')]", - "max-graceful-termination-sec": "[parameters('autoScalerProfileMaxGracefulTerminationSec')]", + "max-empty-bulk-delete": "[format('{0}', parameters('autoScalerProfileMaxEmptyBulkDelete'))]", + "max-graceful-termination-sec": "[format('{0}', parameters('autoScalerProfileMaxGracefulTerminationSec'))]", "max-node-provision-time": "[parameters('autoScalerProfileMaxNodeProvisionTime')]", - "max-total-unready-percentage": "[parameters('autoScalerProfileMaxTotalUnreadyPercentage')]", + "max-total-unready-percentage": "[format('{0}', parameters('autoScalerProfileMaxTotalUnreadyPercentage'))]", "new-pod-scale-up-delay": "[parameters('autoScalerProfileNewPodScaleUpDelay')]", - "ok-total-unready-count": "[parameters('autoScalerProfileOkTotalUnreadyCount')]", + "ok-total-unready-count": "[format('{0}', parameters('autoScalerProfileOkTotalUnreadyCount'))]", "scale-down-delay-after-add": "[parameters('autoScalerProfileScaleDownDelayAfterAdd')]", "scale-down-delay-after-delete": "[parameters('autoScalerProfileScaleDownDelayAfterDelete')]", "scale-down-delay-after-failure": "[parameters('autoScalerProfileScaleDownDelayAfterFailure')]", @@ -1713,7 +1747,7 @@ "privateDNSZone": "[parameters('privateDNSZone')]" }, "azureMonitorProfile": { - "containerInsights": "[if(parameters('enableContainerInsights'), createObject('enabled', parameters('enableContainerInsights'), 'logAnalyticsWorkspaceResourceId', if(not(empty(parameters('monitoringWorkspaceId'))), parameters('monitoringWorkspaceId'), null()), 'disableCustomMetrics', parameters('disableCustomMetrics'), 'disablePrometheusMetricsScraping', parameters('disablePrometheusMetricsScraping'), 'syslogPort', parameters('syslogPort')), null())]", + "containerInsights": "[if(parameters('enableContainerInsights'), createObject('enabled', parameters('enableContainerInsights'), 'logAnalyticsWorkspaceResourceId', if(not(empty(parameters('monitoringWorkspaceResourceId'))), parameters('monitoringWorkspaceResourceId'), null()), 'disableCustomMetrics', parameters('disableCustomMetrics'), 'disablePrometheusMetricsScraping', parameters('disablePrometheusMetricsScraping'), 'syslogPort', parameters('syslogPort')), null())]", "metrics": "[if(parameters('enableAzureMonitorProfileMetrics'), createObject('enabled', parameters('enableAzureMonitorProfileMetrics'), 'kubeStateMetrics', createObject('metricLabelsAllowlist', parameters('metricLabelsAllowlist'), 'metricAnnotationsAllowList', parameters('metricAnnotationsAllowList'))), null())]" }, "podIdentityProfile": { @@ -1723,7 +1757,7 @@ "userAssignedIdentityExceptions": "[parameters('podIdentityProfileUserAssignedIdentityExceptions')]" }, "securityProfile": { - "defender": "[if(parameters('enableAzureDefender'), createObject('securityMonitoring', createObject('enabled', parameters('enableAzureDefender')), 'logAnalyticsWorkspaceResourceId', parameters('monitoringWorkspaceId')), null())]", + "defender": "[if(parameters('enableAzureDefender'), createObject('securityMonitoring', createObject('enabled', parameters('enableAzureDefender')), 'logAnalyticsWorkspaceResourceId', parameters('monitoringWorkspaceResourceId')), null())]", "workloadIdentity": "[if(parameters('enableWorkloadIdentity'), createObject('enabled', parameters('enableWorkloadIdentity')), null())]", "imageCleaner": "[if(parameters('enableImageCleaner'), createObject('enabled', parameters('enableImageCleaner'), 'intervalHours', parameters('imageCleanerIntervalHours')), null())]" }, @@ -1849,10 +1883,9 @@ "name": "managedCluster_maintenanceConfigurations", "count": "[length(coalesce(parameters('maintenanceConfigurations'), createArray()))]" }, - "condition": "[not(empty(coalesce(parameters('maintenanceConfigurations'), createArray())[copyIndex()]))]", "type": "Microsoft.Resources/deployments", "apiVersion": "2022-09-01", - "name": "[format('{0}-ManagedCluster-MaintenanceConfigurations-{1}', uniqueString(deployment().name, parameters('location')), copyIndex())]", + "name": "[format('{0}-ManagedCluster-MaintenanceConfiguration-{1}', uniqueString(deployment().name, parameters('location')), copyIndex())]", "properties": { "expressionEvaluationOptions": { "scope": "inner" @@ -2007,8 +2040,8 @@ "nodeLabels": { "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'nodeLabels')]" }, - "nodePublicIpPrefixId": { - "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'nodePublicIpPrefixId')]" + "nodePublicIpPrefixResourceId": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'nodePublicIpPrefixResourceId')]" }, "nodeTaints": { "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'nodeTaints')]" @@ -2028,8 +2061,8 @@ "osType": { "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'osType')]" }, - "podSubnetId": { - "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'podSubnetId')]" + "podSubnetResourceId": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'podSubnetResourceId')]" }, "proximityPlacementGroupResourceId": { "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'proximityPlacementGroupResourceId')]" @@ -2058,8 +2091,8 @@ "vmSize": { "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'vmSize')]" }, - "vnetSubnetId": { - "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'vnetSubnetId')]" + "vnetSubnetResourceId": { + "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'vnetSubnetResourceId')]" }, "workloadRuntime": { "value": "[tryGet(coalesce(parameters('agentPools'), createArray())[copyIndex()], 'workloadRuntime')]" @@ -2073,7 +2106,7 @@ "_generator": { "name": "bicep", "version": "0.30.23.60470", - "templateHash": "4315564225725874539" + "templateHash": "13856766172443517827" }, "name": "Azure Kubernetes Service (AKS) Managed Cluster Agent Pools", "description": "This module deploys an Azure Kubernetes Service (AKS) Managed Cluster Agent Pool.", @@ -2094,7 +2127,14 @@ }, "availabilityZones": { "type": "array", - "nullable": true, + "items": { + "type": "int" + }, + "defaultValue": [ + 1, + 2, + 3 + ], "metadata": { "description": "Optional. The list of Availability zones to use for nodes. This can only be specified if the AgentPoolType property is \"VirtualMachineScaleSets\"." } @@ -2206,7 +2246,7 @@ "description": "Optional. The node labels to be persisted across all nodes in agent pool." } }, - "nodePublicIpPrefixId": { + "nodePublicIpPrefixResourceId": { "type": "string", "nullable": true, "metadata": { @@ -2270,11 +2310,11 @@ "description": "Optional. The operating system type. The default is Linux." } }, - "podSubnetId": { + "podSubnetResourceId": { "type": "string", "nullable": true, "metadata": { - "description": "Optional. Subnet ID for the pod IPs. If omitted, pod IPs are statically assigned on the node subnet (see vnetSubnetID for more details). This is of the form: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName}." + "description": "Optional. Subnet resource ID for the pod IPs. If omitted, pod IPs are statically assigned on the node subnet (see vnetSubnetID for more details). This is of the form: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.Network/virtualNetworks/{virtualNetworkName}/subnets/{subnetName}." } }, "proximityPlacementGroupResourceId": { @@ -2352,7 +2392,7 @@ "description": "Optional. VM size. VM size availability varies by region. If a node contains insufficient compute resources (memory, cpu, etc) pods might fail to run correctly. For more details on restricted VM sizes, see: /azure/aks/quotas-skus-regions." } }, - "vnetSubnetId": { + "vnetSubnetResourceId": { "type": "string", "nullable": true, "metadata": { @@ -2379,7 +2419,7 @@ "apiVersion": "2023-07-02-preview", "name": "[format('{0}/{1}', parameters('managedClusterName'), parameters('name'))]", "properties": { - "availabilityZones": "[parameters('availabilityZones')]", + "availabilityZones": "[map(coalesce(parameters('availabilityZones'), createArray()), lambda('zone', format('{0}', lambdaVariables('zone'))))]", "count": "[parameters('count')]", "creationData": "[if(not(empty(parameters('sourceResourceId'))), createObject('sourceResourceId', parameters('sourceResourceId')), null())]", "enableAutoScaling": "[parameters('enableAutoScaling')]", @@ -2394,14 +2434,14 @@ "minCount": "[parameters('minCount')]", "mode": "[parameters('mode')]", "nodeLabels": "[parameters('nodeLabels')]", - "nodePublicIPPrefixID": "[parameters('nodePublicIpPrefixId')]", + "nodePublicIPPrefixID": "[parameters('nodePublicIpPrefixResourceId')]", "nodeTaints": "[parameters('nodeTaints')]", "orchestratorVersion": "[parameters('orchestratorVersion')]", "osDiskSizeGB": "[parameters('osDiskSizeGB')]", "osDiskType": "[parameters('osDiskType')]", "osSKU": "[parameters('osSku')]", "osType": "[parameters('osType')]", - "podSubnetID": "[parameters('podSubnetId')]", + "podSubnetID": "[parameters('podSubnetResourceId')]", "proximityPlacementGroupID": "[parameters('proximityPlacementGroupResourceId')]", "scaleDownMode": "[parameters('scaleDownMode')]", "scaleSetEvictionPolicy": "[parameters('scaleSetEvictionPolicy')]", @@ -2413,7 +2453,7 @@ "maxSurge": "[parameters('maxSurge')]" }, "vmSize": "[parameters('vmSize')]", - "vnetSubnetID": "[parameters('vnetSubnetId')]", + "vnetSubnetID": "[parameters('vnetSubnetResourceId')]", "workloadRuntime": "[parameters('workloadRuntime')]" }, "dependsOn": [ diff --git a/avm/res/container-service/managed-cluster/maintenance-configurations/main.json b/avm/res/container-service/managed-cluster/maintenance-configurations/main.json index 3c4f84d104..22e9300b85 100644 --- a/avm/res/container-service/managed-cluster/maintenance-configurations/main.json +++ b/avm/res/container-service/managed-cluster/maintenance-configurations/main.json @@ -4,8 +4,8 @@ "metadata": { "_generator": { "name": "bicep", - "version": "0.29.47.4906", - "templateHash": "12168542117744033419" + "version": "0.30.23.60470", + "templateHash": "2505380725266419010" }, "name": "Azure Kubernetes Service (AKS) Managed Cluster Maintenance Configurations", "description": "This module deploys an Azure Kubernetes Service (AKS) Managed Cluster Maintenance Configurations.", diff --git a/avm/res/container-service/managed-cluster/tests/e2e/automatic/main.test.bicep b/avm/res/container-service/managed-cluster/tests/e2e/automatic/main.test.bicep index cd20ff76e7..7852289f4c 100644 --- a/avm/res/container-service/managed-cluster/tests/e2e/automatic/main.test.bicep +++ b/avm/res/container-service/managed-cluster/tests/e2e/automatic/main.test.bicep @@ -62,8 +62,7 @@ module testDeployment '../../../main.bicep' = [ managedIdentities: { systemAssigned: true } - - primaryAgentPoolProfile: [ + primaryAgentPoolProfiles: [ { name: 'systempool' count: 3 diff --git a/avm/res/container-service/managed-cluster/tests/e2e/azure/main.test.bicep b/avm/res/container-service/managed-cluster/tests/e2e/azure/main.test.bicep index 76e6c50044..d21dfdcb2d 100644 --- a/avm/res/container-service/managed-cluster/tests/e2e/azure/main.test.bicep +++ b/avm/res/container-service/managed-cluster/tests/e2e/azure/main.test.bicep @@ -76,10 +76,10 @@ module testDeployment '../../../main.bicep' = [ params: { location: resourceLocation name: '${namePrefix}${serviceShort}001' - primaryAgentPoolProfile: [ + primaryAgentPoolProfiles: [ { availabilityZones: [ - '3' + 3 ] count: 1 enableAutoScaling: true @@ -95,13 +95,13 @@ module testDeployment '../../../main.bicep' = [ osType: 'Linux' type: 'VirtualMachineScaleSets' vmSize: 'Standard_DS2_v2' - vnetSubnetID: nestedDependencies.outputs.subnetResourceIds[0] + vnetSubnetResourceId: nestedDependencies.outputs.subnetResourceIds[0] } ] agentPools: [ { availabilityZones: [ - '3' + 3 ] count: 2 enableAutoScaling: true @@ -118,12 +118,12 @@ module testDeployment '../../../main.bicep' = [ scaleSetPriority: 'Regular' type: 'VirtualMachineScaleSets' vmSize: 'Standard_DS2_v2' - vnetSubnetID: nestedDependencies.outputs.subnetResourceIds[1] + vnetSubnetResourceId: nestedDependencies.outputs.subnetResourceIds[1] proximityPlacementGroupResourceId: nestedDependencies.outputs.proximityPlacementGroupResourceId } { availabilityZones: [ - '3' + 3 ] count: 2 enableAutoScaling: true @@ -140,7 +140,7 @@ module testDeployment '../../../main.bicep' = [ scaleSetPriority: 'Regular' type: 'VirtualMachineScaleSets' vmSize: 'Standard_DS2_v2' - vnetSubnetID: nestedDependencies.outputs.subnetResourceIds[2] + vnetSubnetResourceId: nestedDependencies.outputs.subnetResourceIds[2] } ] autoUpgradeProfileUpgradeChannel: 'stable' @@ -213,7 +213,7 @@ module testDeployment '../../../main.bicep' = [ } } omsAgentEnabled: true - monitoringWorkspaceId: nestedDependencies.outputs.logAnalyticsWorkspaceResourceId + monitoringWorkspaceResourceId: nestedDependencies.outputs.logAnalyticsWorkspaceResourceId enableAzureDefender: true enableKeyvaultSecretsProvider: true enablePodSecurityPolicy: false diff --git a/avm/res/container-service/managed-cluster/tests/e2e/defaults/main.test.bicep b/avm/res/container-service/managed-cluster/tests/e2e/defaults/main.test.bicep index 60a4103ddd..48b0faca8c 100644 --- a/avm/res/container-service/managed-cluster/tests/e2e/defaults/main.test.bicep +++ b/avm/res/container-service/managed-cluster/tests/e2e/defaults/main.test.bicep @@ -42,7 +42,7 @@ module testDeployment '../../../main.bicep' = [ managedIdentities: { systemAssigned: true } - primaryAgentPoolProfile: [ + primaryAgentPoolProfiles: [ { name: 'systempool' count: 3 diff --git a/avm/res/container-service/managed-cluster/tests/e2e/kubenet/main.test.bicep b/avm/res/container-service/managed-cluster/tests/e2e/kubenet/main.test.bicep index 6b5171e708..03235dc22b 100644 --- a/avm/res/container-service/managed-cluster/tests/e2e/kubenet/main.test.bicep +++ b/avm/res/container-service/managed-cluster/tests/e2e/kubenet/main.test.bicep @@ -67,10 +67,10 @@ module testDeployment '../../../main.bicep' = [ params: { name: '${namePrefix}${serviceShort}001' location: resourceLocation - primaryAgentPoolProfile: [ + primaryAgentPoolProfiles: [ { availabilityZones: [ - '3' + 3 ] count: 1 enableAutoScaling: true @@ -91,7 +91,7 @@ module testDeployment '../../../main.bicep' = [ agentPools: [ { availabilityZones: [ - '3' + 3 ] count: 2 enableAutoScaling: true @@ -111,7 +111,7 @@ module testDeployment '../../../main.bicep' = [ } { availabilityZones: [ - '3' + 3 ] count: 2 enableAutoScaling: true diff --git a/avm/res/container-service/managed-cluster/tests/e2e/priv/main.test.bicep b/avm/res/container-service/managed-cluster/tests/e2e/priv/main.test.bicep index 94bf2b0d23..555258a93b 100644 --- a/avm/res/container-service/managed-cluster/tests/e2e/priv/main.test.bicep +++ b/avm/res/container-service/managed-cluster/tests/e2e/priv/main.test.bicep @@ -55,10 +55,10 @@ module testDeployment '../../../main.bicep' = [ name: '${namePrefix}${serviceShort}001' location: resourceLocation enablePrivateCluster: true - primaryAgentPoolProfile: [ + primaryAgentPoolProfiles: [ { availabilityZones: [ - '3' + 3 ] count: 1 enableAutoScaling: true @@ -74,13 +74,13 @@ module testDeployment '../../../main.bicep' = [ osType: 'Linux' type: 'VirtualMachineScaleSets' vmSize: 'Standard_DS2_v2' - vnetSubnetID: '${nestedDependencies.outputs.vNetResourceId}/subnets/defaultSubnet' + vnetSubnetResourceId: '${nestedDependencies.outputs.vNetResourceId}/subnets/defaultSubnet' } ] agentPools: [ { availabilityZones: [ - '3' + 3 ] count: 2 enableAutoScaling: true @@ -97,11 +97,11 @@ module testDeployment '../../../main.bicep' = [ scaleSetPriority: 'Regular' type: 'VirtualMachineScaleSets' vmSize: 'Standard_DS2_v2' - vnetSubnetID: '${nestedDependencies.outputs.vNetResourceId}/subnets/defaultSubnet' + vnetSubnetResourceId: '${nestedDependencies.outputs.vNetResourceId}/subnets/defaultSubnet' } { availabilityZones: [ - '3' + 3 ] count: 2 enableAutoScaling: true diff --git a/avm/res/container-service/managed-cluster/tests/e2e/waf-aligned/main.test.bicep b/avm/res/container-service/managed-cluster/tests/e2e/waf-aligned/main.test.bicep index bb0487a6a1..4b88dc1c87 100644 --- a/avm/res/container-service/managed-cluster/tests/e2e/waf-aligned/main.test.bicep +++ b/avm/res/container-service/managed-cluster/tests/e2e/waf-aligned/main.test.bicep @@ -71,10 +71,10 @@ module testDeployment '../../../main.bicep' = [ name: '${namePrefix}${serviceShort}001' location: resourceLocation enablePrivateCluster: true - primaryAgentPoolProfile: [ + primaryAgentPoolProfiles: [ { availabilityZones: [ - '3' + 3 ] count: 3 enableAutoScaling: true @@ -90,13 +90,13 @@ module testDeployment '../../../main.bicep' = [ osType: 'Linux' type: 'VirtualMachineScaleSets' vmSize: 'Standard_DS2_v2' - vnetSubnetID: '${nestedDependencies.outputs.vNetResourceId}/subnets/defaultSubnet' + vnetSubnetResourceId: '${nestedDependencies.outputs.vNetResourceId}/subnets/defaultSubnet' } ] agentPools: [ { availabilityZones: [ - '3' + 3 ] count: 3 enableAutoScaling: true @@ -114,11 +114,11 @@ module testDeployment '../../../main.bicep' = [ scaleSetPriority: 'Regular' type: 'VirtualMachineScaleSets' vmSize: 'Standard_DS2_v2' - vnetSubnetID: '${nestedDependencies.outputs.vNetResourceId}/subnets/defaultSubnet' + vnetSubnetResourceId: '${nestedDependencies.outputs.vNetResourceId}/subnets/defaultSubnet' } { availabilityZones: [ - '3' + 3 ] count: 3 enableAutoScaling: true @@ -178,7 +178,7 @@ module testDeployment '../../../main.bicep' = [ dnsServiceIP: '10.10.200.10' serviceCidr: '10.10.200.0/24' omsAgentEnabled: true - monitoringWorkspaceId: diagnosticDependencies.outputs.logAnalyticsWorkspaceResourceId + monitoringWorkspaceResourceId: diagnosticDependencies.outputs.logAnalyticsWorkspaceResourceId disableLocalAccounts: true enableAzureDefender: true diagnosticSettings: [