r/kubernetes_cluster: making the aci_connector_linux & http_application_routing blocks cloud specific

azurerm/internal/services/containers/client/client.go

@@ -4,6 +4,7 @@ import (
  "github.com/Azure/azure-sdk-for-go/services/containerinstance/mgmt/2018-10-01/containerinstance"
  "github.com/Azure/azure-sdk-for-go/services/containerregistry/mgmt/2018-09-01/containerregistry"
  "github.com/Azure/azure-sdk-for-go/services/containerservice/mgmt/2020-02-01/containerservice"
+ "github.com/Azure/go-autorest/autorest/azure"
  "github.com/terraform-providers/terraform-provider-azurerm/azurerm/internal/common"
 )
 
@@ -15,6 +16,8 @@ type Client struct {
  ReplicationsClient *containerregistry.ReplicationsClient
  ServicesClient *containerservice.ContainerServicesClient
  WebhooksClient *containerregistry.WebhooksClient
+
+ Environment azure.Environment
 }
 
 func NewClient(o *common.ClientOptions) *Client {
@@ -48,5 +51,6 @@ func NewClient(o *common.ClientOptions) *Client {
  WebhooksClient: &webhooksClient,
  ReplicationsClient: &replicationsClient,
  ServicesClient: &servicesClient,
+ Environment: o.Environment,
  }
 }

azurerm/internal/services/containers/kubernetes_addons.go

@@ -1,24 +1,42 @@
 package containers
 
 import (
+ "fmt"
  "strings"
 
  "github.com/Azure/azure-sdk-for-go/services/containerservice/mgmt/2020-02-01/containerservice"
+ "github.com/Azure/go-autorest/autorest/azure"
  "github.com/hashicorp/terraform-plugin-sdk/helper/schema"
  "github.com/hashicorp/terraform-plugin-sdk/helper/validation"
- "github.com/terraform-providers/terraform-provider-azurerm/azurerm/helpers/azure"
+ azureHelpers "github.com/terraform-providers/terraform-provider-azurerm/azurerm/helpers/azure"
  "github.com/terraform-providers/terraform-provider-azurerm/azurerm/utils"
 )
 
 const (
+ // note: the casing on these keys is important
  aciConnectorKey = "aciConnectorLinux"
  azurePolicyKey = "azurepolicy"
  kubernetesDashboardKey = "kubeDashboard"
  httpApplicationRoutingKey = "httpApplicationRouting"
  omsAgentKey = "omsagent"
 )
 
-func SchemaKubernetesAddOnProfiles() *schema.Schema {
+// The AKS API hard-codes which add-ons are supported in which environment
+// as such unfortunately we can't just send "disabled" - we need to strip
+// the unsupported addons from the HTTP response. As such this defines
+// the list of unsupported addons in the defined region - e.g. by being
+// omited from this list an addon/environment combination will be supported
+var unsupportedAddonsForEnvironment = map[string][]string{
+ azure.ChinaCloud.Name: {
+ aciConnectorKey, // https://github.com/terraform-providers/terraform-provider-azurerm/issues/5510
+ httpApplicationRoutingKey, // https://github.com/terraform-providers/terraform-provider-azurerm/issues/5960
+ },
+ azure.USGovernmentCloud.Name: {
+ httpApplicationRoutingKey, // https://github.com/terraform-providers/terraform-provider-azurerm/issues/5960
+ },
+}
+
+func schemaKubernetesAddOnProfiles() *schema.Schema {
  return &schema.Schema{
  Type: schema.TypeList,
  MaxItems: 1,
@@ -107,7 +125,7 @@ func SchemaKubernetesAddOnProfiles() *schema.Schema {
  "log_analytics_workspace_id": {
  Type: schema.TypeString,
  Optional: true,
- ValidateFunc: azure.ValidateResourceID,
+ ValidateFunc: azureHelpers.ValidateResourceID,
  },
  },
  },
@@ -117,21 +135,21 @@ func SchemaKubernetesAddOnProfiles() *schema.Schema {
  }
 }
 
-func ExpandKubernetesAddOnProfiles(input []interface{}) map[string]*containerservice.ManagedClusterAddonProfile {
+func expandKubernetesAddOnProfiles(input []interface{}, env azure.Environment) (*map[string]*containerservice.ManagedClusterAddonProfile, error) {
  disabled := containerservice.ManagedClusterAddonProfile{
  Enabled: utils.Bool(false),
  }
 
  profiles := map[string]*containerservice.ManagedClusterAddonProfile{
- // note: the casing on these keys is important
  aciConnectorKey: &disabled,
  azurePolicyKey: &disabled,
  kubernetesDashboardKey: &disabled,
  httpApplicationRoutingKey: &disabled,
  omsAgentKey: &disabled,
  }
+
  if len(input) == 0 {
- return profiles
+ return filterUnsupportedKubernetesAddOns(profiles, env)
  }
 
  profile := input[0].(map[string]interface{})
@@ -141,7 +159,7 @@ func ExpandKubernetesAddOnProfiles(input []interface{}) map[string]*containerser
  if len(httpApplicationRouting) > 0 && httpApplicationRouting[0] != nil {
  value := httpApplicationRouting[0].(map[string]interface{})
  enabled := value["enabled"].(bool)
- addonProfiles["httpApplicationRouting"] = &containerservice.ManagedClusterAddonProfile{
+ addonProfiles[httpApplicationRoutingKey] = &containerservice.ManagedClusterAddonProfile{
  Enabled: utils.Bool(enabled),
  }
  }
@@ -156,7 +174,7 @@ func ExpandKubernetesAddOnProfiles(input []interface{}) map[string]*containerser
  config["logAnalyticsWorkspaceResourceID"] = utils.String(workspaceId.(string))
  }
 
- addonProfiles["omsagent"] = &containerservice.ManagedClusterAddonProfile{
+ addonProfiles[omsAgentKey] = &containerservice.ManagedClusterAddonProfile{
  Enabled: utils.Bool(enabled),
  Config: config,
  }
@@ -172,7 +190,7 @@ func ExpandKubernetesAddOnProfiles(input []interface{}) map[string]*containerser
  config["SubnetName"] = utils.String(subnetName.(string))
  }
 
- addonProfiles["aciConnectorLinux"] = &containerservice.ManagedClusterAddonProfile{
+ addonProfiles[aciConnectorKey] = &containerservice.ManagedClusterAddonProfile{
  Enabled: utils.Bool(enabled),
  Config: config,
  }
@@ -183,7 +201,7 @@ func ExpandKubernetesAddOnProfiles(input []interface{}) map[string]*containerser
  value := kubeDashboard[0].(map[string]interface{})
  enabled := value["enabled"].(bool)
 
- addonProfiles["kubeDashboard"] = &containerservice.ManagedClusterAddonProfile{
+ addonProfiles[kubernetesDashboardKey] = &containerservice.ManagedClusterAddonProfile{
  Enabled: utils.Bool(enabled),
  Config: nil,
  }
@@ -194,16 +212,45 @@ func ExpandKubernetesAddOnProfiles(input []interface{}) map[string]*containerser
  value := azurePolicy[0].(map[string]interface{})
  enabled := value["enabled"].(bool)
 
- addonProfiles["azurepolicy"] = &containerservice.ManagedClusterAddonProfile{
+ addonProfiles[azurePolicyKey] = &containerservice.ManagedClusterAddonProfile{
  Enabled: utils.Bool(enabled),
  Config: nil,
  }
  }
 
- return addonProfiles
+ return filterUnsupportedKubernetesAddOns(addonProfiles, env)
+}
+
+func filterUnsupportedKubernetesAddOns(input map[string]*containerservice.ManagedClusterAddonProfile, env azure.Environment) (*map[string]*containerservice.ManagedClusterAddonProfile, error) {
+ var filter = func(input map[string]*containerservice.ManagedClusterAddonProfile, key string) (*map[string]*containerservice.ManagedClusterAddonProfile, error) {
+ output := input
+ if v, ok := output[key]; ok {
+ if v.Enabled != nil && *v.Enabled {
+ return nil, fmt.Errorf("The addon %q is not supported for a Kubernetes Cluster located in %q", key, env.Name)
+ }
+
+ // otherwise it's disabled by default, so just remove it
+ delete(output, key)
+ }
+
+ return &output, nil
+ }
+
+ output := input
+ if unsupportedAddons, ok := unsupportedAddonsForEnvironment[env.Name]; ok {
+ for _, key := range unsupportedAddons {
+ out, err := filter(output, key)
+ if err != nil {
+ return nil, err
+ }
+
+ output = *out
+ }
+ }
+ return &output, nil
 }
 
-func FlattenKubernetesAddOnProfiles(profile map[string]*containerservice.ManagedClusterAddonProfile) []interface{} {
+func flattenKubernetesAddOnProfiles(profile map[string]*containerservice.ManagedClusterAddonProfile) []interface{} {
  // when the Kubernetes Cluster is updated in the Portal - Azure updates the casing on the keys
  // meaning what's submitted could be different to what's returned..
  var locateInProfile = func(key string) *containerservice.ManagedClusterAddonProfile {

azurerm/internal/services/containers/resource_arm_kubernetes_cluster.go

@@ -72,7 +72,7 @@ func resourceArmKubernetesCluster() *schema.Resource {
  "default_node_pool": SchemaDefaultNodePool(),
 
  // Optional
- "addon_profile": SchemaKubernetesAddOnProfiles(),
+ "addon_profile": schemaKubernetesAddOnProfiles(),
 
  "api_server_authorized_ip_ranges": {
  Type: schema.TypeSet,
@@ -485,6 +485,7 @@ func resourceArmKubernetesCluster() *schema.Resource {
 
 func resourceArmKubernetesClusterCreate(d *schema.ResourceData, meta interface{}) error {
  client := meta.(*clients.Client).Containers.KubernetesClustersClient
+ env := meta.(*clients.Client).Containers.Environment
  ctx, cancel := timeouts.ForCreate(meta.(*clients.Client).StopContext, d)
  defer cancel()
  tenantId := meta.(*clients.Client).Account.TenantId
@@ -524,7 +525,10 @@ func resourceArmKubernetesClusterCreate(d *schema.ResourceData, meta interface{}
  }
 
  addOnProfilesRaw := d.Get("addon_profile").([]interface{})
- addonProfiles := ExpandKubernetesAddOnProfiles(addOnProfilesRaw)
+ addonProfiles, err := expandKubernetesAddOnProfiles(addOnProfilesRaw, env)
+ if err != nil {
+ return err
+ }
 
  networkProfileRaw := d.Get("network_profile").([]interface{})
  networkProfile, err := expandKubernetesClusterNetworkProfile(networkProfileRaw)
@@ -560,7 +564,7 @@ func resourceArmKubernetesClusterCreate(d *schema.ResourceData, meta interface{}
  ManagedClusterProperties: &containerservice.ManagedClusterProperties{
  APIServerAccessProfile: &apiAccessProfile,
  AadProfile: azureADProfile,
- AddonProfiles: addonProfiles,
+ AddonProfiles: *addonProfiles,
  AgentPoolProfiles: agentProfiles,
  DNSPrefix: utils.String(dnsPrefix),
  EnableRBAC: utils.Bool(rbacEnabled),
@@ -622,6 +626,7 @@ func resourceArmKubernetesClusterCreate(d *schema.ResourceData, meta interface{}
 func resourceArmKubernetesClusterUpdate(d *schema.ResourceData, meta interface{}) error {
  nodePoolsClient := meta.(*clients.Client).Containers.AgentPoolsClient
  clusterClient := meta.(*clients.Client).Containers.KubernetesClustersClient
+ env := meta.(*clients.Client).Containers.Environment
  ctx, cancel := timeouts.ForUpdate(meta.(*clients.Client).StopContext, d)
  defer cancel()
  tenantId := meta.(*clients.Client).Account.TenantId
@@ -715,8 +720,12 @@ func resourceArmKubernetesClusterUpdate(d *schema.ResourceData, meta interface{}
  if d.HasChange("addon_profile") {
  updateCluster = true
  addOnProfilesRaw := d.Get("addon_profile").([]interface{})
- addonProfiles := ExpandKubernetesAddOnProfiles(addOnProfilesRaw)
- existing.ManagedClusterProperties.AddonProfiles = addonProfiles
+ addonProfiles, err := expandKubernetesAddOnProfiles(addOnProfilesRaw, env)
+ if err != nil {
+ return err
+ }
+
+ existing.ManagedClusterProperties.AddonProfiles = *addonProfiles
  }
 
  if d.HasChange("api_server_authorized_ip_ranges") {
@@ -887,7 +896,7 @@ func resourceArmKubernetesClusterRead(d *schema.ResourceData, meta interface{})
  d.Set("private_link_enabled", accessProfile.EnablePrivateCluster)
  }
 
- addonProfiles := FlattenKubernetesAddOnProfiles(props.AddonProfiles)
+ addonProfiles := flattenKubernetesAddOnProfiles(props.AddonProfiles)
  if err := d.Set("addon_profile", addonProfiles); err != nil {
  return fmt.Errorf("setting `addon_profile`: %+v", err)
  }

website/allowed-subcategories

@@ -1,7 +1,6 @@
 API Management
 Analysis Services
 App Configuration
-App Platform
 App Service (Web Apps)
 Application Insights
 Authorization
@@ -52,6 +51,7 @@ Redis
 Search
 Security Center
 Service Fabric
+Spring Cloud
 Storage
 Stream Analytics
 Template

website/docs/r/kubernetes_cluster.html.markdown

@@ -146,12 +146,16 @@ A `addon_profile` block supports the following:
 
 * `aci_connector_linux` - (Optional) A `aci_connector_linux` block. For more details, please visit [Create and configure an AKS cluster to use virtual nodes](https://docs.microsoft.com/en-us/azure/aks/virtual-nodes-portal).
 
+-> **NOTE:** At this time ACI Connector's are not supported in Azure China.
+
 * `azure_policy` - (Optional) A `azure_policy` block as defined below. For more details please visit [Understand Azure Policy for Azure Kubernetes Service](https://docs.microsoft.com/en-ie/azure/governance/policy/concepts/rego-for-aks)
 
 -> **NOTE**: Azure Policy for Azure Kubernetes Service is currently in preview and not available to subscriptions that have not [opted-in](https://docs.microsoft.com/en-us/azure/governance/policy/concepts/rego-for-aks?toc=/azure/aks/toc.json) to join `Azure Policy` preview.
 
 * `http_application_routing` - (Optional) A `http_application_routing` block as defined below.
 
+-> **NOTE:** At this time HTTP Application Routing is not supported in Azure China or Azure US Government.
+
 * `kube_dashboard` - (Optional) A `kube_dashboard` block as defined below.
 
 * `oms_agent` - (Optional) A `oms_agent` block as defined below. For more details, please visit [How to onboard Azure Monitor for containers](https://docs.microsoft.com/en-us/azure/monitoring/monitoring-container-insights-onboard).