diff --git a/docs/topics/clusterdefinitions.md b/docs/topics/clusterdefinitions.md index 43a99e9af65..f9db7830bb4 100644 --- a/docs/topics/clusterdefinitions.md +++ b/docs/topics/clusterdefinitions.md @@ -98,6 +98,8 @@ $ aks-engine get-versions | cloudProviderRateLimitQPSWrite | no | QPS for Azure cloudprovider write request rate limiter enforcement. Follows the same defaults calculation as cloudProviderRateLimitQPS. | | cloudProviderDisableOutboundSNAT | no | For clusters w/ Standard LB only: enforces the disabling of outbound NAT for that load balancing rule. See [here](https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-outbound-rules-overview#disablesnat) for more details. Defaults to `false`. | | microsoftAptRepositoryURL | no | You may configure certain Microsoft-curated apt packages to be sourced from a custom repository so long as it acts as a mirror to the data at "https://packages.microsoft.com/" (the default value). | +| enableMultipleStandardLoadBalancers | no | Using multiple standard load balancers per cluster. The `loadBalancerSku` must be `standard`. Default to false. | +| tags | no | Specify the tags which will be applied to all of the resources managed by the cloud provider, with the format `a=b,c=d`. | #### addons diff --git a/parts/k8s/cloud-init/artifacts/cse_config.sh b/parts/k8s/cloud-init/artifacts/cse_config.sh index 8ddb23ed1e5..294787336ea 100755 --- a/parts/k8s/cloud-init/artifacts/cse_config.sh +++ b/parts/k8s/cloud-init/artifacts/cse_config.sh @@ -212,7 +212,9 @@ configureK8s() { "providerVaultName": "${KMS_PROVIDER_VAULT_NAME}", "maximumLoadBalancerRuleCount": ${MAXIMUM_LOADBALANCER_RULE_COUNT}, "providerKeyName": "k8s", - "providerKeyVersion": "" + "providerKeyVersion": "", + "enableMultipleStandardLoadBalancers": ${ENABLE_MULTIPLE_STANDARD_LOAD_BALANCERS}, + "tags": "${TAGS}" } EOF set -x diff --git a/pkg/api/convertertoapi.go b/pkg/api/convertertoapi.go index 7d09f4cb88c..d24ea05afb6 100644 --- a/pkg/api/convertertoapi.go +++ b/pkg/api/convertertoapi.go @@ -315,6 +315,8 @@ func convertVLabsKubernetesConfig(vlabs *vlabs.KubernetesConfig, api *Kubernetes api.CloudProviderDisableOutboundSNAT = vlabs.CloudProviderDisableOutboundSNAT api.KubeReservedCgroup = vlabs.KubeReservedCgroup api.MicrosoftAptRepositoryURL = vlabs.MicrosoftAptRepositoryURL + api.EnableMultipleStandardLoadBalancers = vlabs.EnableMultipleStandardLoadBalancers + api.Tags = vlabs.Tags convertComponentsToAPI(vlabs, api) convertAddonsToAPI(vlabs, api) convertKubeletConfigToAPI(vlabs, api) diff --git a/pkg/api/types.go b/pkg/api/types.go index f4d7d3ad3b0..a4e0f71d565 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -386,92 +386,94 @@ const ( // KubernetesConfig contains the Kubernetes config structure, containing // Kubernetes specific configuration type KubernetesConfig struct { - KubernetesImageBase string `json:"kubernetesImageBase,omitempty"` - KubernetesImageBaseType string `json:"kubernetesImageBaseType,omitempty"` - MCRKubernetesImageBase string `json:"mcrKubernetesImageBase,omitempty"` - ClusterSubnet string `json:"clusterSubnet,omitempty"` - NetworkPolicy string `json:"networkPolicy,omitempty"` - NetworkPlugin string `json:"networkPlugin,omitempty"` - NetworkMode string `json:"networkMode,omitempty"` - ContainerRuntime string `json:"containerRuntime,omitempty"` - MaxPods int `json:"maxPods,omitempty"` - DockerBridgeSubnet string `json:"dockerBridgeSubnet,omitempty"` - DNSServiceIP string `json:"dnsServiceIP,omitempty"` - ServiceCIDR string `json:"serviceCidr,omitempty"` - UseManagedIdentity *bool `json:"useManagedIdentity,omitempty"` - UserAssignedID string `json:"userAssignedID,omitempty"` - UserAssignedClientID string `json:"userAssignedClientID,omitempty"` //Note: cannot be provided in config. Used *only* for transferring this to azure.json. - CustomHyperkubeImage string `json:"customHyperkubeImage,omitempty"` - CustomKubeAPIServerImage string `json:"customKubeAPIServerImage,omitempty"` - CustomKubeControllerManagerImage string `json:"customKubeControllerManagerImage,omitempty"` - CustomKubeProxyImage string `json:"customKubeProxyImage,omitempty"` - CustomKubeSchedulerImage string `json:"customKubeSchedulerImage,omitempty"` - CustomKubeBinaryURL string `json:"customKubeBinaryURL,omitempty"` - DockerEngineVersion string `json:"dockerEngineVersion,omitempty"` // Deprecated - MobyVersion string `json:"mobyVersion,omitempty"` - LinuxMobyURL string `json:"linuxMobyURL,omitempty"` - ContainerdVersion string `json:"containerdVersion,omitempty"` - LinuxContainerdURL string `json:"linuxContainerdURL,omitempty"` - CustomCcmImage string `json:"customCcmImage,omitempty"` // Image for cloud-controller-manager - UseCloudControllerManager *bool `json:"useCloudControllerManager,omitempty"` - CustomWindowsPackageURL string `json:"customWindowsPackageURL,omitempty"` - WindowsNodeBinariesURL string `json:"windowsNodeBinariesURL,omitempty"` - WindowsContainerdURL string `json:"windowsContainerdURL,omitempty"` - WindowsSdnPluginURL string `json:"windowsSdnPluginURL,omitempty"` - UseInstanceMetadata *bool `json:"useInstanceMetadata,omitempty"` - EnableRbac *bool `json:"enableRbac,omitempty"` - EnableSecureKubelet *bool `json:"enableSecureKubelet,omitempty"` - EnableAggregatedAPIs bool `json:"enableAggregatedAPIs,omitempty"` - PrivateCluster *PrivateCluster `json:"privateCluster,omitempty"` - GCHighThreshold int `json:"gchighthreshold,omitempty"` - GCLowThreshold int `json:"gclowthreshold,omitempty"` - EtcdVersion string `json:"etcdVersion,omitempty"` - EtcdDiskSizeGB string `json:"etcdDiskSizeGB,omitempty"` - EtcdStorageLimitGB int `json:"etcdStorageLimitGB,omitempty"` - EtcdEncryptionKey string `json:"etcdEncryptionKey,omitempty"` - EnableDataEncryptionAtRest *bool `json:"enableDataEncryptionAtRest,omitempty"` - EnableEncryptionWithExternalKms *bool `json:"enableEncryptionWithExternalKms,omitempty"` - EnablePodSecurityPolicy *bool `json:"enablePodSecurityPolicy,omitempty"` - Addons []KubernetesAddon `json:"addons,omitempty"` - Components []KubernetesComponent `json:"components,omitempty"` - KubeletConfig map[string]string `json:"kubeletConfig,omitempty"` - ContainerRuntimeConfig map[string]string `json:"containerRuntimeConfig"` - ControllerManagerConfig map[string]string `json:"controllerManagerConfig,omitempty"` - CloudControllerManagerConfig map[string]string `json:"cloudControllerManagerConfig,omitempty"` - APIServerConfig map[string]string `json:"apiServerConfig,omitempty"` - SchedulerConfig map[string]string `json:"schedulerConfig,omitempty"` - PodSecurityPolicyConfig map[string]string `json:"podSecurityPolicyConfig,omitempty"` // Deprecated - KubeReservedCgroup string `json:"kubeReservedCgroup,omitempty"` - CloudProviderBackoffMode string `json:"cloudProviderBackoffMode"` - CloudProviderBackoff *bool `json:"cloudProviderBackoff,omitempty"` - CloudProviderBackoffRetries int `json:"cloudProviderBackoffRetries,omitempty"` - CloudProviderBackoffJitter float64 `json:"cloudProviderBackoffJitter,omitempty"` - CloudProviderBackoffDuration int `json:"cloudProviderBackoffDuration,omitempty"` - CloudProviderBackoffExponent float64 `json:"cloudProviderBackoffExponent,omitempty"` - CloudProviderRateLimit *bool `json:"cloudProviderRateLimit,omitempty"` - CloudProviderRateLimitQPS float64 `json:"cloudProviderRateLimitQPS,omitempty"` - CloudProviderRateLimitQPSWrite float64 `json:"cloudProviderRateLimitQPSWrite,omitempty"` - CloudProviderRateLimitBucket int `json:"cloudProviderRateLimitBucket,omitempty"` - CloudProviderRateLimitBucketWrite int `json:"cloudProviderRateLimitBucketWrite,omitempty"` - CloudProviderDisableOutboundSNAT *bool `json:"cloudProviderDisableOutboundSNAT,omitempty"` - NonMasqueradeCidr string `json:"nonMasqueradeCidr,omitempty"` - NodeStatusUpdateFrequency string `json:"nodeStatusUpdateFrequency,omitempty"` - HardEvictionThreshold string `json:"hardEvictionThreshold,omitempty"` - CtrlMgrNodeMonitorGracePeriod string `json:"ctrlMgrNodeMonitorGracePeriod,omitempty"` - CtrlMgrPodEvictionTimeout string `json:"ctrlMgrPodEvictionTimeout,omitempty"` - CtrlMgrRouteReconciliationPeriod string `json:"ctrlMgrRouteReconciliationPeriod,omitempty"` - LoadBalancerSku string `json:"loadBalancerSku,omitempty"` - ExcludeMasterFromStandardLB *bool `json:"excludeMasterFromStandardLB,omitempty"` - LoadBalancerOutboundIPs *int `json:"loadBalancerOutboundIPs,omitempty"` - AzureCNIVersion string `json:"azureCNIVersion,omitempty"` - AzureCNIURLLinux string `json:"azureCNIURLLinux,omitempty"` - AzureCNIURLWindows string `json:"azureCNIURLWindows,omitempty"` - KeyVaultSku string `json:"keyVaultSku,omitempty"` - MaximumLoadBalancerRuleCount int `json:"maximumLoadBalancerRuleCount,omitempty"` - ProxyMode KubeProxyMode `json:"kubeProxyMode,omitempty"` - PrivateAzureRegistryServer string `json:"privateAzureRegistryServer,omitempty"` - OutboundRuleIdleTimeoutInMinutes int32 `json:"outboundRuleIdleTimeoutInMinutes,omitempty"` - MicrosoftAptRepositoryURL string `json:"microsoftAptRepositoryURL,omitempty"` + KubernetesImageBase string `json:"kubernetesImageBase,omitempty"` + KubernetesImageBaseType string `json:"kubernetesImageBaseType,omitempty"` + MCRKubernetesImageBase string `json:"mcrKubernetesImageBase,omitempty"` + ClusterSubnet string `json:"clusterSubnet,omitempty"` + NetworkPolicy string `json:"networkPolicy,omitempty"` + NetworkPlugin string `json:"networkPlugin,omitempty"` + NetworkMode string `json:"networkMode,omitempty"` + ContainerRuntime string `json:"containerRuntime,omitempty"` + MaxPods int `json:"maxPods,omitempty"` + DockerBridgeSubnet string `json:"dockerBridgeSubnet,omitempty"` + DNSServiceIP string `json:"dnsServiceIP,omitempty"` + ServiceCIDR string `json:"serviceCidr,omitempty"` + UseManagedIdentity *bool `json:"useManagedIdentity,omitempty"` + UserAssignedID string `json:"userAssignedID,omitempty"` + UserAssignedClientID string `json:"userAssignedClientID,omitempty"` //Note: cannot be provided in config. Used *only* for transferring this to azure.json. + CustomHyperkubeImage string `json:"customHyperkubeImage,omitempty"` + CustomKubeAPIServerImage string `json:"customKubeAPIServerImage,omitempty"` + CustomKubeControllerManagerImage string `json:"customKubeControllerManagerImage,omitempty"` + CustomKubeProxyImage string `json:"customKubeProxyImage,omitempty"` + CustomKubeSchedulerImage string `json:"customKubeSchedulerImage,omitempty"` + CustomKubeBinaryURL string `json:"customKubeBinaryURL,omitempty"` + DockerEngineVersion string `json:"dockerEngineVersion,omitempty"` // Deprecated + MobyVersion string `json:"mobyVersion,omitempty"` + LinuxMobyURL string `json:"linuxMobyURL,omitempty"` + ContainerdVersion string `json:"containerdVersion,omitempty"` + LinuxContainerdURL string `json:"linuxContainerdURL,omitempty"` + CustomCcmImage string `json:"customCcmImage,omitempty"` // Image for cloud-controller-manager + UseCloudControllerManager *bool `json:"useCloudControllerManager,omitempty"` + CustomWindowsPackageURL string `json:"customWindowsPackageURL,omitempty"` + WindowsNodeBinariesURL string `json:"windowsNodeBinariesURL,omitempty"` + WindowsContainerdURL string `json:"windowsContainerdURL,omitempty"` + WindowsSdnPluginURL string `json:"windowsSdnPluginURL,omitempty"` + UseInstanceMetadata *bool `json:"useInstanceMetadata,omitempty"` + EnableRbac *bool `json:"enableRbac,omitempty"` + EnableSecureKubelet *bool `json:"enableSecureKubelet,omitempty"` + EnableAggregatedAPIs bool `json:"enableAggregatedAPIs,omitempty"` + PrivateCluster *PrivateCluster `json:"privateCluster,omitempty"` + GCHighThreshold int `json:"gchighthreshold,omitempty"` + GCLowThreshold int `json:"gclowthreshold,omitempty"` + EtcdVersion string `json:"etcdVersion,omitempty"` + EtcdDiskSizeGB string `json:"etcdDiskSizeGB,omitempty"` + EtcdStorageLimitGB int `json:"etcdStorageLimitGB,omitempty"` + EtcdEncryptionKey string `json:"etcdEncryptionKey,omitempty"` + EnableDataEncryptionAtRest *bool `json:"enableDataEncryptionAtRest,omitempty"` + EnableEncryptionWithExternalKms *bool `json:"enableEncryptionWithExternalKms,omitempty"` + EnablePodSecurityPolicy *bool `json:"enablePodSecurityPolicy,omitempty"` + Addons []KubernetesAddon `json:"addons,omitempty"` + Components []KubernetesComponent `json:"components,omitempty"` + KubeletConfig map[string]string `json:"kubeletConfig,omitempty"` + ContainerRuntimeConfig map[string]string `json:"containerRuntimeConfig"` + ControllerManagerConfig map[string]string `json:"controllerManagerConfig,omitempty"` + CloudControllerManagerConfig map[string]string `json:"cloudControllerManagerConfig,omitempty"` + APIServerConfig map[string]string `json:"apiServerConfig,omitempty"` + SchedulerConfig map[string]string `json:"schedulerConfig,omitempty"` + PodSecurityPolicyConfig map[string]string `json:"podSecurityPolicyConfig,omitempty"` // Deprecated + KubeReservedCgroup string `json:"kubeReservedCgroup,omitempty"` + CloudProviderBackoffMode string `json:"cloudProviderBackoffMode"` + CloudProviderBackoff *bool `json:"cloudProviderBackoff,omitempty"` + CloudProviderBackoffRetries int `json:"cloudProviderBackoffRetries,omitempty"` + CloudProviderBackoffJitter float64 `json:"cloudProviderBackoffJitter,omitempty"` + CloudProviderBackoffDuration int `json:"cloudProviderBackoffDuration,omitempty"` + CloudProviderBackoffExponent float64 `json:"cloudProviderBackoffExponent,omitempty"` + CloudProviderRateLimit *bool `json:"cloudProviderRateLimit,omitempty"` + CloudProviderRateLimitQPS float64 `json:"cloudProviderRateLimitQPS,omitempty"` + CloudProviderRateLimitQPSWrite float64 `json:"cloudProviderRateLimitQPSWrite,omitempty"` + CloudProviderRateLimitBucket int `json:"cloudProviderRateLimitBucket,omitempty"` + CloudProviderRateLimitBucketWrite int `json:"cloudProviderRateLimitBucketWrite,omitempty"` + CloudProviderDisableOutboundSNAT *bool `json:"cloudProviderDisableOutboundSNAT,omitempty"` + NonMasqueradeCidr string `json:"nonMasqueradeCidr,omitempty"` + NodeStatusUpdateFrequency string `json:"nodeStatusUpdateFrequency,omitempty"` + HardEvictionThreshold string `json:"hardEvictionThreshold,omitempty"` + CtrlMgrNodeMonitorGracePeriod string `json:"ctrlMgrNodeMonitorGracePeriod,omitempty"` + CtrlMgrPodEvictionTimeout string `json:"ctrlMgrPodEvictionTimeout,omitempty"` + CtrlMgrRouteReconciliationPeriod string `json:"ctrlMgrRouteReconciliationPeriod,omitempty"` + LoadBalancerSku string `json:"loadBalancerSku,omitempty"` + ExcludeMasterFromStandardLB *bool `json:"excludeMasterFromStandardLB,omitempty"` + LoadBalancerOutboundIPs *int `json:"loadBalancerOutboundIPs,omitempty"` + AzureCNIVersion string `json:"azureCNIVersion,omitempty"` + AzureCNIURLLinux string `json:"azureCNIURLLinux,omitempty"` + AzureCNIURLWindows string `json:"azureCNIURLWindows,omitempty"` + KeyVaultSku string `json:"keyVaultSku,omitempty"` + MaximumLoadBalancerRuleCount int `json:"maximumLoadBalancerRuleCount,omitempty"` + ProxyMode KubeProxyMode `json:"kubeProxyMode,omitempty"` + PrivateAzureRegistryServer string `json:"privateAzureRegistryServer,omitempty"` + OutboundRuleIdleTimeoutInMinutes int32 `json:"outboundRuleIdleTimeoutInMinutes,omitempty"` + MicrosoftAptRepositoryURL string `json:"microsoftAptRepositoryURL,omitempty"` + EnableMultipleStandardLoadBalancers *bool `json:"enableMultipleStandardLoadBalancers,omitempty"` + Tags string `json:"tags,omitempty"` } // CustomFile has source as the full absolute source path to a file and dest @@ -2182,61 +2184,63 @@ func (cs *ContainerService) GetProvisionScriptParametersCommon(input ProvisionSc cloudSpecConfig := cs.GetCloudSpecConfig() kubernetesConfig := cs.Properties.OrchestratorProfile.KubernetesConfig parameters := map[string]string{ - "ADMINUSER": cs.Properties.LinuxProfile.AdminUsername, - "ETCD_DOWNLOAD_URL": cloudSpecConfig.KubernetesSpecConfig.EtcdDownloadURLBase, - "ETCD_VERSION": kubernetesConfig.EtcdVersion, - "CONTAINERD_VERSION": kubernetesConfig.ContainerdVersion, - "MOBY_VERSION": kubernetesConfig.MobyVersion, - "TENANT_ID": input.TenantID, - "KUBERNETES_VERSION": cs.Properties.GetKubernetesVersion(), - "HYPERKUBE_URL": cs.Properties.GetKubernetesHyperkubeSpec(), - "APISERVER_PUBLIC_KEY": input.APIServerCertificate, - "SUBSCRIPTION_ID": input.SubscriptionID, - "RESOURCE_GROUP": input.ResourceGroup, - "LOCATION": input.Location, - "VM_TYPE": cs.Properties.GetVMType(), - "SUBNET": cs.Properties.GetSubnetName(), - "NETWORK_SECURITY_GROUP": cs.Properties.GetNSGName(), - "VIRTUAL_NETWORK": cs.Properties.GetVirtualNetworkName(), - "VIRTUAL_NETWORK_RESOURCE_GROUP": cs.Properties.GetVNetResourceGroupName(), - "ROUTE_TABLE": cs.Properties.GetRouteTableName(), - "PRIMARY_AVAILABILITY_SET": cs.Properties.GetPrimaryAvailabilitySetName(), - "PRIMARY_SCALE_SET": cs.Properties.GetPrimaryScaleSetName(), - "SERVICE_PRINCIPAL_CLIENT_ID": input.ClientID, - "SERVICE_PRINCIPAL_CLIENT_SECRET": input.ClientSecret, - "KUBELET_PRIVATE_KEY": input.KubeletPrivateKey, - "NETWORK_PLUGIN": kubernetesConfig.NetworkPlugin, - "NETWORK_POLICY": kubernetesConfig.NetworkPolicy, - "VNET_CNI_PLUGINS_URL": kubernetesConfig.GetAzureCNIURLLinux(cloudSpecConfig), - "CNI_PLUGINS_URL": cloudSpecConfig.KubernetesSpecConfig.CNIPluginsDownloadURL, - "CLOUDPROVIDER_BACKOFF": strconv.FormatBool(to.Bool(kubernetesConfig.CloudProviderBackoff)), - "CLOUDPROVIDER_BACKOFF_MODE": kubernetesConfig.CloudProviderBackoffMode, - "CLOUDPROVIDER_BACKOFF_RETRIES": strconv.Itoa(kubernetesConfig.CloudProviderBackoffRetries), - "CLOUDPROVIDER_BACKOFF_EXPONENT": strconv.FormatFloat(kubernetesConfig.CloudProviderBackoffExponent, 'f', -1, 64), - "CLOUDPROVIDER_BACKOFF_DURATION": strconv.Itoa(kubernetesConfig.CloudProviderBackoffDuration), - "CLOUDPROVIDER_BACKOFF_JITTER": strconv.FormatFloat(kubernetesConfig.CloudProviderBackoffJitter, 'f', -1, 64), - "CLOUDPROVIDER_RATELIMIT": strconv.FormatBool(to.Bool(kubernetesConfig.CloudProviderRateLimit)), - "CLOUDPROVIDER_RATELIMIT_QPS": strconv.FormatFloat(kubernetesConfig.CloudProviderRateLimitQPS, 'f', -1, 64), - "CLOUDPROVIDER_RATELIMIT_QPS_WRITE": strconv.FormatFloat(kubernetesConfig.CloudProviderRateLimitQPSWrite, 'f', -1, 64), - "CLOUDPROVIDER_RATELIMIT_BUCKET": strconv.Itoa(kubernetesConfig.CloudProviderRateLimitBucket), - "CLOUDPROVIDER_RATELIMIT_BUCKET_WRITE": strconv.Itoa(kubernetesConfig.CloudProviderRateLimitBucketWrite), - "LOAD_BALANCER_DISABLE_OUTBOUND_SNAT": strconv.FormatBool(to.Bool(kubernetesConfig.CloudProviderDisableOutboundSNAT)), - "USE_MANAGED_IDENTITY_EXTENSION": strconv.FormatBool(to.Bool(kubernetesConfig.UseManagedIdentity)), - "USE_INSTANCE_METADATA": strconv.FormatBool(to.Bool(kubernetesConfig.UseInstanceMetadata)), - "LOAD_BALANCER_SKU": kubernetesConfig.LoadBalancerSku, - "EXCLUDE_MASTER_FROM_STANDARD_LB": strconv.FormatBool(to.Bool(kubernetesConfig.ExcludeMasterFromStandardLB)), - "MAXIMUM_LOADBALANCER_RULE_COUNT": strconv.Itoa(kubernetesConfig.MaximumLoadBalancerRuleCount), - "CONTAINER_RUNTIME": kubernetesConfig.ContainerRuntime, - "CONTAINERD_DOWNLOAD_URL_BASE": cloudSpecConfig.KubernetesSpecConfig.ContainerdDownloadURLBase, - "KMS_PROVIDER_VAULT_NAME": input.ClusterKeyVaultName, - "IS_IPV6_ENABLED": strconv.FormatBool(cs.Properties.FeatureFlags.IsFeatureEnabled("EnableIPv6Only") || cs.Properties.FeatureFlags.IsFeatureEnabled("EnableIPv6DualStack")), - "AUTHENTICATION_METHOD": cs.Properties.GetCustomCloudAuthenticationMethod(), - "IDENTITY_SYSTEM": cs.Properties.GetCustomCloudIdentitySystem(), - "NETWORK_API_VERSION": APIVersionNetwork, - "NETWORK_MODE": kubernetesConfig.NetworkMode, - "KUBE_BINARY_URL": kubernetesConfig.CustomKubeBinaryURL, - "CUSTOM_HYPERKUBE_IMAGE": kubernetesConfig.CustomHyperkubeImage, - "MS_APT_REPO": kubernetesConfig.MicrosoftAptRepositoryURL, + "ADMINUSER": cs.Properties.LinuxProfile.AdminUsername, + "ETCD_DOWNLOAD_URL": cloudSpecConfig.KubernetesSpecConfig.EtcdDownloadURLBase, + "ETCD_VERSION": kubernetesConfig.EtcdVersion, + "CONTAINERD_VERSION": kubernetesConfig.ContainerdVersion, + "MOBY_VERSION": kubernetesConfig.MobyVersion, + "TENANT_ID": input.TenantID, + "KUBERNETES_VERSION": cs.Properties.GetKubernetesVersion(), + "HYPERKUBE_URL": cs.Properties.GetKubernetesHyperkubeSpec(), + "APISERVER_PUBLIC_KEY": input.APIServerCertificate, + "SUBSCRIPTION_ID": input.SubscriptionID, + "RESOURCE_GROUP": input.ResourceGroup, + "LOCATION": input.Location, + "VM_TYPE": cs.Properties.GetVMType(), + "SUBNET": cs.Properties.GetSubnetName(), + "NETWORK_SECURITY_GROUP": cs.Properties.GetNSGName(), + "VIRTUAL_NETWORK": cs.Properties.GetVirtualNetworkName(), + "VIRTUAL_NETWORK_RESOURCE_GROUP": cs.Properties.GetVNetResourceGroupName(), + "ROUTE_TABLE": cs.Properties.GetRouteTableName(), + "PRIMARY_AVAILABILITY_SET": cs.Properties.GetPrimaryAvailabilitySetName(), + "PRIMARY_SCALE_SET": cs.Properties.GetPrimaryScaleSetName(), + "SERVICE_PRINCIPAL_CLIENT_ID": input.ClientID, + "SERVICE_PRINCIPAL_CLIENT_SECRET": input.ClientSecret, + "KUBELET_PRIVATE_KEY": input.KubeletPrivateKey, + "NETWORK_PLUGIN": kubernetesConfig.NetworkPlugin, + "NETWORK_POLICY": kubernetesConfig.NetworkPolicy, + "VNET_CNI_PLUGINS_URL": kubernetesConfig.GetAzureCNIURLLinux(cloudSpecConfig), + "CNI_PLUGINS_URL": cloudSpecConfig.KubernetesSpecConfig.CNIPluginsDownloadURL, + "CLOUDPROVIDER_BACKOFF": strconv.FormatBool(to.Bool(kubernetesConfig.CloudProviderBackoff)), + "CLOUDPROVIDER_BACKOFF_MODE": kubernetesConfig.CloudProviderBackoffMode, + "CLOUDPROVIDER_BACKOFF_RETRIES": strconv.Itoa(kubernetesConfig.CloudProviderBackoffRetries), + "CLOUDPROVIDER_BACKOFF_EXPONENT": strconv.FormatFloat(kubernetesConfig.CloudProviderBackoffExponent, 'f', -1, 64), + "CLOUDPROVIDER_BACKOFF_DURATION": strconv.Itoa(kubernetesConfig.CloudProviderBackoffDuration), + "CLOUDPROVIDER_BACKOFF_JITTER": strconv.FormatFloat(kubernetesConfig.CloudProviderBackoffJitter, 'f', -1, 64), + "CLOUDPROVIDER_RATELIMIT": strconv.FormatBool(to.Bool(kubernetesConfig.CloudProviderRateLimit)), + "CLOUDPROVIDER_RATELIMIT_QPS": strconv.FormatFloat(kubernetesConfig.CloudProviderRateLimitQPS, 'f', -1, 64), + "CLOUDPROVIDER_RATELIMIT_QPS_WRITE": strconv.FormatFloat(kubernetesConfig.CloudProviderRateLimitQPSWrite, 'f', -1, 64), + "CLOUDPROVIDER_RATELIMIT_BUCKET": strconv.Itoa(kubernetesConfig.CloudProviderRateLimitBucket), + "CLOUDPROVIDER_RATELIMIT_BUCKET_WRITE": strconv.Itoa(kubernetesConfig.CloudProviderRateLimitBucketWrite), + "LOAD_BALANCER_DISABLE_OUTBOUND_SNAT": strconv.FormatBool(to.Bool(kubernetesConfig.CloudProviderDisableOutboundSNAT)), + "USE_MANAGED_IDENTITY_EXTENSION": strconv.FormatBool(to.Bool(kubernetesConfig.UseManagedIdentity)), + "USE_INSTANCE_METADATA": strconv.FormatBool(to.Bool(kubernetesConfig.UseInstanceMetadata)), + "LOAD_BALANCER_SKU": kubernetesConfig.LoadBalancerSku, + "EXCLUDE_MASTER_FROM_STANDARD_LB": strconv.FormatBool(to.Bool(kubernetesConfig.ExcludeMasterFromStandardLB)), + "MAXIMUM_LOADBALANCER_RULE_COUNT": strconv.Itoa(kubernetesConfig.MaximumLoadBalancerRuleCount), + "CONTAINER_RUNTIME": kubernetesConfig.ContainerRuntime, + "CONTAINERD_DOWNLOAD_URL_BASE": cloudSpecConfig.KubernetesSpecConfig.ContainerdDownloadURLBase, + "KMS_PROVIDER_VAULT_NAME": input.ClusterKeyVaultName, + "IS_IPV6_ENABLED": strconv.FormatBool(cs.Properties.FeatureFlags.IsFeatureEnabled("EnableIPv6Only") || cs.Properties.FeatureFlags.IsFeatureEnabled("EnableIPv6DualStack")), + "AUTHENTICATION_METHOD": cs.Properties.GetCustomCloudAuthenticationMethod(), + "IDENTITY_SYSTEM": cs.Properties.GetCustomCloudIdentitySystem(), + "NETWORK_API_VERSION": APIVersionNetwork, + "NETWORK_MODE": kubernetesConfig.NetworkMode, + "KUBE_BINARY_URL": kubernetesConfig.CustomKubeBinaryURL, + "CUSTOM_HYPERKUBE_IMAGE": kubernetesConfig.CustomHyperkubeImage, + "MS_APT_REPO": kubernetesConfig.MicrosoftAptRepositoryURL, + "TAGS": kubernetesConfig.Tags, + "ENABLE_MULTIPLE_STANDARD_LOAD_BALANCERS": strconv.FormatBool(to.Bool(kubernetesConfig.EnableMultipleStandardLoadBalancers)), } keys := make([]string, 0) diff --git a/pkg/api/types_test.go b/pkg/api/types_test.go index 4566f1422f4..fa10e7f30ee 100644 --- a/pkg/api/types_test.go +++ b/pkg/api/types_test.go @@ -6754,7 +6754,7 @@ func TestGetProvisionScriptParametersCommon(t *testing.T) { KubeletPrivateKey: "fakekubeletkey", ClusterKeyVaultName: "", }, - expected: "ADMINUSER=azureuser APISERVER_PUBLIC_KEY=fakecert AUTHENTICATION_METHOD=client_secret CLOUDPROVIDER_BACKOFF=false CLOUDPROVIDER_BACKOFF_DURATION=0 CLOUDPROVIDER_BACKOFF_EXPONENT=0 CLOUDPROVIDER_BACKOFF_JITTER=0 CLOUDPROVIDER_BACKOFF_MODE= CLOUDPROVIDER_BACKOFF_RETRIES=0 CLOUDPROVIDER_RATELIMIT=false CLOUDPROVIDER_RATELIMIT_BUCKET=0 CLOUDPROVIDER_RATELIMIT_BUCKET_WRITE=0 CLOUDPROVIDER_RATELIMIT_QPS=0 CLOUDPROVIDER_RATELIMIT_QPS_WRITE=0 CNI_PLUGINS_URL=https://kubernetesartifacts.azureedge.net/cni-plugins/" + CNIPluginVer + "/binaries/cni-plugins-linux-amd64-" + CNIPluginVer + ".tgz CONTAINERD_DOWNLOAD_URL_BASE=https://storage.googleapis.com/cri-containerd-release/ CONTAINERD_VERSION=" + DefaultContainerdVersion + " CONTAINER_RUNTIME=docker CUSTOM_HYPERKUBE_IMAGE= ETCD_DOWNLOAD_URL=mcr.microsoft.com/oss/etcd-io/ ETCD_VERSION=" + DefaultEtcdVersion + " EXCLUDE_MASTER_FROM_STANDARD_LB=false HYPERKUBE_URL=hyperkube-amd64:v1.16.8 IDENTITY_SYSTEM=azure_ad IS_IPV6_ENABLED=false KMS_PROVIDER_VAULT_NAME= KUBELET_PRIVATE_KEY=fakekubeletkey KUBERNETES_VERSION=1.16.8 KUBE_BINARY_URL= LOAD_BALANCER_DISABLE_OUTBOUND_SNAT=false LOAD_BALANCER_SKU=Basic LOCATION=westus MAXIMUM_LOADBALANCER_RULE_COUNT=0 MOBY_VERSION=" + DefaultMobyVersion + " MS_APT_REPO=" + DefaultMicrosoftAptRepositoryURL + " NETWORK_API_VERSION=2018-08-01 NETWORK_MODE= NETWORK_PLUGIN=kubenet NETWORK_POLICY= NETWORK_SECURITY_GROUP=" + common.LegacyControlPlaneVMPrefix + "-22998975-nsg PRIMARY_AVAILABILITY_SET=agentpool1-availabilitySet-22998975 PRIMARY_SCALE_SET= RESOURCE_GROUP=fakerg ROUTE_TABLE=" + common.LegacyControlPlaneVMPrefix + "-22998975-routetable SERVICE_PRINCIPAL_CLIENT_ID=fakeclientID SERVICE_PRINCIPAL_CLIENT_SECRET=fakeclientSecret SUBNET=k8s-subnet SUBSCRIPTION_ID=fakesubID TENANT_ID=faketenantID USE_INSTANCE_METADATA=false USE_MANAGED_IDENTITY_EXTENSION=false VIRTUAL_NETWORK=k8s-vnet-22998975 VIRTUAL_NETWORK_RESOURCE_GROUP= VM_TYPE=standard VNET_CNI_PLUGINS_URL=https://kubernetesartifacts.azureedge.net/azure-cni/" + AzureCniPluginVerLinux + "/binaries/azure-vnet-cni-linux-amd64-" + AzureCniPluginVerLinux + ".tgz ", + expected: "ADMINUSER=azureuser APISERVER_PUBLIC_KEY=fakecert AUTHENTICATION_METHOD=client_secret CLOUDPROVIDER_BACKOFF=false CLOUDPROVIDER_BACKOFF_DURATION=0 CLOUDPROVIDER_BACKOFF_EXPONENT=0 CLOUDPROVIDER_BACKOFF_JITTER=0 CLOUDPROVIDER_BACKOFF_MODE= CLOUDPROVIDER_BACKOFF_RETRIES=0 CLOUDPROVIDER_RATELIMIT=false CLOUDPROVIDER_RATELIMIT_BUCKET=0 CLOUDPROVIDER_RATELIMIT_BUCKET_WRITE=0 CLOUDPROVIDER_RATELIMIT_QPS=0 CLOUDPROVIDER_RATELIMIT_QPS_WRITE=0 CNI_PLUGINS_URL=https://kubernetesartifacts.azureedge.net/cni-plugins/" + CNIPluginVer + "/binaries/cni-plugins-linux-amd64-" + CNIPluginVer + ".tgz CONTAINERD_DOWNLOAD_URL_BASE=https://storage.googleapis.com/cri-containerd-release/ CONTAINERD_VERSION=" + DefaultContainerdVersion + " CONTAINER_RUNTIME=docker CUSTOM_HYPERKUBE_IMAGE= ENABLE_MULTIPLE_STANDARD_LOAD_BALANCERS=false ETCD_DOWNLOAD_URL=mcr.microsoft.com/oss/etcd-io/ ETCD_VERSION=" + DefaultEtcdVersion + " EXCLUDE_MASTER_FROM_STANDARD_LB=false HYPERKUBE_URL=hyperkube-amd64:v1.16.8 IDENTITY_SYSTEM=azure_ad IS_IPV6_ENABLED=false KMS_PROVIDER_VAULT_NAME= KUBELET_PRIVATE_KEY=fakekubeletkey KUBERNETES_VERSION=1.16.8 KUBE_BINARY_URL= LOAD_BALANCER_DISABLE_OUTBOUND_SNAT=false LOAD_BALANCER_SKU=Basic LOCATION=westus MAXIMUM_LOADBALANCER_RULE_COUNT=0 MOBY_VERSION=" + DefaultMobyVersion + " MS_APT_REPO=" + DefaultMicrosoftAptRepositoryURL + " NETWORK_API_VERSION=2018-08-01 NETWORK_MODE= NETWORK_PLUGIN=kubenet NETWORK_POLICY= NETWORK_SECURITY_GROUP=" + common.LegacyControlPlaneVMPrefix + "-22998975-nsg PRIMARY_AVAILABILITY_SET=agentpool1-availabilitySet-22998975 PRIMARY_SCALE_SET= RESOURCE_GROUP=fakerg ROUTE_TABLE=" + common.LegacyControlPlaneVMPrefix + "-22998975-routetable SERVICE_PRINCIPAL_CLIENT_ID=fakeclientID SERVICE_PRINCIPAL_CLIENT_SECRET=fakeclientSecret SUBNET=k8s-subnet SUBSCRIPTION_ID=fakesubID TAGS= TENANT_ID=faketenantID USE_INSTANCE_METADATA=false USE_MANAGED_IDENTITY_EXTENSION=false VIRTUAL_NETWORK=k8s-vnet-22998975 VIRTUAL_NETWORK_RESOURCE_GROUP= VM_TYPE=standard VNET_CNI_PLUGINS_URL=https://kubernetesartifacts.azureedge.net/azure-cni/" + AzureCniPluginVerLinux + "/binaries/azure-vnet-cni-linux-amd64-" + AzureCniPluginVerLinux + ".tgz ", }, { name: "With ARM variables", @@ -6770,7 +6770,7 @@ func TestGetProvisionScriptParametersCommon(t *testing.T) { KubeletPrivateKey: common.WrapAsParameter("clientPrivateKey"), ClusterKeyVaultName: common.WrapAsARMVariable("clusterKeyvaultName"), }, - expected: "ADMINUSER=azureuser APISERVER_PUBLIC_KEY=',parameters('apiServerCertificate'),' AUTHENTICATION_METHOD=client_secret CLOUDPROVIDER_BACKOFF=false CLOUDPROVIDER_BACKOFF_DURATION=0 CLOUDPROVIDER_BACKOFF_EXPONENT=0 CLOUDPROVIDER_BACKOFF_JITTER=0 CLOUDPROVIDER_BACKOFF_MODE= CLOUDPROVIDER_BACKOFF_RETRIES=0 CLOUDPROVIDER_RATELIMIT=false CLOUDPROVIDER_RATELIMIT_BUCKET=0 CLOUDPROVIDER_RATELIMIT_BUCKET_WRITE=0 CLOUDPROVIDER_RATELIMIT_QPS=0 CLOUDPROVIDER_RATELIMIT_QPS_WRITE=0 CNI_PLUGINS_URL=https://kubernetesartifacts.azureedge.net/cni-plugins/" + CNIPluginVer + "/binaries/cni-plugins-linux-amd64-" + CNIPluginVer + ".tgz CONTAINERD_DOWNLOAD_URL_BASE=https://storage.googleapis.com/cri-containerd-release/ CONTAINERD_VERSION=" + DefaultContainerdVersion + " CONTAINER_RUNTIME=docker CUSTOM_HYPERKUBE_IMAGE= ETCD_DOWNLOAD_URL=mcr.microsoft.com/oss/etcd-io/ ETCD_VERSION=" + DefaultEtcdVersion + " EXCLUDE_MASTER_FROM_STANDARD_LB=false HYPERKUBE_URL=hyperkube-amd64:v1.16.8 IDENTITY_SYSTEM=azure_ad IS_IPV6_ENABLED=false KMS_PROVIDER_VAULT_NAME=',variables('clusterKeyvaultName'),' KUBELET_PRIVATE_KEY=',parameters('clientPrivateKey'),' KUBERNETES_VERSION=1.16.8 KUBE_BINARY_URL= LOAD_BALANCER_DISABLE_OUTBOUND_SNAT=false LOAD_BALANCER_SKU=Basic LOCATION=',variables('location'),' MAXIMUM_LOADBALANCER_RULE_COUNT=0 MOBY_VERSION=" + DefaultMobyVersion + " MS_APT_REPO=" + DefaultMicrosoftAptRepositoryURL + " NETWORK_API_VERSION=2018-08-01 NETWORK_MODE= NETWORK_PLUGIN=kubenet NETWORK_POLICY= NETWORK_SECURITY_GROUP=" + common.LegacyControlPlaneVMPrefix + "-22998975-nsg PRIMARY_AVAILABILITY_SET=agentpool1-availabilitySet-22998975 PRIMARY_SCALE_SET= RESOURCE_GROUP=',variables('resourceGroup'),' ROUTE_TABLE=" + common.LegacyControlPlaneVMPrefix + "-22998975-routetable SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('singleQuote'),'',variables('servicePrincipalClientSecret'),'',variables('singleQuote'),' SUBNET=k8s-subnet SUBSCRIPTION_ID=',variables('subscriptionId'),' TENANT_ID=',variables('tenantID'),' USE_INSTANCE_METADATA=false USE_MANAGED_IDENTITY_EXTENSION=false VIRTUAL_NETWORK=k8s-vnet-22998975 VIRTUAL_NETWORK_RESOURCE_GROUP= VM_TYPE=standard VNET_CNI_PLUGINS_URL=https://kubernetesartifacts.azureedge.net/azure-cni/" + AzureCniPluginVerLinux + "/binaries/azure-vnet-cni-linux-amd64-" + AzureCniPluginVerLinux + ".tgz ", + expected: "ADMINUSER=azureuser APISERVER_PUBLIC_KEY=',parameters('apiServerCertificate'),' AUTHENTICATION_METHOD=client_secret CLOUDPROVIDER_BACKOFF=false CLOUDPROVIDER_BACKOFF_DURATION=0 CLOUDPROVIDER_BACKOFF_EXPONENT=0 CLOUDPROVIDER_BACKOFF_JITTER=0 CLOUDPROVIDER_BACKOFF_MODE= CLOUDPROVIDER_BACKOFF_RETRIES=0 CLOUDPROVIDER_RATELIMIT=false CLOUDPROVIDER_RATELIMIT_BUCKET=0 CLOUDPROVIDER_RATELIMIT_BUCKET_WRITE=0 CLOUDPROVIDER_RATELIMIT_QPS=0 CLOUDPROVIDER_RATELIMIT_QPS_WRITE=0 CNI_PLUGINS_URL=https://kubernetesartifacts.azureedge.net/cni-plugins/" + CNIPluginVer + "/binaries/cni-plugins-linux-amd64-" + CNIPluginVer + ".tgz CONTAINERD_DOWNLOAD_URL_BASE=https://storage.googleapis.com/cri-containerd-release/ CONTAINERD_VERSION=" + DefaultContainerdVersion + " CONTAINER_RUNTIME=docker CUSTOM_HYPERKUBE_IMAGE= ENABLE_MULTIPLE_STANDARD_LOAD_BALANCERS=false ETCD_DOWNLOAD_URL=mcr.microsoft.com/oss/etcd-io/ ETCD_VERSION=" + DefaultEtcdVersion + " EXCLUDE_MASTER_FROM_STANDARD_LB=false HYPERKUBE_URL=hyperkube-amd64:v1.16.8 IDENTITY_SYSTEM=azure_ad IS_IPV6_ENABLED=false KMS_PROVIDER_VAULT_NAME=',variables('clusterKeyvaultName'),' KUBELET_PRIVATE_KEY=',parameters('clientPrivateKey'),' KUBERNETES_VERSION=1.16.8 KUBE_BINARY_URL= LOAD_BALANCER_DISABLE_OUTBOUND_SNAT=false LOAD_BALANCER_SKU=Basic LOCATION=',variables('location'),' MAXIMUM_LOADBALANCER_RULE_COUNT=0 MOBY_VERSION=" + DefaultMobyVersion + " MS_APT_REPO=" + DefaultMicrosoftAptRepositoryURL + " NETWORK_API_VERSION=2018-08-01 NETWORK_MODE= NETWORK_PLUGIN=kubenet NETWORK_POLICY= NETWORK_SECURITY_GROUP=" + common.LegacyControlPlaneVMPrefix + "-22998975-nsg PRIMARY_AVAILABILITY_SET=agentpool1-availabilitySet-22998975 PRIMARY_SCALE_SET= RESOURCE_GROUP=',variables('resourceGroup'),' ROUTE_TABLE=" + common.LegacyControlPlaneVMPrefix + "-22998975-routetable SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('singleQuote'),'',variables('servicePrincipalClientSecret'),'',variables('singleQuote'),' SUBNET=k8s-subnet SUBSCRIPTION_ID=',variables('subscriptionId'),' TAGS= TENANT_ID=',variables('tenantID'),' USE_INSTANCE_METADATA=false USE_MANAGED_IDENTITY_EXTENSION=false VIRTUAL_NETWORK=k8s-vnet-22998975 VIRTUAL_NETWORK_RESOURCE_GROUP= VM_TYPE=standard VNET_CNI_PLUGINS_URL=https://kubernetesartifacts.azureedge.net/azure-cni/" + AzureCniPluginVerLinux + "/binaries/azure-vnet-cni-linux-amd64-" + AzureCniPluginVerLinux + ".tgz ", }, } diff --git a/pkg/api/vlabs/types.go b/pkg/api/vlabs/types.go index 638c566acea..5d35adcfcd2 100644 --- a/pkg/api/vlabs/types.go +++ b/pkg/api/vlabs/types.go @@ -7,11 +7,10 @@ import ( "encoding/json" "strings" - "github.com/Azure/go-autorest/autorest/azure" - "github.com/pkg/errors" - "github.com/Azure/aks-engine/pkg/api/common" + "github.com/Azure/go-autorest/autorest/azure" "github.com/Azure/go-autorest/autorest/to" + "github.com/pkg/errors" ) // ResourcePurchasePlan defines resource plan as required by ARM @@ -324,86 +323,88 @@ const ( // KubernetesConfig contains the Kubernetes config structure, containing // Kubernetes specific configuration type KubernetesConfig struct { - KubernetesImageBase string `json:"kubernetesImageBase,omitempty"` - KubernetesImageBaseType string `json:"kubernetesImageBaseType,omitempty"` - MCRKubernetesImageBase string `json:"mcrKubernetesImageBase,omitempty"` - ClusterSubnet string `json:"clusterSubnet,omitempty"` - DNSServiceIP string `json:"dnsServiceIP,omitempty"` - ServiceCidr string `json:"serviceCidr,omitempty"` - NetworkPolicy string `json:"networkPolicy,omitempty"` - NetworkPlugin string `json:"networkPlugin,omitempty"` - NetworkMode string `json:"networkMode,omitempty"` - ContainerRuntime string `json:"containerRuntime,omitempty"` - MaxPods int `json:"maxPods,omitempty"` - DockerBridgeSubnet string `json:"dockerBridgeSubnet,omitempty"` - UseManagedIdentity *bool `json:"useManagedIdentity,omitempty"` - UserAssignedID string `json:"userAssignedID,omitempty"` - UserAssignedClientID string `json:"userAssignedClientID,omitempty"` //Note: cannot be provided in config. Used *only* for transferring this to azure.json. - CustomHyperkubeImage string `json:"customHyperkubeImage,omitempty"` - CustomKubeAPIServerImage string `json:"customKubeAPIServerImage,omitempty"` - CustomKubeControllerManagerImage string `json:"customKubeControllerManagerImage,omitempty"` - CustomKubeProxyImage string `json:"customKubeProxyImage,omitempty"` - CustomKubeSchedulerImage string `json:"customKubeSchedulerImage,omitempty"` - CustomKubeBinaryURL string `json:"customKubeBinaryURL,omitempty"` - DockerEngineVersion string `json:"dockerEngineVersion,omitempty"` // Deprecated - MobyVersion string `json:"mobyVersion,omitempty"` - LinuxMobyURL string `json:"linuxMobyURL,omitempty"` - ContainerdVersion string `json:"containerdVersion,omitempty"` - LinuxContainerdURL string `json:"linuxContainerdURL,omitempty"` - CustomCcmImage string `json:"customCcmImage,omitempty"` - UseCloudControllerManager *bool `json:"useCloudControllerManager,omitempty"` - CustomWindowsPackageURL string `json:"customWindowsPackageURL,omitempty"` - WindowsNodeBinariesURL string `json:"windowsNodeBinariesURL,omitempty"` - WindowsContainerdURL string `json:"windowsContainerdURL,omitempty"` - WindowsSdnPluginURL string `json:"windowsSdnPluginURL,omitempty"` - UseInstanceMetadata *bool `json:"useInstanceMetadata,omitempty"` - EnableRbac *bool `json:"enableRbac,omitempty"` - EnableSecureKubelet *bool `json:"enableSecureKubelet,omitempty"` - EnableAggregatedAPIs bool `json:"enableAggregatedAPIs,omitempty"` - PrivateCluster *PrivateCluster `json:"privateCluster,omitempty"` - GCHighThreshold int `json:"gchighthreshold,omitempty"` - GCLowThreshold int `json:"gclowthreshold,omitempty"` - EtcdVersion string `json:"etcdVersion,omitempty"` - EtcdDiskSizeGB string `json:"etcdDiskSizeGB,omitempty"` - EtcdStorageLimitGB int `json:"etcdStorageLimitGB,omitempty"` - EtcdEncryptionKey string `json:"etcdEncryptionKey,omitempty"` - EnableDataEncryptionAtRest *bool `json:"enableDataEncryptionAtRest,omitempty"` - EnableEncryptionWithExternalKms *bool `json:"enableEncryptionWithExternalKms,omitempty"` - EnablePodSecurityPolicy *bool `json:"enablePodSecurityPolicy,omitempty"` - Addons []KubernetesAddon `json:"addons,omitempty"` - Components []KubernetesComponent `json:"components,omitempty"` - ContainerRuntimeConfig map[string]string `json:"containerRuntimeConfig,omitempty"` - KubeletConfig map[string]string `json:"kubeletConfig,omitempty"` - ControllerManagerConfig map[string]string `json:"controllerManagerConfig,omitempty"` - CloudControllerManagerConfig map[string]string `json:"cloudControllerManagerConfig,omitempty"` - APIServerConfig map[string]string `json:"apiServerConfig,omitempty"` - SchedulerConfig map[string]string `json:"schedulerConfig,omitempty"` - PodSecurityPolicyConfig map[string]string `json:"podSecurityPolicyConfig,omitempty"` // Deprecated - KubeReservedCgroup string `json:"kubeReservedCgroup,omitempty"` - CloudProviderBackoffMode string `json:"cloudProviderBackoffMode"` - CloudProviderBackoff *bool `json:"cloudProviderBackoff,omitempty"` - CloudProviderBackoffRetries int `json:"cloudProviderBackoffRetries,omitempty"` - CloudProviderBackoffJitter float64 `json:"cloudProviderBackoffJitter,omitempty"` - CloudProviderBackoffDuration int `json:"cloudProviderBackoffDuration,omitempty"` - CloudProviderBackoffExponent float64 `json:"cloudProviderBackoffExponent,omitempty"` - CloudProviderRateLimit *bool `json:"cloudProviderRateLimit,omitempty"` - CloudProviderRateLimitQPS float64 `json:"cloudProviderRateLimitQPS,omitempty"` - CloudProviderRateLimitQPSWrite float64 `json:"cloudProviderRateLimitQPSWrite,omitempty"` - CloudProviderRateLimitBucket int `json:"cloudProviderRateLimitBucket,omitempty"` - CloudProviderRateLimitBucketWrite int `json:"cloudProviderRateLimitBucketWrite,omitempty"` - CloudProviderDisableOutboundSNAT *bool `json:"cloudProviderDisableOutboundSNAT,omitempty"` - LoadBalancerSku string `json:"loadBalancerSku,omitempty"` - ExcludeMasterFromStandardLB *bool `json:"excludeMasterFromStandardLB,omitempty"` - LoadBalancerOutboundIPs *int `json:"loadBalancerOutboundIPs,omitempty"` - AzureCNIVersion string `json:"azureCNIVersion,omitempty"` - AzureCNIURLLinux string `json:"azureCNIURLLinux,omitempty"` - AzureCNIURLWindows string `json:"azureCNIURLWindows,omitempty"` - KeyVaultSku string `json:"keyVaultSku,omitempty"` - MaximumLoadBalancerRuleCount int `json:"maximumLoadBalancerRuleCount,omitempty"` - ProxyMode KubeProxyMode `json:"kubeProxyMode,omitempty"` - PrivateAzureRegistryServer string `json:"privateAzureRegistryServer,omitempty"` - OutboundRuleIdleTimeoutInMinutes int32 `json:"outboundRuleIdleTimeoutInMinutes,omitempty"` - MicrosoftAptRepositoryURL string `json:"microsoftAptRepositoryURL,omitempty"` + KubernetesImageBase string `json:"kubernetesImageBase,omitempty"` + KubernetesImageBaseType string `json:"kubernetesImageBaseType,omitempty"` + MCRKubernetesImageBase string `json:"mcrKubernetesImageBase,omitempty"` + ClusterSubnet string `json:"clusterSubnet,omitempty"` + DNSServiceIP string `json:"dnsServiceIP,omitempty"` + ServiceCidr string `json:"serviceCidr,omitempty"` + NetworkPolicy string `json:"networkPolicy,omitempty"` + NetworkPlugin string `json:"networkPlugin,omitempty"` + NetworkMode string `json:"networkMode,omitempty"` + ContainerRuntime string `json:"containerRuntime,omitempty"` + MaxPods int `json:"maxPods,omitempty"` + DockerBridgeSubnet string `json:"dockerBridgeSubnet,omitempty"` + UseManagedIdentity *bool `json:"useManagedIdentity,omitempty"` + UserAssignedID string `json:"userAssignedID,omitempty"` + UserAssignedClientID string `json:"userAssignedClientID,omitempty"` //Note: cannot be provided in config. Used *only* for transferring this to azure.json. + CustomHyperkubeImage string `json:"customHyperkubeImage,omitempty"` + CustomKubeAPIServerImage string `json:"customKubeAPIServerImage,omitempty"` + CustomKubeControllerManagerImage string `json:"customKubeControllerManagerImage,omitempty"` + CustomKubeProxyImage string `json:"customKubeProxyImage,omitempty"` + CustomKubeSchedulerImage string `json:"customKubeSchedulerImage,omitempty"` + CustomKubeBinaryURL string `json:"customKubeBinaryURL,omitempty"` + DockerEngineVersion string `json:"dockerEngineVersion,omitempty"` // Deprecated + MobyVersion string `json:"mobyVersion,omitempty"` + LinuxMobyURL string `json:"linuxMobyURL,omitempty"` + ContainerdVersion string `json:"containerdVersion,omitempty"` + LinuxContainerdURL string `json:"linuxContainerdURL,omitempty"` + CustomCcmImage string `json:"customCcmImage,omitempty"` + UseCloudControllerManager *bool `json:"useCloudControllerManager,omitempty"` + CustomWindowsPackageURL string `json:"customWindowsPackageURL,omitempty"` + WindowsNodeBinariesURL string `json:"windowsNodeBinariesURL,omitempty"` + WindowsContainerdURL string `json:"windowsContainerdURL,omitempty"` + WindowsSdnPluginURL string `json:"windowsSdnPluginURL,omitempty"` + UseInstanceMetadata *bool `json:"useInstanceMetadata,omitempty"` + EnableRbac *bool `json:"enableRbac,omitempty"` + EnableSecureKubelet *bool `json:"enableSecureKubelet,omitempty"` + EnableAggregatedAPIs bool `json:"enableAggregatedAPIs,omitempty"` + PrivateCluster *PrivateCluster `json:"privateCluster,omitempty"` + GCHighThreshold int `json:"gchighthreshold,omitempty"` + GCLowThreshold int `json:"gclowthreshold,omitempty"` + EtcdVersion string `json:"etcdVersion,omitempty"` + EtcdDiskSizeGB string `json:"etcdDiskSizeGB,omitempty"` + EtcdStorageLimitGB int `json:"etcdStorageLimitGB,omitempty"` + EtcdEncryptionKey string `json:"etcdEncryptionKey,omitempty"` + EnableDataEncryptionAtRest *bool `json:"enableDataEncryptionAtRest,omitempty"` + EnableEncryptionWithExternalKms *bool `json:"enableEncryptionWithExternalKms,omitempty"` + EnablePodSecurityPolicy *bool `json:"enablePodSecurityPolicy,omitempty"` + Addons []KubernetesAddon `json:"addons,omitempty"` + Components []KubernetesComponent `json:"components,omitempty"` + ContainerRuntimeConfig map[string]string `json:"containerRuntimeConfig,omitempty"` + KubeletConfig map[string]string `json:"kubeletConfig,omitempty"` + ControllerManagerConfig map[string]string `json:"controllerManagerConfig,omitempty"` + CloudControllerManagerConfig map[string]string `json:"cloudControllerManagerConfig,omitempty"` + APIServerConfig map[string]string `json:"apiServerConfig,omitempty"` + SchedulerConfig map[string]string `json:"schedulerConfig,omitempty"` + PodSecurityPolicyConfig map[string]string `json:"podSecurityPolicyConfig,omitempty"` // Deprecated + KubeReservedCgroup string `json:"kubeReservedCgroup,omitempty"` + CloudProviderBackoffMode string `json:"cloudProviderBackoffMode"` + CloudProviderBackoff *bool `json:"cloudProviderBackoff,omitempty"` + CloudProviderBackoffRetries int `json:"cloudProviderBackoffRetries,omitempty"` + CloudProviderBackoffJitter float64 `json:"cloudProviderBackoffJitter,omitempty"` + CloudProviderBackoffDuration int `json:"cloudProviderBackoffDuration,omitempty"` + CloudProviderBackoffExponent float64 `json:"cloudProviderBackoffExponent,omitempty"` + CloudProviderRateLimit *bool `json:"cloudProviderRateLimit,omitempty"` + CloudProviderRateLimitQPS float64 `json:"cloudProviderRateLimitQPS,omitempty"` + CloudProviderRateLimitQPSWrite float64 `json:"cloudProviderRateLimitQPSWrite,omitempty"` + CloudProviderRateLimitBucket int `json:"cloudProviderRateLimitBucket,omitempty"` + CloudProviderRateLimitBucketWrite int `json:"cloudProviderRateLimitBucketWrite,omitempty"` + CloudProviderDisableOutboundSNAT *bool `json:"cloudProviderDisableOutboundSNAT,omitempty"` + LoadBalancerSku string `json:"loadBalancerSku,omitempty"` + ExcludeMasterFromStandardLB *bool `json:"excludeMasterFromStandardLB,omitempty"` + LoadBalancerOutboundIPs *int `json:"loadBalancerOutboundIPs,omitempty"` + AzureCNIVersion string `json:"azureCNIVersion,omitempty"` + AzureCNIURLLinux string `json:"azureCNIURLLinux,omitempty"` + AzureCNIURLWindows string `json:"azureCNIURLWindows,omitempty"` + KeyVaultSku string `json:"keyVaultSku,omitempty"` + MaximumLoadBalancerRuleCount int `json:"maximumLoadBalancerRuleCount,omitempty"` + ProxyMode KubeProxyMode `json:"kubeProxyMode,omitempty"` + PrivateAzureRegistryServer string `json:"privateAzureRegistryServer,omitempty"` + OutboundRuleIdleTimeoutInMinutes int32 `json:"outboundRuleIdleTimeoutInMinutes,omitempty"` + MicrosoftAptRepositoryURL string `json:"microsoftAptRepositoryURL,omitempty"` + EnableMultipleStandardLoadBalancers *bool `json:"enableMultipleStandardLoadBalancers,omitempty"` + Tags string `json:"tags,omitempty"` } // CustomFile has source as the full absolute source path to a file and dest diff --git a/pkg/api/vlabs/validate.go b/pkg/api/vlabs/validate.go index 1675a1ae897..5bf51195da8 100644 --- a/pkg/api/vlabs/validate.go +++ b/pkg/api/vlabs/validate.go @@ -1513,6 +1513,13 @@ func (k *KubernetesConfig) Validate(k8sVersion string, hasWindows, ipv6DualStack if e := k.validateKubernetesImageBaseType(); e != nil { return e } + + if to.Bool(k.EnableMultipleStandardLoadBalancers) && !common.IsKubernetesVersionGe(k8sVersion, "1.20.0-beta.1") { + return errors.Errorf("OrchestratorProfile.KubernetesConfig.EnableMultipleStandardLoadBalancers is available since kubernetes version v1.20.0-beta.1, current version is %s", k8sVersion) + } + if k.Tags != "" && !common.IsKubernetesVersionGe(k8sVersion, "1.20.0-beta.1") { + return errors.Errorf("OrchestratorProfile.KubernetesConfig.Tags is available since kubernetes version v1.20.0-beta.1, current version is %s", k8sVersion) + } return k.validateContainerRuntimeConfig() } diff --git a/pkg/api/vlabs/validate_test.go b/pkg/api/vlabs/validate_test.go index 13f0804372d..2981f1c8089 100644 --- a/pkg/api/vlabs/validate_test.go +++ b/pkg/api/vlabs/validate_test.go @@ -734,6 +734,22 @@ func Test_KubernetesConfig_Validate(t *testing.T) { t.Error("should error when dual stack and single stack IPv6 enabled simultaneously") } } + + // Tests that apply to 1.20 and later releases + for _, k8sVersion := range common.GetVersionsLt(common.GetAllSupportedKubernetesVersions(false, false, false), "1.20.0", false, false) { + c := KubernetesConfig{ + EnableMultipleStandardLoadBalancers: to.BoolPtr(true), + } + if err := c.Validate(k8sVersion, false, false, false); err == nil { + t.Errorf("should error when enable multiple standard load balancer before v1.20.0") + } + c = KubernetesConfig{ + Tags: "a=b", + } + if err := c.Validate(k8sVersion, false, false, false); err == nil { + t.Errorf("should error when setting tags before v1.20.0") + } + } } func Test_Properties_ValidateCustomKubeComponent(t *testing.T) { diff --git a/pkg/engine/templates_generated.go b/pkg/engine/templates_generated.go index 610e4bc40b0..b9e08336f79 100644 --- a/pkg/engine/templates_generated.go +++ b/pkg/engine/templates_generated.go @@ -11858,7 +11858,9 @@ configureK8s() { "providerVaultName": "${KMS_PROVIDER_VAULT_NAME}", "maximumLoadBalancerRuleCount": ${MAXIMUM_LOADBALANCER_RULE_COUNT}, "providerKeyName": "k8s", - "providerKeyVersion": "" + "providerKeyVersion": "", + "enableMultipleStandardLoadBalancers": ${ENABLE_MULTIPLE_STANDARD_LOAD_BALANCERS}, + "tags": "${TAGS}" } EOF set -x