diff --git a/.circleci/config.yml b/.circleci/config.yml index fe754f28f4..77a23304ca 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -412,6 +412,70 @@ jobs: path: /go/src/github.com/Azure/acs-engine/_logs - store_artifacts: path: /go/src/github.com/Azure/acs-engine/_output + openshift-3.9-rhel-e2e-vnet: + working_directory: /go/src/github.com/Azure/acs-engine + docker: + - image: registry.svc.ci.openshift.org/ci/acs-engine-tests:v3.9 + environment: + GOPATH: /go + steps: + - checkout + - run: | + echo 'export TIMEOUT=30m' >> $BASH_ENV + echo 'export DISTRO=openshift39_rhel' >> $BASH_ENV + echo 'export LOCATION=eastus' >> $BASH_ENV + echo 'export ORCHESTRATOR_RELEASE=3.9' >> $BASH_ENV + echo 'export ORCHESTRATOR_VERSION=3.9.0' >> $BASH_ENV + echo 'export CLUSTER_DEFINITION=examples/e2e-tests/openshift/definition.json' >> $BASH_ENV + echo 'export CREATE_VNET=true' >> $BASH_ENV + echo 'export CLEANUP_ON_EXIT=${CLEANUP_ON_EXIT}' >> $BASH_ENV + echo 'export RETAIN_SSH=false' >> $BASH_ENV + echo 'export SUBSCRIPTION_ID=${SUBSCRIPTION_ID_E2E_KUBERNETES}' >> $BASH_ENV + echo 'export CLIENT_ID=${SERVICE_PRINCIPAL_CLIENT_ID_E2E_KUBERNETES}' >> $BASH_ENV + echo 'export CLIENT_SECRET=${SERVICE_PRINCIPAL_CLIENT_SECRET_E2E_KUBERNETES}' >> $BASH_ENV + - run: + name: compile + command: make build-binary + - run: + name: ginkgo openshift e2e tests + command: make test-openshift + no_output_timeout: "30m" + - store_artifacts: + path: /go/src/github.com/Azure/acs-engine/_logs + - store_artifacts: + path: /go/src/github.com/Azure/acs-engine/_output + openshift-3.9-centos-e2e: + working_directory: /go/src/github.com/Azure/acs-engine + docker: + - image: registry.svc.ci.openshift.org/ci/acs-engine-tests:v3.9 + environment: + GOPATH: /go + steps: + - checkout + - run: | + echo 'export TIMEOUT=30m' >> $BASH_ENV + echo 'export DISTRO=openshift39_centos' >> $BASH_ENV + echo 'export LOCATION=eastus' >> $BASH_ENV + echo 'export ORCHESTRATOR_RELEASE=3.9' >> $BASH_ENV + echo 'export ORCHESTRATOR_VERSION=3.9.0' >> $BASH_ENV + echo 'export CLUSTER_DEFINITION=examples/openshift.json' >> $BASH_ENV + echo 'export CREATE_VNET=false' >> $BASH_ENV + echo 'export CLEANUP_ON_EXIT=false' >> $BASH_ENV + echo 'export RETAIN_SSH=false' >> $BASH_ENV + echo 'export SUBSCRIPTION_ID=${SUBSCRIPTION_ID_E2E_KUBERNETES}' >> $BASH_ENV + echo 'export CLIENT_ID=${SERVICE_PRINCIPAL_CLIENT_ID_E2E_KUBERNETES}' >> $BASH_ENV + echo 'export CLIENT_SECRET=${SERVICE_PRINCIPAL_CLIENT_SECRET_E2E_KUBERNETES}' >> $BASH_ENV + - run: + name: compile + command: make build-binary + - run: + name: ginkgo openshift e2e tests + command: make test-openshift + no_output_timeout: "30m" + - store_artifacts: + path: /go/src/github.com/Azure/acs-engine/_logs + - store_artifacts: + path: /go/src/github.com/Azure/acs-engine/_output workflows: version: 2 build_and_test_pr: @@ -499,6 +563,18 @@ workflows: filters: branches: ignore: master + - openshift-3.9-rhel-e2e-vnet: + requires: + - pr-e2e-hold + filters: + branches: + ignore: master + - openshift-3.9-centos-e2e: + requires: + - pr-e2e-hold + filters: + branches: + ignore: master - swarm-e2e: requires: - pr-e2e-hold @@ -589,9 +665,21 @@ workflows: filters: branches: only: master + - openshift-3.9-rhel-e2e-vnet: + requires: + - test + filters: + branches: + only: master + - openshift-3.9-centos-e2e: + requires: + - test + filters: + branches: + only: master - dcos-e2e: requires: - test filters: branches: - only: master \ No newline at end of file + only: master diff --git a/cmd/deploy.go b/cmd/deploy.go index 98e20c62e5..e47457bf77 100644 --- a/cmd/deploy.go +++ b/cmd/deploy.go @@ -1,6 +1,7 @@ package cmd import ( + "errors" "fmt" "io/ioutil" "math/rand" @@ -51,6 +52,7 @@ type deployCmd struct { caPrivateKeyPath string classicMode bool parametersOnly bool + set []string // derived containerService *api.ContainerService @@ -71,12 +73,18 @@ func newDeployCmd() *cobra.Command { Short: deployShortDescription, Long: deployLongDescription, RunE: func(cmd *cobra.Command, args []string) error { - if err := dc.validate(cmd, args); err != nil { + if err := dc.validateArgs(cmd, args); err != nil { log.Fatalf(fmt.Sprintf("error validating deployCmd: %s", err.Error())) } - if err := dc.load(cmd, args); err != nil { + if err := dc.mergeAPIModel(); err != nil { + log.Fatalf(fmt.Sprintf("error merging API model in deployCmd: %s", err.Error())) + } + if err := dc.loadAPIModel(cmd, args); err != nil { log.Fatalln("failed to load apimodel: %s", err.Error()) } + if _, _, err := dc.validateApimodel(); err != nil { + log.Fatalln("Failed to validate the apimodel after populating values: %s", err.Error()) + } return dc.run() }, } @@ -91,13 +99,14 @@ func newDeployCmd() *cobra.Command { f.StringVarP(&dc.resourceGroup, "resource-group", "g", "", "resource group to deploy to (will use the DNS prefix from the apimodel if not specified)") f.StringVarP(&dc.location, "location", "l", "", "location to deploy to (required)") f.BoolVarP(&dc.forceOverwrite, "force-overwrite", "f", false, "automatically overwrite existing files in the output directory") + f.StringArrayVar(&dc.set, "set", []string{}, "set values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)") addAuthFlags(&dc.authArgs, f) return deployCmd } -func (dc *deployCmd) validate(cmd *cobra.Command, args []string) error { +func (dc *deployCmd) validateArgs(cmd *cobra.Command, args []string) error { var err error dc.locale, err = i18n.LoadTranslations() @@ -129,7 +138,29 @@ func (dc *deployCmd) validate(cmd *cobra.Command, args []string) error { return nil } -func (dc *deployCmd) load(cmd *cobra.Command, args []string) error { +func (dc *deployCmd) mergeAPIModel() error { + var err error + + // if --set flag has been used + if dc.set != nil && len(dc.set) > 0 { + m := make(map[string]transform.APIModelValue) + transform.MapValues(m, dc.set) + + // overrides the api model and generates a new file + dc.apimodelPath, err = transform.MergeValuesWithAPIModel(dc.apimodelPath, m) + if err != nil { + return fmt.Errorf(fmt.Sprintf("error merging --set values with the api model: %s", err.Error())) + } + + log.Infoln(fmt.Sprintf("new api model file has been generated during merge: %s", dc.apimodelPath)) + } + + return nil +} + +func (dc *deployCmd) loadAPIModel(cmd *cobra.Command, args []string) error { + var caCertificateBytes []byte + var caKeyBytes []byte var err error apiloader := &api.Apiloader{ @@ -144,6 +175,35 @@ func (dc *deployCmd) load(cmd *cobra.Command, args []string) error { return fmt.Errorf(fmt.Sprintf("error parsing the api model: %s", err.Error())) } + if dc.outputDirectory == "" { + if dc.containerService.Properties.MasterProfile != nil { + dc.outputDirectory = path.Join("_output", dc.containerService.Properties.MasterProfile.DNSPrefix) + } else { + dc.outputDirectory = path.Join("_output", dc.containerService.Properties.HostedMasterProfile.DNSPrefix) + } + } + + // consume dc.caCertificatePath and dc.caPrivateKeyPath + if (dc.caCertificatePath != "" && dc.caPrivateKeyPath == "") || (dc.caCertificatePath == "" && dc.caPrivateKeyPath != "") { + return errors.New("--ca-certificate-path and --ca-private-key-path must be specified together") + } + + if dc.caCertificatePath != "" { + if caCertificateBytes, err = ioutil.ReadFile(dc.caCertificatePath); err != nil { + return fmt.Errorf(fmt.Sprintf("failed to read CA certificate file: %s", err.Error())) + } + if caKeyBytes, err = ioutil.ReadFile(dc.caPrivateKeyPath); err != nil { + return fmt.Errorf(fmt.Sprintf("failed to read CA private key file: %s", err.Error())) + } + + prop := dc.containerService.Properties + if prop.CertificateProfile == nil { + prop.CertificateProfile = &api.CertificateProfile{} + } + prop.CertificateProfile.CaCertificate = string(caCertificateBytes) + prop.CertificateProfile.CaPrivateKey = string(caKeyBytes) + } + if dc.containerService.Location == "" { dc.containerService.Location = dc.location } else if dc.containerService.Location != dc.location { @@ -163,11 +223,6 @@ func (dc *deployCmd) load(cmd *cobra.Command, args []string) error { return err } - _, _, err = validateApimodel(apiloader, dc.containerService, dc.apiVersion) - if err != nil { - return fmt.Errorf("Failed to validate the apimodel after populating values: %s", err) - } - dc.random = rand.New(rand.NewSource(time.Now().UnixNano())) return nil @@ -293,9 +348,15 @@ func autofillApimodel(dc *deployCmd) error { return nil } -func validateApimodel(apiloader *api.Apiloader, containerService *api.ContainerService, apiVersion string) (*api.ContainerService, string, error) { +func (dc *deployCmd) validateApimodel() (*api.ContainerService, string, error) { + apiloader := &api.Apiloader{ + Translator: &i18n.Translator{ + Locale: dc.locale, + }, + } + // This isn't terribly elegant, but it's the easiest way to go for now w/o duplicating a bunch of code - rawVersionedAPIModel, err := apiloader.SerializeContainerService(containerService, apiVersion) + rawVersionedAPIModel, err := apiloader.SerializeContainerService(dc.containerService, dc.apiVersion) if err != nil { return nil, "", err } @@ -366,5 +427,10 @@ func (dc *deployCmd) run() error { log.Fatalln(err) } + if dc.containerService.Properties.OrchestratorProfile.OrchestratorType == api.OpenShift { + // TODO: when the Azure client library is updated, read this from the template `masterFQDN` output + fmt.Printf("OpenShift web UI available at https://%s.%s.cloudapp.azure.com:8443/\n", dc.containerService.Properties.MasterProfile.DNSPrefix, dc.location) + } + return nil } diff --git a/cmd/deploy_test.go b/cmd/deploy_test.go index 0c7f89f94d..991db81e04 100644 --- a/cmd/deploy_test.go +++ b/cmd/deploy_test.go @@ -173,7 +173,7 @@ func TestValidate(t *testing.T) { } for _, c := range cases { - err = c.dc.validate(r, c.args) + err = c.dc.validateArgs(r, c.args) if err != nil && c.expectedErr != nil { if err.Error() != c.expectedErr.Error() { t.Fatalf("expected validate deploy command to return error %s, but instead got %s", c.expectedErr.Error(), err.Error()) @@ -447,7 +447,7 @@ func testAutodeployCredentialHandling(t *testing.T, useManagedIdentity bool, cli // cleanup, since auto-populations creates dirs and saves the SSH private key that it might create defer os.RemoveAll(deployCmd.outputDirectory) - cs, _, err = validateApimodel(apiloader, cs, ver) + cs, _, err = deployCmd.validateApimodel() if err != nil { t.Fatalf("unexpected error validating apimodel after populating defaults: %s", err) } @@ -464,3 +464,62 @@ func testAutodeployCredentialHandling(t *testing.T, useManagedIdentity bool, cli } } } + +func testDeployCmdMergeAPIModel(t *testing.T) { + d := &deployCmd{} + d.apimodelPath = "../pkg/acsengine/testdata/simple/kubernetes.json" + err := d.mergeAPIModel() + if err != nil { + t.Fatalf("unexpected error calling mergeAPIModel with no --set flag defined: %s", err.Error()) + } + + d = &deployCmd{} + d.apimodelPath = "../pkg/acsengine/testdata/simple/kubernetes.json" + d.set = []string{"masterProfile.count=3,linuxProfile.adminUsername=testuser"} + err = d.mergeAPIModel() + if err != nil { + t.Fatalf("unexpected error calling mergeAPIModel with one --set flag: %s", err.Error()) + } + + d = &deployCmd{} + d.apimodelPath = "../pkg/acsengine/testdata/simple/kubernetes.json" + d.set = []string{"masterProfile.count=3", "linuxProfile.adminUsername=testuser"} + err = d.mergeAPIModel() + if err != nil { + t.Fatalf("unexpected error calling mergeAPIModel with multiple --set flags: %s", err.Error()) + } + + d = &deployCmd{} + d.apimodelPath = "../pkg/acsengine/testdata/simple/kubernetes.json" + d.set = []string{"agentPoolProfiles[0].count=1"} + err = d.mergeAPIModel() + if err != nil { + t.Fatalf("unexpected error calling mergeAPIModel with one --set flag to override an array property: %s", err.Error()) + } +} + +func testDeployCmdMLoadAPIModel(t *testing.T) { + d := &deployCmd{} + r := &cobra.Command{} + f := r.Flags() + + addAuthFlags(&d.authArgs, f) + + fakeRawSubscriptionID := "6dc93fae-9a76-421f-bbe5-cc6460ea81cb" + fakeSubscriptionID, err := uuid.FromString(fakeRawSubscriptionID) + if err != nil { + t.Fatalf("Invalid SubscriptionId in Test: %s", err) + } + + d.apimodelPath = "../pkg/acsengine/testdata/simple/kubernetes.json" + d.set = []string{"agentPoolProfiles[0].count=1"} + d.SubscriptionID = fakeSubscriptionID + d.rawSubscriptionID = fakeRawSubscriptionID + + d.validateArgs(r, []string{"../pkg/acsengine/testdata/simple/kubernetes.json"}) + d.mergeAPIModel() + err = d.loadAPIModel(r, []string{"../pkg/acsengine/testdata/simple/kubernetes.json"}) + if err != nil { + t.Fatalf("unexpected error loading api model: %s", err.Error()) + } +} diff --git a/docs/clusterdefinition.md b/docs/clusterdefinition.md index d38e8f608e..29de840659 100644 --- a/docs/clusterdefinition.md +++ b/docs/clusterdefinition.md @@ -69,6 +69,7 @@ Here are the valid values for the orchestrator types: |tiller|true|1|Delivers the Helm server-side component: tiller. See https://github.com/kubernetes/helm for more info| |kubernetes-dashboard|true|1|Delivers the kubernetes dashboard component. See https://github.com/kubernetes/dashboard for more info| |rescheduler|false|1|Delivers the kubernetes rescheduler component| +|cluster-autoscaler|false|1|Delivers the kubernetes cluster autoscaler component. See https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler/cloudprovider/azure for more info| To give a bit more info on the `addons` property: We've tried to expose the basic bits of data that allow useful configuration of these cluster features. Here are some example usage patterns that will unpack what `addons` provide: @@ -102,7 +103,7 @@ As you can see above, `addons` is an array child property of `kubernetesConfig`. } ``` -More usefully, let's add some custom configuration to both of the above addons: +More usefully, let's add some custom configuration to the above addons: ``` "kubernetesConfig": { @@ -131,6 +132,22 @@ More usefully, let's add some custom configuration to both of the above addons: "memoryLimits": "512Mi" } ] + }, + { + "name": "cluster-autoscaler", + "containers": [ + { + "name": "cluster-autoscaler", + "cpuRequests": "100m", + "memoryRequests": "300Mi", + "cpuLimits": "100m", + "memoryLimits": "300Mi" + } + ], + "config": { + "maxNodes": "5", + "minNodes": "1" + } } ] } @@ -508,6 +525,13 @@ https://{keyvaultname}.vault.azure.net:443/secrets/{secretName}/{version} |clientId|yes, for Kubernetes clusters|describes the Azure client id. It is recommended to use a separate client ID per cluster| |secret|yes, for Kubernetes clusters|describes the Azure client secret. It is recommended to use a separate client secret per client id| |objectId|optional, for Kubernetes clusters|describes the Azure service principal object id. It is required if enableEncryptionWithExternalKms is true| +|keyvaultSecretRef.vaultId|no, for Kubernetes clusters|describes the vault id of the keyvault to retrieve the service principal secret from. See below for format.| +|keyvaultSecretRef.secretName|no, for Kubernetes clusters|describes the name of the service principal secret in keyvault| +|keyvaultSecretRef.version|no, for Kubernetes clusters|describes the version of the secret to use| + + +format for `keyvaultSecretRef.vaultId`, can be obtained in cli, or found in the portal: +`/subscriptions//resourceGroups//providers/Microsoft.KeyVault/vaults/`. See [keyvault params](../examples/keyvault-params/README.md#service-principal-profile) for an example. ## Cluster Defintions for apiVersion "2016-03-30" @@ -562,12 +586,13 @@ For apiVersion "2016-03-30", a cluster may have only 1 agent pool profiles. |ssh.publicKeys[0].keyData|yes|The public SSH key used for authenticating access to all Linux nodes in the cluster. Here are instructions for [generating a public/private key pair](ssh.md#ssh-key-generation)| ### aadProfile -`linuxProfile` provides [AAD integration](kubernetes.aad.md) configuration for the cluster, currently only available for Kubernetes orchestrator. +`aadProfile` provides [Azure Active Directory integration](kubernetes.aad.md) configuration for the cluster, currently only available for Kubernetes orchestrator. |Name|Required|Description| |---|---|---| |clientAppID|yes|Describes the client AAD application ID| |serverAppID|yes|Describes the server AAD application ID| +|adminGroupID|no|Describes the AAD Group Object ID that will be assigned the cluster-admin RBAC role| |tenantID|no|Describes the AAD tenant ID to use for authentication. If not specified, will use the tenant of the deployment subscription| ### extensionProfiles A cluster can have 0 - N extensions in extension profiles. Extension profiles allow a user to easily add pre-packaged functionality into a cluster. An example would be configuring a monitoring solution on your cluster. You can think of extensions like a marketplace for acs clusters. diff --git a/docs/kubernetes/deploy.md b/docs/kubernetes/deploy.md index ff43391f48..ce65445ddd 100644 --- a/docs/kubernetes/deploy.md +++ b/docs/kubernetes/deploy.md @@ -71,6 +71,22 @@ Administrative note: By default, the directory where acs-engine stores cluster c **Note**: If the cluster is using an existing VNET please see the [Custom VNET](features.md#feat-custom-vnet) feature documentation for additional steps that must be completed after cluster provisioning. +The deploy command lets you override any values under the properties tag (even in arrays) from the cluster definition file without having to update the file. You can use the `--set` flag to do that. For example: + +```bash +acs-engine deploy --resource-group "your-resource-group" \ + --location "westeurope" \ + --subscription-id "your-subscription-id" \ + --api-model "./apimodel.json" \ + --set masterProfile.dnsPrefix="your-dns-prefix-override" \ + --set agentPoolProfiles[0].name="your-agentpool-0-name-override" \ + --set agentPoolProfiles[0].count=1 \ + --set linuxProfile.ssh.publicKeys[0].keyData="ssh-rsa PUBLICKEY azureuser@linuxvm" \ + --set servicePrincipalProfile.clientId="spn-client-id" \ + --set servicePrincipalProfile.secret="spn-client-secret" +``` + + ## ACS Engine the Long Way diff --git a/examples/e2e-tests/openshift/definition.json b/examples/e2e-tests/openshift/definition.json new file mode 100644 index 0000000000..86b3a57150 --- /dev/null +++ b/examples/e2e-tests/openshift/definition.json @@ -0,0 +1,72 @@ +{ + "apiVersion": "vlabs", + "properties": { + "orchestratorProfile": { + "orchestratorType": "OpenShift", + "openShiftConfig": { + "clusterUsername": "", + "clusterPassword": "", + "enableAADAuthentication": false + } + }, + "azProfile": { + "tenantId": "", + "subscriptionId": "", + "resourceGroup": "", + "location": "" + }, + "masterProfile": { + "count": 1, + "dnsPrefix": "", + "imageReference": { + "name": "", + "resourceGroup": "" + }, + "storageProfile": "ManagedDisks", + "vmSize": "Standard_D4s_v3", + "vnetSubnetId": "/subscriptions/SUB_ID/resourceGroups/RG_NAME/providers/Microsoft.Network/virtualNetworks/VNET_NAME/subnets/SUBNET_NAME", + "firstConsecutiveStaticIP": "10.239.0.239" + }, + "agentPoolProfiles": [ + { + "availabilityProfile": "AvailabilitySet", + "count": 1, + "imageReference": { + "name": "", + "resourceGroup": "" + }, + "name": "compute", + "storageProfile": "ManagedDisks", + "vmSize": "Standard_D4s_v3", + "vnetSubnetId": "/subscriptions/SUB_ID/resourceGroups/RG_NAME/providers/Microsoft.Network/virtualNetworks/VNET_NAME/subnets/SUBNET_NAME" + }, + { + "availabilityProfile": "AvailabilitySet", + "count": 1, + "imageReference": { + "name": "", + "resourceGroup": "" + }, + "role": "infra", + "name": "infra", + "storageProfile": "ManagedDisks", + "vmSize": "Standard_D4s_v3", + "vnetSubnetId": "/subscriptions/SUB_ID/resourceGroups/RG_NAME/providers/Microsoft.Network/virtualNetworks/VNET_NAME/subnets/SUBNET_NAME" + } + ], + "linuxProfile": { + "adminUsername": "cloud-user", + "ssh": { + "publicKeys": [ + { + "keyData": "" + } + ] + } + }, + "servicePrincipalProfile": { + "clientId": "", + "secret": "" + } + } +} \ No newline at end of file diff --git a/examples/networkpolicy/README.md b/examples/networkpolicy/README.md index 6b8e80a6d6..b853d57227 100644 --- a/examples/networkpolicy/README.md +++ b/examples/networkpolicy/README.md @@ -18,7 +18,7 @@ The kubernetes-calico deployment template enables Calico networking and policies } ``` -This template will deploy the [v3.0 release](https://docs.projectcalico.org/v3.0/releases/) of [Kubernetes Datastore Install](https://docs.projectcalico.org/v3.0/getting-started/kubernetes/installation/hosted/kubernetes-datastore/) version of calico with the "Calico policy-only with user-supplied networking" which supports kubernetes ingress policies and has some limitations as denoted on the referenced page. +This template will deploy the [v3.1 release](https://docs.projectcalico.org/v3.1/releases/) of [Kubernetes Datastore Install](https://docs.projectcalico.org/v3.1/getting-started/kubernetes/installation/other) version of calico with the "Calico for policy" with user-supplied networking which supports kubernetes ingress policies. > Note: The Typha service and deployment is installed on the cluster, but effectively disabled using the default settings of deployment replicas set to 0 and Typha service name not configured. Typha is recommended to be enabled when scaling to 50+ nodes on the cluster to reduce the load on the Kubernetes API server. If this functionality is desired to be configurable via the API model, please file an issue on Github requesting this feature be added. Otherwise, this can be manually changed via modifying and applying changes with the `/etc/kubernetes/addons/calico-daemonset.yaml` file on every master node in the cluster. @@ -28,6 +28,6 @@ To understand how to deploy this template, please read the baseline [Kubernetes] ### Post installation -Once the template has been successfully deployed, following the [simple policy tutorial](https://docs.projectcalico.org/v3.0/getting-started/kubernetes/tutorials/simple-policy) or the [advanced policy tutorial](https://docs.projectcalico.org/v3.0/getting-started/kubernetes/tutorials/advanced-policy) will help to understand calico networking. +Once the template has been successfully deployed, following the [simple policy tutorial](https://docs.projectcalico.org/v3.1/getting-started/kubernetes/tutorials/simple-policy) or the [advanced policy tutorial](https://docs.projectcalico.org/v3.1/getting-started/kubernetes/tutorials/advanced-policy) will help to understand calico networking. > Note: `ping` (ICMP) traffic is blocked on the cluster by default. Wherever `ping` is used in any tutorial substitute testing access with something like `wget -q --timeout=5 google.com -O -` instead. diff --git a/examples/vnet/kubernetesvnet-customnodesdns.json b/examples/vnet/kubernetesvnet-customnodesdns.json index 1499f6822d..b324148f74 100644 --- a/examples/vnet/kubernetesvnet-customnodesdns.json +++ b/examples/vnet/kubernetesvnet-customnodesdns.json @@ -10,24 +10,37 @@ "masterProfile": { "count": 1, "dnsPrefix": "test", - "vmSize": "Standard_D2_v2", + "vmSize": "Standard_D2_v3", "vnetSubnetId": "/subscriptions/SUB_ID/resourceGroups/RG_NAME/providers/Microsoft.Network/virtualNetworks/VNET_NAME/subnets/SUBNET_NAME", - "firstConsecutiveStaticIP": "10.239.255.239" + "firstConsecutiveStaticIP": "10.239.255.239", + "vnetCidr": "10.22.221.0/24", + "preProvisionExtension": { + "name": "register-dns", + "singleOrAll": "All" + } }, "agentPoolProfiles": [ { - "name": "agentpri", + "name": "staging", "count": 2, - "vmSize": "Standard_D2_v2", + "vmSize": "Standard_D2_v3", "vnetSubnetId": "/subscriptions/SUB_ID/resourceGroups/RG_NAME/providers/Microsoft.Network/virtualNetworks/VNET_NAME/subnets/SUBNET_NAME", - "availabilityProfile": "AvailabilitySet" + "availabilityProfile": "VirtualMachineScaleSets", + "preProvisionExtension": { + "name": "register-dns", + "singleOrAll": "All" + } }, { - "name": "agentpri2", + "name": "production", "count": 2, - "vmSize": "Standard_D2_v2", + "vmSize": "Standard_D2_v3", "vnetSubnetId": "/subscriptions/SUB_ID/resourceGroups/RG_NAME/providers/Microsoft.Network/virtualNetworks/VNET_NAME/subnets/SUBNET_NAME", - "availabilityProfile": "AvailabilitySet" + "availabilityProfile": "VirtualMachineScaleSets", + "preProvisionExtension": { + "name": "register-dns", + "singleOrAll": "All" + } } ], "linuxProfile": { @@ -43,6 +56,15 @@ ] } }, + "extensionProfiles": [ + { + "name": "register-dns", + "version": "v1", + "extensionParameters": "mydomain.com", + "rootURL": "https://raw.githubusercontent.com/Azure/acs-engine/master/extensions/dnsupdate/", + "script": "register-dns.sh" + } + ], "servicePrincipalProfile": { "clientId": "", "secret": "" diff --git a/extensions/dnsupdate/v1/register-dns.sh b/extensions/dnsupdate/v1/register-dns.sh new file mode 100644 index 0000000000..9782188e43 --- /dev/null +++ b/extensions/dnsupdate/v1/register-dns.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# Add support for registering host on dns server. Must allow non-secure updates + +set -e + +DOMAINNAME=$1 + +echo $(date) " - Starting Script" + +cat > /etc/network/if-up.d/register-dns < \$nsupdatecmds + echo "update add \${host}.${DOMAINNAME} 3600 a \${ip}" >> \$nsupdatecmds + echo "send" >> \$nsupdatecmds + + nsupdate \$nsupdatecmds +fi +EOFDHCP + +chmod 755 /etc/network/if-up.d/register-dns + +if ! grep -Fq "${DOMAINNAME}" /etc/dhcp/dhclient.conf +then + echo $(date) " - Adding domain to dhclient.conf" + + echo "supersede domain-name \"${DOMAINNAME}\";" >> /etc/dhcp/dhclient.conf + echo "prepend domain-search \"${DOMAINNAME}\";" >> /etc/dhcp/dhclient.conf +fi + +# service networking restart +echo $(date) " - Restarting network" +sudo ifdown eth0 && sudo ifup eth0 \ No newline at end of file diff --git a/parts/dcos/bstrap/bootstrapcustomdata.yml b/parts/dcos/bstrap/bootstrapcustomdata.yml index 1c57b5493c..2d234013e0 100644 --- a/parts/dcos/bstrap/bootstrapcustomdata.yml +++ b/parts/dcos/bstrap/bootstrapcustomdata.yml @@ -30,10 +30,6 @@ runcmd: - [ ln, -s, /bin/mount, /usr/bin/mount ] - [ ln, -s, /bin/bash, /usr/bin/bash ] - [ ln, -s, /usr/sbin/useradd, /usr/bin/useradd ] - - [ systemctl, disable, --now, resolvconf.service ] - - [ systemctl, mask, --now, lxc-net.service ] - - [ systemctl, disable, --now, unscd.service ] - - [ systemctl, stop, --now, unscd.service ] - /opt/azure/containers/provision.sh - /opt/azure/dcos/init_bootstrap.sh write_files: @@ -60,6 +56,7 @@ write_files: MASTER_IP_LIST resolvers: - 168.63.129.16 + dns_search: owner: root path: /opt/azure/dcos/genconf/config.yaml permissions: '0644' @@ -67,6 +64,11 @@ MASTER_IP_LIST #!/bin/bash source /opt/azure/containers/provision_source.sh + + # update dns_search + dns=$(grep search /etc/resolv.conf | cut -d " " -f 2) + sed -i "/dns_search:/c dns_search: $dns" /opt/azure/dcos/genconf/config.yaml + # install and run bootstrap package cd /opt/azure/dcos retrycmd_if_failure 10 10 120 curl -fsSL -o dcos_generate_config.sh.sha1sum {{{dcosBootstrapURL}}}.sha1sum retry_download 1 1 120 {{{dcosBootstrapURL}}} dcos_generate_config.sh $(cat dcos_generate_config.sh.sha1sum) diff --git a/parts/dcos/dcosWindowsAgentResourcesVmss.t b/parts/dcos/dcosWindowsAgentResourcesVmss.t index 8fd4bb3fea..ef17f773c4 100644 --- a/parts/dcos/dcosWindowsAgentResourcesVmss.t +++ b/parts/dcos/dcosWindowsAgentResourcesVmss.t @@ -124,6 +124,7 @@ "location": "[variables('location')]", "name": "[concat(variables('{{.Name}}VMNamePrefix'), '-vmss')]", "properties": { + "overprovision": false, "upgradePolicy": { "mode": "Manual" }, diff --git a/parts/dcos/dcosWindowsProvision.ps1 b/parts/dcos/dcosWindowsProvision.ps1 index 3c5dd7e7b4..a2933b7eaa 100644 --- a/parts/dcos/dcosWindowsProvision.ps1 +++ b/parts/dcos/dcosWindowsProvision.ps1 @@ -124,7 +124,7 @@ Get-BootstrapScript($download_uri, $download_dir) # Get Mesos Binaries $scriptfile = "DCOSWindowsAgentSetup.ps1" - Write-Log " get script "+ ($download_uri+"/"+$scriptfile) + "and put it "+ ($download_dir+"\"+$scriptfile) + Write-Log "get script $download_uri/$scriptfile and put it $download_dir\$scriptfile" Invoke-WebRequest -Uri ($download_uri+"/"+$scriptfile) -OutFile ($download_dir+"\"+$scriptfile) } @@ -193,6 +193,7 @@ try $run_cmd += ">"+$global:BootstrapInstallDir+"\DCOSWindowsAgentSetup.log 2>&1" Write-Log "run setup script $run_cmd" Invoke-Expression $run_cmd + Write-Log "setup script completed" } else # We must be deploying a master { @@ -202,9 +203,12 @@ try } PREPROVISION_EXTENSION + + Write-Log "Provisioning script succeeded" } catch { + Write-Log "Provisioning script failed" Write-Error $_ exit 1 } diff --git a/parts/dcos/dcosagentresourcesvmss.t b/parts/dcos/dcosagentresourcesvmss.t index d3bf083f83..10f798966a 100644 --- a/parts/dcos/dcosagentresourcesvmss.t +++ b/parts/dcos/dcosagentresourcesvmss.t @@ -110,6 +110,7 @@ "location": "[variables('location')]", "name": "[concat(variables('{{.Name}}VMNamePrefix'), 'vmss')]", "properties": { + "overprovision": false, "upgradePolicy": { "mode": "Manual" }, diff --git a/parts/k8s/addons/azure-cni-networkmonitor.yaml b/parts/k8s/addons/azure-cni-networkmonitor.yaml new file mode 100644 index 0000000000..7bc3127e61 --- /dev/null +++ b/parts/k8s/addons/azure-cni-networkmonitor.yaml @@ -0,0 +1,48 @@ +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: azure-cni-networkmonitor + namespace: kube-system + labels: + app: azure-cnms + addonmanager.kubernetes.io/mode: "EnsureExists" +spec: + selector: + matchLabels: + k8s-app: azure-cnms + template: + metadata: + labels: + k8s-app: azure-cnms + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + tolerations: + - key: CriticalAddonsOnly + operator: Exists + containers: + - name: azure-cnms + image: + securityContext: + privileged: true + env: + - name: HOSTNAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + volumeMounts: + - name: ebtables-rule-repo + mountPath: /var/run + - name: log + mountPath: /var/log + hostNetwork: true + volumes: + - name: log + hostPath: + path: /var/log + type: Directory + - name: ebtables-rule-repo + hostPath: + path: /var/run/ + type: Directory \ No newline at end of file diff --git a/parts/k8s/addons/kubernetesmasteraddons-calico-daemonset.yaml b/parts/k8s/addons/kubernetesmasteraddons-calico-daemonset.yaml index 8149b8553e..85d3bd4d5a 100644 --- a/parts/k8s/addons/kubernetesmasteraddons-calico-daemonset.yaml +++ b/parts/k8s/addons/kubernetesmasteraddons-calico-daemonset.yaml @@ -1,8 +1,8 @@ -# Calico Version v3.1.1 -# https://docs.projectcalico.org/v3.1/releases#v3.1.1 +# Calico Version v3.1.3 +# https://docs.projectcalico.org/v3.1/releases#v3.1.3 # This manifest includes the following component versions: -# calico/node:v3.1.1 -# calico/cni:v3.1.1 +# calico/node:v3.1.3 +# calico/cni:v3.1.3 apiVersion: v1 kind: ServiceAccount @@ -217,7 +217,7 @@ spec: # as a host-networked pod. serviceAccountName: calico-node containers: - - image: quay.io/calico/typha:v0.7.2 + - image: quay.io/calico/typha:v0.7.4 name: calico-typha ports: - containerPort: 5473 @@ -314,7 +314,7 @@ spec: # container programs network policy and routes on each # host. - name: calico-node - image: quay.io/calico/node:v3.1.1 + image: quay.io/calico/node:v3.1.3 env: # Use Kubernetes API as the backing datastore. - name: DATASTORE_TYPE @@ -393,7 +393,7 @@ spec: # This container installs the Calico CNI binaries # and CNI network config file on each node. - name: install-cni - image: quay.io/calico/cni:v3.1.1 + image: quay.io/calico/cni:v3.1.3 command: ["/install-cni.sh"] env: # Name of the CNI config file to create. diff --git a/parts/k8s/addons/kubernetesmasteraddons-kube-dns-deployment.yaml b/parts/k8s/addons/kubernetesmasteraddons-kube-dns-deployment.yaml index 5f736b1589..407dd212a8 100644 --- a/parts/k8s/addons/kubernetesmasteraddons-kube-dns-deployment.yaml +++ b/parts/k8s/addons/kubernetesmasteraddons-kube-dns-deployment.yaml @@ -47,6 +47,8 @@ spec: metadata: annotations: scheduler.alpha.kubernetes.io/critical-pod: "" + prometheus.io/scrape: "true" + prometheus.io/port: "10055" labels: k8s-app: kube-dns kubernetes.io/cluster-service: "true" @@ -78,6 +80,9 @@ spec: - "--dns-port=10053" - "--v=2" - "--config-dir=/kube-dns-config" + env: + - name: PROMETHEUS_PORT + value: "10055" image: livenessProbe: failureThreshold: 5 @@ -96,6 +101,9 @@ spec: - containerPort: 10053 name: dns-tcp-local protocol: TCP + - containerPort: 10055 + name: metrics + protocol: TCP readinessProbe: httpGet: path: "/readiness" diff --git a/parts/k8s/addons/omsagent-daemonset.yaml b/parts/k8s/addons/omsagent-daemonset.yaml index f593c9398b..48e94e1f0a 100644 --- a/parts/k8s/addons/omsagent-daemonset.yaml +++ b/parts/k8s/addons/omsagent-daemonset.yaml @@ -3,11 +3,17 @@ kind: ServiceAccount metadata: name: omsagent namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: omsagent-reader + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile rules: - apiGroups: [""] resources: ["pods", "events", "nodes", "namespaces", "services"] @@ -17,6 +23,9 @@ kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: omsagentclusterrolebinding + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile subjects: - kind: ServiceAccount name: omsagent @@ -26,86 +35,233 @@ roleRef: name: omsagent-reader apiGroup: rbac.authorization.k8s.io --- +kind: ConfigMap +apiVersion: v1 +data: + kube.conf: "# Fluentd config file for OMS Docker - cluster components (kubeAPI)\r\n\r\n#Kubernetes + pod inventory\r\n\r\n\ttype kubepodinventory\r\n\ttag oms.containerinsights.KubePodInventory\r\n\trun_interval + 60s\r\n log_level debug\r\n\r\n\r\n#Kubernetes events\r\n\r\n\ttype + kubeevents\r\n\ttag oms.api.KubeEvents.CollectionTime\r\n\trun_interval 60s\r\n + \ log_level debug\r\n\r\n\r\n#Kubernetes logs\r\n\r\n\ttype kubelogs\r\n\ttag + oms.api.KubeLogs\r\n\trun_interval 60s\r\n\r\n\r\n#Kubernetes services\r\n\r\n\ttype + kubeservices\r\n\ttag oms.api.KubeServices.CollectionTime\r\n\trun_interval 60s\r\n + \ log_level debug\r\n\r\n\r\n#Kubernetes Nodes\r\n\r\n\ttype + kubenodeinventory\r\n\ttag oms.containerinsights.KubeNodeInventory\r\n\trun_interval + 60s\r\n log_level debug\r\n\r\n\r\n#Kubernetes perf\r\n\r\n\ttype + kubeperf\r\n\ttag oms.api.KubePerf\r\n\trun_interval 60s\r\n log_level debug\r\n\r\n\r\n\r\n type out_oms\r\n log_level debug\r\n + \ num_threads 5\r\n buffer_chunk_limit 20m\r\n buffer_type file\r\n buffer_path + %STATE_DIR_WS%/out_oms_kubepods*.buffer\r\n buffer_queue_limit 20\r\n buffer_queue_full_action + drop_oldest_chunk\r\n flush_interval 20s\r\n retry_limit 10\r\n retry_wait + 30s\r\n max_retry_wait 9m\r\n\r\n\r\n\r\n\ttype + out_oms_api\r\n\tlog_level debug\r\n num_threads 5\r\n\tbuffer_chunk_limit 5m\r\n\tbuffer_type + file\r\n\tbuffer_path %STATE_DIR_WS%/out_oms_api_kubeevents*.buffer\r\n\tbuffer_queue_limit + 10\r\n buffer_queue_full_action drop_oldest_chunk\r\n\tflush_interval 20s\r\n\tretry_limit + 10\r\n\tretry_wait 30s\r\n\r\n\r\n\r\n\ttype + out_oms_api\r\n\tlog_level debug\r\n buffer_chunk_limit 10m\r\n\tbuffer_type + file\r\n\tbuffer_path %STATE_DIR_WS%/out_oms_api_kubernetes_logs*.buffer\r\n\tbuffer_queue_limit + 10\r\n\tflush_interval 20s\r\n\tretry_limit 10\r\n\tretry_wait 30s\r\n\r\n\r\n\t \r\n type out_oms_api\r\n log_level debug\r\n num_threads + 5\r\n buffer_chunk_limit 20m\r\n buffer_type file\r\n buffer_path %STATE_DIR_WS%/out_oms_kubeservices*.buffer\r\n + \ buffer_queue_limit 20\r\n buffer_queue_full_action drop_oldest_chunk\r\n flush_interval + 20s\r\n retry_limit 10\r\n retry_wait 30s\r\n max_retry_wait 9m\r\n\r\n\r\n\r\n type out_oms\r\n log_level debug\r\n + \ num_threads 5\r\n buffer_chunk_limit 20m\r\n buffer_type file\r\n buffer_path + %STATE_DIR_WS%/state/out_oms_kubenodes*.buffer\r\n buffer_queue_limit 20\r\n + \ buffer_queue_full_action drop_oldest_chunk\r\n flush_interval 20s\r\n retry_limit + 10\r\n retry_wait 30s\r\n max_retry_wait 9m\r\n\r\n\r\n\t\r\n + \ type out_oms\r\n log_level debug\r\n num_threads 5\r\n buffer_chunk_limit + 20m\r\n buffer_type file\r\n buffer_path %STATE_DIR_WS%/out_oms_kubeperf*.buffer\r\n + \ buffer_queue_limit 20\r\n buffer_queue_full_action drop_oldest_chunk\r\n flush_interval + 20s\r\n retry_limit 10\r\n retry_wait 30s\r\n max_retry_wait 9m\r\n\r\n" +metadata: + name: omsagent-rs-config + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +--- apiVersion: extensions/v1beta1 kind: DaemonSet metadata: - name: omsagent - namespace: kube-system + labels: + component: oms-agent + tier: node + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + name: omsagent + namespace: kube-system spec: - updateStrategy: - type: RollingUpdate - template: - metadata: - labels: - app: omsagent - agentVersion: - dockerProviderVersion: - spec: - serviceAccountName: omsagent - containers: - - name: omsagent - image: - imagePullPolicy: IfNotPresent + selector: + matchLabels: + component: oms-agent + tier: node + template: + metadata: + annotations: + agentVersion: + dockerProviderVersion: + labels: + component: oms-agent + tier: node + spec: + serviceAccountName: omsagent + containers: + - name: omsagent + image: + imagePullPolicy: Always + resources: + requests: + cpu: + memory: + limits: + cpu: + memory: env: - - name: ACS_RESOURCE_NAME - value: "my_acs_cluster_name" - - name: DISABLE_KUBE_SYSTEM_LOG_COLLECTION - value: "true" - - name: WSID - value: - - name: KEY - value: - - name: NODE_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP - securityContext: - privileged: true - ports: - - containerPort: 25225 - protocol: TCP - - containerPort: 25224 - protocol: UDP - volumeMounts: - - mountPath: /var/run/docker.sock - name: docker-sock - - mountPath: /var/log - name: host-log - - mountPath: /var/lib/docker/containers - name: containerlog-path + - name: ACS_RESOURCE_NAME + value: "my_acs_cluster_name" + - name: DISABLE_KUBE_SYSTEM_LOG_COLLECTION + value: "true" + - name: WSID + value: + - name: KEY + value: + - name: NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP livenessProbe: exec: command: - - /bin/bash - - -c - - ps -ef | grep omsagent | grep -v "grep" + - /bin/bash + - -c + - ps -ef | grep omsagent | grep -v "grep" initialDelaySeconds: 60 periodSeconds: 60 - resources: - requests: - cpu: - memory: - limits: - cpu: - memory: - nodeSelector: - beta.kubernetes.io/os: linux + ports: + - containerPort: 25225 + protocol: TCP + - containerPort: 25224 + protocol: UDP + securityContext: + privileged: true + volumeMounts: + - mountPath: /var/run/docker.sock + name: docker-sock + - mountPath: /var/log + name: host-log + - mountPath: /var/lib/docker/containers + name: containerlog-path + nodeSelector: + beta.kubernetes.io/os: linux # Tolerate a NoSchedule taint on master that ACS Engine sets. - tolerations: - - key: "node-role.kubernetes.io/master" - operator: "Equal" - value: "true" - effect: "NoSchedule" - volumes: - - name: docker-sock - hostPath: - path: /var/run/docker.sock - - name: container-hostname - hostPath: - path: /etc/hostname - - name: host-log - hostPath: - path: /var/log - - name: containerlog-path - hostPath: - path: /var/lib/docker/containers \ No newline at end of file + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Equal" + value: "true" + effect: "NoSchedule" + volumes: + - name: docker-sock + hostPath: + path: /var/run/docker.sock + - name: container-hostname + hostPath: + path: /etc/hostname + - name: host-log + hostPath: + path: /var/log + - name: containerlog-path + hostPath: + path: /var/lib/docker/containers + updateStrategy: + type: RollingUpdate +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: omsagent-rs + namespace: kube-system + labels: + component: oms-agent + tier: node + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + spec: + replicas: 1 + selector: + matchLabels: + rsName: "omsagent-rs" + strategy: + type: RollingUpdate + template: + metadata: + labels: + rsName: "omsagent-rs" + annotations: + agentVersion: + dockerProviderVersion: + spec: + serviceAccountName: omsagent + containers: + - name: omsagent + image: + imagePullPolicy: IfNotPresent + resources: + limits: + cpu: + memory: + requests: + cpu: + memory: + env: + - name: NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: AKS_CLUSTER_NAME + value: my_aks_cluster + - name: DISABLE_KUBE_SYSTEM_LOG_COLLECTION + value: "true" + securityContext: + privileged: true + ports: + - containerPort: 25225 + protocol: TCP + - containerPort: 25224 + protocol: UDP + volumeMounts: + - mountPath: /var/run/docker.sock + name: docker-sock + - mountPath: /var/log + name: host-log + - mountPath: /var/lib/docker/containers + name: containerlog-path + - mountPath : /etc/config + name: omsagent-rs-config + livenessProbe: + exec: + command: + - /bin/bash + - -c + - ps -ef | grep omsagent | grep -v "grep" + initialDelaySeconds: 60 + periodSeconds: 60 + nodeSelector: + beta.kubernetes.io/os: linux + kubernetes.io/role: agent + volumes: + - name: docker-sock + hostPath: + path: /var/run/docker.sock + - name: container-hostname + hostPath: + path: /etc/hostname + - name: host-log + hostPath: + path: /var/log + - name: containerlog-path + hostPath: + path: /var/lib/docker/containers + - name: omsagent-rs-config + configMap: + name: omsagent-rs-config \ No newline at end of file diff --git a/parts/k8s/kubernetesagentcustomdata.yml b/parts/k8s/kubernetesagentcustomdata.yml index 0dff9d4cac..d5b6eece1c 100644 --- a/parts/k8s/kubernetesagentcustomdata.yml +++ b/parts/k8s/kubernetesagentcustomdata.yml @@ -90,40 +90,6 @@ write_files: name: localclustercontext current-context: localclustercontext -- path: "/etc/systemd/system/hyperkube-extract.service" - permissions: "0644" - owner: "root" - content: | - [Unit] - Description=kubectl and kubelet extraction - Requires=docker.service - After=docker.service -{{if .IsCoreOS}} - ConditionPathExists=!/opt/kubectl -{{else}} - ConditionPathExists=!/usr/local/bin/kubectl -{{end}} - - [Service] - TimeoutStartSec=0 - Restart=on-failure - RestartSec=5s - ExecStartPre=/bin/mkdir -p /tmp/hyperkubedir - ExecStartPre=/usr/bin/docker pull {{WrapAsVariable "kubernetesHyperkubeSpec"}} - ExecStartPre=/usr/bin/docker run --rm -v /tmp/hyperkubedir:/opt/hyperkubedir {{WrapAsVariable "kubernetesHyperkubeSpec"}} /bin/bash -c "cp /hyperkube /opt/hyperkubedir/" -{{if .IsCoreOS}} - ExecStartPre=/bin/cp /tmp/hyperkubedir/hyperkube /opt/kubelet - ExecStartPre=/bin/mv /tmp/hyperkubedir/hyperkube /opt/kubectl - ExecStart=/bin/chmod a+x /opt/kubelet /opt/kubectl -{{else}} - ExecStartPre=/bin/cp /tmp/hyperkubedir/hyperkube /usr/local/bin/kubelet - ExecStartPre=/bin/mv /tmp/hyperkubedir/hyperkube /usr/local/bin/kubectl - ExecStart=/bin/chmod a+x /usr/local/bin/kubelet /usr/local/bin/kubectl -{{end}} - - [Install] - WantedBy=multi-user.target - - path: "/etc/default/kubelet" permissions: "0644" owner: "root" @@ -148,6 +114,7 @@ AGENT_ARTIFACTS_CONFIG_PLACEHOLDER {{if IsAzureCNI}} # SNAT outbound traffic from pods to destinations outside of VNET. iptables -t nat -A POSTROUTING -m iprange ! --dst-range 168.63.129.16 -m addrtype ! --dst-type local ! -d {{WrapAsVariable "vnetCidr"}} -j MASQUERADE + sed -i "s||{{WrapAsVariable "AzureCNINetworkMonitorImageURL"}}|g" "/etc/kubernetes/addons/azure-cni-networkmonitor.yaml" {{end}} {{if not EnablePodSecurityPolicy}} sed -i "s|apparmor_parser|d|g" "/etc/systemd/system/kubelet.service" diff --git a/parts/k8s/kubernetesagentresourcesvmas.t b/parts/k8s/kubernetesagentresourcesvmas.t index 6952399aba..f2977f339e 100644 --- a/parts/k8s/kubernetesagentresourcesvmas.t +++ b/parts/k8s/kubernetesagentresourcesvmas.t @@ -1,5 +1,5 @@ { -{{if .IsAcceleratedNetworkingEnabled}} +{{if .AcceleratedNetworkingEnabled}} "apiVersion": "[variables('apiVersionAcceleratedNetworking')]", {{else}} "apiVersion": "[variables('apiVersionDefault')]", @@ -32,7 +32,7 @@ "location": "[variables('location')]", "name": "[concat(variables('{{.Name}}VMNamePrefix'), 'nic-', copyIndex(variables('{{.Name}}Offset')))]", "properties": { - "enableAcceleratedNetworking" : "{{.IsAcceleratedNetworkingEnabled}}", + "enableAcceleratedNetworking" : "{{.AcceleratedNetworkingEnabled}}", {{if not IsOpenShift}} {{if .IsCustomVNET}} "networkSecurityGroup": { diff --git a/parts/k8s/kubernetesagentresourcesvmss.t b/parts/k8s/kubernetesagentresourcesvmss.t index 2ec2492d98..0977fd1255 100644 --- a/parts/k8s/kubernetesagentresourcesvmss.t +++ b/parts/k8s/kubernetesagentresourcesvmss.t @@ -53,7 +53,7 @@ "name": "[variables('{{.Name}}VMNamePrefix')]", "properties": { "primary": true, - "enableAcceleratedNetworking" : "{{.IsAcceleratedNetworkingEnabled}}", + "enableAcceleratedNetworking" : "{{.AcceleratedNetworkingEnabled}}", {{if .IsCustomVNET}} "networkSecurityGroup": { "id": "[variables('nsgID')]" diff --git a/parts/k8s/kubernetescustomscript.sh b/parts/k8s/kubernetescustomscript.sh index a7369f6d61..442a1f7c14 100644 --- a/parts/k8s/kubernetescustomscript.sh +++ b/parts/k8s/kubernetescustomscript.sh @@ -160,7 +160,9 @@ function installDeps() { # make sure walinuxagent doesn't get updated in the middle of running this script retrycmd_if_failure 20 5 30 apt-mark hold walinuxagent || exit $ERR_HOLD_WALINUXAGENT # See https://github.com/kubernetes/kubernetes/blob/master/build/debian-hyperkube-base/Dockerfile#L25-L44 - apt_get_install 20 30 300 apt-transport-https ca-certificates iptables iproute2 socat util-linux mount ebtables ethtool init-system-helpers nfs-common ceph-common conntrack glusterfs-client ipset jq || exit $ERR_APT_INSTALL_TIMEOUT + apt_get_install 20 30 300 apt-transport-https ca-certificates iptables iproute2 ebtables socat util-linux mount ethtool init-system-helpers nfs-common ceph-common conntrack glusterfs-client ipset jq cgroup-lite git pigz xz-utils || exit $ERR_APT_INSTALL_TIMEOUT + systemctlEnableAndStart rpcbind + systemctlEnableAndStart rpc-statd } function installDocker() { @@ -169,13 +171,13 @@ function installDocker() { echo "deb ${DOCKER_REPO} ubuntu-xenial main" | sudo tee /etc/apt/sources.list.d/docker.list printf "Package: docker-engine\nPin: version ${DOCKER_ENGINE_VERSION}\nPin-Priority: 550\n" > /etc/apt/preferences.d/docker.pref apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT - apt_get_install 20 30 120 ebtables docker-engine || exit $ERR_DOCKER_INSTALL_TIMEOUT + apt_get_install 20 30 120 docker-engine || exit $ERR_DOCKER_INSTALL_TIMEOUT echo "ExecStartPost=/sbin/iptables -P FORWARD ACCEPT" >> /etc/systemd/system/docker.service.d/exec_start.conf usermod -aG docker ${ADMINUSER} } function runAptDaily() { - retrycmd_if_failure 20 30 60 /usr/lib/apt/apt.systemd.daily || exit $ERR_APT_DAILY_TIMEOUT + /usr/lib/apt/apt.systemd.daily } function generateAggregatedAPICerts() { @@ -306,19 +308,6 @@ function installClearContainersRuntime() { systemctlEnableAndStart cc-proxy } -function installContainerd() { - CRI_CONTAINERD_VERSION="1.1.0" - CONTAINERD_DOWNLOAD_URL="https://storage.googleapis.com/cri-containerd-release/cri-containerd-${CRI_CONTAINERD_VERSION}.linux-amd64.tar.gz" - - CONTAINERD_TGZ_TMP=/tmp/containerd.tar.gz - retrycmd_get_tarball 60 5 "$CONTAINERD_TGZ_TMP" "$CONTAINERD_DOWNLOAD_URL" - tar -xzf "$CONTAINERD_TGZ_TMP" -C / - rm -f "$CONTAINERD_TGZ_TMP" - - echo "Successfully installed cri-containerd..." - setupContainerd -} - function setupContainerd() { echo "Configuring cri-containerd..." @@ -340,6 +329,21 @@ function setupContainerd() { setKubeletOpts " --container-runtime=remote --runtime-request-timeout=15m --container-runtime-endpoint=unix:///run/containerd/containerd.sock" } +function installContainerd() { + CRI_CONTAINERD_VERSION="1.1.0" + CONTAINERD_DOWNLOAD_URL="https://storage.googleapis.com/cri-containerd-release/cri-containerd-${CRI_CONTAINERD_VERSION}.linux-amd64.tar.gz" + + CONTAINERD_TGZ_TMP=/tmp/containerd.tar.gz + retrycmd_get_tarball 60 5 "$CONTAINERD_TGZ_TMP" "$CONTAINERD_DOWNLOAD_URL" + tar -xzf "$CONTAINERD_TGZ_TMP" -C / + rm -f "$CONTAINERD_TGZ_TMP" + + echo "Successfully installed cri-containerd..." + if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]] || [[ "$CONTAINER_RUNTIME" == "containerd" ]]; then + setupContainerd + fi +} + function ensureContainerd() { if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]] || [[ "$CONTAINER_RUNTIME" == "containerd" ]]; then # Enable and start cri-containerd service @@ -362,8 +366,22 @@ function ensureKubelet() { } function extractHyperkube(){ - retrycmd_if_failure 100 1 60 docker pull $HYPERKUBE_URL || exit $ERR_K8S_DOWNLOAD_TIMEOUT - systemctlEnableAndStart hyperkube-extract + TMP_DIR=$(mktemp -d) + retrycmd_if_failure 100 1 30 curl -sSL -o /usr/local/bin/img "https://github.com/genuinetools/img/releases/download/v0.4.6/img-linux-amd64" + chmod +x /usr/local/bin/img + retrycmd_if_failure 100 1 60 img pull $HYPERKUBE_URL || $ERR_K8S_DOWNLOAD_TIMEOUT + path=$(find /tmp/img -name "hyperkube") + + if [[ $OS == $COREOS_OS_NAME ]]; then + cp "$path" "/opt/kubelet" + cp "$path" "/opt/kubectl" + chmod a+x /opt/kubelet /opt/kubectl + else + cp "$path" "/usr/local/bin/kubelet" + cp "$path" "/usr/local/bin/kubectl" + chmod a+x /usr/local/bin/kubelet /usr/local/bin/kubectl + fi + rm -rf /tmp/hyperkube.tar "/tmp/img" } function ensureJournal(){ @@ -494,7 +512,6 @@ fi installDeps installDocker -runAptDaily configureK8s ensureDocker configNetworkPlugin @@ -505,6 +522,10 @@ if [[ ! -z "${MASTER_NODE}" ]]; then echo `date`,`hostname`, configAddonsDone>>/opt/m fi +# containerd needs to be installed before extractHyperkube +# so runc is present. +echo `date`,`hostname`, installContainerdStart>>/opt/m +installContainerd echo `date`,`hostname`, extractHyperkubeStart>>/opt/m extractHyperkube echo `date`,`hostname`, extractHyperkubeDone>>/opt/m @@ -515,10 +536,6 @@ if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]]; then installClearContainersRuntime fi fi -if [[ "$CONTAINER_RUNTIME" == "clear-containers" ]] || [[ "$CONTAINER_RUNTIME" == "containerd" ]]; then - echo `date`,`hostname`, installContainerdStart>>/opt/m - installContainerd -fi echo `date`,`hostname`, ensureContainerdStart>>/opt/m ensureContainerd @@ -552,4 +569,6 @@ if $REBOOTREQUIRED; then # wait 1 minute to restart node, so that the custom script extension can complete echo 'reboot required, rebooting node in 1 minute' /bin/bash -c "shutdown -r 1 &" +else + runAptDaily & fi diff --git a/parts/k8s/kubernetesmastercustomdata.yml b/parts/k8s/kubernetesmastercustomdata.yml index 9e4cdfdeea..d3155a9ca9 100644 --- a/parts/k8s/kubernetesmastercustomdata.yml +++ b/parts/k8s/kubernetesmastercustomdata.yml @@ -144,40 +144,6 @@ MASTER_ADDONS_CONFIG_PLACEHOLDER MASTER_CUSTOM_FILES_PLACEHOLDER -- path: "/etc/systemd/system/hyperkube-extract.service" - permissions: "0644" - owner: "root" - content: | - [Unit] - Description=kubectl and kubelet extraction - Requires=docker.service - After=docker.service -{{if .MasterProfile.IsCoreOS}} - ConditionPathExists=!/opt/kubectl -{{else}} - ConditionPathExists=!/usr/local/bin/kubectl -{{end}} - - [Service] - TimeoutStartSec=0 - Restart=on-failure - RestartSec=5s - ExecStartPre=/bin/mkdir -p /tmp/hyperkubedir - ExecStartPre=/usr/bin/docker pull {{WrapAsVariable "kubernetesHyperkubeSpec"}} - ExecStartPre=/usr/bin/docker run --rm -v /tmp/hyperkubedir:/opt/hyperkubedir {{WrapAsVariable "kubernetesHyperkubeSpec"}} /bin/bash -c "cp /hyperkube /opt/hyperkubedir/" -{{if .MasterProfile.IsCoreOS}} - ExecStartPre=/bin/cp /tmp/hyperkubedir/hyperkube /opt/kubelet - ExecStartPre=/bin/mv /tmp/hyperkubedir/hyperkube /opt/kubectl - ExecStart=/bin/chmod a+x /opt/kubelet /opt/kubectl -{{else}} - ExecStartPre=/bin/cp /tmp/hyperkubedir/hyperkube /usr/local/bin/kubelet - ExecStartPre=/bin/mv /tmp/hyperkubedir/hyperkube /usr/local/bin/kubectl - ExecStart=/bin/chmod a+x /usr/local/bin/kubelet /usr/local/bin/kubectl -{{end}} - - [Install] - WantedBy=multi-user.target - - path: "/etc/default/kubelet" permissions: "0644" owner: "root" @@ -217,6 +183,7 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER {{if IsAzureCNI}} # SNAT outbound traffic from pods to destinations outside of VNET. iptables -t nat -A POSTROUTING -m iprange ! --dst-range 168.63.129.16 -m addrtype ! --dst-type local ! -d {{WrapAsVariable "vnetCidr"}} -j MASQUERADE + sed -i "s||{{WrapAsVariable "AzureCNINetworkMonitorImageURL"}}|g" "/etc/kubernetes/addons/azure-cni-networkmonitor.yaml" {{end}} sed -i "s||{{WrapAsVariable "kubernetesAddonManagerSpec"}}|g" "/etc/kubernetes/manifests/kube-addon-manager.yaml" sed -i "s||{{WrapAsVariable "kubernetesHyperkubeSpec"}}|g" "/etc/kubernetes/manifests/kube-apiserver.yaml" diff --git a/parts/k8s/kubernetesmastervars.t b/parts/k8s/kubernetesmastervars.t index cccaf4987e..e24b60105f 100644 --- a/parts/k8s/kubernetesmastervars.t +++ b/parts/k8s/kubernetesmastervars.t @@ -278,6 +278,7 @@ {{if IsAzureCNI}} "allocateNodeCidrs": false, + "AzureCNINetworkMonitorImageURL": "[parameters('AzureCNINetworkMonitorImageURL')]", {{else}} "allocateNodeCidrs": true, {{end}} diff --git a/parts/k8s/kubernetesparams.t b/parts/k8s/kubernetesparams.t index da020e0041..8e0cb8e409 100644 --- a/parts/k8s/kubernetesparams.t +++ b/parts/k8s/kubernetesparams.t @@ -914,3 +914,12 @@ } } {{end}} + {{if IsAzureCNI}} + ,"AzureCNINetworkMonitorImageURL": { + "defaultValue": "", + "metadata": { + "description": "Azure CNI networkmonitor Image URL" + }, + "type": "string" + } + {{end}} diff --git a/parts/k8s/kubernetesprovisionsource.sh b/parts/k8s/kubernetesprovisionsource.sh index 1ef0265659..58083ab71e 100644 --- a/parts/k8s/kubernetesprovisionsource.sh +++ b/parts/k8s/kubernetesprovisionsource.sh @@ -78,6 +78,7 @@ apt_get_install() { return 1 else sleep $wait_sleep + apt_get_update fi done echo Executed apt-get install --no-install-recommends -y \"$@\" $i times; diff --git a/parts/openshift/release-3.9/openshiftmasterscript.sh b/parts/openshift/release-3.9/openshiftmasterscript.sh index 118c163fa0..cacf656459 100644 --- a/parts/openshift/release-3.9/openshiftmasterscript.sh +++ b/parts/openshift/release-3.9/openshiftmasterscript.sh @@ -30,7 +30,13 @@ else COCKPIT_BASENAME="kubernetes" COCKPIT_VERSION="latest" fi -systemctl restart docker.service + +# TODO: with WALinuxAgent>=v2.2.21 (https://github.com/Azure/WALinuxAgent/pull/1005) +# we should be able to append context=system_u:object_r:container_var_lib_t:s0 +# to ResourceDisk.MountOptions in /etc/waagent.conf and remove this stanza. +systemctl stop docker.service +restorecon -R /var/lib/docker +systemctl start docker.service echo "BOOTSTRAP_CONFIG_NAME=node-config-master" >>/etc/sysconfig/${SERVICE_TYPE}-node diff --git a/parts/openshift/release-3.9/openshiftnodescript.sh b/parts/openshift/release-3.9/openshiftnodescript.sh index b7b64e38d8..af105ed702 100644 --- a/parts/openshift/release-3.9/openshiftnodescript.sh +++ b/parts/openshift/release-3.9/openshiftnodescript.sh @@ -7,6 +7,13 @@ if [ -f "/etc/sysconfig/atomic-openshift-node" ]; then SERVICE_TYPE=atomic-openshift fi +# TODO: with WALinuxAgent>=v2.2.21 (https://github.com/Azure/WALinuxAgent/pull/1005) +# we should be able to append context=system_u:object_r:container_var_lib_t:s0 +# to ResourceDisk.MountOptions in /etc/waagent.conf and remove this stanza. +systemctl stop docker.service +restorecon -R /var/lib/docker +systemctl start docker.service + {{if eq .Role "infra"}} echo "BOOTSTRAP_CONFIG_NAME=node-config-infra" >>/etc/sysconfig/${SERVICE_TYPE}-node {{else}} diff --git a/parts/openshift/unstable/openshiftmasterscript.sh b/parts/openshift/unstable/openshiftmasterscript.sh index 7966788ce8..e0329b3384 100644 --- a/parts/openshift/unstable/openshiftmasterscript.sh +++ b/parts/openshift/unstable/openshiftmasterscript.sh @@ -31,7 +31,13 @@ else COCKPIT_BASENAME="kubernetes" COCKPIT_VERSION="latest" fi -systemctl restart docker.service + +# TODO: with WALinuxAgent>=v2.2.21 (https://github.com/Azure/WALinuxAgent/pull/1005) +# we should be able to append context=system_u:object_r:container_var_lib_t:s0 +# to ResourceDisk.MountOptions in /etc/waagent.conf and remove this stanza. +systemctl stop docker.service +restorecon -R /var/lib/docker +systemctl start docker.service echo "BOOTSTRAP_CONFIG_NAME=node-config-master" >>/etc/sysconfig/${SERVICE_TYPE}-node diff --git a/parts/openshift/unstable/openshiftnodescript.sh b/parts/openshift/unstable/openshiftnodescript.sh index 3878571bb1..a42839397e 100644 --- a/parts/openshift/unstable/openshiftnodescript.sh +++ b/parts/openshift/unstable/openshiftnodescript.sh @@ -7,6 +7,13 @@ if [ -f "/etc/sysconfig/atomic-openshift-node" ]; then SERVICE_TYPE=atomic-openshift fi +# TODO: with WALinuxAgent>=v2.2.21 (https://github.com/Azure/WALinuxAgent/pull/1005) +# we should be able to append context=system_u:object_r:container_var_lib_t:s0 +# to ResourceDisk.MountOptions in /etc/waagent.conf and remove this stanza. +systemctl stop docker.service +restorecon -R /var/lib/docker +systemctl start docker.service + {{if eq .Role "infra"}} echo "BOOTSTRAP_CONFIG_NAME=node-config-infra" >>/etc/sysconfig/${SERVICE_TYPE}-node {{else}} diff --git a/pkg/acsengine/addons.go b/pkg/acsengine/addons.go index 0b429b7b3c..83ffeced38 100644 --- a/pkg/acsengine/addons.go +++ b/pkg/acsengine/addons.go @@ -107,6 +107,11 @@ func kubernetesAddonSettingsInit(profile *api.Properties) []kubernetesFeatureSet "omsagent-daemonset.yaml", profile.OrchestratorProfile.IsContainerMonitoringEnabled(), }, + { + "azure-cni-networkmonitor.yaml", + "azure-cni-networkmonitor.yaml", + profile.OrchestratorProfile.IsAzureCNI(), + }, } } diff --git a/pkg/acsengine/const.go b/pkg/acsengine/const.go index 44f7dccea5..5ed6d8942e 100644 --- a/pkg/acsengine/const.go +++ b/pkg/acsengine/const.go @@ -140,6 +140,8 @@ const ( DefaultNVIDIADevicePluginAddonName = "nvidia-device-plugin" // ContainerMonitoringAddonName is the name of the kubernetes Container Monitoring addon deployment ContainerMonitoringAddonName = "container-monitoring" + // AzureCNINetworkMonitoringAddonName is the name of the Azure CNI networkmonitor addon + AzureCNINetworkMonitoringAddonName = "azure-cni-networkmonitor" // DefaultKubernetesKubeletMaxPods is the max pods per kubelet DefaultKubernetesKubeletMaxPods = 110 // DefaultMasterEtcdServerPort is the default etcd server port for Kubernetes master nodes diff --git a/pkg/acsengine/defaults-apiserver.go b/pkg/acsengine/defaults-apiserver.go index 3ce272e87d..0a1b2bf862 100644 --- a/pkg/acsengine/defaults-apiserver.go +++ b/pkg/acsengine/defaults-apiserver.go @@ -10,7 +10,7 @@ import ( func setAPIServerConfig(cs *api.ContainerService) { o := cs.Properties.OrchestratorProfile - staticLinuxAPIServerConfig := map[string]string{ + staticAPIServerConfig := map[string]string{ "--bind-address": "0.0.0.0", "--advertise-address": "", "--allow-privileged": "true", @@ -36,13 +36,6 @@ func setAPIServerConfig(cs *api.ContainerService) { "--v": "4", } - // Windows apiserver config overrides - // TODO placeholder for specific config overrides for Windows clusters - staticWindowsAPIServerConfig := make(map[string]string) - for key, val := range staticLinuxAPIServerConfig { - staticWindowsAPIServerConfig[key] = val - } - // Default apiserver config defaultAPIServerConfig := map[string]string{ "--audit-log-maxage": "30", @@ -50,14 +43,9 @@ func setAPIServerConfig(cs *api.ContainerService) { "--audit-log-maxsize": "100", } - // Data Encryption at REST configuration - if helpers.IsTrueBoolPointer(o.KubernetesConfig.EnableDataEncryptionAtRest) { - staticLinuxAPIServerConfig["--experimental-encryption-provider-config"] = "/etc/kubernetes/encryption-config.yaml" - } - - // Data Encryption at REST with external KMS configuration - if helpers.IsTrueBoolPointer(o.KubernetesConfig.EnableEncryptionWithExternalKms) { - staticLinuxAPIServerConfig["--experimental-encryption-provider-config"] = "/etc/kubernetes/encryption-config.yaml" + // Data Encryption at REST configuration conditions + if helpers.IsTrueBoolPointer(o.KubernetesConfig.EnableDataEncryptionAtRest) || helpers.IsTrueBoolPointer(o.KubernetesConfig.EnableEncryptionWithExternalKms) { + staticAPIServerConfig["--experimental-encryption-provider-config"] = "/etc/kubernetes/encryption-config.yaml" } // Aggregated API configuration @@ -73,8 +61,8 @@ func setAPIServerConfig(cs *api.ContainerService) { // Enable cloudprovider if we're not using cloud controller manager if !helpers.IsTrueBoolPointer(o.KubernetesConfig.UseCloudControllerManager) { - staticLinuxAPIServerConfig["--cloud-provider"] = "azure" - staticLinuxAPIServerConfig["--cloud-config"] = "/etc/kubernetes/azure.json" + staticAPIServerConfig["--cloud-provider"] = "azure" + staticAPIServerConfig["--cloud-config"] = "/etc/kubernetes/azure.json" } // AAD configuration @@ -91,7 +79,7 @@ func setAPIServerConfig(cs *api.ContainerService) { // Audit Policy configuration if common.IsKubernetesVersionGe(o.OrchestratorVersion, "1.8.0") { - staticLinuxAPIServerConfig["--audit-policy-file"] = "/etc/kubernetes/manifests/audit-policy.yaml" + staticAPIServerConfig["--audit-policy-file"] = "/etc/kubernetes/manifests/audit-policy.yaml" } // RBAC configuration @@ -122,13 +110,7 @@ func setAPIServerConfig(cs *api.ContainerService) { // We don't support user-configurable values for the following, // so any of the value assignments below will override user-provided values - var overrideAPIServerConfig map[string]string - if cs.Properties.HasWindows() { - overrideAPIServerConfig = staticWindowsAPIServerConfig - } else { - overrideAPIServerConfig = staticLinuxAPIServerConfig - } - for key, val := range overrideAPIServerConfig { + for key, val := range staticAPIServerConfig { o.KubernetesConfig.APIServerConfig[key] = val } diff --git a/pkg/acsengine/defaults-cloud-controller-manager.go b/pkg/acsengine/defaults-cloud-controller-manager.go index d53dcded8d..44443defaa 100644 --- a/pkg/acsengine/defaults-cloud-controller-manager.go +++ b/pkg/acsengine/defaults-cloud-controller-manager.go @@ -8,7 +8,7 @@ import ( func setCloudControllerManagerConfig(cs *api.ContainerService) { o := cs.Properties.OrchestratorProfile - staticLinuxCloudControllerManagerConfig := map[string]string{ + staticCloudControllerManagerConfig := map[string]string{ "--allocate-node-cidrs": strconv.FormatBool(!o.IsAzureCNI()), "--configure-cloud-routes": strconv.FormatBool(o.RequireRouteTable()), "--cloud-provider": "azure", @@ -21,18 +21,11 @@ func setCloudControllerManagerConfig(cs *api.ContainerService) { // Set --cluster-name based on appropriate DNS prefix if cs.Properties.MasterProfile != nil { - staticLinuxCloudControllerManagerConfig["--cluster-name"] = cs.Properties.MasterProfile.DNSPrefix + staticCloudControllerManagerConfig["--cluster-name"] = cs.Properties.MasterProfile.DNSPrefix } else if cs.Properties.HostedMasterProfile != nil { - staticLinuxCloudControllerManagerConfig["--cluster-name"] = cs.Properties.HostedMasterProfile.DNSPrefix + staticCloudControllerManagerConfig["--cluster-name"] = cs.Properties.HostedMasterProfile.DNSPrefix } - staticWindowsCloudControllerManagerConfig := make(map[string]string) - for key, val := range staticLinuxCloudControllerManagerConfig { - staticWindowsCloudControllerManagerConfig[key] = val - } - // Windows cloud-controller-manager config overrides - // TODO placeholder for specific config overrides for Windows clusters - // Default cloud-controller-manager config defaultCloudControllerManagerConfig := map[string]string{ "--route-reconciliation-period": DefaultKubernetesCtrlMgrRouteReconciliationPeriod, @@ -53,13 +46,7 @@ func setCloudControllerManagerConfig(cs *api.ContainerService) { // We don't support user-configurable values for the following, // so any of the value assignments below will override user-provided values - var overrideCloudControllerManagerConfig map[string]string - if cs.Properties.HasWindows() { - overrideCloudControllerManagerConfig = staticWindowsCloudControllerManagerConfig - } else { - overrideCloudControllerManagerConfig = staticLinuxCloudControllerManagerConfig - } - for key, val := range overrideCloudControllerManagerConfig { + for key, val := range staticCloudControllerManagerConfig { o.KubernetesConfig.CloudControllerManagerConfig[key] = val } diff --git a/pkg/acsengine/defaults-controller-manager.go b/pkg/acsengine/defaults-controller-manager.go index b16a560e70..516f1f3e85 100644 --- a/pkg/acsengine/defaults-controller-manager.go +++ b/pkg/acsengine/defaults-controller-manager.go @@ -9,7 +9,7 @@ import ( func setControllerManagerConfig(cs *api.ContainerService) { o := cs.Properties.OrchestratorProfile - staticLinuxControllerManagerConfig := map[string]string{ + staticControllerManagerConfig := map[string]string{ "--kubeconfig": "/var/lib/kubelet/kubeconfig", "--allocate-node-cidrs": strconv.FormatBool(!o.IsAzureCNI()), "--configure-cloud-routes": strconv.FormatBool(o.RequireRouteTable()), @@ -25,24 +25,17 @@ func setControllerManagerConfig(cs *api.ContainerService) { // Set --cluster-name based on appropriate DNS prefix if cs.Properties.MasterProfile != nil { - staticLinuxControllerManagerConfig["--cluster-name"] = cs.Properties.MasterProfile.DNSPrefix + staticControllerManagerConfig["--cluster-name"] = cs.Properties.MasterProfile.DNSPrefix } else if cs.Properties.HostedMasterProfile != nil { - staticLinuxControllerManagerConfig["--cluster-name"] = cs.Properties.HostedMasterProfile.DNSPrefix + staticControllerManagerConfig["--cluster-name"] = cs.Properties.HostedMasterProfile.DNSPrefix } // Enable cloudprovider if we're not using cloud controller manager if !helpers.IsTrueBoolPointer(o.KubernetesConfig.UseCloudControllerManager) { - staticLinuxControllerManagerConfig["--cloud-provider"] = "azure" - staticLinuxControllerManagerConfig["--cloud-config"] = "/etc/kubernetes/azure.json" + staticControllerManagerConfig["--cloud-provider"] = "azure" + staticControllerManagerConfig["--cloud-config"] = "/etc/kubernetes/azure.json" } - staticWindowsControllerManagerConfig := make(map[string]string) - for key, val := range staticLinuxControllerManagerConfig { - staticWindowsControllerManagerConfig[key] = val - } - // Windows controller-manager config overrides - // TODO placeholder for specific config overrides for Windows clusters - // Default controller-manager config defaultControllerManagerConfig := map[string]string{ "--node-monitor-grace-period": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod, @@ -70,13 +63,7 @@ func setControllerManagerConfig(cs *api.ContainerService) { // We don't support user-configurable values for the following, // so any of the value assignments below will override user-provided values - var overrideControllerManagerConfig map[string]string - if cs.Properties.HasWindows() { - overrideControllerManagerConfig = staticWindowsControllerManagerConfig - } else { - overrideControllerManagerConfig = staticLinuxControllerManagerConfig - } - for key, val := range overrideControllerManagerConfig { + for key, val := range staticControllerManagerConfig { o.KubernetesConfig.ControllerManagerConfig[key] = val } diff --git a/pkg/acsengine/defaults-scheduler.go b/pkg/acsengine/defaults-scheduler.go index 444a3ce4b3..87e0f6ecb9 100644 --- a/pkg/acsengine/defaults-scheduler.go +++ b/pkg/acsengine/defaults-scheduler.go @@ -4,8 +4,8 @@ import ( "github.com/Azure/acs-engine/pkg/api" ) -// staticLinuxSchedulerConfig is not user-overridable -var staticLinuxSchedulerConfig = map[string]string{ +// staticSchedulerConfig is not user-overridable +var staticSchedulerConfig = map[string]string{ "--kubeconfig": "/var/lib/kubelet/kubeconfig", "--leader-elect": "true", "--profiling": "false", @@ -18,12 +18,6 @@ var defaultSchedulerConfig = map[string]string{ func setSchedulerConfig(cs *api.ContainerService) { o := cs.Properties.OrchestratorProfile - staticWindowsSchedulerConfig := make(map[string]string) - for key, val := range staticLinuxSchedulerConfig { - staticWindowsSchedulerConfig[key] = val - } - // Windows scheduler config overrides - // TODO placeholder for specific config overrides for Windows clusters // If no user-configurable scheduler config values exists, use the defaults if o.KubernetesConfig.SchedulerConfig == nil { @@ -40,13 +34,7 @@ func setSchedulerConfig(cs *api.ContainerService) { // We don't support user-configurable values for the following, // so any of the value assignments below will override user-provided values - var overrideSchedulerConfig map[string]string - if cs.Properties.HasWindows() { - overrideSchedulerConfig = staticWindowsSchedulerConfig - } else { - overrideSchedulerConfig = staticLinuxSchedulerConfig - } - for key, val := range overrideSchedulerConfig { + for key, val := range staticSchedulerConfig { o.KubernetesConfig.SchedulerConfig[key] = val } } diff --git a/pkg/acsengine/defaults-scheduler_test.go b/pkg/acsengine/defaults-scheduler_test.go index 05524e9f07..bae224265d 100644 --- a/pkg/acsengine/defaults-scheduler_test.go +++ b/pkg/acsengine/defaults-scheduler_test.go @@ -8,7 +8,7 @@ func TestSchedulerDefaultConfig(t *testing.T) { cs := createContainerService("testcluster", "1.9.6", 3, 2) setSchedulerConfig(cs) s := cs.Properties.OrchestratorProfile.KubernetesConfig.SchedulerConfig - for key, val := range staticLinuxSchedulerConfig { + for key, val := range staticSchedulerConfig { if val != s[key] { t.Fatalf("got unexpected kube-scheduler static config value for %s. Expected %s, got %s", key, val, s[key]) @@ -46,7 +46,7 @@ func TestSchedulerStaticConfig(t *testing.T) { "--profiling": "user-override", } setSchedulerConfig(cs) - for key, val := range staticLinuxSchedulerConfig { + for key, val := range staticSchedulerConfig { if val != cs.Properties.OrchestratorProfile.KubernetesConfig.SchedulerConfig[key] { t.Fatalf("kube-scheduler static config did not override user values for %s. Expected %s, got %s", key, val, cs.Properties.OrchestratorProfile.KubernetesConfig.SchedulerConfig) diff --git a/pkg/acsengine/defaults.go b/pkg/acsengine/defaults.go index 17939f1e87..8130d3369f 100644 --- a/pkg/acsengine/defaults.go +++ b/pkg/acsengine/defaults.go @@ -21,7 +21,7 @@ const ( // AzureCniPluginVer specifies version of Azure CNI plugin, which has been mirrored from // https://github.com/Azure/azure-container-networking/releases/download/${AZURE_PLUGIN_VER}/azure-vnet-cni-linux-amd64-${AZURE_PLUGIN_VER}.tgz // to https://acs-mirror.azureedge.net/cni/ - AzureCniPluginVer = "v1.0.6" + AzureCniPluginVer = "v1.0.4" // CNIPluginVer specifies the version of CNI implementation // https://github.com/containernetworking/plugins CNIPluginVer = "v0.7.0" @@ -34,6 +34,7 @@ var ( TillerImageBase: "gcrio.azureedge.net/kubernetes-helm/", ACIConnectorImageBase: "microsoft/", NVIDIAImageBase: "nvidia/", + AzureCNIImageBase: "containernetworking/", EtcdDownloadURLBase: "https://acs-mirror.azureedge.net/github-coreos", KubeBinariesSASURLBase: "https://acs-mirror.azureedge.net/wink8s/", WindowsPackageSASURLBase: "https://acs-mirror.azureedge.net/wink8s/", @@ -66,7 +67,7 @@ var ( ImageOffer: "UbuntuServer", ImageSku: "16.04-LTS", ImagePublisher: "Canonical", - ImageVersion: "16.04.201805220", + ImageVersion: "16.04.201806120", } //DefaultRHELOSImageConfig is the RHEL Linux distribution. @@ -320,16 +321,26 @@ var ( Enabled: helpers.PointerToBool(api.DefaultContainerMonitoringAddonEnabled), Config: map[string]string{ "omsAgentVersion": "1.6.0-42", - "dockerProviderVersion": "2.0.0-2", + "dockerProviderVersion": "2.0.0-3", }, Containers: []api.KubernetesContainerSpec{ { Name: "omsagent", - Image: "dockerio.azureedge.net/microsoft/oms:ciprod05082018", + Image: "microsoft/oms:ciprod06072018", CPURequests: "50m", - MemoryRequests: "150Mi", - CPULimits: "50m", - MemoryLimits: "150Mi", + MemoryRequests: "100Mi", + CPULimits: "150m", + MemoryLimits: "500Mi", + }, + }, + } + + // DefaultAzureCNINetworkMonitorAddonsConfig is the default Azure CNI networkmonitor Kubernetes addon Config + DefaultAzureCNINetworkMonitorAddonsConfig = api.KubernetesAddon{ + Name: AzureCNINetworkMonitoringAddonName, + Containers: []api.KubernetesContainerSpec{ + { + Name: AzureCNINetworkMonitoringAddonName, }, }, } @@ -405,8 +416,9 @@ func setOrchestratorDefaults(cs *api.ContainerService) { DefaultMetricsServerAddonsConfig, DefaultNVIDIADevicePluginAddonsConfig, DefaultContainerMonitoringAddonsConfig, + DefaultAzureCNINetworkMonitorAddonsConfig, } - enforceK8sVersionAddonOverrides(o.KubernetesConfig.Addons, o) + enforceK8sAddonOverrides(o.KubernetesConfig.Addons, o) } else { // For each addon, provide default configuration if user didn't provide its own config t := getAddonsIndexByName(o.KubernetesConfig.Addons, DefaultTillerAddonName) @@ -451,6 +463,11 @@ func setOrchestratorDefaults(cs *api.ContainerService) { // Provide default acs-engine config for Container Monitoring o.KubernetesConfig.Addons = append(o.KubernetesConfig.Addons, DefaultContainerMonitoringAddonsConfig) } + aN := getAddonsIndexByName(o.KubernetesConfig.Addons, AzureCNINetworkMonitoringAddonName) + if aN < 0 { + // Provide default acs-engine config for Azure CNI containernetworking Device Plugin + o.KubernetesConfig.Addons = append(o.KubernetesConfig.Addons, DefaultAzureCNINetworkMonitorAddonsConfig) + } } if o.KubernetesConfig.KubernetesImageBase == "" { o.KubernetesConfig.KubernetesImageBase = cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase @@ -554,6 +571,10 @@ func setOrchestratorDefaults(cs *api.ContainerService) { if a.OrchestratorProfile.KubernetesConfig.Addons[cm].IsEnabled(api.DefaultContainerMonitoringAddonEnabled) { a.OrchestratorProfile.KubernetesConfig.Addons[cm] = assignDefaultAddonVals(a.OrchestratorProfile.KubernetesConfig.Addons[cm], DefaultContainerMonitoringAddonsConfig) } + aN := getAddonsIndexByName(a.OrchestratorProfile.KubernetesConfig.Addons, AzureCNINetworkMonitoringAddonName) + if a.OrchestratorProfile.KubernetesConfig.Addons[aN].IsEnabled(a.OrchestratorProfile.IsAzureCNI()) { + a.OrchestratorProfile.KubernetesConfig.Addons[aN] = assignDefaultAddonVals(a.OrchestratorProfile.KubernetesConfig.Addons[aN], DefaultAzureCNINetworkMonitorAddonsConfig) + } if o.KubernetesConfig.PrivateCluster == nil { o.KubernetesConfig.PrivateCluster = &api.PrivateCluster{} @@ -621,9 +642,6 @@ func setOrchestratorDefaults(cs *api.ContainerService) { if o.DcosConfig == nil { o.DcosConfig = &api.DcosConfig{} } - if o.DcosConfig.DcosWindowsBootstrapURL == "" { - o.DcosConfig.DcosWindowsBootstrapURL = DefaultDCOSSpecConfig.DCOSWindowsBootstrapDownloadURL - } dcosSemVer, _ := semver.Make(o.OrchestratorVersion) dcosBootstrapSemVer, _ := semver.Make(common.DCOSVersion1Dot11Dot0) if !dcosSemVer.LT(dcosBootstrapSemVer) { @@ -635,9 +653,6 @@ func setOrchestratorDefaults(cs *api.ContainerService) { } } case api.OpenShift: - if a.MasterProfile.Distro == "" { - a.MasterProfile.Distro = api.RHEL - } kc := a.OrchestratorProfile.OpenShiftConfig.KubernetesConfig if kc == nil { kc = &api.KubernetesConfig{} @@ -675,10 +690,12 @@ func setMasterNetworkDefaults(a *api.Properties, isUpgrade bool) { if a.MasterProfile == nil { return } - - // Set default Distro to Ubuntu - if a.MasterProfile.Distro == "" { - a.MasterProfile.Distro = api.Ubuntu + // don't default Distro for OpenShift + if !a.OrchestratorProfile.IsOpenShift() { + // Set default Distro to Ubuntu + if a.MasterProfile.Distro == "" { + a.MasterProfile.Distro = api.Ubuntu + } } if !a.MasterProfile.IsCustomVNET() { @@ -769,9 +786,13 @@ func setAgentNetworkDefaults(a *api.Properties) { if profile.OSType == "" { profile.OSType = api.Linux } - // set default Distro to Ubuntu - if profile.Distro == "" { - profile.Distro = api.Ubuntu + + // don't default Distro for OpenShift + if !a.OrchestratorProfile.IsOpenShift() { + // Set default Distro to Ubuntu + if profile.Distro == "" { + profile.Distro = api.Ubuntu + } } // Set the default number of IP addresses allocated for agents. @@ -807,8 +828,12 @@ func setStorageDefaults(a *api.Properties) { } if len(profile.AvailabilityProfile) == 0 { profile.AvailabilityProfile = api.VirtualMachineScaleSets + // VMSS is not supported for k8s below 1.10.0 if a.OrchestratorProfile.OrchestratorType == api.Kubernetes && !common.IsKubernetesVersionGe(a.OrchestratorProfile.OrchestratorVersion, "1.10.0") { profile.AvailabilityProfile = api.AvailabilitySet + // VMSS is not supported with instance metadata for k8s below 1.10.2 + } else if a.OrchestratorProfile.OrchestratorType == api.Kubernetes && helpers.IsTrueBoolPointer(a.OrchestratorProfile.KubernetesConfig.UseInstanceMetadata) && !common.IsKubernetesVersionGe(a.OrchestratorProfile.OrchestratorVersion, "1.10.2") { + profile.AvailabilityProfile = api.AvailabilitySet } } if len(profile.ScaleSetEvictionPolicy) == 0 && profile.ScaleSetPriority == api.ScaleSetPriorityLow { @@ -1075,15 +1100,21 @@ func mapToString(valueMap map[string]string) string { return strings.TrimSuffix(buf.String(), ",") } -func enforceK8sVersionAddonOverrides(addons []api.KubernetesAddon, o *api.OrchestratorProfile) { +func enforceK8sAddonOverrides(addons []api.KubernetesAddon, o *api.OrchestratorProfile) { m := getAddonsIndexByName(o.KubernetesConfig.Addons, DefaultMetricsServerAddonName) o.KubernetesConfig.Addons[m].Enabled = k8sVersionMetricsServerAddonEnabled(o) + aN := getAddonsIndexByName(o.KubernetesConfig.Addons, AzureCNINetworkMonitoringAddonName) + o.KubernetesConfig.Addons[aN].Enabled = azureCNINetworkMonitorAddonEnabled(o) } func k8sVersionMetricsServerAddonEnabled(o *api.OrchestratorProfile) *bool { return helpers.PointerToBool(common.IsKubernetesVersionGe(o.OrchestratorVersion, "1.9.0")) } +func azureCNINetworkMonitorAddonEnabled(o *api.OrchestratorProfile) *bool { + return helpers.PointerToBool(o.IsAzureCNI()) +} + func generateEtcdEncryptionKey() string { b := make([]byte, 32) rand.Read(b) diff --git a/pkg/acsengine/defaults_test.go b/pkg/acsengine/defaults_test.go index 773fe004a9..f2578c6cc5 100644 --- a/pkg/acsengine/defaults_test.go +++ b/pkg/acsengine/defaults_test.go @@ -486,7 +486,7 @@ func TestStorageProfile(t *testing.T) { properties.AgentPoolProfiles[0].AvailabilityProfile, api.AvailabilitySet) } - mockCS = getMockBaseContainerService("1.10.0") + mockCS = getMockBaseContainerService("1.10.2") properties = mockCS.Properties properties.OrchestratorProfile.OrchestratorType = "Kubernetes" setPropertiesDefaults(&mockCS, false) @@ -519,6 +519,72 @@ func TestAgentPoolProfile(t *testing.T) { } } +// TestSetComponentsNetworkDefaults covers tests for setMasterNetworkDefaults and setAgentNetworkDefaults +// TODO: Currently this test covers only api.Distro setting. Extend test cases to cover network configuration too. +func TestSetComponentsNetworkDefaults(t *testing.T) { + + var tests = []struct { + name string // test case name + orchestratorProfile api.OrchestratorProfile // orchestrator to be tested + expectedDistro api.Distro // expected result default disto to be used + }{ + { + "ubuntu_kubernetes", + api.OrchestratorProfile{ + OrchestratorType: api.Kubernetes, + }, + api.Ubuntu, + }, + { + "rhel_openshift", + api.OrchestratorProfile{ + OrchestratorType: api.OpenShift, + }, + "", + }, + } + + for _, test := range tests { + mockAPI := getMockAPIProperties("1.0.0") + mockAPI.OrchestratorProfile = &test.orchestratorProfile + setMasterNetworkDefaults(&mockAPI, false) + setAgentNetworkDefaults(&mockAPI) + if mockAPI.MasterProfile.Distro != test.expectedDistro { + t.Fatalf("setMasterNetworkDefaults() test case %v did not return right Distro configurations %v != %v", test.name, mockAPI.MasterProfile.Distro, test.expectedDistro) + } + for _, agent := range mockAPI.AgentPoolProfiles { + if agent.Distro != test.expectedDistro { + t.Fatalf("setAgentNetworkDefaults() test case %v did not return right Distro configurations %v != %v", test.name, agent.Distro, test.expectedDistro) + } + } + } +} + +func TestIsAzureCNINetworkmonitorAddon(t *testing.T) { + mockCS := getMockBaseContainerService("1.10.3") + properties := mockCS.Properties + properties.OrchestratorProfile.OrchestratorType = "Kubernetes" + properties.MasterProfile.Count = 1 + properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin = "azure" + setOrchestratorDefaults(&mockCS) + + i := getAddonsIndexByName(properties.OrchestratorProfile.KubernetesConfig.Addons, AzureCNINetworkMonitoringAddonName) + if !helpers.IsTrueBoolPointer(properties.OrchestratorProfile.KubernetesConfig.Addons[i].Enabled) { + t.Fatalf("Azure CNI network plugin configuration should add Azure CNI networkmonitor addon") + } + mockCS = getMockBaseContainerService("1.10.3") + properties = mockCS.Properties + properties.OrchestratorProfile.OrchestratorType = "Kubernetes" + properties.MasterProfile.Count = 1 + properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin = "kubenet" + setOrchestratorDefaults(&mockCS) + + i = getAddonsIndexByName(properties.OrchestratorProfile.KubernetesConfig.Addons, AzureCNINetworkMonitoringAddonName) + if helpers.IsTrueBoolPointer(properties.OrchestratorProfile.KubernetesConfig.Addons[i].Enabled) { + t.Fatalf("Azure CNI networkmonitor addon should only be present in Azure CNI configurations") + } +} + func getMockAddon(name string) api.KubernetesAddon { return api.KubernetesAddon{ Name: name, @@ -536,20 +602,25 @@ func getMockAddon(name string) api.KubernetesAddon { } func getMockBaseContainerService(orchestratorVersion string) api.ContainerService { + mockAPIProperties := getMockAPIProperties(orchestratorVersion) return api.ContainerService{ - Properties: &api.Properties{ - OrchestratorProfile: &api.OrchestratorProfile{ - OrchestratorVersion: orchestratorVersion, - KubernetesConfig: &api.KubernetesConfig{}, - }, - MasterProfile: &api.MasterProfile{}, - AgentPoolProfiles: []*api.AgentPoolProfile{ - {}, - }, - }, + Properties: &mockAPIProperties, } } +func getMockAPIProperties(orchestratorVersion string) api.Properties { + return api.Properties{ + ProvisioningState: "", + OrchestratorProfile: &api.OrchestratorProfile{ + OrchestratorVersion: orchestratorVersion, + KubernetesConfig: &api.KubernetesConfig{}, + }, + MasterProfile: &api.MasterProfile{}, + AgentPoolProfiles: []*api.AgentPoolProfile{ + {}, + }} +} + func getKubernetesConfigWithFeatureGates(featureGates string) *api.KubernetesConfig { return &api.KubernetesConfig{ KubeletConfig: map[string]string{"--feature-gates": featureGates}, diff --git a/pkg/acsengine/engine.go b/pkg/acsengine/engine.go index 8c3996b452..89b1cd265a 100644 --- a/pkg/acsengine/engine.go +++ b/pkg/acsengine/engine.go @@ -368,6 +368,18 @@ func getDCOSDefaultBootstrapInstallerURL(profile *api.OrchestratorProfile) strin return "" } +func getDCOSDefaultWindowsBootstrapInstallerURL(profile *api.OrchestratorProfile) string { + if profile.OrchestratorType == api.DCOS { + switch profile.OrchestratorVersion { + case common.DCOSVersion1Dot11Dot2: + return "https://dcos-mirror.azureedge.net/dcos-windows/1-11-2" + case common.DCOSVersion1Dot11Dot0: + return "https://dcos-mirror.azureedge.net/dcos-windows/1-11-0" + } + } + return "" +} + func getDCOSDefaultProviderPackageGUID(orchestratorType string, orchestratorVersion string, masterCount int) string { if orchestratorType == api.DCOS { switch orchestratorVersion { diff --git a/pkg/acsengine/k8s_versions.go b/pkg/acsengine/k8s_versions.go index e9913fa049..b917303826 100644 --- a/pkg/acsengine/k8s_versions.go +++ b/pkg/acsengine/k8s_versions.go @@ -9,174 +9,180 @@ import ( var k8sComponentVersions = map[string]map[string]string{ "1.11": { - "dockerEngine": "1.13.*", - "dashboard": "kubernetes-dashboard-amd64:v1.8.3", - "exechealthz": "exechealthz-amd64:1.2", - "addon-resizer": "addon-resizer:1.8.1", - "heapster": "heapster-amd64:v1.5.1", - "metrics-server": "metrics-server-amd64:v0.2.1", - "kube-dns": "k8s-dns-kube-dns-amd64:1.14.8", - "addon-manager": "kube-addon-manager-amd64:v8.6", - "dnsmasq": "k8s-dns-dnsmasq-nanny-amd64:1.14.8", - "pause": "pause-amd64:3.1", - "tiller": "tiller:v2.8.1", - "rescheduler": "rescheduler:v0.3.1", - "aci-connector": "virtual-kubelet:latest", - ContainerMonitoringAddonName: "oms:ciprod05082018", - "nodestatusfreq": DefaultKubernetesNodeStatusUpdateFrequency, - "nodegraceperiod": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod, - "podeviction": DefaultKubernetesCtrlMgrPodEvictionTimeout, - "routeperiod": DefaultKubernetesCtrlMgrRouteReconciliationPeriod, - "backoffretries": strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries), - "backoffjitter": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64), - "backoffduration": strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration), - "backoffexponent": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64), - "ratelimitqps": strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64), - "ratelimitbucket": strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket), - "gchighthreshold": strconv.Itoa(DefaultKubernetesGCHighThreshold), - "gclowthreshold": strconv.Itoa(DefaultKubernetesGCLowThreshold), + "dockerEngine": "1.13.*", + "dashboard": "kubernetes-dashboard-amd64:v1.8.3", + "exechealthz": "exechealthz-amd64:1.2", + "addon-resizer": "addon-resizer:1.8.1", + "heapster": "heapster-amd64:v1.5.1", + "metrics-server": "metrics-server-amd64:v0.2.1", + "kube-dns": "k8s-dns-kube-dns-amd64:1.14.8", + "addon-manager": "kube-addon-manager-amd64:v8.6", + "dnsmasq": "k8s-dns-dnsmasq-nanny-amd64:1.14.8", + "pause": "pause-amd64:3.1", + "tiller": "tiller:v2.8.1", + "rescheduler": "rescheduler:v0.3.1", + "aci-connector": "virtual-kubelet:latest", + ContainerMonitoringAddonName: "oms:ciprod05082018", + AzureCNINetworkMonitoringAddonName: "networkmonitor:v0.0.4", + "nodestatusfreq": DefaultKubernetesNodeStatusUpdateFrequency, + "nodegraceperiod": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod, + "podeviction": DefaultKubernetesCtrlMgrPodEvictionTimeout, + "routeperiod": DefaultKubernetesCtrlMgrRouteReconciliationPeriod, + "backoffretries": strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries), + "backoffjitter": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64), + "backoffduration": strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration), + "backoffexponent": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64), + "ratelimitqps": strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64), + "ratelimitbucket": strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket), + "gchighthreshold": strconv.Itoa(DefaultKubernetesGCHighThreshold), + "gclowthreshold": strconv.Itoa(DefaultKubernetesGCLowThreshold), }, "1.10": { - "dockerEngine": "1.13.*", - "dashboard": "kubernetes-dashboard-amd64:v1.8.3", - "exechealthz": "exechealthz-amd64:1.2", - "addon-resizer": "addon-resizer:1.8.1", - "heapster": "heapster-amd64:v1.5.1", - "metrics-server": "metrics-server-amd64:v0.2.1", - "kube-dns": "k8s-dns-kube-dns-amd64:1.14.8", - "addon-manager": "kube-addon-manager-amd64:v8.6", - "dnsmasq": "k8s-dns-dnsmasq-nanny-amd64:1.14.8", - "pause": "pause-amd64:3.1", - "tiller": "tiller:v2.8.1", - "rescheduler": "rescheduler:v0.3.1", - "aci-connector": "virtual-kubelet:latest", - ContainerMonitoringAddonName: "oms:ciprod05082018", - "cluster-autoscaler": "cluster-autoscaler:v1.2.2", - "nvidia-device-plugin": "k8s-device-plugin:1.10", - "nodestatusfreq": DefaultKubernetesNodeStatusUpdateFrequency, - "nodegraceperiod": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod, - "podeviction": DefaultKubernetesCtrlMgrPodEvictionTimeout, - "routeperiod": DefaultKubernetesCtrlMgrRouteReconciliationPeriod, - "backoffretries": strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries), - "backoffjitter": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64), - "backoffduration": strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration), - "backoffexponent": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64), - "ratelimitqps": strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64), - "ratelimitbucket": strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket), - "gchighthreshold": strconv.Itoa(DefaultKubernetesGCHighThreshold), - "gclowthreshold": strconv.Itoa(DefaultKubernetesGCLowThreshold), + "dockerEngine": "1.13.*", + "dashboard": "kubernetes-dashboard-amd64:v1.8.3", + "exechealthz": "exechealthz-amd64:1.2", + "addon-resizer": "addon-resizer:1.8.1", + "heapster": "heapster-amd64:v1.5.1", + "metrics-server": "metrics-server-amd64:v0.2.1", + "kube-dns": "k8s-dns-kube-dns-amd64:1.14.8", + "addon-manager": "kube-addon-manager-amd64:v8.6", + "dnsmasq": "k8s-dns-dnsmasq-nanny-amd64:1.14.8", + "pause": "pause-amd64:3.1", + "tiller": "tiller:v2.8.1", + "rescheduler": "rescheduler:v0.3.1", + "aci-connector": "virtual-kubelet:latest", + ContainerMonitoringAddonName: "oms:ciprod05082018", + AzureCNINetworkMonitoringAddonName: "networkmonitor:v0.0.4", + "cluster-autoscaler": "cluster-autoscaler:v1.2.2", + "nvidia-device-plugin": "k8s-device-plugin:1.10", + "nodestatusfreq": DefaultKubernetesNodeStatusUpdateFrequency, + "nodegraceperiod": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod, + "podeviction": DefaultKubernetesCtrlMgrPodEvictionTimeout, + "routeperiod": DefaultKubernetesCtrlMgrRouteReconciliationPeriod, + "backoffretries": strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries), + "backoffjitter": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64), + "backoffduration": strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration), + "backoffexponent": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64), + "ratelimitqps": strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64), + "ratelimitbucket": strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket), + "gchighthreshold": strconv.Itoa(DefaultKubernetesGCHighThreshold), + "gclowthreshold": strconv.Itoa(DefaultKubernetesGCLowThreshold), }, "1.9": { - "dockerEngine": "1.13.*", - "dashboard": "kubernetes-dashboard-amd64:v1.8.3", - "exechealthz": "exechealthz-amd64:1.2", - "addon-resizer": "addon-resizer:1.8.1", - "heapster": "heapster-amd64:v1.5.1", - "metrics-server": "metrics-server-amd64:v0.2.1", - "kube-dns": "k8s-dns-kube-dns-amd64:1.14.8", - "addon-manager": "kube-addon-manager-amd64:v8.6", - "dnsmasq": "k8s-dns-dnsmasq-nanny-amd64:1.14.8", - "pause": "pause-amd64:3.1", - "tiller": "tiller:v2.8.1", - "rescheduler": "rescheduler:v0.3.1", - "aci-connector": "virtual-kubelet:latest", - ContainerMonitoringAddonName: "oms:ciprod05082018", - "cluster-autoscaler": "cluster-autoscaler:v1.1.2", - "nodestatusfreq": DefaultKubernetesNodeStatusUpdateFrequency, - "nodegraceperiod": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod, - "podeviction": DefaultKubernetesCtrlMgrPodEvictionTimeout, - "routeperiod": DefaultKubernetesCtrlMgrRouteReconciliationPeriod, - "backoffretries": strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries), - "backoffjitter": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64), - "backoffduration": strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration), - "backoffexponent": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64), - "ratelimitqps": strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64), - "ratelimitbucket": strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket), - "gchighthreshold": strconv.Itoa(DefaultKubernetesGCHighThreshold), - "gclowthreshold": strconv.Itoa(DefaultKubernetesGCLowThreshold), + "dockerEngine": "1.13.*", + "dashboard": "kubernetes-dashboard-amd64:v1.8.3", + "exechealthz": "exechealthz-amd64:1.2", + "addon-resizer": "addon-resizer:1.8.1", + "heapster": "heapster-amd64:v1.5.1", + "metrics-server": "metrics-server-amd64:v0.2.1", + "kube-dns": "k8s-dns-kube-dns-amd64:1.14.8", + "addon-manager": "kube-addon-manager-amd64:v8.6", + "dnsmasq": "k8s-dns-dnsmasq-nanny-amd64:1.14.8", + "pause": "pause-amd64:3.1", + "tiller": "tiller:v2.8.1", + "rescheduler": "rescheduler:v0.3.1", + "aci-connector": "virtual-kubelet:latest", + ContainerMonitoringAddonName: "oms:ciprod05082018", + AzureCNINetworkMonitoringAddonName: "networkmonitor:v0.0.4", + "cluster-autoscaler": "cluster-autoscaler:v1.1.2", + "nodestatusfreq": DefaultKubernetesNodeStatusUpdateFrequency, + "nodegraceperiod": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod, + "podeviction": DefaultKubernetesCtrlMgrPodEvictionTimeout, + "routeperiod": DefaultKubernetesCtrlMgrRouteReconciliationPeriod, + "backoffretries": strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries), + "backoffjitter": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64), + "backoffduration": strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration), + "backoffexponent": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64), + "ratelimitqps": strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64), + "ratelimitbucket": strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket), + "gchighthreshold": strconv.Itoa(DefaultKubernetesGCHighThreshold), + "gclowthreshold": strconv.Itoa(DefaultKubernetesGCLowThreshold), }, "1.8": { - "dockerEngine": "1.13.*", - "dashboard": "kubernetes-dashboard-amd64:v1.8.3", - "exechealthz": "exechealthz-amd64:1.2", - "addon-resizer": "addon-resizer:1.7", - "heapster": "heapster-amd64:v1.5.1", - "metrics-server": "metrics-server-amd64:v0.2.1", - "kube-dns": "k8s-dns-kube-dns-amd64:1.14.8", - "addon-manager": "kube-addon-manager-amd64:v8.6", - "dnsmasq": "k8s-dns-dnsmasq-nanny-amd64:1.14.8", - "pause": "pause-amd64:3.1", - "tiller": "tiller:v2.8.1", - "rescheduler": "rescheduler:v0.3.1", - "aci-connector": "virtual-kubelet:latest", - ContainerMonitoringAddonName: "oms:ciprod05082018", - "nodestatusfreq": DefaultKubernetesNodeStatusUpdateFrequency, - "nodegraceperiod": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod, - "podeviction": DefaultKubernetesCtrlMgrPodEvictionTimeout, - "routeperiod": DefaultKubernetesCtrlMgrRouteReconciliationPeriod, - "backoffretries": strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries), - "backoffjitter": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64), - "backoffduration": strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration), - "backoffexponent": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64), - "ratelimitqps": strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64), - "ratelimitbucket": strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket), - "gchighthreshold": strconv.Itoa(DefaultKubernetesGCHighThreshold), - "gclowthreshold": strconv.Itoa(DefaultKubernetesGCLowThreshold), + "dockerEngine": "1.13.*", + "dashboard": "kubernetes-dashboard-amd64:v1.8.3", + "exechealthz": "exechealthz-amd64:1.2", + "addon-resizer": "addon-resizer:1.7", + "heapster": "heapster-amd64:v1.5.1", + "metrics-server": "metrics-server-amd64:v0.2.1", + "kube-dns": "k8s-dns-kube-dns-amd64:1.14.8", + "addon-manager": "kube-addon-manager-amd64:v8.6", + "dnsmasq": "k8s-dns-dnsmasq-nanny-amd64:1.14.8", + "pause": "pause-amd64:3.1", + "tiller": "tiller:v2.8.1", + "rescheduler": "rescheduler:v0.3.1", + "aci-connector": "virtual-kubelet:latest", + ContainerMonitoringAddonName: "oms:ciprod05082018", + AzureCNINetworkMonitoringAddonName: "networkmonitor:v0.0.4", + "nodestatusfreq": DefaultKubernetesNodeStatusUpdateFrequency, + "nodegraceperiod": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod, + "podeviction": DefaultKubernetesCtrlMgrPodEvictionTimeout, + "routeperiod": DefaultKubernetesCtrlMgrRouteReconciliationPeriod, + "backoffretries": strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries), + "backoffjitter": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64), + "backoffduration": strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration), + "backoffexponent": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64), + "ratelimitqps": strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64), + "ratelimitbucket": strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket), + "gchighthreshold": strconv.Itoa(DefaultKubernetesGCHighThreshold), + "gclowthreshold": strconv.Itoa(DefaultKubernetesGCLowThreshold), }, "1.7": { - "dockerEngine": "1.13.*", - "dashboard": "kubernetes-dashboard-amd64:v1.6.3", - "exechealthz": "exechealthz-amd64:1.2", - "addon-resizer": "addon-resizer:1.7", - "heapster": "heapster-amd64:v1.5.1", - "metrics-server": "metrics-server-amd64:v0.2.1", - "kube-dns": "k8s-dns-kube-dns-amd64:1.14.5", - "addon-manager": "kube-addon-manager-amd64:v8.6", - "dnsmasq": "k8s-dns-dnsmasq-nanny-amd64:1.14.5", - "pause": "pause-amd64:3.1", - "tiller": "tiller:v2.8.1", - "rescheduler": "rescheduler:v0.3.1", - "aci-connector": "virtual-kubelet:latest", - ContainerMonitoringAddonName: "oms:ciprod05082018", - "nodestatusfreq": DefaultKubernetesNodeStatusUpdateFrequency, - "nodegraceperiod": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod, - "podeviction": DefaultKubernetesCtrlMgrPodEvictionTimeout, - "routeperiod": DefaultKubernetesCtrlMgrRouteReconciliationPeriod, - "backoffretries": strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries), - "backoffjitter": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64), - "backoffduration": strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration), - "backoffexponent": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64), - "ratelimitqps": strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64), - "ratelimitbucket": strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket), - "gchighthreshold": strconv.Itoa(DefaultKubernetesGCHighThreshold), - "gclowthreshold": strconv.Itoa(DefaultKubernetesGCLowThreshold), + "dockerEngine": "1.13.*", + "dashboard": "kubernetes-dashboard-amd64:v1.6.3", + "exechealthz": "exechealthz-amd64:1.2", + "addon-resizer": "addon-resizer:1.7", + "heapster": "heapster-amd64:v1.5.1", + "metrics-server": "metrics-server-amd64:v0.2.1", + "kube-dns": "k8s-dns-kube-dns-amd64:1.14.5", + "addon-manager": "kube-addon-manager-amd64:v8.6", + "dnsmasq": "k8s-dns-dnsmasq-nanny-amd64:1.14.5", + "pause": "pause-amd64:3.1", + "tiller": "tiller:v2.8.1", + "rescheduler": "rescheduler:v0.3.1", + "aci-connector": "virtual-kubelet:latest", + ContainerMonitoringAddonName: "oms:ciprod05082018", + AzureCNINetworkMonitoringAddonName: "networkmonitor:v0.0.4", + "nodestatusfreq": DefaultKubernetesNodeStatusUpdateFrequency, + "nodegraceperiod": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod, + "podeviction": DefaultKubernetesCtrlMgrPodEvictionTimeout, + "routeperiod": DefaultKubernetesCtrlMgrRouteReconciliationPeriod, + "backoffretries": strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries), + "backoffjitter": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64), + "backoffduration": strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration), + "backoffexponent": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64), + "ratelimitqps": strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64), + "ratelimitbucket": strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket), + "gchighthreshold": strconv.Itoa(DefaultKubernetesGCHighThreshold), + "gclowthreshold": strconv.Itoa(DefaultKubernetesGCLowThreshold), }, "1.6": { - "dockerEngine": "1.12.*", - "dashboard": "kubernetes-dashboard-amd64:v1.6.3", - "exechealthz": "exechealthz-amd64:1.2", - "addon-resizer": "addon-resizer:1.7", - "heapster": "heapster-amd64:v1.3.0", - "metrics-server": "metrics-server-amd64:v0.2.1", - "kube-dns": "k8s-dns-kube-dns-amd64:1.14.5", - "addon-manager": "kube-addon-manager-amd64:v6.5", - "dnsmasq": "k8s-dns-dnsmasq-nanny-amd64:1.14.5", - "pause": "pause-amd64:3.0", - "tiller": "tiller:v2.8.1", - "rescheduler": "rescheduler:v0.3.1", - "aci-connector": "virtual-kubelet:latest", - "nodestatusfreq": DefaultKubernetesNodeStatusUpdateFrequency, - "nodegraceperiod": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod, - "podeviction": DefaultKubernetesCtrlMgrPodEvictionTimeout, - "routeperiod": DefaultKubernetesCtrlMgrRouteReconciliationPeriod, - "backoffretries": strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries), - "backoffjitter": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64), - "backoffduration": strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration), - "backoffexponent": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64), - "ratelimitqps": strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64), - "ratelimitbucket": strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket), - "gchighthreshold": strconv.Itoa(DefaultKubernetesGCHighThreshold), - "gclowthreshold": strconv.Itoa(DefaultKubernetesGCLowThreshold), + "dockerEngine": "1.12.*", + "dashboard": "kubernetes-dashboard-amd64:v1.6.3", + "exechealthz": "exechealthz-amd64:1.2", + "addon-resizer": "addon-resizer:1.7", + "heapster": "heapster-amd64:v1.3.0", + "metrics-server": "metrics-server-amd64:v0.2.1", + "kube-dns": "k8s-dns-kube-dns-amd64:1.14.5", + "addon-manager": "kube-addon-manager-amd64:v6.5", + "dnsmasq": "k8s-dns-dnsmasq-nanny-amd64:1.14.5", + "pause": "pause-amd64:3.0", + "tiller": "tiller:v2.8.1", + "rescheduler": "rescheduler:v0.3.1", + "aci-connector": "virtual-kubelet:latest", + AzureCNINetworkMonitoringAddonName: "networkmonitor:v0.0.4", + "nodestatusfreq": DefaultKubernetesNodeStatusUpdateFrequency, + "nodegraceperiod": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod, + "podeviction": DefaultKubernetesCtrlMgrPodEvictionTimeout, + "routeperiod": DefaultKubernetesCtrlMgrRouteReconciliationPeriod, + "backoffretries": strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries), + "backoffjitter": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64), + "backoffduration": strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration), + "backoffexponent": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64), + "ratelimitqps": strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64), + "ratelimitbucket": strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket), + "gchighthreshold": strconv.Itoa(DefaultKubernetesGCHighThreshold), + "gclowthreshold": strconv.Itoa(DefaultKubernetesGCLowThreshold), }, } @@ -248,26 +254,27 @@ func getK8sVersionComponents(version string, overrides map[string]string) map[st "addonresizer": k8sComponentVersions["1.11"]["addon-resizer"], "heapster": k8sComponentVersions["1.11"]["heapster"], DefaultMetricsServerAddonName: k8sComponentVersions["1.11"]["metrics-server"], - "dns": k8sComponentVersions["1.11"]["kube-dns"], - "addonmanager": k8sComponentVersions["1.11"]["addon-manager"], - "dnsmasq": k8sComponentVersions["1.11"]["dnsmasq"], - "pause": k8sComponentVersions["1.11"]["pause"], - DefaultTillerAddonName: k8sComponentVersions["1.11"]["tiller"], - DefaultReschedulerAddonName: k8sComponentVersions["1.11"]["rescheduler"], - DefaultACIConnectorAddonName: k8sComponentVersions["1.11"]["aci-connector"], - ContainerMonitoringAddonName: k8sComponentVersions["1.11"][ContainerMonitoringAddonName], - "nodestatusfreq": k8sComponentVersions["1.11"]["nodestatusfreq"], - "nodegraceperiod": k8sComponentVersions["1.11"]["nodegraceperiod"], - "podeviction": k8sComponentVersions["1.11"]["podeviction"], - "routeperiod": k8sComponentVersions["1.11"]["routeperiod"], - "backoffretries": k8sComponentVersions["1.11"]["backoffretries"], - "backoffjitter": k8sComponentVersions["1.11"]["backoffjitter"], - "backoffduration": k8sComponentVersions["1.11"]["backoffduration"], - "backoffexponent": k8sComponentVersions["1.11"]["backoffexponent"], - "ratelimitqps": k8sComponentVersions["1.11"]["ratelimitqps"], - "ratelimitbucket": k8sComponentVersions["1.11"]["ratelimitbucket"], - "gchighthreshold": k8sComponentVersions["1.11"]["gchighthreshold"], - "gclowthreshold": k8sComponentVersions["1.11"]["gclowthreshold"], + "dns": k8sComponentVersions["1.11"]["kube-dns"], + "addonmanager": k8sComponentVersions["1.11"]["addon-manager"], + "dnsmasq": k8sComponentVersions["1.11"]["dnsmasq"], + "pause": k8sComponentVersions["1.11"]["pause"], + DefaultTillerAddonName: k8sComponentVersions["1.11"]["tiller"], + DefaultReschedulerAddonName: k8sComponentVersions["1.11"]["rescheduler"], + DefaultACIConnectorAddonName: k8sComponentVersions["1.11"]["aci-connector"], + ContainerMonitoringAddonName: k8sComponentVersions["1.11"][ContainerMonitoringAddonName], + AzureCNINetworkMonitoringAddonName: k8sComponentVersions["1.11"][AzureCNINetworkMonitoringAddonName], + "nodestatusfreq": k8sComponentVersions["1.11"]["nodestatusfreq"], + "nodegraceperiod": k8sComponentVersions["1.11"]["nodegraceperiod"], + "podeviction": k8sComponentVersions["1.11"]["podeviction"], + "routeperiod": k8sComponentVersions["1.11"]["routeperiod"], + "backoffretries": k8sComponentVersions["1.11"]["backoffretries"], + "backoffjitter": k8sComponentVersions["1.11"]["backoffjitter"], + "backoffduration": k8sComponentVersions["1.11"]["backoffduration"], + "backoffexponent": k8sComponentVersions["1.11"]["backoffexponent"], + "ratelimitqps": k8sComponentVersions["1.11"]["ratelimitqps"], + "ratelimitbucket": k8sComponentVersions["1.11"]["ratelimitbucket"], + "gchighthreshold": k8sComponentVersions["1.11"]["gchighthreshold"], + "gclowthreshold": k8sComponentVersions["1.11"]["gclowthreshold"], } case "1.10": ret = map[string]string{ @@ -287,7 +294,8 @@ func getK8sVersionComponents(version string, overrides map[string]string) map[st DefaultTillerAddonName: k8sComponentVersions["1.10"]["tiller"], DefaultReschedulerAddonName: k8sComponentVersions["1.10"]["rescheduler"], DefaultACIConnectorAddonName: k8sComponentVersions["1.10"]["aci-connector"], - ContainerMonitoringAddonName: k8sComponentVersions["1.11"][ContainerMonitoringAddonName], + ContainerMonitoringAddonName: k8sComponentVersions["1.10"][ContainerMonitoringAddonName], + AzureCNINetworkMonitoringAddonName: k8sComponentVersions["1.10"][AzureCNINetworkMonitoringAddonName], "nodestatusfreq": k8sComponentVersions["1.10"]["nodestatusfreq"], "nodegraceperiod": k8sComponentVersions["1.10"]["nodegraceperiod"], "podeviction": k8sComponentVersions["1.10"]["podeviction"], @@ -314,27 +322,28 @@ func getK8sVersionComponents(version string, overrides map[string]string) map[st "addonresizer": k8sComponentVersions["1.9"]["addon-resizer"], "heapster": k8sComponentVersions["1.9"]["heapster"], DefaultMetricsServerAddonName: k8sComponentVersions["1.9"]["metrics-server"], - "dns": k8sComponentVersions["1.9"]["kube-dns"], - "addonmanager": k8sComponentVersions["1.9"]["addon-manager"], - "dnsmasq": k8sComponentVersions["1.9"]["dnsmasq"], - "pause": k8sComponentVersions["1.9"]["pause"], - DefaultTillerAddonName: k8sComponentVersions["1.9"]["tiller"], - DefaultReschedulerAddonName: k8sComponentVersions["1.9"]["rescheduler"], - DefaultACIConnectorAddonName: k8sComponentVersions["1.9"]["aci-connector"], - ContainerMonitoringAddonName: k8sComponentVersions["1.11"][ContainerMonitoringAddonName], - "nodestatusfreq": k8sComponentVersions["1.9"]["nodestatusfreq"], - "nodegraceperiod": k8sComponentVersions["1.9"]["nodegraceperiod"], - "podeviction": k8sComponentVersions["1.9"]["podeviction"], - "routeperiod": k8sComponentVersions["1.9"]["routeperiod"], - "backoffretries": k8sComponentVersions["1.9"]["backoffretries"], - "backoffjitter": k8sComponentVersions["1.9"]["backoffjitter"], - "backoffduration": k8sComponentVersions["1.9"]["backoffduration"], - "backoffexponent": k8sComponentVersions["1.9"]["backoffexponent"], - "ratelimitqps": k8sComponentVersions["1.9"]["ratelimitqps"], - "ratelimitbucket": k8sComponentVersions["1.9"]["ratelimitbucket"], - "gchighthreshold": k8sComponentVersions["1.9"]["gchighthreshold"], - "gclowthreshold": k8sComponentVersions["1.9"]["gclowthreshold"], - DefaultClusterAutoscalerAddonName: k8sComponentVersions["1.9"]["cluster-autoscaler"], + "dns": k8sComponentVersions["1.9"]["kube-dns"], + "addonmanager": k8sComponentVersions["1.9"]["addon-manager"], + "dnsmasq": k8sComponentVersions["1.9"]["dnsmasq"], + "pause": k8sComponentVersions["1.9"]["pause"], + DefaultTillerAddonName: k8sComponentVersions["1.9"]["tiller"], + DefaultReschedulerAddonName: k8sComponentVersions["1.9"]["rescheduler"], + DefaultACIConnectorAddonName: k8sComponentVersions["1.9"]["aci-connector"], + ContainerMonitoringAddonName: k8sComponentVersions["1.9"][ContainerMonitoringAddonName], + AzureCNINetworkMonitoringAddonName: k8sComponentVersions["1.9"][AzureCNINetworkMonitoringAddonName], + "nodestatusfreq": k8sComponentVersions["1.9"]["nodestatusfreq"], + "nodegraceperiod": k8sComponentVersions["1.9"]["nodegraceperiod"], + "podeviction": k8sComponentVersions["1.9"]["podeviction"], + "routeperiod": k8sComponentVersions["1.9"]["routeperiod"], + "backoffretries": k8sComponentVersions["1.9"]["backoffretries"], + "backoffjitter": k8sComponentVersions["1.9"]["backoffjitter"], + "backoffduration": k8sComponentVersions["1.9"]["backoffduration"], + "backoffexponent": k8sComponentVersions["1.9"]["backoffexponent"], + "ratelimitqps": k8sComponentVersions["1.9"]["ratelimitqps"], + "ratelimitbucket": k8sComponentVersions["1.9"]["ratelimitbucket"], + "gchighthreshold": k8sComponentVersions["1.9"]["gchighthreshold"], + "gclowthreshold": k8sComponentVersions["1.9"]["gclowthreshold"], + DefaultClusterAutoscalerAddonName: k8sComponentVersions["1.9"]["cluster-autoscaler"], } case "1.8": ret = map[string]string{ @@ -347,26 +356,27 @@ func getK8sVersionComponents(version string, overrides map[string]string) map[st "addonresizer": k8sComponentVersions["1.8"]["addon-resizer"], "heapster": k8sComponentVersions["1.8"]["heapster"], DefaultMetricsServerAddonName: k8sComponentVersions["1.8"]["metrics-server"], - "dns": k8sComponentVersions["1.8"]["kube-dns"], - "addonmanager": k8sComponentVersions["1.8"]["addon-manager"], - "dnsmasq": k8sComponentVersions["1.8"]["dnsmasq"], - "pause": k8sComponentVersions["1.8"]["pause"], - DefaultTillerAddonName: k8sComponentVersions["1.8"]["tiller"], - DefaultReschedulerAddonName: k8sComponentVersions["1.8"]["rescheduler"], - DefaultACIConnectorAddonName: k8sComponentVersions["1.8"]["aci-connector"], - ContainerMonitoringAddonName: k8sComponentVersions["1.11"][ContainerMonitoringAddonName], - "nodestatusfreq": k8sComponentVersions["1.8"]["nodestatusfreq"], - "nodegraceperiod": k8sComponentVersions["1.8"]["nodegraceperiod"], - "podeviction": k8sComponentVersions["1.8"]["podeviction"], - "routeperiod": k8sComponentVersions["1.8"]["routeperiod"], - "backoffretries": k8sComponentVersions["1.8"]["backoffretries"], - "backoffjitter": k8sComponentVersions["1.8"]["backoffjitter"], - "backoffduration": k8sComponentVersions["1.8"]["backoffduration"], - "backoffexponent": k8sComponentVersions["1.8"]["backoffexponent"], - "ratelimitqps": k8sComponentVersions["1.8"]["ratelimitqps"], - "ratelimitbucket": k8sComponentVersions["1.8"]["ratelimitbucket"], - "gchighthreshold": k8sComponentVersions["1.8"]["gchighthreshold"], - "gclowthreshold": k8sComponentVersions["1.8"]["gclowthreshold"], + "dns": k8sComponentVersions["1.8"]["kube-dns"], + "addonmanager": k8sComponentVersions["1.8"]["addon-manager"], + "dnsmasq": k8sComponentVersions["1.8"]["dnsmasq"], + "pause": k8sComponentVersions["1.8"]["pause"], + DefaultTillerAddonName: k8sComponentVersions["1.8"]["tiller"], + DefaultReschedulerAddonName: k8sComponentVersions["1.8"]["rescheduler"], + DefaultACIConnectorAddonName: k8sComponentVersions["1.8"]["aci-connector"], + ContainerMonitoringAddonName: k8sComponentVersions["1.8"][ContainerMonitoringAddonName], + AzureCNINetworkMonitoringAddonName: k8sComponentVersions["1.8"][AzureCNINetworkMonitoringAddonName], + "nodestatusfreq": k8sComponentVersions["1.8"]["nodestatusfreq"], + "nodegraceperiod": k8sComponentVersions["1.8"]["nodegraceperiod"], + "podeviction": k8sComponentVersions["1.8"]["podeviction"], + "routeperiod": k8sComponentVersions["1.8"]["routeperiod"], + "backoffretries": k8sComponentVersions["1.8"]["backoffretries"], + "backoffjitter": k8sComponentVersions["1.8"]["backoffjitter"], + "backoffduration": k8sComponentVersions["1.8"]["backoffduration"], + "backoffexponent": k8sComponentVersions["1.8"]["backoffexponent"], + "ratelimitqps": k8sComponentVersions["1.8"]["ratelimitqps"], + "ratelimitbucket": k8sComponentVersions["1.8"]["ratelimitbucket"], + "gchighthreshold": k8sComponentVersions["1.8"]["gchighthreshold"], + "gclowthreshold": k8sComponentVersions["1.8"]["gclowthreshold"], } case "1.7": ret = map[string]string{ @@ -377,26 +387,27 @@ func getK8sVersionComponents(version string, overrides map[string]string) map[st "addonresizer": k8sComponentVersions["1.7"]["addon-resizer"], "heapster": k8sComponentVersions["1.7"]["heapster"], DefaultMetricsServerAddonName: k8sComponentVersions["1.7"]["metrics-server"], - "dns": k8sComponentVersions["1.7"]["kube-dns"], - "addonmanager": k8sComponentVersions["1.7"]["addon-manager"], - "dnsmasq": k8sComponentVersions["1.7"]["dnsmasq"], - "pause": k8sComponentVersions["1.7"]["pause"], - DefaultTillerAddonName: k8sComponentVersions["1.7"]["tiller"], - DefaultReschedulerAddonName: k8sComponentVersions["1.7"]["rescheduler"], - DefaultACIConnectorAddonName: k8sComponentVersions["1.7"]["aci-connector"], - ContainerMonitoringAddonName: k8sComponentVersions["1.11"][ContainerMonitoringAddonName], - "nodestatusfreq": k8sComponentVersions["1.7"]["nodestatusfreq"], - "nodegraceperiod": k8sComponentVersions["1.7"]["nodegraceperiod"], - "podeviction": k8sComponentVersions["1.7"]["podeviction"], - "routeperiod": k8sComponentVersions["1.7"]["routeperiod"], - "backoffretries": k8sComponentVersions["1.7"]["backoffretries"], - "backoffjitter": k8sComponentVersions["1.7"]["backoffjitter"], - "backoffduration": k8sComponentVersions["1.7"]["backoffduration"], - "backoffexponent": k8sComponentVersions["1.7"]["backoffexponent"], - "ratelimitqps": k8sComponentVersions["1.7"]["ratelimitqps"], - "ratelimitbucket": k8sComponentVersions["1.7"]["ratelimitbucket"], - "gchighthreshold": k8sComponentVersions["1.7"]["gchighthreshold"], - "gclowthreshold": k8sComponentVersions["1.7"]["gclowthreshold"], + "dns": k8sComponentVersions["1.7"]["kube-dns"], + "addonmanager": k8sComponentVersions["1.7"]["addon-manager"], + "dnsmasq": k8sComponentVersions["1.7"]["dnsmasq"], + "pause": k8sComponentVersions["1.7"]["pause"], + DefaultTillerAddonName: k8sComponentVersions["1.7"]["tiller"], + DefaultReschedulerAddonName: k8sComponentVersions["1.7"]["rescheduler"], + DefaultACIConnectorAddonName: k8sComponentVersions["1.7"]["aci-connector"], + ContainerMonitoringAddonName: k8sComponentVersions["1.7"][ContainerMonitoringAddonName], + AzureCNINetworkMonitoringAddonName: k8sComponentVersions["1.7"][AzureCNINetworkMonitoringAddonName], + "nodestatusfreq": k8sComponentVersions["1.7"]["nodestatusfreq"], + "nodegraceperiod": k8sComponentVersions["1.7"]["nodegraceperiod"], + "podeviction": k8sComponentVersions["1.7"]["podeviction"], + "routeperiod": k8sComponentVersions["1.7"]["routeperiod"], + "backoffretries": k8sComponentVersions["1.7"]["backoffretries"], + "backoffjitter": k8sComponentVersions["1.7"]["backoffjitter"], + "backoffduration": k8sComponentVersions["1.7"]["backoffduration"], + "backoffexponent": k8sComponentVersions["1.7"]["backoffexponent"], + "ratelimitqps": k8sComponentVersions["1.7"]["ratelimitqps"], + "ratelimitbucket": k8sComponentVersions["1.7"]["ratelimitbucket"], + "gchighthreshold": k8sComponentVersions["1.7"]["gchighthreshold"], + "gclowthreshold": k8sComponentVersions["1.7"]["gclowthreshold"], } case "1.6": ret = map[string]string{ @@ -407,25 +418,26 @@ func getK8sVersionComponents(version string, overrides map[string]string) map[st "addonresizer": k8sComponentVersions["1.6"]["addon-resizer"], "heapster": k8sComponentVersions["1.6"]["heapster"], DefaultMetricsServerAddonName: k8sComponentVersions["1.6"]["metrics-server"], - "dns": k8sComponentVersions["1.6"]["kube-dns"], - "addonmanager": k8sComponentVersions["1.6"]["addon-manager"], - "dnsmasq": k8sComponentVersions["1.6"]["dnsmasq"], - "pause": k8sComponentVersions["1.6"]["pause"], - DefaultTillerAddonName: k8sComponentVersions["1.6"]["tiller"], - DefaultReschedulerAddonName: k8sComponentVersions["1.6"]["rescheduler"], - DefaultACIConnectorAddonName: k8sComponentVersions["1.6"]["aci-connector"], - "nodestatusfreq": k8sComponentVersions["1.6"]["nodestatusfreq"], - "nodegraceperiod": k8sComponentVersions["1.6"]["nodegraceperiod"], - "podeviction": k8sComponentVersions["1.6"]["podeviction"], - "routeperiod": k8sComponentVersions["1.6"]["routeperiod"], - "backoffretries": k8sComponentVersions["1.6"]["backoffretries"], - "backoffjitter": k8sComponentVersions["1.6"]["backoffjitter"], - "backoffduration": k8sComponentVersions["1.6"]["backoffduration"], - "backoffexponent": k8sComponentVersions["1.6"]["backoffexponent"], - "ratelimitqps": k8sComponentVersions["1.6"]["ratelimitqps"], - "ratelimitbucket": k8sComponentVersions["1.6"]["ratelimitbucket"], - "gchighthreshold": k8sComponentVersions["1.6"]["gchighthreshold"], - "gclowthreshold": k8sComponentVersions["1.6"]["gclowthreshold"], + "dns": k8sComponentVersions["1.6"]["kube-dns"], + "addonmanager": k8sComponentVersions["1.6"]["addon-manager"], + "dnsmasq": k8sComponentVersions["1.6"]["dnsmasq"], + "pause": k8sComponentVersions["1.6"]["pause"], + DefaultTillerAddonName: k8sComponentVersions["1.6"]["tiller"], + DefaultReschedulerAddonName: k8sComponentVersions["1.6"]["rescheduler"], + DefaultACIConnectorAddonName: k8sComponentVersions["1.6"]["aci-connector"], + AzureCNINetworkMonitoringAddonName: k8sComponentVersions["1.6"][AzureCNINetworkMonitoringAddonName], + "nodestatusfreq": k8sComponentVersions["1.6"]["nodestatusfreq"], + "nodegraceperiod": k8sComponentVersions["1.6"]["nodegraceperiod"], + "podeviction": k8sComponentVersions["1.6"]["podeviction"], + "routeperiod": k8sComponentVersions["1.6"]["routeperiod"], + "backoffretries": k8sComponentVersions["1.6"]["backoffretries"], + "backoffjitter": k8sComponentVersions["1.6"]["backoffjitter"], + "backoffduration": k8sComponentVersions["1.6"]["backoffduration"], + "backoffexponent": k8sComponentVersions["1.6"]["backoffexponent"], + "ratelimitqps": k8sComponentVersions["1.6"]["ratelimitqps"], + "ratelimitbucket": k8sComponentVersions["1.6"]["ratelimitbucket"], + "gchighthreshold": k8sComponentVersions["1.6"]["gchighthreshold"], + "gclowthreshold": k8sComponentVersions["1.6"]["gclowthreshold"], } default: diff --git a/pkg/acsengine/k8s_versions_test.go b/pkg/acsengine/k8s_versions_test.go index 1272b3298c..a9038d0da7 100644 --- a/pkg/acsengine/k8s_versions_test.go +++ b/pkg/acsengine/k8s_versions_test.go @@ -20,26 +20,27 @@ func TestGetK8sVersionComponents(t *testing.T) { "addonresizer": k8sComponentVersions["1.11"]["addon-resizer"], "heapster": k8sComponentVersions["1.11"]["heapster"], DefaultMetricsServerAddonName: k8sComponentVersions["1.11"]["metrics-server"], - "dns": k8sComponentVersions["1.11"]["kube-dns"], - "addonmanager": k8sComponentVersions["1.11"]["addon-manager"], - "dnsmasq": k8sComponentVersions["1.11"]["dnsmasq"], - "pause": k8sComponentVersions["1.11"]["pause"], - DefaultTillerAddonName: k8sComponentVersions["1.11"]["tiller"], - DefaultReschedulerAddonName: k8sComponentVersions["1.11"]["rescheduler"], - DefaultACIConnectorAddonName: k8sComponentVersions["1.11"]["aci-connector"], - ContainerMonitoringAddonName: k8sComponentVersions["1.11"][ContainerMonitoringAddonName], - "nodestatusfreq": k8sComponentVersions["1.11"]["nodestatusfreq"], - "nodegraceperiod": k8sComponentVersions["1.11"]["nodegraceperiod"], - "podeviction": k8sComponentVersions["1.11"]["podeviction"], - "routeperiod": k8sComponentVersions["1.11"]["routeperiod"], - "backoffretries": k8sComponentVersions["1.11"]["backoffretries"], - "backoffjitter": k8sComponentVersions["1.11"]["backoffjitter"], - "backoffduration": k8sComponentVersions["1.11"]["backoffduration"], - "backoffexponent": k8sComponentVersions["1.11"]["backoffexponent"], - "ratelimitqps": k8sComponentVersions["1.11"]["ratelimitqps"], - "ratelimitbucket": k8sComponentVersions["1.11"]["ratelimitbucket"], - "gchighthreshold": k8sComponentVersions["1.11"]["gchighthreshold"], - "gclowthreshold": k8sComponentVersions["1.11"]["gclowthreshold"], + "dns": k8sComponentVersions["1.11"]["kube-dns"], + "addonmanager": k8sComponentVersions["1.11"]["addon-manager"], + "dnsmasq": k8sComponentVersions["1.11"]["dnsmasq"], + "pause": k8sComponentVersions["1.11"]["pause"], + DefaultTillerAddonName: k8sComponentVersions["1.11"]["tiller"], + DefaultReschedulerAddonName: k8sComponentVersions["1.11"]["rescheduler"], + DefaultACIConnectorAddonName: k8sComponentVersions["1.11"]["aci-connector"], + ContainerMonitoringAddonName: k8sComponentVersions["1.11"][ContainerMonitoringAddonName], + AzureCNINetworkMonitoringAddonName: k8sComponentVersions["1.11"][AzureCNINetworkMonitoringAddonName], + "nodestatusfreq": k8sComponentVersions["1.11"]["nodestatusfreq"], + "nodegraceperiod": k8sComponentVersions["1.11"]["nodegraceperiod"], + "podeviction": k8sComponentVersions["1.11"]["podeviction"], + "routeperiod": k8sComponentVersions["1.11"]["routeperiod"], + "backoffretries": k8sComponentVersions["1.11"]["backoffretries"], + "backoffjitter": k8sComponentVersions["1.11"]["backoffjitter"], + "backoffduration": k8sComponentVersions["1.11"]["backoffduration"], + "backoffexponent": k8sComponentVersions["1.11"]["backoffexponent"], + "ratelimitqps": k8sComponentVersions["1.11"]["ratelimitqps"], + "ratelimitbucket": k8sComponentVersions["1.11"]["ratelimitbucket"], + "gchighthreshold": k8sComponentVersions["1.11"]["gchighthreshold"], + "gclowthreshold": k8sComponentVersions["1.11"]["gclowthreshold"], } for k, v := range oneDotElevenDotZero { @@ -62,27 +63,28 @@ func TestGetK8sVersionComponents(t *testing.T) { "addonresizer": k8sComponentVersions["1.9"]["addon-resizer"], "heapster": k8sComponentVersions["1.9"]["heapster"], DefaultMetricsServerAddonName: k8sComponentVersions["1.9"]["metrics-server"], - "dns": k8sComponentVersions["1.9"]["kube-dns"], - "addonmanager": k8sComponentVersions["1.9"]["addon-manager"], - "dnsmasq": k8sComponentVersions["1.9"]["dnsmasq"], - "pause": k8sComponentVersions["1.9"]["pause"], - DefaultTillerAddonName: k8sComponentVersions["1.9"]["tiller"], - DefaultReschedulerAddonName: k8sComponentVersions["1.9"]["rescheduler"], - DefaultACIConnectorAddonName: k8sComponentVersions["1.9"]["aci-connector"], - ContainerMonitoringAddonName: k8sComponentVersions["1.11"][ContainerMonitoringAddonName], - DefaultClusterAutoscalerAddonName: k8sComponentVersions["1.9"]["cluster-autoscaler"], - "nodestatusfreq": k8sComponentVersions["1.9"]["nodestatusfreq"], - "nodegraceperiod": k8sComponentVersions["1.9"]["nodegraceperiod"], - "podeviction": k8sComponentVersions["1.9"]["podeviction"], - "routeperiod": k8sComponentVersions["1.9"]["routeperiod"], - "backoffretries": k8sComponentVersions["1.9"]["backoffretries"], - "backoffjitter": k8sComponentVersions["1.9"]["backoffjitter"], - "backoffduration": k8sComponentVersions["1.9"]["backoffduration"], - "backoffexponent": k8sComponentVersions["1.9"]["backoffexponent"], - "ratelimitqps": k8sComponentVersions["1.9"]["ratelimitqps"], - "ratelimitbucket": k8sComponentVersions["1.9"]["ratelimitbucket"], - "gchighthreshold": k8sComponentVersions["1.9"]["gchighthreshold"], - "gclowthreshold": k8sComponentVersions["1.9"]["gclowthreshold"], + "dns": k8sComponentVersions["1.9"]["kube-dns"], + "addonmanager": k8sComponentVersions["1.9"]["addon-manager"], + "dnsmasq": k8sComponentVersions["1.9"]["dnsmasq"], + "pause": k8sComponentVersions["1.9"]["pause"], + DefaultTillerAddonName: k8sComponentVersions["1.9"]["tiller"], + DefaultReschedulerAddonName: k8sComponentVersions["1.9"]["rescheduler"], + DefaultACIConnectorAddonName: k8sComponentVersions["1.9"]["aci-connector"], + ContainerMonitoringAddonName: k8sComponentVersions["1.11"][ContainerMonitoringAddonName], + AzureCNINetworkMonitoringAddonName: k8sComponentVersions["1.9"][AzureCNINetworkMonitoringAddonName], + DefaultClusterAutoscalerAddonName: k8sComponentVersions["1.9"]["cluster-autoscaler"], + "nodestatusfreq": k8sComponentVersions["1.9"]["nodestatusfreq"], + "nodegraceperiod": k8sComponentVersions["1.9"]["nodegraceperiod"], + "podeviction": k8sComponentVersions["1.9"]["podeviction"], + "routeperiod": k8sComponentVersions["1.9"]["routeperiod"], + "backoffretries": k8sComponentVersions["1.9"]["backoffretries"], + "backoffjitter": k8sComponentVersions["1.9"]["backoffjitter"], + "backoffduration": k8sComponentVersions["1.9"]["backoffduration"], + "backoffexponent": k8sComponentVersions["1.9"]["backoffexponent"], + "ratelimitqps": k8sComponentVersions["1.9"]["ratelimitqps"], + "ratelimitbucket": k8sComponentVersions["1.9"]["ratelimitbucket"], + "gchighthreshold": k8sComponentVersions["1.9"]["gchighthreshold"], + "gclowthreshold": k8sComponentVersions["1.9"]["gclowthreshold"], } for k, v := range oneDotNineDotThree { @@ -105,26 +107,27 @@ func TestGetK8sVersionComponents(t *testing.T) { "addonresizer": k8sComponentVersions["1.8"]["addon-resizer"], "heapster": k8sComponentVersions["1.8"]["heapster"], DefaultMetricsServerAddonName: k8sComponentVersions["1.8"]["metrics-server"], - "dns": k8sComponentVersions["1.8"]["kube-dns"], - "addonmanager": k8sComponentVersions["1.8"]["addon-manager"], - "dnsmasq": k8sComponentVersions["1.8"]["dnsmasq"], - "pause": k8sComponentVersions["1.8"]["pause"], - DefaultTillerAddonName: k8sComponentVersions["1.8"]["tiller"], - DefaultReschedulerAddonName: k8sComponentVersions["1.8"]["rescheduler"], - DefaultACIConnectorAddonName: k8sComponentVersions["1.8"]["aci-connector"], - ContainerMonitoringAddonName: k8sComponentVersions["1.11"][ContainerMonitoringAddonName], - "nodestatusfreq": k8sComponentVersions["1.8"]["nodestatusfreq"], - "nodegraceperiod": k8sComponentVersions["1.8"]["nodegraceperiod"], - "podeviction": k8sComponentVersions["1.8"]["podeviction"], - "routeperiod": k8sComponentVersions["1.8"]["routeperiod"], - "backoffretries": k8sComponentVersions["1.8"]["backoffretries"], - "backoffjitter": k8sComponentVersions["1.8"]["backoffjitter"], - "backoffduration": k8sComponentVersions["1.8"]["backoffduration"], - "backoffexponent": k8sComponentVersions["1.8"]["backoffexponent"], - "ratelimitqps": k8sComponentVersions["1.8"]["ratelimitqps"], - "ratelimitbucket": k8sComponentVersions["1.8"]["ratelimitbucket"], - "gchighthreshold": k8sComponentVersions["1.8"]["gchighthreshold"], - "gclowthreshold": k8sComponentVersions["1.8"]["gclowthreshold"], + "dns": k8sComponentVersions["1.8"]["kube-dns"], + "addonmanager": k8sComponentVersions["1.8"]["addon-manager"], + "dnsmasq": k8sComponentVersions["1.8"]["dnsmasq"], + "pause": k8sComponentVersions["1.8"]["pause"], + DefaultTillerAddonName: k8sComponentVersions["1.8"]["tiller"], + DefaultReschedulerAddonName: k8sComponentVersions["1.8"]["rescheduler"], + DefaultACIConnectorAddonName: k8sComponentVersions["1.8"]["aci-connector"], + ContainerMonitoringAddonName: k8sComponentVersions["1.11"][ContainerMonitoringAddonName], + AzureCNINetworkMonitoringAddonName: k8sComponentVersions["1.8"][AzureCNINetworkMonitoringAddonName], + "nodestatusfreq": k8sComponentVersions["1.8"]["nodestatusfreq"], + "nodegraceperiod": k8sComponentVersions["1.8"]["nodegraceperiod"], + "podeviction": k8sComponentVersions["1.8"]["podeviction"], + "routeperiod": k8sComponentVersions["1.8"]["routeperiod"], + "backoffretries": k8sComponentVersions["1.8"]["backoffretries"], + "backoffjitter": k8sComponentVersions["1.8"]["backoffjitter"], + "backoffduration": k8sComponentVersions["1.8"]["backoffduration"], + "backoffexponent": k8sComponentVersions["1.8"]["backoffexponent"], + "ratelimitqps": k8sComponentVersions["1.8"]["ratelimitqps"], + "ratelimitbucket": k8sComponentVersions["1.8"]["ratelimitbucket"], + "gchighthreshold": k8sComponentVersions["1.8"]["gchighthreshold"], + "gclowthreshold": k8sComponentVersions["1.8"]["gclowthreshold"], } for k, v := range oneDotEightDotEight { if expected[k] != v { @@ -144,26 +147,27 @@ func TestGetK8sVersionComponents(t *testing.T) { "addonresizer": k8sComponentVersions["1.7"]["addon-resizer"], "heapster": k8sComponentVersions["1.7"]["heapster"], DefaultMetricsServerAddonName: k8sComponentVersions["1.7"]["metrics-server"], - "dns": k8sComponentVersions["1.7"]["kube-dns"], - "addonmanager": k8sComponentVersions["1.7"]["addon-manager"], - "dnsmasq": k8sComponentVersions["1.7"]["dnsmasq"], - "pause": k8sComponentVersions["1.7"]["pause"], - DefaultTillerAddonName: k8sComponentVersions["1.7"]["tiller"], - DefaultReschedulerAddonName: k8sComponentVersions["1.7"]["rescheduler"], - DefaultACIConnectorAddonName: k8sComponentVersions["1.7"]["aci-connector"], - ContainerMonitoringAddonName: k8sComponentVersions["1.11"][ContainerMonitoringAddonName], - "nodestatusfreq": k8sComponentVersions["1.7"]["nodestatusfreq"], - "nodegraceperiod": k8sComponentVersions["1.7"]["nodegraceperiod"], - "podeviction": k8sComponentVersions["1.7"]["podeviction"], - "routeperiod": k8sComponentVersions["1.7"]["routeperiod"], - "backoffretries": k8sComponentVersions["1.7"]["backoffretries"], - "backoffjitter": k8sComponentVersions["1.7"]["backoffjitter"], - "backoffduration": k8sComponentVersions["1.7"]["backoffduration"], - "backoffexponent": k8sComponentVersions["1.7"]["backoffexponent"], - "ratelimitqps": k8sComponentVersions["1.7"]["ratelimitqps"], - "ratelimitbucket": k8sComponentVersions["1.7"]["ratelimitbucket"], - "gchighthreshold": k8sComponentVersions["1.7"]["gchighthreshold"], - "gclowthreshold": k8sComponentVersions["1.7"]["gclowthreshold"], + "dns": k8sComponentVersions["1.7"]["kube-dns"], + "addonmanager": k8sComponentVersions["1.7"]["addon-manager"], + "dnsmasq": k8sComponentVersions["1.7"]["dnsmasq"], + "pause": k8sComponentVersions["1.7"]["pause"], + DefaultTillerAddonName: k8sComponentVersions["1.7"]["tiller"], + DefaultReschedulerAddonName: k8sComponentVersions["1.7"]["rescheduler"], + DefaultACIConnectorAddonName: k8sComponentVersions["1.7"]["aci-connector"], + ContainerMonitoringAddonName: k8sComponentVersions["1.11"][ContainerMonitoringAddonName], + AzureCNINetworkMonitoringAddonName: k8sComponentVersions["1.7"][AzureCNINetworkMonitoringAddonName], + "nodestatusfreq": k8sComponentVersions["1.7"]["nodestatusfreq"], + "nodegraceperiod": k8sComponentVersions["1.7"]["nodegraceperiod"], + "podeviction": k8sComponentVersions["1.7"]["podeviction"], + "routeperiod": k8sComponentVersions["1.7"]["routeperiod"], + "backoffretries": k8sComponentVersions["1.7"]["backoffretries"], + "backoffjitter": k8sComponentVersions["1.7"]["backoffjitter"], + "backoffduration": k8sComponentVersions["1.7"]["backoffduration"], + "backoffexponent": k8sComponentVersions["1.7"]["backoffexponent"], + "ratelimitqps": k8sComponentVersions["1.7"]["ratelimitqps"], + "ratelimitbucket": k8sComponentVersions["1.7"]["ratelimitbucket"], + "gchighthreshold": k8sComponentVersions["1.7"]["gchighthreshold"], + "gclowthreshold": k8sComponentVersions["1.7"]["gclowthreshold"], } for k, v := range oneDotSevenDotZero { if expected[k] != v { @@ -185,27 +189,28 @@ func TestGetK8sVersionComponents(t *testing.T) { "addonresizer": k8sComponentVersions["1.9"]["addon-resizer"], "heapster": k8sComponentVersions["1.9"]["heapster"], DefaultMetricsServerAddonName: k8sComponentVersions["1.9"]["metrics-server"], - "dns": k8sComponentVersions["1.9"]["kube-dns"], - "addonmanager": k8sComponentVersions["1.9"]["addon-manager"], - "dnsmasq": k8sComponentVersions["1.9"]["dnsmasq"], - "pause": k8sComponentVersions["1.9"]["pause"], - DefaultTillerAddonName: k8sComponentVersions["1.9"]["tiller"], - DefaultReschedulerAddonName: k8sComponentVersions["1.9"]["rescheduler"], - DefaultACIConnectorAddonName: k8sComponentVersions["1.9"]["aci-connector"], - ContainerMonitoringAddonName: k8sComponentVersions["1.11"][ContainerMonitoringAddonName], - DefaultClusterAutoscalerAddonName: k8sComponentVersions["1.9"]["cluster-autoscaler"], - "nodestatusfreq": k8sComponentVersions["1.9"]["nodestatusfreq"], - "nodegraceperiod": k8sComponentVersions["1.9"]["nodegraceperiod"], - "podeviction": k8sComponentVersions["1.9"]["podeviction"], - "routeperiod": k8sComponentVersions["1.9"]["routeperiod"], - "backoffretries": k8sComponentVersions["1.9"]["backoffretries"], - "backoffjitter": k8sComponentVersions["1.9"]["backoffjitter"], - "backoffduration": k8sComponentVersions["1.9"]["backoffduration"], - "backoffexponent": k8sComponentVersions["1.9"]["backoffexponent"], - "ratelimitqps": k8sComponentVersions["1.9"]["ratelimitqps"], - "ratelimitbucket": k8sComponentVersions["1.9"]["ratelimitbucket"], - "gchighthreshold": k8sComponentVersions["1.9"]["gchighthreshold"], - "gclowthreshold": k8sComponentVersions["1.9"]["gclowthreshold"], + "dns": k8sComponentVersions["1.9"]["kube-dns"], + "addonmanager": k8sComponentVersions["1.9"]["addon-manager"], + "dnsmasq": k8sComponentVersions["1.9"]["dnsmasq"], + "pause": k8sComponentVersions["1.9"]["pause"], + DefaultTillerAddonName: k8sComponentVersions["1.9"]["tiller"], + DefaultReschedulerAddonName: k8sComponentVersions["1.9"]["rescheduler"], + DefaultACIConnectorAddonName: k8sComponentVersions["1.9"]["aci-connector"], + ContainerMonitoringAddonName: k8sComponentVersions["1.11"][ContainerMonitoringAddonName], + AzureCNINetworkMonitoringAddonName: k8sComponentVersions["1.9"][AzureCNINetworkMonitoringAddonName], + DefaultClusterAutoscalerAddonName: k8sComponentVersions["1.9"]["cluster-autoscaler"], + "nodestatusfreq": k8sComponentVersions["1.9"]["nodestatusfreq"], + "nodegraceperiod": k8sComponentVersions["1.9"]["nodegraceperiod"], + "podeviction": k8sComponentVersions["1.9"]["podeviction"], + "routeperiod": k8sComponentVersions["1.9"]["routeperiod"], + "backoffretries": k8sComponentVersions["1.9"]["backoffretries"], + "backoffjitter": k8sComponentVersions["1.9"]["backoffjitter"], + "backoffduration": k8sComponentVersions["1.9"]["backoffduration"], + "backoffexponent": k8sComponentVersions["1.9"]["backoffexponent"], + "ratelimitqps": k8sComponentVersions["1.9"]["ratelimitqps"], + "ratelimitbucket": k8sComponentVersions["1.9"]["ratelimitbucket"], + "gchighthreshold": k8sComponentVersions["1.9"]["gchighthreshold"], + "gclowthreshold": k8sComponentVersions["1.9"]["gclowthreshold"], } for k, v := range override { if expected[k] != v { diff --git a/pkg/acsengine/params.go b/pkg/acsengine/params.go index 5222ddbc29..5d1e7ca67b 100644 --- a/pkg/acsengine/params.go +++ b/pkg/acsengine/params.go @@ -122,6 +122,7 @@ func getParameters(cs *api.ContainerService, isClassicMode bool, generatorCode s dcosBootstrapURL = cloudSpecConfig.DCOSSpecConfig.DCOS110BootstrapDownloadURL default: dcosBootstrapURL = getDCOSDefaultBootstrapInstallerURL(properties.OrchestratorProfile) + dcosWindowsBootstrapURL = getDCOSDefaultWindowsBootstrapInstallerURL(properties.OrchestratorProfile) } } diff --git a/pkg/acsengine/params_k8s.go b/pkg/acsengine/params_k8s.go index af89a7cd68..1ca849050d 100644 --- a/pkg/acsengine/params_k8s.go +++ b/pkg/acsengine/params_k8s.go @@ -10,18 +10,221 @@ import ( func assignKubernetesParameters(properties *api.Properties, parametersMap paramsMap, cloudSpecConfig AzureEnvironmentSpecConfig, generatorCode string) { + addValue(parametersMap, "generatorCode", generatorCode) if properties.OrchestratorProfile.IsKubernetes() || properties.OrchestratorProfile.IsOpenShift() { k8sVersion := properties.OrchestratorProfile.OrchestratorVersion - kubernetesHyperkubeSpec := properties.OrchestratorProfile.KubernetesConfig.KubernetesImageBase + KubeConfigs[k8sVersion]["hyperkube"] - if properties.OrchestratorProfile.KubernetesConfig.CustomHyperkubeImage != "" { - kubernetesHyperkubeSpec = properties.OrchestratorProfile.KubernetesConfig.CustomHyperkubeImage + dockerEngineVersion := KubeConfigs[k8sVersion]["dockerEngineVersion"] + + if properties.OrchestratorProfile.KubernetesConfig != nil { + if helpers.IsTrueBoolPointer(properties.OrchestratorProfile.KubernetesConfig.UseCloudControllerManager) { + kubernetesCcmSpec := properties.OrchestratorProfile.KubernetesConfig.KubernetesImageBase + KubeConfigs[k8sVersion]["ccm"] + if properties.OrchestratorProfile.KubernetesConfig.CustomCcmImage != "" { + kubernetesCcmSpec = properties.OrchestratorProfile.KubernetesConfig.CustomCcmImage + } + + addValue(parametersMap, "kubernetesCcmImageSpec", kubernetesCcmSpec) + } + + kubernetesHyperkubeSpec := properties.OrchestratorProfile.KubernetesConfig.KubernetesImageBase + KubeConfigs[k8sVersion]["hyperkube"] + if properties.OrchestratorProfile.KubernetesConfig.CustomHyperkubeImage != "" { + kubernetesHyperkubeSpec = properties.OrchestratorProfile.KubernetesConfig.CustomHyperkubeImage + } + + addValue(parametersMap, "kubeDNSServiceIP", properties.OrchestratorProfile.KubernetesConfig.DNSServiceIP) + addValue(parametersMap, "kubeServiceCidr", properties.OrchestratorProfile.KubernetesConfig.ServiceCIDR) + addValue(parametersMap, "kubernetesHyperkubeSpec", kubernetesHyperkubeSpec) + addValue(parametersMap, "kubernetesAddonManagerSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+KubeConfigs[k8sVersion]["addonmanager"]) + addValue(parametersMap, "kubernetesAddonResizerSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+KubeConfigs[k8sVersion]["addonresizer"]) + addValue(parametersMap, "kubernetesDNSMasqSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+KubeConfigs[k8sVersion]["dnsmasq"]) + addValue(parametersMap, "kubernetesExecHealthzSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+KubeConfigs[k8sVersion]["exechealthz"]) + addValue(parametersMap, "kubernetesHeapsterSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+KubeConfigs[k8sVersion]["heapster"]) + tillerAddon := getAddonByName(properties.OrchestratorProfile.KubernetesConfig.Addons, DefaultTillerAddonName) + c := getAddonContainersIndexByName(tillerAddon.Containers, DefaultTillerAddonName) + if c > -1 { + addValue(parametersMap, "kubernetesTillerCPURequests", tillerAddon.Containers[c].CPURequests) + addValue(parametersMap, "kubernetesTillerCPULimit", tillerAddon.Containers[c].CPULimits) + addValue(parametersMap, "kubernetesTillerMemoryRequests", tillerAddon.Containers[c].MemoryRequests) + addValue(parametersMap, "kubernetesTillerMemoryLimit", tillerAddon.Containers[c].MemoryLimits) + addValue(parametersMap, "kubernetesTillerMaxHistory", tillerAddon.Config["max-history"]) + if tillerAddon.Containers[c].Image != "" { + addValue(parametersMap, "kubernetesTillerSpec", tillerAddon.Containers[c].Image) + } else { + addValue(parametersMap, "kubernetesTillerSpec", cloudSpecConfig.KubernetesSpecConfig.TillerImageBase+KubeConfigs[k8sVersion][DefaultTillerAddonName]) + } + } + aciConnectorAddon := getAddonByName(properties.OrchestratorProfile.KubernetesConfig.Addons, DefaultACIConnectorAddonName) + c = getAddonContainersIndexByName(aciConnectorAddon.Containers, DefaultACIConnectorAddonName) + if c > -1 { + addValue(parametersMap, "kubernetesACIConnectorEnabled", helpers.IsTrueBoolPointer(aciConnectorAddon.Enabled)) + addValue(parametersMap, "kubernetesACIConnectorNodeName", aciConnectorAddon.Config["nodeName"]) + addValue(parametersMap, "kubernetesACIConnectorOS", aciConnectorAddon.Config["os"]) + addValue(parametersMap, "kubernetesACIConnectorTaint", aciConnectorAddon.Config["taint"]) + addValue(parametersMap, "kubernetesACIConnectorRegion", aciConnectorAddon.Config["region"]) + addValue(parametersMap, "kubernetesACIConnectorCPURequests", aciConnectorAddon.Containers[c].CPURequests) + addValue(parametersMap, "kubernetesACIConnectorCPULimit", aciConnectorAddon.Containers[c].CPULimits) + addValue(parametersMap, "kubernetesACIConnectorMemoryRequests", aciConnectorAddon.Containers[c].MemoryRequests) + addValue(parametersMap, "kubernetesACIConnectorMemoryLimit", aciConnectorAddon.Containers[c].MemoryLimits) + if aciConnectorAddon.Containers[c].Image != "" { + addValue(parametersMap, "kubernetesACIConnectorSpec", aciConnectorAddon.Containers[c].Image) + } else { + addValue(parametersMap, "kubernetesACIConnectorSpec", cloudSpecConfig.KubernetesSpecConfig.ACIConnectorImageBase+KubeConfigs[k8sVersion][DefaultACIConnectorAddonName]) + } + } + clusterAutoscalerAddon := getAddonByName(properties.OrchestratorProfile.KubernetesConfig.Addons, DefaultClusterAutoscalerAddonName) + c = getAddonContainersIndexByName(clusterAutoscalerAddon.Containers, DefaultClusterAutoscalerAddonName) + if c > -1 { + addValue(parametersMap, "kubernetesClusterAutoscalerCPURequests", clusterAutoscalerAddon.Containers[c].CPURequests) + addValue(parametersMap, "kubernetesClusterAutoscalerCPULimit", clusterAutoscalerAddon.Containers[c].CPULimits) + addValue(parametersMap, "kubernetesClusterAutoscalerMemoryRequests", clusterAutoscalerAddon.Containers[c].MemoryRequests) + addValue(parametersMap, "kubernetesClusterAutoscalerMemoryLimit", clusterAutoscalerAddon.Containers[c].MemoryLimits) + addValue(parametersMap, "kubernetesClusterAutoscalerMinNodes", clusterAutoscalerAddon.Config["minNodes"]) + addValue(parametersMap, "kubernetesClusterAutoscalerMaxNodes", clusterAutoscalerAddon.Config["maxNodes"]) + addValue(parametersMap, "kubernetesClusterAutoscalerEnabled", helpers.IsTrueBoolPointer(clusterAutoscalerAddon.Enabled)) + addValue(parametersMap, "kubernetesClusterAutoscalerUseManagedIdentity", strings.ToLower(strconv.FormatBool(properties.OrchestratorProfile.KubernetesConfig.UseManagedIdentity))) + if clusterAutoscalerAddon.Containers[c].Image != "" { + addValue(parametersMap, "kubernetesClusterAutoscalerSpec", clusterAutoscalerAddon.Containers[c].Image) + } else { + addValue(parametersMap, "kubernetesClusterAutoscalerSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+KubeConfigs[k8sVersion][DefaultClusterAutoscalerAddonName]) + } + } + dashboardAddon := getAddonByName(properties.OrchestratorProfile.KubernetesConfig.Addons, DefaultDashboardAddonName) + c = getAddonContainersIndexByName(dashboardAddon.Containers, DefaultDashboardAddonName) + if c > -1 { + addValue(parametersMap, "kubernetesDashboardCPURequests", dashboardAddon.Containers[c].CPURequests) + addValue(parametersMap, "kubernetesDashboardCPULimit", dashboardAddon.Containers[c].CPULimits) + addValue(parametersMap, "kubernetesDashboardMemoryRequests", dashboardAddon.Containers[c].MemoryRequests) + addValue(parametersMap, "kubernetesDashboardMemoryLimit", dashboardAddon.Containers[c].MemoryLimits) + if dashboardAddon.Containers[c].Image != "" { + addValue(parametersMap, "kubernetesDashboardSpec", dashboardAddon.Containers[c].Image) + } else { + addValue(parametersMap, "kubernetesDashboardSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+KubeConfigs[k8sVersion][DefaultDashboardAddonName]) + } + } + reschedulerAddon := getAddonByName(properties.OrchestratorProfile.KubernetesConfig.Addons, DefaultReschedulerAddonName) + c = getAddonContainersIndexByName(reschedulerAddon.Containers, DefaultReschedulerAddonName) + if c > -1 { + addValue(parametersMap, "kubernetesReschedulerCPURequests", reschedulerAddon.Containers[c].CPURequests) + addValue(parametersMap, "kubernetesReschedulerCPULimit", reschedulerAddon.Containers[c].CPULimits) + addValue(parametersMap, "kubernetesReschedulerMemoryRequests", reschedulerAddon.Containers[c].MemoryRequests) + addValue(parametersMap, "kubernetesReschedulerMemoryLimit", reschedulerAddon.Containers[c].MemoryLimits) + if reschedulerAddon.Containers[c].Image != "" { + addValue(parametersMap, "kubernetesReschedulerSpec", dashboardAddon.Containers[c].Image) + } else { + addValue(parametersMap, "kubernetesReschedulerSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+KubeConfigs[k8sVersion][DefaultReschedulerAddonName]) + } + } + metricsServerAddon := getAddonByName(properties.OrchestratorProfile.KubernetesConfig.Addons, DefaultMetricsServerAddonName) + c = getAddonContainersIndexByName(metricsServerAddon.Containers, DefaultMetricsServerAddonName) + if c > -1 { + if metricsServerAddon.Containers[c].Image != "" { + addValue(parametersMap, "kubernetesMetricsServerSpec", metricsServerAddon.Containers[c].Image) + } else { + addValue(parametersMap, "kubernetesMetricsServerSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+KubeConfigs[k8sVersion][DefaultMetricsServerAddonName]) + } + } + nvidiaDevicePluginAddon := getAddonByName(properties.OrchestratorProfile.KubernetesConfig.Addons, DefaultNVIDIADevicePluginAddonName) + c = getAddonContainersIndexByName(nvidiaDevicePluginAddon.Containers, DefaultNVIDIADevicePluginAddonName) + if c > -1 { + if nvidiaDevicePluginAddon.Containers[c].Image != "" { + addValue(parametersMap, "kubernetesNVIDIADevicePluginSpec", nvidiaDevicePluginAddon.Containers[c].Image) + } else { + addValue(parametersMap, "kubernetesNVIDIADevicePluginSpec", cloudSpecConfig.KubernetesSpecConfig.NVIDIAImageBase+KubeConfigs[k8sVersion][DefaultNVIDIADevicePluginAddonName]) + } + } + containerMonitoringAddon := getAddonByName(properties.OrchestratorProfile.KubernetesConfig.Addons, ContainerMonitoringAddonName) + c = getAddonContainersIndexByName(containerMonitoringAddon.Containers, "omsagent") + if c > -1 { + addValue(parametersMap, "omsAgentVersion", containerMonitoringAddon.Config["omsAgentVersion"]) + addValue(parametersMap, "omsAgentDockerProviderVersion", containerMonitoringAddon.Config["dockerProviderVersion"]) + addValue(parametersMap, "omsAgentWorkspaceGuid", containerMonitoringAddon.Config["workspaceGuid"]) + addValue(parametersMap, "omsAgentWorkspaceKey", containerMonitoringAddon.Config["workspaceKey"]) + addValue(parametersMap, "kubernetesOMSAgentCPURequests", containerMonitoringAddon.Containers[c].CPURequests) + addValue(parametersMap, "kubernetesOMSAgentCPULimit", containerMonitoringAddon.Containers[c].CPULimits) + addValue(parametersMap, "kubernetesOMSAgentMemoryRequests", containerMonitoringAddon.Containers[c].MemoryRequests) + addValue(parametersMap, "kubernetesOMSAgentMemoryLimit", containerMonitoringAddon.Containers[c].MemoryLimits) + if containerMonitoringAddon.Containers[c].Image != "" { + addValue(parametersMap, "omsAgentImage", containerMonitoringAddon.Containers[c].Image) + } else { + addValue(parametersMap, "omsAgentImage", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+KubeConfigs[k8sVersion][ContainerMonitoringAddonName]) + } + } + if properties.OrchestratorProfile.IsAzureCNI() { + azureCNINetworkmonitorAddon := getAddonByName(properties.OrchestratorProfile.KubernetesConfig.Addons, AzureCNINetworkMonitoringAddonName) + c = getAddonContainersIndexByName(azureCNINetworkmonitorAddon.Containers, AzureCNINetworkMonitoringAddonName) + if c > -1 { + if azureCNINetworkmonitorAddon.Containers[c].Image != "" { + addValue(parametersMap, "AzureCNINetworkMonitorImageURL", azureCNINetworkmonitorAddon.Containers[c].Image) + } else { + addValue(parametersMap, "AzureCNINetworkMonitorImageURL", cloudSpecConfig.KubernetesSpecConfig.AzureCNIImageBase+KubeConfigs[k8sVersion][AzureCNINetworkMonitoringAddonName]) + } + } + } + addValue(parametersMap, "kubernetesKubeDNSSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+KubeConfigs[k8sVersion]["dns"]) + addValue(parametersMap, "kubernetesPodInfraContainerSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+KubeConfigs[k8sVersion]["pause"]) + addValue(parametersMap, "cloudProviderBackoff", strconv.FormatBool(properties.OrchestratorProfile.KubernetesConfig.CloudProviderBackoff)) + addValue(parametersMap, "cloudProviderBackoffRetries", strconv.Itoa(properties.OrchestratorProfile.KubernetesConfig.CloudProviderBackoffRetries)) + addValue(parametersMap, "cloudProviderBackoffExponent", strconv.FormatFloat(properties.OrchestratorProfile.KubernetesConfig.CloudProviderBackoffExponent, 'f', -1, 64)) + addValue(parametersMap, "cloudProviderBackoffDuration", strconv.Itoa(properties.OrchestratorProfile.KubernetesConfig.CloudProviderBackoffDuration)) + addValue(parametersMap, "cloudProviderBackoffJitter", strconv.FormatFloat(properties.OrchestratorProfile.KubernetesConfig.CloudProviderBackoffJitter, 'f', -1, 64)) + addValue(parametersMap, "cloudProviderRatelimit", strconv.FormatBool(properties.OrchestratorProfile.KubernetesConfig.CloudProviderRateLimit)) + addValue(parametersMap, "cloudProviderRatelimitQPS", strconv.FormatFloat(properties.OrchestratorProfile.KubernetesConfig.CloudProviderRateLimitQPS, 'f', -1, 64)) + addValue(parametersMap, "cloudProviderRatelimitBucket", strconv.Itoa(properties.OrchestratorProfile.KubernetesConfig.CloudProviderRateLimitBucket)) + addValue(parametersMap, "kubeClusterCidr", properties.OrchestratorProfile.KubernetesConfig.ClusterSubnet) + addValue(parametersMap, "kubernetesNonMasqueradeCidr", properties.OrchestratorProfile.KubernetesConfig.KubeletConfig["--non-masquerade-cidr"]) + addValue(parametersMap, "kubernetesKubeletClusterDomain", properties.OrchestratorProfile.KubernetesConfig.KubeletConfig["--cluster-domain"]) + addValue(parametersMap, "dockerBridgeCidr", properties.OrchestratorProfile.KubernetesConfig.DockerBridgeSubnet) + addValue(parametersMap, "networkPolicy", properties.OrchestratorProfile.KubernetesConfig.NetworkPolicy) + addValue(parametersMap, "networkPlugin", properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin) + addValue(parametersMap, "containerRuntime", properties.OrchestratorProfile.KubernetesConfig.ContainerRuntime) + addValue(parametersMap, "cniPluginsURL", cloudSpecConfig.KubernetesSpecConfig.CNIPluginsDownloadURL) + addValue(parametersMap, "vnetCniLinuxPluginsURL", cloudSpecConfig.KubernetesSpecConfig.VnetCNILinuxPluginsDownloadURL) + addValue(parametersMap, "vnetCniWindowsPluginsURL", cloudSpecConfig.KubernetesSpecConfig.VnetCNIWindowsPluginsDownloadURL) + addValue(parametersMap, "gchighthreshold", properties.OrchestratorProfile.KubernetesConfig.GCHighThreshold) + addValue(parametersMap, "gclowthreshold", properties.OrchestratorProfile.KubernetesConfig.GCLowThreshold) + addValue(parametersMap, "etcdDownloadURLBase", cloudSpecConfig.KubernetesSpecConfig.EtcdDownloadURLBase) + addValue(parametersMap, "etcdVersion", properties.OrchestratorProfile.KubernetesConfig.EtcdVersion) + addValue(parametersMap, "etcdDiskSizeGB", properties.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB) + addValue(parametersMap, "etcdEncryptionKey", properties.OrchestratorProfile.KubernetesConfig.EtcdEncryptionKey) + if properties.OrchestratorProfile.KubernetesConfig.PrivateJumpboxProvision() { + addValue(parametersMap, "jumpboxVMName", properties.OrchestratorProfile.KubernetesConfig.PrivateCluster.JumpboxProfile.Name) + addValue(parametersMap, "jumpboxVMSize", properties.OrchestratorProfile.KubernetesConfig.PrivateCluster.JumpboxProfile.VMSize) + addValue(parametersMap, "jumpboxUsername", properties.OrchestratorProfile.KubernetesConfig.PrivateCluster.JumpboxProfile.Username) + addValue(parametersMap, "jumpboxOSDiskSizeGB", properties.OrchestratorProfile.KubernetesConfig.PrivateCluster.JumpboxProfile.OSDiskSizeGB) + addValue(parametersMap, "jumpboxPublicKey", properties.OrchestratorProfile.KubernetesConfig.PrivateCluster.JumpboxProfile.PublicKey) + addValue(parametersMap, "jumpboxStorageProfile", properties.OrchestratorProfile.KubernetesConfig.PrivateCluster.JumpboxProfile.StorageProfile) + } + + if properties.OrchestratorProfile.KubernetesConfig.DockerEngineVersion != "" { + dockerEngineVersion = properties.OrchestratorProfile.KubernetesConfig.DockerEngineVersion + } } - dockerEngineVersion := KubeConfigs[k8sVersion]["dockerEngineVersion"] - if properties.OrchestratorProfile.KubernetesConfig.DockerEngineVersion != "" { - dockerEngineVersion = properties.OrchestratorProfile.KubernetesConfig.DockerEngineVersion + if properties.OrchestratorProfile.KubernetesConfig == nil || + !properties.OrchestratorProfile.KubernetesConfig.UseManagedIdentity { + + addValue(parametersMap, "servicePrincipalClientId", properties.ServicePrincipalProfile.ClientID) + if properties.ServicePrincipalProfile.KeyvaultSecretRef != nil { + addKeyvaultReference(parametersMap, "servicePrincipalClientSecret", + properties.ServicePrincipalProfile.KeyvaultSecretRef.VaultID, + properties.ServicePrincipalProfile.KeyvaultSecretRef.SecretName, + properties.ServicePrincipalProfile.KeyvaultSecretRef.SecretVersion) + } else { + addValue(parametersMap, "servicePrincipalClientSecret", properties.ServicePrincipalProfile.Secret) + } + + if properties.OrchestratorProfile.KubernetesConfig != nil && helpers.IsTrueBoolPointer(properties.OrchestratorProfile.KubernetesConfig.EnableEncryptionWithExternalKms) && !properties.OrchestratorProfile.KubernetesConfig.UseManagedIdentity && properties.ServicePrincipalProfile.ObjectID != "" { + addValue(parametersMap, "servicePrincipalObjectId", properties.ServicePrincipalProfile.ObjectID) + } + } + + if properties.HostedMasterProfile != nil { + addValue(parametersMap, "orchestratorName", "aks") + } else if properties.OrchestratorProfile.IsOpenShift() { + addValue(parametersMap, "orchestratorName", DefaultOpenshiftOrchestratorName) + } else { + addValue(parametersMap, "orchestratorName", DefaultOrchestratorName) } /** @@ -89,199 +292,11 @@ func assignKubernetesParameters(properties *api.Properties, parametersMap params addValue(parametersMap, "kubernetesEndpoint", properties.HostedMasterProfile.FQDN) } - if helpers.IsTrueBoolPointer(properties.OrchestratorProfile.KubernetesConfig.UseCloudControllerManager) { - kubernetesCcmSpec := properties.OrchestratorProfile.KubernetesConfig.KubernetesImageBase + KubeConfigs[k8sVersion]["ccm"] - if properties.OrchestratorProfile.KubernetesConfig.CustomCcmImage != "" { - kubernetesCcmSpec = properties.OrchestratorProfile.KubernetesConfig.CustomCcmImage - } - - addValue(parametersMap, "kubernetesCcmImageSpec", kubernetesCcmSpec) - } - if !properties.OrchestratorProfile.IsOpenShift() { addValue(parametersMap, "dockerEngineDownloadRepo", cloudSpecConfig.DockerSpecConfig.DockerEngineRepo) addValue(parametersMap, "dockerEngineVersion", dockerEngineVersion) } - addValue(parametersMap, "kubeDNSServiceIP", properties.OrchestratorProfile.KubernetesConfig.DNSServiceIP) - addValue(parametersMap, "kubeServiceCidr", properties.OrchestratorProfile.KubernetesConfig.ServiceCIDR) - addValue(parametersMap, "kubernetesHyperkubeSpec", kubernetesHyperkubeSpec) - addValue(parametersMap, "kubernetesAddonManagerSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+KubeConfigs[k8sVersion]["addonmanager"]) - addValue(parametersMap, "kubernetesAddonResizerSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+KubeConfigs[k8sVersion]["addonresizer"]) - addValue(parametersMap, "kubernetesDNSMasqSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+KubeConfigs[k8sVersion]["dnsmasq"]) - addValue(parametersMap, "kubernetesExecHealthzSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+KubeConfigs[k8sVersion]["exechealthz"]) - addValue(parametersMap, "kubernetesHeapsterSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+KubeConfigs[k8sVersion]["heapster"]) - tillerAddon := getAddonByName(properties.OrchestratorProfile.KubernetesConfig.Addons, DefaultTillerAddonName) - c := getAddonContainersIndexByName(tillerAddon.Containers, DefaultTillerAddonName) - if c > -1 { - addValue(parametersMap, "kubernetesTillerCPURequests", tillerAddon.Containers[c].CPURequests) - addValue(parametersMap, "kubernetesTillerCPULimit", tillerAddon.Containers[c].CPULimits) - addValue(parametersMap, "kubernetesTillerMemoryRequests", tillerAddon.Containers[c].MemoryRequests) - addValue(parametersMap, "kubernetesTillerMemoryLimit", tillerAddon.Containers[c].MemoryLimits) - addValue(parametersMap, "kubernetesTillerMaxHistory", tillerAddon.Config["max-history"]) - if tillerAddon.Containers[c].Image != "" { - addValue(parametersMap, "kubernetesTillerSpec", tillerAddon.Containers[c].Image) - } else { - addValue(parametersMap, "kubernetesTillerSpec", cloudSpecConfig.KubernetesSpecConfig.TillerImageBase+KubeConfigs[k8sVersion][DefaultTillerAddonName]) - } - } - aciConnectorAddon := getAddonByName(properties.OrchestratorProfile.KubernetesConfig.Addons, DefaultACIConnectorAddonName) - c = getAddonContainersIndexByName(aciConnectorAddon.Containers, DefaultACIConnectorAddonName) - if c > -1 { - addValue(parametersMap, "kubernetesACIConnectorEnabled", aciConnectorAddon.Enabled) - addValue(parametersMap, "kubernetesACIConnectorNodeName", aciConnectorAddon.Config["nodeName"]) - addValue(parametersMap, "kubernetesACIConnectorOS", aciConnectorAddon.Config["os"]) - addValue(parametersMap, "kubernetesACIConnectorTaint", aciConnectorAddon.Config["taint"]) - addValue(parametersMap, "kubernetesACIConnectorRegion", aciConnectorAddon.Config["region"]) - addValue(parametersMap, "kubernetesACIConnectorCPURequests", aciConnectorAddon.Containers[c].CPURequests) - addValue(parametersMap, "kubernetesACIConnectorCPULimit", aciConnectorAddon.Containers[c].CPULimits) - addValue(parametersMap, "kubernetesACIConnectorMemoryRequests", aciConnectorAddon.Containers[c].MemoryRequests) - addValue(parametersMap, "kubernetesACIConnectorMemoryLimit", aciConnectorAddon.Containers[c].MemoryLimits) - if aciConnectorAddon.Containers[c].Image != "" { - addValue(parametersMap, "kubernetesACIConnectorSpec", aciConnectorAddon.Containers[c].Image) - } else { - addValue(parametersMap, "kubernetesACIConnectorSpec", cloudSpecConfig.KubernetesSpecConfig.ACIConnectorImageBase+KubeConfigs[k8sVersion][DefaultACIConnectorAddonName]) - } - } - clusterAutoscalerAddon := getAddonByName(properties.OrchestratorProfile.KubernetesConfig.Addons, DefaultClusterAutoscalerAddonName) - c = getAddonContainersIndexByName(clusterAutoscalerAddon.Containers, DefaultClusterAutoscalerAddonName) - if c > -1 { - addValue(parametersMap, "kubernetesClusterAutoscalerCPURequests", clusterAutoscalerAddon.Containers[c].CPURequests) - addValue(parametersMap, "kubernetesClusterAutoscalerCPULimit", clusterAutoscalerAddon.Containers[c].CPULimits) - addValue(parametersMap, "kubernetesClusterAutoscalerMemoryRequests", clusterAutoscalerAddon.Containers[c].MemoryRequests) - addValue(parametersMap, "kubernetesClusterAutoscalerMemoryLimit", clusterAutoscalerAddon.Containers[c].MemoryLimits) - addValue(parametersMap, "kubernetesClusterAutoscalerMinNodes", clusterAutoscalerAddon.Config["minNodes"]) - addValue(parametersMap, "kubernetesClusterAutoscalerMaxNodes", clusterAutoscalerAddon.Config["maxNodes"]) - addValue(parametersMap, "kubernetesClusterAutoscalerEnabled", clusterAutoscalerAddon.Enabled) - addValue(parametersMap, "kubernetesClusterAutoscalerUseManagedIdentity", strings.ToLower(strconv.FormatBool(properties.OrchestratorProfile.KubernetesConfig.UseManagedIdentity))) - if clusterAutoscalerAddon.Containers[c].Image != "" { - addValue(parametersMap, "kubernetesClusterAutoscalerSpec", clusterAutoscalerAddon.Containers[c].Image) - } else { - addValue(parametersMap, "kubernetesClusterAutoscalerSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+KubeConfigs[k8sVersion][DefaultClusterAutoscalerAddonName]) - } - } - dashboardAddon := getAddonByName(properties.OrchestratorProfile.KubernetesConfig.Addons, DefaultDashboardAddonName) - c = getAddonContainersIndexByName(dashboardAddon.Containers, DefaultDashboardAddonName) - if c > -1 { - addValue(parametersMap, "kubernetesDashboardCPURequests", dashboardAddon.Containers[c].CPURequests) - addValue(parametersMap, "kubernetesDashboardCPULimit", dashboardAddon.Containers[c].CPULimits) - addValue(parametersMap, "kubernetesDashboardMemoryRequests", dashboardAddon.Containers[c].MemoryRequests) - addValue(parametersMap, "kubernetesDashboardMemoryLimit", dashboardAddon.Containers[c].MemoryLimits) - if dashboardAddon.Containers[c].Image != "" { - addValue(parametersMap, "kubernetesDashboardSpec", dashboardAddon.Containers[c].Image) - } else { - addValue(parametersMap, "kubernetesDashboardSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+KubeConfigs[k8sVersion][DefaultDashboardAddonName]) - } - } - reschedulerAddon := getAddonByName(properties.OrchestratorProfile.KubernetesConfig.Addons, DefaultReschedulerAddonName) - c = getAddonContainersIndexByName(reschedulerAddon.Containers, DefaultReschedulerAddonName) - if c > -1 { - addValue(parametersMap, "kubernetesReschedulerCPURequests", reschedulerAddon.Containers[c].CPURequests) - addValue(parametersMap, "kubernetesReschedulerCPULimit", reschedulerAddon.Containers[c].CPULimits) - addValue(parametersMap, "kubernetesReschedulerMemoryRequests", reschedulerAddon.Containers[c].MemoryRequests) - addValue(parametersMap, "kubernetesReschedulerMemoryLimit", reschedulerAddon.Containers[c].MemoryLimits) - if reschedulerAddon.Containers[c].Image != "" { - addValue(parametersMap, "kubernetesReschedulerSpec", dashboardAddon.Containers[c].Image) - } else { - addValue(parametersMap, "kubernetesReschedulerSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+KubeConfigs[k8sVersion][DefaultReschedulerAddonName]) - } - } - metricsServerAddon := getAddonByName(properties.OrchestratorProfile.KubernetesConfig.Addons, DefaultMetricsServerAddonName) - c = getAddonContainersIndexByName(metricsServerAddon.Containers, DefaultMetricsServerAddonName) - if c > -1 { - if metricsServerAddon.Containers[c].Image != "" { - addValue(parametersMap, "kubernetesMetricsServerSpec", metricsServerAddon.Containers[c].Image) - } else { - addValue(parametersMap, "kubernetesMetricsServerSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+KubeConfigs[k8sVersion][DefaultMetricsServerAddonName]) - } - } - nvidiaDevicePluginAddon := getAddonByName(properties.OrchestratorProfile.KubernetesConfig.Addons, DefaultNVIDIADevicePluginAddonName) - c = getAddonContainersIndexByName(nvidiaDevicePluginAddon.Containers, DefaultNVIDIADevicePluginAddonName) - if c > -1 { - if nvidiaDevicePluginAddon.Containers[c].Image != "" { - addValue(parametersMap, "kubernetesNVIDIADevicePluginSpec", nvidiaDevicePluginAddon.Containers[c].Image) - } else { - addValue(parametersMap, "kubernetesNVIDIADevicePluginSpec", cloudSpecConfig.KubernetesSpecConfig.NVIDIAImageBase+KubeConfigs[k8sVersion][DefaultNVIDIADevicePluginAddonName]) - } - } - containerMonitoringAddon := getAddonByName(properties.OrchestratorProfile.KubernetesConfig.Addons, ContainerMonitoringAddonName) - c = getAddonContainersIndexByName(containerMonitoringAddon.Containers, "omsagent") - if c > -1 { - addValue(parametersMap, "omsAgentVersion", containerMonitoringAddon.Config["omsAgentVersion"]) - addValue(parametersMap, "omsAgentDockerProviderVersion", containerMonitoringAddon.Config["dockerProviderVersion"]) - addValue(parametersMap, "omsAgentWorkspaceGuid", containerMonitoringAddon.Config["workspaceGuid"]) - addValue(parametersMap, "omsAgentWorkspaceKey", containerMonitoringAddon.Config["workspaceKey"]) - addValue(parametersMap, "kubernetesOMSAgentCPURequests", containerMonitoringAddon.Containers[c].CPURequests) - addValue(parametersMap, "kubernetesOMSAgentCPULimit", containerMonitoringAddon.Containers[c].CPULimits) - addValue(parametersMap, "kubernetesOMSAgentMemoryRequests", containerMonitoringAddon.Containers[c].MemoryRequests) - addValue(parametersMap, "kubernetesOMSAgentMemoryLimit", containerMonitoringAddon.Containers[c].MemoryLimits) - if containerMonitoringAddon.Containers[c].Image != "" { - addValue(parametersMap, "omsAgentImage", containerMonitoringAddon.Containers[c].Image) - } else { - addValue(parametersMap, "omsAgentImage", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+KubeConfigs[k8sVersion][ContainerMonitoringAddonName]) - } - } - addValue(parametersMap, "kubernetesKubeDNSSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+KubeConfigs[k8sVersion]["dns"]) - addValue(parametersMap, "kubernetesPodInfraContainerSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+KubeConfigs[k8sVersion]["pause"]) - addValue(parametersMap, "cloudProviderBackoff", strconv.FormatBool(properties.OrchestratorProfile.KubernetesConfig.CloudProviderBackoff)) - addValue(parametersMap, "cloudProviderBackoffRetries", strconv.Itoa(properties.OrchestratorProfile.KubernetesConfig.CloudProviderBackoffRetries)) - addValue(parametersMap, "cloudProviderBackoffExponent", strconv.FormatFloat(properties.OrchestratorProfile.KubernetesConfig.CloudProviderBackoffExponent, 'f', -1, 64)) - addValue(parametersMap, "cloudProviderBackoffDuration", strconv.Itoa(properties.OrchestratorProfile.KubernetesConfig.CloudProviderBackoffDuration)) - addValue(parametersMap, "cloudProviderBackoffJitter", strconv.FormatFloat(properties.OrchestratorProfile.KubernetesConfig.CloudProviderBackoffJitter, 'f', -1, 64)) - addValue(parametersMap, "cloudProviderRatelimit", strconv.FormatBool(properties.OrchestratorProfile.KubernetesConfig.CloudProviderRateLimit)) - addValue(parametersMap, "cloudProviderRatelimitQPS", strconv.FormatFloat(properties.OrchestratorProfile.KubernetesConfig.CloudProviderRateLimitQPS, 'f', -1, 64)) - addValue(parametersMap, "cloudProviderRatelimitBucket", strconv.Itoa(properties.OrchestratorProfile.KubernetesConfig.CloudProviderRateLimitBucket)) - addValue(parametersMap, "kubeClusterCidr", properties.OrchestratorProfile.KubernetesConfig.ClusterSubnet) - addValue(parametersMap, "kubernetesNonMasqueradeCidr", properties.OrchestratorProfile.KubernetesConfig.KubeletConfig["--non-masquerade-cidr"]) - addValue(parametersMap, "kubernetesKubeletClusterDomain", properties.OrchestratorProfile.KubernetesConfig.KubeletConfig["--cluster-domain"]) - addValue(parametersMap, "generatorCode", generatorCode) - if properties.HostedMasterProfile != nil { - addValue(parametersMap, "orchestratorName", "aks") - } else if properties.OrchestratorProfile.IsOpenShift() { - addValue(parametersMap, "orchestratorName", DefaultOpenshiftOrchestratorName) - } else { - addValue(parametersMap, "orchestratorName", DefaultOrchestratorName) - } - addValue(parametersMap, "dockerBridgeCidr", properties.OrchestratorProfile.KubernetesConfig.DockerBridgeSubnet) - addValue(parametersMap, "networkPolicy", properties.OrchestratorProfile.KubernetesConfig.NetworkPolicy) - addValue(parametersMap, "networkPlugin", properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin) - addValue(parametersMap, "containerRuntime", properties.OrchestratorProfile.KubernetesConfig.ContainerRuntime) - addValue(parametersMap, "cniPluginsURL", cloudSpecConfig.KubernetesSpecConfig.CNIPluginsDownloadURL) - addValue(parametersMap, "vnetCniLinuxPluginsURL", cloudSpecConfig.KubernetesSpecConfig.VnetCNILinuxPluginsDownloadURL) - addValue(parametersMap, "vnetCniWindowsPluginsURL", cloudSpecConfig.KubernetesSpecConfig.VnetCNIWindowsPluginsDownloadURL) - addValue(parametersMap, "gchighthreshold", properties.OrchestratorProfile.KubernetesConfig.GCHighThreshold) - addValue(parametersMap, "gclowthreshold", properties.OrchestratorProfile.KubernetesConfig.GCLowThreshold) - addValue(parametersMap, "etcdDownloadURLBase", cloudSpecConfig.KubernetesSpecConfig.EtcdDownloadURLBase) - addValue(parametersMap, "etcdVersion", properties.OrchestratorProfile.KubernetesConfig.EtcdVersion) - addValue(parametersMap, "etcdDiskSizeGB", properties.OrchestratorProfile.KubernetesConfig.EtcdDiskSizeGB) - addValue(parametersMap, "etcdEncryptionKey", properties.OrchestratorProfile.KubernetesConfig.EtcdEncryptionKey) - if properties.OrchestratorProfile.KubernetesConfig.PrivateJumpboxProvision() { - addValue(parametersMap, "jumpboxVMName", properties.OrchestratorProfile.KubernetesConfig.PrivateCluster.JumpboxProfile.Name) - addValue(parametersMap, "jumpboxVMSize", properties.OrchestratorProfile.KubernetesConfig.PrivateCluster.JumpboxProfile.VMSize) - addValue(parametersMap, "jumpboxUsername", properties.OrchestratorProfile.KubernetesConfig.PrivateCluster.JumpboxProfile.Username) - addValue(parametersMap, "jumpboxOSDiskSizeGB", properties.OrchestratorProfile.KubernetesConfig.PrivateCluster.JumpboxProfile.OSDiskSizeGB) - addValue(parametersMap, "jumpboxPublicKey", properties.OrchestratorProfile.KubernetesConfig.PrivateCluster.JumpboxProfile.PublicKey) - addValue(parametersMap, "jumpboxStorageProfile", properties.OrchestratorProfile.KubernetesConfig.PrivateCluster.JumpboxProfile.StorageProfile) - } - - if properties.OrchestratorProfile.KubernetesConfig == nil || - !properties.OrchestratorProfile.KubernetesConfig.UseManagedIdentity { - - addValue(parametersMap, "servicePrincipalClientId", properties.ServicePrincipalProfile.ClientID) - if properties.ServicePrincipalProfile.KeyvaultSecretRef != nil { - addKeyvaultReference(parametersMap, "servicePrincipalClientSecret", - properties.ServicePrincipalProfile.KeyvaultSecretRef.VaultID, - properties.ServicePrincipalProfile.KeyvaultSecretRef.SecretName, - properties.ServicePrincipalProfile.KeyvaultSecretRef.SecretVersion) - } else { - addValue(parametersMap, "servicePrincipalClientSecret", properties.ServicePrincipalProfile.Secret) - } - - if properties.OrchestratorProfile.KubernetesConfig != nil && helpers.IsTrueBoolPointer(properties.OrchestratorProfile.KubernetesConfig.EnableEncryptionWithExternalKms) && !properties.OrchestratorProfile.KubernetesConfig.UseManagedIdentity && properties.ServicePrincipalProfile.ObjectID != "" { - addValue(parametersMap, "servicePrincipalObjectId", properties.ServicePrincipalProfile.ObjectID) - } - } - if properties.AADProfile != nil { addValue(parametersMap, "aadTenantId", properties.AADProfile.TenantID) if properties.AADProfile.AdminGroupID != "" { diff --git a/pkg/acsengine/params_k8s_test.go b/pkg/acsengine/params_k8s_test.go new file mode 100644 index 0000000000..85ccb7c803 --- /dev/null +++ b/pkg/acsengine/params_k8s_test.go @@ -0,0 +1,46 @@ +package acsengine + +import ( + "path" + "testing" + + "github.com/Azure/acs-engine/pkg/api" + "github.com/Azure/acs-engine/pkg/i18n" + "github.com/leonelquinteros/gotext" +) + +func TestAssignKubernetesParameters(t *testing.T) { + // Initialize locale for translation + locale := gotext.NewLocale(path.Join("..", "..", "translations"), "en_US") + i18n.Initialize(locale) + + apiloader := &api.Apiloader{ + Translator: &i18n.Translator{ + Locale: locale, + }, + } + // iterate the test data directory + apiModelTestFiles := &[]APIModelTestFile{} + if e := IterateTestFilesDirectory(TestDataDir, apiModelTestFiles); e != nil { + t.Error(e.Error()) + return + } + + for _, tuple := range *apiModelTestFiles { + containerService, _, err := apiloader.LoadContainerServiceFromFile(tuple.APIModelFilename, true, false, nil) + if err != nil { + t.Errorf("Loading file %s got error: %s", tuple.APIModelFilename, err.Error()) + continue + } + + parametersMap := paramsMap{} + cloudSpecConfig := getCloudSpecConfig("eastus") + assignKubernetesParameters(containerService.Properties, parametersMap, cloudSpecConfig, DefaultGeneratorCode) + for k, v := range parametersMap { + switch val := v.(paramsMap)["value"].(type) { + case *bool: + t.Errorf("got a pointer to bool in paramsMap value, this is dangerous!: %s: %v", k, val) + } + } + } +} diff --git a/pkg/acsengine/testdata/addons/kubernetes.json b/pkg/acsengine/testdata/addons/kubernetes.json new file mode 100644 index 0000000000..15a80107e0 --- /dev/null +++ b/pkg/acsengine/testdata/addons/kubernetes.json @@ -0,0 +1,124 @@ +{ + "apiVersion": "vlabs", + "plan": {}, + "properties": { + "provisioningState": "", + "orchestratorProfile": { + "orchestratorType": "Kubernetes", + "orchestratorRelease": "1.10", + "kubernetesConfig": { + "addons": [ + { + "name": "tiller", + "enabled": false + }, + { + "name": "kubernetes-dashboard", + "enabled": false + }, + { + "name": "aci-connector", + "enabled": true, + "containers": [ + { + "name": "aci-connector", + "cpuRequests": "50m", + "memoryRequests": "150Mi", + "cpuLimits": "50m", + "memoryLimits": "150Mi" + } + ], + "config": { + "nodeName": "aci-connector", + "os": "Linux", + "region": "westus", + "taint": "azure.com/aci" + } + }, + { + "name": "cluster-autoscaler", + "enabled": true, + "containers": [ + { + "name": "cluster-autoscaler", + "cpuRequests": "100m", + "memoryRequests": "300Mi", + "cpuLimits": "100m", + "memoryLimits": "300Mi" + } + ], + "config": { + "maxNodes": "5", + "minNodes": "1" + } + }, + { + "name": "rescheduler", + "enabled": false, + "containers": [ + { + "name": "rescheduler", + "cpuRequests": "10m", + "memoryRequests": "100Mi", + "cpuLimits": "10m", + "memoryLimits": "100Mi" + } + ] + }, + { + "name": "metrics-server", + "enabled": true, + "containers": [ + { + "name": "metrics-server" + } + ] + } + ] + } + }, + "masterProfile": { + "count": 1, + "dnsPrefix": "masterdns1", + "vmSize": "Standard_D2_v2" + }, + "agentPoolProfiles": [ + { + "name": "agentpool1", + "count": 3, + "vmSize": "Standard_D2_v2" + } + ], + "linuxProfile": { + "adminUsername": "azureuser", + "ssh": { + "publicKeys": [ + { + "keyData": "ssh-rsa PUBLICKEY azureuser@linuxvm" + } + ] + } + }, + "windowsProfile": {}, + "servicePrincipalProfile": { + "clientId": "ServicePrincipalClientID", + "secret": "myServicePrincipalClientSecret" + }, + "certificateProfile": { + "caCertificate": "caCertificate", + "caPrivateKey": "caPrivateKey", + "apiServerCertificate": "apiServerCertificate", + "apiServerPrivateKey": "apiServerPrivateKey", + "clientCertificate": "clientCertificate", + "clientPrivateKey": "clientPrivateKey", + "kubeConfigCertificate": "kubeConfigCertificate", + "kubeConfigPrivateKey": "kubeConfigPrivateKey", + "etcdClientCertificate": "etcdClientCertificate", + "etcdClientPrivateKey": "etcdClientPrivateKey", + "etcdServerCertificate": "etcdServerCertificate", + "etcdServerPrivateKey": "etcdServerPrivateKey", + "etcdPeerCertificates": ["etcdPeerCertificate0"], + "etcdPeerPrivateKeys": ["etcdPeerPrivateKey0"] + } + } +} diff --git a/pkg/acsengine/types.go b/pkg/acsengine/types.go index d1ff203630..38eb0416ce 100644 --- a/pkg/acsengine/types.go +++ b/pkg/acsengine/types.go @@ -51,6 +51,7 @@ type KubernetesSpecConfig struct { TillerImageBase string ACIConnectorImageBase string NVIDIAImageBase string + AzureCNIImageBase string EtcdDownloadURLBase string KubeBinariesSASURLBase string WindowsPackageSASURLBase string diff --git a/pkg/api/agentPoolOnlyApi/v20170831/merge.go b/pkg/api/agentPoolOnlyApi/v20170831/merge.go new file mode 100644 index 0000000000..d315394967 --- /dev/null +++ b/pkg/api/agentPoolOnlyApi/v20170831/merge.go @@ -0,0 +1,44 @@ +package v20170831 + +import ( + "fmt" +) + +// Merge existing ManagedCluster attribute into mc +func (mc *ManagedCluster) Merge(emc *ManagedCluster) error { + + // Merge properties.dnsPrefix + if emc.Properties.DNSPrefix == "" { + return fmt.Errorf("existing ManagedCluster expect properties.dnsPrefix not to be empty") + } + + if mc.Properties.DNSPrefix == "" { + mc.Properties.DNSPrefix = emc.Properties.DNSPrefix + } else if mc.Properties.DNSPrefix != emc.Properties.DNSPrefix { + return fmt.Errorf("change dnsPrefix from %s to %s is not supported", + emc.Properties.DNSPrefix, + mc.Properties.DNSPrefix) + } + // Merge Properties.AgentPoolProfiles + if mc.Properties.AgentPoolProfiles == nil { + mc.Properties.AgentPoolProfiles = emc.Properties.AgentPoolProfiles + } + // Merge Properties.LinuxProfile + if mc.Properties.LinuxProfile == nil { + mc.Properties.LinuxProfile = emc.Properties.LinuxProfile + } + // Merge Properties.WindowsProfile + if mc.Properties.WindowsProfile == nil { + mc.Properties.WindowsProfile = emc.Properties.WindowsProfile + } + // Merge Properties.ServicePrincipalProfile + if mc.Properties.ServicePrincipalProfile == nil { + mc.Properties.ServicePrincipalProfile = emc.Properties.ServicePrincipalProfile + } + // Merge Properties.KubernetesVersion + if mc.Properties.KubernetesVersion == "" { + mc.Properties.KubernetesVersion = emc.Properties.KubernetesVersion + } + + return nil +} diff --git a/pkg/api/agentPoolOnlyApi/v20170831/types.go b/pkg/api/agentPoolOnlyApi/v20170831/types.go index b8c384e5b4..c6269834aa 100644 --- a/pkg/api/agentPoolOnlyApi/v20170831/types.go +++ b/pkg/api/agentPoolOnlyApi/v20170831/types.go @@ -63,7 +63,7 @@ type ManagedClusterAccessProfile struct { // (optional) is the version of the secret (default: the latest version) type ServicePrincipalProfile struct { ClientID string `json:"clientId,omitempty" validate:"required"` - Secret string `json:"secret,omitempty"` + Secret string `json:"secret,omitempty" conform:"redact"` } // LinuxProfile represents the Linux configuration passed to the cluster @@ -83,7 +83,7 @@ type PublicKey struct { // WindowsProfile represents the Windows configuration passed to the cluster type WindowsProfile struct { AdminUsername string `json:"adminUsername,omitempty" validate:"required"` - AdminPassword string `json:"adminPassword,omitempty"` + AdminPassword string `json:"adminPassword,omitempty" conform:"redact"` } // ProvisioningState represents the current state of container service resource. diff --git a/pkg/api/agentPoolOnlyApi/v20170831/validate.go b/pkg/api/agentPoolOnlyApi/v20170831/validate.go index e9c88a114a..062285b685 100644 --- a/pkg/api/agentPoolOnlyApi/v20170831/validate.go +++ b/pkg/api/agentPoolOnlyApi/v20170831/validate.go @@ -68,7 +68,7 @@ func (a *Properties) Validate() error { // Don't need to call validate.Struct(m) // It is handled by Properties.Validate() - if e := validateDNSPrefix(a.DNSPrefix); e != nil { + if e := common.ValidateDNSPrefix(a.DNSPrefix); e != nil { return e } @@ -106,18 +106,6 @@ func validatePoolName(poolName string) error { return nil } -func validateDNSPrefix(dnsName string) error { - dnsNameRegex := `^([A-Za-z][A-Za-z0-9-]{1,43}[A-Za-z0-9])$` - re, err := regexp.Compile(dnsNameRegex) - if err != nil { - return err - } - if !re.MatchString(dnsName) { - return fmt.Errorf("DNSPrefix '%s' is invalid. The DNSPrefix must contain between 3 and 45 characters. The name can contain only letters, numbers, and hyphens. The name must start with a letter and must end with a letter or a number. (length was %d)", dnsName, len(dnsName)) - } - return nil -} - func validateUniqueProfileNames(profiles []*AgentPoolProfile) error { profileNames := make(map[string]bool) for _, profile := range profiles { diff --git a/pkg/api/agentPoolOnlyApi/v20180331/apiloader_test.go b/pkg/api/agentPoolOnlyApi/v20180331/apiloader_test.go index 4c028c055d..a725d4a0d0 100644 --- a/pkg/api/agentPoolOnlyApi/v20180331/apiloader_test.go +++ b/pkg/api/agentPoolOnlyApi/v20180331/apiloader_test.go @@ -95,7 +95,7 @@ var _ = Describe("v20180331 test suite", func() { // ssh key should not be re-generated Expect(sshAutoGenerated).To(BeFalse()) Expect(cs2.Properties.MasterProfile).To(BeNil()) - Expect(cs2.Properties.LinuxProfile).To(BeNil()) + Expect(cs2.Properties.LinuxProfile).NotTo(BeNil()) Expect(cs2.Properties.OrchestratorProfile).NotTo(BeNil()) Expect(cs2.Properties.OrchestratorProfile.OrchestratorVersion).To(Equal(k8sVersions[0])) Expect(cs2.Properties.OrchestratorProfile.KubernetesConfig.NetworkPolicy).To(Equal("")) @@ -105,10 +105,10 @@ var _ = Describe("v20180331 test suite", func() { Expect(cs2.Properties.OrchestratorProfile.KubernetesConfig.DockerBridgeSubnet).To(Equal(api.DefaultDockerBridgeSubnet)) Expect(*cs2.Properties.OrchestratorProfile.KubernetesConfig.EnableRbac).To(Equal(false)) Expect(*cs2.Properties.OrchestratorProfile.KubernetesConfig.EnableSecureKubelet).To(Equal(false)) - Expect(cs2.Properties.ServicePrincipalProfile).To(BeNil()) + Expect(cs2.Properties.ServicePrincipalProfile).NotTo(BeNil()) Expect(cs2.Properties.HostedMasterProfile).NotTo(BeNil()) Expect(cs2.Properties.HostedMasterProfile.DNSPrefix).To(Equal(model.Properties.DNSPrefix)) - Expect(len(cs2.Properties.AgentPoolProfiles)).To(Equal(0)) + Expect(len(cs2.Properties.AgentPoolProfiles)).To(Equal(1)) }) }) @@ -185,7 +185,7 @@ var _ = Describe("v20180331 test suite", func() { // ssh key should not be re-generated Expect(sshAutoGenerated).To(BeFalse()) Expect(cs2.Properties.MasterProfile).To(BeNil()) - Expect(cs2.Properties.LinuxProfile).To(BeNil()) + Expect(cs2.Properties.LinuxProfile).NotTo(BeNil()) Expect(cs2.Properties.OrchestratorProfile).NotTo(BeNil()) Expect(cs2.Properties.OrchestratorProfile.OrchestratorVersion).To(Equal(k8sVersions[0])) Expect(cs2.Properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin).To(Equal("azure")) @@ -195,10 +195,10 @@ var _ = Describe("v20180331 test suite", func() { Expect(cs2.Properties.OrchestratorProfile.KubernetesConfig.DockerBridgeSubnet).To(Equal(dockerBridgeCidr)) Expect(*cs2.Properties.OrchestratorProfile.KubernetesConfig.EnableRbac).To(Equal(false)) Expect(*cs2.Properties.OrchestratorProfile.KubernetesConfig.EnableSecureKubelet).To(Equal(false)) - Expect(cs2.Properties.ServicePrincipalProfile).To(BeNil()) + Expect(cs2.Properties.ServicePrincipalProfile).NotTo(BeNil()) Expect(cs2.Properties.HostedMasterProfile).NotTo(BeNil()) Expect(cs2.Properties.HostedMasterProfile.DNSPrefix).To(Equal(model.Properties.DNSPrefix)) - Expect(len(cs2.Properties.AgentPoolProfiles)).To(Equal(0)) + Expect(len(cs2.Properties.AgentPoolProfiles)).To(Equal(1)) }) }) @@ -279,7 +279,7 @@ var _ = Describe("v20180331 test suite", func() { // ssh key should not be re-generated Expect(sshAutoGenerated).To(BeFalse()) Expect(cs2.Properties.MasterProfile).To(BeNil()) - Expect(cs2.Properties.LinuxProfile).To(BeNil()) + Expect(cs2.Properties.LinuxProfile).NotTo(BeNil()) Expect(cs2.Properties.OrchestratorProfile).NotTo(BeNil()) Expect(cs2.Properties.OrchestratorProfile.OrchestratorVersion).To(Equal(k8sVersions[0])) Expect(cs2.Properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin).To(Equal("azure")) @@ -365,7 +365,7 @@ var _ = Describe("v20180331 test suite", func() { // ssh key should not be re-generated Expect(sshAutoGenerated).To(BeFalse()) Expect(cs2.Properties.MasterProfile).To(BeNil()) - Expect(cs2.Properties.LinuxProfile).To(BeNil()) + Expect(cs2.Properties.LinuxProfile).NotTo(BeNil()) Expect(cs2.Properties.OrchestratorProfile).NotTo(BeNil()) Expect(cs2.Properties.OrchestratorProfile.OrchestratorVersion).To(Equal(k8sVersions[1])) Expect(cs2.Properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin).To(Equal("azure")) @@ -377,7 +377,7 @@ var _ = Describe("v20180331 test suite", func() { Expect(*cs2.Properties.OrchestratorProfile.KubernetesConfig.EnableSecureKubelet).To(Equal(false)) Expect(cs2.Properties.HostedMasterProfile).NotTo(BeNil()) Expect(cs2.Properties.HostedMasterProfile.DNSPrefix).To(Equal(model.Properties.DNSPrefix)) - Expect(len(cs2.Properties.AgentPoolProfiles)).To(Equal(0)) + Expect(len(cs2.Properties.AgentPoolProfiles)).To(Equal(1)) }) }) }) diff --git a/pkg/api/agentPoolOnlyApi/v20180331/merge.go b/pkg/api/agentPoolOnlyApi/v20180331/merge.go index 71cf6477d8..c9a1dbf911 100644 --- a/pkg/api/agentPoolOnlyApi/v20180331/merge.go +++ b/pkg/api/agentPoolOnlyApi/v20180331/merge.go @@ -22,6 +22,27 @@ func (mc *ManagedCluster) Merge(emc *ManagedCluster) error { mc.Properties.DNSPrefix) } + // Merge Properties.AgentPoolProfiles + if mc.Properties.AgentPoolProfiles == nil { + mc.Properties.AgentPoolProfiles = emc.Properties.AgentPoolProfiles + } + // Merge Properties.LinuxProfile + if mc.Properties.LinuxProfile == nil { + mc.Properties.LinuxProfile = emc.Properties.LinuxProfile + } + // Merge Properties.WindowsProfile + if mc.Properties.WindowsProfile == nil { + mc.Properties.WindowsProfile = emc.Properties.WindowsProfile + } + // Merge Properties.ServicePrincipalProfile + if mc.Properties.ServicePrincipalProfile == nil { + mc.Properties.ServicePrincipalProfile = emc.Properties.ServicePrincipalProfile + } + // Merge Properties.KubernetesVersion + if mc.Properties.KubernetesVersion == "" { + mc.Properties.KubernetesVersion = emc.Properties.KubernetesVersion + } + // Merge properties.enableRBAC if emc.Properties.EnableRBAC == nil { return fmt.Errorf("existing ManagedCluster expect properties.enableRBAC not to be nil") diff --git a/pkg/api/agentPoolOnlyApi/v20180331/types.go b/pkg/api/agentPoolOnlyApi/v20180331/types.go index 5b4f9acd63..0ef4b7abf3 100644 --- a/pkg/api/agentPoolOnlyApi/v20180331/types.go +++ b/pkg/api/agentPoolOnlyApi/v20180331/types.go @@ -102,7 +102,7 @@ type ManagedClusterAccessProfile struct { // (optional) is the version of the secret (default: the latest version) type ServicePrincipalProfile struct { ClientID string `json:"clientId,omitempty" validate:"required"` - Secret string `json:"secret,omitempty"` + Secret string `json:"secret,omitempty" conform:"redact"` } // LinuxProfile represents the Linux configuration passed to the cluster @@ -122,7 +122,7 @@ type PublicKey struct { // WindowsProfile represents the Windows configuration passed to the cluster type WindowsProfile struct { AdminUsername string `json:"adminUsername,omitempty" validate:"required"` - AdminPassword string `json:"adminPassword,omitempty"` + AdminPassword string `json:"adminPassword,omitempty" conform:"redact"` } // ProvisioningState represents the current state of container service resource. @@ -206,7 +206,7 @@ type AADProfile struct { // The server AAD application ID. ServerAppID string `json:"serverAppID,omitempty"` // The server AAD application secret - ServerAppSecret string `json:"serverAppSecret,omitempty"` + ServerAppSecret string `json:"serverAppSecret,omitempty" conform:"redact"` // The AAD tenant ID to use for authentication. // If not specified, will use the tenant of the deployment subscription. // Optional diff --git a/pkg/api/agentPoolOnlyApi/v20180331/validate.go b/pkg/api/agentPoolOnlyApi/v20180331/validate.go index e79e311a95..b62830d2ae 100644 --- a/pkg/api/agentPoolOnlyApi/v20180331/validate.go +++ b/pkg/api/agentPoolOnlyApi/v20180331/validate.go @@ -101,7 +101,7 @@ func (a *Properties) Validate() error { // Don't need to call validate.Struct(m) // It is handled by Properties.Validate() - if e := validateDNSPrefix(a.DNSPrefix); e != nil { + if e := common.ValidateDNSPrefix(a.DNSPrefix); e != nil { return e } @@ -148,18 +148,6 @@ func validatePoolName(poolName string) error { return nil } -func validateDNSPrefix(dnsName string) error { - dnsNameRegex := `^([A-Za-z][A-Za-z0-9-]{1,43}[A-Za-z0-9])$` - re, err := regexp.Compile(dnsNameRegex) - if err != nil { - return err - } - if !re.MatchString(dnsName) { - return fmt.Errorf("DNSPrefix '%s' is invalid. The DNSPrefix must contain between 3 and 45 characters. The name can contain only letters, numbers, and hyphens. The name must start with a letter and must end with a letter or a number. (length was %d)", dnsName, len(dnsName)) - } - return nil -} - func validateUniqueProfileNames(profiles []*AgentPoolProfile) error { profileNames := make(map[string]bool) for _, profile := range profiles { diff --git a/pkg/api/agentPoolOnlyApi/vlabs/validate.go b/pkg/api/agentPoolOnlyApi/vlabs/validate.go index e3aeed17b1..0149c3ef23 100644 --- a/pkg/api/agentPoolOnlyApi/vlabs/validate.go +++ b/pkg/api/agentPoolOnlyApi/vlabs/validate.go @@ -68,7 +68,7 @@ func (a *Properties) Validate() error { // Don't need to call validate.Struct(m) // It is handled by Properties.Validate() - if e := validateDNSName(a.DNSPrefix); e != nil { + if e := common.ValidateDNSPrefix(a.DNSPrefix); e != nil { return e } @@ -96,18 +96,6 @@ func validatePoolName(poolName string) error { return nil } -func validateDNSName(dnsName string) error { - dnsNameRegex := `^([A-Za-z][A-Za-z0-9-]{1,43}[A-Za-z0-9])$` - re, err := regexp.Compile(dnsNameRegex) - if err != nil { - return err - } - if !re.MatchString(dnsName) { - return fmt.Errorf("DNS name '%s' is invalid. The DNS name must contain between 3 and 45 characters. The name can contain only letters, numbers, and hyphens. The name must start with a letter and must end with a letter or a number. (length was %d)", dnsName, len(dnsName)) - } - return nil -} - func validateUniqueProfileNames(profiles []*AgentPoolProfile) error { profileNames := make(map[string]bool) for _, profile := range profiles { diff --git a/pkg/api/apiloader.go b/pkg/api/apiloader.go index f23acd9920..9745b22267 100644 --- a/pkg/api/apiloader.go +++ b/pkg/api/apiloader.go @@ -6,6 +6,8 @@ import ( "io/ioutil" "reflect" + "fmt" + "github.com/Azure/acs-engine/pkg/api/agentPoolOnlyApi/v20170831" "github.com/Azure/acs-engine/pkg/api/agentPoolOnlyApi/v20180331" apvlabs "github.com/Azure/acs-engine/pkg/api/agentPoolOnlyApi/vlabs" @@ -82,6 +84,9 @@ func (a *Apiloader) LoadContainerService( } } setContainerServiceDefaultsv20160930(containerService) + if containerService.Properties == nil { + return nil, fmt.Errorf("missing ContainerService Properties") + } if e := containerService.Properties.Validate(); validate && e != nil { return nil, e } @@ -102,6 +107,9 @@ func (a *Apiloader) LoadContainerService( } } setContainerServiceDefaultsv20160330(containerService) + if containerService.Properties == nil { + return nil, fmt.Errorf("missing ContainerService Properties") + } if e := containerService.Properties.Validate(); validate && e != nil { return nil, e } @@ -123,6 +131,9 @@ func (a *Apiloader) LoadContainerService( } } setContainerServiceDefaultsv20170131(containerService) + if containerService.Properties == nil { + return nil, fmt.Errorf("missing ContainerService Properties") + } if e := containerService.Properties.Validate(); validate && e != nil { return nil, e } @@ -143,6 +154,9 @@ func (a *Apiloader) LoadContainerService( return nil, e } } + if containerService.Properties == nil { + return nil, fmt.Errorf("missing ContainerService Properties") + } if e := containerService.Properties.Validate(isUpdate); validate && e != nil { return nil, e } @@ -168,6 +182,9 @@ func (a *Apiloader) LoadContainerService( return nil, e } } + if containerService.Properties == nil { + return nil, fmt.Errorf("missing ContainerService Properties") + } if e := containerService.Properties.Validate(isUpdate); validate && e != nil { return nil, e } @@ -200,10 +217,21 @@ func (a *Apiloader) LoadContainerServiceForAgentPoolOnlyCluster( if e := json.Unmarshal(contents, &managedCluster); e != nil { return nil, IsSSHAutoGenerated, e } - // verify orchestrator version - if len(managedCluster.Properties.KubernetesVersion) > 0 && !common.AllKubernetesSupportedVersions[managedCluster.Properties.KubernetesVersion] { - return nil, IsSSHAutoGenerated, a.Translator.Errorf("The selected orchestrator version '%s' is not supported", managedCluster.Properties.KubernetesVersion) + // verify managedCluster.Properties is not nil for creating case + if managedCluster.Properties == nil { + if !isUpdate { + return nil, false, a.Translator.Errorf("properties object in managed cluster should not be nil") + } + managedCluster.Properties = &v20170831.Properties{} } + + if hasExistingCS { + vemc := ConvertContainerServiceToV20170831AgentPoolOnly(existingContainerService) + if e := managedCluster.Merge(vemc); e != nil { + return nil, IsSSHAutoGenerated, e + } + } + // use defaultKubernetesVersion arg if no version was supplied in the request contents if managedCluster.Properties.KubernetesVersion == "" && defaultKubernetesVersion != "" { if !common.AllKubernetesSupportedVersions[defaultKubernetesVersion] { @@ -211,19 +239,37 @@ func (a *Apiloader) LoadContainerServiceForAgentPoolOnlyCluster( } managedCluster.Properties.KubernetesVersion = defaultKubernetesVersion } + + // verify orchestrator version + if len(managedCluster.Properties.KubernetesVersion) > 0 && !common.AllKubernetesSupportedVersions[managedCluster.Properties.KubernetesVersion] { + return nil, IsSSHAutoGenerated, a.Translator.Errorf("The selected orchestrator version '%s' is not supported", managedCluster.Properties.KubernetesVersion) + } + if e := managedCluster.Properties.Validate(); validate && e != nil { return nil, IsSSHAutoGenerated, e } + return ConvertV20170831AgentPoolOnly(managedCluster), false, nil case v20180331.APIVersion: managedCluster := &v20180331.ManagedCluster{} if e := json.Unmarshal(contents, &managedCluster); e != nil { return nil, IsSSHAutoGenerated, e } - // verify orchestrator version - if len(managedCluster.Properties.KubernetesVersion) > 0 && !common.AllKubernetesSupportedVersions[managedCluster.Properties.KubernetesVersion] { - return nil, IsSSHAutoGenerated, a.Translator.Errorf("The selected orchestrator version '%s' is not supported", managedCluster.Properties.KubernetesVersion) + // verify managedCluster.Properties is not nil for creating case + if managedCluster.Properties == nil { + if !isUpdate { + return nil, false, a.Translator.Errorf("properties object in managed cluster should not be nil") + } + managedCluster.Properties = &v20180331.Properties{} + } + + if hasExistingCS { + vemc := ConvertContainerServiceToV20180331AgentPoolOnly(existingContainerService) + if e := managedCluster.Merge(vemc); e != nil { + return nil, IsSSHAutoGenerated, e + } } + // use defaultKubernetesVersion arg if no version was supplied in the request contents if managedCluster.Properties.KubernetesVersion == "" && defaultKubernetesVersion != "" { if !common.AllKubernetesSupportedVersions[defaultKubernetesVersion] { @@ -236,12 +282,11 @@ func (a *Apiloader) LoadContainerServiceForAgentPoolOnlyCluster( } } - if hasExistingCS { - vemc := ConvertContainerServiceToV20180331AgentPoolOnly(existingContainerService) - if e := managedCluster.Merge(vemc); e != nil { - return nil, IsSSHAutoGenerated, e - } + // verify orchestrator version + if len(managedCluster.Properties.KubernetesVersion) > 0 && !common.AllKubernetesSupportedVersions[managedCluster.Properties.KubernetesVersion] { + return nil, IsSSHAutoGenerated, a.Translator.Errorf("The selected orchestrator version '%s' is not supported", managedCluster.Properties.KubernetesVersion) } + if e := managedCluster.Properties.Validate(); validate && e != nil { return nil, IsSSHAutoGenerated, e } diff --git a/pkg/api/apiloader_test.go b/pkg/api/apiloader_test.go index 2c7c45f6de..0c22053f75 100644 --- a/pkg/api/apiloader_test.go +++ b/pkg/api/apiloader_test.go @@ -1,10 +1,18 @@ package api import ( + "encoding/json" + + "github.com/Azure/acs-engine/pkg/api/agentPoolOnlyApi/v20170831" + "github.com/Azure/acs-engine/pkg/api/agentPoolOnlyApi/v20180331" "github.com/Azure/acs-engine/pkg/api/common" "github.com/Azure/acs-engine/pkg/i18n" "github.com/leonelquinteros/gotext" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "io/ioutil" + "os" "path" "testing" ) @@ -84,4 +92,151 @@ func TestLoadContainerServiceFromFile(t *testing.T) { if containerService.Properties.OrchestratorProfile.OrchestratorVersion != common.GetDefaultKubernetesVersionWindows() { t.Errorf("Failed to set orcherstator version to windows default when it is not set in the json API v20170131, got %s but expected %s", containerService.Properties.OrchestratorProfile.OrchestratorVersion, common.GetDefaultKubernetesVersionWindows()) } + +} + +func TestLoadContainerServiceForAgentPoolOnlyCluster(t *testing.T) { + var _ = Describe("create/update cluster operations", func() { + locale := gotext.NewLocale(path.Join("../../..", "../../..", "translations"), "en_US") + i18n.Initialize(locale) + apiloader := &Apiloader{ + Translator: &i18n.Translator{ + Locale: locale, + }, + } + k8sVersions := common.GetAllSupportedKubernetesVersions() + defaultK8sVersion := common.GetDefaultKubernetesVersion() + + Context("v20180331", func() { + It("it should return error if managed cluster body is empty", func() { + + model := v20180331.ManagedCluster{} + + modelString, _ := json.Marshal(model) + _, _, err := apiloader.LoadContainerServiceForAgentPoolOnlyCluster([]byte(modelString), "2018-03-31", false, false, defaultK8sVersion, nil) + Expect(err).NotTo(BeNil()) + }) + + It("it should merge if managed cluster body is empty and trying to update", func() { + model := v20180331.ManagedCluster{ + Name: "myaks", + Properties: &v20180331.Properties{ + DNSPrefix: "myaks", + KubernetesVersion: k8sVersions[0], + AgentPoolProfiles: []*v20180331.AgentPoolProfile{ + { + Name: "agentpool1", + Count: 3, + VMSize: "Standard_DS2_v2", + OSDiskSizeGB: 0, + StorageProfile: "ManagedDisk", + }, + }, + ServicePrincipalProfile: &v20180331.ServicePrincipalProfile{ + ClientID: "clientID", + Secret: "clientSecret", + }, + }, + } + modelString, _ := json.Marshal(model) + cs, sshAutoGenerated, err := apiloader.LoadContainerServiceForAgentPoolOnlyCluster([]byte(modelString), "2018-03-31", false, false, defaultK8sVersion, nil) + Expect(err).To(BeNil()) + Expect(sshAutoGenerated).To(BeFalse()) + + model2 := v20180331.ManagedCluster{} + modelString2, _ := json.Marshal(model2) + cs2, sshAutoGenerated, err := apiloader.LoadContainerServiceForAgentPoolOnlyCluster([]byte(modelString2), "2018-03-31", false, true, defaultK8sVersion, cs) + + Expect(err).To(BeNil()) + // ssh key should not be re-generated + Expect(sshAutoGenerated).To(BeFalse()) + Expect(cs2.Properties.AgentPoolProfiles).NotTo(BeNil()) + Expect(cs2.Properties.LinuxProfile).NotTo(BeNil()) + Expect(cs2.Properties.WindowsProfile).NotTo(BeNil()) + Expect(cs2.Properties.ServicePrincipalProfile).NotTo(BeNil()) + Expect(cs2.Properties.HostedMasterProfile).NotTo(BeNil()) + Expect(cs2.Properties.HostedMasterProfile.DNSPrefix).To(Equal(model.Properties.DNSPrefix)) + Expect(cs2.Properties.OrchestratorProfile.OrchestratorVersion).To(Equal(k8sVersions[0])) + }) + }) + + Context("20170831", func() { + It("it should return error if managed cluster body is empty", func() { + + model := v20170831.ManagedCluster{} + + modelString, _ := json.Marshal(model) + _, _, err := apiloader.LoadContainerServiceForAgentPoolOnlyCluster([]byte(modelString), "2018-03-31", false, false, defaultK8sVersion, nil) + Expect(err).NotTo(BeNil()) + }) + + It("it should merge if managed cluster body is empty and trying to update", func() { + model := v20170831.ManagedCluster{ + Name: "myaks", + Properties: &v20170831.Properties{ + DNSPrefix: "myaks", + KubernetesVersion: k8sVersions[0], + AgentPoolProfiles: []*v20170831.AgentPoolProfile{ + { + Name: "agentpool1", + Count: 3, + VMSize: "Standard_DS2_v2", + OSDiskSizeGB: 0, + StorageProfile: "ManagedDisk", + }, + }, + ServicePrincipalProfile: &v20170831.ServicePrincipalProfile{ + ClientID: "clientID", + Secret: "clientSecret", + }, + }, + } + modelString, _ := json.Marshal(model) + cs, sshAutoGenerated, err := apiloader.LoadContainerServiceForAgentPoolOnlyCluster([]byte(modelString), "2018-03-31", false, false, defaultK8sVersion, nil) + Expect(err).To(BeNil()) + Expect(sshAutoGenerated).To(BeFalse()) + + model2 := v20170831.ManagedCluster{} + modelString2, _ := json.Marshal(model2) + cs2, sshAutoGenerated, err := apiloader.LoadContainerServiceForAgentPoolOnlyCluster([]byte(modelString2), "2018-03-31", false, true, defaultK8sVersion, cs) + + Expect(err).To(BeNil()) + // ssh key should not be re-generated + Expect(sshAutoGenerated).To(BeFalse()) + Expect(cs2.Properties.AgentPoolProfiles).NotTo(BeNil()) + Expect(cs2.Properties.LinuxProfile).NotTo(BeNil()) + Expect(cs2.Properties.WindowsProfile).NotTo(BeNil()) + Expect(cs2.Properties.ServicePrincipalProfile).NotTo(BeNil()) + Expect(cs2.Properties.HostedMasterProfile).NotTo(BeNil()) + Expect(cs2.Properties.HostedMasterProfile.DNSPrefix).To(Equal(model.Properties.DNSPrefix)) + }) + }) + }) +} + +func TestLoadContainerServiceWithNilProperties(t *testing.T) { + jsonWithoutProperties := `{ + "type": "Microsoft.ContainerService/managedClusters", + "name": "[parameters('clusterName')]", + "apiVersion": "2017-07-01", + "location": "[resourceGroup().location]" + }` + + tmpFile, err := ioutil.TempFile("", "containerService-invalid") + fileName := tmpFile.Name() + defer os.Remove(fileName) + + err = ioutil.WriteFile(fileName, []byte(jsonWithoutProperties), os.ModeAppend) + + apiloader := &Apiloader{} + existingContainerService := &ContainerService{Name: "test", + Properties: &Properties{OrchestratorProfile: &OrchestratorProfile{OrchestratorType: Kubernetes, OrchestratorVersion: "1.6.9"}}} + _, _, err = apiloader.LoadContainerServiceFromFile(fileName, true, false, existingContainerService) + if err == nil { + t.Errorf("Expected error to be thrown") + } + expectedMsg := "missing ContainerService Properties" + if err.Error() != expectedMsg { + t.Errorf("Expected error with message %s but got %s", expectedMsg, err.Error()) + } } diff --git a/pkg/api/common/helper.go b/pkg/api/common/helper.go index 1c1ede318a..0c1182a75c 100644 --- a/pkg/api/common/helper.go +++ b/pkg/api/common/helper.go @@ -2,6 +2,7 @@ package common import ( "fmt" + "regexp" "strings" validator "gopkg.in/go-playground/validator.v9" @@ -51,3 +52,16 @@ func HandleValidationErrors(e validator.ValidationErrors) error { } return fmt.Errorf("Namespace %s is not caught, %+v", ns, e) } + +// ValidateDNSPrefix is a helper function to check that a DNS Prefix is valid +func ValidateDNSPrefix(dnsName string) error { + dnsNameRegex := `^([A-Za-z][A-Za-z0-9-]{1,43}[A-Za-z0-9])$` + re, err := regexp.Compile(dnsNameRegex) + if err != nil { + return err + } + if !re.MatchString(dnsName) { + return fmt.Errorf("DNSPrefix '%s' is invalid. The DNSPrefix must contain between 3 and 45 characters and can contain only letters, numbers, and hyphens. It must start with a letter and must end with a letter or a number. (length was %d)", dnsName, len(dnsName)) + } + return nil +} diff --git a/pkg/api/common/helper_test.go b/pkg/api/common/helper_test.go new file mode 100644 index 0000000000..748d409a34 --- /dev/null +++ b/pkg/api/common/helper_test.go @@ -0,0 +1,57 @@ +package common + +import ( + "fmt" + "testing" +) + +func TestValidateDNSPrefix(t *testing.T) { + cases := []struct { + dnsPrefix string + expectedErr error + }{ + { + "validDnsPrefix", + nil, + }, + { + "", + fmt.Errorf("DNSPrefix '' is invalid. The DNSPrefix must contain between 3 and 45 characters and can contain only letters, numbers, and hyphens. It must start with a letter and must end with a letter or a number. (length was 0)"), + }, + { + "a", + fmt.Errorf("DNSPrefix 'a' is invalid. The DNSPrefix must contain between 3 and 45 characters and can contain only letters, numbers, and hyphens. It must start with a letter and must end with a letter or a number. (length was 1)"), + }, + { + "1234", + fmt.Errorf("DNSPrefix '1234' is invalid. The DNSPrefix must contain between 3 and 45 characters and can contain only letters, numbers, and hyphens. It must start with a letter and must end with a letter or a number. (length was 4)"), + }, + { + "verylongdnsprefixthatismorethan45characterslong", + fmt.Errorf("DNSPrefix 'verylongdnsprefixthatismorethan45characterslong' is invalid. The DNSPrefix must contain between 3 and 45 characters and can contain only letters, numbers, and hyphens. It must start with a letter and must end with a letter or a number. (length was 47)"), + }, + { + "dnswith_special?char", + fmt.Errorf("DNSPrefix 'dnswith_special?char' is invalid. The DNSPrefix must contain between 3 and 45 characters and can contain only letters, numbers, and hyphens. It must start with a letter and must end with a letter or a number. (length was 20)"), + }, + { + "myDNS-1234", + nil, + }, + } + + for _, c := range cases { + err := ValidateDNSPrefix(c.dnsPrefix) + if err != nil && c.expectedErr != nil { + if err.Error() != c.expectedErr.Error() { + t.Fatalf("expected validateDNSPrefix to return error %s, but instead got %s", c.expectedErr.Error(), err.Error()) + } + } else { + if c.expectedErr != nil { + t.Fatalf("expected validateDNSPrefix to return error %s, but instead got no error", c.expectedErr.Error()) + } else if err != nil { + t.Fatalf("expected validateDNSPrefix to return no error, but instead got %s", err.Error()) + } + } + } +} diff --git a/pkg/api/common/versions.go b/pkg/api/common/versions.go index 37893409b0..224d221018 100644 --- a/pkg/api/common/versions.go +++ b/pkg/api/common/versions.go @@ -40,6 +40,7 @@ var AllKubernetesSupportedVersions = map[string]bool{ "1.8.11": true, "1.8.12": true, "1.8.13": true, + "1.8.14": true, "1.9.0": true, "1.9.1": true, "1.9.2": true, @@ -215,6 +216,7 @@ func getAllKubernetesWindowsSupportedVersionsMap() map[string]bool { "1.7.0", "1.7.1", "1.8.13", + "1.8.14", "1.10.0-beta.2", "1.10.0-beta.4", "1.10.0-rc.1", diff --git a/pkg/api/types.go b/pkg/api/types.go index 47ce7d8b6c..5fc5793b40 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -81,7 +81,7 @@ type AzProfile struct { // ServicePrincipalProfile contains the client and secret used by the cluster for Azure Resource CRUD type ServicePrincipalProfile struct { ClientID string `json:"clientId"` - Secret string `json:"secret,omitempty"` + Secret string `json:"secret,omitempty" conform:"redact"` ObjectID string `json:"objectId,omitempty"` KeyvaultSecretRef *KeyvaultSecretRef `json:"keyvaultSecretRef,omitempty"` } @@ -97,33 +97,33 @@ type KeyvaultSecretRef struct { // CertificateProfile represents the definition of the master cluster type CertificateProfile struct { // CaCertificate is the certificate authority certificate. - CaCertificate string `json:"caCertificate,omitempty"` + CaCertificate string `json:"caCertificate,omitempty" conform:"redact"` // CaPrivateKey is the certificate authority key. - CaPrivateKey string `json:"caPrivateKey,omitempty"` + CaPrivateKey string `json:"caPrivateKey,omitempty" conform:"redact"` // ApiServerCertificate is the rest api server certificate, and signed by the CA - APIServerCertificate string `json:"apiServerCertificate,omitempty"` + APIServerCertificate string `json:"apiServerCertificate,omitempty" conform:"redact"` // ApiServerPrivateKey is the rest api server private key, and signed by the CA - APIServerPrivateKey string `json:"apiServerPrivateKey,omitempty"` + APIServerPrivateKey string `json:"apiServerPrivateKey,omitempty" conform:"redact"` // ClientCertificate is the certificate used by the client kubelet services and signed by the CA - ClientCertificate string `json:"clientCertificate,omitempty"` + ClientCertificate string `json:"clientCertificate,omitempty" conform:"redact"` // ClientPrivateKey is the private key used by the client kubelet services and signed by the CA - ClientPrivateKey string `json:"clientPrivateKey,omitempty"` + ClientPrivateKey string `json:"clientPrivateKey,omitempty" conform:"redact"` // KubeConfigCertificate is the client certificate used for kubectl cli and signed by the CA - KubeConfigCertificate string `json:"kubeConfigCertificate,omitempty"` + KubeConfigCertificate string `json:"kubeConfigCertificate,omitempty" conform:"redact"` // KubeConfigPrivateKey is the client private key used for kubectl cli and signed by the CA - KubeConfigPrivateKey string `json:"kubeConfigPrivateKey,omitempty"` + KubeConfigPrivateKey string `json:"kubeConfigPrivateKey,omitempty" conform:"redact"` // EtcdServerCertificate is the server certificate for etcd, and signed by the CA - EtcdServerCertificate string `json:"etcdServerCertificate,omitempty"` + EtcdServerCertificate string `json:"etcdServerCertificate,omitempty" conform:"redact"` // EtcdServerPrivateKey is the server private key for etcd, and signed by the CA - EtcdServerPrivateKey string `json:"etcdServerPrivateKey,omitempty"` + EtcdServerPrivateKey string `json:"etcdServerPrivateKey,omitempty" conform:"redact"` // EtcdClientCertificate is etcd client certificate, and signed by the CA - EtcdClientCertificate string `json:"etcdClientCertificate,omitempty"` + EtcdClientCertificate string `json:"etcdClientCertificate,omitempty" conform:"redact"` // EtcdClientPrivateKey is the etcd client private key, and signed by the CA - EtcdClientPrivateKey string `json:"etcdClientPrivateKey,omitempty"` + EtcdClientPrivateKey string `json:"etcdClientPrivateKey,omitempty" conform:"redact"` // EtcdPeerCertificates is list of etcd peer certificates, and signed by the CA - EtcdPeerCertificates []string `json:"etcdPeerCertificates,omitempty"` + EtcdPeerCertificates []string `json:"etcdPeerCertificates,omitempty" conform:"redact"` // EtcdPeerPrivateKeys is list of etcd peer private keys, and signed by the CA - EtcdPeerPrivateKeys []string `json:"etcdPeerPrivateKeys,omitempty"` + EtcdPeerPrivateKeys []string `json:"etcdPeerPrivateKeys,omitempty" conform:"redact"` } // LinuxProfile represents the linux parameters passed to the cluster @@ -159,7 +159,7 @@ type CustomNodesDNS struct { // WindowsProfile represents the windows parameters passed to the cluster type WindowsProfile struct { AdminUsername string `json:"adminUsername"` - AdminPassword string `json:"adminPassword"` + AdminPassword string `json:"adminPassword" conform:"redact"` ImageVersion string `json:"imageVersion"` WindowsImageSourceURL string `json:"windowsImageSourceURL"` WindowsPublisher string `json:"windowsPublisher"` @@ -443,13 +443,12 @@ type AgentPoolProfile struct { Distro Distro `json:"distro,omitempty"` Role AgentPoolProfileRole `json:"role,omitempty"` AcceleratedNetworkingEnabled bool `json:"acceleratedNetworkingEnabled,omitempty"` - - FQDN string `json:"fqdn,omitempty"` - CustomNodeLabels map[string]string `json:"customNodeLabels,omitempty"` - PreprovisionExtension *Extension `json:"preProvisionExtension"` - Extensions []Extension `json:"extensions"` - KubernetesConfig *KubernetesConfig `json:"kubernetesConfig,omitempty"` - ImageRef *ImageReference `json:"imageReference,omitempty"` + FQDN string `json:"fqdn,omitempty"` + CustomNodeLabels map[string]string `json:"customNodeLabels,omitempty"` + PreprovisionExtension *Extension `json:"preProvisionExtension"` + Extensions []Extension `json:"extensions"` + KubernetesConfig *KubernetesConfig `json:"kubernetesConfig,omitempty"` + ImageRef *ImageReference `json:"imageReference,omitempty"` } // AgentPoolProfileRole represents an agent role @@ -547,7 +546,7 @@ type AADProfile struct { // The server AAD application ID. ServerAppID string `json:"serverAppID,omitempty"` // The server AAD application secret - ServerAppSecret string `json:"serverAppSecret,omitempty"` + ServerAppSecret string `json:"serverAppSecret,omitempty" conform:"redact"` // The AAD tenant ID to use for authentication. // If not specified, will use the tenant of the deployment subscription. // Optional @@ -556,7 +555,7 @@ type AADProfile struct { // cluster-admin RBAC role. // Optional AdminGroupID string `json:"adminGroupID,omitempty"` - // The authenticator to use, either "OIDC" or "Webhook". + // The authenticator to use, either "oidc" or "webhook". Authenticator AuthenticatorType `json:"authenticator"` } @@ -642,7 +641,7 @@ func (p *Properties) HasManagedDisks() bool { return true } } - if p.OrchestratorProfile.KubernetesConfig.PrivateJumpboxProvision() && p.OrchestratorProfile.KubernetesConfig.PrivateCluster.JumpboxProfile.StorageProfile == ManagedDisks { + if p.OrchestratorProfile != nil && p.OrchestratorProfile.KubernetesConfig != nil && p.OrchestratorProfile.KubernetesConfig.PrivateJumpboxProvision() && p.OrchestratorProfile.KubernetesConfig.PrivateCluster.JumpboxProfile.StorageProfile == ManagedDisks { return true } return false @@ -650,7 +649,7 @@ func (p *Properties) HasManagedDisks() bool { // HasStorageAccountDisks returns true if the cluster contains Storage Account Disks func (p *Properties) HasStorageAccountDisks() bool { - if p.OrchestratorProfile.OrchestratorType == OpenShift { + if p.OrchestratorProfile != nil && p.OrchestratorProfile.OrchestratorType == OpenShift { return true } if p.MasterProfile != nil && p.MasterProfile.StorageProfile == StorageAccount { @@ -661,7 +660,7 @@ func (p *Properties) HasStorageAccountDisks() bool { return true } } - if p.OrchestratorProfile.KubernetesConfig.PrivateJumpboxProvision() && p.OrchestratorProfile.KubernetesConfig.PrivateCluster.JumpboxProfile.StorageProfile == StorageAccount { + if p.OrchestratorProfile != nil && p.OrchestratorProfile.KubernetesConfig != nil && p.OrchestratorProfile.KubernetesConfig.PrivateJumpboxProvision() && p.OrchestratorProfile.KubernetesConfig.PrivateCluster.JumpboxProfile.StorageProfile == StorageAccount { return true } return false @@ -769,11 +768,6 @@ func (a *AgentPoolProfile) HasDisks() bool { return len(a.DiskSizesGB) > 0 } -// IsAcceleratedNetworkingEnabled returns true if the customer enabled Accelerated Networking -func (a *AgentPoolProfile) IsAcceleratedNetworkingEnabled() bool { - return a.AcceleratedNetworkingEnabled -} - // HasSecrets returns true if the customer specified secrets to install func (w *WindowsProfile) HasSecrets() bool { return len(w.Secrets) > 0 diff --git a/pkg/api/types_test.go b/pkg/api/types_test.go index fe2df48613..6a6d346587 100644 --- a/pkg/api/types_test.go +++ b/pkg/api/types_test.go @@ -3,6 +3,8 @@ package api import ( "log" "testing" + + "github.com/Azure/acs-engine/pkg/helpers" ) const exampleCustomHyperkubeImage = `example.azurecr.io/example/hyperkube-amd64:custom` @@ -26,6 +28,505 @@ const exampleAPIModel = `{ } ` +func TestOSType(t *testing.T) { + p := Properties{ + MasterProfile: &MasterProfile{ + Distro: RHEL, + }, + AgentPoolProfiles: []*AgentPoolProfile{ + { + OSType: Linux, + }, + { + OSType: Linux, + Distro: RHEL, + }, + }, + } + + if p.HasWindows() { + t.Fatalf("expected HasWindows() to return false but instead returned true") + } + if p.AgentPoolProfiles[0].IsWindows() { + t.Fatalf("expected IsWindows() to return false but instead returned true") + } + + if !p.AgentPoolProfiles[0].IsLinux() { + t.Fatalf("expected IsLinux() to return true but instead returned false") + } + + if p.AgentPoolProfiles[0].IsRHEL() { + t.Fatalf("expected IsRHEL() to return false but instead returned true") + } + + if p.AgentPoolProfiles[0].IsCoreOS() { + t.Fatalf("expected IsCoreOS() to return false but instead returned true") + } + + if !p.AgentPoolProfiles[1].IsRHEL() { + t.Fatalf("expected IsRHEL() to return true but instead returned false") + } + + if p.AgentPoolProfiles[1].IsCoreOS() { + t.Fatalf("expected IsCoreOS() to return false but instead returned true") + } + + if !p.MasterProfile.IsRHEL() { + t.Fatalf("expected IsRHEL() to return true but instead returned false") + } + + if p.MasterProfile.IsCoreOS() { + t.Fatalf("expected IsCoreOS() to return false but instead returned true") + } + + p.MasterProfile.Distro = CoreOS + p.AgentPoolProfiles[0].OSType = Windows + p.AgentPoolProfiles[1].Distro = CoreOS + + if !p.HasWindows() { + t.Fatalf("expected HasWindows() to return true but instead returned false") + } + + if !p.AgentPoolProfiles[0].IsWindows() { + t.Fatalf("expected IsWindows() to return true but instead returned false") + } + + if p.AgentPoolProfiles[0].IsLinux() { + t.Fatalf("expected IsLinux() to return false but instead returned true") + } + + if p.AgentPoolProfiles[0].IsRHEL() { + t.Fatalf("expected IsRHEL() to return false but instead returned true") + } + + if p.AgentPoolProfiles[0].IsCoreOS() { + t.Fatalf("expected IsCoreOS() to return false but instead returned true") + } + + if p.AgentPoolProfiles[1].IsRHEL() { + t.Fatalf("expected IsRHEL() to return false but instead returned true") + } + + if !p.AgentPoolProfiles[1].IsCoreOS() { + t.Fatalf("expected IsCoreOS() to return true but instead returned false") + } + + if p.MasterProfile.IsRHEL() { + t.Fatalf("expected IsRHEL() to return false but instead returned true") + } + + if !p.MasterProfile.IsCoreOS() { + t.Fatalf("expected IsCoreOS() to return true but instead returned false") + } +} + +func TestHasStorageProfile(t *testing.T) { + cases := []struct { + p Properties + expectedHasMD bool + expectedHasSA bool + expectedMasterMD bool + expectedAgent0MD bool + expectedPrivateJB bool + expectedHasDisks bool + }{ + { + p: Properties{ + MasterProfile: &MasterProfile{ + StorageProfile: StorageAccount, + }, + AgentPoolProfiles: []*AgentPoolProfile{ + { + StorageProfile: StorageAccount, + DiskSizesGB: []int{5}, + }, + { + StorageProfile: StorageAccount, + }, + }, + }, + expectedHasMD: false, + expectedHasSA: true, + expectedMasterMD: false, + expectedAgent0MD: false, + expectedHasDisks: true, + }, + { + p: Properties{ + MasterProfile: &MasterProfile{ + StorageProfile: ManagedDisks, + }, + AgentPoolProfiles: []*AgentPoolProfile{ + { + StorageProfile: StorageAccount, + }, + { + StorageProfile: StorageAccount, + }, + }, + }, + expectedHasMD: true, + expectedHasSA: true, + expectedMasterMD: true, + expectedAgent0MD: false, + }, + { + p: Properties{ + MasterProfile: &MasterProfile{ + StorageProfile: StorageAccount, + }, + AgentPoolProfiles: []*AgentPoolProfile{ + { + StorageProfile: ManagedDisks, + }, + { + StorageProfile: StorageAccount, + }, + }, + }, + expectedHasMD: true, + expectedHasSA: true, + expectedMasterMD: false, + expectedAgent0MD: true, + }, + { + p: Properties{ + OrchestratorProfile: &OrchestratorProfile{ + OrchestratorType: Kubernetes, + }, + MasterProfile: &MasterProfile{ + StorageProfile: ManagedDisks, + }, + AgentPoolProfiles: []*AgentPoolProfile{ + { + StorageProfile: ManagedDisks, + }, + { + StorageProfile: ManagedDisks, + }, + }, + }, + expectedHasMD: true, + expectedHasSA: false, + expectedMasterMD: true, + expectedAgent0MD: true, + expectedPrivateJB: false, + }, + { + p: Properties{ + OrchestratorProfile: &OrchestratorProfile{ + OrchestratorType: Kubernetes, + KubernetesConfig: &KubernetesConfig{ + PrivateCluster: &PrivateCluster{ + Enabled: helpers.PointerToBool(true), + JumpboxProfile: &PrivateJumpboxProfile{ + StorageProfile: ManagedDisks, + }, + }, + }, + }, + MasterProfile: &MasterProfile{ + StorageProfile: StorageAccount, + }, + AgentPoolProfiles: []*AgentPoolProfile{ + { + StorageProfile: StorageAccount, + }, + }, + }, + expectedHasMD: true, + expectedHasSA: true, + expectedMasterMD: false, + expectedAgent0MD: false, + expectedPrivateJB: true, + }, + + { + p: Properties{ + OrchestratorProfile: &OrchestratorProfile{ + OrchestratorType: Kubernetes, + KubernetesConfig: &KubernetesConfig{ + PrivateCluster: &PrivateCluster{ + Enabled: helpers.PointerToBool(true), + JumpboxProfile: &PrivateJumpboxProfile{ + StorageProfile: StorageAccount, + }, + }, + }, + }, + MasterProfile: &MasterProfile{ + StorageProfile: ManagedDisks, + }, + AgentPoolProfiles: []*AgentPoolProfile{ + { + StorageProfile: ManagedDisks, + }, + }, + }, + expectedHasMD: true, + expectedHasSA: true, + expectedMasterMD: true, + expectedAgent0MD: true, + expectedPrivateJB: true, + }, + } + + for _, c := range cases { + if c.p.HasManagedDisks() != c.expectedHasMD { + t.Fatalf("expected HasManagedDisks() to return %t but instead returned %t", c.expectedHasMD, c.p.HasManagedDisks()) + } + if c.p.HasStorageAccountDisks() != c.expectedHasSA { + t.Fatalf("expected HasStorageAccountDisks() to return %t but instead returned %t", c.expectedHasSA, c.p.HasStorageAccountDisks()) + } + if c.p.MasterProfile.IsManagedDisks() != c.expectedMasterMD { + t.Fatalf("expected IsManagedDisks() to return %t but instead returned %t", c.expectedMasterMD, c.p.MasterProfile.IsManagedDisks()) + } + if c.p.MasterProfile.IsStorageAccount() == c.expectedMasterMD { + t.Fatalf("expected IsStorageAccount() to return %t but instead returned %t", !c.expectedMasterMD, c.p.MasterProfile.IsStorageAccount()) + } + if c.p.AgentPoolProfiles[0].IsManagedDisks() != c.expectedAgent0MD { + t.Fatalf("expected IsManagedDisks() to return %t but instead returned %t", c.expectedAgent0MD, c.p.AgentPoolProfiles[0].IsManagedDisks()) + } + if c.p.AgentPoolProfiles[0].IsStorageAccount() == c.expectedAgent0MD { + t.Fatalf("expected IsStorageAccount() to return %t but instead returned %t", !c.expectedAgent0MD, c.p.AgentPoolProfiles[0].IsStorageAccount()) + } + if c.p.OrchestratorProfile != nil && c.p.OrchestratorProfile.KubernetesConfig.PrivateJumpboxProvision() != c.expectedPrivateJB { + t.Fatalf("expected PrivateJumpboxProvision() to return %t but instead returned %t", c.expectedPrivateJB, c.p.OrchestratorProfile.KubernetesConfig.PrivateJumpboxProvision()) + } + if c.p.AgentPoolProfiles[0].HasDisks() != c.expectedHasDisks { + t.Fatalf("expected HasDisks() to return %t but instead returned %t", c.expectedHasDisks, c.p.AgentPoolProfiles[0].HasDisks()) + } + } +} + +func TestTotalNodes(t *testing.T) { + cases := []struct { + p Properties + expected int + }{ + { + p: Properties{ + MasterProfile: &MasterProfile{ + Count: 1, + }, + AgentPoolProfiles: []*AgentPoolProfile{ + { + Count: 1, + }, + }, + }, + expected: 2, + }, + { + p: Properties{ + AgentPoolProfiles: []*AgentPoolProfile{ + { + Count: 3, + }, + { + Count: 4, + }, + }, + }, + expected: 7, + }, + { + p: Properties{ + MasterProfile: &MasterProfile{ + Count: 5, + }, + AgentPoolProfiles: []*AgentPoolProfile{ + { + Count: 6, + }, + }, + }, + expected: 11, + }, + } + + for _, c := range cases { + if c.p.TotalNodes() != c.expected { + t.Fatalf("expected TotalNodes() to return %d but instead returned %d", c.expected, c.p.TotalNodes()) + } + } +} + +func TestAvailabilityProfile(t *testing.T) { + cases := []struct { + p Properties + expectedHasVMSS bool + expectedISVMSS bool + expectedIsAS bool + expectedLowPri bool + }{ + { + p: Properties{ + AgentPoolProfiles: []*AgentPoolProfile{ + { + AvailabilityProfile: VirtualMachineScaleSets, + ScaleSetPriority: ScaleSetPriorityLow, + }, + }, + }, + expectedHasVMSS: true, + expectedISVMSS: true, + expectedIsAS: false, + expectedLowPri: true, + }, + { + p: Properties{ + AgentPoolProfiles: []*AgentPoolProfile{ + { + AvailabilityProfile: VirtualMachineScaleSets, + ScaleSetPriority: ScaleSetPriorityRegular, + }, + { + AvailabilityProfile: AvailabilitySet, + }, + }, + }, + expectedHasVMSS: true, + expectedISVMSS: true, + expectedIsAS: false, + expectedLowPri: false, + }, + { + p: Properties{ + AgentPoolProfiles: []*AgentPoolProfile{ + { + AvailabilityProfile: AvailabilitySet, + }, + }, + }, + expectedHasVMSS: false, + expectedISVMSS: false, + expectedIsAS: true, + expectedLowPri: false, + }, + } + + for _, c := range cases { + if c.p.HasVirtualMachineScaleSets() != c.expectedHasVMSS { + t.Fatalf("expected HasVirtualMachineScaleSets() to return %t but instead returned %t", c.expectedHasVMSS, c.p.HasVirtualMachineScaleSets()) + } + if c.p.AgentPoolProfiles[0].IsVirtualMachineScaleSets() != c.expectedISVMSS { + t.Fatalf("expected IsVirtualMachineScaleSets() to return %t but instead returned %t", c.expectedISVMSS, c.p.AgentPoolProfiles[0].IsVirtualMachineScaleSets()) + } + if c.p.AgentPoolProfiles[0].IsAvailabilitySets() != c.expectedIsAS { + t.Fatalf("expected HasVirtualMachineScaleSets() to return %t but instead returned %t", c.expectedIsAS, c.p.AgentPoolProfiles[0].IsAvailabilitySets()) + } + if c.p.AgentPoolProfiles[0].IsLowPriorityScaleSet() != c.expectedLowPri { + t.Fatalf("expected IsLowPriorityScaleSet() to return %t but instead returned %t", c.expectedLowPri, c.p.AgentPoolProfiles[0].IsLowPriorityScaleSet()) + } + } +} + +func TestIsCustomVNET(t *testing.T) { + cases := []struct { + p Properties + expectedMaster bool + expectedAgent bool + }{ + { + p: Properties{ + MasterProfile: &MasterProfile{ + VnetSubnetID: "testSubnet", + }, + AgentPoolProfiles: []*AgentPoolProfile{ + { + VnetSubnetID: "testSubnet", + }, + }, + }, + expectedMaster: true, + expectedAgent: true, + }, + { + p: Properties{ + MasterProfile: &MasterProfile{ + Count: 1, + }, + AgentPoolProfiles: []*AgentPoolProfile{ + { + Count: 1, + }, + { + Count: 1, + }, + }, + }, + expectedMaster: false, + expectedAgent: false, + }, + } + + for _, c := range cases { + if c.p.MasterProfile.IsCustomVNET() != c.expectedMaster { + t.Fatalf("expected IsCustomVnet() to return %t but instead returned %t", c.expectedMaster, c.p.MasterProfile.IsCustomVNET()) + } + if c.p.AgentPoolProfiles[0].IsCustomVNET() != c.expectedAgent { + t.Fatalf("expected IsCustomVnet() to return %t but instead returned %t", c.expectedAgent, c.p.AgentPoolProfiles[0].IsCustomVNET()) + } + } + +} + +func TestRequireRouteTable(t *testing.T) { + cases := []struct { + p Properties + expected bool + }{ + { + p: Properties{ + OrchestratorProfile: &OrchestratorProfile{ + OrchestratorType: DCOS, + }, + }, + expected: false, + }, + { + p: Properties{ + OrchestratorProfile: &OrchestratorProfile{ + OrchestratorType: Kubernetes, + KubernetesConfig: &KubernetesConfig{ + NetworkPolicy: "", + }, + }, + }, + expected: true, + }, + { + p: Properties{ + OrchestratorProfile: &OrchestratorProfile{ + OrchestratorType: Kubernetes, + KubernetesConfig: &KubernetesConfig{ + NetworkPlugin: "azure", + }, + }, + }, + expected: false, + }, + { + p: Properties{ + OrchestratorProfile: &OrchestratorProfile{ + OrchestratorType: Kubernetes, + KubernetesConfig: &KubernetesConfig{ + NetworkPolicy: "cilium", + }, + }, + }, + expected: false, + }, + } + + for _, c := range cases { + if c.p.OrchestratorProfile.RequireRouteTable() != c.expected { + t.Fatalf("expected RequireRouteTable() to return %t but instead got %t", c.expected, c.p.OrchestratorProfile.RequireRouteTable()) + } + } +} + func TestIsAzureCNI(t *testing.T) { k := &KubernetesConfig{ NetworkPlugin: "azure", @@ -55,19 +556,178 @@ func TestIsAzureCNI(t *testing.T) { } } -func TestIsDCOS(t *testing.T) { - dCOSProfile := &OrchestratorProfile{ - OrchestratorType: "DCOS", +func TestOrchestrator(t *testing.T) { + cases := []struct { + p Properties + expectedIsDCOS bool + expectedIsKubernetes bool + expectedIsOpenShift bool + expectedIsSwarmMode bool + }{ + { + p: Properties{ + OrchestratorProfile: &OrchestratorProfile{ + OrchestratorType: DCOS, + }, + }, + expectedIsDCOS: true, + expectedIsKubernetes: false, + expectedIsOpenShift: false, + expectedIsSwarmMode: false, + }, + { + p: Properties{ + OrchestratorProfile: &OrchestratorProfile{ + OrchestratorType: Kubernetes, + }, + }, + expectedIsDCOS: false, + expectedIsKubernetes: true, + expectedIsOpenShift: false, + expectedIsSwarmMode: false, + }, + { + p: Properties{ + OrchestratorProfile: &OrchestratorProfile{ + OrchestratorType: OpenShift, + }, + }, + expectedIsDCOS: false, + expectedIsKubernetes: false, + expectedIsOpenShift: true, + expectedIsSwarmMode: false, + }, + { + p: Properties{ + OrchestratorProfile: &OrchestratorProfile{ + OrchestratorType: SwarmMode, + }, + }, + expectedIsDCOS: false, + expectedIsKubernetes: false, + expectedIsOpenShift: false, + expectedIsSwarmMode: true, + }, + } + + for _, c := range cases { + if c.expectedIsDCOS != c.p.OrchestratorProfile.IsDCOS() { + t.Fatalf("Expected IsDCOS() to be %t with OrchestratorType=%s", c.expectedIsDCOS, c.p.OrchestratorProfile.OrchestratorType) + } + if c.expectedIsKubernetes != c.p.OrchestratorProfile.IsKubernetes() { + t.Fatalf("Expected IsKubernetes() to be %t with OrchestratorType=%s", c.expectedIsKubernetes, c.p.OrchestratorProfile.OrchestratorType) + } + if c.expectedIsOpenShift != c.p.OrchestratorProfile.IsOpenShift() { + t.Fatalf("Expected IsOpenShift() to be %t with OrchestratorType=%s", c.expectedIsOpenShift, c.p.OrchestratorProfile.OrchestratorType) + } + if c.expectedIsSwarmMode != c.p.OrchestratorProfile.IsSwarmMode() { + t.Fatalf("Expected IsSwarmMode() to be %t with OrchestratorType=%s", c.expectedIsSwarmMode, c.p.OrchestratorProfile.OrchestratorType) + } + if c.expectedIsOpenShift && !c.p.HasStorageAccountDisks() { + t.Fatalf("Expected HasStorageAccountDisks() to return true when OrchestratorType is OpenShift") + } + } +} + +func TestWindowsProfile(t *testing.T) { + w := WindowsProfile{} + + if w.HasSecrets() || w.HasCustomImage() { + t.Fatalf("Expected HasSecrets() and HasCustomImage() to return false when WindowsProfile is empty") + } + + w = WindowsProfile{ + Secrets: []KeyVaultSecrets{ + { + SourceVault: &KeyVaultID{"testVault"}, + VaultCertificates: []KeyVaultCertificate{ + { + CertificateURL: "testURL", + CertificateStore: "testStore", + }, + }, + }, + }, + WindowsImageSourceURL: "testCustomImage", + } + + if !(w.HasSecrets() && w.HasCustomImage()) { + t.Fatalf("Expected HasSecrets() and HasCustomImage() to return true") + } +} + +func TestLinuxProfile(t *testing.T) { + l := LinuxProfile{} + + if l.HasSecrets() || l.HasSearchDomain() || l.HasCustomNodesDNS() { + t.Fatalf("Expected HasSecrets(), HasSearchDomain() and HasCustomNodesDNS() to return false when LinuxProfile is empty") + } + + l = LinuxProfile{ + Secrets: []KeyVaultSecrets{ + { + SourceVault: &KeyVaultID{"testVault"}, + VaultCertificates: []KeyVaultCertificate{ + { + CertificateURL: "testURL", + CertificateStore: "testStore", + }, + }, + }, + }, + CustomNodesDNS: &CustomNodesDNS{ + DNSServer: "testDNSServer", + }, + CustomSearchDomain: &CustomSearchDomain{ + Name: "testName", + RealmPassword: "testRealmPassword", + RealmUser: "testRealmUser", + }, + } + + if !(l.HasSecrets() && l.HasSearchDomain() && l.HasCustomNodesDNS()) { + t.Fatalf("Expected HasSecrets(), HasSearchDomain() and HasCustomNodesDNS() to return true") + } +} + +func TestGetAPIServerEtcdAPIVersion(t *testing.T) { + o := OrchestratorProfile{} + + if o.GetAPIServerEtcdAPIVersion() != "" { + t.Fatalf("Expected GetAPIServerEtcdAPIVersion() to return \"\" but instead got %s", o.GetAPIServerEtcdAPIVersion()) + } + + o.KubernetesConfig = &KubernetesConfig{ + EtcdVersion: "3.2.1", + } + + if o.GetAPIServerEtcdAPIVersion() != "etcd3" { + t.Fatalf("Expected GetAPIServerEtcdAPIVersion() to return \"etcd3\" but instead got %s", o.GetAPIServerEtcdAPIVersion()) } - if !dCOSProfile.IsDCOS() { - t.Fatalf("unable to detect DCOS orchestrator profile from OrchestratorType=%s", dCOSProfile.OrchestratorType) + + // invalid version string + o.KubernetesConfig.EtcdVersion = "2.3.8" + if o.GetAPIServerEtcdAPIVersion() != "etcd2" { + t.Fatalf("Expected GetAPIServerEtcdAPIVersion() to return \"etcd2\" but instead got %s", o.GetAPIServerEtcdAPIVersion()) } - kubernetesProfile := &OrchestratorProfile{ - OrchestratorType: "Kubernetes", +} + +func TestHasAadProfile(t *testing.T) { + p := Properties{} + + if p.HasAadProfile() { + t.Fatalf("Expected HasAadProfile() to return false") } - if kubernetesProfile.IsDCOS() { - t.Fatalf("unexpectedly detected DCOS orchestrator profile from OrchestratorType=%s", kubernetesProfile.OrchestratorType) + + p.AADProfile = &AADProfile{ + ClientAppID: "test", + ServerAppID: "test", + } + + if !p.HasAadProfile() { + t.Fatalf("Expected HasAadProfile() to return true") } + } func TestCustomHyperkubeImageField(t *testing.T) { @@ -222,28 +882,31 @@ func TestIsNVIDIADevicePluginEnabled(t *testing.T) { }, }, } - enabled := p.IsNVIDIADevicePluginEnabled() - if enabled == isNSeriesSKU(&p) { - t.Fatalf("KubernetesConfig.IsNVIDIADevicePluginEnabled() should return false with N-series VMs with < k8s 1.10, instead returned %t", enabled) + + if !isNSeriesSKU(&p) { + t.Fatalf("isNSeriesSKU should return true when explicitly using VM Size %s", p.AgentPoolProfiles[0].VMSize) + } + if p.IsNVIDIADevicePluginEnabled() { + t.Fatalf("KubernetesConfig.IsNVIDIADevicePluginEnabled() should return false with N-series VMs with < k8s 1.10, instead returned %t", p.IsNVIDIADevicePluginEnabled()) } - o := p.OrchestratorProfile - o.OrchestratorVersion = "1.10.0" - enabled = p.IsNVIDIADevicePluginEnabled() - if enabled != isNSeriesSKU(&p) { - t.Fatalf("KubernetesConfig.IsNVIDIADevicePluginEnabled() should return %t with N-series VMs with k8s >= 1.10, instead returned %t", isNSeriesSKU(&p), enabled) + p.OrchestratorProfile.OrchestratorVersion = "1.10.0" + if !p.IsNVIDIADevicePluginEnabled() { + t.Fatalf("KubernetesConfig.IsNVIDIADevicePluginEnabled() should return true with N-series VMs with k8s >= 1.10, instead returned %t", p.IsNVIDIADevicePluginEnabled()) } - b := false - c := p.OrchestratorProfile.KubernetesConfig - c.Addons = []KubernetesAddon{ + p.AgentPoolProfiles[0].VMSize = "Standard_D2_v2" + p.OrchestratorProfile.KubernetesConfig.Addons = []KubernetesAddon{ { Name: DefaultNVIDIADevicePluginAddonName, - Enabled: &b, + Enabled: helpers.PointerToBool(false), }, } - enabled = p.IsNVIDIADevicePluginEnabled() - if enabled { + + if isNSeriesSKU(&p) { + t.Fatalf("isNSeriesSKU should return false when explicitly using VM Size %s", p.AgentPoolProfiles[0].VMSize) + } + if p.IsNVIDIADevicePluginEnabled() { t.Fatalf("KubernetesConfig.IsNVIDIADevicePluginEnabled() should return false when explicitly disabled") } } diff --git a/pkg/api/v20160330/validate.go b/pkg/api/v20160330/validate.go index a0d250b16f..deb8778745 100644 --- a/pkg/api/v20160330/validate.go +++ b/pkg/api/v20160330/validate.go @@ -4,6 +4,8 @@ import ( "errors" "fmt" "regexp" + + "github.com/Azure/acs-engine/pkg/api/common" ) // Validate implements APIObject @@ -27,7 +29,7 @@ func (m *MasterProfile) Validate() error { if e := validateName(m.DNSPrefix, "MasterProfile.DNSPrefix"); e != nil { return e } - return validateDNSName(m.DNSPrefix) + return common.ValidateDNSPrefix(m.DNSPrefix) } // Validate implements APIObject @@ -45,7 +47,7 @@ func (a *AgentPoolProfile) Validate() error { return e } if a.DNSPrefix != "" { - if e := validateDNSName(a.DNSPrefix); e != nil { + if e := common.ValidateDNSPrefix(a.DNSPrefix); e != nil { return e } } @@ -129,18 +131,6 @@ func validatePoolName(poolName string) error { return nil } -func validateDNSName(dnsName string) error { - dnsNameRegex := `^([A-Za-z][A-Za-z0-9-]{1,43}[A-Za-z0-9])$` - re, err := regexp.Compile(dnsNameRegex) - if err != nil { - return err - } - if !re.MatchString(dnsName) { - return fmt.Errorf("DNS name '%s' is invalid. The DNS name must contain between 3 and 45 characters. The name can contain only letters, numbers, and hyphens. The name must start with a letter and must end with a letter or a number. (length was %d)", dnsName, len(dnsName)) - } - return nil -} - func validateUniqueProfileNames(profiles []*AgentPoolProfile) error { profileNames := make(map[string]bool) for _, profile := range profiles { diff --git a/pkg/api/v20160930/validate.go b/pkg/api/v20160930/validate.go index 89dbf8c6a7..38dee2d38f 100644 --- a/pkg/api/v20160930/validate.go +++ b/pkg/api/v20160930/validate.go @@ -4,6 +4,8 @@ import ( "errors" "fmt" "regexp" + + "github.com/Azure/acs-engine/pkg/api/common" ) // Validate implements APIObject @@ -28,7 +30,7 @@ func (m *MasterProfile) Validate() error { if e := validateName(m.DNSPrefix, "MasterProfile.DNSPrefix"); e != nil { return e } - return validateDNSName(m.DNSPrefix) + return common.ValidateDNSPrefix(m.DNSPrefix) } // Validate implements APIObject @@ -53,7 +55,7 @@ func (a *AgentPoolProfile) Validate(orchestratorType string) error { } } if a.DNSPrefix != "" { - if e := validateDNSName(a.DNSPrefix); e != nil { + if e := common.ValidateDNSPrefix(a.DNSPrefix); e != nil { return e } } @@ -144,18 +146,6 @@ func validatePoolName(poolName string) error { return nil } -func validateDNSName(dnsName string) error { - dnsNameRegex := `^([A-Za-z][A-Za-z0-9-]{1,43}[A-Za-z0-9])$` - re, err := regexp.Compile(dnsNameRegex) - if err != nil { - return err - } - if !re.MatchString(dnsName) { - return fmt.Errorf("DNS name '%s' is invalid. The DNS name must contain between 3 and 45 characters. The name can contain only letters, numbers, and hyphens. The name must start with a letter and must end with a letter or a number. (length was %d)", dnsName, len(dnsName)) - } - return nil -} - func validateUniqueProfileNames(profiles []*AgentPoolProfile) error { profileNames := make(map[string]bool) for _, profile := range profiles { diff --git a/pkg/api/v20170131/validate.go b/pkg/api/v20170131/validate.go index 26f5957401..beee768ec0 100644 --- a/pkg/api/v20170131/validate.go +++ b/pkg/api/v20170131/validate.go @@ -4,6 +4,8 @@ import ( "errors" "fmt" "regexp" + + "github.com/Azure/acs-engine/pkg/api/common" ) // Validate implements APIObject @@ -29,7 +31,7 @@ func (m *MasterProfile) Validate() error { if e := validateName(m.DNSPrefix, "MasterProfile.DNSPrefix"); e != nil { return e } - return validateDNSName(m.DNSPrefix) + return common.ValidateDNSPrefix(m.DNSPrefix) } // Validate implements APIObject @@ -54,7 +56,7 @@ func (a *AgentPoolProfile) Validate(orchestratorType string) error { } } if a.DNSPrefix != "" { - if e := validateDNSName(a.DNSPrefix); e != nil { + if e := common.ValidateDNSPrefix(a.DNSPrefix); e != nil { return e } } @@ -157,18 +159,6 @@ func validatePoolName(poolName string) error { return nil } -func validateDNSName(dnsName string) error { - dnsNameRegex := `^([A-Za-z][A-Za-z0-9-]{1,43}[A-Za-z0-9])$` - re, err := regexp.Compile(dnsNameRegex) - if err != nil { - return err - } - if !re.MatchString(dnsName) { - return fmt.Errorf("DNS name '%s' is invalid. The DNS name must contain between 3 and 45 characters. The name can contain only letters, numbers, and hyphens. The name must start with a letter and must end with a letter or a number. (length was %d)", dnsName, len(dnsName)) - } - return nil -} - func validateUniqueProfileNames(profiles []*AgentPoolProfile) error { profileNames := make(map[string]bool) for _, profile := range profiles { diff --git a/pkg/api/v20170701/validate.go b/pkg/api/v20170701/validate.go index 11ccb14d0c..bbab64ab4d 100644 --- a/pkg/api/v20170701/validate.go +++ b/pkg/api/v20170701/validate.go @@ -66,7 +66,7 @@ func (o *OrchestratorProfile) Validate(isUpdate, hasWindows bool) error { func (m *MasterProfile) Validate() error { // Don't need to call validate.Struct(m) // It is handled by Properties.Validate() - return validateDNSName(m.DNSPrefix) + return common.ValidateDNSPrefix(m.DNSPrefix) } // Validate implements APIObject @@ -89,7 +89,7 @@ func (a *AgentPoolProfile) Validate(orchestratorType string) error { } } if a.DNSPrefix != "" { - if e := validateDNSName(a.DNSPrefix); e != nil { + if e := common.ValidateDNSPrefix(a.DNSPrefix); e != nil { return e } if len(a.Ports) > 0 { @@ -209,18 +209,6 @@ func validatePoolName(poolName string) error { return nil } -func validateDNSName(dnsName string) error { - dnsNameRegex := `^([A-Za-z][A-Za-z0-9-]{1,43}[A-Za-z0-9])$` - re, err := regexp.Compile(dnsNameRegex) - if err != nil { - return err - } - if !re.MatchString(dnsName) { - return fmt.Errorf("DNS name '%s' is invalid. The DNS name must contain between 3 and 45 characters. The name can contain only letters, numbers, and hyphens. The name must start with a letter and must end with a letter or a number. (length was %d)", dnsName, len(dnsName)) - } - return nil -} - func validateUniqueProfileNames(profiles []*AgentPoolProfile) error { profileNames := make(map[string]bool) for _, profile := range profiles { diff --git a/pkg/api/vlabs/types.go b/pkg/api/vlabs/types.go index 40ed78f402..0ca4e39127 100644 --- a/pkg/api/vlabs/types.go +++ b/pkg/api/vlabs/types.go @@ -609,11 +609,6 @@ func (a *AgentPoolProfile) SetSubnet(subnet string) { a.subnet = subnet } -// IsAcceleratedNetworkingEnabled returns true if the customer enabled Accelerated Networking -func (a *AgentPoolProfile) IsAcceleratedNetworkingEnabled() bool { - return a.AcceleratedNetworkingEnabled -} - // HasSearchDomain returns true if the customer specified secrets to install func (l *LinuxProfile) HasSearchDomain() bool { if l.CustomSearchDomain != nil { diff --git a/pkg/api/vlabs/validate.go b/pkg/api/vlabs/validate.go index b49ea1d11c..0515cf3a74 100644 --- a/pkg/api/vlabs/validate.go +++ b/pkg/api/vlabs/validate.go @@ -304,12 +304,16 @@ func (a *Properties) validateOrchestratorProfile(isUpdate bool) error { func (a *Properties) validateMasterProfile() error { m := a.MasterProfile - if a.OrchestratorProfile.OrchestratorType == OpenShift && m.Count != 1 { - return errors.New("openshift can only deployed with one master") - } - - if a.OrchestratorProfile.OrchestratorType == OpenShift && m.StorageProfile != ManagedDisks { - return errors.New("OpenShift orchestrator supports only ManagedDisks") + if a.OrchestratorProfile.OrchestratorType == OpenShift { + if m.Count != 1 { + return errors.New("openshift can only deployed with one master") + } + if m.VnetSubnetID != "" && m.FirstConsecutiveStaticIP == "" { + return errors.New("when specifying a vnetsubnetid the firstconsecutivestaticip is required") + } + if m.StorageProfile != ManagedDisks { + return errors.New("OpenShift orchestrator supports only ManagedDisks") + } } if m.ImageRef != nil { @@ -317,7 +321,7 @@ func (a *Properties) validateMasterProfile() error { return err } } - return validateDNSName(m.DNSPrefix) + return common.ValidateDNSPrefix(m.DNSPrefix) } func (a *Properties) validateAgentPoolProfiles() error { @@ -690,12 +694,7 @@ func (a *AgentPoolProfile) validateVMSS(o *OrchestratorProfile) error { if *o.KubernetesConfig.UseInstanceMetadata && sv.LT(minVersion) { return fmt.Errorf("VirtualMachineScaleSets with instance metadata is supported for Kubernetes version %s or greater. Please set \"useInstanceMetadata\": false in \"kubernetesConfig\" or set \"orchestratorVersion\" to %s or above", minVersion.String(), minVersion.String()) } - } else { - if sv.LT(minVersion) { - return fmt.Errorf("VirtualMachineScaleSets with instance metadata is supported for Kubernetes version %s or greater. Please set \"useInstanceMetadata\": false in \"kubernetesConfig\" or set \"orchestratorVersion\" to %s or above", minVersion.String(), minVersion.String()) - } } - if (a.AvailabilityProfile == VirtualMachineScaleSets || len(a.AvailabilityProfile) == 0) && a.StorageProfile == StorageAccount { return fmt.Errorf("VirtualMachineScaleSets does not support %s disks. Please specify \"storageProfile\": \"%s\" (recommended) or \"availabilityProfile\": \"%s\"", StorageAccount, ManagedDisks, AvailabilitySet) } @@ -746,7 +745,7 @@ func (a *AgentPoolProfile) validateOrchestratorSpecificProperties(orchestratorTy } if a.DNSPrefix != "" { - if e := validateDNSName(a.DNSPrefix); e != nil { + if e := common.ValidateDNSPrefix(a.DNSPrefix); e != nil { return e } if len(a.Ports) > 0 { @@ -1106,18 +1105,6 @@ func validatePoolOSType(os OSType) error { return nil } -func validateDNSName(dnsName string) error { - dnsNameRegex := `^([A-Za-z][A-Za-z0-9-]{1,43}[A-Za-z0-9])$` - re, err := regexp.Compile(dnsNameRegex) - if err != nil { - return err - } - if !re.MatchString(dnsName) { - return fmt.Errorf("DNS name '%s' is invalid. The DNS name must contain between 3 and 45 characters. The name can contain only letters, numbers, and hyphens. The name must start with a letter and must end with a letter or a number (length was %d)", dnsName, len(dnsName)) - } - return nil -} - func validateUniquePorts(ports []int, name string) error { portMap := make(map[int]bool) for _, port := range ports { diff --git a/pkg/api/vlabs/validate_test.go b/pkg/api/vlabs/validate_test.go index 808856dc45..569de26395 100644 --- a/pkg/api/vlabs/validate_test.go +++ b/pkg/api/vlabs/validate_test.go @@ -971,7 +971,7 @@ func TestMasterProfileValidate(t *testing.T) { masterProfile: MasterProfile{ DNSPrefix: "bad!", }, - expectedErr: "DNS name 'bad!' is invalid. The DNS name must contain between 3 and 45 characters. The name can contain only letters, numbers, and hyphens. The name must start with a letter and must end with a letter or a number (length was 4)", + expectedErr: "DNSPrefix 'bad!' is invalid. The DNSPrefix must contain between 3 and 45 characters and can contain only letters, numbers, and hyphens. It must start with a letter and must end with a letter or a number. (length was 4)", }, { masterProfile: MasterProfile{ @@ -1000,6 +1000,23 @@ func TestMasterProfileValidate(t *testing.T) { }, expectedErr: "openshift can only deployed with one master", }, + { // test existing vnet: run with only specifying vnetsubnetid + orchestratorType: OpenShift, + masterProfile: MasterProfile{ + VnetSubnetID: "testvnetstring", + Count: 1, + }, + expectedErr: "when specifying a vnetsubnetid the firstconsecutivestaticip is required", + }, + { // test existing vnet: run with specifying both vnetsubnetid and firstconsecutivestaticip + orchestratorType: OpenShift, + masterProfile: MasterProfile{ + DNSPrefix: "dummy", + VnetSubnetID: "testvnetstring", + FirstConsecutiveStaticIP: "10.0.0.1", + Count: 1, + }, + }, } for i, test := range tests { diff --git a/pkg/armhelpers/azureclient.go b/pkg/armhelpers/azureclient.go index f32d9fc2b2..0a48e08c89 100644 --- a/pkg/armhelpers/azureclient.go +++ b/pkg/armhelpers/azureclient.go @@ -46,17 +46,18 @@ type AzureClient struct { environment azure.Environment subscriptionID string - authorizationClient authorization.RoleAssignmentsClient - deploymentsClient resources.DeploymentsClient - deploymentOperationsClient resources.DeploymentOperationsClient - resourcesClient resources.GroupClient - storageAccountsClient storage.AccountsClient - interfacesClient network.InterfacesClient - groupsClient resources.GroupsClient - providersClient resources.ProvidersClient - virtualMachinesClient compute.VirtualMachinesClient - virtualMachineScaleSetsClient compute.VirtualMachineScaleSetsClient - disksClient disk.DisksClient + authorizationClient authorization.RoleAssignmentsClient + deploymentsClient resources.DeploymentsClient + deploymentOperationsClient resources.DeploymentOperationsClient + resourcesClient resources.GroupClient + storageAccountsClient storage.AccountsClient + interfacesClient network.InterfacesClient + groupsClient resources.GroupsClient + providersClient resources.ProvidersClient + virtualMachinesClient compute.VirtualMachinesClient + virtualMachineScaleSetsClient compute.VirtualMachineScaleSetsClient + virtualMachineScaleSetVMsClient compute.VirtualMachineScaleSetVMsClient + disksClient disk.DisksClient applicationsClient graphrbac.ApplicationsClient servicePrincipalsClient graphrbac.ServicePrincipalsClient @@ -264,17 +265,18 @@ func getClient(env azure.Environment, subscriptionID, tenantID string, armSpt *a environment: env, subscriptionID: subscriptionID, - authorizationClient: authorization.NewRoleAssignmentsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID), - deploymentsClient: resources.NewDeploymentsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID), - deploymentOperationsClient: resources.NewDeploymentOperationsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID), - resourcesClient: resources.NewGroupClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID), - storageAccountsClient: storage.NewAccountsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID), - interfacesClient: network.NewInterfacesClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID), - groupsClient: resources.NewGroupsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID), - providersClient: resources.NewProvidersClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID), - virtualMachinesClient: compute.NewVirtualMachinesClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID), - virtualMachineScaleSetsClient: compute.NewVirtualMachineScaleSetsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID), - disksClient: disk.NewDisksClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID), + authorizationClient: authorization.NewRoleAssignmentsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID), + deploymentsClient: resources.NewDeploymentsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID), + deploymentOperationsClient: resources.NewDeploymentOperationsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID), + resourcesClient: resources.NewGroupClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID), + storageAccountsClient: storage.NewAccountsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID), + interfacesClient: network.NewInterfacesClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID), + groupsClient: resources.NewGroupsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID), + providersClient: resources.NewProvidersClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID), + virtualMachinesClient: compute.NewVirtualMachinesClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID), + virtualMachineScaleSetsClient: compute.NewVirtualMachineScaleSetsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID), + virtualMachineScaleSetVMsClient: compute.NewVirtualMachineScaleSetVMsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID), + disksClient: disk.NewDisksClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID), applicationsClient: graphrbac.NewApplicationsClientWithBaseURI(env.GraphEndpoint, tenantID), servicePrincipalsClient: graphrbac.NewServicePrincipalsClientWithBaseURI(env.GraphEndpoint, tenantID), @@ -291,6 +293,7 @@ func getClient(env azure.Environment, subscriptionID, tenantID string, armSpt *a c.providersClient.Authorizer = authorizer c.virtualMachinesClient.Authorizer = authorizer c.virtualMachineScaleSetsClient.Authorizer = authorizer + c.virtualMachineScaleSetVMsClient.Authorizer = authorizer c.disksClient.Authorizer = authorizer c.deploymentsClient.PollingDelay = time.Second * 5 diff --git a/pkg/armhelpers/compute.go b/pkg/armhelpers/compute.go index 3bc7a3479f..fde07ed472 100644 --- a/pkg/armhelpers/compute.go +++ b/pkg/armhelpers/compute.go @@ -23,3 +23,26 @@ func (az *AzureClient) DeleteVirtualMachine(resourceGroup, name string, cancel < func (az *AzureClient) ListVirtualMachineScaleSets(resourceGroup string) (compute.VirtualMachineScaleSetListResult, error) { return az.virtualMachineScaleSetsClient.List(resourceGroup) } + +// ListVirtualMachineScaleSetVMs returns the list of VMs per VMSS +func (az *AzureClient) ListVirtualMachineScaleSetVMs(resourceGroup, virtualMachineScaleSet string) (compute.VirtualMachineScaleSetVMListResult, error) { + return az.virtualMachineScaleSetVMsClient.List(resourceGroup, virtualMachineScaleSet, "", "", "") +} + +// DeleteVirtualMachineScaleSetVM deletes a VM in a VMSS +func (az *AzureClient) DeleteVirtualMachineScaleSetVM(resourceGroup, virtualMachineScaleSet, instanceID string, cancel <-chan struct{}) (<-chan compute.OperationStatusResponse, <-chan error) { + return az.virtualMachineScaleSetVMsClient.Delete(resourceGroup, virtualMachineScaleSet, instanceID, cancel) +} + +// SetVirtualMachineScaleSetCapacity sets the VMSS capacity +func (az *AzureClient) SetVirtualMachineScaleSetCapacity(resourceGroup, virtualMachineScaleSet string, sku compute.Sku, location string, cancel <-chan struct{}) (<-chan compute.VirtualMachineScaleSet, <-chan error) { + return az.virtualMachineScaleSetsClient.CreateOrUpdate( + resourceGroup, + virtualMachineScaleSet, + compute.VirtualMachineScaleSet{ + Location: &location, + Sku: &sku, + }, + cancel, + ) +} diff --git a/pkg/armhelpers/interfaces.go b/pkg/armhelpers/interfaces.go index 0f6e812914..55517ee2c8 100644 --- a/pkg/armhelpers/interfaces.go +++ b/pkg/armhelpers/interfaces.go @@ -44,6 +44,15 @@ type ACSEngineClient interface { // ListVirtualMachineScaleSets lists the vmss resources in the resource group ListVirtualMachineScaleSets(resourceGroup string) (compute.VirtualMachineScaleSetListResult, error) + // ListVirtualMachineScaleSetVMs lists the virtual machines contained in a vmss + ListVirtualMachineScaleSetVMs(resourceGroup, virtualMachineScaleSet string) (compute.VirtualMachineScaleSetVMListResult, error) + + // DeleteVirtualMachineScaleSetVM deletes a VM in a VMSS + DeleteVirtualMachineScaleSetVM(resourceGroup, virtualMachineScaleSet, instanceID string, cancel <-chan struct{}) (<-chan compute.OperationStatusResponse, <-chan error) + + // SetVirtualMachineScaleSetCapacity sets the VMSS capacity + SetVirtualMachineScaleSetCapacity(resourceGroup, virtualMachineScaleSet string, sku compute.Sku, location string, cancel <-chan struct{}) (<-chan compute.VirtualMachineScaleSet, <-chan error) + // // STORAGE diff --git a/pkg/armhelpers/mockclients.go b/pkg/armhelpers/mockclients.go index 20cb200706..b4820a9f23 100644 --- a/pkg/armhelpers/mockclients.go +++ b/pkg/armhelpers/mockclients.go @@ -20,21 +20,24 @@ import ( //MockACSEngineClient is an implementation of ACSEngineClient where all requests error out type MockACSEngineClient struct { - FailDeployTemplate bool - FailDeployTemplateQuota bool - FailDeployTemplateConflict bool - FailEnsureResourceGroup bool - FailListVirtualMachines bool - FailListVirtualMachineScaleSets bool - FailGetVirtualMachine bool - FailDeleteVirtualMachine bool - FailGetStorageClient bool - FailDeleteNetworkInterface bool - FailGetKubernetesClient bool - FailListProviders bool - ShouldSupportVMIdentity bool - FailDeleteRoleAssignment bool - MockKubernetesClient *MockKubernetesClient + FailDeployTemplate bool + FailDeployTemplateQuota bool + FailDeployTemplateConflict bool + FailEnsureResourceGroup bool + FailListVirtualMachines bool + FailListVirtualMachineScaleSets bool + FailGetVirtualMachine bool + FailDeleteVirtualMachine bool + FailDeleteVirtualMachineScaleSetVM bool + FailSetVirtualMachineScaleSetCapacity bool + FailListVirtualMachineScaleSetVMs bool + FailGetStorageClient bool + FailDeleteNetworkInterface bool + FailGetKubernetesClient bool + FailListProviders bool + ShouldSupportVMIdentity bool + FailDeleteRoleAssignment bool + MockKubernetesClient *MockKubernetesClient } //MockStorageClient mock implementation of StorageClient @@ -344,6 +347,79 @@ func (mc *MockACSEngineClient) DeleteVirtualMachine(resourceGroup, name string, return respChan, errChan } +//DeleteVirtualMachineScaleSetVM mock +func (mc *MockACSEngineClient) DeleteVirtualMachineScaleSetVM(resourceGroup, virtualMachineScaleSet, instanceID string, cancel <-chan struct{}) (<-chan compute.OperationStatusResponse, <-chan error) { + if mc.FailDeleteVirtualMachineScaleSetVM { + errChan := make(chan error) + respChan := make(chan compute.OperationStatusResponse) + go func() { + defer func() { + close(errChan) + }() + defer func() { + close(respChan) + }() + errChan <- fmt.Errorf("DeleteVirtualMachineScaleSetVM failed") + }() + return respChan, errChan + } + + errChan := make(chan error) + respChan := make(chan compute.OperationStatusResponse) + go func() { + defer func() { + close(errChan) + }() + defer func() { + close(respChan) + }() + errChan <- nil + respChan <- compute.OperationStatusResponse{} + }() + return respChan, errChan +} + +//SetVirtualMachineScaleSetCapacity mock +func (mc *MockACSEngineClient) SetVirtualMachineScaleSetCapacity(resourceGroup, virtualMachineScaleSet string, sku compute.Sku, location string, cancel <-chan struct{}) (<-chan compute.VirtualMachineScaleSet, <-chan error) { + if mc.FailSetVirtualMachineScaleSetCapacity { + errChan := make(chan error) + respChan := make(chan compute.VirtualMachineScaleSet) + go func() { + defer func() { + close(errChan) + }() + defer func() { + close(respChan) + }() + errChan <- fmt.Errorf("SetVirtualMachineScaleSetCapacity failed") + }() + return respChan, errChan + } + + errChan := make(chan error) + respChan := make(chan compute.VirtualMachineScaleSet) + go func() { + defer func() { + close(errChan) + }() + defer func() { + close(respChan) + }() + errChan <- nil + respChan <- compute.VirtualMachineScaleSet{} + }() + return respChan, errChan +} + +//ListVirtualMachineScaleSetVMs mock +func (mc *MockACSEngineClient) ListVirtualMachineScaleSetVMs(resourceGroup, virtualMachineScaleSet string) (compute.VirtualMachineScaleSetVMListResult, error) { + if mc.FailDeleteVirtualMachineScaleSetVM { + return compute.VirtualMachineScaleSetVMListResult{}, fmt.Errorf("DeleteVirtualMachineScaleSetVM failed") + } + + return compute.VirtualMachineScaleSetVMListResult{}, nil +} + //GetStorageClient mock func (mc *MockACSEngineClient) GetStorageClient(resourceGroup, accountName string) (ACSStorageClient, error) { if mc.FailGetStorageClient { diff --git a/pkg/openshift/certgen/release39/templates/bindata.go b/pkg/openshift/certgen/release39/templates/bindata.go index 01f2a4ad73..184ddf2182 100644 --- a/pkg/openshift/certgen/release39/templates/bindata.go +++ b/pkg/openshift/certgen/release39/templates/bindata.go @@ -119,7 +119,7 @@ func masterEtcOriginMasterHtpasswd() (*asset, error) { return a, nil } -var _masterEtcOriginMasterMasterConfigYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xc4\x58\x7b\x6f\x1b\xb9\x11\xff\x5f\x9f\x82\x08\x0e\x48\x52\x74\x57\x92\x9d\xe7\x02\x45\xa1\xda\xce\x45\x38\x3b\x51\x65\xa7\xb8\xa2\x2e\x0e\x14\x39\x5a\x31\xe2\x92\x1b\x3e\x14\x2b\x6e\xbe\x7b\xc1\xc7\xee\x72\xf5\x48\xd2\x5c\x8a\x73\x82\xc4\x4b\xce\x0c\x87\x33\x3f\xce\x0b\xd3\x8a\x69\xcd\xa4\x38\x93\x62\xc9\xca\x62\x80\x50\xcd\x6d\xc9\x92\x6f\x84\xfe\x66\x19\xa7\xe7\xb0\xc4\x96\x1b\x1d\x96\x10\x22\x9e\xc0\x2a\x6c\x98\x14\xcd\x22\x42\xb8\x66\xff\x00\xe5\x24\x16\x68\x33\x6e\x97\x41\x6c\x0a\xf4\xaf\x7f\xb7\xdf\x6b\x26\x68\xd1\x17\x1c\x4e\x6c\x29\x14\x68\x69\x15\x01\xdd\xc9\x46\x88\xb3\x8a\x19\x5d\xa0\xfb\xcf\xc9\xa2\x82\x0f\x16\x74\xb2\xec\xc5\xbe\xdd\x80\x52\x8c\xc2\x77\x2a\x9c\x28\xd8\x4a\x4a\x34\x9c\x49\x3a\x53\xa0\xc1\x7c\x9f\x74\xca\x34\x5e\x70\x28\xd0\x12\x73\x0d\x3b\x87\x46\x83\x4c\xfa\xae\xf1\x44\xb2\x06\xa1\x57\x6c\x69\x72\x26\x87\xd3\x0a\x97\x30\x93\x9c\x91\xed\x77\x3a\xe5\x0e\x88\x75\x94\x73\xcb\x53\x3b\x67\xa8\xc2\x86\xac\xbc\xfc\x89\x10\xd2\x78\x71\x3d\x47\x64\x68\x0d\xdb\x02\x31\x47\xa2\xf3\x9e\x5a\x14\xc4\x36\x6b\x45\x27\x3c\x08\x6d\x30\xb7\x50\xa0\x87\x46\x59\x78\x98\xec\x08\x5c\x41\xd1\xa9\x93\x51\x10\x0c\x68\x42\x20\xc5\xfc\x10\x1c\xb2\x16\x25\x05\xaa\x25\xd5\x47\xb6\x16\xce\x8b\xba\x87\x98\xf7\x40\x4c\x81\x9c\x1e\xc9\xb2\x5e\xb3\xfa\xad\x3f\x89\x7b\x3d\x5e\x61\xc6\xad\x82\x1d\xba\xe0\xa4\xc4\xf8\xd1\x3f\xb8\x2c\x15\x94\xd8\x48\x95\xbc\x25\x25\xef\xb6\x67\x9c\x81\x30\x53\xb1\x94\x41\x77\x02\xca\xbc\x62\xce\xfb\x1d\x4b\xb6\x54\x52\x98\xcc\xd3\xe7\x44\x19\x4f\xb8\x86\xed\x17\xe9\xd6\xb0\x1d\xe0\x9a\x5d\xc2\x06\xb8\x2e\x06\x99\xf3\xed\x8e\xab\xb1\x35\xab\x4e\x9d\xf8\x52\x5e\x03\xa6\xa0\xa2\x32\x5e\xb9\xb3\x49\x81\x12\xc9\x19\xc1\xad\x12\x91\x40\x56\x95\x14\x6f\x70\xd5\x38\x20\x3b\xa2\xd4\x20\x00\xcb\x28\x1c\x4e\x99\x29\x58\xb2\xbb\x8e\xeb\xd7\x6c\x0e\x95\x34\x90\x5d\x38\x9a\xcc\xaf\x96\x4a\xda\x3a\x90\xef\xd3\xfd\xec\x36\xfd\xa2\xd5\xa0\x1c\x52\x8e\x51\xbe\xd3\xa0\x06\x44\x0a\xa3\x24\xe7\x90\x78\x01\x38\x90\xee\x41\x70\x49\xd6\x6f\x3c\xe0\x5a\xd8\x66\x15\xd6\x06\x54\xd6\x31\x3b\xb4\x68\x50\x1b\x46\xe0\xda\xfd\x27\xca\x33\x50\xf1\xb1\x6b\x56\x8a\xc6\x7c\xa9\x37\x23\x7d\x16\xf6\x5b\x03\x26\x7e\xdc\xa1\x70\xfe\x4b\x8e\x2c\xd0\xc3\x3f\x3d\x1c\x10\xa9\xf4\x84\x73\xf9\x11\xe8\x5b\xc5\x4a\x26\xbc\x67\x1f\xfd\x95\x3d\x1e\x0e\xc7\x27\xcf\x6f\xf3\x91\xff\x3b\x7e\x54\xfc\xe7\xf6\xd3\xe3\x76\x8b\x4b\x82\xf9\x4a\x6a\xb3\xb3\x7e\x7f\x8f\xfe\x6e\xa5\x81\x2b\x30\x18\x3d\x62\x82\xc2\x1d\xca\xaf\xfc\x75\xf3\xe9\x4c\xa3\xd1\xe3\xfc\xda\x28\x26\x4a\xf4\xf9\xf3\x0e\xeb\xda\x2e\x40\x09\x30\xa0\x6f\x73\x1a\x62\xd2\xd7\x29\x6e\x73\xbd\x21\xb7\x39\xe1\xd6\x1d\x71\x9b\x7b\xbd\x8e\xb2\x7d\x49\xd9\xfc\xe2\xce\x38\x87\xf3\xa0\xed\x6b\xa9\x8d\xf3\xfe\xbe\x9e\xad\x1b\x8f\xa9\xd9\x17\x1b\x2f\xff\xbf\xc8\xf3\x97\xfa\xc6\xbb\xef\x90\x8d\x9f\x9f\xdc\xe6\xa7\x87\x7d\x76\xe4\xa0\xaf\x58\xaf\xe5\x8a\xeb\x54\xe8\x0e\xea\x0b\x26\xe8\x84\x52\x05\x5a\x17\x68\x94\xfb\x3f\xc5\x8b\xd1\xd3\xd3\xb8\xf7\x06\xcc\x47\xa9\xd6\x05\x32\xa4\x7e\x32\x00\x43\x68\x3f\x3a\x11\x5c\xa0\xf0\x18\x72\xb7\xd9\x05\x82\x0e\xe6\xbd\x6d\xcf\x1c\x49\x5a\x98\x1f\xa0\x70\x50\x47\xc8\x2a\xee\x9f\x6d\x86\x56\xc6\xd4\xba\xf0\xae\x39\xe0\x90\xe2\xe4\xf4\xf9\x4b\xaf\xdd\xb5\x91\x0a\x97\xd0\x5d\xb0\x33\x7b\xdc\x0a\x01\xa6\x48\x36\x72\x26\x0f\x11\xf6\x33\xa0\xb3\xe2\xb5\xb3\xe2\x8e\x98\x34\x95\x1d\x20\x4b\x85\xf8\xe4\xd7\x69\xb6\x94\xaa\xc2\xa6\x40\xd3\xab\xc9\xcf\x17\xbf\xcd\xe6\x17\xaf\xa6\xbf\x0e\xc3\xc7\xcd\x3f\x67\x17\xd9\x4f\xf7\x44\x56\xb5\x14\x20\xcc\xe7\xe2\xa7\xfb\x4d\x90\xe4\x2a\x16\x8e\x0d\x68\xd3\x14\x03\x21\xbf\x04\xa3\xc4\xd4\xe2\xee\xc2\xc1\xec\xbb\x8a\xe0\x6c\x61\x05\xe5\x70\xcc\x4b\x91\xf3\xcb\x8e\xda\x21\x0a\xbe\xaa\xa5\x32\x05\x1a\x8f\x4e\x9e\x8e\x06\x9d\x2d\x53\xb5\x9c\x12\xb8\x66\x2e\x3e\x82\x9a\xa8\xd2\x56\x20\x9a\xfa\x50\x59\x61\x58\x05\x19\x49\xca\xc8\xcc\x51\xeb\xa1\x06\x63\x98\x28\x75\xbe\x7e\xe1\x5c\x35\xdc\x8c\x31\xaf\x57\x78\xfc\x97\x36\xcb\xea\x60\xeb\x6c\x81\xc9\x1a\x04\x6d\xb8\x1d\x1e\x4e\x7b\x04\x15\x50\x86\x33\xb3\xad\xa1\x3b\xa1\xe6\x8c\xf8\x7a\x65\xb8\x11\x34\x4f\x50\x51\x2b\x69\xe4\xc2\x2e\x63\x56\x93\x96\xba\x8c\xb5\x61\x6d\x2a\xcc\xd0\x03\xfc\xc9\x2a\x78\x90\x50\xf4\xf5\x7f\x30\x04\x43\x86\x9e\x28\xfc\x9b\xbb\x7d\x47\xdf\x85\xf1\x1d\x43\xc4\xa7\xec\x43\x3e\x13\x65\xe6\x3c\x94\x2d\x9d\xf5\x7b\x32\xa5\x0f\xf5\xc3\xe0\x90\x61\x78\x75\x0f\x0e\x0a\x58\xc3\xf6\x5b\xf8\xd7\xb0\x7d\xf0\x7f\xb9\x69\x15\x11\x60\x85\x83\x47\xbb\x30\x9d\x15\xe8\xfe\xfe\x6b\x19\xc6\xe3\x8a\x5e\x6c\x98\xcf\xc7\x37\xac\x02\x69\x4d\x81\x84\xe5\xfc\xeb\xd5\x52\x44\x6b\xac\x50\x52\x40\xef\x43\xba\x47\x14\x00\xad\xc9\x0a\xa8\xed\x79\xa8\x39\xb8\xdd\x0a\xc0\x0e\x92\x0e\xd8\xb5\xa5\xcb\xdf\x6b\x5f\xda\xc6\x84\xae\xdf\x48\x0a\x33\xa9\xcc\x1c\x8b\xd2\x15\xb8\x0f\x93\xbd\x6b\xbb\x10\xe0\x6c\xf5\xfc\x24\x3f\xf5\x01\x79\x38\x7e\xe6\xf6\x5d\x59\x4d\x1c\x67\x28\xab\x5c\x7b\x14\x8d\xeb\xd5\xf6\x00\x82\x98\x03\x7f\x69\x71\x7c\x16\xeb\x31\x21\x42\x51\xb3\xd3\xe7\x60\x42\xa0\x76\xdb\x06\x84\xb9\xd9\xd6\x4e\xf0\x37\x3c\x8a\x3f\xa7\x34\xf1\x72\x08\x2d\xac\x72\x71\xe9\xc9\x68\x34\x88\xdd\x45\x23\xf5\x9b\x84\x7a\xa6\x0f\xb5\x2e\xd0\x89\x97\xb0\x7f\x19\xf7\x5b\x0c\x26\xc1\x68\x6d\xf8\xbd\x94\xb2\x76\xef\xff\x0f\xb8\xee\xb3\xdf\x7d\xdd\x53\x2f\x61\xef\x2e\xe9\x6d\x77\x6b\x4f\x2f\x30\xbc\xc1\x88\x82\x99\x5d\x70\x46\xde\xcd\x2f\x8b\x5e\xb6\x3c\x5a\x15\x15\x49\x2e\x75\x58\x74\xcf\x4d\x84\x74\xdf\xc5\xeb\x18\x4d\x62\x19\x70\x36\x3d\x9f\xbb\x18\x9f\x8f\x4f\x5e\x04\x60\x3e\xd9\xa3\x89\x09\x9b\x30\xaa\xf6\x49\x11\x72\x25\x67\x40\xf8\x25\x88\xd2\xac\x0a\xf4\x32\xf1\xf4\x74\x96\x9c\x14\x25\xc5\xaa\x64\xe8\x4c\x74\x98\x3b\x6a\x3d\xf3\x83\x88\x50\xaa\x2b\xa0\x2b\x6c\xba\xda\x27\x93\x1b\x9d\x69\xcf\xd9\x3d\xb5\xfe\xad\x7a\xcf\x4d\xf6\x1b\x21\xac\x35\x98\x1f\x60\xe0\x21\x91\x42\x4b\x0e\xc3\x81\xeb\x62\xb0\x07\x6a\x1b\x45\x2b\x30\x2b\x49\x0b\x84\xad\x71\xa5\x04\xa3\x20\x0c\x33\xdb\x59\x0c\xc5\xba\x18\xdc\xdf\x67\x88\x2d\x51\x7e\x21\xf0\x82\xc3\x64\x72\x3e\xb1\x66\xe5\xa8\x02\xd0\x7c\xbc\xcc\x62\x77\x3c\x71\x51\x18\x4d\xce\x03\x34\x57\x98\x73\xf0\xb1\xa6\x9b\x20\x70\x59\x32\x91\x34\xab\x15\xae\x6b\x26\xca\xab\xa8\x06\xe1\x98\x55\x7e\xa3\x9f\x0c\x8e\x8c\x07\x42\x19\xf2\xb6\x06\x31\x3d\x9f\xee\xa8\xde\xb4\x3e\x21\x54\x9f\xfb\xc8\x9f\x7b\x05\xc3\xfd\xf3\xc9\xe4\x3c\xc6\xf1\xf3\x10\xf5\x3b\xf2\x6b\x20\xca\x85\xc3\xa3\x2c\x81\x20\x65\xc3\xac\x4a\xda\x7e\x46\xd3\x49\x85\xb6\x8b\xf6\xab\x56\xb0\x04\xa5\x80\xbe\x8b\x9d\x62\x4a\x68\x05\xfb\x60\xe1\x37\xb7\xdc\xae\xee\xd2\xf4\x36\xa1\xc2\x8c\xa7\xbb\x7e\x21\x7e\x37\x95\x6c\x34\xa0\x35\x2b\xa9\xd8\x27\xe8\x90\xe4\x9d\x91\x57\x8c\x28\xa9\xe5\xd2\x48\xc1\x99\x70\x49\xb4\x1a\xee\x5e\xfc\x06\x04\x8e\x86\x1a\x7a\x98\x9e\x0c\x5b\x79\xed\x09\x46\xae\x41\xfc\x20\xe9\x5e\x96\xc7\x1e\x08\xda\xc3\xd8\xa5\x6b\x37\x50\x8d\xb5\xfe\x28\x15\xdd\x45\x5a\x0b\xac\x1f\x0b\xb4\xe5\xb1\x6c\xbb\x32\x5e\x13\xda\x03\xe4\xeb\x9b\x99\x5f\x9c\x45\x25\x0f\x40\x33\x26\xd1\xc9\x7e\x79\xfc\xe3\x02\x6b\x23\xeb\xf7\x4a\xd1\xb0\x33\x7d\x6d\x97\xae\xf0\xdd\xa4\x84\x6b\x97\x13\xa8\x4b\x29\x4d\x56\x8a\xdb\x21\x2c\x6a\x2d\xd2\xc5\xf0\x74\xf4\xf1\xfa\x25\x90\x65\x3a\xd0\xe5\x5b\x5c\x39\x40\x7b\x40\xa4\x2a\xb8\xac\xaa\xf5\x8d\x5b\xde\x51\xe3\xc5\xb3\xa6\x18\x68\x31\x7a\x88\xec\xe9\x68\x34\xa8\xb1\xd5\x0e\x85\xdd\x78\x23\x84\xaa\x3a\x19\x9a\xf9\x96\x55\x4a\xa3\x8d\xc2\x75\x98\xa6\x1d\x55\x3e\xf0\x35\x95\x57\x9b\x09\xa6\x62\xa9\xb0\x36\xca\x12\x63\x55\x28\xa5\x6a\x4c\x7a\xd3\x1d\xe6\x48\x52\x9e\xeb\x15\x56\x40\xdb\x91\xe2\x21\xa6\x41\xad\xe4\x7b\x20\x49\x40\x8f\x2d\xba\x2b\xd8\xae\xfd\x44\x49\xaa\x02\x09\x49\x21\x53\x92\x43\xde\xeb\x3f\x87\xae\xd5\xb3\x06\x9a\x9e\x26\x0a\x9b\x87\xf9\xdb\x15\x68\x8d\xdb\x3a\xb1\xbf\x77\x03\x55\xed\xfa\xc1\xb6\x88\x24\x56\x31\xb3\x9d\x70\x2e\x09\x76\x47\x86\x17\x47\x74\xbb\x12\x6b\x4e\x3d\x2a\x86\x27\xcd\xe6\x25\x5e\x00\xd7\x33\x50\xb3\x20\xbc\x40\x4f\xc3\x08\x8d\xd1\x5d\xbe\xf1\xa8\xf9\xc9\xc6\x2f\x9b\x9f\xa1\x5f\x1d\x28\x69\x5d\xab\xd6\xd9\x40\xdb\x05\x95\x15\x76\xaf\xff\xe6\xe2\x6a\x36\x7f\xfb\xee\xe6\x62\x3e\x9d\xe5\x82\xd5\xae\x5f\x8e\x79\x78\x42\x88\x6b\x0f\x3a\x36\x3f\xb6\x0f\xe0\x9c\xbb\x10\x0d\x82\x80\xee\x52\x57\x85\x05\x2e\x81\xb6\xd3\xc5\xac\xb1\xb5\xff\xdd\x4f\x6f\xfd\xc3\x76\xeb\x35\x97\xdb\xaf\xbc\xf2\x5a\xb1\x0d\x36\xf0\xcb\xce\xd4\x0d\x07\xad\x5c\xbd\xe6\xf7\x9b\x3e\xd7\x47\x83\x48\x1c\x8f\xdf\xe3\xf0\x34\x9e\x41\x87\x69\x60\xd3\x9f\x1c\x9c\xb7\x1c\x7c\xe9\xfb\xd3\x97\x03\xbd\x8d\xf6\xbd\x74\xd3\xcc\xb7\xa3\xd9\x76\x0a\xb3\xdb\xe6\x44\xfa\x70\x93\x0a\xdf\x45\x10\xe9\xa9\x78\xc5\x59\xb9\x32\xe1\x25\xb6\x83\xdf\xd8\x70\xf5\x83\xca\x46\x72\x5b\x25\x73\x0c\xba\x15\xb8\x62\xc4\x07\x54\x17\x2d\x98\x28\x43\x7d\x42\x63\xc8\xff\x6f\x00\x00\x00\xff\xff\xce\x50\x55\x89\x34\x1a\x00\x00") +var _masterEtcOriginMasterMasterConfigYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xc4\x59\x79\x6f\x1b\xb9\x15\xff\x5f\x9f\x82\x08\x16\x48\x52\x74\x66\x24\x3b\xe7\x00\x45\xa1\xda\xce\x46\x58\x3b\x51\x65\xa7\xd8\xa2\x2e\x16\x14\xf9\x34\x62\xc4\x21\x27\x3c\x14\x2b\x6e\xbe\x7b\xc1\x63\x4e\x49\x49\x9a\x4d\xb1\xde\x45\xe2\xe1\x3b\xf8\xf8\xde\x8f\xef\x60\x30\x2d\x99\xd6\x4c\x8a\x33\x29\x56\xac\xc8\x47\x08\x55\xdc\x16\xac\xf3\x8d\xd0\xdf\x2c\xe3\xf4\x1c\x56\xd8\x72\xa3\xc3\x12\x42\xc4\x33\x58\x85\x0d\x93\xa2\x5e\x44\x08\x57\xec\x1f\xa0\x9c\xc6\x1c\x6d\x27\xcd\x32\x88\x6d\x8e\xfe\xf5\xef\xe6\x7b\xc3\x04\xcd\xfb\x8a\xc3\x8e\x0d\x87\x02\x2d\xad\x22\xa0\x5b\xdd\x08\x71\x56\x32\xa3\x73\x74\xff\xb9\xb3\xa8\xe0\x83\x05\xdd\x59\xf6\x6a\xdf\x6e\x41\x29\x46\xe1\x3b\x0d\xee\x18\xd8\x68\xea\x58\x38\x97\x74\xae\x40\x83\xf9\x3e\xed\x94\x69\xbc\xe4\x90\xa3\x15\xe6\x1a\x06\x9b\x46\x87\x4c\xfb\xa1\xf1\x4c\xb2\x02\xa1\xd7\x6c\x65\x52\x26\xb3\x59\x89\x0b\x98\x4b\xce\xc8\xee\x3b\x83\x72\x07\xc4\x3a\xce\x85\xe5\x5d\x3f\x27\xa8\xc4\x86\xac\xbd\xfe\xa9\x10\xd2\x78\x75\xbd\x40\x24\x68\x03\xbb\x1c\x31\xc7\xa2\xd3\x9e\x59\x14\xc4\x2e\x69\x54\x77\x64\x10\xda\x62\x6e\x21\x47\x0f\x8d\xb2\xf0\xb0\x43\x11\xb8\x84\xbc\x35\x27\xa1\x20\x18\xd0\x0e\x83\x14\x8b\x43\x70\x48\x1a\x94\xe4\xa8\x92\x54\x1f\x21\x2d\x5d\x14\x75\x0f\x31\xef\x81\x98\x1c\x39\x3b\x3a\xcb\x7a\xc3\xaa\xb7\x7e\x27\xee\xed\x78\x85\x19\xb7\x0a\x06\x7c\x21\x48\x1d\xe7\xc7\xf8\xe0\xa2\x50\x50\x60\x23\x55\xe7\x2e\x29\x79\xb7\x3b\xe3\x0c\x84\x99\x89\x95\x0c\xb6\x13\x50\xe6\x15\x73\xd1\x6f\x45\x92\x95\x92\xc2\x24\x9e\x3f\x25\xca\x78\xc6\x0d\xec\xbe\xc8\xb7\x81\xdd\x08\x57\xec\x12\xb6\xc0\x75\x3e\x4a\x5c\x6c\x07\xa1\xc6\xd6\xac\x5b\x73\xe2\x4d\x79\x0d\x98\x82\x8a\xc6\x78\xe3\xce\xa6\x39\xea\x68\x4e\x08\x6e\x8c\x88\x0c\xb2\x2c\xa5\x78\x83\xcb\x3a\x00\xc9\x11\xa3\x46\x01\x58\x46\xe1\xb0\xcb\x5c\xc1\x8a\xdd\xb5\x52\xbf\x26\x0b\x28\xa5\x81\xe4\xc2\xf1\x24\x7e\xb5\x50\xd2\x56\x81\x7d\x9f\xef\x67\x47\xf4\x8b\x56\x83\x72\x48\x39\xc6\xf9\x4e\x83\x1a\x11\x29\x8c\x92\x9c\x43\x27\x0a\xc0\x81\xb4\x17\x82\x4b\xb2\x79\xe3\x01\xd7\xc0\x36\x29\xb1\x36\xa0\x92\x56\xd8\xa1\x45\x83\xda\x32\x02\xd7\xee\x2f\x51\x9c\x81\x8a\x97\x5d\xb3\x42\xd4\xee\xeb\x46\x33\xf2\x27\x81\xde\x38\xb0\x13\xc7\x01\x87\x8b\x5f\x67\xcb\x1c\x3d\xfc\xd3\xc3\x11\x91\x4a\x4f\x39\x97\x1f\x81\xbe\x55\xac\x60\xc2\x47\xf6\xd1\x5f\xd9\xe3\x2c\x9b\x9c\x3c\xbf\x4d\xc7\xfe\xff\xc9\xa3\xfc\x3f\xb7\x9f\x1e\x37\x24\x2e\x09\xe6\x6b\xa9\xcd\x60\xfd\xfe\x1e\xfd\xdd\x4a\x03\x57\x60\x30\x7a\xc4\x04\x85\x3b\x94\x5e\xf9\xe3\xa6\xb3\xb9\x46\xe3\xc7\xe9\xb5\x51\x4c\x14\xe8\xf3\xe7\x81\xe8\xc6\x2e\x41\x09\x30\xa0\x6f\x53\x1a\x72\xd2\xd7\x39\x6e\x53\xbd\x25\xb7\x29\xe1\xd6\x6d\x71\x9b\x7a\xbb\x8e\x8a\x7d\xc9\xd8\xf4\xe2\xce\xb8\x80\xf3\x60\xed\x6b\xa9\x8d\x8b\xfe\xbe\x9d\x4d\x18\x8f\x99\xd9\x57\x1b\x0f\xff\xbf\xe8\xf3\x87\xfa\xc6\xb3\x0f\xd8\x26\xcf\x4f\x6e\xd3\xd3\xc3\x31\x3b\xb2\xd1\x57\xbc\xd7\x48\xc5\x75\x2a\x74\x0b\xf5\x25\x13\x74\x4a\xa9\x02\xad\x73\x34\x4e\xfd\x7f\xf9\x8b\xf1\xd3\xd3\x48\x7b\x03\xe6\xa3\x54\x9b\x1c\x19\x52\x3d\x19\x81\x21\xb4\x9f\x9d\x08\xce\x51\xb8\x0c\xa9\x23\xb6\x89\xa0\x85\x79\x8f\xec\x85\x23\x4b\x03\xf3\x03\x1c\x0e\xea\x08\x59\xc5\xfd\xb5\x4d\xd0\xda\x98\x4a\xe7\x3e\x34\x07\x02\x92\x9f\x9c\x3e\x7f\xe9\xad\xbb\x36\x52\xe1\x02\xda\x03\xb6\x6e\x8f\xa4\x90\x60\xf2\x0e\x21\x65\xf2\x10\x63\xbf\x02\x3a\x2f\x5e\x3b\x2f\x0e\xd4\x74\x4b\xd9\x01\xb6\xae\x12\x5f\xfc\x5a\xcb\x56\x52\x95\xd8\xe4\x68\x76\x35\xfd\xf9\xe2\xb7\xf9\xe2\xe2\xd5\xec\xd7\x2c\x7c\xdc\xfc\x73\x7e\x91\xfc\x74\x4f\x64\x59\x49\x01\xc2\x7c\xce\x7f\xba\xdf\x06\x4d\xae\x63\xe1\xd8\x80\x36\x75\x33\xc0\x86\x95\xc5\x29\x67\x22\xdc\x85\x05\x14\x4c\x1b\xb5\xab\x9d\x95\x23\x2a\xc9\x06\x54\xa2\x22\xa1\x46\x92\x03\x52\xfe\x74\x3c\x1e\x8f\x42\xbd\x0a\x4e\x8e\xa5\xca\xf9\x86\x83\xd9\x0f\x3d\xc1\xc9\xd2\x0a\xca\xe1\x58\xd4\xa3\xe4\x97\x03\x3f\x60\x0a\xb1\xaf\xa4\x32\x39\x9a\x8c\x4f\x9e\x8e\x47\x6d\x6c\xba\x66\x39\x23\x70\xc5\x5c\xbe\x05\x35\x55\x85\x2d\x41\xd4\xfd\xa6\xb2\xc2\xb0\x12\x12\xd2\x69\x4b\x13\xc7\xad\x33\x0d\xc6\x30\x51\xe8\x74\xf3\xc2\x85\x3e\xdb\x4e\x30\xaf\xd6\x78\xf2\x97\xa6\x6a\xeb\x10\xbb\x64\x89\xc9\x06\x04\xad\xa5\x1d\xbe\x4e\x7b\x0c\x25\x50\x86\x13\xb3\xab\xa0\xdd\xa1\xe2\x8c\xf8\xfe\x27\xdb\x0a\x9a\x76\x50\x56\x29\x69\xe4\xd2\xae\x62\x95\x94\x96\xba\x0a\xb8\x65\x4d\x69\x4d\xd0\x03\xfc\xc9\x2a\x78\xd0\xe1\xe8\xdb\xff\x20\x03\x43\x32\xcf\x14\xfe\x4c\x1d\xdd\xf1\xb7\x65\x61\xe0\x88\x98\x1a\x7c\x09\x61\xa2\x48\x5c\x84\x92\x95\xf3\x7e\x4f\xa7\xf4\xa5\x23\x0b\x01\xc9\xc2\x2d\x7e\x70\x50\xc1\x06\x76\xdf\x22\xbf\x81\xdd\x83\xff\xcb\x49\xcb\x88\x00\x2b\x1c\x3c\x9a\x85\xd9\x3c\x47\xf7\xf7\x5f\xab\x58\x1e\x57\xf4\x62\xcb\x7c\x7d\xbf\x61\x25\x48\x6b\x72\x24\x2c\xe7\x5f\xef\xbe\x22\x5a\x63\xc7\xd3\x05\xf4\x3e\xa4\x7b\x4c\x01\xd0\x9a\xac\x81\xda\x5e\x84\xea\x8d\x1b\x52\x00\x76\xd0\x74\xc0\xaf\x0d\x5f\xfa\x5e\xfb\x56\x39\x36\x08\xfa\x8d\xa4\x30\x97\xca\x2c\xb0\x28\x5c\xc3\xfc\xb0\x43\xbb\xb6\x4b\x01\xce\x57\xcf\x4f\xd2\x53\x9f\xe0\xb3\xc9\x33\x47\x77\x6d\x3a\x71\x92\xa1\x4d\x73\xe3\x56\x74\xae\x37\xdb\x03\x08\x62\x4d\xfd\xa5\xc1\xf1\x59\xec\xef\x84\x08\x4d\xd2\x60\x6e\xc2\x84\x40\xe5\xc8\x06\x84\xb9\xd9\x55\x4e\xf1\x37\x5c\x8a\x3f\x77\x79\xe2\xe1\x10\x5a\x5a\xe5\xf2\xdc\x93\xf1\x78\x14\xa7\x95\x5a\xeb\x37\x29\xf5\x42\x1f\x2a\x9d\xa3\x13\xaf\x61\xff\x30\xee\xb7\x98\x4c\x82\xd3\x9a\x74\x7e\x29\x65\xe5\xee\xff\x1f\x70\xdc\x67\xbf\xfb\xb8\xa7\x5e\xc3\xde\x59\xba\xa7\x1d\xf6\xb2\x5e\x61\xb8\x83\x11\x05\x73\xbb\xe4\x8c\xbc\x5b\x5c\xe6\xbd\xea\x7b\xb4\xcb\xca\x3b\xb5\xd9\x61\xd1\x5d\x37\x11\xda\x87\x36\x5f\xc7\x6c\x12\xdb\x8a\xb3\xd9\xf9\xc2\xe5\xf8\x74\x72\xf2\x22\x00\xf3\xc9\x1e\x4f\x6c\x00\x08\xa3\x6a\x9f\x15\x21\xd7\xc2\x06\x84\x5f\x82\x28\xcc\x3a\x47\x2f\x3b\x91\x9e\xcd\x3b\x3b\x45\x4d\xb1\xcb\xc9\x9c\x8b\x0e\x4b\x47\xab\xe7\xfe\x61\x23\xb4\xfe\x0a\xe8\x1a\x9b\xb6\x97\x4a\xe4\x56\x27\xda\x4b\xb6\x57\xad\x7f\xaa\xde\x75\x93\xfd\xc1\x0a\x6b\x0d\xe6\x07\x38\x38\x23\x52\x68\xc9\x21\x1b\xb9\xa9\x08\x7b\xa0\x36\x59\xb4\x04\xb3\x96\x34\x47\xd8\x1a\xd7\x9a\x30\x0a\xc2\x30\xb3\x9b\xc7\x54\xac\xf3\xd1\xfd\x7d\x82\xd8\x0a\xa5\x17\x02\x2f\x39\x4c\xa7\xe7\x53\x6b\xd6\x8e\x2b\x00\xcd\xe7\xcb\x24\x4e\xdb\x53\x97\x85\xd1\xf4\x3c\x40\x73\x8d\x39\x07\x9f\x6b\xda\x17\x09\x2e\x0b\x26\x3a\xc3\x6f\x89\xab\x8a\x89\xe2\x2a\x9a\x41\x38\x66\xa5\x27\xf4\x8b\xc1\x91\xe7\x86\xd0\x86\xbc\xad\x40\xcc\xce\x67\x03\xd3\xeb\x51\x2a\xa4\xea\x73\x9f\xf9\x53\x6f\x60\x38\x7f\x3a\x9d\x9e\xc7\x3c\x7e\x1e\xb2\x7e\xcb\x7e\x0d\x44\xb9\x74\x78\x54\x24\x30\x74\xc5\x30\x2b\x3b\xcf\x08\x8c\x76\x5f\x3e\xb4\x5d\x36\x5f\x95\x82\x15\x28\x05\xf4\x5d\x9c\x3c\xbb\x8c\x56\xb0\x0f\x16\x7e\x73\xcb\xcd\xea\x90\xa7\x47\x84\x12\x33\xde\xa5\xfa\x85\xf8\x5d\x77\xc6\xd1\x81\xd6\xac\xa5\x62\x9f\xa0\x45\x92\x0f\x46\x5a\x32\xa2\xa4\x96\x2b\x23\x05\x67\xc2\x15\xd1\x32\x1b\x1e\xfc\x06\x04\x8e\x8e\xca\x3c\x4c\x4f\xb2\x46\x5f\xb3\x83\x91\x1b\x10\x3f\x48\xbb\xd7\xe5\xb1\x07\x82\xf6\x30\x76\xe9\xc6\x17\x54\x61\xad\x3f\x4a\x45\x87\x48\x6b\x80\xf5\x63\x81\xb6\x3a\x56\x6d\xd7\xc6\x5b\x42\x7b\x80\x7c\x7d\x33\xf7\x8b\xf3\x68\xe4\x01\x68\xc6\x22\x3a\xdd\x6f\x8f\x7f\x5c\x62\xad\x75\xfd\x5e\x2d\x1a\x06\xaf\xb9\xcd\xd2\x15\xbe\x9b\x16\x70\xed\x6a\x02\x75\x25\xa5\xae\x4a\x91\x1c\xd2\xa2\xd6\xa2\xbb\x18\xae\x8e\x3e\xde\xbf\x04\xb6\x44\x07\xbe\x74\x87\x4b\x07\x68\x0f\x88\xae\x09\xae\xaa\x6a\x7d\xe3\x96\x07\x66\xbc\x78\x56\x37\x03\x0d\x46\x0f\xb1\x3d\x1d\x8f\x47\x15\xb6\xda\xa1\xb0\x7d\x2e\x09\xa9\xaa\x1a\x8c\x4a\x4b\x29\x8d\x36\x0a\x57\x61\x86\x3a\x6a\x7c\x90\xab\x3b\xaf\xa6\x12\xcc\xc4\x4a\x61\x6d\x94\x25\xc6\xaa\xd0\x4a\x55\x98\xf4\x5e\x8b\x98\x63\xe9\xca\x5c\xaf\xb1\x02\xda\x3c\x51\x1e\x12\x1a\x55\x4a\xbe\x07\xd2\x49\xe8\x71\x50\x73\x0d\xdb\xb5\x7f\xa1\x92\x2a\x47\x42\x52\x48\x94\xe4\x90\xf6\xe6\xd9\xcc\x8d\x8e\xd6\x40\x3d\xd3\x44\x65\x8b\xf0\x9e\x77\x05\x5a\xe3\xa6\x4f\xec\xd3\x6e\xa0\xac\xdc\x7c\xd9\x34\x91\xc4\x2a\x66\x76\x53\xce\x25\xc1\x6e\xcb\x70\xe3\x88\x6e\x56\x62\xcf\xa9\xc7\x79\x76\x52\x13\x2f\xf1\x12\xb8\x9e\x83\x9a\x07\xe5\x39\x7a\x1a\x9e\xe4\x18\x1d\xca\x4d\xc6\xf5\x4f\x32\x79\x59\xff\x64\x7e\x75\xa4\xa4\x75\xa3\x5a\xeb\x03\x6d\x97\x54\x96\xd8\xdd\xfe\x9b\x8b\xab\xf9\xe2\xed\xbb\x9b\x8b\xc5\x6c\x9e\x0a\x56\xb9\xf9\x3b\xd6\xe1\x29\x21\x6e\x3c\x68\xc5\xfc\x3f\x03\x04\x70\x2e\x5c\x8a\x06\x41\x40\xb7\xa5\xab\xc4\x02\x17\x40\x9b\xd7\xca\xa4\xf6\xb5\xff\xdd\xbf\x06\xfb\x8b\xed\xd6\x2b\x2e\x77\x5f\xb9\xe5\x95\x62\x5b\x6c\xe0\x97\xc1\x2b\x1e\x0e\x56\xb9\x7e\xcd\xd3\xeb\x39\xd7\x67\x83\xc8\x1c\xb7\xdf\x93\xf0\x3c\x5e\x40\x87\xd7\xc5\x7a\x3e\x39\xf8\x7e\x73\xf0\xa6\xef\xbf\xe6\x1c\x98\x6d\xb4\x9f\xa5\xeb\x61\xbe\x79\xea\x6d\x5e\x75\x86\x63\x4e\xe4\x0f\x27\x29\xf1\x5d\x04\x91\x9e\x89\x57\x9c\x15\x6b\x13\x6e\x62\xf3\x90\x1c\x07\xae\x7e\x52\xd9\x4a\x6e\xcb\xce\xbb\x08\xdd\x09\x5c\x32\xe2\x13\xaa\xcb\x16\x4c\x14\xa1\x3f\xa1\x31\xe5\xff\x37\x00\x00\xff\xff\x96\xdf\x75\xfd\x84\x1a\x00\x00") func masterEtcOriginMasterMasterConfigYamlBytes() ([]byte, error) { return bindataRead( diff --git a/pkg/openshift/certgen/release39/templates/master/etc/origin/master/master-config.yaml b/pkg/openshift/certgen/release39/templates/master/etc/origin/master/master-config.yaml index 3fbcb7579d..9a62edd9dc 100644 --- a/pkg/openshift/certgen/release39/templates/master/etc/origin/master/master-config.yaml +++ b/pkg/openshift/certgen/release39/templates/master/etc/origin/master/master-config.yaml @@ -89,6 +89,8 @@ etcdStorageConfig: imageConfig: format: IMAGE_PREFIX/IMAGE_TYPE-${component}:${version} latest: false +imagePolicyConfig: + internalRegistryHostname: docker-registry.default.svc:5000 kind: MasterConfig kubeletClientInfo: ca: ca-bundle.crt diff --git a/pkg/openshift/certgen/unstable/templates/bindata.go b/pkg/openshift/certgen/unstable/templates/bindata.go index 6610509717..0046d9424c 100644 --- a/pkg/openshift/certgen/unstable/templates/bindata.go +++ b/pkg/openshift/certgen/unstable/templates/bindata.go @@ -118,7 +118,7 @@ func masterEtcOriginMasterHtpasswd() (*asset, error) { return a, nil } -var _masterEtcOriginMasterMasterConfigYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xc4\x58\x7b\x6f\x1b\xb9\x11\xff\x5f\x9f\x82\x08\x0e\x48\x52\x74\x77\x65\x3b\xcf\x05\x8a\x42\xb5\x9d\x8b\x70\x76\xa2\xda\x4e\x71\x45\x5d\x1c\x28\x72\xb4\x62\xc4\x25\x37\x7c\x28\x56\xdc\x7c\xf7\x82\x8f\xdd\xe5\xea\x91\xa4\xb9\xa0\xe7\x04\xb0\x97\x9c\x19\x0e\x67\x7e\x9c\x17\xa6\x35\xd3\x9a\x49\x71\x2a\xc5\x82\x55\xe5\x08\xa1\x86\xdb\x8a\x25\xdf\x08\xfd\xcd\x32\x4e\xcf\x60\x81\x2d\x37\x3a\x2c\x21\x44\x3c\x81\x55\xd8\x30\x29\xda\x45\x84\x70\xc3\xfe\x01\xca\x49\x2c\xd1\xfa\xa8\x5b\x06\xb1\x2e\xd1\xbf\xfe\xdd\x7d\xaf\x98\xa0\xe5\x50\x70\x38\xb1\xa3\x50\xa0\xa5\x55\x04\x74\x2f\x1b\x21\xce\x6a\x66\x74\x89\xee\x3f\x27\x8b\x0a\x3e\x58\xd0\xc9\xb2\x17\xfb\x76\x0d\x4a\x31\x0a\xdf\xa9\x70\xa2\x60\x27\x29\xd1\x70\x26\xe9\x4c\x81\x06\xf3\x7d\xd2\x29\xd3\x78\xce\xa1\x44\x0b\xcc\x35\x6c\x1d\x1a\x0d\x32\x19\xba\xc6\x13\xc9\x06\x84\x5e\xb2\x85\xc9\x99\x2c\xa6\x35\xae\x60\x26\x39\x23\x9b\xef\x74\xca\x1d\x10\xeb\x28\xaf\x2c\x4f\xed\x9c\xa1\x1a\x1b\xb2\xf4\xf2\x27\x42\x48\xe3\xc5\x0d\x1c\x91\xa1\x15\x6c\x4a\xc4\x1c\x89\xce\x07\x6a\x51\x10\x9b\xac\x13\x9d\xf0\x20\xb4\xc6\xdc\x42\x89\x1e\x1a\x65\xe1\x61\xb2\x23\x70\x0d\x65\xaf\x4e\x46\x41\x30\xa0\x09\x81\x14\x57\xfb\xe0\x90\x75\x28\x29\x51\x23\xa9\x3e\xb0\x35\x77\x5e\xd4\x03\xc4\xbc\x07\x62\x4a\xe4\xf4\x48\x96\xf5\x8a\x35\x6f\xfd\x49\xdc\xeb\xf1\x0a\x33\x6e\x15\x6c\xd1\x05\x27\x25\xc6\x8f\xfe\xc1\x55\xa5\xa0\xc2\x46\xaa\xe4\x2d\x29\x79\xb7\x39\xe5\x0c\x84\x99\x8a\x85\x0c\xba\x13\x50\xe6\x15\x73\xde\xef\x59\xb2\x85\x92\xc2\x64\x9e\x3e\x27\xca\x78\xc2\x15\x6c\xbe\x48\xb7\x82\xcd\x08\x37\xec\x02\xd6\xc0\x75\x39\xca\x9c\x6f\xb7\x5c\x8d\xad\x59\xf6\xea\xc4\x97\xf2\x1a\x30\x05\x15\x95\xf1\xca\x9d\x4e\x4a\x94\x48\xce\x08\xee\x94\x88\x04\xb2\xae\xa5\x78\x83\xeb\xd6\x01\xd9\x01\xa5\x46\x01\x58\x46\xe1\x70\xca\x4c\xc1\x82\xdd\xf5\x5c\xbf\x66\x57\x50\x4b\x03\xd9\xb9\xa3\xc9\xfc\x6a\xa5\xa4\x6d\x02\xf9\x2e\xdd\xcf\x6e\xd3\x2f\x5a\x0d\xca\x21\xe5\x10\xe5\x3b\x0d\x6a\x44\xa4\x30\x4a\x72\x0e\x89\x17\x80\x03\xe9\x1f\x04\x97\x64\xf5\xc6\x03\xae\x83\x6d\x56\x63\x6d\x40\x65\x3d\xb3\x43\x8b\x06\xb5\x66\x04\xae\xdd\x2f\x51\x9d\x82\x8a\x8f\x5d\xb3\x4a\xb4\xe6\x4b\xbd\x19\xe9\xb3\xb0\xdf\x19\x30\xf1\xe3\x16\x85\xf3\x5f\x72\x64\x89\x1e\xfe\xe9\xe1\x88\x48\xa5\x27\x9c\xcb\x8f\x40\xdf\x2a\x56\x31\xe1\x3d\xfb\xe8\xaf\xec\x71\x51\x1c\x1d\x3f\xbf\xcd\xc7\xfe\xff\xd1\xa3\xf2\x3f\xb7\x9f\x1e\x77\x5b\x5c\x12\xcc\x97\x52\x9b\xad\xf5\xfb\x7b\xf4\x77\x2b\x0d\x5c\x82\xc1\xe8\x11\x13\x14\xee\x50\x7e\xe9\xaf\x9b\x4f\x67\x1a\x8d\x1f\xe7\xd7\x46\x31\x51\xa1\xcf\x9f\xb7\x58\x57\x76\x0e\x4a\x80\x01\x7d\x9b\xd3\x10\x93\xbe\x4e\x71\x9b\xeb\x35\xb9\xcd\x09\xb7\xee\x88\xdb\xdc\xeb\x75\x90\xed\x4b\xca\xe6\xe7\x77\xc6\x39\x9c\x07\x6d\x5f\x4b\x6d\x9c\xf7\x77\xf5\xec\xdc\x78\x48\xcd\xa1\xd8\x78\xf9\xff\x45\x9e\xbf\xd4\x37\xde\x7d\x8b\xec\xe8\xf9\xf1\x6d\x7e\xb2\xdf\x67\x07\x0e\xfa\x8a\xf5\x3a\xae\xb8\x4e\x85\xee\xa1\x3e\x67\x82\x4e\x28\x55\xa0\x75\x89\xc6\xb9\xff\x57\xbe\x18\x3f\x3d\x89\x7b\x6f\xc0\x7c\x94\x6a\x55\x22\x43\x9a\x27\x23\x30\x84\x0e\xa3\x13\xc1\x25\x0a\x8f\x21\x77\x9b\x7d\x20\xe8\x61\x3e\xd8\xf6\xcc\x91\xa4\x83\xf9\x1e\x0a\x07\x75\x84\xac\xe2\xfe\xd9\x66\x68\x69\x4c\xa3\x4b\xef\x9a\x3d\x0e\x29\x8f\x4f\x9e\xbf\xf4\xda\x5d\x1b\xa9\x70\x05\xfd\x05\x7b\xb3\xc7\xad\x10\x60\xca\x64\x23\x67\x72\x1f\xe1\x30\x03\x3a\x2b\x5e\x3b\x2b\x6e\x89\x49\x53\xd9\x1e\xb2\x54\x88\x4f\x7e\xbd\x66\x0b\xa9\x6a\x6c\x4a\x34\xbd\x9c\xfc\x7c\xfe\xdb\xec\xea\xfc\xd5\xf4\xd7\x22\x7c\xdc\xfc\x73\x76\x9e\xfd\x74\x4f\x64\xdd\x48\x01\xc2\x7c\x2e\x7f\xba\x5f\x07\x49\xae\x62\xe1\xd8\x80\x36\x6d\x31\x10\xf2\x4b\x30\x4a\x4c\x2d\xee\x2e\x1c\xcc\xae\xab\x08\xce\xe6\x56\x50\x0e\x87\xbc\x14\x39\xbf\xec\xa8\x2d\xa2\xe0\xab\x46\x2a\x53\xa2\xa3\xf1\xf1\xd3\xf1\xa8\xb7\x65\xaa\x96\x53\x02\x37\xcc\xc5\x47\x50\x13\x55\xd9\x1a\x44\x5b\x1f\x2a\x2b\x0c\xab\x21\x23\x49\x19\x99\x39\x6a\x5d\x68\x30\x86\x89\x4a\xe7\xab\x17\xce\x55\xc5\xfa\x08\xf3\x66\x89\x8f\xfe\xd2\x65\x59\x1d\x6c\x9d\xcd\x31\x59\x81\xa0\x2d\xb7\xc3\xc3\xc9\x80\xa0\x06\xca\x70\x66\x36\x0d\xf4\x27\x34\x9c\x11\x5f\xaf\x14\x6b\x41\xf3\x04\x15\x8d\x92\x46\xce\xed\x22\x66\x35\x69\xa9\xcb\x58\x6b\xd6\xa5\xc2\x0c\x3d\xc0\x9f\xac\x82\x07\x09\xc5\x50\xff\x07\x05\x18\x52\x48\x1f\x96\x0b\x4f\xd0\x4a\x28\x3c\x67\xee\xc8\x1d\x7b\x1f\xd5\xb7\xec\x12\x5f\xb6\xcf\x00\x4c\x54\x99\x73\x58\xb6\x70\xce\xd8\x77\x44\xf0\x4f\x11\x1e\xe1\x83\xbd\x02\x56\xb0\xf9\x16\xfe\x15\x6c\x1e\xfc\x3f\x2e\x5e\x47\x7c\x58\xe1\xc0\xd3\x2d\x4c\x67\x25\xba\xbf\xff\x5a\xfe\xf1\xa8\xa3\xe7\x6b\xe6\xb3\xf5\x0d\xab\x41\x5a\x53\x22\x61\x39\xff\x7a\x2d\x15\xb1\x1c\xeb\x97\x14\xee\xbb\x80\x1f\x10\x05\xb8\x6b\xb2\x04\x6a\x07\x0e\x6b\x0f\xee\xb6\x02\xec\x83\xa4\x3d\x66\xee\xe8\xf2\xf7\xda\x17\xbe\x31\xdd\xeb\x37\x92\xc2\x4c\x2a\x73\x85\x45\xe5\xca\xdf\x87\xc9\xde\xb5\x9d\x0b\x70\xb6\x7a\x7e\x9c\x9f\xf8\x70\x5d\x1c\x3d\x73\xfb\xae\xe8\x26\x8e\x33\x14\x5d\xae\x79\x8a\xc6\xf5\x6a\x7b\x3c\x41\xcc\x90\xbf\x74\x28\x3f\x8d\xd5\x9a\x10\xa1\xe4\xd9\xea\x82\x30\x21\xd0\xb8\x6d\x03\xc2\xdc\x6c\x1a\x27\xf8\x1b\x9e\xcc\x9f\x53\x9a\x78\x39\x84\xe6\x56\xb9\xa8\xf5\x64\x3c\x1e\xc5\xde\xa3\x95\xfa\x4d\x42\x3d\xd3\x87\x46\x97\xe8\xd8\x4b\xd8\xbd\x8c\xfb\x2b\x86\x9a\x60\xb4\x2e\x38\x5f\x48\xd9\xb8\xe8\xf0\x07\x5c\xf7\xd9\xef\xbe\xee\x89\x97\xb0\x73\x97\xf4\xb6\xdb\x95\xa9\x17\x18\x9e\x64\x44\xc1\xcc\xce\x39\x23\xef\xae\x2e\xca\x41\x2e\x3d\x58\x33\x95\x49\xa6\x75\x58\x74\xcf\x4d\x84\x62\xa0\x8f\xe6\x31\xb8\xc4\x22\xe1\x74\x7a\x76\xe5\x32\x40\x7e\x74\xfc\x22\x00\xf3\xc9\x0e\x4d\x4c\xe7\x84\x51\xb5\x4b\x8a\x90\x2b\x48\x03\xc2\x2f\x40\x54\x66\x59\xa2\x97\x89\xa7\xa7\xb3\xe4\xa4\x28\x29\xd6\x2c\x85\x33\xd1\x7e\xee\xa8\xf5\xcc\x8f\x29\x42\x21\xaf\x80\x2e\xb1\xe9\x2b\xa3\x4c\xae\x75\xa6\x3d\x67\xff\xd4\x86\xb7\x1a\x3c\x37\x39\x6c\x93\xb0\xd6\x60\x7e\x80\x81\x0b\x22\x85\x96\x1c\x8a\x91\xeb\x71\xb0\x07\x6a\x17\x54\x6b\x30\x4b\x49\x4b\x84\xad\x71\x85\x06\xa3\x20\x0c\x33\x9b\x59\x8c\xab\xba\x1c\xdd\xdf\x67\x88\x2d\x50\x7e\x2e\xf0\x9c\xc3\x64\x72\x36\xb1\x66\xe9\xa8\x02\xd0\x7c\xbc\xcc\x62\xef\x3c\x71\x51\x18\x4d\xce\x02\x34\x97\x98\x73\xf0\xb1\xa6\x9f\x2f\x70\x59\x31\x91\xb4\xb2\x35\x6e\x1a\x26\xaa\xcb\xa8\x06\xe1\x98\xd5\x7e\x63\x98\x1b\x0e\x0c\x0f\x42\x91\xf2\xb6\x01\x31\x3d\x9b\x6e\xa9\xde\x36\x46\x21\x54\x9f\xf9\xc8\x9f\x7b\x05\xc3\xfd\xf3\xc9\xe4\x2c\xc6\xf1\xb3\x10\xf5\x7b\xf2\x6b\x20\xca\x85\xc3\x83\x2c\x81\x20\x65\xc3\xac\x4e\x86\x02\x8c\xa6\x73\x0c\x6d\xe7\xdd\x57\xa3\x60\x01\x4a\x01\x7d\x17\xfb\xc8\x94\xd0\x0a\xf6\xc1\xc2\x6f\x6e\xb9\x5b\xdd\xa6\x19\x6c\x42\x8d\x19\x4f\x77\xfd\x42\xfc\x6e\xeb\xdc\x68\x40\x6b\x96\x52\xb1\x4f\xd0\x23\xc9\x3b\x23\xaf\x19\x51\x52\xcb\x85\x91\x82\x33\xe1\x92\x68\x5d\x6c\x5f\xfc\x06\x04\x8e\x86\x2a\x3c\x4c\x8f\x8b\x4e\x5e\x77\x82\x91\x2b\x10\x3f\x48\xba\x97\xe5\xb1\x07\x82\x0e\x30\x76\xe1\x9a\x11\xd4\x60\xad\x3f\x4a\x45\xb7\x91\xd6\x01\xeb\xc7\x02\x6d\x71\x28\xdb\x2e\x8d\xd7\x84\x0e\x00\xf9\xfa\x66\xe6\x17\x67\x51\xc9\x3d\xd0\x8c\x49\x74\xb2\x5b\x3c\xff\xb8\xc0\xda\xca\xfa\xbd\x52\x34\x6c\xcd\x66\xbb\xa5\x4b\x7c\x37\xa9\xe0\xda\xe5\x04\xea\x52\x4a\x9b\x95\xe2\x76\x08\x8b\x5a\x8b\x74\x31\x3c\x1d\x7d\xb8\x7e\x09\x64\x99\x0e\x74\xf9\x06\xd7\x0e\xd0\x1e\x10\xa9\x0a\x2e\xab\x6a\x7d\xe3\x96\xb7\xd4\x78\xf1\xac\x2d\x06\x3a\x8c\xee\x23\x7b\x3a\x1e\x8f\x1a\x6c\xb5\x43\x61\x3f\xfc\x08\xa1\xaa\x49\x46\x6a\xbe\xa1\x95\xd2\x68\xa3\x70\x13\x66\x6d\x07\x95\x0f\x7c\x6d\xe5\xd5\x65\x82\xa9\x58\x28\xac\x8d\xb2\xc4\x58\x15\x4a\xa9\x06\x93\xc1\xec\x87\x39\x92\x94\xe7\x7a\x89\x15\xd0\x6e\xe0\xb8\x8f\x69\xd4\x28\xf9\x1e\x48\x12\xd0\x63\x03\xef\x0a\xb6\x6b\x3f\x6f\x92\xaa\x44\x42\x52\xc8\x94\xe4\x90\x0f\xba\xd3\xc2\x35\x82\xd6\x40\xdb\xf1\x44\x61\x57\x61\x3a\x77\x09\x5a\xe3\xae\x4e\x1c\xee\xdd\x40\xdd\xb8\x6e\xb1\x2b\x22\x89\x55\xcc\x6c\x26\x9c\x4b\x82\xdd\x91\xe1\xc5\x11\xdd\xad\xc4\x9a\x53\x8f\xcb\xe2\xb8\xdd\xbc\xc0\x73\xe0\x7a\x06\x6a\x16\x84\x97\xe8\x69\x18\xb0\x31\xba\xcd\x77\x34\x6e\x7f\xb2\xa3\x97\xed\x4f\xe1\x57\x47\x4a\x5a\xd7\xc8\xf5\x36\xd0\x76\x4e\x65\x8d\xdd\xeb\xbf\x39\xbf\x9c\x5d\xbd\x7d\x77\x73\x7e\x35\x9d\xe5\x82\x35\xae\x9b\x8e\x79\x78\x42\x88\x6b\x0f\x7a\x36\x3f\xd4\x0f\xe0\xbc\x72\x21\x1a\x04\x01\xdd\xa7\xae\x1a\x0b\x5c\x01\xed\x66\x8f\x59\x6b\x6b\xff\xb7\x9f\xed\xfa\x87\xed\xd6\x1b\x2e\x37\x5f\x79\xe5\x8d\x62\x6b\x6c\xe0\x97\xad\x99\x1c\x0e\x5a\xb9\x7a\xcd\xef\xb7\x5d\xb0\x8f\x06\x91\x38\x1e\xbf\xc3\xe1\x69\x3c\x83\x0e\xb3\xc2\xb6\x3f\xd9\x3b\x8d\xd9\xfb\xd2\x77\x67\x33\x7b\x7a\x1b\xed\x3b\xed\xb6\xd5\xef\x06\xb7\xdd\x8c\x66\xbb\xcd\x89\xf4\xe1\x26\x35\xbe\x8b\x20\xd2\x53\xf1\x8a\xb3\x6a\x69\xc2\x4b\xec\xc6\xc2\xb1\xe1\x1a\x06\x95\xb5\xe4\xb6\x4e\xa6\x1c\x74\x23\x70\xcd\x88\x0f\xa8\x2e\x5a\x30\x51\x85\xfa\x84\xc6\x90\xff\xdf\x00\x00\x00\xff\xff\xa5\x5d\xc4\x11\x52\x1a\x00\x00") +var _masterEtcOriginMasterMasterConfigYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xc4\x59\x7b\x6f\x1b\xb9\x11\xff\x5f\x9f\x82\x08\x0e\x48\x52\x74\x77\x25\x3b\xcf\x05\x8a\x42\xb5\x9d\x8b\x70\x76\xa2\xca\x4e\x71\x45\x5d\x1c\x28\x72\xb4\x62\xc4\x25\x37\x7c\x28\x56\xdc\x7c\xf7\x82\x8f\x7d\x4a\x4a\xd2\x5c\xd0\x73\x02\xc4\x4b\xfe\x66\x38\xe4\xfc\x38\x0f\x06\xd3\x92\x69\xcd\xa4\x38\x93\x62\xc5\x8a\x7c\x84\x50\xc5\x6d\xc1\x3a\xdf\x08\xfd\xcd\x32\x4e\xcf\x61\x85\x2d\x37\x3a\x0c\x21\x44\x3c\xc0\x2a\x6c\x98\x14\xf5\x20\x42\xb8\x62\xff\x00\xe5\x34\xe6\x68\x3b\x69\x86\x41\x6c\x73\xf4\xaf\x7f\x37\xdf\x1b\x26\x68\xde\x57\x1c\x56\x6c\x10\x0a\xb4\xb4\x8a\x80\x6e\x75\x23\xc4\x59\xc9\x8c\xce\xd1\xfd\xe7\xce\xa0\x82\x0f\x16\x74\x67\xd8\xab\x7d\xbb\x05\xa5\x18\x85\xef\x34\xb8\x63\x60\xa3\xa9\x63\xe1\x5c\xd2\xb9\x02\x0d\xe6\xfb\xb4\x53\xa6\xf1\x92\x43\x8e\x56\x98\x6b\x18\x2c\x1a\x0f\x64\xda\x77\x8d\x07\xc9\x0a\x84\x5e\xb3\x95\x49\x99\xcc\x66\x25\x2e\x60\x2e\x39\x23\xbb\xef\x74\xca\x1d\x10\xeb\x90\x0b\xcb\xbb\xe7\x9c\xa0\x12\x1b\xb2\xf6\xfa\xa7\x42\x48\xe3\xd5\xf5\x1c\x91\xa0\x0d\xec\x72\xc4\x1c\x44\xa7\x3d\xb3\x28\x88\x5d\xd2\xa8\xee\xc8\x20\xb4\xc5\xdc\x42\x8e\x1e\x1a\x65\xe1\x61\x67\x46\xe0\x12\xf2\xd6\x9c\x84\x82\x60\x40\x3b\x00\x29\x16\x87\xe8\x90\x34\x2c\xc9\x51\x25\xa9\x3e\x32\xb5\x74\x5e\xd4\x3d\xc6\xbc\x07\x62\x72\xe4\xec\xe8\x0c\xeb\x0d\xab\xde\xfa\x95\xb8\xb7\xe3\x15\x66\xdc\x2a\x18\xe0\x82\x93\x3a\x87\x1f\xfd\x83\x8b\x42\x41\x81\x8d\x54\x9d\xbb\xa4\xe4\xdd\xee\x8c\x33\x10\x66\x26\x56\x32\xd8\x4e\x40\x99\x57\xcc\x79\xbf\x15\x49\x56\x4a\x0a\x93\x78\x7c\x4a\x94\xf1\xc0\x0d\xec\xbe\x88\xdb\xc0\x6e\x84\x2b\x76\x09\x5b\xe0\x3a\x1f\x25\xce\xb7\x03\x57\x63\x6b\xd6\xad\x39\xf1\xa6\xbc\x06\x4c\x41\x45\x63\xbc\x71\x67\xd3\x1c\x75\x34\x27\x04\x37\x46\x44\x80\x2c\x4b\x29\xde\xe0\xb2\x76\x40\x72\xc4\xa8\x51\x20\x96\x51\x38\xac\x32\x57\xb0\x62\x77\xad\xd4\xaf\xc9\x02\x4a\x69\x20\xb9\x70\x98\xc4\x8f\x16\x4a\xda\x2a\xc0\xf7\x71\x3f\xbb\x49\x3f\x68\x35\x28\xc7\x94\x63\xc8\x77\x1a\xd4\x88\x48\x61\x94\xe4\x1c\x3a\x5e\x00\x0e\xa4\xbd\x10\x5c\x92\xcd\x1b\x4f\xb8\x86\xb6\x49\x89\xb5\x01\x95\xb4\xc2\x8e\x2d\x1a\xd4\x96\x11\xb8\x76\xff\x88\xe2\x0c\x54\xbc\xec\x9a\x15\xa2\x3e\xbe\xae\x37\x23\x3e\x09\xf3\xcd\x01\x76\xfc\x38\x40\x38\xff\x75\x96\xcc\xd1\xc3\x3f\x3d\x1c\x11\xa9\xf4\x94\x73\xf9\x11\xe8\x5b\xc5\x0a\x26\xbc\x67\x1f\xfd\x95\x3d\xce\xb2\xc9\xc9\xf3\xdb\x74\xec\xff\x4e\x1e\xe5\xff\xb9\xfd\xf4\xb8\x99\xe2\x92\x60\xbe\x96\xda\x0c\xc6\xef\xef\xd1\xdf\xad\x34\x70\x05\x06\xa3\x47\x4c\x50\xb8\x43\xe9\x95\xdf\x6e\x3a\x9b\x6b\x34\x7e\x9c\x5e\x1b\xc5\x44\x81\x3e\x7f\x1e\x88\x6e\xec\x12\x94\x00\x03\xfa\x36\xa5\x21\x26\x7d\x1d\x71\x9b\xea\x2d\xb9\x4d\x09\xb7\x6e\x89\xdb\xd4\xdb\x75\x54\xec\x4b\xc6\xa6\x17\x77\xc6\x39\x9c\x07\x6b\x5f\x4b\x6d\x9c\xf7\xf7\xed\x6c\xdc\x78\xcc\xcc\xbe\xda\xb8\xf9\xff\x45\x9f\xdf\xd4\x37\xee\x7d\x00\x9b\x3c\x3f\xb9\x4d\x4f\x0f\xfb\xec\xc8\x42\x5f\x39\xbd\x46\x2a\x8e\x53\xa1\x5b\xaa\x2f\x99\xa0\x53\x4a\x15\x68\x9d\xa3\x71\xea\xff\xe4\x2f\xc6\x4f\x4f\xe3\xdc\x1b\x30\x1f\xa5\xda\xe4\xc8\x90\xea\xc9\x08\x0c\xa1\xfd\xe8\x44\x70\x8e\xc2\x65\x48\xdd\x64\x1b\x08\x5a\x9a\xf7\xa6\xbd\x70\x84\x34\x34\x3f\x80\x70\x54\x47\xc8\x2a\xee\xaf\x6d\x82\xd6\xc6\x54\x3a\xf7\xae\x39\xe0\x90\xfc\xe4\xf4\xf9\x4b\x6f\xdd\xb5\x91\x0a\x17\xd0\x6e\xb0\x3d\xf6\x38\x15\x02\x4c\xde\x99\x48\x99\x3c\x04\xec\x67\x40\x77\x8a\xd7\xee\x14\x07\x6a\xba\xa9\xec\x00\xac\xab\xc4\x27\xbf\xd6\xb2\x95\x54\x25\x36\x39\x9a\x5d\x4d\x7f\xbe\xf8\x6d\xbe\xb8\x78\x35\xfb\x35\x0b\x1f\x37\xff\x9c\x5f\x24\x3f\xdd\x13\x59\x56\x52\x80\x30\x9f\xf3\x9f\xee\xb7\x41\x93\xab\x58\x38\x36\xa0\x4d\x5d\x0c\xb0\x61\x66\x71\xca\x99\x08\x77\x61\x01\x05\xd3\x46\xed\xea\xc3\xca\x11\x95\x64\x03\x2a\x51\x71\xa2\x66\x92\x23\x52\xfe\x74\x3c\x1e\x8f\x42\xbe\x0a\x87\x1c\x53\x95\x3b\x1b\x0e\x66\xdf\xf5\x04\x27\x4b\x2b\x28\x87\x63\x5e\x8f\x92\x5f\x76\xfc\x00\x14\x7c\x5f\x49\x65\x72\x34\x19\x9f\x3c\x1d\x8f\x5a\xdf\x74\xcd\x72\x46\xe0\x8a\xb9\x78\x0b\x6a\xaa\x0a\x5b\x82\xa8\xeb\x4d\x65\x85\x61\x25\x24\xa4\x53\x96\x26\x0e\xad\x33\x0d\xc6\x30\x51\xe8\x74\xf3\xc2\xb9\x3e\xdb\x4e\x30\xaf\xd6\x78\xf2\x97\x26\x6b\xeb\xe0\xbb\x64\x89\xc9\x06\x04\xad\xa5\x1d\xbf\x4e\x7b\x80\x12\x28\xc3\x89\xd9\x55\xd0\xae\x50\x71\x46\x7c\xfd\x93\x6d\x05\x4d\x3b\x2c\xab\x94\x34\x72\x69\x57\x31\x4b\x4a\x4b\x5d\x06\xdc\xb2\x26\xb5\x26\xe8\x01\xfe\x64\x15\x3c\xe8\x20\xfa\xf6\x3f\xc8\xc0\x90\x4c\xfa\x30\x9f\x79\x40\xad\x21\xf3\x92\xa9\x83\x3b\xf1\x36\x4b\x0c\xce\x25\x46\x0a\x9f\x51\x98\x28\x12\xe7\xb0\x64\xe5\x9c\x71\x68\x89\xe0\x9f\x2c\x5c\xea\x07\x07\x15\x6c\x60\xf7\x2d\xf2\x1b\xd8\x3d\xf8\x7f\x6c\xbc\x8c\xfc\xb0\xc2\x91\xa7\x19\x98\xcd\x73\x74\x7f\xff\xb5\x7c\xe6\x59\x47\x2f\xb6\xcc\x67\xff\x1b\x56\x82\xb4\x26\x47\xc2\x72\xfe\xf5\xda\x2c\x72\x39\xd6\x43\x5d\xba\xef\x13\xbe\x07\x0a\x74\xd7\x64\x0d\xd4\xf6\x1c\x56\x2f\xdc\x4c\x05\xda\x07\x4d\x07\x8e\xb9\xc1\xa5\xef\xb5\x2f\xa4\x63\xf9\xa0\xdf\x48\x0a\x73\xa9\xcc\x02\x8b\xc2\x95\xd3\x0f\x3b\x73\xd7\x76\x29\xc0\x9d\xd5\xf3\x93\xf4\xd4\x87\xff\x6c\xf2\xcc\xcd\xbb\x22\x9e\x38\xc9\x50\xc4\xb9\x66\x2c\x1e\xae\x37\xdb\xf3\x09\x62\xc6\xfd\xa5\x61\xf9\x59\xac\xfe\x84\x08\x25\xd4\xa0\xab\xc2\x84\x40\xe5\xa6\x0d\x08\x73\xb3\xab\x9c\xe2\x6f\xb8\x32\x7f\xee\x62\xe2\xe6\x10\x5a\x5a\xe5\xa2\xe0\x93\xf1\x78\x14\x7b\x99\x5a\xeb\x37\x29\xf5\x42\x1f\x2a\x9d\xa3\x13\xaf\x61\x7f\x33\xee\xb7\x18\x6a\xc2\xa1\x35\xc1\xfe\x52\xca\xca\x45\x87\x3f\x60\xbb\xcf\x7e\xf7\x76\x4f\xbd\x86\xbd\xbd\x74\x77\x3b\xac\x74\xbd\xc2\x70\x25\x23\x0b\xe6\x76\xc9\x19\x79\xb7\xb8\xcc\x7b\xb9\xf9\x68\x0d\x96\x77\x32\xb7\xe3\xa2\xbb\x6e\x22\x14\x17\x6d\x34\x8f\xc1\x25\x16\x1d\x67\xb3\xf3\x85\xcb\x00\xe9\xe4\xe4\x45\x20\xe6\x93\x3d\x4c\x2c\x0f\x08\xa3\x6a\x1f\x8a\x90\x2b\x70\x03\xc3\x2f\x41\x14\x66\x9d\xa3\x97\x1d\x4f\xcf\xe6\x9d\x95\xa2\xa6\x58\x03\x65\xee\x88\x0e\x4b\x47\xab\xe7\xfe\xd9\x23\x34\x06\x0a\xe8\x1a\x9b\xb6\xd2\x4a\xe4\x56\x27\xda\x4b\xb6\x57\xad\xbf\xab\xde\x75\x93\xfd\xb6\x0b\x6b\x0d\xe6\x07\x1c\x70\x46\xa4\xd0\x92\x43\x36\x72\x3d\x13\xf6\x44\x6d\x82\x6a\x09\x66\x2d\x69\x8e\xb0\x35\xae\x70\x61\x14\x84\x61\x66\x37\x8f\x71\x55\xe7\xa3\xfb\xfb\x04\xb1\x15\x4a\x2f\x04\x5e\x72\x98\x4e\xcf\xa7\xd6\xac\x1d\x2a\x10\xcd\xc7\xcb\x24\xf6\xe2\x53\x17\x85\xd1\xf4\x3c\x50\x73\x8d\x39\x07\x1f\x6b\xda\xf7\x0a\x2e\x0b\x26\x3a\xad\x71\x89\xab\x8a\x89\xe2\x2a\x9a\x41\x38\x66\xa5\x9f\xe8\xe7\x86\x23\x8f\x11\xa1\x48\x79\x5b\x81\x98\x9d\xcf\x06\xa6\xd7\x8d\x56\x08\xd5\xe7\x3e\xf2\xa7\xde\xc0\xb0\xff\x74\x3a\x3d\x8f\x71\xfc\x3c\x44\xfd\x16\x7e\x0d\x44\xb9\x70\x78\x54\x24\x00\xba\x62\x98\x95\x9d\x47\x06\x46\xbb\xef\x22\xda\x2e\x9b\xaf\x4a\xc1\x0a\x94\x02\xfa\x2e\xf6\xa5\x5d\xa0\x15\xec\x83\x85\xdf\xdc\x70\x33\x3a\xc4\xf4\x26\xa1\xc4\x8c\x77\x67\xfd\x40\xfc\xae\xeb\xe6\x78\x80\xd6\xac\xa5\x62\x9f\xa0\x65\x92\x77\x46\x5a\x32\xa2\xa4\x96\x2b\x23\x05\x67\xc2\x25\xd1\x32\x1b\x6e\xfc\x06\x04\x8e\x07\x95\x79\x9a\x9e\x64\x8d\xbe\x66\x05\x23\x37\x20\x7e\x90\x76\xaf\xcb\x73\x0f\x04\xed\x71\xec\xd2\x35\x37\xa8\xc2\x5a\x7f\x94\x8a\x0e\x99\xd6\x10\xeb\xc7\x12\x6d\x75\x2c\xdb\xae\x8d\xb7\x84\xf6\x08\xf9\xfa\x66\xee\x07\xe7\xd1\xc8\x03\xd4\x8c\x49\x74\xba\x5f\x3c\xff\xb8\xc0\x5a\xeb\xfa\xbd\x5a\x34\x0c\xde\x7a\x9b\xa1\x2b\x7c\x37\x2d\xe0\xda\xe5\x04\xea\x52\x4a\x9d\x95\xe2\x74\x08\x8b\x5a\x8b\xee\x60\xb8\x3a\xfa\x78\xfd\x12\x60\x89\x0e\xb8\x74\x87\x4b\x47\x68\x4f\x88\xae\x09\x2e\xab\x6a\x7d\xe3\x86\x07\x66\xbc\x78\x56\x17\x03\x0d\x47\x0f\xc1\x9e\x8e\xc7\xa3\x0a\x5b\xed\x58\xd8\x3e\xa6\x84\x50\x55\x0d\x1a\xa9\xa5\x94\x46\x1b\x85\xab\xd0\x61\x1d\x35\x3e\xc8\xd5\x95\x57\x93\x09\x66\x62\xa5\xb0\x36\xca\x12\x63\x55\x28\xa5\x2a\x4c\x7a\x6f\x49\xcc\x41\xba\x32\xd7\x6b\xac\x80\x36\x0f\x98\x87\x84\x46\x95\x92\xef\x81\x74\x02\x7a\x6c\xe3\x5c\xc1\x76\xed\xdf\xaf\xa4\xca\x91\x90\x14\x12\x25\x39\xa4\xbd\x6e\x37\x73\x8d\xa5\x35\x50\x77\x3c\x51\xd9\x22\xbc\xf6\x5d\x81\xd6\xb8\xa9\x13\xfb\x73\x37\x50\x56\xae\xfb\x6c\x8a\x48\x62\x15\x33\xbb\x29\xe7\x92\x60\xb7\x64\xb8\x71\x44\x37\x23\xb1\xe6\xd4\xe3\x3c\x3b\xa9\x27\x2f\xf1\x12\xb8\x9e\x83\x9a\x07\xe5\x39\x7a\x1a\x1e\xec\x18\x1d\xca\x4d\xc6\xf5\x4f\x32\x79\x59\xff\x64\x7e\x74\xa4\xa4\x75\x8d\x5c\x7b\x06\xda\x2e\xa9\x2c\xb1\xbb\xfd\x37\x17\x57\xf3\xc5\xdb\x77\x37\x17\x8b\xd9\x3c\x15\xac\x72\xdd\x79\xcc\xc3\x53\x42\x5c\x7b\xd0\x8a\xf9\xff\x24\x08\xe4\x5c\xb8\x10\x0d\x82\x80\x6e\x53\x57\x89\x05\x2e\x80\x36\x6f\x99\x49\x7d\xd6\xfe\x77\xff\x56\xec\x2f\xb6\x1b\xaf\xb8\xdc\x7d\xe5\x96\x57\x8a\x6d\xb1\x81\x5f\x06\x6f\x7c\x38\x58\xe5\xea\x35\x3f\x5f\x77\xc1\x3e\x1a\x44\x70\x5c\x7e\x4f\xc2\x63\xbc\x80\x0e\x6f\x8f\x75\x7f\x72\xf0\x75\xe7\xe0\x4d\xdf\x7f\xeb\x39\xd0\xdb\x68\xdf\x69\xd7\xad\x7e\xf3\x10\xdc\xbc\xf9\x0c\xdb\x9c\x88\x0f\x3b\x29\xf1\x5d\x24\x91\x9e\x89\x57\x9c\x15\x6b\x13\x6e\x62\xf3\xcc\x1c\x1b\xae\x7e\x50\xd9\x4a\x6e\xcb\xce\xab\x09\xdd\x09\x5c\x32\xe2\x03\xaa\x8b\x16\x4c\x14\xa1\x3e\xa1\x31\xe4\xff\x37\x00\x00\xff\xff\x51\xa6\x87\xe7\xa2\x1a\x00\x00") func masterEtcOriginMasterMasterConfigYamlBytes() ([]byte, error) { return bindataRead( @@ -198,7 +198,7 @@ func masterTmpAnsibleAnsibleSh() (*asset, error) { return a, nil } -var _masterTmpAnsibleAzureLocalMasterInventoryYml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x9c\x57\x6d\x6f\xdb\x38\x12\xfe\x9e\x5f\x31\x70\x0b\xf8\x0e\xa8\x6d\xb4\x5d\xe0\x6e\x05\xf4\x43\x36\xab\x6e\x82\xdd\xc4\x86\xed\x1e\xda\x3b\x1c\x04\x8a\x1a\x59\xbc\x50\x1c\x2d\x49\xd9\xf1\x06\xf9\xef\x87\x11\x29\xc7\x51\x92\x26\xbb\xf9\x14\x99\xf3\xf2\x70\xe6\x99\x17\x4e\x26\x93\x93\x37\xf0\xf9\xe2\xeb\x65\x9a\xc0\xa9\xd9\xfb\x4a\x99\x0d\xe4\xa8\x69\x07\xca\x80\xd0\x7a\x22\x45\xe3\xa0\x12\x0e\x94\x1f\x3b\xd8\x0a\xdd\x22\x58\x6c\xb4\x90\x58\x40\xbe\x07\x87\x05\x8b\xfa\x0a\x4f\xde\x40\xfc\xa3\x06\x8d\xab\x54\xe9\x6b\xe1\x3c\x5a\x27\xad\x6a\xfc\xd4\x55\xb0\xab\x94\xac\x40\x39\x30\xe4\x41\x15\x28\xf4\x3b\xd8\x21\xb8\x8a\x5a\x5d\x40\xa9\x6e\xc0\x57\xc2\x4f\x4f\x34\x49\xa1\x83\x72\x72\x02\x50\x91\xf3\x8e\xff\x01\xe8\x4e\xf8\xfb\x04\x60\x2b\x6c\xfc\x55\x18\xa7\x72\x8d\x99\x24\x63\x50\x7a\x45\x26\x09\x92\x0f\x4e\x9b\xbd\xaf\xc8\x64\xca\x78\xb4\x8d\x45\x36\x0e\xb3\xd6\xd9\x59\xae\xcc\x2c\x1c\x7e\x38\x39\x79\x70\x81\x4c\x39\x36\xea\x85\x32\x68\xd5\x1f\x58\x24\xf0\x59\x68\x87\x03\x29\xa9\x15\x1a\x9f\xe5\xca\x08\xbb\x3f\x32\x4a\xf2\x91\xbd\x5a\x6c\x30\xf3\x62\x93\xc0\x68\xfb\xaf\x74\xb9\xba\x98\x5f\x8d\x86\x42\x3b\xcc\xd9\xab\x23\x8d\x51\xc1\x88\x1a\x13\x18\x5d\x5c\x9e\xfe\x92\x66\x8b\x65\xfa\xf9\xe2\xeb\x2c\x7c\xac\xbf\x2d\xd2\xc9\x0e\xf3\x49\x54\x48\x86\x56\x2d\x6e\xb2\xd6\xea\x04\xc6\xcf\x69\xbf\xbd\x95\x54\x37\x64\xd0\xf8\xbb\xe4\xed\xed\x16\xad\x53\x64\xee\xc6\x03\x54\x21\x1f\x59\x81\xa5\x68\xb5\xcf\x5c\x9b\x17\x54\x0b\x65\x12\x18\xaf\xd3\xcb\xc5\x72\xfe\x65\x9d\x2e\x2f\x16\x53\xa3\x9a\xa9\xa2\x71\x70\x1f\xd9\x15\xff\x5f\x57\xca\xf5\xd9\xce\x11\xfc\xbe\xc1\x4f\xca\x94\x56\xbc\x83\xbc\xf5\x4c\x85\x4a\x6c\x11\x3c\x81\x56\x5b\x84\x9d\xf2\x15\x58\xdc\x28\x32\x41\x0c\x4a\xb2\x60\x68\x17\xcd\xe5\x28\x45\xeb\x10\xa8\x04\x8d\x1b\x21\xf7\x60\x51\x38\x32\x6e\x80\xdc\x52\xcb\xc8\x1d\x6a\x94\x9e\x6c\x02\xe3\x63\xa3\xc3\x7b\xf2\x99\xf3\x76\xff\x7d\xf9\x37\x70\x45\x1d\x85\x61\x57\xed\xc1\xf3\xc5\x94\x03\x01\x85\x2a\x4b\xb4\x68\x3c\x14\xc2\x8b\xee\x8a\xe1\x72\xca\x83\x1a\x02\x6b\x2c\xd5\xe8\x2b\x6c\x5d\x66\xa8\xc0\x23\x8f\xb7\xd1\xe5\x38\x81\x71\xf0\x7a\xd7\x07\x74\x85\x1e\x4e\xff\x68\x2d\x82\x6b\x50\xaa\x52\xc9\xe0\x8a\x63\xc3\x95\x81\x05\xf4\x57\x18\xb8\x0b\xa7\x47\x17\xf4\x64\x99\x5d\x8d\xa5\xad\x2a\xb8\x1c\x46\x82\x0d\x67\xb9\xa6\x7c\xf4\x4a\xe5\x6b\x65\x8a\x04\x46\x94\xff\x0f\xa5\x7f\xad\xd2\xbd\x9b\x4c\x48\x49\xad\xf1\x81\xe1\xe3\x65\xfa\xcb\xc5\x6a\xbd\xfc\x96\xad\xd6\xf3\x25\x33\xf4\xf4\xdf\x5f\x96\x69\x76\x7a\x76\x36\xff\x72\xb5\xbe\x3a\xbd\x4c\x87\xe9\x7a\xbd\x8b\x6b\xdc\xbf\xe8\xe1\xd7\xf4\xdb\x5f\x70\x70\x68\x11\x09\x8c\x7a\xb9\xbf\x10\x0a\x8b\x42\xd7\x09\x8c\x24\x59\x9c\xee\x94\x29\x68\xe7\xa6\x06\xfd\xe8\x89\x5a\x8a\xbf\x9c\x0b\x5b\x80\xa4\x02\x03\x05\x63\x7d\x4c\x1f\xc8\x9c\xb5\x96\x09\xa9\x99\xa6\x08\x52\xb7\x5c\xc7\xe0\xbc\xf0\x08\xc2\x43\x81\x8d\xa6\x7d\xcd\x94\xf5\xaa\x46\x28\x08\x43\x83\x0e\xb5\x58\x21\x30\x37\x5d\x34\x16\xf0\xa3\xc5\x82\xcb\x94\x4f\x43\x5f\x08\xc5\xaa\x45\x8e\xda\x81\x68\x1a\xad\xb0\x00\x61\x98\x8b\xa2\xd8\xb3\x6c\x8e\xf0\x7b\x8b\x56\x61\x11\x4d\x89\x8d\x50\xc6\x79\xc6\xc0\x76\x1a\x52\xc6\x77\xb3\x84\x51\x84\x99\x12\xc1\x75\x63\x21\x08\x69\xb1\xcf\x89\xae\x1d\xd8\xd6\x4c\xe1\x54\x3b\x7a\x17\xcd\xf1\x71\xa8\xf4\x6e\x40\x29\x29\xa0\x4b\x3b\xc4\x8e\x05\xa3\x52\x68\xed\x20\x17\xf2\x7a\xc4\x80\xde\x33\x4e\x4b\x8d\x55\xc2\xa3\xde\xc3\xae\x42\x8b\x20\xdc\xb1\xbd\x98\xad\x83\x45\x4d\x1b\xae\xb7\x18\xa2\x29\xac\x3b\x9d\x9d\x70\x20\xb4\x23\x28\x94\x93\xad\xe3\xf6\x09\x22\x27\xae\xfa\x32\x5a\xeb\xc6\xe7\x91\x3f\x06\x50\x50\xc8\x5a\xec\x73\x9d\x07\x07\x9f\x3e\x41\x68\x76\x5d\xd8\x0f\x4d\x8e\x0d\x44\x5b\x0d\xda\x12\x25\x27\xb4\x44\xd1\xcd\x36\xb6\xd6\xa5\x4b\x98\x23\xe5\x78\xff\x4a\x6d\x2a\xb4\x1c\xc3\x3e\xa6\x4e\xd9\x43\x16\x7a\xbf\x0f\x39\xb3\xc4\xad\x72\xca\x83\x16\x1c\xcf\xbf\x35\xe4\xd8\xcf\x9e\x1b\x9c\xc0\x9a\x8c\x43\x0f\x64\xe1\x2d\xf9\x0a\xed\xdf\x9f\xa1\x7a\x68\xbb\xbd\x83\x04\xde\xbf\x50\x12\xc7\x92\xcf\x77\xca\x8e\xb8\x09\x8c\x1a\x8b\x0e\xcd\xa3\x96\x33\xec\xa9\x78\xd3\x90\x65\x1c\x61\x92\xc6\xe9\x96\xc0\x78\xb1\x9c\x5f\xa6\xeb\xf3\xf4\xcb\x2a\x4b\xbf\x2e\xe6\xcb\x75\xba\xcc\xe2\xec\x1c\x7f\x77\x22\x1b\xe7\x85\xd6\x09\xac\x6d\x8b\xdf\xc1\x19\xfc\x35\x16\x4b\x75\x33\x9c\xdd\x43\xd0\xf7\x35\x98\xf1\xc4\x48\x60\x7c\x7a\xb5\xba\xf8\xe9\xb7\x34\xfb\x39\x5d\xfc\x36\xff\xd6\x0d\xea\x88\xaa\x5f\x67\x1c\xda\xad\x92\x98\xe5\x96\xae\xf9\x7a\x0f\x60\x3d\x34\xdf\x8b\x4a\xe1\x85\xa6\x4d\x80\xf6\x9d\x7d\x22\xca\x4f\xa2\xfc\xd1\x4e\xf1\xe4\x4a\x80\x5e\x16\xbc\x63\xb8\x04\xfe\x33\xaa\xbc\x6f\x5c\x32\x9b\x9d\xcf\x57\x5d\xbf\x4e\x3e\x7c\xfc\xc7\x8f\xa3\xff\x06\xec\x1e\xeb\x86\x29\xf5\x08\xfc\x0b\x80\x7a\xbd\x03\xb2\xa0\x37\x5c\x76\xde\xc0\xd5\x7c\x9d\x26\x61\xcf\x50\x0e\x2c\x16\xad\x29\x84\xf1\x61\x04\x5b\xfc\xbd\x55\xb1\x73\x55\xc2\x14\x1a\x21\xd2\x01\xdc\x35\xee\x20\x47\xbf\x43\x34\xd1\xd4\x60\x97\x9d\xc4\xb8\x73\x67\x9a\x5b\xb5\xe1\xfd\xd8\x14\x30\x3f\x5b\x0c\x82\x82\x37\xa2\x6e\x34\x86\xc5\x91\x33\x7a\xa0\xdc\x68\xbb\x3a\x9f\x2f\xd7\xcc\xb2\x47\x3b\x9f\x24\x79\xdd\xa8\x9e\x09\x68\xef\x89\x73\x36\x3f\xfb\x75\x71\xb1\x7e\x8e\x3a\x8f\x14\x73\xe1\x30\xee\x8b\xbd\xea\x4f\xa7\xab\x94\x73\xf1\xa2\xee\x3d\xd4\x5e\xf5\xe9\x00\xff\x4c\xdd\x84\x28\xb0\x54\x06\x5f\xac\x7c\x70\x74\xdf\xc1\xbd\x70\xd7\x50\x2a\x8d\x7d\xff\xe9\x64\xa7\xfb\x5a\x83\x32\x4e\x15\x61\xe4\x0c\x4d\x82\x25\x8d\x50\x5a\xaa\x9f\xc8\xc8\x4e\x69\xdd\xcf\x94\xd6\x53\x43\x4d\xcb\x64\xe1\x95\xab\x75\xfc\xa2\x79\xca\xe2\x70\x2b\xec\xb2\xc9\x03\x6a\xaf\xcc\xe6\x68\x04\x98\xb6\xce\xd1\xf2\x8e\x79\xd4\x95\x87\x74\x43\x7e\x8c\x28\x91\xeb\xae\x61\xf3\xf3\x49\x58\x66\x8a\x47\x6b\x84\x66\xbe\x3d\x46\xcd\xfe\x76\x38\xb6\xf8\x80\x6f\x61\x17\x8b\xa0\x6b\xe8\xc6\x8b\x27\x10\x5b\x52\x45\x87\x47\x19\xc9\xf3\x9c\xa7\xbe\xf3\x0c\xab\x14\xd2\x43\xa9\x4c\xd1\xe3\x3e\xb8\x0a\x8f\x23\x00\x49\x75\x4d\xa6\xff\xe2\x6f\x53\xaa\x4d\x47\x94\x04\x66\xe8\xe5\x8c\x3a\x42\xcf\x0e\x12\xaf\xe4\x70\x10\xbe\x7f\xa8\xc5\x2b\xb4\xb9\x56\xf2\xd0\x2f\xbb\xc7\xc7\xa1\x2d\xdc\xde\xc2\x34\xbd\x09\x71\xb9\xec\x14\xcf\xc9\x75\xdb\x1f\xdc\xdd\x25\xff\xfc\xe1\x87\x8f\xb3\xa8\x38\x1a\x1a\x14\x8d\xfa\xd3\xc6\xee\x8d\xbc\xa6\x49\xc1\x9b\xf8\x16\xbe\x22\xd0\x64\x36\x68\xc1\x20\x16\xbc\xc9\x38\x0e\xf5\xa3\x24\x4e\x3e\x4e\x7f\x9c\x7e\xf8\x30\x79\x1f\xdf\x2f\x63\x8b\x5d\x61\x90\x09\x84\xef\xdb\xcb\x1e\x7d\x04\xc2\xf4\xb9\x0f\x15\x7f\xc5\xcd\xb7\x87\x32\x3e\x39\x21\x8a\x9d\xd5\x65\x9e\xb2\x90\xad\x67\xdf\xc1\xff\x0f\x00\x00\xff\xff\xa5\x31\x2a\x64\xc4\x0f\x00\x00") +var _masterTmpAnsibleAzureLocalMasterInventoryYml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x9c\x57\x6d\x6f\xe3\x36\x12\xfe\xee\x5f\x31\x50\x16\xf0\x1d\xb0\xb6\xb1\xbb\x05\xee\x2a\x60\x3f\xb8\xa9\xb6\x09\xda\xc4\x86\xed\x3d\xec\x5e\x51\xa8\x14\x35\xb2\x78\xa1\x38\x2a\x49\xd9\x71\x0d\xff\xf7\x03\x45\xca\x71\x94\x78\x93\x36\x9f\x22\x73\x5e\x1e\xce\x3c\xf3\xc2\xd1\x68\x34\xb8\x80\x4f\xd7\x5f\x6e\x92\x18\xa6\x6a\x67\x4b\xa1\xd6\x90\xa1\xa4\x2d\x08\x05\x4c\xca\x11\x67\xb5\x81\x92\x19\x10\x76\x68\x60\xc3\x64\x83\xa0\xb1\x96\x8c\x63\x0e\xd9\x0e\x0c\xe6\x4e\xd4\x96\x38\xb8\x80\xf0\x47\x35\x2a\x53\x8a\xc2\x56\xcc\x58\xd4\x86\x6b\x51\xdb\xb1\x29\x61\x5b\x0a\x5e\x82\x30\xa0\xc8\x82\xc8\x91\xc9\xb7\xb0\x45\x30\x25\x35\x32\x87\x42\xdc\x83\x2d\x99\x1d\x0f\x24\x71\x26\xbd\x72\x3c\x00\x28\xc9\x58\xe3\xfe\x01\x68\x4f\xdc\xf7\x00\x60\xc3\x74\xf8\x95\x29\x23\x32\x89\x29\x27\xa5\x90\x5b\x41\x2a\xf6\x92\x8f\x4e\xeb\x9d\x2d\x49\xa5\x42\x59\xd4\xb5\x46\x67\x1c\x26\x8d\xd1\x93\x4c\xa8\x89\x3f\x7c\x3f\x18\x3c\xba\x40\x2a\x8c\x33\x6a\x99\x50\xa8\xc5\x9f\x98\xc7\xf0\x89\x49\x83\x3d\x29\x2e\x05\x2a\x9b\x66\x42\x31\xbd\x3b\x31\x4a\xfc\x89\xbd\x8a\xad\x31\xb5\x6c\x1d\x43\xb4\xf9\x4f\xb2\x58\x5e\xcf\x6e\xa3\xbe\xd0\x16\x33\xe7\xd5\x90\xc4\xa0\xa0\x58\x85\x31\x44\xd7\x37\xd3\x9f\x92\x74\xbe\x48\x3e\x5d\x7f\x99\xf8\x8f\xd5\xd7\x79\x32\xda\x62\x36\x0a\x0a\x71\xdf\xaa\xc6\x75\xda\x68\x19\xc3\xf0\x9c\xf6\x9b\x3d\xa7\xaa\x26\x85\xca\x1e\xe2\x37\xfb\x0d\x6a\x23\x48\x1d\x86\x3d\x54\x3e\x1f\x69\x8e\x05\x6b\xa4\x4d\x4d\x93\xe5\x54\x31\xa1\x62\x18\xae\x92\x9b\xf9\x62\xf6\x79\x95\x2c\xae\xe7\x63\x25\xea\xb1\xa0\xa1\x77\x1f\xd8\x15\xfe\x5f\x95\xc2\x74\xd9\xce\x10\xec\xae\xc6\x8f\x42\x15\x9a\xbd\x85\xac\xb1\x8e\x0a\x25\xdb\x20\x58\x02\x29\x36\x08\x5b\x61\x4b\xd0\xb8\x16\xa4\xbc\x18\x14\xa4\x41\xd1\x36\x98\xcb\x90\xb3\xc6\x20\x50\x01\x12\xd7\x8c\xef\x40\x23\x33\xa4\x4c\x0f\xb9\xa6\xc6\x21\x37\x28\x91\x5b\xd2\x31\x0c\x4f\x8d\xf6\xef\xe9\xce\x8c\xd5\xbb\x6f\xcb\x5f\xc0\x2d\xb5\x14\x86\x6d\xb9\x03\xeb\x2e\x26\x0c\x30\xc8\x45\x51\xa0\x46\x65\x21\x67\x96\xb5\x57\xf4\x97\x13\x16\x44\x1f\x58\xad\xa9\x42\x5b\x62\x63\x52\x45\x39\x9e\x78\xdc\x07\x97\xc3\x18\x86\xde\xeb\xa1\x0b\xe8\x12\x2d\x4c\xff\x6c\x34\x82\xa9\x91\x8b\x42\x70\xef\xca\xc5\xc6\x55\x06\xe6\xd0\x5d\xa1\xe7\xce\x9f\x9e\x5c\xd0\x92\x76\xec\xaa\x35\x6d\x44\xee\xca\x21\x62\xce\x70\x9a\x49\xca\xa2\x57\x2a\xdf\x09\x95\xc7\x10\x51\xf6\x3f\xe4\xf6\xb5\x4a\x0f\x6e\x52\xc6\x39\x35\xca\x7a\x86\x0f\x17\xc9\x4f\xd7\xcb\xd5\xe2\x6b\xba\x5c\xcd\x16\x8e\xa1\xd3\xff\x7e\x5e\x24\xe9\xf4\xf2\x72\xf6\xf9\x76\x75\x3b\xbd\x49\xfa\xe9\x7a\xbd\x8b\x3b\xdc\xbd\xe8\xe1\xe7\xe4\xeb\xdf\x70\x70\x6c\x11\x31\x44\x9d\xdc\xdf\x08\x85\x46\x26\xab\x18\x22\x4e\x1a\xc7\x5b\xa1\x72\xda\x9a\xb1\x42\x1b\x3d\x53\x4b\xe1\x97\x2b\xa6\x73\xe0\x94\xa3\xa7\x60\xa8\x8f\xf1\x23\x99\xcb\x46\x3b\x42\x4a\x47\x53\x04\x2e\x1b\x57\xc7\x60\x2c\xb3\x08\xcc\x42\x8e\xb5\xa4\x5d\xe5\x28\x6b\x45\x85\x90\x13\xfa\x06\xed\x6b\xb1\x44\x70\xdc\x34\xc1\x98\xc7\x8f\x1a\x73\x57\xa6\xee\xd4\xf7\x05\x5f\xac\x92\x65\x28\x0d\xb0\xba\x96\x02\x73\x60\xca\x71\x91\xe5\x3b\x27\x9b\x21\xfc\xd1\xa0\x16\x98\x07\x53\x6c\xcd\x84\x32\xd6\x61\x70\x76\x6a\x12\xca\xb6\xb3\xc4\xa1\xf0\x33\x25\x80\x6b\xc7\x82\x17\x92\x6c\x97\x11\xdd\x19\xd0\x8d\x1a\xc3\x54\x1a\x7a\x1b\xcc\xb9\x63\x5f\xe9\xed\x80\x12\x9c\x41\x9b\x76\x08\x1d\x0b\xa2\x82\x49\x69\x20\x63\xfc\x2e\x72\x80\xde\x39\x9c\x9a\x6a\x2d\x98\x45\xb9\x83\x6d\x89\x1a\x81\x99\x53\x7b\x21\x5b\x47\x8b\x92\xd6\xae\xde\x42\x88\xc6\xb0\x6a\x75\xb6\xcc\x00\x93\x86\x20\x17\x86\x37\xc6\xb5\x4f\x60\x19\xb9\xaa\x2f\x82\xb5\x76\x7c\x9e\xf8\x73\x00\x72\xf2\x59\x0b\x7d\xae\xf5\x60\xe0\xe3\x47\xf0\xcd\xae\x0d\xfb\xb1\xc9\x39\x03\xc1\x56\x8d\xba\x40\xee\x12\x5a\x20\x6b\x67\x9b\xb3\xd6\xa6\x8b\xa9\x13\xe5\x70\xff\x52\xac\x4b\xd4\x2e\x86\x5d\x4c\x8d\xd0\xc7\x2c\x74\x7e\x1f\x73\x66\x81\x1b\x61\x84\x05\xc9\x5c\x3c\xff\x51\x93\x71\x7e\x76\xae\xc1\x31\xac\x48\x19\xb4\x40\x1a\xde\x90\x2d\x51\xff\xf3\x0c\xd5\x7d\xdb\xed\x1c\xc4\xf0\xee\x85\x92\x38\x95\x3c\xdf\x29\x5b\xe2\xc6\x10\xd5\x1a\x0d\xaa\x27\x2d\xa7\xdf\x53\xf1\xbe\x26\xed\x70\xf8\x49\x1a\xa6\x5b\x0c\xc3\xf9\x62\x76\x93\xac\xae\x92\xcf\xcb\x34\xf9\x32\x9f\x2d\x56\xc9\x22\x0d\xb3\x73\xf8\xcd\x89\xac\x8c\x65\x52\xc6\xb0\xd2\x0d\x7e\x03\xa7\xf7\x57\x6b\x2c\xc4\x7d\x7f\x76\xf7\x41\x3f\xd4\x60\xea\x26\x46\x0c\xc3\xe9\xed\xf2\xfa\x87\x5f\x92\xf4\xc7\x64\xfe\xcb\xec\x6b\x3b\xa8\x03\xaa\x6e\x9d\x31\xa8\x37\x82\x63\x9a\x69\xba\x73\xd7\x7b\x04\xeb\x5b\x82\x0e\x57\x0c\xb0\xdf\xff\x1e\xed\xf7\x20\x53\x66\xb2\xe3\x54\x6f\x0f\x4d\x9a\x0b\x6e\x7f\x3d\x0b\xef\x37\x10\xc5\x79\xf0\x8e\xc0\x43\xd2\x62\x2d\xd4\x10\x50\x1a\x0c\x2e\x7c\x3c\x1a\x2d\xe1\x70\x88\x7e\x3f\x1c\x7a\x21\xe8\x50\x72\x66\x99\xa4\x75\x07\xf3\xec\xce\x13\xe4\x47\x41\xfe\x64\xef\x79\x76\x6d\x41\xcb\x73\xe7\xdc\xc4\xf0\x6b\x54\x5a\x5b\x9b\x78\x32\xb9\x9a\x2d\xdb\x99\x12\xbf\xff\xf0\xaf\xef\xa3\xdf\x7c\x7c\x2d\x56\xb5\xa3\xfd\x99\xb8\x9d\x05\xd4\xe9\x1d\x91\x79\xbd\xfe\x42\x76\x01\xb7\xb3\x55\x12\xfb\x5d\x48\x18\xd0\x98\x37\x2a\x67\xca\xfa\x35\x41\xe3\x1f\x8d\x08\xdd\xb5\x64\x2a\x97\x08\x81\xb2\x60\xee\x70\x0b\x19\xda\x2d\xa2\x0a\xa6\x7a\xfb\xf6\x28\xa4\xdc\x75\xcf\x59\x9b\x81\xb6\x05\xcf\x2e\xe7\xbd\xa0\xe0\x3d\xab\x6a\x89\x7e\xb9\x75\x89\x3b\x96\x45\xb4\x59\x5e\xcd\x16\x2b\x57\x09\x4f\xf6\x52\x4e\xfc\xae\x16\x5d\xc2\x51\x3f\x90\xfb\x72\x76\xf9\xf3\xfc\x7a\x75\x8e\xde\x4f\x14\x33\x66\x30\xec\xb4\x9d\xea\x0f\xd3\x65\xe2\x72\xf1\xa2\xee\x03\xd4\x4e\xf5\xf9\x00\xff\x48\xed\x14\xcb\xb1\x10\x0a\x5f\xec\x4e\x60\xe8\x61\xca\x58\x66\xee\xa0\x10\x12\xbb\x1e\xd9\xca\x8e\x77\x95\x04\xa1\x8c\xc8\xfd\x58\xec\x9b\x04\x4d\x12\xa1\xd0\x54\x3d\x93\x91\xad\x90\xb2\x9b\x7b\x8d\xa5\x9a\xea\xc6\x91\xc5\xad\x85\x8d\x71\xaf\xae\xe7\x2c\xf6\x37\xd7\x36\x9b\x6e\x88\xee\x84\x5a\x9f\x8c\x29\xd5\x54\x19\x6a\xb7\x07\x9f\x4c\x8e\x3e\xdd\xd0\x3d\x98\x04\xcb\x64\x3b\x54\xdc\x13\x8f\x69\xc7\x14\x8b\x5a\x31\xe9\xf8\xf6\x14\xb5\xf3\xb7\xc5\xa1\xc6\x47\x7c\xf3\xfb\x62\x00\x5d\x41\x3b\x02\x2d\x01\xdb\x90\xc8\x5b\x3c\x42\x71\xb7\x73\xb8\xcd\xc4\x58\x07\xab\x60\xdc\x42\x21\x54\xde\xe1\x3e\xba\xf2\x0f\x38\x00\x4e\x55\x45\xaa\xfb\x72\xdf\xaa\x10\xeb\x96\x28\x31\x4c\xd0\xf2\x89\x6f\x29\x93\xa3\xc4\x2b\x39\xec\x85\x1f\x1e\x93\xe1\x0a\x4d\x26\x05\x3f\xf6\xf4\xf6\x81\x74\x6c\x0b\xfb\x3d\x8c\x93\x7b\x1f\x97\x9b\x56\xf1\x8a\x4c\xbb\xa1\xc2\xe1\x10\xff\xfb\xbb\xef\x3e\x4c\x82\x62\xd4\x37\xc8\x6a\xf1\x97\x8d\x3d\x18\x79\x4d\x93\x82\x8b\xf0\x5e\xbf\x25\x90\xa4\xd6\xa8\x41\x21\xe6\x6e\xdb\x32\x2e\xd4\x4f\x92\x38\xfa\x30\xfe\x7e\xfc\xfe\xfd\xe8\x5d\x78\x63\x0d\x35\xb6\x85\x41\xca\x13\xbe\x6b\x2f\x3b\xb4\x01\x88\xa3\xcf\x43\xa8\xdc\x57\xd8\xce\x3b\x28\xc3\xc1\x80\x28\x74\x56\x93\x5a\x4a\x7d\xb6\xce\xbe\xd5\xff\x1f\x00\x00\xff\xff\xe7\x02\x45\x23\x68\x10\x00\x00") func masterTmpAnsibleAzureLocalMasterInventoryYmlBytes() ([]byte, error) { return bindataRead( diff --git a/pkg/openshift/certgen/unstable/templates/master/etc/origin/master/master-config.yaml b/pkg/openshift/certgen/unstable/templates/master/etc/origin/master/master-config.yaml index bb13c9fe59..a4be4587c6 100644 --- a/pkg/openshift/certgen/unstable/templates/master/etc/origin/master/master-config.yaml +++ b/pkg/openshift/certgen/unstable/templates/master/etc/origin/master/master-config.yaml @@ -89,6 +89,8 @@ etcdStorageConfig: imageConfig: format: IMAGE_PREFIX/IMAGE_TYPE-${component}:${version} latest: false +imagePolicyConfig: + internalRegistryHostname: docker-registry.default.svc:5000 kind: MasterConfig kubeletClientInfo: ca: ca-bundle.crt diff --git a/pkg/openshift/certgen/unstable/templates/master/tmp/ansible/azure-local-master-inventory.yml b/pkg/openshift/certgen/unstable/templates/master/tmp/ansible/azure-local-master-inventory.yml index ddf3cfe382..9fb2fd9342 100644 --- a/pkg/openshift/certgen/unstable/templates/master/tmp/ansible/azure-local-master-inventory.yml +++ b/pkg/openshift/certgen/unstable/templates/master/tmp/ansible/azure-local-master-inventory.yml @@ -60,6 +60,7 @@ localmaster: openshift_deployment_type: 'ANSIBLE_DEPLOY_TYPE' ansible_service_broker_install: True + ansible_service_broker_image: {{`"{{ l_asb_default_images_dict[openshift_deployment_type] if openshift_deployment_type == 'origin' else l_asb_image_url }}"`}} openshift_service_catalog_image: "IMAGE_PREFIX/IMAGE_TYPE-service-catalog:vVERSION" openshift_master_etcd_urls: ["https://HOSTNAME:2379"] diff --git a/pkg/operations/kubernetesupgrade/upgradecluster.go b/pkg/operations/kubernetesupgrade/upgradecluster.go index fdd3376f00..fc85767268 100644 --- a/pkg/operations/kubernetesupgrade/upgradecluster.go +++ b/pkg/operations/kubernetesupgrade/upgradecluster.go @@ -27,10 +27,26 @@ type ClusterTopology struct { AgentPoolsToUpgrade map[string]bool AgentPools map[string]*AgentPoolTopology + AgentPoolScaleSetsToUpgrade []AgentPoolScaleSet + MasterVMs *[]compute.VirtualMachine UpgradedMasterVMs *[]compute.VirtualMachine } +// AgentPoolScaleSet contains necessary data required to upgrade a VMSS +type AgentPoolScaleSet struct { + Name string + Sku compute.Sku + Location string + VMsToUpgrade []AgentPoolScaleSetVM +} + +// AgentPoolScaleSetVM represents a VM in a VMSS +type AgentPoolScaleSetVM struct { + Name string + InstanceID string +} + // AgentPoolTopology contains agent VMs in a single pool type AgentPoolTopology struct { Identifier *string @@ -128,6 +144,50 @@ func (uc *UpgradeCluster) getClusterNodeStatus(subscriptionID uuid.UUID, resourc targetOrchestratorTypeVersion := fmt.Sprintf("%s:%s", uc.DataModel.Properties.OrchestratorProfile.OrchestratorType, uc.DataModel.Properties.OrchestratorProfile.OrchestratorVersion) + vmScaleSets, err := uc.Client.ListVirtualMachineScaleSets(resourceGroup) + if err != nil { + return err + } + if vmScaleSets.Value != nil { + for _, vmScaleSet := range *vmScaleSets.Value { + vmScaleSetVMs, err := uc.Client.ListVirtualMachineScaleSetVMs(resourceGroup, *vmScaleSet.Name) + if err != nil { + return err + } + scaleSetToUpgrade := AgentPoolScaleSet{ + Name: *vmScaleSet.Name, + Sku: *vmScaleSet.Sku, + Location: *vmScaleSet.Location, + } + for _, vm := range *vmScaleSetVMs.Value { + if vm.Tags == nil || (*vm.Tags)["orchestrator"] == nil { + uc.Logger.Infof("No tags found for scale set VM: %s skipping.\n", *vm.Name) + continue + } + + scaleSetVMOrchestratorTypeAndVersion := *(*vm.Tags)["orchestrator"] + if scaleSetVMOrchestratorTypeAndVersion != targetOrchestratorTypeVersion { + // This condition is a scale set VM that is an older version and should be handled + uc.Logger.Infof( + "VM %s in VMSS %s has a current tag of %s and a desired tag of %s. Upgrading this node.\n", + *vm.Name, + *vmScaleSet.Name, + scaleSetVMOrchestratorTypeAndVersion, + targetOrchestratorTypeVersion, + ) + scaleSetToUpgrade.VMsToUpgrade = append( + scaleSetToUpgrade.VMsToUpgrade, + AgentPoolScaleSetVM{ + Name: *vm.VirtualMachineScaleSetVMProperties.OsProfile.ComputerName, + InstanceID: *vm.InstanceID, + }, + ) + } + } + uc.AgentPoolScaleSetsToUpgrade = append(uc.AgentPoolScaleSetsToUpgrade, scaleSetToUpgrade) + } + } + for _, vm := range *vmListResult.Value { if vm.Tags == nil || (*vm.Tags)["orchestrator"] == nil { uc.Logger.Infof("No tags found for VM: %s skipping.\n", *vm.Name) diff --git a/pkg/operations/kubernetesupgrade/upgrader.go b/pkg/operations/kubernetesupgrade/upgrader.go index 8cb4c0b0e9..0b9e620a0e 100644 --- a/pkg/operations/kubernetesupgrade/upgrader.go +++ b/pkg/operations/kubernetesupgrade/upgrader.go @@ -10,6 +10,7 @@ import ( "github.com/Azure/acs-engine/pkg/armhelpers" "github.com/Azure/acs-engine/pkg/armhelpers/utils" "github.com/Azure/acs-engine/pkg/i18n" + "github.com/Azure/acs-engine/pkg/operations" "github.com/sirupsen/logrus" "k8s.io/api/core/v1" ) @@ -56,6 +57,10 @@ func (ku *Upgrader) RunUpgrade() error { return err } + if err := ku.upgradeAgentScaleSets(); err != nil { + return err + } + return ku.upgradeAgentPools() } @@ -374,6 +379,113 @@ func (ku *Upgrader) upgradeAgentPools() error { return nil } +func (ku *Upgrader) upgradeAgentScaleSets() error { + for _, vmssToUpgrade := range ku.ClusterTopology.AgentPoolScaleSetsToUpgrade { + ku.logger.Infof("Upgrading VMSS %s", vmssToUpgrade.Name) + + if len(vmssToUpgrade.VMsToUpgrade) == 0 { + ku.logger.Infof("No VMs to upgrade for VMSS %s, skipping", vmssToUpgrade.Name) + continue + } + + newCapacity := *vmssToUpgrade.Sku.Capacity + 1 + ku.logger.Infof( + "VMSS %s current capacity is %d and new capacity will be %d while each node is swapped", + vmssToUpgrade.Name, + *vmssToUpgrade.Sku.Capacity, + newCapacity, + ) + + *vmssToUpgrade.Sku.Capacity = newCapacity + + for _, vmToUpgrade := range vmssToUpgrade.VMsToUpgrade { + success, failure := ku.Client.SetVirtualMachineScaleSetCapacity( + ku.ClusterTopology.ResourceGroup, + vmssToUpgrade.Name, + vmssToUpgrade.Sku, + vmssToUpgrade.Location, + make(chan struct{}), + ) + + select { + case <-success: + ku.logger.Infof("Successfully set capacity for VMSS %s", vmssToUpgrade.Name) + case err := <-failure: + ku.logger.Errorf("Failure to set capacity for VMSS %s", vmssToUpgrade.Name) + return err + } + + // Before we can delete the node we should safely and responsibly drain it + var kubeAPIServerURL string + getClientTimeout := 10 * time.Second + + if ku.DataModel.Properties.HostedMasterProfile != nil { + kubeAPIServerURL = ku.DataModel.Properties.HostedMasterProfile.FQDN + } else { + kubeAPIServerURL = ku.DataModel.Properties.MasterProfile.FQDN + } + client, err := ku.Client.GetKubernetesClient( + kubeAPIServerURL, + ku.kubeConfig, + interval, + getClientTimeout, + ) + if err != nil { + ku.logger.Errorf("Error getting Kubernetes client: %v", err) + return err + } + + ku.logger.Infof("Draining node %s", vmToUpgrade.Name) + err = operations.SafelyDrainNodeWithClient( + client, + ku.logger, + vmToUpgrade.Name, + time.Minute, + ) + if err != nil { + ku.logger.Errorf("Error draining VM in VMSS: %v", err) + return err + } + + ku.logger.Infof( + "Deleting VM %s in VMSS %s", + vmToUpgrade.Name, + vmssToUpgrade.Name, + ) + + // At this point we have our buffer node that will replace the node to delete + // so we can just remove this current node then + res, failure := ku.Client.DeleteVirtualMachineScaleSetVM( + ku.ClusterTopology.ResourceGroup, + vmssToUpgrade.Name, + vmToUpgrade.InstanceID, + make(chan struct{}), + ) + + select { + case <-res: + ku.logger.Infof( + "Successfully deleted VM %s in VMSS %s", + vmToUpgrade.Name, + vmssToUpgrade.Name, + ) + case err := <-failure: + ku.logger.Errorf( + "Failed to delete VM %s in VMSS %s", + vmToUpgrade.Name, + vmssToUpgrade, + ) + return err + } + } + ku.logger.Infof("Completed upgrading VMSS %s", vmssToUpgrade) + } + + ku.logger.Infoln("Completed upgrading all VMSS") + + return nil +} + func (ku *Upgrader) generateUpgradeTemplate(upgradeContainerService *api.ContainerService, acsengineVersion string) (map[string]interface{}, map[string]interface{}, error) { var err error ctx := acsengine.Context{ diff --git a/test/e2e/runner/cli_provisioner.go b/test/e2e/runner/cli_provisioner.go index 73d02e495e..c6c01cb244 100644 --- a/test/e2e/runner/cli_provisioner.go +++ b/test/e2e/runner/cli_provisioner.go @@ -245,7 +245,7 @@ func (cli *CLIProvisioner) FetchProvisioningMetrics(path string, cfg *config.Con agentFiles := []string{"/var/log/azure/cluster-provision.log", "/var/log/cloud-init.log", "/var/log/cloud-init-output.log", "/var/log/syslog", "/var/log/azure/custom-script/handler.log", "/opt/m", "/opt/azure/containers/kubelet.sh", "/opt/azure/containers/provision.sh", - "/opt/azure/provision-ps.log", "/var/log/azure/kubelet-status.log", "/var/log/azure/hyperkube-extract-status.log", + "/opt/azure/provision-ps.log", "/var/log/azure/kubelet-status.log", "/var/log/azure/docker-status.log", "/var/log/azure/systemd-journald-status.log"} masterFiles := agentFiles masterFiles = append(masterFiles, "/opt/azure/containers/mountetcd.sh", "/opt/azure/containers/setup-etcd.sh", "/opt/azure/containers/setup-etcd.log")