From 17070618a0b35521eaa9295403fb6bc3dd63b211 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Thu, 2 Aug 2018 14:34:59 -0700 Subject: [PATCH 1/4] add support for k8s v1.12.0-alpha.1 --- .circleci/config.yml | 39 +++++++++++++++++++++ pkg/acsengine/k8s_versions.go | 66 +++++++++++++++++++++++++++++++++++ pkg/api/common/versions.go | 1 + 3 files changed, 106 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 32ea592a88..7ff2fae72b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -188,6 +188,33 @@ jobs: path: /go/src/github.com/Azure/acs-engine/_logs - store_artifacts: path: /go/src/github.com/Azure/acs-engine/_output + k8s-1.12-release-e2e: + <<: *defaults + steps: + - checkout + - run: | + echo 'export TIMEOUT=20m' >> $BASH_ENV + echo 'export ORCHESTRATOR_RELEASE=1.12' >> $BASH_ENV + echo 'export CLUSTER_DEFINITION=examples/e2e-tests/kubernetes/release/default/definition.json' >> $BASH_ENV + echo 'export CREATE_VNET=true' >> $BASH_ENV + echo 'export ENABLE_KMS_ENCRYPTION=true' >> $BASH_ENV + echo 'export CLEANUP_ON_EXIT=${CLEANUP_ON_EXIT}' >> $BASH_ENV + echo 'export RETAIN_SSH=false' >> $BASH_ENV + echo 'export SUBSCRIPTION_ID=${SUBSCRIPTION_ID_E2E_KUBERNETES}' >> $BASH_ENV + echo 'export CLIENT_ID=${SERVICE_PRINCIPAL_CLIENT_ID_E2E_KUBERNETES}' >> $BASH_ENV + echo 'export CLIENT_SECRET=${SERVICE_PRINCIPAL_CLIENT_SECRET_E2E_KUBERNETES}' >> $BASH_ENV + echo 'export CLIENT_OBJECTID=${SERVICE_PRINCIPAL_OBJECT_ID_E2E_KUBERNETES}' >> $BASH_ENV + - run: + name: compile + command: make build-binary + - run: + name: ginkgo k8s e2e tests + command: make test-kubernetes + no_output_timeout: "30m" + - store_artifacts: + path: /go/src/github.com/Azure/acs-engine/_logs + - store_artifacts: + path: /go/src/github.com/Azure/acs-engine/_output k8s-1.8-release-e2e: <<: *defaults steps: @@ -355,6 +382,12 @@ workflows: filters: branches: ignore: master + - k8s-1.12-release-e2e: + requires: + - pr-e2e-hold + filters: + branches: + ignore: master - dcos-e2e: requires: - pr-e2e-hold @@ -409,6 +442,12 @@ workflows: filters: branches: only: master + - k8s-1.12-release-e2e: + requires: + - test + filters: + branches: + only: master - k8s-windows-1.11-release-e2e: requires: - test diff --git a/pkg/acsengine/k8s_versions.go b/pkg/acsengine/k8s_versions.go index 5cf052f279..7a564c96bc 100644 --- a/pkg/acsengine/k8s_versions.go +++ b/pkg/acsengine/k8s_versions.go @@ -8,6 +8,37 @@ import ( ) var k8sComponentVersions = map[string]map[string]string{ + "1.12": { + "dockerEngine": "1.13.*", + "dashboard": "kubernetes-dashboard-amd64:v1.8.3", + "addon-resizer": "addon-resizer:1.8.1", + "heapster": "heapster-amd64:v1.5.3", + "metrics-server": "metrics-server-amd64:v0.2.1", + "kube-dns": "k8s-dns-kube-dns-amd64:1.14.10", + "addon-manager": "kube-addon-manager-amd64:v8.6", + "dnsmasq": "k8s-dns-dnsmasq-nanny-amd64:1.14.10", + "pause": "pause-amd64:3.1", + "tiller": "tiller:v2.8.1", + "rescheduler": "rescheduler:v0.4.0", + "aci-connector": "virtual-kubelet:latest", + ContainerMonitoringAddonName: "oms:ciprod05082018", + AzureCNINetworkMonitoringAddonName: "networkmonitor:v0.0.4", + "cluster-autoscaler": "cluster-autoscaler:v1.3.0", + NVIDIADevicePluginAddonName: "k8s-device-plugin:1.11", + "k8s-dns-sidecar": "k8s-dns-sidecar-amd64:1.14.10", + "nodestatusfreq": DefaultKubernetesNodeStatusUpdateFrequency, + "nodegraceperiod": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod, + "podeviction": DefaultKubernetesCtrlMgrPodEvictionTimeout, + "routeperiod": DefaultKubernetesCtrlMgrRouteReconciliationPeriod, + "backoffretries": strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries), + "backoffjitter": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64), + "backoffduration": strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration), + "backoffexponent": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64), + "ratelimitqps": strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64), + "ratelimitbucket": strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket), + "gchighthreshold": strconv.Itoa(DefaultKubernetesGCHighThreshold), + "gclowthreshold": strconv.Itoa(DefaultKubernetesGCLowThreshold), + }, "1.11": { "dockerEngine": "1.13.*", "dashboard": "kubernetes-dashboard-amd64:v1.8.3", @@ -247,6 +278,41 @@ func getK8sVersionComponents(version string, overrides map[string]string) map[st majorMinor := strings.Join(s[:2], ".") var ret map[string]string switch majorMinor { + case "1.12": + ret = map[string]string{ + "hyperkube": "hyperkube-amd64:v" + version, + "ccm": "cloud-controller-manager-amd64:v" + version, + "windowszip": "v" + version + "-1int.zip", + "dockerEngineVersion": k8sComponentVersions["1.12"]["dockerEngine"], + DefaultDashboardAddonName: k8sComponentVersions["1.12"]["dashboard"], + "addonresizer": k8sComponentVersions["1.12"]["addon-resizer"], + "heapster": k8sComponentVersions["1.12"]["heapster"], + DefaultMetricsServerAddonName: k8sComponentVersions["1.12"]["metrics-server"], + "dns": k8sComponentVersions["1.12"]["kube-dns"], + "addonmanager": k8sComponentVersions["1.12"]["addon-manager"], + "dnsmasq": k8sComponentVersions["1.12"]["dnsmasq"], + "pause": k8sComponentVersions["1.12"]["pause"], + DefaultTillerAddonName: k8sComponentVersions["1.12"]["tiller"], + DefaultReschedulerAddonName: k8sComponentVersions["1.12"]["rescheduler"], + DefaultACIConnectorAddonName: k8sComponentVersions["1.12"]["aci-connector"], + ContainerMonitoringAddonName: k8sComponentVersions["1.12"][ContainerMonitoringAddonName], + AzureCNINetworkMonitoringAddonName: k8sComponentVersions["1.12"][AzureCNINetworkMonitoringAddonName], + DefaultClusterAutoscalerAddonName: k8sComponentVersions["1.12"]["cluster-autoscaler"], + "k8s-dns-sidecar": k8sComponentVersions["1.12"]["k8s-dns-sidecar"], + "nodestatusfreq": k8sComponentVersions["1.12"]["nodestatusfreq"], + "nodegraceperiod": k8sComponentVersions["1.12"]["nodegraceperiod"], + "podeviction": k8sComponentVersions["1.12"]["podeviction"], + "routeperiod": k8sComponentVersions["1.12"]["routeperiod"], + "backoffretries": k8sComponentVersions["1.12"]["backoffretries"], + "backoffjitter": k8sComponentVersions["1.12"]["backoffjitter"], + "backoffduration": k8sComponentVersions["1.12"]["backoffduration"], + "backoffexponent": k8sComponentVersions["1.12"]["backoffexponent"], + "ratelimitqps": k8sComponentVersions["1.12"]["ratelimitqps"], + "ratelimitbucket": k8sComponentVersions["1.12"]["ratelimitbucket"], + "gchighthreshold": k8sComponentVersions["1.12"]["gchighthreshold"], + "gclowthreshold": k8sComponentVersions["1.12"]["gclowthreshold"], + NVIDIADevicePluginAddonName: k8sComponentVersions["1.12"][NVIDIADevicePluginAddonName], + } case "1.11": ret = map[string]string{ "hyperkube": "hyperkube-amd64:v" + version, diff --git a/pkg/api/common/versions.go b/pkg/api/common/versions.go index 9c7952a0c3..39b8a4d700 100644 --- a/pkg/api/common/versions.go +++ b/pkg/api/common/versions.go @@ -71,6 +71,7 @@ var AllKubernetesSupportedVersions = map[string]bool{ "1.11.0-rc.3": true, "1.11.0": true, "1.11.1": true, + "1.12.0-alpha.1": true, } // GetDefaultKubernetesVersion returns the default Kubernetes version, that is the latest patch of the default release From bdb0ef84362b37014f431f432b2317fde382450f Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Thu, 2 Aug 2018 14:44:13 -0700 Subject: [PATCH 2/4] update cluster-autoscaler for 1.12 See: https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.12.md#v1120-alpha1 --- pkg/acsengine/k8s_versions.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/acsengine/k8s_versions.go b/pkg/acsengine/k8s_versions.go index 7a564c96bc..90e8e4bbba 100644 --- a/pkg/acsengine/k8s_versions.go +++ b/pkg/acsengine/k8s_versions.go @@ -23,7 +23,7 @@ var k8sComponentVersions = map[string]map[string]string{ "aci-connector": "virtual-kubelet:latest", ContainerMonitoringAddonName: "oms:ciprod05082018", AzureCNINetworkMonitoringAddonName: "networkmonitor:v0.0.4", - "cluster-autoscaler": "cluster-autoscaler:v1.3.0", + "cluster-autoscaler": "cluster-autoscaler:v1.3.1", NVIDIADevicePluginAddonName: "k8s-device-plugin:1.11", "k8s-dns-sidecar": "k8s-dns-sidecar-amd64:1.14.10", "nodestatusfreq": DefaultKubernetesNodeStatusUpdateFrequency, From 8a57bb17d77582845b16b7e6fdd5685e652a8f9e Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Thu, 2 Aug 2018 16:45:33 -0700 Subject: [PATCH 3/4] --cadvisor-port not supported in 1.12 --- pkg/acsengine/defaults-kubelet.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/acsengine/defaults-kubelet.go b/pkg/acsengine/defaults-kubelet.go index df5b6cfae6..d7a93d8e43 100644 --- a/pkg/acsengine/defaults-kubelet.go +++ b/pkg/acsengine/defaults-kubelet.go @@ -98,6 +98,13 @@ func setKubeletConfig(cs *api.ContainerService) { } } + // Get rid of values not supported in v1.12 and up + if common.IsKubernetesVersionGe(o.OrchestratorVersion, "1.12.0-alpha.1") { + for _, key := range []string{"--cadvisor-port"} { + delete(o.KubernetesConfig.KubeletConfig, key) + } + } + // Remove secure kubelet flags, if configured if !helpers.IsTrueBoolPointer(o.KubernetesConfig.EnableSecureKubelet) { for _, key := range []string{"--anonymous-auth", "--client-ca-file"} { From bdf136cbd9704af81f1744c678850c81b0cf2ec9 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Thu, 2 Aug 2018 17:03:13 -0700 Subject: [PATCH 4/4] add windows 1.12 tests, remove 1.7 tests --- .circleci/config.yml | 71 ++++++++++++++++++++++---------------------- 1 file changed, 35 insertions(+), 36 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7ff2fae72b..617f9ecfb5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -58,31 +58,6 @@ jobs: path: /go/src/github.com/Azure/acs-engine/_logs - store_artifacts: path: /go/src/github.com/Azure/acs-engine/_output - k8s-1.7-release-e2e: - <<: *defaults - steps: - - checkout - - run: | - echo 'export TIMEOUT=20m' >> $BASH_ENV - echo 'export ORCHESTRATOR_RELEASE=1.7' >> $BASH_ENV - echo 'export CLUSTER_DEFINITION=examples/e2e-tests/kubernetes/release/default/definition.json' >> $BASH_ENV - echo 'export CREATE_VNET=true' >> $BASH_ENV - echo 'export CLEANUP_ON_EXIT=${CLEANUP_ON_EXIT}' >> $BASH_ENV - echo 'export RETAIN_SSH=false' >> $BASH_ENV - echo 'export SUBSCRIPTION_ID=${SUBSCRIPTION_ID_E2E_KUBERNETES}' >> $BASH_ENV - echo 'export CLIENT_ID=${SERVICE_PRINCIPAL_CLIENT_ID_E2E_KUBERNETES}' >> $BASH_ENV - echo 'export CLIENT_SECRET=${SERVICE_PRINCIPAL_CLIENT_SECRET_E2E_KUBERNETES}' >> $BASH_ENV - - run: - name: compile - command: make build-binary - - run: - name: ginkgo k8s e2e tests - command: make test-kubernetes - no_output_timeout: "30m" - - store_artifacts: - path: /go/src/github.com/Azure/acs-engine/_logs - - store_artifacts: - path: /go/src/github.com/Azure/acs-engine/_output k8s-linux-default-e2e: <<: *defaults steps: @@ -313,6 +288,30 @@ jobs: path: /go/src/github.com/Azure/acs-engine/_logs - store_artifacts: path: /go/src/github.com/Azure/acs-engine/_output + k8s-windows-1.12-release-e2e: + <<: *defaults + steps: + - checkout + - run: | + echo 'export TIMEOUT=30m' >> $BASH_ENV + echo 'export ORCHESTRATOR_RELEASE=1.12' >> $BASH_ENV + echo 'export CLUSTER_DEFINITION=examples/e2e-tests/kubernetes/windows/definition.json' >> $BASH_ENV + echo 'export CLEANUP_ON_EXIT=${CLEANUP_ON_EXIT}' >> $BASH_ENV + echo 'export RETAIN_SSH=false' >> $BASH_ENV + echo 'export SUBSCRIPTION_ID=${SUBSCRIPTION_ID_E2E_KUBERNETES}' >> $BASH_ENV + echo 'export CLIENT_ID=${SERVICE_PRINCIPAL_CLIENT_ID_E2E_KUBERNETES}' >> $BASH_ENV + echo 'export CLIENT_SECRET=${SERVICE_PRINCIPAL_CLIENT_SECRET_E2E_KUBERNETES}' >> $BASH_ENV + - run: + name: compile + command: make build-binary + - run: + name: ginkgo k8s windows e2e tests + command: make test-kubernetes + no_output_timeout: "30m" + - store_artifacts: + path: /go/src/github.com/Azure/acs-engine/_logs + - store_artifacts: + path: /go/src/github.com/Azure/acs-engine/_output workflows: version: 2 build_and_test_pr: @@ -334,31 +333,31 @@ workflows: filters: branches: ignore: master - - k8s-1.7-release-e2e: + - k8s-1.8-release-e2e: requires: - pr-e2e-hold filters: branches: ignore: master - - k8s-1.8-release-e2e: + - k8s-windows-1.9-release-e2e: requires: - pr-e2e-hold filters: branches: ignore: master - - k8s-windows-1.9-release-e2e: + - k8s-windows-1.10-release-e2e: requires: - pr-e2e-hold filters: branches: ignore: master - - k8s-windows-1.10-release-e2e: + - k8s-windows-1.11-release-e2e: requires: - pr-e2e-hold filters: branches: ignore: master - - k8s-windows-1.11-release-e2e: + - k8s-windows-1.12-release-e2e: requires: - pr-e2e-hold filters: @@ -400,12 +399,6 @@ workflows: filters: branches: only: master - - k8s-1.7-release-e2e: - requires: - - test - filters: - branches: - only: master - k8s-1.8-release-e2e: requires: - test @@ -454,6 +447,12 @@ workflows: filters: branches: only: master + - k8s-windows-1.12-release-e2e: + requires: + - test + filters: + branches: + only: master - dcos-e2e: requires: - test