diff --git a/README.md b/README.md index 4e270f73f..2fe52afcc 100644 --- a/README.md +++ b/README.md @@ -31,10 +31,6 @@ This extension controller supports the following Kubernetes versions: | Kubernetes 1.19 | 1.19.0+ | [![Gardener v1.19 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.19%20Azure/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.19%20Azure) | | Kubernetes 1.18 | 1.18.0+ | [![Gardener v1.18 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.18%20Azure/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.18%20Azure) | | Kubernetes 1.17 | 1.17.0+ | [![Gardener v1.17 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.17%20Azure/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.17%20Azure) | -| Kubernetes 1.16 | 1.16.0+, except 1.16.2 | [![Gardener v1.16 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.16%20Azure/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.16%20Azure) | -| Kubernetes 1.15 | 1.15.0+, except 1.15.5 | [1] | - -[1] Conformance tests are still executed and validated, unfortunately [no longer shown in TestGrid](https://github.com/kubernetes/test-infra/pull/18509#issuecomment-668204180). Please take a look [here](https://github.com/gardener/gardener/blob/master/docs/usage/supported_k8s_versions.md) to see which versions are supported by Gardener in general. diff --git a/charts/images.yaml b/charts/images.yaml index 5363f51a2..3d327a6ab 100644 --- a/charts/images.yaml +++ b/charts/images.yaml @@ -4,10 +4,6 @@ images: repository: eu.gcr.io/gardener-project/gardener/terraformer-azure tag: "v2.18.1" -- name: cloud-controller-manager - sourceRepository: github.com/kubernetes/kubernetes - repository: k8s.gcr.io/hyperkube - targetVersion: "< 1.17" - name: cloud-controller-manager sourceRepository: github.com/gardener/cloud-provider-azure repository: eu.gcr.io/gardener-project/kubernetes/cloud-provider-azure diff --git a/charts/internal/cloud-provider-config/values.yaml b/charts/internal/cloud-provider-config/values.yaml index 70b404ef5..3193324f4 100644 --- a/charts/internal/cloud-provider-config/values.yaml +++ b/charts/internal/cloud-provider-config/values.yaml @@ -1,4 +1,4 @@ -kubernetesVersion: 1.15.5 +kubernetesVersion: 1.17.0 tenantId: fooTenant subscriptionId: barSub aadClientId: fooClient diff --git a/charts/internal/seed-controlplane/charts/cloud-controller-manager/templates/cloud-controller-manager.yaml b/charts/internal/seed-controlplane/charts/cloud-controller-manager/templates/cloud-controller-manager.yaml index a4666f9e3..b71a918c6 100644 --- a/charts/internal/seed-controlplane/charts/cloud-controller-manager/templates/cloud-controller-manager.yaml +++ b/charts/internal/seed-controlplane/charts/cloud-controller-manager/templates/cloud-controller-manager.yaml @@ -38,11 +38,7 @@ spec: image: {{ index .Values.images "cloud-controller-manager" }} imagePullPolicy: IfNotPresent command: - {{- if semverCompare "< 1.17" .Values.kubernetesVersion }} - - /hyperkube - - cloud-controller-manager - - --allocate-node-cidrs=true - {{- else if semverCompare ">= 1.23" .Values.kubernetesVersion }} + {{- if semverCompare ">= 1.23" .Values.kubernetesVersion }} - /usr/local/bin/cloud-controller-manager - --allocate-node-cidrs=false {{- else }} diff --git a/charts/internal/seed-controlplane/charts/cloud-controller-manager/values.yaml b/charts/internal/seed-controlplane/charts/cloud-controller-manager/values.yaml index 89b92e432..6fe6bbd97 100644 --- a/charts/internal/seed-controlplane/charts/cloud-controller-manager/values.yaml +++ b/charts/internal/seed-controlplane/charts/cloud-controller-manager/values.yaml @@ -1,6 +1,6 @@ replicas: 1 clusterName: shoot-foo-bar -kubernetesVersion: 1.15.5 +kubernetesVersion: 1.23.9 podNetwork: 192.168.0.0/16 podAnnotations: {} podLabels: {} diff --git a/docs/usage-as-end-user.md b/docs/usage-as-end-user.md index 7beac9d3b..ec1b6a406 100644 --- a/docs/usage-as-end-user.md +++ b/docs/usage-as-end-user.md @@ -403,7 +403,7 @@ spec: nodes: 10.250.0.0/16 services: 100.64.0.0/13 kubernetes: - version: 1.16.1 + version: 1.24.3 maintenance: autoUpdate: kubernetesVersion: true @@ -460,7 +460,7 @@ spec: nodes: 10.250.0.0/16 services: 100.64.0.0/13 kubernetes: - version: 1.16.1 + version: 1.24.3 maintenance: autoUpdate: kubernetesVersion: true @@ -532,7 +532,7 @@ spec: nodes: 10.250.0.0/16 services: 100.64.0.0/13 kubernetes: - version: 1.16.1 + version: 1.24.3 maintenance: autoUpdate: kubernetesVersion: true diff --git a/docs/usage-as-operator.md b/docs/usage-as-operator.md index 86922b4b7..6c4341342 100644 --- a/docs/usage-as-operator.md +++ b/docs/usage-as-operator.md @@ -77,9 +77,9 @@ spec: type: azure kubernetes: versions: - - version: 1.16.1 - - version: 1.16.0 - expirationDate: "2020-04-05T01:02:03Z" + - version: 1.24.3 + - version: 1.23.8 + expirationDate: "2022-10-31T23:59:59Z" machineImages: - name: coreos versions: diff --git a/example/10-fake-shoot-controlplane.yaml b/example/10-fake-shoot-controlplane.yaml index 5d0783a36..afa1b7007 100644 --- a/example/10-fake-shoot-controlplane.yaml +++ b/example/10-fake-shoot-controlplane.yaml @@ -126,9 +126,8 @@ spec: spec: containers: - command: - - /hyperkube - - apiserver - - --enable-admission-plugins=Priority,NamespaceLifecycle,LimitRanger,PodSecurityPolicy,ServiceAccount,NodeRestriction,DefaultStorageClass,Initializers,DefaultTolerationSeconds,ResourceQuota,StorageObjectInUseProtection,MutatingAdmissionWebhook,ValidatingAdmissionWebhook + - /usr/local/bin/kube-apiserver + - --enable-admission-plugins=Priority,NamespaceLifecycle,LimitRanger,PodSecurityPolicy,ServiceAccount,NodeRestriction,DefaultStorageClass,DefaultTolerationSeconds,ResourceQuota,StorageObjectInUseProtection,MutatingAdmissionWebhook,ValidatingAdmissionWebhook - --disable-admission-plugins=PersistentVolumeLabel - --allow-privileged=true - --anonymous-auth=false @@ -147,7 +146,7 @@ spec: - --tls-cert-file=/srv/kubernetes/apiserver/kube-apiserver.crt - --tls-private-key-file=/srv/kubernetes/apiserver/kube-apiserver.key - --v=2 - image: k8s.gcr.io/hyperkube:v1.15.6 + image: registry.k8s.io/kube-apiserver:v1.17.17 imagePullPolicy: IfNotPresent name: kube-apiserver ports: diff --git a/example/30-controlplane.yaml b/example/30-controlplane.yaml index 8c31fc0cb..0f24dfc44 100644 --- a/example/30-controlplane.yaml +++ b/example/30-controlplane.yaml @@ -38,7 +38,7 @@ spec: networking: pods: 10.250.0.0/19 kubernetes: - version: 1.15.4 + version: 1.24.3 hibernation: enabled: false status: diff --git a/example/30-worker.yaml b/example/30-worker.yaml index 9f3adbb7b..7af6e583e 100644 --- a/example/30-worker.yaml +++ b/example/30-worker.yaml @@ -36,7 +36,7 @@ spec: kind: Shoot spec: kubernetes: - version: 1.15.4 + version: 1.24.3 status: lastOperation: state: Succeeded diff --git a/hack/api-reference/api.json b/hack/api-reference/api.json index d50497d19..841a7a399 100644 --- a/hack/api-reference/api.json +++ b/hack/api-reference/api.json @@ -9,7 +9,7 @@ "externalPackages": [ { "typeMatchPrefix": "^k8s\\.io/(api|apimachinery/pkg/apis)/", - "docsURLTemplate": "https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.15/#{{lower .TypeIdentifier}}-{{arrIndex .PackageSegments -1}}-{{arrIndex .PackageSegments -2}}" + "docsURLTemplate": "https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#{{lower .TypeIdentifier}}-{{arrIndex .PackageSegments -1}}-{{arrIndex .PackageSegments -2}}" } ], "typeDisplayNamePrefixOverrides": { diff --git a/hack/api-reference/config.json b/hack/api-reference/config.json index 33b5ff4e2..bd0810681 100644 --- a/hack/api-reference/config.json +++ b/hack/api-reference/config.json @@ -17,7 +17,7 @@ }, { "typeMatchPrefix": "^k8s\\.io/(api|apimachinery/pkg/apis)/", - "docsURLTemplate": "https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.15/#{{lower .TypeIdentifier}}-{{arrIndex .PackageSegments -1}}-{{arrIndex .PackageSegments -2}}" + "docsURLTemplate": "https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#{{lower .TypeIdentifier}}-{{arrIndex .PackageSegments -1}}-{{arrIndex .PackageSegments -2}}" }, { "typeMatchPrefix": "github.com/gardener/gardener/extensions/pkg/apis/config", diff --git a/pkg/controller/controlplane/valuesprovider_test.go b/pkg/controller/controlplane/valuesprovider_test.go index 4e4f56600..b13381564 100644 --- a/pkg/controller/controlplane/valuesprovider_test.go +++ b/pkg/controller/controlplane/valuesprovider_test.go @@ -109,7 +109,7 @@ var _ = Describe("ValuesProvider", func() { cidr = "10.250.0.0/19" cloudProviderConfigData = "foo" - k8sVersionLessThan121 = "1.15.4" + k8sVersionLessThan121 = "1.17.1" k8sVersionHigherEqual121 = "1.21.4" enabledTrue = map[string]interface{}{"enabled": true} @@ -491,7 +491,7 @@ var _ = Describe("ValuesProvider", func() { "checksum/configmap-" + azure.CloudProviderDiskConfigName: "", }, "cloudProviderConfig": "", - "kubernetesVersion": "1.15.4", + "kubernetesVersion": "1.17.1", }) globalVpaDisabled = map[string]interface{}{ "vpaEnabled": false, diff --git a/pkg/webhook/controlplane/ensurer.go b/pkg/webhook/controlplane/ensurer.go index d1253dd04..b1ef53c6e 100644 --- a/pkg/webhook/controlplane/ensurer.go +++ b/pkg/webhook/controlplane/ensurer.go @@ -33,7 +33,6 @@ import ( oscutils "github.com/gardener/gardener/pkg/operation/botanist/component/extensions/operatingsystemconfig/utils" kutil "github.com/gardener/gardener/pkg/utils/kubernetes" "github.com/gardener/gardener/pkg/utils/version" - versionutils "github.com/gardener/gardener/pkg/utils/version" "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -335,11 +334,10 @@ func ensureKubeControllerManagerVolumeMounts(c *corev1.Container, version string } c.VolumeMounts = extensionswebhook.EnsureVolumeMountWithName(c.VolumeMounts, cloudProviderConfigVolumeMount) - if mustMountEtcSSLFolder(version) { - c.VolumeMounts = extensionswebhook.EnsureVolumeMountWithName(c.VolumeMounts, etcSSLVolumeMount) - // some distros have symlinks from /etc/ssl/certs to /usr/share/ca-certificates - c.VolumeMounts = extensionswebhook.EnsureVolumeMountWithName(c.VolumeMounts, usrShareCaCertsVolumeMount) - } + + c.VolumeMounts = extensionswebhook.EnsureVolumeMountWithName(c.VolumeMounts, etcSSLVolumeMount) + // some distros have symlinks from /etc/ssl/certs to /usr/share/ca-certificates + c.VolumeMounts = extensionswebhook.EnsureVolumeMountWithName(c.VolumeMounts, usrShareCaCertsVolumeMount) } func ensureKubeAPIServerVolumes(ps *corev1.PodSpec, csiEnabled, csiMigrationComplete bool) { @@ -360,22 +358,10 @@ func ensureKubeControllerManagerVolumes(ps *corev1.PodSpec, version string, csiE } ps.Volumes = extensionswebhook.EnsureVolumeWithName(ps.Volumes, cloudProviderConfigVolume) - if mustMountEtcSSLFolder(version) { - ps.Volumes = extensionswebhook.EnsureVolumeWithName(ps.Volumes, etcSSLVolume) - // some distros have symlinks from /etc/ssl/certs to /usr/share/ca-certificates - ps.Volumes = extensionswebhook.EnsureVolumeWithName(ps.Volumes, usrShareCaCertsVolume) - } -} -// Beginning with 1.17 Gardener no longer uses the hyperkube image for the Kubernetes control plane components. -// The hyperkube image contained all the well-known root CAs, but the dedicated images don't. This is why we -// mount the /etc/ssl folder from the host here. -func mustMountEtcSSLFolder(version string) bool { - k8sVersionAtLeast117, err := versionutils.CompareVersions(version, ">=", "1.17") - if err != nil { - return false - } - return k8sVersionAtLeast117 + ps.Volumes = extensionswebhook.EnsureVolumeWithName(ps.Volumes, etcSSLVolume) + // some distros have symlinks from /etc/ssl/certs to /usr/share/ca-certificates + ps.Volumes = extensionswebhook.EnsureVolumeWithName(ps.Volumes, usrShareCaCertsVolume) } func (e *ensurer) ensureChecksumAnnotations(ctx context.Context, template *corev1.PodTemplateSpec, namespace string, csiEnabled, csiMigrationComplete bool) error { diff --git a/pkg/webhook/controlplane/ensurer_test.go b/pkg/webhook/controlplane/ensurer_test.go index 93fc817f7..6965c4772 100644 --- a/pkg/webhook/controlplane/ensurer_test.go +++ b/pkg/webhook/controlplane/ensurer_test.go @@ -66,20 +66,6 @@ var _ = Describe("Ensurer", func() { c *mockclient.MockClient dummyContext = gcontext.NewGardenContext(nil, nil) - eContextK8s116 = gcontext.NewInternalGardenContext( - &extensionscontroller.Cluster{ - Shoot: &gardencorev1beta1.Shoot{ - Spec: gardencorev1beta1.ShootSpec{ - Kubernetes: gardencorev1beta1.Kubernetes{ - Version: "1.16.0", - }, - }, - Status: gardencorev1beta1.ShootStatus{ - TechnicalID: namespace, - }, - }, - }, - ) eContextK8s117 = gcontext.NewInternalGardenContext( &extensionscontroller.Cluster{ Shoot: &gardencorev1beta1.Shoot{ @@ -178,15 +164,6 @@ var _ = Describe("Ensurer", func() { } }) - It("should add missing elements to kube-apiserver deployment (k8s < 1.17)", func() { - c.EXPECT().Get(ctx, key, &corev1.Secret{}).DoAndReturn(clientGet(secret)) - - err := ensurer.EnsureKubeAPIServerDeployment(ctx, eContextK8s116, dep, nil) - Expect(err).To(Not(HaveOccurred())) - - checkKubeAPIServerDeployment(dep, annotations, "1.16.0", false) - }) - It("should add missing elements to kube-apiserver deployment (k8s >= 1.17, < 1.21)", func() { c.EXPECT().Get(ctx, key, &corev1.Secret{}).DoAndReturn(clientGet(secret)) @@ -242,8 +219,8 @@ var _ = Describe("Ensurer", func() { c.EXPECT().Get(ctx, key, &corev1.Secret{}).DoAndReturn(clientGet(secret)) - Expect(ensurer.EnsureKubeAPIServerDeployment(ctx, eContextK8s116, dep, nil)).To(Not(HaveOccurred())) - checkKubeAPIServerDeployment(dep, annotations, "1.16.0", false) + Expect(ensurer.EnsureKubeAPIServerDeployment(ctx, eContextK8s117, dep, nil)).To(Not(HaveOccurred())) + checkKubeAPIServerDeployment(dep, annotations, "1.17.0", false) }) }) @@ -267,15 +244,6 @@ var _ = Describe("Ensurer", func() { } }) - It("should add missing elements to kube-controller-manager deployment (k8s < 1.17)", func() { - c.EXPECT().Get(ctx, key, &corev1.Secret{}).DoAndReturn(clientGet(secret)) - - err := ensurer.EnsureKubeControllerManagerDeployment(ctx, eContextK8s116, dep, nil) - Expect(err).To(Not(HaveOccurred())) - - checkKubeControllerManagerDeployment(dep, annotations, kubeControllerManagerLabels, "1.16.4", false) - }) - It("should add missing elements to kube-controller-manager deployment (k8s >= 1.17, k8s < 1.21)", func() { c.EXPECT().Get(ctx, key, &corev1.Secret{}).DoAndReturn(clientGet(secret)) @@ -330,9 +298,9 @@ var _ = Describe("Ensurer", func() { c.EXPECT().Get(ctx, key, &corev1.Secret{}).DoAndReturn(clientGet(secret)) - err := ensurer.EnsureKubeControllerManagerDeployment(ctx, eContextK8s116, dep, nil) + err := ensurer.EnsureKubeControllerManagerDeployment(ctx, eContextK8s117, dep, nil) Expect(err).To(Not(HaveOccurred())) - checkKubeControllerManagerDeployment(dep, annotations, kubeControllerManagerLabels, "1.16.0", false) + checkKubeControllerManagerDeployment(dep, annotations, kubeControllerManagerLabels, "1.17.0", false) }) }) @@ -617,7 +585,6 @@ func checkKubeAPIServerDeployment(dep *appsv1.Deployment, annotations map[string } func checkKubeControllerManagerDeployment(dep *appsv1.Deployment, annotations, labels map[string]string, k8sVersion string, needsCSIMigrationCompletedFeatureGates bool) { - k8sVersionLessThan117, _ := version.CompareVersions(k8sVersion, "<", "1.17") k8sVersionAtLeast121, _ := version.CompareVersions(k8sVersion, ">=", "1.21") // Check that the kube-controller-manager container still exists and contains all needed command line args, @@ -634,12 +601,10 @@ func checkKubeControllerManagerDeployment(dep *appsv1.Deployment, annotations, l Expect(dep.Spec.Template.Annotations).To(Equal(annotations)) Expect(dep.Spec.Template.Labels).To(Equal(labels)) Expect(dep.Spec.Template.Spec.Volumes).To(ContainElement(cloudProviderConfigVolume)) - if !k8sVersionLessThan117 { - Expect(c.VolumeMounts).To(ContainElement(etcSSLVolumeMount)) - Expect(dep.Spec.Template.Spec.Volumes).To(ContainElement(etcSSLVolume)) - Expect(c.VolumeMounts).To(ContainElement(usrShareCaCertsVolumeMount)) - Expect(dep.Spec.Template.Spec.Volumes).To(ContainElement(usrShareCaCertsVolume)) - } + Expect(c.VolumeMounts).To(ContainElement(etcSSLVolumeMount)) + Expect(dep.Spec.Template.Spec.Volumes).To(ContainElement(etcSSLVolume)) + Expect(c.VolumeMounts).To(ContainElement(usrShareCaCertsVolumeMount)) + Expect(dep.Spec.Template.Spec.Volumes).To(ContainElement(usrShareCaCertsVolume)) if k8sVersionAtLeast121 { Expect(c.Command).To(ContainElement("--feature-gates=CSIMigration=true,CSIMigrationAzureDisk=true,CSIMigrationAzureFile=true")) }