diff --git a/parts/k8s/addons/metrics-server.yaml b/parts/k8s/addons/metrics-server.yaml index ef1bb6549b..d2d94f16a5 100644 --- a/parts/k8s/addons/metrics-server.yaml +++ b/parts/k8s/addons/metrics-server.yaml @@ -1,68 +1,82 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: metrics-server - namespace: kube-system labels: - kubernetes.io/cluster-service: "true" + k8s-app: metrics-server addonmanager.kubernetes.io/mode: {{GetMode}} + name: metrics-server + namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: system:metrics-server labels: - kubernetes.io/cluster-service: "true" + k8s-app: metrics-server addonmanager.kubernetes.io/mode: {{GetMode}} + rbac.authorization.k8s.io/aggregate-to-admin: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rbac.authorization.k8s.io/aggregate-to-view: "true" + name: system:aggregated-metrics-reader rules: - apiGroups: - - "" + - metrics.k8s.io resources: - pods - nodes - - nodes/stats - - namespaces verbs: - get - list - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + k8s-app: metrics-server + addonmanager.kubernetes.io/mode: {{GetMode}} + name: system:metrics-server +rules: - apiGroups: - - "extensions" + - "" resources: - - deployments + - pods + - nodes + - nodes/stats + - namespaces + - configmaps verbs: - get - list - watch --- apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding +kind: RoleBinding metadata: - name: system:metrics-server labels: - kubernetes.io/cluster-service: "true" + k8s-app: metrics-server addonmanager.kubernetes.io/mode: {{GetMode}} + name: metrics-server-auth-reader + namespace: kube-system roleRef: apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:metrics-server + kind: Role + name: extension-apiserver-authentication-reader subjects: - kind: ServiceAccount name: metrics-server namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding +kind: ClusterRoleBinding metadata: - name: metrics-server-auth-reader - namespace: kube-system labels: - kubernetes.io/cluster-service: "true" + k8s-app: metrics-server addonmanager.kubernetes.io/mode: {{GetMode}} + name: metrics-server:system:auth-delegator roleRef: apiGroup: rbac.authorization.k8s.io - kind: Role - name: extension-apiserver-authentication-reader + kind: ClusterRole + name: system:auth-delegator subjects: - kind: ServiceAccount name: metrics-server @@ -71,14 +85,14 @@ subjects: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: metrics-server:system:auth-delegator labels: - kubernetes.io/cluster-service: "true" + k8s-app: metrics-server addonmanager.kubernetes.io/mode: {{GetMode}} + name: system:metrics-server roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: system:auth-delegator + name: system:metrics-server subjects: - kind: ServiceAccount name: metrics-server @@ -87,65 +101,96 @@ subjects: apiVersion: v1 kind: Service metadata: - name: metrics-server - namespace: kube-system labels: + k8s-app: metrics-server addonmanager.kubernetes.io/mode: {{GetMode}} - kubernetes.io/name: "Metrics-server" - kubernetes.io/cluster-service: "true" + name: metrics-server + namespace: kube-system spec: - selector: - k8s-app: metrics-server ports: - - port: 443 + - name: https + port: 443 protocol: TCP - targetPort: 443 + targetPort: https + selector: + k8s-app: metrics-server --- apiVersion: apps/v1 kind: Deployment metadata: - name: metrics-server - namespace: kube-system labels: k8s-app: metrics-server - kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: {{GetMode}} + name: metrics-server + namespace: kube-system spec: selector: matchLabels: k8s-app: metrics-server + strategy: + rollingUpdate: + maxUnavailable: 0 template: metadata: - name: metrics-server labels: k8s-app: metrics-server spec: - serviceAccountName: metrics-server - priorityClassName: system-cluster-critical containers: - - name: metrics-server + - args: + - --cert-dir=/tmp + - --secure-port=4443 + - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname + - --kubelet-use-node-status-port + - --kubelet-insecure-tls image: {{ContainerImage "metrics-server"}} imagePullPolicy: IfNotPresent - command: - - /metrics-server - - --kubelet-insecure-tls - - --kubelet-preferred-address-types=InternalIP + livenessProbe: + failureThreshold: 3 + httpGet: + path: /livez + port: https + scheme: HTTPS + periodSeconds: 10 + name: metrics-server + ports: + - containerPort: 4443 + name: https + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readyz + port: https + scheme: HTTPS + periodSeconds: 10 + securityContext: + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + volumeMounts: + - mountPath: /tmp + name: tmp-dir nodeSelector: kubernetes.io/os: linux + priorityClassName: system-cluster-critical + serviceAccountName: metrics-server + volumes: + - emptyDir: {} + name: tmp-dir --- apiVersion: apiregistration.k8s.io/v1{{- if not (IsKubernetesVersionGe "1.19.0")}}beta1{{end}} kind: APIService metadata: - name: v1beta1.metrics.k8s.io labels: - kubernetes.io/cluster-service: "true" + k8s-app: metrics-server addonmanager.kubernetes.io/mode: {{GetMode}} + name: v1beta1.metrics.k8s.io spec: + group: metrics.k8s.io + groupPriorityMinimum: 100 + insecureSkipTLSVerify: true service: name: metrics-server namespace: kube-system - group: metrics.k8s.io version: v1beta1 - insecureSkipTLSVerify: true - groupPriorityMinimum: 100 versionPriority: 100 diff --git a/pkg/api/k8s_versions.go b/pkg/api/k8s_versions.go index e892a0e171..f54aa76e91 100644 --- a/pkg/api/k8s_versions.go +++ b/pkg/api/k8s_versions.go @@ -116,43 +116,43 @@ var kubernetesImageBaseVersionedImages = map[string]map[string]map[string]string common.KubernetesImageBaseTypeGCR: { "1.22": { common.AddonResizerComponentName: "addon-resizer:1.8.7", - common.MetricsServerAddonName: "metrics-server/metrics-server:v0.4.1", + common.MetricsServerAddonName: "metrics-server/metrics-server:v0.4.4", common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.5", common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", }, "1.21": { common.AddonResizerComponentName: "addon-resizer:1.8.7", - common.MetricsServerAddonName: "metrics-server/metrics-server:v0.4.1", + common.MetricsServerAddonName: "metrics-server/metrics-server:v0.4.4", common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.3", common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", }, "1.20": { common.AddonResizerComponentName: "addon-resizer:1.8.7", - common.MetricsServerAddonName: "metrics-server/metrics-server:v0.4.1", + common.MetricsServerAddonName: "metrics-server/metrics-server:v0.4.4", common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.3", common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", }, "1.19": { common.AddonResizerComponentName: "addon-resizer:1.8.7", - common.MetricsServerAddonName: "metrics-server/metrics-server:v0.4.1", + common.MetricsServerAddonName: "metrics-server/metrics-server:v0.4.4", common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.3", common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", }, "1.18": { common.AddonResizerComponentName: "addon-resizer:1.8.7", - common.MetricsServerAddonName: "metrics-server/metrics-server:v0.4.1", + common.MetricsServerAddonName: "metrics-server/metrics-server:v0.4.4", common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.3", common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.18.0", }, "1.17": { common.AddonResizerComponentName: "addon-resizer:1.8.7", - common.MetricsServerAddonName: "metrics-server/metrics-server:v0.4.1", + common.MetricsServerAddonName: "metrics-server/metrics-server:v0.4.4", common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.3", common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.17.1", }, "1.16": { common.AddonResizerComponentName: "addon-resizer:1.8.7", - common.MetricsServerAddonName: "metrics-server/metrics-server:v0.4.1", + common.MetricsServerAddonName: "metrics-server/metrics-server:v0.4.4", common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.3", common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.16.4", }, @@ -240,43 +240,43 @@ var kubernetesImageBaseVersionedImages = map[string]map[string]map[string]string common.KubernetesImageBaseTypeMCR: { "1.22": { common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", - common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.3.7", + common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.4.4", common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.5", common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.20.0", }, "1.21": { common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", - common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.3.7", + common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.4.4", common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.3", common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.20.0", }, "1.20": { common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", - common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.3.7", + common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.4.4", common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.3", common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.20.0", }, "1.19": { common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", - common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.3.7", + common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.4.4", common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.3", common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.20.0", }, "1.18": { common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", - common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.3.7", + common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.4.4", common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.3", common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.20.0", }, "1.17": { common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", - common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.3.7", + common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.4.4", common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.3", common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.20.0", }, "1.16": { common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.7", - common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.3.7", + common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.4.4", common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.3", common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.16.7", }, diff --git a/pkg/engine/templates_generated.go b/pkg/engine/templates_generated.go index 773520f88d..4a174c67db 100644 --- a/pkg/engine/templates_generated.go +++ b/pkg/engine/templates_generated.go @@ -9888,68 +9888,82 @@ func k8sAddonsKubernetesDashboardYaml() (*asset, error) { var _k8sAddonsMetricsServerYaml = []byte(`apiVersion: v1 kind: ServiceAccount metadata: - name: metrics-server - namespace: kube-system labels: - kubernetes.io/cluster-service: "true" + k8s-app: metrics-server addonmanager.kubernetes.io/mode: {{GetMode}} + name: metrics-server + namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: system:metrics-server labels: - kubernetes.io/cluster-service: "true" + k8s-app: metrics-server addonmanager.kubernetes.io/mode: {{GetMode}} + rbac.authorization.k8s.io/aggregate-to-admin: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rbac.authorization.k8s.io/aggregate-to-view: "true" + name: system:aggregated-metrics-reader rules: - apiGroups: - - "" + - metrics.k8s.io resources: - pods - nodes - - nodes/stats - - namespaces verbs: - get - list - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + k8s-app: metrics-server + addonmanager.kubernetes.io/mode: {{GetMode}} + name: system:metrics-server +rules: - apiGroups: - - "extensions" + - "" resources: - - deployments + - pods + - nodes + - nodes/stats + - namespaces + - configmaps verbs: - get - list - watch --- apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding +kind: RoleBinding metadata: - name: system:metrics-server labels: - kubernetes.io/cluster-service: "true" + k8s-app: metrics-server addonmanager.kubernetes.io/mode: {{GetMode}} + name: metrics-server-auth-reader + namespace: kube-system roleRef: apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:metrics-server + kind: Role + name: extension-apiserver-authentication-reader subjects: - kind: ServiceAccount name: metrics-server namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding +kind: ClusterRoleBinding metadata: - name: metrics-server-auth-reader - namespace: kube-system labels: - kubernetes.io/cluster-service: "true" + k8s-app: metrics-server addonmanager.kubernetes.io/mode: {{GetMode}} + name: metrics-server:system:auth-delegator roleRef: apiGroup: rbac.authorization.k8s.io - kind: Role - name: extension-apiserver-authentication-reader + kind: ClusterRole + name: system:auth-delegator subjects: - kind: ServiceAccount name: metrics-server @@ -9958,14 +9972,14 @@ subjects: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: metrics-server:system:auth-delegator labels: - kubernetes.io/cluster-service: "true" + k8s-app: metrics-server addonmanager.kubernetes.io/mode: {{GetMode}} + name: system:metrics-server roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: system:auth-delegator + name: system:metrics-server subjects: - kind: ServiceAccount name: metrics-server @@ -9974,67 +9988,98 @@ subjects: apiVersion: v1 kind: Service metadata: - name: metrics-server - namespace: kube-system labels: + k8s-app: metrics-server addonmanager.kubernetes.io/mode: {{GetMode}} - kubernetes.io/name: "Metrics-server" - kubernetes.io/cluster-service: "true" + name: metrics-server + namespace: kube-system spec: - selector: - k8s-app: metrics-server ports: - - port: 443 + - name: https + port: 443 protocol: TCP - targetPort: 443 + targetPort: https + selector: + k8s-app: metrics-server --- apiVersion: apps/v1 kind: Deployment metadata: - name: metrics-server - namespace: kube-system labels: k8s-app: metrics-server - kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: {{GetMode}} + name: metrics-server + namespace: kube-system spec: selector: matchLabels: k8s-app: metrics-server + strategy: + rollingUpdate: + maxUnavailable: 0 template: metadata: - name: metrics-server labels: k8s-app: metrics-server spec: - serviceAccountName: metrics-server - priorityClassName: system-cluster-critical containers: - - name: metrics-server + - args: + - --cert-dir=/tmp + - --secure-port=4443 + - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname + - --kubelet-use-node-status-port + - --kubelet-insecure-tls image: {{ContainerImage "metrics-server"}} imagePullPolicy: IfNotPresent - command: - - /metrics-server - - --kubelet-insecure-tls - - --kubelet-preferred-address-types=InternalIP + livenessProbe: + failureThreshold: 3 + httpGet: + path: /livez + port: https + scheme: HTTPS + periodSeconds: 10 + name: metrics-server + ports: + - containerPort: 4443 + name: https + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readyz + port: https + scheme: HTTPS + periodSeconds: 10 + securityContext: + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + volumeMounts: + - mountPath: /tmp + name: tmp-dir nodeSelector: kubernetes.io/os: linux + priorityClassName: system-cluster-critical + serviceAccountName: metrics-server + volumes: + - emptyDir: {} + name: tmp-dir --- apiVersion: apiregistration.k8s.io/v1{{- if not (IsKubernetesVersionGe "1.19.0")}}beta1{{end}} kind: APIService metadata: - name: v1beta1.metrics.k8s.io labels: - kubernetes.io/cluster-service: "true" + k8s-app: metrics-server addonmanager.kubernetes.io/mode: {{GetMode}} + name: v1beta1.metrics.k8s.io spec: + group: metrics.k8s.io + groupPriorityMinimum: 100 + insecureSkipTLSVerify: true service: name: metrics-server namespace: kube-system - group: metrics.k8s.io version: v1beta1 - insecureSkipTLSVerify: true - groupPriorityMinimum: 100 versionPriority: 100 `) diff --git a/vhd/packer/install-dependencies.sh b/vhd/packer/install-dependencies.sh index ec1d5ae739..16fd4a8c4b 100644 --- a/vhd/packer/install-dependencies.sh +++ b/vhd/packer/install-dependencies.sh @@ -128,7 +128,7 @@ systemctl status docker --no-pager || exit 1 echo "Docker images pre-pulled:" >> ${VHD_LOGS_FILEPATH} METRICS_SERVER_VERSIONS=" -0.3.7 +0.4.4 " for METRICS_SERVER_VERSION in ${METRICS_SERVER_VERSIONS}; do CONTAINER_IMAGE="mcr.microsoft.com/oss/kubernetes/metrics-server:v${METRICS_SERVER_VERSION}"