Merge branch 'master' into feature/virtualbox

* master: (21 commits)
  Add tz to kubespray image
  Add tag to AWS VPC subnets for automatic subnet discovery by load balancers or ingress controllers (kubernetes-sigs#8705)
  Enable external CA mode for control-plane deployment (kubernetes-sigs#8620)
  UpCloud integration (kubernetes-sigs#8653)
  Fixes for Hetzner terraform and Hetzner Cloud  (kubernetes-sigs#8702)
  Add VAGRANT_ANSIBLE_TAGS for normal deployment (kubernetes-sigs#8697)
  Removed quotation of nerdctl_extra_flags. (kubernetes-sigs#8695)
  [calico] add calico apiserver (kubernetes-sigs#8690)
  Add support for kube-vip (kubernetes-sigs#8669)
  Ensure all Kubelet required kernel values are configured when enabling protectKernelDefaults (kubernetes-sigs#8692)
  [cert-manager] Upgrade to v1.8.0 (kubernetes-sigs#8688)
  fix: reset docker was not removing docker properly (kubernetes-sigs#8680)
  Single quotes are missing in jsonpath argument of kubectl get node (kubernetes-sigs#8683)
  split kube_feature_gates variable for different kubernetes components (kubernetes-sigs#8677)
  [crun] upgrade to 1.4.4 (kubernetes-sigs#8675)
  [validate-container-engine] check if kubelet is present was not working (kubernetes-sigs#8679)
  [containerd] upgrade versions to address CVE-2022-24769 (kubernetes-sigs#8671)
  [vsphere_csi] update to 2.5.1 and make external_vsphere_version 7.0u1 by default (kubernetes-sigs#8676)
  [runc] upgrade to 1.1.1 (kubernetes-sigs#8674)
  [nerdctl] upgrade to 0.18.0 (kubernetes-sigs#8672)
  ...

Dockerfile

@@ -1,6 +1,9 @@
 # Use imutable image tags rather than mutable tags (like ubuntu:20.04)
 FROM ubuntu:focal-20220316
 
+ARG TZ=Etc/UTC
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
 RUN apt update -y \
     && apt install -y \
     libssl-dev python3-dev sshpass apt-transport-https jq moreutils \

README.md

@@ -134,7 +134,7 @@ Note: Upstart/SysV init based OS types are not supported.
   - [kubernetes](https://github.com/kubernetes/kubernetes) v1.23.5
   - [etcd](https://github.com/etcd-io/etcd) v3.5.1
   - [docker](https://www.docker.com/) v20.10 (see note)
-  - [containerd](https://containerd.io/) v1.6.1
+  - [containerd](https://containerd.io/) v1.6.2
   - [cri-o](http://cri-o.io/) v1.22 (experimental: see [CRI-O Note](docs/cri-o.md). Only on fedora, ubuntu and centos based OS)
 - Network Plugin
   - [cni-plugins](https://github.com/containernetworking/plugins) v1.0.1

Vagrantfile

@@ -75,6 +75,7 @@ $local_path_provisioner_claim_root ||= "/opt/local-path-provisioner/"
 $libvirt_nested ||= false
 # boolean or string (e.g. "-vvv")
 $ansible_verbosity ||= false
+$ansible_tags ||= ENV['VAGRANT_ANSIBLE_TAGS'] || ""
 
 $playbook ||= "cluster.yml"
 
@@ -256,7 +257,9 @@ Vagrant.configure("2") do |config|
           ansible.host_key_checking = false
           ansible.raw_arguments = ["--forks=#{$num_instances}", "--flush-cache", "-e ansible_become_pass=vagrant"]
           ansible.host_vars = host_vars
-          ansible.tags = ['facts']
+          if $ansible_tags != ""
+            ansible.tags = [$ansible_tags]
+          end
           ansible.groups = {
             "etcd" => ["#{$instance_name_prefix}-[1:#{$etcd_instances}]"],
             "kube_control_plane" => ["#{$instance_name_prefix}-[1:#{$kube_master_instances}]"],

contrib/terraform/aws/modules/vpc/main.tf

@@ -31,7 +31,8 @@ resource "aws_subnet" "cluster-vpc-subnets-public" {
 
   tags = merge(var.default_tags, tomap({
     Name = "kubernetes-${var.aws_cluster_name}-${element(var.aws_avail_zones, count.index)}-public"
-    "kubernetes.io/cluster/${var.aws_cluster_name}" = "member"
+    "kubernetes.io/cluster/${var.aws_cluster_name}" = "shared"
+    "kubernetes.io/role/elb" = "1"
   }))
 }
 
@@ -49,6 +50,8 @@ resource "aws_subnet" "cluster-vpc-subnets-private" {
 
   tags = merge(var.default_tags, tomap({
     Name = "kubernetes-${var.aws_cluster_name}-${element(var.aws_avail_zones, count.index)}-private"
+    "kubernetes.io/cluster/${var.aws_cluster_name}" = "shared"
+    "kubernetes.io/role/internal-elb" = "1"
   }))
 }
 

contrib/terraform/hetzner/README.md

@@ -97,6 +97,7 @@ terraform destroy --var-file default.tfvars ../../contrib/terraform/hetzner
 * `prefix`: Prefix to add to all resources, if set to "" don't set any prefix
 * `ssh_public_keys`: List of public SSH keys to install on all machines
 * `zone`: The zone where to run the cluster
+* `network_zone`: the network zone where the cluster is running
 * `machines`: Machines to provision. Key of this object will be used as the name of the machine
   * `node_type`: The role of this node *(master|worker)*
   * `size`: Size of the VM

contrib/terraform/hetzner/default.tfvars

@@ -1,6 +1,6 @@
 prefix = "default"
 zone   = "hel1"
-
+network_zone = "eu-central"
 inventory_file = "inventory.ini"
 
 ssh_public_keys = [

contrib/terraform/hetzner/main.tf

@@ -10,6 +10,7 @@ module "kubernetes" {
   machines = var.machines
 
   ssh_public_keys = var.ssh_public_keys
+  network_zone = var.network_zone
 
   ssh_whitelist        = var.ssh_whitelist
   api_server_whitelist = var.api_server_whitelist
@@ -34,9 +35,9 @@ data "template_file" "inventory" {
       keys(module.kubernetes.worker_ip_addresses),
       values(module.kubernetes.worker_ip_addresses).*.public_ip,
     values(module.kubernetes.worker_ip_addresses).*.private_ip))
-
     list_master = join("\n", keys(module.kubernetes.master_ip_addresses))
     list_worker = join("\n", keys(module.kubernetes.worker_ip_addresses))
+    network_id = module.kubernetes.network_id
   }
 }
 

contrib/terraform/hetzner/modules/kubernetes-cluster/main.tf

@@ -6,7 +6,7 @@ resource "hcloud_network" "kubernetes" {
 resource "hcloud_network_subnet" "kubernetes" {
   type         = "cloud"
   network_id   = hcloud_network.kubernetes.id
-  network_zone = "eu-central"
+  network_zone = var.network_zone
   ip_range     = var.private_subnet_cidr
 }
 

contrib/terraform/hetzner/modules/kubernetes-cluster/output.tf

@@ -21,3 +21,7 @@ output "worker_ip_addresses" {
 output "cluster_private_network_cidr" {
   value = var.private_subnet_cidr
 }
+
+output "network_id" {
+  value = hcloud_network.kubernetes.id
+}

contrib/terraform/hetzner/modules/kubernetes-cluster/variables.tf

@@ -39,3 +39,6 @@ variable "private_network_cidr" {
 variable "private_subnet_cidr" {
   default = "10.0.10.0/24"
 }
+variable "network_zone" {
+  default = "eu-central"
+}

contrib/terraform/hetzner/templates/inventory.tpl

@@ -14,3 +14,6 @@ ${list_worker}
 [k8s-cluster:children]
 kube-master
 kube-node
+
+[k8s-cluster:vars]
+network_id=${network_id}

contrib/terraform/hetzner/variables.tf

@@ -1,6 +1,10 @@
 variable "zone" {
   description = "The zone where to run the cluster"
 }
+variable "network_zone" {
+  description = "The network zone where the cluster is running"
+  default = "eu-central"
+}
 
 variable "prefix" {
   description = "Prefix for resource names"

contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf

@@ -65,6 +65,11 @@ resource "upcloud_server" "master" {
     network = upcloud_network.private.id
   }
 
+  # Ignore volumes created by csi-driver
+  lifecycle {
+    ignore_changes = [storage_devices]
+  }
+
   dynamic "storage_devices" {
     for_each = {
       for disk_key_name, disk in upcloud_storage.additional_disks :
@@ -114,6 +119,11 @@ resource "upcloud_server" "worker" {
     network = upcloud_network.private.id
   }
 
+  # Ignore volumes created by csi-driver
+  lifecycle {
+    ignore_changes = [storage_devices]
+  }
+
   dynamic "storage_devices" {
     for_each = {
       for disk_key_name, disk in upcloud_storage.additional_disks :

docs/_sidebar.md

@@ -18,6 +18,7 @@
   * [Weave](docs/weave.md)
   * [Multus](docs/multus.md)
 * Ingress
+  * [kube-vip](docs/kube-vip.md)
   * [ALB Ingress](docs/ingress_controller/alb_ingress_controller.md)
   * [MetalLB](docs/metallb.md)
   * [Nginx Ingress](docs/ingress_controller/ingress_nginx.md)

docs/ansible.md

@@ -156,6 +156,7 @@ The following tags are defined in playbooks:
 |                        kubeadm | Roles linked to kubeadm tasks
 |                 kube-apiserver | Configuring static pod kube-apiserver
 |        kube-controller-manager | Configuring static pod kube-controller-manager
+|                       kube-vip | Installing and configuring kube-vip
 |                        kubectl | Installing kubectl and bash completion
 |                        kubelet | Configuring kubelet service
 |                       kube-ovn | Network plugin kube-ovn

docs/ha-mode.md

@@ -29,9 +29,7 @@ configure kubelet and kube-proxy on non-master nodes to use the local internal
 loadbalancer.
 
 If you choose to NOT use the local internal loadbalancer, you will need to
-configure your own loadbalancer to achieve HA. Note that deploying a
-loadbalancer is up to a user and is not covered by ansible roles in Kubespray.
-By default, it only configures a non-HA endpoint, which points to the
+use the [kube-vip](kube-vip.md) ansible role or configure your own loadbalancer to achieve HA. By default, it only configures a non-HA endpoint, which points to the
 `access_ip` or IP address of the first server node in the `kube_control_plane` group.
 It can also configure clients to use endpoints for a given loadbalancer type.
 The following diagram shows how traffic to the apiserver is directed.

docs/kube-vip.md

@@ -0,0 +1,52 @@
+# kube-vip
+
+kube-vip provides Kubernetes clusters with a virtual IP and load balancer for both the control plane (for building a highly-available cluster) and Kubernetes Services of type LoadBalancer without relying on any external hardware or software.
+
+## Install
+
+You have to explicitly enable the kube-vip extension:
+
+```yaml
+kube_vip_enabled: true
+```
+
+You also need to enable
+[kube-vip as HA, Load Balancer, or both](https://kube-vip.chipzoller.dev/docs/installation/static/#kube-vip-as-ha-load-balancer-or-both):
+
+```yaml
+# HA for control-plane, requires a VIP
+kube_vip_controlplane_enabled: true
+kube_vip_address: 10.42.42.42
+loadbalancer_apiserver:
+  address: "{{ kube_vip_address }}"
+  port: 6443
+# kube_vip_interface: ens160
+
+# LoadBalancer for services
+kube_vip_services_enabled: false
+# kube_vip_services_interface: ens320
+```
+
+> Note: When using `kube-vip` as LoadBalancer for services,
+[additionnal manual steps](https://kube-vip.chipzoller.dev/docs/usage/cloud-provider/)
+are needed.
+
+If using [ARP mode](https://kube-vip.chipzoller.dev/docs/installation/static/#arp) :
+
+```yaml
+kube_vip_arp_enabled: true
+```
+
+If using [BGP mode](https://kube-vip.chipzoller.dev/docs/installation/static/#bgp) :
+
+```yaml
+kube_vip_bgp_enabled: true
+kube_vip_local_as: 65000
+kube_vip_bgp_routerid: 192.168.0.2
+kube_vip_bgppeers:
+- 192.168.0.10:65000::false
+- 192.168.0.11:65000::false
+# kube_vip_bgp_peeraddress:
+# kube_vip_bgp_peerpass:
+# kube_vip_bgp_peeras:
+```

docs/vars.md

@@ -88,11 +88,17 @@ following default cluster parameters:
 * *cloud_provider* - Enable extra Kubelet option if operating inside GCE or
   OpenStack (default is unset)
 * *kube_feature_gates* - A list of key=value pairs that describe feature gates for
-  alpha/experimental Kubernetes features. (defaults is `[]`)
+  alpha/experimental Kubernetes features. (defaults is `[]`).
+  Additionally, you can use also the following variables to individually customize your kubernetes components installation (they works exactly like `kube_feature_gates`):
+  * *kube_apiserver_feature_gates*
+  * *kube_controller_feature_gates*
+  * *kube_scheduler_feature_gates*
+  * *kube_proxy_feature_gates*
+  * *kubelet_feature_gates*
 * *kubeadm_feature_gates* - A list of key=value pairs that describe feature gates for
   alpha/experimental Kubeadm features. (defaults is `[]`)
 * *authorization_modes* - A list of [authorization mode](
-https://kubernetes.io/docs/admin/authorization/#using-flags-for-your-authorization-module)
+  https://kubernetes.io/docs/admin/authorization/#using-flags-for-your-authorization-module)
   that the cluster should be configured for. Defaults to `['Node', 'RBAC']`
   (Node and RBAC authorizers).
   Note: `Node` and `RBAC` are enabled by default. Previously deployed clusters can be

inventory/sample/group_vars/all/hcloud.yml

@@ -2,7 +2,7 @@
 # external_hcloud_cloud:
 #   hcloud_api_token: ""
 #   token_secret_name: hcloud
-#
+#   with_networks: false # Use the hcloud controller-manager with networks support https://github.com/hetznercloud/hcloud-cloud-controller-manager#networks-support
 #   service_account_name: cloud-controller-manager
 #
 #   controller_image_tag: "latest"

inventory/sample/group_vars/all/upcloud.yml

@@ -0,0 +1,17 @@
+## Repo for UpClouds csi-driver: https://github.com/UpCloudLtd/upcloud-csi
+## To use UpClouds CSI plugin to provision volumes set this value to true
+## Remember to set UPCLOUD_USERNAME and UPCLOUD_PASSWORD
+# upcloud_csi_enabled: true
+# upcloud_csi_controller_replicas: 1
+## Override used image tags
+# upcloud_csi_provisioner_image_tag: "v3.1.0"
+# upcloud_csi_attacher_image_tag: "v3.4.0"
+# upcloud_csi_resizer_image_tag: "v1.4.0"
+# upcloud_csi_plugin_image_tag: "alpha"
+# upcloud_csi_node_image_tag: "v2.5.0"
+# upcloud_tolerations: []
+## Storage class options
+# expand_persistent_volumes: true
+# storage_classes:
+#   - name: standard
+#     is_default: true

inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml

@@ -238,6 +238,8 @@ podsecuritypolicy_enabled: false
 
 # Make a copy of kubeconfig on the host that runs Ansible in {{ inventory_dir }}/artifacts
 # kubeconfig_localhost: false
+# Use ansible_host as external api ip when copying over kubeconfig.
+# kubeconfig_localhost_ansible_host: false
 # Download kubectl onto the host that runs Ansible in {{ bin_dir }}
 # kubectl_localhost: false
 

inventory/sample/group_vars/k8s_cluster/k8s-net-calico.yml

@@ -112,3 +112,6 @@
 # Under certain situations liveness and readiness probes may need tunning
 # calico_node_livenessprobe_timeout: 10
 # calico_node_readinessprobe_timeout: 10
+
+# Calico apiserver (only with kdd)
+# calico_apiserver_enabled: false