Switch from dockershim to containerd (#624)

* Switch from dockershim to containerd.

* aws-cni dockershim changes

* bump pause image

* Update CHANGELOG.md

* wip

* switch to v1 apiVersion for storage class

* switch to v1 apiVersion for storage class

* bump aws-cni

CHANGELOG.md

@@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+
+- Use containerd socket instead of dockershim in the kubelet config.
+- [AWS] Bump to AWS-cni 1.11.2 and mount containerd socket instead of dockershim one to `aws-node` pods.
+- Bump Pod Infra image to `giantswarm/pause-amd64:3.3`.
+
 ## [10.1.0] - 2022-06-23
 
 ### Added

platforms/aws/giantnetes/variables.tf

@@ -328,7 +328,7 @@ variable "image_pull_progress_deadline" {
 }
 
 variable "pod_infra_image" {
-  default = "giantswarm/pause-amd64:3.1"
+  default = "giantswarm/pause-amd64:3.3"
 }
 
 ### External Kubernetes API Access

platforms/azure/giantnetes/main.tf

@@ -164,6 +164,7 @@ locals {
     "MasterCount"              = var.master_count
     "OIDCIssuerURL"            = "https://${var.oidc_issuer_dns}.${var.base_domain}"
     "PodCIDR"                  = var.pod_cidr
+    "PodInfraImage"            = var.pod_infra_image
     "Provider"                 = "azure"
     "Users"                    = yamldecode(base64decode(jsondecode(data.http.bastion_users.body).content))
     "VaultDomainName"          = "${var.vault_dns}.${var.base_domain}"

platforms/azure/giantnetes/variables.tf

@@ -165,6 +165,10 @@ variable "hyperkube_version" {
   default = "1.22.10"
 }
 
+variable "pod_infra_image" {
+  default = "giantswarm/pause-amd64:3.3"
+}
+
 ### DNS ###
 
 variable "base_domain" {

templates/files/conf/10-use-custom-config.conf

@@ -0,0 +1,3 @@
+[Service]
+ExecStart=
+ExecStart=/usr/bin/containerd

templates/files/conf/containerd-config.toml

@@ -0,0 +1,36 @@
+version = 2
+
+# persistent data location
+root = "/var/lib/containerd"
+# runtime state information
+state = "/run/containerd"
+# set containerd as a subreaper on linux when it is not running as PID 1
+subreaper = true
+# set containerd's OOM score
+oom_score = -999
+disabled_plugins = []
+
+# grpc configuration
+[grpc]
+address = "/run/containerd/containerd.sock"
+# socket uid
+uid = 0
+# socket gid
+gid = 0
+
+[plugins."containerd.runtime.v1.linux"]
+# shim binary name/path
+shim = "containerd-shim"
+# runtime binary name/path
+runtime = "runc"
+# do not use a shim when starting containers, saves on memory but
+# live restore is not supported
+no_shim = false
+
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+# setting runc.options unsets parent settings
+runtime_type = "io.containerd.runc.v2"
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+SystemdCgroup = true
+[plugins."io.containerd.grpc.v1.cri"]
+sandbox_image = "{{.DockerRegistry}}/{{ .PodInfraImage }}"

templates/files/k8s-resource/aws-cni.yaml

@@ -153,7 +153,7 @@ spec:
       tolerations:
         - operator: Exists
       initContainers:
-        - image: {{.DockerRegistry}}/giantswarm/aws-cni-init:v1.10.1-nftables
+        - image: {{.DockerRegistry}}/giantswarm/aws-cni-init:v1.11.2-nftables
           imagePullPolicy: Always
           name: aws-vpc-cni-init
           env:
@@ -167,7 +167,7 @@ spec:
             - mountPath: /host/opt/cni/bin
               name: cni-bin-dir
       containers:
-        - image: {{.DockerRegistry}}/giantswarm/aws-cni:v1.10.1-nftables
+        - image: {{.DockerRegistry}}/giantswarm/aws-cni:v1.11.2-nftables
           imagePullPolicy: Always
           ports:
             - containerPort: 61678
@@ -244,8 +244,8 @@ spec:
               name: log-dir
             - mountPath: /var/run/aws-node
               name: run-dir
-            - mountPath: /var/run/dockershim.sock
-              name: dockershim
+            - mountPath: /var/run/cri.sock
+              name: cri
             - mountPath: /run/xtables.lock
               name: xtables-lock
       volumes:
@@ -267,9 +267,9 @@ spec:
             path: /var/run/aws-node
             type: DirectoryOrCreate
           name: run-dir
-        - name: dockershim
+        - name: cri
           hostPath:
-            path: /var/run/dockershim/dockershim.sock
+            path: /var/run/containerd/containerd.sock
 
 ---
 apiVersion: apiextensions.k8s.io/v1

templates/files/k8s-resource/default-storage-class.yaml

@@ -1,5 +1,5 @@
 {{if eq .Provider "aws" -}}
-apiVersion: storage.k8s.io/v1beta1
+apiVersion: storage.k8s.io/v1
 kind: StorageClass
 metadata:
   name: gp2

templates/master.yaml.tmpl

@@ -555,6 +555,27 @@ storage:
         id: 0
       contents:
         source: "data:text/plain;charset=utf-8;base64,{{ index .Files "conf/ipvs.conf" }}"
+
+    - path : /etc/containerd/config.toml
+      filesystem: root
+      mode: 420
+      user:
+        id: 0
+      group:
+        id: 0
+      contents:
+        source: "data:text/plain;charset=utf-8;base64,{{ index .Files "conf/containerd-config.toml" }}"
+
+    - path : /etc/systemd/system/containerd.service.d/10-use-custom-config.conf
+      filesystem: root
+      mode: 420
+      user:
+        id: 0
+      group:
+        id: 0
+      contents:
+        source: "data:text/plain;charset=utf-8;base64,{{ index .Files "conf/10-use-custom-config.conf" }}"
+
 {{ if eq .Provider "aws" }}
     - path: /etc/systemd/system/etcd3.d/10-require-attach-dep.conf
       filesystem: root
@@ -1186,7 +1207,8 @@ systemd:
       ExecStart=/opt/bin/kubelet \
       --config=/etc/kubernetes/config/kubelet.yaml \
       --node-ip=${DEFAULT_IPV4} \
-      --container-runtime-endpoint=/var/run/dockershim/dockershim.sock \
+      --container-runtime=remote \
+      --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
       --logtostderr=true \
       --cloud-provider=external \
       {{if eq .Provider "aws" -}}

templates/worker.yaml.tmpl

@@ -230,6 +230,26 @@ storage:
       contents:
         source: "data:text/plain;charset=utf-8;base64,{{ index .Files "conf/ipvs.conf" }}"
 
+    - path : /etc/containerd/config.toml
+      filesystem: root
+      mode: 420
+      user:
+        id: 0
+      group:
+        id: 0
+      contents:
+        source: "data:text/plain;charset=utf-8;base64,{{ index .Files "conf/containerd-config.toml" }}"
+
+    - path : /etc/systemd/system/containerd.service.d/10-use-custom-config.conf
+      filesystem: root
+      mode: 420
+      user:
+        id: 0
+      group:
+        id: 0
+      contents:
+        source: "data:text/plain;charset=utf-8;base64,{{ index .Files "conf/10-use-custom-config.conf" }}"
+
 {{ if .LogentriesEnabled }}
     - path: /opt/bin/logentries.sh
       filesystem: root
@@ -668,7 +688,8 @@ systemd:
       ExecStart=/opt/bin/kubelet \
       --config=/etc/kubernetes/config/kubelet.yaml \
       --node-ip=${DEFAULT_IPV4} \
-      --container-runtime-endpoint=/var/run/dockershim/dockershim.sock \
+      --container-runtime=remote \
+      --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
       --logtostderr=true \
       {{if eq .Provider "aws" -}}
       --cloud-provider=external \