Switch from the embedded k3s containerd process to one we manage but …

…still use the k3s supplied binary to allow us to change the containerd root. Add missing findutils package for longhorn. Raise io priority for embedded etcd. Add Pramodhs containerd.go change to skip starting containerd-user in pillar.

Signed-off-by: Andrew Durbin <136004284+andrewd-zededa@users.noreply.github.com>

pkg/kube/Dockerfile

@@ -2,7 +2,7 @@
 
 FROM lfedge/eve-alpine:12487b9900ba40f3ecdadfec2c84799fa34e5014 as build
 ENV PKGS alpine-baselayout musl-utils iproute2 iptables curl openrc \
-         open-iscsi libvirt libvirt-client util-linux grep
+         open-iscsi libvirt libvirt-client util-linux grep findutils
 RUN eve-alpine-deploy.sh
 
 FROM scratch
@@ -11,6 +11,8 @@ COPY cluster-init.sh /usr/bin/
 COPY cgconfig.conf /etc
 # kubevirt yaml files are patched files and will be removed later, look at cluster-init.sh
 COPY kubevirt-operator.yaml /etc
+RUN mkdir -p /etc/containerd
+COPY config-k3s.toml /etc/containerd/
 RUN mkdir -p /etc/rancher/k3s
 COPY config.yaml /etc/rancher/k3s
 WORKDIR /

pkg/kube/cluster-init.sh

@@ -68,6 +68,43 @@ setup_prereqs () {
         check_network_connection
 }
 
+check_start_containerd() {
+        # Needed to get the pods to start
+        if [ ! -L /usr/bin/runc ]; then
+                ln -s /var/lib/rancher/k3s/data/current/bin/runc /usr/bin/runc
+        fi
+        if [ ! -L /usr/bin/containerd-shim-runc-v2 ]; then
+                ln -s /var/lib/rancher/k3s/data/current/bin/containerd-shim-runc-v2 /usr/bin/containerd-shim-runc-v2
+        fi
+
+        if pgrep -f "containerd --config" >> $INSTALL_LOG 2>&1; then
+                logmsg "k3s-containerd is alive"
+        else
+                logmsg "Starting k3s-containerd"
+                mkdir -p /run/containerd-user
+                nohup /var/lib/rancher/k3s/data/current/bin/containerd --config /etc/containerd/config-k3s.toml &
+        fi
+}
+trigger_k3s_selfextraction() {
+        # Analysis of the k3s source shows nearly any cli command will first self-extract a series of binaries.
+        # In our case we're looking for the containerd binary.
+        # k3s check-config appears to be the only cli cmd which doesn't:
+        # - start a long running process/server
+        # - timeout connecting to a socket
+        # - manipulate config/certs
+
+        # When run on the shell this does throw some config errors, its unclear if we need this issues fixed:
+        # - links: aux/ip6tables should link to iptables-detect.sh (fail)
+        #- links: aux/ip6tables-restore should link to iptables-detect.sh (fail)
+        #- links: aux/ip6tables-save should link to iptables-detect.sh (fail)
+        #- links: aux/iptables should link to iptables-detect.sh (fail)
+        #- links: aux/iptables-restore should link to iptables-detect.sh (fail)
+        #- links: aux/iptables-save should link to iptables-detect.sh (fail)
+        #- apparmor: enabled, but apparmor_parser missing (fail)
+        #      - CONFIG_INET_XFRM_MODE_TRANSPORT: missing
+        /usr/bin/k3s check-config >> $INSTALL_LOG 2>&1
+}
+
 # NOTE: We only support zfs storage in production systems because data is persisted on zvol.
 # If ZFS is not available we still go ahead and provide the service but the data is lost on reboot
 # because /var/lib will be on overlayfs. The only reason to allow that is to provide a quick debugging env for developers.
@@ -82,7 +119,7 @@ fi
 setup_prereqs
 
 date >> $INSTALL_LOG
-HOSTNAME=$(/bin/hostname)
+
 #Forever loop every 15 secs
 while true;
 do
@@ -94,10 +131,14 @@ if [ ! -f /var/lib/all_components_initialized ]; then
                 /usr/bin/curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${K3S_VERSION} INSTALL_K3S_SKIP_ENABLE=true INSTALL_K3S_BIN_DIR=/var/lib/k3s/bin sh -
                 ln -s /var/lib/k3s/bin/* /usr/bin
                 logmsg "Initializing K3S version $K3S_VERSION"
+                trigger_k3s_selfextraction
+                check_start_containerd
                 nohup /usr/bin/k3s server --config /etc/rancher/k3s/config.yaml &
                 #wait until k3s is ready
                 logmsg "Looping until k3s is ready"
-                until kubectl get node | grep "$HOSTNAME" | awk '{print $2}' | grep 'Ready'; do sleep 5; done
+                until kubectl get node | grep "$(/bin/hostname)" | awk '{print $2}' | grep 'Ready'; do sleep 5; done
+                # Give the embedded etcd in k3s priority over io as its fsync latencies are critical
+                ionice -c2 -n0 -p "$(pgrep -f "k3s server")"
                 logmsg "k3s is ready on this node"
                 # Default location where clients will look for config
                 ln -s /etc/rancher/k3s/k3s.yaml ~/.kube/config
@@ -129,6 +170,7 @@ if [ ! -f /var/lib/all_components_initialized ]; then
                 touch /var/lib/all_components_initialized
         fi
 else
+        check_start_containerd
         if pgrep k3s >> $INSTALL_LOG 2>&1; then
                 logmsg "k3s is alive "
         else
@@ -137,7 +179,9 @@ else
                 logmsg "Starting k3s server after reboot"
                 nohup /usr/bin/k3s server --config /etc/rancher/k3s/config.yaml &
                 logmsg "Looping until k3s is ready"
-                until kubectl get node | grep "$HOSTNAME" | awk '{print $2}' | grep 'Ready'; do sleep 5; done
+                until kubectl get node | grep "$(/bin/hostname)" | awk '{print $2}' | grep 'Ready'; do sleep 5; done
+                # Give the embedded etcd in k3s priority over io as its fsync latencies are critical
+                ionice -c2 -n0 -p "$(pgrep -f "k3s server")"
                 logmsg "k3s is ready on this node"
                 # Default location where clients will look for config
                 ln -s /etc/rancher/k3s/k3s.yaml ~/.kube/config

pkg/kube/config.yaml

@@ -4,3 +4,6 @@
 write-kubeconfig-mode: "0644"
 cluster-init: true
 log: "/var/lib/rancher/k3s/k3s.log"
+debug: true
+etcd-expose-metrics: true
+container-runtime-endpoint: "/run/containerd-user/containerd.sock"

pkg/pillar/containerd/containerd.go

@@ -28,6 +28,7 @@ import (
 	"github.com/containerd/containerd/snapshots"
 	"github.com/containerd/typeurl"
 	"github.com/lf-edge/edge-containers/pkg/resolver"
+	"github.com/lf-edge/eve/pkg/pillar/base"
 	"github.com/lf-edge/eve/pkg/pillar/types"
 	"github.com/lf-edge/eve/pkg/pillar/vault"
 	"github.com/opencontainers/go-digest"
@@ -49,6 +50,8 @@ const (
 	ctrdSystemServicesNamespace = "services.linuxkit"
 	// ctrdServicesNamespace containerd namespace for running user containers
 	ctrdServicesNamespace = "eve-user-apps"
+	// ctrdKubeServicesNamespace containerd namespace for running user containers in kube-containerd
+	ctrdKubeServicesNamespace = "k8s.io"
 	//containerdRunTime - default runtime of containerd
 	containerdRunTime = "io.containerd.runc.v2"
 	// container config file name
@@ -84,6 +87,9 @@ type Client struct {
 // GetServicesNamespace returns ctrdServicesNamespace
 // The value is used to define the cgroups path of the EVE services
 func GetServicesNamespace() string {
+	if base.IsHVTypeKube() {
+		return ctrdKubeServicesNamespace
+	}
 	return ctrdServicesNamespace
 }
 
@@ -93,6 +99,10 @@ func init() {
 	if vault.ReadPersistType() == types.PersistZFS {
 		defaultSnapshotter = types.ZFSSnapshotter
 	}
+
+	if base.IsHVTypeKube() {
+		defaultSnapshotter = "overlayfs"
+	}
 }
 
 // NewContainerdClient returns a *Client
@@ -635,6 +645,9 @@ func (client *Client) Resolver(ctx context.Context) (resolver.ResolverCloser, er
 // CtrNewUserServicesCtx returns a new user service containerd context
 // and a done func to cancel the context after use.
 func (client *Client) CtrNewUserServicesCtx() (context.Context, context.CancelFunc) {
+	if base.IsHVTypeKube() {
+		return newServiceCtx(ctrdKubeServicesNamespace)
+	}
 	return newServiceCtx(ctrdServicesNamespace)
 }
 
@@ -647,6 +660,9 @@ func (client *Client) CtrNewSystemServicesCtx() (context.Context, context.Cancel
 // CtrNewUserServicesCtxWithLease returns a new user service containerd context with a 24 hrs lease
 // and a done func to delete the lease and cancel the context after use.
 func (client *Client) CtrNewUserServicesCtxWithLease() (context.Context, context.CancelFunc, error) {
+	if base.IsHVTypeKube() {
+		return newServiceCtxWithLease(client.ctrdClient, ctrdKubeServicesNamespace)
+	}
 	return newServiceCtxWithLease(client.ctrdClient, ctrdServicesNamespace)
 }
 
@@ -885,6 +901,10 @@ func (client *Client) UnpackClientImage(clientImage containerd.Image) error {
 
 // StartUserContainerdInstance execute user containerd instance in goroutine
 func StartUserContainerdInstance() error {
+	// In kubevirt env we do not start eve user containerd, we just use that comes with k3s.
+	if base.IsHVTypeKube() {
+		return nil
+	}
 	name := "/usr/bin/containerd"
 	args := []string{"--config", "/etc/containerd/user.toml"}
 	cmd := exec.Command(name, args...)