Adding ephemeral worker nodes to Monsoon

Until now, Typhoon needed Flatcar Linux to be installed on a machine
in order to start working on it. This impedes the implementation of
ephemeral workers, hence the reason for this change. This commit
allows one to set a variable `enable_install` which defaults to true,
and boots the machines live without the need of installing Flatcar
Linux while using a persistent partition for things such as SSH host
keys or Kubernetes certificates. The disk used for this persistent
partition will either be a disk specified with its path in the
controller or worker's definition or the smallest disk available on a
given machine as we do not need to store a lot of information.
This is done by introducing conditions and logic  in the yaml
templates and sending the controller/worker ignition files instead of
the install ignition file as well as making some changes to what is
done through SSH, transferring it to the ignition file.

This means that, if a worker were to become unhealthy, after
investigating the cause of its failure, it can simply be rebooted with
an ipmi `power reset` command or a `systemctl reboot` and it will be
reimaged and reprovisioned with the ignition file, except for some
of the persistent data such as some Kubernetes certificates or
manifests or SSH host keys.

flatcar-linux/kubernetes/butane/controller.yaml

@@ -12,6 +12,7 @@ systemd:
         After=docker.service
         [Service]
         Environment=ETCD_IMAGE=quay.io/coreos/etcd:v3.5.7
+        ConditionPathExists=/etc/etcd/etcd.env
         ExecStartPre=/usr/bin/docker run -d \
           --name etcd \
           --network host \
@@ -57,6 +58,8 @@ systemd:
         RequiredBy=kubelet.service
         RequiredBy=etcd-member.service
     - name: kubelet.service
+      enabled: true
+        # ConditionPathExists=/etc/kubernetes/kubeconfig
       contents: |
         [Unit]
         Description=Kubelet (System Container)
@@ -65,12 +68,13 @@ systemd:
         Wants=rpc-statd.service
         [Service]
         Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.26.3
+        ConditionPathExists=/etc/kubernetes/kubeconfig
         ExecStartPre=/bin/mkdir -p /etc/cni/net.d
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
         ExecStartPre=/bin/mkdir -p /opt/cni/bin
         ExecStartPre=/bin/mkdir -p /var/lib/calico
         ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins
-        ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
+        ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /${etc}/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /${etc}/kubernetes/ca.crt"
         ExecStartPre=/usr/bin/docker run -d \
           --name kubelet \
           --privileged \
@@ -108,7 +112,7 @@ systemd:
         [Unit]
         Description=Kubernetes control plane
         Wants=docker.service
-        After=docker.service
+        After=docker.service kubelet.service
         ConditionPathExists=!/opt/bootstrap/bootstrap.done
         [Service]
         Type=oneshot
@@ -117,27 +121,70 @@ systemd:
         Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.26.3
         ExecStart=/usr/bin/docker run \
             -v /etc/kubernetes/pki:/etc/kubernetes/pki:ro \
-            -v /opt/bootstrap/assets:/assets:ro \
+            -v /${opt}/bootstrap/assets:/assets:ro \
             -v /opt/bootstrap/apply:/apply:ro \
             --entrypoint=/apply \
             $${KUBELET_IMAGE}
         ExecStartPost=/bin/touch /opt/bootstrap/bootstrap.done
         [Install]
         WantedBy=multi-user.target
+    %{ if ! enable_install }
+    - name: update-engine.service
+      mask: true
+    - name: mnt-persist.mount
+      enabled: true
+      contents: |
+        [Unit]
+        Description= Mounting persistent data
+        After=persistent_preparation.service
+        DefaultDependencies=no
+        [Mount]
+        What=/dev/disk/by-label/persistent
+        Where=/mnt/persist
+        Type=ext4
+        Options=defaults
+        [Install]
+        WantedBy=multi-user.target
+    - name: persistent_preparation.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description= A service used to prepare the persistent disk by creating the required partition if it doesn't exist
+        [Service]
+        Type= oneshot
+        ExecStart=/opt/create_persistent_partition
+        RemainAfterExit= false
+        [Install]
+        WantedBy=multi-user.target
+    - name: initialize_data.service
+      enabled: true
+      contents: |
+        [Unit]
+        Description= A service used to verify the existence of or prepare the SSH host keys on a persistent partition
+        After=persistent_preparation.service mnt-persist.mount
+        Requires=persistent_preparation.service mnt-persist.mount
+        Before=sshd.service docker.service
+        [Service]
+        Type= oneshot
+        ExecStart=/opt/initialize_disk
+        RemainAfterExit= false
+        [Install]
+        WantedBy=multi-user.target
+%{ endif }
 storage:
   directories:
     - path: /var/lib/etcd
       mode: 0700
       overwrite: true
-    - path: /etc/kubernetes
+    - path: /${etc}/kubernetes
       mode: 0755
   files:
     - path: /etc/hostname
       mode: 0644
       contents:
         inline:
           ${domain_name}
-    - path: /etc/kubernetes/kubelet.yaml
+    - path: /opt/bootstrap/kubelet.yaml
       mode: 0644
       contents:
         inline: |
@@ -169,23 +216,40 @@ storage:
       contents:
         inline: |
           #!/bin/bash -e
-          mkdir -p -- auth tls/etcd tls/k8s static-manifests manifests/coredns manifests-networking
-          awk '/#####/ {filename=$2; next} {print > filename}' assets
-          mkdir -p /etc/ssl/etcd/etcd
-          mkdir -p /etc/kubernetes/pki
-          mv tls/etcd/{peer*,server*} /etc/ssl/etcd/etcd/
-          mv tls/etcd/etcd-client* /etc/kubernetes/pki/
-          chown -R etcd:etcd /etc/ssl/etcd
-          chmod -R 500 /etc/ssl/etcd
+          mkdir -p -- /home/core/auth /home/core/tls/etcd /home/core/tls/k8s /home/core/static-manifests /home/core/manifests/coredns /home/core/manifests-networking
+          %{ if ! enable_install }
+          PERST_DRIVE='/mnt/persist/'
+          while [ ! -d "$PERST_DRIVE" ]; do
+              sleep 10
+          done
+          mkdir -p /${etc}/kubernetes
+          chmod 755 /${etc}/kubernetes
+          %{ endif }
+          mkdir -p /${etc}/ssl/etcd/etcd
+          mkdir -p /${etc}/kubernetes/pki
+          mkdir -p /${etc}/kubernetes/manifests
+          mkdir -p /${opt}/bootstrap/assets
+          chmod -R 500 /${etc}/ssl/etcd
           chmod -R 700 /var/lib/etcd
-          mv auth/* /etc/kubernetes/pki/
-          mv tls/k8s/* /etc/kubernetes/pki/
-          mkdir -p /etc/kubernetes/manifests
-          mv static-manifests/* /etc/kubernetes/manifests/
-          mkdir -p /opt/bootstrap/assets
-          mv manifests /opt/bootstrap/assets/manifests
-          mv manifests-networking/* /opt/bootstrap/assets/manifests/
-          rm -rf assets auth static-manifests tls manifests-networking
+          chown -R etcd:etcd /${etc}/ssl/etcd
+          if [ ! -f "/${etc}/kubernetes/kubeconfig" ] ; then
+            while [ ! -f "/home/core/kubeconfig" ]; do
+                sleep 1
+              echo "Waiting for SSH file provisioning to finish"
+            done
+            mv /home/core/kubeconfig /${etc}/kubernetes/kubeconfig ;
+            mv /opt/bootstrap/kubelet.yaml /${etc}/kubernetes/kubelet.yaml
+          fi
+          awk '/#####/ {filename=$2; next} {print > filename}' /home/core/assets
+          mv /home/core/tls/etcd/{peer*,server*} /${etc}/ssl/etcd/etcd/
+          mv /home/core/tls/etcd/etcd-client* /${etc}/kubernetes/pki/
+          mv /home/core/auth/* /${etc}/kubernetes/pki/
+          mv /home/core/tls/k8s/* /${etc}/kubernetes/pki/
+          mv /home/core/static-manifests/* /${etc}/kubernetes/manifests/
+          mv /home/core/manifests /${opt}/bootstrap/assets/manifests
+          mv /home/core/manifests-networking/* /${opt}/bootstrap/assets/manifests
+          rm -rf assets auth kubernetes manifests-networking static-manifests tls
+          rm -f /home/core/kubeconfig
     - path: /opt/bootstrap/apply
       mode: 0544
       contents:
@@ -231,6 +295,127 @@ storage:
             ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd/peer.crt
             ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key
             ETCD_PEER_CLIENT_CERT_AUTH=true
+    %{ if ! enable_install }
+    - path: /opt/create_persistent_partition
+      mode: 0544
+      contents:
+        inline: |
+          #!/bin/sh -eu
+          # we verify if the partition we need to work on already exists
+          # if not, we create it and label it
+          if [ -L /dev/disk/by-label/persistent ] ; then
+            echo "Found existing persistent partition"
+            echo "No need to create a new one"
+          else
+            %{ if live_disk != "" }
+            disk=$(realpath ${live_disk})
+            %{ else }
+            disk=$(for device in /sys/block/*; do
+              if readlink "$device" | grep -q virtual; then
+                continue # Skip virtual devices
+              fi
+              echo -en "/dev$${device#/sys/block*}\t"
+              udevadm info -a -p "$device" | awk -F '"' '/ATTR{size}/ { print $2 }'
+            done | sort -nk 2n | cut -f1 | head -1)
+          %{ endif }
+            echo "Could not find persistent partition, preparing one"
+            echo "Selecting disk : $disk"
+            # arguments given to fdisk : 
+            #  n # new partition
+            #  p # primary partition
+            #  1 # partition number 1
+            #  2048  # default - start at beginning of disk
+            #  +500M # 500 MB parttion
+            #  w # and we're done, saving changes
+            echo "Creating persistent partition"
+            fdisk $disk << EOF ;
+              n
+              p
+              1
+              2048
+              +500M
+              w
+          EOF
+            echo "Making ext4 file system"
+            mkfs.ext4 -c "$${disk}1" ;
+            echo "Labeling partition"
+            e2label "$${disk}1" "persistent";
+          fi
+    - path: /opt/initialize_disk
+      mode: 0544
+      contents:
+        inline: |
+          #!/bin/sh -eu
+          echo "Waiting for persistent disk to get mounted"
+          PERST_DRIVE='/mnt/persist/'
+          while [ ! -d "$PERST_DRIVE" ]; do
+              sleep 10
+            echo "Waiting..."
+          done
+          echo "Persistent disk mounted"
+          if [ ! -d /${etc}/sshd-config/ ] ;
+          then
+            echo "Generating SSH keys"
+            mkdir -p /${etc}/sshd-config ;
+            ssh-keygen -q -t rsa -f /${etc}/sshd-config/ssh_host_rsa_key -C "core@localhost" -N "" ;
+            ssh-keygen -q -t dsa -f /${etc}/sshd-config/ssh_host_dsa_key -C "core@localhost" -N "" ;
+            ssh-keygen -q -t ecdsa -f /${etc}/sshd-config/ssh_host_ecdsa_key -C "core@localhost" -N "" ;
+            ssh-keygen -q -t ed25519 -f /${etc}/sshd-config/ssh_host_ed25519_key -C "core@localhost" -N "" ;
+            chmod 744  -R /${etc}/sshd-config
+            chmod 755 /${etc}/sshd-config/ssh_host_rsa_key.pub
+            chmod 755 /${etc}/sshd-config/ssh_host_dsa_key.pub
+            chmod 755 /${etc}/sshd-config/ssh_host_ecdsa_key.pub
+            chmod 755 /${etc}/sshd-config/ssh_host_ed25519_key.pub
+            chmod 600 /${etc}/sshd-config/ssh_host_rsa_key
+            chmod 600 /${etc}/sshd-config/ssh_host_dsa_key
+            chmod 600 /${etc}/sshd-config/ssh_host_ecdsa_key
+            chmod 600 /${etc}/sshd-config/ssh_host_ed25519_key
+            echo "Finished generating SSH keys"
+          fi
+          # replace the empty kubernetes folder with a symlink to our persistent folder 
+          echo "Creating symbolic links"
+          rm -rf /etc/kubernetes
+          ln -s /${etc}/kubernetes /etc/kubernetes
+          rm -rf /etc/ssl/etcd
+          ln -s /${etc}/ssl/etcd /etc/ssl/etcd
+          rm -rf /opt/bootstrap/assets
+          ln -s /${opt}/bootstrap/assets /opt/bootstrap/assets
+    - path: /etc/ssh/sshd_config.d/host_key_config.conf
+      mode: 0444
+      contents:
+        inline: |
+          HostKey /${etc}/sshd-config/ssh_host_rsa_key
+          HostKey /${etc}/sshd-config/ssh_host_dsa_key
+          HostKey /${etc}/sshd-config/ssh_host_ecdsa_key
+          HostKey /${etc}/sshd-config/ssh_host_ed25519_key 
+    - path: /etc/ssh/sshd_config
+      mode: 0444
+      contents:
+        inline: |
+          # Use most defaults for sshd configuration.
+          UsePrivilegeSeparation sandbox
+          Subsystem sftp internal-sftp
+          ClientAliveInterval 180
+          UseDNS no
+          UsePAM yes
+          # handled by PAM
+          PrintLastLog no
+          # handled by PAM
+          PrintMotd no
+          Ciphers chacha20-poly1305@openssh.com,aes128-ctr,aes192-ctr,aes256-ctr,aes128-gcm@openssh.com,aes256-gcm@openssh.com
+          MACs hmac-sha2-256-etm@openssh.com,hmac-sha2-512-etm@openssh.com,hmac-sha2-256,hmac-sha2-512,umac-128-etm@openssh.com,umac-128@openssh.com
+          KexAlgorithms curve25519-sha256,curve25519-sha256@libssh.org,ecdh-sha2-nistp256,ecdh-sha2-nistp384,ecdh-sha2-nistp521,diffie-hellman-group-exchange-sha256,diffie-hellman-group16-sha512,diffie-hellman-group18-sha512,diffie-hellman-group14-sha256
+
+          # Temporarily accept ssh-rsa algorithm for openssh >= 8.8,
+          # until most ssh clients could deprecate ssh-rsa.
+          HostkeyAlgorithms +ssh-rsa
+          PubkeyAcceptedAlgorithms +ssh-rsa
+          Include /etc/ssh/sshd_config.d/*.conf
+
+          PermitRootLogin no
+          PasswordAuthentication no
+          ChallengeResponseAuthentication no
+%{ endif }
 passwd:
   users:
     - name: core

flatcar-linux/kubernetes/profiles.tf

@@ -36,12 +36,11 @@ resource "matchbox_group" "install" {
 resource "matchbox_profile" "install" {
   count = length(var.controllers)
 
-  name   = format("%s-install-%s", var.cluster_name, var.controllers.*.name[count.index])
+  name         = var.enable_install ? format("%s-install-%s", var.cluster_name, var.controllers.*.name[count.index]) : format("%s-controller-%s", var.cluster_name, var.controllers.*.name[count.index])
+  raw_ignition = var.enable_install ? data.ct_config.install[count.index].rendered : data.ct_config.controllers.*.rendered[count.index]
   kernel = local.kernel
   initrd = local.initrd
   args   = concat(local.args, var.kernel_args)
-
-  raw_ignition = data.ct_config.install[count.index].rendered
 }
 
 # Flatcar Linux install
@@ -90,6 +89,10 @@ data "ct_config" "controllers" {
     cluster_dns_service_ip = module.bootstrap.cluster_dns_service_ip
     cluster_domain_suffix  = var.cluster_domain_suffix
     ssh_authorized_key     = var.ssh_authorized_key
+    enable_install         = var.enable_install
+    live_disk              = var.controllers.*.live_disk[count.index]
+    etc                    = var.enable_install ? "etc" : "mnt/persist/etc"
+    opt                    = var.enable_install ? "opt" : "mnt/persist/opt"
   })
   strict   = true
   snippets = lookup(var.snippets, var.controllers.*.name[count.index], [])

flatcar-linux/kubernetes/ssh.tf

@@ -38,7 +38,6 @@ resource "null_resource" "copy-controller-secrets" {
 
   provisioner "remote-exec" {
     inline = [
-      "sudo mv /home/core/kubeconfig /etc/kubernetes/kubeconfig",
       "sudo /opt/bootstrap/layout",
     ]
   }

flatcar-linux/kubernetes/variables.tf

@@ -29,21 +29,26 @@ variable "os_version" {
 
 variable "controllers" {
   type = list(object({
-    name   = string
-    mac    = string
-    domain = string
+    name      = string
+    mac       = string
+    domain    = string
+    live_disk = optional(string, "")
   }))
   description = <<EOD
-List of controller machine details (unique name, identifying MAC address, FQDN)
+List of controller machine details (unique name, identifying MAC address, FQDN) and the
+disk on which a persistent partition should be installed if we live-boot Typhoon.
+If no disk is selected for a persistent partition, we will search for the 
+smallest disk and create this persistent partition on it.
 [{ name = "node1", mac = "52:54:00:a1:9c:ae", domain = "node1.example.com"}]
 EOD
 }
 
 variable "workers" {
   type = list(object({
-    name   = string
-    mac    = string
-    domain = string
+    name      = string
+    mac       = string
+    domain    = string
+    live_disk = optional(string, "")
   }))
   description = <<EOD
 List of worker machine details (unique name, identifying MAC address, FQDN)
@@ -156,6 +161,12 @@ variable "enable_aggregation" {
   default     = true
 }
 
+variable "enable_install" {
+  type        = bool
+  description = "Enable installing Typhoon on disk"
+  default     = true
+}
+
 variable "oem_type" {
   type        = string
   description = <<EOD