kubevirt - encrypt etcd-storage with vault.

docs/BOOTING.md

@@ -132,7 +132,7 @@ in grub.cfg with graphical GRUB menu to get the device to boot again.
     11. `eve_install_skip_rootfs` - do not install rootfs partition onto device. May be selected from graphical GRUB menu.
     12. `eve_install_skip_zfs_checks` - install zfs by skipping minimum requirement checks.
     13. `eve_install_zfs_with_raid_level` - Sets raid level for zfs storage. Valid values are none,raid1,raid5,raid6. Default value is none. This option also applied for the first boot of a live image to prepare zfs persist pool instead of ext4.
-    14. `eve_install_kubevirt_reserve_for_eve_sizeGB` - Amount of space in GB to reserve for eve services in kubevirt based images (This is highly experimental and not supported config, also its an optional parameter and defaults to 20GB if not set)
+    14. `eve_install_kubevirt_etcd_sizeGB` - Size in GB of the etcd-storage zvol.  Defaults to 10GB.
 3. General kernel parameters may be adjusted with `set_global dom0_extra_args "$dom0_extra_args OPTION1=VAL1 OPTION2 "`.
    They will be added to kernel cmdline.
 

pkg/mkimage-raw-efi/install

@@ -22,7 +22,7 @@
 #   eve_install_skip_rootfs
 #   eve_install_skip_zfs_checks
 #   eve_install_zfs_with_raid_level
-#   eve_install_kubevirt_reserve_for_eve_sizeGB
+#   eve_install_kubevirt_etcd_sizeGB
 #
 BAIL_FINAL_CMD=${BAIL_FINAL_CMD:-"exit 1"}
 [ -n "$DEBUG" ] && set -x
@@ -140,7 +140,7 @@ collect_black_box() {
    pillar_run tpmmgr saveTpmInfo "$1"/tpminfo.txt
 }
 
-# prepare_mounts_and_zfs_pool POOL_CREATION_COMMAND_SUFFIX_SUFFIX INSTALL_CLUSTERED_STORAGE RESERVE_EVE_STORAGE_SIZEGB
+# prepare_mounts_and_zfs_pool POOL_CREATION_COMMAND_SUFFIX_SUFFIX INSTALL_CLUSTERED_STORAGE
 prepare_mounts_and_zfs_pool() {
   logmsg "Preparing ZFS pool and mounts"
   [ -e /root/sys ] || mkdir /root/sys && mount -t sysfs sysfs /root/sys
@@ -298,15 +298,7 @@ eve_flavor=$(cat /root/etc/eve-hv-type)
 if [ "$eve_flavor" = "kubevirt" ]; then
    INSTALL_CLUSTERED_STORAGE=true
    INSTALL_ZFS=true
-   RESERVE_EVE_STORAGE_SIZEGB=$(</proc/cmdline tr ' ' '\012' | sed -ne '/^eve_install_kubevirt_reserve_for_eve_sizeGB=/s#^.*=##p')
-
-   #If not set, default it to 20GB
-   # NOTE reserving 20GB for eve services is an experimental change and could be modified in future after analysing data.
-   if [ -z "$RESERVE_EVE_STORAGE_SIZEGB" ]; then
-      RESERVE_EVE_STORAGE_SIZEGB=20
-   fi
-
-   logmsg "Kubevirt image installing ZFS and EVE reserved storage sizeGB $RESERVE_EVE_STORAGE_SIZEGB"
+   logmsg "Kubevirt image installing ZFS"
 fi
 
 

pkg/pillar/base/kubevirt.go

@@ -23,6 +23,12 @@ const (
 	KubeAppNameUUIDSuffixLen = 5
 	// VMIPodNamePrefix : prefix added to name of every pod created to run VM.
 	VMIPodNamePrefix = "virt-launcher-"
+	// InstallOptionEtcdSizeGB grub option at install time.  Size of etcd volume in GB.
+	InstallOptionEtcdSizeGB = "eve_install_kubevirt_etcd_sizeGB"
+	// DefaultEtcdSizeGB default for InstallOptionEtcdSizeGB
+	DefaultEtcdSizeGB uint32 = 10
+	// EtcdVolBlockSizeBytes is the block size for the etcd volume
+	EtcdVolBlockSizeBytes = uint64(4 * 1024)
 )
 
 // IsHVTypeKube - return true if the EVE image is kube cluster type.

pkg/pillar/vault/handler_zfs.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"strconv"
 	"strings"
 	"time"
 
@@ -94,14 +95,14 @@ func (h *ZFSHandler) SetupDefaultVault() error {
 			if zfs.DatasetExist(h.log, types.SealedDataset) {
 				return MountVaultZvol(h.log, types.SealedDataset)
 			}
+			if err := CreateZvolEtcd(h.log, types.EtcdZvol, "", false); err != nil {
+				return fmt.Errorf("error creating zfs etcd zvol %s, error=%v",
+					types.EtcdZvol, err)
+			}
 			if err := CreateZvolVault(h.log, types.SealedDataset, "", false); err != nil {
 				return fmt.Errorf("error creating zfs non-tpm vault %s, error=%v",
 					types.SealedDataset, err)
 			}
-			if err := CreateZvolEtcd(h.log, types.EtcdZvol); err != nil {
-				return fmt.Errorf("error creating zfs etcd zvol %s, error=%v",
-					types.EtcdZvol, err)
-			}
 			return nil
 		}
 		if zfs.DatasetExist(h.log, types.SealedDataset) {
@@ -143,6 +144,13 @@ func (h *ZFSHandler) unlockVault(vaultPath string) error {
 
 	// zfs mount
 	if base.IsHVTypeKube() {
+		// zfs load-key here separately for types.EtcdZvol because we don't mount it here, only in kube.
+		args := []string{"load-key", types.EtcdZvol}
+		if stdOut, stdErr, err := execCmd(types.ZFSBinary, args...); err != nil {
+			h.log.Errorf("Error loading key for etcd vol vault: %v, %s, %s",
+				err, stdOut, stdErr)
+			return err
+		}
 		if err := MountVaultZvol(h.log, vaultPath); err != nil {
 			h.log.Errorf("Error unlocking vault: %v", err)
 			return err
@@ -169,13 +177,13 @@ func (h *ZFSHandler) createVault(vaultPath string) error {
 	defer unstage()
 
 	if base.IsHVTypeKube() {
+		if err := CreateZvolEtcd(h.log, types.EtcdZvol, zfsKeyFile, true); err != nil {
+			return fmt.Errorf("error creating zfs etcd zvol %s, error=%v", types.EtcdZvol, err)
+		}
 		if err := CreateZvolVault(h.log, vaultPath, zfsKeyFile, true); err != nil {
 			h.log.Errorf("Error creating zfs vault %s, error=%v", vaultPath, err)
 			return err
 		}
-		if err := CreateZvolEtcd(h.log, types.EtcdZvol); err != nil {
-			return fmt.Errorf("error creating zfs etcd zvol %s, error=%v", types.EtcdZvol, err)
-		}
 	} else {
 		if err := zfs.CreateVaultDataset(vaultPath, zfsKeyFile); err != nil {
 			h.log.Errorf("Error creating zfs vault %s, error=%v", vaultPath, err)
@@ -374,8 +382,10 @@ func CreateZvolVault(log *base.LogObject, datasetName string, zfsKeyFile string,
 	if err != nil {
 		return fmt.Errorf("Dataset %s available bytes read error: %v", parentDatasetName, err)
 	}
+	// Shift back to allow for alignUp space.
+	sizeBytes = sizeBytes - zfs.VolBlockSizeBytes
 
-	err = zfs.CreateVaultVolumeDataset(log, datasetName, zfsKeyFile, encrypted, sizeBytes)
+	err = zfs.CreateVaultVolumeDataset(log, datasetName, zfsKeyFile, encrypted, sizeBytes, "zstd", zfs.VolBlockSizeBytes)
 	if err != nil {
 		return fmt.Errorf("Vault zvol creation error; %v", err)
 	}
@@ -398,40 +408,49 @@ func CreateZvolVault(log *base.LogObject, datasetName string, zfsKeyFile string,
 }
 
 // CreateZvolEtcd Create and mount an empty vault dataset zvol
-func CreateZvolEtcd(log *base.LogObject, datasetName string) error {
-	// Remaining space in the pool
-	sizeBytes := uint64(0)
-	etcdSizeBytes := uint64(1024 * 1024 * 1024 * 1)
-
-	parentDatasetName := datasetName
-	if strings.Contains(parentDatasetName, "/") {
-		datasetParts := strings.Split(parentDatasetName, "/")
-		parentDatasetName = datasetParts[0]
-	}
-
-	sizeBytes, err := zfs.GetDatasetAvailableBytes(parentDatasetName)
+func CreateZvolEtcd(log *base.LogObject, datasetName string, zfsKeyFile string, encrypted bool) error {
+	etcdSizeGb, err := getEtcdSizeSetting()
 	if err != nil {
-		return fmt.Errorf("Dataset %s available bytes read error: %v", parentDatasetName, err)
-	}
-
-	if sizeBytes > (1024 * 1024 * 1024 * 10) {
-		etcdSizeBytes = 1024 * 1024 * 1024 * 10
+		log.Errorf("Using default %d GB, can't read etcd size setting: %v", etcdSizeGb, err)
 	}
+	etcdSizeBytes := uint64(1024 * 1024 * 1024 * uint64(etcdSizeGb))
 
-	err = zfs.CreateVolumeDataset(log, datasetName, etcdSizeBytes, "off")
+	err = zfs.CreateVaultVolumeDataset(log, datasetName, zfsKeyFile, encrypted, etcdSizeBytes, "off", base.EtcdVolBlockSizeBytes)
 	if err != nil {
-		return fmt.Errorf("Vault zvol creation error; %v", err)
+		return fmt.Errorf("Vault Etcd zvol creation error: %v", err)
 	}
 
 	devPath := zfs.GetZvolPath(datasetName)
 	// Sometimes we wait for /dev path to the zvol to appear
 	// Since this only occurs on first boot, we can afford to be patient
 	if err = waitPath(log, devPath, vaultZvolPathWaitSeconds); err != nil {
-		return fmt.Errorf("Vault zvol dev path missing: %v", err)
+		return fmt.Errorf("Vault Etcd zvol dev path missing: %v", err)
 	}
 
 	if err = formatZvol(log, devPath, vaultFsType); err != nil {
-		return fmt.Errorf("Vault zvol format error: %v", err)
+		return fmt.Errorf("Vault Etcd zvol format error: %v", err)
 	}
 	return nil
 }
+
+func getEtcdSizeSetting() (uint32, error) {
+	size := base.DefaultEtcdSizeGB
+	data, err := os.ReadFile("/proc/cmdline")
+	if err != nil {
+		return size, err
+	}
+	bootArgs := strings.Fields(string(data))
+	for _, arg := range bootArgs {
+		if strings.HasPrefix(arg, base.InstallOptionEtcdSizeGB) {
+			argSplitted := strings.Split(arg, "=")
+			if len(argSplitted) == 2 {
+				valGB, err := strconv.ParseUint(argSplitted[1], 10, 32)
+				if err == nil {
+					return uint32(valGB), nil
+				}
+				return size, err
+			}
+		}
+	}
+	return size, nil
+}

pkg/pillar/volumehandlers/zvolhandler.go

@@ -66,7 +66,7 @@ func (handler *volumeHandlerZVol) PrepareVolume() error {
 		}
 	}
 	zVolName := handler.status.ZVolName()
-	if err := zfs.CreateVolumeDataset(handler.log, zVolName, size, "zstd"); err != nil {
+	if err := zfs.CreateVolumeDataset(handler.log, zVolName, size, "zstd", zfs.VolBlockSizeBytes); err != nil {
 		errStr := fmt.Sprintf("Error creating zfs zvol at %s, error=%v",
 			zVolName, err)
 		handler.log.Error(errStr)

pkg/pillar/zfs/zfs.go

@@ -21,10 +21,8 @@ import (
 	"google.golang.org/protobuf/proto"
 )
 
-const volBlockSize = uint64(16 * 1024)
-
-// taken from mkimage-raw-efi/install kubevirt RESERVE_EVE_STORAGE_SIZEGB
-const reserveEveStorageSizeGb = uint64(20 * 1024 * 1024 * 1024)
+// VolBlockSizeBytes is the default dataset block size in bytes
+const VolBlockSizeBytes = uint64(16 * 1024)
 
 // CreateDatasets - creates all the non-existing parent datasets.
 // Datasets created in this manner are automatically mounted
@@ -151,10 +149,8 @@ func GetZvolPath(datasetName string) string {
 }
 
 // CreateVaultVolumeDataset Create an empty vault zvol
-func CreateVaultVolumeDataset(log *base.LogObject, datasetName string, zfsKeyFile string, encrypted bool, sizeBytes uint64) error {
-	// Shave off reserved + Can't align up if we're already at max space.
-	sizeBytes = sizeBytes - reserveEveStorageSizeGb - (volBlockSize * 1024)
-	alignedSize := alignUpToBlockSize(sizeBytes)
+func CreateVaultVolumeDataset(log *base.LogObject, datasetName string, zfsKeyFile string, encrypted bool, sizeBytes uint64, compressionType string, blockSizeBytes uint64) error {
+	alignedSizeBytes := alignUpToBlockSize(sizeBytes, blockSizeBytes)
 	props := make(map[libzfs.Prop]libzfs.Property)
 
 	if encrypted {
@@ -166,13 +162,13 @@ func CreateVaultVolumeDataset(log *base.LogObject, datasetName string, zfsKeyFil
 			Value: "raw"}
 	}
 	props[libzfs.DatasetPropVolsize] = libzfs.Property{
-		Value: strconv.FormatUint(alignedSize, 10)}
+		Value: strconv.FormatUint(alignedSizeBytes, 10)}
 	props[libzfs.DatasetPropVolblocksize] = libzfs.Property{
-		Value: strconv.FormatUint(volBlockSize, 10)}
+		Value: strconv.FormatUint(blockSizeBytes, 10)}
 	props[libzfs.DatasetPropVolmode] = libzfs.Property{
 		Value: "dev"}
 	props[libzfs.DatasetPropCompression] = libzfs.Property{
-		Value: "zstd"}
+		Value: compressionType}
 
 	dataset, err := libzfs.DatasetCreate(datasetName, libzfs.DatasetTypeVolume, props)
 	if err != nil {
@@ -292,8 +288,8 @@ func SetReserved(datasetName string, percentage uint64) error {
 }
 
 // CreateVolumeDataset creates dataset of zvol type in zfs
-func CreateVolumeDataset(log *base.LogObject, datasetName string, size uint64, compression string) error {
-	alignedSize := alignUpToBlockSize(size)
+func CreateVolumeDataset(log *base.LogObject, datasetName string, sizeBytes uint64, compression string, blockSizeBytes uint64) error {
+	alignedSizeBytes := alignUpToBlockSize(sizeBytes, blockSizeBytes)
 
 	// Create fs datasets if they don't exist
 	if err := CreateDatasets(log, filepath.Dir(datasetName)); err != nil {
@@ -302,11 +298,11 @@ func CreateVolumeDataset(log *base.LogObject, datasetName string, size uint64, c
 
 	props := make(map[libzfs.Prop]libzfs.Property)
 	props[libzfs.DatasetPropVolsize] = libzfs.Property{
-		Value: strconv.FormatUint(alignedSize, 10)}
+		Value: strconv.FormatUint(alignedSizeBytes, 10)}
 	props[libzfs.DatasetPropVolblocksize] = libzfs.Property{
-		Value: strconv.FormatUint(volBlockSize, 10)}
+		Value: strconv.FormatUint(blockSizeBytes, 10)}
 	props[libzfs.DatasetPropReservation] = libzfs.Property{
-		Value: strconv.FormatUint(alignedSize, 10)}
+		Value: strconv.FormatUint(alignedSizeBytes, 10)}
 	props[libzfs.DatasetPropVolmode] = libzfs.Property{
 		Value: "dev"}
 	props[libzfs.DatasetPropLogbias] = libzfs.Property{
@@ -458,8 +454,8 @@ func GetZFSVolumeInfo(device string) (*types.ImgInfo, error) {
 	return &imgInfo, nil
 }
 
-func alignUpToBlockSize(size uint64) uint64 {
-	return (size + volBlockSize - 1) & ^(volBlockSize - 1)
+func alignUpToBlockSize(sizeBytes uint64, blockSizeBytes uint64) uint64 {
+	return (sizeBytes + blockSizeBytes - 1) & ^(blockSizeBytes - 1)
 }
 
 // RemoveVDev removes vdev from the pool