kola: add IBM CEX device test for the s390x build

This kola test is crucial for verifying the security of CEX
hardware-based LUKS encryption on root volume. It guarantees that the
encrypted device employs protected keys to encrypt and decrypt the
volume.

This is essentially testing the enablement done in
coreos/ignition#1820.

To run this, it needs to be on a system with a CEX device with
passthrough enabled and the device's UUID exposed via KOLA_CEX_UUID. See
also coreos/fedora-coreos-pipeline#1010.

Co-authored-by: Jonathan Lebon <jonathan@jlebon.com>

mantle/cmd/kola/options.go

@@ -168,6 +168,8 @@ func init() {
 	bv(&kola.QEMUOptions.SecureExecution, "qemu-secex", false, "Run IBM Secure Execution Image")
 	sv(&kola.QEMUOptions.SecureExecutionIgnitionPubKey, "qemu-secex-ignition-pubkey", "", "Path to Ignition GPG Public Key")
 	sv(&kola.QEMUOptions.SecureExecutionHostKey, "qemu-secex-hostkey", "", "Path to Secure Execution HKD certificate")
+	// s390x CEX-specific options
+	bv(&kola.QEMUOptions.Cex, "qemu-cex", false, "Attach CEX device to guest")
 }
 
 // Sync up the command line options if there is dependency

mantle/cmd/kola/qemuexec.go

@@ -391,6 +391,14 @@ func runQemuExec(cmd *cobra.Command, args []string) error {
 		}
 	}
 
+	// IBM Cex based luks encryption.
+	if kola.QEMUOptions.Cex {
+		err := builder.AddCexDevice()
+		if err != nil {
+			return err
+		}
+	}
+
 	if devshell && !devshellConsole {
 		return runDevShellSSH(ctx, builder, config, sshCommand)
 	}

mantle/kola/tests/ignition/luks.go

@@ -2,13 +2,15 @@ package ignition
 
 import (
 	"fmt"
+	"os"
 	"time"
 
 	coreosarch "github.com/coreos/stream-metadata-go/arch"
 
 	"github.com/coreos/coreos-assembler/mantle/kola"
 	"github.com/coreos/coreos-assembler/mantle/kola/cluster"
 	"github.com/coreos/coreos-assembler/mantle/kola/register"
+	"github.com/coreos/coreos-assembler/mantle/kola/tests/coretest"
 	ut "github.com/coreos/coreos-assembler/mantle/kola/tests/util"
 	"github.com/coreos/coreos-assembler/mantle/platform"
 	"github.com/coreos/coreos-assembler/mantle/platform/conf"
@@ -50,6 +52,20 @@ func init() {
 		ExcludeArchitectures: []string{"s390x"}, // no TPM backend support for s390x
 		Tags:                 []string{"luks", "tpm", "tang", "sss", kola.NeedsInternetTag, "reprovision"},
 	})
+	register.RegisterTest(&register.Test{
+		Run:           runCexTest,
+		ClusterSize:   0,
+		Name:          `luks.cex`,
+		Description:   "Verify that CEX-based rootfs encryption works.",
+		Flags:         []register.Flag{},
+		Platforms:     []string{"qemu"},
+		Architectures: []string{"s390x"},
+		Tags:          []string{"luks", "cex", "reprovision"},
+		NativeFuncs: map[string]register.NativeFuncWrap{
+			"RHCOSGrowpart": register.CreateNativeFuncWrap(coretest.TestRHCOSGrowfs, []string{"fcos"}...),
+			"FCOSGrowpart":  register.CreateNativeFuncWrap(coretest.TestFCOSGrowfs, []string{"rhcos"}...),
+		},
+	})
 }
 
 func setupTangMachine(c cluster.TestCluster) ut.TangServer {
@@ -176,6 +192,74 @@ func runTest(c cluster.TestCluster, tpm2 bool, threshold int, killTangAfterFirst
 	ut.LUKSSanityTest(c, tangd, m, tpm2, killTangAfterFirstBoot, rootPart)
 }
 
+func runCexTest(c cluster.TestCluster) {
+	var err error
+	var m platform.Machine
+
+	// To prevent the test to fail the whole run on s390x machine that does not have Cex Device
+	cex_uuid := os.Getenv("KOLA_CEX_UUID")
+	if cex_uuid == "" {
+		c.Skip("No CEX device found in KOLA_CEX_UUID env var")
+	}
+
+	ignition := conf.Ignition(`{
+		"ignition": {
+			"version": "3.5.0-experimental"
+		},
+		"kernelArguments": {
+			"shouldExist": [
+				"rd.luks.key=/etc/luks/cex.key"
+		]
+		},
+		"storage": {
+			"luks": [
+				{
+					"name": "root",
+					"device": "/dev/disk/by-label/root",
+					"cex": {
+						"enabled": true
+					},
+					"label": "root",
+					"wipeVolume": true
+				}
+			],
+			"filesystems": [
+				{
+					"device": "/dev/mapper/root",
+					"format": "xfs",
+					"wipeFilesystem": true,
+					"label": "root"
+				}
+			]
+		}
+	}`)
+
+	opts := platform.QemuMachineOptions{
+		Cex: true,
+	}
+	opts.MinMemory = 8192
+
+	switch pc := c.Cluster.(type) {
+	case *qemu.Cluster:
+		m, err = pc.NewMachineWithQemuOptions(ignition, opts)
+	default:
+		panic("Unsupported cluster type")
+	}
+
+	// copy over kolet into the machine
+	if err := kola.ScpKolet([]platform.Machine{m}); err != nil {
+		c.Fatal(err)
+	}
+	coretest.LocalTests(c)
+
+	if err != nil {
+		c.Fatalf("Unable to create test machine: %v", err)
+	}
+	rootPart := "/dev/disk/by-partlabel/root"
+
+	ut.LUKSSanityCEXTest(c, m, rootPart)
+}
+
 // Verify that the rootfs is encrypted with Tang
 func luksTangTest(c cluster.TestCluster) {
 	runTest(c, false, 1, false)

mantle/kola/tests/util/luks.go

@@ -79,3 +79,21 @@ func LUKSSanityTest(c cluster.TestCluster, tangd TangServer, m platform.Machine,
 	luksDump = c.MustSSH(m, "sudo cryptsetup luksDump "+rootPart)
 	mustMatch(c, "Cipher: *aes", luksDump)
 }
+
+// LUKSSanityCEXTest verifies that the rootfs is encrypted with Cex based LUKS
+func LUKSSanityCEXTest(c cluster.TestCluster, m platform.Machine, rootPart string) {
+	var err error
+	luksDump := c.MustSSH(m, "sudo cryptsetup luksDump "+rootPart)
+	mustMatch(c, "cipher: paes-*", luksDump)
+	mustNotMatch(c, "Cipher: *cipher_null-ecb", luksDump)
+	mustMatch(c, "0: paes-verification-pattern", luksDump)
+	mustNotMatch(c, "9: *coreos", luksDump)
+
+	err = m.Reboot()
+
+	if err != nil {
+		c.Fatalf("Failed to reboot the machine: %v", err)
+	}
+	luksDump = c.MustSSH(m, "sudo cryptsetup luksDump "+rootPart)
+	mustMatch(c, "cipher: paes-*", luksDump)
+}

mantle/platform/machine/qemu/cluster.go

@@ -150,6 +150,12 @@ func (qc *Cluster) NewMachineWithQemuOptions(userdata *conf.UserData, options pl
 		primaryDisk = *diskp
 	}
 
+	if qc.flight.opts.Cex || options.Cex {
+		if err := builder.AddCexDevice(); err != nil {
+			return nil, err
+		}
+	}
+
 	if qc.flight.opts.Nvme || options.Nvme {
 		primaryDisk.Channel = "nvme"
 	}

mantle/platform/machine/qemu/flight.go

@@ -55,6 +55,9 @@ type Options struct {
 	SecureExecutionIgnitionPubKey string
 	SecureExecutionHostKey        string
 
+	// Option to create IBM cex based luks encryption
+	Cex bool
+
 	*platform.Options
 }
 

mantle/platform/qemu.go

@@ -75,6 +75,7 @@ type QemuMachineOptions struct {
 	OverrideBackingFile string
 	Firmware            string
 	Nvme                bool
+	Cex                 bool
 }
 
 // QEMUMachine represents a qemu instance.
@@ -2063,3 +2064,13 @@ func (builder *QemuBuilder) Close() {
 		os.RemoveAll(builder.tempdir)
 	}
 }
+
+// supports IBM Cex based LUKS encryption if it is s390x host (zKVM/LPAR)
+func (builder *QemuBuilder) AddCexDevice() error {
+	cex_uuid := os.Getenv("KOLA_CEX_UUID")
+	if cex_uuid == "" {
+		return errors.New("cannot add CEX device: KOLA_CEX_UUID env var undefined")
+	}
+	builder.Append("-device", fmt.Sprintf("vfio-ap,sysfsdev=/sys/devices/vfio_ap/matrix/%s", cex_uuid))
+	return nil
+}