Enable cosa/kola run for .iso

Lower the bits to do "inject Ignition via coreos-installer embed iso"
into the qemu layer.  Then teach `cosa run` and the mantle "flight"
code (used by `kola`) to use it automatically if the target
disk ends in `.iso`.

This makes it completely trivial to get a `ssh` shell in an ISO:
`cosa run --qemu-image builds/latest/x86_64/rhcos-46.82.202007011714-0-live.x86_64.iso --memory 819`

And even better means we can start running a lot of the kola tests
directly against the ISO (though not everything will work, e.g.
tests that reboot can't persist state).   I did verify this works:
`kola run --qemu-image builds/latest/x86_64/rhcos-46.82.202007011714-0-live.x86_64.iso --qemu-memory 8192 basic`

mantle/cmd/kola/qemuexec.go

@@ -191,23 +191,27 @@ func runQemuExec(cmd *cobra.Command, args []string) error {
 	defer builder.Close()
 	builder.Firmware = kola.QEMUOptions.Firmware
 	if kola.QEMUOptions.DiskImage != "" {
-		channel := "virtio"
-		if kola.QEMUOptions.Nvme {
-			channel = "nvme"
-		}
-		sectorSize := 0
-		if kola.QEMUOptions.Native4k {
-			sectorSize = 4096
-		}
-		if err = builder.AddPrimaryDisk(&platform.Disk{
-			BackingFile:   kola.QEMUOptions.DiskImage,
-			Channel:       channel,
-			Size:          kola.QEMUOptions.DiskSize,
-			SectorSize:    sectorSize,
-			MultiPathDisk: kola.QEMUOptions.MultiPathDisk,
-			NbdDisk:       kola.QEMUOptions.NbdDisk,
-		}); err != nil {
-			return errors.Wrapf(err, "adding primary disk")
+		if strings.HasSuffix(kola.QEMUOptions.DiskImage, ".iso") {
+			builder.AddInstallIso(kola.QEMUOptions.DiskImage, "")
+		} else {
+			channel := "virtio"
+			if kola.QEMUOptions.Nvme {
+				channel = "nvme"
+			}
+			sectorSize := 0
+			if kola.QEMUOptions.Native4k {
+				sectorSize = 4096
+			}
+			if err = builder.AddPrimaryDisk(&platform.Disk{
+				BackingFile:   kola.QEMUOptions.DiskImage,
+				Channel:       channel,
+				Size:          kola.QEMUOptions.DiskSize,
+				SectorSize:    sectorSize,
+				MultiPathDisk: kola.QEMUOptions.MultiPathDisk,
+				NbdDisk:       kola.QEMUOptions.NbdDisk,
+			}); err != nil {
+				return errors.Wrapf(err, "adding primary disk")
+			}
 		}
 	}
 	builder.Hostname = hostname

mantle/platform/machine/unprivqemu/cluster.go

@@ -19,6 +19,7 @@ import (
 	"os"
 	"path/filepath"
 	"strconv"
+	"strings"
 
 	"sync"
 	"time"
@@ -105,29 +106,33 @@ func (qc *Cluster) NewMachineWithOptions(userdata *conf.UserData, options platfo
 		builder.Memory = int(memory)
 	}
 
-	channel := "virtio"
-	if qc.flight.opts.Nvme {
-		channel = "nvme"
-	}
-	sectorSize := 0
-	if qc.flight.opts.Native4k {
-		sectorSize = 4096
-	}
-	multiPathDisk := false
-	if qc.flight.opts.MultiPathDisk {
-		multiPathDisk = true
-	}
-	primaryDisk := platform.Disk{
-		BackingFile:   qc.flight.opts.DiskImage,
-		Channel:       channel,
-		Size:          qc.flight.opts.DiskSize,
-		SectorSize:    sectorSize,
-		MultiPathDisk: multiPathDisk,
+	if strings.HasSuffix(qc.flight.opts.DiskImage, ".iso") {
+		builder.AddInstallIso(qc.flight.opts.DiskImage, "")
+	} else {
+		channel := "virtio"
+		if qc.flight.opts.Nvme {
+			channel = "nvme"
+		}
+		sectorSize := 0
+		if qc.flight.opts.Native4k {
+			sectorSize = 4096
+		}
+		multiPathDisk := false
+		if qc.flight.opts.MultiPathDisk {
+			multiPathDisk = true
+		}
+		primaryDisk := platform.Disk{
+			BackingFile:   qc.flight.opts.DiskImage,
+			Channel:       channel,
+			Size:          qc.flight.opts.DiskSize,
+			SectorSize:    sectorSize,
+			MultiPathDisk: multiPathDisk,
+		}
+		if err = builder.AddPrimaryDisk(&primaryDisk); err != nil {
+			return nil, errors.Wrapf(err, "adding primary disk")
+		}
 	}
 
-	if err = builder.AddPrimaryDisk(&primaryDisk); err != nil {
-		return nil, errors.Wrapf(err, "adding primary disk")
-	}
 	for _, disk := range options.AdditionalDisks {
 		if err = builder.AddDisk(&disk); err != nil {
 			return nil, errors.Wrapf(err, "adding additional disk")

mantle/platform/metal.go

@@ -640,36 +640,10 @@ WantedBy=multi-user.target
 	}
 	mergedConfig := v3.Merge(inst.liveIgnition, installerConfig)
 	mergedConfig = v3.Merge(mergedConfig, conf.GetAutologin())
-	mergedConfigSerialized, err := conf.SerializeAndMaybeConvert(mergedConfig, inst.IgnitionSpec2)
-	if err != nil {
-		return nil, err
-	}
-
-	isoEmbeddedPath := filepath.Join(tempdir, "test.iso")
-	// TODO ensure this tempdir is underneath cosa tempdir so we can reliably reflink
-	cpcmd := exec.Command("cp", "--reflink=auto", srcisopath, isoEmbeddedPath)
-	cpcmd.Stderr = os.Stderr
-	if err := cpcmd.Run(); err != nil {
-		return nil, errors.Wrapf(err, "copying iso")
-	}
-	instCmd := exec.Command("coreos-installer", "iso", "embed", isoEmbeddedPath)
-	instCmd.Stderr = os.Stderr
-	instCmdStdin, err := instCmd.StdinPipe()
-	if err != nil {
-		return nil, err
-	}
-	go func() {
-		defer instCmdStdin.Close()
-		if _, err := instCmdStdin.Write(mergedConfigSerialized); err != nil {
-			panic(err)
-		}
-	}()
-	if err := instCmd.Run(); err != nil {
-		return nil, errors.Wrapf(err, "running coreos-installer iso embed")
-	}
 
 	qemubuilder := inst.Builder
-	qemubuilder.AddInstallIso(isoEmbeddedPath, "bootindex=2")
+	qemubuilder.SetConfig(mergedConfig, inst.IgnitionSpec2)
+	qemubuilder.AddInstallIso(srcisopath, "bootindex=2")
 
 	if offline {
 		qemubuilder.Append("-nic", "none")

mantle/platform/qemu.go

@@ -66,6 +66,12 @@ type Disk struct {
 	nbdServCmd     exec.Cmd // command to serve the disk
 }
 
+// bootIso is an internal struct used by SetInstallIso()
+type bootIso struct {
+	path      string
+	bootindex string
+}
+
 type QemuInstance struct {
 	qemu               exec.Cmd
 	tmpConfig          string
@@ -220,6 +226,7 @@ type QemuBuilder struct {
 
 	InheritConsole bool
 
+	iso         *bootIso
 	primaryDisk *Disk
 
 	MultiPathDisk bool
@@ -747,22 +754,9 @@ func (builder *QemuBuilder) AddDisk(disk *Disk) error {
 
 // AddInstallIso adds an ISO image
 func (builder *QemuBuilder) AddInstallIso(path string, bootindexStr string) error {
-	// Arches s390x and ppc64le don't support UEFI and use the cdrom option to boot the ISO.
-	// For all other arches we use ide-cd device with bootindex=2 here: the idea is
-	// that during an ISO install, the primary disk isn't bootable, so the bootloader
-	// will fall back to the ISO boot. On reboot when the system is installed, the
-	// primary disk is selected. This allows us to have "boot once" functionality on
-	// both UEFI and BIOS (`-boot once=d` OTOH doesn't work with OVMF).
-
-	// TBD: aarch64 does not support ide-cd but cdrom won't work either.Including it here as it doesn't error out at least.
-	switch system.RpmArch() {
-	case "s390x", "ppc64le", "aarch64":
-		builder.Append("-cdrom", path)
-	default:
-		if bootindexStr != "" {
-			bootindexStr = "," + bootindexStr
-		}
-		builder.Append("-drive", "file="+path+",format=raw,if=none,readonly=on,id=installiso", "-device", "ide-cd,drive=installiso"+bootindexStr)
+	builder.iso = &bootIso{
+		path:      path,
+		bootindex: bootindexStr,
 	}
 	return nil
 }
@@ -887,6 +881,59 @@ func (builder *QemuBuilder) setupUefi(secureBoot bool) error {
 	return nil
 }
 
+func (builder *QemuBuilder) setupIso() error {
+	if builder.ConfigFile != "" {
+		if builder.configInjected {
+			panic("config already injected?")
+		}
+		if err := builder.ensureTempdir(); err != nil {
+			return err
+		}
+		// TODO change to use something like an unlinked tempfile (or O_TMPFILE)
+		// in the same filesystem as the source so that reflinks (if available)
+		// will work
+		isoEmbeddedPath := filepath.Join(builder.tempdir, "install.iso")
+		cpcmd := exec.Command("cp", "--reflink=auto", builder.iso.path, isoEmbeddedPath)
+		cpcmd.Stderr = os.Stderr
+		if err := cpcmd.Run(); err != nil {
+			return errors.Wrapf(err, "copying iso")
+		}
+		configf, err := os.Open(builder.ConfigFile)
+		if err != nil {
+			return err
+		}
+		instCmd := exec.Command("coreos-installer", "iso", "embed", isoEmbeddedPath)
+		instCmd.Stdin = configf
+		instCmd.Stderr = os.Stderr
+		if err := instCmd.Run(); err != nil {
+			return errors.Wrapf(err, "running coreos-installer iso embed")
+		}
+		builder.iso.path = isoEmbeddedPath
+		builder.configInjected = true
+	}
+
+	// Arches s390x and ppc64le don't support UEFI and use the cdrom option to boot the ISO.
+	// For all other arches we use ide-cd device with bootindex=2 here: the idea is
+	// that during an ISO install, the primary disk isn't bootable, so the bootloader
+	// will fall back to the ISO boot. On reboot when the system is installed, the
+	// primary disk is selected. This allows us to have "boot once" functionality on
+	// both UEFI and BIOS (`-boot once=d` OTOH doesn't work with OVMF).
+
+	// TBD: aarch64 does not support ide-cd but cdrom won't work either.Including it here as it doesn't error out at least.
+	switch system.RpmArch() {
+	case "s390x", "ppc64le", "aarch64":
+		builder.Append("-cdrom", builder.iso.path)
+	default:
+		bootindexStr := ""
+		if builder.iso.bootindex != "" {
+			bootindexStr = "," + builder.iso.bootindex
+		}
+		builder.Append("-drive", "file="+builder.iso.path+",format=raw,if=none,readonly=on,id=installiso", "-device", "ide-cd,drive=installiso"+bootindexStr)
+	}
+
+	return nil
+}
+
 // VirtioChannelRead allocates a virtio-serial channel that will appear in
 // the guest as /dev/virtio-ports/<name>.  The guest can write to it, and
 // the host can read.
@@ -1021,14 +1068,20 @@ func (builder *QemuBuilder) Exec() (*QemuInstance, error) {
 	// We never want a popup window
 	argv = append(argv, "-nographic")
 
-	// See AddPrimaryDisk for why we do this lazily
+	// We only render Ignition lazily, because we want to support calling
+	// SetConfig() after AddPrimaryDisk() or AddInstallIso().
+	if builder.iso != nil {
+		if err := builder.setupIso(); err != nil {
+			return nil, err
+		}
+	}
 	if builder.primaryDisk != nil {
 		if err := builder.addDiskImpl(builder.primaryDisk, true); err != nil {
 			return nil, err
 		}
 	}
 
-	// Handle Ignition
+	// Handle Ignition if it wasn't already injected above
 	if builder.ConfigFile != "" && !builder.configInjected {
 		if builder.supportsFwCfg() {
 			builder.Append("-fw_cfg", "name=opt/com.coreos/config,file="+builder.ConfigFile)