From 9696753be34e21410f91cd211e034f5b07059a66 Mon Sep 17 00:00:00 2001 From: Benjamin Somers Date: Wed, 15 Jan 2025 14:24:28 +0000 Subject: [PATCH 01/12] incusd/instance: Split startupHook function Signed-off-by: Benjamin Somers --- .../server/instance/drivers/driver_qemu.go | 31 +++++++++++-------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/internal/server/instance/drivers/driver_qemu.go b/internal/server/instance/drivers/driver_qemu.go index fb43917ca96..90e2825ece1 100644 --- a/internal/server/instance/drivers/driver_qemu.go +++ b/internal/server/instance/drivers/driver_qemu.go @@ -1126,7 +1126,23 @@ func (d *qemu) Start(stateful bool) error { return d.start(stateful, nil) } -// startupHook executes QMP commands and runs startup scriptlets at early, pre-stard and post-start +// runStartupScriptlet runs startup scriptlets at config, early, pre-start and post-start stages. +func (d *qemu) runStartupScriptlet(monitor *qmp.Monitor, stage string) error { + _, ok := d.expandedConfig["raw.qemu.scriptlet"] + if ok { + instanceName := d.Name() + + err := scriptlet.QEMURun(logger.Log, monitor, instanceName, stage) + if err != nil { + err = fmt.Errorf("Failed running QEMU scriptlet at %s stage: %w", stage, err) + return err + } + } + + return nil +} + +// startupHook executes QMP commands and runs startup scriptlets at early, pre-start and post-start // stages. func (d *qemu) startupHook(monitor *qmp.Monitor, stage string) error { commands, ok := d.expandedConfig["raw.qemu.qmp."+stage] @@ -1148,18 +1164,7 @@ func (d *qemu) startupHook(monitor *qmp.Monitor, stage string) error { } } - _, ok = d.expandedConfig["raw.qemu.scriptlet"] - if ok { - instanceName := d.Name() - - err := scriptlet.QEMURun(logger.Log, monitor, instanceName, stage) - if err != nil { - err = fmt.Errorf("Failed running QEMU scriptlet at %s stage: %w", stage, err) - return err - } - } - - return nil + return d.runStartupScriptlet(monitor, stage) } // start starts the instance and can use an existing InstanceOperation lock. From 7e6670025eddda09f31cfb8c10c8d1bc2239e7fe Mon Sep 17 00:00:00 2001 From: Benjamin Somers Date: Wed, 15 Jan 2025 15:18:04 +0000 Subject: [PATCH 02/12] incusd/instance: Pass an *api.Instance to the scriptlet program Signed-off-by: Benjamin Somers --- .../server/instance/drivers/driver_qemu.go | 42 ++++++++++++------- internal/server/scriptlet/qemu.go | 5 ++- 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/internal/server/instance/drivers/driver_qemu.go b/internal/server/instance/drivers/driver_qemu.go index 90e2825ece1..ec83ffb8bde 100644 --- a/internal/server/instance/drivers/driver_qemu.go +++ b/internal/server/instance/drivers/driver_qemu.go @@ -1130,9 +1130,14 @@ func (d *qemu) Start(stateful bool) error { func (d *qemu) runStartupScriptlet(monitor *qmp.Monitor, stage string) error { _, ok := d.expandedConfig["raw.qemu.scriptlet"] if ok { - instanceName := d.Name() + // Render cannot return errors here. + render, _, _ := d.Render() + instanceData, ok := render.(*api.Instance) + if !ok { + return errors.New("Unexpected instance type") + } - err := scriptlet.QEMURun(logger.Log, monitor, instanceName, stage) + err := scriptlet.QEMURun(logger.Log, instanceData, monitor, stage) if err != nil { err = fmt.Errorf("Failed running QEMU scriptlet at %s stage: %w", stage, err) return err @@ -1696,6 +1701,27 @@ func (d *qemu) start(stateful bool, op *operationlock.InstanceOperation) error { qemuCmd = append(qemuCmd, fields...) } + d.cmdArgs = qemuArgs + + // Precompile the QEMU scriptlet + src, ok := d.expandedConfig["raw.qemu.scriptlet"] + if ok { + instanceName := d.Name() + + err := scriptletLoad.QEMUSet(src, instanceName) + if err != nil { + err = fmt.Errorf("Failed loading QEMU scriptlet: %w", err) + return err + } + } + + // Config startup hook. + err = d.runStartupScriptlet(nil, "config") + if err != nil { + op.Done(err) + return err + } + // Run the qemu command via forklimits so we can selectively increase ulimits. forkLimitsCmd := []string{ "forklimits", @@ -1777,18 +1803,6 @@ func (d *qemu) start(stateful bool, op *operationlock.InstanceOperation) error { // onStop hook isn't triggered prematurely (as this function's reverter will clean up on failure to start). monitor.SetOnDisconnectEvent(false) - // Precompile the QEMU scriptlet - src, ok := d.expandedConfig["raw.qemu.scriptlet"] - if ok { - instanceName := d.Name() - - err := scriptletLoad.QEMUSet(src, instanceName) - if err != nil { - err = fmt.Errorf("Failed loading QEMU scriptlet: %w", err) - return err - } - } - // Early startup hook err = d.startupHook(monitor, "early") if err != nil { diff --git a/internal/server/scriptlet/qemu.go b/internal/server/scriptlet/qemu.go index a72f722562d..1df2c1fa4cf 100644 --- a/internal/server/scriptlet/qemu.go +++ b/internal/server/scriptlet/qemu.go @@ -11,11 +11,12 @@ import ( scriptletLoad "github.com/lxc/incus/v6/internal/server/scriptlet/load" "github.com/lxc/incus/v6/internal/server/scriptlet/log" "github.com/lxc/incus/v6/internal/server/scriptlet/marshal" + "github.com/lxc/incus/v6/shared/api" "github.com/lxc/incus/v6/shared/logger" ) // QEMURun runs the QEMU scriptlet. -func QEMURun(l logger.Logger, m *qmp.Monitor, instance string, stage string) error { +func QEMURun(l logger.Logger, instance *api.Instance, m *qmp.Monitor, stage string) error { logFunc := log.CreateLogger(l, "QEMU scriptlet ("+stage+")") runQMPFunc := func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { var command *starlark.Dict @@ -155,7 +156,7 @@ func QEMURun(l logger.Logger, m *qmp.Monitor, instance string, stage string) err "qom_set": makeQOM("qom-set"), } - prog, thread, err := scriptletLoad.QEMUProgram(instance) + prog, thread, err := scriptletLoad.QEMUProgram(instance.Name) if err != nil { return err } From ec4ee7fa7b7bbf49c34f1545a955f88bc58f0b96 Mon Sep 17 00:00:00 2001 From: Benjamin Somers Date: Wed, 15 Jan 2025 15:28:47 +0000 Subject: [PATCH 03/12] incusd/scriptlet/qemu: Add instance parameter to the QEMU scriptlet Signed-off-by: Benjamin Somers --- internal/server/scriptlet/load/load.go | 2 +- internal/server/scriptlet/qemu.go | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/internal/server/scriptlet/load/load.go b/internal/server/scriptlet/load/load.go index 9ba5e30ec04..2f6904d88d9 100644 --- a/internal/server/scriptlet/load/load.go +++ b/internal/server/scriptlet/load/load.go @@ -81,7 +81,7 @@ func QEMUCompile(name string, src string) (*starlark.Program, error) { // QEMUValidate validates the QEMU scriptlet. func QEMUValidate(src string) error { return validate(QEMUCompile, prefixQEMU, src, declaration{ - required("qemu_hook"): {"stage"}, + required("qemu_hook"): {"instance", "stage"}, }) } diff --git a/internal/server/scriptlet/qemu.go b/internal/server/scriptlet/qemu.go index 1df2c1fa4cf..968a8f896fc 100644 --- a/internal/server/scriptlet/qemu.go +++ b/internal/server/scriptlet/qemu.go @@ -174,8 +174,17 @@ func QEMURun(l logger.Logger, instance *api.Instance, m *qmp.Monitor, stage stri return fmt.Errorf("Scriptlet missing qemu_hook function") } + instancev, err := marshal.StarlarkMarshal(instance) + if err != nil { + return fmt.Errorf("Marshalling instance failed: %w", err) + } + // Call starlark function from Go. v, err := starlark.Call(thread, qemuHook, nil, []starlark.Tuple{ + { + starlark.String("instance"), + instancev, + }, { starlark.String("stage"), starlark.String(stage), From 688bb4198256be9cb5cfc5570658d94909cc49be Mon Sep 17 00:00:00 2001 From: Benjamin Somers Date: Wed, 15 Jan 2025 18:00:53 +0000 Subject: [PATCH 04/12] incusd/instance: Rewire QEMU config generation Signed-off-by: Benjamin Somers --- .../server/instance/drivers/driver_qemu.go | 81 +++++++++++-------- 1 file changed, 48 insertions(+), 33 deletions(-) diff --git a/internal/server/instance/drivers/driver_qemu.go b/internal/server/instance/drivers/driver_qemu.go index ec83ffb8bde..39fa2cfe3fa 100644 --- a/internal/server/instance/drivers/driver_qemu.go +++ b/internal/server/instance/drivers/driver_qemu.go @@ -350,6 +350,10 @@ type qemu struct { // Keep a reference to the console socket when switching backends, so we can properly cleanup when switching back to a ring buffer. consoleSocket *net.UnixListener consoleSocketFile *os.File + + // Keep a record of QEMU configuration. + cmdArgs []string + cfg []cfgSection } // getAgentClient returns the current agent client handle. @@ -1568,16 +1572,15 @@ func (d *qemu) start(stateful bool, op *operationlock.InstanceOperation) error { } // Generate the QEMU configuration. - confFile, monHooks, err := d.generateQemuConfigFile(cpuInfo, mountInfo, qemuBus, vsockFD, devConfs, &fdFiles) + monHooks, err := d.generateQemuConfig(cpuInfo, mountInfo, qemuBus, vsockFD, devConfs, &fdFiles) if err != nil { op.Done(err) return err } + confFile := filepath.Join(d.RunPath(), "qemu.conf") // Start QEMU. - qemuCmd := []string{ - "--", - qemuPath, + qemuArgs := []string{ "-S", "-name", d.Name(), "-uuid", instUUID, @@ -1597,7 +1600,7 @@ func (d *qemu) start(stateful bool, op *operationlock.InstanceOperation) error { // If stateful, restore now. if stateful { if d.stateful { - qemuCmd = append(qemuCmd, "-incoming", "defer") + qemuArgs = append(qemuArgs, "-incoming", "defer") } else { // No state to restore, just start as normal. stateful = false @@ -1622,12 +1625,12 @@ func (d *qemu) start(stateful bool, op *operationlock.InstanceOperation) error { // SMBIOS only on x86_64 and aarch64. if d.architectureSupportsUEFI(d.architecture) { - qemuCmd = append(qemuCmd, "-smbios", "type=2,manufacturer=LinuxContainers,product=Incus") + qemuArgs = append(qemuArgs, "-smbios", "type=2,manufacturer=LinuxContainers,product=Incus") } // Attempt to drop privileges (doesn't work when restoring state). if !stateful && d.state.OS.UnprivUser != "" { - qemuCmd = append(qemuCmd, "-runas", d.state.OS.UnprivUser) + qemuArgs = append(qemuArgs, "-runas", d.state.OS.UnprivUser) nvRAMPath := d.nvramPath() if d.architectureSupportsUEFI(d.architecture) && util.PathExists(nvRAMPath) { @@ -1688,7 +1691,7 @@ func (d *qemu) start(stateful bool, op *operationlock.InstanceOperation) error { return err } - qemuCmd = append(qemuCmd, "-mem-path", hugetlb, "-mem-prealloc") + qemuArgs = append(qemuArgs, "-mem-path", hugetlb, "-mem-prealloc") } if d.expandedConfig["raw.qemu"] != "" { @@ -1698,7 +1701,7 @@ func (d *qemu) start(stateful bool, op *operationlock.InstanceOperation) error { return err } - qemuCmd = append(qemuCmd, fields...) + qemuArgs = append(qemuArgs, fields...) } d.cmdArgs = qemuArgs @@ -1722,6 +1725,13 @@ func (d *qemu) start(stateful bool, op *operationlock.InstanceOperation) error { return err } + // Write the config file. + err = d.writeQemuConfigFile(confFile) + if err != nil { + op.Done(err) + return err + } + // Run the qemu command via forklimits so we can selectively increase ulimits. forkLimitsCmd := []string{ "forklimits", @@ -1738,7 +1748,8 @@ func (d *qemu) start(stateful bool, op *operationlock.InstanceOperation) error { } // Log the QEMU command line. - fullCmd := append(forkLimitsCmd, qemuCmd...) + fullCmd := append(forkLimitsCmd, "--", qemuPath) + fullCmd = append(fullCmd, d.cmdArgs...) d.logger.Debug("Starting QEMU", logger.Ctx{"command": fullCmd}) // Setup background process. @@ -3307,16 +3318,15 @@ func (d *qemu) deviceBootPriorities(base int) (map[string]int, error) { return sortedDevs, nil } -// generateQemuConfigFile writes the qemu config file and returns its location. -// It writes the config file inside the VM's log path. -func (d *qemu) generateQemuConfigFile(cpuInfo *cpuTopology, mountInfo *storagePools.MountInfo, busName string, vsockFD int, devConfs []*deviceConfig.RunConfig, fdFiles *[]*os.File) (string, []monitorHook, error) { +// generateQemuConfig generates the QEMU configuration. +func (d *qemu) generateQemuConfig(cpuInfo *cpuTopology, mountInfo *storagePools.MountInfo, busName string, vsockFD int, devConfs []*deviceConfig.RunConfig, fdFiles *[]*os.File) ([]monitorHook, error) { var monHooks []monitorHook cfg := qemuBase(&qemuBaseOpts{d.Architecture()}) err := d.addCPUMemoryConfig(&cfg, cpuInfo) if err != nil { - return "", nil, err + return nil, err } // Parse raw.qemu. @@ -3324,7 +3334,7 @@ func (d *qemu) generateQemuConfigFile(cpuInfo *cpuTopology, mountInfo *storagePo if d.expandedConfig["raw.qemu"] != "" { rawOptions, err = shellquote.Split(d.expandedConfig["raw.qemu"]) if err != nil { - return "", nil, err + return nil, err } } @@ -3336,7 +3346,7 @@ func (d *qemu) generateQemuConfigFile(cpuInfo *cpuTopology, mountInfo *storagePo // This is so the QEMU process can still read/write the file after it has dropped its user privs. nvRAMFile, err := os.Open(d.nvramPath()) if err != nil { - return "", nil, fmt.Errorf("Failed opening NVRAM file: %w", err) + return nil, fmt.Errorf("Failed opening NVRAM file: %w", err) } // Determine expected firmware. @@ -3358,7 +3368,7 @@ func (d *qemu) generateQemuConfigFile(cpuInfo *cpuTopology, mountInfo *storagePo } if efiCode == "" { - return "", nil, fmt.Errorf("Unable to locate matching firmware: %+v", firmwares) + return nil, fmt.Errorf("Unable to locate matching firmware: %+v", firmwares) } driveFirmwareOpts := qemuDriveFirmwareOpts{ @@ -3430,7 +3440,7 @@ func (d *qemu) generateQemuConfigFile(cpuInfo *cpuTopology, mountInfo *storagePo // Existing vsock ID from volatile. vsockID, err := d.getVsockID() if err != nil { - return "", nil, err + return nil, err } devBus, devAddr, multi = bus.allocate(busFunctionGroupGeneric) @@ -3536,7 +3546,7 @@ func (d *qemu) generateQemuConfigFile(cpuInfo *cpuTopology, mountInfo *storagePo if util.IsTrue(d.expandedConfig["security.sev"]) { sevOpts, err := d.setupSEV(fdFiles) if err != nil { - return "", nil, err + return nil, err } if sevOpts != nil { @@ -3581,7 +3591,7 @@ func (d *qemu) generateQemuConfigFile(cpuInfo *cpuTopology, mountInfo *storagePo bootIndexes, err := d.deviceBootPriorities(base) if err != nil { - return "", nil, fmt.Errorf("Error calculating boot indexes: %w", err) + return nil, fmt.Errorf("Error calculating boot indexes: %w", err) } // Record the mounts we are going to do inside the VM using the agent. @@ -3632,7 +3642,7 @@ func (d *qemu) generateQemuConfigFile(cpuInfo *cpuTopology, mountInfo *storagePo } if err != nil { - return "", nil, fmt.Errorf("Failed setting up disk device %q: %w", drive.DevName, err) + return nil, fmt.Errorf("Failed setting up disk device %q: %w", drive.DevName, err) } if monHook != nil { @@ -3660,7 +3670,7 @@ func (d *qemu) generateQemuConfigFile(cpuInfo *cpuTopology, mountInfo *storagePo monHook, err := d.addNetDevConfig(bus.name, qemuDev, bootIndexes, runConf.NetworkInterface) if err != nil { - return "", nil, err + return nil, err } monHooks = append(monHooks, monHook) @@ -3670,7 +3680,7 @@ func (d *qemu) generateQemuConfigFile(cpuInfo *cpuTopology, mountInfo *storagePo if len(runConf.GPUDevice) > 0 { err = d.addGPUDevConfig(&cfg, bus, runConf.GPUDevice) if err != nil { - return "", nil, err + return nil, err } } @@ -3678,7 +3688,7 @@ func (d *qemu) generateQemuConfigFile(cpuInfo *cpuTopology, mountInfo *storagePo if len(runConf.PCIDevice) > 0 { err = d.addPCIDevConfig(&cfg, bus, runConf.PCIDevice) if err != nil { - return "", nil, err + return nil, err } } @@ -3686,7 +3696,7 @@ func (d *qemu) generateQemuConfigFile(cpuInfo *cpuTopology, mountInfo *storagePo for _, usbDev := range runConf.USBDevice { monHook, err := d.addUSBDeviceConfig(usbDev) if err != nil { - return "", nil, err + return nil, err } monHooks = append(monHooks, monHook) @@ -3696,7 +3706,7 @@ func (d *qemu) generateQemuConfigFile(cpuInfo *cpuTopology, mountInfo *storagePo if len(runConf.TPMDevice) > 0 { err = d.addTPMDeviceConfig(&cfg, runConf.TPMDevice, fdFiles) if err != nil { - return "", nil, err + return nil, err } } } @@ -3705,7 +3715,7 @@ func (d *qemu) generateQemuConfigFile(cpuInfo *cpuTopology, mountInfo *storagePo if d.architecture == osarch.ARCH_64BIT_INTEL_X86 { err = d.addVmgenDeviceConfig(&cfg, d.localConfig["volatile.uuid.generation"]) if err != nil { - return "", nil, err + return nil, err } } @@ -3717,21 +3727,26 @@ func (d *qemu) generateQemuConfigFile(cpuInfo *cpuTopology, mountInfo *storagePo // Write the agent mount config. agentMountJSON, err := json.Marshal(agentMounts) if err != nil { - return "", nil, fmt.Errorf("Failed marshalling agent mounts to JSON: %w", err) + return nil, fmt.Errorf("Failed marshalling agent mounts to JSON: %w", err) } agentMountFile := filepath.Join(d.Path(), "config", "agent-mounts.json") err = os.WriteFile(agentMountFile, agentMountJSON, 0400) if err != nil { - return "", nil, fmt.Errorf("Failed writing agent mounts file: %w", err) + return nil, fmt.Errorf("Failed writing agent mounts file: %w", err) } // process any user-specified overrides - cfg = qemuRawCfgOverride(cfg, d.expandedConfig) + d.cfg = qemuRawCfgOverride(cfg, d.expandedConfig) + return monHooks, nil +} + +// writeQemuConfigFile writes the QEMU config file. +// It writes the config file inside the VM's log path. +func (d *qemu) writeQemuConfigFile(configPath string) error { // Write the config file to disk. - sb := qemuStringifyCfg(cfg...) - configPath := filepath.Join(d.RunPath(), "qemu.conf") - return configPath, monHooks, os.WriteFile(configPath, []byte(sb.String()), 0640) + sb := qemuStringifyCfg(d.cfg...) + return os.WriteFile(configPath, []byte(sb.String()), 0640) } // addCPUMemoryConfig adds the qemu config required for setting the number of virtualised CPUs and memory. From 9168da5cd71c0b013801ae4ae48178ed2af642cf Mon Sep 17 00:00:00 2001 From: Benjamin Somers Date: Wed, 15 Jan 2025 18:31:08 +0000 Subject: [PATCH 05/12] incusd/instance: Make QEMU config types public Signed-off-by: Benjamin Somers --- internal/server/instance/drivers/cfg/cfg.go | 14 + .../server/instance/drivers/driver_qemu.go | 91 +-- .../instance/drivers/driver_qemu_bus.go | 10 +- .../drivers/driver_qemu_config_override.go | 108 +-- .../instance/drivers/driver_qemu_templates.go | 650 +++++++++--------- 5 files changed, 441 insertions(+), 432 deletions(-) create mode 100644 internal/server/instance/drivers/cfg/cfg.go diff --git a/internal/server/instance/drivers/cfg/cfg.go b/internal/server/instance/drivers/cfg/cfg.go new file mode 100644 index 00000000000..449c47581cb --- /dev/null +++ b/internal/server/instance/drivers/cfg/cfg.go @@ -0,0 +1,14 @@ +package cfg + +// Entry holds single QEMU configuration Key-Value pairs. +type Entry struct { + Key string + Value string +} + +// Section holds QEMU configuration sections. +type Section struct { + Name string + Comment string + Entries []Entry +} diff --git a/internal/server/instance/drivers/driver_qemu.go b/internal/server/instance/drivers/driver_qemu.go index 39fa2cfe3fa..feb1443316e 100644 --- a/internal/server/instance/drivers/driver_qemu.go +++ b/internal/server/instance/drivers/driver_qemu.go @@ -55,6 +55,7 @@ import ( deviceConfig "github.com/lxc/incus/v6/internal/server/device/config" "github.com/lxc/incus/v6/internal/server/device/nictype" "github.com/lxc/incus/v6/internal/server/instance" + "github.com/lxc/incus/v6/internal/server/instance/drivers/cfg" "github.com/lxc/incus/v6/internal/server/instance/drivers/edk2" "github.com/lxc/incus/v6/internal/server/instance/drivers/qemudefault" "github.com/lxc/incus/v6/internal/server/instance/drivers/qmp" @@ -353,7 +354,7 @@ type qemu struct { // Keep a record of QEMU configuration. cmdArgs []string - cfg []cfgSection + conf []cfg.Section } // getAgentClient returns the current agent client handle. @@ -3322,9 +3323,9 @@ func (d *qemu) deviceBootPriorities(base int) (map[string]int, error) { func (d *qemu) generateQemuConfig(cpuInfo *cpuTopology, mountInfo *storagePools.MountInfo, busName string, vsockFD int, devConfs []*deviceConfig.RunConfig, fdFiles *[]*os.File) ([]monitorHook, error) { var monHooks []monitorHook - cfg := qemuBase(&qemuBaseOpts{d.Architecture()}) + conf := qemuBase(&qemuBaseOpts{d.Architecture()}) - err := d.addCPUMemoryConfig(&cfg, cpuInfo) + err := d.addCPUMemoryConfig(&conf, cpuInfo) if err != nil { return nil, err } @@ -3376,17 +3377,17 @@ func (d *qemu) generateQemuConfig(cpuInfo *cpuTopology, mountInfo *storagePools. nvramPath: fmt.Sprintf("/dev/fd/%d", d.addFileDescriptor(fdFiles, nvRAMFile)), } - cfg = append(cfg, qemuDriveFirmware(&driveFirmwareOpts)...) + conf = append(conf, qemuDriveFirmware(&driveFirmwareOpts)...) } // QMP socket. - cfg = append(cfg, qemuControlSocket(&qemuControlSocketOpts{d.monitorPath()})...) + conf = append(conf, qemuControlSocket(&qemuControlSocketOpts{d.monitorPath()})...) // Console output. - cfg = append(cfg, qemuConsole()...) + conf = append(conf, qemuConsole()...) // Setup the bus allocator. - bus := qemuNewBus(busName, &cfg) + bus := qemuNewBus(busName, &conf) // Now add the fixed set of devices. The multi-function groups used for these fixed internal devices are // specifically chosen to ensure that we consume exactly 4 PCI bus ports (on PCIe bus). This ensures that @@ -3403,7 +3404,7 @@ func (d *qemu) generateQemuConfig(cpuInfo *cpuTopology, mountInfo *storagePools. multifunction: multi, } - cfg = append(cfg, qemuBalloon(&balloonOpts)...) + conf = append(conf, qemuBalloon(&balloonOpts)...) devBus, devAddr, multi = bus.allocate(busFunctionGroupGeneric) rngOpts := qemuDevOpts{ @@ -3413,7 +3414,7 @@ func (d *qemu) generateQemuConfig(cpuInfo *cpuTopology, mountInfo *storagePools. multifunction: multi, } - cfg = append(cfg, qemuRNG(&rngOpts)...) + conf = append(conf, qemuRNG(&rngOpts)...) devBus, devAddr, multi = bus.allocate(busFunctionGroupGeneric) keyboardOpts := qemuDevOpts{ @@ -3423,7 +3424,7 @@ func (d *qemu) generateQemuConfig(cpuInfo *cpuTopology, mountInfo *storagePools. multifunction: multi, } - cfg = append(cfg, qemuKeyboard(&keyboardOpts)...) + conf = append(conf, qemuKeyboard(&keyboardOpts)...) devBus, devAddr, multi = bus.allocate(busFunctionGroupGeneric) tabletOpts := qemuDevOpts{ @@ -3433,7 +3434,7 @@ func (d *qemu) generateQemuConfig(cpuInfo *cpuTopology, mountInfo *storagePools. multifunction: multi, } - cfg = append(cfg, qemuTablet(&tabletOpts)...) + conf = append(conf, qemuTablet(&tabletOpts)...) // Windows doesn't support virtio-vsock. if !strings.Contains(strings.ToLower(d.expandedConfig["image.os"]), "windows") { @@ -3455,7 +3456,7 @@ func (d *qemu) generateQemuConfig(cpuInfo *cpuTopology, mountInfo *storagePools. vsockID: vsockID, } - cfg = append(cfg, qemuVsock(&vsockOpts)...) + conf = append(conf, qemuVsock(&vsockOpts)...) } devBus, devAddr, multi = bus.allocate(busFunctionGroupGeneric) @@ -3470,7 +3471,7 @@ func (d *qemu) generateQemuConfig(cpuInfo *cpuTopology, mountInfo *storagePools. ringbufSizeBytes: qmp.RingbufSize, } - cfg = append(cfg, qemuSerial(&serialOpts)...) + conf = append(conf, qemuSerial(&serialOpts)...) // s390x doesn't really have USB. if d.architecture != osarch.ARCH_64BIT_S390_BIG_ENDIAN { @@ -3482,7 +3483,7 @@ func (d *qemu) generateQemuConfig(cpuInfo *cpuTopology, mountInfo *storagePools. ports: qemuSparseUSBPorts, } - cfg = append(cfg, qemuUSB(&usbOpts)...) + conf = append(conf, qemuUSB(&usbOpts)...) } if util.IsTrue(d.expandedConfig["security.csm"]) { @@ -3502,7 +3503,7 @@ func (d *qemu) generateQemuConfig(cpuInfo *cpuTopology, mountInfo *storagePools. multifunction: multi, } - cfg = append(cfg, qemuSCSI(&scsiOpts)...) + conf = append(conf, qemuSCSI(&scsiOpts)...) // Windows doesn't support virtio-9p. if !strings.Contains(strings.ToLower(d.expandedConfig["image.os"]), "windows") { @@ -3521,7 +3522,7 @@ func (d *qemu) generateQemuConfig(cpuInfo *cpuTopology, mountInfo *storagePools. path: d.configDriveMountPath(), } - cfg = append(cfg, qemuDriveConfig(&driveConfig9pOpts)...) + conf = append(conf, qemuDriveConfig(&driveConfig9pOpts)...) // Pass in the agents if INCUS_AGENT_PATH is set. if util.PathExists(os.Getenv("INCUS_AGENT_PATH")) { @@ -3538,7 +3539,7 @@ func (d *qemu) generateQemuConfig(cpuInfo *cpuTopology, mountInfo *storagePools. path: os.Getenv("INCUS_AGENT_PATH"), } - cfg = append(cfg, qemuDriveConfig(&driveConfig9pOpts)...) + conf = append(conf, qemuDriveConfig(&driveConfig9pOpts)...) } } @@ -3550,14 +3551,14 @@ func (d *qemu) generateQemuConfig(cpuInfo *cpuTopology, mountInfo *storagePools. } if sevOpts != nil { - for i := range cfg { - if cfg[i].name == "machine" { - cfg[i].entries = append(cfg[i].entries, cfgEntry{"memory-encryption", "sev0"}) + for i := range conf { + if conf[i].Name == "machine" { + conf[i].Entries = append(conf[i].Entries, cfg.Entry{Key: "memory-encryption", Value: "sev0"}) break } } - cfg = append(cfg, qemuSEV(sevOpts)...) + conf = append(conf, qemuSEV(sevOpts)...) } } @@ -3581,7 +3582,7 @@ func (d *qemu) generateQemuConfig(cpuInfo *cpuTopology, mountInfo *storagePools. architecture: d.Architecture(), } - cfg = append(cfg, qemuGPU(&gpuOpts)...) + conf = append(conf, qemuGPU(&gpuOpts)...) // Dynamic devices. base := 0 @@ -3636,7 +3637,7 @@ func (d *qemu) generateQemuConfig(cpuInfo *cpuTopology, mountInfo *storagePools. if drive.TargetPath == "/" { monHook, err = d.addRootDriveConfig(qemuDev, mountInfo, bootIndexes, drive) } else if drive.FSType == "9p" { - err = d.addDriveDirConfig(&cfg, bus, fdFiles, &agentMounts, drive) + err = d.addDriveDirConfig(&conf, bus, fdFiles, &agentMounts, drive) } else { monHook, err = d.addDriveConfig(qemuDev, bootIndexes, drive) } @@ -3678,7 +3679,7 @@ func (d *qemu) generateQemuConfig(cpuInfo *cpuTopology, mountInfo *storagePools. // Add GPU device. if len(runConf.GPUDevice) > 0 { - err = d.addGPUDevConfig(&cfg, bus, runConf.GPUDevice) + err = d.addGPUDevConfig(&conf, bus, runConf.GPUDevice) if err != nil { return nil, err } @@ -3686,7 +3687,7 @@ func (d *qemu) generateQemuConfig(cpuInfo *cpuTopology, mountInfo *storagePools. // Add PCI device. if len(runConf.PCIDevice) > 0 { - err = d.addPCIDevConfig(&cfg, bus, runConf.PCIDevice) + err = d.addPCIDevConfig(&conf, bus, runConf.PCIDevice) if err != nil { return nil, err } @@ -3704,7 +3705,7 @@ func (d *qemu) generateQemuConfig(cpuInfo *cpuTopology, mountInfo *storagePools. // Add TPM device. if len(runConf.TPMDevice) > 0 { - err = d.addTPMDeviceConfig(&cfg, runConf.TPMDevice, fdFiles) + err = d.addTPMDeviceConfig(&conf, runConf.TPMDevice, fdFiles) if err != nil { return nil, err } @@ -3713,7 +3714,7 @@ func (d *qemu) generateQemuConfig(cpuInfo *cpuTopology, mountInfo *storagePools. // VM generation ID is only available on x86. if d.architecture == osarch.ARCH_64BIT_INTEL_X86 { - err = d.addVmgenDeviceConfig(&cfg, d.localConfig["volatile.uuid.generation"]) + err = d.addVmgenDeviceConfig(&conf, d.localConfig["volatile.uuid.generation"]) if err != nil { return nil, err } @@ -3737,7 +3738,7 @@ func (d *qemu) generateQemuConfig(cpuInfo *cpuTopology, mountInfo *storagePools. } // process any user-specified overrides - d.cfg = qemuRawCfgOverride(cfg, d.expandedConfig) + d.conf = qemuRawCfgOverride(conf, d.expandedConfig) return monHooks, nil } @@ -3745,13 +3746,13 @@ func (d *qemu) generateQemuConfig(cpuInfo *cpuTopology, mountInfo *storagePools. // It writes the config file inside the VM's log path. func (d *qemu) writeQemuConfigFile(configPath string) error { // Write the config file to disk. - sb := qemuStringifyCfg(d.cfg...) + sb := qemuStringifyCfg(d.conf...) return os.WriteFile(configPath, []byte(sb.String()), 0640) } // addCPUMemoryConfig adds the qemu config required for setting the number of virtualised CPUs and memory. // If sb is nil then no config is written. -func (d *qemu) addCPUMemoryConfig(cfg *[]cfgSection, cpuInfo *cpuTopology) error { +func (d *qemu) addCPUMemoryConfig(conf *[]cfg.Section, cpuInfo *cpuTopology) error { // Figure out what memory object layout we're going to use. // Before v6.0 or if version unknown, we use the "repeated" format, otherwise we use "indexed" format. qemuMemObjectFormat := "repeated" @@ -3877,9 +3878,9 @@ func (d *qemu) addCPUMemoryConfig(cfg *[]cfgSection, cpuInfo *cpuTopology) error nodeMemory := int64(memSizeMB / int64(len(hostNodes))) cpuOpts.memory = nodeMemory - if cfg != nil { - *cfg = append(*cfg, qemuMemory(&qemuMemoryOpts{memSizeMB})...) - *cfg = append(*cfg, qemuCPU(&cpuOpts, cpuPinning)...) + if conf != nil { + *conf = append(*conf, qemuMemory(&qemuMemoryOpts{memSizeMB})...) + *conf = append(*conf, qemuCPU(&cpuOpts, cpuPinning)...) } return nil @@ -3936,7 +3937,7 @@ func (d *qemu) addRootDriveConfig(qemuDev map[string]any, mountInfo *storagePool } // addDriveDirConfig adds the qemu config required for adding a supplementary drive directory share. -func (d *qemu) addDriveDirConfig(cfg *[]cfgSection, bus *qemuBus, fdFiles *[]*os.File, agentMounts *[]instancetype.VMAgentMount, driveConf deviceConfig.MountEntryItem) error { +func (d *qemu) addDriveDirConfig(conf *[]cfg.Section, bus *qemuBus, fdFiles *[]*os.File, agentMounts *[]instancetype.VMAgentMount, driveConf deviceConfig.MountEntryItem) error { mountTag := fmt.Sprintf("incus_%s", driveConf.DevName) agentMount := instancetype.VMAgentMount{ @@ -3992,7 +3993,7 @@ func (d *qemu) addDriveDirConfig(cfg *[]cfgSection, bus *qemuBus, fdFiles *[]*os path: virtiofsdSockPath, protocol: "virtio-fs", } - *cfg = append(*cfg, qemuDriveDir(&driveDirVirtioOpts)...) + *conf = append(*conf, qemuDriveDir(&driveDirVirtioOpts)...) } // Add 9p share config. @@ -4019,7 +4020,7 @@ func (d *qemu) addDriveDirConfig(cfg *[]cfgSection, bus *qemuBus, fdFiles *[]*os readonly: readonly, protocol: "9p", } - *cfg = append(*cfg, qemuDriveDir(&driveDir9pOpts)...) + *conf = append(*conf, qemuDriveDir(&driveDir9pOpts)...) } return nil @@ -4719,7 +4720,7 @@ func (d *qemu) writeNICDevConfig(mtuStr string, devName string, nicName string, } // addPCIDevConfig adds the qemu config required for adding a raw PCI device. -func (d *qemu) addPCIDevConfig(cfg *[]cfgSection, bus *qemuBus, pciConfig []deviceConfig.RunConfigItem) error { +func (d *qemu) addPCIDevConfig(conf *[]cfg.Section, bus *qemuBus, pciConfig []deviceConfig.RunConfigItem) error { var devName, pciSlotName string for _, pciItem := range pciConfig { if pciItem.Key == "devName" { @@ -4740,13 +4741,13 @@ func (d *qemu) addPCIDevConfig(cfg *[]cfgSection, bus *qemuBus, pciConfig []devi devName: devName, pciSlotName: pciSlotName, } - *cfg = append(*cfg, qemuPCIPhysical(&pciPhysicalOpts)...) + *conf = append(*conf, qemuPCIPhysical(&pciPhysicalOpts)...) return nil } // addGPUDevConfig adds the qemu config required for adding a GPU device. -func (d *qemu) addGPUDevConfig(cfg *[]cfgSection, bus *qemuBus, gpuConfig []deviceConfig.RunConfigItem) error { +func (d *qemu) addGPUDevConfig(conf *[]cfg.Section, bus *qemuBus, gpuConfig []deviceConfig.RunConfigItem) error { var devName, pciSlotName, vgpu string for _, gpuItem := range gpuConfig { if gpuItem.Key == "devName" { @@ -4797,7 +4798,7 @@ func (d *qemu) addGPUDevConfig(cfg *[]cfgSection, bus *qemuBus, gpuConfig []devi } // Add main GPU device in VGA mode to qemu config. - *cfg = append(*cfg, qemuGPUDevPhysical(&gpuDevPhysicalOpts)...) + *conf = append(*conf, qemuGPUDevPhysical(&gpuDevPhysicalOpts)...) var iommuGroupPath string @@ -4839,7 +4840,7 @@ func (d *qemu) addGPUDevConfig(cfg *[]cfgSection, bus *qemuBus, gpuConfig []devi vgpu: "", } - *cfg = append(*cfg, qemuGPUDevPhysical(&gpuDevPhysicalOpts)...) + *conf = append(*conf, qemuGPUDevPhysical(&gpuDevPhysicalOpts)...) } return nil @@ -4893,7 +4894,7 @@ func (d *qemu) addUSBDeviceConfig(usbDev deviceConfig.USBDeviceItem) (monitorHoo return monHook, nil } -func (d *qemu) addTPMDeviceConfig(cfg *[]cfgSection, tpmConfig []deviceConfig.RunConfigItem, fdFiles *[]*os.File) error { +func (d *qemu) addTPMDeviceConfig(conf *[]cfg.Section, tpmConfig []deviceConfig.RunConfigItem, fdFiles *[]*os.File) error { var devName, socketPath string for _, tpmItem := range tpmConfig { @@ -4915,16 +4916,16 @@ func (d *qemu) addTPMDeviceConfig(cfg *[]cfgSection, tpmConfig []deviceConfig.Ru devName: devName, path: fmt.Sprintf("/proc/self/fd/%d", tpmFD), } - *cfg = append(*cfg, qemuTPM(&tpmOpts)...) + *conf = append(*conf, qemuTPM(&tpmOpts)...) return nil } -func (d *qemu) addVmgenDeviceConfig(cfg *[]cfgSection, guid string) error { +func (d *qemu) addVmgenDeviceConfig(conf *[]cfg.Section, guid string) error { vmgenIDOpts := qemuVmgenIDOpts{ guid: guid, } - *cfg = append(*cfg, qemuVmgen(&vmgenIDOpts)...) + *conf = append(*conf, qemuVmgen(&vmgenIDOpts)...) return nil } diff --git a/internal/server/instance/drivers/driver_qemu_bus.go b/internal/server/instance/drivers/driver_qemu_bus.go index cb5b9fccdd6..394a6b853d0 100644 --- a/internal/server/instance/drivers/driver_qemu_bus.go +++ b/internal/server/instance/drivers/driver_qemu_bus.go @@ -2,6 +2,8 @@ package drivers import ( "fmt" + + "github.com/lxc/incus/v6/internal/server/instance/drivers/cfg" ) const busFunctionGroupNone = "" // Add a non multi-function port. @@ -18,8 +20,8 @@ type qemuBusEntry struct { } type qemuBus struct { - name string // Bus type. - cfg *[]cfgSection // pointer to cfgSection slice. + name string // Bus type. + cfg *[]cfg.Section // pointer to Section slice. portNum int // Next available port/chassis on the bridge. devNum int // Next available device number on the bridge. @@ -148,10 +150,10 @@ func (a *qemuBus) allocateInternal(multiFunctionGroup string, hotplug bool) (str // qemuNewBus instantiates a new qemu bus allocator. Accepts the type name of the bus and the qemu config builder // which it will use to write root port config entries too as ports are allocated. -func qemuNewBus(name string, cfg *[]cfgSection) *qemuBus { +func qemuNewBus(name string, conf *[]cfg.Section) *qemuBus { a := &qemuBus{ name: name, - cfg: cfg, + cfg: conf, portNum: 0, // No PCIe ports are used in the default config. devNum: 1, // Address 0 is used by the DRAM controller. diff --git a/internal/server/instance/drivers/driver_qemu_config_override.go b/internal/server/instance/drivers/driver_qemu_config_override.go index 45e54a92392..85bf6516816 100644 --- a/internal/server/instance/drivers/driver_qemu_config_override.go +++ b/internal/server/instance/drivers/driver_qemu_config_override.go @@ -5,6 +5,8 @@ import ( "sort" "strconv" "strings" + + "github.com/lxc/incus/v6/internal/server/instance/drivers/cfg" ) const pattern = `\s*(?m:(?:\[([^\]]+)\](?:\[(\d+)\])?)|(?:([^=]+)[ \t]*=[ \t]*(?:"([^"]*)"|([^\n]*)))$)` @@ -19,10 +21,10 @@ type rawConfigKey struct { type configMap map[rawConfigKey]string -func sortedConfigKeys(cfgMap configMap) []rawConfigKey { +func sortedConfigKeys(confMap configMap) []rawConfigKey { rv := []rawConfigKey{} - for k := range cfgMap { + for k := range confMap { rv = append(rv, k) } @@ -111,21 +113,21 @@ func parseConfOverride(confOverride string) configMap { return rv } -func updateEntries(entries []cfgEntry, sk rawConfigKey, cfgMap configMap) []cfgEntry { - rv := []cfgEntry{} +func updateEntries(entries []cfg.Entry, sk rawConfigKey, confMap configMap) []cfg.Entry { + rv := []cfg.Entry{} for _, entry := range entries { - newEntry := cfgEntry{ - key: entry.key, - value: entry.value, + newEntry := cfg.Entry{ + Key: entry.Key, + Value: entry.Value, } - ek := rawConfigKey{sk.sectionName, sk.index, entry.key} - val, ok := cfgMap[ek] + ek := rawConfigKey{sk.sectionName, sk.index, entry.Key} + val, ok := confMap[ek] if ok { // override - delete(cfgMap, ek) - newEntry.value = val + delete(confMap, ek) + newEntry.Value = val } rv = append(rv, newEntry) @@ -134,9 +136,9 @@ func updateEntries(entries []cfgEntry, sk rawConfigKey, cfgMap configMap) []cfgE return rv } -func appendEntries(entries []cfgEntry, sk rawConfigKey, cfgMap configMap) []cfgEntry { +func appendEntries(entries []cfg.Entry, sk rawConfigKey, confMap configMap) []cfg.Entry { // sort to have deterministic output in the appended entries - sortedKeys := sortedConfigKeys(cfgMap) + sortedKeys := sortedConfigKeys(confMap) // processed all modifications for the current section, now // handle new entries for _, rawKey := range sortedKeys { @@ -144,61 +146,61 @@ func appendEntries(entries []cfgEntry, sk rawConfigKey, cfgMap configMap) []cfgE continue } - newEntry := cfgEntry{ - key: rawKey.entryKey, - value: cfgMap[rawKey], + newEntry := cfg.Entry{ + Key: rawKey.entryKey, + Value: confMap[rawKey], } entries = append(entries, newEntry) - delete(cfgMap, rawKey) + delete(confMap, rawKey) } return entries } -func updateSections(cfg []cfgSection, cfgMap configMap) []cfgSection { - newCfg := []cfgSection{} +func updateSections(conf []cfg.Section, confMap configMap) []cfg.Section { + newConf := []cfg.Section{} sectionCounts := map[string]uint{} - for _, section := range cfg { - count, ok := sectionCounts[section.name] + for _, section := range conf { + count, ok := sectionCounts[section.Name] if ok { - sectionCounts[section.name] = count + 1 + sectionCounts[section.Name] = count + 1 } else { - sectionCounts[section.name] = 1 + sectionCounts[section.Name] = 1 } - index := sectionCounts[section.name] - 1 - sk := rawConfigKey{section.name, index, ""} + index := sectionCounts[section.Name] - 1 + sk := rawConfigKey{section.Name, index, ""} - val, ok := cfgMap[sk] + val, ok := confMap[sk] if ok { if val == "" { // deleted section - delete(cfgMap, sk) + delete(confMap, sk) continue } } - newSection := cfgSection{ - name: section.name, - comment: section.comment, + newSection := cfg.Section{ + Name: section.Name, + Comment: section.Comment, } - newSection.entries = updateEntries(section.entries, sk, cfgMap) - newSection.entries = appendEntries(newSection.entries, sk, cfgMap) + newSection.Entries = updateEntries(section.Entries, sk, confMap) + newSection.Entries = appendEntries(newSection.Entries, sk, confMap) - newCfg = append(newCfg, newSection) + newConf = append(newConf, newSection) } - return newCfg + return newConf } -func appendSections(newCfg []cfgSection, cfgMap configMap) []cfgSection { - tmp := map[rawConfigKey]cfgSection{} +func appendSections(newConf []cfg.Section, confMap configMap) []cfg.Section { + tmp := map[rawConfigKey]cfg.Section{} // sort to have deterministic output in the appended entries - sortedKeys := sortedConfigKeys(cfgMap) + sortedKeys := sortedConfigKeys(confMap) for _, k := range sortedKeys { if k.entryKey == "" { @@ -210,13 +212,13 @@ func appendSections(newCfg []cfgSection, cfgMap configMap) []cfgSection { sectionKey := rawConfigKey{k.sectionName, k.index, ""} section, found := tmp[sectionKey] if !found { - section = cfgSection{ - name: k.sectionName, + section = cfg.Section{ + Name: k.sectionName, } } - section.entries = append(section.entries, cfgEntry{ - key: k.entryKey, - value: cfgMap[k], + section.Entries = append(section.Entries, cfg.Entry{ + Key: k.entryKey, + Value: confMap[k], }) tmp[sectionKey] = section } @@ -233,27 +235,27 @@ func appendSections(newCfg []cfgSection, cfgMap configMap) []cfgSection { }) for _, rawSection := range rawSections { - newCfg = append(newCfg, tmp[rawSection]) + newConf = append(newConf, tmp[rawSection]) } - return newCfg + return newConf } -func qemuRawCfgOverride(cfg []cfgSection, expandedConfig map[string]string) []cfgSection { +func qemuRawCfgOverride(conf []cfg.Section, expandedConfig map[string]string) []cfg.Section { confOverride, ok := expandedConfig["raw.qemu.conf"] if !ok { - return cfg + return conf } - cfgMap := parseConfOverride(confOverride) + confMap := parseConfOverride(confOverride) - if len(cfgMap) == 0 { - // If no keys are found, we return the cfg unmodified. - return cfg + if len(confMap) == 0 { + // If no keys are found, we return the conf unmodified. + return conf } - newCfg := updateSections(cfg, cfgMap) - newCfg = appendSections(newCfg, cfgMap) + newConf := updateSections(conf, confMap) + newConf = appendSections(newConf, confMap) - return newCfg + return newConf } diff --git a/internal/server/instance/drivers/driver_qemu_templates.go b/internal/server/instance/drivers/driver_qemu_templates.go index 8b2d0c2ff8e..f7593ffd24c 100644 --- a/internal/server/instance/drivers/driver_qemu_templates.go +++ b/internal/server/instance/drivers/driver_qemu_templates.go @@ -4,35 +4,25 @@ import ( "fmt" "strings" + "github.com/lxc/incus/v6/internal/server/instance/drivers/cfg" "github.com/lxc/incus/v6/internal/server/resources" "github.com/lxc/incus/v6/shared/osarch" ) -type cfgEntry struct { - key string - value string -} - -type cfgSection struct { - name string - comment string - entries []cfgEntry -} - -func qemuStringifyCfg(cfg ...cfgSection) *strings.Builder { +func qemuStringifyCfg(conf ...cfg.Section) *strings.Builder { sb := &strings.Builder{} - for _, section := range cfg { - if section.comment != "" { - sb.WriteString(fmt.Sprintf("# %s\n", section.comment)) + for _, section := range conf { + if section.Comment != "" { + sb.WriteString(fmt.Sprintf("# %s\n", section.Comment)) } - sb.WriteString(fmt.Sprintf("[%s]\n", section.name)) + sb.WriteString(fmt.Sprintf("[%s]\n", section.Name)) - for _, entry := range section.entries { - value := entry.value + for _, entry := range section.Entries { + value := entry.Value if value != "" { - sb.WriteString(fmt.Sprintf("%s = \"%s\"\n", entry.key, value)) + sb.WriteString(fmt.Sprintf("%s = \"%s\"\n", entry.Key, value)) } } @@ -63,7 +53,7 @@ type qemuBaseOpts struct { architecture int } -func qemuBase(opts *qemuBaseOpts) []cfgSection { +func qemuBase(opts *qemuBaseOpts) []cfg.Section { machineType := qemuMachineType(opts.architecture) gicVersion := "" capLargeDecr := "" @@ -75,42 +65,42 @@ func qemuBase(opts *qemuBaseOpts) []cfgSection { capLargeDecr = "off" } - sections := []cfgSection{{ - name: "machine", - comment: "Machine", - entries: []cfgEntry{ - {key: "graphics", value: "off"}, - {key: "type", value: machineType}, - {key: "gic-version", value: gicVersion}, - {key: "cap-large-decr", value: capLargeDecr}, - {key: "accel", value: "kvm"}, - {key: "usb", value: "off"}, + sections := []cfg.Section{{ + Name: "machine", + Comment: "Machine", + Entries: []cfg.Entry{ + {Key: "graphics", Value: "off"}, + {Key: "type", Value: machineType}, + {Key: "gic-version", Value: gicVersion}, + {Key: "cap-large-decr", Value: capLargeDecr}, + {Key: "accel", Value: "kvm"}, + {Key: "usb", Value: "off"}, }, }} if opts.architecture == osarch.ARCH_64BIT_INTEL_X86 { - sections = append(sections, []cfgSection{{ - name: "global", - entries: []cfgEntry{ - {key: "driver", value: "ICH9-LPC"}, - {key: "property", value: "disable_s3"}, - {key: "value", value: "1"}, + sections = append(sections, []cfg.Section{{ + Name: "global", + Entries: []cfg.Entry{ + {Key: "driver", Value: "ICH9-LPC"}, + {Key: "property", Value: "disable_s3"}, + {Key: "value", Value: "1"}, }, }, { - name: "global", - entries: []cfgEntry{ - {key: "driver", value: "ICH9-LPC"}, - {key: "property", value: "disable_s4"}, - {key: "value", value: "1"}, + Name: "global", + Entries: []cfg.Entry{ + {Key: "driver", Value: "ICH9-LPC"}, + {Key: "property", Value: "disable_s4"}, + {Key: "value", Value: "1"}, }, }}...) } return append( sections, - cfgSection{ - name: "boot-opts", - entries: []cfgEntry{{key: "strict", value: "on"}}, + cfg.Section{ + Name: "boot-opts", + Entries: []cfg.Entry{{Key: "strict", Value: "on"}}, }) } @@ -118,11 +108,11 @@ type qemuMemoryOpts struct { memSizeMB int64 } -func qemuMemory(opts *qemuMemoryOpts) []cfgSection { - return []cfgSection{{ - name: "memory", - comment: "Memory", - entries: []cfgEntry{{key: "size", value: fmt.Sprintf("%dM", opts.memSizeMB)}}, +func qemuMemory(opts *qemuMemoryOpts) []cfg.Section { + return []cfg.Section{{ + Name: "memory", + Comment: "Memory", + Entries: []cfg.Entry{{Key: "size", Value: fmt.Sprintf("%dM", opts.memSizeMB)}}, }} } @@ -139,21 +129,21 @@ type qemuDevEntriesOpts struct { ccwName string } -func qemuDeviceEntries(opts *qemuDevEntriesOpts) []cfgEntry { - entries := []cfgEntry{} +func qemuDeviceEntries(opts *qemuDevEntriesOpts) []cfg.Entry { + entries := []cfg.Entry{} if opts.dev.busName == "pci" || opts.dev.busName == "pcie" { - entries = append(entries, []cfgEntry{ - {key: "driver", value: opts.pciName}, - {key: "bus", value: opts.dev.devBus}, - {key: "addr", value: opts.dev.devAddr}, + entries = append(entries, []cfg.Entry{ + {Key: "driver", Value: opts.pciName}, + {Key: "bus", Value: opts.dev.devBus}, + {Key: "addr", Value: opts.dev.devAddr}, }...) } else if opts.dev.busName == "ccw" { - entries = append(entries, cfgEntry{key: "driver", value: opts.ccwName}) + entries = append(entries, cfg.Entry{Key: "driver", Value: opts.ccwName}) } if opts.dev.multifunction { - entries = append(entries, cfgEntry{key: "multifunction", value: "on"}) + entries = append(entries, cfg.Entry{Key: "multifunction", Value: "on"}) } return entries @@ -165,74 +155,74 @@ type qemuSerialOpts struct { ringbufSizeBytes int } -func qemuSerial(opts *qemuSerialOpts) []cfgSection { +func qemuSerial(opts *qemuSerialOpts) []cfg.Section { entriesOpts := qemuDevEntriesOpts{ dev: opts.dev, pciName: "virtio-serial-pci", ccwName: "virtio-serial-ccw", } - return []cfgSection{{ - name: `device "dev-qemu_serial"`, - comment: "Virtual serial bus", - entries: qemuDeviceEntries(&entriesOpts), + return []cfg.Section{{ + Name: `device "dev-qemu_serial"`, + Comment: "Virtual serial bus", + Entries: qemuDeviceEntries(&entriesOpts), }, { // Ring buffer used by the incus agent to report (write) its status to. Incus server will read // its content via QMP using "ringbuf-read" command. - name: fmt.Sprintf(`chardev "%s"`, opts.charDevName), - comment: "Serial identifier", - entries: []cfgEntry{ - {key: "backend", value: "ringbuf"}, - {key: "size", value: fmt.Sprintf("%dB", opts.ringbufSizeBytes)}}, + Name: fmt.Sprintf(`chardev "%s"`, opts.charDevName), + Comment: "Serial identifier", + Entries: []cfg.Entry{ + {Key: "backend", Value: "ringbuf"}, + {Key: "size", Value: fmt.Sprintf("%dB", opts.ringbufSizeBytes)}}, }, { // QEMU serial device connected to the above ring buffer. - name: `device "qemu_serial"`, - entries: []cfgEntry{ - {key: "driver", value: "virtserialport"}, - {key: "name", value: "org.linuxcontainers.incus"}, - {key: "chardev", value: opts.charDevName}, - {key: "bus", value: "dev-qemu_serial.0"}, + Name: `device "qemu_serial"`, + Entries: []cfg.Entry{ + {Key: "driver", Value: "virtserialport"}, + {Key: "name", Value: "org.linuxcontainers.incus"}, + {Key: "chardev", Value: opts.charDevName}, + {Key: "bus", Value: "dev-qemu_serial.0"}, }, }, { // Legacy QEMU serial device, not connected to any ring buffer. Its purpose is to // create a symlink in /dev/virtio-ports/, triggering a udev rule to start incus-agent. // This is necessary for backward compatibility with virtual machines lacking the // updated incus-agent-loader package, which includes updated udev rules and a systemd unit. - name: `device "qemu_serial_legacy"`, - entries: []cfgEntry{ - {key: "driver", value: "virtserialport"}, - {key: "name", value: "org.linuxcontainers.lxd"}, - {key: "bus", value: "dev-qemu_serial.0"}, + Name: `device "qemu_serial_legacy"`, + Entries: []cfg.Entry{ + {Key: "driver", Value: "virtserialport"}, + {Key: "name", Value: "org.linuxcontainers.lxd"}, + {Key: "bus", Value: "dev-qemu_serial.0"}, }, }, { - name: `chardev "qemu_spice-chardev"`, - comment: "Spice agent", - entries: []cfgEntry{ - {key: "backend", value: "spicevmc"}, - {key: "name", value: "vdagent"}, + Name: `chardev "qemu_spice-chardev"`, + Comment: "Spice agent", + Entries: []cfg.Entry{ + {Key: "backend", Value: "spicevmc"}, + {Key: "name", Value: "vdagent"}, }, }, { - name: `device "qemu_spice"`, - entries: []cfgEntry{ - {key: "driver", value: "virtserialport"}, - {key: "name", value: "com.redhat.spice.0"}, - {key: "chardev", value: "qemu_spice-chardev"}, - {key: "bus", value: "dev-qemu_serial.0"}, + Name: `device "qemu_spice"`, + Entries: []cfg.Entry{ + {Key: "driver", Value: "virtserialport"}, + {Key: "name", Value: "com.redhat.spice.0"}, + {Key: "chardev", Value: "qemu_spice-chardev"}, + {Key: "bus", Value: "dev-qemu_serial.0"}, }, }, { - name: `chardev "qemu_spicedir-chardev"`, - comment: "Spice folder", - entries: []cfgEntry{ - {key: "backend", value: "spiceport"}, - {key: "name", value: "org.spice-space.webdav.0"}, + Name: `chardev "qemu_spicedir-chardev"`, + Comment: "Spice folder", + Entries: []cfg.Entry{ + {Key: "backend", Value: "spiceport"}, + {Key: "name", Value: "org.spice-space.webdav.0"}, }, }, { - name: `device "qemu_spicedir"`, - entries: []cfgEntry{ - {key: "driver", value: "virtserialport"}, - {key: "name", value: "org.spice-space.webdav.0"}, - {key: "chardev", value: "qemu_spicedir-chardev"}, - {key: "bus", value: "dev-qemu_serial.0"}, + Name: `device "qemu_spicedir"`, + Entries: []cfg.Entry{ + {Key: "driver", Value: "virtserialport"}, + {Key: "name", Value: "org.spice-space.webdav.0"}, + {Key: "chardev", Value: "qemu_spicedir-chardev"}, + {Key: "bus", Value: "dev-qemu_serial.0"}, }, }} } @@ -244,70 +234,70 @@ type qemuPCIeOpts struct { multifunction bool } -func qemuPCIe(opts *qemuPCIeOpts) []cfgSection { - entries := []cfgEntry{ - {key: "driver", value: "pcie-root-port"}, - {key: "bus", value: "pcie.0"}, - {key: "addr", value: opts.devAddr}, - {key: "chassis", value: fmt.Sprintf("%d", opts.index)}, +func qemuPCIe(opts *qemuPCIeOpts) []cfg.Section { + entries := []cfg.Entry{ + {Key: "driver", Value: "pcie-root-port"}, + {Key: "bus", Value: "pcie.0"}, + {Key: "addr", Value: opts.devAddr}, + {Key: "chassis", Value: fmt.Sprintf("%d", opts.index)}, } if opts.multifunction { - entries = append(entries, cfgEntry{key: "multifunction", value: "on"}) + entries = append(entries, cfg.Entry{Key: "multifunction", Value: "on"}) } - return []cfgSection{{ - name: fmt.Sprintf(`device "%s"`, opts.portName), - entries: entries, + return []cfg.Section{{ + Name: fmt.Sprintf(`device "%s"`, opts.portName), + Entries: entries, }} } -func qemuSCSI(opts *qemuDevOpts) []cfgSection { +func qemuSCSI(opts *qemuDevOpts) []cfg.Section { entriesOpts := qemuDevEntriesOpts{ dev: *opts, pciName: "virtio-scsi-pci", ccwName: "virtio-scsi-ccw", } - return []cfgSection{{ - name: `device "qemu_scsi"`, - comment: "SCSI controller", - entries: qemuDeviceEntries(&entriesOpts), + return []cfg.Section{{ + Name: `device "qemu_scsi"`, + Comment: "SCSI controller", + Entries: qemuDeviceEntries(&entriesOpts), }} } -func qemuBalloon(opts *qemuDevOpts) []cfgSection { +func qemuBalloon(opts *qemuDevOpts) []cfg.Section { entriesOpts := qemuDevEntriesOpts{ dev: *opts, pciName: "virtio-balloon-pci", ccwName: "virtio-balloon-ccw", } - return []cfgSection{{ - name: `device "qemu_balloon"`, - comment: "Balloon driver", - entries: qemuDeviceEntries(&entriesOpts), + return []cfg.Section{{ + Name: `device "qemu_balloon"`, + Comment: "Balloon driver", + Entries: qemuDeviceEntries(&entriesOpts), }} } -func qemuRNG(opts *qemuDevOpts) []cfgSection { +func qemuRNG(opts *qemuDevOpts) []cfg.Section { entriesOpts := qemuDevEntriesOpts{ dev: *opts, pciName: "virtio-rng-pci", ccwName: "virtio-rng-ccw", } - return []cfgSection{{ - name: `object "qemu_rng"`, - comment: "Random number generator", - entries: []cfgEntry{ - {key: "qom-type", value: "rng-random"}, - {key: "filename", value: "/dev/urandom"}, + return []cfg.Section{{ + Name: `object "qemu_rng"`, + Comment: "Random number generator", + Entries: []cfg.Entry{ + {Key: "qom-type", Value: "rng-random"}, + {Key: "filename", Value: "/dev/urandom"}, }, }, { - name: `device "dev-qemu_rng"`, - entries: append(qemuDeviceEntries(&entriesOpts), - cfgEntry{key: "rng", value: "qemu_rng"}), + Name: `device "dev-qemu_rng"`, + Entries: append(qemuDeviceEntries(&entriesOpts), + cfg.Entry{Key: "rng", Value: "qemu_rng"}), }} } @@ -319,22 +309,22 @@ type qemuSevOpts struct { sessionDataFD string } -func qemuSEV(opts *qemuSevOpts) []cfgSection { - entries := []cfgEntry{ - {key: "qom-type", value: "sev-guest"}, - {key: "cbitpos", value: fmt.Sprintf("%d", opts.cbitpos)}, - {key: "reduced-phys-bits", value: fmt.Sprintf("%d", opts.reducedPhysBits)}, - {key: "policy", value: opts.policy}, +func qemuSEV(opts *qemuSevOpts) []cfg.Section { + entries := []cfg.Entry{ + {Key: "qom-type", Value: "sev-guest"}, + {Key: "cbitpos", Value: fmt.Sprintf("%d", opts.cbitpos)}, + {Key: "reduced-phys-bits", Value: fmt.Sprintf("%d", opts.reducedPhysBits)}, + {Key: "policy", Value: opts.policy}, } if opts.dhCertFD != "" && opts.sessionDataFD != "" { - entries = append(entries, cfgEntry{key: "dh-cert-file", value: opts.dhCertFD}, cfgEntry{key: "session-file", value: opts.sessionDataFD}) + entries = append(entries, cfg.Entry{Key: "dh-cert-file", Value: opts.dhCertFD}, cfg.Entry{Key: "session-file", Value: opts.sessionDataFD}) } - return []cfgSection{{ - name: `object "sev0"`, - comment: "Secure Encrypted Virtualization", - entries: entries, + return []cfg.Section{{ + Name: `object "sev0"`, + Comment: "Secure Encrypted Virtualization", + Entries: entries, }} } @@ -344,19 +334,19 @@ type qemuVsockOpts struct { vsockID uint32 } -func qemuVsock(opts *qemuVsockOpts) []cfgSection { +func qemuVsock(opts *qemuVsockOpts) []cfg.Section { entriesOpts := qemuDevEntriesOpts{ dev: opts.dev, pciName: "vhost-vsock-pci", ccwName: "vhost-vsock-ccw", } - return []cfgSection{{ - name: `device "qemu_vsock"`, - comment: "Vsock", - entries: append(qemuDeviceEntries(&entriesOpts), - cfgEntry{key: "guest-cid", value: fmt.Sprintf("%d", opts.vsockID)}, - cfgEntry{key: "vhostfd", value: fmt.Sprintf("%d", opts.vsockFD)}), + return []cfg.Section{{ + Name: `device "qemu_vsock"`, + Comment: "Vsock", + Entries: append(qemuDeviceEntries(&entriesOpts), + cfg.Entry{Key: "guest-cid", Value: fmt.Sprintf("%d", opts.vsockID)}, + cfg.Entry{Key: "vhostfd", Value: fmt.Sprintf("%d", opts.vsockFD)}), }} } @@ -365,7 +355,7 @@ type qemuGpuOpts struct { architecture int } -func qemuGPU(opts *qemuGpuOpts) []cfgSection { +func qemuGPU(opts *qemuGpuOpts) []cfg.Section { var pciName string if opts.architecture == osarch.ARCH_64BIT_INTEL_X86 { @@ -380,38 +370,38 @@ func qemuGPU(opts *qemuGpuOpts) []cfgSection { ccwName: "virtio-gpu-ccw", } - return []cfgSection{{ - name: `device "qemu_gpu"`, - comment: "GPU", - entries: qemuDeviceEntries(&entriesOpts), + return []cfg.Section{{ + Name: `device "qemu_gpu"`, + Comment: "GPU", + Entries: qemuDeviceEntries(&entriesOpts), }} } -func qemuKeyboard(opts *qemuDevOpts) []cfgSection { +func qemuKeyboard(opts *qemuDevOpts) []cfg.Section { entriesOpts := qemuDevEntriesOpts{ dev: *opts, pciName: "virtio-keyboard-pci", ccwName: "virtio-keyboard-ccw", } - return []cfgSection{{ - name: `device "qemu_keyboard"`, - comment: "Input", - entries: qemuDeviceEntries(&entriesOpts), + return []cfg.Section{{ + Name: `device "qemu_keyboard"`, + Comment: "Input", + Entries: qemuDeviceEntries(&entriesOpts), }} } -func qemuTablet(opts *qemuDevOpts) []cfgSection { +func qemuTablet(opts *qemuDevOpts) []cfg.Section { entriesOpts := qemuDevEntriesOpts{ dev: *opts, pciName: "virtio-tablet-pci", ccwName: "virtio-tablet-ccw", } - return []cfgSection{{ - name: `device "qemu_tablet"`, - comment: "Input", - entries: qemuDeviceEntries(&entriesOpts), + return []cfg.Section{{ + Name: `device "qemu_tablet"`, + Comment: "Input", + Entries: qemuDeviceEntries(&entriesOpts), }} } @@ -438,47 +428,47 @@ type qemuCPUOpts struct { qemuMemObjectFormat string } -func qemuCPUNumaHostNode(opts *qemuCPUOpts, index int) []cfgSection { - entries := []cfgEntry{} +func qemuCPUNumaHostNode(opts *qemuCPUOpts, index int) []cfg.Section { + entries := []cfg.Entry{} if opts.hugepages != "" { - entries = append(entries, []cfgEntry{ - {key: "qom-type", value: "memory-backend-file"}, - {key: "mem-path", value: opts.hugepages}, - {key: "prealloc", value: "on"}, - {key: "discard-data", value: "on"}, + entries = append(entries, []cfg.Entry{ + {Key: "qom-type", Value: "memory-backend-file"}, + {Key: "mem-path", Value: opts.hugepages}, + {Key: "prealloc", Value: "on"}, + {Key: "discard-data", Value: "on"}, }...) } else { - entries = append(entries, cfgEntry{key: "qom-type", value: "memory-backend-memfd"}) + entries = append(entries, cfg.Entry{Key: "qom-type", Value: "memory-backend-memfd"}) } - entries = append(entries, cfgEntry{key: "size", value: fmt.Sprintf("%dM", opts.memory)}) + entries = append(entries, cfg.Entry{Key: "size", Value: fmt.Sprintf("%dM", opts.memory)}) - return []cfgSection{{ - name: fmt.Sprintf("object \"mem%d\"", index), - entries: entries, + return []cfg.Section{{ + Name: fmt.Sprintf("object \"mem%d\"", index), + Entries: entries, }, { - name: "numa", - entries: []cfgEntry{ - {key: "type", value: "node"}, - {key: "nodeid", value: fmt.Sprintf("%d", index)}, - {key: "memdev", value: fmt.Sprintf("mem%d", index)}, + Name: "numa", + Entries: []cfg.Entry{ + {Key: "type", Value: "node"}, + {Key: "nodeid", Value: fmt.Sprintf("%d", index)}, + {Key: "memdev", Value: fmt.Sprintf("mem%d", index)}, }, }} } -func qemuCPU(opts *qemuCPUOpts, pinning bool) []cfgSection { - entries := []cfgEntry{ - {key: "cpus", value: fmt.Sprintf("%d", opts.cpuCount)}, +func qemuCPU(opts *qemuCPUOpts, pinning bool) []cfg.Section { + entries := []cfg.Entry{ + {Key: "cpus", Value: fmt.Sprintf("%d", opts.cpuCount)}, } if pinning { - entries = append(entries, cfgEntry{ - key: "sockets", value: fmt.Sprintf("%d", opts.cpuSockets), - }, cfgEntry{ - key: "cores", value: fmt.Sprintf("%d", opts.cpuCores), - }, cfgEntry{ - key: "threads", value: fmt.Sprintf("%d", opts.cpuThreads), + entries = append(entries, cfg.Entry{ + Key: "sockets", Value: fmt.Sprintf("%d", opts.cpuSockets), + }, cfg.Entry{ + Key: "cores", Value: fmt.Sprintf("%d", opts.cpuCores), + }, cfg.Entry{ + Key: "threads", Value: fmt.Sprintf("%d", opts.cpuThreads), }) } else { cpu, err := resources.GetCPU() @@ -496,33 +486,33 @@ func qemuCPU(opts *qemuCPUOpts, pinning bool) []cfgSection { max = opts.cpuCount } - entries = append(entries, cfgEntry{ - key: "maxcpus", value: fmt.Sprintf("%d", max), + entries = append(entries, cfg.Entry{ + Key: "maxcpus", Value: fmt.Sprintf("%d", max), }) } - sections := []cfgSection{{ - name: "smp-opts", - comment: "CPU", - entries: entries, + sections := []cfg.Section{{ + Name: "smp-opts", + Comment: "CPU", + Entries: entries, }} if opts.architecture != "x86_64" { return sections } - share := cfgEntry{key: "share", value: "on"} + share := cfg.Entry{Key: "share", Value: "on"} if len(opts.cpuNumaHostNodes) == 0 { // Add one mem and one numa sections with index 0. numaHostNode := qemuCPUNumaHostNode(opts, 0) // Unconditionally append "share = "on" to the [object "mem0"] section - numaHostNode[0].entries = append(numaHostNode[0].entries, share) + numaHostNode[0].Entries = append(numaHostNode[0].Entries, share) // If NUMA memory restrictions are set, apply them. if len(opts.memoryHostNodes) > 0 { - extraMemEntries := []cfgEntry{{key: "policy", value: "bind"}} + extraMemEntries := []cfg.Entry{{Key: "policy", Value: "bind"}} for index, element := range opts.memoryHostNodes { var hostNodesKey string @@ -532,12 +522,12 @@ func qemuCPU(opts *qemuCPUOpts, pinning bool) []cfgSection { hostNodesKey = "host-nodes" } - hostNode := cfgEntry{key: hostNodesKey, value: fmt.Sprintf("%d", element)} + hostNode := cfg.Entry{Key: hostNodesKey, Value: fmt.Sprintf("%d", element)} extraMemEntries = append(extraMemEntries, hostNode) } // Append the extra entries to the [object "mem{{idx}}"] section. - numaHostNode[0].entries = append(numaHostNode[0].entries, extraMemEntries...) + numaHostNode[0].Entries = append(numaHostNode[0].Entries, extraMemEntries...) } return append(sections, numaHostNode...) @@ -546,7 +536,7 @@ func qemuCPU(opts *qemuCPUOpts, pinning bool) []cfgSection { for index, element := range opts.cpuNumaHostNodes { numaHostNode := qemuCPUNumaHostNode(opts, index) - extraMemEntries := []cfgEntry{{key: "policy", value: "bind"}} + extraMemEntries := []cfg.Entry{{Key: "policy", Value: "bind"}} if opts.hugepages != "" { // append share = "on" only if hugepages is set @@ -560,22 +550,22 @@ func qemuCPU(opts *qemuCPUOpts, pinning bool) []cfgSection { hostNodesKey = "host-nodes" } - hostNode := cfgEntry{key: hostNodesKey, value: fmt.Sprintf("%d", element)} + hostNode := cfg.Entry{Key: hostNodesKey, Value: fmt.Sprintf("%d", element)} extraMemEntries = append(extraMemEntries, hostNode) // append the extra entries to the [object "mem{{idx}}"] section - numaHostNode[0].entries = append(numaHostNode[0].entries, extraMemEntries...) + numaHostNode[0].Entries = append(numaHostNode[0].Entries, extraMemEntries...) sections = append(sections, numaHostNode...) } for _, numa := range opts.cpuNumaMapping { - sections = append(sections, cfgSection{ - name: "numa", - entries: []cfgEntry{ - {key: "type", value: "cpu"}, - {key: "node-id", value: fmt.Sprintf("%d", numa.node)}, - {key: "socket-id", value: fmt.Sprintf("%d", numa.socket)}, - {key: "core-id", value: fmt.Sprintf("%d", numa.core)}, - {key: "thread-id", value: fmt.Sprintf("%d", numa.thread)}, + sections = append(sections, cfg.Section{ + Name: "numa", + Entries: []cfg.Entry{ + {Key: "type", Value: "cpu"}, + {Key: "node-id", Value: fmt.Sprintf("%d", numa.node)}, + {Key: "socket-id", Value: fmt.Sprintf("%d", numa.socket)}, + {Key: "core-id", Value: fmt.Sprintf("%d", numa.core)}, + {Key: "thread-id", Value: fmt.Sprintf("%d", numa.thread)}, }, }) } @@ -587,32 +577,32 @@ type qemuControlSocketOpts struct { path string } -func qemuControlSocket(opts *qemuControlSocketOpts) []cfgSection { - return []cfgSection{{ - name: `chardev "monitor"`, - comment: "Qemu control", - entries: []cfgEntry{ - {key: "backend", value: "socket"}, - {key: "path", value: opts.path}, - {key: "server", value: "on"}, - {key: "wait", value: "off"}, +func qemuControlSocket(opts *qemuControlSocketOpts) []cfg.Section { + return []cfg.Section{{ + Name: `chardev "monitor"`, + Comment: "Qemu control", + Entries: []cfg.Entry{ + {Key: "backend", Value: "socket"}, + {Key: "path", Value: opts.path}, + {Key: "server", Value: "on"}, + {Key: "wait", Value: "off"}, }, }, { - name: "mon", - entries: []cfgEntry{ - {key: "chardev", value: "monitor"}, - {key: "mode", value: "control"}, + Name: "mon", + Entries: []cfg.Entry{ + {Key: "chardev", Value: "monitor"}, + {Key: "mode", Value: "control"}, }, }} } -func qemuConsole() []cfgSection { - return []cfgSection{{ - name: `chardev "console"`, - comment: "Console", - entries: []cfgEntry{ - {key: "backend", value: "ringbuf"}, - {key: "size", value: "1048576"}, +func qemuConsole() []cfg.Section { + return []cfg.Section{{ + Name: `chardev "console"`, + Comment: "Console", + Entries: []cfg.Entry{ + {Key: "backend", Value: "ringbuf"}, + {Key: "size", Value: "1048576"}, }, }} } @@ -622,25 +612,25 @@ type qemuDriveFirmwareOpts struct { nvramPath string } -func qemuDriveFirmware(opts *qemuDriveFirmwareOpts) []cfgSection { - return []cfgSection{{ - name: "drive", - comment: "Firmware (read only)", - entries: []cfgEntry{ - {key: "file", value: opts.roPath}, - {key: "if", value: "pflash"}, - {key: "format", value: "raw"}, - {key: "unit", value: "0"}, - {key: "readonly", value: "on"}, +func qemuDriveFirmware(opts *qemuDriveFirmwareOpts) []cfg.Section { + return []cfg.Section{{ + Name: "drive", + Comment: "Firmware (read only)", + Entries: []cfg.Entry{ + {Key: "file", Value: opts.roPath}, + {Key: "if", Value: "pflash"}, + {Key: "format", Value: "raw"}, + {Key: "unit", Value: "0"}, + {Key: "readonly", Value: "on"}, }, }, { - name: "drive", - comment: "Firmware settings (writable)", - entries: []cfgEntry{ - {key: "file", value: opts.nvramPath}, - {key: "if", value: "pflash"}, - {key: "format", value: "raw"}, - {key: "unit", value: "1"}, + Name: "drive", + Comment: "Firmware settings (writable)", + Entries: []cfg.Entry{ + {Key: "file", Value: opts.nvramPath}, + {Key: "if", Value: "pflash"}, + {Key: "format", Value: "raw"}, + {Key: "unit", Value: "1"}, }, }} } @@ -659,9 +649,9 @@ type qemuHostDriveOpts struct { protocol string } -func qemuHostDrive(opts *qemuHostDriveOpts) []cfgSection { - var extraDeviceEntries []cfgEntry - var driveSection cfgSection +func qemuHostDrive(opts *qemuHostDriveOpts) []cfg.Section { + var extraDeviceEntries []cfg.Entry + var driveSection cfg.Section deviceOpts := qemuDevEntriesOpts{dev: opts.dev} if opts.protocol == "9p" { @@ -672,51 +662,51 @@ func qemuHostDrive(opts *qemuHostDriveOpts) []cfgSection { readonly = "off" } - driveSection = cfgSection{ - name: fmt.Sprintf(`fsdev "%s"`, opts.name), - comment: opts.comment, - entries: []cfgEntry{ - {key: "fsdriver", value: opts.fsdriver}, - {key: "sock_fd", value: opts.sockFd}, - {key: "security_model", value: opts.securityModel}, - {key: "readonly", value: readonly}, - {key: "path", value: opts.path}, + driveSection = cfg.Section{ + Name: fmt.Sprintf(`fsdev "%s"`, opts.name), + Comment: opts.comment, + Entries: []cfg.Entry{ + {Key: "fsdriver", Value: opts.fsdriver}, + {Key: "sock_fd", Value: opts.sockFd}, + {Key: "security_model", Value: opts.securityModel}, + {Key: "readonly", Value: readonly}, + {Key: "path", Value: opts.path}, }, } deviceOpts.pciName = "virtio-9p-pci" deviceOpts.ccwName = "virtio-9p-ccw" - extraDeviceEntries = []cfgEntry{ - {key: "mount_tag", value: opts.mountTag}, - {key: "fsdev", value: opts.name}, + extraDeviceEntries = []cfg.Entry{ + {Key: "mount_tag", Value: opts.mountTag}, + {Key: "fsdev", Value: opts.name}, } } else if opts.protocol == "virtio-fs" { - driveSection = cfgSection{ - name: fmt.Sprintf(`chardev "%s"`, opts.name), - comment: opts.comment, - entries: []cfgEntry{ - {key: "backend", value: "socket"}, - {key: "path", value: opts.path}, + driveSection = cfg.Section{ + Name: fmt.Sprintf(`chardev "%s"`, opts.name), + Comment: opts.comment, + Entries: []cfg.Entry{ + {Key: "backend", Value: "socket"}, + {Key: "path", Value: opts.path}, }, } deviceOpts.pciName = "vhost-user-fs-pci" deviceOpts.ccwName = "vhost-user-fs-ccw" - extraDeviceEntries = []cfgEntry{ - {key: "tag", value: opts.mountTag}, - {key: "chardev", value: opts.name}, + extraDeviceEntries = []cfg.Entry{ + {Key: "tag", Value: opts.mountTag}, + {Key: "chardev", Value: opts.name}, } } else { - return []cfgSection{} + return []cfg.Section{} } - return []cfgSection{ + return []cfg.Section{ driveSection, { - name: fmt.Sprintf(`device "dev-%s%s-%s"`, opts.name, opts.nameSuffix, opts.protocol), - entries: append(qemuDeviceEntries(&deviceOpts), extraDeviceEntries...), + Name: fmt.Sprintf(`device "dev-%s%s-%s"`, opts.name, opts.nameSuffix, opts.protocol), + Entries: append(qemuDeviceEntries(&deviceOpts), extraDeviceEntries...), }, } } @@ -728,7 +718,7 @@ type qemuDriveConfigOpts struct { path string } -func qemuDriveConfig(opts *qemuDriveConfigOpts) []cfgSection { +func qemuDriveConfig(opts *qemuDriveConfigOpts) []cfg.Section { return qemuHostDrive(&qemuHostDriveOpts{ dev: opts.dev, // Devices use "qemu_" prefix indicating that this is a internally named device. @@ -754,7 +744,7 @@ type qemuDriveDirOpts struct { readonly bool } -func qemuDriveDir(opts *qemuDriveDirOpts) []cfgSection { +func qemuDriveDir(opts *qemuDriveDirOpts) []cfg.Section { return qemuHostDrive(&qemuHostDriveOpts{ dev: opts.dev, name: fmt.Sprintf("incus_%s", opts.devName), @@ -774,21 +764,21 @@ type qemuPCIPhysicalOpts struct { pciSlotName string } -func qemuPCIPhysical(opts *qemuPCIPhysicalOpts) []cfgSection { +func qemuPCIPhysical(opts *qemuPCIPhysicalOpts) []cfg.Section { deviceOpts := qemuDevEntriesOpts{ dev: opts.dev, pciName: "vfio-pci", ccwName: "vfio-ccw", } - entries := append(qemuDeviceEntries(&deviceOpts), []cfgEntry{ - {key: "host", value: opts.pciSlotName}, + entries := append(qemuDeviceEntries(&deviceOpts), []cfg.Entry{ + {Key: "host", Value: opts.pciSlotName}, }...) - return []cfgSection{{ - name: fmt.Sprintf(`device "%s%s"`, qemuDeviceIDPrefix, opts.devName), - comment: fmt.Sprintf(`PCI card ("%s" device)`, opts.devName), - entries: entries, + return []cfg.Section{{ + Name: fmt.Sprintf(`device "%s%s"`, qemuDeviceIDPrefix, opts.devName), + Comment: fmt.Sprintf(`PCI card ("%s" device)`, opts.devName), + Entries: entries, }} } @@ -800,7 +790,7 @@ type qemuGPUDevPhysicalOpts struct { vga bool } -func qemuGPUDevPhysical(opts *qemuGPUDevPhysicalOpts) []cfgSection { +func qemuGPUDevPhysical(opts *qemuGPUDevPhysicalOpts) []cfg.Section { deviceOpts := qemuDevEntriesOpts{ dev: opts.dev, pciName: "vfio-pci", @@ -811,19 +801,19 @@ func qemuGPUDevPhysical(opts *qemuGPUDevPhysicalOpts) []cfgSection { if opts.vgpu != "" { sysfsdev := fmt.Sprintf("/sys/bus/mdev/devices/%s", opts.vgpu) - entries = append(entries, cfgEntry{key: "sysfsdev", value: sysfsdev}) + entries = append(entries, cfg.Entry{Key: "sysfsdev", Value: sysfsdev}) } else { - entries = append(entries, cfgEntry{key: "host", value: opts.pciSlotName}) + entries = append(entries, cfg.Entry{Key: "host", Value: opts.pciSlotName}) } if opts.vga { - entries = append(entries, cfgEntry{key: "x-vga", value: "on"}) + entries = append(entries, cfg.Entry{Key: "x-vga", Value: "on"}) } - return []cfgSection{{ - name: fmt.Sprintf(`device "%s%s"`, qemuDeviceIDPrefix, opts.devName), - comment: fmt.Sprintf(`GPU card ("%s" device)`, opts.devName), - entries: entries, + return []cfg.Section{{ + Name: fmt.Sprintf(`device "%s%s"`, qemuDeviceIDPrefix, opts.devName), + Comment: fmt.Sprintf(`GPU card ("%s" device)`, opts.devName), + Entries: entries, }} } @@ -834,7 +824,7 @@ type qemuUSBOpts struct { ports int } -func qemuUSB(opts *qemuUSBOpts) []cfgSection { +func qemuUSB(opts *qemuUSBOpts) []cfg.Section { deviceOpts := qemuDevEntriesOpts{ dev: qemuDevOpts{ busName: "pci", @@ -845,28 +835,28 @@ func qemuUSB(opts *qemuUSBOpts) []cfgSection { pciName: "qemu-xhci", } - sections := []cfgSection{{ - name: `device "qemu_usb"`, - comment: "USB controller", - entries: append(qemuDeviceEntries(&deviceOpts), []cfgEntry{ - {key: "p2", value: fmt.Sprintf("%d", opts.ports)}, - {key: "p3", value: fmt.Sprintf("%d", opts.ports)}, + sections := []cfg.Section{{ + Name: `device "qemu_usb"`, + Comment: "USB controller", + Entries: append(qemuDeviceEntries(&deviceOpts), []cfg.Entry{ + {Key: "p2", Value: fmt.Sprintf("%d", opts.ports)}, + {Key: "p3", Value: fmt.Sprintf("%d", opts.ports)}, }...), }} for i := 1; i <= 3; i++ { chardev := fmt.Sprintf("qemu_spice-usb-chardev%d", i) - sections = append(sections, []cfgSection{{ - name: fmt.Sprintf(`chardev "%s"`, chardev), - entries: []cfgEntry{ - {key: "backend", value: "spicevmc"}, - {key: "name", value: "usbredir"}, + sections = append(sections, []cfg.Section{{ + Name: fmt.Sprintf(`chardev "%s"`, chardev), + Entries: []cfg.Entry{ + {Key: "backend", Value: "spicevmc"}, + {Key: "name", Value: "usbredir"}, }, }, { - name: fmt.Sprintf(`device "qemu_spice-usb%d"`, i), - entries: []cfgEntry{ - {key: "driver", value: "usb-redir"}, - {key: "chardev", value: chardev}, + Name: fmt.Sprintf(`device "qemu_spice-usb%d"`, i), + Entries: []cfg.Entry{ + {Key: "driver", Value: "usb-redir"}, + {Key: "chardev", Value: chardev}, }, }}...) } @@ -879,27 +869,27 @@ type qemuTPMOpts struct { path string } -func qemuTPM(opts *qemuTPMOpts) []cfgSection { +func qemuTPM(opts *qemuTPMOpts) []cfg.Section { chardev := fmt.Sprintf("qemu_tpm-chardev_%s", opts.devName) tpmdev := fmt.Sprintf("qemu_tpm-tpmdev_%s", opts.devName) - return []cfgSection{{ - name: fmt.Sprintf(`chardev "%s"`, chardev), - entries: []cfgEntry{ - {key: "backend", value: "socket"}, - {key: "path", value: opts.path}, + return []cfg.Section{{ + Name: fmt.Sprintf(`chardev "%s"`, chardev), + Entries: []cfg.Entry{ + {Key: "backend", Value: "socket"}, + {Key: "path", Value: opts.path}, }, }, { - name: fmt.Sprintf(`tpmdev "%s"`, tpmdev), - entries: []cfgEntry{ - {key: "type", value: "emulator"}, - {key: "chardev", value: chardev}, + Name: fmt.Sprintf(`tpmdev "%s"`, tpmdev), + Entries: []cfg.Entry{ + {Key: "type", Value: "emulator"}, + {Key: "chardev", Value: chardev}, }, }, { - name: fmt.Sprintf(`device "%s%s"`, qemuDeviceIDPrefix, opts.devName), - entries: []cfgEntry{ - {key: "driver", value: "tpm-crb"}, - {key: "tpmdev", value: tpmdev}, + Name: fmt.Sprintf(`device "%s%s"`, qemuDeviceIDPrefix, opts.devName), + Entries: []cfg.Entry{ + {Key: "driver", Value: "tpm-crb"}, + {Key: "tpmdev", Value: tpmdev}, }, }} } @@ -908,13 +898,13 @@ type qemuVmgenIDOpts struct { guid string } -func qemuVmgen(opts *qemuVmgenIDOpts) []cfgSection { - return []cfgSection{{ - name: `device "vmgenid0"`, - comment: "VM Generation ID", - entries: []cfgEntry{ - {key: "driver", value: "vmgenid"}, - {key: "guid", value: opts.guid}, +func qemuVmgen(opts *qemuVmgenIDOpts) []cfg.Section { + return []cfg.Section{{ + Name: `device "vmgenid0"`, + Comment: "VM Generation ID", + Entries: []cfg.Entry{ + {Key: "driver", Value: "vmgenid"}, + {Key: "guid", Value: opts.guid}, }, }} } From 34626c5f0ec682fae0919970877f34ca0ce2bd3e Mon Sep 17 00:00:00 2001 From: Benjamin Somers Date: Thu, 16 Jan 2025 11:59:59 +0000 Subject: [PATCH 06/12] incusd/scriptlet/qemu: Fix Starlark function name Signed-off-by: Benjamin Somers --- internal/server/scriptlet/qemu.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/server/scriptlet/qemu.go b/internal/server/scriptlet/qemu.go index 968a8f896fc..54966d4b11e 100644 --- a/internal/server/scriptlet/qemu.go +++ b/internal/server/scriptlet/qemu.go @@ -114,7 +114,7 @@ func QEMURun(l logger.Logger, instance *api.Instance, m *qmp.Monitor, stage stri makeQOM := func(funName string) *starlark.Builtin { fun := func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { frame := thread.CallFrame(1) - errPrefix := fmt.Sprintf("%s (%d:%d):", strings.ReplaceAll(funName, "-", "_"), frame.Pos.Line, frame.Pos.Col) + errPrefix := fmt.Sprintf("%s (%d:%d):", b.Name(), frame.Pos.Line, frame.Pos.Col) argsLen := args.Len() if argsLen != 0 { @@ -129,7 +129,7 @@ func QEMURun(l logger.Logger, instance *api.Instance, m *qmp.Monitor, stage stri return rv, nil } - return starlark.NewBuiltin(funName, fun) + return starlark.NewBuiltin(strings.ReplaceAll(funName, "-", "_"), fun) } // Remember to match the entries in scriptletLoad.QEMUCompile() with this list so Starlark can From 12f0e8accdf043da06df3d321e293178eb4f3d25 Mon Sep 17 00:00:00 2001 From: Benjamin Somers Date: Thu, 16 Jan 2025 13:30:54 +0000 Subject: [PATCH 07/12] incusd/scriptlet/qemu: Prevent calling QMP functions at config stage Signed-off-by: Benjamin Somers --- internal/server/scriptlet/qemu.go | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/internal/server/scriptlet/qemu.go b/internal/server/scriptlet/qemu.go index 54966d4b11e..89283adc747 100644 --- a/internal/server/scriptlet/qemu.go +++ b/internal/server/scriptlet/qemu.go @@ -18,10 +18,23 @@ import ( // QEMURun runs the QEMU scriptlet. func QEMURun(l logger.Logger, instance *api.Instance, m *qmp.Monitor, stage string) error { logFunc := log.CreateLogger(l, "QEMU scriptlet ("+stage+")") + + assertQEMUStarted := func(name string) error { + if stage == "config" { + return fmt.Errorf("%s cannot be called at config stage", name) + } + + return nil + } + runQMPFunc := func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { - var command *starlark.Dict + err := assertQEMUStarted(b.Name()) + if err != nil { + return nil, err + } - err := starlark.UnpackArgs(b.Name(), args, kwargs, "command", &command) + var command *starlark.Dict + err = starlark.UnpackArgs(b.Name(), args, kwargs, "command", &command) if err != nil { return nil, err } @@ -85,6 +98,11 @@ func QEMURun(l logger.Logger, instance *api.Instance, m *qmp.Monitor, stage stri } runCommandFunc := func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + err := assertQEMUStarted(b.Name()) + if err != nil { + return nil, err + } + frame := thread.CallFrame(1) errPrefix := fmt.Sprintf("run_command (%d:%d):", frame.Pos.Line, frame.Pos.Col) @@ -113,6 +131,11 @@ func QEMURun(l logger.Logger, instance *api.Instance, m *qmp.Monitor, stage stri makeQOM := func(funName string) *starlark.Builtin { fun := func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + err := assertQEMUStarted(b.Name()) + if err != nil { + return nil, err + } + frame := thread.CallFrame(1) errPrefix := fmt.Sprintf("%s (%d:%d):", b.Name(), frame.Pos.Line, frame.Pos.Col) From 3c5081a73ff612c0880d1cd512945759bb61a1f2 Mon Sep 17 00:00:00 2001 From: Benjamin Somers Date: Fri, 17 Jan 2025 21:15:38 +0000 Subject: [PATCH 08/12] incusd/scriptlet/qemu: Add QEMU configuration getters Signed-off-by: Benjamin Somers --- .../server/instance/drivers/driver_qemu.go | 2 +- internal/server/scriptlet/load/load.go | 4 + internal/server/scriptlet/qemu.go | 83 ++++++++++++++++++- 3 files changed, 84 insertions(+), 5 deletions(-) diff --git a/internal/server/instance/drivers/driver_qemu.go b/internal/server/instance/drivers/driver_qemu.go index feb1443316e..b0d5b26d8a3 100644 --- a/internal/server/instance/drivers/driver_qemu.go +++ b/internal/server/instance/drivers/driver_qemu.go @@ -1142,7 +1142,7 @@ func (d *qemu) runStartupScriptlet(monitor *qmp.Monitor, stage string) error { return errors.New("Unexpected instance type") } - err := scriptlet.QEMURun(logger.Log, instanceData, monitor, stage) + err := scriptlet.QEMURun(logger.Log, instanceData, &d.cmdArgs, &d.conf, monitor, stage) if err != nil { err = fmt.Errorf("Failed running QEMU scriptlet at %s stage: %w", stage, err) return err diff --git a/internal/server/scriptlet/load/load.go b/internal/server/scriptlet/load/load.go index 2f6904d88d9..29920cf95c9 100644 --- a/internal/server/scriptlet/load/load.go +++ b/internal/server/scriptlet/load/load.go @@ -59,6 +59,7 @@ func QEMUCompile(name string, src string) (*starlark.Program, error) { "log_info", "log_warn", "log_error", + "run_qmp", "run_command", "blockdev_add", @@ -75,6 +76,9 @@ func QEMUCompile(name string, src string) (*starlark.Program, error) { "qom_get", "qom_list", "qom_set", + + "get_qemu_cmdline", + "get_qemu_conf", }) } diff --git a/internal/server/scriptlet/qemu.go b/internal/server/scriptlet/qemu.go index 89283adc747..fa36b9f1c65 100644 --- a/internal/server/scriptlet/qemu.go +++ b/internal/server/scriptlet/qemu.go @@ -7,6 +7,7 @@ import ( "go.starlark.net/starlark" + "github.com/lxc/incus/v6/internal/server/instance/drivers/cfg" "github.com/lxc/incus/v6/internal/server/instance/drivers/qmp" scriptletLoad "github.com/lxc/incus/v6/internal/server/scriptlet/load" "github.com/lxc/incus/v6/internal/server/scriptlet/log" @@ -15,10 +16,52 @@ import ( "github.com/lxc/incus/v6/shared/logger" ) +// qemuCfgSection is a temporary struct to hold QEMU configuration sections with Entries of type +// map[string]string instead of []cfg.Entry. This type should be moved to the cfg package in the +// future. +type qemuCfgSection struct { + Name string `json:"name"` + Comment string `json:"comment"` + Entries map[string]string `json:"entries"` +} + +// marshalQEMUConf marshals a configuration into a []map[string]any. +func marshalQEMUConf(conf any) ([]map[string]any, error) { + jsonConf, err := json.Marshal(conf) + if err != nil { + return nil, err + } + + var newConf []map[string]any + err = json.Unmarshal(jsonConf, &newConf) + if err != nil { + return nil, err + } + + return newConf, nil +} + // QEMURun runs the QEMU scriptlet. -func QEMURun(l logger.Logger, instance *api.Instance, m *qmp.Monitor, stage string) error { +func QEMURun(l logger.Logger, instance *api.Instance, cmdArgs *[]string, conf *[]cfg.Section, m *qmp.Monitor, stage string) error { logFunc := log.CreateLogger(l, "QEMU scriptlet ("+stage+")") + // We first convert from []cfg.Section to []qemuCfgSection. This conversion is temporary. + var cfgSectionMaps []qemuCfgSection + for _, section := range *conf { + entries := map[string]string{} + for _, entry := range section.Entries { + entries[entry.Key] = entry.Value + } + + cfgSectionMaps = append(cfgSectionMaps, qemuCfgSection{Name: section.Name, Comment: section.Comment, Entries: entries}) + } + + // We do not want to handle a qemuCfgSection object within our scriptlet, for simplicity. + cfgSections, err := marshalQEMUConf(cfgSectionMaps) + if err != nil { + return err + } + assertQEMUStarted := func(name string) error { if stage == "config" { return fmt.Errorf("%s cannot be called at config stage", name) @@ -155,12 +198,41 @@ func QEMURun(l logger.Logger, instance *api.Instance, m *qmp.Monitor, stage stri return starlark.NewBuiltin(strings.ReplaceAll(funName, "-", "_"), fun) } + getCmdArgsFunc := func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + err := starlark.UnpackArgs(b.Name(), args, kwargs) + if err != nil { + return nil, err + } + + rv, err := marshal.StarlarkMarshal(cmdArgs) + if err != nil { + return nil, fmt.Errorf("Marshalling QEMU command-line arguments failed: %w", err) + } + + return rv, nil + } + + getConfFunc := func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + err := starlark.UnpackArgs(b.Name(), args, kwargs) + if err != nil { + return nil, err + } + + rv, err := marshal.StarlarkMarshal(cfgSections) + if err != nil { + return nil, fmt.Errorf("Marshalling QEMU configuration failed: %w", err) + } + + return rv, nil + } + // Remember to match the entries in scriptletLoad.QEMUCompile() with this list so Starlark can // perform compile time validation of functions used. env := starlark.StringDict{ - "log_info": starlark.NewBuiltin("log_info", logFunc), - "log_warn": starlark.NewBuiltin("log_warn", logFunc), - "log_error": starlark.NewBuiltin("log_error", logFunc), + "log_info": starlark.NewBuiltin("log_info", logFunc), + "log_warn": starlark.NewBuiltin("log_warn", logFunc), + "log_error": starlark.NewBuiltin("log_error", logFunc), + "run_qmp": starlark.NewBuiltin("run_qmp", runQMPFunc), "run_command": starlark.NewBuiltin("run_command", runCommandFunc), "blockdev_add": makeQOM("blockdev-add"), @@ -177,6 +249,9 @@ func QEMURun(l logger.Logger, instance *api.Instance, m *qmp.Monitor, stage stri "qom_get": makeQOM("qom-get"), "qom_list": makeQOM("qom-list"), "qom_set": makeQOM("qom-set"), + + "get_qemu_cmdline": starlark.NewBuiltin("get_qemu_cmdline", getCmdArgsFunc), + "get_qemu_conf": starlark.NewBuiltin("get_qemu_conf", getConfFunc), } prog, thread, err := scriptletLoad.QEMUProgram(instance.Name) From b0bb02c7b0cabd541119c1968fd972cb51a72a28 Mon Sep 17 00:00:00 2001 From: Benjamin Somers Date: Fri, 17 Jan 2025 21:12:58 +0000 Subject: [PATCH 09/12] incusd/scriptlet/qemu: Add QEMU configuration setters Signed-off-by: Benjamin Somers --- internal/server/scriptlet/load/load.go | 2 + internal/server/scriptlet/qemu.go | 152 +++++++++++++++++++++++++ 2 files changed, 154 insertions(+) diff --git a/internal/server/scriptlet/load/load.go b/internal/server/scriptlet/load/load.go index 29920cf95c9..615c7b5dcba 100644 --- a/internal/server/scriptlet/load/load.go +++ b/internal/server/scriptlet/load/load.go @@ -78,7 +78,9 @@ func QEMUCompile(name string, src string) (*starlark.Program, error) { "qom_set", "get_qemu_cmdline", + "set_qemu_cmdline", "get_qemu_conf", + "set_qemu_conf", }) } diff --git a/internal/server/scriptlet/qemu.go b/internal/server/scriptlet/qemu.go index fa36b9f1c65..c7cf7105e6d 100644 --- a/internal/server/scriptlet/qemu.go +++ b/internal/server/scriptlet/qemu.go @@ -2,6 +2,7 @@ package scriptlet import ( "encoding/json" + "errors" "fmt" "strings" @@ -41,6 +42,22 @@ func marshalQEMUConf(conf any) ([]map[string]any, error) { return newConf, nil } +// unmarshalQEMUConf unmarshals a configuration into a []qemuCfgSection. +func unmarshalQEMUConf(conf any) ([]qemuCfgSection, error) { + jsonConf, err := json.Marshal(conf) + if err != nil { + return nil, err + } + + var newConf []qemuCfgSection + err = json.Unmarshal(jsonConf, &newConf) + if err != nil { + return nil, err + } + + return newConf, nil +} + // QEMURun runs the QEMU scriptlet. func QEMURun(l logger.Logger, instance *api.Instance, cmdArgs *[]string, conf *[]cfg.Section, m *qmp.Monitor, stage string) error { logFunc := log.CreateLogger(l, "QEMU scriptlet ("+stage+")") @@ -70,6 +87,14 @@ func QEMURun(l logger.Logger, instance *api.Instance, cmdArgs *[]string, conf *[ return nil } + assertConfigStage := func(name string) error { + if stage != "config" { + return fmt.Errorf("%s can only be called at config stage", name) + } + + return nil + } + runQMPFunc := func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { err := assertQEMUStarted(b.Name()) if err != nil { @@ -212,6 +237,69 @@ func QEMURun(l logger.Logger, instance *api.Instance, cmdArgs *[]string, conf *[ return rv, nil } + setCmdArgsFunc := func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + err := assertConfigStage(b.Name()) + if err != nil { + return nil, err + } + + var newCmdArgsv *starlark.List + err = starlark.UnpackArgs(b.Name(), args, kwargs, "args", &newCmdArgsv) + if err != nil { + return nil, err + } + + newCmdArgsAny, err := marshal.StarlarkUnmarshal(newCmdArgsv) + if err != nil { + return nil, err + } + + newCmdArgsListAny, ok := newCmdArgsAny.([]any) + if !ok { + return nil, fmt.Errorf("%s requires a list of strings", b.Name()) + } + + // Check whether -bios or -kernel are in the new arguments, and convert them to string on the go. + var newFoundBios, newFoundKernel bool + var newCmdArgs []string + for _, argAny := range newCmdArgsListAny { + arg, ok := argAny.(string) + if !ok { + return nil, fmt.Errorf("%s requires a list of strings", b.Name()) + } + + newCmdArgs = append(newCmdArgs, arg) + + if arg == "-bios" { + newFoundBios = true + } else if arg == "-kernel" { + newFoundKernel = true + } + } + + // Check whether -bios or -kernel are in the current arguments + var foundBios, foundKernel bool + for _, arg := range *cmdArgs { + if arg == "-bios" { + foundBios = true + } else if arg == "-kernel" { + foundKernel = true + } + + // If we've found both already, we can break early. + if foundBios && foundKernel { + break + } + } + + if foundBios != newFoundBios || foundKernel != newFoundKernel { + return nil, errors.New("Addition or deletion of -bios or -kernel is unsupported") + } + + *cmdArgs = newCmdArgs + return starlark.None, nil + } + getConfFunc := func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { err := starlark.UnpackArgs(b.Name(), args, kwargs) if err != nil { @@ -226,6 +314,49 @@ func QEMURun(l logger.Logger, instance *api.Instance, cmdArgs *[]string, conf *[ return rv, nil } + setConfFunc := func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + err := assertConfigStage(b.Name()) + if err != nil { + return nil, err + } + + var newConf *starlark.List + err = starlark.UnpackArgs(b.Name(), args, kwargs, "conf", &newConf) + if err != nil { + return nil, err + } + + confAny, err := marshal.StarlarkUnmarshal(newConf) + if err != nil { + return nil, err + } + + confListAny, ok := confAny.([]any) + if !ok { + return nil, fmt.Errorf("%s requires a valid configuration", b.Name()) + } + + var newCfgSections []map[string]any + for _, section := range confListAny { + newSection, ok := section.(map[string]any) + if !ok { + return nil, fmt.Errorf("%s requires a valid configuration", b.Name()) + } + + newCfgSections = append(newCfgSections, newSection) + } + + // We want to further check the configuration structure, by trying to unmarshal it to a + // []qemuCfgSection. + _, err = unmarshalQEMUConf(confAny) + if err != nil { + return nil, fmt.Errorf("%s requires a valid configuration", b.Name()) + } + + cfgSections = newCfgSections + return starlark.None, nil + } + // Remember to match the entries in scriptletLoad.QEMUCompile() with this list so Starlark can // perform compile time validation of functions used. env := starlark.StringDict{ @@ -251,7 +382,9 @@ func QEMURun(l logger.Logger, instance *api.Instance, cmdArgs *[]string, conf *[ "qom_set": makeQOM("qom-set"), "get_qemu_cmdline": starlark.NewBuiltin("get_qemu_cmdline", getCmdArgsFunc), + "set_qemu_cmdline": starlark.NewBuiltin("set_qemu_cmdline", setCmdArgsFunc), "get_qemu_conf": starlark.NewBuiltin("get_qemu_conf", getConfFunc), + "set_qemu_conf": starlark.NewBuiltin("set_qemu_conf", setConfFunc), } prog, thread, err := scriptletLoad.QEMUProgram(instance.Name) @@ -296,5 +429,24 @@ func QEMURun(l logger.Logger, instance *api.Instance, cmdArgs *[]string, conf *[ return fmt.Errorf("Failed with unexpected return value: %v", v) } + // We need to convert the configuration back to a suitable format + cfgSectionMaps, err = unmarshalQEMUConf(cfgSections) + if err != nil { + return err + } + + // We convert back from []qemuCfgSection to []cfg.Section. This conversion is temporary. + var newConf []cfg.Section + for _, section := range cfgSectionMaps { + entries := []cfg.Entry{} + for key, value := range section.Entries { + entries = append(entries, cfg.Entry{Key: key, Value: value}) + } + + newConf = append(newConf, cfg.Section{Name: section.Name, Comment: section.Comment, Entries: entries}) + } + + *conf = newConf + return nil } From f5e5aa65060073e3b6fa11a6b7bd47ed36e88176 Mon Sep 17 00:00:00 2001 From: Benjamin Somers Date: Thu, 16 Jan 2025 17:58:58 +0000 Subject: [PATCH 10/12] api: qemu_scriptlet_config Signed-off-by: Benjamin Somers --- doc/api-extensions.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/api-extensions.md b/doc/api-extensions.md index dd6a7ccc7ca..61fe2bbf3ba 100644 --- a/doc/api-extensions.md +++ b/doc/api-extensions.md @@ -2681,3 +2681,7 @@ It introduces the new `--force` flag for connecting to the instance console. ## `network_ovn_state_addresses` This adds extra fields to the OVN network state struct for the IPv4 and IPv6 addresses used on the uplink. + +## `qemu_scriptlet_config` + +This extends the QEMU scriptlet feature by allowing to modify QEMU configuration before a VM starts, and passing information about the instance to the scriptlet. From 179b54ed9eec36b13ae5f2401b2403c638e4d487 Mon Sep 17 00:00:00 2001 From: Benjamin Somers Date: Thu, 16 Jan 2025 18:17:54 +0000 Subject: [PATCH 11/12] doc/ref/instance_options: Update QEMU scriptlet documentation Signed-off-by: Benjamin Somers --- doc/reference/instance_options.md | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/doc/reference/instance_options.md b/doc/reference/instance_options.md index d10cfebc9e8..c7edd41be84 100644 --- a/doc/reference/instance_options.md +++ b/doc/reference/instance_options.md @@ -342,16 +342,17 @@ Those take a JSON encoded list of QMP commands to run. The hooks correspond to: -- `early` is run prior to any device having been added by Incus through QMP -- `pre-start` is run following Incus having added all its devices by prior to the VM being started -- `post-start` is run immediately following the VM starting up +- `early`, run prior to any device having been added by Incus through QMP, after QEMU has started +- `pre-start`, run following Incus having added all its devices but prior to the VM being started +- `post-start`, run immediately following the VM starting up +### Advanced use For anyone needing dynamic QMP interactions, for example to retrieve the current value of some objects before modifying or generating new objects, it's also possible to attach to those same hooks using a scriptlet. -This is done through `raw.qemu.scriptlet`. The scriptlet must define the `qemu_hook(stage)` function. +This is done through `raw.qemu.scriptlet`. The scriptlet must define the `qemu_hook(instance, stage)` function. The `instance` arguments is an object representing the VM, whose attributes are those of the `api.Instance` struct. The `stage` argument is the name of the hook (`config`, `early`, `pre-start` or `post-start`), with `config` being run before starting QEMU, and the other hooks defined above. The following commands are exposed to that scriptlet: @@ -360,6 +361,10 @@ The following commands are exposed to that scriptlet: - `log_error` will log an `ERROR` message - `run_qmp` will run an arbitrary QMP command (JSON) and return its output - `run_command` will run the specified command with an optional list of arguments and return its output +- `get_qemu_cmdline` will return the list of command-line arguments passed to QEMU +- `set_qemu_cmdline` will set them +- `get_qemu_conf` will return the QEMU configuration file as a dictionary +- `set_qemu_conf` will set it from a dictionary Additionally the following alias commands (internally use `run_command`) are also available to simplify scripts: @@ -378,6 +383,8 @@ Additionally the following alias commands (internally use `run_command`) are als - `qom_list` - `qom_set` +The functions allowing to change QEMU configuration can only be run during the `config` hook. In parallel, the functions running QMP commands cannot be run during the `config` hook. + (instance-options-security)= ## Security policies From be7d5f38588ac6f2d183de631a66ceab48f7d2be Mon Sep 17 00:00:00 2001 From: Benjamin Somers Date: Thu, 16 Jan 2025 20:19:41 +0000 Subject: [PATCH 12/12] incusd/instance: Update unit test Signed-off-by: Benjamin Somers --- .../drivers/driver_qemu_config_test.go | 75 ++++++++++--------- 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/internal/server/instance/drivers/driver_qemu_config_test.go b/internal/server/instance/drivers/driver_qemu_config_test.go index b2bb65c1769..34c24b3c74c 100644 --- a/internal/server/instance/drivers/driver_qemu_config_test.go +++ b/internal/server/instance/drivers/driver_qemu_config_test.go @@ -6,6 +6,7 @@ import ( "strings" "testing" + "github.com/lxc/incus/v6/internal/server/instance/drivers/cfg" "github.com/lxc/incus/v6/shared/osarch" ) @@ -16,7 +17,7 @@ func TestQemuConfigTemplates(t *testing.T) { return strings.TrimSpace(indent.ReplaceAllString(s, "$1")) } - runTest := func(expected string, sections []cfgSection) { + runTest := func(expected string, sections []cfg.Section) { t.Run(expected, func(t *testing.T) { actual := normalize(qemuStringifyCfg(sections...).String()) expected = normalize(expected) @@ -1082,47 +1083,47 @@ func TestQemuConfigTemplates(t *testing.T) { }) t.Run("qemu_raw_cfg_override", func(t *testing.T) { - cfg := []cfgSection{{ - name: "global", - entries: []cfgEntry{ - {key: "driver", value: "ICH9-LPC"}, - {key: "property", value: "disable_s3"}, - {key: "value", value: "1"}, + conf := []cfg.Section{{ + Name: "global", + Entries: []cfg.Entry{ + {Key: "driver", Value: "ICH9-LPC"}, + {Key: "property", Value: "disable_s3"}, + {Key: "value", Value: "1"}, }, }, { - name: "global", - entries: []cfgEntry{ - {key: "driver", value: "ICH9-LPC"}, - {key: "property", value: "disable_s4"}, - {key: "value", value: "1"}, + Name: "global", + Entries: []cfg.Entry{ + {Key: "driver", Value: "ICH9-LPC"}, + {Key: "property", Value: "disable_s4"}, + {Key: "value", Value: "1"}, }, }, { - name: "memory", - entries: []cfgEntry{ - {key: "size", value: "1024M"}, + Name: "memory", + Entries: []cfg.Entry{ + {Key: "size", Value: "1024M"}, }, }, { - name: `device "qemu_gpu"`, - entries: []cfgEntry{ - {key: "driver", value: "virtio-gpu-pci"}, - {key: "bus", value: "qemu_pci3"}, - {key: "addr", value: "00.0"}, + Name: `device "qemu_gpu"`, + Entries: []cfg.Entry{ + {Key: "driver", Value: "virtio-gpu-pci"}, + {Key: "bus", Value: "qemu_pci3"}, + {Key: "addr", Value: "00.0"}, }, }, { - name: `device "qemu_keyboard"`, - entries: []cfgEntry{ - {key: "driver", value: "virtio-keyboard-pci"}, - {key: "bus", value: "qemu_pci2"}, - {key: "addr", value: "00.1"}, + Name: `device "qemu_keyboard"`, + Entries: []cfg.Entry{ + {Key: "driver", Value: "virtio-keyboard-pci"}, + {Key: "bus", Value: "qemu_pci2"}, + {Key: "addr", Value: "00.1"}, }, }} testCases := []struct { - cfg []cfgSection + cfg []cfg.Section overrides map[string]string expected string }{{ // unmodified - cfg, + conf, map[string]string{}, `[global] driver = "ICH9-LPC" @@ -1148,7 +1149,7 @@ func TestQemuConfigTemplates(t *testing.T) { addr = "00.1"`, }, { // override some keys - cfg, + conf, map[string]string{ "raw.qemu.conf": ` [memory] @@ -1181,7 +1182,7 @@ func TestQemuConfigTemplates(t *testing.T) { addr = "00.1"`, }, { // delete some keys - cfg, + conf, map[string]string{ "raw.qemu.conf": ` [device "qemu_keyboard"] @@ -1212,7 +1213,7 @@ func TestQemuConfigTemplates(t *testing.T) { addr = "00.1"`, }, { // add some keys to existing sections - cfg, + conf, map[string]string{ "raw.qemu.conf": ` [memory] @@ -1257,7 +1258,7 @@ func TestQemuConfigTemplates(t *testing.T) { multifunction = "off"`, }, { // edit/add/remove - cfg, + conf, map[string]string{ "raw.qemu.conf": ` [memory] @@ -1291,7 +1292,7 @@ func TestQemuConfigTemplates(t *testing.T) { driver = "virtio-keyboard-pci"`, }, { // delete sections - cfg, + conf, map[string]string{ "raw.qemu.conf": ` [memory] @@ -1309,7 +1310,7 @@ func TestQemuConfigTemplates(t *testing.T) { addr = "00.0"`, }, { // add sections - cfg, + conf, map[string]string{ "raw.qemu.conf": ` [object1] @@ -1363,7 +1364,7 @@ func TestQemuConfigTemplates(t *testing.T) { key6 = "value6"`, }, { // add/remove sections - cfg, + conf, map[string]string{ "raw.qemu.conf": ` [device "qemu_gpu"] @@ -1400,7 +1401,7 @@ func TestQemuConfigTemplates(t *testing.T) { key4 = "value4"`, }, { // edit keys of repeated sections - cfg, + conf, map[string]string{ "raw.qemu.conf": ` [global][1] @@ -1439,7 +1440,7 @@ func TestQemuConfigTemplates(t *testing.T) { addr = "00.1"`, }, { // create multiple sections with same name - cfg, + conf, // note that for appending new sections, all that matters is that // the index is higher than the existing indexes map[string]string{ @@ -1498,7 +1499,7 @@ func TestQemuConfigTemplates(t *testing.T) { k11 = "v11"`, }, { // mix all operations - cfg, + conf, map[string]string{ "raw.qemu.conf": ` [memory]