Merge pull request #148 from williamtheaker/wt.macos_guests

Add support for running macOS guests.

.gitignore

@@ -1,2 +1,3 @@
 /out
 **/.DS_Store
+**/efi-variable-store

README.md

@@ -17,6 +17,9 @@ brew tap cfergeau/crc
 brew install vfkit
 ```
 
+### Building
+
+From the root direction of this repository, run `make`.
 
 ### Usage
 

cmd/vfkit/root.go

@@ -14,7 +14,7 @@ var opts = &cmdline.Options{}
 var rootCmd = &cobra.Command{
 	Use:   "vfkit",
 	Short: "vfkit is a simple hypervisor using Apple's Virtualization framework",
-	Long: `A hypervisor written in Go using Apple's Virtualization framework to run Linux virtual machines.
+	Long: `A hypervisor written in Go using Apple's Virtualization framework to run virtual machines.
                 Complete documentation is available at https://github.com/crc-org/vfkit`,
 	RunE: func(_ *cobra.Command, _ []string) error {
 		if len(opts.LogLevel) > 0 {

doc/quickstart.md

@@ -5,14 +5,14 @@
 vfkit is a macOS command-line-based hypervisor, which uses [Apple's Virtualization Framework](https://developer.apple.com/documentation/virtualization?language=objc) to run virtual machines.
 You start a virtual machine by running vfkit with a set of arguments describing the virtual machine configuration/hardware.
 When vfkit stops, the virtual machine stops running.
-It requires macOS 11 or newer, and runs on both x86_64 and aarch64 Macs.
+It requires macOS 11 or newer, and runs on both Intel and Apple silicon Macs.
 File sharing is only available on macOS 12 or newer.
 UEFI boot and graphical user interface support are only available on macOS 13 or newer.
 
 
 ## Installation
 
-You can get vfkit either by downloading it from [its release page](https://github.com/crc-org/vfkit/releases), or get it from [brew](https://brew.sh/):
+You can either download vfkit from [its release page](https://github.com/crc-org/vfkit/releases), or install it from [brew](https://brew.sh/):
 ```
 # Only the first time
 brew tap cfergeau/crc
@@ -66,6 +66,10 @@ If you are using an image or an older macOS version which does not support UEFI
 This requires a separate kernel, initrd file and kernel command-line arguments.
 Details can be found in the [usage instructions](https://github.com/crc-org/vfkit/blob/main/doc/usage.md#linux-bootloader).
 
+### Adding a GUI
+
+To run a VM with a graphical user interface, append the necessary flags to your vfkit command:
+`--device virtio-input,keyboard --device virtio-input,pointing --device virtio-gpu,width=800,height=600 --gui`
 
 ### Adding a serial console for boot logs
 
@@ -83,7 +87,7 @@ On more verbose images, the boot logs are only shown late in the boot process as
 
 To make the interactions with the virtual machine easier, we can add a virtio-net device to it:
 ```
-    --device virtio-net,nat
+--device virtio-net,nat
 ```
 
 After booting, the Fedora image prints the IP address of the VM in the serial console before the login prompt.
@@ -104,8 +108,9 @@ Once you have a virtual machine up and running, here are some additional feature
 - [host/guest communication over virtio-vsock](https://github.com/crc-org/vfkit/blob/main/doc/usage.md#virtio-vsock-communication)
 - [host/guest file sharing with virtio-fs](https://github.com/crc-org/vfkit/blob/main/doc/usage.md#file-sharing)
 - [Rosetta support to run x86_64 binaries in virtual machines on Apple silicon Macs](https://github.com/crc-org/vfkit/blob/main/doc/usage.md#rosetta)
-- [GUI support](https://github.com/crc-org/vfkit/blob/main/doc/usage.md#enabling-a-graphical-user-interface)
 - [REST API to control the virtual machine](https://github.com/crc-org/vfkit/blob/main/doc/usage.md#restful-service)
 - [user-mode networking with the `gvproxy` command from gvisor-tap-vsock](https://github.com/containers/gvisor-tap-vsock)
 
+Full documentation of vfkit's various features is documented in the [usage guide](https://github.com/crc-org/vfkit/blob/main/doc/usage.md).
+
 Any questions/issues/... with vfkit can be reported [on GitHub](https://github.com/crc-org/vfkit/issues/new).

doc/usage.md

@@ -16,8 +16,7 @@ Set the log-level for VFKit.  Supported values are `debug`, `info`, and `error`.
 - `--restful-URI`
 
 The URI (address) of the RESTful service.  The default is `tcp://localhost:8081`.  Valid schemes are
-`tcp`, `none`, or `unix`.  In the case of unix, the "host" portion would be a path to where the unix domain
-socket will be stored. A scheme of `none` disables the RESTful service.
+`tcp`, `none`, or `unix`.  In the case of unix, the "host" portion would be a path to where the unix domain socket will be stored. A scheme of `none` disables the RESTful service.
 
 ### Virtual Machine Resources
 
@@ -47,7 +46,7 @@ It must be configured to communicate over virtio-vsock.
 
 ## Bootloader Configuration
 
-A bootloader is required to tell vfkit _how_ it should be starting the guest OS.
+A bootloader is required to tell vfkit _how_ it should start the guest OS.
 
 ### Linux bootloader
 
@@ -68,6 +67,21 @@ It allows to specify which kernel and initrd should be used when starting the VM
 
 The kernel command line must be enclosed in `"`, and depending on your shell, they might need to be escaped (`\"`)
 
+### macOS bootloader
+
+#### Description
+
+`--bootloader macos` is required to run macOS VMs. You must use an arm64/Apple silicon device running macOS 12 or later. Due to hardcoded limitations in the Apple Virtualization framework, it's not possible to run more than two macOS VMs at a time. Since macOS guests can't run headlessly, you'll need to enable a GUI, even if you only plan to interact with the VM over SSH.
+
+#### Arguments
+
+- `machineIdentifierPath`: absolute path to a binary property list containing a unique ECID identifier for the VM
+- `hardwareModelPath`: absolute path to a binary property list defining OS version support
+- `auxImagePath`: absolute path to the auxiliary storage file with NVRAM contents and the iBoot bootloader
+
+#### Example
+
+`--bootloader macos,machineIdentifierPath=/Users/virtuser/VM.bundle/MachineIdentifier,hardwareModelPath=/Users/virtuser/VM.bundle/HardwareModel,auxImagePath=/Users/virtuser/VM.bundle/AuxiliaryStorage`
 
 ### EFI bootloader
 
@@ -111,7 +125,7 @@ Kernel command line to use when starting the virtual machine.
 
 ## Device Configuration
 
-Various devices can be added to the virtual machines. They are all paravirtualized devices using VirtIO. They are grouped under the `--device` commande line flag.
+Various devices can be added to the virtual machines. They are all paravirtualized devices using VirtIO. They are grouped under the `--device` command line flag.
 
 
 ### Disk
@@ -124,17 +138,16 @@ See also [vz/CreateDiskImage](https://pkg.go.dev/github.com/Code-Hex/vz/v3#Creat
 
 #### Thin images
 
-Apple Virtualization Framework only support raw disk images and ISO images.
+Apple Virtualization Framework only supports raw disk images and ISO images.
 There is no support for thin image formats such as [qcow2](https://en.wikipedia.org/wiki/Qcow).
 
-However, APFS, the default macOS filesystem has support for sparse files and copy-on-write files, so it offers the main features of thin image format.
+However, APFS, the default macOS filesystem has support for sparse files and copy-on-write files, so it offers the main features of thin image formats.
 
 A sparse raw image can be created/expanded using the `truncate` command or
-using
-[`truncate(2)`](https://manpagez.com/man/2/truncate/).
+using [`truncate(2)`](https://manpagez.com/man/2/truncate/).
 For example, an empty 1GiB disk can be created with `truncate -s 1G
 vfkit.img`. Such an image will only use disk space when content is written to
-it. It initially only uses a few bytes of actual disk space even if it's size
+it. It initially only uses a few bytes of actual disk space even if its size
 is 1G.
 
 A copy-on-write image is a raw image file which references a backing file. Its
@@ -318,11 +331,15 @@ This will share `/Users/virtuser/vfkit` with the guest:
 --device virtio-fs,sharedDir=/Users/virtuser/vfkit/,mountTag=vfkit-share
 ```
 
-The share can then be mounted in the guest with:
+The share can then be mounted in Linux guests with:
 ```
 mount -t virtiofs vfkit-share /mount
 ```
 
+and on macOS with:
+```
+mkdir /tmp/tag && mount_virtiofs vfkit-share /tmp/tag
+```
 
 ### Rosetta
 
@@ -386,34 +403,45 @@ None
 `--device virtio-input,pointing`
 
 
-## RESTful Service
+## RESTful API
+
+To interact with the RESTful API, append a valid scheme to your base command: `--restful-uri tcp://localhost:8081`.
 
 ### Get VM state
 
-Used to obtain the state of the virtual machine that is being run by vfkit.
+Obtain the state of the virtual machine that is being run by vfkit.
+
+Request:
+```HTTP
+GET /vm/state
+```
 
-GET `/vm/state`
-Response: { "state": string, "canStart": bool, "canPause": bool, "canResume": bool, "canStop": bool, "canHardStop": bool }
+Response:
+`{ "state": string, "canStart": bool, "canPause": bool, "canResume": bool, "canStop": bool, "canHardStop": bool }`
 
-> `canHardStop` is only supported on macOS 12 and newer, false will always be returned on older versions.
+`canHardStop` is only supported on macOS 12 and newer, false will always be returned on older versions.
 
 ### Change VM State
 
-Change the state of the virtual machine. Valid states are:
+Change the state of the virtual machine. Valid state values are:
 * HardStop
 * Pause
 * Resume
 * Stop
 
-POST `/vm/state` { "state": "new value"}
-
-Response: http 200
+```HTTP
+POST /vm/state { "state": "new value"}
+```
+Response: HTTP 200
 
 ### Inspect VM
 
 Get description of the virtual machine
 
-GET `/vm/inspect`
+```HTTP
+GET /vm/inspect
+```
+
 Response: { "cpus": uint, "memory": uint64, "devices": []config.VirtIODevice }
 
 ## Enabling a Graphical User Interface

pkg/config/bootloader.go

@@ -30,6 +30,13 @@ type EFIBootloader struct {
 	CreateVariableStore bool `json:"createVariableStore"`
 }
 
+// MacOSBootloader provides necessary objects for booting macOS guests
+type MacOSBootloader struct {
+	MachineIdentifierPath string `json:"machineIdentifierPath"`
+	HardwareModelPath     string `json:"hardwareModelPath"`
+	AuxImagePath          string `json:"auxImagePath"`
+}
+
 // NewLinuxBootloader creates a new bootloader to start a VM with the file at
 // vmlinuzPath as the kernel, kernelCmdLine as the kernel command line, and the
 // file at initrdPath as the initrd. On ARM64, the kernel must be uncompressed
@@ -120,6 +127,28 @@ func (bootloader *EFIBootloader) ToCmdLine() ([]string, error) {
 	return []string{"--bootloader", builder.String()}, nil
 }
 
+func (bootloader *MacOSBootloader) FromOptions(options []option) error {
+	for _, option := range options {
+		switch option.key {
+		case "machineIdentifierPath":
+			bootloader.MachineIdentifierPath = option.value
+		case "hardwareModelPath":
+			bootloader.HardwareModelPath = option.value
+		case "auxImagePath":
+			bootloader.AuxImagePath = option.value
+		default:
+			return fmt.Errorf("unknown option for macOS bootloaders: %s", option.key)
+		}
+	}
+	return nil
+}
+
+func (bootloader *MacOSBootloader) ToCmdLine() ([]string, error) {
+	args := []string{}
+
+	return args, nil
+}
+
 func BootloaderFromCmdLine(optsStrv []string) (Bootloader, error) {
 	var bootloader Bootloader
 
@@ -132,6 +161,8 @@ func BootloaderFromCmdLine(optsStrv []string) (Bootloader, error) {
 		bootloader = &EFIBootloader{}
 	case "linux":
 		bootloader = &LinuxBootloader{}
+	case "macos":
+		bootloader = &MacOSBootloader{}
 	default:
 		return nil, fmt.Errorf("unknown bootloader type: %s", bootloaderType)
 	}

pkg/vf/bootloader.go

@@ -87,6 +87,8 @@ func toVzBootloader(bootloader config.Bootloader) (vz.BootLoader, error) {
 		return toVzLinuxBootloader(b)
 	case *config.EFIBootloader:
 		return toVzEFIBootloader(b)
+	case *config.MacOSBootloader:
+		return toVzMacOSBootloader(b)
 	default:
 		return nil, fmt.Errorf("Unexpected bootloader type: %T", b)
 	}

pkg/vf/virtio.go

@@ -117,22 +117,34 @@ func (dev *VirtioInput) AddToVirtualMachineConfig(vmConfig *VirtualMachineConfig
 	return nil
 }
 
-func (dev *VirtioGPU) toVz() (vz.GraphicsDeviceConfiguration, error) {
+func newVirtioGraphicsDeviceConfiguration(dev *VirtioGPU) (vz.GraphicsDeviceConfiguration, error) {
 	gpuDeviceConfig, err := vz.NewVirtioGraphicsDeviceConfiguration()
 	if err != nil {
-		return nil, fmt.Errorf("failed to initialize virtio graphic device: %w", err)
+		return nil, fmt.Errorf("failed to initialize virtio graphics device: %w", err)
 	}
 	graphicsScanoutConfig, err := vz.NewVirtioGraphicsScanoutConfiguration(int64(dev.Width), int64(dev.Height))
+
 	if err != nil {
 		return nil, fmt.Errorf("failed to create graphics scanout: %w", err)
 	}
+
 	gpuDeviceConfig.SetScanouts(
 		graphicsScanoutConfig,
 	)
 
 	return gpuDeviceConfig, nil
 }
 
+func (dev *VirtioGPU) toVz() (vz.GraphicsDeviceConfiguration, error) {
+	log.Debugf("Setting up graphics device with %vx%v resolution.", dev.Width, dev.Height)
+
+	if PlatformType == "macos" {
+		return newMacGraphicsDeviceConfiguration(dev)
+	}
+	return newVirtioGraphicsDeviceConfiguration(dev)
+
+}
+
 func (dev *VirtioGPU) AddToVirtualMachineConfig(vmConfig *VirtualMachineConfiguration) error {
 	gpuDeviceConfig, err := dev.toVz()
 	if err != nil {
@@ -199,7 +211,7 @@ func (dev *VirtioRng) AddToVirtualMachineConfig(vmConfig *VirtualMachineConfigur
 	return nil
 }
 
-// https://developer.apple.com/documentation/virtualization/running_linux_in_a_virtual_machine?language=objc#:~:text=Configure%20the%20Serial%20Port%20Device%20for%20Standard%20In%20and%20Out
+// https://developer.apple.com/documentation/virtualization/running_linux_in_a_virtual_machine#3880009
 func setRawMode(f *os.File) error {
 	// Get settings for terminal
 	attr, _ := unix.IoctlGetTermios(int(f.Fd()), unix.TIOCGETA)

pkg/vf/vm.go

@@ -12,11 +12,28 @@ type VirtualMachine struct {
 	vfConfig *VirtualMachineConfiguration
 }
 
+var PlatformType string
+
 func NewVirtualMachine(vmConfig config.VirtualMachine) (*VirtualMachine, error) {
 	vfConfig, err := NewVirtualMachineConfiguration(&vmConfig)
 	if err != nil {
 		return nil, err
 	}
+
+	if macosBootloader, ok := vmConfig.Bootloader.(*config.MacOSBootloader); ok {
+		platformConfig, err := NewMacPlatformConfiguration(macosBootloader.MachineIdentifierPath, macosBootloader.HardwareModelPath, macosBootloader.AuxImagePath)
+
+		PlatformType = "macos"
+
+		if err != nil {
+			return nil, err
+		}
+
+		vfConfig.SetPlatformVirtualMachineConfiguration(platformConfig)
+	} else {
+		PlatformType = "linux"
+	}
+
 	return &VirtualMachine{
 		vfConfig: vfConfig,
 	}, nil

pkg/vf/vm_amd64.go

@@ -0,0 +1,20 @@
+package vf
+
+import (
+	"fmt"
+
+	"github.com/Code-Hex/vz/v3"
+	"github.com/crc-org/vfkit/pkg/config"
+)
+
+func NewMacPlatformConfiguration(_, _, _ string) (vz.PlatformConfiguration, error) {
+	return nil, fmt.Errorf("running macOS guests is only supported on ARM devices")
+}
+
+func toVzMacOSBootloader(_ *config.MacOSBootloader) (vz.BootLoader, error) {
+	return nil, fmt.Errorf("running macOS guests is only supported on ARM devices")
+}
+
+func newMacGraphicsDeviceConfiguration(_ *VirtioGPU) (vz.GraphicsDeviceConfiguration, error) {
+	return nil, fmt.Errorf("running macOS guests is only supported on ARM devices")
+}