Extend virtio-gpu to support Venus on macOS

Makefile

@@ -32,7 +32,7 @@ ifeq ($(NET),1)
 endif
 ifeq ($(EFI),1)
 	VARIANT = -efi
-	FEATURE_FLAGS := --features efi
+	FEATURE_FLAGS := --features efi,gpu
 	BUILD_INIT = 0
 endif
 
@@ -86,6 +86,9 @@ ifeq ($(OS),Linux)
 	patchelf --set-soname $(KRUN_SONAME_$(OS)) --output $(LIBRARY_RELEASE_$(OS)) target/release/$(KRUN_BASE_$(OS))
 else
 ifeq ($(EFI),1)
+ifeq ($(OS),Darwin)
+	install_name_tool -id libkrun-efi.dylib target/release/libkrun.dylib
+endif
 	mv target/release/libkrun.dylib target/release/$(KRUN_BASE_$(OS))
 endif
 	cp target/release/$(KRUN_BASE_$(OS)) $(LIBRARY_RELEASE_$(OS))

src/arch/src/aarch64/macos/gic.rs

@@ -11,7 +11,7 @@ pub enum Error {}
 type Result<T> = result::Result<T, Error>;
 
 /// Trait for GIC devices.
-pub trait GICDevice {
+pub trait GICDevice: Send {
     /// Returns an array with GIC device properties
     fn device_properties(&self) -> &[u64];
 

src/devices/src/virtio/device.rs

@@ -125,12 +125,12 @@ pub trait VirtioDevice: AsAny + Send {
     }
 }
 
-pub trait VmmExitObserver {
+pub trait VmmExitObserver: Send {
     /// Callback to finish processing or cleanup the device resources
     fn on_vmm_exit(&mut self) {}
 }
 
-impl<F: Fn()> VmmExitObserver for F {
+impl<F: Fn() + Send> VmmExitObserver for F {
     fn on_vmm_exit(&mut self) {
         self()
     }

src/devices/src/virtio/gpu/device.rs

@@ -17,6 +17,8 @@ use super::defs::uapi::virtio_gpu_config;
 use super::worker::Worker;
 use crate::legacy::Gic;
 use crate::Error as DeviceError;
+#[cfg(target_os = "macos")]
+use hvf::MemoryMapping;
 
 // Control queue.
 pub(crate) const CTL_INDEX: usize = 0;
@@ -26,6 +28,7 @@ pub(crate) const CUR_INDEX: usize = 1;
 // Supported features.
 pub(crate) const AVAIL_FEATURES: u64 = 1u64 << uapi::VIRTIO_F_VERSION_1
     | 1u64 << uapi::VIRTIO_GPU_F_VIRGL
+    | 1u64 << uapi::VIRTIO_GPU_F_RESOURCE_UUID
     | 1u64 << uapi::VIRTIO_GPU_F_RESOURCE_BLOB
     | 1u64 << uapi::VIRTIO_GPU_F_CONTEXT_INIT;
 
@@ -45,10 +48,16 @@ pub struct Gpu {
     irq_line: Option<u32>,
     pub(crate) sender: Option<Sender<u64>>,
     virgl_flags: u32,
+    #[cfg(target_os = "macos")]
+    map_sender: Sender<MemoryMapping>,
 }
 
 impl Gpu {
-    pub(crate) fn with_queues(queues: Vec<VirtQueue>, virgl_flags: u32) -> super::Result<Gpu> {
+    pub(crate) fn with_queues(
+        queues: Vec<VirtQueue>,
+        virgl_flags: u32,
+        #[cfg(target_os = "macos")] map_sender: Sender<MemoryMapping>,
+    ) -> super::Result<Gpu> {
         let mut queue_events = Vec::new();
         for _ in 0..queues.len() {
             queue_events
@@ -74,15 +83,25 @@ impl Gpu {
             irq_line: None,
             sender: None,
             virgl_flags,
+            #[cfg(target_os = "macos")]
+            map_sender,
         })
     }
 
-    pub fn new(virgl_flags: u32) -> super::Result<Gpu> {
+    pub fn new(
+        virgl_flags: u32,
+        #[cfg(target_os = "macos")] map_sender: Sender<MemoryMapping>,
+    ) -> super::Result<Gpu> {
         let queues: Vec<VirtQueue> = defs::QUEUE_SIZES
             .iter()
             .map(|&max_size| VirtQueue::new(max_size))
             .collect();
-        Self::with_queues(queues, virgl_flags)
+        Self::with_queues(
+            queues,
+            virgl_flags,
+            #[cfg(target_os = "macos")]
+            map_sender,
+        )
     }
 
     pub fn id(&self) -> &str {
@@ -268,6 +287,8 @@ impl VirtioDevice for Gpu {
             self.irq_line,
             shm_region,
             self.virgl_flags,
+            #[cfg(target_os = "macos")]
+            self.map_sender.clone(),
         );
         worker.run();
 

src/devices/src/virtio/gpu/virtio_gpu.rs

@@ -1,16 +1,27 @@
 use std::collections::BTreeMap;
 use std::env;
+#[cfg(target_os = "linux")]
 use std::os::fd::AsRawFd;
 use std::path::PathBuf;
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::{Arc, Mutex};
 
+#[cfg(target_os = "macos")]
+use crossbeam_channel::{unbounded, Sender};
+#[cfg(target_os = "macos")]
+use hvf::MemoryMapping;
 use libc::c_void;
+#[cfg(target_os = "macos")]
+use rutabaga_gfx::RUTABAGA_MEM_HANDLE_TYPE_APPLE;
 use rutabaga_gfx::{
     ResourceCreate3D, ResourceCreateBlob, Rutabaga, RutabagaBuilder, RutabagaChannel,
     RutabagaFence, RutabagaFenceHandler, RutabagaIovec, Transfer3D, RUTABAGA_CHANNEL_TYPE_WAYLAND,
+    RUTABAGA_MAP_CACHE_MASK,
+};
+#[cfg(target_os = "linux")]
+use rutabaga_gfx::{
     RUTABAGA_MAP_ACCESS_MASK, RUTABAGA_MAP_ACCESS_READ, RUTABAGA_MAP_ACCESS_RW,
-    RUTABAGA_MAP_ACCESS_WRITE, RUTABAGA_MAP_CACHE_MASK, RUTABAGA_MEM_HANDLE_TYPE_OPAQUE_FD,
+    RUTABAGA_MAP_ACCESS_WRITE, RUTABAGA_MEM_HANDLE_TYPE_OPAQUE_FD,
 };
 use utils::eventfd::EventFd;
 use vm_memory::{GuestAddress, GuestMemory, GuestMemoryMmap, VolatileSlice};
@@ -21,6 +32,7 @@ use super::protocol::{
     GpuResponse, GpuResponsePlaneInfo, VirtioGpuResult, VIRTIO_GPU_BLOB_FLAG_CREATE_GUEST_HANDLE,
     VIRTIO_GPU_BLOB_MEM_HOST3D,
 };
+
 use super::{GpuError, Result};
 use crate::legacy::Gic;
 use crate::virtio::gpu::protocol::VIRTIO_GPU_FLAG_INFO_RING_IDX;
@@ -89,6 +101,8 @@ pub struct VirtioGpu {
     rutabaga: Rutabaga,
     resources: BTreeMap<u32, VirtioGpuResource>,
     fence_state: Arc<Mutex<FenceState>>,
+    #[cfg(target_os = "macos")]
+    map_sender: Sender<MemoryMapping>,
 }
 
 impl VirtioGpu {
@@ -149,6 +163,7 @@ impl VirtioGpu {
         })
     }
 
+    #[allow(clippy::too_many_arguments)]
     pub fn new(
         mem: GuestMemoryMmap,
         queue_ctl: Arc<Mutex<VirtQueue>>,
@@ -157,6 +172,7 @@ impl VirtioGpu {
         intc: Option<Arc<Mutex<Gic>>>,
         irq_line: Option<u32>,
         virgl_flags: u32,
+        #[cfg(target_os = "macos")] map_sender: Sender<MemoryMapping>,
     ) -> Self {
         let xdg_runtime_dir = match env::var("XDG_RUNTIME_DIR") {
             Ok(dir) => dir,
@@ -199,6 +215,8 @@ impl VirtioGpu {
             rutabaga,
             resources: Default::default(),
             fence_state,
+            #[cfg(target_os = "macos")]
+            map_sender,
         }
     }
 
@@ -469,6 +487,7 @@ impl VirtioGpu {
     /// rutabaga as ExternalMapping.
     /// When sandboxing is enabled, external_blob is set and opaque fds must be mapped in the
     /// hypervisor process by Vulkano using metadata provided by Rutabaga::vulkan_info().
+    #[cfg(target_os = "linux")]
     pub fn resource_map_blob(
         &mut self,
         resource_id: u32,
@@ -525,8 +544,62 @@ impl VirtioGpu {
             map_info: map_info & RUTABAGA_MAP_CACHE_MASK,
         })
     }
+    #[cfg(target_os = "macos")]
+    pub fn resource_map_blob(
+        &mut self,
+        resource_id: u32,
+        shm_region: &VirtioShmRegion,
+        offset: u64,
+    ) -> VirtioGpuResult {
+        let resource = self
+            .resources
+            .get_mut(&resource_id)
+            .ok_or(ErrInvalidResourceId)?;
+
+        let map_info = self.rutabaga.map_info(resource_id).map_err(|_| ErrUnspec)?;
+        let map_ptr = self.rutabaga.map_ptr(resource_id).map_err(|_| ErrUnspec)?;
+
+        if let Ok(export) = self.rutabaga.export_blob(resource_id) {
+            if export.handle_type == RUTABAGA_MEM_HANDLE_TYPE_APPLE {
+                if offset + resource.size > shm_region.size as u64 {
+                    error!("mapping DOES NOT FIT");
+                    return Err(ErrUnspec);
+                }
+
+                let guest_addr = shm_region.guest_addr + offset;
+                debug!(
+                    "mapping: map_ptr={:x}, guest_addr={:x}, size={}",
+                    map_ptr, guest_addr, resource.size
+                );
+
+                let (reply_sender, reply_receiver) = unbounded();
+                self.map_sender
+                    .send(MemoryMapping::AddMapping(
+                        reply_sender,
+                        map_ptr,
+                        guest_addr,
+                        resource.size,
+                    ))
+                    .unwrap();
+                if !reply_receiver.recv().unwrap() {
+                    return Err(ErrUnspec);
+                }
+            } else {
+                return Err(ErrUnspec);
+            }
+        } else {
+            return Err(ErrUnspec);
+        }
+
+        resource.shmem_offset = Some(offset);
+        // Access flags not a part of the virtio-gpu spec.
+        Ok(OkMapInfo {
+            map_info: map_info & RUTABAGA_MAP_CACHE_MASK,
+        })
+    }
 
     /// Uses the hypervisor to unmap the blob resource.
+    #[cfg(target_os = "linux")]
     pub fn resource_unmap_blob(
         &mut self,
         resource_id: u32,
@@ -557,6 +630,42 @@ impl VirtioGpu {
 
         resource.shmem_offset = None;
 
+        Ok(OkNoData)
+    }
+    #[cfg(target_os = "macos")]
+    pub fn resource_unmap_blob(
+        &mut self,
+        resource_id: u32,
+        shm_region: &VirtioShmRegion,
+    ) -> VirtioGpuResult {
+        let resource = self
+            .resources
+            .get_mut(&resource_id)
+            .ok_or(ErrInvalidResourceId)?;
+
+        debug!("resource_unmap_blob");
+        let shmem_offset = resource.shmem_offset.ok_or(ErrUnspec)?;
+
+        let guest_addr = shm_region.guest_addr + shmem_offset;
+        debug!(
+            "unmapping: guest_addr={:x}, size={}",
+            guest_addr, resource.size
+        );
+
+        let (reply_sender, reply_receiver) = unbounded();
+        self.map_sender
+            .send(MemoryMapping::RemoveMapping(
+                reply_sender,
+                guest_addr,
+                resource.size,
+            ))
+            .unwrap();
+        if !reply_receiver.recv().unwrap() {
+            return Err(ErrUnspec);
+        }
+
+        resource.shmem_offset = None;
+
         Ok(OkNoData)
     }
 }

src/devices/src/virtio/gpu/worker.rs

@@ -4,6 +4,10 @@ use std::sync::{Arc, Mutex};
 use std::{result, thread};
 
 use crossbeam_channel::Receiver;
+#[cfg(target_os = "macos")]
+use crossbeam_channel::Sender;
+#[cfg(target_os = "macos")]
+use hvf::MemoryMapping;
 use rutabaga_gfx::{
     ResourceCreate3D, ResourceCreateBlob, RutabagaFence, Transfer3D,
     RUTABAGA_PIPE_BIND_RENDER_TARGET, RUTABAGA_PIPE_TEXTURE_2D,
@@ -32,6 +36,8 @@ pub struct Worker {
     irq_line: Option<u32>,
     shm_region: VirtioShmRegion,
     virgl_flags: u32,
+    #[cfg(target_os = "macos")]
+    map_sender: Sender<MemoryMapping>,
 }
 
 impl Worker {
@@ -46,6 +52,7 @@ impl Worker {
         irq_line: Option<u32>,
         shm_region: VirtioShmRegion,
         virgl_flags: u32,
+        #[cfg(target_os = "macos")] map_sender: Sender<MemoryMapping>,
     ) -> Self {
         Self {
             receiver,
@@ -57,6 +64,8 @@ impl Worker {
             irq_line,
             shm_region,
             virgl_flags,
+            #[cfg(target_os = "macos")]
+            map_sender,
         }
     }
 
@@ -73,6 +82,8 @@ impl Worker {
             self.intc.clone(),
             self.irq_line,
             self.virgl_flags,
+            #[cfg(target_os = "macos")]
+            self.map_sender.clone(),
         );
 
         loop {

src/hvf/Cargo.toml

@@ -5,5 +5,6 @@ authors = ["Sergio Lopez <slp@sinrega.org>"]
 edition = "2021"
 
 [dependencies]
+crossbeam-channel = "0.5"
 log = "0.4.0"
-env_logger = "0.9.0"
+env_logger = "0.9.0"