Adding POSTCOPY support

.buildkite/rust-vmm-ci-tests.json

@@ -30,7 +30,7 @@
     },
     {
       "test_name": "unittests-gnu-all-without-xen",
-      "command": "cargo test --workspace --no-default-features --features test-utils,vhost-vsock,vhost-kern,vhost-vdpa,vhost-net,vhost-user,vhost-user-frontend,vhost-user-backend",
+      "command": "cargo test --workspace --no-default-features --features test-utils,vhost-vsock,vhost-kern,vhost-vdpa,vhost-net,vhost-user,vhost-user-frontend,vhost-user-backend,postcopy",
       "platform": [
         "x86_64",
         "aarch64"
@@ -46,7 +46,7 @@
     },
     {
       "test_name": "unittests-musl-all-without-xen",
-      "command": "cargo test --workspace --target {target_platform}-unknown-linux-musl --no-default-features --features test-utils,vhost-vsock,vhost-kern,vhost-vdpa,vhost-net,vhost-user,vhost-user-frontend,vhost-user-backend",
+      "command": "cargo test --workspace --target {target_platform}-unknown-linux-musl --no-default-features --features test-utils,vhost-vsock,vhost-kern,vhost-vdpa,vhost-net,vhost-user,vhost-user-frontend,vhost-user-backend,postcopy",
       "platform": [
         "x86_64",
         "aarch64"
@@ -62,7 +62,7 @@
     },
     {
       "test_name": "clippy-all-without-xen",
-      "command": "cargo clippy --workspace --bins --examples --benches --all-targets --no-default-features --features test-utils,vhost-vsock,vhost-kern,vhost-vdpa,vhost-net,vhost-user,vhost-user-frontend,vhost-user-backend -- -D warnings -D clippy::undocumented_unsafe_blocks",
+      "command": "cargo clippy --workspace --bins --examples --benches --all-targets --no-default-features --features test-utils,vhost-vsock,vhost-kern,vhost-vdpa,vhost-net,vhost-user,vhost-user-frontend,vhost-user-backend,postcopy -- -D warnings -D clippy::undocumented_unsafe_blocks",
       "platform": [
         "x86_64",
         "aarch64"
@@ -78,7 +78,7 @@
     },
     {
       "test_name": "check-warnings-all-without-xen",
-      "command": "RUSTFLAGS=\"-D warnings\" cargo check --all-targets --workspace --no-default-features --features test-utils,vhost-vsock,vhost-kern,vhost-vdpa,vhost-net,vhost-user,vhost-user-frontend,vhost-user-backend",
+      "command": "RUSTFLAGS=\"-D warnings\" cargo check --all-targets --workspace --no-default-features --features test-utils,vhost-vsock,vhost-kern,vhost-vdpa,vhost-net,vhost-user,vhost-user-frontend,vhost-user-backend,postcopy",
       "platform": [
         "x86_64",
         "aarch64"

coverage_config_x86_64.json

@@ -1,5 +1,5 @@
 {
   "coverage_score": 72.12,
   "exclude_path": "vhost/src/vhost_kern/",
-  "crate_features": "vhost/vhost-user-frontend,vhost/vhost-user-backend"
+  "crate_features": "vhost/vhost-user-frontend,vhost/vhost-user-backend,vhost-user-backend/postcopy"
 }

vhost-user-backend/Cargo.toml

@@ -10,10 +10,12 @@ license = "Apache-2.0"
 
 [features]
 xen = ["vm-memory/xen", "vhost/xen"]
+postcopy = ["vhost/postcopy", "userfaultfd"]
 
 [dependencies]
 libc = "0.2.39"
 log = "0.4.17"
+userfaultfd = { version = "0.7.0", optional = true }
 vhost = { path = "../vhost", version = "0.10", features = ["vhost-user-backend"] }
 virtio-bindings = "0.2.1"
 virtio-queue = "0.11.0"

vhost-user-backend/README.md

@@ -98,6 +98,16 @@ impl VhostUserBackendMut for VhostUserService {
 }
 ```
 
+## Postcopy support
+
+To enabled POSTCOPY_* messages support there is a `postcopy` feature.
+Due to how Xen handles memory mappings the `postcopy` feature is not compatible
+with `xen` feature. Enabling both at the same time will result in a compilation error.
+
+`postcopy` feature enables optional `userfaultfd` dependency in order to create and
+interact with `userfaultfd` object. This requires access permission to `/dev/userfaultfd`
+file from the backend.
+
 ## Xen support
 
 Supporting Xen requires special handling while mapping the guest memory. The

vhost-user-backend/src/handler.rs

@@ -6,10 +6,14 @@
 use std::error;
 use std::fs::File;
 use std::io;
+#[cfg(feature = "postcopy")]
+use std::os::fd::FromRawFd;
 use std::os::unix::io::AsRawFd;
 use std::sync::Arc;
 use std::thread;
 
+#[cfg(feature = "postcopy")]
+use userfaultfd::{Uffd, UffdBuilder};
 use vhost::vhost_user::message::{
     VhostTransferStateDirection, VhostTransferStatePhase, VhostUserConfigFlags,
     VhostUserMemoryRegion, VhostUserProtocolFeatures, VhostUserSingleMemoryRegion,
@@ -71,6 +75,8 @@ impl error::Error for VhostUserHandlerError {}
 pub type VhostUserHandlerResult<T> = std::result::Result<T, VhostUserHandlerError>;
 
 struct AddrMapping {
+    #[cfg(feature = "postcopy")]
+    local_addr: u64,
     vmm_addr: u64,
     size: u64,
     gpa_base: u64,
@@ -89,6 +95,8 @@ pub struct VhostUserHandler<T: VhostUserBackend> {
     mappings: Vec<AddrMapping>,
     atomic_mem: GM<T::Bitmap>,
     vrings: Vec<T::Vring>,
+    #[cfg(feature = "postcopy")]
+    uffd: Option<Uffd>,
     worker_threads: Vec<thread::JoinHandle<VringEpollResult<()>>>,
 }
 
@@ -148,6 +156,8 @@ where
             mappings: Vec::new(),
             atomic_mem,
             vrings,
+            #[cfg(feature = "postcopy")]
+            uffd: None,
             worker_threads,
         })
     }
@@ -280,20 +290,21 @@ where
         let mut mappings: Vec<AddrMapping> = Vec::new();
 
         for (region, file) in ctx.iter().zip(files) {
-            regions.push(
-                GuestRegionMmap::new(
-                    region.mmap_region(file)?,
-                    GuestAddress(region.guest_phys_addr),
-                )
-                .map_err(|e| {
-                    VhostUserError::ReqHandlerError(io::Error::new(io::ErrorKind::Other, e))
-                })?,
-            );
+            let guest_region = GuestRegionMmap::new(
+                region.mmap_region(file)?,
+                GuestAddress(region.guest_phys_addr),
+            )
+            .map_err(|e| {
+                VhostUserError::ReqHandlerError(io::Error::new(io::ErrorKind::Other, e))
+            })?;
             mappings.push(AddrMapping {
+                #[cfg(feature = "postcopy")]
+                local_addr: guest_region.as_ptr() as u64,
                 vmm_addr: region.user_addr,
                 size: region.memory_size,
                 gpa_base: region.guest_phys_addr,
             });
+            regions.push(guest_region);
         }
 
         let mem = GuestMemoryMmap::from_regions(regions).map_err(|e| {
@@ -570,6 +581,14 @@ where
             })?,
         );
 
+        let addr_mapping = AddrMapping {
+            #[cfg(feature = "postcopy")]
+            local_addr: guest_region.as_ptr() as u64,
+            vmm_addr: region.user_addr,
+            size: region.memory_size,
+            gpa_base: region.guest_phys_addr,
+        };
+
         let mem = self
             .atomic_mem
             .memory()
@@ -586,11 +605,7 @@ where
                 VhostUserError::ReqHandlerError(io::Error::new(io::ErrorKind::Other, e))
             })?;
 
-        self.mappings.push(AddrMapping {
-            vmm_addr: region.user_addr,
-            size: region.memory_size,
-            gpa_base: region.guest_phys_addr,
-        });
+        self.mappings.push(addr_mapping);
 
         Ok(())
     }
@@ -634,6 +649,68 @@ where
             .check_device_state()
             .map_err(VhostUserError::ReqHandlerError)
     }
+
+    #[cfg(feature = "postcopy")]
+    fn postcopy_advice(&mut self) -> VhostUserResult<File> {
+        let mut uffd_builder = UffdBuilder::new();
+
+        let uffd = uffd_builder
+            .close_on_exec(true)
+            .non_blocking(true)
+            .user_mode_only(false)
+            .create()
+            .map_err(|e| {
+                VhostUserError::ReqHandlerError(io::Error::new(io::ErrorKind::Other, e))
+            })?;
+
+        // We need to duplicate the uffd fd because we need both
+        // to return File with fd and store fd inside uffd.
+        //
+        // SAFETY:
+        // We know that uffd is correctly created.
+        // This means fd inside uffd is also a valid fd.
+        // Duplicating a valid fd is safe.
+        let uffd_dup = unsafe { libc::dup(uffd.as_raw_fd()) };
+        if uffd_dup < 0 {
+            return Err(VhostUserError::ReqHandlerError(io::Error::last_os_error()));
+        }
+
+        // SAFETY:
+        // We know that uffd_dup is a valid fd.
+        let uffd_file = unsafe { File::from_raw_fd(uffd_dup) };
+
+        self.uffd = Some(uffd);
+
+        Ok(uffd_file)
+    }
+
+    #[cfg(feature = "postcopy")]
+    fn postcopy_listen(&mut self) -> VhostUserResult<()> {
+        let Some(ref uffd) = self.uffd else {
+            return Err(VhostUserError::ReqHandlerError(io::Error::new(
+                io::ErrorKind::Other,
+                "No registered UFFD handler",
+            )));
+        };
+
+        for mapping in self.mappings.iter() {
+            uffd.register(
+                mapping.local_addr as *mut libc::c_void,
+                mapping.size as usize,
+            )
+            .map_err(|e| {
+                VhostUserError::ReqHandlerError(io::Error::new(io::ErrorKind::Other, e))
+            })?;
+        }
+
+        Ok(())
+    }
+
+    #[cfg(feature = "postcopy")]
+    fn postcopy_end(&mut self) -> VhostUserResult<()> {
+        self.uffd = None;
+        Ok(())
+    }
 }
 
 impl<T: VhostUserBackend> Drop for VhostUserHandler<T> {

vhost-user-backend/src/lib.rs

@@ -33,6 +33,12 @@ pub use self::vring::{
     VringMutex, VringRwLock, VringState, VringStateGuard, VringStateMutGuard, VringT,
 };
 
+/// Due to the way `xen` handles memory mappings we can not combine it with
+/// `postcopy` feature which relies on persistent memory mappings. Thus we
+/// disallow enabling both features at the same time.
+#[cfg(all(feature = "postcopy", feature = "xen"))]
+compile_error!("Both `postcopy` and `xen` features can not be enabled at the same time.");
+
 /// An alias for `GuestMemoryAtomic<GuestMemoryMmap<B>>` to simplify code.
 type GM<B> = GuestMemoryAtomic<GuestMemoryMmap<B>>;
 

vhost-user-backend/tests/vhost-user-server.rs

@@ -292,3 +292,34 @@ fn vhost_user_get_inflight(path: &Path, barrier: Arc<Barrier>) {
 fn test_vhost_user_get_inflight() {
     vhost_user_server(vhost_user_get_inflight);
 }
+
+#[cfg(feature = "postcopy")]
+fn vhost_user_postcopy_advise(path: &Path, barrier: Arc<Barrier>) {
+    let mut frontend = setup_frontend(path, barrier);
+    let _uffd_file = frontend.postcopy_advise().unwrap();
+}
+
+#[cfg(feature = "postcopy")]
+fn vhost_user_postcopy_listen(path: &Path, barrier: Arc<Barrier>) {
+    let mut frontend = setup_frontend(path, barrier);
+    let _uffd_file = frontend.postcopy_advise().unwrap();
+    frontend.postcopy_listen().unwrap();
+}
+
+#[cfg(feature = "postcopy")]
+fn vhost_user_postcopy_end(path: &Path, barrier: Arc<Barrier>) {
+    let mut frontend = setup_frontend(path, barrier);
+    let _uffd_file = frontend.postcopy_advise().unwrap();
+    frontend.postcopy_listen().unwrap();
+    frontend.postcopy_end().unwrap();
+}
+
+// These tests need an access to the `/dev/userfaultfd`
+// in order to pass.
+#[cfg(feature = "postcopy")]
+#[test]
+fn test_vhost_user_postcopy() {
+    vhost_user_server(vhost_user_postcopy_advise);
+    vhost_user_server(vhost_user_postcopy_listen);
+    vhost_user_server(vhost_user_postcopy_end);
+}

vhost/Cargo.toml

@@ -24,6 +24,7 @@ vhost-user = []
 vhost-user-frontend = ["vhost-user"]
 vhost-user-backend = ["vhost-user"]
 xen = ["vm-memory/xen"]
+postcopy = []
 
 [dependencies]
 bitflags = "2.4"

vhost/README.md

@@ -31,6 +31,12 @@ Frontend is the application that shares its virtqueues, backend is the consumer
 of the virtqueues. Frontend and backend can be either a client (i.e. connecting)
 or server (listening) in the socket communication.
 
+## Postcopy support
+
+To enabled POSTCOPY_* messages support there is a `postcopy` feature.
+Due to how Xen handles memory mappings the `postcopy` feature is not compatible
+with `xen` feature. Enabling both at the same time will result in a compilation error.
+
 ## Xen support
 
 Supporting Xen requires special handling while mapping the guest memory. The

vhost/src/lib.rs

@@ -51,6 +51,12 @@ pub mod vhost_user;
 #[cfg(feature = "vhost-vsock")]
 pub mod vsock;
 
+/// Due to the way `xen` handles memory mappings we can not combine it with
+/// `postcopy` feature which relies on persistent memory mappings. Thus we
+/// disallow enabling both features at the same time.
+#[cfg(all(feature = "postcopy", feature = "xen"))]
+compile_error!("Both `postcopy` and `xen` features can not be enabled at the same time.");
+
 /// Error codes for vhost operations
 #[derive(Debug)]
 pub enum Error {