From e20441b80dcff26419e4f1d8f3279a7702ad3420 Mon Sep 17 00:00:00 2001
From: Hanna Czenczek <hreitz@redhat.com>
Date: Mon, 12 May 2025 17:10:01 +0200
Subject: [PATCH 01/11] Add IoMemory trait
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The existing `GuestMemory` trait is insufficient for representing
virtual memory, as it does not allow specifying the required access
permissions.

Its focus on all guest memory implementations consisting of a relatively
small number of regions is also unsuited for paged virtual memory with a
potentially very lage set of non-continuous mappings.

The new `IoMemory` trait in contrast provides only a small number of
methods that keep the implementing type’s internal structure more
opaque, and every access needs to be accompanied by the required
permissions.

Signed-off-by: Hanna Czenczek <hreitz@redhat.com>
---
 src/guest_memory.rs | 164 ++++++++++++++++++++++++++++++++++++++++++--
 src/lib.rs          |   2 +-
 2 files changed, 159 insertions(+), 7 deletions(-)

diff --git a/src/guest_memory.rs b/src/guest_memory.rs
index 7136db35..bd1328df 100644
--- a/src/guest_memory.rs
+++ b/src/guest_memory.rs
@@ -52,7 +52,7 @@ use std::sync::atomic::Ordering;
 use std::sync::Arc;
 
 use crate::address::{Address, AddressValue};
-use crate::bitmap::MS;
+use crate::bitmap::{BitmapSlice, MS};
 use crate::bytes::{AtomicAccess, Bytes};
 use crate::io::{ReadVolatile, WriteVolatile};
 use crate::volatile_memory::{self, VolatileSlice};
@@ -541,11 +541,7 @@ impl<'a, M: GuestMemory + ?Sized> GuestMemorySliceIterator<'a, M> {
     /// when it encounters an error after successfully producing at least one slice.
     /// Return an error if requesting the first slice returns an error.
     pub fn stop_on_error(self) -> Result<impl Iterator<Item = VolatileSlice<'a, MS<'a, M>>>> {
-        let mut peek = self.peekable();
-        if let Some(err) = peek.next_if(Result::is_err) {
-            return Err(err.unwrap_err());
-        }
-        Ok(peek.filter_map(Result::ok))
+        <Self as IoMemorySliceIterator<'a, MS<'a, M>>>::stop_on_error(self)
     }
 }
 
@@ -566,6 +562,11 @@ impl<'a, M: GuestMemory + ?Sized> Iterator for GuestMemorySliceIterator<'a, M> {
     }
 }
 
+impl<'a, M: GuestMemory + ?Sized> IoMemorySliceIterator<'a, MS<'a, M>>
+    for GuestMemorySliceIterator<'a, M>
+{
+}
+
 /// This iterator continues to return `None` when exhausted.
 ///
 /// [`<Self as Iterator>::next()`](GuestMemorySliceIterator::next) sets `self.count` to 0 when
@@ -723,6 +724,157 @@ impl<T: GuestMemory + ?Sized> Bytes<GuestAddress> for T {
     }
 }
 
+/// Permissions for accessing virtual memory.
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+#[repr(u8)]
+pub enum Permissions {
+    /// No permissions
+    No = 0b00,
+    /// Read-only
+    Read = 0b01,
+    /// Write-only
+    Write = 0b10,
+    /// Allow both reading and writing
+    ReadWrite = 0b11,
+}
+
+impl Permissions {
+    /// Convert the numerical representation into the enum.
+    ///
+    /// # Panics
+    ///
+    /// Panics if `raw` is not a valid representation of any `Permissions` variant.
+    fn from_repr(raw: u8) -> Self {
+        use Permissions::*;
+
+        match raw {
+            value if value == No as u8 => No,
+            value if value == Read as u8 => Read,
+            value if value == Write as u8 => Write,
+            value if value == ReadWrite as u8 => ReadWrite,
+            _ => panic!("{raw:x} is not a valid raw Permissions value"),
+        }
+    }
+
+    /// Check whether the permissions `self` allow the given `access`.
+    pub fn allow(&self, access: Self) -> bool {
+        *self & access == access
+    }
+}
+
+impl std::ops::BitOr for Permissions {
+    type Output = Permissions;
+
+    /// Return the union of `self` and `rhs`.
+    fn bitor(self, rhs: Permissions) -> Self::Output {
+        Self::from_repr(self as u8 | rhs as u8)
+    }
+}
+
+impl std::ops::BitAnd for Permissions {
+    type Output = Permissions;
+
+    /// Return the intersection of `self` and `rhs`.
+    fn bitand(self, rhs: Permissions) -> Self::Output {
+        Self::from_repr(self as u8 & rhs as u8)
+    }
+}
+
+/// Represents virtual I/O memory.
+///
+/// `IoMemory` is generally backed by some “physical” `GuestMemory`, which then consists for
+/// `GuestMemoryRegion` objects.  However, the mapping from I/O virtual addresses (IOVAs) to
+/// physical addresses may be arbitrarily fragmented.  Translation is done via an IOMMU.
+///
+/// Note in contrast to `GuestMemory`:
+/// - Any IOVA range may consist of arbitrarily many underlying ranges in physical memory.
+/// - Accessing an IOVA requires passing the intended access mode, and the IOMMU will check whether
+///   the given access mode is permitted for the given IOVA.
+/// - The translation result for a given IOVA may change over time (i.e. the physical address
+///   associated with an IOVA may change).
+pub trait IoMemory {
+    /// Underlying `GuestMemory` type.
+    type PhysicalMemory: GuestMemory + ?Sized;
+
+    /// Return `true` if `addr..(addr + count)` is accessible with `access`.
+    fn check_range(&self, addr: GuestAddress, count: usize, access: Permissions) -> bool;
+
+    /// Returns a [`VolatileSlice`](struct.VolatileSlice.html) of `count` bytes starting at
+    /// `addr`.
+    ///
+    /// Note that because of the fragmented nature of virtual memory, it can easily happen that the
+    /// range `[addr, addr + count)` is not backed by a continuous region in our own virtual
+    /// memory, which will make generating the slice impossible.
+    ///
+    /// The iterator’s items are wrapped in [`Result`], i.e. there may be errors reported on
+    /// individual items.  If there is no such error, the cumulative length of all items will be
+    /// equal to `count`.  Any error will end iteration immediately, i.e. there are no items past
+    /// the first error.
+    ///
+    /// If `count` is 0, an empty iterator will be returned.
+    fn get_slices<'a>(
+        &'a self,
+        addr: GuestAddress,
+        count: usize,
+        access: Permissions,
+    ) -> Result<impl IoMemorySliceIterator<'a, MS<'a, Self::PhysicalMemory>>>;
+
+    /// If this virtual memory is just a plain `GuestMemory` object underneath without an IOMMU
+    /// translation layer in between, return that `GuestMemory` object.
+    fn physical_memory(&self) -> Option<&Self::PhysicalMemory> {
+        None
+    }
+}
+
+/// Iterates over [`VolatileSlice`]s that together form an I/O memory area.
+///
+/// Returned by [`IoMemory::get_slices()`].
+pub trait IoMemorySliceIterator<'a, B: BitmapSlice>:
+    Iterator<Item = Result<VolatileSlice<'a, B>>> + Sized
+{
+    /// Adapts this [`IoMemorySliceIterator`] to return `None` (e.g. gracefully terminate) when it
+    /// encounters an error after successfully producing at least one slice.
+    /// Return an error if requesting the first slice returns an error.
+    fn stop_on_error(self) -> Result<impl Iterator<Item = VolatileSlice<'a, B>>> {
+        let mut peek = self.peekable();
+        if let Some(err) = peek.next_if(Result::is_err) {
+            return Err(err.unwrap_err());
+        }
+        Ok(peek.filter_map(Result::ok))
+    }
+}
+
+/// Allow accessing every [`GuestMemory`] via [`IoMemory`].
+///
+/// [`IoMemory`] is a generalization of [`GuestMemory`]: Every object implementing the former is a
+/// subset of an object implementing the latter (there always is an underlying [`GuestMemory`]),
+/// with an opaque internal mapping on top, e.g. provided by an IOMMU.
+///
+/// Every [`GuestMemory`] is therefore trivially also an [`IoMemory`], assuming a complete identity
+/// mapping (which we must assume, so that accessing such objects via either trait will yield the
+/// same result): Basically, all [`IoMemory`] methods are implemented as trivial wrappers around
+/// the same [`GuestMemory`] methods (if available), discarding the `access` parameter.
+impl<M: GuestMemory + ?Sized> IoMemory for M {
+    type PhysicalMemory = M;
+
+    fn check_range(&self, addr: GuestAddress, count: usize, _access: Permissions) -> bool {
+        <M as GuestMemory>::check_range(self, addr, count)
+    }
+
+    fn get_slices<'a>(
+        &'a self,
+        addr: GuestAddress,
+        count: usize,
+        _access: Permissions,
+    ) -> Result<impl IoMemorySliceIterator<'a, MS<'a, Self::PhysicalMemory>>> {
+        Ok(<M as GuestMemory>::get_slices(self, addr, count))
+    }
+
+    fn physical_memory(&self) -> Option<&Self::PhysicalMemory> {
+        Some(self)
+    }
+}
+
 #[cfg(test)]
 mod tests {
     #![allow(clippy::undocumented_unsafe_blocks)]
diff --git a/src/lib.rs b/src/lib.rs
index 3d4ad1e9..ef92e47b 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -53,7 +53,7 @@ pub use endian::{Be16, Be32, Be64, BeSize, Le16, Le32, Le64, LeSize};
 pub mod guest_memory;
 pub use guest_memory::{
     Error as GuestMemoryError, FileOffset, GuestAddress, GuestAddressSpace, GuestMemory,
-    GuestUsize, MemoryRegionAddress, Result as GuestMemoryResult,
+    GuestUsize, IoMemory, MemoryRegionAddress, Permissions, Result as GuestMemoryResult,
 };
 
 pub mod region;

From e25ce529c4a4c6981c7e0571f02d481a6ae8ce88 Mon Sep 17 00:00:00 2001
From: Hanna Czenczek <hreitz@redhat.com>
Date: Mon, 12 May 2025 17:10:04 +0200
Subject: [PATCH 02/11] Implement Bytes on IoMemory instead of GuestMemory

Rust only allows us to give one trait the blanket implementations for
`Bytes`.

We want `IoMemory` to be our primary external interface becaue it has
users specify the access permissions they need, and because we can (and
do) provide a blanket `IoMemory` implementation for all `GuestMemory`
types.  Also, while `IoMemory` (as the more general trait) only has a
restricted interface when compared to `GuestMemory`, this interface is
enough to implement `Bytes`; notably, accesses to `IoMemory` require
specifying the access mode, which is naturally trivial for `Bytes`
methods like `read()` or `write()`.

Signed-off-by: Hanna Czenczek <hreitz@redhat.com>
---
 src/guest_memory.rs | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/src/guest_memory.rs b/src/guest_memory.rs
index bd1328df..88051806 100644
--- a/src/guest_memory.rs
+++ b/src/guest_memory.rs
@@ -573,17 +573,22 @@ impl<'a, M: GuestMemory + ?Sized> IoMemorySliceIterator<'a, MS<'a, M>>
 /// returning `None`, ensuring that it will only return `None` from that point on.
 impl<M: GuestMemory + ?Sized> FusedIterator for GuestMemorySliceIterator<'_, M> {}
 
-impl<T: GuestMemory + ?Sized> Bytes<GuestAddress> for T {
+/// Allow accessing [`IoMemory`] (and [`GuestMemory`]) objects via [`Bytes`].
+///
+/// Thanks to the [blanket implementation of `IoMemory` for all `GuestMemory`
+/// types](../io_memory/trait.IoMemory.html#impl-IoMemory-for-M), this blanket implementation
+/// extends to all [`GuestMemory`] types.
+impl<T: IoMemory + ?Sized> Bytes<GuestAddress> for T {
     type E = Error;
 
     fn write(&self, buf: &[u8], addr: GuestAddress) -> Result<usize> {
-        self.get_slices(addr, buf.len())
+        self.get_slices(addr, buf.len(), Permissions::Write)?
             .stop_on_error()?
             .try_fold(0, |acc, slice| Ok(acc + slice.write(&buf[acc..], 0)?))
     }
 
     fn read(&self, buf: &mut [u8], addr: GuestAddress) -> Result<usize> {
-        self.get_slices(addr, buf.len())
+        self.get_slices(addr, buf.len(), Permissions::Read)?
             .stop_on_error()?
             .try_fold(0, |acc, slice| Ok(acc + slice.read(&mut buf[acc..], 0)?))
     }
@@ -649,7 +654,7 @@ impl<T: GuestMemory + ?Sized> Bytes<GuestAddress> for T {
     where
         F: ReadVolatile,
     {
-        self.get_slices(addr, count)
+        self.get_slices(addr, count, Permissions::Write)?
             .stop_on_error()?
             .try_fold(0, |acc, slice| {
                 Ok(acc + slice.read_volatile_from(0, src, slice.len())?)
@@ -679,7 +684,7 @@ impl<T: GuestMemory + ?Sized> Bytes<GuestAddress> for T {
     where
         F: WriteVolatile,
     {
-        self.get_slices(addr, count)
+        self.get_slices(addr, count, Permissions::Read)?
             .stop_on_error()?
             .try_fold(0, |acc, slice| {
                 // For a non-RAM region, reading could have side effects, so we
@@ -706,7 +711,7 @@ impl<T: GuestMemory + ?Sized> Bytes<GuestAddress> for T {
     fn store<O: AtomicAccess>(&self, val: O, addr: GuestAddress, order: Ordering) -> Result<()> {
         // No need to check past the first iterator item: It either has the size of `O`, then there
         // can be no further items; or it does not, and then `VolatileSlice::store()` will fail.
-        self.get_slices(addr, size_of::<O>())
+        self.get_slices(addr, size_of::<O>(), Permissions::Write)?
             .next()
             .unwrap()? // count > 0 never produces an empty iterator
             .store(val, 0, order)
@@ -716,7 +721,7 @@ impl<T: GuestMemory + ?Sized> Bytes<GuestAddress> for T {
     fn load<O: AtomicAccess>(&self, addr: GuestAddress, order: Ordering) -> Result<O> {
         // No need to check past the first iterator item: It either has the size of `O`, then there
         // can be no further items; or it does not, and then `VolatileSlice::store()` will fail.
-        self.get_slices(addr, size_of::<O>())
+        self.get_slices(addr, size_of::<O>(), Permissions::Read)?
             .next()
             .unwrap()? // count > 0 never produces an empty iterator
             .load(0, order)

From e544029ef8cdfe242c527c145e329c07e84d0312 Mon Sep 17 00:00:00 2001
From: Hanna Czenczek <hreitz@redhat.com>
Date: Fri, 15 Aug 2025 16:21:49 +0200
Subject: [PATCH 03/11] Implement GuestAddressSpace for IoMemory
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

We want a trait like `GuestAddressSpace` for `IoMemory`, but just
duplicating it into an `IoAddressSpace` trait is not so easy:

We could rename the current `GuestAddressSpace` to `IoAddressSpace` and
require `M: IoMemory` (instead of `M: GuestMemory`), and then define
`GuestAddressSpace` as:

```rust
pub trait GuestAddressSpace: IoAddressSpace<M: GuestMemory> {}

impl<AS: IoAddressSpace> GuestAddressSpace for AS
where
    AS::M: GuestMemory,
{}
```

But doing just this would break all existing `GuestAddressSpace` users,
as they’d now need to import `IoAddressSpace` to use `memory()`.

(Re-)Adding `GuestAddressSpace::memory()` as

```rust
fn memory(&self) -> <Self as IoAddressSpace>::T {
    IoAddressSpace::memory(self)
}
```

also doesn’t (just) work, as it gets the compiler confused which
`memory()` to use (between `GuestAddressSpace` and `IoAddressSpace`), so
the `IoAddressSpace::memory()` method would need to be called
differently.  However, I would find that a bit silly, and it would also
then later require changes if the user wants to switch from
`GuestMemory` to `IoMemory`.

Instead just changing the `GuestAddressSpace::M: GuestMemory`
requirement to `M: IoMemory` seems easier:
- All callers that just use the `Bytes` interface remain completely
  unchanged,
- It does break users that actually need the `GuestMemory` interface,
  but from what I have seen, that is only the case for
  `vhost::vhost_kern`.  There, we can simply require that
  `<AS as GuestAddressSpace>::M: GuestMemory`.

Signed-off-by: Hanna Czenczek <hreitz@redhat.com>
---
 src/guest_memory.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/guest_memory.rs b/src/guest_memory.rs
index 88051806..0599d5d0 100644
--- a/src/guest_memory.rs
+++ b/src/guest_memory.rs
@@ -223,7 +223,7 @@ impl FileOffset {
 /// ```
 pub trait GuestAddressSpace: Clone {
     /// The type that will be used to access guest memory.
-    type M: GuestMemory;
+    type M: IoMemory;
 
     /// A type that provides access to the memory.
     type T: Clone + Deref<Target = Self::M>;
@@ -234,7 +234,7 @@ pub trait GuestAddressSpace: Clone {
     fn memory(&self) -> Self::T;
 }
 
-impl<M: GuestMemory> GuestAddressSpace for &M {
+impl<M: IoMemory> GuestAddressSpace for &M {
     type M = M;
     type T = Self;
 
@@ -243,7 +243,7 @@ impl<M: GuestMemory> GuestAddressSpace for &M {
     }
 }
 
-impl<M: GuestMemory> GuestAddressSpace for Rc<M> {
+impl<M: IoMemory> GuestAddressSpace for Rc<M> {
     type M = M;
     type T = Self;
 
@@ -252,7 +252,7 @@ impl<M: GuestMemory> GuestAddressSpace for Rc<M> {
     }
 }
 
-impl<M: GuestMemory> GuestAddressSpace for Arc<M> {
+impl<M: IoMemory> GuestAddressSpace for Arc<M> {
     type M = M;
     type T = Self;
 

From 36a79d76faeb364c1100cb21daf0c638ec7fbb2b Mon Sep 17 00:00:00 2001
From: Hanna Czenczek <hreitz@redhat.com>
Date: Fri, 15 Aug 2025 16:48:31 +0200
Subject: [PATCH 04/11] Allow any IoMemory for GuestMemoryAtomic

This simply makes `GuestMemoryAtomic` more general.

(However, this change requires the preceding commit that relaxed the
`GuestAddressSpace::M` requirement from `GuestMemory` to `IoMemory`.)

Signed-off-by: Hanna Czenczek <hreitz@redhat.com>
---
 src/atomic.rs | 34 +++++++++++++++++-----------------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/src/atomic.rs b/src/atomic.rs
index 87a2c1e3..be8c5cf7 100644
--- a/src/atomic.rs
+++ b/src/atomic.rs
@@ -2,7 +2,7 @@
 // Copyright (C) 2020 Red Hat, Inc. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-//! A wrapper over an `ArcSwap<GuestMemory>` struct to support RCU-style mutability.
+//! A wrapper over an `ArcSwap<IoMemory>` struct to support RCU-style mutability.
 //!
 //! With the `backend-atomic` feature enabled, simply replacing `GuestMemoryMmap`
 //! with `GuestMemoryAtomic<GuestMemoryMmap>` will enable support for mutable memory maps.
@@ -15,17 +15,17 @@ use arc_swap::{ArcSwap, Guard};
 use std::ops::Deref;
 use std::sync::{Arc, LockResult, Mutex, MutexGuard, PoisonError};
 
-use crate::{GuestAddressSpace, GuestMemory};
+use crate::{GuestAddressSpace, IoMemory};
 
 /// A fast implementation of a mutable collection of memory regions.
 ///
 /// This implementation uses `ArcSwap` to provide RCU-like snapshotting of the memory map:
-/// every update of the memory map creates a completely new `GuestMemory` object, and
+/// every update of the memory map creates a completely new `IoMemory` object, and
 /// readers will not be blocked because the copies they retrieved will be collected once
 /// no one can access them anymore.  Under the assumption that updates to the memory map
 /// are rare, this allows a very efficient implementation of the `memory()` method.
 #[derive(Debug)]
-pub struct GuestMemoryAtomic<M: GuestMemory> {
+pub struct GuestMemoryAtomic<M: IoMemory> {
     // GuestAddressSpace<M>, which we want to implement, is basically a drop-in
     // replacement for &M.  Therefore, we need to pass to devices the `GuestMemoryAtomic`
     // rather than a reference to it.  To obtain this effect we wrap the actual fields
@@ -34,9 +34,9 @@ pub struct GuestMemoryAtomic<M: GuestMemory> {
     inner: Arc<(ArcSwap<M>, Mutex<()>)>,
 }
 
-impl<M: GuestMemory> From<Arc<M>> for GuestMemoryAtomic<M> {
+impl<M: IoMemory> From<Arc<M>> for GuestMemoryAtomic<M> {
     /// create a new `GuestMemoryAtomic` object whose initial contents come from
-    /// the `map` reference counted `GuestMemory`.
+    /// the `map` reference counted `IoMemory`.
     fn from(map: Arc<M>) -> Self {
         let inner = (ArcSwap::new(map), Mutex::new(()));
         GuestMemoryAtomic {
@@ -45,9 +45,9 @@ impl<M: GuestMemory> From<Arc<M>> for GuestMemoryAtomic<M> {
     }
 }
 
-impl<M: GuestMemory> GuestMemoryAtomic<M> {
+impl<M: IoMemory> GuestMemoryAtomic<M> {
     /// create a new `GuestMemoryAtomic` object whose initial contents come from
-    /// the `map` `GuestMemory`.
+    /// the `map` `IoMemory`.
     pub fn new(map: M) -> Self {
         Arc::new(map).into()
     }
@@ -75,7 +75,7 @@ impl<M: GuestMemory> GuestMemoryAtomic<M> {
     }
 }
 
-impl<M: GuestMemory> Clone for GuestMemoryAtomic<M> {
+impl<M: IoMemory> Clone for GuestMemoryAtomic<M> {
     fn clone(&self) -> Self {
         Self {
             inner: self.inner.clone(),
@@ -83,7 +83,7 @@ impl<M: GuestMemory> Clone for GuestMemoryAtomic<M> {
     }
 }
 
-impl<M: GuestMemory> GuestAddressSpace for GuestMemoryAtomic<M> {
+impl<M: IoMemory> GuestAddressSpace for GuestMemoryAtomic<M> {
     type T = GuestMemoryLoadGuard<M>;
     type M = M;
 
@@ -94,14 +94,14 @@ impl<M: GuestMemory> GuestAddressSpace for GuestMemoryAtomic<M> {
 
 /// A guard that provides temporary access to a `GuestMemoryAtomic`.  This
 /// object is returned from the `memory()` method.  It dereference to
-/// a snapshot of the `GuestMemory`, so it can be used transparently to
+/// a snapshot of the `IoMemory`, so it can be used transparently to
 /// access memory.
 #[derive(Debug)]
-pub struct GuestMemoryLoadGuard<M: GuestMemory> {
+pub struct GuestMemoryLoadGuard<M: IoMemory> {
     guard: Guard<Arc<M>>,
 }
 
-impl<M: GuestMemory> GuestMemoryLoadGuard<M> {
+impl<M: IoMemory> GuestMemoryLoadGuard<M> {
     /// Make a clone of the held pointer and returns it.  This is more
     /// expensive than just using the snapshot, but it allows to hold on
     /// to the snapshot outside the scope of the guard.  It also allows
@@ -112,7 +112,7 @@ impl<M: GuestMemory> GuestMemoryLoadGuard<M> {
     }
 }
 
-impl<M: GuestMemory> Clone for GuestMemoryLoadGuard<M> {
+impl<M: IoMemory> Clone for GuestMemoryLoadGuard<M> {
     fn clone(&self) -> Self {
         GuestMemoryLoadGuard {
             guard: Guard::from_inner(Arc::clone(&*self.guard)),
@@ -120,7 +120,7 @@ impl<M: GuestMemory> Clone for GuestMemoryLoadGuard<M> {
     }
 }
 
-impl<M: GuestMemory> Deref for GuestMemoryLoadGuard<M> {
+impl<M: IoMemory> Deref for GuestMemoryLoadGuard<M> {
     type Target = M;
 
     fn deref(&self) -> &Self::Target {
@@ -133,12 +133,12 @@ impl<M: GuestMemory> Deref for GuestMemoryLoadGuard<M> {
 /// possibly after updating the memory map represented by the
 /// `GuestMemoryAtomic` that created the guard.
 #[derive(Debug)]
-pub struct GuestMemoryExclusiveGuard<'a, M: GuestMemory> {
+pub struct GuestMemoryExclusiveGuard<'a, M: IoMemory> {
     parent: &'a GuestMemoryAtomic<M>,
     _guard: MutexGuard<'a, ()>,
 }
 
-impl<M: GuestMemory> GuestMemoryExclusiveGuard<'_, M> {
+impl<M: IoMemory> GuestMemoryExclusiveGuard<'_, M> {
     /// Replace the memory map in the `GuestMemoryAtomic` that created the guard
     /// with the new memory map, `map`.  The lock is then dropped since this
     /// method consumes the guard.

From 9dad343fa41bb4b7b213cbed8f0158162c92d865 Mon Sep 17 00:00:00 2001
From: Hanna Czenczek <hreitz@redhat.com>
Date: Mon, 12 May 2025 17:10:07 +0200
Subject: [PATCH 05/11] Add Iommu trait and Iotlb struct
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The Iommu trait defines an interface for translating virtual addresses
into addresses in an underlying address space.

It is supposed to do so by internally keeping an instance of the Iotlb
type, updating it with mappings whenever necessary (e.g. when
actively invalidated or when there’s an access failure) from some
internal data source (e.g. for a vhost-user IOMMU, the data comes from
the vhost-user front-end by requesting an update).

In a later commit, we are going to provide an implementation of
`IoMemory` that can use an `Iommu` to provide an I/O virtual address
space.

Note that while I/O virtual memory in practice will be organized in
pages, the vhost-user specification makes no mention of a specific page
size or how to obtain it.  Therefore, we cannot really assume any page
size and have to use plain ranges with byte granularity as mappings
instead.

Signed-off-by: Hanna Czenczek <hreitz@redhat.com>
---
 Cargo.toml          |   2 +
 src/atomic.rs       |   7 +-
 src/guest_memory.rs |   6 +
 src/iommu.rs        | 332 ++++++++++++++++++++++++++++++++++++++++++++
 src/lib.rs          |   5 +
 5 files changed, 349 insertions(+), 3 deletions(-)
 create mode 100644 src/iommu.rs

diff --git a/Cargo.toml b/Cargo.toml
index d13f7bcc..2d71cb09 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -16,6 +16,7 @@ default = ["rawfd"]
 backend-bitmap = ["dep:libc", "dep:winapi"]
 backend-mmap = ["dep:libc", "dep:winapi"]
 backend-atomic = ["arc-swap"]
+iommu = ["dep:rangemap"]
 rawfd = ["dep:libc"]
 xen = ["backend-mmap", "bitflags", "vmm-sys-util"]
 
@@ -23,6 +24,7 @@ xen = ["backend-mmap", "bitflags", "vmm-sys-util"]
 libc = { version = "0.2.39", optional = true }
 arc-swap = { version = "1.0.0", optional = true }
 bitflags = { version = "2.4.0", optional = true }
+rangemap = { version = "1.5.1", optional = true }
 thiserror = "2.0.16"
 vmm-sys-util = { version = ">=0.12.1, <=0.15.0", optional = true }
 
diff --git a/src/atomic.rs b/src/atomic.rs
index be8c5cf7..4496d68a 100644
--- a/src/atomic.rs
+++ b/src/atomic.rs
@@ -151,7 +151,7 @@ impl<M: IoMemory> GuestMemoryExclusiveGuard<'_, M> {
 mod tests {
     use super::*;
     use crate::region::tests::{new_guest_memory_collection_from_regions, Collection, MockRegion};
-    use crate::{GuestAddress, GuestMemory, GuestMemoryRegion, GuestUsize};
+    use crate::{GuestAddress, GuestMemory, GuestMemoryRegion, GuestUsize, IoMemory};
 
     type GuestMemoryMmapAtomic = GuestMemoryAtomic<Collection>;
 
@@ -165,7 +165,8 @@ mod tests {
         let mut iterated_regions = Vec::new();
         let gmm = new_guest_memory_collection_from_regions(&regions).unwrap();
         let gm = GuestMemoryMmapAtomic::new(gmm);
-        let mem = gm.memory();
+        let vmem = gm.memory();
+        let mem = vmem.physical_memory().unwrap();
 
         for region in mem.iter() {
             assert_eq!(region.len(), region_size as GuestUsize);
@@ -184,7 +185,7 @@ mod tests {
             .map(|x| (x.0, x.1))
             .eq(iterated_regions.iter().copied()));
 
-        let mem2 = mem.into_inner();
+        let mem2 = vmem.into_inner();
         for region in mem2.iter() {
             assert_eq!(region.len(), region_size as GuestUsize);
         }
diff --git a/src/guest_memory.rs b/src/guest_memory.rs
index 0599d5d0..28d5019a 100644
--- a/src/guest_memory.rs
+++ b/src/guest_memory.rs
@@ -55,6 +55,8 @@ use crate::address::{Address, AddressValue};
 use crate::bitmap::{BitmapSlice, MS};
 use crate::bytes::{AtomicAccess, Bytes};
 use crate::io::{ReadVolatile, WriteVolatile};
+#[cfg(feature = "iommu")]
+use crate::iommu::Error as IommuError;
 use crate::volatile_memory::{self, VolatileSlice};
 use crate::GuestMemoryRegion;
 
@@ -85,6 +87,10 @@ pub enum Error {
     /// The address to be read by `try_access` is outside the address range.
     #[error("The address to be read by `try_access` is outside the address range")]
     GuestAddressOverflow,
+    #[cfg(feature = "iommu")]
+    /// IOMMU translation error
+    #[error("IOMMU failed to translate guest address: {0}")]
+    IommuError(IommuError),
 }
 
 impl From<volatile_memory::Error> for Error {
diff --git a/src/iommu.rs b/src/iommu.rs
new file mode 100644
index 00000000..cc16ea33
--- /dev/null
+++ b/src/iommu.rs
@@ -0,0 +1,332 @@
+// Copyright (C) 2025 Red Hat. All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE-BSD-3-Clause file.
+//
+// SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause
+
+//! Provide an interface for IOMMUs enabling I/O virtual address (IOVA) translation.
+//!
+//! All IOMMUs consist of an IOTLB ([`Iotlb`]), which is backed by a data source that can deliver
+//! all mappings.  For example, for vhost-user, that data source is the vhost-user front-end; i.e.
+//! IOTLB misses require sending a notification to the front-end and awaiting a reply that supplies
+//! the desired mapping.
+
+use crate::{GuestAddress, Permissions};
+use rangemap::RangeMap;
+use std::cmp;
+use std::fmt::Debug;
+use std::num::Wrapping;
+use std::ops::{Deref, Range};
+
+/// Errors associated with IOMMU address translation.
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+    /// Lookup cannot be resolved.
+    #[error(
+        "Cannot translate I/O virtual address range {:#x}+{}: {reason}",
+        iova_range.base.0,
+        iova_range.length,
+    )]
+    CannotResolve {
+        /// IOVA range that could not be resolved
+        iova_range: IovaRange,
+        /// Some human-readable specifics about the reason
+        reason: String,
+    },
+
+    /// Wanted to translate an IOVA range into a single slice, but the range is fragmented.
+    #[error(
+        "Expected {:#x}+{} to be a continuous I/O virtual address range, but only {continuous_length} bytes are",
+        iova_range.base.0,
+        iova_range.length,
+    )]
+    Fragmented {
+        /// Full IOVA range that was to be translated
+        iova_range: IovaRange,
+        /// Length of the continuous head (i.e. the first fragment)
+        continuous_length: usize,
+    },
+
+    /// IOMMU is not configured correctly, and so cannot translate addresses.
+    #[error("IOMMU not configured correctly, cannot operate: {reason}")]
+    IommuMisconfigured {
+        /// Some human-readable specifics about the misconfiguration
+        reason: String,
+    },
+}
+
+/// An IOMMU, allowing translation of I/O virtual addresses (IOVAs).
+///
+/// Generally, `Iommu` implementaions consist of an [`Iotlb`], which is supposed to be consulted
+/// first for lookup requests.  All misses and access failures then should be resolved by looking
+/// up the affected ranges in the actual IOMMU (which has all current mappings) and putting the
+/// results back into the IOTLB.  A subsequent lookup in the IOTLB should result in a full
+/// translation, which can then be returned.
+pub trait Iommu: Debug + Send + Sync {
+    /// `Deref` type associated with the type that internally wraps the `Iotlb`.
+    ///
+    /// For example, the `Iommu` may keep the `Iotlb` wrapped in an `RwLock`, making this type
+    /// `RwLockReadGuard<'a, Iotlb>`.
+    ///
+    /// We need this specific type instead of a plain reference so that [`IotlbIterator`] can
+    /// actually own the reference and prolong its lifetime.
+    type IotlbGuard<'a>: Deref<Target = Iotlb> + 'a
+    where
+        Self: 'a;
+
+    /// Translate the given range for the given access into the underlying address space.
+    ///
+    /// Any translation request is supposed to be fully served by an internal [`Iotlb`] instance.
+    /// Any misses or access failures should result in a lookup in the full IOMMU structures,
+    /// filling the IOTLB with the results, and then repeating the lookup in there.
+    fn translate(
+        &self,
+        iova: GuestAddress,
+        length: usize,
+        access: Permissions,
+    ) -> Result<IotlbIterator<Self::IotlbGuard<'_>>, Error>;
+}
+
+/// Mapping target in an IOMMU/IOTLB.
+///
+/// This is the data to which each entry in an IOMMU/IOTLB maps.
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+struct IommuMapping {
+    /// Difference between the mapped and the IOVA address, i.e. what to add to an IOVA address to
+    /// get the mapped adrress.
+    ///
+    /// We cannot store the more obvious mapped base address for this range because that would
+    /// allow rangemap to wrongfully merge consecutive map entries if they are a duplicate mapping
+    /// (which does happen).  Storing the difference ensures that entries are only merged when they
+    /// are indeed consecutive.
+    ///
+    /// Note that we make no granularity restrictions (i.e. do not operate on a unit like pages),
+    /// so the source and target address may have arbitrary alignment.  That is why both fields
+    /// here need to be separate and we cannot merge the two bits that are `permissions` with this
+    /// base address into a single `u64` field.
+    target_source_diff: Wrapping<u64>,
+    /// Allowed access for the mapped range
+    permissions: Permissions,
+}
+
+/// Provides an IOTLB.
+///
+/// The IOTLB caches IOMMU mappings.  It must be preemptively updated whenever mappings are
+/// restricted or removed; in contrast, adding mappings or making them more permissive does not
+/// require preemptive updates, as subsequent accesses that violate the previous (more restrictive)
+/// permissions will trigger TLB misses or access failures, which is then supposed to result in an
+/// update from the outer [`Iommu`] object that performs the translation.
+#[derive(Debug, Default)]
+pub struct Iotlb {
+    /// Mappings of which we know.
+    ///
+    /// Note that the vhost(-user) specification makes no mention of a specific page size, even
+    /// though in practice the IOVA address space will be organized in terms of pages.  However, we
+    /// cannot really rely on that (or any specific page size; it could be 4k, the guest page size,
+    /// or the host page size), so we need to be able to handle continuous ranges of any
+    /// granularity.
+    tlb: RangeMap<u64, IommuMapping>,
+}
+
+/// Iterates over a range of valid IOTLB mappings that together constitute a continuous range in
+/// I/O virtual address space.
+///
+/// Returned by [`Iotlb::lookup()`] and [`Iommu::translate()`] in case translation was successful
+/// (i.e. the whole requested range is mapped and permits the given access).
+#[derive(Clone, Debug)]
+pub struct IotlbIterator<D: Deref<Target = Iotlb>> {
+    /// IOTLB that provides these mapings
+    iotlb: D,
+    /// I/O virtual address range left to iterate over
+    range: Range<u64>,
+    /// Requested access permissions
+    access: Permissions,
+}
+
+/// Representation of an IOVA memory range (i.e. in the I/O virtual address space).
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct IovaRange {
+    /// IOVA base address
+    pub base: GuestAddress,
+    /// Length (in bytes) of this range
+    pub length: usize,
+}
+
+/// Representation of a mapped memory range in the underlying address space.
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct MappedRange {
+    /// Base address in the underlying address space
+    pub base: GuestAddress,
+    /// Length (in bytes) of this mapping
+    pub length: usize,
+}
+
+/// Lists the subranges in I/O virtual address space that turned out to not be accessible when
+/// trying to access an IOVA range.
+#[derive(Clone, Debug)]
+pub struct IotlbFails {
+    /// Subranges not mapped at all
+    pub misses: Vec<IovaRange>,
+    /// Subranges that are mapped, but do not allow the requested access mode
+    pub access_fails: Vec<IovaRange>,
+}
+
+impl IommuMapping {
+    /// Create a new mapping.
+    fn new(source_base: u64, target_base: u64, permissions: Permissions) -> Self {
+        IommuMapping {
+            target_source_diff: Wrapping(target_base) - Wrapping(source_base),
+            permissions,
+        }
+    }
+
+    /// Map the given source address (IOVA) to its corresponding target address.
+    fn map(&self, iova: u64) -> u64 {
+        (Wrapping(iova) + self.target_source_diff).0
+    }
+
+    /// Return the permissions for this mapping.
+    fn permissions(&self) -> Permissions {
+        self.permissions
+    }
+}
+
+impl Iotlb {
+    /// Create a new empty instance.
+    pub fn new() -> Self {
+        Default::default()
+    }
+
+    /// Change the mapping of the given IOVA range.
+    pub fn set_mapping(
+        &mut self,
+        iova: GuestAddress,
+        map_to: GuestAddress,
+        length: usize,
+        perm: Permissions,
+    ) -> Result<(), Error> {
+        // Soft TODO: We may want to evict old entries here once the TLB grows to a certain size,
+        // but that will require LRU book-keeping.  However, this is left for the future, because:
+        // - this TLB is not implemented in hardware, so we do not really have strong entry count
+        //   constraints, and
+        // - it seems like at least Linux guests invalidate mappings often, automatically limiting
+        //   our entry count.
+
+        let mapping = IommuMapping::new(iova.0, map_to.0, perm);
+        self.tlb.insert(iova.0..(iova.0 + length as u64), mapping);
+
+        Ok(())
+    }
+
+    /// Remove any mapping in the given IOVA range.
+    pub fn invalidate_mapping(&mut self, iova: GuestAddress, length: usize) {
+        self.tlb.remove(iova.0..(iova.0 + length as u64));
+    }
+
+    /// Remove all mappings.
+    pub fn invalidate_all(&mut self) {
+        self.tlb.clear();
+    }
+
+    /// Perform a lookup for the given range and the given `access` mode.
+    ///
+    /// If the whole range is mapped and accessible, return an iterator over all mappings.
+    ///
+    /// If any part of the range is not mapped or does not permit the given access mode, return an
+    /// `Err(_)` that contains a list of all such subranges.
+    pub fn lookup<D: Deref<Target = Iotlb>>(
+        this: D,
+        iova: GuestAddress,
+        length: usize,
+        access: Permissions,
+    ) -> Result<IotlbIterator<D>, IotlbFails> {
+        let full_range = iova.0..(iova.0 + length as u64);
+
+        let has_misses = this.tlb.gaps(&full_range).any(|_| true);
+        let has_access_fails = this
+            .tlb
+            .overlapping(full_range.clone())
+            .any(|(_, mapping)| !mapping.permissions().allow(access));
+
+        if has_misses || has_access_fails {
+            let misses = this
+                .tlb
+                .gaps(&full_range)
+                .map(|range| {
+                    // Gaps are always cut down to the range given to `gaps()`
+                    debug_assert!(range.start >= full_range.start && range.end <= full_range.end);
+                    range.try_into().unwrap()
+                })
+                .collect::<Vec<_>>();
+
+            let access_fails = this
+                .tlb
+                .overlapping(full_range.clone())
+                .filter(|(_, mapping)| !mapping.permissions().allow(access))
+                .map(|(range, _)| {
+                    let start = cmp::max(range.start, full_range.start);
+                    let end = cmp::min(range.end, full_range.end);
+                    (start..end).try_into().unwrap()
+                })
+                .collect::<Vec<_>>();
+
+            return Err(IotlbFails {
+                misses,
+                access_fails,
+            });
+        }
+
+        Ok(IotlbIterator {
+            iotlb: this,
+            range: full_range,
+            access,
+        })
+    }
+}
+
+impl<D: Deref<Target = Iotlb>> Iterator for IotlbIterator<D> {
+    /// Addresses in the underlying address space
+    type Item = MappedRange;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        // Note that we can expect the whole IOVA range to be mapped with the right access flags.
+        // The `IotlbIterator` is created by `Iotlb::lookup()` only if the whole range is mapped
+        // accessibly; we have a permanent reference to `Iotlb`, so the range cannot be invalidated
+        // in the meantime.
+        // Another note: It is tempting to have `IotlbIterator` wrap around the
+        // `rangemap::Overlapping` iterator, but that just takes a (lifetimed) reference to the
+        // map, not an owned reference (like RwLockReadGuard), which we want to use; so using that
+        // would probably require self-referential structs.
+
+        if self.range.is_empty() {
+            return None;
+        }
+
+        let (range, mapping) = self.iotlb.tlb.get_key_value(&self.range.start).unwrap();
+
+        assert!(mapping.permissions().allow(self.access));
+
+        let mapping_iova_start = self.range.start;
+        let mapping_iova_end = cmp::min(self.range.end, range.end);
+        let mapping_len = mapping_iova_end - mapping_iova_start;
+
+        self.range.start = mapping_iova_end;
+
+        Some(MappedRange {
+            base: GuestAddress(mapping.map(mapping_iova_start)),
+            length: mapping_len.try_into().unwrap(),
+        })
+    }
+}
+
+impl TryFrom<Range<u64>> for IovaRange {
+    type Error = <u64 as TryFrom<usize>>::Error;
+
+    fn try_from(range: Range<u64>) -> Result<Self, Self::Error> {
+        Ok(IovaRange {
+            base: GuestAddress(range.start),
+            length: (range.end - range.start).try_into()?,
+        })
+    }
+}
diff --git a/src/lib.rs b/src/lib.rs
index ef92e47b..9f0fed56 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -64,6 +64,11 @@ pub use region::{
 pub mod io;
 pub use io::{ReadVolatile, WriteVolatile};
 
+#[cfg(feature = "iommu")]
+pub mod iommu;
+#[cfg(feature = "iommu")]
+pub use iommu::{Iommu, Iotlb};
+
 #[cfg(feature = "backend-mmap")]
 pub mod mmap;
 

From f2234a50f0ebb34cc6fba5230cce824e94f14f38 Mon Sep 17 00:00:00 2001
From: Hanna Czenczek <hreitz@redhat.com>
Date: Mon, 12 May 2025 17:10:10 +0200
Subject: [PATCH 06/11] Add IommuMemory

This `IoMemory` type provides an I/O virtual address space by adding an
IOMMU translation layer to an underlying `GuestMemory` object.

Signed-off-by: Hanna Czenczek <hreitz@redhat.com>
---
 src/iommu.rs | 218 ++++++++++++++++++++++++++++++++++++++++++++++++++-
 src/lib.rs   |   2 +-
 2 files changed, 218 insertions(+), 2 deletions(-)

diff --git a/src/iommu.rs b/src/iommu.rs
index cc16ea33..c168c07f 100644
--- a/src/iommu.rs
+++ b/src/iommu.rs
@@ -12,12 +12,17 @@
 //! IOTLB misses require sending a notification to the front-end and awaiting a reply that supplies
 //! the desired mapping.
 
-use crate::{GuestAddress, Permissions};
+use crate::guest_memory::{
+    Error as GuestMemoryError, GuestMemorySliceIterator, IoMemorySliceIterator,
+    Result as GuestMemoryResult,
+};
+use crate::{bitmap, GuestAddress, GuestMemory, IoMemory, Permissions, VolatileSlice};
 use rangemap::RangeMap;
 use std::cmp;
 use std::fmt::Debug;
 use std::num::Wrapping;
 use std::ops::{Deref, Range};
+use std::sync::Arc;
 
 /// Errors associated with IOMMU address translation.
 #[derive(Debug, thiserror::Error)]
@@ -172,6 +177,22 @@ pub struct IotlbFails {
     pub access_fails: Vec<IovaRange>,
 }
 
+/// [`IoMemory`] type that consists of an underlying [`GuestMemory`] object plus an [`Iommu`].
+///
+/// The underlying [`GuestMemory`] is basically the physical memory, and the [`Iommu`] translates
+/// the I/O virtual address space that `IommuMemory` provides into that underlying physical address
+/// space.
+#[derive(Debug, Default)]
+pub struct IommuMemory<M: GuestMemory, I: Iommu> {
+    /// Physical memory
+    inner: M,
+    /// IOMMU to translate IOVAs into physical addresses
+    iommu: Arc<I>,
+    /// Whether the IOMMU is even to be used or not; disabling it makes this a pass-through to
+    /// `inner`.
+    use_iommu: bool,
+}
+
 impl IommuMapping {
     /// Create a new mapping.
     fn new(source_base: u64, target_base: u64, permissions: Permissions) -> Self {
@@ -330,3 +351,198 @@ impl TryFrom<Range<u64>> for IovaRange {
         })
     }
 }
+
+impl<M: GuestMemory, I: Iommu> IommuMemory<M, I> {
+    /// Create a new `IommuMemory` instance.
+    pub fn new(inner: M, iommu: I, use_iommu: bool) -> Self {
+        IommuMemory {
+            inner,
+            iommu: Arc::new(iommu),
+            use_iommu,
+        }
+    }
+
+    /// Create a new version of `self` with the underlying physical memory replaced.
+    ///
+    /// Note that the inner `Arc` reference to the IOMMU is cloned, i.e. both the existing and the
+    /// new `IommuMemory` object will share an IOMMU instance.  (The `use_iommu` flag however is
+    /// copied, so is independent between the two instances.)
+    pub fn inner_replaced(&self, inner: M) -> Self {
+        IommuMemory {
+            inner,
+            iommu: Arc::clone(&self.iommu),
+            use_iommu: self.use_iommu,
+        }
+    }
+
+    /// Enable or disable the IOMMU.
+    ///
+    /// Disabling the IOMMU switches to pass-through mode, where every access is done directly on
+    /// the underlying physical memory.
+    pub fn set_iommu_enabled(&mut self, enabled: bool) {
+        self.use_iommu = enabled;
+    }
+
+    /// Return a reference to the IOMMU.
+    pub fn iommu(&self) -> &Arc<I> {
+        &self.iommu
+    }
+
+    /// Return a reference to the inner physical memory object.
+    pub fn inner(&self) -> &M {
+        &self.inner
+    }
+}
+
+impl<M: GuestMemory + Clone, I: Iommu> Clone for IommuMemory<M, I> {
+    fn clone(&self) -> Self {
+        IommuMemory {
+            inner: self.inner.clone(),
+            iommu: Arc::clone(&self.iommu),
+            use_iommu: self.use_iommu,
+        }
+    }
+}
+
+impl<M: GuestMemory, I: Iommu> IoMemory for IommuMemory<M, I> {
+    type PhysicalMemory = M;
+
+    fn check_range(&self, addr: GuestAddress, count: usize, access: Permissions) -> bool {
+        if !self.use_iommu {
+            return self.inner.check_range(addr, count);
+        }
+
+        let Ok(mut translated_iter) = self.iommu.translate(addr, count, access) else {
+            return false;
+        };
+
+        translated_iter.all(|translated| self.inner.check_range(translated.base, translated.length))
+    }
+
+    fn get_slices<'a>(
+        &'a self,
+        addr: GuestAddress,
+        count: usize,
+        access: Permissions,
+    ) -> GuestMemoryResult<impl IoMemorySliceIterator<'a, bitmap::MS<'a, M>>> {
+        if self.use_iommu {
+            IommuMemorySliceIterator::virt(self, addr, count, access)
+                .map_err(GuestMemoryError::IommuError)
+        } else {
+            Ok(IommuMemorySliceIterator::phys(self, addr, count))
+        }
+    }
+
+    fn physical_memory(&self) -> Option<&Self::PhysicalMemory> {
+        if self.use_iommu {
+            None
+        } else {
+            Some(&self.inner)
+        }
+    }
+}
+
+/// Iterates over [`VolatileSlice`]s that together form an area in an `IommuMemory`.
+///
+/// Returned by [`IommuMemory::get_slices()`]
+#[derive(Debug)]
+pub struct IommuMemorySliceIterator<'a, M: GuestMemory, I: Iommu + 'a> {
+    /// Underlying physical memory (i.e. not the `IommuMemory`)
+    phys_mem: &'a M,
+    /// IOMMU translation result (i.e. remaining physical regions to visit)
+    translation: Option<IotlbIterator<I::IotlbGuard<'a>>>,
+    /// Iterator in the currently visited physical region
+    current_translated_iter: Option<GuestMemorySliceIterator<'a, M>>,
+}
+
+impl<'a, M: GuestMemory, I: Iommu> IommuMemorySliceIterator<'a, M, I> {
+    /// Create an iterator over the physical region `[addr, addr + count)`.
+    ///
+    /// “Physical” means that the IOMMU is not used to translate this address range.  The resulting
+    /// iterator is effectively the same as would be returned by [`GuestMemory::get_slices()`] on
+    /// the underlying physical memory for the given address range.
+    fn phys(mem: &'a IommuMemory<M, I>, addr: GuestAddress, count: usize) -> Self {
+        IommuMemorySliceIterator {
+            phys_mem: &mem.inner,
+            translation: None,
+            current_translated_iter: Some(mem.inner.get_slices(addr, count)),
+        }
+    }
+
+    /// Create an iterator over the IOVA region `[addr, addr + count)`.
+    ///
+    /// This address range is translated using the IOMMU, and the resulting mappings are then
+    /// separately visited via [`GuestMemory::get_slices()`].
+    fn virt(
+        mem: &'a IommuMemory<M, I>,
+        addr: GuestAddress,
+        count: usize,
+        access: Permissions,
+    ) -> Result<Self, Error> {
+        let translation = mem.iommu.translate(addr, count, access)?;
+        Ok(IommuMemorySliceIterator {
+            phys_mem: &mem.inner,
+            translation: Some(translation),
+            current_translated_iter: None,
+        })
+    }
+
+    /// Helper function for [`<Self as Iterator>::next()`](IommuMemorySliceIterator::next).
+    ///
+    /// Get the next slice and update the internal state.  If there is an element left in
+    /// `self.current_translated_iter`, return that; otherwise, move to the next mapping left in
+    /// `self.translation` until there are no more mappings left.
+    ///
+    /// If both fields are `None`, always return `None`.
+    ///
+    /// # Safety
+    ///
+    /// This function never resets `self.current_translated_iter` or `self.translation` to `None`,
+    /// particularly not in case of error; calling this function with these fields not reset after
+    /// an error is ill-defined, so the caller must check the return value, and in case of an
+    /// error, reset these fields to `None`.
+    ///
+    /// (This is why this function exists, so this reset can happen in a single central location.)
+    unsafe fn do_next(
+        &mut self,
+    ) -> Option<GuestMemoryResult<VolatileSlice<'a, bitmap::MS<'a, M>>>> {
+        loop {
+            if let Some(item) = self
+                .current_translated_iter
+                .as_mut()
+                .and_then(|iter| iter.next())
+            {
+                return Some(item);
+            }
+
+            let next_mapping = self.translation.as_mut()?.next()?;
+            self.current_translated_iter = Some(
+                self.phys_mem
+                    .get_slices(next_mapping.base, next_mapping.length),
+            );
+        }
+    }
+}
+
+impl<'a, M: GuestMemory, I: Iommu> Iterator for IommuMemorySliceIterator<'a, M, I> {
+    type Item = GuestMemoryResult<VolatileSlice<'a, bitmap::MS<'a, M>>>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        // SAFETY:
+        // We reset `current_translated_iter` and `translation` to `None` in case of error
+        match unsafe { self.do_next() } {
+            Some(Ok(slice)) => Some(Ok(slice)),
+            other => {
+                // On error (or end), clear both so iteration remains stopped
+                self.current_translated_iter.take();
+                self.translation.take();
+                other
+            }
+        }
+    }
+}
+
+impl<'a, M: GuestMemory, I: Iommu> IoMemorySliceIterator<'a, bitmap::MS<'a, M>>
+    for IommuMemorySliceIterator<'a, M, I>
+{
+}
diff --git a/src/lib.rs b/src/lib.rs
index 9f0fed56..4df2ecee 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -67,7 +67,7 @@ pub use io::{ReadVolatile, WriteVolatile};
 #[cfg(feature = "iommu")]
 pub mod iommu;
 #[cfg(feature = "iommu")]
-pub use iommu::{Iommu, Iotlb};
+pub use iommu::{Iommu, IommuMemory, Iotlb};
 
 #[cfg(feature = "backend-mmap")]
 pub mod mmap;

From ce7dc832092620b83f82305872bee44f7390a9c3 Mon Sep 17 00:00:00 2001
From: Hanna Czenczek <hreitz@redhat.com>
Date: Tue, 13 May 2025 12:01:41 +0200
Subject: [PATCH 07/11] mmap: Wrap MmapRegion in Arc<>
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The vhost-user-backend crate will need to be able to modify all existing
memory regions to use the VMM user address instead of the guest physical
address once the IOMMU feature is switched on, and vice versa.  To do
so, it needs to be able to modify regions’ base address.

Because `GuestMemoryMmap` stores regions wrapped in an `Arc<_>`, we
cannot mutate them after they have been put into the `GuestMemoryMmap`
object; and `MmapRegion` itself is by its nature not clonable.  So to
modify the regions’ base addresses, we need some way to create a new
`GuestRegionMmap` referencing the same `MmapRegion` as another one, but
with a different base address.

We can do that by having `GuestRegionMmap` wrap its `MmapRegion` in an
`Arc`, and adding a method to return a reference to that `Arc`, and a
method to construct a `GuestRegionMmap` object from such a cloned `Arc.`

Signed-off-by: Hanna Czenczek <hreitz@redhat.com>
---
 src/mmap/mod.rs | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/src/mmap/mod.rs b/src/mmap/mod.rs
index 7d133bdf..f18ea5c5 100644
--- a/src/mmap/mod.rs
+++ b/src/mmap/mod.rs
@@ -15,6 +15,7 @@
 use std::borrow::Borrow;
 use std::ops::Deref;
 use std::result;
+use std::sync::Arc;
 
 use crate::address::Address;
 use crate::bitmap::{Bitmap, BS};
@@ -54,7 +55,7 @@ pub use windows::MmapRegion;
 /// in the virtual address space of the calling process.
 #[derive(Debug)]
 pub struct GuestRegionMmap<B = ()> {
-    mapping: MmapRegion<B>,
+    mapping: Arc<MmapRegion<B>>,
     guest_base: GuestAddress,
 }
 
@@ -62,7 +63,7 @@ impl<B> Deref for GuestRegionMmap<B> {
     type Target = MmapRegion<B>;
 
     fn deref(&self) -> &MmapRegion<B> {
-        &self.mapping
+        self.mapping.as_ref()
     }
 }
 
@@ -71,6 +72,11 @@ impl<B: Bitmap> GuestRegionMmap<B> {
     ///
     /// Returns `None` if `guest_base` + `mapping.len()` would overflow.
     pub fn new(mapping: MmapRegion<B>, guest_base: GuestAddress) -> Option<Self> {
+        Self::with_arc(Arc::new(mapping), guest_base)
+    }
+
+    /// Same as [`Self::new()`], but takes an `Arc`-wrapped `mapping`.
+    pub fn with_arc(mapping: Arc<MmapRegion<B>>, guest_base: GuestAddress) -> Option<Self> {
         guest_base
             .0
             .checked_add(mapping.size() as u64)
@@ -79,6 +85,16 @@ impl<B: Bitmap> GuestRegionMmap<B> {
                 guest_base,
             })
     }
+
+    /// Return a reference to the inner `Arc<MmapRegion>` (as opposed to
+    /// [`.deref()`](Self::deref()), which does not reference the `Arc`).
+    ///
+    /// The returned reference can be cloned to construct a new `GuestRegionMmap` with a different
+    /// base address (e.g. when switching between memory address spaces based on the guest physical
+    /// address vs. the VMM userspace virtual address).
+    pub fn get_mapping(&self) -> &Arc<MmapRegion<B>> {
+        &self.mapping
+    }
 }
 
 #[cfg(not(feature = "xen"))]

From 7d42461a7229d6df150c19f3e1800b7ec37f08e1 Mon Sep 17 00:00:00 2001
From: Hanna Czenczek <hreitz@redhat.com>
Date: Tue, 29 Jul 2025 12:28:31 +0200
Subject: [PATCH 08/11] IoMemory: Add IOVA-space bitmap

Without an IOMMU, we have direct access to guest physical addresses
(GPAs).  In order to track our writes to guest memory (during
migration), we log them into dirty bitmaps, and a page's bit index is
its GPA divided by the page size.

With an IOMMU, however, we no longer know the GPA, instead we operate on
I/O virtual addresses (IOVAs) and VMM user-space addresses (VUAs).
Here, the dirty bitmap bit index is the IOVA divided by the page size.

`IoMemory` types contain an internal "physical" memory type that
operates on these VUAs (`IoMemory::PhysicalMemory).  Any bitmap
functionality that this internal type may already have (e.g.
`GuestMemoryMmap` does) cannot be used for dirty bitmap tracking with an
IOMMU because they would use the VUA, but we need to use the IOVA, and
this information is not available on that lower layer.

Therefore, `IoMemory` itself needs to support bitmaps separately from
its inner `PhysicalMemory`, which will be used when the IOMMU is in use.
Add an associated `IoMemory::Bitmap` type and add a bitmap object to
`IommuMemory`.  Ensure that writes to memory dirty that bitmap
appropriately:
- In `try_access()`, if write access was requested, dirty the handled
  region of the bitmap after the access is done.
- In `get_slice()`, replace the `VolatileSlice`'s bitmap (which comes
  from the inner `PhysicalMemory`) by the correct slice of our IOVA
  bitmap before returning it.

Signed-off-by: Hanna Czenczek <hreitz@redhat.com>
---
 src/guest_memory.rs    |  14 ++++--
 src/iommu.rs           | 111 +++++++++++++++++++++++++++++++++++++----
 src/volatile_memory.rs |  11 ++++
 3 files changed, 123 insertions(+), 13 deletions(-)

diff --git a/src/guest_memory.rs b/src/guest_memory.rs
index 28d5019a..1f19fc9f 100644
--- a/src/guest_memory.rs
+++ b/src/guest_memory.rs
@@ -52,7 +52,7 @@ use std::sync::atomic::Ordering;
 use std::sync::Arc;
 
 use crate::address::{Address, AddressValue};
-use crate::bitmap::{BitmapSlice, MS};
+use crate::bitmap::{Bitmap, BitmapSlice, BS, MS};
 use crate::bytes::{AtomicAccess, Bytes};
 use crate::io::{ReadVolatile, WriteVolatile};
 #[cfg(feature = "iommu")]
@@ -771,6 +771,11 @@ impl Permissions {
     pub fn allow(&self, access: Self) -> bool {
         *self & access == access
     }
+
+    /// Check whether the permissions `self` include write access.
+    pub fn has_write(&self) -> bool {
+        *self & Permissions::Write == Permissions::Write
+    }
 }
 
 impl std::ops::BitOr for Permissions {
@@ -806,6 +811,8 @@ impl std::ops::BitAnd for Permissions {
 pub trait IoMemory {
     /// Underlying `GuestMemory` type.
     type PhysicalMemory: GuestMemory + ?Sized;
+    /// Dirty bitmap type for tracking writes to the IOVA address space.
+    type Bitmap: Bitmap;
 
     /// Return `true` if `addr..(addr + count)` is accessible with `access`.
     fn check_range(&self, addr: GuestAddress, count: usize, access: Permissions) -> bool;
@@ -828,7 +835,7 @@ pub trait IoMemory {
         addr: GuestAddress,
         count: usize,
         access: Permissions,
-    ) -> Result<impl IoMemorySliceIterator<'a, MS<'a, Self::PhysicalMemory>>>;
+    ) -> Result<impl IoMemorySliceIterator<'a, BS<'a, Self::Bitmap>>>;
 
     /// If this virtual memory is just a plain `GuestMemory` object underneath without an IOMMU
     /// translation layer in between, return that `GuestMemory` object.
@@ -867,6 +874,7 @@ pub trait IoMemorySliceIterator<'a, B: BitmapSlice>:
 /// the same [`GuestMemory`] methods (if available), discarding the `access` parameter.
 impl<M: GuestMemory + ?Sized> IoMemory for M {
     type PhysicalMemory = M;
+    type Bitmap = <M::R as GuestMemoryRegion>::B;
 
     fn check_range(&self, addr: GuestAddress, count: usize, _access: Permissions) -> bool {
         <M as GuestMemory>::check_range(self, addr, count)
@@ -877,7 +885,7 @@ impl<M: GuestMemory + ?Sized> IoMemory for M {
         addr: GuestAddress,
         count: usize,
         _access: Permissions,
-    ) -> Result<impl IoMemorySliceIterator<'a, MS<'a, Self::PhysicalMemory>>> {
+    ) -> Result<impl IoMemorySliceIterator<'a, BS<'a, Self::Bitmap>>> {
         Ok(<M as GuestMemory>::get_slices(self, addr, count))
     }
 
diff --git a/src/iommu.rs b/src/iommu.rs
index c168c07f..cf36c4a8 100644
--- a/src/iommu.rs
+++ b/src/iommu.rs
@@ -12,14 +12,18 @@
 //! IOTLB misses require sending a notification to the front-end and awaiting a reply that supplies
 //! the desired mapping.
 
+use crate::bitmap::{self, Bitmap};
 use crate::guest_memory::{
     Error as GuestMemoryError, GuestMemorySliceIterator, IoMemorySliceIterator,
     Result as GuestMemoryResult,
 };
-use crate::{bitmap, GuestAddress, GuestMemory, IoMemory, Permissions, VolatileSlice};
+use crate::{
+    Address, GuestAddress, GuestMemory, GuestMemoryRegion, GuestUsize, IoMemory, Permissions,
+    VolatileSlice,
+};
 use rangemap::RangeMap;
 use std::cmp;
-use std::fmt::Debug;
+use std::fmt::{self, Debug};
 use std::num::Wrapping;
 use std::ops::{Deref, Range};
 use std::sync::Arc;
@@ -182,7 +186,15 @@ pub struct IotlbFails {
 /// The underlying [`GuestMemory`] is basically the physical memory, and the [`Iommu`] translates
 /// the I/O virtual address space that `IommuMemory` provides into that underlying physical address
 /// space.
-#[derive(Debug, Default)]
+///
+/// Note on memory write tracking (“logging”):
+/// - When the IOMMU is disabled ([`IommuMemory::set_iommu_enabled()`]), writes to memory are
+///   tracked by the underlying [`GuestMemory`] in its bitmap(s).
+/// - When it is enabled, they are instead tracked in the [`IommuMemory`]’s dirty bitmap; the
+///   offset in the bitmap is calculated from the write’s IOVA.
+///
+/// Therefore, this type should only be used when this is the desired behavior (IOVA-based memory
+/// write logging when IOMMU is used).
 pub struct IommuMemory<M: GuestMemory, I: Iommu> {
     /// Physical memory
     inner: M,
@@ -191,6 +203,8 @@ pub struct IommuMemory<M: GuestMemory, I: Iommu> {
     /// Whether the IOMMU is even to be used or not; disabling it makes this a pass-through to
     /// `inner`.
     use_iommu: bool,
+    /// Dirty bitmap to use for IOVA accesses
+    bitmap: Arc<<M::R as GuestMemoryRegion>::B>,
 }
 
 impl IommuMapping {
@@ -354,27 +368,36 @@ impl TryFrom<Range<u64>> for IovaRange {
 
 impl<M: GuestMemory, I: Iommu> IommuMemory<M, I> {
     /// Create a new `IommuMemory` instance.
-    pub fn new(inner: M, iommu: I, use_iommu: bool) -> Self {
+    pub fn new(inner: M, iommu: I, use_iommu: bool, bitmap: <Self as IoMemory>::Bitmap) -> Self {
         IommuMemory {
             inner,
             iommu: Arc::new(iommu),
             use_iommu,
+            bitmap: Arc::new(bitmap),
         }
     }
 
     /// Create a new version of `self` with the underlying physical memory replaced.
     ///
-    /// Note that the inner `Arc` reference to the IOMMU is cloned, i.e. both the existing and the
-    /// new `IommuMemory` object will share an IOMMU instance.  (The `use_iommu` flag however is
-    /// copied, so is independent between the two instances.)
+    /// Note that the inner `Arc` references to the IOMMU and bitmap are cloned, i.e. both the
+    /// existing and the new `IommuMemory` object will share the IOMMU and bitmap instances.  (The
+    /// `use_iommu` flag however is copied, so is independent between the two instances.)
     pub fn inner_replaced(&self, inner: M) -> Self {
         IommuMemory {
             inner,
             iommu: Arc::clone(&self.iommu),
             use_iommu: self.use_iommu,
+            bitmap: Arc::clone(&self.bitmap),
         }
     }
 
+    /// Return a reference to the IOVA address space's dirty bitmap.
+    ///
+    /// This bitmap tracks write accesses done while the IOMMU is enabled.
+    pub fn bitmap(&self) -> &Arc<<Self as IoMemory>::Bitmap> {
+        &self.bitmap
+    }
+
     /// Enable or disable the IOMMU.
     ///
     /// Disabling the IOMMU switches to pass-through mode, where every access is done directly on
@@ -400,12 +423,42 @@ impl<M: GuestMemory + Clone, I: Iommu> Clone for IommuMemory<M, I> {
             inner: self.inner.clone(),
             iommu: Arc::clone(&self.iommu),
             use_iommu: self.use_iommu,
+            bitmap: Arc::clone(&self.bitmap),
+        }
+    }
+}
+
+impl<M: GuestMemory + Debug, I: Iommu> Debug for IommuMemory<M, I>
+where
+    <M::R as GuestMemoryRegion>::B: Debug,
+{
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("IommuMemory")
+            .field("inner", &self.inner)
+            .field("iommu", &self.iommu)
+            .field("use_iommu", &self.use_iommu)
+            .field("bitmap", &self.bitmap)
+            .finish()
+    }
+}
+
+impl<M: GuestMemory + Default, I: Iommu + Default> Default for IommuMemory<M, I>
+where
+    <M::R as GuestMemoryRegion>::B: Default,
+{
+    fn default() -> Self {
+        IommuMemory {
+            inner: Default::default(),
+            iommu: Default::default(),
+            use_iommu: Default::default(),
+            bitmap: Default::default(),
         }
     }
 }
 
 impl<M: GuestMemory, I: Iommu> IoMemory for IommuMemory<M, I> {
     type PhysicalMemory = M;
+    type Bitmap = <M::R as GuestMemoryRegion>::B;
 
     fn check_range(&self, addr: GuestAddress, count: usize, access: Permissions) -> bool {
         if !self.use_iommu {
@@ -424,7 +477,7 @@ impl<M: GuestMemory, I: Iommu> IoMemory for IommuMemory<M, I> {
         addr: GuestAddress,
         count: usize,
         access: Permissions,
-    ) -> GuestMemoryResult<impl IoMemorySliceIterator<'a, bitmap::MS<'a, M>>> {
+    ) -> GuestMemoryResult<impl IoMemorySliceIterator<'a, bitmap::BS<'a, Self::Bitmap>>> {
         if self.use_iommu {
             IommuMemorySliceIterator::virt(self, addr, count, access)
                 .map_err(GuestMemoryError::IommuError)
@@ -445,8 +498,11 @@ impl<M: GuestMemory, I: Iommu> IoMemory for IommuMemory<M, I> {
 /// Iterates over [`VolatileSlice`]s that together form an area in an `IommuMemory`.
 ///
 /// Returned by [`IommuMemory::get_slices()`]
-#[derive(Debug)]
 pub struct IommuMemorySliceIterator<'a, M: GuestMemory, I: Iommu + 'a> {
+    /// Current IOVA (needed to access the right slice of the IOVA space dirty bitmap)
+    iova: GuestAddress,
+    /// IOVA space dirty bitmap
+    bitmap: Option<&'a <M::R as GuestMemoryRegion>::B>,
     /// Underlying physical memory (i.e. not the `IommuMemory`)
     phys_mem: &'a M,
     /// IOMMU translation result (i.e. remaining physical regions to visit)
@@ -463,6 +519,8 @@ impl<'a, M: GuestMemory, I: Iommu> IommuMemorySliceIterator<'a, M, I> {
     /// the underlying physical memory for the given address range.
     fn phys(mem: &'a IommuMemory<M, I>, addr: GuestAddress, count: usize) -> Self {
         IommuMemorySliceIterator {
+            iova: addr,
+            bitmap: None,
             phys_mem: &mem.inner,
             translation: None,
             current_translated_iter: Some(mem.inner.get_slices(addr, count)),
@@ -481,6 +539,8 @@ impl<'a, M: GuestMemory, I: Iommu> IommuMemorySliceIterator<'a, M, I> {
     ) -> Result<Self, Error> {
         let translation = mem.iommu.translate(addr, count, access)?;
         Ok(IommuMemorySliceIterator {
+            iova: addr,
+            bitmap: Some(mem.bitmap.as_ref()),
             phys_mem: &mem.inner,
             translation: Some(translation),
             current_translated_iter: None,
@@ -512,7 +572,22 @@ impl<'a, M: GuestMemory, I: Iommu> IommuMemorySliceIterator<'a, M, I> {
                 .as_mut()
                 .and_then(|iter| iter.next())
             {
-                return Some(item);
+                let mut item = match item {
+                    Ok(item) => item,
+                    Err(err) => return Some(Err(err)),
+                };
+
+                if let Some(bitmap) = self.bitmap.as_ref() {
+                    let bitmap_slice = bitmap.slice_at(self.iova.0 as usize);
+                    item = item.replace_bitmap(bitmap_slice);
+                }
+
+                self.iova = match self.iova.overflowing_add(item.len() as GuestUsize) {
+                    (x @ GuestAddress(0), _) | (x, false) => x,
+                    (_, true) => return Some(Err(GuestMemoryError::GuestAddressOverflow)),
+                };
+
+                return Some(Ok(item));
             }
 
             let next_mapping = self.translation.as_mut()?.next()?;
@@ -546,3 +621,19 @@ impl<'a, M: GuestMemory, I: Iommu> IoMemorySliceIterator<'a, bitmap::MS<'a, M>>
     for IommuMemorySliceIterator<'a, M, I>
 {
 }
+
+impl<'a, M: GuestMemory + Debug, I: Iommu> Debug for IommuMemorySliceIterator<'a, M, I>
+where
+    I::IotlbGuard<'a>: Debug,
+    <M::R as GuestMemoryRegion>::B: Debug,
+{
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("IommuMemorySliceIterator")
+            .field("iova", &self.iova)
+            .field("bitmap", &self.bitmap)
+            .field("phys_mem", &self.phys_mem)
+            .field("translation", &self.translation)
+            .field("current_translated_iter", &self.current_translated_iter)
+            .finish()
+    }
+}
diff --git a/src/volatile_memory.rs b/src/volatile_memory.rs
index f242fce1..47d60932 100644
--- a/src/volatile_memory.rs
+++ b/src/volatile_memory.rs
@@ -435,6 +435,17 @@ impl<'a, B: BitmapSlice> VolatileSlice<'a, B> {
         }
     }
 
+    /// Replaces the bitmap in `self` by `new_bitmap`.
+    #[cfg(feature = "iommu")]
+    pub(crate) fn replace_bitmap<NB: BitmapSlice>(self, new_bitmap: NB) -> VolatileSlice<'a, NB> {
+        VolatileSlice {
+            addr: self.addr,
+            size: self.size,
+            bitmap: new_bitmap,
+            mmap: self.mmap,
+        }
+    }
+
     /// Returns a guard for the pointer to the underlying memory.
     pub fn ptr_guard(&self) -> PtrGuard {
         PtrGuard::read(self.mmap, self.addr, self.len())

From b628fb9a4974c65735a45271943fad208db1bdb0 Mon Sep 17 00:00:00 2001
From: Hanna Czenczek <hreitz@redhat.com>
Date: Wed, 9 Jul 2025 14:50:35 +0200
Subject: [PATCH 09/11] Add tests for IOMMU functionality
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This commit also adds the iommu feature to the coverage_config feature
list.  (I left the aarch64 coverage value unchanged; I cannot find out
how to get the current value on my system, and it isn’t include in CI.)

Signed-off-by: Hanna Czenczek <hreitz@redhat.com>
---
 coverage_config_aarch64.json |   2 +-
 coverage_config_x86_64.json  |   4 +-
 src/guest_memory.rs          |  56 +++
 src/iommu.rs                 | 856 +++++++++++++++++++++++++++++++++++
 src/mmap/mod.rs              |  25 +
 5 files changed, 940 insertions(+), 3 deletions(-)

diff --git a/coverage_config_aarch64.json b/coverage_config_aarch64.json
index 4aeb3711..7332c6c7 100644
--- a/coverage_config_aarch64.json
+++ b/coverage_config_aarch64.json
@@ -1,5 +1,5 @@
 {
   "coverage_score": 85.2,
   "exclude_path": "mmap/windows.rs",
-  "crate_features": "backend-mmap,backend-atomic,backend-bitmap"
+  "crate_features": "backend-mmap,backend-atomic,backend-bitmap,iommu"
 }
diff --git a/coverage_config_x86_64.json b/coverage_config_x86_64.json
index 1c8745c5..2fa53707 100644
--- a/coverage_config_x86_64.json
+++ b/coverage_config_x86_64.json
@@ -1,5 +1,5 @@
 {
-  "coverage_score": 90.82,
+  "coverage_score": 91.52,
   "exclude_path": "mmap_windows.rs",
-  "crate_features": "backend-mmap,backend-atomic,backend-bitmap"
+  "crate_features": "backend-mmap,backend-atomic,backend-bitmap,iommu"
 }
diff --git a/src/guest_memory.rs b/src/guest_memory.rs
index 1f19fc9f..dabf4bf0 100644
--- a/src/guest_memory.rs
+++ b/src/guest_memory.rs
@@ -897,6 +897,9 @@ impl<M: GuestMemory + ?Sized> IoMemory for M {
 #[cfg(test)]
 mod tests {
     #![allow(clippy::undocumented_unsafe_blocks)]
+
+    // Note that `IoMemory` is tested primarily in src/iommu.rs via `IommuMemory`.
+
     use super::*;
     #[cfg(feature = "backend-mmap")]
     use crate::bytes::ByteValued;
@@ -1137,4 +1140,57 @@ mod tests {
         let r = mem.find_region(addr).unwrap();
         assert_eq!(r.is_hugetlbfs(), None);
     }
+
+    /// Test `Permissions & Permissions`.
+    #[test]
+    fn test_perm_and() {
+        use Permissions::*;
+
+        for p in [No, Read, Write, ReadWrite] {
+            assert_eq!(p & p, p);
+        }
+        for p1 in [No, Read, Write, ReadWrite] {
+            for p2 in [No, Read, Write, ReadWrite] {
+                assert_eq!(p1 & p2, p2 & p1);
+            }
+        }
+        for p in [No, Read, Write, ReadWrite] {
+            assert_eq!(No & p, No);
+        }
+        for p in [No, Read, Write, ReadWrite] {
+            assert_eq!(ReadWrite & p, p);
+        }
+        assert_eq!(Read & Write, No);
+    }
+
+    /// Test `Permissions | Permissions`.
+    #[test]
+    fn test_perm_or() {
+        use Permissions::*;
+
+        for p in [No, Read, Write, ReadWrite] {
+            assert_eq!(p | p, p);
+        }
+        for p1 in [No, Read, Write, ReadWrite] {
+            for p2 in [No, Read, Write, ReadWrite] {
+                assert_eq!(p1 | p2, p2 | p1);
+            }
+        }
+        for p in [No, Read, Write, ReadWrite] {
+            assert_eq!(No | p, p);
+        }
+        for p in [No, Read, Write, ReadWrite] {
+            assert_eq!(ReadWrite | p, ReadWrite);
+        }
+        assert_eq!(Read | Write, ReadWrite);
+    }
+
+    /// Test `Permissions::has_write()`.
+    #[test]
+    fn test_perm_has_write() {
+        assert!(!Permissions::No.has_write());
+        assert!(!Permissions::Read.has_write());
+        assert!(Permissions::Write.has_write());
+        assert!(Permissions::ReadWrite.has_write());
+    }
 }
diff --git a/src/iommu.rs b/src/iommu.rs
index cf36c4a8..3c75d6ec 100644
--- a/src/iommu.rs
+++ b/src/iommu.rs
@@ -637,3 +637,859 @@ where
             .finish()
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::{Error, IotlbIterator, IovaRange, MappedRange};
+    #[cfg(all(feature = "backend-bitmap", feature = "backend-mmap"))]
+    use crate::bitmap::AtomicBitmap;
+    #[cfg(feature = "backend-mmap")]
+    use crate::bitmap::NewBitmap;
+    #[cfg(all(feature = "backend-bitmap", feature = "backend-mmap"))]
+    use crate::GuestMemoryRegion;
+    #[cfg(feature = "backend-mmap")]
+    use crate::{
+        Bytes, GuestMemoryError, GuestMemoryMmap, GuestMemoryResult, IoMemory, IommuMemory,
+    };
+    use crate::{GuestAddress, Iommu, Iotlb, Permissions};
+    use std::fmt::Debug;
+    #[cfg(all(feature = "backend-bitmap", feature = "backend-mmap"))]
+    use std::num::NonZeroUsize;
+    use std::ops::Deref;
+    use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
+    use std::sync::{RwLock, RwLockReadGuard};
+
+    #[derive(Debug)]
+    struct SimpleIommu {
+        iotlb: RwLock<Iotlb>,
+        /// Records the last fail event's base IOVA
+        fail_base: AtomicU64,
+        /// Records the last fail event's length
+        fail_len: AtomicUsize,
+        /// Records whether the last fail event was a miss
+        fail_was_miss: AtomicBool,
+        /// What base physical address to map to on the next fail event (0 means return error)
+        next_map_to: AtomicU64,
+    }
+
+    impl SimpleIommu {
+        #[cfg(feature = "backend-mmap")]
+        fn new() -> Self {
+            SimpleIommu {
+                iotlb: Iotlb::new().into(),
+                fail_base: 0.into(),
+                fail_len: 0.into(),
+                fail_was_miss: false.into(),
+                next_map_to: 0.into(),
+            }
+        }
+
+        #[cfg(feature = "backend-mmap")]
+        fn expect_mapping_request(&self, to_phys: GuestAddress) {
+            // Clear failed range info so it can be tested after the request
+            self.fail_base.store(0, Ordering::Relaxed);
+            self.fail_len.store(0, Ordering::Relaxed);
+            self.next_map_to.store(to_phys.0, Ordering::Relaxed);
+        }
+
+        #[cfg(feature = "backend-mmap")]
+        fn verify_mapping_request(&self, virt: GuestAddress, len: usize, was_miss: bool) {
+            assert_eq!(self.fail_base.load(Ordering::Relaxed), virt.0);
+            assert_eq!(self.fail_len.load(Ordering::Relaxed), len);
+            assert_eq!(self.fail_was_miss.load(Ordering::Relaxed), was_miss);
+        }
+    }
+
+    impl Iommu for SimpleIommu {
+        type IotlbGuard<'a> = RwLockReadGuard<'a, Iotlb>;
+
+        fn translate(
+            &self,
+            iova: GuestAddress,
+            length: usize,
+            access: Permissions,
+        ) -> Result<IotlbIterator<Self::IotlbGuard<'_>>, Error> {
+            loop {
+                let mut fails =
+                    match Iotlb::lookup(self.iotlb.read().unwrap(), iova, length, access) {
+                        Ok(success) => return Ok(success),
+                        Err(fails) => fails,
+                    };
+                let miss = !fails.misses.is_empty();
+                let fail = fails
+                    .misses
+                    .pop()
+                    .or_else(|| fails.access_fails.pop())
+                    .expect("No failure reported, even though a failure happened");
+                self.fail_base.store(fail.base.0, Ordering::Relaxed);
+                self.fail_len.store(fail.length, Ordering::Relaxed);
+                self.fail_was_miss.store(miss, Ordering::Relaxed);
+
+                if !fails.misses.is_empty() || !fails.access_fails.is_empty() {
+                    return Err(Error::CannotResolve {
+                        iova_range: IovaRange { base: iova, length },
+                        reason: "This IOMMU can only handle one failure per access".into(),
+                    });
+                }
+
+                let map_to = self.next_map_to.swap(0, Ordering::Relaxed);
+                if map_to == 0 {
+                    return Err(Error::CannotResolve {
+                        iova_range: IovaRange {
+                            base: fail.base,
+                            length: fail.length,
+                        },
+                        reason: "No mapping provided for failed range".into(),
+                    });
+                }
+
+                self.iotlb.write().unwrap().set_mapping(
+                    fail.base,
+                    GuestAddress(map_to),
+                    fail.length,
+                    access,
+                )?;
+            }
+        }
+    }
+
+    /// Verify that `iova`+`length` is mapped to `expected`.
+    fn verify_hit(
+        iotlb: impl Deref<Target = Iotlb> + Debug,
+        iova: GuestAddress,
+        length: usize,
+        permissions: Permissions,
+        expected: impl IntoIterator<Item = MappedRange>,
+    ) {
+        let mut iter = Iotlb::lookup(iotlb, iova, length, permissions)
+            .inspect_err(|err| panic!("Unexpected lookup error {err:?}"))
+            .unwrap();
+
+        for e in expected {
+            assert_eq!(iter.next(), Some(e));
+        }
+        assert_eq!(iter.next(), None);
+    }
+
+    /// Verify that trying to look up `iova`+`length` results in misses at `expected_misses` and
+    /// access failures (permission-related) at `expected_access_fails`.
+    fn verify_fail(
+        iotlb: impl Deref<Target = Iotlb> + Debug,
+        iova: GuestAddress,
+        length: usize,
+        permissions: Permissions,
+        expected_misses: impl IntoIterator<Item = IovaRange>,
+        expected_access_fails: impl IntoIterator<Item = IovaRange>,
+    ) {
+        let fails = Iotlb::lookup(iotlb, iova, length, permissions)
+            .inspect(|hits| panic!("Expected error on lookup, found {hits:?}"))
+            .unwrap_err();
+
+        let mut miss_iter = fails.misses.into_iter();
+        for e in expected_misses {
+            assert_eq!(miss_iter.next(), Some(e));
+        }
+        assert_eq!(miss_iter.next(), None);
+
+        let mut accf_iter = fails.access_fails.into_iter();
+        for e in expected_access_fails {
+            assert_eq!(accf_iter.next(), Some(e));
+        }
+        assert_eq!(accf_iter.next(), None);
+    }
+
+    /// Enter adjacent IOTLB entries and verify they are merged into a single one.
+    #[test]
+    fn test_iotlb_merge() -> Result<(), Error> {
+        const IOVA: GuestAddress = GuestAddress(42);
+        const PHYS: GuestAddress = GuestAddress(87);
+        const LEN_1: usize = 123;
+        const LEN_2: usize = 234;
+
+        let mut iotlb = Iotlb::new();
+        iotlb.set_mapping(IOVA, PHYS, LEN_1, Permissions::ReadWrite)?;
+        iotlb.set_mapping(
+            GuestAddress(IOVA.0 + LEN_1 as u64),
+            GuestAddress(PHYS.0 + LEN_1 as u64),
+            LEN_2,
+            Permissions::ReadWrite,
+        )?;
+
+        verify_hit(
+            &iotlb,
+            IOVA,
+            LEN_1 + LEN_2,
+            Permissions::ReadWrite,
+            [MappedRange {
+                base: PHYS,
+                length: LEN_1 + LEN_2,
+            }],
+        );
+
+        // Also check just a partial range
+        verify_hit(
+            &iotlb,
+            GuestAddress(IOVA.0 + LEN_1 as u64 - 1),
+            2,
+            Permissions::ReadWrite,
+            [MappedRange {
+                base: GuestAddress(PHYS.0 + LEN_1 as u64 - 1),
+                length: 2,
+            }],
+        );
+
+        Ok(())
+    }
+
+    /// Test that adjacent IOTLB entries that map to the same physical address are not merged into
+    /// a single entry.
+    #[test]
+    fn test_iotlb_nomerge_same_phys() -> Result<(), Error> {
+        const IOVA: GuestAddress = GuestAddress(42);
+        const PHYS: GuestAddress = GuestAddress(87);
+        const LEN_1: usize = 123;
+        const LEN_2: usize = 234;
+
+        let mut iotlb = Iotlb::new();
+        iotlb.set_mapping(IOVA, PHYS, LEN_1, Permissions::ReadWrite)?;
+        iotlb.set_mapping(
+            GuestAddress(IOVA.0 + LEN_1 as u64),
+            PHYS,
+            LEN_2,
+            Permissions::ReadWrite,
+        )?;
+
+        verify_hit(
+            &iotlb,
+            IOVA,
+            LEN_1 + LEN_2,
+            Permissions::ReadWrite,
+            [
+                MappedRange {
+                    base: PHYS,
+                    length: LEN_1,
+                },
+                MappedRange {
+                    base: PHYS,
+                    length: LEN_2,
+                },
+            ],
+        );
+
+        Ok(())
+    }
+
+    /// Test permission handling
+    #[test]
+    fn test_iotlb_perms() -> Result<(), Error> {
+        const IOVA_R: GuestAddress = GuestAddress(42);
+        const PHYS_R: GuestAddress = GuestAddress(87);
+        const LEN_R: usize = 123;
+        const IOVA_W: GuestAddress = GuestAddress(IOVA_R.0 + LEN_R as u64);
+        const PHYS_W: GuestAddress = GuestAddress(PHYS_R.0 + LEN_R as u64);
+        const LEN_W: usize = 234;
+        const IOVA_FULL: GuestAddress = IOVA_R;
+        const LEN_FULL: usize = LEN_R + LEN_W;
+
+        let mut iotlb = Iotlb::new();
+        iotlb.set_mapping(IOVA_R, PHYS_R, LEN_R, Permissions::Read)?;
+        iotlb.set_mapping(IOVA_W, PHYS_W, LEN_W, Permissions::Write)?;
+
+        // Test 1: Access whole range as R+W, should completely fail
+        verify_fail(
+            &iotlb,
+            IOVA_FULL,
+            LEN_FULL,
+            Permissions::ReadWrite,
+            [],
+            [
+                IovaRange {
+                    base: IOVA_R,
+                    length: LEN_R,
+                },
+                IovaRange {
+                    base: IOVA_W,
+                    length: LEN_W,
+                },
+            ],
+        );
+
+        // Test 2: Access whole range as R-only, should fail on second part
+        verify_fail(
+            &iotlb,
+            IOVA_FULL,
+            LEN_FULL,
+            Permissions::Read,
+            [],
+            [IovaRange {
+                base: IOVA_W,
+                length: LEN_W,
+            }],
+        );
+
+        // Test 3: Access whole range W-only, should fail on second part
+        verify_fail(
+            &iotlb,
+            IOVA_FULL,
+            LEN_FULL,
+            Permissions::Write,
+            [],
+            [IovaRange {
+                base: IOVA_R,
+                length: LEN_R,
+            }],
+        );
+
+        // Test 4: Access whole range w/o perms, should succeed
+        verify_hit(
+            &iotlb,
+            IOVA_FULL,
+            LEN_FULL,
+            Permissions::No,
+            [
+                MappedRange {
+                    base: PHYS_R,
+                    length: LEN_R,
+                },
+                MappedRange {
+                    base: PHYS_W,
+                    length: LEN_W,
+                },
+            ],
+        );
+
+        // Test 5: Access R range as R, should succeed
+        verify_hit(
+            &iotlb,
+            IOVA_R,
+            LEN_R,
+            Permissions::Read,
+            [MappedRange {
+                base: PHYS_R,
+                length: LEN_R,
+            }],
+        );
+
+        // Test 6: Access W range as W, should succeed
+        verify_hit(
+            &iotlb,
+            IOVA_W,
+            LEN_W,
+            Permissions::Write,
+            [MappedRange {
+                base: PHYS_W,
+                length: LEN_W,
+            }],
+        );
+
+        Ok(())
+    }
+
+    /// Test IOTLB invalidation
+    #[test]
+    fn test_iotlb_invalidation() -> Result<(), Error> {
+        const IOVA: GuestAddress = GuestAddress(42);
+        const PHYS: GuestAddress = GuestAddress(87);
+        const LEN: usize = 123;
+        const INVAL_OFS: usize = LEN / 2;
+        const INVAL_LEN: usize = 3;
+        const IOVA_AT_INVAL: GuestAddress = GuestAddress(IOVA.0 + INVAL_OFS as u64);
+        const PHYS_AT_INVAL: GuestAddress = GuestAddress(PHYS.0 + INVAL_OFS as u64);
+        const IOVA_POST_INVAL: GuestAddress = GuestAddress(IOVA_AT_INVAL.0 + INVAL_LEN as u64);
+        const PHYS_POST_INVAL: GuestAddress = GuestAddress(PHYS_AT_INVAL.0 + INVAL_LEN as u64);
+        const POST_INVAL_LEN: usize = LEN - INVAL_OFS - INVAL_LEN;
+
+        let mut iotlb = Iotlb::new();
+        iotlb.set_mapping(IOVA, PHYS, LEN, Permissions::ReadWrite)?;
+        verify_hit(
+            &iotlb,
+            IOVA,
+            LEN,
+            Permissions::ReadWrite,
+            [MappedRange {
+                base: PHYS,
+                length: LEN,
+            }],
+        );
+
+        // Invalidate something in the middle; expect mapping at the start, then miss, then further
+        // mapping
+        iotlb.invalidate_mapping(IOVA_AT_INVAL, INVAL_LEN);
+        verify_hit(
+            &iotlb,
+            IOVA,
+            INVAL_OFS,
+            Permissions::ReadWrite,
+            [MappedRange {
+                base: PHYS,
+                length: INVAL_OFS,
+            }],
+        );
+        verify_fail(
+            &iotlb,
+            IOVA,
+            LEN,
+            Permissions::ReadWrite,
+            [IovaRange {
+                base: IOVA_AT_INVAL,
+                length: INVAL_LEN,
+            }],
+            [],
+        );
+        verify_hit(
+            &iotlb,
+            IOVA_POST_INVAL,
+            POST_INVAL_LEN,
+            Permissions::ReadWrite,
+            [MappedRange {
+                base: PHYS_POST_INVAL,
+                length: POST_INVAL_LEN,
+            }],
+        );
+
+        // And invalidate everything; expect full miss
+        iotlb.invalidate_all();
+        verify_fail(
+            &iotlb,
+            IOVA,
+            LEN,
+            Permissions::ReadWrite,
+            [IovaRange {
+                base: IOVA,
+                length: LEN,
+            }],
+            [],
+        );
+
+        Ok(())
+    }
+
+    /// Create `IommuMemory` backed by multiple physical regions, all mapped into a single virtual
+    /// region (if `virt_start`/`virt_perm` are given).
+    ///
+    /// Memory is filled with incrementing (overflowing) bytes, starting with value `value_offset`.
+    #[cfg(feature = "backend-mmap")]
+    fn create_virt_memory<B: NewBitmap>(
+        virt_mapping: Option<(GuestAddress, Permissions)>,
+        value_offset: u8,
+        phys_regions: impl IntoIterator<Item = MappedRange>,
+        bitmap: B,
+    ) -> IommuMemory<GuestMemoryMmap<B>, SimpleIommu> {
+        let phys_ranges = phys_regions
+            .into_iter()
+            .map(|range| (range.base, range.length))
+            .collect::<Vec<(GuestAddress, usize)>>();
+        let phys_mem = GuestMemoryMmap::<B>::from_ranges(&phys_ranges).unwrap();
+
+        let mut byte_val = value_offset;
+        for (base, len) in &phys_ranges {
+            let mut slices = phys_mem
+                .get_slices(*base, *len, Permissions::Write)
+                .inspect_err(|err| panic!("Failed to access memory: {err}"))
+                .unwrap();
+            let slice = slices
+                .next()
+                .unwrap()
+                .inspect_err(|err| panic!("Failed to access memory: {err}"))
+                .unwrap();
+            assert!(slices.next().is_none(), "Expected single slice");
+
+            for i in 0..*len {
+                slice.write(&[byte_val], i).unwrap();
+                byte_val = byte_val.wrapping_add(1);
+            }
+        }
+
+        let mem = IommuMemory::new(phys_mem, SimpleIommu::new(), true, bitmap);
+
+        // IOMMU is in use, this will be `None`
+        assert!(mem.physical_memory().is_none());
+
+        if let Some((mut virt, perm)) = virt_mapping {
+            for (base, len) in phys_ranges {
+                let mut iotlb = mem.iommu().iotlb.write().unwrap();
+                iotlb.set_mapping(virt, base, len, perm).unwrap();
+                virt = GuestAddress(virt.0 + len as u64);
+            }
+        }
+
+        mem
+    }
+
+    /// Verify the byte contents at `start`+`len`.  Assume the initial byte value to be
+    /// `value_offset`.
+    ///
+    /// Each byte is expected to be incremented over the last (as created by
+    /// `create_virt_memory()`).
+    ///
+    /// Return an error if mapping fails, but just panic if there is a content mismatch.
+    #[cfg(feature = "backend-mmap")]
+    fn check_virt_mem_content(
+        mem: &impl IoMemory,
+        start: GuestAddress,
+        len: usize,
+        value_offset: u8,
+    ) -> GuestMemoryResult<()> {
+        let mut ref_value = value_offset;
+        for slice in mem.get_slices(start, len, Permissions::Read)? {
+            let slice = slice?;
+
+            let count = slice.len();
+            let mut data = vec![0u8; count];
+            slice.read(&mut data, 0).unwrap();
+            for val in data {
+                assert_eq!(val, ref_value);
+                ref_value = ref_value.wrapping_add(1);
+            }
+        }
+
+        Ok(())
+    }
+
+    #[cfg(feature = "backend-mmap")]
+    fn verify_virt_mem_content(
+        m: &impl IoMemory,
+        start: GuestAddress,
+        len: usize,
+        value_offset: u8,
+    ) {
+        check_virt_mem_content(m, start, len, value_offset).unwrap();
+    }
+
+    /// Verify that trying to read from `start`+`len` fails (because of `CannotResolve`).
+    ///
+    /// The reported failed-to-map range is checked to be `fail_start`+`fail_len`.  `fail_start`
+    /// defaults to `start`, `fail_len` defaults to the remaining length of the whole mapping
+    /// starting at `fail_start` (i.e. `start + len - fail_start`).
+    #[cfg(feature = "backend-mmap")]
+    fn verify_virt_mem_error(
+        m: &impl IoMemory,
+        start: GuestAddress,
+        len: usize,
+        fail_start: Option<GuestAddress>,
+        fail_len: Option<usize>,
+    ) {
+        let fail_start = fail_start.unwrap_or(start);
+        let fail_len = fail_len.unwrap_or(len - (fail_start.0 - start.0) as usize);
+        let err = check_virt_mem_content(m, start, len, 0).unwrap_err();
+        let GuestMemoryError::IommuError(Error::CannotResolve {
+            iova_range: failed_range,
+            reason: _,
+        }) = err
+        else {
+            panic!("Unexpected error: {err:?}");
+        };
+        assert_eq!(
+            failed_range,
+            IovaRange {
+                base: fail_start,
+                length: fail_len,
+            }
+        );
+    }
+
+    /// Test `IommuMemory`, with pre-filled mappings.
+    #[cfg(feature = "backend-mmap")]
+    #[test]
+    fn test_iommu_memory_pre_mapped() {
+        const PHYS_START_1: GuestAddress = GuestAddress(0x4000);
+        const PHYS_START_2: GuestAddress = GuestAddress(0x8000);
+        const PHYS_LEN: usize = 128;
+        const VIRT_START: GuestAddress = GuestAddress(0x2a000);
+        const VIRT_LEN: usize = PHYS_LEN * 2;
+        const VIRT_POST_MAP: GuestAddress = GuestAddress(VIRT_START.0 + VIRT_LEN as u64);
+
+        let mem = create_virt_memory(
+            Some((VIRT_START, Permissions::Read)),
+            0,
+            [
+                MappedRange {
+                    base: PHYS_START_1,
+                    length: PHYS_LEN,
+                },
+                MappedRange {
+                    base: PHYS_START_2,
+                    length: PHYS_LEN,
+                },
+            ],
+            (),
+        );
+
+        assert!(mem.check_range(VIRT_START, VIRT_LEN, Permissions::No));
+        assert!(mem.check_range(VIRT_START, VIRT_LEN, Permissions::Read));
+        assert!(!mem.check_range(VIRT_START, VIRT_LEN, Permissions::Write));
+        assert!(!mem.check_range(VIRT_START, VIRT_LEN, Permissions::ReadWrite));
+        assert!(!mem.check_range(GuestAddress(VIRT_START.0 - 1), 1, Permissions::No));
+        assert!(!mem.check_range(VIRT_POST_MAP, 1, Permissions::No));
+
+        verify_virt_mem_content(&mem, VIRT_START, VIRT_LEN, 0);
+        verify_virt_mem_error(&mem, GuestAddress(VIRT_START.0 - 1), 1, None, None);
+        verify_virt_mem_error(&mem, VIRT_POST_MAP, 1, None, None);
+        verify_virt_mem_error(&mem, VIRT_START, VIRT_LEN + 1, Some(VIRT_POST_MAP), None);
+    }
+
+    /// Test `IommuMemory`, with mappings created through the IOMMU on the fly.
+    #[cfg(feature = "backend-mmap")]
+    #[test]
+    fn test_iommu_memory_live_mapped() {
+        const PHYS_START_1: GuestAddress = GuestAddress(0x4000);
+        const PHYS_START_2: GuestAddress = GuestAddress(0x8000);
+        const PHYS_LEN: usize = 128;
+        const VIRT_START: GuestAddress = GuestAddress(0x2a000);
+        const VIRT_START_1: GuestAddress = VIRT_START;
+        const VIRT_START_2: GuestAddress = GuestAddress(VIRT_START.0 + PHYS_LEN as u64);
+        const VIRT_LEN: usize = PHYS_LEN * 2;
+        const VIRT_POST_MAP: GuestAddress = GuestAddress(VIRT_START.0 + VIRT_LEN as u64);
+
+        let mem = create_virt_memory(
+            None,
+            0,
+            [
+                MappedRange {
+                    base: PHYS_START_1,
+                    length: PHYS_LEN,
+                },
+                MappedRange {
+                    base: PHYS_START_2,
+                    length: PHYS_LEN,
+                },
+            ],
+            (),
+        );
+
+        assert!(!mem.check_range(VIRT_START, VIRT_LEN, Permissions::No));
+        assert!(!mem.check_range(VIRT_START, VIRT_LEN, Permissions::Read));
+        assert!(!mem.check_range(VIRT_START, VIRT_LEN, Permissions::Write));
+        assert!(!mem.check_range(VIRT_START, VIRT_LEN, Permissions::ReadWrite));
+        assert!(!mem.check_range(GuestAddress(VIRT_START.0 - 1), 1, Permissions::No));
+        assert!(!mem.check_range(VIRT_POST_MAP, 1, Permissions::No));
+
+        verify_virt_mem_error(&mem, VIRT_START, VIRT_LEN, None, None);
+        verify_virt_mem_error(&mem, GuestAddress(VIRT_START.0 - 1), 1, None, None);
+        verify_virt_mem_error(&mem, VIRT_POST_MAP, 1, None, None);
+        verify_virt_mem_error(&mem, VIRT_START, VIRT_LEN + 1, None, None);
+
+        let iommu = mem.iommu();
+
+        // Can only map one region at a time (with `SimpleIommu`), so only access `PHYS_LEN` first,
+        // not `VIRT_LEN`
+        iommu.expect_mapping_request(PHYS_START_1);
+        verify_virt_mem_content(&mem, VIRT_START, PHYS_LEN, 0);
+        iommu.verify_mapping_request(VIRT_START_1, PHYS_LEN, true);
+
+        iommu.expect_mapping_request(PHYS_START_2);
+        verify_virt_mem_content(&mem, VIRT_START, VIRT_LEN, 0);
+        iommu.verify_mapping_request(VIRT_START_2, PHYS_LEN, true);
+
+        // Also check invalid access failure
+        iommu
+            .iotlb
+            .write()
+            .unwrap()
+            .set_mapping(VIRT_START_1, PHYS_START_1, PHYS_LEN, Permissions::Write)
+            .unwrap();
+
+        iommu.expect_mapping_request(PHYS_START_1);
+        verify_virt_mem_content(&mem, VIRT_START, VIRT_LEN, 0);
+        iommu.verify_mapping_request(VIRT_START_1, PHYS_LEN, false);
+    }
+
+    /// Test replacing the physical memory of an `IommuMemory`.
+    #[cfg(feature = "backend-mmap")]
+    #[test]
+    fn test_mem_replace() {
+        const PHYS_START_1: GuestAddress = GuestAddress(0x4000);
+        const PHYS_START_2: GuestAddress = GuestAddress(0x8000);
+        const PHYS_LEN: usize = 128;
+        const VIRT_START: GuestAddress = GuestAddress(0x2a000);
+
+        // Note only one physical region.  `mem2` will have two, to see that this pattern
+        // (`inner_replaced()`) can be used to e.g. extend physical memory.
+        let mem = create_virt_memory(
+            Some((VIRT_START, Permissions::Read)),
+            0,
+            [MappedRange {
+                base: PHYS_START_1,
+                length: PHYS_LEN,
+            }],
+            (),
+        );
+
+        verify_virt_mem_content(&mem, VIRT_START, PHYS_LEN, 0);
+        verify_virt_mem_error(
+            &mem,
+            VIRT_START,
+            PHYS_LEN * 2,
+            Some(GuestAddress(VIRT_START.0 + PHYS_LEN as u64)),
+            None,
+        );
+
+        let mut mem2 = create_virt_memory(
+            Some((VIRT_START, Permissions::Read)),
+            42,
+            [
+                MappedRange {
+                    base: PHYS_START_1,
+                    length: PHYS_LEN,
+                },
+                MappedRange {
+                    base: PHYS_START_2,
+                    length: PHYS_LEN,
+                },
+            ],
+            (),
+        );
+
+        verify_virt_mem_content(&mem2, VIRT_START, PHYS_LEN * 2, 42);
+
+        // Clone `mem` before replacing its physical memory, to see that works
+        let mem_cloned = mem.clone();
+
+        // Use `mem2`'s physical memory for `mem`
+        mem2.set_iommu_enabled(false);
+        let pmem2 = mem2.physical_memory().unwrap();
+        assert!(std::ptr::eq(pmem2, mem2.inner()));
+        let mem = mem.inner_replaced(pmem2.clone());
+
+        // The physical memory has been replaced, but `mem` still uses its old IOMMU, so the
+        // mapping for everything past VIRT_START + PHYS_LEN does not yet exist.
+        mem.iommu().expect_mapping_request(PHYS_START_2);
+        verify_virt_mem_content(&mem, VIRT_START, PHYS_LEN * 2, 42);
+        mem.iommu().verify_mapping_request(
+            GuestAddress(VIRT_START.0 + PHYS_LEN as u64),
+            PHYS_LEN,
+            true,
+        );
+
+        // Verify `mem`'s clone still is the same (though it does use the same IOMMU)
+        verify_virt_mem_content(&mem_cloned, VIRT_START, PHYS_LEN, 0);
+        // See, it's the same IOMMU (i.e. it has a mapping PHYS_START_2):
+        verify_hit(
+            mem_cloned.iommu().iotlb.read().unwrap(),
+            VIRT_START,
+            PHYS_LEN * 2,
+            Permissions::Read,
+            [
+                MappedRange {
+                    base: PHYS_START_1,
+                    length: PHYS_LEN,
+                },
+                MappedRange {
+                    base: PHYS_START_2,
+                    length: PHYS_LEN,
+                },
+            ],
+        );
+        // (But we cannot access that mapping because `mem_cloned`'s physical memory does not
+        // contain that physical range.)
+    }
+
+    /// In `mem`'s dirty bitmap, verify that the given `clean` addresses are clean, and the `dirty`
+    /// addresses are dirty.  Auto-clear the dirty addresses checked.
+    ///
+    /// Cannot import `GuestMemory` in this module, as that would interfere with `IoMemory` for
+    /// methods that have the same name between the two.
+    #[cfg(all(feature = "backend-bitmap", feature = "backend-mmap"))]
+    fn verify_mem_bitmap<
+        M: crate::GuestMemory<R = R>,
+        R: GuestMemoryRegion<B = AtomicBitmap>,
+        I: Iommu,
+    >(
+        mem: &IommuMemory<M, I>,
+        clean: impl IntoIterator<Item = usize>,
+        dirty: impl IntoIterator<Item = usize>,
+    ) {
+        let bitmap = mem.bitmap();
+        for addr in clean {
+            if bitmap.is_addr_set(addr) {
+                panic!("Expected addr {addr:#x} to be clean, but is dirty");
+            }
+        }
+        for addr in dirty {
+            if !bitmap.is_addr_set(addr) {
+                panic!("Expected addr {addr:#x} to be dirty, but is clean");
+            }
+            bitmap.reset_addr_range(addr, 1);
+        }
+    }
+
+    #[cfg(all(feature = "backend-bitmap", feature = "backend-mmap"))]
+    #[test]
+    fn test_dirty_bitmap() {
+        const PAGE_SIZE: usize = 4096;
+        const PHYS_START: GuestAddress = GuestAddress(0x4000);
+        const PHYS_LEN: usize = PAGE_SIZE * 2;
+        const PHYS_PAGE_0: usize = PHYS_START.0 as usize;
+        const PHYS_PAGE_1: usize = PHYS_START.0 as usize + PAGE_SIZE;
+        const VIRT_START: GuestAddress = GuestAddress(0x2a000);
+        const VIRT_PAGE_0: usize = VIRT_START.0 as usize;
+        const VIRT_PAGE_1: usize = VIRT_START.0 as usize + PAGE_SIZE;
+
+        let bitmap = AtomicBitmap::new(
+            VIRT_START.0 as usize + PHYS_LEN,
+            NonZeroUsize::new(PAGE_SIZE).unwrap(),
+        );
+
+        let mem = create_virt_memory(
+            Some((VIRT_START, Permissions::ReadWrite)),
+            0,
+            [MappedRange {
+                base: PHYS_START,
+                length: PHYS_LEN,
+            }],
+            bitmap,
+        );
+
+        // Check bitmap is cleared before everything -- through the whole test, the physical ranges
+        // should remain clean as the bitmap is only supposed to track IOVAs
+        verify_mem_bitmap(
+            &mem,
+            [PHYS_PAGE_0, PHYS_PAGE_1, VIRT_PAGE_0, VIRT_PAGE_1],
+            [],
+        );
+
+        // Just to be sure, check that PHYS_PAGE_0 and PHYS_PAGE_1 technically can be dirtied,
+        // though, or testing them would not be really useful
+        mem.bitmap().set_addr_range(PHYS_PAGE_0, 2 * PAGE_SIZE);
+        verify_mem_bitmap(&mem, [VIRT_PAGE_0, VIRT_PAGE_1], [PHYS_PAGE_0, PHYS_PAGE_1]);
+
+        // Just read from memory, should not dirty bitmap
+        verify_virt_mem_content(&mem, VIRT_START, PHYS_LEN, 0);
+        verify_mem_bitmap(
+            &mem,
+            [PHYS_PAGE_0, PHYS_PAGE_1, VIRT_PAGE_0, VIRT_PAGE_1],
+            [],
+        );
+
+        // Verify that writing to a writeable slice causes dirtying, i.e. that the `VolatileSlice`
+        // returned here correctly dirties the bitmap when written to
+        let mut slices = mem
+            .get_slices(VIRT_START, PHYS_LEN, Permissions::Write)
+            .inspect_err(|err| panic!("Failed to access memory: {err}"))
+            .unwrap();
+        let slice = slices
+            .next()
+            .unwrap()
+            .inspect_err(|err| panic!("Failed to access memory: {err}"))
+            .unwrap();
+        assert!(slices.next().is_none(), "Expected single slice");
+
+        verify_mem_bitmap(
+            &mem,
+            [PHYS_PAGE_0, PHYS_PAGE_1, VIRT_PAGE_0, VIRT_PAGE_1],
+            [],
+        );
+
+        slice
+            .store(42, 0, Ordering::Relaxed)
+            .inspect_err(|err| panic!("Writing to memory failed: {err}"))
+            .unwrap();
+        verify_mem_bitmap(&mem, [PHYS_PAGE_0, PHYS_PAGE_1, VIRT_PAGE_1], [VIRT_PAGE_0]);
+
+        slice
+            .store(23, PAGE_SIZE, Ordering::Relaxed)
+            .inspect_err(|err| panic!("Writing to memory failed: {err}"))
+            .unwrap();
+        verify_mem_bitmap(&mem, [PHYS_PAGE_0, PHYS_PAGE_1, VIRT_PAGE_0], [VIRT_PAGE_1]);
+    }
+}
diff --git a/src/mmap/mod.rs b/src/mmap/mod.rs
index f18ea5c5..708cacca 100644
--- a/src/mmap/mod.rs
+++ b/src/mmap/mod.rs
@@ -733,4 +733,29 @@ mod tests {
                 .unwrap()
         });
     }
+
+    #[test]
+    fn test_change_region_addr() {
+        let addr1 = GuestAddress(0x1000);
+        let addr2 = GuestAddress(0x2000);
+        let gm = GuestMemoryMmap::from_ranges(&[(addr1, 0x1000)]).unwrap();
+
+        assert!(gm.find_region(addr1).is_some());
+        assert!(gm.find_region(addr2).is_none());
+
+        let (gm, region) = gm.remove_region(addr1, 0x1000).unwrap();
+
+        assert!(gm.find_region(addr1).is_none());
+        assert!(gm.find_region(addr2).is_none());
+
+        // Note that the `region` returned by `remove_region` is an `Arc<_>`, so users generally
+        // cannot mutate it (change its base address).  In this test, we can (we could unwrap the
+        // `Arc<_>`), but our users generally cannot, hence why this interface exists.
+        let region = GuestRegionMmap::with_arc(Arc::clone(region.get_mapping()), addr2).unwrap();
+
+        let gm = gm.insert_region(Arc::new(region)).unwrap();
+
+        assert!(gm.find_region(addr1).is_none());
+        assert!(gm.find_region(addr2).is_some());
+    }
 }

From 41ac284d7e7ee13536c8de0b318b0b1bcad61a7a Mon Sep 17 00:00:00 2001
From: Hanna Czenczek <hreitz@redhat.com>
Date: Fri, 30 May 2025 13:04:50 +0200
Subject: [PATCH 10/11] DESIGN: Document I/O virtual memory

Document in DESIGN.md how I/O virtual memory is handled.

Signed-off-by: Hanna Czenczek <hreitz@redhat.com>
---
 DESIGN.md | 30 +++++++++++++++++++++++++++---
 1 file changed, 27 insertions(+), 3 deletions(-)

diff --git a/DESIGN.md b/DESIGN.md
index 5915f50e..c3098d85 100644
--- a/DESIGN.md
+++ b/DESIGN.md
@@ -2,8 +2,8 @@
 
 ## Objectives
 
-- Provide a set of traits for accessing and configuring the physical memory of
-  a virtual machine.
+- Provide a set of traits for accessing and configuring the physical and/or
+  I/O virtual memory of a virtual machine.
 - Provide a clean abstraction of the VM memory such that rust-vmm components
   can use it without depending on the implementation details specific to
   different VMMs.
@@ -122,6 +122,29 @@ let buf = &mut [0u8; 5];
 let result = guest_memory_mmap.write(buf, addr);
 ```
 
+### I/O Virtual Address Space
+
+When using an IOMMU, there no longer is direct access to the guest (physical)
+address space, but instead only to I/O virtual address space.  In this case:
+
+- `IoMemory` replaces `GuestMemory`: It requires specifying the required access
+  permissions (which are relevant for virtual memory).  It also removes
+  interfaces that imply a mostly linear memory layout, because virtual memory is
+  fragmented into many pages instead of few (large) memory regions.
+  - Any `IoMemory` still has a `GuestMemory` inside as the underlying address
+    space, but if an IOMMU is used, that will generally not be guest physical
+    address space.  With vhost-user, for example, it will be the VMM’s user
+    address space instead.
+  - `IommuMemory` as our only actually IOMMU-supporting `IoMemory`
+    implementation uses an `Iommu` object to translate I/O virtual addresses
+    (IOVAs) into VMM user addresses (VUAs), which are then passed to the inner
+    `GuestMemory` implementation (like `GuestMemoryMmap`).
+- `GuestAddress` (for compatibility) refers to an address in any of these
+  address spaces:
+  - Guest physical addresses (GPAs) when no IOMMU is used,
+  - I/O virtual addresses (IOVAs),
+  - VMM user addresses (VUAs).
+
 ### Utilities and Helpers
 
 The following utilities and helper traits/macros are imported from the
@@ -143,7 +166,8 @@ with minor changes:
 - `Address` inherits `AddressValue`
 - `GuestMemoryRegion` inherits `Bytes<MemoryRegionAddress, E = Error>`. The
   `Bytes` trait must be implemented.
-- `GuestMemory` has a generic implementation of `Bytes<GuestAddress>`.
+- `GuestMemory` has a generic implementation of `IoMemory`
+- `IoMemory` has a generic implementation of `Bytes<GuestAddress>`.
 
 **Types**:
 

From c955ffdb663557198363de324915e644bdf4b999 Mon Sep 17 00:00:00 2001
From: Hanna Czenczek <hreitz@redhat.com>
Date: Fri, 30 May 2025 16:49:59 +0200
Subject: [PATCH 11/11] CHANGELOG: Add I/O virtual memory entry

Signed-off-by: Hanna Czenczek <hreitz@redhat.com>
---
 CHANGELOG.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index dfe41067..e5b3c433 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,11 @@
 # Changelog
 
+## Upcoming version
+
+### Added
+
+- \[[#327](https://github.com/rust-vmm/vm-memory/pull/327)\] I/O virtual memory support via `IoMemory`, `IommuMemory`, and `Iommu`/`Iotlb`
+
 ## \[v0.17.0\]
 
 ### Added