From 8e761a1d2c68690736ba840020375c9fad72dba7 Mon Sep 17 00:00:00 2001 From: marc0246 <40955683+marc0246@users.noreply.github.com> Date: Thu, 4 Jul 2024 11:27:25 +0200 Subject: [PATCH] Task graph [1/10]: resource synchronization state tracking --- Cargo.lock | 37 + Cargo.toml | 3 + vulkano-taskgraph/Cargo.toml | 26 + vulkano-taskgraph/LICENSE-APACHE | 1 + vulkano-taskgraph/LICENSE-MIT | 1 + vulkano-taskgraph/src/lib.rs | 957 +++++++++++++ vulkano-taskgraph/src/resource.rs | 1982 +++++++++++++++++++++++++++ vulkano/src/memory/allocator/mod.rs | 9 +- vulkano/src/memory/mod.rs | 5 +- 9 files changed, 3017 insertions(+), 4 deletions(-) create mode 100644 vulkano-taskgraph/Cargo.toml create mode 120000 vulkano-taskgraph/LICENSE-APACHE create mode 120000 vulkano-taskgraph/LICENSE-MIT create mode 100644 vulkano-taskgraph/src/lib.rs create mode 100644 vulkano-taskgraph/src/resource.rs diff --git a/Cargo.lock b/Cargo.lock index 19b7964f73..7f9c37ee0f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -398,6 +398,14 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "concurrent-slotmap" +version = "0.1.0" +source = "git+https://github.com/vulkano-rs/concurrent-slotmap?rev=ba68c6afbc07d322c66f4af0dc675c71d0da392f#ba68c6afbc07d322c66f4af0dc675c71d0da392f" +dependencies = [ + "virtual-buffer", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -1693,6 +1701,12 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rangemap" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60fcc7d6849342eff22c4350c8b9a989ee8ceabc4b481253e8946b9fe83d684" + [[package]] name = "raw-window-handle" version = "0.4.3" @@ -2326,6 +2340,16 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "virtual-buffer" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00bbb9a832cd697a36c2abd5ef58c263b0bc33cdf280f704b895646ed3e9f595" +dependencies = [ + "libc", + "windows-targets 0.52.0", +] + [[package]] name = "vk-parse" version = "0.12.0" @@ -2388,6 +2412,19 @@ dependencies = [ "vulkano", ] +[[package]] +name = "vulkano-taskgraph" +version = "0.34.0" +dependencies = [ + "ash", + "concurrent-slotmap", + "parking_lot", + "rangemap", + "smallvec", + "thread_local", + "vulkano", +] + [[package]] name = "vulkano-util" version = "0.34.0" diff --git a/Cargo.toml b/Cargo.toml index 746f52a87f..7623c8533e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,7 @@ members = [ "vulkano", "vulkano-macros", "vulkano-shaders", + "vulkano-taskgraph", "vulkano-util", # "vulkano-win", ] @@ -41,6 +42,7 @@ ahash = "0.8" # https://github.com/KhronosGroup/Vulkan-Headers/commits/main/registry/vk.xml ash = "0.38.0" bytemuck = "1.9" +concurrent-slotmap = { git = "https://github.com/vulkano-rs/concurrent-slotmap", rev = "ba68c6afbc07d322c66f4af0dc675c71d0da392f" } core-graphics-types = "0.1" crossbeam-queue = "0.3" half = "2.0" @@ -54,6 +56,7 @@ parking_lot = "0.12" proc-macro2 = "1.0" proc-macro-crate = "2.0" quote = "1.0" +rangemap = "1.5" raw-window-handle = "0.6" serde = "1.0" serde_json = "1.0" diff --git a/vulkano-taskgraph/Cargo.toml b/vulkano-taskgraph/Cargo.toml new file mode 100644 index 0000000000..3667a8d14d --- /dev/null +++ b/vulkano-taskgraph/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "vulkano-taskgraph" +version = "0.34.0" +authors = ["The vulkano contributors"] +repository = "https://github.com/vulkano-rs/vulkano/tree/master/vulkano-taskgraph" +description = "Vulkano's task graph implementation" +documentation = "https://docs.rs/vulkano-taskgraph" +readme = "../README.md" +edition = { workspace = true } +rust-version = { workspace = true } +license = { workspace = true } +homepage = { workspace = true } +keywords = { workspace = true } +categories = { workspace = true } + +[dependencies] +ash = { workspace = true } +concurrent-slotmap = { workspace = true } +parking_lot = { workspace = true } +rangemap = { workspace = true } +smallvec = { workspace = true } +thread_local = { workspace = true } +vulkano = { workspace = true } + +[lints] +workspace = true diff --git a/vulkano-taskgraph/LICENSE-APACHE b/vulkano-taskgraph/LICENSE-APACHE new file mode 120000 index 0000000000..965b606f33 --- /dev/null +++ b/vulkano-taskgraph/LICENSE-APACHE @@ -0,0 +1 @@ +../LICENSE-APACHE \ No newline at end of file diff --git a/vulkano-taskgraph/LICENSE-MIT b/vulkano-taskgraph/LICENSE-MIT new file mode 120000 index 0000000000..76219eb72e --- /dev/null +++ b/vulkano-taskgraph/LICENSE-MIT @@ -0,0 +1 @@ +../LICENSE-MIT \ No newline at end of file diff --git a/vulkano-taskgraph/src/lib.rs b/vulkano-taskgraph/src/lib.rs new file mode 100644 index 0000000000..1e920ca788 --- /dev/null +++ b/vulkano-taskgraph/src/lib.rs @@ -0,0 +1,957 @@ +// FIXME: +#![allow(unused)] +#![forbid(unsafe_op_in_unsafe_fn)] + +use ash::vk::{self, Handle}; +use concurrent_slotmap::SlotId; +use parking_lot::RwLock; +use resource::{BufferRange, BufferState, DeathRow, ImageState, Resources, SwapchainState}; +use std::{ + any::{Any, TypeId}, + cell::Cell, + cmp, + error::Error, + fmt, + hash::{Hash, Hasher}, + marker::PhantomData, + ops::{Deref, DerefMut, Range, RangeBounds}, + sync::Arc, + thread, +}; +use vulkano::{ + buffer::{Buffer, BufferContents, BufferMemory, Subbuffer}, + command_buffer::sys::{RawCommandBuffer, RawRecordingCommandBuffer}, + device::DeviceOwned, + image::Image, + memory::{ + allocator::{align_down, align_up}, + DeviceAlignment, MappedMemoryRange, MemoryPropertyFlags, ResourceMemory, + }, + swapchain::Swapchain, + DeviceSize, ValidationError, VulkanError, VulkanObject, +}; + +pub mod resource; + +/// A task represents a unit of work to be recorded to a command buffer. +pub trait Task: Any + Send + Sync { + // Potentially TODO: + // fn update(&mut self, ...) {} + + /// Executes the task, which should record its commands using the provided context. + /// + /// # Safety + /// + /// - Every subresource in the [task's input/output interface] must not be written to + /// concurrently in any other tasks during execution on the device. + /// - Every subresource in the task's input/output interface, if it's a [host access], must not + /// be written to concurrently in any other tasks during execution on the host. + /// - Every subresource in the task's input interface, if it's an [image access], must have had + /// its layout transitioned to the layout specified in the interface. + /// - Every subresource in the task's input interface, if the resource's [sharing mode] is + /// exclusive, must be currently owned by the queue family the task is executing on. + unsafe fn execute(&self, ctx: &mut TaskContext<'_>) -> TaskResult; +} + +impl dyn Task { + /// Returns `true` if `self` is of type `T`. + #[inline] + pub fn is(&self) -> bool { + self.type_id() == TypeId::of::() + } + + /// Returns a reference to the inner value if it is of type `T`, or returns `None` otherwise. + #[inline] + pub fn downcast_ref(&self) -> Option<&T> { + if self.is::() { + // SAFETY: We just checked that the type is correct. + Some(unsafe { self.downcast_unchecked_ref() }) + } else { + None + } + } + + /// Returns a reference to the inner value if it is of type `T`, or returns `None` otherwise. + #[inline] + pub fn downcast_mut(&mut self) -> Option<&mut T> { + if self.is::() { + // SAFETY: We just checked that the type is correct. + Some(unsafe { self.downcast_unchecked_mut() }) + } else { + None + } + } + + /// Returns a reference to the inner value without checking if it is of type `T`. + /// + /// # Safety + /// + /// `self` must be of type `T`. + #[inline] + pub unsafe fn downcast_unchecked_ref(&self) -> &T { + // SAFETY: The caller must guarantee that the type is correct. + unsafe { &*<*const dyn Task>::cast::(self) } + } + + /// Returns a reference to the inner value without checking if it is of type `T`. + /// + /// # Safety + /// + /// `self` must be of type `T`. + #[inline] + pub unsafe fn downcast_unchecked_mut(&mut self) -> &mut T { + // SAFETY: The caller must guarantee that the type is correct. + unsafe { &mut *<*mut dyn Task>::cast::(self) } + } +} + +impl fmt::Debug for dyn Task { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Task").finish_non_exhaustive() + } +} + +/// The context of a task. +/// +/// This gives you access to the current command buffer, resources, as well as resource cleanup. +pub struct TaskContext<'a> { + resources: &'a Resources, + death_row: Cell>, + current_command_buffer: Cell>, + command_buffers: Cell>>>, +} + +impl<'a> TaskContext<'a> { + /// Returns the current raw command buffer for the task. + /// + /// While this method is safe, using the command buffer isn't. You must guarantee that any + /// subresources you use while recording commands are either accounted for in the [task's + /// input/output interface], or that those subresources don't require any synchronization + /// (including layout transitions and queue family ownership transfers), or that no other task + /// is accessing the subresources at the same time without appropriate synchronization. + /// + /// # Panics + /// + /// - Panics if called more than once. + // TODO: We could alternatively to ^ pass two parameters to `Task::execute`. + #[inline] + pub fn raw_command_buffer(&self) -> &'a mut RawRecordingCommandBuffer { + self.current_command_buffer + .take() + .expect("`TaskContext::raw_command_buffer` can only be called once") + } + + /// Pushes a command buffer into the list of command buffers to be executed on the queue. + /// + /// All command buffers will be executed in the order in which they are pushed after the task + /// has finished execution. That means in particular, that commands recorded by the task will + /// start execution before execution of any pushed command buffers starts. + /// + /// # Safety + /// + /// The same safety preconditions apply as outlined in the [`raw_command_buffer`] method. Since + /// the command buffer will be executed on the same queue right after the current command + /// buffer, without any added synchronization, it must be safe to do so. The given command + /// buffer must not do any accesses not accounted for in the [task's input/output interface], + /// or ensure that such accesses are appropriately synchronized. + /// + /// [`raw_command_buffer`]: Self::raw_command_buffer + #[inline] + pub unsafe fn push_command_buffer(&self, command_buffer: Arc) { + let vec = self.command_buffers.take().unwrap(); + vec.push(command_buffer); + self.command_buffers.set(Some(vec)); + } + + /// Extends the list of command buffers to be executed on the queue. + /// + /// This function behaves identically to the [`push_command_buffer`] method, execept that it + /// pushes all command buffers from the given iterator in order. + /// + /// # Safety + /// + /// See the [`push_command_buffer`] method for the safety preconditions. + /// + /// [`push_command_buffer`]: Self::push_command_buffer + #[inline] + pub unsafe fn extend_command_buffers( + &self, + command_buffers: impl IntoIterator>, + ) { + let vec = self.command_buffers.take().unwrap(); + vec.extend(command_buffers); + self.command_buffers.set(Some(vec)); + } + + /// Returns the buffer corresponding to `id`, or returns an error if it isn't present. + #[inline] + pub fn buffer(&self, id: Id) -> TaskResult<&'a BufferState> { + // SAFETY: Ensured by the caller of `Task::execute`. + Ok(unsafe { self.resources.buffer_unprotected(id) }?) + } + + /// Returns the image corresponding to `id`, or returns an error if it isn't present. + #[inline] + pub fn image(&self, id: Id) -> TaskResult<&'a ImageState> { + // SAFETY: Ensured by the caller of `Task::execute`. + Ok(unsafe { self.resources.image_unprotected(id) }?) + } + + /// Returns the swapchain corresponding to `id`, or returns an error if it isn't present. + #[inline] + pub fn swapchain(&self, id: Id) -> TaskResult<&'a SwapchainState> { + // SAFETY: Ensured by the caller of `Task::execute`. + Ok(unsafe { self.resources.swapchain_unprotected(id) }?) + } + + /// Returns the `Resources` collection. + #[inline] + pub fn resources(&self) -> &'a Resources { + self.resources + } + + /// Tries to get read access to a portion of the buffer corresponding to `id`. + /// + /// If host read access of the portion of the buffer is not accounted for in the [task's + /// input/output interface], this method will return an error. + /// + /// If the memory backing the buffer is not [host-coherent], then this method will check a + /// range that is potentially larger than the given range, because the range given to + /// [`invalidate_range`] must be aligned to the [`non_coherent_atom_size`]. This means that for + /// example if your Vulkan implementation reports an atom size of 64, and you tried to put 2 + /// subbuffers of size 32 in the same buffer, one at offset 0 and one at offset 32, while the + /// buffer is backed by non-coherent memory, then invalidating one subbuffer would also + /// invalidate the other subbuffer. This can lead to data races and is therefore not allowed. + /// What you should do in that case is ensure that each subbuffer is aligned to the + /// non-coherent atom size, so in this case one would be at offset 0 and the other at offset + /// 64. + /// + /// If the memory backing the buffer is not managed by vulkano (i.e. the buffer was created + /// by [`RawBuffer::assume_bound`]), then it can't be read using this method and an error will + /// be returned. + /// + /// # Panics + /// + /// - Panics if the alignment of `T` is greater than 64. + /// - Panics if [`Subbuffer::slice`] with the given `range` panics. + /// - Panics if [`Subbuffer::reinterpret`] to the given `T` panics. + /// + /// [host-coherent]: vulkano::memory::MemoryPropertyFlags::HOST_COHERENT + /// [`invalidate_range`]: vulkano::memory::ResourceMemory::invalidate_range + /// [`non_coherent_atom_size`]: vulkano::device::DeviceProperties::non_coherent_atom_size + /// [`RawBuffer::assume_bound`]: vulkano::buffer::sys::RawBuffer::assume_bound + pub fn read_buffer( + &self, + id: Id, + range: impl RangeBounds, + ) -> TaskResult> { + #[cold] + unsafe fn invalidate_subbuffer( + ctx: &TaskContext<'_>, + subbuffer: &Subbuffer<[u8]>, + allocation: &ResourceMemory, + atom_size: DeviceAlignment, + ) -> TaskResult { + // This works because the memory allocator must align allocations to the non-coherent + // atom size when the memory is host-visible but not host-coherent. + let start = align_down(subbuffer.offset(), atom_size); + let end = cmp::min( + align_up(subbuffer.offset() + subbuffer.size(), atom_size), + allocation.size(), + ); + let range = Range { start, end }; + + ctx.validate_read_buffer(subbuffer.buffer(), range.clone())?; + + let memory_range = MappedMemoryRange { + offset: range.start, + size: range.end - range.start, + _ne: crate::NE, + }; + + // SAFETY: + // - We checked that the task has read access to the subbuffer above. + // - The caller must guarantee that the subbuffer falls within the mapped range of + // memory. + // - We ensure that memory mappings are always aligned to the non-coherent atom size for + // non-host-coherent memory, therefore the subbuffer's range aligned to the + // non-coherent atom size must fall within the mapped range of the memory. + unsafe { allocation.invalidate_range_unchecked(memory_range) } + .map_err(HostAccessError::Invalidate)?; + + Ok(()) + } + + assert!(T::LAYOUT.alignment().as_devicesize() <= 64); + + let buffer = self.buffer(id)?.buffer(); + let subbuffer = Subbuffer::from(buffer.clone()) + .slice(range) + .reinterpret::(); + + let allocation = match buffer.memory() { + BufferMemory::Normal(a) => a, + BufferMemory::Sparse => { + todo!("`TaskContext::read_buffer` doesn't support sparse binding yet") + } + BufferMemory::External => { + return Err(TaskError::HostAccess(HostAccessError::Unmanaged)) + } + _ => unreachable!(), + }; + + let mapped_slice = subbuffer.mapped_slice().map_err(|err| match err { + vulkano::sync::HostAccessError::NotHostMapped => HostAccessError::NotHostMapped, + vulkano::sync::HostAccessError::OutOfMappedRange => HostAccessError::OutOfMappedRange, + _ => unreachable!(), + })?; + + let atom_size = allocation.atom_size(); + + if let Some(atom_size) = atom_size { + // SAFETY: + // `subbuffer.mapped_slice()` didn't return an error, which means that the subbuffer + // falls within the mapped range of the memory. + unsafe { invalidate_subbuffer(self, subbuffer.as_bytes(), allocation, atom_size) }?; + } else { + let range = subbuffer.offset()..subbuffer.offset() + subbuffer.size(); + self.validate_write_buffer(buffer, range)?; + } + + // SAFETY: We checked that the task has read access to the subbuffer above, which also + // includes the guarantee that no other tasks can be writing the subbuffer on neither the + // host nor the device. The same task cannot obtain another `BufferWriteGuard` to the + // subbuffer because `TaskContext::write_buffer` requires a mutable reference. + let data = unsafe { &*T::ptr_from_slice(mapped_slice) }; + + Ok(BufferReadGuard { data }) + } + + fn validate_read_buffer(&self, buffer: &Buffer, range: BufferRange) -> TaskResult { + todo!() + } + + /// Gets read access to a portion of the buffer corresponding to `id` without checking if this + /// access is accounted for in the [task's input/output interface]. + /// + /// This method doesn't do any host cache control. If the memory backing the buffer is not + /// [host-coherent], you must call [`invalidate_range`] in order for any device writes to be + /// visible to the host, and must not forget that such flushes must be aligned to the + /// [`non_coherent_atom_size`] and hence the aligned range must be accounted for in the task's + /// input/output interface. + /// + /// If the memory backing the buffer is not managed by vulkano (i.e. the buffer was created + /// by [`RawBuffer::assume_bound`]), then it can't be read using this method and an error will + /// be returned. + /// + /// # Safety + /// + /// This access must be accounted for in the task's input/output interface. + /// + /// # Panics + /// + /// - Panics if the alignment of `T` is greater than 64. + /// - Panics if [`Subbuffer::slice`] with the given `range` panics. + /// - Panics if [`Subbuffer::reinterpret`] to the given `T` panics. + /// + /// [host-coherent]: vulkano::memory::MemoryPropertyFlags::HOST_COHERENT + /// [`invalidate_range`]: vulkano::memory::ResourceMemory::invalidate_range + /// [`non_coherent_atom_size`]: vulkano::device::DeviceProperties::non_coherent_atom_size + /// [`RawBuffer::assume_bound`]: vulkano::buffer::sys::RawBuffer::assume_bound + pub unsafe fn read_buffer_unchecked( + &self, + id: Id, + range: impl RangeBounds, + ) -> TaskResult<&T> { + assert!(T::LAYOUT.alignment().as_devicesize() <= 64); + + let buffer = self.buffer(id)?.buffer(); + let subbuffer = Subbuffer::from(buffer.clone()) + .slice(range) + .reinterpret::(); + + let allocation = match buffer.memory() { + BufferMemory::Normal(a) => a, + BufferMemory::Sparse => { + todo!("`TaskContext::read_buffer_unchecked` doesn't support sparse binding yet"); + } + BufferMemory::External => { + return Err(TaskError::HostAccess(HostAccessError::Unmanaged)); + } + _ => unreachable!(), + }; + + let mapped_slice = subbuffer.mapped_slice().map_err(|err| match err { + vulkano::sync::HostAccessError::NotHostMapped => HostAccessError::NotHostMapped, + vulkano::sync::HostAccessError::OutOfMappedRange => HostAccessError::OutOfMappedRange, + _ => unreachable!(), + })?; + + // SAFETY: The caller must ensure that access to the data is synchronized. + let data = unsafe { &*T::ptr_from_slice(mapped_slice) }; + + Ok(data) + } + + /// Tries to get write access to a portion of the buffer corresponding to `id`. + /// + /// If host write access of the portion of the buffer is not accounted for in the [task's + /// input/output interface], this method will return an error. + /// + /// If the memory backing the buffer is not [host-coherent], then this method will check a + /// range that is potentially larger than the given range, because the range given to + /// [`flush_range`] must be aligned to the [`non_coherent_atom_size`]. This means that for + /// example if your Vulkan implementation reports an atom size of 64, and you tried to put 2 + /// subbuffers of size 32 in the same buffer, one at offset 0 and one at offset 32, while the + /// buffer is backed by non-coherent memory, then invalidating one subbuffer would also + /// invalidate the other subbuffer. This can lead to data races and is therefore not allowed. + /// What you should do in that case is ensure that each subbuffer is aligned to the + /// non-coherent atom size, so in this case one would be at offset 0 and the other at offset + /// 64. + /// + /// If the memory backing the buffer is not managed by vulkano (i.e. the buffer was created + /// by [`RawBuffer::assume_bound`]), then it can't be written using this method and an error + /// will be returned. + /// + /// # Panics + /// + /// - Panics if the alignment of `T` is greater than 64. + /// - Panics if [`Subbuffer::slice`] with the given `range` panics. + /// - Panics if [`Subbuffer::reinterpret`] to the given `T` panics. + /// + /// [host-coherent]: vulkano::memory::MemoryPropertyFlags::HOST_COHERENT + /// [`flush_range`]: vulkano::memory::ResourceMemory::flush_range + /// [`non_coherent_atom_size`]: vulkano::device::DeviceProperties::non_coherent_atom_size + /// [`RawBuffer::assume_bound`]: vulkano::buffer::sys::RawBuffer::assume_bound + pub fn write_buffer( + &mut self, + id: Id, + range: impl RangeBounds, + ) -> TaskResult> { + #[cold] + unsafe fn invalidate_subbuffer( + ctx: &TaskContext<'_>, + subbuffer: &Subbuffer<[u8]>, + allocation: &ResourceMemory, + atom_size: DeviceAlignment, + ) -> TaskResult { + // This works because the memory allocator must align allocations to the non-coherent + // atom size when the memory is host-visible but not host-coherent. + let start = align_down(subbuffer.offset(), atom_size); + let end = cmp::min( + align_up(subbuffer.offset() + subbuffer.size(), atom_size), + allocation.size(), + ); + let range = Range { start, end }; + + ctx.validate_write_buffer(subbuffer.buffer(), range.clone())?; + + let memory_range = MappedMemoryRange { + offset: range.start, + size: range.end - range.start, + _ne: crate::NE, + }; + + // SAFETY: + // - We checked that the task has write access to the subbuffer above. + // - The caller must guarantee that the subbuffer falls within the mapped range of + // memory. + // - We ensure that memory mappings are always aligned to the non-coherent atom size for + // non-host-coherent memory, therefore the subbuffer's range aligned to the + // non-coherent atom size must fall within the mapped range of the memory. + unsafe { allocation.invalidate_range_unchecked(memory_range) } + .map_err(HostAccessError::Invalidate)?; + + Ok(()) + } + + assert!(T::LAYOUT.alignment().as_devicesize() <= 64); + + let buffer = self.buffer(id)?.buffer(); + let subbuffer = Subbuffer::from(buffer.clone()) + .slice(range) + .reinterpret::(); + + let allocation = match buffer.memory() { + BufferMemory::Normal(a) => a, + BufferMemory::Sparse => { + todo!("`TaskContext::write_buffer` doesn't support sparse binding yet"); + } + BufferMemory::External => { + return Err(TaskError::HostAccess(HostAccessError::Unmanaged)); + } + _ => unreachable!(), + }; + + let mapped_slice = subbuffer.mapped_slice().map_err(|err| match err { + vulkano::sync::HostAccessError::NotHostMapped => HostAccessError::NotHostMapped, + vulkano::sync::HostAccessError::OutOfMappedRange => HostAccessError::OutOfMappedRange, + _ => unreachable!(), + })?; + + let atom_size = allocation.atom_size(); + + if let Some(atom_size) = atom_size { + // SAFETY: + // `subbuffer.mapped_slice()` didn't return an error, which means that the subbuffer + // falls within the mapped range of the memory. + unsafe { invalidate_subbuffer(self, subbuffer.as_bytes(), allocation, atom_size) }?; + } else { + let range = subbuffer.offset()..subbuffer.offset() + subbuffer.size(); + self.validate_write_buffer(buffer, range)?; + } + + // SAFETY: We checked that the task has write access to the subbuffer above, which also + // includes the guarantee that no other tasks can be accessing the subbuffer on neither the + // host nor the device. The same task cannot obtain another `BufferWriteGuard` to the + // subbuffer because `TaskContext::write_buffer` requires a mutable reference. + let data = unsafe { &mut *T::ptr_from_slice(mapped_slice) }; + + Ok(BufferWriteGuard { + subbuffer: subbuffer.into_bytes(), + data, + atom_size, + }) + } + + fn validate_write_buffer(&self, buffer: &Buffer, range: BufferRange) -> TaskResult { + todo!() + } + + /// Gets write access to a portion of the buffer corresponding to `id` without checking if this + /// access is accounted for in the [task's input/output interface]. + /// + /// This method doesn't do any host cache control. If the memory backing the buffer is not + /// [host-coherent], you must call [`flush_range`] in order for any writes to be available to + /// the host memory domain, and must not forget that such flushes must be aligned to the + /// [`non_coherent_atom_size`] and hence the aligned range must be accounted for in the task's + /// input/output interface. + /// + /// If the memory backing the buffer is not managed by vulkano (i.e. the buffer was created + /// by [`RawBuffer::assume_bound`]), then it can't be written using this method and an error + /// will be returned. + /// + /// # Safety + /// + /// This access must be accounted for in the task's input/output interface. + /// + /// # Panics + /// + /// - Panics if the alignment of `T` is greater than 64. + /// - Panics if [`Subbuffer::slice`] with the given `range` panics. + /// - Panics if [`Subbuffer::reinterpret`] to the given `T` panics. + /// + /// [host-coherent]: vulkano::memory::MemoryPropertyFlags::HOST_COHERENT + /// [`flush_range`]: vulkano::memory::ResourceMemory::flush_range + /// [`non_coherent_atom_size`]: vulkano::device::DeviceProperties::non_coherent_atom_size + /// [`RawBuffer::assume_bound`]: vulkano::buffer::sys::RawBuffer::assume_bound + pub unsafe fn write_buffer_unchecked( + &mut self, + id: Id, + range: impl RangeBounds, + ) -> TaskResult<&mut T> { + assert!(T::LAYOUT.alignment().as_devicesize() <= 64); + + let buffer = self.buffer(id)?.buffer(); + let subbuffer = Subbuffer::from(buffer.clone()) + .slice(range) + .reinterpret::(); + + let allocation = match buffer.memory() { + BufferMemory::Normal(a) => a, + BufferMemory::Sparse => { + todo!("`TaskContext::write_buffer_unchecked` doesn't support sparse binding yet"); + } + BufferMemory::External => { + return Err(TaskError::HostAccess(HostAccessError::Unmanaged)); + } + _ => unreachable!(), + }; + + let mapped_slice = subbuffer.mapped_slice().map_err(|err| match err { + vulkano::sync::HostAccessError::NotHostMapped => HostAccessError::NotHostMapped, + vulkano::sync::HostAccessError::OutOfMappedRange => HostAccessError::OutOfMappedRange, + _ => unreachable!(), + })?; + + // SAFETY: The caller must ensure that access to the data is synchronized. + let data = unsafe { &mut *T::ptr_from_slice(mapped_slice) }; + + Ok(data) + } + + /// Queues the destruction of the buffer corresponding to `id` after the destruction of the + /// command buffer(s) for this task. + // FIXME: unsafe + #[inline] + pub unsafe fn destroy_buffer(&self, id: Id) -> TaskResult { + let state = unsafe { self.resources.remove_buffer(id) }?; + let death_row = self.death_row.take().unwrap(); + // FIXME: + death_row.push(state.buffer().clone()); + self.death_row.set(Some(death_row)); + + Ok(()) + } + + /// Queues the destruction of the image corresponding to `id` after the destruction of the + /// command buffer(s) for this task. + // FIXME: unsafe + #[inline] + pub unsafe fn destroy_image(&self, id: Id) -> TaskResult { + let state = unsafe { self.resources.remove_image(id) }?; + let death_row = self.death_row.take().unwrap(); + // FIXME: + death_row.push(state.image().clone()); + self.death_row.set(Some(death_row)); + + Ok(()) + } + + /// Queues the destruction of the swapchain corresponding to `id` after the destruction of the + /// command buffer(s) for this task. + // FIXME: unsafe + #[inline] + pub unsafe fn destroy_swapchain(&self, id: Id) -> TaskResult { + let state = unsafe { self.resources.remove_swapchain(id) }?; + let death_row = self.death_row.take().unwrap(); + // FIXME: + death_row.push(state.swapchain().clone()); + self.death_row.set(Some(death_row)); + + Ok(()) + } +} + +/// Allows you to read a subbuffer from the host. +/// +/// This type is created by the [`read_buffer`] method on [`TaskContext`]. +/// +/// [`read_buffer`]: TaskContext::read_buffer +// NOTE(Marc): This type doesn't actually do anything, but exists for forward-compatibility. +#[derive(Debug)] +pub struct BufferReadGuard<'a, T: ?Sized> { + data: &'a T, +} + +impl Deref for BufferReadGuard<'_, T> { + type Target = T; + + #[inline] + fn deref(&self) -> &Self::Target { + self.data + } +} + +/// Allows you to write a subbuffer from the host. +/// +/// This type is created by the [`write_buffer`] method on [`TaskContext`]. +/// +/// [`write_buffer`]: TaskContext::write_buffer +pub struct BufferWriteGuard<'a, T: ?Sized> { + subbuffer: Subbuffer<[u8]>, + data: &'a mut T, + atom_size: Option, +} + +impl Deref for BufferWriteGuard<'_, T> { + type Target = T; + + #[inline] + fn deref(&self) -> &Self::Target { + self.data + } +} + +impl DerefMut for BufferWriteGuard<'_, T> { + #[inline] + fn deref_mut(&mut self) -> &mut Self::Target { + self.data + } +} + +impl Drop for BufferWriteGuard<'_, T> { + #[inline] + fn drop(&mut self) { + #[cold] + fn flush_subbuffer(subbuffer: &Subbuffer<[u8]>, atom_size: DeviceAlignment) { + let allocation = match subbuffer.buffer().memory() { + BufferMemory::Normal(a) => a, + _ => unreachable!(), + }; + let atom_size = allocation.atom_size().unwrap(); + + let memory_range = MappedMemoryRange { + offset: align_down(subbuffer.offset(), atom_size), + size: cmp::min( + align_up(subbuffer.offset() + subbuffer.size(), atom_size), + allocation.size(), + ) - subbuffer.offset(), + _ne: crate::NE, + }; + + // SAFETY: `TaskContext::write_buffer` ensures that the task has write access to this + // subbuffer aligned to the non-coherent atom size. + if let Err(err) = unsafe { allocation.flush_range_unchecked(memory_range) } { + if !thread::panicking() { + panic!("failed to flush buffer write: {err:?}"); + } + } + } + + if let Some(atom_size) = self.atom_size { + flush_subbuffer(&self.subbuffer, atom_size); + } + } +} + +/// The type of result returned by a task. +pub type TaskResult = ::std::result::Result; + +/// Error that can happen inside a task. +#[derive(Debug)] +pub enum TaskError { + InvalidSlot(InvalidSlotError), + HostAccess(HostAccessError), + ValidationError(Box), +} + +impl From for TaskError { + fn from(err: InvalidSlotError) -> Self { + Self::InvalidSlot(err) + } +} + +impl From for TaskError { + fn from(err: HostAccessError) -> Self { + Self::HostAccess(err) + } +} + +impl From> for TaskError { + fn from(err: Box) -> Self { + Self::ValidationError(err) + } +} + +impl fmt::Display for TaskError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let msg = match self { + Self::InvalidSlot(_) => "invalid slot", + Self::HostAccess(_) => "a host access error occured", + Self::ValidationError(_) => "a validation error occured", + }; + + f.write_str(msg) + } +} + +impl Error for TaskError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::InvalidSlot(err) => Some(err), + Self::HostAccess(err) => Some(err), + Self::ValidationError(err) => Some(err), + } + } +} + +/// Error that can happen when trying to retrieve a Vulkan object or state by [`Id`]. +#[derive(Debug)] +pub struct InvalidSlotError { + object_type: vk::ObjectType, + slot: SlotId, +} + +impl InvalidSlotError { + fn new(id: Id) -> Self { + InvalidSlotError { + object_type: O::Handle::TYPE, + slot: id.slot, + } + } +} + +impl fmt::Display for InvalidSlotError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let &InvalidSlotError { object_type, slot } = self; + + write!(f, "invalid slot for object type {object_type:?}: {slot:?}") + } +} + +impl Error for InvalidSlotError {} + +/// Error that can happen when attempting to read or write a resource from the host. +#[derive(Debug)] +pub enum HostAccessError { + Invalidate(VulkanError), + Unmanaged, + NotHostMapped, + OutOfMappedRange, +} + +impl fmt::Display for HostAccessError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let msg = match self { + Self::Invalidate(_) => "invalidating the device memory failed", + Self::Unmanaged => "the resource is not managed by vulkano", + Self::NotHostMapped => "the device memory is not current host-mapped", + Self::OutOfMappedRange => { + "the requested range is not within the currently mapped range of device memory" + } + }; + + f.write_str(msg) + } +} + +impl Error for HostAccessError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Invalidate(err) => Some(err), + _ => None, + } + } +} + +/// A task created from a closure. +/// +/// This loses you the downcasting capabilities, but that should be fine for most use cases. Using +/// a closure is much more ergonomic than having to define a new type for each kind of task. +#[derive(Clone, Copy)] +pub struct AdHocTask(pub F); + +impl From for AdHocTask { + #[inline] + fn from(f: F) -> Self { + AdHocTask(f) + } +} + +impl fmt::Debug for AdHocTask { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("AdHoc").finish_non_exhaustive() + } +} + +impl Task for AdHocTask +where + F: Fn(&mut TaskContext<'_>) -> TaskResult + Send + Sync + 'static, +{ + unsafe fn execute(&self, ctx: &mut TaskContext<'_>) -> TaskResult { + (self.0)(ctx) + } +} + +/// Specifies the type of queue family that a task can be executed on. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum QueueFamilyType { + /// Picks a queue family that supports graphics and transfer operations. + Graphics, + + /// Picks a queue family that supports compute and transfer operations. + Compute, + + /// Picks a queue family that supports transfer operations. + Transfer, + + // TODO: + // VideoDecode, + + // TODO: + // VideoEncode, + /// Picks the queue family of the given index. You should generally avoid this and use one of + /// the other variants, so that the task graph compiler can pick the most optimal queue family + /// indices that still satisfy the supported operations that the tasks require (and also, it's + /// more convenient that way, as there's less to think about). Nevertheless, you may want to + /// use this if you're looking for some very specific outcome. + Specific { index: u32 }, +} + +/// This ID type is used throughout the crate to refer to Vulkan objects such as resource objects +/// and their synchronization state, synchronization object state, and other state. +/// +/// The type parameter denotes the type of object or state being referred to. +/// +/// Note that this ID **is not** globally unique. It is unique in the scope of a logical device. +#[repr(transparent)] +pub struct Id { + slot: SlotId, + marker: PhantomData T>, +} + +impl Id { + fn new(slot: SlotId) -> Self { + Id { + slot, + marker: PhantomData, + } + } + + fn index(self) -> u32 { + self.slot.index() + } +} + +impl Clone for Id { + #[inline] + fn clone(&self) -> Self { + *self + } +} + +impl Copy for Id {} + +impl fmt::Debug for Id { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Id") + .field("generation", &self.slot.generation()) + .field("index", &self.slot.index()) + .finish() + } +} + +impl PartialEq for Id { + #[inline] + fn eq(&self, other: &Self) -> bool { + self.slot == other.slot + } +} + +impl Eq for Id {} + +impl Hash for Id { + #[inline] + fn hash(&self, state: &mut H) { + self.slot.hash(state); + } +} + +/// A reference to some Vulkan object or state. +/// +/// When you use [`Id`] to retrieve something, you can get back a `Ref` with the same type +/// parameter, which you can then dereference to get at the underlying data denoted by the type +/// parameter. +pub struct Ref<'a, T>(concurrent_slotmap::Ref<'a, T>); + +impl Deref for Ref<'_, T> { + type Target = T; + + #[inline] + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl fmt::Debug for Ref<'_, T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&self.0, f) + } +} + +// SAFETY: ZSTs can always be safely produced out of thin air, barring any safety invariants they +// might impose, which in the case of `NonExhaustive` are none. +const NE: vulkano::NonExhaustive = + unsafe { ::std::mem::transmute::<(), ::vulkano::NonExhaustive>(()) }; diff --git a/vulkano-taskgraph/src/resource.rs b/vulkano-taskgraph/src/resource.rs new file mode 100644 index 0000000000..dd657c3d2d --- /dev/null +++ b/vulkano-taskgraph/src/resource.rs @@ -0,0 +1,1982 @@ +//! Synchronization state tracking of all resources. + +use crate::{Id, InvalidSlotError, Ref}; +use ash::vk; +use concurrent_slotmap::{epoch, SlotMap}; +use parking_lot::{Mutex, MutexGuard}; +use rangemap::RangeMap; +use smallvec::SmallVec; +use std::{ + any::Any, + cell::Cell, + fmt, + hash::Hash, + iter::FusedIterator, + mem, + num::NonZeroU32, + ops::Range, + sync::{ + atomic::{AtomicU32, Ordering}, + Arc, + }, +}; +use thread_local::ThreadLocal; +use vulkano::{ + buffer::{AllocateBufferError, Buffer, BufferCreateInfo}, + device::{Device, DeviceOwned}, + image::{ + AllocateImageError, Image, ImageAspects, ImageCreateFlags, ImageCreateInfo, ImageLayout, + ImageMemory, ImageSubresourceRange, + }, + memory::allocator::{AllocationCreateInfo, DeviceLayout, MemoryAllocator}, + swapchain::{Surface, Swapchain, SwapchainCreateInfo}, + sync::{ + fence::{Fence, FenceCreateFlags, FenceCreateInfo}, + semaphore::Semaphore, + AccessFlags, PipelineStages, + }, + DeviceSize, Validated, VulkanError, +}; + +static REGISTERED_DEVICES: Mutex> = Mutex::new(Vec::new()); + +/// Tracks the synchronization state of all resources. +/// +/// There can only exist one `Resources` collection per device, because there must only be one +/// source of truth in regards to the synchronization state of a resource. In a similar vein, each +/// resource in the collection must be unique. +// FIXME: Custom collector +// FIXME: Swapchain recreation +#[derive(Debug)] +pub struct Resources { + memory_allocator: Arc, + + global: epoch::GlobalHandle, + locals: ThreadLocal, + buffers: SlotMap, + images: SlotMap, + swapchains: SlotMap, + flights: SlotMap, +} + +#[derive(Debug)] +pub struct BufferState { + buffer: Arc, + // FIXME: This is terribly inefficient. + last_accesses: Mutex>, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct BufferAccess { + access_type: AccessType, + queue_family_index: u32, +} + +#[derive(Debug)] +pub struct ImageState { + image: Arc, + // FIXME: This is terribly inefficient. + last_accesses: Mutex>, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct ImageAccess { + access_type: AccessType, + layout_type: ImageLayoutType, + queue_family_index: u32, +} + +// FIXME: imported/exported semaphores +#[derive(Debug)] +pub struct SwapchainState { + swapchain: Arc, + images: SmallVec<[Arc; 3]>, + pub(crate) semaphores: SmallVec<[SwapchainSemaphoreState; 3]>, + flight_id: Id, + pub(crate) current_image_index: AtomicU32, +} + +#[derive(Debug)] +pub(crate) struct SwapchainSemaphoreState { + pub(crate) image_available_semaphore: Semaphore, + pub(crate) tasks_complete_semaphore: Semaphore, +} + +// FIXME: imported/exported fences +#[derive(Debug)] +pub struct Flight { + frame_count: NonZeroU32, + current_frame: AtomicU32, + fences: SmallVec<[Fence; 3]>, + pub(crate) state: Mutex, +} + +#[derive(Debug)] +pub(crate) struct FlightState { + pub(crate) swapchains: SmallVec<[Id; 1]>, + pub(crate) death_rows: SmallVec<[DeathRow; 3]>, +} + +pub(crate) type DeathRow = Vec>; + +impl Resources { + /// Creates a new `Resources` collection. + /// + /// # Panics + /// + /// - Panics if `memory_allocator.device()` already has a `Resources` collection associated + /// with it. + #[must_use] + pub fn new( + memory_allocator: Arc, + create_info: ResourcesCreateInfo, + ) -> Self { + let device = memory_allocator.device().clone(); + let mut registered_devices = REGISTERED_DEVICES.lock(); + let device_addr = Arc::as_ptr(&device) as usize; + + assert!( + !registered_devices.contains(&device_addr), + "the device already has a `Resources` collection associated with it", + ); + + registered_devices.push(device_addr); + + let global = epoch::GlobalHandle::new(); + + Resources { + memory_allocator, + locals: ThreadLocal::new(), + buffers: SlotMap::with_global(create_info.max_buffers, global.clone()), + images: SlotMap::with_global(create_info.max_images, global.clone()), + swapchains: SlotMap::with_global(create_info.max_swapchains, global.clone()), + flights: SlotMap::with_global(create_info.max_flights, global.clone()), + global, + } + } + + /// Returns the memory allocator that the collection was created with. + #[inline] + #[must_use] + pub fn memory_allocator(&self) -> &Arc { + &self.memory_allocator + } + + /// Creates a new buffer and adds it to the collection. + /// + /// # Panics + /// + /// - Panics if `create_info.size` is not zero. + /// + /// # Errors + /// + /// - Returns an error when [`Buffer::new`] returns an error. + pub fn create_buffer( + &self, + create_info: BufferCreateInfo, + allocation_info: AllocationCreateInfo, + layout: DeviceLayout, + ) -> Result, Validated> { + let buffer = Buffer::new( + self.memory_allocator.clone(), + create_info, + allocation_info, + layout, + )?; + + // SAFETY: We just created the buffer. + Ok(unsafe { self.add_buffer_unchecked(buffer) }) + } + + /// Creates a new image and adds it to the collection. + /// + /// # Errors + /// + /// - Returns an error when [`Image::new`] returns an error. + pub fn create_image( + &self, + create_info: ImageCreateInfo, + allocation_info: AllocationCreateInfo, + ) -> Result, Validated> { + let image = Image::new(self.memory_allocator.clone(), create_info, allocation_info)?; + + // SAFETY: We just created the image. + Ok(unsafe { self.add_image_unchecked(image) }) + } + + /// Creates a swapchain and adds it to the collection. `flight_id` is the [flight] which will + /// own the swapchain. + /// + /// # Panics + /// + /// - Panics if the instance of `surface` is not the same as that of `self.device()`. + /// - Panics if `create_info.min_image_count` is not greater than or equal to the number of + /// [frames] of the flight corresponding to `flight_id`. + /// + /// # Errors + /// + /// - Returns an error when [`Swapchain::new`] returns an error. + /// - Returns an error when [`add_swapchain`] returns an error. + /// + /// [`add_swapchain`]: Self::add_swapchain + pub fn create_swapchain( + &self, + surface: Arc, + create_info: SwapchainCreateInfo, + flight_id: Id, + ) -> Result, Validated> { + assert_eq!(surface.instance(), self.device().instance()); + + let frames_in_flight = self + .flights + .get(flight_id.slot, self.pin()) + .unwrap() + .frame_count(); + + assert!(create_info.min_image_count >= frames_in_flight); + + let (swapchain, images) = Swapchain::new(self.device().clone(), surface, create_info)?; + + // SAFETY: We just created the swapchain. + Ok(unsafe { self.add_swapchain_unchecked(swapchain, images, flight_id) }?) + } + + /// Creates a new [flight] with `frame_count` [frames] and adds it to the collection. + /// + /// # Panics + /// + /// - Panics if `frame_count` is zero. + /// + /// # Errors + /// + /// - Returns an error when [`Fence::new_unchecked`] returns an error. + pub fn create_flight(&self, frame_count: u32) -> Result, VulkanError> { + let frame_count = + NonZeroU32::new(frame_count).expect("a flight with zero frames is not valid"); + + let fences = (0..frame_count.get()) + .map(|_| { + // SAFETY: The parameters are valid. + unsafe { + Fence::new_unchecked( + self.device().clone(), + FenceCreateInfo { + flags: FenceCreateFlags::SIGNALED, + ..Default::default() + }, + ) + } + }) + .collect::>()?; + + let flight = Flight { + frame_count, + current_frame: AtomicU32::new(0), + fences, + state: Mutex::new(FlightState { + swapchains: SmallVec::new(), + death_rows: (0..frame_count.get()).map(|_| Vec::new()).collect(), + }), + }; + + Ok(Id::new(self.flights.insert(flight, self.pin()))) + } + + /// Adds a buffer to the collection. + /// + /// # Panics + /// + /// - Panics if any other references to the buffer exist. + /// - Panics if the device of `buffer` is not the same as that of `self`. + #[must_use] + pub fn add_buffer(&self, mut buffer: Arc) -> Id { + assert!(Arc::get_mut(&mut buffer).is_some()); + assert_eq!(buffer.device(), self.device()); + + unsafe { self.add_buffer_unchecked(buffer) } + } + + unsafe fn add_buffer_unchecked(&self, buffer: Arc) -> Id { + let state = BufferState { + buffer, + last_accesses: Mutex::new(RangeMap::new()), + }; + + Id::new(self.buffers.insert(state, self.pin())) + } + + /// Adds an image to the collection. + /// + /// # Panics + /// + /// - Panics if any other references to the image exist. + /// - Panics if the device of `image` is not the same as that of `self`. + /// - Panics if `image` is a swapchain image. + #[must_use] + pub fn add_image(&self, mut image: Arc) -> Id { + assert!(Arc::get_mut(&mut image).is_some()); + assert_eq!(image.device(), self.device()); + + assert!( + !matches!(image.memory(), ImageMemory::Swapchain { .. }), + "swapchain images cannot be added like regular images; please use \ + `Resources::add_swapchain` instead", + ); + + unsafe { self.add_image_unchecked(image) } + } + + unsafe fn add_image_unchecked(&self, image: Arc) -> Id { + let state = ImageState { + image, + last_accesses: Mutex::new(RangeMap::new()), + }; + + Id::new(self.images.insert(state, self.pin())) + } + + /// Adds a swapchain to the collection. `(swapchain, images)` must correspond to the value + /// returned by one of the [`Swapchain`] constructors or by [`Swapchain::recreate`]. + /// `flight_id` is the [flight] which will own the swapchain. + /// + /// # Panics + /// + /// - Panics if any other references to the swapchain exist. + /// - Panics if the device of `swapchain` is not the same as that of `self`. + /// - Panics if the `images` don't comprise the images of `swapchain`. + /// - Panics if `flight_id` is invalid. + /// - Panics if `swapchain.image_count()` is not greater than or equal to the number of + /// [frames] of the flight corresponding to `flight_id`. + /// + /// # Errors + /// + /// - Returns an error when [`Semaphore::new_unchecked`] returns an error. + pub fn add_swapchain( + &self, + swapchain: Arc, + images: Vec>, + flight_id: Id, + ) -> Result, VulkanError> { + assert_eq!(swapchain.device(), self.device()); + assert_eq!(images.len(), swapchain.image_count() as usize); + + let frames_in_flight = self + .flights + .get(flight_id.slot, self.pin()) + .unwrap() + .frame_count(); + + assert!(swapchain.image_count() >= frames_in_flight); + + for (index, image) in images.iter().enumerate() { + match image.memory() { + ImageMemory::Swapchain { + swapchain: image_swapchain, + image_index, + } => { + assert_eq!(image_swapchain, &swapchain); + assert_eq!(*image_index as usize, index); + } + _ => panic!("not a swapchain image"), + } + } + + // It is against the safety contract of `Arc::(de,in)crement_strong_count` to call them + // with a pointer obtained through `Arc::as_ptr`, even though that's perfectly safe to do, + // so we have to go through this hoop. + let ptr = Arc::into_raw(swapchain); + let mut swapchain = unsafe { Arc::from_raw(ptr) }; + + // The following is extremely cursed, but as of right now the only way to assert that we + // own the only references to the swapchain. + { + for _ in 0..images.len() { + // SAFETY: The pointer was obtained through `Arc::into_raw` above, and we checked + // that each of the images is a swapchain image belonging to the same swapchain + // also above, which means that there are at least `images.len()` references to + // this swapchain besides `swapchain` itself. + unsafe { Arc::decrement_strong_count(ptr) }; + } + + let we_own_the_only_references = Arc::get_mut(&mut swapchain).is_some(); + + for _ in 0..images.len() { + // SAFETY: Same as the `Arc::decrement_strong_count` above. + unsafe { Arc::increment_strong_count(ptr) }; + } + + assert!(we_own_the_only_references); + } + + unsafe { self.add_swapchain_unchecked(swapchain, images, flight_id) } + } + + unsafe fn add_swapchain_unchecked( + &self, + swapchain: Arc, + images: Vec>, + flight_id: Id, + ) -> Result, VulkanError> { + let guard = &self.pin(); + + let frames_in_flight = self + .flights + .get(flight_id.slot, guard) + .unwrap() + .frame_count(); + + let semaphores = (0..frames_in_flight) + .map(|_| { + Ok(SwapchainSemaphoreState { + // SAFETY: The parameters are valid. + image_available_semaphore: unsafe { + Semaphore::new_unchecked(self.device().clone(), Default::default()) + }?, + // SAFETY: The parameters are valid. + tasks_complete_semaphore: unsafe { + Semaphore::new_unchecked(self.device().clone(), Default::default()) + }?, + }) + }) + .collect::>()?; + + let state = SwapchainState { + swapchain, + images: images.into(), + semaphores, + flight_id, + current_image_index: AtomicU32::new(u32::MAX), + }; + + let id = Id::new(self.swapchains.insert(state, guard)); + + self.flights + .get(flight_id.slot, guard) + .unwrap() + .state + // FIXME: + .lock() + .swapchains + .push(id); + + Ok(id) + } + + /// Removes the buffer corresponding to `id`, or returns `None` if it isn't present. + /// + /// # Safety + /// + /// - Unless the buffer is being kept alive by other means, it must not be in use in any + /// pending command buffer, and if it is used in any command buffer that's in the executable + /// or recording state, that command buffer must never be executed. + pub unsafe fn remove_buffer(&self, id: Id) -> Result> { + self.buffers + .remove(id.slot, self.pin()) + .map(Ref) + .ok_or(InvalidSlotError::new(id)) + } + + /// Removes the image corresponding to `id`, or returns `None` if it isn't present. + /// + /// # Safety + /// + /// - Unless the image is being kept alive by other means, it must not be in use in any pending + /// command buffer, and if it is used in any command buffer that's in the executable or + /// recording state, that command buffer must never be executed. + pub unsafe fn remove_image(&self, id: Id) -> Result> { + self.images + .remove(id.slot, self.pin()) + .map(Ref) + .ok_or(InvalidSlotError::new(id)) + } + + /// Removes the swapchain corresponding to `id`, or returns `None` if it isn't present. + /// + /// # Safety + /// + /// - Unless the swapchain is being kept alive by other means, it must not be in use in any + /// pending command buffer, and if it is used in any command buffer that's in the executable + /// or recording state, that command buffer must never be executed. + pub unsafe fn remove_swapchain(&self, id: Id) -> Result> { + let state = self + .swapchains + .remove(id.slot, self.pin()) + .map(Ref) + .ok_or(InvalidSlotError::new(id))?; + let flight_id = state.flight_id; + + let flight = self.flights.get(flight_id.slot, self.pin()).unwrap(); + // FIXME: + let swapchains = &mut flight.state.lock().swapchains; + let index = swapchains.iter().position(|&x| x == id).unwrap(); + swapchains.remove(index); + + Ok(state) + } + + /// Returns the buffer corresponding to `id`, or returns `None` if it isn't present. + #[inline] + pub fn buffer(&self, id: Id) -> Result> { + self.buffers + .get(id.slot, self.pin()) + .map(Ref) + .ok_or(InvalidSlotError::new(id)) + } + + #[inline] + pub(crate) unsafe fn buffer_unprotected(&self, id: Id) -> Result<&BufferState> { + // SAFETY: Enforced by the caller. + unsafe { self.buffers.get_unprotected(id.slot) }.ok_or(InvalidSlotError::new(id)) + } + + /// Returns the image corresponding to `id`, or returns `None` if it isn't present. + #[inline] + pub fn image(&self, id: Id) -> Result> { + self.images + .get(id.slot, self.pin()) + .map(Ref) + .ok_or(InvalidSlotError::new(id)) + } + + #[inline] + pub(crate) unsafe fn image_unprotected(&self, id: Id) -> Result<&ImageState> { + // SAFETY: Enforced by the caller. + unsafe { self.images.get_unprotected(id.slot) }.ok_or(InvalidSlotError::new(id)) + } + + /// Returns the swapchain corresponding to `id`, or returns `None` if it isn't present. + #[inline] + pub fn swapchain(&self, id: Id) -> Result> { + self.swapchains + .get(id.slot, self.pin()) + .map(Ref) + .ok_or(InvalidSlotError::new(id)) + } + + #[inline] + pub(crate) unsafe fn swapchain_unprotected( + &self, + id: Id, + ) -> Result<&SwapchainState> { + // SAFETY: Enforced by the caller. + unsafe { self.swapchains.get_unprotected(id.slot) }.ok_or(InvalidSlotError::new(id)) + } + + /// Returns the [flight] corresponding to `id`, or returns `None` if it isn't present. + #[inline] + #[must_use] + pub fn flight(&self, id: Id) -> Option> { + self.flights.get(id.slot, self.pin()).map(Ref) + } + + #[inline] + pub(crate) unsafe fn flight_unprotected(&self, id: Id) -> Option<&Flight> { + // SAFETY: Enforced by the caller. + unsafe { self.flights.get_unprotected(id.slot) } + } + + #[inline] + pub(crate) fn pin(&self) -> epoch::Guard<'_> { + self.locals.get_or(|| self.global.register_local()).pin() + } + + pub(crate) fn try_advance_global_and_collect(&self, guard: &epoch::Guard<'_>) { + if guard.try_advance_global() { + self.buffers.try_collect(guard); + self.images.try_collect(guard); + self.swapchains.try_collect(guard); + self.flights.try_collect(guard); + } + } +} + +impl Drop for Resources { + fn drop(&mut self) { + let mut registered_devices = REGISTERED_DEVICES.lock(); + + // This can't panic because there's no way to construct this type without the device's + // address being inserted into the list. + let index = registered_devices + .iter() + .position(|&addr| addr == Arc::as_ptr(self.device()) as usize) + .unwrap(); + + registered_devices.remove(index); + } +} + +unsafe impl DeviceOwned for Resources { + #[inline] + fn device(&self) -> &Arc { + self.memory_allocator.device() + } +} + +impl BufferState { + /// Returns the buffer. + #[inline] + #[must_use] + pub fn buffer(&self) -> &Arc { + &self.buffer + } + + /// Returns all last accesses that overlap the given `range` of the buffer. + /// + /// # Panics + /// + /// - Panics if `range` doesn't denote a valid range of the buffer. + #[inline] + pub fn accesses(&self, range: BufferRange) -> BufferAccesses<'_> { + assert!(range.end <= self.buffer.size()); + assert!(!range.is_empty()); + + BufferAccesses { + inner: MutexGuard::leak(self.last_accesses.lock()).overlapping(range), + // SAFETY: We locked the mutex above. + _guard: unsafe { AccessesGuard::new(&self.last_accesses) }, + } + } + + /// Sets the last access of the given `range` of the buffer. + /// + /// # Safety + /// + /// - `access` must constitute the correct access that was last performed on the `range` of the + /// buffer. + /// + /// # Panics + /// + /// - Panics if `range` is empty. + #[inline] + pub unsafe fn set_access(&self, range: BufferRange, access: BufferAccess) { + self.last_accesses.lock().insert(range, access); + } +} + +impl BufferAccess { + /// Creates a new `BufferAccess`. + #[inline] + pub const fn new(access_type: AccessType, queue_family_index: u32) -> Self { + BufferAccess { + access_type, + queue_family_index, + } + } + + /// Returns the stage mask of this access. + #[inline] + pub const fn stage_mask(&self) -> PipelineStages { + self.access_type.stage_mask() + } + + /// Returns the access mask of this access. + #[inline] + pub const fn access_mask(&self) -> AccessFlags { + self.access_type.access_mask() + } +} + +impl ImageState { + /// Returns the image. + #[inline] + #[must_use] + pub fn image(&self) -> &Arc { + &self.image + } + + /// Returns all last accesses that overlap the given `subresource_range` of the image. + /// + /// # Panics + /// + /// - Panics if `subresource_range` doesn't denote a valid subresource range of the image. + #[inline] + pub fn accesses(&self, subresource_range: ImageSubresourceRange) -> ImageAccesses<'_> { + let subresource_ranges = SubresourceRanges::from_image(&self.image, subresource_range); + let map = MutexGuard::leak(self.last_accesses.lock()); + + ImageAccesses { + mip_levels: self.image.mip_levels(), + array_layers: self.image.array_layers(), + subresource_ranges, + overlapping: map.overlapping(0..0), + map, + // SAFETY: We locked the mutex above. + _guard: unsafe { AccessesGuard::new(&self.last_accesses) }, + } + } + + /// Sets the last access of the given `subresource_range` of the image. + /// + /// # Safety + /// + /// - `access` must constitute the correct access that was last performed on the + /// `subresource_range` of the image. + /// + /// # Panics + /// + /// - Panics if `range` is empty. + #[inline] + pub unsafe fn set_access(&self, subresource_range: ImageSubresourceRange, access: ImageAccess) { + let mut last_accesses = self.last_accesses.lock(); + + for range in SubresourceRanges::from_image(&self.image, subresource_range) { + last_accesses.insert(range, access); + } + } +} + +impl ImageAccess { + /// Creates a new `ImageAccess`. + #[inline] + pub const fn new( + access_type: AccessType, + mut layout_type: ImageLayoutType, + queue_family_index: u32, + ) -> Self { + // Make sure that entries in the tree always compare equal if the effective access is the + // same, so that they can be combined for easier pipeline barrier batching. + if matches!(access_type.image_layout(), ImageLayout::General) { + layout_type = ImageLayoutType::General; + } + + ImageAccess { + access_type, + layout_type, + queue_family_index, + } + } + + /// Returns the stage mask of this access. + #[inline] + #[must_use] + pub const fn stage_mask(&self) -> PipelineStages { + self.access_type.stage_mask() + } + + /// Returns the access mask of this access. + #[inline] + #[must_use] + pub const fn access_mask(&self) -> AccessFlags { + self.access_type.access_mask() + } + + /// Returns the image layout of this access. + #[inline] + #[must_use] + pub const fn image_layout(&self) -> ImageLayout { + if self.layout_type.is_general() { + ImageLayout::General + } else { + self.access_type.image_layout() + } + } +} + +impl SwapchainState { + /// Returns the swapchain. + #[inline] + #[must_use] + pub fn swapchain(&self) -> &Arc { + &self.swapchain + } + + /// Returns the images comprising the swapchain. + #[inline] + #[must_use] + pub fn images(&self) -> &[Arc] { + &self.images + } +} + +impl Flight { + /// Returns the number of [frames] in this [flight]. + #[inline] + #[must_use] + pub fn frame_count(&self) -> u32 { + self.frame_count.get() + } + + /// Returns the index of the current [frame] in [flight]. + #[inline] + #[must_use] + pub fn current_frame(&self) -> u32 { + self.current_frame.load(Ordering::Relaxed) % self.frame_count + } + + /// Returns the fence for the current [frame] in [flight]. + #[inline] + #[must_use] + pub fn current_fence(&self) -> &Fence { + &self.fences[self.current_frame() as usize] + } + + pub(crate) unsafe fn next_frame(&self) { + self.current_frame.fetch_add(1, Ordering::Relaxed); + } +} + +/// Parameters to create a new [`Resources`] collection. +#[derive(Debug)] +pub struct ResourcesCreateInfo { + /// The maximum number of [`Buffer`]s that the collection can hold at once. + pub max_buffers: u32, + + /// The maximum number of [`Image`]s that the collection can hold at once. + pub max_images: u32, + + /// The maximum number of [`Swapchain`]s that the collection can hold at once. + pub max_swapchains: u32, + + /// The maximum number of [`Flight`]s that the collection can hold at once. + pub max_flights: u32, + + pub _ne: vulkano::NonExhaustive, +} + +impl Default for ResourcesCreateInfo { + #[inline] + fn default() -> Self { + ResourcesCreateInfo { + max_buffers: 1 << 24, + max_images: 1 << 24, + max_swapchains: 1 << 8, + max_flights: 1 << 8, + _ne: crate::NE, + } + } +} + +/// A subresource of a buffer that should be accessed. +pub type BufferRange = Range; + +/// An iterator over the last accesses of a buffer subresource. +/// +/// This type is created by the [`accesses`] method on [`BufferState`]. +/// +/// [`accesses`]: BufferState::accesses +pub struct BufferAccesses<'a> { + inner: rangemap::map::Overlapping<'a, DeviceSize, BufferAccess, Range>, + _guard: AccessesGuard<'a, BufferAccess>, +} + +impl<'a> Iterator for BufferAccesses<'a> { + type Item = (BufferRange, &'a BufferAccess); + + #[inline] + fn next(&mut self) -> Option { + self.inner + .next() + .map(|(range, access)| (range.clone(), access)) + } +} + +impl FusedIterator for BufferAccesses<'_> {} + +/// An iterator over the last accesses of an image subresource. +/// +/// This type is created by the [`accesses`] method on [`ImageState`]. +/// +/// [`accesses`]: ImageState::accesses +pub struct ImageAccesses<'a> { + mip_levels: u32, + array_layers: u32, + subresource_ranges: SubresourceRanges, + overlapping: rangemap::map::Overlapping<'a, DeviceSize, ImageAccess, Range>, + map: &'a RangeMap, + _guard: AccessesGuard<'a, ImageAccess>, +} + +impl<'a> Iterator for ImageAccesses<'a> { + type Item = (ImageSubresourceRange, &'a ImageAccess); + + #[inline] + fn next(&mut self) -> Option { + if let Some((range, access)) = self.overlapping.next() { + let subresource_range = + range_to_subresources(range.clone(), self.mip_levels, self.array_layers); + + Some((subresource_range, access)) + } else if let Some(range) = self.subresource_ranges.next() { + self.overlapping = self.map.overlapping(range); + + self.next() + } else { + None + } + } +} + +impl FusedIterator for ImageAccesses<'_> {} + +struct AccessesGuard<'a, V> { + mutex: &'a Mutex>, +} + +impl<'a, V> AccessesGuard<'a, V> { + unsafe fn new(mutex: &'a Mutex>) -> Self { + AccessesGuard { mutex } + } +} + +impl Drop for AccessesGuard<'_, V> { + fn drop(&mut self) { + // SAFETY: Enforced by the caller of `AccessesGuard::new`. + unsafe { self.mutex.force_unlock() } + } +} + +const _: () = assert!(mem::size_of::() == mem::size_of::()); + +#[derive(Clone)] +struct SubresourceRanges { + aspects: u32, + mip_levels: Range, + array_layers: Range, + aspect_size: DeviceSize, + mip_level_size: DeviceSize, + aspect_offset: DeviceSize, + mip_level_offset: DeviceSize, + granularity: SubresourceRangeGranularity, +} + +#[derive(Clone, Copy)] +enum SubresourceRangeGranularity { + Aspect, + MipLevel, + ArrayLayer, +} + +impl SubresourceRanges { + fn from_image(image: &Image, mut subresource_range: ImageSubresourceRange) -> Self { + assert!(image.format().aspects().contains(subresource_range.aspects)); + + if image.flags().intersects(ImageCreateFlags::DISJOINT) + && subresource_range.aspects.intersects(ImageAspects::COLOR) + { + subresource_range.aspects -= ImageAspects::COLOR; + subresource_range.aspects |= match image.format().planes().len() { + 2 => ImageAspects::PLANE_0 | ImageAspects::PLANE_1, + 3 => ImageAspects::PLANE_0 | ImageAspects::PLANE_1 | ImageAspects::PLANE_2, + _ => unreachable!(), + }; + } + + SubresourceRanges::new(subresource_range, image.mip_levels(), image.array_layers()) + } + + fn new( + subresource_range: ImageSubresourceRange, + image_mip_levels: u32, + image_array_layers: u32, + ) -> Self { + assert!(subresource_range.mip_levels.end <= image_mip_levels); + assert!(subresource_range.array_layers.end <= image_array_layers); + assert!(!subresource_range.mip_levels.is_empty()); + assert!(!subresource_range.array_layers.is_empty()); + + let mip_level_size = DeviceSize::from(image_array_layers); + let mip_levels = DeviceSize::from(subresource_range.mip_levels.start) * mip_level_size + ..DeviceSize::from(subresource_range.mip_levels.end) * mip_level_size; + let aspect_size = mip_level_size * DeviceSize::from(image_mip_levels); + let aspect_offset = 0; + let mip_level_offset = mip_levels.end - mip_level_size; + + let granularity = if subresource_range.array_layers != (0..image_array_layers) { + SubresourceRangeGranularity::ArrayLayer + } else if subresource_range.mip_levels != (0..image_mip_levels) { + SubresourceRangeGranularity::MipLevel + } else { + SubresourceRangeGranularity::Aspect + }; + + SubresourceRanges { + aspects: vk::ImageAspectFlags::from(subresource_range.aspects).as_raw(), + mip_levels, + array_layers: subresource_range.array_layers, + aspect_size, + mip_level_size, + aspect_offset, + mip_level_offset, + granularity, + } + } + + fn skip_unset_aspects(&mut self) { + let aspect_count = self.aspects.trailing_zeros(); + self.aspects >>= aspect_count; + self.aspect_offset += DeviceSize::from(aspect_count) * self.aspect_size; + } + + fn next_aspect(&mut self) { + self.aspects >>= 1; + self.aspect_offset += self.aspect_size; + } +} + +impl Iterator for SubresourceRanges { + type Item = Range; + + fn next(&mut self) -> Option { + if self.aspects != 0 { + match self.granularity { + SubresourceRangeGranularity::Aspect => { + self.skip_unset_aspects(); + + let aspect_count = DeviceSize::from(self.aspects.trailing_ones()); + let start = self.aspect_offset; + let end = self.aspect_offset + aspect_count * self.aspect_size; + + self.aspects >>= aspect_count; + self.aspect_offset += aspect_count * self.aspect_size; + + Some(Range { start, end }) + } + SubresourceRangeGranularity::MipLevel => { + self.skip_unset_aspects(); + + let start = self.aspect_offset + self.mip_levels.start; + let end = self.aspect_offset + self.mip_levels.end; + + self.next_aspect(); + + Some(Range { start, end }) + } + SubresourceRangeGranularity::ArrayLayer => { + self.mip_level_offset += self.mip_level_size; + + if self.mip_level_offset == self.mip_levels.end { + self.mip_level_offset = self.mip_levels.start; + self.skip_unset_aspects(); + } + + let offset = self.aspect_offset + self.mip_level_offset; + let start = offset + DeviceSize::from(self.array_layers.start); + let end = offset + DeviceSize::from(self.array_layers.end); + + if self.mip_level_offset == self.mip_levels.end - self.mip_level_size { + self.next_aspect(); + } + + Some(Range { start, end }) + } + } + } else { + None + } + } +} + +fn range_to_subresources( + mut range: Range, + image_mip_levels: u32, + image_array_layers: u32, +) -> ImageSubresourceRange { + debug_assert!(!range.is_empty()); + + let aspect_size = DeviceSize::from(image_mip_levels) * DeviceSize::from(image_array_layers); + let mip_level_size = DeviceSize::from(image_array_layers); + + if range.end - range.start > aspect_size { + debug_assert!(range.start % aspect_size == 0); + debug_assert!(range.end % aspect_size == 0); + + let aspect_start = (range.start / aspect_size) as u32; + let aspect_end = (range.end / aspect_size) as u32; + let aspects = u32::MAX >> (u32::BITS - (aspect_end - aspect_start)) << aspect_start; + + ImageSubresourceRange { + aspects: vk::ImageAspectFlags::from_raw(aspects).into(), + mip_levels: 0..image_mip_levels, + array_layers: 0..image_array_layers, + } + } else { + let aspect_index = (range.start / aspect_size) as u32; + range.start %= aspect_size; + range.end %= aspect_size; + + // Wraparound + if range.end == 0 { + range.end = aspect_size; + } + + if range.end - range.start > mip_level_size { + debug_assert!(range.start % mip_level_size == 0); + debug_assert!(range.end % mip_level_size == 0); + + let mip_level_start = (range.start / mip_level_size) as u32; + let mip_level_end = (range.end / mip_level_size) as u32; + + ImageSubresourceRange { + aspects: vk::ImageAspectFlags::from_raw(1 << aspect_index).into(), + mip_levels: mip_level_start..mip_level_end, + array_layers: 0..image_array_layers, + } + } else { + let mip_level = (range.start / mip_level_size) as u32; + range.start %= mip_level_size; + range.end %= mip_level_size; + + // Wraparound + if range.end == 0 { + range.end = mip_level_size; + } + + let array_layer_start = range.start as u32; + let array_layer_end = range.end as u32; + + ImageSubresourceRange { + aspects: vk::ImageAspectFlags::from_raw(1 << aspect_index).into(), + mip_levels: mip_level..mip_level + 1, + array_layers: array_layer_start..array_layer_end, + } + } + } +} + +macro_rules! access_types { + ( + $( + $(#[$meta:meta])* + $name:ident { + stage_mask: $($stage_flag:ident)|+, + access_mask: $($access_flag:ident)|+, + image_layout: $image_layout:ident, + } + )* + ) => { + /// Specifies which type of access is performed on a subresource. + #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] + #[non_exhaustive] + pub enum AccessType { + $( + $(#[$meta])* + $name, + )* + } + + impl AccessType { + /// Returns the stage mask of this type of access. + #[inline] + #[must_use] + pub const fn stage_mask(self) -> PipelineStages { + match self { + $( + Self::$name => PipelineStages::empty() + $(.union(PipelineStages::$stage_flag))+, + )* + } + } + + /// Returns the access mask of this type of access. + #[inline] + #[must_use] + pub const fn access_mask(self) -> AccessFlags { + match self { + $( + Self::$name => AccessFlags::empty() + $(.union(AccessFlags::$access_flag))+, + )* + } + } + + /// Returns the optimal image layout for this type of access, if any. + #[inline] + #[must_use] + pub const fn image_layout(self) -> ImageLayout { + match self { + $( + Self::$name => ImageLayout::$image_layout, + )* + } + } + } + }; +} + +access_types! { + IndirectCommandRead { + stage_mask: DRAW_INDIRECT, + access_mask: INDIRECT_COMMAND_READ, + image_layout: Undefined, + } + + IndexRead { + stage_mask: INDEX_INPUT, + access_mask: INDEX_READ, + image_layout: Undefined, + } + + VertexAttributeRead { + stage_mask: VERTEX_ATTRIBUTE_INPUT, + access_mask: VERTEX_ATTRIBUTE_READ, + image_layout: Undefined, + } + + VertexShaderUniformRead { + stage_mask: VERTEX_SHADER, + access_mask: UNIFORM_READ, + image_layout: Undefined, + } + + VertexShaderSampledRead { + stage_mask: VERTEX_SHADER, + access_mask: SHADER_SAMPLED_READ, + image_layout: ShaderReadOnlyOptimal, + } + + VertexShaderStorageRead { + stage_mask: VERTEX_SHADER, + access_mask: SHADER_STORAGE_READ, + image_layout: General, + } + + VertexShaderStorageWrite { + stage_mask: VERTEX_SHADER, + access_mask: SHADER_STORAGE_WRITE, + image_layout: General, + } + + VertexShaderAccelerationStructureRead { + stage_mask: VERTEX_SHADER, + access_mask: ACCELERATION_STRUCTURE_READ, + image_layout: Undefined, + } + + TessellationControlShaderUniformRead { + stage_mask: TESSELLATION_CONTROL_SHADER, + access_mask: UNIFORM_READ, + image_layout: Undefined, + } + + TessellationControlShaderSampledRead { + stage_mask: TESSELLATION_CONTROL_SHADER, + access_mask: SHADER_SAMPLED_READ, + image_layout: ShaderReadOnlyOptimal, + } + + TessellationControlShaderStorageRead { + stage_mask: TESSELLATION_CONTROL_SHADER, + access_mask: SHADER_STORAGE_READ, + image_layout: General, + } + + TessellationControlShaderStorageWrite { + stage_mask: TESSELLATION_CONTROL_SHADER, + access_mask: SHADER_STORAGE_WRITE, + image_layout: General, + } + + TessellationControlShaderAccelerationStructureRead { + stage_mask: TESSELLATION_CONTROL_SHADER, + access_mask: ACCELERATION_STRUCTURE_READ, + image_layout: Undefined, + } + + TessellationEvaluationShaderUniformRead { + stage_mask: TESSELLATION_EVALUATION_SHADER, + access_mask: UNIFORM_READ, + image_layout: Undefined, + } + + TessellationEvaluationShaderSampledRead { + stage_mask: TESSELLATION_EVALUATION_SHADER, + access_mask: SHADER_SAMPLED_READ, + image_layout: ShaderReadOnlyOptimal, + } + + TessellationEvaluationShaderStorageRead { + stage_mask: TESSELLATION_EVALUATION_SHADER, + access_mask: SHADER_STORAGE_READ, + image_layout: General, + } + + TessellationEvaluationShaderStorageWrite { + stage_mask: TESSELLATION_EVALUATION_SHADER, + access_mask: SHADER_STORAGE_WRITE, + image_layout: General, + } + + TessellationEvaluationShaderAccelerationStructureRead { + stage_mask: TESSELLATION_EVALUATION_SHADER, + access_mask: ACCELERATION_STRUCTURE_READ, + image_layout: Undefined, + } + + GeometryShaderUniformRead { + stage_mask: GEOMETRY_SHADER, + access_mask: UNIFORM_READ, + image_layout: Undefined, + } + + GeometryShaderSampledRead { + stage_mask: GEOMETRY_SHADER, + access_mask: SHADER_SAMPLED_READ, + image_layout: ShaderReadOnlyOptimal, + } + + GeometryShaderStorageRead { + stage_mask: GEOMETRY_SHADER, + access_mask: SHADER_STORAGE_READ, + image_layout: General, + } + + GeometryShaderStorageWrite { + stage_mask: GEOMETRY_SHADER, + access_mask: SHADER_STORAGE_WRITE, + image_layout: General, + } + + GeometryShaderAccelerationStructureRead { + stage_mask: GEOMETRY_SHADER, + access_mask: ACCELERATION_STRUCTURE_READ, + image_layout: Undefined, + } + + FragmentShaderUniformRead { + stage_mask: FRAGMENT_SHADER, + access_mask: UNIFORM_READ, + image_layout: Undefined, + } + + FragmentShaderColorInputAttachmentRead { + stage_mask: FRAGMENT_SHADER, + access_mask: INPUT_ATTACHMENT_READ, + image_layout: ShaderReadOnlyOptimal, + } + + FragmentShaderDepthStencilInputAttachmentRead { + stage_mask: FRAGMENT_SHADER, + access_mask: INPUT_ATTACHMENT_READ, + image_layout: DepthStencilReadOnlyOptimal, + } + + FragmentShaderSampledRead { + stage_mask: FRAGMENT_SHADER, + access_mask: SHADER_SAMPLED_READ, + image_layout: ShaderReadOnlyOptimal, + } + + FragmentShaderStorageRead { + stage_mask: FRAGMENT_SHADER, + access_mask: SHADER_STORAGE_READ, + image_layout: General, + } + + FragmentShaderStorageWrite { + stage_mask: FRAGMENT_SHADER, + access_mask: SHADER_STORAGE_WRITE, + image_layout: General, + } + + FragmentShaderAccelerationStructureRead { + stage_mask: FRAGMENT_SHADER, + access_mask: ACCELERATION_STRUCTURE_READ, + image_layout: Undefined, + } + + DepthStencilAttachmentRead { + stage_mask: EARLY_FRAGMENT_TESTS | LATE_FRAGMENT_TESTS, + access_mask: DEPTH_STENCIL_ATTACHMENT_READ, + image_layout: DepthStencilReadOnlyOptimal, + } + + DepthStencilAttachmentWrite { + stage_mask: EARLY_FRAGMENT_TESTS | LATE_FRAGMENT_TESTS, + access_mask: DEPTH_STENCIL_ATTACHMENT_WRITE, + image_layout: DepthStencilAttachmentOptimal, + } + + DepthAttachmentWriteStencilReadOnly { + stage_mask: EARLY_FRAGMENT_TESTS | LATE_FRAGMENT_TESTS, + access_mask: DEPTH_STENCIL_ATTACHMENT_READ | DEPTH_STENCIL_ATTACHMENT_WRITE, + image_layout: DepthAttachmentStencilReadOnlyOptimal, + } + + DepthReadOnlyStencilAttachmentWrite { + stage_mask: EARLY_FRAGMENT_TESTS | LATE_FRAGMENT_TESTS, + access_mask: DEPTH_STENCIL_ATTACHMENT_READ | DEPTH_STENCIL_ATTACHMENT_WRITE, + image_layout: DepthReadOnlyStencilAttachmentOptimal, + } + + ColorAttachmentRead { + stage_mask: COLOR_ATTACHMENT_OUTPUT, + access_mask: COLOR_ATTACHMENT_READ, + image_layout: ColorAttachmentOptimal, + } + + ColorAttachmentWrite { + stage_mask: COLOR_ATTACHMENT_OUTPUT, + access_mask: COLOR_ATTACHMENT_WRITE, + image_layout: ColorAttachmentOptimal, + } + + ColorAttachmentReadWrite { + stage_mask: COLOR_ATTACHMENT_OUTPUT, + access_mask: COLOR_ATTACHMENT_READ | COLOR_ATTACHMENT_WRITE, + image_layout: ColorAttachmentOptimal, + } + + ComputeShaderUniformRead { + stage_mask: COMPUTE_SHADER, + access_mask: UNIFORM_READ, + image_layout: Undefined, + } + + ComputeShaderSampledRead { + stage_mask: COMPUTE_SHADER, + access_mask: SHADER_SAMPLED_READ, + image_layout: ShaderReadOnlyOptimal, + } + + ComputeShaderStorageRead { + stage_mask: COMPUTE_SHADER, + access_mask: SHADER_STORAGE_READ, + image_layout: General, + } + + ComputeShaderStorageWrite { + stage_mask: COMPUTE_SHADER, + access_mask: SHADER_STORAGE_WRITE, + image_layout: General, + } + + ComputeShaderAccelerationStructureRead { + stage_mask: COMPUTE_SHADER, + access_mask: ACCELERATION_STRUCTURE_READ, + image_layout: Undefined, + } + + HostRead { + stage_mask: HOST, + access_mask: HOST_READ, + image_layout: General, + } + + HostWrite { + stage_mask: HOST, + access_mask: HOST_WRITE, + image_layout: General, + } + + CopyTransferRead { + stage_mask: COPY, + access_mask: TRANSFER_READ, + image_layout: TransferSrcOptimal, + } + + CopyTransferWrite { + stage_mask: COPY, + access_mask: TRANSFER_WRITE, + image_layout: TransferDstOptimal, + } + + BlitTransferRead { + stage_mask: BLIT, + access_mask: TRANSFER_READ, + image_layout: TransferSrcOptimal, + } + + BlitTransferWrite { + stage_mask: BLIT, + access_mask: TRANSFER_WRITE, + image_layout: TransferDstOptimal, + } + + ResolveTransferRead { + stage_mask: RESOLVE, + access_mask: TRANSFER_READ, + image_layout: TransferSrcOptimal, + } + + ResolveTransferWrite { + stage_mask: RESOLVE, + access_mask: TRANSFER_WRITE, + image_layout: TransferDstOptimal, + } + + ClearTransferWrite { + stage_mask: CLEAR, + access_mask: TRANSFER_WRITE, + image_layout: TransferDstOptimal, + } + + AccelerationStructureCopyTransferRead { + stage_mask: ACCELERATION_STRUCTURE_COPY, + access_mask: TRANSFER_READ, + image_layout: Undefined, + } + + AccelerationStructureCopyTrasferWrite { + stage_mask: ACCELERATION_STRUCTURE_COPY, + access_mask: TRANSFER_WRITE, + image_layout: Undefined, + } + + // TODO: + // VideoDecodeRead { + // stage_mask: VIDEO_DECODE, + // access_mask: VIDEO_DECODE_READ, + // image_layout: Undefined, + // } + + // TODO: + // VideoDecodeWrite { + // stage_mask: VIDEO_DECODE, + // access_mask: VIDEO_DECODE_WRITE, + // image_layout: VideoDecodeDst, + // } + + // TODO: + // VideoDecodeDpbRead { + // stage_mask: VIDEO_DECODE, + // access_mask: VIDEO_DECODE_READ, + // image_layout: VideoDecodeDpb, + // } + + // TODO: + // VideoDecodeDpbWrite { + // stage_mask: VIDEO_DECODE, + // access_mask: VIDEO_DECODE_WRITE, + // image_layout: VideoDecodeDpb, + // } + + // TODO: + // VideoEncodeRead { + // stage_mask: VIDEO_ENCODE, + // access_mask: VIDEO_ENCODE_READ, + // image_layout: VideoEncodeSrc, + // } + + // TODO: + // VideoEncodeWrite { + // stage_mask: VIDEO_ENCODE, + // access_mask: VIDEO_ENCODE_WRITE, + // image_layout: Undefined, + // } + + // TODO: + // VideoEncodeDpbRead { + // stage_mask: VIDEO_ENCODE, + // access_mask: VIDEO_ENCODE_READ, + // image_layout: VideoEncodeDpb, + // } + + // TODO: + // VideoEncodeDpbWrite { + // stage_mask: VIDEO_ENCODE, + // access_mask: VIDEO_ENCODE_WRITE, + // image_layout: VideoEncodeDpb, + // } + + // TODO: + // RayTracingShaderUniformRead { + // stage_mask: RAY_TRACING_SHADER, + // access_mask: UNIFORM_READ, + // image_layout: Undefined, + // } + + // TODO: + // RayTracingShaderColorInputAttachmentRead { + // stage_mask: RAY_TRACING_SHADER, + // access_mask: INPUT_ATTACHMENT_READ, + // image_layout: ShaderReadOnlyOptimal, + // } + + // TODO: + // RayTracingShaderDepthStencilInputAttachmentRead { + // stage_mask: RAY_TRACING_SHADER, + // access_mask: INPUT_ATTACHMENT_READ, + // image_layout: DepthStencilReadOnlyOptimal, + // } + + // TODO: + // RayTracingShaderSampledRead { + // stage_mask: RAY_TRACING_SHADER, + // access_mask: SHADER_SAMPLED_READ, + // image_layout: ShaderReadOnlyOptimal, + // } + + // TODO: + // RayTracingShaderStorageRead { + // stage_mask: RAY_TRACING_SHADER, + // access_mask: SHADER_STORAGE_READ, + // image_layout: General, + // } + + // TODO: + // RayTracingShaderStorageWrite { + // stage_mask: RAY_TRACING_SHADER, + // access_mask: SHADER_STORAGE_WRITE, + // image_layout: General, + // } + + // TODO: + // RayTracingShaderBindingTableRead { + // stage_mask: RAY_TRACING_SHADER, + // access_mask: SHADER_BINDING_TABLE_READ, + // image_layout: Undefined, + // } + + // TODO: + // RayTracingShaderAccelerationStructureRead { + // stage_mask: RAY_TRACING_SHADER, + // access_mask: ACCELERATION_STRUCTURE_READ, + // image_layout: Undefined, + // } + + TaskShaderUniformRead { + stage_mask: TASK_SHADER, + access_mask: UNIFORM_READ, + image_layout: Undefined, + } + + TaskShaderSampledRead { + stage_mask: TASK_SHADER, + access_mask: SHADER_SAMPLED_READ, + image_layout: ShaderReadOnlyOptimal, + } + + TaskShaderStorageRead { + stage_mask: TASK_SHADER, + access_mask: SHADER_STORAGE_READ, + image_layout: General, + } + + TaskShaderStorageWrite { + stage_mask: TASK_SHADER, + access_mask: SHADER_STORAGE_WRITE, + image_layout: General, + } + + TaskShaderAccelerationStructureRead { + stage_mask: TASK_SHADER, + access_mask: ACCELERATION_STRUCTURE_READ, + image_layout: Undefined, + } + + MeshShaderUniformRead { + stage_mask: MESH_SHADER, + access_mask: UNIFORM_READ, + image_layout: Undefined, + } + + MeshShaderSampledRead { + stage_mask: MESH_SHADER, + access_mask: SHADER_SAMPLED_READ, + image_layout: ShaderReadOnlyOptimal, + } + + MeshShaderStorageRead { + stage_mask: MESH_SHADER, + access_mask: SHADER_STORAGE_READ, + image_layout: General, + } + + MeshShaderStorageWrite { + stage_mask: MESH_SHADER, + access_mask: SHADER_STORAGE_WRITE, + image_layout: General, + } + + MeshShaderAccelerationStructureRead { + stage_mask: MESH_SHADER, + access_mask: ACCELERATION_STRUCTURE_READ, + image_layout: Undefined, + } + + AccelerationStructureBuildShaderRead { + stage_mask: ACCELERATION_STRUCTURE_BUILD, + access_mask: SHADER_READ, + image_layout: Undefined, + } + + AccelerationStructureBuildAccelerationStructureRead { + stage_mask: ACCELERATION_STRUCTURE_BUILD, + access_mask: ACCELERATION_STRUCTURE_READ, + image_layout: Undefined, + } + + AccelerationStructureBuildAccelerationStructureWrite { + stage_mask: ACCELERATION_STRUCTURE_BUILD, + access_mask: ACCELERATION_STRUCTURE_WRITE, + image_layout: Undefined, + } + + AccelerationStructureCopyAccelerationStructureRead { + stage_mask: ACCELERATION_STRUCTURE_COPY, + access_mask: ACCELERATION_STRUCTURE_READ, + image_layout: Undefined, + } + + AccelerationStructureCopyAccelerationStructureWrite { + stage_mask: ACCELERATION_STRUCTURE_COPY, + access_mask: ACCELERATION_STRUCTURE_WRITE, + image_layout: Undefined, + } + + /// Only use this for prototyping or debugging please. 🔫 Please. 🔫 + General { + stage_mask: ALL_COMMANDS, + access_mask: MEMORY_READ | MEMORY_WRITE, + image_layout: General, + } +} + +/// Specifies which type of layout an image subresource is accessed in. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum ImageLayoutType { + /// The image is accessed in an optimal layout. This is what you should be using most of the + /// time. + /// + /// The optimal layout depends on the access. For instance, for color attachment output, the + /// only valid optimal layout is [`ColorAttachmentOptimal`]. For transfer sources, it's + /// [`TransferSrcOptimal`], and so on. Some accesses don't have an optimal layout for them. In + /// such cases, using this option makes no difference, as the [general layout] will always be + /// used. + /// + /// [`ColorAttachmentOptimal`]: ImageLayout::ColorAttachmentOptimal + /// [`TransferSrcOptimal`]: ImageLayout::TransferSrcOptimal + /// [general layout]: ImageLayout::General + Optimal, + + /// The image is accessed in the [general layout]. This layout may be less efficient to access + /// on some hardware than an optimal layout. However, you may still want to use it in certain + /// cases if you want to minimize the number of layout transitions. + /// + /// [general layout]: ImageLayout::General + General, +} + +impl ImageLayoutType { + /// Returns `true` if the layout type is `Optimal`. + #[inline] + #[must_use] + pub const fn is_optimal(self) -> bool { + matches!(self, ImageLayoutType::Optimal) + } + + /// Returns `true` if the layout type is `General`. + #[inline] + #[must_use] + pub const fn is_general(self) -> bool { + matches!(self, ImageLayoutType::General) + } +} + +type Result = ::std::result::Result; + +#[allow(clippy::erasing_op, clippy::identity_op)] +#[cfg(test)] +mod tests { + use super::*; + use vulkano::image::ImageAspects; + + #[test] + fn subresource_ranges_aspect_granularity() { + let mut iter = SubresourceRanges::new( + ImageSubresourceRange { + aspects: ImageAspects::COLOR, + mip_levels: 0..4, + array_layers: 0..8, + }, + 4, + 8, + ); + + assert_eq!(iter.next(), Some(0 * 32..1 * 32)); + assert_eq!(iter.next(), None); + + let mut iter = SubresourceRanges::new( + ImageSubresourceRange { + aspects: ImageAspects::DEPTH | ImageAspects::STENCIL, + mip_levels: 0..2, + array_layers: 0..12, + }, + 2, + 12, + ); + + assert_eq!(iter.next(), Some(1 * 24..3 * 24)); + assert_eq!(iter.next(), None); + + let mut iter = SubresourceRanges::new( + ImageSubresourceRange { + aspects: ImageAspects::COLOR | ImageAspects::METADATA | ImageAspects::PLANE_0, + mip_levels: 0..5, + array_layers: 0..10, + }, + 5, + 10, + ); + + assert_eq!(iter.next(), Some(0 * 50..1 * 50)); + assert_eq!(iter.next(), Some(3 * 50..5 * 50)); + assert_eq!(iter.next(), None); + + let mut iter = SubresourceRanges::new( + ImageSubresourceRange { + aspects: ImageAspects::COLOR + | ImageAspects::DEPTH + | ImageAspects::STENCIL + | ImageAspects::PLANE_0 + | ImageAspects::PLANE_2 + | ImageAspects::MEMORY_PLANE_2 + | ImageAspects::MEMORY_PLANE_3, + mip_levels: 0..3, + array_layers: 0..20, + }, + 3, + 20, + ); + + assert_eq!(iter.next(), Some(0 * 60..3 * 60)); + assert_eq!(iter.next(), Some(4 * 60..5 * 60)); + assert_eq!(iter.next(), Some(6 * 60..7 * 60)); + assert_eq!(iter.next(), Some(9 * 60..11 * 60)); + assert_eq!(iter.next(), None); + } + + #[test] + fn subresource_ranges_mip_level_granularity() { + let mut iter = SubresourceRanges::new( + ImageSubresourceRange { + aspects: ImageAspects::DEPTH, + mip_levels: 1..3, + array_layers: 0..8, + }, + 5, + 8, + ); + + assert_eq!(iter.next(), Some(1 * 40 + 1 * 8..1 * 40 + 3 * 8)); + assert_eq!(iter.next(), None); + + let mut iter = SubresourceRanges::new( + ImageSubresourceRange { + aspects: ImageAspects::PLANE_0 | ImageAspects::PLANE_1 | ImageAspects::PLANE_2, + mip_levels: 1..3, + array_layers: 0..12, + }, + 3, + 12, + ); + + assert_eq!(iter.next(), Some(4 * 36 + 1 * 12..4 * 36 + 3 * 12)); + assert_eq!(iter.next(), Some(5 * 36 + 1 * 12..5 * 36 + 3 * 12)); + assert_eq!(iter.next(), Some(6 * 36 + 1 * 12..6 * 36 + 3 * 12)); + assert_eq!(iter.next(), None); + + let mut iter = SubresourceRanges::new( + ImageSubresourceRange { + aspects: ImageAspects::DEPTH + | ImageAspects::STENCIL + | ImageAspects::PLANE_0 + | ImageAspects::PLANE_1 + | ImageAspects::PLANE_2, + mip_levels: 1..3, + array_layers: 0..10, + }, + 4, + 10, + ); + + dbg!(iter.clone().collect::>()); + + assert_eq!(iter.next(), Some(1 * 40 + 1 * 10..1 * 40 + 3 * 10)); + assert_eq!(iter.next(), Some(2 * 40 + 1 * 10..2 * 40 + 3 * 10)); + assert_eq!(iter.next(), Some(4 * 40 + 1 * 10..4 * 40 + 3 * 10)); + assert_eq!(iter.next(), Some(5 * 40 + 1 * 10..5 * 40 + 3 * 10)); + assert_eq!(iter.next(), Some(6 * 40 + 1 * 10..6 * 40 + 3 * 10)); + assert_eq!(iter.next(), None); + + let mut iter = SubresourceRanges::new( + ImageSubresourceRange { + aspects: ImageAspects::METADATA + | ImageAspects::PLANE_2 + | ImageAspects::MEMORY_PLANE_1, + mip_levels: 2..4, + array_layers: 0..6, + }, + 4, + 6, + ); + + assert_eq!(iter.next(), Some(3 * 24 + 2 * 6..3 * 24 + 4 * 6)); + assert_eq!(iter.next(), Some(6 * 24 + 2 * 6..6 * 24 + 4 * 6)); + assert_eq!(iter.next(), Some(8 * 24 + 2 * 6..8 * 24 + 4 * 6)); + assert_eq!(iter.next(), None); + } + + #[test] + fn subresource_ranges_array_layer_granularity() { + let mut iter = SubresourceRanges::new( + ImageSubresourceRange { + aspects: ImageAspects::STENCIL, + mip_levels: 0..4, + array_layers: 2..9, + }, + 4, + 10, + ); + + assert_eq!(iter.next(), Some(2 * 40 + 0 * 10 + 2..2 * 40 + 0 * 10 + 9)); + assert_eq!(iter.next(), Some(2 * 40 + 1 * 10 + 2..2 * 40 + 1 * 10 + 9)); + assert_eq!(iter.next(), Some(2 * 40 + 2 * 10 + 2..2 * 40 + 2 * 10 + 9)); + assert_eq!(iter.next(), Some(2 * 40 + 3 * 10 + 2..2 * 40 + 3 * 10 + 9)); + assert_eq!(iter.next(), None); + + let mut iter = SubresourceRanges::new( + ImageSubresourceRange { + aspects: ImageAspects::COLOR | ImageAspects::METADATA, + mip_levels: 1..3, + array_layers: 3..8, + }, + 3, + 8, + ); + + assert_eq!(iter.next(), Some(0 * 24 + 1 * 8 + 3..0 * 24 + 1 * 8 + 8)); + assert_eq!(iter.next(), Some(0 * 24 + 2 * 8 + 3..0 * 24 + 2 * 8 + 8)); + assert_eq!(iter.next(), Some(3 * 24 + 1 * 8 + 3..3 * 24 + 1 * 8 + 8)); + assert_eq!(iter.next(), Some(3 * 24 + 2 * 8 + 3..3 * 24 + 2 * 8 + 8)); + assert_eq!(iter.next(), None); + + let mut iter = SubresourceRanges::new( + ImageSubresourceRange { + aspects: ImageAspects::DEPTH | ImageAspects::PLANE_0 | ImageAspects::PLANE_1, + mip_levels: 1..3, + array_layers: 2..4, + }, + 5, + 6, + ); + + assert_eq!(iter.next(), Some(1 * 30 + 1 * 6 + 2..1 * 30 + 1 * 6 + 4)); + assert_eq!(iter.next(), Some(1 * 30 + 2 * 6 + 2..1 * 30 + 2 * 6 + 4)); + assert_eq!(iter.next(), Some(4 * 30 + 1 * 6 + 2..4 * 30 + 1 * 6 + 4)); + assert_eq!(iter.next(), Some(4 * 30 + 2 * 6 + 2..4 * 30 + 2 * 6 + 4)); + assert_eq!(iter.next(), Some(5 * 30 + 1 * 6 + 2..5 * 30 + 1 * 6 + 4)); + assert_eq!(iter.next(), Some(5 * 30 + 2 * 6 + 2..5 * 30 + 2 * 6 + 4)); + assert_eq!(iter.next(), None); + + let mut iter = SubresourceRanges::new( + ImageSubresourceRange { + aspects: ImageAspects::PLANE_2 + | ImageAspects::MEMORY_PLANE_0 + | ImageAspects::MEMORY_PLANE_1 + | ImageAspects::MEMORY_PLANE_2, + mip_levels: 5..6, + array_layers: 0..3, + }, + 8, + 4, + ); + + assert_eq!(iter.next(), Some(6 * 32 + 5 * 4 + 0..6 * 32 + 5 * 4 + 3)); + assert_eq!(iter.next(), Some(7 * 32 + 5 * 4 + 0..7 * 32 + 5 * 4 + 3)); + assert_eq!(iter.next(), Some(8 * 32 + 5 * 4 + 0..8 * 32 + 5 * 4 + 3)); + assert_eq!(iter.next(), Some(9 * 32 + 5 * 4 + 0..9 * 32 + 5 * 4 + 3)); + assert_eq!(iter.next(), None); + } +} diff --git a/vulkano/src/memory/allocator/mod.rs b/vulkano/src/memory/allocator/mod.rs index 44d4462a90..cb32fbb2ae 100644 --- a/vulkano/src/memory/allocator/mod.rs +++ b/vulkano/src/memory/allocator/mod.rs @@ -1838,14 +1838,17 @@ impl Default for GenericMemoryAllocatorCreateInfo<'_> { } } -/// > **Note**: Returns `0` on overflow. +/// Returns the smallest value greater or equal to `val` that is a multiple of `alignment`. +/// +/// > **Note**: Returns zero on overflow. #[inline(always)] -pub(crate) const fn align_up(val: DeviceSize, alignment: DeviceAlignment) -> DeviceSize { +pub const fn align_up(val: DeviceSize, alignment: DeviceAlignment) -> DeviceSize { align_down(val.wrapping_add(alignment.as_devicesize() - 1), alignment) } +/// Returns the largest value smaller or equal to `val` that is a multiple of `alignment`. #[inline(always)] -pub(crate) const fn align_down(val: DeviceSize, alignment: DeviceAlignment) -> DeviceSize { +pub const fn align_down(val: DeviceSize, alignment: DeviceAlignment) -> DeviceSize { val & !(alignment.as_devicesize() - 1) } diff --git a/vulkano/src/memory/mod.rs b/vulkano/src/memory/mod.rs index 4c73cab0d0..444f579070 100644 --- a/vulkano/src/memory/mod.rs +++ b/vulkano/src/memory/mod.rs @@ -318,7 +318,10 @@ impl ResourceMemory { } } - pub(crate) fn atom_size(&self) -> Option { + // TODO: Expose (in a better way). + #[doc(hidden)] + #[inline] + pub fn atom_size(&self) -> Option { let memory = self.device_memory(); (!memory.is_coherent()).then_some(memory.atom_size())