Tracking Optimization and Rewrite (#2662)

* Texture state full is dead code

* temp

* temp2 - next is lifetime only

* temp3 - all trackers in place

* temp4 - continue muddling along

* temp5 - doodle doodle

* temp6 - continuous progress is continuous

* temp7 - pro-gress

* temp8 - Add refcounts to trackers

* temp9 - Soundness, generics, refcounts, and epochs

* temp10 - Compiling?!1?!1?!1

* temp11 - Add bind group state optimization

* temp12 - Safety and adding size setting

* temp13 - unsafe

* temp14 - Abstract all the state transitions

* temp15 - It verks! kinda...

* temp16 - it more verks!

* temp17 - debugging and unit tests

* temp18 - fixing unfixed things

* temp19 - hmmm

* temp20 - curious about line deltas

* temp21 - working texture trackers

* temp22 - write merge/update/barrier for buffers

* temp23 - cleanup and buffers

* temp24 - clippy cleanup

* temp25 - Add inline markers on buffer functions

* temp26 - Fix buffer trackers

* temp27 - fixed texture insert to handle both sides

* temp28 - document tracker and usages

* temp29 - document track/mod.rs

* temp30 - convert STORAGE_WRITE to STORAGE_READ_WRITE

* temp31 - Add some debug asserts to make sure we can't insert invalid states

* temp32 - clippy is opinionated sometimes

* temp33 - renaming and documentation

* temp34 - logging

wgpu-core/Cargo.toml

@@ -26,6 +26,7 @@ vulkan-portability = ["hal/vulkan"]
 [dependencies]
 arrayvec = "0.7"
 bitflags = "1.0"
+bit-vec = "0.6"
 codespan-reporting = "0.11"
 copyless = "0.1"
 fxhash = "0.2"

wgpu-core/src/binding_model.rs

@@ -1,10 +1,10 @@
 use crate::{
  device::{DeviceError, MissingDownlevelFlags, MissingFeatures, SHADER_STAGE_COUNT},
  error::{ErrorFormatter, PrettyError},
- hub::Resource,
- id::{BindGroupLayoutId, BufferId, DeviceId, SamplerId, TextureViewId, Valid},
+ hub::{HalApi, Resource},
+ id::{BindGroupLayoutId, BufferId, DeviceId, SamplerId, TextureId, TextureViewId, Valid},
  init_tracker::{BufferInitTrackerAction, TextureInitTrackerAction},
- track::{TrackerSet, UsageConflict, DUMMY_SELECTOR},
+ track::{BindGroupStates, UsageConflict},
  validation::{MissingBufferUsageError, MissingTextureUsageError},
  FastHashMap, Label, LifeGuard, MultiRefCount, Stored,
 };
@@ -16,10 +16,7 @@ use serde::Deserialize;
 #[cfg(feature = "trace")]
 use serde::Serialize;
 
-use std::{
- borrow::{Borrow, Cow},
- ops::Range,
-};
+use std::{borrow::Cow, ops::Range};
 
 use thiserror::Error;
 
@@ -63,6 +60,8 @@ pub enum CreateBindGroupError {
  InvalidBuffer(BufferId),
  #[error("texture view {0:?} is invalid")]
  InvalidTextureView(TextureViewId),
+ #[error("texture {0:?} is invalid")]
+ InvalidTexture(TextureId),
  #[error("sampler {0:?} is invalid")]
  InvalidSampler(SamplerId),
  #[error(
@@ -709,13 +708,12 @@ pub(crate) fn buffer_binding_type_alignment(
  }
 }
 
-#[derive(Debug)]
-pub struct BindGroup<A: hal::Api> {
+pub struct BindGroup<A: HalApi> {
  pub(crate) raw: A::BindGroup,
  pub(crate) device_id: Stored<DeviceId>,
  pub(crate) layout_id: Valid<BindGroupLayoutId>,
  pub(crate) life_guard: LifeGuard,
- pub(crate) used: TrackerSet,
+ pub(crate) used: BindGroupStates<A>,
  pub(crate) used_buffer_ranges: Vec<BufferInitTrackerAction>,
  pub(crate) used_texture_ranges: Vec<TextureInitTrackerAction>,
  pub(crate) dynamic_binding_info: Vec<BindGroupDynamicBindingData>,
@@ -724,7 +722,7 @@ pub struct BindGroup<A: hal::Api> {
  pub(crate) late_buffer_binding_sizes: Vec<wgt::BufferSize>,
 }
 
-impl<A: hal::Api> BindGroup<A> {
+impl<A: HalApi> BindGroup<A> {
  pub(crate) fn validate_dynamic_bindings(
  &self,
  offsets: &[wgt::DynamicOffset],
@@ -766,13 +764,7 @@ impl<A: hal::Api> BindGroup<A> {
  }
 }
 
-impl<A: hal::Api> Borrow<()> for BindGroup<A> {
- fn borrow(&self) -> &() {
- &DUMMY_SELECTOR
- }
-}
-
-impl<A: hal::Api> Resource for BindGroup<A> {
+impl<A: HalApi> Resource for BindGroup<A> {
  const TYPE: &'static str = "BindGroup";
 
  fn life_guard(&self) -> &LifeGuard {

wgpu-core/src/command/bundle.rs

@@ -34,7 +34,7 @@ invalidations or index format changes.
 #![allow(clippy::reversed_empty_ranges)]
 
 use crate::{
- binding_model::buffer_binding_type_alignment,
+ binding_model::{self, buffer_binding_type_alignment},
  command::{
  BasePass, BindGroupStateChange, DrawError, MapPassErr, PassErrorScope, RenderCommand,
  RenderCommandError, StateChange,
@@ -48,8 +48,9 @@ use crate::{
  hub::{GlobalIdentityHandlerFactory, HalApi, Hub, Resource, Storage, Token},
  id,
  init_tracker::{BufferInitTrackerAction, MemoryInitKind, TextureInitTrackerAction},
- pipeline::PipelineFlags,
- track::{TrackerSet, UsageConflict},
+ pipeline::{self, PipelineFlags},
+ resource,
+ track::RenderBundleScope,
  validation::check_buffer_usage,
  Label, LabelHelpers, LifeGuard, Stored,
 };
@@ -117,7 +118,7 @@ impl RenderBundleEncoder {
  },
  sample_count: {
  let sc = desc.sample_count;
- if sc == 0 || sc > 32 || !conv::is_power_of_two(sc) {
+ if sc == 0 || sc > 32 || !conv::is_power_of_two_u32(sc) {
  return Err(CreateRenderBundleError::InvalidSampleCount(sc));
  }
  sc
@@ -177,20 +178,28 @@ impl RenderBundleEncoder {
  /// and accumulate buffer and texture initialization actions.
  ///
  /// [`ExecuteBundle`]: RenderCommand::ExecuteBundle
- pub(crate) fn finish<A: hal::Api, G: GlobalIdentityHandlerFactory>(
+ pub(crate) fn finish<A: HalApi, G: GlobalIdentityHandlerFactory>(
  self,
  desc: &RenderBundleDescriptor,
  device: &Device<A>,
  hub: &Hub<A, G>,
  token: &mut Token<Device<A>>,
- ) -> Result<RenderBundle, RenderBundleError> {
+ ) -> Result<RenderBundle<A>, RenderBundleError> {
  let (pipeline_layout_guard, mut token) = hub.pipeline_layouts.read(token);
  let (bind_group_guard, mut token) = hub.bind_groups.read(&mut token);
  let (pipeline_guard, mut token) = hub.render_pipelines.read(&mut token);
- let (buffer_guard, _) = hub.buffers.read(&mut token);
+ let (query_set_guard, mut token) = hub.query_sets.read(&mut token);
+ let (buffer_guard, mut token) = hub.buffers.read(&mut token);
+ let (texture_guard, _) = hub.textures.read(&mut token);
 
  let mut state = State {
- trackers: TrackerSet::new(self.parent_id.backend()),
+ trackers: RenderBundleScope::new(
+ &*buffer_guard,
+ &*texture_guard,
+ &*bind_group_guard,
+ &*pipeline_guard,
+ &*query_set_guard,
+ ),
  index: IndexState::new(),
  vertex: (0..hal::MAX_VERTEX_BUFFERS)
  .map(|_| VertexState::new())
@@ -234,11 +243,11 @@ impl RenderBundleEncoder {
  next_dynamic_offset = offsets_range.end;
  let offsets = &base.dynamic_offsets[offsets_range.clone()];
 
- let bind_group = state
+ let bind_group: &binding_model::BindGroup<A> = state
  .trackers
  .bind_groups
- .use_extend(&*bind_group_guard, bind_group_id, (), ())
- .map_err(|_| RenderCommandError::InvalidBindGroup(bind_group_id))
+ .add_single(&*bind_group_guard, bind_group_id)
+ .ok_or(RenderCommandError::InvalidBindGroup(bind_group_id))
  .map_pass_err(scope)?;
  if bind_group.dynamic_binding_info.len() != offsets.len() {
  return Err(RenderCommandError::InvalidDynamicOffsetCount {
@@ -268,10 +277,12 @@ impl RenderBundleEncoder {
  texture_memory_init_actions.extend_from_slice(&bind_group.used_texture_ranges);
 
  state.set_bind_group(index, bind_group_id, bind_group.layout_id, offsets_range);
- state
- .trackers
- .merge_extend_stateful(&bind_group.used)
- .map_pass_err(scope)?;
+ unsafe {
+ state
+ .trackers
+ .merge_bind_group(&*texture_guard, &bind_group.used)
+ .map_pass_err(scope)?
+ };
  //Note: stateless trackers are not merged: the lifetime reference
  // is held to the bind group itself.
  }
@@ -280,11 +291,11 @@ impl RenderBundleEncoder {
 
  state.pipeline = Some(pipeline_id);
 
- let pipeline = state
+ let pipeline: &pipeline::RenderPipeline<A> = state
  .trackers
- .render_pipes
- .use_extend(&*pipeline_guard, pipeline_id, (), ())
- .map_err(|_| RenderCommandError::InvalidPipeline(pipeline_id))
+ .render_pipelines
+ .add_single(&*pipeline_guard, pipeline_id)
+ .ok_or(RenderCommandError::InvalidPipeline(pipeline_id))
  .map_pass_err(scope)?;
 
  self.context
@@ -320,11 +331,11 @@ impl RenderBundleEncoder {
  size,
  } => {
  let scope = PassErrorScope::SetIndexBuffer(buffer_id);
- let buffer = state
+ let buffer: &resource::Buffer<A> = state
  .trackers
  .buffers
- .use_extend(&*buffer_guard, buffer_id, (), hal::BufferUses::INDEX)
- .unwrap();
+ .merge_single(&*buffer_guard, buffer_id, hal::BufferUses::INDEX)
+ .map_pass_err(scope)?;
  check_buffer_usage(buffer.usage, wgt::BufferUsages::INDEX)
  .map_pass_err(scope)?;
 
@@ -347,11 +358,11 @@ impl RenderBundleEncoder {
  size,
  } => {
  let scope = PassErrorScope::SetVertexBuffer(buffer_id);
- let buffer = state
+ let buffer: &resource::Buffer<A> = state
  .trackers
  .buffers
- .use_extend(&*buffer_guard, buffer_id, (), hal::BufferUses::VERTEX)
- .unwrap();
+ .merge_single(&*buffer_guard, buffer_id, hal::BufferUses::VERTEX)
+ .map_pass_err(scope)?;
  check_buffer_usage(buffer.usage, wgt::BufferUsages::VERTEX)
  .map_pass_err(scope)?;
 
@@ -472,11 +483,11 @@ impl RenderBundleEncoder {
  .require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION)
  .map_pass_err(scope)?;
 
- let buffer = state
+ let buffer: &resource::Buffer<A> = state
  .trackers
  .buffers
- .use_extend(&*buffer_guard, buffer_id, (), hal::BufferUses::INDIRECT)
- .unwrap();
+ .merge_single(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT)
+ .map_pass_err(scope)?;
  check_buffer_usage(buffer.usage, wgt::BufferUsages::INDIRECT)
  .map_pass_err(scope)?;
 
@@ -505,11 +516,10 @@ impl RenderBundleEncoder {
  .require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION)
  .map_pass_err(scope)?;
 
- let buffer = state
+ let buffer: &resource::Buffer<A> = state
  .trackers
  .buffers
- .use_extend(&*buffer_guard, buffer_id, (), hal::BufferUses::INDIRECT)
- .map_err(|err| RenderCommandError::Buffer(buffer_id, err))
+ .merge_single(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT)
  .map_pass_err(scope)?;
  check_buffer_usage(buffer.usage, wgt::BufferUsages::INDIRECT)
  .map_pass_err(scope)?;
@@ -612,24 +622,23 @@ pub type RenderBundleDescriptor<'a> = wgt::RenderBundleDescriptor<Label<'a>>;
 //Note: here, `RenderBundle` is just wrapping a raw stream of render commands.
 // The plan is to back it by an actual Vulkan secondary buffer, D3D12 Bundle,
 // or Metal indirect command buffer.
-#[derive(Debug)]
-pub struct RenderBundle {
+pub struct RenderBundle<A: HalApi> {
  // Normalized command stream. It can be executed verbatim,
  // without re-binding anything on the pipeline change.
  base: BasePass<RenderCommand>,
  pub(super) is_ds_read_only: bool,
  pub(crate) device_id: Stored<id::DeviceId>,
- pub(crate) used: TrackerSet,
+ pub(crate) used: RenderBundleScope<A>,
  pub(super) buffer_memory_init_actions: Vec<BufferInitTrackerAction>,
  pub(super) texture_memory_init_actions: Vec<TextureInitTrackerAction>,
  pub(super) context: RenderPassContext,
  pub(crate) life_guard: LifeGuard,
 }
 
-unsafe impl Send for RenderBundle {}
-unsafe impl Sync for RenderBundle {}
+unsafe impl<A: HalApi> Send for RenderBundle<A> {}
+unsafe impl<A: HalApi> Sync for RenderBundle<A> {}
 
-impl RenderBundle {
+impl<A: HalApi> RenderBundle<A> {
  /// Actually encode the contents into a native command buffer.
  ///
  /// This is partially duplicating the logic of `command_encoder_run_render_pass`.
@@ -639,7 +648,7 @@ impl RenderBundle {
  /// Note that the function isn't expected to fail, generally.
  /// All the validation has already been done by this point.
  /// The only failure condition is if some of the used buffers are destroyed.
- pub(super) unsafe fn execute<A: HalApi>(
+ pub(super) unsafe fn execute(
  &self,
  raw: &mut A::CommandEncoder,
  pipeline_layout_guard: &Storage<
@@ -828,7 +837,7 @@ impl RenderBundle {
  }
 }
 
-impl Resource for RenderBundle {
+impl<A: HalApi> Resource for RenderBundle<A> {
  const TYPE: &'static str = "RenderBundle";
 
  fn life_guard(&self) -> &LifeGuard {
@@ -1027,10 +1036,9 @@ struct VertexLimitState {
 /// [`SetIndexBuffer`] to the simulated state stored here, and then
 /// calls the `flush_foo` methods before draw calls to produce the
 /// update commands we actually need.
-#[derive(Debug)]
-struct State {
+struct State<A: HalApi> {
  /// Resources used by this bundle. This will become [`RenderBundle::used`].
- trackers: TrackerSet,
+ trackers: RenderBundleScope<A>,
 
  /// The current index buffer. We flush this state before indexed
  /// draw commands.
@@ -1056,7 +1064,7 @@ struct State {
  pipeline: Option<id::RenderPipelineId>,
 }
 
-impl State {
+impl<A: HalApi> State<A> {
  fn vertex_limits(&self) -> VertexLimitState {
  let mut vert_state = VertexLimitState {
  vertex_limit: u32::MAX,
@@ -1235,8 +1243,6 @@ pub(super) enum RenderBundleErrorInner {
  #[error(transparent)]
  RenderCommand(RenderCommandError),
  #[error(transparent)]
- ResourceUsageConflict(#[from] UsageConflict),
- #[error(transparent)]
  Draw(#[from] DrawError),
  #[error(transparent)]
  MissingDownlevelFlags(#[from] MissingDownlevelFlags),