Support sparse binding in Vulkan for buffers and opaque images (#1237)

* WIP: Support sparse binding

* WIP: cache sparse binding info on the tracing side

* WIP: recreate buffer sparse binding and opaque image sparse binding

* WIP: update the state block in the trace side

* Log error when the memories for sparse bindings for opaque images or buffers do not exist during recreating state

core/cc/interval_list.h

@@ -118,6 +118,10 @@ class CustomIntervalList {
     // end() returns the pointer to one-past the last interval in the list.
     inline const T* end() const;
 
+    // operator[] returns the const reference to the element at the specified
+    // location pos.
+    inline const T& operator[](size_t pos) const;
+
 protected:
     // rangeFirst returns the index of the first interval + bias that touches or
     // exceeds start.
@@ -240,6 +244,11 @@ inline const T* CustomIntervalList<T>::end() const {
     }
 }
 
+template<typename T>
+inline const T& CustomIntervalList<T>::operator[](size_t pos) const {
+  return mIntervals[pos];
+}
+
 template<typename T>
 inline ssize_t CustomIntervalList<T>::rangeFirst(interval_unit_type start, interval_unit_type bias) const {
     ssize_t l = 0;

gapii/cc/vulkan_extras.inl

@@ -142,7 +142,9 @@ void SpyOverride_RecreateImage(
     VkMemoryRequirements* pMemoryRequirements, uint32_t sparseMemoryRequirementCount,
     VkSparseImageMemoryRequirements* pSparseMemoryRequirements) {}
 void SpyOverride_RecreateBindImageMemory(VkDevice, VkImage, VkDeviceMemory,
-                                         VkDeviceSize offset) {}
+                                         VkDeviceSize offset,
+                                         uint32_t bindCount,
+                                         VkSparseMemoryBind* binds) {}
 void SpyOverride_RecreateImageData(VkDevice, VkImage,
                                    uint32_t /*VkImageLayout*/,
                                    uint32_t hostMemoryIndex, VkQueue,
@@ -165,7 +167,9 @@ void SpyOverride_RecreateComputePipeline(VkDevice, VkPipelineCache,
                                          VkPipeline*) {}
 void SpyOverride_RecreateBuffer(VkDevice, VkBufferCreateInfo*, VkBuffer*) {}
 void SpyOverride_RecreateBindBufferMemory(VkDevice, VkBuffer, VkDeviceMemory,
-                                          VkDeviceSize offset) {}
+                                          VkDeviceSize offset,
+                                          uint32_t bindCount,
+                                          VkSparseMemoryBind* binds) {}
 void SpyOverride_RecreateBufferData(VkDevice, VkBuffer,
                                     uint32_t hostBufferMemoryIndex, VkQueue,
                                     void* data) {}
@@ -208,3 +212,52 @@ uint32_t SpyOverride_createImageAndCacheMemoryRequirements(
 void SpyOverride_cacheImageSparseMemoryRequirements(
     VkDevice device, VkImage image, uint32_t count,
     VkSparseImageMemoryRequirements* pSparseMemoryRequirements);
+
+
+class SparseBindingInterval {
+ public:
+  SparseBindingInterval(const VkSparseMemoryBind& bind)
+      : resourceOffset_(bind.mresourceOffset),
+        size_(bind.msize),
+        memory_(bind.mmemory),
+        memoryOffset_(bind.mmemoryOffset),
+        flags_(bind.mflags) {}
+  SparseBindingInterval(const SparseBindingInterval&) = default;
+  SparseBindingInterval(SparseBindingInterval&&) = default;
+  SparseBindingInterval& operator=(const SparseBindingInterval&) = default;
+  SparseBindingInterval& operator=(SparseBindingInterval&&) = default;
+
+  VkSparseMemoryBind sparseMemoryBind() const {
+    return VkSparseMemoryBind(resourceOffset_, size_, memory_, memoryOffset_,
+                              flags_);
+  }
+
+  using interval_unit_type = VkDeviceSize;
+  inline VkDeviceSize start() const { return resourceOffset_; }
+  inline VkDeviceSize end() const { return resourceOffset_ + size_; }
+  inline void adjust(VkDeviceSize start, VkDeviceSize end) {
+    VkDeviceSize new_size = end - start;
+    if (start > resourceOffset_) {
+      VkDeviceSize x = start - resourceOffset_;
+      resourceOffset_ += x;
+      memoryOffset_ += x;
+    } else {
+      VkDeviceSize x = resourceOffset_ - start;
+      resourceOffset_ -= x;
+      memoryOffset_ -= x;
+    }
+    size_ = new_size;
+  }
+
+ private:
+  VkDeviceSize resourceOffset_;
+  VkDeviceSize size_;
+  VkDeviceMemory memory_;
+  VkDeviceSize memoryOffset_;
+  VkSparseMemoryBindFlags flags_;
+};
+
+using SparseBindingList = core::CustomIntervalList<SparseBindingInterval>;
+
+std::unordered_map<VkBuffer, SparseBindingList> mBufferSparseBindings;
+std::unordered_map<VkImage, SparseBindingList> mOpaqueImageSparseBindings;

gapii/cc/vulkan_mid_execution.cpp

@@ -708,7 +708,12 @@ void VulkanSpy::EnumerateVulkanResources(CallObserver* observer) {
       VkBuffer copy_buffer;
       VkDeviceMemory copy_memory;
 
-      if (buffer.second->mMemory) {
+      bool denseBound = buffer.second->mMemory != nullptr;
+      bool sparseBound = (mBufferSparseBindings.find(buffer.first) !=
+                          mBufferSparseBindings.end()) &&
+                         (mBufferSparseBindings[buffer.first].count() > 0);
+
+      if (denseBound || sparseBound) {
         need_to_clean_up_temps = true;
         VkPhysicalDeviceMemoryProperties properties;
         mImports.mVkInstanceFunctions[instance]
@@ -836,11 +841,19 @@ void VulkanSpy::EnumerateVulkanResources(CallObserver* observer) {
         device_functions.vkDestroyCommandPool(device, pool, nullptr);
       }
 
+      uint32_t sparseBindCount = sparseBound ? mBufferSparseBindings[buffer.first].count() : 0;
+      std::vector<VkSparseMemoryBind> sparseBinds;
+      for (const auto& b : mBufferSparseBindings[buffer.first]) {
+        sparseBinds.emplace_back(b.sparseMemoryBind());
+      }
+
       RecreateBindBufferMemory(
           observer, buffer.second->mDevice, buffer.second->mVulkanHandle,
-          buffer.second->mMemory ? buffer.second->mMemory->mVulkanHandle
-                                 : VkDeviceMemory(0),
-          buffer.second->mMemoryOffset);
+          denseBound ? buffer.second->mMemory->mVulkanHandle
+                     : VkDeviceMemory(0),
+          buffer.second->mMemoryOffset,
+          sparseBound ? mBufferSparseBindings[buffer.first].count() : 0,
+          sparseBound ? sparseBinds.data() : nullptr);
 
       RecreateBufferData(observer, buffer.second->mDevice,
                          buffer.second->mVulkanHandle, host_buffer_memory_index,
@@ -920,7 +933,13 @@ void VulkanSpy::EnumerateVulkanResources(CallObserver* observer) {
 
       uint32_t imageLayout = info.mLayout;
 
-      if (image.second->mBoundMemory &&
+      bool denseBound = image.second->mBoundMemory != nullptr;
+      bool opaqueSparseBound =
+          (mOpaqueImageSparseBindings.find(image.first) !=
+           mOpaqueImageSparseBindings.end()) &&
+          (mOpaqueImageSparseBindings[image.first].count() > 0);
+
+      if ((denseBound || opaqueSparseBound) &&
           info.mSamples == VkSampleCountFlagBits::VK_SAMPLE_COUNT_1_BIT &&
           // Don't capture images with undefined layout. The resulting data
           // itself will be undefined.
@@ -1113,11 +1132,19 @@ void VulkanSpy::EnumerateVulkanResources(CallObserver* observer) {
         device_functions.vkDestroyCommandPool(device, pool, nullptr);
       }
 
+      uint32_t opaqueSparseBindCount = opaqueSparseBound ? mOpaqueImageSparseBindings[image.first].count() : 0;
+      std::vector<VkSparseMemoryBind> opaqueSparseBinds;
+      for (const auto& b : mOpaqueImageSparseBindings[image.first]) {
+        opaqueSparseBinds.emplace_back(b.sparseMemoryBind());
+      }
+
       RecreateBindImageMemory(
           observer, image.second->mDevice, image.second->mVulkanHandle,
-          image.second->mBoundMemory ? image.second->mBoundMemory->mVulkanHandle
-                                     : VkDeviceMemory(0),
-          image.second->mBoundMemoryOffset);
+          denseBound ? image.second->mBoundMemory->mVulkanHandle
+                     : VkDeviceMemory(0),
+          image.second->mBoundMemoryOffset,
+          opaqueSparseBound ? opaqueSparseBindCount : 0,
+          opaqueSparseBound ? opaqueSparseBinds.data() : nullptr);
 
       RecreateImageData(observer, image.second->mDevice,
                         image.second->mVulkanHandle, imageLayout,

gapis/api/vulkan/CMakeFiles.cmake

@@ -38,6 +38,8 @@ set(files
     resolvables.pb.go
     resolvables.proto
     resources.go
+    sparse_binding_test.go
+    sparse_bindings.go
     state.go
     vulkan.go
     vulkan_terminator.go

gapis/api/vulkan/custom_replay.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"strings"
 
+	"github.com/google/gapid/core/log"
 	"github.com/google/gapid/gapis/api"
 	"github.com/google/gapid/gapis/memory"
 	"github.com/google/gapid/gapis/replay/builder"
@@ -1173,6 +1174,53 @@ func (a *RecreateBindImageMemory) Mutate(ctx context.Context, id api.CmdID, s *a
 			return err
 		}
 	}
+	if a.OpaqueSparseBindCount > 0 {
+		cb := CommandBuilder{Thread: a.thread}
+		for _, bind := range a.POpaqueSparseBinds.Slice(0, uint64(a.OpaqueSparseBindCount), s.MemoryLayout).MustRead(ctx, a, s, nil) {
+			if !GetState(s).DeviceMemories.Contains(bind.Memory) {
+				// TODO: Move this message to report view
+				log.E(ctx, "Sparse memory binding for opaque image: %v, Memory: %v does not exist.", a.Image, bind.Memory)
+			}
+		}
+		opaqueMemBindInfo := VkSparseImageOpaqueMemoryBindInfo{
+			Image:     a.Image,
+			BindCount: a.OpaqueSparseBindCount,
+			PBinds:    a.POpaqueSparseBinds,
+		}
+		opaqueMemBindInfoData := s.AllocDataOrPanic(ctx, opaqueMemBindInfo)
+		defer opaqueMemBindInfoData.Free()
+		queueBindInfo := VkBindSparseInfo{
+			SType:                VkStructureType_VK_STRUCTURE_TYPE_BIND_SPARSE_INFO,
+			PNext:                NewVoidᶜᵖ(memory.Nullptr),
+			WaitSemaphoreCount:   0,
+			PWaitSemaphores:      NewVkSemaphoreᶜᵖ(memory.Nullptr),
+			BufferBindCount:      0,
+			PBufferBinds:         NewVkSparseBufferMemoryBindInfoᶜᵖ(memory.Nullptr),
+			ImageOpaqueBindCount: 1,
+			PImageOpaqueBinds:    NewVkSparseImageOpaqueMemoryBindInfoᶜᵖ(opaqueMemBindInfoData.Ptr()),
+			ImageBindCount:       0,
+			PImageBinds:          NewVkSparseImageMemoryBindInfoᶜᵖ(memory.Nullptr),
+			SignalSemaphoreCount: 0,
+			PSignalSemaphores:    NewVkSemaphoreᶜᵖ(memory.Nullptr),
+		}
+		queueBindInfoData := s.AllocDataOrPanic(ctx, queueBindInfo)
+		defer queueBindInfoData.Free()
+
+		queue := findSupportedQueueForDevice(a.Device, s, VkQueueFlags(VkQueueFlagBits_VK_QUEUE_SPARSE_BINDING_BIT))
+		err := cb.VkQueueBindSparse(
+			queue,
+			1,
+			queueBindInfoData.Ptr(),
+			VkFence(0),
+			VkResult_VK_SUCCESS,
+		).AddRead(
+			queueBindInfoData.Data(),
+		).AddRead(
+			opaqueMemBindInfoData.Data(),
+		).Mutate(ctx, id, s, b)
+
+		return err
+	}
 	return nil
 }
 
@@ -1343,6 +1391,53 @@ func (a *RecreateBindBufferMemory) Mutate(ctx context.Context, id api.CmdID, s *
 			return err
 		}
 	}
+	if a.SparseBindCount > 0 {
+		cb := CommandBuilder{Thread: a.thread}
+		for _, bind := range a.PSparseBinds.Slice(0, uint64(a.SparseBindCount), s.MemoryLayout).MustRead(ctx, a, s, nil) {
+			if !GetState(s).DeviceMemories.Contains(bind.Memory) {
+				// TODO: Move this message to report view
+				log.E(ctx, "Sparse memory binding for buffer: %v, Memory: %v does not exist.", a.Buffer, bind.Memory)
+			}
+		}
+		bufMemBindInfo := VkSparseBufferMemoryBindInfo{
+			Buffer:    a.Buffer,
+			BindCount: a.SparseBindCount,
+			PBinds:    a.PSparseBinds,
+		}
+		bufMemBindInfoData := s.AllocDataOrPanic(ctx, bufMemBindInfo)
+		defer bufMemBindInfoData.Free()
+		queueBindInfo := VkBindSparseInfo{
+			SType:                VkStructureType_VK_STRUCTURE_TYPE_BIND_SPARSE_INFO,
+			PNext:                NewVoidᶜᵖ(memory.Nullptr),
+			WaitSemaphoreCount:   0,
+			PWaitSemaphores:      NewVkSemaphoreᶜᵖ(memory.Nullptr),
+			BufferBindCount:      1,
+			PBufferBinds:         NewVkSparseBufferMemoryBindInfoᶜᵖ(bufMemBindInfoData.Ptr()),
+			ImageOpaqueBindCount: 0,
+			PImageOpaqueBinds:    NewVkSparseImageOpaqueMemoryBindInfoᶜᵖ(memory.Nullptr),
+			ImageBindCount:       0,
+			PImageBinds:          NewVkSparseImageMemoryBindInfoᶜᵖ(memory.Nullptr),
+			SignalSemaphoreCount: 0,
+			PSignalSemaphores:    NewVkSemaphoreᶜᵖ(memory.Nullptr),
+		}
+		queueBindInfoData := s.AllocDataOrPanic(ctx, queueBindInfo)
+		defer queueBindInfoData.Free()
+
+		queue := findSupportedQueueForDevice(a.Device, s, VkQueueFlags(VkQueueFlagBits_VK_QUEUE_SPARSE_BINDING_BIT))
+		err := cb.VkQueueBindSparse(
+			queue,
+			1,
+			queueBindInfoData.Ptr(),
+			VkFence(0),
+			VkResult_VK_SUCCESS,
+		).AddRead(
+			queueBindInfoData.Data(),
+		).AddRead(
+			bufMemBindInfoData.Data(),
+		).Mutate(ctx, id, s, b)
+
+		return err
+	}
 	return nil
 }
 
@@ -1453,6 +1548,21 @@ func findGraphicsAndComputeQueueForDevice(device VkDevice, s *api.GlobalState) V
 	return backupQueue
 }
 
+// Returns a queue capable of the sorts of operations specified in the flags.
+// If such a queue cannot be found, returns a VkQueue(0).
+func findSupportedQueueForDevice(device VkDevice, s *api.GlobalState, flags VkQueueFlags) VkQueue {
+	c := GetState(s)
+	for _, v := range c.Queues {
+		if v.Device == device {
+			family := c.PhysicalDevices[c.Devices[device].PhysicalDevice].QueueFamilyProperties[v.Family]
+			if uint32(family.QueueFlags)&uint32(flags) == uint32(flags) {
+				return v.VulkanHandle
+			}
+		}
+	}
+	return VkQueue(0)
+}
+
 func (a *RecreateQueryPool) Mutate(ctx context.Context, id api.CmdID, s *api.GlobalState, b *builder.Builder) error {
 	defer EnterRecreate(ctx, s)()
 	l := s.MemoryLayout

gapis/api/vulkan/externs.go

@@ -907,6 +907,9 @@ func (e externs) execPendingCommands(queue VkQueue) {
 			if command.SemaphoreUpdate == SemaphoreUpdate_Unsignal {
 				o.Semaphores[command.Semaphore].Signaled = false
 			}
+			if command.SparseBinds != nil {
+				bindSparse(e.ctx, e.s, command.SparseBinds)
+			}
 			if command.Buffer == VkCommandBuffer(0) {
 				continue
 			}
@@ -1027,3 +1030,47 @@ func (e externs) popAndPushMarkerForNextSubpass(nextSubpass uint32) {
 		GetState(e.s).pushMarkerGroup(name, true, RenderPassMarker)
 	}
 }
+
+func bindSparse(ctx context.Context, s *api.GlobalState, binds *QueuedSparseBinds) {
+	st := GetState(s)
+	for buffer, binds := range binds.BufferBinds {
+		for _, bind := range binds.SparseMemoryBinds {
+			if _, ok := st.bufferSparseBindings[buffer]; !ok {
+				st.bufferSparseBindings[buffer] = sparseBindingList{}
+			}
+			st.bufferSparseBindings[buffer] = addBinding(
+				st.bufferSparseBindings[buffer], bind)
+		}
+		// update the data for UI
+		bufObj := st.Buffers.Get(buffer)
+		for i := 0; i < len(st.bufferSparseBindings[buffer]) || i < len(bufObj.SparseMemoryBindings); i++ {
+			if i >= len(st.bufferSparseBindings[buffer]) {
+				delete(bufObj.SparseMemoryBindings, uint32(i))
+			}
+			bufObj.SparseMemoryBindings[uint32(i)] = st.bufferSparseBindings[buffer][i]
+		}
+	}
+	for image, binds := range binds.OpaqueImageBinds {
+		for _, bind := range binds.SparseMemoryBinds {
+			if _, ok := st.opaqueImageSparseBindings[image]; !ok {
+				st.opaqueImageSparseBindings[image] = sparseBindingList{}
+			}
+			st.opaqueImageSparseBindings[image] = addBinding(
+				st.opaqueImageSparseBindings[image], bind)
+		}
+		// update the data for UI
+		imgObj := st.Images.Get(image)
+		for i := 0; i < len(st.opaqueImageSparseBindings[image]) || i < len(imgObj.OpaqueSparseMemoryBindings); i++ {
+			if i >= len(st.opaqueImageSparseBindings[image]) {
+				delete(imgObj.OpaqueSparseMemoryBindings, uint32(i))
+			}
+			imgObj.OpaqueSparseMemoryBindings[uint32(i)] = st.opaqueImageSparseBindings[image][i]
+		}
+	}
+	for image, binds := range binds.ImageBinds {
+		for _, bind := range binds.SparseImageMemoryBinds {
+			log.W(ctx, "sparse binding: image: %v, bindinfo: %v", image, bind)
+			log.W(ctx, "Image sparse residency binding is currently not supported")
+		}
+	}
+}