[Sparse #1] Track memory referencing of sparse resources into queues

* Normal resources have a single memory bind that can be added to frame
  references when the resource is. Sparse resources don't have that easy
  single mapping.
* Calls to MarkResourceFrameReferenced with a 0 ResourceId aren't a
  real problem, but we have to be careful when dirtying resources.
* We don't have to dirty memory that's sparsely bound as we'll dirty it
  as soon as its sparsely bound. We do have to worry about making sure
  that memory is correctly frame referenced.
* To do this command buffers track the sparsemapping structures (will be
  created for all sparse resources) that are referenced underneath them.
  Then at queue submit time the current sparse mapping is iterated and
  all bound memory marked frame referenced.
* For most cases where a buffer is bound or similar, we can directly add
  the sparsemapping from the resource record. For descriptor sets we
  must do this indirectly - by marking any bindframerefs in the descset
  record with a bit indicating that the associated resource is sparse,
  then at submit time when iterating the bindframerefs, taking any with
  the bit set and looking up its record to find the sparsemapping.

renderdoc/core/resource_manager.cpp

@@ -45,6 +45,7 @@ namespace ResourceIDGen
 
 void ResourceRecord::MarkResourceFrameReferenced(ResourceId id, FrameRefType refType)
 {
+	if(id == ResourceId()) return;
 	ResourceManager<void*,void*,ResourceRecord>::MarkReferenced(m_FrameRefs, id, refType);
 }
 

renderdoc/driver/vulkan/vk_manager.cpp

@@ -287,6 +287,22 @@ void VulkanResourceManager::SerialiseImageStates(map<ResourceId, ImageLayouts> &
 	}
 }
 
+void VulkanResourceManager::MarkSparseMapReferenced(SparseMapping *sparse)
+{
+	if(sparse == NULL)
+	{
+		RDCERR("Unexpected NULL sparse mapping");
+		return;
+	}
+
+	for(size_t i=0; i < sparse->opaquemappings.size(); i++)
+		MarkResourceFrameReferenced(GetResID(sparse->opaquemappings[i].first), eFrameRef_Read);
+
+	for(int a=0; a < VK_IMAGE_ASPECT_NUM; a++)
+			for(VkDeviceSize i=0; sparse->pages[a] && i < sparse->imgdim.width*sparse->imgdim.height*sparse->imgdim.depth; i++)
+				MarkResourceFrameReferenced(GetResID(sparse->pages[a][i].first), eFrameRef_Read);
+}
+
 void VulkanResourceManager::ApplyTransitions(vector< pair<ResourceId, ImageRegionState> > &trans, map<ResourceId, ImageLayouts> &states)
 {
 	TRDBG("Applying %u transitions", (uint32_t)trans.size());

renderdoc/driver/vulkan/vk_manager.h

@@ -228,6 +228,8 @@ class VulkanResourceManager : public ResourceManager<WrappedVkRes*, TypedRealHan
 			delete GetWrapped(obj);
 		}
 
+		// helper for sparse mappings
+		void MarkSparseMapReferenced(SparseMapping *sparse);
 	private:
 		bool SerialisableResource(ResourceId id, VkResourceRecord *record);
 

renderdoc/driver/vulkan/vk_resources.h

@@ -575,14 +575,40 @@ struct SwapchainInfo
 	vector<SwapImage> images;
 	uint32_t lastPresent;
 };
-
+
+struct SparseMapping
+{
+	SparseMapping()
+	{
+		RDCEraseEl(imgdim);
+		RDCEraseEl(pagedim);
+		RDCEraseEl(pages);
+	}
+
+	// for buffers or non-sparse-resident images (bound with opaque mappings)
+	vector< pair<VkDeviceMemory, VkDeviceSize> > opaquemappings;
+
+	// for sparse resident images:
+	// total image size (in pages)
+	VkExtent3D imgdim;
+	// size of a page
+	VkExtent3D pagedim;
+	// pagetable per image aspect (some may be NULL)
+	// in order of width first, then height, then depth
+	pair<VkDeviceMemory, VkDeviceSize> *pages[VK_IMAGE_ASPECT_NUM];
+};
+
 struct CmdBufferRecordingInfo
 {
 	VkDevice device;
 	VkCmdBufferCreateInfo createInfo;
 
 	vector< pair<ResourceId, ImageRegionState> > imgtransitions;
 
+	// sparse resources referenced by this command buffer (at submit time
+	// need to go through the sparse mapping and reference all memory)
+	set<SparseMapping *> sparse;
+
 	// a list of all resources dirtied by this command buffer
 	set<ResourceId> dirtied;
 
@@ -640,19 +666,20 @@ struct VkResourceRecord : public ResourceRecord
 			cmdInfo->boundDescSets.swap(bakedCommands->cmdInfo->boundDescSets);
 			cmdInfo->imgtransitions.swap(bakedCommands->cmdInfo->imgtransitions);
 			cmdInfo->subcmds.swap(bakedCommands->cmdInfo->subcmds);
+			cmdInfo->sparse.swap(bakedCommands->cmdInfo->sparse);
 		}
 
-		void AddBindFrameRef(ResourceId id, FrameRefType ref)
+		void AddBindFrameRef(ResourceId id, FrameRefType ref, bool hasSparse = false)
 		{
 			if(id == ResourceId())
 			{
 				RDCERR("Unexpected NULL resource ID being added as a bind frame ref");
 				return;
 			}
 
-			if(bindFrameRefs[id].first == 0)
+			if((bindFrameRefs[id].first&~SPARSE_REF_BIT) == 0)
 			{
-				bindFrameRefs[id] = std::make_pair(1, ref);
+				bindFrameRefs[id] = std::make_pair(1 | (hasSparse ? SPARSE_REF_BIT : 0), ref);
 			}
 			else
 			{
@@ -669,7 +696,9 @@ struct VkResourceRecord : public ResourceRecord
 			// deleted since it was bound.
 			if(id == ResourceId()) return;
 
-			if(--bindFrameRefs[id].first == 0)
+			--bindFrameRefs[id].first;
+
+			if((bindFrameRefs[id].first&~SPARSE_REF_BIT) == 0)
 				bindFrameRefs.erase(id);
 		}
 
@@ -691,6 +720,7 @@ struct VkResourceRecord : public ResourceRecord
 		// creation or use, this can always be determined
 		ResourceId baseResource;
 		ResourceId baseResourceMem; // for image views, we need to point to both the image and mem
+		SparseMapping *sparseInfo;
 
 		// framebuffers are the only object that can point to multiple resources
 		// (as each attachment has an image).
@@ -715,7 +745,10 @@ struct VkResourceRecord : public ResourceRecord
 		// contains the framerefs (ref counted) for the bound resources
 		// in the binding slots. Updated when updating descriptor sets
 		// and then applied in a block on descriptor set bind.
-		map<ResourceId, pair<int, FrameRefType> > bindFrameRefs;
+		// the refcount has the high-bit set if this resource has sparse
+		// mapping information
+		static const uint32_t SPARSE_REF_BIT = 0x80000000;
+		map<ResourceId, pair<uint32_t, FrameRefType> > bindFrameRefs;
 };
 
 struct ImageLayouts

renderdoc/driver/vulkan/wrappers/vk_cmd_funcs.cpp

@@ -678,7 +678,10 @@ void WrappedVulkan::vkCmdBeginRenderPass(
 		{
 			if(fb->imageAttachments[i] == NULL) break;
 			record->MarkResourceFrameReferenced(fb->imageAttachments[i]->baseResource, eFrameRef_Write);
-			record->MarkResourceFrameReferenced(fb->imageAttachments[i]->baseResourceMem, eFrameRef_Read);
+			if(fb->imageAttachments[i]->baseResourceMem != ResourceId())
+				record->MarkResourceFrameReferenced(fb->imageAttachments[i]->baseResourceMem, eFrameRef_Read);
+			if(fb->imageAttachments[i]->sparseInfo)
+				record->cmdInfo->sparse.insert(fb->imageAttachments[i]->sparseInfo);
 			record->cmdInfo->dirtied.insert(fb->imageAttachments[i]->baseResource);
 		}
 	}
@@ -1334,6 +1337,8 @@ void WrappedVulkan::vkCmdBindVertexBuffers(
 		{
 			record->MarkResourceFrameReferenced(GetResID(pBuffers[i]), eFrameRef_Read);
 			record->MarkResourceFrameReferenced(GetRecord(pBuffers[i])->baseResource, eFrameRef_Read);
+			if(GetRecord(pBuffers[i])->sparseInfo)
+				record->cmdInfo->sparse.insert(GetRecord(pBuffers[i])->sparseInfo);
 		}
 	}
 }
@@ -1402,6 +1407,8 @@ void WrappedVulkan::vkCmdBindIndexBuffer(
 		record->AddChunk(scope.Get());
 		record->MarkResourceFrameReferenced(GetResID(buffer), eFrameRef_Read);
 		record->MarkResourceFrameReferenced(GetRecord(buffer)->baseResource, eFrameRef_Read);
+		if(GetRecord(buffer)->sparseInfo)
+			record->cmdInfo->sparse.insert(GetRecord(buffer)->sparseInfo);
 	}
 }
 
@@ -1470,7 +1477,10 @@ void WrappedVulkan::vkCmdUpdateBuffer(
 		// mark buffer just as read, and memory behind as write & dirtied
 		record->MarkResourceFrameReferenced(buf->GetResourceID(), eFrameRef_Read);
 		record->MarkResourceFrameReferenced(buf->baseResource, eFrameRef_Write);
-		record->cmdInfo->dirtied.insert(buf->baseResource);
+		if(buf->baseResource != ResourceId())
+			record->cmdInfo->dirtied.insert(buf->baseResource);
+		if(buf->sparseInfo)
+			record->cmdInfo->sparse.insert(buf->sparseInfo);
 	}
 }
 
@@ -1537,7 +1547,10 @@ void WrappedVulkan::vkCmdFillBuffer(
 		// mark buffer just as read, and memory behind as write & dirtied
 		record->MarkResourceFrameReferenced(buf->GetResourceID(), eFrameRef_Read);
 		record->MarkResourceFrameReferenced(buf->baseResource, eFrameRef_Write);
-		record->cmdInfo->dirtied.insert(buf->baseResource);
+		if(buf->baseResource != ResourceId())
+			record->cmdInfo->dirtied.insert(buf->baseResource);
+		if(buf->sparseInfo)
+			record->cmdInfo->sparse.insert(buf->sparseInfo);
 	}
 }
 
@@ -1833,7 +1846,10 @@ void WrappedVulkan::vkCmdWriteTimestamp(
 		// mark buffer just as read, and memory behind as write & dirtied
 		record->MarkResourceFrameReferenced(buf->GetResourceID(), eFrameRef_Read);
 		record->MarkResourceFrameReferenced(buf->baseResource, eFrameRef_Write);
-		record->cmdInfo->dirtied.insert(buf->baseResource);
+		if(buf->baseResource != ResourceId())
+			record->cmdInfo->dirtied.insert(buf->baseResource);
+		if(buf->sparseInfo)
+			record->cmdInfo->sparse.insert(buf->sparseInfo);
 	}
 }
 
@@ -1912,7 +1928,10 @@ void WrappedVulkan::vkCmdCopyQueryPoolResults(
 		// mark buffer just as read, and memory behind as write & dirtied
 		record->MarkResourceFrameReferenced(buf->GetResourceID(), eFrameRef_Read);
 		record->MarkResourceFrameReferenced(buf->baseResource, eFrameRef_Write);
-		record->cmdInfo->dirtied.insert(buf->baseResource);
+		if(buf->baseResource != ResourceId())
+			record->cmdInfo->dirtied.insert(buf->baseResource);
+		if(buf->sparseInfo)
+			record->cmdInfo->sparse.insert(buf->sparseInfo);
 	}
 }
 

renderdoc/driver/vulkan/wrappers/vk_descriptor_funcs.cpp

@@ -623,13 +623,15 @@ void WrappedVulkan::vkUpdateDescriptorSets(
 				if(bind.bufferView != VK_NULL_HANDLE)
 				{
 					record->RemoveBindFrameRef(GetResID(bind.bufferView));
-					record->RemoveBindFrameRef(GetRecord(bind.bufferView)->baseResource);
+					if(GetRecord(bind.bufferView)->baseResource != ResourceId())
+						record->RemoveBindFrameRef(GetRecord(bind.bufferView)->baseResource);
 				}
 				if(bind.imageView != VK_NULL_HANDLE)
 				{
 					record->RemoveBindFrameRef(GetResID(bind.imageView));
 					record->RemoveBindFrameRef(GetRecord(bind.imageView)->baseResource);
-					record->RemoveBindFrameRef(GetRecord(bind.imageView)->baseResourceMem);
+					if(GetRecord(bind.imageView)->baseResourceMem != ResourceId())
+						record->RemoveBindFrameRef(GetRecord(bind.imageView)->baseResourceMem);
 				}
 				if(bind.sampler != VK_NULL_HANDLE)
 				{
@@ -638,30 +640,34 @@ void WrappedVulkan::vkUpdateDescriptorSets(
 				if(bind.bufferInfo.buffer != VK_NULL_HANDLE)
 				{
 					record->RemoveBindFrameRef(GetResID(bind.bufferInfo.buffer));
-					record->RemoveBindFrameRef(GetRecord(bind.bufferInfo.buffer)->baseResource);
+					if(GetRecord(bind.bufferInfo.buffer)->baseResource != ResourceId())
+						record->RemoveBindFrameRef(GetRecord(bind.bufferInfo.buffer)->baseResource);
 				}
 
 				bind = pDescriptorWrites[i].pDescriptors[d];
 
 				if(bind.bufferView != VK_NULL_HANDLE)
 				{
-					record->AddBindFrameRef(GetResID(bind.bufferView), eFrameRef_Read);
-					record->AddBindFrameRef(GetRecord(bind.bufferView)->baseResource, ref);
+					record->AddBindFrameRef(GetResID(bind.bufferView), eFrameRef_Read, GetRecord(bind.bufferView)->sparseInfo != NULL);
+					if(GetRecord(bind.bufferView)->baseResource != ResourceId())
+						record->AddBindFrameRef(GetRecord(bind.bufferView)->baseResource, ref);
 				}
 				if(bind.imageView != VK_NULL_HANDLE)
 				{
-					record->AddBindFrameRef(GetResID(bind.imageView), eFrameRef_Read);
+					record->AddBindFrameRef(GetResID(bind.imageView), eFrameRef_Read, GetRecord(bind.imageView)->sparseInfo != NULL);
 					record->AddBindFrameRef(GetRecord(bind.imageView)->baseResource, ref);
-					record->AddBindFrameRef(GetRecord(bind.imageView)->baseResourceMem, eFrameRef_Read);
+					if(GetRecord(bind.imageView)->baseResourceMem != ResourceId())
+						record->AddBindFrameRef(GetRecord(bind.imageView)->baseResourceMem, eFrameRef_Read);
 				}
 				if(bind.sampler != VK_NULL_HANDLE)
 				{
 					record->AddBindFrameRef(GetResID(bind.sampler), ref);
 				}
 				if(bind.bufferInfo.buffer != VK_NULL_HANDLE)
 				{
-					record->AddBindFrameRef(GetResID(bind.bufferInfo.buffer), eFrameRef_Read);
-					record->AddBindFrameRef(GetRecord(bind.bufferInfo.buffer)->baseResource, ref);
+					record->AddBindFrameRef(GetResID(bind.bufferInfo.buffer), eFrameRef_Read, GetRecord(bind.bufferInfo.buffer)->sparseInfo != NULL);
+					if(GetRecord(bind.bufferInfo.buffer)->baseResource != ResourceId())
+						record->AddBindFrameRef(GetRecord(bind.bufferInfo.buffer)->baseResource, ref);
 				}
 			}
 		}

renderdoc/driver/vulkan/wrappers/vk_draw_funcs.cpp

@@ -180,6 +180,10 @@ void WrappedVulkan::vkCmdBlitImage(
 		record->MarkResourceFrameReferenced(GetResID(destImage), eFrameRef_Write);
 		record->MarkResourceFrameReferenced(GetRecord(destImage)->baseResource, eFrameRef_Read);
 		record->cmdInfo->dirtied.insert(GetResID(destImage));
+		if(GetRecord(srcImage)->sparseInfo)
+			record->cmdInfo->sparse.insert(GetRecord(srcImage)->sparseInfo);
+		if(GetRecord(destImage)->sparseInfo)
+			record->cmdInfo->sparse.insert(GetRecord(destImage)->sparseInfo);
 	}
 }
 
@@ -257,6 +261,10 @@ void WrappedVulkan::vkCmdResolveImage(
 		record->MarkResourceFrameReferenced(GetResID(destImage), eFrameRef_Write);
 		record->MarkResourceFrameReferenced(GetRecord(destImage)->baseResource, eFrameRef_Read);
 		record->cmdInfo->dirtied.insert(GetResID(destImage));
+		if(GetRecord(srcImage)->sparseInfo)
+			record->cmdInfo->sparse.insert(GetRecord(srcImage)->sparseInfo);
+		if(GetRecord(destImage)->sparseInfo)
+			record->cmdInfo->sparse.insert(GetRecord(destImage)->sparseInfo);
 	}
 }
 
@@ -333,6 +341,10 @@ void WrappedVulkan::vkCmdCopyImage(
 		record->MarkResourceFrameReferenced(GetResID(destImage), eFrameRef_Write);
 		record->MarkResourceFrameReferenced(GetRecord(destImage)->baseResource, eFrameRef_Read);
 		record->cmdInfo->dirtied.insert(GetResID(destImage));
+		if(GetRecord(srcImage)->sparseInfo)
+			record->cmdInfo->sparse.insert(GetRecord(srcImage)->sparseInfo);
+		if(GetRecord(destImage)->sparseInfo)
+			record->cmdInfo->sparse.insert(GetRecord(destImage)->sparseInfo);
 	}
 }
 
@@ -423,6 +435,10 @@ void WrappedVulkan::vkCmdCopyBufferToImage(
 		record->MarkResourceFrameReferenced(GetResID(destImage), eFrameRef_Write);
 		record->MarkResourceFrameReferenced(GetRecord(destImage)->baseResource, eFrameRef_Read);
 		record->cmdInfo->dirtied.insert(GetResID(destImage));
+		if(GetRecord(srcBuffer)->sparseInfo)
+			record->cmdInfo->sparse.insert(GetRecord(srcBuffer)->sparseInfo);
+		if(GetRecord(destImage)->sparseInfo)
+			record->cmdInfo->sparse.insert(GetRecord(destImage)->sparseInfo);
 	}
 }
 
@@ -500,7 +516,12 @@ void WrappedVulkan::vkCmdCopyImageToBuffer(
 		// mark buffer just as read, and memory behind as write & dirtied
 		record->MarkResourceFrameReferenced(buf->GetResourceID(), eFrameRef_Read);
 		record->MarkResourceFrameReferenced(buf->baseResource, eFrameRef_Write);
-		record->cmdInfo->dirtied.insert(buf->baseResource);
+		if(buf->baseResource != ResourceId())
+			record->cmdInfo->dirtied.insert(buf->baseResource);
+		if(GetRecord(srcImage)->sparseInfo)
+			record->cmdInfo->sparse.insert(GetRecord(srcImage)->sparseInfo);
+		if(buf->sparseInfo)
+			record->cmdInfo->sparse.insert(buf->sparseInfo);
 	}
 }
 
@@ -589,7 +610,12 @@ void WrappedVulkan::vkCmdCopyBuffer(
 		// mark buffer just as read, and memory behind as write & dirtied
 		record->MarkResourceFrameReferenced(buf->GetResourceID(), eFrameRef_Read);
 		record->MarkResourceFrameReferenced(buf->baseResource, eFrameRef_Write);
-		record->cmdInfo->dirtied.insert(buf->baseResource);
+		if(buf->baseResource != ResourceId())
+			record->cmdInfo->dirtied.insert(buf->baseResource);
+		if(GetRecord(srcBuffer)->sparseInfo)
+			record->cmdInfo->sparse.insert(GetRecord(srcBuffer)->sparseInfo);
+		if(buf->sparseInfo)
+			record->cmdInfo->sparse.insert(buf->sparseInfo);
 	}
 }
 
@@ -658,6 +684,8 @@ void WrappedVulkan::vkCmdClearColorImage(
 		record->AddChunk(scope.Get());
 		record->MarkResourceFrameReferenced(GetResID(image), eFrameRef_Write);
 		record->MarkResourceFrameReferenced(GetRecord(image)->baseResource, eFrameRef_Read);
+		if(GetRecord(image)->sparseInfo)
+			record->cmdInfo->sparse.insert(GetRecord(image)->sparseInfo);
 	}
 }
 
@@ -725,6 +753,8 @@ void WrappedVulkan::vkCmdClearDepthStencilImage(
 		record->AddChunk(scope.Get());
 		record->MarkResourceFrameReferenced(GetResID(image), eFrameRef_Write);
 		record->MarkResourceFrameReferenced(GetRecord(image)->baseResource, eFrameRef_Read);
+		if(GetRecord(image)->sparseInfo)
+			record->cmdInfo->sparse.insert(GetRecord(image)->sparseInfo);
 	}
 }
 

renderdoc/driver/vulkan/wrappers/vk_queue_funcs.cpp

@@ -399,8 +399,18 @@ VkResult WrappedVulkan::vkQueueSubmit(
 				{
 					refdIDs.insert(refit->first);
 					GetResourceManager()->MarkResourceFrameReferenced(refit->first, refit->second.second);
+
+					if(refit->second.first & VkResourceRecord::SPARSE_REF_BIT)
+					{
+						VkResourceRecord *record = GetResourceManager()->GetResourceRecord(refit->first);
+
+						GetResourceManager()->MarkSparseMapReferenced(record->sparseInfo);
+					}
 				}
 			}
+
+			for(auto it = record->bakedCommands->cmdInfo->sparse.begin(); it != record->bakedCommands->cmdInfo->sparse.end(); ++it)
+				GetResourceManager()->MarkSparseMapReferenced(*it);
 
 			// pull in frame refs from this baked command buffer
 			record->bakedCommands->AddResourceReferences(GetResourceManager());