Fix invalid usage by waiting on CommandBuffer fence before resetting …

…CommandBuffer (#1795)

* Add fences to per-image copy resource
* Give fence to QueueSubmit to signal
* Wait on fence before resetting commandbuffer
* Add device to QueueInfo
* Use device from QueueInfo or as passed in

framework/decode/vulkan_object_info.h

@@ -314,6 +314,7 @@ struct DeviceInfo : public VulkanObjectInfo<VkDevice>
 
 struct QueueInfo : public VulkanObjectInfo<VkQueue>
 {
+ VkDevice parent{ VK_NULL_HANDLE };
  std::unordered_map<uint32_t, size_t> array_counts;
  uint32_t family_index;
  uint32_t queue_index;

framework/decode/vulkan_replay_consumer_base.cpp

@@ -3015,6 +3015,7 @@ void VulkanReplayConsumerBase::OverrideGetDeviceQueue(PFN_vkGetDeviceQueue
  // This is necessary for the virtual swapchain to determine which command buffer to use when
  // Bliting the images on the Presenting Queue.
  auto queue_info = reinterpret_cast<QueueInfo*>(pQueue->GetConsumerData(0));
+ queue_info->parent = device;
  queue_info->family_index = queueFamilyIndex;
  queue_info->queue_index = queueIndex;
 }
@@ -3038,6 +3039,7 @@ void VulkanReplayConsumerBase::OverrideGetDeviceQueue2(PFN_vkGetDeviceQueue2
  // This is necessary for the virtual swapchain to determine which command buffer to use when
  // Bliting the images on the Presenting Queue.
  auto queue_info = reinterpret_cast<QueueInfo*>(pQueue->GetConsumerData(0));
+ queue_info->parent = device;
  queue_info->family_index = in_pQueueInfo->queueFamilyIndex;
  queue_info->queue_index = in_pQueueInfo->queueIndex;
 }

framework/decode/vulkan_virtual_swapchain.cpp

@@ -134,6 +134,10 @@ void VulkanVirtualSwapchain::CleanSwapchainResourceData(const DeviceInfo*
  {
  device_table_->DestroySemaphore(device, semaphore, nullptr);
  }
+ for (auto& fence : copy_cmd_data.second.fences)
+ {
+ device_table_->DestroyFence(device, fence, nullptr);
+ }
  }
 
  swapchain_resources_.erase(swapchain);
@@ -362,6 +366,31 @@ VkResult VulkanVirtualSwapchain::CreateSwapchainResourceData(const DeviceInfo*
  copy_cmd_data.semaphores[ii] = semaphore;
  }
  }
+ uint32_t fence_count = static_cast<uint32_t>(copy_cmd_data.fences.size());
+ if (fence_count < capture_image_count)
+ {
+ copy_cmd_data.fences.resize(capture_image_count);
+
+ for (uint32_t ii = fence_count; ii < capture_image_count; ++ii)
+ {
+ VkFenceCreateInfo fence_create_info = {
+ VK_STRUCTURE_TYPE_FENCE_CREATE_INFO, // sType
+ nullptr, // pNext
+ VK_FENCE_CREATE_SIGNALED_BIT // flags
+ };
+
+ VkFence fence = VK_NULL_HANDLE;
+ result = device_table_->CreateFence(device, &fence_create_info, nullptr, &fence);
+ if (result != VK_SUCCESS)
+ {
+ GFXRECON_LOG_ERROR("Virtual swapchain failed creating internal copy fence for "
+ "swapchain (ID = %" PRIu64 ")",
+ swapchain_info->capture_id);
+ return result;
+ }
+ copy_cmd_data.fences[ii] = fence;
+ }
+ }
  }
  }
  }
@@ -431,7 +460,18 @@ VkResult VulkanVirtualSwapchain::CreateSwapchainResourceData(const DeviceInfo*
  begin_info.pInheritanceInfo = nullptr;
 
  auto command_buffer = swapchain_resources->copy_cmd_data[copy_queue_family_index].command_buffers[0];
+ auto copy_fence = swapchain_resources->copy_cmd_data[copy_queue_family_index].fences[0];
 
+ result = device_table_->WaitForFences(device, 1, &copy_fence, VK_TRUE, ~0UL);
+ if (result != VK_SUCCESS)
+ {
+ return result;
+ }
+ result = device_table_->ResetFences(device, 1, &copy_fence);
+ if (result != VK_SUCCESS)
+ {
+ return result;
+ }
  result = device_table_->ResetCommandBuffer(command_buffer, 0);
  if (result != VK_SUCCESS)
  {
@@ -500,7 +540,7 @@ VkResult VulkanVirtualSwapchain::CreateSwapchainResourceData(const DeviceInfo*
  submit_info.commandBufferCount = 1;
  submit_info.pCommandBuffers = &command_buffer;
 
- result = device_table_->QueueSubmit(initial_copy_queue, 1, &submit_info, VK_NULL_HANDLE);
+ result = device_table_->QueueSubmit(initial_copy_queue, 1, &submit_info, copy_fence);
  if (result != VK_SUCCESS)
  {
  GFXRECON_LOG_ERROR(
@@ -673,6 +713,7 @@ VkResult VulkanVirtualSwapchain::QueuePresentKHR(VkResult
  return func(queue_info->handle, present_info);
  }
 
+ VkDevice device = queue_info->parent;
  VkQueue queue = queue_info->handle;
  uint32_t queue_family_index = queue_info->family_index;
 
@@ -769,6 +810,7 @@ VkResult VulkanVirtualSwapchain::QueuePresentKHR(VkResult
  auto& copy_cmd_data = swapchain_resources->copy_cmd_data[queue_family_index];
  auto command_buffer = copy_cmd_data.command_buffers[capture_image_index];
  auto copy_semaphore = copy_cmd_data.semaphores[capture_image_index];
+ auto copy_fence = copy_cmd_data.fences[capture_image_index];
 
  std::vector<VkSemaphore> wait_semaphores;
  std::vector<VkSemaphore> signal_semaphores;
@@ -789,6 +831,16 @@ VkResult VulkanVirtualSwapchain::QueuePresentKHR(VkResult
  present_wait_semaphores.emplace_back(copy_semaphore);
  }
 
+ result = device_table_->WaitForFences(device, 1, &copy_fence, VK_TRUE, ~0UL);
+ if (result != VK_SUCCESS)
+ {
+ return result;
+ }
+ result = device_table_->ResetFences(device, 1, &copy_fence);
+ if (result != VK_SUCCESS)
+ {
+ return result;
+ }
  result = device_table_->ResetCommandBuffer(command_buffer, 0);
  if (result != VK_SUCCESS)
  {
@@ -892,7 +944,7 @@ VkResult VulkanVirtualSwapchain::QueuePresentKHR(VkResult
  submit_info.commandBufferCount = 1;
  submit_info.pCommandBuffers = &command_buffer;
 
- result = device_table_->QueueSubmit(queue, 1, &submit_info, VK_NULL_HANDLE);
+ result = device_table_->QueueSubmit(queue, 1, &submit_info, copy_fence);
 
  if (result != VK_SUCCESS)
  {

framework/decode/vulkan_virtual_swapchain.h

@@ -133,6 +133,7 @@ class VulkanVirtualSwapchain : public VulkanSwapchain
  {
  VkCommandPool command_pool;
  std::vector<VkCommandBuffer> command_buffers;
+ std::vector<VkFence> fences;
  std::vector<VkSemaphore> semaphores;
  };