Skip to content

Commit

Permalink
vulkan: reset fences instead create/destroy (#7169)
Browse files Browse the repository at this point in the history 
We allocate all the fences beforehand to reduce calls to
vkCreateFence.

Also remove blocking code in `getFenceStatus` since there is
not a usecase that would require that.
  • Loading branch information
@poweifeng
poweifeng authored Sep 14, 2023
1 parent 2114995 commit d4d03e4
Show file tree
Hide file tree
Showing 3 changed files with 75 additions and 71 deletions.
123 changes: 64 additions & 59 deletions filament/backend/src/vulkan/VulkanCommands.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,23 +35,14 @@ namespace filament::backend {

using Timestamp = VulkanGroupMarkers::Timestamp;

VulkanCmdFence::VulkanCmdFence(VkDevice device, bool signaled) : device(device) {
VkFenceCreateInfo fenceCreateInfo { .sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO };
if (signaled) {
fenceCreateInfo.flags = VK_FENCE_CREATE_SIGNALED_BIT;
}
vkCreateFence(device, &fenceCreateInfo, VKALLOC, &fence);

VulkanCmdFence::VulkanCmdFence(VkFence ifence)
: fence(ifence) {
// Internally we use the VK_INCOMPLETE status to mean "not yet submitted". When this fence gets
// submitted, its status changes to VK_NOT_READY. Finally, when the GPU actually finishes
// executing the command buffer, the status changes to VK_SUCCESS.
status.store(VK_INCOMPLETE);
}

VulkanCmdFence::~VulkanCmdFence() {
vkDestroyFence(device, fence, VKALLOC);
}

VulkanCommandBuffer::VulkanCommandBuffer(VulkanResourceAllocator* allocator, VkDevice device,
VkCommandPool pool)
: mResourceManager(allocator) {
Expand All @@ -64,19 +55,18 @@ VulkanCommandBuffer::VulkanCommandBuffer(VulkanResourceAllocator* allocator, VkD
};

// The buffer allocated here will be implicitly reset when vkBeginCommandBuffer is called.
vkAllocateCommandBuffers(device, &allocateInfo, &mBuffer);

// We don't need to deallocate since destroying the pool will free all of the buffers.
vkAllocateCommandBuffers(device, &allocateInfo, &mBuffer);
}

CommandBufferObserver::~CommandBufferObserver() {}

static VkCommandPool createPool(VkDevice device, uint32_t queueFamilyIndex) {
VkCommandPoolCreateInfo createInfo = {
.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO,
.flags =
VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT | VK_COMMAND_POOL_CREATE_TRANSIENT_BIT,
.queueFamilyIndex = queueFamilyIndex,
.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO,
.flags =
VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT | VK_COMMAND_POOL_CREATE_TRANSIENT_BIT,
.queueFamilyIndex = queueFamilyIndex,
};
VkCommandPool pool;
vkCreateCommandPool(device, &createInfo, VKALLOC, &pool);
Expand Down Expand Up @@ -123,7 +113,7 @@ std::pair<std::string, Timestamp> VulkanGroupMarkers::top() const {
assert_invariant(!empty());
auto const marker = mMarkers.back();
#if FILAMENT_VULKAN_VERBOSE
auto const topTimestamp = mTimestamps.top();
auto const topTimestamp = mTimestamps.front();
return std::make_pair(marker, topTimestamp);
#else
return std::make_pair(marker, Timestamp{});
Expand All @@ -146,6 +136,11 @@ VulkanCommands::VulkanCommands(VkDevice device, VkQueue queue, uint32_t queueFam
vkCreateSemaphore(mDevice, &sci, nullptr, &semaphore);
}

VkFenceCreateInfo fenceCreateInfo{.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};
for (auto& fence: mFences) {
vkCreateFence(device, &fenceCreateInfo, VKALLOC, &fence);
}

for (size_t i = 0; i < CAPACITY; ++i) {
mStorage[i] = std::make_unique<VulkanCommandBuffer>(allocator, mDevice, mPool);
}
Expand All @@ -155,9 +150,12 @@ VulkanCommands::~VulkanCommands() {
wait();
gc();
vkDestroyCommandPool(mDevice, mPool, VKALLOC);
for (VkSemaphore sema : mSubmissionSignals) {
for (VkSemaphore sema: mSubmissionSignals) {
vkDestroySemaphore(mDevice, sema, VKALLOC);
}
for (VkFence fence: mFences) {
vkDestroyFence(mDevice, fence, VKALLOC);
}
}

VulkanCommandBuffer& VulkanCommands::get() {
Expand All @@ -170,9 +168,9 @@ VulkanCommandBuffer& VulkanCommands::get() {
// presenting the swap chain or waiting on a fence.
while (mAvailableBufferCount == 0) {
#if VK_REPORT_STALLS
slog.i << "VulkanCommands has stalled. "
<< "If this occurs frequently, consider increasing VK_MAX_COMMAND_BUFFERS."
<< io::endl;
slog.i << "VulkanCommands has stalled. "
<< "If this occurs frequently, consider increasing VK_MAX_COMMAND_BUFFERS."
<< io::endl;
#endif
wait();
gc();
Expand All @@ -195,12 +193,12 @@ VulkanCommandBuffer& VulkanCommands::get() {
// Note that the fence wrapper uses shared_ptr because a DriverAPI fence can also have ownership
// over it. The destruction of the low-level fence occurs either in VulkanCommands::gc(), or in
// VulkanDriver::destroyFence(), both of which are safe spots.
currentbuf->fence = std::make_shared<VulkanCmdFence>(mDevice);
currentbuf->fence = std::make_shared<VulkanCmdFence>(mFences[mCurrentCommandBufferIndex]);

// Begin writing into the command buffer.
const VkCommandBufferBeginInfo binfo {
.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT,
const VkCommandBufferBeginInfo binfo{
.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT,
};
vkBeginCommandBuffer(currentbuf->buffer(), &binfo);

Expand All @@ -215,7 +213,6 @@ VulkanCommandBuffer& VulkanCommands::get() {
auto [marker, time] = mCarriedOverMarkers->pop();
pushGroupMarker(marker.c_str(), time);
}

return *currentbuf;
}

Expand All @@ -225,7 +222,6 @@ bool VulkanCommands::flush() {
return false;
}


// Before actually submitting, we need to pop any leftover group markers.
// Note that this needs to occur before vkEndCommandBuffer.
while (mGroupMarkers && !mGroupMarkers->empty()) {
Expand All @@ -252,26 +248,26 @@ bool VulkanCommands::flush() {
// the only safe option because the previously submitted command buffer might have set up some
// state that the new command buffer depends on.
VkPipelineStageFlags waitDestStageMasks[2] = {
VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
};

VkSemaphore signals[2] = {
VK_NULL_HANDLE,
VK_NULL_HANDLE,
VK_NULL_HANDLE,
VK_NULL_HANDLE,
};

VkCommandBuffer const cmdbuffer = currentbuf->buffer();

VkSubmitInfo submitInfo {
.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO,
.waitSemaphoreCount = 0,
.pWaitSemaphores = signals,
.pWaitDstStageMask = waitDestStageMasks,
.commandBufferCount = 1,
.pCommandBuffers = &cmdbuffer,
.signalSemaphoreCount = 1u,
.pSignalSemaphores = &renderingFinished,
VkSubmitInfo submitInfo{
.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO,
.waitSemaphoreCount = 0,
.pWaitSemaphores = signals,
.pWaitDstStageMask = waitDestStageMasks,
.commandBufferCount = 1,
.pCommandBuffers = &cmdbuffer,
.signalSemaphoreCount = 1u,
.pSignalSemaphores = &renderingFinished,
};

if (mSubmissionSignal) {
Expand All @@ -289,9 +285,9 @@ bool VulkanCommands::flush() {

#if FILAMENT_VULKAN_VERBOSE
slog.i << "Submitting cmdbuffer=" << cmdbuffer
<< " wait=(" << signals[0] << ", " << signals[1] << ") "
<< " signal=" << renderingFinished
<< io::endl;
<< " wait=(" << signals[0] << ", " << signals[1] << ") "
<< " signal=" << renderingFinished
<< io::endl;
#endif

auto& cmdfence = currentbuf->fence;
Expand All @@ -303,7 +299,7 @@ bool VulkanCommands::flush() {

#if FILAMENT_VULKAN_VERBOSE
if (result != VK_SUCCESS) {
utils::slog.d <<"Failed command buffer submission result: " << result << utils::io::endl;
utils::slog.d << "Failed command buffer submission result: " << result << utils::io::endl;
}
#endif
assert_invariant(result == VK_SUCCESS);
Expand Down Expand Up @@ -343,23 +339,32 @@ void VulkanCommands::wait() {
}
if (count > 0) {
vkWaitForFences(mDevice, count, fences, VK_TRUE, UINT64_MAX);
vkResetFences(mDevice, count, fences);
}
}

void VulkanCommands::gc() {
VkFence fences[CAPACITY];
size_t count = 0;

for (size_t i = 0; i < CAPACITY; i++) {
auto wrapper = mStorage[i].get();
if (wrapper->buffer() == VK_NULL_HANDLE) {
continue;
}
VkResult const result = vkWaitForFences(mDevice, 1, &wrapper->fence->fence, VK_TRUE, 0);
VkResult const result = vkGetFenceStatus(mDevice, wrapper->fence->fence);
if (result != VK_SUCCESS) {
continue;
}
fences[count++] = wrapper->fence->fence;
wrapper->fence->status.store(VK_SUCCESS);
wrapper->reset();
mAvailableBufferCount++;
}

if (count > 0) {
vkResetFences(mDevice, count, fences);
}
}

void VulkanCommands::updateFences() {
Expand Down Expand Up @@ -411,19 +416,19 @@ void VulkanCommands::pushGroupMarker(char const* str, VulkanGroupMarkers::Timest
}

void VulkanCommands::popGroupMarker() {
assert_invariant(mGroupMarkers);
assert_invariant(mGroupMarkers);

if (!mGroupMarkers->empty()) {
VkCommandBuffer const cmdbuffer = get().buffer();
#if FILAMENT_VULKAN_VERBOSE
auto const [marker, startTime] = mGroupMarkers->pop();
auto const endTime = std::chrono::high_resolution_clock::now();
std::chrono::duration<double> diff = endTime - startTime;
utils::slog.d << "<---- " << marker << " elapsed: " << (diff.count() * 1000) << " ms\n"
<< utils::io::flush;
#else
mGroupMarkers->pop();
#endif
#if FILAMENT_VULKAN_VERBOSE
auto const [marker, startTime] = mGroupMarkers->pop();
auto const endTime = std::chrono::high_resolution_clock::now();
std::chrono::duration<double> diff = endTime - startTime;
utils::slog.d << "<---- " << marker << " elapsed: " << (diff.count() * 1000) << " ms\n"
<< utils::io::flush;
#else
mGroupMarkers->pop();
#endif

if (mContext->isDebugUtilsSupported()) {
vkCmdEndDebugUtilsLabelEXT(cmdbuffer);
Expand All @@ -449,9 +454,9 @@ void VulkanCommands::insertEventMarker(char const* string, uint32_t len) {
vkCmdInsertDebugUtilsLabelEXT(cmdbuffer, &labelInfo);
} else if (mContext->isDebugMarkersSupported()) {
VkDebugMarkerMarkerInfoEXT markerInfo = {
.sType = VK_STRUCTURE_TYPE_DEBUG_MARKER_MARKER_INFO_EXT,
.pMarkerName = string,
.color = {0.0f, 1.0f, 0.0f, 1.0f},
.sType = VK_STRUCTURE_TYPE_DEBUG_MARKER_MARKER_INFO_EXT,
.pMarkerName = string,
.color = {0.0f, 1.0f, 0.0f, 1.0f},
};
vkCmdDebugMarkerInsertEXT(cmdbuffer, &markerInfo);
}
Expand Down
6 changes: 3 additions & 3 deletions filament/backend/src/vulkan/VulkanCommands.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,9 +58,8 @@ class VulkanGroupMarkers {

// Wrapper to enable use of shared_ptr for implementing shared ownership of low-level Vulkan fences.
struct VulkanCmdFence {
VulkanCmdFence(VkDevice device, bool signaled = false);
~VulkanCmdFence();
const VkDevice device;
VulkanCmdFence(VkFence ifence);
~VulkanCmdFence() = default;
VkFence fence;
utils::Condition condition;
utils::Mutex mutex;
Expand Down Expand Up @@ -192,6 +191,7 @@ class VulkanCommands {
VkSemaphore mSubmissionSignal = {};
VkSemaphore mInjectedSignal = {};
utils::FixedCapacityVector<std::unique_ptr<VulkanCommandBuffer>> mStorage;
VkFence mFences[CAPACITY] = {};
VkSemaphore mSubmissionSignals[CAPACITY] = {};
uint8_t mAvailableBufferCount = CAPACITY;
CommandBufferObserver* mObserver = nullptr;
Expand Down
17 changes: 8 additions & 9 deletions filament/backend/src/vulkan/VulkanDriver.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -691,16 +691,15 @@ FenceStatus VulkanDriver::getFenceStatus(Handle<HwFence> fh) {
// Internally we use the VK_INCOMPLETE status to mean "not yet submitted".
// When this fence gets submitted, its status changes to VK_NOT_READY.
std::unique_lock<utils::Mutex> lock(cmdfence->mutex);
if (cmdfence->status.load() == VK_INCOMPLETE) {
// This will obviously timeout if Filament creates a fence and immediately waits on it
// without calling endFrame() or commit().
cmdfence->condition.wait(lock);
} else {
lock.unlock();
if (cmdfence->status.load() == VK_SUCCESS) {
return FenceStatus::CONDITION_SATISFIED;
}
VkResult result =
vkWaitForFences(mPlatform->getDevice(), 1, &cmdfence->fence, VK_TRUE, 0);
return result == VK_SUCCESS ? FenceStatus::CONDITION_SATISFIED : FenceStatus::TIMEOUT_EXPIRED;

// Two other states are possible:
// - VK_INCOMPLETE: the corresponding buffer has not yet been submitted.
// - VK_NOT_READY: the buffer has been submitted but not yet signaled.
// In either case, we return TIMEOUT_EXPIRED to indicate the fence has not been signaled.
return FenceStatus::TIMEOUT_EXPIRED;
}

// We create all textures using VK_IMAGE_TILING_OPTIMAL, so our definition of "supported" is that
Expand Down

0 comments on commit d4d03e4

Please sign in to comment.