Timeline semaphores

include/ppx/config.h

@@ -146,6 +146,7 @@ enum Result
     ERROR_GRFX_INVALID_BINDING_NUMBER               = -1024,
     ERROR_GRFX_INVALID_SET_NUMBER                   = -1025,
     ERROR_GRFX_OPERATION_NOT_PERMITTED              = -1026,
+    ERROR_GRFX_INVALID_SEMAPHORE_TYPE               = -1027,
 
     ERROR_IMAGE_FILE_LOAD_FAILED               = -2000,
     ERROR_IMAGE_FILE_SAVE_FAILED               = -2001,

include/ppx/grfx/dx12/dx12_queue.h

@@ -35,6 +35,9 @@ class Queue
 
     virtual Result Submit(const grfx::SubmitInfo* pSubmitInfo) override;
 
+    virtual Result QueueWait(grfx::Semaphore* pSemaphore, uint64_t value) override;
+    virtual Result QueueSignal(grfx::Semaphore* pSemaphore, uint64_t value) override;
+
     virtual Result GetTimestampFrequency(uint64_t* pFrequency) const override;
 
 protected:

include/ppx/grfx/dx12/dx12_sync.h

@@ -61,13 +61,19 @@ class Semaphore
     UINT64 GetNextSignalValue();
     UINT64 GetWaitForValue() const;
 
+private:
+    virtual Result   TimelineWait(uint64_t value, uint64_t timeout) const override;
+    virtual Result   TimelineSignal(uint64_t value) const override;
+    virtual uint64_t TimelineCounterValue() const override;
+
 protected:
     virtual Result CreateApiObjects(const grfx::SemaphoreCreateInfo* pCreateInfo) override;
     virtual void   DestroyApiObjects() override;
 
 private:
     D3D12FencePtr mFence;
-    UINT64        mValue = 0;
+    HANDLE        mWaitEventHandle = nullptr;
+    UINT64        mValue           = 0;
 };
 
 } // namespace dx12

include/ppx/grfx/grfx_enums.h

@@ -441,6 +441,12 @@ enum SampleCount
     SAMPLE_COUNT_64 = 64,
 };
 
+enum SemaphoreType
+{
+    SEMAPHORE_TYPE_BINARY   = 0,
+    SEMAPHORE_TYPE_TIMELINE = 1,
+};
+
 enum ShaderStageBits
 {
     SHADER_STAGE_UNDEFINED    = 0x00000000,

include/ppx/grfx/grfx_queue.h

@@ -27,8 +27,10 @@ struct SubmitInfo
     const grfx::CommandBuffer* const* ppCommandBuffers     = nullptr;
     uint32_t                          waitSemaphoreCount   = 0;
     const grfx::Semaphore* const*     ppWaitSemaphores     = nullptr;
+    std::vector<uint64_t>             waitValues           = {}; // Use 0 if index is binary semaphore
     uint32_t                          signalSemaphoreCount = 0;
     grfx::Semaphore**                 ppSignalSemaphores   = nullptr;
+    std::vector<uint64_t>             signalValues         = {}; // Use 0 if index is binary smeaphore
     grfx::Fence*                      pFence               = nullptr;
 };
 
@@ -63,6 +65,10 @@ class Queue
 
     virtual Result Submit(const grfx::SubmitInfo* pSubmitInfo) = 0;
 
+    // Timeline semaphore functions
+    virtual Result QueueWait(grfx::Semaphore* pSemaphore, uint64_t value)   = 0;
+    virtual Result QueueSignal(grfx::Semaphore* pSemaphore, uint64_t value) = 0;
+
     // GPU timestamp frequency counter in ticks per second
     virtual Result GetTimestampFrequency(uint64_t* pFrequency) const = 0;
 

include/ppx/grfx/grfx_sync.h

@@ -56,6 +56,8 @@ class Fence
 //!
 struct SemaphoreCreateInfo
 {
+    grfx::SemaphoreType semaphoreType = grfx::SEMAPHORE_TYPE_BINARY;
+    uint64_t            initialValue  = 0; // Timeline semaphore only
 };
 
 //! @class Semaphore
@@ -68,12 +70,36 @@ class Semaphore
     Semaphore() {}
     virtual ~Semaphore() {}
 
+    grfx::SemaphoreType GetSemaphoreType() const { return mCreateInfo.semaphoreType; }
+    bool                IsBinary() const { return mCreateInfo.semaphoreType == grfx::SEMAPHORE_TYPE_BINARY; }
+    bool                IsTimeline() const { return mCreateInfo.semaphoreType == grfx::SEMAPHORE_TYPE_TIMELINE; }
+
+    // Timeline semaphore wait
+    Result Wait(uint64_t value, uint64_t timeout = UINT64_MAX) const;
+
+    // Timeline semaphore signal
+    //
+    // WARNING: Signaling a value less than what's already been signaled can
+    //          cause a block or a race condition.
+    //
+    Result Signal(uint64_t value) const;
+
+    // Returns current timeline semaphore value
+    uint64_t GetCounterValue() const;
+
+private:
+    virtual Result   TimelineWait(uint64_t value, uint64_t timeout) const = 0;
+    virtual Result   TimelineSignal(uint64_t value) const                 = 0;
+    virtual uint64_t TimelineCounterValue() const                         = 0;
+
 protected:
     virtual Result CreateApiObjects(const grfx::SemaphoreCreateInfo* pCreateInfo) = 0;
     virtual void   DestroyApiObjects()                                            = 0;
     friend class grfx::Device;
 };
 
+// -------------------------------------------------------------------------------------------------
+
 } // namespace grfx
 } // namespace ppx
 

include/ppx/grfx/vk/vk_device.h

@@ -50,6 +50,10 @@ class Device
         uint32_t    firstQuery,
         uint32_t    queryCount) const;
 
+    VkResult WaitSemaphores(const VkSemaphoreWaitInfo* pWaitInfo, uint64_t timeout) const;
+    VkResult SignalSemaphore(const VkSemaphoreSignalInfo* pSignalInfo);
+    VkResult GetSemaphoreCounterValue(VkSemaphore semaphore, uint64_t* pValue);
+
     uint32_t                GetGraphicsQueueFamilyIndex() const { return mGraphicsQueueFamilyIndex; }
     uint32_t                GetComputeQueueFamilyIndex() const { return mComputeQueueFamilyIndex; }
     uint32_t                GetTransferQueueFamilyIndex() const { return mTransferQueueFamilyIndex; }
@@ -113,6 +117,9 @@ class Device
     bool                                           mHasUnrestrictedDepthRange                  = false;
     bool                                           mHasDynamicRendering                        = false;
     PFN_vkResetQueryPoolEXT                        mFnResetQueryPoolEXT                        = nullptr;
+    PFN_vkWaitSemaphores                           mFnWaitSemaphores                           = nullptr;
+    PFN_vkSignalSemaphore                          mFnSignalSemaphore                          = nullptr;
+    PFN_vkGetSemaphoreCounterValue                 mFnGetSemaphoreCounterValue                 = nullptr;
     uint32_t                                       mGraphicsQueueFamilyIndex                   = 0;
     uint32_t                                       mComputeQueueFamilyIndex                    = 0;
     uint32_t                                       mTransferQueueFamilyIndex                   = 0;

include/ppx/grfx/vk/vk_queue.h

@@ -37,6 +37,9 @@ class Queue
 
     virtual Result Submit(const grfx::SubmitInfo* pSubmitInfo) override;
 
+    virtual Result QueueWait(grfx::Semaphore* pSemaphore, uint64_t value) override;
+    virtual Result QueueSignal(grfx::Semaphore* pSemaphore, uint64_t value) override;
+
     virtual Result GetTimestampFrequency(uint64_t* pFrequency) const override;
 
     VkResult TransitionImageLayout(
@@ -57,6 +60,8 @@ class Queue
 private:
     VkQueuePtr       mQueue;
     VkCommandPoolPtr mTransientPool;
+    std::mutex       mQueueMutex;
+    std::mutex       mCommandPoolMutex;
 };
 
 } // namespace vk

include/ppx/grfx/vk/vk_sync.h

@@ -59,12 +59,35 @@ class Semaphore
 
     VkSemaphorePtr GetVkSemaphore() const { return mSemaphore; }
 
+private:
+    virtual Result   TimelineWait(uint64_t value, uint64_t timeout) const override;
+    virtual Result   TimelineSignal(uint64_t value) const override;
+    virtual uint64_t TimelineCounterValue() const override;
+
 protected:
     virtual Result CreateApiObjects(const grfx::SemaphoreCreateInfo* pCreateInfo) override;
     virtual void   DestroyApiObjects() override;
 
 private:
-    VkSemaphorePtr mSemaphore;
+    //
+    // Why are we storing timeline semaphore signaled values?
+    //
+    // Direct3D allows fence objects to signal a value if the value is
+    // equal to or greater than what's already been signaled.
+    //
+    // Vulkan does not:
+    //   https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/VkSemaphoreSignalInfo.html#VUID-VkSemaphoreSignalInfo-value-03258
+    //
+    // This is unfortunate, because there are cases where an application
+    // may need to signal a value that is equal to what's been signaled.
+    //
+    // Even though it's possible to get the current value, add 1 to it,
+    // and then signal it - this can create a different problem where a value
+    // is signaled too soon and a write-after-read hazard
+    // possibly gets introduced.
+    //
+    mutable uint64_t mTimelineSignaledValue = 0;
+    VkSemaphorePtr   mSemaphore;
 };
 
 } // namespace vk

projects/CMakeLists.txt

@@ -49,6 +49,7 @@ add_subdirectory(alloc)
 add_subdirectory(fishtornado)
 add_subdirectory(fluid_simulation)
 add_subdirectory(oit_demo)
+add_subdirectory(timeline_semaphore)
 
 if (PPX_BUILD_XR)
 add_subdirectory(cube_xr)

projects/timeline_semaphore/CMakeLists.txt

@@ -0,0 +1,22 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+cmake_minimum_required(VERSION 3.0 FATAL_ERROR)
+
+project(timeline_semaphore)
+
+add_samples_for_all_apis(
+    NAME ${PROJECT_NAME}
+    SOURCES "main.cpp"
+    SHADER_DEPENDENCIES
+    "shader_text_draw")

projects/timeline_semaphore/README.md

@@ -0,0 +1,9 @@
+# Timeline semaphore
+
+Example of how to use timeline semaphores.
+
+## Shaders
+
+Shader          | Purpose for this project
+--------------- | ----------------------------
+`TextDraw.hlsl` | Draw colored text.