Add support for VK_KHR_sampler_ycbcr_conversion

This adds ability for applications to import Android Hardware Buffers (AHBs) as OpenGL images which in turn can be sampled from and/or written. This was specifically tested with the common use case of importing a buffer created by an media decoder and using that as a texture source to include that video content on the screen. Tested with: - Angry Birds 2 video player (for ads) requires YUV conversion. - Basic Media Decoder example: https://github.com/android/media-samples/tree/master/BasicMediaDecoder Bug: b/155487768 Change-Id: I9255450f81aa4daa2aace7205d4f6c3f225abcca Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/2175103 Commit-Queue: Courtney Goeltzenleuchter <courtneygo@google.com> Reviewed-by: Courtney Goeltzenleuchter <courtneygo@google.com> Reviewed-by: Tim Van Patten <timvp@google.com> Reviewed-by: Jamie Madill <jmadill@chromium.org>
flutter · Jul 9, 2020 · f61272f · f61272f
1 parent b549be9
commit f61272f
Show file tree

Hide file tree

Showing 26 changed files with 448 additions and 69 deletions.
diff --git a/include/platform/FeaturesVk.h b/include/platform/FeaturesVk.h
@@ -122,11 +122,11 @@ struct FeaturesVk : FeatureSetBase
 
     // Whether the VkDevice supports the VK_FUCHSIA_external_memory
     // extension, on which the GL_ANGLE_memory_object_fuchsia extension can be layered.
-    angle::Feature supportsExternalMemoryFuchsia = {
+    Feature supportsExternalMemoryFuchsia = {
         "supports_external_memory_fuchsia", FeatureCategory::VulkanFeatures,
         "VkDevice supports the VK_FUCHSIA_external_memory extension", &members};
 
-    angle::Feature supportsFilteringPrecision = {
+    Feature supportsFilteringPrecision = {
         "supports_filtering_precision_google", FeatureCategory::VulkanFeatures,
         "VkDevice supports the VK_GOOGLE_sampler_filtering_precision extension", &members};
 
@@ -172,6 +172,12 @@ struct FeaturesVk : FeatureSetBase
         "supports_shader_stencil_export", FeatureCategory::VulkanFeatures,
         "VkDevice supports the VK_EXT_shader_stencil_export extension", &members};
 
+    // Whether the VkDevice supports the VK_KHR_sampler_ycbcr_conversion extension, which is needed
+    // to support Ycbcr conversion with external images.
+    Feature supportsYUVSamplerConversion = {
+        "supports_yuv_sampler_conversion", FeatureCategory::VulkanFeatures,
+        "VkDevice supports the VK_KHR_sampler_ycbcr_conversion extension", &members};
+
     // Where VK_EXT_transform_feedback is not support, an emulation path is used.
     // http://anglebug.com/3205
     Feature emulateTransformFeedback = {

diff --git a/scripts/code_generation_hashes/Vulkan_format.json b/scripts/code_generation_hashes/Vulkan_format.json
@@ -4,9 +4,9 @@
   "src/libANGLE/renderer/angle_format_map.json":
     "c79d833aea7007c7d0d51cdaa9b265a6",
   "src/libANGLE/renderer/vulkan/gen_vk_format_table.py":
-    "d8a0f2278c09a49049a73930b9da3719",
+    "54a7374f93f17da1386600027acca7a3",
   "src/libANGLE/renderer/vulkan/vk_format_map.json":
     "738c8dc36fbe212669944e88ae918f9c",
   "src/libANGLE/renderer/vulkan/vk_format_table_autogen.cpp":
-    "383749c786e957bdbbbce9d8b5c2441a"
+    "c7e01cbf1eded87f4ed9f3bc8b7a567c"
 }
diff --git a/src/common/vulkan/vk_headers.h b/src/common/vulkan/vk_headers.h
@@ -64,6 +64,10 @@ extern PFN_vkImportFenceFdKHR vkImportFenceFdKHR;
 extern PFN_vkGetPhysicalDeviceExternalSemaphorePropertiesKHR
     vkGetPhysicalDeviceExternalSemaphorePropertiesKHR;
 
+// VK_KHR_sampler_ycbcr_conversion
+extern PFN_vkCreateSamplerYcbcrConversionKHR vkCreateSamplerYcbcrConversionKHR;
+extern PFN_vkDestroySamplerYcbcrConversionKHR vkDestroySamplerYcbcrConversionKHR;
+
 #    if defined(ANGLE_PLATFORM_FUCHSIA)
 // VK_FUCHSIA_imagepipe_surface
 extern PFN_vkCreateImagePipeSurfaceFUCHSIA vkCreateImagePipeSurfaceFUCHSIA;

diff --git a/src/libANGLE/renderer/vulkan/ContextVk.cpp b/src/libANGLE/renderer/vulkan/ContextVk.cpp
@@ -3748,6 +3748,7 @@ angle::Result ContextVk::updateActiveTextures(const gl::Context *context)
     const gl::ActiveTextureMask &activeTextures    = executable->getActiveSamplersMask();
     const gl::ActiveTextureTypeArray &textureTypes = executable->getActiveSamplerTypes();
 
+    bool haveImmutableSampler = false;
     for (size_t textureUnit : activeTextures)
     {
         gl::Texture *texture        = textures[textureUnit];
@@ -3776,13 +3777,24 @@ angle::Result ContextVk::updateActiveTextures(const gl::Context *context)
             samplerSerial = samplerVk->getSerial();
         }
 
+        if (textureVk->getImage().hasImmutableSampler())
+        {
+            haveImmutableSampler = true;
+        }
+
         mActiveTextures[textureUnit].texture = textureVk;
         mActiveTextures[textureUnit].sampler = samplerVk;
         // Cache serials from sampler and texture, but re-use texture if no sampler bound
         ASSERT(textureVk != nullptr);
         mActiveTexturesDesc.update(textureUnit, textureVk->getSerial(), samplerSerial);
     }
 
+    if (haveImmutableSampler)
+    {
+        ANGLE_TRY(mExecutable->updatePipelineLayout(context, &mActiveTextures));
+        invalidateCurrentGraphicsPipeline();
+    }
+
     return angle::Result::Continue;
 }
 

diff --git a/src/libANGLE/renderer/vulkan/ImageVk.cpp b/src/libANGLE/renderer/vulkan/ImageVk.cpp
@@ -97,9 +97,13 @@ egl::Error ImageVk::initialize(const egl::Display *display)
             return egl::EglBadAccess();
         }
 
+        // start with some reasonable alignment that's safe for the case where intendedFormatID is
+        // FormatID::NONE
+        size_t alignment = mImage->getFormat().getValidImageCopyBufferAlignment();
+
         // Make sure a staging buffer is ready to use to upload data
-        mImage->initStagingBuffer(renderer, mImage->getFormat().getImageCopyBufferAlignment(),
-                                  vk::kStagingBufferFlags, vk::kStagingBufferSize);
+        mImage->initStagingBuffer(renderer, alignment, vk::kStagingBufferFlags,
+                                  vk::kStagingBufferSize);
 
         mOwnsImage = false;
 

diff --git a/src/libANGLE/renderer/vulkan/MemoryObjectVk.cpp b/src/libANGLE/renderer/vulkan/MemoryObjectVk.cpp
@@ -245,7 +245,7 @@ angle::Result MemoryObjectVk::createImage(ContextVk *contextVk,
 
     VkMemoryPropertyFlags flags = 0;
     ANGLE_TRY(image->initExternalMemory(contextVk, renderer->getMemoryProperties(),
-                                        externalMemoryRequirements, importMemoryInfo,
+                                        externalMemoryRequirements, nullptr, importMemoryInfo,
                                         renderer->getQueueFamilyIndex(), flags));
 
     return angle::Result::Continue;

diff --git a/src/libANGLE/renderer/vulkan/ProgramExecutableVk.cpp b/src/libANGLE/renderer/vulkan/ProgramExecutableVk.cpp
@@ -446,9 +446,11 @@ void ProgramExecutableVk::addImageDescriptorSetDesc(const gl::ProgramExecutable
     }
 }
 
-void ProgramExecutableVk::addTextureDescriptorSetDesc(const gl::ProgramState &programState,
-                                                      bool useOldRewriteStructSamplers,
-                                                      vk::DescriptorSetLayoutDesc *descOut)
+void ProgramExecutableVk::addTextureDescriptorSetDesc(
+    const gl::ProgramState &programState,
+    bool useOldRewriteStructSamplers,
+    const gl::ActiveTextureArray<vk::TextureUnit> *activeTextures,
+    vk::DescriptorSetLayoutDesc *descOut)
 {
     const std::vector<gl::SamplerBinding> &samplerBindings = programState.getSamplerBindings();
     const std::vector<gl::LinkedUniform> &uniforms         = programState.getUniforms();
@@ -493,8 +495,25 @@ void ProgramExecutableVk::addTextureDescriptorSetDesc(const gl::ProgramState &pr
             ShaderInterfaceVariableInfo &info = mVariableInfoMap[shaderType][samplerName];
             VkShaderStageFlags activeStages   = gl_vk::kShaderStageMap[shaderType];
 
-            descOut->update(info.binding, VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, arraySize,
-                            activeStages, nullptr);
+            // TODO: https://issuetracker.google.com/issues/158215272: how do we handle array of
+            // immutable samplers?
+            GLuint textureUnit = samplerBinding.boundTextureUnits[0];
+            if (activeTextures &&
+                ((*activeTextures)[textureUnit].texture->getImage().hasImmutableSampler()))
+            {
+                ASSERT(samplerBinding.boundTextureUnits.size() == 1);
+                // Always take the texture's sampler, that's only way to get to yuv conversion for
+                // externalFormat
+                const vk::Sampler &immutableSampler =
+                    (*activeTextures)[textureUnit].texture->getSampler();
+                descOut->update(info.binding, VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, arraySize,
+                                activeStages, &immutableSampler);
+            }
+            else
+            {
+                descOut->update(info.binding, VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, arraySize,
+                                activeStages, nullptr);
+            }
         }
     }
 }
@@ -610,7 +629,11 @@ angle::Result ProgramExecutableVk::getComputePipeline(ContextVk *contextVk,
     return shaderProgram->getComputePipeline(contextVk, getPipelineLayout(), pipelineOut);
 }
 
-angle::Result ProgramExecutableVk::createPipelineLayout(const gl::Context *glContext)
+// updatePipelineLayout is used to create the DescriptorSetLayout(s) and PipelineLayout and update
+// them when we discover that an immutable sampler is in use.
+angle::Result ProgramExecutableVk::updatePipelineLayout(
+    const gl::Context *glContext,
+    gl::ActiveTextureArray<vk::TextureUnit> *activeTextures)
 {
     const gl::State &glState                   = glContext->getState();
     ContextVk *contextVk                       = vk::GetImpl(glContext);
@@ -621,13 +644,12 @@ angle::Result ProgramExecutableVk::createPipelineLayout(const gl::Context *glCon
     gl::ShaderMap<const gl::ProgramState *> programStates;
     fillProgramStateMap(contextVk, &programStates);
 
-    reset(contextVk);
-
     // Store a reference to the pipeline and descriptor set layouts. This will create them if they
     // don't already exist in the cache.
 
     // Default uniforms and transform feedback:
     vk::DescriptorSetLayoutDesc uniformsAndXfbSetDesc;
+    mNumDefaultUniformDescriptors = 0;
     for (const gl::ShaderType shaderType : linkedShaderStages)
     {
         const std::string uniformBlockName = kDefaultUniformNames[shaderType];
@@ -693,7 +715,7 @@ angle::Result ProgramExecutableVk::createPipelineLayout(const gl::Context *glCon
         const gl::ProgramState *programState = programStates[shaderType];
         ASSERT(programState);
         addTextureDescriptorSetDesc(*programState, contextVk->useOldRewriteStructSamplers(),
-                                    &texturesSetDesc);
+                                    activeTextures, &texturesSetDesc);
     }
 
     ANGLE_TRY(renderer->getDescriptorSetLayout(contextVk, texturesSetDesc,
@@ -721,6 +743,22 @@ angle::Result ProgramExecutableVk::createPipelineLayout(const gl::Context *glCon
     ANGLE_TRY(renderer->getPipelineLayout(contextVk, pipelineLayoutDesc, mDescriptorSetLayouts,
                                           &mPipelineLayout));
 
+    return angle::Result::Continue;
+}
+
+angle::Result ProgramExecutableVk::createPipelineLayout(const gl::Context *glContext)
+{
+    ContextVk *contextVk                       = vk::GetImpl(glContext);
+    RendererVk *renderer                       = contextVk->getRenderer();
+    const gl::ProgramExecutable &glExecutable  = getGlExecutable();
+    const gl::ShaderBitSet &linkedShaderStages = glExecutable.getLinkedShaderStages();
+    gl::ShaderMap<const gl::ProgramState *> programStates;
+    fillProgramStateMap(contextVk, &programStates);
+
+    reset(contextVk);
+
+    ANGLE_TRY(updatePipelineLayout(glContext, nullptr));
+
     // Initialize descriptor pools.
     std::array<VkDescriptorPoolSize, 2> uniformAndXfbSetSize = {
         {{VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC,
@@ -1302,6 +1340,10 @@ angle::Result ProgramExecutableVk::updateTexturesDescriptorSet(ContextVk *contex
                         textureVk->getReadImageViewAndRecordUse(contextVk).getHandle();
                 }
 
+                if (textureVk->getImage().hasImmutableSampler())
+                {
+                    imageInfos[arrayElement].sampler = textureVk->getSampler().getHandle();
+                }
                 ShaderInterfaceVariableInfoMap &variableInfoMap = mVariableInfoMap[shaderType];
                 const std::string samplerName =
                     contextVk->getRenderer()->getFeatures().forceOldRewriteStructSamplers.enabled

diff --git a/src/libANGLE/renderer/vulkan/ProgramExecutableVk.h b/src/libANGLE/renderer/vulkan/ProgramExecutableVk.h
@@ -137,6 +137,8 @@ class ProgramExecutableVk
 
     const vk::PipelineLayout &getPipelineLayout() const { return mPipelineLayout.get(); }
     angle::Result createPipelineLayout(const gl::Context *glContext);
+    angle::Result updatePipelineLayout(const gl::Context *glContext,
+                                       gl::ActiveTextureArray<vk::TextureUnit> *activeTextures);
 
     angle::Result updateTexturesDescriptorSet(ContextVk *contextVk);
     angle::Result updateShaderResourcesDescriptorSet(ContextVk *contextVk,
@@ -182,6 +184,7 @@ class ProgramExecutableVk
                                    vk::DescriptorSetLayoutDesc *descOut);
     void addTextureDescriptorSetDesc(const gl::ProgramState &programState,
                                      bool useOldRewriteStructSamplers,
+                                     const gl::ActiveTextureArray<vk::TextureUnit> *activeTextures,
                                      vk::DescriptorSetLayoutDesc *descOut);
 
     void updateDefaultUniformsDescriptorSet(

diff --git a/src/libANGLE/renderer/vulkan/RendererVk.cpp b/src/libANGLE/renderer/vulkan/RendererVk.cpp
@@ -492,6 +492,7 @@ void RendererVk::onDestroy()
 
     mPipelineCache.destroy(mDevice);
     mSamplerCache.destroy(this);
+    mYuvConversionCache.destroy(this);
     mTheNullBuffer.destroy(this);
 
     mAllocator.destroy();
@@ -886,6 +887,10 @@ void RendererVk::queryDeviceExtensionFeatures(const ExtensionNameList &deviceExt
     mExternalSemaphoreProperties       = {};
     mExternalSemaphoreProperties.sType = VK_STRUCTURE_TYPE_EXTERNAL_SEMAPHORE_PROPERTIES;
 
+    mSamplerYcbcrConversionFeatures = {};
+    mSamplerYcbcrConversionFeatures.sType =
+        VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SAMPLER_YCBCR_CONVERSION_FEATURES;
+
     if (!vkGetPhysicalDeviceProperties2KHR || !vkGetPhysicalDeviceFeatures2KHR)
     {
         return;
@@ -935,6 +940,12 @@ void RendererVk::queryDeviceExtensionFeatures(const ExtensionNameList &deviceExt
         vk::AddToPNextChain(&deviceProperties, &mExternalMemoryHostProperties);
     }
 
+    // Query Ycbcr conversion properties
+    if (ExtensionFound(VK_KHR_SAMPLER_YCBCR_CONVERSION_EXTENSION_NAME, deviceExtensionNames))
+    {
+        vk::AddToPNextChain(&deviceFeatures, &mSamplerYcbcrConversionFeatures);
+    }
+
     // Query subgroup properties
     vk::AddToPNextChain(&deviceProperties, &mSubgroupProperties);
 
@@ -972,6 +983,13 @@ void RendererVk::queryDeviceExtensionFeatures(const ExtensionNameList &deviceExt
     mIndexTypeUint8Features.pNext           = nullptr;
     mSubgroupProperties.pNext               = nullptr;
     mExternalMemoryHostProperties.pNext     = nullptr;
+    mLineRasterizationFeatures.pNext        = nullptr;
+    mProvokingVertexFeatures.pNext          = nullptr;
+    mVertexAttributeDivisorFeatures.pNext   = nullptr;
+    mVertexAttributeDivisorProperties.pNext = nullptr;
+    mTransformFeedbackFeatures.pNext        = nullptr;
+    mIndexTypeUint8Features.pNext           = nullptr;
+    mSamplerYcbcrConversionFeatures.pNext   = nullptr;
 }
 
 angle::Result RendererVk::initializeDevice(DisplayVk *displayVk, uint32_t queueFamilyIndex)
@@ -1274,6 +1292,12 @@ angle::Result RendererVk::initializeDevice(DisplayVk *displayVk, uint32_t queueF
 #endif  // !defined(ANGLE_SHARED_LIBVULKAN)
     }
 
+    if (getFeatures().supportsYUVSamplerConversion.enabled)
+    {
+        enabledDeviceExtensions.push_back(VK_KHR_SAMPLER_YCBCR_CONVERSION_EXTENSION_NAME);
+        vk::AddToPNextChain(&createInfo, &mSamplerYcbcrConversionFeatures);
+    }
+
     createInfo.sType                 = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO;
     createInfo.flags                 = 0;
     createInfo.queueCreateInfoCount  = 1;
@@ -1325,6 +1349,10 @@ angle::Result RendererVk::initializeDevice(DisplayVk *displayVk, uint32_t queueF
     {
         InitTransformFeedbackEXTFunctions(mDevice);
     }
+    if (getFeatures().supportsYUVSamplerConversion.enabled)
+    {
+        InitSamplerYcbcrKHRFunctions(mDevice);
+    }
 #endif  // !defined(ANGLE_SHARED_LIBVULKAN)
 
     // Initialize the vulkan pipeline cache.
@@ -1730,6 +1758,9 @@ void RendererVk::initFeatures(DisplayVk *displayVk, const ExtensionNameList &dev
     // Currently disabled by default: http://anglebug.com/4324
     ANGLE_FEATURE_CONDITION(&mFeatures, enableCommandProcessingThread, false);
 
+    ANGLE_FEATURE_CONDITION(&mFeatures, supportsYUVSamplerConversion,
+                            mSamplerYcbcrConversionFeatures.samplerYcbcrConversion != VK_FALSE);
+
     angle::PlatformMethods *platform = ANGLEPlatformCurrent();
     platform->overrideFeaturesVk(platform, &mFeatures);
 

diff --git a/src/libANGLE/renderer/vulkan/RendererVk.h b/src/libANGLE/renderer/vulkan/RendererVk.h
@@ -247,6 +247,7 @@ class RendererVk : angle::NonCopyable
     bool enableDebugUtils() const { return mEnableDebugUtils; }
 
     SamplerCache &getSamplerCache() { return mSamplerCache; }
+    SamplerYcbcrConversionCache &getYuvConversionCache() { return mYuvConversionCache; }
     vk::ActiveHandleCounter &getActiveHandleCounts() { return mActiveHandleCounts; }
 
     // Queue commands to worker thread for processing
@@ -309,6 +310,7 @@ class RendererVk : angle::NonCopyable
     VkPhysicalDeviceExternalMemoryHostPropertiesEXT mExternalMemoryHostProperties;
     VkExternalFenceProperties mExternalFenceProperties;
     VkExternalSemaphoreProperties mExternalSemaphoreProperties;
+    VkPhysicalDeviceSamplerYcbcrConversionFeatures mSamplerYcbcrConversionFeatures;
     std::vector<VkQueueFamilyProperties> mQueueFamilyProperties;
     std::mutex mQueueMutex;
     angle::PackedEnumMap<egl::ContextPriority, VkQueue> mQueues;
@@ -382,6 +384,7 @@ class RendererVk : angle::NonCopyable
 
     vk::Allocator mAllocator;
     SamplerCache mSamplerCache;
+    SamplerYcbcrConversionCache mYuvConversionCache;
     vk::ActiveHandleCounter mActiveHandleCounts;
 
     // Vulkan does not allow binding a null vertex buffer. We use a dummy as a placeholder.

diff --git a/src/libANGLE/renderer/vulkan/SamplerVk.cpp b/src/libANGLE/renderer/vulkan/SamplerVk.cpp
@@ -39,7 +39,7 @@ angle::Result SamplerVk::syncState(const gl::Context *context, const bool dirty)
         mSampler.reset();
     }
 
-    vk::SamplerDesc desc(mState, false);
+    vk::SamplerDesc desc(mState, false, 0);
     ANGLE_TRY(renderer->getSamplerCache().getSampler(contextVk, desc, &mSampler));
 
     // Regenerate the serial on a sampler change.

diff --git a/src/libANGLE/renderer/vulkan/SurfaceVk.cpp b/src/libANGLE/renderer/vulkan/SurfaceVk.cpp
@@ -210,9 +210,9 @@ angle::Result OffscreenSurfaceVk::AttachmentImage::initializeWithExternalMemory(
     image.getImage().getMemoryRequirements(renderer->getDevice(), &externalMemoryRequirements);
 
     VkMemoryPropertyFlags flags = 0;
-    ANGLE_TRY(image.initExternalMemory(displayVk, renderer->getMemoryProperties(),
-                                       externalMemoryRequirements, &importMemoryHostPointerInfo,
-                                       VK_QUEUE_FAMILY_EXTERNAL, flags));
+    ANGLE_TRY(image.initExternalMemory(
+        displayVk, renderer->getMemoryProperties(), externalMemoryRequirements, nullptr,
+        &importMemoryHostPointerInfo, VK_QUEUE_FAMILY_EXTERNAL, flags));
 
     return angle::Result::Continue;
 }

diff --git a/src/libANGLE/renderer/vulkan/TextureVk.cpp b/src/libANGLE/renderer/vulkan/TextureVk.cpp
@@ -1884,7 +1884,8 @@ angle::Result TextureVk::syncState(const gl::Context *context,
                                  mImage->getLevelCount(), layerCount));
     }
 
-    vk::SamplerDesc samplerDesc(mState.getSamplerState(), mState.isStencilMode());
+    vk::SamplerDesc samplerDesc(mState.getSamplerState(), mState.isStencilMode(),
+                                mImage->getExternalFormat());
     ANGLE_TRY(renderer->getSamplerCache().getSampler(contextVk, samplerDesc, &mSampler));
 
     // Regenerate the serial on a sampler change.

diff --git a/src/libANGLE/renderer/vulkan/TextureVk.h b/src/libANGLE/renderer/vulkan/TextureVk.h
@@ -413,7 +413,7 @@ class TextureVk : public TextureImpl, public angle::ObserverInterface
     // reallocated independently of |mImage| on state changes.
     vk::ImageViewHelper mImageViews;
 
-    // |mSampler| contains the relevant Vulkan sampler states reprensenting the OpenGL Texture
+    // |mSampler| contains the relevant Vulkan sampler states representing the OpenGL Texture
     // sampling states for the Texture.
     vk::BindingPointer<vk::Sampler> mSampler;