Add ability to override texture format in ExternalTexture (#1443)

In D3D, some textures have multiple planes (e.g., NV12). In order to access these planes, the format for the D3D shader resource view must be different than the format of the texture. See NV12 section of the [DXGI_FORMAT doc](https://learn.microsoft.com/en-us/windows/win32/api/dxgiformat/ne-dxgiformat-dxgi_format). This change makes it possible to override the texture format to something different than the one from the texture itself when creating the external texture.
BabylonJS · Oct 21, 2024 · df19d3d · df19d3d
1 parent 58adc7d
commit df19d3d
Show file tree

Hide file tree

Showing 10 changed files with 36 additions and 29 deletions.
diff --git a/Core/Graphics/Include/RendererType/D3D11/Babylon/Graphics/RendererType.h b/Core/Graphics/Include/RendererType/D3D11/Babylon/Graphics/RendererType.h
@@ -6,6 +6,7 @@ namespace Babylon::Graphics
 {
     using DeviceT = ID3D11Device*;
     using TextureT = ID3D11Resource*;
+    using TextureFormatT = DXGI_FORMAT;
 
     struct DeviceConfiguration
     {

diff --git a/Core/Graphics/Include/RendererType/D3D12/Babylon/Graphics/RendererType.h b/Core/Graphics/Include/RendererType/D3D12/Babylon/Graphics/RendererType.h
@@ -6,6 +6,7 @@ namespace Babylon::Graphics
 {
     using DeviceT = ID3D12Device*;
     using TextureT = ID3D12Resource*;
+    using TextureFormatT = DXGI_FORMAT;
 
     struct DeviceConfiguration
     {

diff --git a/Core/Graphics/Include/RendererType/Metal/Babylon/Graphics/RendererType.h b/Core/Graphics/Include/RendererType/Metal/Babylon/Graphics/RendererType.h
@@ -6,6 +6,7 @@ namespace Babylon::Graphics
 {
     using DeviceT = id<MTLDevice>;
     using TextureT = id<MTLTexture>;
+    using TextureFormatT = MTLPixelFormat;
 
     struct DeviceConfiguration
     {

diff --git a/Core/Graphics/Include/RendererType/OpenGL/Babylon/Graphics/RendererType.h b/Core/Graphics/Include/RendererType/OpenGL/Babylon/Graphics/RendererType.h
@@ -6,6 +6,7 @@ namespace Babylon::Graphics
 {
     using DeviceT = EGLContext;
     using TextureT = unsigned int;
+    using TextureFormatT = unsigned int;
 
     struct DeviceConfiguration
     {

diff --git a/Plugins/ExternalTexture/Include/Babylon/Plugins/ExternalTexture.h b/Plugins/ExternalTexture/Include/Babylon/Plugins/ExternalTexture.h
@@ -3,14 +3,15 @@
 #include <Babylon/JsRuntime.h>
 #include <Babylon/Graphics/RendererType.h>
 #include <memory>
+#include <optional>
 
 namespace Babylon::Plugins
 {
     class ExternalTexture final
     {
     public:
         // NOTE: Must call from the Graphics thread.
-        ExternalTexture(Graphics::TextureT);
+        ExternalTexture(Graphics::TextureT, std::optional<Graphics::TextureFormatT> = {});
         ~ExternalTexture();
 
         // Copy semantics
@@ -34,7 +35,7 @@ namespace Babylon::Plugins
         // Updates to a new texture.
         // Texture attributes (width, height, format, etc.) must match.
         // NOTE: Must call from the Graphics thread.
-        void Update(Graphics::TextureT);
+        void Update(Graphics::TextureT, std::optional<Graphics::TextureFormatT> = {});
 
     private:
         class Impl;

diff --git a/Plugins/ExternalTexture/Source/ExternalTexture_D3D11.cpp b/Plugins/ExternalTexture/Source/ExternalTexture_D3D11.cpp
@@ -158,16 +158,16 @@ namespace Babylon::Plugins
     {
     public:
         // Implemented in ExternalTexture_Shared.h
-        Impl(Graphics::TextureT);
-        void Update(Graphics::TextureT);
+        Impl(Graphics::TextureT, std::optional<Graphics::TextureFormatT>);
+        void Update(Graphics::TextureT, std::optional<Graphics::TextureFormatT>);
 
         uintptr_t Ptr() const
         {
             return reinterpret_cast<uintptr_t>(m_ptr.get());
         }
 
     private:
-        static void GetInfo(Graphics::TextureT ptr, Info& info)
+        static void GetInfo(Graphics::TextureT ptr, std::optional<Graphics::TextureFormatT> overrideFormat, Info& info)
         {
             winrt::com_ptr<ID3D11Resource> resource;
             resource.copy_from(ptr);
@@ -196,13 +196,14 @@ namespace Babylon::Plugins
                 }
             }
 
+            DXGI_FORMAT targetFormat = overrideFormat.has_value() ? overrideFormat.value() : desc.Format;
             for (int i = 0; i < BX_COUNTOF(s_textureFormat); ++i)
             {
                 const auto& format = s_textureFormat[i];
-                if (format.m_fmt == desc.Format || format.m_fmtSrgb == desc.Format)
+                if (format.m_fmt == targetFormat || format.m_fmtSrgb == targetFormat)
                 {
                     info.Format = static_cast<bgfx::TextureFormat::Enum>(i);
-                    if (format.m_fmtSrgb == desc.Format)
+                    if (format.m_fmtSrgb == targetFormat)
                     {
                         info.Flags |= BGFX_TEXTURE_SRGB;
                     }

diff --git a/Plugins/ExternalTexture/Source/ExternalTexture_D3D12.cpp b/Plugins/ExternalTexture/Source/ExternalTexture_D3D12.cpp
@@ -158,16 +158,16 @@ namespace Babylon::Plugins
     {
     public:
         // Implemented in ExternalTexture_Shared.h
-        Impl(Graphics::TextureT);
-        void Update(Graphics::TextureT);
+        Impl(Graphics::TextureT, std::optional<Graphics::TextureFormatT>);
+        void Update(Graphics::TextureT, std::optional<Graphics::TextureFormatT>);
 
         uintptr_t Ptr() const
         {
             return reinterpret_cast<uintptr_t>(m_ptr.get());
         }
 
     private:
-        static void GetInfo(Graphics::TextureT ptr, Info& info)
+        static void GetInfo(Graphics::TextureT ptr, std::optional<Graphics::TextureFormatT> overrideFormat, Info& info)
         {
             D3D12_RESOURCE_DESC desc = ptr->GetDesc();
 
@@ -190,13 +190,14 @@ namespace Babylon::Plugins
                 }
             }
 
+            DXGI_FORMAT targetFormat = overrideFormat.has_value() ? overrideFormat.value() : desc.Format;
             for (int i = 0; i < BX_COUNTOF(s_textureFormat); ++i)
             {
                 const auto& format = s_textureFormat[i];
-                if (format.m_fmt == desc.Format || format.m_fmtSrgb == desc.Format)
+                if (format.m_fmt == targetFormat || format.m_fmtSrgb == targetFormat)
                 {
                     info.Format = static_cast<bgfx::TextureFormat::Enum>(i);
-                    if (format.m_fmtSrgb == desc.Format)
+                    if (format.m_fmtSrgb == targetFormat)
                     {
                         info.Flags |= BGFX_TEXTURE_SRGB;
                     }

diff --git a/Plugins/ExternalTexture/Source/ExternalTexture_Metal.mm b/Plugins/ExternalTexture/Source/ExternalTexture_Metal.mm
@@ -144,16 +144,16 @@
     {
     public:
         // Implemented in ExternalTexture_Shared.h
-        Impl(Graphics::TextureT);
-        void Update(Graphics::TextureT);
+        Impl(Graphics::TextureT, std::optional<Graphics::TextureFormatT>);
+        void Update(Graphics::TextureT, std::optional<Graphics::TextureFormatT>);
 
         uintptr_t Ptr() const
         {
             return reinterpret_cast<uintptr_t>(m_ptr);
         }
 
     private:
-        static void GetInfo(Graphics::TextureT ptr, Info& info)
+        static void GetInfo(Graphics::TextureT ptr, std::optional<Graphics::TextureFormatT> overrideFormat, Info& info)
         {
             if (ptr.textureType != MTLTextureType2D && ptr.textureType != MTLTextureType2DMultisample)
             {
@@ -174,14 +174,14 @@ static void GetInfo(Graphics::TextureT ptr, Info& info)
                 }
             }
 
-            const auto pixelFormat = ptr.pixelFormat;
+            const auto targetFormat = overrideFormat.has_value() ? overrideFormat.value() : ptr.pixelFormat;
             for (size_t i = 0; i < BX_COUNTOF(s_textureFormat); ++i)
             {
                 const auto& format = s_textureFormat[i];
-                if (format.m_fmt == pixelFormat || format.m_fmtSrgb == pixelFormat)
+                if (format.m_fmt == targetFormat || format.m_fmtSrgb == targetFormat)
                 {
                     info.Format = static_cast<bgfx::TextureFormat::Enum>(i);
-                    if (format.m_fmtSrgb == pixelFormat)
+                    if (format.m_fmtSrgb == targetFormat)
                     {
                         info.Flags |= BGFX_TEXTURE_SRGB;
                     }

diff --git a/Plugins/ExternalTexture/Source/ExternalTexture_OpenGL.cpp b/Plugins/ExternalTexture/Source/ExternalTexture_OpenGL.cpp
@@ -15,16 +15,16 @@ namespace Babylon::Plugins
     {
     public:
         // Implemented in ExternalTexture_Shared.h
-        Impl(Graphics::TextureT);
-        void Update(Graphics::TextureT);
+        Impl(Graphics::TextureT, std::optional<Graphics::TextureFormatT>);
+        void Update(Graphics::TextureT, std::optional<Graphics::TextureFormatT>);
 
         uintptr_t Ptr() const
         {
             throw std::runtime_error{"not implemented"};
         }
 
     private:
-        void GetInfo(Graphics::TextureT, Info&)
+        static void GetInfo(Graphics::TextureT, std::optional<Graphics::TextureFormatT>, Info&)
         {
             throw std::runtime_error{"not implemented"};
         }

diff --git a/Plugins/ExternalTexture/Source/ExternalTexture_Shared.h b/Plugins/ExternalTexture/Source/ExternalTexture_Shared.h
@@ -2,9 +2,9 @@
 
 namespace Babylon::Plugins
 {
-    ExternalTexture::Impl::Impl(Graphics::TextureT ptr)
+    ExternalTexture::Impl::Impl(Graphics::TextureT ptr, std::optional<Graphics::TextureFormatT> overrideFormat)
     {
-        GetInfo(ptr, m_info);
+        GetInfo(ptr, overrideFormat, m_info);
 
         if (m_info.MipLevels != 1 && m_info.MipLevels != 0 && !IsFullMipChain(m_info.MipLevels, m_info.Width, m_info.Height))
         {
@@ -14,10 +14,10 @@ namespace Babylon::Plugins
         Assign(ptr);
     }
 
-    void ExternalTexture::Impl::Update(Graphics::TextureT ptr)
+    void ExternalTexture::Impl::Update(Graphics::TextureT ptr, std::optional<Graphics::TextureFormatT> overrideFormat)
     {
         Info info;
-        GetInfo(ptr, info);
+        GetInfo(ptr, overrideFormat, info);
 
         if (info.Width != m_info.Width || info.Height != m_info.Height || info.MipLevels != m_info.MipLevels)
         {
@@ -33,8 +33,8 @@ namespace Babylon::Plugins
         UpdateHandles(Ptr());
     }
 
-    ExternalTexture::ExternalTexture(Graphics::TextureT ptr)
-        : m_impl{std::make_unique<Impl>(ptr)}
+    ExternalTexture::ExternalTexture(Graphics::TextureT ptr, std::optional<Graphics::TextureFormatT> overrideFormat)
+        : m_impl{std::make_unique<Impl>(ptr, overrideFormat)}
     {
     }
 
@@ -119,8 +119,8 @@ namespace Babylon::Plugins
         return promise;
     }
 
-    void ExternalTexture::Update(Graphics::TextureT ptr)
+    void ExternalTexture::Update(Graphics::TextureT ptr, std::optional<Graphics::TextureFormatT> overrideFormat)
     {
-        m_impl->Update(ptr);
+        m_impl->Update(ptr, overrideFormat);
     }
 }