D3D9: Handle binding mismatching texture types

src/d3d9/d3d9_common_texture.h

@@ -319,7 +319,7 @@ namespace dxvk {
       return std::exchange(m_transitionedToHazardLayout, true);
     }
 
-    D3DRESOURCETYPE GetType() {
+    D3DRESOURCETYPE GetType() const {
       return m_type;
     }
 
@@ -441,6 +441,10 @@ namespace dxvk {
     static VkImageType GetImageTypeFromResourceType(
             D3DRESOURCETYPE  Dimension);
 
+    static VkImageViewType GetImageViewTypeFromResourceType(
+            D3DRESOURCETYPE  Dimension,
+            UINT             Layer);
+
      /**
      * \brief Tracks sequence number for a given subresource
      *
@@ -553,10 +557,6 @@ namespace dxvk {
 
     void ExportImageInfo();
 
-    static VkImageViewType GetImageViewTypeFromResourceType(
-            D3DRESOURCETYPE  Dimension,
-            UINT             Layer);
-
   };
 
 }

src/d3d9/d3d9_device.cpp

@@ -3372,10 +3372,17 @@ namespace dxvk {
 
       BindShader<DxsoProgramTypes::VertexShader>(GetCommonShader(shader));
       m_vsShaderMasks = newShader->GetShaderMask();
+
+      UpdateTextureTypeMismatchesForShader(newShader, m_vsShaderMasks.samplerMask, caps::MaxTexturesPS + 1);
     }
-    else
+    else {
       m_vsShaderMasks = D3D9ShaderMasks();
 
+      // Fixed function vertex shaders don't support sampling textures.
+      m_dirtyTextures = m_vsShaderMasks.samplerMask & m_mismatchingTextureTypes;
+      m_mismatchingTextureTypes &= ~m_vsShaderMasks.samplerMask;
+    }
+
     m_flags.set(D3D9DeviceFlag::DirtyInputLayout);
 
     return D3D_OK;
@@ -3730,13 +3737,19 @@ namespace dxvk {
 
       BindShader<DxsoProgramTypes::PixelShader>(newShader);
       newShaderMasks = newShader->GetShaderMask();
+
+      UpdateTextureTypeMismatchesForShader(newShader, newShaderMasks.samplerMask, 0);
     }
     else {
       // TODO: What fixed function textures are in use?
       // Currently we are making all 8 of them as in use here.
 
       // The RT output is always 0 for fixed function.
       newShaderMasks = FixedFunctionMask;
+
+      // Fixed function always uses spec constants to decide the texture type.
+      m_dirtyTextures = newShaderMasks.samplerMask & m_mismatchingTextureTypes;
+      m_mismatchingTextureTypes &= ~newShaderMasks.samplerMask;
     }
 
     // If we have any RTs we would have bound to the the FB
@@ -4341,12 +4354,12 @@ namespace dxvk {
       const uint32_t textureType = newTexture != nullptr
         ? uint32_t(newTexture->GetType() - D3DRTYPE_TEXTURE)
         : 0;
-      // There are 4 texture types, so we need 2 bits.
+      // There are 3 texture types, so we need 2 bits.
       const uint32_t offset = StateSampler * 2;
       const uint32_t textureBitMask = 0b11u       << offset;
       const uint32_t textureBits    = textureType << offset;
 
-      // In fixed function shaders and SM < 3 we put the type mask
+      // In fixed function shaders and SM < 2 we put the type mask
       // into a spec constant to select the used sampler type.
       m_textureTypes &= ~textureBitMask;
       m_textureTypes |=  textureBits;
@@ -6123,11 +6136,12 @@ namespace dxvk {
   inline void D3D9DeviceEx::UpdateTextureBitmasks(uint32_t index, DWORD combinedUsage) {
     const uint32_t bit = 1 << index;
 
-    m_activeTextureRTs       &= ~bit;
-    m_activeTextureDSs       &= ~bit;
-    m_activeTextures         &= ~bit;
-    m_activeTexturesToUpload &= ~bit;
-    m_activeTexturesToGen    &= ~bit;
+    m_activeTextureRTs        &= ~bit;
+    m_activeTextureDSs        &= ~bit;
+    m_activeTextures          &= ~bit;
+    m_activeTexturesToUpload  &= ~bit;
+    m_activeTexturesToGen     &= ~bit;
+    m_mismatchingTextureTypes &= ~bit;
 
     auto tex = GetCommonTexture(m_state.textures[index]);
     if (tex != nullptr) {
@@ -6168,6 +6182,8 @@ namespace dxvk {
 
       if (unlikely(m_fetch4Enabled & bit))
         UpdateActiveFetch4(index);
+
+      UpdateTextureTypeMismatchesForTexture(index);
     } else {
       if (unlikely(m_fetch4 & bit))
         UpdateActiveFetch4(index);
@@ -6322,6 +6338,66 @@ namespace dxvk {
   }
 
 
+  void D3D9DeviceEx::UpdateTextureTypeMismatchesForShader(const D3D9CommonShader* shader, uint32_t shaderSamplerMask, uint32_t shaderSamplerOffset) {
+    if (unlikely(shader->GetInfo().majorVersion() < 2)) {
+      // SM 1 shaders don't define the texture type in the shader.
+      // We always use spec constants for those.
+      m_dirtyTextures = shaderSamplerMask & m_mismatchingTextureTypes;
+      m_mismatchingTextureTypes &= ~shaderSamplerMask;
+      return;
+    }
+
+    for (uint32_t i : bit::BitMask(shaderSamplerMask)) {
+      const D3D9CommonTexture* texture = GetCommonTexture(m_state.textures[i]);
+      if (unlikely(texture == nullptr)) {
+        // Unbound textures are not mismatching texture types
+        m_dirtyTextures |= m_mismatchingTextureTypes & (1 << i);
+        m_mismatchingTextureTypes &= ~(1 << i);
+        continue;
+      }
+
+      VkImageViewType boundViewType  = D3D9CommonTexture::GetImageViewTypeFromResourceType(texture->GetType(), D3D9CommonTexture::AllLayers);
+      VkImageViewType shaderViewType = shader->GetImageViewType(i - shaderSamplerOffset);
+      if (unlikely(boundViewType != shaderViewType)) {
+        m_dirtyTextures |= 1 << i;
+        m_mismatchingTextureTypes |= 1 << i;
+      } else {
+        // The texture type is no longer mismatching, make sure we bind the texture now.
+        m_dirtyTextures |= m_mismatchingTextureTypes & (1 << i);
+        m_mismatchingTextureTypes &= ~(1 << i);
+      }
+    }
+  }
+
+
+  void D3D9DeviceEx::UpdateTextureTypeMismatchesForTexture(uint32_t stateSampler) {
+    uint32_t shaderTextureIndex;
+    const D3D9CommonShader* shader;
+    if (unlikely(stateSampler > caps::MaxTexturesPS + 1)) {
+      shader = GetCommonShader(m_state.vertexShader);
+      shaderTextureIndex = stateSampler - caps::MaxTexturesPS - 1;
+    } else {
+      shader = GetCommonShader(m_state.pixelShader);
+      shaderTextureIndex = stateSampler;
+    }
+
+    if (unlikely(shader == nullptr || shader->GetInfo().majorVersion() < 2)) {
+      return;
+    }
+
+    const D3D9CommonTexture* tex = GetCommonTexture(m_state.textures[stateSampler]);
+    VkImageViewType boundViewType  = D3D9CommonTexture::GetImageViewTypeFromResourceType(tex->GetType(), D3D9CommonTexture::AllLayers);
+    VkImageViewType shaderViewType = shader->GetImageViewType(shaderTextureIndex);
+    // D3D9 does not have 1D textures. The value of VIEW_TYPE_1D is 0
+    // which is the default when there is no declaration for the type.
+    bool shaderUsesTexture = shaderViewType != VkImageViewType(0);
+    if (unlikely(boundViewType != shaderViewType && shaderUsesTexture)) {
+      const uint32_t samplerBit = 1u << stateSampler;
+      m_mismatchingTextureTypes |= 1 << samplerBit;
+    }
+  }
+
+
   void D3D9DeviceEx::GenerateTextureMips(uint32_t mask) {
     for (uint32_t texIdx : bit::BitMask(mask)) {
       // Guaranteed to not be nullptr...
@@ -7056,8 +7132,8 @@ namespace dxvk {
 
 
   void D3D9DeviceEx::UndirtyTextures(uint32_t usedMask) {
-    const uint32_t activeMask   = usedMask &  m_activeTextures;
-    const uint32_t inactiveMask = usedMask & ~m_activeTextures;
+    const uint32_t activeMask   = usedMask &  (m_activeTextures & ~m_mismatchingTextureTypes);
+    const uint32_t inactiveMask = usedMask & (~m_activeTextures | m_mismatchingTextureTypes);
 
     for (uint32_t i : bit::BitMask(activeMask))
       BindTexture(i);

src/d3d9/d3d9_device.h

@@ -817,6 +817,10 @@ namespace dxvk {
 
     void UpdateActiveFetch4(uint32_t stateSampler);
 
+    void UpdateTextureTypeMismatchesForShader(const D3D9CommonShader* shader, uint32_t shaderSamplerMask, uint32_t shaderSamplerOffset);
+
+    void UpdateTextureTypeMismatchesForTexture(uint32_t stateSampler);
+
     void UploadManagedTexture(D3D9CommonTexture* pResource);
 
     void UploadManagedTextures(uint32_t mask);
@@ -1461,6 +1465,7 @@ namespace dxvk {
     uint32_t                        m_drefClamp = 0;
     uint32_t                        m_cubeTextures = 0;
     uint32_t                        m_textureTypes = 0;
+    uint32_t                        m_mismatchingTextureTypes = 0;
     uint32_t                        m_projectionBitfield  = 0;
 
     uint32_t                        m_dirtySamplerStates = 0;

src/d3d9/d3d9_shader.cpp

@@ -57,6 +57,7 @@ namespace dxvk {
     m_shader       = pModule->compile(*pDxsoModuleInfo, name, AnalysisInfo, constantLayout);
     m_isgn         = pModule->isgn();
     m_usedSamplers = pModule->usedSamplers();
+    m_textureTypes = pModule->textureTypes();
 
     // Shift up these sampler bits so we can just
     // do an or per-draw in the device.

src/d3d9/d3d9_shader.h

@@ -54,11 +54,18 @@ namespace dxvk {
 
     uint32_t GetMaxDefinedConstant() const { return m_maxDefinedConst; }
 
+    VkImageViewType GetImageViewType(uint32_t samplerSlot) const {
+      const uint32_t offset = samplerSlot * 2;
+      const uint32_t mask = 0b11;
+      return static_cast<VkImageViewType>((m_textureTypes >> offset) & mask);
+    }
+
   private:
 
     DxsoIsgn              m_isgn;
     uint32_t              m_usedSamplers;
     uint32_t              m_usedRTs;
+    uint32_t              m_textureTypes;
 
     DxsoProgramInfo       m_info;
     DxsoShaderMetaInfo    m_meta;

src/dxso/dxso_compiler.cpp

@@ -41,6 +41,7 @@ namespace dxvk {
 
     m_usedSamplers = 0;
     m_usedRTs      = 0;
+    m_textureTypes = 0;
     m_rRegs.reserve(DxsoMaxTempRegs);
 
     for (uint32_t i = 0; i < m_rRegs.size(); i++)
@@ -762,6 +763,10 @@ namespace dxvk {
         // We could also be depth compared!
         DclSampler(idx, binding, samplerType, true, implicit);
       }
+
+      const uint32_t offset = idx * 2;
+      uint32_t textureBits = uint32_t(viewType);
+      m_textureTypes |= textureBits << offset;
     }
     else {
       // Could be any of these!

src/dxso/dxso_compiler.h

@@ -248,6 +248,7 @@ namespace dxvk {
     uint32_t usedSamplers() const { return m_usedSamplers; }
     uint32_t usedRTs() const { return m_usedRTs; }
     uint32_t maxDefinedConstant() const { return m_maxDefinedConstant; }
+    uint32_t textureTypes() const { return m_textureTypes; }
 
   private:
 
@@ -358,6 +359,8 @@ namespace dxvk {
     uint32_t m_usedSamplers;
     uint32_t m_usedRTs;
 
+    uint32_t m_textureTypes;
+
     uint32_t m_specUbo = 0;
 
     uint32_t m_rsBlock = 0;

src/dxso/dxso_module.cpp

@@ -38,6 +38,7 @@ namespace dxvk {
     m_constants       = compiler->constants();
     m_maxDefinedConst = compiler->maxDefinedConstant();
     m_usedSamplers    = compiler->usedSamplers();
+    m_textureTypes    = compiler->textureTypes();
 
     compiler->finalize();
 

src/dxso/dxso_module.h

@@ -61,6 +61,8 @@ namespace dxvk {
 
     uint32_t maxDefinedConstant() { return m_maxDefinedConst; }
 
+    uint32_t textureTypes() { return m_textureTypes; }
+
   private:
 
     void runCompiler(
@@ -77,6 +79,7 @@ namespace dxvk {
     DxsoIsgn        m_isgn;
     uint32_t        m_usedSamplers;
     uint32_t        m_usedRTs;
+    uint32_t        m_textureTypes;
 
     DxsoShaderMetaInfo   m_meta;
     uint32_t             m_maxDefinedConst;

src/util/config/config.cpp

@@ -1007,10 +1007,6 @@ namespace dxvk {
       { "d3d9.customVendorId",              "10de" },
       { "d3d9.cachedDynamicBuffers",        "True" },
     }} },
-    /* Alpha Protocol - Rids unwanted reflections  */
-    { R"(\\APGame\.exe$)", {{
-      { "d3d9.forceSamplerTypeSpecConstants", "True" },
-    }} },
     /* Dark Sector - Crashes in places             */
     { R"(\\DS\.exe$)", {{
       { "d3d9.textureMemory",                "0" },