Merge pull request #2033 from Autodesk/vlasovi/MAYA-106075

MAYA-106075 - USD selection highlighting doesn't take Color Settings into consideration

lib/mayaUsd/render/vp2RenderDelegate/basisCurves.cpp

@@ -1206,9 +1206,10 @@ void HdVP2BasisCurves::_UpdateDrawItem(
             // it needs to display both wireframe color and selection highlight
             // with one color vertex buffer.
             if (drawItem->ContainsUsage(HdVP2DrawItem::kSelectionHighlight)) {
-                const MColor colors[] = { drawScene.GetWireframeColor(),
-                                          drawScene.GetSelectionHighlightColor(false),
-                                          drawScene.GetSelectionHighlightColor(true) };
+                const MColor colors[]
+                    = { drawScene.GetWireframeColor(),
+                        drawScene.GetSelectionHighlightColor(HdPrimTypeTokens->basisCurves),
+                        drawScene.GetSelectionHighlightColor() };
 
                 // Store the indices to colors.
                 std::vector<unsigned char> colorIndices;
@@ -1264,9 +1265,10 @@ void HdVP2BasisCurves::_UpdateDrawItem(
                 int  primitiveStride = 0;
 
                 const MColor& color
-                    = (_selectionStatus != kUnselected
-                           ? drawScene.GetSelectionHighlightColor(_selectionStatus == kFullyLead)
-                           : drawScene.GetWireframeColor());
+                    = (_selectionStatus != kUnselected ? drawScene.GetSelectionHighlightColor(
+                           _selectionStatus == kFullyLead ? TfToken()
+                                                          : HdPrimTypeTokens->basisCurves)
+                                                       : drawScene.GetWireframeColor());
 
                 if (desc.geomStyle == HdBasisCurvesGeomStylePatch) {
                     if (_selectionStatus != kUnselected) {

lib/mayaUsd/render/vp2RenderDelegate/mesh.cpp

@@ -2138,10 +2138,11 @@ void HdVP2Mesh::_UpdateDrawItem(
             // render item.
 
             // Set up the source color buffers.
-            const MColor wireframeColors[] = { drawScene.GetWireframeColor(),
-                                               drawScene.GetSelectionHighlightColor(false),
-                                               drawScene.GetSelectionHighlightColor(true) };
-            bool         useWireframeColors = stateToCommit._instanceColorParam == kSolidColorStr;
+            const MColor wireframeColors[]
+                = { drawScene.GetWireframeColor(),
+                    drawScene.GetSelectionHighlightColor(HdPrimTypeTokens->mesh),
+                    drawScene.GetSelectionHighlightColor() };
+            bool useWireframeColors = stateToCommit._instanceColorParam == kSolidColorStr;
 
             MFloatArray*    shadedColors = nullptr;
             HdInterpolation colorInterpolation = HdInterpolationConstant;
@@ -2254,9 +2255,9 @@ void HdVP2Mesh::_UpdateDrawItem(
         if ((itemDirtyBits & DirtySelectionHighlight)
             && drawItem->ContainsUsage(HdVP2DrawItem::kSelectionHighlight)) {
             const MColor& color
-                = (_selectionStatus != kUnselected
-                       ? drawScene.GetSelectionHighlightColor(_selectionStatus == kFullyLead)
-                       : drawScene.GetWireframeColor());
+                = (_selectionStatus != kUnselected ? drawScene.GetSelectionHighlightColor(
+                       _selectionStatus == kFullyLead ? TfToken() : HdPrimTypeTokens->mesh)
+                                                   : drawScene.GetWireframeColor());
 
             MHWRender::MShaderInstance* shader = _delegate->Get3dSolidShader(color);
             if (shader != nullptr && shader != drawItemData._shader) {

lib/mayaUsd/render/vp2RenderDelegate/proxyRenderDelegate.cpp

@@ -47,6 +47,7 @@
 #include <pxr/usd/usd/prim.h>
 #include <pxr/usdImaging/usdImaging/delegate.h>
 
+#include <maya/MColorPickerUtilities.h>
 #include <maya/MEventMessage.h>
 #include <maya/MFileIO.h>
 #include <maya/MFnPluginData.h>
@@ -1488,7 +1489,7 @@ GfVec3f ProxyRenderDelegate::GetCurveDefaultColor()
 {
     MDoubleArray curveColorResult;
     {
-        std::lock_guard<std::mutex> _(_mayaCommandEngineMutex);
+        std::lock_guard<std::mutex> mutexGuard(_mayaCommandEngineMutex);
         MGlobal::executeCommand(
             "int $index = `displayColor -q -dormant \"curve\"`; colorIndex -q $index;",
             curveColorResult);
@@ -1497,18 +1498,75 @@ GfVec3f ProxyRenderDelegate::GetCurveDefaultColor()
     if (curveColorResult.length() == 3) {
         return GfVec3f(curveColorResult[0], curveColorResult[1], curveColorResult[2]);
     } else {
+        TF_WARN("Failed to obtain curve default color");
         // In case of an error, return the default navy-blue color
         return GfVec3f(0.000f, 0.016f, 0.376f);
     }
 }
 
 //! \brief
-const MColor& ProxyRenderDelegate::GetSelectionHighlightColor(bool lead) const
+MColor ProxyRenderDelegate::GetSelectionHighlightColor(const TfToken& className)
 {
-    static const MColor kLeadColor(0.056f, 1.0f, 0.366f, 1.0f);
-    static const MColor kActiveColor(1.0f, 1.0f, 1.0f, 1.0f);
+    static const MColor kDefaultLeadColor(0.056f, 1.0f, 0.366f, 1.0f);
+    static const MColor kDefaultActiveColor(1.0f, 1.0f, 1.0f, 1.0f);
+
+    // Prepare to construct the query command.
+    bool        fromPalette = true;
+    const char* queryName = "unsupported";
+    if (className.IsEmpty()) {
+        fromPalette = false;
+        queryName = "lead";
+    } else if (className == HdPrimTypeTokens->mesh) {
+        fromPalette = false;
+        queryName = "polymeshActive";
+    } else if (className == HdPrimTypeTokens->basisCurves) {
+        queryName = "curve";
+    }
+
+    // Construct the query command string.
+    MString queryCommand;
+    if (fromPalette) {
+        queryCommand = "int $index = `displayColor -q -active \"";
+        queryCommand += queryName;
+        queryCommand += "\"`; colorIndex -q $index;";
+    } else {
+        queryCommand = "displayRGBColor -q \"";
+        queryCommand += queryName;
+        queryCommand += "\"";
+    }
+
+    // Query and return the selection color.
+    {
+        MDoubleArray                colorResult;
+        std::lock_guard<std::mutex> mutexGuard(_mayaCommandEngineMutex);
+        MGlobal::executeCommand(queryCommand, colorResult);
+
+        if (colorResult.length() == 3) {
+            MColor color(colorResult[0], colorResult[1], colorResult[2]);
+
+            if (className.IsEmpty()) {
+                // The 'lead' color is returned in display space, so we need to convert it to
+                // rendering space. However, function MColorPickerUtilities::applyViewTransform
+                // is supported only starting from Maya 2023, so in opposite case we just return
+                // the default lead color.
+#if MAYA_API_VERSION >= 20230000
+                return MColorPickerUtilities::applyViewTransform(
+                    color, MColorPickerUtilities::kInverse);
+#else
+                return kDefaultLeadColor;
+#endif
+            } else {
+                return color;
+            }
+        } else {
+            TF_WARN(
+                "Failed to obtain selection highlight color for '%s' objects",
+                className.IsEmpty() ? "lead" : className.GetString().c_str());
+        }
+    }
 
-    return lead ? kLeadColor : kActiveColor;
+    // In case of any failure, return the default color
+    return className.IsEmpty() ? kDefaultLeadColor : kDefaultActiveColor;
 }
 
 bool ProxyRenderDelegate::DrawRenderTag(const TfToken& renderTag) const

lib/mayaUsd/render/vp2RenderDelegate/proxyRenderDelegate.h

@@ -173,8 +173,10 @@ class ProxyRenderDelegate : public MHWRender::MPxSubSceneOverride
     MAYAUSD_CORE_PUBLIC
     GfVec3f GetCurveDefaultColor();
 
+    // Returns the selection highlight color for a given HdPrim type.
+    // If className is empty, returns the lead highlight color.
     MAYAUSD_CORE_PUBLIC
-    const MColor& GetSelectionHighlightColor(bool lead) const;
+    MColor GetSelectionHighlightColor(const TfToken& className = TfToken());
 
     MAYAUSD_CORE_PUBLIC
     const HdSelection::PrimSelectionState* GetLeadSelectionState(const SdfPath& path) const;