Add handling of pInvokeMarshallingRequired and suppressGCTransition

Author: janvorli

src/coreclr/interpreter/compiler.cpp

@@ -2810,14 +2810,19 @@ void InterpCompiler::EmitPushLdvirtftn(int thisVar, CORINFO_RESOLVED_TOKEN* pRes
     m_pLastNewIns->info.pCallInfo->pCallArgs = callArgs;
 }
 
-void InterpCompiler::EmitCalli(bool isTailCall, void* calliCookie, int callIFunctionPointerVar, CorInfoCallConv callConv)
+void InterpCompiler::EmitCalli(bool isTailCall, void* calliCookie, int callIFunctionPointerVar, CORINFO_SIG_INFO* callSiteSig)
 {
     AddIns(isTailCall ? INTOP_CALLI_TAIL : INTOP_CALLI);
     m_pLastNewIns->data[0] = GetDataItemIndex(calliCookie);
     // data[1] is set to 1 if the calli is calling a pinvoke, 0 otherwise
-    m_pLastNewIns->data[1] = (callConv == CORINFO_CALLCONV_DEFAULT || callConv == CORINFO_CALLCONV_VARARG) ? 0 : 1;
+    bool suppressGCTransition = false;
+    m_compHnd->getUnmanagedCallConv(nullptr, callSiteSig, &suppressGCTransition);
+    bool isPInvoke = (callSiteSig->callConv != CORINFO_CALLCONV_DEFAULT && callSiteSig->callConv != CORINFO_CALLCONV_VARARG);
+    bool isPInvokeMarshalled = isPInvoke && m_compHnd->pInvokeMarshalingRequired(NULL, callSiteSig);
+    m_pLastNewIns->data[1] = (suppressGCTransition ? (int32_t)CalliFlags::SuppressGCTransition : 0) |
+                             (isPInvoke ? (int32_t)CalliFlags::PInvoke : 0) |
+                             (isPInvokeMarshalled ? (int32_t)CalliFlags::PInvokeMarshalled : 0);
     m_pLastNewIns->SetSVars2(CALL_ARGS_SVAR, callIFunctionPointerVar);
-
 }
 
 void InterpCompiler::EmitCall(CORINFO_RESOLVED_TOKEN* pConstrainedToken, bool readonly, bool tailcall, bool newObj, bool isCalli)
@@ -3225,7 +3230,7 @@ void InterpCompiler::EmitCall(CORINFO_RESOLVED_TOKEN* pConstrainedToken, bool re
             }
             else if (isCalli)
             {
-                EmitCalli(tailcall, calliCookie, callIFunctionPointerVar, callInfo.sig.getCallConv());
+                EmitCalli(tailcall, calliCookie, callIFunctionPointerVar, &callInfo.sig);
             }
             else
             {
@@ -3282,7 +3287,7 @@ void InterpCompiler::EmitCall(CORINFO_RESOLVED_TOKEN* pConstrainedToken, bool re
 
             calliCookie = m_compHnd->GetCookieForInterpreterCalliSig(&callInfo.sig);
 
-            EmitCalli(tailcall, calliCookie, codePointerLookupResult, callInfo.sig.getCallConv());
+            EmitCalli(tailcall, calliCookie, codePointerLookupResult, &callInfo.sig);
             break;
         }
         case CORINFO_VIRTUALCALL_VTABLE:
@@ -3315,7 +3320,7 @@ void InterpCompiler::EmitCall(CORINFO_RESOLVED_TOKEN* pConstrainedToken, bool re
 
                 calliCookie = m_compHnd->GetCookieForInterpreterCalliSig(&callInfo.sig);
 
-                EmitCalli(tailcall, calliCookie, synthesizedLdvirtftnPtrVar, callInfo.sig.getCallConv());
+                EmitCalli(tailcall, calliCookie, synthesizedLdvirtftnPtrVar, &callInfo.sig);
             }
             else
             {

src/coreclr/interpreter/compiler.h

@@ -711,7 +711,7 @@ class InterpCompiler
     void    EmitShiftOp(int32_t opBase);
     void    EmitCompareOp(int32_t opBase);
     void    EmitCall(CORINFO_RESOLVED_TOKEN* pConstrainedToken, bool readonly, bool tailcall, bool newObj, bool isCalli);
-    void    EmitCalli(bool isTailCall, void* calliCookie, int callIFunctionPointerVar, CorInfoCallConv callConv);
+    void    EmitCalli(bool isTailCall, void* calliCookie, int callIFunctionPointerVar, CORINFO_SIG_INFO* callSiteSig);
     bool    EmitNamedIntrinsicCall(NamedIntrinsic ni, CORINFO_CLASS_HANDLE clsHnd, CORINFO_METHOD_HANDLE method, CORINFO_SIG_INFO sig);
     void    EmitLdind(InterpType type, CORINFO_CLASS_HANDLE clsHnd, int32_t offset);
     void    EmitStind(InterpType type, CORINFO_CLASS_HANDLE clsHnd, int32_t offset, bool reverseSVarOrder);

src/coreclr/interpreter/interpretershared.h

@@ -171,4 +171,12 @@ enum class PInvokeCallFlags : int32_t
     SuppressGCTransition = 1 << 1, // The pinvoke is marked by the SuppressGCTransition attribute
 };
 
+enum class CalliFlags : int32_t
+{
+    None = 0,
+    SuppressGCTransition = 1 << 1, // The call is marked by the SuppressGCTransition attribute
+    PInvoke = 1 << 2, // The call is a PInvoke call
+    PInvokeMarshalled = 1 << 3, // The call is a PInvoke call with marshaling
+};
+
 #endif

src/coreclr/vm/interpexec.cpp

@@ -2009,7 +2009,7 @@ void InterpExecMethod(InterpreterFrame *pInterpreterFrame, InterpMethodContextFr
                     callArgsOffset = ip[2];
                     int32_t calliFunctionPointerVar = ip[3];
                     int32_t calliCookie = ip[4];
-                    bool isPInvoke = ip[5] != 0;
+                    int32_t flags = ip[5];
 
                     CallStubHeader *pCallStub = (CallStubHeader*)pMethod->pDataItems[calliCookie];
                     ip += 6;
@@ -2018,7 +2018,7 @@ void InterpExecMethod(InterpreterFrame *pInterpreterFrame, InterpMethodContextFr
                     pFrame->ip = ip;
 
                     // Interpreter-FIXME: isTailcall
-                    if (isPInvoke)
+                    if ((flags & (int32_t)CalliFlags::PInvoke) && !(flags & (int32_t)CalliFlags::SuppressGCTransition) && !(flags & (int32_t)CalliFlags::PInvokeMarshalled))
                     {
                         InvokePInvokeCalliStub(LOCAL_VAR(calliFunctionPointerVar, PCODE), pCallStub, stack, pFrame, stack + callArgsOffset, stack + returnOffset);
                     }