Implement portable tailcall helpers (#341)

* Implement portable tailcall helpers

This implements tailcall-via-help support for all platforms supported by
the runtime. In this new mechanism the JIT asks the runtime for help
whenever it realizes it will need a helper to perform a tailcall, i.e.
when it sees an explicit tail. prefixed call that it cannot make into a
fast jump-based tailcall.

The runtime created two important IL stubs to help the JIT in performing
the necessary tailcalls. One IL stub is used to store the args for the
tailcall, while the other is used to dispatch the actual tailcall
itself. The JIT will then transform the call from

return tail. F(a1, ..., an);

to

IL_STUB_StoreTailCallArgs(a1, ..., an);
T result;
IL_STUB_DispatchTailCalls(..., &result);
return result;

The dispatcher is written in such a way that it is able to dispatch
multiple tailcalls in a row when tailcalled functions also perform
tailcalls. To do this, the JIT helps the dispatcher detect if the
caller's caller is also a dispatcher. When this is the case the
dispatcher returns to let the previous dispatcher perform the tailcall
with the currently stored args. This allows the frame to unwind and
ensures that sequences of tailcalls do not grow the stack more than by a
constant factor.

Due to this unwinding the args cannot be stored on the stack and are
instead stored in TLS. The GC is made specially of this buffer as the
args can be anything, including interior pointers.

The control-flow when performing the new tailcalls is nonstandard, so
this also changes the debugger to support proper stepping into/over/out
of tailcalled functions when they go through the new dispatcher.

x86's tailcalling mechanism does not change.

This commit also includes the following changes:

* Update design doc for helper-based tailcalls

* Change lowering of GT_LABEL on arm.

Generate movw/movt instead of adr on arm. adr on arm allows offsets
up to 4k, which may not be enough. In particular,
IL_STUB_CallTailCallTarget uses GT_LABEL before argument setup code
and it can be more than 4k.

* Add COMPlus_FastTailCalls environment variable.

COMPlus_FastTailCalls controls whether fast tail calls are allowed.
If COMPlus_FastTailCalls is 0, fast tail calls are not allowed even for
tail-prefixed calls. Only helper-based calls are allowed. This is useful
for testing helper-based calls.

Co-authored-by: Eugene Rozenfeld <erozen@microsoft.com>

src/coreclr/src/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs

@@ -286,8 +286,16 @@ public static IntPtr AllocateTypeAssociatedMemory(Type type, int size)
 
         [DllImport(RuntimeHelpers.QCall)]
         private static extern IntPtr AllocateTypeAssociatedMemoryInternal(QCallTypeHandle type, uint size);
-    }
 
+        [MethodImpl(MethodImplOptions.InternalCall)]
+        private static extern IntPtr AllocTailCallArgBuffer(int size, IntPtr gcDesc);
+
+        [MethodImpl(MethodImplOptions.InternalCall)]
+        private static extern void FreeTailCallArgBuffer();
+
+        [MethodImpl(MethodImplOptions.InternalCall)]
+        private static unsafe extern TailCallTls* GetTailCallInfo(IntPtr retAddrSlot, IntPtr* retAddr);
+    }
     // Helper class to assist with unsafe pinning of arbitrary objects.
     // It's used by VM code.
     internal class RawData
@@ -421,4 +429,24 @@ public int MultiDimensionalArrayRank
             }
         }
     }
+
+    // Helper structs used for tail calls via helper.
+    [StructLayout(LayoutKind.Sequential)]
+    internal unsafe struct PortableTailCallFrame
+    {
+        public PortableTailCallFrame* Prev;
+        public IntPtr TailCallAwareReturnAddress;
+        public IntPtr NextCall;
+    }
+
+    [StructLayout(LayoutKind.Sequential)]
+    internal unsafe struct TailCallTls
+    {
+        public PortableTailCallFrame* Frame;
+        public IntPtr ArgBuffer;
+        private IntPtr _argBufferSize;
+        private IntPtr _argBufferGCDesc;
+        private fixed byte _argBufferInline[64];
+    }
+
 }

src/coreclr/src/System.Private.CoreLib/src/System/StubHelpers.cs

@@ -1878,5 +1878,8 @@ internal static void CheckStringLength(uint length)
         [MethodImpl(MethodImplOptions.InternalCall)]
         internal static extern void MulticastDebuggerTraceHelper(object o, int count);
 #endif
+
+        [MethodImplAttribute(MethodImplOptions.InternalCall)]
+        internal static extern IntPtr NextCallReturnAddress();
     }  // class StubHelpers
 }

src/coreclr/src/ToolBox/superpmi/superpmi-shared/icorjitinfoimpl.h

@@ -883,6 +883,12 @@ void MethodCompileComplete(CORINFO_METHOD_HANDLE methHnd);
 // return a thunk that will copy the arguments for the given signature.
 void* getTailCallCopyArgsThunk(CORINFO_SIG_INFO* pSig, CorInfoHelperTailCallSpecialHandling flags);
 
+bool getTailCallHelpers(
+    CORINFO_RESOLVED_TOKEN* callToken,
+    CORINFO_SIG_INFO* sig,
+    CORINFO_GET_TAILCALL_HELPERS_FLAGS flags,
+    CORINFO_TAILCALL_HELPERS* pResult);
+
 bool convertPInvokeCalliToCall(CORINFO_RESOLVED_TOKEN * pResolvedToken, bool fMustConvert);
 
 // get a block of memory for the code, readonly data, and read-write data

src/coreclr/src/ToolBox/superpmi/superpmi-shared/lwmlist.h

@@ -122,6 +122,7 @@ LWM(GetSharedCCtorHelper, DWORDLONG, DWORD)
 LWM(GetStringConfigValue, DWORD, DWORD)
 LWM(GetSystemVAmd64PassStructInRegisterDescriptor, DWORDLONG, Agnostic_GetSystemVAmd64PassStructInRegisterDescriptor)
 LWM(GetTailCallCopyArgsThunk, Agnostic_GetTailCallCopyArgsThunk, DWORDLONG)
+LWM(GetTailCallHelpers, Agnostic_GetTailCallHelpers, Agnostic_CORINFO_TAILCALL_HELPERS)
 LWM(GetThreadTLSIndex, DWORD, DLD)
 LWM(GetTokenTypeAsHandle, GetTokenTypeAsHandleValue, DWORDLONG)
 LWM(GetTypeForBox, DWORDLONG, DWORDLONG)

src/coreclr/src/ToolBox/superpmi/superpmi-shared/methodcontext.cpp

@@ -5956,12 +5956,14 @@ void MethodContext::recGetTailCallCopyArgsThunk(CORINFO_SIG_INFO*
     GetTailCallCopyArgsThunk->Add(key, (DWORDLONG)result);
     DEBUG_REC(dmpGetTailCallCopyArgsThunk(key, (DWORDLONG)result));
 }
+
 void MethodContext::dmpGetTailCallCopyArgsThunk(const Agnostic_GetTailCallCopyArgsThunk& key, DWORDLONG value)
 {
     printf("GetTailCallCopyArgsThunk key sig%s flg-%08X",
            SpmiDumpHelper::DumpAgnostic_CORINFO_SIG_INFO(key.Sig).c_str(), key.flags);
     printf(", value res-%016llX", value);
 }
+
 void* MethodContext::repGetTailCallCopyArgsThunk(CORINFO_SIG_INFO* pSig, CorInfoHelperTailCallSpecialHandling flags)
 {
     AssertCodeMsg(GetTailCallCopyArgsThunk != nullptr, EXCEPTIONCODE_MC, "Didn't find anything for ...");
@@ -5980,6 +5982,78 @@ void* MethodContext::repGetTailCallCopyArgsThunk(CORINFO_SIG_INFO* pSig, CorInfo
     return result;
 }
 
+void MethodContext::recGetTailCallHelpers(
+    CORINFO_RESOLVED_TOKEN* callToken,
+    CORINFO_SIG_INFO* sig,
+    CORINFO_GET_TAILCALL_HELPERS_FLAGS flags,
+    CORINFO_TAILCALL_HELPERS* pResult)
+{
+    if (GetTailCallHelpers == nullptr)
+        GetTailCallHelpers = new LightWeightMap<Agnostic_GetTailCallHelpers, Agnostic_CORINFO_TAILCALL_HELPERS>();
+
+    Agnostic_GetTailCallHelpers key;
+    ZeroMemory(&key, sizeof(Agnostic_GetTailCallHelpers));
+
+    key.callToken = SpmiRecordsHelper::StoreAgnostic_CORINFO_RESOLVED_TOKEN(callToken, GetTailCallHelpers);
+    key.sig = SpmiRecordsHelper::StoreAgnostic_CORINFO_SIG_INFO(*sig, GetTailCallHelpers);
+    key.flags = (DWORD)flags;
+
+    Agnostic_CORINFO_TAILCALL_HELPERS value;
+    ZeroMemory(&value, sizeof(Agnostic_CORINFO_TAILCALL_HELPERS));
+
+    value.result = pResult != nullptr;
+    if (pResult != nullptr)
+    {
+        value.flags = (DWORD)pResult->flags;
+        value.hStoreArgs = (DWORDLONG)pResult->hStoreArgs;
+        value.hCallTarget = (DWORDLONG)pResult->hCallTarget;
+        value.hDispatcher = (DWORDLONG)pResult->hDispatcher;
+    }
+    GetTailCallHelpers->Add(key, value);
+    DEBUG_REC(dmpGetTailCallHelpers(key, value));
+}
+
+void MethodContext::dmpGetTailCallHelpers(const Agnostic_GetTailCallHelpers& key, const Agnostic_CORINFO_TAILCALL_HELPERS& value)
+{
+    printf("GetTailCallHelpers key callToken-%s sig-%s flg-%08X",
+        SpmiDumpHelper::DumpAgnostic_CORINFO_RESOLVED_TOKEN(key.callToken).c_str(),
+        SpmiDumpHelper::DumpAgnostic_CORINFO_SIG_INFO(key.sig).c_str(),
+        key.flags);
+    printf(", value result-%s flg-%08X hStoreArgs-%016llX hCallTarget-%016llX hDispatcher-%016llX",
+        value.result ? "true" : "false",
+        value.flags,
+        value.hStoreArgs,
+        value.hCallTarget,
+        value.hDispatcher);
+}
+
+bool MethodContext::repGetTailCallHelpers(
+    CORINFO_RESOLVED_TOKEN* callToken,
+    CORINFO_SIG_INFO* sig,
+    CORINFO_GET_TAILCALL_HELPERS_FLAGS flags,
+    CORINFO_TAILCALL_HELPERS* pResult)
+{
+    AssertCodeMsg(GetTailCallHelpers != nullptr, EXCEPTIONCODE_MC, "Didn't find anything for ...");
+
+    Agnostic_GetTailCallHelpers key;
+    ZeroMemory(&key, sizeof(Agnostic_GetTailCallHelpers));
+    key.callToken = SpmiRecordsHelper::RestoreAgnostic_CORINFO_RESOLVED_TOKEN(callToken, GetTailCallHelpers);
+    key.sig = SpmiRecordsHelper::RestoreAgnostic_CORINFO_SIG_INFO(*sig, GetTailCallHelpers);
+    key.flags = (DWORD)flags;
+
+    AssertCodeMsg(GetTailCallHelpers->GetIndex(key) != -1, EXCEPTIONCODE_MC, "Could not find matching tail call helper call");
+    Agnostic_CORINFO_TAILCALL_HELPERS value = GetTailCallHelpers->Get(key);
+    if (!value.result)
+        return false;
+
+    pResult->flags = (CORINFO_TAILCALL_HELPERS_FLAGS)value.flags;
+    pResult->hStoreArgs = (CORINFO_METHOD_HANDLE)value.hStoreArgs;
+    pResult->hCallTarget = (CORINFO_METHOD_HANDLE)value.hCallTarget;
+    pResult->hDispatcher = (CORINFO_METHOD_HANDLE)value.hDispatcher;
+    DEBUG_REP(dmpGetTailCallHelpers(key, value));
+    return true;
+}
+
 void MethodContext::recGetMethodDefFromMethod(CORINFO_METHOD_HANDLE hMethod, mdMethodDef result)
 {
     if (GetMethodDefFromMethod == nullptr)

src/coreclr/src/ToolBox/superpmi/superpmi-shared/methodcontext.h

@@ -441,6 +441,20 @@ class MethodContext
         Agnostic_CORINFO_SIG_INFO Sig;
         DWORD                     flags;
     };
+    struct Agnostic_GetTailCallHelpers
+    {
+        Agnostic_CORINFO_RESOLVED_TOKEN callToken;
+        Agnostic_CORINFO_SIG_INFO sig;
+        DWORD flags;
+    };
+    struct Agnostic_CORINFO_TAILCALL_HELPERS
+    {
+        bool result;
+        DWORD flags;
+        DWORDLONG hStoreArgs;
+        DWORDLONG hCallTarget;
+        DWORDLONG hDispatcher;
+    };
     struct Agnostic_GetArgClass_Value
     {
         DWORDLONG result;
@@ -1259,6 +1273,18 @@ class MethodContext
     void dmpGetTailCallCopyArgsThunk(const Agnostic_GetTailCallCopyArgsThunk& key, DWORDLONG value);
     void* repGetTailCallCopyArgsThunk(CORINFO_SIG_INFO* pSig, CorInfoHelperTailCallSpecialHandling flags);
 
+    void recGetTailCallHelpers(
+        CORINFO_RESOLVED_TOKEN* callToken,
+        CORINFO_SIG_INFO* sig,
+        CORINFO_GET_TAILCALL_HELPERS_FLAGS flags,
+        CORINFO_TAILCALL_HELPERS* pResult);
+    void dmpGetTailCallHelpers(const Agnostic_GetTailCallHelpers& key, const Agnostic_CORINFO_TAILCALL_HELPERS& value);
+    bool repGetTailCallHelpers(
+        CORINFO_RESOLVED_TOKEN* callToken,
+        CORINFO_SIG_INFO* sig,
+        CORINFO_GET_TAILCALL_HELPERS_FLAGS flags,
+        CORINFO_TAILCALL_HELPERS* pResult);
+
     void recGetMethodDefFromMethod(CORINFO_METHOD_HANDLE hMethod, mdMethodDef result);
     void dmpGetMethodDefFromMethod(DWORDLONG key, DWORD value);
     mdMethodDef repGetMethodDefFromMethod(CORINFO_METHOD_HANDLE hMethod);
@@ -1320,7 +1346,7 @@ class MethodContext
 };
 
 // ********************* Please keep this up-to-date to ease adding more ***************
-// Highest packet number: 177
+// Highest packet number: 178
 // *************************************************************************************
 enum mcPackets
 {
@@ -1439,6 +1465,7 @@ enum mcPackets
     Packet_GetSecurityPrologHelper                       = 85, // Retired 2/18/2020
     Packet_GetSharedCCtorHelper                          = 86,
     Packet_GetTailCallCopyArgsThunk                      = 87,
+    Packet_GetTailCallHelpers                            = 178, // Added 3/18/2020
     Packet_GetThreadTLSIndex                             = 88,
     Packet_GetTokenTypeAsHandle                          = 89,
     Packet_GetTypeForBox                                 = 90,

src/coreclr/src/ToolBox/superpmi/superpmi-shim-collector/icorjitinfo.cpp

@@ -1866,6 +1866,21 @@ void* interceptor_ICJI::getTailCallCopyArgsThunk(CORINFO_SIG_INFO* pSig, CorInfo
     return result;
 }
 
+bool interceptor_ICJI::getTailCallHelpers(
+        CORINFO_RESOLVED_TOKEN* callToken,
+        CORINFO_SIG_INFO* sig,
+        CORINFO_GET_TAILCALL_HELPERS_FLAGS flags,
+        CORINFO_TAILCALL_HELPERS* pResult)
+{
+    mc->cr->AddCall("getTailCallHelpers");
+    bool result = original_ICorJitInfo->getTailCallHelpers(callToken, sig, flags, pResult);
+    if (result)
+        mc->recGetTailCallHelpers(callToken, sig, flags, pResult);
+    else
+        mc->recGetTailCallHelpers(callToken, sig, flags, nullptr);
+    return result;
+}
+
 // Stuff directly on ICorJitInfo
 
 // Returns extended flags for a particular compilation instance.

src/coreclr/src/ToolBox/superpmi/superpmi-shim-counter/icorjitinfo.cpp

@@ -1449,6 +1449,16 @@ void* interceptor_ICJI::getTailCallCopyArgsThunk(CORINFO_SIG_INFO* pSig, CorInfo
     return original_ICorJitInfo->getTailCallCopyArgsThunk(pSig, flags);
 }
 
+bool interceptor_ICJI::getTailCallHelpers(
+        CORINFO_RESOLVED_TOKEN* callToken,
+        CORINFO_SIG_INFO* sig,
+        CORINFO_GET_TAILCALL_HELPERS_FLAGS flags,
+        CORINFO_TAILCALL_HELPERS* pResult)
+{
+    mcs->AddCall("getTailCallHelpers");
+    return original_ICorJitInfo->getTailCallHelpers(callToken, sig, flags, pResult);
+}
+
 // Stuff directly on ICorJitInfo
 
 // Returns extended flags for a particular compilation instance.

src/coreclr/src/ToolBox/superpmi/superpmi-shim-simple/icorjitinfo.cpp

@@ -1291,6 +1291,15 @@ void* interceptor_ICJI::getTailCallCopyArgsThunk(CORINFO_SIG_INFO* pSig, CorInfo
     return original_ICorJitInfo->getTailCallCopyArgsThunk(pSig, flags);
 }
 
+bool interceptor_ICJI::getTailCallHelpers(
+        CORINFO_RESOLVED_TOKEN* callToken,
+        CORINFO_SIG_INFO* sig,
+        CORINFO_GET_TAILCALL_HELPERS_FLAGS flags,
+        CORINFO_TAILCALL_HELPERS* pResult)
+{
+    return original_ICorJitInfo->getTailCallHelpers(callToken, sig, flags, pResult);
+}
+
 // Stuff directly on ICorJitInfo
 
 // Returns extended flags for a particular compilation instance.

src/coreclr/src/ToolBox/superpmi/superpmi/icorjitinfo.cpp

@@ -1547,6 +1547,16 @@ void* MyICJI::getTailCallCopyArgsThunk(CORINFO_SIG_INFO* pSig, CorInfoHelperTail
     return jitInstance->mc->repGetTailCallCopyArgsThunk(pSig, flags);
 }
 
+bool MyICJI::getTailCallHelpers(
+        CORINFO_RESOLVED_TOKEN* callToken,
+        CORINFO_SIG_INFO* sig,
+        CORINFO_GET_TAILCALL_HELPERS_FLAGS flags,
+        CORINFO_TAILCALL_HELPERS* pResult)
+{
+    jitInstance->mc->cr->AddCall("getTailCallHelpers");
+    return jitInstance->mc->repGetTailCallHelpers(callToken, sig, flags, pResult);
+}
+
 bool MyICJI::convertPInvokeCalliToCall(CORINFO_RESOLVED_TOKEN* pResolvedToken, bool fMustConvert)
 {
     jitInstance->mc->cr->AddCall("convertPInvokeCalliToCall");