Cached interface dispatch for coreclr

docs/design/coreclr/botr/clr-abi.md

@@ -114,7 +114,7 @@ ARM64-only: When a method returns a structure that is larger than 16 bytes the c
 
 ## Hidden parameters
 
-*Stub dispatch* - when a virtual call uses a VSD stub, rather than back-patching the calling code (or disassembling it), the JIT must place the address of the stub used to load the call target, the "stub indirection cell", in (x86) `EAX` / (AMD64) `R11` / (AMD64 NativeAOT ABI) `R10` / (ARM) `R4` / (ARM NativeAOT ABI) `R12` / (ARM64) `R11`. In the JIT, this is encapsulated in the `VirtualStubParamInfo` class.
+*Stub dispatch* - when a virtual call uses a VSD stub, rather than back-patching the calling code (or disassembling it), the JIT must place the address of the stub used to load the call target, the "stub indirection cell", in (x86) `EAX` / (AMD64) `R11` / (ARM) `R4` / (ARM NativeAOT ABI) `R12` / (ARM64) `R11`. In the JIT, this is encapsulated in the `VirtualStubParamInfo` class.
 
 *Calli Pinvoke* - The VM wants the address of the PInvoke in (AMD64) `R10` / (ARM) `R12` / (ARM64) `R14` (In the JIT: `REG_PINVOKE_TARGET_PARAM`), and the signature (the pinvoke cookie) in (AMD64) `R11` / (ARM) `R4` / (ARM64) `R15` (in the JIT: `REG_PINVOKE_COOKIE_PARAM`).
 
@@ -812,7 +812,7 @@ Therefore it will expand all indirect calls via the validation helper and a manu
 ## CFG details for x64
 
 On x64, `CORINFO_HELP_VALIDATE_INDIRECT_CALL` takes the call address in `rcx`.
-In addition to the usual registers it also preserves all float registers and `rcx` and `r10`; furthermore, shadow stack space is not required to be allocated.
+In addition to the usual registers it also preserves all float registers, `rcx`, and `r10`; furthermore, shadow stack space is not required to be allocated.
 
 `CORINFO_HELP_DISPATCH_INDIRECT_CALL` takes the call address in `rax` and it reserves the right to use and trash `r10` and `r11`.
 The JIT uses the dispatch helper on x64 whenever possible as it is expected that the code size benefits outweighs the less accurate branch prediction.

src/coreclr/clrfeatures.cmake

@@ -49,4 +49,24 @@ endif()
 
 if (CLR_CMAKE_TARGET_WIN32)
   set(FEATURE_TYPEEQUIVALENCE 1)
-endif(CLR_CMAKE_TARGET_WIN32)
+endif(CLR_CMAKE_TARGET_WIN32)
+
+
+if (CLR_CMAKE_TARGET_MACCATALYST OR CLR_CMAKE_TARGET_IOS OR CLR_CMAKE_TARGET_TVOS)
+  set(FEATURE_CORECLR_CACHED_INTERFACE_DISPATCH 1)
+  set(FEATURE_CORECLR_VIRTUAL_STUB_DISPATCH 0)
+else()
+  # Enable cached interface dispatch so that we can test/debug it more easily on non-embedded scenarios (set DOTNET_UseCachedInterfaceDispatch=1)
+  # Only enable in chk/debug builds as this support isn't intended for retail use elsewhere
+  if (CLR_CMAKE_TARGET_ARCH_AMD64 OR CLR_CMAKE_TARGET_ARCH_ARM64)
+    set(FEATURE_CORECLR_CACHED_INTERFACE_DISPATCH $<IF:$<CONFIG:Debug,Checked>,1,0>)
+  else()
+    set(FEATURE_CORECLR_CACHED_INTERFACE_DISPATCH 0)
+  endif()
+  set(FEATURE_CORECLR_VIRTUAL_STUB_DISPATCH 1)
+endif()
+
+if (CLR_CMAKE_HOST_UNIX AND CLR_CMAKE_HOST_ARCH_AMD64)
+  # Allow 16 byte compare-exchange (cmpxchg16b)
+  add_compile_options($<${FEATURE_CORECLR_CACHED_INTERFACE_DISPATCH}:-mcx16>)
+endif()

src/coreclr/crossgen-corelib.proj

@@ -118,7 +118,10 @@
       <CrossGenDllCmd Condition="'$(TargetsAndroid)' == 'true'">$(CrossGenDllCmd) --targetos:linux</CrossGenDllCmd>
       <CrossGenDllCmd Condition="'$(UsingToolIbcOptimization)' != 'true' and '$(EnableNgenOptimization)' == 'true'">$(CrossGenDllCmd) -m:$(MergedMibcPath) --embed-pgo-data</CrossGenDllCmd>
       <CrossGenDllCmd>$(CrossGenDllCmd) -O</CrossGenDllCmd>
+      <!-- Enable type and field layout verification to make it easier to catch when the crossgen2 type layout engine and the runtime disagree on layout conventions -->
       <CrossGenDllCmd  Condition="'$(Configuration)' == 'Debug' or '$(Configuration)' == 'Checked'">$(CrossGenDllCmd) --verify-type-and-field-layout</CrossGenDllCmd>
+      <!-- Enable Cached Interface Dispatch layout rules for the StubDispatch import section. This allows testing of this path on platforms that do not require CachedInterface Dispatch -->
+      <CrossGenDllCmd  Condition="'$(Configuration)' == 'Debug' or '$(Configuration)' == 'Checked'">$(CrossGenDllCmd) --enable-cached-interface-dispatch-support</CrossGenDllCmd>
       <CrossGenDllCmd>$(CrossGenDllCmd) @(CoreLib)</CrossGenDllCmd>
     </PropertyGroup>
 

src/coreclr/debug/CMakeLists.txt

@@ -1,3 +1,7 @@
+
+add_compile_definitions($<${FEATURE_CORECLR_CACHED_INTERFACE_DISPATCH}:FEATURE_CACHED_INTERFACE_DISPATCH>)
+add_compile_definitions($<${FEATURE_CORECLR_VIRTUAL_STUB_DISPATCH}:FEATURE_VIRTUAL_STUB_DISPATCH>)
+
 add_subdirectory(daccess)
 add_subdirectory(ee)
 add_subdirectory(di)

src/coreclr/debug/daccess/dacdbiimpl.cpp

@@ -3544,7 +3544,9 @@ void DacDbiInterfaceImpl::EnumerateMemRangesForLoaderAllocator(PTR_LoaderAllocat
     if (pVcsMgr)
     {
         if (pVcsMgr->indcell_heap != NULL) heapsToEnumerate.Push(pVcsMgr->indcell_heap);
+#ifdef FEATURE_VIRTUAL_STUB_DISPATCH
         if (pVcsMgr->cache_entry_heap != NULL) heapsToEnumerate.Push(pVcsMgr->cache_entry_heap);
+#endif // FEATURE_VIRTUAL_STUB_DISPATCH
     }
 
     TADDR rangeAccumAsTaddr = TO_TADDR(rangeAcummulator);

src/coreclr/debug/daccess/request.cpp

@@ -3620,14 +3620,19 @@ ClrDataAccess::TraverseVirtCallStubHeap(CLRDATA_ADDRESS pAppDomain, VCSHeapType
                 break;
 
             case CacheEntryHeap:
+#ifdef FEATURE_VIRTUAL_STUB_DISPATCH
+                // The existence of the CacheEntryHeap is part of the SOS api surface, but currently
+                // when FEATURE_VIRTUAL_STUB_DISPATCH is not defined, the CacheEntryHeap is not created
+                // so its commented out in that situation, but is not considered to be a E_INVALIDARG.
                 pLoaderHeap = pVcsMgr->cache_entry_heap;
+#endif // FEATURE_VIRTUAL_STUB_DISPATCH
                 break;
 
             default:
                 hr = E_INVALIDARG;
         }
 
-        if (SUCCEEDED(hr))
+        if (SUCCEEDED(hr) && (pLoaderHeap != NULL))
         {
             hr = TraverseLoaderHeapBlock(pLoaderHeap->m_pFirstBlock, pFunc);
         }
@@ -3670,7 +3675,9 @@ static const char *LoaderAllocatorLoaderHeapNames[] =
     "FixupPrecodeHeap",
     "NewStubPrecodeHeap",
     "IndcellHeap",
+#ifdef FEATURE_VIRTUAL_STUB_DISPATCH
     "CacheEntryHeap",
+#endif // FEATURE_VIRTUAL_STUB_DISPATCH
 };
 
 
@@ -3714,7 +3721,9 @@ HRESULT ClrDataAccess::GetLoaderAllocatorHeaps(CLRDATA_ADDRESS loaderAllocatorAd
             else
             {
                 pLoaderHeaps[i++] = HOST_CDADDR(pVcsMgr->indcell_heap);
+#ifdef FEATURE_VIRTUAL_STUB_DISPATCH
                 pLoaderHeaps[i++] = HOST_CDADDR(pVcsMgr->cache_entry_heap);
+#endif // FEATURE_VIRTUAL_STUB_DISPATCH
             }
 
             // All of the above are "LoaderHeap" and not the ExplicitControl version.

src/coreclr/inc/CrstTypes.def

@@ -529,3 +529,6 @@ End
 Crst PerfMap
     AcquiredAfter CodeVersioning AssemblyList
 End
+
+Crst InterfaceDispatchGlobalLists
+End

src/coreclr/inc/clrconfigvalues.h

@@ -581,6 +581,7 @@ RETAIL_CONFIG_DWORD_INFO(EXTERNAL_VirtualCallStubLogging, W("VirtualCallStubLogg
 CONFIG_DWORD_INFO(INTERNAL_VirtualCallStubMissCount, W("VirtualCallStubMissCount"), 100, "Used only when STUB_LOGGING is defined, which by default is not.")
 CONFIG_DWORD_INFO(INTERNAL_VirtualCallStubResetCacheCounter, W("VirtualCallStubResetCacheCounter"), 0, "Used only when STUB_LOGGING is defined, which by default is not.")
 CONFIG_DWORD_INFO(INTERNAL_VirtualCallStubResetCacheIncr, W("VirtualCallStubResetCacheIncr"), 0, "Used only when STUB_LOGGING is defined, which by default is not.")
+CONFIG_DWORD_INFO(INTERNAL_UseCachedInterfaceDispatch, W("UseCachedInterfaceDispatch"), 0, "If cached interface dispatch is compiled in, use that instead of virtual stub dispatch")
 
 ///
 /// Watson

src/coreclr/inc/crsttypes_generated.h

@@ -59,66 +59,67 @@ enum CrstType
     CrstILStubGen = 41,
     CrstInlineTrackingMap = 42,
     CrstInstMethodHashTable = 43,
-    CrstInterop = 44,
-    CrstInteropData = 45,
-    CrstIsJMCMethod = 46,
-    CrstISymUnmanagedReader = 47,
-    CrstJit = 48,
-    CrstJitInlineTrackingMap = 49,
-    CrstJitPatchpoint = 50,
-    CrstJumpStubCache = 51,
-    CrstLeafLock = 52,
-    CrstListLock = 53,
-    CrstLoaderAllocator = 54,
-    CrstLoaderAllocatorReferences = 55,
-    CrstLoaderHeap = 56,
-    CrstManagedObjectWrapperMap = 57,
-    CrstMethodDescBackpatchInfoTracker = 58,
-    CrstMethodTableExposedObject = 59,
-    CrstModule = 60,
-    CrstModuleLookupTable = 61,
-    CrstMulticoreJitHash = 62,
-    CrstMulticoreJitManager = 63,
-    CrstNativeImageEagerFixups = 64,
-    CrstNativeImageLoad = 65,
-    CrstNotifyGdb = 66,
-    CrstPEImage = 67,
-    CrstPendingTypeLoadEntry = 68,
-    CrstPerfMap = 69,
-    CrstPgoData = 70,
-    CrstPinnedByrefValidation = 71,
-    CrstPinnedHeapHandleTable = 72,
-    CrstProfilerGCRefDataFreeList = 73,
-    CrstProfilingAPIStatus = 74,
-    CrstRCWCache = 75,
-    CrstRCWCleanupList = 76,
-    CrstReadyToRunEntryPointToMethodDescMap = 77,
-    CrstReflection = 78,
-    CrstReJITGlobalRequest = 79,
-    CrstRetThunkCache = 80,
-    CrstSigConvert = 81,
-    CrstSingleUseLock = 82,
-    CrstStressLog = 83,
-    CrstStubCache = 84,
-    CrstStubDispatchCache = 85,
-    CrstSyncBlockCache = 86,
-    CrstSyncHashLock = 87,
-    CrstSystemDomain = 88,
-    CrstSystemDomainDelayedUnloadList = 89,
-    CrstThreadIdDispenser = 90,
-    CrstThreadLocalStorageLock = 91,
-    CrstThreadStore = 92,
-    CrstTieredCompilation = 93,
-    CrstTypeEquivalenceMap = 94,
-    CrstTypeIDMap = 95,
-    CrstUMEntryThunkCache = 96,
-    CrstUMEntryThunkFreeListLock = 97,
-    CrstUniqueStack = 98,
-    CrstUnresolvedClassLock = 99,
-    CrstUnwindInfoTableLock = 100,
-    CrstVSDIndirectionCellLock = 101,
-    CrstWrapperTemplate = 102,
-    kNumberOfCrstTypes = 103
+    CrstInterfaceDispatchGlobalLists = 44,
+    CrstInterop = 45,
+    CrstInteropData = 46,
+    CrstIsJMCMethod = 47,
+    CrstISymUnmanagedReader = 48,
+    CrstJit = 49,
+    CrstJitInlineTrackingMap = 50,
+    CrstJitPatchpoint = 51,
+    CrstJumpStubCache = 52,
+    CrstLeafLock = 53,
+    CrstListLock = 54,
+    CrstLoaderAllocator = 55,
+    CrstLoaderAllocatorReferences = 56,
+    CrstLoaderHeap = 57,
+    CrstManagedObjectWrapperMap = 58,
+    CrstMethodDescBackpatchInfoTracker = 59,
+    CrstMethodTableExposedObject = 60,
+    CrstModule = 61,
+    CrstModuleLookupTable = 62,
+    CrstMulticoreJitHash = 63,
+    CrstMulticoreJitManager = 64,
+    CrstNativeImageEagerFixups = 65,
+    CrstNativeImageLoad = 66,
+    CrstNotifyGdb = 67,
+    CrstPEImage = 68,
+    CrstPendingTypeLoadEntry = 69,
+    CrstPerfMap = 70,
+    CrstPgoData = 71,
+    CrstPinnedByrefValidation = 72,
+    CrstPinnedHeapHandleTable = 73,
+    CrstProfilerGCRefDataFreeList = 74,
+    CrstProfilingAPIStatus = 75,
+    CrstRCWCache = 76,
+    CrstRCWCleanupList = 77,
+    CrstReadyToRunEntryPointToMethodDescMap = 78,
+    CrstReflection = 79,
+    CrstReJITGlobalRequest = 80,
+    CrstRetThunkCache = 81,
+    CrstSigConvert = 82,
+    CrstSingleUseLock = 83,
+    CrstStressLog = 84,
+    CrstStubCache = 85,
+    CrstStubDispatchCache = 86,
+    CrstSyncBlockCache = 87,
+    CrstSyncHashLock = 88,
+    CrstSystemDomain = 89,
+    CrstSystemDomainDelayedUnloadList = 90,
+    CrstThreadIdDispenser = 91,
+    CrstThreadLocalStorageLock = 92,
+    CrstThreadStore = 93,
+    CrstTieredCompilation = 94,
+    CrstTypeEquivalenceMap = 95,
+    CrstTypeIDMap = 96,
+    CrstUMEntryThunkCache = 97,
+    CrstUMEntryThunkFreeListLock = 98,
+    CrstUniqueStack = 99,
+    CrstUnresolvedClassLock = 100,
+    CrstUnwindInfoTableLock = 101,
+    CrstVSDIndirectionCellLock = 102,
+    CrstWrapperTemplate = 103,
+    kNumberOfCrstTypes = 104
 };
 
 #endif // __CRST_TYPES_INCLUDED
@@ -173,6 +174,7 @@ int g_rgCrstLevelMap[] =
     6,          // CrstILStubGen
     2,          // CrstInlineTrackingMap
     18,         // CrstInstMethodHashTable
+    0,          // CrstInterfaceDispatchGlobalLists
     21,         // CrstInterop
     9,          // CrstInteropData
     0,          // CrstIsJMCMethod
@@ -281,6 +283,7 @@ LPCSTR g_rgCrstNameMap[] =
     "CrstILStubGen",
     "CrstInlineTrackingMap",
     "CrstInstMethodHashTable",
+    "CrstInterfaceDispatchGlobalLists",
     "CrstInterop",
     "CrstInteropData",
     "CrstIsJMCMethod",

src/coreclr/jit/compiler.h

@@ -8408,16 +8408,8 @@ class Compiler
             reg     = REG_EAX;
             regMask = RBM_EAX;
 #elif defined(TARGET_AMD64)
-            if (isNativeAOT)
-            {
-                reg     = REG_R10;
-                regMask = RBM_R10;
-            }
-            else
-            {
-                reg     = REG_R11;
-                regMask = RBM_R11;
-            }
+            reg     = REG_R11;
+            regMask = RBM_R11;
 #elif defined(TARGET_ARM)
             if (isNativeAOT)
             {

src/coreclr/minipal/minipal.h

@@ -76,3 +76,16 @@ class VMToOSInterface
     //  true if it succeeded, false if it failed
     static bool ReleaseRWMapping(void* pStart, size_t size);
 };
+
+#if defined(HOST_64BIT) && defined(FEATURE_CACHED_INTERFACE_DISPATCH)
+EXTERN_C uint8_t _InterlockedCompareExchange128(int64_t volatile *, int64_t, int64_t, int64_t *);
+
+#if defined(HOST_WINDOWS)
+#pragma intrinsic(_InterlockedCompareExchange128)
+#endif
+
+FORCEINLINE uint8_t PalInterlockedCompareExchange128(_Inout_ int64_t volatile *pDst, int64_t iValueHigh, int64_t iValueLow, int64_t *pComparandAndResult)
+{
+    return _InterlockedCompareExchange128(pDst, iValueHigh, iValueLow, pComparandAndResult);
+}
+#endif // defined(HOST_64BIT) && defined(FEATURE_CACHED_INTERFACE_DISPATCH)

src/coreclr/nativeaot/CMakeLists.txt

@@ -23,7 +23,7 @@ if(CLR_CMAKE_HOST_UNIX)
   endif(CLR_CMAKE_TARGET_APPLE)
 
   if(CLR_CMAKE_TARGET_ARCH_AMD64 OR CLR_CMAKE_TARGET_ARCH_I386)
-    # Allow 16 byte compare-exchange
+    # Allow 16 byte compare-exchange (cmpxchg16b)
     add_compile_options(-mcx16)
   endif(CLR_CMAKE_TARGET_ARCH_AMD64 OR CLR_CMAKE_TARGET_ARCH_I386)
 endif (CLR_CMAKE_HOST_UNIX)

src/coreclr/nativeaot/Runtime/CMakeLists.txt

@@ -1,9 +1,11 @@
 set(GC_DIR ../../gc)
+set(RUNTIME_DIR ../../runtime)
 
 set(COMMON_RUNTIME_SOURCES
     allocheap.cpp
     rhassert.cpp
-    CachedInterfaceDispatch.cpp
+    ${RUNTIME_DIR}/CachedInterfaceDispatch.cpp
+    CachedInterfaceDispatchAot.cpp
     Crst.cpp
     DebugHeader.cpp
     MethodTable.cpp
@@ -76,6 +78,7 @@ include_directories(.)
 include_directories(${GC_DIR})
 include_directories(${GC_DIR}/env)
 include_directories(${CMAKE_CURRENT_BINARY_DIR}/eventpipe/inc)
+include_directories(${RUNTIME_DIR})
 
 if (WIN32)
   set(GC_HEADERS
@@ -208,11 +211,17 @@ list(APPEND RUNTIME_SOURCES_ARCH_ASM
   ${ARCH_SOURCES_DIR}/MiscStubs.${ASM_SUFFIX}
   ${ARCH_SOURCES_DIR}/PInvoke.${ASM_SUFFIX}
   ${ARCH_SOURCES_DIR}/InteropThunksHelpers.${ASM_SUFFIX}
-  ${ARCH_SOURCES_DIR}/StubDispatch.${ASM_SUFFIX}
+  ${RUNTIME_DIR}/${ARCH_SOURCES_DIR}/StubDispatch.${ASM_SUFFIX}
   ${ARCH_SOURCES_DIR}/UniversalTransition.${ASM_SUFFIX}
   ${ARCH_SOURCES_DIR}/WriteBarriers.${ASM_SUFFIX}
 )
 
+if (CLR_CMAKE_TARGET_ARCH_AMD64 OR CLR_CMAKE_TARGET_ARCH_ARM64)
+  list(APPEND RUNTIME_SOURCES_ARCH_ASM
+    ${ARCH_SOURCES_DIR}/CachedInterfaceDispatchAot.${ASM_SUFFIX}
+  )
+endif ()
+
 # Add architecture specific folder for looking up headers.
 convert_to_absolute_path(ARCH_SOURCES_DIR ${ARCH_SOURCES_DIR})
 include_directories(${ARCH_SOURCES_DIR})
@@ -289,7 +298,7 @@ if (CLR_CMAKE_TARGET_UNIX)
 
 endif(CLR_CMAKE_TARGET_UNIX)
 
-set(RUNTIME_DIR ${CMAKE_CURRENT_SOURCE_DIR})
+set(NATIVEAOT_RUNTIME_DIR ${CMAKE_CURRENT_SOURCE_DIR})
 
 list(APPEND COMMON_RUNTIME_SOURCES ${GC_HEADERS})