Windows/Arm64: Use 8.1 atomic instructions if they are available (#70921

) * Use Fast compareexchange, acquire/release * working windows version * remove printf * some more #ifdef * fix some #ifdef * optimize EnterObjMonitorHelperSpin * Remove FastInterlockedCompareExchange64 for now
dotnet · Jun 20, 2022 · 6be8d27 · 6be8d27
1 parent 9876d8c
commit 6be8d27
Show file tree

Hide file tree

Showing 6 changed files with 103 additions and 0 deletions.
diff --git a/src/coreclr/utilcode/util.cpp b/src/coreclr/utilcode/util.cpp
@@ -23,6 +23,13 @@
 
 #ifndef DACCESS_COMPILE
 UINT32 g_nClrInstanceId = 0;
+
+#if defined(TARGET_WINDOWS) && defined(TARGET_ARM64)
+// Flag to check if atomics feature is available on
+// the machine
+bool g_arm64_atomics_present = false;
+#endif
+
 #endif //!DACCESS_COMPILE
 
 //*****************************************************************************

diff --git a/src/coreclr/vm/codeman.cpp b/src/coreclr/vm/codeman.cpp
@@ -1566,6 +1566,7 @@ void EEJitManager::SetCpuInfo()
     if (IsProcessorFeaturePresent(PF_ARM_V81_ATOMIC_INSTRUCTIONS_AVAILABLE))
     {
         CPUCompileFlags.Set(InstructionSet_Atomics);
+        g_arm64_atomics_present = true;
     }
 
     // PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE (43)

diff --git a/src/coreclr/vm/syncblk.cpp b/src/coreclr/vm/syncblk.cpp
@@ -1652,7 +1652,11 @@ AwareLock::EnterHelperResult ObjHeader::EnterObjMonitorHelperSpin(Thread* pCurTh
             }
 
             LONG newValue = oldValue | tid;
+#if defined(TARGET_WINDOWS) && defined(TARGET_ARM64)
+            if (FastInterlockedCompareExchangeAcquire((LONG*)&m_SyncBlockValue, newValue, oldValue) == oldValue)
+#else
             if (InterlockedCompareExchangeAcquire((LONG*)&m_SyncBlockValue, newValue, oldValue) == oldValue)
+#endif
             {
                 return AwareLock::EnterHelperResult_Entered;
             }

diff --git a/src/coreclr/vm/syncblk.h b/src/coreclr/vm/syncblk.h
@@ -382,13 +382,21 @@ class AwareLock
         LockState CompareExchange(LockState toState, LockState fromState)
         {
             LIMITED_METHOD_CONTRACT;
+#if defined(TARGET_WINDOWS) && defined(TARGET_ARM64)
+            return (UINT32)FastInterlockedCompareExchange((LONG *)&m_state, (LONG)toState, (LONG)fromState);
+#else
             return (UINT32)InterlockedCompareExchange((LONG *)&m_state, (LONG)toState, (LONG)fromState);
+#endif
         }
 
         LockState CompareExchangeAcquire(LockState toState, LockState fromState)
         {
             LIMITED_METHOD_CONTRACT;
+#if defined(TARGET_WINDOWS) && defined(TARGET_ARM64)
+            return (UINT32)FastInterlockedCompareExchangeAcquire((LONG *)&m_state, (LONG)toState, (LONG)fromState);
+#else
             return (UINT32)InterlockedCompareExchangeAcquire((LONG *)&m_state, (LONG)toState, (LONG)fromState);
+#endif
         }
 
     public:

diff --git a/src/coreclr/vm/syncblk.inl b/src/coreclr/vm/syncblk.inl
@@ -602,7 +602,11 @@ FORCEINLINE AwareLock::EnterHelperResult ObjHeader::EnterObjMonitorHelper(Thread
         }
 
         LONG newValue = oldValue | tid;
+#if defined(TARGET_WINDOWS) && defined(TARGET_ARM64)
+        if (FastInterlockedCompareExchangeAcquire((LONG*)&m_SyncBlockValue, newValue, oldValue) == oldValue)
+#else   
         if (InterlockedCompareExchangeAcquire((LONG*)&m_SyncBlockValue, newValue, oldValue) == oldValue)
+#endif
         {
             return AwareLock::EnterHelperResult_Entered;
         }
@@ -650,7 +654,11 @@ FORCEINLINE AwareLock::EnterHelperResult ObjHeader::EnterObjMonitorHelper(Thread
         return AwareLock::EnterHelperResult_UseSlowPath;
     }
 
+#if defined(TARGET_WINDOWS) && defined(TARGET_ARM64)
+    if (FastInterlockedCompareExchangeAcquire((LONG*)&m_SyncBlockValue, newValue, oldValue) == oldValue)
+#else
     if (InterlockedCompareExchangeAcquire((LONG*)&m_SyncBlockValue, newValue, oldValue) == oldValue)
+#endif
     {
         return AwareLock::EnterHelperResult_Entered;
     }
@@ -723,7 +731,12 @@ FORCEINLINE AwareLock::LeaveHelperAction ObjHeader::LeaveObjMonitorHelper(Thread
         {
             // We are leaving the lock
             DWORD newValue = (syncBlockValue & (~SBLK_MASK_LOCK_THREADID));
+
+#if defined(TARGET_WINDOWS) && defined(TARGET_ARM64)
+            if (FastInterlockedCompareExchangeRelease((LONG*)&m_SyncBlockValue, newValue, syncBlockValue) != (LONG)syncBlockValue)
+#else
             if (InterlockedCompareExchangeRelease((LONG*)&m_SyncBlockValue, newValue, syncBlockValue) != (LONG)syncBlockValue)
+#endif
             {
                 return AwareLock::LeaveHelperAction_Yield;
             }
@@ -732,7 +745,11 @@ FORCEINLINE AwareLock::LeaveHelperAction ObjHeader::LeaveObjMonitorHelper(Thread
         {
             // recursion and ThinLock
             DWORD newValue = syncBlockValue - SBLK_LOCK_RECLEVEL_INC;
+#if defined(TARGET_WINDOWS) && defined(TARGET_ARM64)
+            if (FastInterlockedCompareExchangeRelease((LONG*)&m_SyncBlockValue, newValue, syncBlockValue) != (LONG)syncBlockValue)
+#else
             if (InterlockedCompareExchangeRelease((LONG*)&m_SyncBlockValue, newValue, syncBlockValue) != (LONG)syncBlockValue)
+#endif
             {
                 return AwareLock::LeaveHelperAction_Yield;
             }

diff --git a/src/coreclr/vm/util.hpp b/src/coreclr/vm/util.hpp
@@ -25,6 +25,14 @@
 #define MAX_CACHE_LINE_SIZE 64
 #endif
 
+#ifndef DACCESS_COMPILE
+#if defined(TARGET_WINDOWS) && defined(TARGET_ARM64)
+// Flag to check if atomics feature is available on
+// the machine
+extern bool g_arm64_atomics_present;
+#endif
+#endif
+
 #ifndef TARGET_UNIX
 // Copied from malloc.h: don't want to bring in the whole header file.
 void * __cdecl _alloca(size_t);
@@ -71,6 +79,64 @@ BOOL inline FitsInU4(unsigned __int64 val)
     return val == (unsigned __int64)(unsigned __int32)val;
 }
 
+#if defined(DACCESS_COMPILE)
+#define FastInterlockedCompareExchange InterlockedCompareExchange
+#define FastInterlockedCompareExchangeAcquire InterlockedCompareExchangeAcquire
+#define FastInterlockedCompareExchangeRelease InterlockedCompareExchangeRelease
+#else
+
+#if defined(TARGET_WINDOWS) && defined(TARGET_ARM64)
+
+FORCEINLINE LONG  FastInterlockedCompareExchange(
+    LONG volatile *Destination,
+    LONG Exchange,
+    LONG Comperand)
+{
+    if (g_arm64_atomics_present)
+    {
+        return (LONG) __casal32((unsigned __int32*) Destination, (unsigned  __int32)Comperand, (unsigned __int32)Exchange);
+    }
+    else
+    {
+        return InterlockedCompareExchange(Destination, Exchange, Comperand);
+    }
+}
+
+FORCEINLINE LONG FastInterlockedCompareExchangeAcquire(
+  IN OUT LONG volatile *Destination,
+  IN LONG Exchange,
+  IN LONG Comperand
+)
+{
+    if (g_arm64_atomics_present)
+    {
+        return (LONG) __casa32((unsigned __int32*) Destination, (unsigned  __int32)Comperand, (unsigned __int32)Exchange);
+    }
+    else
+    {
+        return InterlockedCompareExchangeAcquire(Destination, Exchange, Comperand);
+    }
+}
+
+FORCEINLINE LONG FastInterlockedCompareExchangeRelease(
+  IN OUT LONG volatile *Destination,
+  IN LONG Exchange,
+  IN LONG Comperand
+)
+{
+    if (g_arm64_atomics_present)
+    {
+        return (LONG) __casl32((unsigned __int32*) Destination, (unsigned  __int32)Comperand, (unsigned __int32)Exchange);
+    }
+    else
+    {
+        return InterlockedCompareExchangeRelease(Destination, Exchange, Comperand);
+    }
+}
+
+#endif // defined(TARGET_WINDOWS) && defined(TARGET_ARM64)
+
+#endif //defined(DACCESS_COMPILE)
 
 
 //************************************************************************