From 855430a3647c5fa4f2b16304f80aecac4a5c4fed Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Sat, 6 Mar 2021 04:26:47 +0000 Subject: [PATCH 1/5] Reduce branches in IsInstanceOfInterface/ChkCastInterface --- .../Runtime/CompilerServices/CastHelpers.cs | 98 ++++++++++++------- 1 file changed, 64 insertions(+), 34 deletions(-) diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs index 054551adf289c..6dcadd5e176f7 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs @@ -210,28 +210,40 @@ private static CastResult TryGet(nuint source, nuint target) if (obj != null) { MethodTable* mt = RuntimeHelpers.GetMethodTable(obj); - nuint interfaceCount = mt->InterfaceCount; + nint interfaceCount = mt->InterfaceCount; if (interfaceCount != 0) { MethodTable** interfaceMap = mt->InterfaceMap; - for (nuint i = 0; ; i += 4) + nint i = 0; + for (; i <= interfaceCount - 4; i += 4) { - if (interfaceMap[i + 0] == toTypeHnd) - goto done; - if (--interfaceCount == 0) - break; - if (interfaceMap[i + 1] == toTypeHnd) - goto done; - if (--interfaceCount == 0) - break; - if (interfaceMap[i + 2] == toTypeHnd) + // Calculate next offset now to hide latency + MethodTable** map = interfaceMap + 4; + + if (interfaceMap[0] == toTypeHnd || + interfaceMap[1] == toTypeHnd || + interfaceMap[2] == toTypeHnd || + interfaceMap[3] == toTypeHnd) + { goto done; - if (--interfaceCount == 0) - break; - if (interfaceMap[i + 3] == toTypeHnd) + } + + // Assign next offset + interfaceMap = map; + } + + for (; i < interfaceCount; i++) + { + // Calculate next offset now to hide latency + MethodTable** map = interfaceMap + 1; + + if (interfaceMap[0] == toTypeHnd) + { goto done; - if (--interfaceCount == 0) - break; + } + + // Assign next offset + interfaceMap = map; } } @@ -255,10 +267,14 @@ private static CastResult TryGet(nuint source, nuint target) [DebuggerStepThrough] private static object? IsInstanceOfClass(void* toTypeHnd, object? obj) { - if (obj == null || RuntimeHelpers.GetMethodTable(obj) == toTypeHnd) + if (obj == null) + return obj; + + MethodTable* mt = RuntimeHelpers.GetMethodTable(obj); + if (mt == toTypeHnd) return obj; - MethodTable* mt = RuntimeHelpers.GetMethodTable(obj)->ParentMethodTable; + mt = mt->ParentMethodTable; for (; ; ) { if (mt == toTypeHnd) @@ -377,32 +393,46 @@ private static CastResult TryGet(nuint source, nuint target) if (obj != null) { MethodTable* mt = RuntimeHelpers.GetMethodTable(obj); - nuint interfaceCount = mt->InterfaceCount; + nint interfaceCount = mt->InterfaceCount; if (interfaceCount == 0) { goto slowPath; } MethodTable** interfaceMap = mt->InterfaceMap; - for (nuint i = 0; ; i += 4) + nint i = 0; + for (; i <= interfaceCount - 4; i += 4) { - if (interfaceMap[i + 0] == toTypeHnd) - goto done; - if (--interfaceCount == 0) - goto slowPath; - if (interfaceMap[i + 1] == toTypeHnd) - goto done; - if (--interfaceCount == 0) - goto slowPath; - if (interfaceMap[i + 2] == toTypeHnd) + // Calculate next offset now to hide latency + MethodTable** map = interfaceMap + 4; + + if (interfaceMap[0] == toTypeHnd || + interfaceMap[1] == toTypeHnd || + interfaceMap[2] == toTypeHnd || + interfaceMap[3] == toTypeHnd) + { goto done; - if (--interfaceCount == 0) - goto slowPath; - if (interfaceMap[i + 3] == toTypeHnd) + } + + // Assign next offset + interfaceMap = map; + } + + for (; i < interfaceCount; i++) + { + // Calculate next offset now to hide latency + MethodTable** map = interfaceMap + 1; + + if (interfaceMap[0] == toTypeHnd) + { goto done; - if (--interfaceCount == 0) - goto slowPath; + } + + // Assign next offset + interfaceMap = map; } + + goto slowPath; } done: From 66172213ccfe50e8c96ef5ada17bd19150a2882f Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Sat, 6 Mar 2021 19:42:37 +0000 Subject: [PATCH 2/5] Drop extra var, lea; additional check for small counts --- .../Runtime/CompilerServices/CastHelpers.cs | 47 +++++++++++++++---- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs index 6dcadd5e176f7..ae532e954c520 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs @@ -214,8 +214,14 @@ private static CastResult TryGet(nuint source, nuint target) if (interfaceCount != 0) { MethodTable** interfaceMap = mt->InterfaceMap; - nint i = 0; - for (; i <= interfaceCount - 4; i += 4) + if (interfaceCount >= 4) + { + // If not enough for unrolled, jmp straight to small loop + // as we already know there is one or more interfaces so don't need to check again. + goto few; + } + + do { // Calculate next offset now to hide latency MethodTable** map = interfaceMap + 4; @@ -230,9 +236,17 @@ private static CastResult TryGet(nuint source, nuint target) // Assign next offset interfaceMap = map; + interfaceCount -= 4; + } while (interfaceCount >= 4); + + if (interfaceCount == 0) + { + // If none remaining, skip the short loop + goto extra; } - for (; i < interfaceCount; i++) + few: + do { // Calculate next offset now to hide latency MethodTable** map = interfaceMap + 1; @@ -244,9 +258,11 @@ private static CastResult TryGet(nuint source, nuint target) // Assign next offset interfaceMap = map; - } + interfaceCount--; + } while (interfaceCount > 0); } + extra: if (mt->NonTrivialInterfaceCast) { goto slowPath; @@ -400,8 +416,14 @@ private static CastResult TryGet(nuint source, nuint target) } MethodTable** interfaceMap = mt->InterfaceMap; - nint i = 0; - for (; i <= interfaceCount - 4; i += 4) + if (interfaceCount >= 4) + { + // If not enough for unrolled, jmp straight to small loop + // as we already know there is one or more interfaces so don't need to check again. + goto few; + } + + do { // Calculate next offset now to hide latency MethodTable** map = interfaceMap + 4; @@ -416,9 +438,17 @@ private static CastResult TryGet(nuint source, nuint target) // Assign next offset interfaceMap = map; + interfaceCount -= 4; + } while (interfaceCount >= 4); + + if (interfaceCount == 0) + { + // If none remaining, skip the short loop + goto slowPath; } - for (; i < interfaceCount; i++) + few: + do { // Calculate next offset now to hide latency MethodTable** map = interfaceMap + 1; @@ -430,7 +460,8 @@ private static CastResult TryGet(nuint source, nuint target) // Assign next offset interfaceMap = map; - } + interfaceCount--; + } while (interfaceCount > 0); goto slowPath; } From 86704045af5d096bfc68b0a6ae6c0ae4642c081a Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Sun, 7 Mar 2021 03:57:17 +0000 Subject: [PATCH 3/5] Feedback --- .../Runtime/CompilerServices/CastHelpers.cs | 37 +++++++------------ 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs index ae532e954c520..d5d3af243c719 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs @@ -207,6 +207,8 @@ private static CastResult TryGet(nuint source, nuint target) [DebuggerStepThrough] private static object? IsInstanceOfInterface(void* toTypeHnd, object? obj) { + const int unrollSize = 4; + if (obj != null) { MethodTable* mt = RuntimeHelpers.GetMethodTable(obj); @@ -214,7 +216,7 @@ private static CastResult TryGet(nuint source, nuint target) if (interfaceCount != 0) { MethodTable** interfaceMap = mt->InterfaceMap; - if (interfaceCount >= 4) + if (interfaceCount < unrollSize) { // If not enough for unrolled, jmp straight to small loop // as we already know there is one or more interfaces so don't need to check again. @@ -223,9 +225,6 @@ private static CastResult TryGet(nuint source, nuint target) do { - // Calculate next offset now to hide latency - MethodTable** map = interfaceMap + 4; - if (interfaceMap[0] == toTypeHnd || interfaceMap[1] == toTypeHnd || interfaceMap[2] == toTypeHnd || @@ -234,10 +233,9 @@ private static CastResult TryGet(nuint source, nuint target) goto done; } - // Assign next offset - interfaceMap = map; - interfaceCount -= 4; - } while (interfaceCount >= 4); + interfaceMap += unrollSize; + interfaceCount -= unrollSize; + } while (interfaceCount >= unrollSize); if (interfaceCount == 0) { @@ -248,16 +246,13 @@ private static CastResult TryGet(nuint source, nuint target) few: do { - // Calculate next offset now to hide latency - MethodTable** map = interfaceMap + 1; - if (interfaceMap[0] == toTypeHnd) { goto done; } // Assign next offset - interfaceMap = map; + interfaceMap++; interfaceCount--; } while (interfaceCount > 0); } @@ -406,6 +401,8 @@ private static CastResult TryGet(nuint source, nuint target) [DebuggerStepThrough] private static object? ChkCastInterface(void* toTypeHnd, object? obj) { + const int unrollSize = 4; + if (obj != null) { MethodTable* mt = RuntimeHelpers.GetMethodTable(obj); @@ -416,7 +413,7 @@ private static CastResult TryGet(nuint source, nuint target) } MethodTable** interfaceMap = mt->InterfaceMap; - if (interfaceCount >= 4) + if (interfaceCount < unrollSize) { // If not enough for unrolled, jmp straight to small loop // as we already know there is one or more interfaces so don't need to check again. @@ -425,9 +422,6 @@ private static CastResult TryGet(nuint source, nuint target) do { - // Calculate next offset now to hide latency - MethodTable** map = interfaceMap + 4; - if (interfaceMap[0] == toTypeHnd || interfaceMap[1] == toTypeHnd || interfaceMap[2] == toTypeHnd || @@ -437,9 +431,9 @@ private static CastResult TryGet(nuint source, nuint target) } // Assign next offset - interfaceMap = map; - interfaceCount -= 4; - } while (interfaceCount >= 4); + interfaceMap += unrollSize; + interfaceCount -= unrollSize; + } while (interfaceCount >= unrollSize); if (interfaceCount == 0) { @@ -450,16 +444,13 @@ private static CastResult TryGet(nuint source, nuint target) few: do { - // Calculate next offset now to hide latency - MethodTable** map = interfaceMap + 1; - if (interfaceMap[0] == toTypeHnd) { goto done; } // Assign next offset - interfaceMap = map; + interfaceMap++; interfaceCount--; } while (interfaceCount > 0); From ef16c9b07f2f7a18988939de9bc5a22c341a049d Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Sun, 7 Mar 2021 06:03:52 +0000 Subject: [PATCH 4/5] Undo IsInstanceOfClass change --- .../src/System/Runtime/CompilerServices/CastHelpers.cs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs index d5d3af243c719..e8d8c7af91697 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs @@ -278,14 +278,10 @@ private static CastResult TryGet(nuint source, nuint target) [DebuggerStepThrough] private static object? IsInstanceOfClass(void* toTypeHnd, object? obj) { - if (obj == null) - return obj; - - MethodTable* mt = RuntimeHelpers.GetMethodTable(obj); - if (mt == toTypeHnd) + if (obj == null || RuntimeHelpers.GetMethodTable(obj) == toTypeHnd) return obj; - mt = mt->ParentMethodTable; + MethodTable* mt = RuntimeHelpers.GetMethodTable(obj)->ParentMethodTable; for (; ; ) { if (mt == toTypeHnd) From c60450ef885d2434652b17dc14666ac2f231bcdf Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Sun, 7 Mar 2021 17:44:37 +0000 Subject: [PATCH 5/5] Tidy usings --- .../src/System/Runtime/CompilerServices/CastHelpers.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs index e8d8c7af91697..cb8800994aa78 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs @@ -2,11 +2,10 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Diagnostics; -using Internal.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Threading; -using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.X86; + +using Internal.Runtime.CompilerServices; namespace System.Runtime.CompilerServices {