Skip to content

Commit

Permalink
[NativeAOT] Using the same CastCache implementation as in CoreClr (#8…
Browse files Browse the repository at this point in the history 
…4430)

* Getter

* different limits on debug/release

* tweaks

* remove now unnecessary CrstCastCache

* implement flushing

* move coreclr castcache to a separate file

* Unified CastCache implementation

* comments and cleanups

* couple more cleanups

* trivial implementation of the cast cache for the Test.Corlib

* use Numerics.BitOperations for bit math
  • Loading branch information
@VSadov
VSadov authored Apr 7, 2023
1 parent 84065a8 commit 38b81ba
Show file tree
Hide file tree
Showing 12 changed files with 536 additions and 479 deletions.
153 changes: 6 additions & 147 deletions src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,155 +3,14 @@

using System.Diagnostics;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;

namespace System.Runtime.CompilerServices
{
internal static unsafe class CastHelpers
{
private static int[]? s_table;

[DebuggerDisplay("Source = {_source}; Target = {_targetAndResult & ~1}; Result = {_targetAndResult & 1}; VersionNum = {_version & ((1 << 29) - 1)}; Distance = {_version >> 29};")]
[StructLayout(LayoutKind.Sequential)]
private struct CastCacheEntry
{
// version has the following structure:
// [ distance:3bit | versionNum:29bit ]
//
// distance is how many iterations the entry is from it ideal position.
// we use that for preemption.
//
// versionNum is a monotonicaly increasing numerical tag.
// Writer "claims" entry by atomically incrementing the tag. Thus odd number indicates an entry in progress.
// Upon completion of adding an entry the tag is incremented again making it even. Even number indicates a complete entry.
//
// Readers will read the version twice before and after retrieving the entry.
// To have a usable entry both reads must yield the same even version.
//
internal int _version;
internal nuint _source;
// pointers have unused lower bits due to alignment, we use one for the result
internal nuint _targetAndResult;
};

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int KeyToBucket(ref int tableData, nuint source, nuint target)
{
// upper bits of addresses do not vary much, so to reduce loss due to cancelling out,
// we do `rotl(source, <half-size>) ^ target` for mixing inputs.
// then we use fibonacci hashing to reduce the value to desired size.

int hashShift = HashShift(ref tableData);
#if TARGET_64BIT
ulong hash = BitOperations.RotateLeft((ulong)source, 32) ^ (ulong)target;
return (int)((hash * 11400714819323198485ul) >> hashShift);
#else
uint hash = BitOperations.RotateLeft((uint)source, 16) ^ (uint)target;
return (int)((hash * 2654435769u) >> hashShift);
#endif
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static ref int TableData(int[] table)
{
// element 0 is used for embedded aux data
return ref MemoryMarshal.GetArrayDataReference(table);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static ref CastCacheEntry Element(ref int tableData, int index)
{
// element 0 is used for embedded aux data, skip it
return ref Unsafe.Add(ref Unsafe.As<int, CastCacheEntry>(ref tableData), index + 1);
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int HashShift(ref int tableData)
{
return tableData;
}

// TableMask is "size - 1"
// we need that more often that we need size
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int TableMask(ref int tableData)
{
return Unsafe.Add(ref tableData, 1);
}

private enum CastResult
{
CannotCast = 0,
CanCast = 1,
MaybeCast = 2
}

// NOTE!!
// This is a copy of C++ implementation in castcache.cpp
// Keep the copies, if possible, in sync.
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static CastResult TryGet(nuint source, nuint target)
{
const int BUCKET_SIZE = 8;

// table is initialized and updated by native code that guarantees it is not null.
ref int tableData = ref TableData(s_table!);

int index = KeyToBucket(ref tableData, source, target);
for (int i = 0; i < BUCKET_SIZE;)
{
ref CastCacheEntry pEntry = ref Element(ref tableData, index);

// must read in this order: version -> [entry parts] -> version
// if version is odd or changes, the entry is inconsistent and thus ignored
int version = Volatile.Read(ref pEntry._version);
nuint entrySource = pEntry._source;

// mask the lower version bit to make it even.
// This way we can check if version is odd or changing in just one compare.
version &= ~1;

if (entrySource == source)
{
nuint entryTargetAndResult = pEntry._targetAndResult;
// target never has its lower bit set.
// a matching entryTargetAndResult would the have same bits, except for the lowest one, which is the result.
entryTargetAndResult ^= target;
if (entryTargetAndResult <= 1)
{
// make sure 'version' is loaded after 'source' and 'targetAndResults'
//
// We can either:
// - use acquires for both _source and _targetAndResults or
// - issue a load barrier before reading _version
// benchmarks on available hardware show that use of a read barrier is cheaper.
Interlocked.ReadMemoryBarrier();
if (version != pEntry._version)
{
// oh, so close, the entry is in inconsistent state.
// it is either changing or has changed while we were reading.
// treat it as a miss.
break;
}

return (CastResult)entryTargetAndResult;
}
}

if (version == 0)
{
// the rest of the bucket is unclaimed, no point to search further
break;
}

// quadratic reprobe
i++;
index = (index + i) & TableMask(ref tableData);
}
return CastResult.MaybeCast;
}

[MethodImpl(MethodImplOptions.InternalCall)]
private static extern object IsInstanceOfAny_NoCacheLookup(void* toTypeHnd, object obj);

Expand All @@ -177,7 +36,7 @@ private static CastResult TryGet(nuint source, nuint target)
void* mt = RuntimeHelpers.GetMethodTable(obj);
if (mt != toTypeHnd)
{
CastResult result = TryGet((nuint)mt, (nuint)toTypeHnd);
CastResult result = CastCache.TryGet((nuint)mt, (nuint)toTypeHnd);
if (result == CastResult.CanCast)
{
// do nothing
Expand Down Expand Up @@ -327,7 +186,7 @@ private static CastResult TryGet(nuint source, nuint target)
[MethodImpl(MethodImplOptions.NoInlining)]
private static object? IsInstance_Helper(void* toTypeHnd, object obj)
{
CastResult result = TryGet((nuint)RuntimeHelpers.GetMethodTable(obj), (nuint)toTypeHnd);
CastResult result = CastCache.TryGet((nuint)RuntimeHelpers.GetMethodTable(obj), (nuint)toTypeHnd);
if (result == CastResult.CanCast)
{
return obj;
Expand Down Expand Up @@ -356,7 +215,7 @@ private static CastResult TryGet(nuint source, nuint target)
void* mt = RuntimeHelpers.GetMethodTable(obj);
if (mt != toTypeHnd)
{
result = TryGet((nuint)mt, (nuint)toTypeHnd);
result = CastCache.TryGet((nuint)mt, (nuint)toTypeHnd);
if (result != CastResult.CanCast)
{
goto slowPath;
Expand All @@ -380,7 +239,7 @@ private static CastResult TryGet(nuint source, nuint target)
[MethodImpl(MethodImplOptions.NoInlining)]
private static object? ChkCast_Helper(void* toTypeHnd, object obj)
{
CastResult result = TryGet((nuint)RuntimeHelpers.GetMethodTable(obj), (nuint)toTypeHnd);
CastResult result = CastCache.TryGet((nuint)RuntimeHelpers.GetMethodTable(obj), (nuint)toTypeHnd);
if (result == CastResult.CanCast)
{
return obj;
Expand Down Expand Up @@ -597,7 +456,7 @@ private static void StelemRef(Array array, nint index, object? obj)
[MethodImpl(MethodImplOptions.NoInlining)]
private static void StelemRef_Helper(ref object? element, void* elementType, object obj)
{
CastResult result = TryGet((nuint)RuntimeHelpers.GetMethodTable(obj), (nuint)elementType);
CastResult result = CastCache.TryGet((nuint)RuntimeHelpers.GetMethodTable(obj), (nuint)elementType);
if (result == CastResult.CanCast)
{
WriteBarrier(ref element, obj);
Expand Down
8 changes: 0 additions & 8 deletions src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/InternalCalls.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -306,14 +306,6 @@ internal static extern unsafe IntPtr RhpCallPropagateExceptionCallback(
[UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvCdecl) })]
internal static extern void RhpSignalFinalizationComplete();

[DllImport(Redhawk.BaseName)]
[UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvCdecl) })]
internal static extern void RhpAcquireCastCacheLock();

[DllImport(Redhawk.BaseName)]
[UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvCdecl) })]
internal static extern void RhpReleaseCastCacheLock();

[DllImport(Redhawk.BaseName)]
[UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvCdecl) })]
internal static extern ulong RhpGetTickCount64();
Expand Down
Loading

0 comments on commit 38b81ba

Please sign in to comment.