Skip to content

Commit fab69ef

Browse files
EgorBojkotas
EgorBo
and
jkotas
authored
Move memset/memcpy/memzero jit helpers to SpanHelpers (#98623)
--------- Co-authored-by: Jan Kotas <jkotas@microsoft.com>
1 parent 79dd9ba commit fab69ef

File tree

68 files changed

+807
-1204
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

68 files changed

+807
-1204
lines changed

‎docs/design/coreclr/botr/guide-for-porting.md

-6
Original file line numberDiff line numberDiff line change
@@ -413,12 +413,6 @@ Here is an annotated list of the stubs implemented for Unix on Arm64.
413413
Today use of this feature on Unix requires hand-written IL. On Windows
414414
this feature is commonly used by C++/CLI
415415

416-
3. EH Correctness. Some helpers are written in assembly to provide well known
417-
locations for NullReferenceExceptions to be generated out of a SIGSEGV
418-
signal.
419-
420-
1. `JIT_MemSet`, and `JIT_MemCpy` have this requirement
421-
422416
#### cgencpu.h
423417

424418
This header is included by various code in the VM directory. It provides a large

‎src/coreclr/System.Private.CoreLib/src/System/Array.CoreCLR.cs

+2-2
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ private static unsafe void CopyImpl(Array sourceArray, int sourceIndex, Array de
7474
if (pMT->ContainsGCPointers)
7575
Buffer.BulkMoveWithWriteBarrier(ref dst, ref src, byteCount);
7676
else
77-
Buffer.Memmove(ref dst, ref src, byteCount);
77+
SpanHelpers.Memmove(ref dst, ref src, byteCount);
7878

7979
// GC.KeepAlive(sourceArray) not required. pMT kept alive via sourceArray
8080
return;
@@ -184,7 +184,7 @@ private static unsafe void CopyImplUnBoxEachElement(Array sourceArray, int sourc
184184
}
185185
else
186186
{
187-
Buffer.Memmove(ref dest, ref obj.GetRawData(), destSize);
187+
SpanHelpers.Memmove(ref dest, ref obj.GetRawData(), destSize);
188188
}
189189
}
190190
}

‎src/coreclr/System.Private.CoreLib/src/System/Object.CoreCLR.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ protected internal unsafe object MemberwiseClone()
3030
if (RuntimeHelpers.GetMethodTable(clone)->ContainsGCPointers)
3131
Buffer.BulkMoveWithWriteBarrier(ref dst, ref src, byteCount);
3232
else
33-
Buffer.Memmove(ref dst, ref src, byteCount);
33+
SpanHelpers.Memmove(ref dst, ref src, byteCount);
3434

3535
return clone;
3636
}

‎src/coreclr/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshal.CoreCLR.cs

+2-2
Original file line numberDiff line numberDiff line change
@@ -266,7 +266,7 @@ public static unsafe void StructureToPtr(object structure, IntPtr ptr, bool fDel
266266
}
267267
else
268268
{
269-
Buffer.Memmove(ref *(byte*)ptr, ref structure.GetRawData(), size);
269+
SpanHelpers.Memmove(ref *(byte*)ptr, ref structure.GetRawData(), size);
270270
}
271271
}
272272

@@ -291,7 +291,7 @@ private static unsafe void PtrToStructureHelper(IntPtr ptr, object structure, bo
291291
}
292292
else
293293
{
294-
Buffer.Memmove(ref structure.GetRawData(), ref *(byte*)ptr, size);
294+
SpanHelpers.Memmove(ref structure.GetRawData(), ref *(byte*)ptr, size);
295295
}
296296
}
297297

‎src/coreclr/System.Private.CoreLib/src/System/String.CoreCLR.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ internal static unsafe void InternalCopy(string src, IntPtr dest, int len)
3939
{
4040
if (len != 0)
4141
{
42-
Buffer.Memmove(ref *(byte*)dest, ref Unsafe.As<char, byte>(ref src.GetRawStringData()), (nuint)len);
42+
SpanHelpers.Memmove(ref *(byte*)dest, ref Unsafe.As<char, byte>(ref src.GetRawStringData()), (nuint)len);
4343
}
4444
}
4545

‎src/coreclr/System.Private.CoreLib/src/System/StubHelpers.cs

+5-5
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ internal static unsafe IntPtr ConvertToNative(int flags, string strManaged, IntP
103103
// + 1 for the null character from the user. + 1 for the null character we put in.
104104
pbNativeBuffer = (byte*)Marshal.AllocCoTaskMem(nb + 2);
105105

106-
Buffer.Memmove(ref *pbNativeBuffer, ref MemoryMarshal.GetArrayDataReference(bytes), (nuint)nb);
106+
SpanHelpers.Memmove(ref *pbNativeBuffer, ref MemoryMarshal.GetArrayDataReference(bytes), (nuint)nb);
107107
}
108108
}
109109

@@ -360,7 +360,7 @@ internal static unsafe IntPtr ConvertToNative(string strManaged, bool fBestFit,
360360

361361
Debug.Assert(nbytesused >= 0 && nbytesused < nbytes, "Insufficient buffer allocated in VBByValStrMarshaler.ConvertToNative");
362362

363-
Buffer.Memmove(ref *pNative, ref MemoryMarshal.GetArrayDataReference(bytes), (nuint)nbytesused);
363+
SpanHelpers.Memmove(ref *pNative, ref MemoryMarshal.GetArrayDataReference(bytes), (nuint)nbytesused);
364364

365365
pNative[nbytesused] = 0;
366366
*pLength = nbytesused;
@@ -409,7 +409,7 @@ internal static unsafe IntPtr ConvertToNative(int flags, string strManaged)
409409
IntPtr bstr = Marshal.AllocBSTRByteLen(length);
410410
if (bytes != null)
411411
{
412-
Buffer.Memmove(ref *(byte*)bstr, ref MemoryMarshal.GetArrayDataReference(bytes), length);
412+
SpanHelpers.Memmove(ref *(byte*)bstr, ref MemoryMarshal.GetArrayDataReference(bytes), length);
413413
}
414414

415415
return bstr;
@@ -1484,7 +1484,7 @@ internal static unsafe void FmtClassUpdateNativeInternal(object obj, byte* pNati
14841484
}
14851485
else
14861486
{
1487-
Buffer.Memmove(ref *pNative, ref obj.GetRawData(), size);
1487+
SpanHelpers.Memmove(ref *pNative, ref obj.GetRawData(), size);
14881488
}
14891489
}
14901490

@@ -1503,7 +1503,7 @@ internal static unsafe void FmtClassUpdateCLRInternal(object obj, byte* pNative)
15031503
}
15041504
else
15051505
{
1506-
Buffer.Memmove(ref obj.GetRawData(), ref *pNative, size);
1506+
SpanHelpers.Memmove(ref obj.GetRawData(), ref *pNative, size);
15071507
}
15081508
}
15091509

‎src/coreclr/inc/corinfo.h

+3
Original file line numberDiff line numberDiff line change
@@ -572,7 +572,10 @@ enum CorInfoHelpFunc
572572
CORINFO_HELP_INIT_PINVOKE_FRAME, // initialize an inlined PInvoke Frame for the JIT-compiler
573573

574574
CORINFO_HELP_MEMSET, // Init block of memory
575+
CORINFO_HELP_MEMZERO, // Init block of memory with zeroes
575576
CORINFO_HELP_MEMCPY, // Copy block of memory
577+
CORINFO_HELP_NATIVE_MEMSET, // Init block of memory using native memset (not safe for pDst being null,
578+
// not safe for unbounded size, does not trigger GC)
576579

577580
CORINFO_HELP_RUNTIMEHANDLE_METHOD, // determine a type/field/method handle at run-time
578581
CORINFO_HELP_RUNTIMEHANDLE_METHOD_LOG, // determine a type/field/method handle at run-time, with IBC logging

‎src/coreclr/inc/jiteeversionguid.h

+5-5
Original file line numberDiff line numberDiff line change
@@ -43,11 +43,11 @@ typedef const GUID *LPCGUID;
4343
#define GUID_DEFINED
4444
#endif // !GUID_DEFINED
4545

46-
constexpr GUID JITEEVersionIdentifier = { /* 1f30d12b-38f1-4f1e-a08a-831def882aa4 */
47-
0x1f30d12b,
48-
0x38f1,
49-
0x4f1e,
50-
{0xa0, 0x8a, 0x83, 0x1d, 0xef, 0x88, 0x2a, 0xa4}
46+
constexpr GUID JITEEVersionIdentifier = { /* 86eab154-5d93-4fad-bc07-e94fd9268b70 */
47+
0x86eab154,
48+
0x5d93,
49+
0x4fad,
50+
{0xbc, 0x07, 0xe9, 0x4f, 0xd9, 0x26, 0x8b, 0x70}
5151
};
5252

5353
//////////////////////////////////////////////////////////////////////////////////////////////////////////

‎src/coreclr/inc/jithelpers.h

+4-7
Original file line numberDiff line numberDiff line change
@@ -235,13 +235,10 @@
235235
DYNAMICJITHELPER(CORINFO_HELP_INIT_PINVOKE_FRAME, NULL, CORINFO_HELP_SIG_REG_ONLY)
236236
#endif
237237

238-
#ifdef TARGET_X86
239-
JITHELPER(CORINFO_HELP_MEMSET, NULL, CORINFO_HELP_SIG_CANNOT_USE_ALIGN_STUB)
240-
JITHELPER(CORINFO_HELP_MEMCPY, NULL, CORINFO_HELP_SIG_CANNOT_USE_ALIGN_STUB)
241-
#else
242-
JITHELPER(CORINFO_HELP_MEMSET, JIT_MemSet, CORINFO_HELP_SIG_REG_ONLY)
243-
JITHELPER(CORINFO_HELP_MEMCPY, JIT_MemCpy, CORINFO_HELP_SIG_REG_ONLY)
244-
#endif
238+
DYNAMICJITHELPER(CORINFO_HELP_MEMSET, NULL, CORINFO_HELP_SIG_REG_ONLY)
239+
DYNAMICJITHELPER(CORINFO_HELP_MEMZERO, NULL, CORINFO_HELP_SIG_REG_ONLY)
240+
DYNAMICJITHELPER(CORINFO_HELP_MEMCPY, NULL, CORINFO_HELP_SIG_REG_ONLY)
241+
JITHELPER(CORINFO_HELP_NATIVE_MEMSET, Jit_NativeMemSet, CORINFO_HELP_SIG_REG_ONLY)
245242

246243
// Generics
247244
JITHELPER(CORINFO_HELP_RUNTIMEHANDLE_METHOD, JIT_GenericHandleMethod, CORINFO_HELP_SIG_REG_ONLY)

‎src/coreclr/inc/readytorun.h

+6-6
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
// If you update this, ensure you run `git grep MINIMUM_READYTORUN_MAJOR_VERSION`
2121
// and handle pending work.
2222
#define READYTORUN_MAJOR_VERSION 0x0009
23-
#define READYTORUN_MINOR_VERSION 0x0001
23+
#define READYTORUN_MINOR_VERSION 0x0002
2424

2525
#define MINIMUM_READYTORUN_MAJOR_VERSION 0x009
2626

@@ -33,6 +33,8 @@
3333
// R2R Version 8.0 Changes the alignment of the Int128 type
3434
// R2R Version 9.0 adds support for the Vector512 type
3535
// R2R Version 9.1 adds new helpers to allocate objects on frozen segments
36+
// R2R Version 9.2 adds MemZero and NativeMemSet helpers
37+
3638

3739
struct READYTORUN_CORE_HEADER
3840
{
@@ -325,7 +327,9 @@ enum ReadyToRunHelper
325327
READYTORUN_HELPER_Stelem_Ref = 0x38,
326328
READYTORUN_HELPER_Ldelema_Ref = 0x39,
327329

328-
READYTORUN_HELPER_MemSet = 0x40,
330+
READYTORUN_HELPER_MemZero = 0x3E,
331+
READYTORUN_HELPER_MemSet = 0x3F,
332+
READYTORUN_HELPER_NativeMemSet = 0x40,
329333
READYTORUN_HELPER_MemCpy = 0x41,
330334

331335
// PInvoke helpers
@@ -441,10 +445,6 @@ enum ReadyToRunHelper
441445
READYTORUN_HELPER_StackProbe = 0x111,
442446

443447
READYTORUN_HELPER_GetCurrentManagedThreadId = 0x112,
444-
445-
// Array helpers for use with native ints
446-
READYTORUN_HELPER_Stelem_Ref_I = 0x113,
447-
READYTORUN_HELPER_Ldelema_Ref_I = 0x114,
448448
};
449449

450450
#include "readytoruninstructionset.h"

‎src/coreclr/inc/readytorunhelpers.h

+2
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@ HELPER(READYTORUN_HELPER_Stelem_Ref, CORINFO_HELP_ARRADDR_ST,
2929
HELPER(READYTORUN_HELPER_Ldelema_Ref, CORINFO_HELP_LDELEMA_REF, )
3030

3131
HELPER(READYTORUN_HELPER_MemSet, CORINFO_HELP_MEMSET, )
32+
HELPER(READYTORUN_HELPER_MemZero, CORINFO_HELP_MEMZERO, )
33+
HELPER(READYTORUN_HELPER_NativeMemSet, CORINFO_HELP_NATIVE_MEMSET, )
3234
HELPER(READYTORUN_HELPER_MemCpy, CORINFO_HELP_MEMCPY, )
3335

3436
HELPER(READYTORUN_HELPER_LogMethodEnter, CORINFO_HELP_BBT_FCN_ENTER, )

‎src/coreclr/jit/codegencommon.cpp

+3-1
Original file line numberDiff line numberDiff line change
@@ -8403,7 +8403,9 @@ void CodeGen::genPoisonFrame(regMaskTP regLiveIn)
84038403
GetEmitter()->emitIns_R_S(INS_lea, EA_PTRSIZE, REG_ARG_0, (int)varNum, 0);
84048404
instGen_Set_Reg_To_Imm(EA_4BYTE, REG_ARG_1, static_cast<char>(poisonVal));
84058405
instGen_Set_Reg_To_Imm(EA_PTRSIZE, REG_ARG_2, size);
8406-
genEmitHelperCall(CORINFO_HELP_MEMSET, 0, EA_UNKNOWN);
8406+
8407+
// Call non-managed memset
8408+
genEmitHelperCall(CORINFO_HELP_NATIVE_MEMSET, 0, EA_UNKNOWN);
84078409
// May kill REG_SCRATCH, so we need to reload it.
84088410
hasPoisonImm = false;
84098411
#endif

‎src/coreclr/jit/fgbasic.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -1332,7 +1332,7 @@ void Compiler::fgFindJumpTargets(const BYTE* codeAddr, IL_OFFSET codeSize, Fixed
13321332
case NI_System_SpanHelpers_ClearWithoutReferences:
13331333
case NI_System_SpanHelpers_Fill:
13341334
case NI_System_SpanHelpers_SequenceEqual:
1335-
case NI_System_Buffer_Memmove:
1335+
case NI_System_SpanHelpers_Memmove:
13361336
{
13371337
if (FgStack::IsConstArgument(pushedStack.Top(), impInlineInfo))
13381338
{

‎src/coreclr/jit/fgprofile.cpp

+3-3
Original file line numberDiff line numberDiff line change
@@ -1947,7 +1947,7 @@ class ValueHistogramProbeVisitor final : public GenTreeVisitor<ValueHistogramPro
19471947
if (node->IsCall() && node->AsCall()->IsSpecialIntrinsic())
19481948
{
19491949
const NamedIntrinsic ni = m_compiler->lookupNamedIntrinsic(node->AsCall()->gtCallMethHnd);
1950-
if ((ni == NI_System_Buffer_Memmove) || (ni == NI_System_SpanHelpers_SequenceEqual))
1950+
if ((ni == NI_System_SpanHelpers_Memmove) || (ni == NI_System_SpanHelpers_SequenceEqual))
19511951
{
19521952
m_functor(m_compiler, node);
19531953
}
@@ -2274,7 +2274,7 @@ class ValueHistogramProbeInserter
22742274
return;
22752275
}
22762276

2277-
assert(node->AsCall()->IsSpecialIntrinsic(compiler, NI_System_Buffer_Memmove) ||
2277+
assert(node->AsCall()->IsSpecialIntrinsic(compiler, NI_System_SpanHelpers_Memmove) ||
22782278
node->AsCall()->IsSpecialIntrinsic(compiler, NI_System_SpanHelpers_SequenceEqual));
22792279

22802280
const ICorJitInfo::PgoInstrumentationSchema& countEntry = m_schema[*m_currentSchemaIndex];
@@ -2540,7 +2540,7 @@ PhaseStatus Compiler::fgPrepareToInstrumentMethod()
25402540
// These are marked as [Intrinsic] only to be handled (unrolled) for constant inputs.
25412541
// In other cases they have large managed implementations we want to profile.
25422542
case NI_System_String_Equals:
2543-
case NI_System_Buffer_Memmove:
2543+
case NI_System_SpanHelpers_Memmove:
25442544
case NI_System_MemoryExtensions_Equals:
25452545
case NI_System_MemoryExtensions_SequenceEqual:
25462546
case NI_System_MemoryExtensions_StartsWith:

‎src/coreclr/jit/importer.cpp

+23-7
Original file line numberDiff line numberDiff line change
@@ -10295,9 +10295,9 @@ void Compiler::impImportBlockCode(BasicBlock* block)
1029510295
}
1029610296
#endif
1029710297

10298-
op3 = impPopStack().val; // Size
10299-
op2 = impPopStack().val; // Value / Src addr
10300-
op1 = impPopStack().val; // Dst addr
10298+
op3 = gtFoldExpr(impPopStack().val); // Size
10299+
op2 = gtFoldExpr(impPopStack().val); // Value / Src addr
10300+
op1 = impPopStack().val; // Dst addr
1030110301

1030210302
if (op3->IsCnsIntOrI())
1030310303
{
@@ -10343,18 +10343,34 @@ void Compiler::impImportBlockCode(BasicBlock* block)
1034310343
// TODO: enable for X86 as well, it currently doesn't support memset/memcpy helpers
1034410344
// Then, get rid of GT_STORE_DYN_BLK entirely.
1034510345
#ifndef TARGET_X86
10346-
const unsigned helper = opcode == CEE_INITBLK ? CORINFO_HELP_MEMSET : CORINFO_HELP_MEMCPY;
10346+
GenTreeCall* call;
10347+
if (opcode == CEE_INITBLK)
10348+
{
10349+
// value is zero -> memzero, otherwise -> memset
10350+
if (op2->IsIntegralConst(0))
10351+
{
10352+
call = gtNewHelperCallNode(CORINFO_HELP_MEMZERO, TYP_VOID, op1, op3);
10353+
}
10354+
else
10355+
{
10356+
call = gtNewHelperCallNode(CORINFO_HELP_MEMSET, TYP_VOID, op1, op2, op3);
10357+
}
10358+
}
10359+
else
10360+
{
10361+
call = gtNewHelperCallNode(CORINFO_HELP_MEMCPY, TYP_VOID, op1, op2, op3);
10362+
}
10363+
1034710364
if (isVolatile)
1034810365
{
1034910366
// Wrap with memory barriers: full-barrier + call + load-barrier
1035010367
impAppendTree(gtNewMemoryBarrier(), CHECK_SPILL_ALL, impCurStmtDI);
10351-
impAppendTree(gtNewHelperCallNode(helper, TYP_VOID, op1, op2, op3), CHECK_SPILL_ALL,
10352-
impCurStmtDI);
10368+
impAppendTree(call, CHECK_SPILL_ALL, impCurStmtDI);
1035310369
op1 = gtNewMemoryBarrier(true);
1035410370
}
1035510371
else
1035610372
{
10357-
op1 = gtNewHelperCallNode(helper, TYP_VOID, op1, op2, op3);
10373+
op1 = call;
1035810374
}
1035910375
#else
1036010376
if (opcode == CEE_INITBLK)

‎src/coreclr/jit/importercalls.cpp

+8-11
Original file line numberDiff line numberDiff line change
@@ -1290,7 +1290,7 @@ var_types Compiler::impImportCall(OPCODE opcode,
12901290
impAppendTree(call, verCurrentState.esStackDepth - 1, impCurStmtDI);
12911291
}
12921292
else if (JitConfig.JitProfileValues() && call->IsCall() &&
1293-
call->AsCall()->IsSpecialIntrinsic(this, NI_System_Buffer_Memmove))
1293+
call->AsCall()->IsSpecialIntrinsic(this, NI_System_SpanHelpers_Memmove))
12941294
{
12951295
if (opts.IsOptimizedWithProfile())
12961296
{
@@ -1555,7 +1555,7 @@ GenTree* Compiler::impDuplicateWithProfiledArg(GenTreeCall* call, IL_OFFSET ilOf
15551555
unsigned argNum = 0;
15561556
ssize_t minValue = 0;
15571557
ssize_t maxValue = 0;
1558-
if (call->IsSpecialIntrinsic(this, NI_System_Buffer_Memmove))
1558+
if (call->IsSpecialIntrinsic(this, NI_System_SpanHelpers_Memmove))
15591559
{
15601560
// dst(0), src(1), len(2)
15611561
argNum = 2;
@@ -2761,7 +2761,7 @@ GenTree* Compiler::impIntrinsic(GenTree* newobjThis,
27612761
betterToExpand = true;
27622762
break;
27632763

2764-
case NI_System_Buffer_Memmove:
2764+
case NI_System_SpanHelpers_Memmove:
27652765
case NI_System_SpanHelpers_SequenceEqual:
27662766
// We're going to instrument these
27672767
betterToExpand = opts.IsInstrumented();
@@ -3983,7 +3983,7 @@ GenTree* Compiler::impIntrinsic(GenTree* newobjThis,
39833983
case NI_System_Text_UTF8Encoding_UTF8EncodingSealed_ReadUtf8:
39843984
case NI_System_SpanHelpers_SequenceEqual:
39853985
case NI_System_SpanHelpers_ClearWithoutReferences:
3986-
case NI_System_Buffer_Memmove:
3986+
case NI_System_SpanHelpers_Memmove:
39873987
{
39883988
if (sig->sigInst.methInstCount == 0)
39893989
{
@@ -8874,13 +8874,6 @@ NamedIntrinsic Compiler::lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method)
88748874
result = NI_System_BitConverter_Int64BitsToDouble;
88758875
}
88768876
}
8877-
else if (strcmp(className, "Buffer") == 0)
8878-
{
8879-
if (strcmp(methodName, "Memmove") == 0)
8880-
{
8881-
result = NI_System_Buffer_Memmove;
8882-
}
8883-
}
88848877
break;
88858878
}
88868879

@@ -9040,6 +9033,10 @@ NamedIntrinsic Compiler::lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method)
90409033
{
90419034
result = NI_System_SpanHelpers_ClearWithoutReferences;
90429035
}
9036+
else if (strcmp(methodName, "Memmove") == 0)
9037+
{
9038+
result = NI_System_SpanHelpers_Memmove;
9039+
}
90439040
}
90449041
else if (strcmp(className, "String") == 0)
90459042
{

0 commit comments

Comments
 (0)