JIT: Switch GS cookie check temp reg to use r10 for tailcalls (#120941)

rax was used before, but rax is used as an argument for the indirection
cell for R2R tailcalls. r10 has a use for some pinvokes, but we do not
tailcall for these.

Fix #120780

Author: jakobbotsch

src/coreclr/jit/codegencommon.cpp

@@ -2519,17 +2519,17 @@ CorInfoHelpFunc CodeGenInterface::genWriteBarrierHelperForWriteBarrierForm(GCInf
 //
 regMaskTP CodeGenInterface::genGetGSCookieTempRegs(bool tailCall)
 {
-#ifdef TARGET_XARCH
+#ifdef TARGET_AMD64
     if (tailCall)
     {
-        // If we are tailcalling then return registers are available for use
-        return RBM_RAX;
+        // If we are tailcalling then arg regs cannot be used. For both SysV and winx64 that
+        // leaves rax, r10, r11. rax and r11 are used for indirection cells, so we pick r10.
+        return RBM_R10;
     }
-
-#ifdef TARGET_AMD64
     // Otherwise on x64 (win-x64 and SysV) r8 is never used for return values
     return RBM_R8;
-#else
+#elif TARGET_X86
+    assert(!tailCall);
     // On x86 it's more difficult: we have only eax, ecx and edx available as volatile
     // registers, and all of them may be used for return values (longs + async continuation).
     if (compiler->compIsAsync())
@@ -2540,7 +2540,6 @@ regMaskTP CodeGenInterface::genGetGSCookieTempRegs(bool tailCall)
 
     // Outside async ecx is not used for returns.
     return RBM_ECX;
-#endif
 #else
     return RBM_GSCOOKIE_TMP;
 #endif

src/coreclr/jit/morph.cpp

@@ -1773,6 +1773,11 @@ void CallArgs::AddFinalArgsAndDetermineABIInfo(Compiler* comp, GenTreeCall* call
         InsertAfterThisOrFirst(comp, NewCallArg::Primitive(arg).WellKnown(WellKnownArg::PInvokeCookie));
         // put destination into R10/EAX
         arg = comp->gtClone(call->gtCallAddr, true);
+        // On x64 the pinvoke target is passed in r11 which is the same
+        // register as the gs cookie check may use. That would be a problem if
+        // this was a tailcall, but we do not tailcall functions with
+        // non-standard added args except indirection cells currently.
+        assert(!call->IsFastTailCall());
         InsertAfterThisOrFirst(comp, NewCallArg::Primitive(arg).WellKnown(WellKnownArg::PInvokeTarget));
 
         // finally change this call to a helper call