Async instantiating and unboxing thunks

docs/design/coreclr/botr/runtime-async-codegen.md

@@ -98,10 +98,10 @@ This is what is used in non-async functions when calling an async function. Gene
 ```
 (result, continuation) = call func(NULL /\* Continuation argument \*/, args)
 place result onto IL evaluation stack
-Place continuation into a local for access using the StubHelpers.AsyncCallContinuation() helper function.
+Place continuation into a local for access using the AsyncHelpers.AsyncCallContinuation() helper function.
 ```
 
-Implement an intrinsic for StubHelpers.AsyncCallContinuation() which will load the most recent value stored into the continuation local.
+Implement an intrinsic for AsyncHelpers.AsyncCallContinuation() which will load the most recent value stored into the continuation local.
 
 # Behavior of ContinuationContextHandling
 

src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs

@@ -198,6 +198,10 @@ public static partial class AsyncHelpers
         [Intrinsic]
         private static Continuation? AsyncCallContinuation() => throw new UnreachableException();
 
+        // Indicate that an upcoming await should be done as a "tail await" that does not introduce a new suspension point.
+        [Intrinsic]
+        private static void TailAwait() => throw new UnreachableException();
+
         // Used during suspensions to hold the continuation chain and on what we are waiting.
         // Methods like FinalizeTaskReturningThunk will unlink the state and wrap into a Task.
         private struct RuntimeAsyncAwaitState

src/coreclr/jit/async.h

@@ -72,6 +72,9 @@ class AsyncTransformation
     BasicBlock*                   m_lastResumptionBB        = nullptr;
     BasicBlock*                   m_sharedReturnBB          = nullptr;
 
+    void        TransformTailAwait(BasicBlock* block, GenTreeCall* call, BasicBlock** remainder);
+    BasicBlock* CreateTailAwaitSuspension(BasicBlock* block, GenTreeCall* call);
+
     bool IsLive(unsigned lclNum);
     void Transform(BasicBlock*               block,
                    GenTreeCall*              call,
@@ -98,7 +101,7 @@ class AsyncTransformation
                                           bool                           needsKeepAlive,
                                           jitstd::vector<LiveLocalInfo>& liveLocals);
 
-    CallDefinitionInfo CanonicalizeCallDefinition(BasicBlock* block, GenTreeCall* call, AsyncLiveness& life);
+    CallDefinitionInfo CanonicalizeCallDefinition(BasicBlock* block, GenTreeCall* call, AsyncLiveness* life);
 
     BasicBlock* CreateSuspension(
         BasicBlock* block, GenTreeCall* call, unsigned stateNum, AsyncLiveness& life, const ContinuationLayout& layout);
@@ -110,7 +113,6 @@ class AsyncTransformation
     void         CreateCheckAndSuspendAfterCall(BasicBlock*               block,
                                                 GenTreeCall*              call,
                                                 const CallDefinitionInfo& callDefInfo,
-                                                AsyncLiveness&            life,
                                                 BasicBlock*               suspendBB,
                                                 BasicBlock**              remainder);
     BasicBlock*  CreateResumption(BasicBlock*               block,
@@ -137,9 +139,12 @@ class AsyncTransformation
                                    var_types    storeType,
                                    GenTreeFlags indirFlags = GTF_IND_NONFAULTING);
 
-    void     CreateDebugInfoForSuspensionPoint(const ContinuationLayout& layout);
-    unsigned GetResultBaseVar();
-    unsigned GetExceptionVar();
+    void        CreateDebugInfoForSuspensionPoint(const ContinuationLayout& layout);
+    unsigned    GetReturnedContinuationVar();
+    unsigned    GetNewContinuationVar();
+    unsigned    GetResultBaseVar();
+    unsigned    GetExceptionVar();
+    BasicBlock* GetSharedReturnBB();
 
     void CreateResumptionSwitch();
 

src/coreclr/jit/compiler.h

@@ -4985,6 +4985,8 @@ class Compiler
 
     bool impCanReimport;
 
+    bool m_nextAwaitIsTail = false;
+
     bool impSpillStackEntry(unsigned level,
                             unsigned varNum
 #ifdef DEBUG

src/coreclr/jit/gentree.h

@@ -4363,6 +4363,10 @@ struct AsyncCallInfo
     // configured and whether it is a task await or custom await. This field
     // records that behavior.
     ContinuationContextHandling ContinuationContextHandling = ContinuationContextHandling::None;
+
+    // Tail awaits do not generate suspension points and the JIT instead
+    // directly returns the callee's continuation to the caller.
+    bool IsTailAwait = false;
 };
 
 // Return type descriptor of a GT_CALL node.

src/coreclr/jit/importercalls.cpp

@@ -3390,6 +3390,17 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE    clsHnd,
         return nullptr;
     }
 
+    if (ni == NI_System_Runtime_CompilerServices_AsyncHelpers_TailAwait)
+    {
+        if ((info.compMethodInfo->options & CORINFO_ASYNC_SAVE_CONTEXTS) != 0)
+        {
+            BADCODE("TailAwait is not supported in async methods that capture contexts");
+        }
+
+        m_nextAwaitIsTail = true;
+        return gtNewNothingNode();
+    }
+
     bool betterToExpand = false;
 
     // Allow some lightweight intrinsics in Tier0 which can improve throughput
@@ -6900,6 +6911,13 @@ void Compiler::impSetupAsyncCall(GenTreeCall* call, OPCODE opcode, unsigned pref
         JITDUMP("Call is an async non-task await\n");
     }
 
+    if (m_nextAwaitIsTail)
+    {
+        asyncInfo.ContinuationContextHandling = ContinuationContextHandling::None;
+        asyncInfo.IsTailAwait                 = true;
+        m_nextAwaitIsTail                     = false;
+    }
+
     call->AsCall()->SetIsAsync(new (this, CMK_Async) AsyncCallInfo(asyncInfo));
 }
 
@@ -10899,6 +10917,10 @@ NamedIntrinsic Compiler::lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method)
                             {
                                 result = NI_System_Runtime_CompilerServices_AsyncHelpers_AsyncCallContinuation;
                             }
+                            else if (strcmp(methodName, "TailAwait") == 0)
+                            {
+                                result = NI_System_Runtime_CompilerServices_AsyncHelpers_TailAwait;
+                            }
                         }
                         else if (strcmp(className, "StaticsHelpers") == 0)
                         {

src/coreclr/jit/namedintrinsiclist.h

@@ -129,6 +129,7 @@ enum NamedIntrinsic : unsigned short
     NI_System_Runtime_CompilerServices_AsyncHelpers_AsyncSuspend,
     NI_System_Runtime_CompilerServices_AsyncHelpers_Await,
     NI_System_Runtime_CompilerServices_AsyncHelpers_AsyncCallContinuation,
+    NI_System_Runtime_CompilerServices_AsyncHelpers_TailAwait,
 
     NI_System_Runtime_CompilerServices_StaticsHelpers_VolatileReadAsByref,
 

src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs

@@ -4423,7 +4423,12 @@ private uint getJitFlags(ref CORJIT_FLAGS flags, uint sizeInBytes)
                 flags.Set(CorJitFlag.CORJIT_FLAG_SOFTFP_ABI);
             }
 
-            if (this.MethodBeingCompiled.IsAsyncCall())
+            if (this.MethodBeingCompiled.IsAsyncCall()
+#if !READYTORUN
+                || (_compilation.TypeSystemContext.IsSpecialUnboxingThunk(this.MethodBeingCompiled) && _compilation.TypeSystemContext.GetTargetOfSpecialUnboxingThunk(this.MethodBeingCompiled).IsAsyncCall())
+                || (_compilation.TypeSystemContext.IsDefaultInterfaceMethodImplementationInstantiationThunk(this.MethodBeingCompiled) && _compilation.TypeSystemContext.GetTargetOfDefaultInterfaceMethodImplementationInstantiationThunk(this.MethodBeingCompiled).IsAsyncCall())
+#endif
+                )
             {
                 flags.Set(CorJitFlag.CORJIT_FLAG_ASYNC);
             }

src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/CompilerTypeSystemContext.BoxedTypes.cs

@@ -396,16 +396,6 @@ public override MethodIL EmitIL()
                         Array.Empty<object>());
                 }
 
-                // TODO: (async) https://github.com/dotnet/runtime/issues/121781
-                if (_targetMethod.IsAsyncCall())
-                {
-                    ILEmitter e = new ILEmitter();
-                    ILCodeStream c = e.NewCodeStream();
-
-                    c.EmitCallThrowHelper(e, Context.GetCoreLibEntryPoint("System.Runtime"u8, "InternalCalls"u8, "RhpFallbackFailFast"u8, null));
-                    return e.Link(this);
-                }
-
                 // Generate the unboxing stub. This loosely corresponds to following C#:
                 // return BoxedValue.InstanceMethod(this.m_pEEType, [rest of parameters])
 
@@ -431,6 +421,11 @@ public override MethodIL EmitIL()
                     codeStream.EmitLdArg(i + 1);
                 }
 
+                if (_targetMethod.IsAsyncCall())
+                {
+                    codeStream.Emit(ILOpcode.call, emit.NewToken(Context.GetCoreLibEntryPoint("System.Runtime.CompilerServices"u8, "AsyncHelpers"u8, "TailAwait"u8, null)));
+                }
+
                 codeStream.Emit(ILOpcode.call, emit.NewToken(_targetMethod.InstantiateAsOpen()));
                 codeStream.Emit(ILOpcode.ret);
 
@@ -490,7 +485,8 @@ public override MethodIL EmitIL()
                         Array.Empty<object>());
                 }
 
-                // TODO: (async) https://github.com/dotnet/runtime/issues/121781
+                // TODO: mirror what was done in the commit that introduced this comment. Not doing it in that
+                // commit since this can't be tested in dotnet/runtime repo main right now.
                 if (_targetMethod.IsAsyncCall())
                 {
                     ILEmitter e = new ILEmitter();

src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/CompilerTypeSystemContext.InterfaceThunks.cs

@@ -143,6 +143,22 @@ protected override DefaultInterfaceMethodImplementationInstantiationThunk Create
         }
         private DefaultInterfaceMethodImplementationInstantiationThunkHashtable _dimThunkHashtable = new DefaultInterfaceMethodImplementationInstantiationThunkHashtable();
 
+        /// <summary>
+        /// Does a method represent a default interface method specialization thunk?
+        /// </summary>
+        public bool IsDefaultInterfaceMethodImplementationInstantiationThunk(MethodDesc method)
+        {
+            return method is DefaultInterfaceMethodImplementationInstantiationThunk;
+        }
+
+        /// <summary>
+        /// Convert from a default interface method specialization thunk to the actual target method.
+        /// </summary>
+        public MethodDesc GetTargetOfDefaultInterfaceMethodImplementationInstantiationThunk(MethodDesc method)
+        {
+            return ((DefaultInterfaceMethodImplementationInstantiationThunk)method).TargetMethod;
+        }
+
         /// <summary>
         /// Represents a thunk to call shared instance method on generic interfaces.
         /// </summary>
@@ -202,16 +218,6 @@ public override string DiagnosticName
 
             public override MethodIL EmitIL()
             {
-                // TODO: (async) https://github.com/dotnet/runtime/issues/121781
-                if (_targetMethod.IsAsyncCall())
-                {
-                    ILEmitter e = new ILEmitter();
-                    ILCodeStream c = e.NewCodeStream();
-
-                    c.EmitCallThrowHelper(e, Context.GetCoreLibEntryPoint("System.Runtime"u8, "InternalCalls"u8, "RhpFallbackFailFast"u8, null));
-                    return e.Link(this);
-                }
-
                 // Generate the instantiating stub. This loosely corresponds to following C#:
                 // return Interface.Method(this, GetOrdinalInterface(this.m_pEEType, Index), [rest of parameters])
 
@@ -250,6 +256,11 @@ public override MethodIL EmitIL()
                     codeStream.EmitLdArg(i + 1);
                 }
 
+                if (_targetMethod.IsAsyncCall())
+                {
+                    codeStream.Emit(ILOpcode.call, emit.NewToken(Context.GetCoreLibEntryPoint("System.Runtime.CompilerServices"u8, "AsyncHelpers"u8, "TailAwait"u8, null)));
+                }
+
                 codeStream.Emit(ILOpcode.call, emit.NewToken(_targetMethod));
                 codeStream.Emit(ILOpcode.ret);
 

src/coreclr/vm/asyncthunks.cpp

@@ -74,7 +74,7 @@ void MethodDesc::EmitTaskReturningThunk(MethodDesc* pAsyncCallVariant, MetaSig&
     //   {
     //     T result = Inner(args);
     //     // call an intrisic to see if the call above produced a continuation
-    //     if (StubHelpers.AsyncCallContinuation() == null)
+    //     if (AsyncHelpers.AsyncCallContinuation() == null)
     //       return Task.FromResult(result);
     //
     //     return FinalizeTaskReturningThunk();

src/tests/async/inst-unbox-thunks/inst-unbox-thunks.cs

@@ -141,28 +141,24 @@ public static void ManyArgUnbox()
     }
 
     [Fact]
-    [ActiveIssue("https://github.com/dotnet/runtime/issues/121781", typeof(TestLibrary.Utilities), nameof(TestLibrary.Utilities.IsNativeAot))]
     public static void NoArgGenericUnbox()
     {
         Assert.Equal("System.String", CallStruct1M0().Result);
     }
 
     [Fact]
-    [ActiveIssue("https://github.com/dotnet/runtime/issues/121781", typeof(TestLibrary.Utilities), nameof(TestLibrary.Utilities.IsNativeAot))]
     public static void ManyArgGenericUnbox()
     {
         Assert.Equal("System.String", CallStruct1M1().Result);
     }
 
     [Fact]
-    [ActiveIssue("https://github.com/dotnet/runtime/issues/121781", typeof(TestLibrary.Utilities), nameof(TestLibrary.Utilities.IsNativeAot))]
     public static void NoArgGenericInstantiating()
     {
         Assert.Equal("System.String", CallStruct1M0b().Result);
     }
 
     [Fact]
-    [ActiveIssue("https://github.com/dotnet/runtime/issues/121781", typeof(TestLibrary.Utilities), nameof(TestLibrary.Utilities.IsNativeAot))]
     public static void ManyArgGenericInstantiating()
     {
         Assert.Equal("System.String", CallStruct1M1b().Result);
@@ -247,14 +243,12 @@ static async Task<string> CallClass3M1()
     }
 
     [Fact]
-    [ActiveIssue("https://github.com/dotnet/runtime/issues/121781", typeof(TestLibrary.Utilities), nameof(TestLibrary.Utilities.IsNativeAot))]
     public static void NoArgDefaultMethod()
     {
         Assert.Equal("System.String", CallClass3M0().Result);
     }
 
     [Fact]
-    [ActiveIssue("https://github.com/dotnet/runtime/issues/121781", typeof(TestLibrary.Utilities), nameof(TestLibrary.Utilities.IsNativeAot))]
     public static void ManyArgDefaultMethod()
     {
         Assert.Equal("System.String", CallClass3M1().Result);