diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs index 19ea178f6bf1f9..60f08f9149815f 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs @@ -150,6 +150,16 @@ public static partial class AsyncHelpers [Intrinsic] private static void AsyncSuspend(Continuation continuation) => throw new UnreachableException(); + [Intrinsic] + [BypassReadyToRun] + internal static Continuation? AsyncCallContinuation() => throw new UnreachableException(); // Unconditionally expanded intrinsic + + // Set call continuation argument for an upcoming call with runtime async calling convention. + // Can only be used shortly before the call (with no interfering control flow instructions before the call). + // If used anywhere in the method it must be present before _all_ calls with runtime async calling convention. + [Intrinsic] + internal static void SetAsyncCallContinuationArg(Continuation continuation) => throw new UnreachableException(); + // Used during suspensions to hold the continuation chain and on what we are waiting. // Methods like FinalizeTaskReturningThunk will unlink the state and wrap into a Task. private struct RuntimeAsyncAwaitState diff --git a/src/coreclr/System.Private.CoreLib/src/System/StubHelpers.cs b/src/coreclr/System.Private.CoreLib/src/System/StubHelpers.cs index acb629e66e3662..6d61edd19ebc4a 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/StubHelpers.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/StubHelpers.cs @@ -1610,9 +1610,6 @@ internal static void MulticastDebuggerTraceHelper(object o, int count) [Intrinsic] internal static IntPtr NextCallReturnAddress() => throw new UnreachableException(); // Unconditionally expanded intrinsic - - [Intrinsic] - internal static Continuation? AsyncCallContinuation() => throw new UnreachableException(); // Unconditionally expanded intrinsic } // class StubHelpers #if FEATURE_COMINTEROP diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index f1eb605e1e951c..fc8f731ee8d1f5 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -484,6 +484,11 @@ bool AsyncLiveness::IsLocalCaptureUnnecessary(unsigned lclNum) return true; } + if (lclNum == m_comp->lvaNextAsyncCallContArgVar) + { + return true; + } + return false; } diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 5f51bd68e138db..ed485796cb519e 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -3976,6 +3976,8 @@ class Compiler bool lvaInlineeReturnSpillTempFreshlyCreated = false; // True if the temp was freshly created for the inlinee return + unsigned lvaNextAsyncCallContArgVar = BAD_VAR_NUM; + #if FEATURE_FIXED_OUT_ARGS unsigned lvaOutgoingArgSpaceVar = BAD_VAR_NUM; // var that represents outgoing argument space PhasedVar lvaOutgoingArgSpaceSize; // size of fixed outgoing argument space diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index fe620f6f332b87..b9d49d5655f500 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -2265,7 +2265,7 @@ bool GenTreeCall::HasSideEffects(Compiler* compiler, bool ignoreExceptions, bool // those cases the JIT does not know (and does not need to know) which arg is // the async continuation. // -// The VM also uses the StubHelpers.AsyncCallContinuation() intrinsic in the +// The VM also uses the AsyncHelpers.AsyncCallContinuation() intrinsic in the // stubs discussed above. The JIT must take care in those cases to still mark // the preceding call as an async call; this is required for correct LSRA // behavior and GC reporting around the returned async continuation. This is diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index 42dbb08ee7043f..f706bba22efc77 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -875,7 +875,10 @@ var_types Compiler::impImportCall(OPCODE opcode, // stubs and the VM inserts the arg itself. if (call->AsCall()->IsAsync() && (opcode != CEE_CALLI)) { - call->AsCall()->gtArgs.PushFront(this, NewCallArg::Primitive(gtNewNull(), TYP_REF) + GenTree* arg = lvaNextAsyncCallContArgVar != BAD_VAR_NUM + ? (GenTree*)gtNewLclVarNode(lvaNextAsyncCallContArgVar) + : gtNewNull(); + call->AsCall()->gtArgs.PushFront(this, NewCallArg::Primitive(arg, TYP_REF) .WellKnown(WellKnownArg::AsyncContinuation)); } @@ -891,7 +894,10 @@ var_types Compiler::impImportCall(OPCODE opcode, // stubs and the VM inserts the arg itself. if (call->AsCall()->IsAsync() && (opcode != CEE_CALLI)) { - call->AsCall()->gtArgs.PushBack(this, NewCallArg::Primitive(gtNewNull(), TYP_REF) + GenTree* arg = lvaNextAsyncCallContArgVar != BAD_VAR_NUM + ? (GenTree*)gtNewLclVarNode(lvaNextAsyncCallContArgVar) + : gtNewNull(); + call->AsCall()->gtArgs.PushBack(this, NewCallArg::Primitive(arg, TYP_REF) .WellKnown(WellKnownArg::AsyncContinuation)); } @@ -3323,7 +3329,7 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd, return new (this, GT_LABEL) GenTree(GT_LABEL, TYP_I_IMPL); } - if (ni == NI_System_StubHelpers_AsyncCallContinuation) + if (ni == NI_System_Runtime_CompilerServices_AsyncHelpers_AsyncCallContinuation) { GenTree* node = new (this, GT_ASYNC_CONTINUATION) GenTree(GT_ASYNC_CONTINUATION, TYP_REF); node->SetHasOrderingSideEffect(); @@ -3332,6 +3338,18 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE clsHnd, return node; } + if (ni == NI_System_Runtime_CompilerServices_AsyncHelpers_SetAsyncCallContinuationArg) + { + if (lvaNextAsyncCallContArgVar == BAD_VAR_NUM) + { + lvaNextAsyncCallContArgVar = lvaGrabTemp(false DEBUGARG("Async call continuation arg")); + lvaGetDesc(lvaNextAsyncCallContArgVar)->lvType = TYP_REF; + } + + GenTree* node = gtNewStoreLclVarNode(lvaNextAsyncCallContArgVar, impPopStack().val); + return node; + } + if (ni == NI_System_Runtime_CompilerServices_AsyncHelpers_AsyncSuspend) { GenTree* node = gtNewOperNode(GT_RETURN_SUSPEND, TYP_VOID, impPopStack().val); @@ -10872,6 +10890,14 @@ NamedIntrinsic Compiler::lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method) { result = NI_System_Runtime_CompilerServices_AsyncHelpers_Await; } + else if (strcmp(methodName, "AsyncCallContinuation") == 0) + { + result = NI_System_Runtime_CompilerServices_AsyncHelpers_AsyncCallContinuation; + } + else if (strcmp(methodName, "SetAsyncCallContinuationArg") == 0) + { + result = NI_System_Runtime_CompilerServices_AsyncHelpers_SetAsyncCallContinuationArg; + } } else if (strcmp(className, "StaticsHelpers") == 0) { @@ -11129,10 +11155,6 @@ NamedIntrinsic Compiler::lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method) { result = NI_System_StubHelpers_NextCallReturnAddress; } - else if (strcmp(methodName, "AsyncCallContinuation") == 0) - { - result = NI_System_StubHelpers_AsyncCallContinuation; - } } } else if (strcmp(namespaceName, "Text") == 0) diff --git a/src/coreclr/jit/namedintrinsiclist.h b/src/coreclr/jit/namedintrinsiclist.h index aa56003f25acc1..dd884ac5e575f4 100644 --- a/src/coreclr/jit/namedintrinsiclist.h +++ b/src/coreclr/jit/namedintrinsiclist.h @@ -107,7 +107,6 @@ enum NamedIntrinsic : unsigned short NI_System_RuntimeType_get_TypeHandle, NI_System_StubHelpers_GetStubContext, NI_System_StubHelpers_NextCallReturnAddress, - NI_System_StubHelpers_AsyncCallContinuation, NI_Array_Address, NI_Array_Get, @@ -126,6 +125,8 @@ enum NamedIntrinsic : unsigned short NI_System_Runtime_CompilerServices_AsyncHelpers_AsyncSuspend, NI_System_Runtime_CompilerServices_AsyncHelpers_Await, + NI_System_Runtime_CompilerServices_AsyncHelpers_AsyncCallContinuation, + NI_System_Runtime_CompilerServices_AsyncHelpers_SetAsyncCallContinuationArg, NI_System_Runtime_CompilerServices_StaticsHelpers_VolatileReadAsByref, diff --git a/src/coreclr/tools/Common/TypeSystem/IL/Stubs/AsyncResumptionStub.cs b/src/coreclr/tools/Common/TypeSystem/IL/Stubs/AsyncResumptionStub.cs index 29504bb6834fcf..724420eac5004d 100644 --- a/src/coreclr/tools/Common/TypeSystem/IL/Stubs/AsyncResumptionStub.cs +++ b/src/coreclr/tools/Common/TypeSystem/IL/Stubs/AsyncResumptionStub.cs @@ -18,8 +18,7 @@ public partial class AsyncResumptionStub : ILStubMethod public AsyncResumptionStub(MethodDesc owningMethod) { - Debug.Assert(owningMethod.IsAsyncVariant() - || (owningMethod.IsAsync && !owningMethod.Signature.ReturnsTaskOrValueTask())); + Debug.Assert(owningMethod.IsAsyncCall()); _owningMethod = owningMethod; } @@ -41,13 +40,62 @@ private MethodSignature InitializeSignature() public override MethodIL EmitIL() { - var emitter = new ILEmitter(); - ILCodeStream codeStream = emitter.NewCodeStream(); + ILEmitter ilEmitter = new ILEmitter(); + ILCodeStream ilStream = ilEmitter.NewCodeStream(); - // TODO: match getAsyncResumptionStub from CoreCLR VM - codeStream.EmitCallThrowHelper(emitter, Context.GetHelperEntryPoint("ThrowHelpers"u8, "ThrowNotSupportedException"u8)); + // Ported from jitinterface.cpp CEEJitInfo::getAsyncResumptionStub + if (!_owningMethod.Signature.IsStatic) + { + if (_owningMethod.OwningType.IsValueType) + { + ilStream.EmitLdc(0); + ilStream.Emit(ILOpcode.conv_u); + } + else + { + ilStream.Emit(ILOpcode.ldnull); + } + } - return emitter.Link(this); + foreach (var param in _owningMethod.Signature) + { + var local = ilEmitter.NewLocal(param); + ilStream.EmitLdLoca(local); + ilStream.Emit(ILOpcode.initobj, ilEmitter.NewToken(param)); + ilStream.EmitLdLoc(local); + } + ilStream.Emit(ILOpcode.ldftn, ilEmitter.NewToken(_owningMethod)); + ilStream.Emit(ILOpcode.calli, ilEmitter.NewToken(this.Signature)); + + bool returnsVoid = _owningMethod.Signature.ReturnType != Context.GetWellKnownType(WellKnownType.Void); + Internal.IL.Stubs.ILLocalVariable resultLocal = default; + if (!returnsVoid) + { + resultLocal = ilEmitter.NewLocal(_owningMethod.Signature.ReturnType); + ilStream.EmitStLoc(resultLocal); + } + + MethodDesc asyncCallContinuation = Context.SystemModule.GetKnownType("System.Runtime.CompilerServices"u8, "AsyncHelpers"u8) + .GetKnownMethod("AsyncCallContinuation"u8, null); + TypeDesc continuation = Context.SystemModule.GetKnownType("System.Runtime.CompilerServices"u8, "Continuation"u8); + var newContinuationLocal = ilEmitter.NewLocal(continuation); + ilStream.Emit(ILOpcode.call, ilEmitter.NewToken(asyncCallContinuation)); + ilStream.EmitStLoc(newContinuationLocal); + + if (!returnsVoid) + { + var doneResult = ilEmitter.NewCodeLabel(); + ilStream.EmitLdLoc(newContinuationLocal); + ilStream.Emit(ILOpcode.brtrue, doneResult); + ilStream.EmitLdArg(1); + ilStream.EmitLdLoc(resultLocal); + ilStream.Emit(ILOpcode.stobj, ilEmitter.NewToken(_owningMethod.Signature.ReturnType)); + ilStream.EmitLabel(doneResult); + } + ilStream.EmitLdLoc(newContinuationLocal); + ilStream.Emit(ILOpcode.ret); + + return ilEmitter.Link(this); } } } diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/ILCompiler.ReadyToRun.csproj b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/ILCompiler.ReadyToRun.csproj index d326121cb517ee..8a8e116c8f5c0f 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/ILCompiler.ReadyToRun.csproj +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/ILCompiler.ReadyToRun.csproj @@ -45,6 +45,9 @@ + + + diff --git a/src/coreclr/vm/corelib.h b/src/coreclr/vm/corelib.h index e0cde61d373d8e..b2dae8c9ed9e6b 100644 --- a/src/coreclr/vm/corelib.h +++ b/src/coreclr/vm/corelib.h @@ -747,8 +747,9 @@ DEFINE_METHOD(ASYNC_HELPERS, COMPLETED_TASK, CompletedTask, NoSi DEFINE_METHOD(ASYNC_HELPERS, CAPTURE_EXECUTION_CONTEXT, CaptureExecutionContext, NoSig) DEFINE_METHOD(ASYNC_HELPERS, RESTORE_EXECUTION_CONTEXT, RestoreExecutionContext, NoSig) DEFINE_METHOD(ASYNC_HELPERS, CAPTURE_CONTINUATION_CONTEXT, CaptureContinuationContext, NoSig) -DEFINE_METHOD(ASYNC_HELPERS, CAPTURE_CONTEXTS, CaptureContexts, NoSig) -DEFINE_METHOD(ASYNC_HELPERS, RESTORE_CONTEXTS, RestoreContexts, NoSig) +DEFINE_METHOD(ASYNC_HELPERS, CAPTURE_CONTEXTS, CaptureContexts, NoSig) +DEFINE_METHOD(ASYNC_HELPERS, RESTORE_CONTEXTS, RestoreContexts, NoSig) +DEFINE_METHOD(ASYNC_HELPERS, ASYNC_CALL_CONTINUATION, AsyncCallContinuation, SM_RetContinuation) #ifdef TARGET_BROWSER DEFINE_METHOD(ASYNC_HELPERS, HANDLE_ASYNC_ENTRYPOINT, HandleAsyncEntryPoint, SM_TaskOfInt_RetInt) @@ -1098,7 +1099,6 @@ DEFINE_METHOD(STUBHELPERS, VALIDATE_BYREF, Validate DEFINE_METHOD(STUBHELPERS, GET_STUB_CONTEXT, GetStubContext, SM_RetIntPtr) DEFINE_METHOD(STUBHELPERS, LOG_PINNED_ARGUMENT, LogPinnedArgument, SM_IntPtr_IntPtr_RetVoid) DEFINE_METHOD(STUBHELPERS, NEXT_CALL_RETURN_ADDRESS, NextCallReturnAddress, SM_RetIntPtr) -DEFINE_METHOD(STUBHELPERS, ASYNC_CALL_CONTINUATION, AsyncCallContinuation, SM_RetContinuation) DEFINE_METHOD(STUBHELPERS, SAFE_HANDLE_ADD_REF, SafeHandleAddRef, SM_SafeHandle_RefBool_RetIntPtr) DEFINE_METHOD(STUBHELPERS, SAFE_HANDLE_RELEASE, SafeHandleRelease, SM_SafeHandle_RetVoid)