Skip to content

Commit b8c97b9

Browse files
nate-chandlernate-chandler
authored
Merge pull request #80361 from nate-chandler/general-coro/20250327/1
[CoroutineAccessors] Open code alloc/dealloc functions.
2 parents 0222f90 + b38d3ee commit b8c97b9

File tree

11 files changed

+223
-163
lines changed

11 files changed

+223
-163
lines changed

‎include/swift/Runtime/Coro.h‎

Lines changed: 0 additions & 32 deletions
This file was deleted.

‎include/swift/Runtime/RuntimeFunctions.def‎

Lines changed: 0 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -3008,26 +3008,6 @@ FUNCTION(MemsetS, c, memset_s, C_CC, AlwaysAvailable,
30083008
EFFECT(RuntimeEffect::NoEffect),
30093009
UNKNOWN_MEMEFFECTS)
30103010

3011-
// void *swift_coro_alloc(CoroAllocator *, size_t size);
3012-
FUNCTION(CoroAlloc,
3013-
Swift, swift_coro_alloc, SwiftCC,
3014-
CoroutineAccessorsAvailability,
3015-
RETURNS(Int8PtrTy),
3016-
ARGS(CoroAllocatorPtrTy, SizeTy),
3017-
NO_ATTRS,
3018-
EFFECT(RuntimeEffect::Allocating, RuntimeEffect::Concurrency),
3019-
UNKNOWN_MEMEFFECTS)
3020-
3021-
// void swift_coro_dealloc(CoroAllocator *, void *ptr);
3022-
FUNCTION(CoroDealloc,
3023-
Swift, swift_coro_dealloc, SwiftCC,
3024-
CoroutineAccessorsAvailability,
3025-
RETURNS(VoidTy),
3026-
ARGS(CoroAllocatorPtrTy, Int8PtrTy),
3027-
NO_ATTRS,
3028-
EFFECT(RuntimeEffect::Deallocating, RuntimeEffect::Concurrency),
3029-
UNKNOWN_MEMEFFECTS)
3030-
30313011
#undef RETURNS
30323012
#undef ARGS
30333013
#undef ATTRS

‎lib/IRGen/GenCall.cpp‎

Lines changed: 145 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -5118,68 +5118,133 @@ void irgen::emitYieldManyCoroutineEntry(
51185118
allocFn, deallocFn, {});
51195119
}
51205120

5121-
static llvm::Constant *getCoroAllocWrapperFn(IRGenModule &IGM) {
5121+
static llvm::Constant *getCoroAllocFn(IRGenModule &IGM) {
5122+
auto isSwiftCoroCCAvailable = IGM.SwiftCoroCC == llvm::CallingConv::SwiftCoro;
51225123
return IGM.getOrCreateHelperFunction(
5123-
"__swift_coro_alloc_", IGM.Int8PtrTy,
5124-
{IGM.CoroAllocatorPtrTy, IGM.SizeTy},
5125-
[](IRGenFunction &IGF) {
5124+
"_swift_coro_alloc", IGM.Int8PtrTy, {IGM.CoroAllocatorPtrTy, IGM.SizeTy},
5125+
[isSwiftCoroCCAvailable](IRGenFunction &IGF) {
51265126
auto parameters = IGF.collectParameters();
51275127
auto *allocator = parameters.claimNext();
51285128
auto *size = parameters.claimNext();
5129-
auto *nullAllocator = IGF.Builder.CreateCmp(
5130-
llvm::CmpInst::Predicate::ICMP_EQ, allocator,
5131-
llvm::ConstantPointerNull::get(
5132-
cast<llvm::PointerType>(allocator->getType())));
5133-
auto *poplessReturn = IGF.createBasicBlock("coro.return.popless");
5134-
auto *normalReturn = IGF.createBasicBlock("coro.return.normal");
5135-
IGF.Builder.CreateCondBr(nullAllocator, poplessReturn, normalReturn);
5136-
IGF.Builder.emitBlock(poplessReturn);
5137-
// Emit the dynamic alloca.
5138-
auto *alloca =
5139-
IGF.Builder.IRBuilderBase::CreateAlloca(IGF.IGM.Int8Ty, size);
5140-
alloca->setAlignment(llvm::Align(MaximumAlignment));
5141-
auto *retPopless = IGF.Builder.CreateIntrinsic(
5142-
IGF.IGM.VoidTy, llvm::Intrinsic::ret_popless, {});
5143-
retPopless->setTailCallKind(llvm::CallInst::TailCallKind::TCK_MustTail);
5144-
IGF.Builder.CreateRet(alloca);
5145-
IGF.Builder.emitBlock(normalReturn);
5146-
auto *call = IGF.Builder.CreateCall(
5147-
IGF.IGM.getCoroAllocFunctionPointer(), {allocator, size});
5129+
if (isSwiftCoroCCAvailable) {
5130+
// swiftcorocc is available, so if there's no allocator pointer,
5131+
// allocate storage on the stack and return a pointer to it without
5132+
// popping the stack.
5133+
auto *nullAllocator = IGF.Builder.CreateCmp(
5134+
llvm::CmpInst::Predicate::ICMP_EQ, allocator,
5135+
llvm::ConstantPointerNull::get(
5136+
cast<llvm::PointerType>(allocator->getType())));
5137+
auto *poplessReturn = IGF.createBasicBlock("popless");
5138+
auto *normalReturn = IGF.createBasicBlock("normal");
5139+
IGF.Builder.CreateCondBr(nullAllocator, poplessReturn, normalReturn);
5140+
IGF.Builder.emitBlock(poplessReturn);
5141+
// Emit the dynamic alloca.
5142+
auto *alloca =
5143+
IGF.Builder.IRBuilderBase::CreateAlloca(IGF.IGM.Int8Ty, size);
5144+
alloca->setAlignment(llvm::Align(MaximumAlignment));
5145+
auto *retPopless = IGF.Builder.CreateIntrinsic(
5146+
IGF.IGM.VoidTy, llvm::Intrinsic::ret_popless, {});
5147+
retPopless->setTailCallKind(
5148+
llvm::CallInst::TailCallKind::TCK_MustTail);
5149+
IGF.Builder.CreateRet(alloca);
5150+
// Start emitting the "normal" block.
5151+
IGF.Builder.emitBlock(normalReturn);
5152+
}
5153+
auto *calleePtr = IGF.Builder.CreateInBoundsGEP(
5154+
IGF.IGM.CoroAllocatorTy, allocator,
5155+
{llvm::ConstantInt::get(IGF.IGM.Int32Ty, 0),
5156+
llvm::ConstantInt::get(IGF.IGM.Int32Ty, 1)});
5157+
auto *callee = IGF.Builder.CreateLoad(
5158+
Address(calleePtr, IGF.IGM.CoroAllocateFnTy->getPointerTo(),
5159+
IGF.IGM.getPointerAlignment()),
5160+
"allocate_fn");
5161+
auto fnPtr = FunctionPointer::createUnsigned(
5162+
FunctionPointer::Kind::Function, callee,
5163+
Signature(cast<llvm::FunctionType>(IGF.IGM.CoroAllocateFnTy), {},
5164+
IGF.IGM.SwiftCC));
5165+
auto *call = IGF.Builder.CreateCall(fnPtr, {size});
5166+
call->setDoesNotThrow();
5167+
call->setCallingConv(IGF.IGM.SwiftCC);
51485168
IGF.Builder.CreateRet(call);
51495169
},
5150-
/*setIsNoInline=*/false,
5170+
/*setIsNoInline=*/true,
51515171
/*forPrologue=*/false,
51525172
/*isPerformanceConstraint=*/false,
5153-
/*optionalLinkageOverride=*/nullptr, llvm::CallingConv::SwiftCoro);
5173+
/*optionalLinkageOverride=*/nullptr, IGM.SwiftCoroCC);
51545174
}
51555175

5156-
static llvm::Constant *getCoroDeallocWrapperFn(IRGenModule &IGM) {
5176+
static llvm::Constant *getCoroDeallocFn(IRGenModule &IGM) {
5177+
auto isSwiftCoroCCAvailable = IGM.SwiftCoroCC == llvm::CallingConv::SwiftCoro;
51575178
return IGM.getOrCreateHelperFunction(
5158-
"__swift_coro_dealloc_", IGM.VoidTy,
5179+
"_swift_coro_dealloc", IGM.VoidTy,
51595180
{IGM.CoroAllocatorPtrTy, IGM.Int8PtrTy},
5160-
[](IRGenFunction &IGF) {
5181+
[isSwiftCoroCCAvailable](IRGenFunction &IGF) {
51615182
auto parameters = IGF.collectParameters();
51625183
auto *allocator = parameters.claimNext();
51635184
auto *ptr = parameters.claimNext();
5164-
auto *nullAllocator = IGF.Builder.CreateCmp(
5165-
llvm::CmpInst::Predicate::ICMP_EQ, allocator,
5166-
llvm::ConstantPointerNull::get(
5167-
cast<llvm::PointerType>(allocator->getType())));
5168-
auto *bailBlock = IGF.createBasicBlock("bail");
5169-
auto *forwardBlock = IGF.createBasicBlock("forward");
5170-
IGF.Builder.CreateCondBr(nullAllocator, bailBlock, forwardBlock);
5171-
IGF.Builder.emitBlock(bailBlock);
5172-
// Emit the dynamic alloca.
5185+
if (isSwiftCoroCCAvailable) {
5186+
// swiftcorocc is available, so if there's no allocator pointer,
5187+
// storage was allocated on the stack which will be naturally cleaned
5188+
// up when the coroutine's frame is "freed".
5189+
auto *nullAllocator = IGF.Builder.CreateCmp(
5190+
llvm::CmpInst::Predicate::ICMP_EQ, allocator,
5191+
llvm::ConstantPointerNull::get(
5192+
cast<llvm::PointerType>(allocator->getType())));
5193+
auto *bailBlock = IGF.createBasicBlock("null_allocator");
5194+
auto *normalBlock = IGF.createBasicBlock("nonnull_allocator");
5195+
IGF.Builder.CreateCondBr(nullAllocator, bailBlock, normalBlock);
5196+
IGF.Builder.emitBlock(bailBlock);
5197+
// Nothing to do here.
5198+
IGF.Builder.CreateRetVoid();
5199+
// Start emitting the "normal" block.
5200+
IGF.Builder.emitBlock(normalBlock);
5201+
}
5202+
auto shouldDeallocateImmediatelyFlag = CoroAllocatorFlags(0);
5203+
shouldDeallocateImmediatelyFlag.setShouldDeallocateImmediately(true);
5204+
auto *flagsPtr = IGF.Builder.CreateInBoundsGEP(
5205+
IGF.IGM.CoroAllocatorTy, allocator,
5206+
{llvm::ConstantInt::get(IGF.IGM.Int32Ty, 0),
5207+
llvm::ConstantInt::get(IGF.IGM.Int32Ty, 0)});
5208+
auto *flags = IGF.Builder.CreateLoad(
5209+
Address(flagsPtr, IGF.IGM.Int32Ty, Alignment(4)), "");
5210+
auto *deallocDeferringAllocator = IGF.Builder.CreateAnd(
5211+
flags,
5212+
llvm::APInt(IGF.IGM.Int32Ty->getBitWidth(),
5213+
shouldDeallocateImmediatelyFlag.getOpaqueValue()));
5214+
auto *isDeallocDeferringAllocator = IGF.Builder.CreateICmpNE(
5215+
deallocDeferringAllocator,
5216+
llvm::ConstantInt::get(IGF.IGM.Int32Ty, 0));
5217+
auto *deferringAllocatorBlock =
5218+
IGF.createBasicBlock("deferring_allocator");
5219+
auto *normalBlock = IGF.createBasicBlock("normal");
5220+
IGF.Builder.CreateCondBr(isDeallocDeferringAllocator,
5221+
deferringAllocatorBlock, normalBlock);
5222+
IGF.Builder.emitBlock(deferringAllocatorBlock);
5223+
// Nothing to do here.
51735224
IGF.Builder.CreateRetVoid();
5174-
IGF.Builder.emitBlock(forwardBlock);
5175-
IGF.Builder.CreateCall(
5176-
IGF.IGM.getCoroDeallocFunctionPointer(), {allocator, ptr});
5225+
// Start emitting the "normal" block.
5226+
IGF.Builder.emitBlock(normalBlock);
5227+
auto *calleePtr = IGF.Builder.CreateInBoundsGEP(
5228+
IGF.IGM.CoroAllocatorTy, allocator,
5229+
{llvm::ConstantInt::get(IGF.IGM.Int32Ty, 0),
5230+
llvm::ConstantInt::get(IGF.IGM.Int32Ty, 2)});
5231+
auto *callee = IGF.Builder.CreateLoad(
5232+
Address(calleePtr, IGF.IGM.CoroDeallocateFnTy->getPointerTo(),
5233+
IGF.IGM.getPointerAlignment()),
5234+
"deallocate_fn");
5235+
auto fnPtr = FunctionPointer::createUnsigned(
5236+
FunctionPointer::Kind::Function, callee,
5237+
Signature(cast<llvm::FunctionType>(IGF.IGM.CoroDeallocateFnTy), {},
5238+
IGF.IGM.SwiftCC));
5239+
auto *call = IGF.Builder.CreateCall(fnPtr, {ptr});
5240+
call->setDoesNotThrow();
5241+
call->setCallingConv(IGF.IGM.SwiftCC);
51775242
IGF.Builder.CreateRetVoid();
51785243
},
5179-
/*setIsNoInline=*/false,
5244+
/*setIsNoInline=*/true,
51805245
/*forPrologue=*/false,
51815246
/*isPerformanceConstraint=*/false,
5182-
/*optionalLinkageOverride=*/nullptr, llvm::CallingConv::SwiftCoro);
5247+
/*optionalLinkageOverride=*/nullptr, IGM.SwiftCoroCC);
51835248
}
51845249

51855250
void irgen::emitYieldOnce2CoroutineEntry(IRGenFunction &IGF,
@@ -5188,14 +5253,8 @@ void irgen::emitYieldOnce2CoroutineEntry(IRGenFunction &IGF,
51885253
llvm::Value *allocator,
51895254
llvm::GlobalVariable *cfp) {
51905255
IGF.setCoroutineAllocator(allocator);
5191-
auto isSwiftCoroCCAvailable =
5192-
IGF.IGM.SwiftCoroCC == llvm::CallingConv::SwiftCoro;
5193-
auto allocFn = IGF.IGM.getOpaquePtr(isSwiftCoroCCAvailable
5194-
? getCoroAllocWrapperFn(IGF.IGM)
5195-
: IGF.IGM.getCoroAllocFn());
5196-
auto deallocFn = IGF.IGM.getOpaquePtr(isSwiftCoroCCAvailable
5197-
? getCoroDeallocWrapperFn(IGF.IGM)
5198-
: IGF.IGM.getCoroDeallocFn());
5256+
auto allocFn = IGF.IGM.getOpaquePtr(getCoroAllocFn(IGF.IGM));
5257+
auto deallocFn = IGF.IGM.getOpaquePtr(getCoroDeallocFn(IGF.IGM));
51995258
emitRetconCoroutineEntry(
52005259
IGF, fnType, buffer, llvm::Intrinsic::coro_id_retcon_once_dynamic,
52015260
Size(-1) /*dynamic-to-IRGen size*/, IGF.IGM.getCoroStaticFrameAlignment(),
@@ -5230,6 +5289,40 @@ Address irgen::emitAllocYieldManyCoroutineBuffer(IRGenFunction &IGF) {
52305289
getYieldManyCoroutineBufferAlignment(IGF.IGM));
52315290
}
52325291

5292+
static llvm::Constant *getAddrOfSwiftCCMalloc(IRGenModule &IGM) {
5293+
auto mallocFnPtr = IGM.getMallocFunctionPointer();
5294+
auto sig = mallocFnPtr.getSignature();
5295+
if (sig.getCallingConv() == IGM.SwiftCC) {
5296+
return IGM.getMallocFn();
5297+
}
5298+
return IGM.getOrCreateHelperFunction(
5299+
"_swift_malloc", sig.getType()->getReturnType(), sig.getType()->params(),
5300+
[](IRGenFunction &IGF) {
5301+
auto parameters = IGF.collectParameters();
5302+
auto *size = parameters.claimNext();
5303+
auto malloc = IGF.IGM.getMallocFunctionPointer();
5304+
auto *call = IGF.Builder.CreateCall(malloc, {size});
5305+
IGF.Builder.CreateRet(call);
5306+
});
5307+
}
5308+
5309+
static llvm::Constant *getAddrOfSwiftCCFree(IRGenModule &IGM) {
5310+
auto freeFnPtr = IGM.getFreeFunctionPointer();
5311+
auto sig = freeFnPtr.getSignature();
5312+
if (sig.getCallingConv() == IGM.SwiftCC) {
5313+
return IGM.getFreeFn();
5314+
}
5315+
return IGM.getOrCreateHelperFunction(
5316+
"_swift_free", sig.getType()->getReturnType(), sig.getType()->params(),
5317+
[](IRGenFunction &IGF) {
5318+
auto parameters = IGF.collectParameters();
5319+
auto *ptr = parameters.claimNext();
5320+
auto free = IGF.IGM.getFreeFunctionPointer();
5321+
IGF.Builder.CreateCall(free, {ptr});
5322+
IGF.Builder.CreateRetVoid();
5323+
});
5324+
}
5325+
52335326
static llvm::Constant *getAddrOfGlobalCoroAllocator(
52345327
IRGenModule &IGM, CoroAllocatorKind kind, bool shouldDeallocateImmediately,
52355328
llvm::Constant *allocFn, llvm::Constant *deallocFn) {
@@ -5251,7 +5344,8 @@ static llvm::Constant *getAddrOfGlobalCoroAllocator(
52515344
llvm::Constant *IRGenModule::getAddrOfGlobalCoroMallocAllocator() {
52525345
return getAddrOfGlobalCoroAllocator(*this, CoroAllocatorKind::Malloc,
52535346
/*shouldDeallocateImmediately=*/true,
5254-
getMallocFn(), getFreeFn());
5347+
getAddrOfSwiftCCMalloc(*this),
5348+
getAddrOfSwiftCCFree(*this));
52555349
}
52565350
llvm::Constant *IRGenModule::getAddrOfGlobalCoroAsyncTaskAllocator() {
52575351
return getAddrOfGlobalCoroAllocator(*this, CoroAllocatorKind::Async,

‎lib/IRGen/IRGenModule.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -629,7 +629,7 @@ IRGenModule::IRGenModule(IRGenerator &irgen,
629629
if (isCoroCCSupported) {
630630
SwiftCoroCC = llvm::CallingConv::SwiftCoro;
631631
} else {
632-
SwiftCoroCC = llvm::CallingConv::Swift;
632+
SwiftCoroCC = SwiftCC;
633633
}
634634

635635
if (opts.DebugInfoLevel > IRGenDebugInfoLevel::None)

‎stdlib/public/Concurrency/TaskAlloc.cpp‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020
#include "TaskPrivate.h"
2121
#include "swift/ABI/Task.h"
2222
#include "swift/Runtime/Concurrency.h"
23-
#include "swift/Runtime/Coro.h"
2423

2524
#include <stdlib.h>
2625

‎stdlib/public/runtime/CMakeLists.txt‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,6 @@ set(swift_runtime_sources
3838
Bincompat.cpp
3939
BytecodeLayouts.cpp
4040
Casting.cpp
41-
Coro.cpp
4241
CrashReporter.cpp
4342
Demangle.cpp
4443
DynamicCast.cpp

‎stdlib/public/runtime/Coro.cpp‎

Lines changed: 0 additions & 32 deletions
This file was deleted.

0 commit comments

Comments
 (0)