Skip to content

Commit 7c8cd13

Browse files
jckarterjckarter
authored
Merge pull request #76704 from jckarter/atomic-layout-6.0
[6.0] IRGen: Set a "not bitwise borrowable" bit in value witnesses for `@_rawLayout` types.
2 parents ef70db4 + cad6bec commit 7c8cd13

16 files changed

+204
-81
lines changed

‎include/swift/ABI/MetadataValues.h‎

Lines changed: 33 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -168,17 +168,18 @@ class TargetValueWitnessFlags {
168168
// flags for the struct. (The "non-inline" and "has-extra-inhabitants" bits
169169
// still require additional fixup.)
170170
enum : uint32_t {
171-
AlignmentMask = 0x000000FF,
172-
// unused 0x0000FF00,
173-
IsNonPOD = 0x00010000,
174-
IsNonInline = 0x00020000,
175-
// unused 0x00040000,
176-
HasSpareBits = 0x00080000,
177-
IsNonBitwiseTakable = 0x00100000,
178-
HasEnumWitnesses = 0x00200000,
179-
Incomplete = 0x00400000,
180-
IsNonCopyable = 0x00800000,
181-
// unused 0xFF000000,
171+
AlignmentMask = 0x000000FF,
172+
// unused 0x0000FF00,
173+
IsNonPOD = 0x00010000,
174+
IsNonInline = 0x00020000,
175+
// unused 0x00040000,
176+
HasSpareBits = 0x00080000,
177+
IsNonBitwiseTakable = 0x00100000,
178+
HasEnumWitnesses = 0x00200000,
179+
Incomplete = 0x00400000,
180+
IsNonCopyable = 0x00800000,
181+
IsNonBitwiseBorrowable = 0x01000000,
182+
// unused 0xFE000000,
182183
};
183184

184185
static constexpr const uint32_t MaxNumExtraInhabitants = 0x7FFFFFFF;
@@ -243,6 +244,27 @@ class TargetValueWitnessFlags {
243244
(isBT ? 0 : IsNonBitwiseTakable));
244245
}
245246

247+
/// True if values of this type can be passed by value when borrowed.
248+
/// If this bit is true, then borrows of the value are independent of the
249+
/// value's address, so a value can be passed in registers or memcpy'd
250+
/// while borrowed. This is in contrast to Rust, for instance, where a
251+
/// `&T` type is always represented as a pointer, and borrowing a
252+
/// value always moves the borrowed value into memory.
253+
bool isBitwiseBorrowable() const {
254+
/// This bit was introduced with Swift 6; prior to the introduction of
255+
/// `Atomic` and `Mutex`, a type was always bitwise-borrowable if it
256+
/// was bitwise-takable. Compilers and runtimes before Swift 6 would
257+
/// never set the `IsNonBitwiseBorrowable` bit in the value witness
258+
/// table, but any type that sets `IsNonBitwiseTakable` is definitely
259+
/// not bitwise borrowable.
260+
return isBitwiseTakable()
261+
&& !(Data & IsNonBitwiseBorrowable);
262+
}
263+
constexpr TargetValueWitnessFlags withBitwiseBorrowable(bool isBB) const {
264+
return TargetValueWitnessFlags((Data & ~IsNonBitwiseBorrowable) |
265+
(isBB ? 0 : IsNonBitwiseBorrowable));
266+
}
267+
246268
/// True if values of this type can be copied.
247269
bool isCopyable() const { return !(Data & IsNonCopyable); }
248270
constexpr TargetValueWitnessFlags withCopyable(bool isCopyable) const {

‎lib/IRGen/GenEnum.cpp‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -3906,8 +3906,8 @@ namespace {
39063906
} else if (allSingleRefcount
39073907
&& ElementsWithNoPayload.size() <= 1) {
39083908
CopyDestroyKind = TaggedRefcounted;
3909-
} else if (this->EnumImplStrategy::BitwiseTakable == IsBitwiseTakable &&
3910-
Copyable == IsCopyable) {
3909+
} else if (this->EnumImplStrategy::BitwiseTakable == IsBitwiseTakableAndBorrowable
3910+
&& Copyable == IsCopyable) {
39113911
CopyDestroyKind = BitwiseTakable;
39123912
}
39133913
}
@@ -6365,7 +6365,7 @@ EnumImplStrategy::get(TypeConverter &TC, SILType type, EnumDecl *theEnum) {
63656365
? IsNotTriviallyDestroyable : IsTriviallyDestroyable;
63666366
auto copyable = !theEnum->canBeCopyable()
63676367
? IsNotCopyable : IsCopyable;
6368-
auto bitwiseTakable = IsBitwiseTakable; // FIXME: will there be check here?
6368+
auto bitwiseTakable = IsBitwiseTakableAndBorrowable; // FIXME: will there be check here?
63696369
bool allowFixedLayoutOptimizations = true;
63706370
std::vector<Element> elementsWithPayload;
63716371
std::vector<Element> elementsWithNoPayload;
@@ -6377,7 +6377,7 @@ EnumImplStrategy::get(TypeConverter &TC, SILType type, EnumDecl *theEnum) {
63776377
payloadTI.isTriviallyDestroyable(ResilienceExpansion::Maximal);
63786378
copyable = copyable & payloadTI.isCopyable(ResilienceExpansion::Maximal);
63796379
bitwiseTakable = bitwiseTakable &
6380-
payloadTI.isBitwiseTakable(ResilienceExpansion::Maximal);
6380+
payloadTI.getBitwiseTakable(ResilienceExpansion::Maximal);
63816381
};
63826382

63836383
if (TC.IGM.isResilient(theEnum, ResilienceExpansion::Minimal))
@@ -6825,7 +6825,8 @@ EnumImplStrategy::getFixedEnumTypeInfo(llvm::StructType *T, Size S,
68256825
AlwaysFixedSize);
68266826
break;
68276827
case Loadable:
6828-
assert(isBT && "loadable enum not bitwise takable?!");
6828+
assert(isBT == IsBitwiseTakableAndBorrowable
6829+
&& "loadable enum not bitwise takable?!");
68296830
mutableTI = new LoadableEnumTypeInfo(*this, T, S, std::move(SB), A,
68306831
isTriviallyDestroyable,
68316832
isCopyable,
@@ -7038,7 +7039,7 @@ TypeInfo *SinglePayloadEnumImplStrategy::completeFixedLayout(
70387039
getFixedEnumTypeInfo(
70397040
enumTy, Size(sizeWithTag), spareBits.build(), alignment,
70407041
deinit & payloadTI.isTriviallyDestroyable(ResilienceExpansion::Maximal),
7041-
payloadTI.isBitwiseTakable(ResilienceExpansion::Maximal),
7042+
payloadTI.getBitwiseTakable(ResilienceExpansion::Maximal),
70427043
copyable);
70437044
if (TIK >= Loadable && CopyDestroyKind == Normal) {
70447045
computePayloadTypesAndTagType(TC.IGM, *TI, PayloadTypesAndTagType);
@@ -7073,7 +7074,7 @@ TypeInfo *SinglePayloadEnumImplStrategy::completeDynamicLayout(
70737074
return registerEnumTypeInfo(new NonFixedEnumTypeInfo(*this, enumTy,
70747075
alignment,
70757076
deinit & payloadTI.isTriviallyDestroyable(ResilienceExpansion::Maximal),
7076-
payloadTI.isBitwiseTakable(ResilienceExpansion::Maximal),
7077+
payloadTI.getBitwiseTakable(ResilienceExpansion::Maximal),
70777078
copyable,
70787079
enumAccessible));
70797080
}
@@ -7108,16 +7109,15 @@ MultiPayloadEnumImplStrategy::completeFixedLayout(TypeConverter &TC,
71087109
? IsNotCopyable : IsCopyable;
71097110
auto isTriviallyDestroyable = theEnum->getValueTypeDestructor()
71107111
? IsNotTriviallyDestroyable : IsTriviallyDestroyable;
7111-
IsBitwiseTakable_t isBT = IsBitwiseTakable;
7112+
IsBitwiseTakable_t isBT = IsBitwiseTakableAndBorrowable;
71127113
PayloadSize = 0;
71137114
for (auto &elt : ElementsWithPayload) {
71147115
auto &fixedPayloadTI = cast<FixedTypeInfo>(*elt.ti);
71157116
if (fixedPayloadTI.getFixedAlignment() > worstAlignment)
71167117
worstAlignment = fixedPayloadTI.getFixedAlignment();
71177118
if (!fixedPayloadTI.isTriviallyDestroyable(ResilienceExpansion::Maximal))
71187119
isTriviallyDestroyable = IsNotTriviallyDestroyable;
7119-
if (!fixedPayloadTI.isBitwiseTakable(ResilienceExpansion::Maximal))
7120-
isBT = IsNotBitwiseTakable;
7120+
isBT &= fixedPayloadTI.getBitwiseTakable(ResilienceExpansion::Maximal);
71217121

71227122
unsigned payloadBytes = fixedPayloadTI.getFixedSize().getValue();
71237123
unsigned payloadBits = fixedPayloadTI.getFixedSize().getValueInBits();
@@ -7274,12 +7274,12 @@ TypeInfo *MultiPayloadEnumImplStrategy::completeDynamicLayout(
72747274
Alignment alignment(1);
72757275
auto td = theEnum->getValueTypeDestructor()
72767276
? IsNotTriviallyDestroyable : IsTriviallyDestroyable;
7277-
auto bt = IsBitwiseTakable;
7277+
auto bt = IsBitwiseTakableAndBorrowable;
72787278
for (auto &element : ElementsWithPayload) {
72797279
auto &payloadTI = *element.ti;
72807280
alignment = std::max(alignment, payloadTI.getBestKnownAlignment());
72817281
td &= payloadTI.isTriviallyDestroyable(ResilienceExpansion::Maximal);
7282-
bt &= payloadTI.isBitwiseTakable(ResilienceExpansion::Maximal);
7282+
bt &= payloadTI.getBitwiseTakable(ResilienceExpansion::Maximal);
72837283
}
72847284

72857285
applyLayoutAttributes(TC.IGM, theEnum, /*fixed*/false, alignment);

‎lib/IRGen/GenExistential.cpp‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -897,11 +897,16 @@ class OpaqueExistentialTypeInfo final :
897897

898898
OpaqueExistentialTypeInfo(ArrayRef<const ProtocolDecl *> protocols,
899899
llvm::Type *ty, Size size,
900+
IsCopyable_t copyable,
900901
SpareBitVector &&spareBits,
901902
Alignment align)
902903
: super(protocols, ty, size,
903904
std::move(spareBits), align,
904-
IsNotTriviallyDestroyable, IsBitwiseTakable, IsCopyable,
905+
IsNotTriviallyDestroyable,
906+
// Copyable existentials are bitwise-takable and borrowable.
907+
// Noncopyable existentials are bitwise-takable only.
908+
copyable ? IsBitwiseTakableAndBorrowable : IsBitwiseTakableOnly,
909+
copyable,
905910
IsFixedSize) {}
906911

907912
public:
@@ -1645,14 +1650,19 @@ static const TypeInfo *createExistentialTypeInfo(IRGenModule &IGM, CanType T) {
16451650
OpaqueExistentialLayout opaque(protosWithWitnessTables.size());
16461651
Alignment align = opaque.getAlignment(IGM);
16471652
Size size = opaque.getSize(IGM);
1653+
IsCopyable_t copyable = T->isNoncopyable() ? IsNotCopyable : IsCopyable;
1654+
16481655
// There are spare bits in the metadata pointer and witness table pointers
16491656
// consistent with a native object reference.
1650-
// TODO: There are spare bits we could theoretically use in the type metadata
1651-
// and witness table pointers, but opaque existentials are currently address-
1657+
// NB: There are spare bits we could theoretically use in the type metadata
1658+
// and witness table pointers, but opaque existentials are address-
16521659
// only, and we can't soundly take advantage of spare bits for in-memory
16531660
// representations.
1661+
// Maybe an ABI break that made opaque existentials loadable could use those
1662+
// bits, though.
16541663
auto spareBits = SpareBitVector::getConstant(size.getValueInBits(), false);
16551664
return OpaqueExistentialTypeInfo::create(protosWithWitnessTables, type, size,
1665+
copyable,
16561666
std::move(spareBits),
16571667
align);
16581668
}

‎lib/IRGen/GenFunc.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -577,7 +577,7 @@ const TypeInfo *TypeConverter::convertBlockStorageType(SILBlockStorageType *T) {
577577

578578
size = captureOffset + fixedCapture->getFixedSize();
579579
pod = fixedCapture->isTriviallyDestroyable(ResilienceExpansion::Maximal);
580-
bt = fixedCapture->isBitwiseTakable(ResilienceExpansion::Maximal);
580+
bt = fixedCapture->getBitwiseTakable(ResilienceExpansion::Maximal);
581581
}
582582

583583
llvm::Type *storageElts[] = {

‎lib/IRGen/GenReflection.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1137,7 +1137,7 @@ class FixedTypeMetadataBuilder : public ReflectionMetadataBuilder {
11371137

11381138
auto alignment = ti->getFixedAlignment().getValue();
11391139
unsigned bitwiseTakable =
1140-
(ti->isBitwiseTakable(ResilienceExpansion::Minimal) == IsBitwiseTakable
1140+
(ti->getBitwiseTakable(ResilienceExpansion::Minimal) >= IsBitwiseTakableOnly
11411141
? 1 : 0);
11421142
B.addInt32(alignment | (bitwiseTakable << 16));
11431143

‎lib/IRGen/GenType.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -233,7 +233,7 @@ llvm::Value *FixedTypeInfo::getIsTriviallyDestroyable(IRGenFunction &IGF, SILTyp
233233
}
234234
llvm::Value *FixedTypeInfo::getIsBitwiseTakable(IRGenFunction &IGF, SILType T) const {
235235
return llvm::ConstantInt::get(IGF.IGM.Int1Ty,
236-
isBitwiseTakable(ResilienceExpansion::Maximal) == IsBitwiseTakable);
236+
getBitwiseTakable(ResilienceExpansion::Maximal) >= IsBitwiseTakableOnly);
237237
}
238238
llvm::Constant *FixedTypeInfo::getStaticStride(IRGenModule &IGM) const {
239239
return IGM.getSize(getFixedStride());

‎lib/IRGen/GenValueWitness.cpp‎

Lines changed: 32 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -847,12 +847,22 @@ ValueWitnessFlags getValueWitnessFlags(const TypeInfo *TI, SILType concreteType,
847847
bool isInline = packing == FixedPacking::OffsetZero;
848848
bool isBitwiseTakable =
849849
fixedTI->isBitwiseTakable(ResilienceExpansion::Maximal);
850+
bool isBitwiseBorrowable =
851+
fixedTI->isBitwiseBorrowable(ResilienceExpansion::Maximal);
850852
assert(isBitwiseTakable || !isInline);
851853
flags = flags.withAlignment(fixedTI->getFixedAlignment().getValue())
852854
.withPOD(fixedTI->isTriviallyDestroyable(ResilienceExpansion::Maximal))
853855
.withCopyable(fixedTI->isCopyable(ResilienceExpansion::Maximal))
854856
.withInlineStorage(isInline)
855-
.withBitwiseTakable(isBitwiseTakable);
857+
.withBitwiseTakable(isBitwiseTakable)
858+
// the IsNotBitwiseBorrowable bit only needs to be set if the
859+
// type is bitwise-takable but not bitwise-borrowable, since
860+
// a type must be bitwise-takable to be bitwise-borrowable.
861+
//
862+
// Swift prior to version 6 didn't have the
863+
// IsNotBitwiseBorrowable bit, so to avoid unnecessary variation
864+
// in metadata output, we only set the bit when needed.
865+
.withBitwiseBorrowable(!isBitwiseTakable || isBitwiseBorrowable);
856866
} else {
857867
flags = flags.withIncomplete(true);
858868
}
@@ -1530,12 +1540,12 @@ llvm::Constant *IRGenModule::emitFixedTypeLayout(CanType t,
15301540
unsigned align = ti.getFixedAlignment().getValue();
15311541

15321542
bool pod = ti.isTriviallyDestroyable(ResilienceExpansion::Maximal);
1533-
bool bt = ti.isBitwiseTakable(ResilienceExpansion::Maximal);
1543+
IsBitwiseTakable_t bt = ti.getBitwiseTakable(ResilienceExpansion::Maximal);
15341544
unsigned numExtraInhabitants = ti.getFixedExtraInhabitantCount(*this);
15351545

15361546
// Try to use common type layouts exported by the runtime.
15371547
llvm::Constant *commonValueWitnessTable = nullptr;
1538-
if (pod && bt && numExtraInhabitants == 0) {
1548+
if (pod && bt == IsBitwiseTakableAndBorrowable && numExtraInhabitants == 0) {
15391549
if (size == 0)
15401550
commonValueWitnessTable =
15411551
getAddrOfValueWitnessTable(Context.TheEmptyTupleType);
@@ -1560,7 +1570,7 @@ llvm::Constant *IRGenModule::emitFixedTypeLayout(CanType t,
15601570

15611571
// Otherwise, see if a layout has been emitted with these characteristics
15621572
// already.
1563-
FixedLayoutKey key{size, numExtraInhabitants, align, pod, bt};
1573+
FixedLayoutKey key{size, numExtraInhabitants, align, pod, unsigned(bt)};
15641574

15651575
auto found = PrivateFixedLayouts.find(key);
15661576
if (found != PrivateFixedLayouts.end())
@@ -1576,16 +1586,29 @@ llvm::Constant *IRGenModule::emitFixedTypeLayout(CanType t,
15761586
addValueWitness(*this, witnesses, witness, packing, t, silTy, ti);
15771587
}
15781588

1589+
auto pod_bt_string = [](bool pod, IsBitwiseTakable_t bt) -> StringRef {
1590+
if (pod) {
1591+
return "_pod";
1592+
}
1593+
switch (bt) {
1594+
case IsNotBitwiseTakable:
1595+
return "";
1596+
case IsBitwiseTakableOnly:
1597+
return "_bt_nbb";
1598+
case IsBitwiseTakableAndBorrowable:
1599+
return "_bt";
1600+
}
1601+
};
1602+
15791603
auto layoutVar
15801604
= witnesses.finishAndCreateGlobal(
15811605
"type_layout_" + llvm::Twine(size)
15821606
+ "_" + llvm::Twine(align)
15831607
+ "_" + llvm::Twine::utohexstr(numExtraInhabitants)
1584-
+ (pod ? "_pod" :
1585-
bt ? "_bt" : ""),
1586-
getPointerAlignment(),
1587-
/*constant*/ true,
1588-
llvm::GlobalValue::PrivateLinkage);
1608+
+ pod_bt_string(pod, bt),
1609+
getPointerAlignment(),
1610+
/*constant*/ true,
1611+
llvm::GlobalValue::PrivateLinkage);
15891612

15901613
// Cast to the standard currency type for type layouts.
15911614
auto layout = llvm::ConstantExpr::getBitCast(layoutVar, Int8PtrPtrTy);

‎lib/IRGen/IRGen.h‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -80,9 +80,15 @@ inline IsLoadable_t &operator&=(IsLoadable_t &l, IsLoadable_t r) {
8080
return (l = (l & r));
8181
}
8282

83-
enum IsBitwiseTakable_t : bool { IsNotBitwiseTakable, IsBitwiseTakable };
83+
enum IsBitwiseTakable_t : uint8_t {
84+
IsNotBitwiseTakable = 0,
85+
// The type is bitwise-takable, but borrows are pinned to memory.
86+
IsBitwiseTakableOnly = 1,
87+
// The type is bitwise-takable and -borrowable.
88+
IsBitwiseTakableAndBorrowable = 3,
89+
};
8490
inline IsBitwiseTakable_t operator&(IsBitwiseTakable_t l, IsBitwiseTakable_t r) {
85-
return IsBitwiseTakable_t(unsigned(l) & unsigned(r));
91+
return IsBitwiseTakable_t(std::min(unsigned(l), unsigned(r)));
8692
}
8793
inline IsBitwiseTakable_t &operator&=(IsBitwiseTakable_t &l, IsBitwiseTakable_t r) {
8894
return (l = (l & r));

‎lib/IRGen/IRGenModule.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1363,7 +1363,7 @@ class IRGenModule {
13631363
unsigned numExtraInhabitants;
13641364
unsigned align: 16;
13651365
unsigned pod: 1;
1366-
unsigned bitwiseTakable: 1;
1366+
unsigned bitwiseTakable: 2;
13671367
};
13681368
friend struct ::llvm::DenseMapInfo<swift::irgen::IRGenModule::FixedLayoutKey>;
13691369
llvm::DenseMap<FixedLayoutKey, llvm::Constant *> PrivateFixedLayouts;

‎lib/IRGen/LoadableTypeInfo.h‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -64,8 +64,9 @@ class LoadableTypeInfo : public FixedTypeInfo {
6464
IsFixedSize_t alwaysFixedSize,
6565
SpecialTypeInfoKind stik = SpecialTypeInfoKind::Loadable)
6666
: FixedTypeInfo(type, size, spareBits, align, pod,
67-
// All currently implemented loadable types are bitwise-takable.
68-
IsBitwiseTakable,
67+
// All currently implemented loadable types are
68+
// bitwise-takable and -borrowable.
69+
IsBitwiseTakableAndBorrowable,
6970
copy, alwaysFixedSize, stik) {
7071
assert(isLoadable());
7172
}
@@ -78,8 +79,9 @@ class LoadableTypeInfo : public FixedTypeInfo {
7879
IsFixedSize_t alwaysFixedSize,
7980
SpecialTypeInfoKind stik = SpecialTypeInfoKind::Loadable)
8081
: FixedTypeInfo(type, size, std::move(spareBits), align, pod,
81-
// All currently implemented loadable types are bitwise-takable.
82-
IsBitwiseTakable,
82+
// All currently implemented loadable types are
83+
// bitwise-takable and borrowable.
84+
IsBitwiseTakableAndBorrowable,
8385
copy, alwaysFixedSize, stik) {
8486
assert(isLoadable());
8587
}

0 commit comments

Comments
 (0)