Skip to content

Commit 4e3f665

Browse files
jckarterjckarter
authored
Merge pull request #76688 from jckarter/atomic-layout
IRGen: Set a "not bitwise borrowable" bit in value witnesses for `@_rawLayout` types.
2 parents ec962fc + 57a56e5 commit 4e3f665

16 files changed

+214
-91
lines changed

‎include/swift/ABI/MetadataValues.h‎

Lines changed: 33 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -168,17 +168,18 @@ class TargetValueWitnessFlags {
168168
// flags for the struct. (The "non-inline" and "has-extra-inhabitants" bits
169169
// still require additional fixup.)
170170
enum : uint32_t {
171-
AlignmentMask = 0x000000FF,
172-
// unused 0x0000FF00,
173-
IsNonPOD = 0x00010000,
174-
IsNonInline = 0x00020000,
175-
// unused 0x00040000,
176-
HasSpareBits = 0x00080000,
177-
IsNonBitwiseTakable = 0x00100000,
178-
HasEnumWitnesses = 0x00200000,
179-
Incomplete = 0x00400000,
180-
IsNonCopyable = 0x00800000,
181-
// unused 0xFF000000,
171+
AlignmentMask = 0x000000FF,
172+
// unused 0x0000FF00,
173+
IsNonPOD = 0x00010000,
174+
IsNonInline = 0x00020000,
175+
// unused 0x00040000,
176+
HasSpareBits = 0x00080000,
177+
IsNonBitwiseTakable = 0x00100000,
178+
HasEnumWitnesses = 0x00200000,
179+
Incomplete = 0x00400000,
180+
IsNonCopyable = 0x00800000,
181+
IsNonBitwiseBorrowable = 0x01000000,
182+
// unused 0xFE000000,
182183
};
183184

184185
static constexpr const uint32_t MaxNumExtraInhabitants = 0x7FFFFFFF;
@@ -243,6 +244,27 @@ class TargetValueWitnessFlags {
243244
(isBT ? 0 : IsNonBitwiseTakable));
244245
}
245246

247+
/// True if values of this type can be passed by value when borrowed.
248+
/// If this bit is true, then borrows of the value are independent of the
249+
/// value's address, so a value can be passed in registers or memcpy'd
250+
/// while borrowed. This is in contrast to Rust, for instance, where a
251+
/// `&T` type is always represented as a pointer, and borrowing a
252+
/// value always moves the borrowed value into memory.
253+
bool isBitwiseBorrowable() const {
254+
/// This bit was introduced with Swift 6; prior to the introduction of
255+
/// `Atomic` and `Mutex`, a type was always bitwise-borrowable if it
256+
/// was bitwise-takable. Compilers and runtimes before Swift 6 would
257+
/// never set the `IsNonBitwiseBorrowable` bit in the value witness
258+
/// table, but any type that sets `IsNonBitwiseTakable` is definitely
259+
/// not bitwise borrowable.
260+
return isBitwiseTakable()
261+
&& !(Data & IsNonBitwiseBorrowable);
262+
}
263+
constexpr TargetValueWitnessFlags withBitwiseBorrowable(bool isBB) const {
264+
return TargetValueWitnessFlags((Data & ~IsNonBitwiseBorrowable) |
265+
(isBB ? 0 : IsNonBitwiseBorrowable));
266+
}
267+
246268
/// True if values of this type can be copied.
247269
bool isCopyable() const { return !(Data & IsNonCopyable); }
248270
constexpr TargetValueWitnessFlags withCopyable(bool isCopyable) const {

‎lib/IRGen/GenEnum.cpp‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -3927,8 +3927,8 @@ namespace {
39273927
} else if (allSingleRefcount
39283928
&& ElementsWithNoPayload.size() <= 1) {
39293929
CopyDestroyKind = TaggedRefcounted;
3930-
} else if (this->EnumImplStrategy::BitwiseTakable == IsBitwiseTakable &&
3931-
Copyable == IsCopyable) {
3930+
} else if (this->EnumImplStrategy::BitwiseTakable == IsBitwiseTakableAndBorrowable
3931+
&& Copyable == IsCopyable) {
39323932
CopyDestroyKind = BitwiseTakable;
39333933
}
39343934
}
@@ -6396,7 +6396,7 @@ EnumImplStrategy::get(TypeConverter &TC, SILType type, EnumDecl *theEnum) {
63966396
? IsNotTriviallyDestroyable : IsTriviallyDestroyable;
63976397
auto copyable = !theEnum->canBeCopyable()
63986398
? IsNotCopyable : IsCopyable;
6399-
auto bitwiseTakable = IsBitwiseTakable; // FIXME: will there be check here?
6399+
auto bitwiseTakable = IsBitwiseTakableAndBorrowable; // FIXME: will there be check here?
64006400
bool allowFixedLayoutOptimizations = true;
64016401
std::vector<Element> elementsWithPayload;
64026402
std::vector<Element> elementsWithNoPayload;
@@ -6408,7 +6408,7 @@ EnumImplStrategy::get(TypeConverter &TC, SILType type, EnumDecl *theEnum) {
64086408
payloadTI.isTriviallyDestroyable(ResilienceExpansion::Maximal);
64096409
copyable = copyable & payloadTI.isCopyable(ResilienceExpansion::Maximal);
64106410
bitwiseTakable = bitwiseTakable &
6411-
payloadTI.isBitwiseTakable(ResilienceExpansion::Maximal);
6411+
payloadTI.getBitwiseTakable(ResilienceExpansion::Maximal);
64126412
};
64136413

64146414
if (TC.IGM.isResilient(theEnum, ResilienceExpansion::Minimal))
@@ -6861,7 +6861,8 @@ EnumImplStrategy::getFixedEnumTypeInfo(llvm::StructType *T, Size S,
68616861
abiAccessible);
68626862
break;
68636863
case Loadable:
6864-
assert(isBT && "loadable enum not bitwise takable?!");
6864+
assert(isBT == IsBitwiseTakableAndBorrowable
6865+
&& "loadable enum not bitwise takable?!");
68656866
mutableTI = new LoadableEnumTypeInfo(*this, T, S, std::move(SB), A,
68666867
isTriviallyDestroyable,
68676868
isCopyable,
@@ -7084,7 +7085,7 @@ TypeInfo *SinglePayloadEnumImplStrategy::completeFixedLayout(
70847085
getFixedEnumTypeInfo(
70857086
enumTy, Size(sizeWithTag), spareBits.build(), alignment,
70867087
deinit & payloadTI.isTriviallyDestroyable(ResilienceExpansion::Maximal),
7087-
payloadTI.isBitwiseTakable(ResilienceExpansion::Maximal),
7088+
payloadTI.getBitwiseTakable(ResilienceExpansion::Maximal),
70887089
copyable, isABIAccessible);
70897090

70907091
if (TIK >= Loadable && CopyDestroyKind == Normal) {
@@ -7120,7 +7121,7 @@ TypeInfo *SinglePayloadEnumImplStrategy::completeDynamicLayout(
71207121
return registerEnumTypeInfo(new NonFixedEnumTypeInfo(*this, enumTy,
71217122
alignment,
71227123
deinit & payloadTI.isTriviallyDestroyable(ResilienceExpansion::Maximal),
7123-
payloadTI.isBitwiseTakable(ResilienceExpansion::Maximal),
7124+
payloadTI.getBitwiseTakable(ResilienceExpansion::Maximal),
71247125
copyable,
71257126
enumAccessible));
71267127
}
@@ -7155,16 +7156,15 @@ MultiPayloadEnumImplStrategy::completeFixedLayout(TypeConverter &TC,
71557156
? IsNotCopyable : IsCopyable;
71567157
auto isTriviallyDestroyable = theEnum->getValueTypeDestructor()
71577158
? IsNotTriviallyDestroyable : IsTriviallyDestroyable;
7158-
IsBitwiseTakable_t isBT = IsBitwiseTakable;
7159+
IsBitwiseTakable_t isBT = IsBitwiseTakableAndBorrowable;
71597160
PayloadSize = 0;
71607161
for (auto &elt : ElementsWithPayload) {
71617162
auto &fixedPayloadTI = cast<FixedTypeInfo>(*elt.ti);
71627163
if (fixedPayloadTI.getFixedAlignment() > worstAlignment)
71637164
worstAlignment = fixedPayloadTI.getFixedAlignment();
71647165
if (!fixedPayloadTI.isTriviallyDestroyable(ResilienceExpansion::Maximal))
71657166
isTriviallyDestroyable = IsNotTriviallyDestroyable;
7166-
if (!fixedPayloadTI.isBitwiseTakable(ResilienceExpansion::Maximal))
7167-
isBT = IsNotBitwiseTakable;
7167+
isBT &= fixedPayloadTI.getBitwiseTakable(ResilienceExpansion::Maximal);
71687168

71697169
unsigned payloadBytes = fixedPayloadTI.getFixedSize().getValue();
71707170
unsigned payloadBits = fixedPayloadTI.getFixedSize().getValueInBits();
@@ -7324,12 +7324,12 @@ TypeInfo *MultiPayloadEnumImplStrategy::completeDynamicLayout(
73247324
Alignment alignment(1);
73257325
auto td = theEnum->getValueTypeDestructor()
73267326
? IsNotTriviallyDestroyable : IsTriviallyDestroyable;
7327-
auto bt = IsBitwiseTakable;
7327+
auto bt = IsBitwiseTakableAndBorrowable;
73287328
for (auto &element : ElementsWithPayload) {
73297329
auto &payloadTI = *element.ti;
73307330
alignment = std::max(alignment, payloadTI.getBestKnownAlignment());
73317331
td &= payloadTI.isTriviallyDestroyable(ResilienceExpansion::Maximal);
7332-
bt &= payloadTI.isBitwiseTakable(ResilienceExpansion::Maximal);
7332+
bt &= payloadTI.getBitwiseTakable(ResilienceExpansion::Maximal);
73337333
}
73347334

73357335
applyLayoutAttributes(TC.IGM, theEnum, /*fixed*/false, alignment);

‎lib/IRGen/GenExistential.cpp‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -900,11 +900,16 @@ class OpaqueExistentialTypeInfo final :
900900

901901
OpaqueExistentialTypeInfo(ArrayRef<const ProtocolDecl *> protocols,
902902
llvm::Type *ty, Size size,
903+
IsCopyable_t copyable,
903904
SpareBitVector &&spareBits,
904905
Alignment align)
905906
: super(protocols, ty, size,
906907
std::move(spareBits), align,
907-
IsNotTriviallyDestroyable, IsBitwiseTakable, IsCopyable,
908+
IsNotTriviallyDestroyable,
909+
// Copyable existentials are bitwise-takable and borrowable.
910+
// Noncopyable existentials are bitwise-takable only.
911+
copyable ? IsBitwiseTakableAndBorrowable : IsBitwiseTakableOnly,
912+
copyable,
908913
IsFixedSize, IsABIAccessible) {}
909914

910915
public:
@@ -1649,14 +1654,19 @@ static const TypeInfo *createExistentialTypeInfo(IRGenModule &IGM, CanType T) {
16491654
OpaqueExistentialLayout opaque(protosWithWitnessTables.size());
16501655
Alignment align = opaque.getAlignment(IGM);
16511656
Size size = opaque.getSize(IGM);
1657+
IsCopyable_t copyable = T->isNoncopyable() ? IsNotCopyable : IsCopyable;
1658+
16521659
// There are spare bits in the metadata pointer and witness table pointers
16531660
// consistent with a native object reference.
1654-
// TODO: There are spare bits we could theoretically use in the type metadata
1655-
// and witness table pointers, but opaque existentials are currently address-
1661+
// NB: There are spare bits we could theoretically use in the type metadata
1662+
// and witness table pointers, but opaque existentials are address-
16561663
// only, and we can't soundly take advantage of spare bits for in-memory
16571664
// representations.
1665+
// Maybe an ABI break that made opaque existentials loadable could use those
1666+
// bits, though.
16581667
auto spareBits = SpareBitVector::getConstant(size.getValueInBits(), false);
16591668
return OpaqueExistentialTypeInfo::create(protosWithWitnessTables, type, size,
1669+
copyable,
16601670
std::move(spareBits),
16611671
align);
16621672
}

‎lib/IRGen/GenFunc.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -579,7 +579,7 @@ const TypeInfo *TypeConverter::convertBlockStorageType(SILBlockStorageType *T) {
579579

580580
size = captureOffset + fixedCapture->getFixedSize();
581581
pod = fixedCapture->isTriviallyDestroyable(ResilienceExpansion::Maximal);
582-
bt = fixedCapture->isBitwiseTakable(ResilienceExpansion::Maximal);
582+
bt = fixedCapture->getBitwiseTakable(ResilienceExpansion::Maximal);
583583
}
584584

585585
llvm::Type *storageElts[] = {

‎lib/IRGen/GenReflection.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1144,7 +1144,7 @@ class FixedTypeMetadataBuilder : public ReflectionMetadataBuilder {
11441144

11451145
auto alignment = ti->getFixedAlignment().getValue();
11461146
unsigned bitwiseTakable =
1147-
(ti->isBitwiseTakable(ResilienceExpansion::Minimal) == IsBitwiseTakable
1147+
(ti->getBitwiseTakable(ResilienceExpansion::Minimal) >= IsBitwiseTakableOnly
11481148
? 1 : 0);
11491149
B.addInt32(alignment | (bitwiseTakable << 16));
11501150

‎lib/IRGen/GenType.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -236,7 +236,7 @@ llvm::Value *FixedTypeInfo::getIsTriviallyDestroyable(IRGenFunction &IGF, SILTyp
236236
}
237237
llvm::Value *FixedTypeInfo::getIsBitwiseTakable(IRGenFunction &IGF, SILType T) const {
238238
return llvm::ConstantInt::get(IGF.IGM.Int1Ty,
239-
isBitwiseTakable(ResilienceExpansion::Maximal) == IsBitwiseTakable);
239+
getBitwiseTakable(ResilienceExpansion::Maximal) >= IsBitwiseTakableOnly);
240240
}
241241
llvm::Constant *FixedTypeInfo::getStaticStride(IRGenModule &IGM) const {
242242
return IGM.getSize(getFixedStride());

‎lib/IRGen/GenValueWitness.cpp‎

Lines changed: 32 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -849,12 +849,22 @@ ValueWitnessFlags getValueWitnessFlags(const TypeInfo *TI, SILType concreteType,
849849
bool isInline = packing == FixedPacking::OffsetZero;
850850
bool isBitwiseTakable =
851851
fixedTI->isBitwiseTakable(ResilienceExpansion::Maximal);
852+
bool isBitwiseBorrowable =
853+
fixedTI->isBitwiseBorrowable(ResilienceExpansion::Maximal);
852854
assert(isBitwiseTakable || !isInline);
853855
flags = flags.withAlignment(fixedTI->getFixedAlignment().getValue())
854856
.withPOD(fixedTI->isTriviallyDestroyable(ResilienceExpansion::Maximal))
855857
.withCopyable(fixedTI->isCopyable(ResilienceExpansion::Maximal))
856858
.withInlineStorage(isInline)
857-
.withBitwiseTakable(isBitwiseTakable);
859+
.withBitwiseTakable(isBitwiseTakable)
860+
// the IsNotBitwiseBorrowable bit only needs to be set if the
861+
// type is bitwise-takable but not bitwise-borrowable, since
862+
// a type must be bitwise-takable to be bitwise-borrowable.
863+
//
864+
// Swift prior to version 6 didn't have the
865+
// IsNotBitwiseBorrowable bit, so to avoid unnecessary variation
866+
// in metadata output, we only set the bit when needed.
867+
.withBitwiseBorrowable(!isBitwiseTakable || isBitwiseBorrowable);
858868
} else {
859869
flags = flags.withIncomplete(true);
860870
}
@@ -1532,12 +1542,12 @@ llvm::Constant *IRGenModule::emitFixedTypeLayout(CanType t,
15321542
unsigned align = ti.getFixedAlignment().getValue();
15331543

15341544
bool pod = ti.isTriviallyDestroyable(ResilienceExpansion::Maximal);
1535-
bool bt = ti.isBitwiseTakable(ResilienceExpansion::Maximal);
1545+
IsBitwiseTakable_t bt = ti.getBitwiseTakable(ResilienceExpansion::Maximal);
15361546
unsigned numExtraInhabitants = ti.getFixedExtraInhabitantCount(*this);
15371547

15381548
// Try to use common type layouts exported by the runtime.
15391549
llvm::Constant *commonValueWitnessTable = nullptr;
1540-
if (pod && bt && numExtraInhabitants == 0) {
1550+
if (pod && bt == IsBitwiseTakableAndBorrowable && numExtraInhabitants == 0) {
15411551
if (size == 0)
15421552
commonValueWitnessTable =
15431553
getAddrOfValueWitnessTable(Context.TheEmptyTupleType);
@@ -1562,7 +1572,7 @@ llvm::Constant *IRGenModule::emitFixedTypeLayout(CanType t,
15621572

15631573
// Otherwise, see if a layout has been emitted with these characteristics
15641574
// already.
1565-
FixedLayoutKey key{size, numExtraInhabitants, align, pod, bt};
1575+
FixedLayoutKey key{size, numExtraInhabitants, align, pod, unsigned(bt)};
15661576

15671577
auto found = PrivateFixedLayouts.find(key);
15681578
if (found != PrivateFixedLayouts.end())
@@ -1578,16 +1588,29 @@ llvm::Constant *IRGenModule::emitFixedTypeLayout(CanType t,
15781588
addValueWitness(*this, witnesses, witness, packing, t, silTy, ti);
15791589
}
15801590

1591+
auto pod_bt_string = [](bool pod, IsBitwiseTakable_t bt) -> StringRef {
1592+
if (pod) {
1593+
return "_pod";
1594+
}
1595+
switch (bt) {
1596+
case IsNotBitwiseTakable:
1597+
return "";
1598+
case IsBitwiseTakableOnly:
1599+
return "_bt_nbb";
1600+
case IsBitwiseTakableAndBorrowable:
1601+
return "_bt";
1602+
}
1603+
};
1604+
15811605
auto layoutVar
15821606
= witnesses.finishAndCreateGlobal(
15831607
"type_layout_" + llvm::Twine(size)
15841608
+ "_" + llvm::Twine(align)
15851609
+ "_" + llvm::Twine::utohexstr(numExtraInhabitants)
1586-
+ (pod ? "_pod" :
1587-
bt ? "_bt" : ""),
1588-
getPointerAlignment(),
1589-
/*constant*/ true,
1590-
llvm::GlobalValue::PrivateLinkage);
1610+
+ pod_bt_string(pod, bt),
1611+
getPointerAlignment(),
1612+
/*constant*/ true,
1613+
llvm::GlobalValue::PrivateLinkage);
15911614

15921615
// Cast to the standard currency type for type layouts.
15931616
auto layout = llvm::ConstantExpr::getBitCast(layoutVar, Int8PtrPtrTy);

‎lib/IRGen/IRGen.h‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -80,9 +80,15 @@ inline IsLoadable_t &operator&=(IsLoadable_t &l, IsLoadable_t r) {
8080
return (l = (l & r));
8181
}
8282

83-
enum IsBitwiseTakable_t : bool { IsNotBitwiseTakable, IsBitwiseTakable };
83+
enum IsBitwiseTakable_t : uint8_t {
84+
IsNotBitwiseTakable = 0,
85+
// The type is bitwise-takable, but borrows are pinned to memory.
86+
IsBitwiseTakableOnly = 1,
87+
// The type is bitwise-takable and -borrowable.
88+
IsBitwiseTakableAndBorrowable = 3,
89+
};
8490
inline IsBitwiseTakable_t operator&(IsBitwiseTakable_t l, IsBitwiseTakable_t r) {
85-
return IsBitwiseTakable_t(unsigned(l) & unsigned(r));
91+
return IsBitwiseTakable_t(std::min(unsigned(l), unsigned(r)));
8692
}
8793
inline IsBitwiseTakable_t &operator&=(IsBitwiseTakable_t &l, IsBitwiseTakable_t r) {
8894
return (l = (l & r));

‎lib/IRGen/IRGenModule.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1368,7 +1368,7 @@ class IRGenModule {
13681368
unsigned numExtraInhabitants;
13691369
unsigned align: 16;
13701370
unsigned pod: 1;
1371-
unsigned bitwiseTakable: 1;
1371+
unsigned bitwiseTakable: 2;
13721372
};
13731373
friend struct ::llvm::DenseMapInfo<swift::irgen::IRGenModule::FixedLayoutKey>;
13741374
llvm::DenseMap<FixedLayoutKey, llvm::Constant *> PrivateFixedLayouts;

‎lib/IRGen/LoadableTypeInfo.h‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,9 @@ class LoadableTypeInfo : public FixedTypeInfo {
6565
IsABIAccessible_t isABIAccessible,
6666
SpecialTypeInfoKind stik = SpecialTypeInfoKind::Loadable)
6767
: FixedTypeInfo(type, size, spareBits, align, pod,
68-
// All currently implemented loadable types are bitwise-takable.
69-
IsBitwiseTakable,
68+
// All currently implemented loadable types are
69+
// bitwise-takable and -borrowable.
70+
IsBitwiseTakableAndBorrowable,
7071
copy, alwaysFixedSize, isABIAccessible, stik) {
7172
assert(isLoadable());
7273
}
@@ -80,8 +81,9 @@ class LoadableTypeInfo : public FixedTypeInfo {
8081
IsABIAccessible_t isABIAccessible,
8182
SpecialTypeInfoKind stik = SpecialTypeInfoKind::Loadable)
8283
: FixedTypeInfo(type, size, std::move(spareBits), align, pod,
83-
// All currently implemented loadable types are bitwise-takable.
84-
IsBitwiseTakable,
84+
// All currently implemented loadable types are
85+
// bitwise-takable and borrowable.
86+
IsBitwiseTakableAndBorrowable,
8587
copy, alwaysFixedSize, isABIAccessible, stik) {
8688
assert(isLoadable());
8789
}

0 commit comments

Comments
 (0)