Skip to content

Commit d7a5e2b

Browse files
compnerdcompnerd
authored
Merge pull request #42143 from compnerd/malign
runtime: allow over-aligned types in the runtime
2 parents de0e227 + 8a0079d commit d7a5e2b

File tree

10 files changed

+186
-16
lines changed

10 files changed

+186
-16
lines changed

‎include/swift/Runtime/Atomic.h‎

Lines changed: 62 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818
#define SWIFT_RUNTIME_ATOMIC_H
1919

2020
#include "swift/Runtime/Config.h"
21+
#include "swift/Runtime/Heap.h"
22+
2123
#include <assert.h>
2224
#include <atomic>
2325
#if defined(_WIN64)
@@ -44,13 +46,72 @@
4446
namespace swift {
4547
namespace impl {
4648

49+
// FIXME: why can we not use the definitions from Heap.h? It seems that we
50+
// would fail to collapse the structure down in that case and end up with size
51+
// differences.
52+
template <std::size_t Alignment_>
53+
struct requires_aligned_alloc {
54+
#if defined(__cpp_aligned_new)
55+
// If we have C++17 or newer we can use the alignment aware allocation
56+
// implicitly.
57+
static constexpr const bool value = false;
58+
#else
59+
#if defined(__STDCPP_DEFAULT_NEW_ALIGNMENT__)
60+
static constexpr const bool value =
61+
Alignment_ > std::alignment_of<std::max_align_t>::value &&
62+
Alignment_ > __STDCPP_DEFAULT_NEW_ALIGNMENT__;
63+
#else
64+
static constexpr const bool value =
65+
Alignment_ > std::alignment_of<std::max_align_t>::value;
66+
#endif
67+
#endif
68+
};
69+
70+
template <std::size_t Alignment_,
71+
bool = requires_aligned_alloc<Alignment_>::value>
72+
struct aligned_alloc;
73+
74+
template <std::size_t Alignment_>
75+
struct aligned_alloc<Alignment_, false> {};
76+
77+
template <std::size_t Alignment_>
78+
struct aligned_alloc<Alignment_, true> {
79+
[[nodiscard]] void *operator new(std::size_t size) noexcept {
80+
#if defined(_WIN32)
81+
return _aligned_malloc(size, Alignment_);
82+
#else
83+
static_assert(Alignment_ >= sizeof(void *),
84+
"posix_memalign requires minimal alignment of pointer");
85+
void *ptr = nullptr;
86+
(void)posix_memalign(&ptr, Alignment_, size);
87+
return ptr;
88+
#endif
89+
}
90+
91+
void operator delete(void *ptr) noexcept {
92+
#if defined(_WIN32)
93+
_aligned_free(ptr);
94+
#else
95+
free(ptr);
96+
#endif
97+
}
98+
99+
#if defined(_WIN32)
100+
// FIXME: why is this even needed? This is not permitted as per the C++
101+
// standrd new.delete.placement (§17.6.3.4).
102+
[[nodiscard]] void *operator new(std::size_t size, void *where) noexcept {
103+
return ::operator new(size, where);
104+
}
105+
#endif
106+
};
107+
47108
/// The default implementation for swift::atomic<T>, which just wraps
48109
/// std::atomic with minor differences.
49110
///
50111
/// TODO: should we make this use non-atomic operations when the runtime
51112
/// is single-threaded?
52113
template <class Value, size_t Size = sizeof(Value)>
53-
class alignas(Size) atomic_impl {
114+
class alignas(Size) atomic_impl : public aligned_alloc<Size> {
54115
std::atomic<Value> value;
55116
public:
56117
constexpr atomic_impl(Value value) : value(value) {}

‎include/swift/Runtime/AtomicWaitQueue.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
#ifndef SWIFT_RUNTIME_ATOMICWAITQUEUE_H
2020
#define SWIFT_RUNTIME_ATOMICWAITQUEUE_H
2121

22+
#include "swift/Runtime/Heap.h"
2223
#include "swift/Runtime/Mutex.h"
2324
#include <assert.h>
2425

@@ -425,7 +426,8 @@ class AtomicWaitQueue {
425426
template <class... Args>
426427
static Impl *createNewQueue(Args &&...args) {
427428
#if !defined(__cpp_aligned_new)
428-
static_assert(std::alignment_of<Impl>::value <= __STDCPP_DEFAULT_NEW_ALIGNMENT__,
429+
static_assert(!swift::requires_aligned_alloc<std::alignment_of<Impl>::value>::value ||
430+
is_aligned_alloc_aware<Impl>::value,
429431
"type is over-aligned for non-alignment aware operator new");
430432
#endif
431433
auto queue = new Impl(std::forward<Args>(args)...);

‎include/swift/Runtime/Heap.h‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,86 @@
1717
#ifndef SWIFT_RUNTIME_HEAP_H
1818
#define SWIFT_RUNTIME_HEAP_H
1919

20+
#include <cstddef>
21+
#include <cstdint>
22+
#include <cstdlib>
23+
#include <type_traits>
24+
25+
#if defined(_WIN32)
26+
#include <malloc.h>
27+
#endif
28+
29+
namespace swift {
30+
namespace {
31+
// This is C++17 and newer, so we simply re-define it. Since the codebase is
32+
// C++14, asume that DR1558 is accounted for and that unused parameters in alias
33+
// templates are guaranteed to ensure SFINAE and are not ignored.
34+
template <typename ...>
35+
using void_t = void;
36+
37+
template <typename T, typename = void>
38+
struct is_aligned_alloc_aware : std::false_type {};
39+
40+
template <typename T>
41+
struct is_aligned_alloc_aware<T, void_t<decltype(T::operator new(0))>>
42+
: std::true_type {};
43+
}
44+
45+
template <std::size_t Alignment_>
46+
struct requires_aligned_alloc {
47+
#if defined(__cpp_aligned_new)
48+
// If we have C++17 or newer we can use the alignment aware allocation
49+
// implicitly.
50+
static constexpr const bool value = false;
51+
#else
52+
#if defined(__STDCPP_DEFAULT_NEW_ALIGNMENT__)
53+
static constexpr const bool value =
54+
Alignment_ > std::alignment_of<std::max_align_t>::value &&
55+
Alignment_ > __STDCPP_DEFAULT_NEW_ALIGNMENT__;
56+
#else
57+
static constexpr const bool value =
58+
Alignment_ > std::alignment_of<std::max_align_t>::value;
59+
#endif
60+
#endif
61+
};
62+
63+
template <std::size_t Alignment_,
64+
bool = requires_aligned_alloc<Alignment_>::value>
65+
struct aligned_alloc;
66+
67+
template <std::size_t Alignment_>
68+
struct aligned_alloc<Alignment_, false> {};
69+
70+
template <std::size_t Alignment_>
71+
struct aligned_alloc<Alignment_, true> {
72+
[[nodiscard]] void *operator new(std::size_t size) noexcept {
73+
#if defined(_WIN32)
74+
return _aligned_malloc(size, Alignment_);
75+
#else
76+
static_assert(Alignment_ >= sizeof(void *),
77+
"posix_memalign requires minimal alignment of pointer");
78+
void *ptr = nullptr;
79+
(void)posix_memalign(&ptr, Alignment_, size);
80+
return ptr;
81+
#endif
82+
}
83+
84+
void operator delete(void *ptr) noexcept {
85+
#if defined(_WIN32)
86+
_aligned_free(ptr);
87+
#else
88+
free(ptr);
89+
#endif
90+
}
91+
92+
#if defined(_WIN32)
93+
// FIXME: why is this even needed? This is not permitted as per the C++
94+
// standrd new.delete.placement (§17.6.3.4).
95+
[[nodiscard]] void *operator new(std::size_t size, void *where) noexcept {
96+
return ::operator new(size, where);
97+
}
98+
#endif
99+
};
100+
}
101+
20102
#endif // SWIFT_RUNTIME_HEAP_H

‎stdlib/public/Concurrency/Actor.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -616,7 +616,8 @@ class JobRef {
616616
/// achieved through careful arrangement of the storage for this in the
617617
/// DefaultActorImpl. The additional alignment requirements are
618618
/// enforced by static asserts below.
619-
class alignas(sizeof(void *) * 2) ActiveActorStatus {
619+
class alignas(2 * sizeof(void *)) ActiveActorStatus
620+
: public swift::aligned_alloc<2 * sizeof(void *)> {
620621
#if SWIFT_CONCURRENCY_ENABLE_PRIORITY_ESCALATION && SWIFT_POINTER_IS_4_BYTES
621622
uint32_t Flags;
622623
dispatch_lock_t DrainLock;

‎stdlib/public/Concurrency/AsyncLet.cpp‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
#include "swift/ABI/Task.h"
2222
#include "swift/ABI/TaskOptions.h"
2323
#include "swift/Runtime/Mutex.h"
24+
#include "swift/Runtime/Heap.h"
2425
#include "swift/Runtime/HeapObject.h"
2526
#include "llvm/ADT/PointerIntPair.h"
2627
#include "TaskPrivate.h"
@@ -33,7 +34,9 @@
3334
using namespace swift;
3435

3536
namespace {
36-
class alignas(Alignment_AsyncLet) AsyncLetImpl: public ChildTaskStatusRecord {
37+
class alignas(Alignment_AsyncLet) AsyncLetImpl
38+
: public swift::aligned_alloc<Alignment_AsyncLet>,
39+
public ChildTaskStatusRecord {
3740
public:
3841
// This is where we could define a Status or other types important for async-let
3942

‎stdlib/public/Concurrency/TaskPrivate.h‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
#include "swift/Runtime/DispatchShims.h"
2727
#include "swift/Runtime/Error.h"
2828
#include "swift/Runtime/Exclusivity.h"
29+
#include "swift/Runtime/Heap.h"
2930
#include "swift/Runtime/HeapObject.h"
3031
#include <atomic>
3132

@@ -524,6 +525,12 @@ class alignas(2 * sizeof(void*)) ActiveTaskStatus {
524525
#endif
525526
static_assert(sizeof(ActiveTaskStatus) == ACTIVE_TASK_STATUS_SIZE,
526527
"ActiveTaskStatus is of incorrect size");
528+
#if !defined(_WIN64)
529+
static_assert(sizeof(swift::atomic<ActiveTaskStatus>) == sizeof(std::atomic<ActiveTaskStatus>),
530+
"swift::atomic pads std::atomic, memory aliasing invariants violated");
531+
#endif
532+
static_assert(sizeof(swift::atomic<ActiveTaskStatus>) == sizeof(ActiveTaskStatus),
533+
"swift::atomic pads ActiveTaskStatus, memory aliasing invariants violated");
527534

528535
/// The size of an allocator slab. We want the full allocation to fit into a
529536
/// 1024-byte malloc quantum. We subtract off the slab header size, plus a
@@ -545,7 +552,7 @@ struct AsyncTask::PrivateStorage {
545552

546553
/// Storage for the ActiveTaskStatus. See doc for ActiveTaskStatus for size
547554
/// and alignment requirements.
548-
alignas(2 * sizeof(void *)) char StatusStorage[sizeof(ActiveTaskStatus)];
555+
alignas(alignof(ActiveTaskStatus)) char StatusStorage[sizeof(ActiveTaskStatus)];
549556

550557
/// The allocator for the task stack.
551558
/// Currently 2 words + 8 bytes.

‎stdlib/public/SwiftShims/RefCount.h‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ typedef InlineRefCountsPlaceholder InlineRefCounts;
3737
#include "swift/Runtime/Atomic.h"
3838
#include "swift/Runtime/Config.h"
3939
#include "swift/Runtime/Debug.h"
40+
#include "swift/Runtime/Heap.h"
4041

4142

4243
/*
@@ -625,8 +626,9 @@ class RefCountBitsT {
625626

626627
typedef RefCountBitsT<RefCountIsInline> InlineRefCountBits;
627628

628-
class alignas(sizeof(void*) * 2) SideTableRefCountBits : public RefCountBitsT<RefCountNotInline>
629-
{
629+
class alignas(2 * sizeof(void*)) SideTableRefCountBits
630+
: public swift::aligned_alloc<2 * sizeof(void *)>,
631+
public RefCountBitsT<RefCountNotInline> {
630632
uint32_t weakBits;
631633

632634
public:

‎stdlib/public/runtime/KnownMetadata.cpp‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
//===----------------------------------------------------------------------===//
1616

1717
#include "swift/Runtime/Metadata.h"
18+
#include "swift/Runtime/Heap.h"
1819
#include "swift/Runtime/HeapObject.h"
1920
#include "swift/Runtime/Numeric.h"
2021
#include "MetadataImpl.h"
@@ -40,19 +41,19 @@ OpaqueValue *swift::swift_copyPOD(OpaqueValue *dest, OpaqueValue *src,
4041
namespace {
4142
// A type sized and aligned the way Swift wants Int128 (and Float80/Float128)
4243
// to be sized and aligned.
43-
struct alignas(16) int128_like {
44+
struct alignas(16) int128_like : swift::aligned_alloc<16> {
4445
char data[16];
4546
};
4647

4748
static_assert(MaximumAlignment == 16, "max alignment was hardcoded");
48-
struct alignas(16) int256_like {
49+
struct alignas(16) int256_like : swift::aligned_alloc<16> {
4950
char data[32];
5051
};
51-
struct alignas(16) int512_like {
52+
struct alignas(16) int512_like : swift::aligned_alloc<16> {
5253
char data[64];
5354
};
5455

55-
struct alignas(16) float80_like {
56+
struct alignas(16) float80_like : swift::aligned_alloc<16> {
5657
char data[10];
5758
};
5859
} // end anonymous namespace
@@ -89,7 +90,8 @@ namespace ctypes {
8990
// Types that are defined in the _Concurrency library
9091

9192
// Default actor storage type.
92-
struct alignas(2 * alignof(void*)) BD {
93+
struct alignas(2 * alignof(void*)) BD
94+
: swift::aligned_alloc<2 * alignof(void*)> {
9395
void *storage[NumWords_DefaultActor];
9496
};
9597

‎stdlib/public/runtime/Metadata.cpp‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
#include "swift/Runtime/Casting.h"
2525
#include "swift/Runtime/EnvironmentVariables.h"
2626
#include "swift/Runtime/ExistentialContainer.h"
27+
#include "swift/Runtime/Heap.h"
2728
#include "swift/Runtime/HeapObject.h"
2829
#include "swift/Runtime/Mutex.h"
2930
#include "swift/Runtime/Once.h"
@@ -6153,10 +6154,16 @@ void swift::blockOnMetadataDependency(MetadataDependency root,
61536154
#if !SWIFT_STDLIB_PASSTHROUGH_METADATA_ALLOCATOR
61546155

61556156
namespace {
6156-
struct alignas(sizeof(uintptr_t) * 2) PoolRange {
6157+
struct alignas(2 * sizeof(uintptr_t)) PoolRange
6158+
: swift::aligned_alloc<2 * sizeof(uintptr_t)> {
61576159
static constexpr uintptr_t PageSize = 16 * 1024;
61586160
static constexpr uintptr_t MaxPoolAllocationSize = PageSize / 2;
61596161

6162+
constexpr PoolRange(char *Begin, size_t Remaining)
6163+
: Begin(Begin), Remaining(Remaining) {}
6164+
6165+
PoolRange() : Begin(nullptr), Remaining(0) {}
6166+
61606167
/// The start of the allocation.
61616168
char *Begin;
61626169

‎stdlib/public/runtime/MetadataCache.h‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
#include "llvm/ADT/Hashing.h"
1616
#include "llvm/ADT/STLExtras.h"
1717
#include "swift/Runtime/Concurrent.h"
18+
#include "swift/Runtime/Heap.h"
1819
#include "swift/Runtime/Metadata.h"
1920
#include "swift/Runtime/Mutex.h"
2021
#include "swift/Runtime/AtomicWaitQueue.h"
@@ -623,8 +624,9 @@ const size_t PrivateMetadataTrackingAlignment = 16;
623624

624625
/// The wait queue object that we create for metadata that are
625626
/// being actively initialized right now.
626-
struct alignas(PrivateMetadataTrackingAlignment) MetadataWaitQueue :
627-
public AtomicWaitQueue<MetadataWaitQueue, ConcurrencyControl::LockType> {
627+
struct alignas(PrivateMetadataTrackingAlignment) MetadataWaitQueue
628+
: swift::aligned_alloc<PrivateMetadataTrackingAlignment>,
629+
public AtomicWaitQueue<MetadataWaitQueue, ConcurrencyControl::LockType> {
628630

629631
/// A pointer to the completion context being used to complete this
630632
/// metadata. This is only actually filled in if:
@@ -683,7 +685,8 @@ struct alignas(PrivateMetadataTrackingAlignment) MetadataWaitQueue :
683685

684686
/// A record used to store information about an attempt to
685687
/// complete a metadata when there's no active worker thread.
686-
struct alignas(PrivateMetadataTrackingAlignment) SuspendedMetadataCompletion {
688+
struct alignas(PrivateMetadataTrackingAlignment) SuspendedMetadataCompletion
689+
: swift::aligned_alloc<PrivateMetadataTrackingAlignment> {
687690
MetadataDependency BlockingDependency;
688691
std::unique_ptr<PrivateMetadataCompletionContext> PersistentContext;
689692

0 commit comments

Comments
 (0)