Merge pull request #41222 from compnerd/spinning

Runtime: change atomic implementation for `AllocationPool`

Author: compnerd

include/swift/Runtime/Atomic.h

@@ -72,6 +72,13 @@ class alignas(Size) atomic_impl {
     return value.compare_exchange_weak(oldValue, newValue, successOrder,
                                        failureOrder);
   }
+
+  bool compare_exchange_strong(Value &oldValue, Value newValue,
+                               std::memory_order successOrder,
+                               std::memory_order failureOrder) {
+    return value.compare_exchange_strong(oldValue, newValue, successOrder,
+                                         failureOrder);
+  }
 };
 
 #if defined(_WIN64)
@@ -128,11 +135,14 @@ class alignas(2 * sizeof(void*)) atomic_impl<Value, 2 * sizeof(void*)> {
   bool compare_exchange_weak(Value &oldValue, Value newValue,
                              std::memory_order successOrder,
                              std::memory_order failureOrder) {
-    assert(failureOrder == std::memory_order_relaxed ||
-           failureOrder == std::memory_order_acquire ||
-           failureOrder == std::memory_order_consume);
-    assert(successOrder == std::memory_order_relaxed ||
-           successOrder == std::memory_order_release);
+    // We do not have weak CAS intrinsics, fallback to strong
+    return compare_exchange_strong(oldValue, newValue, successOrder,
+                                   failureOrder);
+  }
+
+  bool compare_exchange_strong(Value &oldValue, Value newValue,
+                               std::memory_order successOrder,
+                               std::memory_order failureOrder) {
 #if SWIFT_HAS_MSVC_ARM_ATOMICS
     if (successOrder == std::memory_order_relaxed &&
         failureOrder != std::memory_order_acquire) {
@@ -180,7 +190,7 @@ class alignas(2 * sizeof(void*)) atomic_impl<Value, 2 * sizeof(void*)> {
 template <class T>
 class atomic : public impl::atomic_impl<T> {
 public:
-  atomic(T value) : impl::atomic_impl<T>(value) {}
+  constexpr atomic(T value) : impl::atomic_impl<T>(value) {}
 };
 
 } // end namespace swift

stdlib/public/runtime/Metadata.cpp

@@ -5918,7 +5918,7 @@ namespace {
 alignas(void *) static struct {
   char Pool[InitialPoolSize];
 } InitialAllocationPool;
-static std::atomic<PoolRange>
+static swift::atomic<PoolRange>
 AllocationPool{PoolRange{InitialAllocationPool.Pool,
                          sizeof(InitialAllocationPool.Pool)}};
 
@@ -6071,10 +6071,9 @@ void *MetadataAllocator::Allocate(size_t size, size_t alignment) {
     }
 
     // Swap in the new state.
-    if (std::atomic_compare_exchange_weak_explicit(&AllocationPool,
-                                                   &curState, newState,
-                                              std::memory_order_relaxed,
-                                              std::memory_order_relaxed)) {
+    if (AllocationPool.compare_exchange_weak(curState, newState,
+                                             std::memory_order_relaxed,
+                                             std::memory_order_relaxed)) {
       // If that succeeded, we've successfully allocated.
       __msan_allocated_memory(allocation, sizeWithHeader);
       __asan_unpoison_memory_region(allocation, sizeWithHeader);
@@ -6129,11 +6128,10 @@ void MetadataAllocator::Deallocate(const void *allocation, size_t size,
   // don't bother trying again; we'll just leak the allocation.
   PoolRange newState = { reinterpret_cast<char*>(const_cast<void*>(allocation)),
                          curState.Remaining + size };
-  (void)
-    std::atomic_compare_exchange_strong_explicit(&AllocationPool,
-                                                 &curState, newState,
-                                                 std::memory_order_relaxed,
-                                                 std::memory_order_relaxed);
+
+  AllocationPool.compare_exchange_weak(curState, newState,
+                                       std::memory_order_relaxed,
+                                       std::memory_order_relaxed);
 }
 
 #endif