[Concurrency] Eliminate StatusRecordLockRecord.

Move to a recursive lock inline in the Task. This avoids the need to allocate a lock record and simplifies the code somewhat.

Change Task's OpaquePrivateStorage to compute its size at build time based on the sizes of its components, rather than having it be a fixed size. It appears that the fixed size was intended to be part of the ABI, but that didn't happen and we're free to change this size. We need to expand it slightly when using pthread_mutex as the recursive lock, as pthread_mutex is pretty big. Other recursive locks allow it to shrink slightly.

We don't have a recursive mutex in our Threading support code, so add a RecursiveMutex type.

rdar://113898653

Author: mikeash

include/swift/ABI/MetadataValues.h

@@ -62,9 +62,6 @@ enum {
   /// in a non-default distributed actor.
   NumWords_NonDefaultDistributedActor = 12,
 
-  /// The number of words in a task.
-  NumWords_AsyncTask = 24,
-
   /// The number of words in a task group.
   NumWords_TaskGroup = 32,
 

include/swift/ABI/Task.h

@@ -298,7 +298,18 @@ class AsyncTask : public Job {
 
   /// Private storage for the use of the runtime.
   struct alignas(2 * alignof(void*)) OpaquePrivateStorage {
-    void *Storage[14];
+#if SWIFT_CONCURRENCY_ENABLE_PRIORITY_ESCALATION && SWIFT_POINTER_IS_4_BYTES
+    static constexpr size_t ActiveTaskStatusSize = 4 * sizeof(void *);
+#else
+    static constexpr size_t ActiveTaskStatusSize = 4 * sizeof(void *);
+#endif
+
+    // Private storage is currently 6 pointers, 16 bytes of non-pointer data,
+    // the ActiveTaskStatus, and a RecursiveMutex.
+    static constexpr size_t PrivateStorageSize =
+        6 * sizeof(void *) + 16 + ActiveTaskStatusSize + sizeof(RecursiveMutex);
+
+    void *Storage[PrivateStorageSize];
 
     /// Initialize this storage during the creation of a task.
     void initialize(JobPriority basePri);
@@ -755,8 +766,6 @@ class AsyncTask : public Job {
 };
 
 // The compiler will eventually assume these.
-static_assert(sizeof(AsyncTask) == NumWords_AsyncTask * sizeof(void*),
-              "AsyncTask size is wrong");
 static_assert(alignof(AsyncTask) == 2 * alignof(void*),
               "AsyncTask alignment is wrong");
 #pragma clang diagnostic push

include/swift/Threading/Impl/C11.h

@@ -147,6 +147,25 @@ inline void lazy_mutex_unsafe_unlock(lazy_mutex_handle &handle) {
   (void)::mtx_unlock(&handle.mutex);
 }
 
+// .. Recursive mutex support .................................................
+
+using recursive_mutex_handle = ::mtx_t;
+
+inline void recursive_mutex_init(recursive_mutex_handle &handle,
+                                 bool checked = false) {
+  SWIFT_C11THREADS_CHECK(::mtx_init(&handle, ::mtx_recursive));
+}
+inline void recursive_mutex_destroy(recursive_mutex_handle &handle) {
+  ::mtx_destroy(&handle);
+}
+
+inline void recursive_mutex_lock(recursive_mutex_handle &handle) {
+  SWIFT_C11THREADS_CHECK(::mtx_lock(&handle));
+}
+inline void recursive_mutex_unlock(recursive_mutex_handle &handle) {
+  SWIFT_C11THREADS_CHECK(::mtx_unlock(&handle));
+}
+
 // .. ConditionVariable support ..............................................
 
 struct cond_handle {

include/swift/Threading/Impl/Darwin.h

@@ -129,6 +129,42 @@ inline void lazy_mutex_unsafe_unlock(lazy_mutex_handle &handle) {
   ::os_unfair_lock_unlock(&handle);
 }
 
+// .. Recursive mutex support .................................................
+
+// The os_unfair_recursive_lock interface is stable, but not in the SDK. Bring
+// our own definitions for what we need.
+
+#define OS_UNFAIR_RECURSIVE_LOCK_INIT                                          \
+  (os_unfair_recursive_lock{OS_UNFAIR_LOCK_INIT, 0})
+
+typedef struct os_unfair_recursive_lock_s {
+  os_unfair_lock ourl_lock;
+  uint32_t ourl_count;
+} os_unfair_recursive_lock, *os_unfair_recursive_lock_t;
+
+using recursive_mutex_handle = os_unfair_recursive_lock;
+
+extern "C" void
+os_unfair_recursive_lock_lock_with_options(os_unfair_recursive_lock_t lock,
+                                           uint32_t options);
+
+extern "C" void
+os_unfair_recursive_lock_unlock(os_unfair_recursive_lock_t lock);
+
+inline void recursive_mutex_init(recursive_mutex_handle &handle,
+                                 bool checked = false) {
+  handle = OS_UNFAIR_RECURSIVE_LOCK_INIT;
+}
+inline void recursive_mutex_destroy(recursive_mutex_handle &handle) {}
+
+inline void recursive_mutex_lock(recursive_mutex_handle &handle) {
+  os_unfair_recursive_lock_lock_with_options(&handle, 0);
+}
+
+inline void recursive_mutex_unlock(recursive_mutex_handle &handle) {
+  os_unfair_recursive_lock_unlock(&handle);
+}
+
 // .. ConditionVariable support ..............................................
 
 struct cond_handle {

include/swift/Threading/Impl/Linux.h

@@ -134,6 +134,29 @@ inline void lazy_mutex_unsafe_unlock(lazy_mutex_handle &handle) {
   (void)::pthread_mutex_unlock(&handle);
 }
 
+// .. Recursive mutex support ................................................
+
+using recursive_mutex_handle = ::pthread_mutex_t;
+
+inline void recursive_mutex_init(mutex_handle &handle, bool checked = false) {
+  ::pthread_mutexattr_t attr;
+  SWIFT_LINUXTHREADS_CHECK(::pthread_mutexattr_init(&attr));
+  SWIFT_LINUXTHREADS_CHECK(
+      ::pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE));
+  SWIFT_LINUXTHREADS_CHECK(::pthread_mutex_init(&handle, &attr));
+  SWIFT_LINUXTHREADS_CHECK(::pthread_mutexattr_destroy(&attr));
+}
+inline void recursive_mutex_destroy(mutex_handle &handle) {
+  SWIFT_LINUXTHREADS_CHECK(::pthread_mutex_destroy(&handle));
+}
+
+inline void recursive_mutex_lock(mutex_handle &handle) {
+  SWIFT_LINUXTHREADS_CHECK(::pthread_mutex_lock(&handle));
+}
+inline void recursive_mutex_unlock(mutex_handle &handle) {
+  SWIFT_LINUXTHREADS_CHECK(::pthread_mutex_unlock(&handle));
+}
+
 // .. ConditionVariable support ..............................................
 
 struct cond_handle {

include/swift/Threading/Impl/Nothreads.h

@@ -60,6 +60,16 @@ inline bool lazy_mutex_try_lock(lazy_mutex_handle &handle) { return true; }
 inline void lazy_mutex_unsafe_lock(lazy_mutex_handle &handle) {}
 inline void lazy_mutex_unsafe_unlock(lazy_mutex_handle &handle) {}
 
+// .. Recursive mutex support .................................................
+
+using recursive_mutex_handle = unsigned;
+
+inline void recursive_mutex_init(recursive_mutex_handle &handle,
+                                 bool checked = false) {}
+inline void recursive_mutex_destroy(recursive_mutex_handle &handle) {}
+inline void recursive_mutex_lock(recursive_mutex_handle &handle) {}
+inline void recursive_mutex_unlock(recursive_mutex_handle &handle) {}
+
 // .. ConditionVariable support ..............................................
 
 using cond_handle = unsigned;

include/swift/Threading/Impl/Pthreads.h

@@ -130,6 +130,29 @@ inline void lazy_mutex_unsafe_unlock(lazy_mutex_handle &handle) {
   (void)::pthread_mutex_unlock(&handle);
 }
 
+// .. Recursive mutex support ................................................
+
+using recursive_mutex_handle = ::pthread_mutex_t;
+
+inline void recursive_mutex_init(mutex_handle &handle, bool checked = false) {
+  ::pthread_mutexattr_t attr;
+  SWIFT_PTHREADS_CHECK(::pthread_mutexattr_init(&attr));
+  SWIFT_PTHREADS_CHECK(
+      ::pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE));
+  SWIFT_PTHREADS_CHECK(::pthread_mutex_init(&handle, &attr));
+  SWIFT_PTHREADS_CHECK(::pthread_mutexattr_destroy(&attr));
+}
+inline void recursive_mutex_destroy(mutex_handle &handle) {
+  SWIFT_PTHREADS_CHECK(::pthread_mutex_destroy(&handle));
+}
+
+inline void recursive_mutex_lock(mutex_handle &handle) {
+  SWIFT_PTHREADS_CHECK(::pthread_mutex_lock(&handle));
+}
+inline void recursive_mutex_unlock(mutex_handle &handle) {
+  SWIFT_PTHREADS_CHECK(::pthread_mutex_unlock(&handle));
+}
+
 // .. ConditionVariable support ..............................................
 
 struct cond_handle {

include/swift/Threading/Mutex.h

@@ -241,6 +241,48 @@ class IndirectMutex {
 using SmallMutex =
     std::conditional_t<sizeof(Mutex) <= sizeof(void *), Mutex, IndirectMutex>;
 
+/// A recursive variant of Mutex.
+class RecursiveMutex {
+
+  RecursiveMutex(const Mutex &) = delete;
+  RecursiveMutex &operator=(const Mutex &) = delete;
+  RecursiveMutex(Mutex &&) = delete;
+  RecursiveMutex &operator=(Mutex &&) = delete;
+
+public:
+  /// Constructs a non-recursive mutex.
+  ///
+  /// If `checked` is true the mutex will attempt to check for misuse and
+  /// fatalError when detected. If `checked` is false (the default) the
+  /// mutex will make little to no effort to check for misuse (more efficient).
+  explicit RecursiveMutex(bool checked = false) {
+    threading_impl::recursive_mutex_init(Handle, checked);
+  }
+  ~RecursiveMutex() { threading_impl::recursive_mutex_destroy(Handle); }
+
+  /// The lock() method has the following properties:
+  /// - Behaves as an atomic operation.
+  /// - Blocks the calling thread until exclusive ownership of the mutex
+  ///   can be obtained.
+  /// - Prior m.unlock() operations on the same mutex synchronize-with
+  ///   this lock operation.
+  /// - Does not throw exceptions but will halt on error (fatalError).
+  void lock() { threading_impl::recursive_mutex_lock(Handle); }
+
+  /// The unlock() method has the following properties:
+  /// - Behaves as an atomic operation.
+  /// - Releases the calling thread's ownership of the mutex and
+  ///   synchronizes-with the subsequent successful lock operations on
+  ///   the same object.
+  /// - The behavior is undefined if the calling thread does not own
+  ///   the mutex.
+  /// - Does not throw exceptions but will halt on error (fatalError).
+  void unlock() { threading_impl::recursive_mutex_unlock(Handle); }
+
+protected:
+  threading_impl::recursive_mutex_handle Handle;
+};
+
 } // namespace swift
 
 #endif // SWIFT_THREADING_MUTEX_H

stdlib/public/Concurrency/TaskPrivate.h

@@ -29,6 +29,7 @@
 #include "swift/Runtime/Error.h"
 #include "swift/Runtime/Exclusivity.h"
 #include "swift/Runtime/HeapObject.h"
+#include "swift/Threading/Mutex.h"
 #include "swift/Threading/Thread.h"
 #include "swift/Threading/ThreadSanitizer.h"
 #include <atomic>
@@ -598,27 +599,25 @@ class alignas(2 * sizeof(void*)) ActiveTaskStatus {
 #endif
   }
 
-  /// Is there a lock on the linked list of status records?
+  /// Does some thread hold the status record lock?
   bool isStatusRecordLocked() const { return Flags & IsStatusRecordLocked; }
-  ActiveTaskStatus withLockingRecord(TaskStatusRecord *lockRecord) const {
+  ActiveTaskStatus withStatusRecordLocked() const {
     assert(!isStatusRecordLocked());
-    assert(lockRecord->Parent == Record);
 #if SWIFT_CONCURRENCY_ENABLE_PRIORITY_ESCALATION
-    return ActiveTaskStatus(lockRecord, Flags | IsStatusRecordLocked, ExecutionLock);
+    return ActiveTaskStatus(Record, Flags | IsStatusRecordLocked,
+                            ExecutionLock);
 #else
-    return ActiveTaskStatus(lockRecord, Flags | IsStatusRecordLocked);
+    return ActiveTaskStatus(Record, Flags | IsStatusRecordLocked);
 #endif
   }
-  ActiveTaskStatus withoutLockingRecord() const {
+  ActiveTaskStatus withoutStatusRecordLocked() const {
     assert(isStatusRecordLocked());
-    assert(Record->getKind() == TaskStatusRecordKind::Private_RecordLock);
 
-    // Remove the lock record, and put the next one as the head
-    auto newRecord = Record->Parent;
 #if SWIFT_CONCURRENCY_ENABLE_PRIORITY_ESCALATION
-    return ActiveTaskStatus(newRecord, Flags & ~IsStatusRecordLocked, ExecutionLock);
+    return ActiveTaskStatus(Record, Flags & ~IsStatusRecordLocked,
+                            ExecutionLock);
 #else
-    return ActiveTaskStatus(newRecord, Flags & ~IsStatusRecordLocked);
+    return ActiveTaskStatus(Record, Flags & ~IsStatusRecordLocked);
 #endif
   }
 
@@ -779,6 +778,9 @@ struct AsyncTask::PrivateStorage {
   /// async task stack when it is needed.
   TaskDependencyStatusRecord *dependencyRecord = nullptr;
 
+  // The lock used to protect more complicated operations on the task status.
+  RecursiveMutex statusLock;
+
   // Always create an async task with max priority in ActiveTaskStatus = base
   // priority. It will be updated later if needed.
   PrivateStorage(JobPriority basePri)