[Concurrency] Make task IDs 64-bit.

The 32-bit identifier in Job is locked down at this point, so we expand the ID by storing the top 32 bits separately inside AsyncTask::PrivateStorage.

rdar://85167409

Author: mikeash

include/swift/ABI/Task.h

@@ -250,7 +250,7 @@ class AsyncTask : public Job {
     : Job(flags, run, metadata, captureCurrentVoucher),
       ResumeContext(initialContext) {
     assert(flags.isAsyncTask());
-    Id = getNextTaskId();
+    setTaskId();
   }
 
   /// Create a task with "immortal" reference counts.
@@ -265,11 +265,14 @@ class AsyncTask : public Job {
     : Job(flags, run, metadata, immortal, captureCurrentVoucher),
       ResumeContext(initialContext) {
     assert(flags.isAsyncTask());
-    Id = getNextTaskId();
+    setTaskId();
   }
 
   ~AsyncTask();
 
+  /// Set the task's ID field to the next task ID.
+  void setTaskId();
+
   /// Given that we've already fully established the job context
   /// in the current thread, start running this task.  To establish
   /// the job context correctly, call swift_job_run or
@@ -566,14 +569,6 @@ class AsyncTask : public Job {
     return reinterpret_cast<AsyncTask *&>(
         SchedulerPrivate[NextWaitingTaskIndex]);
   }
-
-  /// Get the next non-zero Task ID.
-  uint32_t getNextTaskId() {
-    static std::atomic<uint32_t> Id(1);
-    uint32_t Next = Id.fetch_add(1, std::memory_order_relaxed);
-    if (Next == 0) Next = Id.fetch_add(1, std::memory_order_relaxed);
-    return Next;
-  }
 };
 
 // The compiler will eventually assume these.

stdlib/public/Concurrency/Task.cpp

@@ -243,6 +243,19 @@ AsyncTask::~AsyncTask() {
   Private.destroy();
 }
 
+void AsyncTask::setTaskId() {
+  static std::atomic<uint64_t> NextId(1);
+
+  // We want the 32-bit Job::Id to be non-zero, so loop if we happen upon zero.
+  uint64_t Fetched;
+  do {
+    Fetched = NextId.fetch_add(1, std::memory_order_relaxed);
+    Id = Fetched & 0xffffffff;
+  } while (Id == 0);
+
+  _private().Id = (Fetched >> 32) & 0xffffffff;
+}
+
 SWIFT_CC(swift)
 static void destroyTask(SWIFT_CONTEXT HeapObject *obj) {
   auto task = static_cast<AsyncTask*>(obj);

stdlib/public/Concurrency/TaskPrivate.h

@@ -294,6 +294,9 @@ struct AsyncTask::PrivateStorage {
   /// state.
   uintptr_t ExclusivityAccessSet[2] = {0, 0};
 
+  /// The top 32 bits of the task ID. The bottom 32 bits are in Job::Id.
+  uint32_t Id;
+
   PrivateStorage(JobFlags flags)
       : Status(ActiveTaskStatus(flags)), Local(TaskLocal::Storage()) {}