[Concurrency] Refactor executors so they are in their own object module.

C++ executor implementations were `#include`ed into `GlobalExecutor.cpp`,
which makes it difficult to replace the global executor when using the
Embedded Concurrency library.  Refactor things so that they build into
separate objects, which means replacing them is just a matter of writing
the relevant functions yourself.

rdar://135380149

Author: al45tair

include/swift/Runtime/Concurrency.h

@@ -720,6 +720,13 @@ void swift_task_enqueueGlobal(Job *job);
 SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift)
 void swift_task_checkIsolated(SerialExecutorRef executor);
 
+/// Invoke a Swift executor's `checkIsolated` implementation; returns
+/// `true` if it invoked the Swift implementation, `false` otherwise.
+/// Executors will want to call this from their `swift_task_checkIsolatedImpl`
+/// implementation.
+SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift)
+bool swift_task_invokeSwiftCheckIsolated(SerialExecutorRef executor);
+
 /// A count in nanoseconds.
 using JobDelay = unsigned long long;
 
@@ -760,71 +767,29 @@ void swift_task_enqueueOnDispatchQueue(Job *job, HeapObject *queue);
 
 #endif
 
-/// A hook to take over global enqueuing.
-typedef SWIFT_CC(swift) void (*swift_task_enqueueGlobal_original)(Job *job);
-SWIFT_EXPORT_FROM(swift_Concurrency)
-SWIFT_CC(swift) void (*swift_task_enqueueGlobal_hook)(
-    Job *job, swift_task_enqueueGlobal_original original);
-
-/// A hook to take over global enqueuing with delay.
-typedef SWIFT_CC(swift) void (*swift_task_enqueueGlobalWithDelay_original)(
-    unsigned long long delay, Job *job);
-SWIFT_EXPORT_FROM(swift_Concurrency)
-SWIFT_CC(swift) void (*swift_task_enqueueGlobalWithDelay_hook)(
-    unsigned long long delay, Job *job,
-    swift_task_enqueueGlobalWithDelay_original original);
-
-typedef SWIFT_CC(swift) void (*swift_task_enqueueGlobalWithDeadline_original)(
-    long long sec,
-    long long nsec,
-    long long tsec,
-    long long tnsec,
-    int clock, Job *job);
-SWIFT_EXPORT_FROM(swift_Concurrency)
-SWIFT_CC(swift) void (*swift_task_enqueueGlobalWithDeadline_hook)(
-    long long sec,
-    long long nsec,
-    long long tsec,
-    long long tnsec,
-    int clock, Job *job,
-    swift_task_enqueueGlobalWithDeadline_original original);
-
-typedef SWIFT_CC(swift) void (*swift_task_checkIsolated_original)(SerialExecutorRef executor);
-SWIFT_EXPORT_FROM(swift_Concurrency)
-SWIFT_CC(swift) void (*swift_task_checkIsolated_hook)(
-    SerialExecutorRef executor, swift_task_checkIsolated_original original);
-
-
-typedef SWIFT_CC(swift) bool (*swift_task_isOnExecutor_original)(
-    HeapObject *executor,
-    const Metadata *selfType,
-    const SerialExecutorWitnessTable *wtable);
-SWIFT_EXPORT_FROM(swift_Concurrency)
-SWIFT_CC(swift) bool (*swift_task_isOnExecutor_hook)(
-    HeapObject *executor,
-    const Metadata *selfType,
-    const SerialExecutorWitnessTable *wtable,
-    swift_task_isOnExecutor_original original);
-
-/// A hook to take over main executor enqueueing.
-typedef SWIFT_CC(swift) void (*swift_task_enqueueMainExecutor_original)(
-    Job *job);
-SWIFT_EXPORT_FROM(swift_Concurrency)
-SWIFT_CC(swift) void (*swift_task_enqueueMainExecutor_hook)(
-    Job *job, swift_task_enqueueMainExecutor_original original);
-
-/// A hook to override the entrypoint to the main runloop used to drive the
-/// concurrency runtime and drain the main queue. This function must not return.
-/// Note: If the hook is wrapping the original function and the `compatOverride`
-///       is passed in, the `original` function pointer must be passed into the
-///       compatibility override function as the original function.
-typedef SWIFT_CC(swift) void (*swift_task_asyncMainDrainQueue_original)();
-typedef SWIFT_CC(swift) void (*swift_task_asyncMainDrainQueue_override)(
-    swift_task_asyncMainDrainQueue_original original);
-SWIFT_EXPORT_FROM(swift_Concurrency)
-SWIFT_CC(swift) void (*swift_task_asyncMainDrainQueue_hook)(
-    swift_task_asyncMainDrainQueue_original original,
-    swift_task_asyncMainDrainQueue_override compatOverride);
+// Declare all the hooks
+#define SWIFT_CONCURRENCY_HOOK(returnType, name, ...)                   \
+  typedef SWIFT_CC(swift) returnType (*name##_original)(__VA_ARGS__);   \
+  typedef SWIFT_CC(swift) returnType                                    \
+    (*name##_hook_t)(__VA_ARGS__, name##_original original);            \
+  SWIFT_EXPORT_FROM(swift_Concurrency) name##_hook_t name##_hook
+
+#define SWIFT_CONCURRENCY_HOOK0(returnType, name)                       \
+  typedef SWIFT_CC(swift) returnType (*name##_original)();              \
+  typedef SWIFT_CC(swift) returnType                                    \
+    (*name##_hook_t)(name##_original original);                         \
+  SWIFT_EXPORT_FROM(swift_Concurrency) name##_hook_t name##_hook
+
+#define SWIFT_CONCURRENCY_HOOK_OVERRIDE0(returnType, name)              \
+  typedef SWIFT_CC(swift) returnType (*name##_original)();              \
+  typedef SWIFT_CC(swift) returnType (*name##_override)(                \
+    name##_original original);                                          \
+  typedef SWIFT_CC(swift) returnType                                    \
+    (*name##_hook_t)(name##_original original,                          \
+                     name##_override compatOverride);                   \
+  SWIFT_EXPORT_FROM(swift_Concurrency) name##_hook_t name##_hook
+
+#include "ConcurrencyHooks.def"
 
 /// Initialize the runtime storage for a default actor.
 SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift)
@@ -951,6 +916,10 @@ SerialExecutorRef swift_task_getCurrentExecutor(void);
 SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift)
 SerialExecutorRef swift_task_getMainExecutor(void);
 
+/// Test if an executor is the main executor.
+SWIFT_EXPORT_FROM(swift_Concurrency) SWIFT_CC(swift)
+bool swift_task_isMainExecutor(SerialExecutorRef executor);
+
 /// Return the preferred task executor of the current task,
 /// or ``TaskExecutorRef::undefined()`` if no preference.
 ///

include/swift/Runtime/ConcurrencyHooks.def

@@ -0,0 +1,75 @@
+//===--- Concurrency.h - Runtime interface for concurrency ------*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2024 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+//
+// Hooks for concurrency
+//
+//===----------------------------------------------------------------------===//
+
+// To use this file, define the following macros:
+//
+//   SWIFT_CONCURRENCY_HOOK(returnType, name, ...)
+//   SWIFT_CONCURRENCY_HOOK0(returnType, name)
+//   SWIFT_CONCURRENCY_HOOK_OVERRIDE0(returnType, name)
+//
+// Then include the file somewhere.
+
+#ifndef SWIFT_CONCURRENCY_HOOK
+#define SWIFT_CONCURRENCY_HOOK(returnType, name, ...)
+#endif
+
+#ifndef SWIFT_CONCURRENCY_HOOK0
+#define SWIFT_CONCURRENCY_HOOK0(returnType, name)
+#endif
+
+#ifndef SWIFT_CONCURRENCY_HOOK_OVERRIDE0
+#define SWIFT_CONCURRENCY_HOOK_OVERRIDE0(returnType, name)
+#endif
+
+// .............................................................................
+
+SWIFT_CONCURRENCY_HOOK(void, swift_task_enqueueGlobal, Job *job);
+
+SWIFT_CONCURRENCY_HOOK(void, swift_task_enqueueGlobalWithDelay,
+                 unsigned long long delay, Job *job);
+
+SWIFT_CONCURRENCY_HOOK(void, swift_task_enqueueGlobalWithDeadline,
+    long long sec,
+    long long nsec,
+    long long tsec,
+    long long tnsec,
+    int clock, Job *job);
+
+SWIFT_CONCURRENCY_HOOK(void, swift_task_checkIsolated, SerialExecutorRef executor);
+
+SWIFT_CONCURRENCY_HOOK(bool, swift_task_isOnExecutor,
+    HeapObject *executor,
+    const Metadata *selfType,
+    const SerialExecutorWitnessTable *wtable);
+
+SWIFT_CONCURRENCY_HOOK(void, swift_task_enqueueMainExecutor, Job *job);
+
+SWIFT_CONCURRENCY_HOOK0(SerialExecutorRef, swift_task_getMainExecutor);
+
+SWIFT_CONCURRENCY_HOOK(bool, swift_task_isMainExecutor, SerialExecutorRef);
+
+/// A hook to override the entrypoint to the main runloop used to drive the
+/// concurrency runtime and drain the main queue. This function must not return.
+/// Note: If the hook is wrapping the original function and the `compatOverride`
+///       is passed in, the `original` function pointer must be passed into the
+///       compatibility override function as the original function.
+SWIFT_CONCURRENCY_HOOK_OVERRIDE0(void, swift_task_asyncMainDrainQueue);
+
+// .............................................................................
+
+#undef SWIFT_CONCURRENCY_HOOK
+#undef SWIFT_CONCURRENCY_HOOK0
+#undef SWIFT_CONCURRENCY_HOOK_OVERRIDE0

stdlib/public/Concurrency/Actor.cpp

@@ -43,6 +43,7 @@
 #define SWIFT_OBJC_INTEROP 1
 #endif
 #include "llvm/ADT/PointerIntPair.h"
+
 #include "TaskPrivate.h"
 #include "VoucherSupport.h"
 

stdlib/public/Concurrency/CMakeLists.txt

@@ -43,10 +43,6 @@ else()
   message(FATAL_ERROR "Invalid value for SWIFT_CONCURRENCY_GLOBAL_EXECUTOR (\"${SWIFT_CONCURRENCY_GLOBAL_EXECUTOR}\").")
 endif()
 
-if(NOT SWIFT_CONCURRENCY_USES_DISPATCH)
-
-endif()
-
 if(NOT swift_concurrency_async_fp_mode)
   set(swift_concurrency_async_fp_mode "always")
 endif()
@@ -78,6 +74,7 @@ set(SWIFT_RUNTIME_CONCURRENCY_C_SOURCES
   AsyncLet.cpp
   Clock.cpp
   GlobalExecutor.cpp
+  Hooks.cpp
   EmbeddedSupport.cpp
   Error.cpp
   Setup.cpp
@@ -92,6 +89,27 @@ set(SWIFT_RUNTIME_CONCURRENCY_C_SOURCES
   linker-support/magic-symbols-for-install-name.c
 )
 
+if("${SWIFT_CONCURRENCY_GLOBAL_EXECUTOR}" STREQUAL "dispatch")
+  set(SWIFT_RUNTIME_CONCURRENCY_EXECUTOR_SOURCES
+    DispatchGlobalExecutor.cpp
+  )
+elseif("${SWIFT_CONCURRENCY_GLOBAL_EXECUTOR}" STREQUAL "singlethreaded")
+  set(SWIFT_RUNTIME_CONCURRENCY_EXECUTOR_SOURCES
+    CooperativeGlobalExecutor.cpp
+  )
+elseif("${SWIFT_CONCURRENCY_GLOBAL_EXECUTOR}" STREQUAL "hooked" OR
+    "${SWIFT_CONCURRENCY_GLOBAL_EXECUTOR}" STREQUAL "none")
+  set(SWIFT_RUNTIME_CONCURRENCY_EXECUTOR_SOURCES
+    NonDispatchGlobalExecutor.cpp
+  )
+endif()
+
+set(LLVM_OPTIONAL_SOURCES
+  CooperativeGlobalExecutor.cpp
+  DispatchGlobalExecutor.cpp
+  NonDispatchGlobalExecutor.cpp
+)
+
 set(SWIFT_RUNTIME_CONCURRENCY_SWIFT_SOURCES
   Actor.swift
   AsyncLet.swift
@@ -158,6 +176,7 @@ set(SWIFT_RUNTIME_CONCURRENCY_SWIFT_SOURCES
 
 add_swift_target_library(swift_Concurrency ${SWIFT_STDLIB_LIBRARY_BUILD_TYPES} IS_STDLIB
   ${SWIFT_RUNTIME_CONCURRENCY_C_SOURCES}
+  ${SWIFT_RUNTIME_CONCURRENCY_EXECUTOR_SOURCES}
   ${SWIFT_RUNTIME_CONCURRENCY_SWIFT_SOURCES}
 
   SWIFT_MODULE_DEPENDS_ANDROID Android
@@ -230,7 +249,7 @@ if(SWIFT_SHOULD_BUILD_EMBEDDED_STDLIB AND SWIFT_SHOULD_BUILD_EMBEDDED_CONCURRENC
       set(extra_c_compile_flags -D__MACH__ -D__APPLE__ -ffreestanding)
       set(extra_swift_compile_flags -Xcc -D__MACH__ -Xcc -D__APPLE__ -Xcc -ffreestanding)
     endif()
-    
+
     set(SWIFT_SDK_embedded_THREADING_PACKAGE none)
     set(SWIFT_SDK_embedded_ARCH_${mod}_MODULE "${mod}")
     set(SWIFT_SDK_embedded_LIB_SUBDIR "embedded")
@@ -243,6 +262,8 @@ if(SWIFT_SHOULD_BUILD_EMBEDDED_STDLIB AND SWIFT_SHOULD_BUILD_EMBEDDED_CONCURRENC
       IS_STDLIB IS_FRAGILE
 
       ${SWIFT_RUNTIME_CONCURRENCY_C_SOURCES}
+      CooperativeGlobalExecutor.cpp
+
       # TODO: Only a subset of Swift Concurrency .swift sources, for now.
       Actor.swift
       AsyncLet.swift

stdlib/public/Concurrency/CooperativeGlobalExecutor.inc renamed to stdlib/public/Concurrency/CooperativeGlobalExecutor.cpp

@@ -23,6 +23,8 @@
 ///
 ///===------------------------------------------------------------------===///
 
+#include "swift/Runtime/Concurrency.h"
+
 #include <chrono>
 #ifndef SWIFT_THREADING_NONE
 # include <thread>
@@ -37,6 +39,10 @@
 # define NSEC_PER_SEC 1000000000ull
 #endif
 
+#include "ExecutorHooks.h"
+
+using namespace swift;
+
 namespace {
 
 struct JobQueueTraits {
@@ -104,14 +110,14 @@ static Job *DelayedJobQueue = nullptr;
 
 /// Insert a job into the cooperative global queue.
 SWIFT_CC(swift)
-static void swift_task_enqueueGlobalImpl(Job *job) {
+void swift::swift_task_enqueueGlobalImpl(Job *job) {
   assert(job && "no job provided");
   JobQueue.enqueue(job);
 }
 
 /// Enqueues a task on the main executor.
 SWIFT_CC(swift)
-static void swift_task_enqueueMainExecutorImpl(Job *job) {
+void swift::swift_task_enqueueMainExecutorImpl(Job *job) {
   // The cooperative executor does not distinguish between the main
   // queue and the global queue.
   swift_task_enqueueGlobalImpl(job);
@@ -136,15 +142,13 @@ static void insertDelayedJob(Job *newJob, JobDeadline deadline) {
 }
 
 SWIFT_CC(swift)
-static void swift_task_checkIsolatedImpl(SerialExecutorRef executor) {
-  _task_serialExecutor_checkIsolated(
-      executor.getIdentity(), swift_getObjectType(executor.getIdentity()),
-      executor.getSerialExecutorWitnessTable());
+void swift::swift_task_checkIsolatedImpl(SerialExecutorRef executor) {
+  swift_task_invokeSwiftCheckIsolated(executor);
 }
 
 /// Insert a job into the cooperative global queue with a delay.
 SWIFT_CC(swift)
-static void swift_task_enqueueGlobalWithDelayImpl(JobDelay delay,
+void swift::swift_task_enqueueGlobalWithDelayImpl(JobDelay delay,
                                                   Job *newJob) {
   assert(newJob && "no job provided");
 
@@ -157,7 +161,7 @@ static void swift_task_enqueueGlobalWithDelayImpl(JobDelay delay,
 }
 
 SWIFT_CC(swift)
-static void swift_task_enqueueGlobalWithDeadlineImpl(long long sec,
+void swift::swift_task_enqueueGlobalWithDeadlineImpl(long long sec,
                                                      long long nsec,
                                                      long long tsec,
                                                      long long tnsec,
@@ -252,3 +256,13 @@ swift_task_donateThreadToGlobalExecutorUntil(bool (*condition)(void *),
     swift_job_run(job, SerialExecutorRef::generic());
   }
 }
+
+SWIFT_CC(swift)
+SerialExecutorRef swift::swift_task_getMainExecutorImpl() {
+  return SerialExecutorRef::generic();
+}
+
+SWIFT_CC(swift)
+bool swift::swift_task_isMainExecutorImpl(SerialExecutorRef executor) {
+  return executor.isGeneric();
+}