Merge pull request #63192 from etcwilde/ewilde/fix-backdeploy-compat56-crash

Fixes for the compat56 library

Author: etcwilde

lib/ASTGen/CMakeLists.txt

@@ -34,6 +34,19 @@ if (SWIFT_SWIFT_PARSER)
     Sources/ASTGen/Types.swift
   )
 
+  # The compat56 library is not available in current toolchains. The stage-0
+  # compiler will build fine since the builder compiler is not aware of the 56
+  # compat library, but the stage-1 and subsequent stage compilers will fail as
+  # the stage-0 compiler is aware and will attempt to include the appropriate
+  # compatibility library. We should turn this back on once we are building the
+  # compiler correctly.
+  # Note: This is safe at the moment because the 5.6 compat library only
+  #       contains concurrency runtime fixes, and the compiler frontend does not
+  #       use concurrency at the moment.
+  target_compile_options(swiftASTGen PRIVATE
+    $<$<COMPILE_LANGUAGE:Swift>:-runtime-compatibility-version>
+    $<$<COMPILE_LANGUAGE:Swift>:none>)
+
   # Set the appropriate target triple.
   if(SWIFT_HOST_VARIANT_SDK IN_LIST SWIFT_DARWIN_PLATFORMS)
     set(DEPLOYMENT_VERSION "${SWIFT_SDK_${SWIFT_HOST_VARIANT_SDK}_DEPLOYMENT_VERSION}")

stdlib/public/BackDeployConcurrency/Actor.cpp

@@ -1228,7 +1228,7 @@ void DefaultActorImpl::giveUpThread(RunningJobInfo runner) {
     }
 
 #define LOG_STATE_TRANSITION                                                   \
-  SWIFT_TASK_DEBUG_LOG("actor %p transitioned from %zx to %zx (%s)\n", this,   \
+  SWIFT_TASK_DEBUG_LOG("actor %p transitioned from %zx to %zx (%s)", this,     \
                        oldState.Flags.getOpaqueValue(),                        \
                        newState.Flags.getOpaqueValue(), __FUNCTION__)
     LOG_STATE_TRANSITION;

stdlib/public/BackDeployConcurrency/TaskGroup.cpp

@@ -479,7 +479,7 @@ static void swift_taskGroup_initializeImpl(TaskGroup *group, const Metadata *T)
 // ==== add / attachChild ------------------------------------------------------
 
 void TaskGroup::addChildTask(AsyncTask *child) {
-  SWIFT_TASK_DEBUG_LOG("attach child task = %p to group = %p", child, group);
+  SWIFT_TASK_DEBUG_LOG("attach child task = %p to group = %p", child, this);
 
   // The counterpart of this (detachChild) is performed by the group itself,
   // when it offers the completed (child) task's value to a waiting task -

stdlib/public/BackDeployConcurrency/TaskPrivate.h

@@ -47,7 +47,7 @@ namespace swift {
 #if 0
 #define SWIFT_TASK_DEBUG_LOG(fmt, ...)                                         \
   fprintf(stderr, "[%lu] [%s:%d](%s) " fmt "\n",                               \
-          (unsigned long)Thread::current()::platformThreadId(),                \
+          (unsigned long)Thread::current().platformThreadId(),                 \
           __FILE__, __LINE__, __FUNCTION__,                                    \
           __VA_ARGS__)
 #else

stdlib/toolchain/Compatibility56/CMakeLists.txt

@@ -2,6 +2,10 @@ set(library_name "swiftCompatibility56")
 
 include_directories("include/" "${SWIFT_STDLIB_SOURCE_DIR}")
 
+set(CMAKE_C_VISIBILITY_PRESET hidden)
+set(CMAKE_CXX_VISIBILITY_PRESET hidden)
+set(CMAKE_VISIBILITY_INLINES_HIDDEN YES)
+
 add_compile_definitions(SWIFT_COMPATIBILITY56)
 add_swift_target_library("${library_name}" STATIC
   Overrides.cpp

stdlib/toolchain/Compatibility56/Concurrency/Task.cpp

@@ -15,6 +15,7 @@ FutureFragment::Status AsyncTask::waitFuture(AsyncTask *waitingTask,
                                              TaskContinuationFunction *resumeFn,
                                              AsyncContext *callerContext,
                                              OpaqueValue *result) {
+  SWIFT_TASK_DEBUG_LOG("compat 56 task task %p", this);
   using Status = FutureFragment::Status;
   using WaitQueueItem = FutureFragment::WaitQueueItem;
 

stdlib/toolchain/Compatibility56/Overrides.cpp

@@ -39,13 +39,15 @@ struct ConcurrencyOverrideSection {
 
 #undef OVERRIDE
 
+__attribute__((visibility("hidden")))
 ConcurrencyOverrideSection Swift56ConcurrencyOverrides
 __attribute__((used, section("__DATA,__s_async_hook"))) = {
   .version = 0,
   .task_future_wait = swift56override_swift_task_future_wait,
   .task_future_wait_throwing = swift56override_swift_task_future_wait_throwing,
 };
 
+__attribute__((visibility("hidden")))
 RuntimeOverrideSection Swift56RuntimeOverrides
 __attribute__((used, section("__DATA,__swift56_hooks"))) = {
   .version = 0,

stdlib/toolchain/Compatibility56/include/Concurrency/Error.h

@@ -23,6 +23,7 @@
 
 namespace swift {
 
+__attribute__((visibility("hidden")))
 SWIFT_NORETURN void swift_Concurrency_fatalError(uint32_t flags, const char *format, ...);
 
 } // namespace swift

stdlib/toolchain/Compatibility56/include/Concurrency/Task.h

@@ -17,6 +17,7 @@
 #ifndef SWIFT_ABI_TASK_BACKDEPLOY56_H
 #define SWIFT_ABI_TASK_BACKDEPLOY56_H
 
+#include "Concurrency/TaskLocal.h"
 #include "Executor.h"
 #include "swift/ABI/HeapObject.h"
 #include "swift/ABI/Metadata.h"
@@ -26,7 +27,7 @@
 #include "VoucherShims.h"
 #include "swift/Basic/STLExtras.h"
 #include <bitset>
-#include <queue>
+#include <queue> // TODO: remove and replace with our own mpsc
 
 namespace swift {
 class AsyncTask;
@@ -275,13 +276,6 @@ class AsyncTask : public Job {
   void setTaskId();
   uint64_t getTaskId();
 
-  /// Get the task's resume function, for logging purposes only. This will
-  /// attempt to see through the various adapters that are sometimes used, and
-  /// failing that will return ResumeTask. The returned function pointer may
-  /// have a different signature than ResumeTask, and it's only for identifying
-  /// code associated with the task.
-  const void *getResumeFunctionForLogging();
-
   /// Given that we've already fully established the job context
   /// in the current thread, start running this task.  To establish
   /// the job context correctly, call swift_job_run or
@@ -313,34 +307,44 @@ class AsyncTask : public Job {
   ///
   /// Generally this should be done immediately after updating
   /// ActiveTask.
+  __attribute__((visibility("hidden")))
   void flagAsRunning();
+  __attribute__((visibility("hidden")))
   void flagAsRunning_slow();
 
-  /// Flag that this task is now suspended.
+  /// Flag that this task is now suspended.  This can update the
+  /// priority stored in the job flags if the priority hsa been
+  /// escalated.  Generally this should be done immediately after
+  /// clearing ActiveTask and immediately before enqueuing the task
+  /// somewhere.  TODO: record where the task is enqueued if
+  /// possible.
+  __attribute__((visibility("hidden")))
   void flagAsSuspended();
+  __attribute__((visibility("hidden")))
   void flagAsSuspended_slow();
 
-  /// Flag that the task is to be enqueued on the provided executor and actually
-  /// enqueue it
-  void flagAsAndEnqueueOnExecutor(ExecutorRef newExecutor);
-
   /// Flag that this task is now completed. This normally does not do anything
   /// but can be used to locally insert logging.
+  __attribute__((visibility("hidden")))
   void flagAsCompleted();
 
   /// Check whether this task has been cancelled.
   /// Checking this is, of course, inherently race-prone on its own.
+  __attribute__((visibility("hidden")))
   bool isCancelled() const;
 
   // ==== Task Local Values ----------------------------------------------------
 
+  __attribute__((visibility("hidden")))
   void localValuePush(const HeapObject *key,
                       /* +1 */ OpaqueValue *value,
                       const Metadata *valueType);
 
+  __attribute__((visibility("hidden")))
   OpaqueValue *localValueGet(const HeapObject *key);
 
   /// Returns true if storage has still more bindings.
+  __attribute__((visibility("hidden")))
   bool localValuePop();
 
   // ==== Child Fragment -------------------------------------------------------
@@ -566,8 +570,9 @@ class AsyncTask : public Job {
   /// \c Executing, then \c waitingTask has been added to the
   /// wait queue and will be scheduled when the future completes. Otherwise,
   /// the future has completed and can be queried.
-  /// The waiting task's async context will be initialized with the parameters if
+  /// The waiting task's async context will be intialized with the parameters if
   /// the current's task state is executing.
+  __attribute__((visibility("hidden")))
   FutureFragment::Status waitFuture(AsyncTask *waitingTask,
                                     AsyncContext *waitingTaskContext,
                                     TaskContinuationFunction *resumeFn,
@@ -614,6 +619,65 @@ inline void Job::runInFullyEstablishedContext() {
 
 // ==== ------------------------------------------------------------------------
 
+/// The Swift5.6 AsyncContextKind for the AsyncContext.
+/// Note that these were removed in Swift5.7
+/// (aca744b21165a20655502b563a6fa54c2c83efdf).
+/// Kinds of async context.
+enum class AsyncContextKind {
+  /// An ordinary asynchronous function.
+  Ordinary         = 0,
+
+  /// A context which can yield to its caller.
+  Yielding         = 1,
+
+  /// A continuation context.
+  Continuation     = 2,
+
+  // Other kinds are reserved for interesting special
+  // intermediate contexts.
+
+  // Kinds >= 192 are private to the implementation.
+  First_Reserved = 192
+};
+
+
+/// The Swift5.6 AsyncContextFlags for the AsyncContext.
+/// Note that these were removed in Swift5.7
+/// (aca744b21165a20655502b563a6fa54c2c83efdf).
+/// Flags for async contexts.
+class AsyncContextFlags : public FlagSet<uint32_t> {
+public:
+  enum {
+    Kind                = 0,
+    Kind_width          = 8,
+
+    CanThrow            = 8,
+
+    // Kind-specific flags should grow down from 31.
+
+    Continuation_IsExecutorSwitchForced = 31,
+  };
+
+  explicit AsyncContextFlags(uint32_t bits) : FlagSet(bits) {}
+  constexpr AsyncContextFlags() {}
+  AsyncContextFlags(AsyncContextKind kind) {
+    setKind(kind);
+  }
+
+  /// The kind of context this represents.
+  FLAGSET_DEFINE_FIELD_ACCESSORS(Kind, Kind_width, AsyncContextKind,
+                                 getKind, setKind)
+
+  /// Whether this context is permitted to throw.
+  FLAGSET_DEFINE_FLAG_ACCESSORS(CanThrow, canThrow, setCanThrow)
+
+  /// See AsyncContinuationFlags::isExecutorSwitchForced.
+  FLAGSET_DEFINE_FLAG_ACCESSORS(Continuation_IsExecutorSwitchForced,
+                                continuation_isExecutorSwitchForced,
+                                continuation_setIsExecutorSwitchForced)
+};
+
+
 /// An asynchronous context within a task.  Generally contexts are
 /// allocated using the task-local stack alloc/dealloc operations, but
 /// there's no guarantee of that, and the ABI is designed to permit
@@ -634,9 +698,19 @@ class alignas(MaximumAlignment) AsyncContext {
   TaskContinuationFunction * __ptrauth_swift_async_context_resume
     ResumeParent;
 
-  AsyncContext(TaskContinuationFunction *resumeParent,
+  /// Flags describing this context.
+  ///
+  /// Note that this field is only 32 bits; any alignment padding
+  /// following this on 64-bit platforms can be freely used by the
+  /// function.  If the function is a yielding function, that padding
+  /// is of course interrupted by the YieldToParent field.
+  AsyncContextFlags Flags;
+
+  AsyncContext(AsyncContextFlags flags,
+               TaskContinuationFunction *resumeParent,
                AsyncContext *parent)
-    : Parent(parent), ResumeParent(resumeParent) {}
+    : Parent(parent), ResumeParent(resumeParent),
+      Flags(flags) {}
 
   AsyncContext(const AsyncContext &) = delete;
   AsyncContext &operator=(const AsyncContext &) = delete;
@@ -660,66 +734,48 @@ class YieldingAsyncContext : public AsyncContext {
   TaskContinuationFunction * __ptrauth_swift_async_context_yield
     YieldToParent;
 
-  YieldingAsyncContext(TaskContinuationFunction *resumeParent,
+  YieldingAsyncContext(AsyncContextFlags flags,
+                       TaskContinuationFunction *resumeParent,
                        TaskContinuationFunction *yieldToParent,
                        AsyncContext *parent)
-    : AsyncContext(resumeParent, parent),
+    : AsyncContext(flags, resumeParent, parent),
       YieldToParent(yieldToParent) {}
+
+  static bool classof(const AsyncContext *context) {
+    return context->Flags.getKind() == AsyncContextKind::Yielding;
+  }
 };
 
 /// An async context that can be resumed as a continuation.
 class ContinuationAsyncContext : public AsyncContext {
 public:
-  class FlagsType : public FlagSet<size_t> {
-  public:
-    enum {
-      CanThrow = 0,
-      IsExecutorSwitchForced = 1,
-    };
-
-    explicit FlagsType(size_t bits) : FlagSet(bits) {}
-    constexpr FlagsType() {}
-
-    /// Whether this is a throwing continuation.
-    FLAGSET_DEFINE_FLAG_ACCESSORS(CanThrow,
-                                  canThrow,
-                                  setCanThrow)
-
-    /// See AsyncContinuationFlags::isExecutorSwitchForced().
-    FLAGSET_DEFINE_FLAG_ACCESSORS(IsExecutorSwitchForced,
-                                  isExecutorSwitchForced,
-                                  setIsExecutorSwitchForced)
-  };
-
-  /// Flags for the continuation.  Not public ABI.
-  FlagsType Flags;
-
   /// An atomic object used to ensure that a continuation is not
   /// scheduled immediately during a resume if it hasn't yet been
-  /// awaited by the function which set it up.  Not public ABI.
+  /// awaited by the function which set it up.
   std::atomic<ContinuationStatus> AwaitSynchronization;
 
   /// The error result value of the continuation.
   /// This should be null-initialized when setting up the continuation.
   /// Throwing resumers must overwrite this with a non-null value.
-  /// Public ABI.
   SwiftError *ErrorResult;
 
   /// A pointer to the normal result value of the continuation.
   /// Normal resumers must initialize this before resuming.
-  /// Public ABI.
   OpaqueValue *NormalResult;
 
   /// The executor that should be resumed to.
-  /// Public ABI.
   ExecutorRef ResumeToExecutor;
 
   void setErrorResult(SwiftError *error) {
     ErrorResult = error;
   }
 
   bool isExecutorSwitchForced() const {
-    return Flags.isExecutorSwitchForced();
+    return Flags.continuation_isExecutorSwitchForced();
+  }
+
+  static bool classof(const AsyncContext *context) {
+    return context->Flags.getKind() == AsyncContextKind::Continuation;
   }
 };
 

stdlib/toolchain/Compatibility56/include/Concurrency/TaskPrivate.h

@@ -29,7 +29,25 @@
 
 namespace swift {
 
+#if 0
+using ThreadID = decltype(pthread_self());
+
+inline ThreadID _swift_get_thread_id() {
+#if defined(_WIN32)
+  return GetCurrentThreadId();
+#else
+  return pthread_self();
+#endif
+}
+
+#define SWIFT_TASK_DEBUG_LOG(fmt, ...)                                         \
+  fprintf(stderr, "[%lu] [%s:%d](%s) " fmt "\n",                               \
+          (unsigned long)_swift_get_thread_id(),                               \
+          __FILE__, __LINE__, __FUNCTION__,                                    \
+          __VA_ARGS__)
+#else
 #define SWIFT_TASK_DEBUG_LOG(fmt ...) (void)0
+#endif
 
 /// Allocate task-local memory on behalf of a specific task,
 /// not necessarily the current one.  Generally this should only be
@@ -91,18 +109,23 @@ class TaskFutureWaitAsyncContext : public AsyncContext {
 
 /// Adopt the voucher stored in `task`. This removes the voucher from the task
 /// and adopts it on the current thread.
+__attribute__((visibility("hidden")))
 void adoptTaskVoucher(AsyncTask *task);
 
 /// Restore the voucher for `task`. This un-adopts the current thread's voucher
 /// and stores it back into the task again.
+__attribute__((visibility("hidden")))
 void restoreTaskVoucher(AsyncTask *task);
 
 /// release() establishes a happens-before relation with a preceding acquire()
 /// on the same address.
+__attribute__((visibility("hidden")))
 void _swift_tsan_acquire(void *addr);
+__attribute__((visibility("hidden")))
 void _swift_tsan_release(void *addr);
 
 /// Clear the active task reference for the current thread.
+__attribute__((visibility("hidden")))
 AsyncTask *_swift_task_clearCurrent();
 
 /// The current state of a task's status records.