[Runtime] Fix strong and unowned refcount overflow on 32-bit.

Fix overflow detection on unowned refcounts so that we create a side table when incrementing from 126. Implement strong refcount overflow to the side table.

The unowned refcount is never supposed to be 127, because that (sometimes) represents the immortal refcount. We attempt to detect that by checking newValue == Offsets::UnownedRefCountMask, but the mask is shifted so that condition is never true. We managed to hit the side table case when incrementing from 127, because it looks like the immortal case. But that broke when we fixed immortal side table initialization in b41079a8f54ae2d61c68cdda46c74232084af020. With that change, we now create an immortal side table when overflowing the unowned refcount, then try to increment the unowned refcount in that immortal side table, which traps.

rdar://123788910

Author: mikeash

stdlib/public/SwiftShims/swift/shims/RefCount.h

@@ -397,7 +397,7 @@ class RefCountBitsT {
   bool isOverflowingUnownedRefCount(uint32_t oldValue, uint32_t inc) const {
     auto newValue = getUnownedRefCount();
     return newValue != oldValue + inc ||
-      newValue == Offsets::UnownedRefCountMask;
+      newValue == Offsets::UnownedRefCountMask >> Offsets::UnownedRefCountShift;
   }
 
   SWIFT_ALWAYS_INLINE
@@ -582,6 +582,12 @@ class RefCountBitsT {
     }
 #endif
 
+    // If we're decrementing by more than 1, then this underflow might end up
+    // subtracting away an existing 1 in UseSlowRC. Check that separately. This
+    // check should be constant folded away for the swift_release case.
+    if (dec > 1 && getUseSlowRC())
+      return false;
+
     // This deliberately underflows by borrowing from the UseSlowRC field.
     bits -= BitsType(dec) << Offsets::StrongExtraRefCountShift;
     return (SignedBitsType(bits) >= 0);
@@ -1007,8 +1013,8 @@ class RefCounts {
   bool doDecrementSlow(RefCountBits oldbits, uint32_t dec) {
     RefCountBits newbits;
 
-    // constant propagation will remove this in swift_release, it should only
-    // be present in swift_release_n
+    // Constant propagation will remove this in swift_release, it should only
+    // be present in swift_release_n.
     if (dec != 1 && oldbits.isImmortal(true)) {
       return false;
     }
@@ -1306,6 +1312,8 @@ class RefCounts {
     return refCounts.load(std::memory_order_relaxed).getBitsValue();
   }
 
+  void dump() const;
+
   private:
   HeapObject *getHeapObject();
 
@@ -1505,6 +1513,10 @@ class HeapObjectSideTableEntry {
     immutableCOWBuffer = immutable;
   }
 #endif
+
+  void dumpRefCounts() {
+    refCounts.dump();
+  }
 };
 
 

stdlib/public/runtime/RefCount.cpp

@@ -14,8 +14,56 @@
 
 namespace swift {
 
-template <typename RefCountBits>
-HeapObject *RefCounts<RefCountBits>::incrementSlow(RefCountBits oldbits,
+// Return an object's side table, allocating it if necessary.
+// Returns null if the object is deiniting.
+// SideTableRefCountBits specialization intentionally does not exist.
+template <>
+HeapObjectSideTableEntry* RefCounts<InlineRefCountBits>::allocateSideTable(bool failIfDeiniting)
+{
+  auto oldbits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME);
+
+  // Preflight failures before allocating a new side table.
+  if (oldbits.hasSideTable()) {
+    // Already have a side table. Return it.
+    return oldbits.getSideTable();
+  }
+  else if (failIfDeiniting && oldbits.getIsDeiniting()) {
+    // Already past the start of deinit. Do nothing.
+    return nullptr;
+  }
+
+  // Preflight passed. Allocate a side table.
+
+  // FIXME: custom side table allocator
+  auto side = swift_cxx_newObject<HeapObjectSideTableEntry>(getHeapObject());
+
+  auto newbits = InlineRefCountBits(side);
+
+  do {
+    if (oldbits.hasSideTable()) {
+      // Already have a side table. Return it and delete ours.
+      // Read before delete to streamline barriers.
+      auto result = oldbits.getSideTable();
+      swift_cxx_deleteObject(side);
+      return result;
+    }
+    else if (failIfDeiniting && oldbits.getIsDeiniting()) {
+      // Already past the start of deinit. Do nothing.
+      return nullptr;
+    }
+
+    side->initRefCounts(oldbits);
+
+  } while (! refCounts.compare_exchange_weak(oldbits, newbits,
+                                             std::memory_order_release,
+                                             std::memory_order_relaxed));
+
+  return side;
+}
+
+
+template <>
+HeapObject *RefCounts<InlineRefCountBits>::incrementSlow(InlineRefCountBits oldbits,
                                                    uint32_t n) {
   if (oldbits.isImmortal(false)) {
     return getHeapObject();
@@ -25,21 +73,28 @@ HeapObject *RefCounts<RefCountBits>::incrementSlow(RefCountBits oldbits,
     auto side = oldbits.getSideTable();
     side->incrementStrong(n);
   }
+  else {
+    // Overflow into a new side table.
+    auto side = allocateSideTable(false);
+    side->incrementStrong(n);
+  }
+  return getHeapObject();
+}
+template <>
+HeapObject *RefCounts<SideTableRefCountBits>::incrementSlow(SideTableRefCountBits oldbits,
+                                                uint32_t n) {
+  if (oldbits.isImmortal(false)) {
+    return getHeapObject();
+  }
   else {
     // Retain count overflow.
     swift::swift_abortRetainOverflow();
   }
   return getHeapObject();
 }
-template HeapObject *
-RefCounts<InlineRefCountBits>::incrementSlow(InlineRefCountBits oldbits,
-                                             uint32_t n);
-template HeapObject *
-RefCounts<SideTableRefCountBits>::incrementSlow(SideTableRefCountBits oldbits,
-                                                uint32_t n);
 
-template <typename RefCountBits>
-void RefCounts<RefCountBits>::incrementNonAtomicSlow(RefCountBits oldbits,
+template <>
+void RefCounts<InlineRefCountBits>::incrementNonAtomicSlow(InlineRefCountBits oldbits,
                                                      uint32_t n) {
   if (oldbits.isImmortal(false)) {
     return;
@@ -48,12 +103,20 @@ void RefCounts<RefCountBits>::incrementNonAtomicSlow(RefCountBits oldbits,
     // Out-of-line slow path.
     auto side = oldbits.getSideTable();
     side->incrementStrong(n);  // FIXME: can there be a nonatomic impl?
+  } else {
+    // Overflow into a new side table.
+    auto side = allocateSideTable(false);
+    side->incrementStrong(n);  // FIXME: can there be a nonatomic impl?
+  }
+}
+template <>
+void RefCounts<SideTableRefCountBits>::incrementNonAtomicSlow(SideTableRefCountBits oldbits, uint32_t n) {
+  if (oldbits.isImmortal(false)) {
+    return;
   } else {
     swift::swift_abortRetainOverflow();
   }
 }
-template void RefCounts<InlineRefCountBits>::incrementNonAtomicSlow(InlineRefCountBits oldbits, uint32_t n);
-template void RefCounts<SideTableRefCountBits>::incrementNonAtomicSlow(SideTableRefCountBits oldbits, uint32_t n);
 
 template <typename RefCountBits>
 bool RefCounts<RefCountBits>::tryIncrementSlow(RefCountBits oldbits) {
@@ -81,53 +144,6 @@ bool RefCounts<RefCountBits>::tryIncrementNonAtomicSlow(RefCountBits oldbits) {
 template bool RefCounts<InlineRefCountBits>::tryIncrementNonAtomicSlow(InlineRefCountBits oldbits);
 template bool RefCounts<SideTableRefCountBits>::tryIncrementNonAtomicSlow(SideTableRefCountBits oldbits);
 
-// Return an object's side table, allocating it if necessary.
-// Returns null if the object is deiniting.
-// SideTableRefCountBits specialization intentionally does not exist.
-template <>
-HeapObjectSideTableEntry* RefCounts<InlineRefCountBits>::allocateSideTable(bool failIfDeiniting)
-{
-  auto oldbits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME);
-  
-  // Preflight failures before allocating a new side table.
-  if (oldbits.hasSideTable()) {
-    // Already have a side table. Return it.
-    return oldbits.getSideTable();
-  } 
-  else if (failIfDeiniting && oldbits.getIsDeiniting()) {
-    // Already past the start of deinit. Do nothing.
-    return nullptr;
-  }
-
-  // Preflight passed. Allocate a side table.
-  
-  // FIXME: custom side table allocator
-  auto side = swift_cxx_newObject<HeapObjectSideTableEntry>(getHeapObject());
-  
-  auto newbits = InlineRefCountBits(side);
-  
-  do {
-    if (oldbits.hasSideTable()) {
-      // Already have a side table. Return it and delete ours.
-      // Read before delete to streamline barriers.
-      auto result = oldbits.getSideTable();
-      swift_cxx_deleteObject(side);
-      return result;
-    }
-    else if (failIfDeiniting && oldbits.getIsDeiniting()) {
-      // Already past the start of deinit. Do nothing.
-      return nullptr;
-    }
-    
-    side->initRefCounts(oldbits);
-    
-  } while (! refCounts.compare_exchange_weak(oldbits, newbits,
-                                             std::memory_order_release,
-                                             std::memory_order_relaxed));
-  return side;
-}
-
-
 // SideTableRefCountBits specialization intentionally does not exist.
 template <>
 HeapObjectSideTableEntry* RefCounts<InlineRefCountBits>::formWeakReference()
@@ -183,6 +199,17 @@ bool RefCounts<InlineRefCountBits>::setIsImmutableCOWBuffer(bool immutable) {
 
 #endif
 
+template <typename RefCountBits>
+void RefCounts<RefCountBits>::dump() const {
+  printf("Location: %p\n", this);
+  printf("Strong Ref Count: %d.\n", getCount());
+  printf("Unowned Ref Count: %d.\n", getUnownedCount());
+  printf("Weak Ref Count: %d.\n", getWeakCount());
+  printf("RefCount Side Table: %p.\n", getSideTable());
+  printf("Is Deiniting: %s.\n", isDeiniting() ? "true" : "false");
+  printf("Is Immortal: %s.\n", refCounts.load().isImmortal(false) ? "true" : "false");
+}
+
 // namespace swift
 } // namespace swift
 

test/Interpreter/Inputs/retain_release_wrappers.c

@@ -0,0 +1,46 @@
+#include <stdint.h>
+
+void *swift_retain_n(void *, uint32_t);
+void swift_release_n(void *, uint32_t);
+void *swift_nonatomic_retain_n(void *, uint32_t);
+void swift_nonatomic_release_n(void *, uint32_t);
+
+void *swift_unownedRetain_n(void *, uint32_t);
+void swift_unownedRelease_n(void *, uint32_t);
+void *swift_nonatomic_unownedRetain_n(void *, uint32_t);
+void swift_nonatomic_unownedRelease_n(void *, uint32_t);
+
+// Wrappers so we can call these from Swift without upsetting the ARC optimizer.
+void *wrapper_swift_retain_n(void *obj, uint32_t n) {
+  return swift_retain_n(obj, n);
+}
+
+void wrapper_swift_release_n(void *obj, uint32_t n) {
+  swift_release_n(obj, n);
+}
+
+void *wrapper_swift_nonatomic_retain_n(void *obj, uint32_t n) {
+  return swift_nonatomic_retain_n(obj, n);
+}
+
+void wrapper_swift_nonatomic_release_n(void *obj, uint32_t n) {
+  swift_nonatomic_release_n(obj, n);
+}
+
+void *wrapper_swift_unownedRetain_n(void *obj, uint32_t n) {
+  return swift_unownedRetain_n(obj, n);
+}
+
+void wrapper_swift_unownedRelease_n(void *obj, uint32_t n) {
+  swift_unownedRelease_n(obj, n);
+}
+
+void *wrapper_swift_nonatomic_unownedRetain_n(void *obj, uint32_t n) {
+  return swift_nonatomic_unownedRetain_n(obj, n);
+}
+
+void wrapper_swift_nonatomic_unownedRelease_n(void *obj, uint32_t n) {
+  swift_nonatomic_unownedRelease_n(obj, n);
+}
+
+