Lazily allocate+initialize locks.

m-ou-se · m-ou-se · commit 6a417d482899 · 2022-06-03T17:04:14.000+02:00
diff --git a/library/std/src/sys/sgx/condvar.rs b/library/std/src/sys/sgx/condvar.rs
@@ -1,4 +1,5 @@
 use crate::sys::locks::Mutex;
+use crate::sys_common::lazy_box::{LazyBox, LazyInit};
 use crate::time::Duration;
 
 use super::waitqueue::{SpinMutex, WaitQueue, WaitVariable};
@@ -7,16 +8,19 @@ pub struct Condvar {
     inner: SpinMutex<WaitVariable<()>>,
 }
 
-pub type MovableCondvar = Box<Condvar>;
+pub(crate) type MovableCondvar = LazyBox<Condvar>;
+
+impl LazyInit for Condvar {
+    fn init() -> Box<Self> {
+        Box::new(Self::new())
+    }
+}
 
 impl Condvar {
     pub const fn new() -> Condvar {
         Condvar { inner: SpinMutex::new(WaitVariable::new(())) }
     }
 
-    #[inline]
-    pub unsafe fn init(&mut self) {}
-
     #[inline]
     pub unsafe fn notify_one(&self) {
         let _ = WaitQueue::notify_one(self.inner.lock());
diff --git a/library/std/src/sys/sgx/mutex.rs b/library/std/src/sys/sgx/mutex.rs
@@ -1,11 +1,18 @@
 use super::waitqueue::{try_lock_or_false, SpinMutex, WaitQueue, WaitVariable};
+use crate::sys_common::lazy_box::{LazyBox, LazyInit};
 
 pub struct Mutex {
     inner: SpinMutex<WaitVariable<bool>>,
 }
 
 // not movable: see UnsafeList implementation
-pub type MovableMutex = Box<Mutex>;
+pub(crate) type MovableMutex = LazyBox<Mutex>;
+
+impl LazyInit for Mutex {
+    fn init() -> Box<Self> {
+        Box::new(Self::new())
+    }
+}
 
 // Implementation according to “Operating Systems: Three Easy Pieces”, chapter 28
 impl Mutex {
diff --git a/library/std/src/sys/sgx/rwlock.rs b/library/std/src/sys/sgx/rwlock.rs
@@ -2,6 +2,7 @@
 mod tests;
 
 use crate::num::NonZeroUsize;
+use crate::sys_common::lazy_box::{LazyBox, LazyInit};
 
 use super::waitqueue::{
     try_lock_or_false, NotifiedTcs, SpinMutex, SpinMutexGuard, WaitQueue, WaitVariable,
@@ -13,7 +14,13 @@ pub struct RwLock {
     writer: SpinMutex<WaitVariable<bool>>,
 }
 
-pub type MovableRwLock = Box<RwLock>;
+pub(crate) type MovableRwLock = LazyBox<RwLock>;
+
+impl LazyInit for RwLock {
+    fn init() -> Box<Self> {
+        Box::new(Self::new())
+    }
+}
 
 // Check at compile time that RwLock size matches C definition (see test_c_rwlock_initializer below)
 //
diff --git a/library/std/src/sys/unix/locks/futex.rs b/library/std/src/sys/unix/locks/futex.rs
@@ -115,9 +115,6 @@ impl Condvar {
         Self { futex: AtomicU32::new(0) }
     }
 
-    #[inline]
-    pub unsafe fn init(&mut self) {}
-
     // All the memory orderings here are `Relaxed`,
     // because synchronization is done by unlocking and locking the mutex.
 
diff --git a/library/std/src/sys/unix/locks/mod.rs b/library/std/src/sys/unix/locks/mod.rs
@@ -9,14 +9,14 @@ cfg_if::cfg_if! {
     ))] {
         mod futex;
         mod futex_rwlock;
-        pub use futex::{Mutex, MovableMutex, Condvar, MovableCondvar};
-        pub use futex_rwlock::{RwLock, MovableRwLock};
+        pub(crate) use futex::{Mutex, MovableMutex, MovableCondvar};
+        pub(crate) use futex_rwlock::{RwLock, MovableRwLock};
     } else {
         mod pthread_mutex;
         mod pthread_rwlock;
         mod pthread_condvar;
-        pub use pthread_mutex::{Mutex, MovableMutex};
-        pub use pthread_rwlock::{RwLock, MovableRwLock};
-        pub use pthread_condvar::{Condvar, MovableCondvar};
+        pub(crate) use pthread_mutex::{Mutex, MovableMutex};
+        pub(crate) use pthread_rwlock::{RwLock, MovableRwLock};
+        pub(crate) use pthread_condvar::MovableCondvar;
     }
 }
diff --git a/library/std/src/sys/unix/locks/pthread_condvar.rs b/library/std/src/sys/unix/locks/pthread_condvar.rs
@@ -1,12 +1,13 @@
 use crate::cell::UnsafeCell;
 use crate::sys::locks::{pthread_mutex, Mutex};
+use crate::sys_common::lazy_box::{LazyBox, LazyInit};
 use crate::time::Duration;
 
 pub struct Condvar {
     inner: UnsafeCell<libc::pthread_cond_t>,
 }
 
-pub type MovableCondvar = Box<Condvar>;
+pub(crate) type MovableCondvar = LazyBox<Condvar>;
 
 unsafe impl Send for Condvar {}
 unsafe impl Sync for Condvar {}
@@ -18,6 +19,14 @@ fn saturating_cast_to_time_t(value: u64) -> libc::time_t {
     if value > <libc::time_t>::MAX as u64 { <libc::time_t>::MAX } else { value as libc::time_t }
 }
 
+impl LazyInit for Condvar {
+    fn init() -> Box<Self> {
+        let mut condvar = Box::new(Self::new());
+        unsafe { condvar.init() };
+        condvar
+    }
+}
+
 impl Condvar {
     pub const fn new() -> Condvar {
         // Might be moved and address is changing it is better to avoid
@@ -32,14 +41,14 @@ impl Condvar {
         target_os = "android",
         target_os = "redox"
     ))]
-    pub unsafe fn init(&mut self) {}
+    unsafe fn init(&mut self) {}
 
     // NOTE: ESP-IDF's PTHREAD_COND_INITIALIZER support is not released yet
     // So on that platform, init() should always be called
     // Moreover, that platform does not have pthread_condattr_setclock support,
     // hence that initialization should be skipped as well
     #[cfg(target_os = "espidf")]
-    pub unsafe fn init(&mut self) {
+    unsafe fn init(&mut self) {
         let r = libc::pthread_cond_init(self.inner.get(), crate::ptr::null());
         assert_eq!(r, 0);
     }
@@ -52,7 +61,7 @@ impl Condvar {
         target_os = "redox",
         target_os = "espidf"
     )))]
-    pub unsafe fn init(&mut self) {
+    unsafe fn init(&mut self) {
         use crate::mem::MaybeUninit;
         let mut attr = MaybeUninit::<libc::pthread_condattr_t>::uninit();
         let r = libc::pthread_condattr_init(attr.as_mut_ptr());
diff --git a/library/std/src/sys/unix/locks/pthread_mutex.rs b/library/std/src/sys/unix/locks/pthread_mutex.rs
@@ -1,12 +1,13 @@
 use crate::cell::UnsafeCell;
 use crate::mem::MaybeUninit;
 use crate::sys::cvt_nz;
+use crate::sys_common::lazy_box::{LazyBox, LazyInit};
 
 pub struct Mutex {
     inner: UnsafeCell<libc::pthread_mutex_t>,
 }
 
-pub type MovableMutex = Box<Mutex>;
+pub(crate) type MovableMutex = LazyBox<Mutex>;
 
 #[inline]
 pub unsafe fn raw(m: &Mutex) -> *mut libc::pthread_mutex_t {
@@ -16,6 +17,14 @@ pub unsafe fn raw(m: &Mutex) -> *mut libc::pthread_mutex_t {
 unsafe impl Send for Mutex {}
 unsafe impl Sync for Mutex {}
 
+impl LazyInit for Mutex {
+    fn init() -> Box<Self> {
+        let mut mutex = Box::new(Self::new());
+        unsafe { mutex.init() };
+        mutex
+    }
+}
+
 impl Mutex {
     pub const fn new() -> Mutex {
         // Might be moved to a different address, so it is better to avoid
diff --git a/library/std/src/sys/unix/locks/pthread_rwlock.rs b/library/std/src/sys/unix/locks/pthread_rwlock.rs
@@ -1,17 +1,24 @@
 use crate::cell::UnsafeCell;
 use crate::sync::atomic::{AtomicUsize, Ordering};
+use crate::sys_common::lazy_box::{LazyBox, LazyInit};
 
 pub struct RwLock {
     inner: UnsafeCell<libc::pthread_rwlock_t>,
     write_locked: UnsafeCell<bool>, // guarded by the `inner` RwLock
     num_readers: AtomicUsize,
 }
 
-pub type MovableRwLock = Box<RwLock>;
+pub(crate) type MovableRwLock = LazyBox<RwLock>;
 
 unsafe impl Send for RwLock {}
 unsafe impl Sync for RwLock {}
 
+impl LazyInit for RwLock {
+    fn init() -> Box<Self> {
+        Box::new(Self::new())
+    }
+}
+
 impl RwLock {
     pub const fn new() -> RwLock {
         RwLock {
diff --git a/library/std/src/sys/unsupported/locks/condvar.rs b/library/std/src/sys/unsupported/locks/condvar.rs
@@ -10,9 +10,6 @@ impl Condvar {
         Condvar {}
     }
 
-    #[inline]
-    pub unsafe fn init(&mut self) {}
-
     #[inline]
     pub unsafe fn notify_one(&self) {}
 
diff --git a/library/std/src/sys/wasm/mod.rs b/library/std/src/sys/wasm/mod.rs
@@ -54,8 +54,8 @@ cfg_if::cfg_if! {
             #![allow(unsafe_op_in_unsafe_fn)]
             mod futex;
             mod futex_rwlock;
-            pub use futex::{Mutex, MovableMutex, Condvar, MovableCondvar};
-            pub use futex_rwlock::{RwLock, MovableRwLock};
+            pub(crate) use futex::{Mutex, MovableMutex, Condvar, MovableCondvar};
+            pub(crate) use futex_rwlock::{RwLock, MovableRwLock};
         }
         #[path = "atomics/futex.rs"]
         pub mod futex;
diff --git a/library/std/src/sys/windows/locks/condvar.rs b/library/std/src/sys/windows/locks/condvar.rs
@@ -18,9 +18,6 @@ impl Condvar {
         Condvar { inner: UnsafeCell::new(c::CONDITION_VARIABLE_INIT) }
     }
 
-    #[inline]
-    pub unsafe fn init(&mut self) {}
-
     #[inline]
     pub unsafe fn wait(&self, mutex: &Mutex) {
         let r = c::SleepConditionVariableSRW(self.inner.get(), mutex::raw(mutex), c::INFINITE, 0);
diff --git a/library/std/src/sys_common/condvar.rs b/library/std/src/sys_common/condvar.rs
@@ -15,9 +15,7 @@ pub struct Condvar {
 impl Condvar {
     /// Creates a new condition variable for use.
     pub fn new() -> Self {
-        let mut c = imp::MovableCondvar::from(imp::Condvar::new());
-        unsafe { c.init() };
-        Self { inner: c, check: CondvarCheck::new() }
+        Self { inner: imp::MovableCondvar::new(), check: CondvarCheck::new() }
     }
 
     /// Signals one waiter on this condition variable to wake up.
diff --git a/library/std/src/sys_common/condvar/check.rs b/library/std/src/sys_common/condvar/check.rs
@@ -1,6 +1,7 @@
 use crate::ptr;
 use crate::sync::atomic::{AtomicPtr, Ordering};
 use crate::sys::locks as imp;
+use crate::sys_common::lazy_box::{LazyBox, LazyInit};
 use crate::sys_common::mutex::MovableMutex;
 
 pub trait CondvarCheck {
@@ -9,7 +10,7 @@ pub trait CondvarCheck {
 
 /// For boxed mutexes, a `Condvar` will check it's only ever used with the same
 /// mutex, based on its (stable) address.
-impl CondvarCheck for Box<imp::Mutex> {
+impl<T: LazyInit> CondvarCheck for LazyBox<T> {
     type Check = SameMutexCheck;
 }
 
diff --git a/library/std/src/sys_common/lazy_box.rs b/library/std/src/sys_common/lazy_box.rs
@@ -0,0 +1,77 @@
+#![allow(dead_code)] // Only used on some platforms.
+
+// This is used to wrap pthread {Mutex, Condvar, RwLock} in.
+
+use crate::marker::PhantomData;
+use crate::ops::{Deref, DerefMut};
+use crate::ptr::null_mut;
+use crate::sync::atomic::{
+    AtomicPtr,
+    Ordering::{AcqRel, Acquire},
+};
+
+pub(crate) struct LazyBox<T: LazyInit> {
+    ptr: AtomicPtr<T>,
+    _phantom: PhantomData<T>,
+}
+
+pub(crate) trait LazyInit {
+    /// This is called before the box is allocated, to provide the value to
+    /// move into the new box.
+    ///
+    /// It might be called more than once per LazyBox, as multiple threads
+    /// might race to initialize it concurrently, each constructing and initializing
+    /// their own box. (All but one of them will be destroyed right after.)
+    fn init() -> Box<Self>;
+}
+
+impl<T: LazyInit> LazyBox<T> {
+    #[inline]
+    pub const fn new() -> Self {
+        Self { ptr: AtomicPtr::new(null_mut()), _phantom: PhantomData }
+    }
+
+    #[inline]
+    fn get_pointer(&self) -> *mut T {
+        let ptr = self.ptr.load(Acquire);
+        if ptr.is_null() { self.initialize() } else { ptr }
+    }
+
+    #[cold]
+    fn initialize(&self) -> *mut T {
+        let new_ptr = Box::into_raw(T::init());
+        match self.ptr.compare_exchange(null_mut(), new_ptr, AcqRel, Acquire) {
+            Ok(_) => new_ptr,
+            Err(ptr) => {
+                // Lost the race to another thread.
+                // Drop the box we created, and use the one from the other thread instead.
+                drop(unsafe { Box::from_raw(new_ptr) });
+                ptr
+            }
+        }
+    }
+}
+
+impl<T: LazyInit> Deref for LazyBox<T> {
+    type Target = T;
+    #[inline]
+    fn deref(&self) -> &T {
+        unsafe { &*self.get_pointer() }
+    }
+}
+
+impl<T: LazyInit> DerefMut for LazyBox<T> {
+    #[inline]
+    fn deref_mut(&mut self) -> &mut T {
+        unsafe { &mut *self.get_pointer() }
+    }
+}
+
+impl<T: LazyInit> Drop for LazyBox<T> {
+    fn drop(&mut self) {
+        let ptr = *self.ptr.get_mut();
+        if !ptr.is_null() {
+            drop(unsafe { Box::from_raw(ptr) });
+        }
+    }
+}
diff --git a/library/std/src/sys_common/mod.rs b/library/std/src/sys_common/mod.rs
@@ -24,6 +24,7 @@ pub mod backtrace;
 pub mod condvar;
 pub mod fs;
 pub mod io;
+pub mod lazy_box;
 pub mod memchr;
 pub mod mutex;
 pub mod process;
diff --git a/library/std/src/sys_common/mutex.rs b/library/std/src/sys_common/mutex.rs
@@ -61,9 +61,7 @@ unsafe impl Sync for MovableMutex {}
 impl MovableMutex {
     /// Creates a new mutex.
     pub fn new() -> Self {
-        let mut mutex = imp::MovableMutex::from(imp::Mutex::new());
-        unsafe { mutex.init() };
-        Self(mutex)
+        Self(imp::MovableMutex::new())
     }
 
     pub(super) fn raw(&self) -> &imp::Mutex {
diff --git a/library/std/src/sys_common/rwlock.rs b/library/std/src/sys_common/rwlock.rs
@@ -74,7 +74,7 @@ pub struct MovableRwLock(imp::MovableRwLock);
 impl MovableRwLock {
     /// Creates a new reader-writer lock for use.
     pub fn new() -> Self {
-        Self(imp::MovableRwLock::from(imp::RwLock::new()))
+        Self(imp::MovableRwLock::new())
     }
 
     /// Acquires shared access to the underlying lock, blocking the current

Original file line number	Diff line number	Diff line change
`@@ -115,9 +115,6 @@ impl Condvar {`
`115`	`115`	`Self { futex: AtomicU32::new(0) }`
`116`	`116`	`}`
`117`	`117`
`118`		`- #[inline]`
`119`		`- pub unsafe fn init(&mut self) {}`
`120`		`-`
`121`	`118`	// All the memory orderings here are `Relaxed`,
`122`	`119`	`// because synchronization is done by unlocking and locking the mutex.`
`123`	`120`
Original file line number	Diff line number	Diff line change
`@@ -10,9 +10,6 @@ impl Condvar {`
`10`	`10`	`Condvar {}`
`11`	`11`	`}`
`12`	`12`
`13`		`- #[inline]`
`14`		`- pub unsafe fn init(&mut self) {}`
`15`		`-`
`16`	`13`	`#[inline]`
`17`	`14`	`pub unsafe fn notify_one(&self) {}`
`18`	`15`