From 1c82aa75941aeef3db1fc401ca41df97d17d0652 Mon Sep 17 00:00:00 2001 From: joboet Date: Sat, 9 Mar 2024 12:46:11 +0100 Subject: [PATCH 1/2] std: allow `weak` in item position on Apple platforms --- library/std/src/sys/fd/unix.rs | 8 ++++---- library/std/src/sys/pal/unix/weak/dlsym.rs | 9 ++++----- library/std/src/sys/pal/unix/weak/tests.rs | 4 ++-- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/library/std/src/sys/fd/unix.rs b/library/std/src/sys/fd/unix.rs index 2b2dfe48e89e2..ac1913bfc9d1d 100644 --- a/library/std/src/sys/fd/unix.rs +++ b/library/std/src/sys/fd/unix.rs @@ -326,9 +326,9 @@ impl FileDesc { ); match preadv.get() { - Some(preadv) => { + Some(read) => { let ret = cvt(unsafe { - preadv( + read( self.as_raw_fd(), bufs.as_mut_ptr() as *mut libc::iovec as *const libc::iovec, cmp::min(bufs.len(), max_iov()) as libc::c_int, @@ -532,9 +532,9 @@ impl FileDesc { ); match pwritev.get() { - Some(pwritev) => { + Some(read) => { let ret = cvt(unsafe { - pwritev( + read( self.as_raw_fd(), bufs.as_ptr() as *const libc::iovec, cmp::min(bufs.len(), max_iov()) as libc::c_int, diff --git a/library/std/src/sys/pal/unix/weak/dlsym.rs b/library/std/src/sys/pal/unix/weak/dlsym.rs index 4967b93cc52b5..3ee5260620c1b 100644 --- a/library/std/src/sys/pal/unix/weak/dlsym.rs +++ b/library/std/src/sys/pal/unix/weak/dlsym.rs @@ -8,8 +8,9 @@ use crate::{mem, ptr}; mod tests; pub(crate) macro weak { - (fn $name:ident($($param:ident : $t:ty),* $(,)?) -> $ret:ty;) => ( - static DLSYM: DlsymWeak $ret> = { + ($v:vis fn $name:ident($($param:ident : $t:ty),* $(,)?) -> $ret:ty;) => { + #[allow(non_upper_case_globals)] + $v static $name: DlsymWeak $ret> = { let Ok(name) = CStr::from_bytes_with_nul(concat!(stringify!($name), '\0').as_bytes()) else { panic!("symbol name may not contain NUL") }; @@ -20,9 +21,7 @@ pub(crate) macro weak { // the function pointer be unsafe. unsafe { DlsymWeak::new(name) } }; - - let $name = &DLSYM; - ) + } } pub(crate) struct DlsymWeak { diff --git a/library/std/src/sys/pal/unix/weak/tests.rs b/library/std/src/sys/pal/unix/weak/tests.rs index 90d52095694a5..244bcb7f59f6c 100644 --- a/library/std/src/sys/pal/unix/weak/tests.rs +++ b/library/std/src/sys/pal/unix/weak/tests.rs @@ -11,8 +11,8 @@ fn weak_existing() { fn strlen(cs: *const c_char) -> usize; } - let strlen = strlen.get().unwrap(); - assert_eq!(unsafe { strlen(TEST_STRING.as_ptr()) }, TEST_STRING.count_bytes()); + let len = strlen.get().unwrap(); + assert_eq!(unsafe { len(TEST_STRING.as_ptr()) }, TEST_STRING.count_bytes()); } #[test] From dc7c2b686ce5c5e4c26a74bb0958e304b4e25757 Mon Sep 17 00:00:00 2001 From: joboet Date: Mon, 3 Feb 2025 16:46:22 +0100 Subject: [PATCH 2/2] std: use futex for everything but `Mutex` on Apple platforms --- library/std/src/sys/pal/unix/futex.rs | 201 ++++++++++++++++++ library/std/src/sys/pal/unix/sync/mod.rs | 2 + library/std/src/sys/sync/condvar/mod.rs | 1 + library/std/src/sys/sync/once/mod.rs | 4 + library/std/src/sys/sync/rwlock/mod.rs | 1 + .../std/src/sys/sync/thread_parking/darwin.rs | 130 ----------- .../std/src/sys/sync/thread_parking/mod.rs | 1 + triagebot.toml | 1 - 8 files changed, 210 insertions(+), 131 deletions(-) delete mode 100644 library/std/src/sys/sync/thread_parking/darwin.rs diff --git a/library/std/src/sys/pal/unix/futex.rs b/library/std/src/sys/pal/unix/futex.rs index 265067d84d502..f9f3d5fc6f289 100644 --- a/library/std/src/sys/pal/unix/futex.rs +++ b/library/std/src/sys/pal/unix/futex.rs @@ -1,5 +1,6 @@ #![cfg(any( target_os = "linux", + target_vendor = "apple", target_os = "android", all(target_os = "emscripten", target_feature = "atomics"), target_os = "freebsd", @@ -147,6 +148,206 @@ pub fn futex_wake_all(futex: &Atomic) { }; } +/// With macOS version 14.4, Apple introduced a public futex API. Unfortunately, +/// our minimum supported version is 10.12, so we need a fallback API. Luckily +/// for us, the underlying syscalls have been available since exactly that +/// version, so we just use those when needed. This is private API however, +/// which means we need to take care to avoid breakage if the syscall is removed +/// and to avoid apps being rejected from the App Store. To do this, we use weak +/// linkage emulation for both the public and the private API. Experiments +/// indicate that this way of referencing private symbols is not flagged by the +/// App Store checks, see +/// https://github.com/rust-lang/rust/pull/122408#issuecomment-3403989895 +/// +/// See https://developer.apple.com/documentation/os/os_sync_wait_on_address?language=objc +/// for documentation of the public API and +/// https://github.com/apple-oss-distributions/xnu/blob/1031c584a5e37aff177559b9f69dbd3c8c3fd30a/bsd/sys/ulock.h#L69 +/// for the header file of the private API, along with its usage in libpthread +/// https://github.com/apple-oss-distributions/libpthread/blob/d8c4e3c212553d3e0f5d76bb7d45a8acd61302dc/src/pthread_cond.c#L463 +#[cfg(target_vendor = "apple")] +mod apple { + use crate::ffi::{c_int, c_void}; + use crate::sys::pal::weak::weak; + + pub const OS_CLOCK_MACH_ABSOLUTE_TIME: u32 = 32; + pub const OS_SYNC_WAIT_ON_ADDRESS_NONE: u32 = 0; + pub const OS_SYNC_WAKE_BY_ADDRESS_NONE: u32 = 0; + + pub const UL_COMPARE_AND_WAIT: u32 = 1; + pub const ULF_WAKE_ALL: u32 = 0x100; + // The syscalls support directly returning errors instead of going through errno. + pub const ULF_NO_ERRNO: u32 = 0x1000000; + + // These functions appeared with macOS 14.4, iOS 17.4, tvOS 17.4, watchOS 10.4, visionOS 1.1. + weak! { + pub fn os_sync_wait_on_address(addr: *mut c_void, value: u64, size: usize, flags: u32) -> c_int; + } + + weak! { + pub fn os_sync_wait_on_address_with_timeout(addr: *mut c_void, value: u64, size: usize, flags: u32, clockid: u32, timeout_ns: u64) -> c_int; + } + + weak! { + pub fn os_sync_wake_by_address_any(addr: *mut c_void, size: usize, flags: u32) -> c_int; + } + + weak! { + pub fn os_sync_wake_by_address_all(addr: *mut c_void, size: usize, flags: u32) -> c_int; + } + + // This syscall appeared with macOS 11.0. + // It is used to support nanosecond precision for timeouts, among other features. + weak! { + pub fn __ulock_wait2(operation: u32, addr: *mut c_void, value: u64, timeout: u64, value2: u64) -> c_int; + } + + // These syscalls appeared with macOS 10.12. + weak! { + pub fn __ulock_wait(operation: u32, addr: *mut c_void, value: u64, timeout: u32) -> c_int; + } + + weak! { + pub fn __ulock_wake(operation: u32, addr: *mut c_void, wake_value: u64) -> c_int; + } +} + +#[cfg(target_vendor = "apple")] +pub fn futex_wait(futex: &Atomic, expected: u32, timeout: Option) -> bool { + use apple::*; + + use crate::mem::size_of; + + let addr = futex.as_ptr().cast(); + let value = expected as u64; + let size = size_of::(); + if let Some(timeout) = timeout { + let timeout_ns = timeout.as_nanos().clamp(1, u64::MAX as u128) as u64; + + if let Some(wait) = os_sync_wait_on_address_with_timeout.get() { + let r = unsafe { + wait( + addr, + value, + size, + OS_SYNC_WAIT_ON_ADDRESS_NONE, + OS_CLOCK_MACH_ABSOLUTE_TIME, + timeout_ns, + ) + }; + + // We promote spurious wakeups (reported as EINTR) to normal ones for + // simplicity. + r != -1 || super::os::errno() != libc::ETIMEDOUT + } else if let Some(wait) = __ulock_wait2.get() { + let r = unsafe { wait(UL_COMPARE_AND_WAIT | ULF_NO_ERRNO, addr, value, timeout_ns, 0) }; + + r != -libc::ETIMEDOUT + } else if let Some(wait) = __ulock_wait.get() { + let (timeout_us, truncated) = match timeout.as_micros().try_into() { + Ok(timeout_us) => (u32::max(timeout_us, 1), false), + Err(_) => (u32::MAX, true), + }; + + let r = unsafe { wait(UL_COMPARE_AND_WAIT | ULF_NO_ERRNO, addr, value, timeout_us) }; + + // Report truncation as a spurious wakeup instead of a timeout. + // Truncation occurs for timeout durations larger than 4295 s + // ≈ 1 hour, so it should be considered. + r != -libc::ETIMEDOUT || truncated + } else { + rtabort!("your system is below the minimum supported version of Rust"); + } + } else { + if let Some(wait) = os_sync_wait_on_address.get() { + unsafe { wait(addr, value, size, OS_SYNC_WAIT_ON_ADDRESS_NONE) }; + } else if let Some(wait) = __ulock_wait2.get() { + unsafe { wait(UL_COMPARE_AND_WAIT | ULF_NO_ERRNO, addr, value, 0, 0) }; + } else if let Some(wait) = __ulock_wait.get() { + unsafe { wait(UL_COMPARE_AND_WAIT | ULF_NO_ERRNO, addr, value, 0) }; + } else { + rtabort!("your system is below the minimum supported version of Rust"); + } + + true + } +} + +#[cfg(target_vendor = "apple")] +pub fn futex_wake(futex: &Atomic) -> bool { + use apple::*; + + use crate::io::Error; + use crate::mem::size_of; + + let addr = futex.as_ptr().cast(); + if let Some(wake) = os_sync_wake_by_address_any.get() { + let r = unsafe { wake(addr, size_of::(), OS_SYNC_WAKE_BY_ADDRESS_NONE) }; + if r == 0 { + true + } else { + match super::os::errno() { + // There were no waiters to wake up. + libc::ENOENT => false, + err => rtabort!("__ulock_wake failed: {}", Error::from_raw_os_error(err)), + } + } + } else if let Some(wake) = __ulock_wake.get() { + // Judging by its use in pthreads, __ulock_wake can get interrupted, so + // retry until either waking up a waiter or failing because there are no + // waiters (ENOENT). + loop { + let r = unsafe { wake(UL_COMPARE_AND_WAIT | ULF_NO_ERRNO, addr, 0) }; + + if r >= 0 { + return true; + } else { + match -r { + libc::ENOENT => return false, + libc::EINTR => continue, + err => rtabort!("__ulock_wake failed: {}", Error::from_raw_os_error(err)), + } + } + } + } else { + rtabort!("your system is below the minimum supported version of Rust"); + } +} + +#[cfg(target_vendor = "apple")] +pub fn futex_wake_all(futex: &Atomic) { + use apple::*; + + use crate::io::Error; + use crate::mem::size_of; + + let addr = futex.as_ptr().cast(); + + if let Some(wake) = os_sync_wake_by_address_all.get() { + unsafe { + wake(addr, size_of::(), OS_SYNC_WAKE_BY_ADDRESS_NONE); + } + } else if let Some(wake) = __ulock_wake.get() { + // Judging by its use in pthreads, __ulock_wake can get interrupted, so + // retry until either waking up a waiter or failing because there are no + // waiters (ENOENT). + loop { + let r = unsafe { wake(UL_COMPARE_AND_WAIT | ULF_WAKE_ALL | ULF_NO_ERRNO, addr, 0) }; + + if r >= 0 { + return; + } else { + match -r { + libc::ENOENT => return, + libc::EINTR => continue, + err => panic!("__ulock_wake failed: {}", Error::from_raw_os_error(err)), + } + } + } + } else { + panic!("your system is below the minimum supported version of Rust"); + } +} + #[cfg(target_os = "openbsd")] pub fn futex_wait(futex: &Atomic, expected: u32, timeout: Option) -> bool { use super::time::Timespec; diff --git a/library/std/src/sys/pal/unix/sync/mod.rs b/library/std/src/sys/pal/unix/sync/mod.rs index b430ff5d8ef5f..17f5b50e14728 100644 --- a/library/std/src/sys/pal/unix/sync/mod.rs +++ b/library/std/src/sys/pal/unix/sync/mod.rs @@ -9,8 +9,10 @@ )))] #![forbid(unsafe_op_in_unsafe_fn)] +#[cfg(not(target_vendor = "apple"))] mod condvar; mod mutex; +#[cfg(not(target_vendor = "apple"))] pub use condvar::Condvar; pub use mutex::Mutex; diff --git a/library/std/src/sys/sync/condvar/mod.rs b/library/std/src/sys/sync/condvar/mod.rs index 83cf0ae629851..e149ebafa4332 100644 --- a/library/std/src/sys/sync/condvar/mod.rs +++ b/library/std/src/sys/sync/condvar/mod.rs @@ -2,6 +2,7 @@ cfg_select! { any( all(target_os = "windows", not(target_vendor="win7")), target_os = "linux", + target_vendor = "apple", target_os = "android", target_os = "freebsd", target_os = "openbsd", diff --git a/library/std/src/sys/sync/once/mod.rs b/library/std/src/sys/sync/once/mod.rs index aeea884b9f617..933b61fcdfdd5 100644 --- a/library/std/src/sys/sync/once/mod.rs +++ b/library/std/src/sys/sync/once/mod.rs @@ -19,6 +19,10 @@ cfg_select! { target_os = "dragonfly", target_os = "fuchsia", target_os = "hermit", + target_os = "macos", + target_os = "ios", + target_os = "tvos", + target_os = "watchos", ) => { mod futex; pub use futex::{Once, OnceState}; diff --git a/library/std/src/sys/sync/rwlock/mod.rs b/library/std/src/sys/sync/rwlock/mod.rs index ab5715bf2de33..3a4c834c755a2 100644 --- a/library/std/src/sys/sync/rwlock/mod.rs +++ b/library/std/src/sys/sync/rwlock/mod.rs @@ -2,6 +2,7 @@ cfg_select! { any( all(target_os = "windows", not(target_vendor = "win7")), target_os = "linux", + target_vendor = "apple", target_os = "android", target_os = "freebsd", target_os = "openbsd", diff --git a/library/std/src/sys/sync/thread_parking/darwin.rs b/library/std/src/sys/sync/thread_parking/darwin.rs deleted file mode 100644 index b9bcc538c65ab..0000000000000 --- a/library/std/src/sys/sync/thread_parking/darwin.rs +++ /dev/null @@ -1,130 +0,0 @@ -//! Thread parking for Darwin-based systems. -//! -//! Darwin actually has futex syscalls (`__ulock_wait`/`__ulock_wake`), but they -//! cannot be used in `std` because they are non-public (their use will lead to -//! rejection from the App Store). -//! -//! Therefore, we need to look for other synchronization primitives. Luckily, Darwin -//! supports semaphores, which allow us to implement the behavior we need with -//! only one primitive (as opposed to a mutex-condvar pair). We use the semaphore -//! provided by libdispatch, as the underlying Mach semaphore is only dubiously -//! public. - -#![allow(non_camel_case_types)] - -use crate::pin::Pin; -use crate::sync::atomic::Ordering::{Acquire, Release}; -use crate::sync::atomic::{Atomic, AtomicI8}; -use crate::time::Duration; - -type dispatch_semaphore_t = *mut crate::ffi::c_void; -type dispatch_time_t = u64; - -const DISPATCH_TIME_NOW: dispatch_time_t = 0; -const DISPATCH_TIME_FOREVER: dispatch_time_t = !0; - -// Contained in libSystem.dylib, which is linked by default. -unsafe extern "C" { - fn dispatch_time(when: dispatch_time_t, delta: i64) -> dispatch_time_t; - fn dispatch_semaphore_create(val: isize) -> dispatch_semaphore_t; - fn dispatch_semaphore_wait(dsema: dispatch_semaphore_t, timeout: dispatch_time_t) -> isize; - fn dispatch_semaphore_signal(dsema: dispatch_semaphore_t) -> isize; - fn dispatch_release(object: *mut crate::ffi::c_void); -} - -const EMPTY: i8 = 0; -const NOTIFIED: i8 = 1; -const PARKED: i8 = -1; - -pub struct Parker { - semaphore: dispatch_semaphore_t, - state: Atomic, -} - -unsafe impl Sync for Parker {} -unsafe impl Send for Parker {} - -impl Parker { - pub unsafe fn new_in_place(parker: *mut Parker) { - let semaphore = dispatch_semaphore_create(0); - assert!( - !semaphore.is_null(), - "failed to create dispatch semaphore for thread synchronization" - ); - parker.write(Parker { semaphore, state: AtomicI8::new(EMPTY) }) - } - - // Does not need `Pin`, but other implementation do. - pub unsafe fn park(self: Pin<&Self>) { - // The semaphore counter must be zero at this point, because unparking - // threads will not actually increase it until we signalled that we - // are waiting. - - // Change NOTIFIED to EMPTY and EMPTY to PARKED. - if self.state.fetch_sub(1, Acquire) == NOTIFIED { - return; - } - - // Another thread may increase the semaphore counter from this point on. - // If it is faster than us, we will decrement it again immediately below. - // If we are faster, we wait. - - // Ensure that the semaphore counter has actually been decremented, even - // if the call timed out for some reason. - while dispatch_semaphore_wait(self.semaphore, DISPATCH_TIME_FOREVER) != 0 {} - - // At this point, the semaphore counter is zero again. - - // We were definitely woken up, so we don't need to check the state. - // Still, we need to reset the state using a swap to observe the state - // change with acquire ordering. - self.state.swap(EMPTY, Acquire); - } - - // Does not need `Pin`, but other implementation do. - pub unsafe fn park_timeout(self: Pin<&Self>, dur: Duration) { - if self.state.fetch_sub(1, Acquire) == NOTIFIED { - return; - } - - let nanos = dur.as_nanos().try_into().unwrap_or(i64::MAX); - let timeout = dispatch_time(DISPATCH_TIME_NOW, nanos); - - let timeout = dispatch_semaphore_wait(self.semaphore, timeout) != 0; - - let state = self.state.swap(EMPTY, Acquire); - if state == NOTIFIED && timeout { - // If the state was NOTIFIED but semaphore_wait returned without - // decrementing the count because of a timeout, it means another - // thread is about to call semaphore_signal. We must wait for that - // to happen to ensure the semaphore count is reset. - while dispatch_semaphore_wait(self.semaphore, DISPATCH_TIME_FOREVER) != 0 {} - } else { - // Either a timeout occurred and we reset the state before any thread - // tried to wake us up, or we were woken up and reset the state, - // making sure to observe the state change with acquire ordering. - // Either way, the semaphore counter is now zero again. - } - } - - // Does not need `Pin`, but other implementation do. - pub fn unpark(self: Pin<&Self>) { - let state = self.state.swap(NOTIFIED, Release); - if state == PARKED { - unsafe { - dispatch_semaphore_signal(self.semaphore); - } - } - } -} - -impl Drop for Parker { - fn drop(&mut self) { - // SAFETY: - // We always ensure that the semaphore count is reset, so this will - // never cause an exception. - unsafe { - dispatch_release(self.semaphore); - } - } -} diff --git a/library/std/src/sys/sync/thread_parking/mod.rs b/library/std/src/sys/sync/thread_parking/mod.rs index e8a9dc884f816..f8259974662f9 100644 --- a/library/std/src/sys/sync/thread_parking/mod.rs +++ b/library/std/src/sys/sync/thread_parking/mod.rs @@ -2,6 +2,7 @@ cfg_select! { any( all(target_os = "windows", not(target_vendor = "win7")), target_os = "linux", + target_vendor = "apple", target_os = "android", all(target_arch = "wasm32", target_feature = "atomics"), target_os = "freebsd", diff --git a/triagebot.toml b/triagebot.toml index 5a96d96b346d4..3f38e2f2cb4ae 100644 --- a/triagebot.toml +++ b/triagebot.toml @@ -385,7 +385,6 @@ trigger_files = [ trigger_files = [ "library/std/src/os/darwin", "library/std/src/sys/platform_version/darwin", - "library/std/src/sys/sync/thread_parking/darwin.rs", "compiler/rustc_target/src/spec/base/apple", ]