From 57f88d4a79735ce6436a81328bd1d1a50bca9a45 Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Mon, 4 Aug 2025 21:55:18 -0400 Subject: [PATCH 1/8] compiler-builtins: Change gating for outline atomic symbols Build outline atomic symbols on all targets that have `outline-atomics` enabled, rather than only on Linux. Since this is no longer OS-specific, also rename the module. --- .../src/{aarch64_linux.rs => aarch64_outline_atomics.rs} | 8 ++++---- library/compiler-builtins/compiler-builtins/src/lib.rs | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) rename library/compiler-builtins/compiler-builtins/src/{aarch64_linux.rs => aarch64_outline_atomics.rs} (97%) diff --git a/library/compiler-builtins/compiler-builtins/src/aarch64_linux.rs b/library/compiler-builtins/compiler-builtins/src/aarch64_outline_atomics.rs similarity index 97% rename from library/compiler-builtins/compiler-builtins/src/aarch64_linux.rs rename to library/compiler-builtins/compiler-builtins/src/aarch64_outline_atomics.rs index 01d7fb4732918..df0cf76502223 100644 --- a/library/compiler-builtins/compiler-builtins/src/aarch64_linux.rs +++ b/library/compiler-builtins/compiler-builtins/src/aarch64_outline_atomics.rs @@ -196,7 +196,7 @@ macro_rules! compare_and_swap { "cbnz w17, 0b", "1:", "ret", - have_lse = sym crate::aarch64_linux::HAVE_LSE_ATOMICS, + have_lse = sym crate::aarch64_outline_atomics::HAVE_LSE_ATOMICS, } } } @@ -228,7 +228,7 @@ macro_rules! compare_and_swap_i128 { "cbnz w15, 0b", "1:", "ret", - have_lse = sym crate::aarch64_linux::HAVE_LSE_ATOMICS, + have_lse = sym crate::aarch64_outline_atomics::HAVE_LSE_ATOMICS, } } } @@ -256,7 +256,7 @@ macro_rules! swap { concat!(stxr!($ordering, $bytes), " w17, ", reg!($bytes, 16), ", [x1]"), "cbnz w17, 0b", "ret", - have_lse = sym crate::aarch64_linux::HAVE_LSE_ATOMICS, + have_lse = sym crate::aarch64_outline_atomics::HAVE_LSE_ATOMICS, } } } @@ -286,7 +286,7 @@ macro_rules! fetch_op { concat!(stxr!($ordering, $bytes), " w15, ", reg!($bytes, 17), ", [x1]"), "cbnz w15, 0b", "ret", - have_lse = sym crate::aarch64_linux::HAVE_LSE_ATOMICS, + have_lse = sym crate::aarch64_outline_atomics::HAVE_LSE_ATOMICS, } } } diff --git a/library/compiler-builtins/compiler-builtins/src/lib.rs b/library/compiler-builtins/compiler-builtins/src/lib.rs index b111dc0bd18f3..581d5c24fa127 100644 --- a/library/compiler-builtins/compiler-builtins/src/lib.rs +++ b/library/compiler-builtins/compiler-builtins/src/lib.rs @@ -56,8 +56,8 @@ pub mod arm; #[cfg(any(target_arch = "aarch64", target_arch = "arm64ec"))] pub mod aarch64; -#[cfg(all(target_arch = "aarch64", target_os = "linux"))] -pub mod aarch64_linux; +#[cfg(all(target_arch = "aarch64", target_feature = "outline-atomics"))] +pub mod aarch64_outline_atomics; #[cfg(all( kernel_user_helpers, From 9cf6ece63ea3c1e619f4372b04ac083b80eacead Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Mon, 4 Aug 2025 21:55:18 -0400 Subject: [PATCH 2/8] Make the `RUST_LSE_INIT` constructor cross-platform Change the gating and link sections to enable this for any platforms that enable `outline-atomics`, rather than only Linux. Additionally, no longer run this if LSE is available, since in this case the outline versions will never be called. --- library/std/src/sys/configure_builtins.rs | 35 ++++++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/library/std/src/sys/configure_builtins.rs b/library/std/src/sys/configure_builtins.rs index 9d776b778dcbe..63a1d544d62a9 100644 --- a/library/std/src/sys/configure_builtins.rs +++ b/library/std/src/sys/configure_builtins.rs @@ -1,8 +1,35 @@ -/// Hook into .init_array to enable LSE atomic operations at startup, if -/// supported. -#[cfg(all(target_arch = "aarch64", target_os = "linux", not(feature = "compiler-builtins-c")))] +/// Enable LSE atomic operations at startup, if supported. +/// +/// Linker sections are based on what [`ctor`] does, with priorities to run slightly before user +/// code: +/// +/// - Apple uses the section `__mod_init_func`, `mod_init_funcs` is needed to set +/// `S_MOD_INIT_FUNC_POINTERS`. There doesn't seem to be a way to indicate priorities. +/// - Windows uses `.CRT$XCT`, which is run before user constructors (these should use `.CRT$XCU`). +/// - ELF uses `.init_array` with a priority of 90, which runs before our `ARGV_INIT_ARRAY` +/// initializer (priority 99). Both are within the 0-100 implementation-reserved range, per docs +/// for the [`prio-ctor-dtor`] warning, and this matches compiler-rt's `CONSTRUCTOR_PRIORITY`. +/// +/// To save startup time, the initializer is only run if outline atomic routines from +/// compiler-builtins may be used. If LSE is known to be available then the calls are never +/// emitted, and if we build the C intrinsics then it has its own initializer using the symbol +/// `__aarch64_have_lse_atomics`. +/// +/// [`ctor`]: https://github.com/mmastrac/rust-ctor/blob/63382b833ddcbfb8b064f4e86bfa1ed4026ff356/shared/src/macros/mod.rs#L522-L534 +/// [`prio-ctor-dtor`]: https://gcc.gnu.org/onlinedocs/gcc/Warning-Options.html +#[cfg(all( + target_arch = "aarch64", + target_feature = "outline-atomics", + not(target_feature = "lse"), + not(feature = "compiler-builtins-c"), +))] #[used] -#[unsafe(link_section = ".init_array.90")] +#[cfg_attr(target_vendor = "apple", unsafe(link_section = "__DATA,__mod_init_func,mod_init_funcs"))] +#[cfg_attr(target_os = "windows", unsafe(link_section = ".CRT$XCT"))] +#[cfg_attr( + not(any(target_vendor = "apple", target_os = "windows")), + unsafe(link_section = ".init_array.90") +)] static RUST_LSE_INIT: extern "C" fn() = { extern "C" fn init_lse() { use crate::arch; From dafcd25c5932c398edec10e76fc57b31efb6470c Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Fri, 5 Sep 2025 15:20:36 -0400 Subject: [PATCH 3/8] Enable `outline-atomics` by default on AArch64 Apple platforms Darwin and the `-sim` targets already have a baseline with LSE. Enable `outline-atomics` on all other Apple targets here. --- compiler/rustc_target/src/spec/targets/aarch64_apple_ios.rs | 2 +- compiler/rustc_target/src/spec/targets/aarch64_apple_tvos.rs | 2 +- .../rustc_target/src/spec/targets/aarch64_apple_visionos.rs | 2 +- compiler/rustc_target/src/spec/targets/aarch64_apple_watchos.rs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/compiler/rustc_target/src/spec/targets/aarch64_apple_ios.rs b/compiler/rustc_target/src/spec/targets/aarch64_apple_ios.rs index 3b522c34522b0..2d35263e06757 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_apple_ios.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_apple_ios.rs @@ -16,7 +16,7 @@ pub(crate) fn target() -> Target { .into(), arch, options: TargetOptions { - features: "+neon,+fp-armv8,+apple-a7".into(), + features: "+neon,+fp-armv8,+apple-a7,+outline-atomics".into(), max_atomic_width: Some(128), supported_sanitizers: SanitizerSet::ADDRESS | SanitizerSet::THREAD, ..opts diff --git a/compiler/rustc_target/src/spec/targets/aarch64_apple_tvos.rs b/compiler/rustc_target/src/spec/targets/aarch64_apple_tvos.rs index 7aef6f96e1c5f..09c0e17ca0d7c 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_apple_tvos.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_apple_tvos.rs @@ -16,7 +16,7 @@ pub(crate) fn target() -> Target { .into(), arch, options: TargetOptions { - features: "+neon,+fp-armv8,+apple-a7".into(), + features: "+neon,+fp-armv8,+apple-a7,+outline-atomics".into(), max_atomic_width: Some(128), ..opts }, diff --git a/compiler/rustc_target/src/spec/targets/aarch64_apple_visionos.rs b/compiler/rustc_target/src/spec/targets/aarch64_apple_visionos.rs index 22ce52e637f3b..bf583275488f2 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_apple_visionos.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_apple_visionos.rs @@ -16,7 +16,7 @@ pub(crate) fn target() -> Target { .into(), arch, options: TargetOptions { - features: "+neon,+fp-armv8,+apple-a16".into(), + features: "+neon,+fp-armv8,+apple-a16,+outline-atomics".into(), max_atomic_width: Some(128), supported_sanitizers: SanitizerSet::ADDRESS | SanitizerSet::THREAD, ..opts diff --git a/compiler/rustc_target/src/spec/targets/aarch64_apple_watchos.rs b/compiler/rustc_target/src/spec/targets/aarch64_apple_watchos.rs index 2e88f95f1dd81..8c36ba3070613 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_apple_watchos.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_apple_watchos.rs @@ -16,7 +16,7 @@ pub(crate) fn target() -> Target { .into(), arch, options: TargetOptions { - features: "+v8a,+neon,+fp-armv8,+apple-a7".into(), + features: "+v8a,+neon,+fp-armv8,+apple-a7,+outline-atomics".into(), max_atomic_width: Some(128), dynamic_linking: false, position_independent_executables: true, From f6d958f4fc903c53ea2fb28a489e9b56b41f1d36 Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Fri, 5 Sep 2025 15:20:59 -0400 Subject: [PATCH 4/8] Enable `outline-atomics` by default on AArch64 Windows platforms Windows has a similar flag `/forceInterlockedFunctions`, which uses names such as `_InterlockedAdd64_rel`. --- .../rustc_target/src/spec/targets/aarch64_pc_windows_gnullvm.rs | 2 +- .../rustc_target/src/spec/targets/aarch64_pc_windows_msvc.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/rustc_target/src/spec/targets/aarch64_pc_windows_gnullvm.rs b/compiler/rustc_target/src/spec/targets/aarch64_pc_windows_gnullvm.rs index eee668cc67edb..468a3eaf6cc95 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_pc_windows_gnullvm.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_pc_windows_gnullvm.rs @@ -3,7 +3,7 @@ use crate::spec::{FramePointer, Target, TargetMetadata, base}; pub(crate) fn target() -> Target { let mut base = base::windows_gnullvm::opts(); base.max_atomic_width = Some(128); - base.features = "+v8a,+neon,+fp-armv8".into(); + base.features = "+v8a,+neon,+fp-armv8,+outline-atomics".into(); base.linker = Some("aarch64-w64-mingw32-clang".into()); // Microsoft recommends enabling frame pointers on Arm64 Windows. diff --git a/compiler/rustc_target/src/spec/targets/aarch64_pc_windows_msvc.rs b/compiler/rustc_target/src/spec/targets/aarch64_pc_windows_msvc.rs index cd55576ef8167..b921459ceca9b 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_pc_windows_msvc.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_pc_windows_msvc.rs @@ -3,7 +3,7 @@ use crate::spec::{FramePointer, Target, TargetMetadata, base}; pub(crate) fn target() -> Target { let mut base = base::windows_msvc::opts(); base.max_atomic_width = Some(128); - base.features = "+v8a,+neon,+fp-armv8".into(); + base.features = "+v8a,+neon,+fp-armv8,+outline-atomics".into(); // Microsoft recommends enabling frame pointers on Arm64 Windows. // From https://learn.microsoft.com/en-us/cpp/build/arm64-windows-abi-conventions?view=msvc-170#integer-registers From d49a46a2315e2017dfb328bd55b4f2771ccc1355 Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Fri, 5 Sep 2025 15:23:00 -0400 Subject: [PATCH 5/8] Enable `outline-atomics` by default on AArch64 Android Per LLVM commit c5e7e649d537 ("[AArch64][Clang][Linux] Enable out-of-line atomics by default.") [1], Clang enables these on Android. Thus, do the same in Rust. [1]: https://github.com/llvm/llvm-project/commit/c5e7e649d537067de --- compiler/rustc_target/src/spec/targets/aarch64_linux_android.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/rustc_target/src/spec/targets/aarch64_linux_android.rs b/compiler/rustc_target/src/spec/targets/aarch64_linux_android.rs index d8651f305fe94..1cfa43b53a716 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_linux_android.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_linux_android.rs @@ -21,7 +21,7 @@ pub(crate) fn target() -> Target { max_atomic_width: Some(128), // As documented in https://developer.android.com/ndk/guides/cpu-features.html // the neon (ASIMD) and FP must exist on all android aarch64 targets. - features: "+v8a,+neon,+fp-armv8".into(), + features: "+v8a,+neon,+fp-armv8,+outline-atomics".into(), // the AAPCS64 expects use of non-leaf frame pointers per // https://github.com/ARM-software/abi-aa/blob/4492d1570eb70c8fd146623e0db65b2d241f12e7/aapcs64/aapcs64.rst#the-frame-pointer // and we tend to encounter interesting bugs in AArch64 unwinding code if we do not From d8eb386471735dbe22d21176df8182f825cdac5a Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Fri, 5 Sep 2025 15:27:17 -0400 Subject: [PATCH 6/8] Enable `outline-atomics` by default on AArch64 FreeBSD Clang does not currently have this enabled on FreeBSD, but there doesn't seem to be any specific reason not to. Thus, enable it here. --- .../rustc_target/src/spec/targets/aarch64_unknown_freebsd.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/rustc_target/src/spec/targets/aarch64_unknown_freebsd.rs b/compiler/rustc_target/src/spec/targets/aarch64_unknown_freebsd.rs index 7306a75aa2274..8c8d8ef1a560e 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_unknown_freebsd.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_unknown_freebsd.rs @@ -13,7 +13,7 @@ pub(crate) fn target() -> Target { data_layout: "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), arch: "aarch64".into(), options: TargetOptions { - features: "+v8a".into(), + features: "+v8a,+outline-atomics".into(), max_atomic_width: Some(128), stack_probes: StackProbeType::Inline, supported_sanitizers: SanitizerSet::ADDRESS From 0f135f73765f346c116c9145f181474d31781a29 Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Fri, 5 Sep 2025 15:34:47 -0400 Subject: [PATCH 7/8] Enable `outline-atomics` by default on AArch64 Fuchsia Clang has done this by default since LLVM commit 1a963d3278c2 ("[Driver] Make -moutline-atomics default for aarch64-fuchsia targets"), [1], so do the same here. [1]: https://github.com/llvm/llvm-project/commit/1a963d3278c2daab7e12125371442cd129c09954 --- .../rustc_target/src/spec/targets/aarch64_unknown_fuchsia.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/rustc_target/src/spec/targets/aarch64_unknown_fuchsia.rs b/compiler/rustc_target/src/spec/targets/aarch64_unknown_fuchsia.rs index 8366b6d9bd824..773fa9fd4dd06 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_unknown_fuchsia.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_unknown_fuchsia.rs @@ -5,7 +5,7 @@ use crate::spec::{ pub(crate) fn target() -> Target { let mut base = base::fuchsia::opts(); base.cpu = "generic".into(); - base.features = "+v8a,+crc,+aes,+sha2,+neon".into(); + base.features = "+v8a,+crc,+aes,+sha2,+neon,+outline-atomics".into(); base.max_atomic_width = Some(128); base.stack_probes = StackProbeType::Inline; base.supported_sanitizers = SanitizerSet::ADDRESS From b18a6c6817458b127f9f341484277687f999e82a Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Fri, 5 Sep 2025 15:34:57 -0400 Subject: [PATCH 8/8] Enable `outline-atomics` by default on AArch64 OpenBSD Clang has recently started doing this, as of LLVM commit 5d774ec8d183 ("[Driver] Enable outline atomics for OpenBSD/aarch64") [1]. Thus, do the same here. [1]: https://github.com/llvm/llvm-project/commit/5d774ec8d183cbbb243c57f50d891822211d3ec2 --- .../rustc_target/src/spec/targets/aarch64_unknown_openbsd.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/rustc_target/src/spec/targets/aarch64_unknown_openbsd.rs b/compiler/rustc_target/src/spec/targets/aarch64_unknown_openbsd.rs index c23006adad6d8..78953b66e4898 100644 --- a/compiler/rustc_target/src/spec/targets/aarch64_unknown_openbsd.rs +++ b/compiler/rustc_target/src/spec/targets/aarch64_unknown_openbsd.rs @@ -13,7 +13,7 @@ pub(crate) fn target() -> Target { data_layout: "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(), arch: "aarch64".into(), options: TargetOptions { - features: "+v8a".into(), + features: "+v8a,+outline-atomics".into(), max_atomic_width: Some(128), stack_probes: StackProbeType::Inline, ..base::openbsd::opts()