Auto merge of rust-lang#130679 - saethlin:inline-usually, r=<try>

Add inline(usually)

r? `@ghost`

I'm looking into what kind of things could recover the perf improvement detected in rust-lang#121417 (comment)

Author: bors

compiler/rustc_attr/src/builtin.rs

@@ -46,6 +46,7 @@ pub enum InlineAttr {
     Hint,
     Always,
     Never,
+    Usually,
 }
 
 #[derive(Clone, Encodable, Decodable, Debug, PartialEq, Eq, HashStable_Generic)]

compiler/rustc_codegen_gcc/src/attributes.rs

@@ -30,6 +30,7 @@ fn inline_attr<'gcc, 'tcx>(
                 None
             }
         }
+        InlineAttr::Usually => Some(FnAttribute::Inline),
         InlineAttr::None => None,
     }
 }

compiler/rustc_codegen_llvm/src/attributes.rs

@@ -47,6 +47,9 @@ fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll
             }
         }
         InlineAttr::None => None,
+        InlineAttr::Usually => {
+            Some(llvm::CreateAttrStringValue(cx.llcx, "function-inline-cost", "0"))
+        }
     }
 }
 

compiler/rustc_codegen_ssa/src/codegen_attrs.rs

@@ -525,9 +525,11 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
                         .emit();
                     InlineAttr::None
                 } else if list_contains_name(items, sym::always) {
-                    InlineAttr::Always
+                    InlineAttr::Usually
                 } else if list_contains_name(items, sym::never) {
                     InlineAttr::Never
+                } else if list_contains_name(items, sym::usually) {
+                    InlineAttr::Usually
                 } else {
                     struct_span_code_err!(tcx.dcx(), items[0].span(), E0535, "invalid argument")
                         .with_help("valid inline arguments are `always` and `never`")

compiler/rustc_mir_transform/src/cross_crate_inline.rs

@@ -46,7 +46,7 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
     // #[inline(never)] to force code generation.
     match codegen_fn_attrs.inline {
         InlineAttr::Never => return false,
-        InlineAttr::Hint | InlineAttr::Always => return true,
+        InlineAttr::Hint | InlineAttr::Always | InlineAttr::Usually => return true,
         _ => {}
     }
 

compiler/rustc_mir_transform/src/inline.rs

@@ -106,7 +106,7 @@ fn inline<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool {
         changed: false,
         caller_is_inline_forwarder: matches!(
             codegen_fn_attrs.inline,
-            InlineAttr::Hint | InlineAttr::Always
+            InlineAttr::Hint | InlineAttr::Always | InlineAttr::Usually
         ) && body_is_forwarder(body),
     };
     let blocks = START_BLOCK..body.basic_blocks.next_index();

compiler/rustc_span/src/symbol.rs

@@ -2101,6 +2101,7 @@ symbols! {
         usize_legacy_fn_max_value,
         usize_legacy_fn_min_value,
         usize_legacy_mod,
+        usually,
         va_arg,
         va_copy,
         va_end,

library/core/src/alloc/layout.rs

@@ -84,7 +84,8 @@ impl Layout {
         true
     }
 
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]
+    #[cfg_attr(not(bootstrap), inline(usually))]
     const fn max_size_for_align(align: Alignment) -> usize {
         // (power-of-two implies align != 0.)
 

library/core/src/alloc/mod.rs

@@ -360,7 +360,8 @@ pub unsafe trait Allocator {
     /// Creates a "by reference" adapter for this instance of `Allocator`.
     ///
     /// The returned adapter also implements `Allocator` and will simply borrow this.
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]
+    #[cfg_attr(not(bootstrap), inline(usually))]
     fn by_ref(&self) -> &Self
     where
         Self: Sized,

library/core/src/cell.rs

@@ -813,12 +813,14 @@ fn panic_already_mutably_borrowed(err: BorrowError) -> ! {
 type BorrowFlag = isize;
 const UNUSED: BorrowFlag = 0;
 
-#[inline(always)]
+#[cfg_attr(bootstrap, inline(always))]
+#[cfg_attr(not(bootstrap), inline(usually))]
 fn is_writing(x: BorrowFlag) -> bool {
     x < UNUSED
 }
 
-#[inline(always)]
+#[cfg_attr(bootstrap, inline(always))]
+#[cfg_attr(not(bootstrap), inline(usually))]
 fn is_reading(x: BorrowFlag) -> bool {
     x > UNUSED
 }
@@ -2079,7 +2081,8 @@ impl<T> UnsafeCell<T> {
     /// ```
     #[stable(feature = "rust1", since = "1.0.0")]
     #[rustc_const_stable(feature = "const_unsafe_cell_new", since = "1.32.0")]
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]
+    #[cfg_attr(not(bootstrap), inline(usually))]
     pub const fn new(value: T) -> UnsafeCell<T> {
         UnsafeCell { value }
     }
@@ -2095,7 +2098,8 @@ impl<T> UnsafeCell<T> {
     ///
     /// let five = uc.into_inner();
     /// ```
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]
+    #[cfg_attr(not(bootstrap), inline(usually))]
     #[stable(feature = "rust1", since = "1.0.0")]
     // When this is const stabilized, please remove `primitive_into_inner` below.
     #[rustc_const_unstable(feature = "const_cell_into_inner", issue = "78729")]
@@ -2119,7 +2123,8 @@ impl<T: ?Sized> UnsafeCell<T> {
     /// *uc.get_mut() -= 1;
     /// assert_eq!(*uc.get_mut(), 41);
     /// ```
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]
+    #[cfg_attr(not(bootstrap), inline(usually))]
     #[unstable(feature = "unsafe_cell_from_mut", issue = "111645")]
     pub const fn from_mut(value: &mut T) -> &mut UnsafeCell<T> {
         // SAFETY: `UnsafeCell<T>` has the same memory layout as `T` due to #[repr(transparent)].
@@ -2142,7 +2147,8 @@ impl<T: ?Sized> UnsafeCell<T> {
     ///
     /// let five = uc.get();
     /// ```
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]
+    #[cfg_attr(not(bootstrap), inline(usually))]
     #[stable(feature = "rust1", since = "1.0.0")]
     #[rustc_const_stable(feature = "const_unsafecell_get", since = "1.32.0")]
     #[rustc_never_returns_null_ptr]
@@ -2168,7 +2174,8 @@ impl<T: ?Sized> UnsafeCell<T> {
     ///
     /// assert_eq!(*c.get_mut(), 6);
     /// ```
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]
+    #[cfg_attr(not(bootstrap), inline(usually))]
     #[stable(feature = "unsafe_cell_get_mut", since = "1.50.0")]
     #[rustc_const_unstable(feature = "const_unsafecell_get_mut", issue = "88836")]
     pub const fn get_mut(&mut self) -> &mut T {
@@ -2203,7 +2210,8 @@ impl<T: ?Sized> UnsafeCell<T> {
     ///
     /// assert_eq!(uc.into_inner(), 5);
     /// ```
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]
+    #[cfg_attr(not(bootstrap), inline(usually))]
     #[stable(feature = "unsafe_cell_raw_get", since = "1.56.0")]
     #[rustc_const_stable(feature = "unsafe_cell_raw_get", since = "1.56.0")]
     #[rustc_diagnostic_item = "unsafe_cell_raw_get"]