Auto merge of rust-lang#130679 - saethlin:inline-usually, r=<try>

Add inline(usually)

r? `@ghost`

I'm looking into what kind of things could recover the perf improvement detected in rust-lang#121417 (comment)

Author: bors

compiler/rustc_attr/src/builtin.rs

@@ -46,6 +46,7 @@ pub enum InlineAttr {
     Hint,
     Always,
     Never,
+    Usually,
 }
 
 #[derive(Clone, Encodable, Decodable, Debug, PartialEq, Eq, HashStable_Generic)]

compiler/rustc_codegen_llvm/src/attributes.rs

@@ -47,6 +47,9 @@ fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll
             }
         }
         InlineAttr::None => None,
+        InlineAttr::Usually => {
+            Some(llvm::CreateAttrStringValue(cx.llcx, "function-inline-cost", "0"))
+        }
     }
 }
 

compiler/rustc_codegen_ssa/src/codegen_attrs.rs

@@ -546,6 +546,8 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
                     InlineAttr::Always
                 } else if list_contains_name(items, sym::never) {
                     InlineAttr::Never
+                } else if list_contains_name(items, sym::usually) {
+                    InlineAttr::Usually
                 } else {
                     struct_span_code_err!(tcx.dcx(), items[0].span(), E0535, "invalid argument")
                         .with_help("valid inline arguments are `always` and `never`")

compiler/rustc_middle/src/mir/mono.rs

@@ -138,7 +138,7 @@ impl<'tcx> MonoItem<'tcx> {
                 // conflict with upstream crates as it could be an exported
                 // symbol.
                 match tcx.codegen_fn_attrs(instance.def_id()).inline {
-                    InlineAttr::Always => InstantiationMode::LocalCopy,
+                    InlineAttr::Always | InlineAttr::Usually => InstantiationMode::LocalCopy,
                     _ => InstantiationMode::GloballyShared { may_conflict: true },
                 }
             }

compiler/rustc_mir_transform/src/cross_crate_inline.rs

@@ -46,7 +46,7 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
     // #[inline(never)] to force code generation.
     match codegen_fn_attrs.inline {
         InlineAttr::Never => return false,
-        InlineAttr::Hint | InlineAttr::Always => return true,
+        InlineAttr::Hint | InlineAttr::Always | InlineAttr::Usually => return true,
         _ => {}
     }
 

compiler/rustc_mir_transform/src/inline.rs

@@ -106,7 +106,7 @@ fn inline<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool {
         changed: false,
         caller_is_inline_forwarder: matches!(
             codegen_fn_attrs.inline,
-            InlineAttr::Hint | InlineAttr::Always
+            InlineAttr::Hint | InlineAttr::Always | InlineAttr::Usually
         ) && body_is_forwarder(body),
     };
     let blocks = START_BLOCK..body.basic_blocks.next_index();

compiler/rustc_span/src/symbol.rs

@@ -2101,6 +2101,7 @@ symbols! {
         usize_legacy_fn_max_value,
         usize_legacy_fn_min_value,
         usize_legacy_mod,
+        usually,
         va_arg,
         va_copy,
         va_end,

library/core/src/option.rs

@@ -958,7 +958,8 @@ impl<T> Option<T> {
     /// let x: Option<&str> = None;
     /// assert_eq!(x.unwrap(), "air"); // fails
     /// ```
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]
+    #[cfg_attr(not(bootstrap), inline(usually))]
     #[track_caller]
     #[stable(feature = "rust1", since = "1.0.0")]
     #[cfg_attr(not(test), rustc_diagnostic_item = "option_unwrap")]

library/core/src/result.rs

@@ -1091,7 +1091,8 @@ impl<T, E> Result<T, E> {
     /// let x: Result<u32, &str> = Err("emergency failure");
     /// x.unwrap(); // panics with `emergency failure`
     /// ```
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]
+    #[cfg_attr(not(bootstrap), inline(usually))]
     #[track_caller]
     #[stable(feature = "rust1", since = "1.0.0")]
     pub fn unwrap(self) -> T

library/core/src/slice/index.rs

@@ -11,7 +11,8 @@ where
 {
     type Output = I::Output;
 
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]
+    #[cfg_attr(not(bootstrap), inline(usually))]
     fn index(&self, index: I) -> &I::Output {
         index.index(self)
     }
@@ -22,7 +23,8 @@ impl<T, I> ops::IndexMut<I> for [T]
 where
     I: SliceIndex<[T]>,
 {
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]
+    #[cfg_attr(not(bootstrap), inline(usually))]
     fn index_mut(&mut self, index: I) -> &mut I::Output {
         index.index_mut(self)
     }
@@ -455,7 +457,8 @@ unsafe impl<T> SliceIndex<[T]> for ops::Range<usize> {
         }
     }
 
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]
+    #[cfg_attr(not(bootstrap), inline(usually))]
     fn index(self, slice: &[T]) -> &[T] {
         // Using checked_sub is a safe way to get `SubUnchecked` in MIR
         let Some(new_len) = usize::checked_sub(self.end, self.start) else {
@@ -507,7 +510,8 @@ unsafe impl<T> SliceIndex<[T]> for range::Range<usize> {
         unsafe { ops::Range::from(self).get_unchecked_mut(slice) }
     }
 
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]
+    #[cfg_attr(not(bootstrap), inline(usually))]
     fn index(self, slice: &[T]) -> &[T] {
         ops::Range::from(self).index(slice)
     }
@@ -545,7 +549,8 @@ unsafe impl<T> SliceIndex<[T]> for ops::RangeTo<usize> {
         unsafe { (0..self.end).get_unchecked_mut(slice) }
     }
 
-    #[inline(always)]
+    #[cfg_attr(bootstrap, inline(always))]
+    #[cfg_attr(not(bootstrap), inline(usually))]
     fn index(self, slice: &[T]) -> &[T] {
         (0..self.end).index(slice)
     }