Auto merge of #144483 - scottmcm:early-inline, r=<try>

EXPERIMENT: Use the force-inline pass for more stuff

Author: rust-bors[bot]

compiler/rustc_attr_parsing/src/attributes/inline.rs

@@ -93,3 +93,16 @@ impl<S: Stage> SingleAttributeParser<S> for RustcForceInlineParser {
         ))
     }
 }
+
+pub(crate) struct RustcEarlyInlineParser;
+
+impl<S: Stage> SingleAttributeParser<S> for RustcEarlyInlineParser {
+    const PATH: &'static [Symbol] = &[sym::rustc_early_inline];
+    const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepOutermost;
+    const ON_DUPLICATE: OnDuplicate<S> = OnDuplicate::WarnButFutureError;
+    const TEMPLATE: AttributeTemplate = template!(Word);
+
+    fn convert(cx: &mut AcceptContext<'_, '_, S>, _args: &ArgParser<'_>) -> Option<AttributeKind> {
+        Some(AttributeKind::Inline(InlineAttr::Early, cx.attr_span))
+    }
+}

compiler/rustc_attr_parsing/src/context.rs

@@ -23,7 +23,7 @@ use crate::attributes::codegen_attrs::{
 use crate::attributes::confusables::ConfusablesParser;
 use crate::attributes::deprecation::DeprecationParser;
 use crate::attributes::dummy::DummyParser;
-use crate::attributes::inline::{InlineParser, RustcForceInlineParser};
+use crate::attributes::inline::{InlineParser, RustcEarlyInlineParser, RustcForceInlineParser};
 use crate::attributes::link_attrs::{
     ExportStableParser, FfiConstParser, FfiPureParser, LinkNameParser, LinkOrdinalParser,
     LinkSectionParser, StdInternalSymbolParser,
@@ -169,6 +169,7 @@ attribute_parsers!(
         Single<PathAttributeParser>,
         Single<ProcMacroDeriveParser>,
         Single<RustcBuiltinMacroParser>,
+        Single<RustcEarlyInlineParser>,
         Single<RustcForceInlineParser>,
         Single<RustcLayoutScalarValidRangeEnd>,
         Single<RustcLayoutScalarValidRangeStart>,

compiler/rustc_codegen_gcc/src/attributes.rs

@@ -67,7 +67,7 @@ fn inline_attr<'gcc, 'tcx>(
                 Some(FnAttribute::AlwaysInline)
             }
         }
-        InlineAttr::Hint => Some(FnAttribute::Inline),
+        InlineAttr::Hint | InlineAttr::Early => Some(FnAttribute::Inline),
         InlineAttr::Force { .. } => Some(FnAttribute::AlwaysInline),
         InlineAttr::Never => {
             if cx.sess().target.arch != "amdgpu" {

compiler/rustc_codegen_llvm/src/attributes.rs

@@ -52,7 +52,9 @@ fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll
         return Some(AttributeKind::NoInline.create_attr(cx.llcx));
     }
     match inline {
-        InlineAttr::Hint => Some(AttributeKind::InlineHint.create_attr(cx.llcx)),
+        InlineAttr::Hint | InlineAttr::Early => {
+            Some(AttributeKind::InlineHint.create_attr(cx.llcx))
+        }
         InlineAttr::Always | InlineAttr::Force { .. } => {
             Some(AttributeKind::AlwaysInline.create_attr(cx.llcx))
         }

compiler/rustc_codegen_ssa/src/codegen_attrs.rs

@@ -473,7 +473,7 @@ fn check_result(
 
     // warn that inline has no effect when no_sanitize is present
     if !codegen_fn_attrs.no_sanitize.is_empty()
-        && codegen_fn_attrs.inline.always()
+        && codegen_fn_attrs.inline.always_in_codegen()
         && let (Some(no_sanitize_span), Some(inline_span)) =
             (interesting_spans.no_sanitize, interesting_spans.inline)
     {

compiler/rustc_feature/src/builtin_attrs.rs

@@ -1123,6 +1123,11 @@ pub static BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
         rustc_no_mir_inline, Normal, template!(Word), WarnFollowing, EncodeCrossCrate::Yes,
         "`#[rustc_no_mir_inline]` prevents the MIR inliner from inlining a function while not affecting codegen"
     ),
+    rustc_attr!(
+        rustc_early_inline, Normal, template!(Word), WarnFollowing, EncodeCrossCrate::Yes,
+        "`#[rustc_early_inline]` inlines non-generic calls to trivial methods even in debug, \
+         while still allowing them to be codegen'd for generic calls"
+    ),
     rustc_attr!(
         rustc_force_inline, Normal, template!(Word, NameValueStr: "reason"), WarnFollowing, EncodeCrossCrate::Yes,
         "`#[rustc_force_inline]` forces a free function to be inlined"

compiler/rustc_hir/src/attrs/data_structures.rs

@@ -24,13 +24,20 @@ pub enum InlineAttr {
         attr_span: Span,
         reason: Option<Symbol>,
     },
+    /// `#[rustc_early_inline]` will always inline calls to a known impl in MIR.
+    ///
+    /// You can think of this as either
+    /// - Force, but without the "do not codegen as a function ever" restriction.
+    /// - Always, but only for MIR.
+    Early,
 }
 
 impl InlineAttr {
-    pub fn always(&self) -> bool {
+    pub fn always_in_codegen(&self) -> bool {
         match self {
-            InlineAttr::Always | InlineAttr::Force { .. } => true,
-            InlineAttr::None | InlineAttr::Hint | InlineAttr::Never => false,
+            InlineAttr::Always => true,
+            InlineAttr::None | InlineAttr::Hint | InlineAttr::Early | InlineAttr::Never => false,
+            InlineAttr::Force { .. } => panic!("Shouldn't be codegen'ing {self:?}"),
         }
     }
 }
@@ -342,7 +349,7 @@ pub enum AttributeKind {
         reason: Option<Symbol>,
     },
 
-    /// Represents `#[inline]` and `#[rustc_force_inline]`.
+    /// Represents `#[inline]` and `#[rustc_force_inline]` and `#[rustc_early_inline]`.
     Inline(InlineAttr, Span),
 
     /// Represents `#[link_name]`.

compiler/rustc_middle/src/mir/mono.rs

@@ -205,7 +205,7 @@ impl<'tcx> MonoItem<'tcx> {
 
         // To ensure that #[inline(always)] can be inlined as much as possible, especially in unoptimized
         // builds, we always select LocalCopy.
-        if codegen_fn_attrs.inline.always() {
+        if codegen_fn_attrs.inline.always_in_codegen() {
             return InstantiationMode::LocalCopy;
         }
 

compiler/rustc_mir_transform/src/cross_crate_inline.rs

@@ -46,8 +46,10 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
     // #[inline(never)] to force code generation.
     match codegen_fn_attrs.inline {
         InlineAttr::Never => return false,
-        InlineAttr::Hint | InlineAttr::Always | InlineAttr::Force { .. } => return true,
-        _ => {}
+        InlineAttr::Hint | InlineAttr::Always | InlineAttr::Early | InlineAttr::Force { .. } => {
+            return true;
+        }
+        InlineAttr::None => {}
     }
 
     // If the crate is likely to be mostly unused, use cross-crate inlining to defer codegen until

compiler/rustc_mir_transform/src/inline.rs

@@ -77,7 +77,7 @@ pub struct ForceInline;
 
 impl ForceInline {
     pub fn should_run_pass_for_callee<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> bool {
-        matches!(tcx.codegen_fn_attrs(def_id).inline, InlineAttr::Force { .. })
+        matches!(tcx.codegen_fn_attrs(def_id).inline, InlineAttr::Force { .. } | InlineAttr::Early)
     }
 }
 
@@ -195,7 +195,7 @@ impl<'tcx> Inliner<'tcx> for ForceInliner<'tcx> {
         &self,
         callee_attrs: &CodegenFnAttrs,
     ) -> Result<(), &'static str> {
-        debug_assert_matches!(callee_attrs.inline, InlineAttr::Force { .. });
+        debug_assert_matches!(callee_attrs.inline, InlineAttr::Force { .. } | InlineAttr::Early);
         Ok(())
     }
 
@@ -247,23 +247,26 @@ impl<'tcx> Inliner<'tcx> for ForceInliner<'tcx> {
 
     fn on_inline_failure(&self, callsite: &CallSite<'tcx>, reason: &'static str) {
         let tcx = self.tcx();
-        let InlineAttr::Force { attr_span, reason: justification } =
-            tcx.codegen_fn_attrs(callsite.callee.def_id()).inline
-        else {
-            bug!("called on item without required inlining");
-        };
-
-        let call_span = callsite.source_info.span;
-        tcx.dcx().emit_err(crate::errors::ForceInlineFailure {
-            call_span,
-            attr_span,
-            caller_span: tcx.def_span(self.def_id),
-            caller: tcx.def_path_str(self.def_id),
-            callee_span: tcx.def_span(callsite.callee.def_id()),
-            callee: tcx.def_path_str(callsite.callee.def_id()),
-            reason,
-            justification: justification.map(|sym| crate::errors::ForceInlineJustification { sym }),
-        });
+        match tcx.codegen_fn_attrs(callsite.callee.def_id()).inline {
+            InlineAttr::Early => {
+                // Ok, we don't actually mind if this fails.
+            }
+            InlineAttr::Force { attr_span, reason: justification } => {
+                let call_span = callsite.source_info.span;
+                tcx.dcx().emit_err(crate::errors::ForceInlineFailure {
+                    call_span,
+                    attr_span,
+                    caller_span: tcx.def_span(self.def_id),
+                    caller: tcx.def_path_str(self.def_id),
+                    callee_span: tcx.def_span(callsite.callee.def_id()),
+                    callee: tcx.def_path_str(callsite.callee.def_id()),
+                    reason,
+                    justification: justification
+                        .map(|sym| crate::errors::ForceInlineJustification { sym }),
+                });
+            }
+            _ => bug!("called on item without required inlining"),
+        }
     }
 }