Always consider inlining for trivial functions

Author: saethlin

compiler/rustc_mir_transform/src/inline.rs

@@ -10,7 +10,6 @@ use rustc_middle::mir::visit::*;
 use rustc_middle::mir::*;
 use rustc_middle::ty::TypeVisitableExt;
 use rustc_middle::ty::{self, Instance, InstanceDef, ParamEnv, Ty, TyCtxt};
-use rustc_session::config::OptLevel;
 use rustc_target::abi::FieldIdx;
 use rustc_target::spec::abi::Abi;
 
@@ -47,12 +46,8 @@ impl<'tcx> MirPass<'tcx> for Inline {
         }
 
         match sess.mir_opt_level() {
-            0 | 1 => false,
-            2 => {
-                (sess.opts.optimize == OptLevel::Default
-                    || sess.opts.optimize == OptLevel::Aggressive)
-                    && sess.opts.incremental == None
-            }
+            0 => false,
+            1 | 2 => sess.opts.incremental.is_none(),
             _ => true,
         }
     }
@@ -471,7 +466,11 @@ impl<'tcx> Inliner<'tcx> {
         if callee_body.basic_blocks.len() <= 3 {
             threshold += threshold / 4;
         }
-        debug!("    final inline threshold = {}", threshold);
+
+        if tcx.sess.mir_opt_level() == 1 {
+            threshold = 2 * INSTR_COST;
+        }
+        debug!("final inline threshold = {}", threshold);
 
         // FIXME: Give a bonus to functions with only a single caller
 
@@ -481,6 +480,7 @@ impl<'tcx> Inliner<'tcx> {
             instance: callsite.callee,
             callee_body,
             cost: 0,
+            threshold,
         };
 
         // Traverse the MIR manually so we can account for the effects of inlining on the CFG.
@@ -493,6 +493,10 @@ impl<'tcx> Inliner<'tcx> {
 
             let blk = &callee_body.basic_blocks[bb];
             checker.visit_basic_block_data(bb, blk);
+            if callee_attrs.inline != InlineAttr::Always && checker.cost > threshold {
+                debug!("checker cost exceeded threshold partway through");
+                return Err("cost above threshold");
+            }
 
             let term = blk.terminator();
             if let TerminatorKind::Drop { ref place, target, unwind, replace: _ } = term.kind {
@@ -805,12 +809,17 @@ struct CostChecker<'b, 'tcx> {
     tcx: TyCtxt<'tcx>,
     param_env: ParamEnv<'tcx>,
     cost: usize,
+    threshold: usize,
     callee_body: &'b Body<'tcx>,
     instance: ty::Instance<'tcx>,
 }
 
 impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> {
     fn visit_statement(&mut self, statement: &Statement<'tcx>, _: Location) {
+        if self.cost > self.threshold {
+            return;
+        }
+
         // Don't count StorageLive/StorageDead in the inlining cost.
         match statement.kind {
             StatementKind::StorageLive(_)

tests/codegen-units/polymorphization/unused_type_parameters.rs

@@ -1,4 +1,4 @@
-// compile-flags:-Zpolymorphize=on -Zprint-mono-items=lazy -Copt-level=1
+// compile-flags:-Zpolymorphize=on -Zprint-mono-items=lazy -Copt-level=1 -Zmir-opt-level=0
 
 #![crate_type = "rlib"]
 

tests/debuginfo/function-names.rs

@@ -1,7 +1,7 @@
 // Function names are formatted differently in old versions of GDB
 // min-gdb-version: 10.1
 
-// compile-flags:-g
+// compile-flags: -g -Zmir-opt-level=0
 
 // === GDB TESTS ===================================================================================
 

tests/mir-opt/inline/trivial_fn_debug_build.rs

@@ -0,0 +1,22 @@
+// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
+// compile-flags: -Copt-level=0 -Zmir-opt-level=1
+#![crate_type = "lib"]
+
+// Test that we still inline trivial functions even in a debug build
+
+pub struct Thing {
+    inner: u8,
+}
+
+impl Thing {
+    #[inline]
+    pub fn get(&self) -> u8 {
+        self.inner
+    }
+}
+
+// EMIT_MIR trivial_fn_debug_build.wrapper.Inline.diff
+// EMIT_MIR trivial_fn_debug_build.wrapper.PreCodegen.after.mir
+pub fn wrapper(t: &Thing) -> u8 {
+    t.get()
+}

tests/mir-opt/inline/trivial_fn_debug_build.wrapper.Inline.panic-abort.diff

@@ -0,0 +1,24 @@
+- // MIR for `wrapper` before Inline
++ // MIR for `wrapper` after Inline
+  
+  fn wrapper(_1: &Thing) -> u8 {
+      debug t => _1;
+      let mut _0: u8;
+      let mut _2: &Thing;
++     scope 1 (inlined Thing::get) {
++         debug self => _2;
++     }
+  
+      bb0: {
+          StorageLive(_2);
+          _2 = &(*_1);
+-         _0 = Thing::get(move _2) -> [return: bb1, unwind unreachable];
+-     }
+- 
+-     bb1: {
++         _0 = ((*_2).0: u8);
+          StorageDead(_2);
+          return;
+      }
+  }
+  

tests/mir-opt/inline/trivial_fn_debug_build.wrapper.Inline.panic-unwind.diff

@@ -0,0 +1,24 @@
+- // MIR for `wrapper` before Inline
++ // MIR for `wrapper` after Inline
+  
+  fn wrapper(_1: &Thing) -> u8 {
+      debug t => _1;
+      let mut _0: u8;
+      let mut _2: &Thing;
++     scope 1 (inlined Thing::get) {
++         debug self => _2;
++     }
+  
+      bb0: {
+          StorageLive(_2);
+          _2 = &(*_1);
+-         _0 = Thing::get(move _2) -> [return: bb1, unwind continue];
+-     }
+- 
+-     bb1: {
++         _0 = ((*_2).0: u8);
+          StorageDead(_2);
+          return;
+      }
+  }
+  

tests/mir-opt/inline/trivial_fn_debug_build.wrapper.PreCodegen.after.panic-abort.mir

@@ -0,0 +1,14 @@
+// MIR for `wrapper` after PreCodegen
+
+fn wrapper(_1: &Thing) -> u8 {
+    debug t => _1;
+    let mut _0: u8;
+    scope 1 (inlined Thing::get) {
+        debug self => _1;
+    }
+
+    bb0: {
+        _0 = ((*_1).0: u8);
+        return;
+    }
+}

tests/mir-opt/inline/trivial_fn_debug_build.wrapper.PreCodegen.after.panic-unwind.mir

@@ -0,0 +1,14 @@
+// MIR for `wrapper` after PreCodegen
+
+fn wrapper(_1: &Thing) -> u8 {
+    debug t => _1;
+    let mut _0: u8;
+    scope 1 (inlined Thing::get) {
+        debug self => _1;
+    }
+
+    bb0: {
+        _0 = ((*_1).0: u8);
+        return;
+    }
+}

tests/ui-fulldeps/stable-mir/crate-info.rs

@@ -59,7 +59,7 @@ fn test_stable_mir(_tcx: TyCtxt<'_>) -> ControlFlow<()> {
     let foo_bar = get_item(&items, (DefKind::Fn, "foo_bar")).unwrap();
     let body = foo_bar.body();
     assert_eq!(body.locals.len(), 5);
-    assert_eq!(body.blocks.len(), 4);
+    assert_eq!(body.blocks.len(), 3);
     let block = &body.blocks[0];
     match &block.terminator.kind {
         stable_mir::mir::TerminatorKind::Call { .. } => {}

tests/ui/consts/const-eval/index-out-of-bounds-never-type.rs

@@ -1,4 +1,5 @@
 // build-fail
+// compile-flags: -Zmir-opt-level=0
 
 // Regression test for #66975
 #![warn(unconditional_panic)]