Enable MIR inlining of very small functions in debug builds

Author: saethlin

compiler/rustc_mir_transform/src/inline.rs

@@ -10,7 +10,6 @@ use rustc_middle::mir::visit::*;
 use rustc_middle::mir::*;
 use rustc_middle::ty::TypeVisitableExt;
 use rustc_middle::ty::{self, Instance, InstanceDef, ParamEnv, Ty, TyCtxt};
-use rustc_session::config::OptLevel;
 use rustc_target::abi::FieldIdx;
 use rustc_target::spec::abi::Abi;
 
@@ -47,12 +46,8 @@ impl<'tcx> MirPass<'tcx> for Inline {
         }
 
         match sess.mir_opt_level() {
-            0 | 1 => false,
-            2 => {
-                (sess.opts.optimize == OptLevel::Default
-                    || sess.opts.optimize == OptLevel::Aggressive)
-                    && sess.opts.incremental == None
-            }
+            0 => false,
+            1 | 2 => sess.opts.incremental.is_none(),
             _ => true,
         }
     }
@@ -70,6 +65,14 @@ impl<'tcx> MirPass<'tcx> for Inline {
 }
 
 fn inline<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool {
+    let has_call = body
+        .basic_blocks
+        .iter()
+        .any(|b| matches!(b.terminator().kind, TerminatorKind::Call { .. }));
+    if !has_call {
+        return false;
+    }
+
     let def_id = body.source.def_id().expect_local();
 
     // Only do inlining into fn bodies.
@@ -475,7 +478,16 @@ impl<'tcx> Inliner<'tcx> {
         if callee_body.basic_blocks.len() <= 3 {
             threshold += threshold / 4;
         }
-        debug!("    final inline threshold = {}", threshold);
+
+        // At MIR opt level 1 we only want to do a tiny bit of inlining.
+        // 2 * INSTR_COST is the cost of one statement plus a return terminator.
+        if tcx.sess.mir_opt_level() == 1 {
+            threshold = 2 * INSTR_COST;
+            if callee_body.basic_blocks.len() > 1 {
+                return Err("cost above threshold");
+            }
+        }
+        debug!("final inline threshold = {}", threshold);
 
         // FIXME: Give a bonus to functions with only a single caller
 

tests/codegen-units/polymorphization/unused_type_parameters.rs

@@ -1,4 +1,4 @@
-// compile-flags:-Zpolymorphize=on -Zprint-mono-items=lazy -Copt-level=1
+// compile-flags:-Zpolymorphize=on -Zprint-mono-items=lazy -Copt-level=1 -Zmir-opt-level=0
 
 #![crate_type = "rlib"]
 

tests/coverage-map/status-quo/async.cov-map

@@ -283,22 +283,19 @@ Number of file 0 mappings: 5
 - Code(Zero) at (prev + 2, 1) to (start + 0, 2)
 
 Function name: async::l
-Raw bytes (37): 0x[01, 01, 04, 01, 07, 09, 05, 09, 0f, 05, 02, 05, 01, 53, 01, 01, 0c, 02, 02, 0e, 00, 10, 05, 01, 0e, 00, 10, 09, 01, 0e, 00, 10, 0b, 02, 01, 00, 02]
+Raw bytes (33): 0x[01, 01, 02, 09, 07, 05, 00, 05, 01, 53, 01, 01, 0c, 00, 02, 0e, 00, 10, 00, 01, 0e, 00, 10, 09, 01, 0e, 00, 10, 03, 02, 01, 00, 02]
 Number of files: 1
 - file 0 => global file 1
-Number of expressions: 4
-- expression 0 operands: lhs = Counter(0), rhs = Expression(1, Add)
-- expression 1 operands: lhs = Counter(2), rhs = Counter(1)
-- expression 2 operands: lhs = Counter(2), rhs = Expression(3, Add)
-- expression 3 operands: lhs = Counter(1), rhs = Expression(0, Sub)
+Number of expressions: 2
+- expression 0 operands: lhs = Counter(2), rhs = Expression(1, Add)
+- expression 1 operands: lhs = Counter(1), rhs = Zero
 Number of file 0 mappings: 5
 - Code(Counter(0)) at (prev + 83, 1) to (start + 1, 12)
-- Code(Expression(0, Sub)) at (prev + 2, 14) to (start + 0, 16)
-    = (c0 - (c2 + c1))
-- Code(Counter(1)) at (prev + 1, 14) to (start + 0, 16)
+- Code(Zero) at (prev + 2, 14) to (start + 0, 16)
+- Code(Zero) at (prev + 1, 14) to (start + 0, 16)
 - Code(Counter(2)) at (prev + 1, 14) to (start + 0, 16)
-- Code(Expression(2, Add)) at (prev + 2, 1) to (start + 0, 2)
-    = (c2 + (c1 + (c0 - (c2 + c1))))
+- Code(Expression(0, Add)) at (prev + 2, 1) to (start + 0, 2)
+    = (c2 + (c1 + Zero))
 
 Function name: async::m
 Raw bytes (9): 0x[01, 01, 00, 01, 01, 5b, 01, 00, 19]

tests/debuginfo/function-names.rs

@@ -1,7 +1,7 @@
 // Function names are formatted differently in old versions of GDB
 // min-gdb-version: 10.1
 
-// compile-flags:-g
+// compile-flags: -g -Zmir-opt-level=0
 
 // === GDB TESTS ===================================================================================
 

tests/mir-opt/inline/trivial_fn_debug_build.rs

@@ -0,0 +1,22 @@
+// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
+// compile-flags: -Copt-level=0 -Zmir-opt-level=1
+#![crate_type = "lib"]
+
+// Test that we still inline trivial functions even in a debug build
+
+pub struct Thing {
+    inner: u8,
+}
+
+impl Thing {
+    #[inline]
+    pub fn get(&self) -> u8 {
+        self.inner
+    }
+}
+
+// EMIT_MIR trivial_fn_debug_build.wrapper.Inline.diff
+// EMIT_MIR trivial_fn_debug_build.wrapper.PreCodegen.after.mir
+pub fn wrapper(t: &Thing) -> u8 {
+    t.get()
+}

tests/mir-opt/inline/trivial_fn_debug_build.wrapper.Inline.panic-abort.diff

@@ -0,0 +1,24 @@
+- // MIR for `wrapper` before Inline
++ // MIR for `wrapper` after Inline
+  
+  fn wrapper(_1: &Thing) -> u8 {
+      debug t => _1;
+      let mut _0: u8;
+      let mut _2: &Thing;
++     scope 1 (inlined Thing::get) {
++         debug self => _2;
++     }
+  
+      bb0: {
+          StorageLive(_2);
+          _2 = &(*_1);
+-         _0 = Thing::get(move _2) -> [return: bb1, unwind unreachable];
+-     }
+- 
+-     bb1: {
++         _0 = ((*_2).0: u8);
+          StorageDead(_2);
+          return;
+      }
+  }
+  

tests/mir-opt/inline/trivial_fn_debug_build.wrapper.Inline.panic-unwind.diff

@@ -0,0 +1,24 @@
+- // MIR for `wrapper` before Inline
++ // MIR for `wrapper` after Inline
+  
+  fn wrapper(_1: &Thing) -> u8 {
+      debug t => _1;
+      let mut _0: u8;
+      let mut _2: &Thing;
++     scope 1 (inlined Thing::get) {
++         debug self => _2;
++     }
+  
+      bb0: {
+          StorageLive(_2);
+          _2 = &(*_1);
+-         _0 = Thing::get(move _2) -> [return: bb1, unwind continue];
+-     }
+- 
+-     bb1: {
++         _0 = ((*_2).0: u8);
+          StorageDead(_2);
+          return;
+      }
+  }
+  

tests/mir-opt/inline/trivial_fn_debug_build.wrapper.PreCodegen.after.panic-abort.mir

@@ -0,0 +1,14 @@
+// MIR for `wrapper` after PreCodegen
+
+fn wrapper(_1: &Thing) -> u8 {
+    debug t => _1;
+    let mut _0: u8;
+    scope 1 (inlined Thing::get) {
+        debug self => _1;
+    }
+
+    bb0: {
+        _0 = ((*_1).0: u8);
+        return;
+    }
+}

tests/mir-opt/inline/trivial_fn_debug_build.wrapper.PreCodegen.after.panic-unwind.mir

@@ -0,0 +1,14 @@
+// MIR for `wrapper` after PreCodegen
+
+fn wrapper(_1: &Thing) -> u8 {
+    debug t => _1;
+    let mut _0: u8;
+    scope 1 (inlined Thing::get) {
+        debug self => _1;
+    }
+
+    bb0: {
+        _0 = ((*_1).0: u8);
+        return;
+    }
+}

tests/run-make/sepcomp-cci-copies/Makefile

@@ -8,5 +8,5 @@ include ../tools.mk
 all:
 	$(RUSTC) cci_lib.rs
 	$(RUSTC) foo.rs --emit=llvm-ir -C codegen-units=6 \
-		-Z inline-in-all-cgus
+		-Z inline-in-all-cgus -Zmir-opt-level=0
 	[ "$$(cat "$(TMPDIR)"/foo.*.ll | grep -c define\ .*cci_fn)" -eq "2" ]