Use optimized_mir_summary.

Author: cjgillot

compiler/rustc_middle/src/ty/mod.rs

@@ -17,7 +17,7 @@ pub use self::IntVarValue::*;
 pub use self::Variance::*;
 use crate::metadata::ModChild;
 use crate::middle::privacy::AccessLevels;
-use crate::mir::{Body, GeneratorLayout};
+use crate::mir::{self, Body, GeneratorLayout};
 use crate::traits::{self, Reveal};
 use crate::ty;
 use crate::ty::fast_reject::SimplifiedType;
@@ -2160,6 +2160,38 @@ impl<'tcx> TyCtxt<'tcx> {
         }
     }
 
+    /// Returns the possibly-auto-generated MIR of a `(DefId, Subst)` pair.
+    pub fn instance_mir_summary(self, instance: ty::InstanceDef<'tcx>) -> mir::Summary {
+        match instance {
+            ty::InstanceDef::Item(def) => match self.def_kind(def.did) {
+                DefKind::Const
+                | DefKind::Static(..)
+                | DefKind::AssocConst
+                | DefKind::Ctor(..)
+                | DefKind::AnonConst
+                | DefKind::InlineConst => {
+                    mir::Summary { inlining_cost: 0, bbcount: 0, diverges: false }
+                }
+                // If the caller wants `mir_for_ctfe` of a function they should not be using
+                // `instance_mir`, so we'll assume const fn also wants the optimized version.
+                _ => {
+                    assert_eq!(def.const_param_did, None);
+                    self.optimized_mir_summary(def.did)
+                }
+            },
+            ty::InstanceDef::VTableShim(..)
+            | ty::InstanceDef::ReifyShim(..)
+            | ty::InstanceDef::Intrinsic(..)
+            | ty::InstanceDef::FnPtrShim(..)
+            | ty::InstanceDef::Virtual(..)
+            | ty::InstanceDef::ClosureOnceShim { .. }
+            | ty::InstanceDef::DropGlue(..)
+            | ty::InstanceDef::CloneShim(..) => {
+                mir::Summary { inlining_cost: 0, bbcount: 0, diverges: false }
+            }
+        }
+    }
+
     // FIXME(@lcnr): Remove this function.
     pub fn get_attrs_unchecked(self, did: DefId) -> &'tcx [ast::Attribute] {
         if let Some(did) = did.as_local() {

compiler/rustc_mir_transform/src/inline.rs

@@ -151,8 +151,7 @@ impl<'tcx> Inliner<'tcx> {
         let callee_attrs = self.tcx.codegen_fn_attrs(callsite.callee.def_id());
         self.check_codegen_attributes(callsite, callee_attrs)?;
         self.check_mir_is_available(caller_body, &callsite.callee)?;
-        let callee_body = self.tcx.instance_mir(callsite.callee.def);
-        self.check_mir_body(callsite, callee_body, callee_attrs)?;
+        let callee_body = self.check_mir_body(callsite, callee_attrs)?;
 
         if !self.tcx.consider_optimizing(|| {
             format!("Inline {:?} into {:?}", callsite.callee, caller_body.source)
@@ -385,43 +384,54 @@ impl<'tcx> Inliner<'tcx> {
 
     /// Returns inlining decision that is based on the examination of callee MIR body.
     /// Assumes that codegen attributes have been checked for compatibility already.
-    #[instrument(level = "debug", skip(self, callee_body))]
+    #[instrument(level = "debug", skip(self))]
     fn check_mir_body(
         &self,
         callsite: &CallSite<'tcx>,
-        callee_body: &Body<'tcx>,
         callee_attrs: &CodegenFnAttrs,
-    ) -> Result<(), &'static str> {
-        let cost_info = body_cost(self.tcx, self.param_env, callee_body, |ty| {
-            callsite.callee.subst_mir(self.tcx, &ty)
-        });
+    ) -> Result<&'tcx Body<'tcx>, &'static str> {
+        if let InlineAttr::Always = callee_attrs.inline {
+            debug!("INLINING {:?} because inline(always)", callsite,);
+            let callee_body = self.tcx.instance_mir(callsite.callee.def);
+            return Ok(callee_body);
+        }
 
         let mut threshold = if callee_attrs.requests_inline() {
             self.tcx.sess.opts.unstable_opts.inline_mir_hint_threshold.unwrap_or(100)
         } else {
             self.tcx.sess.opts.unstable_opts.inline_mir_threshold.unwrap_or(50)
         };
 
+        let callee_summary = self.tcx.instance_mir_summary(callsite.callee.def);
         // Give a bonus functions with a small number of blocks,
         // We normally have two or three blocks for even
         // very small functions.
-        if cost_info.bbcount <= 3 {
+        if callee_summary.bbcount <= 3 {
             threshold += threshold / 4;
         }
+        if callee_summary.diverges {
+            threshold = 0;
+        }
         debug!("    final inline threshold = {}", threshold);
 
-        if cost_info.diverges {
-            threshold = 0;
+        // Fast reject based on unsubstituted MIR.
+        if callee_summary.inlining_cost >= 2 * threshold {
+            return Err("summary cost above threshold");
         }
 
+        let callee_body = self.tcx.instance_mir(callsite.callee.def);
+        let cost_info = body_cost(self.tcx, self.param_env, callee_body, |ty| {
+            callsite.callee.subst_mir(self.tcx, &ty)
+        });
+
         let cost = cost_info.cost;
 
         if let InlineAttr::Always = callee_attrs.inline {
             debug!("INLINING {:?} because inline(always) [cost={}]", callsite, cost);
-            Ok(())
+            Ok(callee_body)
         } else if cost <= threshold {
             debug!("INLINING {:?} [cost={} <= threshold={}]", callsite, cost, threshold);
-            Ok(())
+            Ok(callee_body)
         } else {
             debug!("NOT inlining {:?} [cost={} > threshold={}]", callsite, cost, threshold);
             Err("cost above threshold")