Skip to content

Commit 591a39a

Browse files
Dylan-DPCDylan-DPC
authored
Rollup merge of rust-lang#78966 - tmiasko:inline-never, r=oli-obk
Never inline C variadics, cold functions, functions with incompatible attributes ... ... and fix generator inlining. Closes rust-lang#67863. Closes rust-lang#78859.
2 parents 09cafd7 + 2a010dd commit 591a39a

9 files changed

+238
-61
lines changed

‎compiler/rustc_mir/src/transform/inline.rs‎

Lines changed: 19 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ use rustc_index::vec::Idx;
77
use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs};
88
use rustc_middle::mir::visit::*;
99
use rustc_middle::mir::*;
10+
use rustc_middle::ty::subst::Subst;
1011
use rustc_middle::ty::{self, ConstKind, Instance, InstanceDef, ParamEnv, Ty, TyCtxt};
1112
use rustc_span::{hygiene::ExpnKind, ExpnData, Span};
1213
use rustc_target::spec::abi::Abi;
@@ -28,6 +29,7 @@ pub struct Inline;
2829
#[derive(Copy, Clone, Debug)]
2930
struct CallSite<'tcx> {
3031
callee: Instance<'tcx>,
32+
fn_sig: ty::PolyFnSig<'tcx>,
3133
block: BasicBlock,
3234
target: Option<BasicBlock>,
3335
source_info: SourceInfo,
@@ -173,22 +175,23 @@ impl Inliner<'tcx> {
173175

174176
// Only consider direct calls to functions
175177
let terminator = bb_data.terminator();
176-
if let TerminatorKind::Call { func: ref op, ref destination, .. } = terminator.kind {
177-
if let ty::FnDef(callee_def_id, substs) = *op.ty(caller_body, self.tcx).kind() {
178-
// To resolve an instance its substs have to be fully normalized, so
179-
// we do this here.
180-
let normalized_substs = self.tcx.normalize_erasing_regions(self.param_env, substs);
178+
if let TerminatorKind::Call { ref func, ref destination, .. } = terminator.kind {
179+
let func_ty = func.ty(caller_body, self.tcx);
180+
if let ty::FnDef(def_id, substs) = *func_ty.kind() {
181+
// To resolve an instance its substs have to be fully normalized.
182+
let substs = self.tcx.normalize_erasing_regions(self.param_env, substs);
181183
let callee =
182-
Instance::resolve(self.tcx, self.param_env, callee_def_id, normalized_substs)
183-
.ok()
184-
.flatten()?;
184+
Instance::resolve(self.tcx, self.param_env, def_id, substs).ok().flatten()?;
185185

186186
if let InstanceDef::Virtual(..) | InstanceDef::Intrinsic(_) = callee.def {
187187
return None;
188188
}
189189

190+
let fn_sig = self.tcx.fn_sig(def_id).subst(self.tcx, substs);
191+
190192
return Some(CallSite {
191193
callee,
194+
fn_sig,
192195
block: bb,
193196
target: destination.map(|(_, target)| target),
194197
source_info: terminator.source_info,
@@ -203,9 +206,8 @@ impl Inliner<'tcx> {
203206
debug!("should_inline({:?})", callsite);
204207
let tcx = self.tcx;
205208

206-
// Cannot inline generators which haven't been transformed yet
207-
if callee_body.yield_ty.is_some() {
208-
debug!(" yield ty present - not inlining");
209+
if callsite.fn_sig.c_variadic() {
210+
debug!("callee is variadic - not inlining");
209211
return false;
210212
}
211213

@@ -218,11 +220,7 @@ impl Inliner<'tcx> {
218220
return false;
219221
}
220222

221-
let self_no_sanitize =
222-
self.codegen_fn_attrs.no_sanitize & self.tcx.sess.opts.debugging_opts.sanitizer;
223-
let callee_no_sanitize =
224-
codegen_fn_attrs.no_sanitize & self.tcx.sess.opts.debugging_opts.sanitizer;
225-
if self_no_sanitize != callee_no_sanitize {
223+
if self.codegen_fn_attrs.no_sanitize != codegen_fn_attrs.no_sanitize {
226224
debug!("`callee has incompatible no_sanitize attribute - not inlining");
227225
return false;
228226
}
@@ -256,9 +254,9 @@ impl Inliner<'tcx> {
256254
self.tcx.sess.opts.debugging_opts.inline_mir_threshold
257255
};
258256

259-
// Significantly lower the threshold for inlining cold functions
260257
if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::COLD) {
261-
threshold /= 5;
258+
debug!("#[cold] present - not inlining");
259+
return false;
262260
}
263261

264262
// Give a bonus functions with a small number of blocks,
@@ -447,7 +445,7 @@ impl Inliner<'tcx> {
447445
};
448446

449447
// Copy the arguments if needed.
450-
let args: Vec<_> = self.make_call_args(args, &callsite, caller_body);
448+
let args: Vec<_> = self.make_call_args(args, &callsite, caller_body, &callee_body);
451449

452450
let mut integrator = Integrator {
453451
args: &args,
@@ -528,6 +526,7 @@ impl Inliner<'tcx> {
528526
args: Vec<Operand<'tcx>>,
529527
callsite: &CallSite<'tcx>,
530528
caller_body: &mut Body<'tcx>,
529+
callee_body: &Body<'tcx>,
531530
) -> Vec<Local> {
532531
let tcx = self.tcx;
533532

@@ -554,9 +553,7 @@ impl Inliner<'tcx> {
554553
// tmp2 = tuple_tmp.2
555554
//
556555
// and the vector is `[closure_ref, tmp0, tmp1, tmp2]`.
557-
// FIXME(eddyb) make this check for `"rust-call"` ABI combined with
558-
// `callee_body.spread_arg == None`, instead of special-casing closures.
559-
if tcx.is_closure(callsite.callee.def_id()) {
556+
if callsite.fn_sig.abi() == Abi::RustCall && callee_body.spread_arg.is_none() {
560557
let mut args = args.into_iter();
561558
let self_ = self.create_temp_if_necessary(args.next().unwrap(), callsite, caller_body);
562559
let tuple = self.create_temp_if_necessary(args.next().unwrap(), callsite, caller_body);

‎src/test/mir-opt/inline/inline-compatibility.rs‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,11 @@
11
// Checks that only functions with compatible attributes are inlined.
22
//
33
// only-x86_64
4-
// needs-sanitizer-address
5-
// compile-flags: -Zsanitizer=address
64

75
#![crate_type = "lib"]
86
#![feature(no_sanitize)]
97
#![feature(target_feature_11)]
8+
#![feature(c_variadic)]
109

1110
// EMIT_MIR inline_compatibility.inlined_target_feature.Inline.diff
1211
#[target_feature(enable = "sse2")]
@@ -35,5 +34,22 @@ pub unsafe fn not_inlined_no_sanitize() {
3534
pub unsafe fn target_feature() {}
3635

3736
#[inline]
38-
#[no_sanitize(address, memory)]
37+
#[no_sanitize(address)]
3938
pub unsafe fn no_sanitize() {}
39+
40+
// EMIT_MIR inline_compatibility.not_inlined_c_variadic.Inline.diff
41+
pub unsafe fn not_inlined_c_variadic() {
42+
let s = sum(4u32, 4u32, 30u32, 200u32, 1000u32);
43+
}
44+
45+
#[no_mangle]
46+
#[inline(always)]
47+
unsafe extern "C" fn sum(n: u32, mut vs: ...) -> u32 {
48+
let mut s = 0;
49+
let mut i = 0;
50+
while i != n {
51+
s += vs.arg::<u32>();
52+
i += 1;
53+
}
54+
s
55+
}

‎src/test/mir-opt/inline/inline-generator.rs‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
// ignore-wasm32-bare compiled with panic=abort by default
2+
#![feature(generators, generator_trait)]
3+
4+
use std::ops::Generator;
5+
use std::pin::Pin;
6+
7+
// EMIT_MIR inline_generator.main.Inline.diff
8+
fn main() {
9+
let _r = Pin::new(&mut g()).resume(false);
10+
}
11+
12+
#[inline(always)]
13+
pub fn g() -> impl Generator<bool> {
14+
#[inline(always)]
15+
|a| { yield if a { 7 } else { 13 } }
16+
}

‎src/test/mir-opt/inline/inline_compatibility.inlined_no_sanitize.Inline.diff‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,24 +2,24 @@
22
+ // MIR for `inlined_no_sanitize` after Inline
33

44
fn inlined_no_sanitize() -> () {
5-
let mut _0: (); // return place in scope 0 at $DIR/inline-compatibility.rs:24:37: 24:37
6-
let _1: (); // in scope 0 at $DIR/inline-compatibility.rs:25:5: 25:18
7-
+ scope 1 (inlined no_sanitize) { // at $DIR/inline-compatibility.rs:25:5: 25:18
5+
let mut _0: (); // return place in scope 0 at $DIR/inline-compatibility.rs:23:37: 23:37
6+
let _1: (); // in scope 0 at $DIR/inline-compatibility.rs:24:5: 24:18
7+
+ scope 1 (inlined no_sanitize) { // at $DIR/inline-compatibility.rs:24:5: 24:18
88
+ }
99

1010
bb0: {
11-
StorageLive(_1); // scope 0 at $DIR/inline-compatibility.rs:25:5: 25:18
12-
- _1 = no_sanitize() -> bb1; // scope 0 at $DIR/inline-compatibility.rs:25:5: 25:18
11+
StorageLive(_1); // scope 0 at $DIR/inline-compatibility.rs:24:5: 24:18
12+
- _1 = no_sanitize() -> bb1; // scope 0 at $DIR/inline-compatibility.rs:24:5: 24:18
1313
- // mir::Constant
14-
- // + span: $DIR/inline-compatibility.rs:25:5: 25:16
14+
- // + span: $DIR/inline-compatibility.rs:24:5: 24:16
1515
- // + literal: Const { ty: unsafe fn() {no_sanitize}, val: Value(Scalar(<ZST>)) }
1616
- }
1717
-
1818
- bb1: {
19-
+ _1 = const (); // scope 1 at $DIR/inline-compatibility.rs:25:5: 25:18
20-
StorageDead(_1); // scope 0 at $DIR/inline-compatibility.rs:25:18: 25:19
21-
_0 = const (); // scope 0 at $DIR/inline-compatibility.rs:24:37: 26:2
22-
return; // scope 0 at $DIR/inline-compatibility.rs:26:2: 26:2
19+
+ _1 = const (); // scope 1 at $DIR/inline-compatibility.rs:24:5: 24:18
20+
StorageDead(_1); // scope 0 at $DIR/inline-compatibility.rs:24:18: 24:19
21+
_0 = const (); // scope 0 at $DIR/inline-compatibility.rs:23:37: 25:2
22+
return; // scope 0 at $DIR/inline-compatibility.rs:25:2: 25:2
2323
}
2424
}
2525

‎src/test/mir-opt/inline/inline_compatibility.inlined_target_feature.Inline.diff‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,24 +2,24 @@
22
+ // MIR for `inlined_target_feature` after Inline
33

44
fn inlined_target_feature() -> () {
5-
let mut _0: (); // return place in scope 0 at $DIR/inline-compatibility.rs:13:40: 13:40
6-
let _1: (); // in scope 0 at $DIR/inline-compatibility.rs:14:5: 14:21
7-
+ scope 1 (inlined target_feature) { // at $DIR/inline-compatibility.rs:14:5: 14:21
5+
let mut _0: (); // return place in scope 0 at $DIR/inline-compatibility.rs:12:40: 12:40
6+
let _1: (); // in scope 0 at $DIR/inline-compatibility.rs:13:5: 13:21
7+
+ scope 1 (inlined target_feature) { // at $DIR/inline-compatibility.rs:13:5: 13:21
88
+ }
99

1010
bb0: {
11-
StorageLive(_1); // scope 0 at $DIR/inline-compatibility.rs:14:5: 14:21
12-
- _1 = target_feature() -> bb1; // scope 0 at $DIR/inline-compatibility.rs:14:5: 14:21
11+
StorageLive(_1); // scope 0 at $DIR/inline-compatibility.rs:13:5: 13:21
12+
- _1 = target_feature() -> bb1; // scope 0 at $DIR/inline-compatibility.rs:13:5: 13:21
1313
- // mir::Constant
14-
- // + span: $DIR/inline-compatibility.rs:14:5: 14:19
14+
- // + span: $DIR/inline-compatibility.rs:13:5: 13:19
1515
- // + literal: Const { ty: unsafe fn() {target_feature}, val: Value(Scalar(<ZST>)) }
1616
- }
1717
-
1818
- bb1: {
19-
+ _1 = const (); // scope 1 at $DIR/inline-compatibility.rs:14:5: 14:21
20-
StorageDead(_1); // scope 0 at $DIR/inline-compatibility.rs:14:21: 14:22
21-
_0 = const (); // scope 0 at $DIR/inline-compatibility.rs:13:40: 15:2
22-
return; // scope 0 at $DIR/inline-compatibility.rs:15:2: 15:2
19+
+ _1 = const (); // scope 1 at $DIR/inline-compatibility.rs:13:5: 13:21
20+
StorageDead(_1); // scope 0 at $DIR/inline-compatibility.rs:13:21: 13:22
21+
_0 = const (); // scope 0 at $DIR/inline-compatibility.rs:12:40: 14:2
22+
return; // scope 0 at $DIR/inline-compatibility.rs:14:2: 14:2
2323
}
2424
}
2525

‎src/test/mir-opt/inline/inline_compatibility.not_inlined_c_variadic.Inline.diff‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
- // MIR for `not_inlined_c_variadic` before Inline
2+
+ // MIR for `not_inlined_c_variadic` after Inline
3+
4+
fn not_inlined_c_variadic() -> () {
5+
let mut _0: (); // return place in scope 0 at $DIR/inline-compatibility.rs:41:40: 41:40
6+
let _1: u32; // in scope 0 at $DIR/inline-compatibility.rs:42:9: 42:10
7+
scope 1 {
8+
debug s => _1; // in scope 1 at $DIR/inline-compatibility.rs:42:9: 42:10
9+
}
10+
11+
bb0: {
12+
StorageLive(_1); // scope 0 at $DIR/inline-compatibility.rs:42:9: 42:10
13+
_1 = sum(const 4_u32, const 4_u32, const 30_u32, const 200_u32, const 1000_u32) -> bb1; // scope 0 at $DIR/inline-compatibility.rs:42:13: 42:52
14+
// mir::Constant
15+
// + span: $DIR/inline-compatibility.rs:42:13: 42:16
16+
// + literal: Const { ty: unsafe extern "C" fn(u32, ...) -> u32 {sum}, val: Value(Scalar(<ZST>)) }
17+
}
18+
19+
bb1: {
20+
_0 = const (); // scope 0 at $DIR/inline-compatibility.rs:41:40: 43:2
21+
StorageDead(_1); // scope 0 at $DIR/inline-compatibility.rs:43:1: 43:2
22+
return; // scope 0 at $DIR/inline-compatibility.rs:43:2: 43:2
23+
}
24+
}
25+

‎src/test/mir-opt/inline/inline_compatibility.not_inlined_no_sanitize.Inline.diff‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,21 +2,21 @@
22
+ // MIR for `not_inlined_no_sanitize` after Inline
33

44
fn not_inlined_no_sanitize() -> () {
5-
let mut _0: (); // return place in scope 0 at $DIR/inline-compatibility.rs:29:41: 29:41
6-
let _1: (); // in scope 0 at $DIR/inline-compatibility.rs:30:5: 30:18
5+
let mut _0: (); // return place in scope 0 at $DIR/inline-compatibility.rs:28:41: 28:41
6+
let _1: (); // in scope 0 at $DIR/inline-compatibility.rs:29:5: 29:18
77

88
bb0: {
9-
StorageLive(_1); // scope 0 at $DIR/inline-compatibility.rs:30:5: 30:18
10-
_1 = no_sanitize() -> bb1; // scope 0 at $DIR/inline-compatibility.rs:30:5: 30:18
9+
StorageLive(_1); // scope 0 at $DIR/inline-compatibility.rs:29:5: 29:18
10+
_1 = no_sanitize() -> bb1; // scope 0 at $DIR/inline-compatibility.rs:29:5: 29:18
1111
// mir::Constant
12-
// + span: $DIR/inline-compatibility.rs:30:5: 30:16
12+
// + span: $DIR/inline-compatibility.rs:29:5: 29:16
1313
// + literal: Const { ty: unsafe fn() {no_sanitize}, val: Value(Scalar(<ZST>)) }
1414
}
1515

1616
bb1: {
17-
StorageDead(_1); // scope 0 at $DIR/inline-compatibility.rs:30:18: 30:19
18-
_0 = const (); // scope 0 at $DIR/inline-compatibility.rs:29:41: 31:2
19-
return; // scope 0 at $DIR/inline-compatibility.rs:31:2: 31:2
17+
StorageDead(_1); // scope 0 at $DIR/inline-compatibility.rs:29:18: 29:19
18+
_0 = const (); // scope 0 at $DIR/inline-compatibility.rs:28:41: 30:2
19+
return; // scope 0 at $DIR/inline-compatibility.rs:30:2: 30:2
2020
}
2121
}
2222

‎src/test/mir-opt/inline/inline_compatibility.not_inlined_target_feature.Inline.diff‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,21 +2,21 @@
22
+ // MIR for `not_inlined_target_feature` after Inline
33

44
fn not_inlined_target_feature() -> () {
5-
let mut _0: (); // return place in scope 0 at $DIR/inline-compatibility.rs:18:44: 18:44
6-
let _1: (); // in scope 0 at $DIR/inline-compatibility.rs:19:5: 19:21
5+
let mut _0: (); // return place in scope 0 at $DIR/inline-compatibility.rs:17:44: 17:44
6+
let _1: (); // in scope 0 at $DIR/inline-compatibility.rs:18:5: 18:21
77

88
bb0: {
9-
StorageLive(_1); // scope 0 at $DIR/inline-compatibility.rs:19:5: 19:21
10-
_1 = target_feature() -> bb1; // scope 0 at $DIR/inline-compatibility.rs:19:5: 19:21
9+
StorageLive(_1); // scope 0 at $DIR/inline-compatibility.rs:18:5: 18:21
10+
_1 = target_feature() -> bb1; // scope 0 at $DIR/inline-compatibility.rs:18:5: 18:21
1111
// mir::Constant
12-
// + span: $DIR/inline-compatibility.rs:19:5: 19:19
12+
// + span: $DIR/inline-compatibility.rs:18:5: 18:19
1313
// + literal: Const { ty: unsafe fn() {target_feature}, val: Value(Scalar(<ZST>)) }
1414
}
1515

1616
bb1: {
17-
StorageDead(_1); // scope 0 at $DIR/inline-compatibility.rs:19:21: 19:22
18-
_0 = const (); // scope 0 at $DIR/inline-compatibility.rs:18:44: 20:2
19-
return; // scope 0 at $DIR/inline-compatibility.rs:20:2: 20:2
17+
StorageDead(_1); // scope 0 at $DIR/inline-compatibility.rs:18:21: 18:22
18+
_0 = const (); // scope 0 at $DIR/inline-compatibility.rs:17:44: 19:2
19+
return; // scope 0 at $DIR/inline-compatibility.rs:19:2: 19:2
2020
}
2121
}
2222

0 commit comments

Comments
 (0)