Skip to content

Commit 67d1617

Browse files
borsbors
committed
Auto merge of #106364 - JakobDegen:top-down-inlining, r=cjgillot
Reenable limited top-down MIR inlining Reverts most of #105119 and uses an alternative strategy to prevent exponential blowup. Specifically, we allow doing top-down inlining up to depth at most five, and for at most one call site per nested body. r? `@cjgillot`
2 parents d6f99e5 + ee6503a commit 67d1617

11 files changed

+220
-97
lines changed

‎compiler/rustc_mir_transform/src/inline.rs‎

Lines changed: 36 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
//! Inlining pass for MIR functions
22
use crate::deref_separator::deref_finder;
33
use rustc_attr::InlineAttr;
4+
use rustc_hir::def_id::DefId;
45
use rustc_index::bit_set::BitSet;
56
use rustc_index::vec::Idx;
67
use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs};
@@ -27,6 +28,8 @@ const RESUME_PENALTY: usize = 45;
2728

2829
const UNKNOWN_SIZE_COST: usize = 10;
2930

31+
const TOP_DOWN_DEPTH_LIMIT: usize = 5;
32+
3033
pub struct Inline;
3134

3235
#[derive(Copy, Clone, Debug)]
@@ -86,8 +89,13 @@ fn inline<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool {
8689

8790
let param_env = tcx.param_env_reveal_all_normalized(def_id);
8891

89-
let mut this =
90-
Inliner { tcx, param_env, codegen_fn_attrs: tcx.codegen_fn_attrs(def_id), changed: false };
92+
let mut this = Inliner {
93+
tcx,
94+
param_env,
95+
codegen_fn_attrs: tcx.codegen_fn_attrs(def_id),
96+
history: Vec::new(),
97+
changed: false,
98+
};
9199
let blocks = BasicBlock::new(0)..body.basic_blocks.next_index();
92100
this.process_blocks(body, blocks);
93101
this.changed
@@ -98,12 +106,26 @@ struct Inliner<'tcx> {
98106
param_env: ParamEnv<'tcx>,
99107
/// Caller codegen attributes.
100108
codegen_fn_attrs: &'tcx CodegenFnAttrs,
109+
/// Stack of inlined instances.
110+
/// We only check the `DefId` and not the substs because we want to
111+
/// avoid inlining cases of polymorphic recursion.
112+
/// The number of `DefId`s is finite, so checking history is enough
113+
/// to ensure that we do not loop endlessly while inlining.
114+
history: Vec<DefId>,
101115
/// Indicates that the caller body has been modified.
102116
changed: bool,
103117
}
104118

105119
impl<'tcx> Inliner<'tcx> {
106120
fn process_blocks(&mut self, caller_body: &mut Body<'tcx>, blocks: Range<BasicBlock>) {
121+
// How many callsites in this body are we allowed to inline? We need to limit this in order
122+
// to prevent super-linear growth in MIR size
123+
let inline_limit = match self.history.len() {
124+
0 => usize::MAX,
125+
1..=TOP_DOWN_DEPTH_LIMIT => 1,
126+
_ => return,
127+
};
128+
let mut inlined_count = 0;
107129
for bb in blocks {
108130
let bb_data = &caller_body[bb];
109131
if bb_data.is_cleanup {
@@ -122,12 +144,16 @@ impl<'tcx> Inliner<'tcx> {
122144
debug!("not-inlined {} [{}]", callsite.callee, reason);
123145
continue;
124146
}
125-
Ok(_) => {
147+
Ok(new_blocks) => {
126148
debug!("inlined {}", callsite.callee);
127149
self.changed = true;
128-
// We could process the blocks returned by `try_inlining` here. However, that
129-
// leads to exponential compile times due to the top-down nature of this kind
130-
// of inlining.
150+
inlined_count += 1;
151+
if inlined_count == inline_limit {
152+
return;
153+
}
154+
self.history.push(callsite.callee.def_id());
155+
self.process_blocks(caller_body, new_blocks);
156+
self.history.pop();
131157
}
132158
}
133159
}
@@ -301,6 +327,10 @@ impl<'tcx> Inliner<'tcx> {
301327
return None;
302328
}
303329

330+
if self.history.contains(&callee.def_id()) {
331+
return None;
332+
}
333+
304334
let fn_sig = self.tcx.bound_fn_sig(def_id).subst(self.tcx, substs);
305335
let source_info = SourceInfo { span: fn_span, ..terminator.source_info };
306336

‎src/test/mir-opt/inline/cycle.g.Inline.diff‎

Lines changed: 11 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@
1010
+ let _3: (); // in scope 1 at $DIR/cycle.rs:6:5: 6:8
1111
+ let mut _4: &fn() {main}; // in scope 1 at $DIR/cycle.rs:6:5: 6:6
1212
+ let mut _5: (); // in scope 1 at $DIR/cycle.rs:6:5: 6:8
13+
+ scope 2 (inlined <fn() {main} as Fn<()>>::call - shim(fn() {main})) { // at $DIR/cycle.rs:6:5: 6:8
14+
+ }
1315
+ }
1416

1517
bb0: {
@@ -27,10 +29,7 @@
2729
+ StorageLive(_4); // scope 1 at $DIR/cycle.rs:6:5: 6:6
2830
+ _4 = &_2; // scope 1 at $DIR/cycle.rs:6:5: 6:6
2931
+ StorageLive(_5); // scope 1 at $DIR/cycle.rs:6:5: 6:8
30-
+ _3 = <fn() {main} as Fn<()>>::call(move _4, move _5) -> [return: bb2, unwind: bb3]; // scope 1 at $DIR/cycle.rs:6:5: 6:8
31-
+ // mir::Constant
32-
+ // + span: $DIR/cycle.rs:6:5: 6:6
33-
+ // + literal: Const { ty: for<'a> extern "rust-call" fn(&'a fn() {main}, ()) -> <fn() {main} as FnOnce<()>>::Output {<fn() {main} as Fn<()>>::call}, val: Value(<ZST>) }
32+
+ _3 = move (*_4)() -> [return: bb4, unwind: bb2]; // scope 2 at $SRC_DIR/core/src/ops/function.rs:LL:COL
3433
}
3534

3635
bb1: {
@@ -40,19 +39,19 @@
4039
return; // scope 0 at $DIR/cycle.rs:+2:2: +2:2
4140
+ }
4241
+
43-
+ bb2: {
44-
+ StorageDead(_5); // scope 1 at $DIR/cycle.rs:6:7: 6:8
45-
+ StorageDead(_4); // scope 1 at $DIR/cycle.rs:6:7: 6:8
46-
+ StorageDead(_3); // scope 1 at $DIR/cycle.rs:6:8: 6:9
47-
+ drop(_2) -> bb1; // scope 1 at $DIR/cycle.rs:7:1: 7:2
42+
+ bb2 (cleanup): {
43+
+ drop(_2) -> bb3; // scope 1 at $DIR/cycle.rs:7:1: 7:2
4844
+ }
4945
+
5046
+ bb3 (cleanup): {
51-
+ drop(_2) -> bb4; // scope 1 at $DIR/cycle.rs:7:1: 7:2
47+
+ resume; // scope 1 at $DIR/cycle.rs:5:1: 7:2
5248
+ }
5349
+
54-
+ bb4 (cleanup): {
55-
+ resume; // scope 1 at $DIR/cycle.rs:5:1: 7:2
50+
+ bb4: {
51+
+ StorageDead(_5); // scope 1 at $DIR/cycle.rs:6:7: 6:8
52+
+ StorageDead(_4); // scope 1 at $DIR/cycle.rs:6:7: 6:8
53+
+ StorageDead(_3); // scope 1 at $DIR/cycle.rs:6:8: 6:9
54+
+ drop(_2) -> bb1; // scope 1 at $DIR/cycle.rs:7:1: 7:2
5655
}
5756
}
5857

‎src/test/mir-opt/inline/cycle.main.Inline.diff‎

Lines changed: 11 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@
1010
+ let _3: (); // in scope 1 at $DIR/cycle.rs:6:5: 6:8
1111
+ let mut _4: &fn() {g}; // in scope 1 at $DIR/cycle.rs:6:5: 6:6
1212
+ let mut _5: (); // in scope 1 at $DIR/cycle.rs:6:5: 6:8
13+
+ scope 2 (inlined <fn() {g} as Fn<()>>::call - shim(fn() {g})) { // at $DIR/cycle.rs:6:5: 6:8
14+
+ }
1315
+ }
1416

1517
bb0: {
@@ -27,10 +29,7 @@
2729
+ StorageLive(_4); // scope 1 at $DIR/cycle.rs:6:5: 6:6
2830
+ _4 = &_2; // scope 1 at $DIR/cycle.rs:6:5: 6:6
2931
+ StorageLive(_5); // scope 1 at $DIR/cycle.rs:6:5: 6:8
30-
+ _3 = <fn() {g} as Fn<()>>::call(move _4, move _5) -> [return: bb2, unwind: bb3]; // scope 1 at $DIR/cycle.rs:6:5: 6:8
31-
+ // mir::Constant
32-
+ // + span: $DIR/cycle.rs:6:5: 6:6
33-
+ // + literal: Const { ty: for<'a> extern "rust-call" fn(&'a fn() {g}, ()) -> <fn() {g} as FnOnce<()>>::Output {<fn() {g} as Fn<()>>::call}, val: Value(<ZST>) }
32+
+ _3 = move (*_4)() -> [return: bb4, unwind: bb2]; // scope 2 at $SRC_DIR/core/src/ops/function.rs:LL:COL
3433
}
3534

3635
bb1: {
@@ -40,19 +39,19 @@
4039
return; // scope 0 at $DIR/cycle.rs:+2:2: +2:2
4140
+ }
4241
+
43-
+ bb2: {
44-
+ StorageDead(_5); // scope 1 at $DIR/cycle.rs:6:7: 6:8
45-
+ StorageDead(_4); // scope 1 at $DIR/cycle.rs:6:7: 6:8
46-
+ StorageDead(_3); // scope 1 at $DIR/cycle.rs:6:8: 6:9
47-
+ drop(_2) -> bb1; // scope 1 at $DIR/cycle.rs:7:1: 7:2
42+
+ bb2 (cleanup): {
43+
+ drop(_2) -> bb3; // scope 1 at $DIR/cycle.rs:7:1: 7:2
4844
+ }
4945
+
5046
+ bb3 (cleanup): {
51-
+ drop(_2) -> bb4; // scope 1 at $DIR/cycle.rs:7:1: 7:2
47+
+ resume; // scope 1 at $DIR/cycle.rs:5:1: 7:2
5248
+ }
5349
+
54-
+ bb4 (cleanup): {
55-
+ resume; // scope 1 at $DIR/cycle.rs:5:1: 7:2
50+
+ bb4: {
51+
+ StorageDead(_5); // scope 1 at $DIR/cycle.rs:6:7: 6:8
52+
+ StorageDead(_4); // scope 1 at $DIR/cycle.rs:6:7: 6:8
53+
+ StorageDead(_3); // scope 1 at $DIR/cycle.rs:6:8: 6:9
54+
+ drop(_2) -> bb1; // scope 1 at $DIR/cycle.rs:7:1: 7:2
5655
}
5756
}
5857

‎src/test/mir-opt/inline/exponential_runtime.main.Inline.diff‎

Lines changed: 39 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -8,43 +8,68 @@
88
+ let _2: (); // in scope 1 at $DIR/exponential_runtime.rs:73:9: 73:25
99
+ let _3: (); // in scope 1 at $DIR/exponential_runtime.rs:74:9: 74:25
1010
+ let _4: (); // in scope 1 at $DIR/exponential_runtime.rs:75:9: 75:25
11+
+ scope 2 (inlined <() as F>::call) { // at $DIR/exponential_runtime.rs:73:9: 73:25
12+
+ let _5: (); // in scope 2 at $DIR/exponential_runtime.rs:61:9: 61:25
13+
+ let _6: (); // in scope 2 at $DIR/exponential_runtime.rs:62:9: 62:25
14+
+ let _7: (); // in scope 2 at $DIR/exponential_runtime.rs:63:9: 63:25
15+
+ }
1116
+ }
1217

1318
bb0: {
1419
StorageLive(_1); // scope 0 at $DIR/exponential_runtime.rs:+1:5: +1:22
1520
- _1 = <() as G>::call() -> bb1; // scope 0 at $DIR/exponential_runtime.rs:+1:5: +1:22
1621
+ StorageLive(_2); // scope 1 at $DIR/exponential_runtime.rs:73:9: 73:25
17-
+ _2 = <() as F>::call() -> bb1; // scope 1 at $DIR/exponential_runtime.rs:73:9: 73:25
22+
+ StorageLive(_5); // scope 2 at $DIR/exponential_runtime.rs:61:9: 61:25
23+
+ _5 = <() as E>::call() -> bb3; // scope 2 at $DIR/exponential_runtime.rs:61:9: 61:25
1824
// mir::Constant
1925
- // + span: $DIR/exponential_runtime.rs:86:5: 86:20
2026
- // + literal: Const { ty: fn() {<() as G>::call}, val: Value(<ZST>) }
21-
+ // + span: $DIR/exponential_runtime.rs:73:9: 73:23
22-
+ // + literal: Const { ty: fn() {<() as F>::call}, val: Value(<ZST>) }
27+
+ // + span: $DIR/exponential_runtime.rs:61:9: 61:23
28+
+ // + literal: Const { ty: fn() {<() as E>::call}, val: Value(<ZST>) }
2329
}
2430

2531
bb1: {
26-
+ StorageDead(_2); // scope 1 at $DIR/exponential_runtime.rs:73:25: 73:26
27-
+ StorageLive(_3); // scope 1 at $DIR/exponential_runtime.rs:74:9: 74:25
28-
+ _3 = <() as F>::call() -> bb2; // scope 1 at $DIR/exponential_runtime.rs:74:9: 74:25
29-
+ // mir::Constant
30-
+ // + span: $DIR/exponential_runtime.rs:74:9: 74:23
31-
+ // + literal: Const { ty: fn() {<() as F>::call}, val: Value(<ZST>) }
32-
+ }
33-
+
34-
+ bb2: {
3532
+ StorageDead(_3); // scope 1 at $DIR/exponential_runtime.rs:74:25: 74:26
3633
+ StorageLive(_4); // scope 1 at $DIR/exponential_runtime.rs:75:9: 75:25
37-
+ _4 = <() as F>::call() -> bb3; // scope 1 at $DIR/exponential_runtime.rs:75:9: 75:25
34+
+ _4 = <() as F>::call() -> bb2; // scope 1 at $DIR/exponential_runtime.rs:75:9: 75:25
3835
+ // mir::Constant
3936
+ // + span: $DIR/exponential_runtime.rs:75:9: 75:23
4037
+ // + literal: Const { ty: fn() {<() as F>::call}, val: Value(<ZST>) }
4138
+ }
4239
+
43-
+ bb3: {
40+
+ bb2: {
4441
+ StorageDead(_4); // scope 1 at $DIR/exponential_runtime.rs:75:25: 75:26
4542
StorageDead(_1); // scope 0 at $DIR/exponential_runtime.rs:+1:22: +1:23
4643
_0 = const (); // scope 0 at $DIR/exponential_runtime.rs:+0:11: +2:2
4744
return; // scope 0 at $DIR/exponential_runtime.rs:+2:2: +2:2
45+
+ }
46+
+
47+
+ bb3: {
48+
+ StorageDead(_5); // scope 2 at $DIR/exponential_runtime.rs:61:25: 61:26
49+
+ StorageLive(_6); // scope 2 at $DIR/exponential_runtime.rs:62:9: 62:25
50+
+ _6 = <() as E>::call() -> bb4; // scope 2 at $DIR/exponential_runtime.rs:62:9: 62:25
51+
+ // mir::Constant
52+
+ // + span: $DIR/exponential_runtime.rs:62:9: 62:23
53+
+ // + literal: Const { ty: fn() {<() as E>::call}, val: Value(<ZST>) }
54+
+ }
55+
+
56+
+ bb4: {
57+
+ StorageDead(_6); // scope 2 at $DIR/exponential_runtime.rs:62:25: 62:26
58+
+ StorageLive(_7); // scope 2 at $DIR/exponential_runtime.rs:63:9: 63:25
59+
+ _7 = <() as E>::call() -> bb5; // scope 2 at $DIR/exponential_runtime.rs:63:9: 63:25
60+
+ // mir::Constant
61+
+ // + span: $DIR/exponential_runtime.rs:63:9: 63:23
62+
+ // + literal: Const { ty: fn() {<() as E>::call}, val: Value(<ZST>) }
63+
+ }
64+
+
65+
+ bb5: {
66+
+ StorageDead(_7); // scope 2 at $DIR/exponential_runtime.rs:63:25: 63:26
67+
+ StorageDead(_2); // scope 1 at $DIR/exponential_runtime.rs:73:25: 73:26
68+
+ StorageLive(_3); // scope 1 at $DIR/exponential_runtime.rs:74:9: 74:25
69+
+ _3 = <() as F>::call() -> bb1; // scope 1 at $DIR/exponential_runtime.rs:74:9: 74:25
70+
+ // mir::Constant
71+
+ // + span: $DIR/exponential_runtime.rs:74:9: 74:23
72+
+ // + literal: Const { ty: fn() {<() as F>::call}, val: Value(<ZST>) }
4873
}
4974
}
5075

‎src/test/mir-opt/inline/inline_cycle.one.Inline.diff‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,17 +5,20 @@
55
let mut _0: (); // return place in scope 0 at $DIR/inline_cycle.rs:+0:10: +0:10
66
let _1: (); // in scope 0 at $DIR/inline_cycle.rs:+1:5: +1:24
77
+ scope 1 (inlined <C as Call>::call) { // at $DIR/inline_cycle.rs:14:5: 14:24
8+
+ scope 2 (inlined <A<C> as Call>::call) { // at $DIR/inline_cycle.rs:43:9: 43:23
9+
+ scope 3 (inlined <B<C> as Call>::call) { // at $DIR/inline_cycle.rs:28:9: 28:31
10+
+ }
11+
+ }
812
+ }
913

1014
bb0: {
1115
StorageLive(_1); // scope 0 at $DIR/inline_cycle.rs:+1:5: +1:24
1216
- _1 = <C as Call>::call() -> bb1; // scope 0 at $DIR/inline_cycle.rs:+1:5: +1:24
13-
+ _1 = <A<C> as Call>::call() -> bb1; // scope 1 at $DIR/inline_cycle.rs:43:9: 43:23
17+
+ _1 = <C as Call>::call() -> bb1; // scope 3 at $DIR/inline_cycle.rs:36:9: 36:28
1418
// mir::Constant
1519
- // + span: $DIR/inline_cycle.rs:14:5: 14:22
16-
- // + literal: Const { ty: fn() {<C as Call>::call}, val: Value(<ZST>) }
17-
+ // + span: $DIR/inline_cycle.rs:43:9: 43:21
18-
+ // + literal: Const { ty: fn() {<A<C> as Call>::call}, val: Value(<ZST>) }
20+
+ // + span: $DIR/inline_cycle.rs:36:9: 36:26
21+
// + literal: Const { ty: fn() {<C as Call>::call}, val: Value(<ZST>) }
1922
}
2023

2124
bb1: {

‎src/test/mir-opt/inline/inline_cycle.two.Inline.diff‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99
+ debug f => _2; // in scope 1 at $DIR/inline_cycle.rs:53:22: 53:23
1010
+ let _3: (); // in scope 1 at $DIR/inline_cycle.rs:54:5: 54:8
1111
+ let mut _4: (); // in scope 1 at $DIR/inline_cycle.rs:54:5: 54:8
12+
+ scope 2 (inlined <fn() {f} as FnOnce<()>>::call_once - shim(fn() {f})) { // at $DIR/inline_cycle.rs:54:5: 54:8
13+
+ }
1214
+ }
1315

1416
bb0: {
@@ -24,10 +26,7 @@
2426
// + literal: Const { ty: fn() {f}, val: Value(<ZST>) }
2527
+ StorageLive(_3); // scope 1 at $DIR/inline_cycle.rs:54:5: 54:8
2628
+ StorageLive(_4); // scope 1 at $DIR/inline_cycle.rs:54:5: 54:8
27-
+ _3 = <fn() {f} as FnOnce<()>>::call_once(move _2, move _4) -> bb1; // scope 1 at $DIR/inline_cycle.rs:54:5: 54:8
28-
+ // mir::Constant
29-
+ // + span: $DIR/inline_cycle.rs:54:5: 54:6
30-
+ // + literal: Const { ty: extern "rust-call" fn(fn() {f}, ()) -> <fn() {f} as FnOnce<()>>::Output {<fn() {f} as FnOnce<()>>::call_once}, val: Value(<ZST>) }
29+
+ _3 = move _2() -> bb1; // scope 2 at $SRC_DIR/core/src/ops/function.rs:LL:COL
3130
}
3231

3332
bb1: {

‎src/test/mir-opt/inline/inline_cycle_generic.main.Inline.diff‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,18 +6,21 @@
66
let _1: (); // in scope 0 at $DIR/inline_cycle_generic.rs:+1:5: +1:24
77
+ scope 1 (inlined <C as Call>::call) { // at $DIR/inline_cycle_generic.rs:9:5: 9:24
88
+ scope 2 (inlined <B<A> as Call>::call) { // at $DIR/inline_cycle_generic.rs:38:9: 38:31
9+
+ scope 3 (inlined <A as Call>::call) { // at $DIR/inline_cycle_generic.rs:31:9: 31:28
10+
+ scope 4 (inlined <B<C> as Call>::call) { // at $DIR/inline_cycle_generic.rs:23:9: 23:31
11+
+ }
12+
+ }
913
+ }
1014
+ }
1115

1216
bb0: {
1317
StorageLive(_1); // scope 0 at $DIR/inline_cycle_generic.rs:+1:5: +1:24
1418
- _1 = <C as Call>::call() -> bb1; // scope 0 at $DIR/inline_cycle_generic.rs:+1:5: +1:24
15-
+ _1 = <A as Call>::call() -> bb1; // scope 2 at $DIR/inline_cycle_generic.rs:31:9: 31:28
19+
+ _1 = <C as Call>::call() -> bb1; // scope 4 at $DIR/inline_cycle_generic.rs:31:9: 31:28
1620
// mir::Constant
1721
- // + span: $DIR/inline_cycle_generic.rs:9:5: 9:22
18-
- // + literal: Const { ty: fn() {<C as Call>::call}, val: Value(<ZST>) }
1922
+ // + span: $DIR/inline_cycle_generic.rs:31:9: 31:26
20-
+ // + literal: Const { ty: fn() {<A as Call>::call}, val: Value(<ZST>) }
23+
// + literal: Const { ty: fn() {<C as Call>::call}, val: Value(<ZST>) }
2124
}
2225

2326
bb1: {

0 commit comments

Comments
 (0)