From 175fa55c7b3530dd74ecee2fbaeccc7d4eae296b Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Sat, 21 Jun 2025 09:03:40 +0000 Subject: [PATCH 1/9] Fortify test. --- ...g.aggregate.JumpThreading.panic-abort.diff | 47 +++++++++---------- ....aggregate.JumpThreading.panic-unwind.diff | 47 +++++++++---------- tests/mir-opt/jump_threading.rs | 37 ++++++++++++++- 3 files changed, 81 insertions(+), 50 deletions(-) diff --git a/tests/mir-opt/jump_threading.aggregate.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.aggregate.JumpThreading.panic-abort.diff index a7551c3fb5b79..89d04c557f125 100644 --- a/tests/mir-opt/jump_threading.aggregate.JumpThreading.panic-abort.diff +++ b/tests/mir-opt/jump_threading.aggregate.JumpThreading.panic-abort.diff @@ -1,51 +1,50 @@ - // MIR for `aggregate` before JumpThreading + // MIR for `aggregate` after JumpThreading - fn aggregate(_1: u8) -> u8 { - debug x => _1; + fn aggregate() -> u8 { let mut _0: u8; + let _1: u8; let _2: u8; - let _3: u8; - let mut _4: (u8, u8); - let mut _5: bool; - let mut _6: u8; + let mut _3: (u8, u8); + let mut _4: bool; + let mut _5: u8; scope 1 { - debug a => _2; - debug b => _3; + debug a => _1; + debug b => _2; } bb0: { - StorageLive(_4); - _4 = const aggregate::FOO; - StorageLive(_2); - _2 = copy (_4.0: u8); StorageLive(_3); - _3 = copy (_4.1: u8); - StorageDead(_4); + _3 = const aggregate::FOO; + StorageLive(_1); + _1 = copy (_3.0: u8); + StorageLive(_2); + _2 = copy (_3.1: u8); + StorageDead(_3); + StorageLive(_4); StorageLive(_5); - StorageLive(_6); - _6 = copy _2; - _5 = Eq(move _6, const 7_u8); -- switchInt(move _5) -> [0: bb2, otherwise: bb1]; + _5 = copy _1; + _4 = Eq(move _5, const 7_u8); +- switchInt(move _4) -> [0: bb2, otherwise: bb1]; + goto -> bb2; } bb1: { - StorageDead(_6); - _0 = copy _3; + StorageDead(_5); + _0 = copy _2; goto -> bb3; } bb2: { - StorageDead(_6); - _0 = copy _2; + StorageDead(_5); + _0 = copy _1; goto -> bb3; } bb3: { - StorageDead(_5); - StorageDead(_3); + StorageDead(_4); StorageDead(_2); + StorageDead(_1); return; } } diff --git a/tests/mir-opt/jump_threading.aggregate.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.aggregate.JumpThreading.panic-unwind.diff index a7551c3fb5b79..89d04c557f125 100644 --- a/tests/mir-opt/jump_threading.aggregate.JumpThreading.panic-unwind.diff +++ b/tests/mir-opt/jump_threading.aggregate.JumpThreading.panic-unwind.diff @@ -1,51 +1,50 @@ - // MIR for `aggregate` before JumpThreading + // MIR for `aggregate` after JumpThreading - fn aggregate(_1: u8) -> u8 { - debug x => _1; + fn aggregate() -> u8 { let mut _0: u8; + let _1: u8; let _2: u8; - let _3: u8; - let mut _4: (u8, u8); - let mut _5: bool; - let mut _6: u8; + let mut _3: (u8, u8); + let mut _4: bool; + let mut _5: u8; scope 1 { - debug a => _2; - debug b => _3; + debug a => _1; + debug b => _2; } bb0: { - StorageLive(_4); - _4 = const aggregate::FOO; - StorageLive(_2); - _2 = copy (_4.0: u8); StorageLive(_3); - _3 = copy (_4.1: u8); - StorageDead(_4); + _3 = const aggregate::FOO; + StorageLive(_1); + _1 = copy (_3.0: u8); + StorageLive(_2); + _2 = copy (_3.1: u8); + StorageDead(_3); + StorageLive(_4); StorageLive(_5); - StorageLive(_6); - _6 = copy _2; - _5 = Eq(move _6, const 7_u8); -- switchInt(move _5) -> [0: bb2, otherwise: bb1]; + _5 = copy _1; + _4 = Eq(move _5, const 7_u8); +- switchInt(move _4) -> [0: bb2, otherwise: bb1]; + goto -> bb2; } bb1: { - StorageDead(_6); - _0 = copy _3; + StorageDead(_5); + _0 = copy _2; goto -> bb3; } bb2: { - StorageDead(_6); - _0 = copy _2; + StorageDead(_5); + _0 = copy _1; goto -> bb3; } bb3: { - StorageDead(_5); - StorageDead(_3); + StorageDead(_4); StorageDead(_2); + StorageDead(_1); return; } } diff --git a/tests/mir-opt/jump_threading.rs b/tests/mir-opt/jump_threading.rs index 009e1060700c1..b1d9602bd46c1 100644 --- a/tests/mir-opt/jump_threading.rs +++ b/tests/mir-opt/jump_threading.rs @@ -449,9 +449,17 @@ fn disappearing_bb(x: u8) -> u8 { } /// Verify that we can thread jumps when we assign from an aggregate constant. -fn aggregate(x: u8) -> u8 { +fn aggregate() -> u8 { // CHECK-LABEL: fn aggregate( + // CHECK: debug a => [[a:_.*]]; + // CHECK: debug b => [[b:_.*]]; // CHECK-NOT: switchInt( + // CHECK: [[a2:_.*]] = copy [[a]]; + // CHECK: {{_.*}} = Eq(move [[a2]], const 7_u8); + // CHECK-NEXT: goto -> [[bb:bb.*]]; + // CHECK: [[bb]]: { + // CHECK-NOT: } + // CHECK: _0 = copy [[a]]; const FOO: (u8, u8) = (5, 13); @@ -508,7 +516,16 @@ fn assume(a: u8, b: bool) -> u8 { /// Verify that jump threading succeeds seeing through copies of aggregates. fn aggregate_copy() -> u32 { // CHECK-LABEL: fn aggregate_copy( + // CHECK: debug a => [[a:_.*]]; + // CHECK: debug b => [[b:_.*]]; + // CHECK: debug c => [[c:_.*]]; // CHECK-NOT: switchInt( + // CHECK: [[c2:_.*]] = copy [[c]]; + // CHECK: {{_.*}} = Eq(move [[c2]], const 2_u32); + // CHECK-NEXT: goto -> [[bb:bb.*]]; + // CHECK: [[bb]]: { + // CHECK-NOT: } + // CHECK: _0 = const 13_u32; const Foo: (u32, u32) = (5, 3); @@ -532,6 +549,14 @@ fn floats() -> u32 { pub fn bitwise_not() -> i32 { // CHECK-LABEL: fn bitwise_not( + // CHECK: debug a => [[a:_.*]]; + // CHECK: [[a2:_.*]] = copy [[a]]; + // CHECK: [[not:_.*]] = Not(move [[a2]]); + // CHECK: {{_.*}} = Eq(move [[not]], const 0_i32); + // CHECK-NEXT: goto -> [[bb:bb.*]]; + // CHECK: [[bb]]: { + // CHECK-NOT: } + // CHECK: _0 = const 0_i32; // Test for #131195, which was optimizing `!a == b` into `a != b`. let a = 1; @@ -540,6 +565,14 @@ pub fn bitwise_not() -> i32 { pub fn logical_not() -> i32 { // CHECK-LABEL: fn logical_not( + // CHECK: debug a => [[a:_.*]]; + // CHECK: [[a2:_.*]] = copy [[a]]; + // CHECK: [[not:_.*]] = Not(move [[a2]]); + // CHECK: {{_.*}} = Eq(move [[not]], const true); + // CHECK-NEXT: goto -> [[bb:bb.*]]; + // CHECK: [[bb]]: { + // CHECK-NOT: } + // CHECK: _0 = const 1_i32; let a = false; if !a == true { 1 } else { 0 } @@ -557,7 +590,7 @@ fn main() { mutable_ref(); renumbered_bb(true); disappearing_bb(7); - aggregate(7); + aggregate(); assume(7, false); floats(); bitwise_not(); From 7cd001a69bb50795755b28924974a6c65ef9ad02 Mon Sep 17 00:00:00 2001 From: Camille Gillot Date: Sun, 21 Sep 2025 02:10:54 +0000 Subject: [PATCH 2/9] Add tests. --- tests/mir-opt/jump_threading.rs | 27 +++++++++ ...g.two_reads.JumpThreading.panic-abort.diff | 55 +++++++++++++++++++ ....two_reads.JumpThreading.panic-unwind.diff | 55 +++++++++++++++++++ 3 files changed, 137 insertions(+) create mode 100644 tests/mir-opt/jump_threading.two_reads.JumpThreading.panic-abort.diff create mode 100644 tests/mir-opt/jump_threading.two_reads.JumpThreading.panic-unwind.diff diff --git a/tests/mir-opt/jump_threading.rs b/tests/mir-opt/jump_threading.rs index b1d9602bd46c1..ab8840db3837b 100644 --- a/tests/mir-opt/jump_threading.rs +++ b/tests/mir-opt/jump_threading.rs @@ -77,6 +77,32 @@ fn identity(x: Result) -> Result { Ok(x?) } +fn two_reads() -> i32 { + // CHECK-LABEL: fn two_reads( + // CHECK: debug a => [[a:_.*]]; + // CHECK: debug b => [[b:_.*]]; + // CHECK: debug c => [[c:_.*]]; + // CHECK: bb0: { + // CHECK: [[a]] = const 2_i32; + // CHECK: [[b]] = copy [[a]]; + // CHECK: [[c]] = copy [[a]]; + // CHECK: [[tmp:_.*]] = copy [[c]]; + // CHECK: [[eq:_.*]] = Eq(move [[tmp]], const 2_i32); + // CHECK: switchInt(move [[eq]]) -> [0: bb2, otherwise: bb1]; + // CHECK: bb1: { + // CHECK: _0 = const 0_i32; + // CHECK: goto -> bb3; + // CHECK: bb2: { + // CHECK: _0 = const 1_i32; + // CHECK: goto -> bb3; + // CHECK: bb3: { + // CHECK: return; + let a = 2; + let b = a; + let c = a; + if c == 2 { 0 } else { 1 } +} + enum DFA { A, B, @@ -599,6 +625,7 @@ fn main() { // EMIT_MIR jump_threading.too_complex.JumpThreading.diff // EMIT_MIR jump_threading.identity.JumpThreading.diff +// EMIT_MIR jump_threading.two_reads.JumpThreading.diff // EMIT_MIR jump_threading.custom_discr.JumpThreading.diff // EMIT_MIR jump_threading.dfa.JumpThreading.diff // EMIT_MIR jump_threading.multiple_match.JumpThreading.diff diff --git a/tests/mir-opt/jump_threading.two_reads.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.two_reads.JumpThreading.panic-abort.diff new file mode 100644 index 0000000000000..9962a5724b3f7 --- /dev/null +++ b/tests/mir-opt/jump_threading.two_reads.JumpThreading.panic-abort.diff @@ -0,0 +1,55 @@ +- // MIR for `two_reads` before JumpThreading ++ // MIR for `two_reads` after JumpThreading + + fn two_reads() -> i32 { + let mut _0: i32; + let _1: i32; + let mut _4: bool; + let mut _5: i32; + scope 1 { + debug a => _1; + let _2: i32; + scope 2 { + debug b => _2; + let _3: i32; + scope 3 { + debug c => _3; + } + } + } + + bb0: { + StorageLive(_1); + _1 = const 2_i32; + StorageLive(_2); + _2 = copy _1; + StorageLive(_3); + _3 = copy _1; + StorageLive(_4); + StorageLive(_5); + _5 = copy _3; + _4 = Eq(move _5, const 2_i32); + switchInt(move _4) -> [0: bb2, otherwise: bb1]; + } + + bb1: { + StorageDead(_5); + _0 = const 0_i32; + goto -> bb3; + } + + bb2: { + StorageDead(_5); + _0 = const 1_i32; + goto -> bb3; + } + + bb3: { + StorageDead(_4); + StorageDead(_3); + StorageDead(_2); + StorageDead(_1); + return; + } + } + diff --git a/tests/mir-opt/jump_threading.two_reads.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.two_reads.JumpThreading.panic-unwind.diff new file mode 100644 index 0000000000000..9962a5724b3f7 --- /dev/null +++ b/tests/mir-opt/jump_threading.two_reads.JumpThreading.panic-unwind.diff @@ -0,0 +1,55 @@ +- // MIR for `two_reads` before JumpThreading ++ // MIR for `two_reads` after JumpThreading + + fn two_reads() -> i32 { + let mut _0: i32; + let _1: i32; + let mut _4: bool; + let mut _5: i32; + scope 1 { + debug a => _1; + let _2: i32; + scope 2 { + debug b => _2; + let _3: i32; + scope 3 { + debug c => _3; + } + } + } + + bb0: { + StorageLive(_1); + _1 = const 2_i32; + StorageLive(_2); + _2 = copy _1; + StorageLive(_3); + _3 = copy _1; + StorageLive(_4); + StorageLive(_5); + _5 = copy _3; + _4 = Eq(move _5, const 2_i32); + switchInt(move _4) -> [0: bb2, otherwise: bb1]; + } + + bb1: { + StorageDead(_5); + _0 = const 0_i32; + goto -> bb3; + } + + bb2: { + StorageDead(_5); + _0 = const 1_i32; + goto -> bb3; + } + + bb3: { + StorageDead(_4); + StorageDead(_3); + StorageDead(_2); + StorageDead(_1); + return; + } + } + From 0bb02e2823617840895f4256fe9c53f20ad32466 Mon Sep 17 00:00:00 2001 From: Camille Gillot Date: Sun, 21 Sep 2025 02:12:06 +0000 Subject: [PATCH 3/9] Extend value_analysis API. --- .../rustc_mir_dataflow/src/value_analysis.rs | 70 +++++++++++++------ 1 file changed, 50 insertions(+), 20 deletions(-) diff --git a/compiler/rustc_mir_dataflow/src/value_analysis.rs b/compiler/rustc_mir_dataflow/src/value_analysis.rs index bf5ec8f459e61..969e9b3a4dd82 100644 --- a/compiler/rustc_mir_dataflow/src/value_analysis.rs +++ b/compiler/rustc_mir_dataflow/src/value_analysis.rs @@ -24,9 +24,7 @@ rustc_index::newtype_index!( rustc_index::newtype_index!( /// This index uniquely identifies a tracked place and therefore a slot in [`State`]. - /// - /// It is an implementation detail of this module. - struct ValueIndex {} + pub struct ValueIndex {} ); /// See [`State`]. @@ -211,22 +209,9 @@ impl State { /// The target place must have been flooded before calling this method. pub fn insert_place_idx(&mut self, target: PlaceIndex, source: PlaceIndex, map: &Map<'_>) { let State::Reachable(values) = self else { return }; - - // If both places are tracked, we copy the value to the target. - // If the target is tracked, but the source is not, we do nothing, as invalidation has - // already been performed. - if let Some(target_value) = map.places[target].value_index - && let Some(source_value) = map.places[source].value_index - { - values.insert(target_value, values.get(source_value).clone()); - } - for target_child in map.children(target) { - // Try to find corresponding child and recurse. Reasoning is similar as above. - let projection = map.places[target_child].proj_elem.unwrap(); - if let Some(source_child) = map.projections.get(&(source, projection)) { - self.insert_place_idx(target_child, *source_child, map); - } - } + map.for_each_value_pair(target, source, &mut |target, source| { + values.insert(target, values.get(source).clone()); + }); } /// Helper method to interpret `target = result`. @@ -677,6 +662,26 @@ impl<'tcx> Map<'tcx> { self.find_extra(place, [TrackElem::DerefLen]) } + /// Locates the value corresponding to the given place. + pub fn value(&self, place: PlaceIndex) -> Option { + self.places[place].value_index + } + + /// Locates the value corresponding to the given place. + pub fn find_value(&self, place: PlaceRef<'_>) -> Option { + self.value(self.find(place)?) + } + + /// Locates the value corresponding to the given discriminant. + pub fn find_discr_value(&self, place: PlaceRef<'_>) -> Option { + self.value(self.find_discr(place)?) + } + + /// Locates the value corresponding to the given length. + pub fn find_len_value(&self, place: PlaceRef<'_>) -> Option { + self.value(self.find_len(place)?) + } + /// Iterate over all direct children. fn children(&self, parent: PlaceIndex) -> impl Iterator { Children::new(self, parent) @@ -689,7 +694,7 @@ impl<'tcx> Map<'tcx> { /// /// `tail_elem` allows to support discriminants that are not a place in MIR, but that we track /// as such. - fn for_each_aliasing_place( + pub fn for_each_aliasing_place( &self, place: PlaceRef<'_>, tail_elem: Option, @@ -778,6 +783,31 @@ impl<'tcx> Map<'tcx> { } } } + + /// Recursively iterates on each value contained in `target`, paired with matching projection + /// inside `source`. + pub fn for_each_value_pair( + &self, + target: PlaceIndex, + source: PlaceIndex, + f: &mut impl FnMut(ValueIndex, ValueIndex), + ) { + // If both places are tracked, we copy the value to the target. + // If the target is tracked, but the source is not, we do nothing, as invalidation has + // already been performed. + if let Some(target_value) = self.places[target].value_index + && let Some(source_value) = self.places[source].value_index + { + f(target_value, source_value) + } + for target_child in self.children(target) { + // Try to find corresponding child and recurse. Reasoning is similar as above. + let projection = self.places[target_child].proj_elem.unwrap(); + if let Some(source_child) = self.projections.get(&(source, projection)) { + self.for_each_value_pair(target_child, *source_child, f); + } + } + } } /// This is the information tracked for every [`PlaceIndex`] and is stored by [`Map`]. From 46b47a35f9a89123f08cccbed71e380ae8393d1a Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Thu, 19 Jun 2025 22:42:05 +0000 Subject: [PATCH 4/9] Use a simpler condition set in jump threading. --- .../rustc_mir_transform/src/jump_threading.rs | 249 ++++++++++------- tests/coverage/conditions.cov-map | 254 ++++++++---------- tests/mir-opt/jump_threading.rs | 2 +- ...g.two_reads.JumpThreading.panic-abort.diff | 3 +- ....two_reads.JumpThreading.panic-unwind.diff | 3 +- 5 files changed, 266 insertions(+), 245 deletions(-) diff --git a/compiler/rustc_mir_transform/src/jump_threading.rs b/compiler/rustc_mir_transform/src/jump_threading.rs index 492f5ca82a07e..9b420ab18112b 100644 --- a/compiler/rustc_mir_transform/src/jump_threading.rs +++ b/compiler/rustc_mir_transform/src/jump_threading.rs @@ -38,7 +38,7 @@ use rustc_arena::DroplessArena; use rustc_const_eval::const_eval::DummyMachine; use rustc_const_eval::interpret::{ImmTy, Immediate, InterpCx, OpTy, Projectable}; -use rustc_data_structures::fx::FxHashSet; +use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_index::IndexVec; use rustc_index::bit_set::DenseBitSet; use rustc_middle::bug; @@ -47,7 +47,7 @@ use rustc_middle::mir::visit::Visitor; use rustc_middle::mir::*; use rustc_middle::ty::{self, ScalarInt, TyCtxt}; use rustc_mir_dataflow::lattice::HasBottom; -use rustc_mir_dataflow::value_analysis::{Map, PlaceIndex, State, TrackElem}; +use rustc_mir_dataflow::value_analysis::{Map, PlaceIndex, TrackElem, ValueIndex}; use rustc_span::DUMMY_SP; use tracing::{debug, instrument, trace}; @@ -134,6 +134,7 @@ struct TOFinder<'a, 'tcx> { /// to `value`, jump to `target`. #[derive(Copy, Clone, Debug)] struct Condition { + place: ValueIndex, value: ScalarInt, polarity: Polarity, target: BasicBlock, @@ -146,8 +147,14 @@ enum Polarity { } impl Condition { - fn matches(&self, value: ScalarInt) -> bool { - (self.value == value) == (self.polarity == Polarity::Eq) + fn matches(&self, place: ValueIndex, value: ScalarInt) -> bool { + self.place == place && (self.value == value) == (self.polarity == Polarity::Eq) + } + + fn into_opportunity(self, match_bb: Option) -> ThreadingOpportunity { + trace!(?self, "registering"); + let chain = match_bb.into_iter().collect(); + ThreadingOpportunity { chain, target: self.target } } } @@ -163,29 +170,50 @@ impl HasBottom for ConditionSet<'_> { } impl<'a> ConditionSet<'a> { + fn is_empty(self) -> bool { + self.0.is_empty() + } + fn iter(self) -> impl Iterator { self.0.iter().copied() } - fn iter_matches(self, value: ScalarInt) -> impl Iterator { - self.iter().filter(move |c| c.matches(value)) + fn iter_matches(self, place: ValueIndex, value: ScalarInt) -> impl Iterator { + self.iter().filter(move |c| c.matches(place, value)) + } + + fn register_matches( + &self, + place: ValueIndex, + value: ScalarInt, + match_bb: Option, + opportunities: &mut Vec, + ) { + self.iter_matches(place, value) + .for_each(|cond| opportunities.push(cond.into_opportunity(match_bb))) + } + + fn filter(self, arena: &'a DroplessArena, f: impl Fn(Condition) -> bool) -> ConditionSet<'a> { + let set = arena.alloc_from_iter(self.iter().filter(|&c| f(c))); + ConditionSet(set) } - fn map( + fn filter_map( self, arena: &'a DroplessArena, f: impl Fn(Condition) -> Option, - ) -> Option> { - let set = arena.try_alloc_from_iter(self.iter().map(|c| f(c).ok_or(()))).ok()?; - Some(ConditionSet(set)) + ) -> ConditionSet<'a> { + let set = arena.alloc_from_iter(self.iter().filter_map(|c| f(c))); + ConditionSet(set) } -} -impl<'a, 'tcx> TOFinder<'a, 'tcx> { - fn is_empty(&self, state: &State>) -> bool { - state.all_bottom() + fn map(self, arena: &'a DroplessArena, f: impl Fn(Condition) -> Condition) -> ConditionSet<'a> { + let set = arena.alloc_from_iter(self.iter().map(|c| f(c))); + ConditionSet(set) } +} +impl<'a, 'tcx> TOFinder<'a, 'tcx> { /// Recursion entry point to find threading opportunities. #[instrument(level = "trace", skip(self))] fn start_from_switch(&mut self, bb: BasicBlock) { @@ -200,27 +228,24 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { let discr_ty = discr.ty(self.body, self.tcx).ty; let Ok(discr_layout) = self.ecx.layout_of(discr_ty) else { return }; - let Some(discr) = self.map.find(discr.as_ref()) else { return }; + let Some(discr) = self.map.find_value(discr.as_ref()) else { return }; debug!(?discr); let cost = CostChecker::new(self.tcx, self.typing_env, None, self.body); - let mut state = State::new_reachable(); let conds = if let Some((value, then, else_)) = targets.as_static_if() { let Some(value) = ScalarInt::try_from_uint(value, discr_layout.size) else { return }; self.arena.alloc_from_iter([ - Condition { value, polarity: Polarity::Eq, target: then }, - Condition { value, polarity: Polarity::Ne, target: else_ }, + Condition { place: discr, value, polarity: Polarity::Eq, target: then }, + Condition { place: discr, value, polarity: Polarity::Ne, target: else_ }, ]) } else { self.arena.alloc_from_iter(targets.iter().filter_map(|(value, target)| { let value = ScalarInt::try_from_uint(value, discr_layout.size)?; - Some(Condition { value, polarity: Polarity::Eq, target }) + Some(Condition { place: discr, value, polarity: Polarity::Eq, target }) })) }; - let conds = ConditionSet(conds); - state.insert_value_idx(discr, conds, &self.map); - + let state = ConditionSet(conds); self.find_opportunity(bb, state, cost, 0) } @@ -230,7 +255,7 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { fn find_opportunity( &mut self, bb: BasicBlock, - mut state: State>, + mut state: ConditionSet<'a>, mut cost: CostChecker<'_, 'tcx>, depth: usize, ) { @@ -243,7 +268,7 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { for (statement_index, stmt) in self.body.basic_blocks[bb].statements.iter().enumerate().rev() { - if self.is_empty(&state) { + if state.is_empty() { return; } @@ -260,11 +285,11 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { // _1 = 5 // Whatever happens here, it won't change the result of a `SwitchInt`. // _1 = 6 if let Some((lhs, tail)) = self.mutated_statement(stmt) { - state.flood_with_tail_elem(lhs.as_ref(), tail, &self.map, ConditionSet::BOTTOM); + self.flood_state(lhs, tail, &mut state); } } - if self.is_empty(&state) || depth >= MAX_BACKTRACK { + if state.is_empty() || depth >= MAX_BACKTRACK { return; } @@ -280,13 +305,13 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { self.process_switch_int(discr, targets, bb, &mut state); self.find_opportunity(pred, state, cost, depth + 1); } - _ => self.recurse_through_terminator(pred, || state, &cost, depth), + _ => self.recurse_through_terminator(pred, state, &cost, depth), } } else if let &[ref predecessors @ .., last_pred] = &predecessors[..] { for &pred in predecessors { - self.recurse_through_terminator(pred, || state.clone(), &cost, depth); + self.recurse_through_terminator(pred, state, &cost, depth); } - self.recurse_through_terminator(last_pred, || state, &cost, depth); + self.recurse_through_terminator(last_pred, state, &cost, depth); } let new_tos = &mut self.opportunities[last_non_rec..]; @@ -313,6 +338,23 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { } } + /// Remove all conditions in the state that alias given place. + fn flood_state( + &self, + place: Place<'tcx>, + extra_elem: Option, + state: &mut ConditionSet<'a>, + ) { + let mut places_to_exclude = FxHashSet::default(); + self.map.for_each_aliasing_place(place.as_ref(), extra_elem, &mut |vi| { + places_to_exclude.insert(vi); + }); + if places_to_exclude.is_empty() { + return; + } + *state = state.filter(self.arena, |c| !places_to_exclude.contains(&c.place)); + } + /// Extract the mutated place from a statement. /// /// This method returns the `Place` so we can flood the state in case of a partial assignment. @@ -359,17 +401,12 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { bb: BasicBlock, lhs: PlaceIndex, rhs: ImmTy<'tcx>, - state: &mut State>, + state: &mut ConditionSet<'a>, ) { - let register_opportunity = |c: Condition| { - debug!(?bb, ?c.target, "register"); - self.opportunities.push(ThreadingOpportunity { chain: vec![bb], target: c.target }) - }; - - if let Some(conditions) = state.try_get_idx(lhs, &self.map) + if let Some(lhs) = self.map.value(lhs) && let Immediate::Scalar(Scalar::Int(int)) = *rhs { - conditions.iter_matches(int).for_each(register_opportunity); + state.register_matches(lhs, int, Some(bb), &mut self.opportunities); } } @@ -380,7 +417,7 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { bb: BasicBlock, lhs: PlaceIndex, constant: OpTy<'tcx>, - state: &mut State>, + state: &mut ConditionSet<'a>, ) { self.map.for_each_projection_value( lhs, @@ -402,27 +439,38 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { } }, &mut |place, op| { - if let Some(conditions) = state.try_get_idx(place, &self.map) + if let Some(place) = self.map.value(place) && let Some(imm) = self.ecx.read_immediate_raw(op).discard_err() && let Some(imm) = imm.right() && let Immediate::Scalar(Scalar::Int(int)) = *imm { - conditions.iter_matches(int).for_each(|c: Condition| { - self.opportunities - .push(ThreadingOpportunity { chain: vec![bb], target: c.target }) - }) + state.register_matches(place, int, Some(bb), &mut self.opportunities); } }, ); } + #[instrument(level = "trace", skip(self))] + fn process_copy(&mut self, lhs: PlaceIndex, rhs: PlaceIndex, state: &mut ConditionSet<'a>) { + let mut renames = FxHashMap::default(); + self.map.for_each_value_pair(rhs, lhs, &mut |rhs, lhs| { + renames.insert(lhs, rhs); + }); + *state = state.map(self.arena, |mut c| { + if let Some(rhs) = renames.get(&c.place) { + c.place = *rhs + } + c + }); + } + #[instrument(level = "trace", skip(self))] fn process_operand( &mut self, bb: BasicBlock, lhs: PlaceIndex, rhs: &Operand<'tcx>, - state: &mut State>, + state: &mut ConditionSet<'a>, ) { match rhs { // If we expect `lhs ?= A`, we have an opportunity if we assume `constant == A`. @@ -437,7 +485,7 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { // Transfer the conditions on the copied rhs. Operand::Move(rhs) | Operand::Copy(rhs) => { let Some(rhs) = self.map.find(rhs.as_ref()) else { return }; - state.insert_place_idx(rhs, lhs, &self.map); + self.process_copy(lhs, rhs, state) } } } @@ -447,16 +495,16 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { &mut self, bb: BasicBlock, lhs_place: &Place<'tcx>, - rhs: &Rvalue<'tcx>, - state: &mut State>, + rvalue: &Rvalue<'tcx>, + state: &mut ConditionSet<'a>, ) { let Some(lhs) = self.map.find(lhs_place.as_ref()) else { return }; - match rhs { + match rvalue { Rvalue::Use(operand) => self.process_operand(bb, lhs, operand, state), // Transfer the conditions on the copy rhs. Rvalue::Discriminant(rhs) => { let Some(rhs) = self.map.find_discr(rhs.as_ref()) else { return }; - state.insert_place_idx(rhs, lhs, &self.map); + self.process_copy(lhs, rhs, state) } // If we expect `lhs ?= A`, we have an opportunity if we assume `constant == A`. Rvalue::Aggregate(box kind, operands) => { @@ -488,32 +536,30 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { } } // Transfer the conditions on the copy rhs, after inverting the value of the condition. - Rvalue::UnaryOp(UnOp::Not, Operand::Move(place) | Operand::Copy(place)) => { - let layout = self.ecx.layout_of(place.ty(self.body, self.tcx).ty).unwrap(); - let Some(conditions) = state.try_get_idx(lhs, &self.map) else { return }; - let Some(place) = self.map.find(place.as_ref()) else { return }; - let Some(conds) = conditions.map(self.arena, |mut cond| { - cond.value = self - .ecx - .unary_op(UnOp::Not, &ImmTy::from_scalar_int(cond.value, layout)) - .discard_err()? - .to_scalar_int() - .discard_err()?; + Rvalue::UnaryOp(UnOp::Not, Operand::Move(operand) | Operand::Copy(operand)) => { + let layout = self.ecx.layout_of(operand.ty(self.body, self.tcx).ty).unwrap(); + let Some(lhs) = self.map.value(lhs) else { return }; + let Some(operand) = self.map.find_value(operand.as_ref()) else { return }; + *state = state.filter_map(self.arena, |mut cond| { + if cond.place == lhs { + cond.place = operand; + cond.value = self + .ecx + .unary_op(UnOp::Not, &ImmTy::from_scalar_int(cond.value, layout)) + .discard_err()? + .to_scalar_int() + .discard_err()?; + } Some(cond) - }) else { - return; - }; - state.insert_value_idx(place, conds, &self.map); + }); } // We expect `lhs ?= A`. We found `lhs = Eq(rhs, B)`. // Create a condition on `rhs ?= B`. Rvalue::BinaryOp( op, - box (Operand::Move(place) | Operand::Copy(place), Operand::Constant(value)) - | box (Operand::Constant(value), Operand::Move(place) | Operand::Copy(place)), + box (Operand::Move(operand) | Operand::Copy(operand), Operand::Constant(value)) + | box (Operand::Constant(value), Operand::Move(operand) | Operand::Copy(operand)), ) => { - let Some(conditions) = state.try_get_idx(lhs, &self.map) else { return }; - let Some(place) = self.map.find(place.as_ref()) else { return }; let equals = match op { BinOp::Eq => ScalarInt::TRUE, BinOp::Ne => ScalarInt::FALSE, @@ -526,20 +572,28 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { // Avoid handling them, though this could be extended in the future. return; } + let Some(lhs) = self.map.value(lhs) else { return }; + let Some(operand) = self.map.find_value(operand.as_ref()) else { return }; let Some(value) = value.const_.try_eval_scalar_int(self.tcx, self.typing_env) else { return; }; - let Some(conds) = conditions.map(self.arena, |c| { - Some(Condition { - value, - polarity: if c.matches(equals) { Polarity::Eq } else { Polarity::Ne }, - ..c - }) - }) else { - return; - }; - state.insert_value_idx(place, conds, &self.map); + *state = state.map(self.arena, |c| { + if c.place == lhs { + Condition { + place: operand, + value, + polarity: if c.matches(lhs, equals) { + Polarity::Eq + } else { + Polarity::Ne + }, + ..c + } + } else { + c + } + }); } _ => {} @@ -551,13 +605,8 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { &mut self, bb: BasicBlock, stmt: &Statement<'tcx>, - state: &mut State>, + state: &mut ConditionSet<'a>, ) { - let register_opportunity = |c: Condition| { - debug!(?bb, ?c.target, "register"); - self.opportunities.push(ThreadingOpportunity { chain: vec![bb], target: c.target }) - }; - // Below, `lhs` is the return value of `mutated_statement`, // the place to which `conditions` apply. @@ -581,8 +630,8 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume( Operand::Copy(place) | Operand::Move(place), )) => { - let Some(conditions) = state.try_get(place.as_ref(), &self.map) else { return }; - conditions.iter_matches(ScalarInt::TRUE).for_each(register_opportunity) + let Some(place) = self.map.find_value(place.as_ref()) else { return }; + state.register_matches(place, ScalarInt::TRUE, Some(bb), &mut self.opportunities) } StatementKind::Assign(box (lhs_place, rhs)) => { self.process_assign(bb, lhs_place, rhs, state) @@ -595,8 +644,7 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { fn recurse_through_terminator( &mut self, bb: BasicBlock, - // Pass a closure that may clone the state, as we don't want to do it each time. - state: impl FnOnce() -> State>, + mut state: ConditionSet<'a>, cost: &CostChecker<'_, 'tcx>, depth: usize, ) { @@ -627,9 +675,8 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { }; // We can recurse through this terminator. - let mut state = state(); if let Some(place_to_flood) = place_to_flood { - state.flood_with(place_to_flood.as_ref(), &self.map, ConditionSet::BOTTOM); + self.flood_state(place_to_flood, None, &mut state); } self.find_opportunity(bb, state, cost.clone(), depth + 1) } @@ -640,7 +687,7 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { discr: &Operand<'tcx>, targets: &SwitchTargets, target_bb: BasicBlock, - state: &mut State>, + state: &mut ConditionSet<'a>, ) { debug_assert_ne!(target_bb, START_BLOCK); debug_assert_eq!(self.body.basic_blocks.predecessors()[target_bb].len(), 1); @@ -650,7 +697,7 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { let Ok(discr_layout) = self.ecx.layout_of(discr_ty) else { return; }; - let Some(conditions) = state.try_get(discr.as_ref(), &self.map) else { return }; + let Some(discr) = self.map.find_value(discr.as_ref()) else { return }; if let Some((value, _)) = targets.iter().find(|&(_, target)| target == target_bb) { let Some(value) = ScalarInt::try_from_uint(value, discr_layout.size) else { return }; @@ -660,10 +707,10 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { // through the `SwitchInt` before arriving here. Therefore, we know that // `discr == value`. If one condition can be fulfilled by `discr == value`, // that's an opportunity. - for c in conditions.iter_matches(value) { - debug!(?target_bb, ?c.target, "register"); - self.opportunities.push(ThreadingOpportunity { chain: vec![], target: c.target }); - } + // + // The TO starts with `target_bb`, which will be added by `find_opportunity`, so we + // start with an empty bb chain. + state.register_matches(discr, value, None, &mut self.opportunities); } else if let Some((value, _, else_bb)) = targets.as_static_if() && target_bb == else_bb { @@ -672,11 +719,11 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { // We only know that `discr != value`. That's much weaker information than // the equality we had in the previous arm. All we can conclude is that // the replacement condition `discr != value` can be threaded, and nothing else. - for c in conditions.iter() { - if c.value == value && c.polarity == Polarity::Ne { - debug!(?target_bb, ?c.target, "register"); - self.opportunities - .push(ThreadingOpportunity { chain: vec![], target: c.target }); + for c in state.iter() { + if c.place == discr && c.value == value && c.polarity == Polarity::Ne { + // The TO starts with `target_bb`, which will be added by `find_opportunity`, + // so we start with an empty bb chain. + self.opportunities.push(c.into_opportunity(None)); } } } diff --git a/tests/coverage/conditions.cov-map b/tests/coverage/conditions.cov-map index 29d9604085ede..0047d52c89fab 100644 --- a/tests/coverage/conditions.cov-map +++ b/tests/coverage/conditions.cov-map @@ -1,95 +1,75 @@ Function name: conditions::main -Raw bytes (656): 0x[01, 01, 57, 05, 09, 01, 05, 09, 5d, 09, 27, 5d, 61, 27, 65, 5d, 61, 09, 23, 27, 65, 5d, 61, 01, 03, 03, 0d, 11, 51, 11, 4f, 51, 55, 4f, 59, 51, 55, 11, 4b, 4f, 59, 51, 55, 03, 9f, 01, 0d, 11, 0d, 11, 0d, 11, 0d, 11, 0d, 11, 0d, 11, 0d, 11, 9f, 01, 15, 0d, 11, 19, 45, 19, 97, 01, 45, 49, 97, 01, 4d, 45, 49, 19, 93, 01, 97, 01, 4d, 45, 49, 9f, 01, 9b, 02, 0d, 11, 15, 19, 15, 19, 15, 19, 15, 19, 15, 19, 1d, 21, 15, 19, 9b, 02, 1d, 15, 19, 21, 39, 21, e3, 01, 39, 3d, e3, 01, 41, 39, 3d, 21, df, 01, e3, 01, 41, 39, 3d, 9b, 02, d7, 02, 15, 19, 1d, 21, 9b, 02, d7, 02, 15, 19, 1d, 21, 9b, 02, d7, 02, 15, 19, 1d, 21, 9b, 02, d7, 02, 15, 19, 1d, 21, 9b, 02, d7, 02, 15, 19, 1d, 21, 25, 29, 1d, 21, d7, 02, 25, 1d, 21, 29, 2d, 29, cf, 02, 2d, 31, cf, 02, 35, 2d, 31, 29, cb, 02, cf, 02, 35, 2d, 31, d7, 02, db, 02, 1d, 21, 25, 29, 53, 01, 03, 01, 00, 0a, 01, 01, 09, 00, 16, 01, 00, 19, 00, 1a, 01, 01, 08, 00, 0c, 01, 00, 0d, 02, 06, 00, 02, 05, 00, 06, 03, 03, 09, 00, 0a, 01, 00, 10, 00, 1d, 05, 01, 09, 00, 17, 05, 01, 09, 00, 0a, 06, 01, 0f, 00, 1c, 09, 01, 0c, 00, 19, 0a, 00, 1d, 00, 2a, 0e, 00, 2e, 00, 3c, 23, 00, 3d, 02, 0a, 1e, 02, 09, 00, 0a, 09, 01, 09, 00, 17, 09, 01, 09, 00, 12, 2a, 02, 09, 00, 0f, 03, 03, 09, 00, 16, 03, 00, 19, 00, 1a, 03, 01, 08, 00, 0c, 03, 00, 0d, 02, 06, 00, 02, 05, 00, 06, 03, 02, 08, 00, 15, 0d, 00, 16, 02, 06, 2e, 02, 0f, 00, 1c, 11, 01, 0c, 00, 19, 32, 00, 1d, 00, 2a, 36, 00, 2e, 00, 3c, 4b, 00, 3d, 02, 0a, 46, 02, 09, 00, 0a, 11, 01, 09, 00, 17, 52, 02, 09, 00, 0f, 9f, 01, 03, 08, 00, 0c, 9f, 01, 01, 0d, 00, 1a, 9f, 01, 00, 1d, 00, 1e, 9f, 01, 01, 0c, 00, 10, 9f, 01, 00, 11, 02, 0a, 00, 02, 09, 00, 0a, 9f, 01, 02, 0c, 00, 19, 15, 00, 1a, 02, 0a, 72, 04, 11, 00, 1e, 19, 01, 10, 00, 1d, 7a, 00, 21, 00, 2e, 7e, 00, 32, 00, 40, 93, 01, 00, 41, 02, 0e, 8e, 01, 02, 0d, 00, 0e, 19, 01, 0d, 00, 1b, 9a, 01, 02, 0d, 00, 13, 00, 02, 05, 00, 06, 9b, 02, 02, 09, 00, 16, 9b, 02, 00, 19, 00, 1a, 9b, 02, 01, 08, 00, 0c, 9b, 02, 00, 0d, 02, 06, 00, 02, 05, 00, 06, d7, 02, 02, 09, 00, 0a, 9b, 02, 00, 10, 00, 1d, 1d, 00, 1e, 02, 06, be, 01, 02, 0f, 00, 1c, 21, 01, 0c, 00, 19, c6, 01, 00, 1d, 00, 2a, ca, 01, 00, 2e, 00, 3c, df, 01, 00, 3d, 02, 0a, da, 01, 02, 09, 00, 0a, 21, 01, 09, 00, 17, 96, 02, 02, 0d, 00, 20, 96, 02, 00, 23, 00, 2c, 96, 02, 01, 09, 00, 11, 96, 02, 00, 12, 00, 1b, 96, 02, 01, 09, 00, 0f, db, 02, 03, 09, 00, 0a, d7, 02, 00, 10, 00, 1d, 25, 00, 1e, 02, 06, aa, 02, 02, 0f, 00, 1c, 29, 01, 0c, 00, 19, b2, 02, 00, 1d, 00, 2a, b6, 02, 00, 2e, 00, 3c, cb, 02, 00, 3d, 02, 0a, c6, 02, 02, 09, 00, 0a, 29, 01, 09, 00, 17, d2, 02, 02, 09, 00, 0f, 01, 02, 01, 00, 02] +Raw bytes (595): 0x[01, 01, 43, 05, 09, 01, 05, 09, 51, 09, 13, 51, 55, 01, 03, 03, 0d, 11, 49, 11, 27, 49, 4d, 03, 63, 0d, 11, 0d, 11, 0d, 11, 0d, 11, 0d, 11, 0d, 11, 0d, 11, 63, 15, 0d, 11, 19, 41, 19, 5b, 41, 45, 63, cb, 01, 0d, 11, 15, 19, 15, 19, 15, 19, 15, 19, 15, 19, 1d, 21, 15, 19, cb, 01, 1d, 15, 19, 21, 39, 21, 93, 01, 39, 3d, cb, 01, 87, 02, 15, 19, 1d, 21, cb, 01, 87, 02, 15, 19, 1d, 21, cb, 01, 87, 02, 15, 19, 1d, 21, cb, 01, 87, 02, 15, 19, 1d, 21, cb, 01, 87, 02, 15, 19, 1d, 21, 25, 29, 1d, 21, 87, 02, 25, 1d, 21, 29, 2d, 29, ff, 01, 2d, 31, ff, 01, 35, 2d, 31, 29, fb, 01, ff, 01, 35, 2d, 31, 87, 02, 8b, 02, 1d, 21, 25, 29, 53, 01, 03, 01, 00, 0a, 01, 01, 09, 00, 16, 01, 00, 19, 00, 1a, 01, 01, 08, 00, 0c, 01, 00, 0d, 02, 06, 00, 02, 05, 00, 06, 03, 03, 09, 00, 0a, 01, 00, 10, 00, 1d, 05, 01, 09, 00, 17, 05, 01, 09, 00, 0a, 06, 01, 0f, 00, 1c, 09, 01, 0c, 00, 19, 0a, 00, 1d, 00, 2a, 0e, 00, 2e, 00, 3c, 09, 00, 3d, 02, 0a, 00, 02, 09, 00, 0a, 09, 01, 09, 00, 17, 09, 01, 09, 00, 12, 16, 02, 09, 00, 0f, 03, 03, 09, 00, 16, 03, 00, 19, 00, 1a, 03, 01, 08, 00, 0c, 03, 00, 0d, 02, 06, 00, 02, 05, 00, 06, 03, 02, 08, 00, 15, 0d, 00, 16, 02, 06, 1a, 02, 0f, 00, 1c, 11, 01, 0c, 00, 19, 1e, 00, 1d, 00, 2a, 22, 00, 2e, 00, 3c, 11, 00, 3d, 02, 0a, 00, 02, 09, 00, 0a, 11, 01, 09, 00, 17, 2a, 02, 09, 00, 0f, 63, 03, 08, 00, 0c, 63, 01, 0d, 00, 1a, 63, 00, 1d, 00, 1e, 63, 01, 0c, 00, 10, 63, 00, 11, 02, 0a, 00, 02, 09, 00, 0a, 63, 02, 0c, 00, 19, 15, 00, 1a, 02, 0a, 4a, 04, 11, 00, 1e, 19, 01, 10, 00, 1d, 52, 00, 21, 00, 2e, 56, 00, 32, 00, 40, 19, 00, 41, 02, 0e, 00, 02, 0d, 00, 0e, 19, 01, 0d, 00, 1b, 5e, 02, 0d, 00, 13, 00, 02, 05, 00, 06, cb, 01, 02, 09, 00, 16, cb, 01, 00, 19, 00, 1a, cb, 01, 01, 08, 00, 0c, cb, 01, 00, 0d, 02, 06, 00, 02, 05, 00, 06, 87, 02, 02, 09, 00, 0a, cb, 01, 00, 10, 00, 1d, 1d, 00, 1e, 02, 06, 82, 01, 02, 0f, 00, 1c, 21, 01, 0c, 00, 19, 8a, 01, 00, 1d, 00, 2a, 8e, 01, 00, 2e, 00, 3c, 21, 00, 3d, 02, 0a, 00, 02, 09, 00, 0a, 21, 01, 09, 00, 17, c6, 01, 02, 0d, 00, 20, c6, 01, 00, 23, 00, 2c, c6, 01, 01, 09, 00, 11, c6, 01, 00, 12, 00, 1b, c6, 01, 01, 09, 00, 0f, 8b, 02, 03, 09, 00, 0a, 87, 02, 00, 10, 00, 1d, 25, 00, 1e, 02, 06, da, 01, 02, 0f, 00, 1c, 29, 01, 0c, 00, 19, e2, 01, 00, 1d, 00, 2a, e6, 01, 00, 2e, 00, 3c, fb, 01, 00, 3d, 02, 0a, f6, 01, 02, 09, 00, 0a, 29, 01, 09, 00, 17, 82, 02, 02, 09, 00, 0f, 01, 02, 01, 00, 02] Number of files: 1 - file 0 => $DIR/conditions.rs -Number of expressions: 87 +Number of expressions: 67 - expression 0 operands: lhs = Counter(1), rhs = Counter(2) - expression 1 operands: lhs = Counter(0), rhs = Counter(1) -- expression 2 operands: lhs = Counter(2), rhs = Counter(23) -- expression 3 operands: lhs = Counter(2), rhs = Expression(9, Add) -- expression 4 operands: lhs = Counter(23), rhs = Counter(24) -- expression 5 operands: lhs = Expression(9, Add), rhs = Counter(25) -- expression 6 operands: lhs = Counter(23), rhs = Counter(24) -- expression 7 operands: lhs = Counter(2), rhs = Expression(8, Add) -- expression 8 operands: lhs = Expression(9, Add), rhs = Counter(25) -- expression 9 operands: lhs = Counter(23), rhs = Counter(24) -- expression 10 operands: lhs = Counter(0), rhs = Expression(0, Add) -- expression 11 operands: lhs = Expression(0, Add), rhs = Counter(3) -- expression 12 operands: lhs = Counter(4), rhs = Counter(20) -- expression 13 operands: lhs = Counter(4), rhs = Expression(19, Add) -- expression 14 operands: lhs = Counter(20), rhs = Counter(21) -- expression 15 operands: lhs = Expression(19, Add), rhs = Counter(22) -- expression 16 operands: lhs = Counter(20), rhs = Counter(21) -- expression 17 operands: lhs = Counter(4), rhs = Expression(18, Add) -- expression 18 operands: lhs = Expression(19, Add), rhs = Counter(22) -- expression 19 operands: lhs = Counter(20), rhs = Counter(21) -- expression 20 operands: lhs = Expression(0, Add), rhs = Expression(39, Add) -- expression 21 operands: lhs = Counter(3), rhs = Counter(4) -- expression 22 operands: lhs = Counter(3), rhs = Counter(4) -- expression 23 operands: lhs = Counter(3), rhs = Counter(4) +- expression 2 operands: lhs = Counter(2), rhs = Counter(20) +- expression 3 operands: lhs = Counter(2), rhs = Expression(4, Add) +- expression 4 operands: lhs = Counter(20), rhs = Counter(21) +- expression 5 operands: lhs = Counter(0), rhs = Expression(0, Add) +- expression 6 operands: lhs = Expression(0, Add), rhs = Counter(3) +- expression 7 operands: lhs = Counter(4), rhs = Counter(18) +- expression 8 operands: lhs = Counter(4), rhs = Expression(9, Add) +- expression 9 operands: lhs = Counter(18), rhs = Counter(19) +- expression 10 operands: lhs = Expression(0, Add), rhs = Expression(24, Add) +- expression 11 operands: lhs = Counter(3), rhs = Counter(4) +- expression 12 operands: lhs = Counter(3), rhs = Counter(4) +- expression 13 operands: lhs = Counter(3), rhs = Counter(4) +- expression 14 operands: lhs = Counter(3), rhs = Counter(4) +- expression 15 operands: lhs = Counter(3), rhs = Counter(4) +- expression 16 operands: lhs = Counter(3), rhs = Counter(4) +- expression 17 operands: lhs = Counter(3), rhs = Counter(4) +- expression 18 operands: lhs = Expression(24, Add), rhs = Counter(5) +- expression 19 operands: lhs = Counter(3), rhs = Counter(4) +- expression 20 operands: lhs = Counter(6), rhs = Counter(16) +- expression 21 operands: lhs = Counter(6), rhs = Expression(22, Add) +- expression 22 operands: lhs = Counter(16), rhs = Counter(17) +- expression 23 operands: lhs = Expression(24, Add), rhs = Expression(50, Add) - expression 24 operands: lhs = Counter(3), rhs = Counter(4) -- expression 25 operands: lhs = Counter(3), rhs = Counter(4) -- expression 26 operands: lhs = Counter(3), rhs = Counter(4) -- expression 27 operands: lhs = Counter(3), rhs = Counter(4) -- expression 28 operands: lhs = Expression(39, Add), rhs = Counter(5) -- expression 29 operands: lhs = Counter(3), rhs = Counter(4) -- expression 30 operands: lhs = Counter(6), rhs = Counter(17) -- expression 31 operands: lhs = Counter(6), rhs = Expression(37, Add) -- expression 32 operands: lhs = Counter(17), rhs = Counter(18) -- expression 33 operands: lhs = Expression(37, Add), rhs = Counter(19) -- expression 34 operands: lhs = Counter(17), rhs = Counter(18) -- expression 35 operands: lhs = Counter(6), rhs = Expression(36, Add) -- expression 36 operands: lhs = Expression(37, Add), rhs = Counter(19) -- expression 37 operands: lhs = Counter(17), rhs = Counter(18) -- expression 38 operands: lhs = Expression(39, Add), rhs = Expression(70, Add) -- expression 39 operands: lhs = Counter(3), rhs = Counter(4) -- expression 40 operands: lhs = Counter(5), rhs = Counter(6) +- expression 25 operands: lhs = Counter(5), rhs = Counter(6) +- expression 26 operands: lhs = Counter(5), rhs = Counter(6) +- expression 27 operands: lhs = Counter(5), rhs = Counter(6) +- expression 28 operands: lhs = Counter(5), rhs = Counter(6) +- expression 29 operands: lhs = Counter(5), rhs = Counter(6) +- expression 30 operands: lhs = Counter(7), rhs = Counter(8) +- expression 31 operands: lhs = Counter(5), rhs = Counter(6) +- expression 32 operands: lhs = Expression(50, Add), rhs = Counter(7) +- expression 33 operands: lhs = Counter(5), rhs = Counter(6) +- expression 34 operands: lhs = Counter(8), rhs = Counter(14) +- expression 35 operands: lhs = Counter(8), rhs = Expression(36, Add) +- expression 36 operands: lhs = Counter(14), rhs = Counter(15) +- expression 37 operands: lhs = Expression(50, Add), rhs = Expression(65, Add) +- expression 38 operands: lhs = Counter(5), rhs = Counter(6) +- expression 39 operands: lhs = Counter(7), rhs = Counter(8) +- expression 40 operands: lhs = Expression(50, Add), rhs = Expression(65, Add) - expression 41 operands: lhs = Counter(5), rhs = Counter(6) -- expression 42 operands: lhs = Counter(5), rhs = Counter(6) -- expression 43 operands: lhs = Counter(5), rhs = Counter(6) +- expression 42 operands: lhs = Counter(7), rhs = Counter(8) +- expression 43 operands: lhs = Expression(50, Add), rhs = Expression(65, Add) - expression 44 operands: lhs = Counter(5), rhs = Counter(6) - expression 45 operands: lhs = Counter(7), rhs = Counter(8) -- expression 46 operands: lhs = Counter(5), rhs = Counter(6) -- expression 47 operands: lhs = Expression(70, Add), rhs = Counter(7) -- expression 48 operands: lhs = Counter(5), rhs = Counter(6) -- expression 49 operands: lhs = Counter(8), rhs = Counter(14) -- expression 50 operands: lhs = Counter(8), rhs = Expression(56, Add) -- expression 51 operands: lhs = Counter(14), rhs = Counter(15) -- expression 52 operands: lhs = Expression(56, Add), rhs = Counter(16) -- expression 53 operands: lhs = Counter(14), rhs = Counter(15) -- expression 54 operands: lhs = Counter(8), rhs = Expression(55, Add) -- expression 55 operands: lhs = Expression(56, Add), rhs = Counter(16) -- expression 56 operands: lhs = Counter(14), rhs = Counter(15) -- expression 57 operands: lhs = Expression(70, Add), rhs = Expression(85, Add) -- expression 58 operands: lhs = Counter(5), rhs = Counter(6) -- expression 59 operands: lhs = Counter(7), rhs = Counter(8) -- expression 60 operands: lhs = Expression(70, Add), rhs = Expression(85, Add) -- expression 61 operands: lhs = Counter(5), rhs = Counter(6) -- expression 62 operands: lhs = Counter(7), rhs = Counter(8) -- expression 63 operands: lhs = Expression(70, Add), rhs = Expression(85, Add) -- expression 64 operands: lhs = Counter(5), rhs = Counter(6) +- expression 46 operands: lhs = Expression(50, Add), rhs = Expression(65, Add) +- expression 47 operands: lhs = Counter(5), rhs = Counter(6) +- expression 48 operands: lhs = Counter(7), rhs = Counter(8) +- expression 49 operands: lhs = Expression(50, Add), rhs = Expression(65, Add) +- expression 50 operands: lhs = Counter(5), rhs = Counter(6) +- expression 51 operands: lhs = Counter(7), rhs = Counter(8) +- expression 52 operands: lhs = Counter(9), rhs = Counter(10) +- expression 53 operands: lhs = Counter(7), rhs = Counter(8) +- expression 54 operands: lhs = Expression(65, Add), rhs = Counter(9) +- expression 55 operands: lhs = Counter(7), rhs = Counter(8) +- expression 56 operands: lhs = Counter(10), rhs = Counter(11) +- expression 57 operands: lhs = Counter(10), rhs = Expression(63, Add) +- expression 58 operands: lhs = Counter(11), rhs = Counter(12) +- expression 59 operands: lhs = Expression(63, Add), rhs = Counter(13) +- expression 60 operands: lhs = Counter(11), rhs = Counter(12) +- expression 61 operands: lhs = Counter(10), rhs = Expression(62, Add) +- expression 62 operands: lhs = Expression(63, Add), rhs = Counter(13) +- expression 63 operands: lhs = Counter(11), rhs = Counter(12) +- expression 64 operands: lhs = Expression(65, Add), rhs = Expression(66, Add) - expression 65 operands: lhs = Counter(7), rhs = Counter(8) -- expression 66 operands: lhs = Expression(70, Add), rhs = Expression(85, Add) -- expression 67 operands: lhs = Counter(5), rhs = Counter(6) -- expression 68 operands: lhs = Counter(7), rhs = Counter(8) -- expression 69 operands: lhs = Expression(70, Add), rhs = Expression(85, Add) -- expression 70 operands: lhs = Counter(5), rhs = Counter(6) -- expression 71 operands: lhs = Counter(7), rhs = Counter(8) -- expression 72 operands: lhs = Counter(9), rhs = Counter(10) -- expression 73 operands: lhs = Counter(7), rhs = Counter(8) -- expression 74 operands: lhs = Expression(85, Add), rhs = Counter(9) -- expression 75 operands: lhs = Counter(7), rhs = Counter(8) -- expression 76 operands: lhs = Counter(10), rhs = Counter(11) -- expression 77 operands: lhs = Counter(10), rhs = Expression(83, Add) -- expression 78 operands: lhs = Counter(11), rhs = Counter(12) -- expression 79 operands: lhs = Expression(83, Add), rhs = Counter(13) -- expression 80 operands: lhs = Counter(11), rhs = Counter(12) -- expression 81 operands: lhs = Counter(10), rhs = Expression(82, Add) -- expression 82 operands: lhs = Expression(83, Add), rhs = Counter(13) -- expression 83 operands: lhs = Counter(11), rhs = Counter(12) -- expression 84 operands: lhs = Expression(85, Add), rhs = Expression(86, Add) -- expression 85 operands: lhs = Counter(7), rhs = Counter(8) -- expression 86 operands: lhs = Counter(9), rhs = Counter(10) +- expression 66 operands: lhs = Counter(9), rhs = Counter(10) Number of file 0 mappings: 83 - Code(Counter(0)) at (prev + 3, 1) to (start + 0, 10) - Code(Counter(0)) at (prev + 1, 9) to (start + 0, 22) @@ -106,16 +86,14 @@ Number of file 0 mappings: 83 = (c0 - c1) - Code(Counter(2)) at (prev + 1, 12) to (start + 0, 25) - Code(Expression(2, Sub)) at (prev + 0, 29) to (start + 0, 42) - = (c2 - c23) + = (c2 - c20) - Code(Expression(3, Sub)) at (prev + 0, 46) to (start + 0, 60) - = (c2 - (c23 + c24)) -- Code(Expression(8, Add)) at (prev + 0, 61) to (start + 2, 10) - = ((c23 + c24) + c25) -- Code(Expression(7, Sub)) at (prev + 2, 9) to (start + 0, 10) - = (c2 - ((c23 + c24) + c25)) + = (c2 - (c20 + c21)) +- Code(Counter(2)) at (prev + 0, 61) to (start + 2, 10) +- Code(Zero) at (prev + 2, 9) to (start + 0, 10) - Code(Counter(2)) at (prev + 1, 9) to (start + 0, 23) - Code(Counter(2)) at (prev + 1, 9) to (start + 0, 18) -- Code(Expression(10, Sub)) at (prev + 2, 9) to (start + 0, 15) +- Code(Expression(5, Sub)) at (prev + 2, 9) to (start + 0, 15) = (c0 - (c1 + c2)) - Code(Expression(0, Add)) at (prev + 3, 9) to (start + 0, 22) = (c1 + c2) @@ -129,103 +107,97 @@ Number of file 0 mappings: 83 - Code(Expression(0, Add)) at (prev + 2, 8) to (start + 0, 21) = (c1 + c2) - Code(Counter(3)) at (prev + 0, 22) to (start + 2, 6) -- Code(Expression(11, Sub)) at (prev + 2, 15) to (start + 0, 28) +- Code(Expression(6, Sub)) at (prev + 2, 15) to (start + 0, 28) = ((c1 + c2) - c3) - Code(Counter(4)) at (prev + 1, 12) to (start + 0, 25) -- Code(Expression(12, Sub)) at (prev + 0, 29) to (start + 0, 42) - = (c4 - c20) -- Code(Expression(13, Sub)) at (prev + 0, 46) to (start + 0, 60) - = (c4 - (c20 + c21)) -- Code(Expression(18, Add)) at (prev + 0, 61) to (start + 2, 10) - = ((c20 + c21) + c22) -- Code(Expression(17, Sub)) at (prev + 2, 9) to (start + 0, 10) - = (c4 - ((c20 + c21) + c22)) +- Code(Expression(7, Sub)) at (prev + 0, 29) to (start + 0, 42) + = (c4 - c18) +- Code(Expression(8, Sub)) at (prev + 0, 46) to (start + 0, 60) + = (c4 - (c18 + c19)) +- Code(Counter(4)) at (prev + 0, 61) to (start + 2, 10) +- Code(Zero) at (prev + 2, 9) to (start + 0, 10) - Code(Counter(4)) at (prev + 1, 9) to (start + 0, 23) -- Code(Expression(20, Sub)) at (prev + 2, 9) to (start + 0, 15) +- Code(Expression(10, Sub)) at (prev + 2, 9) to (start + 0, 15) = ((c1 + c2) - (c3 + c4)) -- Code(Expression(39, Add)) at (prev + 3, 8) to (start + 0, 12) +- Code(Expression(24, Add)) at (prev + 3, 8) to (start + 0, 12) = (c3 + c4) -- Code(Expression(39, Add)) at (prev + 1, 13) to (start + 0, 26) +- Code(Expression(24, Add)) at (prev + 1, 13) to (start + 0, 26) = (c3 + c4) -- Code(Expression(39, Add)) at (prev + 0, 29) to (start + 0, 30) +- Code(Expression(24, Add)) at (prev + 0, 29) to (start + 0, 30) = (c3 + c4) -- Code(Expression(39, Add)) at (prev + 1, 12) to (start + 0, 16) +- Code(Expression(24, Add)) at (prev + 1, 12) to (start + 0, 16) = (c3 + c4) -- Code(Expression(39, Add)) at (prev + 0, 17) to (start + 2, 10) +- Code(Expression(24, Add)) at (prev + 0, 17) to (start + 2, 10) = (c3 + c4) - Code(Zero) at (prev + 2, 9) to (start + 0, 10) -- Code(Expression(39, Add)) at (prev + 2, 12) to (start + 0, 25) +- Code(Expression(24, Add)) at (prev + 2, 12) to (start + 0, 25) = (c3 + c4) - Code(Counter(5)) at (prev + 0, 26) to (start + 2, 10) -- Code(Expression(28, Sub)) at (prev + 4, 17) to (start + 0, 30) +- Code(Expression(18, Sub)) at (prev + 4, 17) to (start + 0, 30) = ((c3 + c4) - c5) - Code(Counter(6)) at (prev + 1, 16) to (start + 0, 29) -- Code(Expression(30, Sub)) at (prev + 0, 33) to (start + 0, 46) - = (c6 - c17) -- Code(Expression(31, Sub)) at (prev + 0, 50) to (start + 0, 64) - = (c6 - (c17 + c18)) -- Code(Expression(36, Add)) at (prev + 0, 65) to (start + 2, 14) - = ((c17 + c18) + c19) -- Code(Expression(35, Sub)) at (prev + 2, 13) to (start + 0, 14) - = (c6 - ((c17 + c18) + c19)) +- Code(Expression(20, Sub)) at (prev + 0, 33) to (start + 0, 46) + = (c6 - c16) +- Code(Expression(21, Sub)) at (prev + 0, 50) to (start + 0, 64) + = (c6 - (c16 + c17)) +- Code(Counter(6)) at (prev + 0, 65) to (start + 2, 14) +- Code(Zero) at (prev + 2, 13) to (start + 0, 14) - Code(Counter(6)) at (prev + 1, 13) to (start + 0, 27) -- Code(Expression(38, Sub)) at (prev + 2, 13) to (start + 0, 19) +- Code(Expression(23, Sub)) at (prev + 2, 13) to (start + 0, 19) = ((c3 + c4) - (c5 + c6)) - Code(Zero) at (prev + 2, 5) to (start + 0, 6) -- Code(Expression(70, Add)) at (prev + 2, 9) to (start + 0, 22) +- Code(Expression(50, Add)) at (prev + 2, 9) to (start + 0, 22) = (c5 + c6) -- Code(Expression(70, Add)) at (prev + 0, 25) to (start + 0, 26) +- Code(Expression(50, Add)) at (prev + 0, 25) to (start + 0, 26) = (c5 + c6) -- Code(Expression(70, Add)) at (prev + 1, 8) to (start + 0, 12) +- Code(Expression(50, Add)) at (prev + 1, 8) to (start + 0, 12) = (c5 + c6) -- Code(Expression(70, Add)) at (prev + 0, 13) to (start + 2, 6) +- Code(Expression(50, Add)) at (prev + 0, 13) to (start + 2, 6) = (c5 + c6) - Code(Zero) at (prev + 2, 5) to (start + 0, 6) -- Code(Expression(85, Add)) at (prev + 2, 9) to (start + 0, 10) +- Code(Expression(65, Add)) at (prev + 2, 9) to (start + 0, 10) = (c7 + c8) -- Code(Expression(70, Add)) at (prev + 0, 16) to (start + 0, 29) +- Code(Expression(50, Add)) at (prev + 0, 16) to (start + 0, 29) = (c5 + c6) - Code(Counter(7)) at (prev + 0, 30) to (start + 2, 6) -- Code(Expression(47, Sub)) at (prev + 2, 15) to (start + 0, 28) +- Code(Expression(32, Sub)) at (prev + 2, 15) to (start + 0, 28) = ((c5 + c6) - c7) - Code(Counter(8)) at (prev + 1, 12) to (start + 0, 25) -- Code(Expression(49, Sub)) at (prev + 0, 29) to (start + 0, 42) +- Code(Expression(34, Sub)) at (prev + 0, 29) to (start + 0, 42) = (c8 - c14) -- Code(Expression(50, Sub)) at (prev + 0, 46) to (start + 0, 60) +- Code(Expression(35, Sub)) at (prev + 0, 46) to (start + 0, 60) = (c8 - (c14 + c15)) -- Code(Expression(55, Add)) at (prev + 0, 61) to (start + 2, 10) - = ((c14 + c15) + c16) -- Code(Expression(54, Sub)) at (prev + 2, 9) to (start + 0, 10) - = (c8 - ((c14 + c15) + c16)) +- Code(Counter(8)) at (prev + 0, 61) to (start + 2, 10) +- Code(Zero) at (prev + 2, 9) to (start + 0, 10) - Code(Counter(8)) at (prev + 1, 9) to (start + 0, 23) -- Code(Expression(69, Sub)) at (prev + 2, 13) to (start + 0, 32) +- Code(Expression(49, Sub)) at (prev + 2, 13) to (start + 0, 32) = ((c5 + c6) - (c7 + c8)) -- Code(Expression(69, Sub)) at (prev + 0, 35) to (start + 0, 44) +- Code(Expression(49, Sub)) at (prev + 0, 35) to (start + 0, 44) = ((c5 + c6) - (c7 + c8)) -- Code(Expression(69, Sub)) at (prev + 1, 9) to (start + 0, 17) +- Code(Expression(49, Sub)) at (prev + 1, 9) to (start + 0, 17) = ((c5 + c6) - (c7 + c8)) -- Code(Expression(69, Sub)) at (prev + 0, 18) to (start + 0, 27) +- Code(Expression(49, Sub)) at (prev + 0, 18) to (start + 0, 27) = ((c5 + c6) - (c7 + c8)) -- Code(Expression(69, Sub)) at (prev + 1, 9) to (start + 0, 15) +- Code(Expression(49, Sub)) at (prev + 1, 9) to (start + 0, 15) = ((c5 + c6) - (c7 + c8)) -- Code(Expression(86, Add)) at (prev + 3, 9) to (start + 0, 10) +- Code(Expression(66, Add)) at (prev + 3, 9) to (start + 0, 10) = (c9 + c10) -- Code(Expression(85, Add)) at (prev + 0, 16) to (start + 0, 29) +- Code(Expression(65, Add)) at (prev + 0, 16) to (start + 0, 29) = (c7 + c8) - Code(Counter(9)) at (prev + 0, 30) to (start + 2, 6) -- Code(Expression(74, Sub)) at (prev + 2, 15) to (start + 0, 28) +- Code(Expression(54, Sub)) at (prev + 2, 15) to (start + 0, 28) = ((c7 + c8) - c9) - Code(Counter(10)) at (prev + 1, 12) to (start + 0, 25) -- Code(Expression(76, Sub)) at (prev + 0, 29) to (start + 0, 42) +- Code(Expression(56, Sub)) at (prev + 0, 29) to (start + 0, 42) = (c10 - c11) -- Code(Expression(77, Sub)) at (prev + 0, 46) to (start + 0, 60) +- Code(Expression(57, Sub)) at (prev + 0, 46) to (start + 0, 60) = (c10 - (c11 + c12)) -- Code(Expression(82, Add)) at (prev + 0, 61) to (start + 2, 10) +- Code(Expression(62, Add)) at (prev + 0, 61) to (start + 2, 10) = ((c11 + c12) + c13) -- Code(Expression(81, Sub)) at (prev + 2, 9) to (start + 0, 10) +- Code(Expression(61, Sub)) at (prev + 2, 9) to (start + 0, 10) = (c10 - ((c11 + c12) + c13)) - Code(Counter(10)) at (prev + 1, 9) to (start + 0, 23) -- Code(Expression(84, Sub)) at (prev + 2, 9) to (start + 0, 15) +- Code(Expression(64, Sub)) at (prev + 2, 9) to (start + 0, 15) = ((c7 + c8) - (c9 + c10)) - Code(Counter(0)) at (prev + 2, 1) to (start + 0, 2) Highest counter ID seen: c10 diff --git a/tests/mir-opt/jump_threading.rs b/tests/mir-opt/jump_threading.rs index ab8840db3837b..ad46a02b83f40 100644 --- a/tests/mir-opt/jump_threading.rs +++ b/tests/mir-opt/jump_threading.rs @@ -88,7 +88,7 @@ fn two_reads() -> i32 { // CHECK: [[c]] = copy [[a]]; // CHECK: [[tmp:_.*]] = copy [[c]]; // CHECK: [[eq:_.*]] = Eq(move [[tmp]], const 2_i32); - // CHECK: switchInt(move [[eq]]) -> [0: bb2, otherwise: bb1]; + // CHECK: goto -> bb1; // CHECK: bb1: { // CHECK: _0 = const 0_i32; // CHECK: goto -> bb3; diff --git a/tests/mir-opt/jump_threading.two_reads.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.two_reads.JumpThreading.panic-abort.diff index 9962a5724b3f7..090e9d37b4f93 100644 --- a/tests/mir-opt/jump_threading.two_reads.JumpThreading.panic-abort.diff +++ b/tests/mir-opt/jump_threading.two_reads.JumpThreading.panic-abort.diff @@ -29,7 +29,8 @@ StorageLive(_5); _5 = copy _3; _4 = Eq(move _5, const 2_i32); - switchInt(move _4) -> [0: bb2, otherwise: bb1]; +- switchInt(move _4) -> [0: bb2, otherwise: bb1]; ++ goto -> bb1; } bb1: { diff --git a/tests/mir-opt/jump_threading.two_reads.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.two_reads.JumpThreading.panic-unwind.diff index 9962a5724b3f7..090e9d37b4f93 100644 --- a/tests/mir-opt/jump_threading.two_reads.JumpThreading.panic-unwind.diff +++ b/tests/mir-opt/jump_threading.two_reads.JumpThreading.panic-unwind.diff @@ -29,7 +29,8 @@ StorageLive(_5); _5 = copy _3; _4 = Eq(move _5, const 2_i32); - switchInt(move _4) -> [0: bb2, otherwise: bb1]; +- switchInt(move _4) -> [0: bb2, otherwise: bb1]; ++ goto -> bb1; } bb1: { From 9019b8c616674bcc229598741cf11e58b5ba94aa Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Sat, 21 Jun 2025 23:32:54 +0000 Subject: [PATCH 5/9] Do not flood if state is empty. --- compiler/rustc_mir_transform/src/jump_threading.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/compiler/rustc_mir_transform/src/jump_threading.rs b/compiler/rustc_mir_transform/src/jump_threading.rs index 9b420ab18112b..72c003e9c5753 100644 --- a/compiler/rustc_mir_transform/src/jump_threading.rs +++ b/compiler/rustc_mir_transform/src/jump_threading.rs @@ -345,6 +345,9 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { extra_elem: Option, state: &mut ConditionSet<'a>, ) { + if state.is_empty() { + return; + } let mut places_to_exclude = FxHashSet::default(); self.map.for_each_aliasing_place(place.as_ref(), extra_elem, &mut |vi| { places_to_exclude.insert(vi); From be6b5e2c4131faf2373cb74efc33bfff0a610d2d Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Sun, 22 Jun 2025 08:38:50 +0000 Subject: [PATCH 6/9] Skip process_constant if state has no matching value. --- compiler/rustc_mir_dataflow/src/value_analysis.rs | 10 +++++++--- compiler/rustc_mir_transform/src/jump_threading.rs | 4 ++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/compiler/rustc_mir_dataflow/src/value_analysis.rs b/compiler/rustc_mir_dataflow/src/value_analysis.rs index 969e9b3a4dd82..b1ff7ffc60edd 100644 --- a/compiler/rustc_mir_dataflow/src/value_analysis.rs +++ b/compiler/rustc_mir_dataflow/src/value_analysis.rs @@ -750,11 +750,15 @@ impl<'tcx> Map<'tcx> { } } + /// Return the range of value indices inside this place. + pub fn values_inside(&self, root: PlaceIndex) -> &[ValueIndex] { + let range = self.inner_values[root].clone(); + &self.inner_values_buffer[range] + } + /// Invoke a function on each value in the given place and all descendants. fn for_each_value_inside(&self, root: PlaceIndex, f: &mut impl FnMut(ValueIndex)) { - let range = self.inner_values[root].clone(); - let values = &self.inner_values_buffer[range]; - for &v in values { + for &v in self.values_inside(root) { f(v) } } diff --git a/compiler/rustc_mir_transform/src/jump_threading.rs b/compiler/rustc_mir_transform/src/jump_threading.rs index 72c003e9c5753..1e793249c86b4 100644 --- a/compiler/rustc_mir_transform/src/jump_threading.rs +++ b/compiler/rustc_mir_transform/src/jump_threading.rs @@ -422,6 +422,10 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { constant: OpTy<'tcx>, state: &mut ConditionSet<'a>, ) { + let values_inside = self.map.values_inside(lhs); + if !state.iter().any(|cond| values_inside.contains(&cond.place)) { + return; + } self.map.for_each_projection_value( lhs, constant, From 9596dd35871b21c7ee19d81ad4f55aed105c1ac3 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Thu, 19 Jun 2025 21:24:45 +0000 Subject: [PATCH 7/9] Maintain a graph of fulfilled conditions. --- .../rustc_mir_transform/src/jump_threading.rs | 932 ++++++++++-------- ..._conditions.JumpThreading.panic-abort.diff | 335 +++++++ ...conditions.JumpThreading.panic-unwind.diff | 349 +++++++ ...ustom_discr.JumpThreading.panic-abort.diff | 12 +- ...stom_discr.JumpThreading.panic-unwind.diff | 12 +- ...ppearing_bb.JumpThreading.panic-abort.diff | 41 +- ...pearing_bb.JumpThreading.panic-unwind.diff | 41 +- ...icate_chain.JumpThreading.panic-abort.diff | 19 +- ...cate_chain.JumpThreading.panic-unwind.diff | 19 +- ...ng.identity.JumpThreading.panic-abort.diff | 15 +- ...g.identity.JumpThreading.panic-unwind.diff | 15 +- ...tiple_match.JumpThreading.panic-abort.diff | 19 +- ...iple_match.JumpThreading.panic-unwind.diff | 19 +- ...numbered_bb.JumpThreading.panic-abort.diff | 17 +- ...umbered_bb.JumpThreading.panic-unwind.diff | 17 +- tests/mir-opt/jump_threading.rs | 122 ++- ...too_complex.JumpThreading.panic-abort.diff | 11 +- ...oo_complex.JumpThreading.panic-unwind.diff | 11 +- .../derived_ord.demo_le.PreCodegen.after.mir | 44 +- tests/mir-opt/pre-codegen/derived_ord.rs | 33 +- ...ated_loop.PreCodegen.after.panic-abort.mir | 14 +- ...ward_loop.PreCodegen.after.panic-abort.mir | 14 +- ...ard_loop.PreCodegen.after.panic-unwind.mir | 14 +- ...e_const_switch.identity.JumpThreading.diff | 14 +- ...onst_switch.too_complex.JumpThreading.diff | 11 +- 25 files changed, 1546 insertions(+), 604 deletions(-) create mode 100644 tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-abort.diff create mode 100644 tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-unwind.diff diff --git a/compiler/rustc_mir_transform/src/jump_threading.rs b/compiler/rustc_mir_transform/src/jump_threading.rs index 1e793249c86b4..a99297bf2f9ad 100644 --- a/compiler/rustc_mir_transform/src/jump_threading.rs +++ b/compiler/rustc_mir_transform/src/jump_threading.rs @@ -7,46 +7,63 @@ //! ------------/ \-------- ------------ //! //! -//! We proceed by walking the cfg backwards starting from each `SwitchInt` terminator, -//! looking for assignments that will turn the `SwitchInt` into a simple `Goto`. +//! This implementation is heavily inspired by the work outlined in [libfirm]. //! -//! The algorithm maintains a set of replacement conditions: -//! - `conditions[place]` contains `Condition { value, polarity: Eq, target }` -//! if assigning `value` to `place` turns the `SwitchInt` into `Goto { target }`. -//! - `conditions[place]` contains `Condition { value, polarity: Ne, target }` -//! if assigning anything different from `value` to `place` turns the `SwitchInt` -//! into `Goto { target }`. +//! The general algorithm proceeds in two phases: (1) walk the CFG backwards to construct a +//! graph of threading conditions, and (2) propagate fulfilled conditions forward by duplicating +//! blocks. +//! +//! # 1. Condition graph construction //! //! In this file, we denote as `place ?= value` the existence of a replacement condition //! on `place` with given `value`, irrespective of the polarity and target of that //! replacement condition. //! -//! We then walk the CFG backwards transforming the set of conditions. -//! When we find a fulfilling assignment, we record a `ThreadingOpportunity`. -//! All `ThreadingOpportunity`s are applied to the body, by duplicating blocks if required. +//! Inside a block, we associate with each condition `c` a set of targets: +//! - `Goto(target)` if fulfilling `c` changes the terminator into a `Goto { target }`; +//! - `Chain(target, c2)` if fulfilling `c` means that `c2` is fulfilled inside `target`. +//! +//! Before walking a block `bb`, we construct the exit set of condition from its successors. +//! For each condition `c` in a successor `s`, we record that fulfilling `c` in `bb` will fulfill +//! `c` in `s`, as a `Chain(s, c)` condition. +//! +//! When encountering a `switchInt(place) -> [value: bb...]` terminator, we also record a +//! `place == value` condition for each `value`, and associate a `Goto(target)` condition. //! -//! The optimization search can be very heavy, as it performs a DFS on MIR starting from -//! each `SwitchInt` terminator. To manage the complexity, we: -//! - bound the maximum depth by a constant `MAX_BACKTRACK`; -//! - we only traverse `Goto` terminators. +//! Then, we walk the statements backwards, transforming the set of conditions along the way, +//! resulting in a set of conditions at the block entry. //! //! We try to avoid creating irreducible control-flow by not threading through a loop header. //! -//! Likewise, applying the optimisation can create a lot of new MIR, so we bound the instruction +//! Applying the optimisation can create a lot of new MIR, so we bound the instruction //! cost by `MAX_COST`. +//! +//! # 2. Block duplication +//! +//! We now have the set of fulfilled conditions inside each block and their targets. +//! +//! For each block `bb` in reverse postorder, we apply in turn the target associated with each +//! fulfilled condition: +//! - for `Goto(target)`, change the terminator of `bb` into a `Goto { target }`; +//! - for `Chain(target, cond)`, duplicate `target` into a new block which fulfills the same +//! conditions and also fulfills `cond`. This is made efficient by maintaining a map of duplicates, +//! `duplicate[(target, cond)]` to avoid cloning blocks multiple times. +//! +//! [libfirm]: + +use std::cell::OnceCell; -use rustc_arena::DroplessArena; +use itertools::Itertools as _; use rustc_const_eval::const_eval::DummyMachine; use rustc_const_eval::interpret::{ImmTy, Immediate, InterpCx, OpTy, Projectable}; -use rustc_data_structures::fx::{FxHashMap, FxHashSet}; +use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexSet}; use rustc_index::IndexVec; -use rustc_index::bit_set::DenseBitSet; +use rustc_index::bit_set::{DenseBitSet, GrowableBitSet}; use rustc_middle::bug; use rustc_middle::mir::interpret::Scalar; use rustc_middle::mir::visit::Visitor; use rustc_middle::mir::*; use rustc_middle::ty::{self, ScalarInt, TyCtxt}; -use rustc_mir_dataflow::lattice::HasBottom; use rustc_mir_dataflow::value_analysis::{Map, PlaceIndex, TrackElem, ValueIndex}; use rustc_span::DUMMY_SP; use tracing::{debug, instrument, trace}; @@ -55,8 +72,7 @@ use crate::cost_checker::CostChecker; pub(super) struct JumpThreading; -const MAX_BACKTRACK: usize = 5; -const MAX_COST: usize = 100; +const MAX_COST: u8 = 100; const MAX_PLACES: usize = 100; impl<'tcx> crate::MirPass<'tcx> for JumpThreading { @@ -76,33 +92,51 @@ impl<'tcx> crate::MirPass<'tcx> for JumpThreading { } let typing_env = body.typing_env(tcx); - let arena = &DroplessArena::default(); let mut finder = TOFinder { tcx, typing_env, ecx: InterpCx::new(tcx, DUMMY_SP, typing_env, DummyMachine), body, - arena, map: Map::new(tcx, body, Some(MAX_PLACES)), maybe_loop_headers: loops::maybe_loop_headers(body), - opportunities: Vec::new(), + entry_states: IndexVec::from_elem(ConditionSet::default(), &body.basic_blocks), + costs: IndexVec::from_elem(OnceCell::new(), &body.basic_blocks), }; - for (bb, _) in traversal::preorder(body) { - finder.start_from_switch(bb); - } + for (bb, bbdata) in traversal::postorder(body) { + if bbdata.is_cleanup { + continue; + } - let opportunities = finder.opportunities; - debug!(?opportunities); - if opportunities.is_empty() { - return; + let mut state = finder.populate_from_outgoing_edges(bb); + trace!("output_states[{bb:?}] = {state:?}"); + + finder.process_terminator(bb, &mut state); + trace!("pre_terminator_states[{bb:?}] = {state:?}"); + + for stmt in bbdata.statements.iter().rev() { + if state.is_empty() { + break; + } + + finder.process_statement(stmt, &mut state); + + // When a statement mutates a place, assignments to that place that happen + // above the mutation cannot fulfill a condition. + // _1 = 5 // Whatever happens here, it won't change the result of a `SwitchInt`. + // _1 = 6 + if let Some((lhs, tail)) = finder.mutated_statement(stmt) { + finder.flood_state(lhs, tail, &mut state); + } + } + + trace!("entry_states[{bb:?}] = {state:?}"); + finder.entry_states[bb] = state; } - // Verify that we do not thread through a loop header. - for to in opportunities.iter() { - assert!(to.chain.iter().all(|&block| !finder.maybe_loop_headers.contains(block))); + if let Some(opportunities) = OpportunitySet::new(body, finder.entry_states) { + opportunities.apply(); } - OpportunitySet::new(body, opportunities).apply(body); } fn is_required(&self) -> bool { @@ -110,14 +144,6 @@ impl<'tcx> crate::MirPass<'tcx> for JumpThreading { } } -#[derive(Debug)] -struct ThreadingOpportunity { - /// The list of `BasicBlock`s from the one that found the opportunity to the `SwitchInt`. - chain: Vec, - /// The `SwitchInt` will be replaced by `Goto { target }`. - target: BasicBlock, -} - struct TOFinder<'a, 'tcx> { tcx: TyCtxt<'tcx>, typing_env: ty::TypingEnv<'tcx>, @@ -125,22 +151,31 @@ struct TOFinder<'a, 'tcx> { body: &'a Body<'tcx>, map: Map<'tcx>, maybe_loop_headers: DenseBitSet, - /// We use an arena to avoid cloning the slices when cloning `state`. - arena: &'a DroplessArena, - opportunities: Vec, + /// This stores the state of each visited block on entry, + /// and the current state of the block being visited. + // Invariant: for each `bb`, each condition in `entry_states[bb]` has a `chain` that + // starts with `bb`. + entry_states: IndexVec, + /// Pre-computed cost of duplicating each block. + costs: IndexVec>, +} + +rustc_index::newtype_index! { + #[derive(Ord, PartialOrd)] + #[debug_format = "_c{}"] + struct ConditionIndex {} } /// Represent the following statement. If we can prove that the current local is equal/not-equal /// to `value`, jump to `target`. -#[derive(Copy, Clone, Debug)] +#[derive(Copy, Clone, Debug, Hash, Eq, PartialEq)] struct Condition { place: ValueIndex, value: ScalarInt, polarity: Polarity, - target: BasicBlock, } -#[derive(Copy, Clone, Debug, Eq, PartialEq)] +#[derive(Copy, Clone, Debug, Hash, Eq, PartialEq)] enum Polarity { Ne, Eq, @@ -150,192 +185,176 @@ impl Condition { fn matches(&self, place: ValueIndex, value: ScalarInt) -> bool { self.place == place && (self.value == value) == (self.polarity == Polarity::Eq) } - - fn into_opportunity(self, match_bb: Option) -> ThreadingOpportunity { - trace!(?self, "registering"); - let chain = match_bb.into_iter().collect(); - ThreadingOpportunity { chain, target: self.target } - } } -#[derive(Copy, Clone, Debug)] -struct ConditionSet<'a>(&'a [Condition]); +/// Represent the effect of fulfilling a condition. +#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +enum EdgeEffect { + /// If the condition is fulfilled, replace the current block's terminator by a single goto. + Goto { target: BasicBlock }, + /// If the condition is fulfilled, fulfill the condition `succ_condition` in `succ_block`. + Chain { succ_block: BasicBlock, succ_condition: ConditionIndex }, +} -impl HasBottom for ConditionSet<'_> { - const BOTTOM: Self = ConditionSet(&[]); +impl EdgeEffect { + fn block(self) -> BasicBlock { + match self { + EdgeEffect::Goto { target: bb } | EdgeEffect::Chain { succ_block: bb, .. } => bb, + } + } - fn is_bottom(&self) -> bool { - self.0.is_empty() + fn replace_block(&mut self, target: BasicBlock, new_target: BasicBlock) { + match self { + EdgeEffect::Goto { target: bb } | EdgeEffect::Chain { succ_block: bb, .. } => { + if *bb == target { + *bb = new_target + } + } + } } } -impl<'a> ConditionSet<'a> { - fn is_empty(self) -> bool { - self.0.is_empty() +#[derive(Clone, Debug, Default)] +struct ConditionSet { + active: Vec<(ConditionIndex, Condition)>, + fulfilled: Vec, + targets: IndexVec>, + costs: IndexVec, +} + +impl ConditionSet { + fn is_empty(&self) -> bool { + self.active.is_empty() } - fn iter(self) -> impl Iterator { - self.0.iter().copied() + #[tracing::instrument(level = "trace", skip(self))] + fn push_condition(&mut self, c: Condition, target: BasicBlock) { + let index = self.targets.push(vec![EdgeEffect::Goto { target }]); + self.costs.push(0); + self.active.push((index, c)); } - fn iter_matches(self, place: ValueIndex, value: ScalarInt) -> impl Iterator { - self.iter().filter(move |c| c.matches(place, value)) + /// Register fulfilled condition and remove it from the set. + fn fulfill_if(&mut self, f: impl Fn(Condition, &Vec) -> bool) { + self.active.retain(|&(index, condition)| { + let targets = &self.targets[index]; + if f(condition, targets) { + trace!(?index, ?condition, "fulfill"); + self.fulfilled.push(index); + false + } else { + true + } + }) } - fn register_matches( - &self, - place: ValueIndex, - value: ScalarInt, - match_bb: Option, - opportunities: &mut Vec, - ) { - self.iter_matches(place, value) - .for_each(|cond| opportunities.push(cond.into_opportunity(match_bb))) + /// Register fulfilled condition and remove them from the set. + fn fulfill_matches(&mut self, place: ValueIndex, value: ScalarInt) { + self.fulfill_if(|c, _| c.matches(place, value)) } - fn filter(self, arena: &'a DroplessArena, f: impl Fn(Condition) -> bool) -> ConditionSet<'a> { - let set = arena.alloc_from_iter(self.iter().filter(|&c| f(c))); - ConditionSet(set) + fn retain(&mut self, mut f: impl FnMut(Condition) -> bool) { + self.active.retain(|&(_, c)| f(c)) } - fn filter_map( - self, - arena: &'a DroplessArena, - f: impl Fn(Condition) -> Option, - ) -> ConditionSet<'a> { - let set = arena.alloc_from_iter(self.iter().filter_map(|c| f(c))); - ConditionSet(set) + fn retain_mut(&mut self, mut f: impl FnMut(Condition) -> Option) { + self.active.retain_mut(|(_, c)| { + if let Some(new) = f(*c) { + *c = new; + true + } else { + false + } + }) } - fn map(self, arena: &'a DroplessArena, f: impl Fn(Condition) -> Condition) -> ConditionSet<'a> { - let set = arena.alloc_from_iter(self.iter().map(|c| f(c))); - ConditionSet(set) + fn for_each_mut(&mut self, f: impl Fn(&mut Condition)) { + for (_, c) in &mut self.active { + f(c) + } } } impl<'a, 'tcx> TOFinder<'a, 'tcx> { - /// Recursion entry point to find threading opportunities. + /// Construct the condition set for `bb` from the terminator, without executing its effect. #[instrument(level = "trace", skip(self))] - fn start_from_switch(&mut self, bb: BasicBlock) { + fn populate_from_outgoing_edges(&mut self, bb: BasicBlock) -> ConditionSet { let bbdata = &self.body[bb]; - if bbdata.is_cleanup || self.maybe_loop_headers.contains(bb) { - return; - } - let Some((discr, targets)) = bbdata.terminator().kind.as_switch() else { return }; - let Some(discr) = discr.place() else { return }; - debug!(?discr, ?bb); - - let discr_ty = discr.ty(self.body, self.tcx).ty; - let Ok(discr_layout) = self.ecx.layout_of(discr_ty) else { return }; - - let Some(discr) = self.map.find_value(discr.as_ref()) else { return }; - debug!(?discr); - let cost = CostChecker::new(self.tcx, self.typing_env, None, self.body); + // This should be the first time we populate `entry_states[bb]`. + debug_assert!(self.entry_states[bb].is_empty()); - let conds = if let Some((value, then, else_)) = targets.as_static_if() { - let Some(value) = ScalarInt::try_from_uint(value, discr_layout.size) else { return }; - self.arena.alloc_from_iter([ - Condition { place: discr, value, polarity: Polarity::Eq, target: then }, - Condition { place: discr, value, polarity: Polarity::Ne, target: else_ }, - ]) - } else { - self.arena.alloc_from_iter(targets.iter().filter_map(|(value, target)| { - let value = ScalarInt::try_from_uint(value, discr_layout.size)?; - Some(Condition { place: discr, value, polarity: Polarity::Eq, target }) - })) + let state_len = + bbdata.terminator().successors().map(|succ| self.entry_states[succ].active.len()).sum(); + let mut state = ConditionSet { + active: Vec::with_capacity(state_len), + targets: IndexVec::with_capacity(state_len), + fulfilled: Vec::new(), + costs: IndexVec::with_capacity(state_len), }; - let state = ConditionSet(conds); - self.find_opportunity(bb, state, cost, 0) - } - - /// Recursively walk statements backwards from this bb's terminator to find threading - /// opportunities. - #[instrument(level = "trace", skip(self, cost), ret)] - fn find_opportunity( - &mut self, - bb: BasicBlock, - mut state: ConditionSet<'a>, - mut cost: CostChecker<'_, 'tcx>, - depth: usize, - ) { - // Do not thread through loop headers. - if self.maybe_loop_headers.contains(bb) { - return; - } - debug!(cost = ?cost.cost()); - for (statement_index, stmt) in - self.body.basic_blocks[bb].statements.iter().enumerate().rev() - { - if state.is_empty() { - return; + // Use an index-set to deduplicate conditions coming from different successor blocks. + let mut known_conditions = + FxIndexSet::with_capacity_and_hasher(state_len, Default::default()); + let mut insert = |condition, succ_block, succ_condition, cost| { + let (index, new) = known_conditions.insert_full(condition); + let index = ConditionIndex::from_usize(index); + if new { + state.active.push((index, condition)); + let _index = state.targets.push(Vec::new()); + debug_assert_eq!(_index, index); + let _index = state.costs.push(u8::MAX); + debug_assert_eq!(_index, index); } + let target = EdgeEffect::Chain { succ_block, succ_condition }; + debug_assert!( + !state.targets[index].contains(&target), + "duplicate targets for index={index:?} as {target:?} targets={:#?}", + &state.targets[index], + ); + state.targets[index].push(target); + state.costs[index] = std::cmp::min(state.costs[index], cost); + }; - cost.visit_statement(stmt, Location { block: bb, statement_index }); - if cost.cost() > MAX_COST { - return; + // A given block may have several times the same successor. + let mut seen = FxHashSet::default(); + for succ in bbdata.terminator().successors() { + if !seen.insert(succ) { + continue; } - // Attempt to turn the `current_condition` on `lhs` into a condition on another place. - self.process_statement(bb, stmt, &mut state); - - // When a statement mutates a place, assignments to that place that happen - // above the mutation cannot fulfill a condition. - // _1 = 5 // Whatever happens here, it won't change the result of a `SwitchInt`. - // _1 = 6 - if let Some((lhs, tail)) = self.mutated_statement(stmt) { - self.flood_state(lhs, tail, &mut state); + // Do not thread through loop headers. + if self.maybe_loop_headers.contains(succ) { + continue; } - } - - if state.is_empty() || depth >= MAX_BACKTRACK { - return; - } - let last_non_rec = self.opportunities.len(); - - let predecessors = &self.body.basic_blocks.predecessors()[bb]; - if let &[pred] = &predecessors[..] - && bb != START_BLOCK - { - let term = self.body.basic_blocks[pred].terminator(); - match term.kind { - TerminatorKind::SwitchInt { ref discr, ref targets } => { - self.process_switch_int(discr, targets, bb, &mut state); - self.find_opportunity(pred, state, cost, depth + 1); + let succ_cost = self.cost(succ); + for &(succ_index, cond) in self.entry_states[succ].active.iter() { + let cost = self.entry_states[succ].costs[succ_index]; + if let Ok(cost) = ((cost as usize) + succ_cost).try_into() + && cost < MAX_COST + { + insert(cond, succ, succ_index, cost); } - _ => self.recurse_through_terminator(pred, state, &cost, depth), } - } else if let &[ref predecessors @ .., last_pred] = &predecessors[..] { - for &pred in predecessors { - self.recurse_through_terminator(pred, state, &cost, depth); - } - self.recurse_through_terminator(last_pred, state, &cost, depth); } - let new_tos = &mut self.opportunities[last_non_rec..]; - debug!(?new_tos); + let num_conditions = known_conditions.len(); + debug_assert_eq!(num_conditions, state.active.len()); + debug_assert_eq!(num_conditions, state.targets.len()); + debug_assert_eq!(num_conditions, state.costs.len()); + state.fulfilled.reserve(num_conditions); - // Try to deduplicate threading opportunities. - if new_tos.len() > 1 - && new_tos.len() == predecessors.len() - && predecessors - .iter() - .zip(new_tos.iter()) - .all(|(&pred, to)| to.chain == &[pred] && to.target == new_tos[0].target) - { - // All predecessors have a threading opportunity, and they all point to the same block. - debug!(?new_tos, "dedup"); - let first = &mut new_tos[0]; - *first = ThreadingOpportunity { chain: vec![bb], target: first.target }; - self.opportunities.truncate(last_non_rec + 1); - return; - } + state + } - for op in self.opportunities[last_non_rec..].iter_mut() { - op.chain.push(bb); - } + fn cost(&self, bb: BasicBlock) -> usize { + *self.costs[bb].get_or_init(|| { + let bbdata = &self.body[bb]; + let mut cost = CostChecker::new(self.tcx, self.typing_env, None, self.body); + cost.visit_basic_block_data(bb, bbdata); + cost.cost() + }) } /// Remove all conditions in the state that alias given place. @@ -343,7 +362,7 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { &self, place: Place<'tcx>, extra_elem: Option, - state: &mut ConditionSet<'a>, + state: &mut ConditionSet, ) { if state.is_empty() { return; @@ -352,10 +371,11 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { self.map.for_each_aliasing_place(place.as_ref(), extra_elem, &mut |vi| { places_to_exclude.insert(vi); }); + trace!(?places_to_exclude, "flood_state"); if places_to_exclude.is_empty() { return; } - *state = state.filter(self.arena, |c| !places_to_exclude.contains(&c.place)); + state.retain(|c| !places_to_exclude.contains(&c.place)); } /// Extract the mutated place from a statement. @@ -398,32 +418,25 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { } } - #[instrument(level = "trace", skip(self))] - fn process_immediate( - &mut self, - bb: BasicBlock, - lhs: PlaceIndex, - rhs: ImmTy<'tcx>, - state: &mut ConditionSet<'a>, - ) { + #[instrument(level = "trace", skip(self, state))] + fn process_immediate(&mut self, lhs: PlaceIndex, rhs: ImmTy<'tcx>, state: &mut ConditionSet) { if let Some(lhs) = self.map.value(lhs) && let Immediate::Scalar(Scalar::Int(int)) = *rhs { - state.register_matches(lhs, int, Some(bb), &mut self.opportunities); + state.fulfill_matches(lhs, int) } } /// If we expect `lhs ?= A`, we have an opportunity if we assume `constant == A`. - #[instrument(level = "trace", skip(self))] + #[instrument(level = "trace", skip(self, state))] fn process_constant( &mut self, - bb: BasicBlock, lhs: PlaceIndex, constant: OpTy<'tcx>, - state: &mut ConditionSet<'a>, + state: &mut ConditionSet, ) { let values_inside = self.map.values_inside(lhs); - if !state.iter().any(|cond| values_inside.contains(&cond.place)) { + if !state.active.iter().any(|&(_, cond)| values_inside.contains(&cond.place)) { return; } self.map.for_each_projection_value( @@ -451,34 +464,27 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { && let Some(imm) = imm.right() && let Immediate::Scalar(Scalar::Int(int)) = *imm { - state.register_matches(place, int, Some(bb), &mut self.opportunities); + state.fulfill_matches(place, int) } }, ); } - #[instrument(level = "trace", skip(self))] - fn process_copy(&mut self, lhs: PlaceIndex, rhs: PlaceIndex, state: &mut ConditionSet<'a>) { + #[instrument(level = "trace", skip(self, state))] + fn process_copy(&mut self, lhs: PlaceIndex, rhs: PlaceIndex, state: &mut ConditionSet) { let mut renames = FxHashMap::default(); self.map.for_each_value_pair(rhs, lhs, &mut |rhs, lhs| { renames.insert(lhs, rhs); }); - *state = state.map(self.arena, |mut c| { + state.for_each_mut(|c| { if let Some(rhs) = renames.get(&c.place) { c.place = *rhs } - c }); } - #[instrument(level = "trace", skip(self))] - fn process_operand( - &mut self, - bb: BasicBlock, - lhs: PlaceIndex, - rhs: &Operand<'tcx>, - state: &mut ConditionSet<'a>, - ) { + #[instrument(level = "trace", skip(self, state))] + fn process_operand(&mut self, lhs: PlaceIndex, rhs: &Operand<'tcx>, state: &mut ConditionSet) { match rhs { // If we expect `lhs ?= A`, we have an opportunity if we assume `constant == A`. Operand::Constant(constant) => { @@ -487,7 +493,7 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { else { return; }; - self.process_constant(bb, lhs, constant, state); + self.process_constant(lhs, constant, state); } // Transfer the conditions on the copied rhs. Operand::Move(rhs) | Operand::Copy(rhs) => { @@ -497,17 +503,16 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { } } - #[instrument(level = "trace", skip(self))] + #[instrument(level = "trace", skip(self, state))] fn process_assign( &mut self, - bb: BasicBlock, lhs_place: &Place<'tcx>, rvalue: &Rvalue<'tcx>, - state: &mut ConditionSet<'a>, + state: &mut ConditionSet, ) { let Some(lhs) = self.map.find(lhs_place.as_ref()) else { return }; match rvalue { - Rvalue::Use(operand) => self.process_operand(bb, lhs, operand, state), + Rvalue::Use(operand) => self.process_operand(lhs, operand, state), // Transfer the conditions on the copy rhs. Rvalue::Discriminant(rhs) => { let Some(rhs) = self.map.find_discr(rhs.as_ref()) else { return }; @@ -526,7 +531,7 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { .discriminant_for_variant(agg_ty, *variant_index) .discard_err() { - self.process_immediate(bb, discr_target, discr_value, state); + self.process_immediate(discr_target, discr_value, state); } if let Some(idx) = self.map.apply(lhs, TrackElem::Variant(*variant_index)) { idx @@ -538,7 +543,7 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { }; for (field_index, operand) in operands.iter_enumerated() { if let Some(field) = self.map.apply(lhs, TrackElem::Field(field_index)) { - self.process_operand(bb, field, operand, state); + self.process_operand(field, operand, state); } } } @@ -547,17 +552,18 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { let layout = self.ecx.layout_of(operand.ty(self.body, self.tcx).ty).unwrap(); let Some(lhs) = self.map.value(lhs) else { return }; let Some(operand) = self.map.find_value(operand.as_ref()) else { return }; - *state = state.filter_map(self.arena, |mut cond| { - if cond.place == lhs { - cond.place = operand; - cond.value = self + state.retain_mut(|mut c| { + if c.place == lhs { + let value = self .ecx - .unary_op(UnOp::Not, &ImmTy::from_scalar_int(cond.value, layout)) + .unary_op(UnOp::Not, &ImmTy::from_scalar_int(c.value, layout)) .discard_err()? .to_scalar_int() .discard_err()?; + c.place = operand; + c.value = value; } - Some(cond) + Some(c) }); } // We expect `lhs ?= A`. We found `lhs = Eq(rhs, B)`. @@ -585,20 +591,13 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { else { return; }; - *state = state.map(self.arena, |c| { + state.for_each_mut(|c| { if c.place == lhs { - Condition { - place: operand, - value, - polarity: if c.matches(lhs, equals) { - Polarity::Eq - } else { - Polarity::Ne - }, - ..c - } - } else { - c + let polarity = + if c.matches(lhs, equals) { Polarity::Eq } else { Polarity::Ne }; + c.place = operand; + c.value = value; + c.polarity = polarity; } }); } @@ -607,13 +606,8 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { } } - #[instrument(level = "trace", skip(self))] - fn process_statement( - &mut self, - bb: BasicBlock, - stmt: &Statement<'tcx>, - state: &mut ConditionSet<'a>, - ) { + #[instrument(level = "trace", skip(self, state))] + fn process_statement(&mut self, stmt: &Statement<'tcx>, state: &mut ConditionSet) { // Below, `lhs` is the return value of `mutated_statement`, // the place to which `conditions` apply. @@ -631,61 +625,59 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { else { return; }; - self.process_immediate(bb, discr_target, discr, state) + self.process_immediate(discr_target, discr, state) } // If we expect `lhs ?= true`, we have an opportunity if we assume `lhs == true`. StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume( Operand::Copy(place) | Operand::Move(place), )) => { let Some(place) = self.map.find_value(place.as_ref()) else { return }; - state.register_matches(place, ScalarInt::TRUE, Some(bb), &mut self.opportunities) + state.fulfill_matches(place, ScalarInt::TRUE); } StatementKind::Assign(box (lhs_place, rhs)) => { - self.process_assign(bb, lhs_place, rhs, state) + self.process_assign(lhs_place, rhs, state) } _ => {} } } - #[instrument(level = "trace", skip(self, state, cost))] - fn recurse_through_terminator( - &mut self, - bb: BasicBlock, - mut state: ConditionSet<'a>, - cost: &CostChecker<'_, 'tcx>, - depth: usize, - ) { + /// Execute the terminator for block `bb` into state `entry_states[bb]`. + #[instrument(level = "trace", skip(self, state))] + fn process_terminator(&mut self, bb: BasicBlock, state: &mut ConditionSet) { let term = self.body.basic_blocks[bb].terminator(); let place_to_flood = match term.kind { - // We come from a target, so those are not possible. - TerminatorKind::UnwindResume - | TerminatorKind::UnwindTerminate(_) - | TerminatorKind::Return - | TerminatorKind::TailCall { .. } - | TerminatorKind::Unreachable - | TerminatorKind::CoroutineDrop => bug!("{term:?} has no terminators"), // Disallowed during optimizations. TerminatorKind::FalseEdge { .. } | TerminatorKind::FalseUnwind { .. } | TerminatorKind::Yield { .. } => bug!("{term:?} invalid"), // Cannot reason about inline asm. - TerminatorKind::InlineAsm { .. } => return, + TerminatorKind::InlineAsm { .. } => { + state.active.clear(); + return; + } // `SwitchInt` is handled specially. - TerminatorKind::SwitchInt { .. } => return, - // We can recurse, no thing particular to do. - TerminatorKind::Goto { .. } => None, + TerminatorKind::SwitchInt { ref discr, ref targets } => { + return self.process_switch_int(discr, targets, state); + } + // These do not modify memory. + TerminatorKind::UnwindResume + | TerminatorKind::UnwindTerminate(_) + | TerminatorKind::Return + | TerminatorKind::Unreachable + | TerminatorKind::CoroutineDrop + // Assertions can be no-op at codegen time, so treat them as such. + | TerminatorKind::Assert { .. } + | TerminatorKind::Goto { .. } => None, // Flood the overwritten place, and progress through. TerminatorKind::Drop { place: destination, .. } | TerminatorKind::Call { destination, .. } => Some(destination), - // Ignore, as this can be a no-op at codegen time. - TerminatorKind::Assert { .. } => None, + TerminatorKind::TailCall { .. } => Some(RETURN_PLACE.into()), }; - // We can recurse through this terminator. + // This terminator modifies `place_to_flood`, cleanup the associated conditions. if let Some(place_to_flood) = place_to_flood { - self.flood_state(place_to_flood, None, &mut state); + self.flood_state(place_to_flood, None, state); } - self.find_opportunity(bb, state, cost.clone(), depth + 1) } #[instrument(level = "trace", skip(self))] @@ -693,194 +685,262 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { &mut self, discr: &Operand<'tcx>, targets: &SwitchTargets, - target_bb: BasicBlock, - state: &mut ConditionSet<'a>, + state: &mut ConditionSet, ) { - debug_assert_ne!(target_bb, START_BLOCK); - debug_assert_eq!(self.body.basic_blocks.predecessors()[target_bb].len(), 1); - let Some(discr) = discr.place() else { return }; + let Some(discr_idx) = self.map.find_value(discr.as_ref()) else { return }; + let discr_ty = discr.ty(self.body, self.tcx).ty; - let Ok(discr_layout) = self.ecx.layout_of(discr_ty) else { - return; - }; - let Some(discr) = self.map.find_value(discr.as_ref()) else { return }; + let Ok(discr_layout) = self.ecx.layout_of(discr_ty) else { return }; - if let Some((value, _)) = targets.iter().find(|&(_, target)| target == target_bb) { - let Some(value) = ScalarInt::try_from_uint(value, discr_layout.size) else { return }; - debug_assert_eq!(targets.iter().filter(|&(_, target)| target == target_bb).count(), 1); - - // We are inside `target_bb`. Since we have a single predecessor, we know we passed - // through the `SwitchInt` before arriving here. Therefore, we know that - // `discr == value`. If one condition can be fulfilled by `discr == value`, - // that's an opportunity. - // - // The TO starts with `target_bb`, which will be added by `find_opportunity`, so we - // start with an empty bb chain. - state.register_matches(discr, value, None, &mut self.opportunities); - } else if let Some((value, _, else_bb)) = targets.as_static_if() - && target_bb == else_bb - { - let Some(value) = ScalarInt::try_from_uint(value, discr_layout.size) else { return }; + // Attempt to fulfill a condition using an outgoing branch's condition. + // Only support the case where there are no duplicated outgoing edges. + if targets.is_distinct() { + for &(index, c) in state.active.iter() { + if c.place != discr_idx { + continue; + } - // We only know that `discr != value`. That's much weaker information than - // the equality we had in the previous arm. All we can conclude is that - // the replacement condition `discr != value` can be threaded, and nothing else. - for c in state.iter() { - if c.place == discr && c.value == value && c.polarity == Polarity::Ne { - // The TO starts with `target_bb`, which will be added by `find_opportunity`, - // so we start with an empty bb chain. - self.opportunities.push(c.into_opportunity(None)); + // Set of blocks `t` such that the edge `bb -> t` fulfills `c`. + let mut edges_fulfilling_condition = FxHashSet::default(); + + // On edge `bb -> tgt`, we know that `discr_idx == branch`. + for (branch, tgt) in targets.iter() { + if let Some(branch) = ScalarInt::try_from_uint(branch, discr_layout.size) + && c.matches(discr_idx, branch) + { + edges_fulfilling_condition.insert(tgt); + } } - } - } - } -} -struct OpportunitySet { - opportunities: Vec, - /// For each bb, give the TOs in which it appears. The pair corresponds to the index - /// in `opportunities` and the index in `ThreadingOpportunity::chain`. - involving_tos: IndexVec>, - /// Cache the number of predecessors for each block, as we clear the basic block cache.. - predecessors: IndexVec, -} + // On edge `bb -> otherwise`, we only know that `discr` is different from all the + // constants in the switch. That's much weaker information than the equality we + // had in the previous arm. All we can conclude is that the replacement condition + // `discr != value` can be threaded, and nothing else. + if c.polarity == Polarity::Ne + && let Ok(value) = c.value.try_to_bits(discr_layout.size) + && targets.all_values().contains(&value.into()) + { + edges_fulfilling_condition.insert(targets.otherwise()); + } -impl OpportunitySet { - fn new(body: &Body<'_>, opportunities: Vec) -> OpportunitySet { - let mut involving_tos = IndexVec::from_elem(Vec::new(), &body.basic_blocks); - for (index, to) in opportunities.iter().enumerate() { - for (ibb, &bb) in to.chain.iter().enumerate() { - involving_tos[bb].push((index, ibb)); + // Register that jumping to a `t` fulfills condition `c`. + // This does *not* mean that `c` is fulfilled in this block: inserting `index` in + // `fulfilled` is wrong if we have targets that jump to other blocks. + let condition_targets = &state.targets[index]; + + let new_edges: Vec<_> = condition_targets + .iter() + .copied() + .filter(|&target| match target { + EdgeEffect::Goto { .. } => false, + EdgeEffect::Chain { succ_block, .. } => { + edges_fulfilling_condition.contains(&succ_block) + } + }) + .collect(); + + if new_edges.len() == condition_targets.len() { + // If `new_edges == condition_targets`, do not bother creating a new + // `ConditionIndex`, we can use the existing one. + state.fulfilled.push(index); + } else { + // Fulfilling `index` may thread conditions that we do not want, + // so create a brand new index to immediately mark fulfilled. + let index = state.targets.push(new_edges); + let _index = state.costs.push(0); + debug_assert_eq!(_index, index); + state.fulfilled.push(index); + } } - involving_tos[to.target].push((index, to.chain.len())); } - let predecessors = predecessor_count(body); - OpportunitySet { opportunities, involving_tos, predecessors } - } - /// Apply the opportunities on the graph. - fn apply(&mut self, body: &mut Body<'_>) { - for i in 0..self.opportunities.len() { - self.apply_once(i, body); + // Introduce additional conditions of the form `discr ?= value` for each value in targets. + let mut mk_condition = |value, polarity, target| { + let c = Condition { place: discr_idx, value, polarity }; + state.push_condition(c, target); + }; + if let Some((value, then_, else_)) = targets.as_static_if() { + // We have an `if`, generate both `discr == value` and `discr != value`. + let Some(value) = ScalarInt::try_from_uint(value, discr_layout.size) else { return }; + mk_condition(value, Polarity::Eq, then_); + mk_condition(value, Polarity::Ne, else_); + } else { + // We have a general switch and we cannot express `discr != value0 && discr != value1`, + // so we only generate equality predicates. + for (value, target) in targets.iter() { + if let Some(value) = ScalarInt::try_from_uint(value, discr_layout.size) { + mk_condition(value, Polarity::Eq, target); + } + } } } +} - #[instrument(level = "trace", skip(self, body))] - fn apply_once(&mut self, index: usize, body: &mut Body<'_>) { - debug!(?self.predecessors); - debug!(?self.involving_tos); +struct OpportunitySet<'a, 'tcx> { + basic_blocks: &'a mut IndexVec>, + entry_states: IndexVec, + /// Cache duplicated block. When cloning a basic block `bb` to fulfill a condition `c`, + /// record the target of this `bb with c` edge. + duplicates: FxHashMap<(BasicBlock, ConditionIndex), BasicBlock>, +} - // Check that `predecessors` satisfies its invariant. - debug_assert_eq!(self.predecessors, predecessor_count(body)); +impl<'a, 'tcx> OpportunitySet<'a, 'tcx> { + fn new( + body: &'a mut Body<'tcx>, + mut entry_states: IndexVec, + ) -> Option> { + trace!(def_id = ?body.source.def_id(), "apply"); - // Remove the TO from the vector to allow modifying the other ones later. - let op = &mut self.opportunities[index]; - debug!(?op); - let op_chain = std::mem::take(&mut op.chain); - let op_target = op.target; - debug_assert_eq!(op_chain.len(), op_chain.iter().collect::>().len()); + if entry_states.iter().all(|state| state.fulfilled.is_empty()) { + return None; + } - let Some((current, chain)) = op_chain.split_first() else { return }; + // Free some memory, because we will need to clone condition sets. + for state in entry_states.iter_mut() { + state.active = Default::default(); + state.costs = Default::default(); + } + let duplicates = Default::default(); let basic_blocks = body.basic_blocks.as_mut(); + Some(OpportunitySet { basic_blocks, entry_states, duplicates }) + } - // Invariant: the control-flow is well-formed at the end of each iteration. - let mut current = *current; - for &succ in chain { - debug!(?current, ?succ); + /// Apply the opportunities on the graph. + #[instrument(level = "debug", skip(self))] + fn apply(mut self) { + let mut worklist = Vec::with_capacity(self.basic_blocks.len()); + worklist.push(START_BLOCK); - // `succ` must be a successor of `current`. If it is not, this means this TO is not - // satisfiable and a previous TO erased this edge, so we bail out. - if !basic_blocks[current].terminator().successors().any(|s| s == succ) { - debug!("impossible"); - return; - } + // Use a `GrowableBitSet` and not a `DenseBitSet` as we are adding blocks. + let mut visited = GrowableBitSet::with_capacity(self.basic_blocks.len()); - // Fast path: `succ` is only used once, so we can reuse it directly. - if self.predecessors[succ] == 1 { - debug!("single"); - current = succ; + while let Some(bb) = worklist.pop() { + if !visited.insert(bb) { continue; } - let new_succ = basic_blocks.push(basic_blocks[succ].clone()); - debug!(?new_succ); + self.apply_once(bb); - // Replace `succ` by `new_succ` where it appears. - let mut num_edges = 0; - basic_blocks[current].terminator_mut().successors_mut(|s| { - if *s == succ { - *s = new_succ; - num_edges += 1; - } - }); - - // Update predecessors with the new block. - let _new_succ = self.predecessors.push(num_edges); - debug_assert_eq!(new_succ, _new_succ); - self.predecessors[succ] -= num_edges; - self.update_predecessor_count(basic_blocks[new_succ].terminator(), Update::Incr); - - // Replace the `current -> succ` edge by `current -> new_succ` in all the following - // TOs. This is necessary to avoid trying to thread through a non-existing edge. We - // use `involving_tos` here to avoid traversing the full set of TOs on each iteration. - let mut new_involved = Vec::new(); - for &(to_index, in_to_index) in &self.involving_tos[current] { - // That TO has already been applied, do nothing. - if to_index <= index { - continue; - } + // `apply_once` may have modified the terminator of `bb`. + // Only visit actual successors. + worklist.extend(self.basic_blocks[bb].terminator().successors()); + } + } - let other_to = &mut self.opportunities[to_index]; - if other_to.chain.get(in_to_index) != Some(¤t) { - continue; + /// Apply the opportunities on `bb`. + #[instrument(level = "debug", skip(self))] + fn apply_once(&mut self, bb: BasicBlock) { + let state = &mut self.entry_states[bb]; + trace!(?state); + + // We are modifying the `bb` in-place. Once a `EdgeEffect` has been applied, + // it does not need to be applied again. + let mut targets: Vec<_> = state + .fulfilled + .iter() + .flat_map(|&index| std::mem::take(&mut state.targets[index])) + .collect(); + targets.sort(); + targets.dedup(); + trace!(?targets); + + // Use a while-pop to allow modifying `targets` from inside the loop. + targets.reverse(); + while let Some(target) = targets.pop() { + debug!(?target); + trace!(term = ?self.basic_blocks[bb].terminator().kind); + + // By construction, `target.block()` is a successor of `bb`. + // When applying targets, we may change the set of successors. + // The match below updates the set of targets for consistency. + debug_assert!( + self.basic_blocks[bb].terminator().successors().contains(&target.block()), + "missing {target:?} in successors for {bb:?}, term={:?}", + self.basic_blocks[bb].terminator(), + ); + + match target { + EdgeEffect::Goto { target } => { + self.apply_goto(bb, target); + + // We now have `target` as single successor. Drop all other target blocks. + targets.retain(|t| t.block() == target); + // Also do this on targets that may be applied by a duplicate of `bb`. + for ts in self.entry_states[bb].targets.iter_mut() { + ts.retain(|t| t.block() == target); + } } - let s = other_to.chain.get_mut(in_to_index + 1).unwrap_or(&mut other_to.target); - if *s == succ { - // `other_to` references the `current -> succ` edge, so replace `succ`. - *s = new_succ; - new_involved.push((to_index, in_to_index + 1)); + EdgeEffect::Chain { succ_block, succ_condition } => { + let new_succ_block = self.apply_chain(bb, succ_block, succ_condition); + + // We have a new name for `target`, ensure it is correctly applied. + if let Some(new_succ_block) = new_succ_block { + for t in targets.iter_mut() { + t.replace_block(succ_block, new_succ_block) + } + // Also do this on targets that may be applied by a duplicate of `bb`. + for t in + self.entry_states[bb].targets.iter_mut().flat_map(|ts| ts.iter_mut()) + { + t.replace_block(succ_block, new_succ_block) + } + } } } - // The TOs that we just updated now reference `new_succ`. Update `involving_tos` - // in case we need to duplicate an edge starting at `new_succ` later. - let _new_succ = self.involving_tos.push(new_involved); - debug_assert_eq!(new_succ, _new_succ); - - current = new_succ; + trace!(post_term = ?self.basic_blocks[bb].terminator().kind); } + } - let current = &mut basic_blocks[current]; - self.update_predecessor_count(current.terminator(), Update::Decr); - current.terminator_mut().kind = TerminatorKind::Goto { target: op_target }; - self.predecessors[op_target] += 1; + #[instrument(level = "debug", skip(self))] + fn apply_goto(&mut self, bb: BasicBlock, target: BasicBlock) { + self.basic_blocks[bb].terminator_mut().kind = TerminatorKind::Goto { target }; } - fn update_predecessor_count(&mut self, terminator: &Terminator<'_>, incr: Update) { - match incr { - Update::Incr => { - for s in terminator.successors() { - self.predecessors[s] += 1; - } - } - Update::Decr => { - for s in terminator.successors() { - self.predecessors[s] -= 1; - } - } + #[instrument(level = "debug", skip(self), ret)] + fn apply_chain( + &mut self, + bb: BasicBlock, + target: BasicBlock, + condition: ConditionIndex, + ) -> Option { + if self.entry_states[target].fulfilled.contains(&condition) { + // `target` already fulfills `condition`, so we do not need to thread anything. + trace!("fulfilled"); + return None; } - } -} -fn predecessor_count(body: &Body<'_>) -> IndexVec { - let mut predecessors: IndexVec<_, _> = - body.basic_blocks.predecessors().iter().map(|ps| ps.len()).collect(); - predecessors[START_BLOCK] += 1; // Account for the implicit entry edge. - predecessors -} + // We may be tempted to modify `target` in-place to avoid a clone. This is wrong. + // We may still have edges from other blocks to `target` that have not been created yet. + // For instance because we may be threading an edge coming from `bb`, + // or `target` may be a block duplicate for which we may still create predecessors. + + let new_target = *self.duplicates.entry((target, condition)).or_insert_with(|| { + // If we already have a duplicate of `target` which fulfills `condition`, reuse it. + // Otherwise, we clone a new bb to such ends. + let new_target = self.basic_blocks.push(self.basic_blocks[target].clone()); + trace!(?target, ?new_target, ?condition, "clone"); + + // By definition, `new_target` fulfills the same condition as `target`, with + // `condition` added. + let mut condition_set = self.entry_states[target].clone(); + condition_set.fulfilled.push(condition); + let _new_target = self.entry_states.push(condition_set); + debug_assert_eq!(new_target, _new_target); + + new_target + }); + trace!(?target, ?new_target, ?condition, "reuse"); -enum Update { - Incr, - Decr, + // Replace `target` by `new_target` where it appears. + // This changes exactly `direct_count` edges. + self.basic_blocks[bb].terminator_mut().successors_mut(|s| { + if *s == target { + *s = new_target; + } + }); + + Some(new_target) + } } diff --git a/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-abort.diff new file mode 100644 index 0000000000000..4c1f25ada0760 --- /dev/null +++ b/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-abort.diff @@ -0,0 +1,335 @@ +- // MIR for `chained_conditions` before JumpThreading ++ // MIR for `chained_conditions` after JumpThreading + + fn chained_conditions() -> u8 { + let mut _0: u8; + let _1: chained_conditions::BacktraceStyle; + let mut _2: std::option::Option; + let mut _3: &std::option::Option; + let mut _4: isize; + let _5: std::string::String; + let _6: &std::string::String; + let mut _7: bool; + let mut _8: &&std::string::String; + let _9: &std::string::String; + let mut _10: &&str; + let _11: &str; + let _12: std::string::String; + let _13: &std::string::String; + let mut _14: bool; + let mut _15: &&std::string::String; + let _16: &std::string::String; + let mut _17: &&str; + let _18: &str; + let mut _19: isize; + let mut _20: &&str; + let mut _21: &&str; + let mut _22: bool; + let mut _23: bool; + let mut _24: isize; + let mut _25: isize; + let mut _26: isize; + scope 1 { + debug format => _1; + } + scope 2 { + debug x => _5; + debug x => _6; + } + scope 3 { + debug x => _12; + debug x => _13; + } + scope 4 (inlined std::cmp::impls:: for &String>::eq) { + let mut _27: &std::string::String; + let mut _28: &str; + } + scope 5 (inlined std::cmp::impls:: for &String>::eq) { + let mut _29: &std::string::String; + let mut _30: &str; + } + + bb0: { + _22 = const false; + _23 = const false; + StorageLive(_1); + StorageLive(_2); + _2 = env_var() -> [return: bb1, unwind unreachable]; + } + + bb1: { + _22 = const true; + _23 = const true; + _4 = discriminant(_2); +- switchInt(move _4) -> [0: bb3, 1: bb4, otherwise: bb2]; ++ switchInt(move _4) -> [0: bb21, 1: bb4, otherwise: bb2]; + } + + bb2: { + unreachable; + } + + bb3: { + _1 = chained_conditions::BacktraceStyle::Off; + goto -> bb18; + } + + bb4: { + StorageLive(_6); + _6 = &((_2 as Some).0: std::string::String); + StorageLive(_7); + StorageLive(_8); + StorageLive(_9); + _9 = &(*_6); + _8 = &_9; + StorageLive(_10); + _21 = const chained_conditions::promoted[1]; + _10 = &(*_21); + StorageLive(_27); + StorageLive(_28); + _27 = copy (*_8); + _28 = copy (*_10); + _7 = >::eq(move _27, move _28) -> [return: bb19, unwind unreachable]; + } + + bb5: { + StorageDead(_10); + StorageDead(_9); + StorageDead(_8); + StorageDead(_7); + StorageLive(_5); + _23 = const false; + _5 = move ((_2 as Some).0: std::string::String); + _1 = chained_conditions::BacktraceStyle::Full; +- drop(_5) -> [return: bb7, unwind unreachable]; ++ drop(_5) -> [return: bb23, unwind unreachable]; + } + + bb6: { + StorageDead(_10); + StorageDead(_9); + StorageDead(_8); + StorageDead(_7); + StorageDead(_6); + StorageLive(_13); + _13 = &((_2 as Some).0: std::string::String); + StorageLive(_14); + StorageLive(_15); + StorageLive(_16); + _16 = &(*_13); + _15 = &_16; + StorageLive(_17); + _20 = const chained_conditions::promoted[0]; + _17 = &(*_20); + StorageLive(_29); + StorageLive(_30); + _29 = copy (*_15); + _30 = copy (*_17); + _14 = >::eq(move _29, move _30) -> [return: bb20, unwind unreachable]; + } + + bb7: { + StorageDead(_5); + StorageDead(_6); + goto -> bb18; + } + + bb8: { + StorageDead(_17); + StorageDead(_16); + StorageDead(_15); + StorageDead(_14); + StorageLive(_12); + _23 = const false; + _12 = move ((_2 as Some).0: std::string::String); + _1 = chained_conditions::BacktraceStyle::Off; +- drop(_12) -> [return: bb10, unwind unreachable]; ++ drop(_12) -> [return: bb30, unwind unreachable]; + } + + bb9: { + StorageDead(_17); + StorageDead(_16); + StorageDead(_15); + StorageDead(_14); + StorageDead(_13); + _1 = chained_conditions::BacktraceStyle::Short; +- goto -> bb18; ++ goto -> bb36; + } + + bb10: { + StorageDead(_12); + StorageDead(_13); + goto -> bb18; + } + + bb11: { + _0 = const 3_u8; + goto -> bb14; + } + + bb12: { + _0 = const 2_u8; + goto -> bb14; + } + + bb13: { + _0 = const 1_u8; + goto -> bb14; + } + + bb14: { + StorageDead(_1); + return; + } + + bb15: { + _22 = const false; + _23 = const false; + StorageDead(_2); + _19 = discriminant(_1); + switchInt(move _19) -> [0: bb13, 1: bb12, 2: bb11, otherwise: bb2]; + } + + bb16: { + switchInt(copy _23) -> [0: bb15, otherwise: bb17]; + } + + bb17: { + drop(((_2 as Some).0: std::string::String)) -> [return: bb15, unwind unreachable]; + } + + bb18: { + _24 = discriminant(_2); + switchInt(move _24) -> [1: bb16, otherwise: bb15]; + } + + bb19: { + StorageDead(_28); + StorageDead(_27); + switchInt(move _7) -> [0: bb6, otherwise: bb5]; + } + + bb20: { + StorageDead(_30); + StorageDead(_29); + switchInt(move _14) -> [0: bb9, otherwise: bb8]; ++ } ++ ++ bb21: { ++ _1 = chained_conditions::BacktraceStyle::Off; ++ goto -> bb40; ++ } ++ ++ bb22: { ++ StorageDead(_5); ++ StorageDead(_6); ++ goto -> bb18; ++ } ++ ++ bb23: { ++ StorageDead(_5); ++ StorageDead(_6); ++ goto -> bb25; ++ } ++ ++ bb24: { ++ _24 = discriminant(_2); ++ switchInt(move _24) -> [1: bb16, otherwise: bb15]; ++ } ++ ++ bb25: { ++ _24 = discriminant(_2); ++ switchInt(move _24) -> [1: bb28, otherwise: bb26]; ++ } ++ ++ bb26: { ++ _22 = const false; ++ _23 = const false; ++ StorageDead(_2); ++ _19 = discriminant(_1); ++ goto -> bb11; ++ } ++ ++ bb27: { ++ switchInt(copy _23) -> [0: bb15, otherwise: bb17]; ++ } ++ ++ bb28: { ++ goto -> bb26; ++ } ++ ++ bb29: { ++ StorageDead(_12); ++ StorageDead(_13); ++ goto -> bb18; ++ } ++ ++ bb30: { ++ StorageDead(_12); ++ StorageDead(_13); ++ goto -> bb32; ++ } ++ ++ bb31: { ++ _24 = discriminant(_2); ++ switchInt(move _24) -> [1: bb16, otherwise: bb15]; ++ } ++ ++ bb32: { ++ _24 = discriminant(_2); ++ switchInt(move _24) -> [1: bb35, otherwise: bb33]; ++ } ++ ++ bb33: { ++ _22 = const false; ++ _23 = const false; ++ StorageDead(_2); ++ _19 = discriminant(_1); ++ goto -> bb13; ++ } ++ ++ bb34: { ++ switchInt(copy _23) -> [0: bb15, otherwise: bb17]; ++ } ++ ++ bb35: { ++ goto -> bb33; ++ } ++ ++ bb36: { ++ _24 = discriminant(_2); ++ switchInt(move _24) -> [1: bb38, otherwise: bb37]; ++ } ++ ++ bb37: { ++ _22 = const false; ++ _23 = const false; ++ StorageDead(_2); ++ _19 = discriminant(_1); ++ goto -> bb12; ++ } ++ ++ bb38: { ++ switchInt(copy _23) -> [0: bb37, otherwise: bb39]; ++ } ++ ++ bb39: { ++ drop(((_2 as Some).0: std::string::String)) -> [return: bb37, unwind unreachable]; ++ } ++ ++ bb40: { ++ _24 = discriminant(_2); ++ switchInt(move _24) -> [1: bb41, otherwise: bb33]; ++ } ++ ++ bb41: { ++ goto -> bb42; ++ } ++ ++ bb42: { ++ drop(((_2 as Some).0: std::string::String)) -> [return: bb33, unwind unreachable]; + } + } + diff --git a/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-unwind.diff new file mode 100644 index 0000000000000..e45cb1714a99a --- /dev/null +++ b/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-unwind.diff @@ -0,0 +1,349 @@ +- // MIR for `chained_conditions` before JumpThreading ++ // MIR for `chained_conditions` after JumpThreading + + fn chained_conditions() -> u8 { + let mut _0: u8; + let _1: chained_conditions::BacktraceStyle; + let mut _2: std::option::Option; + let mut _3: &std::option::Option; + let mut _4: isize; + let _5: std::string::String; + let _6: &std::string::String; + let mut _7: bool; + let mut _8: &&std::string::String; + let _9: &std::string::String; + let mut _10: &&str; + let _11: &str; + let _12: std::string::String; + let _13: &std::string::String; + let mut _14: bool; + let mut _15: &&std::string::String; + let _16: &std::string::String; + let mut _17: &&str; + let _18: &str; + let mut _19: isize; + let mut _20: &&str; + let mut _21: &&str; + let mut _22: bool; + let mut _23: bool; + let mut _24: isize; + let mut _25: isize; + let mut _26: isize; + scope 1 { + debug format => _1; + } + scope 2 { + debug x => _5; + debug x => _6; + } + scope 3 { + debug x => _12; + debug x => _13; + } + scope 4 (inlined std::cmp::impls:: for &String>::eq) { + let mut _27: &std::string::String; + let mut _28: &str; + } + scope 5 (inlined std::cmp::impls:: for &String>::eq) { + let mut _29: &std::string::String; + let mut _30: &str; + } + + bb0: { + _22 = const false; + _23 = const false; + StorageLive(_1); + StorageLive(_2); + _22 = const true; + _23 = const true; +- _2 = env_var() -> [return: bb1, unwind continue]; ++ _2 = env_var() -> [return: bb25, unwind continue]; + } + + bb1: { + _4 = discriminant(_2); + switchInt(move _4) -> [0: bb3, 1: bb4, otherwise: bb2]; + } + + bb2: { + unreachable; + } + + bb3: { + _1 = chained_conditions::BacktraceStyle::Off; + goto -> bb19; + } + + bb4: { + StorageLive(_6); + _6 = &((_2 as Some).0: std::string::String); + StorageLive(_7); + StorageLive(_8); + StorageLive(_9); + _9 = &(*_6); + _8 = &_9; + StorageLive(_10); + _21 = const chained_conditions::promoted[1]; + _10 = &(*_21); + StorageLive(_27); + StorageLive(_28); + _27 = copy (*_8); + _28 = copy (*_10); + _7 = >::eq(move _27, move _28) -> [return: bb23, unwind: bb22]; + } + + bb5: { + StorageDead(_10); + StorageDead(_9); + StorageDead(_8); + StorageDead(_7); + StorageLive(_5); + _23 = const false; + _5 = move ((_2 as Some).0: std::string::String); + _1 = chained_conditions::BacktraceStyle::Full; +- drop(_5) -> [return: bb7, unwind: bb22]; ++ drop(_5) -> [return: bb28, unwind: bb22]; + } + + bb6: { + StorageDead(_10); + StorageDead(_9); + StorageDead(_8); + StorageDead(_7); + StorageDead(_6); + StorageLive(_13); + _13 = &((_2 as Some).0: std::string::String); + StorageLive(_14); + StorageLive(_15); + StorageLive(_16); + _16 = &(*_13); + _15 = &_16; + StorageLive(_17); + _20 = const chained_conditions::promoted[0]; + _17 = &(*_20); + StorageLive(_29); + StorageLive(_30); + _29 = copy (*_15); + _30 = copy (*_17); + _14 = >::eq(move _29, move _30) -> [return: bb24, unwind: bb22]; + } + + bb7: { + StorageDead(_5); + StorageDead(_6); + goto -> bb19; + } + + bb8: { + StorageDead(_17); + StorageDead(_16); + StorageDead(_15); + StorageDead(_14); + StorageLive(_12); + _23 = const false; + _12 = move ((_2 as Some).0: std::string::String); + _1 = chained_conditions::BacktraceStyle::Off; +- drop(_12) -> [return: bb10, unwind: bb22]; ++ drop(_12) -> [return: bb35, unwind: bb22]; + } + + bb9: { + StorageDead(_17); + StorageDead(_16); + StorageDead(_15); + StorageDead(_14); + StorageDead(_13); + _1 = chained_conditions::BacktraceStyle::Short; +- goto -> bb19; ++ goto -> bb41; + } + + bb10: { + StorageDead(_12); + StorageDead(_13); + goto -> bb19; + } + + bb11: { + _0 = const 3_u8; + goto -> bb14; + } + + bb12: { + _0 = const 2_u8; + goto -> bb14; + } + + bb13: { + _0 = const 1_u8; + goto -> bb14; + } + + bb14: { + StorageDead(_1); + return; + } + + bb15 (cleanup): { + resume; + } + + bb16: { + _22 = const false; + _23 = const false; + StorageDead(_2); + _19 = discriminant(_1); + switchInt(move _19) -> [0: bb13, 1: bb12, 2: bb11, otherwise: bb2]; + } + + bb17: { + switchInt(copy _23) -> [0: bb16, otherwise: bb18]; + } + + bb18: { + drop(((_2 as Some).0: std::string::String)) -> [return: bb16, unwind: bb15]; + } + + bb19: { + _24 = discriminant(_2); + switchInt(move _24) -> [1: bb17, otherwise: bb16]; + } + + bb20 (cleanup): { + switchInt(copy _23) -> [0: bb15, otherwise: bb21]; + } + + bb21 (cleanup): { + drop(((_2 as Some).0: std::string::String)) -> [return: bb15, unwind terminate(cleanup)]; + } + + bb22 (cleanup): { + _26 = discriminant(_2); + switchInt(move _26) -> [1: bb20, otherwise: bb15]; + } + + bb23: { + StorageDead(_28); + StorageDead(_27); + switchInt(move _7) -> [0: bb6, otherwise: bb5]; + } + + bb24: { + StorageDead(_30); + StorageDead(_29); + switchInt(move _14) -> [0: bb9, otherwise: bb8]; ++ } ++ ++ bb25: { ++ _4 = discriminant(_2); ++ switchInt(move _4) -> [0: bb26, 1: bb4, otherwise: bb2]; ++ } ++ ++ bb26: { ++ _1 = chained_conditions::BacktraceStyle::Off; ++ goto -> bb44; ++ } ++ ++ bb27: { ++ StorageDead(_5); ++ StorageDead(_6); ++ goto -> bb19; ++ } ++ ++ bb28: { ++ StorageDead(_5); ++ StorageDead(_6); ++ goto -> bb30; ++ } ++ ++ bb29: { ++ _24 = discriminant(_2); ++ switchInt(move _24) -> [1: bb17, otherwise: bb16]; ++ } ++ ++ bb30: { ++ _24 = discriminant(_2); ++ switchInt(move _24) -> [1: bb33, otherwise: bb31]; ++ } ++ ++ bb31: { ++ _22 = const false; ++ _23 = const false; ++ StorageDead(_2); ++ _19 = discriminant(_1); ++ goto -> bb11; ++ } ++ ++ bb32: { ++ switchInt(copy _23) -> [0: bb16, otherwise: bb18]; ++ } ++ ++ bb33: { ++ goto -> bb31; ++ } ++ ++ bb34: { ++ StorageDead(_12); ++ StorageDead(_13); ++ goto -> bb19; ++ } ++ ++ bb35: { ++ StorageDead(_12); ++ StorageDead(_13); ++ goto -> bb37; ++ } ++ ++ bb36: { ++ _24 = discriminant(_2); ++ switchInt(move _24) -> [1: bb17, otherwise: bb16]; ++ } ++ ++ bb37: { ++ _24 = discriminant(_2); ++ switchInt(move _24) -> [1: bb40, otherwise: bb38]; ++ } ++ ++ bb38: { ++ _22 = const false; ++ _23 = const false; ++ StorageDead(_2); ++ _19 = discriminant(_1); ++ goto -> bb13; ++ } ++ ++ bb39: { ++ switchInt(copy _23) -> [0: bb16, otherwise: bb18]; ++ } ++ ++ bb40: { ++ goto -> bb38; ++ } ++ ++ bb41: { ++ _24 = discriminant(_2); ++ switchInt(move _24) -> [1: bb43, otherwise: bb42]; ++ } ++ ++ bb42: { ++ _22 = const false; ++ _23 = const false; ++ StorageDead(_2); ++ _19 = discriminant(_1); ++ goto -> bb12; ++ } ++ ++ bb43: { ++ switchInt(copy _23) -> [0: bb42, otherwise: bb18]; ++ } ++ ++ bb44: { ++ _24 = discriminant(_2); ++ switchInt(move _24) -> [1: bb45, otherwise: bb38]; ++ } ++ ++ bb45: { ++ goto -> bb18; + } + } + diff --git a/tests/mir-opt/jump_threading.custom_discr.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.custom_discr.JumpThreading.panic-abort.diff index a86371794ebe9..0b11a0d188fb8 100644 --- a/tests/mir-opt/jump_threading.custom_discr.JumpThreading.panic-abort.diff +++ b/tests/mir-opt/jump_threading.custom_discr.JumpThreading.panic-abort.diff @@ -23,14 +23,14 @@ bb2: { _2 = CustomDiscr::B; - goto -> bb3; +- goto -> bb3; ++ goto -> bb8; } bb3: { StorageDead(_3); _4 = discriminant(_2); -- switchInt(move _4) -> [35: bb5, otherwise: bb4]; -+ goto -> bb4; + switchInt(move _4) -> [35: bb5, otherwise: bb4]; } bb4: { @@ -52,6 +52,12 @@ + StorageDead(_3); + _4 = discriminant(_2); + goto -> bb5; ++ } ++ ++ bb8: { ++ StorageDead(_3); ++ _4 = discriminant(_2); ++ goto -> bb4; } } diff --git a/tests/mir-opt/jump_threading.custom_discr.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.custom_discr.JumpThreading.panic-unwind.diff index a86371794ebe9..0b11a0d188fb8 100644 --- a/tests/mir-opt/jump_threading.custom_discr.JumpThreading.panic-unwind.diff +++ b/tests/mir-opt/jump_threading.custom_discr.JumpThreading.panic-unwind.diff @@ -23,14 +23,14 @@ bb2: { _2 = CustomDiscr::B; - goto -> bb3; +- goto -> bb3; ++ goto -> bb8; } bb3: { StorageDead(_3); _4 = discriminant(_2); -- switchInt(move _4) -> [35: bb5, otherwise: bb4]; -+ goto -> bb4; + switchInt(move _4) -> [35: bb5, otherwise: bb4]; } bb4: { @@ -52,6 +52,12 @@ + StorageDead(_3); + _4 = discriminant(_2); + goto -> bb5; ++ } ++ ++ bb8: { ++ StorageDead(_3); ++ _4 = discriminant(_2); ++ goto -> bb4; } } diff --git a/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-abort.diff index d17f2752f58ec..4933e42d0f40d 100644 --- a/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-abort.diff +++ b/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-abort.diff @@ -9,13 +9,13 @@ bb0: { _2 = const true; _3 = const true; - switchInt(copy _1) -> [0: bb3, 1: bb3, 2: bb1, otherwise: bb2]; +- switchInt(copy _1) -> [0: bb3, 1: bb3, 2: bb1, otherwise: bb2]; ++ switchInt(copy _1) -> [0: bb10, 1: bb10, 2: bb9, otherwise: bb2]; } bb1: { _3 = const false; -- goto -> bb4; -+ goto -> bb9; + goto -> bb4; } bb2: { @@ -40,8 +40,7 @@ } bb7: { -- goto -> bb5; -+ goto -> bb10; + goto -> bb5; } bb8: { @@ -49,10 +48,40 @@ + } + + bb9: { -+ goto -> bb5; ++ _3 = const false; ++ goto -> bb12; + } + + bb10: { ++ _2 = const false; ++ goto -> bb15; ++ } ++ ++ bb11: { ++ switchInt(copy _3) -> [0: bb5, otherwise: bb7]; ++ } ++ ++ bb12: { ++ goto -> bb13; ++ } ++ ++ bb13: { ++ goto -> bb8; ++ } ++ ++ bb14: { ++ switchInt(copy _3) -> [0: bb5, otherwise: bb7]; ++ } ++ ++ bb15: { ++ goto -> bb16; ++ } ++ ++ bb16: { ++ goto -> bb17; ++ } ++ ++ bb17: { goto -> bb6; } } diff --git a/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-unwind.diff index d17f2752f58ec..4933e42d0f40d 100644 --- a/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-unwind.diff +++ b/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-unwind.diff @@ -9,13 +9,13 @@ bb0: { _2 = const true; _3 = const true; - switchInt(copy _1) -> [0: bb3, 1: bb3, 2: bb1, otherwise: bb2]; +- switchInt(copy _1) -> [0: bb3, 1: bb3, 2: bb1, otherwise: bb2]; ++ switchInt(copy _1) -> [0: bb10, 1: bb10, 2: bb9, otherwise: bb2]; } bb1: { _3 = const false; -- goto -> bb4; -+ goto -> bb9; + goto -> bb4; } bb2: { @@ -40,8 +40,7 @@ } bb7: { -- goto -> bb5; -+ goto -> bb10; + goto -> bb5; } bb8: { @@ -49,10 +48,40 @@ + } + + bb9: { -+ goto -> bb5; ++ _3 = const false; ++ goto -> bb12; + } + + bb10: { ++ _2 = const false; ++ goto -> bb15; ++ } ++ ++ bb11: { ++ switchInt(copy _3) -> [0: bb5, otherwise: bb7]; ++ } ++ ++ bb12: { ++ goto -> bb13; ++ } ++ ++ bb13: { ++ goto -> bb8; ++ } ++ ++ bb14: { ++ switchInt(copy _3) -> [0: bb5, otherwise: bb7]; ++ } ++ ++ bb15: { ++ goto -> bb16; ++ } ++ ++ bb16: { ++ goto -> bb17; ++ } ++ ++ bb17: { goto -> bb6; } } diff --git a/tests/mir-opt/jump_threading.duplicate_chain.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.duplicate_chain.JumpThreading.panic-abort.diff index 083a6e7487a05..d17e7d9744432 100644 --- a/tests/mir-opt/jump_threading.duplicate_chain.JumpThreading.panic-abort.diff +++ b/tests/mir-opt/jump_threading.duplicate_chain.JumpThreading.panic-abort.diff @@ -13,12 +13,14 @@ bb1: { _2 = const 5_u8; - goto -> bb3; +- goto -> bb3; ++ goto -> bb7; } bb2: { _2 = const 5_u8; - goto -> bb3; +- goto -> bb3; ++ goto -> bb7; } bb3: { @@ -28,8 +30,7 @@ bb4: { _4 = const 15_i32; -- switchInt(copy _2) -> [5: bb5, otherwise: bb6]; -+ goto -> bb5; + switchInt(copy _2) -> [5: bb5, otherwise: bb6]; } bb5: { @@ -40,6 +41,16 @@ bb6: { _0 = const 9_u8; return; ++ } ++ ++ bb7: { ++ _3 = const 13_i32; ++ goto -> bb8; ++ } ++ ++ bb8: { ++ _4 = const 15_i32; ++ goto -> bb5; } } diff --git a/tests/mir-opt/jump_threading.duplicate_chain.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.duplicate_chain.JumpThreading.panic-unwind.diff index 083a6e7487a05..d17e7d9744432 100644 --- a/tests/mir-opt/jump_threading.duplicate_chain.JumpThreading.panic-unwind.diff +++ b/tests/mir-opt/jump_threading.duplicate_chain.JumpThreading.panic-unwind.diff @@ -13,12 +13,14 @@ bb1: { _2 = const 5_u8; - goto -> bb3; +- goto -> bb3; ++ goto -> bb7; } bb2: { _2 = const 5_u8; - goto -> bb3; +- goto -> bb3; ++ goto -> bb7; } bb3: { @@ -28,8 +30,7 @@ bb4: { _4 = const 15_i32; -- switchInt(copy _2) -> [5: bb5, otherwise: bb6]; -+ goto -> bb5; + switchInt(copy _2) -> [5: bb5, otherwise: bb6]; } bb5: { @@ -40,6 +41,16 @@ bb6: { _0 = const 9_u8; return; ++ } ++ ++ bb7: { ++ _3 = const 13_i32; ++ goto -> bb8; ++ } ++ ++ bb8: { ++ _4 = const 15_i32; ++ goto -> bb5; } } diff --git a/tests/mir-opt/jump_threading.identity.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.identity.JumpThreading.panic-abort.diff index 79599f856115d..97b8d484194f5 100644 --- a/tests/mir-opt/jump_threading.identity.JumpThreading.panic-abort.diff +++ b/tests/mir-opt/jump_threading.identity.JumpThreading.panic-abort.diff @@ -97,8 +97,7 @@ StorageDead(_10); StorageDead(_4); _5 = discriminant(_3); -- switchInt(move _5) -> [0: bb2, 1: bb3, otherwise: bb1]; -+ goto -> bb2; + switchInt(move _5) -> [0: bb2, 1: bb3, otherwise: bb1]; } bb6: { @@ -114,7 +113,8 @@ bb7: { _11 = move ((_4 as Ok).0: i32); _3 = ControlFlow::, i32>::Continue(copy _11); - goto -> bb5; +- goto -> bb5; ++ goto -> bb9; + } + + bb8: { @@ -124,6 +124,15 @@ + StorageDead(_4); + _5 = discriminant(_3); + goto -> bb3; ++ } ++ ++ bb9: { ++ StorageDead(_12); ++ StorageDead(_11); ++ StorageDead(_10); ++ StorageDead(_4); ++ _5 = discriminant(_3); ++ goto -> bb2; } } diff --git a/tests/mir-opt/jump_threading.identity.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.identity.JumpThreading.panic-unwind.diff index 79599f856115d..97b8d484194f5 100644 --- a/tests/mir-opt/jump_threading.identity.JumpThreading.panic-unwind.diff +++ b/tests/mir-opt/jump_threading.identity.JumpThreading.panic-unwind.diff @@ -97,8 +97,7 @@ StorageDead(_10); StorageDead(_4); _5 = discriminant(_3); -- switchInt(move _5) -> [0: bb2, 1: bb3, otherwise: bb1]; -+ goto -> bb2; + switchInt(move _5) -> [0: bb2, 1: bb3, otherwise: bb1]; } bb6: { @@ -114,7 +113,8 @@ bb7: { _11 = move ((_4 as Ok).0: i32); _3 = ControlFlow::, i32>::Continue(copy _11); - goto -> bb5; +- goto -> bb5; ++ goto -> bb9; + } + + bb8: { @@ -124,6 +124,15 @@ + StorageDead(_4); + _5 = discriminant(_3); + goto -> bb3; ++ } ++ ++ bb9: { ++ StorageDead(_12); ++ StorageDead(_11); ++ StorageDead(_10); ++ StorageDead(_4); ++ _5 = discriminant(_3); ++ goto -> bb2; } } diff --git a/tests/mir-opt/jump_threading.multiple_match.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.multiple_match.JumpThreading.panic-abort.diff index 09c0ad6d48560..271b7f00078c2 100644 --- a/tests/mir-opt/jump_threading.multiple_match.JumpThreading.panic-abort.diff +++ b/tests/mir-opt/jump_threading.multiple_match.JumpThreading.panic-abort.diff @@ -7,19 +7,18 @@ let mut _3: u8; bb0: { - switchInt(copy _1) -> [3: bb1, otherwise: bb2]; +- switchInt(copy _1) -> [3: bb1, otherwise: bb2]; ++ switchInt(copy _1) -> [3: bb9, otherwise: bb10]; } bb1: { _2 = copy _1; -- switchInt(copy _2) -> [3: bb3, otherwise: bb4]; -+ goto -> bb3; + switchInt(copy _2) -> [3: bb3, otherwise: bb4]; } bb2: { _3 = copy _1; -- switchInt(copy _3) -> [3: bb5, otherwise: bb6]; -+ goto -> bb6; + switchInt(copy _3) -> [3: bb5, otherwise: bb6]; } bb3: { @@ -49,6 +48,16 @@ bb8: { _0 = const 11_u8; return; ++ } ++ ++ bb9: { ++ _2 = copy _1; ++ goto -> bb3; ++ } ++ ++ bb10: { ++ _3 = copy _1; ++ goto -> bb6; } } diff --git a/tests/mir-opt/jump_threading.multiple_match.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.multiple_match.JumpThreading.panic-unwind.diff index 09c0ad6d48560..271b7f00078c2 100644 --- a/tests/mir-opt/jump_threading.multiple_match.JumpThreading.panic-unwind.diff +++ b/tests/mir-opt/jump_threading.multiple_match.JumpThreading.panic-unwind.diff @@ -7,19 +7,18 @@ let mut _3: u8; bb0: { - switchInt(copy _1) -> [3: bb1, otherwise: bb2]; +- switchInt(copy _1) -> [3: bb1, otherwise: bb2]; ++ switchInt(copy _1) -> [3: bb9, otherwise: bb10]; } bb1: { _2 = copy _1; -- switchInt(copy _2) -> [3: bb3, otherwise: bb4]; -+ goto -> bb3; + switchInt(copy _2) -> [3: bb3, otherwise: bb4]; } bb2: { _3 = copy _1; -- switchInt(copy _3) -> [3: bb5, otherwise: bb6]; -+ goto -> bb6; + switchInt(copy _3) -> [3: bb5, otherwise: bb6]; } bb3: { @@ -49,6 +48,16 @@ bb8: { _0 = const 11_u8; return; ++ } ++ ++ bb9: { ++ _2 = copy _1; ++ goto -> bb3; ++ } ++ ++ bb10: { ++ _3 = copy _1; ++ goto -> bb6; } } diff --git a/tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-abort.diff index 9a8bdc8f4d952..4f80acd073384 100644 --- a/tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-abort.diff +++ b/tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-abort.diff @@ -8,13 +8,13 @@ bb0: { _3 = const false; - switchInt(copy _1) -> [1: bb1, otherwise: bb2]; +- switchInt(copy _1) -> [1: bb1, otherwise: bb2]; ++ switchInt(copy _1) -> [1: bb8, otherwise: bb2]; } bb1: { _2 = const false; -- goto -> bb3; -+ goto -> bb8; + goto -> bb3; } bb2: { @@ -47,10 +47,19 @@ + } + + bb8: { -+ goto -> bb9; ++ _2 = const false; ++ goto -> bb10; + } + + bb9: { ++ switchInt(copy _2) -> [0: bb4, otherwise: bb5]; ++ } ++ ++ bb10: { ++ goto -> bb11; ++ } ++ ++ bb11: { + goto -> bb6; } } diff --git a/tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-unwind.diff index 9a8bdc8f4d952..4f80acd073384 100644 --- a/tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-unwind.diff +++ b/tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-unwind.diff @@ -8,13 +8,13 @@ bb0: { _3 = const false; - switchInt(copy _1) -> [1: bb1, otherwise: bb2]; +- switchInt(copy _1) -> [1: bb1, otherwise: bb2]; ++ switchInt(copy _1) -> [1: bb8, otherwise: bb2]; } bb1: { _2 = const false; -- goto -> bb3; -+ goto -> bb8; + goto -> bb3; } bb2: { @@ -47,10 +47,19 @@ + } + + bb8: { -+ goto -> bb9; ++ _2 = const false; ++ goto -> bb10; + } + + bb9: { ++ switchInt(copy _2) -> [0: bb4, otherwise: bb5]; ++ } ++ ++ bb10: { ++ goto -> bb11; ++ } ++ ++ bb11: { + goto -> bb6; } } diff --git a/tests/mir-opt/jump_threading.rs b/tests/mir-opt/jump_threading.rs index ad46a02b83f40..e841643e7ce5c 100644 --- a/tests/mir-opt/jump_threading.rs +++ b/tests/mir-opt/jump_threading.rs @@ -19,9 +19,9 @@ fn too_complex(x: Result) -> Option { // CHECK: goto -> bb8; // CHECK: bb3: { // CHECK: [[controlflow]] = ControlFlow::::Continue( - // CHECK: goto -> bb4; + // CHECK: goto -> bb9; // CHECK: bb4: { - // CHECK: goto -> bb6; + // CHECK: switchInt(move _8) -> [0: bb6, 1: bb5, otherwise: bb1]; // CHECK: bb5: { // CHECK: {{_.*}} = copy (([[controlflow]] as Break).0: usize); // CHECK: _0 = Option::::None; @@ -34,6 +34,8 @@ fn too_complex(x: Result) -> Option { // CHECK: return; // CHECK: bb8: { // CHECK: goto -> bb5; + // CHECK: bb9: { + // CHECK: goto -> bb6; match { match x { Ok(v) => ControlFlow::Continue(v), @@ -63,7 +65,7 @@ fn identity(x: Result) -> Result { // CHECK: bb4: { // CHECK: return; // CHECK: bb5: { - // CHECK: goto -> bb2; + // CHECK: switchInt(move _5) -> [0: bb2, 1: bb3, otherwise: bb1]; // CHECK: bb6: { // CHECK: {{_.*}} = move (([[x]] as Err).0: i32); // CHECK: [[controlflow]] = ControlFlow::, i32>::Break( @@ -71,9 +73,11 @@ fn identity(x: Result) -> Result { // CHECK: bb7: { // CHECK: {{_.*}} = move (([[x]] as Ok).0: i32); // CHECK: [[controlflow]] = ControlFlow::, i32>::Continue( - // CHECK: goto -> bb5; + // CHECK: goto -> bb9; // CHECK: bb8: { // CHECK: goto -> bb3; + // CHECK: bb9: { + // CHECK: goto -> bb2; Ok(x?) } @@ -160,9 +164,9 @@ fn custom_discr(x: bool) -> u8 { // CHECK: goto -> bb7; // CHECK: bb2: { // CHECK: {{_.*}} = CustomDiscr::B; - // CHECK: goto -> bb3; + // CHECK: goto -> bb8; // CHECK: bb3: { - // CHECK: goto -> bb4; + // CHECK: switchInt(move _4) -> [35: bb5, otherwise: bb4]; // CHECK: bb4: { // CHECK: _0 = const 13_u8; // CHECK: goto -> bb6; @@ -173,6 +177,8 @@ fn custom_discr(x: bool) -> u8 { // CHECK: return; // CHECK: bb7: { // CHECK: goto -> bb5; + // CHECK: bb8: { + // CHECK: goto -> bb4; match if x { CustomDiscr::A } else { CustomDiscr::B } { CustomDiscr::A => 5, _ => 13, @@ -185,22 +191,18 @@ fn multiple_match(x: u8) -> u8 { mir! { { // CHECK: bb0: { - // CHECK: switchInt(copy [[x:_.*]]) -> [3: bb1, otherwise: bb2]; + // CHECK: switchInt(copy [[x:_.*]]) -> [3: bb9, otherwise: bb10]; match x { 3 => bb1, _ => bb2 } } bb1 = { // We know `x == 3`, so we can take `bb3`. // CHECK: bb1: { - // CHECK: {{_.*}} = copy [[x]]; - // CHECK: goto -> bb3; let y = x; match y { 3 => bb3, _ => bb4 } } bb2 = { // We know `x != 3`, so we can take `bb6`. // CHECK: bb2: { - // CHECK: [[z:_.*]] = copy [[x]]; - // CHECK: goto -> bb6; let z = x; match z { 3 => bb5, _ => bb6 } } @@ -228,7 +230,7 @@ fn multiple_match(x: u8) -> u8 { bb6 = { // We know `z != 3`, so we CANNOT take `bb7`. // CHECK: bb6: { - // CHECK: switchInt(copy [[z]]) -> [1: bb7, otherwise: bb8]; + // CHECK: switchInt(copy [[z:_.*]]) -> [1: bb7, otherwise: bb8]; match z { 1 => bb7, _ => bb8 } } bb7 = { @@ -245,6 +247,14 @@ fn multiple_match(x: u8) -> u8 { RET = 11; Return() } + // We know `x == 3`, so we can take `bb3`. + // CHECK: bb9: { + // CHECK: {{_.*}} = copy [[x]]; + // CHECK: goto -> bb3; + // We know `x != 3`, so we can take `bb6`. + // CHECK: bb10: { + // CHECK: [[z]] = copy [[x]]; + // CHECK: goto -> bb6; } } @@ -263,14 +273,14 @@ fn duplicate_chain(x: bool) -> u8 { bb1 = { // CHECK: bb1: { // CHECK: [[a:_.*]] = const 5_u8; - // CHECK: goto -> bb3; + // CHECK: goto -> bb7; a = 5; Goto(bb3) } bb2 = { // CHECK: bb2: { // CHECK: [[a]] = const 5_u8; - // CHECK: goto -> bb3; + // CHECK: goto -> bb7; a = 5; Goto(bb3) } @@ -284,8 +294,7 @@ fn duplicate_chain(x: bool) -> u8 { bb4 = { // CHECK: bb4: { // CHECK: {{_.*}} = const 15_i32; - // CHECK-NOT: switchInt( - // CHECK: goto -> bb5; + // CHECK: switchInt(copy _2) -> [5: bb5, otherwise: bb6]; let c = 15; match a { 5 => bb5, _ => bb6 } } @@ -303,6 +312,12 @@ fn duplicate_chain(x: bool) -> u8 { RET = 9; Return() } + // CHECK: bb7: { + // CHECK-NEXT: {{_.*}} = const 13_i32; + // CHECK-NEXT: goto -> bb8; + // CHECK: bb8: { + // CHECK-NEXT: {{_.*}} = const 15_i32; + // CHECK-NEXT: goto -> bb5; } } @@ -368,13 +383,13 @@ fn renumbered_bb(x: bool) -> u8 { let b: bool; { // CHECK: bb0: { - // CHECK: switchInt({{.*}}) -> [1: bb1, otherwise: bb2]; + // CHECK: switchInt({{.*}}) -> [1: bb8, otherwise: bb2]; b = false; match x { true => bb1, _ => bb2 } } bb1 = { // CHECK: bb1: { - // CHECK: goto -> bb8; + // CHECK: goto -> bb3; a = false; Goto(bb3) } @@ -413,12 +428,18 @@ fn renumbered_bb(x: bool) -> u8 { RET = 11; Return() } - // Duplicate of bb3. + // Duplicate of bb1. // CHECK: bb8: { - // CHECK-NEXT: goto -> bb9; - // Duplicate of bb4. + // CHECK: goto -> bb10; + // Duplicate of bb3. // CHECK: bb9: { - // CHECK-NEXT: goto -> bb6; + // CHECK: switchInt(copy _2) -> [0: bb4, otherwise: bb5]; + // Duplicate of bb9. + // CHECK: bb10: { + // CHECK: goto -> bb11; + // Duplicate of bb4. + // CHECK: bb11: { + // CHECK: goto -> bb6; } } @@ -433,22 +454,26 @@ fn disappearing_bb(x: u8) -> u8 { let a: bool; let b: bool; { + // CHECK: bb0: { a = true; b = true; + // CHECK: switchInt({{.*}}) -> [0: bb10, 1: bb10, 2: bb9, otherwise: bb2]; match x { 0 => bb3, 1 => bb3, 2 => bb1, _ => bb2 } } bb1 = { // CHECK: bb1: { - // CHECK: goto -> bb9; + // CHECK: goto -> bb4; b = false; Goto(bb4) } bb2 = { + // CHECK: bb2: { + // CHECK: unreachable; Unreachable() } bb3 = { // CHECK: bb3: { - // CHECK: goto -> bb10; + // CHECK: goto -> bb4; a = false; Goto(bb4) } @@ -468,9 +493,23 @@ fn disappearing_bb(x: u8) -> u8 { Goto(bb6) } // CHECK: bb9: { - // CHECK: goto -> bb5; + // CHECK: goto -> bb12; // CHECK: bb10: { - // CHECK: goto -> bb6; + // CHECK: goto -> bb15; + // CHECK: bb11: { + // CHECK: switchInt(copy _3) -> [0: bb5, otherwise: bb7]; + // CHECK: bb12: { + // CHECK: goto -> bb13; + // CHECK: bb13: { + // CHECK: goto -> bb8; + // CHECK: bb14: { + // CHECK: switchInt(copy _3) -> [0: bb5, otherwise: bb7]; + // CHECK: bb15: { + // CHECK: goto -> bb16; + // CHECK: bb16: { + // CHECK: goto -> bb17; + // CHECK: bb17: { + // CHECK: goto -> bb6; } } @@ -604,6 +643,36 @@ pub fn logical_not() -> i32 { if !a == true { 1 } else { 0 } } +/// Verify that we correctly handle threading multiple conditions on the same bb. +/// One version of the implementation was buggy and mutated a bb that would be duplicated later. +fn chained_conditions() -> u8 { + // CHECK-LABEL: fn chained_conditions( + + #[inline(never)] + fn env_var() -> Option { + None + } + + enum BacktraceStyle { + Off, + Short, + Full, + }; + + let format = match env_var() { + Some(x) if &x == "full" => BacktraceStyle::Full, + Some(x) if &x == "0" => BacktraceStyle::Off, + Some(_) => BacktraceStyle::Short, + None => BacktraceStyle::Off, + }; + + match format { + BacktraceStyle::Off => 1, + BacktraceStyle::Short => 2, + BacktraceStyle::Full => 3, + } +} + fn main() { // CHECK-LABEL: fn main( too_complex(Ok(0)); @@ -640,3 +709,4 @@ fn main() { // EMIT_MIR jump_threading.floats.JumpThreading.diff // EMIT_MIR jump_threading.bitwise_not.JumpThreading.diff // EMIT_MIR jump_threading.logical_not.JumpThreading.diff +// EMIT_MIR jump_threading.chained_conditions.JumpThreading.diff diff --git a/tests/mir-opt/jump_threading.too_complex.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.too_complex.JumpThreading.panic-abort.diff index 7de359298922b..6e0cf5ec41abd 100644 --- a/tests/mir-opt/jump_threading.too_complex.JumpThreading.panic-abort.diff +++ b/tests/mir-opt/jump_threading.too_complex.JumpThreading.panic-abort.diff @@ -57,13 +57,13 @@ _2 = ControlFlow::::Continue(move _5); StorageDead(_5); StorageDead(_4); - goto -> bb4; +- goto -> bb4; ++ goto -> bb9; } bb4: { _8 = discriminant(_2); -- switchInt(move _8) -> [0: bb6, 1: bb5, otherwise: bb1]; -+ goto -> bb6; + switchInt(move _8) -> [0: bb6, 1: bb5, otherwise: bb1]; } bb5: { @@ -93,6 +93,11 @@ + bb8: { + _8 = discriminant(_2); + goto -> bb5; ++ } ++ ++ bb9: { ++ _8 = discriminant(_2); ++ goto -> bb6; } } diff --git a/tests/mir-opt/jump_threading.too_complex.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.too_complex.JumpThreading.panic-unwind.diff index 7de359298922b..6e0cf5ec41abd 100644 --- a/tests/mir-opt/jump_threading.too_complex.JumpThreading.panic-unwind.diff +++ b/tests/mir-opt/jump_threading.too_complex.JumpThreading.panic-unwind.diff @@ -57,13 +57,13 @@ _2 = ControlFlow::::Continue(move _5); StorageDead(_5); StorageDead(_4); - goto -> bb4; +- goto -> bb4; ++ goto -> bb9; } bb4: { _8 = discriminant(_2); -- switchInt(move _8) -> [0: bb6, 1: bb5, otherwise: bb1]; -+ goto -> bb6; + switchInt(move _8) -> [0: bb6, 1: bb5, otherwise: bb1]; } bb5: { @@ -93,6 +93,11 @@ + bb8: { + _8 = discriminant(_2); + goto -> bb5; ++ } ++ ++ bb9: { ++ _8 = discriminant(_2); ++ goto -> bb6; } } diff --git a/tests/mir-opt/pre-codegen/derived_ord.demo_le.PreCodegen.after.mir b/tests/mir-opt/pre-codegen/derived_ord.demo_le.PreCodegen.after.mir index e235fa35c0238..578aff4f7129d 100644 --- a/tests/mir-opt/pre-codegen/derived_ord.demo_le.PreCodegen.after.mir +++ b/tests/mir-opt/pre-codegen/derived_ord.demo_le.PreCodegen.after.mir @@ -7,12 +7,11 @@ fn demo_le(_1: &MultiField, _2: &MultiField) -> bool { scope 1 (inlined ::le) { let mut _6: std::option::Option; scope 2 (inlined Option::::is_some_and:: bool {std::cmp::Ordering::is_le}>) { - let mut _11: isize; - let _12: std::cmp::Ordering; + let _11: std::cmp::Ordering; scope 3 { scope 4 (inlined bool {std::cmp::Ordering::is_le} as FnOnce<(std::cmp::Ordering,)>>::call_once - shim(fn(std::cmp::Ordering) -> bool {std::cmp::Ordering::is_le})) { scope 5 (inlined std::cmp::Ordering::is_le) { - let mut _13: i8; + let mut _12: i8; scope 6 (inlined std::cmp::Ordering::as_raw) { } } @@ -37,7 +36,7 @@ fn demo_le(_1: &MultiField, _2: &MultiField) -> bool { } bb0: { - StorageLive(_12); + StorageLive(_11); StorageLive(_6); StorageLive(_5); StorageLive(_7); @@ -64,42 +63,19 @@ fn demo_le(_1: &MultiField, _2: &MultiField) -> bool { StorageDead(_8); _6 = Option::::Some(move _10); StorageDead(_10); - StorageDead(_7); - StorageDead(_5); - StorageLive(_11); - goto -> bb4; + goto -> bb2; } bb2: { StorageDead(_7); StorageDead(_5); - StorageLive(_11); - _11 = discriminant(_6); - switchInt(move _11) -> [0: bb3, 1: bb4, otherwise: bb6]; - } - - bb3: { - _0 = const false; - goto -> bb5; - } - - bb4: { - _12 = move ((_6 as Some).0: std::cmp::Ordering); - StorageLive(_13); - _13 = discriminant(_12); - _0 = Le(move _13, const 0_i8); - StorageDead(_13); - goto -> bb5; - } - - bb5: { - StorageDead(_11); - StorageDead(_6); + _11 = move ((_6 as Some).0: std::cmp::Ordering); + StorageLive(_12); + _12 = discriminant(_11); + _0 = Le(move _12, const 0_i8); StorageDead(_12); + StorageDead(_6); + StorageDead(_11); return; } - - bb6: { - unreachable; - } } diff --git a/tests/mir-opt/pre-codegen/derived_ord.rs b/tests/mir-opt/pre-codegen/derived_ord.rs index 823e0f6d09c2b..a67756c22d466 100644 --- a/tests/mir-opt/pre-codegen/derived_ord.rs +++ b/tests/mir-opt/pre-codegen/derived_ord.rs @@ -13,22 +13,23 @@ pub fn demo_le(a: &MultiField, b: &MultiField) -> bool { // CHECK: inlined{{.+}}is_some_and // CHECK: inlined ::partial_cmp - // CHECK: [[A0:_[0-9]+]] = copy ((*_1).0: char); - // CHECK: [[B0:_[0-9]+]] = copy ((*_2).0: char); - // CHECK: Cmp(move [[A0]], move [[B0]]); - - // CHECK: [[D0:_[0-9]+]] = discriminant({{.+}}); - // CHECK: switchInt(move [[D0]]) -> [0: bb{{[0-9]+}}, otherwise: bb{{[0-9]+}}]; - - // CHECK: [[A1:_[0-9]+]] = copy ((*_1).1: i16); - // CHECK: [[B1:_[0-9]+]] = copy ((*_2).1: i16); - // CHECK: Cmp(move [[A1]], move [[B1]]); - - // CHECK: [[D1:_[0-9]+]] = discriminant({{.+}}); - // CHECK: switchInt(move [[D1]]) -> [0: bb{{[0-9]+}}, 1: bb{{[0-9]+}}, otherwise: bb{{[0-9]+}}]; - - // CHECK: [[D2:_[0-9]+]] = discriminant({{.+}}); - // CHECK: _0 = Le(move [[D2]], const 0_i8); + // CHECK: bb0: { + // CHECK: [[A0:_[0-9]+]] = copy ((*_1).0: char); + // CHECK: [[B0:_[0-9]+]] = copy ((*_2).0: char); + // CHECK: Cmp(move [[A0]], move [[B0]]); + // CHECK: [[D0:_[0-9]+]] = discriminant({{.+}}); + // CHECK: switchInt(move [[D0]]) -> [0: bb1, otherwise: bb2]; + + // CHECK: bb1: { + // CHECK: [[A1:_[0-9]+]] = copy ((*_1).1: i16); + // CHECK: [[B1:_[0-9]+]] = copy ((*_2).1: i16); + // CHECK: Cmp(move [[A1]], move [[B1]]); + // CHECK: goto -> bb2; + + // CHECK: bb2: { + // CHECK: [[D2:_[0-9]+]] = discriminant({{.+}}); + // CHECK: _0 = Le(move [[D2]], const 0_i8); + // CHECK: return; *a <= *b } diff --git a/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir index 104987b0fdda9..f72611b7cb8e3 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir @@ -211,12 +211,6 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { bb7: { StorageDead(_16); - StorageDead(_22); - StorageDead(_13); - StorageDead(_20); - StorageDead(_19); - StorageDead(_12); - StorageDead(_11); goto -> bb10; } @@ -226,16 +220,16 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () { } bb9: { + goto -> bb10; + } + + bb10: { StorageDead(_22); StorageDead(_13); StorageDead(_20); StorageDead(_19); StorageDead(_12); StorageDead(_11); - goto -> bb10; - } - - bb10: { StorageDead(_23); StorageDead(_26); StorageDead(_25); diff --git a/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-abort.mir index 4d0e3548e7d6b..b210efb1f46c2 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-abort.mir @@ -173,12 +173,6 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { bb7: { StorageDead(_15); - StorageDead(_21); - StorageDead(_12); - StorageDead(_19); - StorageDead(_18); - StorageDead(_11); - StorageDead(_10); goto -> bb10; } @@ -188,16 +182,16 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { } bb9: { + goto -> bb10; + } + + bb10: { StorageDead(_21); StorageDead(_12); StorageDead(_19); StorageDead(_18); StorageDead(_11); StorageDead(_10); - goto -> bb10; - } - - bb10: { StorageDead(_22); drop(_2) -> [return: bb11, unwind unreachable]; } diff --git a/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-unwind.mir index 2b5d8c27d7109..ab6e2bf0b36b3 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-unwind.mir @@ -173,12 +173,6 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { bb7: { StorageDead(_15); - StorageDead(_21); - StorageDead(_12); - StorageDead(_19); - StorageDead(_18); - StorageDead(_11); - StorageDead(_10); goto -> bb10; } @@ -188,16 +182,16 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () { } bb9: { + goto -> bb10; + } + + bb10: { StorageDead(_21); StorageDead(_12); StorageDead(_19); StorageDead(_18); StorageDead(_11); StorageDead(_10); - goto -> bb10; - } - - bb10: { StorageDead(_22); drop(_2) -> [return: bb11, unwind continue]; } diff --git a/tests/mir-opt/separate_const_switch.identity.JumpThreading.diff b/tests/mir-opt/separate_const_switch.identity.JumpThreading.diff index ce9d812701a8f..34f451fc698c7 100644 --- a/tests/mir-opt/separate_const_switch.identity.JumpThreading.diff +++ b/tests/mir-opt/separate_const_switch.identity.JumpThreading.diff @@ -69,8 +69,7 @@ StorageDead(_7); StorageDead(_6); _3 = discriminant(_2); -- switchInt(move _3) -> [0: bb2, 1: bb3, otherwise: bb1]; -+ goto -> bb2; + switchInt(move _3) -> [0: bb2, 1: bb3, otherwise: bb1]; } bb5: { @@ -86,7 +85,8 @@ bb6: { _7 = copy ((_1 as Ok).0: i32); _2 = ControlFlow::, i32>::Continue(copy _7); - goto -> bb4; +- goto -> bb4; ++ goto -> bb8; + } + + bb7: { @@ -95,6 +95,14 @@ + StorageDead(_6); + _3 = discriminant(_2); + goto -> bb3; ++ } ++ ++ bb8: { ++ StorageDead(_8); ++ StorageDead(_7); ++ StorageDead(_6); ++ _3 = discriminant(_2); ++ goto -> bb2; } } diff --git a/tests/mir-opt/separate_const_switch.too_complex.JumpThreading.diff b/tests/mir-opt/separate_const_switch.too_complex.JumpThreading.diff index c88c63e0c1334..794c28ab46da0 100644 --- a/tests/mir-opt/separate_const_switch.too_complex.JumpThreading.diff +++ b/tests/mir-opt/separate_const_switch.too_complex.JumpThreading.diff @@ -44,13 +44,13 @@ bb3: { _4 = copy ((_1 as Ok).0: i32); _2 = ControlFlow::::Continue(copy _4); - goto -> bb4; +- goto -> bb4; ++ goto -> bb9; } bb4: { _6 = discriminant(_2); -- switchInt(move _6) -> [0: bb6, 1: bb5, otherwise: bb1]; -+ goto -> bb6; + switchInt(move _6) -> [0: bb6, 1: bb5, otherwise: bb1]; } bb5: { @@ -75,6 +75,11 @@ + bb8: { + _6 = discriminant(_2); + goto -> bb5; ++ } ++ ++ bb9: { ++ _6 = discriminant(_2); ++ goto -> bb6; } } From 120d2154d84e018d9da6947c8ac13d2a29ca402c Mon Sep 17 00:00:00 2001 From: Camille Gillot Date: Sun, 14 Sep 2025 19:48:25 +0000 Subject: [PATCH 8/9] Simplify condition graph. --- .../rustc_mir_transform/src/jump_threading.rs | 90 +++++++++++++++- ..._conditions.JumpThreading.panic-abort.diff | 101 +++++++----------- ...conditions.JumpThreading.panic-unwind.diff | 96 ++++++----------- ...ppearing_bb.JumpThreading.panic-abort.diff | 31 ++---- ...pearing_bb.JumpThreading.panic-unwind.diff | 31 ++---- ...icate_chain.JumpThreading.panic-abort.diff | 19 +--- ...cate_chain.JumpThreading.panic-unwind.diff | 19 +--- ...tiple_match.JumpThreading.panic-abort.diff | 19 +--- ...iple_match.JumpThreading.panic-unwind.diff | 19 +--- ...numbered_bb.JumpThreading.panic-abort.diff | 15 +-- ...umbered_bb.JumpThreading.panic-unwind.diff | 15 +-- tests/mir-opt/jump_threading.rs | 67 +++++------- 12 files changed, 231 insertions(+), 291 deletions(-) diff --git a/compiler/rustc_mir_transform/src/jump_threading.rs b/compiler/rustc_mir_transform/src/jump_threading.rs index a99297bf2f9ad..1b6a0203519c4 100644 --- a/compiler/rustc_mir_transform/src/jump_threading.rs +++ b/compiler/rustc_mir_transform/src/jump_threading.rs @@ -134,7 +134,10 @@ impl<'tcx> crate::MirPass<'tcx> for JumpThreading { finder.entry_states[bb] = state; } - if let Some(opportunities) = OpportunitySet::new(body, finder.entry_states) { + let mut entry_states = finder.entry_states; + simplify_conditions(body, &mut entry_states); + + if let Some(opportunities) = OpportunitySet::new(body, entry_states) { opportunities.apply(); } } @@ -777,6 +780,91 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { } } +/// Propagate fulfilled conditions forward in the CFG to reduce the amount of duplication. +#[instrument(level = "debug", skip(body, entry_states))] +fn simplify_conditions(body: &Body<'_>, entry_states: &mut IndexVec) { + let basic_blocks = &body.basic_blocks; + let reverse_postorder = basic_blocks.reverse_postorder(); + + // Start by computing the number of *incoming edges* for each block. + // We do not use the cached `basic_blocks.predecessors` as we only want reachable predecessors. + let mut predecessors = IndexVec::from_elem(0, &entry_states); + predecessors[START_BLOCK] = 1; // Account for the implicit entry edge. + for &bb in reverse_postorder { + let term = basic_blocks[bb].terminator(); + for s in term.successors() { + predecessors[s] += 1; + } + } + + // Compute the number of edges into each block that carry each condition. + let mut fulfill_in_pred_count = IndexVec::from_fn_n( + |bb: BasicBlock| IndexVec::from_elem_n(0, entry_states[bb].targets.len()), + entry_states.len(), + ); + + // By traversing in RPO, we increase the likelihood to visit predecessors before successors. + for &bb in reverse_postorder { + let preds = predecessors[bb]; + trace!(?bb, ?preds); + + // We have removed all the input edges towards this block. Just skip visiting it. + if preds == 0 { + continue; + } + + let state = &mut entry_states[bb]; + trace!(?state); + + // Conditions that are fulfilled in all the predecessors, are fulfilled in `bb`. + trace!(fulfilled_count = ?fulfill_in_pred_count[bb]); + for (condition, &cond_preds) in fulfill_in_pred_count[bb].iter_enumerated() { + if cond_preds == preds { + trace!(?condition); + state.fulfilled.push(condition); + } + } + + // We want to count how many times each condition is fulfilled, + // so ensure we are not counting the same edge twice. + let mut targets: Vec<_> = state + .fulfilled + .iter() + .flat_map(|&index| state.targets[index].iter().copied()) + .collect(); + targets.sort(); + targets.dedup(); + trace!(?targets); + + // We may modify the set of successors by applying edges, so track them here. + let mut successors = basic_blocks[bb].terminator().successors().collect::>(); + + targets.reverse(); + while let Some(target) = targets.pop() { + match target { + EdgeEffect::Goto { target } => { + // We update the count of predecessors. If target or any successor has not been + // processed yet, this increases the likelihood we find something relevant. + predecessors[target] += 1; + for &s in successors.iter() { + predecessors[s] -= 1; + } + // Only process edges that still exist. + targets.retain(|t| t.block() == target); + successors.clear(); + successors.push(target); + } + EdgeEffect::Chain { succ_block, succ_condition } => { + // `predecessors` is the number of incoming *edges* in each block. + // Count the number of edges that apply `succ_condition` into `succ_block`. + let count = successors.iter().filter(|&&s| s == succ_block).count(); + fulfill_in_pred_count[succ_block][succ_condition] += count; + } + } + } + } +} + struct OpportunitySet<'a, 'tcx> { basic_blocks: &'a mut IndexVec>, entry_states: IndexVec, diff --git a/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-abort.diff index 4c1f25ada0760..bbe9ba806dc72 100644 --- a/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-abort.diff +++ b/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-abort.diff @@ -61,8 +61,7 @@ _22 = const true; _23 = const true; _4 = discriminant(_2); -- switchInt(move _4) -> [0: bb3, 1: bb4, otherwise: bb2]; -+ switchInt(move _4) -> [0: bb21, 1: bb4, otherwise: bb2]; + switchInt(move _4) -> [0: bb3, 1: bb4, otherwise: bb2]; } bb2: { @@ -71,7 +70,8 @@ bb3: { _1 = chained_conditions::BacktraceStyle::Off; - goto -> bb18; +- goto -> bb18; ++ goto -> bb35; } bb4: { @@ -101,8 +101,7 @@ _23 = const false; _5 = move ((_2 as Some).0: std::string::String); _1 = chained_conditions::BacktraceStyle::Full; -- drop(_5) -> [return: bb7, unwind unreachable]; -+ drop(_5) -> [return: bb23, unwind unreachable]; + drop(_5) -> [return: bb7, unwind unreachable]; } bb6: { @@ -131,7 +130,8 @@ bb7: { StorageDead(_5); StorageDead(_6); - goto -> bb18; +- goto -> bb18; ++ goto -> bb22; } bb8: { @@ -143,8 +143,7 @@ _23 = const false; _12 = move ((_2 as Some).0: std::string::String); _1 = chained_conditions::BacktraceStyle::Off; -- drop(_12) -> [return: bb10, unwind unreachable]; -+ drop(_12) -> [return: bb30, unwind unreachable]; + drop(_12) -> [return: bb10, unwind unreachable]; } bb9: { @@ -155,13 +154,14 @@ StorageDead(_13); _1 = chained_conditions::BacktraceStyle::Short; - goto -> bb18; -+ goto -> bb36; ++ goto -> bb31; } bb10: { StorageDead(_12); StorageDead(_13); - goto -> bb18; +- goto -> bb18; ++ goto -> bb27; } bb11: { @@ -218,33 +218,16 @@ + } + + bb21: { -+ _1 = chained_conditions::BacktraceStyle::Off; -+ goto -> bb40; -+ } -+ -+ bb22: { -+ StorageDead(_5); -+ StorageDead(_6); -+ goto -> bb18; -+ } -+ -+ bb23: { -+ StorageDead(_5); -+ StorageDead(_6); -+ goto -> bb25; -+ } -+ -+ bb24: { + _24 = discriminant(_2); + switchInt(move _24) -> [1: bb16, otherwise: bb15]; + } + -+ bb25: { ++ bb22: { + _24 = discriminant(_2); -+ switchInt(move _24) -> [1: bb28, otherwise: bb26]; ++ switchInt(move _24) -> [1: bb25, otherwise: bb23]; + } + -+ bb26: { ++ bb23: { + _22 = const false; + _23 = const false; + StorageDead(_2); @@ -252,37 +235,25 @@ + goto -> bb11; + } + -+ bb27: { ++ bb24: { + switchInt(copy _23) -> [0: bb15, otherwise: bb17]; + } + -+ bb28: { -+ goto -> bb26; -+ } -+ -+ bb29: { -+ StorageDead(_12); -+ StorageDead(_13); -+ goto -> bb18; -+ } -+ -+ bb30: { -+ StorageDead(_12); -+ StorageDead(_13); -+ goto -> bb32; ++ bb25: { ++ goto -> bb23; + } + -+ bb31: { ++ bb26: { + _24 = discriminant(_2); + switchInt(move _24) -> [1: bb16, otherwise: bb15]; + } + -+ bb32: { ++ bb27: { + _24 = discriminant(_2); -+ switchInt(move _24) -> [1: bb35, otherwise: bb33]; ++ switchInt(move _24) -> [1: bb30, otherwise: bb28]; + } + -+ bb33: { ++ bb28: { + _22 = const false; + _23 = const false; + StorageDead(_2); @@ -290,20 +261,20 @@ + goto -> bb13; + } + -+ bb34: { ++ bb29: { + switchInt(copy _23) -> [0: bb15, otherwise: bb17]; + } + -+ bb35: { -+ goto -> bb33; ++ bb30: { ++ goto -> bb28; + } + -+ bb36: { ++ bb31: { + _24 = discriminant(_2); -+ switchInt(move _24) -> [1: bb38, otherwise: bb37]; ++ switchInt(move _24) -> [1: bb33, otherwise: bb32]; + } + -+ bb37: { ++ bb32: { + _22 = const false; + _23 = const false; + StorageDead(_2); @@ -311,25 +282,25 @@ + goto -> bb12; + } + -+ bb38: { -+ switchInt(copy _23) -> [0: bb37, otherwise: bb39]; ++ bb33: { ++ switchInt(copy _23) -> [0: bb32, otherwise: bb34]; + } + -+ bb39: { -+ drop(((_2 as Some).0: std::string::String)) -> [return: bb37, unwind unreachable]; ++ bb34: { ++ drop(((_2 as Some).0: std::string::String)) -> [return: bb32, unwind unreachable]; + } + -+ bb40: { ++ bb35: { + _24 = discriminant(_2); -+ switchInt(move _24) -> [1: bb41, otherwise: bb33]; ++ switchInt(move _24) -> [1: bb36, otherwise: bb28]; + } + -+ bb41: { -+ goto -> bb42; ++ bb36: { ++ goto -> bb37; + } + -+ bb42: { -+ drop(((_2 as Some).0: std::string::String)) -> [return: bb33, unwind unreachable]; ++ bb37: { ++ drop(((_2 as Some).0: std::string::String)) -> [return: bb28, unwind unreachable]; } } diff --git a/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-unwind.diff index e45cb1714a99a..a40e4446aef17 100644 --- a/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-unwind.diff +++ b/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-unwind.diff @@ -56,8 +56,7 @@ StorageLive(_2); _22 = const true; _23 = const true; -- _2 = env_var() -> [return: bb1, unwind continue]; -+ _2 = env_var() -> [return: bb25, unwind continue]; + _2 = env_var() -> [return: bb1, unwind continue]; } bb1: { @@ -71,7 +70,8 @@ bb3: { _1 = chained_conditions::BacktraceStyle::Off; - goto -> bb19; +- goto -> bb19; ++ goto -> bb38; } bb4: { @@ -101,8 +101,7 @@ _23 = const false; _5 = move ((_2 as Some).0: std::string::String); _1 = chained_conditions::BacktraceStyle::Full; -- drop(_5) -> [return: bb7, unwind: bb22]; -+ drop(_5) -> [return: bb28, unwind: bb22]; + drop(_5) -> [return: bb7, unwind: bb22]; } bb6: { @@ -131,7 +130,8 @@ bb7: { StorageDead(_5); StorageDead(_6); - goto -> bb19; +- goto -> bb19; ++ goto -> bb26; } bb8: { @@ -143,8 +143,7 @@ _23 = const false; _12 = move ((_2 as Some).0: std::string::String); _1 = chained_conditions::BacktraceStyle::Off; -- drop(_12) -> [return: bb10, unwind: bb22]; -+ drop(_12) -> [return: bb35, unwind: bb22]; + drop(_12) -> [return: bb10, unwind: bb22]; } bb9: { @@ -155,13 +154,14 @@ StorageDead(_13); _1 = chained_conditions::BacktraceStyle::Short; - goto -> bb19; -+ goto -> bb41; ++ goto -> bb35; } bb10: { StorageDead(_12); StorageDead(_13); - goto -> bb19; +- goto -> bb19; ++ goto -> bb31; } bb11: { @@ -235,38 +235,16 @@ + } + + bb25: { -+ _4 = discriminant(_2); -+ switchInt(move _4) -> [0: bb26, 1: bb4, otherwise: bb2]; -+ } -+ -+ bb26: { -+ _1 = chained_conditions::BacktraceStyle::Off; -+ goto -> bb44; -+ } -+ -+ bb27: { -+ StorageDead(_5); -+ StorageDead(_6); -+ goto -> bb19; -+ } -+ -+ bb28: { -+ StorageDead(_5); -+ StorageDead(_6); -+ goto -> bb30; -+ } -+ -+ bb29: { + _24 = discriminant(_2); + switchInt(move _24) -> [1: bb17, otherwise: bb16]; + } + -+ bb30: { ++ bb26: { + _24 = discriminant(_2); -+ switchInt(move _24) -> [1: bb33, otherwise: bb31]; ++ switchInt(move _24) -> [1: bb29, otherwise: bb27]; + } + -+ bb31: { ++ bb27: { + _22 = const false; + _23 = const false; + StorageDead(_2); @@ -274,37 +252,25 @@ + goto -> bb11; + } + -+ bb32: { ++ bb28: { + switchInt(copy _23) -> [0: bb16, otherwise: bb18]; + } + -+ bb33: { -+ goto -> bb31; -+ } -+ -+ bb34: { -+ StorageDead(_12); -+ StorageDead(_13); -+ goto -> bb19; -+ } -+ -+ bb35: { -+ StorageDead(_12); -+ StorageDead(_13); -+ goto -> bb37; ++ bb29: { ++ goto -> bb27; + } + -+ bb36: { ++ bb30: { + _24 = discriminant(_2); + switchInt(move _24) -> [1: bb17, otherwise: bb16]; + } + -+ bb37: { ++ bb31: { + _24 = discriminant(_2); -+ switchInt(move _24) -> [1: bb40, otherwise: bb38]; ++ switchInt(move _24) -> [1: bb34, otherwise: bb32]; + } + -+ bb38: { ++ bb32: { + _22 = const false; + _23 = const false; + StorageDead(_2); @@ -312,20 +278,20 @@ + goto -> bb13; + } + -+ bb39: { ++ bb33: { + switchInt(copy _23) -> [0: bb16, otherwise: bb18]; + } + -+ bb40: { -+ goto -> bb38; ++ bb34: { ++ goto -> bb32; + } + -+ bb41: { ++ bb35: { + _24 = discriminant(_2); -+ switchInt(move _24) -> [1: bb43, otherwise: bb42]; ++ switchInt(move _24) -> [1: bb37, otherwise: bb36]; + } + -+ bb42: { ++ bb36: { + _22 = const false; + _23 = const false; + StorageDead(_2); @@ -333,16 +299,16 @@ + goto -> bb12; + } + -+ bb43: { -+ switchInt(copy _23) -> [0: bb42, otherwise: bb18]; ++ bb37: { ++ switchInt(copy _23) -> [0: bb36, otherwise: bb18]; + } + -+ bb44: { ++ bb38: { + _24 = discriminant(_2); -+ switchInt(move _24) -> [1: bb45, otherwise: bb38]; ++ switchInt(move _24) -> [1: bb39, otherwise: bb32]; + } + -+ bb45: { ++ bb39: { + goto -> bb18; } } diff --git a/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-abort.diff index 4933e42d0f40d..4955d05214ff4 100644 --- a/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-abort.diff +++ b/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-abort.diff @@ -9,13 +9,13 @@ bb0: { _2 = const true; _3 = const true; -- switchInt(copy _1) -> [0: bb3, 1: bb3, 2: bb1, otherwise: bb2]; -+ switchInt(copy _1) -> [0: bb10, 1: bb10, 2: bb9, otherwise: bb2]; + switchInt(copy _1) -> [0: bb3, 1: bb3, 2: bb1, otherwise: bb2]; } bb1: { _3 = const false; - goto -> bb4; +- goto -> bb4; ++ goto -> bb10; } bb2: { @@ -24,7 +24,8 @@ bb3: { _2 = const false; - goto -> bb4; +- goto -> bb4; ++ goto -> bb13; } bb4: { @@ -48,40 +49,30 @@ + } + + bb9: { -+ _3 = const false; -+ goto -> bb12; ++ switchInt(copy _3) -> [0: bb5, otherwise: bb7]; + } + + bb10: { -+ _2 = const false; -+ goto -> bb15; ++ goto -> bb11; + } + + bb11: { -+ switchInt(copy _3) -> [0: bb5, otherwise: bb7]; ++ goto -> bb8; + } + + bb12: { -+ goto -> bb13; ++ switchInt(copy _3) -> [0: bb5, otherwise: bb7]; + } + + bb13: { -+ goto -> bb8; ++ goto -> bb14; + } + + bb14: { -+ switchInt(copy _3) -> [0: bb5, otherwise: bb7]; ++ goto -> bb15; + } + + bb15: { -+ goto -> bb16; -+ } -+ -+ bb16: { -+ goto -> bb17; -+ } -+ -+ bb17: { goto -> bb6; } } diff --git a/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-unwind.diff index 4933e42d0f40d..4955d05214ff4 100644 --- a/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-unwind.diff +++ b/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-unwind.diff @@ -9,13 +9,13 @@ bb0: { _2 = const true; _3 = const true; -- switchInt(copy _1) -> [0: bb3, 1: bb3, 2: bb1, otherwise: bb2]; -+ switchInt(copy _1) -> [0: bb10, 1: bb10, 2: bb9, otherwise: bb2]; + switchInt(copy _1) -> [0: bb3, 1: bb3, 2: bb1, otherwise: bb2]; } bb1: { _3 = const false; - goto -> bb4; +- goto -> bb4; ++ goto -> bb10; } bb2: { @@ -24,7 +24,8 @@ bb3: { _2 = const false; - goto -> bb4; +- goto -> bb4; ++ goto -> bb13; } bb4: { @@ -48,40 +49,30 @@ + } + + bb9: { -+ _3 = const false; -+ goto -> bb12; ++ switchInt(copy _3) -> [0: bb5, otherwise: bb7]; + } + + bb10: { -+ _2 = const false; -+ goto -> bb15; ++ goto -> bb11; + } + + bb11: { -+ switchInt(copy _3) -> [0: bb5, otherwise: bb7]; ++ goto -> bb8; + } + + bb12: { -+ goto -> bb13; ++ switchInt(copy _3) -> [0: bb5, otherwise: bb7]; + } + + bb13: { -+ goto -> bb8; ++ goto -> bb14; + } + + bb14: { -+ switchInt(copy _3) -> [0: bb5, otherwise: bb7]; ++ goto -> bb15; + } + + bb15: { -+ goto -> bb16; -+ } -+ -+ bb16: { -+ goto -> bb17; -+ } -+ -+ bb17: { goto -> bb6; } } diff --git a/tests/mir-opt/jump_threading.duplicate_chain.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.duplicate_chain.JumpThreading.panic-abort.diff index d17e7d9744432..083a6e7487a05 100644 --- a/tests/mir-opt/jump_threading.duplicate_chain.JumpThreading.panic-abort.diff +++ b/tests/mir-opt/jump_threading.duplicate_chain.JumpThreading.panic-abort.diff @@ -13,14 +13,12 @@ bb1: { _2 = const 5_u8; -- goto -> bb3; -+ goto -> bb7; + goto -> bb3; } bb2: { _2 = const 5_u8; -- goto -> bb3; -+ goto -> bb7; + goto -> bb3; } bb3: { @@ -30,7 +28,8 @@ bb4: { _4 = const 15_i32; - switchInt(copy _2) -> [5: bb5, otherwise: bb6]; +- switchInt(copy _2) -> [5: bb5, otherwise: bb6]; ++ goto -> bb5; } bb5: { @@ -41,16 +40,6 @@ bb6: { _0 = const 9_u8; return; -+ } -+ -+ bb7: { -+ _3 = const 13_i32; -+ goto -> bb8; -+ } -+ -+ bb8: { -+ _4 = const 15_i32; -+ goto -> bb5; } } diff --git a/tests/mir-opt/jump_threading.duplicate_chain.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.duplicate_chain.JumpThreading.panic-unwind.diff index d17e7d9744432..083a6e7487a05 100644 --- a/tests/mir-opt/jump_threading.duplicate_chain.JumpThreading.panic-unwind.diff +++ b/tests/mir-opt/jump_threading.duplicate_chain.JumpThreading.panic-unwind.diff @@ -13,14 +13,12 @@ bb1: { _2 = const 5_u8; -- goto -> bb3; -+ goto -> bb7; + goto -> bb3; } bb2: { _2 = const 5_u8; -- goto -> bb3; -+ goto -> bb7; + goto -> bb3; } bb3: { @@ -30,7 +28,8 @@ bb4: { _4 = const 15_i32; - switchInt(copy _2) -> [5: bb5, otherwise: bb6]; +- switchInt(copy _2) -> [5: bb5, otherwise: bb6]; ++ goto -> bb5; } bb5: { @@ -41,16 +40,6 @@ bb6: { _0 = const 9_u8; return; -+ } -+ -+ bb7: { -+ _3 = const 13_i32; -+ goto -> bb8; -+ } -+ -+ bb8: { -+ _4 = const 15_i32; -+ goto -> bb5; } } diff --git a/tests/mir-opt/jump_threading.multiple_match.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.multiple_match.JumpThreading.panic-abort.diff index 271b7f00078c2..09c0ad6d48560 100644 --- a/tests/mir-opt/jump_threading.multiple_match.JumpThreading.panic-abort.diff +++ b/tests/mir-opt/jump_threading.multiple_match.JumpThreading.panic-abort.diff @@ -7,18 +7,19 @@ let mut _3: u8; bb0: { -- switchInt(copy _1) -> [3: bb1, otherwise: bb2]; -+ switchInt(copy _1) -> [3: bb9, otherwise: bb10]; + switchInt(copy _1) -> [3: bb1, otherwise: bb2]; } bb1: { _2 = copy _1; - switchInt(copy _2) -> [3: bb3, otherwise: bb4]; +- switchInt(copy _2) -> [3: bb3, otherwise: bb4]; ++ goto -> bb3; } bb2: { _3 = copy _1; - switchInt(copy _3) -> [3: bb5, otherwise: bb6]; +- switchInt(copy _3) -> [3: bb5, otherwise: bb6]; ++ goto -> bb6; } bb3: { @@ -48,16 +49,6 @@ bb8: { _0 = const 11_u8; return; -+ } -+ -+ bb9: { -+ _2 = copy _1; -+ goto -> bb3; -+ } -+ -+ bb10: { -+ _3 = copy _1; -+ goto -> bb6; } } diff --git a/tests/mir-opt/jump_threading.multiple_match.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.multiple_match.JumpThreading.panic-unwind.diff index 271b7f00078c2..09c0ad6d48560 100644 --- a/tests/mir-opt/jump_threading.multiple_match.JumpThreading.panic-unwind.diff +++ b/tests/mir-opt/jump_threading.multiple_match.JumpThreading.panic-unwind.diff @@ -7,18 +7,19 @@ let mut _3: u8; bb0: { -- switchInt(copy _1) -> [3: bb1, otherwise: bb2]; -+ switchInt(copy _1) -> [3: bb9, otherwise: bb10]; + switchInt(copy _1) -> [3: bb1, otherwise: bb2]; } bb1: { _2 = copy _1; - switchInt(copy _2) -> [3: bb3, otherwise: bb4]; +- switchInt(copy _2) -> [3: bb3, otherwise: bb4]; ++ goto -> bb3; } bb2: { _3 = copy _1; - switchInt(copy _3) -> [3: bb5, otherwise: bb6]; +- switchInt(copy _3) -> [3: bb5, otherwise: bb6]; ++ goto -> bb6; } bb3: { @@ -48,16 +49,6 @@ bb8: { _0 = const 11_u8; return; -+ } -+ -+ bb9: { -+ _2 = copy _1; -+ goto -> bb3; -+ } -+ -+ bb10: { -+ _3 = copy _1; -+ goto -> bb6; } } diff --git a/tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-abort.diff index 4f80acd073384..c8edd704ff9bd 100644 --- a/tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-abort.diff +++ b/tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-abort.diff @@ -8,13 +8,13 @@ bb0: { _3 = const false; -- switchInt(copy _1) -> [1: bb1, otherwise: bb2]; -+ switchInt(copy _1) -> [1: bb8, otherwise: bb2]; + switchInt(copy _1) -> [1: bb1, otherwise: bb2]; } bb1: { _2 = const false; - goto -> bb3; +- goto -> bb3; ++ goto -> bb9; } bb2: { @@ -47,19 +47,14 @@ + } + + bb8: { -+ _2 = const false; -+ goto -> bb10; ++ switchInt(copy _2) -> [0: bb4, otherwise: bb5]; + } + + bb9: { -+ switchInt(copy _2) -> [0: bb4, otherwise: bb5]; ++ goto -> bb10; + } + + bb10: { -+ goto -> bb11; -+ } -+ -+ bb11: { + goto -> bb6; } } diff --git a/tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-unwind.diff index 4f80acd073384..c8edd704ff9bd 100644 --- a/tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-unwind.diff +++ b/tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-unwind.diff @@ -8,13 +8,13 @@ bb0: { _3 = const false; -- switchInt(copy _1) -> [1: bb1, otherwise: bb2]; -+ switchInt(copy _1) -> [1: bb8, otherwise: bb2]; + switchInt(copy _1) -> [1: bb1, otherwise: bb2]; } bb1: { _2 = const false; - goto -> bb3; +- goto -> bb3; ++ goto -> bb9; } bb2: { @@ -47,19 +47,14 @@ + } + + bb8: { -+ _2 = const false; -+ goto -> bb10; ++ switchInt(copy _2) -> [0: bb4, otherwise: bb5]; + } + + bb9: { -+ switchInt(copy _2) -> [0: bb4, otherwise: bb5]; ++ goto -> bb10; + } + + bb10: { -+ goto -> bb11; -+ } -+ -+ bb11: { + goto -> bb6; } } diff --git a/tests/mir-opt/jump_threading.rs b/tests/mir-opt/jump_threading.rs index e841643e7ce5c..39a2f16c5ad6f 100644 --- a/tests/mir-opt/jump_threading.rs +++ b/tests/mir-opt/jump_threading.rs @@ -191,18 +191,22 @@ fn multiple_match(x: u8) -> u8 { mir! { { // CHECK: bb0: { - // CHECK: switchInt(copy [[x:_.*]]) -> [3: bb9, otherwise: bb10]; + // CHECK: switchInt(copy [[x:_.*]]) -> [3: bb1, otherwise: bb2]; match x { 3 => bb1, _ => bb2 } } bb1 = { // We know `x == 3`, so we can take `bb3`. // CHECK: bb1: { + // CHECK: {{_.*}} = copy [[x]]; + // CHECK: goto -> bb3; let y = x; match y { 3 => bb3, _ => bb4 } } bb2 = { // We know `x != 3`, so we can take `bb6`. // CHECK: bb2: { + // CHECK: [[z:_.*]] = copy [[x]]; + // CHECK: goto -> bb6; let z = x; match z { 3 => bb5, _ => bb6 } } @@ -230,7 +234,7 @@ fn multiple_match(x: u8) -> u8 { bb6 = { // We know `z != 3`, so we CANNOT take `bb7`. // CHECK: bb6: { - // CHECK: switchInt(copy [[z:_.*]]) -> [1: bb7, otherwise: bb8]; + // CHECK: switchInt(copy [[z]]) -> [1: bb7, otherwise: bb8]; match z { 1 => bb7, _ => bb8 } } bb7 = { @@ -247,14 +251,6 @@ fn multiple_match(x: u8) -> u8 { RET = 11; Return() } - // We know `x == 3`, so we can take `bb3`. - // CHECK: bb9: { - // CHECK: {{_.*}} = copy [[x]]; - // CHECK: goto -> bb3; - // We know `x != 3`, so we can take `bb6`. - // CHECK: bb10: { - // CHECK: [[z]] = copy [[x]]; - // CHECK: goto -> bb6; } } @@ -273,14 +269,14 @@ fn duplicate_chain(x: bool) -> u8 { bb1 = { // CHECK: bb1: { // CHECK: [[a:_.*]] = const 5_u8; - // CHECK: goto -> bb7; + // CHECK: goto -> bb3; a = 5; Goto(bb3) } bb2 = { // CHECK: bb2: { // CHECK: [[a]] = const 5_u8; - // CHECK: goto -> bb7; + // CHECK: goto -> bb3; a = 5; Goto(bb3) } @@ -294,7 +290,7 @@ fn duplicate_chain(x: bool) -> u8 { bb4 = { // CHECK: bb4: { // CHECK: {{_.*}} = const 15_i32; - // CHECK: switchInt(copy _2) -> [5: bb5, otherwise: bb6]; + // CHECK: goto -> bb5; let c = 15; match a { 5 => bb5, _ => bb6 } } @@ -312,12 +308,6 @@ fn duplicate_chain(x: bool) -> u8 { RET = 9; Return() } - // CHECK: bb7: { - // CHECK-NEXT: {{_.*}} = const 13_i32; - // CHECK-NEXT: goto -> bb8; - // CHECK: bb8: { - // CHECK-NEXT: {{_.*}} = const 15_i32; - // CHECK-NEXT: goto -> bb5; } } @@ -383,13 +373,13 @@ fn renumbered_bb(x: bool) -> u8 { let b: bool; { // CHECK: bb0: { - // CHECK: switchInt({{.*}}) -> [1: bb8, otherwise: bb2]; + // CHECK: switchInt({{.*}}) -> [1: bb1, otherwise: bb2]; b = false; match x { true => bb1, _ => bb2 } } bb1 = { // CHECK: bb1: { - // CHECK: goto -> bb3; + // CHECK: goto -> bb9; a = false; Goto(bb3) } @@ -428,17 +418,14 @@ fn renumbered_bb(x: bool) -> u8 { RET = 11; Return() } - // Duplicate of bb1. - // CHECK: bb8: { - // CHECK: goto -> bb10; // Duplicate of bb3. - // CHECK: bb9: { + // CHECK: bb8: { // CHECK: switchInt(copy _2) -> [0: bb4, otherwise: bb5]; - // Duplicate of bb9. - // CHECK: bb10: { - // CHECK: goto -> bb11; + // Duplicate of bb8. + // CHECK: bb9: { + // CHECK: goto -> bb10; // Duplicate of bb4. - // CHECK: bb11: { + // CHECK: bb10: { // CHECK: goto -> bb6; } } @@ -457,12 +444,12 @@ fn disappearing_bb(x: u8) -> u8 { // CHECK: bb0: { a = true; b = true; - // CHECK: switchInt({{.*}}) -> [0: bb10, 1: bb10, 2: bb9, otherwise: bb2]; + // CHECK: switchInt({{.*}}) -> [0: bb3, 1: bb3, 2: bb1, otherwise: bb2]; match x { 0 => bb3, 1 => bb3, 2 => bb1, _ => bb2 } } bb1 = { // CHECK: bb1: { - // CHECK: goto -> bb4; + // CHECK: goto -> bb10; b = false; Goto(bb4) } @@ -473,7 +460,7 @@ fn disappearing_bb(x: u8) -> u8 { } bb3 = { // CHECK: bb3: { - // CHECK: goto -> bb4; + // CHECK: goto -> bb13; a = false; Goto(bb4) } @@ -493,22 +480,18 @@ fn disappearing_bb(x: u8) -> u8 { Goto(bb6) } // CHECK: bb9: { - // CHECK: goto -> bb12; + // CHECK: switchInt(copy _3) -> [0: bb5, otherwise: bb7]; // CHECK: bb10: { - // CHECK: goto -> bb15; + // CHECK: goto -> bb11; // CHECK: bb11: { - // CHECK: switchInt(copy _3) -> [0: bb5, otherwise: bb7]; + // CHECK: goto -> bb8; // CHECK: bb12: { - // CHECK: goto -> bb13; + // CHECK: switchInt(copy _3) -> [0: bb5, otherwise: bb7]; // CHECK: bb13: { - // CHECK: goto -> bb8; + // CHECK: goto -> bb14; // CHECK: bb14: { - // CHECK: switchInt(copy _3) -> [0: bb5, otherwise: bb7]; + // CHECK: goto -> bb15; // CHECK: bb15: { - // CHECK: goto -> bb16; - // CHECK: bb16: { - // CHECK: goto -> bb17; - // CHECK: bb17: { // CHECK: goto -> bb6; } } From 87ac07f73bffcf0ef1c4b804d0ebaa4c9e80c012 Mon Sep 17 00:00:00 2001 From: Camille Gillot Date: Sat, 20 Sep 2025 13:40:38 +0000 Subject: [PATCH 9/9] Filter costly chains after simplification. --- .../rustc_mir_transform/src/jump_threading.rs | 111 +++++++++++++----- ..._conditions.JumpThreading.panic-abort.diff | 86 ++------------ ...conditions.JumpThreading.panic-unwind.diff | 76 ++---------- 3 files changed, 96 insertions(+), 177 deletions(-) diff --git a/compiler/rustc_mir_transform/src/jump_threading.rs b/compiler/rustc_mir_transform/src/jump_threading.rs index 1b6a0203519c4..c021e7d4c3ae9 100644 --- a/compiler/rustc_mir_transform/src/jump_threading.rs +++ b/compiler/rustc_mir_transform/src/jump_threading.rs @@ -51,8 +51,6 @@ //! //! [libfirm]: -use std::cell::OnceCell; - use itertools::Itertools as _; use rustc_const_eval::const_eval::DummyMachine; use rustc_const_eval::interpret::{ImmTy, Immediate, InterpCx, OpTy, Projectable}; @@ -100,7 +98,6 @@ impl<'tcx> crate::MirPass<'tcx> for JumpThreading { map: Map::new(tcx, body, Some(MAX_PLACES)), maybe_loop_headers: loops::maybe_loop_headers(body), entry_states: IndexVec::from_elem(ConditionSet::default(), &body.basic_blocks), - costs: IndexVec::from_elem(OnceCell::new(), &body.basic_blocks), }; for (bb, bbdata) in traversal::postorder(body) { @@ -136,6 +133,7 @@ impl<'tcx> crate::MirPass<'tcx> for JumpThreading { let mut entry_states = finder.entry_states; simplify_conditions(body, &mut entry_states); + remove_costly_conditions(tcx, typing_env, body, &mut entry_states); if let Some(opportunities) = OpportunitySet::new(body, entry_states) { opportunities.apply(); @@ -159,8 +157,6 @@ struct TOFinder<'a, 'tcx> { // Invariant: for each `bb`, each condition in `entry_states[bb]` has a `chain` that // starts with `bb`. entry_states: IndexVec, - /// Pre-computed cost of duplicating each block. - costs: IndexVec>, } rustc_index::newtype_index! { @@ -222,7 +218,6 @@ struct ConditionSet { active: Vec<(ConditionIndex, Condition)>, fulfilled: Vec, targets: IndexVec>, - costs: IndexVec, } impl ConditionSet { @@ -233,7 +228,6 @@ impl ConditionSet { #[tracing::instrument(level = "trace", skip(self))] fn push_condition(&mut self, c: Condition, target: BasicBlock) { let index = self.targets.push(vec![EdgeEffect::Goto { target }]); - self.costs.push(0); self.active.push((index, c)); } @@ -293,21 +287,18 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { active: Vec::with_capacity(state_len), targets: IndexVec::with_capacity(state_len), fulfilled: Vec::new(), - costs: IndexVec::with_capacity(state_len), }; // Use an index-set to deduplicate conditions coming from different successor blocks. let mut known_conditions = FxIndexSet::with_capacity_and_hasher(state_len, Default::default()); - let mut insert = |condition, succ_block, succ_condition, cost| { + let mut insert = |condition, succ_block, succ_condition| { let (index, new) = known_conditions.insert_full(condition); let index = ConditionIndex::from_usize(index); if new { state.active.push((index, condition)); let _index = state.targets.push(Vec::new()); debug_assert_eq!(_index, index); - let _index = state.costs.push(u8::MAX); - debug_assert_eq!(_index, index); } let target = EdgeEffect::Chain { succ_block, succ_condition }; debug_assert!( @@ -316,7 +307,6 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { &state.targets[index], ); state.targets[index].push(target); - state.costs[index] = std::cmp::min(state.costs[index], cost); }; // A given block may have several times the same successor. @@ -331,35 +321,19 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { continue; } - let succ_cost = self.cost(succ); for &(succ_index, cond) in self.entry_states[succ].active.iter() { - let cost = self.entry_states[succ].costs[succ_index]; - if let Ok(cost) = ((cost as usize) + succ_cost).try_into() - && cost < MAX_COST - { - insert(cond, succ, succ_index, cost); - } + insert(cond, succ, succ_index); } } let num_conditions = known_conditions.len(); debug_assert_eq!(num_conditions, state.active.len()); debug_assert_eq!(num_conditions, state.targets.len()); - debug_assert_eq!(num_conditions, state.costs.len()); state.fulfilled.reserve(num_conditions); state } - fn cost(&self, bb: BasicBlock) -> usize { - *self.costs[bb].get_or_init(|| { - let bbdata = &self.body[bb]; - let mut cost = CostChecker::new(self.tcx, self.typing_env, None, self.body); - cost.visit_basic_block_data(bb, bbdata); - cost.cost() - }) - } - /// Remove all conditions in the state that alias given place. fn flood_state( &self, @@ -751,8 +725,6 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { // Fulfilling `index` may thread conditions that we do not want, // so create a brand new index to immediately mark fulfilled. let index = state.targets.push(new_edges); - let _index = state.costs.push(0); - debug_assert_eq!(_index, index); state.fulfilled.push(index); } } @@ -865,6 +837,82 @@ fn simplify_conditions(body: &Body<'_>, entry_states: &mut IndexVec( + tcx: TyCtxt<'tcx>, + typing_env: ty::TypingEnv<'tcx>, + body: &Body<'tcx>, + entry_states: &mut IndexVec, +) { + let basic_blocks = &body.basic_blocks; + + let mut costs = IndexVec::from_elem(None, basic_blocks); + let mut cost = |bb: BasicBlock| -> u8 { + let c = *costs[bb].get_or_insert_with(|| { + let bbdata = &basic_blocks[bb]; + let mut cost = CostChecker::new(tcx, typing_env, None, body); + cost.visit_basic_block_data(bb, bbdata); + cost.cost().try_into().unwrap_or(MAX_COST) + }); + trace!("cost[{bb:?}] = {c}"); + c + }; + + // Initialize costs with `MAX_COST`: if we have a cycle, the cyclic `bb` has infinite costs. + let mut condition_cost = IndexVec::from_fn_n( + |bb: BasicBlock| IndexVec::from_elem_n(MAX_COST, entry_states[bb].targets.len()), + entry_states.len(), + ); + + let reverse_postorder = basic_blocks.reverse_postorder(); + + for &bb in reverse_postorder.iter().rev() { + let state = &entry_states[bb]; + trace!(?bb, ?state); + + let mut current_costs = IndexVec::from_elem(0u8, &state.targets); + + for (condition, targets) in state.targets.iter_enumerated() { + for &target in targets { + match target { + // A `Goto` has cost 0. + EdgeEffect::Goto { .. } => {} + // Chaining into an already-fulfilled condition is nop. + EdgeEffect::Chain { succ_block, succ_condition } + if entry_states[succ_block].fulfilled.contains(&succ_condition) => {} + // When chaining, use `cost[succ_block][succ_condition] + cost(succ_block)`. + EdgeEffect::Chain { succ_block, succ_condition } => { + // Cost associated with duplicating `succ_block`. + let duplication_cost = cost(succ_block); + // Cost associated with the rest of the chain. + let target_cost = + *condition_cost[succ_block].get(succ_condition).unwrap_or(&MAX_COST); + let cost = current_costs[condition] + .saturating_add(duplication_cost) + .saturating_add(target_cost); + trace!(?condition, ?succ_block, ?duplication_cost, ?target_cost); + current_costs[condition] = cost; + } + } + } + } + + trace!("condition_cost[{bb:?}] = {:?}", current_costs); + condition_cost[bb] = current_costs; + } + + trace!(?condition_cost); + + for &bb in reverse_postorder { + for (index, targets) in entry_states[bb].targets.iter_enumerated_mut() { + if condition_cost[bb][index] >= MAX_COST { + trace!(?bb, ?index, ?targets, c = ?condition_cost[bb][index], "remove"); + targets.clear() + } + } + } +} + struct OpportunitySet<'a, 'tcx> { basic_blocks: &'a mut IndexVec>, entry_states: IndexVec, @@ -887,7 +935,6 @@ impl<'a, 'tcx> OpportunitySet<'a, 'tcx> { // Free some memory, because we will need to clone condition sets. for state in entry_states.iter_mut() { state.active = Default::default(); - state.costs = Default::default(); } let duplicates = Default::default(); let basic_blocks = body.basic_blocks.as_mut(); diff --git a/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-abort.diff index bbe9ba806dc72..f09a187bfaf8d 100644 --- a/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-abort.diff +++ b/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-abort.diff @@ -71,7 +71,7 @@ bb3: { _1 = chained_conditions::BacktraceStyle::Off; - goto -> bb18; -+ goto -> bb35; ++ goto -> bb23; } bb4: { @@ -131,7 +131,7 @@ StorageDead(_5); StorageDead(_6); - goto -> bb18; -+ goto -> bb22; ++ goto -> bb21; } bb8: { @@ -154,14 +154,14 @@ StorageDead(_13); _1 = chained_conditions::BacktraceStyle::Short; - goto -> bb18; -+ goto -> bb31; ++ goto -> bb23; } bb10: { StorageDead(_12); StorageDead(_13); - goto -> bb18; -+ goto -> bb27; ++ goto -> bb21; } bb11: { @@ -219,88 +219,20 @@ + + bb21: { + _24 = discriminant(_2); -+ switchInt(move _24) -> [1: bb16, otherwise: bb15]; ++ switchInt(move _24) -> [1: bb22, otherwise: bb15]; + } + + bb22: { -+ _24 = discriminant(_2); -+ switchInt(move _24) -> [1: bb25, otherwise: bb23]; ++ goto -> bb15; + } + + bb23: { -+ _22 = const false; -+ _23 = const false; -+ StorageDead(_2); -+ _19 = discriminant(_1); -+ goto -> bb11; -+ } -+ -+ bb24: { -+ switchInt(copy _23) -> [0: bb15, otherwise: bb17]; -+ } -+ -+ bb25: { -+ goto -> bb23; -+ } -+ -+ bb26: { -+ _24 = discriminant(_2); -+ switchInt(move _24) -> [1: bb16, otherwise: bb15]; -+ } -+ -+ bb27: { + _24 = discriminant(_2); -+ switchInt(move _24) -> [1: bb30, otherwise: bb28]; -+ } -+ -+ bb28: { -+ _22 = const false; -+ _23 = const false; -+ StorageDead(_2); -+ _19 = discriminant(_1); -+ goto -> bb13; -+ } -+ -+ bb29: { -+ switchInt(copy _23) -> [0: bb15, otherwise: bb17]; ++ switchInt(move _24) -> [1: bb24, otherwise: bb15]; + } + -+ bb30: { -+ goto -> bb28; -+ } -+ -+ bb31: { -+ _24 = discriminant(_2); -+ switchInt(move _24) -> [1: bb33, otherwise: bb32]; -+ } -+ -+ bb32: { -+ _22 = const false; -+ _23 = const false; -+ StorageDead(_2); -+ _19 = discriminant(_1); -+ goto -> bb12; -+ } -+ -+ bb33: { -+ switchInt(copy _23) -> [0: bb32, otherwise: bb34]; -+ } -+ -+ bb34: { -+ drop(((_2 as Some).0: std::string::String)) -> [return: bb32, unwind unreachable]; -+ } -+ -+ bb35: { -+ _24 = discriminant(_2); -+ switchInt(move _24) -> [1: bb36, otherwise: bb28]; -+ } -+ -+ bb36: { -+ goto -> bb37; -+ } -+ -+ bb37: { -+ drop(((_2 as Some).0: std::string::String)) -> [return: bb28, unwind unreachable]; ++ bb24: { ++ goto -> bb17; } } diff --git a/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-unwind.diff index a40e4446aef17..afd40c1862c3c 100644 --- a/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-unwind.diff +++ b/tests/mir-opt/jump_threading.chained_conditions.JumpThreading.panic-unwind.diff @@ -71,7 +71,7 @@ bb3: { _1 = chained_conditions::BacktraceStyle::Off; - goto -> bb19; -+ goto -> bb38; ++ goto -> bb27; } bb4: { @@ -131,7 +131,7 @@ StorageDead(_5); StorageDead(_6); - goto -> bb19; -+ goto -> bb26; ++ goto -> bb25; } bb8: { @@ -154,14 +154,14 @@ StorageDead(_13); _1 = chained_conditions::BacktraceStyle::Short; - goto -> bb19; -+ goto -> bb35; ++ goto -> bb27; } bb10: { StorageDead(_12); StorageDead(_13); - goto -> bb19; -+ goto -> bb31; ++ goto -> bb25; } bb11: { @@ -236,79 +236,19 @@ + + bb25: { + _24 = discriminant(_2); -+ switchInt(move _24) -> [1: bb17, otherwise: bb16]; ++ switchInt(move _24) -> [1: bb26, otherwise: bb16]; + } + + bb26: { -+ _24 = discriminant(_2); -+ switchInt(move _24) -> [1: bb29, otherwise: bb27]; ++ goto -> bb16; + } + + bb27: { -+ _22 = const false; -+ _23 = const false; -+ StorageDead(_2); -+ _19 = discriminant(_1); -+ goto -> bb11; -+ } -+ -+ bb28: { -+ switchInt(copy _23) -> [0: bb16, otherwise: bb18]; -+ } -+ -+ bb29: { -+ goto -> bb27; -+ } -+ -+ bb30: { -+ _24 = discriminant(_2); -+ switchInt(move _24) -> [1: bb17, otherwise: bb16]; -+ } -+ -+ bb31: { + _24 = discriminant(_2); -+ switchInt(move _24) -> [1: bb34, otherwise: bb32]; -+ } -+ -+ bb32: { -+ _22 = const false; -+ _23 = const false; -+ StorageDead(_2); -+ _19 = discriminant(_1); -+ goto -> bb13; ++ switchInt(move _24) -> [1: bb28, otherwise: bb16]; + } + -+ bb33: { -+ switchInt(copy _23) -> [0: bb16, otherwise: bb18]; -+ } -+ -+ bb34: { -+ goto -> bb32; -+ } -+ -+ bb35: { -+ _24 = discriminant(_2); -+ switchInt(move _24) -> [1: bb37, otherwise: bb36]; -+ } -+ -+ bb36: { -+ _22 = const false; -+ _23 = const false; -+ StorageDead(_2); -+ _19 = discriminant(_1); -+ goto -> bb12; -+ } -+ -+ bb37: { -+ switchInt(copy _23) -> [0: bb36, otherwise: bb18]; -+ } -+ -+ bb38: { -+ _24 = discriminant(_2); -+ switchInt(move _24) -> [1: bb39, otherwise: bb32]; -+ } -+ -+ bb39: { ++ bb28: { + goto -> bb18; } }