@@ -541,142 +541,141 @@ impl<'tcx> Inliner<'tcx> {
541541 mut callee_body : Body < ' tcx > ,
542542 ) {
543543 let terminator = caller_body[ callsite. block ] . terminator . take ( ) . unwrap ( ) ;
544- match terminator. kind {
545- TerminatorKind :: Call { args, destination, unwind, .. } => {
546- // If the call is something like `a[*i] = f(i)`, where
547- // `i : &mut usize`, then just duplicating the `a[*i]`
548- // Place could result in two different locations if `f`
549- // writes to `i`. To prevent this we need to create a temporary
550- // borrow of the place and pass the destination as `*temp` instead.
551- fn dest_needs_borrow ( place : Place < ' _ > ) -> bool {
552- for elem in place. projection . iter ( ) {
553- match elem {
554- ProjectionElem :: Deref | ProjectionElem :: Index ( _) => return true ,
555- _ => { }
556- }
557- }
544+ let TerminatorKind :: Call { args, destination, unwind, .. } = terminator. kind else {
545+ bug ! ( "unexpected terminator kind {:?}" , terminator. kind) ;
546+ } ;
558547
559- false
548+ // If the call is something like `a[*i] = f(i)`, where
549+ // `i : &mut usize`, then just duplicating the `a[*i]`
550+ // Place could result in two different locations if `f`
551+ // writes to `i`. To prevent this we need to create a temporary
552+ // borrow of the place and pass the destination as `*temp` instead.
553+ fn dest_needs_borrow ( place : Place < ' _ > ) -> bool {
554+ for elem in place. projection . iter ( ) {
555+ match elem {
556+ ProjectionElem :: Deref | ProjectionElem :: Index ( _) => return true ,
557+ _ => { }
560558 }
559+ }
561560
562- let dest = if dest_needs_borrow ( destination) {
563- trace ! ( "creating temp for return destination" ) ;
564- let dest = Rvalue :: Ref (
565- self . tcx . lifetimes . re_erased ,
566- BorrowKind :: Mut { kind : MutBorrowKind :: Default } ,
567- destination,
568- ) ;
569- let dest_ty = dest. ty ( caller_body, self . tcx ) ;
570- let temp = Place :: from ( self . new_call_temp ( caller_body, & callsite, dest_ty) ) ;
571- caller_body[ callsite. block ] . statements . push ( Statement {
572- source_info : callsite. source_info ,
573- kind : StatementKind :: Assign ( Box :: new ( ( temp, dest) ) ) ,
574- } ) ;
575- self . tcx . mk_place_deref ( temp)
576- } else {
577- destination
578- } ;
561+ false
562+ }
579563
580- // Always create a local to hold the destination, as `RETURN_PLACE` may appear
581- // where a full `Place` is not allowed.
582- let ( remap_destination, destination_local) = if let Some ( d) = dest. as_local ( ) {
583- ( false , d)
584- } else {
585- (
586- true ,
587- self . new_call_temp (
588- caller_body,
589- & callsite,
590- destination. ty ( caller_body, self . tcx ) . ty ,
591- ) ,
592- )
593- } ;
564+ let dest = if dest_needs_borrow ( destination) {
565+ trace ! ( "creating temp for return destination" ) ;
566+ let dest = Rvalue :: Ref (
567+ self . tcx . lifetimes . re_erased ,
568+ BorrowKind :: Mut { kind : MutBorrowKind :: Default } ,
569+ destination,
570+ ) ;
571+ let dest_ty = dest. ty ( caller_body, self . tcx ) ;
572+ let temp = Place :: from ( self . new_call_temp ( caller_body, & callsite, dest_ty) ) ;
573+ caller_body[ callsite. block ] . statements . push ( Statement {
574+ source_info : callsite. source_info ,
575+ kind : StatementKind :: Assign ( Box :: new ( ( temp, dest) ) ) ,
576+ } ) ;
577+ self . tcx . mk_place_deref ( temp)
578+ } else {
579+ destination
580+ } ;
594581
595- // Copy the arguments if needed.
596- let args: Vec < _ > = self . make_call_args ( args, & callsite, caller_body, & callee_body) ;
597-
598- let mut integrator = Integrator {
599- args : & args,
600- new_locals : Local :: new ( caller_body. local_decls . len ( ) ) ..,
601- new_scopes : SourceScope :: new ( caller_body. source_scopes . len ( ) ) ..,
602- new_blocks : BasicBlock :: new ( caller_body. basic_blocks . len ( ) ) ..,
603- destination : destination_local,
604- callsite_scope : caller_body. source_scopes [ callsite. source_info . scope ] . clone ( ) ,
605- callsite,
606- cleanup_block : unwind,
607- in_cleanup_block : false ,
608- tcx : self . tcx ,
609- always_live_locals : BitSet :: new_filled ( callee_body. local_decls . len ( ) ) ,
610- } ;
582+ // Always create a local to hold the destination, as `RETURN_PLACE` may appear
583+ // where a full `Place` is not allowed.
584+ let ( remap_destination, destination_local) = if let Some ( d) = dest. as_local ( ) {
585+ ( false , d)
586+ } else {
587+ (
588+ true ,
589+ self . new_call_temp (
590+ caller_body,
591+ & callsite,
592+ destination. ty ( caller_body, self . tcx ) . ty ,
593+ ) ,
594+ )
595+ } ;
611596
612- // Map all `Local`s, `SourceScope`s and `BasicBlock`s to new ones
613- // (or existing ones, in a few special cases) in the caller.
614- integrator. visit_body ( & mut callee_body) ;
615-
616- // If there are any locals without storage markers, give them storage only for the
617- // duration of the call.
618- for local in callee_body. vars_and_temps_iter ( ) {
619- if integrator. always_live_locals . contains ( local) {
620- let new_local = integrator. map_local ( local) ;
621- caller_body[ callsite. block ] . statements . push ( Statement {
622- source_info : callsite. source_info ,
623- kind : StatementKind :: StorageLive ( new_local) ,
624- } ) ;
625- }
626- }
627- if let Some ( block) = callsite. target {
628- // To avoid repeated O(n) insert, push any new statements to the end and rotate
629- // the slice once.
630- let mut n = 0 ;
631- if remap_destination {
632- caller_body[ block] . statements . push ( Statement {
633- source_info : callsite. source_info ,
634- kind : StatementKind :: Assign ( Box :: new ( (
635- dest,
636- Rvalue :: Use ( Operand :: Move ( destination_local. into ( ) ) ) ,
637- ) ) ) ,
638- } ) ;
639- n += 1 ;
640- }
641- for local in callee_body. vars_and_temps_iter ( ) . rev ( ) {
642- if integrator. always_live_locals . contains ( local) {
643- let new_local = integrator. map_local ( local) ;
644- caller_body[ block] . statements . push ( Statement {
645- source_info : callsite. source_info ,
646- kind : StatementKind :: StorageDead ( new_local) ,
647- } ) ;
648- n += 1 ;
649- }
650- }
651- caller_body[ block] . statements . rotate_right ( n) ;
652- }
597+ // Copy the arguments if needed.
598+ let args: Vec < _ > = self . make_call_args ( args, & callsite, caller_body, & callee_body) ;
599+
600+ let mut integrator = Integrator {
601+ args : & args,
602+ new_locals : Local :: new ( caller_body. local_decls . len ( ) ) ..,
603+ new_scopes : SourceScope :: new ( caller_body. source_scopes . len ( ) ) ..,
604+ new_blocks : BasicBlock :: new ( caller_body. basic_blocks . len ( ) ) ..,
605+ destination : destination_local,
606+ callsite_scope : caller_body. source_scopes [ callsite. source_info . scope ] . clone ( ) ,
607+ callsite,
608+ cleanup_block : unwind,
609+ in_cleanup_block : false ,
610+ tcx : self . tcx ,
611+ always_live_locals : BitSet :: new_filled ( callee_body. local_decls . len ( ) ) ,
612+ } ;
653613
654- // Insert all of the (mapped) parts of the callee body into the caller.
655- caller_body. local_decls . extend ( callee_body. drain_vars_and_temps ( ) ) ;
656- caller_body. source_scopes . extend ( & mut callee_body. source_scopes . drain ( ..) ) ;
657- caller_body. var_debug_info . append ( & mut callee_body. var_debug_info ) ;
658- caller_body. basic_blocks_mut ( ) . extend ( callee_body. basic_blocks_mut ( ) . drain ( ..) ) ;
614+ // Map all `Local`s, `SourceScope`s and `BasicBlock`s to new ones
615+ // (or existing ones, in a few special cases) in the caller.
616+ integrator. visit_body ( & mut callee_body) ;
659617
660- caller_body[ callsite. block ] . terminator = Some ( Terminator {
618+ // If there are any locals without storage markers, give them storage only for the
619+ // duration of the call.
620+ for local in callee_body. vars_and_temps_iter ( ) {
621+ if integrator. always_live_locals . contains ( local) {
622+ let new_local = integrator. map_local ( local) ;
623+ caller_body[ callsite. block ] . statements . push ( Statement {
661624 source_info : callsite. source_info ,
662- kind : TerminatorKind :: Goto { target : integrator . map_block ( START_BLOCK ) } ,
625+ kind : StatementKind :: StorageLive ( new_local ) ,
663626 } ) ;
664-
665- // Copy only unevaluated constants from the callee_body into the caller_body.
666- // Although we are only pushing `ConstKind::Unevaluated` consts to
667- // `required_consts`, here we may not only have `ConstKind::Unevaluated`
668- // because we are calling `instantiate_and_normalize_erasing_regions`.
669- caller_body. required_consts . extend (
670- callee_body. required_consts . iter ( ) . copied ( ) . filter ( |& ct| match ct. const_ {
671- Const :: Ty ( _) => {
672- bug ! ( "should never encounter ty::UnevaluatedConst in `required_consts`" )
673- }
674- Const :: Val ( ..) | Const :: Unevaluated ( ..) => true ,
675- } ) ,
676- ) ;
677627 }
678- kind => bug ! ( "unexpected terminator kind {:?}" , kind) ,
679628 }
629+ if let Some ( block) = callsite. target {
630+ // To avoid repeated O(n) insert, push any new statements to the end and rotate
631+ // the slice once.
632+ let mut n = 0 ;
633+ if remap_destination {
634+ caller_body[ block] . statements . push ( Statement {
635+ source_info : callsite. source_info ,
636+ kind : StatementKind :: Assign ( Box :: new ( (
637+ dest,
638+ Rvalue :: Use ( Operand :: Move ( destination_local. into ( ) ) ) ,
639+ ) ) ) ,
640+ } ) ;
641+ n += 1 ;
642+ }
643+ for local in callee_body. vars_and_temps_iter ( ) . rev ( ) {
644+ if integrator. always_live_locals . contains ( local) {
645+ let new_local = integrator. map_local ( local) ;
646+ caller_body[ block] . statements . push ( Statement {
647+ source_info : callsite. source_info ,
648+ kind : StatementKind :: StorageDead ( new_local) ,
649+ } ) ;
650+ n += 1 ;
651+ }
652+ }
653+ caller_body[ block] . statements . rotate_right ( n) ;
654+ }
655+
656+ // Insert all of the (mapped) parts of the callee body into the caller.
657+ caller_body. local_decls . extend ( callee_body. drain_vars_and_temps ( ) ) ;
658+ caller_body. source_scopes . extend ( & mut callee_body. source_scopes . drain ( ..) ) ;
659+ caller_body. var_debug_info . append ( & mut callee_body. var_debug_info ) ;
660+ caller_body. basic_blocks_mut ( ) . extend ( callee_body. basic_blocks_mut ( ) . drain ( ..) ) ;
661+
662+ caller_body[ callsite. block ] . terminator = Some ( Terminator {
663+ source_info : callsite. source_info ,
664+ kind : TerminatorKind :: Goto { target : integrator. map_block ( START_BLOCK ) } ,
665+ } ) ;
666+
667+ // Copy only unevaluated constants from the callee_body into the caller_body.
668+ // Although we are only pushing `ConstKind::Unevaluated` consts to
669+ // `required_consts`, here we may not only have `ConstKind::Unevaluated`
670+ // because we are calling `instantiate_and_normalize_erasing_regions`.
671+ caller_body. required_consts . extend ( callee_body. required_consts . iter ( ) . copied ( ) . filter (
672+ |& ct| match ct. const_ {
673+ Const :: Ty ( _) => {
674+ bug ! ( "should never encounter ty::UnevaluatedConst in `required_consts`" )
675+ }
676+ Const :: Val ( ..) | Const :: Unevaluated ( ..) => true ,
677+ } ,
678+ ) ) ;
680679 }
681680
682681 fn make_call_args (
0 commit comments