Implement profiling for compiler-generated move/copy operations (MIR transform)

compiler/rustc_borrowck/src/lib.rs

@@ -904,22 +904,14 @@ impl<'a, 'tcx> ResultsVisitor<'tcx, Borrowck<'a, 'tcx>> for MirBorrowckCtxt<'a,
                     state,
                 );
             }
-            TerminatorKind::Call {
-                func,
-                args,
-                destination,
-                target: _,
-                unwind: _,
-                call_source: _,
-                fn_span: _,
-            } => {
+            TerminatorKind::Call { func, args, destination, .. } => {
                 self.consume_operand(loc, (func, span), state);
                 for arg in args {
                     self.consume_operand(loc, (&arg.node, arg.span), state);
                 }
                 self.mutate_place(loc, (*destination, span), Deep, state);
             }
-            TerminatorKind::TailCall { func, args, fn_span: _ } => {
+            TerminatorKind::TailCall { func, args, .. } => {
                 self.consume_operand(loc, (func, span), state);
                 for arg in args {
                     self.consume_operand(loc, (&arg.node, arg.span), state);

compiler/rustc_borrowck/src/polonius/legacy/loan_invalidations.rs

@@ -118,15 +118,7 @@ impl<'a, 'tcx> Visitor<'tcx> for LoanInvalidationsGenerator<'a, 'tcx> {
                     LocalMutationIsAllowed::Yes,
                 );
             }
-            TerminatorKind::Call {
-                func,
-                args,
-                destination,
-                target: _,
-                unwind: _,
-                call_source: _,
-                fn_span: _,
-            } => {
+            TerminatorKind::Call { func, args, destination, .. } => {
                 self.consume_operand(location, func);
                 for arg in args {
                     self.consume_operand(location, &arg.node);

compiler/rustc_codegen_cranelift/src/base.rs

@@ -478,15 +478,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
                     switch.emit(&mut fx.bcx, discr, otherwise_block);
                 }
             }
-            TerminatorKind::Call {
-                func,
-                args,
-                destination,
-                target,
-                fn_span,
-                unwind,
-                call_source: _,
-            } => {
+            TerminatorKind::Call { func, args, destination, target, fn_span, unwind, .. } => {
                 fx.tcx.prof.generic_activity("codegen call").run(|| {
                     crate::abi::codegen_terminator_call(
                         fx,

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -1151,6 +1151,19 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 _ => {}
             }
 
+            // Look up stored SourceInfo for this argument if it exists (from annotate_moves pass)
+            let arg_source_info = match &terminator.kind {
+                mir::TerminatorKind::Call { arg_move_source_info, .. }
+                | mir::TerminatorKind::TailCall { arg_move_source_info, .. } => {
+                    arg_move_source_info.as_ref().and_then(|infos| infos.get(i).copied().flatten())
+                }
+                _ => None,
+            };
+
+            if let Some(arg_source_info) = arg_source_info {
+                self.set_debug_loc(bx, arg_source_info);
+            }
+
             self.codegen_argument(
                 bx,
                 op,
@@ -1444,8 +1457,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 destination,
                 target,
                 unwind,
-                call_source: _,
                 fn_span,
+                ..
             } => self.codegen_call_terminator(
                 helper,
                 bx,
@@ -1459,7 +1472,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 CallKind::Normal,
                 mergeable_succ(),
             ),
-            mir::TerminatorKind::TailCall { ref func, ref args, fn_span } => self
+            mir::TerminatorKind::TailCall { ref func, ref args, fn_span, .. } => self
                 .codegen_call_terminator(
                     helper,
                     bx,

compiler/rustc_const_eval/src/interpret/step.rs

@@ -544,15 +544,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                 self.go_to_block(target_block);
             }
 
-            Call {
-                ref func,
-                ref args,
-                destination,
-                target,
-                unwind,
-                call_source: _,
-                fn_span: _,
-            } => {
+            Call { ref func, ref args, destination, target, unwind, .. } => {
                 let old_stack = self.frame_idx();
                 let old_loc = self.frame().loc;
 
@@ -576,7 +568,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                 }
             }
 
-            TailCall { ref func, ref args, fn_span: _ } => {
+            TailCall { ref func, ref args, .. } => {
                 let old_frame_idx = self.frame_idx();
 
                 let EvaluatedCalleeAndArgs { callee, args, fn_sig, fn_abi, with_caller_location } =

compiler/rustc_middle/src/mir/mod.rs

@@ -1664,11 +1664,11 @@ mod size_asserts {
 
     use super::*;
     // tidy-alphabetical-start
-    static_assert_size!(BasicBlockData<'_>, 128);
+    static_assert_size!(BasicBlockData<'_>, 144);
     static_assert_size!(LocalDecl<'_>, 40);
     static_assert_size!(SourceScopeData<'_>, 64);
     static_assert_size!(Statement<'_>, 32);
-    static_assert_size!(Terminator<'_>, 96);
+    static_assert_size!(Terminator<'_>, 112);
     static_assert_size!(VarDebugInfo<'_>, 88);
     // tidy-alphabetical-end
 }

compiler/rustc_middle/src/mir/syntax.rs

@@ -16,7 +16,7 @@ use rustc_span::{Span, Symbol};
 use rustc_target::asm::InlineAsmRegOrRegClass;
 use smallvec::SmallVec;
 
-use super::{BasicBlock, Const, Local, UserTypeProjection};
+use super::{BasicBlock, Const, Local, SourceInfo, UserTypeProjection};
 use crate::mir::coverage::CoverageKind;
 use crate::ty::adjustment::PointerCoercion;
 use crate::ty::{self, GenericArgsRef, List, Region, Ty, UserTypeAnnotationIndex};
@@ -812,7 +812,7 @@ pub enum TerminatorKind<'tcx> {
     ///
     /// [#71117]: https://github.com/rust-lang/rust/issues/71117
     Call {
-        /// The function that’s being called.
+        /// The function that's being called.
         func: Operand<'tcx>,
         /// Arguments the function is called with.
         /// These are owned by the callee, which is free to modify them.
@@ -832,6 +832,13 @@ pub enum TerminatorKind<'tcx> {
         /// This `Span` is the span of the function, without the dot and receiver
         /// e.g. `foo(a, b)` in `x.foo(a, b)`
         fn_span: Span,
+        /// Optional array of source info for move operations in the arguments.
+        /// This is used to make the move operations appear in profilers as if they
+        /// were inlined from compiler_move intrinsics.
+        /// If None, no special source info is used.
+        /// If Some, the array has the same length as args, with None for arguments
+        /// that don't need special source info.
+        arg_move_source_info: Option<Box<[Option<SourceInfo>]>>,
     },
 
     /// Tail call.
@@ -851,7 +858,7 @@ pub enum TerminatorKind<'tcx> {
     /// [`Call`]: TerminatorKind::Call
     /// [`Return`]: TerminatorKind::Return
     TailCall {
-        /// The function that’s being called.
+        /// The function that's being called.
         func: Operand<'tcx>,
         /// Arguments the function is called with.
         /// These are owned by the callee, which is free to modify them.
@@ -862,6 +869,13 @@ pub enum TerminatorKind<'tcx> {
         /// This `Span` is the span of the function, without the dot and receiver
         /// (e.g. `foo(a, b)` in `x.foo(a, b)`
         fn_span: Span,
+        /// Optional array of source info for move operations in the arguments.
+        /// This is used to make the move operations appear in profilers as if they
+        /// were inlined from compiler_move intrinsics.
+        /// If None, no special source info is used.
+        /// If Some, the array has the same length as args, with None for arguments
+        /// that don't need special source info.
+        arg_move_source_info: Option<Box<[Option<SourceInfo>]>>,
     },
 
     /// Evaluates the operand, which must have type `bool`. If it is not equal to `expected`,
@@ -1734,6 +1748,6 @@ mod size_asserts {
     static_assert_size!(PlaceElem<'_>, 24);
     static_assert_size!(Rvalue<'_>, 40);
     static_assert_size!(StatementKind<'_>, 16);
-    static_assert_size!(TerminatorKind<'_>, 80);
+    static_assert_size!(TerminatorKind<'_>, 96);
     // tidy-alphabetical-end
 }

compiler/rustc_middle/src/mir/terminator.rs

@@ -779,15 +779,7 @@ impl<'tcx> TerminatorKind<'tcx> {
                 }
             }
 
-            Call {
-                unwind,
-                destination,
-                ref target,
-                func: _,
-                args: _,
-                fn_span: _,
-                call_source: _,
-            } => TerminatorEdges::AssignOnReturn {
+            Call { unwind, destination, ref target, .. } => TerminatorEdges::AssignOnReturn {
                 return_: target.as_ref().map(slice::from_ref).unwrap_or_default(),
                 cleanup: unwind.cleanup_block(),
                 place: CallReturnPlaces::Call(destination),

compiler/rustc_middle/src/mir/visit.rs

@@ -551,10 +551,8 @@ macro_rules! make_mir_visitor {
                         func,
                         args,
                         destination,
-                        target: _,
-                        unwind: _,
-                        call_source: _,
                         fn_span,
+                        ..
                     } => {
                         self.visit_span($(& $mutability)? *fn_span);
                         self.visit_operand(func, location);
@@ -568,7 +566,7 @@ macro_rules! make_mir_visitor {
                         );
                     }
 
-                    TerminatorKind::TailCall { func, args, fn_span } => {
+                    TerminatorKind::TailCall { func, args, fn_span, .. } => {
                         self.visit_span($(& $mutability)? *fn_span);
                         self.visit_operand(func, location);
                         for arg in args {

compiler/rustc_mir_build/src/builder/custom/parse/instruction.rs

@@ -194,6 +194,7 @@ impl<'a, 'tcx> ParseCtxt<'a, 'tcx> {
                         CallSource::OverloadedOperator
                     },
                     fn_span: *fn_span,
+                    arg_move_source_info: None,
                 })
             },
         )
@@ -213,6 +214,7 @@ impl<'a, 'tcx> ParseCtxt<'a, 'tcx> {
                     func: fun,
                     args,
                     fn_span: *fn_span,
+                    arg_move_source_info: None,
                 })
             },
         )

compiler/rustc_mir_build/src/builder/expr/as_rvalue.rs

@@ -166,6 +166,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         unwind: UnwindAction::Continue,
                         call_source: CallSource::Misc,
                         fn_span: expr_span,
+                        arg_move_source_info: None,
                     },
                 );
                 this.diverge_from(block);

compiler/rustc_mir_build/src/builder/expr/into.rs

@@ -408,6 +408,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                             CallSource::OverloadedOperator
                         },
                         fn_span,
+                        arg_move_source_info: None,
                     },
                 );
                 this.diverge_from(block);
@@ -451,6 +452,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                             unwind: UnwindAction::Unreachable,
                             call_source: CallSource::Use,
                             fn_span: expr_span,
+                            arg_move_source_info: None,
                         },
                     );
                     success.unit()

compiler/rustc_mir_build/src/builder/expr/stmt.rs

@@ -135,7 +135,12 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     this.cfg.terminate(
                         block,
                         source_info,
-                        TerminatorKind::TailCall { func: fun, args, fn_span },
+                        TerminatorKind::TailCall {
+                            func: fun,
+                            args,
+                            fn_span,
+                            arg_move_source_info: None,
+                        },
                     );
 
                     this.cfg.start_new_block().unit()

compiler/rustc_mir_build/src/builder/matches/test.rs

@@ -399,6 +399,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 unwind: UnwindAction::Continue,
                 call_source: CallSource::Misc,
                 fn_span: source_info.span,
+                arg_move_source_info: None,
             },
         );
     }
@@ -475,6 +476,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 unwind: UnwindAction::Continue,
                 call_source: CallSource::MatchCmp,
                 fn_span: source_info.span,
+                arg_move_source_info: None,
             },
         );
         self.diverge_from(block);

compiler/rustc_mir_dataflow/src/move_paths/builder.rs

@@ -459,9 +459,7 @@ impl<'a, 'tcx, F: Fn(Ty<'tcx>) -> bool> MoveDataBuilder<'a, 'tcx, F> {
                 ref args,
                 destination,
                 target,
-                unwind: _,
-                call_source: _,
-                fn_span: _,
+                ..
             } => {
                 self.gather_operand(func);
                 for arg in args {