|
| 1 | +//! Annotation pass for move/copy operations. |
| 2 | +//! |
| 3 | +//! This pass modifies the source scopes of statements containing `Operand::Move` and `Operand::Copy` |
| 4 | +//! to make them appear as if they were inlined from `compiler_move()` and `compiler_copy()` intrinsic |
| 5 | +//! functions. This creates the illusion that moves/copies are function calls in debuggers and |
| 6 | +//! profilers, making them visible for performance analysis. |
| 7 | +//! |
| 8 | +//! The pass leverages the existing inlining infrastructure by creating synthetic `SourceScopeData` |
| 9 | +//! with the `inlined` field set to point to the appropriate intrinsic function. |
| 10 | +
|
| 11 | +use rustc_hir::def_id::DefId; |
| 12 | +use rustc_index::IndexVec; |
| 13 | +use rustc_middle::mir::*; |
| 14 | +use rustc_middle::ty::{self, Instance, Ty, TyCtxt, TypingEnv}; |
| 15 | +use rustc_session::config::DebugInfo; |
| 16 | +use rustc_span::sym; |
| 17 | + |
| 18 | +/// Default minimum size in bytes for move/copy operations to be annotated. Set to 64+1 bytes |
| 19 | +/// (typical cache line size) to focus on potentially expensive operations. |
| 20 | +const DEFAULT_ANNOTATE_MOVES_SIZE_LIMIT: u64 = 65; |
| 21 | + |
| 22 | +/// Bundle up parameters into a structure to make repeated calling neater |
| 23 | +struct Params<'a, 'tcx> { |
| 24 | + tcx: TyCtxt<'tcx>, |
| 25 | + source_scopes: &'a mut IndexVec<SourceScope, SourceScopeData<'tcx>>, |
| 26 | + local_decls: &'a IndexVec<Local, LocalDecl<'tcx>>, |
| 27 | + typing_env: TypingEnv<'tcx>, |
| 28 | + size_limit: u64, |
| 29 | +} |
| 30 | + |
| 31 | +/// MIR transform that annotates move/copy operations for profiler visibility. |
| 32 | +pub(crate) struct AnnotateMoves { |
| 33 | + compiler_copy: Option<DefId>, |
| 34 | + compiler_move: Option<DefId>, |
| 35 | +} |
| 36 | + |
| 37 | +impl<'tcx> crate::MirPass<'tcx> for AnnotateMoves { |
| 38 | + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { |
| 39 | + sess.opts.unstable_opts.annotate_moves.is_enabled() |
| 40 | + && sess.opts.debuginfo != DebugInfo::None |
| 41 | + } |
| 42 | + |
| 43 | + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { |
| 44 | + // Skip promoted MIR bodies to avoid recursion |
| 45 | + if body.source.promoted.is_some() { |
| 46 | + return; |
| 47 | + } |
| 48 | + |
| 49 | + let typing_env = body.typing_env(tcx); |
| 50 | + let size_limit = tcx |
| 51 | + .sess |
| 52 | + .opts |
| 53 | + .unstable_opts |
| 54 | + .annotate_moves |
| 55 | + .size_limit() |
| 56 | + .unwrap_or(DEFAULT_ANNOTATE_MOVES_SIZE_LIMIT); |
| 57 | + |
| 58 | + // Common params, including selectively borrowing the bits of Body we need to avoid |
| 59 | + // mut/non-mut aliasing problems. |
| 60 | + let mut params = Params { |
| 61 | + tcx, |
| 62 | + source_scopes: &mut body.source_scopes, |
| 63 | + local_decls: &body.local_decls, |
| 64 | + typing_env, |
| 65 | + size_limit, |
| 66 | + }; |
| 67 | + |
| 68 | + // Process each basic block |
| 69 | + for block_data in body.basic_blocks.as_mut() { |
| 70 | + for stmt in &mut block_data.statements { |
| 71 | + let source_info = &mut stmt.source_info; |
| 72 | + |
| 73 | + if let StatementKind::Assign(box (_, rvalue)) = &stmt.kind { |
| 74 | + // Save the original scope before processing any operands |
| 75 | + // This prevents chaining when multiple operands are processed |
| 76 | + let original_scope = source_info.scope; |
| 77 | + |
| 78 | + match rvalue { |
| 79 | + Rvalue::Use(op) |
| 80 | + | Rvalue::Repeat(op, _) |
| 81 | + | Rvalue::Cast(_, op, _) |
| 82 | + | Rvalue::UnaryOp(_, op) => { |
| 83 | + self.annotate_move(&mut params, source_info, original_scope, op); |
| 84 | + } |
| 85 | + Rvalue::BinaryOp(_, box (lop, rop)) => { |
| 86 | + self.annotate_move(&mut params, source_info, original_scope, lop); |
| 87 | + self.annotate_move(&mut params, source_info, original_scope, rop); |
| 88 | + } |
| 89 | + Rvalue::Aggregate(_, ops) => { |
| 90 | + for op in ops { |
| 91 | + self.annotate_move(&mut params, source_info, original_scope, op); |
| 92 | + } |
| 93 | + } |
| 94 | + Rvalue::Ref(..) |
| 95 | + | Rvalue::ThreadLocalRef(..) |
| 96 | + | Rvalue::RawPtr(..) |
| 97 | + | Rvalue::NullaryOp(..) |
| 98 | + | Rvalue::Discriminant(..) |
| 99 | + | Rvalue::CopyForDeref(..) |
| 100 | + | Rvalue::ShallowInitBox(..) |
| 101 | + | Rvalue::WrapUnsafeBinder(..) => {} // No operands to instrument |
| 102 | + } |
| 103 | + } |
| 104 | + } |
| 105 | + |
| 106 | + // Process terminator operands |
| 107 | + if let Some(terminator) = &mut block_data.terminator { |
| 108 | + let source_info = &mut terminator.source_info; |
| 109 | + // Save the original scope before processing any operands |
| 110 | + let original_scope = source_info.scope; |
| 111 | + |
| 112 | + match &terminator.kind { |
| 113 | + TerminatorKind::Call { func, args, .. } |
| 114 | + | TerminatorKind::TailCall { func, args, .. } => { |
| 115 | + self.annotate_move(&mut params, source_info, original_scope, func); |
| 116 | + for arg in &*args { |
| 117 | + self.annotate_move(&mut params, source_info, original_scope, &arg.node); |
| 118 | + } |
| 119 | + } |
| 120 | + TerminatorKind::SwitchInt { discr: op, .. } |
| 121 | + | TerminatorKind::Assert { cond: op, .. } |
| 122 | + | TerminatorKind::Yield { value: op, .. } => { |
| 123 | + self.annotate_move(&mut params, source_info, original_scope, op); |
| 124 | + } |
| 125 | + TerminatorKind::InlineAsm { operands, .. } => { |
| 126 | + for op in &**operands { |
| 127 | + match op { |
| 128 | + InlineAsmOperand::In { value, .. } |
| 129 | + | InlineAsmOperand::InOut { in_value: value, .. } => { |
| 130 | + self.annotate_move( |
| 131 | + &mut params, |
| 132 | + source_info, |
| 133 | + original_scope, |
| 134 | + value, |
| 135 | + ); |
| 136 | + } |
| 137 | + // Const, SymFn, SymStatic, Out, and Label don't have Operands we care about |
| 138 | + _ => {} |
| 139 | + } |
| 140 | + } |
| 141 | + } |
| 142 | + _ => {} // Other terminators don't have operands |
| 143 | + } |
| 144 | + } |
| 145 | + } |
| 146 | + } |
| 147 | + |
| 148 | + fn is_required(&self) -> bool { |
| 149 | + false // Optional optimization/instrumentation pass |
| 150 | + } |
| 151 | +} |
| 152 | + |
| 153 | +impl AnnotateMoves { |
| 154 | + pub(crate) fn new<'tcx>(tcx: TyCtxt<'tcx>) -> Self { |
| 155 | + let compiler_copy = tcx.get_diagnostic_item(sym::compiler_copy); |
| 156 | + let compiler_move = tcx.get_diagnostic_item(sym::compiler_move); |
| 157 | + |
| 158 | + Self { compiler_copy, compiler_move } |
| 159 | + } |
| 160 | + |
| 161 | + /// If this is a Move or Copy of a concrete type, update its debug info to make it look like it |
| 162 | + /// was inlined from `core::profiling::compiler_move`/`compiler_copy`. |
| 163 | + /// |
| 164 | + /// Takes an explicit `original_scope` to use as the parent scope, which prevents chaining |
| 165 | + /// when multiple operands in the same statement are processed. |
| 166 | + /// |
| 167 | + /// The statement's span is NOT modified, so profilers will show the move at its actual |
| 168 | + /// source location rather than at profiling.rs. This provides more useful context about |
| 169 | + /// where the move occurs in the user's code. |
| 170 | + fn annotate_move<'tcx>( |
| 171 | + &self, |
| 172 | + params: &mut Params<'_, 'tcx>, |
| 173 | + source_info: &mut SourceInfo, |
| 174 | + original_scope: SourceScope, |
| 175 | + op: &Operand<'tcx>, |
| 176 | + ) { |
| 177 | + let (place, Some(profiling_marker)) = (match op { |
| 178 | + Operand::Move(place) => (place, self.compiler_move), |
| 179 | + Operand::Copy(place) => (place, self.compiler_copy), |
| 180 | + _ => return, |
| 181 | + }) else { |
| 182 | + return; |
| 183 | + }; |
| 184 | + let Params { tcx, typing_env, local_decls, size_limit, source_scopes } = params; |
| 185 | + |
| 186 | + if let Some(type_size) = |
| 187 | + self.should_annotate_operation(*tcx, *typing_env, local_decls, place, *size_limit) |
| 188 | + { |
| 189 | + let ty = place.ty(*local_decls, *tcx).ty; |
| 190 | + let callsite_span = source_info.span; |
| 191 | + let new_scope = self.create_inlined_scope( |
| 192 | + *tcx, |
| 193 | + *typing_env, |
| 194 | + source_scopes, |
| 195 | + original_scope, |
| 196 | + callsite_span, |
| 197 | + profiling_marker, |
| 198 | + ty, |
| 199 | + type_size, |
| 200 | + ); |
| 201 | + source_info.scope = new_scope; |
| 202 | + // Note: We deliberately do NOT modify source_info.span. |
| 203 | + // Keeping the original span means profilers show the actual source location |
| 204 | + // of the move/copy, which is more useful than showing profiling.rs:13. |
| 205 | + // The scope change is sufficient to make the move appear as an inlined call |
| 206 | + // to compiler_move/copy in the profiler. |
| 207 | + } |
| 208 | + } |
| 209 | + |
| 210 | + /// Determines if an operation should be annotated based on type characteristics. |
| 211 | + /// Returns Some(size) if it should be annotated, None otherwise. |
| 212 | + fn should_annotate_operation<'tcx>( |
| 213 | + &self, |
| 214 | + tcx: TyCtxt<'tcx>, |
| 215 | + typing_env: ty::TypingEnv<'tcx>, |
| 216 | + local_decls: &rustc_index::IndexVec<Local, LocalDecl<'tcx>>, |
| 217 | + place: &Place<'tcx>, |
| 218 | + size_limit: u64, |
| 219 | + ) -> Option<u64> { |
| 220 | + let ty = place.ty(local_decls, tcx).ty; |
| 221 | + let layout = match tcx.layout_of(typing_env.as_query_input(ty)) { |
| 222 | + Ok(layout) => layout, |
| 223 | + Err(err) => { |
| 224 | + tracing::info!("Failed to get layout of {ty:?}: {err}"); |
| 225 | + return None; |
| 226 | + } |
| 227 | + }; |
| 228 | + |
| 229 | + let size = layout.size.bytes(); |
| 230 | + |
| 231 | + // 1. Skip ZST types (no actual move/copy happens) |
| 232 | + if layout.is_zst() { |
| 233 | + return None; |
| 234 | + } |
| 235 | + |
| 236 | + // 2. Check size threshold (only annotate large moves/copies) |
| 237 | + if size < size_limit { |
| 238 | + return None; |
| 239 | + } |
| 240 | + |
| 241 | + // 3. Skip scalar/vector types that won't generate memcpy |
| 242 | + match layout.layout.backend_repr { |
| 243 | + rustc_abi::BackendRepr::Scalar(_) |
| 244 | + | rustc_abi::BackendRepr::ScalarPair(_, _) |
| 245 | + | rustc_abi::BackendRepr::SimdVector { .. } => None, |
| 246 | + _ => Some(size), |
| 247 | + } |
| 248 | + } |
| 249 | + |
| 250 | + /// Creates an inlined scope that makes operations appear to come from |
| 251 | + /// the specified compiler intrinsic function. |
| 252 | + fn create_inlined_scope<'tcx>( |
| 253 | + &self, |
| 254 | + tcx: TyCtxt<'tcx>, |
| 255 | + typing_env: TypingEnv<'tcx>, |
| 256 | + source_scopes: &mut IndexVec<SourceScope, SourceScopeData<'tcx>>, |
| 257 | + original_scope: SourceScope, |
| 258 | + callsite_span: rustc_span::Span, |
| 259 | + profiling_def_id: DefId, |
| 260 | + ty: Ty<'tcx>, |
| 261 | + type_size: u64, |
| 262 | + ) -> SourceScope { |
| 263 | + // Monomorphize the profiling marker for the actual type being moved/copied + size const |
| 264 | + // parameter compiler_move<T, const SIZE: usize> or compiler_copy<T, const SIZE: usize> |
| 265 | + let size_const = ty::Const::from_target_usize(tcx, type_size); |
| 266 | + let generic_args = tcx.mk_args(&[ty.into(), size_const.into()]); |
| 267 | + let profiling_instance = Instance::expect_resolve( |
| 268 | + tcx, |
| 269 | + typing_env, |
| 270 | + profiling_def_id, |
| 271 | + generic_args, |
| 272 | + callsite_span, |
| 273 | + ); |
| 274 | + |
| 275 | + // Get the profiling marker's definition span to use as the scope's span |
| 276 | + // This ensures the file_start_pos/file_end_pos in the DebugScope match the DIScope's file |
| 277 | + let profiling_span = tcx.def_span(profiling_def_id); |
| 278 | + |
| 279 | + // Create new inlined scope that makes the operation appear to come from the profiling |
| 280 | + // marker |
| 281 | + let inlined_scope_data = SourceScopeData { |
| 282 | + // Use profiling_span so file bounds match the DIScope (profiling.rs) |
| 283 | + // This prevents DILexicalBlockFile mismatches that would show profiling.rs |
| 284 | + // with incorrect line numbers |
| 285 | + span: profiling_span, |
| 286 | + parent_scope: Some(original_scope), |
| 287 | + |
| 288 | + // The inlined field shows: (what was inlined, where it was called from) |
| 289 | + // - profiling_instance: the compiler_move/copy function that was "inlined" |
| 290 | + // - callsite_span: where the move/copy actually occurs in the user's code |
| 291 | + inlined: Some((profiling_instance, callsite_span)), |
| 292 | + |
| 293 | + // Proper inlined scope chaining to maintain debug info hierarchy |
| 294 | + // We need to find the first non-compiler_move inlined scope in the chain |
| 295 | + inlined_parent_scope: { |
| 296 | + let mut scope = original_scope; |
| 297 | + loop { |
| 298 | + let scope_data = &source_scopes[scope]; |
| 299 | + if let Some((instance, _)) = scope_data.inlined { |
| 300 | + // Check if this is a compiler_move/copy scope we created |
| 301 | + if let Some(def_id) = instance.def_id().as_local() { |
| 302 | + let is_compiler_move = tcx.get_diagnostic_item(sym::compiler_move) |
| 303 | + == Some(def_id.to_def_id()); |
| 304 | + let is_compiler_copy = tcx.get_diagnostic_item(sym::compiler_copy) |
| 305 | + == Some(def_id.to_def_id()); |
| 306 | + |
| 307 | + if is_compiler_move || is_compiler_copy { |
| 308 | + // This is one of our scopes, skip it and look at its inlined_parent_scope |
| 309 | + if let Some(parent) = scope_data.inlined_parent_scope { |
| 310 | + scope = parent; |
| 311 | + continue; |
| 312 | + } else { |
| 313 | + // No more parents, this is fine |
| 314 | + break None; |
| 315 | + } |
| 316 | + } |
| 317 | + } |
| 318 | + // This is a real inlined scope (not compiler_move/copy), use it |
| 319 | + break Some(scope); |
| 320 | + } else { |
| 321 | + // Not an inlined scope, use its inlined_parent_scope |
| 322 | + break scope_data.inlined_parent_scope; |
| 323 | + } |
| 324 | + } |
| 325 | + }, |
| 326 | + |
| 327 | + local_data: ClearCrossCrate::Clear, |
| 328 | + }; |
| 329 | + |
| 330 | + // Add the new scope and return it |
| 331 | + source_scopes.push(inlined_scope_data) |
| 332 | + } |
| 333 | +} |
0 commit comments