@@ -5,7 +5,6 @@ use rustc_data_structures::fx::FxHashMap;
55use rustc_data_structures:: graph:: DirectedGraph ;
66use rustc_index:: IndexVec ;
77use rustc_index:: bit_set:: BitSet ;
8- use rustc_middle:: bug;
98use rustc_middle:: mir:: coverage:: { CounterId , CovTerm , Expression , ExpressionId , Op } ;
109use tracing:: { debug, debug_span, instrument} ;
1110
@@ -58,13 +57,13 @@ pub(super) struct CoverageCounters {
5857 counter_increment_sites : IndexVec < CounterId , CounterIncrementSite > ,
5958
6059 /// Coverage counters/expressions that are associated with individual BCBs.
61- bcb_counters : IndexVec < BasicCoverageBlock , Option < BcbCounter > > ,
60+ node_counters : IndexVec < BasicCoverageBlock , Option < BcbCounter > > ,
6261 /// Coverage counters/expressions that are associated with the control-flow
6362 /// edge between two BCBs.
6463 ///
6564 /// We currently don't iterate over this map, but if we do in the future,
6665 /// switch it back to `FxIndexMap` to avoid query stability hazards.
67- bcb_edge_counters : FxHashMap < ( BasicCoverageBlock , BasicCoverageBlock ) , BcbCounter > ,
66+ edge_counters : FxHashMap < ( BasicCoverageBlock , BasicCoverageBlock ) , BcbCounter > ,
6867
6968 /// Table of expression data, associating each expression ID with its
7069 /// corresponding operator (+ or -) and its LHS/RHS operands.
@@ -78,20 +77,20 @@ impl CoverageCounters {
7877 /// Ensures that each BCB node needing a counter has one, by creating physical
7978 /// counters or counter expressions for nodes and edges as required.
8079 pub ( super ) fn make_bcb_counters (
81- basic_coverage_blocks : & CoverageGraph ,
80+ graph : & CoverageGraph ,
8281 bcb_needs_counter : & BitSet < BasicCoverageBlock > ,
8382 ) -> Self {
84- let mut counters = MakeBcbCounters :: new ( basic_coverage_blocks , bcb_needs_counter) ;
85- counters . make_bcb_counters ( ) ;
83+ let mut builder = CountersBuilder :: new ( graph , bcb_needs_counter) ;
84+ builder . make_bcb_counters ( ) ;
8685
87- counters . coverage_counters
86+ builder . counters
8887 }
8988
9089 fn with_num_bcbs ( num_bcbs : usize ) -> Self {
9190 Self {
9291 counter_increment_sites : IndexVec :: new ( ) ,
93- bcb_counters : IndexVec :: from_elem_n ( None , num_bcbs) ,
94- bcb_edge_counters : FxHashMap :: default ( ) ,
92+ node_counters : IndexVec :: from_elem_n ( None , num_bcbs) ,
93+ edge_counters : FxHashMap :: default ( ) ,
9594 expressions : IndexVec :: new ( ) ,
9695 expressions_memo : FxHashMap :: default ( ) ,
9796 }
@@ -187,35 +186,31 @@ impl CoverageCounters {
187186 self . counter_increment_sites . len ( )
188187 }
189188
190- fn set_bcb_counter ( & mut self , bcb : BasicCoverageBlock , counter_kind : BcbCounter ) -> BcbCounter {
191- if let Some ( replaced) = self . bcb_counters [ bcb] . replace ( counter_kind) {
192- bug ! (
193- "attempt to set a BasicCoverageBlock coverage counter more than once; \
194- {bcb:?} already had counter {replaced:?}",
195- ) ;
196- } else {
197- counter_kind
198- }
189+ fn set_node_counter ( & mut self , bcb : BasicCoverageBlock , counter : BcbCounter ) -> BcbCounter {
190+ let existing = self . node_counters [ bcb] . replace ( counter) ;
191+ assert ! (
192+ existing. is_none( ) ,
193+ "node {bcb:?} already has a counter: {existing:?} => {counter:?}"
194+ ) ;
195+ counter
199196 }
200197
201- fn set_bcb_edge_counter (
198+ fn set_edge_counter (
202199 & mut self ,
203200 from_bcb : BasicCoverageBlock ,
204201 to_bcb : BasicCoverageBlock ,
205- counter_kind : BcbCounter ,
202+ counter : BcbCounter ,
206203 ) -> BcbCounter {
207- if let Some ( replaced) = self . bcb_edge_counters . insert ( ( from_bcb, to_bcb) , counter_kind) {
208- bug ! (
209- "attempt to set an edge counter more than once; from_bcb: \
210- {from_bcb:?} already had counter {replaced:?}",
211- ) ;
212- } else {
213- counter_kind
214- }
204+ let existing = self . edge_counters . insert ( ( from_bcb, to_bcb) , counter) ;
205+ assert ! (
206+ existing. is_none( ) ,
207+ "edge ({from_bcb:?} -> {to_bcb:?}) already has a counter: {existing:?} => {counter:?}"
208+ ) ;
209+ counter
215210 }
216211
217212 pub ( super ) fn term_for_bcb ( & self , bcb : BasicCoverageBlock ) -> Option < CovTerm > {
218- self . bcb_counters [ bcb] . map ( |counter| counter. as_term ( ) )
213+ self . node_counters [ bcb] . map ( |counter| counter. as_term ( ) )
219214 }
220215
221216 /// Returns an iterator over all the nodes/edges in the coverage graph that
@@ -232,7 +227,7 @@ impl CoverageCounters {
232227 pub ( super ) fn bcb_nodes_with_coverage_expressions (
233228 & self ,
234229 ) -> impl Iterator < Item = ( BasicCoverageBlock , ExpressionId ) > + Captures < ' _ > {
235- self . bcb_counters . iter_enumerated ( ) . filter_map ( |( bcb, & counter_kind ) | match counter_kind {
230+ self . node_counters . iter_enumerated ( ) . filter_map ( |( bcb, & counter ) | match counter {
236231 // Yield the BCB along with its associated expression ID.
237232 Some ( BcbCounter :: Expression { id } ) => Some ( ( bcb, id) ) ,
238233 // This BCB is associated with a counter or nothing, so skip it.
@@ -259,22 +254,20 @@ impl CoverageCounters {
259254 }
260255}
261256
262- /// Helper struct that allows counter creation to inspect the BCB graph.
263- struct MakeBcbCounters < ' a > {
264- coverage_counters : CoverageCounters ,
265- basic_coverage_blocks : & ' a CoverageGraph ,
257+ /// Helper struct that allows counter creation to inspect the BCB graph, and
258+ /// the set of nodes that need counters.
259+ struct CountersBuilder < ' a > {
260+ counters : CoverageCounters ,
261+ graph : & ' a CoverageGraph ,
266262 bcb_needs_counter : & ' a BitSet < BasicCoverageBlock > ,
267263}
268264
269- impl < ' a > MakeBcbCounters < ' a > {
270- fn new (
271- basic_coverage_blocks : & ' a CoverageGraph ,
272- bcb_needs_counter : & ' a BitSet < BasicCoverageBlock > ,
273- ) -> Self {
274- assert_eq ! ( basic_coverage_blocks. num_nodes( ) , bcb_needs_counter. domain_size( ) ) ;
265+ impl < ' a > CountersBuilder < ' a > {
266+ fn new ( graph : & ' a CoverageGraph , bcb_needs_counter : & ' a BitSet < BasicCoverageBlock > ) -> Self {
267+ assert_eq ! ( graph. num_nodes( ) , bcb_needs_counter. domain_size( ) ) ;
275268 Self {
276- coverage_counters : CoverageCounters :: with_num_bcbs ( basic_coverage_blocks . num_nodes ( ) ) ,
277- basic_coverage_blocks ,
269+ counters : CoverageCounters :: with_num_bcbs ( graph . num_nodes ( ) ) ,
270+ graph ,
278271 bcb_needs_counter,
279272 }
280273 }
@@ -289,7 +282,7 @@ impl<'a> MakeBcbCounters<'a> {
289282 // nodes within the loop are visited before visiting any nodes outside
290283 // the loop. It also keeps track of which loop(s) the traversal is
291284 // currently inside.
292- let mut traversal = TraverseCoverageGraphWithLoops :: new ( self . basic_coverage_blocks ) ;
285+ let mut traversal = TraverseCoverageGraphWithLoops :: new ( self . graph ) ;
293286 while let Some ( bcb) = traversal. next ( ) {
294287 let _span = debug_span ! ( "traversal" , ?bcb) . entered ( ) ;
295288 if self . bcb_needs_counter . contains ( bcb) {
@@ -316,26 +309,26 @@ impl<'a> MakeBcbCounters<'a> {
316309 // We might also use that counter to compute one of the out-edge counters.
317310 let node_counter = self . get_or_make_node_counter ( from_bcb) ;
318311
319- let successors = self . basic_coverage_blocks . successors [ from_bcb] . as_slice ( ) ;
312+ let successors = self . graph . successors [ from_bcb] . as_slice ( ) ;
320313
321314 // If this node's out-edges won't sum to the node's counter,
322315 // then there's no reason to create edge counters here.
323- if !self . basic_coverage_blocks [ from_bcb] . is_out_summable {
316+ if !self . graph [ from_bcb] . is_out_summable {
324317 return ;
325318 }
326319
327320 // When choosing which out-edge should be given a counter expression, ignore edges that
328321 // already have counters, or could use the existing counter of their target node.
329322 let out_edge_has_counter = |to_bcb| {
330- if self . coverage_counters . bcb_edge_counters . contains_key ( & ( from_bcb, to_bcb) ) {
323+ if self . counters . edge_counters . contains_key ( & ( from_bcb, to_bcb) ) {
331324 return true ;
332325 }
333- self . basic_coverage_blocks . sole_predecessor ( to_bcb) == Some ( from_bcb)
334- && self . coverage_counters . bcb_counters [ to_bcb] . is_some ( )
326+ self . graph . sole_predecessor ( to_bcb) == Some ( from_bcb)
327+ && self . counters . node_counters [ to_bcb] . is_some ( )
335328 } ;
336329
337330 // Determine the set of out-edges that could benefit from being given an expression.
338- let candidate_successors = self . basic_coverage_blocks . successors [ from_bcb]
331+ let candidate_successors = self . graph . successors [ from_bcb]
339332 . iter ( )
340333 . copied ( )
341334 . filter ( |& to_bcb| !out_edge_has_counter ( to_bcb) )
@@ -344,7 +337,7 @@ impl<'a> MakeBcbCounters<'a> {
344337
345338 // If there are out-edges without counters, choose one to be given an expression
346339 // (computed from this node and the other out-edges) instead of a physical counter.
347- let Some ( expression_to_bcb ) =
340+ let Some ( target_bcb ) =
348341 self . choose_out_edge_for_expression ( traversal, & candidate_successors)
349342 else {
350343 return ;
@@ -357,49 +350,44 @@ impl<'a> MakeBcbCounters<'a> {
357350 . iter ( )
358351 . copied ( )
359352 // Skip the chosen edge, since we'll calculate its count from this sum.
360- . filter ( |& to_bcb| to_bcb != expression_to_bcb )
353+ . filter ( |& edge_target_bcb| edge_target_bcb != target_bcb )
361354 . map ( |to_bcb| self . get_or_make_edge_counter ( from_bcb, to_bcb) )
362355 . collect :: < Vec < _ > > ( ) ;
363- let Some ( sum_of_all_other_out_edges) =
364- self . coverage_counters . make_sum ( & other_out_edge_counters)
356+ let Some ( sum_of_all_other_out_edges) = self . counters . make_sum ( & other_out_edge_counters)
365357 else {
366358 return ;
367359 } ;
368360
369361 // Now create an expression for the chosen edge, by taking the counter
370362 // for its source node and subtracting the sum of its sibling out-edges.
371- let expression = self . coverage_counters . make_expression (
372- node_counter,
373- Op :: Subtract ,
374- sum_of_all_other_out_edges,
375- ) ;
363+ let expression =
364+ self . counters . make_expression ( node_counter, Op :: Subtract , sum_of_all_other_out_edges) ;
376365
377- debug ! ( "{expression_to_bcb :?} gets an expression: {expression:?}" ) ;
378- self . coverage_counters . set_bcb_edge_counter ( from_bcb, expression_to_bcb , expression) ;
366+ debug ! ( "{target_bcb :?} gets an expression: {expression:?}" ) ;
367+ self . counters . set_edge_counter ( from_bcb, target_bcb , expression) ;
379368 }
380369
381370 #[ instrument( level = "debug" , skip( self ) ) ]
382371 fn get_or_make_node_counter ( & mut self , bcb : BasicCoverageBlock ) -> BcbCounter {
383372 // If the BCB already has a counter, return it.
384- if let Some ( counter_kind ) = self . coverage_counters . bcb_counters [ bcb] {
385- debug ! ( "{bcb:?} already has a counter: {counter_kind :?}" ) ;
386- return counter_kind ;
373+ if let Some ( counter ) = self . counters . node_counters [ bcb] {
374+ debug ! ( "{bcb:?} already has a counter: {counter :?}" ) ;
375+ return counter ;
387376 }
388377
389378 let counter = self . make_node_counter_inner ( bcb) ;
390- self . coverage_counters . set_bcb_counter ( bcb, counter)
379+ self . counters . set_node_counter ( bcb, counter)
391380 }
392381
393382 fn make_node_counter_inner ( & mut self , bcb : BasicCoverageBlock ) -> BcbCounter {
394383 // If the node's sole in-edge already has a counter, use that.
395- if let Some ( sole_pred) = self . basic_coverage_blocks . sole_predecessor ( bcb)
396- && let Some ( & edge_counter) =
397- self . coverage_counters . bcb_edge_counters . get ( & ( sole_pred, bcb) )
384+ if let Some ( sole_pred) = self . graph . sole_predecessor ( bcb)
385+ && let Some ( & edge_counter) = self . counters . edge_counters . get ( & ( sole_pred, bcb) )
398386 {
399387 return edge_counter;
400388 }
401389
402- let predecessors = self . basic_coverage_blocks . predecessors [ bcb] . as_slice ( ) ;
390+ let predecessors = self . graph . predecessors [ bcb] . as_slice ( ) ;
403391
404392 // Handle cases where we can't compute a node's count from its in-edges:
405393 // - START_BCB has no in-edges, so taking the sum would panic (or be wrong).
@@ -408,7 +396,7 @@ impl<'a> MakeBcbCounters<'a> {
408396 // leading to infinite recursion.
409397 if predecessors. len ( ) <= 1 || predecessors. contains ( & bcb) {
410398 debug ! ( ?bcb, ?predecessors, "node has <=1 predecessors or is its own predecessor" ) ;
411- let counter = self . coverage_counters . make_phys_node_counter ( bcb) ;
399+ let counter = self . counters . make_phys_node_counter ( bcb) ;
412400 debug ! ( ?bcb, ?counter, "node gets a physical counter" ) ;
413401 return counter;
414402 }
@@ -420,10 +408,8 @@ impl<'a> MakeBcbCounters<'a> {
420408 . copied ( )
421409 . map ( |from_bcb| self . get_or_make_edge_counter ( from_bcb, bcb) )
422410 . collect :: < Vec < _ > > ( ) ;
423- let sum_of_in_edges: BcbCounter = self
424- . coverage_counters
425- . make_sum ( & in_edge_counters)
426- . expect ( "there must be at least one in-edge" ) ;
411+ let sum_of_in_edges: BcbCounter =
412+ self . counters . make_sum ( & in_edge_counters) . expect ( "there must be at least one in-edge" ) ;
427413
428414 debug ! ( "{bcb:?} gets a new counter (sum of predecessor counters): {sum_of_in_edges:?}" ) ;
429415 sum_of_in_edges
@@ -436,15 +422,13 @@ impl<'a> MakeBcbCounters<'a> {
436422 to_bcb : BasicCoverageBlock ,
437423 ) -> BcbCounter {
438424 // If the edge already has a counter, return it.
439- if let Some ( & counter_kind) =
440- self . coverage_counters . bcb_edge_counters . get ( & ( from_bcb, to_bcb) )
441- {
442- debug ! ( "Edge {from_bcb:?}->{to_bcb:?} already has a counter: {counter_kind:?}" ) ;
443- return counter_kind;
425+ if let Some ( & counter) = self . counters . edge_counters . get ( & ( from_bcb, to_bcb) ) {
426+ debug ! ( "Edge {from_bcb:?}->{to_bcb:?} already has a counter: {counter:?}" ) ;
427+ return counter;
444428 }
445429
446430 let counter = self . make_edge_counter_inner ( from_bcb, to_bcb) ;
447- self . coverage_counters . set_bcb_edge_counter ( from_bcb, to_bcb, counter)
431+ self . counters . set_edge_counter ( from_bcb, to_bcb, counter)
448432 }
449433
450434 fn make_edge_counter_inner (
@@ -454,7 +438,7 @@ impl<'a> MakeBcbCounters<'a> {
454438 ) -> BcbCounter {
455439 // If the target node has exactly one in-edge (i.e. this one), then just
456440 // use the node's counter, since it will have the same value.
457- if let Some ( sole_pred) = self . basic_coverage_blocks . sole_predecessor ( to_bcb) {
441+ if let Some ( sole_pred) = self . graph . sole_predecessor ( to_bcb) {
458442 assert_eq ! ( sole_pred, from_bcb) ;
459443 // This call must take care not to invoke `get_or_make_edge` for
460444 // this edge, since that would result in infinite recursion!
@@ -463,13 +447,13 @@ impl<'a> MakeBcbCounters<'a> {
463447
464448 // If the source node has exactly one out-edge (i.e. this one) and would have
465449 // the same execution count as that edge, then just use the node's counter.
466- if let Some ( simple_succ) = self . basic_coverage_blocks . simple_successor ( from_bcb) {
450+ if let Some ( simple_succ) = self . graph . simple_successor ( from_bcb) {
467451 assert_eq ! ( simple_succ, to_bcb) ;
468452 return self . get_or_make_node_counter ( from_bcb) ;
469453 }
470454
471455 // Make a new counter to count this edge.
472- let counter = self . coverage_counters . make_phys_edge_counter ( from_bcb, to_bcb) ;
456+ let counter = self . counters . make_phys_edge_counter ( from_bcb, to_bcb) ;
473457 debug ! ( ?from_bcb, ?to_bcb, ?counter, "edge gets a physical counter" ) ;
474458 counter
475459 }
@@ -516,9 +500,9 @@ impl<'a> MakeBcbCounters<'a> {
516500 for & target_bcb in candidate_successors {
517501 // An edge is a reloop edge if its target dominates any BCB that has
518502 // an edge back to the loop header. (Otherwise it's an exit edge.)
519- let is_reloop_edge = reloop_bcbs. iter ( ) . any ( | & reloop_bcb| {
520- self . basic_coverage_blocks . dominates ( target_bcb , reloop_bcb )
521- } ) ;
503+ let is_reloop_edge = reloop_bcbs
504+ . iter ( )
505+ . any ( | & reloop_bcb| self . graph . dominates ( target_bcb , reloop_bcb ) ) ;
522506 if is_reloop_edge {
523507 // We found a good out-edge to be given an expression.
524508 return Some ( target_bcb) ;
0 commit comments