@@ -5,7 +5,6 @@ use rustc_data_structures::fx::FxHashMap;
55use rustc_data_structures:: graph:: DirectedGraph ;
66use rustc_index:: IndexVec ;
77use rustc_index:: bit_set:: BitSet ;
8- use rustc_middle:: bug;
98use rustc_middle:: mir:: coverage:: { CounterId , CovTerm , Expression , ExpressionId , Op } ;
109use tracing:: { debug, debug_span, instrument} ;
1110
@@ -58,13 +57,13 @@ pub(super) struct CoverageCounters {
5857 counter_increment_sites : IndexVec < CounterId , CounterIncrementSite > ,
5958
6059 /// Coverage counters/expressions that are associated with individual BCBs.
61- bcb_counters : IndexVec < BasicCoverageBlock , Option < BcbCounter > > ,
60+ node_counters : IndexVec < BasicCoverageBlock , Option < BcbCounter > > ,
6261 /// Coverage counters/expressions that are associated with the control-flow
6362 /// edge between two BCBs.
6463 ///
6564 /// We currently don't iterate over this map, but if we do in the future,
6665 /// switch it back to `FxIndexMap` to avoid query stability hazards.
67- bcb_edge_counters : FxHashMap < ( BasicCoverageBlock , BasicCoverageBlock ) , BcbCounter > ,
66+ edge_counters : FxHashMap < ( BasicCoverageBlock , BasicCoverageBlock ) , BcbCounter > ,
6867
6968 /// Table of expression data, associating each expression ID with its
7069 /// corresponding operator (+ or -) and its LHS/RHS operands.
@@ -78,20 +77,20 @@ impl CoverageCounters {
7877 /// Ensures that each BCB node needing a counter has one, by creating physical
7978 /// counters or counter expressions for nodes and edges as required.
8079 pub ( super ) fn make_bcb_counters (
81- basic_coverage_blocks : & CoverageGraph ,
80+ graph : & CoverageGraph ,
8281 bcb_needs_counter : & BitSet < BasicCoverageBlock > ,
8382 ) -> Self {
84- let mut counters = MakeBcbCounters :: new ( basic_coverage_blocks , bcb_needs_counter) ;
85- counters . make_bcb_counters ( ) ;
83+ let mut builder = CountersBuilder :: new ( graph , bcb_needs_counter) ;
84+ builder . make_bcb_counters ( ) ;
8685
87- counters . coverage_counters
86+ builder . counters
8887 }
8988
9089 fn with_num_bcbs ( num_bcbs : usize ) -> Self {
9190 Self {
9291 counter_increment_sites : IndexVec :: new ( ) ,
93- bcb_counters : IndexVec :: from_elem_n ( None , num_bcbs) ,
94- bcb_edge_counters : FxHashMap :: default ( ) ,
92+ node_counters : IndexVec :: from_elem_n ( None , num_bcbs) ,
93+ edge_counters : FxHashMap :: default ( ) ,
9594 expressions : IndexVec :: new ( ) ,
9695 expressions_memo : FxHashMap :: default ( ) ,
9796 }
@@ -189,35 +188,31 @@ impl CoverageCounters {
189188 self . counter_increment_sites . len ( )
190189 }
191190
192- fn set_bcb_counter ( & mut self , bcb : BasicCoverageBlock , counter_kind : BcbCounter ) -> BcbCounter {
193- if let Some ( replaced) = self . bcb_counters [ bcb] . replace ( counter_kind) {
194- bug ! (
195- "attempt to set a BasicCoverageBlock coverage counter more than once; \
196- {bcb:?} already had counter {replaced:?}",
197- ) ;
198- } else {
199- counter_kind
200- }
191+ fn set_node_counter ( & mut self , bcb : BasicCoverageBlock , counter : BcbCounter ) -> BcbCounter {
192+ let existing = self . node_counters [ bcb] . replace ( counter) ;
193+ assert ! (
194+ existing. is_none( ) ,
195+ "node {bcb:?} already has a counter: {existing:?} => {counter:?}"
196+ ) ;
197+ counter
201198 }
202199
203- fn set_bcb_edge_counter (
200+ fn set_edge_counter (
204201 & mut self ,
205202 from_bcb : BasicCoverageBlock ,
206203 to_bcb : BasicCoverageBlock ,
207- counter_kind : BcbCounter ,
204+ counter : BcbCounter ,
208205 ) -> BcbCounter {
209- if let Some ( replaced) = self . bcb_edge_counters . insert ( ( from_bcb, to_bcb) , counter_kind) {
210- bug ! (
211- "attempt to set an edge counter more than once; from_bcb: \
212- {from_bcb:?} already had counter {replaced:?}",
213- ) ;
214- } else {
215- counter_kind
216- }
206+ let existing = self . edge_counters . insert ( ( from_bcb, to_bcb) , counter) ;
207+ assert ! (
208+ existing. is_none( ) ,
209+ "edge ({from_bcb:?} -> {to_bcb:?}) already has a counter: {existing:?} => {counter:?}"
210+ ) ;
211+ counter
217212 }
218213
219214 pub ( super ) fn term_for_bcb ( & self , bcb : BasicCoverageBlock ) -> Option < CovTerm > {
220- self . bcb_counters [ bcb] . map ( |counter| counter. as_term ( ) )
215+ self . node_counters [ bcb] . map ( |counter| counter. as_term ( ) )
221216 }
222217
223218 /// Returns an iterator over all the nodes/edges in the coverage graph that
@@ -234,7 +229,7 @@ impl CoverageCounters {
234229 pub ( super ) fn bcb_nodes_with_coverage_expressions (
235230 & self ,
236231 ) -> impl Iterator < Item = ( BasicCoverageBlock , ExpressionId ) > + Captures < ' _ > {
237- self . bcb_counters . iter_enumerated ( ) . filter_map ( |( bcb, & counter_kind ) | match counter_kind {
232+ self . node_counters . iter_enumerated ( ) . filter_map ( |( bcb, & counter ) | match counter {
238233 // Yield the BCB along with its associated expression ID.
239234 Some ( BcbCounter :: Expression { id } ) => Some ( ( bcb, id) ) ,
240235 // This BCB is associated with a counter or nothing, so skip it.
@@ -261,22 +256,20 @@ impl CoverageCounters {
261256 }
262257}
263258
264- /// Helper struct that allows counter creation to inspect the BCB graph.
265- struct MakeBcbCounters < ' a > {
266- coverage_counters : CoverageCounters ,
267- basic_coverage_blocks : & ' a CoverageGraph ,
259+ /// Helper struct that allows counter creation to inspect the BCB graph, and
260+ /// the set of nodes that need counters.
261+ struct CountersBuilder < ' a > {
262+ counters : CoverageCounters ,
263+ graph : & ' a CoverageGraph ,
268264 bcb_needs_counter : & ' a BitSet < BasicCoverageBlock > ,
269265}
270266
271- impl < ' a > MakeBcbCounters < ' a > {
272- fn new (
273- basic_coverage_blocks : & ' a CoverageGraph ,
274- bcb_needs_counter : & ' a BitSet < BasicCoverageBlock > ,
275- ) -> Self {
276- assert_eq ! ( basic_coverage_blocks. num_nodes( ) , bcb_needs_counter. domain_size( ) ) ;
267+ impl < ' a > CountersBuilder < ' a > {
268+ fn new ( graph : & ' a CoverageGraph , bcb_needs_counter : & ' a BitSet < BasicCoverageBlock > ) -> Self {
269+ assert_eq ! ( graph. num_nodes( ) , bcb_needs_counter. domain_size( ) ) ;
277270 Self {
278- coverage_counters : CoverageCounters :: with_num_bcbs ( basic_coverage_blocks . num_nodes ( ) ) ,
279- basic_coverage_blocks ,
271+ counters : CoverageCounters :: with_num_bcbs ( graph . num_nodes ( ) ) ,
272+ graph ,
280273 bcb_needs_counter,
281274 }
282275 }
@@ -291,7 +284,7 @@ impl<'a> MakeBcbCounters<'a> {
291284 // nodes within the loop are visited before visiting any nodes outside
292285 // the loop. It also keeps track of which loop(s) the traversal is
293286 // currently inside.
294- let mut traversal = TraverseCoverageGraphWithLoops :: new ( self . basic_coverage_blocks ) ;
287+ let mut traversal = TraverseCoverageGraphWithLoops :: new ( self . graph ) ;
295288 while let Some ( bcb) = traversal. next ( ) {
296289 let _span = debug_span ! ( "traversal" , ?bcb) . entered ( ) ;
297290 if self . bcb_needs_counter . contains ( bcb) {
@@ -318,26 +311,26 @@ impl<'a> MakeBcbCounters<'a> {
318311 // We might also use that counter to compute one of the out-edge counters.
319312 let node_counter = self . get_or_make_node_counter ( from_bcb) ;
320313
321- let successors = self . basic_coverage_blocks . successors [ from_bcb] . as_slice ( ) ;
314+ let successors = self . graph . successors [ from_bcb] . as_slice ( ) ;
322315
323316 // If this node's out-edges won't sum to the node's counter,
324317 // then there's no reason to create edge counters here.
325- if !self . basic_coverage_blocks [ from_bcb] . is_out_summable {
318+ if !self . graph [ from_bcb] . is_out_summable {
326319 return ;
327320 }
328321
329322 // When choosing which out-edge should be given a counter expression, ignore edges that
330323 // already have counters, or could use the existing counter of their target node.
331324 let out_edge_has_counter = |to_bcb| {
332- if self . coverage_counters . bcb_edge_counters . contains_key ( & ( from_bcb, to_bcb) ) {
325+ if self . counters . edge_counters . contains_key ( & ( from_bcb, to_bcb) ) {
333326 return true ;
334327 }
335- self . basic_coverage_blocks . sole_predecessor ( to_bcb) == Some ( from_bcb)
336- && self . coverage_counters . bcb_counters [ to_bcb] . is_some ( )
328+ self . graph . sole_predecessor ( to_bcb) == Some ( from_bcb)
329+ && self . counters . node_counters [ to_bcb] . is_some ( )
337330 } ;
338331
339332 // Determine the set of out-edges that could benefit from being given an expression.
340- let candidate_successors = self . basic_coverage_blocks . successors [ from_bcb]
333+ let candidate_successors = self . graph . successors [ from_bcb]
341334 . iter ( )
342335 . copied ( )
343336 . filter ( |& to_bcb| !out_edge_has_counter ( to_bcb) )
@@ -346,7 +339,7 @@ impl<'a> MakeBcbCounters<'a> {
346339
347340 // If there are out-edges without counters, choose one to be given an expression
348341 // (computed from this node and the other out-edges) instead of a physical counter.
349- let Some ( expression_to_bcb ) =
342+ let Some ( target_bcb ) =
350343 self . choose_out_edge_for_expression ( traversal, & candidate_successors)
351344 else {
352345 return ;
@@ -359,49 +352,44 @@ impl<'a> MakeBcbCounters<'a> {
359352 . iter ( )
360353 . copied ( )
361354 // Skip the chosen edge, since we'll calculate its count from this sum.
362- . filter ( |& to_bcb| to_bcb != expression_to_bcb )
355+ . filter ( |& edge_target_bcb| edge_target_bcb != target_bcb )
363356 . map ( |to_bcb| self . get_or_make_edge_counter ( from_bcb, to_bcb) )
364357 . collect :: < Vec < _ > > ( ) ;
365- let Some ( sum_of_all_other_out_edges) =
366- self . coverage_counters . make_sum ( & other_out_edge_counters)
358+ let Some ( sum_of_all_other_out_edges) = self . counters . make_sum ( & other_out_edge_counters)
367359 else {
368360 return ;
369361 } ;
370362
371363 // Now create an expression for the chosen edge, by taking the counter
372364 // for its source node and subtracting the sum of its sibling out-edges.
373- let expression = self . coverage_counters . make_expression (
374- node_counter,
375- Op :: Subtract ,
376- sum_of_all_other_out_edges,
377- ) ;
365+ let expression =
366+ self . counters . make_expression ( node_counter, Op :: Subtract , sum_of_all_other_out_edges) ;
378367
379- debug ! ( "{expression_to_bcb :?} gets an expression: {expression:?}" ) ;
380- self . coverage_counters . set_bcb_edge_counter ( from_bcb, expression_to_bcb , expression) ;
368+ debug ! ( "{target_bcb :?} gets an expression: {expression:?}" ) ;
369+ self . counters . set_edge_counter ( from_bcb, target_bcb , expression) ;
381370 }
382371
383372 #[ instrument( level = "debug" , skip( self ) ) ]
384373 fn get_or_make_node_counter ( & mut self , bcb : BasicCoverageBlock ) -> BcbCounter {
385374 // If the BCB already has a counter, return it.
386- if let Some ( counter_kind ) = self . coverage_counters . bcb_counters [ bcb] {
387- debug ! ( "{bcb:?} already has a counter: {counter_kind :?}" ) ;
388- return counter_kind ;
375+ if let Some ( counter ) = self . counters . node_counters [ bcb] {
376+ debug ! ( "{bcb:?} already has a counter: {counter :?}" ) ;
377+ return counter ;
389378 }
390379
391380 let counter = self . make_node_counter_inner ( bcb) ;
392- self . coverage_counters . set_bcb_counter ( bcb, counter)
381+ self . counters . set_node_counter ( bcb, counter)
393382 }
394383
395384 fn make_node_counter_inner ( & mut self , bcb : BasicCoverageBlock ) -> BcbCounter {
396385 // If the node's sole in-edge already has a counter, use that.
397- if let Some ( sole_pred) = self . basic_coverage_blocks . sole_predecessor ( bcb)
398- && let Some ( & edge_counter) =
399- self . coverage_counters . bcb_edge_counters . get ( & ( sole_pred, bcb) )
386+ if let Some ( sole_pred) = self . graph . sole_predecessor ( bcb)
387+ && let Some ( & edge_counter) = self . counters . edge_counters . get ( & ( sole_pred, bcb) )
400388 {
401389 return edge_counter;
402390 }
403391
404- let predecessors = self . basic_coverage_blocks . predecessors [ bcb] . as_slice ( ) ;
392+ let predecessors = self . graph . predecessors [ bcb] . as_slice ( ) ;
405393
406394 // Handle cases where we can't compute a node's count from its in-edges:
407395 // - START_BCB has no in-edges, so taking the sum would panic (or be wrong).
@@ -410,7 +398,7 @@ impl<'a> MakeBcbCounters<'a> {
410398 // leading to infinite recursion.
411399 if predecessors. len ( ) <= 1 || predecessors. contains ( & bcb) {
412400 debug ! ( ?bcb, ?predecessors, "node has <=1 predecessors or is its own predecessor" ) ;
413- let counter = self . coverage_counters . make_phys_node_counter ( bcb) ;
401+ let counter = self . counters . make_phys_node_counter ( bcb) ;
414402 debug ! ( ?bcb, ?counter, "node gets a physical counter" ) ;
415403 return counter;
416404 }
@@ -422,10 +410,8 @@ impl<'a> MakeBcbCounters<'a> {
422410 . copied ( )
423411 . map ( |from_bcb| self . get_or_make_edge_counter ( from_bcb, bcb) )
424412 . collect :: < Vec < _ > > ( ) ;
425- let sum_of_in_edges: BcbCounter = self
426- . coverage_counters
427- . make_sum ( & in_edge_counters)
428- . expect ( "there must be at least one in-edge" ) ;
413+ let sum_of_in_edges: BcbCounter =
414+ self . counters . make_sum ( & in_edge_counters) . expect ( "there must be at least one in-edge" ) ;
429415
430416 debug ! ( "{bcb:?} gets a new counter (sum of predecessor counters): {sum_of_in_edges:?}" ) ;
431417 sum_of_in_edges
@@ -438,15 +424,13 @@ impl<'a> MakeBcbCounters<'a> {
438424 to_bcb : BasicCoverageBlock ,
439425 ) -> BcbCounter {
440426 // If the edge already has a counter, return it.
441- if let Some ( & counter_kind) =
442- self . coverage_counters . bcb_edge_counters . get ( & ( from_bcb, to_bcb) )
443- {
444- debug ! ( "Edge {from_bcb:?}->{to_bcb:?} already has a counter: {counter_kind:?}" ) ;
445- return counter_kind;
427+ if let Some ( & counter) = self . counters . edge_counters . get ( & ( from_bcb, to_bcb) ) {
428+ debug ! ( "Edge {from_bcb:?}->{to_bcb:?} already has a counter: {counter:?}" ) ;
429+ return counter;
446430 }
447431
448432 let counter = self . make_edge_counter_inner ( from_bcb, to_bcb) ;
449- self . coverage_counters . set_bcb_edge_counter ( from_bcb, to_bcb, counter)
433+ self . counters . set_edge_counter ( from_bcb, to_bcb, counter)
450434 }
451435
452436 fn make_edge_counter_inner (
@@ -456,7 +440,7 @@ impl<'a> MakeBcbCounters<'a> {
456440 ) -> BcbCounter {
457441 // If the target node has exactly one in-edge (i.e. this one), then just
458442 // use the node's counter, since it will have the same value.
459- if let Some ( sole_pred) = self . basic_coverage_blocks . sole_predecessor ( to_bcb) {
443+ if let Some ( sole_pred) = self . graph . sole_predecessor ( to_bcb) {
460444 assert_eq ! ( sole_pred, from_bcb) ;
461445 // This call must take care not to invoke `get_or_make_edge` for
462446 // this edge, since that would result in infinite recursion!
@@ -465,13 +449,13 @@ impl<'a> MakeBcbCounters<'a> {
465449
466450 // If the source node has exactly one out-edge (i.e. this one) and would have
467451 // the same execution count as that edge, then just use the node's counter.
468- if let Some ( simple_succ) = self . basic_coverage_blocks . simple_successor ( from_bcb) {
452+ if let Some ( simple_succ) = self . graph . simple_successor ( from_bcb) {
469453 assert_eq ! ( simple_succ, to_bcb) ;
470454 return self . get_or_make_node_counter ( from_bcb) ;
471455 }
472456
473457 // Make a new counter to count this edge.
474- let counter = self . coverage_counters . make_phys_edge_counter ( from_bcb, to_bcb) ;
458+ let counter = self . counters . make_phys_edge_counter ( from_bcb, to_bcb) ;
475459 debug ! ( ?from_bcb, ?to_bcb, ?counter, "edge gets a physical counter" ) ;
476460 counter
477461 }
@@ -518,9 +502,9 @@ impl<'a> MakeBcbCounters<'a> {
518502 for & target_bcb in candidate_successors {
519503 // An edge is a reloop edge if its target dominates any BCB that has
520504 // an edge back to the loop header. (Otherwise it's an exit edge.)
521- let is_reloop_edge = reloop_bcbs. iter ( ) . any ( | & reloop_bcb| {
522- self . basic_coverage_blocks . dominates ( target_bcb , reloop_bcb )
523- } ) ;
505+ let is_reloop_edge = reloop_bcbs
506+ . iter ( )
507+ . any ( | & reloop_bcb| self . graph . dominates ( target_bcb , reloop_bcb ) ) ;
524508 if is_reloop_edge {
525509 // We found a good out-edge to be given an expression.
526510 return Some ( target_bcb) ;
0 commit comments