@@ -5,7 +5,6 @@ use rustc_data_structures::fx::FxHashMap;
5
5
use rustc_data_structures:: graph:: DirectedGraph ;
6
6
use rustc_index:: IndexVec ;
7
7
use rustc_index:: bit_set:: BitSet ;
8
- use rustc_middle:: bug;
9
8
use rustc_middle:: mir:: coverage:: { CounterId , CovTerm , Expression , ExpressionId , Op } ;
10
9
use tracing:: { debug, debug_span, instrument} ;
11
10
@@ -58,13 +57,13 @@ pub(super) struct CoverageCounters {
58
57
counter_increment_sites : IndexVec < CounterId , CounterIncrementSite > ,
59
58
60
59
/// Coverage counters/expressions that are associated with individual BCBs.
61
- bcb_counters : IndexVec < BasicCoverageBlock , Option < BcbCounter > > ,
60
+ node_counters : IndexVec < BasicCoverageBlock , Option < BcbCounter > > ,
62
61
/// Coverage counters/expressions that are associated with the control-flow
63
62
/// edge between two BCBs.
64
63
///
65
64
/// We currently don't iterate over this map, but if we do in the future,
66
65
/// switch it back to `FxIndexMap` to avoid query stability hazards.
67
- bcb_edge_counters : FxHashMap < ( BasicCoverageBlock , BasicCoverageBlock ) , BcbCounter > ,
66
+ edge_counters : FxHashMap < ( BasicCoverageBlock , BasicCoverageBlock ) , BcbCounter > ,
68
67
69
68
/// Table of expression data, associating each expression ID with its
70
69
/// corresponding operator (+ or -) and its LHS/RHS operands.
@@ -78,20 +77,20 @@ impl CoverageCounters {
78
77
/// Ensures that each BCB node needing a counter has one, by creating physical
79
78
/// counters or counter expressions for nodes and edges as required.
80
79
pub ( super ) fn make_bcb_counters (
81
- basic_coverage_blocks : & CoverageGraph ,
80
+ graph : & CoverageGraph ,
82
81
bcb_needs_counter : & BitSet < BasicCoverageBlock > ,
83
82
) -> Self {
84
- let mut counters = MakeBcbCounters :: new ( basic_coverage_blocks , bcb_needs_counter) ;
85
- counters . make_bcb_counters ( ) ;
83
+ let mut builder = CountersBuilder :: new ( graph , bcb_needs_counter) ;
84
+ builder . make_bcb_counters ( ) ;
86
85
87
- counters . coverage_counters
86
+ builder . counters
88
87
}
89
88
90
89
fn with_num_bcbs ( num_bcbs : usize ) -> Self {
91
90
Self {
92
91
counter_increment_sites : IndexVec :: new ( ) ,
93
- bcb_counters : IndexVec :: from_elem_n ( None , num_bcbs) ,
94
- bcb_edge_counters : FxHashMap :: default ( ) ,
92
+ node_counters : IndexVec :: from_elem_n ( None , num_bcbs) ,
93
+ edge_counters : FxHashMap :: default ( ) ,
95
94
expressions : IndexVec :: new ( ) ,
96
95
expressions_memo : FxHashMap :: default ( ) ,
97
96
}
@@ -189,35 +188,31 @@ impl CoverageCounters {
189
188
self . counter_increment_sites . len ( )
190
189
}
191
190
192
- fn set_bcb_counter ( & mut self , bcb : BasicCoverageBlock , counter_kind : BcbCounter ) -> BcbCounter {
193
- if let Some ( replaced) = self . bcb_counters [ bcb] . replace ( counter_kind) {
194
- bug ! (
195
- "attempt to set a BasicCoverageBlock coverage counter more than once; \
196
- {bcb:?} already had counter {replaced:?}",
197
- ) ;
198
- } else {
199
- counter_kind
200
- }
191
+ fn set_node_counter ( & mut self , bcb : BasicCoverageBlock , counter : BcbCounter ) -> BcbCounter {
192
+ let existing = self . node_counters [ bcb] . replace ( counter) ;
193
+ assert ! (
194
+ existing. is_none( ) ,
195
+ "node {bcb:?} already has a counter: {existing:?} => {counter:?}"
196
+ ) ;
197
+ counter
201
198
}
202
199
203
- fn set_bcb_edge_counter (
200
+ fn set_edge_counter (
204
201
& mut self ,
205
202
from_bcb : BasicCoverageBlock ,
206
203
to_bcb : BasicCoverageBlock ,
207
- counter_kind : BcbCounter ,
204
+ counter : BcbCounter ,
208
205
) -> BcbCounter {
209
- if let Some ( replaced) = self . bcb_edge_counters . insert ( ( from_bcb, to_bcb) , counter_kind) {
210
- bug ! (
211
- "attempt to set an edge counter more than once; from_bcb: \
212
- {from_bcb:?} already had counter {replaced:?}",
213
- ) ;
214
- } else {
215
- counter_kind
216
- }
206
+ let existing = self . edge_counters . insert ( ( from_bcb, to_bcb) , counter) ;
207
+ assert ! (
208
+ existing. is_none( ) ,
209
+ "edge ({from_bcb:?} -> {to_bcb:?}) already has a counter: {existing:?} => {counter:?}"
210
+ ) ;
211
+ counter
217
212
}
218
213
219
214
pub ( super ) fn term_for_bcb ( & self , bcb : BasicCoverageBlock ) -> Option < CovTerm > {
220
- self . bcb_counters [ bcb] . map ( |counter| counter. as_term ( ) )
215
+ self . node_counters [ bcb] . map ( |counter| counter. as_term ( ) )
221
216
}
222
217
223
218
/// Returns an iterator over all the nodes/edges in the coverage graph that
@@ -234,7 +229,7 @@ impl CoverageCounters {
234
229
pub ( super ) fn bcb_nodes_with_coverage_expressions (
235
230
& self ,
236
231
) -> impl Iterator < Item = ( BasicCoverageBlock , ExpressionId ) > + Captures < ' _ > {
237
- self . bcb_counters . iter_enumerated ( ) . filter_map ( |( bcb, & counter_kind ) | match counter_kind {
232
+ self . node_counters . iter_enumerated ( ) . filter_map ( |( bcb, & counter ) | match counter {
238
233
// Yield the BCB along with its associated expression ID.
239
234
Some ( BcbCounter :: Expression { id } ) => Some ( ( bcb, id) ) ,
240
235
// This BCB is associated with a counter or nothing, so skip it.
@@ -261,22 +256,20 @@ impl CoverageCounters {
261
256
}
262
257
}
263
258
264
- /// Helper struct that allows counter creation to inspect the BCB graph.
265
- struct MakeBcbCounters < ' a > {
266
- coverage_counters : CoverageCounters ,
267
- basic_coverage_blocks : & ' a CoverageGraph ,
259
+ /// Helper struct that allows counter creation to inspect the BCB graph, and
260
+ /// the set of nodes that need counters.
261
+ struct CountersBuilder < ' a > {
262
+ counters : CoverageCounters ,
263
+ graph : & ' a CoverageGraph ,
268
264
bcb_needs_counter : & ' a BitSet < BasicCoverageBlock > ,
269
265
}
270
266
271
- impl < ' a > MakeBcbCounters < ' a > {
272
- fn new (
273
- basic_coverage_blocks : & ' a CoverageGraph ,
274
- bcb_needs_counter : & ' a BitSet < BasicCoverageBlock > ,
275
- ) -> Self {
276
- assert_eq ! ( basic_coverage_blocks. num_nodes( ) , bcb_needs_counter. domain_size( ) ) ;
267
+ impl < ' a > CountersBuilder < ' a > {
268
+ fn new ( graph : & ' a CoverageGraph , bcb_needs_counter : & ' a BitSet < BasicCoverageBlock > ) -> Self {
269
+ assert_eq ! ( graph. num_nodes( ) , bcb_needs_counter. domain_size( ) ) ;
277
270
Self {
278
- coverage_counters : CoverageCounters :: with_num_bcbs ( basic_coverage_blocks . num_nodes ( ) ) ,
279
- basic_coverage_blocks ,
271
+ counters : CoverageCounters :: with_num_bcbs ( graph . num_nodes ( ) ) ,
272
+ graph ,
280
273
bcb_needs_counter,
281
274
}
282
275
}
@@ -291,7 +284,7 @@ impl<'a> MakeBcbCounters<'a> {
291
284
// nodes within the loop are visited before visiting any nodes outside
292
285
// the loop. It also keeps track of which loop(s) the traversal is
293
286
// currently inside.
294
- let mut traversal = TraverseCoverageGraphWithLoops :: new ( self . basic_coverage_blocks ) ;
287
+ let mut traversal = TraverseCoverageGraphWithLoops :: new ( self . graph ) ;
295
288
while let Some ( bcb) = traversal. next ( ) {
296
289
let _span = debug_span ! ( "traversal" , ?bcb) . entered ( ) ;
297
290
if self . bcb_needs_counter . contains ( bcb) {
@@ -318,26 +311,26 @@ impl<'a> MakeBcbCounters<'a> {
318
311
// We might also use that counter to compute one of the out-edge counters.
319
312
let node_counter = self . get_or_make_node_counter ( from_bcb) ;
320
313
321
- let successors = self . basic_coverage_blocks . successors [ from_bcb] . as_slice ( ) ;
314
+ let successors = self . graph . successors [ from_bcb] . as_slice ( ) ;
322
315
323
316
// If this node's out-edges won't sum to the node's counter,
324
317
// then there's no reason to create edge counters here.
325
- if !self . basic_coverage_blocks [ from_bcb] . is_out_summable {
318
+ if !self . graph [ from_bcb] . is_out_summable {
326
319
return ;
327
320
}
328
321
329
322
// When choosing which out-edge should be given a counter expression, ignore edges that
330
323
// already have counters, or could use the existing counter of their target node.
331
324
let out_edge_has_counter = |to_bcb| {
332
- if self . coverage_counters . bcb_edge_counters . contains_key ( & ( from_bcb, to_bcb) ) {
325
+ if self . counters . edge_counters . contains_key ( & ( from_bcb, to_bcb) ) {
333
326
return true ;
334
327
}
335
- self . basic_coverage_blocks . sole_predecessor ( to_bcb) == Some ( from_bcb)
336
- && self . coverage_counters . bcb_counters [ to_bcb] . is_some ( )
328
+ self . graph . sole_predecessor ( to_bcb) == Some ( from_bcb)
329
+ && self . counters . node_counters [ to_bcb] . is_some ( )
337
330
} ;
338
331
339
332
// Determine the set of out-edges that could benefit from being given an expression.
340
- let candidate_successors = self . basic_coverage_blocks . successors [ from_bcb]
333
+ let candidate_successors = self . graph . successors [ from_bcb]
341
334
. iter ( )
342
335
. copied ( )
343
336
. filter ( |& to_bcb| !out_edge_has_counter ( to_bcb) )
@@ -346,7 +339,7 @@ impl<'a> MakeBcbCounters<'a> {
346
339
347
340
// If there are out-edges without counters, choose one to be given an expression
348
341
// (computed from this node and the other out-edges) instead of a physical counter.
349
- let Some ( expression_to_bcb ) =
342
+ let Some ( target_bcb ) =
350
343
self . choose_out_edge_for_expression ( traversal, & candidate_successors)
351
344
else {
352
345
return ;
@@ -359,49 +352,44 @@ impl<'a> MakeBcbCounters<'a> {
359
352
. iter ( )
360
353
. copied ( )
361
354
// Skip the chosen edge, since we'll calculate its count from this sum.
362
- . filter ( |& to_bcb| to_bcb != expression_to_bcb )
355
+ . filter ( |& edge_target_bcb| edge_target_bcb != target_bcb )
363
356
. map ( |to_bcb| self . get_or_make_edge_counter ( from_bcb, to_bcb) )
364
357
. collect :: < Vec < _ > > ( ) ;
365
- let Some ( sum_of_all_other_out_edges) =
366
- self . coverage_counters . make_sum ( & other_out_edge_counters)
358
+ let Some ( sum_of_all_other_out_edges) = self . counters . make_sum ( & other_out_edge_counters)
367
359
else {
368
360
return ;
369
361
} ;
370
362
371
363
// Now create an expression for the chosen edge, by taking the counter
372
364
// for its source node and subtracting the sum of its sibling out-edges.
373
- let expression = self . coverage_counters . make_expression (
374
- node_counter,
375
- Op :: Subtract ,
376
- sum_of_all_other_out_edges,
377
- ) ;
365
+ let expression =
366
+ self . counters . make_expression ( node_counter, Op :: Subtract , sum_of_all_other_out_edges) ;
378
367
379
- debug ! ( "{expression_to_bcb :?} gets an expression: {expression:?}" ) ;
380
- self . coverage_counters . set_bcb_edge_counter ( from_bcb, expression_to_bcb , expression) ;
368
+ debug ! ( "{target_bcb :?} gets an expression: {expression:?}" ) ;
369
+ self . counters . set_edge_counter ( from_bcb, target_bcb , expression) ;
381
370
}
382
371
383
372
#[ instrument( level = "debug" , skip( self ) ) ]
384
373
fn get_or_make_node_counter ( & mut self , bcb : BasicCoverageBlock ) -> BcbCounter {
385
374
// If the BCB already has a counter, return it.
386
- if let Some ( counter_kind ) = self . coverage_counters . bcb_counters [ bcb] {
387
- debug ! ( "{bcb:?} already has a counter: {counter_kind :?}" ) ;
388
- return counter_kind ;
375
+ if let Some ( counter ) = self . counters . node_counters [ bcb] {
376
+ debug ! ( "{bcb:?} already has a counter: {counter :?}" ) ;
377
+ return counter ;
389
378
}
390
379
391
380
let counter = self . make_node_counter_inner ( bcb) ;
392
- self . coverage_counters . set_bcb_counter ( bcb, counter)
381
+ self . counters . set_node_counter ( bcb, counter)
393
382
}
394
383
395
384
fn make_node_counter_inner ( & mut self , bcb : BasicCoverageBlock ) -> BcbCounter {
396
385
// If the node's sole in-edge already has a counter, use that.
397
- if let Some ( sole_pred) = self . basic_coverage_blocks . sole_predecessor ( bcb)
398
- && let Some ( & edge_counter) =
399
- self . coverage_counters . bcb_edge_counters . get ( & ( sole_pred, bcb) )
386
+ if let Some ( sole_pred) = self . graph . sole_predecessor ( bcb)
387
+ && let Some ( & edge_counter) = self . counters . edge_counters . get ( & ( sole_pred, bcb) )
400
388
{
401
389
return edge_counter;
402
390
}
403
391
404
- let predecessors = self . basic_coverage_blocks . predecessors [ bcb] . as_slice ( ) ;
392
+ let predecessors = self . graph . predecessors [ bcb] . as_slice ( ) ;
405
393
406
394
// Handle cases where we can't compute a node's count from its in-edges:
407
395
// - START_BCB has no in-edges, so taking the sum would panic (or be wrong).
@@ -410,7 +398,7 @@ impl<'a> MakeBcbCounters<'a> {
410
398
// leading to infinite recursion.
411
399
if predecessors. len ( ) <= 1 || predecessors. contains ( & bcb) {
412
400
debug ! ( ?bcb, ?predecessors, "node has <=1 predecessors or is its own predecessor" ) ;
413
- let counter = self . coverage_counters . make_phys_node_counter ( bcb) ;
401
+ let counter = self . counters . make_phys_node_counter ( bcb) ;
414
402
debug ! ( ?bcb, ?counter, "node gets a physical counter" ) ;
415
403
return counter;
416
404
}
@@ -422,10 +410,8 @@ impl<'a> MakeBcbCounters<'a> {
422
410
. copied ( )
423
411
. map ( |from_bcb| self . get_or_make_edge_counter ( from_bcb, bcb) )
424
412
. collect :: < Vec < _ > > ( ) ;
425
- let sum_of_in_edges: BcbCounter = self
426
- . coverage_counters
427
- . make_sum ( & in_edge_counters)
428
- . expect ( "there must be at least one in-edge" ) ;
413
+ let sum_of_in_edges: BcbCounter =
414
+ self . counters . make_sum ( & in_edge_counters) . expect ( "there must be at least one in-edge" ) ;
429
415
430
416
debug ! ( "{bcb:?} gets a new counter (sum of predecessor counters): {sum_of_in_edges:?}" ) ;
431
417
sum_of_in_edges
@@ -438,15 +424,13 @@ impl<'a> MakeBcbCounters<'a> {
438
424
to_bcb : BasicCoverageBlock ,
439
425
) -> BcbCounter {
440
426
// If the edge already has a counter, return it.
441
- if let Some ( & counter_kind) =
442
- self . coverage_counters . bcb_edge_counters . get ( & ( from_bcb, to_bcb) )
443
- {
444
- debug ! ( "Edge {from_bcb:?}->{to_bcb:?} already has a counter: {counter_kind:?}" ) ;
445
- return counter_kind;
427
+ if let Some ( & counter) = self . counters . edge_counters . get ( & ( from_bcb, to_bcb) ) {
428
+ debug ! ( "Edge {from_bcb:?}->{to_bcb:?} already has a counter: {counter:?}" ) ;
429
+ return counter;
446
430
}
447
431
448
432
let counter = self . make_edge_counter_inner ( from_bcb, to_bcb) ;
449
- self . coverage_counters . set_bcb_edge_counter ( from_bcb, to_bcb, counter)
433
+ self . counters . set_edge_counter ( from_bcb, to_bcb, counter)
450
434
}
451
435
452
436
fn make_edge_counter_inner (
@@ -456,7 +440,7 @@ impl<'a> MakeBcbCounters<'a> {
456
440
) -> BcbCounter {
457
441
// If the target node has exactly one in-edge (i.e. this one), then just
458
442
// use the node's counter, since it will have the same value.
459
- if let Some ( sole_pred) = self . basic_coverage_blocks . sole_predecessor ( to_bcb) {
443
+ if let Some ( sole_pred) = self . graph . sole_predecessor ( to_bcb) {
460
444
assert_eq ! ( sole_pred, from_bcb) ;
461
445
// This call must take care not to invoke `get_or_make_edge` for
462
446
// this edge, since that would result in infinite recursion!
@@ -465,13 +449,13 @@ impl<'a> MakeBcbCounters<'a> {
465
449
466
450
// If the source node has exactly one out-edge (i.e. this one) and would have
467
451
// the same execution count as that edge, then just use the node's counter.
468
- if let Some ( simple_succ) = self . basic_coverage_blocks . simple_successor ( from_bcb) {
452
+ if let Some ( simple_succ) = self . graph . simple_successor ( from_bcb) {
469
453
assert_eq ! ( simple_succ, to_bcb) ;
470
454
return self . get_or_make_node_counter ( from_bcb) ;
471
455
}
472
456
473
457
// Make a new counter to count this edge.
474
- let counter = self . coverage_counters . make_phys_edge_counter ( from_bcb, to_bcb) ;
458
+ let counter = self . counters . make_phys_edge_counter ( from_bcb, to_bcb) ;
475
459
debug ! ( ?from_bcb, ?to_bcb, ?counter, "edge gets a physical counter" ) ;
476
460
counter
477
461
}
@@ -518,9 +502,9 @@ impl<'a> MakeBcbCounters<'a> {
518
502
for & target_bcb in candidate_successors {
519
503
// An edge is a reloop edge if its target dominates any BCB that has
520
504
// an edge back to the loop header. (Otherwise it's an exit edge.)
521
- let is_reloop_edge = reloop_bcbs. iter ( ) . any ( | & reloop_bcb| {
522
- self . basic_coverage_blocks . dominates ( target_bcb , reloop_bcb )
523
- } ) ;
505
+ let is_reloop_edge = reloop_bcbs
506
+ . iter ( )
507
+ . any ( | & reloop_bcb| self . graph . dominates ( target_bcb , reloop_bcb ) ) ;
524
508
if is_reloop_edge {
525
509
// We found a good out-edge to be given an expression.
526
510
return Some ( target_bcb) ;
0 commit comments