@@ -107,7 +107,8 @@ use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
107
107
use rustc_middle:: middle:: exported_symbols:: { SymbolExportInfo , SymbolExportLevel } ;
108
108
use rustc_middle:: mir;
109
109
use rustc_middle:: mir:: mono:: {
110
- CodegenUnit , CodegenUnitNameBuilder , InstantiationMode , Linkage , MonoItem , Visibility ,
110
+ CodegenUnit , CodegenUnitNameBuilder , InstantiationMode , Linkage , MonoItem , MonoItemData ,
111
+ Visibility ,
111
112
} ;
112
113
use rustc_middle:: query:: Providers ;
113
114
use rustc_middle:: ty:: print:: { characteristic_def_id_of_type, with_no_trimmed_paths} ;
@@ -130,11 +131,6 @@ struct PlacedMonoItems<'tcx> {
130
131
codegen_units : Vec < CodegenUnit < ' tcx > > ,
131
132
132
133
internalization_candidates : FxHashSet < MonoItem < ' tcx > > ,
133
-
134
- /// These must be obtained when the iterator in `partition` runs. They
135
- /// can't be obtained later because some inlined functions might not be
136
- /// reachable.
137
- unique_inlined_stats : ( usize , usize ) ,
138
134
}
139
135
140
136
// The output CGUs are sorted by name.
@@ -152,11 +148,11 @@ where
152
148
153
149
// Place all mono items into a codegen unit. `place_mono_items` is
154
150
// responsible for initializing the CGU size estimates.
155
- let PlacedMonoItems { mut codegen_units, internalization_candidates, unique_inlined_stats } = {
151
+ let PlacedMonoItems { mut codegen_units, internalization_candidates } = {
156
152
let _prof_timer = tcx. prof . generic_activity ( "cgu_partitioning_place_items" ) ;
157
153
let placed = place_mono_items ( cx, mono_items) ;
158
154
159
- debug_dump ( tcx, "PLACE" , & placed. codegen_units , placed . unique_inlined_stats ) ;
155
+ debug_dump ( tcx, "PLACE" , & placed. codegen_units ) ;
160
156
161
157
placed
162
158
} ;
@@ -167,7 +163,7 @@ where
167
163
{
168
164
let _prof_timer = tcx. prof . generic_activity ( "cgu_partitioning_merge_cgus" ) ;
169
165
merge_codegen_units ( cx, & mut codegen_units) ;
170
- debug_dump ( tcx, "MERGE" , & codegen_units, unique_inlined_stats ) ;
166
+ debug_dump ( tcx, "MERGE" , & codegen_units) ;
171
167
}
172
168
173
169
// Make as many symbols "internal" as possible, so LLVM has more freedom to
@@ -176,7 +172,7 @@ where
176
172
let _prof_timer = tcx. prof . generic_activity ( "cgu_partitioning_internalize_symbols" ) ;
177
173
internalize_symbols ( cx, & mut codegen_units, internalization_candidates) ;
178
174
179
- debug_dump ( tcx, "INTERNALIZE" , & codegen_units, unique_inlined_stats ) ;
175
+ debug_dump ( tcx, "INTERNALIZE" , & codegen_units) ;
180
176
}
181
177
182
178
// Mark one CGU for dead code, if necessary.
@@ -216,18 +212,12 @@ where
216
212
let cgu_name_builder = & mut CodegenUnitNameBuilder :: new ( cx. tcx ) ;
217
213
let cgu_name_cache = & mut FxHashMap :: default ( ) ;
218
214
219
- let mut num_unique_inlined_items = 0 ;
220
- let mut unique_inlined_items_size = 0 ;
221
215
for mono_item in mono_items {
222
216
// Handle only root items directly here. Inlined items are handled at
223
217
// the bottom of the loop based on reachability.
224
218
match mono_item. instantiation_mode ( cx. tcx ) {
225
219
InstantiationMode :: GloballyShared { .. } => { }
226
- InstantiationMode :: LocalCopy => {
227
- num_unique_inlined_items += 1 ;
228
- unique_inlined_items_size += mono_item. size_estimate ( cx. tcx ) ;
229
- continue ;
230
- }
220
+ InstantiationMode :: LocalCopy => continue ,
231
221
}
232
222
233
223
let characteristic_def_id = characteristic_def_id_of_mono_item ( cx. tcx , mono_item) ;
@@ -256,8 +246,9 @@ where
256
246
if visibility == Visibility :: Hidden && can_be_internalized {
257
247
internalization_candidates. insert ( mono_item) ;
258
248
}
249
+ let size_estimate = mono_item. size_estimate ( cx. tcx ) ;
259
250
260
- cgu. items_mut ( ) . insert ( mono_item, ( linkage, visibility) ) ;
251
+ cgu. items_mut ( ) . insert ( mono_item, MonoItemData { linkage, visibility, size_estimate } ) ;
261
252
262
253
// Get all inlined items that are reachable from `mono_item` without
263
254
// going via another root item. This includes drop-glue, functions from
@@ -271,7 +262,11 @@ where
271
262
// the `insert` will be a no-op.
272
263
for inlined_item in reachable_inlined_items {
273
264
// This is a CGU-private copy.
274
- cgu. items_mut ( ) . insert ( inlined_item, ( Linkage :: Internal , Visibility :: Default ) ) ;
265
+ cgu. items_mut ( ) . entry ( inlined_item) . or_insert_with ( || MonoItemData {
266
+ linkage : Linkage :: Internal ,
267
+ visibility : Visibility :: Default ,
268
+ size_estimate : inlined_item. size_estimate ( cx. tcx ) ,
269
+ } ) ;
275
270
}
276
271
}
277
272
@@ -286,14 +281,10 @@ where
286
281
codegen_units. sort_by ( |a, b| a. name ( ) . as_str ( ) . cmp ( b. name ( ) . as_str ( ) ) ) ;
287
282
288
283
for cgu in codegen_units. iter_mut ( ) {
289
- cgu. compute_size_estimate ( cx . tcx ) ;
284
+ cgu. compute_size_estimate ( ) ;
290
285
}
291
286
292
- return PlacedMonoItems {
293
- codegen_units,
294
- internalization_candidates,
295
- unique_inlined_stats : ( num_unique_inlined_items, unique_inlined_items_size) ,
296
- } ;
287
+ return PlacedMonoItems { codegen_units, internalization_candidates } ;
297
288
298
289
fn get_reachable_inlined_items < ' tcx > (
299
290
tcx : TyCtxt < ' tcx > ,
@@ -349,7 +340,7 @@ fn merge_codegen_units<'tcx>(
349
340
&& codegen_units. iter ( ) . any ( |cgu| cgu. size_estimate ( ) < NON_INCR_MIN_CGU_SIZE ) )
350
341
{
351
342
// Sort small cgus to the back.
352
- codegen_units. sort_by_cached_key ( |cgu| cmp:: Reverse ( cgu. size_estimate ( ) ) ) ;
343
+ codegen_units. sort_by_key ( |cgu| cmp:: Reverse ( cgu. size_estimate ( ) ) ) ;
353
344
354
345
let mut smallest = codegen_units. pop ( ) . unwrap ( ) ;
355
346
let second_smallest = codegen_units. last_mut ( ) . unwrap ( ) ;
@@ -358,7 +349,7 @@ fn merge_codegen_units<'tcx>(
358
349
// may be duplicate inlined items, in which case the destination CGU is
359
350
// unaffected. Recalculate size estimates afterwards.
360
351
second_smallest. items_mut ( ) . extend ( smallest. items_mut ( ) . drain ( ) ) ;
361
- second_smallest. compute_size_estimate ( cx . tcx ) ;
352
+ second_smallest. compute_size_estimate ( ) ;
362
353
363
354
// Record that `second_smallest` now contains all the stuff that was
364
355
// in `smallest` before.
@@ -492,7 +483,7 @@ fn internalize_symbols<'tcx>(
492
483
for cgu in codegen_units {
493
484
let home_cgu = MonoItemPlacement :: SingleCgu ( cgu. name ( ) ) ;
494
485
495
- for ( item, linkage_and_visibility ) in cgu. items_mut ( ) {
486
+ for ( item, data ) in cgu. items_mut ( ) {
496
487
if !internalization_candidates. contains ( item) {
497
488
// This item is no candidate for internalizing, so skip it.
498
489
continue ;
@@ -520,7 +511,8 @@ fn internalize_symbols<'tcx>(
520
511
521
512
// If we got here, we did not find any uses from other CGUs, so
522
513
// it's fine to make this monomorphization internal.
523
- * linkage_and_visibility = ( Linkage :: Internal , Visibility :: Default ) ;
514
+ data. linkage = Linkage :: Internal ;
515
+ data. visibility = Visibility :: Default ;
524
516
}
525
517
}
526
518
}
@@ -537,7 +529,7 @@ fn mark_code_coverage_dead_code_cgu<'tcx>(codegen_units: &mut [CodegenUnit<'tcx>
537
529
// function symbols to be included via `-u` or `/include` linker args.
538
530
let dead_code_cgu = codegen_units
539
531
. iter_mut ( )
540
- . filter ( |cgu| cgu. items ( ) . iter ( ) . any ( |( _, ( linkage , _ ) ) | * linkage == Linkage :: External ) )
532
+ . filter ( |cgu| cgu. items ( ) . iter ( ) . any ( |( _, data ) | data . linkage == Linkage :: External ) )
541
533
. min_by_key ( |cgu| cgu. size_estimate ( ) ) ;
542
534
543
535
// If there are no CGUs that have externally linked items, then we just
@@ -851,12 +843,7 @@ fn default_visibility(tcx: TyCtxt<'_>, id: DefId, is_generic: bool) -> Visibilit
851
843
}
852
844
}
853
845
854
- fn debug_dump < ' a , ' tcx : ' a > (
855
- tcx : TyCtxt < ' tcx > ,
856
- label : & str ,
857
- cgus : & [ CodegenUnit < ' tcx > ] ,
858
- ( unique_inlined_items, unique_inlined_size) : ( usize , usize ) ,
859
- ) {
846
+ fn debug_dump < ' a , ' tcx : ' a > ( tcx : TyCtxt < ' tcx > , label : & str , cgus : & [ CodegenUnit < ' tcx > ] ) {
860
847
let dump = move || {
861
848
use std:: fmt:: Write ;
862
849
@@ -865,28 +852,36 @@ fn debug_dump<'a, 'tcx: 'a>(
865
852
866
853
// Note: every unique root item is placed exactly once, so the number
867
854
// of unique root items always equals the number of placed root items.
855
+ //
856
+ // Also, unreached inlined items won't be counted here. This is fine.
857
+
858
+ let mut inlined_items = FxHashSet :: default ( ) ;
868
859
869
860
let mut root_items = 0 ;
870
- // unique_inlined_items is passed in above.
861
+ let mut unique_inlined_items = 0 ;
871
862
let mut placed_inlined_items = 0 ;
872
863
873
864
let mut root_size = 0 ;
874
- // unique_inlined_size is passed in above.
865
+ let mut unique_inlined_size = 0 ;
875
866
let mut placed_inlined_size = 0 ;
876
867
877
868
for cgu in cgus. iter ( ) {
878
869
num_cgus += 1 ;
879
870
all_cgu_sizes. push ( cgu. size_estimate ( ) ) ;
880
871
881
- for ( item, _ ) in cgu. items ( ) {
872
+ for ( item, data ) in cgu. items ( ) {
882
873
match item. instantiation_mode ( tcx) {
883
874
InstantiationMode :: GloballyShared { .. } => {
884
875
root_items += 1 ;
885
- root_size += item . size_estimate ( tcx ) ;
876
+ root_size += data . size_estimate ;
886
877
}
887
878
InstantiationMode :: LocalCopy => {
879
+ if inlined_items. insert ( item) {
880
+ unique_inlined_items += 1 ;
881
+ unique_inlined_size += data. size_estimate ;
882
+ }
888
883
placed_inlined_items += 1 ;
889
- placed_inlined_size += item . size_estimate ( tcx ) ;
884
+ placed_inlined_size += data . size_estimate ;
890
885
}
891
886
}
892
887
}
@@ -928,7 +923,7 @@ fn debug_dump<'a, 'tcx: 'a>(
928
923
let mean_size = size as f64 / num_items as f64 ;
929
924
930
925
let mut placed_item_sizes: Vec < _ > =
931
- cgu. items ( ) . iter ( ) . map ( |( item , _ ) | item . size_estimate ( tcx ) ) . collect ( ) ;
926
+ cgu. items ( ) . values ( ) . map ( |data| data . size_estimate ) . collect ( ) ;
932
927
placed_item_sizes. sort_unstable_by_key ( |& n| cmp:: Reverse ( n) ) ;
933
928
let sizes = list ( & placed_item_sizes) ;
934
929
@@ -937,15 +932,16 @@ fn debug_dump<'a, 'tcx: 'a>(
937
932
let _ =
938
933
writeln ! ( s, " - items: {num_items}, mean size: {mean_size:.1}, sizes: {sizes}" , ) ;
939
934
940
- for ( item, linkage) in cgu. items_in_deterministic_order ( tcx) {
935
+ for ( item, data) in cgu. items_in_deterministic_order ( tcx) {
936
+ let linkage = data. linkage ;
941
937
let symbol_name = item. symbol_name ( tcx) . name ;
942
938
let symbol_hash_start = symbol_name. rfind ( 'h' ) ;
943
939
let symbol_hash = symbol_hash_start. map_or ( "<no hash>" , |i| & symbol_name[ i..] ) ;
944
- let size = item. size_estimate ( tcx) ;
945
940
let kind = match item. instantiation_mode ( tcx) {
946
941
InstantiationMode :: GloballyShared { .. } => "root" ,
947
942
InstantiationMode :: LocalCopy => "inlined" ,
948
943
} ;
944
+ let size = data. size_estimate ;
949
945
let _ = with_no_trimmed_paths ! ( writeln!(
950
946
s,
951
947
" - {item} [{linkage:?}] [{symbol_hash}] ({kind}, size: {size})"
@@ -1100,8 +1096,8 @@ fn collect_and_partition_mono_items(tcx: TyCtxt<'_>, (): ()) -> (&DefIdSet, &[Co
1100
1096
let mut item_to_cgus: FxHashMap < _ , Vec < _ > > = Default :: default ( ) ;
1101
1097
1102
1098
for cgu in codegen_units {
1103
- for ( & mono_item, & linkage ) in cgu. items ( ) {
1104
- item_to_cgus. entry ( mono_item) . or_default ( ) . push ( ( cgu. name ( ) , linkage) ) ;
1099
+ for ( & mono_item, & data ) in cgu. items ( ) {
1100
+ item_to_cgus. entry ( mono_item) . or_default ( ) . push ( ( cgu. name ( ) , data . linkage ) ) ;
1105
1101
}
1106
1102
}
1107
1103
@@ -1114,7 +1110,7 @@ fn collect_and_partition_mono_items(tcx: TyCtxt<'_>, (): ()) -> (&DefIdSet, &[Co
1114
1110
let cgus = item_to_cgus. get_mut ( i) . unwrap_or ( & mut empty) ;
1115
1111
cgus. sort_by_key ( |( name, _) | * name) ;
1116
1112
cgus. dedup ( ) ;
1117
- for & ( ref cgu_name, ( linkage, _ ) ) in cgus. iter ( ) {
1113
+ for & ( ref cgu_name, linkage) in cgus. iter ( ) {
1118
1114
output. push ( ' ' ) ;
1119
1115
output. push_str ( cgu_name. as_str ( ) ) ;
1120
1116
0 commit comments