Skip to content

Commit 88eced5

Browse files
eddybeddyb
committed
rustc: precompute the largest Niche and store it in LayoutDetails.
1 parent dfbf464 commit 88eced5

File tree

2 files changed

+120
-90
lines changed

2 files changed

+120
-90
lines changed

‎src/librustc/ty/layout.rs

+100-90
Original file line numberDiff line numberDiff line change
@@ -246,13 +246,22 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
246246
let align = a.value.align(dl).max(b_align).max(dl.aggregate_align);
247247
let b_offset = a.value.size(dl).align_to(b_align.abi);
248248
let size = (b_offset + b.value.size(dl)).align_to(align.abi);
249+
250+
// HACK(nox): We iter on `b` and then `a` because `max_by_key`
251+
// returns the last maximum.
252+
let largest_niche = Niche::from_scalar(dl, b_offset, b.clone())
253+
.into_iter()
254+
.chain(Niche::from_scalar(dl, Size::ZERO, a.clone()))
255+
.max_by_key(|niche| niche.available(dl));
256+
249257
LayoutDetails {
250258
variants: Variants::Single { index: VariantIdx::new(0) },
251259
fields: FieldPlacement::Arbitrary {
252260
offsets: vec![Size::ZERO, b_offset],
253261
memory_index: vec![0, 1]
254262
},
255263
abi: Abi::ScalarPair(a, b),
264+
largest_niche,
256265
align,
257266
size
258267
}
@@ -321,6 +330,8 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
321330

322331

323332
let mut offset = Size::ZERO;
333+
let mut largest_niche = None;
334+
let mut largest_niche_available = 0;
324335

325336
if let StructKind::Prefixed(prefix_size, prefix_align) = kind {
326337
let prefix_align = if packed {
@@ -355,6 +366,15 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
355366
debug!("univariant offset: {:?} field: {:#?}", offset, field);
356367
offsets[i as usize] = offset;
357368

369+
if let Some(mut niche) = field.largest_niche.clone() {
370+
let available = niche.available(dl);
371+
if available > largest_niche_available {
372+
largest_niche_available = available;
373+
niche.offset += offset;
374+
largest_niche = Some(niche);
375+
}
376+
}
377+
358378
offset = offset.checked_add(field.size, dl)
359379
.ok_or(LayoutError::SizeOverflow(ty))?;
360380
}
@@ -466,6 +486,7 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
466486
memory_index
467487
},
468488
abi,
489+
largest_niche,
469490
align,
470491
size
471492
})
@@ -525,6 +546,7 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
525546
variants: Variants::Single { index: VariantIdx::new(0) },
526547
fields: FieldPlacement::Union(0),
527548
abi: Abi::Uninhabited,
549+
largest_niche: None,
528550
align: dl.i8_align,
529551
size: Size::ZERO
530552
})
@@ -583,13 +605,20 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
583605
Abi::Aggregate { sized: true }
584606
};
585607

608+
let largest_niche = if count != 0 {
609+
element.largest_niche.clone()
610+
} else {
611+
None
612+
};
613+
586614
tcx.intern_layout(LayoutDetails {
587615
variants: Variants::Single { index: VariantIdx::new(0) },
588616
fields: FieldPlacement::Array {
589617
stride: element.size,
590618
count
591619
},
592620
abi,
621+
largest_niche,
593622
align: element.align,
594623
size
595624
})
@@ -603,6 +632,7 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
603632
count: 0
604633
},
605634
abi: Abi::Aggregate { sized: false },
635+
largest_niche: None,
606636
align: element.align,
607637
size: Size::ZERO
608638
})
@@ -615,6 +645,7 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
615645
count: 0
616646
},
617647
abi: Abi::Aggregate { sized: false },
648+
largest_niche: None,
618649
align: dl.i8_align,
619650
size: Size::ZERO
620651
})
@@ -683,6 +714,7 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
683714
element: scalar,
684715
count
685716
},
717+
largest_niche: element.largest_niche.clone(),
686718
size,
687719
align,
688720
})
@@ -768,6 +800,7 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
768800
variants: Variants::Single { index },
769801
fields: FieldPlacement::Union(variants[index].len()),
770802
abi,
803+
largest_niche: None,
771804
align,
772805
size: size.align_to(align.abi)
773806
}));
@@ -829,14 +862,38 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
829862
// `#[rustc_layout_scalar_valid_range(n)]`
830863
// attribute to widen the range of anything as that would probably
831864
// result in UB somewhere
865+
// FIXME(eddyb) the asserts are probably not needed,
866+
// as larger validity ranges would result in missed
867+
// optimizations, *not* wrongly assuming the inner
868+
// value is valid. e.g. unions enlarge validity ranges,
869+
// because the values may be uninitialized.
832870
if let Bound::Included(start) = start {
871+
// FIXME(eddyb) this might be incorrect - it doesn't
872+
// account for wrap-around (end < start) ranges.
833873
assert!(*scalar.valid_range.start() <= start);
834874
scalar.valid_range = start..=*scalar.valid_range.end();
835875
}
836876
if let Bound::Included(end) = end {
877+
// FIXME(eddyb) this might be incorrect - it doesn't
878+
// account for wrap-around (end < start) ranges.
837879
assert!(*scalar.valid_range.end() >= end);
838880
scalar.valid_range = *scalar.valid_range.start()..=end;
839881
}
882+
883+
// Update `largest_niche` if we have introduced a larger niche.
884+
let niche = Niche::from_scalar(dl, Size::ZERO, scalar.clone());
885+
if let Some(niche) = niche {
886+
match &st.largest_niche {
887+
Some(largest_niche) => {
888+
// Replace the existing niche even if they're equal,
889+
// because this one is at a lower offset.
890+
if largest_niche.available(dl) <= niche.available(dl) {
891+
st.largest_niche = Some(niche);
892+
}
893+
}
894+
None => st.largest_niche = Some(niche),
895+
}
896+
}
840897
}
841898
_ => assert!(
842899
start == Bound::Unbounded && end == Bound::Unbounded,
@@ -845,6 +902,7 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
845902
st,
846903
),
847904
}
905+
848906
return Ok(tcx.intern_layout(st));
849907
}
850908

@@ -886,8 +944,10 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
886944
let count = (
887945
niche_variants.end().as_u32() - niche_variants.start().as_u32() + 1
888946
) as u128;
947+
// FIXME(#62691) use the largest niche across all fields,
948+
// not just the first one.
889949
for (field_index, &field) in variants[i].iter().enumerate() {
890-
let niche = match self.find_niche(field)? {
950+
let niche = match &field.largest_niche {
891951
Some(niche) => niche,
892952
_ => continue,
893953
};
@@ -937,6 +997,10 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
937997
abi = Abi::Uninhabited;
938998
}
939999

1000+
1001+
let largest_niche =
1002+
Niche::from_scalar(dl, offset, niche_scalar.clone());
1003+
9401004
return Ok(tcx.intern_layout(LayoutDetails {
9411005
variants: Variants::Multiple {
9421006
discr: niche_scalar,
@@ -953,6 +1017,7 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
9531017
memory_index: vec![0]
9541018
},
9551019
abi,
1020+
largest_niche,
9561021
size,
9571022
align,
9581023
}));
@@ -1164,6 +1229,8 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
11641229
abi = Abi::Uninhabited;
11651230
}
11661231

1232+
let largest_niche = Niche::from_scalar(dl, Size::ZERO, tag.clone());
1233+
11671234
tcx.intern_layout(LayoutDetails {
11681235
variants: Variants::Multiple {
11691236
discr: tag,
@@ -1175,6 +1242,7 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
11751242
offsets: vec![Size::ZERO],
11761243
memory_index: vec![0]
11771244
},
1245+
largest_niche,
11781246
abi,
11791247
align,
11801248
size
@@ -1332,16 +1400,31 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
13321400
// locals as part of the prefix. We compute the layout of all of
13331401
// these fields at once to get optimal packing.
13341402
let discr_index = substs.prefix_tys(def_id, tcx).count();
1335-
let promoted_tys =
1336-
ineligible_locals.iter().map(|local| subst_field(info.field_tys[local]));
1337-
let prefix_tys = substs.prefix_tys(def_id, tcx)
1338-
.chain(iter::once(substs.discr_ty(tcx)))
1339-
.chain(promoted_tys);
1340-
let prefix = self.univariant_uninterned(
1403+
// FIXME(eddyb) set the correct vaidity range for the discriminant.
1404+
let discr_layout = self.layout_of(substs.discr_ty(tcx))?;
1405+
let discr = match &discr_layout.abi {
1406+
Abi::Scalar(s) => s.clone(),
1407+
_ => bug!(),
1408+
};
1409+
// FIXME(eddyb) wrap each promoted type in `MaybeUninit` so that they
1410+
// don't poison the `largest_niche` or `abi` fields of `prefix`.
1411+
let promoted_layouts = ineligible_locals.iter()
1412+
.map(|local| subst_field(info.field_tys[local]))
1413+
.map(|ty| self.layout_of(ty));
1414+
let prefix_layouts = substs.prefix_tys(def_id, tcx)
1415+
.map(|ty| self.layout_of(ty))
1416+
.chain(iter::once(Ok(discr_layout)))
1417+
.chain(promoted_layouts)
1418+
.collect::<Result<Vec<_>, _>>()?;
1419+
let mut prefix = self.univariant_uninterned(
13411420
ty,
1342-
&prefix_tys.map(|ty| self.layout_of(ty)).collect::<Result<Vec<_>, _>>()?,
1421+
&prefix_layouts,
13431422
&ReprOptions::default(),
1344-
StructKind::AlwaysSized)?;
1423+
StructKind::AlwaysSized,
1424+
)?;
1425+
// FIXME(eddyb) need `MaybeUninit` around promoted types (see above).
1426+
prefix.largest_niche = None;
1427+
13451428
let (prefix_size, prefix_align) = (prefix.size, prefix.align);
13461429

13471430
// Split the prefix layout into the "outer" fields (upvars and
@@ -1463,10 +1546,6 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
14631546
} else {
14641547
Abi::Aggregate { sized: true }
14651548
};
1466-
let discr = match &self.layout_of(substs.discr_ty(tcx))?.abi {
1467-
Abi::Scalar(s) => s.clone(),
1468-
_ => bug!(),
1469-
};
14701549

14711550
let layout = tcx.intern_layout(LayoutDetails {
14721551
variants: Variants::Multiple {
@@ -1477,6 +1556,7 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
14771556
},
14781557
fields: outer_fields,
14791558
abi,
1559+
largest_niche: prefix.largest_niche,
14801560
size,
14811561
align,
14821562
});
@@ -1950,6 +2030,7 @@ where
19502030
variants: Variants::Single { index: variant_index },
19512031
fields: FieldPlacement::Union(fields),
19522032
abi: Abi::Uninhabited,
2033+
largest_niche: None,
19532034
align: tcx.data_layout.i8_align,
19542035
size: Size::ZERO
19552036
})
@@ -2222,83 +2303,6 @@ where
22222303
}
22232304
}
22242305

2225-
impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
2226-
/// Find the offset of a niche leaf field, starting from
2227-
/// the given type and recursing through aggregates.
2228-
// FIXME(eddyb) traverse already optimized enums.
2229-
fn find_niche(&self, layout: TyLayout<'tcx>) -> Result<Option<Niche>, LayoutError<'tcx>> {
2230-
let scalar_niche = |scalar: &Scalar, offset| {
2231-
let niche = Niche { offset, scalar: scalar.clone() };
2232-
if niche.available(self) > 0 {
2233-
Some(niche)
2234-
} else {
2235-
None
2236-
}
2237-
};
2238-
2239-
// Locals variables which live across yields are stored
2240-
// in the generator type as fields. These may be uninitialized
2241-
// so we don't look for niches there.
2242-
if let ty::Generator(..) = layout.ty.sty {
2243-
return Ok(None);
2244-
}
2245-
2246-
match layout.abi {
2247-
Abi::Scalar(ref scalar) => {
2248-
return Ok(scalar_niche(scalar, Size::ZERO));
2249-
}
2250-
Abi::ScalarPair(ref a, ref b) => {
2251-
// HACK(nox): We iter on `b` and then `a` because `max_by_key`
2252-
// returns the last maximum.
2253-
let niche = iter::once(
2254-
(b, a.value.size(self).align_to(b.value.align(self).abi))
2255-
)
2256-
.chain(iter::once((a, Size::ZERO)))
2257-
.filter_map(|(scalar, offset)| scalar_niche(scalar, offset))
2258-
.max_by_key(|niche| niche.available(self));
2259-
return Ok(niche);
2260-
}
2261-
Abi::Vector { ref element, .. } => {
2262-
return Ok(scalar_niche(element, Size::ZERO));
2263-
}
2264-
_ => {}
2265-
}
2266-
2267-
// Perhaps one of the fields is non-zero, let's recurse and find out.
2268-
if let FieldPlacement::Union(_) = layout.fields {
2269-
// Only Rust enums have safe-to-inspect fields
2270-
// (a discriminant), other unions are unsafe.
2271-
if let Variants::Single { .. } = layout.variants {
2272-
return Ok(None);
2273-
}
2274-
}
2275-
if let FieldPlacement::Array { count: original_64_bit_count, .. } = layout.fields {
2276-
// rust-lang/rust#57038: avoid ICE within FieldPlacement::count when count too big
2277-
if original_64_bit_count > usize::max_value() as u64 {
2278-
return Err(LayoutError::SizeOverflow(layout.ty));
2279-
}
2280-
if layout.fields.count() > 0 {
2281-
return self.find_niche(layout.field(self, 0)?);
2282-
} else {
2283-
return Ok(None);
2284-
}
2285-
}
2286-
let mut niche = None;
2287-
let mut available = 0;
2288-
for i in 0..layout.fields.count() {
2289-
if let Some(mut c) = self.find_niche(layout.field(self, i)?)? {
2290-
let c_available = c.available(self);
2291-
if c_available > available {
2292-
available = c_available;
2293-
c.offset += layout.fields.offset(i);
2294-
niche = Some(c);
2295-
}
2296-
}
2297-
}
2298-
Ok(niche)
2299-
}
2300-
}
2301-
23022306
impl<'a> HashStable<StableHashingContext<'a>> for Variants {
23032307
fn hash_stable<W: StableHasherResult>(&self,
23042308
hcx: &mut StableHashingContext<'a>,
@@ -2419,10 +2423,16 @@ impl<'a> HashStable<StableHashingContext<'a>> for Scalar {
24192423
}
24202424
}
24212425

2426+
impl_stable_hash_for!(struct crate::ty::layout::Niche {
2427+
offset,
2428+
scalar
2429+
});
2430+
24222431
impl_stable_hash_for!(struct crate::ty::layout::LayoutDetails {
24232432
variants,
24242433
fields,
24252434
abi,
2435+
largest_niche,
24262436
size,
24272437
align
24282438
});

0 commit comments

Comments
 (0)