Skip to content

Commit ad758f8

Browse files
ogoffarterikdesjardins
ogoffart
authored and
erikdesjardins
committed
Use the niche optimisation if other enum variants are small enough
* Put the largest niche first * Add test from issue 63866 * Add test for enum of sized/unsized * Prefer fields that are already earlier in the struct
1 parent 6f9a8a7 commit ad758f8

File tree

5 files changed

+237
-65
lines changed

5 files changed

+237
-65
lines changed

‎compiler/rustc_middle/src/ty/layout.rs‎

Lines changed: 135 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -285,6 +285,31 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
285285

286286
let mut align = if pack.is_some() { dl.i8_align } else { dl.aggregate_align };
287287

288+
let largest_niche_index = if matches!(kind, StructKind::Prefixed{..}) || repr.hide_niche() {
289+
None
290+
} else {
291+
fields
292+
.iter()
293+
.enumerate()
294+
.filter_map(|(i, &field)| field.largest_niche.as_ref().map(|n| (i, n)))
295+
.max_by_key(|(i, niche)| {
296+
(
297+
niche.available(dl),
298+
// Prefer niches that occur earlier in their respective field, to maximize space after the niche.
299+
cmp::Reverse(niche.offset),
300+
// Prefer fields that occur earlier in the struct, to avoid reordering fields unnecessarily.
301+
cmp::Reverse(*i),
302+
)
303+
})
304+
.map(|(i, _)| i as u32)
305+
};
306+
307+
// inverse_memory_index holds field indices by increasing memory offset.
308+
// That is, if field 5 has offset 0, the first element of inverse_memory_index is 5.
309+
// We now write field offsets to the corresponding offset slot;
310+
// field 5 with offset 0 puts 0 in offsets[5].
311+
// At the bottom of this function, we invert `inverse_memory_index` to
312+
// produce `memory_index` (see `invert_mapping`).
288313
let mut inverse_memory_index: Vec<u32> = (0..fields.len() as u32).collect();
289314

290315
let optimize = !repr.inhibit_struct_field_reordering_opt();
@@ -298,10 +323,15 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
298323
match kind {
299324
StructKind::AlwaysSized | StructKind::MaybeUnsized => {
300325
optimizing.sort_by_key(|&x| {
301-
// Place ZSTs first to avoid "interesting offsets",
302-
// especially with only one or two non-ZST fields.
303326
let f = &fields[x as usize];
304-
(!f.is_zst(), cmp::Reverse(field_align(f)))
327+
(
328+
// Place ZSTs first to avoid "interesting offsets",
329+
// especially with only one or two non-ZST fields.
330+
!f.is_zst(),
331+
cmp::Reverse(field_align(f)),
332+
// Try to put the largest niche earlier.
333+
Some(x) != largest_niche_index,
334+
)
305335
});
306336
}
307337
StructKind::Prefixed(..) => {
@@ -310,20 +340,29 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
310340
optimizing.sort_by_key(|&x| field_align(&fields[x as usize]));
311341
}
312342
}
313-
}
314343

315-
// inverse_memory_index holds field indices by increasing memory offset.
316-
// That is, if field 5 has offset 0, the first element of inverse_memory_index is 5.
317-
// We now write field offsets to the corresponding offset slot;
318-
// field 5 with offset 0 puts 0 in offsets[5].
319-
// At the bottom of this function, we invert `inverse_memory_index` to
320-
// produce `memory_index` (see `invert_mapping`).
344+
// Rotate index array to put the largest niche first. Then reverse the ones with larger
345+
// alignment. Since it is already the first amongst the types with the same alignment,
346+
// this will just move some of the potential padding within the structure.
347+
if let (Some(niche_index), StructKind::AlwaysSized) = (largest_niche_index, kind) {
348+
// ZSTs are always first, and the largest niche is not one, so we can unwrap
349+
let first_non_zst = inverse_memory_index
350+
.iter()
351+
.position(|&x| !fields[x as usize].is_zst())
352+
.unwrap();
353+
let non_zsts = &mut inverse_memory_index[first_non_zst..];
354+
let pivot = non_zsts.iter().position(|&x| x == niche_index).unwrap();
355+
non_zsts.rotate_left(pivot);
356+
let pivot = non_zsts.len() - pivot;
357+
non_zsts[pivot..].reverse();
358+
debug_assert_eq!(non_zsts[0], niche_index);
359+
}
360+
}
321361

322362
let mut sized = true;
323363
let mut offsets = vec![Size::ZERO; fields.len()];
324364
let mut offset = Size::ZERO;
325365
let mut largest_niche = None;
326-
let mut largest_niche_available = 0;
327366

328367
if let StructKind::Prefixed(prefix_size, prefix_align) = kind {
329368
let prefix_align =
@@ -354,15 +393,10 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
354393
debug!("univariant offset: {:?} field: {:#?}", offset, field);
355394
offsets[i as usize] = offset;
356395

357-
if !repr.hide_niche() {
358-
if let Some(mut niche) = field.largest_niche.clone() {
359-
let available = niche.available(dl);
360-
if available > largest_niche_available {
361-
largest_niche_available = available;
362-
niche.offset += offset;
363-
largest_niche = Some(niche);
364-
}
365-
}
396+
if largest_niche_index == Some(i) {
397+
let mut niche = field.largest_niche.clone().unwrap();
398+
niche.offset += offset;
399+
largest_niche = Some(niche)
366400
}
367401

368402
offset = offset.checked_add(field.size, dl).ok_or(LayoutError::SizeOverflow(ty))?;
@@ -864,72 +898,123 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
864898
if !def.repr.inhibit_enum_layout_opt() && no_explicit_discriminants {
865899
let mut dataful_variant = None;
866900
let mut niche_variants = VariantIdx::MAX..=VariantIdx::new(0);
901+
let mut max_size = Size::ZERO;
902+
let mut second_max_size = Size::ZERO;
903+
let mut align = dl.aggregate_align;
904+
905+
// The size computations below assume that the padding is minimum.
906+
// This is the case when fields are re-ordered.
907+
let struct_reordering_opt = !def.repr.inhibit_struct_field_reordering_opt();
908+
909+
let mut extend_niche_range = |d| {
910+
niche_variants =
911+
*niche_variants.start().min(&d)..=*niche_variants.end().max(&d);
912+
};
867913

868-
// Find one non-ZST variant.
869-
'variants: for (v, fields) in variants.iter_enumerated() {
914+
// Find the largest and second largest variant.
915+
for (v, fields) in variants.iter_enumerated() {
870916
if absent(fields) {
871-
continue 'variants;
917+
continue;
872918
}
873-
for f in fields {
874-
if !f.is_zst() {
875-
if dataful_variant.is_none() {
876-
dataful_variant = Some(v);
877-
continue 'variants;
878-
} else {
879-
dataful_variant = None;
880-
break 'variants;
881-
}
919+
let mut size = Size::ZERO;
920+
for &f in fields {
921+
align = align.max(f.align);
922+
size += f.size;
923+
}
924+
if size > max_size {
925+
second_max_size = max_size;
926+
max_size = size;
927+
if let Some(d) = dataful_variant {
928+
extend_niche_range(d);
882929
}
930+
dataful_variant = Some(v);
931+
} else if size == max_size {
932+
if let Some(d) = dataful_variant {
933+
extend_niche_range(d);
934+
}
935+
dataful_variant = None;
936+
extend_niche_range(v);
937+
} else {
938+
second_max_size = second_max_size.max(size);
939+
extend_niche_range(v);
883940
}
884-
niche_variants = *niche_variants.start().min(&v)..=v;
885941
}
886942

887943
if niche_variants.start() > niche_variants.end() {
888944
dataful_variant = None;
889945
}
890946

891-
if let Some(i) = dataful_variant {
947+
if let Some(dataful_variant) = dataful_variant {
892948
let count = (niche_variants.end().as_u32()
893949
- niche_variants.start().as_u32()
894950
+ 1) as u128;
895951

896952
// Find the field with the largest niche
897-
let niche_candidate = variants[i]
953+
let niche_candidate = variants[dataful_variant]
898954
.iter()
899955
.enumerate()
900956
.filter_map(|(j, &field)| Some((j, field.largest_niche.as_ref()?)))
901-
.max_by_key(|(_, niche)| niche.available(dl));
957+
.max_by_key(|(_, n)| (n.available(dl), cmp::Reverse(n.offset)))
958+
.and_then(|(field_index, niche)| {
959+
if !struct_reordering_opt && second_max_size > Size::ZERO {
960+
return None;
961+
}
962+
// make sure there is enough room for the other variants
963+
if max_size - (niche.offset + niche.scalar.value.size(dl))
964+
< second_max_size
965+
{
966+
return None;
967+
}
968+
Some((field_index, niche, niche.reserve(self, count)?))
969+
});
902970

903971
if let Some((field_index, niche, (niche_start, niche_scalar))) =
904-
niche_candidate.and_then(|(field_index, niche)| {
905-
Some((field_index, niche, niche.reserve(self, count)?))
906-
})
972+
niche_candidate
907973
{
908-
let mut align = dl.aggregate_align;
974+
let prefix = niche.offset + niche.scalar.value.size(dl);
909975
let st = variants
910976
.iter_enumerated()
911977
.map(|(j, v)| {
912978
let mut st = self.univariant_uninterned(
913979
ty,
914980
v,
915981
&def.repr,
916-
StructKind::AlwaysSized,
982+
if j == dataful_variant || second_max_size == Size::ZERO {
983+
StructKind::AlwaysSized
984+
} else {
985+
StructKind::Prefixed(
986+
prefix,
987+
Align::from_bytes(1).unwrap(),
988+
)
989+
},
917990
)?;
918991
st.variants = Variants::Single { index: j };
919992

920-
align = align.max(st.align);
921-
993+
debug_assert_eq!(align, align.max(st.align));
922994
Ok(st)
923995
})
924996
.collect::<Result<IndexVec<VariantIdx, _>, _>>()?;
925997

926-
let offset = st[i].fields.offset(field_index) + niche.offset;
927-
let size = st[i].size;
998+
let offset = if struct_reordering_opt {
999+
debug_assert_eq!(
1000+
st[dataful_variant].fields.offset(field_index),
1001+
Size::ZERO
1002+
);
1003+
niche.offset
1004+
} else {
1005+
st[dataful_variant].fields.offset(field_index) + niche.offset
1006+
};
1007+
1008+
let size = st[dataful_variant].size.align_to(align.abi);
1009+
debug_assert!(
1010+
!struct_reordering_opt || size == max_size.align_to(align.abi)
1011+
);
1012+
debug_assert!(st.iter().all(|v| v.size <= size));
9281013

9291014
let abi = if st.iter().all(|v| v.abi.is_uninhabited()) {
9301015
Abi::Uninhabited
931-
} else {
932-
match st[i].abi {
1016+
} else if second_max_size == Size::ZERO {
1017+
match st[dataful_variant].abi {
9331018
Abi::Scalar(_) => Abi::Scalar(niche_scalar.clone()),
9341019
Abi::ScalarPair(ref first, ref second) => {
9351020
// We need to use scalar_unit to reset the
@@ -951,6 +1036,8 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
9511036
}
9521037
_ => Abi::Aggregate { sized: true },
9531038
}
1039+
} else {
1040+
Abi::Aggregate { sized: true }
9541041
};
9551042

9561043
let largest_niche =
@@ -960,7 +1047,7 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
9601047
variants: Variants::Multiple {
9611048
tag: niche_scalar,
9621049
tag_encoding: TagEncoding::Niche {
963-
dataful_variant: i,
1050+
dataful_variant,
9641051
niche_variants,
9651052
niche_start,
9661053
},

‎compiler/rustc_mir_build/src/thir/mod.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ crate enum StmtKind<'tcx> {
9797

9898
// `Expr` is used a lot. Make sure it doesn't unintentionally get bigger.
9999
#[cfg(target_arch = "x86_64")]
100-
rustc_data_structures::static_assert_size!(Expr<'_>, 168);
100+
rustc_data_structures::static_assert_size!(Expr<'_>, if cfg!(bootstrap) { 168 } else { 160 });
101101

102102
/// The Thir trait implementor lowers their expressions (`&'tcx H::Expr`)
103103
/// into instances of this `Expr` enum. This lowering can be done

‎src/test/ui/consts/const-eval/const_transmute.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,10 @@ impl Drop for Foo {
3333
}
3434

3535
#[derive(Copy, Clone)]
36+
#[repr(C)]
3637
struct Fat<'a>(&'a Foo, &'static VTable);
3738

39+
#[repr(C)]
3840
struct VTable {
3941
drop: Option<for<'a> fn(&'a mut Foo)>,
4042
size: usize,

‎src/test/ui/print_type_sizes/padding.stdout‎

Lines changed: 14 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,21 @@
1-
print-type-size type: `E1`: 12 bytes, alignment: 4 bytes
2-
print-type-size discriminant: 1 bytes
3-
print-type-size variant `B`: 11 bytes
4-
print-type-size padding: 3 bytes
5-
print-type-size field `.0`: 8 bytes, alignment: 4 bytes
6-
print-type-size variant `A`: 7 bytes
7-
print-type-size field `.1`: 1 bytes
1+
print-type-size type: `E1`: 8 bytes, alignment: 4 bytes
2+
print-type-size variant `A`: 8 bytes
3+
print-type-size padding: 1 bytes
4+
print-type-size field `.1`: 1 bytes, alignment: 1 bytes
85
print-type-size padding: 2 bytes
96
print-type-size field `.0`: 4 bytes, alignment: 4 bytes
10-
print-type-size type: `E2`: 12 bytes, alignment: 4 bytes
11-
print-type-size discriminant: 1 bytes
12-
print-type-size variant `B`: 11 bytes
13-
print-type-size padding: 3 bytes
14-
print-type-size field `.0`: 8 bytes, alignment: 4 bytes
15-
print-type-size variant `A`: 7 bytes
16-
print-type-size field `.0`: 1 bytes
7+
print-type-size variant `B`: 8 bytes
8+
print-type-size field `.0`: 8 bytes
9+
print-type-size type: `E2`: 8 bytes, alignment: 4 bytes
10+
print-type-size variant `A`: 8 bytes
11+
print-type-size padding: 1 bytes
12+
print-type-size field `.0`: 1 bytes, alignment: 1 bytes
1713
print-type-size padding: 2 bytes
1814
print-type-size field `.1`: 4 bytes, alignment: 4 bytes
15+
print-type-size variant `B`: 8 bytes
16+
print-type-size field `.0`: 8 bytes
1917
print-type-size type: `S`: 8 bytes, alignment: 4 bytes
20-
print-type-size field `.g`: 4 bytes
2118
print-type-size field `.a`: 1 bytes
2219
print-type-size field `.b`: 1 bytes
23-
print-type-size end padding: 2 bytes
20+
print-type-size padding: 2 bytes
21+
print-type-size field `.g`: 4 bytes, alignment: 4 bytes

0 commit comments

Comments
 (0)