Skip to content

Commit d58273b

Browse files
ogoffartogoffart
committed
Use the niche optimisation if other enum variant are small enough
Reduce the size of enums which have one bigger variant with a niche
1 parent 059b19a commit d58273b

File tree

6 files changed

+137
-48
lines changed

6 files changed

+137
-48
lines changed

‎src/librustc_middle/ty/layout.rs‎

Lines changed: 101 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -898,56 +898,118 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
898898
if !def.repr.inhibit_enum_layout_opt() && no_explicit_discriminants {
899899
let mut dataful_variant = None;
900900
let mut niche_variants = VariantIdx::MAX..=VariantIdx::new(0);
901+
let mut max_size = Size::ZERO;
902+
let mut second_max_size = Size::ZERO;
903+
904+
// The size computations below assume that the padding is minimum.
905+
// This is the case when fields are re-ordered.
906+
let struct_reordering_opt = !def.repr.inhibit_struct_field_reordering_opt();
907+
if struct_reordering_opt {
908+
let mut extend_niche_range = |d| {
909+
niche_variants =
910+
*niche_variants.start().min(&d)..=*niche_variants.end().max(&d);
911+
};
912+
let mut align = dl.aggregate_align;
901913

902-
// Find one non-ZST variant.
903-
'variants: for (v, fields) in variants.iter_enumerated() {
904-
if absent(fields) {
905-
continue 'variants;
914+
// Find the largest and second largest variant.
915+
for (v, fields) in variants.iter_enumerated() {
916+
if absent(fields) {
917+
continue;
918+
}
919+
let mut size = Size::ZERO;
920+
for &f in fields {
921+
align = align.max(f.align);
922+
size += f.size;
923+
}
924+
if size > max_size {
925+
second_max_size = max_size;
926+
max_size = size;
927+
if let Some(d) = dataful_variant {
928+
extend_niche_range(d);
929+
}
930+
dataful_variant = Some(v);
931+
} else if size == max_size {
932+
if let Some(d) = dataful_variant {
933+
extend_niche_range(d);
934+
}
935+
dataful_variant = None;
936+
extend_niche_range(v);
937+
} else {
938+
second_max_size = second_max_size.max(size);
939+
extend_niche_range(v);
940+
}
906941
}
907-
for f in fields {
908-
if !f.is_zst() {
909-
if dataful_variant.is_none() {
910-
dataful_variant = Some(v);
911-
continue 'variants;
912-
} else {
913-
dataful_variant = None;
914-
break 'variants;
942+
if !max_size.is_aligned(align.abi) {
943+
// Don't perform niche optimisation if there is padding anyway
944+
dataful_variant = None;
945+
}
946+
} else {
947+
// Find one non-ZST variant.
948+
'variants: for (v, fields) in variants.iter_enumerated() {
949+
if absent(fields) {
950+
continue 'variants;
951+
}
952+
for f in fields {
953+
if !f.is_zst() {
954+
if dataful_variant.is_none() {
955+
dataful_variant = Some(v);
956+
continue 'variants;
957+
} else {
958+
dataful_variant = None;
959+
break 'variants;
960+
}
915961
}
962+
niche_variants = *niche_variants.start().min(&v)..=v
916963
}
917964
}
918-
niche_variants = *niche_variants.start().min(&v)..=v;
919965
}
920966

921967
if niche_variants.start() > niche_variants.end() {
922968
dataful_variant = None;
923969
}
924970

925-
if let Some(i) = dataful_variant {
971+
if let Some(dataful_variant) = dataful_variant {
926972
let count = (niche_variants.end().as_u32()
927973
- niche_variants.start().as_u32()
928974
+ 1) as u128;
929975

930976
// Find the field with the largest niche
931-
let niche_candidate = variants[i]
977+
let niche_candidate = variants[dataful_variant]
932978
.iter()
933979
.enumerate()
934980
.filter_map(|(j, &field)| Some((j, field.largest_niche.as_ref()?)))
935-
.max_by_key(|(_, niche)| niche.available(dl));
981+
.max_by_key(|(_, n)| (n.available(dl), cmp::Reverse(n.offset)))
982+
.and_then(|(field_index, niche)| {
983+
// make sure there is enough room for the other variants
984+
if struct_reordering_opt
985+
&& max_size - (niche.offset + niche.scalar.value.size(dl))
986+
< second_max_size
987+
{
988+
return None;
989+
}
990+
Some((field_index, niche, niche.reserve(self, count)?))
991+
});
936992

937993
if let Some((field_index, niche, (niche_start, niche_scalar))) =
938-
niche_candidate.and_then(|(field_index, niche)| {
939-
Some((field_index, niche, niche.reserve(self, count)?))
940-
})
994+
niche_candidate
941995
{
942996
let mut align = dl.aggregate_align;
997+
let prefix = niche.offset + niche.scalar.value.size(dl);
943998
let st = variants
944999
.iter_enumerated()
9451000
.map(|(j, v)| {
9461001
let mut st = self.univariant_uninterned(
9471002
ty,
9481003
v,
9491004
&def.repr,
950-
StructKind::AlwaysSized,
1005+
if j == dataful_variant || second_max_size == Size::ZERO {
1006+
StructKind::AlwaysSized
1007+
} else {
1008+
StructKind::Prefixed(
1009+
prefix,
1010+
Align::from_bytes(1).unwrap(),
1011+
)
1012+
},
9511013
)?;
9521014
st.variants = Variants::Single { index: j };
9531015

@@ -957,21 +1019,30 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
9571019
})
9581020
.collect::<Result<IndexVec<VariantIdx, _>, _>>()?;
9591021

960-
let offset = st[i].fields.offset(field_index) + niche.offset;
961-
let size = st[i].size;
1022+
let niche_offset = if struct_reordering_opt {
1023+
debug_assert_eq!(
1024+
st[dataful_variant].fields.offset(field_index),
1025+
Size::ZERO
1026+
);
1027+
niche.offset
1028+
} else {
1029+
st[dataful_variant].fields.offset(field_index) + niche.offset
1030+
};
1031+
let size = st[dataful_variant].size;
1032+
debug_assert!(st.iter().all(|v| { v.size <= size }));
9621033

9631034
let abi = if st.iter().all(|v| v.abi.is_uninhabited()) {
9641035
Abi::Uninhabited
965-
} else {
966-
match st[i].abi {
1036+
} else if second_max_size == Size::ZERO {
1037+
match st[dataful_variant].abi {
9671038
Abi::Scalar(_) => Abi::Scalar(niche_scalar.clone()),
9681039
Abi::ScalarPair(ref first, ref second) => {
9691040
// We need to use scalar_unit to reset the
9701041
// valid range to the maximal one for that
9711042
// primitive, because only the niche is
9721043
// guaranteed to be initialised, not the
9731044
// other primitive.
974-
if offset.bytes() == 0 {
1045+
if niche_offset.bytes() == 0 {
9751046
Abi::ScalarPair(
9761047
niche_scalar.clone(),
9771048
scalar_unit(second.value),
@@ -985,24 +1056,26 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
9851056
}
9861057
_ => Abi::Aggregate { sized: true },
9871058
}
1059+
} else {
1060+
Abi::Aggregate { sized: true }
9881061
};
9891062

9901063
let largest_niche =
991-
Niche::from_scalar(dl, offset, niche_scalar.clone());
1064+
Niche::from_scalar(dl, niche_offset, niche_scalar.clone());
9921065

9931066
return Ok(tcx.intern_layout(Layout {
9941067
variants: Variants::Multiple {
9951068
discr: niche_scalar,
9961069
discr_kind: DiscriminantKind::Niche {
997-
dataful_variant: i,
1070+
dataful_variant,
9981071
niche_variants,
9991072
niche_start,
10001073
},
10011074
discr_index: 0,
10021075
variants: st,
10031076
},
10041077
fields: FieldsShape::Arbitrary {
1005-
offsets: vec![offset],
1078+
offsets: vec![niche_offset],
10061079
memory_index: vec![0],
10071080
},
10081081
abi,

‎src/librustc_middle/ty/sty.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2204,7 +2204,7 @@ pub struct Const<'tcx> {
22042204
}
22052205

22062206
#[cfg(target_arch = "x86_64")]
2207-
static_assert_size!(Const<'_>, 48);
2207+
static_assert_size!(Const<'_>, if cfg!(bootstrap) { 48 } else { 40 });
22082208

22092209
impl<'tcx> Const<'tcx> {
22102210
/// Literals and const generic parameters are eagerly converted to a constant, everything else
@@ -2432,7 +2432,7 @@ pub enum ConstKind<'tcx> {
24322432
}
24332433

24342434
#[cfg(target_arch = "x86_64")]
2435-
static_assert_size!(ConstKind<'_>, 40);
2435+
static_assert_size!(ConstKind<'_>, if cfg!(bootstrap) { 40 } else { 32 });
24362436

24372437
impl<'tcx> ConstKind<'tcx> {
24382438
#[inline]

‎src/librustc_mir_build/hair/mod.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ crate enum StmtKind<'tcx> {
9595

9696
// `Expr` is used a lot. Make sure it doesn't unintentionally get bigger.
9797
#[cfg(target_arch = "x86_64")]
98-
rustc_data_structures::static_assert_size!(Expr<'_>, 168);
98+
rustc_data_structures::static_assert_size!(Expr<'_>, if cfg!(bootstrap) { 168 } else { 160 });
9999

100100
/// The Hair trait implementor lowers their expressions (`&'tcx H::Expr`)
101101
/// into instances of this `Expr` enum. This lowering can be done

‎src/test/ui/print_type_sizes/niche-filling.stdout‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,12 @@ print-type-size variant `Some`: 12 bytes
88
print-type-size field `.0`: 12 bytes
99
print-type-size variant `None`: 0 bytes
1010
print-type-size type: `EmbeddedDiscr`: 8 bytes, alignment: 4 bytes
11+
print-type-size discriminant: 1 bytes
1112
print-type-size variant `Record`: 7 bytes
12-
print-type-size field `.val`: 4 bytes
13-
print-type-size field `.post`: 2 bytes
1413
print-type-size field `.pre`: 1 bytes
14+
print-type-size field `.post`: 2 bytes
15+
print-type-size field `.val`: 4 bytes
1516
print-type-size variant `None`: 0 bytes
16-
print-type-size end padding: 1 bytes
1717
print-type-size type: `MyOption<Union1<std::num::NonZeroU32>>`: 8 bytes, alignment: 4 bytes
1818
print-type-size discriminant: 4 bytes
1919
print-type-size variant `Some`: 4 bytes

‎src/test/ui/print_type_sizes/padding.stdout‎

Lines changed: 12 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,19 @@
1-
print-type-size type: `E1`: 12 bytes, alignment: 4 bytes
2-
print-type-size discriminant: 1 bytes
3-
print-type-size variant `B`: 11 bytes
4-
print-type-size padding: 3 bytes
5-
print-type-size field `.0`: 8 bytes, alignment: 4 bytes
6-
print-type-size variant `A`: 7 bytes
7-
print-type-size field `.1`: 1 bytes
1+
print-type-size type: `E1`: 8 bytes, alignment: 4 bytes
2+
print-type-size variant `A`: 8 bytes
3+
print-type-size padding: 1 bytes
4+
print-type-size field `.1`: 1 bytes, alignment: 1 bytes
85
print-type-size padding: 2 bytes
96
print-type-size field `.0`: 4 bytes, alignment: 4 bytes
10-
print-type-size type: `E2`: 12 bytes, alignment: 4 bytes
11-
print-type-size discriminant: 1 bytes
12-
print-type-size variant `B`: 11 bytes
13-
print-type-size padding: 3 bytes
14-
print-type-size field `.0`: 8 bytes, alignment: 4 bytes
15-
print-type-size variant `A`: 7 bytes
16-
print-type-size field `.0`: 1 bytes
7+
print-type-size variant `B`: 8 bytes
8+
print-type-size field `.0`: 8 bytes
9+
print-type-size type: `E2`: 8 bytes, alignment: 4 bytes
10+
print-type-size variant `A`: 8 bytes
11+
print-type-size padding: 1 bytes
12+
print-type-size field `.0`: 1 bytes, alignment: 1 bytes
1713
print-type-size padding: 2 bytes
1814
print-type-size field `.1`: 4 bytes, alignment: 4 bytes
15+
print-type-size variant `B`: 8 bytes
16+
print-type-size field `.0`: 8 bytes
1917
print-type-size type: `S`: 8 bytes, alignment: 4 bytes
2018
print-type-size field `.b`: 1 bytes
2119
print-type-size field `.a`: 1 bytes

‎src/test/ui/type-sizes.rs‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,16 @@ enum Option2<A, B> {
102102
None
103103
}
104104

105+
struct BoolInTheMiddle(std::num::NonZeroU16, bool, u8);
106+
107+
enum NicheWithData {
108+
A,
109+
B([u16; 5]),
110+
Largest { a1: u32, a2: BoolInTheMiddle, a3: u32 },
111+
C,
112+
D(u32, u32),
113+
}
114+
105115
pub fn main() {
106116
assert_eq!(size_of::<u8>(), 1 as usize);
107117
assert_eq!(size_of::<u32>(), 4 as usize);
@@ -149,4 +159,12 @@ pub fn main() {
149159
struct S1{ a: u16, b: std::num::NonZeroU16, c: u16, d: u8, e: u32, f: u64, g:[u8;2] }
150160
assert_eq!(size_of::<S1>(), 24);
151161
assert_eq!(size_of::<Option<S1>>(), 24);
162+
163+
assert_eq!(size_of::<NicheWithData>(), 12);
164+
assert_eq!(size_of::<Option<NicheWithData>>(), 12);
165+
assert_eq!(size_of::<Option<Option<NicheWithData>>>(), 12);
166+
assert_eq!(
167+
size_of::<Option<Option2<&(), Option<NicheWithData>>>>(),
168+
size_of::<(&(), NicheWithData)>()
169+
);
152170
}

0 commit comments

Comments
 (0)