clean up struct layout code

Author: Lukas Markeffsky

compiler/rustc_abi/src/layout.rs

@@ -134,7 +134,7 @@ pub trait LayoutCalculator {
         scalar_valid_range: (Bound<u128>, Bound<u128>),
         discr_range_of_repr: impl Fn(i128, i128) -> (Integer, bool),
         discriminants: impl Iterator<Item = (VariantIdx, i128)>,
-        niche_optimize_enum: bool,
+        dont_niche_optimize_enum: bool,
         always_sized: bool,
     ) -> Option<LayoutS> {
         let dl = self.current_data_layout();
@@ -183,10 +183,10 @@ pub trait LayoutCalculator {
             // (Typechecking will reject discriminant-sizing attrs.)
 
             let v = present_first;
-            let kind = if is_enum || variants[v].is_empty() {
+            let kind = if is_enum || variants[v].is_empty() || always_sized {
                 StructKind::AlwaysSized
             } else {
-                if !always_sized { StructKind::MaybeUnsized } else { StructKind::AlwaysSized }
+                StructKind::MaybeUnsized
             };
 
             let mut st = self.univariant(dl, &variants[v], repr, kind)?;
@@ -280,7 +280,7 @@ pub trait LayoutCalculator {
         }
 
         let calculate_niche_filling_layout = || -> Option<TmpLayout> {
-            if niche_optimize_enum {
+            if dont_niche_optimize_enum {
                 return None;
             }
 

compiler/rustc_ty_utils/src/layout.rs

@@ -463,38 +463,43 @@ fn layout_of_uncached<'tcx>(
                 ));
             }
 
-            tcx.mk_layout(
-                cx.layout_of_struct_or_enum(
-                    &def.repr(),
-                    &variants,
-                    def.is_enum(),
-                    def.is_unsafe_cell(),
-                    tcx.layout_scalar_valid_range(def.did()),
-                    |min, max| Integer::repr_discr(tcx, ty, &def.repr(), min, max),
-                    def.is_enum()
-                        .then(|| def.discriminants(tcx).map(|(v, d)| (v, d.val as i128)))
-                        .into_iter()
-                        .flatten(),
-                    def.repr().inhibit_enum_layout_opt()
-                        || def
-                            .variants()
-                            .iter_enumerated()
-                            .any(|(i, v)| v.discr != ty::VariantDiscr::Relative(i.as_u32())),
-                    {
-                        let param_env = tcx.param_env(def.did());
-                        def.is_struct()
-                            && match def.variants().iter().next().and_then(|x| x.fields.raw.last())
-                            {
-                                Some(last_field) => tcx
-                                    .type_of(last_field.did)
-                                    .subst_identity()
-                                    .is_sized(tcx, param_env),
-                                None => false,
-                            }
-                    },
-                )
-                .ok_or_else(|| error(cx, LayoutError::SizeOverflow(ty)))?,
-            )
+            let get_discriminant_type =
+                |min, max| Integer::repr_discr(tcx, ty, &def.repr(), min, max);
+
+            let discriminants_iter = || {
+                def.is_enum()
+                    .then(|| def.discriminants(tcx).map(|(v, d)| (v, d.val as i128)))
+                    .into_iter()
+                    .flatten()
+            };
+
+            let dont_niche_optimize_enum = def.repr().inhibit_enum_layout_opt()
+                || def
+                    .variants()
+                    .iter_enumerated()
+                    .any(|(i, v)| v.discr != ty::VariantDiscr::Relative(i.as_u32()));
+
+            let maybe_unsized = def.is_struct()
+                && def.non_enum_variant().fields.raw.last().is_some_and(|last_field| {
+                    let param_env = tcx.param_env(def.did());
+                    !tcx.type_of(last_field.did).subst_identity().is_sized(tcx, param_env)
+                });
+
+            let Some(layout) = cx.layout_of_struct_or_enum(
+                &def.repr(),
+                &variants,
+                def.is_enum(),
+                def.is_unsafe_cell(),
+                tcx.layout_scalar_valid_range(def.did()),
+                get_discriminant_type,
+                discriminants_iter(),
+                dont_niche_optimize_enum,
+                !maybe_unsized,
+            ) else {
+                return Err(error(cx, LayoutError::SizeOverflow(ty)));
+            };
+
+            tcx.mk_layout(layout)
         }
 
         // Types with no meaningful known layout.