Use niche-filling optimization even when multiple variants have data.

Fixes #46213

Author: Michael Benfield

compiler/rustc_ast/src/ast.rs

@@ -3075,7 +3075,8 @@ mod size_asserts {
     static_assert_size!(Block, 48);
     static_assert_size!(Expr, 104);
     static_assert_size!(ExprKind, 72);
-    static_assert_size!(Fn, 192);
+    #[cfg(not(bootstrap))]
+    static_assert_size!(Fn, 184);
     static_assert_size!(ForeignItem, 96);
     static_assert_size!(ForeignItemKind, 24);
     static_assert_size!(GenericArg, 24);

compiler/rustc_const_eval/src/interpret/operand.rs

@@ -780,13 +780,13 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
 }
 
 // Some nodes are used a lot. Make sure they don't unintentionally get bigger.
-#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64", not(bootstrap)))]
 mod size_asserts {
     use super::*;
     use rustc_data_structures::static_assert_size;
     // These are in alphabetical order, which is easy to maintain.
-    static_assert_size!(Immediate, 56);
-    static_assert_size!(ImmTy<'_>, 72);
-    static_assert_size!(Operand, 64);
-    static_assert_size!(OpTy<'_>, 88);
+    static_assert_size!(Immediate, 48);
+    static_assert_size!(ImmTy<'_>, 64);
+    static_assert_size!(Operand, 56);
+    static_assert_size!(OpTy<'_>, 80);
 }

compiler/rustc_const_eval/src/interpret/place.rs

@@ -890,6 +890,8 @@ mod size_asserts {
     static_assert_size!(MemPlaceMeta, 24);
     static_assert_size!(MemPlace, 40);
     static_assert_size!(MPlaceTy<'_>, 64);
-    static_assert_size!(Place, 48);
-    static_assert_size!(PlaceTy<'_>, 72);
+    #[cfg(not(bootstrap))]
+    static_assert_size!(Place, 40);
+    #[cfg(not(bootstrap))]
+    static_assert_size!(PlaceTy<'_>, 64);
 }

compiler/rustc_data_structures/src/obligation_forest/mod.rs

@@ -117,6 +117,10 @@ pub trait ObligationProcessor {
 }
 
 /// The result type used by `process_obligation`.
+// `repr(C)` to inhibit the niche filling optimization. Otherwise, the `match` appearing
+// in `process_obligations` is significantly slower, which can substantially affect
+// benchmarks like `rustc-perf`'s inflate and keccak.
+#[repr(C)]
 #[derive(Debug)]
 pub enum ProcessResult<O, E> {
     Unchanged,

compiler/rustc_errors/src/lib.rs

@@ -69,8 +69,8 @@ pub type PResult<'a, T> = Result<T, DiagnosticBuilder<'a, ErrorGuaranteed>>;
 // (See also the comment on `DiagnosticBuilder`'s `diagnostic` field.)
 #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 rustc_data_structures::static_assert_size!(PResult<'_, ()>, 16);
-#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
-rustc_data_structures::static_assert_size!(PResult<'_, bool>, 24);
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64", not(bootstrap)))]
+rustc_data_structures::static_assert_size!(PResult<'_, bool>, 16);
 
 #[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, Encodable, Decodable)]
 pub enum SuggestionStyle {

compiler/rustc_hir/src/hir.rs

@@ -3473,12 +3473,15 @@ mod size_asserts {
     static_assert_size!(FnDecl<'_>, 40);
     static_assert_size!(ForeignItem<'_>, 72);
     static_assert_size!(ForeignItemKind<'_>, 40);
-    static_assert_size!(GenericArg<'_>, 40);
+    #[cfg(not(bootstrap))]
+    static_assert_size!(GenericArg<'_>, 32);
     static_assert_size!(GenericBound<'_>, 48);
     static_assert_size!(Generics<'_>, 56);
     static_assert_size!(Impl<'_>, 80);
-    static_assert_size!(ImplItem<'_>, 88);
-    static_assert_size!(ImplItemKind<'_>, 40);
+    #[cfg(not(bootstrap))]
+    static_assert_size!(ImplItem<'_>, 80);
+    #[cfg(not(bootstrap))]
+    static_assert_size!(ImplItemKind<'_>, 32);
     static_assert_size!(Item<'_>, 80);
     static_assert_size!(ItemKind<'_>, 48);
     static_assert_size!(Local<'_>, 64);
@@ -3490,8 +3493,10 @@ mod size_asserts {
     static_assert_size!(QPath<'_>, 24);
     static_assert_size!(Stmt<'_>, 32);
     static_assert_size!(StmtKind<'_>, 16);
-    static_assert_size!(TraitItem<'_>, 96);
-    static_assert_size!(TraitItemKind<'_>, 56);
+    #[cfg(not(bootstrap))]
+    static_assert_size!(TraitItem<'static>, 88);
+    #[cfg(not(bootstrap))]
+    static_assert_size!(TraitItemKind<'_>, 48);
     static_assert_size!(Ty<'_>, 72);
     static_assert_size!(TyKind<'_>, 56);
 }

compiler/rustc_middle/src/mir/syntax.rs

@@ -1231,7 +1231,8 @@ pub enum BinOp {
 mod size_asserts {
     use super::*;
     // These are in alphabetical order, which is easy to maintain.
-    static_assert_size!(AggregateKind<'_>, 48);
+    #[cfg(not(bootstrap))]
+    static_assert_size!(AggregateKind<'_>, 40);
     static_assert_size!(Operand<'_>, 24);
     static_assert_size!(Place<'_>, 16);
     static_assert_size!(PlaceElem<'_>, 24);

compiler/rustc_middle/src/thir.rs

@@ -825,8 +825,12 @@ mod size_asserts {
     static_assert_size!(Block, 56);
     static_assert_size!(Expr<'_>, 64);
     static_assert_size!(ExprKind<'_>, 40);
-    static_assert_size!(Pat<'_>, 72);
-    static_assert_size!(PatKind<'_>, 56);
-    static_assert_size!(Stmt<'_>, 56);
-    static_assert_size!(StmtKind<'_>, 48);
+    #[cfg(not(bootstrap))]
+    static_assert_size!(Pat<'_>, 64);
+    #[cfg(not(bootstrap))]
+    static_assert_size!(PatKind<'_>, 48);
+    #[cfg(not(bootstrap))]
+    static_assert_size!(Stmt<'_>, 48);
+    #[cfg(not(bootstrap))]
+    static_assert_size!(StmtKind<'_>, 40);
 }