inhibit layout randomization for Box

Author: the8472

compiler/rustc_abi/src/layout.rs

@@ -968,8 +968,8 @@ fn univariant<
     let mut align = if pack.is_some() { dl.i8_align } else { dl.aggregate_align };
     let mut max_repr_align = repr.align;
     let mut inverse_memory_index: IndexVec<u32, FieldIdx> = fields.indices().collect();
-    let optimize = !repr.inhibit_struct_field_reordering();
-    if optimize && fields.len() > 1 {
+    let optimize_field_order = !repr.inhibit_struct_field_reordering();
+    if optimize_field_order && fields.len() > 1 {
         let end = if let StructKind::MaybeUnsized = kind { fields.len() - 1 } else { fields.len() };
         let optimizing = &mut inverse_memory_index.raw[..end];
         let fields_excluding_tail = &fields.raw[..end];
@@ -1176,7 +1176,7 @@ fn univariant<
     // If field 5 has offset 0, offsets[0] is 5, and memory_index[5] should be 0.
     // Field 5 would be the first element, so memory_index is i:
     // Note: if we didn't optimize, it's already right.
-    let memory_index = if optimize {
+    let memory_index = if optimize_field_order {
         inverse_memory_index.invert_bijective_mapping()
     } else {
         debug_assert!(inverse_memory_index.iter().copied().eq(fields.indices()));
@@ -1189,6 +1189,9 @@ fn univariant<
     }
     let mut layout_of_single_non_zst_field = None;
     let mut abi = Abi::Aggregate { sized };
+
+    let optimize_abi = !repr.inhibit_newtype_abi_optimization();
+
     // Try to make this a Scalar/ScalarPair.
     if sized && size.bytes() > 0 {
         // We skip *all* ZST here and later check if we are good in terms of alignment.
@@ -1205,7 +1208,7 @@ fn univariant<
                     match field.abi {
                         // For plain scalars, or vectors of them, we can't unpack
                         // newtypes for `#[repr(C)]`, as that affects C ABIs.
-                        Abi::Scalar(_) | Abi::Vector { .. } if optimize => {
+                        Abi::Scalar(_) | Abi::Vector { .. } if optimize_abi => {
                             abi = field.abi;
                         }
                         // But scalar pairs are Rust-specific and get

compiler/rustc_abi/src/lib.rs

@@ -43,14 +43,17 @@ bitflags! {
         const IS_SIMD            = 1 << 1;
         const IS_TRANSPARENT     = 1 << 2;
         // Internal only for now. If true, don't reorder fields.
+        // On its own it does not prevent ABI optimizations.
         const IS_LINEAR          = 1 << 3;
-        // If true, the type's layout can be randomized using
-        // the seed stored in `ReprOptions.field_shuffle_seed`
+        // If true, the type's crate has opted into layout randomization.
+        // Other flags can still inhibit reordering and thus randomization.
+        // The seed stored in `ReprOptions.field_shuffle_seed`.
         const RANDOMIZE_LAYOUT   = 1 << 4;
         // Any of these flags being set prevent field reordering optimisation.
-        const IS_UNOPTIMISABLE   = ReprFlags::IS_C.bits()
+        const FIELD_ORDER_UNOPTIMIZABLE   = ReprFlags::IS_C.bits()
                                  | ReprFlags::IS_SIMD.bits()
                                  | ReprFlags::IS_LINEAR.bits();
+        const ABI_UNOPTIMIZABLE = ReprFlags::IS_C.bits() | ReprFlags::IS_SIMD.bits();
     }
 }
 
@@ -139,10 +142,14 @@ impl ReprOptions {
         self.c() || self.int.is_some()
     }
 
+    pub fn inhibit_newtype_abi_optimization(&self) -> bool {
+        self.flags.intersects(ReprFlags::ABI_UNOPTIMIZABLE)
+    }
+
     /// Returns `true` if this `#[repr()]` guarantees a fixed field order,
     /// e.g. `repr(C)` or `repr(<int>)`.
     pub fn inhibit_struct_field_reordering(&self) -> bool {
-        self.flags.intersects(ReprFlags::IS_UNOPTIMISABLE) || self.int.is_some()
+        self.flags.intersects(ReprFlags::FIELD_ORDER_UNOPTIMIZABLE) || self.int.is_some()
     }
 
     /// Returns `true` if this type is valid for reordering and `-Z randomize-layout`

compiler/rustc_middle/src/ty/mod.rs

@@ -35,6 +35,7 @@ use rustc_data_structures::tagged_ptr::CopyTaggedPtr;
 use rustc_errors::{Diag, ErrorGuaranteed, StashKey};
 use rustc_hir::def::{CtorKind, CtorOf, DefKind, DocLinkResMap, LifetimeRes, Res};
 use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, LocalDefId, LocalDefIdMap};
+use rustc_hir::LangItem;
 use rustc_index::IndexVec;
 use rustc_macros::{
     extension, Decodable, Encodable, HashStable, TyDecodable, TyEncodable, TypeFoldable,
@@ -1570,8 +1571,15 @@ impl<'tcx> TyCtxt<'tcx> {
             flags.insert(ReprFlags::RANDOMIZE_LAYOUT);
         }
 
+        // box is special, on the one hand the compiler assumes an ordered layout, with the pointer
+        // always at offset zero. On the other hand we want scalar abi optimizations.
+        let is_box = self.is_lang_item(did.to_def_id(), LangItem::OwnedBox);
+
         // This is here instead of layout because the choice must make it into metadata.
-        if !self.consider_optimizing(|| format!("Reorder fields of {:?}", self.def_path_str(did))) {
+        if is_box
+            || !self
+                .consider_optimizing(|| format!("Reorder fields of {:?}", self.def_path_str(did)))
+        {
             flags.insert(ReprFlags::IS_LINEAR);
         }