Foo<T> != Foo<U> under layout randomization

previously field ordering was using the same seed for all instances of Foo,
now we pass seed values through the layout tree so that not only
the struct itself affects layout but also its fields

Author: the8472

compiler/rustc_abi/src/layout.rs

@@ -119,6 +119,8 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
             .chain(Niche::from_scalar(dl, Size::ZERO, a))
             .max_by_key(|niche| niche.available(dl));
 
+        let combined_seed = a.size(&self.cx).bytes().wrapping_add(b.size(&self.cx).bytes());
+
         LayoutData {
             variants: Variants::Single { index: VariantIdx::new(0) },
             fields: FieldsShape::Arbitrary {
@@ -131,6 +133,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
             size,
             max_repr_align: None,
             unadjusted_abi_align: align.abi,
+            randomization_seed: combined_seed,
         }
     }
 
@@ -222,6 +225,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
             size: Size::ZERO,
             max_repr_align: None,
             unadjusted_abi_align: dl.i8_align.abi,
+            randomization_seed: 0,
         }
     }
 
@@ -384,6 +388,11 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
             return Err(LayoutCalculatorError::EmptyUnion);
         };
 
+        let combined_seed = only_variant
+            .iter()
+            .map(|v| v.randomization_seed)
+            .fold(repr.field_shuffle_seed, |acc, seed| acc.wrapping_add(seed));
+
         Ok(LayoutData {
             variants: Variants::Single { index: only_variant_idx },
             fields: FieldsShape::Union(union_field_count),
@@ -393,6 +402,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
             size: size.align_to(align.abi),
             max_repr_align,
             unadjusted_abi_align,
+            randomization_seed: combined_seed,
         })
     }
 
@@ -649,6 +659,11 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
                 BackendRepr::Memory { sized: true }
             };
 
+            let combined_seed = variant_layouts
+                .iter()
+                .map(|v| v.randomization_seed)
+                .fold(repr.field_shuffle_seed, |acc, seed| acc.wrapping_add(seed));
+
             let layout = LayoutData {
                 variants: Variants::Multiple {
                     tag: niche_scalar,
@@ -670,6 +685,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
                 align,
                 max_repr_align,
                 unadjusted_abi_align,
+                randomization_seed: combined_seed,
             };
 
             Some(TmpLayout { layout, variants: variant_layouts })
@@ -960,6 +976,11 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
 
         let largest_niche = Niche::from_scalar(dl, Size::ZERO, tag);
 
+        let combined_seed = layout_variants
+            .iter()
+            .map(|v| v.randomization_seed)
+            .fold(repr.field_shuffle_seed, |acc, seed| acc.wrapping_add(seed));
+
         let tagged_layout = LayoutData {
             variants: Variants::Multiple {
                 tag,
@@ -977,6 +998,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
             size,
             max_repr_align,
             unadjusted_abi_align,
+            randomization_seed: combined_seed,
         };
 
         let tagged_layout = TmpLayout { layout: tagged_layout, variants: layout_variants };
@@ -1028,6 +1050,8 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
         let mut align = if pack.is_some() { dl.i8_align } else { dl.aggregate_align };
         let mut max_repr_align = repr.align;
         let mut inverse_memory_index: IndexVec<u32, FieldIdx> = fields.indices().collect();
+        let field_seed =
+            fields.raw.iter().fold(0u64, |acc, f| acc.wrapping_add(f.randomization_seed));
         let optimize_field_order = !repr.inhibit_struct_field_reordering();
         if optimize_field_order && fields.len() > 1 {
             let end =
@@ -1043,10 +1067,12 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
                 {
                     use rand::SeedableRng;
                     use rand::seq::SliceRandom;
+                    //let field_entropy = fields_excluding_tail.iter().map(|f| f.).sum();
                     // `ReprOptions.field_shuffle_seed` is a deterministic seed we can use to randomize field
                     // ordering.
-                    let mut rng =
-                        rand_xoshiro::Xoshiro128StarStar::seed_from_u64(repr.field_shuffle_seed);
+                    let mut rng = rand_xoshiro::Xoshiro128StarStar::seed_from_u64(
+                        field_seed.wrapping_add(repr.field_shuffle_seed),
+                    );
 
                     // Shuffle the ordering of the fields.
                     optimizing.shuffle(&mut rng);
@@ -1343,6 +1369,13 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
             unadjusted_abi_align
         };
 
+        // a transparent struct only has a single field, so its seed should be the same as the one we pass forward
+        let seed = if repr.transparent() {
+            field_seed
+        } else {
+            field_seed.wrapping_add(repr.field_shuffle_seed)
+        };
+
         Ok(LayoutData {
             variants: Variants::Single { index: VariantIdx::new(0) },
             fields: FieldsShape::Arbitrary { offsets, memory_index },
@@ -1352,6 +1385,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
             size,
             max_repr_align,
             unadjusted_abi_align,
+            randomization_seed: seed,
         })
     }
 

compiler/rustc_abi/src/lib.rs

@@ -1667,6 +1667,9 @@ pub struct LayoutData<FieldIdx: Idx, VariantIdx: Idx> {
     /// Only used on aarch64-linux, where the argument passing ABI ignores the requested alignment
     /// in some cases.
     pub unadjusted_abi_align: Align,
+
+    /// The randomization seed based on this type's own repr and its fields'.
+    pub randomization_seed: u64,
 }
 
 impl<FieldIdx: Idx, VariantIdx: Idx> LayoutData<FieldIdx, VariantIdx> {
@@ -1687,6 +1690,14 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutData<FieldIdx, VariantIdx> {
         let largest_niche = Niche::from_scalar(cx, Size::ZERO, scalar);
         let size = scalar.size(cx);
         let align = scalar.align(cx);
+
+        let seed_extra = match scalar.primitive() {
+            Primitive::Int(_, true) => 1,
+            Primitive::Int(_, false) => 2,
+            Primitive::Float(_) => 3,
+            Primitive::Pointer(_) => 4,
+        };
+
         LayoutData {
             variants: Variants::Single { index: VariantIdx::new(0) },
             fields: FieldsShape::Primitive,
@@ -1696,6 +1707,7 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutData<FieldIdx, VariantIdx> {
             align,
             max_repr_align: None,
             unadjusted_abi_align: align.abi,
+            randomization_seed: size.bytes().wrapping_add(seed_extra << 32),
         }
     }
 }
@@ -1718,6 +1730,7 @@ where
             variants,
             max_repr_align,
             unadjusted_abi_align,
+            ref randomization_seed,
         } = self;
         f.debug_struct("Layout")
             .field("size", size)
@@ -1728,6 +1741,7 @@ where
             .field("variants", variants)
             .field("max_repr_align", max_repr_align)
             .field("unadjusted_abi_align", unadjusted_abi_align)
+            .field("randomization_seed", randomization_seed)
             .finish()
     }
 }

compiler/rustc_middle/src/ty/layout.rs

@@ -761,6 +761,7 @@ where
                     size: Size::ZERO,
                     max_repr_align: None,
                     unadjusted_abi_align: tcx.data_layout.i8_align.abi,
+                    randomization_seed: 0,
                 })
             }
 

compiler/rustc_ty_utils/src/layout.rs

@@ -348,6 +348,7 @@ fn layout_of_uncached<'tcx>(
                 size,
                 max_repr_align: None,
                 unadjusted_abi_align: element.align.abi,
+                randomization_seed: element.randomization_seed.wrapping_add(count),
             })
         }
         ty::Slice(element) => {
@@ -361,6 +362,8 @@ fn layout_of_uncached<'tcx>(
                 size: Size::ZERO,
                 max_repr_align: None,
                 unadjusted_abi_align: element.align.abi,
+                // adding a randomly chosen value to distinguish slices
+                randomization_seed: element.randomization_seed.wrapping_add(0x2dcba99c39784102),
             })
         }
         ty::Str => tcx.mk_layout(LayoutData {
@@ -372,6 +375,8 @@ fn layout_of_uncached<'tcx>(
             size: Size::ZERO,
             max_repr_align: None,
             unadjusted_abi_align: dl.i8_align.abi,
+            // another random value
+            randomization_seed: 0xc1325f37d127be22,
         }),
 
         // Odd unit types.
@@ -543,6 +548,7 @@ fn layout_of_uncached<'tcx>(
                 align,
                 max_repr_align: None,
                 unadjusted_abi_align: align.abi,
+                randomization_seed: e_ly.randomization_seed.wrapping_add(e_len),
             })
         }
 
@@ -995,6 +1001,9 @@ fn coroutine_layout<'tcx>(
         BackendRepr::Memory { sized: true }
     };
 
+    // this is similar to how ReprOptions populates its field_shuffle_seed
+    let def_hash = tcx.def_path_hash(def_id).0.to_smaller_hash().as_u64();
+
     let layout = tcx.mk_layout(LayoutData {
         variants: Variants::Multiple {
             tag,
@@ -1015,6 +1024,7 @@ fn coroutine_layout<'tcx>(
         align,
         max_repr_align: None,
         unadjusted_abi_align: align.abi,
+        randomization_seed: def_hash,
     });
     debug!("coroutine layout ({:?}): {:#?}", ty, layout);
     Ok(layout)

tests/ui/layout/randomize.rs

@@ -0,0 +1,29 @@
+//@ build-pass
+//@ revisions: normal randomize-layout
+//@ [randomize-layout]compile-flags: -Zrandomize-layout
+
+#![crate_type = "lib"]
+
+struct Foo<T>(u32, T, u8);
+
+struct Wrapper<T>(T);
+
+#[repr(transparent)]
+struct TransparentWrapper(u16);
+
+const _: () = {
+    // behavior of the current implementation, not guaranteed
+    #[cfg(not(randomize_layout))]
+    assert!(std::mem::offset_of!(Foo::<u16>, 1) == std::mem::offset_of!(Foo::<Wrapper<u16>>, 1));
+
+    // under randomization Foo<T> != Foo<U>
+    #[cfg(randomize_layout)]
+    assert!(std::mem::offset_of!(Foo::<u16>, 1) != std::mem::offset_of!(Foo::<Wrapper<u16>>, 1));
+
+    // but repr(transparent) should make them the same again.
+    // maybe not strictly guaranteed? but UCG has been leaning in that direction at least
+    #[cfg(randomize_layout)]
+    assert!(
+        std::mem::offset_of!(Foo::<u16>, 1) == std::mem::offset_of!(Foo::<TransparentWrapper>, 1)
+    );
+};