fix safe-transmute handling of enums

Author: RalfJung

compiler/rustc_abi/src/lib.rs

@@ -1505,7 +1505,11 @@ impl BackendRepr {
 #[cfg_attr(feature = "nightly", derive(HashStable_Generic))]
 pub enum Variants<FieldIdx: Idx, VariantIdx: Idx> {
     /// Single enum variants, structs/tuples, unions, and all non-ADTs.
-    Single { index: VariantIdx },
+    Single {
+        /// Always 0 for non-enums/generators.
+        /// For enums without a variant, this is an invalid index!
+        index: VariantIdx,
+    },
 
     /// Enum-likes with more than one variant: each variant comes with
     /// a *discriminant* (usually the same as the variant index but the user can

compiler/rustc_const_eval/src/interpret/discriminant.rs

@@ -70,7 +70,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                 if ty.is_enum() {
                     // Hilariously, `Single` is used even for 0-variant enums.
                     // (See https://github.com/rust-lang/rust/issues/89765).
-                    if matches!(ty.kind(), ty::Adt(def, ..) if def.variants().is_empty()) {
+                    if ty.ty_adt_def().unwrap().variants().is_empty() {
                         throw_ub!(UninhabitedEnumVariantRead(index))
                     }
                     // For consistency with `write_discriminant`, and to make sure that

compiler/rustc_middle/src/query/mod.rs

@@ -1086,6 +1086,8 @@ rustc_queries! {
     }
 
     /// Computes the tag (if any) for a given type and variant.
+    /// `None` means that the variant doesn't need a tag (because it is niched).
+    /// Will panic for uninhabited variants.
     query tag_for_variant(
         key: (Ty<'tcx>, abi::VariantIdx)
     ) -> Option<ty::ScalarInt> {

compiler/rustc_transmute/src/layout/tree.rs

@@ -319,38 +319,35 @@ pub(crate) mod rustc {
         ) -> Result<Self, Err> {
             assert!(def.is_enum());
 
-            // Computes the variant of a given index.
-            let layout_of_variant = |index, encoding: Option<TagEncoding<VariantIdx>>| {
-                let tag = cx.tcx().tag_for_variant((cx.tcx().erase_regions(ty), index));
-                let variant_def = Def::Variant(def.variant(index));
-                let variant_layout = ty_variant(cx, (ty, layout), index);
-                Self::from_variant(
-                    variant_def,
-                    tag.map(|tag| (tag, index, encoding.unwrap())),
-                    (ty, variant_layout),
-                    layout.size,
-                    cx,
-                )
-            };
+            // Computes the layout of a variant.
+            let layout_of_variant =
+                |index, encoding: Option<TagEncoding<VariantIdx>>| -> Result<Self, Err> {
+                    let variant_layout = ty_variant(cx, (ty, layout), index);
+                    if variant_layout.is_uninhabited() {
+                        return Ok(Self::uninhabited());
+                    }
+                    let tag = cx.tcx().tag_for_variant((cx.tcx().erase_regions(ty), index));
+                    let variant_def = Def::Variant(def.variant(index));
+                    Self::from_variant(
+                        variant_def,
+                        tag.map(|tag| (tag, index, encoding.unwrap())),
+                        (ty, variant_layout),
+                        layout.size,
+                        cx,
+                    )
+                };
 
-            // We consider three kinds of enums, each demanding a different
-            // treatment of their layout computation:
-            // 1. enums that are uninhabited ZSTs
-            // 2. enums that delegate their layout to a variant
-            // 3. enums with multiple variants
             match layout.variants() {
-                Variants::Single { .. } if layout.is_uninhabited() && layout.size == Size::ZERO => {
-                    // The layout representation of uninhabited, ZST enums is
-                    // defined to be like that of the `!` type, as opposed of a
-                    // typical enum. Consequently, they cannot be descended into
-                    // as if they typical enums. We therefore special-case this
-                    // scenario and simply return an uninhabited `Tree`.
-                    Ok(Self::uninhabited())
-                }
                 Variants::Single { index } => {
-                    // `Variants::Single` on enums with variants denotes that
-                    // the enum delegates its layout to the variant at `index`.
-                    layout_of_variant(*index, None)
+                    // Hilariously, `Single` is used even for 0-variant enums;
+                    // `index` is just junk in that case.
+                    if ty.ty_adt_def().unwrap().variants().is_empty() {
+                        Ok(Self::uninhabited())
+                    } else {
+                        // `Variants::Single` on enums with variants denotes that
+                        // the enum delegates its layout to the variant at `index`.
+                        layout_of_variant(*index, None)
+                    }
                 }
                 Variants::Multiple { tag, tag_encoding, tag_field, .. } => {
                     // `Variants::Multiple` denotes an enum with multiple
@@ -369,7 +366,7 @@ pub(crate) mod rustc {
                         },
                     )?;
 
-                    return Ok(Self::def(Def::Adt(def)).then(variants));
+                    Ok(Self::def(Def::Adt(def)).then(variants))
                 }
             }
         }

tests/crashes/126267.rs

@@ -91,3 +91,19 @@ fn distant_void() {
     assert::is_maybe_transmutable::<DistantVoid, &'static Void>();
     assert::is_maybe_transmutable::<u128, DistantVoid>(); //~ ERROR: cannot be safely transmuted
 }
+
+fn issue_126267() {
+    pub enum ApiError {}
+    pub struct TokioError {
+        b: bool,
+    }
+    pub enum Error {
+        Api { source: ApiError }, // this variant is uninhabited
+        Ethereum,
+        Tokio { source: TokioError },
+    }
+
+    struct Src;
+    type Dst = Error;
+    assert::is_maybe_transmutable::<Src, Dst>(); //~ERROR: cannot be safely transmuted
+}

tests/ui/transmutability/uninhabited.rs

@@ -110,7 +110,29 @@ LL | |             }
 LL | |         }>
    | |__________^ required by this bound in `is_maybe_transmutable`
 
-error: aborting due to 7 previous errors
+error[E0277]: `Src` cannot be safely transmuted into `issue_126267::Error`
+  --> $DIR/uninhabited.rs:108:42
+   |
+LL |     assert::is_maybe_transmutable::<Src, Dst>();
+   |                                          ^^^ the size of `Src` is smaller than the size of `issue_126267::Error`
+   |
+note: required by a bound in `is_maybe_transmutable`
+  --> $DIR/uninhabited.rs:10:14
+   |
+LL |       pub fn is_maybe_transmutable<Src, Dst>()
+   |              --------------------- required by a bound in this function
+LL |       where
+LL |           Dst: TransmuteFrom<Src, {
+   |  ______________^
+LL | |             Assume {
+LL | |                 alignment: true,
+LL | |                 lifetimes: true,
+...  |
+LL | |             }
+LL | |         }>
+   | |__________^ required by this bound in `is_maybe_transmutable`
+
+error: aborting due to 8 previous errors
 
 Some errors have detailed explanations: E0080, E0277.
 For more information about an error, try `rustc --explain E0080`.