transmutability: shift abstraction boundary

Previously, `rustc_transmute`'s layout representations were genericized
over `R`, a reference. Now, it's instead genericized over
representations of type and region. This allows us to move reference
transmutability logic from `rustc_trait_selection` to
`rustc_transmutability` (and thus unit test it independently of the
compiler), and — in a follow-up PR — will make it possible to support
analyzing function pointer transmutability with minimal surgery.

Author: jswrenn

compiler/rustc_trait_selection/src/error_reporting/traits/fulfillment_errors.rs

@@ -2553,32 +2553,31 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> {
                         rustc_transmute::Reason::SrcIsNotYetSupported => {
                             format!("analyzing the transmutability of `{src}` is not yet supported")
                         }
-
                         rustc_transmute::Reason::DstIsNotYetSupported => {
                             format!("analyzing the transmutability of `{dst}` is not yet supported")
                         }
-
                         rustc_transmute::Reason::DstIsBitIncompatible => {
                             format!(
                                 "at least one value of `{src}` isn't a bit-valid value of `{dst}`"
                             )
                         }
-
                         rustc_transmute::Reason::DstUninhabited => {
                             format!("`{dst}` is uninhabited")
                         }
-
                         rustc_transmute::Reason::DstMayHaveSafetyInvariants => {
                             format!("`{dst}` may carry safety invariants")
                         }
                         rustc_transmute::Reason::DstIsTooBig => {
                             format!("the size of `{src}` is smaller than the size of `{dst}`")
                         }
-                        rustc_transmute::Reason::DstRefIsTooBig { src, dst } => {
-                            let src_size = src.size;
-                            let dst_size = dst.size;
+                        rustc_transmute::Reason::DstRefIsTooBig {
+                            src,
+                            src_size,
+                            dst,
+                            dst_size,
+                        } => {
                             format!(
-                                "the referent size of `{src}` ({src_size} bytes) \
+                                "the size of `{src}` ({src_size} bytes) \
                         is smaller than that of `{dst}` ({dst_size} bytes)"
                             )
                         }

compiler/rustc_trait_selection/src/traits/select/confirmation.rs

@@ -9,13 +9,12 @@
 
 use std::ops::ControlFlow;
 
-use rustc_ast::Mutability;
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_hir::lang_items::LangItem;
 use rustc_infer::infer::{DefineOpaqueTypes, HigherRankedType, InferOk};
 use rustc_infer::traits::ObligationCauseCode;
 use rustc_middle::traits::{BuiltinImplSource, SignatureMismatchData};
-use rustc_middle::ty::{self, GenericArgsRef, Ty, TyCtxt, Upcast, elaborate};
+use rustc_middle::ty::{self, GenericArgsRef, Region, Ty, TyCtxt, Upcast, elaborate};
 use rustc_middle::{bug, span_bug};
 use rustc_span::def_id::DefId;
 use thin_vec::thin_vec;
@@ -291,99 +290,12 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
     ) -> Result<PredicateObligations<'tcx>, SelectionError<'tcx>> {
         use rustc_transmute::{Answer, Assume, Condition};
 
-        /// Generate sub-obligations for reference-to-reference transmutations.
-        fn reference_obligations<'tcx>(
-            tcx: TyCtxt<'tcx>,
-            obligation: &PolyTraitObligation<'tcx>,
-            (src_lifetime, src_ty, src_mut): (ty::Region<'tcx>, Ty<'tcx>, Mutability),
-            (dst_lifetime, dst_ty, dst_mut): (ty::Region<'tcx>, Ty<'tcx>, Mutability),
-            assume: Assume,
-        ) -> PredicateObligations<'tcx> {
-            let make_transmute_obl = |src, dst| {
-                let transmute_trait = obligation.predicate.def_id();
-                let assume = obligation.predicate.skip_binder().trait_ref.args.const_at(2);
-                let trait_ref = ty::TraitRef::new(
-                    tcx,
-                    transmute_trait,
-                    [
-                        ty::GenericArg::from(dst),
-                        ty::GenericArg::from(src),
-                        ty::GenericArg::from(assume),
-                    ],
-                );
-                Obligation::with_depth(
-                    tcx,
-                    obligation.cause.clone(),
-                    obligation.recursion_depth + 1,
-                    obligation.param_env,
-                    trait_ref,
-                )
-            };
-
-            let make_freeze_obl = |ty| {
-                let trait_ref = ty::TraitRef::new(
-                    tcx,
-                    tcx.require_lang_item(LangItem::Freeze, None),
-                    [ty::GenericArg::from(ty)],
-                );
-                Obligation::with_depth(
-                    tcx,
-                    obligation.cause.clone(),
-                    obligation.recursion_depth + 1,
-                    obligation.param_env,
-                    trait_ref,
-                )
-            };
-
-            let make_outlives_obl = |target, region| {
-                let outlives = ty::OutlivesPredicate(target, region);
-                Obligation::with_depth(
-                    tcx,
-                    obligation.cause.clone(),
-                    obligation.recursion_depth + 1,
-                    obligation.param_env,
-                    outlives,
-                )
-            };
-
-            // Given a transmutation from `&'a (mut) Src` and `&'dst (mut) Dst`,
-            // it is always the case that `Src` must be transmutable into `Dst`,
-            // and that that `'src` must outlive `'dst`.
-            let mut obls = PredicateObligations::with_capacity(1);
-            obls.push(make_transmute_obl(src_ty, dst_ty));
-            if !assume.lifetimes {
-                obls.push(make_outlives_obl(src_lifetime, dst_lifetime));
-            }
-
-            // Given a transmutation from `&Src`, both `Src` and `Dst` must be
-            // `Freeze`, otherwise, using the transmuted value could lead to
-            // data races.
-            if src_mut == Mutability::Not {
-                obls.extend([make_freeze_obl(src_ty), make_freeze_obl(dst_ty)])
-            }
-
-            // Given a transmutation into `&'dst mut Dst`, it also must be the
-            // case that `Dst` is transmutable into `Src`. For example,
-            // transmuting bool -> u8 is OK as long as you can't update that u8
-            // to be > 1, because you could later transmute the u8 back to a
-            // bool and get undefined behavior. It also must be the case that
-            // `'dst` lives exactly as long as `'src`.
-            if dst_mut == Mutability::Mut {
-                obls.push(make_transmute_obl(dst_ty, src_ty));
-                if !assume.lifetimes {
-                    obls.push(make_outlives_obl(dst_lifetime, src_lifetime));
-                }
-            }
-
-            obls
-        }
-
         /// Flatten the `Condition` tree into a conjunction of obligations.
         #[instrument(level = "debug", skip(tcx, obligation))]
         fn flatten_answer_tree<'tcx>(
             tcx: TyCtxt<'tcx>,
             obligation: &PolyTraitObligation<'tcx>,
-            cond: Condition<rustc_transmute::layout::rustc::Ref<'tcx>>,
+            cond: Condition<Region<'tcx>, Ty<'tcx>>,
             assume: Assume,
         ) -> PredicateObligations<'tcx> {
             match cond {
@@ -393,13 +305,50 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                     .into_iter()
                     .flat_map(|cond| flatten_answer_tree(tcx, obligation, cond, assume))
                     .collect(),
-                Condition::IfTransmutable { src, dst } => reference_obligations(
-                    tcx,
-                    obligation,
-                    (src.lifetime, src.ty, src.mutability),
-                    (dst.lifetime, dst.ty, dst.mutability),
-                    assume,
-                ),
+                Condition::Immutable { ty } => {
+                    let trait_ref = ty::TraitRef::new(
+                        tcx,
+                        tcx.require_lang_item(LangItem::Freeze, None),
+                        [ty::GenericArg::from(ty)],
+                    );
+                    thin_vec![Obligation::with_depth(
+                        tcx,
+                        obligation.cause.clone(),
+                        obligation.recursion_depth + 1,
+                        obligation.param_env,
+                        trait_ref,
+                    )]
+                }
+                Condition::Outlives { long, short } => {
+                    let outlives = ty::OutlivesPredicate(long, short);
+                    thin_vec![Obligation::with_depth(
+                        tcx,
+                        obligation.cause.clone(),
+                        obligation.recursion_depth + 1,
+                        obligation.param_env,
+                        outlives,
+                    )]
+                }
+                Condition::Transmutable { src, dst } => {
+                    let transmute_trait = obligation.predicate.def_id();
+                    let assume = obligation.predicate.skip_binder().trait_ref.args.const_at(2);
+                    let trait_ref = ty::TraitRef::new(
+                        tcx,
+                        transmute_trait,
+                        [
+                            ty::GenericArg::from(dst),
+                            ty::GenericArg::from(src),
+                            ty::GenericArg::from(assume),
+                        ],
+                    );
+                    thin_vec![Obligation::with_depth(
+                        tcx,
+                        obligation.cause.clone(),
+                        obligation.recursion_depth + 1,
+                        obligation.param_env,
+                        trait_ref,
+                    )]
+                }
             }
         }
 

compiler/rustc_transmute/src/layout/dfa.rs

@@ -2,32 +2,35 @@ use std::fmt;
 use std::iter::Peekable;
 use std::sync::atomic::{AtomicU32, Ordering};
 
-use super::{Byte, Ref, Tree, Uninhabited};
+use super::{Byte, Reference, Region, Tree, Type, Uninhabited};
 use crate::{Map, Set};
 
 #[derive(PartialEq)]
 #[cfg_attr(test, derive(Clone))]
-pub(crate) struct Dfa<R>
+pub(crate) struct Dfa<R, T>
 where
-    R: Ref,
+    R: Region,
+    T: Type,
 {
-    pub(crate) transitions: Map<State, Transitions<R>>,
+    pub(crate) transitions: Map<State, Transitions<R, T>>,
     pub(crate) start: State,
     pub(crate) accept: State,
 }
 
 #[derive(PartialEq, Clone, Debug)]
-pub(crate) struct Transitions<R>
+pub(crate) struct Transitions<R, T>
 where
-    R: Ref,
+    R: Region,
+    T: Type,
 {
     byte_transitions: EdgeSet<State>,
-    ref_transitions: Map<R, State>,
+    ref_transitions: Map<Reference<R, T>, State>,
 }
 
-impl<R> Default for Transitions<R>
+impl<R, T> Default for Transitions<R, T>
 where
-    R: Ref,
+    R: Region,
+    T: Type,
 {
     fn default() -> Self {
         Self { byte_transitions: EdgeSet::empty(), ref_transitions: Map::default() }
@@ -51,9 +54,10 @@ impl fmt::Debug for State {
     }
 }
 
-impl<R> Dfa<R>
+impl<R, T> Dfa<R, T>
 where
-    R: Ref,
+    R: Region,
+    T: Type,
 {
     #[cfg(test)]
     pub(crate) fn bool() -> Self {
@@ -64,7 +68,7 @@ where
     }
 
     pub(crate) fn unit() -> Self {
-        let transitions: Map<State, Transitions<R>> = Map::default();
+        let transitions: Map<State, Transitions<R, T>> = Map::default();
         let start = State::new();
         let accept = start;
 
@@ -78,21 +82,21 @@ where
         })
     }
 
-    pub(crate) fn from_ref(r: R) -> Self {
+    pub(crate) fn from_ref(r: Reference<R, T>) -> Self {
         Self::from_transitions(|accept| Transitions {
             byte_transitions: EdgeSet::empty(),
             ref_transitions: [(r, accept)].into_iter().collect(),
         })
     }
 
-    fn from_transitions(f: impl FnOnce(State) -> Transitions<R>) -> Self {
+    fn from_transitions(f: impl FnOnce(State) -> Transitions<R, T>) -> Self {
         let start = State::new();
         let accept = State::new();
 
         Self { transitions: [(start, f(accept))].into_iter().collect(), start, accept }
     }
 
-    pub(crate) fn from_tree(tree: Tree<!, R>) -> Result<Self, Uninhabited> {
+    pub(crate) fn from_tree(tree: Tree<!, R, T>) -> Result<Self, Uninhabited> {
         Ok(match tree {
             Tree::Byte(b) => Self::from_byte(b),
             Tree::Ref(r) => Self::from_ref(r),
@@ -125,7 +129,7 @@ where
         let start = self.start;
         let accept = other.accept;
 
-        let mut transitions: Map<State, Transitions<R>> = self.transitions;
+        let mut transitions: Map<State, Transitions<R, T>> = self.transitions;
 
         for (source, transition) in other.transitions {
             let fix_state = |state| if state == other.start { self.accept } else { state };
@@ -169,7 +173,7 @@ where
         };
 
         let start = mapped((Some(a.start), Some(b.start)));
-        let mut transitions: Map<State, Transitions<R>> = Map::default();
+        let mut transitions: Map<State, Transitions<R, T>> = Map::default();
         let empty_transitions = Transitions::default();
 
         struct WorkQueue {
@@ -257,7 +261,7 @@ where
             .flat_map(|transitions| transitions.byte_transitions.iter())
     }
 
-    pub(crate) fn refs_from(&self, start: State) -> impl Iterator<Item = (R, State)> {
+    pub(crate) fn refs_from(&self, start: State) -> impl Iterator<Item = (Reference<R, T>, State)> {
         self.transitions
             .get(&start)
             .into_iter()
@@ -297,9 +301,10 @@ where
 }
 
 /// Serialize the DFA using the Graphviz DOT format.
-impl<R> fmt::Debug for Dfa<R>
+impl<R, T> fmt::Debug for Dfa<R, T>
 where
-    R: Ref,
+    R: Region,
+    T: Type,
 {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         writeln!(f, "digraph {{")?;