Safe Transmute: Change Answer type to Result

This patch updates the `Answer` type from `rustc_transmute` so that it just a
type alias to `Result`. This makes it so that the standard methods for `Result`
can be used to process the `Answer` tree, including being able to make use of
the `?` operator on `Answer`s.

Also, remove some unused functions

Author: bryangarza

compiler/rustc_trait_selection/src/solve/eval_ctxt.rs

@@ -675,11 +675,11 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
             scope,
             assume,
         ) {
-            rustc_transmute::Answer::Yes => Ok(Certainty::Yes),
-            rustc_transmute::Answer::No(_)
-            | rustc_transmute::Answer::IfTransmutable { .. }
-            | rustc_transmute::Answer::IfAll(_)
-            | rustc_transmute::Answer::IfAny(_) => Err(NoSolution),
+            Ok(None) => Ok(Certainty::Yes),
+            Err(_)
+            | Ok(Some(rustc_transmute::Condition::IfTransmutable { .. }))
+            | Ok(Some(rustc_transmute::Condition::IfAll(_)))
+            | Ok(Some(rustc_transmute::Condition::IfAny(_))) => Err(NoSolution),
         }
     }
 }

compiler/rustc_trait_selection/src/traits/error_reporting/mod.rs

@@ -2751,13 +2751,14 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
             rustc_transmute::Assume::from_const(self.infcx.tcx, obligation.param_env, trait_ref.substs.const_at(3)) else {
                 span_bug!(span, "Unable to construct rustc_transmute::Assume where it was previously possible");
             };
+        // FIXME(bryangarza): Need to flatten here too
         match rustc_transmute::TransmuteTypeEnv::new(self.infcx).is_transmutable(
             obligation.cause,
             src_and_dst,
             scope,
             assume,
         ) {
-            rustc_transmute::Answer::No(reason) => {
+            Err(reason) => {
                 let dst = trait_ref.substs.type_at(0);
                 let src = trait_ref.substs.type_at(1);
                 let custom_err_msg = format!(
@@ -2795,7 +2796,7 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                 (custom_err_msg, Some(reason_msg))
             }
             // Should never get a Yes at this point! We already ran it before, and did not get a Yes.
-            rustc_transmute::Answer::Yes => span_bug!(
+            Ok(None) => span_bug!(
                 span,
                 "Inconsistent rustc_transmute::is_transmutable(...) result, got Yes",
             ),

compiler/rustc_trait_selection/src/traits/select/confirmation.rs

@@ -279,23 +279,24 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         ImplSourceBuiltinData { nested: obligations }
     }
 
-    #[instrument(skip(self))]
+    #[instrument(level = "debug", skip(self))]
     fn confirm_transmutability_candidate(
         &mut self,
         obligation: &TraitObligation<'tcx>,
     ) -> Result<ImplSourceBuiltinData<PredicateObligation<'tcx>>, SelectionError<'tcx>> {
+        #[instrument(level = "debug", skip(tcx, obligation, predicate))]
         fn flatten_answer_tree<'tcx>(
             tcx: TyCtxt<'tcx>,
             obligation: &TraitObligation<'tcx>,
             predicate: TraitPredicate<'tcx>,
             answer: rustc_transmute::Answer<rustc_transmute::layout::rustc::Ref<'tcx>>,
         ) -> Result<Vec<PredicateObligation<'tcx>>, SelectionError<'tcx>> {
             match answer {
-                rustc_transmute::Answer::Yes => Ok(vec![]),
-                rustc_transmute::Answer::No(_) => Err(Unimplemented),
+                Ok(None) => Ok(vec![]),
+                Err(_) => Err(Unimplemented),
                 // FIXME(bryangarza): Add separate `IfAny` case, instead of treating as `IfAll`
-                rustc_transmute::Answer::IfAll(answers)
-                | rustc_transmute::Answer::IfAny(answers) => {
+                Ok(Some(rustc_transmute::Condition::IfAll(answers)))
+                | Ok(Some(rustc_transmute::Condition::IfAny(answers))) => {
                     let mut nested = vec![];
                     for flattened in answers
                         .into_iter()
@@ -305,7 +306,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                     }
                     Ok(nested)
                 }
-                rustc_transmute::Answer::IfTransmutable { src, dst } => {
+                Ok(Some(rustc_transmute::Condition::IfTransmutable { src, dst })) => {
                     let trait_def_id = obligation.predicate.def_id();
                     let scope = predicate.trait_ref.substs.type_at(2);
                     let assume_const = predicate.trait_ref.substs.const_at(3);
@@ -334,8 +335,6 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             }
         }
 
-        debug!(?obligation, "confirm_transmutability_candidate");
-
         // We erase regions here because transmutability calls layout queries,
         // which does not handle inference regions and doesn't particularly
         // care about other regions. Erasing late-bound regions is equivalent
@@ -352,21 +351,21 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             return Err(Unimplemented);
         };
 
+        let dst = predicate.trait_ref.substs.type_at(0);
+        let src = predicate.trait_ref.substs.type_at(1);
         let mut transmute_env = rustc_transmute::TransmuteTypeEnv::new(self.infcx);
         let maybe_transmutable = transmute_env.is_transmutable(
             obligation.cause.clone(),
-            rustc_transmute::Types {
-                dst: predicate.trait_ref.substs.type_at(0),
-                src: predicate.trait_ref.substs.type_at(1),
-            },
+            rustc_transmute::Types { dst, src },
             predicate.trait_ref.substs.type_at(2),
             assume,
         );
 
-        info!(?maybe_transmutable);
-        let nested = flatten_answer_tree(self.tcx(), obligation, predicate, maybe_transmutable)?;
-        info!(?nested);
-        Ok(ImplSourceBuiltinData { nested })
+        debug!(?src, ?dst);
+        let fully_flattened =
+            flatten_answer_tree(self.tcx(), obligation, predicate, maybe_transmutable)?;
+        debug!(?fully_flattened);
+        Ok(ImplSourceBuiltinData { nested: fully_flattened })
     }
 
     /// This handles the case where an `auto trait Foo` impl is being used.

compiler/rustc_transmute/src/lib.rs

@@ -19,15 +19,12 @@ pub struct Assume {
     pub validity: bool,
 }
 
-/// The type encodes answers to the question: "Are these types transmutable?"
-#[derive(Debug, Hash, Eq, PartialEq, PartialOrd, Ord, Clone)]
-pub enum Answer<R> {
-    /// `Src` is transmutable into `Dst`.
-    Yes,
-
-    /// `Src` is NOT transmutable into `Dst`.
-    No(Reason),
+/// Either we have an error, or we have an optional Condition that must hold.
+pub type Answer<R> = Result<Option<Condition<R>>, Reason>;
 
+/// A condition which must hold for safe transmutation to be possible
+#[derive(Debug, Hash, Eq, PartialEq, Clone)]
+pub enum Condition<R> {
     /// `Src` is transmutable into `Dst`, if `src` is transmutable into `dst`.
     IfTransmutable { src: R, dst: R },