check that UnsizeCoerce may hold before trying unsizing coercion

this prevents us from trying unsizing coercion in cases like
`*const W<dyn T>` -> `*const dyn T`, where it would later cause a
compilation error since `W<dyn T>: Sized` and `W<dyn T>: T` do not hold.

Author: WaffleLapkin

compiler/rustc_hir_typeck/src/coercion.rs

@@ -41,14 +41,14 @@ use rustc_abi::ExternAbi;
 use rustc_attr_parsing::InlineAttr;
 use rustc_errors::codes::*;
 use rustc_errors::{Applicability, Diag, struct_span_code_err};
-use rustc_hir as hir;
 use rustc_hir::def_id::{DefId, LocalDefId};
+use rustc_hir::{self as hir, LangItem};
 use rustc_hir_analysis::hir_ty_lowering::HirTyLowerer;
 use rustc_infer::infer::relate::RelateResult;
 use rustc_infer::infer::{Coercion, DefineOpaqueTypes, InferOk, InferResult};
 use rustc_infer::traits::{
     IfExpressionCause, MatchExpressionArmCause, Obligation, PredicateObligation,
-    PredicateObligations,
+    PredicateObligations, ScrubbedTraitError, TraitEngine,
 };
 use rustc_middle::lint::in_external_macro;
 use rustc_middle::span_bug;
@@ -58,13 +58,14 @@ use rustc_middle::ty::adjustment::{
 };
 use rustc_middle::ty::error::TypeError;
 use rustc_middle::ty::visit::TypeVisitableExt;
-use rustc_middle::ty::{self, GenericArgsRef, Ty, TyCtxt};
+use rustc_middle::ty::{self, AliasTy, GenericArgsRef, Ty, TyCtxt};
 use rustc_session::parse::feature_err;
 use rustc_span::{BytePos, DUMMY_SP, DesugaringKind, Span, sym};
 use rustc_trait_selection::infer::InferCtxtExt as _;
 use rustc_trait_selection::traits::query::evaluate_obligation::InferCtxtExt;
 use rustc_trait_selection::traits::{
     self, NormalizeExt, ObligationCause, ObligationCauseCode, ObligationCtxt,
+    StructurallyNormalizeExt,
 };
 use smallvec::{SmallVec, smallvec};
 use tracing::{debug, instrument};
@@ -582,6 +583,64 @@ impl<'f, 'tcx> Coerce<'f, 'tcx> {
 
         // Create an obligation for `Source: CoerceUnsized<Target>`.
         let cause = self.cause(self.cause.span, ObligationCauseCode::Coercion { source, target });
+        let root_obligation = Obligation::new(
+            self.tcx,
+            cause.clone(),
+            self.fcx.param_env,
+            ty::TraitRef::new(self.tcx, coerce_unsized_did, [coerce_source, coerce_target]),
+        );
+
+        // If the root `Source: CoerceUnsized<Target>` obligation can't possibly hold,
+        // we don't have to assume that this is unsizing coercion (it will always lead to an error)
+        //
+        // However, we don't want to bail early all the time, since the unholdable obligations
+        // may be interesting for diagnostics (such as trying to coerce `&T` to `&dyn Id<This = U>`)
+        // ...
+        if !self.infcx.predicate_may_hold(&root_obligation) {
+            if let Some(dyn_metadata_adt_def_id) = self.tcx.lang_items().get(LangItem::DynMetadata)
+                && let Some(metadata_type_def_id) = self.tcx.lang_items().get(LangItem::Metadata)
+            {
+                // ... so we only bail if there may be another way to convert the types.
+                //
+                // If both types are raw pointers to a (wrapper over a) trait object,
+                // this might be a cast like `*const W<dyn Trait> -> *const dyn Trait`.
+                // So it's better to bail and try that. (even if the cast is not possible, for
+                // example due to vtables not matching, cast diagnostic will likely still be better)
+                self.probe(|_| {
+                    use traits::TraitEngineExt;
+                    let mut fulfill_cx = <dyn TraitEngine<'_, ScrubbedTraitError<'_>>>::new(self);
+                    let mut normalize = |ty| {
+                        self.at(&cause, self.fcx.param_env)
+                            .structurally_normalize(ty, &mut *fulfill_cx)
+                    };
+
+                    // N.B. use `target`, not `coerce_target` (the latter is a var)
+                    if let &ty::RawPtr(source_pointee, _) = coerce_source.kind()
+                        && let &ty::RawPtr(target_pointee, _) = target.kind()
+                        && let Ok(source_metadata) = normalize(Ty::new_alias(
+                            self.tcx,
+                            ty::AliasTyKind::Projection,
+                            AliasTy::new(self.tcx, metadata_type_def_id, [source_pointee]),
+                        ))
+                        && source_metadata
+                            .ty_adt_def()
+                            .is_some_and(|d| d.did() == dyn_metadata_adt_def_id)
+                        && let Ok(target_metadata) = normalize(Ty::new_alias(
+                            self.tcx,
+                            ty::AliasTyKind::Projection,
+                            AliasTy::new(self.tcx, metadata_type_def_id, [target_pointee]),
+                        ))
+                        && target_metadata
+                            .ty_adt_def()
+                            .is_some_and(|d| d.did() == dyn_metadata_adt_def_id)
+                    {
+                        return Err(TypeError::Mismatch);
+                    }
+
+                    Ok(())
+                })?;
+            }
+        }
 
         // Use a FIFO queue for this custom fulfillment procedure.
         //
@@ -590,12 +649,7 @@ impl<'f, 'tcx> Coerce<'f, 'tcx> {
         // and almost never more than 3. By using a SmallVec we avoid an
         // allocation, at the (very small) cost of (occasionally) having to
         // shift subsequent elements down when removing the front element.
-        let mut queue: SmallVec<[PredicateObligation<'tcx>; 4]> = smallvec![Obligation::new(
-            self.tcx,
-            cause,
-            self.fcx.param_env,
-            ty::TraitRef::new(self.tcx, coerce_unsized_did, [coerce_source, coerce_target])
-        )];
+        let mut queue: SmallVec<[PredicateObligation<'tcx>; 4]> = smallvec![root_obligation];
 
         let mut has_unsized_tuple_coercion = false;
         let mut has_trait_upcasting_coercion = None;