check normalization overflow in monomorphization

Author: adwinwhite

compiler/rustc_middle/src/mir/mono.rs

@@ -47,6 +47,9 @@ pub enum InstantiationMode {
     LocalCopy,
 }
 
+#[derive(PartialEq, Eq, Clone, Copy, Debug, Hash, HashStable, TyEncodable, TyDecodable)]
+pub struct NormalizationErrorInMono;
+
 #[derive(PartialEq, Eq, Clone, Copy, Debug, Hash, HashStable, TyEncodable, TyDecodable)]
 pub enum MonoItem<'tcx> {
     Fn(Instance<'tcx>),

compiler/rustc_middle/src/query/erase.rs

@@ -3,8 +3,10 @@ use std::intrinsics::transmute_unchecked;
 use std::mem::MaybeUninit;
 
 use rustc_span::ErrorGuaranteed;
+use rustc_span::source_map::Spanned;
 
 use crate::mir::interpret::EvalToValTreeResult;
+use crate::mir::mono::{MonoItem, NormalizationErrorInMono};
 use crate::query::CyclePlaceholder;
 use crate::traits::solve;
 use crate::ty::adjustment::CoerceUnsizedInfo;
@@ -171,6 +173,17 @@ impl EraseType for Result<ty::EarlyBinder<'_, Ty<'_>>, CyclePlaceholder> {
     type Result = [u8; size_of::<Result<ty::EarlyBinder<'static, Ty<'_>>, CyclePlaceholder>>()];
 }
 
+impl EraseType
+    for Result<(&'_ [Spanned<MonoItem<'_>>], &'_ [Spanned<MonoItem<'_>>]), NormalizationErrorInMono>
+{
+    type Result = [u8; size_of::<
+        Result<
+            (&'static [Spanned<MonoItem<'static>>], &'static [Spanned<MonoItem<'static>>]),
+            NormalizationErrorInMono,
+        >,
+    >()];
+}
+
 impl<T> EraseType for Option<&'_ T> {
     type Result = [u8; size_of::<Option<&'static ()>>()];
 }

compiler/rustc_middle/src/query/mod.rs

@@ -117,7 +117,9 @@ use crate::mir::interpret::{
     EvalStaticInitializerRawResult, EvalToAllocationRawResult, EvalToConstValueResult,
     EvalToValTreeResult, GlobalId, LitToConstInput,
 };
-use crate::mir::mono::{CodegenUnit, CollectionMode, MonoItem, MonoItemPartitions};
+use crate::mir::mono::{
+    CodegenUnit, CollectionMode, MonoItem, MonoItemPartitions, NormalizationErrorInMono,
+};
 use crate::query::erase::{Erase, erase, restore};
 use crate::query::plumbing::{
     CyclePlaceholder, DynamicQuery, query_ensure, query_ensure_error_guaranteed, query_get_at,
@@ -2704,7 +2706,7 @@ rustc_queries! {
         desc { "functions to skip for move-size check" }
     }
 
-    query items_of_instance(key: (ty::Instance<'tcx>, CollectionMode)) -> (&'tcx [Spanned<MonoItem<'tcx>>], &'tcx [Spanned<MonoItem<'tcx>>]) {
+    query items_of_instance(key: (ty::Instance<'tcx>, CollectionMode)) -> Result<(&'tcx [Spanned<MonoItem<'tcx>>], &'tcx [Spanned<MonoItem<'tcx>>]), NormalizationErrorInMono> {
         desc { "collecting items used by `{}`", key.0 }
         cache_on_disk_if { true }
     }

compiler/rustc_monomorphize/src/collector.rs

@@ -208,6 +208,7 @@
 mod autodiff;
 
 use std::cell::OnceCell;
+use std::ops::ControlFlow;
 
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_data_structures::sync::{MTLock, par_for_each_in};
@@ -220,15 +221,17 @@ use rustc_hir::lang_items::LangItem;
 use rustc_hir::limit::Limit;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::mir::interpret::{AllocId, ErrorHandled, GlobalAlloc, Scalar};
-use rustc_middle::mir::mono::{CollectionMode, InstantiationMode, MonoItem};
+use rustc_middle::mir::mono::{
+    CollectionMode, InstantiationMode, MonoItem, NormalizationErrorInMono,
+};
 use rustc_middle::mir::visit::Visitor as MirVisitor;
-use rustc_middle::mir::{self, Location, MentionedItem, traversal};
+use rustc_middle::mir::{self, Body, Location, MentionedItem, traversal};
 use rustc_middle::query::TyCtxtAt;
 use rustc_middle::ty::adjustment::{CustomCoerceUnsized, PointerCoercion};
 use rustc_middle::ty::layout::ValidityRequirement;
 use rustc_middle::ty::{
     self, GenericArgs, GenericParamDefKind, Instance, InstanceKind, Ty, TyCtxt, TypeFoldable,
-    TypeVisitableExt, VtblEntry,
+    TypeVisitable, TypeVisitableExt, TypeVisitor, VtblEntry,
 };
 use rustc_middle::util::Providers;
 use rustc_middle::{bug, span_bug};
@@ -474,7 +477,22 @@ fn collect_items_rec<'tcx>(
             ));
 
             rustc_data_structures::stack::ensure_sufficient_stack(|| {
-                let (used, mentioned) = tcx.items_of_instance((instance, mode));
+                let Ok((used, mentioned)) = tcx.items_of_instance((instance, mode)) else {
+                    // Normalization errors here are usually due to trait solving overflow.
+                    // FIXME: I assume that there are few type errors at post-analysis stage, but not
+                    // entirely sure.
+                    // We have to emit the error outside of `items_of_instance` to access the
+                    // span of the `starting_item`.
+                    let def_id = instance.def_id();
+                    let def_span = tcx.def_span(def_id);
+                    let def_path_str = tcx.def_path_str(def_id);
+                    tcx.dcx().emit_fatal(RecursionLimit {
+                        span: starting_item.span,
+                        instance,
+                        def_span,
+                        def_path_str,
+                    });
+                };
                 used_items.extend(used.into_iter().copied());
                 mentioned_items.extend(mentioned.into_iter().copied());
             });
@@ -603,6 +621,35 @@ fn collect_items_rec<'tcx>(
     }
 }
 
+// Check whether we can normalize every type in the instantiated MIR body.
+fn check_normalization_error<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    instance: Instance<'tcx>,
+    body: &Body<'tcx>,
+) -> Result<(), NormalizationErrorInMono> {
+    struct NormalizationChecker<'tcx> {
+        tcx: TyCtxt<'tcx>,
+        instance: Instance<'tcx>,
+    }
+    impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for NormalizationChecker<'tcx> {
+        type Result = ControlFlow<()>;
+
+        fn visit_ty(&mut self, t: Ty<'tcx>) -> Self::Result {
+            match self.instance.try_instantiate_mir_and_normalize_erasing_regions(
+                self.tcx,
+                ty::TypingEnv::fully_monomorphized(),
+                ty::EarlyBinder::bind(t),
+            ) {
+                Ok(_) => ControlFlow::Continue(()),
+                Err(_) => ControlFlow::Break(()),
+            }
+        }
+    }
+
+    let mut checker = NormalizationChecker { tcx, instance };
+    if body.visit_with(&mut checker).is_break() { Err(NormalizationErrorInMono) } else { Ok(()) }
+}
+
 fn check_recursion_limit<'tcx>(
     tcx: TyCtxt<'tcx>,
     instance: Instance<'tcx>,
@@ -1259,11 +1306,15 @@ fn collect_items_of_instance<'tcx>(
     tcx: TyCtxt<'tcx>,
     instance: Instance<'tcx>,
     mode: CollectionMode,
-) -> (MonoItems<'tcx>, MonoItems<'tcx>) {
+) -> Result<(MonoItems<'tcx>, MonoItems<'tcx>), NormalizationErrorInMono> {
     // This item is getting monomorphized, do mono-time checks.
+    let body = tcx.instance_mir(instance.def);
+    // Plenty of code paths later assume that everything can be normalized. So we have to check
+    // normalization first.
+    // We choose to emit the error outside to provide helpful diagnostics.
+    check_normalization_error(tcx, instance, body)?;
     tcx.ensure_ok().check_mono_item(instance);
 
-    let body = tcx.instance_mir(instance.def);
     // Naively, in "used" collection mode, all functions get added to *both* `used_items` and
     // `mentioned_items`. Mentioned items processing will then notice that they have already been
     // visited, but at that point each mentioned item has been monomorphized, added to the
@@ -1313,19 +1364,22 @@ fn collect_items_of_instance<'tcx>(
         }
     }
 
-    (used_items, mentioned_items)
+    Ok((used_items, mentioned_items))
 }
 
 fn items_of_instance<'tcx>(
     tcx: TyCtxt<'tcx>,
     (instance, mode): (Instance<'tcx>, CollectionMode),
-) -> (&'tcx [Spanned<MonoItem<'tcx>>], &'tcx [Spanned<MonoItem<'tcx>>]) {
-    let (used_items, mentioned_items) = collect_items_of_instance(tcx, instance, mode);
+) -> Result<
+    (&'tcx [Spanned<MonoItem<'tcx>>], &'tcx [Spanned<MonoItem<'tcx>>]),
+    NormalizationErrorInMono,
+> {
+    let (used_items, mentioned_items) = collect_items_of_instance(tcx, instance, mode)?;
 
     let used_items = tcx.arena.alloc_from_iter(used_items);
     let mentioned_items = tcx.arena.alloc_from_iter(mentioned_items);
 
-    (used_items, mentioned_items)
+    Ok((used_items, mentioned_items))
 }
 
 /// `item` must be already monomorphized.

compiler/rustc_traits/src/normalize_erasing_regions.rs

@@ -42,7 +42,18 @@ fn try_normalize_after_erasing_regions<'tcx, T: TypeFoldable<TyCtxt<'tcx>> + Par
             // us a test case.
             debug_assert_eq!(normalized_value, resolved_value);
             let erased = infcx.tcx.erase_and_anonymize_regions(resolved_value);
-            debug_assert!(!erased.has_infer(), "{erased:?}");
+            if infcx.next_trait_solver() {
+                debug_assert!(!erased.has_infer(), "{erased:?}");
+            } else {
+                // The old solver returns an ty var with the failed obligation in case of
+                // selection error. And when the obligation is re-tried, the error should be
+                // reported. However in case of overflow error, the obligation may be fulfilled
+                // due to the original depth being dropped.
+                // In conclusion, overflow results in an unconstrained ty var.
+                if erased.has_infer() {
+                    return Err(NoSolution);
+                }
+            }
             Ok(erased)
         }
         Err(NoSolution) => Err(NoSolution),

tests/crashes/105275.rs renamed to tests/ui/codegen/normalization-overflow/recursion-issue-105275.rs

@@ -1,9 +1,10 @@
-//@ known-bug: #105275
+//@ build-fail
 //@ compile-flags: -Copt-level=0
 
 pub fn encode_num<Writer: ExampleWriter>(n: u32, mut writer: Writer) -> Result<(), Writer::Error> {
     if n > 15 {
         encode_num(n / 16, &mut writer)?;
+        //~^ ERROR: reached the recursion limit while instantiating
     }
     Ok(())
 }

tests/ui/codegen/normalization-overflow/recursion-issue-105275.stderr

@@ -0,0 +1,14 @@
+error: reached the recursion limit while instantiating `encode_num::<&mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut Error>`
+  --> $DIR/recursion-issue-105275.rs:6:9
+   |
+LL |         encode_num(n / 16, &mut writer)?;
+   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+note: `encode_num` defined here
+  --> $DIR/recursion-issue-105275.rs:4:1
+   |
+LL | pub fn encode_num<Writer: ExampleWriter>(n: u32, mut writer: Writer) -> Result<(), Writer::Error> {
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+error: aborting due to 1 previous error
+

tests/crashes/105937.rs renamed to tests/ui/codegen/normalization-overflow/recursion-issue-105937.rs

@@ -1,9 +1,10 @@
-//@ known-bug: #105937
+//@ build-fail
 //@ compile-flags: -Copt-level=0
 
 pub fn encode_num<Writer: ExampleWriter>(n: u32, mut writer: Writer) -> Result<(), Writer::Error> {
     if n > 15 {
         encode_num(n / 16, &mut writer)?;
+        //~^ ERROR: reached the recursion limit while instantiating
     }
     Ok(())
 }

tests/ui/codegen/normalization-overflow/recursion-issue-105937.stderr

@@ -0,0 +1,14 @@
+error: reached the recursion limit while instantiating `encode_num::<&mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut &mut Error>`
+  --> $DIR/recursion-issue-105937.rs:6:9
+   |
+LL |         encode_num(n / 16, &mut writer)?;
+   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+note: `encode_num` defined here
+  --> $DIR/recursion-issue-105937.rs:4:1
+   |
+LL | pub fn encode_num<Writer: ExampleWriter>(n: u32, mut writer: Writer) -> Result<(), Writer::Error> {
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+error: aborting due to 1 previous error
+

tests/crashes/117696-1.rs renamed to tests/ui/codegen/normalization-overflow/recursion-issue-117696-1.rs

@@ -1,4 +1,4 @@
-//@ known-bug: #117696
+//@ build-fail
 fn main() {
     let mut it = (Empty);
     rec(&mut it);
@@ -23,6 +23,7 @@ where
 {
     if () == () {
         T::count(it);
+        //~^ ERROR: reached the recursion limit while instantiating
     } else {
         rec(identity(&mut it))
     }