Auto merge of rust-lang#39680 - canndrew:uninhabited_from-infinite-loop, r=arielb1

Add recursion limit to inhabitedness check

Fixes rust-lang#39489.
Add test aswell.

Author: bors

src/librustc/ty/inhabitedness/mod.rs

@@ -8,7 +8,7 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use util::nodemap::FxHashSet;
+use util::nodemap::{FxHashMap, FxHashSet};
 use ty::context::TyCtxt;
 use ty::{AdtDef, VariantDef, FieldDef, TyS};
 use ty::{DefId, Substs};
@@ -66,27 +66,21 @@ impl<'a, 'gcx, 'tcx> AdtDef {
     /// Calculate the forest of DefIds from which this adt is visibly uninhabited.
     pub fn uninhabited_from(
                 &self,
-                visited: &mut FxHashSet<(DefId, &'tcx Substs<'tcx>)>,
+                visited: &mut FxHashMap<DefId, FxHashSet<&'tcx Substs<'tcx>>>,
                 tcx: TyCtxt<'a, 'gcx, 'tcx>,
                 substs: &'tcx Substs<'tcx>) -> DefIdForest
     {
-        if !visited.insert((self.did, substs)) {
-            return DefIdForest::empty();
-        }
-
-        let ret = DefIdForest::intersection(tcx, self.variants.iter().map(|v| {
+        DefIdForest::intersection(tcx, self.variants.iter().map(|v| {
             v.uninhabited_from(visited, tcx, substs, self.adt_kind())
-        }));
-        visited.remove(&(self.did, substs));
-        ret
+        }))
     }
 }
 
 impl<'a, 'gcx, 'tcx> VariantDef {
     /// Calculate the forest of DefIds from which this variant is visibly uninhabited.
     pub fn uninhabited_from(
                 &self,
-                visited: &mut FxHashSet<(DefId, &'tcx Substs<'tcx>)>,
+                visited: &mut FxHashMap<DefId, FxHashSet<&'tcx Substs<'tcx>>>,
                 tcx: TyCtxt<'a, 'gcx, 'tcx>,
                 substs: &'tcx Substs<'tcx>,
                 adt_kind: AdtKind) -> DefIdForest
@@ -115,12 +109,14 @@ impl<'a, 'gcx, 'tcx> FieldDef {
     /// Calculate the forest of DefIds from which this field is visibly uninhabited.
     pub fn uninhabited_from(
                 &self,
-                visited: &mut FxHashSet<(DefId, &'tcx Substs<'tcx>)>,
+                visited: &mut FxHashMap<DefId, FxHashSet<&'tcx Substs<'tcx>>>,
                 tcx: TyCtxt<'a, 'gcx, 'tcx>,
                 substs: &'tcx Substs<'tcx>,
                 is_enum: bool) -> DefIdForest
     {
-        let mut data_uninhabitedness = move || self.ty(tcx, substs).uninhabited_from(visited, tcx);
+        let mut data_uninhabitedness = move || {
+            self.ty(tcx, substs).uninhabited_from(visited, tcx)
+        };
         // FIXME(canndrew): Currently enum fields are (incorrectly) stored with
         // Visibility::Invisible so we need to override self.vis if we're
         // dealing with an enum.
@@ -144,7 +140,7 @@ impl<'a, 'gcx, 'tcx> TyS<'tcx> {
     /// Calculate the forest of DefIds from which this type is visibly uninhabited.
     pub fn uninhabited_from(
                 &self,
-                visited: &mut FxHashSet<(DefId, &'tcx Substs<'tcx>)>,
+                visited: &mut FxHashMap<DefId, FxHashSet<&'tcx Substs<'tcx>>>,
                 tcx: TyCtxt<'a, 'gcx, 'tcx>) -> DefIdForest
     {
         match tcx.lift_to_global(&self) {
@@ -169,12 +165,37 @@ impl<'a, 'gcx, 'tcx> TyS<'tcx> {
 
     fn uninhabited_from_inner(
                 &self,
-                visited: &mut FxHashSet<(DefId, &'tcx Substs<'tcx>)>,
+                visited: &mut FxHashMap<DefId, FxHashSet<&'tcx Substs<'tcx>>>,
                 tcx: TyCtxt<'a, 'gcx, 'tcx>) -> DefIdForest
     {
         match self.sty {
             TyAdt(def, substs) => {
-                def.uninhabited_from(visited, tcx, substs)
+                {
+                    let mut substs_set = visited.entry(def.did).or_insert(FxHashSet::default());
+                    if !substs_set.insert(substs) {
+                        // We are already calculating the inhabitedness of this type.
+                        // The type must contain a reference to itself. Break the
+                        // infinite loop.
+                        return DefIdForest::empty();
+                    }
+                    if substs_set.len() >= tcx.sess.recursion_limit.get() / 4 {
+                        // We have gone very deep, reinstantiating this ADT inside
+                        // itself with different type arguments. We are probably
+                        // hitting an infinite loop. For example, it's possible to write:
+                        //                a type Foo<T>
+                        //      which contains a Foo<(T, T)>
+                        //      which contains a Foo<((T, T), (T, T))>
+                        //      which contains a Foo<(((T, T), (T, T)), ((T, T), (T, T)))>
+                        //      etc.
+                        let error = format!("reached recursion limit while checking
+                                             inhabitedness of `{}`", self);
+                        tcx.sess.fatal(&error);
+                    }
+                }
+                let ret = def.uninhabited_from(visited, tcx, substs);
+                let mut substs_set = visited.get_mut(&def.did).unwrap();
+                substs_set.remove(substs);
+                ret
             },
 
             TyNever => DefIdForest::full(tcx),

src/librustc/ty/sty.rs

@@ -24,7 +24,7 @@ use std::cmp::Ordering;
 use syntax::abi;
 use syntax::ast::{self, Name};
 use syntax::symbol::{keywords, InternedString};
-use util::nodemap::FxHashSet;
+use util::nodemap::FxHashMap;
 
 use serialize;
 
@@ -1018,7 +1018,7 @@ impl<'a, 'gcx, 'tcx> TyS<'tcx> {
     /// This code should only compile in modules where the uninhabitedness of Foo is
     /// visible.
     pub fn is_uninhabited_from(&self, module: DefId, tcx: TyCtxt<'a, 'gcx, 'tcx>) -> bool {
-        let mut visited = FxHashSet::default();
+        let mut visited = FxHashMap::default();
         let forest = self.uninhabited_from(&mut visited, tcx);
 
         // To check whether this type is uninhabited at all (not just from the

src/librustc_const_eval/_match.rs

@@ -17,7 +17,7 @@ use eval::{compare_const_vals};
 
 use rustc_const_math::ConstInt;
 
-use rustc_data_structures::fx::{FxHashMap, FxHashSet};
+use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::indexed_vec::Idx;
 
 use pattern::{FieldPattern, Pattern, PatternKind};
@@ -404,7 +404,7 @@ fn all_constructors<'a, 'tcx: 'a>(cx: &mut MatchCheckCtxt<'a, 'tcx>,
         }
         ty::TyAdt(def, substs) if def.is_enum() && def.variants.len() != 1 => {
             def.variants.iter().filter_map(|v| {
-                let mut visited = FxHashSet::default();
+                let mut visited = FxHashMap::default();
                 let forest = v.uninhabited_from(&mut visited,
                                                 cx.tcx, substs,
                                                 AdtKind::Enum);

src/librustc_mir/build/matches/simplify.rs

@@ -26,7 +26,7 @@ use build::{BlockAnd, BlockAndExtension, Builder};
 use build::matches::{Binding, MatchPair, Candidate};
 use hair::*;
 use rustc::mir::*;
-use rustc_data_structures::fx::FxHashSet;
+use rustc_data_structures::fx::FxHashMap;
 
 use std::mem;
 
@@ -102,7 +102,7 @@ impl<'a, 'gcx, 'tcx> Builder<'a, 'gcx, 'tcx> {
                 if self.hir.tcx().sess.features.borrow().never_type {
                     let irrefutable = adt_def.variants.iter().enumerate().all(|(i, v)| {
                         i == variant_index || {
-                            let mut visited = FxHashSet::default();
+                            let mut visited = FxHashMap::default();
                             let node_set = v.uninhabited_from(&mut visited,
                                                               self.hir.tcx(),
                                                               substs,

src/test/compile-fail/inhabitedness-infinite-loop.rs

@@ -0,0 +1,25 @@
+// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// error-pattern:reached recursion limit
+
+#![feature(never_type)]
+
+struct Foo<'a, T: 'a> {
+    ph: std::marker::PhantomData<T>,
+    foo: &'a Foo<'a, (T, T)>,
+}
+
+fn wub(f: Foo<!>) {
+    match f {}
+}
+
+fn main() {}
+