Avoid looping when inferring types with recursive lower bounds in constraints

i3627.scala caused the typer to loop in several different ways. One was in isSubType,
the other in wildApprox.

The isSubType loop can be fixed by leaving monitoring on once the limit was reached
for the remainder of the top-level subtype check. I am not exactly sure what caused
the loop but the observation was a very long or infinite oscillation around the monitoring
limit. Leaving monitoring on is certainly not incorrect - the previous trick to turn
it off for recursions under the limit was purely for optimization. On the other hand
I am not sure whether the new scheme is sufficient, or whether we can still have
the situation of infinite recursions that stay under the depth limit.

The wildApprox loop was a simple bug (the `seen` value was not propagated correctly
to the TypeMap). The fix also removed a case in wildApproxBounds that had no clear purpose.

Author: odersky

compiler/src/dotty/tools/dotc/core/TypeComparer.scala

@@ -24,6 +24,7 @@ class TypeComparer(initctx: Context) extends DotClass with ConstraintHandling {
 
   private[this] var pendingSubTypes: mutable.Set[(Type, Type)] = null
   private[this] var recCount = 0
+  private[this] var monitored = false
 
   private[this] var needsGc = false
 
@@ -101,9 +102,11 @@ class TypeComparer(initctx: Context) extends DotClass with ConstraintHandling {
     if (tp2 eq NoType) return false
     if ((tp2 eq tp1) || (tp2 eq WildcardType)) return true
     try isSubType(tp1, tp2)
-    finally
+    finally {
+      monitored = false
       if (Config.checkConstraintsSatisfiable)
         assert(isSatisfiable, constraint.show)
+    }
   }
 
   protected def isSubType(tp1: Type, tp2: Type): Boolean = trace(s"isSubType ${traceInfo(tp1, tp2)}", subtyping) {
@@ -114,9 +117,8 @@ class TypeComparer(initctx: Context) extends DotClass with ConstraintHandling {
       val savedSuccessCount = successCount
       try {
         recCount = recCount + 1
-        val result =
-          if (recCount < Config.LogPendingSubTypesThreshold) firstTry(tp1, tp2)
-          else monitoredIsSubType(tp1, tp2)
+        if (recCount >= Config.LogPendingSubTypesThreshold) monitored = true
+        val result = if (monitored) monitoredIsSubType(tp1, tp2) else firstTry(tp1, tp2)
         recCount = recCount - 1
         if (!result) state.resetConstraintTo(saved)
         else if (recCount == 0 && needsGc) {

compiler/src/dotty/tools/dotc/typer/ProtoTypes.scala

@@ -495,9 +495,7 @@ object ProtoTypes {
       tp.derivedTypeAlias(wildApprox(tp.alias, theMap, seen))
     case tp @ TypeParamRef(poly, pnum) =>
       def wildApproxBounds(bounds: TypeBounds) =
-        if (bounds.lo.isInstanceOf[NamedType] && bounds.hi.isInstanceOf[NamedType])
-          WildcardType(wildApprox(bounds, theMap, seen).bounds)
-        else if (seen.contains(tp)) WildcardType
+        if (seen.contains(tp)) WildcardType
         else WildcardType(wildApprox(bounds, theMap, seen + tp).bounds)
       def unconstrainedApprox = wildApproxBounds(poly.paramInfos(pnum))
       def approxPoly =
@@ -544,7 +542,8 @@ object ProtoTypes {
     case  _: ThisType | _: BoundType | NoPrefix => // default case, inlined for speed
       tp
     case _ =>
-      (if (theMap != null) theMap else new WildApproxMap(seen)).mapOver(tp)
+      (if (theMap != null && seen.eq(theMap.seen)) theMap else new WildApproxMap(seen))
+        .mapOver(tp)
   }
 
   @sharable object AssignProto extends UncachedGroundType with MatchAlways

tests/neg/i3627.scala

@@ -0,0 +1,9 @@
+trait Comparinator[T] {
+  def sort[T](x: Comparinator[_ >: T]) = ()
+  sort((a: Int) => true) // error
+}
+
+trait Comparinator2[T >: U, U] {
+  def sort[TT](x: Comparinator2[_ >: TT, U]) = ()
+  sort((a: Int) => true) // error
+}