Merge pull request #14851 from dotty-staging/fix-2887

Guard against stackoverflows when instantiating HK-Lambdas

Author: odersky

compiler/src/dotty/tools/dotc/core/TypeApplications.scala

@@ -343,11 +343,15 @@ class TypeApplications(val self: Type) extends AnyVal {
               }
             }
             if ((dealiased eq stripped) || followAlias)
-              try {
+              try
                 val instantiated = dealiased.instantiate(args)
                 if (followAlias) instantiated.normalized else instantiated
-              }
-              catch { case ex: IndexOutOfBoundsException => AppliedType(self, args) }
+              catch
+                case ex: IndexOutOfBoundsException =>
+                  AppliedType(self, args)
+                case ex: Throwable =>
+                  handleRecursive("try to instantiate", i"$dealiased[$args%, %]", ex)
+
             else AppliedType(self, args)
           }
           else dealiased.resType match {

tests/neg/i2887b.check

@@ -0,0 +1,30 @@
+-- Error: tests/neg/i2887b.scala:11:7 ----------------------------------------------------------------------------------
+11 |    baz(new A { type S[X[_] <: [_] =>> Any, Y[_]] = [Z] =>> X[Z][Y[Z]]; type I[X] = X })(1) // error
+   |       ^
+   |       Recursion limit exceeded.
+   |       Maybe there is an illegal cyclic reference?
+   |       If that's not the case, you could also try to increase the stacksize using the -Xss JVM option.
+   |       A recurring operation is (inner to outer):
+   |
+   |         try to instantiate Z[Z]
+   |         try to instantiate Z[Z]
+   |         try to instantiate Z[Z]
+   |         try to instantiate Z[Z]
+   |         try to instantiate Z[Z]
+   |         try to instantiate Z[Z]
+   |         try to instantiate Z[Z]
+   |         try to instantiate Z[Z]
+   |         try to instantiate Z[Z]
+   |         try to instantiate Z[Z]
+   |         ...
+   |
+   |         try to instantiate Z[Z]
+   |         try to instantiate Z[Z]
+   |         try to instantiate Z[Z]
+   |         try to instantiate Z[Z]
+   |         try to instantiate Z[Z]
+   |         try to instantiate Z[Z]
+   |         try to instantiate Z[Z]
+   |         try to instantiate Z[Z]
+   |         try to instantiate Z[Z]
+   |         try to instantiate Z[Z]

tests/neg/i2887b.scala

@@ -1,13 +1,13 @@
-trait A { type S[X[_] <: [_] => Any, Y[_]] <: [_] => Any; type I[_] } // error // error
-trait B { type S[X[_],Y[_]]; type I[_] <: [_] => Any } // error
+trait A { type S[X[_] <: [_] =>> Any, Y[_]] <: [_] =>> Any; type I[_] }
+trait B { type S[X[_],Y[_]]; type I[_] <: [_] =>> Any }
 trait C { type M <: B }
 trait D { type M >: A }
 
 object Test {
   def test(x: C with D): Unit = {
-    def foo(a: A, b: B)(z: a.S[b.I,a.I][b.S[a.I,a.I]]) = z // error
+    def foo(a: A, b: B)(z: a.S[b.I,a.I][b.S[a.I,a.I]]) = z
     def bar(a: A, y: x.M) = foo(a,y)
     def baz(a: A) = bar(a, a)
-    baz(new A { type S[X[_] <: [_] => Any, Y[_]] = [Z] => X[Z][Y[Z]]; type I[X] = X })(1) // error // error
+    baz(new A { type S[X[_] <: [_] =>> Any, Y[_]] = [Z] =>> X[Z][Y[Z]]; type I[X] = X })(1) // error
   }
-}
+}