Fix #11234: avoid cycles in unifying F-bounded type parameters

In tests/pos/11234.scala, we have the following F-bounded constraints:

    bounds =
	A0 <: Foo[LazyRef(A0)]
	A <: Foo[LazyRef(A)]
    ordering =
	A <: A0

As `Foo[T]` is non-variant, at some point we will add `A0 <: A`, it
will trigger unification of `A0` and `A`. The unification will call
`Foo[A0].&(Foo[A])`, which in turn calls `TypeComparer.glb(Foo[A0],
Foo[A])`.  The call `glb(Foo[A0], Foo[A])` in a fresh TypeComparer
will in turn add `A0 <: A` thus trigger the unification again.

We need to perform substitution before merge the bounds.

Author: liufengyun

compiler/src/dotty/tools/dotc/core/OrderingConstraint.scala

@@ -393,7 +393,9 @@ class OrderingConstraint(private val boundsMap: ParamBounds,
     order(this, param1, param2).checkNonCyclic()
 
   def unify(p1: TypeParamRef, p2: TypeParamRef)(using Context): This =
-    val p1Bounds = (nonParamBounds(p1) & nonParamBounds(p2)).substParam(p2, p1)
+    val bound1 = nonParamBounds(p1).substParam(p2, p1)
+    val bound2 = nonParamBounds(p2).substParam(p2, p1)
+    val p1Bounds = bound1 & bound2
     updateEntry(p1, p1Bounds).replace(p2, p1)
 
 // ---------- Replacements and Removals -------------------------------------

compiler/src/dotty/tools/dotc/core/TypeComparer.scala

@@ -390,6 +390,10 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
         }
         def compareTypeParamRef =
           assumedTrue(tp1) ||
+          tp2.match {
+            case tp2: TypeParamRef => constraint.isLess(tp1, tp2)
+            case _ => false
+          } ||
           isSubTypeWhenFrozen(bounds(tp1).hi, tp2) || {
             if (canConstrain(tp1) && !approx.high)
               addConstraint(tp1, tp2, fromBelow = false) && flagNothingBound
@@ -540,11 +544,7 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
           // widening in `fourthTry` before adding to the constraint.
           if (frozenConstraint) recur(tp1, bounds(tp2).lo)
           else isSubTypeWhenFrozen(tp1, tp2)
-        alwaysTrue ||
-        frozenConstraint && (tp1 match {
-          case tp1: TypeParamRef => constraint.isLess(tp1, tp2)
-          case _ => false
-        }) || {
+        alwaysTrue || {
           if (canConstrain(tp2) && !approx.low)
             addConstraint(tp2, tp1.widenExpr, fromBelow = true)
           else fourthTry
@@ -2122,7 +2122,7 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
     }
 
   private def andTypeGen(tp1: Type, tp2: Type, op: (Type, Type) => Type,
-      original: (Type, Type) => Type = _ & _, isErased: Boolean = ctx.erasedTypes): Type = trace(s"glb(${tp1.show}, ${tp2.show})", subtyping, show = true) {
+      original: (Type, Type) => Type = _ & _, isErased: Boolean = ctx.erasedTypes): Type = trace(s"andTypeGen(${tp1.show}, ${tp2.show})", subtyping, show = true) {
     val t1 = distributeAnd(tp1, tp2)
     if (t1.exists) t1
     else {

compiler/src/dotty/tools/dotc/core/TypeOps.scala

@@ -3,7 +3,7 @@ package dotc
 package core
 
 import Contexts._, Types._, Symbols._, Names._, Flags._
-import Denotations._, SymDenotations._
+import SymDenotations._
 import util.Spans._
 import util.Stats
 import NameKinds.DepParamName

tests/pos/i11234.scala

@@ -0,0 +1,19 @@
+trait Foo[A <: Foo[A]]
+trait FooCreator[A <: Foo[A]] {
+  def createFoo(): A
+}
+
+trait FooWrapper {
+  type A <: Foo[A]
+  def foo: A
+}
+
+object FooWrapper {
+  def apply[A0 <: Foo[A0]](toWrap: A0): FooWrapper { type A = A0 } = new FooWrapper {
+    type A = A0
+    def foo: A0 = toWrap
+  }
+}
+
+def error(fooWrapper: FooWrapper, processor: [A <: Foo[A]] => A => A): FooWrapper =
+  FooWrapper(processor(fooWrapper.foo))