Avoid inf recursion in provablyDisjointClasses

Author: dwijnand

compiler/src/dotty/tools/dotc/core/TypeComparer.scala

@@ -3201,14 +3201,15 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
             result
         end existsCommonBaseTypeWithDisjointArguments
 
-        provablyDisjointClasses(cls1, cls2)
+        provablyDisjointClasses(cls1, cls2, seen = null)
           || existsCommonBaseTypeWithDisjointArguments
     end match
   }
 
-  private def provablyDisjointClasses(cls1: Symbol, cls2: Symbol)(using Context): Boolean =
+  private def provablyDisjointClasses(cls1: Symbol, cls2: Symbol, seen: util.HashSet[Symbol] | Null)(using Context): Boolean =
     def isDecomposable(cls: Symbol): Boolean =
-      cls.is(Sealed) && !cls.hasAnonymousChild
+      if seen != null && seen.contains(cls) then false
+      else cls.is(Sealed) && !cls.hasAnonymousChild
 
     def decompose(cls: Symbol): List[Symbol] =
       cls.children.flatMap { child =>
@@ -3217,6 +3218,13 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
         else child :: Nil
       }.filter(child => child.exists && child != cls)
 
+    inline def seeing(inline cls: Symbol)(inline thunk: util.HashSet[Symbol] => Boolean) =
+      val seen1 = if seen == null then new util.HashSet[Symbol] else seen
+      try
+        seen1 += cls
+        thunk(seen1)
+      finally seen1 -= cls
+
     def eitherDerivesFromOther(cls1: Symbol, cls2: Symbol): Boolean =
       cls1.derivesFrom(cls2) || cls2.derivesFrom(cls1)
 
@@ -3239,9 +3247,11 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
         // instantiations of `cls1` (terms of the form `new cls1`) are not
         // of type `tp2`. Therefore, we can safely decompose `cls1` using
         // `.children`, even if `cls1` is non abstract.
-        decompose(cls1).forall(x => provablyDisjointClasses(x, cls2))
+        seeing(cls1): seen1 =>
+          decompose(cls1).forall(x => provablyDisjointClasses(x, cls2, seen1))
       else if (isDecomposable(cls2))
-        decompose(cls2).forall(x => provablyDisjointClasses(cls1, x))
+        seeing(cls2): seen1 =>
+          decompose(cls2).forall(x => provablyDisjointClasses(cls1, x, seen1))
       else
         false
   end provablyDisjointClasses

tests/pos/i22266.scala

@@ -0,0 +1,22 @@
+sealed trait NonPolygon
+sealed trait Polygon
+
+sealed trait SymmetryAspect
+sealed trait RotationalSymmetry      extends SymmetryAspect
+sealed trait MaybeRotationalSymmetry extends SymmetryAspect
+
+enum Shape:
+  case Circle   extends Shape with NonPolygon with RotationalSymmetry
+  case Triangle extends Shape with Polygon with MaybeRotationalSymmetry
+  case Square   extends Shape with Polygon with RotationalSymmetry
+
+object Shape:
+
+  def hasPolygon(
+    rotationalSyms: Vector[Shape & RotationalSymmetry],
+    maybeSyms: Vector[Shape & MaybeRotationalSymmetry]
+  ): Boolean =
+    val all = rotationalSyms.concat(maybeSyms)
+    all.exists:
+      case _: Polygon => true
+      case _          => false