Heal occurrences of => T between ElimByName and Erasure (#19558)

There's a window of vulnerability between ElimByName and Erasure where
some ExprTypes `=> T` that appear as parameters of function types are
not yet converted to by-name functions `() ?=> T`. These would cause an
assertion violation when used as operands of & or |. We fix this on the
fly when forming these types in TypeComparer. As explained in
ElimByName, we can't fix it beforehand by mapping all occurrences of `=>
T` to `() ?=> T` since that could lead to cycles.

Fixes #19548

Author: odersky

compiler/src/dotty/tools/dotc/core/TypeComparer.scala

@@ -2318,40 +2318,35 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
     else if !tp2.exists || (tp2 eq WildcardType) then tp1
     else if tp1.isAny && !tp2.isLambdaSub || tp1.isAnyKind || isBottom(tp2) then tp2
     else if tp2.isAny && !tp1.isLambdaSub || tp2.isAnyKind || isBottom(tp1) then tp1
-    else tp2 match
-      case tp2: LazyRef =>
-        glb(tp1, tp2.ref)
-      case _ =>
-        tp1 match
-          case tp1: LazyRef =>
-            glb(tp1.ref, tp2)
-          case _ =>
-            val tp1a = dropIfSuper(tp1, tp2)
-            if tp1a ne tp1 then glb(tp1a, tp2)
-            else
-              val tp2a = dropIfSuper(tp2, tp1)
-              if tp2a ne tp2 then glb(tp1, tp2a)
-              else tp2 match // normalize to disjunctive normal form if possible.
-                case tp2 @ OrType(tp21, tp22) =>
-                  lub(tp1 & tp21, tp1 & tp22, isSoft = tp2.isSoft)
-                case _ =>
-                  tp1 match
-                    case tp1 @ OrType(tp11, tp12) =>
-                      lub(tp11 & tp2, tp12 & tp2, isSoft = tp1.isSoft)
-                    case tp1: ConstantType =>
-                      tp2 match
-                        case tp2: ConstantType =>
-                          // Make use of the fact that the intersection of two constant types
-                          // types which are not subtypes of each other is known to be empty.
-                          // Note: The same does not apply to singleton types in general.
-                          // E.g. we could have a pattern match against `x.type & y.type`
-                          // which might succeed if `x` and `y` happen to be the same ref
-                          // at run time. It would not work to replace that with `Nothing`.
-                          // However, maybe we can still apply the replacement to
-                          // types which are not explicitly written.
-                          NothingType
-                        case _ => andType(tp1, tp2)
+    else
+      def mergedGlb(tp1: Type, tp2: Type): Type =
+        val tp1a = dropIfSuper(tp1, tp2)
+        if tp1a ne tp1 then glb(tp1a, tp2)
+        else
+          val tp2a = dropIfSuper(tp2, tp1)
+          if tp2a ne tp2 then glb(tp1, tp2a)
+          else tp2 match // normalize to disjunctive normal form if possible.
+            case tp2 @ OrType(tp21, tp22) =>
+              lub(tp1 & tp21, tp1 & tp22, isSoft = tp2.isSoft)
+            case _ =>
+              tp1 match
+                case tp1 @ OrType(tp11, tp12) =>
+                  lub(tp11 & tp2, tp12 & tp2, isSoft = tp1.isSoft)
+                case tp1: ConstantType =>
+                  tp2 match
+                    case tp2: ConstantType =>
+                      // Make use of the fact that the intersection of two constant types
+                      // types which are not subtypes of each other is known to be empty.
+                      // Note: The same does not apply to singleton types in general.
+                      // E.g. we could have a pattern match against `x.type & y.type`
+                      // which might succeed if `x` and `y` happen to be the same ref
+                      // at run time. It would not work to replace that with `Nothing`.
+                      // However, maybe we can still apply the replacement to
+                      // types which are not explicitly written.
+                      NothingType
                     case _ => andType(tp1, tp2)
+                case _ => andType(tp1, tp2)
+      mergedGlb(dropExpr(tp1.stripLazyRef), dropExpr(tp2.stripLazyRef))
   }
 
   def widenInUnions(using Context): Boolean =
@@ -2390,7 +2385,7 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
         if ((tp1 ne tp1w) || (tp2 ne tp2w)) lub(tp1w, tp2w, canConstrain = canConstrain, isSoft = isSoft)
         else orType(tp1w, tp2w, isSoft = isSoft) // no need to check subtypes again
       }
-      mergedLub(tp1.stripLazyRef, tp2.stripLazyRef)
+      mergedLub(dropExpr(tp1.stripLazyRef), dropExpr(tp2.stripLazyRef))
   }
 
   /** Try to produce joint arguments for a lub `A[T_1, ..., T_n] | A[T_1', ..., T_n']` using
@@ -2508,6 +2503,19 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
         NoType
     }
 
+  /** There's a window of vulnerability between ElimByName and Erasure where some
+   *  ExprTypes `=> T` that appear as parameters of function types are not yet converted
+   *  to by-name functions `() ?=> T`. These would cause an assertion violation when
+   *  used as operands of glb or lub. We fix this on the fly here. As explained in
+   *  ElimByName, we can't fix it beforehand by mapping all occurrences of `=> T` to
+   *  `() ?=> T` since that could lead to cycles.
+   */
+  private def dropExpr(tp: Type): Type = tp match
+    case ExprType(rt) if (Phases.elimByNamePhase <= ctx.phase) && !ctx.erasedTypes =>
+      defn.ByNameFunction(rt)
+    case _ =>
+      tp
+
   private def andTypeGen(tp1: Type, tp2: Type, op: (Type, Type) => Type,
       original: (Type, Type) => Type = _ & _, isErased: Boolean = ctx.erasedTypes): Type = trace(s"andTypeGen(${tp1.show}, ${tp2.show})", subtyping, show = true) {
     val t1 = distributeAnd(tp1, tp2)

tests/pos/i19548.scala

@@ -0,0 +1,2 @@
+def byName[T](p: => T): T = p
+val test = (if ??? then byName else (??? : ((=> Int) => Int))) (42)