Fix #9568: Disallow partially undefined types as byname anchors

Require that the type that a by-name implicit can recursively implement
must be fully defined.

Author: odersky

compiler/src/dotty/tools/dotc/typer/Implicits.scala

@@ -247,11 +247,9 @@ object Implicits:
             candidates += Candidate(ref, ckind, level)
 
         if considerExtension then
-          val tryExtension = tryCandidate(extensionOnly = true)
-          companionRefs.foreach(tryExtension)
+          companionRefs.foreach(tryCandidate(extensionOnly = true))
         if refs.nonEmpty then
-          val tryGiven = tryCandidate(extensionOnly = false)
-          refs.foreach(tryGiven)
+          refs.foreach(tryCandidate(extensionOnly = false))
         candidates.toList
     }
   }
@@ -488,9 +486,10 @@ object Implicits:
 
   class DivergingImplicit(ref: TermRef,
                           val expectedType: Type,
-                          val argument: Tree) extends SearchFailureType {
+                          val argument: Tree,
+                          addendum: => String = "") extends SearchFailureType {
     def explanation(using Context): String =
-      em"${err.refStr(ref)} produces a diverging implicit search when trying to $qualify"
+      em"${err.refStr(ref)} produces a diverging implicit search when trying to $qualify$addendum"
   }
 
   class FailedExtension(extApp: Tree, val expectedType: Type) extends SearchFailureType:
@@ -1101,7 +1100,13 @@ trait Implicits:
       */
     def tryImplicit(cand: Candidate, contextual: Boolean): SearchResult =
       if checkDivergence(cand) then
-        SearchFailure(new DivergingImplicit(cand.ref, wideProto, argument))
+        val addendum = ctx.searchHistory.disqualifiedType match
+          case NoType => ""
+          case disTp =>
+            em""".
+                |Note that open search type $disTp cannot be re-used as a by-name implicit parameter
+                |since it is not fully defined"""
+        SearchFailure(new DivergingImplicit(cand.ref, wideProto, argument, addendum))
       else {
         val history = ctx.searchHistory.nest(cand, pt)
         val result =
@@ -1371,8 +1376,12 @@ trait Implicits:
         history match
           case prev @ OpenSearch(cand1, tp, outer) =>
             if cand1.ref eq cand.ref then
+              def checkFullyDefined =
+                val result = isFullyDefined(tp, ForceDegree.failBottom)
+                if !result then prev.disqualified = true
+                result
               lazy val wildTp = wildApprox(tp.widenExpr)
-              if belowByname && (wildTp <:< wildPt) then
+              if belowByname && (wildTp <:< wildPt) && checkFullyDefined then
                 false
               else if prev.typeSize > ptSize || prev.coveringSet != ptCoveringSet then
                 loop(outer, tp.isByName || belowByname)
@@ -1464,6 +1473,15 @@ abstract class SearchHistory:
   def defineBynameImplicit(tpe: Type, result: SearchSuccess)(using Context): SearchResult =
     root.defineBynameImplicit(tpe, result)
 
+  /** There is a qualifying call-by-name parameter in the history
+   *  that cannot be used since is it not fully defined. Used for error reporting.
+   */
+  def disqualifiedType: Type =
+    def loop(h: SearchHistory): Type = h match
+      case h: OpenSearch => if h.disqualified then h.pt else loop(h.outer)
+      case _ => NoType
+    loop(this)
+
   // This is NOOP unless at the root of this search history.
   def emitDictionary(span: Span, result: SearchResult)(using Context): SearchResult = result
 
@@ -1483,6 +1501,10 @@ case class OpenSearch(cand: Candidate, pt: Type, outer: SearchHistory)(using Con
   // An example is in neg/9504.scala
   lazy val typeSize = pt.typeSize
   lazy val coveringSet = pt.coveringSet
+
+  // Set if there would be a qualifying call-by-name parameter
+  // that cannot be used since is it not fully defined
+  var disqualified: Boolean = false
 end OpenSearch
 
 /**

tests/neg/i3452.scala

@@ -6,9 +6,7 @@ object Test {
   implicit def case1[F[_]](implicit t: => TC[F[Any]]): TC[Tuple2K[[_] =>> Any, F, Any]] = ???
   implicit def case2[A, F[_]](implicit r: TC[F[Any]]): TC[A] = ???
 
-  // Disabled because it leads to an infinite loop in implicit search
-  // this is probably the same issue as https://github.com/lampepfl/dotty/issues/9568
-  // implicitly[TC[Int]] // was: error
+  implicitly[TC[Int]] // error
 }
 
 object Test1 {

tests/neg/i9568.check

@@ -0,0 +1,11 @@
+-- Error: tests/neg/i9568.scala:13:10 ----------------------------------------------------------------------------------
+13 |  blaMonad.foo(bla) // error: diverges
+   |          ^
+   |no implicit argument of type => Monad[([_$3] =>> Any)] was found for parameter ev of method blaMonad in object Test.
+   |I found:
+   |
+   |    Test.blaMonad[F, S](/* missing */summon[Monad[F]])
+   |
+   |But method blaMonad in object Test produces a diverging implicit search when trying to match type Monad[F].
+   |Note that open search type => Monad[([X] =>> Bla[F, X])] cannot be re-used as a by-name implicit parameter
+   |since it is not fully defined.

tests/neg/i9568.scala

@@ -0,0 +1,14 @@
+trait Monad[F[_]] {
+  def foo[A](fa: F[A]): Unit = {}
+}
+
+class Bla[F[_], A]
+
+object Test {
+  type Id[A] = A
+
+  val bla: Bla[Id, Unit] = ???
+  implicit def blaMonad[F[_], S](implicit ev: => Monad[F]): Monad[({type L[X] = Bla[F, X]})#L] = ???
+
+  blaMonad.foo(bla) // error: diverges
+}

tests/pos/i9568.scala

@@ -0,0 +1,25 @@
+trait Monad[F[_]] {
+  def foo[A](fa: F[A]): Unit = {}
+}
+
+class Bla[F[_], A]
+
+object Test1 {
+  type Id[A] = A
+
+  val bla: Bla[Id, Unit] = ???
+  implicit def blaMonad[F[_]: Monad, S]: Monad[({type L[X] = Bla[F, X]})#L] = ???
+  implicit def idMonad: Monad[Id] = ???
+
+  blaMonad.foo(bla) // does not diverge
+}
+
+object Test2 {
+  type Id[A] = A
+
+  val bla: Bla[Id, Unit] = ???
+  implicit def blaMonad[F[_], S](implicit ev: => Monad[F]): Monad[({type L[X] = Bla[F, X]})#L] = ???
+  implicit def idMonad: Monad[Id] = ???
+
+  blaMonad.foo(bla) // does not diverge
+}