Avoid assertion failure when forcing LazyRef while printing

Fixes #12650

The fix that matters for #12650 is the one in Checking. The others generally increase
robustness of printing diagnostics when triggering a LazyRef recursion.

Author: odersky

compiler/src/dotty/tools/dotc/core/TypeComparer.scala

@@ -2396,19 +2396,21 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
     }
 
   /** Show subtype goal that led to an assertion failure */
-  def showGoal(tp1: Type, tp2: Type)(using Context): Unit = {
-    report.echo(i"assertion failure for ${show(tp1)} <:< ${show(tp2)}, frozen = $frozenConstraint")
-    def explainPoly(tp: Type) = tp match {
-      case tp: TypeParamRef => report.echo(s"TypeParamRef ${tp.show} found in ${tp.binder.show}")
-      case tp: TypeRef if tp.symbol.exists => report.echo(s"typeref ${tp.show} found in ${tp.symbol.owner.show}")
-      case tp: TypeVar => report.echo(s"typevar ${tp.show}, origin = ${tp.origin}")
-      case _ => report.echo(s"${tp.show} is a ${tp.getClass}")
-    }
-    if (Config.verboseExplainSubtype) {
-      explainPoly(tp1)
-      explainPoly(tp2)
-    }
-  }
+  def showGoal(tp1: Type, tp2: Type)(using Context): Unit =
+    try
+      report.echo(i"assertion failure for ${show(tp1)} <:< ${show(tp2)}, frozen = $frozenConstraint")
+      def explainPoly(tp: Type) = tp match {
+        case tp: TypeParamRef => report.echo(s"TypeParamRef ${tp.show} found in ${tp.binder.show}")
+        case tp: TypeRef if tp.symbol.exists => report.echo(s"typeref ${tp.show} found in ${tp.symbol.owner.show}")
+        case tp: TypeVar => report.echo(s"typevar ${tp.show}, origin = ${tp.origin}")
+        case _ => report.echo(s"${tp.show} is a ${tp.getClass}")
+      }
+      if (Config.verboseExplainSubtype) {
+        explainPoly(tp1)
+        explainPoly(tp2)
+      }
+    catch case NonFatal(ex) =>
+      report.echo(s"assertion failure [[cannot display since $ex was thrown]]")
 
   /** Record statistics about the total number of subtype checks
    *  and the number of "successful" subtype checks, i.e. checks

compiler/src/dotty/tools/dotc/core/Types.scala

@@ -2833,7 +2833,9 @@ object Types {
         if myRef == null then
           // if errors were reported previously handle this by throwing a CyclicReference
           // instead of crashing immediately. A test case is neg/i6057.scala.
-          assert(ctx.mode.is(Mode.CheckCyclic) || ctx.reporter.errorsReported)
+          assert(ctx.mode.is(Mode.CheckCyclic)
+              || ctx.mode.is(Mode.Printing)
+              || ctx.reporter.errorsReported)
           throw CyclicReference(NoDenotation)
       else
         computed = true

compiler/src/dotty/tools/dotc/typer/Checking.scala

@@ -403,11 +403,11 @@ object Checking {
       return
 
     def qualifies(sym: Symbol) = sym.name.isTypeName && !sym.is(Private)
-    val abstractTypeNames =
-      for (parent <- parents; mbr <- parent.abstractTypeMembers if qualifies(mbr.symbol))
-      yield mbr.name.asTypeName
-
     withMode(Mode.CheckCyclic) {
+      val abstractTypeNames =
+        for (parent <- parents; mbr <- parent.abstractTypeMembers if qualifies(mbr.symbol))
+        yield mbr.name.asTypeName
+
       for name <- abstractTypeNames do
         try
           val mbr = joint.member(name)

compiler/test/dotty/tools/dotc/CompilationTests.scala

@@ -147,6 +147,7 @@ class CompilationTests {
       compileFile("tests/neg-custom-args/i3882.scala", allowDeepSubtypes),
       compileFile("tests/neg-custom-args/i4372.scala", allowDeepSubtypes),
       compileFile("tests/neg-custom-args/i1754.scala", allowDeepSubtypes),
+      compileFile("tests/neg-custom-args/i12650.scala", allowDeepSubtypes),
       compileFile("tests/neg-custom-args/i9517.scala", defaultOptions.and("-Xprint-types")),
       compileFile("tests/neg-custom-args/i11637.scala", defaultOptions.and("-explain")),
       compileFile("tests/neg-custom-args/interop-polytypes.scala", allowDeepSubtypes.and("-Yexplicit-nulls")),

tests/neg-custom-args/i12650.check

@@ -0,0 +1,4 @@
+-- Error: tests/neg-custom-args/i12650.scala:2:58 ----------------------------------------------------------------------
+2 |  type This <: FooBase { type This <: FooBase.this.This } & FooBase { type This <: FooBase.this.This } // error
+  |               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  |               cyclic reference involving type This

tests/neg-custom-args/i12650.scala

@@ -0,0 +1,3 @@
+trait FooBase {
+  type This <: FooBase { type This <: FooBase.this.This } & FooBase { type This <: FooBase.this.This } // error
+}