Introduce best-effort compilation for IDEs

2 new experimental options are introduces for the compiler:
`-Ybest-effort-dir` and `-Ywith-best-effort-tasty`. A related
Best Effort TASTy format, a TASTy aligned file format able to hold some
errored trees. Behaviour of the options and the format is documented as
part of this commit in the `best-effort-compilation.md` docs file.

Author: jchyb

compiler/src/dotty/tools/backend/jvm/GenBCode.scala

@@ -16,6 +16,8 @@ class GenBCode extends Phase { self =>
 
   override def description: String = GenBCode.description
 
+  override def isRunnable(using Context) = super.isRunnable && !ctx.usesBestEffortTasty
+
   private val superCallsMap = new MutableSymbolMap[Set[ClassSymbol]]
   def registerSuperCall(sym: Symbol, calls: ClassSymbol): Unit = {
     val old = superCallsMap.getOrElse(sym, Set.empty)

compiler/src/dotty/tools/backend/sjs/GenSJSIR.scala

@@ -12,7 +12,7 @@ class GenSJSIR extends Phase {
   override def description: String = GenSJSIR.description
 
   override def isRunnable(using Context): Boolean =
-    super.isRunnable && ctx.settings.scalajs.value
+    super.isRunnable && ctx.settings.scalajs.value && !ctx.usesBestEffortTasty
 
   def run(using Context): Unit =
     new JSCodeGen().run()

compiler/src/dotty/tools/dotc/Driver.scala

@@ -100,8 +100,8 @@ class Driver {
             None
           else file.extension match
             case "jar" => Some(file.path)
-            case "tasty" =>
-              TastyFileUtil.getClassPath(file) match
+            case "tasty" | "betasty" =>
+              TastyFileUtil.getClassPath(file, ctx.withBestEffortTasty) match
                 case Some(classpath) => Some(classpath)
                 case _ =>
                   report.error(em"Could not load classname from: ${file.path}")

compiler/src/dotty/tools/dotc/Run.scala

@@ -225,6 +225,13 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
       if (ctx.settings.YtestPickler.value) List("pickler")
       else ctx.settings.YstopAfter.value
 
+    var forceReachPhaseMaybe =
+      if (ctx.isBestEffort) Some("typer")
+      else None
+
+    var reachedSemanticDB = false
+    var reachedPickler = false
+
     val pluginPlan = ctx.base.addPluginPhases(ctx.base.phasePlan)
     val phases = ctx.base.fusePhases(pluginPlan,
       ctx.settings.Yskip.value, ctx.settings.YstopBefore.value, stopAfter, ctx.settings.Ycheck.value)
@@ -239,7 +246,7 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
       var phasesWereAdjusted = false
 
       for (phase <- ctx.base.allPhases)
-        if (phase.isRunnable)
+        if (phase.isRunnable || forceReachPhaseMaybe.nonEmpty)
           Stats.trackTime(s"$phase ms ") {
             val start = System.currentTimeMillis
             val profileBefore = profiler.beforePhase(phase)
@@ -249,6 +256,24 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
               for (unit <- units)
                 lastPrintedTree =
                   printTree(lastPrintedTree)(using ctx.fresh.setPhase(phase.next).setCompilationUnit(unit))
+            
+            forceReachPhaseMaybe match {
+              case Some(forceReachPhase) if phase.phaseName == forceReachPhase =>
+                forceReachPhaseMaybe = None
+              case _ =>
+            }
+
+            if phase.phaseName == "extractSemanticDB" then reachedSemanticDB = true
+            if phase.phaseName == "pickler" then reachedPickler = true
+
+            if !reachedSemanticDB && forceReachPhaseMaybe.isEmpty && ctx.reporter.hasErrors && ctx.isBestEffort then
+              ctx.base.allPhases.find(_.phaseName == "extractSemanticDB").foreach(_.runOn(units))
+              reachedSemanticDB = true
+
+            if !reachedPickler && forceReachPhaseMaybe.isEmpty && ctx.reporter.hasErrors && ctx.isBestEffort then
+              ctx.base.allPhases.find(_.phaseName == "pickler").foreach(_.runOn(units))
+              reachedPickler = true
+
             report.informTime(s"$phase ", start)
             Stats.record(s"total trees at end of $phase", ast.Trees.ntrees)
             for (unit <- units)

compiler/src/dotty/tools/dotc/ast/TreeInfo.scala

@@ -886,12 +886,12 @@ trait TypedTreeInfo extends TreeInfo[Type] { self: Trees.Instance[Type] =>
       else cpy.PackageDef(tree)(pid, slicedStats) :: Nil
     case tdef: TypeDef =>
       val sym = tdef.symbol
-      assert(sym.isClass)
+      if !ctx.isBestEffort then assert(sym.isClass)
       if (cls == sym || cls == sym.linkedClass) tdef :: Nil
       else Nil
     case vdef: ValDef =>
       val sym = vdef.symbol
-      assert(sym.is(Module))
+      if !ctx.isBestEffort then assert(sym.is(Module))
       if (cls == sym.companionClass || cls == sym.moduleClass) vdef :: Nil
       else Nil
     case tree =>

compiler/src/dotty/tools/dotc/ast/tpd.scala

@@ -48,7 +48,7 @@ object tpd extends Trees.Instance[Type] with TypedTreeInfo {
     case _: RefTree | _: GenericApply | _: Inlined | _: Hole =>
       ta.assignType(untpd.Apply(fn, args), fn, args)
     case _ =>
-      assert(ctx.reporter.errorsReported)
+      assert(ctx.isBestEffort || ctx.usesBestEffortTasty || ctx.reporter.errorsReported)
       ta.assignType(untpd.Apply(fn, args), fn, args)
 
   def TypeApply(fn: Tree, args: List[Tree])(using Context): TypeApply = fn match
@@ -57,7 +57,7 @@ object tpd extends Trees.Instance[Type] with TypedTreeInfo {
     case _: RefTree | _: GenericApply =>
       ta.assignType(untpd.TypeApply(fn, args), fn, args)
     case _ =>
-      assert(ctx.reporter.errorsReported)
+      assert(ctx.isBestEffort || ctx.usesBestEffortTasty || ctx.reporter.errorsReported)
       ta.assignType(untpd.TypeApply(fn, args), fn, args)
 
   def Literal(const: Constant)(using Context): Literal =

compiler/src/dotty/tools/dotc/classpath/DirectoryClassPath.scala

@@ -286,7 +286,7 @@ case class DirectoryClassPath(dir: JFile) extends JFileDirectoryLookup[ClassFile
   }
 
   protected def createFileEntry(file: AbstractFile): ClassFileEntryImpl = ClassFileEntryImpl(file)
-  protected def isMatchingFile(f: JFile): Boolean = f.isClass
+  protected def isMatchingFile(f: JFile): Boolean = f.isClass || f.isBestEffortTasty
 
   private[dotty] def classes(inPackage: PackageName): Seq[ClassFileEntry] = files(inPackage)
 }

compiler/src/dotty/tools/dotc/classpath/FileUtils.scala

@@ -37,6 +37,10 @@ object FileUtils {
 
     def isClass: Boolean = file.isFile && file.getName.endsWith(".class") && !file.getName.endsWith("$class.class")
       // FIXME: drop last condition when we stop being compatible with Scala 2.11
+
+    def isTasty: Boolean = file.isFile && file.getName.endsWith(".tasty")
+
+    def isBestEffortTasty: Boolean = file.isFile && file.getName.endsWith(".betasty")
   }
 
   private val SUFFIX_CLASS = ".class"

compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -376,6 +376,9 @@ private sealed trait YSettings:
       //.withPostSetHook( _ => YprofileEnabled.value = true )
   val YprofileRunGcBetweenPhases: Setting[List[String]] = PhasesSetting("-Yprofile-run-gc", "Run a GC between phases - this allows heap size to be accurate at the expense of more time. Specify a list of phases, or *", "_")
       //.withPostSetHook( _ => YprofileEnabled.value = true )
+  
+  val YbestEffortDir: Setting[String] = StringSetting("-Ybest-effort-dir", "dir", "Enable best-effort compilation attempting to produce tasty in case of failure to specified directory, as part of the pickler phase.", "")
+  val YwithBestEffortTasty: Setting[Boolean] = BooleanSetting("-Ywith-best-effort-tasty", "Allow to compile from a best effort tasty files. If such file is used, the compiler will stop after the pickler phase.")
 
   // Experimental language features
   val YnoKindPolymorphism: Setting[Boolean] = BooleanSetting("-Yno-kind-polymorphism", "Disable kind polymorphism.")

compiler/src/dotty/tools/dotc/core/Contexts.scala

@@ -453,6 +453,18 @@ object Contexts {
     /** Is the explicit nulls option set? */
     def explicitNulls: Boolean = base.settings.YexplicitNulls.value
 
+    /** Is best-effort-dir option set? */
+    def isBestEffort: Boolean = base.settings.YbestEffortDir.value.nonEmpty
+
+    /** Is the from-best-effort-tasty option set to true? */
+    def withBestEffortTasty: Boolean = base.settings.YwithBestEffortTasty.value
+
+    /** Were any best effort tasty dependencies used during compilation? */
+    def usesBestEffortTasty: Boolean = base.usedBestEffortTasty
+
+    /** Confirm that a best effort tasty dependency was used during compilation. */
+    def setUsesBestEffortTasty(): Unit = base.usedBestEffortTasty = true
+
     /** A fresh clone of this context embedded in this context. */
     def fresh: FreshContext = freshOver(this)
 
@@ -939,6 +951,9 @@ object Contexts {
     val sources: util.HashMap[AbstractFile, SourceFile] = util.HashMap[AbstractFile, SourceFile]()
     val files: util.HashMap[TermName, AbstractFile] = util.HashMap()
 
+    /** Was best effort file used during compilation? */
+    private[core] var usedBestEffortTasty = false
+
     // Types state
     /** A table for hash consing unique types */
     private[core] val uniques: Uniques = Uniques()