Remove custom path for best effort compilation artifacts

Now best efforttasty files will be always saved in the
`<output>/META-INF/best-effort` directory, similar to the semanticdb
files.

Author: jchyb

compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -376,9 +376,9 @@ private sealed trait YSettings:
       //.withPostSetHook( _ => YprofileEnabled.value = true )
   val YprofileRunGcBetweenPhases: Setting[List[String]] = PhasesSetting("-Yprofile-run-gc", "Run a GC between phases - this allows heap size to be accurate at the expense of more time. Specify a list of phases, or *", "_")
       //.withPostSetHook( _ => YprofileEnabled.value = true )
-  
-  val YbestEffortDir: Setting[String] = StringSetting("-Ybest-effort-dir", "dir", "Enable best-effort compilation attempting to produce tasty in case of failure to specified directory, as part of the pickler phase.", "")
-  val YwithBestEffortTasty: Setting[Boolean] = BooleanSetting("-Ywith-best-effort-tasty", "Allow to compile from a best effort tasty files. If such file is used, the compiler will stop after the pickler phase.")
+
+  val YbestEffort: Setting[Boolean] = BooleanSetting("-Ybest-effort", "Enable best-effort compilation attempting to produce tasty to the META-INF/best-effort directory, regardless of errors, as part of the pickler phase.")
+  val YwithBestEffortTasty: Setting[Boolean] = BooleanSetting("-Ywith-best-effort-tasty", "Allow to compile from best-effort tasty files. If such file is used, the compiler will stop after the pickler phase.")
 
   // Experimental language features
   val YnoKindPolymorphism: Setting[Boolean] = BooleanSetting("-Yno-kind-polymorphism", "Disable kind polymorphism.")

compiler/src/dotty/tools/dotc/core/Contexts.scala

@@ -454,7 +454,7 @@ object Contexts {
     def explicitNulls: Boolean = base.settings.YexplicitNulls.value
 
     /** Is best-effort-dir option set? */
-    def isBestEffort: Boolean = base.settings.YbestEffortDir.value.nonEmpty
+    def isBestEffort: Boolean = base.settings.YbestEffort.value
 
     /** Is the from-best-effort-tasty option set to true? */
     def withBestEffortTasty: Boolean = base.settings.YwithBestEffortTasty.value

compiler/src/dotty/tools/dotc/core/tasty/BestEffortTastyWriter.scala

@@ -11,7 +11,7 @@ import dotty.tools.dotc.core.Contexts.Context
 
 object BestEffortTastyWriter:
 
-  def write(dir: JPath, units: List[CompilationUnit])(using Context): Unit = 
+  def write(dir: JPath, units: List[CompilationUnit])(using Context): Unit =
     if JFiles.exists(dir) then JFiles.createDirectories(dir)
 
     units.foreach { unit =>
@@ -37,7 +37,7 @@ object BestEffortTastyWriter:
       case last :: Nil =>
         val name = last.stripSuffix("$")
         acc.resolve(s"$name.betasty")
-      case pkg :: tail => 
+      case pkg :: tail =>
         val next = acc.resolve(pkg)
         if !JFiles.exists(next) then JFiles.createDirectory(next)
         outputPath(tail, next)

compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala

@@ -51,7 +51,7 @@ import scala.annotation.internal.sharable
  *  @param reader              the reader from which to unpickle
  *  @param posUnpicklerOpt     the unpickler for positions, if it exists
  *  @param commentUnpicklerOpt the unpickler for comments, if it exists
- *  @param isBestEffortTasty   decides whether to unpickle as a Best Effort TASTy 
+ *  @param isBestEffortTasty   decides whether to unpickle as a Best Effort TASTy
  */
 class TreeUnpickler(reader: TastyReader,
                     nameAtRef: NameTable,
@@ -1015,7 +1015,7 @@ class TreeUnpickler(reader: TastyReader,
      *                   but skip constructor arguments. Return any trees that were partially
      *                   parsed in this way as InferredTypeTrees.
      *  @param  end      specifies last address that can be read. Should be set for Best Effort
-     *                   TASTy compilation, otherwise should be None 
+     *                   TASTy compilation, otherwise should be None
      */
     def readParents(withArgs: Boolean, end: Option[Addr])(using Context): List[Tree] =
       collectWhile(nextByte != SELFDEF && nextByte != DEFDEF && end.map(_ != currentAddr).getOrElse(true)) {

compiler/src/dotty/tools/dotc/transform/Pickler.scala

@@ -16,7 +16,8 @@ import reporting.{ThrowingReporter, Profile, Message}
 import collection.mutable
 import util.concurrent.{Executor, Future}
 import compiletime.uninitialized
-import java.nio.file.Paths
+import java.nio.file.Files
+import scala.language.unsafeNulls
 
 object Pickler {
   val name: String = "pickler"
@@ -162,8 +163,12 @@ class Pickler extends Phase {
             .addMode(Mode.ReadPositions)
             .addMode(Mode.PrintShowExceptions))
     if ctx.isBestEffort then
-      val dir = Paths.get(ctx.settings.YbestEffortDir.value)
-      BestEffortTastyWriter.write(dir.nn, units)
+      val outpath =
+        ctx.settings.outputDir.value.jpath.toAbsolutePath.normalize
+          .resolve("META-INF")
+          .resolve("best-effort")
+      Files.createDirectories(outpath)
+      BestEffortTastyWriter.write(outpath.nn, units)
     result
   }
 

compiler/test/dotty/tools/vulpix/ParallelTesting.scala

@@ -641,13 +641,13 @@ trait ParallelTesting extends RunnerOrchestration { self =>
         }
 
       reporter
-    
+
     protected def compileFromBestEffortTasty(flags0: TestFlags, suppressErrors: Boolean, targetDir: JFile): TestReporter = {
       val classes = flattenFiles(targetDir).filter(isBestEffortTastyFile).map(_.toString)
       val flags = flags0 and "-from-tasty" and "-Ywith-best-effort-tasty"
       val reporter = mkReporter
       val driver = new Driver
-      
+
       driver.process(flags.all ++ classes, reporter = reporter)
 
       reporter
@@ -660,7 +660,7 @@ trait ParallelTesting extends RunnerOrchestration { self =>
       println((flags.all ++ files0.map(_.toString) ++ Array(bestEffortDir.toString)).mkString(" "))
 
       val args = Array("-classpath", flags.defaultClassPath + ":" + bestEffortDir.toString) ++ flags.options
-      
+
       driver.process(args ++ files0.map(_.toString), reporter = reporter)
 
       reporter
@@ -1271,20 +1271,6 @@ trait ParallelTesting extends RunnerOrchestration { self =>
     targetDir
   }
 
-  private def createBestEffortDirForDir(d: JFile, sourceDir: JFile, outDir: String): JFile = {
-    val uniqueSubdir = s"${d.getName}-best-effort"
-    val bestEffortDir = new JFile(outDir + s"${sourceDir.getName}${JFile.separatorChar}$uniqueSubdir")
-    bestEffortDir.mkdirs()
-    bestEffortDir
-  }
-
-  private def createBestEffortDirForFile(file: JFile, sourceDir: JFile, outDir: String): JFile = {
-    val uniqueSubdir = s"${file.getName.substring(0, file.getName.lastIndexOf('.'))}-best-effort"
-    val bestEffortDir = new JFile(outDir + s"${sourceDir.getName}${JFile.separatorChar}$uniqueSubdir")
-    bestEffortDir.mkdirs()
-    bestEffortDir
-  }
-
   /** Make sure that directory string is as expected */
   private def checkRequirements(f: String, sourceDir: JFile, outDir: String): Unit = {
     require(sourceDir.isDirectory && sourceDir.exists, "passed non-directory to `compileFilesInDir`: " + sourceDir)
@@ -1487,16 +1473,16 @@ trait ParallelTesting extends RunnerOrchestration { self =>
   }
 
   /** A two step compilation test for best effort compilation pickling and unpickling.
-   * 
-   *  First, erroring neg test files are compiled with the `-Ybest-effort-dir` option
-   *  to a best effort directory. If successful, then the produced Best Effort TASTy is
-   *  re-compiled with '-Ywith-best-effort-tasty' to test the TastyReader for Best Effort TASTy.
+   *
+   *  First, erroring neg test files are compiled with the `-Ybest-effort` option.
+   *  If successful, then the produced Best Effort TASTy is re-compiled with
+   *  '-Ywith-best-effort-tasty' to test the TastyReader for Best Effort TASTy.
    */
   def compileBestEffortTastyInDir(f: String, flags: TestFlags, picklingFilter: FileFilter, unpicklingFilter: FileFilter)(
       implicit testGroup: TestGroup): BestEffortCompilationTest = {
-    val bestEffortFlag = "-Ybest-effort-dir"
-    assert(!flags.options.contains(bestEffortFlag), "Best effort directory should not be added manually")
-    
+    val bestEffortFlag = "-Ybest-effort"
+    assert(!flags.options.contains(bestEffortFlag), "Best effort compilation flag should not be added manually")
+
     val outDir = defaultOutputDir + testGroup + JFile.separator
     val sourceDir = new JFile(f)
     checkRequirements(f, sourceDir, outDir)
@@ -1510,15 +1496,14 @@ trait ParallelTesting extends RunnerOrchestration { self =>
       name: String,
       file: JFile,
       flags: TestFlags,
-      bestEffortDir: JFile,
       outputDir: JFile
-    ) extends JointCompilationSource(name, Array(file), flags.and(bestEffortFlag).and(bestEffortDir.toString), outputDir) {
+    ) extends JointCompilationSource(name, Array(file), flags.and(bestEffortFlag), outputDir) {
       override def buildInstructions(errors: Int, warnings: Int): String = {
         s"""|
             |Test '$title' compiled with a compiler crash,
             |the test can be reproduced by running:
             |
-            |  sbt "scalac -Ybest-effort-dir $bestEffortDir $file"
+            |  sbt "scalac -Ybest-effort $file"
             |
             |These tests can be disabled by adding `${file.getName}` to `compiler${JFile.separator}test${JFile.separator}dotc${JFile.separator}neg-best-effort-pickling.blacklist`
             |""".stripMargin
@@ -1533,7 +1518,7 @@ trait ParallelTesting extends RunnerOrchestration { self =>
     ) extends JointCompilationSource(name, Array(file), flags, bestEffortDir, fromTasty = FromBestEffortTasty) {
 
       override def buildInstructions(errors: Int, warnings: Int): String = {
-        def beTastyFiles(file: JFile): Array[JFile] = 
+        def beTastyFiles(file: JFile): Array[JFile] =
           file.listFiles.flatMap { innerFile =>
             if (innerFile.isDirectory) beTastyFiles(innerFile)
             else if (isBestEffortTastyFile(innerFile)) Array(innerFile)
@@ -1544,7 +1529,7 @@ trait ParallelTesting extends RunnerOrchestration { self =>
             |Test '$title' compiled with a compiler crash,
             |the test can be reproduced by running:
             |
-            |  sbt "scalac -Ybest-effort-dir $bestEffortDir $file"
+            |  sbt "scalac -Ybest-effort $file"
             |  sbt "scalac --from-tasty -Ywith-best-effort-tasty $beTastyFilesString"
             |
             |These tests can be disabled by adding `${file.getName}` to `compiler${JFile.separator}test${JFile.separator}dotc${JFile.separator}neg-best-effort-unpickling.blacklist`
@@ -1553,20 +1538,22 @@ trait ParallelTesting extends RunnerOrchestration { self =>
       }
     }
 
-    val (bestEffortTargets, bestEffortDirs) =
+    val (bestEffortTargets, targetAndBestEffortDirs) =
       filteredPicklingFiles.map { f =>
-        val bestEffortDir = createBestEffortDirForFile(f, sourceDir, outDir)
         val outputDir = createOutputDirsForFile(f, sourceDir, outDir)
+        val bestEffortDir = new JFile(outputDir, s"META-INF${JFile.separator}best-effort")
         (
-          BestEffortCompilation(testGroup.name, f, flags, bestEffortDir, outputDir),
-          bestEffortDir
+          BestEffortCompilation(testGroup.name, f, flags, outputDir),
+          (f, bestEffortDir)
         )
       }.unzip
+    val (_, bestEffortDirs) = targetAndBestEffortDirs.unzip
+    val fileToBestEffortDirMap = targetAndBestEffortDirs.toMap
 
     val picklingSet = filteredPicklingFiles.toSet
     val fromTastyTargets =
       filteredUnpicklingFiles.filter(picklingSet.contains(_)).map { f =>
-        val bestEffortDir = createBestEffortDirForFile(f, sourceDir, outDir)
+        val bestEffortDir = fileToBestEffortDirMap(f)
         new JointCompilationSourceFromBestEffortTasty(testGroup.name, f, flags, bestEffortDir)
       }
 
@@ -1579,13 +1566,13 @@ trait ParallelTesting extends RunnerOrchestration { self =>
   }
 
   /** A two step integration test for best effort compilation.
-   * 
-   *  Directories found in the directory `f` represent separate tests and must contain 
+   *
+   *  Directories found in the directory `f` represent separate tests and must contain
    *  the 'err' and 'main' directories. First the (erroring) contents of the 'err'
-   *  directory are compiled with the `Ybest-effort-dir` option with a best effort directory.
+   *  directory are compiled with the `Ybest-effort` option.
    *  Then, are the contents of 'main' are compiled with the previous best effort directory
    *  on the classpath using the option `-Ywith-best-effort-tasty`.
-   * 
+   *
    */
   def compileBestEffortIntegration(f: String, flags: TestFlags)(implicit testGroup: TestGroup) = {
     val sourceDir = new JFile(f)
@@ -1599,24 +1586,23 @@ trait ParallelTesting extends RunnerOrchestration { self =>
       val step1SourceFiles = step1SourceDir.listFiles
       val step2SourceFiles = step2SourceDir.listFiles
 
-      val outDir = defaultOutputDir + testGroup + JFile.separator
+      val outDir = defaultOutputDir + testGroup + JFile.separator + dir.getName().toString + JFile.separator
 
       val step1OutDir = createOutputDirsForDir(step1SourceDir, step1SourceDir, outDir)
       val step2OutDir = createOutputDirsForDir(step2SourceDir, step2SourceDir, outDir)
 
-      val bestEffortDir = createBestEffortDirForDir(dir, sourceDir, outDir)
-
       val step1Compilation = JointCompilationSource(
-        testGroup.name, step1SourceFiles, flags.and("-Ybest-effort-dir").and(bestEffortDir.toString), step1OutDir, fromTasty = NotFromTasty
+        testGroup.name, step1SourceFiles, flags.and("-Ybest-effort"), step1OutDir, fromTasty = NotFromTasty
         )
+
+      val bestEffortDir = new JFile(step1OutDir, s"META-INF${JFile.separator}best-effort")
+
       val step2Compilation = JointCompilationSource(
-        testGroup.name, step2SourceFiles, flags.and("-Ywith-best-effort-tasty"), step1OutDir, fromTasty = WithBestEffortTasty(bestEffortDir)
+        testGroup.name, step2SourceFiles, flags.and("-Ywith-best-effort-tasty"), step2OutDir, fromTasty = WithBestEffortTasty(bestEffortDir)
         )
       (step1Compilation, step2Compilation, bestEffortDir)
     }.unzip3
-    println(step1Targets)
-    println(step2Targets)
-    
+
     BestEffortCompilationTest(
       new CompilationTest(step1Targets).keepOutput,
       new CompilationTest(step2Targets).keepOutput,
@@ -1653,7 +1639,7 @@ trait ParallelTesting extends RunnerOrchestration { self =>
   }
 
   class BestEffortCompilationTest(step1: CompilationTest, step2: CompilationTest, bestEffortDirs: List[JFile], shouldDelete: Boolean)(implicit testGroup: TestGroup) {
-  
+
     def checkNoCrash()(implicit summaryReport: SummaryReporting): this.type = {
       step1.checkNoCrash() // Compile all files to generate the class files with best effort tasty
       step2.checkNoCrash() // Compile with best effort tasty
@@ -1717,7 +1703,7 @@ object ParallelTesting {
 
   def isTastyFile(f: JFile): Boolean =
     f.getName.endsWith(".tasty")
-  
+
   def isBestEffortTastyFile(f: JFile): Boolean =
     f.getName.endsWith(".betasty")
 

docs/_docs/internals/best-effort-compilation.md

@@ -7,12 +7,12 @@ Best-effort compilation is a compilation mode introduced with the aim of improvi
 tast6y-like artifacts and semanticdb files in erroring programs.
 
 It is composed of two experimental compiler options:
-* `-Ybest-effort-dir dir` produces Best Effort TASTy (`.betasty`) files to a separete `dir` best effort target directory
-* `-Ywith-best-effort-tasty` allows to read BestEffortTASTy, and if such file is read from the classpath then
-limits compilation to the frontend phases
+* `-Ybest-effort` produces Best Effort TASTy (`.betasty`) files to the `META-INF/best-effort` directory
+* `-Ywith-best-effort-tasty` allows to read Best Effort TASTy files, and if such file is read from the classpath then
+limits compilation to the frontend phases (for use in IDEs)
 
 This feature aims to force through to the typer phase regardless of errors, and then serialize tasty-like files
-obtained from the error trees into the best effort directory and also serialize semanticdb as normal.
+obtained from the error trees into the best effort directory (`META-INF/best-effort`) and also serialize semanticdb as normal.
 
 The exact execution pattern is as follows:
 
@@ -47,7 +47,7 @@ Pickler (with added printing of best effort tasty to the best effort target dire
 End compilation                      Execute latter passes
 ```
 
-This is because the IDE is able to retrieve useful info even when skipping phases like PostTyper. 
+This is because the IDE is able to retrieve useful info even when skipping phases like PostTyper.
 
 This execution structure where we skip phases depending on the errors found is motivated by the desire
 to avoid additionally handling errored trees in as many phases as possible, therefore also decreasing
@@ -56,12 +56,12 @@ with errors from typer and usually the IDE is able to retrieve enough informatio
 
 `-Ywith-best-effort-tasty` option allows reading Best Effort TASTy files from classpath. If such file is read, then
 the compiler is disallowed from proceeding to any non-frontend phase. This is to be used either in combination with
-`-Ybest-effort-dir` option to produce Best Effort TASTy using failing dependencies, or in the Presentation Compiler
+`-Ybest-effort` option to produce Best Effort TASTy using failing dependencies, or in the Presentation Compiler
 to access symbols derived from failing projects.
 
 ## Best Effort TASTy format
 
-The Best Effort TASTy (`.betasty`) format is a file format produced by the compiler when the `-Ybest-effort-dir` option
+The Best Effort TASTy (`.betasty`) format is a file format produced by the compiler when the `-Ybest-effort` option
 is used. It is characterised by a different header and an addition of the `ERRORtype` type, which represents errored types in
 the compiler. The Best Effort TASTy format also informally extends the regular TASTy grammar to allow the handling of as
 large amount of incorrect trees produced by the compiler as possible. Because of this informality and unlike the regular
@@ -70,7 +70,7 @@ TASTy format, the BETASTy format should stay internal to the compiler.
 ## Testing
 
 The testing procedure reuses the `tests/neg` negative tests that are usually meant to produce errors. First they are compiled
-with the `-Ybest-effort-dir` option (testing the TreePickler for errored trees), then later, the tree is reconstructed using
+with the `-Ybest-effort` option (testing the TreePickler for errored trees), then later, the tree is reconstructed using
 the previously created Best Effort TASTy, with `-Yread-tasty` and `-Ywith-best-effort-tasty` options. This is to test the
 TreeUnpickler for those Best Effort TASTy files.