Backport "Fix declaring product of straight-to-jar compilation" to LTS (#21164)

WojciechMazur · web-flow · commit be60505ac18e · 2024-07-11T10:04:45.000+02:00
Backports #20592 to the LTS branch. PR submitted by the release tooling. [skip ci]
diff --git a/compiler/src/dotty/tools/backend/jvm/ClassfileWriters.scala b/compiler/src/dotty/tools/backend/jvm/ClassfileWriters.scala
@@ -1,6 +1,6 @@
 package dotty.tools.backend.jvm
 
-import java.io.{DataOutputStream, IOException, BufferedOutputStream, FileOutputStream}
+import java.io.{DataOutputStream, File, IOException, BufferedOutputStream, FileOutputStream}
 import java.nio.ByteBuffer
 import java.nio.channels.{ClosedByInterruptException, FileChannel}
 import java.nio.charset.StandardCharsets.UTF_8
@@ -12,7 +12,7 @@ import java.util.zip.{CRC32, Deflater, ZipEntry, ZipOutputStream}
 
 import dotty.tools.dotc.core.Contexts.*
 import dotty.tools.dotc.core.Decorators.em
-import dotty.tools.io.{AbstractFile, PlainFile}
+import dotty.tools.io.{AbstractFile, PlainFile, VirtualFile}
 import dotty.tools.io.PlainFile.toPlainFile
 import BTypes.InternalName
 import scala.util.chaining._
@@ -22,7 +22,6 @@ import scala.language.unsafeNulls
 
 
 class ClassfileWriters(frontendAccess: PostProcessorFrontendAccess) {
-  type NullableFile =  AbstractFile | Null
   import frontendAccess.{compilerSettings, backendReporting}
 
   sealed trait TastyWriter {
@@ -42,7 +41,7 @@ class ClassfileWriters(frontendAccess: PostProcessorFrontendAccess) {
     /**
      * Write a classfile
      */
-    def writeClass(name: InternalName, bytes: Array[Byte], sourceFile: AbstractFile): NullableFile
+    def writeClass(name: InternalName, bytes: Array[Byte], sourceFile: AbstractFile): AbstractFile
 
 
     /**
@@ -87,7 +86,7 @@ class ClassfileWriters(frontendAccess: PostProcessorFrontendAccess) {
     }
 
     private final class SingleClassWriter(underlying: FileWriter) extends ClassfileWriter {
-      override def writeClass(className: InternalName, bytes: Array[Byte], sourceFile: AbstractFile): NullableFile = {
+      override def writeClass(className: InternalName, bytes: Array[Byte], sourceFile: AbstractFile): AbstractFile = {
         underlying.writeFile(classRelativePath(className), bytes)
       }
       override def writeTasty(className: InternalName, bytes: Array[Byte], sourceFile: AbstractFile): Unit = {
@@ -99,7 +98,7 @@ class ClassfileWriters(frontendAccess: PostProcessorFrontendAccess) {
     }
 
     private final class DebugClassWriter(basic: ClassfileWriter, dump: FileWriter) extends ClassfileWriter {
-      override def writeClass(className: InternalName, bytes: Array[Byte], sourceFile: AbstractFile): NullableFile = {
+      override def writeClass(className: InternalName, bytes: Array[Byte], sourceFile: AbstractFile): AbstractFile = {
         val outFile = basic.writeClass(className, bytes, sourceFile)
         dump.writeFile(classRelativePath(className), bytes)
         outFile
@@ -117,7 +116,7 @@ class ClassfileWriters(frontendAccess: PostProcessorFrontendAccess) {
   }
 
   sealed trait FileWriter {
-    def writeFile(relativePath: String, bytes: Array[Byte]): NullableFile
+    def writeFile(relativePath: String, bytes: Array[Byte]): AbstractFile
     def close(): Unit
   }
 
@@ -161,7 +160,7 @@ class ClassfileWriters(frontendAccess: PostProcessorFrontendAccess) {
 
     lazy val crc = new CRC32
 
-    override def writeFile(relativePath: String, bytes: Array[Byte]): NullableFile = this.synchronized {
+    override def writeFile(relativePath: String, bytes: Array[Byte]): AbstractFile = this.synchronized {
       val entry = new ZipEntry(relativePath)
       if (storeOnly) {
         // When using compression method `STORED`, the ZIP spec requires the CRC and compressed/
@@ -178,7 +177,13 @@ class ClassfileWriters(frontendAccess: PostProcessorFrontendAccess) {
       jarWriter.putNextEntry(entry)
       try jarWriter.write(bytes, 0, bytes.length)
       finally jarWriter.flush()
-      null
+      // important detail here, even on Windows, Zinc expects the separator within the jar
+      // to be the system default, (even if in the actual jar file the entry always uses '/').
+      // see https://github.com/sbt/zinc/blob/dcddc1f9cfe542d738582c43f4840e17c053ce81/internal/compiler-bridge/src/main/scala/xsbt/JarUtils.scala#L47
+      val pathInJar = 
+        if File.separatorChar == '/' then relativePath
+        else relativePath.replace('/', File.separatorChar)
+      PlainFile.toPlainFile(Paths.get(s"${file.absolutePath}!$pathInJar"))
     }
 
     override def close(): Unit = this.synchronized(jarWriter.close())
@@ -226,7 +231,7 @@ class ClassfileWriters(frontendAccess: PostProcessorFrontendAccess) {
     private val fastOpenOptions = util.EnumSet.of(StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)
     private val fallbackOpenOptions = util.EnumSet.of(StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING)
 
-    override def writeFile(relativePath: String, bytes: Array[Byte]): NullableFile = {
+    override def writeFile(relativePath: String, bytes: Array[Byte]): AbstractFile = {
       val path = base.resolve(relativePath)
       try {
         ensureDirForPath(base, path)
@@ -275,7 +280,7 @@ class ClassfileWriters(frontendAccess: PostProcessorFrontendAccess) {
       finally out.close()
     }
 
-    override def writeFile(relativePath: String, bytes: Array[Byte]):NullableFile = {
+    override def writeFile(relativePath: String, bytes: Array[Byte]): AbstractFile = {
       val outFile = getFile(base, relativePath)
       writeBytes(outFile, bytes)
       outFile
diff --git a/compiler/src/dotty/tools/backend/jvm/PostProcessor.scala b/compiler/src/dotty/tools/backend/jvm/PostProcessor.scala
@@ -48,7 +48,7 @@ class PostProcessor(val frontendAccess: PostProcessorFrontendAccess, val bTypes:
       if AsmUtils.traceSerializedClassEnabled && internalName.contains(AsmUtils.traceSerializedClassPattern) then
         AsmUtils.traceClass(bytes)
       val clsFile = classfileWriter.writeClass(internalName, bytes, sourceFile)
-      if clsFile != null then clazz.onFileCreated(clsFile)
+      clazz.onFileCreated(clsFile)
   }
 
   def sendToDisk(tasty: GeneratedTasty, sourceFile: AbstractFile): Unit = {
diff --git a/compiler/src/dotty/tools/io/JarArchive.scala b/compiler/src/dotty/tools/io/JarArchive.scala
@@ -10,10 +10,12 @@ import scala.jdk.CollectionConverters.*
  * This class implements an [[AbstractFile]] backed by a jar
  * that be can used as the compiler's output directory.
  */
-class JarArchive private (root: Directory) extends PlainDirectory(root) {
-  def close(): Unit = jpath.getFileSystem().close()
+class JarArchive private (val jarPath: Path, root: Directory) extends PlainDirectory(root) {
+  def close(): Unit = this.synchronized(jpath.getFileSystem().close())
   def allFileNames(): Iterator[String] =
     java.nio.file.Files.walk(jpath).iterator().asScala.map(_.toString)
+
+  override def toString: String = jarPath.toString
 }
 
 object JarArchive {
@@ -39,6 +41,6 @@ object JarArchive {
       }
     }
     val root = fs.getRootDirectories().iterator.next()
-    new JarArchive(Directory(root))
+    new JarArchive(path, Directory(root))
   }
 }
diff --git a/sbt-bridge/test/xsbt/ExtractUsedNamesSpecification.scala b/sbt-bridge/test/xsbt/ExtractUsedNamesSpecification.scala
@@ -1,7 +1,6 @@
 package xsbt
 
 import xsbti.UseScope
-import ScalaCompilerForUnitTesting.Callbacks
 
 import org.junit.{ Test, Ignore }
 import org.junit.Assert._
@@ -227,9 +226,9 @@ class ExtractUsedNamesSpecification {
 
     def findPatMatUsages(in: String): Set[String] = {
       val compilerForTesting = new ScalaCompilerForUnitTesting
-      val (_, Callbacks(callback, _)) =
+      val output =
         compilerForTesting.compileSrcs(List(List(sealedClass, in)))
-      val clientNames = callback.usedNamesAndScopes.view.filterKeys(!_.startsWith("base."))
+      val clientNames = output.analysis.usedNamesAndScopes.view.filterKeys(!_.startsWith("base."))
 
       val names: Set[String] = clientNames.flatMap {
         case (_, usages) =>
diff --git a/sbt-bridge/test/xsbt/ProductsSpecification.scala b/sbt-bridge/test/xsbt/ProductsSpecification.scala
@@ -0,0 +1,34 @@
+package xsbt
+
+import org.junit.Assert.*
+import org.junit.Ignore
+import org.junit.Test
+
+import java.io.File
+import java.nio.file.Path
+import java.nio.file.Paths
+
+class ProductsSpecification {
+
+  @Test
+  def extractProductsFromJar = {
+    val src =
+      """package example
+        |
+        |class A {
+        |  class B
+        |  def foo =
+        |    class C
+        |}""".stripMargin
+    val output = compiler.compileSrcsToJar(src)
+    val srcFile = output.srcFiles.head
+    val products = output.analysis.productClassesToSources.filter(_._2 == srcFile).keys.toSet
+    
+    def toPathInJar(className: String): Path =
+      Paths.get(s"${output.classesOutput}!${className.replace('.', File.separatorChar)}.class")
+    val expected = Set("example.A", "example.A$B", "example.A$C$1").map(toPathInJar)
+    assertEquals(products, expected)
+  }
+
+  private def compiler = new ScalaCompilerForUnitTesting
+}
diff --git a/sbt-bridge/test/xsbt/ScalaCompilerForUnitTesting.scala b/sbt-bridge/test/xsbt/ScalaCompilerForUnitTesting.scala
@@ -1,22 +1,19 @@
 /** Adapted from https://github.com/sbt/sbt/blob/0.13/compile/interface/src/test/scala/xsbt/ScalaCompilerForUnitTesting.scala */
 package xsbt
 
-import xsbti.compile.{CompileProgress, SingleOutput}
-import java.io.File
-import xsbti._
-import sbt.io.IO
-import xsbti.api.{ ClassLike, Def, DependencyContext }
-import DependencyContext._
-import xsbt.api.SameAPI
-import sbt.internal.util.ConsoleLogger
-import dotty.tools.io.PlainFile.toPlainFile
 import dotty.tools.xsbt.CompilerBridge
+import sbt.io.IO
+import xsbti.*
+import xsbti.api.ClassLike
+import xsbti.api.DependencyContext.*
+import xsbti.compile.SingleOutput
+
+import java.io.File
+import java.nio.file.Path
 
 import TestCallback.ExtractedClassDependencies
-import ScalaCompilerForUnitTesting.Callbacks
 
-object ScalaCompilerForUnitTesting:
-  case class Callbacks(analysis: TestCallback, progress: TestCompileProgress)
+case class CompileOutput(srcFiles: Seq[VirtualFileRef], classesOutput: Path, analysis: TestCallback, progress: TestCompileProgress)
 
 /**
  * Provides common functionality needed for unit tests that require compiling
@@ -25,38 +22,33 @@ object ScalaCompilerForUnitTesting:
 class ScalaCompilerForUnitTesting {
 
   def extractEnteredPhases(srcs: String*): Seq[List[String]] = {
-    val (tempSrcFiles, Callbacks(_, testProgress)) = compileSrcs(srcs: _*)
-    val run = testProgress.runs.head
-    tempSrcFiles.map(src => run.unitPhases(src.id))
+    val output = compileSrcs(srcs*)
+    val run = output.progress.runs.head
+    output.srcFiles.map(src => run.unitPhases(src.id))
   }
 
-  def extractTotal(srcs: String*)(extraSourcePath: String*): Int = {
-    val (tempSrcFiles, Callbacks(_, testProgress)) = compileSrcs(List(srcs.toList), extraSourcePath.toList)
-    val run = testProgress.runs.head
-    run.total
-  }
+  def extractTotal(srcs: String*)(extraSourcePath: String*): Int =
+    compileSrcs(List(srcs.toList), extraSourcePath.toList).progress.runs.head.total
 
-  def extractProgressPhases(srcs: String*): List[String] = {
-    val (_, Callbacks(_, testProgress)) = compileSrcs(srcs: _*)
-    testProgress.runs.head.phases
-  }
+  def extractProgressPhases(srcs: String*): List[String] =
+    compileSrcs(srcs*).progress.runs.head.phases
 
   /**
    * Compiles given source code using Scala compiler and returns API representation
    * extracted by ExtractAPI class.
    */
   def extractApiFromSrc(src: String): Seq[ClassLike] = {
-    val (Seq(tempSrcFile), Callbacks(analysisCallback, _)) = compileSrcs(src)
-    analysisCallback.apis(tempSrcFile)
+    val output = compileSrcs(src)
+    output.analysis.apis(output.srcFiles.head)
   }
 
   /**
    * Compiles given source code using Scala compiler and returns API representation
    * extracted by ExtractAPI class.
    */
   def extractApisFromSrcs(srcs: List[String]*): Seq[Seq[ClassLike]] = {
-    val (tempSrcFiles, Callbacks(analysisCallback, _)) = compileSrcs(srcs.toList)
-    tempSrcFiles.map(analysisCallback.apis)
+    val output = compileSrcs(srcs.toList)
+    output.srcFiles.map(output.analysis.apis)
   }
 
   /**
@@ -73,15 +65,16 @@ class ScalaCompilerForUnitTesting {
       assertDefaultScope: Boolean = true
   ): Map[String, Set[String]] = {
     // we drop temp src file corresponding to the definition src file
-    val (Seq(_, tempSrcFile), Callbacks(analysisCallback, _)) = compileSrcs(definitionSrc, actualSrc)
+    val output = compileSrcs(definitionSrc, actualSrc)
+    val analysis = output.analysis
 
     if (assertDefaultScope) for {
-      (className, used) <- analysisCallback.usedNamesAndScopes
-      analysisCallback.TestUsedName(name, scopes) <- used
+      (className, used) <- analysis.usedNamesAndScopes
+      analysis.TestUsedName(name, scopes) <- used
     } assert(scopes.size() == 1 && scopes.contains(UseScope.Default), s"$className uses $name in $scopes")
 
-    val classesInActualSrc = analysisCallback.classNames(tempSrcFile).map(_._1)
-    classesInActualSrc.map(className => className -> analysisCallback.usedNames(className)).toMap
+    val classesInActualSrc = analysis.classNames(output.srcFiles.head).map(_._1)
+    classesInActualSrc.map(className => className -> analysis.usedNames(className)).toMap
   }
 
   /**
@@ -91,11 +84,11 @@ class ScalaCompilerForUnitTesting {
    * Only the names used in the last src file are returned.
    */
   def extractUsedNamesFromSrc(sources: String*): Map[String, Set[String]] = {
-    val (srcFiles, Callbacks(analysisCallback, _)) = compileSrcs(sources: _*)
-    srcFiles
+    val output = compileSrcs(sources*)
+    output.srcFiles
       .map { srcFile =>
-        val classesInSrc = analysisCallback.classNames(srcFile).map(_._1)
-        classesInSrc.map(className => className -> analysisCallback.usedNames(className)).toMap
+        val classesInSrc = output.analysis.classNames(srcFile).map(_._1)
+        classesInSrc.map(className => className -> output.analysis.usedNames(className)).toMap
       }
       .reduce(_ ++ _)
   }
@@ -113,15 +106,15 @@ class ScalaCompilerForUnitTesting {
    * file system-independent way of testing dependencies between source code "files".
    */
   def extractDependenciesFromSrcs(srcs: List[List[String]]): ExtractedClassDependencies = {
-    val (_, Callbacks(testCallback, _)) = compileSrcs(srcs)
+    val analysis = compileSrcs(srcs).analysis
 
-    val memberRefDeps = testCallback.classDependencies collect {
+    val memberRefDeps = analysis.classDependencies collect {
       case (target, src, DependencyByMemberRef) => (src, target)
     }
-    val inheritanceDeps = testCallback.classDependencies collect {
+    val inheritanceDeps = analysis.classDependencies collect {
       case (target, src, DependencyByInheritance) => (src, target)
     }
-    val localInheritanceDeps = testCallback.classDependencies collect {
+    val localInheritanceDeps = analysis.classDependencies collect {
       case (target, src, LocalDependencyByInheritance) => (src, target)
     }
     ExtractedClassDependencies.fromPairs(memberRefDeps, inheritanceDeps, localInheritanceDeps)
@@ -142,12 +135,20 @@ class ScalaCompilerForUnitTesting {
    * The sequence of temporary files corresponding to passed snippets and analysis
    * callback is returned as a result.
    */
-  def compileSrcs(groupedSrcs: List[List[String]], sourcePath: List[String] = Nil): (Seq[VirtualFile], Callbacks) = {
+  def compileSrcs(groupedSrcs: List[List[String]], sourcePath: List[String] = Nil, compileToJar: Boolean = false): CompileOutput = {
       val temp = IO.createTemporaryDirectory
       val analysisCallback = new TestCallback
       val testProgress = new TestCompileProgress
-      val classesDir = new File(temp, "classes")
-      classesDir.mkdir()
+      val classesOutput =
+        if (compileToJar) {
+          val jar = new File(temp, "classes.jar")
+          jar.createNewFile()
+          jar
+        } else {
+          val dir = new File(temp, "classes")
+          dir.mkdir()
+          dir
+        }
 
       val bridge = new CompilerBridge
 
@@ -164,16 +165,16 @@ class ScalaCompilerForUnitTesting {
         }
 
         val virtualSrcFiles = srcFiles.toArray
-        val classesDirPath = classesDir.getAbsolutePath.toString
+        val classesOutputPath = classesOutput.getAbsolutePath()
         val output = new SingleOutput:
-          def getOutputDirectory() = classesDir
+          def getOutputDirectory() = classesOutput
 
         val maybeSourcePath = if extraFiles.isEmpty then Nil else List("-sourcepath", temp.getAbsolutePath.toString)
 
         bridge.run(
           virtualSrcFiles,
           new TestDependencyChanges,
-          Array("-Yforce-sbt-phases", "-classpath", classesDirPath, "-usejavacp", "-d", classesDirPath) ++ maybeSourcePath,
+          Array("-Yforce-sbt-phases", "-classpath", classesOutputPath, "-usejavacp", "-d", classesOutputPath) ++ maybeSourcePath,
           output,
           analysisCallback,
           new TestReporter,
@@ -185,13 +186,16 @@ class ScalaCompilerForUnitTesting {
 
         srcFiles
       }
-      (files.flatten.toSeq, Callbacks(analysisCallback, testProgress))
+      CompileOutput(files.flatten.toSeq, classesOutput.toPath, analysisCallback, testProgress)
   }
 
-  def compileSrcs(srcs: String*): (Seq[VirtualFile], Callbacks) = {
+  def compileSrcs(srcs: String*): CompileOutput = {
     compileSrcs(List(srcs.toList))
   }
 
+  def compileSrcsToJar(srcs: String*): CompileOutput =
+    compileSrcs(List(srcs.toList), compileToJar = true)
+
   private def prepareSrcFile(baseDir: File, fileName: String, src: String): VirtualFile = {
     val srcFile = new File(baseDir, fileName)
     IO.write(srcFile, src)

Original file line number	Diff line number	Diff line change
`@@ -48,7 +48,7 @@ class PostProcessor(val frontendAccess: PostProcessorFrontendAccess, val bTypes:`
`48`	`48`	`if AsmUtils.traceSerializedClassEnabled && internalName.contains(AsmUtils.traceSerializedClassPattern) then`
`49`	`49`	`AsmUtils.traceClass(bytes)`
`50`	`50`	`val clsFile = classfileWriter.writeClass(internalName, bytes, sourceFile)`
`51`		`- if clsFile != null then clazz.onFileCreated(clsFile)`
	`51`	`+ clazz.onFileCreated(clsFile)`
`52`	`52`	`}`
`53`	`53`
`54`	`54`	`def sendToDisk(tasty: GeneratedTasty, sourceFile: AbstractFile): Unit = {`
Original file line number	Diff line number	Diff line change
`@@ -10,10 +10,12 @@ import scala.jdk.CollectionConverters.*`
`10`	`10`	`* This class implements an [[AbstractFile]] backed by a jar`
`11`	`11`	`* that be can used as the compiler's output directory.`
`12`	`12`	`*/`
`13`		`-class JarArchive private (root: Directory) extends PlainDirectory(root) {`
`14`		`- def close(): Unit = jpath.getFileSystem().close()`
	`13`	`+class JarArchive private (val jarPath: Path, root: Directory) extends PlainDirectory(root) {`
	`14`	`+ def close(): Unit = this.synchronized(jpath.getFileSystem().close())`
`15`	`15`	`def allFileNames(): Iterator[String] =`
`16`	`16`	`java.nio.file.Files.walk(jpath).iterator().asScala.map(_.toString)`
	`17`	`+`
	`18`	`+ override def toString: String = jarPath.toString`
`17`	`19`	`}`
`18`	`20`
`19`	`21`	`object JarArchive {`
`@@ -39,6 +41,6 @@ object JarArchive {`
`39`	`41`	`}`
`40`	`42`	`}`
`41`	`43`	`val root = fs.getRootDirectories().iterator.next()`
`42`		`- new JarArchive(Directory(root))`
	`44`	`+ new JarArchive(path, Directory(root))`
`43`	`45`	`}`
`44`	`46`	`}`