Fix declaring product of straight-to-jar compilation (#20592)

Currently the straight-to-JAR compilation does not declare any class
file to Zinc, which prevents Zinc from compiling downstream modules
incrementally.

For the context, we would like to enable straight-to-JAR compilation in
sbt 2.x, and put remote caching on top of it.

Author: bishabosha

compiler/src/dotty/tools/backend/jvm/ClassfileWriters.scala

@@ -1,6 +1,6 @@
 package dotty.tools.backend.jvm
 
-import java.io.{DataOutputStream, IOException, BufferedOutputStream, FileOutputStream}
+import java.io.{DataOutputStream, File, IOException, BufferedOutputStream, FileOutputStream}
 import java.nio.ByteBuffer
 import java.nio.channels.{ClosedByInterruptException, FileChannel}
 import java.nio.charset.StandardCharsets.UTF_8
@@ -12,7 +12,7 @@ import java.util.zip.{CRC32, Deflater, ZipEntry, ZipOutputStream}
 
 import dotty.tools.dotc.core.Contexts.*
 import dotty.tools.dotc.core.Decorators.em
-import dotty.tools.io.{AbstractFile, PlainFile}
+import dotty.tools.io.{AbstractFile, PlainFile, VirtualFile}
 import dotty.tools.io.PlainFile.toPlainFile
 import BTypes.InternalName
 import scala.util.chaining.*
@@ -26,7 +26,6 @@ import scala.language.unsafeNulls
  * Until then, any changes to this file should be copied to `dotty.tools.io.FileWriters` as well.
  */
 class ClassfileWriters(frontendAccess: PostProcessorFrontendAccess) {
-  type NullableFile =  AbstractFile | Null
   import frontendAccess.{compilerSettings, backendReporting}
 
   sealed trait TastyWriter {
@@ -46,7 +45,7 @@ class ClassfileWriters(frontendAccess: PostProcessorFrontendAccess) {
     /**
      * Write a classfile
      */
-    def writeClass(name: InternalName, bytes: Array[Byte], sourceFile: AbstractFile): NullableFile
+    def writeClass(name: InternalName, bytes: Array[Byte], sourceFile: AbstractFile): AbstractFile
 
 
     /**
@@ -91,7 +90,7 @@ class ClassfileWriters(frontendAccess: PostProcessorFrontendAccess) {
     }
 
     private final class SingleClassWriter(underlying: FileWriter) extends ClassfileWriter {
-      override def writeClass(className: InternalName, bytes: Array[Byte], sourceFile: AbstractFile): NullableFile = {
+      override def writeClass(className: InternalName, bytes: Array[Byte], sourceFile: AbstractFile): AbstractFile = {
         underlying.writeFile(classRelativePath(className), bytes)
       }
       override def writeTasty(className: InternalName, bytes: Array[Byte], sourceFile: AbstractFile): Unit = {
@@ -103,7 +102,7 @@ class ClassfileWriters(frontendAccess: PostProcessorFrontendAccess) {
     }
 
     private final class DebugClassWriter(basic: ClassfileWriter, dump: FileWriter) extends ClassfileWriter {
-      override def writeClass(className: InternalName, bytes: Array[Byte], sourceFile: AbstractFile): NullableFile = {
+      override def writeClass(className: InternalName, bytes: Array[Byte], sourceFile: AbstractFile): AbstractFile = {
         val outFile = basic.writeClass(className, bytes, sourceFile)
         dump.writeFile(classRelativePath(className), bytes)
         outFile
@@ -121,7 +120,7 @@ class ClassfileWriters(frontendAccess: PostProcessorFrontendAccess) {
   }
 
   sealed trait FileWriter {
-    def writeFile(relativePath: String, bytes: Array[Byte]): NullableFile
+    def writeFile(relativePath: String, bytes: Array[Byte]): AbstractFile
     def close(): Unit
   }
 
@@ -165,7 +164,7 @@ class ClassfileWriters(frontendAccess: PostProcessorFrontendAccess) {
 
     lazy val crc = new CRC32
 
-    override def writeFile(relativePath: String, bytes: Array[Byte]): NullableFile = this.synchronized {
+    override def writeFile(relativePath: String, bytes: Array[Byte]): AbstractFile = this.synchronized {
       val entry = new ZipEntry(relativePath)
       if (storeOnly) {
         // When using compression method `STORED`, the ZIP spec requires the CRC and compressed/
@@ -182,7 +181,13 @@ class ClassfileWriters(frontendAccess: PostProcessorFrontendAccess) {
       jarWriter.putNextEntry(entry)
       try jarWriter.write(bytes, 0, bytes.length)
       finally jarWriter.flush()
-      null
+      // important detail here, even on Windows, Zinc expects the separator within the jar
+      // to be the system default, (even if in the actual jar file the entry always uses '/').
+      // see https://github.com/sbt/zinc/blob/dcddc1f9cfe542d738582c43f4840e17c053ce81/internal/compiler-bridge/src/main/scala/xsbt/JarUtils.scala#L47
+      val pathInJar = 
+        if File.separatorChar == '/' then relativePath
+        else relativePath.replace('/', File.separatorChar)
+      PlainFile.toPlainFile(Paths.get(s"${file.absolutePath}!$pathInJar"))
     }
 
     override def close(): Unit = this.synchronized(jarWriter.close())
@@ -230,7 +235,7 @@ class ClassfileWriters(frontendAccess: PostProcessorFrontendAccess) {
     private val fastOpenOptions = util.EnumSet.of(StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)
     private val fallbackOpenOptions = util.EnumSet.of(StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING)
 
-    override def writeFile(relativePath: String, bytes: Array[Byte]): NullableFile = {
+    override def writeFile(relativePath: String, bytes: Array[Byte]): AbstractFile = {
       val path = base.resolve(relativePath)
       try {
         ensureDirForPath(base, path)
@@ -279,7 +284,7 @@ class ClassfileWriters(frontendAccess: PostProcessorFrontendAccess) {
       finally out.close()
     }
 
-    override def writeFile(relativePath: String, bytes: Array[Byte]):NullableFile = {
+    override def writeFile(relativePath: String, bytes: Array[Byte]): AbstractFile = {
       val outFile = getFile(base, relativePath)
       writeBytes(outFile, bytes)
       outFile

compiler/src/dotty/tools/backend/jvm/PostProcessor.scala

@@ -44,11 +44,11 @@ class PostProcessor(val frontendAccess: PostProcessorFrontendAccess, val bTypes:
           backendReporting.error(em"Error while emitting $internalName\n${ex.getMessage}")
           null
 
-     if bytes != null then
-        if (AsmUtils.traceSerializedClassEnabled && internalName.contains(AsmUtils.traceSerializedClassPattern))
-          AsmUtils.traceClass(bytes)
-        val clsFile = classfileWriter.writeClass(internalName, bytes, sourceFile)
-        if clsFile != null then clazz.onFileCreated(clsFile)
+    if bytes != null then
+      if AsmUtils.traceSerializedClassEnabled && internalName.contains(AsmUtils.traceSerializedClassPattern) then
+        AsmUtils.traceClass(bytes)
+      val clsFile = classfileWriter.writeClass(internalName, bytes, sourceFile)
+      clazz.onFileCreated(clsFile)
   }
 
   def sendToDisk(tasty: GeneratedTasty, sourceFile: AbstractFile): Unit = {

compiler/src/dotty/tools/io/JarArchive.scala

@@ -10,11 +10,13 @@ import scala.jdk.CollectionConverters.*
  * This class implements an [[AbstractFile]] backed by a jar
  * that be can used as the compiler's output directory.
  */
-class JarArchive private (root: Directory) extends PlainDirectory(root) {
+class JarArchive private (val jarPath: Path, root: Directory) extends PlainDirectory(root) {
   def close(): Unit = this.synchronized(jpath.getFileSystem().close())
   override def exists: Boolean = jpath.getFileSystem().isOpen() && super.exists
   def allFileNames(): Iterator[String] =
     java.nio.file.Files.walk(jpath).iterator().asScala.map(_.toString)
+
+  override def toString: String = jarPath.toString
 }
 
 object JarArchive {
@@ -40,6 +42,6 @@ object JarArchive {
       }
     }
     val root = fs.getRootDirectories().iterator.next()
-    new JarArchive(Directory(root))
+    new JarArchive(path, Directory(root))
   }
 }

sbt-bridge/test/xsbt/ExtractUsedNamesSpecification.scala

@@ -1,7 +1,6 @@
 package xsbt
 
 import xsbti.UseScope
-import ScalaCompilerForUnitTesting.Callbacks
 
 import org.junit.{ Test, Ignore }
 import org.junit.Assert._
@@ -227,9 +226,9 @@ class ExtractUsedNamesSpecification {
 
     def findPatMatUsages(in: String): Set[String] = {
       val compilerForTesting = new ScalaCompilerForUnitTesting
-      val (_, Callbacks(callback, _)) =
+      val output =
         compilerForTesting.compileSrcs(List(List(sealedClass, in)))
-      val clientNames = callback.usedNamesAndScopes.view.filterKeys(!_.startsWith("base."))
+      val clientNames = output.analysis.usedNamesAndScopes.view.filterKeys(!_.startsWith("base."))
 
       val names: Set[String] = clientNames.flatMap {
         case (_, usages) =>

sbt-bridge/test/xsbt/ProductsSpecification.scala

@@ -0,0 +1,34 @@
+package xsbt
+
+import org.junit.Assert.*
+import org.junit.Ignore
+import org.junit.Test
+
+import java.io.File
+import java.nio.file.Path
+import java.nio.file.Paths
+
+class ProductsSpecification {
+
+  @Test
+  def extractProductsFromJar = {
+    val src =
+      """package example
+        |
+        |class A {
+        |  class B
+        |  def foo =
+        |    class C
+        |}""".stripMargin
+    val output = compiler.compileSrcsToJar(src)
+    val srcFile = output.srcFiles.head
+    val products = output.analysis.productClassesToSources.filter(_._2 == srcFile).keys.toSet
+    
+    def toPathInJar(className: String): Path =
+      Paths.get(s"${output.classesOutput}!${className.replace('.', File.separatorChar)}.class")
+    val expected = Set("example.A", "example.A$B", "example.A$C$1").map(toPathInJar)
+    assertEquals(products, expected)
+  }
+
+  private def compiler = new ScalaCompilerForUnitTesting
+}