Merge pull request #9619 from dotty-staging/parallel-position-pickler

Parallelize position pickling

Author: odersky

compiler/src/dotty/tools/backend/jvm/GenBCode.scala

@@ -235,15 +235,15 @@ class GenBCodePipeline(val int: DottyBackendInterface)(using Context) extends BC
             if (!ctx.settings.YemitTastyInClass.value) {
               val outTastyFile = getFileForClassfile(outF, store.name, ".tasty")
               val outstream = new DataOutputStream(outTastyFile.bufferedOutput)
-              try outstream.write(binary)
+              try outstream.write(binary())
               catch case ex: ClosedByInterruptException =>
                 try
                   outTastyFile.delete() // don't leave an empty or half-written tastyfile around after an interrupt
                 catch case _: Throwable =>
                 throw ex
               finally outstream.close()
 
-              val uuid = new TastyHeaderUnpickler(binary).readHeader()
+              val uuid = new TastyHeaderUnpickler(binary()).readHeader()
               val lo = uuid.getMostSignificantBits
               val hi = uuid.getLeastSignificantBits
 
@@ -257,7 +257,7 @@ class GenBCodePipeline(val int: DottyBackendInterface)(using Context) extends BC
               // Create an empty file to signal that a tasty section exist in the corresponding .class
               // This is much cheaper and simpler to check than doing classfile parsing
               getFileForClassfile(outF, store.name, ".hasTasty")
-              binary
+              binary()
             }
           val dataAttr = createJAttribute(nme.TASTYATTR.mangledString, tasty, 0, tasty.length)
           store.visitAttribute(dataAttr)

compiler/src/dotty/tools/dotc/CompilationUnit.scala

@@ -30,7 +30,7 @@ class CompilationUnit protected (val source: SourceFile) {
   var sourceVersion: Option[SourceVersion] = None
 
   /** Pickled TASTY binaries, indexed by class. */
-  var pickled: Map[ClassSymbol, Array[Byte]] = Map()
+  var pickled: Map[ClassSymbol, () => Array[Byte]] = Map()
 
   /** The fresh name creator for the current unit.
    *  FIXME(#7661): This is not fine-grained enough to enable reproducible builds,

compiler/src/dotty/tools/dotc/core/quoted/PickledQuotes.scala

@@ -168,8 +168,6 @@ object PickledQuotes {
     val treePkl = pickler.treePkl
     treePkl.pickle(tree :: Nil)
     treePkl.compactify()
-    pickler.addrOfTree = treePkl.buf.addrOfTree
-    pickler.addrOfSym = treePkl.addrOfSym
     if (tree.span.exists)
       new PositionPickler(pickler, treePkl.buf.addrOfTree).picklePositions(tree :: Nil)
 

compiler/src/dotty/tools/dotc/core/tasty/TastyPickler.scala

@@ -65,18 +65,5 @@ class TastyPickler(val rootCls: ClassSymbol) {
     all.bytes
   }
 
-  /** The address in the TASTY file of a given tree, or None if unknown.
-   *  Note that trees are looked up by reference equality,
-   *  so one can reliably use this function only directly after `pickler`.
-   */
-  var addrOfTree: tpd.Tree => Addr = (_ => NoAddr)
-
-  /**
-   * Addresses in TASTY file of symbols, stored by pickling.
-   * Note that trees are checked for reference equality,
-   * so one can reliably use this function only dirrectly after `pickler`
-   */
-  var addrOfSym: Symbol => Option[Addr] = (_ => None)
-
   val treePkl: TreePickler = new TreePickler(this)
 }

compiler/src/dotty/tools/dotc/core/tasty/TreeBuffer.scala

@@ -21,7 +21,10 @@ class TreeBuffer extends TastyBuffer(50000) {
   private var delta: Array[Int] = _
   private var numOffsets = 0
 
-  /** A map from tree unique ids to the address index at which a tree is pickled. */
+  /** A map from tree unique ids to the address index at which a tree is pickled.
+   *  Note that trees are looked up by reference equality,
+   *  so one can reliably use this function only directly after `pickler`.
+   */
   private val addrOfTree = SparseIntArray()
 
   def registerTreeAddr(tree: Tree): Addr =

compiler/src/dotty/tools/dotc/decompiler/DecompilationPrinter.scala

@@ -39,7 +39,7 @@ class DecompilationPrinter extends Phase {
   private def printToOutput(out: PrintStream)(using Context): Unit = {
     val unit = ctx.compilationUnit
     if (ctx.settings.printTasty.value)
-      println(new TastyPrinter(unit.pickled.head._2).printContents())
+      println(new TastyPrinter(unit.pickled.head._2()).printContents())
     else {
       val unitFile = unit.source.toString.replace("\\", "/").replace(".class", ".tasty")
       out.println(s"/** Decompiled from $unitFile */")

compiler/src/dotty/tools/dotc/decompiler/IDEDecompilerDriver.scala

@@ -35,7 +35,7 @@ class IDEDecompilerDriver(val settings: List[String]) extends dotc.Driver {
       val unit = ctx.run.units.head
 
       val decompiled = ReflectionImpl.showTree(unit.tpdTree)
-      val tree = new TastyHTMLPrinter(unit.pickled.head._2).printContents()
+      val tree = new TastyHTMLPrinter(unit.pickled.head._2()).printContents()
 
       reporter.removeBufferedMessages.foreach(message => System.err.println(message))
       (tree, decompiled)

compiler/src/dotty/tools/dotc/fromtasty/ReadTasty.scala

@@ -40,7 +40,7 @@ class ReadTasty extends Phase {
               if (cls.rootTree.isEmpty) None
               else {
                 val unit = CompilationUnit(cls, cls.rootTree, forceTrees = true)
-                unit.pickled += (cls -> unpickler.unpickler.bytes)
+                unit.pickled += (cls -> (() => unpickler.unpickler.bytes))
                 Some(unit)
               }
             case tree: Tree[?] =>

compiler/src/dotty/tools/dotc/transform/Pickler.scala

@@ -13,6 +13,8 @@ import Symbols._
 import Flags.Module
 import reporting.ThrowingReporter
 import collection.mutable
+import scala.concurrent.{Future, Await, ExecutionContext}
+import scala.concurrent.duration.Duration
 
 object Pickler {
   val name: String = "pickler"
@@ -55,35 +57,37 @@ class Pickler extends Phase {
     }
     {
       val pickler = new TastyPickler(cls)
-      if (ctx.settings.YtestPickler.value) {
+      if ctx.settings.YtestPickler.value then
         beforePickling(cls) = tree.show
         picklers(cls) = pickler
-      }
       val treePkl = pickler.treePkl
       treePkl.pickle(tree :: Nil)
-      treePkl.compactify()
-      pickler.addrOfTree = treePkl.buf.addrOfTree
-      pickler.addrOfSym = treePkl.addrOfSym
-      if (tree.span.exists)
-        new PositionPickler(pickler, treePkl.buf.addrOfTree).picklePositions(tree :: Nil)
-
-      if (!ctx.settings.YdropComments.value)
-        new CommentPickler(pickler, treePkl.buf.addrOfTree).pickleComment(tree)
-
-      // other pickle sections go here.
-      val pickled = pickler.assembleParts()
-      unit.pickled += (cls -> pickled)
-
-      def rawBytes = // not needed right now, but useful to print raw format.
-        pickled.iterator.grouped(10).toList.zipWithIndex.map {
-          case (row, i) => s"${i}0: ${row.mkString(" ")}"
-        }
-
-      // println(i"rawBytes = \n$rawBytes%\n%") // DEBUG
-      if (pickling ne noPrinter) {
-        println(i"**** pickled info of $cls")
-        println(new TastyPrinter(pickled).printContents())
-      }
+      val pickledF = Future {
+        treePkl.compactify()
+        if tree.span.exists then
+          new PositionPickler(pickler, treePkl.buf.addrOfTree).picklePositions(tree :: Nil)
+
+        if !ctx.settings.YdropComments.value then
+          new CommentPickler(pickler, treePkl.buf.addrOfTree).pickleComment(tree)
+
+        val pickled = pickler.assembleParts()
+
+        def rawBytes = // not needed right now, but useful to print raw format.
+          pickled.iterator.grouped(10).toList.zipWithIndex.map {
+            case (row, i) => s"${i}0: ${row.mkString(" ")}"
+          }
+
+        // println(i"rawBytes = \n$rawBytes%\n%") // DEBUG
+        if pickling ne noPrinter then
+          pickling.synchronized {
+            println(i"**** pickled info of $cls")
+            println(new TastyPrinter(pickled).printContents())
+          }
+        pickled
+      }(using ExecutionContext.global)
+      def force(): Array[Byte] = Await.result(pickledF, Duration.Inf)
+      if ctx.settings.YtestPickler.value then force()
+      unit.pickled += (cls -> force)
     }
   }