Merge pull request #3819 from dotty-staging/encode-large-quotes

Encode large quotes

Author: nicolasstucki

compiler/src/dotty/tools/dotc/core/quoted/PickledQuotes.scala

@@ -11,16 +11,22 @@ import dotty.tools.dotc.core.Symbols._
 import dotty.tools.dotc.core.tasty.{TastyPickler, TastyPrinter, TastyString}
 import dotty.tools.dotc.interpreter.RawQuoted
 
+import scala.runtime.quoted.Unpickler.Pickled
+
 object PickledQuotes {
   import tpd._
 
-  /** Pickle the quote into a TASTY string */
-  def pickleQuote(tree: Tree)(implicit ctx: Context): String = {
-    if (ctx.reporter.hasErrors) "<error>"
+  /** Pickle the quote into strings */
+  def pickleQuote(tree: Tree)(implicit ctx: Context): Tree = {
+    if (ctx.reporter.hasErrors) Literal(Constant("<error>"))
     else {
       val encapsulated = encapsulateQuote(tree)
       val pickled = pickle(encapsulated)
-      TastyString.tastyToString(pickled)
+      TastyString.pickle(pickled).foldRight[Tree](ref(defn.NilModule)) { (x, acc) =>
+        acc.select("::".toTermName)
+          .appliedToType(defn.StringType)
+          .appliedTo(Literal(Constant(x)))
+      }
     }
   }
 
@@ -33,7 +39,7 @@ object PickledQuotes {
 
   /** Unpickle the tree contained in the TastyQuoted */
   private def unpickleQuote(expr: quoted.TastyQuoted)(implicit ctx: Context): Tree = {
-    val tastyBytes = TastyString.stringToTasty(expr.tasty)
+    val tastyBytes = TastyString.unpickle(expr.tasty)
     val unpickled = unpickle(tastyBytes, expr.args)
     unpickled match {
       case PackageDef(_, (vdef: ValDef) :: Nil) => vdef.rhs

compiler/src/dotty/tools/dotc/core/tasty/TastyString.scala

@@ -1,17 +1,32 @@
 package dotty.tools.dotc.core.tasty
 
+import scala.runtime.quoted.Unpickler.Pickled
+
 /** Utils for String representation of TASTY */
 object TastyString {
 
+  // Conservative encoding, each byte is encoded in a char
+  // TODO improve encoding compression
+  private final val maxStringSize = 65535 / 2
+
+  /** Encode TASTY bytes into an Seq of String */
+  def pickle(bytes: Array[Byte]): Pickled =
+    bytes.sliding(maxStringSize, maxStringSize).map(bytesToString).toList
+
   /** Decode the TASTY String into TASTY bytes */
-  def stringToTasty(str: String): Array[Byte] = {
-    val bytes = new Array[Byte](str.length)
-    for (i <- str.indices) bytes(i) = str.charAt(i).toByte
+  def unpickle(strings: Pickled): Array[Byte] = {
+    val bytes = new Array[Byte](strings.map(_.length).sum)
+    var i = 0
+    for (str <- strings; j <- str.indices) {
+      bytes(i) = str.charAt(j).toByte
+      i += 1
+    }
     bytes
   }
 
-  /** Encode TASTY bytes into a TASTY String */
-  def tastyToString(bytes: Array[Byte]): String = {
+  /** Encode bytes into a String */
+  private def bytesToString(bytes: Array[Byte]): String = {
+    assert(bytes.length <= maxStringSize)
     val chars = new Array[Char](bytes.length)
     for (i <- bytes.indices) chars(i) = (bytes(i) & 0xff).toChar
     new String(chars)

compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala

@@ -269,7 +269,7 @@ class ReifyQuotes extends MacroTransform {
         ref(if (isType) defn.Unpickler_unpickleType else defn.Unpickler_unpickleExpr)
           .appliedToType(if (isType) body1.tpe else body1.tpe.widen)
           .appliedTo(
-            Literal(Constant(PickledQuotes.pickleQuote(body1))),
+            PickledQuotes.pickleQuote(body1),
             SeqLiteral(splices, TypeTree(defn.AnyType)))
       }
     }.withPos(quote.pos)

compiler/test/dotty/tools/dotc/CompilationTests.scala

@@ -201,6 +201,7 @@ class CompilationTests extends ParallelTesting {
     compileFile("../tests/run-special/quote-run-constants.scala", defaultRunWithCompilerOptions) +
     compileFile("../tests/run-special/quote-run.scala", defaultRunWithCompilerOptions) +
     compileFile("../tests/run-special/quote-run-2.scala", defaultRunWithCompilerOptions) +
+    compileFile("../tests/run-special/quote-run-large.scala", defaultRunWithCompilerOptions) +
     compileFile("../tests/run-special/quote-run-staged-interpreter.scala", defaultRunWithCompilerOptions)
   }.checkRuns()
 

library/src/scala/runtime/quoted/Unpickler.scala

@@ -5,10 +5,10 @@ import scala.quoted._
 /** Provides methods to unpickle `Expr` and `Type` trees. */
 object Unpickler {
 
-  /** Representation of pickled trees. For now it's String, but it
-   *  should be changed to some kind of TASTY bundle.
+  /** Representation of pickled trees. For now a List[String],
+   *  but it should be changed to some kind of TASTY bundle.
    */
-  type Pickled = String
+  type Pickled = List[String]
 
   /** Unpickle `repr` which represents a pickled `Expr` tree,
    *  replacing splice nodes with `args`

tests/run-special/quote-run-large.check

@@ -0,0 +1 @@
+Foo5000(5)