Add source file attribute to TASTY files

This introduces the first non-boolean attribute to the TASTY format. We
Split the tags into a set of boolean tags and string tags to handle This
new kind of attribute. We keep some tags as unknown to allow for additional
kinds of tags in the future.

Author: nicolasstucki

compiler/src/dotty/tools/dotc/core/tasty/AttributePickler.scala

@@ -12,12 +12,18 @@ object AttributePickler:
     pickler: TastyPickler,
     buf: TastyBuffer
   ): Unit =
-    if attributes.booleanTags.nonEmpty then
-      pickler.newSection(AttributesSection, buf)
+    pickler.newSection(AttributesSection, buf)
 
     for tag <- attributes.booleanTags do
+      assert(tag < TastyFormat.firstStringAttrTag, "Not a boolean attribute tag: " + tag)
       buf.writeByte(tag)
 
+    assert(attributes.stringTagValues.exists(_._1 == TastyFormat.SOURCEFILEattr))
+    for (tag, value) <- attributes.stringTagValues do
+      assert(TastyFormat.firstStringAttrTag <= tag && tag < TastyFormat.firstUnassignedAttrTag, "Not a string attribute tag: " + tag)
+      buf.writeByte(tag)
+      buf.writeUtf8(value)
+
   end pickleAttributes
 
 end AttributePickler

compiler/src/dotty/tools/dotc/core/tasty/AttributeUnpickler.scala

@@ -3,6 +3,7 @@ package core.tasty
 
 import scala.language.unsafeNulls
 import scala.collection.immutable.BitSet
+import scala.collection.immutable.TreeMap
 
 import dotty.tools.tasty.{TastyFormat, TastyReader, TastyBuffer}
 
@@ -11,11 +12,18 @@ class AttributeUnpickler(reader: TastyReader):
 
   lazy val attributes: Attributes = {
     val booleanTags = BitSet.newBuilder
+    val stringTagValue = List.newBuilder[(Int, String)]
 
     while !isAtEnd do
-      booleanTags += readByte()
+      val tag = readByte()
+      if tag < TastyFormat.firstStringAttrTag then
+        booleanTags += tag
+      else if tag < TastyFormat.firstUnassignedAttrTag then
+        stringTagValue += tag -> readUtf8()
+      else
+        assert(false, "unknown attribute tag: " + tag)
 
-    new Attributes(booleanTags.result())
+    new Attributes(booleanTags.result(), stringTagValue.result())
   }
 
 end AttributeUnpickler

compiler/src/dotty/tools/dotc/core/tasty/Attributes.scala

@@ -3,20 +3,24 @@ package dotty.tools.dotc.core.tasty
 import dotty.tools.tasty.TastyFormat.*
 
 import scala.collection.immutable.BitSet
+import scala.collection.immutable.TreeMap
 
 class Attributes private[tasty](
   private[tasty] val booleanTags: BitSet,
+  private[tasty] val stringTagValues: List[(Int, String)],
 ) {
   def scala2StandardLibrary: Boolean = booleanTags(SCALA2STANDARDLIBRARYattr)
   def explicitNulls: Boolean = booleanTags(EXPLICITNULLSattr)
   def captureChecked: Boolean = booleanTags(CAPTURECHECKEDattr)
   def withPureFuns: Boolean = booleanTags(WITHPUREFUNSattr)
   def isJava: Boolean = booleanTags(JAVAattr)
   def isOutline: Boolean = booleanTags(OUTLINEattr)
+  def sourceFile: Option[String] = stringTagValues.find(_._1 == SOURCEFILEattr).map(_._2)
 }
 
 object Attributes:
   def apply(
+    sourceFile: String,
     scala2StandardLibrary: Boolean,
     explicitNulls: Boolean,
     captureChecked: Boolean,
@@ -31,8 +35,12 @@ object Attributes:
     if withPureFuns then booleanTags += WITHPUREFUNSattr
     if isJava then booleanTags += JAVAattr
     if isOutline then booleanTags += OUTLINEattr
-    new Attributes(booleanTags.result())
+
+    val stringTagValues = List.newBuilder[(Int, String)]
+    stringTagValues += SOURCEFILEattr -> sourceFile
+
+    new Attributes(booleanTags.result(), stringTagValues.result())
   end apply
 
   val empty: Attributes =
-    new Attributes(BitSet.empty)
+    new Attributes(BitSet.empty, Nil)

compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala

@@ -233,6 +233,10 @@ class TastyPrinter(bytes: Array[Byte]) {
 
       for tag <- attributes.booleanTags do
         sb.append("  ").append(attributeTagToString(tag)).append("\n")
+      for (tag, value) <- attributes.stringTagValues do
+        sb.append("  ").append(attributeTagToString(tag))
+          .append(" ").append(value).append("\n")
+      sb.result
     }
   }
 

compiler/src/dotty/tools/dotc/transform/Pickler.scala

@@ -96,12 +96,16 @@ class Pickler extends Phase {
     do
       if ctx.settings.YtestPickler.value then beforePickling(cls) = tree.show
 
+      val sourceRelativePath =
+        val reference = ctx.settings.sourceroot.value
+        util.SourceFile.relativePath(unit.source, reference)
       val isJavaAttr = unit.isJava // we must always set JAVAattr when pickling Java sources
       if isJavaAttr then
         // assert that Java sources didn't reach Pickler without `-Yjava-tasty`.
         assert(ctx.settings.YjavaTasty.value, "unexpected Java source file without -Yjava-tasty")
       val isOutline = isJavaAttr // TODO: later we may want outline for Scala sources too
       val attributes = Attributes(
+        sourceFile = sourceRelativePath,
         scala2StandardLibrary = ctx.settings.YcompileScala2Library.value,
         explicitNulls = ctx.settings.YexplicitNulls.value,
         captureChecked = Feature.ccEnabled,

tasty/src/dotty/tools/tasty/TastyFormat.scala

@@ -236,11 +236,11 @@ Note: The signature of a SELECTin or TERMREFin node is the signature of the sele
 
 Note: Tree tags are grouped into 5 categories that determine what follows, and thus allow to compute the size of the tagged tree in a generic way.
 ```none
-  Category 1 (tags 1-59)   :  tag
-  Category 2 (tags 60-89)  :  tag Nat
-  Category 3 (tags 90-109) :  tag AST
-  Category 4 (tags 110-127):  tag Nat AST
-  Category 5 (tags 128-255):  tag Length <payload>
+  Tree Category 1 (tags 1-59)   :  tag
+  Tree Category 2 (tags 60-89)  :  tag Nat
+  Tree Category 3 (tags 90-109) :  tag AST
+  Tree Category 4 (tags 110-127):  tag Nat AST
+  Tree Category 5 (tags 128-255):  tag Length <payload>
 ```
 
 Standard-Section: "Positions" LinesSizes Assoc*
@@ -278,6 +278,13 @@ Standard Section: "Attributes" Attribute*
                   WITHPUREFUNSattr
                   JAVAattr
                   OUTLINEattr
+                  SOURCEFILEattr Utf8
+```
+
+Note: Attribute tags are grouped into 2 categories that determine what follows, and thus allow to compute the size of the tagged tree in a generic way.
+```none
+  Attribute Category 1 (tags 1-64)  :  tag
+  Attribute Category 2 (tags 65-128):  tag UTF8
 ```
 
 **************************************************************************************/
@@ -441,9 +448,9 @@ object TastyFormat {
 
   final val SOURCE = 4
 
- // AST tags
-  // Cat. 1:    tag
+  // AST tags
 
+  // Tree Cat. 1:    tag
   final val firstSimpleTreeTag = UNITconst
   // final val ??? = 1
   final val UNITconst = 2
@@ -492,8 +499,8 @@ object TastyFormat {
   final val EMPTYCLAUSE = 45
   final val SPLITCLAUSE = 46
 
-  // Cat. 2:    tag Nat
-
+  // Tree Cat. 2:    tag Nat
+  final val firstNatTreeTag = SHAREDterm
   final val SHAREDterm = 60
   final val SHAREDtype = 61
   final val TERMREFdirect = 62
@@ -512,8 +519,8 @@ object TastyFormat {
   final val IMPORTED = 75
   final val RENAMED = 76
 
-  // Cat. 3:    tag AST
-
+  // Tree Cat. 3:    tag AST
+  final val firstASTTreeTag = THIS
   final val THIS = 90
   final val QUALTHIS = 91
   final val CLASSconst = 92
@@ -531,8 +538,8 @@ object TastyFormat {
   final val ELIDED = 104
 
 
-  // Cat. 4:    tag Nat AST
-
+  // Tree Cat. 4:    tag Nat AST
+  final val firstNatASTTreeTag = IDENT
   final val IDENT = 110
   final val IDENTtpt = 111
   final val SELECT = 112
@@ -544,8 +551,8 @@ object TastyFormat {
   final val SELFDEF = 118
   final val NAMEDARG = 119
 
-  // Cat. 5:    tag Length ...
-
+  // Tree Cat. 5:    tag Length ...
+  final val firstLengthTreeTag = PACKAGE
   final val PACKAGE = 128
   final val VALDEF = 129
   final val DEFDEF = 130
@@ -607,21 +614,25 @@ object TastyFormat {
 
   final val HOLE = 255
 
-  final val firstNatTreeTag = SHAREDterm
-  final val firstASTTreeTag = THIS
-  final val firstNatASTTreeTag = IDENT
-  final val firstLengthTreeTag = PACKAGE
-
-
   // Attributes tags
 
+  // Attr Cat. 1: tag
+  final val firstBooleanAttrTag = SCALA2STANDARDLIBRARYattr
   final val SCALA2STANDARDLIBRARYattr = 1
   final val EXPLICITNULLSattr = 2
   final val CAPTURECHECKEDattr = 3
   final val WITHPUREFUNSattr = 4
   final val JAVAattr = 5
   final val OUTLINEattr = 6
 
+  // Attr Cat. 2: tag UTF8
+  final val firstStringAttrTag = SOURCEFILEattr
+  final val SOURCEFILEattr = 128
+
+  // Attr Cat. 3: unassigned
+  final val firstUnassignedAttrTag = 129
+
+
   /** Useful for debugging */
   def isLegalTag(tag: Int): Boolean =
     firstSimpleTreeTag <= tag && tag <= SPLITCLAUSE ||
@@ -845,6 +856,7 @@ object TastyFormat {
     case WITHPUREFUNSattr => "WITHPUREFUNSattr"
     case JAVAattr => "JAVAattr"
     case OUTLINEattr => "OUTLINEattr"
+    case SOURCEFILEattr => "SOURCEFILEattr"
   }
 
   /** @return If non-negative, the number of leading references (represented as nats) of a length/trees entry.