TASTy header format changes

Here we adds fields for experimental and tooling versions.

- ExperimentalVersion: Nat, when reading non-zero and
  we are non-zero, the file must be at same version.
  Otherwise if we are non-zero, the file must have
  zero experimental version, and must be of strictly
  lower minor version.
- ToolingVersion: String, an arbitrary UTF-8 string
  that says the name+version of the tooling that
  produced the file. We could reinterpret to mean
  the "minimum" version required to read that file,
  but that would require more process to record that.

Author: bishabosha

compiler/src/dotty/tools/dotc/core/tasty/TastyPickler.scala

@@ -12,6 +12,15 @@ import core.Symbols.{Symbol, ClassSymbol}
 import ast.tpd
 import Decorators._
 
+object TastyPickler {
+
+  private val versionStringBytes = {
+    val compilerString = s"Scala compiler ${config.Properties.versionString}"
+    compilerString.getBytes(java.nio.charset.StandardCharsets.UTF_8)
+  }
+
+}
+
 class TastyPickler(val rootCls: ClassSymbol) {
 
   private val sections = new mutable.ArrayBuffer[(NameRef, TastyBuffer)]
@@ -37,10 +46,13 @@ class TastyPickler(val rootCls: ClassSymbol) {
     val uuidHi: Long = otherSectionHashes.fold(0L)(_ ^ _)
 
     val headerBuffer = {
-      val buf = new TastyBuffer(header.length + 24)
+      val buf = new TastyBuffer(header.length + TastyPickler.versionStringBytes.length + 32)
       for (ch <- header) buf.writeByte(ch.toByte)
       buf.writeNat(MajorVersion)
       buf.writeNat(MinorVersion)
+      buf.writeNat(ExperimentalVersion)
+      buf.writeNat(TastyPickler.versionStringBytes.length)
+      buf.writeBytes(TastyPickler.versionStringBytes, TastyPickler.versionStringBytes.length)
       buf.writeUncompressedLong(uuidLow)
       buf.writeUncompressedLong(uuidHi)
       buf

compiler/test/dotty/tools/dotc/CompilationTests.scala

@@ -247,7 +247,7 @@ class CompilationTests {
           //  "-source", "3.1",  // TODO: re-enable once we allow : @unchecked in pattern definitions. Right now, lots of narrowing pattern definitions fail.
           ))(libGroup)
 
-    val tastyCoreSources = sources(Paths.get("tasty/src")) ++ sources(Paths.get("tasty/src-bootstrapped"))
+    val tastyCoreSources = sources(Paths.get("tasty/src"))
     val tastyCore = compileList("tastyCore", tastyCoreSources, opt)(tastyCoreGroup)
 
     val compilerSources = sources(Paths.get("compiler/src")) ++ sources(Paths.get("compiler/src-bootstrapped"))

tasty/src-bootstrapped/dotty/tools/tasty/TastyHeaderUnpickler.scala

@@ -19,10 +19,11 @@ Micro-syntax:
 
 Macro-format:
 
-  File          = Header majorVersion_Nat minorVersion_Nat UUID
+  File          = Header majorVersion_Nat minorVersion_Nat experimentalVersion_Nat VersionString UUID
                   nameTable_Length Name* Section*
   Header        = 0x5CA1AB1F
   UUID          = Byte*16                 -- random UUID
+  VersionString = Length UTF8-CodePoint*  -- string that represents the compiler that produced the TASTy
 
   Section       = NameRef Length Bytes
   Length        = Nat                     -- length of rest of entry in bytes
@@ -263,8 +264,25 @@ Standard Section: "Comments" Comment*
 object TastyFormat {
 
   final val header: Array[Int] = Array(0x5C, 0xA1, 0xAB, 0x1F)
-  val MajorVersion: Int = 27
-  val MinorVersion: Int = 0
+
+  /** Each increment of the MajorVersion begins a new series of backward compatible TASTy versions
+   *  - The MajorVersion is and will always be the first value in a TASTy document after the header bytes.
+   *  - A TASTy document can then be further parsed if and only if the MajorVersion read
+   *    from a TASTy file is equal to or less than this value.
+   *  - i.e. `MajorVersion` and `header` are the only stable parts of TASTy that can be used to decide how to parse
+   *    the rest of the document.
+   */
+  final val MajorVersion: Int = 28
+
+  /** The last minor version to break forward compatability */
+  final val MinorVersion: Int = 0
+
+  /** A transitionary marker:
+   *  - 0 means that `MinorVersion` is from a final release of TASTy
+   *  - A positive number means TASTy was produced by an experimental compiler, released in between
+   *    increasing `MinorVersion`.
+   */
+  final val ExperimentalVersion: Int = 1
 
   final val ASTsSection = "ASTs"
   final val PositionsSection = "Positions"

tasty/src-non-bootstrapped/dotty/tools/tasty/TastyHeaderUnpickler.scala

@@ -0,0 +1,136 @@
+package dotty.tools.tasty
+
+import java.util.UUID
+
+import TastyFormat.{MajorVersion, MinorVersion, ExperimentalVersion, header}
+
+/**
+ * The Tasty Header consists of four fields:
+ * - uuid
+ *   - contains a hash of the sections of the TASTy file
+ * - majorVersion
+ *   - matching the TASTy format version that last broke backwards compatibility
+ * - minorVersion
+ *   - matching the TASTy format version that last broke forward compatibility
+ * - experimentalVersion
+ *   - 0 for final compiler version
+ *   - positive for between minor versions and forward compatibility
+ *     is broken since the previous stable version.
+ * - toolingVersion
+ *   - arbitrary string representing the tooling that produced the TASTy
+ */
+sealed abstract case class TastyHeader(
+  uuid: UUID,
+  majorVersion: Int,
+  minorVersion: Int,
+  experimentalVersion: Int,
+  toolingVersion: String // String could lead to a lot of duplication, perhaps cache headers?
+)
+
+class TastyHeaderUnpickler(reader: TastyReader) {
+  import TastyHeaderUnpickler._
+  import reader._
+
+  def this(bytes: Array[Byte]) = this(new TastyReader(bytes))
+
+  /** reads and verifies the TASTy version, extracting the UUID */
+  def readHeader(): UUID =
+    readFullHeader().uuid
+
+  /** reads and verifies the TASTy version, extracting the whole header */
+  def readFullHeader(): TastyHeader = {
+
+    for (i <- 0 until header.length)
+      check(readByte() == header(i), "not a TASTy file")
+    val fileMajor = readNat()
+    if (fileMajor <= 27) { // old behavior before `tasty-core` 3.0.0-M4
+      val fileMinor = readNat()
+      val signature = signatureString(expectedMinor, fileMajor, fileMinor.toString)
+      throw new UnpickleException(signature + backIncompatAddendum + myAddendum)
+    }
+    else {
+      val fileMinor = readNat()
+      val fileExperimental = readNat()
+      val toolingVersion = {
+        val length = readNat()
+        val start = currentAddr
+        val end = start + length
+        goto(end)
+        new String(bytes, start.index, length) // NOTE: new string object here (should probably cache as it leaks)
+      }
+
+      val validVersion = ( // inlined from `dotty.tools.tasty.TastyVersionFormatTest.TastyVersion.<:<`
+        fileMajor == MajorVersion && (
+          if (fileExperimental == ExperimentalVersion) {
+            if (ExperimentalVersion == 0) {
+              fileMinor <= MinorVersion
+            }
+            else {
+              fileMinor == MinorVersion
+            }
+          }
+          else {
+            fileExperimental == 0 && fileMinor < MinorVersion
+          }
+        )
+      )
+      check(validVersion, {
+        val foundMinor = showMinorVersion(fileMinor, fileExperimental)
+        val producedByAddendum = s"\nThe TASTy file was produced by $toolingVersion.$myAddendum"
+        val signature = signatureString(expectedMinor, fileMajor, foundMinor)
+        if (fileExperimental != 0)
+          signature + unstableAddendum + producedByAddendum
+        else if (fileMajor < MajorVersion)
+          signature + backIncompatAddendum + producedByAddendum
+        else
+          signature + forwardIncompatAddendum + producedByAddendum
+      }
+      )
+      val uuid = new UUID(readUncompressedLong(), readUncompressedLong())
+      new TastyHeader(uuid, fileMajor, fileMinor, fileExperimental, toolingVersion) {}
+    }
+  }
+
+  def isAtEnd: Boolean = reader.isAtEnd
+
+  private def check(cond: Boolean, msg: => String): Unit = {
+    if (!cond) throw new UnpickleException(msg)
+  }
+}
+
+object TastyHeaderUnpickler {
+
+  private def showMinorVersion(min: Int, exp: Int) = {
+    val expStr = if (exp == 0) "" else s" [unstable release: $exp]"
+    s"$min$expStr"
+  }
+
+  private def expectedMinor = showMinorVersion(MinorVersion, ExperimentalVersion)
+
+  private def myAddendum = {
+    if (ExperimentalVersion > 0)
+      "\nNote that your tooling is currently using an unstable TASTy version."
+    else
+      ""
+  }
+
+  private def signatureString(minorVersion: String, fileMajor: Int, fileMinor: String) = (
+    s"""TASTy signature has wrong version.
+      | expected: {majorVersion: $MajorVersion, minorVersion: $minorVersion}
+      | found   : {majorVersion: $fileMajor, minorVersion: $fileMinor}
+      |
+      |""".stripMargin
+  )
+
+  private def unstableAddendum =
+    """This TASTy file was produced by an unstable release.
+      |To read this TASTy file, your tooling must be at the same version.""".stripMargin
+
+  private def backIncompatAddendum =
+    """This TASTy file was produced by a backwards incompatible release.
+      |Please recompile this TASTy with a later version.""".stripMargin
+
+  private def forwardIncompatAddendum =
+    """This TASTy file was produced by a forwards incompatible release.
+      |To read this TASTy file, please upgrade your tooling.""".stripMargin
+}

tasty/src/dotty/tools/tasty/TastyFormat.scala

@@ -0,0 +1,84 @@
+package dotty.tools.tasty
+
+import org.junit.Assert._
+import org.junit.{Test, Ignore}
+
+import TastyFormat._
+import TastyBuffer._
+
+@Ignore // comment if you want to experiment with error messages
+class TastyHeaderUnpicklerTest {
+
+  import TastyHeaderUnpicklerTest._
+
+  @Test def vanilla: Unit = {
+    runTest(MajorVersion, MinorVersion, ExperimentalVersion, "Scala compiler 3.0.0-M4-bin-SNAPSHOT-git-12345")
+  }
+
+  @Test def failBumpExperimental: Unit = {
+    (runTest(MajorVersion, MinorVersion, ExperimentalVersion + 1, "Scala compiler 3.0.0-M4-bin-SNAPSHOT-git-12345"))
+  }
+
+  @Test def failBumpMinor: Unit = {
+    (runTest(MajorVersion, MinorVersion + 1, ExperimentalVersion, "Scala compiler 3.1.0-RC1"))
+  }
+
+  @Test def failBumpMajor: Unit = {
+    (runTest(MajorVersion + 1, MinorVersion, ExperimentalVersion, "Scala compiler 4.0.0-M1"))
+  }
+
+  @Test def failBumpMajorFinal: Unit = {
+    (runTest(MajorVersion + 1, MinorVersion, 0, "Scala compiler 4.0.0"))
+  }
+
+  @Test def okSubtractExperimental: Unit = {
+    (runTest(MajorVersion, MinorVersion, ExperimentalVersion - 1, "Scala compiler 3.0.0"))
+  }
+
+  @Test def okSubtractMinor: Unit = {
+    (runTest(MajorVersion, MinorVersion - 1, ExperimentalVersion, "Scala compiler 3.0.0-M4-bin-SNAPSHOT-git-12345"))
+  }
+
+  @Test def failSubtractMajor: Unit = {
+    (runTest(MajorVersion - 1, MinorVersion, ExperimentalVersion, "Scala compiler 3.0.0-M4-bin-SNAPSHOT-git-12345"))
+  }
+
+}
+
+object TastyHeaderUnpicklerTest {
+
+
+  def fillHeader(maj: Int, min: Int, exp: Int, compiler: String): TastyBuffer = {
+    val compilerBytes = compiler.getBytes(java.nio.charset.StandardCharsets.UTF_8)
+    val buf = new TastyBuffer(header.length + 32 + compilerBytes.length)
+    for (ch <- header) buf.writeByte(ch.toByte)
+    buf.writeNat(maj)
+    buf.writeNat(min)
+    buf.writeNat(exp)
+    buf.writeNat(compilerBytes.length)
+    buf.writeBytes(compilerBytes, compilerBytes.length)
+    buf.writeUncompressedLong(237478l)
+    buf.writeUncompressedLong(324789l)
+    buf
+  }
+
+  def runTest(maj: Int, min: Int, exp: Int, compiler: String): Unit = {
+    val headerBuffer = fillHeader(maj, min, exp, compiler)
+    val bs = headerBuffer.bytes.clone
+
+    val hr = new TastyHeaderUnpickler(bs)
+
+    hr.readFullHeader()
+  }
+
+  def expectUnpickleError(op: => Unit) = {
+    try {
+      op
+      fail()
+    }
+    catch {
+      case err: UnpickleException => ()
+    }
+  }
+
+}