Add FromTasty compiler tests

Fixes FromTasty when info already loaded

Author: nicolasstucki

compiler/src/dotty/tools/dotc/CompilationUnit.scala

@@ -1,8 +1,7 @@
 package dotty.tools
 package dotc
 
-import dotty.tools.dotc.core.Types.Type
-import dotty.tools.dotc.core.tasty.{TastyUnpickler, TastyBuffer, TastyPickler}
+import dotty.tools.dotc.core.Types.Type // Do not remove me #3383
 import util.SourceFile
 import ast.{tpd, untpd}
 import dotty.tools.dotc.ast.tpd.{ Tree, TreeTraverser }
@@ -28,6 +27,7 @@ object CompilationUnit {
 
   /** Make a compilation unit for top class `clsd` with the contends of the `unpickled` */
   def mkCompilationUnit(clsd: ClassDenotation, unpickled: Tree, forceTrees: Boolean)(implicit ctx: Context): CompilationUnit = {
+    assert(!unpickled.isEmpty, unpickled)
     val unit1 = new CompilationUnit(new SourceFile(clsd.symbol.sourceFile, Seq()))
     unit1.tpdTree = unpickled
     if (forceTrees)

compiler/src/dotty/tools/dotc/FromTasty.scala

@@ -69,23 +69,28 @@ object FromTasty extends Driver {
 
     def readTASTY(unit: CompilationUnit)(implicit ctx: Context): CompilationUnit = unit match {
       case unit: TASTYCompilationUnit =>
+        assert(ctx.settings.YretainTrees.value)
         val className = unit.className.toTypeName
-        val clsd = ctx.base.staticRef(className)
-        def cannotUnpickle(reason: String) = {
-          ctx.error(s"class $className cannot be unpickled because $reason")
-          unit
-        }
-        clsd match {
+        ctx.base.staticRef(className) match {
           case clsd: ClassDenotation =>
-            clsd.infoOrCompleter match {
+            def cannotUnpickle(reason: String) =
+              ctx.error(s"class $className cannot be unpickled because $reason")
+            def tryToLoad = clsd.infoOrCompleter match {
               case info: ClassfileLoader =>
                 info.load(clsd)
-                val unpickled = clsd.symbol.asClass.tree
-                if (unpickled != null) CompilationUnit.mkCompilationUnit(clsd, unpickled, forceTrees = true)
-                else cannotUnpickle(s"its class file ${info.classfile} does not have a TASTY attribute")
+                Option(clsd.symbol.asClass.tree).orElse {
+                  cannotUnpickle(s"its class file ${info.classfile} does not have a TASTY attribute")
+                  None
+                }
+
               case info =>
                 cannotUnpickle(s"its info of type ${info.getClass} is not a ClassfileLoader")
+                None
             }
+            Option(clsd.symbol.asClass.tree).orElse(tryToLoad)
+              .map(unpickled => CompilationUnit.mkCompilationUnit(clsd, unpickled, forceTrees = true))
+              .getOrElse(unit)
+
           case _ =>
             ctx.error(s"class not found: $className")
             unit

compiler/test/dotty/tools/dotc/FromTastyTests.scala

@@ -0,0 +1,55 @@
+package dotty
+package tools
+package dotc
+
+import org.junit.{AfterClass, Test}
+import vulpix._
+
+import scala.concurrent.duration._
+
+class FromTastyTests extends ParallelTesting {
+  import TestConfiguration._
+  import FromTastyTests._
+
+  // Test suite configuration --------------------------------------------------
+
+  def maxDuration = 30.seconds
+  def numberOfSlaves = 5
+  def safeMode = Properties.testsSafeMode
+  def isInteractive = SummaryReport.isInteractive
+  def testFilter = Properties.testsFilter
+
+
+  @Test def posTestFromTasty: Unit = {
+    implicit val testGroup: TestGroup = TestGroup("posTestFromTasty")
+    val (step1, step2) = {
+      // compileTastyInDir("../tests/pos", defaultOptions) + // FIXME
+      compileTastyInDir("../tests/pos-from-tasty", defaultOptions) +
+      compileTasty("../tests/pos-from-tasty/simpleClass.scala", defaultOptions)
+    }
+    step1.keepOutput.checkCompile() // Compile all files to generate the class files with tasty
+    step2.keepOutput.checkCompile() // Compile from tasty
+    (step1 + step2).delete()
+  }
+
+  @Test def runTestFromTasty: Unit = {
+    implicit val testGroup: TestGroup = TestGroup("runTestFromTasty")
+    val (step1, step2) = {
+      // compileTastyInDir("../tests/run", defaultOptions) + // FIXME
+      compileTastyInDir("../tests/run-from-tasty", defaultOptions)
+    }
+    step1.keepOutput.checkCompile() // Compile all files to generate the class files with tasty
+    step2.keepOutput.checkRuns() // Compile from tasty and run the result
+    (step1 + step2).delete()
+  }
+
+  private implicit class tastyCompilationTuples(tup: (CompilationTest, CompilationTest)) {
+    def +(that: (CompilationTest, CompilationTest)): (CompilationTest, CompilationTest) =
+      (tup._1 + that._1, tup._2 + that._2)
+  }
+}
+
+object FromTastyTests {
+  implicit val summaryReport: SummaryReporting = new SummaryReport
+  @AfterClass def cleanup(): Unit = summaryReport.echoSummary()
+}

compiler/test/dotty/tools/vulpix/ParallelTesting.scala

@@ -132,7 +132,8 @@ trait ParallelTesting extends RunnerOrchestration { self =>
     name: String,
     files: Array[JFile],
     flags: TestFlags,
-    outDir: JFile
+    outDir: JFile,
+    fromTasty: Boolean = false
   ) extends TestSource {
     def sourceFiles: Array[JFile] = files.filter(isSourceFile)
 
@@ -213,7 +214,7 @@ trait ParallelTesting extends RunnerOrchestration { self =>
     private val filteredSources =
       if (!testFilter.isDefined) testSources
       else testSources.filter {
-        case JointCompilationSource(_, files, _, _) =>
+        case JointCompilationSource(_, files, _, _, _) =>
           files.exists(file => file.getAbsolutePath.contains(testFilter.get))
         case SeparateCompilationSource(_, dir, _, _) =>
           dir.getAbsolutePath.contains(testFilter.get)
@@ -336,15 +337,6 @@ trait ParallelTesting extends RunnerOrchestration { self =>
 
       val files: Array[JFile] = files0.flatMap(flattenFiles)
 
-      def addOutDir(xs: Array[String]): Array[String] = {
-        val (beforeCp, cpAndAfter) = xs.toList.span(_ != "-classpath")
-        if (cpAndAfter.nonEmpty) {
-          val (cp :: cpArg :: rest) = cpAndAfter
-          (beforeCp ++ (cp :: (cpArg + s":${targetDir.getAbsolutePath}") :: rest)).toArray
-        }
-        else (beforeCp ++ ("-classpath" :: targetDir.getAbsolutePath :: Nil)).toArray
-      }
-
       def compileWithJavac(fs: Array[String]) = if (fs.nonEmpty) {
         val fullArgs = Array(
           "javac",
@@ -379,7 +371,7 @@ trait ParallelTesting extends RunnerOrchestration { self =>
             }
         }
 
-      val allArgs = addOutDir(flags.all)
+      val allArgs = flags.withClasspath(targetDir.getAbsolutePath).all
 
       // Compile with a try to catch any StackTrace generated by the compiler:
       try {
@@ -400,6 +392,30 @@ trait ParallelTesting extends RunnerOrchestration { self =>
       reporter
     }
 
+    protected def compileFromTasty(flags0: TestFlags, suppressErrors: Boolean, targetDir: JFile): TestReporter = {
+      val tastyOutput = new JFile(targetDir.getPath + "_from-tasty")
+      tastyOutput.mkdir()
+      val flags = flags0 and ("-d", tastyOutput.getAbsolutePath)
+
+      def hasTastyFileToClassName(f: JFile): String =
+        targetDir.toPath.relativize(f.toPath).toString.dropRight(".hasTasty".length).replace('/', '.')
+      val classes = flattenFiles(targetDir).filter(isHasTastyFile).map(hasTastyFileToClassName)
+
+      val reporter =
+        TestReporter.reporter(realStdout, logLevel =
+          if (suppressErrors || suppressAllOutput) ERROR + 1 else ERROR)
+
+      // Compile with a try to catch any StackTrace generated by the compiler:
+      try {
+        dotc.FromTasty.process(flags.all ++ classes, reporter = reporter)
+      }
+      catch {
+        case NonFatal(ex) => reporter.logStackTrace(ex)
+      }
+
+      reporter
+    }
+
     private[ParallelTesting] def executeTestSuite(): this.type = {
       assert(_testSourcesCompleted == 0, "not allowed to re-use a `CompileRun`")
 
@@ -440,24 +456,30 @@ trait ParallelTesting extends RunnerOrchestration { self =>
 
       this
     }
+
+    /** Returns all files in directory or the file if not a directory */
+    private def flattenFiles(f: JFile): Array[JFile] =
+      if (f.isDirectory) f.listFiles.flatMap(flattenFiles)
+      else Array(f)
   }
 
   private final class PosTest(testSources: List[TestSource], times: Int, threadLimit: Option[Int], suppressAllOutput: Boolean)(implicit summaryReport: SummaryReporting)
   extends Test(testSources, times, threadLimit, suppressAllOutput) {
     protected def encapsulatedCompilation(testSource: TestSource) = new LoggedRunnable {
       def checkTestSource(): Unit = tryCompile(testSource) {
         testSource match {
-          case testSource @ JointCompilationSource(_, files, flags, outDir) => {
-            val reporter = compile(testSource.sourceFiles, flags, false, outDir)
+          case testSource @ JointCompilationSource(_, files, flags, outDir, fromTasty) =>
+            val reporter =
+              if (fromTasty) compileFromTasty(flags, false, outDir)
+              else compile(testSource.sourceFiles, flags, false, outDir)
             registerCompletion(reporter.errorCount)
 
             if (reporter.compilerCrashed || reporter.errorCount > 0) {
               logReporterContents(reporter)
               logBuildInstructions(reporter, testSource, reporter.errorCount, reporter.warningCount)
             }
-          }
 
-          case testSource @ SeparateCompilationSource(_, dir, flags, outDir) => {
+          case testSource @ SeparateCompilationSource(_, dir, flags, outDir) =>
             val reporters = testSource.compilationGroups.map(files => compile(files, flags, false, outDir))
             val compilerCrashed = reporters.exists(_.compilerCrashed)
             val errorCount = reporters.foldLeft(0) { (acc, reporter) =>
@@ -475,7 +497,6 @@ trait ParallelTesting extends RunnerOrchestration { self =>
               reporters.foreach(logReporterContents)
               logBuildInstructions(reporters.head, testSource, errorCount, warningCount)
             }
-          }
         }
       }
     }
@@ -546,7 +567,7 @@ trait ParallelTesting extends RunnerOrchestration { self =>
     protected def encapsulatedCompilation(testSource: TestSource) = new LoggedRunnable {
       def checkTestSource(): Unit = tryCompile(testSource) {
         val (compilerCrashed, errorCount, warningCount, verifier: Function0[Unit]) = testSource match {
-          case testSource @ JointCompilationSource(_, files, flags, outDir) => {
+          case testSource @ JointCompilationSource(_, files, flags, outDir, fromTasty) =>
             val checkFile = files.flatMap { file =>
               if (file.isDirectory) Nil
               else {
@@ -556,17 +577,18 @@ trait ParallelTesting extends RunnerOrchestration { self =>
                 else Nil
               }
             }.headOption
-            val reporter = compile(testSource.sourceFiles, flags, false, outDir)
+            val reporter =
+              if (fromTasty) compileFromTasty(flags, false, outDir)
+              else compile(testSource.sourceFiles, flags, false, outDir)
 
             if (reporter.compilerCrashed || reporter.errorCount > 0) {
               logReporterContents(reporter)
               logBuildInstructions(reporter, testSource, reporter.errorCount, reporter.warningCount)
             }
 
             (reporter.compilerCrashed, reporter.errorCount, reporter.warningCount, () => verifyOutput(checkFile, outDir, testSource, reporter.warningCount))
-          }
 
-          case testSource @ SeparateCompilationSource(_, dir, flags, outDir) => {
+          case testSource @ SeparateCompilationSource(_, dir, flags, outDir) =>
             val checkFile = new JFile(dir.getAbsolutePath.reverse.dropWhile(_ == '/').reverse + ".check")
             val reporters = testSource.compilationGroups.map(compile(_, flags, false, outDir))
             val compilerCrashed = reporters.exists(_.compilerCrashed)
@@ -584,7 +606,6 @@ trait ParallelTesting extends RunnerOrchestration { self =>
             }
 
             (compilerCrashed, errorCount, warningCount, () => verifyOutput(Some(checkFile), outDir, testSource, warningCount))
-          }
         }
 
         if (!compilerCrashed && errorCount == 0) verifier()
@@ -655,7 +676,7 @@ trait ParallelTesting extends RunnerOrchestration { self =>
         }
 
         val (compilerCrashed, expectedErrors, actualErrors, hasMissingAnnotations, errorMap) = testSource match {
-          case testSource @ JointCompilationSource(_, files, flags, outDir) => {
+          case testSource @ JointCompilationSource(_, files, flags, outDir, fromTasty) =>
             val sourceFiles = testSource.sourceFiles
             val (errorMap, expectedErrors) = getErrorMapAndExpectedCount(sourceFiles)
             val reporter = compile(sourceFiles, flags, true, outDir)
@@ -665,7 +686,6 @@ trait ParallelTesting extends RunnerOrchestration { self =>
               logReporterContents(reporter)
 
             (reporter.compilerCrashed, expectedErrors, actualErrors, () => getMissingExpectedErrors(errorMap, reporter.errors), errorMap)
-          }
 
           case testSource @ SeparateCompilationSource(_, dir, flags, outDir) => {
             val compilationGroups = testSource.compilationGroups
@@ -941,7 +961,7 @@ trait ParallelTesting extends RunnerOrchestration { self =>
      */
     def copyToTarget(): CompilationTest = new CompilationTest (
       targets.map {
-        case target @ JointCompilationSource(_, files, _, outDir) =>
+        case target @ JointCompilationSource(_, files, _, outDir, _) =>
           target.copy(files = files.map(copyToDir(outDir,_)))
         case target @ SeparateCompilationSource(_, dir, _, outDir) =>
           target.copy(dir = copyToDir(outDir, dir))
@@ -1056,6 +1076,30 @@ trait ParallelTesting extends RunnerOrchestration { self =>
     new CompilationTest(target)
   }
 
+  /** Compiles a single file from the string path `f` using the supplied flags */
+  def compileTasty(f: String, flags: TestFlags)(implicit testGroup: TestGroup): (CompilationTest, CompilationTest) = {
+    val sourceFile = new JFile(f)
+    val parent = sourceFile.getParentFile
+    val outDir =
+      defaultOutputDir + testGroup + "/" +
+        sourceFile.getName.substring(0, sourceFile.getName.lastIndexOf('.')) + "/"
+
+    require(
+      sourceFile.exists && !sourceFile.isDirectory &&
+        (parent ne null) && parent.exists && parent.isDirectory,
+      s"Source file: $f, didn't exist"
+    )
+    val tastySource = createOutputDirsForFile(sourceFile, parent, outDir)
+    val target = JointCompilationSource(
+      testGroup.name,
+      Array(sourceFile),
+      flags.withClasspath(tastySource.getPath) and "-Yretain-trees",
+      tastySource,
+      fromTasty = true
+    )
+    (compileFile(f, flags), new CompilationTest(target))
+  }
+
   /** Compiles a directory `f` using the supplied `flags`. This method does
    *  deep compilation, that is - it compiles all files and subdirectories
    *  contained within the directory `f`.
@@ -1137,6 +1181,44 @@ trait ParallelTesting extends RunnerOrchestration { self =>
     new CompilationTest(targets)
   }
 
+  /** This function compiles the files and folders contained within directory
+   *  `f` in a specific way. Once compiled, they are recompiled/run from tasty as sources.
+   *
+   *  - Each file is compiled separately as a single compilation run
+   *  - Each directory is compiled as a `SeparateCompilationTaret`, in this
+   *    target all files are grouped according to the file suffix `_X` where `X`
+   *    is a number. These groups are then ordered in ascending order based on
+   *    the value of `X` and each group is compiled one after the other.
+   *
+   *  For this function to work as expected, we use the same convention for
+   *  directory layout as the old partest. That is:
+   *
+   *  - Single files can have an associated check-file with the same name (but
+   *    with file extension `.check`)
+   *  - Directories can have an associated check-file, where the check file has
+   *    the same name as the directory (with the file extension `.check`)
+   */
+  def compileTastyInDir(f: String, flags0: TestFlags)(implicit testGroup: TestGroup): (CompilationTest, CompilationTest) = {
+    val outDir = defaultOutputDir + testGroup + "/"
+    val flags = flags0 and "-Yretain-trees"
+    val sourceDir = new JFile(f)
+    checkRequirements(f, sourceDir, outDir)
+
+    val (dirs, files) = compilationTargets(sourceDir)
+
+    val targets =
+      files.map { f =>
+        val classpath = createOutputDirsForFile(f, sourceDir, outDir)
+        JointCompilationSource(testGroup.name, Array(f), flags.withClasspath(classpath.getPath), classpath, fromTasty = true)
+      }
+    // TODO add SeparateCompilationSource from tasty?
+
+    // Create a CompilationTest and let the user decide whether to execute a pos or a neg test
+    val generateClassFiles = compileFilesInDir(f, flags0)
+    (generateClassFiles, new CompilationTest(targets))
+  }
+
+
   /** This function behaves similar to `compileFilesInDir` but it ignores
    *  sub-directories and as such, does **not** perform separate compilation
    *  tests.
@@ -1165,4 +1247,7 @@ object ParallelTesting {
     val name = f.getName
     name.endsWith(".scala") || name.endsWith(".java")
   }
+
+  def isHasTastyFile(f: JFile): Boolean =
+    f.getName.endsWith(".hasTasty")
 }

compiler/test/dotty/tools/vulpix/TestFlags.scala

@@ -11,6 +11,9 @@ final case class TestFlags(
   def without(flags: String*): TestFlags =
     TestFlags(defaultClassPath, runClassPath, options diff flags)
 
+  def withClasspath(classPath: String): TestFlags =
+    TestFlags(s"$defaultClassPath:$classPath", runClassPath, options)
+
   def all: Array[String] = Array("-classpath", defaultClassPath) ++ options
 }
 

tests/pos-from-tasty/simpleClass.scala

@@ -0,0 +1,5 @@
+package foo
+
+class A
+
+class B extends A

tests/run-from-tasty/innerClass.check

@@ -0,0 +1,2 @@
+foo.A
+foo.A$B