Merge pull request #4266 from dotty-staging/compile-decompiled-code

Test compilation of decompiled code

Author: nicolasstucki

compiler/src/dotty/tools/dotc/decompiler/DecompilationPrinter.scala

@@ -24,7 +24,7 @@ class DecompilationPrinter extends Phase {
       var os: OutputStream = null
       var ps: PrintStream = null
       try {
-        os = File(outputDir + ".decompiled").outputStream(append = true)
+        os = File(outputDir + "/decompiled.scala").outputStream(append = true)
         ps = new PrintStream(os)
         printToOutput(ps)
       } finally {
@@ -38,21 +38,15 @@ class DecompilationPrinter extends Phase {
     val unit = ctx.compilationUnit
     val pageWidth = ctx.settings.pageWidth.value
     val printLines = ctx.settings.printLines.value
-    val doubleLine = "=" * pageWidth
-    val line = "-" * pageWidth
-
-    out.println(doubleLine)
-    out.println(unit.source)
-    out.println(line)
 
+    out.println(s"/** Decompiled from $unit */")
     val printer = new DecompilerPrinter(ctx)
-
     out.println(printer.toText(unit.tpdTree).mkString(pageWidth, printLines))
-    out.println(line)
 
     if (ctx.settings.printTasty.value) {
+      out.println("/*")
       new TastyPrinter(unit.pickled.head._2).printContents()
-      out.println(line)
+      out.println("*/")
     }
   }
 }

compiler/src/dotty/tools/dotc/decompiler/Main.scala

@@ -14,13 +14,14 @@ object Main extends dotc.Driver {
     assert(ctx.settings.fromTasty.value)
     val outputDir = ctx.settings.outputDir.value
     if (outputDir != ".")
-      Files.deleteIfExists(Paths.get(outputDir + ".decompiled"))
+      Files.deleteIfExists(Paths.get(outputDir + "/decompiled.scala"))
     new TASTYDecompiler
   }
 
   override def setup(args0: Array[String], rootCtx: Context): (List[String], Context) = {
     var args = args0.filter(a => a != "-decompile")
-    args = if (args.contains("-from-tasty")) args else "-from-tasty" +: args
+    if (!args.contains("-from-tasty")) args = "-from-tasty" +: args
+    if (args.contains("-d")) args = "-color:never" +: args
     super.setup(args, rootCtx)
   }
 }

compiler/src/dotty/tools/dotc/printing/DecompilerPrinter.scala

@@ -1,7 +1,7 @@
 package dotty.tools.dotc.printing
 
 import dotty.tools.dotc.ast.Trees._
-import dotty.tools.dotc.ast.untpd.{PackageDef, Template, TypeDef}
+import dotty.tools.dotc.ast.untpd.{Tree, PackageDef, Template, TypeDef}
 import dotty.tools.dotc.ast.{Trees, untpd}
 import dotty.tools.dotc.printing.Texts._
 import dotty.tools.dotc.core.Contexts._
@@ -55,8 +55,21 @@ class DecompilerPrinter(_ctx: Context) extends RefinedPrinter(_ctx) {
   }
 
   override protected def toTextTemplate(impl: Template, ofNew: Boolean = false): Text = {
-    val impl1 = impl.copy(parents = impl.parents.filterNot(_.symbol.maybeOwner == defn.ObjectClass))
-    super.toTextTemplate(impl1, ofNew)
+    def isSynthetic(parent: Tree): Boolean = {
+      val sym = parent.symbol
+      sym.maybeOwner == defn.ObjectClass ||
+      (sym == defn.ProductClass && impl.symbol.owner.is(Case))
+    }
+    val parents = impl.parents.filterNot(isSynthetic)
+
+    // We don't print self type and constructor for objects
+    val isObject = impl.constr.symbol.owner.is(Module)
+    if (isObject) {
+      val parentsText = keywordText(" extends") ~~ Text(parents.map(constrText), keywordStr(" with "))
+      val bodyText = " {" ~~ toTextGlobal(impl.body, "\n") ~ "}"
+      parentsText.provided(parents.nonEmpty) ~ bodyText
+    }
+    else super.toTextTemplate(impl.copy(parents = parents), ofNew)
   }
 
   override protected def typeApplyText[T >: Untyped](tree: TypeApply[T]): Text = {

compiler/src/dotty/tools/dotc/printing/RefinedPrinter.scala

@@ -651,8 +651,7 @@ class RefinedPrinter(_ctx: Context) extends PlainPrinter(_ctx) {
     val selfText = {
       val selfName = if (self.name == nme.WILDCARD) keywordStr("this") else self.name.toString
       (selfName ~ optText(self.tpt)(": " ~ _) ~ " =>").close
-    } provided !self.isEmpty
-
+    }.provided(!self.isEmpty)
     val body = if (ctx.settings.YtestPickler.value) {
       // Pickling/unpickling reorders the body members, so we need to homogenize
       val (params, rest) = impl.body partition {
@@ -664,9 +663,9 @@ class RefinedPrinter(_ctx: Context) extends PlainPrinter(_ctx) {
       params ::: rest
     } else impl.body
 
-    val bodyText = "{" ~~ selfText ~~ toTextGlobal(primaryConstrs ::: body, "\n") ~ "}"
+    val bodyText = " {" ~~ selfText ~~ toTextGlobal(primaryConstrs ::: body, "\n") ~ "}"
 
-    prefix ~ (keywordText(" extends") provided (!ofNew && parents.nonEmpty)) ~~ parentsText ~~ bodyText
+    prefix ~ keywordText(" extends").provided(!ofNew && parents.nonEmpty) ~~ parentsText ~ bodyText
   }
 
   protected def templateText(tree: TypeDef, impl: Template): Text = {

compiler/test/dotty/tools/dotc/FromTastyTests.scala

@@ -26,7 +26,7 @@ class FromTastyTests extends ParallelTesting {
     // > dotc -Ythrough-tasty -Ycheck:all <source>
 
     implicit val testGroup: TestGroup = TestGroup("posTestFromTasty")
-    val (step1, step2, step3) = compileTastyInDir("tests/pos", defaultOptions,
+    compileTastyInDir("tests/pos", defaultOptions,
       blacklist = Set(
         // Wrong number of arguments (only on bootstrapped)
         "i3130b.scala",
@@ -36,12 +36,11 @@ class FromTastyTests extends ParallelTesting {
 
         // MatchError in SymDenotation.sourceModule on a ThisType
         "t3612.scala",
+      ),
+      recompilationBlacklist = Set(
+        "simpleCaseObject"
       )
-    )
-    step1.checkCompile() // Compile all files to generate the class files with tasty
-    step2.checkCompile() // Compile from tasty
-    step3.checkCompile() // Decompile from tasty
-    (step1 + step2 + step3).delete()
+    ).checkCompile()
   }
 
   @Test def runTestFromTasty: Unit = {
@@ -51,21 +50,13 @@ class FromTastyTests extends ParallelTesting {
     // > dotr Test
 
     implicit val testGroup: TestGroup = TestGroup("runTestFromTasty")
-    val (step1, step2, step3) = compileTastyInDir("tests/run", defaultOptions,
-       blacklist = Set(
-         // Closure type miss match
-         "eff-dependent.scala",
-       )
-    )
-    step1.checkCompile() // Compile all files to generate the class files with tasty
-    step2.checkRuns() // Compile from tasty and run the result
-    step3.checkCompile() // Decompile from tasty
-    (step1 + step2 + step3).delete()
-  }
-
-  private implicit class tastyCompilationTuples(tup: (CompilationTest, CompilationTest)) {
-    def +(that: (CompilationTest, CompilationTest)): (CompilationTest, CompilationTest) =
-      (tup._1 + that._1, tup._2 + that._2)
+    compileTastyInDir("tests/run", defaultOptions,
+      blacklist = Set(
+        // Closure type miss match
+        "eff-dependent.scala",
+      ),
+      recompilationBlacklist = Set()
+    ).checkRuns()
   }
 }
 

compiler/test/dotty/tools/vulpix/ParallelTesting.scala

@@ -2,27 +2,27 @@ package dotty
 package tools
 package vulpix
 
-import java.io.{ File => JFile }
+import java.io.{File => JFile}
 import java.text.SimpleDateFormat
 import java.util.HashMap
 import java.nio.file.StandardCopyOption.REPLACE_EXISTING
-import java.nio.file.{ Files, Path, Paths, NoSuchFileException }
-import java.util.concurrent.{ Executors => JExecutors, TimeUnit, TimeoutException }
+import java.nio.file.{Files, NoSuchFileException, Path, Paths}
+import java.util.concurrent.{TimeUnit, TimeoutException, Executors => JExecutors}
 
 import scala.io.Source
 import scala.util.control.NonFatal
 import scala.util.Try
 import scala.collection.mutable
 import scala.util.matching.Regex
 import scala.util.Random
-
 import dotc.core.Contexts._
-import dotc.reporting.{ Reporter, TestReporter }
+import dotc.reporting.{Reporter, TestReporter}
 import dotc.reporting.diagnostic.MessageContainer
 import dotc.interfaces.Diagnostic.ERROR
 import dotc.util.DiffUtil
-import dotc.{ Driver, Compiler }
+import dotc.{Compiler, Driver}
 import dotc.decompiler
+import dotty.tools.vulpix.TestConfiguration.defaultOptions
 
 /** A parallel testing suite whose goal is to integrate nicely with JUnit
  *
@@ -425,16 +425,17 @@ trait ParallelTesting extends RunnerOrchestration { self =>
       reporter
     }
 
-    protected def decompile(flags0: TestFlags, suppressErrors: Boolean, targetDir: JFile): TestReporter = {
-      val decompilationOutput = new JFile(targetDir.getPath)
-      decompilationOutput.mkdir()
+    protected def decompile(flags0: TestFlags, suppressErrors: Boolean, targetDir0: JFile): TestReporter = {
+      val targetDir = new JFile(targetDir0.getParent + "_decompiled")
+      val decompilationOutput = new JFile(targetDir + "/" + targetDir0.getName)
+      decompilationOutput.mkdirs()
       val flags =
         flags0 and ("-d", decompilationOutput.getAbsolutePath) and
         "-decompile" and "-pagewidth" and "80"
 
       def hasTastyFileToClassName(f: JFile): String =
-        targetDir.toPath.relativize(f.toPath).toString.dropRight(".hasTasty".length).replace('/', '.')
-      val classes = flattenFiles(targetDir).filter(isHasTastyFile).map(hasTastyFileToClassName).sorted
+        targetDir0.toPath.relativize(f.toPath).toString.dropRight(".hasTasty".length).replace('/', '.')
+      val classes = flattenFiles(targetDir0).filter(isHasTastyFile).map(hasTastyFileToClassName).sorted
 
       val reporter =
         TestReporter.reporter(realStdout, logLevel =
@@ -522,7 +523,7 @@ trait ParallelTesting extends RunnerOrchestration { self =>
                 checkFileOpt match {
                   case Some(checkFile) =>
                     val stripTrailingWhitespaces = "(.*\\S|)\\s+".r
-                    val output = Source.fromFile(outDir + ".decompiled").getLines().map {line =>
+                    val output = Source.fromFile(outDir.getParent + "_decompiled/" + outDir.getName + "/decompiled.scala").getLines().map {line =>
                       stripTrailingWhitespaces.unapplySeq(line).map(_.head).getOrElse(line)
                     }.mkString("\n")
 
@@ -1257,8 +1258,8 @@ trait ParallelTesting extends RunnerOrchestration { self =>
    *  Tests in the first part of the tuple must be executed before the second.
    *  Both testsRequires explicit delete().
    */
-  def compileTastyInDir(f: String, flags0: TestFlags, blacklist: Set[String] = Set.empty)(
-      implicit testGroup: TestGroup): (CompilationTest, CompilationTest, CompilationTest) = {
+  def compileTastyInDir(f: String, flags0: TestFlags, blacklist: Set[String], recompilationBlacklist: Set[String])(
+      implicit testGroup: TestGroup): TastyCompilationTest = {
     val outDir = defaultOutputDir + testGroup + "/"
     val flags = flags0 and "-Yretain-trees"
     val sourceDir = new JFile(f)
@@ -1284,13 +1285,52 @@ trait ParallelTesting extends RunnerOrchestration { self =>
     // Create a CompilationTest and let the user decide whether to execute a pos or a neg test
     val generateClassFiles = compileFilesInDir(f, flags0, blacklist)
 
-    (
+    val decompilationDir = outDir + sourceDir.getName + "_decompiled"
+
+    new TastyCompilationTest(
       generateClassFiles.keepOutput,
       new CompilationTest(targets).keepOutput,
-      new CompilationTest(targets2).keepOutput
+      new CompilationTest(targets2).keepOutput,
+      recompilationBlacklist,
+      decompilationDir,
+      shouldDelete = true
     )
   }
 
+  class TastyCompilationTest(step1: CompilationTest, step2: CompilationTest, step3: CompilationTest,
+      recompilationBlacklist: Set[String], decompilationDir: String, shouldDelete: Boolean)(implicit testGroup: TestGroup) {
+
+    def keepOutput: TastyCompilationTest =
+      new TastyCompilationTest(step1, step2, step3, recompilationBlacklist, decompilationDir, shouldDelete)
+
+    def checkCompile()(implicit summaryReport: SummaryReporting): this.type = {
+      step1.checkCompile() // Compile all files to generate the class files with tasty
+      step2.checkCompile() // Compile from tasty
+      step3.checkCompile() // Decompile from tasty
+
+      val step4 = compileFilesInDir(decompilationDir, defaultOptions, recompilationBlacklist).keepOutput
+      step4.checkCompile() // Recompile decompiled code
+
+      if (shouldDelete)
+        (step1 + step2 + step3 + step4).delete()
+
+      this
+    }
+
+    def checkRuns()(implicit summaryReport: SummaryReporting): this.type = {
+      step1.checkCompile() // Compile all files to generate the class files with tasty
+      step2.checkRuns() // Compile from tasty
+      step3.checkCompile() // Decompile from tasty
+
+      val step4 = compileFilesInDir(decompilationDir, defaultOptions, recompilationBlacklist).keepOutput
+      step4.checkRuns() // Recompile decompiled code
+
+      if (shouldDelete)
+        (step1 + step2 + step3 + step4).delete()
+
+      this
+    }
+  }
 
   /** This function behaves similar to `compileFilesInDir` but it ignores
    *  sub-directories and as such, does **not** perform separate compilation

tests/pos/lambda.decompiled

@@ -1,6 +1,4 @@
-================================================================================
-out/posTestFromTasty/pos/lambda/foo/Foo.class
---------------------------------------------------------------------------------
+/** Decompiled from out/posTestFromTasty/pos/lambda/foo/Foo.class */
 package foo {
   class Foo() {
     {
@@ -11,5 +9,4 @@ package foo {
     }
     val a: Int => Int = (x: Int) => x.*(x)
   }
-}
---------------------------------------------------------------------------------
+}

tests/pos/methodTypes.decompiled

@@ -1,9 +1,6 @@
-================================================================================
-out/posTestFromTasty/pos/methodTypes/Foo.class
---------------------------------------------------------------------------------
+/** Decompiled from out/posTestFromTasty/pos/methodTypes/Foo.class */
 class Foo() {
   val x: Int = 1
   def y: Int = 2
   def z(): Int = 3
-}
---------------------------------------------------------------------------------
+}

tests/pos/simpleCaseObject.decompiled

@@ -1,8 +1,6 @@
-================================================================================
-out/posTestFromTasty/pos/simpleCaseObject/foo/Foo.class
---------------------------------------------------------------------------------
+/** Decompiled from out/posTestFromTasty/pos/simpleCaseObject/foo/Foo.class */
 package foo {
-  case object Foo() extends _root_.scala.Product { this: foo.Foo.type =>
+  case object Foo {
     override def hashCode(): Int = 1045991777
     override def toString(): String = "Foo"
     override def canEqual(that: Any): Boolean =
@@ -15,5 +13,4 @@ package foo {
           case _ => throw new IndexOutOfBoundsException(n.toString())
         }
   }
-}
---------------------------------------------------------------------------------
+}

tests/pos/simpleClass-2.decompiled

@@ -1,14 +1,8 @@
-================================================================================
-out/posTestFromTasty/pos/simpleClass-2/foo/A.class
---------------------------------------------------------------------------------
+/** Decompiled from out/posTestFromTasty/pos/simpleClass-2/foo/A.class */
 package foo {
   class A() extends foo.B() {}
 }
---------------------------------------------------------------------------------
-================================================================================
-out/posTestFromTasty/pos/simpleClass-2/foo/B.class
---------------------------------------------------------------------------------
+/** Decompiled from out/posTestFromTasty/pos/simpleClass-2/foo/B.class */
 package foo {
   class B() {}
-}
---------------------------------------------------------------------------------
+}