Merge pull request #5894 from dotty-staging/add-update-checkfiles

Add `updateCheckFiles` to override tests check files with output

Author: nicolasstucki

compiler/test/dotc/comptest.scala

@@ -11,6 +11,7 @@ object comptest extends ParallelTesting {
   def safeMode = false
   def isInteractive = true
   def testFilter = None
+  def updateCheckFiles: Boolean = false
 
   val posDir = "./tests/pos/"
   val negDir = "./tests/neg/"

compiler/test/dotty/Properties.scala

@@ -20,6 +20,9 @@ object Properties {
    */
   val testsFilter: Option[String] = sys.props.get("dotty.tests.filter")
 
+  /** Tests should override the checkfiles with the current output */
+  val testsUpdateCheckfile: Boolean = propIsNullOrTrue("dotty.tests.updateCheckfiles")
+
   /** When set, the run tests are only compiled - not run, a warning will be
    *  issued
    */

compiler/test/dotty/tools/dotc/BootstrappedOnlyCompilationTests.scala

@@ -7,13 +7,8 @@ import org.junit.Assert._
 import org.junit.Assume._
 import org.junit.experimental.categories.Category
 
-import java.nio.file._
-import java.util.stream.{ Stream => JStream }
-import scala.collection.JavaConverters._
-import scala.util.matching.Regex
 import scala.concurrent.duration._
 import vulpix._
-import dotty.tools.io.JFile
 
 @Category(Array(classOf[BootstrappedOnlyTests]))
 class BootstrappedOnlyCompilationTests extends ParallelTesting {
@@ -28,6 +23,7 @@ class BootstrappedOnlyCompilationTests extends ParallelTesting {
   def safeMode = Properties.testsSafeMode
   def isInteractive = SummaryReport.isInteractive
   def testFilter = Properties.testsFilter
+  def updateCheckFiles: Boolean = Properties.testsUpdateCheckfile
 
   // Positive tests ------------------------------------------------------------
 

compiler/test/dotty/tools/dotc/CompilationTests.scala

@@ -27,6 +27,7 @@ class CompilationTests extends ParallelTesting {
   def safeMode = Properties.testsSafeMode
   def isInteractive = SummaryReport.isInteractive
   def testFilter = Properties.testsFilter
+  def updateCheckFiles: Boolean = Properties.testsUpdateCheckfile
 
   // Positive tests ------------------------------------------------------------
 

compiler/test/dotty/tools/dotc/FromTastyTests.scala

@@ -20,7 +20,7 @@ class FromTastyTests extends ParallelTesting {
   def safeMode = Properties.testsSafeMode
   def isInteractive = SummaryReport.isInteractive
   def testFilter = Properties.testsFilter
-
+  def updateCheckFiles: Boolean = Properties.testsUpdateCheckfile
 
   @Test def posTestFromTasty: Unit = {
     // Can be reproduced with

compiler/test/dotty/tools/dotc/IdempotencyTests.scala

@@ -24,6 +24,7 @@ class IdempotencyTests extends ParallelTesting {
   def safeMode = Properties.testsSafeMode
   def isInteractive = SummaryReport.isInteractive
   def testFilter = Properties.testsFilter
+  def updateCheckFiles: Boolean = Properties.testsUpdateCheckfile
 
   @Category(Array(classOf[SlowTests]))
   @Test def idempotency: Unit = {

compiler/test/dotty/tools/vulpix/ParallelTesting.scala

@@ -46,6 +46,9 @@ trait ParallelTesting extends RunnerOrchestration { self =>
    */
   def testFilter: Option[String]
 
+  /** Tests should override the checkfiles with the current output */
+  def updateCheckFiles: Boolean
+
   /** A test source whose files or directory of files is to be compiled
    *  in a specific way defined by the `Test`
    */
@@ -505,6 +508,12 @@ trait ParallelTesting extends RunnerOrchestration { self =>
       this
     }
 
+    protected def updateCheckFile(checkFile: JFile, lines: Seq[String]): Unit = {
+      val outFile = dotty.tools.io.File(checkFile.toPath)
+      outFile.writeAll(lines.mkString("", EOL, EOL))
+      echo("Updated checkfile: " + checkFile.getPath)
+    }
+
     /** Returns all files in directory or the file if not a directory */
     private def flattenFiles(f: JFile): Array[JFile] =
       if (f.isDirectory) f.listFiles.flatMap(flattenFiles)
@@ -537,28 +546,32 @@ trait ParallelTesting extends RunnerOrchestration { self =>
                     val output = Source.fromFile(outDir.getParent + "_decompiled" + JFile.separator + outDir.getName
                       + JFile.separator + "decompiled.scala", "UTF-8").getLines().map {line =>
                       stripTrailingWhitespaces.unapplySeq(line).map(_.head).getOrElse(line)
-                    }.toList
+                    }.filter(!_.startsWith(ignoredFilePathLine)).toList
 
-                    val check: String = Source.fromFile(checkFile, "UTF-8").getLines().filter(!_.startsWith(ignoredFilePathLine))
+                    val check: String = Source.fromFile(checkFile, "UTF-8").getLines()
                       .mkString(EOL)
 
-                    if (output.filter(!_.startsWith(ignoredFilePathLine)).mkString(EOL) != check) {
+                    if (output.mkString(EOL) != check) {
                       val outFile = dotty.tools.io.File(checkFile.toPath).addExtension(".out")
-                      outFile.writeAll(output.mkString(EOL))
-                      val msg =
-                        s"""Output differed for test $name, use the following command to see the diff:
-                           |  > diff $checkFile $outFile
+                      if (updateCheckFiles) {
+                        updateCheckFile(checkFile, output)
+                      } else {
+                        outFile.writeAll(output.mkString("", EOL, ""))
+                        val msg =
+                          s"""Output differed for test $name, use the following command to see the diff:
+                             |  > diff $checkFile $outFile
                         """.stripMargin
 
-                      echo(msg)
-                      addFailureInstruction(msg)
+                        echo(msg)
+                        addFailureInstruction(msg)
 
-                      // Print build instructions to file and summary:
-                      val buildInstr = testSource.buildInstructions(0, rep.warningCount)
-                      addFailureInstruction(buildInstr)
+                        // Print build instructions to file and summary:
+                        val buildInstr = testSource.buildInstructions(0, rep.warningCount)
+                        addFailureInstruction(buildInstr)
 
-                      // Fail target:
-                      failTestSource(testSource)
+                        // Fail target:
+                        failTestSource(testSource)
+                      }
                     }
                   case _ =>
                 }
@@ -631,6 +644,9 @@ trait ParallelTesting extends RunnerOrchestration { self =>
 
             // Fail target:
             failTestSource(testSource)
+
+            if (updateCheckFiles)
+              updateCheckFile(checkFile.get, outputLines)
           }
         }
 
@@ -766,7 +782,11 @@ trait ParallelTesting extends RunnerOrchestration { self =>
         }
         def checkFileTest(sourceName: String, checkFile: JFile, actual: List[String]) = {
           val expexted = Source.fromFile(checkFile, "UTF-8").getLines().toList
-          diffMessage(sourceName, actual, expexted).foreach(fail)
+          for (msg <- diffMessage(sourceName, actual, expexted)) {
+            fail(msg)
+            if (updateCheckFiles)
+              updateCheckFile(checkFile, actual)
+          }
         }
 
         val (compilerCrashed, expectedErrors, actualErrors, hasMissingAnnotations, errorMap) = testSource match {
@@ -1371,7 +1391,7 @@ trait ParallelTesting extends RunnerOrchestration { self =>
             |Test '$title' compiled with $errors error(s) and $warnings warning(s),
             |the test can be reproduced by running:
             |
-            |  sbt "testFromTasty $file"
+            |  sbt "testCompilation --from-tasty $file"
             |
             |This tests can be disabled by adding `${file.getName}` to `compiler${JFile.separator}test${JFile.separator}dotc${JFile.separator}$runOrPos-$listName.blacklist`
             |

compiler/test/dotty/tools/vulpix/SummaryReport.scala

@@ -40,6 +40,7 @@ trait SummaryReporting {
 
   /** Echoes contents of `it` to file *immediately* then flushes */
   def echoToLog(it: Iterator[String]): Unit
+
 }
 
 /** A summary report that doesn't do anything */
@@ -53,6 +54,7 @@ final class NoSummaryReport extends SummaryReporting {
   def echoSummary(): Unit = ()
   def echoToLog(msg: String): Unit = ()
   def echoToLog(it: Iterator[String]): Unit = ()
+  def updateCheckFiles: Boolean = false
 }
 
 /** A summary report that logs to both stdout and the `TestReporter.logWriter`

compiler/test/dotty/tools/vulpix/VulpixMetaTests.scala

@@ -4,7 +4,6 @@ package vulpix
 import org.junit.Test
 import org.junit.experimental.categories.Category
 import scala.concurrent.duration._
-import dotty.Properties
 import TestConfiguration._
 
 /** Meta tests for the Vulpix test suite. This test follows the structure of
@@ -19,6 +18,7 @@ class VulpixMetaTests extends ParallelTesting {
   def safeMode = false // Don't fork a new VM after each run test
   def isInteractive = false // Don't beautify output for interactive use.
   def testFilter = None // Run all the tests.
+  def updateCheckFiles: Boolean = false
 
   implicit val summaryReport: SummaryReporting = new SummaryReport
   implicit def testGroup: TestGroup = TestGroup("VulpixMetaTests")

compiler/test/dotty/tools/vulpix/VulpixUnitTests.scala

@@ -21,6 +21,7 @@ class VulpixUnitTests extends ParallelTesting {
   def safeMode = sys.env.get("SAFEMODE").isDefined
   def isInteractive = !sys.env.contains("DRONE")
   def testFilter = None
+  def updateCheckFiles: Boolean = false
 
   // To fail with something else than an AssertionError
   def fail(): Unit = throw new Exception("didn't fail properly")