Fix #5743: Add check files to neg tests

The check files contains the error messages positions and messages

Author: nicolasstucki

compiler/test/dotty/tools/vulpix/ParallelTesting.scala

@@ -616,23 +616,12 @@ trait ParallelTesting extends RunnerOrchestration { self =>
       else runMain(testSource.runClassPath) match {
         case Success(_) if !checkFile.isDefined || !checkFile.get.exists => // success!
         case Success(output) => {
-          val outputLines = output.linesIterator.toArray :+ DiffUtil.EOF
-          val checkLines: Array[String] = Source.fromFile(checkFile.get, "UTF-8").getLines().toArray :+ DiffUtil.EOF
+          val outputLines = output.linesIterator.toSeq
+          val checkLines: Seq[String] = Source.fromFile(checkFile.get, "UTF-8").getLines().toSeq
           val sourceTitle = testSource.title
 
-          def linesMatch =
-            outputLines
-            .zip(checkLines)
-            .forall { case (x, y) => x == y }
+          diffMessage(sourceTitle, outputLines, checkLines).foreach { msg =>
 
-          if (outputLines.length != checkLines.length || !linesMatch) {
-            // Print diff to files and summary:
-            val diff = DiffUtil.mkColoredLineDiff(checkLines, outputLines)
-
-            val msg =
-              s"""|Output from '$sourceTitle' did not match check file.
-                  |Diff (expected on the left, actual right):
-                  |""".stripMargin + diff + "\n"
             echo(msg)
             addFailureInstruction(msg)
 
@@ -765,13 +754,36 @@ trait ParallelTesting extends RunnerOrchestration { self =>
           }
         }
 
+        def fail(msg: String): Unit = {
+          echo(msg)
+          failTestSource(testSource)
+        }
+
+        def reporterOutputLines(reporter: TestReporter): List[String] = {
+          reporter.allErrors.flatMap { error =>
+            (error.pos.span.toString + " in " + error.pos.source.file.name) :: error.getMessage().split("\n").toList
+          }
+        }
+        def checkFileTest(sourceName: String, checkFile: JFile, actual: List[String]) = {
+          if (checkFile.exists) {
+            val expexted = Source.fromFile(checkFile, "UTF-8").getLines().toList
+            diffMessage(sourceName, actual, expexted).foreach(fail)
+          }
+        }
+
         val (compilerCrashed, expectedErrors, actualErrors, hasMissingAnnotations, errorMap) = testSource match {
           case testSource @ JointCompilationSource(_, files, flags, outDir, fromTasty, decompilation) =>
             val sourceFiles = testSource.sourceFiles
             val (errorMap, expectedErrors) = getErrorMapAndExpectedCount(sourceFiles)
             val reporter = compile(sourceFiles, flags, true, outDir)
             val actualErrors = reporter.errorCount
-
+            files.foreach { file =>
+              if (file.isDirectory) Nil
+              else {
+                val checkFile = new JFile(file.getAbsolutePath.reverse.dropWhile(_ != '.').reverse + "check")
+                checkFileTest(testSource.title, checkFile, reporterOutputLines(reporter))
+              }
+            }
             if (reporter.compilerCrashed || actualErrors > 0)
               logReporterContents(reporter)
 
@@ -788,14 +800,13 @@ trait ParallelTesting extends RunnerOrchestration { self =>
             if (actualErrors > 0)
               reporters.foreach(logReporterContents)
 
+            val checkFile = new JFile(dir.getAbsolutePath.reverse.dropWhile(_ == JFile.separatorChar).reverse + ".check")
+            checkFileTest(testSource.title, checkFile, reporters.flatMap(reporter => reporterOutputLines(reporter)))
+
             (compilerCrashed, expectedErrors, actualErrors, () => getMissingExpectedErrors(errorMap, errors), errorMap)
           }
         }
 
-        def fail(msg: String): Unit = {
-          echo(msg)
-          failTestSource(testSource)
-        }
 
         if (compilerCrashed)
           fail(s"Compiler crashed when compiling: ${testSource.title}")
@@ -839,6 +850,24 @@ trait ParallelTesting extends RunnerOrchestration { self =>
     }
   }
 
+  def diffMessage(sourceTitle: String, outputLines: Seq[String], checkLines: Seq[String]): Option[String] = {
+    def linesMatch =
+      outputLines
+        .zip(checkLines)
+        .forall { case (x, y) => x == y }
+
+    if (outputLines.length != checkLines.length || !linesMatch) {
+      // Print diff to files and summary:
+      val diff = DiffUtil.mkColoredLineDiff(checkLines :+ DiffUtil.EOF, outputLines :+ DiffUtil.EOF)
+
+      Some(
+        s"""|Output from '$sourceTitle' did not match check file.
+            |Diff (expected on the left, actual right):
+            |""".stripMargin + diff + "\n")
+    } else None
+
+  }
+
   /** The `CompilationTest` is the main interface to `ParallelTesting`, it
    *  can be instantiated via one of the following methods:
    *

tests/neg/classOf.check

@@ -0,0 +1,6 @@
+[181..202] in classOf.scala
+Test.C{I = String} is not a class type
+[116..117] in classOf.scala
+T is not a class type
+[72..73] in classOf.scala
+T is not a class type

tests/neg/i4382.check

@@ -0,0 +1,8 @@
+[199..207] in i4382.scala
+unreducible application of higher-kinded type App.HkAbs to wildcard arguments
+[149..155] in i4382.scala
+unreducible application of higher-kinded type App.HkU to wildcard arguments
+[97..103] in i4382.scala
+unreducible application of higher-kinded type App.HkL to wildcard arguments
+[46..51] in i4382.scala
+unreducible application of higher-kinded type App.Id to wildcard arguments

tests/neg/i5311.check

@@ -0,0 +1,3 @@
+<267..267> in i5311.scala
+Found:    s.T[Int] => s.T[Int]
+Required: m.Foo

tests/neg/kinds2.check

@@ -0,0 +1,2 @@
+[123..124] in kinds2.scala
+missing type parameter(s) for C

tests/neg/quote-error-2.check

@@ -0,0 +1,2 @@
+[48..51..58] in Test_2.scala
+foo cannot be called with false

tests/neg/quote-error.check

@@ -0,0 +1,2 @@
+[48..51..58] in Test_2.scala
+foo cannot be called with false

tests/neg/subtyping.check

@@ -0,0 +1,14 @@
+<230..230> in subtyping.scala
+Cannot prove that a.T <:< a.U..
+I found:
+
+    $conforms[Nothing]
+
+But method $conforms in object Predef does not match type a.T <:< a.U.
+<106..106> in subtyping.scala
+Cannot prove that B#X <:< A#X..
+I found:
+
+    $conforms[Nothing]
+
+But method $conforms in object Predef does not match type B#X <:< A#X.

tests/neg/t6663.check

@@ -1,6 +1,3 @@
-t6663.scala:16: error: type mismatch;
- found   : String
- required: Int
-  var v = new C(42).foo[String].get :Int
-                                ^
-one error found
+[473..495..498] in t6663.scala
+Found:    String
+Required: Int