Merge pull request #3621 from Gedochao/feature/run-multiple-test-frameworks

Run all found test frameworks, rather than just one

Author: Gedochao

modules/build/src/main/scala/scala/build/internal/Runner.scala

@@ -3,17 +3,21 @@ package scala.build.internal
 import coursier.jvm.Execve
 import org.scalajs.jsenv.jsdomnodejs.JSDOMNodeJSEnv
 import org.scalajs.jsenv.nodejs.NodeJSEnv
-import org.scalajs.jsenv.{Input, RunConfig}
+import org.scalajs.jsenv.{Input, JSEnv, RunConfig}
+import org.scalajs.testing.adapter.TestAdapter as ScalaJsTestAdapter
 import sbt.testing.{Framework, Status}
 
 import java.io.File
 import java.nio.file.{Files, Path, Paths}
 
 import scala.build.EitherCps.{either, value}
 import scala.build.Logger
-import scala.build.errors._
+import scala.build.Ops.EitherSeqOps
+import scala.build.errors.*
 import scala.build.internals.EnvVar
+import scala.build.testrunner.FrameworkUtils.*
 import scala.build.testrunner.{AsmTestRunner, TestRunner}
+import scala.scalanative.testinterface.adapter.TestAdapter as ScalaNativeTestAdapter
 import scala.util.{Failure, Properties, Success}
 
 object Runner {
@@ -238,22 +242,20 @@ object Runner {
     sourceMap: Boolean = false,
     esModule: Boolean = false
   ): Either[BuildException, Process] = either {
-
-    import logger.{log, debug}
-
-    val nodePath = value(findInPath("node").map(_.toString).toRight(NodeNotFoundError()))
-
-    if (!jsDom && allowExecve && Execve.available()) {
-
+    val nodePath: String =
+      value(findInPath("node")
+        .map(_.toString)
+        .toRight(NodeNotFoundError()))
+    if !jsDom && allowExecve && Execve.available() then {
       val command = Seq(nodePath, entrypoint.getAbsolutePath) ++ args
 
-      log(
+      logger.log(
         s"Running ${command.mkString(" ")}",
         "  Running" + System.lineSeparator() +
           command.iterator.map(_ + System.lineSeparator()).mkString
       )
 
-      debug("execve available")
+      logger.debug("execve available")
       Execve.execve(
         command.head,
         "node" +: command.tail.toArray,
@@ -262,40 +264,36 @@ object Runner {
       sys.error("should not happen")
     }
     else {
-
       val nodeArgs =
         // Scala.js runs apps by piping JS to node.
         // If we need to pass arguments, we must first make the piped input explicit
         // with "-", and we pass the user's arguments after that.
-        if (args.isEmpty) Nil
-        else "-" :: args.toList
+        if args.isEmpty then Nil else "-" :: args.toList
       val envJs =
-        if (jsDom)
+        if jsDom then
           new JSDOMNodeJSEnv(
             JSDOMNodeJSEnv.Config()
               .withExecutable(nodePath)
               .withArgs(nodeArgs)
               .withEnv(Map.empty)
           )
-        else new NodeJSEnv(
-          NodeJSEnv.Config()
-            .withExecutable(nodePath)
-            .withArgs(nodeArgs)
-            .withEnv(Map.empty)
-            .withSourceMap(sourceMap)
-        )
+        else
+          new NodeJSEnv(
+            NodeJSEnv.Config()
+              .withExecutable(nodePath)
+              .withArgs(nodeArgs)
+              .withEnv(Map.empty)
+              .withSourceMap(sourceMap)
+          )
 
-      val inputs = Seq(
-        if (esModule) Input.ESModule(entrypoint.toPath)
-        else Input.Script(entrypoint.toPath)
-      )
+      val inputs =
+        Seq(if esModule then Input.ESModule(entrypoint.toPath) else Input.Script(entrypoint.toPath))
 
       val config    = RunConfig().withLogger(logger.scalaJsLogger)
       val processJs = envJs.start(inputs, config)
 
       processJs.future.value.foreach {
-        case Failure(t) =>
-          throw new Exception(t)
+        case Failure(t) => throw new Exception(t)
         case Success(_) =>
       }
 
@@ -346,32 +344,30 @@ object Runner {
 
   private def runTests(
     classPath: Seq[Path],
-    framework: Framework,
+    frameworks: Seq[Framework],
     requireTests: Boolean,
     args: Seq[String],
     parentInspector: AsmTestRunner.ParentInspector
-  ): Either[NoTestsRun, Boolean] = {
-
-    val taskDefs =
-      AsmTestRunner.taskDefs(
-        classPath,
-        keepJars = false,
-        framework.fingerprints().toIndexedSeq,
-        parentInspector
-      ).toArray
-
-    val runner       = framework.runner(args.toArray, Array(), null)
-    val initialTasks = runner.tasks(taskDefs)
-    val events       = TestRunner.runTasks(initialTasks.toIndexedSeq, System.out)
-
-    val doneMsg = runner.done()
-    if (doneMsg.nonEmpty)
-      System.out.println(doneMsg)
-
-    if (requireTests && events.isEmpty)
-      Left(new NoTestsRun)
-    else
-      Right {
+  ): Either[NoTestsRun, Boolean] = frameworks
+    .flatMap { framework =>
+      val taskDefs =
+        AsmTestRunner.taskDefs(
+          classPath,
+          keepJars = false,
+          framework.fingerprints().toIndexedSeq,
+          parentInspector
+        ).toArray
+
+      val runner       = framework.runner(args.toArray, Array(), null)
+      val initialTasks = runner.tasks(taskDefs)
+      val events       = TestRunner.runTasks(initialTasks.toIndexedSeq, System.out)
+
+      val doneMsg = runner.done()
+      if doneMsg.nonEmpty then System.out.println(doneMsg)
+      events
+    } match {
+    case events if requireTests && events.isEmpty => Left(new NoTestsRun)
+    case events => Right {
         !events.exists { ev =>
           ev.status == Status.Error ||
           ev.status == Status.Failure ||
@@ -380,22 +376,30 @@ object Runner {
       }
   }
 
-  def frameworkName(
+  def frameworkNames(
     classPath: Seq[Path],
-    parentInspector: AsmTestRunner.ParentInspector
-  ): Either[NoTestFrameworkFoundError, String] = {
-    val fwOpt = AsmTestRunner.findFrameworkService(classPath)
-      .orElse {
-        AsmTestRunner.findFramework(
-          classPath,
-          TestRunner.commonTestFrameworks,
-          parentInspector
-        )
-      }
-    fwOpt match {
-      case Some(fw) => Right(fw.replace('/', '.').replace('\\', '.'))
-      case None     => Left(new NoTestFrameworkFoundError)
-    }
+    parentInspector: AsmTestRunner.ParentInspector,
+    logger: Logger
+  ): Either[NoTestFrameworkFoundError, Seq[String]] = {
+    logger.debug("Looking for test framework services on the classpath...")
+    val foundFrameworkServices =
+      AsmTestRunner.findFrameworkServices(classPath)
+        .map(_.replace('/', '.').replace('\\', '.'))
+    logger.debug(s"Found ${foundFrameworkServices.length} test framework services.")
+    if foundFrameworkServices.nonEmpty then
+      logger.debug(s"  - ${foundFrameworkServices.mkString("\n  - ")}")
+    logger.debug("Looking for more test frameworks on the classpath...")
+    val foundFrameworks =
+      AsmTestRunner.findFrameworks(classPath, TestRunner.commonTestFrameworks, parentInspector)
+        .map(_.replace('/', '.').replace('\\', '.'))
+    logger.debug(s"Found ${foundFrameworks.length} additional test frameworks")
+    if foundFrameworks.nonEmpty then
+      logger.debug(s"  - ${foundFrameworks.mkString("\n  - ")}")
+    val frameworks: Seq[String] = foundFrameworkServices ++ foundFrameworks
+    logger.log(s"Found ${frameworks.length} test frameworks in total")
+    if frameworks.nonEmpty then
+      logger.debug(s"  - ${frameworks.mkString("\n  - ")}")
+    if frameworks.nonEmpty then Right(frameworks) else Left(new NoTestFrameworkFoundError)
   }
 
   def testJs(
@@ -410,57 +414,72 @@ object Runner {
   ): Either[TestError, Int] = either {
     import org.scalajs.jsenv.Input
     import org.scalajs.jsenv.nodejs.NodeJSEnv
-    import org.scalajs.testing.adapter.TestAdapter
+    logger.debug("Preparing to run tests with Scala.js...")
+    logger.debug(s"Scala.js tests class path: $classPath")
     val nodePath = findInPath("node").fold("node")(_.toString)
-    val jsEnv =
-      if (jsDom)
+    logger.debug(s"Node found at $nodePath")
+    val jsEnv: JSEnv =
+      if jsDom then {
+        logger.log("Loading JS environment with JS DOM...")
         new JSDOMNodeJSEnv(
           JSDOMNodeJSEnv.Config()
             .withExecutable(nodePath)
             .withArgs(Nil)
             .withEnv(Map.empty)
         )
-      else new NodeJSEnv(
-        NodeJSEnv.Config()
-          .withExecutable(nodePath)
-          .withArgs(Nil)
-          .withEnv(Map.empty)
-          .withSourceMap(NodeJSEnv.SourceMap.Disable)
-      )
-    val adapterConfig = TestAdapter.Config().withLogger(logger.scalaJsLogger)
-    val inputs = Seq(
-      if (esModule) Input.ESModule(entrypoint.toPath)
-      else Input.Script(entrypoint.toPath)
-    )
-    var adapter: TestAdapter = null
+      }
+      else {
+        logger.log("Loading JS environment with Node...")
+        new NodeJSEnv(
+          NodeJSEnv.Config()
+            .withExecutable(nodePath)
+            .withArgs(Nil)
+            .withEnv(Map.empty)
+            .withSourceMap(NodeJSEnv.SourceMap.Disable)
+        )
+      }
+    val adapterConfig = ScalaJsTestAdapter.Config().withLogger(logger.scalaJsLogger)
+    val inputs =
+      Seq(if esModule then Input.ESModule(entrypoint.toPath) else Input.Script(entrypoint.toPath))
+    var adapter: ScalaJsTestAdapter = null
 
     logger.debug(s"JS tests class path: $classPath")
 
     val parentInspector = new AsmTestRunner.ParentInspector(classPath)
-    val frameworkName0 = testFrameworkOpt match {
-      case Some(fw) => fw
-      case None     => value(frameworkName(classPath, parentInspector))
+    val foundFrameworkNames: List[String] = testFrameworkOpt match {
+      case some @ Some(_) => some.toList
+      case None           => value(frameworkNames(classPath, parentInspector, logger)).toList
     }
 
     val res =
       try {
-        adapter = new TestAdapter(jsEnv, inputs, adapterConfig)
-
-        val frameworks = adapter.loadFrameworks(List(List(frameworkName0))).flatten
+        adapter = new ScalaJsTestAdapter(jsEnv, inputs, adapterConfig)
+
+        val loadedFrameworks =
+          adapter
+            .loadFrameworks(foundFrameworkNames.map(List(_)))
+            .flatten
+            .distinctBy(_.name())
+
+        val finalTestFrameworks =
+          loadedFrameworks
+            .filter(
+              !_.name().toLowerCase.contains("junit") ||
+              !loadedFrameworks.exists(_.name().toLowerCase.contains("munit"))
+            )
+        if finalTestFrameworks.nonEmpty then
+          logger.log(
+            s"""Final list of test frameworks found:
+               |  - ${finalTestFrameworks.map(_.description).mkString("\n  - ")}
+               |""".stripMargin
+          )
 
-        if (frameworks.isEmpty)
-          Left(new NoFrameworkFoundByBridgeError)
-        else if (frameworks.length > 1)
-          Left(new TooManyFrameworksFoundByBridgeError)
-        else {
-          val framework = frameworks.head
-          runTests(classPath, framework, requireTests, args, parentInspector)
-        }
+        if finalTestFrameworks.isEmpty then Left(new NoFrameworkFoundByBridgeError)
+        else runTests(classPath, finalTestFrameworks, requireTests, args, parentInspector)
       }
-      finally if (adapter != null) adapter.close()
+      finally if adapter != null then adapter.close()
 
-    if (value(res)) 0
-    else 1
+    if value(res) then 0 else 1
   }
 
   def testNative(
@@ -471,42 +490,61 @@ object Runner {
     args: Seq[String],
     logger: Logger
   ): Either[TestError, Int] = either {
-
-    import scala.scalanative.testinterface.adapter.TestAdapter
-
+    logger.debug("Preparing to run tests with Scala Native...")
     logger.debug(s"Native tests class path: $classPath")
 
     val parentInspector = new AsmTestRunner.ParentInspector(classPath)
-    val frameworkName0 = frameworkNameOpt match {
-      case Some(fw) => fw
-      case None     => value(frameworkName(classPath, parentInspector))
+    val foundFrameworkNames: List[String] = frameworkNameOpt match {
+      case Some(fw) => List(fw)
+      case None     => value(frameworkNames(classPath, parentInspector, logger)).toList
     }
 
-    val config = TestAdapter.Config()
+    val config = ScalaNativeTestAdapter.Config()
       .withBinaryFile(launcher)
-      .withEnvVars(sys.env.toMap)
+      .withEnvVars(sys.env)
       .withLogger(logger.scalaNativeTestLogger)
 
-    var adapter: TestAdapter = null
+    var adapter: ScalaNativeTestAdapter = null
 
     val res =
       try {
-        adapter = new TestAdapter(config)
+        adapter = new ScalaNativeTestAdapter(config)
+
+        val loadedFrameworks =
+          adapter
+            .loadFrameworks(foundFrameworkNames.map(List(_)))
+            .flatten
+            .distinctBy(_.name())
+
+        val finalTestFrameworks =
+          loadedFrameworks
+            // .filter(
+            //  _.name() != "Scala Native JUnit test framework" ||
+            //    !loadedFrameworks.exists(_.name() == "munit")
+            // )
+            // TODO: add support for JUnit and then only hardcode filtering it out when passed with munit
+            // https://github.com/VirtusLab/scala-cli/issues/3627
+            .filter(_.name() != "Scala Native JUnit test framework")
+        if finalTestFrameworks.nonEmpty then
+          logger.log(
+            s"""Final list of test frameworks found:
+               |  - ${finalTestFrameworks.map(_.description).mkString("\n  - ")}
+               |""".stripMargin
+          )
 
-        val frameworks = adapter.loadFrameworks(List(List(frameworkName0))).flatten
+        val skippedFrameworks = loadedFrameworks.diff(finalTestFrameworks)
+        if skippedFrameworks.nonEmpty then
+          logger.log(
+            s"""The following test frameworks have been filtered out:
+               |  - ${skippedFrameworks.map(_.description).mkString("\n  - ")}
+               |""".stripMargin
+          )
 
-        if (frameworks.isEmpty)
-          Left(new NoFrameworkFoundByBridgeError)
-        else if (frameworks.length > 1)
-          Left(new TooManyFrameworksFoundByBridgeError)
-        else {
-          val framework = frameworks.head
-          runTests(classPath, framework, requireTests, args, parentInspector)
-        }
+        if finalTestFrameworks.isEmpty then Left(new NoFrameworkFoundByBridgeError)
+        else runTests(classPath, finalTestFrameworks, requireTests, args, parentInspector)
       }
-      finally if (adapter != null) adapter.close()
+      finally if adapter != null then adapter.close()
 
-    if (value(res)) 0
-    else 1
+    if value(res) then 0 else 1
   }
 }