Fix Scala 2 nightly test failures by tagging them as flaky or skipping (#3064)

Author: Gedochao

modules/build/src/test/scala/scala/build/tests/BuildOptionsTests.scala

@@ -33,6 +33,7 @@ import scala.build.internal.Regexes.scala3LtsRegex
 import scala.build.errors.ScalaVersionError
 
 class BuildOptionsTests extends TestUtil.ScalaCliBuildSuite {
+  override def munitFlakyOK: Boolean = TestUtil.isCI
 
   val extraRepoTmpDir = os.temp.dir(prefix = "scala-cli-tests-extra-repo-")
   val directories     = Directories.under(extraRepoTmpDir)
@@ -217,7 +218,7 @@ class BuildOptionsTests extends TestUtil.ScalaCliBuildSuite {
     )
   }
 
-  test("-S 2.nightly option works") {
+  test("-S 2.nightly option works".flaky) {
     val options = BuildOptions(
       scalaOptions = ScalaOptions(
         scalaVersion = Some(MaybeScalaVersion("2.nightly"))
@@ -230,7 +231,7 @@ class BuildOptionsTests extends TestUtil.ScalaCliBuildSuite {
     )
   }
 
-  test("-S 2.13.nightly option works") {
+  test("-S 2.13.nightly option works".flaky) {
     val options = BuildOptions(
       scalaOptions = ScalaOptions(
         scalaVersion = Some(MaybeScalaVersion("2.13.nightly"))
@@ -256,7 +257,7 @@ class BuildOptionsTests extends TestUtil.ScalaCliBuildSuite {
     )
   }
 
-  test("-S 2.12.nightly option works") {
+  test("-S 2.12.nightly option works".flaky) {
     val options = BuildOptions(
       scalaOptions = ScalaOptions(
         scalaVersion = Some(MaybeScalaVersion("2.12.nightly"))
@@ -269,7 +270,7 @@ class BuildOptionsTests extends TestUtil.ScalaCliBuildSuite {
     )
   }
 
-  test("-S 2.13.9-bin-4505094 option works without repo specification") {
+  test("-S 2.13.9-bin-4505094 option works without repo specification".flaky) {
     val options = BuildOptions(
       scalaOptions = ScalaOptions(
         scalaVersion = Some(MaybeScalaVersion("2.13.9-bin-4505094"))

modules/integration/src/test/scala/scala/cli/integration/ReplTests213.scala

@@ -5,7 +5,7 @@ import com.eed3si9n.expecty.Expecty.expect
 import scala.util.Properties
 
 class ReplTests213 extends ReplTestDefinitions with Test213 {
-  test("repl custom repositories work") {
+  test("repl custom repositories work".flaky) {
     TestInputs.empty.fromRoot { root =>
       os.proc(
         TestUtil.cli,

modules/integration/src/test/scala/scala/cli/integration/RunScalacCompatTestDefinitions.scala

@@ -377,8 +377,6 @@ trait RunScalacCompatTestDefinitions {
     }
   }
 
-  override def munitFlakyOK: Boolean = TestUtil.isCI && Properties.isMac && TestUtil.isNativeCli
-
   def verifyScala212VersionCompiles(scalaPatchVersion: String): Unit = {
     TestInputs(os.rel / "s.sc" -> "println(util.Properties.versionNumberString)").fromRoot {
       root =>

modules/integration/src/test/scala/scala/cli/integration/RunTests212.scala

@@ -26,7 +26,7 @@ class RunTests212 extends RunTestDefinitions with Test212 {
       )
     }
   }
-  test("2.12.nightly") {
+  test("2.12.nightly".flaky) {
     TestInputs(os.rel / "sample.sc" -> "println(util.Properties.versionNumberString)").fromRoot {
       root =>
         val res =

modules/integration/src/test/scala/scala/cli/integration/RunTests213.scala

@@ -3,7 +3,7 @@ package scala.cli.integration
 import com.eed3si9n.expecty.Expecty.expect
 
 class RunTests213 extends RunTestDefinitions with Test213 {
-  test("ensure typesafe PR validation snapshot Scala versions are supported") {
+  test("ensure typesafe PR validation snapshot Scala versions are supported".flaky) {
     TestInputs(os.rel / "sample.sc" -> "println(util.Properties.versionNumberString)").fromRoot {
       root =>
         val snapshotScalaVersion = "2.13.11-bin-89f3de5-SNAPSHOT"
@@ -22,7 +22,7 @@ class RunTests213 extends RunTestDefinitions with Test213 {
         expect(res.out.trim() == "2.13.11-20221128-143922-89f3de5")
     }
   }
-  test("2.13.nightly") {
+  test("2.13.nightly".flaky) {
     TestInputs(os.rel / "sample.sc" -> "println(util.Properties.versionNumberString)").fromRoot {
       root =>
         val res =

modules/integration/src/test/scala/scala/cli/integration/ScalaCliSuite.scala

@@ -32,6 +32,8 @@ abstract class ScalaCliSuite extends munit.FunSuite {
     Option(System.getenv("SCALA_CLI_IT_GROUP"))
       .flatMap(_.toIntOption)
       .exists(_ != group.idx)
+
+  override def munitFlakyOK: Boolean = TestUtil.isCI
 }
 
 object ScalaCliSuite {

modules/integration/src/test/scala/scala/cli/integration/SipScalaTests.scala

@@ -665,7 +665,7 @@ class SipScalaTests extends ScalaCliSuite with SbtTestHelper with MillTestHelper
     }
 
     test(
-      s"default Scala version ($sv2) coming straight from a predefined local repository $withBloopString"
+      s"default Scala version ($sv2) coming straight from a predefined local repository $withBloopString".flaky
     ) {
       TestInputs(
         os.rel / "simple.sc" -> "println(scala.util.Properties.versionNumberString)"

website/docs/commands/compile.md

@@ -99,7 +99,7 @@ having to pass their addresses as input after the `--repo` flag.
 
 For compiling with the latest Scala 2 nightly build:
 
-```bash
+```bash ignore
 scala-cli Hello.scala -S 2.nightly
 ```
 
@@ -111,21 +111,21 @@ scala-cli Hello.scala -S 3.nightly
 
 For compiling with a specific nightly build you have the full version:
 
-```bash
+```bash ignore
 scala-cli Hello.scala -S 2.13.9-bin-4505094
 ```
 
 For setting this inside scala files, use [`using` directives](../guides/introduction/using-directives.md):
 
-```scala compile
+```scala
 //> using scala 2.nightly
 ```
 
 ```scala compile
 //> using scala 3.nightly
 ```
 
-```scala compile
+```scala
 //> using scala 2.13.9-bin-4505094
 ```