Retry more occasionally flaky tests on the CI (#3331)

Author: Gedochao

modules/integration/src/test/scala/scala/cli/integration/BspTestDefinitions.scala

@@ -2169,16 +2169,25 @@ abstract class BspTestDefinitions extends ScalaCliSuite with TestScalaVersionArg
 
   for { cliVersion <- Seq("1.5.0", "1.5.0-19-g932866db6-SNAPSHOT", "1.0.0") }
     test(s"setup-ide doesn't pass unrecognised arguments to old --cli-versions: $cliVersion") {
-      val scriptName = "cli-version.sc"
-      val inputs = TestInputs(
-        os.rel / scriptName -> s"""println("Hello from launcher v$cliVersion"""
-      )
-      inputs.fromRoot { root =>
-        val r =
-          os.proc(TestUtil.cli, "--cli-version", cliVersion, "setup-ide", scriptName, extraOptions)
-            .call(cwd = root, stderr = os.Pipe, check = false)
-        expect(!r.err.text().contains("Unrecognized argument"))
-        expect(r.exitCode == 0)
+      TestUtil.retryOnCi() {
+        val scriptName = "cli-version.sc"
+        val inputs = TestInputs(
+          os.rel / scriptName -> s"""println("Hello from launcher v$cliVersion"""
+        )
+        inputs.fromRoot { root =>
+          val r =
+            os.proc(
+              TestUtil.cli,
+              "--cli-version",
+              cliVersion,
+              "setup-ide",
+              scriptName,
+              extraOptions
+            )
+              .call(cwd = root, stderr = os.Pipe, check = false)
+          expect(!r.err.text().contains("Unrecognized argument"))
+          expect(r.exitCode == 0)
+        }
       }
     }
 

modules/integration/src/test/scala/scala/cli/integration/ExportCommonTestDefinitions.scala

@@ -99,23 +99,33 @@ trait ExportCommonTestDefinitions { _: ScalaCliSuite & TestScalaVersionArgs =>
 
   if (runExportTests) {
     test("JVM") {
-      jvmTest(runMainArgs(Some("Main")), runTestsArgs(Some("Main")), mainClassName = "Main")
+      TestUtil.retryOnCi() {
+        jvmTest(runMainArgs(Some("Main")), runTestsArgs(Some("Main")), mainClassName = "Main")
+      }
     }
     test("extra source from a directive introducing a dependency") {
-      extraSourceFromDirectiveWithExtraDependency("Main", "Main.scala")
+      TestUtil.retryOnCi() {
+        extraSourceFromDirectiveWithExtraDependency("Main", "Main.scala")
+      }
     }
     test("extra source passed both via directive and from command line") {
-      extraSourceFromDirectiveWithExtraDependency("Main", ".")
+      TestUtil.retryOnCi() {
+        extraSourceFromDirectiveWithExtraDependency("Main", ".")
+      }
     }
     scalaVersionsInDir.foreach { scalaV =>
       test(s"check export for project with scala version in directive as $scalaV") {
-        scalaVersionTest(scalaV, "Main")
+        TestUtil.retryOnCi() {
+          scalaVersionTest(scalaV, "Main")
+        }
       }
     }
 
     test("just test scope") {
       // Keeping the test name ends with Test to support maven convention
-      justTestScope("MyTest")
+      TestUtil.retryOnCi() {
+        justTestScope("MyTest")
+      }
     }
 
   }

modules/integration/src/test/scala/scala/cli/integration/ExportMillTestDefinitions.scala

@@ -81,28 +81,36 @@ abstract class ExportMillTestDefinitions extends ScalaCliSuite
 
   if (runExportTests) {
     test("JVM custom project name") {
-      val customProjectName = "newproject"
-      jvmTest(
-        mainArgs = Seq(s"$customProjectName.run"),
-        testArgs = Seq(s"$customProjectName.test"),
-        extraExportArgs = Seq("-p", customProjectName),
-        mainClassName = "Hello"
-      )
+      TestUtil.retryOnCi() {
+        val customProjectName = "newproject"
+        jvmTest(
+          mainArgs = Seq(s"$customProjectName.run"),
+          testArgs = Seq(s"$customProjectName.test"),
+          extraExportArgs = Seq("-p", customProjectName),
+          mainClassName = "Hello"
+        )
+      }
     }
     test("JVM scalac options") {
-      jvmTestScalacOptions("Hello")
+      TestUtil.retryOnCi() {
+        jvmTestScalacOptions("Hello")
+      }
     }
   }
   if (runExportTests && !actualScalaVersion.startsWith("3."))
     test("JVM with compiler plugin") {
-      jvmTestCompilerPlugin("Hello")
+      TestUtil.retryOnCi() {
+        jvmTestCompilerPlugin("Hello")
+      }
     }
 
   test("Scala Native") {
     // FIXME this should be adjusted to Scala Native 0.5.x syntax once Mill gets support for it
-    simpleTest(
-      ExportTestProjects.nativeTest(actualScalaVersion, useNative04Syntax = true),
-      mainClass = None
-    )
+    TestUtil.retryOnCi() {
+      simpleTest(
+        ExportTestProjects.nativeTest(actualScalaVersion, useNative04Syntax = true),
+        mainClass = None
+      )
+    }
   }
 }

modules/integration/src/test/scala/scala/cli/integration/ExportSbtTestDefinitions.scala

@@ -23,6 +23,8 @@ abstract class ExportSbtTestDefinitions extends ScalaCliSuite
   override def runTestsArgs(mainClass: Option[String]): Seq[String] = Seq("test")
 
   test("Scala Native") {
-    simpleTest(ExportTestProjects.nativeTest(actualScalaVersion), mainClass = None)
+    TestUtil.retryOnCi() {
+      simpleTest(ExportTestProjects.nativeTest(actualScalaVersion), mainClass = None)
+    }
   }
 }

modules/integration/src/test/scala/scala/cli/integration/ExportScalaOrientedBuildToolsTestDefinitions.scala

@@ -73,16 +73,24 @@ trait ExportScalaOrientedBuildToolsTestDefinitions {
 
   if (runExportTests) {
     test("compile-time only for jsoniter macros") {
-      compileOnlyTest("main")
+      TestUtil.retryOnCi() {
+        compileOnlyTest("main")
+      }
     }
     test("Scala.js") {
-      simpleTest(ExportTestProjects.jsTest(actualScalaVersion), mainClass = None)
+      TestUtil.retryOnCi() {
+        simpleTest(ExportTestProjects.jsTest(actualScalaVersion), mainClass = None)
+      }
     }
     test("zio test") {
-      testZioTest("ZioSpec")
+      TestUtil.retryOnCi() {
+        testZioTest("ZioSpec")
+      }
     }
     test("Ensure test framework NPE is not thrown when depending on logback") {
-      logbackBugCase("main")
+      TestUtil.retryOnCi() {
+        logbackBugCase("main")
+      }
     }
   }
 }

modules/integration/src/test/scala/scala/cli/integration/PackageTestDefinitions.scala

@@ -506,21 +506,29 @@ abstract class PackageTestDefinitions extends ScalaCliSuite with TestScalaVersio
 
   if (!Properties.isWin && actualScalaVersion.startsWith("2.13")) {
     test("simple native") {
-      simpleNativeTest()
+      TestUtil.retryOnCi() {
+        simpleNativeTest()
+      }
     }
     test("dynamic library native") {
-      libraryNativeTest(shared = true)
+      TestUtil.retryOnCi() {
+        libraryNativeTest(shared = true)
+      }
     }
 
     test("dynamic library native override from command line") {
-      libraryNativeTest(shared = false, commandLineShared = Some(true))
+      TestUtil.retryOnCi() {
+        libraryNativeTest(shared = false, commandLineShared = Some(true))
+      }
     }
 
     // To produce a static library, `LLVM_BIN` environment variable needs to be
     // present (for `llvm-ar` utility)
     if (sys.env.contains("LLVM_BIN"))
       test("shared library native") {
-        libraryNativeTest(shared = false)
+        TestUtil.retryOnCi() {
+          libraryNativeTest(shared = false)
+        }
       }
 
   }
@@ -1102,7 +1110,9 @@ abstract class PackageTestDefinitions extends ScalaCliSuite with TestScalaVersio
 
   if (Properties.isLinux)
     test("pass java options to docker") {
-      javaOptionsDockerTest()
+      TestUtil.retryOnCi() {
+        javaOptionsDockerTest()
+      }
     }
 
   test("default values in help") {