Retry some more flaky tests on the CI (#3382)

Author: Gedochao

modules/integration/src/test/scala/scala/cli/integration/GitHubTests.scala

@@ -51,20 +51,24 @@ class GitHubTests extends ScalaCliSuite {
     }
   }
 
-  override def munitFlakyOK: Boolean = TestUtil.isCI && Properties.isMac
+  override def munitFlakyOK: Boolean = TestUtil.isCI
+
+  def createSecret(): Unit = {
+    try
+      createSecretTest()
+    catch {
+      case e: UnsatisfiedLinkError if e.getMessage.contains("libsodium") =>
+        fail("libsodium, couldn't be loaded")
+    }
+
+  }
 
   // currently having issues loading libsodium from the static launcher
   // that launcher is mainly meant to be used on CIs or from docker, missing
   // that feature shouldn't be a big deal there
   if (TestUtil.cliKind != "native-static")
-    test("create secret".flaky) {
-      try
-        createSecretTest()
-      catch {
-        case e: UnsatisfiedLinkError if e.getMessage.contains("libsodium") =>
-          fail("libsodium, couldn't be loaded")
-      }
-    }
+    if (Properties.isMac) test("create secret".flaky)(createSecret())
+    else test("create secret")(TestUtil.retryOnCi()(createSecret()))
 
 }
 

modules/integration/src/test/scala/scala/cli/integration/RunGistTestDefinitions.scala

@@ -46,14 +46,16 @@ trait RunGistTestDefinitions { _: RunTestDefinitions =>
   }
 
   test("Github Gists Script URL") {
-    val url =
-      "https://gist.github.com/alexarchambault/7b4ec20c4033690dd750ffd601e540ec"
-    val message = "Hello"
-    emptyInputs.fromRoot { root =>
-      val output = os.proc(TestUtil.cli, extraOptions, escapedUrls(url))
-        .call(cwd = root)
-        .out.trim()
-      expect(output == message)
+    TestUtil.retryOnCi() {
+      val url =
+        "https://gist.github.com/alexarchambault/7b4ec20c4033690dd750ffd601e540ec"
+      val message = "Hello"
+      emptyInputs.fromRoot { root =>
+        val output = os.proc(TestUtil.cli, extraOptions, escapedUrls(url))
+          .call(cwd = root)
+          .out.trim()
+        expect(output == message)
+      }
     }
   }
 }

modules/integration/src/test/scala/scala/cli/integration/RunTestDefinitions.scala

@@ -1905,7 +1905,7 @@ abstract class RunTestDefinitions
     test(
       s"offline mode should fail on missing artifacts (with Scala $actualAnnouncedScalaVersion)"
     ) {
-      TestUtil.retryOnCi() {
+      TestUtil.retryOnCi(maxAttempts = 5) {
         // Kill bloop deamon to test scalac fallback
         os.proc(TestUtil.cli, "--power", "bloop", "exit")
           .call(cwd = os.pwd)

modules/integration/src/test/scala/scala/cli/integration/TestTestDefinitions.scala

@@ -556,70 +556,77 @@ abstract class TestTestDefinitions extends ScalaCliSuite with TestScalaVersionAr
 
   for ((platformName, platformArgs) <- platforms)
     test(s"custom test framework $platformName") {
-      val inputs = TestInputs(
-        os.rel / "MyTests.test.scala" ->
-          s"""//> using dep com.lihaoyi::utest::$utestVersion
-             |
-             |package mytests
-             |import utest._
-             |
-             |object MyTests extends TestSuite {
-             |  val tests = Tests {
-             |    test("foo") {
-             |      assert(2 + 2 == 4)
-             |      println("Hello from " + "tests")
-             |    }
-             |  }
-             |}
-             |""".stripMargin,
-        os.rel / "CustomFramework.test.scala" ->
-          """package custom
-            |
-            |class CustomFramework extends utest.runner.Framework {
-            |  override def setup(): Unit =
-            |    println("Hello from CustomFramework")
-            |}
-            |""".stripMargin
-      )
-      inputs.fromRoot { root =>
-        val baseRes = os.proc(TestUtil.cli, "test", extraOptions, platformArgs, ".")
-          .call(cwd = root)
-        val baseOutput = baseRes.out.text()
-        expect(baseOutput.contains("Hello from tests"))
-        expect(!baseOutput.contains("Hello from CustomFramework"))
-
-        val cmd = Seq[os.Shellable](
-          TestUtil.cli,
-          "test",
-          extraOptions,
-          platformArgs,
-          ".",
-          "--test-framework",
-          "custom.CustomFramework"
+      TestUtil.retryOnCi(maxAttempts = if (platformName.contains("native")) 3 else 1) {
+        val inputs = TestInputs(
+          os.rel / "MyTests.test.scala" ->
+            s"""//> using dep com.lihaoyi::utest::$utestVersion
+               |
+               |package mytests
+               |import utest._
+               |
+               |object MyTests extends TestSuite {
+               |  val tests = Tests {
+               |    test("foo") {
+               |      assert(2 + 2 == 4)
+               |      println("Hello from " + "tests")
+               |    }
+               |  }
+               |}
+               |""".stripMargin,
+          os.rel / "CustomFramework.test.scala" ->
+            """package custom
+              |
+              |class CustomFramework extends utest.runner.Framework {
+              |  override def setup(): Unit =
+              |    println("Hello from CustomFramework")
+              |}
+              |""".stripMargin
         )
-        val res    = os.proc(cmd).call(cwd = root)
-        val output = res.out.text()
-        expect(output.contains("Hello from tests"))
-        expect(output.contains("Hello from CustomFramework"))
+        inputs.fromRoot { root =>
+          val baseRes = os.proc(TestUtil.cli, "test", extraOptions, platformArgs, ".")
+            .call(cwd = root)
+          val baseOutput = baseRes.out.text()
+          expect(baseOutput.contains("Hello from tests"))
+          expect(!baseOutput.contains("Hello from CustomFramework"))
+
+          val cmd = Seq[os.Shellable](
+            TestUtil.cli,
+            "test",
+            extraOptions,
+            platformArgs,
+            ".",
+            "--test-framework",
+            "custom.CustomFramework"
+          )
+          val res    = os.proc(cmd).call(cwd = root)
+          val output = res.out.text()
+          expect(output.contains("Hello from tests"))
+          expect(output.contains("Hello from CustomFramework"))
+        }
       }
     }
 
   for ((platformName, platformArgs) <- platforms)
     test(s"Fail if no tests were run $platformName") {
-      val inputs = TestInputs(
-        os.rel / "MyTests.test.scala" ->
-          s"""//> using dep org.scalameta::munit::$munitVersion
-             |
-             |object MyTests
-             |""".stripMargin
-      )
+      TestUtil.retryOnCi(maxAttempts = if (platformName.contains("native")) 3 else 1) {
+        val inputs = TestInputs(
+          os.rel / "MyTests.test.scala" ->
+            s"""//> using dep org.scalameta::munit::$munitVersion
+               |
+               |object MyTests
+               |""".stripMargin
+        )
 
-      inputs.fromRoot { root =>
-        val res = os.proc(TestUtil.cli, "test", extraOptions, "--require-tests", platformArgs, ".")
-          .call(cwd = root, stderr = os.Pipe, mergeErrIntoOut = true, check = false)
-        expect(res.exitCode != 0)
-        val output = res.out.text()
-        expect(output.contains("Error: no tests were run") || output.contains("No tests were run"))
+        inputs.fromRoot { root =>
+          val res =
+            os.proc(TestUtil.cli, "test", extraOptions, "--require-tests", platformArgs, ".")
+              .call(cwd = root, stderr = os.Pipe, mergeErrIntoOut = true, check = false)
+          expect(res.exitCode != 0)
+          val output = res.out.text()
+          expect(
+            output.contains("Error: no tests were run") || output.contains("No tests were run")
+          )
+        }
       }
     }
 

modules/integration/src/test/scala/scala/cli/integration/UpdateTests.scala

@@ -108,7 +108,7 @@ class UpdateTests extends ScalaCliSuite {
 
   if (!Properties.isWin && Constants.ghOrg == "VirtusLab" && Constants.ghName == "scala-cli")
     test("updating dummy scala-cli using update command") {
-      runUpdate()
+      TestUtil.retryOnCi()(runUpdate())
     }
 
   test("run update before run/test/compile should not return exit code") {