mongodb · rozza · Oct 1, 2025
@@ -262,7 +262,7 @@ functions:
         working_dir: "src"
         script: |
           ${PREPARE_SHELL}
-          SCALA_VERSION="${SCALA}" SPARK_VERSION="${SPARK}" .evergreen/run-static-checks.sh
+          .evergreen/run-static-checks.sh
 
   "run unit tests":
     - command: shell.exec
@@ -326,15 +326,6 @@ post:
 variables:
   - &run-on
     run_on: "ubuntu2004-small"
-  - &exclude-spec
-    # Scala 2.12 only works with 3.1.3 spark
-    # Scala 2.13 doesn't work on 3.1.3 spark
-    exclude_spec:
-      - { scala: "2.12", spark: "3.2.4" }
-      - { scala: "2.12", spark: "3.3.4" }
-      - { scala: "2.12", spark: "3.4.2" }
-      - { scala: "2.12", spark: "3.5.1" }
-      - { scala: "2.13", spark: "3.1.3" }
 
 tasks:
   - name: "static-checks-task"
@@ -412,76 +403,29 @@ axes:
         variables:
           TOPOLOGY: "sharded_cluster"
 
-  - id: "scala"
-    display_name: "Scala"
-    values:
-      - id: "2.12"
-        display_name: "Scala 2.12"
-        variables:
-          SCALA: "2.12"
-      - id: "2.13"
-        display_name: "Scala 2.13"
-        variables:
-          SCALA: "2.13"
-
-  - id: "spark"
-    display_name: "Spark"
-    values:
-      - id: "3.1.3"
-        display_name: "Spark 3.1.3"
-        variables:
-          SPARK: "3.1.3"
-      - id: "3.2.4"
-        display_name: "Spark 3.2.4"
-        variables:
-          SPARK: "3.2.4"
-      - id: "3.3.4"
-        display_name: "Spark 3.3.4"
-        variables:
-          SPARK: "3.3.4"
-      - id: "3.4.2"
-        display_name: "Spark 3.4.2"
-        variables:
-          SPARK: "3.4.2"
-      - id: "3.5.1"
-        display_name: "Spark 3.5.1"
-        variables:
-          SPARK: "3.5.1"
-
 buildvariants:
 
-  - matrix_name: "static-checks"
-    matrix_spec: { scala: "*", spark: "*" }
-    display_name: "Static checks: ${scala} ${spark}"
-    <<: *exclude-spec
+  - name: "static-checks"
+    display_name: "Static checks"
     tags: ["static-check"]
     tasks:
       - name: "static-checks-task"
 
-  - matrix_name: "unit-tests"
-    matrix_spec: { scala: "*", spark: "*" }
-    <<: *exclude-spec
-    display_name: "Units tests: ${scala} ${spark}"
+  - name: "unit-tests"
+    display_name: "Units tests"
     tags: ["unit-test"]
     tasks:
       - name: "unit-test-task"
 
-  - matrix_name: "integration-tests-2-12"
-    matrix_spec: { scala: "2.12", spark: "3.1.3", version: ["7.0"], topology: "replicaset"}
-    display_name: "Integration tests: ${scala} ${spark} ${version} ${topology}"
-    tags: ["integration-test"]
-    tasks:
-      - name: "integration-test-task"
-
   - matrix_name: "integration-tests-2-13"
-    matrix_spec: { scala: "2.13", spark: ["3.2.4", "3.5.1"], version: ["4.4", "7.0", "latest"], topology: "replicaset" }
-    display_name: "Integration tests: ${scala} ${spark} ${version} ${topology}"
+    matrix_spec: { version: ["4.4", "7.0", "latest"], topology: "replicaset" }
+    display_name: "Integration tests: ${version} ${topology}"
     tags: ["integration-test"]
     tasks:
       - name: "integration-test-task"
 
   - matrix_name: "integration-tests-2-13-sharded"
-    matrix_spec: { scala: "2.13", spark: [ "3.2.4", "3.5.1" ], version: ["4.4", "7.0", "latest" ], topology: "sharded" }
+    matrix_spec: { version: ["4.4", "7.0", "latest" ], topology: "sharded" }
     display_name: "Integration tests: ${scala} ${spark} ${version} ${topology}"
     tags: [ "sharded-integration-test" ]
     tasks:

@@ -10,8 +10,8 @@ set -o errexit  # Exit the script with error if any of the commands fail
 
 echo "Publishing"
 
-export JDK11="/opt/java/jdk11"
-export JAVA_HOME=$JDK11
+export JDK17="/opt/java/jdk17"
+export JAVA_HOME=$JDK17
 
 RELEASE=${RELEASE:false}
 
@@ -29,5 +29,4 @@ fi
 SYSTEM_PROPERTIES="-Dorg.gradle.internal.publish.checksums.insecure=true"
 
 ./gradlew -version
-./gradlew ${SYSTEM_PROPERTIES} --stacktrace --info ${TASK} -DscalaVersion=2.12 -DsparkVersion=3.1.2
-./gradlew ${SYSTEM_PROPERTIES} --stacktrace --info ${TASK} -DscalaVersion=2.13 -DsparkVersion=3.2.2
+./gradlew ${SYSTEM_PROPERTIES} --stacktrace --info ${TASK}
@@ -5,13 +5,9 @@ set -o errexit  # Exit the script with error if any of the commands fail
 
 # Supported/used environment variables:
 # MONGODB_URI     Set the suggested connection MONGODB_URI (including credentials and topology info)
-# SCALA_VERSION   The Scala version to compile with
-# SPARK_VERSION   The spark version to test against
 MONGODB_URI=${MONGODB_URI:-}
-SCALA_VERSION=${SCALA_VERSION:-2.12}
-SPARK_VERSION=${SPARK_VERSION:-3.1.2}
 
-export JAVA_HOME="/opt/java/jdk11"
+export JAVA_HOME="/opt/java/jdk17"
 
 ############################################
 #            Main Program                  #
@@ -21,4 +17,4 @@ export JAVA_HOME="/opt/java/jdk11"
 echo "Running tests connecting to $MONGODB_URI on JDK${JAVA_VERSION}"
 
 ./gradlew -version
-./gradlew -Dorg.mongodb.test.uri=${MONGODB_URI} --stacktrace --info integrationTest -DscalaVersion=$SCALA_VERSION -DsparkVersion=$SPARK_VERSION
+./gradlew -Dorg.mongodb.test.uri=${MONGODB_URI} --stacktrace --info integrationTest
@@ -5,13 +5,9 @@ set -o errexit  # Exit the script with error if any of the commands fail
 
 # Supported/used environment variables:
 # MONGODB_URI     Set the suggested connection MONGODB_URI (including credentials and topology info)
-# SCALA_VERSION   The Scala version to compile with
-# SPARK_VERSION   The spark version to test against
 MONGODB_URI=${MONGODB_URI:-}
-SCALA_VERSION=${SCALA_VERSION:-2.12}
-SPARK_VERSION=${SPARK_VERSION:-3.1.2}
 
-export JAVA_HOME="/opt/java/jdk11"
+export JAVA_HOME="/opt/java/jdk17"
 
 ############################################
 #            Main Program                  #
@@ -21,4 +17,4 @@ export JAVA_HOME="/opt/java/jdk11"
 echo "Running tests connecting to $MONGODB_URI on JDK${JAVA_VERSION}"
 
 ./gradlew -version
-./gradlew -Dorg.mongodb.test.uri=${MONGODB_URI} --stacktrace --info integrationTest --tests "com.mongodb.spark.sql.connector.read.partitioner.ShardedPartitionerTest" -DscalaVersion=$SCALA_VERSION -DsparkVersion=$SPARK_VERSION
+./gradlew -Dorg.mongodb.test.uri=${MONGODB_URI} --stacktrace --info integrationTest --tests "com.mongodb.spark.sql.connector.read.partitioner.ShardedPartitionerTest"
@@ -4,12 +4,7 @@ set -o xtrace   # Write all commands first to stderr
 set -o errexit  # Exit the script with error if any of the commands fail
 
 # Supported/used environment variables:
-# SCALA_VERSION   The Scala version to compile with
-# SPARK_VERSION   The spark version to test against
-SCALA_VERSION=${SCALA_VERSION:-2.12}
-SPARK_VERSION=${SPARK_VERSION:-3.1.2}
-
-export JAVA_HOME="/opt/java/jdk11"
+export JAVA_HOME="/opt/java/jdk17"
 
 ############################################
 #            Main Program                  #
@@ -19,4 +14,4 @@ echo "Compiling and running checks"
 
 # We always compile with the latest version of java
 ./gradlew -version
-./gradlew -PxmlReports.enabled=true --info -x test -x integrationTest clean check jar testClasses javadoc -DscalaVersion=$SCALA_VERSION -DsparkVersion=$SPARK_VERSION
+./gradlew -PxmlReports.enabled=true --info -x test -x integrationTest clean check jar testClasses javadoc
@@ -4,12 +4,8 @@ set -o xtrace   # Write all commands first to stderr
 set -o errexit  # Exit the script with error if any of the commands fail
 
 # Supported/used environment variables:
-# SCALA_VERSION   The Scala version to compile with
-# SPARK_VERSION   The spark version to test against
-SCALA_VERSION=${SCALA_VERSION:-2.12}
-SPARK_VERSION=${SPARK_VERSION:-3.1.2}
 
-export JAVA_HOME="/opt/java/jdk11"
+export JAVA_HOME="/opt/java/jdk17"
 
 ############################################
 #            Main Program                  #
@@ -18,4 +14,4 @@ export JAVA_HOME="/opt/java/jdk11"
 echo "Running unit tests on JDK${JAVA_VERSION}"
 
 ./gradlew -version
-./gradlew --stacktrace --info test -DscalaVersion=$SCALA_VERSION -DsparkVersion=$SPARK_VERSION
+./gradlew --stacktrace --info test
@@ -14,7 +14,9 @@
  * limitations under the License.
  */
 
-import java.io.ByteArrayOutputStream
+import com.github.spotbugs.snom.Confidence
+import com.github.spotbugs.snom.Effort
+import com.github.spotbugs.snom.SpotBugsTask
 import java.time.Duration
 
 buildscript {
@@ -25,34 +27,28 @@ buildscript {
 
 plugins {
     idea
-    `java-library`
+    java
     `maven-publish`
     signing
     checkstyle
     id("com.github.gmazzo.buildconfig") version "3.0.2"
-    id("com.github.spotbugs") version "4.7.9"
+    id("com.github.spotbugs") version "6.4.2"
     id("com.diffplug.spotless") version "6.19.0"
-    id("com.github.johnrengelman.shadow") version "7.0.0"
+    id("com.github.johnrengelman.shadow") version "8.1.1"
     id("io.github.gradle-nexus.publish-plugin") version "2.0.0"
 }
 
-version = "10.5.1-SNAPSHOT"
+version = "11.0.0-SNAPSHOT"
 group = "org.mongodb.spark"
 
 description = "The official MongoDB Apache Spark Connect Connector."
 
-java {
-    sourceCompatibility = JavaVersion.VERSION_1_8
-    targetCompatibility = JavaVersion.VERSION_1_8
-}
-
 repositories {
     mavenCentral()
 }
 
-// Usage: ./gradlew -DscalaVersion=2.12 -DsparkVersion=3.1.4
-val scalaVersion = System.getProperty("scalaVersion", "2.13")
-val sparkVersion = System.getProperty("sparkVersion", "3.5.1")
+val scalaVersion = "2.13"
+val sparkVersion = "4.0.1"
 
 extra.apply {
     set("annotationsVersion", "22.0.0")
@@ -66,14 +62,13 @@ extra.apply {
     set("mockitoVersion", "3.12.4")
 
     // Integration test dependencies
-    set("commons-lang3", "3.12.0")
+    set("commons-lang3", "3.18.0")
 }
 
 sourceSets {
     main {
         java {
-            val scalaInteropSrcDir = if (scalaVersion == "2.12") "java_scala_212" else "java_scala_213"
-            srcDirs("src/main/java", "src/main/$scalaInteropSrcDir")
+            srcDirs("src/main/java", "src/main/java_scala_213")
         }
     }
 }
@@ -97,22 +92,17 @@ dependencies {
     testImplementation("org.apache.spark:spark-streaming_$scalaVersion:$sparkVersion")
 
     // Unit Tests
-    testImplementation(platform("org.junit:junit-bom:5.8.1"))
+    testImplementation(platform("org.junit:junit-bom:5.13.4"))
     testImplementation("org.junit.jupiter:junit-jupiter")
     testImplementation("org.mockito:mockito-junit-jupiter:${project.extra["mockitoVersion"]}")
     testImplementation("org.apiguardian:apiguardian-api:1.1.2") // https://github.com/gradle/gradle/issues/18627
+    testRuntimeOnly("org.junit.platform:junit-platform-launcher")
 
     // Integration Tests
     testImplementation("org.apache.commons:commons-lang3:${project.extra["commons-lang3"]}")
     testImplementation("org.jetbrains:annotations:${project.extra["annotationsVersion"]}")
 }
 
-val defaultJdkVersion: Int = 11
-
-java {
-    toolchain.languageVersion.set(JavaLanguageVersion.of(defaultJdkVersion))
-}
-
 tasks.withType<JavaCompile> {
     options.encoding = "UTF-8"
     options.release.set(8)
@@ -121,22 +111,29 @@ tasks.withType<JavaCompile> {
 // ===========================
 //     Build Config
 // ===========================
+// Gets the git version
 val gitVersion: String by lazy {
-    val describeStdOut = ByteArrayOutputStream()
-    exec {
-        commandLine = listOf("git", "describe", "--tags", "--always", "--dirty")
-        standardOutput = describeStdOut
-    }
-    describeStdOut.toString().substring(1).trim()
+    providers
+        .exec {
+            isIgnoreExitValue = true
+            commandLine("git", "describe", "--tags", "--always", "--dirty")
+        }
+        .standardOutput
+        .asText
+        .map { it.trim().removePrefix("r") }
+        .getOrElse("UNKNOWN")
 }
 
 val gitDiffNameOnly: String by lazy {
-    val describeStdOut = ByteArrayOutputStream()
-    exec {
-        commandLine = listOf("git", "diff", "--name-only")
-        standardOutput = describeStdOut
-    }
-    describeStdOut.toString().replaceIndent(" - ")
+    providers
+        .exec {
+            isIgnoreExitValue = true
+            commandLine("git", "diff", "--name-only")
+        }
+        .standardOutput
+        .asText
+        .map { it.trim().replaceIndent("-") }
+        .getOrElse(" ")
 }
 
 buildConfig {
@@ -156,7 +153,7 @@ sourceSets.create("integrationTest") {
     runtimeClasspath += output + compileClasspath + sourceSets["test"].runtimeClasspath
 }
 
-tasks.create("integrationTest", Test::class.java) {
+tasks.register<Test>("integrationTest") {
     description = "Runs the integration tests"
     group = "verification"
     testClassesDirs = sourceSets["integrationTest"].output.classesDirs
@@ -173,13 +170,7 @@ tasks.withType<Test> {
         events("passed", "skipped", "failed")
     }
 
-    val javaVersion: Int = (project.findProperty("javaVersion") as String? ?: defaultJdkVersion.toString()).toInt()
     logger.info("Running tests using JDK$javaVersion")
-    javaLauncher.set(
-        javaToolchains.launcherFor {
-            languageVersion.set(JavaLanguageVersion.of(javaVersion))
-        },
-    )
 
     systemProperties(mapOf("org.mongodb.test.uri" to System.getProperty("org.mongodb.test.uri", "")))
 
@@ -190,6 +181,7 @@ tasks.withType<Test> {
             executable = javaExecutablesPath.absolutePath
         }
     }
+    jvmArgs("--add-opens=java.base/sun.util.calendar=ALL-UNNAMED")
 
     addTestListener(object : TestListener {
         override fun beforeTest(testDescriptor: TestDescriptor?) {}
@@ -226,14 +218,14 @@ checkstyle {
 spotbugs {
     excludeFilter.set(project.file("config/spotbugs/exclude.xml"))
     showProgress.set(true)
-    setReportLevel("high")
-    setEffort("max")
+    reportLevel.set(Confidence.HIGH)
+    effort.set(Effort.MAX)
 }
 
-tasks.withType<com.github.spotbugs.snom.SpotBugsTask> {
-    enabled = baseName.equals("main")
-    reports.maybeCreate("html").isEnabled = !project.hasProperty("xmlReports.enabled")
-    reports.maybeCreate("xml").isEnabled = project.hasProperty("xmlReports.enabled")
+tasks.withType<SpotBugsTask> {
+    enabled = getBaseName().equals("main", ignoreCase = true)
+    reports.maybeCreate("html").getRequired().set(!project.hasProperty("xmlReports.enabled"))
+    reports.maybeCreate("xml").getRequired().set(project.hasProperty("xmlReports.enabled"))
 }
 
 // Spotless is used to lint and reformat source files.