Upgrade to Elasticsearch 9.1.4. As part of this:

- Upgrade Java to JVM 21
- Upgrade Gradle to 8.11
- Fix all deprecations
- Fix integrations tests on Elasticsearch and re-enable them

Author: johnnyshields

.github/workflows/build.yml

@@ -14,6 +14,7 @@ jobs:
     strategy:
       matrix:
         es-version:
+          - 'es:9.1.4'
           - 'es:8.15.2'
           - 'es:8.14.3'
           - 'es:8.13.4'
@@ -46,7 +47,7 @@ jobs:
           - 'os:2.7.0'
           - 'os:2.6.0'
     env:
-      mainJob: ${{ matrix.es-version == 'es:8.15.2' }}
+      mainJob: ${{ matrix.es-version == 'es:9.1.4' }}
       sudachiVersion: 20241021
       sudachiKind: core
     continue-on-error: true

README.md

@@ -18,19 +18,20 @@ Check [changelog](./CHANGELOG.md) for more.
 
 1. Build analysis-sudachi.
 ```
-   $ ./gradlew -PengineVersion=es:8.15.2 build
+   $ ./gradlew -PengineVersion=es:9.1.4 build
 ```
 
 Use `-PengineVersion=os:2.18.0` for OpenSearch.
 
 ## Supported ElasticSearch versions
 
-1. 8.0.* until 8.15.* supported, integration tests in CI
-2. 7.17.* (latest patch version) - supported, integration tests in CI
-3. 7.11.* until 7.16.* - best effort support, not tested in CI
-4. 7.10.* integration tests for the latest patch version
-5. 7.9.* and below - not tested in CI at all, may be broken
-6. 7.3.* and below - broken, not supported
+1. 9.0.* until 9.1.* supported, integration tests in CI
+2. 8.0.* until 8.15.* supported, integration tests in CI
+3. 7.17.* (latest patch version) - supported, integration tests in CI
+4. 7.11.* until 7.16.* - best effort support, not tested in CI
+5. 7.10.* integration tests for the latest patch version
+6. 7.9.* and below - not tested in CI at all, may be broken
+7. 7.3.* and below - broken, not supported
 
 ## Supported OpenSearch versions
 
@@ -43,11 +44,11 @@ Use `-PengineVersion=os:2.18.0` for OpenSearch.
 
    a. Using the release package
    ```
-   $ bin/elasticsearch-plugin install https://github.com/WorksApplications/elasticsearch-sudachi/releases/download/v3.1.1/analysis-sudachi-8.13.4-3.1.1.zip
+   $ bin/elasticsearch-plugin install https://github.com/WorksApplications/elasticsearch-sudachi/releases/download/v3.1.1/analysis-sudachi-9.1.4-3.1.1.zip
    ```
    b. Using self-build package
    ```
-   $ bin/elasticsearch-plugin install file:///path/to/analysis-sudachi-8.13.4-3.1.1.zip
+   $ bin/elasticsearch-plugin install file:///path/to/analysis-sudachi-9.1.4-3.1.1.zip
    ```
    (Specify the absolute path in URI format)
 3. Download sudachi dictionary archive from https://github.com/WorksApplications/SudachiDict

build.gradle

@@ -2,9 +2,9 @@ import org.jetbrains.kotlin.gradle.dsl.JvmTarget
 
 plugins {
     id 'java-library'
-    id 'org.jetbrains.kotlin.jvm' version '1.8.0'
-    id "org.jetbrains.kotlin.plugin.serialization" version "1.8.0"
-    id 'com.diffplug.spotless' version '6.16.0'
+    id 'org.jetbrains.kotlin.jvm' version '2.0.0'
+    id "org.jetbrains.kotlin.plugin.serialization" version '2.0.0'
+    id 'com.diffplug.spotless' version '6.25.0'
     id 'org.sonarqube' version '4.0.0.2929'
     id("org.jetbrains.kotlinx.kover") version "0.7.0"
     id 'com.worksap.nlp.sudachi.esc'
@@ -13,15 +13,20 @@ plugins {
 }
 
 group = 'com.worksap.nlp'
-archivesBaseName = 'analysis-sudachi'
+base.archivesName.set('analysis-sudachi')
 version = properties["pluginVersion"]
 
+java {
+    sourceCompatibility = JavaVersion.VERSION_21
+    targetCompatibility = JavaVersion.VERSION_21
+}
+
 compileKotlin {
-    compilerOptions.jvmTarget.set(JvmTarget.JVM_11)
+    compilerOptions.jvmTarget.set(JvmTarget.JVM_21)
 }
 
 compileTestKotlin {
-    compilerOptions.jvmTarget.set(JvmTarget.JVM_11)
+    compilerOptions.jvmTarget.set(JvmTarget.JVM_21)
 }
 
 configurations {
@@ -84,7 +89,7 @@ def packageSpiJars = tasks.register('packageSpiJars', Copy) {
 def distZip = tasks.register('distZip', Zip) {
     var esKind = sudachiEs.kind.get()
     dependsOn embedVersion, packageJars, packageSpiJars
-    archiveBaseName.set("${esKind.engine.kind}-${esKind.version}-$archivesBaseName")
+    archiveBaseName.set("${esKind.engine.kind}-${esKind.version}-${base.archivesName.get()}")
     from("build/package/${version}/${esKind.engine.kind}-${esKind.version}", 'LICENSE', 'README.md')
 }
 

buildSrc/src/main/groovy/com/worksap/nlp/tools/EsConventions.groovy

@@ -8,7 +8,7 @@ class EsConventions implements Plugin<Project> {
     @Override
     void apply(Project target) {
         target.tasks.withType(JavaCompile).configureEach {
-            options.release.set(11)
+            options.release.set(21)
             options.encoding = 'UTF-8'
         }
 

buildSrc/src/main/groovy/com/worksap/nlp/tools/EsTestEnvPlugin.groovy

@@ -103,6 +103,10 @@ class StringProvider implements Provider<String>, Serializable {
         throw new IllegalStateException("not implemented")
     }
 
+    @Override
+    Provider<String> filter(org.gradle.api.specs.Spec<? super String> spec) {
+        throw new IllegalStateException("not implemented")
+    }
 
     @Override
     String toString() {
@@ -153,9 +157,10 @@ class EsTestEnvPlugin implements Plugin<Project> {
         target.gradle.taskGraph.whenReady {
             boolean shouldRun = false
             if (target.plugins.findPlugin(EsSudachiPlugin.class) != null) {
-                shouldRun = shouldTestsRun(target.extensions.getByType(EsExtension).kind.get())
+                def kind = target.extensions.getByType(EsExtension).kind.get()
+                shouldRun = shouldTestsRun(kind)
             }
-            target.tasks.findAll().forEach { Task task ->
+            target.tasks.withType(Test).forEach { Test task ->
                 task.onlyIf { shouldRun }
             }
         }

buildSrc/src/main/groovy/com/worksap/nlp/tools/engines.groovy

@@ -13,6 +13,7 @@ enum EsSupport implements EngineSupport {
     Es84("es-8.04"),
     Es810("es-8.10"),
     Es812("es-8.12"),
+    Es90("es-9.00"),
 
     String tag
     List<String> keys
@@ -41,6 +42,8 @@ enum EsSupport implements EngineSupport {
             return Es810
         } else if (vers.ge(8, 12) && vers.lt(9, 0)) {
             return Es812
+        } else if (vers.ge(9, 0)) {
+            return Es90
         } else {
             throw new IllegalArgumentException("unsupported ElasticSearch version: " + vers.raw)
         }

gradle.properties

@@ -1,8 +1,8 @@
-# elasticsearch versions: 8.15.2, 8.14.3, 8.13.4, 8.12.2, 8.11.4, 8.10.4, 8.9.2,
+# elasticsearch versions: 9.1.4, 8.15.2, 8.14.3, 8.13.4, 8.12.2, 8.11.4, 8.10.4, 8.9.2,
 #     8.8.1, 8.6.2, 8.5.3, 8.4.3, 8.2.3, 7.17.24, 7.14.2, 7.10.2
-# opensearch version: 2.18.0, 2.17.1, 2.16.0, 2.15.0, 2.14.0, 2.13.0, 2.12.0, 2.11.1, 
+# opensearch version: 2.18.0, 2.17.1, 2.16.0, 2.15.0, 2.14.0, 2.13.0, 2.12.0, 2.11.1,
 #     2.10.0, 2.9.0, 2.8.0, 2.7.0, 2.6.0
-engineVersion=es:8.15.2
+engineVersion=es:9.1.4
 org.gradle.jvmargs=-XX:MaxMetaspaceSize=350m \
   --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
   --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.1.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.11-bin.zip
 networkTimeout=10000
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

integration/build.gradle

@@ -13,22 +13,33 @@ plugins {
 
 version = properties["pluginVersion"]
 
+java {
+    sourceCompatibility = JavaVersion.VERSION_21
+    targetCompatibility = JavaVersion.VERSION_21
+}
+
 compileKotlin {
-    compilerOptions.jvmTarget.set(JvmTarget.JVM_11)
+    compilerOptions.jvmTarget.set(JvmTarget.JVM_21)
 }
 
 compileTestKotlin {
-    compilerOptions.jvmTarget.set(JvmTarget.JVM_11)
+    compilerOptions.jvmTarget.set(JvmTarget.JVM_21)
 }
 
 configurations { buildSudachiDict }
 
 dependencies {
     buildSudachiDict (project(':spi'))
-    compileOnly(project(':'))
-    compileOnly(project(':spi'))
-    testCompileOnly(project(':testlib'))
-    testCompileOnly(project(':subplugin'))
+    testImplementation(project(':'))
+    testImplementation(project(':spi'))
+    testImplementation(project(':testlib'))
+    testImplementation(project(':subplugin'))
+    // Add ICU plugin JARs for ES 9.x dynamic loading
+    testRuntimeOnly files({
+        fileTree(dir: new File(project.buildDir, "cache/icu-plugin-extracted"), include: '**/*.jar')
+    }) {
+        builtBy 'extractIcuPlugin'
+    }
     testImplementation('junit:junit:4.13.1') {
         transitive = false
     }
@@ -65,6 +76,12 @@ def downloadIcuPlugin = tasks.register('downloadIcuPlugin', Download.class) {
     overwrite(false)
 }
 
+def extractIcuPlugin = tasks.register('extractIcuPlugin', Copy) {
+    dependsOn downloadIcuPlugin
+    from zipTree(downloadIcuPlugin.get().dest)
+    into new File(project.buildDir, "cache/icu-plugin-extracted")
+}
+
 esTestEnv {
     def esKind = sudachiEs.kind.get()
     def packageDir = rootDir.toPath().resolve("build/package/${version}/${esKind.engine.kind}-${esKind.version}")
@@ -77,7 +94,6 @@ esTestEnv {
 }
 
 test {
-    onlyIf { ! (sudachiEs.isEs() && sudachiEs.kind.get().parsedVersion().ge(8, 9)) }
     dependsOn(
             ':packageJars',
             ':packageSpiJars',
@@ -86,6 +102,7 @@ test {
             compileSystemDictionary,
             ':testlib:jar',
             downloadIcuPlugin,
+            extractIcuPlugin,
             ':subplugin:distZip'
     )
     systemProperty("tests.security.manager", true)
@@ -94,7 +111,7 @@ test {
 
 def distZip = tasks.register('distZip', Zip) {
     var esKind = sudachiEs.kind.get()
-    archiveBaseName.set("${esKind.engine.kind}-${esKind.version}-$archivesBaseName")
+    archiveBaseName.set("${esKind.engine.kind}-${esKind.version}-${project.name}")
     from(
         project(':subplugin').packageJars.outputs.files,
         project(':subplugin').embedVersion.outputs.files,

integration/src/test/ext/es-9.00-ge/SudachiInSearchEngineEnv.kt

@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2022-2025 Works Applications Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.worksap.nlp.elasticsearch.sudachi
+
+import org.elasticsearch.indices.analysis.AnalysisModule
+import org.elasticsearch.plugins.AnalysisPlugin
+import org.elasticsearch.plugins.scanners.StablePluginsRegistry
+
+// For ES 9.x tests, we manually instantiate plugins since PluginsService API changed
+class TestPluginsService(private val plugins: List<AnalysisPlugin>) {
+  fun <T> filterPlugins(clazz: Class<T>): List<T> = plugins.filterIsInstance(clazz)
+}
+
+typealias PluginsServiceAlias = TestPluginsService
+
+private fun loadTestPlugins(): List<AnalysisPlugin> {
+  return listOfNotNull(
+      tryLoadPlugin("com.worksap.nlp.elasticsearch.sudachi.plugin.AnalysisSudachiPlugin", true),
+      tryLoadPlugin("org.elasticsearch.plugin.analysis.icu.AnalysisICUPlugin", false))
+}
+
+private fun tryLoadPlugin(className: String, requiresSettings: Boolean): AnalysisPlugin? {
+  return try {
+    val clazz = Class.forName(className)
+    if (requiresSettings) {
+      clazz.getConstructor(org.elasticsearch.common.settings.Settings::class.java).newInstance(null)
+          as AnalysisPlugin
+    } else {
+      clazz.getDeclaredConstructor().newInstance() as AnalysisPlugin
+    }
+  } catch (e: Exception) {
+    null // Plugin not available or failed to load
+  }
+}
+
+fun SudachiInSearchEngineEnv.makePluginService() = TestPluginsService(loadTestPlugins())
+
+fun SudachiInSearchEngineEnv.makeAnalysisModule() =
+    AnalysisModule(environment(), loadTestPlugins(), StablePluginsRegistry())