Merge branch 'master' into feat/data-loader/import-replace-storage

Author: inv-jishnu

.github/workflows/ci.yaml

@@ -254,6 +254,136 @@ jobs:
           name: cassandra_3.11_integration_test_reports_${{ matrix.mode.label }}
           path: core/build/reports/tests/integrationTestCassandra
 
+  integration-test-for-cassandra-4-1:
+    name: Cassandra 4.1 integration test (${{ matrix.mode.label }})
+    runs-on: ubuntu-latest
+
+    services:
+      cassandra:
+        image: cassandra:4.1
+        env:
+          MAX_HEAP_SIZE: 2048m
+          HEAP_NEWSIZE: 512m
+        ports:
+          - 9042:9042
+
+    strategy:
+      fail-fast: false
+      matrix:
+        mode:
+          - label: default
+            group_commit_enabled: false
+          - label: with_group_commit
+            group_commit_enabled: true
+
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Set up JDK ${{ env.JAVA_VERSION }} (${{ env.JAVA_VENDOR }})
+        uses: actions/setup-java@v5
+        with:
+          java-version: ${{ env.JAVA_VERSION }}
+          distribution: ${{ env.JAVA_VENDOR }}
+
+      - name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}) to run integration test
+        uses: actions/setup-java@v5
+        if: ${{ env.SET_UP_INT_TEST_RUNTIME_NON_ORACLE_JDK == 'true'}}
+        with:
+          java-version: ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}
+          distribution: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}
+
+      - name: Login to Oracle container registry
+        uses: docker/login-action@v3
+        if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
+        with:
+          registry: container-registry.oracle.com
+          username: ${{ secrets.OCR_USERNAME }}
+          password: ${{ secrets.OCR_TOKEN }}
+
+      - name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (oracle) to run the integration test
+        if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
+        run: |
+          container_id=$(docker create "container-registry.oracle.com/java/jdk:${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}")
+          docker cp -L "$container_id:/usr/java/default" /usr/lib/jvm/oracle-jdk && docker rm "$container_id"
+
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@v5
+
+      - name: Execute Gradle 'integrationTestCassandra' task
+        run: ./gradlew integrationTestCassandra ${{ matrix.mode.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }}
+
+      - name: Upload Gradle test reports
+        if: always()
+        uses: actions/upload-artifact@v5
+        with:
+          name: cassandra_4.1_integration_test_reports_${{ matrix.mode.label }}
+          path: core/build/reports/tests/integrationTestCassandra
+
+  integration-test-for-cassandra-5-0:
+    name: Cassandra 5.0 integration test (${{ matrix.mode.label }})
+    runs-on: ubuntu-latest
+
+    services:
+      cassandra:
+        image: cassandra:5.0
+        env:
+          MAX_HEAP_SIZE: 2048m
+          HEAP_NEWSIZE: 512m
+        ports:
+          - 9042:9042
+
+    strategy:
+      fail-fast: false
+      matrix:
+        mode:
+          - label: default
+            group_commit_enabled: false
+          - label: with_group_commit
+            group_commit_enabled: true
+
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Set up JDK ${{ env.JAVA_VERSION }} (${{ env.JAVA_VENDOR }})
+        uses: actions/setup-java@v5
+        with:
+          java-version: ${{ env.JAVA_VERSION }}
+          distribution: ${{ env.JAVA_VENDOR }}
+
+      - name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}) to run integration test
+        uses: actions/setup-java@v5
+        if: ${{ env.SET_UP_INT_TEST_RUNTIME_NON_ORACLE_JDK == 'true'}}
+        with:
+          java-version: ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}
+          distribution: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}
+
+      - name: Login to Oracle container registry
+        uses: docker/login-action@v3
+        if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
+        with:
+          registry: container-registry.oracle.com
+          username: ${{ secrets.OCR_USERNAME }}
+          password: ${{ secrets.OCR_TOKEN }}
+
+      - name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (oracle) to run the integration test
+        if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
+        run: |
+          container_id=$(docker create "container-registry.oracle.com/java/jdk:${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}")
+          docker cp -L "$container_id:/usr/java/default" /usr/lib/jvm/oracle-jdk && docker rm "$container_id"
+
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@v5
+
+      - name: Execute Gradle 'integrationTestCassandra' task
+        run: ./gradlew integrationTestCassandra ${{ matrix.mode.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }}
+
+      - name: Upload Gradle test reports
+        if: always()
+        uses: actions/upload-artifact@v5
+        with:
+          name: cassandra_5.0_integration_test_reports_${{ matrix.mode.label }}
+          path: core/build/reports/tests/integrationTestCassandra
+
   integration-test-for-cosmos:
     name: Cosmos DB integration test (${{ matrix.mode.label }})
     runs-on: windows-latest

build.gradle

@@ -26,7 +26,7 @@ subprojects {
         guiceVersion = '5.1.0'
         guavaVersion = '32.1.3-jre'
         slf4jVersion = '1.7.36'
-        cassandraDriverVersion = '3.11.5'
+        cassandraDriverVersion = '3.12.1'
         azureCosmosVersion = '4.75.0'
         azureBlobStorageVersion = '12.32.0'
         jooqVersion = '3.14.16'

core/build.gradle

@@ -167,7 +167,7 @@ dependencies {
     implementation "com.google.guava:guava:${guavaVersion}"
     implementation "com.google.inject:guice:${guiceVersion}"
     implementation "org.slf4j:slf4j-api:${slf4jVersion}"
-    implementation "com.datastax.cassandra:cassandra-driver-core:${cassandraDriverVersion}"
+    implementation "org.apache.cassandra:cassandra-driver-core:${cassandraDriverVersion}"
     implementation "com.azure:azure-cosmos:${azureCosmosVersion}"
     implementation "com.azure:azure-storage-blob:${azureBlobStorageVersion}"
     implementation "org.jooq:jooq:${jooqVersion}"

core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitAdminIntegrationTestWithObjectStorage.java

@@ -27,10 +27,6 @@ protected AdminTestUtils getAdminTestUtils(String testName) {
     return new ObjectStorageAdminTestUtils(getProperties(testName));
   }
 
-  @Override
-  @Disabled("Temporarily disabled because it includes DML operations")
-  public void truncateTable_ShouldTruncateProperly() {}
-
   @Override
   @Disabled("Object Storage does not support index-related operations")
   public void createIndex_ForAllDataTypesWithExistingData_ShouldCreateIndexesCorrectly() {}

core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitCrossPartitionScanIntegrationTestWithObjectStorage.java

@@ -0,0 +1,23 @@
+package com.scalar.db.storage.objectstorage;
+
+import com.scalar.db.transaction.consensuscommit.ConsensusCommitConfig;
+import com.scalar.db.transaction.consensuscommit.ConsensusCommitCrossPartitionScanIntegrationTestBase;
+import java.util.Properties;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+
+public class ConsensusCommitCrossPartitionScanIntegrationTestWithObjectStorage
+    extends ConsensusCommitCrossPartitionScanIntegrationTestBase {
+
+  @Override
+  protected Properties getProps(String testName) {
+    Properties properties = ConsensusCommitObjectStorageEnv.getProperties(testName);
+    properties.setProperty(ConsensusCommitConfig.ISOLATION_LEVEL, "SERIALIZABLE");
+    return properties;
+  }
+
+  @Test
+  @Override
+  @Disabled("Cross-partition scan with ordering is not supported in Object Storage")
+  public void scan_CrossPartitionScanWithOrderingGivenForCommittedRecord_ShouldReturnRecords() {}
+}

core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitIntegrationTestWithObjectStorage.java

@@ -0,0 +1,51 @@
+package com.scalar.db.storage.objectstorage;
+
+import com.scalar.db.api.TableMetadata;
+import com.scalar.db.io.DataType;
+import com.scalar.db.transaction.consensuscommit.ConsensusCommitIntegrationTestBase;
+import java.util.Properties;
+import org.junit.jupiter.api.Disabled;
+
+public class ConsensusCommitIntegrationTestWithObjectStorage
+    extends ConsensusCommitIntegrationTestBase {
+
+  @Override
+  protected TableMetadata getTableMetadata() {
+    return TableMetadata.newBuilder()
+        .addColumn(ACCOUNT_ID, DataType.INT)
+        .addColumn(ACCOUNT_TYPE, DataType.INT)
+        .addColumn(BALANCE, DataType.INT)
+        .addColumn(SOME_COLUMN, DataType.INT)
+        .addColumn(BOOLEAN_COL, DataType.BOOLEAN)
+        .addColumn(BIGINT_COL, DataType.BIGINT)
+        .addColumn(FLOAT_COL, DataType.FLOAT)
+        .addColumn(DOUBLE_COL, DataType.DOUBLE)
+        .addColumn(TEXT_COL, DataType.TEXT)
+        .addColumn(BLOB_COL, DataType.BLOB)
+        .addColumn(DATE_COL, DataType.DATE)
+        .addColumn(TIME_COL, DataType.TIME)
+        .addColumn(TIMESTAMPTZ_COL, DataType.TIMESTAMPTZ)
+        .addColumn(TIMESTAMP_COL, DataType.TIMESTAMP)
+        .addPartitionKey(ACCOUNT_ID)
+        .addClusteringKey(ACCOUNT_TYPE)
+        .build();
+  }
+
+  @Override
+  protected Properties getProps(String testName) {
+    return ConsensusCommitObjectStorageEnv.getProperties(testName);
+  }
+
+  @Override
+  @Disabled("Object Storage does not support index-related operations")
+  public void get_GetGivenForIndexColumn_ShouldReturnRecords() {}
+
+  @Override
+  @Disabled("Object Storage does not support index-related operations")
+  public void scanOrGetScanner_ScanGivenForIndexColumn_ShouldReturnRecords(ScanType scanType) {}
+
+  @Override
+  @Disabled("Object Storage does not support index-related operations")
+  public void scanOrGetScanner_ScanGivenForIndexColumnWithConjunctions_ShouldReturnRecords(
+      ScanType scanType) {}
+}

core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitNullMetadataIntegrationTestWithObjectStorage.java

@@ -0,0 +1,13 @@
+package com.scalar.db.storage.objectstorage;
+
+import com.scalar.db.transaction.consensuscommit.ConsensusCommitNullMetadataIntegrationTestBase;
+import java.util.Properties;
+
+public class ConsensusCommitNullMetadataIntegrationTestWithObjectStorage
+    extends ConsensusCommitNullMetadataIntegrationTestBase {
+
+  @Override
+  protected Properties getProperties(String testName) {
+    return ConsensusCommitObjectStorageEnv.getProperties(testName);
+  }
+}

core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitObjectStorageEnv.java

@@ -0,0 +1,22 @@
+package com.scalar.db.storage.objectstorage;
+
+import com.scalar.db.transaction.consensuscommit.ConsensusCommitTestUtils;
+import java.util.Map;
+import java.util.Properties;
+
+public class ConsensusCommitObjectStorageEnv {
+  private ConsensusCommitObjectStorageEnv() {}
+
+  public static Properties getProperties(String testName) {
+    Properties properties = ObjectStorageEnv.getProperties(testName);
+
+    // Add testName as a coordinator schema suffix
+    ConsensusCommitTestUtils.addSuffixToCoordinatorNamespace(properties, testName);
+
+    return ConsensusCommitTestUtils.loadConsensusCommitProperties(properties);
+  }
+
+  public static Map<String, String> getCreationOptions() {
+    return ObjectStorageEnv.getCreationOptions();
+  }
+}