Skip to content

Commit a496e3e

Browse files
feeblefakieKodaiD
feeblefakie
and
KodaiD
authored
Backport to branch(3) : Add validation for secondary indexes in table metadata with Object Storage (#3138)
Co-authored-by: Kodai Doki <52027276+KodaiD@users.noreply.github.com> Co-authored-by: Kodai Doki <kodai.doki@scalar-labs.com>
1 parent 10c7694 commit a496e3e

9 files changed

+115
-33
lines changed

‎core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitAdminIntegrationTestWithObjectStorage.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package com.scalar.db.storage.objectstorage;
22

3+
import com.scalar.db.api.TableMetadata;
34
import com.scalar.db.config.DatabaseConfig;
45
import com.scalar.db.transaction.consensuscommit.ConsensusCommitAdminIntegrationTestBase;
56
import com.scalar.db.transaction.consensuscommit.ConsensusCommitConfig;
@@ -15,6 +16,14 @@ protected Properties getProps(String testName) {
1516
return ObjectStorageEnv.getProperties(testName);
1617
}
1718

19+
@Override
20+
protected TableMetadata getTableMetadata() {
21+
return TableMetadata.newBuilder(TABLE_METADATA)
22+
.removeSecondaryIndex(COL_NAME5)
23+
.removeSecondaryIndex(COL_NAME6)
24+
.build();
25+
}
26+
1827
@Override
1928
protected String getSystemNamespaceName(Properties properties) {
2029
return ObjectStorageUtils.getObjectStorageConfig(new DatabaseConfig(properties))

‎core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageAdminCaseSensitivityIntegrationTest.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
package com.scalar.db.storage.objectstorage;
22

33
import com.scalar.db.api.DistributedStorageAdminCaseSensitivityIntegrationTestBase;
4+
import com.scalar.db.api.Scan;
5+
import com.scalar.db.api.TableMetadata;
46
import com.scalar.db.config.DatabaseConfig;
7+
import com.scalar.db.io.DataType;
58
import com.scalar.db.util.AdminTestUtils;
69
import java.util.Map;
710
import java.util.Properties;
@@ -21,6 +24,30 @@ protected String getSystemNamespaceName(Properties properties) {
2124
.getMetadataNamespace();
2225
}
2326

27+
@Override
28+
protected TableMetadata getTableMetadata() {
29+
return TableMetadata.newBuilder()
30+
.addColumn(getColumnName1(), DataType.INT)
31+
.addColumn(getColumnName2(), DataType.TEXT)
32+
.addColumn(getColumnName3(), DataType.TEXT)
33+
.addColumn(getColumnName4(), DataType.INT)
34+
.addColumn(getColumnName5(), DataType.INT)
35+
.addColumn(getColumnName6(), DataType.TEXT)
36+
.addColumn(getColumnName7(), DataType.BIGINT)
37+
.addColumn(getColumnName8(), DataType.FLOAT)
38+
.addColumn(getColumnName9(), DataType.DOUBLE)
39+
.addColumn(getColumnName10(), DataType.BOOLEAN)
40+
.addColumn(getColumnName11(), DataType.BLOB)
41+
.addColumn(getColumnName12(), DataType.DATE)
42+
.addColumn(getColumnName13(), DataType.TIME)
43+
.addColumn(getColumnName14(), DataType.TIMESTAMPTZ)
44+
.addPartitionKey(getColumnName2())
45+
.addPartitionKey(getColumnName1())
46+
.addClusteringKey(getColumnName4(), Scan.Ordering.Order.ASC)
47+
.addClusteringKey(getColumnName3(), Scan.Ordering.Order.DESC)
48+
.build();
49+
}
50+
2451
@Override
2552
protected Map<String, String> getCreationOptions() {
2653
return ObjectStorageEnv.getCreationOptions();

‎core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageAdminIntegrationTest.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
package com.scalar.db.storage.objectstorage;
22

33
import com.scalar.db.api.DistributedStorageAdminIntegrationTestBase;
4+
import com.scalar.db.api.Scan;
5+
import com.scalar.db.api.TableMetadata;
46
import com.scalar.db.config.DatabaseConfig;
7+
import com.scalar.db.io.DataType;
58
import com.scalar.db.util.AdminTestUtils;
69
import java.util.Properties;
710
import org.junit.jupiter.api.Disabled;
@@ -19,6 +22,30 @@ protected String getSystemNamespaceName(Properties properties) {
1922
.getMetadataNamespace();
2023
}
2124

25+
@Override
26+
protected TableMetadata getTableMetadata() {
27+
return TableMetadata.newBuilder()
28+
.addColumn(getColumnName1(), DataType.INT)
29+
.addColumn(getColumnName2(), DataType.TEXT)
30+
.addColumn(getColumnName3(), DataType.TEXT)
31+
.addColumn(getColumnName4(), DataType.INT)
32+
.addColumn(getColumnName5(), DataType.INT)
33+
.addColumn(getColumnName6(), DataType.TEXT)
34+
.addColumn(getColumnName7(), DataType.BIGINT)
35+
.addColumn(getColumnName8(), DataType.FLOAT)
36+
.addColumn(getColumnName9(), DataType.DOUBLE)
37+
.addColumn(getColumnName10(), DataType.BOOLEAN)
38+
.addColumn(getColumnName11(), DataType.BLOB)
39+
.addColumn(getColumnName12(), DataType.DATE)
40+
.addColumn(getColumnName13(), DataType.TIME)
41+
.addColumn(getColumnName14(), DataType.TIMESTAMPTZ)
42+
.addPartitionKey(getColumnName2())
43+
.addPartitionKey(getColumnName1())
44+
.addClusteringKey(getColumnName4(), Scan.Ordering.Order.ASC)
45+
.addClusteringKey(getColumnName3(), Scan.Ordering.Order.DESC)
46+
.build();
47+
}
48+
2249
@Override
2350
protected boolean isIndexOnBooleanColumnSupported() {
2451
return false;

‎core/src/integration-test/java/com/scalar/db/storage/objectstorage/SingleCrudOperationTransactionAdminIntegrationTestWithObjectStorage.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package com.scalar.db.storage.objectstorage;
22

3+
import com.scalar.db.api.TableMetadata;
34
import com.scalar.db.config.DatabaseConfig;
45
import com.scalar.db.transaction.singlecrudoperation.SingleCrudOperationTransactionAdminIntegrationTestBase;
56
import java.util.Properties;
@@ -19,6 +20,14 @@ protected String getSystemNamespaceName(Properties properties) {
1920
.getMetadataNamespace();
2021
}
2122

23+
@Override
24+
protected TableMetadata getTableMetadata() {
25+
return TableMetadata.newBuilder(TABLE_METADATA)
26+
.removeSecondaryIndex(COL_NAME5)
27+
.removeSecondaryIndex(COL_NAME6)
28+
.build();
29+
}
30+
2231
@Override
2332
@Disabled("Temporarily disabled because it includes DML operations")
2433
public void truncateTable_ShouldTruncateProperly() {}

‎core/src/main/java/com/scalar/db/storage/objectstorage/ObjectStorageAdmin.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,7 @@ public void createTable(
8888
String namespace, String table, TableMetadata metadata, Map<String, String> options)
8989
throws ExecutionException {
9090
try {
91+
checkTableMetadata(metadata);
9192
// Insert the table metadata
9293
String tableMetadataKey = getTableMetadataKey(namespace, table);
9394
Map<String, String> readVersionMap = new HashMap<>();
@@ -246,6 +247,7 @@ public void repairTable(
246247
String namespace, String table, TableMetadata metadata, Map<String, String> options)
247248
throws ExecutionException {
248249
try {
250+
checkTableMetadata(metadata);
249251
// Upsert the table metadata
250252
String tableMetadataKey = getTableMetadataKey(namespace, table);
251253
Map<String, String> readVersionMap = new HashMap<>();
@@ -445,4 +447,12 @@ private static String getTableNameFromTableMetadataKey(String tableMetadataKey)
445447
}
446448
return parts.get(1);
447449
}
450+
451+
private void checkTableMetadata(TableMetadata metadata) {
452+
Set<String> secondaryIndexNames = metadata.getSecondaryIndexNames();
453+
if (!secondaryIndexNames.isEmpty()) {
454+
throw new IllegalArgumentException(
455+
CoreError.OBJECT_STORAGE_INDEX_NOT_SUPPORTED.buildMessage());
456+
}
457+
}
448458
}

‎integration-test/src/main/java/com/scalar/db/api/DistributedStorageAdminIntegrationTestBase.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -331,9 +331,9 @@ public void getTableMetadata_CorrectTableGiven_ShouldReturnCorrectMetadata()
331331
assertThat(tableMetadata.getClusteringOrder(getColumnName14())).isNull();
332332
assertThat(tableMetadata.getClusteringOrder(getColumnName15())).isNull();
333333

334-
assertThat(tableMetadata.getSecondaryIndexNames().size()).isEqualTo(2);
335-
assertThat(tableMetadata.getSecondaryIndexNames().contains(getColumnName5())).isTrue();
336-
assertThat(tableMetadata.getSecondaryIndexNames().contains(getColumnName6())).isTrue();
334+
Set<String> expectedSecondaryIndexNames = getTableMetadata().getSecondaryIndexNames();
335+
assertThat(tableMetadata.getSecondaryIndexNames())
336+
.containsExactlyInAnyOrderElementsOf(expectedSecondaryIndexNames);
337337
}
338338

339339
@Test

‎integration-test/src/main/java/com/scalar/db/api/DistributedStorageAdminRepairTableIntegrationTestBase.java‎

Lines changed: 15 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -28,21 +28,21 @@ public abstract class DistributedStorageAdminRepairTableIntegrationTestBase {
2828
private static final String NAMESPACE = "int_test_" + TEST_NAME;
2929

3030
private static final String TABLE = "test_table";
31-
private static final String COL_NAME1 = "c1";
32-
private static final String COL_NAME2 = "c2";
33-
private static final String COL_NAME3 = "c3";
34-
private static final String COL_NAME4 = "c4";
35-
private static final String COL_NAME5 = "c5";
36-
private static final String COL_NAME6 = "c6";
37-
private static final String COL_NAME7 = "c7";
38-
private static final String COL_NAME8 = "c8";
39-
private static final String COL_NAME9 = "c9";
40-
private static final String COL_NAME10 = "c10";
41-
private static final String COL_NAME11 = "c11";
42-
private static final String COL_NAME12 = "c12";
43-
private static final String COL_NAME13 = "c13";
44-
private static final String COL_NAME14 = "c14";
45-
private static final String COL_NAME15 = "c15";
31+
protected static final String COL_NAME1 = "c1";
32+
protected static final String COL_NAME2 = "c2";
33+
protected static final String COL_NAME3 = "c3";
34+
protected static final String COL_NAME4 = "c4";
35+
protected static final String COL_NAME5 = "c5";
36+
protected static final String COL_NAME6 = "c6";
37+
protected static final String COL_NAME7 = "c7";
38+
protected static final String COL_NAME8 = "c8";
39+
protected static final String COL_NAME9 = "c9";
40+
protected static final String COL_NAME10 = "c10";
41+
protected static final String COL_NAME11 = "c11";
42+
protected static final String COL_NAME12 = "c12";
43+
protected static final String COL_NAME13 = "c13";
44+
protected static final String COL_NAME14 = "c14";
45+
protected static final String COL_NAME15 = "c15";
4646

4747
protected DistributedStorageAdmin admin;
4848

@@ -73,8 +73,6 @@ protected TableMetadata getTableMetadata() {
7373
.addPartitionKey(COL_NAME1)
7474
.addClusteringKey(COL_NAME4, Scan.Ordering.Order.ASC)
7575
.addClusteringKey(COL_NAME3, Scan.Ordering.Order.DESC)
76-
.addSecondaryIndex(COL_NAME5)
77-
.addSecondaryIndex(COL_NAME6)
7876
.build();
7977
return builder.build();
8078
}

‎integration-test/src/main/java/com/scalar/db/api/DistributedTransactionAdminIntegrationTestBase.java‎

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,10 @@ protected void initialize(String testName) throws Exception {}
111111

112112
protected abstract Properties getProperties(String testName);
113113

114+
protected TableMetadata getTableMetadata() {
115+
return TABLE_METADATA;
116+
}
117+
114118
protected String getNamespaceBaseName() {
115119
return NAMESPACE_BASE_NAME;
116120
}
@@ -130,7 +134,7 @@ private void createTables() throws ExecutionException {
130134
for (String namespace : Arrays.asList(namespace1, namespace2)) {
131135
admin.createNamespace(namespace, true, options);
132136
for (String table : Arrays.asList(TABLE1, TABLE2, TABLE3)) {
133-
admin.createTable(namespace, table, TABLE_METADATA, true, options);
137+
admin.createTable(namespace, table, getTableMetadata(), true, options);
134138
}
135139
}
136140
admin.createCoordinatorTables(true, options);
@@ -248,9 +252,9 @@ public void getTableMetadata_CorrectTableGiven_ShouldReturnCorrectMetadata()
248252
assertThat(tableMetadata.getClusteringOrder(COL_NAME14)).isNull();
249253
assertThat(tableMetadata.getClusteringOrder(COL_NAME15)).isNull();
250254

251-
assertThat(tableMetadata.getSecondaryIndexNames().size()).isEqualTo(2);
252-
assertThat(tableMetadata.getSecondaryIndexNames().contains(COL_NAME5)).isTrue();
253-
assertThat(tableMetadata.getSecondaryIndexNames().contains(COL_NAME6)).isTrue();
255+
Set<String> expectedSecondaryIndexNames = getTableMetadata().getSecondaryIndexNames();
256+
assertThat(tableMetadata.getSecondaryIndexNames())
257+
.containsExactlyInAnyOrderElementsOf(expectedSecondaryIndexNames);
254258
}
255259

256260
@Test
@@ -329,7 +333,7 @@ public void dropNamespace_ForNonEmptyNamespace_ShouldThrowIllegalArgumentExcepti
329333
try {
330334
// Arrange
331335
admin.createNamespace(namespace3);
332-
admin.createTable(namespace3, TABLE1, TABLE_METADATA);
336+
admin.createTable(namespace3, TABLE1, getTableMetadata());
333337

334338
// Act Assert
335339
assertThatThrownBy(() -> admin.dropNamespace(namespace3))
@@ -356,7 +360,7 @@ public void createTable_ForNonExistingTable_ShouldCreateTableProperly()
356360
Map<String, String> options = getCreationOptions();
357361

358362
// Act
359-
admin.createTable(namespace1, TABLE4, TABLE_METADATA, options);
363+
admin.createTable(namespace1, TABLE4, getTableMetadata(), options);
360364

361365
// Assert
362366
assertThat(admin.tableExists(namespace1, TABLE4)).isTrue();
@@ -370,7 +374,7 @@ public void createTable_ForExistingTable_ShouldThrowIllegalArgumentException() {
370374
// Arrange
371375

372376
// Act Assert
373-
assertThatThrownBy(() -> admin.createTable(namespace1, TABLE1, TABLE_METADATA))
377+
assertThatThrownBy(() -> admin.createTable(namespace1, TABLE1, getTableMetadata()))
374378
.isInstanceOf(IllegalArgumentException.class);
375379
}
376380

@@ -379,7 +383,7 @@ public void createTable_ForNonExistingNamespace_ShouldThrowIllegalArgumentExcept
379383
// Arrange
380384

381385
// Act Assert
382-
assertThatThrownBy(() -> admin.createTable(namespace3, TABLE1, TABLE_METADATA))
386+
assertThatThrownBy(() -> admin.createTable(namespace3, TABLE1, getTableMetadata()))
383387
.isInstanceOf(IllegalArgumentException.class);
384388
}
385389

@@ -388,7 +392,7 @@ public void createTable_IfNotExists_ForExistingNamespace_ShouldNotThrowAnyExcept
388392
// Arrange
389393

390394
// Act Assert
391-
assertThatCode(() -> admin.createTable(namespace1, TABLE1, TABLE_METADATA, true))
395+
assertThatCode(() -> admin.createTable(namespace1, TABLE1, getTableMetadata(), true))
392396
.doesNotThrowAnyException();
393397
}
394398

@@ -397,7 +401,7 @@ public void dropTable_ForExistingTable_ShouldDropTableProperly() throws Executio
397401
try {
398402
// Arrange
399403
Map<String, String> options = getCreationOptions();
400-
admin.createTable(namespace1, TABLE4, TABLE_METADATA, options);
404+
admin.createTable(namespace1, TABLE4, getTableMetadata(), options);
401405

402406
// Act
403407
admin.dropTable(namespace1, TABLE4);
@@ -436,7 +440,7 @@ public void truncateTable_ShouldTruncateProperly()
436440
try {
437441
// Arrange
438442
Map<String, String> options = getCreationOptions();
439-
admin.createTable(namespace1, table, TABLE_METADATA, true, options);
443+
admin.createTable(namespace1, table, getTableMetadata(), true, options);
440444
Key partitionKey = Key.of(COL_NAME2, "aaa", COL_NAME1, 1);
441445
Key clusteringKey = Key.of(COL_NAME4, 2, COL_NAME3, "bbb");
442446
transactionalInsert(

‎integration-test/src/main/java/com/scalar/db/api/DistributedTransactionAdminRepairTableIntegrationTestBase.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,8 +68,6 @@ protected TableMetadata getTableMetadata() {
6868
.addPartitionKey(COL_NAME1)
6969
.addClusteringKey(COL_NAME4, Scan.Ordering.Order.ASC)
7070
.addClusteringKey(COL_NAME3, Scan.Ordering.Order.DESC)
71-
.addSecondaryIndex(COL_NAME5)
72-
.addSecondaryIndex(COL_NAME6)
7371
.build();
7472
return builder.build();
7573
}

0 commit comments

Comments
 (0)