Skip to content

Commit ad27355

Browse files
KodaiDKodaiD
authored
Add validation for secondary indexes in table metadata with Object Storage (#3127)
1 parent e01f5a1 commit ad27355

11 files changed

+177
-33
lines changed

‎core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitAdminIntegrationTestWithObjectStorage.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package com.scalar.db.storage.objectstorage;
22

3+
import com.scalar.db.api.TableMetadata;
34
import com.scalar.db.transaction.consensuscommit.ConsensusCommitAdminIntegrationTestBase;
45
import com.scalar.db.util.AdminTestUtils;
56
import java.util.Properties;
@@ -13,6 +14,14 @@ protected Properties getProps(String testName) {
1314
return ObjectStorageEnv.getProperties(testName);
1415
}
1516

17+
@Override
18+
protected TableMetadata getTableMetadata() {
19+
return TableMetadata.newBuilder(TABLE_METADATA)
20+
.removeSecondaryIndex(COL_NAME5)
21+
.removeSecondaryIndex(COL_NAME6)
22+
.build();
23+
}
24+
1625
@Override
1726
protected AdminTestUtils getAdminTestUtils(String testName) {
1827
return new ObjectStorageAdminTestUtils(getProperties(testName));

‎core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitAdminRepairIntegrationTestWithObjectStorage.java‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,39 @@
11
package com.scalar.db.storage.objectstorage;
22

3+
import com.scalar.db.api.Scan;
4+
import com.scalar.db.api.TableMetadata;
5+
import com.scalar.db.io.DataType;
36
import com.scalar.db.transaction.consensuscommit.ConsensusCommitAdminRepairIntegrationTestBase;
47
import java.util.Properties;
58

69
public class ConsensusCommitAdminRepairIntegrationTestWithObjectStorage
710
extends ConsensusCommitAdminRepairIntegrationTestBase {
811

12+
@Override
13+
protected TableMetadata getTableMetadata() {
14+
return TableMetadata.newBuilder()
15+
.addColumn(COL_NAME1, DataType.INT)
16+
.addColumn(COL_NAME2, DataType.TEXT)
17+
.addColumn(COL_NAME3, DataType.TEXT)
18+
.addColumn(COL_NAME4, DataType.INT)
19+
.addColumn(COL_NAME5, DataType.INT)
20+
.addColumn(COL_NAME6, DataType.TEXT)
21+
.addColumn(COL_NAME7, DataType.BIGINT)
22+
.addColumn(COL_NAME8, DataType.FLOAT)
23+
.addColumn(COL_NAME9, DataType.DOUBLE)
24+
.addColumn(COL_NAME10, DataType.BOOLEAN)
25+
.addColumn(COL_NAME11, DataType.BLOB)
26+
.addColumn(COL_NAME12, DataType.DATE)
27+
.addColumn(COL_NAME13, DataType.TIME)
28+
.addColumn(COL_NAME14, DataType.TIMESTAMPTZ)
29+
.addColumn(COL_NAME15, DataType.TIMESTAMP)
30+
.addPartitionKey(COL_NAME2)
31+
.addPartitionKey(COL_NAME1)
32+
.addClusteringKey(COL_NAME4, Scan.Ordering.Order.ASC)
33+
.addClusteringKey(COL_NAME3, Scan.Ordering.Order.DESC)
34+
.build();
35+
}
36+
937
@Override
1038
protected Properties getProps(String testName) {
1139
return ObjectStorageEnv.getProperties(testName);

‎core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageAdminCaseSensitivityIntegrationTest.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
package com.scalar.db.storage.objectstorage;
22

33
import com.scalar.db.api.DistributedStorageAdminCaseSensitivityIntegrationTestBase;
4+
import com.scalar.db.api.Scan;
5+
import com.scalar.db.api.TableMetadata;
6+
import com.scalar.db.io.DataType;
47
import com.scalar.db.util.AdminTestUtils;
58
import java.util.Map;
69
import java.util.Properties;
@@ -14,6 +17,30 @@ protected Properties getProperties(String testName) {
1417
return ObjectStorageEnv.getProperties(testName);
1518
}
1619

20+
@Override
21+
protected TableMetadata getTableMetadata() {
22+
return TableMetadata.newBuilder()
23+
.addColumn(getColumnName1(), DataType.INT)
24+
.addColumn(getColumnName2(), DataType.TEXT)
25+
.addColumn(getColumnName3(), DataType.TEXT)
26+
.addColumn(getColumnName4(), DataType.INT)
27+
.addColumn(getColumnName5(), DataType.INT)
28+
.addColumn(getColumnName6(), DataType.TEXT)
29+
.addColumn(getColumnName7(), DataType.BIGINT)
30+
.addColumn(getColumnName8(), DataType.FLOAT)
31+
.addColumn(getColumnName9(), DataType.DOUBLE)
32+
.addColumn(getColumnName10(), DataType.BOOLEAN)
33+
.addColumn(getColumnName11(), DataType.BLOB)
34+
.addColumn(getColumnName12(), DataType.DATE)
35+
.addColumn(getColumnName13(), DataType.TIME)
36+
.addColumn(getColumnName14(), DataType.TIMESTAMPTZ)
37+
.addPartitionKey(getColumnName2())
38+
.addPartitionKey(getColumnName1())
39+
.addClusteringKey(getColumnName4(), Scan.Ordering.Order.ASC)
40+
.addClusteringKey(getColumnName3(), Scan.Ordering.Order.DESC)
41+
.build();
42+
}
43+
1744
@Override
1845
protected Map<String, String> getCreationOptions() {
1946
return ObjectStorageEnv.getCreationOptions();

‎core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageAdminIntegrationTest.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
package com.scalar.db.storage.objectstorage;
22

33
import com.scalar.db.api.DistributedStorageAdminIntegrationTestBase;
4+
import com.scalar.db.api.Scan;
5+
import com.scalar.db.api.TableMetadata;
6+
import com.scalar.db.io.DataType;
47
import com.scalar.db.util.AdminTestUtils;
58
import java.util.Properties;
69
import org.junit.jupiter.api.Disabled;
@@ -12,6 +15,30 @@ protected Properties getProperties(String testName) {
1215
return ObjectStorageEnv.getProperties(testName);
1316
}
1417

18+
@Override
19+
protected TableMetadata getTableMetadata() {
20+
return TableMetadata.newBuilder()
21+
.addColumn(getColumnName1(), DataType.INT)
22+
.addColumn(getColumnName2(), DataType.TEXT)
23+
.addColumn(getColumnName3(), DataType.TEXT)
24+
.addColumn(getColumnName4(), DataType.INT)
25+
.addColumn(getColumnName5(), DataType.INT)
26+
.addColumn(getColumnName6(), DataType.TEXT)
27+
.addColumn(getColumnName7(), DataType.BIGINT)
28+
.addColumn(getColumnName8(), DataType.FLOAT)
29+
.addColumn(getColumnName9(), DataType.DOUBLE)
30+
.addColumn(getColumnName10(), DataType.BOOLEAN)
31+
.addColumn(getColumnName11(), DataType.BLOB)
32+
.addColumn(getColumnName12(), DataType.DATE)
33+
.addColumn(getColumnName13(), DataType.TIME)
34+
.addColumn(getColumnName14(), DataType.TIMESTAMPTZ)
35+
.addPartitionKey(getColumnName2())
36+
.addPartitionKey(getColumnName1())
37+
.addClusteringKey(getColumnName4(), Scan.Ordering.Order.ASC)
38+
.addClusteringKey(getColumnName3(), Scan.Ordering.Order.DESC)
39+
.build();
40+
}
41+
1542
@Override
1643
protected boolean isIndexOnBooleanColumnSupported() {
1744
return false;

‎core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageAdminRepairIntegrationTest.java‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,40 @@
11
package com.scalar.db.storage.objectstorage;
22

33
import com.scalar.db.api.DistributedStorageAdminRepairIntegrationTestBase;
4+
import com.scalar.db.api.Scan;
5+
import com.scalar.db.api.TableMetadata;
6+
import com.scalar.db.io.DataType;
47
import java.util.Properties;
58
import org.junit.jupiter.api.Disabled;
69

710
public class ObjectStorageAdminRepairIntegrationTest
811
extends DistributedStorageAdminRepairIntegrationTestBase {
912

13+
@Override
14+
protected TableMetadata getTableMetadata() {
15+
return TableMetadata.newBuilder()
16+
.addColumn(COL_NAME1, DataType.INT)
17+
.addColumn(COL_NAME2, DataType.TEXT)
18+
.addColumn(COL_NAME3, DataType.TEXT)
19+
.addColumn(COL_NAME4, DataType.INT)
20+
.addColumn(COL_NAME5, DataType.INT)
21+
.addColumn(COL_NAME6, DataType.TEXT)
22+
.addColumn(COL_NAME7, DataType.BIGINT)
23+
.addColumn(COL_NAME8, DataType.FLOAT)
24+
.addColumn(COL_NAME9, DataType.DOUBLE)
25+
.addColumn(COL_NAME10, DataType.BOOLEAN)
26+
.addColumn(COL_NAME11, DataType.BLOB)
27+
.addColumn(COL_NAME12, DataType.DATE)
28+
.addColumn(COL_NAME13, DataType.TIME)
29+
.addColumn(COL_NAME14, DataType.TIMESTAMPTZ)
30+
.addColumn(COL_NAME15, DataType.TIMESTAMP)
31+
.addPartitionKey(COL_NAME2)
32+
.addPartitionKey(COL_NAME1)
33+
.addClusteringKey(COL_NAME4, Scan.Ordering.Order.ASC)
34+
.addClusteringKey(COL_NAME3, Scan.Ordering.Order.DESC)
35+
.build();
36+
}
37+
1038
@Override
1139
protected Properties getProperties(String testName) {
1240
return ObjectStorageEnv.getProperties(testName);

‎core/src/integration-test/java/com/scalar/db/storage/objectstorage/SingleCrudOperationTransactionAdminIntegrationTestWithObjectStorage.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package com.scalar.db.storage.objectstorage;
22

3+
import com.scalar.db.api.TableMetadata;
34
import com.scalar.db.transaction.singlecrudoperation.SingleCrudOperationTransactionAdminIntegrationTestBase;
45
import java.util.Properties;
56
import org.junit.jupiter.api.Disabled;
@@ -12,6 +13,14 @@ protected Properties getProps(String testName) {
1213
return ObjectStorageEnv.getProperties(testName);
1314
}
1415

16+
@Override
17+
protected TableMetadata getTableMetadata() {
18+
return TableMetadata.newBuilder(TABLE_METADATA)
19+
.removeSecondaryIndex(COL_NAME5)
20+
.removeSecondaryIndex(COL_NAME6)
21+
.build();
22+
}
23+
1524
@Override
1625
@Disabled("Temporarily disabled because it includes DML operations")
1726
public void truncateTable_ShouldTruncateProperly() {}

‎core/src/main/java/com/scalar/db/storage/objectstorage/ObjectStorageAdmin.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@ public void createTable(
8787
String namespace, String table, TableMetadata metadata, Map<String, String> options)
8888
throws ExecutionException {
8989
try {
90+
checkTableMetadata(metadata);
9091
// Insert the table metadata
9192
String tableMetadataKey = getTableMetadataKey(namespace, table);
9293
Map<String, String> readVersionMap = new HashMap<>();
@@ -268,6 +269,7 @@ public void repairTable(
268269
String namespace, String table, TableMetadata metadata, Map<String, String> options)
269270
throws ExecutionException {
270271
try {
272+
checkTableMetadata(metadata);
271273
// Upsert the table metadata
272274
String tableMetadataKey = getTableMetadataKey(namespace, table);
273275
Map<String, String> readVersionMap = new HashMap<>();
@@ -470,4 +472,12 @@ private static String getTableNameFromTableMetadataKey(String tableMetadataKey)
470472
}
471473
return parts.get(1);
472474
}
475+
476+
private void checkTableMetadata(TableMetadata metadata) {
477+
Set<String> secondaryIndexNames = metadata.getSecondaryIndexNames();
478+
if (!secondaryIndexNames.isEmpty()) {
479+
throw new IllegalArgumentException(
480+
CoreError.OBJECT_STORAGE_INDEX_NOT_SUPPORTED.buildMessage());
481+
}
482+
}
473483
}

‎integration-test/src/main/java/com/scalar/db/api/DistributedStorageAdminIntegrationTestBase.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -330,9 +330,9 @@ public void getTableMetadata_CorrectTableGiven_ShouldReturnCorrectMetadata()
330330
assertThat(tableMetadata.getClusteringOrder(getColumnName14())).isNull();
331331
assertThat(tableMetadata.getClusteringOrder(getColumnName15())).isNull();
332332

333-
assertThat(tableMetadata.getSecondaryIndexNames().size()).isEqualTo(2);
334-
assertThat(tableMetadata.getSecondaryIndexNames().contains(getColumnName5())).isTrue();
335-
assertThat(tableMetadata.getSecondaryIndexNames().contains(getColumnName6())).isTrue();
333+
Set<String> expectedSecondaryIndexNames = getTableMetadata().getSecondaryIndexNames();
334+
assertThat(tableMetadata.getSecondaryIndexNames())
335+
.containsExactlyInAnyOrderElementsOf(expectedSecondaryIndexNames);
336336
}
337337

338338
@Test

‎integration-test/src/main/java/com/scalar/db/api/DistributedStorageAdminRepairIntegrationTestBase.java‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -27,21 +27,21 @@ public abstract class DistributedStorageAdminRepairIntegrationTestBase {
2727
private static final String TEST_NAME = "storage_admin_repair";
2828
private static final String NAMESPACE = "int_test_" + TEST_NAME;
2929
private static final String TABLE = "test_table";
30-
private static final String COL_NAME1 = "c1";
31-
private static final String COL_NAME2 = "c2";
32-
private static final String COL_NAME3 = "c3";
33-
private static final String COL_NAME4 = "c4";
34-
private static final String COL_NAME5 = "c5";
35-
private static final String COL_NAME6 = "c6";
36-
private static final String COL_NAME7 = "c7";
37-
private static final String COL_NAME8 = "c8";
38-
private static final String COL_NAME9 = "c9";
39-
private static final String COL_NAME10 = "c10";
40-
private static final String COL_NAME11 = "c11";
41-
private static final String COL_NAME12 = "c12";
42-
private static final String COL_NAME13 = "c13";
43-
private static final String COL_NAME14 = "c14";
44-
private static final String COL_NAME15 = "c15";
30+
protected static final String COL_NAME1 = "c1";
31+
protected static final String COL_NAME2 = "c2";
32+
protected static final String COL_NAME3 = "c3";
33+
protected static final String COL_NAME4 = "c4";
34+
protected static final String COL_NAME5 = "c5";
35+
protected static final String COL_NAME6 = "c6";
36+
protected static final String COL_NAME7 = "c7";
37+
protected static final String COL_NAME8 = "c8";
38+
protected static final String COL_NAME9 = "c9";
39+
protected static final String COL_NAME10 = "c10";
40+
protected static final String COL_NAME11 = "c11";
41+
protected static final String COL_NAME12 = "c12";
42+
protected static final String COL_NAME13 = "c13";
43+
protected static final String COL_NAME14 = "c14";
44+
protected static final String COL_NAME15 = "c15";
4545

4646
protected DistributedStorageAdmin admin;
4747

‎integration-test/src/main/java/com/scalar/db/api/DistributedTransactionAdminIntegrationTestBase.java‎

Lines changed: 17 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,10 @@ protected void initialize(String testName) throws Exception {}
111111

112112
protected abstract Properties getProperties(String testName);
113113

114+
protected TableMetadata getTableMetadata() {
115+
return TABLE_METADATA;
116+
}
117+
114118
protected String getNamespaceBaseName() {
115119
return NAMESPACE_BASE_NAME;
116120
}
@@ -120,7 +124,7 @@ private void createTables() throws ExecutionException {
120124
for (String namespace : Arrays.asList(namespace1, namespace2)) {
121125
admin.createNamespace(namespace, true, options);
122126
for (String table : Arrays.asList(TABLE1, TABLE2, TABLE3)) {
123-
admin.createTable(namespace, table, TABLE_METADATA, true, options);
127+
admin.createTable(namespace, table, getTableMetadata(), true, options);
124128
}
125129
}
126130
admin.createCoordinatorTables(true, options);
@@ -240,9 +244,9 @@ public void getTableMetadata_CorrectTableGiven_ShouldReturnCorrectMetadata()
240244
assertThat(tableMetadata.getClusteringOrder(COL_NAME14)).isNull();
241245
assertThat(tableMetadata.getClusteringOrder(COL_NAME15)).isNull();
242246

243-
assertThat(tableMetadata.getSecondaryIndexNames().size()).isEqualTo(2);
244-
assertThat(tableMetadata.getSecondaryIndexNames().contains(COL_NAME5)).isTrue();
245-
assertThat(tableMetadata.getSecondaryIndexNames().contains(COL_NAME6)).isTrue();
247+
Set<String> expectedSecondaryIndexNames = getTableMetadata().getSecondaryIndexNames();
248+
assertThat(tableMetadata.getSecondaryIndexNames())
249+
.containsExactlyInAnyOrderElementsOf(expectedSecondaryIndexNames);
246250
}
247251

248252
@Test
@@ -322,7 +326,7 @@ public void dropNamespace_ForNonEmptyNamespace_ShouldThrowIllegalArgumentExcepti
322326
try {
323327
// Arrange
324328
admin.createNamespace(namespace3, getCreationOptions());
325-
admin.createTable(namespace3, TABLE1, TABLE_METADATA, getCreationOptions());
329+
admin.createTable(namespace3, TABLE1, getTableMetadata(), getCreationOptions());
326330

327331
// Act Assert
328332
assertThatThrownBy(() -> admin.dropNamespace(namespace3))
@@ -349,7 +353,7 @@ public void createTable_ForNonExistingTable_ShouldCreateTableProperly()
349353
Map<String, String> options = getCreationOptions();
350354

351355
// Act
352-
admin.createTable(namespace1, TABLE4, TABLE_METADATA, options);
356+
admin.createTable(namespace1, TABLE4, getTableMetadata(), options);
353357

354358
// Assert
355359
assertThat(admin.tableExists(namespace1, TABLE4)).isTrue();
@@ -364,7 +368,7 @@ public void createTable_ForExistingTable_ShouldThrowIllegalArgumentException() {
364368

365369
// Act Assert
366370
assertThatThrownBy(
367-
() -> admin.createTable(namespace1, TABLE1, TABLE_METADATA, getCreationOptions()))
371+
() -> admin.createTable(namespace1, TABLE1, getTableMetadata(), getCreationOptions()))
368372
.isInstanceOf(IllegalArgumentException.class);
369373
}
370374

@@ -374,7 +378,7 @@ public void createTable_ForNonExistingNamespace_ShouldThrowIllegalArgumentExcept
374378

375379
// Act Assert
376380
assertThatThrownBy(
377-
() -> admin.createTable(namespace3, TABLE1, TABLE_METADATA, getCreationOptions()))
381+
() -> admin.createTable(namespace3, TABLE1, getTableMetadata(), getCreationOptions()))
378382
.isInstanceOf(IllegalArgumentException.class);
379383
}
380384

@@ -384,7 +388,9 @@ public void createTable_IfNotExists_ForExistingNamespace_ShouldNotThrowAnyExcept
384388

385389
// Act Assert
386390
assertThatCode(
387-
() -> admin.createTable(namespace1, TABLE1, TABLE_METADATA, true, getCreationOptions()))
391+
() ->
392+
admin.createTable(
393+
namespace1, TABLE1, getTableMetadata(), true, getCreationOptions()))
388394
.doesNotThrowAnyException();
389395
}
390396

@@ -393,7 +399,7 @@ public void dropTable_ForExistingTable_ShouldDropTableProperly() throws Executio
393399
try {
394400
// Arrange
395401
Map<String, String> options = getCreationOptions();
396-
admin.createTable(namespace1, TABLE4, TABLE_METADATA, options);
402+
admin.createTable(namespace1, TABLE4, getTableMetadata(), options);
397403

398404
// Act
399405
admin.dropTable(namespace1, TABLE4);
@@ -432,7 +438,7 @@ public void truncateTable_ShouldTruncateProperly()
432438
try {
433439
// Arrange
434440
Map<String, String> options = getCreationOptions();
435-
admin.createTable(namespace1, table, TABLE_METADATA, true, options);
441+
admin.createTable(namespace1, table, getTableMetadata(), true, options);
436442
Key partitionKey = Key.of(COL_NAME2, "aaa", COL_NAME1, 1);
437443
Key clusteringKey = Key.of(COL_NAME4, 2, COL_NAME3, "bbb");
438444
transactionalInsert(

0 commit comments

Comments
 (0)