diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePath.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePath.java index ff228ef1be..f9e4b7e216 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePath.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePath.java @@ -410,4 +410,26 @@ default RecordCursor exportAllData(@Nonnull FDBRecordContext @Nonnull ScanProperties scanProperties) { throw new UnsupportedOperationException("exportAllData is not supported"); } + + /** + * Imports the provided data exported via {@link #exportAllData} into this {@code KeySpacePath}. + *

+ * This will validate that any data provided in {@code dataToImport} has a path that should be in this path, + * or one of the sub-directories, if not, the future will complete exceptionally with + * {@link RecordCoreIllegalImportDataException}. + * If there is any data already existing under this path, the new data will overwrite if the keys are the same. + * This will use the logical values in the {@link DataInKeySpacePath#getPath()} and + * {@link DataInKeySpacePath#getRemainder()} to determine the key, rather + * than the raw key, meaning that this will work even if the data was exported from a different cluster. + * Note, this will not correct for any cluster-specific data, other than {@link DirectoryLayerDirectory} data; + * for example, if you have versionstamps, that data will not align on the destination. + *

+ * @param context the transaction context in which to save the data + * @param dataToImport the data to be saved to the database + * @return a future to be completed once all data has been important. + */ + @API(API.Status.EXPERIMENTAL) + @Nonnull + CompletableFuture importData(@Nonnull FDBRecordContext context, + @Nonnull Iterable dataToImport); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImpl.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImpl.java index 794672fd04..cdbccae613 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImpl.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImpl.java @@ -316,6 +316,43 @@ public RecordCursor exportAllData(@Nonnull FDBRecordContext 1); } + @Nonnull + @Override + public CompletableFuture importData(@Nonnull FDBRecordContext context, + @Nonnull Iterable dataToImport) { + return toTupleAsync(context).thenCompose(targetTuple -> { + List> importFutures = new ArrayList<>(); + + for (DataInKeySpacePath dataItem : dataToImport) { + CompletableFuture importFuture = dataItem.getPath().toTupleAsync(context).thenCompose(itemPathTuple -> { + // Validate that this data belongs under this path + if (!TupleHelpers.isPrefix(targetTuple, itemPathTuple)) { + throw new RecordCoreIllegalImportDataException( + "Data item path does not belong under target path", + "target", targetTuple, + "item", itemPathTuple); + } + + // Reconstruct the key using the path and remainder + Tuple keyTuple = itemPathTuple; + if (dataItem.getRemainder() != null) { + keyTuple = keyTuple.addAll(dataItem.getRemainder()); + } + + // Store the data + byte[] keyBytes = keyTuple.pack(); + byte[] valueBytes = dataItem.getValue(); + context.ensureActive().set(keyBytes, valueBytes); + + return AsyncUtil.DONE; + }); + importFutures.add(importFuture); + } + + return AsyncUtil.whenAll(importFutures); + }); + } + /** * Returns this path properly wrapped in whatever implementation the directory the path is contained in dictates. */ diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathWrapper.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathWrapper.java index 1cec9787ea..baa020ddf3 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathWrapper.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathWrapper.java @@ -202,4 +202,11 @@ public RecordCursor exportAllData(@Nonnull FDBRecordContext @Nonnull ScanProperties scanProperties) { return inner.exportAllData(context, continuation, scanProperties); } + + @Nonnull + @Override + public CompletableFuture importData(@Nonnull FDBRecordContext context, + @Nonnull Iterable dataToImport) { + return inner.importData(context, dataToImport); + } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/RecordCoreIllegalImportDataException.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/RecordCoreIllegalImportDataException.java new file mode 100644 index 0000000000..7879eefab7 --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/RecordCoreIllegalImportDataException.java @@ -0,0 +1,36 @@ +/* + * RecordCoreIllegalImportDataException.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.provider.foundationdb.keyspace; + +import com.apple.foundationdb.record.RecordCoreArgumentException; + +import javax.annotation.Nonnull; + +/** + * Thrown if the data being imported into {@link KeySpacePath#importData} does not belong in that path. + */ +public class RecordCoreIllegalImportDataException extends RecordCoreArgumentException { + private static final long serialVersionUID = 1L; + + public RecordCoreIllegalImportDataException(@Nonnull final String msg, @Nonnull final Object... keyValue) { + super(msg, keyValue); + } +} diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathDataExportTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathDataExportTest.java index 1ff39187dd..44d8f1c6ae 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathDataExportTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathDataExportTest.java @@ -49,7 +49,6 @@ import java.util.Map; import java.util.Set; import java.util.UUID; -import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -69,7 +68,7 @@ class KeySpacePathDataExportTest { final FDBDatabaseExtension dbExtension = new FDBDatabaseExtension(); @Test - void exportAllDataFromSimplePath() throws ExecutionException, InterruptedException { + void exportAllDataFromSimplePath() { KeySpace root = new KeySpace( new KeySpaceDirectory("root", KeyType.STRING, UUID.randomUUID().toString()) .addSubdirectory(new KeySpaceDirectory("level1", KeyType.LONG))); @@ -83,13 +82,14 @@ void exportAllDataFromSimplePath() throws ExecutionException, InterruptedExcepti // Add data at different levels for (int i = 0; i < 5; i++) { - Tuple key = basePath.add("level1", (long) i).toTuple(context); + final KeySpacePath path = basePath.add("level1", (long)i); + Tuple key = path.toTuple(context); tr.set(key.pack(), Tuple.from("value" + i).pack()); // Add some sub-data under each key for (int j = 0; j < 3; j++) { - Tuple subKey = key.add("sub" + j); - tr.set(subKey.pack(), Tuple.from("subvalue" + i + "_" + j).pack()); + tr.set(path.toSubspace(context).pack(Tuple.from("sub" + j)), + Tuple.from("subvalue" + i + "_" + j).pack()); } } context.commit(); @@ -103,16 +103,19 @@ void exportAllDataFromSimplePath() throws ExecutionException, InterruptedExcepti // Should have 5 main entries + 15 sub-entries = 20 total assertEquals(20, allData.size()); + assertThat(allData) + .allSatisfy(data -> + assertThat(data.getPath().getDirectoryName()).isEqualTo("level1")); + // Verify the data is sorted by key - for (int i = 1; i < allData.size(); i++) { - assertTrue(getKey(allData.get(i - 1), context).compareTo(getKey(allData.get(i), context)) < 0); - } + assertThat(allData.stream().map(data -> getKey(data, context)).collect(Collectors.toList())) + .isSorted(); } } // `toTuple` does not include the remainder, I'm not sure if that is intentional, or an oversight. - private Tuple getKey(final DataInKeySpacePath dataInKeySpacePath, final FDBRecordContext context) throws ExecutionException, InterruptedException { - final ResolvedKeySpacePath resolvedKeySpacePath = dataInKeySpacePath.getPath().toResolvedPathAsync(context).get(); + private Tuple getKey(final DataInKeySpacePath dataInKeySpacePath, final FDBRecordContext context) { + final ResolvedKeySpacePath resolvedKeySpacePath = dataInKeySpacePath.getPath().toResolvedPathAsync(context).join(); if (dataInKeySpacePath.getRemainder() != null) { return resolvedKeySpacePath.toTuple().addAll(dataInKeySpacePath.getRemainder()); } else { @@ -524,9 +527,7 @@ private static void exportWithContinuations(final KeySpacePath pathToExport, final RecordCursor cursor = pathToExport.exportAllData(context, continuation.toBytes(), scanProperties); final AtomicReference> tupleResult = new AtomicReference<>(); - final List batch = cursor.map(dataInPath -> { - return Tuple.fromBytes(dataInPath.getValue()); - }).asList(tupleResult).join(); + final List batch = cursor.map(dataInPath -> Tuple.fromBytes(dataInPath.getValue())).asList(tupleResult).join(); actual.add(batch); continuation = tupleResult.get().getContinuation(); } @@ -578,7 +579,7 @@ void exportAllDataThroughKeySpacePathWrapper() { } @Test - void exportAllDataThroughKeySpacePathWrapperResolvedPaths() { + void exportAllDataThroughKeySpacePathWrapperRemainders() { final FDBDatabase database = dbExtension.getDatabase(); final EnvironmentKeySpace keySpace = EnvironmentKeySpace.setupSampleData(database); diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java new file mode 100644 index 0000000000..86f49bf469 --- /dev/null +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java @@ -0,0 +1,419 @@ +/* + * KeySpacePathImportDataTest.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.provider.foundationdb.keyspace; + +import com.apple.foundationdb.Transaction; +import com.apple.foundationdb.record.ScanProperties; +import com.apple.foundationdb.record.provider.foundationdb.FDBDatabase; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext; +import com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpaceDirectory.KeyType; +import com.apple.foundationdb.record.test.FDBDatabaseExtension; +import com.apple.foundationdb.tuple.Tuple; +import com.apple.test.Tags; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import javax.annotation.Nonnull; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.CompletionException; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests for {@link KeySpacePath#importData(FDBRecordContext, Iterable)}. + */ +@Tag(Tags.RequiresFDB) +class KeySpacePathImportDataTest { + @RegisterExtension + final FDBDatabaseExtension dbExtension = new FDBDatabaseExtension(); + private FDBDatabase database; + private List databases; + + @BeforeEach + void setUp() { + databases = dbExtension.getDatabases(2); + database = databases.get(0); + } + + @Test + void importComprehensiveData() { + // Test importing data covering ALL KeyType enum values in a single complex directory structure: + // - NULL, BYTES, STRING, LONG, FLOAT, DOUBLE, BOOLEAN, UUID + // - Constant value directories + // - Complex multi-level hierarchy + // - Different data types in remainder elements + // Given that the majority of complexity is in the components under the importData method, this is basically + // an integration test + final String rootUuid = UUID.randomUUID().toString(); + byte[] binaryId = {0x01, 0x02, 0x03, (byte) 0xFF, (byte) 0xFE}; + UUID memberId = UUID.randomUUID(); + + KeySpace root = new KeySpace( + new KeySpaceDirectory("company", KeyType.STRING, rootUuid) + .addSubdirectory(new KeySpaceDirectory("version", KeyType.LONG, 1L) + .addSubdirectory(new KeySpaceDirectory("department", KeyType.STRING) + .addSubdirectory(new KeySpaceDirectory("employee_id", KeyType.LONG) + .addSubdirectory(new KeySpaceDirectory("binary_data", KeyType.BYTES) + .addSubdirectory(new KeySpaceDirectory("null_section", KeyType.NULL) + .addSubdirectory(new KeySpaceDirectory("member", KeyType.UUID) + .addSubdirectory(new KeySpaceDirectory("active", KeyType.BOOLEAN) + .addSubdirectory(new KeySpaceDirectory("rating", KeyType.FLOAT)))))))))); + + + // Create comprehensive test data covering ALL KeyType values + KeySpacePath basePath = root.path("company").add("version").add("department", "engineering"); + + // Build paths using all KeyType values + KeySpacePath emp1Path = basePath.add("employee_id", 100L) + .add("binary_data", binaryId) + .add("null_section") + .add("member", memberId) + .add("active", true) + .add("rating", 4.5f); + + KeySpacePath emp2Path = basePath.add("employee_id", 200L) + .add("binary_data", binaryId) + .add("null_section") + .add("member", memberId) + .add("active", false) + .add("rating", 3.8f); + + try (FDBRecordContext context = database.openContext()) { + setInPath(emp1Path, context, Tuple.from("profile", "name"), "John Doe"); + setInPath(emp2Path, context, Tuple.from("profile", "name"), "Jane Smith"); + setInPath(emp1Path, context, Tuple.from("salary"), 75000); + setInPath(emp1Path, context, Tuple.from("info", 42, true, "complex"), "Complex Test"); + + byte[] binaryKey = emp1Path.toSubspace(context).pack(Tuple.from("binary_metadata")); + context.ensureActive().set(binaryKey, "binary_test_data".getBytes()); + + context.commit(); + } + + copyData(root.path("company"), root.path("company")); + + // Verify all different KeyType values were handled correctly during import + try (FDBRecordContext context = database.openContext()) { + assertEquals(Tuple.from("John Doe"), + getTupleFromPath(context, emp1Path, Tuple.from("profile", "name"))); + assertEquals(Tuple.from("Jane Smith"), + getTupleFromPath(context, emp2Path, Tuple.from("profile", "name"))); + assertEquals(Tuple.from(75000), + getTupleFromPath(context, emp1Path, Tuple.from("salary"))); + assertEquals(Tuple.from("Complex Test"), + getTupleFromPath(context, emp1Path, Tuple.from("info", 42, true, "complex"))); + + // Verify BYTES data (raw binary, not in tuple) + byte[] binaryKey = emp1Path.toSubspace(context).pack(Tuple.from("binary_metadata")); + assertArrayEquals("binary_test_data".getBytes(), context.ensureActive().get(binaryKey).join()); + } + } + + @Test + void importEmptyData() { + // Test importing an empty collection of data + // Should complete successfully without modifying the data under the path + KeySpace root = new KeySpace( + new KeySpaceDirectory("test", KeyType.STRING, UUID.randomUUID().toString())); + + KeySpacePath testPath = root.path("test"); + importData(database, testPath, Collections.emptyList()); // should not throw any exception + + assertTrue(getExportedData(testPath).isEmpty(), + "there should not have been any data created"); + } + + @Test + void importOverwriteExistingData() { + // Test importing data that overwrites existing keys + // Should verify that new data replaces old data when keys match + final String rootUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new KeySpaceDirectory("root", KeyType.STRING, rootUuid) + .addSubdirectory(new KeySpaceDirectory("data", KeyType.LONG))); + + final KeySpacePath dataPath = root.path("root").add("data", 1L); + setSingleKey(dataPath, Tuple.from("record"), Tuple.from("original_value")); + setSingleKey(dataPath, Tuple.from("other"), Tuple.from("other_value")); + + // Create import data with same key but different value + List importData = new ArrayList<>(); + importData.add(new DataInKeySpacePath(dataPath, + Tuple.from("record"), Tuple.from("new_value").pack())); + + // Verify we can re-import the data multiple times + importData(database, root.path("root"), importData); + importData(database, root.path("root"), importData); + importData(database, root.path("root"), importData); + + // Verify the data was overwritten + verifySingleKey(dataPath, Tuple.from("record"), Tuple.from("new_value")); + verifySingleKey(dataPath, Tuple.from("other"), Tuple.from("other_value")); + } + + @Test + void importDataWithDirectoryLayer() { + // Test importing data into a keyspace using DirectoryLayer directories + // Should verify that DirectoryLayer mappings work correctly during import + final String tenantUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new DirectoryLayerDirectory("tenant", tenantUuid) + .addSubdirectory(new KeySpaceDirectory("user_id", KeyType.LONG))); + + final KeySpacePath dataPath = root.path("tenant").add("user_id", 999L); + setSingleKey(dataPath, Tuple.from("data"), Tuple.from("directory_test")); + + copyData(root.path("tenant"), root.path("tenant")); + + verifySingleKey(dataPath, Tuple.from("data"), Tuple.from("directory_test")); + } + + @Test + void importDataWithMismatchedPath() { + // Test importing data that doesn't belong to the target path + // Should throw RecordCoreIllegalImportDataException + final String root1Uuid = UUID.randomUUID().toString(); + final String root2Uuid = UUID.randomUUID().toString(); + + KeySpace keySpace = new KeySpace( + new KeySpaceDirectory("root1", KeyType.STRING, root1Uuid) + .addSubdirectory(new KeySpaceDirectory("data", KeyType.LONG)), + new KeySpaceDirectory("root2", KeyType.STRING, root2Uuid) + .addSubdirectory(new KeySpaceDirectory("data", KeyType.LONG))); + + setSingleKey(keySpace.path("root1").add("data", 1L), Tuple.from("record"), Tuple.from("other")); + + // Now try to import that into root2 + assertBadImport(keySpace.path("root1"), keySpace.path("root2")); + } + + @Test + void importDataWithInvalidPath() { + // Test importing data with paths that don't exist in the keyspace + // Should throw RecordCoreIllegalImportDataException + final String root1Uuid = UUID.randomUUID().toString(); + final String root2Uuid = UUID.randomUUID().toString(); + + KeySpace keySpace1 = new KeySpace( + new KeySpaceDirectory("root1", KeyType.STRING, root1Uuid) + .addSubdirectory(new KeySpaceDirectory("data", KeyType.LONG))); + + KeySpace keySpace2 = new KeySpace( + new KeySpaceDirectory("root2", KeyType.STRING, root2Uuid) + .addSubdirectory(new KeySpaceDirectory("data", KeyType.LONG))); + + setSingleKey(keySpace1.path("root1").add("data", 1L), Tuple.from("record"), Tuple.from("other")); + + // Now try to import that into root2 + assertBadImport(keySpace1.path("root1"), keySpace2.path("root2")); + } + + @Test + void importDataWithSubdirectoryPath() { + // Test importing data where the target path is a subdirectory of the import path + // Should succeed only if all the data is in the subdirectory + final String rootUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new KeySpaceDirectory("root", KeyType.STRING, rootUuid) + .addSubdirectory(new KeySpaceDirectory("level1", KeyType.LONG))); + + KeySpacePath level1Path = root.path("root").add("level1", 1L); + + setSingleKey(level1Path, Tuple.from("item1"), Tuple.from("value1")); + + // Export from root, import to subdirectory + copyData(root.path("root"), level1Path); + + verifySingleKey(level1Path, Tuple.from("item1"), Tuple.from("value1")); + } + + @Test + void importDataWithSubdirectoryPathFailure() { + // Test importing data where the target path is a subdirectory of the import path + // Should succeed only if all the data is in the subdirectory + final String rootUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new KeySpaceDirectory("root", KeyType.STRING, rootUuid) + .addSubdirectory(new KeySpaceDirectory("level1", KeyType.LONG))); + + KeySpacePath level1Path = root.path("root").add("level1", 1L); + + setSingleKey(level1Path, Tuple.from("item1"), Tuple.from("value1")); + setSingleKey(root.path("root").add("level1", 2L), Tuple.from("item1"), Tuple.from("value1")); + + // Export from root, import to subdirectory + assertBadImport(root.path("root"), level1Path); + } + + @Test + void importDataWithPartialMismatch() { + // Test importing data where the target path is a parent of some import data paths + // Should throw RecordCoreIllegalImportDataException for paths outside the target + final String root1Uuid = UUID.randomUUID().toString(); + final String root2Uuid = UUID.randomUUID().toString(); + + KeySpace keySpace = new KeySpace( + new KeySpaceDirectory("root1", KeyType.STRING, root1Uuid) + .addSubdirectory(new KeySpaceDirectory("child", KeyType.LONG)), + new KeySpaceDirectory("root2", KeyType.STRING, root2Uuid) + .addSubdirectory(new KeySpaceDirectory("child", KeyType.LONG))); + + KeySpacePath path1 = keySpace.path("root1").add("child", 1L); + KeySpacePath path2 = keySpace.path("root2").add("child", 2L); + setSingleKey(path1, Tuple.from("data"), Tuple.from("data1")); + setSingleKey(path2, Tuple.from("data"), Tuple.from("data2")); + + List mixedData = new ArrayList<>(); + mixedData.addAll(getExportedData(path1)); + mixedData.addAll(getExportedData(path2)); + + assertBadImport(keySpace.path("root1"), mixedData); + } + + @Test + void importDataWithWrapperClasses() { + // Test importing data using wrapper classes like EnvironmentKeySpace + // Should verify that wrapper functionality works correctly with import + final EnvironmentKeySpace keySpace = EnvironmentKeySpace.setupSampleData(database); + + EnvironmentKeySpace.DataPath dataStore = keySpace.root().userid(100L).application("app1").dataStore(); + + copyData(keySpace.root(), keySpace.root()); + + verifySingleKey(dataStore, Tuple.from("record2", 0), Tuple.from("user100_app1_data2_0")); + } + + @Test + void importDataWithDuplicateKeys() { + // Test importing data where the same key appears multiple times in the input + // Should verify that the last value wins for duplicate keys + final String rootUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new KeySpaceDirectory("root", KeyType.STRING, rootUuid) + .addSubdirectory(new KeySpaceDirectory("data", KeyType.LONG))); + + + KeySpacePath dataPath = root.path("root").add("data", 1L); + + // Create multiple DataInKeySpacePath objects with same key but different values + List duplicateData = Arrays.asList( + new DataInKeySpacePath(dataPath, Tuple.from("item"), Tuple.from("first_value").pack()), + new DataInKeySpacePath(dataPath, Tuple.from("item"), Tuple.from("second_value").pack()), + new DataInKeySpacePath(dataPath, Tuple.from("item"), Tuple.from("final_value").pack()) + ); + + try (FDBRecordContext context = database.openContext()) { + root.path("root").importData(context, duplicateData).join(); + context.commit(); + } + + verifySingleKey(dataPath, Tuple.from("item"), Tuple.from("final_value")); + } + + private void setSingleKey(KeySpacePath path, Tuple remainder, Tuple value) { + try (FDBRecordContext context = database.openContext()) { + byte[] key = path.toSubspace(context).pack(remainder); + context.ensureActive().set(key, value.pack()); + context.commit(); + } + } + + private void verifySingleKey(KeySpacePath path, Tuple remainder, Tuple expected) { + try (FDBRecordContext context = database.openContext()) { + byte[] key = path.toSubspace(context).pack(remainder); + assertEquals(expected, Tuple.fromBytes(context.ensureActive().get(key).join())); + } + } + + private static void setInPath(final KeySpacePath path, final FDBRecordContext context, + final Tuple remainder, final Object value) { + byte[] key = path.toSubspace(context).pack(remainder); + context.ensureActive().set(key, Tuple.from(value).pack()); + } + + private static Tuple getTupleFromPath(final FDBRecordContext context, final KeySpacePath path, final Tuple remainder) { + byte[] key = path.toSubspace(context).pack(remainder); + return Tuple.fromBytes(context.ensureActive().get(key).join()); + } + + private void copyData(final KeySpacePath sourcePath, KeySpacePath destinationPath) { + // Export the data + final List exportedData = getExportedData(sourcePath); + + if (databases.size() > 1) { + database = databases.get(1); + } else { + // Clear the data and import it back + clearPath(database, sourcePath); + } + + // Import the data + importData(database, destinationPath, exportedData); + } + + private static void importData(final FDBDatabase database, final KeySpacePath path, final List exportedData) { + try (FDBRecordContext context = database.openContext()) { + path.importData(context, exportedData).join(); + context.commit(); + } + } + + private void assertBadImport(KeySpacePath sourcePath, KeySpacePath destinationPath) { + List exportedData = getExportedData(sourcePath); + assertBadImport(destinationPath, exportedData); + } + + private void assertBadImport(final KeySpacePath path, final List invalidData) { + try (FDBRecordContext context = database.openContext()) { + Assertions.assertThatThrownBy(() -> path.importData(context, invalidData).join()) + .isInstanceOf(CompletionException.class) + .hasCauseInstanceOf(RecordCoreIllegalImportDataException.class); + } + } + + private void clearPath(final FDBDatabase database, final KeySpacePath path) { + try (FDBRecordContext context = database.openContext()) { + Transaction tr = context.ensureActive(); + tr.clear(path.toSubspace(context).range()); + context.commit(); + } + // just an extra check to make sure the test is working as expected + assertTrue(getExportedData(path).isEmpty(), + "Clearing should remove all the data"); + } + + @Nonnull + private List getExportedData(final KeySpacePath path) { + try (FDBRecordContext context = database.openContext()) { + return path.exportAllData(context, null, ScanProperties.FORWARD_SCAN) + .asList().join(); + } + } +} diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/test/FDBDatabaseExtension.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/test/FDBDatabaseExtension.java index a1309989f7..15fe3234a2 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/test/FDBDatabaseExtension.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/test/FDBDatabaseExtension.java @@ -38,8 +38,12 @@ import javax.annotation.Nullable; import java.util.HashMap; import java.util.Map; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.Objects; import java.util.concurrent.Executor; +import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -161,6 +165,18 @@ public FDBDatabase getDatabase(@Nullable String clusterFile) { }); } + /** + * Return a random subset of the databases available. + * @param count the number of desired databases + * @return a random subset of the databases available. This may be less than {@code count} if there aren't that many + * databases available. + */ + public List getDatabases(int count) { + List clusterFiles = new ArrayList<>(FDBTestEnvironment.allClusterFiles()); + Collections.shuffle(clusterFiles); + return clusterFiles.stream().limit(count).map(this::getDatabase).collect(Collectors.toList()); + } + public void checkForOpenContexts() { for (final Map.Entry clusterFileToDatabase : databases.entrySet()) { assertEquals(0, clusterFileToDatabase.getValue().warnAndCloseOldTrackedOpenContexts(0),