From 178cd61a23dd95941019b7f7a67f94b1b09385d9 Mon Sep 17 00:00:00 2001 From: Scott Dugas Date: Fri, 5 Sep 2025 14:30:25 -0400 Subject: [PATCH 01/21] Initial pass at KeySpacePath.importData --- .../foundationdb/keyspace/KeySpacePath.java | 17 + .../keyspace/KeySpacePathImpl.java | 37 + .../keyspace/KeySpacePathWrapper.java | 7 + .../RecordCoreIllegalImportDataException.java | 33 + .../keyspace/KeySpacePathImportDataTest.java | 795 ++++++++++++++++++ 5 files changed, 889 insertions(+) create mode 100644 fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/RecordCoreIllegalImportDataException.java create mode 100644 fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePath.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePath.java index ff228ef1be..562c3e03e4 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePath.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePath.java @@ -410,4 +410,21 @@ default RecordCursor exportAllData(@Nonnull FDBRecordContext @Nonnull ScanProperties scanProperties) { throw new UnsupportedOperationException("exportAllData is not supported"); } + + /** + * Imports the provided data exported via {@link #exportAllData} into this {@code KeySpacePath}. + * This will validate that any data provided in {@code dataToImport} has a path that should be in this path, + * or one of the sub-directories, if not the future will complete exceptionally with + * {@link RecordCoreIllegalImportDataException}. + * If there is any data already existing under this path, the new data will overwrite if the keys are the same. + * This will use the logical value in the {@link DataInKeySpacePath#getResolvedPath()} to determine the key, rather + * than the raw key, meaning that this will work even if the data was exported from a different cluster. + * @param context the transaction context in which to save the data + * @param dataToImport the data to be saved to the database + * @return a future to be completed once all data has been important. + */ + @API(API.Status.EXPERIMENTAL) + @Nonnull + CompletableFuture importData(@Nonnull FDBRecordContext context, + @Nonnull Iterable dataToImport); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImpl.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImpl.java index 794672fd04..d5e36d8f4e 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImpl.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImpl.java @@ -316,6 +316,43 @@ public RecordCursor exportAllData(@Nonnull FDBRecordContext 1); } + @Nonnull + @Override + public CompletableFuture importData(@Nonnull FDBRecordContext context, + @Nonnull Iterable dataToImport) { + return toTupleAsync(context).thenCompose(targetTuple -> { + List> importFutures = new ArrayList<>(); + + for (DataInKeySpacePath dataItem : dataToImport) { + CompletableFuture importFuture = dataItem.getResolvedPath().thenCompose(resolvedPath -> { + // Validate that this data belongs under this path + Tuple itemTuple = resolvedPath.toTuple(); + if (!TupleHelpers.isPrefix(targetTuple, itemTuple)) { + throw new RecordCoreIllegalImportDataException( + "Data item path does not belong under target path", + "target", targetTuple, "item", itemTuple); + } + + // Reconstruct the key using logical values from the resolved path + Tuple keyTuple = itemTuple; + if (resolvedPath.getRemainder() != null) { + keyTuple = keyTuple.addAll(resolvedPath.getRemainder()); + } + + // Store the data + byte[] keyBytes = keyTuple.pack(); + byte[] valueBytes = dataItem.getRawKeyValue().getValue(); + context.ensureActive().set(keyBytes, valueBytes); + + return AsyncUtil.DONE; + }); + importFutures.add(importFuture); + } + + return AsyncUtil.whenAll(importFutures); + }); + } + /** * Returns this path properly wrapped in whatever implementation the directory the path is contained in dictates. */ diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathWrapper.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathWrapper.java index 1cec9787ea..baa020ddf3 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathWrapper.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathWrapper.java @@ -202,4 +202,11 @@ public RecordCursor exportAllData(@Nonnull FDBRecordContext @Nonnull ScanProperties scanProperties) { return inner.exportAllData(context, continuation, scanProperties); } + + @Nonnull + @Override + public CompletableFuture importData(@Nonnull FDBRecordContext context, + @Nonnull Iterable dataToImport) { + return inner.importData(context, dataToImport); + } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/RecordCoreIllegalImportDataException.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/RecordCoreIllegalImportDataException.java new file mode 100644 index 0000000000..250e0bcbce --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/RecordCoreIllegalImportDataException.java @@ -0,0 +1,33 @@ +/* + * RecordCoreIllegalImportDataException.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.provider.foundationdb.keyspace; + +import com.apple.foundationdb.record.RecordCoreArgumentException; + +import javax.annotation.Nonnull; + +public class RecordCoreIllegalImportDataException extends RecordCoreArgumentException { + private static final long serialVersionUID = 1L; + + public RecordCoreIllegalImportDataException(@Nonnull final String msg, @Nonnull final Object... keyValue) { + super(msg, keyValue); + } +} diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java new file mode 100644 index 0000000000..cf54dca193 --- /dev/null +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java @@ -0,0 +1,795 @@ +/* + * KeySpacePathImportDataTest.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.provider.foundationdb.keyspace; + +import com.apple.foundationdb.KeyValue; +import com.apple.foundationdb.Transaction; +import com.apple.foundationdb.record.RecordCoreArgumentException; +import com.apple.foundationdb.record.ScanProperties; +import com.apple.foundationdb.record.provider.foundationdb.FDBDatabase; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext; +import com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpaceDirectory.KeyType; +import com.apple.foundationdb.record.test.FDBDatabaseExtension; +import com.apple.foundationdb.subspace.Subspace; +import com.apple.foundationdb.tuple.Tuple; +import com.apple.test.Tags; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import javax.annotation.Nonnull; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests for {@link KeySpacePath#importData(FDBRecordContext, Iterable)}. + */ +@Tag(Tags.RequiresFDB) +class KeySpacePathImportDataTest { + @RegisterExtension + final FDBDatabaseExtension dbExtension = new FDBDatabaseExtension(); + private FDBDatabase database; + + @BeforeEach + void setUp() { + database = dbExtension.getDatabase(); + } + + @Test + void importBasicData() { + // Test importing basic data into a simple keyspace path + // Should validate that data is correctly stored using logical values + // rather than raw keys, allowing import across different clusters + final String companyUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new KeySpaceDirectory("company", KeyType.STRING, companyUuid) + .addSubdirectory(new KeySpaceDirectory("department", KeyType.STRING) + .addSubdirectory(new KeySpaceDirectory("employee_id", KeyType.LONG)))); + + try (FDBRecordContext context = database.openContext()) { + Transaction tr = context.ensureActive(); + + // Create some test data to export and then import + KeySpacePath deptPath = root.path("company").add("department", "engineering"); + KeySpacePath emp1Path = deptPath.add("employee_id", 100L); + KeySpacePath emp2Path = deptPath.add("employee_id", 200L); + + // Store original data with remainder elements + byte[] key1 = emp1Path.toSubspace(context).pack(Tuple.from("profile", "name")); + byte[] value1 = Tuple.from("John Doe").pack(); + byte[] key2 = emp2Path.toSubspace(context).pack(Tuple.from("profile", "name")); + byte[] value2 = Tuple.from("Jane Smith").pack(); + + tr.set(key1, value1); + tr.set(key2, value2); + context.commit(); + } + + // Export the data + final List exportedData = getExportedData(database, root.path("company").add("department", "engineering")); + + // Clear the data and import it back + clearPath(database, root.path("company")); + + // Import the data + importData(database, root.path("company").add("department", "engineering"), exportedData); + + // Verify the data was imported correctly + try (FDBRecordContext context = database.openContext()) { + KeySpacePath emp1Path = root.path("company").add("department", "engineering").add("employee_id", 100L); + KeySpacePath emp2Path = root.path("company").add("department", "engineering").add("employee_id", 200L); + + byte[] key1 = emp1Path.toSubspace(context).pack(Tuple.from("profile", "name")); + byte[] key2 = emp2Path.toSubspace(context).pack(Tuple.from("profile", "name")); + + byte[] value1 = getJoin(context, key1); + byte[] value2 = getJoin(context, key2); + + assertEquals("John Doe", Tuple.fromBytes(value1).getString(0)); + assertEquals("Jane Smith", Tuple.fromBytes(value2).getString(0)); + } + } + + @Test + void importEmptyData() { + // Test importing an empty collection of data + // Should complete successfully without modifying the data under the path + KeySpace root = new KeySpace( + new KeySpaceDirectory("test", KeyType.STRING, UUID.randomUUID().toString())); + + try (FDBRecordContext context = database.openContext()) { + KeySpacePath testPath = root.path("test"); + + // Import empty data - should complete successfully + CompletableFuture importFuture = testPath.importData(context, Collections.emptyList()); + importFuture.join(); // Should not throw any exception + + context.commit(); + } + + try (FDBRecordContext context = database.openContext()) { + KeySpacePath testPath = root.path("test"); + assertTrue( + testPath.exportAllData(context, null, ScanProperties.FORWARD_SCAN).first().join().isEmpty(), + "There should not have been any data created"); + } + + } + + @Test + void importOverwriteExistingData() { + // Test importing data that overwrites existing keys + // Should verify that new data replaces old data when keys match + final String rootUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new KeySpaceDirectory("root", KeyType.STRING, rootUuid) + .addSubdirectory(new KeySpaceDirectory("data", KeyType.LONG))); + + + try (FDBRecordContext context = database.openContext()) { + Transaction tr = context.ensureActive(); + + // Set initial data + KeySpacePath dataPath = root.path("root").add("data", 1L); + final Subspace subspace = dataPath.toSubspace(context); + byte[] key = subspace.pack(Tuple.from("record")); + tr.set(key, Tuple.from("original_value").pack()); + tr.set(subspace.pack("other"), Tuple.from("other_value").pack()); + context.commit(); + } + + // Create import data with same key but different value + List importData = new ArrayList<>(); + try (FDBRecordContext context = database.openContext()) { + KeySpacePath dataPath = root.path("root").add("data", 1L); + byte[] key = dataPath.toSubspace(context).pack(Tuple.from("record")); + KeyValue kv = new KeyValue(key, Tuple.from("new_value").pack()); + importData.add(new DataInKeySpacePath(root.path("root"), kv, context)); + } + + // Verify we can re-import the data multiple times + importData(database, root.path("root"), importData); + importData(database, root.path("root"), importData); + importData(database, root.path("root"), importData); + + // Verify the data was overwritten + try (FDBRecordContext context = database.openContext()) { + KeySpacePath dataPath = root.path("root").add("data", 1L); + final Subspace subspace = dataPath.toSubspace(context); + byte[] key = subspace.pack(Tuple.from("record")); + byte[] value = getJoin(context, key); + assertEquals(Tuple.from("new_value"), Tuple.fromBytes(value)); + assertEquals(Tuple.from("other_value"), Tuple.fromBytes(getJoin(context, subspace.pack("other")))); + } + } + + private static byte[] getJoin(final FDBRecordContext context, final byte[] key) { + return context.ensureActive().get(key).join(); + } + + @Test + void importDataWithConstantValues() { + // Test importing data into a keyspace with constant directory values + // Should verify that constant values are handled correctly during import + final String appUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new KeySpaceDirectory("application", KeyType.STRING, appUuid) + .addSubdirectory(new KeySpaceDirectory("version", KeyType.LONG, 1L) + .addSubdirectory(new KeySpaceDirectory("data", KeyType.STRING)))); + + + try (FDBRecordContext context = database.openContext()) { + Transaction tr = context.ensureActive(); + + // Create test data using constant values + KeySpacePath dataPath = root.path("application").add("version").add("data", "config"); + byte[] key = dataPath.toSubspace(context).pack(Tuple.from("setting")); + tr.set(key, Tuple.from("value1").pack()); + context.commit(); + } + + // Export and import the data + final List exportedData = getExportedData(database, root.path("application")); + + // Clear and import + clearPath(database, root.path("application")); + + importData(database, root.path("application"), exportedData); + + // Verify the data + try (FDBRecordContext context = database.openContext()) { + KeySpacePath dataPath = root.path("application").add("version").add("data", "config"); + byte[] key = dataPath.toSubspace(context).pack(Tuple.from("setting")); + byte[] value = getJoin(context, key); + assertEquals("value1", Tuple.fromBytes(value).getString(0)); + } + } + + private static void clearPath(final FDBDatabase database, final KeySpacePath root) { + try (FDBRecordContext context = database.openContext()) { + Transaction tr = context.ensureActive(); + tr.clear(root.toSubspace(context).range()); + context.commit(); + } + } + + @Test + void importDataWithDirectoryLayer() { + // Test importing data into a keyspace using DirectoryLayer directories + // Should verify that DirectoryLayer mappings work correctly during import + final String tenantUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new DirectoryLayerDirectory("tenant", tenantUuid) + .addSubdirectory(new KeySpaceDirectory("user_id", KeyType.LONG))); + + + try (FDBRecordContext context = database.openContext()) { + Transaction tr = context.ensureActive(); + + KeySpacePath userPath = root.path("tenant").add("user_id", 999L); + byte[] key = userPath.toSubspace(context).pack(Tuple.from("data")); + tr.set(key, Tuple.from("directory_test").pack()); + context.commit(); + } + + // Export and import + List exportedData = new ArrayList<>(); + try (FDBRecordContext context = database.openContext()) { + root.path("tenant").exportAllData(context, null, ScanProperties.FORWARD_SCAN) + .forEach(exportedData::add).join(); + } + + clearPath(database, root.path("tenant")); + + try (FDBRecordContext context = database.openContext()) { + root.path("tenant").importData(context, exportedData).join(); + context.commit(); + } + + // Verify + try (FDBRecordContext context = database.openContext()) { + KeySpacePath userPath = root.path("tenant").add("user_id", 999L); + byte[] key = userPath.toSubspace(context).pack(Tuple.from("data")); + byte[] value = getJoin(context, key); + assertEquals("directory_test", Tuple.fromBytes(value).getString(0)); + } + } + + @Test + void importDataWithBinaryValues() { + // Test importing data containing binary (byte array) values + // Should verify that binary data is imported correctly + final String storeUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new KeySpaceDirectory("binary_store", KeyType.STRING, storeUuid) + .addSubdirectory(new KeySpaceDirectory("blob_id", KeyType.BYTES))); + + + byte[] blobId = {0x01, 0x02, 0x03, (byte) 0xFF, (byte) 0xFE}; + + try (FDBRecordContext context = database.openContext()) { + Transaction tr = context.ensureActive(); + + KeySpacePath blobPath = root.path("binary_store").add("blob_id", blobId); + byte[] key = blobPath.toSubspace(context).pack(Tuple.from("metadata")); + tr.set(key, "binary_test_data".getBytes()); + context.commit(); + } + + // Export and import + List exportedData = new ArrayList<>(); + try (FDBRecordContext context = database.openContext()) { + root.path("binary_store").exportAllData(context, null, ScanProperties.FORWARD_SCAN) + .forEach(exportedData::add).join(); + } + + clearPath(database, root.path("binary_store")); + + try (FDBRecordContext context = database.openContext()) { + root.path("binary_store").importData(context, exportedData).join(); + context.commit(); + } + + // Verify + try (FDBRecordContext context = database.openContext()) { + KeySpacePath blobPath = root.path("binary_store").add("blob_id", blobId); + byte[] key = blobPath.toSubspace(context).pack(Tuple.from("metadata")); + byte[] value = getJoin(context, key); + assertArrayEquals("binary_test_data".getBytes(), value); + } + } + + @Test + void importDataWithNullKeyType() { + // Test importing data into a keyspace with NULL key type directories + // Should verify that NULL directories are handled correctly + final String baseUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new KeySpaceDirectory("base", KeyType.STRING, baseUuid) + .addSubdirectory(new KeySpaceDirectory("null_dir", KeyType.NULL))); + + + try (FDBRecordContext context = database.openContext()) { + Transaction tr = context.ensureActive(); + + KeySpacePath nullPath = root.path("base").add("null_dir"); + byte[] key = nullPath.toSubspace(context).pack(Tuple.from("item")); + tr.set(key, Tuple.from("null_test").pack()); + context.commit(); + } + + // Export and import + List exportedData = new ArrayList<>(); + try (FDBRecordContext context = database.openContext()) { + root.path("base").exportAllData(context, null, ScanProperties.FORWARD_SCAN) + .forEach(exportedData::add).join(); + } + + clearPath(database, root.path("base")); + + try (FDBRecordContext context = database.openContext()) { + root.path("base").importData(context, exportedData).join(); + context.commit(); + } + + // Verify + try (FDBRecordContext context = database.openContext()) { + KeySpacePath nullPath = root.path("base").add("null_dir"); + byte[] key = nullPath.toSubspace(context).pack(Tuple.from("item")); + byte[] value = getJoin(context, key); + assertEquals("null_test", Tuple.fromBytes(value).getString(0)); + } + } + + @Test + void importDataWithComplexHierarchy() { + // Test importing data into a multi-level directory hierarchy + // Should verify that nested paths are reconstructed correctly + final String rootUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new KeySpaceDirectory("org", KeyType.STRING, rootUuid) + .addSubdirectory(new KeySpaceDirectory("dept", KeyType.STRING) + .addSubdirectory(new KeySpaceDirectory("team", KeyType.LONG) + .addSubdirectory(new KeySpaceDirectory("member", KeyType.UUID))))); + + UUID memberId = UUID.randomUUID(); + + try (FDBRecordContext context = database.openContext()) { + Transaction tr = context.ensureActive(); + + KeySpacePath memberPath = root.path("org").add("dept", "engineering") + .add("team", 42L).add("member", memberId); + byte[] key = memberPath.toSubspace(context).pack(Tuple.from("info", "name")); + tr.set(key, Tuple.from("Complex Test").pack()); + context.commit(); + } + + // Export and import + List exportedData = new ArrayList<>(); + try (FDBRecordContext context = database.openContext()) { + root.path("org").exportAllData(context, null, ScanProperties.FORWARD_SCAN) + .forEach(exportedData::add).join(); + } + + clearPath(database, root.path("org")); + + try (FDBRecordContext context = database.openContext()) { + root.path("org").importData(context, exportedData).join(); + context.commit(); + } + + // Verify + try (FDBRecordContext context = database.openContext()) { + KeySpacePath memberPath = root.path("org").add("dept", "engineering") + .add("team", 42L).add("member", memberId); + byte[] key = memberPath.toSubspace(context).pack(Tuple.from("info", "name")); + byte[] value = getJoin(context, key); + assertEquals("Complex Test", Tuple.fromBytes(value).getString(0)); + } + } + + @Test + void importDataFromDifferentCluster() { + // Test importing data that was exported from a different cluster + // Should verify that logical values allow cross-cluster import + // This simulates the scenario where DirectoryLayer mappings differ + final String tenantUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new DirectoryLayerDirectory("tenant", tenantUuid) + .addSubdirectory(new KeySpaceDirectory("user_id", KeyType.LONG))); + + + // Create and export data in one "cluster" context + List exportedData = new ArrayList<>(); + try (FDBRecordContext context1 = database.openContext()) { + Transaction tr = context1.ensureActive(); + + KeySpacePath userPath = root.path("tenant").add("user_id", 123L); + byte[] key = userPath.toSubspace(context1).pack(Tuple.from("profile")); + tr.set(key, Tuple.from("cross_cluster_test").pack()); + + root.path("tenant").exportAllData(context1, null, ScanProperties.FORWARD_SCAN) + .forEach(exportedData::add).join(); + context1.commit(); + } + + // Clear the data to simulate different cluster + clearPath(database, root.path("tenant")); + + // Import in a different "cluster" context + try (FDBRecordContext context2 = database.openContext()) { + root.path("tenant").importData(context2, exportedData).join(); + context2.commit(); + } + + // Verify the data was imported using logical values + try (FDBRecordContext context = database.openContext()) { + KeySpacePath userPath = root.path("tenant").add("user_id", 123L); + byte[] key = userPath.toSubspace(context).pack(Tuple.from("profile")); + byte[] value = getJoin(context, key); + assertEquals("cross_cluster_test", Tuple.fromBytes(value).getString(0)); + } + } + + @Test + void importDataWithMismatchedPath() { + // Test importing data that doesn't belong to the target path + // Should throw RecordCoreIllegalImportDataException + final String root1Uuid = UUID.randomUUID().toString(); + final String root2Uuid = UUID.randomUUID().toString(); + + KeySpace keySpace1 = new KeySpace( + new KeySpaceDirectory("root1", KeyType.STRING, root1Uuid) + .addSubdirectory(new KeySpaceDirectory("data", KeyType.LONG))); + + KeySpace keySpace2 = new KeySpace( + new KeySpaceDirectory("root2", KeyType.STRING, root2Uuid) + .addSubdirectory(new KeySpaceDirectory("data", KeyType.LONG))); + + + // Create data in keySpace2 + List exportedData = new ArrayList<>(); + try (FDBRecordContext context = database.openContext()) { + Transaction tr = context.ensureActive(); + KeySpacePath dataPath = keySpace2.path("root2").add("data", 1L); + byte[] key = dataPath.toSubspace(context).pack(Tuple.from("record")); + tr.set(key, Tuple.from("value").pack()); + + KeyValue kv = new KeyValue(key, Tuple.from("value").pack()); + exportedData.add(new DataInKeySpacePath(keySpace2.path("root2"), kv, context)); + context.commit(); + } + + // Try to import into keySpace1 - should fail + try (FDBRecordContext context = database.openContext()) { + KeySpacePath root1Path = keySpace1.path("root1"); + + CompletionException completionException = assertThrows(CompletionException.class, () -> + root1Path.importData(context, exportedData).join() + ); + assertTrue(completionException.getCause() instanceof RecordCoreIllegalImportDataException); + } + } + + @Test + void importDataWithInvalidPath() { + // Test importing data with paths that don't exist in the keyspace + // Should throw RecordCoreIllegalImportDataException + final String rootUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new KeySpaceDirectory("root", KeyType.STRING, rootUuid) + .addSubdirectory(new KeySpaceDirectory("valid", KeyType.LONG))); + + + try (FDBRecordContext context = database.openContext()) { + // Create a key that doesn't follow the keyspace structure + byte[] invalidKey = Tuple.from("completely", "different", "structure").pack(); + KeyValue kv = new KeyValue(invalidKey, Tuple.from("value").pack()); + + List invalidData = Collections.singletonList( + new DataInKeySpacePath(root.path("root"), kv, context) + ); + + CompletionException completionException = assertThrows(CompletionException.class, () -> + root.path("root").importData(context, invalidData).join() + ); + // The cause might be RecordCoreIllegalImportDataException or RecordCoreArgumentException for invalid paths + assertTrue(completionException.getCause() instanceof RecordCoreArgumentException); + } + } + + @Test + void importDataWithSubdirectoryPath() { + // Test importing data where the target path is a subdirectory of the import path + // Should succeed and import only data that belongs to subdirectories + final String rootUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new KeySpaceDirectory("root", KeyType.STRING, rootUuid) + .addSubdirectory(new KeySpaceDirectory("level1", KeyType.LONG) + .addSubdirectory(new KeySpaceDirectory("level2", KeyType.STRING)))); + + + try (FDBRecordContext context = database.openContext()) { + Transaction tr = context.ensureActive(); + + // Create data at different levels + KeySpacePath level1Path = root.path("root").add("level1", 1L); + KeySpacePath level2Path = level1Path.add("level2", "data"); + + byte[] key1 = level1Path.toSubspace(context).pack(Tuple.from("item1")); + byte[] key2 = level2Path.toSubspace(context).pack(Tuple.from("item2")); + + tr.set(key1, Tuple.from("value1").pack()); + tr.set(key2, Tuple.from("value2").pack()); + context.commit(); + } + + // Export from root, import to subdirectory + List exportedData = new ArrayList<>(); + try (FDBRecordContext context = database.openContext()) { + root.path("root").exportAllData(context, null, ScanProperties.FORWARD_SCAN) + .forEach(exportedData::add).join(); + } + + clearPath(database, root.path("root")); + + // Import only to level1 subdirectory + importData(database, root.path("root").add("level1", 1L), exportedData); + + // Verify only level1 and below data exists + try (FDBRecordContext context = database.openContext()) { + KeySpacePath level2Path = root.path("root").add("level1", 1L).add("level2", "data"); + byte[] key2 = level2Path.toSubspace(context).pack(Tuple.from("item2")); + byte[] value2 = getJoin(context, key2); + assertEquals("value2", Tuple.fromBytes(value2).getString(0)); + } + } + + @Test + void importDataWithParentPath() { + // Test importing data where the target path is a parent of some import data paths + // Should throw RecordCoreIllegalImportDataException for paths outside the target + final String root1Uuid = UUID.randomUUID().toString(); + final String root2Uuid = UUID.randomUUID().toString(); + + KeySpace keySpace1 = new KeySpace( + new KeySpaceDirectory("root1", KeyType.STRING, root1Uuid) + .addSubdirectory(new KeySpaceDirectory("child", KeyType.LONG))); + + KeySpace keySpace2 = new KeySpace( + new KeySpaceDirectory("root2", KeyType.STRING, root2Uuid) + .addSubdirectory(new KeySpaceDirectory("child", KeyType.LONG))); + + + // Create data in both keyspaces + List mixedData = new ArrayList<>(); + try (FDBRecordContext context = database.openContext()) { + Transaction tr = context.ensureActive(); + + KeySpacePath path1 = keySpace1.path("root1").add("child", 1L); + KeySpacePath path2 = keySpace2.path("root2").add("child", 2L); + + byte[] key1 = path1.toSubspace(context).pack(Tuple.from("data")); + byte[] key2 = path2.toSubspace(context).pack(Tuple.from("data")); + + tr.set(key1, Tuple.from("data1").pack()); + tr.set(key2, Tuple.from("data2").pack()); + + KeyValue kv1 = new KeyValue(key1, Tuple.from("data1").pack()); + KeyValue kv2 = new KeyValue(key2, Tuple.from("data2").pack()); + + mixedData.add(new DataInKeySpacePath(keySpace1.path("root1"), kv1, context)); + mixedData.add(new DataInKeySpacePath(keySpace2.path("root2"), kv2, context)); + + context.commit(); + } + + // Try to import mixed data into keySpace1 - should fail due to keySpace2 data + try (FDBRecordContext context = database.openContext()) { + CompletionException completionException = assertThrows(CompletionException.class, () -> + keySpace1.path("root1").importData(context, mixedData).join() + ); + assertTrue(completionException.getCause() instanceof RecordCoreIllegalImportDataException); + } + } + + @Test + void importDataWithWrapperClasses() { + // Test importing data using wrapper classes like EnvironmentKeySpace + // Should verify that wrapper functionality works correctly with import + final EnvironmentKeySpace keySpace = EnvironmentKeySpace.setupSampleData(database); + + // Export some data + List exportedData = new ArrayList<>(); + try (FDBRecordContext context = database.openContext()) { + EnvironmentKeySpace.DataPath dataStore = keySpace.root().userid(100L).application("app1").dataStore(); + dataStore.exportAllData(context, null, ScanProperties.FORWARD_SCAN) + .forEach(exportedData::add).join(); + } + + // Clear and import back + clearPath(database, keySpace.root()); + + try (FDBRecordContext context = database.openContext()) { + keySpace.root().importData(context, exportedData).join(); + context.commit(); + } + + // Verify data exists + try (FDBRecordContext context = database.openContext()) { + EnvironmentKeySpace.DataPath dataStore = keySpace.root().userid(100L).application("app1").dataStore(); + byte[] key = dataStore.toSubspace(context).pack(Tuple.from("record2", 0)); + byte[] value = getJoin(context, key); + assertTrue(value.length > 0); + } + } + + @Test + void importDataWithDuplicateKeys() { + // Test importing data where the same key appears multiple times in the input + // Should verify that the last value wins for duplicate keys + final String rootUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new KeySpaceDirectory("root", KeyType.STRING, rootUuid) + .addSubdirectory(new KeySpaceDirectory("data", KeyType.LONG))); + + + try (FDBRecordContext context = database.openContext()) { + KeySpacePath dataPath = root.path("root").add("data", 1L); + byte[] key = dataPath.toSubspace(context).pack(Tuple.from("item")); + + // Create multiple DataInKeySpacePath objects with same key but different values + List duplicateData = Arrays.asList( + new DataInKeySpacePath(root.path("root"), + new KeyValue(key, Tuple.from("first_value").pack()), context), + new DataInKeySpacePath(root.path("root"), + new KeyValue(key, Tuple.from("second_value").pack()), context), + new DataInKeySpacePath(root.path("root"), + new KeyValue(key, Tuple.from("final_value").pack()), context) + ); + + root.path("root").importData(context, duplicateData).join(); + context.commit(); + } + + // Verify the final value is stored + try (FDBRecordContext context = database.openContext()) { + KeySpacePath dataPath = root.path("root").add("data", 1L); + byte[] key = dataPath.toSubspace(context).pack(Tuple.from("item")); + byte[] value = getJoin(context, key); + assertEquals("final_value", Tuple.fromBytes(value).getString(0)); + } + } + + @Test + void importDataRollback() { + // Test import operation that fails and needs to be rolled back + // Should verify that partial imports are properly cleaned up + final String root1Uuid = UUID.randomUUID().toString(); + final String root2Uuid = UUID.randomUUID().toString(); + + KeySpace keySpace1 = new KeySpace( + new KeySpaceDirectory("root1", KeyType.STRING, root1Uuid) + .addSubdirectory(new KeySpaceDirectory("data", KeyType.LONG))); + + KeySpace keySpace2 = new KeySpace( + new KeySpaceDirectory("root2", KeyType.STRING, root2Uuid) + .addSubdirectory(new KeySpaceDirectory("data", KeyType.LONG))); + + + try (FDBRecordContext context = database.openContext()) { + Transaction tr = context.ensureActive(); + + // Create mixed data that will partially succeed then fail + KeySpacePath validPath = keySpace1.path("root1").add("data", 1L); + KeySpacePath invalidPath = keySpace2.path("root2").add("data", 2L); + + byte[] validKey = validPath.toSubspace(context).pack(Tuple.from("item")); + byte[] invalidKey = invalidPath.toSubspace(context).pack(Tuple.from("item")); + + List mixedData = Arrays.asList( + new DataInKeySpacePath(keySpace1.path("root1"), + new KeyValue(validKey, Tuple.from("valid").pack()), context), + new DataInKeySpacePath(keySpace2.path("root2"), + new KeyValue(invalidKey, Tuple.from("invalid").pack()), context) + ); + + // This should fail and rollback + CompletionException completionException = assertThrows(CompletionException.class, () -> + keySpace1.path("root1").importData(context, mixedData).join() + ); + assertTrue(completionException.getCause() instanceof RecordCoreIllegalImportDataException); + } + + // Verify no data was written after the failed transaction + try (FDBRecordContext context = database.openContext()) { + KeySpacePath validPath = keySpace1.path("root1").add("data", 1L); + byte[] validKey = validPath.toSubspace(context).pack(Tuple.from("item")); + byte[] value = getJoin(context, validKey); + assertTrue(value == null || value.length == 0); + } + } + + @Test + void importDataValidation() { + // Test various edge cases for data validation + // Should verify that malformed DataInKeySpacePath objects are rejected + final String rootUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new KeySpaceDirectory("root", KeyType.STRING, rootUuid) + .addSubdirectory(new KeySpaceDirectory("data", KeyType.LONG))); + + + try (FDBRecordContext context = database.openContext()) { + KeySpacePath rootPath = root.path("root"); + + // Test with empty iterable + rootPath.importData(context, Collections.emptyList()).join(); + + // Test with null remainder (should work) + KeySpacePath dataPath = rootPath.add("data", 1L); + byte[] key = dataPath.toSubspace(context).pack(); // No remainder + KeyValue kv = new KeyValue(key, Tuple.from("test").pack()); + List validData = Collections.singletonList( + new DataInKeySpacePath(rootPath, kv, context) + ); + + rootPath.importData(context, validData).join(); + context.commit(); + } + + // Verify the data was stored + try (FDBRecordContext context = database.openContext()) { + KeySpacePath dataPath = root.path("root").add("data", 1L); + byte[] key = dataPath.toSubspace(context).pack(); + byte[] value = getJoin(context, key); + assertEquals("test", Tuple.fromBytes(value).getString(0)); + } + } + + private static void importData(final FDBDatabase database, final KeySpacePath path, final List exportedData) { + try (FDBRecordContext context = database.openContext()) { + path.importData(context, exportedData).join(); + context.commit(); + } + } + + @Nonnull + private static List getExportedData(final FDBDatabase database, final KeySpacePath path) { + List exportedData = new ArrayList<>(); + try (FDBRecordContext context = database.openContext()) { + path.exportAllData(context, null, ScanProperties.FORWARD_SCAN) + .forEach(exportedData::add).join(); + } + return exportedData; + } +} From 938d2401eb80507adead7f4408bdc8ed050e4400 Mon Sep 17 00:00:00 2001 From: Scott Dugas Date: Mon, 8 Sep 2025 15:59:54 -0400 Subject: [PATCH 02/21] Cleanup some of the tests for importing data --- .../keyspace/KeySpacePathImportDataTest.java | 105 ++++++++---------- 1 file changed, 44 insertions(+), 61 deletions(-) diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java index cf54dca193..84f2b4f619 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java @@ -42,7 +42,6 @@ import java.util.Collections; import java.util.List; import java.util.UUID; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import static org.junit.jupiter.api.Assertions.assertArrayEquals; @@ -111,11 +110,11 @@ void importBasicData() { byte[] key1 = emp1Path.toSubspace(context).pack(Tuple.from("profile", "name")); byte[] key2 = emp2Path.toSubspace(context).pack(Tuple.from("profile", "name")); - byte[] value1 = getJoin(context, key1); - byte[] value2 = getJoin(context, key2); + Tuple value1 = getTuple(context, key1); + Tuple value2 = getTuple(context, key2); - assertEquals("John Doe", Tuple.fromBytes(value1).getString(0)); - assertEquals("Jane Smith", Tuple.fromBytes(value2).getString(0)); + assertEquals("John Doe", value1.getString(0)); + assertEquals("Jane Smith", value2.getString(0)); } } @@ -126,15 +125,7 @@ void importEmptyData() { KeySpace root = new KeySpace( new KeySpaceDirectory("test", KeyType.STRING, UUID.randomUUID().toString())); - try (FDBRecordContext context = database.openContext()) { - KeySpacePath testPath = root.path("test"); - - // Import empty data - should complete successfully - CompletableFuture importFuture = testPath.importData(context, Collections.emptyList()); - importFuture.join(); // Should not throw any exception - - context.commit(); - } + importData(database, root.path("test"), Collections.emptyList()); // should not throw any exception try (FDBRecordContext context = database.openContext()) { KeySpacePath testPath = root.path("test"); @@ -154,7 +145,6 @@ void importOverwriteExistingData() { new KeySpaceDirectory("root", KeyType.STRING, rootUuid) .addSubdirectory(new KeySpaceDirectory("data", KeyType.LONG))); - try (FDBRecordContext context = database.openContext()) { Transaction tr = context.ensureActive(); @@ -186,16 +176,11 @@ void importOverwriteExistingData() { KeySpacePath dataPath = root.path("root").add("data", 1L); final Subspace subspace = dataPath.toSubspace(context); byte[] key = subspace.pack(Tuple.from("record")); - byte[] value = getJoin(context, key); - assertEquals(Tuple.from("new_value"), Tuple.fromBytes(value)); - assertEquals(Tuple.from("other_value"), Tuple.fromBytes(getJoin(context, subspace.pack("other")))); + assertEquals(Tuple.from("new_value"), getTuple(context, key)); + assertEquals(Tuple.from("other_value"), getTuple(context, subspace.pack("other"))); } } - private static byte[] getJoin(final FDBRecordContext context, final byte[] key) { - return context.ensureActive().get(key).join(); - } - @Test void importDataWithConstantValues() { // Test importing data into a keyspace with constant directory values @@ -229,16 +214,8 @@ void importDataWithConstantValues() { try (FDBRecordContext context = database.openContext()) { KeySpacePath dataPath = root.path("application").add("version").add("data", "config"); byte[] key = dataPath.toSubspace(context).pack(Tuple.from("setting")); - byte[] value = getJoin(context, key); - assertEquals("value1", Tuple.fromBytes(value).getString(0)); - } - } - - private static void clearPath(final FDBDatabase database, final KeySpacePath root) { - try (FDBRecordContext context = database.openContext()) { - Transaction tr = context.ensureActive(); - tr.clear(root.toSubspace(context).range()); - context.commit(); + Tuple value = getTuple(context, key); + assertEquals("value1", value.getString(0)); } } @@ -279,8 +256,8 @@ void importDataWithDirectoryLayer() { try (FDBRecordContext context = database.openContext()) { KeySpacePath userPath = root.path("tenant").add("user_id", 999L); byte[] key = userPath.toSubspace(context).pack(Tuple.from("data")); - byte[] value = getJoin(context, key); - assertEquals("directory_test", Tuple.fromBytes(value).getString(0)); + Tuple value = getTuple(context, key); + assertEquals("directory_test", value.getString(0)); } } @@ -314,10 +291,7 @@ void importDataWithBinaryValues() { clearPath(database, root.path("binary_store")); - try (FDBRecordContext context = database.openContext()) { - root.path("binary_store").importData(context, exportedData).join(); - context.commit(); - } + importData(database, root.path("binary_store"), exportedData); // Verify try (FDBRecordContext context = database.openContext()) { @@ -365,8 +339,8 @@ void importDataWithNullKeyType() { try (FDBRecordContext context = database.openContext()) { KeySpacePath nullPath = root.path("base").add("null_dir"); byte[] key = nullPath.toSubspace(context).pack(Tuple.from("item")); - byte[] value = getJoin(context, key); - assertEquals("null_test", Tuple.fromBytes(value).getString(0)); + Tuple value = getTuple(context, key); + assertEquals("null_test", value.getString(0)); } } @@ -394,26 +368,16 @@ void importDataWithComplexHierarchy() { } // Export and import - List exportedData = new ArrayList<>(); - try (FDBRecordContext context = database.openContext()) { - root.path("org").exportAllData(context, null, ScanProperties.FORWARD_SCAN) - .forEach(exportedData::add).join(); - } - + List exportedData = getExportedData(database, root.path("org")); clearPath(database, root.path("org")); - - try (FDBRecordContext context = database.openContext()) { - root.path("org").importData(context, exportedData).join(); - context.commit(); - } + importData(database, root.path("org"), exportedData); // Verify try (FDBRecordContext context = database.openContext()) { KeySpacePath memberPath = root.path("org").add("dept", "engineering") .add("team", 42L).add("member", memberId); byte[] key = memberPath.toSubspace(context).pack(Tuple.from("info", "name")); - byte[] value = getJoin(context, key); - assertEquals("Complex Test", Tuple.fromBytes(value).getString(0)); + assertEquals(Tuple.from("Complex Test"), getTuple(context, key)); } } @@ -455,8 +419,8 @@ void importDataFromDifferentCluster() { try (FDBRecordContext context = database.openContext()) { KeySpacePath userPath = root.path("tenant").add("user_id", 123L); byte[] key = userPath.toSubspace(context).pack(Tuple.from("profile")); - byte[] value = getJoin(context, key); - assertEquals("cross_cluster_test", Tuple.fromBytes(value).getString(0)); + Tuple value = getTuple(context, key); + assertEquals("cross_cluster_test", value.getString(0)); } } @@ -569,8 +533,8 @@ void importDataWithSubdirectoryPath() { try (FDBRecordContext context = database.openContext()) { KeySpacePath level2Path = root.path("root").add("level1", 1L).add("level2", "data"); byte[] key2 = level2Path.toSubspace(context).pack(Tuple.from("item2")); - byte[] value2 = getJoin(context, key2); - assertEquals("value2", Tuple.fromBytes(value2).getString(0)); + Tuple value2 = getTuple(context, key2); + assertEquals("value2", value2.getString(0)); } } @@ -685,8 +649,8 @@ void importDataWithDuplicateKeys() { try (FDBRecordContext context = database.openContext()) { KeySpacePath dataPath = root.path("root").add("data", 1L); byte[] key = dataPath.toSubspace(context).pack(Tuple.from("item")); - byte[] value = getJoin(context, key); - assertEquals("final_value", Tuple.fromBytes(value).getString(0)); + Tuple value = getTuple(context, key); + assertEquals("final_value", value.getString(0)); } } @@ -771,11 +735,19 @@ void importDataValidation() { try (FDBRecordContext context = database.openContext()) { KeySpacePath dataPath = root.path("root").add("data", 1L); byte[] key = dataPath.toSubspace(context).pack(); - byte[] value = getJoin(context, key); - assertEquals("test", Tuple.fromBytes(value).getString(0)); + Tuple value = getTuple(context, key); + assertEquals("test", value.getString(0)); } } + private static Tuple getTuple(final FDBRecordContext context, final byte[] key) { + return Tuple.fromBytes(context.ensureActive().get(key).join()); + } + + private static byte[] getJoin(final FDBRecordContext context, final byte[] key) { + return context.ensureActive().get(key).join(); + } + private static void importData(final FDBDatabase database, final KeySpacePath path, final List exportedData) { try (FDBRecordContext context = database.openContext()) { path.importData(context, exportedData).join(); @@ -783,6 +755,17 @@ private static void importData(final FDBDatabase database, final KeySpacePath pa } } + private static void clearPath(final FDBDatabase database, final KeySpacePath path) { + try (FDBRecordContext context = database.openContext()) { + Transaction tr = context.ensureActive(); + tr.clear(path.toSubspace(context).range()); + context.commit(); + } + // just an extra check to make sure the test is working as expected + assertTrue(getExportedData(database, path).isEmpty(), + "Clearing should remove all the data"); + } + @Nonnull private static List getExportedData(final FDBDatabase database, final KeySpacePath path) { List exportedData = new ArrayList<>(); From bee180607bef369465c66314e2a9f45194ca6983 Mon Sep 17 00:00:00 2001 From: Scott Dugas Date: Mon, 8 Sep 2025 17:20:15 -0400 Subject: [PATCH 03/21] Cleanup import tests --- .../keyspace/KeySpacePathImportDataTest.java | 667 +++++------------- 1 file changed, 168 insertions(+), 499 deletions(-) diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java index 84f2b4f619..2fa8530f51 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java @@ -22,15 +22,14 @@ import com.apple.foundationdb.KeyValue; import com.apple.foundationdb.Transaction; -import com.apple.foundationdb.record.RecordCoreArgumentException; import com.apple.foundationdb.record.ScanProperties; import com.apple.foundationdb.record.provider.foundationdb.FDBDatabase; import com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext; import com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpaceDirectory.KeyType; import com.apple.foundationdb.record.test.FDBDatabaseExtension; -import com.apple.foundationdb.subspace.Subspace; import com.apple.foundationdb.tuple.Tuple; import com.apple.test.Tags; +import org.assertj.core.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; @@ -46,7 +45,6 @@ import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -64,57 +62,95 @@ void setUp() { } @Test - void importBasicData() { - // Test importing basic data into a simple keyspace path - // Should validate that data is correctly stored using logical values - // rather than raw keys, allowing import across different clusters - final String companyUuid = UUID.randomUUID().toString(); + void importComprehensiveData() { + // Test importing data covering ALL KeyType enum values in a single complex directory structure: + // - NULL, BYTES, STRING, LONG, FLOAT, DOUBLE, BOOLEAN, UUID + // - Constant value directories + // - Complex multi-level hierarchy + // - Different data types in remainder elements + // Given that the majority of complexity is in the components under the importData method, this is basically + // an integration test + final String rootUuid = UUID.randomUUID().toString(); + byte[] binaryId = {0x01, 0x02, 0x03, (byte) 0xFF, (byte) 0xFE}; + UUID memberId = UUID.randomUUID(); + KeySpace root = new KeySpace( - new KeySpaceDirectory("company", KeyType.STRING, companyUuid) - .addSubdirectory(new KeySpaceDirectory("department", KeyType.STRING) - .addSubdirectory(new KeySpaceDirectory("employee_id", KeyType.LONG)))); + new KeySpaceDirectory("company", KeyType.STRING, rootUuid) // STRING with constant + .addSubdirectory(new KeySpaceDirectory("version", KeyType.LONG, 1L) // LONG with constant + .addSubdirectory(new KeySpaceDirectory("department", KeyType.STRING) // STRING variable + .addSubdirectory(new KeySpaceDirectory("employee_id", KeyType.LONG) // LONG variable + .addSubdirectory(new KeySpaceDirectory("binary_data", KeyType.BYTES) // BYTES + .addSubdirectory(new KeySpaceDirectory("null_section", KeyType.NULL) // NULL + .addSubdirectory(new KeySpaceDirectory("member", KeyType.UUID) // UUID + .addSubdirectory(new KeySpaceDirectory("active", KeyType.BOOLEAN) // BOOLEAN + .addSubdirectory(new KeySpaceDirectory("rating", KeyType.FLOAT)))))))))); // FLOAT + + + // Create comprehensive test data covering ALL KeyType values + KeySpacePath basePath = root.path("company").add("version").add("department", "engineering"); + + // Build paths using all KeyType values + KeySpacePath emp1Path = basePath.add("employee_id", 100L) + .add("binary_data", binaryId) + .add("null_section") + .add("member", memberId) + .add("active", true) + .add("rating", 4.5f); + + KeySpacePath emp2Path = basePath.add("employee_id", 200L) + .add("binary_data", binaryId) + .add("null_section") + .add("member", memberId) + .add("active", false) + .add("rating", 3.8f); try (FDBRecordContext context = database.openContext()) { Transaction tr = context.ensureActive(); - // Create some test data to export and then import - KeySpacePath deptPath = root.path("company").add("department", "engineering"); - KeySpacePath emp1Path = deptPath.add("employee_id", 100L); - KeySpacePath emp2Path = deptPath.add("employee_id", 200L); - - // Store original data with remainder elements byte[] key1 = emp1Path.toSubspace(context).pack(Tuple.from("profile", "name")); - byte[] value1 = Tuple.from("John Doe").pack(); + setToTuple(tr, key1, "John Doe"); + byte[] key2 = emp2Path.toSubspace(context).pack(Tuple.from("profile", "name")); - byte[] value2 = Tuple.from("Jane Smith").pack(); + setToTuple(tr, key2, "Jane Smith"); + + byte[] longKey = emp1Path.toSubspace(context).pack(Tuple.from("salary")); + setToTuple(tr, longKey, 75000); + + byte[] binaryKey = emp1Path.toSubspace(context).pack(Tuple.from("binary_metadata")); + tr.set(binaryKey, "binary_test_data".getBytes()); + + byte[] complexKey = emp1Path.toSubspace(context).pack(Tuple.from("info", 42, true, "complex")); + setToTuple(tr, complexKey, "Complex Test"); - tr.set(key1, value1); - tr.set(key2, value2); context.commit(); } // Export the data - final List exportedData = getExportedData(database, root.path("company").add("department", "engineering")); + final List exportedData = getExportedData(root.path("company")); // Clear the data and import it back clearPath(database, root.path("company")); // Import the data - importData(database, root.path("company").add("department", "engineering"), exportedData); + importData(database, root.path("company"), exportedData); - // Verify the data was imported correctly + // Verify all different KeyType values were handled correctly during import try (FDBRecordContext context = database.openContext()) { - KeySpacePath emp1Path = root.path("company").add("department", "engineering").add("employee_id", 100L); - KeySpacePath emp2Path = root.path("company").add("department", "engineering").add("employee_id", 200L); - byte[] key1 = emp1Path.toSubspace(context).pack(Tuple.from("profile", "name")); byte[] key2 = emp2Path.toSubspace(context).pack(Tuple.from("profile", "name")); - - Tuple value1 = getTuple(context, key1); - Tuple value2 = getTuple(context, key2); - - assertEquals("John Doe", value1.getString(0)); - assertEquals("Jane Smith", value2.getString(0)); + assertEquals(Tuple.from("John Doe"), getTuple(context, key1)); + assertEquals(Tuple.from("Jane Smith"), getTuple(context, key2)); + + byte[] longKey = emp1Path.toSubspace(context).pack(Tuple.from("salary")); + assertEquals(Tuple.from(75000), getTuple(context, longKey)); + + // Verify BYTES data (raw binary, not in tuple) + byte[] binaryKey = emp1Path.toSubspace(context).pack(Tuple.from("binary_metadata")); + assertArrayEquals("binary_test_data".getBytes(), getValue(context, binaryKey)); + + // Verify complex hierarchy with mixed types in remainder (LONG, BOOLEAN, STRING) + byte[] complexKey = emp1Path.toSubspace(context).pack(Tuple.from("info", 42, true, "complex")); + assertEquals(Tuple.from("Complex Test"), getTuple(context, complexKey)); } } @@ -125,15 +161,11 @@ void importEmptyData() { KeySpace root = new KeySpace( new KeySpaceDirectory("test", KeyType.STRING, UUID.randomUUID().toString())); - importData(database, root.path("test"), Collections.emptyList()); // should not throw any exception - - try (FDBRecordContext context = database.openContext()) { - KeySpacePath testPath = root.path("test"); - assertTrue( - testPath.exportAllData(context, null, ScanProperties.FORWARD_SCAN).first().join().isEmpty(), - "There should not have been any data created"); - } + KeySpacePath testPath = root.path("test"); + importData(database, testPath, Collections.emptyList()); // should not throw any exception + assertTrue(getExportedData(testPath).isEmpty(), + "there should not have been any data created"); } @Test @@ -145,22 +177,13 @@ void importOverwriteExistingData() { new KeySpaceDirectory("root", KeyType.STRING, rootUuid) .addSubdirectory(new KeySpaceDirectory("data", KeyType.LONG))); - try (FDBRecordContext context = database.openContext()) { - Transaction tr = context.ensureActive(); - - // Set initial data - KeySpacePath dataPath = root.path("root").add("data", 1L); - final Subspace subspace = dataPath.toSubspace(context); - byte[] key = subspace.pack(Tuple.from("record")); - tr.set(key, Tuple.from("original_value").pack()); - tr.set(subspace.pack("other"), Tuple.from("other_value").pack()); - context.commit(); - } + final KeySpacePath dataPath = root.path("root").add("data", 1L); + setSingleKey(dataPath, Tuple.from("record"), Tuple.from("original_value")); + setSingleKey(dataPath, Tuple.from("other"), Tuple.from("other_value")); // Create import data with same key but different value List importData = new ArrayList<>(); try (FDBRecordContext context = database.openContext()) { - KeySpacePath dataPath = root.path("root").add("data", 1L); byte[] key = dataPath.toSubspace(context).pack(Tuple.from("record")); KeyValue kv = new KeyValue(key, Tuple.from("new_value").pack()); importData.add(new DataInKeySpacePath(root.path("root"), kv, context)); @@ -172,51 +195,8 @@ void importOverwriteExistingData() { importData(database, root.path("root"), importData); // Verify the data was overwritten - try (FDBRecordContext context = database.openContext()) { - KeySpacePath dataPath = root.path("root").add("data", 1L); - final Subspace subspace = dataPath.toSubspace(context); - byte[] key = subspace.pack(Tuple.from("record")); - assertEquals(Tuple.from("new_value"), getTuple(context, key)); - assertEquals(Tuple.from("other_value"), getTuple(context, subspace.pack("other"))); - } - } - - @Test - void importDataWithConstantValues() { - // Test importing data into a keyspace with constant directory values - // Should verify that constant values are handled correctly during import - final String appUuid = UUID.randomUUID().toString(); - KeySpace root = new KeySpace( - new KeySpaceDirectory("application", KeyType.STRING, appUuid) - .addSubdirectory(new KeySpaceDirectory("version", KeyType.LONG, 1L) - .addSubdirectory(new KeySpaceDirectory("data", KeyType.STRING)))); - - - try (FDBRecordContext context = database.openContext()) { - Transaction tr = context.ensureActive(); - - // Create test data using constant values - KeySpacePath dataPath = root.path("application").add("version").add("data", "config"); - byte[] key = dataPath.toSubspace(context).pack(Tuple.from("setting")); - tr.set(key, Tuple.from("value1").pack()); - context.commit(); - } - - // Export and import the data - final List exportedData = getExportedData(database, root.path("application")); - - // Clear and import - clearPath(database, root.path("application")); - - importData(database, root.path("application"), exportedData); - - // Verify the data - try (FDBRecordContext context = database.openContext()) { - KeySpacePath dataPath = root.path("application").add("version").add("data", "config"); - byte[] key = dataPath.toSubspace(context).pack(Tuple.from("setting")); - Tuple value = getTuple(context, key); - assertEquals("value1", value.getString(0)); - } + verifySingleKey(dataPath, Tuple.from("record"), Tuple.from("new_value")); + verifySingleKey(dataPath, Tuple.from("other"), Tuple.from("other_value")); } @Test @@ -228,362 +208,130 @@ void importDataWithDirectoryLayer() { new DirectoryLayerDirectory("tenant", tenantUuid) .addSubdirectory(new KeySpaceDirectory("user_id", KeyType.LONG))); + final KeySpacePath dataPath = root.path("tenant").add("user_id", 999L); + setSingleKey(dataPath, Tuple.from("data"), Tuple.from("directory_test")); - try (FDBRecordContext context = database.openContext()) { - Transaction tr = context.ensureActive(); - - KeySpacePath userPath = root.path("tenant").add("user_id", 999L); - byte[] key = userPath.toSubspace(context).pack(Tuple.from("data")); - tr.set(key, Tuple.from("directory_test").pack()); - context.commit(); - } - - // Export and import - List exportedData = new ArrayList<>(); - try (FDBRecordContext context = database.openContext()) { - root.path("tenant").exportAllData(context, null, ScanProperties.FORWARD_SCAN) - .forEach(exportedData::add).join(); - } + List exportedData = getExportedData(root.path("tenant")); clearPath(database, root.path("tenant")); - try (FDBRecordContext context = database.openContext()) { - root.path("tenant").importData(context, exportedData).join(); - context.commit(); - } + importData(database, root.path("tenant"), exportedData); - // Verify - try (FDBRecordContext context = database.openContext()) { - KeySpacePath userPath = root.path("tenant").add("user_id", 999L); - byte[] key = userPath.toSubspace(context).pack(Tuple.from("data")); - Tuple value = getTuple(context, key); - assertEquals("directory_test", value.getString(0)); - } + verifySingleKey(dataPath, Tuple.from("data"), Tuple.from("directory_test")); } @Test - void importDataWithBinaryValues() { - // Test importing data containing binary (byte array) values - // Should verify that binary data is imported correctly - final String storeUuid = UUID.randomUUID().toString(); - KeySpace root = new KeySpace( - new KeySpaceDirectory("binary_store", KeyType.STRING, storeUuid) - .addSubdirectory(new KeySpaceDirectory("blob_id", KeyType.BYTES))); - + void importDataWithMismatchedPath() { + // Test importing data that doesn't belong to the target path + // Should throw RecordCoreIllegalImportDataException + final String root1Uuid = UUID.randomUUID().toString(); + final String root2Uuid = UUID.randomUUID().toString(); - byte[] blobId = {0x01, 0x02, 0x03, (byte) 0xFF, (byte) 0xFE}; - - try (FDBRecordContext context = database.openContext()) { - Transaction tr = context.ensureActive(); - - KeySpacePath blobPath = root.path("binary_store").add("blob_id", blobId); - byte[] key = blobPath.toSubspace(context).pack(Tuple.from("metadata")); - tr.set(key, "binary_test_data".getBytes()); - context.commit(); - } - - // Export and import - List exportedData = new ArrayList<>(); - try (FDBRecordContext context = database.openContext()) { - root.path("binary_store").exportAllData(context, null, ScanProperties.FORWARD_SCAN) - .forEach(exportedData::add).join(); - } - - clearPath(database, root.path("binary_store")); - - importData(database, root.path("binary_store"), exportedData); - - // Verify - try (FDBRecordContext context = database.openContext()) { - KeySpacePath blobPath = root.path("binary_store").add("blob_id", blobId); - byte[] key = blobPath.toSubspace(context).pack(Tuple.from("metadata")); - byte[] value = getJoin(context, key); - assertArrayEquals("binary_test_data".getBytes(), value); - } - } - - @Test - void importDataWithNullKeyType() { - // Test importing data into a keyspace with NULL key type directories - // Should verify that NULL directories are handled correctly - final String baseUuid = UUID.randomUUID().toString(); - KeySpace root = new KeySpace( - new KeySpaceDirectory("base", KeyType.STRING, baseUuid) - .addSubdirectory(new KeySpaceDirectory("null_dir", KeyType.NULL))); - - - try (FDBRecordContext context = database.openContext()) { - Transaction tr = context.ensureActive(); - - KeySpacePath nullPath = root.path("base").add("null_dir"); - byte[] key = nullPath.toSubspace(context).pack(Tuple.from("item")); - tr.set(key, Tuple.from("null_test").pack()); - context.commit(); - } - - // Export and import - List exportedData = new ArrayList<>(); - try (FDBRecordContext context = database.openContext()) { - root.path("base").exportAllData(context, null, ScanProperties.FORWARD_SCAN) - .forEach(exportedData::add).join(); - } - - clearPath(database, root.path("base")); - - try (FDBRecordContext context = database.openContext()) { - root.path("base").importData(context, exportedData).join(); - context.commit(); - } - - // Verify - try (FDBRecordContext context = database.openContext()) { - KeySpacePath nullPath = root.path("base").add("null_dir"); - byte[] key = nullPath.toSubspace(context).pack(Tuple.from("item")); - Tuple value = getTuple(context, key); - assertEquals("null_test", value.getString(0)); - } - } - - @Test - void importDataWithComplexHierarchy() { - // Test importing data into a multi-level directory hierarchy - // Should verify that nested paths are reconstructed correctly - final String rootUuid = UUID.randomUUID().toString(); - KeySpace root = new KeySpace( - new KeySpaceDirectory("org", KeyType.STRING, rootUuid) - .addSubdirectory(new KeySpaceDirectory("dept", KeyType.STRING) - .addSubdirectory(new KeySpaceDirectory("team", KeyType.LONG) - .addSubdirectory(new KeySpaceDirectory("member", KeyType.UUID))))); - - UUID memberId = UUID.randomUUID(); - - try (FDBRecordContext context = database.openContext()) { - Transaction tr = context.ensureActive(); - - KeySpacePath memberPath = root.path("org").add("dept", "engineering") - .add("team", 42L).add("member", memberId); - byte[] key = memberPath.toSubspace(context).pack(Tuple.from("info", "name")); - tr.set(key, Tuple.from("Complex Test").pack()); - context.commit(); - } - - // Export and import - List exportedData = getExportedData(database, root.path("org")); - clearPath(database, root.path("org")); - importData(database, root.path("org"), exportedData); - - // Verify - try (FDBRecordContext context = database.openContext()) { - KeySpacePath memberPath = root.path("org").add("dept", "engineering") - .add("team", 42L).add("member", memberId); - byte[] key = memberPath.toSubspace(context).pack(Tuple.from("info", "name")); - assertEquals(Tuple.from("Complex Test"), getTuple(context, key)); - } - } - - @Test - void importDataFromDifferentCluster() { - // Test importing data that was exported from a different cluster - // Should verify that logical values allow cross-cluster import - // This simulates the scenario where DirectoryLayer mappings differ - final String tenantUuid = UUID.randomUUID().toString(); - KeySpace root = new KeySpace( - new DirectoryLayerDirectory("tenant", tenantUuid) - .addSubdirectory(new KeySpaceDirectory("user_id", KeyType.LONG))); - - - // Create and export data in one "cluster" context - List exportedData = new ArrayList<>(); - try (FDBRecordContext context1 = database.openContext()) { - Transaction tr = context1.ensureActive(); - - KeySpacePath userPath = root.path("tenant").add("user_id", 123L); - byte[] key = userPath.toSubspace(context1).pack(Tuple.from("profile")); - tr.set(key, Tuple.from("cross_cluster_test").pack()); - - root.path("tenant").exportAllData(context1, null, ScanProperties.FORWARD_SCAN) - .forEach(exportedData::add).join(); - context1.commit(); - } + KeySpace keySpace = new KeySpace( + new KeySpaceDirectory("root1", KeyType.STRING, root1Uuid) + .addSubdirectory(new KeySpaceDirectory("data", KeyType.LONG)), + new KeySpaceDirectory("root2", KeyType.STRING, root2Uuid) + .addSubdirectory(new KeySpaceDirectory("data", KeyType.LONG))); - // Clear the data to simulate different cluster - clearPath(database, root.path("tenant")); + setSingleKey(keySpace.path("root1").add("data", 1L), Tuple.from("record"), Tuple.from("other")); - // Import in a different "cluster" context - try (FDBRecordContext context2 = database.openContext()) { - root.path("tenant").importData(context2, exportedData).join(); - context2.commit(); - } - - // Verify the data was imported using logical values - try (FDBRecordContext context = database.openContext()) { - KeySpacePath userPath = root.path("tenant").add("user_id", 123L); - byte[] key = userPath.toSubspace(context).pack(Tuple.from("profile")); - Tuple value = getTuple(context, key); - assertEquals("cross_cluster_test", value.getString(0)); - } + // Now try to ipmort that into keySpace2 + List exportedData = getExportedData(keySpace.path("root1")); + assertBadImport(keySpace.path("root2"), exportedData); } @Test - void importDataWithMismatchedPath() { - // Test importing data that doesn't belong to the target path + void importDataWithInvalidPath() { + // Test importing data with paths that don't exist in the keyspace // Should throw RecordCoreIllegalImportDataException final String root1Uuid = UUID.randomUUID().toString(); final String root2Uuid = UUID.randomUUID().toString(); - + KeySpace keySpace1 = new KeySpace( new KeySpaceDirectory("root1", KeyType.STRING, root1Uuid) .addSubdirectory(new KeySpaceDirectory("data", KeyType.LONG))); - + KeySpace keySpace2 = new KeySpace( new KeySpaceDirectory("root2", KeyType.STRING, root2Uuid) .addSubdirectory(new KeySpaceDirectory("data", KeyType.LONG))); + setSingleKey(keySpace1.path("root1").add("data", 1L), Tuple.from("record"), Tuple.from("other")); - // Create data in keySpace2 - List exportedData = new ArrayList<>(); - try (FDBRecordContext context = database.openContext()) { - Transaction tr = context.ensureActive(); - KeySpacePath dataPath = keySpace2.path("root2").add("data", 1L); - byte[] key = dataPath.toSubspace(context).pack(Tuple.from("record")); - tr.set(key, Tuple.from("value").pack()); - - KeyValue kv = new KeyValue(key, Tuple.from("value").pack()); - exportedData.add(new DataInKeySpacePath(keySpace2.path("root2"), kv, context)); - context.commit(); - } - - // Try to import into keySpace1 - should fail - try (FDBRecordContext context = database.openContext()) { - KeySpacePath root1Path = keySpace1.path("root1"); - - CompletionException completionException = assertThrows(CompletionException.class, () -> - root1Path.importData(context, exportedData).join() - ); - assertTrue(completionException.getCause() instanceof RecordCoreIllegalImportDataException); - } + // Now try to ipmort that into keySpace2 + List exportedData = getExportedData(keySpace1.path("root1")); + assertBadImport(keySpace2.path("root2"), exportedData); } @Test - void importDataWithInvalidPath() { - // Test importing data with paths that don't exist in the keyspace - // Should throw RecordCoreIllegalImportDataException + void importDataWithSubdirectoryPath() { + // Test importing data where the target path is a subdirectory of the import path + // Should succeed only if all the data is in the subdirectory final String rootUuid = UUID.randomUUID().toString(); KeySpace root = new KeySpace( new KeySpaceDirectory("root", KeyType.STRING, rootUuid) - .addSubdirectory(new KeySpaceDirectory("valid", KeyType.LONG))); + .addSubdirectory(new KeySpaceDirectory("level1", KeyType.LONG))); + KeySpacePath level1Path = root.path("root").add("level1", 1L); - try (FDBRecordContext context = database.openContext()) { - // Create a key that doesn't follow the keyspace structure - byte[] invalidKey = Tuple.from("completely", "different", "structure").pack(); - KeyValue kv = new KeyValue(invalidKey, Tuple.from("value").pack()); - - List invalidData = Collections.singletonList( - new DataInKeySpacePath(root.path("root"), kv, context) - ); + setSingleKey(level1Path, Tuple.from("item1"), Tuple.from("value1")); - CompletionException completionException = assertThrows(CompletionException.class, () -> - root.path("root").importData(context, invalidData).join() - ); - // The cause might be RecordCoreIllegalImportDataException or RecordCoreArgumentException for invalid paths - assertTrue(completionException.getCause() instanceof RecordCoreArgumentException); - } + // Export from root, import to subdirectory + List exportedData = getExportedData(root.path("root")); + + clearPath(database, root.path("root")); + + // Import only to level1 subdirectory + importData(database, level1Path, exportedData); + + verifySingleKey(level1Path, Tuple.from("item1"), Tuple.from("value1")); } @Test - void importDataWithSubdirectoryPath() { + void importDataWithSubdirectoryPathFailure() { // Test importing data where the target path is a subdirectory of the import path - // Should succeed and import only data that belongs to subdirectories + // Should succeed only if all the data is in the subdirectory final String rootUuid = UUID.randomUUID().toString(); KeySpace root = new KeySpace( new KeySpaceDirectory("root", KeyType.STRING, rootUuid) - .addSubdirectory(new KeySpaceDirectory("level1", KeyType.LONG) - .addSubdirectory(new KeySpaceDirectory("level2", KeyType.STRING)))); + .addSubdirectory(new KeySpaceDirectory("level1", KeyType.LONG))); + KeySpacePath level1Path = root.path("root").add("level1", 1L); - try (FDBRecordContext context = database.openContext()) { - Transaction tr = context.ensureActive(); - - // Create data at different levels - KeySpacePath level1Path = root.path("root").add("level1", 1L); - KeySpacePath level2Path = level1Path.add("level2", "data"); - - byte[] key1 = level1Path.toSubspace(context).pack(Tuple.from("item1")); - byte[] key2 = level2Path.toSubspace(context).pack(Tuple.from("item2")); - - tr.set(key1, Tuple.from("value1").pack()); - tr.set(key2, Tuple.from("value2").pack()); - context.commit(); - } + setSingleKey(level1Path, Tuple.from("item1"), Tuple.from("value1")); + setSingleKey(root.path("root").add("level1", 2L), Tuple.from("item1"), Tuple.from("value1")); // Export from root, import to subdirectory - List exportedData = new ArrayList<>(); - try (FDBRecordContext context = database.openContext()) { - root.path("root").exportAllData(context, null, ScanProperties.FORWARD_SCAN) - .forEach(exportedData::add).join(); - } + List exportedData = getExportedData(root.path("root")); clearPath(database, root.path("root")); // Import only to level1 subdirectory - importData(database, root.path("root").add("level1", 1L), exportedData); - - // Verify only level1 and below data exists - try (FDBRecordContext context = database.openContext()) { - KeySpacePath level2Path = root.path("root").add("level1", 1L).add("level2", "data"); - byte[] key2 = level2Path.toSubspace(context).pack(Tuple.from("item2")); - Tuple value2 = getTuple(context, key2); - assertEquals("value2", value2.getString(0)); - } + assertBadImport(level1Path, exportedData); } @Test - void importDataWithParentPath() { + void importDataWithPartialMismatch() { // Test importing data where the target path is a parent of some import data paths // Should throw RecordCoreIllegalImportDataException for paths outside the target final String root1Uuid = UUID.randomUUID().toString(); final String root2Uuid = UUID.randomUUID().toString(); - KeySpace keySpace1 = new KeySpace( + KeySpace keySpace = new KeySpace( new KeySpaceDirectory("root1", KeyType.STRING, root1Uuid) - .addSubdirectory(new KeySpaceDirectory("child", KeyType.LONG))); - - KeySpace keySpace2 = new KeySpace( + .addSubdirectory(new KeySpaceDirectory("child", KeyType.LONG)), new KeySpaceDirectory("root2", KeyType.STRING, root2Uuid) .addSubdirectory(new KeySpaceDirectory("child", KeyType.LONG))); + KeySpacePath path1 = keySpace.path("root1").add("child", 1L); + KeySpacePath path2 = keySpace.path("root2").add("child", 2L); + setSingleKey(path1, Tuple.from("data"), Tuple.from("data1")); + setSingleKey(path2, Tuple.from("data"), Tuple.from("data2")); - // Create data in both keyspaces List mixedData = new ArrayList<>(); - try (FDBRecordContext context = database.openContext()) { - Transaction tr = context.ensureActive(); - - KeySpacePath path1 = keySpace1.path("root1").add("child", 1L); - KeySpacePath path2 = keySpace2.path("root2").add("child", 2L); - - byte[] key1 = path1.toSubspace(context).pack(Tuple.from("data")); - byte[] key2 = path2.toSubspace(context).pack(Tuple.from("data")); - - tr.set(key1, Tuple.from("data1").pack()); - tr.set(key2, Tuple.from("data2").pack()); - - KeyValue kv1 = new KeyValue(key1, Tuple.from("data1").pack()); - KeyValue kv2 = new KeyValue(key2, Tuple.from("data2").pack()); - - mixedData.add(new DataInKeySpacePath(keySpace1.path("root1"), kv1, context)); - mixedData.add(new DataInKeySpacePath(keySpace2.path("root2"), kv2, context)); - - context.commit(); - } + mixedData.addAll(getExportedData(path1)); + mixedData.addAll(getExportedData(path2)); - // Try to import mixed data into keySpace1 - should fail due to keySpace2 data - try (FDBRecordContext context = database.openContext()) { - CompletionException completionException = assertThrows(CompletionException.class, () -> - keySpace1.path("root1").importData(context, mixedData).join() - ); - assertTrue(completionException.getCause() instanceof RecordCoreIllegalImportDataException); - } + assertBadImport(keySpace.path("root1"), mixedData); } @Test @@ -592,29 +340,13 @@ void importDataWithWrapperClasses() { // Should verify that wrapper functionality works correctly with import final EnvironmentKeySpace keySpace = EnvironmentKeySpace.setupSampleData(database); - // Export some data - List exportedData = new ArrayList<>(); - try (FDBRecordContext context = database.openContext()) { - EnvironmentKeySpace.DataPath dataStore = keySpace.root().userid(100L).application("app1").dataStore(); - dataStore.exportAllData(context, null, ScanProperties.FORWARD_SCAN) - .forEach(exportedData::add).join(); - } + EnvironmentKeySpace.DataPath dataStore = keySpace.root().userid(100L).application("app1").dataStore(); + List exportedData = getExportedData(dataStore); - // Clear and import back clearPath(database, keySpace.root()); - try (FDBRecordContext context = database.openContext()) { - keySpace.root().importData(context, exportedData).join(); - context.commit(); - } - - // Verify data exists - try (FDBRecordContext context = database.openContext()) { - EnvironmentKeySpace.DataPath dataStore = keySpace.root().userid(100L).application("app1").dataStore(); - byte[] key = dataStore.toSubspace(context).pack(Tuple.from("record2", 0)); - byte[] value = getJoin(context, key); - assertTrue(value.length > 0); - } + importData(database, keySpace.root(), exportedData); + verifySingleKey(dataStore, Tuple.from("record2", 0), Tuple.from("user100_app1_data2_0")); } @Test @@ -627,8 +359,8 @@ void importDataWithDuplicateKeys() { .addSubdirectory(new KeySpaceDirectory("data", KeyType.LONG))); + KeySpacePath dataPath = root.path("root").add("data", 1L); try (FDBRecordContext context = database.openContext()) { - KeySpacePath dataPath = root.path("root").add("data", 1L); byte[] key = dataPath.toSubspace(context).pack(Tuple.from("item")); // Create multiple DataInKeySpacePath objects with same key but different values @@ -645,106 +377,34 @@ void importDataWithDuplicateKeys() { context.commit(); } - // Verify the final value is stored - try (FDBRecordContext context = database.openContext()) { - KeySpacePath dataPath = root.path("root").add("data", 1L); - byte[] key = dataPath.toSubspace(context).pack(Tuple.from("item")); - Tuple value = getTuple(context, key); - assertEquals("final_value", value.getString(0)); - } + verifySingleKey(dataPath, Tuple.from("item"), Tuple.from("final_value")); } - @Test - void importDataRollback() { - // Test import operation that fails and needs to be rolled back - // Should verify that partial imports are properly cleaned up - final String root1Uuid = UUID.randomUUID().toString(); - final String root2Uuid = UUID.randomUUID().toString(); - - KeySpace keySpace1 = new KeySpace( - new KeySpaceDirectory("root1", KeyType.STRING, root1Uuid) - .addSubdirectory(new KeySpaceDirectory("data", KeyType.LONG))); - - KeySpace keySpace2 = new KeySpace( - new KeySpaceDirectory("root2", KeyType.STRING, root2Uuid) - .addSubdirectory(new KeySpaceDirectory("data", KeyType.LONG))); - - + private void setSingleKey(KeySpacePath path, Tuple remainder, Tuple value) { try (FDBRecordContext context = database.openContext()) { - Transaction tr = context.ensureActive(); - - // Create mixed data that will partially succeed then fail - KeySpacePath validPath = keySpace1.path("root1").add("data", 1L); - KeySpacePath invalidPath = keySpace2.path("root2").add("data", 2L); - - byte[] validKey = validPath.toSubspace(context).pack(Tuple.from("item")); - byte[] invalidKey = invalidPath.toSubspace(context).pack(Tuple.from("item")); - - List mixedData = Arrays.asList( - new DataInKeySpacePath(keySpace1.path("root1"), - new KeyValue(validKey, Tuple.from("valid").pack()), context), - new DataInKeySpacePath(keySpace2.path("root2"), - new KeyValue(invalidKey, Tuple.from("invalid").pack()), context) - ); - - // This should fail and rollback - CompletionException completionException = assertThrows(CompletionException.class, () -> - keySpace1.path("root1").importData(context, mixedData).join() - ); - assertTrue(completionException.getCause() instanceof RecordCoreIllegalImportDataException); - } - - // Verify no data was written after the failed transaction - try (FDBRecordContext context = database.openContext()) { - KeySpacePath validPath = keySpace1.path("root1").add("data", 1L); - byte[] validKey = validPath.toSubspace(context).pack(Tuple.from("item")); - byte[] value = getJoin(context, validKey); - assertTrue(value == null || value.length == 0); + byte[] key = path.toSubspace(context).pack(remainder); + context.ensureActive().set(key, value.pack()); + context.commit(); } } - @Test - void importDataValidation() { - // Test various edge cases for data validation - // Should verify that malformed DataInKeySpacePath objects are rejected - final String rootUuid = UUID.randomUUID().toString(); - KeySpace root = new KeySpace( - new KeySpaceDirectory("root", KeyType.STRING, rootUuid) - .addSubdirectory(new KeySpaceDirectory("data", KeyType.LONG))); - - + private void verifySingleKey(KeySpacePath path, Tuple remainder, Tuple expected) { try (FDBRecordContext context = database.openContext()) { - KeySpacePath rootPath = root.path("root"); - - // Test with empty iterable - rootPath.importData(context, Collections.emptyList()).join(); - - // Test with null remainder (should work) - KeySpacePath dataPath = rootPath.add("data", 1L); - byte[] key = dataPath.toSubspace(context).pack(); // No remainder - KeyValue kv = new KeyValue(key, Tuple.from("test").pack()); - List validData = Collections.singletonList( - new DataInKeySpacePath(rootPath, kv, context) - ); - - rootPath.importData(context, validData).join(); - context.commit(); + byte[] key = path.toSubspace(context).pack(remainder); + assertEquals(expected, getTuple(context, key)); } + } - // Verify the data was stored - try (FDBRecordContext context = database.openContext()) { - KeySpacePath dataPath = root.path("root").add("data", 1L); - byte[] key = dataPath.toSubspace(context).pack(); - Tuple value = getTuple(context, key); - assertEquals("test", value.getString(0)); - } + private static void setToTuple(final Transaction tr, final byte[] key1, Object items) { + byte[] value1 = Tuple.from(items).pack(); + tr.set(key1, value1); } private static Tuple getTuple(final FDBRecordContext context, final byte[] key) { return Tuple.fromBytes(context.ensureActive().get(key).join()); } - private static byte[] getJoin(final FDBRecordContext context, final byte[] key) { + private static byte[] getValue(final FDBRecordContext context, final byte[] key) { return context.ensureActive().get(key).join(); } @@ -755,19 +415,28 @@ private static void importData(final FDBDatabase database, final KeySpacePath pa } } - private static void clearPath(final FDBDatabase database, final KeySpacePath path) { + private void assertBadImport(final KeySpacePath path, final List invalidData) { + // Try to import into keySpace1 - should fail + try (FDBRecordContext context = database.openContext()) { + Assertions.assertThatThrownBy(() -> path.importData(context, invalidData).join()) + .isInstanceOf(CompletionException.class) + .hasCauseInstanceOf(RecordCoreIllegalImportDataException.class); + } + } + + private void clearPath(final FDBDatabase database, final KeySpacePath path) { try (FDBRecordContext context = database.openContext()) { Transaction tr = context.ensureActive(); tr.clear(path.toSubspace(context).range()); context.commit(); } // just an extra check to make sure the test is working as expected - assertTrue(getExportedData(database, path).isEmpty(), + assertTrue(getExportedData(path).isEmpty(), "Clearing should remove all the data"); } @Nonnull - private static List getExportedData(final FDBDatabase database, final KeySpacePath path) { + private List getExportedData(final KeySpacePath path) { List exportedData = new ArrayList<>(); try (FDBRecordContext context = database.openContext()) { path.exportAllData(context, null, ScanProperties.FORWARD_SCAN) From 44d609b67d821b5e70f0122f77c730c9639f340f Mon Sep 17 00:00:00 2001 From: Scott Dugas Date: Mon, 8 Sep 2025 17:30:53 -0400 Subject: [PATCH 04/21] A little more test cleanup --- .../foundationdb/keyspace/KeySpacePathImportDataTest.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java index 2fa8530f51..5dc24bcbd8 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java @@ -146,7 +146,7 @@ void importComprehensiveData() { // Verify BYTES data (raw binary, not in tuple) byte[] binaryKey = emp1Path.toSubspace(context).pack(Tuple.from("binary_metadata")); - assertArrayEquals("binary_test_data".getBytes(), getValue(context, binaryKey)); + assertArrayEquals("binary_test_data".getBytes(), context.ensureActive().get(binaryKey).join()); // Verify complex hierarchy with mixed types in remainder (LONG, BOOLEAN, STRING) byte[] complexKey = emp1Path.toSubspace(context).pack(Tuple.from("info", 42, true, "complex")); @@ -404,10 +404,6 @@ private static Tuple getTuple(final FDBRecordContext context, final byte[] key) return Tuple.fromBytes(context.ensureActive().get(key).join()); } - private static byte[] getValue(final FDBRecordContext context, final byte[] key) { - return context.ensureActive().get(key).join(); - } - private static void importData(final FDBDatabase database, final KeySpacePath path, final List exportedData) { try (FDBRecordContext context = database.openContext()) { path.importData(context, exportedData).join(); From b8104974aeea9bff69c0b657ae503b89f31898eb Mon Sep 17 00:00:00 2001 From: Scott Dugas Date: Fri, 24 Oct 2025 14:08:13 -0400 Subject: [PATCH 05/21] Respond to api change on main for DataInKeySpacePath --- .../record/provider/foundationdb/keyspace/KeySpacePathImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImpl.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImpl.java index d5e36d8f4e..db0e65612a 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImpl.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImpl.java @@ -341,7 +341,7 @@ public CompletableFuture importData(@Nonnull FDBRecordContext context, // Store the data byte[] keyBytes = keyTuple.pack(); - byte[] valueBytes = dataItem.getRawKeyValue().getValue(); + byte[] valueBytes = dataItem.getValue(); context.ensureActive().set(keyBytes, valueBytes); return AsyncUtil.DONE; From 9132184e3b9187d1debb4944a54a954bb262ee85 Mon Sep 17 00:00:00 2001 From: Scott Dugas Date: Fri, 24 Oct 2025 14:08:31 -0400 Subject: [PATCH 06/21] Extract helper for export+import --- .../keyspace/KeySpacePathImportDataTest.java | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java index 5dc24bcbd8..8b3f71db2e 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java @@ -125,14 +125,7 @@ void importComprehensiveData() { context.commit(); } - // Export the data - final List exportedData = getExportedData(root.path("company")); - - // Clear the data and import it back - clearPath(database, root.path("company")); - - // Import the data - importData(database, root.path("company"), exportedData); + copyData(root.path("company")); // Verify all different KeyType values were handled correctly during import try (FDBRecordContext context = database.openContext()) { @@ -211,11 +204,7 @@ void importDataWithDirectoryLayer() { final KeySpacePath dataPath = root.path("tenant").add("user_id", 999L); setSingleKey(dataPath, Tuple.from("data"), Tuple.from("directory_test")); - List exportedData = getExportedData(root.path("tenant")); - - clearPath(database, root.path("tenant")); - - importData(database, root.path("tenant"), exportedData); + copyData(root.path("tenant")); verifySingleKey(dataPath, Tuple.from("data"), Tuple.from("directory_test")); } @@ -341,11 +330,9 @@ void importDataWithWrapperClasses() { final EnvironmentKeySpace keySpace = EnvironmentKeySpace.setupSampleData(database); EnvironmentKeySpace.DataPath dataStore = keySpace.root().userid(100L).application("app1").dataStore(); - List exportedData = getExportedData(dataStore); - clearPath(database, keySpace.root()); + copyData(keySpace.root()); - importData(database, keySpace.root(), exportedData); verifySingleKey(dataStore, Tuple.from("record2", 0), Tuple.from("user100_app1_data2_0")); } @@ -404,6 +391,17 @@ private static Tuple getTuple(final FDBRecordContext context, final byte[] key) return Tuple.fromBytes(context.ensureActive().get(key).join()); } + private void copyData(final KeySpacePath path) { + // Export the data + final List exportedData = getExportedData(path); + + // Clear the data and import it back + clearPath(database, path); + + // Import the data + importData(database, path, exportedData); + } + private static void importData(final FDBDatabase database, final KeySpacePath path, final List exportedData) { try (FDBRecordContext context = database.openContext()) { path.importData(context, exportedData).join(); From f63ae18658c7bedef7e9add5411674fd32ae4ba9 Mon Sep 17 00:00:00 2001 From: Scott Dugas Date: Fri, 24 Oct 2025 14:28:47 -0400 Subject: [PATCH 07/21] Change the import test to use 2 clusters if available --- .../keyspace/KeySpacePathImportDataTest.java | 12 +++++++++--- .../record/test/FDBDatabaseExtension.java | 10 ++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java index 8b3f71db2e..a42f08b8c5 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java @@ -55,10 +55,12 @@ class KeySpacePathImportDataTest { @RegisterExtension final FDBDatabaseExtension dbExtension = new FDBDatabaseExtension(); private FDBDatabase database; + private List databases; @BeforeEach void setUp() { - database = dbExtension.getDatabase(); + databases = dbExtension.getDatabases(2); + database = databases.get(0); } @Test @@ -395,8 +397,12 @@ private void copyData(final KeySpacePath path) { // Export the data final List exportedData = getExportedData(path); - // Clear the data and import it back - clearPath(database, path); + if (databases.size() > 1) { + database = databases.get(1); + } else { + // Clear the data and import it back + clearPath(database, path); + } // Import the data importData(database, path, exportedData); diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/test/FDBDatabaseExtension.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/test/FDBDatabaseExtension.java index a1309989f7..42a9ee58d1 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/test/FDBDatabaseExtension.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/test/FDBDatabaseExtension.java @@ -38,8 +38,12 @@ import javax.annotation.Nullable; import java.util.HashMap; import java.util.Map; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.Objects; import java.util.concurrent.Executor; +import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -161,6 +165,12 @@ public FDBDatabase getDatabase(@Nullable String clusterFile) { }); } + public List getDatabases(int count) { + List clusterFiles = new ArrayList<>(FDBTestEnvironment.allClusterFiles()); + Collections.shuffle(clusterFiles); + return clusterFiles.stream().limit(count).map(this::getDatabase).collect(Collectors.toList()); + } + public void checkForOpenContexts() { for (final Map.Entry clusterFileToDatabase : databases.entrySet()) { assertEquals(0, clusterFileToDatabase.getValue().warnAndCloseOldTrackedOpenContexts(0), From 714faf3f1e6799822d58c0380cd4bf37f5c06084 Mon Sep 17 00:00:00 2001 From: Scott Dugas Date: Mon, 27 Oct 2025 20:39:52 -0400 Subject: [PATCH 08/21] Respond to DataInKeySpacePath not having Resolved on main (after rebase) --- .../foundationdb/keyspace/KeySpacePath.java | 3 +- .../keyspace/KeySpacePathImpl.java | 25 ++++++++--------- .../keyspace/KeySpacePathDataExportTest.java | 7 +++-- .../keyspace/KeySpacePathImportDataTest.java | 28 +++++++------------ 4 files changed, 28 insertions(+), 35 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePath.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePath.java index 562c3e03e4..6490e04870 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePath.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePath.java @@ -417,7 +417,8 @@ default RecordCursor exportAllData(@Nonnull FDBRecordContext * or one of the sub-directories, if not the future will complete exceptionally with * {@link RecordCoreIllegalImportDataException}. * If there is any data already existing under this path, the new data will overwrite if the keys are the same. - * This will use the logical value in the {@link DataInKeySpacePath#getResolvedPath()} to determine the key, rather + * This will use the logical values in the {@link DataInKeySpacePath#getPath()} and + * {@link DataInKeySpacePath#getRemainder()} to determine the key, rather * than the raw key, meaning that this will work even if the data was exported from a different cluster. * @param context the transaction context in which to save the data * @param dataToImport the data to be saved to the database diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImpl.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImpl.java index db0e65612a..0076b70233 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImpl.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImpl.java @@ -322,33 +322,32 @@ public CompletableFuture importData(@Nonnull FDBRecordContext context, @Nonnull Iterable dataToImport) { return toTupleAsync(context).thenCompose(targetTuple -> { List> importFutures = new ArrayList<>(); - + for (DataInKeySpacePath dataItem : dataToImport) { - CompletableFuture importFuture = dataItem.getResolvedPath().thenCompose(resolvedPath -> { + CompletableFuture importFuture = dataItem.getPath().toTupleAsync(context).thenCompose(itemPathTuple -> { // Validate that this data belongs under this path - Tuple itemTuple = resolvedPath.toTuple(); - if (!TupleHelpers.isPrefix(targetTuple, itemTuple)) { + if (!TupleHelpers.isPrefix(targetTuple, itemPathTuple)) { throw new RecordCoreIllegalImportDataException( "Data item path does not belong under target path", - "target", targetTuple, "item", itemTuple); + "target", targetTuple, "item", itemPathTuple); } - - // Reconstruct the key using logical values from the resolved path - Tuple keyTuple = itemTuple; - if (resolvedPath.getRemainder() != null) { - keyTuple = keyTuple.addAll(resolvedPath.getRemainder()); + + // Reconstruct the key using the path and remainder + Tuple keyTuple = itemPathTuple; + if (dataItem.getRemainder() != null) { + keyTuple = keyTuple.addAll(dataItem.getRemainder()); } - + // Store the data byte[] keyBytes = keyTuple.pack(); byte[] valueBytes = dataItem.getValue(); context.ensureActive().set(keyBytes, valueBytes); - + return AsyncUtil.DONE; }); importFutures.add(importFuture); } - + return AsyncUtil.whenAll(importFutures); }); } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathDataExportTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathDataExportTest.java index 1ff39187dd..fca7445dfa 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathDataExportTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathDataExportTest.java @@ -635,7 +635,7 @@ private static List exportAllData(final KeySpacePath pathToE final List reversed = pathToExport.exportAllData(context, null, ScanProperties.REVERSE_SCAN) .asList().join(); Collections.reverse(reversed); - assertDataInKeySpacePathEquals(asSingleExport, reversed); + assertDataInKeySpacePathEquals(context, asSingleExport, reversed); // Assert continuations work correctly final ScanProperties scanProperties = ScanProperties.FORWARD_SCAN.with(props -> props.setReturnedRowLimit(1)); @@ -655,11 +655,12 @@ private static List exportAllData(final KeySpacePath pathToE } } - assertDataInKeySpacePathEquals(asSingleExport, asContinuations); + assertDataInKeySpacePathEquals(context, asSingleExport, asContinuations); return asSingleExport; } - private static void assertDataInKeySpacePathEquals(final List expectedList, + private static void assertDataInKeySpacePathEquals(final FDBRecordContext context, + final List expectedList, final List actualList) { assertThat(actualList).zipSatisfy(expectedList, (actual, other) -> { diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java index a42f08b8c5..8df2ff5f30 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java @@ -20,7 +20,6 @@ package com.apple.foundationdb.record.provider.foundationdb.keyspace; -import com.apple.foundationdb.KeyValue; import com.apple.foundationdb.Transaction; import com.apple.foundationdb.record.ScanProperties; import com.apple.foundationdb.record.provider.foundationdb.FDBDatabase; @@ -178,11 +177,8 @@ void importOverwriteExistingData() { // Create import data with same key but different value List importData = new ArrayList<>(); - try (FDBRecordContext context = database.openContext()) { - byte[] key = dataPath.toSubspace(context).pack(Tuple.from("record")); - KeyValue kv = new KeyValue(key, Tuple.from("new_value").pack()); - importData.add(new DataInKeySpacePath(root.path("root"), kv, context)); - } + importData.add(new DataInKeySpacePath(dataPath, + Tuple.from("record"), Tuple.from("new_value").pack())); // Verify we can re-import the data multiple times importData(database, root.path("root"), importData); @@ -349,19 +345,15 @@ void importDataWithDuplicateKeys() { KeySpacePath dataPath = root.path("root").add("data", 1L); - try (FDBRecordContext context = database.openContext()) { - byte[] key = dataPath.toSubspace(context).pack(Tuple.from("item")); - - // Create multiple DataInKeySpacePath objects with same key but different values - List duplicateData = Arrays.asList( - new DataInKeySpacePath(root.path("root"), - new KeyValue(key, Tuple.from("first_value").pack()), context), - new DataInKeySpacePath(root.path("root"), - new KeyValue(key, Tuple.from("second_value").pack()), context), - new DataInKeySpacePath(root.path("root"), - new KeyValue(key, Tuple.from("final_value").pack()), context) - ); + // Create multiple DataInKeySpacePath objects with same key but different values + List duplicateData = Arrays.asList( + new DataInKeySpacePath(dataPath, Tuple.from("item"), Tuple.from("first_value").pack()), + new DataInKeySpacePath(dataPath, Tuple.from("item"), Tuple.from("second_value").pack()), + new DataInKeySpacePath(dataPath, Tuple.from("item"), Tuple.from("final_value").pack()) + ); + + try (FDBRecordContext context = database.openContext()) { root.path("root").importData(context, duplicateData).join(); context.commit(); } From c255044fce087ae656fa2a6cf3902a3c7c59d6bf Mon Sep 17 00:00:00 2001 From: Scott Dugas Date: Mon, 27 Oct 2025 20:41:59 -0400 Subject: [PATCH 09/21] Cleanup some of the export tests --- .../keyspace/KeySpacePathDataExportTest.java | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathDataExportTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathDataExportTest.java index fca7445dfa..2e7292e6cf 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathDataExportTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathDataExportTest.java @@ -49,7 +49,6 @@ import java.util.Map; import java.util.Set; import java.util.UUID; -import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -69,7 +68,7 @@ class KeySpacePathDataExportTest { final FDBDatabaseExtension dbExtension = new FDBDatabaseExtension(); @Test - void exportAllDataFromSimplePath() throws ExecutionException, InterruptedException { + void exportAllDataFromSimplePath() { KeySpace root = new KeySpace( new KeySpaceDirectory("root", KeyType.STRING, UUID.randomUUID().toString()) .addSubdirectory(new KeySpaceDirectory("level1", KeyType.LONG))); @@ -83,13 +82,14 @@ void exportAllDataFromSimplePath() throws ExecutionException, InterruptedExcepti // Add data at different levels for (int i = 0; i < 5; i++) { - Tuple key = basePath.add("level1", (long) i).toTuple(context); + final KeySpacePath path = basePath.add("level1", (long)i); + Tuple key = path.toTuple(context); tr.set(key.pack(), Tuple.from("value" + i).pack()); // Add some sub-data under each key for (int j = 0; j < 3; j++) { - Tuple subKey = key.add("sub" + j); - tr.set(subKey.pack(), Tuple.from("subvalue" + i + "_" + j).pack()); + tr.set(path.toSubspace(context).pack(Tuple.from("sub" + j)), + Tuple.from("subvalue" + i + "_" + j).pack()); } } context.commit(); @@ -103,16 +103,19 @@ void exportAllDataFromSimplePath() throws ExecutionException, InterruptedExcepti // Should have 5 main entries + 15 sub-entries = 20 total assertEquals(20, allData.size()); + assertThat(allData) + .allSatisfy(data -> + assertThat(data.getPath().getDirectoryName()).isEqualTo("level1")); + // Verify the data is sorted by key - for (int i = 1; i < allData.size(); i++) { - assertTrue(getKey(allData.get(i - 1), context).compareTo(getKey(allData.get(i), context)) < 0); - } + assertThat(allData.stream().map(data -> getKey(data, context)).collect(Collectors.toList())) + .isSorted(); } } // `toTuple` does not include the remainder, I'm not sure if that is intentional, or an oversight. - private Tuple getKey(final DataInKeySpacePath dataInKeySpacePath, final FDBRecordContext context) throws ExecutionException, InterruptedException { - final ResolvedKeySpacePath resolvedKeySpacePath = dataInKeySpacePath.getPath().toResolvedPathAsync(context).get(); + private Tuple getKey(final DataInKeySpacePath dataInKeySpacePath, final FDBRecordContext context) { + final ResolvedKeySpacePath resolvedKeySpacePath = dataInKeySpacePath.getPath().toResolvedPathAsync(context).join(); if (dataInKeySpacePath.getRemainder() != null) { return resolvedKeySpacePath.toTuple().addAll(dataInKeySpacePath.getRemainder()); } else { @@ -524,9 +527,7 @@ private static void exportWithContinuations(final KeySpacePath pathToExport, final RecordCursor cursor = pathToExport.exportAllData(context, continuation.toBytes(), scanProperties); final AtomicReference> tupleResult = new AtomicReference<>(); - final List batch = cursor.map(dataInPath -> { - return Tuple.fromBytes(dataInPath.getValue()); - }).asList(tupleResult).join(); + final List batch = cursor.map(dataInPath -> Tuple.fromBytes(dataInPath.getValue())).asList(tupleResult).join(); actual.add(batch); continuation = tupleResult.get().getContinuation(); } @@ -578,7 +579,7 @@ void exportAllDataThroughKeySpacePathWrapper() { } @Test - void exportAllDataThroughKeySpacePathWrapperResolvedPaths() { + void exportAllDataThroughKeySpacePathWrapperRemainders() { final FDBDatabase database = dbExtension.getDatabase(); final EnvironmentKeySpace keySpace = EnvironmentKeySpace.setupSampleData(database); From f30cb54c002de9b3828c0583eee769355ab8215e Mon Sep 17 00:00:00 2001 From: Scott Dugas Date: Fri, 31 Oct 2025 11:07:07 -0700 Subject: [PATCH 10/21] Fix minor typo --- .../foundationdb/keyspace/KeySpacePathImportDataTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java index 8df2ff5f30..710cbdbe25 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java @@ -244,7 +244,7 @@ void importDataWithInvalidPath() { setSingleKey(keySpace1.path("root1").add("data", 1L), Tuple.from("record"), Tuple.from("other")); - // Now try to ipmort that into keySpace2 + // Now try to import that into keySpace2 List exportedData = getExportedData(keySpace1.path("root1")); assertBadImport(keySpace2.path("root2"), exportedData); } From 287ff3fe1be26cacda70382f78c9af3bb0a273e1 Mon Sep 17 00:00:00 2001 From: Scott Dugas Date: Fri, 31 Oct 2025 12:24:41 -0700 Subject: [PATCH 11/21] Reduce some duplication in import tests --- .../keyspace/KeySpacePathImportDataTest.java | 41 ++++++++----------- 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java index 710cbdbe25..7ede0f35b2 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java @@ -126,7 +126,7 @@ void importComprehensiveData() { context.commit(); } - copyData(root.path("company")); + copyData(root.path("company"), root.path("company")); // Verify all different KeyType values were handled correctly during import try (FDBRecordContext context = database.openContext()) { @@ -202,7 +202,7 @@ void importDataWithDirectoryLayer() { final KeySpacePath dataPath = root.path("tenant").add("user_id", 999L); setSingleKey(dataPath, Tuple.from("data"), Tuple.from("directory_test")); - copyData(root.path("tenant")); + copyData(root.path("tenant"), root.path("tenant")); verifySingleKey(dataPath, Tuple.from("data"), Tuple.from("directory_test")); } @@ -222,9 +222,8 @@ void importDataWithMismatchedPath() { setSingleKey(keySpace.path("root1").add("data", 1L), Tuple.from("record"), Tuple.from("other")); - // Now try to ipmort that into keySpace2 - List exportedData = getExportedData(keySpace.path("root1")); - assertBadImport(keySpace.path("root2"), exportedData); + // Now try to import that into keySpace2 + assertBadImport(keySpace.path("root1"), keySpace.path("root2")); } @Test @@ -245,8 +244,7 @@ void importDataWithInvalidPath() { setSingleKey(keySpace1.path("root1").add("data", 1L), Tuple.from("record"), Tuple.from("other")); // Now try to import that into keySpace2 - List exportedData = getExportedData(keySpace1.path("root1")); - assertBadImport(keySpace2.path("root2"), exportedData); + assertBadImport(keySpace1.path("root1"), keySpace2.path("root2")); } @Test @@ -263,12 +261,7 @@ void importDataWithSubdirectoryPath() { setSingleKey(level1Path, Tuple.from("item1"), Tuple.from("value1")); // Export from root, import to subdirectory - List exportedData = getExportedData(root.path("root")); - - clearPath(database, root.path("root")); - - // Import only to level1 subdirectory - importData(database, level1Path, exportedData); + copyData(root.path("root"), level1Path); verifySingleKey(level1Path, Tuple.from("item1"), Tuple.from("value1")); } @@ -288,12 +281,7 @@ void importDataWithSubdirectoryPathFailure() { setSingleKey(root.path("root").add("level1", 2L), Tuple.from("item1"), Tuple.from("value1")); // Export from root, import to subdirectory - List exportedData = getExportedData(root.path("root")); - - clearPath(database, root.path("root")); - - // Import only to level1 subdirectory - assertBadImport(level1Path, exportedData); + assertBadImport(root.path("root"), level1Path); } @Test @@ -329,7 +317,7 @@ void importDataWithWrapperClasses() { EnvironmentKeySpace.DataPath dataStore = keySpace.root().userid(100L).application("app1").dataStore(); - copyData(keySpace.root()); + copyData(keySpace.root(), keySpace.root()); verifySingleKey(dataStore, Tuple.from("record2", 0), Tuple.from("user100_app1_data2_0")); } @@ -385,19 +373,19 @@ private static Tuple getTuple(final FDBRecordContext context, final byte[] key) return Tuple.fromBytes(context.ensureActive().get(key).join()); } - private void copyData(final KeySpacePath path) { + private void copyData(final KeySpacePath sourcePath, KeySpacePath destinationPath) { // Export the data - final List exportedData = getExportedData(path); + final List exportedData = getExportedData(sourcePath); if (databases.size() > 1) { database = databases.get(1); } else { // Clear the data and import it back - clearPath(database, path); + clearPath(database, sourcePath); } // Import the data - importData(database, path, exportedData); + importData(database, destinationPath, exportedData); } private static void importData(final FDBDatabase database, final KeySpacePath path, final List exportedData) { @@ -407,6 +395,11 @@ private static void importData(final FDBDatabase database, final KeySpacePath pa } } + private void assertBadImport(KeySpacePath sourcePath, KeySpacePath destinationPath) { + List exportedData = getExportedData(sourcePath); + assertBadImport(destinationPath, exportedData); + } + private void assertBadImport(final KeySpacePath path, final List invalidData) { // Try to import into keySpace1 - should fail try (FDBRecordContext context = database.openContext()) { From 2cac9f305c9cf9b772faf01719f711967a36aef6 Mon Sep 17 00:00:00 2001 From: Scott Dugas Date: Sun, 9 Nov 2025 15:58:13 -0500 Subject: [PATCH 12/21] Remove unused parameter --- .../foundationdb/keyspace/KeySpacePathDataExportTest.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathDataExportTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathDataExportTest.java index 2e7292e6cf..44d8f1c6ae 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathDataExportTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathDataExportTest.java @@ -636,7 +636,7 @@ private static List exportAllData(final KeySpacePath pathToE final List reversed = pathToExport.exportAllData(context, null, ScanProperties.REVERSE_SCAN) .asList().join(); Collections.reverse(reversed); - assertDataInKeySpacePathEquals(context, asSingleExport, reversed); + assertDataInKeySpacePathEquals(asSingleExport, reversed); // Assert continuations work correctly final ScanProperties scanProperties = ScanProperties.FORWARD_SCAN.with(props -> props.setReturnedRowLimit(1)); @@ -656,12 +656,11 @@ private static List exportAllData(final KeySpacePath pathToE } } - assertDataInKeySpacePathEquals(context, asSingleExport, asContinuations); + assertDataInKeySpacePathEquals(asSingleExport, asContinuations); return asSingleExport; } - private static void assertDataInKeySpacePathEquals(final FDBRecordContext context, - final List expectedList, + private static void assertDataInKeySpacePathEquals(final List expectedList, final List actualList) { assertThat(actualList).zipSatisfy(expectedList, (actual, other) -> { From 77d85a2d948e93ee146b58baaa95411336a2e66d Mon Sep 17 00:00:00 2001 From: Scott Dugas Date: Sun, 9 Nov 2025 16:10:06 -0500 Subject: [PATCH 13/21] Create better helpers to simplify test --- .../keyspace/KeySpacePathImportDataTest.java | 68 ++++++++----------- 1 file changed, 29 insertions(+), 39 deletions(-) diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java index 7ede0f35b2..8300b0e51f 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java @@ -76,15 +76,15 @@ void importComprehensiveData() { UUID memberId = UUID.randomUUID(); KeySpace root = new KeySpace( - new KeySpaceDirectory("company", KeyType.STRING, rootUuid) // STRING with constant - .addSubdirectory(new KeySpaceDirectory("version", KeyType.LONG, 1L) // LONG with constant - .addSubdirectory(new KeySpaceDirectory("department", KeyType.STRING) // STRING variable - .addSubdirectory(new KeySpaceDirectory("employee_id", KeyType.LONG) // LONG variable - .addSubdirectory(new KeySpaceDirectory("binary_data", KeyType.BYTES) // BYTES - .addSubdirectory(new KeySpaceDirectory("null_section", KeyType.NULL) // NULL - .addSubdirectory(new KeySpaceDirectory("member", KeyType.UUID) // UUID - .addSubdirectory(new KeySpaceDirectory("active", KeyType.BOOLEAN) // BOOLEAN - .addSubdirectory(new KeySpaceDirectory("rating", KeyType.FLOAT)))))))))); // FLOAT + new KeySpaceDirectory("company", KeyType.STRING, rootUuid) + .addSubdirectory(new KeySpaceDirectory("version", KeyType.LONG, 1L) + .addSubdirectory(new KeySpaceDirectory("department", KeyType.STRING) + .addSubdirectory(new KeySpaceDirectory("employee_id", KeyType.LONG) + .addSubdirectory(new KeySpaceDirectory("binary_data", KeyType.BYTES) + .addSubdirectory(new KeySpaceDirectory("null_section", KeyType.NULL) + .addSubdirectory(new KeySpaceDirectory("member", KeyType.UUID) + .addSubdirectory(new KeySpaceDirectory("active", KeyType.BOOLEAN) + .addSubdirectory(new KeySpaceDirectory("rating", KeyType.FLOAT)))))))))); // Create comprehensive test data covering ALL KeyType values @@ -106,22 +106,13 @@ void importComprehensiveData() { .add("rating", 3.8f); try (FDBRecordContext context = database.openContext()) { - Transaction tr = context.ensureActive(); - - byte[] key1 = emp1Path.toSubspace(context).pack(Tuple.from("profile", "name")); - setToTuple(tr, key1, "John Doe"); - - byte[] key2 = emp2Path.toSubspace(context).pack(Tuple.from("profile", "name")); - setToTuple(tr, key2, "Jane Smith"); - - byte[] longKey = emp1Path.toSubspace(context).pack(Tuple.from("salary")); - setToTuple(tr, longKey, 75000); + setInPath(emp1Path, context, Tuple.from("profile", "name"), "John Doe"); + setInPath(emp2Path, context, Tuple.from("profile", "name"), "Jane Smith"); + setInPath(emp1Path, context, Tuple.from("salary"), 75000); + setInPath(emp1Path, context, Tuple.from("info", 42, true, "complex"), "Complex Test"); byte[] binaryKey = emp1Path.toSubspace(context).pack(Tuple.from("binary_metadata")); - tr.set(binaryKey, "binary_test_data".getBytes()); - - byte[] complexKey = emp1Path.toSubspace(context).pack(Tuple.from("info", 42, true, "complex")); - setToTuple(tr, complexKey, "Complex Test"); + context.ensureActive().set(binaryKey, "binary_test_data".getBytes()); context.commit(); } @@ -130,21 +121,18 @@ void importComprehensiveData() { // Verify all different KeyType values were handled correctly during import try (FDBRecordContext context = database.openContext()) { - byte[] key1 = emp1Path.toSubspace(context).pack(Tuple.from("profile", "name")); - byte[] key2 = emp2Path.toSubspace(context).pack(Tuple.from("profile", "name")); - assertEquals(Tuple.from("John Doe"), getTuple(context, key1)); - assertEquals(Tuple.from("Jane Smith"), getTuple(context, key2)); - - byte[] longKey = emp1Path.toSubspace(context).pack(Tuple.from("salary")); - assertEquals(Tuple.from(75000), getTuple(context, longKey)); + assertEquals(Tuple.from("John Doe"), + getTupleFromPath(context, emp1Path, Tuple.from("profile", "name"))); + assertEquals(Tuple.from("Jane Smith"), + getTupleFromPath(context, emp2Path, Tuple.from("profile", "name"))); + assertEquals(Tuple.from(75000), + getTupleFromPath(context, emp1Path, Tuple.from("salary"))); + assertEquals(Tuple.from("Complex Test"), + getTupleFromPath(context, emp1Path, Tuple.from("info", 42, true, "complex"))); // Verify BYTES data (raw binary, not in tuple) byte[] binaryKey = emp1Path.toSubspace(context).pack(Tuple.from("binary_metadata")); assertArrayEquals("binary_test_data".getBytes(), context.ensureActive().get(binaryKey).join()); - - // Verify complex hierarchy with mixed types in remainder (LONG, BOOLEAN, STRING) - byte[] complexKey = emp1Path.toSubspace(context).pack(Tuple.from("info", 42, true, "complex")); - assertEquals(Tuple.from("Complex Test"), getTuple(context, complexKey)); } } @@ -360,16 +348,18 @@ private void setSingleKey(KeySpacePath path, Tuple remainder, Tuple value) { private void verifySingleKey(KeySpacePath path, Tuple remainder, Tuple expected) { try (FDBRecordContext context = database.openContext()) { byte[] key = path.toSubspace(context).pack(remainder); - assertEquals(expected, getTuple(context, key)); + assertEquals(expected, Tuple.fromBytes(context.ensureActive().get(key).join())); } } - private static void setToTuple(final Transaction tr, final byte[] key1, Object items) { - byte[] value1 = Tuple.from(items).pack(); - tr.set(key1, value1); + private static void setInPath(final KeySpacePath path, final FDBRecordContext context, + final Tuple remainder, final Object value) { + byte[] key = path.toSubspace(context).pack(remainder); + context.ensureActive().set(key, Tuple.from(value).pack()); } - private static Tuple getTuple(final FDBRecordContext context, final byte[] key) { + private static Tuple getTupleFromPath(final FDBRecordContext context, final KeySpacePath path, final Tuple remainder) { + byte[] key = path.toSubspace(context).pack(remainder); return Tuple.fromBytes(context.ensureActive().get(key).join()); } From afc98fb37a68f1590505af5110cef462bd27f004 Mon Sep 17 00:00:00 2001 From: Scott Dugas Date: Mon, 10 Nov 2025 11:05:32 -0500 Subject: [PATCH 14/21] Minor cleanup after self-review --- .../provider/foundationdb/keyspace/KeySpacePath.java | 6 +++++- .../foundationdb/keyspace/KeySpacePathImpl.java | 3 ++- .../RecordCoreIllegalImportDataException.java | 3 +++ .../keyspace/KeySpacePathImportDataTest.java | 11 ++++------- .../record/test/FDBDatabaseExtension.java | 6 ++++++ 5 files changed, 20 insertions(+), 9 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePath.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePath.java index 6490e04870..f9e4b7e216 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePath.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePath.java @@ -413,13 +413,17 @@ default RecordCursor exportAllData(@Nonnull FDBRecordContext /** * Imports the provided data exported via {@link #exportAllData} into this {@code KeySpacePath}. + *

* This will validate that any data provided in {@code dataToImport} has a path that should be in this path, - * or one of the sub-directories, if not the future will complete exceptionally with + * or one of the sub-directories, if not, the future will complete exceptionally with * {@link RecordCoreIllegalImportDataException}. * If there is any data already existing under this path, the new data will overwrite if the keys are the same. * This will use the logical values in the {@link DataInKeySpacePath#getPath()} and * {@link DataInKeySpacePath#getRemainder()} to determine the key, rather * than the raw key, meaning that this will work even if the data was exported from a different cluster. + * Note, this will not correct for any cluster-specific data, other than {@link DirectoryLayerDirectory} data; + * for example, if you have versionstamps, that data will not align on the destination. + *

* @param context the transaction context in which to save the data * @param dataToImport the data to be saved to the database * @return a future to be completed once all data has been important. diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImpl.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImpl.java index 0076b70233..cdbccae613 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImpl.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImpl.java @@ -329,7 +329,8 @@ public CompletableFuture importData(@Nonnull FDBRecordContext context, if (!TupleHelpers.isPrefix(targetTuple, itemPathTuple)) { throw new RecordCoreIllegalImportDataException( "Data item path does not belong under target path", - "target", targetTuple, "item", itemPathTuple); + "target", targetTuple, + "item", itemPathTuple); } // Reconstruct the key using the path and remainder diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/RecordCoreIllegalImportDataException.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/RecordCoreIllegalImportDataException.java index 250e0bcbce..7879eefab7 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/RecordCoreIllegalImportDataException.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/RecordCoreIllegalImportDataException.java @@ -24,6 +24,9 @@ import javax.annotation.Nonnull; +/** + * Thrown if the data being imported into {@link KeySpacePath#importData} does not belong in that path. + */ public class RecordCoreIllegalImportDataException extends RecordCoreArgumentException { private static final long serialVersionUID = 1L; diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java index 8300b0e51f..86f49bf469 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java @@ -210,7 +210,7 @@ void importDataWithMismatchedPath() { setSingleKey(keySpace.path("root1").add("data", 1L), Tuple.from("record"), Tuple.from("other")); - // Now try to import that into keySpace2 + // Now try to import that into root2 assertBadImport(keySpace.path("root1"), keySpace.path("root2")); } @@ -231,7 +231,7 @@ void importDataWithInvalidPath() { setSingleKey(keySpace1.path("root1").add("data", 1L), Tuple.from("record"), Tuple.from("other")); - // Now try to import that into keySpace2 + // Now try to import that into root2 assertBadImport(keySpace1.path("root1"), keySpace2.path("root2")); } @@ -391,7 +391,6 @@ private void assertBadImport(KeySpacePath sourcePath, KeySpacePath destinationPa } private void assertBadImport(final KeySpacePath path, final List invalidData) { - // Try to import into keySpace1 - should fail try (FDBRecordContext context = database.openContext()) { Assertions.assertThatThrownBy(() -> path.importData(context, invalidData).join()) .isInstanceOf(CompletionException.class) @@ -412,11 +411,9 @@ private void clearPath(final FDBDatabase database, final KeySpacePath path) { @Nonnull private List getExportedData(final KeySpacePath path) { - List exportedData = new ArrayList<>(); try (FDBRecordContext context = database.openContext()) { - path.exportAllData(context, null, ScanProperties.FORWARD_SCAN) - .forEach(exportedData::add).join(); + return path.exportAllData(context, null, ScanProperties.FORWARD_SCAN) + .asList().join(); } - return exportedData; } } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/test/FDBDatabaseExtension.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/test/FDBDatabaseExtension.java index 42a9ee58d1..15fe3234a2 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/test/FDBDatabaseExtension.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/test/FDBDatabaseExtension.java @@ -165,6 +165,12 @@ public FDBDatabase getDatabase(@Nullable String clusterFile) { }); } + /** + * Return a random subset of the databases available. + * @param count the number of desired databases + * @return a random subset of the databases available. This may be less than {@code count} if there aren't that many + * databases available. + */ public List getDatabases(int count) { List clusterFiles = new ArrayList<>(FDBTestEnvironment.allClusterFiles()); Collections.shuffle(clusterFiles); From 7dfcb346c0886f9f57a96f46dccd0c362ba59360 Mon Sep 17 00:00:00 2001 From: Scott Dugas Date: Thu, 13 Nov 2025 11:43:25 -0500 Subject: [PATCH 15/21] Instrument the time to import each data entry --- .../record/provider/foundationdb/FDBStoreTimer.java | 2 ++ .../provider/foundationdb/keyspace/KeySpacePathImpl.java | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/FDBStoreTimer.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/FDBStoreTimer.java index c31508c8d2..387669244b 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/FDBStoreTimer.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/FDBStoreTimer.java @@ -182,6 +182,8 @@ public enum Events implements StoreTimer.Event { RANGE_SET_CONTAINS("range set contains key"), /** The amount of time checking if a {@link com.google.common.collect.RangeSet} is empty. */ RANGE_SET_IS_EMPTY("range set is empty"), + /** The amount of time importing a single KeyValue into a path. */ + IMPORT_DATA("import KeyValue"), /** The amount of time spent clearing the space taken by an index that has been removed from the meta-data. */ REMOVE_FORMER_INDEX("remove former index"), diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImpl.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImpl.java index cdbccae613..74b61d69c4 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImpl.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImpl.java @@ -29,6 +29,7 @@ import com.apple.foundationdb.record.cursors.LazyCursor; import com.apple.foundationdb.record.logging.LogMessageKeys; import com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext; +import com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer; import com.apple.foundationdb.record.provider.foundationdb.KeyValueCursor; import com.apple.foundationdb.subspace.Subspace; import com.apple.foundationdb.tuple.ByteArrayUtil; @@ -346,7 +347,7 @@ public CompletableFuture importData(@Nonnull FDBRecordContext context, return AsyncUtil.DONE; }); - importFutures.add(importFuture); + importFutures.add(context.instrument(FDBStoreTimer.Events.IMPORT_DATA, importFuture)); } return AsyncUtil.whenAll(importFutures); From e6b605ee2c6dd219dfbe6f17443efd3b55953d0d Mon Sep 17 00:00:00 2001 From: Scott Dugas Date: Thu, 13 Nov 2025 11:55:51 -0500 Subject: [PATCH 16/21] Change KeySpacePathImportDataTest to have separate DB fields In doing this, I had to rework the test for overwriting data, and in doing so, I decided it would be better to have 3 tests. --- .../keyspace/KeySpacePathImportDataTest.java | 123 ++++++++++++------ 1 file changed, 85 insertions(+), 38 deletions(-) diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java index 86f49bf469..02b3285e70 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java @@ -53,13 +53,18 @@ class KeySpacePathImportDataTest { @RegisterExtension final FDBDatabaseExtension dbExtension = new FDBDatabaseExtension(); - private FDBDatabase database; - private List databases; + private FDBDatabase sourceDatabase; + private FDBDatabase destinationDatabase; @BeforeEach void setUp() { - databases = dbExtension.getDatabases(2); - database = databases.get(0); + final List databases = dbExtension.getDatabases(2); + sourceDatabase = databases.get(0); + if (databases.size() > 1) { + destinationDatabase = databases.get(1); + } else { + destinationDatabase = sourceDatabase; + } } @Test @@ -105,7 +110,7 @@ void importComprehensiveData() { .add("active", false) .add("rating", 3.8f); - try (FDBRecordContext context = database.openContext()) { + try (FDBRecordContext context = sourceDatabase.openContext()) { setInPath(emp1Path, context, Tuple.from("profile", "name"), "John Doe"); setInPath(emp2Path, context, Tuple.from("profile", "name"), "Jane Smith"); setInPath(emp1Path, context, Tuple.from("salary"), 75000); @@ -120,7 +125,7 @@ void importComprehensiveData() { copyData(root.path("company"), root.path("company")); // Verify all different KeyType values were handled correctly during import - try (FDBRecordContext context = database.openContext()) { + try (FDBRecordContext context = destinationDatabase.openContext()) { assertEquals(Tuple.from("John Doe"), getTupleFromPath(context, emp1Path, Tuple.from("profile", "name"))); assertEquals(Tuple.from("Jane Smith"), @@ -144,7 +149,7 @@ void importEmptyData() { new KeySpaceDirectory("test", KeyType.STRING, UUID.randomUUID().toString())); KeySpacePath testPath = root.path("test"); - importData(database, testPath, Collections.emptyList()); // should not throw any exception + importData(testPath, Collections.emptyList()); // should not throw any exception assertTrue(getExportedData(testPath).isEmpty(), "there should not have been any data created"); @@ -153,31 +158,75 @@ void importEmptyData() { @Test void importOverwriteExistingData() { // Test importing data that overwrites existing keys - // Should verify that new data replaces old data when keys match final String rootUuid = UUID.randomUUID().toString(); KeySpace root = new KeySpace( new KeySpaceDirectory("root", KeyType.STRING, rootUuid) .addSubdirectory(new KeySpaceDirectory("data", KeyType.LONG))); final KeySpacePath dataPath = root.path("root").add("data", 1L); - setSingleKey(dataPath, Tuple.from("record"), Tuple.from("original_value")); - setSingleKey(dataPath, Tuple.from("other"), Tuple.from("other_value")); - // Create import data with same key but different value + // Import one value + importData(root.path("root"), List.of(new DataInKeySpacePath(dataPath, + Tuple.from("record"), Tuple.from("old_value").pack()))); + verifySingleKey(destinationDatabase, dataPath, Tuple.from("record"), Tuple.from("old_value")); + + // Import a different value + importData(root.path("root"), List.of(new DataInKeySpacePath(dataPath, + Tuple.from("record"), Tuple.from("new_value").pack()))); + + // Verify the data was overwritten + verifySingleKey(destinationDatabase, dataPath, Tuple.from("record"), Tuple.from("new_value")); + } + + @Test + void leaveExistingData() { + // Test importing data leaves other data under the path untouched + final String rootUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new KeySpaceDirectory("root", KeyType.STRING, rootUuid) + .addSubdirectory(new KeySpaceDirectory("data", KeyType.LONG))); + + final KeySpacePath dataPath = root.path("root").add("data", 1L); + + // Write one key + final Tuple remainder1 = Tuple.from("recordA"); + final Tuple value1 = Tuple.from("old_value"); + importData(root.path("root"), List.of(new DataInKeySpacePath(dataPath, + remainder1, value1.pack()))); + verifySingleKey(destinationDatabase, dataPath, remainder1, value1); + + // write a different key + final Tuple remainder2 = Tuple.from("recordB"); + final Tuple value2 = Tuple.from("new_value"); + importData(root.path("root"), List.of(new DataInKeySpacePath(dataPath, + remainder2, value2.pack()))); + + // Verify the data was overwritten + verifySingleKey(destinationDatabase, dataPath, remainder1, value1); + verifySingleKey(destinationDatabase, dataPath, remainder2, value2); + } + + @Test + void reimport() { + final String rootUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new KeySpaceDirectory("root", KeyType.STRING, rootUuid) + .addSubdirectory(new KeySpaceDirectory("data", KeyType.LONG))); + + final KeySpacePath dataPath = root.path("root").add("data", 1L); List importData = new ArrayList<>(); - importData.add(new DataInKeySpacePath(dataPath, - Tuple.from("record"), Tuple.from("new_value").pack())); + final Tuple remainder = Tuple.from("record"); + importData.add(new DataInKeySpacePath(dataPath, remainder, Tuple.from("value").pack())); // Verify we can re-import the data multiple times - importData(database, root.path("root"), importData); - importData(database, root.path("root"), importData); - importData(database, root.path("root"), importData); + importData(root.path("root"), importData); + importData(root.path("root"), importData); + importData(root.path("root"), importData); - // Verify the data was overwritten - verifySingleKey(dataPath, Tuple.from("record"), Tuple.from("new_value")); - verifySingleKey(dataPath, Tuple.from("other"), Tuple.from("other_value")); + verifySingleKey(destinationDatabase, dataPath, remainder, Tuple.from("value")); } + @Test void importDataWithDirectoryLayer() { // Test importing data into a keyspace using DirectoryLayer directories @@ -192,7 +241,7 @@ void importDataWithDirectoryLayer() { copyData(root.path("tenant"), root.path("tenant")); - verifySingleKey(dataPath, Tuple.from("data"), Tuple.from("directory_test")); + verifySingleKey(destinationDatabase, dataPath, Tuple.from("data"), Tuple.from("directory_test")); } @Test @@ -251,7 +300,7 @@ void importDataWithSubdirectoryPath() { // Export from root, import to subdirectory copyData(root.path("root"), level1Path); - verifySingleKey(level1Path, Tuple.from("item1"), Tuple.from("value1")); + verifySingleKey(destinationDatabase, level1Path, Tuple.from("item1"), Tuple.from("value1")); } @Test @@ -301,13 +350,13 @@ void importDataWithPartialMismatch() { void importDataWithWrapperClasses() { // Test importing data using wrapper classes like EnvironmentKeySpace // Should verify that wrapper functionality works correctly with import - final EnvironmentKeySpace keySpace = EnvironmentKeySpace.setupSampleData(database); + final EnvironmentKeySpace keySpace = EnvironmentKeySpace.setupSampleData(sourceDatabase); EnvironmentKeySpace.DataPath dataStore = keySpace.root().userid(100L).application("app1").dataStore(); copyData(keySpace.root(), keySpace.root()); - verifySingleKey(dataStore, Tuple.from("record2", 0), Tuple.from("user100_app1_data2_0")); + verifySingleKey(destinationDatabase, dataStore, Tuple.from("record2", 0), Tuple.from("user100_app1_data2_0")); } @Test @@ -329,23 +378,23 @@ void importDataWithDuplicateKeys() { new DataInKeySpacePath(dataPath, Tuple.from("item"), Tuple.from("final_value").pack()) ); - try (FDBRecordContext context = database.openContext()) { + try (FDBRecordContext context = destinationDatabase.openContext()) { root.path("root").importData(context, duplicateData).join(); context.commit(); } - verifySingleKey(dataPath, Tuple.from("item"), Tuple.from("final_value")); + verifySingleKey(destinationDatabase, dataPath, Tuple.from("item"), Tuple.from("final_value")); } private void setSingleKey(KeySpacePath path, Tuple remainder, Tuple value) { - try (FDBRecordContext context = database.openContext()) { + try (FDBRecordContext context = sourceDatabase.openContext()) { byte[] key = path.toSubspace(context).pack(remainder); context.ensureActive().set(key, value.pack()); context.commit(); } } - private void verifySingleKey(KeySpacePath path, Tuple remainder, Tuple expected) { + private void verifySingleKey(FDBDatabase database, KeySpacePath path, Tuple remainder, Tuple expected) { try (FDBRecordContext context = database.openContext()) { byte[] key = path.toSubspace(context).pack(remainder); assertEquals(expected, Tuple.fromBytes(context.ensureActive().get(key).join())); @@ -367,19 +416,17 @@ private void copyData(final KeySpacePath sourcePath, KeySpacePath destinationPat // Export the data final List exportedData = getExportedData(sourcePath); - if (databases.size() > 1) { - database = databases.get(1); - } else { + if (sourceDatabase == destinationDatabase) { // Clear the data and import it back - clearPath(database, sourcePath); + clearSourcePath(sourcePath); } // Import the data - importData(database, destinationPath, exportedData); + importData(destinationPath, exportedData); } - private static void importData(final FDBDatabase database, final KeySpacePath path, final List exportedData) { - try (FDBRecordContext context = database.openContext()) { + private void importData(final KeySpacePath path, final List exportedData) { + try (FDBRecordContext context = destinationDatabase.openContext()) { path.importData(context, exportedData).join(); context.commit(); } @@ -391,15 +438,15 @@ private void assertBadImport(KeySpacePath sourcePath, KeySpacePath destinationPa } private void assertBadImport(final KeySpacePath path, final List invalidData) { - try (FDBRecordContext context = database.openContext()) { + try (FDBRecordContext context = destinationDatabase.openContext()) { Assertions.assertThatThrownBy(() -> path.importData(context, invalidData).join()) .isInstanceOf(CompletionException.class) .hasCauseInstanceOf(RecordCoreIllegalImportDataException.class); } } - private void clearPath(final FDBDatabase database, final KeySpacePath path) { - try (FDBRecordContext context = database.openContext()) { + private void clearSourcePath(final KeySpacePath path) { + try (FDBRecordContext context = sourceDatabase.openContext()) { Transaction tr = context.ensureActive(); tr.clear(path.toSubspace(context).range()); context.commit(); @@ -411,7 +458,7 @@ private void clearPath(final FDBDatabase database, final KeySpacePath path) { @Nonnull private List getExportedData(final KeySpacePath path) { - try (FDBRecordContext context = database.openContext()) { + try (FDBRecordContext context = sourceDatabase.openContext()) { return path.exportAllData(context, null, ScanProperties.FORWARD_SCAN) .asList().join(); } From 7a4e12768bf49afc9c5116ecc876ea98b616cbdf Mon Sep 17 00:00:00 2001 From: Scott Dugas Date: Thu, 13 Nov 2025 12:04:46 -0500 Subject: [PATCH 17/21] Change tests that copyData to all be parameterized Now all will both run by copying back to the same cluster, and copying between clusters. --- .../keyspace/KeySpacePathImportDataTest.java | 91 +++++++++++-------- 1 file changed, 55 insertions(+), 36 deletions(-) diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java index 02b3285e70..520f4e5c85 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java @@ -33,6 +33,8 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import javax.annotation.Nonnull; import java.util.ArrayList; @@ -42,6 +44,7 @@ import java.util.UUID; import java.util.concurrent.CompletionException; +import static org.assertj.core.api.Assumptions.assumeThat; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -67,8 +70,9 @@ void setUp() { } } - @Test - void importComprehensiveData() { + @ParameterizedTest + @EnumSource(CopyConfig.class) + void importComprehensiveData(CopyConfig copyConfig) { // Test importing data covering ALL KeyType enum values in a single complex directory structure: // - NULL, BYTES, STRING, LONG, FLOAT, DOUBLE, BOOLEAN, UUID // - Constant value directories @@ -122,10 +126,10 @@ void importComprehensiveData() { context.commit(); } - copyData(root.path("company"), root.path("company")); + final FDBDatabase targetDatabase = copyData(root.path("company"), root.path("company"), copyConfig); // Verify all different KeyType values were handled correctly during import - try (FDBRecordContext context = destinationDatabase.openContext()) { + try (FDBRecordContext context = targetDatabase.openContext()) { assertEquals(Tuple.from("John Doe"), getTupleFromPath(context, emp1Path, Tuple.from("profile", "name"))); assertEquals(Tuple.from("Jane Smith"), @@ -149,7 +153,7 @@ void importEmptyData() { new KeySpaceDirectory("test", KeyType.STRING, UUID.randomUUID().toString())); KeySpacePath testPath = root.path("test"); - importData(testPath, Collections.emptyList()); // should not throw any exception + importData(destinationDatabase, testPath, Collections.emptyList()); // should not throw any exception assertTrue(getExportedData(testPath).isEmpty(), "there should not have been any data created"); @@ -166,12 +170,12 @@ void importOverwriteExistingData() { final KeySpacePath dataPath = root.path("root").add("data", 1L); // Import one value - importData(root.path("root"), List.of(new DataInKeySpacePath(dataPath, + importData(destinationDatabase, root.path("root"), List.of(new DataInKeySpacePath(dataPath, Tuple.from("record"), Tuple.from("old_value").pack()))); verifySingleKey(destinationDatabase, dataPath, Tuple.from("record"), Tuple.from("old_value")); // Import a different value - importData(root.path("root"), List.of(new DataInKeySpacePath(dataPath, + importData(destinationDatabase, root.path("root"), List.of(new DataInKeySpacePath(dataPath, Tuple.from("record"), Tuple.from("new_value").pack()))); // Verify the data was overwritten @@ -191,14 +195,14 @@ void leaveExistingData() { // Write one key final Tuple remainder1 = Tuple.from("recordA"); final Tuple value1 = Tuple.from("old_value"); - importData(root.path("root"), List.of(new DataInKeySpacePath(dataPath, + importData(destinationDatabase, root.path("root"), List.of(new DataInKeySpacePath(dataPath, remainder1, value1.pack()))); verifySingleKey(destinationDatabase, dataPath, remainder1, value1); // write a different key final Tuple remainder2 = Tuple.from("recordB"); final Tuple value2 = Tuple.from("new_value"); - importData(root.path("root"), List.of(new DataInKeySpacePath(dataPath, + importData(destinationDatabase, root.path("root"), List.of(new DataInKeySpacePath(dataPath, remainder2, value2.pack()))); // Verify the data was overwritten @@ -219,16 +223,16 @@ void reimport() { importData.add(new DataInKeySpacePath(dataPath, remainder, Tuple.from("value").pack())); // Verify we can re-import the data multiple times - importData(root.path("root"), importData); - importData(root.path("root"), importData); - importData(root.path("root"), importData); + importData(destinationDatabase, root.path("root"), importData); + importData(destinationDatabase, root.path("root"), importData); + importData(destinationDatabase, root.path("root"), importData); verifySingleKey(destinationDatabase, dataPath, remainder, Tuple.from("value")); } - - @Test - void importDataWithDirectoryLayer() { + @ParameterizedTest + @EnumSource(CopyConfig.class) + void importDataWithDirectoryLayer(final CopyConfig copyConfig) { // Test importing data into a keyspace using DirectoryLayer directories // Should verify that DirectoryLayer mappings work correctly during import final String tenantUuid = UUID.randomUUID().toString(); @@ -239,9 +243,9 @@ void importDataWithDirectoryLayer() { final KeySpacePath dataPath = root.path("tenant").add("user_id", 999L); setSingleKey(dataPath, Tuple.from("data"), Tuple.from("directory_test")); - copyData(root.path("tenant"), root.path("tenant")); + final FDBDatabase targetDatabase = copyData(root.path("tenant"), root.path("tenant"), copyConfig); - verifySingleKey(destinationDatabase, dataPath, Tuple.from("data"), Tuple.from("directory_test")); + verifySingleKey(targetDatabase, dataPath, Tuple.from("data"), Tuple.from("directory_test")); } @Test @@ -284,8 +288,9 @@ void importDataWithInvalidPath() { assertBadImport(keySpace1.path("root1"), keySpace2.path("root2")); } - @Test - void importDataWithSubdirectoryPath() { + @ParameterizedTest + @EnumSource(CopyConfig.class) + void importDataWithSubdirectoryPath(final CopyConfig copyConfig) { // Test importing data where the target path is a subdirectory of the import path // Should succeed only if all the data is in the subdirectory final String rootUuid = UUID.randomUUID().toString(); @@ -298,9 +303,9 @@ void importDataWithSubdirectoryPath() { setSingleKey(level1Path, Tuple.from("item1"), Tuple.from("value1")); // Export from root, import to subdirectory - copyData(root.path("root"), level1Path); + final FDBDatabase targetDatabase = copyData(root.path("root"), level1Path, copyConfig); - verifySingleKey(destinationDatabase, level1Path, Tuple.from("item1"), Tuple.from("value1")); + verifySingleKey(targetDatabase, level1Path, Tuple.from("item1"), Tuple.from("value1")); } @Test @@ -346,17 +351,18 @@ void importDataWithPartialMismatch() { assertBadImport(keySpace.path("root1"), mixedData); } - @Test - void importDataWithWrapperClasses() { + @ParameterizedTest + @EnumSource(CopyConfig.class) + void importDataWithWrapperClasses(final CopyConfig copyConfig) { // Test importing data using wrapper classes like EnvironmentKeySpace // Should verify that wrapper functionality works correctly with import final EnvironmentKeySpace keySpace = EnvironmentKeySpace.setupSampleData(sourceDatabase); EnvironmentKeySpace.DataPath dataStore = keySpace.root().userid(100L).application("app1").dataStore(); - copyData(keySpace.root(), keySpace.root()); + final FDBDatabase targetDatabase = copyData(keySpace.root(), keySpace.root(), copyConfig); - verifySingleKey(destinationDatabase, dataStore, Tuple.from("record2", 0), Tuple.from("user100_app1_data2_0")); + verifySingleKey(targetDatabase, dataStore, Tuple.from("record2", 0), Tuple.from("user100_app1_data2_0")); } @Test @@ -378,10 +384,7 @@ void importDataWithDuplicateKeys() { new DataInKeySpacePath(dataPath, Tuple.from("item"), Tuple.from("final_value").pack()) ); - try (FDBRecordContext context = destinationDatabase.openContext()) { - root.path("root").importData(context, duplicateData).join(); - context.commit(); - } + importData(destinationDatabase, root.path("root"), duplicateData); verifySingleKey(destinationDatabase, dataPath, Tuple.from("item"), Tuple.from("final_value")); } @@ -412,21 +415,37 @@ private static Tuple getTupleFromPath(final FDBRecordContext context, final KeyS return Tuple.fromBytes(context.ensureActive().get(key).join()); } - private void copyData(final KeySpacePath sourcePath, KeySpacePath destinationPath) { + enum CopyConfig { + WithinCluster, + BetweenClusters + } + + private FDBDatabase copyData(final KeySpacePath sourcePath, KeySpacePath destinationPath, final CopyConfig copyConfig) { // Export the data final List exportedData = getExportedData(sourcePath); - if (sourceDatabase == destinationDatabase) { - // Clear the data and import it back - clearSourcePath(sourcePath); + FDBDatabase targetDatabase; + switch (copyConfig) { + case WithinCluster: + // Clear the data and import it back + clearSourcePath(sourcePath); + targetDatabase = sourceDatabase; + break; + case BetweenClusters: + assumeThat(destinationDatabase).isNotEqualTo(sourceDatabase); + targetDatabase = destinationDatabase; + break; + default: + throw new IllegalStateException("Unexpected value: " + copyConfig); } // Import the data - importData(destinationPath, exportedData); + importData(targetDatabase, destinationPath, exportedData); + return targetDatabase; } - private void importData(final KeySpacePath path, final List exportedData) { - try (FDBRecordContext context = destinationDatabase.openContext()) { + private void importData(FDBDatabase targetDatabase, final KeySpacePath path, final List exportedData) { + try (FDBRecordContext context = targetDatabase.openContext()) { path.importData(context, exportedData).join(); context.commit(); } From 40055dde268e1190703bf7cedf7948c93025f152 Mon Sep 17 00:00:00 2001 From: Scott Dugas Date: Thu, 13 Nov 2025 16:59:20 -0500 Subject: [PATCH 18/21] Test Importing a lot of data at once --- .../keyspace/KeySpacePathImportDataTest.java | 68 ++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java index 520f4e5c85..56c3d93ec3 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java @@ -23,10 +23,12 @@ import com.apple.foundationdb.Transaction; import com.apple.foundationdb.record.ScanProperties; import com.apple.foundationdb.record.provider.foundationdb.FDBDatabase; +import com.apple.foundationdb.record.provider.foundationdb.FDBExceptions; import com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext; import com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpaceDirectory.KeyType; import com.apple.foundationdb.record.test.FDBDatabaseExtension; import com.apple.foundationdb.tuple.Tuple; +import com.apple.test.BooleanSource; import com.apple.test.Tags; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -40,6 +42,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Iterator; import java.util.List; import java.util.UUID; import java.util.concurrent.CompletionException; @@ -47,6 +50,7 @@ import static org.assertj.core.api.Assumptions.assumeThat; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -389,6 +393,31 @@ void importDataWithDuplicateKeys() { verifySingleKey(destinationDatabase, dataPath, Tuple.from("item"), Tuple.from("final_value")); } + @ParameterizedTest + @BooleanSource({"manyPaths", "useDirectoryLayer"}) + void importTooMuchData(boolean manyPaths, boolean useDirectoryLayer) { + // Test importing too much data within a single path + // we do it both with 1 path, and many paths, because resolving a directory layer requires reads to the + // database which should slow things down, and result in different errors + // using a standard path we can write data until we get OOM, and if we have a more reasonable limit, we'll + // get a transaction-too-large. + assumeThat(!manyPaths || !useDirectoryLayer) + .as("https://github.com/FoundationDB/fdb-record-layer/issues/3751") + .isTrue(); + final String rootUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new KeySpaceDirectory("root", KeyType.STRING, rootUuid) + .addSubdirectory(useDirectoryLayer + ? new DirectoryLayerDirectory("data") + : new KeySpaceDirectory("data", KeyType.STRING))); + + final KeySpacePath rootPath = root.path("root"); + + final DataGenerator data = new DataGenerator(rootPath, 100_000, manyPaths); + assertThrows(FDBExceptions.FDBStoreTransactionSizeException.class, () -> + importData(destinationDatabase, rootPath, data)); + } + private void setSingleKey(KeySpacePath path, Tuple remainder, Tuple value) { try (FDBRecordContext context = sourceDatabase.openContext()) { byte[] key = path.toSubspace(context).pack(remainder); @@ -444,7 +473,7 @@ private FDBDatabase copyData(final KeySpacePath sourcePath, KeySpacePath destina return targetDatabase; } - private void importData(FDBDatabase targetDatabase, final KeySpacePath path, final List exportedData) { + private void importData(FDBDatabase targetDatabase, final KeySpacePath path, final Iterable exportedData) { try (FDBRecordContext context = targetDatabase.openContext()) { path.importData(context, exportedData).join(); context.commit(); @@ -482,4 +511,41 @@ private List getExportedData(final KeySpacePath path) { .asList().join(); } } + + private static class DataGenerator implements Iterable { + + private final KeySpacePath rootPath; + private final int limit; + private final boolean manyPaths; + + public DataGenerator(final KeySpacePath rootPath, int limit, final boolean manyPaths) { + this.rootPath = rootPath; + this.limit = limit; + this.manyPaths = manyPaths; + } + + @Override + public Iterator iterator() { + return new Iterator<>() { + int counter = 0; + + @Override + public boolean hasNext() { + return counter < limit; + } + + @Override + public DataInKeySpacePath next() { + counter++; + // importing 100_000 different paths in a transaction is unreasonable, beyond what we need for + // https://github.com/FoundationDB/fdb-record-layer/issues/3751 + final KeySpacePath path = manyPaths ? rootPath.add("data", "path " + counter % 1_000) + : rootPath.add("data", "OnlyPath"); + return new DataInKeySpacePath( + path, + Tuple.from(counter), Tuple.from("value", counter).pack()); + } + }; + } + } } From 4c9b11e4228b8368e63d2ae9fd0f0847f5452d1d Mon Sep 17 00:00:00 2001 From: Scott Dugas Date: Thu, 13 Nov 2025 17:24:54 -0500 Subject: [PATCH 19/21] Add a test of importing a bunch of data Also needed equals & hashCode & toString on DataInKeySpacePathTest so added tests for those --- .../keyspace/DataInKeySpacePath.java | 21 ++++ .../keyspace/DataInKeySpacePathTest.java | 103 ++++++++++++++++++ .../keyspace/KeySpacePathImportDataTest.java | 36 ++++-- 3 files changed, 152 insertions(+), 8 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/DataInKeySpacePath.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/DataInKeySpacePath.java index f2d2284730..afbd34599a 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/DataInKeySpacePath.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/DataInKeySpacePath.java @@ -24,10 +24,13 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.RecordCoreArgumentException; import com.apple.foundationdb.record.logging.LogMessageKeys; +import com.apple.foundationdb.tuple.ByteArrayUtil2; import com.apple.foundationdb.tuple.Tuple; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.Arrays; +import java.util.Objects; /** * Class representing a {@link KeyValue} pair within in {@link KeySpacePath}. @@ -67,4 +70,22 @@ public Tuple getRemainder() { return remainder; } + @Override + public boolean equals(final Object o) { + if (o == null || getClass() != o.getClass()) { + return false; + } + final DataInKeySpacePath that = (DataInKeySpacePath)o; + return Objects.equals(path, that.path) && Objects.equals(remainder, that.remainder) && Arrays.equals(value, that.value); + } + + @Override + public int hashCode() { + return Objects.hash(path, remainder, Arrays.hashCode(value)); + } + + @Override + public String toString() { + return path + "+" + remainder + "->" + ByteArrayUtil2.loggable(value); + } } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/DataInKeySpacePathTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/DataInKeySpacePathTest.java index 6e8f0fb0b7..33fd2bdff7 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/DataInKeySpacePathTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/DataInKeySpacePathTest.java @@ -31,9 +31,11 @@ import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * Tests for {@link DataInKeySpacePath}. @@ -118,4 +120,105 @@ void nullValue() { assertThrows(RecordCoreArgumentException.class, () -> new DataInKeySpacePath(testPath, null, valueBytes)); } + + @Test + void testEquals() { + KeySpace root = new KeySpace( + new KeySpaceDirectory("test", KeyType.STRING, UUID.randomUUID().toString())); + + KeySpacePath testPath = root.path("test"); + Tuple remainder1 = Tuple.from("key1", "key2"); + byte[] value1 = Tuple.from("value1").pack(); + + DataInKeySpacePath data1 = new DataInKeySpacePath(testPath, remainder1, value1); + DataInKeySpacePath data2 = new DataInKeySpacePath(testPath, remainder1, value1); + + // Reflexive: object equals itself + assertEquals(data1, data1); + + // Symmetric: a.equals(b) implies b.equals(a) + assertEquals(data1, data2); + assertEquals(data2, data1); + + // Test with different remainder + Tuple remainder2 = Tuple.from("different", "key"); + DataInKeySpacePath data3 = new DataInKeySpacePath(testPath, remainder2, value1); + assertNotEquals(data1, data3); + + // Test with different value + byte[] value2 = Tuple.from("value2").pack(); + DataInKeySpacePath data4 = new DataInKeySpacePath(testPath, remainder1, value2); + assertNotEquals(data1, data4); + + // Test with different path + KeySpace root2 = new KeySpace( + new KeySpaceDirectory("test", KeyType.STRING, UUID.randomUUID().toString())); + KeySpacePath testPath2 = root2.path("test"); + DataInKeySpacePath data5 = new DataInKeySpacePath(testPath2, remainder1, value1); + assertNotEquals(data1, data5); + + // Test with null remainder + DataInKeySpacePath data6 = new DataInKeySpacePath(testPath, null, value1); + DataInKeySpacePath data7 = new DataInKeySpacePath(testPath, null, value1); + assertEquals(data6, data7); + assertNotEquals(data1, data6); + + // Test with null object + assertNotEquals(data1, null); + + // Test with different class + assertNotEquals(data1, "not a DataInKeySpacePath"); + } + + @Test + void testHashCode() { + KeySpace root = new KeySpace( + new KeySpaceDirectory("test", KeyType.STRING, UUID.randomUUID().toString())); + + KeySpacePath testPath = root.path("test"); + Tuple remainder = Tuple.from("key1", "key2"); + byte[] value = Tuple.from("value1").pack(); + + DataInKeySpacePath data1 = new DataInKeySpacePath(testPath, remainder, value); + DataInKeySpacePath data2 = new DataInKeySpacePath(testPath, remainder, value); + + // Equal objects must have equal hash codes + assertEquals(data1.hashCode(), data2.hashCode()); + + // Test with null remainder + DataInKeySpacePath data3 = new DataInKeySpacePath(testPath, null, value); + DataInKeySpacePath data4 = new DataInKeySpacePath(testPath, null, value); + assertEquals(data3.hashCode(), data4.hashCode()); + + // Different objects should generally have different hash codes (not required, but good practice) + Tuple remainder2 = Tuple.from("different", "key"); + DataInKeySpacePath data5 = new DataInKeySpacePath(testPath, remainder2, value); + assertNotEquals(data1.hashCode(), data5.hashCode()); + } + + @Test + void testToString() { + final String rootUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new KeySpaceDirectory("test", KeyType.STRING, rootUuid)); + + KeySpacePath testPath = root.path("test"); + Tuple remainder = Tuple.from("key1", "key2"); + byte[] value = Tuple.from("value1").pack(); + + DataInKeySpacePath data = new DataInKeySpacePath(testPath, remainder, value); + + String result = data.toString(); + + // Verify the string contains expected components + assertTrue(result.contains(rootUuid)); + assertTrue(result.contains("test")); + assertTrue(result.contains("key1"), "toString should contain remainder elements"); + assertTrue(result.contains("key2"), "toString should contain remainder elements"); + + // Test with null remainder + DataInKeySpacePath dataWithNullRemainder = new DataInKeySpacePath(testPath, null, value); + String resultWithNull = dataWithNullRemainder.toString(); + assertTrue(resultWithNull.contains("null"), "toString should contain 'null' for null remainder"); + } } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java index 56c3d93ec3..9952422626 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java @@ -47,6 +47,7 @@ import java.util.UUID; import java.util.concurrent.CompletionException; +import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assumptions.assumeThat; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -159,7 +160,7 @@ void importEmptyData() { KeySpacePath testPath = root.path("test"); importData(destinationDatabase, testPath, Collections.emptyList()); // should not throw any exception - assertTrue(getExportedData(testPath).isEmpty(), + assertTrue(getExportedData(sourceDatabase, testPath).isEmpty(), "there should not have been any data created"); } @@ -349,8 +350,8 @@ void importDataWithPartialMismatch() { setSingleKey(path2, Tuple.from("data"), Tuple.from("data2")); List mixedData = new ArrayList<>(); - mixedData.addAll(getExportedData(path1)); - mixedData.addAll(getExportedData(path2)); + mixedData.addAll(getExportedData(sourceDatabase, path1)); + mixedData.addAll(getExportedData(sourceDatabase, path2)); assertBadImport(keySpace.path("root1"), mixedData); } @@ -418,6 +419,25 @@ void importTooMuchData(boolean manyPaths, boolean useDirectoryLayer) { importData(destinationDatabase, rootPath, data)); } + @ParameterizedTest + @BooleanSource({"manyPaths", "useDirectoryLayer"}) + void importALotOfData(boolean manyPaths, boolean useDirectoryLayer) { + // Test importing a lot of data within a single path + final String rootUuid = UUID.randomUUID().toString(); + KeySpace root = new KeySpace( + new KeySpaceDirectory("root", KeyType.STRING, rootUuid) + .addSubdirectory(useDirectoryLayer + ? new DirectoryLayerDirectory("data") + : new KeySpaceDirectory("data", KeyType.STRING))); + + final KeySpacePath rootPath = root.path("root"); + + final DataGenerator data = new DataGenerator(rootPath, 10_000, manyPaths); + importData(destinationDatabase, rootPath, data); + final List exportedData = getExportedData(destinationDatabase, rootPath); + assertThat(exportedData).containsExactlyInAnyOrderElementsOf(data); + } + private void setSingleKey(KeySpacePath path, Tuple remainder, Tuple value) { try (FDBRecordContext context = sourceDatabase.openContext()) { byte[] key = path.toSubspace(context).pack(remainder); @@ -451,7 +471,7 @@ enum CopyConfig { private FDBDatabase copyData(final KeySpacePath sourcePath, KeySpacePath destinationPath, final CopyConfig copyConfig) { // Export the data - final List exportedData = getExportedData(sourcePath); + final List exportedData = getExportedData(sourceDatabase, sourcePath); FDBDatabase targetDatabase; switch (copyConfig) { @@ -481,7 +501,7 @@ private void importData(FDBDatabase targetDatabase, final KeySpacePath path, fin } private void assertBadImport(KeySpacePath sourcePath, KeySpacePath destinationPath) { - List exportedData = getExportedData(sourcePath); + List exportedData = getExportedData(sourceDatabase, sourcePath); assertBadImport(destinationPath, exportedData); } @@ -500,13 +520,13 @@ private void clearSourcePath(final KeySpacePath path) { context.commit(); } // just an extra check to make sure the test is working as expected - assertTrue(getExportedData(path).isEmpty(), + assertTrue(getExportedData(sourceDatabase, path).isEmpty(), "Clearing should remove all the data"); } @Nonnull - private List getExportedData(final KeySpacePath path) { - try (FDBRecordContext context = sourceDatabase.openContext()) { + private List getExportedData(FDBDatabase targetDatabase, final KeySpacePath path) { + try (FDBRecordContext context = targetDatabase.openContext()) { return path.exportAllData(context, null, ScanProperties.FORWARD_SCAN) .asList().join(); } From 1a237631d7f087ed433419d528b0acce01072a37 Mon Sep 17 00:00:00 2001 From: Scott Dugas Date: Fri, 14 Nov 2025 09:13:15 -0500 Subject: [PATCH 20/21] Add comment to clearPath about how it doesn't clear DirectoryLayer --- .../foundationdb/keyspace/KeySpacePathImportDataTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java index 9952422626..cdeb7042b6 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java @@ -514,6 +514,7 @@ private void assertBadImport(final KeySpacePath path, final List Date: Fri, 14 Nov 2025 09:14:53 -0500 Subject: [PATCH 21/21] Rename FDBDatabaseExtension.getDatabases -> getRandomDatabaseSubset --- .../foundationdb/keyspace/KeySpacePathImportDataTest.java | 2 +- .../foundationdb/record/test/FDBDatabaseExtension.java | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java index cdeb7042b6..1e98127f1b 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImportDataTest.java @@ -66,7 +66,7 @@ class KeySpacePathImportDataTest { @BeforeEach void setUp() { - final List databases = dbExtension.getDatabases(2); + final List databases = dbExtension.getRandomDatabaseSubset(2); sourceDatabase = databases.get(0); if (databases.size() > 1) { destinationDatabase = databases.get(1); diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/test/FDBDatabaseExtension.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/test/FDBDatabaseExtension.java index 15fe3234a2..282f09ebbe 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/test/FDBDatabaseExtension.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/test/FDBDatabaseExtension.java @@ -167,14 +167,14 @@ public FDBDatabase getDatabase(@Nullable String clusterFile) { /** * Return a random subset of the databases available. - * @param count the number of desired databases - * @return a random subset of the databases available. This may be less than {@code count} if there aren't that many + * @param maxCount the number of desired databases + * @return a random subset of the databases available. This may be less than {@code maxCount} if there aren't that many * databases available. */ - public List getDatabases(int count) { + public List getRandomDatabaseSubset(int maxCount) { List clusterFiles = new ArrayList<>(FDBTestEnvironment.allClusterFiles()); Collections.shuffle(clusterFiles); - return clusterFiles.stream().limit(count).map(this::getDatabase).collect(Collectors.toList()); + return clusterFiles.stream().limit(maxCount).map(this::getDatabase).collect(Collectors.toList()); } public void checkForOpenContexts() {