Add support for testing against multiple FDB clusters (#3575)

This changes the way we specify cluster files in tests. It replaces the
`fdb-environment.properties` with `fdb-environment.yaml`. The new format
allows for specifying multiple cluster files.

The majority of tests now randomly chose a file at the start of the test
from the list of the available ones.
The following sub-projects set the environment variable in the gradle
task because getting the project under test to support custom cluster
files seemed like a bigger task:
- fdb-relational-cli
- fdb-relational-jdbc

Our CI builds also, now, setup two FDB clusters, and create an
`fdb-environment.properties`.

I added one new "test" to validate this all works. It connects to two
different clusters. It `assume`s that there are at least two clusters.
At some point in the future we will want to make that a stronger
contract to ensure that a bug in our CI setup doesn't cause us to not
run all the multi-cluster tests.

I created Draft PR:
#3623 to ensure
that everything still works if you don't specify `fdb-environment.yaml`
I also created this PR off the same version of main to confirm the test
counts didn't change:
#3621 Apparently we
do merge main to do the tests so PR #3621 saw fewer tests than this
branch when it was last run. I re-ran and they may be the same depending
on whether new tests were merged. I validated some of the differences,
and they look to be expected.

Resolves: #3574

Author: ScottDugas

.gitignore

@@ -80,6 +80,7 @@ protogen/
 
 # Local FDB settings
 fdb-environment.properties
+fdb-environment.yaml
 
 # Docker local support
 run/

actions/setup-fdb/action.yml

@@ -31,12 +31,57 @@ runs:
     - name: Install FDB Server
       shell: bash
       run: sudo dpkg -i ~/.fdb-cache/${{ steps.fdb_filenames.outputs.client_deb }} ~/.fdb-cache/${{ steps.fdb_filenames.outputs.server_deb }}
-    - name: Fix FDB Network Addresses
+    - name: Stop default fdb
       shell: bash
-      run: sudo sed -i -e "s/public_address = auto:\$ID/public_address = 127.0.0.1:\$ID/g" -e "s/listen_address = public/listen_address = 0.0.0.0:\$ID/g" /etc/foundationdb/foundationdb.conf
-    - name: Start FDB Server
+      run: sudo service foundationdb stop
+
+    - name: Create cluster1 config
+      shell: bash
+      run: |
+        sudo cp /etc/foundationdb/foundationdb.conf /etc/foundationdb/foundationdb1.conf
+
+        sudo sed -i -e "s/\/etc\/foundationdb\/fdb.cluster/\/etc\/foundationdb\/fdb1.cluster/g" \
+        -e "s/\/var\/log\/foundationdb/\/var\/log\/foundationdb1/" \
+        -e "s/fdbserver.4500/fdbserver.4600/g" \
+        /etc/foundationdb/foundationdb1.conf
+        
+        sudo bash -c "echo 'fdb1:$(mktemp -u XXXXXXXX)@127.0.0.1:4600' > /etc/foundationdb/fdb1.cluster"
+    - name: Create cluster2 config
       shell: bash
-      run: sudo /usr/lib/foundationdb/fdbmonitor /etc/foundationdb/foundationdb.conf --daemonize
-    - name: Switch FDB to SSD
+      run: |
+        sudo cp /etc/foundationdb/foundationdb.conf /etc/foundationdb/foundationdb2.conf
+        
+        sudo sed -i -e "s/\/etc\/foundationdb\/fdb.cluster/\/etc\/foundationdb\/fdb2.cluster/g" \
+        -e "s/\/var\/log\/foundationdb/\/var\/log\/foundationdb2/" \
+        -e "s/fdbserver.4500/fdbserver.4700/g" \
+        /etc/foundationdb/foundationdb2.conf
+        
+        sudo bash -c "echo 'fdb2:$(mktemp -u XXXXXXXX)@127.0.0.1:4700' > /etc/foundationdb/fdb2.cluster"
+
+    - name: create dirs & set permissions
       shell: bash
-      run: fdbcli --exec "configure single ssd storage_migration_type=aggressive; status"
+      run: |
+        sudo mkdir /var/log/foundationdb1 /var/log/foundationdb2
+        sudo chown foundationdb:foundationdb /etc/foundationdb/fdb1.cluster /etc/foundationdb/fdb2.cluster /var/log/foundationdb1 /var/log/foundationdb2
+        sudo chmod 664 /etc/foundationdb/fdb1.cluster /etc/foundationdb/fdb2.cluster
+        sudo chmod 700 /var/log/foundationdb1 /var/log/foundationdb2
+
+    - name: Start FDB Server 1
+      shell: bash
+      run: sudo /usr/lib/foundationdb/fdbmonitor --conffile /etc/foundationdb/foundationdb1.conf --lockfile /var/run/fdbmonitor1.pid --loggroup fdb1 --daemonize
+    - name: Start FDB Server 2
+      shell: bash
+      run: sudo /usr/lib/foundationdb/fdbmonitor --conffile /etc/foundationdb/foundationdb2.conf --lockfile /var/run/fdbmonitor2.pid --loggroup fdb2 --daemonize
+
+    - name: Switch FDB 1 to SSD
+      shell: bash
+      run: fdbcli -C /etc/foundationdb/fdb1.cluster --exec "configure new single ssd storage_migration_type=aggressive; status"
+    - name: Switch FDB 2 to SSD
+      shell: bash
+      run: fdbcli -C /etc/foundationdb/fdb2.cluster --exec "configure new single ssd storage_migration_type=aggressive; status"
+    - name: Create fdb-environment.yaml
+      shell: bash
+      run: |
+        echo "clusterFiles: " >> fdb-environment.yaml
+        echo "  - /etc/foundationdb/fdb1.cluster" >> fdb-environment.yaml
+        echo "  - /etc/foundationdb/fdb2.cluster" >> fdb-environment.yaml

build.gradle

@@ -18,6 +18,10 @@
  * limitations under the License.
  */
 
+import org.yaml.snakeyaml.Yaml
+
+import org.apache.tools.ant.taskdefs.condition.Os
+
 buildscript {
     repositories {
         if (Boolean.parseBoolean(mavenLocalEnabled)) {
@@ -29,6 +33,7 @@ buildscript {
 
     dependencies {
         classpath 'org.jboss.tattletale:tattletale:1.2.0.Beta2'
+        classpath libs.snakeyaml
     }
 }
 
@@ -341,6 +346,8 @@ if (!JavaVersion.current().isJava8Compatible()) {
     throw new Exception("Java 8 is required to build fdb-record-layer")
 }
 
+// fdb-environment.properties is the old way we configured the library path and cluster file, it does not scale well
+// to multiple cluster files, so is being replaced with a yaml file.
 def fdbEnvironmentFile = new File("${rootProject.projectDir}/fdb-environment.properties")
 if (fdbEnvironmentFile.exists()) {
     fdbEnvironmentFile.eachLine { line ->
@@ -356,3 +363,15 @@ if (!ext.fdbEnvironment.isEmpty()) {
         }
     }
 }
+
+def fdbEnvironmentYamlFile = new File("${rootProject.projectDir}/fdb-environment.yaml")
+if (fdbEnvironmentYamlFile.exists()) {
+    def libraryPath = new Yaml().loadAll(fdbEnvironmentYamlFile.newInputStream()).first().libraryPath
+    def libraryPathVariable = Os.isFamily(Os.FAMILY_MAC) ? "DYLD_LIBRARY_PATH" : "LD_LIBRARY_PATH"
+    allprojects {
+        tasks.withType(Test) { task ->
+            task.environment([(libraryPathVariable): libraryPath])
+            task.environment(FDB_ENVIRONMENT_YAML: fdbEnvironmentYamlFile.absolutePath)
+        }
+    }
+}

docker-local/start.sh

@@ -95,6 +95,13 @@ FDB_CLUSTER_FILE=${FDB_CLUSTER_FILE}
 ${LIBRARY_PATH}=${RUNDIR}
 EOF
 
+cat >${ROOTDIR}/fdb-environment.yaml <<EOF
+# docker-local
+clusterFiles:
+  - ${FDB_CLUSTER_FILE}
+libraryPath: ${RUNDIR}
+EOF
+
 cat ${ROOTDIR}/fdb-environment.properties
 
 echo "Docker-based FDB cluster is now up."

docs/sphinx/source/Building.md

@@ -63,7 +63,24 @@ If you enable the local repo in whatever uses the Record Layer, the following wi
 ./gradlew publishToMavenLocal -PpublishBuild=true
 ```
 
-## Configuring tests for a non-standard FDB cluster file location
+## Configuring tests for a non-standard FDB cluster file location, or to run against multiple clusters
+
+If a file `fdb-environment.yaml` exists in the root of the working directory, it contains the configuration for the C API library
+and a list of cluster files. Most tests will chose randomly from the provided cluster files when testing.
+
+Here is an example file:
+``` yaml
+libraryPath: /Users/scott/fdb/bin/fdb-server-7.3.42-macos_arm64/lib
+clusterFiles:
+  - /Users/scott/fdb/data/fdb-one.cluster
+  - /Users/scott/fdb/data/fdb-two.cluster
+```
+
+
+
+### deprecated properties file
+
+[ this is being replaced by the yaml file described above, to better support multiple cluster files ]
 
 If a file `fdb-environment.properties` exists in the root of the working directory, it contains environment variables that specify where to find
 the local FDB. These settings will apply when running inside IntelliJ as well as from a command line and so are easier to manage than a shell script.

fdb-extensions/fdb-extensions.gradle

@@ -32,11 +32,13 @@ dependencies {
 
     testImplementation project(':fdb-test-utils')
     testImplementation(libs.bundles.test.impl)
+    testImplementation(libs.snakeyaml)
     testRuntimeOnly(libs.bundles.test.runtime)
     testCompileOnly(libs.bundles.test.compileOnly)
     testAnnotationProcessor(libs.autoService)
 
     testFixturesImplementation(libs.bundles.test.impl)
+    testFixturesImplementation(libs.snakeyaml)
     testFixturesCompileOnly(libs.bundles.test.compileOnly)
     testFixturesImplementation(libs.slf4j.api)
     testFixturesAnnotationProcessor(libs.autoService)

fdb-record-layer-core/fdb-record-layer-core.gradle

@@ -44,6 +44,7 @@ dependencies {
     testImplementation(libs.junit.platform)
     testImplementation(libs.protobuf.util)
     testCompileOnly(libs.spotbugs.annotations)
+    testImplementation(libs.snakeyaml)
 
     testAnnotationProcessor(libs.autoService)
 }

fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/BlockingInAsyncDetectionTest.java

@@ -24,6 +24,7 @@
 import com.apple.foundationdb.record.RecordCoreException;
 import com.apple.foundationdb.record.TestHelpers;
 import com.apple.foundationdb.record.test.FDBDatabaseExtension;
+import com.apple.foundationdb.test.FDBTestEnvironment;
 import com.apple.test.Tags;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
@@ -57,7 +58,7 @@ void testBlockingInAsyncException() {
         // Make sure that we aren't holding on to previously created databases
         factory.clear();
 
-        FDBDatabase database = factory.getDatabase();
+        FDBDatabase database = factory.getDatabase(FDBTestEnvironment.randomClusterFile());
         assertEquals(BlockingInAsyncDetection.IGNORE_COMPLETE_EXCEPTION_BLOCKING, database.getBlockingInAsyncDetection());
         assertThrows(BlockingInAsyncException.class, () -> callAsyncBlocking(database));
     }
@@ -68,7 +69,7 @@ void testBlockingInAsyncWarning() {
         factory.setBlockingInAsyncDetection(BlockingInAsyncDetection.IGNORE_COMPLETE_WARN_BLOCKING);
         factory.clear();
 
-        FDBDatabase database = factory.getDatabase();
+        FDBDatabase database = factory.getDatabase(FDBTestEnvironment.randomClusterFile());
         TestHelpers.assertLogs(FDBDatabase.class, FDBDatabase.BLOCKING_IN_ASYNC_CONTEXT_MESSAGE,
                 () -> {
                     callAsyncBlocking(database, true);
@@ -82,7 +83,7 @@ void testCompletedBlockingInAsyncWarning() {
         factory.setBlockingInAsyncDetection(BlockingInAsyncDetection.WARN_COMPLETE_EXCEPTION_BLOCKING);
         factory.clear();
 
-        FDBDatabase database = factory.getDatabase();
+        FDBDatabase database = factory.getDatabase(FDBTestEnvironment.randomClusterFile());
         TestHelpers.assertLogs(FDBDatabase.class, FDBDatabase.BLOCKING_IN_ASYNC_CONTEXT_MESSAGE,
                 () -> database.asyncToSync(new FDBStoreTimer(), FDBStoreTimer.Waits.WAIT_ERROR_CHECK,
                         CompletableFuture.supplyAsync(() ->
@@ -95,7 +96,7 @@ void testBlockingCreatingAsyncDetection() {
         factory.setBlockingInAsyncDetection(BlockingInAsyncDetection.WARN_COMPLETE_EXCEPTION_BLOCKING);
         factory.clear();
 
-        FDBDatabase database = factory.getDatabase();
+        FDBDatabase database = factory.getDatabase(FDBTestEnvironment.randomClusterFile());
         TestHelpers.assertLogs(FDBDatabase.class, FDBDatabase.BLOCKING_RETURNING_ASYNC_MESSAGE,
                 () -> returnAnAsync(database, MoreAsyncUtil.delayedFuture(200L, TimeUnit.MILLISECONDS, database.getScheduledExecutor())));
     }
@@ -106,7 +107,7 @@ void testCompletedBlockingCreatingAsyncDetection() {
         factory.setBlockingInAsyncDetection(BlockingInAsyncDetection.WARN_COMPLETE_EXCEPTION_BLOCKING);
         factory.clear();
 
-        FDBDatabase database = factory.getDatabase();
+        FDBDatabase database = factory.getDatabase(FDBTestEnvironment.randomClusterFile());
         TestHelpers.assertDidNotLog(FDBDatabase.class, FDBDatabase.BLOCKING_RETURNING_ASYNC_MESSAGE,
                 () -> returnAnAsync(database, CompletableFuture.completedFuture(10L)));
     }

fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBDatabaseTest.java

@@ -36,6 +36,7 @@
 import com.apple.foundationdb.record.test.TestKeySpace;
 import com.apple.foundationdb.record.test.TestKeySpacePathManagerExtension;
 import com.apple.foundationdb.subspace.Subspace;
+import com.apple.foundationdb.test.FDBTestEnvironment;
 import com.apple.foundationdb.tuple.Tuple;
 import com.apple.test.BooleanSource;
 import com.apple.test.Tags;
@@ -54,6 +55,7 @@
 
 import javax.annotation.Nonnull;
 import java.io.IOException;
+import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
@@ -66,6 +68,7 @@
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.lessThan;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
 import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -240,7 +243,7 @@ void testJoinNowOnNonCompletedFuture(BlockingInAsyncDetection behavior) {
         if (behavior.throwExceptionOnBlocking()) {
             assertThrows(BlockingInAsyncException.class, () -> database.joinNow(new CompletableFuture<>()));
         } else {
-            FDBDatabase database2 = factory.getDatabase();
+            FDBDatabase database2 = factory.getDatabase(database.getClusterFile());
             TestHelpers.assertLogs(FDBDatabase.class, FDBDatabase.BLOCKING_FOR_FUTURE_MESSAGE, () -> {
                 long val = database2.joinNow(MoreAsyncUtil.delayedFuture(100, TimeUnit.MILLISECONDS, database2.getScheduledExecutor())
                         .thenApply(vignore -> 1066L));
@@ -494,4 +497,31 @@ void cannotChangeAPIVersionAfterInit() {
             assertEquals(initApiVersion, database.getAPIVersion());
         }
     }
+
+    @Test
+    void canAccessMultipleClusters() {
+        FDBTestEnvironment.assumeClusterCount(2);
+        final FDBDatabase database0 = dbExtension.getDatabase(0);
+        final FDBDatabase database1 = dbExtension.getDatabase(1);
+        final byte[] key = Tuple.from(UUID.randomUUID()).pack();
+        final byte[] value0 = Tuple.from("cluster0").pack();
+        final byte[] value1 = Tuple.from("cluster1").pack();
+        try (FDBRecordContext context0 = database0.openContext()) {
+            context0.ensureActive().set(key, value0);
+            context0.commit();
+        }
+        try (FDBRecordContext context1 = database1.openContext()) {
+            assertNull(context1.ensureActive().get(key).join());
+            context1.ensureActive().set(key, value1);
+            context1.commit();
+        }
+        try (FDBRecordContext context0 = database0.openContext()) {
+            assertArrayEquals(value0, context0.ensureActive().get(key).join());
+            context0.commit();
+        }
+        try (FDBRecordContext context1 = database1.openContext()) {
+            assertArrayEquals(value1, context1.ensureActive().get(key).join());
+            context1.commit();
+        }
+    }
 }

fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordContextTest.java

@@ -34,6 +34,7 @@
 import com.apple.foundationdb.record.test.TestKeySpacePathManagerExtension;
 import com.apple.foundationdb.record.util.pair.Pair;
 import com.apple.foundationdb.subspace.Subspace;
+import com.apple.foundationdb.test.FDBTestEnvironment;
 import com.apple.foundationdb.tuple.ByteArrayUtil;
 import com.apple.foundationdb.tuple.ByteArrayUtil2;
 import com.apple.foundationdb.tuple.Tuple;
@@ -433,7 +434,7 @@ public void logWithoutSettingId() {
             FDBDatabaseFactory factory = fdb.getFactory();
             factory.setTransactionIsTracedSupplier(() -> false);
             factory.clear();
-            fdb = factory.getDatabase();
+            fdb = factory.getDatabase(FDBTestEnvironment.randomClusterFile());
         }
         try (FDBRecordContext context = fdb.openContext()) {
             RecordCoreException err = assertThrows(RecordCoreException.class, context::logTransaction);
@@ -665,7 +666,8 @@ public void contextExecutor() {
             factory.setContextExecutor(exec -> new ThreadIdRestoringExecutor(exec, myThreadId));
             factory.clear();
 
-            FDBDatabase database = factory.getDatabase();
+
+            FDBDatabase database = factory.getDatabase(FDBTestEnvironment.randomClusterFile());
             try (FDBRecordContext context = database.openContext()) {
                 context.ensureActive().get(new byte[] { 0 }).thenAccept( value -> {
                     assertEquals(myThreadId, ThreadId.get());