Skip to content

Commit 3d0e968

Browse files
committed
Merge branch 'main' into keyspace-import
2 parents 70a2134 + 3392b57 commit 3d0e968

File tree

101 files changed

+7299
-141
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

101 files changed

+7299
-141
lines changed

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@ protogen/
8080

8181
# Local FDB settings
8282
fdb-environment.properties
83+
fdb-environment.yaml
8384

8485
# Docker local support
8586
run/

ACKNOWLEDGEMENTS

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -216,3 +216,27 @@ Unicode, Inc (ICU4J)
216216
Creative Commons Attribution 4.0 License (GeoNames)
217217

218218
https://creativecommons.org/licenses/by/4.0/
219+
220+
Christian Heina (HALF4J)
221+
222+
Copyright 2023 Christian Heina
223+
224+
Licensed under the Apache License, Version 2.0 (the "License");
225+
you may not use this file except in compliance with the License.
226+
You may obtain a copy of the License at
227+
228+
http://www.apache.org/licenses/LICENSE-2.0
229+
230+
Unless required by applicable law or agreed to in writing, software
231+
distributed under the License is distributed on an "AS IS" BASIS,
232+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
233+
See the License for the specific language governing permissions and
234+
limitations under the License.
235+
236+
Jianyang Gao, Yutong Gou, Yuexuan Xu, Yongyi Yang, Cheng Long, Raymond Chi-Wing Wong,
237+
"Practical and Asymptotically Optimal Quantization of High-Dimensional Vectors in Euclidean Space for
238+
Approximate Nearest Neighbor Search",
239+
SIGMOD 2025, available at https://arxiv.org/abs/2409.09913
240+
241+
Yutong Gou, Jianyang Gao, Yuexuan Xu, Jifan Shi and Zhonghao Yang
242+
https://github.com/VectorDB-NTU/RaBitQ-Library/blob/main/LICENSE

actions/setup-fdb/action.yml

Lines changed: 51 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -31,12 +31,57 @@ runs:
3131
- name: Install FDB Server
3232
shell: bash
3333
run: sudo dpkg -i ~/.fdb-cache/${{ steps.fdb_filenames.outputs.client_deb }} ~/.fdb-cache/${{ steps.fdb_filenames.outputs.server_deb }}
34-
- name: Fix FDB Network Addresses
34+
- name: Stop default fdb
3535
shell: bash
36-
run: sudo sed -i -e "s/public_address = auto:\$ID/public_address = 127.0.0.1:\$ID/g" -e "s/listen_address = public/listen_address = 0.0.0.0:\$ID/g" /etc/foundationdb/foundationdb.conf
37-
- name: Start FDB Server
36+
run: sudo service foundationdb stop
37+
38+
- name: Create cluster1 config
39+
shell: bash
40+
run: |
41+
sudo cp /etc/foundationdb/foundationdb.conf /etc/foundationdb/foundationdb1.conf
42+
43+
sudo sed -i -e "s/\/etc\/foundationdb\/fdb.cluster/\/etc\/foundationdb\/fdb1.cluster/g" \
44+
-e "s/\/var\/log\/foundationdb/\/var\/log\/foundationdb1/" \
45+
-e "s/fdbserver.4500/fdbserver.4600/g" \
46+
/etc/foundationdb/foundationdb1.conf
47+
48+
sudo bash -c "echo 'fdb1:$(mktemp -u XXXXXXXX)@127.0.0.1:4600' > /etc/foundationdb/fdb1.cluster"
49+
- name: Create cluster2 config
3850
shell: bash
39-
run: sudo /usr/lib/foundationdb/fdbmonitor /etc/foundationdb/foundationdb.conf --daemonize
40-
- name: Switch FDB to SSD
51+
run: |
52+
sudo cp /etc/foundationdb/foundationdb.conf /etc/foundationdb/foundationdb2.conf
53+
54+
sudo sed -i -e "s/\/etc\/foundationdb\/fdb.cluster/\/etc\/foundationdb\/fdb2.cluster/g" \
55+
-e "s/\/var\/log\/foundationdb/\/var\/log\/foundationdb2/" \
56+
-e "s/fdbserver.4500/fdbserver.4700/g" \
57+
/etc/foundationdb/foundationdb2.conf
58+
59+
sudo bash -c "echo 'fdb2:$(mktemp -u XXXXXXXX)@127.0.0.1:4700' > /etc/foundationdb/fdb2.cluster"
60+
61+
- name: create dirs & set permissions
4162
shell: bash
42-
run: fdbcli --exec "configure single ssd storage_migration_type=aggressive; status"
63+
run: |
64+
sudo mkdir /var/log/foundationdb1 /var/log/foundationdb2
65+
sudo chown foundationdb:foundationdb /etc/foundationdb/fdb1.cluster /etc/foundationdb/fdb2.cluster /var/log/foundationdb1 /var/log/foundationdb2
66+
sudo chmod 664 /etc/foundationdb/fdb1.cluster /etc/foundationdb/fdb2.cluster
67+
sudo chmod 700 /var/log/foundationdb1 /var/log/foundationdb2
68+
69+
- name: Start FDB Server 1
70+
shell: bash
71+
run: sudo /usr/lib/foundationdb/fdbmonitor --conffile /etc/foundationdb/foundationdb1.conf --lockfile /var/run/fdbmonitor1.pid --loggroup fdb1 --daemonize
72+
- name: Start FDB Server 2
73+
shell: bash
74+
run: sudo /usr/lib/foundationdb/fdbmonitor --conffile /etc/foundationdb/foundationdb2.conf --lockfile /var/run/fdbmonitor2.pid --loggroup fdb2 --daemonize
75+
76+
- name: Switch FDB 1 to SSD
77+
shell: bash
78+
run: fdbcli -C /etc/foundationdb/fdb1.cluster --exec "configure new single ssd storage_migration_type=aggressive; status"
79+
- name: Switch FDB 2 to SSD
80+
shell: bash
81+
run: fdbcli -C /etc/foundationdb/fdb2.cluster --exec "configure new single ssd storage_migration_type=aggressive; status"
82+
- name: Create fdb-environment.yaml
83+
shell: bash
84+
run: |
85+
echo "clusterFiles: " >> fdb-environment.yaml
86+
echo " - /etc/foundationdb/fdb1.cluster" >> fdb-environment.yaml
87+
echo " - /etc/foundationdb/fdb2.cluster" >> fdb-environment.yaml

build.gradle

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,10 @@
1818
* limitations under the License.
1919
*/
2020

21+
import org.yaml.snakeyaml.Yaml
22+
23+
import org.apache.tools.ant.taskdefs.condition.Os
24+
2125
buildscript {
2226
repositories {
2327
if (Boolean.parseBoolean(mavenLocalEnabled)) {
@@ -29,6 +33,7 @@ buildscript {
2933

3034
dependencies {
3135
classpath 'org.jboss.tattletale:tattletale:1.2.0.Beta2'
36+
classpath libs.snakeyaml
3237
}
3338
}
3439

@@ -341,6 +346,8 @@ if (!JavaVersion.current().isJava8Compatible()) {
341346
throw new Exception("Java 8 is required to build fdb-record-layer")
342347
}
343348

349+
// fdb-environment.properties is the old way we configured the library path and cluster file, it does not scale well
350+
// to multiple cluster files, so is being replaced with a yaml file.
344351
def fdbEnvironmentFile = new File("${rootProject.projectDir}/fdb-environment.properties")
345352
if (fdbEnvironmentFile.exists()) {
346353
fdbEnvironmentFile.eachLine { line ->
@@ -356,3 +363,15 @@ if (!ext.fdbEnvironment.isEmpty()) {
356363
}
357364
}
358365
}
366+
367+
def fdbEnvironmentYamlFile = new File("${rootProject.projectDir}/fdb-environment.yaml")
368+
if (fdbEnvironmentYamlFile.exists()) {
369+
def libraryPath = new Yaml().loadAll(fdbEnvironmentYamlFile.newInputStream()).first().libraryPath
370+
def libraryPathVariable = Os.isFamily(Os.FAMILY_MAC) ? "DYLD_LIBRARY_PATH" : "LD_LIBRARY_PATH"
371+
allprojects {
372+
tasks.withType(Test) { task ->
373+
task.environment([(libraryPathVariable): libraryPath])
374+
task.environment(FDB_ENVIRONMENT_YAML: fdbEnvironmentYamlFile.absolutePath)
375+
}
376+
}
377+
}

docker-local/start.sh

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,13 @@ FDB_CLUSTER_FILE=${FDB_CLUSTER_FILE}
9595
${LIBRARY_PATH}=${RUNDIR}
9696
EOF
9797

98+
cat >${ROOTDIR}/fdb-environment.yaml <<EOF
99+
# docker-local
100+
clusterFiles:
101+
- ${FDB_CLUSTER_FILE}
102+
libraryPath: ${RUNDIR}
103+
EOF
104+
98105
cat ${ROOTDIR}/fdb-environment.properties
99106

100107
echo "Docker-based FDB cluster is now up."

docs/sphinx/source/Building.md

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,24 @@ If you enable the local repo in whatever uses the Record Layer, the following wi
6363
./gradlew publishToMavenLocal -PpublishBuild=true
6464
```
6565

66-
## Configuring tests for a non-standard FDB cluster file location
66+
## Configuring tests for a non-standard FDB cluster file location, or to run against multiple clusters
67+
68+
If a file `fdb-environment.yaml` exists in the root of the working directory, it contains the configuration for the C API library
69+
and a list of cluster files. Most tests will chose randomly from the provided cluster files when testing.
70+
71+
Here is an example file:
72+
``` yaml
73+
libraryPath: /Users/scott/fdb/bin/fdb-server-7.3.42-macos_arm64/lib
74+
clusterFiles:
75+
- /Users/scott/fdb/data/fdb-one.cluster
76+
- /Users/scott/fdb/data/fdb-two.cluster
77+
```
78+
79+
80+
81+
### deprecated properties file
82+
83+
[ this is being replaced by the yaml file described above, to better support multiple cluster files ]
6784
6885
If a file `fdb-environment.properties` exists in the root of the working directory, it contains environment variables that specify where to find
6986
the local FDB. These settings will apply when running inside IntelliJ as well as from a command line and so are easier to manage than a shell script.

fdb-extensions/fdb-extensions.gradle

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,16 +32,50 @@ dependencies {
3232

3333
testImplementation project(':fdb-test-utils')
3434
testImplementation(libs.bundles.test.impl)
35+
testImplementation(libs.snakeyaml)
3536
testRuntimeOnly(libs.bundles.test.runtime)
3637
testCompileOnly(libs.bundles.test.compileOnly)
3738
testAnnotationProcessor(libs.autoService)
3839

3940
testFixturesImplementation(libs.bundles.test.impl)
41+
testFixturesImplementation(libs.snakeyaml)
4042
testFixturesCompileOnly(libs.bundles.test.compileOnly)
4143
testFixturesImplementation(libs.slf4j.api)
4244
testFixturesAnnotationProcessor(libs.autoService)
4345
}
4446

47+
def siftSmallFile = layout.buildDirectory.file('downloads/siftsmall.tar.gz')
48+
def extractDir = layout.buildDirectory.dir("extracted")
49+
50+
// Task that downloads the CSV exactly once unless it changed
51+
tasks.register('downloadSiftSmall', de.undercouch.gradle.tasks.download.Download) {
52+
src 'https://huggingface.co/datasets/vecdata/siftsmall/resolve/3106e1b83049c44713b1ce06942d0ab474bbdfb6/siftsmall.tar.gz'
53+
dest siftSmallFile.get().asFile
54+
onlyIfModified true
55+
tempAndMove true
56+
retries 3
57+
}
58+
59+
tasks.register('extractSiftSmall', Copy) {
60+
dependsOn 'downloadSiftSmall'
61+
from(tarTree(resources.gzip(siftSmallFile)))
62+
into extractDir
63+
64+
doLast {
65+
println "Extracted files into: ${extractDir.get().asFile}"
66+
fileTree(extractDir).visit { details ->
67+
if (!details.isDirectory()) {
68+
println " - ${details.file}"
69+
}
70+
}
71+
}
72+
}
73+
74+
test {
75+
dependsOn tasks.named('extractSiftSmall')
76+
inputs.dir extractDir
77+
}
78+
4579
publishing {
4680
publications {
4781
library(MavenPublication) {

fdb-extensions/src/main/java/com/apple/foundationdb/async/MoreAsyncUtil.java

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,14 @@
2323
import com.apple.foundationdb.annotation.API;
2424
import com.apple.foundationdb.util.LoggableException;
2525
import com.google.common.base.Suppliers;
26+
import com.google.common.collect.Lists;
2627
import com.google.common.util.concurrent.ThreadFactoryBuilder;
2728

2829
import javax.annotation.Nonnull;
2930
import javax.annotation.Nullable;
3031
import java.util.ArrayDeque;
3132
import java.util.ArrayList;
33+
import java.util.Arrays;
3234
import java.util.Collections;
3335
import java.util.Iterator;
3436
import java.util.List;
@@ -42,9 +44,13 @@
4244
import java.util.concurrent.ScheduledThreadPoolExecutor;
4345
import java.util.concurrent.ThreadFactory;
4446
import java.util.concurrent.TimeUnit;
47+
import java.util.concurrent.atomic.AtomicInteger;
48+
import java.util.concurrent.atomic.AtomicReference;
4549
import java.util.function.BiConsumer;
4650
import java.util.function.BiFunction;
4751
import java.util.function.Function;
52+
import java.util.function.IntPredicate;
53+
import java.util.function.IntUnaryOperator;
4854
import java.util.function.Predicate;
4955
import java.util.function.Supplier;
5056

@@ -1051,6 +1057,93 @@ public static CompletableFuture<Void> swallowException(@Nonnull CompletableFutur
10511057
return result;
10521058
}
10531059

1060+
/**
1061+
* Method that provides the functionality of a for loop, however, in an asynchronous way. The result of this method
1062+
* is a {@link CompletableFuture} that represents the result of the last iteration of the loop body.
1063+
* @param startI an integer analogous to the starting value of a loop variable in a for loop
1064+
* @param startU an object of some type {@code U} that represents some initial state that is passed to the loop's
1065+
* initial state
1066+
* @param conditionPredicate a predicate on the loop variable that must be true before the next iteration is
1067+
* entered; analogous to the condition in a for loop
1068+
* @param stepFunction a unary operator used for modifying the loop variable after each iteration
1069+
* @param body a bi-function to be called for each iteration; this function is initially invoked using
1070+
* {@code startI} and {@code startU}; the result of the body is then passed into the next iterator's body
1071+
* together with a new value for the loop variable. In this way callers can access state inside an iteration
1072+
* that was computed in a previous iteration.
1073+
* @param executor the executor
1074+
* @param <U> the type of the result of the body {@link BiFunction}
1075+
* @return a {@link CompletableFuture} containing the result of the last iteration's body invocation.
1076+
*/
1077+
@Nonnull
1078+
public static <U> CompletableFuture<U> forLoop(final int startI, @Nullable final U startU,
1079+
@Nonnull final IntPredicate conditionPredicate,
1080+
@Nonnull final IntUnaryOperator stepFunction,
1081+
@Nonnull final BiFunction<Integer, U, CompletableFuture<U>> body,
1082+
@Nonnull final Executor executor) {
1083+
final AtomicInteger loopVariableAtomic = new AtomicInteger(startI);
1084+
final AtomicReference<U> lastResultAtomic = new AtomicReference<>(startU);
1085+
return whileTrue(() -> {
1086+
final int loopVariable = loopVariableAtomic.get();
1087+
if (!conditionPredicate.test(loopVariable)) {
1088+
return AsyncUtil.READY_FALSE;
1089+
}
1090+
return body.apply(loopVariable, lastResultAtomic.get())
1091+
.thenApply(result -> {
1092+
loopVariableAtomic.set(stepFunction.applyAsInt(loopVariable));
1093+
lastResultAtomic.set(result);
1094+
return true;
1095+
});
1096+
}, executor).thenApply(ignored -> lastResultAtomic.get());
1097+
}
1098+
1099+
/**
1100+
* Method to iterate over some items, for each of which a body is executed asynchronously. The result of each such
1101+
* executed is then collected in a list and returned as a {@link CompletableFuture} over that list.
1102+
* @param items the items to iterate over
1103+
* @param body a function to be called for each item
1104+
* @param parallelism the maximum degree of parallelism this method should use
1105+
* @param executor the executor
1106+
* @param <T> the type of item
1107+
* @param <U> the type of the result
1108+
* @return a {@link CompletableFuture} containing a list of results collected from the individual body invocations
1109+
*/
1110+
@Nonnull
1111+
@SuppressWarnings("unchecked")
1112+
public static <T, U> CompletableFuture<List<U>> forEach(@Nonnull final Iterable<T> items,
1113+
@Nonnull final Function<T, CompletableFuture<U>> body,
1114+
final int parallelism,
1115+
@Nonnull final Executor executor) {
1116+
// this deque is only modified by once upon creation
1117+
final ArrayDeque<T> toBeProcessed = new ArrayDeque<>();
1118+
for (final T item : items) {
1119+
toBeProcessed.addLast(item);
1120+
}
1121+
1122+
final List<CompletableFuture<Void>> working = Lists.newArrayList();
1123+
final AtomicInteger indexAtomic = new AtomicInteger(0);
1124+
final Object[] resultArray = new Object[toBeProcessed.size()];
1125+
1126+
return whileTrue(() -> {
1127+
working.removeIf(CompletableFuture::isDone);
1128+
1129+
while (working.size() <= parallelism) {
1130+
final T currentItem = toBeProcessed.pollFirst();
1131+
if (currentItem == null) {
1132+
break;
1133+
}
1134+
1135+
final int index = indexAtomic.getAndIncrement();
1136+
working.add(body.apply(currentItem)
1137+
.thenAccept(result -> resultArray[index] = result));
1138+
}
1139+
1140+
if (working.isEmpty()) {
1141+
return AsyncUtil.READY_FALSE;
1142+
}
1143+
return whenAny(working).thenApply(ignored -> true);
1144+
}, executor).thenApply(ignored -> Arrays.asList((U[])resultArray));
1145+
}
1146+
10541147
/**
10551148
* A {@code Boolean} function that is always true.
10561149
* @param <T> the type of the (ignored) argument to the function

fdb-extensions/src/main/java/com/apple/foundationdb/async/rtree/StorageAdapter.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@
3636
* Storage adapter used for serialization and deserialization of nodes.
3737
*/
3838
interface StorageAdapter {
39-
4039
/**
4140
* Get the {@link RTree.Config} associated with this storage adapter.
4241
* @return the configuration used by this storage adapter

0 commit comments

Comments
 (0)