From 5d933b3eac45672e78683d64dc84763684b0a070 Mon Sep 17 00:00:00 2001 From: Yuvaraj Loganathan Date: Tue, 3 Dec 2019 15:28:52 +0530 Subject: [PATCH 01/10] Upgraded janus to 0.4.0 fdb to 6.2.7 --- README.md | 2 ++ pom.xml | 9 +++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index f08f71c..05f51eb 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,8 @@ JanusGraph, coupled with the FoundationDB storage adapter provides the following |FDB Storage Adapter|JanusGraph|FoundationDB| |-:|-:|-:| |0.1.0|0.3.0|5.2.5| +|0.2.0|0.4.0|6.2.7| + # Getting started diff --git a/pom.xml b/pom.xml index f0ef473..b9d4c3f 100644 --- a/pom.xml +++ b/pom.xml @@ -13,8 +13,8 @@ 3.6.2 1.8 - 0.3.0 - 5.2.5 + 0.4.0 + 6.2.10 2.10 false true @@ -66,10 +66,11 @@ test - com.apple - foundationdb + org.foundationdb + fdb-java ${foundationdb.version} + com.palantir.docker.compose docker-compose-rule-junit4 From ee47aee8ae9352f01952e86f5cbb62f941894755 Mon Sep 17 00:00:00 2001 From: nischal kumar Date: Fri, 13 Dec 2019 14:00:50 +0530 Subject: [PATCH 02/10] logs added --- .../janusgraph/diskstorage/foundationdb/FoundationDBTx.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/com/experoinc/janusgraph/diskstorage/foundationdb/FoundationDBTx.java b/src/main/java/com/experoinc/janusgraph/diskstorage/foundationdb/FoundationDBTx.java index 785a9b5..c9fd2dc 100644 --- a/src/main/java/com/experoinc/janusgraph/diskstorage/foundationdb/FoundationDBTx.java +++ b/src/main/java/com/experoinc/janusgraph/diskstorage/foundationdb/FoundationDBTx.java @@ -118,6 +118,7 @@ public synchronized void commit() throws BackendException { failing = false; break; } catch (IllegalStateException | ExecutionException e) { + log.warn("failed to commit transaction", e); if (isolationLevel.equals(IsolationLevel.SERIALIZABLE) || isolationLevel.equals(IsolationLevel.READ_COMMITTED_NO_WRITE)) { break; @@ -155,6 +156,7 @@ public byte[] get(final byte[] key) throws PermanentBackendException { failing = false; break; } catch (ExecutionException e) { + log.warn("failed to get ", e); this.restart(); } catch (Exception e) { throw new PermanentBackendException(e); @@ -178,6 +180,7 @@ public List getRange(final byte[] startKey, final byte[] endKey, failing = false; break; } catch (ExecutionException e) { + log.warn("failed to getRange", e); if (txCtr.get() == startTxId) this.restart(); } catch (Exception e) { From e1dd800b8eb90e8605c72f197dcfd798b75a0812 Mon Sep 17 00:00:00 2001 From: nischal kumar Date: Thu, 19 Dec 2019 18:34:21 +0530 Subject: [PATCH 03/10] adding logs to debug interruppted exception --- .../janusgraph/diskstorage/foundationdb/FoundationDBTx.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/experoinc/janusgraph/diskstorage/foundationdb/FoundationDBTx.java b/src/main/java/com/experoinc/janusgraph/diskstorage/foundationdb/FoundationDBTx.java index c9fd2dc..894c28e 100644 --- a/src/main/java/com/experoinc/janusgraph/diskstorage/foundationdb/FoundationDBTx.java +++ b/src/main/java/com/experoinc/janusgraph/diskstorage/foundationdb/FoundationDBTx.java @@ -232,6 +232,7 @@ public synchronized Map> getMultiRange(final List Date: Fri, 20 Dec 2019 13:55:01 +0530 Subject: [PATCH 04/10] adding logs at getRange --- .../janusgraph/diskstorage/foundationdb/FoundationDBTx.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/experoinc/janusgraph/diskstorage/foundationdb/FoundationDBTx.java b/src/main/java/com/experoinc/janusgraph/diskstorage/foundationdb/FoundationDBTx.java index 894c28e..f163a1f 100644 --- a/src/main/java/com/experoinc/janusgraph/diskstorage/foundationdb/FoundationDBTx.java +++ b/src/main/java/com/experoinc/janusgraph/diskstorage/foundationdb/FoundationDBTx.java @@ -184,6 +184,7 @@ public List getRange(final byte[] startKey, final byte[] endKey, if (txCtr.get() == startTxId) this.restart(); } catch (Exception e) { + log.error("raising backend exception for startKey {} endKey {} limit", startKey, endKey, limit, e); throw new PermanentBackendException(e); } } From a35347672e0243f909224e2c2a30280b3606e4d5 Mon Sep 17 00:00:00 2001 From: nischal kumar Date: Thu, 26 Dec 2019 15:12:58 +0530 Subject: [PATCH 05/10] added more exception logs (#3) * added more exception logs * corrected typo Co-authored-by: hari-om-888 --- .../janusgraph/diskstorage/foundationdb/FoundationDBTx.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/com/experoinc/janusgraph/diskstorage/foundationdb/FoundationDBTx.java b/src/main/java/com/experoinc/janusgraph/diskstorage/foundationdb/FoundationDBTx.java index f163a1f..6c76a13 100644 --- a/src/main/java/com/experoinc/janusgraph/diskstorage/foundationdb/FoundationDBTx.java +++ b/src/main/java/com/experoinc/janusgraph/diskstorage/foundationdb/FoundationDBTx.java @@ -90,6 +90,7 @@ public synchronized void rollback() throws BackendException { tx.close(); tx = null; } catch (Exception e) { + log.error("failed to rollback", e); throw new PermanentBackendException(e); } finally { if (tx != null) @@ -125,6 +126,7 @@ public synchronized void commit() throws BackendException { } restart(); } catch (Exception e) { + log.error("failed to commit", e); throw new PermanentBackendException(e); } } @@ -159,6 +161,7 @@ public byte[] get(final byte[] key) throws PermanentBackendException { log.warn("failed to get ", e); this.restart(); } catch (Exception e) { + log.error("failed to get key {}", key, e); throw new PermanentBackendException(e); } } From 48b56d3c76c0d24fc569d69399b090aed15e6105 Mon Sep 17 00:00:00 2001 From: nischal kumar Date: Fri, 27 Dec 2019 02:49:11 +0530 Subject: [PATCH 06/10] supporting READ_COMMITTED_NO_WRITE --- .../foundationdb/FoundationDBTx.java | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/experoinc/janusgraph/diskstorage/foundationdb/FoundationDBTx.java b/src/main/java/com/experoinc/janusgraph/diskstorage/foundationdb/FoundationDBTx.java index 6c76a13..405571d 100644 --- a/src/main/java/com/experoinc/janusgraph/diskstorage/foundationdb/FoundationDBTx.java +++ b/src/main/java/com/experoinc/janusgraph/diskstorage/foundationdb/FoundationDBTx.java @@ -1,9 +1,6 @@ package com.experoinc.janusgraph.diskstorage.foundationdb; -import com.apple.foundationdb.Database; -import com.apple.foundationdb.KeyValue; -import com.apple.foundationdb.Range; -import com.apple.foundationdb.Transaction; +import com.apple.foundationdb.*; import org.janusgraph.diskstorage.BackendException; import org.janusgraph.diskstorage.BaseTransactionConfig; import org.janusgraph.diskstorage.PermanentBackendException; @@ -154,7 +151,8 @@ public byte[] get(final byte[] key) throws PermanentBackendException { byte[] value = null; for (int i = 0; i < maxRuns; i++) { try { - value = this.tx.get(key).get(); + ReadTransaction transaction = getTransaction(this.isolationLevel, this.tx); + value = transaction.get(key).get(); failing = false; break; } catch (ExecutionException e) { @@ -178,7 +176,9 @@ public List getRange(final byte[] startKey, final byte[] endKey, for (int i = 0; i < maxRuns; i++) { final int startTxId = txCtr.get(); try { - result = tx.getRange(new Range(startKey, endKey), limit).asList().get(); + ReadTransaction transaction = getTransaction(isolationLevel, this.tx); + + result = transaction.getRange(new Range(startKey, endKey), limit).asList().get(); if (result == null) return Collections.emptyList(); failing = false; break; @@ -197,6 +197,14 @@ public List getRange(final byte[] startKey, final byte[] endKey, return result; } + private T getTransaction(IsolationLevel isolationLevel, Transaction tx) { + if(IsolationLevel.READ_COMMITTED_NO_WRITE.equals(isolationLevel)) { + return (T)tx.snapshot(); + } else { + return (T)tx; + } + } + public synchronized Map> getMultiRange(final List queries) throws PermanentBackendException { Map> resultMap = new ConcurrentHashMap<>(); From 4c9294a033f4d0c627d32b3e09fcace5dd1c1258 Mon Sep 17 00:00:00 2001 From: nischal kumar Date: Fri, 27 Dec 2019 02:54:28 +0530 Subject: [PATCH 07/10] allowing dirty reads for READ_COMMITTED_WITH_WRITE as well --- .../janusgraph/diskstorage/foundationdb/FoundationDBTx.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/experoinc/janusgraph/diskstorage/foundationdb/FoundationDBTx.java b/src/main/java/com/experoinc/janusgraph/diskstorage/foundationdb/FoundationDBTx.java index 405571d..b06c6a3 100644 --- a/src/main/java/com/experoinc/janusgraph/diskstorage/foundationdb/FoundationDBTx.java +++ b/src/main/java/com/experoinc/janusgraph/diskstorage/foundationdb/FoundationDBTx.java @@ -198,7 +198,8 @@ public List getRange(final byte[] startKey, final byte[] endKey, } private T getTransaction(IsolationLevel isolationLevel, Transaction tx) { - if(IsolationLevel.READ_COMMITTED_NO_WRITE.equals(isolationLevel)) { + if(IsolationLevel.READ_COMMITTED_NO_WRITE.equals(isolationLevel) + || IsolationLevel.READ_COMMITTED_WITH_WRITE.equals(isolationLevel)) { return (T)tx.snapshot(); } else { return (T)tx; From 5b340beaaef88237c0b90e59d30f4225917ed439 Mon Sep 17 00:00:00 2001 From: nischal kumar Date: Fri, 27 Dec 2019 02:55:33 +0530 Subject: [PATCH 08/10] adding formatting before codeReview --- .../janusgraph/diskstorage/foundationdb/FoundationDBTx.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/com/experoinc/janusgraph/diskstorage/foundationdb/FoundationDBTx.java b/src/main/java/com/experoinc/janusgraph/diskstorage/foundationdb/FoundationDBTx.java index b06c6a3..c6e4422 100644 --- a/src/main/java/com/experoinc/janusgraph/diskstorage/foundationdb/FoundationDBTx.java +++ b/src/main/java/com/experoinc/janusgraph/diskstorage/foundationdb/FoundationDBTx.java @@ -177,7 +177,6 @@ public List getRange(final byte[] startKey, final byte[] endKey, final int startTxId = txCtr.get(); try { ReadTransaction transaction = getTransaction(isolationLevel, this.tx); - result = transaction.getRange(new Range(startKey, endKey), limit).asList().get(); if (result == null) return Collections.emptyList(); failing = false; From d6639d9fc4e15e569f80b0d8eab31813afc8b4d0 Mon Sep 17 00:00:00 2001 From: nischal kumar Date: Fri, 27 Dec 2019 17:06:34 +0530 Subject: [PATCH 09/10] formatting imports --- .../diskstorage/foundationdb/FoundationDBTx.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/experoinc/janusgraph/diskstorage/foundationdb/FoundationDBTx.java b/src/main/java/com/experoinc/janusgraph/diskstorage/foundationdb/FoundationDBTx.java index c6e4422..33a2131 100644 --- a/src/main/java/com/experoinc/janusgraph/diskstorage/foundationdb/FoundationDBTx.java +++ b/src/main/java/com/experoinc/janusgraph/diskstorage/foundationdb/FoundationDBTx.java @@ -1,6 +1,10 @@ package com.experoinc.janusgraph.diskstorage.foundationdb; -import com.apple.foundationdb.*; +import com.apple.foundationdb.Database; +import com.apple.foundationdb.KeyValue; +import com.apple.foundationdb.ReadTransaction; +import com.apple.foundationdb.Transaction; +import com.apple.foundationdb.Range; import org.janusgraph.diskstorage.BackendException; import org.janusgraph.diskstorage.BaseTransactionConfig; import org.janusgraph.diskstorage.PermanentBackendException; @@ -170,7 +174,7 @@ public byte[] get(final byte[] key) throws PermanentBackendException { } public List getRange(final byte[] startKey, final byte[] endKey, - final int limit) throws PermanentBackendException { + final int limit) throws PermanentBackendException { boolean failing = true; List result = Collections.emptyList(); for (int i = 0; i < maxRuns; i++) { From 714ddba8d7e62b90d46cd63b2b7dd6aba445ec44 Mon Sep 17 00:00:00 2001 From: nischal kumar Date: Tue, 7 Jan 2020 19:51:07 +0530 Subject: [PATCH 10/10] moving towards iterator --- .../FoundationDBKeyValueStore.java | 57 +++++++------------ .../foundationdb/FoundationDBTx.java | 40 +++++-------- 2 files changed, 34 insertions(+), 63 deletions(-) diff --git a/src/main/java/com/experoinc/janusgraph/diskstorage/foundationdb/FoundationDBKeyValueStore.java b/src/main/java/com/experoinc/janusgraph/diskstorage/foundationdb/FoundationDBKeyValueStore.java index 5f7ee85..733058b 100644 --- a/src/main/java/com/experoinc/janusgraph/diskstorage/foundationdb/FoundationDBKeyValueStore.java +++ b/src/main/java/com/experoinc/janusgraph/diskstorage/foundationdb/FoundationDBKeyValueStore.java @@ -121,44 +121,45 @@ public RecordIterator getSlice(KVQuery query, StoreTransaction tx final StaticBuffer keyStart = query.getStart(); final StaticBuffer keyEnd = query.getEnd(); final KeySelector selector = query.getKeySelector(); - final List result = new ArrayList<>(); final byte[] foundKey = db.pack(keyStart.as(ENTRY_FACTORY)); final byte[] endKey = db.pack(keyEnd.as(ENTRY_FACTORY)); try { - final List results = tx.getRange(foundKey, endKey, query.getLimit()); - - for (final KeyValue keyValue : results) { - StaticBuffer key = getBuffer(db.unpack(keyValue.getKey()).getBytes(0)); - if (selector.include(key)) - result.add(new KeyValueEntry(key, getBuffer(keyValue.getValue()))); - } + final Iterator results = tx.getRange(foundKey, endKey, query.getLimit()); + return new FoundationDBRecordIterator(results, selector); } catch (Exception e) { throw new PermanentBackendException(e); } - - log.trace("db={}, op=getSlice, tx={}, resultcount={}", name, txh, result.size()); - - return new FoundationDBRecordIterator(result); } private class FoundationDBRecordIterator implements RecordIterator { - private final Iterator entries; + private final Iterator entries; + private final KeySelector selector; + private KeyValueEntry keyValueEntry; - public FoundationDBRecordIterator(final List result) { - this.entries = result.iterator(); + public FoundationDBRecordIterator(final Iterator result, final KeySelector keySelector) { + this.entries = result; + this.selector = keySelector; } @Override public boolean hasNext() { - return entries.hasNext(); + while (entries.hasNext()) { + KeyValue keyValue = entries.next(); + StaticBuffer key = getBuffer(db.unpack(keyValue.getKey()).getBytes(0)); + if (selector.include(key)) { + this.keyValueEntry = new KeyValueEntry(key, getBuffer(keyValue.getValue())); + return true; + } + } + return false; } @Override public KeyValueEntry next() { - return entries.next(); + return this.keyValueEntry; } @Override @@ -172,39 +173,23 @@ public void remove() { } @Override - public Map> getSlices(List queries, StoreTransaction txh) throws BackendException { + public Map> getSlices(List queries, StoreTransaction txh) throws BackendException { log.trace("beginning db={}, op=getSlice, tx={}", name, txh); FoundationDBTx tx = getTransaction(txh); final Map> resultMap = new ConcurrentHashMap<>(); - final List> futures = new ArrayList<>(); - try { - final List preppedQueries = new LinkedList<>(); for (final KVQuery query : queries) { final StaticBuffer keyStart = query.getStart(); final StaticBuffer keyEnd = query.getEnd(); final KeySelector selector = query.getKeySelector(); final byte[] foundKey = db.pack(keyStart.as(ENTRY_FACTORY)); final byte[] endKey = db.pack(keyEnd.as(ENTRY_FACTORY)); - preppedQueries.add(new Object[]{query, foundKey, endKey}); - } - final Map> result = tx.getMultiRange(preppedQueries); - - for (Map.Entry> entry : result.entrySet()) { - final List results = new ArrayList<>(); - for (final KeyValue keyValue : entry.getValue()) { - final StaticBuffer key = getBuffer(db.unpack(keyValue.getKey()).getBytes(0)); - if (entry.getKey().getKeySelector().include(key)) - results.add(new KeyValueEntry(key, getBuffer(keyValue.getValue()))); - } - resultMap.put(entry.getKey(), new FoundationDBRecordIterator(results)); - + resultMap.put(query, new FoundationDBRecordIterator(tx.getRange(foundKey, endKey, query.getLimit()), selector)); } + return resultMap; } catch (Exception e) { throw new PermanentBackendException(e); } - - return resultMap; } @Override diff --git a/src/main/java/com/experoinc/janusgraph/diskstorage/foundationdb/FoundationDBTx.java b/src/main/java/com/experoinc/janusgraph/diskstorage/foundationdb/FoundationDBTx.java index 33a2131..0c5164e 100644 --- a/src/main/java/com/experoinc/janusgraph/diskstorage/foundationdb/FoundationDBTx.java +++ b/src/main/java/com/experoinc/janusgraph/diskstorage/foundationdb/FoundationDBTx.java @@ -5,6 +5,7 @@ import com.apple.foundationdb.ReadTransaction; import com.apple.foundationdb.Transaction; import com.apple.foundationdb.Range; +import com.apple.foundationdb.async.AsyncIterable; import org.janusgraph.diskstorage.BackendException; import org.janusgraph.diskstorage.BaseTransactionConfig; import org.janusgraph.diskstorage.PermanentBackendException; @@ -13,10 +14,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; @@ -173,31 +171,19 @@ public byte[] get(final byte[] key) throws PermanentBackendException { return value; } - public List getRange(final byte[] startKey, final byte[] endKey, - final int limit) throws PermanentBackendException { - boolean failing = true; - List result = Collections.emptyList(); - for (int i = 0; i < maxRuns; i++) { - final int startTxId = txCtr.get(); - try { - ReadTransaction transaction = getTransaction(isolationLevel, this.tx); - result = transaction.getRange(new Range(startKey, endKey), limit).asList().get(); - if (result == null) return Collections.emptyList(); - failing = false; - break; - } catch (ExecutionException e) { - log.warn("failed to getRange", e); - if (txCtr.get() == startTxId) - this.restart(); - } catch (Exception e) { - log.error("raising backend exception for startKey {} endKey {} limit", startKey, endKey, limit, e); - throw new PermanentBackendException(e); + public Iterator getRange(final byte[] startKey, final byte[] endKey, + final int limit) throws PermanentBackendException { + try { + ReadTransaction transaction = getTransaction(isolationLevel, this.tx); + AsyncIterable result = transaction.getRange(new Range(startKey, endKey), limit); + if (result == null) { + return Collections.emptyIterator(); } + return result.iterator(); + } catch (Exception e) { + log.error("raising backend exception for startKey {} endKey {} limit", startKey, endKey, limit, e); + throw new PermanentBackendException(e); } - if (failing) { - throw new PermanentBackendException("Max transaction reset count exceeded"); - } - return result; } private T getTransaction(IsolationLevel isolationLevel, Transaction tx) {