First sketch of MDB_DUPSORT.

Author: kenwenzel

core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbRecordIterator.java

@@ -11,6 +11,7 @@
 package org.eclipse.rdf4j.sail.lmdb;
 
 import static org.eclipse.rdf4j.sail.lmdb.LmdbUtil.E;
+import static org.lwjgl.util.lmdb.LMDB.MDB_GET_BOTH_RANGE;
 import static org.lwjgl.util.lmdb.LMDB.MDB_NEXT;
 import static org.lwjgl.util.lmdb.LMDB.MDB_NOTFOUND;
 import static org.lwjgl.util.lmdb.LMDB.MDB_SET;
@@ -21,6 +22,7 @@
 import static org.lwjgl.util.lmdb.LMDB.mdb_cursor_get;
 import static org.lwjgl.util.lmdb.LMDB.mdb_cursor_open;
 import static org.lwjgl.util.lmdb.LMDB.mdb_cursor_renew;
+import static org.lwjgl.util.lmdb.LMDB.mdb_dcmp;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
@@ -53,6 +55,7 @@ class LmdbRecordIterator implements RecordIterator {
 	private final long cursor;
 
 	private final MDBVal maxKey;
+	private final MDBVal maxValue;
 
 	private final boolean matchValues;
 	private GroupMatcher groupMatcher;
@@ -73,44 +76,60 @@ class LmdbRecordIterator implements RecordIterator {
 
 	private ByteBuffer minKeyBuf;
 
+	private ByteBuffer minValueBuf;
+
 	private ByteBuffer maxKeyBuf;
 
+	private ByteBuffer maxValueBuf;
+
 	private int lastResult;
 
 	private final long[] quad;
-	private final long[] originalQuad;
+	private final long[] patternQuad;
 
 	private boolean fetchNext = false;
 
 	private final StampedLongAdderLockManager txnLockManager;
 
 	private final Thread ownerThread = Thread.currentThread();
 
-	LmdbRecordIterator(TripleIndex index, boolean rangeSearch, long subj, long pred, long obj,
+	private final int indexScore;
+
+	LmdbRecordIterator(TripleIndex index, int indexScore, long subj, long pred, long obj,
 			long context, boolean explicit, Txn txnRef) throws IOException {
 		this.subj = subj;
 		this.pred = pred;
 		this.obj = obj;
 		this.context = context;
-		this.originalQuad = new long[] { subj, pred, obj, context };
+		this.patternQuad = new long[] { subj, pred, obj, context };
 		this.quad = new long[] { subj, pred, obj, context };
 		this.pool = Pool.get();
 		this.keyData = pool.getVal();
 		this.valueData = pool.getVal();
 		this.index = index;
-		if (rangeSearch) {
+		this.indexScore = indexScore;
+		// prepare min and max keys if index can be used
+		// otherwise, leave as null to indicate full scan
+		if (indexScore > 0) {
 			minKeyBuf = pool.getKeyBuffer();
-			index.getMinKey(minKeyBuf, subj, pred, obj, context);
+			minValueBuf = pool.getKeyBuffer();
+			index.getMinEntry(minKeyBuf, minValueBuf, subj, pred, obj, context);
 			minKeyBuf.flip();
+			minValueBuf.flip();
 
 			this.maxKey = pool.getVal();
+			this.maxValue = pool.getVal();
 			this.maxKeyBuf = pool.getKeyBuffer();
-			index.getMaxKey(maxKeyBuf, subj, pred, obj, context);
+			this.maxValueBuf = pool.getKeyBuffer();
+			index.getMaxEntry(maxKeyBuf, maxValueBuf, subj, pred, obj, context);
 			maxKeyBuf.flip();
+			maxValueBuf.flip();
 			this.maxKey.mv_data(maxKeyBuf);
+			this.maxValue.mv_data(maxValueBuf);
 		} else {
 			minKeyBuf = null;
 			this.maxKey = null;
+			this.maxValue = null;
 		}
 
 		this.matchValues = subj > 0 || pred > 0 || obj > 0 || context >= 0;
@@ -163,13 +182,15 @@ public long[] next() {
 						minKeyBuf = pool.getKeyBuffer();
 					}
 					minKeyBuf.clear();
-					index.toKey(minKeyBuf, quad[0], quad[1], quad[2], quad[3]);
+					index.toEntry(minKeyBuf, minValueBuf, quad[0], quad[1], quad[2], quad[3]);
 					minKeyBuf.flip();
+					minValueBuf.flip();
 					keyData.mv_data(minKeyBuf);
-					lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_SET);
-					if (lastResult != MDB_SUCCESS) {
-						// use MDB_SET_RANGE if key was deleted
-						lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_SET_RANGE);
+					// use set range if entry was deleted
+					lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_SET_RANGE);
+					if (lastResult == MDB_SUCCESS) {
+						valueData.mv_data(minValueBuf);
+						lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_GET_BOTH_RANGE);
 					}
 					if (lastResult != MDB_SUCCESS) {
 						closeInternal(false);
@@ -187,7 +208,12 @@ public long[] next() {
 				if (minKeyBuf != null) {
 					// set cursor to min key
 					keyData.mv_data(minKeyBuf);
-					lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_SET_RANGE);
+					lastResult = indexScore >= 2 ? MDB_SUCCESS
+							: mdb_cursor_get(cursor, keyData, valueData, MDB_SET_RANGE);
+					if (lastResult == MDB_SUCCESS) {
+						valueData.mv_data(minValueBuf);
+						lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_GET_BOTH_RANGE);
+					}
 				} else {
 					// set cursor to first item
 					lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_NEXT);
@@ -196,14 +222,17 @@ public long[] next() {
 
 			while (lastResult == MDB_SUCCESS) {
 				// if (maxKey != null && TripleStore.COMPARATOR.compare(keyData.mv_data(), maxKey.mv_data()) > 0) {
-				if (maxKey != null && mdb_cmp(txn, dbi, keyData, maxKey) > 0) {
+				int keyDiff;
+				if (maxKey != null &&
+						(keyDiff = mdb_cmp(txn, dbi, keyData, maxKey)) >= 0
+						&& (keyDiff > 0 || mdb_dcmp(txn, dbi, valueData, maxValue) > 0)) {
 					lastResult = MDB_NOTFOUND;
-				} else if (matches()) {
+				} else if (notMatches()) {
 					// value doesn't match search key/mask, fetch next value
 					lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_NEXT);
 				} else {
 					// Matching value found
-					index.keyToQuad(keyData.mv_data(), originalQuad, quad);
+					index.entryToQuad(keyData.mv_data(), valueData.mv_data(), patternQuad, quad);
 					// fetch next value
 					fetchNext = true;
 					return quad;
@@ -216,13 +245,13 @@ public long[] next() {
 		}
 	}
 
-	private boolean matches() {
-
+	private boolean notMatches() {
 		if (groupMatcher != null) {
-			return !this.groupMatcher.matches(keyData.mv_data());
+			return !this.groupMatcher.matches(keyData.mv_data(), valueData.mv_data());
 		} else if (matchValues) {
+			// lazy init of group matcher
 			this.groupMatcher = index.createMatcher(subj, pred, obj, context);
-			return !this.groupMatcher.matches(keyData.mv_data());
+			return !this.groupMatcher.matches(keyData.mv_data(), valueData.mv_data());
 		} else {
 			return false;
 		}
@@ -247,10 +276,13 @@ private void closeInternal(boolean maybeCalledAsync) {
 					pool.free(valueData);
 					if (minKeyBuf != null) {
 						pool.free(minKeyBuf);
+						pool.free(minValueBuf);
 					}
 					if (maxKey != null) {
 						pool.free(maxKeyBuf);
 						pool.free(maxKey);
+						pool.free(maxValueBuf);
+						pool.free(maxValue);
 					}
 				}
 			} finally {