working on new ID based join iterator

Author: hmottestad

core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbRecordIterator.java

@@ -57,6 +57,7 @@ class LmdbRecordIterator implements RecordIterator {
 
 	private boolean matchValues;
 	private GroupMatcher groupMatcher;
+	private GroupMatcher matcherForEvaluation;
 
 	/**
 	 * True when late-bound variables exist beyond the contiguous prefix of the chosen index order, requiring
@@ -131,7 +132,7 @@ void initialize(TripleIndex index, KeyBuilder keyBuilder, boolean rangeSearch, l
 			long context, boolean explicit, Txn txnRef, long[] quadReuse, ByteBuffer minKeyBufParam,
 			ByteBuffer maxKeyBufParam) throws IOException {
 		if (initialized && !closed) {
-			throw new IllegalStateException("Cannot initialize LMDB record iterator while it is open");
+			// prepareForReuse();
 		}
 		initializeInternal(index, keyBuilder, rangeSearch, subj, pred, obj, context, explicit, txnRef, quadReuse,
 				minKeyBufParam, maxKeyBufParam);
@@ -141,12 +142,27 @@ void initialize(TripleIndex index, KeyBuilder keyBuilder, boolean rangeSearch, l
 	private void initializeInternal(TripleIndex index, KeyBuilder keyBuilder, boolean rangeSearch, long subj,
 			long pred, long obj, long context, boolean explicit, Txn txnRef, long[] quadReuse,
 			ByteBuffer minKeyBufParam, ByteBuffer maxKeyBufParam) throws IOException {
+
+//		if (!initialized) {
+//			System.out.println();
+//		} else {
+//			System.out.println();
+//		}
+
 		this.index = index;
+		long prevSubj = this.subj;
+		long prevPred = this.pred;
+		long prevObj = this.obj;
+		long prevContext = this.context;
+
 		this.subj = subj;
 		this.pred = pred;
 		this.obj = obj;
 		this.context = context;
 
+		boolean prevExternalMinKeyBuf = this.externalMinKeyBuf;
+		boolean prevExternalMaxKeyBuf = this.externalMaxKeyBuf;
+
 		if (quadReuse != null && quadReuse.length >= 4) {
 			this.quad = quadReuse;
 		} else if (this.quad == null || this.quad.length < 4) {
@@ -166,20 +182,36 @@ private void initializeInternal(TripleIndex index, KeyBuilder keyBuilder, boolea
 
 		if (rangeSearch) {
 			this.externalMinKeyBuf = minKeyBufParam != null;
-			this.minKeyBuf = externalMinKeyBuf ? minKeyBufParam : pool.getKeyBuffer();
+			if (externalMinKeyBuf) {
+				this.minKeyBuf = minKeyBufParam;
+			} else {
+				if (this.minKeyBuf == null || prevExternalMinKeyBuf) {
+					this.minKeyBuf = pool.getKeyBuffer();
+				} else {
+					this.minKeyBuf.clear();
+				}
+			}
 			minKeyBuf.clear();
 			if (keyBuilder != null) {
 				keyBuilder.writeMin(minKeyBuf);
 			} else {
-				index.getMinKey(minKeyBuf, subj, pred, obj, context);
+				index.getMinKey(minKeyBuf, subj, pred, obj, context, prevSubj, prevPred, prevObj, prevContext);
 			}
 			minKeyBuf.flip();
 
 			if (this.maxKey == null) {
 				this.maxKey = pool.getVal();
 			}
 			this.externalMaxKeyBuf = maxKeyBufParam != null;
-			this.maxKeyBuf = externalMaxKeyBuf ? maxKeyBufParam : pool.getKeyBuffer();
+			if (externalMaxKeyBuf) {
+				this.maxKeyBuf = maxKeyBufParam;
+			} else {
+				if (this.maxKeyBuf == null || prevExternalMaxKeyBuf) {
+					this.maxKeyBuf = pool.getKeyBuffer();
+				} else {
+					this.maxKeyBuf.clear();
+				}
+			}
 			maxKeyBuf.clear();
 			if (keyBuilder != null) {
 				keyBuilder.writeMax(maxKeyBuf);
@@ -193,24 +225,38 @@ private void initializeInternal(TripleIndex index, KeyBuilder keyBuilder, boolea
 				pool.free(maxKey);
 				this.maxKey = null;
 			}
-			if (this.maxKeyBuf != null && !externalMaxKeyBuf) {
+			if (this.maxKeyBuf != null && !prevExternalMaxKeyBuf) {
 				pool.free(maxKeyBuf);
+				this.maxKeyBuf = null;
 			}
 			this.externalMaxKeyBuf = maxKeyBufParam != null;
 			this.maxKeyBuf = externalMaxKeyBuf ? maxKeyBufParam : null;
 
 			if (subj > 0 || pred > 0 || obj > 0 || context >= 0) {
 				this.externalMinKeyBuf = minKeyBufParam != null;
-				this.minKeyBuf = externalMinKeyBuf ? minKeyBufParam : pool.getKeyBuffer();
+				if (externalMinKeyBuf) {
+					this.minKeyBuf = minKeyBufParam;
+				} else {
+					if (this.minKeyBuf == null || prevExternalMinKeyBuf) {
+						this.minKeyBuf = pool.getKeyBuffer();
+					} else {
+						this.minKeyBuf.clear();
+					}
+				}
 				minKeyBuf.clear();
-				index.getMinKey(minKeyBuf, subj, pred, obj, context);
+				index.getMinKey(minKeyBuf, subj, pred, obj, context, prevSubj, prevPred, prevObj, prevContext);
 				minKeyBuf.flip();
 			} else {
-				if (this.minKeyBuf != null && !externalMinKeyBuf) {
+				if (this.minKeyBuf != null && !prevExternalMinKeyBuf) {
 					pool.free(minKeyBuf);
+					this.minKeyBuf = null;
 				}
 				this.externalMinKeyBuf = minKeyBufParam != null;
-				this.minKeyBuf = externalMinKeyBuf ? minKeyBufParam : null;
+				if (externalMinKeyBuf) {
+					this.minKeyBuf = minKeyBufParam;
+				} else {
+					this.minKeyBuf = null;
+				}
 			}
 		}
 
@@ -219,13 +265,16 @@ private void initializeInternal(TripleIndex index, KeyBuilder keyBuilder, boolea
 		int boundCount = (subj > 0 ? 1 : 0) + (pred > 0 ? 1 : 0) + (obj > 0 ? 1 : 0) + (context >= 0 ? 1 : 0);
 		this.needMatcher = boundCount > prefixLen;
 		this.groupMatcher = null;
+		this.matcherForEvaluation = null;
 		this.fetchNext = false;
 		this.lastResult = MDB_SUCCESS;
 		this.closed = false;
 
-		this.dbi = index.getDB(explicit);
-		this.txnRef = txnRef;
-		this.txnLockManager = txnRef.lockManager();
+		if (!initialized) {
+			this.dbi = index.getDB(explicit);
+			this.txnRef = txnRef;
+			this.txnLockManager = txnRef.lockManager();
+		}
 
 		long readStamp;
 		try {
@@ -239,9 +288,12 @@ private void initializeInternal(TripleIndex index, KeyBuilder keyBuilder, boolea
 
 			// Try to reuse a pooled cursor only for read-only transactions; otherwise open a new one
 			if (txnRef.isReadOnly()) {
-				long pooled = pool.getCursor(dbi, index);
-				if (pooled != 0L) {
-					long c = pooled;
+				if (cursor == 0L) {
+					cursor = pool.getCursor(dbi, index);
+				}
+
+				if (cursor != 0L) {
+					long c = cursor;
 					try {
 						E(mdb_cursor_renew(txn, c));
 					} catch (IOException renewEx) {
@@ -262,6 +314,11 @@ private void initializeInternal(TripleIndex index, KeyBuilder keyBuilder, boolea
 					}
 				}
 			} else {
+				if (cursor != 0L) {
+					pool.freeCursor(dbi, index, cursor);
+					cursor = 0L;
+				}
+
 				try (MemoryStack stack = MemoryStack.stackPush()) {
 					PointerBuffer pp = stack.mallocPointer(1);
 					E(mdb_cursor_open(txn, dbi, pp));
@@ -276,7 +333,6 @@ private void initializeInternal(TripleIndex index, KeyBuilder keyBuilder, boolea
 	@Override
 	public long[] next() {
 		if (closed) {
-			log.debug("Calling next() on an LmdbRecordIterator that is already closed, returning null");
 			return null;
 		}
 		StampedLongAdderLockManager manager = txnLockManager;
@@ -345,7 +401,7 @@ public long[] next() {
 					return quad;
 				}
 			}
-			closeInternal(false);
+//			closeInternal(false);
 			return null;
 		} finally {
 			manager.unlockRead(readStamp);
@@ -358,16 +414,39 @@ private boolean matches() {
 			return false;
 		}
 
-		if (groupMatcher != null) {
-			return !this.groupMatcher.matches(keyData.mv_data());
+		if (matcherForEvaluation != null) {
+			return !matcherForEvaluation.matches(keyData.mv_data());
 		} else if (matchValues) {
-			this.groupMatcher = index.createMatcher(subj, pred, obj, context);
-			return !this.groupMatcher.matches(keyData.mv_data());
+			matcherForEvaluation = index.createMatcher(subj, pred, obj, context);
+			groupMatcher = matcherForEvaluation;
+			return !matcherForEvaluation.matches(keyData.mv_data());
 		} else {
 			return false;
 		}
 	}
 
+	private void prepareForReuse() {
+		if (cursor != 0L && txnRef != null) {
+			if (txnRef.isReadOnly()) {
+				pool.freeCursor(dbi, index, cursor);
+			} else {
+				mdb_cursor_close(cursor);
+			}
+		}
+		cursor = 0L;
+		groupMatcher = null;
+		matcherForEvaluation = null;
+		fetchNext = false;
+		lastResult = MDB_SUCCESS;
+		matchValues = false;
+		needMatcher = false;
+		txnRef = null;
+		txn = 0L;
+		txnRefVersion = 0L;
+		txnLockManager = null;
+		closed = true;
+	}
+
 	private void closeInternal(boolean maybeCalledAsync) {
 		StampedLongAdderLockManager manager = this.txnLockManager;
 		if (closed) {
@@ -414,7 +493,10 @@ private void closeInternal(boolean maybeCalledAsync) {
 			maxKeyBuf = null;
 			externalMinKeyBuf = false;
 			externalMaxKeyBuf = false;
-			groupMatcher = null;
+			if (maybeCalledAsync) {
+				groupMatcher = null;
+			}
+			matcherForEvaluation = null;
 			fetchNext = false;
 			lastResult = 0;
 			matchValues = false;