added the ability to skip to a given object when searching the store by class, by reference to an object, etc.

Author: sbazerque

src/storage/backends/Backend.ts

@@ -32,10 +32,25 @@ interface Backend {
 
     loadTerminalOpsForMutable(hash: Hash) : Promise<{lastOp: Hash, terminalOps: Array<Hash>} | undefined>;
 
+    // The BackendSearchResults struct returned by the following three contains two strings, start & end, that can be used to
+    // fetch more search results, for example by using the "end" string in params.start in another call to the search function.
+
+    // The common usage is then call searchBy___(...) first, using an arbitary size limit, and then repeatedly use the result.end
+    // to make more calls like searcgBy___(... {start: result.end}) to get all the results in fixed-sized batches.
+
+    // These index values are always strings and can be compared lexicographically.
+
     searchByClass(className: string, params? : BackendSearchParams) : Promise<BackendSearchResults>;
     searchByReference(referringPath: string, referencedHash: Hash, params? : BackendSearchParams) : Promise<BackendSearchResults>;
     searchByReferencingClass(referringClassName: string, referringPath: string, referencedHash: Hash, params? : BackendSearchParams) : Promise<BackendSearchResults>;
 
+    // the fowllowing 3 return the "start" parameter for the search functions above (the one that goes into params.start) so that
+    // the results will fast forward to startObject, skipping all the previous entries
+
+    skipToObjectByClass(className: string, startObject: Hash) : Promise<string|undefined>;
+    skipToObjectByReference(referringPath: string, referencedHash: Hash, startObject: Hash) : Promise<string|undefined>;
+    skipToObjectByReferencingClass(referringClassName: string, referringPath: string, referencedHash: Hash, startObject: Hash) : Promise<string|undefined>;
+
     close(): void;
 
     setStoredObjectCallback(objectStoreCallback: (literal: Literal) => Promise<void>): void;

src/storage/backends/IdbBackend.ts

@@ -414,6 +414,21 @@ class IdbBackend implements Backend {
     async loadLastCheckpoint(): Promise<StateCheckpoint|undefined> {
         throw new Error('Method not implemented.');
     }
+
+    skipToObjectByClass(className: string, startObject: Hash): Promise<string | undefined> {
+        className; startObject
+        throw new Error('Method not implemented.');
+    }
+
+    skipToObjectByReference(referringPath: string, referencedHash: string, startObject: Hash): Promise<string | undefined> {
+        referringPath; referencedHash; startObject;
+        throw new Error('Method not implemented.');
+    }
+
+    skipToObjectByReferencingClass(referringClassName: string, referringPath: string, referencedHash: string, startObject: Hash): Promise<string | undefined> {
+        referringClassName; referringPath; referencedHash; startObject;
+        throw new Error('Method not implemented.');
+    }
 }
 
 Store.registerBackend(IdbBackend.backendName, (dbName: string) => new IdbBackend(dbName));

src/storage/backends/MemoryBackend.ts

@@ -212,6 +212,21 @@ class MemoryBackend implements Backend {
         return this.searchByIndex(key, this.repr.sortedReferencingClassIndex, params);
     }
 
+    skipToObjectByClass(className: string, startObject: Hash): Promise<string|undefined> {
+        return this.skipToObjectByIndex(className, this.repr.sortedClassIndex, startObject);
+    }
+    
+    skipToObjectByReference(referringPath: string, referencedHash: string, startObject: Hash): Promise<string|undefined> {
+        let key =  referringPath + '#' + referencedHash;
+        return this.skipToObjectByIndex(key, this.repr.sortedReferenceIndex, startObject);
+    }
+    
+    skipToObjectByReferencingClass(referringClassName: string, referringPath: string, referencedHash: string, startObject: Hash): Promise<string|undefined> {
+        let key = referringClassName + '.' + referringPath + '#' + referencedHash;
+        return this.skipToObjectByIndex(key, this.repr.sortedReferencingClassIndex, startObject);
+
+    }
+
     async loadOpHeader(opHash: string): Promise<StoredOpHeader | undefined> {
         return this.repr.opCausalHistories.get(opHash);
     }
@@ -220,32 +235,45 @@ class MemoryBackend implements Backend {
         return this.repr.opCausalHistoriesByHash.get(causalHistoryHash);
     }
 
+    private async skipToObjectByIndex(key: string, sortedIndex: Map<string, Hash[]>, objectHash: Hash): Promise<string|undefined> {
+
+        let classHashes = sortedIndex.get(key);
+
+        if (classHashes !== undefined) {
+            let idx = classHashes.indexOf(objectHash);
+
+            if (idx >= 0) {
+                return MemoryBackend.toStringIndex(idx);
+            }
+        }
+
+        return undefined;
+
+    }
+
     private async searchByIndex(key: string, sortedIndex: Map<string, Hash[]>, params?: BackendSearchParams | undefined): Promise<BackendSearchResults> {
 
         let classHashes = sortedIndex.get(key);
 
-
         if (classHashes === undefined) {
             return { items: [], start: '', end: ''}
         } else {
             let order = (params === undefined || params.order === undefined) ? 'asc' : params.order.toLowerCase();
 
             let segment;
 
-
-
             if (order === 'desc') {
                 classHashes.reverse();
             }
 
             let start = 0;
 
             if (params !== undefined && params.start !== undefined) {
-                start = Number.parseInt(params.start);
+                start = MemoryBackend.fromStringIndex(params.start);
             }
 
             if (start >= classHashes.length) {
-                return { items: [], start: classHashes.length.toString(), end: classHashes.length.toString()}
+                return { items: [], start: MemoryBackend.toStringIndex(classHashes.length), end: MemoryBackend.toStringIndex(classHashes.length)}
             }
 
             let end = classHashes.length
@@ -256,12 +284,20 @@ class MemoryBackend implements Backend {
 
             let result:Literal[] =  segment.map((hash: Hash) => this.repr.objects.get(hash)?.literal as Literal);
 
-            return { start: start.toString(), end: end.toString(), items: result };
+            return { start: MemoryBackend.toStringIndex(start), end: MemoryBackend.toStringIndex(end), items: result };
 
         }
 
     }
 
+    private static toStringIndex(idx: number): string {
+        return idx.toString(16).padStart(16, '0');
+    }
+
+    private static fromStringIndex(idxString: string): number {
+        return Number.parseInt(idxString, 16);
+    }
+
     async storeCheckpoint(checkpoint: StateCheckpoint): Promise<void> {
         this.repr.checkpoints.set(checkpoint.mutableObject, checkpoint);
     }

src/storage/store/Store.ts

@@ -610,6 +610,18 @@ class Store {
         return this.loadSearchResults(searchResults, context);
     }
 
+    async skipToObjectByClass(className: string, object: Hash): Promise<string|undefined> {
+        return this.backend.skipToObjectByClass(className, object);
+    }
+
+    async skipToObjectByReference(referringPath: string, referencedHash: Hash, object: Hash): Promise <string|undefined> {
+        return this.backend.skipToObjectByReference(referringPath, referencedHash, object);
+    }
+
+    async skipToObjectByReferencingClass(referringClassName: string, referringPath: string, referencedHash: Hash, object: Hash): Promise<string|undefined> {
+        return this.backend.skipToObjectByReferencingClass(referringClassName, referringPath, referencedHash, object);
+    }
+
     async loadOpHeader(opHash: Hash): Promise<OpHeader | undefined> {
         const stored = await this.backend.loadOpHeader(opHash);