added a start for a checkpointing implementation

Author: sbazerque

src/data/history/HistoryLog.ts

@@ -17,8 +17,8 @@ class StateTransition<T extends MutableObject> extends MutationOp {
         super(log);
 
         this.mutableHash = mutableHash;
-        this.start   = start;
-        this.end     = end;
+        this.start       = start;
+        this.end         = end;
 
         this.info     = info;
     }

src/data/model/mutable/MutableObject.ts

@@ -28,15 +28,24 @@ enum MutableContentEvents {
 
 const ContentChangeEventActions: Array<string> = [MutableContentEvents.AddObject, MutableContentEvents.RemoveObject];
 
-type MutableObjectConfig = {supportsUndo?: boolean};
+type MutableObjectConfig = {supportsUndo?: boolean, supportsCheckpoints?: boolean, checkpointFreq?: number};
+
+type StateCheckpoint = {
+    mutableObject: Hash,
+    terminalOpHashes: Array<Hash>, 
+    allAppliedOps: Array<Hash>, 
+    activeCascInvsPerOp: Array<[Hash, Array<Hash>]>, 
+    exportedState: any
+};
 
 abstract class MutableObject extends HashedObject {
 
     static controlLog = new Logger(MutableObject.name, LogLevel.INFO)
     static prevOpsComputationLog = new Logger(MutableObject.name, LogLevel.INFO);
 
     readonly _acceptedMutationOpClasses : Array<string>;
-    
+    readonly _supportsCheckpoints: boolean;
+
     _allAppliedOps : Set<Hash>;
     _terminalOps   : Map<Hash, MutationOp>;
 
@@ -68,6 +77,8 @@ abstract class MutableObject extends HashedObject {
                 acceptedOpClasses.push(CascadedInvalidateOp.className);
             }
         }
+
+        this._supportsCheckpoints = config?.supportsCheckpoints || false;
 
         this._acceptedMutationOpClasses = acceptedOpClasses;
         this._boundToStore = false;
@@ -112,6 +123,17 @@ abstract class MutableObject extends HashedObject {
     abstract getMutableContents(): MultiMap<Hash, HashedObject>;
     abstract getMutableContentByHash(hash: Hash): Set<HashedObject>;
 
+    // override the following two to support checkpointing (and pass apropriate params in config to constructor)
+
+    exportMutableState(): any {
+        throw new Error(this.getClassName() + ': this class does not support state packing')
+    }
+
+    importMutableState(state: any) {
+        state;
+        throw new Error(this.getClassName() + ': this class does not support applying packed state')
+    }
+
     /*
     // override if appropiate
     async undo(op: MutationOp): Promise<boolean> {
@@ -224,6 +246,17 @@ abstract class MutableObject extends HashedObject {
 
         await super.loadAllChanges(batchSize, context);
 
+        if (this._supportsCheckpoints) {
+            const checkpoint = await this.getStore().loadLastCheckpoint(this.getLastHash());
+
+            if (checkpoint !== undefined) {
+                this.restoreCheckpoint(checkpoint);
+
+                // TODO: find a way to get the correct "start" parameter for loadByReference below
+                //       to make it ignore all the ops in the checkpoint
+            }
+        }
+
         let results = await this.getStore()
                                 .loadByReference(
                                     'targetObject', 
@@ -510,6 +543,55 @@ abstract class MutableObject extends HashedObject {
         this._unsavedOps.push(op);
     }
 
+    // checkpointing
+
+    async saveCheckpoint() {
+
+        await this.saveQueuedOps();
+
+        const check: StateCheckpoint = {
+            mutableObject: this.getLastHash(),
+            terminalOpHashes: Array.from(this._terminalOps.keys()), 
+            allAppliedOps: Array.from(this._allAppliedOps), 
+            activeCascInvsPerOp: Array.from(this._activeCascInvsPerOp.entries()).map((v: [Hash, Set<Hash>]) => [v[0], Array.from(v[1].values())]),
+            exportedState: this.exportMutableState()
+        };
+
+        await this.getStore().saveCheckpoint(check);
+
+    }
+
+    async restoreCheckpoint(checkpoint: StateCheckpoint) {
+        if (this.getLastHash() !== checkpoint.mutableObject) {
+            throw new Error('Trying to apply a state checkpoint to ' + this.getLastHash() + ', but the checkpoint is for ' + checkpoint.mutableObject);
+        }
+
+        const terminalOps = new Map<Hash, MutationOp>();
+
+        for (const opHash of checkpoint.terminalOpHashes.values()) {
+            const op = await this.loadOp(opHash);
+
+            if (op === undefined) {
+                throw new Error('Cannot apply checkpoint to ' + this.getLastHash() + ', missing op: ' + opHash);
+            }
+
+            terminalOps.set(opHash, op);
+        }
+
+        this._terminalOps = terminalOps;
+        this._allAppliedOps = new Set(checkpoint.allAppliedOps.values());
+        this._activeCascInvsPerOp = new MultiMap();
+
+        for (const [k, vs] of checkpoint.activeCascInvsPerOp) {
+            for (const v of vs) {
+                this._activeCascInvsPerOp.add(k, v);
+            }
+        }
+
+        this.importMutableState(checkpoint.exportedState);
+    }
+
+
     literalizeInContext(context: Context, path: string, flags?: Array<string>) : Hash {
 
         if (flags === undefined) {
@@ -710,4 +792,5 @@ abstract class MutableObject extends HashedObject {
 }
 
 export { MutableObject, MutableContentEvents };
-export type { MutableObjectConfig };
+export type { MutableObjectConfig };
+export type { StateCheckpoint };

src/storage/backends/Backend.ts

@@ -1,6 +1,7 @@
 import { Literal } from 'data/model/literals/LiteralUtils';
 import { Hash } from 'data/model/hashing/Hashing';
 import { StoredOpHeader } from '../store/Store';
+import { StateCheckpoint } from 'data/model';
 
 type BackendSearchParams = {order?: 'asc'|'desc'|undefined, start?: string, limit?: number};
 type BackendSearchResults = {items : Array<Literal>, start?: string, end?: string };
@@ -23,6 +24,9 @@ interface Backend {
     store(literal : Literal, history?: StoredOpHeader) : Promise<void>;
     load(hash: Hash) : Promise<Storable | undefined>;
 
+    storeCheckpoint(checkpoint: StateCheckpoint): Promise<void>;
+    loadLastCheckpoint(mutableObject: Hash): Promise<StateCheckpoint|undefined>;
+
     loadOpHeader(opHash: Hash) : Promise<StoredOpHeader | undefined>;
     loadOpHeaderByHeaderHash(causalHistoryHash: Hash) : Promise<StoredOpHeader | undefined>;
 

src/storage/backends/IdbBackend.ts

@@ -3,7 +3,7 @@ import { openDB, IDBPDatabase } from 'idb';
 
 import { Logger, LogLevel } from 'util/logging';
 
-import { Literal, Hash, HashedSet, HashReference } from 'data/model';
+import { Literal, Hash, HashedSet, HashReference, StateCheckpoint } from 'data/model';
 
 import { Backend, BackendSearchParams, BackendSearchResults, Storable } from './Backend'; 
 import { Store, StoredOpHeader } from 'storage/store/Store';
@@ -403,6 +403,17 @@ class IdbBackend implements Backend {
             return false;
         }
     }
+
+    // TODO: add a STORE for the checkpoints, for now limited to max idb value size
+
+    async storeCheckpoint(checkpoint: StateCheckpoint): Promise<void> {
+        checkpoint;
+        throw new Error('Method not implemented.');
+    }
+
+    async loadLastCheckpoint(): Promise<StateCheckpoint|undefined> {
+        throw new Error('Method not implemented.');
+    }
 }
 
 Store.registerBackend(IdbBackend.backendName, (dbName: string) => new IdbBackend(dbName));

src/storage/backends/MemoryBackend.ts

@@ -1,5 +1,5 @@
 import { Backend, BackendSearchParams, BackendSearchResults, Storable } from './Backend';
-import { Literal, Hash, HashReference, HashedSet } from 'data/model';
+import { Literal, Hash, HashReference, HashedSet, StateCheckpoint } from 'data/model';
 import { MultiMap } from 'util/multimap';
 import { Store, StoredOpHeader } from 'storage/store/Store';
 import { LiteralUtils } from 'data/model/literals/LiteralUtils';
@@ -24,7 +24,8 @@ type MemoryRepr = {
     terminalOps: MultiMap<Hash, Hash>,
     lastOps: Map<Hash, Hash>,
     opCausalHistories: Map<Hash, MemOpCausalHistoryFormat>,
-    opCausalHistoriesByHash: Map<Hash, MemOpCausalHistoryFormat>
+    opCausalHistoriesByHash: Map<Hash, MemOpCausalHistoryFormat>,
+    checkpoints: Map<Hash, StateCheckpoint>
 }
 
 class MemoryBackend implements Backend {
@@ -75,7 +76,8 @@ class MemoryBackend implements Backend {
                 terminalOps: new MultiMap(),
                 lastOps: new Map(),
                 opCausalHistories: new Map(),
-                opCausalHistoriesByHash: new Map()
+                opCausalHistoriesByHash: new Map(),
+                checkpoints: new Map()
             }
         }
 
@@ -260,6 +262,14 @@ class MemoryBackend implements Backend {
 
     }
 
+    async storeCheckpoint(checkpoint: StateCheckpoint): Promise<void> {
+        this.repr.checkpoints.set(checkpoint.mutableObject, checkpoint);
+    }
+    
+    async loadLastCheckpoint(mutableObject: Hash): Promise<StateCheckpoint|undefined> {
+        return this.repr.checkpoints.get(mutableObject);
+    }
+
     async ready(): Promise<void> {
 
     }

src/storage/store/Store.ts

@@ -1,5 +1,5 @@
 import { Backend, BackendSearchParams, BackendSearchResults, Storable } from '../backends/Backend'; 
-import { HashedObject, MutableObject, Literal, Context, HashReference, MutationOp, HashedSet, LiteralUtils } from 'data/model';
+import { HashedObject, MutableObject, Literal, Context, HashReference, MutationOp, HashedSet, LiteralUtils, StateCheckpoint } from 'data/model';
 import { Hash } from 'data/model/hashing/Hashing';
 
 import { MultiMap } from 'util/multimap';
@@ -921,6 +921,13 @@ class Store {
         this.backend.close();
     }
 
+    async saveCheckpoint(checkpoint: StateCheckpoint) {
+        await this.backend.storeCheckpoint(checkpoint);
+    }
+
+    async loadLastCheckpoint(mutableObject: Hash) {
+        return this.backend.loadLastCheckpoint(mutableObject);
+    }
 }
 
 export { Store, StoredOpHeader, LoadResults };