feat(json-crdt): 🎸 implement file encoding and decoding

Author: streamich

src/json-crdt/file/File.ts

@@ -2,6 +2,7 @@ import {Model} from '../model';
 import {PatchLog} from './PatchLog';
 import {FileModelEncoding} from './constants';
 import {Encoder as SidecarEncoder} from '../codec/sidecar/binary/Encoder';
+import {Decoder as SidecarDecoder} from '../codec/sidecar/binary/Decoder';
 import {Encoder as StructuralEncoderCompact} from '../codec/structural/compact/Encoder';
 import {Encoder as StructuralEncoderVerbose} from '../codec/structural/verbose/Encoder';
 import {encode as encodeCompact} from '../../json-crdt-patch/codec/compact/encode';
@@ -10,15 +11,55 @@ import {Writer} from '../../util/buffers/Writer';
 import {CborEncoder} from '../../json-pack/cbor/CborEncoder';
 import {JsonEncoder} from '../../json-pack/json/JsonEncoder';
 import {printTree} from '../../util/print/printTree';
+import {decodeModel, decodeNdjsonComponents, decodePatch, decodeSeqCborComponents} from './util';
 import type * as types from './types';
 import type {Printable} from '../../util/print/types';
 
 export class File implements Printable {
+  public static unserialize(components: types.FileReadSequence): File {
+    const [view, metadata, model, history, ...frontier] = components;
+    const modelFormat = metadata[1];
+    let decodedModel: Model<any> | null = null;
+    if (model && modelFormat !== FileModelEncoding.None) {
+      const isSidecar = modelFormat === FileModelEncoding.SidecarBinary;
+      if (isSidecar) {
+        const decoder = new SidecarDecoder();
+        if (!(model instanceof Uint8Array)) throw new Error('NOT_BLOB');
+        decodedModel = decoder.decode(view, model);
+      } else {
+        decodedModel = decodeModel(model);
+      }
+    }
+    let log: PatchLog | null = null;
+    if (history) {
+      const [start, patches] = history;
+      if (start) {
+        const startModel = decodeModel(start);
+        log = new PatchLog(startModel);
+        for (const patch of patches) log.push(decodePatch(patch));
+      }
+    }
+    if (!log) throw new Error('NO_HISTORY');
+    if (!decodedModel) decodedModel = log.replayToEnd();
+    const file = new File(decodedModel, log);
+    return file;
+  }
+
+  public static fromNdjson(blob: Uint8Array): File {
+    const components = decodeNdjsonComponents(blob);
+    return File.unserialize(components as types.FileReadSequence);
+  }
+
+  public static fromSeqCbor(blob: Uint8Array): File {
+    const components = decodeSeqCborComponents(blob);
+    return File.unserialize(components as types.FileReadSequence);
+  }
+
   public static fromModel(model: Model<any>): File {
     return new File(model, PatchLog.fromModel(model));
   }
 
-  constructor(public readonly model: Model, public readonly history: PatchLog) {}
+  constructor(public readonly model: Model, public readonly log: PatchLog) {}
 
   public serialize(params: types.FileSerializeParams = {}): types.FileWriteSequence {
     const view = this.model.view();
@@ -48,33 +89,41 @@ export class File implements Printable {
         model = new StructuralEncoderVerbose().encode(this.model);
         break;
       }
+      case 'none': {
+        metadata[1] = FileModelEncoding.None;
+        model = null;
+        break;
+      }
       default:
         throw new Error(`Invalid model format: ${modelFormat}`);
     }
     const history: types.FileWriteSequenceHistory = [null, []];
     const patchFormat = params.history ?? 'binary';
     switch (patchFormat) {
       case 'binary': {
-        history[0] = this.history.start.toBinary();
-        this.history.patches.forEach(({v}) => {
+        history[0] = this.log.start.toBinary();
+        this.log.patches.forEach(({v}) => {
           history[1].push(v.toBinary());
         });
         break;
       }
       case 'compact': {
-        history[0] = new StructuralEncoderCompact().encode(this.history.start);
-        this.history.patches.forEach(({v}) => {
+        history[0] = new StructuralEncoderCompact().encode(this.log.start);
+        this.log.patches.forEach(({v}) => {
           history[1].push(encodeCompact(v));
         });
         break;
       }
       case 'verbose': {
-        history[0] = new StructuralEncoderVerbose().encode(this.history.start);
-        this.history.patches.forEach(({v}) => {
+        history[0] = new StructuralEncoderVerbose().encode(this.log.start);
+        this.log.patches.forEach(({v}) => {
           history[1].push(encodeVerbose(v));
         });
         break;
       }
+      case 'none': {
+        break;
+      }
       default:
         throw new Error(`Invalid history format: ${patchFormat}`);
     }
@@ -104,6 +153,6 @@ export class File implements Printable {
   // ---------------------------------------------------------------- Printable
 
   public toString(tab?: string) {
-    return `file` + printTree(tab, [(tab) => this.model.toString(tab), () => '', (tab) => this.history.toString(tab)]);
+    return `file` + printTree(tab, [(tab) => this.model.toString(tab), () => '', (tab) => this.log.toString(tab)]);
   }
 }

src/json-crdt/file/PatchLog.ts

@@ -25,6 +25,12 @@ export class PatchLog implements Printable {
     this.patches.set(id, patch);
   }
 
+  public replayToEnd(): Model {
+    const model = this.start.clone();
+    this.patches.forEach(({v}) => model.applyPatch(v));
+    return model;
+  }
+
   // ---------------------------------------------------------------- Printable
 
   public toString(tab?: string) {

src/json-crdt/file/__tests__/File.spec.ts

@@ -18,11 +18,11 @@ test('can create File from new model', () => {
     }),
   );
   const file = File.fromModel(model);
-  expect(file.history.start.view()).toBe(undefined);
+  expect(file.log.start.view()).toBe(undefined);
   expect(file.model.view()).toEqual({
     foo: 'bar',
   });
-  expect(file.history.start.clock.sid).toBe(file.model.clock.sid);
+  expect(file.log.start.clock.sid).toBe(file.model.clock.sid);
 });
 
 test.todo('patches are flushed and stored in memory');
@@ -46,4 +46,26 @@ describe('.toBinary()', () => {
       expect(view).toEqual({foo: 'bar'});
     });
   });
+
+  describe('can decode from blob', () => {
+    test('.ndjson', () => {
+      const {file} = setup({foo: 'bar'});
+      const blob = file.toBinary({format: 'ndjson', model: 'compact', history: 'compact'});
+      const file2 = File.fromNdjson(blob);
+      expect(file2.model.view()).toEqual({foo: 'bar'});
+      expect(file2.model !== file.model).toBe(true);
+      expect(file.log.start.view()).toEqual(undefined);
+      expect(file.log.replayToEnd().view()).toEqual({foo: 'bar'});
+    });
+
+    test('.seq.cbor', () => {
+      const {file} = setup({foo: 'bar'});
+      const blob = file.toBinary({format: 'seq.cbor', model: 'binary', history: 'binary'});
+      const file2 = File.fromSeqCbor(blob);
+      expect(file2.model.view()).toEqual({foo: 'bar'});
+      expect(file2.model !== file.model).toBe(true);
+      expect(file.log.start.view()).toEqual(undefined);
+      expect(file.log.replayToEnd().view()).toEqual({foo: 'bar'});
+    });
+  });
 });

src/json-crdt/file/types.ts

@@ -15,8 +15,8 @@ export type FileReadSequence = [...FileWriteSequence, ...frontier: Array<Uint8Ar
 
 export interface FileSerializeParams {
   noView?: boolean;
-  model?: 'sidecar' | 'binary' | 'compact' | 'verbose';
-  history?: 'binary' | 'compact' | 'verbose';
+  model?: 'sidecar' | 'binary' | 'compact' | 'verbose' | 'none';
+  history?: 'binary' | 'compact' | 'verbose' | 'none';
 }
 
 export interface FileEncodingParams extends FileSerializeParams {

src/json-crdt/file/util.ts

@@ -0,0 +1,52 @@
+import {JsonDecoder} from '../../json-pack/json/JsonDecoder';
+import {CborDecoder} from '../../json-pack/cbor/CborDecoder';
+import {Model} from '../model';
+import {Decoder as StructuralDecoderCompact} from '../codec/structural/compact/Decoder';
+import {Decoder as StructuralDecoderVerbose} from '../codec/structural/verbose/Decoder';
+import {decode as decodeCompact} from '../../json-crdt-patch/codec/compact/decode';
+import {decode as decodeVerbose} from '../../json-crdt-patch/codec/verbose/decode';
+import {Patch} from '../../json-crdt-patch';
+import type {JsonCrdtCompactDocument} from '../codec/structural/compact';
+import type {JsonCrdtVerboseDocument} from '../codec/structural/verbose';
+import type {CompactCodecPatch} from '../../json-crdt-patch/codec/compact';
+import type {JsonCodecPatch} from '../../json-crdt-patch/codec/verbose';
+
+export const decodeNdjsonComponents = (blob: Uint8Array): unknown[] => {
+  const decoder = new JsonDecoder();
+  const reader = decoder.reader;
+  reader.reset(blob);
+  const components: unknown[] = [];
+  while (reader.x < blob.length) {
+    components.push(decoder.readAny());
+    const nl = reader.u8();
+    if (nl !== '\n'.charCodeAt(0)) throw new Error('NDJSON_UNEXPECTED_NEWLINE');
+  }
+  return components;
+};
+
+export const decodeSeqCborComponents = (blob: Uint8Array): unknown[] => {
+  const decoder = new CborDecoder();
+  const reader = decoder.reader;
+  reader.reset(blob);
+  const components: unknown[] = [];
+  while (reader.x < blob.length) components.push(decoder.val());
+  return components;
+};
+
+export const decodeModel = (serialized: unknown): Model => {
+  if (!serialized) throw new Error('NO_MODEL');
+  if (serialized instanceof Uint8Array) return Model.fromBinary(serialized);
+  if (Array.isArray(serialized))
+    return new StructuralDecoderCompact().decode(<JsonCrdtCompactDocument>serialized);
+  if (typeof serialized === 'object')
+    return new StructuralDecoderVerbose().decode(<JsonCrdtVerboseDocument>serialized);
+  throw new Error('UNKNOWN_MODEL');
+};
+
+export const decodePatch = (serialized: unknown): Patch => {
+  if (!serialized) throw new Error('NO_MODEL');
+  if (serialized instanceof Uint8Array) return Patch.fromBinary(serialized);
+  if (Array.isArray(serialized)) return decodeCompact(<CompactCodecPatch>serialized);
+  if (typeof serialized === 'object') return decodeVerbose(<JsonCodecPatch>serialized);
+  throw new Error('UNKNOWN_MODEL');
+};