Merge pull request #800 from streamich/peritext-annotation-registry

Peritext annotation registry

Author: streamich

src/json-crdt-extensions/peritext/__tests__/Peritext.render-block.spec.ts

@@ -47,8 +47,6 @@ const runInlineSlicesTests = (
       editor.saved.insMarker(['p'], {foo: 'bar'});
       expect(view()).toMatchInlineSnapshot(`
 "<>
-  <0>
-    "" {  }
   <p> { foo = "bar" }
     "abcdefghijklmnopqrstuvwxyz" {  }
 "

src/json-crdt-extensions/peritext/block/Fragment.ts

@@ -80,6 +80,8 @@ export class Fragment extends Range implements Printable, Stateful {
     let pair: ReturnType<typeof iterator>;
     while ((pair = iterator())) {
       const [p1, p2] = pair;
+      const skipFirstVirtualBlock = !p1 && this.start.isAbsStart() && p2 && p2.viewPos() === 0;
+      if (skipFirstVirtualBlock) continue;
       const type = p1 ? p1.type() : CommonSliceType.p;
       const path = type instanceof Array ? type : [type];
       const block = this.insertBlock(parent, path, p1, p2);

src/json-crdt-extensions/peritext/block/types.ts

@@ -1,10 +1,15 @@
+import type {SliceBehavior} from '../slice/constants';
+
 export type PeritextMlNode = string | PeritextMlElement;
-export type PeritextMlElement = [
-  tag: string | number,
-  attrs: null | PeritextMlAttributes,
+
+export type PeritextMlElement<Tag extends string | number = string | number, Data = unknown, Inline = boolean> = [
+  tag: Tag,
+  attrs: null | PeritextMlAttributes<Data, Inline>,
   ...children: PeritextMlNode[],
 ];
-export interface PeritextMlAttributes {
-  inline?: boolean;
-  data?: unknown;
+
+export interface PeritextMlAttributes<Data = unknown, Inline = boolean> {
+  data?: Data;
+  inline?: Inline;
+  behavior?: SliceBehavior;
 }

src/json-crdt-extensions/peritext/lazy/__tests__/import-export-html.spec.ts

@@ -0,0 +1,14 @@
+import {setupKit} from '../../__tests__/setup';
+import {CommonSliceType} from '../../slice';
+import {fromHtml, toViewRange} from '../import-html';
+
+test('a single paragraph', () => {
+  const {peritext} = setupKit();
+  const html = '<p>Hello world</p>';
+  const peritextMl = fromHtml(html);
+  const rangeView = toViewRange(peritextMl);
+  peritext.editor.import(0, rangeView);
+  peritext.refresh();
+  const json = peritext.blocks.toJson();
+  expect(json).toEqual(['', null, [CommonSliceType.p, null, 'Hello world']]);
+});

src/json-crdt-extensions/peritext/lazy/__tests__/import-html.spec.ts

@@ -0,0 +1,100 @@
+import {Anchor} from '../../rga/constants';
+import {CommonSliceType} from '../../slice';
+import {SliceBehavior, SliceHeaderShift} from '../../slice/constants';
+import {fromHtml, toViewRange} from '../import-html';
+
+describe('.fromHtml()', () => {
+  test('a single paragraph', () => {
+    const html = '<p>Hello world</p>';
+    const peritextMl = fromHtml(html);
+    expect(peritextMl).toEqual(['', null, [CommonSliceType.p, null, 'Hello world']]);
+  });
+
+  test('a paragraph with trailing text', () => {
+    const html = '<p>Hello world</p> more text';
+    const peritextMl = fromHtml(html);
+    expect(peritextMl).toEqual(['', null, [CommonSliceType.p, null, 'Hello world'], ' more text']);
+  });
+
+  test('text formatted as italic', () => {
+    const html = '<p>Hello world</p>\n<p><em>italic</em> text, <i>more italic</i></p>';
+    const peritextMl = fromHtml(html);
+    expect(peritextMl).toEqual([
+      '',
+      null,
+      [CommonSliceType.p, null, 'Hello world'],
+      '\n',
+      [
+        CommonSliceType.p,
+        null,
+        [CommonSliceType.i, {behavior: SliceBehavior.One, inline: true}, 'italic'],
+        ' text, ',
+        [CommonSliceType.i, {behavior: SliceBehavior.One, inline: true}, 'more italic'],
+      ],
+    ]);
+  });
+});
+
+describe('.toViewRange()', () => {
+  test('plain text', () => {
+    const html = 'this is plain text';
+    const peritextMl = fromHtml(html);
+    const view = toViewRange(peritextMl);
+    expect(view).toEqual(['this is plain text', 0, []]);
+  });
+
+  test('a single paragraph', () => {
+    const html = '<p>Hello world</p>';
+    const peritextMl = fromHtml(html);
+    const view = toViewRange(peritextMl);
+    expect(view).toEqual(['\nHello world', 0, [[0, 0, 0, 0]]]);
+  });
+
+  test('two consecutive paragraphs', () => {
+    const html = '<p>Hello world</p><p>Goodbye world</p>';
+    const peritextMl = fromHtml(html);
+    const view = toViewRange(peritextMl);
+    expect(view).toEqual([
+      '\nHello world\nGoodbye world',
+      0,
+      [
+        [0, 0, 0, 0],
+        [0, 12, 12, 0],
+      ],
+    ]);
+  });
+
+  test('two paragraphs with whitespace gap', () => {
+    const html = '  <p>Hello world</p>\n  <p>Goodbye world</p>';
+    const peritextMl = fromHtml(html);
+    const view = toViewRange(peritextMl);
+    expect(view).toEqual([
+      '\nHello world\nGoodbye world',
+      0,
+      [
+        [0, 0, 0, 0],
+        [0, 12, 12, 0],
+      ],
+    ]);
+  });
+
+  test('single inline annotation', () => {
+    const html = 'here is some <em>italic</em> text';
+    const peritextMl = fromHtml(html);
+    const view = toViewRange(peritextMl);
+    expect(view).toEqual([
+      'here is some italic text',
+      0,
+      [
+        [
+          (SliceBehavior.One << SliceHeaderShift.Behavior) +
+            (Anchor.Before << SliceHeaderShift.X1Anchor) +
+            (Anchor.After << SliceHeaderShift.X2Anchor),
+          13,
+          19,
+          CommonSliceType.i,
+        ],
+      ],
+    ]);
+  });
+});

src/json-crdt-extensions/peritext/lazy/export-markdown.ts

@@ -6,9 +6,7 @@ import type {PeritextMlNode} from '../block/types';
 
 export const toMdast = (json: PeritextMlNode): IRoot => {
   const hast = toHast(json);
-  // console.log(hast);
   const mdast = _toMdast(hast) as IRoot;
-  // console.log(mdast);
   return mdast;
 };
 

src/json-crdt-extensions/peritext/lazy/import-html.ts

@@ -0,0 +1,117 @@
+import {html as _html} from 'very-small-parser/lib/html';
+import {fromHast as _fromHast} from 'very-small-parser/lib/html/json-ml/fromHast';
+import {SliceTypeName} from '../slice';
+import {registry as defaultRegistry} from '../registry/registry';
+import {SliceBehavior, SliceHeaderShift} from '../slice/constants';
+import {Anchor} from '../rga/constants';
+import type {JsonMlNode} from 'very-small-parser/lib/html/json-ml/types';
+import type {THtmlToken} from 'very-small-parser/lib/html/types';
+import type {PeritextMlNode} from '../block/types';
+import type {SliceRegistry} from '../registry/SliceRegistry';
+import type {ViewRange, ViewSlice} from '../editor/types';
+
+/**
+ * Flattens a {@link PeritextMlNode} tree structure into a {@link ViewRange}
+ * flat string with annotation ranges.
+ */
+class ViewRangeBuilder {
+  private text = '';
+  private slices: ViewSlice[] = [];
+
+  constructor(private registry: SliceRegistry) {}
+
+  private build0(node: PeritextMlNode, depth = 0): void {
+    const skipWhitespace = depth < 2;
+    if (typeof node === 'string') {
+      if (skipWhitespace && !node.trim()) return;
+      this.text += node;
+      return;
+    }
+    const [type, attr] = node;
+    const start = this.text.length;
+    const length = node.length;
+    const inline = !!attr?.inline;
+    if (!!type || type === 0) {
+      let end: number = 0,
+        header: number = 0;
+      if (!inline) {
+        this.text += '\n';
+        end = start;
+        header =
+          (SliceBehavior.Marker << SliceHeaderShift.Behavior) +
+          (Anchor.Before << SliceHeaderShift.X1Anchor) +
+          (Anchor.Before << SliceHeaderShift.X2Anchor);
+        const slice: ViewSlice = [header, start, end, type];
+        const data = attr?.data;
+        if (data) slice.push(data);
+        this.slices.push(slice);
+      }
+    }
+    for (let i = 2; i < length; i++) this.build0(node[i] as PeritextMlNode, depth + 1);
+    if (!!type || type === 0) {
+      let end: number = 0,
+        header: number = 0;
+      if (inline) {
+        end = this.text.length;
+        const behavior: SliceBehavior = attr?.behavior ?? SliceBehavior.Many;
+        header =
+          (behavior << SliceHeaderShift.Behavior) +
+          (Anchor.Before << SliceHeaderShift.X1Anchor) +
+          (Anchor.After << SliceHeaderShift.X2Anchor);
+        const slice: ViewSlice = [header, start, end, type];
+        const data = attr?.data;
+        if (data) slice.push(data);
+        this.slices.push(slice);
+      }
+    }
+  }
+
+  public build(node: PeritextMlNode): ViewRange {
+    this.build0(node);
+    const view: ViewRange = [this.text, 0, this.slices];
+    return view;
+  }
+}
+
+export const toViewRange = (node: PeritextMlNode, registry: SliceRegistry = defaultRegistry): ViewRange =>
+  new ViewRangeBuilder(registry).build(node);
+
+export const fromJsonMl = (jsonml: JsonMlNode, registry: SliceRegistry = defaultRegistry): PeritextMlNode => {
+  if (typeof jsonml === 'string') return jsonml;
+  const tag = jsonml[0];
+  const length = jsonml.length;
+  const node: PeritextMlNode = [tag, null];
+  for (let i = 2; i < length; i++) node.push(fromJsonMl(jsonml[i] as JsonMlNode, registry));
+  const res = registry.fromHtml(jsonml);
+  if (res) {
+    node[0] = res[0];
+    node[1] = res[1];
+  } else {
+    node[0] = SliceTypeName[tag as any] ?? tag;
+    const attr = jsonml[1] || {};
+    let data = null;
+    if (attr['data-attr'] !== void 0) {
+      try {
+        data = JSON.parse(attr['data-attr']);
+      } catch {}
+    }
+    const inline = attr['data-inline'] === 'true';
+    if (data || inline) node[1] = {data, inline};
+  }
+  if (typeof node[0] === 'number' && node[0] < 0) {
+    const attr = node[1] || {};
+    attr.inline = true;
+    node[1] = attr;
+  }
+  return node;
+};
+
+export const fromHast = (hast: THtmlToken, registry?: SliceRegistry): PeritextMlNode => {
+  const jsonml = _fromHast(hast);
+  return fromJsonMl(jsonml, registry);
+};
+
+export const fromHtml = (html: string, registry?: SliceRegistry): PeritextMlNode => {
+  const hast = _html.parsef(html);
+  return fromHast(hast, registry);
+};

src/json-crdt-extensions/peritext/lazy/import-markdown.ts

@@ -0,0 +1,19 @@
+import {toHast} from 'very-small-parser/lib/markdown/block/toHast';
+import {block} from 'very-small-parser/lib/markdown/block';
+import {fromHast as _fromHast} from 'very-small-parser/lib/html/json-ml/fromHast';
+import {registry as defaultRegistry} from '../registry/registry';
+import {fromHast} from './import-html';
+import type {IRoot} from 'very-small-parser/lib/markdown/block/types';
+import type {PeritextMlNode} from '../block/types';
+import type {SliceRegistry} from '../registry/SliceRegistry';
+
+export const fromMdast = (mdast: IRoot, registry: SliceRegistry = defaultRegistry): PeritextMlNode => {
+  const hast = toHast(mdast);
+  const node = fromHast(hast, registry);
+  return node;
+};
+
+export const fromMarkdown = (markdown: string, registry?: SliceRegistry): PeritextMlNode => {
+  const mdast = block.parsef(markdown);
+  return fromMdast(mdast, registry);
+};

src/json-crdt-extensions/peritext/registry/SliceRegistry.ts

@@ -0,0 +1,61 @@
+import type {PeritextMlElement} from '../block/types';
+import type {NodeBuilder} from '../../../json-crdt-patch';
+import {SliceBehavior} from '../slice/constants';
+import type {JsonMlElement} from 'very-small-parser/lib/html/json-ml/types';
+import type {FromHtmlConverter, SliceTypeDefinition, ToHtmlConverter} from './types';
+
+export class SliceRegistry {
+  private map: Map<string | number, SliceTypeDefinition<any, any, any>> = new Map();
+  private toHtmlMap: Map<string | number, ToHtmlConverter<any>> = new Map();
+  private fromHtmlMap: Map<string, [def: SliceTypeDefinition<any, any, any>, converter: FromHtmlConverter][]> =
+    new Map();
+
+  public add<Type extends number | string, Schema extends NodeBuilder, Inline extends boolean = true>(
+    def: SliceTypeDefinition<Type, Schema, Inline>,
+  ): void {
+    const {type, toHtml, fromHtml} = def;
+    this.map.set(type, def);
+    if (toHtml) this.toHtmlMap.set(type, toHtml);
+    if (fromHtml) {
+      const fromHtmlMap = this.fromHtmlMap;
+      for (const htmlTag in fromHtml) {
+        const converter = fromHtml[htmlTag];
+        const converters = fromHtmlMap.get(htmlTag) ?? [];
+        converters.push([def, converter]);
+        fromHtmlMap.set(htmlTag, converters);
+      }
+    }
+  }
+
+  public def<Type extends number | string, Schema extends NodeBuilder, Inline extends boolean = true>(
+    type: Type,
+    schema: Schema,
+    behavior: SliceBehavior,
+    inline: boolean,
+    rest: Omit<SliceTypeDefinition<Type, Schema, Inline>, 'type' | 'schema' | 'behavior' | 'inline'> = {},
+  ): void {
+    this.add({type, schema, behavior, inline, ...rest});
+  }
+
+  public toHtml(el: PeritextMlElement): ReturnType<ToHtmlConverter<any>> | undefined {
+    const converter = this.toHtmlMap.get(el[0]);
+    return converter ? converter(el) : undefined;
+  }
+
+  public fromHtml(el: JsonMlElement): PeritextMlElement | undefined {
+    const tag = el[0] + '';
+    const converters = this.fromHtmlMap.get(tag);
+    if (converters) {
+      for (const [def, converter] of converters) {
+        const result = converter(el);
+        if (result) {
+          const attr = result[1] ?? (result[1] = {});
+          attr.inline = def.inline ?? def.type < 0;
+          attr.behavior = !attr.inline ? SliceBehavior.Marker : (def.behavior ?? SliceBehavior.Many);
+          return result;
+        }
+      }
+    }
+    return;
+  }
+}