Merge pull request #835 from streamich/normalize-html-import

streamich · web-flow · commit a4ee0c3f1935 · 2025-03-20T15:19:54.000+01:00
Normalize HTML import
diff --git a/src/json-crdt-extensions/peritext/registry/SliceRegistry.ts b/src/json-crdt-extensions/peritext/registry/SliceRegistry.ts
@@ -1,9 +1,13 @@
+import {SliceBehavior} from '../slice/constants';
+import {CommonSliceType} from '../slice';
 import type {PeritextMlElement} from '../block/types';
 import type {NodeBuilder} from '../../../json-crdt-patch';
-import {SliceBehavior} from '../slice/constants';
 import type {JsonMlElement} from 'very-small-parser/lib/html/json-ml/types';
 import type {FromHtmlConverter, SliceTypeDefinition, ToHtmlConverter} from './types';
 
+/**
+ * @todo Consider moving the registry under the `/transfer` directory.
+ */
 export class SliceRegistry {
   private map: Map<string | number, SliceTypeDefinition<any, any, any>> = new Map();
   private toHtmlMap: Map<string | number, ToHtmlConverter<any>> = new Map();
@@ -14,17 +18,20 @@ export class SliceRegistry {
     def: SliceTypeDefinition<Type, Schema, Inline>,
   ): void {
     const {type, toHtml, fromHtml} = def;
-    this.map.set(type, def);
+    const fromHtmlMap = this.fromHtmlMap;
     if (toHtml) this.toHtmlMap.set(type, toHtml);
     if (fromHtml) {
-      const fromHtmlMap = this.fromHtmlMap;
       for (const htmlTag in fromHtml) {
         const converter = fromHtml[htmlTag];
         const converters = fromHtmlMap.get(htmlTag) ?? [];
         converters.push([def, converter]);
         fromHtmlMap.set(htmlTag, converters);
       }
     }
+    const tag = CommonSliceType[type as any];
+    if (tag && typeof tag === 'string') {
+      fromHtmlMap.set(tag, [[def, () => [type, null]]]);
+    }
   }
 
   public def<Type extends number | string, Schema extends NodeBuilder, Inline extends boolean = true>(
diff --git a/src/json-crdt-extensions/peritext/transfer/__tests__/PeritextDataTransfer.spec.ts b/src/json-crdt-extensions/peritext/transfer/__tests__/PeritextDataTransfer.spec.ts
@@ -195,7 +195,7 @@ describe('Markdown', () => {
     expect(md2).toBe('ab\n\nc__123__\n\n++456++d');
   });
 
-  test.skip('can insert Markdown with inline line breaks', () => {
+  test('can insert Markdown with inline line breaks', () => {
     const {peritext, transfer} = setup();
     const md = '1\n2\n3';
     peritext.strApi().ins(0, 'ab');
@@ -204,9 +204,72 @@ describe('Markdown', () => {
     peritext.refresh();
     const all = peritext.rangeAll()!;
     const html = transfer.toHtml(all);
-    console.log(html);
-    expect(html).toBe('<p>a123b</p>');
+    expect(html).toBe('<p>a1 2 3b</p>');
     const md2 = transfer.toMarkdown(all);
-    expect(md2).toBe('a123b');
+    expect(md2).toBe('a1 2 3b');
+  });
+
+  test('can insert a blockquote and a paragraph into empty string', () => {
+    const {peritext, transfer} = setup();
+    const md = '> blockquote';
+    transfer.fromMarkdown(0, md);
+    peritext.refresh();
+    const all = peritext.rangeAll()!;
+    const html = transfer.toHtml(all);
+    expect(html).toBe('<blockquote><p>blockquote</p></blockquote>');
+    const md2 = transfer.toMarkdown(all);
+    console.log(md2);
+    expect(md2).toBe('> blockquote');
+  });
+
+  test('can insert a blockquote', () => {
+    const {peritext, transfer} = setup();
+    const md = '> blockquote';
+    peritext.strApi().ins(0, 'ab');
+    peritext.refresh();
+    transfer.fromMarkdown(1, md);
+    peritext.refresh();
+    const all = peritext.rangeAll()!;
+    const html = transfer.toHtml(all);
+    expect(html).toBe('<p>a</p><blockquote><p>blockquoteb</p></blockquote>');
+    const md2 = transfer.toMarkdown(all);
+    expect(md2).toBe('a\n\n> blockquoteb');
+  });
+
+  test('can insert a blockquote and a paragraph', () => {
+    const {peritext, transfer} = setup();
+    const md = '> blockquote\n\nparagraph';
+    peritext.strApi().ins(0, 'ab');
+    peritext.refresh();
+    transfer.fromMarkdown(1, md);
+    peritext.refresh();
+    const all = peritext.rangeAll()!;
+    const html = transfer.toHtml(all);
+    expect(html).toBe('<p>a</p><blockquote><p>blockquote</p></blockquote><p>paragraphb</p>');
+    const md2 = transfer.toMarkdown(all);
+    expect(md2).toBe('a\n\n> blockquote\n\nparagraphb');
+  });
+
+  test('can insert realistic 3 paragraphs of Markdown', () => {
+    const {peritext, transfer} = setup();
+    const md =
+      'The German __automotive sector__ is in the process of _cutting ' +
+      'thousands of jobs_ as it grapples with a global shift toward electric vehicles ' +
+      '— a transformation Musk himself has been at the forefront of.' +
+      '\n\n' +
+      '> To be or not to be, that is the question.' +
+      '\n\n' +
+      'A `ClipboardEvent` is dispatched for copy, cut, and paste events, and it contains ' +
+      'a `clipboardData` property of type `DataTransfer`. The `DataTransfer` object ' +
+      'is used by the Clipboard Events API to hold multiple representations of data.';
+    peritext.strApi().ins(0, 'ab');
+    peritext.refresh();
+    transfer.fromMarkdown(1, md);
+    peritext.refresh();
+    // console.log(peritext.blocks + '');
+    const all = peritext.rangeAll()!;
+    const md2 = transfer.toMarkdown(all);
+    // console.log(md2);
+    expect(md2).toBe('a' + md + 'b');
   });
 });
diff --git a/src/json-crdt-extensions/peritext/transfer/__tests__/import-html.spec.ts b/src/json-crdt-extensions/peritext/transfer/__tests__/import-html.spec.ts
@@ -33,6 +33,57 @@ describe('.fromHtml()', () => {
       ],
     ]);
   });
+
+  test('can import a single <blockquote> block', () => {
+    const html = '<blockquote>2b||!2b</blockquote>';
+    const peritextMl = fromHtml(html);
+    expect(peritextMl).toEqual(['', null, [CommonSliceType.blockquote, null, '2b||!2b']]);
+  });
+
+  test('can import a single <blockquote> block with nested single <p>', () => {
+    const html = '<blockquote><p>2b||!2b</p></blockquote>';
+    const peritextMl = fromHtml(html);
+    expect(peritextMl).toEqual(['', null, [CommonSliceType.blockquote, null, [CommonSliceType.p, null, '2b||!2b']]]);
+  });
+
+  test('can import a single <blockquote> block after a <p> block', () => {
+    const html = '<p>123</p><blockquote>2b||!2b</blockquote>';
+    const peritextMl = fromHtml(html);
+    expect(peritextMl).toEqual([
+      '',
+      null,
+      [CommonSliceType.p, null, '123'],
+      [CommonSliceType.blockquote, null, '2b||!2b'],
+    ]);
+  });
+
+  test('can import a single <blockquote> block with nested single <p>, after a <p> block', () => {
+    const html = '<p>123</p><blockquote><p>2b||!2b</p></blockquote>';
+    const peritextMl = fromHtml(html);
+    expect(peritextMl).toEqual([
+      '',
+      null,
+      [CommonSliceType.p, null, '123'],
+      [CommonSliceType.blockquote, null, [CommonSliceType.p, null, '2b||!2b']],
+    ]);
+  });
+
+  test('can import a single <blockquote> block with nested single <p>, after a <p> block with inline formatting', () => {
+    const html = '<p><b>1</b><code>2</code>3</p><blockquote><p>2b||!2b</p></blockquote>';
+    const peritextMl = fromHtml(html);
+    expect(peritextMl).toEqual([
+      '',
+      null,
+      [
+        CommonSliceType.p,
+        null,
+        [CommonSliceType.b, {behavior: SliceBehavior.One, inline: true}, '1'],
+        [CommonSliceType.code, {behavior: SliceBehavior.One, inline: true}, '2'],
+        '3',
+      ],
+      [CommonSliceType.blockquote, null, [CommonSliceType.p, null, '2b||!2b']],
+    ]);
+  });
 });
 
 describe('.toViewRange()', () => {
diff --git a/src/json-crdt-extensions/peritext/transfer/__tests__/import-markdown.spec.ts b/src/json-crdt-extensions/peritext/transfer/__tests__/import-markdown.spec.ts
@@ -0,0 +1,52 @@
+import {CommonSliceType} from '../../slice';
+import {SliceBehavior} from '../../slice/constants';
+import {fromMarkdown} from '../import-markdown';
+
+describe('fromMarkdown()', () => {
+  test('a single paragraph', () => {
+    const text = 'Hello world';
+    const peritextMl = fromMarkdown(text);
+    expect(peritextMl).toEqual(['', null, [CommonSliceType.p, null, 'Hello world']]);
+  });
+
+  test('multi-block realistic example', () => {
+    const text =
+      'The German __automotive sector__ is in the process of *cutting \n' +
+      'thousands of jobs* as it grapples with a global shift toward electric vehicles \n' +
+      '— a transformation Musk himself has been at the forefront of.\n' +
+      '\n' +
+      '> To be or not to be, that is the question.\n' +
+      '\n' +
+      'A `ClipboardEvent` is dispatched for copy, cut, and paste events, and it contains \n' +
+      'a `clipboardData` property of type `DataTransfer`. The `DataTransfer` object \n' +
+      'is used by the Clipboard Events API to hold multiple representations of data.\n';
+    const peritextMl = fromMarkdown(text);
+    expect(peritextMl).toEqual([
+      '',
+      null,
+      [
+        CommonSliceType.p,
+        null,
+        'The German ',
+        [CommonSliceType.b, {behavior: SliceBehavior.One, inline: true}, 'automotive sector'],
+        ' is in the process of ',
+        [CommonSliceType.i, {behavior: SliceBehavior.One, inline: true}, 'cutting thousands of jobs'],
+        ' as it grapples with a global shift toward electric vehicles — a transformation Musk himself has been at the forefront of.',
+      ],
+      [CommonSliceType.blockquote, null, [CommonSliceType.p, null, 'To be or not to be, that is the question.']],
+      [
+        CommonSliceType.p,
+        null,
+        'A ',
+        [CommonSliceType.code, {behavior: SliceBehavior.One, inline: true}, 'ClipboardEvent'],
+        ' is dispatched for copy, cut, and paste events, and it contains a ',
+        [CommonSliceType.code, {behavior: SliceBehavior.One, inline: true}, 'clipboardData'],
+        ' property of type ',
+        [CommonSliceType.code, {behavior: SliceBehavior.One, inline: true}, 'DataTransfer'],
+        '. The ',
+        [CommonSliceType.code, {behavior: SliceBehavior.One, inline: true}, 'DataTransfer'],
+        ' object is used by the Clipboard Events API to hold multiple representations of data.',
+      ],
+    ]);
+  });
+});
diff --git a/src/json-crdt-extensions/peritext/transfer/import-html.ts b/src/json-crdt-extensions/peritext/transfer/import-html.ts
@@ -14,6 +14,15 @@ import type {SliceRegistry} from '../registry/SliceRegistry';
 import type {ViewStyle, ViewRange, ViewSlice} from '../editor/types';
 import type {ClipboardData} from './export-html';
 
+/**
+ * @todo Implement HTML normalization function, ensure that:
+ *
+ * - <blockquote> and <p> nodes are treated correctly, especially when sole node
+ *   is nested.
+ * - list nodes are treated correctly.
+ * - <svg> nodes are converted to Base64 and inlined as data URL images.
+ */
+
 /**
  * Flattens a {@link PeritextMlNode} tree structure into a {@link ViewRange}
  * flat string with annotation ranges.
@@ -22,31 +31,33 @@ class ViewRangeBuilder {
   private text = '';
   private slices: ViewSlice[] = [];
 
-  private build0(node: PeritextMlNode, depth = 0): void {
-    const skipWhitespace = depth < 2;
+  private build0(node: PeritextMlNode, path: (string | number)[]): boolean {
+    const skipWhitespace = path.length < 2;
     if (typeof node === 'string') {
-      if (skipWhitespace && !node.trim()) return;
+      if (skipWhitespace && !node.trim()) return false;
       this.text += node;
-      return;
+      return false;
     }
     const [type, attr] = node;
     const start = this.text.length;
     const length = node.length;
     const inline = !!attr?.inline;
-    const hasType = !!type || type === 0;
-    if (hasType && !inline) {
+    const hasType = type === 0 || !!type;
+    const firstChild = node[2] as PeritextMlNode;
+    const isFirstChildInline = firstChild && (typeof firstChild === 'string' || firstChild[1]?.inline);
+    if (hasType && !inline && isFirstChildInline) {
       this.text += '\n';
       const header =
         (SliceBehavior.Marker << SliceHeaderShift.Behavior) +
         (Anchor.Before << SliceHeaderShift.X1Anchor) +
         (Anchor.Before << SliceHeaderShift.X2Anchor);
-      const slice: ViewSlice = [header, start, start, type];
+      const slice: ViewSlice = [header, start, start, path.length ? [...path, type] : type];
       const data = attr?.data;
       if (data) slice.push(data);
       this.slices.push(slice);
     }
-    for (let i = 2; i < length; i++) this.build0(node[i] as PeritextMlNode, depth + 1);
-    if (hasType) {
+    for (let i = 2; i < length; i++) this.build0(node[i] as PeritextMlNode, type === '' ? path : [...path, type]);
+    if (hasType && inline) {
       let end: number = 0,
         header: number = 0;
       if (inline) {
@@ -62,10 +73,11 @@ class ViewRangeBuilder {
         this.slices.push(slice);
       }
     }
+    return false;
   }
 
   public build(node: PeritextMlNode): ViewRange {
-    this.build0(node);
+    this.build0(node, []);
     const view: ViewRange = [this.text, 0, this.slices];
     return view;
   }
@@ -115,7 +127,7 @@ export const fromJsonMl = (jsonml: JsonMlNode, registry: SliceRegistry = default
     node[0] = res[0];
     node[1] = res[1];
   } else {
-    node[0] = SliceTypeName[tag as any] ?? tag;
+    if (typeof tag === 'string') node[0] = SliceTypeName[tag as any] ?? tag;
     const attr = jsonml[1] || {};
     let data = null;
     if (attr['data-attr'] !== void 0) {
@@ -126,15 +138,8 @@ export const fromJsonMl = (jsonml: JsonMlNode, registry: SliceRegistry = default
     const inline = inlineHtmlTag || attr['data-inline'] === 'true';
     if (data || inline) node[1] = {data, inline};
   }
-  if (typeof node[0] === 'number' && node[0] < 0) {
-    const attr = node[1] || {};
-    attr.inline = true;
-    node[1] = attr;
-  }
-  if (node.length < 3) {
-    const attr = node[1] || {};
-    if (attr.inline) return '';
-  }
+  if (typeof node[0] === 'number' && node[0] < 0) (node[1] ||= {}).inline = true;
+  if (node.length < 3 && (node[1] || {}).inline) return '';
   return node;
 };
 
