Fix handling of spaces, tabs around line endings

Closes GH-41.

Author: wooorm

lib/handle/heading.js

@@ -39,6 +39,14 @@ export function heading(node, _, context) {
   const subexit = context.enter('phrasing')
   let value = containerPhrasing(node, context, {before: '# ', after: '\n'})
 
+  if (/^[\t ]/.test(value)) {
+    value =
+      '&#x' +
+      value.charCodeAt(0).toString(16).toUpperCase() +
+      ';' +
+      value.slice(1)
+  }
+
   value = value ? sequence + ' ' + value : sequence
 
   if (context.options.closeAtx) {

lib/unsafe.js

@@ -4,6 +4,8 @@
 
 /** @type {Array.<Unsafe>} */
 export const unsafe = [
+  {character: '\t', after: '[\\r\\n]', inConstruct: 'phrasing'},
+  {character: '\t', before: '[\\r\\n]', inConstruct: 'phrasing'},
   {
     character: '\t',
     inConstruct: ['codeFencedLangGraveAccent', 'codeFencedLangTilde']
@@ -30,6 +32,8 @@ export const unsafe = [
       'headingAtx'
     ]
   },
+  {character: ' ', after: '[\\r\\n]', inConstruct: 'phrasing'},
+  {character: ' ', before: '[\\r\\n]', inConstruct: 'phrasing'},
   {
     character: ' ',
     inConstruct: ['codeFencedLangGraveAccent', 'codeFencedLangTilde']
@@ -88,10 +92,7 @@ export const unsafe = [
   // Note: typical escapes are handled in `safe`!
   {character: '\\', after: '[\\r\\n]', inConstruct: 'phrasing'},
   // A right bracket can exit labels.
-  {
-    character: ']',
-    inConstruct: ['label', 'reference']
-  },
+  {character: ']', inConstruct: ['label', 'reference']},
   // Caret is not used in markdown for constructs.
   // An underscore can start emphasis, strong, or a thematic break.
   {atBreak: true, character: '_'},

lib/util/safe.js

@@ -71,11 +71,15 @@ export function safe(context, input, config) {
     // the next character, and the next character is definitly being escaped,
     // then skip this escape.
     if (
-      position + 1 < end &&
-      positions[index + 1] === position + 1 &&
-      infos[position].after &&
-      !infos[position + 1].before &&
-      !infos[position + 1].after
+      (position + 1 < end &&
+        positions[index + 1] === position + 1 &&
+        infos[position].after &&
+        !infos[position + 1].before &&
+        !infos[position + 1].after) ||
+      (positions[index - 1] === position - 1 &&
+        infos[position].before &&
+        !infos[position - 1].before &&
+        !infos[position - 1].after)
     ) {
       continue
     }

test/index.js

@@ -1210,6 +1210,50 @@ test('heading', (t) => {
     'should not escape a `#` in a heading (2)'
   )
 
+  t.equal(
+    to({type: 'heading', depth: 1, children: [{type: 'text', value: '  a'}]}),
+    '#   a\n',
+    'should encode a space at the start of an atx heading'
+  )
+
+  t.equal(
+    to({type: 'heading', depth: 1, children: [{type: 'text', value: '\t\ta'}]}),
+    '# 	\ta\n',
+    'should encode a tab at the start of an atx heading'
+  )
+
+  t.equal(
+    to({type: 'heading', depth: 1, children: [{type: 'text', value: 'a  '}]}),
+    '# a  \n',
+    'should encode a space at the end of an atx heading'
+  )
+
+  t.equal(
+    to({type: 'heading', depth: 1, children: [{type: 'text', value: 'a\t\t'}]}),
+    '# a\t	\n',
+    'should encode a tab at the end of an atx heading'
+  )
+
+  t.equal(
+    to({
+      type: 'heading',
+      depth: 1,
+      children: [{type: 'text', value: 'a \n b'}]
+    }),
+    'a \n b\n=======\n',
+    'should encode spaces around a line ending in a setext heading'
+  )
+
+  t.equal(
+    to({
+      type: 'heading',
+      depth: 3,
+      children: [{type: 'text', value: 'a \n b'}]
+    }),
+    '### a 
 b\n',
+    'should not need to encode spaces around a line ending in an atx heading (because the line ending is encoded)'
+  )
+
   t.end()
 })
 
@@ -1507,7 +1551,7 @@ test('imageReference', (t) => {
   t.end()
 })
 
-test('Code text', (t) => {
+test('code (text)', (t) => {
   // @ts-expect-error: `value` missing.
   t.equal(to({type: 'inlineCode'}), '``\n', 'should support an empty code text')
 
@@ -2733,6 +2777,42 @@ test('paragraph', (t) => {
     'should support a paragraph'
   )
 
+  t.equal(
+    to({type: 'paragraph', children: [{type: 'text', value: '  a'}]}),
+    '  a\n',
+    'should encode spaces at the start of paragraphs'
+  )
+
+  t.equal(
+    to({type: 'paragraph', children: [{type: 'text', value: 'a  '}]}),
+    'a  \n',
+    'should encode spaces at the end of paragraphs'
+  )
+
+  t.equal(
+    to({type: 'paragraph', children: [{type: 'text', value: '\t\ta'}]}),
+    '	\ta\n',
+    'should encode tabs at the start of paragraphs'
+  )
+
+  t.equal(
+    to({type: 'paragraph', children: [{type: 'text', value: 'a\t\t'}]}),
+    'a\t	\n',
+    'should encode tabs at the end of paragraphs'
+  )
+
+  t.equal(
+    to({type: 'paragraph', children: [{type: 'text', value: 'a  \n  b'}]}),
+    'a  \n  b\n',
+    'should encode spaces around line endings in paragraphs'
+  )
+
+  t.equal(
+    to({type: 'paragraph', children: [{type: 'text', value: 'a\t\t\n\t\tb'}]}),
+    'a\t	\n	\tb\n',
+    'should encode spaces around line endings in paragraphs'
+  )
+
   t.end()
 })
 
@@ -2769,6 +2849,7 @@ test('text', (t) => {
   t.equal(to({type: 'text'}), '', 'should support a void text')
   t.equal(to({type: 'text', value: ''}), '', 'should support an empty text')
   t.equal(to({type: 'text', value: 'a\nb'}), 'a\nb\n', 'should support text')
+
   t.end()
 })
 
@@ -3488,5 +3569,21 @@ test('roundtrip', (t) => {
     'should roundtrip different lists w/ `bulletOrderedOther` and lists that could turn into thematic breaks (6)'
   )
 
+  doc = ' \n'
+
+  t.equal(to(from(doc)), doc, 'should roundtrip a single encoded space')
+
+  doc = '	\n'
+
+  t.equal(to(from(doc)), doc, 'should roundtrip a single encoded tab')
+
+  doc = '  a  \n	\tb\t	\n'
+
+  t.equal(
+    to(from(doc)),
+    doc,
+    'should roundtrip encoded spaces and tabs where needed'
+  )
+
   t.end()
 })