Merge pull request #743 from Milly/Web.JSON-fix-char-encodes

Web.JSON: fix char encodes

Author: ujihisa

autoload/vital/__vital__/Web/JSON.vim

@@ -1,6 +1,43 @@
 let s:save_cpo = &cpo
 set cpo&vim
 
+let s:control_chars = {
+    \   '\': '\\',
+    \   '"': '\"',
+    \   "\x01": '\u0001',
+    \   "\x02": '\u0002',
+    \   "\x03": '\u0003',
+    \   "\x04": '\u0004',
+    \   "\x05": '\u0005',
+    \   "\x06": '\u0006',
+    \   "\x07": '\u0007',
+    \   "\x08": '\b',
+    \   "\x09": '\t',
+    \   "\x0a": '\n',
+    \   "\x0b": '\u000b',
+    \   "\x0c": '\f',
+    \   "\x0d": '\r',
+    \   "\x0e": '\u000e',
+    \   "\x0f": '\u000f',
+    \   "\x10": '\u0010',
+    \   "\x11": '\u0011',
+    \   "\x12": '\u0012',
+    \   "\x13": '\u0013',
+    \   "\x14": '\u0014',
+    \   "\x15": '\u0015',
+    \   "\x16": '\u0016',
+    \   "\x17": '\u0017',
+    \   "\x18": '\u0018',
+    \   "\x19": '\u0019',
+    \   "\x1a": '\u001a',
+    \   "\x1b": '\u001b',
+    \   "\x1c": '\u001c',
+    \   "\x1d": '\u001d',
+    \   "\x1e": '\u001e',
+    \   "\x1f": '\u001f',
+    \ }
+lockvar s:control_chars
+
 function! s:_true() abort
   return 1
 endfunction
@@ -26,7 +63,6 @@ function! s:_resolve(val, prefix) abort
   return a:val
 endfunction
 
-
 function! s:_vital_created(module) abort
   " define constant variables
   if !exists('s:const')
@@ -59,6 +95,10 @@ function! s:decode(json, ...) abort
   let json = join(split(json, "\n"), '')
   let json = substitute(json, '\\u34;', '\\"', 'g')
   let json = substitute(json, '\\u\(\x\x\x\x\)', '\=s:string.nr2enc_char("0x".submatch(1))', 'g')
+  " convert surrogate pair
+  let json = substitute(json, '\([\uD800-\uDBFF]\)\([\uDC00-\uDFFF]\)',
+        \ '\=nr2char(0x10000+and(0x7ff,char2nr(submatch(1)))*0x400+and(0x3ff,char2nr(submatch(2))))',
+        \ 'g')
   if settings.use_token
     let prefix = '__Web.JSON__'
     while stridx(json, prefix) != -1
@@ -83,11 +123,9 @@ function! s:encode(val, ...) abort
   if type(a:val) == 0
     return a:val
   elseif type(a:val) == 1
-    let json = '"' . escape(a:val, '\"') . '"'
-    let json = substitute(json, "\r", '\\r', 'g')
-    let json = substitute(json, "\n", '\\n', 'g')
-    let json = substitute(json, "\t", '\\t', 'g')
-    return iconv(json, &encoding, 'utf-8')
+    let s = substitute(a:val, '[\x01-\x1f\\"]', '\=s:control_chars[submatch(0)]', 'g')
+    let s = iconv(s, &encoding, 'utf-8')
+    return '"' . s . '"'
   elseif type(a:val) == 2
     if s:const.true == a:val
       return 'true'

test/Web/JSON.vimspec

@@ -29,7 +29,17 @@ Describe Web.JSON
             \ JSON.decode('"He said \"I''m a vimmer\""'),
             \ 'He said "I''m a vimmer"'
             \)
-      " there should be iconv tests as well
+      " control chars
+      Assert Equals(JSON.decode('"\u0001\u0002\u0003\u0004\u0005\u0006\u0007"'), "\x01\x02\x03\x04\x05\x06\x07")
+      Assert Equals(JSON.decode('"\b\t\n\u000b\f\r\u000e\u000f"'), "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f")
+      Assert Equals(JSON.decode('"\u0010\u0011\u0012\u0013\u0014\u0015\u0016\u0017"'), "\x10\x11\x12\x13\x14\x15\x16\x17")
+      Assert Equals(JSON.decode('"\u0018\u0019\u001a\u001b\u001c\u001d\u001e\u001f"'), "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f")
+      " multibyte
+      Assert Equals(JSON.decode('"s¢cĴgё"'), "s¢cĴgё")
+      " UTF-16 surrogate pair
+      Assert Equals(JSON.decode('"\ud83c\udf63"'), "\xf0\x9f\x8d\xa3")
+      " unpaired UTF-16 surrogate
+      Assert Equals(JSON.decode('"\ud83c\u00a0"'), "\ud83c\u00a0")
     End
 
     It decodes lists
@@ -91,7 +101,15 @@ Describe Web.JSON
             \ JSON.encode('He said "I''m a vimmer"'),
             \ '"He said \"I''m a vimmer\""'
             \)
-      " there should be iconv tests as well
+      " control chars
+      Assert Equals(JSON.encode("\x01\x02\x03\x04\x05\x06\x07"), '"\u0001\u0002\u0003\u0004\u0005\u0006\u0007"')
+      Assert Equals(JSON.encode("\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"), '"\b\t\n\u000b\f\r\u000e\u000f"')
+      Assert Equals(JSON.encode("\x10\x11\x12\x13\x14\x15\x16\x17"), '"\u0010\u0011\u0012\u0013\u0014\u0015\u0016\u0017"')
+      Assert Equals(JSON.encode("\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"), '"\u0018\u0019\u001a\u001b\u001c\u001d\u001e\u001f"')
+      " multibyte
+      Assert Equals(JSON.encode("s¢cĴgё"), '"s¢cĴgё"')
+      " UTF-16 surrogate pair
+      Assert Equals(JSON.encode("\xf0\x9f\x8d\xa3"), "\"\xf0\x9f\x8d\xa3\"")
     End
 
     It encodes lists