Merge remote-tracking branch 'origin/master' into const

Author: tyru

.travis.yml

@@ -3,21 +3,25 @@ dist: xenial
 git:
   depth: 10
 
-matrix:
+jobs:
   include:
-    - env:
+    - name: Vim 7.4
+      env:
       - VIM_VERSION=v7.4
       - MAKE_TARGET=test
       - TEST_PROFILE=vim-profile-v7.4.txt
-    - env:
+    - name: Vim v8
+      env:
       - VIM_VERSION=v8.0.0000
       - MAKE_TARGET=test
       - TEST_PROFILE=vim-profile-v8.0.txt
-    - env:
+    - name: Vim master
+      env:
       - VIM_VERSION=master
       - MAKE_TARGET=test
       - TEST_PROFILE=vim-profile-master.txt
-    - env:
+    - name: Installed Vim with checks
+      env:
       - VIM_VERSION=installed
       - MAKE_TARGET="clean_compiled check js/test py/test test/node_position/test_position.out"
       - TEST_PROFILE=vim-profile-installed.txt

autoload/vimlparser.vim

@@ -3003,7 +3003,7 @@ function! s:ExprTokenizer.get_dict_literal_key()
   let r = self.reader
   let c = r.peek()
   if !s:isalnum(c) && c != '_' && c != '-'
-    throw s:Err(printf('unexpected token: %s', token.value), token.pos)
+    throw s:Err(printf('unexpected character: %s', c), self.reader.getpos())
   endif
   let s = c
   call self.reader.seek_cur(1)
@@ -5414,16 +5414,16 @@ function! s:RegexpParser.get_token_sq_c()
       return ['\n', 0]
     elseif c ==# 'r'
       call self.reader.seek_cur(1)
-      return ['\r', char2nr("\r")]
+      return ['\r', 13]
     elseif c ==# 't'
       call self.reader.seek_cur(1)
-      return ['\t', char2nr("\t")]
+      return ['\t', 9]
     elseif c ==# 'e'
       call self.reader.seek_cur(1)
-      return ['\e', char2nr("\e")]
+      return ['\e', 27]
     elseif c ==# 'b'
       call self.reader.seek_cur(1)
-      return ['\b', char2nr("\b")]
+      return ['\b', 8]
     elseif stridx(']^-\', c) != -1
       call self.reader.seek_cur(1)
       return ['\' . c, char2nr(c)]

js/vimlfunc.js

@@ -155,7 +155,12 @@ function viml_keys(obj) {
 
 function viml_len(obj) {
     if (typeof obj === 'string') {
-      return encodeURIComponent(obj).replace(/%../g, ' ').length;
+      var len = 0;
+      for (var i = 0; i < obj.length; i++) {
+          var c = obj.charCodeAt(i);
+          len += c < 128 ? 1 : ((c > 127) && (c < 2048)) ? 2 : 3;
+      }
+      return len;
     }
     return obj.length;
 }

js/vimlparser.js

@@ -155,7 +155,12 @@ function viml_keys(obj) {
 
 function viml_len(obj) {
     if (typeof obj === 'string') {
-      return encodeURIComponent(obj).replace(/%../g, ' ').length;
+      var len = 0;
+      for (var i = 0; i < obj.length; i++) {
+          var c = obj.charCodeAt(i);
+          len += c < 128 ? 1 : ((c > 127) && (c < 2048)) ? 2 : 3;
+      }
+      return len;
     }
     return obj.length;
 }
@@ -2794,7 +2799,7 @@ ExprTokenizer.prototype.get_dict_literal_key = function() {
     var r = this.reader;
     var c = r.peek();
     if (!isalnum(c) && c != "_" && c != "-") {
-        throw Err(viml_printf("unexpected token: %s", token.value), token.pos);
+        throw Err(viml_printf("unexpected character: %s", c), this.reader.getpos());
     }
     var s = c;
     this.reader.seek_cur(1);
@@ -5503,19 +5508,19 @@ RegexpParser.prototype.get_token_sq_c = function() {
         }
         else if (c == "r") {
             this.reader.seek_cur(1);
-            return ["\\r", viml_char2nr("\r")];
+            return ["\\r", 13];
         }
         else if (c == "t") {
             this.reader.seek_cur(1);
-            return ["\\t", viml_char2nr("\t")];
+            return ["\\t", 9];
         }
         else if (c == "e") {
             this.reader.seek_cur(1);
-            return ["\\e", viml_char2nr("\e")];
+            return ["\\e", 27];
         }
         else if (c == "b") {
             this.reader.seek_cur(1);
-            return ["\\b", viml_char2nr("\b")];
+            return ["\\b", 8];
         }
         else if (viml_stridx("]^-\\", c) != -1) {
             this.reader.seek_cur(1);

py/vimlfunc.py

@@ -132,7 +132,11 @@ def viml_keys(obj):
 
 def viml_len(obj):
     if type(obj) is str:
-        return len(obj.encode('utf-8'))
+        if sys.version_info < (3, 0):
+            b = bytes(obj)
+        else:
+            b = bytes(obj, 'utf8')
+        return len(b)
     return len(obj)
 
 def viml_printf(*args):

py/vimlparser.py

@@ -132,7 +132,11 @@ def viml_keys(obj):
 
 def viml_len(obj):
     if type(obj) is str:
-        return len(obj.encode('utf-8'))
+        if sys.version_info < (3, 0):
+            b = bytes(obj)
+        else:
+            b = bytes(obj, 'utf8')
+        return len(b)
     return len(obj)
 
 def viml_printf(*args):
@@ -2208,7 +2212,7 @@ def get_dict_literal_key(self):
         r = self.reader
         c = r.peek()
         if not isalnum(c) and c != "_" and c != "-":
-            raise VimLParserException(Err(viml_printf("unexpected token: %s", token.value), token.pos))
+            raise VimLParserException(Err(viml_printf("unexpected character: %s", c), self.reader.getpos()))
         s = c
         self.reader.seek_cur(1)
         while TRUE:
@@ -4265,16 +4269,16 @@ def get_token_sq_c(self):
                 return ["\\n", 0]
             elif c == "r":
                 self.reader.seek_cur(1)
-                return ["\\r", viml_char2nr("\r")]
+                return ["\\r", 13]
             elif c == "t":
                 self.reader.seek_cur(1)
-                return ["\\t", viml_char2nr("\t")]
+                return ["\\t", 9]
             elif c == "e":
                 self.reader.seek_cur(1)
-                return ["\\e", viml_char2nr("\e")]
+                return ["\\e", 27]
             elif c == "b":
                 self.reader.seek_cur(1)
-                return ["\\b", viml_char2nr("\b")]
+                return ["\\b", 8]
             elif viml_stridx("]^-\\", c) != -1:
                 self.reader.seek_cur(1)
                 return ["\\" + c, viml_char2nr(c)]

test/test1.ok

@@ -53,6 +53,7 @@
 (let %= a 5)
 (let ..= a 'foo')
 (echo (concat (concat 'foo' 'bar') 'baz'))
+(let = a '🐥')
 (const = a 1)
 (const = (a b) (list 1 2))
 (const = (a b . c) (list 1 2 3))

test/test1.vim

@@ -58,6 +58,7 @@ let a /= 4
 let a %= 5
 let a ..= 'foo'
 echo ('foo' .. 'bar')..'baz'
+let a = '🐥'
 const a = 1
 const [a, b] = [1, 2]
 const [a, b; c] = [1, 2, 3]