1+ import json
2+ from ..compat import unichr
13from .error import LanguageError
24
35__all__ = ['Token' , 'Lexer' , 'TokenKind' ,
@@ -101,7 +103,7 @@ def get_token_kind_desc(kind):
101103def char_code_at (s , pos ):
102104 if 0 <= pos < len (s ):
103105 return ord (s [pos ])
104- return None
106+ return 0
105107
106108
107109PUNCT_CODE_TO_KIND = {
@@ -153,7 +155,7 @@ def read_token(source, from_position):
153155
154156 raise LanguageError (
155157 source , position ,
156- u'Unexpected character "{}" ' .format (body [position ]))
158+ u'Unexpected character {} ' .format (json . dumps ( body [position ]) ))
157159
158160
159161def position_after_whitespace (body , start_position ):
@@ -176,7 +178,7 @@ def position_after_whitespace(body, start_position):
176178 position += 1
177179 while position < body_length :
178180 code = char_code_at (body , position )
179- if code is None or code in (10 , 13 , 0x2028 , 0x2029 ):
181+ if not code or code in (10 , 13 , 0x2028 , 0x2029 ):
180182 break
181183 position += 1
182184 else :
@@ -273,7 +275,7 @@ def read_string(source, start):
273275
274276 while position < len (body ):
275277 code = char_code_at (body , position )
276- if code is None or code in (34 , 10 , 13 , 0x2028 , 0x2029 ):
278+ if not code or code in (34 , 10 , 13 , 0x2028 , 0x2029 ):
277279 break
278280 position += 1
279281 if code == 92 : # \
@@ -349,7 +351,7 @@ def read_name(source, position):
349351 code = None
350352 while end != body_length :
351353 code = char_code_at (body , end )
352- if code is None or not (
354+ if not code or not (
353355 code == 95 or # _
354356 48 <= code <= 57 or # 0-9
355357 65 <= code <= 90 or # A-Z
0 commit comments