@@ -26,15 +26,15 @@ class Lexer(object):
2626 }
2727
2828 def tokenize (self , expression ):
29- self ._init_expr (expression )
29+ self ._initialize_for_expression (expression )
3030 while self ._current is not None :
3131 if self ._current in self .SIMPLE_TOKENS :
3232 yield {'type' : self .SIMPLE_TOKENS [self ._current ],
3333 'value' : self ._current ,
34- 'start' : self ._pos , 'end' : self ._pos + 1 }
34+ 'start' : self ._position , 'end' : self ._position + 1 }
3535 self ._next ()
3636 elif self ._current in self .START_IDENTIFIER :
37- start = self ._pos
37+ start = self ._position
3838 buff = self ._current
3939 while self ._next () in self .VALID_IDENTIFIER :
4040 buff += self ._current
@@ -43,7 +43,7 @@ def tokenize(self, expression):
4343 elif self ._current in self .WHITESPACE :
4444 self ._next ()
4545 elif self ._current == '[' :
46- start = self ._pos
46+ start = self ._position
4747 next_char = self ._next ()
4848 if next_char == ']' :
4949 self ._next ()
@@ -63,7 +63,7 @@ def tokenize(self, expression):
6363 elif self ._current == '`' :
6464 yield self ._consume_literal ()
6565 elif self ._current in self .START_NUMBER :
66- start = self ._pos
66+ start = self ._position
6767 buff = self ._current
6868 while self ._next () in self .VALID_NUMBER :
6969 buff += self ._current
@@ -80,31 +80,33 @@ def tokenize(self, expression):
8080 elif self ._current == '=' :
8181 yield self ._match_or_else ('=' , 'eq' , 'unknown' )
8282 else :
83- raise LexerError (lexer_position = self ._pos ,
83+ raise LexerError (lexer_position = self ._position ,
8484 lexer_value = self ._current ,
8585 message = "Unknown token %s" % self ._current )
8686 yield {'type' : 'eof' , 'value' : '' ,
87- 'start' : self ._len , 'end' : self ._len }
87+ 'start' : self ._length , 'end' : self ._length }
8888
89- def _init_expr (self , expression ):
89+ def _initialize_for_expression (self , expression ):
9090 if not expression :
9191 raise EmptyExpressionError ()
92- self ._pos = 0
92+ self ._position = 0
9393 self ._expression = expression
9494 self ._chars = list (self ._expression )
95- self ._current = self ._chars [self ._pos ]
96- self ._len = len (self ._expression )
95+ self ._current = self ._chars [self ._position ]
96+ self ._length = len (self ._expression )
9797
9898 def _next (self ):
99- if self ._pos == self ._len - 1 :
99+ if self ._position == self ._length - 1 :
100100 self ._current = None
101101 else :
102- self ._pos += 1
103- self ._current = self ._chars [self ._pos ]
102+ self ._position += 1
103+ self ._current = self ._chars [self ._position ]
104104 return self ._current
105105
106- def _in_delimiter (self , delimiter ):
107- start = self ._pos
106+ def _consume_until (self , delimiter ):
107+ # Consume until the delimiter is reached,
108+ # allowing for the delimiter to be escaped with "\".
109+ start = self ._position
108110 buff = ''
109111 self ._next ()
110112 while self ._current != delimiter :
@@ -122,8 +124,8 @@ def _in_delimiter(self, delimiter):
122124 return buff
123125
124126 def _consume_literal (self ):
125- start = self ._pos
126- lexeme = self ._in_delimiter ('`' )
127+ start = self ._position
128+ lexeme = self ._consume_until ('`' )
127129 lexeme = lexeme .replace ('\\ `' , '`' )
128130 try :
129131 # Assume it is valid JSON and attempt to parse.
@@ -139,15 +141,15 @@ def _consume_literal(self):
139141 raise LexerError (lexer_position = start ,
140142 lexer_value = self ._expression ,
141143 message = "Bad token %s" % lexeme )
142- token_len = self ._pos - start
144+ token_len = self ._position - start
143145 return {'type' : 'literal' , 'value' : parsed_json ,
144146 'start' : start , 'end' : token_len }
145147
146148 def _consume_quoted_identifier (self ):
147- start = self ._pos
148- lexeme = '"' + self ._in_delimiter ('"' ) + '"'
149+ start = self ._position
150+ lexeme = '"' + self ._consume_until ('"' ) + '"'
149151 try :
150- token_len = self ._pos - start
152+ token_len = self ._position - start
151153 return {'type' : 'quoted_identifier' , 'value' : loads (lexeme ),
152154 'start' : start , 'end' : token_len }
153155 except ValueError as e :
@@ -157,14 +159,14 @@ def _consume_quoted_identifier(self):
157159 message = error_message )
158160
159161 def _consume_raw_string_literal (self ):
160- start = self ._pos
161- lexeme = self ._in_delimiter ("'" )
162- token_len = self ._pos - start
162+ start = self ._position
163+ lexeme = self ._consume_until ("'" )
164+ token_len = self ._position - start
163165 return {'type' : 'literal' , 'value' : lexeme ,
164166 'start' : start , 'end' : token_len }
165167
166168 def _match_or_else (self , expected , match_type , else_type ):
167- start = self ._pos
169+ start = self ._position
168170 current = self ._current
169171 next_char = self ._next ()
170172 if next_char == expected :
0 commit comments