Merge branch 'more-testing' into develop

* more-testing:
  Add additional hypothesis tests for functions
  Assert more properties on parsed AST result
  Fix lexer error values

Author: jamesls

extra/test_hypothesis.py

@@ -4,6 +4,7 @@
 # via env var for longer runs in travis.
 import os
 import sys
+import numbers
 
 from nose.plugins.skip import SkipTest
 from hypothesis import given, settings, assume, HealthCheck
@@ -12,19 +13,22 @@
 from jmespath import lexer
 from jmespath import parser
 from jmespath import exceptions
+from jmespath.functions import Functions
 
 
 if sys.version_info[:2] == (2, 6):
     raise RuntimeError("Hypothesis tests are not supported on python2.6. "
                        "Use python2.7, or python3.3 and greater.")
 
 
+JSON_NUMBERS = (st.integers() | st.floats(allow_nan=False,
+                                          allow_infinity=False))
+
 RANDOM_JSON = st.recursive(
-    st.floats() | st.booleans() | st.text() | st.none(),
+    JSON_NUMBERS | st.booleans() | st.text() | st.none(),
     lambda children: st.lists(children) | st.dictionaries(st.text(), children)
 )
 
-
 MAX_EXAMPLES = int(os.environ.get('JP_MAX_EXAMPLES', 1000))
 BASE_SETTINGS = {
     'max_examples': MAX_EXAMPLES,
@@ -40,11 +44,31 @@
 def test_lexer_api(expr):
     try:
         tokens = list(lexer.Lexer().tokenize(expr))
-    except exceptions.JMESPathError as e:
+    except exceptions.EmptyExpressionError:
+        return
+    except exceptions.LexerError as e:
+        assert e.lex_position >= 0, e.lex_position
+        assert e.lex_position < len(expr), e.lex_position
+        if expr:
+            assert expr[e.lex_position] == e.token_value[0], (
+                "Lex position does not match first token char.\n"
+                "Expression: %s\n%s != %s" % (expr, expr[e.lex_position],
+                                              e.token_value[0])
+            )
         return
     except Exception as e:
         raise AssertionError("Non JMESPathError raised: %s" % e)
     assert isinstance(tokens, list)
+    # Token starting positions must be unique, can't have two
+    # tokens with the same start position.
+    start_locations = [t['start'] for t in tokens]
+    assert len(set(start_locations)) == len(start_locations), (
+        "Tokens must have unique starting locations.")
+    # Starting positions must be increasing (i.e sorted).
+    assert sorted(start_locations) == start_locations, (
+        "Tokens must have increasing start locations.")
+    # Last token is always EOF.
+    assert tokens[-1]['type'] == 'eof'
 
 
 @settings(**BASE_SETTINGS)
@@ -65,6 +89,9 @@ def test_parser_api_from_str(expr):
     except Exception as e:
         raise AssertionError("Non JMESPathError raised: %s" % e)
     assert isinstance(ast.parsed, dict)
+    assert 'type' in ast.parsed
+    assert 'children' in ast.parsed
+    assert isinstance(ast.parsed['children'], list)
 
 
 @settings(**BASE_SETTINGS)
@@ -82,3 +109,32 @@ def test_search_api(expr, data):
         return
     except Exception as e:
         raise AssertionError("Non JMESPathError raised: %s" % e)
+
+
+# Additional property tests for functions.
+
+@given(arg=JSON_NUMBERS)
+def test_abs(arg):
+    assert Functions().call_function('abs', [arg]) >= 0
+
+
+@given(arg=st.lists(JSON_NUMBERS))
+def test_avg(arg):
+    result = Functions().call_function('avg', [arg])
+    if result is not None:
+        assert isinstance(result, numbers.Number)
+
+
+@given(arg=st.lists(st.floats() | st.booleans() | st.text() | st.none(),
+                    min_size=1))
+def test_not_null(arg):
+    result = Functions().call_function('not_null', arg)
+    if result is not None:
+        assert result in arg
+
+
+@given(arg=RANDOM_JSON)
+def test_to_number(arg):
+    result = Functions().call_function('to_number', [arg])
+    if result is not None:
+        assert isinstance(result, numbers.Number)

jmespath/functions.py

@@ -16,6 +16,7 @@
     'str': 'string',
     'float': 'number',
     'int': 'number',
+    'long': 'number',
     'OrderedDict': 'object',
     '_Projection': 'array',
     '_Expression': 'expref',
@@ -29,7 +30,7 @@
     'object': ('dict', 'OrderedDict',),
     'null': ('None',),
     'string': ('unicode', 'str'),
-    'number': ('float', 'int'),
+    'number': ('float', 'int', 'long'),
     'expref': ('_Expression',),
 }
 

jmespath/lexer.py

@@ -92,10 +92,17 @@ def tokenize(self, expression):
                         'start': self._position - 1, 'end': self._position}
                     self._next()
                 else:
+                    if self._current is None:
+                        # If we're at the EOF, we never advanced
+                        # the position so we don't need to rewind
+                        # it back one location.
+                        position = self._position
+                    else:
+                        position = self._position - 1
                     raise LexerError(
-                        lexer_position=self._position - 1,
+                        lexer_position=position,
                         lexer_value='=',
-                        message="Unknown token =")
+                        message="Unknown token '='")
             else:
                 raise LexerError(lexer_position=self._position,
                                  lexer_value=self._current,
@@ -138,8 +145,9 @@ def _consume_until(self, delimiter):
                 buff += '\\'
                 self._next()
             if self._current is None:
+                # We're at the EOF.
                 raise LexerError(lexer_position=start,
-                                 lexer_value=self._expression,
+                                 lexer_value=self._expression[start:],
                                  message="Unclosed %s delimiter" % delimiter)
             buff += self._current
             self._next()
@@ -162,7 +170,7 @@ def _consume_literal(self):
                               PendingDeprecationWarning)
             except ValueError:
                 raise LexerError(lexer_position=start,
-                                 lexer_value=self._expression,
+                                 lexer_value=self._expression[start:],
                                  message="Bad token %s" % lexeme)
         token_len = self._position - start
         return {'type': 'literal', 'value': parsed_json,