Yajl-Py 2.0.2

pykler · pykler · commit 3cb48f949df3 · 2011-06-28T21:18:39.000-03:00
Details:

    Massive changes to support Yajl 2. There is a slight API breaking
    change in YajlParser and YajlGen:
    YajlParser: no longer is configured by __init__, it is not
      configured by attaching attributes to the parser before calling
      parse. Attribute names are similar to that of yajl
    Yajl Gen: indent is now changed to indent_string to comply with yajl
      naming scheme. Now accpets all yajl gen config with names similar
      to that of yajl's
diff --git a/examples/README.rst b/examples/README.rst
@@ -4,7 +4,7 @@ To run the sample parser, from within this dir, run::
 
 the example can also be run from ../ as::
     
-    python example/yajl_py_example.py <doc.json
+    python examples/yajl_py_example.py <doc.json
 
 In this directory you will also find a python implementation of
 json_reformat, named json_reformat.py, it takes the same parameters as
diff --git a/examples/json_reformat.py b/examples/json_reformat.py
@@ -15,12 +15,12 @@ class ReformatContentHandler(YajlContentHandler):
     '''
     Content handler to reformat a json file using yajl_gen
     '''
-    def __init__(self, beautify=True, indent="  "):
+    def __init__(self, beautify=True, indent_string="  "):
         self.out = sys.stdout
         self.beautify = beautify
-        self.indent = indent
+        self.indent_string = indent_string
     def parse_start(self):
-        self.g = YajlGen(beautify=self.beautify, indent=self.indent)
+        self.g = YajlGen(beautify=self.beautify, indent_string=self.indent_string)
     def parse_buf(self):
         self.out.write(self.g.yajl_gen_get_buf())
     def parse_complete(self):
@@ -54,12 +54,13 @@ def main():
         dest="beautify", action="store_false", default=True,
         help="minimize json rather than beautify (default)")
     opt_parser.add_option("-u",
-        dest="check_utf8", action='store_false', default=True,
+        dest="dont_validate_strings", action='store_true', default=False,
         help="allow invalid UTF8 inside strings during parsing")
     (options, args) = opt_parser.parse_args()
-    yajl_parser = YajlParser(
-        ReformatContentHandler(beautify=options.beautify),
-        check_utf8=options.check_utf8)
+    ch = ReformatContentHandler(beautify=options.beautify)
+    yajl_parser = YajlParser(ch)
+    yajl_parser.allow_comments = True # let's allow comments by default
+    yajl_parser.dont_validate_strings = options.dont_validate_strings
     yajl_parser.parse()
 
 if __name__ == "__main__":
diff --git a/examples/json_verify.py b/examples/json_verify.py
@@ -22,12 +22,12 @@ def main():
         dest="allow_comments", action="store_true", default=False,
         help="allow comments")
     opt_parser.add_option("-u",
-        dest="check_utf8", action='store_false', default=True,
+        dest="dont_validate_strings", action='store_true', default=False,
         help="allow invalid utf8 inside strings")
     (options, args) = opt_parser.parse_args()
-    yajl_parser = YajlParser(
-        allow_comments=options.allow_comments,
-        check_utf8=options.check_utf8)
+    yajl_parser = YajlParser()
+    yajl_parser.allow_comments = options.allow_comments
+    yajl_parser.dont_validate_strings = options.dont_validate_strings
     retval = 0
     try:
         yajl_parser.parse()
diff --git a/tests/cases/ac_difficult_json_c_test_case_with_comments.json b/tests/cases/ac_difficult_json_c_test_case_with_comments.json
diff --git a/tests/cases/ac_difficult_json_c_test_case_with_comments.json.gold b/tests/cases/ac_difficult_json_c_test_case_with_comments.json.gold
diff --git a/tests/cases/ac_simple_with_comments.json b/tests/cases/ac_simple_with_comments.json
diff --git a/tests/cases/ac_simple_with_comments.json.gold b/tests/cases/ac_simple_with_comments.json.gold
@@ -0,0 +1,9 @@
+map open '{'
+key: 'this'
+string: 'is'
+key: 'really'
+string: 'simple'
+key: 'json'
+string: 'right?'
+map close '}'
+memory leaks:	0
diff --git a/tests/cases/ag_false_then_garbage.json b/tests/cases/ag_false_then_garbage.json
diff --git a/tests/cases/ag_false_then_garbage.json.gold b/tests/cases/ag_false_then_garbage.json.gold
diff --git a/tests/cases/ag_null_then_garbage.json b/tests/cases/ag_null_then_garbage.json
diff --git a/tests/cases/ag_null_then_garbage.json.gold b/tests/cases/ag_null_then_garbage.json.gold
diff --git a/tests/cases/ag_true_then_garbage.json b/tests/cases/ag_true_then_garbage.json
diff --git a/tests/cases/ag_true_then_garbage.json.gold b/tests/cases/ag_true_then_garbage.json.gold
diff --git a/tests/cases/am_eof.json b/tests/cases/am_eof.json
@@ -0,0 +1 @@
+{ "123":
diff --git a/tests/cases/am_eof.json.gold b/tests/cases/am_eof.json.gold
@@ -0,0 +1,4 @@
+map open '{'
+key: '123'
+parse error: premature EOF
+memory leaks:	0
diff --git a/tests/cases/am_integers.json b/tests/cases/am_integers.json
@@ -0,0 +1 @@
+1221 21
diff --git a/tests/cases/am_integers.json.gold b/tests/cases/am_integers.json.gold
@@ -0,0 +1,3 @@
+integer: 1221
+integer: 21
+memory leaks:	0
diff --git a/tests/cases/am_multiple.json b/tests/cases/am_multiple.json
@@ -0,0 +1,3 @@
+
+{}
+{}
diff --git a/tests/cases/am_multiple.json.gold b/tests/cases/am_multiple.json.gold
@@ -0,0 +1,5 @@
+map open '{'
+map close '}'
+map open '{'
+map close '}'
+memory leaks:	0
diff --git a/tests/cases/am_stuff.json b/tests/cases/am_stuff.json
@@ -0,0 +1,7 @@
+{}
+[]
+[]
+"sdfasd"
+123
+{ "123" : 123 }
+3.1e124
diff --git a/tests/cases/am_stuff.json.gold b/tests/cases/am_stuff.json.gold
@@ -0,0 +1,14 @@
+map open '{'
+map close '}'
+array open '['
+array close ']'
+array open '['
+array close ']'
+string: 'sdfasd'
+integer: 123
+map open '{'
+key: '123'
+integer: 123
+map close '}'
+double: 3.1e+124
+memory leaks:	0
diff --git a/tests/cases/ap_array_open.json b/tests/cases/ap_array_open.json
diff --git a/tests/cases/ap_array_open.json.gold b/tests/cases/ap_array_open.json.gold
diff --git a/tests/cases/ap_eof_str.json b/tests/cases/ap_eof_str.json
@@ -0,0 +1 @@
+"abc
diff --git a/tests/cases/ap_eof_str.json.gold b/tests/cases/ap_eof_str.json.gold
@@ -0,0 +1 @@
+memory leaks:	0
diff --git a/tests/cases/ap_map_open.json b/tests/cases/ap_map_open.json
diff --git a/tests/cases/ap_map_open.json.gold b/tests/cases/ap_map_open.json.gold
diff --git a/tests/cases/ap_partial_ok.json b/tests/cases/ap_partial_ok.json
@@ -0,0 +1 @@
+[ "foo", "bar"
diff --git a/tests/cases/ap_partial_ok.json.gold b/tests/cases/ap_partial_ok.json.gold
@@ -0,0 +1,4 @@
+array open '['
+string: 'foo'
+string: 'bar'
+memory leaks:	0
diff --git a/tests/cases/bignums.json b/tests/cases/bignums.json
@@ -0,0 +1 @@
+[ 9223372036854775807, -9223372036854775807 ]
diff --git a/tests/cases/bignums.json.gold b/tests/cases/bignums.json.gold
@@ -0,0 +1,5 @@
+array open '['
+integer: 9223372036854775807
+integer: -9223372036854775807
+array close ']'
+memory leaks:	0
diff --git a/tests/cases/dc_simple_with_comments.json.gold b/tests/cases/dc_simple_with_comments.json.gold
diff --git a/tests/cases/fg_false_then_garbage.json b/tests/cases/fg_false_then_garbage.json
@@ -0,0 +1 @@
+falsex
diff --git a/tests/cases/fg_false_then_garbage.json.gold b/tests/cases/fg_false_then_garbage.json.gold
@@ -0,0 +1,3 @@
+bool: false
+parse error: trailing garbage
+memory leaks:	0
diff --git a/tests/cases/fg_issue_7.json b/tests/cases/fg_issue_7.json
@@ -0,0 +1 @@
+2009-10-20@20:38:21.539575
diff --git a/tests/cases/fg_issue_7.json.gold b/tests/cases/fg_issue_7.json.gold
@@ -0,0 +1,3 @@
+integer: 2009
+parse error: trailing garbage
+memory leaks:	0
diff --git a/tests/cases/fg_null_then_garbage.json b/tests/cases/fg_null_then_garbage.json
@@ -0,0 +1 @@
+nullx
diff --git a/tests/cases/fg_null_then_garbage.json.gold b/tests/cases/fg_null_then_garbage.json.gold
@@ -0,0 +1,3 @@
+null
+parse error: trailing garbage
+memory leaks:	0
diff --git a/tests/cases/fg_true_then_garbage.json b/tests/cases/fg_true_then_garbage.json
@@ -0,0 +1 @@
+truex
diff --git a/tests/cases/fg_true_then_garbage.json.gold b/tests/cases/fg_true_then_garbage.json.gold
@@ -0,0 +1,3 @@
+bool: true
+parse error: trailing garbage
+memory leaks:	0
diff --git a/tests/cases/high_overflow.json b/tests/cases/high_overflow.json
@@ -0,0 +1 @@
+9223372036854775808
diff --git a/tests/cases/high_overflow.json.gold b/tests/cases/high_overflow.json.gold
@@ -0,0 +1,2 @@
+parse error: integer overflow
+memory leaks:	0
diff --git a/tests/cases/low_overflow.json b/tests/cases/low_overflow.json
@@ -0,0 +1 @@
+-9223372036854775808
diff --git a/tests/cases/low_overflow.json.gold b/tests/cases/low_overflow.json.gold
@@ -0,0 +1,2 @@
+parse error: integer overflow
+memory leaks:	0
diff --git a/tests/cases/multiple.json b/tests/cases/multiple.json
@@ -0,0 +1,3 @@
+
+{}
+{}
diff --git a/tests/cases/multiple.json.gold b/tests/cases/multiple.json.gold
@@ -0,0 +1,4 @@
+map open '{'
+map close '}'
+parse error: trailing garbage
+memory leaks:	0
diff --git a/tests/cases/np_partial_bad.json b/tests/cases/np_partial_bad.json
@@ -0,0 +1 @@
+[ "foo", "bar"
diff --git a/tests/cases/np_partial_bad.json.gold b/tests/cases/np_partial_bad.json.gold
@@ -0,0 +1,5 @@
+array open '['
+string: 'foo'
+string: 'bar'
+parse error: premature EOF
+memory leaks:	0
diff --git a/tests/cases/simple_with_comments.json.gold b/tests/cases/simple_with_comments.json.gold
@@ -1,9 +1,5 @@
 map open '{'
 key: 'this'
 string: 'is'
-key: 'really'
-string: 'simple'
-key: 'json'
-string: 'right?'
-map close '}'
+lexical error: probable comment found in input text, comments are not enabled.
 memory leaks:	0
diff --git a/tests/test_yajl.py b/tests/test_yajl.py
@@ -90,7 +90,7 @@ def test_allCallbacksExceptNumber(self):
 
     def test_largestNumberAcceptableInIntegerCallback(self):
         good_json = StringIO('[%s]' %(sys.maxint))
-        bad_json = StringIO('[%s]' %(sys.maxint + 1))
+        bad_json = StringIO('[%s]' %(sys.maxint+1))
         self.mock('self.content_handler.yajl_integer')
         parser = yajl.YajlParser(self.content_handler)
         parser.parse(good_json)
@@ -144,13 +144,13 @@ class YajlGenTests(MockTestCase):
     Testing :class:`YajlGen` works as expected
     '''
     def test_YajlGen_callsYajlGenFreeWhenDone(self):
-        self.mock('yajl.yajl_gen.byref', tracker=None)
+        self.mock('yajl.yajl.yajl_gen_config', tracker=None)
         self.mock('yajl.yajl.yajl_gen_alloc')
         self.mock('yajl.yajl.yajl_gen_free')
         g = yajl.YajlGen()
         del g
         self.assertSameTrace(
-            'Called yajl.yajl.yajl_gen_alloc(None, None)\n'
+            'Called yajl.yajl.yajl_gen_alloc(None)\n'
             'Called yajl.yajl.yajl_gen_free(None)\n'
         )
 
@@ -186,19 +186,19 @@ def test_YajlGen_beautifyOutput(self):
         results = self._yajl_gen_sample(g)
         self.assertEqual(
             '{\n'
-            '  "a": [\n'
-            '    null,\n'
-            '    true,\n'
-            '    1,\n'
-            '    -6.5,\n'
-            '    3,\n'
-            '    "b"\n'
-            '  ]\n'
+            '    "a": [\n'
+            '        null,\n'
+            '        true,\n'
+            '        1,\n'
+            '        -6.5,\n'
+            '        3,\n'
+            '        "b"\n'
+            '    ]\n'
             '}\n',
             ''.join(results))
 
     def test_YajlGen_indentOutput(self):
-        g = yajl.YajlGen(beautify=True, indent="\t")
+        g = yajl.YajlGen(beautify=True, indent_string="\t")
         results = self._yajl_gen_sample(g)
         self.assertEqual(
             '{\n'
diff --git a/tests/test_yajl_c.py b/tests/test_yajl_c.py
@@ -53,13 +53,21 @@ def resetOutput(self):
 
 def _make_test(filename, testname):
     def test(self):
-        kwargs = {}
-        if testname.startswith('dc_'):
-            kwargs['allow_comments'] = False
+        config_key = None
+        config = lambda yp: setattr(yp, config_key, True)
+        if testname.startswith('ac_'):
+            config_key = 'allow_comments'
+        if testname.startswith('ag_'):
+            config_key = 'allow_trailing_garbage'
+        if testname.startswith('am_'):
+            config_key = 'allow_multiple_values'
+        if testname.startswith('ap_'):
+            config_key = 'allow_partial_values'
         for buf_siz in range(1, 32):
             # try out a range of buffer_sizes to stress stream parsing
-            kwargs['buf_siz'] = buf_siz
-            parser = yajl.YajlParser(self.content_handler, **kwargs)
+            parser = yajl.YajlParser(self.content_handler, buf_siz=buf_siz)
+            if config_key:
+                config(parser)
             with open(filename) as f:
                 try:
                     parser.parse(f)
diff --git a/yajl/__init__.py b/yajl/__init__.py
@@ -14,7 +14,7 @@
 from yajl_parse import *
 from yajl_gen import *
 
-__version__ = '1.0.12'
+__version__ = '2.0.2' 
 yajl_version = get_yajl_version()
 
 def check_yajl_version():
diff --git a/yajl/yajl_common.py b/yajl/yajl_common.py
@@ -29,7 +29,8 @@ def load_yajl():
     :returns: The yajl shared object
     :raises OSError: when libyajl cannot be loaded
     '''
-    for yajlso in 'libyajl.so', 'libyajl.dylib':
+    for ftype in '', '.so', '.dylib':
+        yajlso = 'libyajl%s' %(ftype)
         try:
             return cdll.LoadLibrary(yajlso)
         except OSError:
@@ -50,11 +51,15 @@ def get_yajl_version():
 yajl = load_yajl()
 
 yajl.yajl_alloc.restype = c_void_p
-yajl.yajl_alloc.argtypes = [c_void_p, c_void_p, c_void_p, c_void_p]
+yajl.yajl_alloc.argtypes = [c_void_p, c_void_p, c_void_p]
+yajl.yajl_config.restype = c_int
+# yajl.yajl_config.argtypes = [c_void_p, c_int, ...]
 yajl.yajl_free.argtypes = [c_void_p]
 yajl.yajl_parse.restype = c_int
-yajl.yajl_parse.argtypes = [c_void_p, c_char_p, c_int]
-yajl.yajl_parse_complete.restype = c_int
-yajl.yajl_parse_complete.argtypes = [c_void_p]
+yajl.yajl_parse.argtypes = [c_void_p, c_char_p, c_size_t]
+yajl.yajl_complete_parse.restype = c_int
+yajl.yajl_complete_parse.argtypes = [c_void_p]
 yajl.yajl_get_error.restype = c_char_p
-yajl.yajl_get_error.argtypes = [c_void_p, c_int, c_char_p, c_int]
+yajl.yajl_get_error.argtypes = [c_void_p, c_int, c_char_p, c_size_t]
+yajl.yajl_get_bytes_consumed.restype = c_uint
+yajl.yajl_get_bytes_consumed.argtypes = [c_void_p, c_char_p]
diff --git a/yajl/yajl_gen.py b/yajl/yajl_gen.py
@@ -13,14 +13,17 @@
     4: 'yajl_gen_generation_complete',
     5: 'yajl_gen_invalid_number',
     6: 'yajl_gen_no_buf',
+    7: 'yajl_gen_invalid_string',
 }
 yajl_gen_status_ok = 0
 
-class yajl_gen_config(Structure):
-    _fields_ = [
-        ("beautify", c_uint),
-        ("indentString", c_char_p),
-    ]
+(
+yajl_gen_beautify,
+yajl_gen_indent_string,
+yajl_gen_print_callback,
+yajl_gen_validate_utf8,
+yajl_gen_escape_solidus,
+) = map(c_int, [2**x for x in range(5)])
 
 class YajlGenException(YajlError):
     pass
@@ -30,7 +33,7 @@ class YajlGen(object):
     Yajl Generator - json formatting using yajl_gen
 
     '''
-    def __init__(self, beautify=True, indent="  "):
+    def __init__(self, **kwargs):
         '''
         :param beautify: To pretty print json (or not)
         :type beautify: bool
@@ -43,8 +46,17 @@ def __init__(self, beautify=True, indent="  "):
             :cfunc:`yajl.yajl_gen_alloc`.
             This should not be used directly.
         '''
-        conf = yajl_gen_config(beautify, indent)
-        self.g = yajl.yajl_gen_alloc(byref(conf), None)
+        self.g = yajl.yajl_gen_alloc(None)
+        config_map = dict([
+            ('beautify', yajl_gen_beautify),
+            ('indent_string', yajl_gen_indent_string),
+            ('print_callback', yajl_gen_print_callback),
+            ('validate_utf8', yajl_gen_validate_utf8),
+            ('gen_escape_solidus', yajl_gen_escape_solidus),
+        ])
+        for k,v in kwargs.items():
+            yajl.yajl_gen_config(self.g, config_map[k], v)
+
     def __del__(self):
         yajl.yajl_gen_free(self.g)
     def _assert_retval(self, retval):
@@ -97,7 +109,7 @@ def yajl_gen_integer(self, n):
         :param n: number to be jsonified
         :type n: int
         '''
-        self._dispatch('yajl_gen_integer', c_long(n))
+        self._dispatch('yajl_gen_integer', c_longlong(n))
     def yajl_gen_double(self, n):
         ''' 
         :param n: number to be jsonified
diff --git a/yajl/yajl_parse.py b/yajl/yajl_parse.py
