Support for json multipath ($) (#1663)

Author: chayim

redis/commands/helpers.py

@@ -17,7 +17,10 @@ def list_or_args(keys, args):
 
 def nativestr(x):
     """Return the decoded binary string, or a string, depending on type."""
-    return x.decode("utf-8", "replace") if isinstance(x, bytes) else x
+    r = x.decode("utf-8", "replace") if isinstance(x, bytes) else x
+    if r == 'null':
+        return
+    return r
 
 
 def delist(x):

redis/commands/json/__init__.py

@@ -1,11 +1,10 @@
-from json import JSONDecoder, JSONEncoder
+from json import JSONDecoder, JSONEncoder, JSONDecodeError
 
 from .decoders import (
-    int_or_list,
-    int_or_none
+    decode_list,
+    bulk_of_jsons,
 )
-from .helpers import bulk_of_jsons
-from ..helpers import nativestr, delist
+from ..helpers import nativestr
 from .commands import JSONCommands
 
 
@@ -46,19 +45,19 @@ def __init__(
             "JSON.SET": lambda r: r and nativestr(r) == "OK",
             "JSON.NUMINCRBY": self._decode,
             "JSON.NUMMULTBY": self._decode,
-            "JSON.TOGGLE": lambda b: b == b"true",
-            "JSON.STRAPPEND": int,
-            "JSON.STRLEN": int,
-            "JSON.ARRAPPEND": int,
-            "JSON.ARRINDEX": int,
-            "JSON.ARRINSERT": int,
-            "JSON.ARRLEN": int_or_none,
+            "JSON.TOGGLE": self._decode,
+            "JSON.STRAPPEND": self._decode,
+            "JSON.STRLEN": self._decode,
+            "JSON.ARRAPPEND": self._decode,
+            "JSON.ARRINDEX": self._decode,
+            "JSON.ARRINSERT": self._decode,
+            "JSON.ARRLEN": self._decode,
             "JSON.ARRPOP": self._decode,
-            "JSON.ARRTRIM": int,
-            "JSON.OBJLEN": int,
-            "JSON.OBJKEYS": delist,
-            # "JSON.RESP": delist,
-            "JSON.DEBUG": int_or_list,
+            "JSON.ARRTRIM": self._decode,
+            "JSON.OBJLEN": self._decode,
+            "JSON.OBJKEYS": self._decode,
+            "JSON.RESP": self._decode,
+            "JSON.DEBUG": self._decode,
         }
 
         self.client = client
@@ -77,9 +76,17 @@ def _decode(self, obj):
             return obj
 
         try:
-            return self.__decoder__.decode(obj)
+            x = self.__decoder__.decode(obj)
+            if x is None:
+                raise TypeError
+            return x
         except TypeError:
-            return self.__decoder__.decode(obj.decode())
+            try:
+                return self.__decoder__.decode(obj.decode())
+            except AttributeError:
+                return decode_list(obj)
+        except (AttributeError, JSONDecodeError):
+            return decode_list(obj)
 
     def _encode(self, obj):
         """Get the encoder."""

redis/commands/json/commands.py

@@ -1,5 +1,5 @@
 from .path import Path
-from .helpers import decode_dict_keys
+from .decoders import decode_dict_keys
 from deprecated import deprecated
 from redis.exceptions import DataError
 
@@ -192,7 +192,7 @@ def strappend(self, name, value, path=Path.rootPath()):
         the key name, the path is determined to be the first. If a single
         option is passed, then the rootpath (i.e Path.rootPath()) is used.
         """
-        pieces = [name, str(path), value]
+        pieces = [name, str(path), self._encode(value)]
         return self.execute_command(
             "JSON.STRAPPEND", *pieces
         )

redis/commands/json/decoders.py

@@ -1,12 +1,59 @@
-def int_or_list(b):
-    if isinstance(b, int):
-        return b
-    else:
-        return b
+from ..helpers import nativestr
+import re
+import copy
+
 
+def bulk_of_jsons(d):
+    """Replace serialized JSON values with objects in a
+    bulk array response (list).
+    """
 
-def int_or_none(b):
-    if b is None:
-        return None
-    if isinstance(b, int):
+    def _f(b):
+        for index, item in enumerate(b):
+            if item is not None:
+                b[index] = d(item)
         return b
+
+    return _f
+
+
+def decode_dict_keys(obj):
+    """Decode the keys of the given dictionary with utf-8."""
+    newobj = copy.copy(obj)
+    for k in obj.keys():
+        if isinstance(k, bytes):
+            newobj[k.decode("utf-8")] = newobj[k]
+            newobj.pop(k)
+    return newobj
+
+
+def unstring(obj):
+    """
+    Attempt to parse string to native integer formats.
+    One can't simply call int/float in a try/catch because there is a
+    semantic difference between (for example) 15.0 and 15.
+    """
+    floatreg = '^\\d+.\\d+$'
+    match = re.findall(floatreg, obj)
+    if match != []:
+        return float(match[0])
+
+    intreg = "^\\d+$"
+    match = re.findall(intreg, obj)
+    if match != []:
+        return int(match[0])
+    return obj
+
+
+def decode_list(b):
+    """
+    Given a non-deserializable object, make a best effort to
+    return a useful set of results.
+    """
+    if isinstance(b, list):
+        return [nativestr(obj) for obj in b]
+    elif isinstance(b, bytes):
+        return unstring(nativestr(b))
+    elif isinstance(b, str):
+        return unstring(b)
+    return b

redis/commands/json/helpers.py

@@ -126,7 +126,8 @@ def teardown():
 @pytest.fixture()
 def modclient(request, **kwargs):
     rmurl = request.config.getoption('--redismod-url')
-    with _get_client(redis.Redis, request, from_url=rmurl, **kwargs) as client:
+    with _get_client(redis.Redis, request, from_url=rmurl,
+                     decode_responses=True, **kwargs) as client:
         yield client