Fix bug in null matching on queryoptimizer

Author: Jibola

django_mongodb_backend/query_conversion/expression_converters.py

@@ -44,8 +44,12 @@ def convert(cls, args):
             ):
                 field_name = field_expr[1:]  # Remove the $ prefix.
                 if cls.operator == "$eq":
-                    return {field_name: value}
-                return {field_name: {cls.operator: value}}
+                    query = {field_name: value}
+                else:
+                    query = {field_name: {cls.operator: value}}
+                if value is None:
+                    query = {"$and": [{field_name: {"$exists": True}}, query]}
+                return query
         return None
 
 
@@ -102,7 +106,12 @@ def convert(cls, in_args):
                 if isinstance(values, (list, tuple, set)) and all(
                     cls.is_simple_value(v) for v in values
                 ):
-                    return {field_name: {"$in": values}}
+                    core_check = {field_name: {"$in": values}}
+                    return (
+                        {"$and": [{field_name: {"$exists": True}}, core_check]}
+                        if None in values
+                        else core_check
+                    )
         return None
 
 

tests/expression_converter_/test_op_expressions.py

@@ -7,6 +7,12 @@
 from django_mongodb_backend.query_conversion.expression_converters import convert_expression
 
 
+def _wrap_condition_if_null(_type, condition, path):
+    if _type is None:
+        return {"$and": [{path: {"$exists": True}}, condition]}
+    return condition
+
+
 class ConversionTestCase(SimpleTestCase):
     CONVERTIBLE_TYPES = {
         "int": 42,
@@ -53,10 +59,14 @@ def test_no_conversion_dict_value(self):
         self.assertNotOptimizable({"$eq": ["$status", {"$gt": 5}]})
 
     def _test_conversion_valid_type(self, _type):
-        self.assertConversionEqual({"$eq": ["$age", _type]}, {"age": _type})
+        self.assertConversionEqual(
+            {"$eq": ["$age", _type]}, _wrap_condition_if_null(_type, {"age": _type}, "age")
+        )
 
     def _test_conversion_valid_array_type(self, _type):
-        self.assertConversionEqual({"$eq": ["$age", _type]}, {"age": _type})
+        self.assertConversionEqual(
+            {"$eq": ["$age", _type]}, _wrap_condition_if_null(_type, {"age": _type}, "age")
+        )
 
     def test_conversion_various_types(self):
         self._test_conversion_various_types(self._test_conversion_valid_type)
@@ -78,7 +88,10 @@ def test_no_conversion_dict_value(self):
         self.assertNotOptimizable({"$in": ["$status", [{"bad": "val"}]]})
 
     def _test_conversion_valid_type(self, _type):
-        self.assertConversionEqual({"$in": ["$age", [_type]]}, {"age": {"$in": [_type]}})
+        self.assertConversionEqual(
+            {"$in": ["$age", [_type]]},
+            _wrap_condition_if_null(_type, {"age": {"$in": [_type]}}, "age"),
+        )
 
     def test_conversion_various_types(self):
         for _type, val in self.CONVERTIBLE_TYPES.items():
@@ -170,7 +183,10 @@ def test_no_conversion_dict_value(self):
         self.assertNotOptimizable({"$gt": ["$price", {}]})
 
     def _test_conversion_valid_type(self, _type):
-        self.assertConversionEqual({"$gt": ["$price", _type]}, {"price": {"$gt": _type}})
+        self.assertConversionEqual(
+            {"$gt": ["$price", _type]},
+            _wrap_condition_if_null(_type, {"price": {"$gt": _type}}, "price"),
+        )
 
     def test_conversion_various_types(self):
         self._test_conversion_various_types(self._test_conversion_valid_type)
@@ -193,7 +209,7 @@ def test_no_conversion_dict_value(self):
     def _test_conversion_valid_type(self, _type):
         expr = {"$gte": ["$price", _type]}
         expected = {"price": {"$gte": _type}}
-        self.assertConversionEqual(expr, expected)
+        self.assertConversionEqual(expr, _wrap_condition_if_null(_type, expected, "price"))
 
     def test_conversion_various_types(self):
         self._test_conversion_various_types(self._test_conversion_valid_type)
@@ -210,7 +226,10 @@ def test_no_conversion_dict_value(self):
         self.assertNotOptimizable({"$lt": ["$price", {}]})
 
     def _test_conversion_valid_type(self, _type):
-        self.assertConversionEqual({"$lt": ["$price", _type]}, {"price": {"$lt": _type}})
+        self.assertConversionEqual(
+            {"$lt": ["$price", _type]},
+            _wrap_condition_if_null(_type, {"price": {"$lt": _type}}, "price"),
+        )
 
     def test_conversion_various_types(self):
         self._test_conversion_various_types(self._test_conversion_valid_type)
@@ -227,7 +246,10 @@ def test_no_conversion_dict_value(self):
         self.assertNotOptimizable({"$lte": ["$price", {}]})
 
     def _test_conversion_valid_type(self, _type):
-        self.assertConversionEqual({"$lte": ["$price", _type]}, {"price": {"$lte": _type}})
+        self.assertConversionEqual(
+            {"$lte": ["$price", _type]},
+            _wrap_condition_if_null(_type, {"price": {"$lte": _type}}, "price"),
+        )
 
     def test_conversion_various_types(self):
         self._test_conversion_various_types(self._test_conversion_valid_type)