Functional approach solution

Author: WaVEV

django_mongodb_backend/base.py

@@ -20,7 +20,7 @@
 from .features import DatabaseFeatures
 from .introspection import DatabaseIntrospection
 from .operations import DatabaseOperations
-from .query_utils import regex_match
+from .query_utils import regex_expr, regex_match
 from .schema import DatabaseSchemaEditor
 from .utils import OperationDebugWrapper
 from .validation import DatabaseValidation
@@ -108,7 +108,12 @@ def _isnull_operator(a, b):
         }
         return is_null if b else {"$not": is_null}
 
-    mongo_operators = {
+    def _isnull_operator_match(a, b):
+        if b:
+            return {"$or": [{a: {"$exists": False}}, {a: None}]}
+        return {"$and": [{a: {"$exists": True}}, {a: {"$ne": None}}]}
+
+    mongo_operators_expr = {
         "exact": lambda a, b: {"$eq": [a, b]},
         "gt": lambda a, b: {"$gt": [a, b]},
         "gte": lambda a, b: {"$gte": [a, b]},
@@ -118,19 +123,56 @@ def _isnull_operator(a, b):
         "lte": lambda a, b: {
             "$and": [{"$lte": [a, b]}, DatabaseWrapper._isnull_operator(a, False)]
         },
-        "in": lambda a, b: {"$in": [a, b]},
+        "in": lambda a, b: {"$in": (a, b)},
         "isnull": _isnull_operator,
         "range": lambda a, b: {
             "$and": [
                 {"$or": [DatabaseWrapper._isnull_operator(b[0], True), {"$gte": [a, b[0]]}]},
                 {"$or": [DatabaseWrapper._isnull_operator(b[1], True), {"$lte": [a, b[1]]}]},
             ]
         },
-        "iexact": lambda a, b: regex_match(a, ("^", b, {"$literal": "$"}), insensitive=True),
-        "startswith": lambda a, b: regex_match(a, ("^", b)),
-        "istartswith": lambda a, b: regex_match(a, ("^", b), insensitive=True),
-        "endswith": lambda a, b: regex_match(a, (b, {"$literal": "$"})),
-        "iendswith": lambda a, b: regex_match(a, (b, {"$literal": "$"}), insensitive=True),
+        "iexact": lambda a, b: regex_expr(a, ("^", b, {"$literal": "$"}), insensitive=True),
+        "startswith": lambda a, b: regex_expr(a, ("^", b)),
+        "istartswith": lambda a, b: regex_expr(a, ("^", b), insensitive=True),
+        "endswith": lambda a, b: regex_expr(a, (b, {"$literal": "$"})),
+        "iendswith": lambda a, b: regex_expr(a, (b, {"$literal": "$"}), insensitive=True),
+        "contains": lambda a, b: regex_expr(a, b),
+        "icontains": lambda a, b: regex_expr(a, b, insensitive=True),
+        "regex": lambda a, b: regex_expr(a, b),
+        "iregex": lambda a, b: regex_expr(a, b, insensitive=True),
+    }
+
+    def range_match(a, b):
+        ## TODO: MAKE A TEST TO TEST WHEN BOTH ENDS ARE NONE. WHAT SHALL I RETURN?
+        conditions = []
+        if b[0] is not None:
+            conditions.append({a: {"$gte": b[0]}})
+        if b[1] is not None:
+            conditions.append({a: {"$lte": b[1]}})
+        if not conditions:
+            return {"$literal": True}
+        return {"$and": conditions}
+
+    mongo_operators_match = {
+        "exact": lambda a, b: {a: b},
+        "gt": lambda a, b: {a: {"$gt": b}},
+        "gte": lambda a, b: {a: {"$gte": b}},
+        # MongoDB considers null less than zero. Exclude null values to match
+        # SQL behavior.
+        "lt": lambda a, b: {
+            "$and": [{a: {"$lt": b}}, DatabaseWrapper._isnull_operator_match(a, False)]
+        },
+        "lte": lambda a, b: {
+            "$and": [{a: {"$lte": b}}, DatabaseWrapper._isnull_operator_match(a, False)]
+        },
+        "in": lambda a, b: {a: {"$in": tuple(b)}},
+        "isnull": _isnull_operator_match,
+        "range": range_match,
+        "iexact": lambda a, b: regex_match(a, f"^{b}$", insensitive=True),
+        "startswith": lambda a, b: regex_match(a, f"^{b}"),
+        "istartswith": lambda a, b: regex_match(a, f"^{b}", insensitive=True),
+        "endswith": lambda a, b: regex_match(a, f"{b}$"),
+        "iendswith": lambda a, b: regex_match(a, f"{b}$", insensitive=True),
         "contains": lambda a, b: regex_match(a, b),
         "icontains": lambda a, b: regex_match(a, b, insensitive=True),
         "regex": lambda a, b: regex_match(a, b),

django_mongodb_backend/compiler.py

@@ -327,14 +327,14 @@ def pre_sql_setup(self, with_col_aliases=False):
             pipeline = self._build_aggregation_pipeline(ids, group)
             if self.having:
                 having = self.having.replace_expressions(all_replacements).as_mql(
-                    self, self.connection
+                    self, self.connection, as_path=True
                 )
                 # Add HAVING subqueries.
                 for query in self.subqueries or ():
                     pipeline.extend(query.get_pipeline())
                 # Remove the added subqueries.
                 self.subqueries = []
-                pipeline.append({"$match": {"$expr": having}})
+                pipeline.append({"$match": having})
             self.aggregation_pipeline = pipeline
         self.annotations = {
             target: expr.replace_expressions(all_replacements)
@@ -481,11 +481,11 @@ def build_query(self, columns=None):
             query.lookup_pipeline = self.get_lookup_pipeline()
             where = self.get_where()
             try:
-                expr = where.as_mql(self, self.connection) if where else {}
+                expr = where.as_mql(self, self.connection, as_path=True) if where else {}
             except FullResultSet:
                 query.match_mql = {}
             else:
-                query.match_mql = {"$expr": expr}
+                query.match_mql = expr
         if extra_fields:
             query.extra_fields = self.get_project_fields(extra_fields, force_expression=True)
         query.subqueries = self.subqueries
@@ -643,7 +643,9 @@ def get_combinator_queries(self):
             for alias, expr in self.columns:
                 # Unfold foreign fields.
                 if isinstance(expr, Col) and expr.alias != self.collection_name:
-                    ids[expr.alias][expr.target.column] = expr.as_mql(self, self.connection)
+                    ids[expr.alias][expr.target.column] = expr.as_mql(
+                        self, self.connection, as_path=False
+                    )
                 else:
                     ids[alias] = f"${alias}"
             # Convert defaultdict to dict so it doesn't appear as
@@ -707,16 +709,16 @@ def get_project_fields(self, columns=None, ordering=None, force_expression=False
                     # For brevity/simplicity, project {"field_name": 1}
                     # instead of {"field_name": "$field_name"}.
                     if isinstance(expr, Col) and name == expr.target.column and not force_expression
-                    else expr.as_mql(self, self.connection)
+                    else expr.as_mql(self, self.connection, as_path=False)
                 )
             except EmptyResultSet:
                 empty_result_set_value = getattr(expr, "empty_result_set_value", NotImplemented)
                 value = (
                     False if empty_result_set_value is NotImplemented else empty_result_set_value
                 )
-                fields[collection][name] = Value(value).as_mql(self, self.connection)
+                fields[collection][name] = Value(value).as_mql(self, self.connection, as_path=False)
             except FullResultSet:
-                fields[collection][name] = Value(True).as_mql(self, self.connection)
+                fields[collection][name] = Value(True).as_mql(self, self.connection, as_path=False)
         # Annotations (stored in None) and the main collection's fields
         # should appear in the top-level of the fields dict.
         fields.update(fields.pop(None, {}))

django_mongodb_backend/expressions/builtins.py

@@ -6,13 +6,15 @@
 from django.core.exceptions import EmptyResultSet, FullResultSet
 from django.db import NotSupportedError
 from django.db.models.expressions import (
+    BaseExpression,
     Case,
     Col,
     ColPairs,
     CombinedExpression,
     Exists,
     ExpressionList,
     ExpressionWrapper,
+    Func,
     NegatedExpression,
     OrderBy,
     RawSQL,
@@ -23,17 +25,19 @@
     Value,
     When,
 )
+from django.db.models.fields.json import KeyTransform
 from django.db.models.sql import Query
 
-from django_mongodb_backend.query_utils import process_lhs
+from django_mongodb_backend.fields.array import Array
+from django_mongodb_backend.query_utils import is_direct_value, process_lhs
 
 
-def case(self, compiler, connection):
+def case(self, compiler, connection, as_path=False):
     case_parts = []
     for case in self.cases:
         case_mql = {}
         try:
-            case_mql["case"] = case.as_mql(compiler, connection)
+            case_mql["case"] = case.as_mql(compiler, connection, as_path=False)
         except EmptyResultSet:
             continue
         except FullResultSet:
@@ -45,12 +49,16 @@ def case(self, compiler, connection):
         default_mql = self.default.as_mql(compiler, connection)
     if not case_parts:
         return default_mql
-    return {
+    expr = {
         "$switch": {
             "branches": case_parts,
             "default": default_mql,
         }
     }
+    if as_path:
+        return {"$expr": expr}
+
+    return expr
 
 
 def col(self, compiler, connection, as_path=False):  # noqa: ARG001
@@ -76,34 +84,34 @@ def col(self, compiler, connection, as_path=False):  # noqa: ARG001
     return f"{prefix}{self.target.column}"
 
 
-def col_pairs(self, compiler, connection):
+def col_pairs(self, compiler, connection, as_path=False):
     cols = self.get_cols()
     if len(cols) > 1:
         raise NotSupportedError("ColPairs is not supported.")
-    return cols[0].as_mql(compiler, connection)
+    return cols[0].as_mql(compiler, connection, as_path=as_path)
 
 
-def combined_expression(self, compiler, connection):
+def combined_expression(self, compiler, connection, as_path=False):
     expressions = [
-        self.lhs.as_mql(compiler, connection),
-        self.rhs.as_mql(compiler, connection),
+        self.lhs.as_mql(compiler, connection, as_path=as_path),
+        self.rhs.as_mql(compiler, connection, as_path=as_path),
     ]
     return connection.ops.combine_expression(self.connector, expressions)
 
 
-def expression_wrapper(self, compiler, connection):
-    return self.expression.as_mql(compiler, connection)
+def expression_wrapper(self, compiler, connection, as_path=False):
+    return self.expression.as_mql(compiler, connection, as_path=as_path)
 
 
-def negated_expression(self, compiler, connection):
-    return {"$not": expression_wrapper(self, compiler, connection)}
+def negated_expression(self, compiler, connection, as_path=False):
+    return {"$not": expression_wrapper(self, compiler, connection, as_path=as_path)}
 
 
 def order_by(self, compiler, connection):
     return self.expression.as_mql(compiler, connection)
 
 
-def query(self, compiler, connection, get_wrapping_pipeline=None):
+def query(self, compiler, connection, get_wrapping_pipeline=None, as_path=False):
     subquery_compiler = self.get_compiler(connection=connection)
     subquery_compiler.pre_sql_setup(with_col_aliases=False)
     field_name, expr = subquery_compiler.columns[0]
@@ -145,14 +153,16 @@ def query(self, compiler, connection, get_wrapping_pipeline=None):
         # Erase project_fields since the required value is projected above.
         subquery.project_fields = None
     compiler.subqueries.append(subquery)
+    if as_path:
+        return f"{table_output}.{field_name}"
     return f"${table_output}.{field_name}"
 
 
 def raw_sql(self, compiler, connection):  # noqa: ARG001
     raise NotSupportedError("RawSQL is not supported on MongoDB.")
 
 
-def ref(self, compiler, connection):  # noqa: ARG001
+def ref(self, compiler, connection, as_path=False):  # noqa: ARG001
     prefix = (
         f"{self.source.alias}."
         if isinstance(self.source, Col) and self.source.alias != compiler.collection_name
@@ -162,32 +172,47 @@ def ref(self, compiler, connection):  # noqa: ARG001
         refs, _ = compiler.columns[self.ordinal - 1]
     else:
         refs = self.refs
-    return f"${prefix}{refs}"
+    if not as_path:
+        prefix = f"${prefix}"
+    return f"{prefix}{refs}"
 
 
-def star(self, compiler, connection):  # noqa: ARG001
+def star(self, compiler, connection, **extra):  # noqa: ARG001
     return {"$literal": True}
 
 
-def subquery(self, compiler, connection, get_wrapping_pipeline=None):
-    return self.query.as_mql(compiler, connection, get_wrapping_pipeline=get_wrapping_pipeline)
+def subquery(self, compiler, connection, get_wrapping_pipeline=None, as_path=False):
+    expr = self.query.as_mql(
+        compiler, connection, get_wrapping_pipeline=get_wrapping_pipeline, as_path=False
+    )
+    if as_path:
+        return {"$expr": expr}
+    return expr
 
 
-def exists(self, compiler, connection, get_wrapping_pipeline=None):
+def exists(self, compiler, connection, get_wrapping_pipeline=None, as_path=False):
     try:
-        lhs_mql = subquery(self, compiler, connection, get_wrapping_pipeline=get_wrapping_pipeline)
+        lhs_mql = subquery(
+            self,
+            compiler,
+            connection,
+            get_wrapping_pipeline=get_wrapping_pipeline,
+            as_path=as_path,
+        )
     except EmptyResultSet:
         return Value(False).as_mql(compiler, connection)
-    return connection.mongo_operators["isnull"](lhs_mql, False)
+    if as_path:
+        return {"$expr": connection.mongo_operators_match["isnull"](lhs_mql, False)}
+    return connection.mongo_operators_expr["isnull"](lhs_mql, False)
 
 
-def when(self, compiler, connection):
-    return self.condition.as_mql(compiler, connection)
+def when(self, compiler, connection, as_path=False):
+    return self.condition.as_mql(compiler, connection, as_path=as_path)
 
 
-def value(self, compiler, connection):  # noqa: ARG001
+def value(self, compiler, connection, as_path=False):  # noqa: ARG001
     value = self.value
-    if isinstance(value, (list, int)):
+    if isinstance(value, (list, int)) and not as_path:
         # Wrap lists & numbers in $literal to prevent ambiguity when Value
         # appears in $project.
         return {"$literal": value}
@@ -209,6 +234,36 @@ def value(self, compiler, connection):  # noqa: ARG001
     return value
 
 
+@staticmethod
+def _is_constant_value(value):
+    if isinstance(value, list | Array):
+        iterable = value.get_source_expressions() if isinstance(value, Array) else value
+        return all(_is_constant_value(e) for e in iterable)
+    if is_direct_value(value):
+        return True
+    return isinstance(value, Func | Value) and not (
+        value.contains_aggregate
+        or value.contains_over_clause
+        or value.contains_column_references
+        or value.contains_subquery
+    )
+
+
+@staticmethod
+def _is_simple_column(lhs):
+    while isinstance(lhs, KeyTransform):
+        if "." in getattr(lhs, "key_name", ""):
+            return False
+        lhs = lhs.lhs
+    col = lhs.source if isinstance(lhs, Ref) else lhs
+    # Foreign columns from parent cannot be addressed as single match
+    return isinstance(col, Col) and col.alias is not None
+
+
+def _is_simple_expression(self):
+    return self.is_simple_column(self.lhs) and self.is_constant_value(self.rhs)
+
+
 def register_expressions():
     Case.as_mql = case
     Col.as_mql = col
@@ -227,3 +282,6 @@ def register_expressions():
     Subquery.as_mql = subquery
     When.as_mql = when
     Value.as_mql = value
+    BaseExpression.is_simple_expression = _is_simple_expression
+    BaseExpression.is_simple_column = _is_simple_column
+    BaseExpression.is_constant_value = _is_constant_value

django_mongodb_backend/expressions/search.py

@@ -933,10 +933,12 @@ def __str__(self):
     def __repr__(self):
         return f"SearchText({self.lhs}, {self.rhs})"
 
-    def as_mql(self, compiler, connection):
-        lhs_mql = process_lhs(self, compiler, connection)
-        value = process_rhs(self, compiler, connection)
-        return {"$gte": [lhs_mql, value]}
+    def as_mql(self, compiler, connection, as_path=False):
+        lhs_mql = process_lhs(self, compiler, connection, as_path=as_path)
+        value = process_rhs(self, compiler, connection, as_path=as_path)
+        if as_path:
+            return {lhs_mql: {"$gte": value}}
+        return {"$expr": {"$gte": [lhs_mql, value]}}
 
 
 CharField.register_lookup(SearchTextLookup)

django_mongodb_backend/features.py

@@ -90,9 +90,6 @@ class DatabaseFeatures(GISFeatures, BaseDatabaseFeatures):
         "auth_tests.test_views.LoginTest.test_login_session_without_hash_session_key",
         # GenericRelation.value_to_string() assumes integer pk.
         "contenttypes_tests.test_fields.GenericRelationTests.test_value_to_string",
-        # icontains doesn't work on ArrayField:
-        # Unsupported conversion from array to string in $convert
-        "model_fields_.test_arrayfield.QueryingTests.test_icontains",
         # ArrayField's contained_by lookup crashes with Exists: "both operands "
         # of $setIsSubset must be arrays. Second argument is of type: null"
         # https://jira.mongodb.org/browse/SERVER-99186