json filtering

Author: NatalyaGrigoreva

fastapi_jsonapi/data_layers/sqla/orm.py

@@ -175,8 +175,7 @@ async def apply_relationships(
         :param action_trigger: indicates which one operation triggered relationships applying
         :return:
         """
-        to_one: dict = {}
-        to_many: dict = {}
+        to_one, to_many = {}, {}
         relationships: BaseModel = data_create.relationships
         if relationships is None:
             return to_one, to_many

fastapi_jsonapi/types_metadata/custom_filter_sql.py

@@ -1,12 +1,21 @@
+import contextlib
+import logging
 from dataclasses import dataclass
-from typing import Generic, TypeVar, Union, cast
+from typing import Any, Generic, TypeVar, Union, cast
+
+import orjson as json
 
 # noinspection PyProtectedMember
 from pydantic.fields import FieldInfo
 from sqlalchemy import func
+from sqlalchemy.dialects.postgresql import JSONB as JSONB_SQLA
 from sqlalchemy.orm import InstrumentedAttribute
 from sqlalchemy.sql.expression import BinaryExpression, BooleanClauseList
 
+from fastapi_jsonapi.exceptions import InvalidFilters
+
+log = logging.getLogger(__name__)
+
 ColumnType = TypeVar("ColumnType")
 ExpressionType = TypeVar("ExpressionType")
 
@@ -29,6 +38,68 @@ class CustomFilterSQLA(CustomFilterSQL[InstrumentedAttribute, Union[BinaryExpres
     """Base class for custom SQLAlchemy filters"""
 
 
+def _get_pg_jsonb_contains_expression(
+    model_column: InstrumentedAttribute,
+    value: Any,
+) -> BinaryExpression:
+    with contextlib.suppress(ValueError):
+        value = json.loads(value)
+
+    return model_column.cast(JSONB_SQLA).op("@>")(value)
+
+
+def _get_sqlite_json_contains_expression(
+    model_column: InstrumentedAttribute,
+    value: Any,
+) -> BinaryExpression:
+    with contextlib.suppress(ValueError):
+        value = json.loads(value)
+
+    return model_column.ilike(value)
+
+
+def _get_pg_jsonb_ilike_expression(
+    model_column: InstrumentedAttribute,
+    value: list,
+    operator: str,
+) -> BinaryExpression:
+    try:
+        target_field, regex = value
+    except ValueError:
+        msg = f'The "value" field has to be list of two values for op `{operator}`'
+        raise InvalidFilters(msg)
+
+    if isinstance(regex, (list, dict)):
+        return model_column[target_field].cast(JSONB_SQLA).op("@>")(regex)
+    elif isinstance(regex, bool):
+        regex = f"{regex}".lower()
+    else:
+        regex = f"{regex}"
+
+    return model_column.op("->>")(target_field).ilike(regex)
+
+
+def _get_sqlite_json_ilike_expression(
+    model_column: InstrumentedAttribute,
+    value: list,
+    operator: str,
+) -> BinaryExpression:
+    try:
+        target_field, regex = value
+    except ValueError:
+        msg = f'The "value" field has to be list of two values for op `{operator}`'
+        raise InvalidFilters(msg)
+
+    if isinstance(regex, (list, dict)):
+        regex = json.dumps(regex).decode()
+    elif isinstance(regex, bool):
+        return model_column.op("->>")(target_field).is_(regex)
+    else:
+        regex = f"{regex}"
+
+    return model_column.op("->>")(target_field).ilike(regex)
+
+
 class LowerEqualsFilterSQL(CustomFilterSQLA):
     def get_expression(
         self,
@@ -43,17 +114,76 @@ def get_expression(
         )
 
 
-# TODO: tests coverage
-class JSONBContainsFilterSQL(CustomFilterSQLA):
+class PGJSONContainsFilterSQL(CustomFilterSQLA):
     def get_expression(
         self,
         schema_field: FieldInfo,
         model_column: InstrumentedAttribute,
-        value: str,
+        value: Any,
+        operator: str,
+    ) -> BinaryExpression:
+        return _get_pg_jsonb_contains_expression(model_column, value)
+
+
+class PGJSONBContainsFilterSQL(CustomFilterSQLA):
+    def get_expression(
+        self,
+        schema_field: FieldInfo,
+        model_column: InstrumentedAttribute,
+        value: Any,
+        operator: str,
+    ) -> BinaryExpression:
+        return _get_pg_jsonb_contains_expression(model_column, value)
+
+
+class PGJSONIlikeFilterSQL(CustomFilterSQLA):
+    def get_expression(
+        self,
+        schema_field: FieldInfo,
+        model_column: InstrumentedAttribute,
+        value: list[str],
+        operator: str,
+    ) -> BinaryExpression:
+        return _get_pg_jsonb_ilike_expression(model_column, value, operator)
+
+
+class PGJSONBIlikeFilterSQL(CustomFilterSQLA):
+    def get_expression(
+        self,
+        schema_field: FieldInfo,
+        model_column: InstrumentedAttribute,
+        value: list[str],
+        operator: str,
+    ) -> BinaryExpression:
+        return _get_pg_jsonb_ilike_expression(model_column, value, operator)
+
+
+class SQLiteJSONContainsFilterSQL(CustomFilterSQLA):
+    def get_expression(
+        self,
+        schema_field: FieldInfo,
+        model_column: InstrumentedAttribute,
+        value: Any,
+        operator: str,
+    ) -> BinaryExpression:
+        return _get_sqlite_json_contains_expression(model_column, value)
+
+
+class SQLiteJSONIlikeFilterSQL(CustomFilterSQLA):
+    def get_expression(
+        self,
+        schema_field: FieldInfo,
+        model_column: InstrumentedAttribute,
+        value: list[str],
         operator: str,
     ) -> BinaryExpression:
-        return model_column.op("@>")(value)
+        return _get_sqlite_json_ilike_expression(model_column, value, operator)
 
 
 sql_filter_lower_equals = LowerEqualsFilterSQL(op="lower_equals")
-sql_filter_jsonb_contains = JSONBContainsFilterSQL(op="jsonb_contains")
+sql_filter_pg_json_contains = PGJSONContainsFilterSQL(op="pg_json_contains")
+sql_filter_pg_jsonb_contains = PGJSONBContainsFilterSQL(op="pg_jsonb_contains")
+sql_filter_pg_json_ilike = PGJSONIlikeFilterSQL(op="pg_json_ilike")
+sql_filter_pg_jsonb_ilike = PGJSONBIlikeFilterSQL(op="pg_jsonb_ilike")
+sql_filter_sqlite_json_contains = SQLiteJSONContainsFilterSQL(op="sqlite_json_contains")
+sql_filter_sqlite_json_ilike = SQLiteJSONIlikeFilterSQL(op="sqlite_json_ilike")

tests/conftest.py

@@ -35,6 +35,8 @@
     parent_1,
     parent_2,
     parent_3,
+    task_1,
+    task_2,
     user_1,
     user_1_bio,
     user_1_comments_for_u2_posts,

tests/fixtures/entities.py

@@ -15,6 +15,8 @@
     UserBio,
     Workplace,
 )
+from tests.common import is_postgres_tests
+from tests.fixtures.models import Task
 from tests.misc.utils import fake
 
 
@@ -47,11 +49,7 @@ async def create_user(async_session: AsyncSession, **fields) -> User:
 
 
 async def create_user_bio(async_session: AsyncSession, user: User, **fields) -> UserBio:
-    fields = {
-        "user": user,
-        **fields,
-    }
-    user_bio = build_user_bio(**fields)
+    user_bio = build_user_bio(user=user, **fields)
     async_session.add(user_bio)
     await async_session.commit()
     return user_bio
@@ -120,16 +118,17 @@ async def user_2_bio(async_session: AsyncSession, user_2: User) -> UserBio:
     )
 
 
-async def build_post(async_session: AsyncSession, user: User, **fields) -> Post:
+def build_post(user: User, **fields) -> Post:
     fields = {
         "title": fake.name(),
         "body": fake.sentence(),
         **fields,
     }
-    post = Post(
-        user=user,
-        **fields,
-    )
+    return Post(user=user, **fields)
+
+
+async def create_post(async_session: AsyncSession, user: User, **fields) -> Post:
+    post = build_post(user, **fields)
     async_session.add(post)
     await async_session.commit()
     return post
@@ -244,21 +243,25 @@ async def factory(name: Optional[str] = None) -> Computer:
     return factory
 
 
-async def build_post_comment(
-    async_session: AsyncSession,
-    user: User,
-    post: Post,
-    **fields,
-) -> PostComment:
+def build_post_comment(user: User, post: Post, **fields) -> PostComment:
     fields = {
         "text": fake.sentence(),
         **fields,
     }
-    post_comment = PostComment(
+    return PostComment(
         user=user,
         post=post,
         **fields,
     )
+
+
+async def create_post_comment(
+    async_session: AsyncSession,
+    user: User,
+    post: Post,
+    **fields,
+) -> PostComment:
+    post_comment = build_post_comment(user=user, post=post, **fields)
     async_session.add(post_comment)
     await async_session.commit()
     return post_comment
@@ -478,8 +481,59 @@ async def p2_c3_association(
     await async_session.commit()
 
 
-async def build_workplace(async_session: AsyncSession, **fields):
-    workplace = Workplace(**fields)
+def build_task(**fields):
+    return Task(**fields)
+
+
+async def create_task(async_session: AsyncSession, **fields):
+    task = build_task(**fields)
+    async_session.add(task)
+    await async_session.commit()
+    return task
+
+
+@async_fixture()
+async def task_1(
+    async_session: AsyncSession,
+):
+    fields = {
+        "task_ids_list_json": [1, 2, 3],
+        "task_ids_dict_json": {"completed": [1, 2, 3], "count": 1, "is_complete": True},
+    }
+    if is_postgres_tests():
+        fields.update(
+            {
+                "task_ids_list_jsonb": ["a", "b", "c"],
+                "task_ids_dict_jsonb": {"completed": ["a", "b", "c"], "count": 2, "is_complete": True},
+            },
+        )
+    yield await create_task(async_session, **fields)
+
+
+@async_fixture()
+async def task_2(
+    async_session: AsyncSession,
+):
+    fields = {
+        "task_ids_list_json": [4, 5, 6],
+        "task_ids_dict_json": {"completed": [4, 5, 6], "count": 3, "is_complete": False},
+    }
+    if is_postgres_tests():
+        fields.update(
+            {
+                "task_ids_list_jsonb": ["d", "e", "f"],
+                "task_ids_dict_jsonb": {"completed": ["d", "e", "f"], "count": 4, "is_complete": False},
+            },
+        )
+    yield await create_task(async_session, **fields)
+
+
+def build_workplace(**fields):
+    return Workplace(**fields)
+
+
+async def create_workplace(async_session: AsyncSession, **fields):
+    workplace = build_workplace(**fields)
     async_session.add(workplace)
     await async_session.commit()
     return workplace
@@ -489,11 +543,11 @@ async def build_workplace(async_session: AsyncSession, **fields):
 async def workplace_1(
     async_session: AsyncSession,
 ):
-    yield await build_workplace(async_session, name="workplace_1")
+    yield await create_workplace(async_session, name="workplace_1")
 
 
 @async_fixture()
 async def workplace_2(
     async_session: AsyncSession,
 ):
-    yield await build_workplace(async_session, name="workplace_2")
+    yield await create_workplace(async_session, name="workplace_2")

tests/fixtures/models/task.py

@@ -1,13 +1,19 @@
 from typing import Optional
 
 from sqlalchemy import JSON
+from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.orm import Mapped, mapped_column
 
 from examples.api_for_sqlalchemy.models.base import Base
+from tests.common import is_postgres_tests
 
 
 class Task(Base):
     __tablename__ = "tasks"
 
-    task_ids_dict: Mapped[Optional[dict]] = mapped_column(JSON, unique=False)
-    task_ids_list: Mapped[Optional[list]] = mapped_column(JSON, unique=False)
+    task_ids_dict_json: Mapped[Optional[dict]] = mapped_column(JSON, unique=False)
+    task_ids_list_json: Mapped[Optional[list]] = mapped_column(JSON, unique=False)
+
+    if is_postgres_tests():
+        task_ids_dict_jsonb: Mapped[Optional[dict]] = mapped_column(JSONB, unique=False)
+        task_ids_list_jsonb: Mapped[Optional[list]] = mapped_column(JSONB, unique=False)