From c356c1be0fde7a8c98ca6f2c32c7568985f0cf86 Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Wed, 25 Jun 2025 15:20:54 -0400 Subject: [PATCH 01/16] INTPYTHON-527 Add Queryable Encryption support --- .github/workflows/encrypted_settings.py | 40 +++ .github/workflows/mongodb_settings.py | 3 +- .github/workflows/test-python-atlas.yml | 26 +- django_mongodb_backend/__init__.py | 2 + django_mongodb_backend/base.py | 17 +- django_mongodb_backend/creation.py | 16 +- django_mongodb_backend/features.py | 29 +- django_mongodb_backend/fields/__init__.py | 52 +++ django_mongodb_backend/fields/encryption.py | 130 +++++++ .../commands/showencryptedfieldsmap.py | 35 ++ django_mongodb_backend/routers.py | 25 +- django_mongodb_backend/schema.py | 94 ++++- django_mongodb_backend/utils.py | 20 ++ docs/howto/index.rst | 1 + docs/howto/queryable-encryption.rst | 335 ++++++++++++++++++ docs/index.rst | 2 + docs/ref/django-admin.rst | 23 ++ docs/ref/index.rst | 2 + docs/ref/models/encrypted-fields.rst | 128 +++++++ docs/ref/settings.rst | 45 +++ docs/ref/utils.rst | 39 ++ docs/releases/5.2.x.rst | 1 + docs/topics/index.rst | 1 + docs/topics/known-issues.rst | 2 + docs/topics/queryable-encryption.rst | 136 +++++++ pyproject.toml | 1 + tests/backend_/test_features.py | 80 +++++ tests/encryption_/__init__.py | 0 tests/encryption_/models.py | 167 +++++++++ tests/encryption_/test_base.py | 21 ++ tests/encryption_/test_fields.py | 251 +++++++++++++ tests/encryption_/test_management.py | 111 ++++++ tests/encryption_/test_schema.py | 140 ++++++++ tests/raw_query_/test_raw_aggregate.py | 2 +- 34 files changed, 1966 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/encrypted_settings.py create mode 100644 django_mongodb_backend/fields/encryption.py create mode 100644 django_mongodb_backend/management/commands/showencryptedfieldsmap.py create mode 100644 docs/howto/queryable-encryption.rst create mode 100644 docs/ref/models/encrypted-fields.rst create mode 100644 docs/ref/settings.rst create mode 100644 docs/topics/queryable-encryption.rst create mode 100644 tests/encryption_/__init__.py create mode 100644 tests/encryption_/models.py create mode 100644 tests/encryption_/test_base.py create mode 100644 tests/encryption_/test_fields.py create mode 100644 tests/encryption_/test_management.py create mode 100644 tests/encryption_/test_schema.py diff --git a/.github/workflows/encrypted_settings.py b/.github/workflows/encrypted_settings.py new file mode 100644 index 000000000..02dc3447d --- /dev/null +++ b/.github/workflows/encrypted_settings.py @@ -0,0 +1,40 @@ +# Settings for django_mongodb_backend/tests when encryption is supported. +import os + +from mongodb_settings import * # noqa: F403 +from pymongo.encryption import AutoEncryptionOpts + +DATABASES["encrypted"] = { # noqa: F405 + "ENGINE": "django_mongodb_backend", + "NAME": "djangotests_encrypted", + "OPTIONS": { + "auto_encryption_opts": AutoEncryptionOpts( + key_vault_namespace="djangotests_encrypted.__keyVault", + kms_providers={"local": {"key": os.urandom(96)}}, + ), + "directConnection": True, + }, + "KMS_CREDENTIALS": {}, +} + + +class EncryptedRouter: + def db_for_read(self, model, **hints): + if model._meta.app_label == "encryption_": + return "encrypted" + return None + + db_for_write = db_for_read + + def allow_migrate(self, db, app_label, model_name=None, **hints): + # The encryption_ app's models are only created in the encrypted + # database. + if app_label == "encryption_": + return db == "encrypted" + # Don't create other app's models in the encrypted database. + if db == "encrypted": + return False + return None + + +DATABASE_ROUTERS.append(EncryptedRouter()) # noqa: F405 diff --git a/.github/workflows/mongodb_settings.py b/.github/workflows/mongodb_settings.py index 4dce3c0d5..619bdcd95 100644 --- a/.github/workflows/mongodb_settings.py +++ b/.github/workflows/mongodb_settings.py @@ -1,4 +1,5 @@ -# Settings for django_mongodb_backend/tests. +# Settings for django_mongodb_backend/tests when encryption isn't supported. from django_settings import * # noqa: F403 +DATABASES["encrypted"] = {} # noqa: F405 DATABASE_ROUTERS = ["django_mongodb_backend.routers.MongoRouter"] diff --git a/.github/workflows/test-python-atlas.yml b/.github/workflows/test-python-atlas.yml index e98d2512d..359a6b90b 100644 --- a/.github/workflows/test-python-atlas.yml +++ b/.github/workflows/test-python-atlas.yml @@ -28,7 +28,7 @@ jobs: - name: install django-mongodb-backend run: | pip3 install --upgrade pip - pip3 install -e . + pip3 install -e .[encryption] - name: Checkout Django uses: actions/checkout@v5 with: @@ -51,8 +51,30 @@ jobs: run: cp .github/workflows/runtests.py django_repo/tests/runtests_.py - name: Start local Atlas working-directory: . - run: bash .github/workflows/start_local_atlas.sh mongodb/mongodb-atlas-local:7 + run: bash .github/workflows/start_local_atlas.sh mongodb/mongodb-atlas-local:8.0.15 + - name: Install mongosh + run: | + wget -q https://downloads.mongodb.com/compass/mongosh-2.2.10-linux-x64.tgz + tar -xzf mongosh-*-linux-x64.tgz + sudo cp mongosh-*-linux-x64/bin/mongosh /usr/local/bin/ + mongosh --version + - name: Install mongocryptd from Enterprise tarball + run: | + curl -sSL -o mongodb-enterprise.tgz "https://downloads.mongodb.com/linux/mongodb-linux-x86_64-enterprise-ubuntu2204-8.0.15.tgz" + tar -xzf mongodb-enterprise.tgz + sudo cp mongodb-linux-x86_64-enterprise-ubuntu2204-8.0.15/bin/mongocryptd /usr/local/bin/ + - name: Start mongocryptd + run: | + nohup mongocryptd --logpath=/tmp/mongocryptd.log & + - name: Verify MongoDB installation + run: | + mongosh --eval 'db.runCommand({ connectionStatus: 1 })' + - name: Verify mongocryptd is running + run: | + pgrep mongocryptd - name: Run tests run: python3 django_repo/tests/runtests_.py permissions: contents: read + env: + DJANGO_SETTINGS_MODULE: "encrypted_settings" diff --git a/django_mongodb_backend/__init__.py b/django_mongodb_backend/__init__.py index 577a4f104..752d72802 100644 --- a/django_mongodb_backend/__init__.py +++ b/django_mongodb_backend/__init__.py @@ -14,6 +14,7 @@ from .indexes import register_indexes # noqa: E402 from .lookups import register_lookups # noqa: E402 from .query import register_nodes # noqa: E402 +from .routers import register_routers # noqa: E402 __all__ = ["parse_uri"] @@ -25,3 +26,4 @@ register_indexes() register_lookups() register_nodes() +register_routers() diff --git a/django_mongodb_backend/base.py b/django_mongodb_backend/base.py index 88c2a1189..1284d9e0f 100644 --- a/django_mongodb_backend/base.py +++ b/django_mongodb_backend/base.py @@ -11,6 +11,7 @@ from django.utils.functional import cached_property from pymongo.collection import Collection from pymongo.driver_info import DriverInfo +from pymongo.encryption import ClientEncryption from pymongo.mongo_client import MongoClient from pymongo.uri_parser import parse_uri @@ -241,6 +242,16 @@ def get_database(self): return OperationDebugWrapper(self) return self.database + @cached_property + def client_encryption(self): + auto_encryption_opts = self.connection._options.auto_encryption_opts + return ClientEncryption( + auto_encryption_opts._kms_providers, + auto_encryption_opts._key_vault_namespace, + self.connection, + self.connection.codec_options, + ) + @cached_property def database(self): """Connect to the database the first time it's accessed.""" @@ -325,7 +336,11 @@ def cursor(self): def get_database_version(self): """Return a tuple of the database's version.""" - return tuple(self.connection.server_info()["versionArray"]) + # TODO: Remove this workaround and replace with + # `tuple(self.connection.server_info()["versionArray"])` when the minimum + # supported version of pymongocrypt is >= 1.14.2 and PYTHON-5429 is resolved. + # See: https://jira.mongodb.org/browse/PYTHON-5429 + return tuple(self.connection.admin.command("buildInfo")["versionArray"]) ## Transaction API for django_mongodb_backend.transaction.atomic() @async_unsafe diff --git a/django_mongodb_backend/creation.py b/django_mongodb_backend/creation.py index c8002b2c4..a1d45277e 100644 --- a/django_mongodb_backend/creation.py +++ b/django_mongodb_backend/creation.py @@ -1,5 +1,5 @@ from django.conf import settings -from django.db.backends.base.creation import BaseDatabaseCreation +from django.db.backends.base.creation import TEST_DATABASE_PREFIX, BaseDatabaseCreation class DatabaseCreation(BaseDatabaseCreation): @@ -7,6 +7,14 @@ def _execute_create_test_db(self, cursor, parameters, keepdb=False): # Close the connection (which may point to the non-test database) so # that a new connection to the test database can be established later. self.connection.close_pool() + # Use a test _key_vault_namespace. This assumes the key vault database + # is the same as the encrypted database so that _destroy_test_db() can + # reset the collection by dropping it. + opts = self.connection.settings_dict["OPTIONS"].get("auto_encryption_opts") + if opts: + self.connection.settings_dict["OPTIONS"][ + "auto_encryption_opts" + ]._key_vault_namespace = TEST_DATABASE_PREFIX + opts._key_vault_namespace if not keepdb: self._destroy_test_db(parameters["dbname"], verbosity=0) @@ -24,3 +32,9 @@ def destroy_test_db(self, old_database_name=None, verbosity=1, keepdb=False, suf super().destroy_test_db(old_database_name, verbosity, keepdb, suffix) # Close the connection to the test database. self.connection.close_pool() + # Restore the original _key_vault_namespace. + opts = self.connection.settings_dict["OPTIONS"].get("auto_encryption_opts") + if opts: + self.connection.settings_dict["OPTIONS"][ + "auto_encryption_opts" + ]._key_vault_namespace = opts._key_vault_namespace[len(TEST_DATABASE_PREFIX) :] diff --git a/django_mongodb_backend/features.py b/django_mongodb_backend/features.py index 18a048bf6..6f4c1e8f5 100644 --- a/django_mongodb_backend/features.py +++ b/django_mongodb_backend/features.py @@ -588,9 +588,21 @@ def django_test_skips(self): skips.update(self._django_test_skips) return skips + @cached_property + def mongodb_version(self): + return self.connection.get_database_version() # e.g., (6, 3, 0) + @cached_property def is_mongodb_6_3(self): - return self.connection.get_database_version() >= (6, 3) + return self.mongodb_version >= (6, 3) + + @cached_property + def is_mongodb_7_0(self): + return self.mongodb_version >= (7, 0) + + @cached_property + def is_mongodb_8_0(self): + return self.mongodb_version >= (8, 0) @cached_property def supports_atlas_search(self): @@ -620,3 +632,18 @@ def _supports_transactions(self): hello = client.command("hello") # a replica set or a sharded cluster return "setName" in hello or hello.get("msg") == "isdbgrid" + + @cached_property + def supports_queryable_encryption(self): + """ + Queryable Encryption requires a MongoDB 8.0 or later replica set or sharded + cluster, as well as MongoDB Atlas or Enterprise. + """ + self.connection.ensure_connection() + build_info = self.connection.connection.admin.command("buildInfo") + is_enterprise = "enterprise" in build_info.get("modules") + return ( + (is_enterprise or self.supports_atlas_search) + and self._supports_transactions + and self.is_mongodb_8_0 + ) diff --git a/django_mongodb_backend/fields/__init__.py b/django_mongodb_backend/fields/__init__.py index 0c95afd69..6cc4bcc18 100644 --- a/django_mongodb_backend/fields/__init__.py +++ b/django_mongodb_backend/fields/__init__.py @@ -3,6 +3,33 @@ from .duration import register_duration_field from .embedded_model import EmbeddedModelField from .embedded_model_array import EmbeddedModelArrayField +from .encryption import ( + EncryptedArrayField, + EncryptedBigIntegerField, + EncryptedBinaryField, + EncryptedBooleanField, + EncryptedCharField, + EncryptedDateField, + EncryptedDateTimeField, + EncryptedDecimalField, + EncryptedDurationField, + EncryptedEmailField, + EncryptedEmbeddedModelArrayField, + EncryptedEmbeddedModelField, + EncryptedFieldMixin, + EncryptedFloatField, + EncryptedGenericIPAddressField, + EncryptedIntegerField, + EncryptedObjectIdField, + EncryptedPositiveBigIntegerField, + EncryptedPositiveIntegerField, + EncryptedPositiveSmallIntegerField, + EncryptedSmallIntegerField, + EncryptedTextField, + EncryptedTimeField, + EncryptedURLField, + EncryptedUUIDField, +) from .json import register_json_field from .objectid import ObjectIdField from .polymorphic_embedded_model import PolymorphicEmbeddedModelField @@ -12,6 +39,31 @@ "ArrayField", "EmbeddedModelArrayField", "EmbeddedModelField", + "EncryptedArrayField", + "EncryptedBigIntegerField", + "EncryptedBinaryField", + "EncryptedBooleanField", + "EncryptedCharField", + "EncryptedDateField", + "EncryptedDateTimeField", + "EncryptedDecimalField", + "EncryptedDurationField", + "EncryptedEmailField", + "EncryptedEmbeddedModelArrayField", + "EncryptedEmbeddedModelField", + "EncryptedFieldMixin", + "EncryptedFloatField", + "EncryptedGenericIPAddressField", + "EncryptedIntegerField", + "EncryptedObjectIdField", + "EncryptedPositiveBigIntegerField", + "EncryptedPositiveIntegerField", + "EncryptedPositiveSmallIntegerField", + "EncryptedSmallIntegerField", + "EncryptedTextField", + "EncryptedTimeField", + "EncryptedURLField", + "EncryptedUUIDField", "ObjectIdAutoField", "ObjectIdField", "PolymorphicEmbeddedModelArrayField", diff --git a/django_mongodb_backend/fields/encryption.py b/django_mongodb_backend/fields/encryption.py new file mode 100644 index 000000000..a4bf7769f --- /dev/null +++ b/django_mongodb_backend/fields/encryption.py @@ -0,0 +1,130 @@ +from django.db import models + +from django_mongodb_backend.fields import ArrayField, EmbeddedModelArrayField, EmbeddedModelField +from django_mongodb_backend.fields.objectid import ObjectIdField + + +class EncryptedFieldMixin: + encrypted = True + + def __init__(self, *args, queries=None, db_index=False, null=False, unique=False, **kwargs): + if db_index: + raise ValueError("'db_index=True' is not supported on encrypted fields.") + if null: + raise ValueError("'null=True' is not supported on encrypted fields.") + if unique: + raise ValueError("'unique=True' is not supported on encrypted fields.") + self.queries = queries + super().__init__(*args, **kwargs) + + def deconstruct(self): + name, path, args, kwargs = super().deconstruct() + + if self.queries is not None: + kwargs["queries"] = self.queries + + if path.startswith("django_mongodb_backend.fields.encryption"): + path = path.replace( + "django_mongodb_backend.fields.encryption", + "django_mongodb_backend.fields", + ) + + return name, path, args, kwargs + + +# Django fields +class EncryptedBinaryField(EncryptedFieldMixin, models.BinaryField): + pass + + +class EncryptedBigIntegerField(EncryptedFieldMixin, models.BigIntegerField): + pass + + +class EncryptedBooleanField(EncryptedFieldMixin, models.BooleanField): + pass + + +class EncryptedCharField(EncryptedFieldMixin, models.CharField): + pass + + +class EncryptedDateField(EncryptedFieldMixin, models.DateField): + pass + + +class EncryptedDateTimeField(EncryptedFieldMixin, models.DateTimeField): + pass + + +class EncryptedDecimalField(EncryptedFieldMixin, models.DecimalField): + pass + + +class EncryptedDurationField(EncryptedFieldMixin, models.DurationField): + pass + + +class EncryptedEmailField(EncryptedFieldMixin, models.EmailField): + pass + + +class EncryptedFloatField(EncryptedFieldMixin, models.FloatField): + pass + + +class EncryptedGenericIPAddressField(EncryptedFieldMixin, models.GenericIPAddressField): + pass + + +class EncryptedIntegerField(EncryptedFieldMixin, models.IntegerField): + pass + + +class EncryptedPositiveBigIntegerField(EncryptedFieldMixin, models.PositiveBigIntegerField): + pass + + +class EncryptedPositiveIntegerField(EncryptedFieldMixin, models.PositiveIntegerField): + pass + + +class EncryptedPositiveSmallIntegerField(EncryptedFieldMixin, models.PositiveSmallIntegerField): + pass + + +class EncryptedSmallIntegerField(EncryptedFieldMixin, models.SmallIntegerField): + pass + + +class EncryptedTextField(EncryptedFieldMixin, models.TextField): + pass + + +class EncryptedTimeField(EncryptedFieldMixin, models.TimeField): + pass + + +class EncryptedURLField(EncryptedFieldMixin, models.URLField): + pass + + +class EncryptedUUIDField(EncryptedFieldMixin, models.UUIDField): + pass + + +# MongoDB fields +class EncryptedArrayField(EncryptedFieldMixin, ArrayField): + pass + + +class EncryptedEmbeddedModelArrayField(EncryptedFieldMixin, EmbeddedModelArrayField): + pass + + +class EncryptedEmbeddedModelField(EncryptedFieldMixin, EmbeddedModelField): + pass + + +class EncryptedObjectIdField(EncryptedFieldMixin, ObjectIdField): + pass diff --git a/django_mongodb_backend/management/commands/showencryptedfieldsmap.py b/django_mongodb_backend/management/commands/showencryptedfieldsmap.py new file mode 100644 index 000000000..5cd864c77 --- /dev/null +++ b/django_mongodb_backend/management/commands/showencryptedfieldsmap.py @@ -0,0 +1,35 @@ +from bson import json_util +from django.apps import apps +from django.core.management.base import BaseCommand +from django.db import DEFAULT_DB_ALIAS, connections, router + +from django_mongodb_backend.utils import model_has_encrypted_fields + + +class Command(BaseCommand): + help = """ + Shows the mapping of encrypted fields to field attributes, including data + type, data keys and query types. The output can be used to set + ``encrypted_fields_map`` in ``AutoEncryptionOpts``. + """ + + def add_arguments(self, parser): + parser.add_argument( + "--database", + default=DEFAULT_DB_ALIAS, + help=""" + Specifies the database to use. Defaults to ``default``.""", + ) + + def handle(self, *args, **options): + db = options["database"] + connection = connections[db] + connection.ensure_connection() + encrypted_fields_map = {} + with connection.schema_editor() as editor: + for app_config in apps.get_app_configs(): + for model in router.get_migratable_models(app_config, db): + if model_has_encrypted_fields(model): + fields = editor._get_encrypted_fields(model) + encrypted_fields_map[model._meta.db_table] = fields + self.stdout.write(json_util.dumps(encrypted_fields_map, indent=2)) diff --git a/django_mongodb_backend/routers.py b/django_mongodb_backend/routers.py index 60e54bbd8..b17f4b021 100644 --- a/django_mongodb_backend/routers.py +++ b/django_mongodb_backend/routers.py @@ -1,6 +1,6 @@ from django.apps import apps - -from django_mongodb_backend.models import EmbeddedModel +from django.core.exceptions import ImproperlyConfigured +from django.db.utils import ConnectionRouter class MongoRouter: @@ -9,6 +9,8 @@ def allow_migrate(self, db, app_label, model_name=None, **hints): EmbeddedModels don't have their own collection and must be ignored by dumpdata. """ + from django_mongodb_backend.models import EmbeddedModel # noqa: PLC0415 + if not model_name: return None try: @@ -16,3 +18,22 @@ def allow_migrate(self, db, app_label, model_name=None, **hints): except LookupError: return None return False if issubclass(model, EmbeddedModel) else None + + +# This function is intended to be monkey-patched as a method of ConnectionRouter. +def kms_provider(self, model, *args, **kwargs): + """ + Return the Key Management Service (KMS) provider for a given model. + + Call each router's kms_provider() method (if present), and return the + first non-None result. Raise ImproperlyConfigured if no provider is found. + """ + for router in self.routers: + func = getattr(router, "kms_provider", None) + if func and callable(func) and (result := func(model, *args, **kwargs)): + return result + raise ImproperlyConfigured("No kms_provider found in database routers.") + + +def register_routers(): + ConnectionRouter.kms_provider = kms_provider diff --git a/django_mongodb_backend/schema.py b/django_mongodb_backend/schema.py index 9bcaecc63..3ab83d540 100644 --- a/django_mongodb_backend/schema.py +++ b/django_mongodb_backend/schema.py @@ -1,5 +1,7 @@ from time import monotonic, sleep +from django.core.exceptions import ImproperlyConfigured +from django.db import router from django.db.backends.base.schema import BaseDatabaseSchemaEditor from django.db.models import Index, UniqueConstraint from pymongo.operations import SearchIndexModel @@ -9,7 +11,7 @@ from .fields import EmbeddedModelField from .gis.schema import GISSchemaEditor from .query import wrap_database_errors -from .utils import OperationCollector +from .utils import OperationCollector, model_has_encrypted_fields def ignore_embedded_models(func): @@ -44,7 +46,7 @@ def get_database(self): @wrap_database_errors @ignore_embedded_models def create_model(self, model): - self.get_database().create_collection(model._meta.db_table) + self._create_collection(model) self._create_model_indexes(model) # Make implicit M2M tables. for field in model._meta.local_many_to_many: @@ -452,6 +454,94 @@ def wait_until_index_dropped(collection, index_name, timeout=60, interval=0.5): sleep(interval) raise TimeoutError(f"Index {index_name} not dropped after {timeout} seconds.") + def _create_collection(self, model): + """ + Create a collection for the model. + If the model has encrypted fields, build (or retrieve) the encrypted_fields schema. + """ + db = self.get_database() + db_table = model._meta.db_table + + if model_has_encrypted_fields(model): + # Encrypted path + client = self.connection.connection + auto_encryption_opts = getattr(client._options, "auto_encryption_opts", None) + if not auto_encryption_opts: + raise ImproperlyConfigured( + f"Encrypted fields found but DATABASES['{self.connection.alias}']['OPTIONS'] " + "is missing auto_encryption_opts." + ) + encrypted_fields = self._get_encrypted_fields(model) + db.create_collection(db_table, encryptedFields=encrypted_fields) + else: + # Unencrypted path + db.create_collection(db_table) + + def _get_encrypted_fields(self, model, key_alt_name=None, path_prefix=None): + """ + Recursively collect encryption schema data for only encrypted fields in a model. + Returns None if no encrypted fields are found anywhere in the model hierarchy. + """ + connection = self.connection + client = connection.connection + fields = model._meta.fields + key_alt_name = key_alt_name or model._meta.db_table + path_prefix = path_prefix or "" + + options = client._options + auto_encryption_opts = options.auto_encryption_opts + + key_vault_db, key_vault_coll = auto_encryption_opts._key_vault_namespace.split(".", 1) + key_vault_collection = client[key_vault_db][key_vault_coll] + + # Create partial unique index on keyAltNames + key_vault_collection.create_index( + "keyAltNames", unique=True, partialFilterExpression={"keyAltNames": {"$exists": True}} + ) + + kms_provider = router.kms_provider(model) + master_key = connection.settings_dict.get("KMS_CREDENTIALS", {}).get(kms_provider) + client_encryption = self.connection.client_encryption + + field_list = [] + + for field in fields: + new_key_alt_name = f"{key_alt_name}.{field.column}" + path = f"{path_prefix}.{field.column}" if path_prefix else field.column + + if isinstance(field, EmbeddedModelField) and not getattr(field, "encrypted", False): + embedded_result = self._get_encrypted_fields( + field.embedded_model, + key_alt_name=new_key_alt_name, + path_prefix=path, + ) + if embedded_result: + field_list.extend(embedded_result["fields"]) + continue + + if getattr(field, "encrypted", False): + bson_type = field.db_type(connection) + data_key = key_vault_collection.find_one({"keyAltNames": new_key_alt_name}) + if data_key: + data_key = data_key["_id"] + else: + data_key = client_encryption.create_data_key( + kms_provider=kms_provider, + master_key=master_key, + key_alt_names=[new_key_alt_name], + ) + field_dict = { + "bsonType": bson_type, + "path": path, + "keyId": data_key, + } + queries = getattr(field, "queries", None) + if queries: + field_dict["queries"] = queries + field_list.append(field_dict) + + return {"fields": field_list} if field_list else None + # GISSchemaEditor extends some SchemaEditor methods. class DatabaseSchemaEditor(GISSchemaEditor, BaseSchemaEditor): diff --git a/django_mongodb_backend/utils.py b/django_mongodb_backend/utils.py index 0240250cf..2afaaba0e 100644 --- a/django_mongodb_backend/utils.py +++ b/django_mongodb_backend/utils.py @@ -193,3 +193,23 @@ def wrapper(self, *args, **kwargs): self.log(method, args, kwargs) return wrapper + + +def model_has_encrypted_fields(model): + """ + Recursively check if this model or any embedded models contain encrypted fields. + Returns True if encryption is found anywhere in the hierarchy. + """ + from django_mongodb_backend.fields import EmbeddedModelField # noqa: PLC0415 + + for field in model._meta.fields: + if getattr(field, "encrypted", False): + return True + + # Recursively check embedded models. + if isinstance(field, EmbeddedModelField) and model_has_encrypted_fields( + field.embedded_model + ): + return True + + return False diff --git a/docs/howto/index.rst b/docs/howto/index.rst index 95d7ef632..8451960ef 100644 --- a/docs/howto/index.rst +++ b/docs/howto/index.rst @@ -11,3 +11,4 @@ Project configuration :maxdepth: 1 contrib-apps + queryable-encryption diff --git a/docs/howto/queryable-encryption.rst b/docs/howto/queryable-encryption.rst new file mode 100644 index 000000000..69d83fd09 --- /dev/null +++ b/docs/howto/queryable-encryption.rst @@ -0,0 +1,335 @@ +================================ +Configuring Queryable Encryption +================================ + +.. versionadded:: 5.2.3 + +:doc:`manual:core/queryable-encryption` is a powerful MongoDB feature that +allows you to encrypt sensitive fields in your database while still supporting +queries on that encrypted data. + +This section will guide you through the process of configuring Queryable +Encryption in your Django project. + +.. admonition:: MongoDB requirements + + Queryable Encryption can be used with MongoDB replica sets or sharded + clusters running version 8.0 or later. Standalone instances are not + supported. The following table summarizes which MongoDB server products + support each Queryable Encryption mechanism. + + - :ref:`manual:qe-compatibility-reference` + +Installation +============ + +In addition to the :doc:`installation ` and :doc:`configuration +` steps for Django MongoDB Backend, Queryable +Encryption requires encryption support and a Key Management Service (KMS). + +You can install encryption support with the following command:: + + pip install django-mongodb-backend[encryption] + +.. _qe-configuring-databases-setting: + +Configuring the ``DATABASES`` setting +===================================== + +In addition to :ref:`configuring-databases-setting`, you must also configure an +encrypted database in your :setting:`django:DATABASES` setting. + +This database will be used to store encrypted fields in your models. The +following example shows how to configure an encrypted database using the +:class:`AutoEncryptionOpts ` from the +:mod:`encryption_options ` module. + +This example uses a local KMS provider and a key vault namespace for storing +encryption keys. + +.. code-block:: python + + import os + + from pymongo.encryption_options import AutoEncryptionOpts + + DATABASES = { + "default": { + "ENGINE": "django_mongodb_backend", + "HOST": "mongodb+srv://cluster0.example.mongodb.net", + "NAME": "my_database", + "USER": "my_user", + "PASSWORD": "my_password", + "PORT": 27017, + "OPTIONS": { + "retryWrites": "true", + "w": "majority", + "tls": "false", + }, + }, + "encrypted": { + "ENGINE": "django_mongodb_backend", + "HOST": "mongodb+srv://cluster0.example.mongodb.net", + "NAME": "encrypted", + "USER": "my_user", + "PASSWORD": "my_password", + "PORT": 27017, + "OPTIONS": { + "auto_encryption_opts": AutoEncryptionOpts( + key_vault_namespace="encrypted.keyvault", + kms_providers={"local": {"key": os.urandom(96)}}, + ) + }, + }, + } + +.. _qe-configuring-database-routers-setting: + +Configuring the ``DATABASE_ROUTERS`` setting +============================================ + +Similar to :ref:`configuring-database-routers-setting` for using :doc:`embedded +models `, to use Queryable Encryption you must also +configure the :setting:`django:DATABASE_ROUTERS` setting to route queries to the +encrypted database. + +This is done by adding a custom router that routes queries to the encrypted +database based on the model's metadata. The following example shows how to +configure a custom router for Queryable Encryption: + +.. code-block:: python + + class EncryptedRouter: + """ + A router for routing queries to the encrypted database for Queryable + Encryption. + """ + + def db_for_read(self, model, **hints): + if model._meta.app_label == "myapp": + return "encrypted" + return None + + db_for_write = db_for_read + + def allow_migrate(self, db, app_label, model_name=None, **hints): + if app_label == "myapp": + return db == "encrypted" + # Don't create other app's models in the encrypted database. + if db == "encrypted": + return False + return None + + def kms_provider(self, model, **hints): + return "local" + + + DATABASE_ROUTERS = [EncryptedRouter] + +.. _qe-configuring-kms: + +Configuring the Key Management Service (KMS) +============================================ + +To use Queryable Encryption, you must configure a Key Management Service (KMS). +The KMS is responsible for managing the encryption keys used to encrypt and +decrypt data. The following table summarizes the available KMS configuration +options followed by an example of how to use them. + ++-------------------------------------------------------------------------+--------------------------------------------------------+ +| :setting:`KMS_CREDENTIALS ` | A dictionary of Key Management Service (KMS) | +| | credentials configured in the | +| | :setting:`django:DATABASES` setting. | ++-------------------------------------------------------------------------+--------------------------------------------------------+ +| :class:`kms_providers ` | A dictionary of KMS provider credentials used to | +| | access the KMS with | +| | :setting:`KMS_CREDENTIALS `. | ++-------------------------------------------------------------------------+--------------------------------------------------------+ +| ``kms_provider`` | A single KMS provider name | +| | configured in your custom database | +| | router. | ++-------------------------------------------------------------------------+--------------------------------------------------------+ + +Example of KMS configuration with AWS KMS: + +.. code-block:: python + + from pymongo.encryption_options import AutoEncryptionOpts + + DATABASES = { + "encrypted": { + "ENGINE": "django_mongodb_backend", + "HOST": "mongodb+srv://cluster0.example.mongodb.net", + "NAME": "encrypted", + "USER": "my_user", + "PASSWORD": "my_password", + "PORT": 27017, + "OPTIONS": { + "auto_encryption_opts": AutoEncryptionOpts( + key_vault_namespace="encrypted.keyvault", + kms_providers={ + "aws": { + "accessKeyId": "your-access-key-id", + "secretAccessKey": "your-secret-access-key", + } + }, + ) + }, + "KMS_CREDENTIALS": { + "aws": { + "key": os.getenv("AWS_KEY_ARN", ""), + "region": os.getenv("AWS_KEY_REGION", ""), + }, + }, + }, + } + + + class EncryptedRouter: + # ... + def kms_provider(self, model, **hints): + return "aws" + +.. _qe-configuring-encrypted-fields-map: + +Configuring the ``encrypted_fields_map`` +======================================== + +When you :ref:`configure an encrypted database connection +` without specifying an +``encrypted_fields_map``, Django MongoDB Backend will create encrypted +collections for you when you run ``python manage.py migrate``. + +Encryption keys for encrypted fields are stored in the key vault +:ref:`specified in the Django settings `. To see the keys +created by Django MongoDB Backend, along with the entire schema, you can run the +:djadmin:`showencryptedfieldsmap` command:: + + $ python manage.py showencryptedfieldsmap --database encrypted + +Use the output of the :djadmin:`showencryptedfieldsmap` command to set the +``encrypted_fields_map`` in +:class:`pymongo.encryption_options.AutoEncryptionOpts` in your Django settings. + +.. code-block:: python + + from pymongo.encryption_options import AutoEncryptionOpts + from bson import json_util + + DATABASES = { + "encrypted": { + "ENGINE": "django_mongodb_backend", + "HOST": "mongodb+srv://cluster0.example.mongodb.net", + "NAME": "encrypted", + "USER": "my_user", + "PASSWORD": "my_password", + "PORT": 27017, + "OPTIONS": { + "auto_encryption_opts": AutoEncryptionOpts( + key_vault_namespace="encrypted.keyvault", + kms_providers={ + "aws": { + "accessKeyId": "your-access-key-id", + "secretAccessKey": "your-secret-access-key", + } + }, + encrypted_fields_map=json_util.loads( + """{ + "encrypt_patient": { + "fields": [ + { + "bsonType": "string", + "path": "patient_record.ssn", + "keyId": { + "$binary": { + "base64": "2MA29LaARIOqymYHGmi2mQ==", + "subType": "04" + } + }, + "queries": { + "queryType": "equality" + } + }, + ] + } + }""" + ), + ) + }, + }, + } + +Configuring the Automatic Encryption Shared Library +=================================================== + +The :ref:`manual:qe-reference-shared-library` is a preferred alternative to +:ref:`manual:qe-mongocryptd` and does not require you to start another process +to perform automatic encryption. + +In practice, if you use Atlas or Enterprise MongoDB, ``mongocryptd`` is already +configured for you, however in such cases the shared library is still +recommended for use with Queryable Encryption. + +You can :ref:`download the shared library +` from the +:ref:`manual:enterprise-official-packages` and configure it in your Django +settings using the ``crypt_shared_lib_path`` option in +:class:`pymongo.encryption_options.AutoEncryptionOpts`. The following example +shows how to configure the shared library in your Django settings: + +.. code-block:: python + + from pymongo.encryption_options import AutoEncryptionOpts + + DATABASES = { + "encrypted": { + "ENGINE": "django_mongodb_backend", + "HOST": "mongodb+srv://cluster0.example.mongodb.net", + "NAME": "encrypted", + "USER": "my_user", + "PASSWORD": "my_password", + "PORT": 27017, + "OPTIONS": { + "auto_encryption_opts": AutoEncryptionOpts( + key_vault_namespace="encrypted.keyvault", + kms_providers={ + "aws": { + "accessKeyId": "your-access-key-id", + "secretAccessKey": "your-secret-access-key", + } + }, + encrypted_fields_map=json_util.loads( + """{ + "encrypt_patient": { + "fields": [ + { + "bsonType": "string", + "path": "patient_record.ssn", + "keyId": { + "$binary": { + "base64": "2MA29LaARIOqymYHGmi2mQ==", + "subType": "04" + } + }, + "queries": { + "queryType": "equality" + } + }, + ] + } + }""" + ), + crypt_shared_lib_path="/path/to/mongo_crypt_shared_v1.dylib", + ) + }, + "KMS_CREDENTIALS": { + "aws": { + "key": os.getenv("AWS_KEY_ARN", ""), + "region": os.getenv("AWS_KEY_REGION", ""), + }, + }, + }, + } + +You are now ready to :doc:`start developing applications +` with Queryable Encryption! diff --git a/docs/index.rst b/docs/index.rst index a5a60e84f..5f5e93108 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -46,10 +46,12 @@ Models - :doc:`ref/database` - :doc:`ref/contrib/gis` - :doc:`ref/django-admin` +- :doc:`ref/models/encrypted-fields` **Topic guides:** - :doc:`topics/embedded-models` +- :doc:`topics/queryable-encryption` - :doc:`topics/transactions` Forms diff --git a/docs/ref/django-admin.rst b/docs/ref/django-admin.rst index a491714cf..1e111eee8 100644 --- a/docs/ref/django-admin.rst +++ b/docs/ref/django-admin.rst @@ -13,3 +13,26 @@ in the :setting:`INSTALLED_APPS` setting. Available commands ================== + +``showencryptedfieldsmap`` +-------------------------- + +.. versionadded:: 5.2.3 + +.. django-admin:: showencryptedfieldsmap + + This command shows the mapping of encrypted fields to attributes including + data type, data keys and query types. Its output can be used to set the + :ref:`encrypted_fields_map ` argument + in :class:`AutoEncryptionOpts + `. + + .. django-admin-option:: --database DATABASE + + Specifies the database to use. Defaults to ``default``. + + To show the encrypted fields map for a database named ``encrypted``, run: + + .. code-block:: console + + $ python manage.py showencryptedfieldsmap --database encrypted diff --git a/docs/ref/index.rst b/docs/ref/index.rst index 94a11a2a8..47b27d466 100644 --- a/docs/ref/index.rst +++ b/docs/ref/index.rst @@ -9,5 +9,7 @@ API reference forms contrib/index database + models/encrypted-fields django-admin utils + settings diff --git a/docs/ref/models/encrypted-fields.rst b/docs/ref/models/encrypted-fields.rst new file mode 100644 index 000000000..07a17b94c --- /dev/null +++ b/docs/ref/models/encrypted-fields.rst @@ -0,0 +1,128 @@ +================ +Encrypted fields +================ + +.. versionadded:: 5.2.3 + +Django MongoDB Backend supports :doc:`manual:core/queryable-encryption`. + +See :doc:`/howto/queryable-encryption` for more information on how to use +Queryable Encryption with Django MongoDB Backend. + +See the :doc:`/topics/queryable-encryption` topic guide for +more information on developing applications with Queryable Encryption. + +The following Django fields are supported by Django MongoDB Backend for use with +Queryable Encryption. + ++----------------------------------------+------------------------------------------------------+ +| Encrypted Field | Django Field | ++========================================+======================================================+ +| ``EncryptedBigIntegerField`` | :class:`~django.db.models.BigIntegerField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedBinaryField`` | :class:`~django.db.models.BinaryField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedBooleanField`` | :class:`~django.db.models.BooleanField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedCharField`` | :class:`~django.db.models.CharField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedDateField`` | :class:`~django.db.models.DateField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedDateTimeField`` | :class:`~django.db.models.DateTimeField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedDecimalField`` | :class:`~django.db.models.DecimalField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedDurationField`` | :class:`~django.db.models.DurationField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedFloatField`` | :class:`~django.db.models.FloatField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedGenericIPAddressField`` | :class:`~django.db.models.GenericIPAddressField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedIntegerField`` | :class:`~django.db.models.IntegerField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedPositiveIntegerField`` | :class:`~django.db.models.PositiveIntegerField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedPositiveBigIntegerField`` | :class:`~django.db.models.PositiveBigIntegerField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedPositiveSmallIntegerField`` | :class:`~django.db.models.PositiveSmallIntegerField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedSmallIntegerField`` | :class:`~django.db.models.SmallIntegerField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedTextField`` | :class:`~django.db.models.TextField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedTimeField`` | :class:`~django.db.models.TimeField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedURLField`` | :class:`~django.db.models.URLField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedUUIDField`` | :class:`~django.db.models.UUIDField` | ++----------------------------------------+------------------------------------------------------+ + +The following MongoDB-specific fields are supported by Django MongoDB Backend +for use with Queryable Encryption. + ++----------------------------------------+------------------------------------------------------+ +| Encrypted Field | MongoDB Field | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedArrayField`` | :class:`~.fields.ArrayField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedEmbeddedModelArrayField`` | :class:`~.fields.EmbeddedModelArrayField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedEmbeddedModelField`` | :class:`~.fields.EmbeddedModelField` | ++----------------------------------------+------------------------------------------------------+ +| ``EncryptedObjectIdField`` | :class:`~.fields.ObjectIdField` | ++----------------------------------------+------------------------------------------------------+ + +The following fields are supported by Django MongoDB Backend but not by +Queryable Encryption. + ++--------------------------------------+--------------------------------------------------------------------------------------------------------------------+ +| Field | Limitation | ++--------------------------------------+--------------------------------------------------------------------------------------------------------------------+ +| :class:`~django.db.models.SlugField` | :ref:`Queryable Encryption does not support TTL Indexes or Unique Indexes ` | ++--------------------------------------+--------------------------------------------------------------------------------------------------------------------+ + +Limitations +=========== + +MongoDB imposes some restrictions on encrypted fields: + +* They cannot be indexed. +* They cannot be part of a unique constraint. +* They cannot be null. + +``EncryptedFieldMixin`` +======================= + +.. class:: EncryptedFieldMixin + + .. versionadded:: 5.2.3 + + A mixin that can be used to create custom encrypted fields with Queryable + Encryption. + + To create an encrypted field, inherit from ``EncryptedFieldMixin`` and + your custom field class: + + .. code-block:: python + + from django.db import models + from django_mongodb_backend.fields import EncryptedFieldMixin + from myapp.fields import MyField + + + class MyEncryptedField(EncryptedFieldMixin, MyField): + pass + + + You can then use your custom encrypted field in a model, specifying the + desired query types: + + .. code-block:: python + + class MyModel(models.Model): + my_encrypted_field = MyEncryptedField( + queries={"queryType": "equality"}, + ) + my_encrypted_field_too = MyEncryptedField( + queries={"queryType": "range"}, + ) diff --git a/docs/ref/settings.rst b/docs/ref/settings.rst new file mode 100644 index 000000000..233515262 --- /dev/null +++ b/docs/ref/settings.rst @@ -0,0 +1,45 @@ +======== +Settings +======== + +.. _queryable-encryption-settings: + +Queryable Encryption +==================== + +The following :setting:`django:DATABASES` inner options support configuration of +Key Management Service (KMS) credentials for Queryable Encryption. + +.. setting:: DATABASE-KMS-CREDENTIALS + +``KMS_CREDENTIALS`` +------------------- + +Default: ``{}`` (empty dictionary) + +A dictionary of Key Management Service (KMS) credential key-value pairs. These +credentials are required to access your KMS provider (such as AWS KMS, Azure Key +Vault, or GCP KMS) for encrypting and decrypting data using Queryable +Encryption. + +For example after :doc:`/howto/queryable-encryption`, to configure AWS KMS, +Azure Key Vault, or GCP KMS credentials, you can set ``KMS_CREDENTIALS`` in +your :setting:`django:DATABASES` settings as follows: + +.. code-block:: python + + DATABASES["encrypted"]["KMS_CREDENTIALS"] = { + "aws": { + "key": os.getenv("AWS_KEY_ARN", ""), + "region": os.getenv("AWS_KEY_REGION", ""), + }, + "azure": { + "key": os.getenv("AZURE_KEY_VAULT_URL", ""), + "client_id": os.getenv("AZURE_CLIENT_ID", ""), + "client_secret": os.getenv("AZURE_CLIENT_SECRET", ""), + }, + "gcp": { + "key": os.getenv("GCP_KEY_NAME", ""), + "project_id": os.getenv("GCP_PROJECT_ID", ""), + }, + } diff --git a/docs/ref/utils.rst b/docs/ref/utils.rst index 5cdb0ccf3..18021d261 100644 --- a/docs/ref/utils.rst +++ b/docs/ref/utils.rst @@ -48,3 +48,42 @@ following parts can be considered stable. But for maximum flexibility, construct :setting:`DATABASES` manually as described in :ref:`configuring-databases-setting`. + +.. versionadded:: 5.2.3 + +``model_has_encrypted_fields()`` +================================= + +.. function:: model_has_encrypted_fields(model) + + Returns ``True`` if the given Django model has any fields that use + encrypted models. + + Example usage in a :ref:`database router + `:: + + from django_mongodb_backend.utils import model_has_encrypted_fields + + class EncryptedRouter: + def db_for_read(self, model, **hints): + if model_has_encrypted_fields(model): + return "encrypted" + return "default" + + def db_for_write(self, model, **hints): + if model_has_encrypted_fields(model): + return "encrypted" + return "default" + + def allow_migrate(self, db, app_label, model_name=None, **hints): + if hints.get("model"): + if model_has_encrypted_fields(hints["model"]): + return db == "encrypted" + else: + return db == "default" + return None + + def kms_provider(self, model): + if model_has_encrypted_fields(model): + return "local" + return None diff --git a/docs/releases/5.2.x.rst b/docs/releases/5.2.x.rst index bed6a6cac..1a0828c81 100644 --- a/docs/releases/5.2.x.rst +++ b/docs/releases/5.2.x.rst @@ -124,6 +124,7 @@ New features :class:`~.fields.PolymorphicEmbeddedModelArrayField` for storing a model instance or list of model instances that may be of more than one model class. - Added :doc:`GeoDjango support `. +- Added :doc:`Queryable Encryption support `. Backwards incompatible changes ------------------------------ diff --git a/docs/topics/index.rst b/docs/topics/index.rst index 6e06b8125..a02b35239 100644 --- a/docs/topics/index.rst +++ b/docs/topics/index.rst @@ -9,5 +9,6 @@ know: :maxdepth: 2 embedded-models + queryable-encryption transactions known-issues diff --git a/docs/topics/known-issues.rst b/docs/topics/known-issues.rst index 96cd6dec1..2d17b352a 100644 --- a/docs/topics/known-issues.rst +++ b/docs/topics/known-issues.rst @@ -26,6 +26,8 @@ Model fields - :class:`~django.db.models.CompositePrimaryKey` - :class:`~django.db.models.GeneratedField` +.. _known-issues-limitations-querying: + Querying ======== diff --git a/docs/topics/queryable-encryption.rst b/docs/topics/queryable-encryption.rst new file mode 100644 index 000000000..75efef7e0 --- /dev/null +++ b/docs/topics/queryable-encryption.rst @@ -0,0 +1,136 @@ +==================== +Queryable Encryption +==================== + +.. versionadded:: 5.2.3 + +Once you have successfully set up MongoDB Queryable Encryption as described in +:doc:`the installation guide `, you can start +using encrypted fields in your Django models. + +Encrypted fields +================ + +The basics +---------- + +:doc:`Encrypted fields ` may be used to protect +sensitive data like social security numbers, credit card information, or +personal health information. With Queryable Encryption, you can also perform +queries on certain encrypted fields. To use encrypted fields in your models, +import the necessary field types from ``django_mongodb_backend.models`` and +define your models as usual. + +Here's the `Python Queryable Encryption Tutorial`_ example implemented in +Django: + +.. code-block:: python + + # myapp/models.py + from django.db import models + from django_mongodb_backend.models import EmbeddedModel + from django_mongodb_backend.fields import ( + EmbeddedModelField, + EncryptedCharField, + EncryptedEmbeddedModelField, + ) + + + class Patient(models.Model): + patient_name = models.CharField(max_length=255) + patient_id = models.BigIntegerField() + patient_record = EmbeddedModelField("PatientRecord") + + def __str__(self): + return f"{self.patient_name} ({self.patient_id})" + + + class PatientRecord(EmbeddedModel): + ssn = EncryptedCharField(max_length=11) + billing = EncryptedEmbeddedModelField("Billing") + bill_amount = models.DecimalField(max_digits=10, decimal_places=2) + + + class Billing(EmbeddedModel): + cc_type = models.CharField(max_length=50) + cc_number = models.CharField(max_length=20) + + +Once you have defined your models, create the migrations with ``python manage.py +makemigrations`` and run the migrations with ``python manage.py migrate``. Then +create and manipulate instances of the data just like any other Django model +data. The fields will automatically handle encryption and decryption, ensuring +that sensitive data is stored securely in the database. + +.. TODO + +.. code-block:: console + + $ python manage.py shell + >>> from myapp.models import Patient, PatientRecord, Billing + >>> billing = Billing(cc_type="Visa", cc_number="4111111111111111") + >>> patient_record = PatientRecord(ssn="123-45-6789", billing=billing) + >>> patient = Patient.objects.create( + patient_name="John Doe", + patient_id=123456789, + patient_record=patient_record, + ) + +.. code-block:: console + + >>> john = Patient.objects.get(name="John Doe") + >>> john.patient_record.ssn + '123-45-6789' + +.. code-block:: console + + >>> john.patient_record.ssn + Binary(b'\x0e\x97sv\xecY\x19Jp\x81\xf1\\\x9cz\t1\r\x02...', 6) + +Querying encrypted fields +------------------------- + +In order to query encrypted fields, you must define the queryable encryption +query type in the model field definition. For example, if you want to query the +``ssn`` field for equality, you can define it as follows: + +.. code-block:: python + + class PatientRecord(EmbeddedModel): + ssn = EncryptedCharField(max_length=11, queries={"queryType": "equality"}) + billing = EncryptedEmbeddedModelField("Billing") + bill_amount = models.DecimalField(max_digits=10, decimal_places=2) + +.. _qe-available-query-types: + +Available query types +~~~~~~~~~~~~~~~~~~~~~ + +The ``queries`` option should be a dictionary that specifies the type of queries +that can be performed on the field. The :ref:`available query types +` are as follows: + +- ``equality``: Supports equality queries. +- ``range``: Supports range queries. + +You can configure an encrypted field for either equality or range queries, but +not both. + +Now you can perform queries on the ``ssn`` field using the defined query type. +For example, to find a patient by their SSN, you can do the following:: + + from myapp.models import Patient + + >>> patient = Patient.objects.get(patient_record__ssn="123-45-6789") + >>> patient.name + 'Bob' + +QuerySet limitations +~~~~~~~~~~~~~~~~~~~~ + +In addition to :ref:`Django MongoDB Backend's QuerySet limitations +`, + +.. TODO + +.. _Python Queryable Encryption Tutorial: https://github.com/mongodb/docs/tree/main/content/manual/manual/source/includes/qe-tutorials/python diff --git a/pyproject.toml b/pyproject.toml index 0549f02ef..b4f4841ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ docs = [ "furo>=2025.7.19", "sphinx-copybutton", ] +encryption = ["pymongo[encryption]"] [project.urls] Homepage = "https://www.mongodb.org" diff --git a/tests/backend_/test_features.py b/tests/backend_/test_features.py index 05959fa70..d505c7fab 100644 --- a/tests/backend_/test_features.py +++ b/tests/backend_/test_features.py @@ -44,3 +44,83 @@ def mocked_command(command): with patch("pymongo.synchronous.database.Database.command", wraps=mocked_command): self.assertIs(connection.features._supports_transactions, False) + + +class SupportsQueryableEncryptionTests(TestCase): + def setUp(self): + # Clear the cached property. + connection.features.__dict__.pop("supports_queryable_encryption", None) + # Must initialize the feature before patching it. + connection.features._supports_transactions # noqa: B018 + + def tearDown(self): + del connection.features.supports_queryable_encryption + + @staticmethod + def enterprise_response(command): + if command == "buildInfo": + return {"modules": ["enterprise"]} + raise Exception("Unexpected command") + + @staticmethod + def non_enterprise_response(command): + if command == "buildInfo": + return {"modules": []} + raise Exception("Unexpected command") + + def test_supported_on_atlas(self): + """Supported on MongoDB 8.0+ Atlas replica set or sharded cluster.""" + with ( + patch( + "pymongo.synchronous.database.Database.command", wraps=self.non_enterprise_response + ), + patch("django.db.connection.features.supports_atlas_search", True), + patch("django.db.connection.features._supports_transactions", True), + patch("django.db.connection.features.is_mongodb_8_0", True), + ): + self.assertIs(connection.features.supports_queryable_encryption, True) + + def test_supported_on_enterprise(self): + """Supported on MongoDB 8.0+ Enterprise replica set or sharded cluster.""" + with ( + patch("pymongo.synchronous.database.Database.command", wraps=self.enterprise_response), + patch("django.db.connection.features.supports_atlas_search", False), + patch("django.db.connection.features._supports_transactions", True), + patch("django.db.connection.features.is_mongodb_8_0", True), + ): + self.assertIs(connection.features.supports_queryable_encryption, True) + + def test_atlas_or_enterprise_required(self): + """Not supported on MongoDB Community Edition.""" + with ( + patch( + "pymongo.synchronous.database.Database.command", wraps=self.non_enterprise_response + ), + patch("django.db.connection.features.supports_atlas_search", False), + patch("django.db.connection.features._supports_transactions", True), + patch("django.db.connection.features.is_mongodb_8_0", True), + ): + self.assertIs(connection.features.supports_queryable_encryption, False) + + def test_transactions_required(self): + """ + Not supported if database isn't a replica set or sharded cluster + (i.e. DatabaseFeatures._supports_transactions = False). + """ + with ( + patch("pymongo.synchronous.database.Database.command", wraps=self.enterprise_response), + patch("django.db.connection.features.supports_atlas_search", False), + patch("django.db.connection.features._supports_transactions", False), + patch("django.db.connection.features.is_mongodb_8_0", True), + ): + self.assertIs(connection.features.supports_queryable_encryption, False) + + def test_mongodb_8_0_required(self): + """Not supported on MongoDB < 8.0""" + with ( + patch("pymongo.synchronous.database.Database.command", wraps=self.enterprise_response), + patch("django.db.connection.features.supports_atlas_search", False), + patch("django.db.connection.features._supports_transactions", True), + patch("django.db.connection.features.is_mongodb_8_0", False), + ): + self.assertIs(connection.features.supports_queryable_encryption, False) diff --git a/tests/encryption_/__init__.py b/tests/encryption_/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/encryption_/models.py b/tests/encryption_/models.py new file mode 100644 index 000000000..930794c39 --- /dev/null +++ b/tests/encryption_/models.py @@ -0,0 +1,167 @@ +from django.db import models + +from django_mongodb_backend.fields import ( + EmbeddedModelField, + EncryptedArrayField, + EncryptedBigIntegerField, + EncryptedBinaryField, + EncryptedBooleanField, + EncryptedCharField, + EncryptedDateField, + EncryptedDateTimeField, + EncryptedDecimalField, + EncryptedDurationField, + EncryptedEmailField, + EncryptedEmbeddedModelArrayField, + EncryptedEmbeddedModelField, + EncryptedFloatField, + EncryptedGenericIPAddressField, + EncryptedIntegerField, + EncryptedObjectIdField, + EncryptedPositiveBigIntegerField, + EncryptedPositiveIntegerField, + EncryptedPositiveSmallIntegerField, + EncryptedSmallIntegerField, + EncryptedTextField, + EncryptedTimeField, + EncryptedURLField, + EncryptedUUIDField, +) +from django_mongodb_backend.models import EmbeddedModel + + +class EncryptedTestModel(models.Model): + class Meta: + abstract = True + required_db_features = {"supports_queryable_encryption"} + + +# Array models +class ArrayModel(EncryptedTestModel): + values = EncryptedArrayField( + models.IntegerField(), + size=5, + ) + + +# Embedded models +class Patient(EncryptedTestModel): + patient_name = models.CharField(max_length=255) + patient_id = models.BigIntegerField() + patient_record = EmbeddedModelField("PatientRecord") + + def __str__(self): + return f"{self.patient_name} ({self.patient_id})" + + +class PatientRecord(EmbeddedModel): + ssn = EncryptedCharField(max_length=11, queries={"queryType": "equality"}) + billing = EncryptedEmbeddedModelField("Billing") + bill_amount = models.DecimalField(max_digits=10, decimal_places=2) + + +class Billing(EmbeddedModel): + cc_type = models.CharField(max_length=50) + cc_number = models.CharField(max_length=20) + + +# Embedded array models +class Actor(EmbeddedModel): + name = models.CharField(max_length=100) + + +class Movie(EncryptedTestModel): + title = models.CharField(max_length=200) + plot = models.TextField(blank=True) + runtime = models.IntegerField(default=0) + released = models.DateTimeField("release date") + cast = EncryptedEmbeddedModelArrayField(Actor) + + def __str__(self): + return self.title + + +# Equality-queryable field models +class BinaryModel(EncryptedTestModel): + value = EncryptedBinaryField(queries={"queryType": "equality"}) + + +class BooleanModel(EncryptedTestModel): + value = EncryptedBooleanField(queries={"queryType": "equality"}) + + +class CharModel(EncryptedTestModel): + value = EncryptedCharField(max_length=255, queries={"queryType": "equality"}) + + +class EmailModel(EncryptedTestModel): + value = EncryptedEmailField(max_length=255, queries={"queryType": "equality"}) + + +class GenericIPAddressModel(EncryptedTestModel): + value = EncryptedGenericIPAddressField(queries={"queryType": "equality"}) + + +class ObjectIdModel(EncryptedTestModel): + value = EncryptedObjectIdField(queries={"queryType": "equality"}) + + +class TextModel(EncryptedTestModel): + value = EncryptedTextField(queries={"queryType": "equality"}) + + +class URLModel(EncryptedTestModel): + value = EncryptedURLField(max_length=500, queries={"queryType": "equality"}) + + +class UUIDModel(EncryptedTestModel): + value = EncryptedUUIDField(queries={"queryType": "equality"}) + + +# Range-queryable field models +class BigIntegerModel(EncryptedTestModel): + value = EncryptedBigIntegerField(queries={"queryType": "range"}) + + +class DateModel(EncryptedTestModel): + value = EncryptedDateField(queries={"queryType": "range"}) + + +class DateTimeModel(EncryptedTestModel): + value = EncryptedDateTimeField(queries={"queryType": "range"}) + + +class DecimalModel(EncryptedTestModel): + value = EncryptedDecimalField(max_digits=10, decimal_places=2, queries={"queryType": "range"}) + + +class DurationModel(EncryptedTestModel): + value = EncryptedDurationField(queries={"queryType": "range"}) + + +class FloatModel(EncryptedTestModel): + value = EncryptedFloatField(queries={"queryType": "range"}) + + +class IntegerModel(EncryptedTestModel): + value = EncryptedIntegerField(queries={"queryType": "range"}) + + +class PositiveBigIntegerModel(EncryptedTestModel): + value = EncryptedPositiveBigIntegerField(queries={"queryType": "range"}) + + +class PositiveIntegerModel(EncryptedTestModel): + value = EncryptedPositiveIntegerField(queries={"queryType": "range"}) + + +class PositiveSmallIntegerModel(EncryptedTestModel): + value = EncryptedPositiveSmallIntegerField(queries={"queryType": "range"}) + + +class SmallIntegerModel(EncryptedTestModel): + value = EncryptedSmallIntegerField(queries={"queryType": "range"}) + + +class TimeModel(EncryptedTestModel): + value = EncryptedTimeField(queries={"queryType": "range"}) diff --git a/tests/encryption_/test_base.py b/tests/encryption_/test_base.py new file mode 100644 index 000000000..0c165d19a --- /dev/null +++ b/tests/encryption_/test_base.py @@ -0,0 +1,21 @@ +import pymongo +from bson.binary import Binary +from django.conf import settings +from django.db import connections +from django.test import TestCase, skipUnlessDBFeature + + +@skipUnlessDBFeature("supports_queryable_encryption") +class EncryptionTestCase(TestCase): + databases = {"default", "encrypted"} + maxDiff = None + + def assertEncrypted(self, model, field): + # Access encrypted database from an unencrypted connection + conn_params = connections["default"].get_connection_params() + db_name = settings.DATABASES["encrypted"]["NAME"] + with pymongo.MongoClient(**conn_params) as new_connection: + db = new_connection[db_name] + collection = db[model._meta.db_table] + data = collection.find_one({}, {field: 1, "_id": 0}) + self.assertIsInstance(data[field], Binary) diff --git a/tests/encryption_/test_fields.py b/tests/encryption_/test_fields.py new file mode 100644 index 000000000..c4a11af98 --- /dev/null +++ b/tests/encryption_/test_fields.py @@ -0,0 +1,251 @@ +import datetime +import uuid +from decimal import Decimal + +from bson import ObjectId + +from django_mongodb_backend.fields import EncryptedCharField, EncryptedIntegerField + +from .models import ( + Actor, + ArrayModel, + BigIntegerModel, + Billing, + BinaryModel, + BooleanModel, + CharModel, + DateModel, + DateTimeModel, + DecimalModel, + DurationModel, + EmailModel, + FloatModel, + GenericIPAddressModel, + IntegerModel, + Movie, + ObjectIdModel, + Patient, + PatientRecord, + PositiveBigIntegerModel, + PositiveIntegerModel, + PositiveSmallIntegerModel, + SmallIntegerModel, + TextModel, + TimeModel, + URLModel, + UUIDModel, +) +from .test_base import EncryptionTestCase + + +class ArrayModelTests(EncryptionTestCase): + def setUp(self): + self.array_model = ArrayModel.objects.create(values=[1, 2, 3, 4, 5]) + + def test_array(self): + array_model = ArrayModel.objects.get(id=self.array_model.id) + self.assertEqual(array_model.values, [1, 2, 3, 4, 5]) + self.assertEncrypted(self.array_model, "values") + + +class EmbeddedModelTests(EncryptionTestCase): + def setUp(self): + self.billing = Billing(cc_type="Visa", cc_number="4111111111111111") + self.patient_record = PatientRecord(ssn="123-45-6789", billing=self.billing) + self.patient = Patient.objects.create( + patient_name="John Doe", patient_id=123456789, patient_record=self.patient_record + ) + + def test_object(self): + patient = Patient.objects.get(id=self.patient.id) + self.assertEqual(patient.patient_record.ssn, "123-45-6789") + self.assertEqual(patient.patient_record.billing.cc_type, "Visa") + self.assertEqual(patient.patient_record.billing.cc_number, "4111111111111111") + + +class EmbeddedModelArrayTests(EncryptionTestCase): + def setUp(self): + self.actor1 = Actor(name="Actor One") + self.actor2 = Actor(name="Actor Two") + self.movie = Movie.objects.create( + title="Sample Movie", + cast=[self.actor1, self.actor2], + released=datetime.date(2024, 6, 1), + ) + + def test_array(self): + movie = Movie.objects.get(id=self.movie.id) + self.assertEqual(len(movie.cast), 2) + self.assertEqual(movie.cast[0].name, "Actor One") + self.assertEqual(movie.cast[1].name, "Actor Two") + self.assertEncrypted(movie, "cast") + + +class FieldTests(EncryptionTestCase): + def assertEquality(self, model_cls, val): + model_cls.objects.create(value=val) + fetched = model_cls.objects.get(value=val) + self.assertEqual(fetched.value, val) + + def assertRange(self, model_cls, *, low, high, threshold): + model_cls.objects.create(value=low) + model_cls.objects.create(value=high) + self.assertEqual(model_cls.objects.get(value=low).value, low) + self.assertEqual(model_cls.objects.get(value=high).value, high) + objs = list(model_cls.objects.filter(value__gt=threshold)) + self.assertEqual(len(objs), 1) + self.assertEqual(objs[0].value, high) + + # Equality-only fields + def test_binary(self): + self.assertEquality(BinaryModel, b"\x00\x01\x02") + self.assertEncrypted(BinaryModel, "value") + + def test_boolean(self): + self.assertEquality(BooleanModel, True) + self.assertEncrypted(BooleanModel, "value") + + def test_char(self): + self.assertEquality(CharModel, "hello") + self.assertEncrypted(CharModel, "value") + + def test_email(self): + self.assertEquality(EmailModel, "test@example.com") + self.assertEncrypted(EmailModel, "value") + + def test_ip(self): + self.assertEquality(GenericIPAddressModel, "192.168.0.1") + self.assertEncrypted(GenericIPAddressModel, "value") + + def test_objectid(self): + self.assertEquality(ObjectIdModel, ObjectId()) + self.assertEncrypted(ObjectIdModel, "value") + + def test_text(self): + self.assertEquality(TextModel, "some text") + self.assertEncrypted(TextModel, "value") + + def test_url(self): + self.assertEquality(URLModel, "https://example.com") + self.assertEncrypted(URLModel, "value") + + def test_uuid(self): + self.assertEquality(UUIDModel, uuid.uuid4()) + self.assertEncrypted(UUIDModel, "value") + + # Range fields + def test_big_integer(self): + self.assertRange(BigIntegerModel, low=100, high=200, threshold=150) + self.assertEncrypted(BigIntegerModel, "value") + + def test_date(self): + self.assertRange( + DateModel, + low=datetime.date(2024, 6, 1), + high=datetime.date(2024, 6, 10), + threshold=datetime.date(2024, 6, 5), + ) + self.assertEncrypted(DateModel, "value") + + def test_datetime(self): + self.assertRange( + DateTimeModel, + low=datetime.datetime(2024, 6, 1, 12, 0), + high=datetime.datetime(2024, 6, 2, 12, 0), + threshold=datetime.datetime(2024, 6, 2, 0, 0), + ) + self.assertEncrypted(DateTimeModel, "value") + + def test_decimal(self): + self.assertRange( + DecimalModel, + low=Decimal("123.45"), + high=Decimal("200.50"), + threshold=Decimal("150"), + ) + self.assertEncrypted(DecimalModel, "value") + + def test_duration(self): + self.assertRange( + DurationModel, + low=datetime.timedelta(days=3), + high=datetime.timedelta(days=10), + threshold=datetime.timedelta(days=5), + ) + self.assertEncrypted(DurationModel, "value") + + def test_float(self): + self.assertRange(FloatModel, low=1.23, high=4.56, threshold=3.0) + self.assertEncrypted(FloatModel, "value") + + def test_integer(self): + self.assertRange(IntegerModel, low=5, high=10, threshold=7) + self.assertEncrypted(IntegerModel, "value") + + def test_positive_big_integer(self): + self.assertRange(PositiveBigIntegerModel, low=100, high=500, threshold=200) + self.assertEncrypted(PositiveBigIntegerModel, "value") + + def test_positive_integer(self): + self.assertRange(PositiveIntegerModel, low=10, high=20, threshold=15) + self.assertEncrypted(PositiveIntegerModel, "value") + + def test_positive_small_integer(self): + self.assertRange(PositiveSmallIntegerModel, low=5, high=8, threshold=6) + self.assertEncrypted(PositiveSmallIntegerModel, "value") + + def test_small_integer(self): + self.assertRange(SmallIntegerModel, low=-5, high=2, threshold=0) + self.assertEncrypted(SmallIntegerModel, "value") + + def test_time(self): + self.assertRange( + TimeModel, + low=datetime.time(10, 0), + high=datetime.time(15, 0), + threshold=datetime.time(12, 0), + ) + self.assertEncrypted(TimeModel, "value") + + +class FieldMixinTests(EncryptionTestCase): + def test_db_index(self): + msg = "'db_index=True' is not supported on encrypted fields." + with self.assertRaisesMessage(ValueError, msg): + EncryptedIntegerField(db_index=True) + + def test_null(self): + msg = "'null=True' is not supported on encrypted fields." + with self.assertRaisesMessage(ValueError, msg): + EncryptedIntegerField(null=True) + + def test_unique(self): + msg = "'unique=True' is not supported on encrypted fields." + with self.assertRaisesMessage(ValueError, msg): + EncryptedIntegerField(unique=True) + + def test_deconstruct_preserves_queries_and_rewrites_path(self): + field = EncryptedCharField(max_length=50, queries={"field": "value"}) + field.name = "ssn" + name, path, args, kwargs = field.deconstruct() + + # Name is preserved + self.assertEqual(name, "ssn") + + # Path is rewritten from 'encrypted_model' to regular fields path + self.assertEqual(path, "django_mongodb_backend.fields.EncryptedCharField") + + # No positional args for CharField + self.assertEqual(args, []) + + # Queries value is preserved in kwargs + self.assertIn("queries", kwargs) + self.assertEqual(kwargs["queries"], {"field": "value"}) + + # Reconstruct from deconstruct output + new_field = EncryptedCharField(*args, **kwargs) + + # Reconstructed field is equivalent + self.assertEqual(new_field.queries, field.queries) + self.assertIsNot(new_field, field) + self.assertEqual(new_field.max_length, field.max_length) diff --git a/tests/encryption_/test_management.py b/tests/encryption_/test_management.py new file mode 100644 index 000000000..ceaa4fca9 --- /dev/null +++ b/tests/encryption_/test_management.py @@ -0,0 +1,111 @@ +from io import StringIO + +from bson import json_util +from django.core.management import call_command +from django.test import modify_settings + +from .test_base import EncryptionTestCase + + +@modify_settings(INSTALLED_APPS={"prepend": "django_mongodb_backend"}) +class CommandTests(EncryptionTestCase): + # Expected encrypted field maps for all Encrypted* models + expected_maps = { + "encryption__patient": { + "fields": [ + { + "bsonType": "string", + "path": "patient_record.ssn", + "queries": {"queryType": "equality"}, + }, + {"bsonType": "object", "path": "patient_record.billing"}, + ] + }, + # Equality-queryable fields + "encryption__binarymodel": { + "fields": [ + {"bsonType": "binData", "path": "value", "queries": {"queryType": "equality"}} + ] + }, + "encryption__booleanmodel": { + "fields": [{"bsonType": "bool", "path": "value", "queries": {"queryType": "equality"}}] + }, + "encryption__charmodel": { + "fields": [ + {"bsonType": "string", "path": "value", "queries": {"queryType": "equality"}} + ] + }, + "encryption__emailmodel": { + "fields": [ + {"bsonType": "string", "path": "value", "queries": {"queryType": "equality"}} + ] + }, + "encryption__genericipaddressmodel": { + "fields": [ + {"bsonType": "string", "path": "value", "queries": {"queryType": "equality"}} + ] + }, + "encryption__textmodel": { + "fields": [ + {"bsonType": "string", "path": "value", "queries": {"queryType": "equality"}} + ] + }, + "encryption__urlmodel": { + "fields": [ + {"bsonType": "string", "path": "value", "queries": {"queryType": "equality"}} + ] + }, + # Range-queryable fields + "encryption__bigintegermodel": { + "fields": [{"bsonType": "long", "path": "value", "queries": {"queryType": "range"}}] + }, + "encryption__datemodel": { + "fields": [{"bsonType": "date", "path": "value", "queries": {"queryType": "range"}}] + }, + "encryption__datetimemodel": { + "fields": [{"bsonType": "date", "path": "value", "queries": {"queryType": "range"}}] + }, + "encryption__decimalmodel": { + "fields": [{"bsonType": "decimal", "path": "value", "queries": {"queryType": "range"}}] + }, + "encryption__durationmodel": { + "fields": [{"bsonType": "long", "path": "value", "queries": {"queryType": "range"}}] + }, + "encryption__floatmodel": { + "fields": [{"bsonType": "double", "path": "value", "queries": {"queryType": "range"}}] + }, + "encryption__integermodel": { + "fields": [{"bsonType": "long", "path": "value", "queries": {"queryType": "range"}}] + }, + "encryption__positivebigintegermodel": { + "fields": [{"bsonType": "long", "path": "value", "queries": {"queryType": "range"}}] + }, + "encryption__positiveintegermodel": { + "fields": [{"bsonType": "long", "path": "value", "queries": {"queryType": "range"}}] + }, + "encryption__positivesmallintegermodel": { + "fields": [{"bsonType": "int", "path": "value", "queries": {"queryType": "range"}}] + }, + "encryption__smallintegermodel": { + "fields": [{"bsonType": "int", "path": "value", "queries": {"queryType": "range"}}] + }, + "encryption__timemodel": { + "fields": [{"bsonType": "date", "path": "value", "queries": {"queryType": "range"}}] + }, + } + + def _compare_output(self, expected, actual): + for field in actual["fields"]: + field.pop("keyId", None) # remove dynamic keyId + self.assertEqual(expected, actual) + + def test_show_encrypted_fields_map(self): + out = StringIO() + call_command("showencryptedfieldsmap", "--database", "encrypted", verbosity=0, stdout=out) + command_output = json_util.loads(out.getvalue()) + + # Loop through each expected model + for model_key, expected in self.expected_maps.items(): + with self.subTest(model=model_key): + self.assertIn(model_key, command_output) + self._compare_output(expected, command_output[model_key]) diff --git a/tests/encryption_/test_schema.py b/tests/encryption_/test_schema.py new file mode 100644 index 000000000..d7cfb5a06 --- /dev/null +++ b/tests/encryption_/test_schema.py @@ -0,0 +1,140 @@ +from bson.binary import Binary +from django.db import connections + +from . import models +from .test_base import EncryptionTestCase + + +class SchemaTests(EncryptionTestCase): + # Expected encrypted fields map per model + expected_map = { + "Patient": { + "fields": [ + { + "bsonType": "string", + "path": "patient_record.ssn", + "queries": {"queryType": "equality"}, + }, + {"bsonType": "object", "path": "patient_record.billing"}, + ] + }, + "BinaryModel": { + "fields": [ + {"bsonType": "binData", "path": "value", "queries": {"queryType": "equality"}} + ] + }, + "BooleanModel": { + "fields": [{"bsonType": "bool", "path": "value", "queries": {"queryType": "equality"}}] + }, + "CharModel": { + "fields": [ + {"bsonType": "string", "path": "value", "queries": {"queryType": "equality"}} + ] + }, + "EmailModel": { + "fields": [ + {"bsonType": "string", "path": "value", "queries": {"queryType": "equality"}} + ] + }, + "GenericIPAddressModel": { + "fields": [ + {"bsonType": "string", "path": "value", "queries": {"queryType": "equality"}} + ] + }, + "TextModel": { + "fields": [ + {"bsonType": "string", "path": "value", "queries": {"queryType": "equality"}} + ] + }, + "URLModel": { + "fields": [ + {"bsonType": "string", "path": "value", "queries": {"queryType": "equality"}} + ] + }, + "BigIntegerModel": { + "fields": [{"bsonType": "long", "path": "value", "queries": {"queryType": "range"}}] + }, + "DateModel": { + "fields": [{"bsonType": "date", "path": "value", "queries": {"queryType": "range"}}] + }, + "DateTimeModel": { + "fields": [{"bsonType": "date", "path": "value", "queries": {"queryType": "range"}}] + }, + "DecimalModel": { + "fields": [{"bsonType": "decimal", "path": "value", "queries": {"queryType": "range"}}] + }, + "DurationModel": { + "fields": [{"bsonType": "long", "path": "value", "queries": {"queryType": "range"}}] + }, + "FloatModel": { + "fields": [{"bsonType": "double", "path": "value", "queries": {"queryType": "range"}}] + }, + "IntegerModel": { + "fields": [{"bsonType": "long", "path": "value", "queries": {"queryType": "range"}}] + }, + "PositiveBigIntegerModel": { + "fields": [{"bsonType": "long", "path": "value", "queries": {"queryType": "range"}}] + }, + "PositiveIntegerModel": { + "fields": [{"bsonType": "long", "path": "value", "queries": {"queryType": "range"}}] + }, + "PositiveSmallIntegerModel": { + "fields": [{"bsonType": "int", "path": "value", "queries": {"queryType": "range"}}] + }, + "SmallIntegerModel": { + "fields": [{"bsonType": "int", "path": "value", "queries": {"queryType": "range"}}] + }, + "TimeModel": { + "fields": [{"bsonType": "date", "path": "value", "queries": {"queryType": "range"}}] + }, + } + + def test_get_encrypted_fields_all_models(self): + """ + Loops through all models, + checks their encrypted fields map from the schema editor, + and compares to expected BSON type & queries mapping. + """ + connection = connections["encrypted"] + + for model_name, expected in self.expected_map.items(): + with self.subTest(model=model_name): + model_class = getattr(models, model_name) + with connection.schema_editor() as editor: + client = connection.connection + encrypted_fields = editor._get_encrypted_fields(model_class, client) + for field in encrypted_fields["fields"]: + field.pop("keyId", None) # Remove dynamic value + self.assertEqual(encrypted_fields, expected) + + def test_key_creation_and_lookup(self): + """ + Use _get_encrypted_fields to + generate and store a data key in the vault, then + query the vault with the keyAltName. + """ + connection = connections["encrypted"] + client = connection.connection + auto_encryption_opts = client._options.auto_encryption_opts + + key_vault_db, key_vault_coll = auto_encryption_opts._key_vault_namespace.split(".", 1) + vault_coll = client[key_vault_db][key_vault_coll] + + model_class = models.CharModel + test_key_alt_name = f"{model_class._meta.db_table}.value" + vault_coll.delete_many({"keyAltNames": test_key_alt_name}) + + with connection.schema_editor() as editor: + encrypted_fields = editor._get_encrypted_fields(model_class) + + # Validate schema contains a keyId for our field + self.assertTrue(encrypted_fields["fields"]) + field_info = encrypted_fields["fields"][0] + self.assertEqual(field_info["path"], "value") + self.assertIsInstance(field_info["keyId"], Binary) + + # Lookup in key vault by the keyAltName created + key_doc = vault_coll.find_one({"keyAltNames": test_key_alt_name}) + self.assertIsNotNone(key_doc, "Key should exist in vault") + self.assertEqual(key_doc["_id"], field_info["keyId"]) + self.assertIn(test_key_alt_name, key_doc["keyAltNames"]) diff --git a/tests/raw_query_/test_raw_aggregate.py b/tests/raw_query_/test_raw_aggregate.py index 99dcd5faf..96df2f925 100644 --- a/tests/raw_query_/test_raw_aggregate.py +++ b/tests/raw_query_/test_raw_aggregate.py @@ -111,7 +111,7 @@ def assertAnnotations(self, results, expected_annotations): self.assertEqual(getattr(result, annotation), value) def test_rawqueryset_repr(self): - queryset = RawQuerySet(pipeline=[]) + queryset = RawQuerySet(pipeline=[], model=Book) self.assertEqual(repr(queryset), "") self.assertEqual(repr(queryset.query), "") From 1cc37abc491c402a8244812bf5a695187f64fbba Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Fri, 17 Oct 2025 13:50:45 -0400 Subject: [PATCH 02/16] Remove duplicate limitations section Added in 892bb6bae9d3563f97efc26123cbb0c521fbca40 (unless we need to call out queryset limitations in addition) --- docs/topics/queryable-encryption.rst | 8 -------- 1 file changed, 8 deletions(-) diff --git a/docs/topics/queryable-encryption.rst b/docs/topics/queryable-encryption.rst index 75efef7e0..3f025c720 100644 --- a/docs/topics/queryable-encryption.rst +++ b/docs/topics/queryable-encryption.rst @@ -125,12 +125,4 @@ For example, to find a patient by their SSN, you can do the following:: >>> patient.name 'Bob' -QuerySet limitations -~~~~~~~~~~~~~~~~~~~~ - -In addition to :ref:`Django MongoDB Backend's QuerySet limitations -`, - -.. TODO - .. _Python Queryable Encryption Tutorial: https://github.com/mongodb/docs/tree/main/content/manual/manual/source/includes/qe-tutorials/python From e1c13f5255de2d0bcc4b9660d4a37b12eb367b5d Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Fri, 17 Oct 2025 18:21:33 -0400 Subject: [PATCH 03/16] Add admonition about range queries vs. lookups --- docs/topics/queryable-encryption.rst | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/topics/queryable-encryption.rst b/docs/topics/queryable-encryption.rst index 3f025c720..affc38853 100644 --- a/docs/topics/queryable-encryption.rst +++ b/docs/topics/queryable-encryption.rst @@ -123,6 +123,24 @@ For example, to find a patient by their SSN, you can do the following:: >>> patient = Patient.objects.get(patient_record__ssn="123-45-6789") >>> patient.name - 'Bob' + 'John Doe' + +.. admonition:: Range queries vs. lookups + + Range queries in Queryable Encryption are different from Django's + :ref:`range lookups ` + Range queries allow you to perform comparisons on encrypted fields, + while Django's range lookups are used for filtering based on a range of + values. + + For example, if you have an encrypted field that supports range queries, you + can perform a query like this:: + + from myapp.models import Patient + + >>> patients = Patient.objects.filter(patient_record__ssn__gte="123-45-0000", + ... patient_record__ssn__lte="123-45-9999") + + This will return all patients whose SSN falls within the specified range. .. _Python Queryable Encryption Tutorial: https://github.com/mongodb/docs/tree/main/content/manual/manual/source/includes/qe-tutorials/python From 39259ffff8e026f5aad01d4259c4e5274f7c786e Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Fri, 17 Oct 2025 21:36:10 -0400 Subject: [PATCH 04/16] Document Query types vs. Django lookups --- docs/topics/queryable-encryption.rst | 70 +++++++++++----------------- 1 file changed, 26 insertions(+), 44 deletions(-) diff --git a/docs/topics/queryable-encryption.rst b/docs/topics/queryable-encryption.rst index affc38853..897b78246 100644 --- a/docs/topics/queryable-encryption.rst +++ b/docs/topics/queryable-encryption.rst @@ -60,32 +60,8 @@ Once you have defined your models, create the migrations with ``python manage.py makemigrations`` and run the migrations with ``python manage.py migrate``. Then create and manipulate instances of the data just like any other Django model data. The fields will automatically handle encryption and decryption, ensuring -that sensitive data is stored securely in the database. - -.. TODO - -.. code-block:: console - - $ python manage.py shell - >>> from myapp.models import Patient, PatientRecord, Billing - >>> billing = Billing(cc_type="Visa", cc_number="4111111111111111") - >>> patient_record = PatientRecord(ssn="123-45-6789", billing=billing) - >>> patient = Patient.objects.create( - patient_name="John Doe", - patient_id=123456789, - patient_record=patient_record, - ) - -.. code-block:: console - - >>> john = Patient.objects.get(name="John Doe") - >>> john.patient_record.ssn - '123-45-6789' - -.. code-block:: console - - >>> john.patient_record.ssn - Binary(b'\x0e\x97sv\xecY\x19Jp\x81\xf1\\\x9cz\t1\r\x02...', 6) +that :ref:`sensitive data is stored securely in the database +`. Querying encrypted fields ------------------------- @@ -101,6 +77,14 @@ query type in the model field definition. For example, if you want to query the billing = EncryptedEmbeddedModelField("Billing") bill_amount = models.DecimalField(max_digits=10, decimal_places=2) +Then you can perform a query like this: + +.. code-block:: console + + >>> patient = Patient.objects.get(patient_record__ssn="123-45-6789") + >>> patient.name + 'John Doe' + .. _qe-available-query-types: Available query types @@ -116,31 +100,29 @@ that can be performed on the field. The :ref:`available query types You can configure an encrypted field for either equality or range queries, but not both. -Now you can perform queries on the ``ssn`` field using the defined query type. -For example, to find a patient by their SSN, you can do the following:: +.. admonition:: Query types vs. Django lookups - from myapp.models import Patient + Range queries in Queryable Encryption are different from Django's + :ref:`range lookups `. Range queries allow you to + perform comparisons on encrypted fields, while Django's range lookups are + used for filtering based on a range of values. - >>> patient = Patient.objects.get(patient_record__ssn="123-45-6789") - >>> patient.name - 'John Doe' +If you have an encrypted field that supports range queries like this: -.. admonition:: Range queries vs. lookups +.. code-block:: python - Range queries in Queryable Encryption are different from Django's - :ref:`range lookups ` - Range queries allow you to perform comparisons on encrypted fields, - while Django's range lookups are used for filtering based on a range of - values. + class PatientRecord(EmbeddedModel): + ssn = EncryptedCharField(max_length=11, queries={"queryType": "range"}) + billing = EncryptedEmbeddedModelField("Billing") + bill_amount = models.DecimalField(max_digits=10, decimal_places=2) - For example, if you have an encrypted field that supports range queries, you - can perform a query like this:: +You can perform a query like this: - from myapp.models import Patient +.. code-block:: console - >>> patients = Patient.objects.filter(patient_record__ssn__gte="123-45-0000", - ... patient_record__ssn__lte="123-45-9999") + >>> patients = Patient.objects.filter(patient_record__ssn__gte="123-45-0000", + ... patient_record__ssn__lte="123-45-9999") - This will return all patients whose SSN falls within the specified range. +This will return all patients whose SSN falls within the specified range. .. _Python Queryable Encryption Tutorial: https://github.com/mongodb/docs/tree/main/content/manual/manual/source/includes/qe-tutorials/python From db188cfda65dafd1ea12a506ee503bc1d8d1a515 Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Tue, 21 Oct 2025 17:36:39 -0400 Subject: [PATCH 05/16] Don't forget to migrate with `--database encrypted` --- docs/howto/queryable-encryption.rst | 3 ++- docs/topics/queryable-encryption.rst | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/howto/queryable-encryption.rst b/docs/howto/queryable-encryption.rst index 69d83fd09..5086b1e0b 100644 --- a/docs/howto/queryable-encryption.rst +++ b/docs/howto/queryable-encryption.rst @@ -198,7 +198,8 @@ Configuring the ``encrypted_fields_map`` When you :ref:`configure an encrypted database connection ` without specifying an ``encrypted_fields_map``, Django MongoDB Backend will create encrypted -collections for you when you run ``python manage.py migrate``. +collections for you when you run ``python manage.py migrate --database +encrypted``. Encryption keys for encrypted fields are stored in the key vault :ref:`specified in the Django settings `. To see the keys diff --git a/docs/topics/queryable-encryption.rst b/docs/topics/queryable-encryption.rst index 897b78246..ff6d1e3a8 100644 --- a/docs/topics/queryable-encryption.rst +++ b/docs/topics/queryable-encryption.rst @@ -57,11 +57,11 @@ Django: Once you have defined your models, create the migrations with ``python manage.py -makemigrations`` and run the migrations with ``python manage.py migrate``. Then -create and manipulate instances of the data just like any other Django model -data. The fields will automatically handle encryption and decryption, ensuring -that :ref:`sensitive data is stored securely in the database -`. +makemigrations`` and run the migrations with ``python manage.py migrate +--database encrypted``. Then create and manipulate instances of the data just +like any other Django model data. The fields will automatically handle +encryption and decryption, ensuring that :ref:`sensitive data is stored securely +in the database `. Querying encrypted fields ------------------------- From 0154ca41f6d3e25db6a09f3b0fafe8dca1740938 Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Tue, 21 Oct 2025 21:28:53 -0400 Subject: [PATCH 06/16] Use same keyvault namespace as QE tutorial It doesn't really matter but may as well be consistent with tutorial when possible. Curiously and confusingly the CSFLE spec uses "keyvault.datakeys". --- docs/howto/queryable-encryption.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/howto/queryable-encryption.rst b/docs/howto/queryable-encryption.rst index 5086b1e0b..5c56f524e 100644 --- a/docs/howto/queryable-encryption.rst +++ b/docs/howto/queryable-encryption.rst @@ -76,7 +76,7 @@ encryption keys. "PORT": 27017, "OPTIONS": { "auto_encryption_opts": AutoEncryptionOpts( - key_vault_namespace="encrypted.keyvault", + key_vault_namespace="encryption.__keyVault", kms_providers={"local": {"key": os.urandom(96)}}, ) }, @@ -166,7 +166,7 @@ Example of KMS configuration with AWS KMS: "PORT": 27017, "OPTIONS": { "auto_encryption_opts": AutoEncryptionOpts( - key_vault_namespace="encrypted.keyvault", + key_vault_namespace="encryption.__keyVault", kms_providers={ "aws": { "accessKeyId": "your-access-key-id", @@ -227,7 +227,7 @@ Use the output of the :djadmin:`showencryptedfieldsmap` command to set the "PORT": 27017, "OPTIONS": { "auto_encryption_opts": AutoEncryptionOpts( - key_vault_namespace="encrypted.keyvault", + key_vault_namespace="encryption.__keyVault", kms_providers={ "aws": { "accessKeyId": "your-access-key-id", @@ -292,7 +292,7 @@ shows how to configure the shared library in your Django settings: "PORT": 27017, "OPTIONS": { "auto_encryption_opts": AutoEncryptionOpts( - key_vault_namespace="encrypted.keyvault", + key_vault_namespace="encryption.__keyVault", kms_providers={ "aws": { "accessKeyId": "your-access-key-id", From 5f01a8297f749904531e121a9dc2f558027bfadd Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Tue, 21 Oct 2025 23:10:11 -0400 Subject: [PATCH 07/16] Update docs --- docs/howto/queryable-encryption.rst | 221 ++++++++++++--------------- docs/ref/models/encrypted-fields.rst | 4 +- docs/topics/queryable-encryption.rst | 38 +++-- 3 files changed, 125 insertions(+), 138 deletions(-) diff --git a/docs/howto/queryable-encryption.rst b/docs/howto/queryable-encryption.rst index 5c56f524e..1d0b7ce85 100644 --- a/docs/howto/queryable-encryption.rst +++ b/docs/howto/queryable-encryption.rst @@ -24,28 +24,36 @@ Installation ============ In addition to the :doc:`installation ` and :doc:`configuration -` steps for Django MongoDB Backend, Queryable -Encryption requires encryption support and a Key Management Service (KMS). +` steps required to use Django MongoDB Backend, Queryable +Encryption has additional dependencies. You can install these dependencies +by using the ``encryption`` extra when installing ``django-mongodb-backend``: -You can install encryption support with the following command:: +.. code-block:: console - pip install django-mongodb-backend[encryption] + $ pip install django-mongodb-backend[encryption] .. _qe-configuring-databases-setting: Configuring the ``DATABASES`` setting ===================================== -In addition to :ref:`configuring-databases-setting`, you must also configure an -encrypted database in your :setting:`django:DATABASES` setting. +In addition to the :ref:`database settings ` +required to use Django MongoDB Backend, Queryable Encryption requires you to +configure a separate encrypted database connection in your +:setting:`django:DATABASES` setting. -This database will be used to store encrypted fields in your models. The -following example shows how to configure an encrypted database using the -:class:`AutoEncryptionOpts ` from the -:mod:`encryption_options ` module. +.. admonition:: Encrypted database -This example uses a local KMS provider and a key vault namespace for storing -encryption keys. + An encrypted database is a separate database connection in your + :setting:`django:DATABASES` setting that is configured to use PyMongo's + :class:`automatic encryption + `. + +The following example shows how to +configure an encrypted database using the :class:`AutoEncryptionOpts +` from the +:mod:`encryption_options ` module with a local KMS +provider and encryption keys stored in the ``encryption.__keyVault`` collection. .. code-block:: python @@ -58,19 +66,12 @@ encryption keys. "ENGINE": "django_mongodb_backend", "HOST": "mongodb+srv://cluster0.example.mongodb.net", "NAME": "my_database", - "USER": "my_user", - "PASSWORD": "my_password", - "PORT": 27017, - "OPTIONS": { - "retryWrites": "true", - "w": "majority", - "tls": "false", - }, + # ... }, "encrypted": { "ENGINE": "django_mongodb_backend", "HOST": "mongodb+srv://cluster0.example.mongodb.net", - "NAME": "encrypted", + "NAME": "my_database_encrypted", "USER": "my_user", "PASSWORD": "my_password", "PORT": 27017, @@ -83,55 +84,69 @@ encryption keys. }, } +.. admonition:: Local KMS provider key + + In the example above, a random key is generated for the local KMS provider + using ``os.urandom(96)``. In a production environment, you should securely + :ref:`store and manage your encryption keys + `. + .. _qe-configuring-database-routers-setting: Configuring the ``DATABASE_ROUTERS`` setting ============================================ -Similar to :ref:`configuring-database-routers-setting` for using :doc:`embedded -models `, to use Queryable Encryption you must also -configure the :setting:`django:DATABASE_ROUTERS` setting to route queries to the -encrypted database. +Similar to configuring the :ref:`DATABASE_ROUTERS +` setting for +:doc:`embedded models `, Queryable Encryption +requires a :setting:`DATABASE_ROUTERS ` setting to +route database operations to the encrypted database. -This is done by adding a custom router that routes queries to the encrypted -database based on the model's metadata. The following example shows how to -configure a custom router for Queryable Encryption: +The following example shows how to configure a router for the "myapp" +application that routes database operations to the encrypted database for all +models in that application. The router also specifies the :ref:`KMS provider +` to use. .. code-block:: python + # myapp/routers.py class EncryptedRouter: - """ - A router for routing queries to the encrypted database for Queryable - Encryption. - """ - - def db_for_read(self, model, **hints): - if model._meta.app_label == "myapp": - return "encrypted" - return None - - db_for_write = db_for_read - def allow_migrate(self, db, app_label, model_name=None, **hints): if app_label == "myapp": return db == "encrypted" - # Don't create other app's models in the encrypted database. + # Prevent migrations on the encrypted database for other apps if db == "encrypted": return False return None + def db_for_read(self, model, **hints): + if model._meta.app_label == "myapp": + return "encrypted" + return None + def kms_provider(self, model, **hints): return "local" + db_for_write = db_for_read + +Then in your Django settings, add the custom database router to the +:setting:`django:DATABASE_ROUTERS` setting: - DATABASE_ROUTERS = [EncryptedRouter] +.. code-block:: python + + # settings.py + DATABASE_ROUTERS = ["myapp.routers.EncryptedRouter"] .. _qe-configuring-kms: Configuring the Key Management Service (KMS) ============================================ -To use Queryable Encryption, you must configure a Key Management Service (KMS). +To use Queryable Encryption, you must configure a Key Management Service (KMS) +to store and manage your encryption keys. Django MongoDB Backend allows you to +configure multiple KMS providers and select the appropriate provider for each +model using a custom database router. + The KMS is responsible for managing the encryption keys used to encrypt and decrypt data. The following table summarizes the available KMS configuration options followed by an example of how to use them. @@ -142,15 +157,15 @@ options followed by an example of how to use them. | | :setting:`django:DATABASES` setting. | +-------------------------------------------------------------------------+--------------------------------------------------------+ | :class:`kms_providers ` | A dictionary of KMS provider credentials used to | -| | access the KMS with | -| | :setting:`KMS_CREDENTIALS `. | +| | access the KMS with ``kms_provider``. | +-------------------------------------------------------------------------+--------------------------------------------------------+ -| ``kms_provider`` | A single KMS provider name | +| :ref:`kms_provider ` | A single KMS provider name | | | configured in your custom database | | | router. | +-------------------------------------------------------------------------+--------------------------------------------------------+ -Example of KMS configuration with AWS KMS: +Example of KMS configuration with ``aws`` in your :class:`kms_providers +` setting: .. code-block:: python @@ -158,22 +173,17 @@ Example of KMS configuration with AWS KMS: DATABASES = { "encrypted": { - "ENGINE": "django_mongodb_backend", - "HOST": "mongodb+srv://cluster0.example.mongodb.net", - "NAME": "encrypted", - "USER": "my_user", - "PASSWORD": "my_password", - "PORT": 27017, + # ... "OPTIONS": { "auto_encryption_opts": AutoEncryptionOpts( - key_vault_namespace="encryption.__keyVault", + # ... kms_providers={ "aws": { "accessKeyId": "your-access-key-id", "secretAccessKey": "your-secret-access-key", - } + }, }, - ) + ), }, "KMS_CREDENTIALS": { "aws": { @@ -184,6 +194,10 @@ Example of KMS configuration with AWS KMS: }, } +In your :ref:`custom database router `, +specify the KMS provider to use for the models in your application: + +.. code-block:: python class EncryptedRouter: # ... @@ -192,25 +206,25 @@ Example of KMS configuration with AWS KMS: .. _qe-configuring-encrypted-fields-map: -Configuring the ``encrypted_fields_map`` -======================================== +Configuring the ``encrypted_fields_map`` option +=============================================== -When you :ref:`configure an encrypted database connection -` without specifying an +When you configure the :ref:`DATABASES ` +setting for Queryable Encryption *without* specifying an ``encrypted_fields_map``, Django MongoDB Backend will create encrypted -collections for you when you run ``python manage.py migrate --database -encrypted``. +collections, including encryption keys, when you :ref:`run migrations for models +that have encrypted fields `. -Encryption keys for encrypted fields are stored in the key vault -:ref:`specified in the Django settings `. To see the keys -created by Django MongoDB Backend, along with the entire schema, you can run the +Encryption keys for encrypted fields are stored in the key vault specified in +the :ref:`DATABASES ` setting. To see the keys created by +Django MongoDB Backend, along with the entire schema, you can run the :djadmin:`showencryptedfieldsmap` command:: $ python manage.py showencryptedfieldsmap --database encrypted -Use the output of the :djadmin:`showencryptedfieldsmap` command to set the -``encrypted_fields_map`` in -:class:`pymongo.encryption_options.AutoEncryptionOpts` in your Django settings. +Use the output of :djadmin:`showencryptedfieldsmap` to set the +``encrypted_fields_map`` in :class:`AutoEncryptionOpts +` in your Django settings. .. code-block:: python @@ -219,21 +233,10 @@ Use the output of the :djadmin:`showencryptedfieldsmap` command to set the DATABASES = { "encrypted": { - "ENGINE": "django_mongodb_backend", - "HOST": "mongodb+srv://cluster0.example.mongodb.net", - "NAME": "encrypted", - "USER": "my_user", - "PASSWORD": "my_password", - "PORT": 27017, + # ... "OPTIONS": { "auto_encryption_opts": AutoEncryptionOpts( - key_vault_namespace="encryption.__keyVault", - kms_providers={ - "aws": { - "accessKeyId": "your-access-key-id", - "secretAccessKey": "your-secret-access-key", - } - }, + # ... encrypted_fields_map=json_util.loads( """{ "encrypt_patient": { @@ -260,6 +263,13 @@ Use the output of the :djadmin:`showencryptedfieldsmap` command to set the }, } + +.. admonition:: Security consideration + + Supplying an encrypted fields map provides more security than relying on an + encrypted fields map obtained from the server. It protects against a + malicious server advertising a false encrypted fields map. + Configuring the Automatic Encryption Shared Library =================================================== @@ -275,8 +285,10 @@ You can :ref:`download the shared library ` from the :ref:`manual:enterprise-official-packages` and configure it in your Django settings using the ``crypt_shared_lib_path`` option in -:class:`pymongo.encryption_options.AutoEncryptionOpts`. The following example -shows how to configure the shared library in your Django settings: +:class:`AutoEncryptionOpts `. + +The following example shows how to configure the shared library in your Django +settings: .. code-block:: python @@ -284,51 +296,14 @@ shows how to configure the shared library in your Django settings: DATABASES = { "encrypted": { - "ENGINE": "django_mongodb_backend", - "HOST": "mongodb+srv://cluster0.example.mongodb.net", - "NAME": "encrypted", - "USER": "my_user", - "PASSWORD": "my_password", - "PORT": 27017, + # ... "OPTIONS": { "auto_encryption_opts": AutoEncryptionOpts( - key_vault_namespace="encryption.__keyVault", - kms_providers={ - "aws": { - "accessKeyId": "your-access-key-id", - "secretAccessKey": "your-secret-access-key", - } - }, - encrypted_fields_map=json_util.loads( - """{ - "encrypt_patient": { - "fields": [ - { - "bsonType": "string", - "path": "patient_record.ssn", - "keyId": { - "$binary": { - "base64": "2MA29LaARIOqymYHGmi2mQ==", - "subType": "04" - } - }, - "queries": { - "queryType": "equality" - } - }, - ] - } - }""" - ), + # ... crypt_shared_lib_path="/path/to/mongo_crypt_shared_v1.dylib", ) }, - "KMS_CREDENTIALS": { - "aws": { - "key": os.getenv("AWS_KEY_ARN", ""), - "region": os.getenv("AWS_KEY_REGION", ""), - }, - }, + # ... }, } diff --git a/docs/ref/models/encrypted-fields.rst b/docs/ref/models/encrypted-fields.rst index 07a17b94c..e8158a0f7 100644 --- a/docs/ref/models/encrypted-fields.rst +++ b/docs/ref/models/encrypted-fields.rst @@ -72,8 +72,8 @@ for use with Queryable Encryption. | ``EncryptedObjectIdField`` | :class:`~.fields.ObjectIdField` | +----------------------------------------+------------------------------------------------------+ -The following fields are supported by Django MongoDB Backend but not by -Queryable Encryption. +The following fields are supported by Django MongoDB Backend but not supported +by Queryable Encryption. +--------------------------------------+--------------------------------------------------------------------------------------------------------------------+ | Field | Limitation | diff --git a/docs/topics/queryable-encryption.rst b/docs/topics/queryable-encryption.rst index ff6d1e3a8..713a27490 100644 --- a/docs/topics/queryable-encryption.rst +++ b/docs/topics/queryable-encryption.rst @@ -55,13 +55,27 @@ Django: cc_type = models.CharField(max_length=50) cc_number = models.CharField(max_length=20) +.. _qe-migrations: -Once you have defined your models, create the migrations with ``python manage.py -makemigrations`` and run the migrations with ``python manage.py migrate ---database encrypted``. Then create and manipulate instances of the data just -like any other Django model data. The fields will automatically handle -encryption and decryption, ensuring that :ref:`sensitive data is stored securely -in the database `. +Migrations +---------- + +Once you have defined your models, create migrations with: + +.. code-block:: console + + $ python manage.py makemigrations + +Then run the migrations with: + +.. code-block:: console + + $ python manage.py migrate --database encrypted + +Now create and manipulate instances of the data just like any other Django +model data. The fields will automatically handle encryption and decryption, +ensuring that :ref:`sensitive data is stored securely in the database +`. Querying encrypted fields ------------------------- @@ -91,14 +105,12 @@ Available query types ~~~~~~~~~~~~~~~~~~~~~ The ``queries`` option should be a dictionary that specifies the type of queries -that can be performed on the field. The :ref:`available query types -` are as follows: - -- ``equality``: Supports equality queries. -- ``range``: Supports range queries. +that can be performed on the field. Of the :ref:`available query types +` Django MongoDB Backend currently +supports: -You can configure an encrypted field for either equality or range queries, but -not both. +- ``equality`` +- ``range`` .. admonition:: Query types vs. Django lookups From 531e673ba04b85e7918f886ef640a4496f1add71 Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Thu, 23 Oct 2025 16:12:13 -0400 Subject: [PATCH 08/16] Include a note about db routers in QE topic guide --- docs/topics/queryable-encryption.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/topics/queryable-encryption.rst b/docs/topics/queryable-encryption.rst index 713a27490..c31513fd9 100644 --- a/docs/topics/queryable-encryption.rst +++ b/docs/topics/queryable-encryption.rst @@ -77,6 +77,24 @@ model data. The fields will automatically handle encryption and decryption, ensuring that :ref:`sensitive data is stored securely in the database `. +Routers +------- + +The example above requires a :ref:`database router +` to direct operations on models with +encrypted fields to the appropriate database. It also requires the use of a +:ref:`router for embedded models `. Here +is an example that includes both: + +.. code-block:: python + + # myproject/settings.py + DATABASE_ROUTERS = [ + "django_mongodb_backend.routers.MongoRouter", + "myproject.routers.EncryptedRouter", + ] + + Querying encrypted fields ------------------------- From 43b8b959756c4a0bec25a6f993a8a23391840144 Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Fri, 24 Oct 2025 12:29:53 -0400 Subject: [PATCH 09/16] Add EncryptedModelAdmin for encrypted fields Paginator not working yet. --- django_mongodb_backend/admin.py | 32 +++++++++++++++++ docs/howto/queryable-encryption.rst | 21 ++++++++++++ docs/ref/contrib/admin.rst | 53 +++++++++++++++++++++++++++++ docs/ref/contrib/index.rst | 1 + docs/ref/utils.rst | 3 +- 5 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 django_mongodb_backend/admin.py create mode 100644 docs/ref/contrib/admin.rst diff --git a/django_mongodb_backend/admin.py b/django_mongodb_backend/admin.py new file mode 100644 index 000000000..bd720532d --- /dev/null +++ b/django_mongodb_backend/admin.py @@ -0,0 +1,32 @@ +from django.contrib import admin +from django.contrib.admin.views.main import ChangeList + + +class EncryptedPaginator: + # TODO: Implement pagination for encrypted data. This paginator + # currently returns all results in a single page. + def __init__(self, queryset, per_page): + self.queryset = queryset + self.per_page = per_page + + def page(self, number): + results = list(self.queryset) + has_next = False + return results, has_next + + +class EncryptedChangeList(ChangeList): + def get_results(self, request): + paginator = EncryptedPaginator(self.queryset, self.list_per_page) + self.result_list, _ = paginator.page(self.page_num + 1) + + self.result_count = len(self.result_list) + self.full_result_count = self.result_count + + self.can_show_all = True + self.multi_page = False + + +class EncryptedModelAdmin(admin.ModelAdmin): + def get_changelist(self, request, **kwargs): + return EncryptedChangeList diff --git a/docs/howto/queryable-encryption.rst b/docs/howto/queryable-encryption.rst index 1d0b7ce85..2ef0a0870 100644 --- a/docs/howto/queryable-encryption.rst +++ b/docs/howto/queryable-encryption.rst @@ -307,5 +307,26 @@ settings: }, } +Configuring the ``EncryptedModelAdmin`` +======================================= + +When using the :doc:`the Django admin site ` +with models that have encrypted fields, use the :class:`EncryptedModelAdmin` +class to ensure that encrypted fields are handled correctly. To do this, inherit +from :class:`EncryptedModelAdmin` in your admin classes instead of the standard +:class:`~django.contrib.admin.ModelAdmin`. + +.. code-block:: python + + # myapp/admin.py + from django.contrib import admin + from .models import Patient + from django_mongodb_backend.admin import EncryptedModelAdmin + + + @admin.register(Patient) + class PatientAdmin(EncryptedModelAdmin): + pass + You are now ready to :doc:`start developing applications ` with Queryable Encryption! diff --git a/docs/ref/contrib/admin.rst b/docs/ref/contrib/admin.rst new file mode 100644 index 000000000..eaea6f921 --- /dev/null +++ b/docs/ref/contrib/admin.rst @@ -0,0 +1,53 @@ +===== +Admin +===== + +Django MongoDB Backend supports the Django admin interface. To enable it, ensure +that you have :ref:`specified the default pk field +` for the +:class:`~django.contrib.admin.apps.AdminConfig` class as described in the +:doc:`Getting Started ` guide. + +``EncryptedModelAdmin`` +======================= + +.. class:: EncryptedModelAdmin + + .. versionadded:: 5.2.3 + + A :class:`~django.contrib.admin.ModelAdmin` subclass that supports models + with encrypted fields. Use this class as a base class for your model's admin + class to ensure that encrypted fields are handled correctly in the admin + interface. + + Define a model with encrypted fields: + + .. code-block:: python + + # myapp/models.py + from django.db import models + from django_mongodb_backend.fields import EmbeddedModelField + + + class Patient(models.Model): + patient_name = models.CharField(max_length=255) + patient_id = models.BigIntegerField() + patient_record = EmbeddedModelField("PatientRecord") + + def __str__(self): + return f"{self.patient_name} ({self.patient_id})" + + Register it with the Django admin using the ``EncryptedModelAdmin`` as shown + below: + + .. code-block:: python + + # myapp/admin.py + from django.contrib import admin + from django_mongodb_backend.admin import EncryptedModelAdmin + from .models import Patient + + + @admin.register(Patient) + class PatientAdmin(EncryptedModelAdmin): + pass diff --git a/docs/ref/contrib/index.rst b/docs/ref/contrib/index.rst index 82cb1b88f..044f059c2 100644 --- a/docs/ref/contrib/index.rst +++ b/docs/ref/contrib/index.rst @@ -7,4 +7,5 @@ Notes for Django's :doc:`django:ref/contrib/index` live here. .. toctree:: :maxdepth: 1 + admin gis diff --git a/docs/ref/utils.rst b/docs/ref/utils.rst index 18021d261..cf6d9c81c 100644 --- a/docs/ref/utils.rst +++ b/docs/ref/utils.rst @@ -49,13 +49,14 @@ following parts can be considered stable. But for maximum flexibility, construct :setting:`DATABASES` manually as described in :ref:`configuring-databases-setting`. -.. versionadded:: 5.2.3 ``model_has_encrypted_fields()`` ================================= .. function:: model_has_encrypted_fields(model) + .. versionadded:: 5.2.3 + Returns ``True`` if the given Django model has any fields that use encrypted models. From 746aeab2781cd052a2b2ffbb580a1fb05de4cae8 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Fri, 24 Oct 2025 17:20:40 -0400 Subject: [PATCH 10/16] simplify create_data_key() call --- django_mongodb_backend/schema.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django_mongodb_backend/schema.py b/django_mongodb_backend/schema.py index 3ab83d540..4c5863896 100644 --- a/django_mongodb_backend/schema.py +++ b/django_mongodb_backend/schema.py @@ -500,7 +500,7 @@ def _get_encrypted_fields(self, model, key_alt_name=None, path_prefix=None): ) kms_provider = router.kms_provider(model) - master_key = connection.settings_dict.get("KMS_CREDENTIALS", {}).get(kms_provider) + master_key = connection.settings_dict.get("KMS_CREDENTIALS").get(kms_provider) client_encryption = self.connection.client_encryption field_list = [] @@ -527,8 +527,8 @@ def _get_encrypted_fields(self, model, key_alt_name=None, path_prefix=None): else: data_key = client_encryption.create_data_key( kms_provider=kms_provider, - master_key=master_key, key_alt_names=[new_key_alt_name], + master_key=master_key, ) field_dict = { "bsonType": bson_type, From 9a9d4cc421702f1bdfaf74df37ce5fd5f0c44210 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Fri, 24 Oct 2025 17:31:35 -0400 Subject: [PATCH 11/16] Make router.kms_provider() unneeded if only a single provider is configured --- django_mongodb_backend/schema.py | 11 +++++++++-- docs/howto/queryable-encryption.rst | 11 +++++------ docs/ref/utils.rst | 5 ----- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/django_mongodb_backend/schema.py b/django_mongodb_backend/schema.py index 4c5863896..49e0e22f6 100644 --- a/django_mongodb_backend/schema.py +++ b/django_mongodb_backend/schema.py @@ -498,8 +498,15 @@ def _get_encrypted_fields(self, model, key_alt_name=None, path_prefix=None): key_vault_collection.create_index( "keyAltNames", unique=True, partialFilterExpression={"keyAltNames": {"$exists": True}} ) - - kms_provider = router.kms_provider(model) + # Select the KMS provider. + kms_providers = auto_encryption_opts._kms_providers + if len(kms_providers) == 1: + # If one provider is configured, no need to consult the router. + kms_provider = next(iter(kms_providers.keys())) + else: + # Otherwise, call the user-defined router.kms_provider(). + kms_provider = router.kms_provider(model) + # Providing master_key raises an error for the local provider. master_key = connection.settings_dict.get("KMS_CREDENTIALS").get(kms_provider) client_encryption = self.connection.client_encryption diff --git a/docs/howto/queryable-encryption.rst b/docs/howto/queryable-encryption.rst index 2ef0a0870..914499440 100644 --- a/docs/howto/queryable-encryption.rst +++ b/docs/howto/queryable-encryption.rst @@ -124,9 +124,6 @@ models in that application. The router also specifies the :ref:`KMS provider return "encrypted" return None - def kms_provider(self, model, **hints): - return "local" - db_for_write = db_for_read Then in your Django settings, add the custom database router to the @@ -194,10 +191,12 @@ Example of KMS configuration with ``aws`` in your :class:`kms_providers }, } -In your :ref:`custom database router `, -specify the KMS provider to use for the models in your application: +(TODO: If there's a use case for multiple providers, motivate with a use case +and add a test.) -.. code-block:: python +If you've configured multiple KMS providers, you must define logic to determine +the provider for each model in your :ref:`database router +`:: class EncryptedRouter: # ... diff --git a/docs/ref/utils.rst b/docs/ref/utils.rst index cf6d9c81c..30bc7b1b5 100644 --- a/docs/ref/utils.rst +++ b/docs/ref/utils.rst @@ -83,8 +83,3 @@ following parts can be considered stable. else: return db == "default" return None - - def kms_provider(self, model): - if model_has_encrypted_fields(model): - return "local" - return None From d22bb82c18fc47096083dd68541c7159035f38f7 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Fri, 24 Oct 2025 17:44:12 -0400 Subject: [PATCH 12/16] cosmetic edits to _get_encrypted_fields() --- django_mongodb_backend/schema.py | 55 ++++++++++++++------------------ 1 file changed, 24 insertions(+), 31 deletions(-) diff --git a/django_mongodb_backend/schema.py b/django_mongodb_backend/schema.py index 49e0e22f6..96cda7a6a 100644 --- a/django_mongodb_backend/schema.py +++ b/django_mongodb_backend/schema.py @@ -477,24 +477,22 @@ def _create_collection(self, model): # Unencrypted path db.create_collection(db_table) - def _get_encrypted_fields(self, model, key_alt_name=None, path_prefix=None): + def _get_encrypted_fields(self, model, key_alt_name_prefix=None, path_prefix=None): """ - Recursively collect encryption schema data for only encrypted fields in a model. - Returns None if no encrypted fields are found anywhere in the model hierarchy. + Return the encrypted fields map for the given model. The "prefix" + arguments are used when this method is called recursively on embedded + models. """ connection = self.connection client = connection.connection - fields = model._meta.fields - key_alt_name = key_alt_name or model._meta.db_table + key_alt_name_prefix = key_alt_name_prefix or model._meta.db_table path_prefix = path_prefix or "" - - options = client._options - auto_encryption_opts = options.auto_encryption_opts - - key_vault_db, key_vault_coll = auto_encryption_opts._key_vault_namespace.split(".", 1) - key_vault_collection = client[key_vault_db][key_vault_coll] - - # Create partial unique index on keyAltNames + auto_encryption_opts = client._options.auto_encryption_opts + key_vault_db, key_vault_collection = auto_encryption_opts._key_vault_namespace.split(".", 1) + key_vault_collection = client[key_vault_db][key_vault_collection] + # Create partial unique index on keyAltNames. + # TODO: find a better place for this. It only needs to run once for an + # application's lifetime. key_vault_collection.create_index( "keyAltNames", unique=True, partialFilterExpression={"keyAltNames": {"$exists": True}} ) @@ -506,48 +504,43 @@ def _get_encrypted_fields(self, model, key_alt_name=None, path_prefix=None): else: # Otherwise, call the user-defined router.kms_provider(). kms_provider = router.kms_provider(model) - # Providing master_key raises an error for the local provider. master_key = connection.settings_dict.get("KMS_CREDENTIALS").get(kms_provider) - client_encryption = self.connection.client_encryption - + # Generate the encrypted fields map. field_list = [] - - for field in fields: - new_key_alt_name = f"{key_alt_name}.{field.column}" + for field in model._meta.fields: + key_alt_name = f"{key_alt_name_prefix}.{field.column}" path = f"{path_prefix}.{field.column}" if path_prefix else field.column - + # Check non-encrypted EmbeddedModelFields for encrypted fields. if isinstance(field, EmbeddedModelField) and not getattr(field, "encrypted", False): embedded_result = self._get_encrypted_fields( field.embedded_model, - key_alt_name=new_key_alt_name, + key_alt_name_prefix=key_alt_name, path_prefix=path, ) + # An EmbeddedModelField may not have any encrypted fields. if embedded_result: field_list.extend(embedded_result["fields"]) continue - + # Populate data for encrypted field. if getattr(field, "encrypted", False): - bson_type = field.db_type(connection) - data_key = key_vault_collection.find_one({"keyAltNames": new_key_alt_name}) + data_key = key_vault_collection.find_one({"keyAltNames": key_alt_name}) if data_key: data_key = data_key["_id"] else: - data_key = client_encryption.create_data_key( + data_key = connection.client_encryption.create_data_key( kms_provider=kms_provider, - key_alt_names=[new_key_alt_name], + key_alt_names=[key_alt_name], master_key=master_key, ) field_dict = { - "bsonType": bson_type, + "bsonType": field.db_type(connection), "path": path, "keyId": data_key, } - queries = getattr(field, "queries", None) - if queries: + if queries := getattr(field, "queries", None): field_dict["queries"] = queries field_list.append(field_dict) - - return {"fields": field_list} if field_list else None + return {"fields": field_list} # GISSchemaEditor extends some SchemaEditor methods. From 51ac968f911a38d7cedc27afb962024d9864d4a4 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Fri, 24 Oct 2025 20:13:42 -0400 Subject: [PATCH 13/16] Use ORM to manage encryption keys in tests --- tests/encryption_/models.py | 8 ++++++++ tests/encryption_/test_schema.py | 31 ++++++++++++------------------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/tests/encryption_/models.py b/tests/encryption_/models.py index 930794c39..c266d37dc 100644 --- a/tests/encryption_/models.py +++ b/tests/encryption_/models.py @@ -165,3 +165,11 @@ class SmallIntegerModel(EncryptedTestModel): class TimeModel(EncryptedTestModel): value = EncryptedTimeField(queries={"queryType": "range"}) + + +class EncryptionKey(models.Model): + key_alt_name = models.CharField(max_length=500, db_column="keyAltNames") + + class Meta: + db_table = "__keyVault" + managed = False diff --git a/tests/encryption_/test_schema.py b/tests/encryption_/test_schema.py index d7cfb5a06..f00c6ecbf 100644 --- a/tests/encryption_/test_schema.py +++ b/tests/encryption_/test_schema.py @@ -2,6 +2,7 @@ from django.db import connections from . import models +from .models import EncryptionKey from .test_base import EncryptionTestCase @@ -113,28 +114,20 @@ def test_key_creation_and_lookup(self): generate and store a data key in the vault, then query the vault with the keyAltName. """ - connection = connections["encrypted"] - client = connection.connection - auto_encryption_opts = client._options.auto_encryption_opts - - key_vault_db, key_vault_coll = auto_encryption_opts._key_vault_namespace.split(".", 1) - vault_coll = client[key_vault_db][key_vault_coll] - model_class = models.CharModel test_key_alt_name = f"{model_class._meta.db_table}.value" - vault_coll.delete_many({"keyAltNames": test_key_alt_name}) - - with connection.schema_editor() as editor: + # Delete the test key and verify it's gone. + EncryptionKey.objects.filter(key_alt_name=test_key_alt_name).delete() + with self.assertRaises(EncryptionKey.DoesNotExist): + EncryptionKey.objects.get(key_alt_name=test_key_alt_name) + # Regenerate the keyId. + with connections["encrypted"].schema_editor() as editor: encrypted_fields = editor._get_encrypted_fields(model_class) - - # Validate schema contains a keyId for our field - self.assertTrue(encrypted_fields["fields"]) + # Validate schema contains a keyId for the field. field_info = encrypted_fields["fields"][0] self.assertEqual(field_info["path"], "value") self.assertIsInstance(field_info["keyId"], Binary) - - # Lookup in key vault by the keyAltName created - key_doc = vault_coll.find_one({"keyAltNames": test_key_alt_name}) - self.assertIsNotNone(key_doc, "Key should exist in vault") - self.assertEqual(key_doc["_id"], field_info["keyId"]) - self.assertIn(test_key_alt_name, key_doc["keyAltNames"]) + # Lookup in key vault by the keyAltName. + key = EncryptionKey.objects.get(key_alt_name=test_key_alt_name) + self.assertEqual(key.id, field_info["keyId"]) + self.assertEqual(key.key_alt_name, [test_key_alt_name]) From 9f9507d513913a1df4b6c9ce0539940d3ad1fac5 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Sat, 25 Oct 2025 15:40:25 -0400 Subject: [PATCH 14/16] Prevent showencryptedfieldsmap from creating data keys --- .../commands/showencryptedfieldsmap.py | 2 +- django_mongodb_backend/schema.py | 25 +++++++++++++------ django_mongodb_backend/utils.py | 1 + tests/encryption_/test_management.py | 22 +++++++++++++++- tests/encryption_/test_schema.py | 11 +++++--- 5 files changed, 47 insertions(+), 14 deletions(-) diff --git a/django_mongodb_backend/management/commands/showencryptedfieldsmap.py b/django_mongodb_backend/management/commands/showencryptedfieldsmap.py index 5cd864c77..2873eadca 100644 --- a/django_mongodb_backend/management/commands/showencryptedfieldsmap.py +++ b/django_mongodb_backend/management/commands/showencryptedfieldsmap.py @@ -30,6 +30,6 @@ def handle(self, *args, **options): for app_config in apps.get_app_configs(): for model in router.get_migratable_models(app_config, db): if model_has_encrypted_fields(model): - fields = editor._get_encrypted_fields(model) + fields = editor._get_encrypted_fields(model, create_data_keys=False) encrypted_fields_map[model._meta.db_table] = fields self.stdout.write(json_util.dumps(encrypted_fields_map, indent=2)) diff --git a/django_mongodb_backend/schema.py b/django_mongodb_backend/schema.py index 96cda7a6a..dc021bae2 100644 --- a/django_mongodb_backend/schema.py +++ b/django_mongodb_backend/schema.py @@ -477,7 +477,9 @@ def _create_collection(self, model): # Unencrypted path db.create_collection(db_table) - def _get_encrypted_fields(self, model, key_alt_name_prefix=None, path_prefix=None): + def _get_encrypted_fields( + self, model, *, key_alt_name_prefix=None, path_prefix=None, create_data_keys=True + ): """ Return the encrypted fields map for the given model. The "prefix" arguments are used when this method is called recursively on embedded @@ -488,12 +490,12 @@ def _get_encrypted_fields(self, model, key_alt_name_prefix=None, path_prefix=Non key_alt_name_prefix = key_alt_name_prefix or model._meta.db_table path_prefix = path_prefix or "" auto_encryption_opts = client._options.auto_encryption_opts - key_vault_db, key_vault_collection = auto_encryption_opts._key_vault_namespace.split(".", 1) - key_vault_collection = client[key_vault_db][key_vault_collection] + _, key_vault_collection = auto_encryption_opts._key_vault_namespace.split(".", 1) + key_vault = self.get_collection(key_vault_collection) # Create partial unique index on keyAltNames. # TODO: find a better place for this. It only needs to run once for an # application's lifetime. - key_vault_collection.create_index( + key_vault.create_index( "keyAltNames", unique=True, partialFilterExpression={"keyAltNames": {"$exists": True}} ) # Select the KMS provider. @@ -516,6 +518,7 @@ def _get_encrypted_fields(self, model, key_alt_name_prefix=None, path_prefix=Non field.embedded_model, key_alt_name_prefix=key_alt_name, path_prefix=path, + create_data_keys=create_data_keys, ) # An EmbeddedModelField may not have any encrypted fields. if embedded_result: @@ -523,15 +526,21 @@ def _get_encrypted_fields(self, model, key_alt_name_prefix=None, path_prefix=Non continue # Populate data for encrypted field. if getattr(field, "encrypted", False): - data_key = key_vault_collection.find_one({"keyAltNames": key_alt_name}) - if data_key: - data_key = data_key["_id"] - else: + if create_data_keys: data_key = connection.client_encryption.create_data_key( kms_provider=kms_provider, key_alt_names=[key_alt_name], master_key=master_key, ) + else: + data_key = key_vault.find_one({"keyAltNames": key_alt_name}) + if data_key: + data_key = data_key["_id"] + else: + raise ImproperlyConfigured( + f"Encryption key {key_alt_name} not found. Have " + f"migrated the {model} model?" + ) field_dict = { "bsonType": field.db_type(connection), "path": path, diff --git a/django_mongodb_backend/utils.py b/django_mongodb_backend/utils.py index 2afaaba0e..c655c8bc0 100644 --- a/django_mongodb_backend/utils.py +++ b/django_mongodb_backend/utils.py @@ -118,6 +118,7 @@ class OperationDebugWrapper: "create_indexes", "create_search_index", "drop", + "find_one", "index_information", "insert_many", "delete_many", diff --git a/tests/encryption_/test_management.py b/tests/encryption_/test_management.py index ceaa4fca9..c43092039 100644 --- a/tests/encryption_/test_management.py +++ b/tests/encryption_/test_management.py @@ -1,9 +1,12 @@ from io import StringIO from bson import json_util +from django.core.exceptions import ImproperlyConfigured from django.core.management import call_command +from django.db import connections from django.test import modify_settings +from .models import EncryptionKey from .test_base import EncryptionTestCase @@ -96,7 +99,7 @@ class CommandTests(EncryptionTestCase): def _compare_output(self, expected, actual): for field in actual["fields"]: - field.pop("keyId", None) # remove dynamic keyId + field.pop("keyId") # remove dynamic keyId self.assertEqual(expected, actual) def test_show_encrypted_fields_map(self): @@ -109,3 +112,20 @@ def test_show_encrypted_fields_map(self): with self.subTest(model=model_key): self.assertIn(model_key, command_output) self._compare_output(expected, command_output[model_key]) + + def test_missing_key(self): + test_key = "encryption__patient.patient_record.ssn" + msg = ( + f"Encryption key {test_key} not found. Have migrated the " + " model?" + ) + EncryptionKey.objects.filter(key_alt_name=test_key).delete() + try: + with self.assertRaisesMessage(ImproperlyConfigured, msg): + call_command("showencryptedfieldsmap", "--database", "encrypted", verbosity=0) + finally: + # Replace the deleted key. + connections["encrypted"].client_encryption.create_data_key( + kms_provider="local", + key_alt_names=[test_key], + ) diff --git a/tests/encryption_/test_schema.py b/tests/encryption_/test_schema.py index f00c6ecbf..0fc42f9a3 100644 --- a/tests/encryption_/test_schema.py +++ b/tests/encryption_/test_schema.py @@ -96,16 +96,19 @@ def test_get_encrypted_fields_all_models(self): checks their encrypted fields map from the schema editor, and compares to expected BSON type & queries mapping. """ + # Deleting all keys is only correct only if this test includes all + # test models. This test may not be needed since it's tested when the + # test runner migrates all models. If any subTest fails, the key vault + # will be left in an inconsistent state. + EncryptionKey.objects.all().delete() connection = connections["encrypted"] - for model_name, expected in self.expected_map.items(): with self.subTest(model=model_name): model_class = getattr(models, model_name) with connection.schema_editor() as editor: - client = connection.connection - encrypted_fields = editor._get_encrypted_fields(model_class, client) + encrypted_fields = editor._get_encrypted_fields(model_class) for field in encrypted_fields["fields"]: - field.pop("keyId", None) # Remove dynamic value + field.pop("keyId") # Remove dynamic value self.assertEqual(encrypted_fields, expected) def test_key_creation_and_lookup(self): From ad70f5964b79bb8250a8b2d767c95d638c584aef Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Tue, 28 Oct 2025 18:58:35 -0400 Subject: [PATCH 15/16] Fix pagination Existing approaches to "no count" pagination do not seem to include support for pagination in the Django admin, so: - Subclass Django's core Paginator - override count - Subclass Django's contrib admin view ChangeList.get_results - Use len instead of count - Ruff fixes for try/except - Subclass Django's contrib ModelAdmin - override get_paginator - override get_changelist --- django_mongodb_backend/admin.py | 60 +++++++++++++++++++++++---------- 1 file changed, 42 insertions(+), 18 deletions(-) diff --git a/django_mongodb_backend/admin.py b/django_mongodb_backend/admin.py index bd720532d..20c04a5f3 100644 --- a/django_mongodb_backend/admin.py +++ b/django_mongodb_backend/admin.py @@ -1,32 +1,56 @@ from django.contrib import admin +from django.contrib.admin.options import IncorrectLookupParameters from django.contrib.admin.views.main import ChangeList +from django.core.paginator import InvalidPage, Paginator +from django.utils.functional import cached_property -class EncryptedPaginator: - # TODO: Implement pagination for encrypted data. This paginator - # currently returns all results in a single page. - def __init__(self, queryset, per_page): - self.queryset = queryset - self.per_page = per_page - - def page(self, number): - results = list(self.queryset) - has_next = False - return results, has_next +class EncryptedPaginator(Paginator): + @cached_property + def count(self): + return len(self.object_list) class EncryptedChangeList(ChangeList): def get_results(self, request): - paginator = EncryptedPaginator(self.queryset, self.list_per_page) - self.result_list, _ = paginator.page(self.page_num + 1) + """ + This is django.contrib.admin.views.main.ChangeList.get_results with + a single modification to avoid COUNT queries. + """ + paginator = self.model_admin.get_paginator(request, self.queryset, self.list_per_page) + result_count = paginator.count + if self.model_admin.show_full_result_count: + # Modification: avoid COUNT query by using len() on the root queryset + full_result_count = len(self.root_queryset) + else: + full_result_count = None + can_show_all = result_count <= self.list_max_show_all + multi_page = result_count > self.list_per_page + if (self.show_all and can_show_all) or not multi_page: + result_list = self.queryset._clone() + else: + try: + result_list = paginator.page(self.page_num).object_list + except InvalidPage as err: + raise IncorrectLookupParameters from err + self.result_count = result_count + self.show_full_result_count = self.model_admin.show_full_result_count + self.show_admin_actions = not self.show_full_result_count or bool(full_result_count) + self.full_result_count = full_result_count + self.result_list = result_list + self.can_show_all = can_show_all + self.multi_page = multi_page + self.paginator = paginator - self.result_count = len(self.result_list) - self.full_result_count = self.result_count - self.can_show_all = True - self.multi_page = False +class EncryptedModelAdmin(admin.ModelAdmin): + """ + A ModelAdmin that uses EncryptedPaginator and EncryptedChangeList + to avoid COUNT queries in the admin changelist. + """ + def get_paginator(self, request, queryset, per_page): + return EncryptedPaginator(queryset, per_page) -class EncryptedModelAdmin(admin.ModelAdmin): def get_changelist(self, request, **kwargs): return EncryptedChangeList From b0f79ef33b3d8c85afeb7404a95835e06abc652f Mon Sep 17 00:00:00 2001 From: "Jeffrey A. Clark" Date: Tue, 28 Oct 2025 22:21:34 -0400 Subject: [PATCH 16/16] Code review fixes --- django_mongodb_backend/base.py | 4 ---- django_mongodb_backend/features.py | 4 ---- .../commands/showencryptedfieldsmap.py | 2 +- docs/ref/contrib/admin.rst | 23 ++----------------- 4 files changed, 3 insertions(+), 30 deletions(-) diff --git a/django_mongodb_backend/base.py b/django_mongodb_backend/base.py index 1284d9e0f..b44152741 100644 --- a/django_mongodb_backend/base.py +++ b/django_mongodb_backend/base.py @@ -336,10 +336,6 @@ def cursor(self): def get_database_version(self): """Return a tuple of the database's version.""" - # TODO: Remove this workaround and replace with - # `tuple(self.connection.server_info()["versionArray"])` when the minimum - # supported version of pymongocrypt is >= 1.14.2 and PYTHON-5429 is resolved. - # See: https://jira.mongodb.org/browse/PYTHON-5429 return tuple(self.connection.admin.command("buildInfo")["versionArray"]) ## Transaction API for django_mongodb_backend.transaction.atomic() diff --git a/django_mongodb_backend/features.py b/django_mongodb_backend/features.py index 6f4c1e8f5..332b69d4e 100644 --- a/django_mongodb_backend/features.py +++ b/django_mongodb_backend/features.py @@ -596,10 +596,6 @@ def mongodb_version(self): def is_mongodb_6_3(self): return self.mongodb_version >= (6, 3) - @cached_property - def is_mongodb_7_0(self): - return self.mongodb_version >= (7, 0) - @cached_property def is_mongodb_8_0(self): return self.mongodb_version >= (8, 0) diff --git a/django_mongodb_backend/management/commands/showencryptedfieldsmap.py b/django_mongodb_backend/management/commands/showencryptedfieldsmap.py index 2873eadca..017fabde5 100644 --- a/django_mongodb_backend/management/commands/showencryptedfieldsmap.py +++ b/django_mongodb_backend/management/commands/showencryptedfieldsmap.py @@ -32,4 +32,4 @@ def handle(self, *args, **options): if model_has_encrypted_fields(model): fields = editor._get_encrypted_fields(model, create_data_keys=False) encrypted_fields_map[model._meta.db_table] = fields - self.stdout.write(json_util.dumps(encrypted_fields_map, indent=2)) + self.stdout.write(json_util.dumps(encrypted_fields_map, indent=4)) diff --git a/docs/ref/contrib/admin.rst b/docs/ref/contrib/admin.rst index eaea6f921..d38c7b161 100644 --- a/docs/ref/contrib/admin.rst +++ b/docs/ref/contrib/admin.rst @@ -20,27 +20,8 @@ that you have :ref:`specified the default pk field class to ensure that encrypted fields are handled correctly in the admin interface. - Define a model with encrypted fields: - - .. code-block:: python - - # myapp/models.py - from django.db import models - from django_mongodb_backend.fields import EmbeddedModelField - - - class Patient(models.Model): - patient_name = models.CharField(max_length=255) - patient_id = models.BigIntegerField() - patient_record = EmbeddedModelField("PatientRecord") - - def __str__(self): - return f"{self.patient_name} ({self.patient_id})" - - Register it with the Django admin using the ``EncryptedModelAdmin`` as shown - below: - - .. code-block:: python + Register encrypted models with the Django admin using the + ``EncryptedModelAdmin`` as shown below:: # myapp/admin.py from django.contrib import admin