INTPYTHON-765 Fix crash loading embedded models with missing fields that use database converters

Author: timgraham

django_mongodb_backend/operations.py

@@ -184,6 +184,8 @@ def convert_embeddedmodelfield_value(self, value, expression, connection):
         if value is not None:
             # Apply database converters to each field of the embedded model.
             for field in expression.output_field.embedded_model._meta.fields:
+                if field.attname not in value:
+                    continue
                 field_expr = Expression(output_field=field)
                 converters = connection.ops.get_db_converters(
                     field_expr
@@ -204,6 +206,8 @@ def convert_polymorphicembeddedmodelfield_value(self, value, expression, connect
             model_class = expression.output_field._get_model_from_label(value["_label"])
             # Apply database converters to each field of the embedded model.
             for field in model_class._meta.fields:
+                if field.attname not in value:
+                    continue
                 field_expr = Expression(output_field=field)
                 converters = connection.ops.get_db_converters(
                     field_expr

docs/releases/5.2.x.rst

@@ -15,7 +15,10 @@ New features
 Bug fixes
 ---------
 
-- ...
+- Fixed a ``KeyError`` crash when loading models with embedded model subfields
+  that use a database converter, if the subfield isn't present in the data
+  (e.g. data not written by Django, or after a field was added to an existing
+  embedded model).
 
 Deprecated features
 -------------------

tests/model_fields_/test_embedded_model.py

@@ -2,7 +2,7 @@
 from datetime import timedelta
 
 from django.core.exceptions import FieldDoesNotExist, ValidationError
-from django.db import models
+from django.db import connection, models
 from django.db.models import (
     Exists,
     ExpressionWrapper,
@@ -107,6 +107,15 @@ def test_pre_save(self):
         self.assertEqual(obj.data.auto_now_add, auto_now_add)
         self.assertGreater(obj.data.auto_now, auto_now_two)
 
+    def test_missing_field_in_data(self):
+        """
+        Loading a model with an EmbeddedModelField that has a missing subfield
+        (e.g. data not written by Django) that uses a database converter (in
+        this case, integer is an IntegerField) doesn't crash.
+        """
+        connection.database.model_fields__holder.update_many({}, {"$unset": {"data.integer": ""}})
+        self.assertIsNone(Holder.objects.first().data.integer)
+
 
 class QueryingTests(TestCase):
     @classmethod

tests/model_fields_/test_embedded_model_array.py

@@ -62,6 +62,18 @@ def test_save_load_null(self):
         movie = Movie.objects.get(title="Lion King")
         self.assertIsNone(movie.reviews)
 
+    def test_missing_field_in_data(self):
+        """
+        Loading a model with an EmbeddedModelArrayField that has a missing
+        subfield (e.g. data not written by Django) that uses a database
+        converter (in this case, rating is an IntegerField) doesn't crash.
+        """
+        Movie.objects.create(title="Lion King", reviews=[Review(title="The best", rating=10)])
+        connection.database.model_fields__movie.update_many(
+            {}, {"$unset": {"reviews.$[].rating": ""}}
+        )
+        self.assertIsNone(Movie.objects.first().reviews[0].rating)
+
 
 class QueryingTests(TestCase):
     @classmethod

tests/model_fields_/test_polymorphic_embedded_model.py

@@ -2,7 +2,7 @@
 from decimal import Decimal
 
 from django.core.exceptions import FieldDoesNotExist, ValidationError
-from django.db import models
+from django.db import connection, models
 from django.test import SimpleTestCase, TestCase
 from django.test.utils import isolate_apps
 
@@ -91,6 +91,15 @@ def test_pre_save(self):
         # simultaneously.
         self.assertAlmostEqual(updated_at, created_at, delta=timedelta(microseconds=1000))
 
+    def test_missing_field_in_data(self):
+        """
+        Loading a model with a PolymorphicEmbeddedModelField that has a missing
+        subfield (e.g. data not written by Django) that uses a database
+        converter (in this case, weight is a DecimalField) doesn't crash.
+        """
+        connection.database.model_fields__person.update_many({}, {"$unset": {"pet.weight": ""}})
+        self.assertIsNone(Person.objects.first().pet.weight)
+
 
 class QueryingTests(TestCase):
     @classmethod

tests/model_fields_/test_polymorphic_embedded_model_array.py

@@ -1,7 +1,7 @@
 from decimal import Decimal
 
 from django.core.exceptions import FieldDoesNotExist
-from django.db import models
+from django.db import connection, models
 from django.test import SimpleTestCase, TestCase
 from django.test.utils import isolate_apps
 
@@ -62,6 +62,16 @@ def test_save_load_null(self):
         owner = Owner.objects.get(name="Bob")
         self.assertIsNone(owner.pets)
 
+    def test_missing_field_in_data(self):
+        """
+        Loading a model with a PolymorphicEmbeddedModelArrayField that has a
+        missing subfield (e.g. data not written by Django) that uses a database
+        converter (in this case, weight is a DecimalField) doesn't crash.
+        """
+        Owner.objects.create(name="Bob", pets=[Cat(name="Phoebe", weight="3.5")])
+        connection.database.model_fields__owner.update_many({}, {"$unset": {"pets.$[].weight": ""}})
+        self.assertIsNone(Owner.objects.first().pets[0].weight)
+
 
 class QueryingTests(TestCase):
     @classmethod