allow sorting keys on to_json and to_python by passing in sort_keys

Author: aezomz

python/pydantic_core/_pydantic_core.pyi

@@ -305,6 +305,7 @@ class SchemaSerializer:
         exclude_unset: bool = False,
         exclude_defaults: bool = False,
         exclude_none: bool = False,
+        sort_keys: bool = False,
         round_trip: bool = False,
         warnings: bool | Literal['none', 'warn', 'error'] = True,
         fallback: Callable[[Any], Any] | None = None,
@@ -326,6 +327,7 @@ class SchemaSerializer:
             exclude_defaults: Whether to exclude fields that are equal to their default value.
             exclude_none: Whether to exclude fields that have a value of `None`.
             round_trip: Whether to enable serialization and validation round-trip support.
+            sort_keys: Whether to sort dictionary keys at the root level.
             warnings: How to handle invalid fields. False/"none" ignores them, True/"warn" logs errors,
                 "error" raises a [`PydanticSerializationError`][pydantic_core.PydanticSerializationError].
             fallback: A function to call when an unknown value is encountered,
@@ -352,6 +354,7 @@ class SchemaSerializer:
         exclude_defaults: bool = False,
         exclude_none: bool = False,
         round_trip: bool = False,
+        sort_keys: bool = False,
         warnings: bool | Literal['none', 'warn', 'error'] = True,
         fallback: Callable[[Any], Any] | None = None,
         serialize_as_any: bool = False,
@@ -371,6 +374,7 @@ class SchemaSerializer:
             exclude_defaults: Whether to exclude fields that are equal to their default value.
             exclude_none: Whether to exclude fields that have a value of `None`.
             round_trip: Whether to enable serialization and validation round-trip support.
+            sort_keys: Whether to sort dictionary keys at the root level.
             warnings: How to handle invalid fields. False/"none" ignores them, True/"warn" logs errors,
                 "error" raises a [`PydanticSerializationError`][pydantic_core.PydanticSerializationError].
             fallback: A function to call when an unknown value is encountered,
@@ -395,6 +399,7 @@ def to_json(
     by_alias: bool | None = None,
     exclude_none: bool = False,
     round_trip: bool = False,
+    sort_keys: bool = False,
     timedelta_mode: Literal['iso8601', 'float'] = 'iso8601',
     bytes_mode: Literal['utf8', 'base64', 'hex'] = 'utf8',
     inf_nan_mode: Literal['null', 'constants', 'strings'] = 'constants',
@@ -416,6 +421,7 @@ def to_json(
         by_alias: Whether to use the alias names of fields.
         exclude_none: Whether to exclude fields that have a value of `None`.
         round_trip: Whether to enable serialization and validation round-trip support.
+        sort_keys: Whether to sort dictionary keys at the root level.
         timedelta_mode: How to serialize `timedelta` objects, either `'iso8601'` or `'float'`.
         bytes_mode: How to serialize `bytes` objects, either `'utf8'`, `'base64'`, or `'hex'`.
         inf_nan_mode: How to serialize `Infinity`, `-Infinity` and `NaN` values, either `'null'`, `'constants'`, or `'strings'`.
@@ -471,6 +477,7 @@ def to_jsonable_python(
     by_alias: bool | None = None,
     exclude_none: bool = False,
     round_trip: bool = False,
+    sort_keys: bool = False,
     timedelta_mode: Literal['iso8601', 'float'] = 'iso8601',
     bytes_mode: Literal['utf8', 'base64', 'hex'] = 'utf8',
     inf_nan_mode: Literal['null', 'constants', 'strings'] = 'constants',
@@ -492,6 +499,7 @@ def to_jsonable_python(
         by_alias: Whether to use the alias names of fields.
         exclude_none: Whether to exclude fields that have a value of `None`.
         round_trip: Whether to enable serialization and validation round-trip support.
+        sort_keys: Whether to sort dictionary keys at the root level.
         timedelta_mode: How to serialize `timedelta` objects, either `'iso8601'` or `'float'`.
         bytes_mode: How to serialize `bytes` objects, either `'utf8'`, `'base64'`, or `'hex'`.
         inf_nan_mode: How to serialize `Infinity`, `-Infinity` and `NaN` values, either `'null'`, `'constants'`, or `'strings'`.

python/pydantic_core/core_schema.py

@@ -151,6 +151,9 @@ def serialize_as_any(self) -> bool: ...
     @property
     def round_trip(self) -> bool: ...
 
+    @property
+    def sort_keys(self) -> bool: ...
+
     def mode_is_json(self) -> bool: ...
 
     def __str__(self) -> str: ...

src/errors/validation_exception.rs

@@ -347,6 +347,7 @@ impl ValidationError {
             None,
             false,
             false,
+            false,
             true,
             None,
             DuckTypingSerMode::SchemaBased,

src/serializers/extra.rs

@@ -86,6 +86,7 @@ impl SerializationState {
         by_alias: Option<bool>,
         exclude_none: bool,
         round_trip: bool,
+        sort_keys: bool,
         serialize_unknown: bool,
         fallback: Option<&'py Bound<'_, PyAny>>,
         duck_typing_ser_mode: DuckTypingSerMode,
@@ -100,6 +101,7 @@ impl SerializationState {
             false,
             exclude_none,
             round_trip,
+            sort_keys,
             &self.config,
             &self.rec_guard,
             serialize_unknown,
@@ -126,6 +128,7 @@ pub(crate) struct Extra<'a> {
     pub exclude_defaults: bool,
     pub exclude_none: bool,
     pub round_trip: bool,
+    pub sort_keys: bool,
     pub config: &'a SerializationConfig,
     pub rec_guard: &'a SerRecursionState,
     // the next two are used for union logic
@@ -152,6 +155,7 @@ impl<'a> Extra<'a> {
         exclude_defaults: bool,
         exclude_none: bool,
         round_trip: bool,
+        sort_keys: bool,
         config: &'a SerializationConfig,
         rec_guard: &'a SerRecursionState,
         serialize_unknown: bool,
@@ -168,6 +172,7 @@ impl<'a> Extra<'a> {
             exclude_defaults,
             exclude_none,
             round_trip,
+            sort_keys,
             config,
             rec_guard,
             check: SerCheck::None,
@@ -236,6 +241,7 @@ pub(crate) struct ExtraOwned {
     exclude_defaults: bool,
     exclude_none: bool,
     round_trip: bool,
+    sort_keys: bool,
     config: SerializationConfig,
     rec_guard: SerRecursionState,
     check: SerCheck,
@@ -257,6 +263,7 @@ impl ExtraOwned {
             exclude_defaults: extra.exclude_defaults,
             exclude_none: extra.exclude_none,
             round_trip: extra.round_trip,
+            sort_keys: extra.sort_keys,
             config: extra.config.clone(),
             rec_guard: extra.rec_guard.clone(),
             check: extra.check,
@@ -279,6 +286,7 @@ impl ExtraOwned {
             exclude_defaults: self.exclude_defaults,
             exclude_none: self.exclude_none,
             round_trip: self.round_trip,
+            sort_keys: self.sort_keys,
             config: &self.config,
             rec_guard: &self.rec_guard,
             check: self.check,

src/serializers/fields.rs

@@ -156,9 +156,12 @@ impl GeneralFieldsSerializer {
         let output_dict = PyDict::new(py);
         let mut used_req_fields: usize = 0;
 
-        // NOTE! we maintain the order of the input dict assuming that's right
-        for result in main_iter {
-            let (key, value) = result?;
+        let mut items = main_iter.collect::<PyResult<Vec<_>>>()?;
+        if extra.sort_keys {
+            items.sort_by_cached_key(|(key, _)| key_str(key).unwrap_or_default().to_string());
+        }
+
+        for (key, value) in items {
             let key_str = key_str(&key)?;
             let op_field = self.fields.get(key_str);
             if extra.exclude_none && value.is_none() {
@@ -242,8 +245,11 @@ impl GeneralFieldsSerializer {
         // we don't both with `used_fields` here because on unions, `to_python(..., mode='json')` is used
         let mut map = serializer.serialize_map(Some(expected_len))?;
 
-        for result in main_iter {
-            let (key, value) = result.map_err(py_err_se_err)?;
+        let mut items = main_iter.collect::<PyResult<Vec<_>>>().map_err(py_err_se_err)?;
+        if extra.sort_keys {
+            items.sort_by_cached_key(|(key, _)| key_str(key).unwrap_or_default().to_string());
+        }
+        for (key, value) in items {
             if extra.exclude_none && value.is_none() {
                 continue;
             }

src/serializers/infer.rs

@@ -103,6 +103,7 @@ pub(crate) fn infer_to_python_known(
             extra.exclude_defaults,
             extra.exclude_none,
             extra.round_trip,
+            extra.sort_keys,
             extra.rec_guard,
             extra.serialize_unknown,
             extra.fallback,
@@ -492,6 +493,7 @@ pub(crate) fn infer_serialize_known<S: Serializer>(
                 extra.exclude_defaults,
                 extra.exclude_none,
                 extra.round_trip,
+                extra.sort_keys,
                 extra.rec_guard,
                 extra.serialize_unknown,
                 extra.fallback,

src/serializers/mod.rs

@@ -60,6 +60,7 @@ impl SchemaSerializer {
         exclude_defaults: bool,
         exclude_none: bool,
         round_trip: bool,
+        sort_keys: bool,
         rec_guard: &'a SerRecursionState,
         serialize_unknown: bool,
         fallback: Option<&'a Bound<'a, PyAny>>,
@@ -75,6 +76,7 @@ impl SchemaSerializer {
             exclude_defaults,
             exclude_none,
             round_trip,
+            sort_keys,
             &self.config,
             rec_guard,
             serialize_unknown,
@@ -107,8 +109,8 @@ impl SchemaSerializer {
 
     #[allow(clippy::too_many_arguments)]
     #[pyo3(signature = (value, *, mode = None, include = None, exclude = None, by_alias = None,
-        exclude_unset = false, exclude_defaults = false, exclude_none = false, round_trip = false, warnings = WarningsArg::Bool(true),
-        fallback = None, serialize_as_any = false, context = None))]
+        exclude_unset = false, exclude_defaults = false, exclude_none = false, round_trip = false, sort_keys = false,
+        warnings = WarningsArg::Bool(true), fallback = None, serialize_as_any = false, context = None))]
     pub fn to_python(
         &self,
         py: Python,
@@ -121,6 +123,7 @@ impl SchemaSerializer {
         exclude_defaults: bool,
         exclude_none: bool,
         round_trip: bool,
+        sort_keys: bool,
         warnings: WarningsArg,
         fallback: Option<&Bound<'_, PyAny>>,
         serialize_as_any: bool,
@@ -143,6 +146,7 @@ impl SchemaSerializer {
             exclude_defaults,
             exclude_none,
             round_trip,
+            sort_keys,
             &rec_guard,
             false,
             fallback,
@@ -156,7 +160,7 @@ impl SchemaSerializer {
 
     #[allow(clippy::too_many_arguments)]
     #[pyo3(signature = (value, *, indent = None, include = None, exclude = None, by_alias = None,
-        exclude_unset = false, exclude_defaults = false, exclude_none = false, round_trip = false, warnings = WarningsArg::Bool(true),
+        exclude_unset = false, exclude_defaults = false, exclude_none = false, round_trip = false, sort_keys = false, warnings = WarningsArg::Bool(true),
         fallback = None, serialize_as_any = false, context = None))]
     pub fn to_json(
         &self,
@@ -170,6 +174,7 @@ impl SchemaSerializer {
         exclude_defaults: bool,
         exclude_none: bool,
         round_trip: bool,
+        sort_keys: bool,
         warnings: WarningsArg,
         fallback: Option<&Bound<'_, PyAny>>,
         serialize_as_any: bool,
@@ -191,6 +196,7 @@ impl SchemaSerializer {
             exclude_defaults,
             exclude_none,
             round_trip,
+            sort_keys,
             &rec_guard,
             false,
             fallback,
@@ -240,7 +246,7 @@ impl SchemaSerializer {
 #[allow(clippy::too_many_arguments)]
 #[pyfunction]
 #[pyo3(signature = (value, *, indent = None, include = None, exclude = None, by_alias = None,
-    exclude_none = false, round_trip = false, timedelta_mode = "iso8601", bytes_mode = "utf8",
+    exclude_none = false, round_trip = false, sort_keys = false, timedelta_mode = "iso8601", bytes_mode = "utf8",
     inf_nan_mode = "constants", serialize_unknown = false, fallback = None, serialize_as_any = false,
     context = None))]
 pub fn to_json(
@@ -252,6 +258,7 @@ pub fn to_json(
     by_alias: Option<bool>,
     exclude_none: bool,
     round_trip: bool,
+    sort_keys: bool,
     timedelta_mode: &str,
     bytes_mode: &str,
     inf_nan_mode: &str,
@@ -268,6 +275,7 @@ pub fn to_json(
         by_alias,
         exclude_none,
         round_trip,
+        sort_keys,
         serialize_unknown,
         fallback,
         duck_typing_ser_mode,
@@ -283,7 +291,7 @@ pub fn to_json(
 #[allow(clippy::too_many_arguments)]
 #[pyfunction]
 #[pyo3(signature = (value, *, include = None, exclude = None, by_alias = None, exclude_none = false, round_trip = false,
-    timedelta_mode = "iso8601", bytes_mode = "utf8", inf_nan_mode = "constants", serialize_unknown = false, fallback = None,
+    sort_keys = false, timedelta_mode = "iso8601", bytes_mode = "utf8", inf_nan_mode = "constants", serialize_unknown = false, fallback = None,
     serialize_as_any = false, context = None))]
 pub fn to_jsonable_python(
     py: Python,
@@ -293,6 +301,7 @@ pub fn to_jsonable_python(
     by_alias: Option<bool>,
     exclude_none: bool,
     round_trip: bool,
+    sort_keys: bool,
     timedelta_mode: &str,
     bytes_mode: &str,
     inf_nan_mode: &str,
@@ -309,6 +318,7 @@ pub fn to_jsonable_python(
         by_alias,
         exclude_none,
         round_trip,
+        sort_keys,
         serialize_unknown,
         fallback,
         duck_typing_ser_mode,

tests/serializers/test_model.py

@@ -12,6 +12,7 @@
 
 import pytest
 from dirty_equals import IsJson
+from inline_snapshot import snapshot
 
 from pydantic_core import (
     PydanticSerializationError,
@@ -991,22 +992,22 @@ def test_extra():
     class MyModel:
         # this is not required, but it avoids `__pydantic_fields_set__` being included in `__dict__`
         __slots__ = '__dict__', '__pydantic_fields_set__', '__pydantic_extra__', '__pydantic_private__'
-        field_a: str
         field_b: int
+        field_a: str
 
     schema = core_schema.model_schema(
         MyModel,
         core_schema.model_fields_schema(
             {
-                'field_a': core_schema.model_field(core_schema.bytes_schema()),
                 'field_b': core_schema.model_field(core_schema.int_schema()),
+                'field_a': core_schema.model_field(core_schema.bytes_schema()),
             },
             extra_behavior='allow',
         ),
         extra_behavior='allow',
     )
     v = SchemaValidator(schema)
-    m = v.validate_python({'field_a': b'test', 'field_b': 12, 'field_c': 'extra'})
+    m = v.validate_python({'field_b': 12, 'field_a': b'test', 'field_c': 'extra'})
     assert isinstance(m, MyModel)
     assert m.__dict__ == {'field_a': b'test', 'field_b': 12}
     assert m.__pydantic_extra__ == {'field_c': 'extra'}
@@ -1015,33 +1016,37 @@ class MyModel:
     s = SchemaSerializer(schema)
     assert 'mode:ModelExtra' in plain_repr(s)
     assert 'has_extra:true' in plain_repr(s)
-    assert s.to_python(m) == {'field_a': b'test', 'field_b': 12, 'field_c': 'extra'}
+    assert s.to_python(m, sort_keys=False) == snapshot({'field_a': b'test', 'field_b': 12, 'field_c': 'extra'})
+    assert s.to_json(m, sort_keys=True) == b'{"field_a":"test","field_b":12,"field_c":"extra"}'
     assert s.to_python(m, mode='json') == {'field_a': 'test', 'field_b': 12, 'field_c': 'extra'}
-    assert s.to_json(m) == b'{"field_a":"test","field_b":12,"field_c":"extra"}'
 
-    # test filtering
+    # # test filtering
     m = v.validate_python({'field_a': b'test', 'field_b': 12, 'field_c': None, 'field_d': [1, 2, 3]})
     assert isinstance(m, MyModel)
     assert m.__dict__ == {'field_a': b'test', 'field_b': 12}
     assert m.__pydantic_extra__ == {'field_c': None, 'field_d': [1, 2, 3]}
     assert m.__pydantic_fields_set__ == {'field_a', 'field_b', 'field_c', 'field_d'}
 
     assert s.to_python(m) == {'field_a': b'test', 'field_b': 12, 'field_c': None, 'field_d': [1, 2, 3]}
-    assert s.to_json(m) == b'{"field_a":"test","field_b":12,"field_c":null,"field_d":[1,2,3]}'
+    assert s.to_json(m) == b'{"field_b":12,"field_a":"test","field_c":null,"field_d":[1,2,3]}'
 
     assert s.to_python(m, exclude_none=True) == {'field_a': b'test', 'field_b': 12, 'field_d': [1, 2, 3]}
-    assert s.to_json(m, exclude_none=True) == b'{"field_a":"test","field_b":12,"field_d":[1,2,3]}'
+    assert s.to_json(m, exclude_none=True) == b'{"field_b":12,"field_a":"test","field_d":[1,2,3]}'
 
     assert s.to_python(m, exclude={'field_c'}) == {'field_a': b'test', 'field_b': 12, 'field_d': [1, 2, 3]}
-    assert s.to_json(m, exclude={'field_c'}) == b'{"field_a":"test","field_b":12,"field_d":[1,2,3]}'
+    assert s.to_json(m, exclude={'field_c'}) == b'{"field_b":12,"field_a":"test","field_d":[1,2,3]}'
 
     assert s.to_python(m, exclude={'field_d': [0]}) == {
         'field_a': b'test',
         'field_b': 12,
         'field_c': None,
         'field_d': [2, 3],
     }
-    assert s.to_json(m, exclude={'field_d': [0]}) == b'{"field_a":"test","field_b":12,"field_c":null,"field_d":[2,3]}'
+    assert s.to_json(m, exclude={'field_d': [0]}) == b'{"field_b":12,"field_a":"test","field_c":null,"field_d":[2,3]}'
+    assert (
+        s.to_json(m, exclude={'field_d': [0]}, sort_keys=True)
+        == b'{"field_a":"test","field_b":12,"field_c":null,"field_d":[2,3]}'
+    )
 
 
 def test_extra_config():