recursive sort dictionary

Author: aezomz

python/pydantic_core/_pydantic_core.pyi

@@ -305,7 +305,7 @@ class SchemaSerializer:
         exclude_unset: bool = False,
         exclude_defaults: bool = False,
         exclude_none: bool = False,
-        sort_keys: bool = False,
+        sort_keys: Literal['recursive', 'top-level', 'unsorted'] = 'unsorted',
         round_trip: bool = False,
         warnings: bool | Literal['none', 'warn', 'error'] = True,
         fallback: Callable[[Any], Any] | None = None,
@@ -327,7 +327,7 @@ class SchemaSerializer:
             exclude_defaults: Whether to exclude fields that are equal to their default value.
             exclude_none: Whether to exclude fields that have a value of `None`.
             round_trip: Whether to enable serialization and validation round-trip support.
-            sort_keys: Whether to sort dictionary keys at the root level.
+            sort_keys: Whether to sort dictionary keys, either `'recursive'`, `'top-level'`, or `'unsorted'`.
             warnings: How to handle invalid fields. False/"none" ignores them, True/"warn" logs errors,
                 "error" raises a [`PydanticSerializationError`][pydantic_core.PydanticSerializationError].
             fallback: A function to call when an unknown value is encountered,
@@ -354,7 +354,7 @@ class SchemaSerializer:
         exclude_defaults: bool = False,
         exclude_none: bool = False,
         round_trip: bool = False,
-        sort_keys: bool = False,
+        sort_keys: Literal['recursive', 'top-level', 'unsorted'] = 'unsorted',
         warnings: bool | Literal['none', 'warn', 'error'] = True,
         fallback: Callable[[Any], Any] | None = None,
         serialize_as_any: bool = False,
@@ -374,7 +374,7 @@ class SchemaSerializer:
             exclude_defaults: Whether to exclude fields that are equal to their default value.
             exclude_none: Whether to exclude fields that have a value of `None`.
             round_trip: Whether to enable serialization and validation round-trip support.
-            sort_keys: Whether to sort dictionary keys at the root level.
+            sort_keys: Whether to sort dictionary keys, either `'recursive'`, `'top-level'`, or `'unsorted'`.
             warnings: How to handle invalid fields. False/"none" ignores them, True/"warn" logs errors,
                 "error" raises a [`PydanticSerializationError`][pydantic_core.PydanticSerializationError].
             fallback: A function to call when an unknown value is encountered,
@@ -399,7 +399,7 @@ def to_json(
     by_alias: bool | None = None,
     exclude_none: bool = False,
     round_trip: bool = False,
-    sort_keys: bool = False,
+    sort_keys: Literal['recursive', 'top-level', 'unsorted'] = 'unsorted',
     timedelta_mode: Literal['iso8601', 'float'] = 'iso8601',
     bytes_mode: Literal['utf8', 'base64', 'hex'] = 'utf8',
     inf_nan_mode: Literal['null', 'constants', 'strings'] = 'constants',
@@ -421,7 +421,7 @@ def to_json(
         by_alias: Whether to use the alias names of fields.
         exclude_none: Whether to exclude fields that have a value of `None`.
         round_trip: Whether to enable serialization and validation round-trip support.
-        sort_keys: Whether to sort dictionary keys at the root level.
+        sort_keys: Whether to sort dictionary keys, either `'recursive'`, `'top-level'`, or `'unsorted'`.
         timedelta_mode: How to serialize `timedelta` objects, either `'iso8601'` or `'float'`.
         bytes_mode: How to serialize `bytes` objects, either `'utf8'`, `'base64'`, or `'hex'`.
         inf_nan_mode: How to serialize `Infinity`, `-Infinity` and `NaN` values, either `'null'`, `'constants'`, or `'strings'`.
@@ -477,7 +477,7 @@ def to_jsonable_python(
     by_alias: bool | None = None,
     exclude_none: bool = False,
     round_trip: bool = False,
-    sort_keys: bool = False,
+    sort_keys: Literal['recursive', 'top-level', 'unsorted'] = 'unsorted',
     timedelta_mode: Literal['iso8601', 'float'] = 'iso8601',
     bytes_mode: Literal['utf8', 'base64', 'hex'] = 'utf8',
     inf_nan_mode: Literal['null', 'constants', 'strings'] = 'constants',

src/errors/validation_exception.rs

@@ -18,7 +18,7 @@ use crate::build_tools::py_schema_error_type;
 use crate::errors::LocItem;
 use crate::get_pydantic_version;
 use crate::input::InputType;
-use crate::serializers::{DuckTypingSerMode, Extra, SerMode, SerializationState};
+use crate::serializers::{DuckTypingSerMode, Extra, SerMode, SerializationState, SortKeysMode};
 use crate::tools::{safe_repr, write_truncated_to_limited_bytes, SchemaDict};
 
 use super::line_error::ValLineError;
@@ -347,7 +347,7 @@ impl ValidationError {
             None,
             false,
             false,
-            false,
+            &SortKeysMode::Unsorted,
             true,
             None,
             DuckTypingSerMode::SchemaBased,

src/serializers/extra.rs

@@ -86,7 +86,7 @@ impl SerializationState {
         by_alias: Option<bool>,
         exclude_none: bool,
         round_trip: bool,
-        sort_keys: bool,
+        sort_keys: &'py SortKeysMode,
         serialize_unknown: bool,
         fallback: Option<&'py Bound<'_, PyAny>>,
         duck_typing_ser_mode: DuckTypingSerMode,
@@ -128,7 +128,7 @@ pub(crate) struct Extra<'a> {
     pub exclude_defaults: bool,
     pub exclude_none: bool,
     pub round_trip: bool,
-    pub sort_keys: bool,
+    pub sort_keys: &'a SortKeysMode,
     pub config: &'a SerializationConfig,
     pub rec_guard: &'a SerRecursionState,
     // the next two are used for union logic
@@ -155,7 +155,7 @@ impl<'a> Extra<'a> {
         exclude_defaults: bool,
         exclude_none: bool,
         round_trip: bool,
-        sort_keys: bool,
+        sort_keys: &'a SortKeysMode,
         config: &'a SerializationConfig,
         rec_guard: &'a SerRecursionState,
         serialize_unknown: bool,
@@ -241,7 +241,7 @@ pub(crate) struct ExtraOwned {
     exclude_defaults: bool,
     exclude_none: bool,
     round_trip: bool,
-    sort_keys: bool,
+    sort_keys: SortKeysMode,
     config: SerializationConfig,
     rec_guard: SerRecursionState,
     check: SerCheck,
@@ -263,7 +263,7 @@ impl ExtraOwned {
             exclude_defaults: extra.exclude_defaults,
             exclude_none: extra.exclude_none,
             round_trip: extra.round_trip,
-            sort_keys: extra.sort_keys,
+            sort_keys: *extra.sort_keys,
             config: extra.config.clone(),
             rec_guard: extra.rec_guard.clone(),
             check: extra.check,
@@ -286,7 +286,7 @@ impl ExtraOwned {
             exclude_defaults: self.exclude_defaults,
             exclude_none: self.exclude_none,
             round_trip: self.round_trip,
-            sort_keys: self.sort_keys,
+            sort_keys: &self.sort_keys,
             config: &self.config,
             rec_guard: &self.rec_guard,
             check: self.check,
@@ -387,6 +387,58 @@ impl From<bool> for WarningsMode {
     }
 }
 
+// #[derive(Debug, Clone, Copy, Eq, PartialEq)]
+#[derive(Debug, Clone, Copy)]
+pub enum SortKeysMode {
+    Recursive,
+    TopLevel,
+    Unsorted,
+}
+
+impl<'py> FromPyObject<'py> for SortKeysMode {
+    fn extract_bound(ob: &Bound<'py, PyAny>) -> PyResult<SortKeysMode> {
+        if let Ok(str_mode) = ob.extract::<&str>() {
+            match str_mode {
+                "recursive" => Ok(Self::Recursive),
+                "top-level" => Ok(Self::TopLevel),
+                "unsorted" => Ok(Self::Unsorted),
+                _ => Err(PyValueError::new_err(
+                    "Invalid sort_keys parameter, should be `'recursive'`, `'top-level'`, `'unsorted'`",
+                )),
+            }
+        } else {
+            Err(PyTypeError::new_err(
+                "Invalid warnings parameter, should be `'none'`, `'warn'`, `'error'` or a `bool`",
+            ))
+        }
+    }
+}
+
+impl From<&str> for SortKeysMode {
+    fn from(s: &str) -> Self {
+        match s {
+            "recursive" => SortKeysMode::Recursive,
+            "top-level" => SortKeysMode::TopLevel,
+            "unsorted" => SortKeysMode::Unsorted,
+            _ => SortKeysMode::Unsorted,
+        }
+    }
+}
+
+impl<'py> IntoPyObject<'py> for &'_ SortKeysMode {
+    type Target = PyString;
+    type Output = Bound<'py, PyString>;
+    type Error = Infallible;
+
+    fn into_pyobject(self, py: Python<'py>) -> Result<Self::Output, Self::Error> {
+        match self {
+            SortKeysMode::Recursive => Ok(intern!(py, "recursive").clone()),
+            SortKeysMode::TopLevel => Ok(intern!(py, "top-level").clone()),
+            SortKeysMode::Unsorted => Ok(intern!(py, "unsorted").clone()),
+        }
+    }
+}
+
 #[cfg_attr(debug_assertions, derive(Debug))]
 pub(crate) struct CollectWarnings {
     mode: WarningsMode,

src/serializers/fields.rs

@@ -13,7 +13,7 @@ use crate::PydanticSerializationUnexpectedValue;
 
 use super::computed_fields::ComputedFields;
 use super::errors::py_err_se_err;
-use super::extra::Extra;
+use super::extra::{Extra, SortKeysMode};
 use super::filter::SchemaFilter;
 use super::infer::{infer_json_key, infer_serialize, infer_to_python, SerializeInfer};
 use super::shared::PydanticSerializer;
@@ -156,7 +156,7 @@ impl GeneralFieldsSerializer {
         let output_dict = PyDict::new(py);
         let mut used_req_fields: usize = 0;
 
-        if !extra.sort_keys {
+        if matches!(extra.sort_keys, SortKeysMode::Unsorted) {
             for result in main_iter {
                 let (key, value) = result?;
                 if let Some(is_required) =
@@ -201,6 +201,23 @@ impl GeneralFieldsSerializer {
         }
     }
 
+    fn sort_dict_recursive<'py>(py: Python<'py>, value: &Bound<'py, PyAny>) -> PyResult<Bound<'py, PyDict>> {
+        let dict = value.downcast::<PyDict>()?;
+        let mut items = dict_items(dict).collect::<PyResult<Vec<_>>>()?;
+        items.sort_by_cached_key(|(key, _)| key_str(key).unwrap_or_default().to_string());
+
+        let sorted_dict = PyDict::new(py);
+        for (k, v) in items {
+            if v.downcast::<PyDict>().is_ok() {
+                let sorted_v = Self::sort_dict_recursive(py, &v)?;
+                sorted_dict.set_item(k, sorted_v)?;
+            } else {
+                sorted_dict.set_item(k, v)?;
+            }
+        }
+        Ok(sorted_dict)
+    }
+
     fn process_field_entry_python<'py>(
         &self,
         key: &Bound<'py, PyAny>,
@@ -235,10 +252,21 @@ impl GeneralFieldsSerializer {
             if let Some(field) = op_field {
                 if let Some(ref serializer) = field.serializer {
                     if !exclude_default(value, &field_extra, serializer)? {
-                        let value =
-                            serializer.to_python(value, next_include.as_ref(), next_exclude.as_ref(), &field_extra)?;
+                        let processed_value = if matches!(extra.sort_keys, SortKeysMode::Recursive)
+                            && value.downcast::<PyDict>().is_ok()
+                        {
+                            let sorted_dict = Self::sort_dict_recursive(value.py(), value)?;
+                            serializer.to_python(
+                                sorted_dict.as_ref(),
+                                next_include.as_ref(),
+                                next_exclude.as_ref(),
+                                &field_extra,
+                            )?
+                        } else {
+                            serializer.to_python(value, next_include.as_ref(), next_exclude.as_ref(), &field_extra)?
+                        };
                         let output_key = field.get_key_py(output_dict.py(), &field_extra);
-                        output_dict.set_item(output_key, value)?;
+                        output_dict.set_item(output_key, processed_value)?;
                     }
                 }
 
@@ -247,13 +275,33 @@ impl GeneralFieldsSerializer {
                 }
                 return Ok(Some(false));
             } else if self.mode == FieldsMode::TypedDictAllow {
-                let value = match &self.extra_serializer {
-                    Some(serializer) => {
-                        serializer.to_python(value, next_include.as_ref(), next_exclude.as_ref(), &field_extra)?
+                let processed_value = if matches!(extra.sort_keys, SortKeysMode::Recursive)
+                    && value.downcast::<PyDict>().is_ok()
+                {
+                    let sorted_dict = Self::sort_dict_recursive(value.py(), value)?;
+                    match &self.extra_serializer {
+                        Some(serializer) => serializer.to_python(
+                            sorted_dict.as_ref(),
+                            next_include.as_ref(),
+                            next_exclude.as_ref(),
+                            &field_extra,
+                        )?,
+                        None => infer_to_python(
+                            sorted_dict.as_ref(),
+                            next_include.as_ref(),
+                            next_exclude.as_ref(),
+                            &field_extra,
+                        )?,
+                    }
+                } else {
+                    match &self.extra_serializer {
+                        Some(serializer) => {
+                            serializer.to_python(value, next_include.as_ref(), next_exclude.as_ref(), &field_extra)?
+                        }
+                        None => infer_to_python(value, next_include.as_ref(), next_exclude.as_ref(), &field_extra)?,
                     }
-                    None => infer_to_python(value, next_include.as_ref(), next_exclude.as_ref(), &field_extra)?,
                 };
-                output_dict.set_item(key, value)?;
+                output_dict.set_item(key, processed_value)?;
                 return Ok(None);
             } else if field_extra.check == SerCheck::Strict {
                 return Err(PydanticSerializationUnexpectedValue::new(
@@ -281,7 +329,7 @@ impl GeneralFieldsSerializer {
         // we don't both with `used_fields` here because on unions, `to_python(..., mode='json')` is used
         let mut map = serializer.serialize_map(Some(expected_len))?;
 
-        if !extra.sort_keys {
+        if matches!(extra.sort_keys, SortKeysMode::Unsorted) {
             for result in main_iter {
                 let (key, value) = result.map_err(py_err_se_err)?;
                 self.process_field_entry::<S>(&key, &value, &mut map, include, exclude, &extra)?;
@@ -308,32 +356,56 @@ impl GeneralFieldsSerializer {
         if extra.exclude_none && value.is_none() {
             return Ok(());
         }
-        let key_str = key_str(key).map_err(py_err_se_err)?;
+        let field_key_str = key_str(key).map_err(py_err_se_err)?;
         let field_extra = Extra {
-            field_name: Some(key_str),
+            field_name: Some(field_key_str),
             ..*extra
         };
 
         let filter = self.filter.key_filter(key, include, exclude).map_err(py_err_se_err)?;
         if let Some((next_include, next_exclude)) = filter {
-            if let Some(field) = self.fields.get(key_str) {
+            if let Some(field) = self.fields.get(field_key_str) {
                 if let Some(ref serializer) = field.serializer {
                     if !exclude_default(value, &field_extra, serializer).map_err(py_err_se_err)? {
-                        let s = PydanticSerializer::new(
-                            value,
-                            serializer,
-                            next_include.as_ref(),
-                            next_exclude.as_ref(),
-                            &field_extra,
-                        );
-                        let output_key = field.get_key_json(key_str, &field_extra);
-                        map.serialize_entry(&output_key, &s)?;
+                        if matches!(extra.sort_keys, SortKeysMode::Recursive) && value.downcast::<PyDict>().is_ok() {
+                            let sorted_dict = Self::sort_dict_recursive(value.py(), value).map_err(py_err_se_err)?;
+                            let s = PydanticSerializer::new(
+                                sorted_dict.as_ref(),
+                                serializer,
+                                next_include.as_ref(),
+                                next_exclude.as_ref(),
+                                &field_extra,
+                            );
+                            let output_key = field.get_key_json(field_key_str, &field_extra);
+                            map.serialize_entry(&output_key, &s)?;
+                        } else {
+                            let s = PydanticSerializer::new(
+                                value,
+                                serializer,
+                                next_include.as_ref(),
+                                next_exclude.as_ref(),
+                                &field_extra,
+                            );
+                            let output_key = field.get_key_json(field_key_str, &field_extra);
+                            map.serialize_entry(&output_key, &s)?;
+                        }
                     }
                 }
             } else if self.mode == FieldsMode::TypedDictAllow {
                 let output_key = infer_json_key(key, &field_extra).map_err(py_err_se_err)?;
-                let s = SerializeInfer::new(value, next_include.as_ref(), next_exclude.as_ref(), &field_extra);
-                map.serialize_entry(&output_key, &s)?;
+                if matches!(extra.sort_keys, SortKeysMode::Recursive) && value.downcast::<PyDict>().is_ok() {
+                    let sorted_dict = Self::sort_dict_recursive(value.py(), value).map_err(py_err_se_err)?;
+                    let s = SerializeInfer::new(
+                        sorted_dict.as_ref(),
+                        next_include.as_ref(),
+                        next_exclude.as_ref(),
+                        &field_extra,
+                    );
+                    map.serialize_entry(&output_key, &s)?;
+                } else {
+                    let s = SerializeInfer::new(value, next_include.as_ref(), next_exclude.as_ref(), &field_extra);
+                    map.serialize_entry(&output_key, &s)?;
+                }
             }
             // no error case here since unions (which need the error case) use `to_python(..., mode='json')`
         }