respect field_serializer when using serialize_as_any=True

Author: davidhewitt

src/serializers/fields.rs

@@ -11,6 +11,8 @@ use smallvec::SmallVec;
 
 use crate::common::missing_sentinel::get_missing_sentinel_object;
 use crate::serializers::extra::SerCheck;
+use crate::serializers::type_serializers::any::AnySerializer;
+use crate::serializers::type_serializers::function::{FunctionPlainSerializer, FunctionWrapSerializer};
 use crate::PydanticSerializationUnexpectedValue;
 
 use super::computed_fields::ComputedFields;
@@ -190,31 +192,26 @@ impl GeneralFieldsSerializer {
                 ..extra
             };
             if let Some((next_include, next_exclude)) = self.filter.key_filter(&key, include, exclude)? {
-                if let Some(field) = op_field {
-                    if let Some(ref serializer) = field.serializer {
-                        if exclude_default(&value, &field_extra, serializer)? {
-                            continue;
-                        }
-                        if serialization_exclude_if(field.serialization_exclude_if.as_ref(), &value)? {
-                            continue;
-                        }
-                        let value =
-                            serializer.to_python(&value, next_include.as_ref(), next_exclude.as_ref(), &field_extra)?;
-                        let output_key = field.get_key_py(output_dict.py(), &field_extra);
-                        output_dict.set_item(output_key, value)?;
-                    }
+                let (key, serializer) = if let Some(field) = op_field {
+                    let serializer = Self::prepare_value(&value, field, &field_extra)?;
 
                     if field.required {
                         used_req_fields += 1;
                     }
-                } else if self.mode == FieldsMode::TypedDictAllow {
-                    let value = match &self.extra_serializer {
-                        Some(serializer) => {
-                            serializer.to_python(&value, next_include.as_ref(), next_exclude.as_ref(), &field_extra)?
-                        }
-                        _ => infer_to_python(&value, next_include.as_ref(), next_exclude.as_ref(), &field_extra)?,
+
+                    let Some(serializer) = serializer else {
+                        continue;
                     };
-                    output_dict.set_item(key, value)?;
+
+                    (field.get_key_py(output_dict.py(), &field_extra), serializer)
+                } else if self.mode == FieldsMode::TypedDictAllow {
+                    let serializer = self
+                        .extra_serializer
+                        .as_ref()
+                        // If using `serialize_as_any`, extras are always inferred
+                        .filter(|_| !extra.serialize_as_any)
+                        .unwrap_or_else(|| AnySerializer::get());
+                    (&key, serializer)
                 } else if field_extra.check == SerCheck::Strict {
                     return Err(PydanticSerializationUnexpectedValue::new(
                         Some(format!("Unexpected field `{key}`")),
@@ -223,7 +220,18 @@ impl GeneralFieldsSerializer {
                         None,
                     )
                     .to_py_err());
-                }
+                } else {
+                    continue;
+                };
+
+                // Use `no_infer` here because the `serialize_as_any` logic has been handled in `prepare_value`
+                let value = serializer.to_python_no_infer(
+                    &value,
+                    next_include.as_ref(),
+                    next_exclude.as_ref(),
+                    &field_extra,
+                )?;
+                output_dict.set_item(key, value)?;
             }
         }
 
@@ -257,7 +265,7 @@ impl GeneralFieldsSerializer {
         extra: Extra,
     ) -> Result<S::SerializeMap, S::Error> {
         // NOTE! As above, we maintain the order of the input dict assuming that's right
-        // we don't both with `used_fields` here because on unions, `to_python(..., mode='json')` is used
+        // we don't both with `used_req_fields` here because on unions, `to_python(..., mode='json')` is used
         let mut map = serializer.serialize_map(Some(expected_len))?;
 
         for result in main_iter {
@@ -278,26 +286,23 @@ impl GeneralFieldsSerializer {
             let filter = self.filter.key_filter(&key, include, exclude).map_err(py_err_se_err)?;
             if let Some((next_include, next_exclude)) = filter {
                 if let Some(field) = self.fields.get(key_str) {
-                    if let Some(ref serializer) = field.serializer {
-                        if exclude_default(&value, &field_extra, serializer).map_err(py_err_se_err)? {
-                            continue;
-                        }
-                        if serialization_exclude_if(field.serialization_exclude_if.as_ref(), &value)
-                            .map_err(py_err_se_err)?
-                        {
-                            continue;
-                        }
-                        let s = PydanticSerializer::new(
-                            &value,
-                            serializer,
-                            next_include.as_ref(),
-                            next_exclude.as_ref(),
-                            &field_extra,
-                        );
-                        let output_key = field.get_key_json(key_str, &field_extra);
-                        map.serialize_entry(&output_key, &s)?;
-                    }
+                    let Some(serializer) = Self::prepare_value(&value, field, &field_extra).map_err(py_err_se_err)?
+                    else {
+                        continue;
+                    };
+
+                    // Use `no_infer` here because the `serialize_as_any` logic has been handled in `prepare_value`
+                    let s = PydanticSerializer::new_no_infer(
+                        &value,
+                        serializer,
+                        next_include.as_ref(),
+                        next_exclude.as_ref(),
+                        &field_extra,
+                    );
+                    let output_key = field.get_key_json(key_str, &field_extra);
+                    map.serialize_entry(&output_key, &s)?;
                 } else if self.mode == FieldsMode::TypedDictAllow {
+                    // FIXME: why is `extra_serializer` not used here when `serialize_as_any` is not set?
                     let output_key = infer_json_key(&key, &field_extra).map_err(py_err_se_err)?;
                     let s = SerializeInfer::new(&value, next_include.as_ref(), next_exclude.as_ref(), &field_extra);
                     map.serialize_entry(&output_key, &s)?;
@@ -308,6 +313,49 @@ impl GeneralFieldsSerializer {
         Ok(map)
     }
 
+    /// Gets the serializer to use for a field, applying `serialize_as_any` logic and applying any
+    /// field-level exclusionsg
+    fn prepare_value<'s>(
+        value: &Bound<'_, PyAny>,
+        field: &'s SerField,
+        field_extra: &Extra,
+    ) -> PyResult<Option<&'s Arc<CombinedSerializer>>> {
+        let Some(serializer) = field.serializer.as_ref() else {
+            // field excluded at schema level
+            return Ok(None);
+        };
+
+        if exclude_default(value, field_extra, serializer)? {
+            return Ok(None);
+        }
+
+        // FIXME: should `exclude_if` be applied to extra fields too?
+        if serialization_exclude_if(field.serialization_exclude_if.as_ref(), value)? {
+            return Ok(None);
+        }
+
+        Ok(Some(
+            if field_extra.serialize_as_any &&
+            // if serialize_as_any is set, we ensure that field serializers are
+            // still used, because this would match the `SerializeAsAny` annotation
+            // on a field
+            !matches!(
+                serializer.as_ref(),
+                CombinedSerializer::Function(FunctionPlainSerializer {
+                    is_field_serializer: true,
+                    ..
+                }) | CombinedSerializer::FunctionWrap(FunctionWrapSerializer {
+                    is_field_serializer: true,
+                    ..
+                })
+            ) {
+                AnySerializer::get()
+            } else {
+                serializer
+            },
+        ))
+    }
+
     pub(crate) fn add_computed_fields_python(
         &self,
         model: Option<&Bound<'_, PyAny>>,
@@ -425,7 +473,7 @@ impl TypeSerializer for GeneralFieldsSerializer {
             _ => self.fields.len() + option_length!(extra_dict) + self.computed_field_count(),
         };
         // NOTE! As above, we maintain the order of the input dict assuming that's right
-        // we don't both with `used_fields` here because on unions, `to_python(..., mode='json')` is used
+        // we don't both with `used_req_fields` here because on unions, `to_python(..., mode='json')` is used
         let mut map = self.main_serde_serialize(
             dict_items(&main_dict),
             expected_len,

src/serializers/shared.rs

@@ -18,6 +18,7 @@ use crate::build_tools::py_schema_error_type;
 use crate::definitions::DefinitionsBuilder;
 use crate::py_gc::PyGcTraverse;
 use crate::serializers::ser::PythonSerializer;
+use crate::serializers::type_serializers::any::AnySerializer;
 use crate::tools::{py_err, SchemaDict};
 
 use super::errors::se_err_py_err;
@@ -418,6 +419,27 @@ impl<'py> PydanticSerializer<'py> {
         include: Option<&'py Bound<'py, PyAny>>,
         exclude: Option<&'py Bound<'py, PyAny>>,
         extra: &'py Extra<'py>,
+    ) -> Self {
+        Self {
+            value,
+            serializer: if extra.serialize_as_any {
+                AnySerializer::get()
+            } else {
+                serializer
+            },
+            include,
+            exclude,
+            extra,
+        }
+    }
+
+    /// Same as above but will not fall back to type inference when `serialize_as_any` is set
+    pub(crate) fn new_no_infer(
+        value: &'py Bound<'py, PyAny>,
+        serializer: &'py CombinedSerializer,
+        include: Option<&'py Bound<'py, PyAny>>,
+        exclude: Option<&'py Bound<'py, PyAny>>,
+        extra: &'py Extra<'py>,
     ) -> Self {
         Self {
             value,
@@ -431,8 +453,9 @@ impl<'py> PydanticSerializer<'py> {
 
 impl Serialize for PydanticSerializer<'_> {
     fn serialize<S: serde::ser::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
+        // inference is handled in the constructor
         self.serializer
-            .serde_serialize(self.value, serializer, self.include, self.exclude, self.extra)
+            .serde_serialize_no_infer(self.value, serializer, self.include, self.exclude, self.extra)
     }
 }
 

src/serializers/type_serializers/function.rs

@@ -81,7 +81,7 @@ pub struct FunctionPlainSerializer {
     // fallback serializer - used when when_used decides that this serializer should not be used
     fallback_serializer: Option<Arc<CombinedSerializer>>,
     when_used: WhenUsed,
-    is_field_serializer: bool,
+    pub(crate) is_field_serializer: bool,
     info_arg: bool,
 }
 
@@ -334,7 +334,7 @@ pub struct FunctionWrapSerializer {
     function_name: String,
     return_serializer: Arc<CombinedSerializer>,
     when_used: WhenUsed,
-    is_field_serializer: bool,
+    pub(crate) is_field_serializer: bool,
     info_arg: bool,
 }
 

tests/serializers/test_serialize_as_any.py

@@ -1,6 +1,7 @@
 from dataclasses import dataclass
-from typing import Optional
+from typing import Callable, Optional
 
+import pytest
 from typing_extensions import TypedDict
 
 from pydantic_core import SchemaSerializer, SchemaValidator, core_schema
@@ -370,3 +371,52 @@ def a_model_serializer(self, handler, info):
     assert MyModel.__pydantic_serializer__.to_python(instance, serialize_as_any=True) == {
         'a_field_wrapped': {'an_inner_field': 1},
     }
+
+
+@pytest.fixture(params=['model', 'dataclass'])
+def container_schema_builder(
+    request: pytest.FixtureRequest,
+) -> Callable[[dict[str, core_schema.CoreSchema]], core_schema.CoreSchema]:
+    if request.param == 'model':
+        return lambda fields: core_schema.model_schema(
+            cls=type('Test', (), {}),
+            schema=core_schema.model_fields_schema(
+                fields={k: core_schema.model_field(schema=v) for k, v in fields.items()},
+            ),
+        )
+    elif request.param == 'dataclass':
+        return lambda fields: core_schema.dataclass_schema(
+            cls=dataclass(type('Test', (), {})),
+            schema=core_schema.dataclass_args_schema(
+                'Test',
+                fields=[core_schema.dataclass_field(name=k, schema=v) for k, v in fields.items()],
+            ),
+            fields=[k for k in fields.keys()],
+        )
+    else:
+        raise ValueError(f'Unknown container type {request.param}')
+
+
+def test_serialize_as_any_with_field_serializer(container_schema_builder) -> None:
+    # https://github.com/pydantic/pydantic/issues/12379
+
+    schema = container_schema_builder(
+        {
+            'value': core_schema.int_schema(
+                serialization=core_schema.plain_serializer_function_ser_schema(
+                    lambda model, v: v * 2, is_field_serializer=True
+                )
+            )
+        }
+    )
+
+    v = SchemaValidator(schema).validate_python({'value': 123})
+    cls = type(v)
+    s = SchemaSerializer(schema)
+    # necessary to ensure that type inference will pick up the serializer
+    cls.__pydantic_serializer__ = s
+
+    assert s.to_python(v, serialize_as_any=False) == {'value': 246}
+    assert s.to_python(v, serialize_as_any=True) == {'value': 246}
+    assert s.to_json(v, serialize_as_any=False) == b'{"value":246}'
+    assert s.to_json(v, serialize_as_any=True) == b'{"value":246}'