diff --git a/benches/main.rs b/benches/main.rs index 86812edb2..6d3fd0951 100644 --- a/benches/main.rs +++ b/benches/main.rs @@ -28,11 +28,19 @@ fn ints_json(bench: &mut Bencher) { Python::with_gil(|py| { let validator = build_schema_validator(py, "{'type': 'int'}"); - let result = validator.validate_json(py, &json(py, "123"), None, None, None).unwrap(); + let result = validator + .validate_json(py, &json(py, "123"), None, None, None, false) + .unwrap(); let result_int: i64 = result.extract(py).unwrap(); assert_eq!(result_int, 123); - bench.iter(|| black_box(validator.validate_json(py, &json(py, "123"), None, None, None).unwrap())) + bench.iter(|| { + black_box( + validator + .validate_json(py, &json(py, "123"), None, None, None, false) + .unwrap(), + ) + }) }) } @@ -42,12 +50,20 @@ fn ints_python(bench: &mut Bencher) { let validator = build_schema_validator(py, "{'type': 'int'}"); let input = 123_i64.into_py(py).into_bound(py); - let result = validator.validate_python(py, &input, None, None, None, None).unwrap(); + let result = validator + .validate_python(py, &input, None, None, None, None, false) + .unwrap(); let result_int: i64 = result.extract(py).unwrap(); assert_eq!(result_int, 123); let input = black_box(input); - bench.iter(|| black_box(validator.validate_python(py, &input, None, None, None, None).unwrap())) + bench.iter(|| { + black_box( + validator + .validate_python(py, &input, None, None, None, None, false) + .unwrap(), + ) + }) }) } @@ -60,7 +76,13 @@ fn list_int_json(bench: &mut Bencher) { (0..100).map(|x| x.to_string()).collect::>().join(",") ); - bench.iter(|| black_box(validator.validate_json(py, &json(py, &code), None, None, None).unwrap())) + bench.iter(|| { + black_box( + validator + .validate_json(py, &json(py, &code), None, None, None, false) + .unwrap(), + ) + }) }) } @@ -81,7 +103,9 @@ fn list_int_python(bench: &mut Bencher) { let (validator, input) = list_int_input(py); let input = black_box(input.bind(py)); bench.iter(|| { - let v = validator.validate_python(py, &input, None, None, None, None).unwrap(); + let v = validator + .validate_python(py, &input, None, None, None, None, false) + .unwrap(); black_box(v) }) }) @@ -114,7 +138,7 @@ fn list_error_json(bench: &mut Bencher) { .join(", ") ); - match validator.validate_json(py, &json(py, &code), None, None, None) { + match validator.validate_json(py, &json(py, &code), None, None, None, false) { Ok(_) => panic!("unexpectedly valid"), Err(e) => { let v = e.value_bound(py); @@ -126,7 +150,7 @@ fn list_error_json(bench: &mut Bencher) { }; bench.iter( - || match validator.validate_json(py, &json(py, &code), None, None, None) { + || match validator.validate_json(py, &json(py, &code), None, None, None, false) { Ok(_) => panic!("unexpectedly valid"), Err(e) => black_box(e), }, @@ -146,7 +170,7 @@ fn list_error_python_input(py: Python<'_>) -> (SchemaValidator, PyObject) { let input = py.eval_bound(&code, None, None).unwrap().extract().unwrap(); - match validator.validate_python(py, &input, None, None, None, None) { + match validator.validate_python(py, &input, None, None, None, None, false) { Ok(_) => panic!("unexpectedly valid"), Err(e) => { let v = e.value_bound(py); @@ -166,7 +190,7 @@ fn list_error_python(bench: &mut Bencher) { let input = black_box(input.bind(py)); bench.iter(|| { - let result = validator.validate_python(py, &input, None, None, None, None); + let result = validator.validate_python(py, &input, None, None, None, None, false); match result { Ok(_) => panic!("unexpectedly valid"), @@ -199,7 +223,13 @@ fn list_any_json(bench: &mut Bencher) { (0..100).map(|x| x.to_string()).collect::>().join(",") ); - bench.iter(|| black_box(validator.validate_json(py, &json(py, &code), None, None, None).unwrap())) + bench.iter(|| { + black_box( + validator + .validate_json(py, &json(py, &code), None, None, None, false) + .unwrap(), + ) + }) }) } @@ -214,7 +244,9 @@ fn list_any_python(bench: &mut Bencher) { let input = py.eval_bound(&code, None, None).unwrap().to_object(py); let input = black_box(input.bind(py)); bench.iter(|| { - let v = validator.validate_python(py, &input, None, None, None, None).unwrap(); + let v = validator + .validate_python(py, &input, None, None, None, None, false) + .unwrap(); black_box(v) }) }) @@ -244,7 +276,13 @@ fn dict_json(bench: &mut Bencher) { .join(", ") ); - bench.iter(|| black_box(validator.validate_json(py, &json(py, &code), None, None, None).unwrap())) + bench.iter(|| { + black_box( + validator + .validate_json(py, &json(py, &code), None, None, None, false) + .unwrap(), + ) + }) }) } @@ -266,7 +304,9 @@ fn dict_python(bench: &mut Bencher) { let input = py.eval_bound(&code, None, None).unwrap().to_object(py); let input = black_box(input.bind(py)); bench.iter(|| { - let v = validator.validate_python(py, &input, None, None, None, None).unwrap(); + let v = validator + .validate_python(py, &input, None, None, None, None, false) + .unwrap(); black_box(v) }) }) @@ -294,7 +334,7 @@ fn dict_value_error(bench: &mut Bencher) { let input = py.eval_bound(&code, None, None).unwrap().to_object(py).into_bound(py); - match validator.validate_python(py, &input, None, None, None, None) { + match validator.validate_python(py, &input, None, None, None, None, false) { Ok(_) => panic!("unexpectedly valid"), Err(e) => { let v = e.value_bound(py); @@ -307,7 +347,7 @@ fn dict_value_error(bench: &mut Bencher) { let input = black_box(input); bench.iter(|| { - let result = validator.validate_python(py, &input, None, None, None, None); + let result = validator.validate_python(py, &input, None, None, None, None, false); match result { Ok(_) => panic!("unexpectedly valid"), @@ -342,7 +382,13 @@ fn typed_dict_json(bench: &mut Bencher) { let code = r#"{"a": 1, "b": 2, "c": 3, "d": 4, "e": 5, "f": 6, "g": 7, "h": 8, "i": 9, "j": 0}"#.to_string(); - bench.iter(|| black_box(validator.validate_json(py, &json(py, &code), None, None, None).unwrap())) + bench.iter(|| { + black_box( + validator + .validate_json(py, &json(py, &code), None, None, None, false) + .unwrap(), + ) + }) }) } @@ -373,7 +419,9 @@ fn typed_dict_python(bench: &mut Bencher) { let input = py.eval_bound(&code, None, None).unwrap().to_object(py); let input = black_box(input.bind(py)); bench.iter(|| { - let v = validator.validate_python(py, &input, None, None, None, None).unwrap(); + let v = validator + .validate_python(py, &input, None, None, None, None, false) + .unwrap(); black_box(v) }) }) @@ -413,7 +461,7 @@ fn typed_dict_deep_error(bench: &mut Bencher) { let input = py.eval_bound(code, None, None).unwrap().to_object(py); let input = black_box(input.bind(py)); - match validator.validate_python(py, &input, None, None, None, None) { + match validator.validate_python(py, &input, None, None, None, None, false) { Ok(_) => panic!("unexpectedly valid"), Err(e) => { let v = e.value_bound(py); @@ -425,7 +473,7 @@ fn typed_dict_deep_error(bench: &mut Bencher) { }; bench.iter(|| { - let result = validator.validate_python(py, &input, None, None, None, None); + let result = validator.validate_python(py, &input, None, None, None, None, false); match result { Ok(_) => panic!("unexpectedly valid"), @@ -450,7 +498,11 @@ fn complete_model(bench: &mut Bencher) { let input = black_box(input); bench.iter(|| { - black_box(validator.validate_python(py, &input, None, None, None, None).unwrap()); + black_box( + validator + .validate_python(py, &input, None, None, None, None, false) + .unwrap(), + ); }) }) } @@ -469,10 +521,16 @@ fn nested_model_using_definitions(bench: &mut Bencher) { let input = complete_schema.call_method0("input_data_valid").unwrap(); let input = black_box(input); - validator.validate_python(py, &input, None, None, None, None).unwrap(); + validator + .validate_python(py, &input, None, None, None, None, false) + .unwrap(); bench.iter(|| { - black_box(validator.validate_python(py, &input, None, None, None, None).unwrap()); + black_box( + validator + .validate_python(py, &input, None, None, None, None, false) + .unwrap(), + ); }) }) } @@ -491,10 +549,16 @@ fn nested_model_inlined(bench: &mut Bencher) { let input = complete_schema.call_method0("input_data_valid").unwrap(); let input = black_box(input); - validator.validate_python(py, &input, None, None, None, None).unwrap(); + validator + .validate_python(py, &input, None, None, None, None, false) + .unwrap(); bench.iter(|| { - black_box(validator.validate_python(py, &input, None, None, None, None).unwrap()); + black_box( + validator + .validate_python(py, &input, None, None, None, None, false) + .unwrap(), + ); }) }) } @@ -506,12 +570,20 @@ fn literal_ints_few_python(bench: &mut Bencher) { let input = 4_i64.into_py(py); let input = input.bind(py); - let result = validator.validate_python(py, &input, None, None, None, None).unwrap(); + let result = validator + .validate_python(py, &input, None, None, None, None, false) + .unwrap(); let result_int: i64 = result.extract(py).unwrap(); assert_eq!(result_int, 4); let input = black_box(input); - bench.iter(|| black_box(validator.validate_python(py, &input, None, None, None, None).unwrap())) + bench.iter(|| { + black_box( + validator + .validate_python(py, &input, None, None, None, None, false) + .unwrap(), + ) + }) }) } @@ -523,12 +595,20 @@ fn literal_strings_few_small_python(bench: &mut Bencher) { let input = py.eval_bound("'4'", None, None).unwrap(); let input = input.to_object(py).into_bound(py); let input_str: String = input.extract().unwrap(); - let result = validator.validate_python(py, &input, None, None, None, None).unwrap(); + let result = validator + .validate_python(py, &input, None, None, None, None, false) + .unwrap(); let result_str: String = result.extract(py).unwrap(); assert_eq!(result_str, input_str); let input = black_box(input); - bench.iter(|| black_box(validator.validate_python(py, &input, None, None, None, None).unwrap())) + bench.iter(|| { + black_box( + validator + .validate_python(py, &input, None, None, None, None, false) + .unwrap(), + ) + }) }) } @@ -543,12 +623,20 @@ fn literal_strings_few_large_python(bench: &mut Bencher) { let input = py.eval_bound("'a' * 25 + '4'", None, None).unwrap(); let input = input.to_object(py).into_bound(py); let input_str: String = input.extract().unwrap(); - let result = validator.validate_python(py, &input, None, None, None, None).unwrap(); + let result = validator + .validate_python(py, &input, None, None, None, None, false) + .unwrap(); let result_str: String = result.extract(py).unwrap(); assert_eq!(result_str, input_str); let input = black_box(input); - bench.iter(|| black_box(validator.validate_python(py, &input, None, None, None, None).unwrap())) + bench.iter(|| { + black_box( + validator + .validate_python(py, &input, None, None, None, None, false) + .unwrap(), + ) + }) }) } @@ -579,11 +667,19 @@ class Foo(Enum): let input = py.eval_bound("Foo.v4", Some(&globals), None).unwrap(); let input = input.to_object(py).into_bound(py); - let result = validator.validate_python(py, &input, None, None, None, None).unwrap(); + let result = validator + .validate_python(py, &input, None, None, None, None, false) + .unwrap(); assert!(input.eq(result).unwrap()); let input = black_box(input); - bench.iter(|| black_box(validator.validate_python(py, &input, None, None, None, None).unwrap())) + bench.iter(|| { + black_box( + validator + .validate_python(py, &input, None, None, None, None, false) + .unwrap(), + ) + }) }) } @@ -593,12 +689,20 @@ fn literal_ints_many_python(bench: &mut Bencher) { let validator = build_schema_validator(py, "{'type': 'literal', 'expected': list(range(100))}"); let input = 99_i64.into_py(py).into_bound(py); - let result = validator.validate_python(py, &input, None, None, None, None).unwrap(); + let result = validator + .validate_python(py, &input, None, None, None, None, false) + .unwrap(); let result_int: i64 = result.extract(py).unwrap(); assert_eq!(result_int, 99); let input = black_box(input); - bench.iter(|| black_box(validator.validate_python(py, &input, None, None, None, None).unwrap())) + bench.iter(|| { + black_box( + validator + .validate_python(py, &input, None, None, None, None, false) + .unwrap(), + ) + }) }) } @@ -610,12 +714,20 @@ fn literal_strings_many_small_python(bench: &mut Bencher) { let input = py.eval_bound("'99'", None, None).unwrap(); let input = input.to_object(py).into_bound(py); let input_str: String = input.extract().unwrap(); - let result = validator.validate_python(py, &input, None, None, None, None).unwrap(); + let result = validator + .validate_python(py, &input, None, None, None, None, false) + .unwrap(); let result_str: String = result.extract(py).unwrap(); assert_eq!(result_str, input_str); let input = black_box(input); - bench.iter(|| black_box(validator.validate_python(py, &input, None, None, None, None).unwrap())) + bench.iter(|| { + black_box( + validator + .validate_python(py, &input, None, None, None, None, false) + .unwrap(), + ) + }) }) } @@ -630,12 +742,20 @@ fn literal_strings_many_large_python(bench: &mut Bencher) { let input = py.eval_bound("'a' * 25 + '99'", None, None).unwrap(); let input = input.to_object(py).into_bound(py); let input_str: String = input.extract().unwrap(); - let result = validator.validate_python(py, &input, None, None, None, None).unwrap(); + let result = validator + .validate_python(py, &input, None, None, None, None, false) + .unwrap(); let result_str: String = result.extract(py).unwrap(); assert_eq!(result_str, input_str); let input = black_box(input); - bench.iter(|| black_box(validator.validate_python(py, &input, None, None, None, None).unwrap())) + bench.iter(|| { + black_box( + validator + .validate_python(py, &input, None, None, None, None, false) + .unwrap(), + ) + }) }) } @@ -646,12 +766,20 @@ fn literal_ints_many_json(bench: &mut Bencher) { let input_json = py.eval_bound("'99'", None, None).unwrap(); let input_json = input_json.to_object(py).into_bound(py); - let result = validator.validate_json(py, &input_json, None, None, None).unwrap(); + let result = validator + .validate_json(py, &input_json, None, None, None, false) + .unwrap(); let result_int: i64 = result.extract(py).unwrap(); assert_eq!(result_int, 99); let input_json = black_box(input_json); - bench.iter(|| black_box(validator.validate_json(py, &input_json, None, None, None).unwrap())) + bench.iter(|| { + black_box( + validator + .validate_json(py, &input_json, None, None, None, false) + .unwrap(), + ) + }) }) } @@ -668,12 +796,20 @@ fn literal_strings_many_large_json(bench: &mut Bencher) { let input_json = py.eval_bound("'\"' + 'a' * 25 + '99' + '\"'", None, None).unwrap(); let input_json = input_json.to_object(py).into_bound(py); let input_str: String = input.extract().unwrap(); - let result = validator.validate_json(py, &input_json, None, None, None).unwrap(); + let result = validator + .validate_json(py, &input_json, None, None, None, false) + .unwrap(); let result_str: String = result.extract(py).unwrap(); assert_eq!(result_str, input_str); let input_json = black_box(input_json); - bench.iter(|| black_box(validator.validate_json(py, &input_json, None, None, None).unwrap())) + bench.iter(|| { + black_box( + validator + .validate_json(py, &input_json, None, None, None, false) + .unwrap(), + ) + }) }) } @@ -706,12 +842,20 @@ class Foo(Enum): let input = py.eval_bound("'null'", None, None).unwrap(); let input = input.to_object(py).into_bound(py); let input_str: String = input.extract().unwrap(); - let result = validator.validate_python(py, &input, None, None, None, None).unwrap(); + let result = validator + .validate_python(py, &input, None, None, None, None, false) + .unwrap(); let result_str: String = result.extract(py).unwrap(); assert_eq!(result_str, input_str); let input = black_box(input); - bench.iter(|| black_box(validator.validate_python(py, &input, None, None, None, None).unwrap())) + bench.iter(|| { + black_box( + validator + .validate_python(py, &input, None, None, None, None, false) + .unwrap(), + ) + }) } // Int @@ -719,34 +863,58 @@ class Foo(Enum): let input = py.eval_bound("-1", None, None).unwrap(); let input = input.to_object(py).into_bound(py); let input_int: i64 = input.extract().unwrap(); - let result = validator.validate_python(py, &input, None, None, None, None).unwrap(); + let result = validator + .validate_python(py, &input, None, None, None, None, false) + .unwrap(); let result_int: i64 = result.extract(py).unwrap(); assert_eq!(result_int, input_int); let input = black_box(input); - bench.iter(|| black_box(validator.validate_python(py, &input, None, None, None, None).unwrap())) + bench.iter(|| { + black_box( + validator + .validate_python(py, &input, None, None, None, None, false) + .unwrap(), + ) + }) } // None { let input = py.eval_bound("None", None, None).unwrap(); let input = input.to_object(py).into_bound(py); - let result = validator.validate_python(py, &input, None, None, None, None).unwrap(); + let result = validator + .validate_python(py, &input, None, None, None, None, false) + .unwrap(); assert!(input.eq(result).unwrap()); let input = black_box(input); - bench.iter(|| black_box(validator.validate_python(py, &input, None, None, None, None).unwrap())) + bench.iter(|| { + black_box( + validator + .validate_python(py, &input, None, None, None, None, false) + .unwrap(), + ) + }) } // Enum { let input = py.eval_bound("Foo.v4", Some(&globals), None).unwrap(); let input = input.to_object(py).into_bound(py); - let result = validator.validate_python(py, &input, None, None, None, None).unwrap(); + let result = validator + .validate_python(py, &input, None, None, None, None, false) + .unwrap(); assert!(input.eq(result).unwrap()); let input = black_box(input); - bench.iter(|| black_box(validator.validate_python(py, &input, None, None, None, None).unwrap())) + bench.iter(|| { + black_box( + validator + .validate_python(py, &input, None, None, None, None, false) + .unwrap(), + ) + }) } }) } diff --git a/python/pydantic_core/_pydantic_core.pyi b/python/pydantic_core/_pydantic_core.pyi index d696ac407..69bfb64a8 100644 --- a/python/pydantic_core/_pydantic_core.pyi +++ b/python/pydantic_core/_pydantic_core.pyi @@ -96,6 +96,7 @@ class SchemaValidator: from_attributes: bool | None = None, context: Any | None = None, self_instance: Any | None = None, + allow_partial: bool = False, ) -> Any: """ Validate a Python object against the schema and return the validated object. @@ -110,6 +111,8 @@ class SchemaValidator: [`info.context`][pydantic_core.core_schema.ValidationInfo.context]. self_instance: An instance of a model set attributes on from validation, this is used when running validation from the `__init__` method of a model. + allow_partial: Whether to allow partial validation; if `True` errors in the last element of sequences + and mappings are ignored. Raises: ValidationError: If validation fails. @@ -143,6 +146,7 @@ class SchemaValidator: strict: bool | None = None, context: Any | None = None, self_instance: Any | None = None, + allow_partial: bool = False, ) -> Any: """ Validate JSON data directly against the schema and return the validated Python object. @@ -160,6 +164,8 @@ class SchemaValidator: context: The context to use for validation, this is passed to functional validators as [`info.context`][pydantic_core.core_schema.ValidationInfo.context]. self_instance: An instance of a model set attributes on from validation. + allow_partial: Whether to allow partial validation; if `True` incomplete JSON will be parsed successfully + and errors in the last element of sequences and mappings are ignored. Raises: ValidationError: If validation fails or if the JSON data is invalid. @@ -168,7 +174,14 @@ class SchemaValidator: Returns: The validated Python object. """ - def validate_strings(self, input: _StringInput, *, strict: bool | None = None, context: Any | None = None) -> Any: + def validate_strings( + self, + input: _StringInput, + *, + strict: bool | None = None, + context: Any | None = None, + allow_partial: bool = False, + ) -> Any: """ Validate a string against the schema and return the validated Python object. @@ -181,6 +194,8 @@ class SchemaValidator: If `None`, the value of [`CoreConfig.strict`][pydantic_core.core_schema.CoreConfig] is used. context: The context to use for validation, this is passed to functional validators as [`info.context`][pydantic_core.core_schema.ValidationInfo.context]. + allow_partial: Whether to allow partial validation; if `True` errors in the last element of sequences + and mappings are ignored. Raises: ValidationError: If validation fails or if the JSON data is invalid. diff --git a/python/pydantic_core/core_schema.py b/python/pydantic_core/core_schema.py index 5a8646fca..c023a5635 100644 --- a/python/pydantic_core/core_schema.py +++ b/python/pydantic_core/core_schema.py @@ -2840,7 +2840,7 @@ def typed_dict_field( Args: schema: The schema to use for the field - required: Whether the field is required + required: Whether the field is required, otherwise uses the value from `total` on the typed dict validation_alias: The alias(es) to use to find the field in the validation data serialization_alias: The alias to use as a key when serializing serialization_exclude: Whether to exclude the field when serializing @@ -2916,7 +2916,7 @@ class MyTypedDict(TypedDict): ref: optional unique identifier of the schema, used to reference the schema in other places metadata: Any other information you want to include with the schema, not used by pydantic-core extra_behavior: The extra behavior to use for the typed dict - total: Whether the typed dict is total + total: Whether the typed dict is total, otherwise uses `typed_dict_total` from config populate_by_name: Whether the typed dict should populate by name serialization: Custom serialization schema """ diff --git a/src/errors/line_error.rs b/src/errors/line_error.rs index c3d2b66dc..03e770cc3 100644 --- a/src/errors/line_error.rs +++ b/src/errors/line_error.rs @@ -145,6 +145,14 @@ impl ValLineError { self.error_type = error_type; self } + + pub fn first_loc_item(&self) -> Option<&LocItem> { + match &self.location { + Location::Empty => None, + // last because order is reversed + Location::List(loc_items) => loc_items.last(), + } + } } #[cfg_attr(debug_assertions, derive(Debug))] diff --git a/src/errors/location.rs b/src/errors/location.rs index 07e1623d7..e9e20ab5d 100644 --- a/src/errors/location.rs +++ b/src/errors/location.rs @@ -12,7 +12,7 @@ use crate::lookup_key::{LookupPath, PathItem}; /// Used to store individual items of the error location, e.g. a string for key/field names /// or a number for array indices. -#[derive(Clone)] +#[derive(Clone, Eq, PartialEq)] #[cfg_attr(debug_assertions, derive(Debug))] pub enum LocItem { /// string type key, used to identify items from a dict or anything that implements `__getitem__` diff --git a/src/input/input_abstract.rs b/src/input/input_abstract.rs index 01deea4bc..05473c702 100644 --- a/src/input/input_abstract.rs +++ b/src/input/input_abstract.rs @@ -236,6 +236,8 @@ pub trait ValidatedDict<'py> { &'a self, consumer: impl ConsumeIterator, Self::Item<'a>)>, Output = R>, ) -> ValResult; + // used in partial mode to check all errors occurred in the last value + fn last_key(&self) -> Option>; } /// For validations from a list @@ -276,6 +278,9 @@ impl<'py> ValidatedDict<'py> for Never { ) -> ValResult { unreachable!() } + fn last_key(&self) -> Option> { + unreachable!() + } } impl<'py> ValidatedList<'py> for Never { diff --git a/src/input/input_json.rs b/src/input/input_json.rs index 7c77928d0..d69874613 100644 --- a/src/input/input_json.rs +++ b/src/input/input_json.rs @@ -516,6 +516,10 @@ impl<'py, 'data> ValidatedDict<'py> for &'_ JsonObject<'data> { ) -> ValResult { Ok(consumer.consume_iterator(LazyIndexMap::iter(self).map(|(k, v)| Ok((k.as_ref(), v))))) } + + fn last_key(&self) -> Option> { + self.keys().last().map(AsRef::as_ref) + } } impl<'a, 'py, 'data> ValidatedList<'py> for &'a JsonArray<'data> { diff --git a/src/input/input_python.rs b/src/input/input_python.rs index b9d95b8bf..4fb88e99c 100644 --- a/src/input/input_python.rs +++ b/src/input/input_python.rs @@ -823,6 +823,21 @@ impl<'py> ValidatedDict<'py> for GenericPyMapping<'_, 'py> { Self::GetAttr(obj, _) => Ok(consumer.consume_iterator(iterate_attributes(obj)?)), } } + + fn last_key(&self) -> Option> { + match self { + Self::Dict(dict) => dict.keys().iter().last(), + // see https://github.com/pydantic/pydantic-core/pull/1512#discussion_r1826057970 + Self::Mapping(mapping) => mapping + .call_method0(intern!(mapping.py(), "keys")) + .ok()? + .iter() + .ok()? + .last()? + .ok(), + Self::GetAttr(_, _) => None, + } + } } /// Container for all the collections (sized iterable containers) types, which diff --git a/src/input/input_string.rs b/src/input/input_string.rs index 1e6b14825..2ac218658 100644 --- a/src/input/input_string.rs +++ b/src/input/input_string.rs @@ -303,4 +303,12 @@ impl<'py> ValidatedDict<'py> for StringMappingDict<'py> { .map(|(key, val)| Ok((StringMapping::new_key(key)?, StringMapping::new_value(val)?))), )) } + + fn last_key(&self) -> Option> { + self.0 + .keys() + .iter() + .last() + .and_then(|key| StringMapping::new_key(key).ok()) + } } diff --git a/src/input/return_enums.rs b/src/input/return_enums.rs index 888dff714..d5e31b97b 100644 --- a/src/input/return_enums.rs +++ b/src/input/return_enums.rs @@ -128,7 +128,9 @@ pub(crate) fn validate_iter_to_vec<'py>( ) -> ValResult> { let mut output: Vec = Vec::with_capacity(capacity); let mut errors: Vec = Vec::new(); - for (index, item_result) in iter.enumerate() { + + for (index, is_last_partial, item_result) in state.enumerate_last_partial(iter) { + state.allow_partial = is_last_partial; let item = item_result.map_err(|e| any_next_error!(py, e, max_length_check.input, index))?; match validator.validate(py, item.borrow_input(), state) { Ok(item) => { @@ -137,9 +139,11 @@ pub(crate) fn validate_iter_to_vec<'py>( } Err(ValError::LineErrors(line_errors)) => { max_length_check.incr()?; - errors.extend(line_errors.into_iter().map(|err| err.with_outer_location(index))); - if fail_fast { - break; + if !is_last_partial { + errors.extend(line_errors.into_iter().map(|err| err.with_outer_location(index))); + if fail_fast { + return Err(ValError::LineErrors(errors)); + } } } Err(ValError::Omit) => (), @@ -197,7 +201,9 @@ pub(crate) fn validate_iter_to_set<'py>( fail_fast: bool, ) -> ValResult<()> { let mut errors: Vec = Vec::new(); - for (index, item_result) in iter.enumerate() { + + for (index, is_last_partial, item_result) in state.enumerate_last_partial(iter) { + state.allow_partial = is_last_partial; let item = item_result.map_err(|e| any_next_error!(py, e, input, index))?; match validator.validate(py, item.borrow_input(), state) { Ok(item) => { @@ -220,13 +226,15 @@ pub(crate) fn validate_iter_to_set<'py>( } } Err(ValError::LineErrors(line_errors)) => { - errors.extend(line_errors.into_iter().map(|err| err.with_outer_location(index))); + if !is_last_partial { + errors.extend(line_errors.into_iter().map(|err| err.with_outer_location(index))); + } } Err(ValError::Omit) => (), Err(err) => return Err(err), } if fail_fast && !errors.is_empty() { - break; + return Err(ValError::LineErrors(errors)); } } diff --git a/src/serializers/fields.rs b/src/serializers/fields.rs index 32c16b18a..6cd76c36b 100644 --- a/src/serializers/fields.rs +++ b/src/serializers/fields.rs @@ -53,7 +53,7 @@ impl SerField { } } - pub fn get_key_py<'py>(&'py self, py: Python<'py>, extra: &Extra) -> &Bound<'py, PyAny> { + pub fn get_key_py<'py>(&self, py: Python<'py>, extra: &Extra) -> &Bound<'py, PyAny> { if extra.by_alias { if let Some(ref alias_py) = self.alias_py { return alias_py.bind(py); diff --git a/src/url.rs b/src/url.rs index 881347f25..3ba2c8451 100644 --- a/src/url.rs +++ b/src/url.rs @@ -45,7 +45,7 @@ impl PyUrl { pub fn py_new(py: Python, url: &Bound<'_, PyAny>) -> PyResult { let schema_obj = SCHEMA_DEFINITION_URL .get_or_init(py, || build_schema_validator(py, "url")) - .validate_python(py, url, None, None, None, None)?; + .validate_python(py, url, None, None, None, None, false)?; schema_obj.extract(py) } @@ -225,7 +225,7 @@ impl PyMultiHostUrl { pub fn py_new(py: Python, url: &Bound<'_, PyAny>) -> PyResult { let schema_obj = SCHEMA_DEFINITION_MULTI_HOST_URL .get_or_init(py, || build_schema_validator(py, "multi-host-url")) - .validate_python(py, url, None, None, None, None)?; + .validate_python(py, url, None, None, None, None, false)?; schema_obj.extract(py) } diff --git a/src/validators/arguments.rs b/src/validators/arguments.rs index 22f870c56..0d90af5d3 100644 --- a/src/validators/arguments.rs +++ b/src/validators/arguments.rs @@ -188,6 +188,9 @@ impl Validator for ArgumentsValidator { input: &(impl Input<'py> + ?Sized), state: &mut ValidationState<'_, 'py>, ) -> ValResult { + // this validator does not yet support partial validation, disable it to avoid incorrect results + state.allow_partial = false; + let args = input.validate_args()?; let mut output_args: Vec = Vec::with_capacity(self.positional_params_count); diff --git a/src/validators/dataclass.rs b/src/validators/dataclass.rs index 4bece7ea5..86211b7b2 100644 --- a/src/validators/dataclass.rs +++ b/src/validators/dataclass.rs @@ -145,6 +145,9 @@ impl Validator for DataclassArgsValidator { input: &(impl Input<'py> + ?Sized), state: &mut ValidationState<'_, 'py>, ) -> ValResult { + // this validator does not yet support partial validation, disable it to avoid incorrect results + state.allow_partial = false; + let args = input.validate_dataclass_args(&self.dataclass_name)?; let output_dict = PyDict::new_bound(py); diff --git a/src/validators/definitions.rs b/src/validators/definitions.rs index a96138c48..34d0edef4 100644 --- a/src/validators/definitions.rs +++ b/src/validators/definitions.rs @@ -75,6 +75,9 @@ impl Validator for DefinitionRefValidator { input: &(impl Input<'py> + ?Sized), state: &mut ValidationState<'_, 'py>, ) -> ValResult { + // this validator does not yet support partial validation, disable it to avoid incorrect results + state.allow_partial = false; + self.definition.read(|validator| { let validator = validator.unwrap(); if let Some(id) = input.as_python().map(py_identity) { diff --git a/src/validators/dict.rs b/src/validators/dict.rs index ea0000236..adfadc6fa 100644 --- a/src/validators/dict.rs +++ b/src/validators/dict.rs @@ -110,7 +110,8 @@ where let output = PyDict::new_bound(self.py); let mut errors: Vec = Vec::new(); - for item_result in iterator { + for (_, is_last_partial, item_result) in self.state.enumerate_last_partial(iterator) { + self.state.allow_partial = false; let (key, value) = item_result?; let output_key = match self.key_validator.validate(self.py, key.borrow_input(), self.state) { Ok(value) => Some(value), @@ -124,19 +125,20 @@ where Err(ValError::Omit) => continue, Err(err) => return Err(err), }; + self.state.allow_partial = is_last_partial; let output_value = match self.value_validator.validate(self.py, value.borrow_input(), self.state) { - Ok(value) => Some(value), + Ok(value) => value, Err(ValError::LineErrors(line_errors)) => { - for err in line_errors { - errors.push(err.with_outer_location(key.clone())); + if !is_last_partial { + errors.extend(line_errors.into_iter().map(|err| err.with_outer_location(key.clone()))); } - None + continue; } Err(ValError::Omit) => continue, Err(err) => return Err(err), }; - if let (Some(key), Some(value)) = (output_key, output_value) { - output.set_item(key, value)?; + if let Some(key) = output_key { + output.set_item(key, output_value)?; } } diff --git a/src/validators/generator.rs b/src/validators/generator.rs index de9949a8c..12241f205 100644 --- a/src/validators/generator.rs +++ b/src/validators/generator.rs @@ -66,6 +66,9 @@ impl Validator for GeneratorValidator { input: &(impl Input<'py> + ?Sized), state: &mut ValidationState<'_, 'py>, ) -> ValResult { + // this validator does not yet support partial validation, disable it to avoid incorrect results + state.allow_partial = false; + let iterator = input.validate_iter()?.into_static(); let validator = self.item_validator.as_ref().map(|v| { InternalValidator::new( @@ -279,7 +282,7 @@ impl InternalValidator { self_instance: self.self_instance.as_ref().map(|data| data.bind(py)), cache_str: self.cache_str, }; - let mut state = ValidationState::new(extra, &mut self.recursion_guard); + let mut state = ValidationState::new(extra, &mut self.recursion_guard, false); state.exactness = self.exactness; let result = self .validator @@ -314,7 +317,7 @@ impl InternalValidator { self_instance: self.self_instance.as_ref().map(|data| data.bind(py)), cache_str: self.cache_str, }; - let mut state = ValidationState::new(extra, &mut self.recursion_guard); + let mut state = ValidationState::new(extra, &mut self.recursion_guard, false); state.exactness = self.exactness; let result = self.validator.validate(py, input, &mut state).map_err(|e| { ValidationError::from_val_error( diff --git a/src/validators/json.rs b/src/validators/json.rs index 79a36e058..9ba13dca1 100644 --- a/src/validators/json.rs +++ b/src/validators/json.rs @@ -1,9 +1,8 @@ -use jiter::FloatMode; use pyo3::intern; use pyo3::prelude::*; use pyo3::types::PyDict; -use jiter::{JsonValue, PartialMode, PythonParse}; +use jiter::{FloatMode, JsonValue, PartialMode, PythonParse}; use crate::errors::{ErrorType, ErrorTypeDefaults, ValError, ValLineError, ValResult}; use crate::input::{EitherBytes, Input, InputType, ValidationMatch}; @@ -60,7 +59,8 @@ impl Validator for JsonValidator { let json_bytes = json_either_bytes.as_slice(); match self.validator { Some(ref validator) => { - let json_value = JsonValue::parse(json_bytes, true).map_err(|e| map_json_err(input, e, json_bytes))?; + let json_value = JsonValue::parse_with_config(json_bytes, true, state.allow_partial) + .map_err(|e| map_json_err(input, e, json_bytes))?; let mut json_state = state.rebind_extra(|e| { e.input_type = InputType::Json; }); @@ -70,7 +70,11 @@ impl Validator for JsonValidator { let parse_builder = PythonParse { allow_inf_nan: true, cache_mode: state.cache_str(), - partial_mode: PartialMode::Off, + partial_mode: if state.allow_partial { + PartialMode::TrailingStrings + } else { + PartialMode::Off + }, catch_duplicate_keys: false, float_mode: FloatMode::Float, }; diff --git a/src/validators/mod.rs b/src/validators/mod.rs index fcd64ebf2..c0c58adcf 100644 --- a/src/validators/mod.rs +++ b/src/validators/mod.rs @@ -164,7 +164,8 @@ impl SchemaValidator { Ok((cls, init_args)) } - #[pyo3(signature = (input, *, strict=None, from_attributes=None, context=None, self_instance=None))] + #[allow(clippy::too_many_arguments)] + #[pyo3(signature = (input, *, strict=None, from_attributes=None, context=None, self_instance=None, allow_partial=false))] pub fn validate_python( &self, py: Python, @@ -173,6 +174,7 @@ impl SchemaValidator { from_attributes: Option, context: Option<&Bound<'_, PyAny>>, self_instance: Option<&Bound<'_, PyAny>>, + allow_partial: bool, ) -> PyResult { self._validate( py, @@ -182,6 +184,7 @@ impl SchemaValidator { from_attributes, context, self_instance, + allow_partial, ) .map_err(|e| self.prepare_validation_err(py, e, InputType::Python)) } @@ -204,6 +207,7 @@ impl SchemaValidator { from_attributes, context, self_instance, + false, ) { Ok(_) => Ok(true), Err(ValError::InternalErr(err)) => Err(err), @@ -213,7 +217,7 @@ impl SchemaValidator { } } - #[pyo3(signature = (input, *, strict=None, context=None, self_instance=None))] + #[pyo3(signature = (input, *, strict=None, context=None, self_instance=None, allow_partial=false))] pub fn validate_json( &self, py: Python, @@ -221,6 +225,7 @@ impl SchemaValidator { strict: Option, context: Option<&Bound<'_, PyAny>>, self_instance: Option<&Bound<'_, PyAny>>, + allow_partial: bool, ) -> PyResult { let r = match json::validate_json_bytes(input) { Ok(v_match) => self._validate_json( @@ -230,24 +235,26 @@ impl SchemaValidator { strict, context, self_instance, + allow_partial, ), Err(err) => Err(err), }; r.map_err(|e| self.prepare_validation_err(py, e, InputType::Json)) } - #[pyo3(signature = (input, *, strict=None, context=None))] + #[pyo3(signature = (input, *, strict=None, context=None, allow_partial=false))] pub fn validate_strings( &self, py: Python, input: Bound<'_, PyAny>, strict: Option, context: Option<&Bound<'_, PyAny>>, + allow_partial: bool, ) -> PyResult { let t = InputType::String; let string_mapping = StringMapping::new_value(input).map_err(|e| self.prepare_validation_err(py, e, t))?; - match self._validate(py, &string_mapping, t, strict, None, context, None) { + match self._validate(py, &string_mapping, t, strict, None, context, None, allow_partial) { Ok(r) => Ok(r), Err(e) => Err(self.prepare_validation_err(py, e, t)), } @@ -276,7 +283,7 @@ impl SchemaValidator { }; let guard = &mut RecursionState::default(); - let mut state = ValidationState::new(extra, guard); + let mut state = ValidationState::new(extra, guard, false); self.validator .validate_assignment(py, &obj, field_name, &field_value, &mut state) .map_err(|e| self.prepare_validation_err(py, e, InputType::Python)) @@ -299,7 +306,7 @@ impl SchemaValidator { cache_str: self.cache_str, }; let recursion_guard = &mut RecursionState::default(); - let mut state = ValidationState::new(extra, recursion_guard); + let mut state = ValidationState::new(extra, recursion_guard, false); let r = self.validator.default_value(py, None::, &mut state); match r { Ok(maybe_default) => match maybe_default { @@ -345,6 +352,7 @@ impl SchemaValidator { from_attributes: Option, context: Option<&Bound<'py, PyAny>>, self_instance: Option<&Bound<'py, PyAny>>, + allow_partial: bool, ) -> ValResult { let mut recursion_guard = RecursionState::default(); let mut state = ValidationState::new( @@ -357,10 +365,12 @@ impl SchemaValidator { self.cache_str, ), &mut recursion_guard, + allow_partial, ); self.validator.validate(py, input, &mut state) } + #[allow(clippy::too_many_arguments)] fn _validate_json( &self, py: Python, @@ -369,10 +379,20 @@ impl SchemaValidator { strict: Option, context: Option<&Bound<'_, PyAny>>, self_instance: Option<&Bound<'_, PyAny>>, + allow_partial: bool, ) -> ValResult { - let json_value = - jiter::JsonValue::parse(json_data, true).map_err(|e| json::map_json_err(input, e, json_data))?; - self._validate(py, &json_value, InputType::Json, strict, None, context, self_instance) + let json_value = jiter::JsonValue::parse_with_config(json_data, true, allow_partial) + .map_err(|e| json::map_json_err(input, e, json_data))?; + self._validate( + py, + &json_value, + InputType::Json, + strict, + None, + context, + self_instance, + allow_partial, + ) } fn prepare_validation_err(&self, py: Python, error: ValError, input_type: InputType) -> PyErr { @@ -410,6 +430,7 @@ impl<'py> SelfValidator<'py> { let mut state = ValidationState::new( Extra::new(strict, None, None, None, InputType::Python, true.into()), &mut recursion_guard, + false, ); match self.validator.validator.validate(py, schema, &mut state) { Ok(schema_obj) => Ok(schema_obj.into_bound(py)), diff --git a/src/validators/model_fields.rs b/src/validators/model_fields.rs index 7ecd1d353..eb4829c7f 100644 --- a/src/validators/model_fields.rs +++ b/src/validators/model_fields.rs @@ -120,6 +120,9 @@ impl Validator for ModelFieldsValidator { input: &(impl Input<'py> + ?Sized), state: &mut ValidationState<'_, 'py>, ) -> ValResult { + // this validator does not yet support partial validation, disable it to avoid incorrect results + state.allow_partial = false; + let strict = state.strict_or(self.strict); let from_attributes = state.extra().from_attributes.unwrap_or(self.from_attributes); diff --git a/src/validators/tuple.rs b/src/validators/tuple.rs index 8acbf2beb..2b544d417 100644 --- a/src/validators/tuple.rs +++ b/src/validators/tuple.rs @@ -272,6 +272,9 @@ impl Validator for TupleValidator { input: &(impl Input<'py> + ?Sized), state: &mut ValidationState<'_, 'py>, ) -> ValResult { + // this validator does not yet support partial validation, disable it to avoid incorrect results + state.allow_partial = false; + let collection = input.validate_tuple(state.strict_or(self.strict))?.unpack(state); let actual_length = collection.len(); diff --git a/src/validators/typed_dict.rs b/src/validators/typed_dict.rs index f72d969a8..949cff9a5 100644 --- a/src/validators/typed_dict.rs +++ b/src/validators/typed_dict.rs @@ -124,7 +124,6 @@ impl BuildValidator for TypedDictValidator { required, }); } - Ok(Self { fields, extra_behavior, @@ -154,6 +153,12 @@ impl Validator for TypedDictValidator { let output_dict = PyDict::new_bound(py); let mut errors: Vec = Vec::with_capacity(self.fields.len()); + let partial_last_key = if state.allow_partial { + dict.last_key().map(Into::into) + } else { + None + }; + // we only care about which keys have been used if we're iterating over the object for extra after // the first pass let mut used_keys: Option> = @@ -171,8 +176,11 @@ impl Validator for TypedDictValidator { let op_key_value = match dict.get_item(&field.lookup_key) { Ok(v) => v, Err(ValError::LineErrors(line_errors)) => { - for err in line_errors { - errors.push(err.with_outer_location(&field.name)); + let field_loc: LocItem = field.name.clone().into(); + if partial_last_key.as_ref() == Some(&field_loc) { + for err in line_errors { + errors.push(err.with_outer_location(field_loc.clone())); + } } continue; } @@ -184,6 +192,13 @@ impl Validator for TypedDictValidator { // extra logic either way used_keys.insert(lookup_path.first_key()); } + let is_last_partial = if let Some(ref last_key) = partial_last_key { + let first_key_loc: LocItem = lookup_path.first_key().into(); + &first_key_loc == last_key + } else { + false + }; + state.allow_partial = is_last_partial; match field.validator.validate(py, value.borrow_input(), state) { Ok(value) => { output_dict.set_item(&field.name_py, value)?; @@ -191,8 +206,10 @@ impl Validator for TypedDictValidator { } Err(ValError::Omit) => continue, Err(ValError::LineErrors(line_errors)) => { - for err in line_errors { - errors.push(lookup_path.apply_error_loc(err, self.loc_by_alias, &field.name)); + if !is_last_partial || field.required { + for err in line_errors { + errors.push(lookup_path.apply_error_loc(err, self.loc_by_alias, &field.name)); + } } } Err(err) => return Err(err), @@ -242,6 +259,7 @@ impl Validator for TypedDictValidator { output_dict: &'a Bound<'py, PyDict>, state: &'a mut ValidationState<'s, 'py>, extra_behavior: ExtraBehavior, + partial_last_key: Option, } impl<'py, Key, Value> ConsumeIterator> for ValidateExtras<'_, '_, 'py> @@ -289,13 +307,20 @@ impl Validator for TypedDictValidator { ExtraBehavior::Allow => { let py_key = either_str.as_py_string(self.py, self.state.cache_str()); if let Some(validator) = self.extras_validator { + let last_partial = self.partial_last_key.as_ref().map_or(false, |last_key| { + let key_loc: LocItem = raw_key.clone().into(); + &key_loc == last_key + }); + self.state.allow_partial = last_partial; match validator.validate(self.py, value, self.state) { Ok(value) => { self.output_dict.set_item(py_key, value)?; } Err(ValError::LineErrors(line_errors)) => { - for err in line_errors { - self.errors.push(err.with_outer_location(raw_key.clone())); + if !last_partial { + for err in line_errors { + self.errors.push(err.with_outer_location(raw_key.clone())); + } } } Err(err) => return Err(err), @@ -319,13 +344,14 @@ impl Validator for TypedDictValidator { output_dict: &output_dict, state, extra_behavior: self.extra_behavior, + partial_last_key, })??; } - if !errors.is_empty() { - Err(ValError::LineErrors(errors)) - } else { + if errors.is_empty() { Ok(output_dict.to_object(py)) + } else { + Err(ValError::LineErrors(errors)) } } diff --git a/src/validators/validation_state.rs b/src/validators/validation_state.rs index b125cd316..b21284a99 100644 --- a/src/validators/validation_state.rs +++ b/src/validators/validation_state.rs @@ -23,16 +23,19 @@ pub struct ValidationState<'a, 'py> { // `model_fields_set` attached to a model. `model_fields_set` includes extra fields // when extra='allow', whereas this tally does not. pub fields_set_count: Option, + // True if `allow_partial=true` and we're validating the last element of a sequence or mapping. + pub allow_partial: bool, // deliberately make Extra readonly extra: Extra<'a, 'py>, } impl<'a, 'py> ValidationState<'a, 'py> { - pub fn new(extra: Extra<'a, 'py>, recursion_guard: &'a mut RecursionState) -> Self { + pub fn new(extra: Extra<'a, 'py>, recursion_guard: &'a mut RecursionState, allow_partial: bool) -> Self { Self { recursion_guard, // Don't care about exactness unless doing union validation exactness: None, fields_set_count: None, + allow_partial, extra, } } @@ -53,6 +56,10 @@ impl<'a, 'py> ValidationState<'a, 'py> { &self.extra } + pub fn enumerate_last_partial(&self, iter: impl Iterator) -> impl Iterator { + EnumerateLastPartial::new(iter, self.allow_partial) + } + pub fn strict_or(&self, default: bool) -> bool { self.extra.strict.unwrap_or(default) } @@ -117,3 +124,37 @@ impl Drop for ValidationStateWithReboundExtra<'_, '_, '_> { std::mem::swap(&mut self.state.extra, &mut self.old_extra); } } + +/// Similar to `iter.enumerate()` but also returns a bool indicating if we're at the last element. +pub struct EnumerateLastPartial { + iter: I, + index: usize, + next_item: Option, + allow_partial: bool, +} +impl EnumerateLastPartial { + pub fn new(mut iter: I, allow_partial: bool) -> Self { + let next_item = iter.next(); + Self { + iter, + index: 0, + next_item, + allow_partial, + } + } +} + +impl Iterator for EnumerateLastPartial { + type Item = (usize, bool, I::Item); + + fn next(&mut self) -> Option { + let a = std::mem::replace(&mut self.next_item, self.iter.next())?; + let i = self.index; + self.index += 1; + Some((i, self.allow_partial && self.next_item.is_none(), a)) + } + + fn size_hint(&self) -> (usize, Option) { + self.iter.size_hint() + } +} diff --git a/tests/emscripten_runner.js b/tests/emscripten_runner.js index c3c53834e..a49c1b6ad 100644 --- a/tests/emscripten_runner.js +++ b/tests/emscripten_runner.js @@ -86,7 +86,7 @@ async function main() { FS.mkdir('/test_dir'); FS.mount(FS.filesystems.NODEFS, {root: path.join(root_dir, 'tests')}, '/test_dir'); FS.chdir('/test_dir'); - await pyodide.loadPackage(['micropip', 'pytest']); + await pyodide.loadPackage(['micropip', 'pytest', 'pygments']); // language=python errcode = await pyodide.runPythonAsync(` import micropip @@ -98,6 +98,7 @@ import importlib await micropip.install([ 'dirty-equals', + 'inline-snapshot', 'hypothesis', 'pytest-speed', 'pytest-mock', diff --git a/tests/requirements.txt b/tests/requirements.txt index 5ee5ebfda..3fab8cc74 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,6 +1,7 @@ backports.zoneinfo==0.2.1;python_version<"3.9" coverage==7.6.1 dirty-equals==0.8.0 +inline-snapshot==0.13.3 hypothesis==6.111.2 # pandas doesn't offer prebuilt wheels for all versions and platforms we test in CI e.g. aarch64 musllinux pandas==2.1.3; python_version >= "3.9" and python_version < "3.13" and implementation_name == "cpython" and platform_machine == 'x86_64' diff --git a/tests/test.rs b/tests/test.rs index 1a4a87ef3..fb141d377 100644 --- a/tests/test.rs +++ b/tests/test.rs @@ -129,7 +129,7 @@ json_input = '{"a": "something"}' let json_input = locals.get_item("json_input").unwrap().unwrap(); let binding = SchemaValidator::py_new(py, &schema, None) .unwrap() - .validate_json(py, &json_input, None, None, None) + .validate_json(py, &json_input, None, None, None, false) .unwrap(); let validation_result: Bound<'_, PyAny> = binding.extract(py).unwrap(); let repr = format!("{}", validation_result.repr().unwrap()); diff --git a/tests/validators/test_allow_partial.py b/tests/validators/test_allow_partial.py new file mode 100644 index 000000000..e147ff62d --- /dev/null +++ b/tests/validators/test_allow_partial.py @@ -0,0 +1,313 @@ +from typing import Mapping + +import pytest +from dirty_equals import IsStrictDict +from inline_snapshot import snapshot + +from pydantic_core import SchemaValidator, ValidationError, core_schema + + +def test_list(): + v = SchemaValidator( + core_schema.list_schema( + core_schema.tuple_positional_schema([core_schema.int_schema(), core_schema.int_schema()]), + ) + ) + assert v.validate_python([[1, 2], [3, 4]]) == [(1, 2), (3, 4)] + assert v.validate_python([[1, 2], [3, 4]], allow_partial=True) == [(1, 2), (3, 4)] + with pytest.raises(ValidationError) as exc_info: + v.validate_python([[1, 2], 'wrong']) + assert exc_info.value.errors(include_url=False) == snapshot( + [ + { + 'type': 'tuple_type', + 'loc': (1,), + 'msg': 'Input should be a valid tuple', + 'input': 'wrong', + } + ] + ) + assert v.validate_python([[1, 2], 'wrong'], allow_partial=True) == [(1, 2)] + assert v.validate_python([[1, 2], []], allow_partial=True) == [(1, 2)] + assert v.validate_python([[1, 2], [3]], allow_partial=True) == [(1, 2)] + assert v.validate_python([[1, 2], [3, 'x']], allow_partial=True) == [(1, 2)] + with pytest.raises(ValidationError, match='Input should be a valid tuple'): + v.validate_python([[1, 2], 'wrong', [3, 4]]) + with pytest.raises(ValidationError, match='Input should be a valid tuple'): + v.validate_python([[1, 2], 'wrong', 'wrong']) + assert v.validate_json(b'[[1, 2], [3, 4]]', allow_partial=True) == [(1, 2), (3, 4)] + assert v.validate_json(b'[[1, 2], [3,', allow_partial=True) == [(1, 2)] + + +@pytest.mark.parametrize('collection_type', [core_schema.set_schema, core_schema.frozenset_schema]) +def test_set_frozenset(collection_type): + v = SchemaValidator( + collection_type( + core_schema.tuple_positional_schema([core_schema.int_schema(), core_schema.int_schema()]), + ) + ) + assert v.validate_python([[1, 2], [3, 4]]) == snapshot({(1, 2), (3, 4)}) + assert v.validate_python([[1, 2], [3, 4]], allow_partial=True) == snapshot({(1, 2), (3, 4)}) + with pytest.raises(ValidationError) as exc_info: + v.validate_python([[1, 2], 'wrong']) + assert exc_info.value.errors(include_url=False) == snapshot( + [ + { + 'type': 'tuple_type', + 'loc': (1,), + 'msg': 'Input should be a valid tuple', + 'input': 'wrong', + } + ] + ) + assert v.validate_python([[1, 2], 'wrong'], allow_partial=True) == snapshot({(1, 2)}) + assert v.validate_python([[1, 2], [3, 4], 'wrong'], allow_partial=True) == snapshot({(1, 2), (3, 4)}) + assert v.validate_python([[1, 2], []], allow_partial=True) == snapshot({(1, 2)}) + assert v.validate_python([[1, 2], [3]], allow_partial=True) == snapshot({(1, 2)}) + assert v.validate_python([[1, 2], [3, 'x']], allow_partial=True) == snapshot({(1, 2)}) + with pytest.raises(ValidationError, match='Input should be a valid tuple'): + v.validate_python([[1, 2], 'wrong', [3, 4]]) + with pytest.raises(ValidationError, match='Input should be a valid tuple'): + v.validate_python([[1, 2], 'wrong', 'wrong']) + + +class MyMapping(Mapping): + def __init__(self, d): + self._d = d + + def __getitem__(self, key): + return self._d[key] + + def __iter__(self): + return iter(self._d) + + def __len__(self): + return len(self._d) + + +def test_dict(): + v = SchemaValidator(core_schema.dict_schema(core_schema.int_schema(), core_schema.int_schema())) + assert v.validate_python({'1': 2, 3: '4'}) == snapshot({1: 2, 3: 4}) + assert v.validate_python({'1': 2, 3: '4'}, allow_partial=True) == snapshot({1: 2, 3: 4}) + assert v.validate_python(MyMapping({'1': 2, 3: '4'}), allow_partial=True) == snapshot({1: 2, 3: 4}) + with pytest.raises(ValidationError) as exc_info: + v.validate_python({'1': 2, 3: 'wrong'}) + assert exc_info.value.errors(include_url=False) == snapshot( + [ + { + 'type': 'int_parsing', + 'loc': (3,), + 'msg': 'Input should be a valid integer, unable to parse string as an integer', + 'input': 'wrong', + } + ] + ) + assert v.validate_python({'1': 2, 3: 'x'}, allow_partial=True) == snapshot({1: 2}) + assert v.validate_python(MyMapping({'1': 2, 3: 'x'}), allow_partial=True) == snapshot({1: 2}) + assert v.validate_python({'1': 2, 3: 4, 5: '6', 7: 'x'}, allow_partial=True) == snapshot({1: 2, 3: 4, 5: 6}) + with pytest.raises(ValidationError, match='Input should be a valid integer'): + v.validate_python({'1': 2, 3: 4, 5: 'x', 7: '8'}) + with pytest.raises(ValidationError, match='Input should be a valid integer'): + v.validate_python({'1': 2, 3: 4, 5: 'x', 7: 'x'}) + with pytest.raises(ValidationError, match='Input should be a valid integer'): + v.validate_python({'1': 2, 3: 4, 'x': 6}) + + +def test_dict_list(): + v = SchemaValidator( + core_schema.dict_schema(core_schema.int_schema(), core_schema.list_schema(core_schema.int_schema(ge=10))) + ) + assert v.validate_python({'1': [20, 30], 3: [40, '50']}, allow_partial=True) == snapshot({1: [20, 30], 3: [40, 50]}) + assert v.validate_python({'1': [20, 30], 3: [40, 5]}, allow_partial=True) == snapshot({1: [20, 30], 3: [40]}) + + with pytest.raises(ValidationError, match=r'1\.1\s+Input should be greater than or equal to 10'): + v.validate_python({'1': [20, 3], 3: [40, 50]}, allow_partial=True) + + +def test_partial_typed_dict(): + v = SchemaValidator( + core_schema.typed_dict_schema( + { + 'a': core_schema.typed_dict_field(core_schema.int_schema(gt=10)), + 'b': core_schema.typed_dict_field(core_schema.int_schema(gt=10)), + 'c': core_schema.typed_dict_field(core_schema.int_schema(gt=10)), + }, + total=False, + ) + ) + + assert v.validate_python({'a': 11, 'b': '12', 'c': 13}) == snapshot(IsStrictDict(a=11, b=12, c=13)) + assert v.validate_python({'a': 11, 'c': 13, 'b': '12'}) == snapshot(IsStrictDict(a=11, b=12, c=13)) + assert v.validate_python(MyMapping({'a': 11, 'c': 13, 'b': '12'})) == snapshot(IsStrictDict(a=11, b=12, c=13)) + + assert v.validate_python({'a': 11, 'b': '12', 'c': 13}, allow_partial=True) == snapshot({'a': 11, 'b': 12, 'c': 13}) + with pytest.raises(ValidationError) as exc_info: + v.validate_python({'a': 11, 'b': '12', 'c': 1}) + assert exc_info.value.errors(include_url=False) == snapshot( + [ + { + 'type': 'greater_than', + 'loc': ('c',), + 'msg': 'Input should be greater than 10', + 'input': 1, + 'ctx': {'gt': 10}, + } + ] + ) + assert v.validate_python({'a': 11, 'b': '12', 'c': 1}, allow_partial=True) == snapshot(IsStrictDict(a=11, b=12)) + assert v.validate_python(MyMapping({'a': 11, 'b': '12', 'c': 1}), allow_partial=True) == snapshot( + IsStrictDict(a=11, b=12) + ) + assert v.validate_python({'a': 11, 'c': 13, 'b': 1}, allow_partial=True) == snapshot(IsStrictDict(a=11, c=13)) + with pytest.raises(ValidationError) as exc_info: + v.validate_python({'a': 11, 'c': 1, 'b': 12}, allow_partial=True) + assert exc_info.value.errors(include_url=False) == snapshot( + [ + { + 'type': 'greater_than', + 'loc': ('c',), + 'msg': 'Input should be greater than 10', + 'input': 1, + 'ctx': {'gt': 10}, + } + ] + ) + with pytest.raises(ValidationError, match=r'c\s+Input should be greater than 10'): + v.validate_python(MyMapping({'a': 11, 'c': 1, 'b': 12}), allow_partial=True) + + # validate strings + assert v.validate_strings({'a': '11', 'b': '22'}) == snapshot({'a': 11, 'b': 22}) + with pytest.raises(ValidationError, match='Input should be greater than 10'): + v.validate_strings({'a': '11', 'b': '2'}) + assert v.validate_strings({'a': '11', 'b': '2'}, allow_partial=True) == snapshot({'a': 11}) + + assert v.validate_json(b'{"b": "12", "a": 11, "c": 13}', allow_partial=True) == IsStrictDict(a=11, b=12, c=13) + assert v.validate_json(b'{"b": "12", "a": 11, "c": 13', allow_partial=True) == IsStrictDict(a=11, b=12, c=13) + assert v.validate_json(b'{"a": 11, "b": "12", "c": 1', allow_partial=True) == IsStrictDict(a=11, b=12) + assert v.validate_json(b'{"a": 11, "b": "12", "c":', allow_partial=True) == IsStrictDict(a=11, b=12) + assert v.validate_json(b'{"a": 11, "b": "12", "c"', allow_partial=True) == IsStrictDict(a=11, b=12) + assert v.validate_json(b'{"a": 11, "b": "12", "c', allow_partial=True) == IsStrictDict(a=11, b=12) + assert v.validate_json(b'{"a": 11, "b": "12", "', allow_partial=True) == IsStrictDict(a=11, b=12) + assert v.validate_json(b'{"a": 11, "b": "12", ', allow_partial=True) == IsStrictDict(a=11, b=12) + assert v.validate_json(b'{"a": 11, "b": "12",', allow_partial=True) == IsStrictDict(a=11, b=12) + assert v.validate_json(b'{"a": 11, "b": "12"', allow_partial=True) == IsStrictDict(a=11, b=12) + + +def test_non_partial_typed_dict(): + v = SchemaValidator( + core_schema.typed_dict_schema( + { + 'a': core_schema.typed_dict_field(core_schema.int_schema(gt=10)), + 'b': core_schema.typed_dict_field(core_schema.int_schema(gt=10), required=True), + 'c': core_schema.typed_dict_field(core_schema.int_schema(gt=10)), + }, + total=False, + ) + ) + + assert v.validate_python({'a': 11, 'b': '12', 'c': 13}) == snapshot({'a': 11, 'b': 12, 'c': 13}) + with pytest.raises(ValidationError, match='Input should be greater than 10'): + v.validate_python({'a': 11, 'b': '12', 'c': 1}) + assert v.validate_python({'a': 11, 'b': '12', 'c': 1}, allow_partial=True) == snapshot({'a': 11, 'b': 12}) + with pytest.raises(ValidationError, match=r'b\s+Field required'): + v.validate_python({'a': 11, 'c': 12}, allow_partial=True) + with pytest.raises(ValidationError, match=r'b\s+Input should be greater than 10'): + v.validate_python({'a': 11, 'c': 12, 'b': 1}, allow_partial=True) + + +def test_double_nested(): + v = SchemaValidator( + core_schema.typed_dict_schema( + { + 'a': core_schema.typed_dict_field(core_schema.int_schema(gt=10)), + 'b': core_schema.typed_dict_field( + core_schema.list_schema( + core_schema.dict_schema(core_schema.str_schema(), core_schema.int_schema(ge=10)) + ) + ), + }, + total=False, + ) + ) + assert v.validate_python({'a': 11, 'b': [{'a': 10, 'b': 20}, {'a': 30, 'b': 40}]}) == snapshot( + {'a': 11, 'b': [{'a': 10, 'b': 20}, {'a': 30, 'b': 40}]} + ) + assert v.validate_python({'a': 11, 'b': [{'a': 10, 'b': 20}, {'a': 30, 'b': 4}]}, allow_partial=True) == snapshot( + {'a': 11, 'b': [{'a': 10, 'b': 20}, {'a': 30}]} + ) + assert v.validate_python({'a': 11, 'b': [{'a': 10, 'b': 20}, {'a': 30, 123: 4}]}, allow_partial=True) == snapshot( + {'a': 11, 'b': [{'a': 10, 'b': 20}]} + ) + # the first element of the list is invalid, so the whole list is invalid + assert v.validate_python({'a': 11, 'b': [{'a': 10, 'b': 2}, {'a': 30}]}, allow_partial=True) == snapshot({'a': 11}) + with pytest.raises(ValidationError, match=r'b\.0\.b\s+Input should be greater than or equal to 10'): + v.validate_python({'b': [{'a': 10, 'b': 2}, {'a': 30}], 'a': 11}, allow_partial=True) + + with pytest.raises(ValidationError, match=r'b\.1\.a\s+Input should be greater than or equal to 10'): + v.validate_python({'b': [{'a': 10, 'b': 20}, {'a': 3}], 'a': 11}, allow_partial=True) + + assert v.validate_python({'a': 11, 'b': [{'a': 1, 'b': 20}, {'a': 3, 'b': 40}]}, allow_partial=True) == snapshot( + {'a': 11} + ) + json = b'{"a": 11, "b": [{"a": 10, "b": 20}, {"a": 30, "b": 40}]}' + assert v.validate_json(json, allow_partial=True) == snapshot( + {'a': 11, 'b': [{'a': 10, 'b': 20}, {'a': 30, 'b': 40}]} + ) + for i in range(1, len(json)): + value = v.validate_json(json[:i], allow_partial=True) + assert isinstance(value, dict) + + +def test_tuple_list(): + """Tuples don't support partial, so behaviour should be disabled.""" + v = SchemaValidator( + core_schema.tuple_positional_schema( + [core_schema.list_schema(core_schema.int_schema()), core_schema.int_schema()] + ), + ) + assert v.validate_python([['1', '2'], '3'], allow_partial=True) == snapshot(([1, 2], 3)) + with pytest.raises(ValidationError, match=r'1\s+Input should be a valid integer'): + v.validate_python([['1', '2'], 'x'], allow_partial=True) + with pytest.raises(ValidationError, match=r'0\.1\s+Input should be a valid integer'): + v.validate_python([['1', 'x'], '2'], allow_partial=True) + + +def test_dataclass(): + """Tuples don't support partial, so behaviour should be disabled.""" + + schema = core_schema.dataclass_args_schema( + 'MyDataclass', + [ + core_schema.dataclass_field(name='a', schema=core_schema.str_schema(), kw_only=False), + core_schema.dataclass_field( + name='b', schema=core_schema.list_schema(core_schema.str_schema(min_length=2)), kw_only=False + ), + ], + ) + v = SchemaValidator(schema) + assert v.validate_python({'a': 'x', 'b': ['ab', 'cd']}) == snapshot(({'a': 'x', 'b': ['ab', 'cd']}, None)) + assert v.validate_python({'a': 'x', 'b': ['ab', 'cd']}, allow_partial=True) == snapshot( + ({'a': 'x', 'b': ['ab', 'cd']}, None) + ) + with pytest.raises(ValidationError, match=r'b\.1\s+String should have at least 2 characters'): + v.validate_python({'a': 'x', 'b': ['ab', 'c']}, allow_partial=True) + + +def test_nullable(): + v = SchemaValidator(core_schema.nullable_schema(core_schema.list_schema(core_schema.str_schema(min_length=2)))) + + assert v.validate_python(None, allow_partial=True) is None + assert v.validate_python(['ab', 'cd'], allow_partial=True) == ['ab', 'cd'] + assert v.validate_python(['ab', 'c'], allow_partial=True) == ['ab'] + + +@pytest.mark.parametrize( + 'json_nested_type', [None, core_schema.dict_schema(core_schema.str_schema(), core_schema.int_schema())] +) +def test_json(json_nested_type): + v = SchemaValidator(core_schema.list_schema(core_schema.json_schema(json_nested_type))) + + assert v.validate_python(['{"a": 1}', '{"b": 2}']) == snapshot([{'a': 1}, {'b': 2}]) + assert v.validate_python(['{"a": 1}', '{"b": 2}'], allow_partial=True) == snapshot([{'a': 1}, {'b': 2}]) + assert v.validate_python(['{"a": 1}', 'xxx'], allow_partial=True) == snapshot([{'a': 1}]) + assert v.validate_python(['{"a": 1}', '{"b": 2'], allow_partial=True) == snapshot([{'a': 1}, {'b': 2}])