diff --git a/.mypy-stubtest-allowlist b/.mypy-stubtest-allowlist index 9c6f4bb3d..8c7d6666a 100644 --- a/.mypy-stubtest-allowlist +++ b/.mypy-stubtest-allowlist @@ -1,7 +1,10 @@ # TODO: don't want to expose this staticmethod, requires https://github.com/PyO3/pyo3/issues/2384 pydantic_core._pydantic_core.PydanticUndefinedType.new -# As per #1240, from_json has custom logic to coverage the `cache_strings` kwarg +# See #1540 for discussion pydantic_core._pydantic_core.from_json +pydantic_core._pydantic_core.SchemaValidator.validate_python +pydantic_core._pydantic_core.SchemaValidator.validate_json +pydantic_core._pydantic_core.SchemaValidator.validate_strings # the `warnings` kwarg for SchemaSerializer functions has custom logic pydantic_core._pydantic_core.SchemaSerializer.to_python pydantic_core._pydantic_core.SchemaSerializer.to_json diff --git a/Cargo.lock b/Cargo.lock index 6381d4c5e..b34342509 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -297,9 +297,9 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "jiter" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f69107091dc861900882c599876b095d9db7f71064b6914d6c68b284d19f9ee8" +checksum = "07f69a121b68af57bc10f151f3f67444a64d1d3a0eb48b042801ea917a38dd25" dependencies = [ "ahash", "bitvec", diff --git a/Cargo.toml b/Cargo.toml index 625e474ec..d68bd2701 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,7 +46,7 @@ base64 = "0.22.1" num-bigint = "0.4.6" python3-dll-a = "0.2.10" uuid = "1.11.0" -jiter = { version = "0.7", features = ["python"] } +jiter = { version = "0.7.1", features = ["python"] } hex = "0.4.3" [lib] diff --git a/benches/main.rs b/benches/main.rs index 6d3fd0951..69157e2d3 100644 --- a/benches/main.rs +++ b/benches/main.rs @@ -29,7 +29,7 @@ fn ints_json(bench: &mut Bencher) { let validator = build_schema_validator(py, "{'type': 'int'}"); let result = validator - .validate_json(py, &json(py, "123"), None, None, None, false) + .validate_json(py, &json(py, "123"), None, None, None, false.into()) .unwrap(); let result_int: i64 = result.extract(py).unwrap(); assert_eq!(result_int, 123); @@ -37,7 +37,7 @@ fn ints_json(bench: &mut Bencher) { bench.iter(|| { black_box( validator - .validate_json(py, &json(py, "123"), None, None, None, false) + .validate_json(py, &json(py, "123"), None, None, None, false.into()) .unwrap(), ) }) @@ -51,7 +51,7 @@ fn ints_python(bench: &mut Bencher) { let input = 123_i64.into_py(py).into_bound(py); let result = validator - .validate_python(py, &input, None, None, None, None, false) + .validate_python(py, &input, None, None, None, None, false.into()) .unwrap(); let result_int: i64 = result.extract(py).unwrap(); assert_eq!(result_int, 123); @@ -60,7 +60,7 @@ fn ints_python(bench: &mut Bencher) { bench.iter(|| { black_box( validator - .validate_python(py, &input, None, None, None, None, false) + .validate_python(py, &input, None, None, None, None, false.into()) .unwrap(), ) }) @@ -79,7 +79,7 @@ fn list_int_json(bench: &mut Bencher) { bench.iter(|| { black_box( validator - .validate_json(py, &json(py, &code), None, None, None, false) + .validate_json(py, &json(py, &code), None, None, None, false.into()) .unwrap(), ) }) @@ -104,7 +104,7 @@ fn list_int_python(bench: &mut Bencher) { let input = black_box(input.bind(py)); bench.iter(|| { let v = validator - .validate_python(py, &input, None, None, None, None, false) + .validate_python(py, &input, None, None, None, None, false.into()) .unwrap(); black_box(v) }) @@ -138,7 +138,7 @@ fn list_error_json(bench: &mut Bencher) { .join(", ") ); - match validator.validate_json(py, &json(py, &code), None, None, None, false) { + match validator.validate_json(py, &json(py, &code), None, None, None, false.into()) { Ok(_) => panic!("unexpectedly valid"), Err(e) => { let v = e.value_bound(py); @@ -150,7 +150,7 @@ fn list_error_json(bench: &mut Bencher) { }; bench.iter( - || match validator.validate_json(py, &json(py, &code), None, None, None, false) { + || match validator.validate_json(py, &json(py, &code), None, None, None, false.into()) { Ok(_) => panic!("unexpectedly valid"), Err(e) => black_box(e), }, @@ -170,7 +170,7 @@ fn list_error_python_input(py: Python<'_>) -> (SchemaValidator, PyObject) { let input = py.eval_bound(&code, None, None).unwrap().extract().unwrap(); - match validator.validate_python(py, &input, None, None, None, None, false) { + match validator.validate_python(py, &input, None, None, None, None, false.into()) { Ok(_) => panic!("unexpectedly valid"), Err(e) => { let v = e.value_bound(py); @@ -190,7 +190,7 @@ fn list_error_python(bench: &mut Bencher) { let input = black_box(input.bind(py)); bench.iter(|| { - let result = validator.validate_python(py, &input, None, None, None, None, false); + let result = validator.validate_python(py, &input, None, None, None, None, false.into()); match result { Ok(_) => panic!("unexpectedly valid"), @@ -226,7 +226,7 @@ fn list_any_json(bench: &mut Bencher) { bench.iter(|| { black_box( validator - .validate_json(py, &json(py, &code), None, None, None, false) + .validate_json(py, &json(py, &code), None, None, None, false.into()) .unwrap(), ) }) @@ -245,7 +245,7 @@ fn list_any_python(bench: &mut Bencher) { let input = black_box(input.bind(py)); bench.iter(|| { let v = validator - .validate_python(py, &input, None, None, None, None, false) + .validate_python(py, &input, None, None, None, None, false.into()) .unwrap(); black_box(v) }) @@ -279,7 +279,7 @@ fn dict_json(bench: &mut Bencher) { bench.iter(|| { black_box( validator - .validate_json(py, &json(py, &code), None, None, None, false) + .validate_json(py, &json(py, &code), None, None, None, false.into()) .unwrap(), ) }) @@ -305,7 +305,7 @@ fn dict_python(bench: &mut Bencher) { let input = black_box(input.bind(py)); bench.iter(|| { let v = validator - .validate_python(py, &input, None, None, None, None, false) + .validate_python(py, &input, None, None, None, None, false.into()) .unwrap(); black_box(v) }) @@ -334,7 +334,7 @@ fn dict_value_error(bench: &mut Bencher) { let input = py.eval_bound(&code, None, None).unwrap().to_object(py).into_bound(py); - match validator.validate_python(py, &input, None, None, None, None, false) { + match validator.validate_python(py, &input, None, None, None, None, false.into()) { Ok(_) => panic!("unexpectedly valid"), Err(e) => { let v = e.value_bound(py); @@ -347,7 +347,7 @@ fn dict_value_error(bench: &mut Bencher) { let input = black_box(input); bench.iter(|| { - let result = validator.validate_python(py, &input, None, None, None, None, false); + let result = validator.validate_python(py, &input, None, None, None, None, false.into()); match result { Ok(_) => panic!("unexpectedly valid"), @@ -385,7 +385,7 @@ fn typed_dict_json(bench: &mut Bencher) { bench.iter(|| { black_box( validator - .validate_json(py, &json(py, &code), None, None, None, false) + .validate_json(py, &json(py, &code), None, None, None, false.into()) .unwrap(), ) }) @@ -420,7 +420,7 @@ fn typed_dict_python(bench: &mut Bencher) { let input = black_box(input.bind(py)); bench.iter(|| { let v = validator - .validate_python(py, &input, None, None, None, None, false) + .validate_python(py, &input, None, None, None, None, false.into()) .unwrap(); black_box(v) }) @@ -461,7 +461,7 @@ fn typed_dict_deep_error(bench: &mut Bencher) { let input = py.eval_bound(code, None, None).unwrap().to_object(py); let input = black_box(input.bind(py)); - match validator.validate_python(py, &input, None, None, None, None, false) { + match validator.validate_python(py, &input, None, None, None, None, false.into()) { Ok(_) => panic!("unexpectedly valid"), Err(e) => { let v = e.value_bound(py); @@ -473,7 +473,7 @@ fn typed_dict_deep_error(bench: &mut Bencher) { }; bench.iter(|| { - let result = validator.validate_python(py, &input, None, None, None, None, false); + let result = validator.validate_python(py, &input, None, None, None, None, false.into()); match result { Ok(_) => panic!("unexpectedly valid"), @@ -500,7 +500,7 @@ fn complete_model(bench: &mut Bencher) { bench.iter(|| { black_box( validator - .validate_python(py, &input, None, None, None, None, false) + .validate_python(py, &input, None, None, None, None, false.into()) .unwrap(), ); }) @@ -522,13 +522,13 @@ fn nested_model_using_definitions(bench: &mut Bencher) { let input = black_box(input); validator - .validate_python(py, &input, None, None, None, None, false) + .validate_python(py, &input, None, None, None, None, false.into()) .unwrap(); bench.iter(|| { black_box( validator - .validate_python(py, &input, None, None, None, None, false) + .validate_python(py, &input, None, None, None, None, false.into()) .unwrap(), ); }) @@ -550,13 +550,13 @@ fn nested_model_inlined(bench: &mut Bencher) { let input = black_box(input); validator - .validate_python(py, &input, None, None, None, None, false) + .validate_python(py, &input, None, None, None, None, false.into()) .unwrap(); bench.iter(|| { black_box( validator - .validate_python(py, &input, None, None, None, None, false) + .validate_python(py, &input, None, None, None, None, false.into()) .unwrap(), ); }) @@ -571,7 +571,7 @@ fn literal_ints_few_python(bench: &mut Bencher) { let input = 4_i64.into_py(py); let input = input.bind(py); let result = validator - .validate_python(py, &input, None, None, None, None, false) + .validate_python(py, &input, None, None, None, None, false.into()) .unwrap(); let result_int: i64 = result.extract(py).unwrap(); assert_eq!(result_int, 4); @@ -580,7 +580,7 @@ fn literal_ints_few_python(bench: &mut Bencher) { bench.iter(|| { black_box( validator - .validate_python(py, &input, None, None, None, None, false) + .validate_python(py, &input, None, None, None, None, false.into()) .unwrap(), ) }) @@ -596,7 +596,7 @@ fn literal_strings_few_small_python(bench: &mut Bencher) { let input = input.to_object(py).into_bound(py); let input_str: String = input.extract().unwrap(); let result = validator - .validate_python(py, &input, None, None, None, None, false) + .validate_python(py, &input, None, None, None, None, false.into()) .unwrap(); let result_str: String = result.extract(py).unwrap(); assert_eq!(result_str, input_str); @@ -605,7 +605,7 @@ fn literal_strings_few_small_python(bench: &mut Bencher) { bench.iter(|| { black_box( validator - .validate_python(py, &input, None, None, None, None, false) + .validate_python(py, &input, None, None, None, None, false.into()) .unwrap(), ) }) @@ -624,7 +624,7 @@ fn literal_strings_few_large_python(bench: &mut Bencher) { let input = input.to_object(py).into_bound(py); let input_str: String = input.extract().unwrap(); let result = validator - .validate_python(py, &input, None, None, None, None, false) + .validate_python(py, &input, None, None, None, None, false.into()) .unwrap(); let result_str: String = result.extract(py).unwrap(); assert_eq!(result_str, input_str); @@ -633,7 +633,7 @@ fn literal_strings_few_large_python(bench: &mut Bencher) { bench.iter(|| { black_box( validator - .validate_python(py, &input, None, None, None, None, false) + .validate_python(py, &input, None, None, None, None, false.into()) .unwrap(), ) }) @@ -668,7 +668,7 @@ class Foo(Enum): let input = py.eval_bound("Foo.v4", Some(&globals), None).unwrap(); let input = input.to_object(py).into_bound(py); let result = validator - .validate_python(py, &input, None, None, None, None, false) + .validate_python(py, &input, None, None, None, None, false.into()) .unwrap(); assert!(input.eq(result).unwrap()); @@ -676,7 +676,7 @@ class Foo(Enum): bench.iter(|| { black_box( validator - .validate_python(py, &input, None, None, None, None, false) + .validate_python(py, &input, None, None, None, None, false.into()) .unwrap(), ) }) @@ -690,7 +690,7 @@ fn literal_ints_many_python(bench: &mut Bencher) { let input = 99_i64.into_py(py).into_bound(py); let result = validator - .validate_python(py, &input, None, None, None, None, false) + .validate_python(py, &input, None, None, None, None, false.into()) .unwrap(); let result_int: i64 = result.extract(py).unwrap(); assert_eq!(result_int, 99); @@ -699,7 +699,7 @@ fn literal_ints_many_python(bench: &mut Bencher) { bench.iter(|| { black_box( validator - .validate_python(py, &input, None, None, None, None, false) + .validate_python(py, &input, None, None, None, None, false.into()) .unwrap(), ) }) @@ -715,7 +715,7 @@ fn literal_strings_many_small_python(bench: &mut Bencher) { let input = input.to_object(py).into_bound(py); let input_str: String = input.extract().unwrap(); let result = validator - .validate_python(py, &input, None, None, None, None, false) + .validate_python(py, &input, None, None, None, None, false.into()) .unwrap(); let result_str: String = result.extract(py).unwrap(); assert_eq!(result_str, input_str); @@ -724,7 +724,7 @@ fn literal_strings_many_small_python(bench: &mut Bencher) { bench.iter(|| { black_box( validator - .validate_python(py, &input, None, None, None, None, false) + .validate_python(py, &input, None, None, None, None, false.into()) .unwrap(), ) }) @@ -743,7 +743,7 @@ fn literal_strings_many_large_python(bench: &mut Bencher) { let input = input.to_object(py).into_bound(py); let input_str: String = input.extract().unwrap(); let result = validator - .validate_python(py, &input, None, None, None, None, false) + .validate_python(py, &input, None, None, None, None, false.into()) .unwrap(); let result_str: String = result.extract(py).unwrap(); assert_eq!(result_str, input_str); @@ -752,7 +752,7 @@ fn literal_strings_many_large_python(bench: &mut Bencher) { bench.iter(|| { black_box( validator - .validate_python(py, &input, None, None, None, None, false) + .validate_python(py, &input, None, None, None, None, false.into()) .unwrap(), ) }) @@ -767,7 +767,7 @@ fn literal_ints_many_json(bench: &mut Bencher) { let input_json = py.eval_bound("'99'", None, None).unwrap(); let input_json = input_json.to_object(py).into_bound(py); let result = validator - .validate_json(py, &input_json, None, None, None, false) + .validate_json(py, &input_json, None, None, None, false.into()) .unwrap(); let result_int: i64 = result.extract(py).unwrap(); assert_eq!(result_int, 99); @@ -776,7 +776,7 @@ fn literal_ints_many_json(bench: &mut Bencher) { bench.iter(|| { black_box( validator - .validate_json(py, &input_json, None, None, None, false) + .validate_json(py, &input_json, None, None, None, false.into()) .unwrap(), ) }) @@ -797,7 +797,7 @@ fn literal_strings_many_large_json(bench: &mut Bencher) { let input_json = input_json.to_object(py).into_bound(py); let input_str: String = input.extract().unwrap(); let result = validator - .validate_json(py, &input_json, None, None, None, false) + .validate_json(py, &input_json, None, None, None, false.into()) .unwrap(); let result_str: String = result.extract(py).unwrap(); assert_eq!(result_str, input_str); @@ -806,7 +806,7 @@ fn literal_strings_many_large_json(bench: &mut Bencher) { bench.iter(|| { black_box( validator - .validate_json(py, &input_json, None, None, None, false) + .validate_json(py, &input_json, None, None, None, false.into()) .unwrap(), ) }) @@ -843,7 +843,7 @@ class Foo(Enum): let input = input.to_object(py).into_bound(py); let input_str: String = input.extract().unwrap(); let result = validator - .validate_python(py, &input, None, None, None, None, false) + .validate_python(py, &input, None, None, None, None, false.into()) .unwrap(); let result_str: String = result.extract(py).unwrap(); assert_eq!(result_str, input_str); @@ -852,7 +852,7 @@ class Foo(Enum): bench.iter(|| { black_box( validator - .validate_python(py, &input, None, None, None, None, false) + .validate_python(py, &input, None, None, None, None, false.into()) .unwrap(), ) }) @@ -864,7 +864,7 @@ class Foo(Enum): let input = input.to_object(py).into_bound(py); let input_int: i64 = input.extract().unwrap(); let result = validator - .validate_python(py, &input, None, None, None, None, false) + .validate_python(py, &input, None, None, None, None, false.into()) .unwrap(); let result_int: i64 = result.extract(py).unwrap(); assert_eq!(result_int, input_int); @@ -873,7 +873,7 @@ class Foo(Enum): bench.iter(|| { black_box( validator - .validate_python(py, &input, None, None, None, None, false) + .validate_python(py, &input, None, None, None, None, false.into()) .unwrap(), ) }) @@ -884,7 +884,7 @@ class Foo(Enum): let input = py.eval_bound("None", None, None).unwrap(); let input = input.to_object(py).into_bound(py); let result = validator - .validate_python(py, &input, None, None, None, None, false) + .validate_python(py, &input, None, None, None, None, false.into()) .unwrap(); assert!(input.eq(result).unwrap()); @@ -892,7 +892,7 @@ class Foo(Enum): bench.iter(|| { black_box( validator - .validate_python(py, &input, None, None, None, None, false) + .validate_python(py, &input, None, None, None, None, false.into()) .unwrap(), ) }) @@ -903,7 +903,7 @@ class Foo(Enum): let input = py.eval_bound("Foo.v4", Some(&globals), None).unwrap(); let input = input.to_object(py).into_bound(py); let result = validator - .validate_python(py, &input, None, None, None, None, false) + .validate_python(py, &input, None, None, None, None, false.into()) .unwrap(); assert!(input.eq(result).unwrap()); @@ -911,7 +911,7 @@ class Foo(Enum): bench.iter(|| { black_box( validator - .validate_python(py, &input, None, None, None, None, false) + .validate_python(py, &input, None, None, None, None, false.into()) .unwrap(), ) }) diff --git a/pyproject.toml b/pyproject.toml index 9d8e5f046..99f0fce73 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -109,3 +109,6 @@ require_change_file = false [tool.pyright] include = ['pydantic_core', 'tests/test_typing.py'] reportUnnecessaryTypeIgnoreComment = true + +[tool.inline-snapshot.shortcuts] +fix = ["create", "fix"] diff --git a/python/pydantic_core/_pydantic_core.pyi b/python/pydantic_core/_pydantic_core.pyi index 718db625d..f3103f28f 100644 --- a/python/pydantic_core/_pydantic_core.pyi +++ b/python/pydantic_core/_pydantic_core.pyi @@ -96,7 +96,7 @@ class SchemaValidator: from_attributes: bool | None = None, context: Any | None = None, self_instance: Any | None = None, - allow_partial: bool = False, + allow_partial: bool | Literal['off', 'on', 'trailing-strings'] = False, ) -> Any: """ Validate a Python object against the schema and return the validated object. @@ -113,6 +113,7 @@ class SchemaValidator: validation from the `__init__` method of a model. allow_partial: Whether to allow partial validation; if `True` errors in the last element of sequences and mappings are ignored. + `'trailing-strings'` means any final unfinished JSON string is included in the result. Raises: ValidationError: If validation fails. @@ -146,7 +147,7 @@ class SchemaValidator: strict: bool | None = None, context: Any | None = None, self_instance: Any | None = None, - allow_partial: bool = False, + allow_partial: bool | Literal['off', 'on', 'trailing-strings'] = False, ) -> Any: """ Validate JSON data directly against the schema and return the validated Python object. @@ -166,6 +167,7 @@ class SchemaValidator: self_instance: An instance of a model set attributes on from validation. allow_partial: Whether to allow partial validation; if `True` incomplete JSON will be parsed successfully and errors in the last element of sequences and mappings are ignored. + `'trailing-strings'` means any final unfinished JSON string is included in the result. Raises: ValidationError: If validation fails or if the JSON data is invalid. @@ -180,7 +182,7 @@ class SchemaValidator: *, strict: bool | None = None, context: Any | None = None, - allow_partial: bool = False, + allow_partial: bool | Literal['off', 'on', 'trailing-strings'] = False, ) -> Any: """ Validate a string against the schema and return the validated Python object. @@ -196,6 +198,7 @@ class SchemaValidator: [`info.context`][pydantic_core.core_schema.ValidationInfo.context]. allow_partial: Whether to allow partial validation; if `True` errors in the last element of sequences and mappings are ignored. + `'trailing-strings'` means any final unfinished JSON string is included in the result. Raises: ValidationError: If validation fails or if the JSON data is invalid. @@ -433,6 +436,7 @@ def from_json( `all/True` means cache all strings, `keys` means cache only dict keys, `none/False` means no caching. allow_partial: Whether to allow partial deserialization, if `True` JSON data is returned if the end of the input is reached before the full object is deserialized, e.g. `["aa", "bb", "c` would return `['aa', 'bb']`. + `'trailing-strings'` means any final unfinished JSON string is included in the result. Raises: ValueError: If deserialization fails. diff --git a/src/input/return_enums.rs b/src/input/return_enums.rs index d5e31b97b..c91a6e2fd 100644 --- a/src/input/return_enums.rs +++ b/src/input/return_enums.rs @@ -3,7 +3,7 @@ use std::cmp::Ordering; use std::ops::Rem; use std::str::FromStr; -use jiter::{JsonArray, JsonValue, StringCacheMode}; +use jiter::{JsonArray, JsonValue, PartialMode, StringCacheMode}; use num_bigint::BigInt; use pyo3::exceptions::PyTypeError; @@ -128,9 +128,13 @@ pub(crate) fn validate_iter_to_vec<'py>( ) -> ValResult> { let mut output: Vec = Vec::with_capacity(capacity); let mut errors: Vec = Vec::new(); + let allow_partial = state.allow_partial; for (index, is_last_partial, item_result) in state.enumerate_last_partial(iter) { - state.allow_partial = is_last_partial; + state.allow_partial = match is_last_partial { + true => allow_partial, + false => PartialMode::Off, + }; let item = item_result.map_err(|e| any_next_error!(py, e, max_length_check.input, index))?; match validator.validate(py, item.borrow_input(), state) { Ok(item) => { @@ -202,8 +206,13 @@ pub(crate) fn validate_iter_to_set<'py>( ) -> ValResult<()> { let mut errors: Vec = Vec::new(); + let allow_partial = state.allow_partial; + for (index, is_last_partial, item_result) in state.enumerate_last_partial(iter) { - state.allow_partial = is_last_partial; + state.allow_partial = match is_last_partial { + true => allow_partial, + false => PartialMode::Off, + }; let item = item_result.map_err(|e| any_next_error!(py, e, input, index))?; match validator.validate(py, item.borrow_input(), state) { Ok(item) => { diff --git a/src/url.rs b/src/url.rs index 3ba2c8451..dda583af5 100644 --- a/src/url.rs +++ b/src/url.rs @@ -45,7 +45,7 @@ impl PyUrl { pub fn py_new(py: Python, url: &Bound<'_, PyAny>) -> PyResult { let schema_obj = SCHEMA_DEFINITION_URL .get_or_init(py, || build_schema_validator(py, "url")) - .validate_python(py, url, None, None, None, None, false)?; + .validate_python(py, url, None, None, None, None, false.into())?; schema_obj.extract(py) } @@ -225,7 +225,7 @@ impl PyMultiHostUrl { pub fn py_new(py: Python, url: &Bound<'_, PyAny>) -> PyResult { let schema_obj = SCHEMA_DEFINITION_MULTI_HOST_URL .get_or_init(py, || build_schema_validator(py, "multi-host-url")) - .validate_python(py, url, None, None, None, None, false)?; + .validate_python(py, url, None, None, None, None, false.into())?; schema_obj.extract(py) } diff --git a/src/validators/arguments.rs b/src/validators/arguments.rs index 0d90af5d3..2ffd49102 100644 --- a/src/validators/arguments.rs +++ b/src/validators/arguments.rs @@ -11,7 +11,6 @@ use crate::build_tools::{schema_or_config_same, ExtraBehavior}; use crate::errors::{ErrorTypeDefaults, ValError, ValLineError, ValResult}; use crate::input::{Arguments, BorrowInput, Input, KeywordArgs, PositionalArgs, ValidationMatch}; use crate::lookup_key::LookupKey; - use crate::tools::SchemaDict; use super::validation_state::ValidationState; @@ -189,7 +188,7 @@ impl Validator for ArgumentsValidator { state: &mut ValidationState<'_, 'py>, ) -> ValResult { // this validator does not yet support partial validation, disable it to avoid incorrect results - state.allow_partial = false; + state.allow_partial = false.into(); let args = input.validate_args()?; diff --git a/src/validators/dataclass.rs b/src/validators/dataclass.rs index 86211b7b2..b41de429f 100644 --- a/src/validators/dataclass.rs +++ b/src/validators/dataclass.rs @@ -146,7 +146,7 @@ impl Validator for DataclassArgsValidator { state: &mut ValidationState<'_, 'py>, ) -> ValResult { // this validator does not yet support partial validation, disable it to avoid incorrect results - state.allow_partial = false; + state.allow_partial = false.into(); let args = input.validate_dataclass_args(&self.dataclass_name)?; diff --git a/src/validators/definitions.rs b/src/validators/definitions.rs index 34d0edef4..e0511f7fe 100644 --- a/src/validators/definitions.rs +++ b/src/validators/definitions.rs @@ -76,7 +76,7 @@ impl Validator for DefinitionRefValidator { state: &mut ValidationState<'_, 'py>, ) -> ValResult { // this validator does not yet support partial validation, disable it to avoid incorrect results - state.allow_partial = false; + state.allow_partial = false.into(); self.definition.read(|validator| { let validator = validator.unwrap(); diff --git a/src/validators/dict.rs b/src/validators/dict.rs index adfadc6fa..985fe95b6 100644 --- a/src/validators/dict.rs +++ b/src/validators/dict.rs @@ -109,9 +109,10 @@ where fn consume_iterator(self, iterator: impl Iterator>) -> ValResult { let output = PyDict::new_bound(self.py); let mut errors: Vec = Vec::new(); + let allow_partial = self.state.allow_partial; for (_, is_last_partial, item_result) in self.state.enumerate_last_partial(iterator) { - self.state.allow_partial = false; + self.state.allow_partial = false.into(); let (key, value) = item_result?; let output_key = match self.key_validator.validate(self.py, key.borrow_input(), self.state) { Ok(value) => Some(value), @@ -125,7 +126,10 @@ where Err(ValError::Omit) => continue, Err(err) => return Err(err), }; - self.state.allow_partial = is_last_partial; + self.state.allow_partial = match is_last_partial { + true => allow_partial, + false => false.into(), + }; let output_value = match self.value_validator.validate(self.py, value.borrow_input(), self.state) { Ok(value) => value, Err(ValError::LineErrors(line_errors)) => { diff --git a/src/validators/generator.rs b/src/validators/generator.rs index 12241f205..1e8d41c85 100644 --- a/src/validators/generator.rs +++ b/src/validators/generator.rs @@ -67,7 +67,7 @@ impl Validator for GeneratorValidator { state: &mut ValidationState<'_, 'py>, ) -> ValResult { // this validator does not yet support partial validation, disable it to avoid incorrect results - state.allow_partial = false; + state.allow_partial = false.into(); let iterator = input.validate_iter()?.into_static(); let validator = self.item_validator.as_ref().map(|v| { @@ -282,7 +282,7 @@ impl InternalValidator { self_instance: self.self_instance.as_ref().map(|data| data.bind(py)), cache_str: self.cache_str, }; - let mut state = ValidationState::new(extra, &mut self.recursion_guard, false); + let mut state = ValidationState::new(extra, &mut self.recursion_guard, false.into()); state.exactness = self.exactness; let result = self .validator @@ -317,7 +317,7 @@ impl InternalValidator { self_instance: self.self_instance.as_ref().map(|data| data.bind(py)), cache_str: self.cache_str, }; - let mut state = ValidationState::new(extra, &mut self.recursion_guard, false); + let mut state = ValidationState::new(extra, &mut self.recursion_guard, false.into()); state.exactness = self.exactness; let result = self.validator.validate(py, input, &mut state).map_err(|e| { ValidationError::from_val_error( diff --git a/src/validators/json.rs b/src/validators/json.rs index 9ba13dca1..daad6fbff 100644 --- a/src/validators/json.rs +++ b/src/validators/json.rs @@ -2,7 +2,7 @@ use pyo3::intern; use pyo3::prelude::*; use pyo3::types::PyDict; -use jiter::{FloatMode, JsonValue, PartialMode, PythonParse}; +use jiter::{FloatMode, JsonValue, PythonParse}; use crate::errors::{ErrorType, ErrorTypeDefaults, ValError, ValLineError, ValResult}; use crate::input::{EitherBytes, Input, InputType, ValidationMatch}; @@ -70,11 +70,7 @@ impl Validator for JsonValidator { let parse_builder = PythonParse { allow_inf_nan: true, cache_mode: state.cache_str(), - partial_mode: if state.allow_partial { - PartialMode::TrailingStrings - } else { - PartialMode::Off - }, + partial_mode: state.allow_partial, catch_duplicate_keys: false, float_mode: FloatMode::Float, }; diff --git a/src/validators/mod.rs b/src/validators/mod.rs index c0c58adcf..10a4d6e34 100644 --- a/src/validators/mod.rs +++ b/src/validators/mod.rs @@ -1,7 +1,7 @@ use std::fmt::Debug; use enum_dispatch::enum_dispatch; -use jiter::StringCacheMode; +use jiter::{PartialMode, StringCacheMode}; use pyo3::exceptions::PyTypeError; use pyo3::prelude::*; @@ -165,7 +165,7 @@ impl SchemaValidator { } #[allow(clippy::too_many_arguments)] - #[pyo3(signature = (input, *, strict=None, from_attributes=None, context=None, self_instance=None, allow_partial=false))] + #[pyo3(signature = (input, *, strict=None, from_attributes=None, context=None, self_instance=None, allow_partial=PartialMode::Off))] pub fn validate_python( &self, py: Python, @@ -174,7 +174,7 @@ impl SchemaValidator { from_attributes: Option, context: Option<&Bound<'_, PyAny>>, self_instance: Option<&Bound<'_, PyAny>>, - allow_partial: bool, + allow_partial: PartialMode, ) -> PyResult { self._validate( py, @@ -207,7 +207,7 @@ impl SchemaValidator { from_attributes, context, self_instance, - false, + false.into(), ) { Ok(_) => Ok(true), Err(ValError::InternalErr(err)) => Err(err), @@ -217,7 +217,7 @@ impl SchemaValidator { } } - #[pyo3(signature = (input, *, strict=None, context=None, self_instance=None, allow_partial=false))] + #[pyo3(signature = (input, *, strict=None, context=None, self_instance=None, allow_partial=PartialMode::Off))] pub fn validate_json( &self, py: Python, @@ -225,7 +225,7 @@ impl SchemaValidator { strict: Option, context: Option<&Bound<'_, PyAny>>, self_instance: Option<&Bound<'_, PyAny>>, - allow_partial: bool, + allow_partial: PartialMode, ) -> PyResult { let r = match json::validate_json_bytes(input) { Ok(v_match) => self._validate_json( @@ -242,14 +242,14 @@ impl SchemaValidator { r.map_err(|e| self.prepare_validation_err(py, e, InputType::Json)) } - #[pyo3(signature = (input, *, strict=None, context=None, allow_partial=false))] + #[pyo3(signature = (input, *, strict=None, context=None, allow_partial=PartialMode::Off))] pub fn validate_strings( &self, py: Python, input: Bound<'_, PyAny>, strict: Option, context: Option<&Bound<'_, PyAny>>, - allow_partial: bool, + allow_partial: PartialMode, ) -> PyResult { let t = InputType::String; let string_mapping = StringMapping::new_value(input).map_err(|e| self.prepare_validation_err(py, e, t))?; @@ -283,7 +283,7 @@ impl SchemaValidator { }; let guard = &mut RecursionState::default(); - let mut state = ValidationState::new(extra, guard, false); + let mut state = ValidationState::new(extra, guard, false.into()); self.validator .validate_assignment(py, &obj, field_name, &field_value, &mut state) .map_err(|e| self.prepare_validation_err(py, e, InputType::Python)) @@ -306,7 +306,7 @@ impl SchemaValidator { cache_str: self.cache_str, }; let recursion_guard = &mut RecursionState::default(); - let mut state = ValidationState::new(extra, recursion_guard, false); + let mut state = ValidationState::new(extra, recursion_guard, false.into()); let r = self.validator.default_value(py, None::, &mut state); match r { Ok(maybe_default) => match maybe_default { @@ -352,7 +352,7 @@ impl SchemaValidator { from_attributes: Option, context: Option<&Bound<'py, PyAny>>, self_instance: Option<&Bound<'py, PyAny>>, - allow_partial: bool, + allow_partial: PartialMode, ) -> ValResult { let mut recursion_guard = RecursionState::default(); let mut state = ValidationState::new( @@ -379,7 +379,7 @@ impl SchemaValidator { strict: Option, context: Option<&Bound<'_, PyAny>>, self_instance: Option<&Bound<'_, PyAny>>, - allow_partial: bool, + allow_partial: PartialMode, ) -> ValResult { let json_value = jiter::JsonValue::parse_with_config(json_data, true, allow_partial) .map_err(|e| json::map_json_err(input, e, json_data))?; @@ -430,7 +430,7 @@ impl<'py> SelfValidator<'py> { let mut state = ValidationState::new( Extra::new(strict, None, None, None, InputType::Python, true.into()), &mut recursion_guard, - false, + false.into(), ); match self.validator.validator.validate(py, schema, &mut state) { Ok(schema_obj) => Ok(schema_obj.into_bound(py)), diff --git a/src/validators/model_fields.rs b/src/validators/model_fields.rs index 39f905d84..7aba7c8e3 100644 --- a/src/validators/model_fields.rs +++ b/src/validators/model_fields.rs @@ -121,7 +121,7 @@ impl Validator for ModelFieldsValidator { state: &mut ValidationState<'_, 'py>, ) -> ValResult { // this validator does not yet support partial validation, disable it to avoid incorrect results - state.allow_partial = false; + state.allow_partial = false.into(); let strict = state.strict_or(self.strict); let from_attributes = state.extra().from_attributes.unwrap_or(self.from_attributes); diff --git a/src/validators/tuple.rs b/src/validators/tuple.rs index 2b544d417..60e777e01 100644 --- a/src/validators/tuple.rs +++ b/src/validators/tuple.rs @@ -1,7 +1,8 @@ +use std::collections::VecDeque; + use pyo3::intern; use pyo3::prelude::*; use pyo3::types::{PyDict, PyList, PyTuple}; -use std::collections::VecDeque; use crate::build_tools::is_strict; use crate::errors::{py_err_string, ErrorType, ErrorTypeDefaults, ValError, ValLineError, ValResult}; @@ -273,7 +274,7 @@ impl Validator for TupleValidator { state: &mut ValidationState<'_, 'py>, ) -> ValResult { // this validator does not yet support partial validation, disable it to avoid incorrect results - state.allow_partial = false; + state.allow_partial = false.into(); let collection = input.validate_tuple(state.strict_or(self.strict))?.unpack(state); let actual_length = collection.len(); diff --git a/src/validators/typed_dict.rs b/src/validators/typed_dict.rs index 949cff9a5..0c127b93b 100644 --- a/src/validators/typed_dict.rs +++ b/src/validators/typed_dict.rs @@ -2,8 +2,6 @@ use pyo3::intern; use pyo3::prelude::*; use pyo3::types::{PyDict, PyString}; -use ahash::AHashSet; - use crate::build_tools::py_schema_err; use crate::build_tools::{is_strict, schema_or_config, schema_or_config_same, ExtraBehavior}; use crate::errors::LocItem; @@ -14,6 +12,8 @@ use crate::input::ValidationMatch; use crate::input::{Input, ValidatedDict}; use crate::lookup_key::LookupKey; use crate::tools::SchemaDict; +use ahash::AHashSet; +use jiter::PartialMode; use super::{build_validator, BuildValidator, CombinedValidator, DefinitionsBuilder, ValidationState, Validator}; @@ -153,11 +153,12 @@ impl Validator for TypedDictValidator { let output_dict = PyDict::new_bound(py); let mut errors: Vec = Vec::with_capacity(self.fields.len()); - let partial_last_key = if state.allow_partial { + let partial_last_key = if state.allow_partial.is_active() { dict.last_key().map(Into::into) } else { None }; + let allow_partial = state.allow_partial; // we only care about which keys have been used if we're iterating over the object for extra after // the first pass @@ -198,7 +199,10 @@ impl Validator for TypedDictValidator { } else { false }; - state.allow_partial = is_last_partial; + state.allow_partial = match is_last_partial { + true => allow_partial, + false => false.into(), + }; match field.validator.validate(py, value.borrow_input(), state) { Ok(value) => { output_dict.set_item(&field.name_py, value)?; @@ -260,6 +264,7 @@ impl Validator for TypedDictValidator { state: &'a mut ValidationState<'s, 'py>, extra_behavior: ExtraBehavior, partial_last_key: Option, + allow_partial: PartialMode, } impl<'py, Key, Value> ConsumeIterator> for ValidateExtras<'_, '_, 'py> @@ -311,7 +316,10 @@ impl Validator for TypedDictValidator { let key_loc: LocItem = raw_key.clone().into(); &key_loc == last_key }); - self.state.allow_partial = last_partial; + self.state.allow_partial = match last_partial { + true => self.allow_partial, + false => false.into(), + }; match validator.validate(self.py, value, self.state) { Ok(value) => { self.output_dict.set_item(py_key, value)?; @@ -345,6 +353,7 @@ impl Validator for TypedDictValidator { state, extra_behavior: self.extra_behavior, partial_last_key, + allow_partial, })??; } diff --git a/src/validators/validation_state.rs b/src/validators/validation_state.rs index b21284a99..8ee41f5de 100644 --- a/src/validators/validation_state.rs +++ b/src/validators/validation_state.rs @@ -1,7 +1,7 @@ use pyo3::prelude::*; use pyo3::types::PyString; -use jiter::StringCacheMode; +use jiter::{PartialMode, StringCacheMode}; use crate::recursion_guard::{ContainsRecursionState, RecursionState}; use crate::tools::new_py_string; @@ -24,13 +24,13 @@ pub struct ValidationState<'a, 'py> { // when extra='allow', whereas this tally does not. pub fields_set_count: Option, // True if `allow_partial=true` and we're validating the last element of a sequence or mapping. - pub allow_partial: bool, + pub allow_partial: PartialMode, // deliberately make Extra readonly extra: Extra<'a, 'py>, } impl<'a, 'py> ValidationState<'a, 'py> { - pub fn new(extra: Extra<'a, 'py>, recursion_guard: &'a mut RecursionState, allow_partial: bool) -> Self { + pub fn new(extra: Extra<'a, 'py>, recursion_guard: &'a mut RecursionState, allow_partial: PartialMode) -> Self { Self { recursion_guard, // Don't care about exactness unless doing union validation exactness: None, @@ -130,10 +130,10 @@ pub struct EnumerateLastPartial { iter: I, index: usize, next_item: Option, - allow_partial: bool, + allow_partial: PartialMode, } impl EnumerateLastPartial { - pub fn new(mut iter: I, allow_partial: bool) -> Self { + pub fn new(mut iter: I, allow_partial: PartialMode) -> Self { let next_item = iter.next(); Self { iter, @@ -151,7 +151,7 @@ impl Iterator for EnumerateLastPartial { let a = std::mem::replace(&mut self.next_item, self.iter.next())?; let i = self.index; self.index += 1; - Some((i, self.allow_partial && self.next_item.is_none(), a)) + Some((i, self.allow_partial.is_active() && self.next_item.is_none(), a)) } fn size_hint(&self) -> (usize, Option) { diff --git a/tests/test.rs b/tests/test.rs index fb141d377..dbc7a21e7 100644 --- a/tests/test.rs +++ b/tests/test.rs @@ -129,7 +129,7 @@ json_input = '{"a": "something"}' let json_input = locals.get_item("json_input").unwrap().unwrap(); let binding = SchemaValidator::py_new(py, &schema, None) .unwrap() - .validate_json(py, &json_input, None, None, None, false) + .validate_json(py, &json_input, None, None, None, false.into()) .unwrap(); let validation_result: Bound<'_, PyAny> = binding.extract(py).unwrap(); let repr = format!("{}", validation_result.repr().unwrap()); diff --git a/tests/validators/test_allow_partial.py b/tests/validators/test_allow_partial.py index e147ff62d..2cd5a2735 100644 --- a/tests/validators/test_allow_partial.py +++ b/tests/validators/test_allow_partial.py @@ -299,6 +299,11 @@ def test_nullable(): assert v.validate_python(None, allow_partial=True) is None assert v.validate_python(['ab', 'cd'], allow_partial=True) == ['ab', 'cd'] assert v.validate_python(['ab', 'c'], allow_partial=True) == ['ab'] + assert v.validate_json('["ab", "cd"]', allow_partial=True) == ['ab', 'cd'] + assert v.validate_json('["ab", "cd', allow_partial=True) == ['ab'] + assert v.validate_json('["ab", "cd', allow_partial='trailing-strings') == ['ab', 'cd'] + assert v.validate_json('["ab", "c', allow_partial=True) == ['ab'] + assert v.validate_json('["ab", "c', allow_partial='trailing-strings') == ['ab'] @pytest.mark.parametrize( @@ -311,3 +316,23 @@ def test_json(json_nested_type): assert v.validate_python(['{"a": 1}', '{"b": 2}'], allow_partial=True) == snapshot([{'a': 1}, {'b': 2}]) assert v.validate_python(['{"a": 1}', 'xxx'], allow_partial=True) == snapshot([{'a': 1}]) assert v.validate_python(['{"a": 1}', '{"b": 2'], allow_partial=True) == snapshot([{'a': 1}, {'b': 2}]) + assert v.validate_json('["{\\"a\\": 1}", "{\\"b\\": 2}', allow_partial='trailing-strings') == snapshot( + [{'a': 1}, {'b': 2}] + ) + assert v.validate_json('["{\\"a\\": 1}", "{\\"b\\": 2', allow_partial='trailing-strings') == snapshot( + [{'a': 1}, {'b': 2}] + ) + + +def test_json_trailing_strings(): + v = SchemaValidator(core_schema.list_schema(core_schema.json_schema())) + assert v.validate_python(['{"a": 1}', '{"b": "x'], allow_partial=True) == snapshot([{'a': 1}, {}]) + assert v.validate_python(['{"a": 1}', '{"b": "x'], allow_partial='trailing-strings') == snapshot( + [{'a': 1}, {'b': 'x'}] + ) + + assert v.validate_json('["{\\"a\\": 1}", "{\\"b\\": 2}"]') == snapshot([{'a': 1}, {'b': 2}]) + assert v.validate_json('["{\\"a\\": 1}", "{\\"b\\": 2, \\"c\\": \\"x', allow_partial=True) == snapshot([{'a': 1}]) + assert v.validate_json( + '["{\\"a\\": 1}", "{\\"b\\": 2, \\"c\\": \\"x', allow_partial='trailing-strings' + ) == snapshot([{'a': 1}, {'b': 2, 'c': 'x'}])