From 97543d0c2d7588d9c2855c329bcc8c2b1107a404 Mon Sep 17 00:00:00 2001 From: Youmu Date: Wed, 3 Jul 2024 16:41:13 -0400 Subject: [PATCH] Implement MapSkipError, skipping un-deserializable entries. --- serde_with/src/de/skip_error.rs | 69 +++++++ .../src/guide/serde_as_transformations.md | 61 ++++-- serde_with/src/lib.rs | 58 ++++++ serde_with/src/ser/skip_error.rs | 27 +++ serde_with/tests/hashbrown_0_14.rs | 89 +++++++++ serde_with/tests/indexmap_1.rs | 89 +++++++++ serde_with/tests/indexmap_2.rs | 89 +++++++++ serde_with/tests/serde_as/lib.rs | 174 ++++++++++++++++++ 8 files changed, 636 insertions(+), 20 deletions(-) diff --git a/serde_with/src/de/skip_error.rs b/serde_with/src/de/skip_error.rs index a50d4bd4..e6b915d7 100644 --- a/serde_with/src/de/skip_error.rs +++ b/serde_with/src/de/skip_error.rs @@ -1,5 +1,13 @@ +use super::impls::macros::foreach_map; use crate::prelude::*; +#[cfg(feature = "hashbrown_0_14")] +use hashbrown_0_14::HashMap as HashbrownMap014; +#[cfg(feature = "indexmap_1")] +use indexmap_1::IndexMap; +#[cfg(feature = "indexmap_2")] +use indexmap_2::IndexMap as IndexMap2; + enum GoodOrError { Good(T), // Only here to consume the TAs generic @@ -73,3 +81,64 @@ where deserializer.deserialize_seq(visitor) } } + +struct MapSkipErrorVisitor(PhantomData<(MAP, K, KAs, V, VAs)>); + +impl<'de, MAP, K, KAs, V, VAs> Visitor<'de> for MapSkipErrorVisitor +where + MAP: FromIterator<(K, V)>, + KAs: DeserializeAs<'de, K>, + VAs: DeserializeAs<'de, V>, +{ + type Value = MAP; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a map") + } + + #[inline] + fn visit_map(self, access: A) -> Result + where + A: MapAccess<'de>, + { + type KVPair = (GoodOrError, GoodOrError); + utils::MapIter::new(access) + .filter_map(|res: Result, A::Error>| match res { + Ok((GoodOrError::Good(key), GoodOrError::Good(value))) => Some(Ok((key, value))), + Ok(_) => None, + Err(err) => Some(Err(err)), + }) + .collect() + } +} + +#[cfg(feature = "alloc")] +macro_rules! map_impl { + ( + $ty:ident < K $(: $kbound1:ident $(+ $kbound2:ident)*)?, V $(, $typaram:ident : $bound1:ident $(+ $bound2:ident)*)* >, + $with_capacity:expr + ) => { + impl<'de, K, V, KAs, VAs $(, $typaram)*> DeserializeAs<'de, $ty> + for MapSkipError + where + KAs: DeserializeAs<'de, K>, + VAs: DeserializeAs<'de, V>, + $(K: $kbound1 $(+ $kbound2)*,)? + $($typaram: $bound1 $(+ $bound2)*),* + { + fn deserialize_as(deserializer: D) -> Result<$ty, D::Error> + where + D: Deserializer<'de>, + { + deserializer.deserialize_map(MapSkipErrorVisitor::< + $ty, + K, + KAs, + V, + VAs, + >(PhantomData)) + } + } + }; +} +foreach_map!(map_impl); diff --git a/serde_with/src/guide/serde_as_transformations.md b/serde_with/src/guide/serde_as_transformations.md index 6768fa3c..20ed710d 100644 --- a/serde_with/src/guide/serde_as_transformations.md +++ b/serde_with/src/guide/serde_as_transformations.md @@ -11,26 +11,27 @@ This page lists the transformations implemented in this crate and supported by ` 7. [Convert to an intermediate type using `TryInto`](#convert-to-an-intermediate-type-using-tryinto) 8. [`Default` from `null`](#default-from-null) 9. [De/Serialize into `Vec`, ignoring errors](#deserialize-into-vec-ignoring-errors) -10. [De/Serialize with `FromStr` and `Display`](#deserialize-with-fromstr-and-display) -11. [`Duration` as seconds](#duration-as-seconds) -12. [Hex encode bytes](#hex-encode-bytes) -13. [Ignore deserialization errors](#ignore-deserialization-errors) -14. [`Maps` to `Vec` of enums](#maps-to-vec-of-enums) -15. [`Maps` to `Vec` of tuples](#maps-to-vec-of-tuples) -16. [`NaiveDateTime` like UTC timestamp](#naivedatetime-like-utc-timestamp) -17. [`None` as empty `String`](#none-as-empty-string) -18. [One or many elements into `Vec`](#one-or-many-elements-into-vec) -19. [Overwrite existing set values](#overwrite-existing-set-values) -20. [Pick first successful deserialization](#pick-first-successful-deserialization) -21. [Prefer the first map key when duplicates exist](#prefer-the-first-map-key-when-duplicates-exist) -22. [Prevent duplicate map keys](#prevent-duplicate-map-keys) -23. [Prevent duplicate set values](#prevent-duplicate-set-values) -24. [Struct fields as map keys](#struct-fields-as-map-keys) -25. [Timestamps as seconds since UNIX epoch](#timestamps-as-seconds-since-unix-epoch) -26. [Value into JSON String](#value-into-json-string) -27. [`Vec` of tuples to `Maps`](#vec-of-tuples-to-maps) -28. [Well-known time formats for `OffsetDateTime`](#well-known-time-formats-for-offsetdatetime) -29. [De/Serialize depending on `De/Serializer::is_human_readable`](#deserialize-depending-on-deserializeris_human_readable) +10. [De/Serialize into a map, ignoring errors](#deserialize-into-a-map-ignoring-errors) +11. [De/Serialize with `FromStr` and `Display`](#deserialize-with-fromstr-and-display) +12. [`Duration` as seconds](#duration-as-seconds) +13. [Hex encode bytes](#hex-encode-bytes) +14. [Ignore deserialization errors](#ignore-deserialization-errors) +15. [`Maps` to `Vec` of enums](#maps-to-vec-of-enums) +16. [`Maps` to `Vec` of tuples](#maps-to-vec-of-tuples) +17. [`NaiveDateTime` like UTC timestamp](#naivedatetime-like-utc-timestamp) +18. [`None` as empty `String`](#none-as-empty-string) +19. [One or many elements into `Vec`](#one-or-many-elements-into-vec) +20. [Overwrite existing set values](#overwrite-existing-set-values) +21. [Pick first successful deserialization](#pick-first-successful-deserialization) +22. [Prefer the first map key when duplicates exist](#prefer-the-first-map-key-when-duplicates-exist) +23. [Prevent duplicate map keys](#prevent-duplicate-map-keys) +24. [Prevent duplicate set values](#prevent-duplicate-set-values) +25. [Struct fields as map keys](#struct-fields-as-map-keys) +26. [Timestamps as seconds since UNIX epoch](#timestamps-as-seconds-since-unix-epoch) +27. [Value into JSON String](#value-into-json-string) +28. [`Vec` of tuples to `Maps`](#vec-of-tuples-to-maps) +29. [Well-known time formats for `OffsetDateTime`](#well-known-time-formats-for-offsetdatetime) +30. [De/Serialize depending on `De/Serializer::is_human_readable`](#deserialize-depending-on-deserializeris_human_readable) ## Base64 encode bytes @@ -181,6 +182,25 @@ colors: Vec, // => vec![Blue, Green] ``` +## De/Serialize into a map, ignoring errors + +[`MapSkipError`] + +For formats with heterogeneously typed maps, we can collect only the elements where both key and value are deserializable. +This is also useful in conjunction to `#[serde(flatten)]` to ingore some entries when capturing additional fields. + +```ignore +// JSON +"value": {"0": "v0", "5": "v5", "str": "str", "10": 2}, + +// Rust +#[serde_as(as = "MapSkipError")] +value: BTreeMap, + +// Only deserializes entries with a numerical key and a string value, i.e., +{0 => "v0", 5 => "v5"} +``` + ## De/Serialize with `FromStr` and `Display` Useful if a type implements `FromStr` / `Display` but not `Deserialize` / `Serialize`. @@ -614,3 +634,4 @@ value: u128, [`TimestampSecondsWithFrac`]: crate::TimestampSecondsWithFrac [`TryFromInto`]: crate::TryFromInto [`VecSkipError`]: crate::VecSkipError +[`MapSkipError`]: crate::MapSkipError diff --git a/serde_with/src/lib.rs b/serde_with/src/lib.rs index 70b5cd28..c64f1f6a 100644 --- a/serde_with/src/lib.rs +++ b/serde_with/src/lib.rs @@ -2205,6 +2205,64 @@ pub struct BorrowCow; #[cfg(feature = "alloc")] pub struct VecSkipError(PhantomData); +/// Deserialize a map, skipping keys and values which fail to deserialize. +/// +/// By default serde terminates if it fails to deserialize a key or a value when deserializing +/// a map. Sometimes a map has heterogeneous keys or values but we only care about some specific +/// types, and it is desirable to skip entries on errors. +/// +/// It is especially useful in conjunction to `#[serde(flatten)]` to capture a map mixed in with +/// other entries which we don't want to exhaust in the type definition. +/// +/// The serialization behavior is identical to the underlying map. +/// +/// The implementation supports both the [`HashMap`] and the [`BTreeMap`] from the standard library. +/// +/// [`BTreeMap`]: std::collections::BTreeMap +/// [`HashMap`]: std::collections::HashMap +/// +/// # Examples +/// +/// ```rust +/// # #[cfg(feature = "macros")] { +/// # use serde::{Deserialize, Serialize}; +/// # use std::collections::BTreeMap; +/// # use serde_with::{serde_as, DisplayFromStr, MapSkipError}; +/// # +/// #[serde_as] +/// # #[derive(Debug, PartialEq)] +/// #[derive(Deserialize, Serialize)] +/// struct VersionNames { +/// yanked: Vec, +/// #[serde_as(as = "MapSkipError")] +/// #[serde(flatten)] +/// names: BTreeMap, +/// } +/// +/// let data = VersionNames { +/// yanked: vec![2, 5], +/// names: BTreeMap::from_iter([ +/// (0u16, "v0".to_string()), +/// (1, "v1".to_string()), +/// (4, "v4".to_string()) +/// ]), +/// }; +/// let source_json = r#"{ +/// "0": "v0", +/// "1": "v1", +/// "4": "v4", +/// "yanked": [2, 5], +/// "last_updated": 1704085200 +/// }"#; +/// let data_json = r#"{"yanked":[2,5],"0":"v0","1":"v1","4":"v4"}"#; +/// // Ensure serialization and deserialization produce the expected results +/// assert_eq!(data_json, serde_json::to_string(&data).unwrap()); +/// assert_eq!(data, serde_json::from_str(source_json).unwrap()); +/// # } +/// ``` +#[cfg(feature = "alloc")] +pub struct MapSkipError(PhantomData<(K, V)>); + /// Deserialize a boolean from a number /// /// Deserialize a number (of `u8`) and turn it into a boolean. diff --git a/serde_with/src/ser/skip_error.rs b/serde_with/src/ser/skip_error.rs index 02509cec..80066dd2 100644 --- a/serde_with/src/ser/skip_error.rs +++ b/serde_with/src/ser/skip_error.rs @@ -1,5 +1,13 @@ +use super::impls::macros::foreach_map; use crate::prelude::*; +#[cfg(feature = "hashbrown_0_14")] +use hashbrown_0_14::HashMap as HashbrownMap014; +#[cfg(feature = "indexmap_1")] +use indexmap_1::IndexMap; +#[cfg(feature = "indexmap_2")] +use indexmap_2::IndexMap as IndexMap2; + impl SerializeAs> for VecSkipError where U: SerializeAs, @@ -11,3 +19,22 @@ where Vec::::serialize_as(source, serializer) } } + +macro_rules! map_skip_error_handling { + ($tyorig:ident < K, V $(, $typaram:ident : $bound:ident)* >) => { + impl SerializeAs<$tyorig> for MapSkipError + where + KAs: SerializeAs, + VAs: SerializeAs, + $($typaram: ?Sized + $bound,)* + { + fn serialize_as(value: &$tyorig, serializer: S) -> Result + where + S: Serializer, + { + <$tyorig>::serialize_as(value, serializer) + } + } + } +} +foreach_map!(map_skip_error_handling); diff --git a/serde_with/tests/hashbrown_0_14.rs b/serde_with/tests/hashbrown_0_14.rs index 2f1267e8..9e522e79 100644 --- a/serde_with/tests/hashbrown_0_14.rs +++ b/serde_with/tests/hashbrown_0_14.rs @@ -291,3 +291,92 @@ fn prohibit_duplicate_value_hashset() { expect![[r#"invalid entry: found duplicate value at line 1 column 15"#]], ); } + +#[test] +fn test_map_skip_error_hashmap() { + use serde_with::MapSkipError; + + #[serde_as] + #[derive(Debug, PartialEq, Deserialize, Serialize)] + struct S { + tag: String, + #[serde_as(as = "MapSkipError")] + values: HashMap, + } + + check_deserialization( + S { + tag: "type".into(), + values: [(0, 1), (10, 20)].into_iter().collect(), + }, + r#" + { + "tag":"type", + "values": { + "0": 1, + "str": 2, + "3": "str", + "4": [10, 11], + "5": {}, + "10": 20 + } + }"#, + ); + is_equal( + S { + tag: "round-trip".into(), + values: [(0, 0), (255, 255)].into_iter().collect(), + }, + expect![[r#" + { + "tag": "round-trip", + "values": { + "255": 255, + "0": 0 + } + }"#]], + ); +} + +#[test] +fn test_map_skip_error_hashmap_flatten() { + use serde_with::MapSkipError; + + #[serde_as] + #[derive(Debug, PartialEq, Deserialize, Serialize)] + struct S { + tag: String, + #[serde_as(as = "MapSkipError")] + #[serde(flatten)] + values: HashMap, + } + + check_deserialization( + S { + tag: "type".into(), + values: [(0, 1), (10, 20)].into_iter().collect(), + }, + r#" + { + "tag":"type", + "0": 1, + "str": 2, + "3": "str", + "4": [10, 11], + "5": {}, + "10": 20 + }"#, + ); + is_equal( + S { + tag: "round-trip".into(), + values: [(0, 0), (255, 255)].into_iter().collect(), + }, + expect![[r#" + { + "tag": "round-trip", + "255": 255, + "0": 0 + }"#]], + ); +} diff --git a/serde_with/tests/indexmap_1.rs b/serde_with/tests/indexmap_1.rs index afb7ce82..49f50fe1 100644 --- a/serde_with/tests/indexmap_1.rs +++ b/serde_with/tests/indexmap_1.rs @@ -251,3 +251,92 @@ fn prohibit_duplicate_value_indexset() { expect![[r#"invalid entry: found duplicate value at line 1 column 15"#]], ); } + +#[test] +fn test_map_skip_error_indexmap() { + use serde_with::MapSkipError; + + #[serde_as] + #[derive(Debug, PartialEq, Deserialize, Serialize)] + struct S { + tag: String, + #[serde_as(as = "MapSkipError")] + values: IndexMap, + } + + check_deserialization( + S { + tag: "type".into(), + values: [(0, 1), (10, 20)].into_iter().collect(), + }, + r#" + { + "tag":"type", + "values": { + "0": 1, + "str": 2, + "3": "str", + "4": [10, 11], + "5": {}, + "10": 20 + } + }"#, + ); + is_equal( + S { + tag: "round-trip".into(), + values: [(0, 0), (255, 255)].into_iter().collect(), + }, + expect![[r#" + { + "tag": "round-trip", + "values": { + "0": 0, + "255": 255 + } + }"#]], + ); +} + +#[test] +fn test_map_skip_error_indexmap_flatten() { + use serde_with::MapSkipError; + + #[serde_as] + #[derive(Debug, PartialEq, Deserialize, Serialize)] + struct S { + tag: String, + #[serde_as(as = "MapSkipError")] + #[serde(flatten)] + values: IndexMap, + } + + check_deserialization( + S { + tag: "type".into(), + values: [(0, 1), (10, 20)].into_iter().collect(), + }, + r#" + { + "tag":"type", + "0": 1, + "str": 2, + "3": "str", + "4": [10, 11], + "5": {}, + "10": 20 + }"#, + ); + is_equal( + S { + tag: "round-trip".into(), + values: [(0, 0), (255, 255)].into_iter().collect(), + }, + expect![[r#" + { + "tag": "round-trip", + "0": 0, + "255": 255 + }"#]], + ); +} diff --git a/serde_with/tests/indexmap_2.rs b/serde_with/tests/indexmap_2.rs index 61edb10a..91de9b3b 100644 --- a/serde_with/tests/indexmap_2.rs +++ b/serde_with/tests/indexmap_2.rs @@ -251,3 +251,92 @@ fn prohibit_duplicate_value_indexset() { expect![[r#"invalid entry: found duplicate value at line 1 column 15"#]], ); } + +#[test] +fn test_map_skip_error_indexmap() { + use serde_with::MapSkipError; + + #[serde_as] + #[derive(Debug, PartialEq, Deserialize, Serialize)] + struct S { + tag: String, + #[serde_as(as = "MapSkipError")] + values: IndexMap, + } + + check_deserialization( + S { + tag: "type".into(), + values: [(0, 1), (10, 20)].into_iter().collect(), + }, + r#" + { + "tag":"type", + "values": { + "0": 1, + "str": 2, + "3": "str", + "4": [10, 11], + "5": {}, + "10": 20 + } + }"#, + ); + is_equal( + S { + tag: "round-trip".into(), + values: [(0, 0), (255, 255)].into_iter().collect(), + }, + expect![[r#" + { + "tag": "round-trip", + "values": { + "0": 0, + "255": 255 + } + }"#]], + ); +} + +#[test] +fn test_map_skip_error_indexmap_flatten() { + use serde_with::MapSkipError; + + #[serde_as] + #[derive(Debug, PartialEq, Deserialize, Serialize)] + struct S { + tag: String, + #[serde_as(as = "MapSkipError")] + #[serde(flatten)] + values: IndexMap, + } + + check_deserialization( + S { + tag: "type".into(), + values: [(0, 1), (10, 20)].into_iter().collect(), + }, + r#" + { + "tag":"type", + "0": 1, + "str": 2, + "3": "str", + "4": [10, 11], + "5": {}, + "10": 20 + }"#, + ); + is_equal( + S { + tag: "round-trip".into(), + values: [(0, 0), (255, 255)].into_iter().collect(), + }, + expect![[r#" + { + "tag": "round-trip", + "0": 0, + "255": 255 + }"#]], + ); +} diff --git a/serde_with/tests/serde_as/lib.rs b/serde_with/tests/serde_as/lib.rs index c728ceea..2f525775 100644 --- a/serde_with/tests/serde_as/lib.rs +++ b/serde_with/tests/serde_as/lib.rs @@ -588,6 +588,180 @@ fn test_vec_skip_error() { ); } +#[test] +fn test_map_skip_error_btreemap() { + use serde_with::MapSkipError; + + #[serde_as] + #[derive(Debug, PartialEq, Deserialize, Serialize)] + struct S { + tag: String, + #[serde_as(as = "MapSkipError")] + values: BTreeMap, + } + + check_deserialization( + S { + tag: "type".into(), + values: [(0, 1), (10, 20)].into_iter().collect(), + }, + r#" + { + "tag":"type", + "values": { + "0": 1, + "str": 2, + "3": "str", + "4": [10, 11], + "5": {}, + "10": 20 + } + }"#, + ); + is_equal( + S { + tag: "round-trip".into(), + values: [(0, 0), (255, 255)].into_iter().collect(), + }, + expect![[r#" + { + "tag": "round-trip", + "values": { + "0": 0, + "255": 255 + } + }"#]], + ); +} + +#[test] +fn test_map_skip_error_btreemap_flatten() { + use serde_with::MapSkipError; + + #[serde_as] + #[derive(Debug, PartialEq, Deserialize, Serialize)] + struct S { + tag: String, + #[serde_as(as = "MapSkipError")] + #[serde(flatten)] + values: BTreeMap, + } + + check_deserialization( + S { + tag: "type".into(), + values: [(0, 1), (10, 20)].into_iter().collect(), + }, + r#" + { + "tag":"type", + "0": 1, + "str": 2, + "3": "str", + "4": [10, 11], + "5": {}, + "10": 20 + }"#, + ); + is_equal( + S { + tag: "round-trip".into(), + values: [(0, 0), (255, 255)].into_iter().collect(), + }, + expect![[r#" + { + "tag": "round-trip", + "0": 0, + "255": 255 + }"#]], + ); +} + +#[test] +fn test_map_skip_error_hashmap() { + use serde_with::MapSkipError; + + #[serde_as] + #[derive(Debug, PartialEq, Deserialize, Serialize)] + struct S { + tag: String, + #[serde_as(as = "MapSkipError")] + values: HashMap, + } + + check_deserialization( + S { + tag: "type".into(), + values: [(0, 1)].into_iter().collect(), + }, + r#" + { + "tag":"type", + "values": { + "0": 1, + "str": 2, + "3": "str", + "4": [10, 11], + "5": {} + } + }"#, + ); + is_equal( + S { + tag: "round-trip".into(), + values: [(255, 0)].into_iter().collect(), + }, + expect![[r#" + { + "tag": "round-trip", + "values": { + "255": 0 + } + }"#]], + ); +} + +#[test] +fn test_map_skip_error_hashmap_flatten() { + use serde_with::MapSkipError; + + #[serde_as] + #[derive(Debug, PartialEq, Deserialize, Serialize)] + struct S { + tag: String, + #[serde_as(as = "MapSkipError")] + #[serde(flatten)] + values: HashMap, + } + + check_deserialization( + S { + tag: "type".into(), + values: [(0, 1)].into_iter().collect(), + }, + r#" + { + "tag":"type", + "0": 1, + "str": 2, + "3": "str", + "4": [10, 11], + "5": {} + }"#, + ); + is_equal( + S { + tag: "round-trip".into(), + values: [(255, 0)].into_iter().collect(), + }, + expect![[r#" + { + "tag": "round-trip", + "255": 0 + }"#]], + ); +} + #[test] fn test_serialize_reference() { #[serde_as]