From 5b359d9e5e55722ab73fac28b0729bb0472704ed Mon Sep 17 00:00:00 2001 From: Lachlan Sneff Date: Fri, 1 Jul 2022 13:56:25 -0700 Subject: [PATCH 1/8] Add function for computing the postcard serialized size of a value --- src/lib.rs | 2 + src/ser/mod.rs | 91 +++++++--- src/ser/serializer.rs | 8 +- src/ser/size.rs | 412 ++++++++++++++++++++++++++++++++++++++++++ src/varint.rs | 65 +++++++ 5 files changed, 551 insertions(+), 27 deletions(-) create mode 100644 src/ser/size.rs diff --git a/src/lib.rs b/src/lib.rs index a8d8159..b3cccf0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -57,6 +57,8 @@ pub mod experimental { /// Compile time max-serialization size calculation #[cfg(feature = "experimental-derive")] pub mod max_size { + pub use crate::ser::serialized_size; + // NOTE: This is the trait... pub use crate::max_size::MaxSize; // NOTE: ...and this is the derive macro diff --git a/src/ser/mod.rs b/src/ser/mod.rs index 17177c3..316073a 100644 --- a/src/ser/mod.rs +++ b/src/ser/mod.rs @@ -18,6 +18,7 @@ use crate::ser::serializer::Serializer; pub mod flavors; pub(crate) mod serializer; +mod size; /// Serialize a `T` to the given slice, with the resulting slice containing /// data in a serialized then COBS encoded format. The terminating sentinel @@ -276,6 +277,16 @@ where .map_err(|_| Error::SerializeBufferFull) } +/// Compute the size of the postcard serialization of `T`. +#[cfg(any(test, feature = "experimental-derive"))] +pub fn serialized_size(value: &T) -> Result +where + T: Serialize + ?Sized, +{ + let mut sizer = size::Sizer { total: 0 }; + value.serialize(&mut sizer).map(|_| sizer.total) +} + #[cfg(feature = "heapless")] #[cfg(test)] mod test { @@ -291,6 +302,7 @@ mod test { fn ser_u8() { let output: Vec = to_vec(&0x05u8).unwrap(); assert!(&[5] == output.deref()); + assert!(output.len() == serialized_size(&0x05u8).unwrap()); assert!(output.len() <= Vec::::POSTCARD_MAX_SIZE); } @@ -299,6 +311,7 @@ mod test { const SZ: usize = varint_max::(); let output: Vec = to_vec(&0xA5C7u16).unwrap(); assert_eq!(&[0xC7, 0xCB, 0x02], output.deref()); + assert!(output.len() == serialized_size(&0xA5C7u16).unwrap()); assert!(output.len() <= Vec::::POSTCARD_MAX_SIZE); } @@ -307,6 +320,7 @@ mod test { const SZ: usize = varint_max::(); let output: Vec = to_vec(&0xCDAB3412u32).unwrap(); assert_eq!(&[0x92, 0xE8, 0xAC, 0xED, 0x0C], output.deref()); + assert!(output.len() == serialized_size(&0xCDAB3412u32).unwrap()); assert!(output.len() <= Vec::::POSTCARD_MAX_SIZE); } @@ -318,6 +332,7 @@ mod test { &[0xEF, 0x9B, 0xAF, 0x85, 0x89, 0xCF, 0x95, 0x9A, 0x12], output.deref() ); + assert!(output.len() == serialized_size(&0x1234_5678_90AB_CDEFu64).unwrap()); assert!(output.len() <= Vec::::POSTCARD_MAX_SIZE); } @@ -332,6 +347,10 @@ mod test { ], output.deref() ); + assert!( + output.len() + == serialized_size(&0x1234_5678_90AB_CDEF_1234_5678_90AB_CDEFu128).unwrap() + ); assert!(output.len() <= Vec::::POSTCARD_MAX_SIZE); } @@ -357,14 +376,14 @@ mod test { #[test] fn ser_struct_unsigned() { const SZ: usize = BasicU8S::POSTCARD_MAX_SIZE; - let output: Vec = to_vec(&BasicU8S { + let input = BasicU8S { st: 0xABCD, ei: 0xFE, ote: 0x1234_4321_ABCD_DCBA_1234_4321_ABCD_DCBA, sf: 0x1234_4321_ABCD_DCBA, tt: 0xACAC_ACAC, - }) - .unwrap(); + }; + let output: Vec = to_vec(&input).unwrap(); assert_eq!( &[ @@ -374,6 +393,7 @@ mod test { ], output.deref() ); + assert!(output.len() == serialized_size(&input).unwrap()); assert!(output.len() <= BasicU8S::POSTCARD_MAX_SIZE); } @@ -385,6 +405,7 @@ mod test { &[0x08, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08], output.deref() ); + assert!(output.len() == serialized_size(&input).unwrap()); let mut input: Vec = Vec::new(); for i in 0..1024 { @@ -405,6 +426,7 @@ mod test { let output: Vec = to_vec(input).unwrap(); assert_eq!(0x10, output.deref()[0]); assert_eq!(input.as_bytes(), &output.deref()[1..]); + assert!(output.len() == serialized_size(&input).unwrap()); let mut input: String<1024> = String::new(); for _ in 0..256 { @@ -468,48 +490,60 @@ mod test { #[test] fn enums() { - let output: Vec = to_vec(&BasicEnum::Bim).unwrap(); + let input = BasicEnum::Bim; + let output: Vec = to_vec(&input).unwrap(); assert_eq!(&[0x01], output.deref()); + assert!(output.len() == serialized_size(&input).unwrap()); - let output: Vec() }> = - to_vec(&DataEnum::Bim(u64::max_value())).unwrap(); + let input = DataEnum::Bim(u64::max_value()); + let output: Vec() }> = to_vec(&input).unwrap(); assert_eq!( &[0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01], output.deref() ); + assert!(output.len() == serialized_size(&input).unwrap()); - let output: Vec() }> = - to_vec(&DataEnum::Bib(u16::max_value())).unwrap(); + let input = DataEnum::Bib(u16::max_value()); + let output: Vec() }> = to_vec(&input).unwrap(); assert_eq!(&[0x00, 0xFF, 0xFF, 0x03], output.deref()); + assert!(output.len() == serialized_size(&input).unwrap()); - let output: Vec = to_vec(&DataEnum::Bap(u8::max_value())).unwrap(); + let input = DataEnum::Bap(u8::max_value()); + let output: Vec = to_vec(&input).unwrap(); assert_eq!(&[0x02, 0xFF], output.deref()); + assert!(output.len() == serialized_size(&input).unwrap()); - let output: Vec = to_vec(&DataEnum::Kim(EnumStruct { + let input = DataEnum::Kim(EnumStruct { eight: 0xF0, sixt: 0xACAC, - })) - .unwrap(); + }); + let output: Vec = to_vec(&input).unwrap(); assert_eq!(&[0x03, 0xF0, 0xAC, 0xD9, 0x02], output.deref()); + assert!(output.len() == serialized_size(&input).unwrap()); - let output: Vec = to_vec(&DataEnum::Chi { + let input = DataEnum::Chi { a: 0x0F, b: 0xC7C7C7C7, - }) - .unwrap(); + }; + let output: Vec = to_vec(&input).unwrap(); assert_eq!(&[0x04, 0x0F, 0xC7, 0x8F, 0x9F, 0xBE, 0x0C], output.deref()); + assert!(output.len() == serialized_size(&input).unwrap()); - let output: Vec = to_vec(&DataEnum::Sho(0x6969, 0x07)).unwrap(); + let input = DataEnum::Sho(0x6969, 0x07); + let output: Vec = to_vec(&input).unwrap(); assert_eq!(&[0x05, 0xE9, 0xD2, 0x01, 0x07], output.deref()); + assert!(output.len() == serialized_size(&input).unwrap()); } #[test] fn tuples() { - let output: Vec = to_vec(&(1u8, 10u32, "Hello!")).unwrap(); + let input = (1u8, 10u32, "Hello!"); + let output: Vec = to_vec(&input).unwrap(); assert_eq!( &[1u8, 0x0A, 0x06, b'H', b'e', b'l', b'l', b'o', b'!'], output.deref() - ) + ); + assert!(output.len() == serialized_size(&input).unwrap()); } #[test] @@ -517,11 +551,13 @@ mod test { let x: &[u8; 32] = &[0u8; 32]; let output: Vec = to_vec(x).unwrap(); assert_eq!(output.len(), 32); + assert!(output.len() == serialized_size(&x).unwrap()); assert!(<[u8; 32] as MaxSize>::POSTCARD_MAX_SIZE <= output.len()); let x: &[u8] = &[0u8; 32]; let output: Vec = to_vec(x).unwrap(); assert_eq!(output.len(), 33); + assert!(output.len() == serialized_size(&x).unwrap()); } #[derive(Serialize)] @@ -532,11 +568,15 @@ mod test { #[test] fn structs() { - let output: Vec = to_vec(&NewTypeStruct(5)).unwrap(); + let input = NewTypeStruct(5); + let output: Vec = to_vec(&input).unwrap(); assert_eq!(&[0x05], output.deref()); + assert!(output.len() == serialized_size(&input).unwrap()); - let output: Vec = to_vec(&TupleStruct((0xA0, 0x1234))).unwrap(); + let input = TupleStruct((0xA0, 0x1234)); + let output: Vec = to_vec(&input).unwrap(); assert_eq!(&[0xA0, 0xB4, 0x24], output.deref()); + assert!(output.len() == serialized_size(&input).unwrap()); } #[derive(Serialize, Deserialize, Eq, PartialEq, Debug)] @@ -549,22 +589,24 @@ mod test { fn ref_struct() { let message = "hElLo"; let bytes = [0x01, 0x10, 0x02, 0x20]; - let output: Vec = to_vec(&RefStruct { + let input = RefStruct { bytes: &bytes, str_s: message, - }) - .unwrap(); + }; + let output: Vec = to_vec(&input).unwrap(); assert_eq!( &[0x04, 0x01, 0x10, 0x02, 0x20, 0x05, b'h', b'E', b'l', b'L', b'o',], output.deref() ); + assert!(output.len() == serialized_size(&input).unwrap()); } #[test] fn unit() { let output: Vec = to_vec(&()).unwrap(); assert_eq!(output.len(), 0); + assert!(output.len() == serialized_size(&()).unwrap()); } #[test] @@ -573,11 +615,13 @@ mod test { input.extend_from_slice(&[0x01, 0x02, 0x03, 0x04]).unwrap(); let output: Vec = to_vec(&input).unwrap(); assert_eq!(&[0x04, 0x01, 0x02, 0x03, 0x04], output.deref()); + assert!(output.len() == serialized_size(&input).unwrap()); let mut input: String<8> = String::new(); write!(&mut input, "helLO!").unwrap(); let output: Vec = to_vec(&input).unwrap(); assert_eq!(&[0x06, b'h', b'e', b'l', b'L', b'O', b'!'], output.deref()); + assert!(output.len() == serialized_size(&input).unwrap()); let mut input: FnvIndexMap = FnvIndexMap::new(); input.insert(0x01, 0x05).unwrap(); @@ -589,6 +633,7 @@ mod test { &[0x04, 0x01, 0x05, 0x02, 0x06, 0x03, 0x07, 0x04, 0x08], output.deref() ); + assert!(output.len() == serialized_size(&input).unwrap()); } #[test] diff --git a/src/ser/serializer.rs b/src/ser/serializer.rs index fc85ffd..6ac1139 100644 --- a/src/ser/serializer.rs +++ b/src/ser/serializer.rs @@ -547,18 +547,18 @@ where } } -fn zig_zag_i16(n: i16) -> u16 { +pub(crate) fn zig_zag_i16(n: i16) -> u16 { ((n << 1) ^ (n >> 15)) as u16 } -fn zig_zag_i32(n: i32) -> u32 { +pub(crate) fn zig_zag_i32(n: i32) -> u32 { ((n << 1) ^ (n >> 31)) as u32 } -fn zig_zag_i64(n: i64) -> u64 { +pub(crate) fn zig_zag_i64(n: i64) -> u64 { ((n << 1) ^ (n >> 63)) as u64 } -fn zig_zag_i128(n: i128) -> u128 { +pub(crate) fn zig_zag_i128(n: i128) -> u128 { ((n << 1) ^ (n >> 127)) as u128 } diff --git a/src/ser/size.rs b/src/ser/size.rs new file mode 100644 index 0000000..b46c8f5 --- /dev/null +++ b/src/ser/size.rs @@ -0,0 +1,412 @@ +use serde::{ser, Serialize}; + +use crate::error::{Error, Result}; +use crate::ser::serializer::{zig_zag_i128, zig_zag_i16, zig_zag_i32, zig_zag_i64}; +use crate::varint::*; + +pub struct Sizer { + pub total: usize, +} + +impl<'a> ser::Serializer for &'a mut Sizer { + type Ok = (); + + type Error = Error; + + // Associated types for keeping track of additional state while serializing + // compound data structures like sequences and maps. In this case no + // additional state is required beyond what is already stored in the + // Serializer struct. + type SerializeSeq = Self; + type SerializeTuple = Self; + type SerializeTupleStruct = Self; + type SerializeTupleVariant = Self; + type SerializeMap = Self; + type SerializeStruct = Self; + type SerializeStructVariant = Self; + + #[inline] + fn is_human_readable(&self) -> bool { + false + } + + #[inline] + fn serialize_bool(self, _v: bool) -> Result<()> { + self.serialize_u8(0) + } + + #[inline] + fn serialize_i8(self, _v: i8) -> Result<()> { + self.serialize_u8(0) + } + + #[inline] + fn serialize_i16(self, v: i16) -> Result<()> { + let zzv = zig_zag_i16(v); + self.total += varint_size_u16(zzv); + Ok(()) + } + + #[inline] + fn serialize_i32(self, v: i32) -> Result<()> { + let zzv = zig_zag_i32(v); + self.total += varint_size_u32(zzv); + Ok(()) + } + + #[inline] + fn serialize_i64(self, v: i64) -> Result<()> { + let zzv = zig_zag_i64(v); + self.total += varint_size_u64(zzv); + Ok(()) + } + + #[inline] + fn serialize_i128(self, v: i128) -> Result<()> { + let zzv = zig_zag_i128(v); + self.total += varint_size_u128(zzv); + Ok(()) + } + + #[inline] + fn serialize_u8(self, _v: u8) -> Result<()> { + self.total += 1; + Ok(()) + } + + #[inline] + fn serialize_u16(self, v: u16) -> Result<()> { + self.total += varint_size_u16(v); + Ok(()) + } + + #[inline] + fn serialize_u32(self, v: u32) -> Result<()> { + self.total += varint_size_u32(v); + Ok(()) + } + + #[inline] + fn serialize_u64(self, v: u64) -> Result<()> { + self.total += varint_size_u64(v); + Ok(()) + } + + #[inline] + fn serialize_u128(self, v: u128) -> Result<()> { + self.total += varint_size_u128(v); + Ok(()) + } + + #[inline] + fn serialize_f32(self, _v: f32) -> Result<()> { + self.total += 4; + Ok(()) + } + + #[inline] + fn serialize_f64(self, _v: f64) -> Result<()> { + self.total += 8; + Ok(()) + } + + #[inline] + fn serialize_char(self, v: char) -> Result<()> { + let mut buf = [0u8; 4]; + let strsl = v.encode_utf8(&mut buf); + strsl.serialize(self) + } + + #[inline] + fn serialize_str(self, v: &str) -> Result<()> { + self.total += varint_size_usize(v.len()); + self.total += v.len(); + Ok(()) + } + + #[inline] + fn serialize_bytes(self, v: &[u8]) -> Result<()> { + self.total += varint_size_usize(v.len()); + self.total += v.len(); + Ok(()) + } + + #[inline] + fn serialize_none(self) -> Result<()> { + self.serialize_u8(0) + } + + #[inline] + fn serialize_some(self, value: &T) -> Result<()> + where + T: ?Sized + Serialize, + { + self.serialize_u8(1)?; + value.serialize(self) + } + + #[inline] + fn serialize_unit(self) -> Result<()> { + Ok(()) + } + + #[inline] + fn serialize_unit_struct(self, _name: &'static str) -> Result<()> { + Ok(()) + } + + #[inline] + fn serialize_unit_variant( + self, + _name: &'static str, + variant_index: u32, + _variant: &'static str, + ) -> Result<()> { + self.total += varint_size_u32(variant_index); + Ok(()) + } + + #[inline] + fn serialize_newtype_struct(self, _name: &'static str, value: &T) -> Result<()> + where + T: ?Sized + Serialize, + { + value.serialize(self) + } + + #[inline] + fn serialize_newtype_variant( + self, + _name: &'static str, + variant_index: u32, + _variant: &'static str, + value: &T, + ) -> Result<()> + where + T: ?Sized + Serialize, + { + self.total += varint_size_u32(variant_index); + value.serialize(self) + } + + #[inline] + fn serialize_seq(self, len: Option) -> Result { + self.total += varint_size_usize(len.ok_or(Error::SerializeSeqLengthUnknown)?); + Ok(self) + } + + #[inline] + fn serialize_tuple(self, _len: usize) -> Result { + Ok(self) + } + + #[inline] + fn serialize_tuple_struct( + self, + _name: &'static str, + _len: usize, + ) -> Result { + Ok(self) + } + + #[inline] + fn serialize_tuple_variant( + self, + _name: &'static str, + variant_index: u32, + _variant: &'static str, + _len: usize, + ) -> Result { + self.total += varint_size_u32(variant_index); + Ok(self) + } + + #[inline] + fn serialize_map(self, len: Option) -> Result { + self.total += varint_size_usize(len.ok_or(Error::SerializeSeqLengthUnknown)?); + Ok(self) + } + + #[inline] + fn serialize_struct(self, _name: &'static str, _len: usize) -> Result { + Ok(self) + } + + #[inline] + fn serialize_struct_variant( + self, + _name: &'static str, + variant_index: u32, + _variant: &'static str, + _len: usize, + ) -> Result { + self.total += varint_size_u32(variant_index); + Ok(self) + } + + #[inline] + fn collect_str(self, value: &T) -> Result + where + T: core::fmt::Display, + { + use core::fmt::Write; + + struct CountWriter { + ct: usize, + } + impl Write for CountWriter { + fn write_str(&mut self, s: &str) -> core::result::Result<(), core::fmt::Error> { + self.ct += s.as_bytes().len(); + Ok(()) + } + } + + let mut ctr = CountWriter { ct: 0 }; + + // This is the first pass through, where we just count the length of the + // data that we are given + write!(&mut ctr, "{}", value).map_err(|_| Error::CollectStrError)?; + let len = ctr.ct; + self.total += varint_size_usize(len); + + Ok(()) + } +} + +impl<'a> ser::SerializeSeq for &'a mut Sizer { + // Must match the `Ok` type of the serializer. + type Ok = (); + // Must match the `Error` type of the serializer. + type Error = Error; + + // Serialize a single element of the sequence. + #[inline] + fn serialize_element(&mut self, value: &T) -> Result<()> + where + T: ?Sized + Serialize, + { + value.serialize(&mut **self) + } + + // Close the sequence. + #[inline] + fn end(self) -> Result<()> { + Ok(()) + } +} + +impl<'a> ser::SerializeTuple for &'a mut Sizer { + type Ok = (); + type Error = Error; + + #[inline] + fn serialize_element(&mut self, value: &T) -> Result<()> + where + T: ?Sized + Serialize, + { + value.serialize(&mut **self) + } + + #[inline] + fn end(self) -> Result<()> { + Ok(()) + } +} + +impl<'a> ser::SerializeTupleStruct for &'a mut Sizer { + type Ok = (); + type Error = Error; + + #[inline] + fn serialize_field(&mut self, value: &T) -> Result<()> + where + T: ?Sized + Serialize, + { + value.serialize(&mut **self) + } + + #[inline] + fn end(self) -> Result<()> { + Ok(()) + } +} + +impl<'a> ser::SerializeTupleVariant for &'a mut Sizer { + type Ok = (); + type Error = Error; + + #[inline] + fn serialize_field(&mut self, value: &T) -> Result<()> + where + T: ?Sized + Serialize, + { + value.serialize(&mut **self) + } + + #[inline] + fn end(self) -> Result<()> { + Ok(()) + } +} + +impl<'a> ser::SerializeMap for &'a mut Sizer { + type Ok = (); + type Error = Error; + + #[inline] + fn serialize_key(&mut self, key: &T) -> Result<()> + where + T: ?Sized + Serialize, + { + key.serialize(&mut **self) + } + + #[inline] + fn serialize_value(&mut self, value: &T) -> Result<()> + where + T: ?Sized + Serialize, + { + value.serialize(&mut **self) + } + + #[inline] + fn end(self) -> Result<()> { + Ok(()) + } +} + +impl<'a> ser::SerializeStruct for &'a mut Sizer { + type Ok = (); + type Error = Error; + + #[inline] + fn serialize_field(&mut self, _key: &'static str, value: &T) -> Result<()> + where + T: ?Sized + Serialize, + { + value.serialize(&mut **self) + } + + #[inline] + fn end(self) -> Result<()> { + Ok(()) + } +} + +impl<'a> ser::SerializeStructVariant for &'a mut Sizer { + type Ok = (); + type Error = Error; + + #[inline] + fn serialize_field(&mut self, _key: &'static str, value: &T) -> Result<()> + where + T: ?Sized + Serialize, + { + value.serialize(&mut **self) + } + + #[inline] + fn end(self) -> Result<()> { + Ok(()) + } +} diff --git a/src/varint.rs b/src/varint.rs index e50a99a..1309060 100644 --- a/src/varint.rs +++ b/src/varint.rs @@ -22,6 +22,19 @@ pub const fn max_of_last_byte() -> u8 { (1 << extra_bits) - 1 } +pub fn varint_size_usize(n: usize) -> usize { + let mut value = n; + for i in 0..varint_max::() { + if value < 128 { + return i + 1; + } + + value >>= 7; + } + debug_assert_eq!(value, 0); + varint_max::() +} + #[inline] pub fn varint_usize(n: usize, out: &mut [u8; varint_max::()]) -> &mut [u8] { let mut value = n; @@ -38,6 +51,19 @@ pub fn varint_usize(n: usize, out: &mut [u8; varint_max::()]) -> &mut [u8 &mut out[..] } +pub fn varint_size_u16(n: u16) -> usize { + let mut value = n; + for i in 0..varint_max::() { + if value < 128 { + return i + 1; + } + + value >>= 7; + } + debug_assert_eq!(value, 0); + varint_max::() +} + #[inline] pub fn varint_u16(n: u16, out: &mut [u8; varint_max::()]) -> &mut [u8] { let mut value = n; @@ -54,6 +80,19 @@ pub fn varint_u16(n: u16, out: &mut [u8; varint_max::()]) -> &mut [u8] { &mut out[..] } +pub fn varint_size_u32(n: u32) -> usize { + let mut value = n; + for i in 0..varint_max::() { + if value < 128 { + return i + 1; + } + + value >>= 7; + } + debug_assert_eq!(value, 0); + varint_max::() +} + #[inline] pub fn varint_u32(n: u32, out: &mut [u8; varint_max::()]) -> &mut [u8] { let mut value = n; @@ -70,6 +109,19 @@ pub fn varint_u32(n: u32, out: &mut [u8; varint_max::()]) -> &mut [u8] { &mut out[..] } +pub fn varint_size_u64(n: u64) -> usize { + let mut value = n; + for i in 0..varint_max::() { + if value < 128 { + return i + 1; + } + + value >>= 7; + } + debug_assert_eq!(value, 0); + varint_max::() +} + #[inline] pub fn varint_u64(n: u64, out: &mut [u8; varint_max::()]) -> &mut [u8] { let mut value = n; @@ -86,6 +138,19 @@ pub fn varint_u64(n: u64, out: &mut [u8; varint_max::()]) -> &mut [u8] { &mut out[..] } +pub fn varint_size_u128(n: u128) -> usize { + let mut value = n; + for i in 0..varint_max::() { + if value < 128 { + return i + 1; + } + + value >>= 7; + } + debug_assert_eq!(value, 0); + varint_max::() +} + #[inline] pub fn varint_u128(n: u128, out: &mut [u8; varint_max::()]) -> &mut [u8] { let mut value = n; From 159e2ec7596050f6557c48e03baffb68b5a57f0e Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Wed, 25 Jan 2023 18:40:57 +0100 Subject: [PATCH 2/8] add some doc comments --- src/ser/size.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ser/size.rs b/src/ser/size.rs index b46c8f5..b321b62 100644 --- a/src/ser/size.rs +++ b/src/ser/size.rs @@ -4,7 +4,9 @@ use crate::error::{Error, Result}; use crate::ser::serializer::{zig_zag_i128, zig_zag_i16, zig_zag_i32, zig_zag_i64}; use crate::varint::*; +/// A `Serializer` to compute the size of a serialization. pub struct Sizer { + /// The number of bytes needed in total. pub total: usize, } From 096443602d1ef077f9923f92cddaec148cd0786b Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Wed, 25 Jan 2023 18:45:19 +0100 Subject: [PATCH 3/8] cleanup api --- src/ser/mod.rs | 4 ++-- src/ser/size.rs | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/ser/mod.rs b/src/ser/mod.rs index 316073a..9bb645c 100644 --- a/src/ser/mod.rs +++ b/src/ser/mod.rs @@ -283,8 +283,8 @@ pub fn serialized_size(value: &T) -> Result where T: Serialize + ?Sized, { - let mut sizer = size::Sizer { total: 0 }; - value.serialize(&mut sizer).map(|_| sizer.total) + let mut sizer = size::Sizer::default(); + value.serialize(&mut sizer).map(|_| sizer.size()) } #[cfg(feature = "heapless")] diff --git a/src/ser/size.rs b/src/ser/size.rs index b321b62..db6e7ea 100644 --- a/src/ser/size.rs +++ b/src/ser/size.rs @@ -5,9 +5,17 @@ use crate::ser::serializer::{zig_zag_i128, zig_zag_i16, zig_zag_i32, zig_zag_i64 use crate::varint::*; /// A `Serializer` to compute the size of a serialization. +#[derive(Default)] pub struct Sizer { /// The number of bytes needed in total. - pub total: usize, + total: usize, +} + +impl Sizer { + /// The number of bytes needed to store the serializations so far. + pub fn size(&self) -> usize { + self.total + } } impl<'a> ser::Serializer for &'a mut Sizer { From 87ddf0005d704f7e6f602c85b6401ead31077a5a Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Wed, 25 Jan 2023 19:52:00 +0100 Subject: [PATCH 4/8] simplify and use a flavor --- src/ser/flavors.rs | 26 +++ src/ser/mod.rs | 4 +- src/ser/size.rs | 422 --------------------------------------------- src/varint.rs | 69 +------- 4 files changed, 29 insertions(+), 492 deletions(-) delete mode 100644 src/ser/size.rs diff --git a/src/ser/flavors.rs b/src/ser/flavors.rs index 752475d..695fcbd 100644 --- a/src/ser/flavors.rs +++ b/src/ser/flavors.rs @@ -410,3 +410,29 @@ where self.flav.finalize() } } + +/// The `Size` flavor is a measurement flavor, storing the serialized bytes length into a plain. +#[derive(Default)] +pub struct Size { + size: usize, +} + +impl Flavor for Size { + type Output = usize; + + #[inline(always)] + fn try_push(&mut self, _b: u8) -> Result<()> { + self.size += 1; + Ok(()) + } + + #[inline(always)] + fn try_extend(&mut self, b: &[u8]) -> Result<()> { + self.size += b.len(); + Ok(()) + } + + fn finalize(self) -> Result { + Ok(self.size) + } +} diff --git a/src/ser/mod.rs b/src/ser/mod.rs index 9bb645c..8c9d6b9 100644 --- a/src/ser/mod.rs +++ b/src/ser/mod.rs @@ -18,7 +18,6 @@ use crate::ser::serializer::Serializer; pub mod flavors; pub(crate) mod serializer; -mod size; /// Serialize a `T` to the given slice, with the resulting slice containing /// data in a serialized then COBS encoded format. The terminating sentinel @@ -283,8 +282,7 @@ pub fn serialized_size(value: &T) -> Result where T: Serialize + ?Sized, { - let mut sizer = size::Sizer::default(); - value.serialize(&mut sizer).map(|_| sizer.size()) + serialize_with_flavor::(value, flavors::Size::default()) } #[cfg(feature = "heapless")] diff --git a/src/ser/size.rs b/src/ser/size.rs deleted file mode 100644 index db6e7ea..0000000 --- a/src/ser/size.rs +++ /dev/null @@ -1,422 +0,0 @@ -use serde::{ser, Serialize}; - -use crate::error::{Error, Result}; -use crate::ser::serializer::{zig_zag_i128, zig_zag_i16, zig_zag_i32, zig_zag_i64}; -use crate::varint::*; - -/// A `Serializer` to compute the size of a serialization. -#[derive(Default)] -pub struct Sizer { - /// The number of bytes needed in total. - total: usize, -} - -impl Sizer { - /// The number of bytes needed to store the serializations so far. - pub fn size(&self) -> usize { - self.total - } -} - -impl<'a> ser::Serializer for &'a mut Sizer { - type Ok = (); - - type Error = Error; - - // Associated types for keeping track of additional state while serializing - // compound data structures like sequences and maps. In this case no - // additional state is required beyond what is already stored in the - // Serializer struct. - type SerializeSeq = Self; - type SerializeTuple = Self; - type SerializeTupleStruct = Self; - type SerializeTupleVariant = Self; - type SerializeMap = Self; - type SerializeStruct = Self; - type SerializeStructVariant = Self; - - #[inline] - fn is_human_readable(&self) -> bool { - false - } - - #[inline] - fn serialize_bool(self, _v: bool) -> Result<()> { - self.serialize_u8(0) - } - - #[inline] - fn serialize_i8(self, _v: i8) -> Result<()> { - self.serialize_u8(0) - } - - #[inline] - fn serialize_i16(self, v: i16) -> Result<()> { - let zzv = zig_zag_i16(v); - self.total += varint_size_u16(zzv); - Ok(()) - } - - #[inline] - fn serialize_i32(self, v: i32) -> Result<()> { - let zzv = zig_zag_i32(v); - self.total += varint_size_u32(zzv); - Ok(()) - } - - #[inline] - fn serialize_i64(self, v: i64) -> Result<()> { - let zzv = zig_zag_i64(v); - self.total += varint_size_u64(zzv); - Ok(()) - } - - #[inline] - fn serialize_i128(self, v: i128) -> Result<()> { - let zzv = zig_zag_i128(v); - self.total += varint_size_u128(zzv); - Ok(()) - } - - #[inline] - fn serialize_u8(self, _v: u8) -> Result<()> { - self.total += 1; - Ok(()) - } - - #[inline] - fn serialize_u16(self, v: u16) -> Result<()> { - self.total += varint_size_u16(v); - Ok(()) - } - - #[inline] - fn serialize_u32(self, v: u32) -> Result<()> { - self.total += varint_size_u32(v); - Ok(()) - } - - #[inline] - fn serialize_u64(self, v: u64) -> Result<()> { - self.total += varint_size_u64(v); - Ok(()) - } - - #[inline] - fn serialize_u128(self, v: u128) -> Result<()> { - self.total += varint_size_u128(v); - Ok(()) - } - - #[inline] - fn serialize_f32(self, _v: f32) -> Result<()> { - self.total += 4; - Ok(()) - } - - #[inline] - fn serialize_f64(self, _v: f64) -> Result<()> { - self.total += 8; - Ok(()) - } - - #[inline] - fn serialize_char(self, v: char) -> Result<()> { - let mut buf = [0u8; 4]; - let strsl = v.encode_utf8(&mut buf); - strsl.serialize(self) - } - - #[inline] - fn serialize_str(self, v: &str) -> Result<()> { - self.total += varint_size_usize(v.len()); - self.total += v.len(); - Ok(()) - } - - #[inline] - fn serialize_bytes(self, v: &[u8]) -> Result<()> { - self.total += varint_size_usize(v.len()); - self.total += v.len(); - Ok(()) - } - - #[inline] - fn serialize_none(self) -> Result<()> { - self.serialize_u8(0) - } - - #[inline] - fn serialize_some(self, value: &T) -> Result<()> - where - T: ?Sized + Serialize, - { - self.serialize_u8(1)?; - value.serialize(self) - } - - #[inline] - fn serialize_unit(self) -> Result<()> { - Ok(()) - } - - #[inline] - fn serialize_unit_struct(self, _name: &'static str) -> Result<()> { - Ok(()) - } - - #[inline] - fn serialize_unit_variant( - self, - _name: &'static str, - variant_index: u32, - _variant: &'static str, - ) -> Result<()> { - self.total += varint_size_u32(variant_index); - Ok(()) - } - - #[inline] - fn serialize_newtype_struct(self, _name: &'static str, value: &T) -> Result<()> - where - T: ?Sized + Serialize, - { - value.serialize(self) - } - - #[inline] - fn serialize_newtype_variant( - self, - _name: &'static str, - variant_index: u32, - _variant: &'static str, - value: &T, - ) -> Result<()> - where - T: ?Sized + Serialize, - { - self.total += varint_size_u32(variant_index); - value.serialize(self) - } - - #[inline] - fn serialize_seq(self, len: Option) -> Result { - self.total += varint_size_usize(len.ok_or(Error::SerializeSeqLengthUnknown)?); - Ok(self) - } - - #[inline] - fn serialize_tuple(self, _len: usize) -> Result { - Ok(self) - } - - #[inline] - fn serialize_tuple_struct( - self, - _name: &'static str, - _len: usize, - ) -> Result { - Ok(self) - } - - #[inline] - fn serialize_tuple_variant( - self, - _name: &'static str, - variant_index: u32, - _variant: &'static str, - _len: usize, - ) -> Result { - self.total += varint_size_u32(variant_index); - Ok(self) - } - - #[inline] - fn serialize_map(self, len: Option) -> Result { - self.total += varint_size_usize(len.ok_or(Error::SerializeSeqLengthUnknown)?); - Ok(self) - } - - #[inline] - fn serialize_struct(self, _name: &'static str, _len: usize) -> Result { - Ok(self) - } - - #[inline] - fn serialize_struct_variant( - self, - _name: &'static str, - variant_index: u32, - _variant: &'static str, - _len: usize, - ) -> Result { - self.total += varint_size_u32(variant_index); - Ok(self) - } - - #[inline] - fn collect_str(self, value: &T) -> Result - where - T: core::fmt::Display, - { - use core::fmt::Write; - - struct CountWriter { - ct: usize, - } - impl Write for CountWriter { - fn write_str(&mut self, s: &str) -> core::result::Result<(), core::fmt::Error> { - self.ct += s.as_bytes().len(); - Ok(()) - } - } - - let mut ctr = CountWriter { ct: 0 }; - - // This is the first pass through, where we just count the length of the - // data that we are given - write!(&mut ctr, "{}", value).map_err(|_| Error::CollectStrError)?; - let len = ctr.ct; - self.total += varint_size_usize(len); - - Ok(()) - } -} - -impl<'a> ser::SerializeSeq for &'a mut Sizer { - // Must match the `Ok` type of the serializer. - type Ok = (); - // Must match the `Error` type of the serializer. - type Error = Error; - - // Serialize a single element of the sequence. - #[inline] - fn serialize_element(&mut self, value: &T) -> Result<()> - where - T: ?Sized + Serialize, - { - value.serialize(&mut **self) - } - - // Close the sequence. - #[inline] - fn end(self) -> Result<()> { - Ok(()) - } -} - -impl<'a> ser::SerializeTuple for &'a mut Sizer { - type Ok = (); - type Error = Error; - - #[inline] - fn serialize_element(&mut self, value: &T) -> Result<()> - where - T: ?Sized + Serialize, - { - value.serialize(&mut **self) - } - - #[inline] - fn end(self) -> Result<()> { - Ok(()) - } -} - -impl<'a> ser::SerializeTupleStruct for &'a mut Sizer { - type Ok = (); - type Error = Error; - - #[inline] - fn serialize_field(&mut self, value: &T) -> Result<()> - where - T: ?Sized + Serialize, - { - value.serialize(&mut **self) - } - - #[inline] - fn end(self) -> Result<()> { - Ok(()) - } -} - -impl<'a> ser::SerializeTupleVariant for &'a mut Sizer { - type Ok = (); - type Error = Error; - - #[inline] - fn serialize_field(&mut self, value: &T) -> Result<()> - where - T: ?Sized + Serialize, - { - value.serialize(&mut **self) - } - - #[inline] - fn end(self) -> Result<()> { - Ok(()) - } -} - -impl<'a> ser::SerializeMap for &'a mut Sizer { - type Ok = (); - type Error = Error; - - #[inline] - fn serialize_key(&mut self, key: &T) -> Result<()> - where - T: ?Sized + Serialize, - { - key.serialize(&mut **self) - } - - #[inline] - fn serialize_value(&mut self, value: &T) -> Result<()> - where - T: ?Sized + Serialize, - { - value.serialize(&mut **self) - } - - #[inline] - fn end(self) -> Result<()> { - Ok(()) - } -} - -impl<'a> ser::SerializeStruct for &'a mut Sizer { - type Ok = (); - type Error = Error; - - #[inline] - fn serialize_field(&mut self, _key: &'static str, value: &T) -> Result<()> - where - T: ?Sized + Serialize, - { - value.serialize(&mut **self) - } - - #[inline] - fn end(self) -> Result<()> { - Ok(()) - } -} - -impl<'a> ser::SerializeStructVariant for &'a mut Sizer { - type Ok = (); - type Error = Error; - - #[inline] - fn serialize_field(&mut self, _key: &'static str, value: &T) -> Result<()> - where - T: ?Sized + Serialize, - { - value.serialize(&mut **self) - } - - #[inline] - fn end(self) -> Result<()> { - Ok(()) - } -} diff --git a/src/varint.rs b/src/varint.rs index 1309060..f63d620 100644 --- a/src/varint.rs +++ b/src/varint.rs @@ -22,19 +22,6 @@ pub const fn max_of_last_byte() -> u8 { (1 << extra_bits) - 1 } -pub fn varint_size_usize(n: usize) -> usize { - let mut value = n; - for i in 0..varint_max::() { - if value < 128 { - return i + 1; - } - - value >>= 7; - } - debug_assert_eq!(value, 0); - varint_max::() -} - #[inline] pub fn varint_usize(n: usize, out: &mut [u8; varint_max::()]) -> &mut [u8] { let mut value = n; @@ -51,19 +38,6 @@ pub fn varint_usize(n: usize, out: &mut [u8; varint_max::()]) -> &mut [u8 &mut out[..] } -pub fn varint_size_u16(n: u16) -> usize { - let mut value = n; - for i in 0..varint_max::() { - if value < 128 { - return i + 1; - } - - value >>= 7; - } - debug_assert_eq!(value, 0); - varint_max::() -} - #[inline] pub fn varint_u16(n: u16, out: &mut [u8; varint_max::()]) -> &mut [u8] { let mut value = n; @@ -80,19 +54,6 @@ pub fn varint_u16(n: u16, out: &mut [u8; varint_max::()]) -> &mut [u8] { &mut out[..] } -pub fn varint_size_u32(n: u32) -> usize { - let mut value = n; - for i in 0..varint_max::() { - if value < 128 { - return i + 1; - } - - value >>= 7; - } - debug_assert_eq!(value, 0); - varint_max::() -} - #[inline] pub fn varint_u32(n: u32, out: &mut [u8; varint_max::()]) -> &mut [u8] { let mut value = n; @@ -109,19 +70,6 @@ pub fn varint_u32(n: u32, out: &mut [u8; varint_max::()]) -> &mut [u8] { &mut out[..] } -pub fn varint_size_u64(n: u64) -> usize { - let mut value = n; - for i in 0..varint_max::() { - if value < 128 { - return i + 1; - } - - value >>= 7; - } - debug_assert_eq!(value, 0); - varint_max::() -} - #[inline] pub fn varint_u64(n: u64, out: &mut [u8; varint_max::()]) -> &mut [u8] { let mut value = n; @@ -137,21 +85,8 @@ pub fn varint_u64(n: u64, out: &mut [u8; varint_max::()]) -> &mut [u8] { debug_assert_eq!(value, 0); &mut out[..] } - -pub fn varint_size_u128(n: u128) -> usize { - let mut value = n; - for i in 0..varint_max::() { - if value < 128 { - return i + 1; - } - - value >>= 7; - } - debug_assert_eq!(value, 0); - varint_max::() -} - -#[inline] +# +[inline] pub fn varint_u128(n: u128, out: &mut [u8; varint_max::()]) -> &mut [u8] { let mut value = n; for i in 0..varint_max::() { From d4e0cd899f82ada1d2df7c18b7786a398c401972 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Wed, 25 Jan 2023 19:54:09 +0100 Subject: [PATCH 5/8] cleanup visibility --- src/lib.rs | 4 ++-- src/ser/mod.rs | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index b3cccf0..96c6422 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -57,14 +57,14 @@ pub mod experimental { /// Compile time max-serialization size calculation #[cfg(feature = "experimental-derive")] pub mod max_size { - pub use crate::ser::serialized_size; - // NOTE: This is the trait... pub use crate::max_size::MaxSize; // NOTE: ...and this is the derive macro pub use postcard_derive::MaxSize; } + pub use crate::ser::serialized_size; + /// Compile time Schema generation #[cfg(feature = "experimental-derive")] pub mod schema { diff --git a/src/ser/mod.rs b/src/ser/mod.rs index 8c9d6b9..95d330e 100644 --- a/src/ser/mod.rs +++ b/src/ser/mod.rs @@ -277,7 +277,6 @@ where } /// Compute the size of the postcard serialization of `T`. -#[cfg(any(test, feature = "experimental-derive"))] pub fn serialized_size(value: &T) -> Result where T: Serialize + ?Sized, From e2ab20bb4c67033916fe0b2e8273dc753f33814a Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Wed, 25 Jan 2023 19:55:05 +0100 Subject: [PATCH 6/8] cleanup --- src/ser/serializer.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ser/serializer.rs b/src/ser/serializer.rs index 6ac1139..fc85ffd 100644 --- a/src/ser/serializer.rs +++ b/src/ser/serializer.rs @@ -547,18 +547,18 @@ where } } -pub(crate) fn zig_zag_i16(n: i16) -> u16 { +fn zig_zag_i16(n: i16) -> u16 { ((n << 1) ^ (n >> 15)) as u16 } -pub(crate) fn zig_zag_i32(n: i32) -> u32 { +fn zig_zag_i32(n: i32) -> u32 { ((n << 1) ^ (n >> 31)) as u32 } -pub(crate) fn zig_zag_i64(n: i64) -> u64 { +fn zig_zag_i64(n: i64) -> u64 { ((n << 1) ^ (n >> 63)) as u64 } -pub(crate) fn zig_zag_i128(n: i128) -> u128 { +fn zig_zag_i128(n: i128) -> u128 { ((n << 1) ^ (n >> 127)) as u128 } From 02de03fc29ac4ac4412b56cd37a8412ebaef472d Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Wed, 25 Jan 2023 19:55:25 +0100 Subject: [PATCH 7/8] ... --- src/varint.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/varint.rs b/src/varint.rs index f63d620..e50a99a 100644 --- a/src/varint.rs +++ b/src/varint.rs @@ -85,8 +85,8 @@ pub fn varint_u64(n: u64, out: &mut [u8; varint_max::()]) -> &mut [u8] { debug_assert_eq!(value, 0); &mut out[..] } -# -[inline] + +#[inline] pub fn varint_u128(n: u128, out: &mut [u8; varint_max::()]) -> &mut [u8] { let mut value = n; for i in 0..varint_max::() { From 9573f9fdd42d45c5305692f4e5cf08cd5056a25a Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Wed, 25 Jan 2023 22:26:40 +0100 Subject: [PATCH 8/8] fixup docs --- src/ser/flavors.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/ser/flavors.rs b/src/ser/flavors.rs index 695fcbd..f82c8c0 100644 --- a/src/ser/flavors.rs +++ b/src/ser/flavors.rs @@ -411,7 +411,17 @@ where } } -/// The `Size` flavor is a measurement flavor, storing the serialized bytes length into a plain. +/// The `Size` flavor is a measurement flavor, which accumulates the number of bytes needed to +/// serialize the data. +/// +/// ``` +/// use postcard::{serialize_with_flavor, ser_flavors}; +/// +/// let value = false; +/// let size = serialize_with_flavor(&value, ser_flavors::Size::default()).unwrap(); +/// +/// assert_eq!(size, 1); +/// ``` #[derive(Default)] pub struct Size { size: usize,