From 503f16fe8c57d4bc2446f4c4ef0349e44acd4991 Mon Sep 17 00:00:00 2001 From: max143672 Date: Tue, 8 Aug 2023 22:14:33 +0400 Subject: [PATCH 1/9] Add serde_bytes module for handling byte arrays This commit introduces a new module called `serde_bytes`. The new module contains two inner modules: `ser` and `de`. These are for Serde's `Serialize` and `Deserialize`traits respectively. The `ser` module has a `serialize` function for human-readable and non-human-readable versions of byte arrays. The `de` module essentially does the opposite and contains a function to deserialize the byte arrays. They both utilize faster_hex library for efficient conversion. This was done to abstract away byte array serialization/deserialization logic to a separate unit of the project, enhancing modularity and code organization. --- utils/src/lib.rs | 1 + utils/src/serde_bytes/de.rs | 24 ++++++++++++++++++++++++ utils/src/serde_bytes/mod.rs | 20 ++++++++++++++++++++ utils/src/serde_bytes/ser.rs | 24 ++++++++++++++++++++++++ 4 files changed, 69 insertions(+) create mode 100644 utils/src/serde_bytes/de.rs create mode 100644 utils/src/serde_bytes/mod.rs create mode 100644 utils/src/serde_bytes/ser.rs diff --git a/utils/src/lib.rs b/utils/src/lib.rs index fa41e1535..895bc79d5 100644 --- a/utils/src/lib.rs +++ b/utils/src/lib.rs @@ -8,6 +8,7 @@ pub mod iter; pub mod networking; pub mod option; pub mod refs; +pub mod serde_bytes; pub mod sim; pub mod sync; pub mod triggers; diff --git a/utils/src/serde_bytes/de.rs b/utils/src/serde_bytes/de.rs new file mode 100644 index 000000000..957b34b61 --- /dev/null +++ b/utils/src/serde_bytes/de.rs @@ -0,0 +1,24 @@ +use crate::hex::FromHex; +use serde::Deserializer; + +pub trait Deserialize<'de>: Sized + FromHex + From<&'de [u8]> { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>; +} + +impl<'de, T: FromHex + From<&'de [u8]>> Deserialize<'de> for T { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + if deserializer.is_human_readable() { + let s: &str = serde::Deserialize::deserialize(deserializer)?; + Ok(T::from_hex(s).map_err(serde::de::Error::custom)?) + } else { + // serde::Deserialize for &[u8] is already optimized, so simply forward to that. + let bytes: &[u8] = serde::Deserialize::deserialize(deserializer)?; + Ok(T::from(bytes)) + } + } +} diff --git a/utils/src/serde_bytes/mod.rs b/utils/src/serde_bytes/mod.rs new file mode 100644 index 000000000..fa20694df --- /dev/null +++ b/utils/src/serde_bytes/mod.rs @@ -0,0 +1,20 @@ +mod de; +mod ser; + +pub use crate::serde_bytes::de::Deserialize; +pub use crate::serde_bytes::ser::Serialize; + +pub fn serialize(bytes: &T, serializer: S) -> Result +where + T: ?Sized + Serialize, + S: serde::Serializer, +{ + Serialize::serialize(bytes, serializer) +} +pub fn deserialize<'de, T, D>(deserializer: D) -> Result +where + T: Deserialize<'de>, + D: serde::Deserializer<'de>, +{ + Deserialize::deserialize(deserializer) +} diff --git a/utils/src/serde_bytes/ser.rs b/utils/src/serde_bytes/ser.rs new file mode 100644 index 000000000..67e28a54e --- /dev/null +++ b/utils/src/serde_bytes/ser.rs @@ -0,0 +1,24 @@ +use serde::Serializer; +use std::str::{self}; + +pub trait Serialize { + #[allow(missing_docs)] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer; +} + +impl> Serialize for T { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + if serializer.is_human_readable() { + let mut hex = vec![0u8; self.as_ref().len() * 2]; + faster_hex::hex_encode(&self.as_ref(), &mut hex[..]).map_err(serde::ser::Error::custom)?; + serializer.serialize_str(unsafe { str::from_utf8_unchecked(&hex) }) + } else { + serializer.serialize_bytes(self.as_ref()) + } + } +} From 0eeb865765a7940dd3d967ce4091d750a042897b Mon Sep 17 00:00:00 2001 From: max143672 Date: Tue, 8 Aug 2023 23:01:04 +0400 Subject: [PATCH 2/9] Improve error handling in serde_bytes implementation The trait Deserialize in utils/src/serde_bytes/de.rs has been updated to use TryFrom rather than From. The implementation has been updated accordingly. --- utils/src/serde_bytes/de.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/utils/src/serde_bytes/de.rs b/utils/src/serde_bytes/de.rs index 957b34b61..7643414e7 100644 --- a/utils/src/serde_bytes/de.rs +++ b/utils/src/serde_bytes/de.rs @@ -1,13 +1,17 @@ use crate::hex::FromHex; use serde::Deserializer; +use std::fmt::Display; -pub trait Deserialize<'de>: Sized + FromHex + From<&'de [u8]> { +pub trait Deserialize<'de>: Sized + FromHex + TryFrom<&'de [u8]> { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>; } -impl<'de, T: FromHex + From<&'de [u8]>> Deserialize<'de> for T { +impl<'de, T: FromHex + TryFrom<&'de [u8]>> Deserialize<'de> for T +where + >::Error: Display, +{ fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, @@ -17,8 +21,7 @@ impl<'de, T: FromHex + From<&'de [u8]>> Deserialize<'de> for T { Ok(T::from_hex(s).map_err(serde::de::Error::custom)?) } else { // serde::Deserialize for &[u8] is already optimized, so simply forward to that. - let bytes: &[u8] = serde::Deserialize::deserialize(deserializer)?; - Ok(T::from(bytes)) + serde::Deserialize::deserialize(deserializer).and_then(|bts| T::try_from(bts).map_err(serde::de::Error::custom)) } } } From d6ba2a3210aa700665c9971cdd72a814d593e552 Mon Sep 17 00:00:00 2001 From: max143672 Date: Tue, 8 Aug 2023 23:11:20 +0400 Subject: [PATCH 3/9] Update serde serialization and remove unnecessary dependencies use utils::serde_bytes instead of external crate serde_bytes --- Cargo.lock | 10 ---------- consensus/core/Cargo.toml | 2 +- consensus/core/src/tx.rs | 7 ++++++- crypto/hashes/src/lib.rs | 8 ++++++++ 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 01f864a4c..60806ed73 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2113,7 +2113,6 @@ dependencies = [ "secp256k1", "serde", "serde-wasm-bindgen 0.4.5", - "serde_bytes", "serde_json", "smallvec", "thiserror", @@ -4424,15 +4423,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "serde_bytes" -version = "0.11.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab33ec92f677585af6d88c65593ae2375adde54efdbf16d597f2cbc7a6d368ff" -dependencies = [ - "serde", -] - [[package]] name = "serde_derive" version = "1.0.177" diff --git a/consensus/core/Cargo.toml b/consensus/core/Cargo.toml index ad4b07a08..efc16331f 100644 --- a/consensus/core/Cargo.toml +++ b/consensus/core/Cargo.toml @@ -35,7 +35,7 @@ workflow-wasm.workspace = true rand.workspace = true workflow-log.workspace = true workflow-core.workspace = true -serde_bytes.workspace = true +#serde_bytes.workspace = true # secp256k1 = { version = "0.24", features = ["global-context", "rand-std", "serde"] } # secp256k1 = { version = "0.24", features = ["global-context", "rand-std"] } diff --git a/consensus/core/src/tx.rs b/consensus/core/src/tx.rs index ec1f75f02..8e2f975ad 100644 --- a/consensus/core/src/tx.rs +++ b/consensus/core/src/tx.rs @@ -1,5 +1,7 @@ use borsh::{BorshDeserialize, BorshSchema, BorshSerialize}; -use kaspa_utils::hex::*; +use kaspa_utils::{hex::*, serde_bytes}; +// use kaspa_utils::hex::*; + use serde::{Deserialize, Deserializer, Serialize, Serializer}; use smallvec::SmallVec; use std::{ @@ -200,6 +202,7 @@ pub type TransactionIndexType = u32; #[derive(Eq, Hash, PartialEq, Debug, Copy, Clone, Serialize, Deserialize, BorshSerialize, BorshDeserialize, BorshSchema)] #[serde(rename_all = "camelCase")] pub struct TransactionOutpoint { + #[serde(with = "serde_bytes")] pub transaction_id: TransactionId, pub index: TransactionIndexType, } @@ -255,6 +258,7 @@ pub struct Transaction { pub inputs: Vec, pub outputs: Vec, pub lock_time: u64, + #[serde(with = "serde_bytes")] pub subnetwork_id: SubnetworkId, pub gas: u64, #[serde(with = "serde_bytes")] @@ -262,6 +266,7 @@ pub struct Transaction { // A field that is used to cache the transaction ID. // Always use the corresponding self.id() instead of accessing this field directly + #[serde(with = "serde_bytes")] // todo should it be in serde outcome? id: TransactionId, } diff --git a/crypto/hashes/src/lib.rs b/crypto/hashes/src/lib.rs index 437b0d176..0ac2b3e36 100644 --- a/crypto/hashes/src/lib.rs +++ b/crypto/hashes/src/lib.rs @@ -27,6 +27,14 @@ pub use hashers::*; #[wasm_bindgen] pub struct Hash([u8; HASH_SIZE]); +impl TryFrom<&[u8]> for Hash { + type Error = TryFromSliceError; + + fn try_from(value: &[u8]) -> Result { + Hash::try_from_slice(value) + } +} + impl Hash { #[inline(always)] pub const fn from_bytes(bytes: [u8; HASH_SIZE]) -> Self { From 27b8e1a0094bd76e34b12f31652fe8fc416e11e1 Mon Sep 17 00:00:00 2001 From: max143672 Date: Wed, 9 Aug 2023 18:32:17 +0400 Subject: [PATCH 4/9] tmp --- consensus/core/Cargo.toml | 1 - consensus/core/src/subnets.rs | 6 +++ consensus/core/src/tx.rs | 59 ++++++++++++++++++++++++++---- utils/src/lib.rs | 1 + utils/src/serde_bytes_fixed/de.rs | 27 ++++++++++++++ utils/src/serde_bytes_fixed/mod.rs | 20 ++++++++++ utils/src/serde_bytes_fixed/ser.rs | 28 ++++++++++++++ 7 files changed, 133 insertions(+), 9 deletions(-) create mode 100644 utils/src/serde_bytes_fixed/de.rs create mode 100644 utils/src/serde_bytes_fixed/mod.rs create mode 100644 utils/src/serde_bytes_fixed/ser.rs diff --git a/consensus/core/Cargo.toml b/consensus/core/Cargo.toml index efc16331f..0c6d0dd2c 100644 --- a/consensus/core/Cargo.toml +++ b/consensus/core/Cargo.toml @@ -35,7 +35,6 @@ workflow-wasm.workspace = true rand.workspace = true workflow-log.workspace = true workflow-core.workspace = true -#serde_bytes.workspace = true # secp256k1 = { version = "0.24", features = ["global-context", "rand-std", "serde"] } # secp256k1 = { version = "0.24", features = ["global-context", "rand-std"] } diff --git a/consensus/core/src/subnets.rs b/consensus/core/src/subnets.rs index 5a6f0fb0d..69260f2e0 100644 --- a/consensus/core/src/subnets.rs +++ b/consensus/core/src/subnets.rs @@ -14,6 +14,12 @@ pub const SUBNETWORK_ID_SIZE: usize = 20; )] pub struct SubnetworkId([u8; SUBNETWORK_ID_SIZE]); +impl AsRef<[u8; 20]> for SubnetworkId { + fn as_ref(&self) -> &[u8; 20] { + &self.0 + } +} + impl AsRef<[u8]> for SubnetworkId { fn as_ref(&self) -> &[u8] { &self.0 diff --git a/consensus/core/src/tx.rs b/consensus/core/src/tx.rs index 8e2f975ad..02551b021 100644 --- a/consensus/core/src/tx.rs +++ b/consensus/core/src/tx.rs @@ -1,6 +1,5 @@ use borsh::{BorshDeserialize, BorshSchema, BorshSerialize}; -use kaspa_utils::{hex::*, serde_bytes}; -// use kaspa_utils::hex::*; +use kaspa_utils::{hex::*, serde_bytes, serde_bytes_fixed}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use smallvec::SmallVec; @@ -202,7 +201,6 @@ pub type TransactionIndexType = u32; #[derive(Eq, Hash, PartialEq, Debug, Copy, Clone, Serialize, Deserialize, BorshSerialize, BorshDeserialize, BorshSchema)] #[serde(rename_all = "camelCase")] pub struct TransactionOutpoint { - #[serde(with = "serde_bytes")] pub transaction_id: TransactionId, pub index: TransactionIndexType, } @@ -258,7 +256,7 @@ pub struct Transaction { pub inputs: Vec, pub outputs: Vec, pub lock_time: u64, - #[serde(with = "serde_bytes")] + #[serde(with = "serde_bytes_fixed")] pub subnetwork_id: SubnetworkId, pub gas: u64, #[serde(with = "serde_bytes")] @@ -266,7 +264,6 @@ pub struct Transaction { // A field that is used to cache the transaction ID. // Always use the corresponding self.id() instead of accessing this field directly - #[serde(with = "serde_bytes")] // todo should it be in serde outcome? id: TransactionId, } @@ -525,7 +522,7 @@ pub type SignableTransaction = MutableTransaction; #[cfg(test)] mod tests { use super::*; - use crate::subnets::SUBNETWORK_ID_COINBASE; + use consensus_core::subnets::SUBNETWORK_ID_COINBASE; use smallvec::smallvec; fn test_transaction() -> Transaction { @@ -594,7 +591,7 @@ mod tests { let bts = bincode::serialize(&tx).unwrap(); // standard, based on https://github.com/kaspanet/rusty-kaspa/commit/7e947a06d2434daf4bc7064d4cd87dc1984b56fe - let kos_bytes = vec![ + let expected_bts = vec![ 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 22, 94, 56, 232, 179, 145, 69, 149, 217, 198, 65, 243, 184, 238, 194, 243, 70, 17, 137, 107, 130, 26, 104, 59, 122, 78, 222, 254, 44, 0, 0, 0, 250, 255, 255, 255, 32, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 2, 0, 0, 0, 0, 0, 0, 0, 3, 75, @@ -612,10 +609,56 @@ mod tests { 64, 98, 49, 45, 0, 77, 32, 25, 122, 77, 15, 211, 252, 61, 210, 82, 177, 39, 153, 127, 33, 188, 172, 138, 38, 67, 75, 241, 176, ]; - assert_eq!(kos_bytes, bts); + assert_eq!(expected_bts, bts); assert_eq!(tx, bincode::deserialize(&bts).unwrap()); } + #[test] + fn test_transaction_json() { + let tx = test_transaction(); + let str = serde_json::to_string_pretty(&tx).unwrap(); + let expected_str = r#"{ + "version": 1, + "inputs": [ + { + "previousOutpoint": { + "transactionId": "165e38e8b3914595d9c641f3b8eec2f34611896b821a683b7a4edefe2c000000", + "index": 4294967290 + }, + "signatureScript": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "sequence": 2, + "sigOpCount": 3 + }, + { + "previousOutpoint": { + "transactionId": "4bb07535dfd58e0b3cd64fd7155280872a0471bcf83095526ace0e38c6000000", + "index": 4294967291 + }, + "signatureScript": "202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f", + "sequence": 4, + "sigOpCount": 5 + } + ], + "outputs": [ + { + "value": 6, + "scriptPublicKey": "000076a921032f7e430aa4c9d159437e84b975dc76d9003bf0922cf3aa4528464bab780dba5e" + }, + { + "value": 7, + "scriptPublicKey": "000076a921032f7e430aa4c9d159437e84b975dc76d9003bf0922cf3aa4528464bab780dba5e" + } + ], + "lockTime": 8, + "subnetworkId": "0100000000000000000000000000000000000000", + "gas": 9, + "payload": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f60616263", + "id": "4592c14062312d004d20197a4d0fd3fc3dd252b127997f21bcac8a26434bf1b0" +}"#; + assert_eq!(expected_str, str); + assert_eq!(tx, serde_json::from_str(&str).unwrap()); + } + #[test] fn test_spk_serde_json() { let vec = (0..SCRIPT_VECTOR_SIZE as u8).collect::>(); diff --git a/utils/src/lib.rs b/utils/src/lib.rs index 895bc79d5..1e09e77c7 100644 --- a/utils/src/lib.rs +++ b/utils/src/lib.rs @@ -9,6 +9,7 @@ pub mod networking; pub mod option; pub mod refs; pub mod serde_bytes; +pub mod serde_bytes_fixed; pub mod sim; pub mod sync; pub mod triggers; diff --git a/utils/src/serde_bytes_fixed/de.rs b/utils/src/serde_bytes_fixed/de.rs new file mode 100644 index 000000000..7643414e7 --- /dev/null +++ b/utils/src/serde_bytes_fixed/de.rs @@ -0,0 +1,27 @@ +use crate::hex::FromHex; +use serde::Deserializer; +use std::fmt::Display; + +pub trait Deserialize<'de>: Sized + FromHex + TryFrom<&'de [u8]> { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>; +} + +impl<'de, T: FromHex + TryFrom<&'de [u8]>> Deserialize<'de> for T +where + >::Error: Display, +{ + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + if deserializer.is_human_readable() { + let s: &str = serde::Deserialize::deserialize(deserializer)?; + Ok(T::from_hex(s).map_err(serde::de::Error::custom)?) + } else { + // serde::Deserialize for &[u8] is already optimized, so simply forward to that. + serde::Deserialize::deserialize(deserializer).and_then(|bts| T::try_from(bts).map_err(serde::de::Error::custom)) + } + } +} diff --git a/utils/src/serde_bytes_fixed/mod.rs b/utils/src/serde_bytes_fixed/mod.rs new file mode 100644 index 000000000..cfed898e6 --- /dev/null +++ b/utils/src/serde_bytes_fixed/mod.rs @@ -0,0 +1,20 @@ +mod de; +mod ser; + +pub use crate::serde_bytes_fixed::de::Deserialize; +pub use crate::serde_bytes_fixed::ser::Serialize; + +pub fn serialize(bytes: &T, serializer: S) -> Result +where + T: ?Sized + Serialize, + S: serde::Serializer, +{ + Serialize::serialize(bytes, serializer) +} +pub fn deserialize<'de, T, D>(deserializer: D) -> Result +where + T: Deserialize<'de>, + D: serde::Deserializer<'de>, +{ + Deserialize::deserialize(deserializer) +} diff --git a/utils/src/serde_bytes_fixed/ser.rs b/utils/src/serde_bytes_fixed/ser.rs new file mode 100644 index 000000000..cde223677 --- /dev/null +++ b/utils/src/serde_bytes_fixed/ser.rs @@ -0,0 +1,28 @@ +use serde::Serializer; +use std::str::{self}; +use serde::ser::SerializeTuple; + +pub trait Serialize { + #[allow(missing_docs)] + fn serialize(&self, serializer: S) -> Result + where + S: Serializer; +} + +impl> Serialize for T { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + if serializer.is_human_readable() { + let mut hex = vec![0u8; self.as_ref().len() * 2]; + faster_hex::hex_encode(self.as_ref(), &mut hex[..]).map_err(serde::ser::Error::custom)?; + serializer.serialize_str(unsafe { str::from_utf8_unchecked(&hex) }) + } else { + let t = serializer.serialize_tuple(self.as_ref().len())?; + + t.serialize_element() + serializer.serialize_bytes(self.as_ref()) + } + } +} From 2d8d84b606383af6172b80e17ae65baabd683053 Mon Sep 17 00:00:00 2001 From: max143672 Date: Thu, 10 Aug 2023 12:10:06 +0400 Subject: [PATCH 5/9] "Implement improved serialization and deserialization for fixed-size bytes" This update implements a more efficient and concise way to serialize and deserialize fixed-size bytes. It replaces previous code with a new approach that leverages macros and generic traits. The changes were made across multiple files including "subnets.rs", "tx.rs" and several in the "serde_bytes_fixed" directory. The new approach uses the macro "serde_impl_ser_fixed_bytes" and the trait "serialize", which together streamline the process of serialization. For deserialization, the "deserialize" trait and the macro "serde_impl_deser_fixed_bytes" were used. These new implementations are designed to support both human-readable and binary formats. This allows for more flexibility and efficiency in handling different types of data. In addition, they improve code coherence and maintainability, making the codebase easier to understand and modify in the future. --- consensus/core/src/subnets.rs | 15 ++- consensus/core/src/tx.rs | 2 + crypto/hashes/src/lib.rs | 118 +++-------------- utils/src/serde_bytes_fixed/de.rs | 200 ++++++++++++++++++++++++++--- utils/src/serde_bytes_fixed/mod.rs | 32 ++++- utils/src/serde_bytes_fixed/ser.rs | 46 ++++++- 6 files changed, 278 insertions(+), 135 deletions(-) diff --git a/consensus/core/src/subnets.rs b/consensus/core/src/subnets.rs index 69260f2e0..95542283f 100644 --- a/consensus/core/src/subnets.rs +++ b/consensus/core/src/subnets.rs @@ -3,19 +3,16 @@ use std::str::{self, FromStr}; use borsh::{BorshDeserialize, BorshSchema, BorshSerialize}; use kaspa_utils::hex::{FromHex, ToHex}; -use serde::{Deserialize, Serialize}; /// The size of the array used to store subnetwork IDs. pub const SUBNETWORK_ID_SIZE: usize = 20; /// The domain representation of a Subnetwork ID -#[derive( - Debug, Clone, Default, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize, BorshSerialize, BorshDeserialize, BorshSchema, -)] +#[derive(Debug, Clone, Default, Eq, PartialEq, Ord, PartialOrd, Hash, BorshSerialize, BorshDeserialize, BorshSchema)] pub struct SubnetworkId([u8; SUBNETWORK_ID_SIZE]); -impl AsRef<[u8; 20]> for SubnetworkId { - fn as_ref(&self) -> &[u8; 20] { +impl AsRef<[u8; SUBNETWORK_ID_SIZE]> for SubnetworkId { + fn as_ref(&self) -> &[u8; SUBNETWORK_ID_SIZE] { &self.0 } } @@ -26,6 +23,12 @@ impl AsRef<[u8]> for SubnetworkId { } } +impl From<[u8; SUBNETWORK_ID_SIZE]> for SubnetworkId { + fn from(value: [u8; SUBNETWORK_ID_SIZE]) -> Self { + Self::from_bytes(value) + } +} + impl SubnetworkId { pub const fn from_byte(b: u8) -> SubnetworkId { let mut bytes = [0u8; SUBNETWORK_ID_SIZE]; diff --git a/consensus/core/src/tx.rs b/consensus/core/src/tx.rs index 02551b021..81960421c 100644 --- a/consensus/core/src/tx.rs +++ b/consensus/core/src/tx.rs @@ -201,6 +201,7 @@ pub type TransactionIndexType = u32; #[derive(Eq, Hash, PartialEq, Debug, Copy, Clone, Serialize, Deserialize, BorshSerialize, BorshDeserialize, BorshSchema)] #[serde(rename_all = "camelCase")] pub struct TransactionOutpoint { + #[serde(with = "serde_bytes_fixed")] pub transaction_id: TransactionId, pub index: TransactionIndexType, } @@ -264,6 +265,7 @@ pub struct Transaction { // A field that is used to cache the transaction ID. // Always use the corresponding self.id() instead of accessing this field directly + #[serde(with = "serde_bytes_fixed")] id: TransactionId, } diff --git a/crypto/hashes/src/lib.rs b/crypto/hashes/src/lib.rs index 0ac2b3e36..46adcc375 100644 --- a/crypto/hashes/src/lib.rs +++ b/crypto/hashes/src/lib.rs @@ -2,13 +2,10 @@ mod hashers; mod pow_hashers; use borsh::{BorshDeserialize, BorshSchema, BorshSerialize}; -use kaspa_utils::hex::{FromHex, ToHex}; -use serde::{ - de::Error as DeserializeError, - de::{SeqAccess, Visitor}, - Deserialize, Deserializer, Serialize, Serializer, +use kaspa_utils::{ + hex::{FromHex, ToHex}, + serde_impl_deser_fixed_bytes, serde_impl_ser_fixed_bytes, }; -use std::marker::PhantomData; use std::{ array::TryFromSliceError, fmt::{Debug, Display, Formatter}, @@ -27,6 +24,15 @@ pub use hashers::*; #[wasm_bindgen] pub struct Hash([u8; HASH_SIZE]); +serde_impl_ser_fixed_bytes!(Hash, HASH_SIZE); +serde_impl_deser_fixed_bytes!(Hash, HASH_SIZE); + +impl From<[u8; HASH_SIZE]> for Hash { + fn from(value: [u8; HASH_SIZE]) -> Self { + Hash(value) + } +} + impl TryFrom<&[u8]> for Hash { type Error = TryFromSliceError; @@ -134,6 +140,13 @@ impl From for Hash { } } +impl AsRef<[u8; HASH_SIZE]> for Hash { + #[inline(always)] + fn as_ref(&self) -> &[u8; HASH_SIZE] { + &self.0 + } +} + impl AsRef<[u8]> for Hash { #[inline(always)] fn as_ref(&self) -> &[u8] { @@ -153,99 +166,6 @@ impl FromHex for Hash { Self::from_str(hex_str) } } -impl Serialize for Hash { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - if serializer.is_human_readable() { - let mut hex = [0u8; HASH_SIZE * 2]; - faster_hex::hex_encode(&self.0, &mut hex).expect("The output is exactly twice the size of the input"); - serializer.serialize_str(str::from_utf8(&hex).expect("hex is always valid UTF-8")) - } else { - serializer.serialize_newtype_struct("Hash", &self.0) - } - } -} - -struct HashVisitor<'de> { - marker: PhantomData, - lifetime: PhantomData<&'de ()>, -} -impl<'de> Visitor<'de> for HashVisitor<'de> { - type Value = Hash; - - fn expecting(&self, formatter: &mut Formatter) -> std::fmt::Result { - formatter.write_str("tuple struct Hash") - } - #[inline] - fn visit_str(self, v: &str) -> Result - where - E: DeserializeError, - { - Hash::try_from_slice(v.as_bytes()).map_err(serde::de::Error::custom) - } - #[inline] - fn visit_borrowed_str(self, v: &'de str) -> Result - where - E: DeserializeError, - { - Hash::try_from_slice(v.as_bytes()).map_err(serde::de::Error::custom) - } - #[inline] - fn visit_bytes(self, v: &[u8]) -> Result - where - E: DeserializeError, - { - Hash::try_from_slice(v).map_err(serde::de::Error::custom) - } - #[inline] - fn visit_borrowed_bytes(self, v: &'de [u8]) -> Result - where - E: DeserializeError, - { - Hash::try_from_slice(v).map_err(serde::de::Error::custom) - } - #[inline] - fn visit_byte_buf(self, v: Vec) -> Result - where - E: DeserializeError, - { - Hash::try_from_slice(&v).map_err(serde::de::Error::custom) - } - #[inline] - fn visit_newtype_struct(self, deserializer: D) -> Result - where - D: Deserializer<'de>, - { - <[u8; HASH_SIZE] as serde::Deserialize>::deserialize(deserializer).map(Hash) - } - #[inline] - fn visit_seq(self, mut seq: A) -> Result - where - A: SeqAccess<'de>, - { - let Some(value) = seq.next_element()? else { - return Err(serde::de::Error::invalid_length(0usize, &"tuple struct Hash with 1 element")); - } ; - Ok(Hash(value)) - } -} -// _serde::Deserializer::deserialize_newtype_struct(__deserializer, "Hash", __Visitor { marker: _serde::__private::PhantomData::, lifetime: _serde::__private::PhantomData }) - -impl<'de> Deserialize<'de> for Hash { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - if deserializer.is_human_readable() { - let s = ::deserialize(deserializer)?; - FromStr::from_str(&s).map_err(serde::de::Error::custom) - } else { - deserializer.deserialize_newtype_struct("Hash", HashVisitor { marker: Default::default(), lifetime: Default::default() }) - } - } -} #[wasm_bindgen] impl Hash { diff --git a/utils/src/serde_bytes_fixed/de.rs b/utils/src/serde_bytes_fixed/de.rs index 7643414e7..d81679195 100644 --- a/utils/src/serde_bytes_fixed/de.rs +++ b/utils/src/serde_bytes_fixed/de.rs @@ -1,27 +1,185 @@ -use crate::hex::FromHex; -use serde::Deserializer; -use std::fmt::Display; - -pub trait Deserialize<'de>: Sized + FromHex + TryFrom<&'de [u8]> { +/// Trait for deserialization of fixed-size byte arrays, newtype structs, +/// or any other custom types that can implement `From<[u8; N]>`. +/// Implementers should also provide implementation for `crate::hex::FromHex`. +pub trait Deserialize<'de, const N: usize>: Sized + crate::hex::FromHex + From<[u8; N]> { + /// Deserialize given `deserializer` into `Self`. fn deserialize(deserializer: D) -> Result where - D: Deserializer<'de>; + D: serde::Deserializer<'de>; } -impl<'de, T: FromHex + TryFrom<&'de [u8]>> Deserialize<'de> for T -where - >::Error: Display, -{ - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - if deserializer.is_human_readable() { - let s: &str = serde::Deserialize::deserialize(deserializer)?; - Ok(T::from_hex(s).map_err(serde::de::Error::custom)?) - } else { - // serde::Deserialize for &[u8] is already optimized, so simply forward to that. - serde::Deserialize::deserialize(deserializer).and_then(|bts| T::try_from(bts).map_err(serde::de::Error::custom)) +#[macro_export] +/// Macro to generate impl of `Deserialize` for types `T`, which are +/// capable of being constructed from byte arrays of fixed size, +/// or newtype structs or other types based on [`From<[u8; $size]>`]. +macro_rules! deser_fixed_bytes { + ($size: expr) => { + impl<'de, T: $crate::hex::FromHex + From<[u8; $size]>> Deserialize<'de, $size> for T { + /// Deserialization function for types `T` + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + struct FixedBytesVisitor<'de> { + marker: std::marker::PhantomData<[u8; $size]>, + lifetime: std::marker::PhantomData<&'de ()>, + } + impl<'de> serde::de::Visitor<'de> for FixedBytesVisitor<'de> { + type Value = [u8; $size]; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(formatter, "an byte array of size {}", $size) + } + #[inline] + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + let v: [u8; $size] = v.as_bytes().try_into().map_err(serde::de::Error::custom)?; + Ok(Self::Value::from(v)) + } + #[inline] + fn visit_borrowed_str(self, v: &'de str) -> Result + where + E: serde::de::Error, + { + let v: [u8; $size] = v.as_bytes().try_into().map_err(serde::de::Error::custom)?; + Ok(Self::Value::from(v)) + } + #[inline] + fn visit_bytes(self, v: &[u8]) -> Result + where + E: serde::de::Error, + { + let v: [u8; $size] = v.try_into().map_err(serde::de::Error::custom)?; + Ok(Self::Value::from(v)) + } + #[inline] + fn visit_borrowed_bytes(self, v: &'de [u8]) -> Result + where + E: serde::de::Error, + { + let v: [u8; $size] = v.try_into().map_err(serde::de::Error::custom)?; + Ok(Self::Value::from(v)) + } + + #[inline] + fn visit_newtype_struct(self, deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + <[u8; $size] as serde::Deserialize>::deserialize(deserializer).map(|v| Self::Value::from(v)) + } + #[inline] + fn visit_seq(self, mut seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + let Some(value): Option<[u8; $size]> = seq.next_element()? else { + return Err(serde::de::Error::invalid_length(0usize, &"tuple struct fixed array with 1 element")); + } ; + Ok(Self::Value::from(value)) + } + } + + if deserializer.is_human_readable() { + let s: &str = serde::Deserialize::deserialize(deserializer)?; + Ok(T::from_hex(s).map_err(serde::de::Error::custom)?) + } else { + deserializer + .deserialize_tuple($size, FixedBytesVisitor { marker: Default::default(), lifetime: Default::default() }) + .map(Into::into) + } + } + } + }; +} + +// Calling the macro to generate `Deserialize` implementation for byte arrays of size 20 and 32, +// and for newtype structs or other types extendable from these fixed sized arrays. +deser_fixed_bytes!(20); +deser_fixed_bytes!(32); + +#[macro_export] +/// Macro to provide serde::Deserialize implementations for types `$t` +/// which can be constructed from byte arrays of fixed size. +/// The resulting structure will support deserialization from human-readable +/// formats using hex::FromHex, as well as binary formats. +macro_rules! serde_impl_deser_fixed_bytes { + ($t: ty, $size: expr) => { + impl<'de> serde::Deserialize<'de> for $t { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + struct MyVisitor<'de> { + marker: std::marker::PhantomData<$t>, + lifetime: std::marker::PhantomData<&'de ()>, + } + impl<'de> serde::de::Visitor<'de> for MyVisitor<'de> { + type Value = $t; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(formatter, "an byte array of size {} to stringify!($i)", $size) + } + #[inline] + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + let v: [u8; $size] = v.as_bytes().try_into().map_err(serde::de::Error::custom)?; + Ok(Self::Value::from(v)) + } + #[inline] + fn visit_borrowed_str(self, v: &'de str) -> Result + where + E: serde::de::Error, + { + let v: [u8; $size] = v.as_bytes().try_into().map_err(serde::de::Error::custom)?; + Ok(Self::Value::from(v)) + } + #[inline] + fn visit_bytes(self, v: &[u8]) -> Result + where + E: serde::de::Error, + { + let v: [u8; $size] = v.try_into().map_err(serde::de::Error::custom)?; + Ok(Self::Value::from(v)) + } + #[inline] + fn visit_borrowed_bytes(self, v: &'de [u8]) -> Result + where + E: serde::de::Error, + { + let v: [u8; $size] = v.try_into().map_err(serde::de::Error::custom)?; + Ok(Self::Value::from(v)) + } + + #[inline] + fn visit_newtype_struct(self, deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + <[u8; $size] as serde::Deserialize>::deserialize(deserializer).map(|v| Self::Value::from(v)) + } + #[inline] + fn visit_seq(self, mut seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + let Some(value): Option<[u8; $size]> = seq.next_element()? else { + return Err(serde::de::Error::invalid_length(0usize, &"tuple struct fixed array with 1 element")); + } ; + Ok(Self::Value::from(value)) + } + } + if deserializer.is_human_readable() { + let s: &str = serde::Deserialize::deserialize(deserializer)?; + Ok($crate::hex::FromHex::from_hex(s).map_err(serde::de::Error::custom)?) + } else { + deserializer.deserialize_newtype_struct(stringify!($i), MyVisitor { marker: Default::default(), lifetime: Default::default() }) + } + } } - } + }; } diff --git a/utils/src/serde_bytes_fixed/mod.rs b/utils/src/serde_bytes_fixed/mod.rs index cfed898e6..51a56a4aa 100644 --- a/utils/src/serde_bytes_fixed/mod.rs +++ b/utils/src/serde_bytes_fixed/mod.rs @@ -3,6 +3,7 @@ mod ser; pub use crate::serde_bytes_fixed::de::Deserialize; pub use crate::serde_bytes_fixed::ser::Serialize; +use crate::{serde_impl_deser_fixed_bytes, serde_impl_ser_fixed_bytes}; pub fn serialize(bytes: &T, serializer: S) -> Result where @@ -11,10 +12,37 @@ where { Serialize::serialize(bytes, serializer) } -pub fn deserialize<'de, T, D>(deserializer: D) -> Result + +pub fn deserialize<'de, T, D, const N: usize>(deserializer: D) -> Result where - T: Deserialize<'de>, + T: Deserialize<'de, N>, D: serde::Deserializer<'de>, { Deserialize::deserialize(deserializer) } + +struct MyBox([u8; 20]); + +impl AsRef<[u8; 20]> for MyBox { + fn as_ref(&self) -> &[u8; 20] { + &self.0 + } +} + +impl crate::hex::FromHex for MyBox { + type Error = faster_hex::Error; + fn from_hex(hex_str: &str) -> Result { + let mut bytes = [0u8; 20]; + faster_hex::hex_decode(hex_str.as_bytes(), &mut bytes)?; + Ok(MyBox(bytes)) + } +} + +impl From<[u8; 20]> for MyBox { + fn from(value: [u8; 20]) -> Self { + MyBox(value) + } +} + +serde_impl_ser_fixed_bytes!(MyBox, 20); +serde_impl_deser_fixed_bytes!(MyBox, 20); diff --git a/utils/src/serde_bytes_fixed/ser.rs b/utils/src/serde_bytes_fixed/ser.rs index cde223677..7dd343976 100644 --- a/utils/src/serde_bytes_fixed/ser.rs +++ b/utils/src/serde_bytes_fixed/ser.rs @@ -1,14 +1,16 @@ -use serde::Serializer; -use std::str::{self}; use serde::ser::SerializeTuple; +use serde::Serializer; +use std::str; +/// Trait for serialization of types which can be referenced as fixed-size byte arrays. pub trait Serialize { - #[allow(missing_docs)] + /// Serialize `self` using the provided Serializer. fn serialize(&self, serializer: S) -> Result where S: Serializer; } +/// Implementation of Serialize trait for types that can be referenced as fixed-size byte array. impl> Serialize for T { fn serialize(&self, serializer: S) -> Result where @@ -19,10 +21,40 @@ impl> Serialize for T { faster_hex::hex_encode(self.as_ref(), &mut hex[..]).map_err(serde::ser::Error::custom)?; serializer.serialize_str(unsafe { str::from_utf8_unchecked(&hex) }) } else { - let t = serializer.serialize_tuple(self.as_ref().len())?; - - t.serialize_element() - serializer.serialize_bytes(self.as_ref()) + let mut t = serializer.serialize_tuple(self.as_ref().len())?; + for v in self.as_ref() { + t.serialize_element(v)?; + } + t.end() } } } + +#[macro_export] +/// Macro to generate serde::Serialize implementation for types `$t` that can be referenced as a byte array of fixed size, +/// The resulting structure will support serialization into human-readable formats using hex encoding, +/// as well as binary formats. +macro_rules! serde_impl_ser_fixed_bytes { + ($t: ty, $size: expr) => { + impl serde::Serialize for $t { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let len = std::convert::AsRef::<[u8; $size]>::as_ref(self).len(); + if serializer.is_human_readable() { + let mut hex = vec![0u8; len * 2]; + faster_hex::hex_encode(&std::convert::AsRef::<[u8; $size]>::as_ref(self)[..], &mut hex[..]) + .map_err(serde::ser::Error::custom)?; + serializer.serialize_str(unsafe { std::str::from_utf8_unchecked(&hex) }) + } else { + let mut t = serializer.serialize_tuple(len)?; + for v in std::convert::AsRef::<[u8; $size]>::as_ref(self) { + serde::ser::SerializeTuple::serialize_element(&mut t, v)? + } + serde::ser::SerializeTuple::end(t) + } + } + } + }; +} From 622ecdce8a5987c51d3b2b2edf6d0467d071c08e Mon Sep 17 00:00:00 2001 From: max143672 Date: Thu, 10 Aug 2023 12:33:11 +0400 Subject: [PATCH 6/9] "Implement serialization of SubnetworkId with fixed bytes" SubnetworkId is now serialized using newly imported serde utilities, serde_impl_ser_fixed_bytes and serde_impl_deser_fixed_bytes instead of serde_bytes_fixed attribute. --- consensus/core/src/subnets.rs | 4 ++++ consensus/core/src/tx.rs | 1 - protocol/p2p/src/convert/subnets.rs | 2 +- protocol/p2p/src/convert/tx.rs | 2 +- utils/src/serde_bytes/ser.rs | 2 +- 5 files changed, 7 insertions(+), 4 deletions(-) diff --git a/consensus/core/src/subnets.rs b/consensus/core/src/subnets.rs index 95542283f..63623ab86 100644 --- a/consensus/core/src/subnets.rs +++ b/consensus/core/src/subnets.rs @@ -3,6 +3,7 @@ use std::str::{self, FromStr}; use borsh::{BorshDeserialize, BorshSchema, BorshSerialize}; use kaspa_utils::hex::{FromHex, ToHex}; +use kaspa_utils::{serde_impl_deser_fixed_bytes, serde_impl_ser_fixed_bytes}; /// The size of the array used to store subnetwork IDs. pub const SUBNETWORK_ID_SIZE: usize = 20; @@ -11,6 +12,9 @@ pub const SUBNETWORK_ID_SIZE: usize = 20; #[derive(Debug, Clone, Default, Eq, PartialEq, Ord, PartialOrd, Hash, BorshSerialize, BorshDeserialize, BorshSchema)] pub struct SubnetworkId([u8; SUBNETWORK_ID_SIZE]); +serde_impl_ser_fixed_bytes!(SubnetworkId, SUBNETWORK_ID_SIZE); +serde_impl_deser_fixed_bytes!(SubnetworkId, SUBNETWORK_ID_SIZE); + impl AsRef<[u8; SUBNETWORK_ID_SIZE]> for SubnetworkId { fn as_ref(&self) -> &[u8; SUBNETWORK_ID_SIZE] { &self.0 diff --git a/consensus/core/src/tx.rs b/consensus/core/src/tx.rs index 81960421c..06b8b4943 100644 --- a/consensus/core/src/tx.rs +++ b/consensus/core/src/tx.rs @@ -257,7 +257,6 @@ pub struct Transaction { pub inputs: Vec, pub outputs: Vec, pub lock_time: u64, - #[serde(with = "serde_bytes_fixed")] pub subnetwork_id: SubnetworkId, pub gas: u64, #[serde(with = "serde_bytes")] diff --git a/protocol/p2p/src/convert/subnets.rs b/protocol/p2p/src/convert/subnets.rs index cfc1d4298..6fbe78d69 100644 --- a/protocol/p2p/src/convert/subnets.rs +++ b/protocol/p2p/src/convert/subnets.rs @@ -9,7 +9,7 @@ use super::error::ConversionError; impl From for protowire::SubnetworkId { fn from(item: SubnetworkId) -> Self { - Self { bytes: item.as_ref().to_vec() } + Self { bytes: >::as_ref(&item).to_vec() } } } diff --git a/protocol/p2p/src/convert/tx.rs b/protocol/p2p/src/convert/tx.rs index 15afec7b9..614396872 100644 --- a/protocol/p2p/src/convert/tx.rs +++ b/protocol/p2p/src/convert/tx.rs @@ -24,7 +24,7 @@ impl From<&Hash> for protowire::TransactionId { impl From<&SubnetworkId> for protowire::SubnetworkId { fn from(id: &SubnetworkId) -> Self { - Self { bytes: Vec::from(id.as_ref()) } + Self { bytes: Vec::from(>::as_ref(id)) } } } diff --git a/utils/src/serde_bytes/ser.rs b/utils/src/serde_bytes/ser.rs index 67e28a54e..3206764a8 100644 --- a/utils/src/serde_bytes/ser.rs +++ b/utils/src/serde_bytes/ser.rs @@ -15,7 +15,7 @@ impl> Serialize for T { { if serializer.is_human_readable() { let mut hex = vec![0u8; self.as_ref().len() * 2]; - faster_hex::hex_encode(&self.as_ref(), &mut hex[..]).map_err(serde::ser::Error::custom)?; + faster_hex::hex_encode(self.as_ref(), &mut hex[..]).map_err(serde::ser::Error::custom)?; serializer.serialize_str(unsafe { str::from_utf8_unchecked(&hex) }) } else { serializer.serialize_bytes(self.as_ref()) From 3cbe95f7ecf28f1ef9aba4ba7b5c98fa8a557d4a Mon Sep 17 00:00:00 2001 From: max143672 Date: Thu, 10 Aug 2023 12:52:46 +0400 Subject: [PATCH 7/9] Update formatting of deserialization error message Modified the "expecting" function in serde_bytes_fixed/de.rs to provide clearer error messages when deserializing byte arrays. The error message now includes the expected data type in addition to the array size. --- utils/src/serde_bytes_fixed/de.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utils/src/serde_bytes_fixed/de.rs b/utils/src/serde_bytes_fixed/de.rs index d81679195..27b82d820 100644 --- a/utils/src/serde_bytes_fixed/de.rs +++ b/utils/src/serde_bytes_fixed/de.rs @@ -120,7 +120,8 @@ macro_rules! serde_impl_deser_fixed_bytes { type Value = $t; fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(formatter, "an byte array of size {} to stringify!($i)", $size) + write!(formatter, "an byte array of size {} to ", $size)?; + write!(formatter, "{}", stringify!($t)) } #[inline] fn visit_str(self, v: &str) -> Result From 2cf7f4c75093574d04683dd706ae5bdeb8cd3d85 Mon Sep 17 00:00:00 2001 From: max143672 Date: Fri, 11 Aug 2023 11:36:07 +0400 Subject: [PATCH 8/9] Refactor deserialization of hexadecimal strings Reduced code redundancy in serde_bytes_fixed/de.rs and serde_bytes/de.rs by introducing a FromHexVisitor struct which handles converting hexadecimal strings to their respective types. This makes the code more maintainable and easier to test. Moreover, implemented FromHex trait to ScriptPublicKey, which allows us to use our new visitor in deserialization of ScriptPublicKey. --- consensus/core/src/tx.rs | 14 ++++-- utils/src/serde_bytes/de.rs | 79 ++++++++++++++++++++++++++++--- utils/src/serde_bytes/mod.rs | 2 +- utils/src/serde_bytes_fixed/de.rs | 6 +-- 4 files changed, 87 insertions(+), 14 deletions(-) diff --git a/consensus/core/src/tx.rs b/consensus/core/src/tx.rs index 06b8b4943..721360ba5 100644 --- a/consensus/core/src/tx.rs +++ b/consensus/core/src/tx.rs @@ -32,6 +32,7 @@ pub type ScriptVec = SmallVec<[u8; SCRIPT_VECTOR_SIZE]>; /// Represents the ScriptPublicKey Version pub type ScriptPublicKeyVersion = u16; +use kaspa_utils::serde_bytes::FromHexVisitor; /// Alias the `smallvec!` macro to ease maintenance pub use smallvec::smallvec as scriptvec; @@ -46,6 +47,14 @@ pub struct ScriptPublicKey { script: ScriptVec, // Kept private to preserve read-only semantics } +impl FromHex for ScriptPublicKey { + type Error = faster_hex::Error; + + fn from_hex(hex_str: &str) -> Result { + ScriptPublicKey::from_str(hex_str) + } +} + #[derive(Default, Debug, PartialEq, Eq, Serialize, Deserialize, Clone, Hash)] #[serde(rename_all = "camelCase")] #[serde(rename = "ScriptPublicKey")] @@ -76,8 +85,7 @@ impl<'de: 'a, 'a> Deserialize<'de> for ScriptPublicKey { D: Deserializer<'de>, { if deserializer.is_human_readable() { - let s = <&str as Deserialize>::deserialize(deserializer)?; - FromStr::from_str(s).map_err(serde::de::Error::custom) + deserializer.deserialize_any(FromHexVisitor::default()) } else { ScriptPublicKeyInternal::deserialize(deserializer) .map(|ScriptPublicKeyInternal { script, version }| Self { version, script: SmallVec::from_slice(script) }) @@ -664,7 +672,7 @@ mod tests { fn test_spk_serde_json() { let vec = (0..SCRIPT_VECTOR_SIZE as u8).collect::>(); let spk = ScriptPublicKey::from_vec(0xc0de, vec.clone()); - let hex = serde_json::to_string(&spk).unwrap(); + let hex: String = serde_json::to_string(&spk).unwrap(); assert_eq!("\"c0de000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20212223\"", hex); let spk = serde_json::from_str::(&hex).unwrap(); assert_eq!(spk.version, 0xc0de); diff --git a/utils/src/serde_bytes/de.rs b/utils/src/serde_bytes/de.rs index 7643414e7..e89c625f7 100644 --- a/utils/src/serde_bytes/de.rs +++ b/utils/src/serde_bytes/de.rs @@ -1,11 +1,10 @@ use crate::hex::FromHex; -use serde::Deserializer; -use std::fmt::Display; +use std::{fmt::Display, str}; pub trait Deserialize<'de>: Sized + FromHex + TryFrom<&'de [u8]> { fn deserialize(deserializer: D) -> Result where - D: Deserializer<'de>; + D: serde::Deserializer<'de>; } impl<'de, T: FromHex + TryFrom<&'de [u8]>> Deserialize<'de> for T @@ -14,14 +13,82 @@ where { fn deserialize(deserializer: D) -> Result where - D: Deserializer<'de>, + D: serde::Deserializer<'de>, { if deserializer.is_human_readable() { - let s: &str = serde::Deserialize::deserialize(deserializer)?; - Ok(T::from_hex(s).map_err(serde::de::Error::custom)?) + deserializer.deserialize_any(FromHexVisitor::default()) } else { // serde::Deserialize for &[u8] is already optimized, so simply forward to that. serde::Deserialize::deserialize(deserializer).and_then(|bts| T::try_from(bts).map_err(serde::de::Error::custom)) } } } + +pub struct FromHexVisitor<'de, T: FromHex> { + marker: std::marker::PhantomData, + lifetime: std::marker::PhantomData<&'de ()>, +} + +impl<'de, T: FromHex> Default for FromHexVisitor<'de, T> { + fn default() -> Self { + Self { marker: Default::default(), lifetime: Default::default() } + } +} + +impl<'de, T: FromHex> serde::de::Visitor<'de> for FromHexVisitor<'de, T> { + type Value = T; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(formatter, "string, str or slice, vec of bytes") + } + #[inline] + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + FromHex::from_hex(v).map_err(serde::de::Error::custom) + } + + #[inline] + fn visit_borrowed_str(self, v: &'de str) -> Result + where + E: serde::de::Error, + { + FromHex::from_hex(v).map_err(serde::de::Error::custom) + } + + #[inline] + fn visit_string(self, v: String) -> Result + where + E: serde::de::Error, + { + FromHex::from_hex(&v).map_err(serde::de::Error::custom) + } + + #[inline] + fn visit_bytes(self, v: &[u8]) -> Result + where + E: serde::de::Error, + { + let str = str::from_utf8(v).map_err(serde::de::Error::custom)?; + FromHex::from_hex(str).map_err(serde::de::Error::custom) + } + + #[inline] + fn visit_borrowed_bytes(self, v: &'de [u8]) -> Result + where + E: serde::de::Error, + { + let str = str::from_utf8(v).map_err(serde::de::Error::custom)?; + FromHex::from_hex(str).map_err(serde::de::Error::custom) + } + + #[inline] + fn visit_byte_buf(self, v: Vec) -> Result + where + E: serde::de::Error, + { + let str = str::from_utf8(&v).map_err(serde::de::Error::custom)?; + FromHex::from_hex(str).map_err(serde::de::Error::custom) + } +} diff --git a/utils/src/serde_bytes/mod.rs b/utils/src/serde_bytes/mod.rs index fa20694df..993786a32 100644 --- a/utils/src/serde_bytes/mod.rs +++ b/utils/src/serde_bytes/mod.rs @@ -1,7 +1,7 @@ mod de; mod ser; -pub use crate::serde_bytes::de::Deserialize; +pub use crate::serde_bytes::de::{Deserialize, FromHexVisitor}; pub use crate::serde_bytes::ser::Serialize; pub fn serialize(bytes: &T, serializer: S) -> Result diff --git a/utils/src/serde_bytes_fixed/de.rs b/utils/src/serde_bytes_fixed/de.rs index 27b82d820..496bbd044 100644 --- a/utils/src/serde_bytes_fixed/de.rs +++ b/utils/src/serde_bytes_fixed/de.rs @@ -83,8 +83,7 @@ macro_rules! deser_fixed_bytes { } if deserializer.is_human_readable() { - let s: &str = serde::Deserialize::deserialize(deserializer)?; - Ok(T::from_hex(s).map_err(serde::de::Error::custom)?) + deserializer.deserialize_any($crate::serde_bytes::FromHexVisitor::default()) } else { deserializer .deserialize_tuple($size, FixedBytesVisitor { marker: Default::default(), lifetime: Default::default() }) @@ -175,8 +174,7 @@ macro_rules! serde_impl_deser_fixed_bytes { } } if deserializer.is_human_readable() { - let s: &str = serde::Deserialize::deserialize(deserializer)?; - Ok($crate::hex::FromHex::from_hex(s).map_err(serde::de::Error::custom)?) + deserializer.deserialize_any($crate::serde_bytes::FromHexVisitor::default()) } else { deserializer.deserialize_newtype_struct(stringify!($i), MyVisitor { marker: Default::default(), lifetime: Default::default() }) } From 915acdb580ace77ff135c6f35b09fff7883d2939 Mon Sep 17 00:00:00 2001 From: max143672 Date: Fri, 11 Aug 2023 14:58:38 +0400 Subject: [PATCH 9/9] Update deserialization for fixed-size byte arrays Fixed-size byte arrays now should implement AsRef<[u8; N]> in addition to FromHex and From<[u8; N]> to fit the constraints. This modification is found in the kaspa-utils library. This also extends the functionality of the Deserialize trait in kaspa-utils which will now support the deserialization of byte arrays of a fixed size. By doing so, the code maintaining the 'serde_impl_deser_fixed_bytes' and 'serde_impl_ser_fixed_bytes' macros to implement serialization and deserialization for byte arrays of a certain size is simplified. The method works by referencing the byte array as the fixed-size array which makes it possible to use a reference implementation instead of the usual implementation. Added new Deserialize trait implementation for byte arrays of length 0 to 32 and replaced fixed_bytes with fixed_bytes_ref for module hashes and tx in crypto and consensus/core respectively. Modified the implementation of the Serialize trait for types that can be referenced as a fixed-sized byte array which is now possible for byte arrays from a length of 0 to 32. An extended set of examples demonstrating how to use the proposed serialization/deserialization has been included. The purpose of these changes is to simplify the implementation and usage of serialization/deserialization for byte arrays of a fixed size, and to extend this functionality to a larger range of possible array sizes. --- Cargo.lock | 3 + consensus/core/src/subnets.rs | 6 +- consensus/core/src/tx.rs | 8 +- crypto/hashes/src/lib.rs | 6 +- utils/Cargo.toml | 3 + utils/src/hex.rs | 25 ++++ utils/src/lib.rs | 172 +++++++++++++++++++++++++ utils/src/serde_bytes/de.rs | 2 +- utils/src/serde_bytes_fixed/de.rs | 99 ++------------ utils/src/serde_bytes_fixed/mod.rs | 27 ---- utils/src/serde_bytes_fixed/ser.rs | 40 +----- utils/src/serde_bytes_fixed_ref/de.rs | 86 +++++++++++++ utils/src/serde_bytes_fixed_ref/mod.rs | 21 +++ utils/src/serde_bytes_fixed_ref/ser.rs | 60 +++++++++ 14 files changed, 395 insertions(+), 163 deletions(-) create mode 100644 utils/src/serde_bytes_fixed_ref/de.rs create mode 100644 utils/src/serde_bytes_fixed_ref/mod.rs create mode 100644 utils/src/serde_bytes_fixed_ref/ser.rs diff --git a/Cargo.lock b/Cargo.lock index 60806ed73..65305cb1c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2749,6 +2749,7 @@ version = "0.1.6" dependencies = [ "async-channel", "async-trait", + "bincode", "borsh", "criterion", "event-listener", @@ -2758,7 +2759,9 @@ dependencies = [ "itertools 0.10.5", "rand 0.8.5", "serde", + "serde_json", "smallvec", + "thiserror", "tokio", "triggered", "uuid 1.4.1", diff --git a/consensus/core/src/subnets.rs b/consensus/core/src/subnets.rs index 63623ab86..0f1c81d5a 100644 --- a/consensus/core/src/subnets.rs +++ b/consensus/core/src/subnets.rs @@ -3,7 +3,7 @@ use std::str::{self, FromStr}; use borsh::{BorshDeserialize, BorshSchema, BorshSerialize}; use kaspa_utils::hex::{FromHex, ToHex}; -use kaspa_utils::{serde_impl_deser_fixed_bytes, serde_impl_ser_fixed_bytes}; +use kaspa_utils::{serde_impl_deser_fixed_bytes_ref, serde_impl_ser_fixed_bytes_ref}; /// The size of the array used to store subnetwork IDs. pub const SUBNETWORK_ID_SIZE: usize = 20; @@ -12,8 +12,8 @@ pub const SUBNETWORK_ID_SIZE: usize = 20; #[derive(Debug, Clone, Default, Eq, PartialEq, Ord, PartialOrd, Hash, BorshSerialize, BorshDeserialize, BorshSchema)] pub struct SubnetworkId([u8; SUBNETWORK_ID_SIZE]); -serde_impl_ser_fixed_bytes!(SubnetworkId, SUBNETWORK_ID_SIZE); -serde_impl_deser_fixed_bytes!(SubnetworkId, SUBNETWORK_ID_SIZE); +serde_impl_ser_fixed_bytes_ref!(SubnetworkId, SUBNETWORK_ID_SIZE); +serde_impl_deser_fixed_bytes_ref!(SubnetworkId, SUBNETWORK_ID_SIZE); impl AsRef<[u8; SUBNETWORK_ID_SIZE]> for SubnetworkId { fn as_ref(&self) -> &[u8; SUBNETWORK_ID_SIZE] { diff --git a/consensus/core/src/tx.rs b/consensus/core/src/tx.rs index 721360ba5..2c9eeb5e8 100644 --- a/consensus/core/src/tx.rs +++ b/consensus/core/src/tx.rs @@ -1,5 +1,5 @@ use borsh::{BorshDeserialize, BorshSchema, BorshSerialize}; -use kaspa_utils::{hex::*, serde_bytes, serde_bytes_fixed}; +use kaspa_utils::{hex::*, serde_bytes, serde_bytes_fixed_ref}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use smallvec::SmallVec; @@ -85,7 +85,7 @@ impl<'de: 'a, 'a> Deserialize<'de> for ScriptPublicKey { D: Deserializer<'de>, { if deserializer.is_human_readable() { - deserializer.deserialize_any(FromHexVisitor::default()) + deserializer.deserialize_str(FromHexVisitor::default()) } else { ScriptPublicKeyInternal::deserialize(deserializer) .map(|ScriptPublicKeyInternal { script, version }| Self { version, script: SmallVec::from_slice(script) }) @@ -209,7 +209,7 @@ pub type TransactionIndexType = u32; #[derive(Eq, Hash, PartialEq, Debug, Copy, Clone, Serialize, Deserialize, BorshSerialize, BorshDeserialize, BorshSchema)] #[serde(rename_all = "camelCase")] pub struct TransactionOutpoint { - #[serde(with = "serde_bytes_fixed")] + #[serde(with = "serde_bytes_fixed_ref")] pub transaction_id: TransactionId, pub index: TransactionIndexType, } @@ -272,7 +272,7 @@ pub struct Transaction { // A field that is used to cache the transaction ID. // Always use the corresponding self.id() instead of accessing this field directly - #[serde(with = "serde_bytes_fixed")] + #[serde(with = "serde_bytes_fixed_ref")] id: TransactionId, } diff --git a/crypto/hashes/src/lib.rs b/crypto/hashes/src/lib.rs index 46adcc375..827ddfb49 100644 --- a/crypto/hashes/src/lib.rs +++ b/crypto/hashes/src/lib.rs @@ -4,7 +4,7 @@ mod pow_hashers; use borsh::{BorshDeserialize, BorshSchema, BorshSerialize}; use kaspa_utils::{ hex::{FromHex, ToHex}, - serde_impl_deser_fixed_bytes, serde_impl_ser_fixed_bytes, + serde_impl_deser_fixed_bytes_ref, serde_impl_ser_fixed_bytes_ref, }; use std::{ array::TryFromSliceError, @@ -24,8 +24,8 @@ pub use hashers::*; #[wasm_bindgen] pub struct Hash([u8; HASH_SIZE]); -serde_impl_ser_fixed_bytes!(Hash, HASH_SIZE); -serde_impl_deser_fixed_bytes!(Hash, HASH_SIZE); +serde_impl_ser_fixed_bytes_ref!(Hash, HASH_SIZE); +serde_impl_deser_fixed_bytes_ref!(Hash, HASH_SIZE); impl From<[u8; HASH_SIZE]> for Hash { fn from(value: [u8; HASH_SIZE]) -> Self { diff --git a/utils/Cargo.toml b/utils/Cargo.toml index 2836b7de3..046d62adc 100644 --- a/utils/Cargo.toml +++ b/utils/Cargo.toml @@ -15,12 +15,15 @@ borsh.workspace = true faster-hex.workspace = true smallvec.workspace = true itertools.workspace = true +thiserror.workspace = true triggered = "0.1" event-listener = "2.5.3" ipnet = "2.8.0" [dev-dependencies] +bincode.workspace = true +serde_json.workspace = true async-trait.workspace = true futures-util.workspace = true tokio = { workspace = true, features = ["rt", "time", "macros"] } diff --git a/utils/src/hex.rs b/utils/src/hex.rs index 76a6e5cf9..beb0e08ab 100644 --- a/utils/src/hex.rs +++ b/utils/src/hex.rs @@ -1,4 +1,5 @@ use serde::{Deserialize, Deserializer, Serializer}; +use std::fmt::Debug; // use consensus_core::BlueWorkType; use smallvec::{smallvec, SmallVec}; use std::str; @@ -71,6 +72,30 @@ impl FromHex for Vec { } } +#[derive(Debug, thiserror::Error)] +pub enum FixedArrayError { + #[error(transparent)] + Deserialize(#[from] faster_hex::Error), + #[error("unexpected length of hex string. actual: {0}")] + WrongLength(usize), +} + +/// Little endian format of full content +/// (so string lengths must be even). +impl FromHex for [u8; N] { + type Error = FixedArrayError; + fn from_hex(hex_str: &str) -> Result { + let len = hex_str.len(); + if len != N * 2 { + return Err(Self::Error::WrongLength(len)); + } + + let mut bytes = [0u8; N]; + faster_hex::hex_decode(hex_str.as_bytes(), bytes.as_mut_slice())?; + Ok(bytes) + } +} + /// Little endian format of full content /// (so string lengths are always even). impl> ToHex for SmallVec { diff --git a/utils/src/lib.rs b/utils/src/lib.rs index 1e09e77c7..371e52001 100644 --- a/utils/src/lib.rs +++ b/utils/src/lib.rs @@ -8,8 +8,180 @@ pub mod iter; pub mod networking; pub mod option; pub mod refs; + +/// # Examples +/// ## Implement serde::Serialize/serde::Deserialize for the Vec field of the struct +/// ``` +/// #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +/// struct MyStructVec { +/// #[serde(with = "kaspa_utils::serde_bytes")] +/// v: Vec, +/// } +/// let v = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]; +/// let len = v.len(); +/// let test_struct = MyStructVec { v: v.clone() }; +/// +/// // Serialize using bincode +/// let encoded = bincode::serialize(&test_struct).unwrap(); +/// assert_eq!(encoded, len.to_le_bytes().into_iter().chain(v.into_iter()).collect::>()); +/// // Deserialize using bincode +/// let decoded: MyStructVec = bincode::deserialize(&encoded).unwrap(); +/// assert_eq!(test_struct, decoded); +/// +/// let expected_str = r#"{"v":"000102030405060708090a0b0c0d0e0f10111213"}"#; +/// // Serialize using serde_json +/// let json = serde_json::to_string(&test_struct).unwrap(); +/// assert_eq!(expected_str, json); +/// // Deserialize using serde_json +/// let from_json: MyStructVec = serde_json::from_str(&json).unwrap(); +/// assert_eq!(test_struct, from_json); +/// ``` +/// ## Implement serde::Serialize/serde::Deserialize for the SmallVec field of the struct +/// ``` +/// use smallvec::{smallvec, SmallVec}; +/// #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +/// struct MyStructSmallVec { +/// #[serde(with = "kaspa_utils::serde_bytes")] +/// v: SmallVec<[u8; 19]>, +/// } +/// let v = smallvec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]; +/// let len = v.len(); +/// let test_struct = MyStructSmallVec { v: v.clone() }; +/// +/// // Serialize using bincode +/// let encoded = bincode::serialize(&test_struct).unwrap(); +/// assert_eq!(encoded, len.to_le_bytes().into_iter().chain(v.into_iter()).collect::>()); +/// // Deserialize using bincode +/// let decoded: MyStructSmallVec = bincode::deserialize(&encoded).unwrap(); +/// assert_eq!(test_struct, decoded); +/// +/// let expected_str = r#"{"v":"000102030405060708090a0b0c0d0e0f10111213"}"#; +/// // Serialize using serde_json +/// let json = serde_json::to_string(&test_struct).unwrap(); +/// assert_eq!(expected_str, json); +/// // Deserialize using serde_json +/// let from_json: MyStructSmallVec = serde_json::from_str(&json).unwrap(); +/// assert_eq!(test_struct, from_json); +/// ``` pub mod serde_bytes; + +/// # Examples +/// +/// ``` +/// #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +/// struct TestStruct { +/// #[serde(with = "kaspa_utils::serde_bytes_fixed")] +/// v: [u8; 20], +/// } +/// let test_struct = TestStruct { v: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19] }; +/// +/// // Serialize using bincode +/// let encoded = bincode::serialize(&test_struct).unwrap(); +/// assert_eq!(encoded, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]); +/// // Deserialize using bincode +/// let decoded: TestStruct = bincode::deserialize(&encoded).unwrap(); +/// assert_eq!(test_struct, decoded); +/// +/// let expected_str = r#"{"v":"000102030405060708090a0b0c0d0e0f10111213"}"#; +/// // Serialize using serde_json +/// let json = serde_json::to_string(&test_struct).unwrap(); +/// assert_eq!(expected_str, json); +/// // Deserialize using serde_json +/// let from_json: TestStruct = serde_json::from_str(&json).unwrap(); +/// assert_eq!(test_struct, from_json); +/// ``` pub mod serde_bytes_fixed; +/// # Examples +/// ## Implement serde::Serialize/serde::Deserialize using declarative macro +/// ``` +/// #[derive(Debug, Copy, Clone, PartialEq, Eq)] +/// struct MyStruct([u8; 20]); +/// +/// impl AsRef<[u8; 20]> for MyStruct { +/// fn as_ref(&self) -> &[u8; 20] { +/// &self.0 +/// } +/// } +/// impl kaspa_utils::hex::FromHex for MyStruct { +/// type Error = faster_hex::Error; +/// fn from_hex(hex_str: &str) -> Result { +/// let mut bytes = [0u8; 20]; +/// faster_hex::hex_decode(hex_str.as_bytes(), &mut bytes)?; +/// Ok(MyStruct(bytes)) +/// } +/// } +/// impl From<[u8; 20]> for MyStruct { +/// fn from(value: [u8; 20]) -> Self { +/// MyStruct(value) +/// } +/// } +/// kaspa_utils::serde_impl_ser_fixed_bytes_ref!(MyStruct, 20); +/// kaspa_utils::serde_impl_deser_fixed_bytes_ref!(MyStruct, 20); +/// +/// let test_struct = MyStruct([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]); +/// let expected_str = r#""000102030405060708090a0b0c0d0e0f10111213""#; +/// // Serialize using serde_json +/// let json = serde_json::to_string(&test_struct).unwrap(); +/// assert_eq!(expected_str, json); +/// // Deserialize using serde_json +/// let from_json: MyStruct = serde_json::from_str(&json).unwrap(); +/// assert_eq!(test_struct, from_json); +/// // Serialize using bincode +/// let encoded = bincode::serialize(&test_struct).unwrap(); +/// assert_eq!(encoded, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]); +/// // Deserialize using bincode +/// let decoded: MyStruct = bincode::deserialize(&encoded).unwrap(); +/// assert_eq!(test_struct, decoded); +/// ``` +/// +/// ## Implement serde::Serialize/serde::Deserialize for the field of the struct +/// ``` +/// #[derive(Debug, Copy, Clone, PartialEq, Eq)] +/// struct MyStruct([u8; 20]); +/// +/// impl AsRef<[u8; 20]> for MyStruct { +/// fn as_ref(&self) -> &[u8; 20] { +/// &self.0 +/// } +/// } +/// impl kaspa_utils::hex::FromHex for MyStruct { +/// type Error = faster_hex::Error; +/// fn from_hex(hex_str: &str) -> Result { +/// let mut bytes = [0u8; 20]; +/// faster_hex::hex_decode(hex_str.as_bytes(), &mut bytes)?; +/// Ok(MyStruct(bytes)) +/// } +/// } +/// impl From<[u8; 20]> for MyStruct { +/// fn from(value: [u8; 20]) -> Self { +/// MyStruct(value) +/// } +/// } +/// +/// #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +/// struct TestStruct { +/// #[serde(with = "kaspa_utils::serde_bytes_fixed_ref")] +/// v: MyStruct, +/// } +/// +/// let test_struct = TestStruct { v: MyStruct([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]) }; +/// +/// // Serialize using bincode +/// let encoded = bincode::serialize(&test_struct).unwrap(); +/// assert_eq!(encoded, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]); +/// // Deserialize using bincode +/// let decoded: TestStruct = bincode::deserialize(&encoded).unwrap(); +/// assert_eq!(test_struct, decoded); +/// +/// let expected_str = r#"{"v":"000102030405060708090a0b0c0d0e0f10111213"}"#; +/// // Serialize using serde_json +/// let json = serde_json::to_string(&test_struct).unwrap(); +/// assert_eq!(expected_str, json); +/// // Deserialize using serde_json +/// let from_json: TestStruct = serde_json::from_str(&json).unwrap(); +/// assert_eq!(test_struct, from_json); +/// ``` +pub mod serde_bytes_fixed_ref; pub mod sim; pub mod sync; pub mod triggers; diff --git a/utils/src/serde_bytes/de.rs b/utils/src/serde_bytes/de.rs index e89c625f7..66064bfe3 100644 --- a/utils/src/serde_bytes/de.rs +++ b/utils/src/serde_bytes/de.rs @@ -16,7 +16,7 @@ where D: serde::Deserializer<'de>, { if deserializer.is_human_readable() { - deserializer.deserialize_any(FromHexVisitor::default()) + deserializer.deserialize_str(FromHexVisitor::default()) } else { // serde::Deserialize for &[u8] is already optimized, so simply forward to that. serde::Deserialize::deserialize(deserializer).and_then(|bts| T::try_from(bts).map_err(serde::de::Error::custom)) diff --git a/utils/src/serde_bytes_fixed/de.rs b/utils/src/serde_bytes_fixed/de.rs index 496bbd044..8df52cab9 100644 --- a/utils/src/serde_bytes_fixed/de.rs +++ b/utils/src/serde_bytes_fixed/de.rs @@ -1,6 +1,7 @@ /// Trait for deserialization of fixed-size byte arrays, newtype structs, /// or any other custom types that can implement `From<[u8; N]>`. /// Implementers should also provide implementation for `crate::hex::FromHex`. +/// N must be in range: (0..=32), this is serde limitation pub trait Deserialize<'de, const N: usize>: Sized + crate::hex::FromHex + From<[u8; N]> { /// Deserialize given `deserializer` into `Self`. fn deserialize(deserializer: D) -> Result @@ -8,13 +9,12 @@ pub trait Deserialize<'de, const N: usize>: Sized + crate::hex::FromHex + From<[ D: serde::Deserializer<'de>; } -#[macro_export] /// Macro to generate impl of `Deserialize` for types `T`, which are /// capable of being constructed from byte arrays of fixed size, /// or newtype structs or other types based on [`From<[u8; $size]>`]. macro_rules! deser_fixed_bytes { ($size: expr) => { - impl<'de, T: $crate::hex::FromHex + From<[u8; $size]>> Deserialize<'de, $size> for T { + impl<'de, T: $crate::hex::FromHex + From<[u8; $size]>> $crate::serde_bytes_fixed::Deserialize<'de, $size> for T { /// Deserialization function for types `T` fn deserialize(deserializer: D) -> Result where @@ -83,7 +83,7 @@ macro_rules! deser_fixed_bytes { } if deserializer.is_human_readable() { - deserializer.deserialize_any($crate::serde_bytes::FromHexVisitor::default()) + deserializer.deserialize_str($crate::serde_bytes::FromHexVisitor::default()) } else { deserializer .deserialize_tuple($size, FixedBytesVisitor { marker: Default::default(), lifetime: Default::default() }) @@ -94,91 +94,10 @@ macro_rules! deser_fixed_bytes { }; } -// Calling the macro to generate `Deserialize` implementation for byte arrays of size 20 and 32, -// and for newtype structs or other types extendable from these fixed sized arrays. -deser_fixed_bytes!(20); -deser_fixed_bytes!(32); - -#[macro_export] -/// Macro to provide serde::Deserialize implementations for types `$t` -/// which can be constructed from byte arrays of fixed size. -/// The resulting structure will support deserialization from human-readable -/// formats using hex::FromHex, as well as binary formats. -macro_rules! serde_impl_deser_fixed_bytes { - ($t: ty, $size: expr) => { - impl<'de> serde::Deserialize<'de> for $t { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - struct MyVisitor<'de> { - marker: std::marker::PhantomData<$t>, - lifetime: std::marker::PhantomData<&'de ()>, - } - impl<'de> serde::de::Visitor<'de> for MyVisitor<'de> { - type Value = $t; - - fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(formatter, "an byte array of size {} to ", $size)?; - write!(formatter, "{}", stringify!($t)) - } - #[inline] - fn visit_str(self, v: &str) -> Result - where - E: serde::de::Error, - { - let v: [u8; $size] = v.as_bytes().try_into().map_err(serde::de::Error::custom)?; - Ok(Self::Value::from(v)) - } - #[inline] - fn visit_borrowed_str(self, v: &'de str) -> Result - where - E: serde::de::Error, - { - let v: [u8; $size] = v.as_bytes().try_into().map_err(serde::de::Error::custom)?; - Ok(Self::Value::from(v)) - } - #[inline] - fn visit_bytes(self, v: &[u8]) -> Result - where - E: serde::de::Error, - { - let v: [u8; $size] = v.try_into().map_err(serde::de::Error::custom)?; - Ok(Self::Value::from(v)) - } - #[inline] - fn visit_borrowed_bytes(self, v: &'de [u8]) -> Result - where - E: serde::de::Error, - { - let v: [u8; $size] = v.try_into().map_err(serde::de::Error::custom)?; - Ok(Self::Value::from(v)) - } - - #[inline] - fn visit_newtype_struct(self, deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - <[u8; $size] as serde::Deserialize>::deserialize(deserializer).map(|v| Self::Value::from(v)) - } - #[inline] - fn visit_seq(self, mut seq: A) -> Result - where - A: serde::de::SeqAccess<'de>, - { - let Some(value): Option<[u8; $size]> = seq.next_element()? else { - return Err(serde::de::Error::invalid_length(0usize, &"tuple struct fixed array with 1 element")); - } ; - Ok(Self::Value::from(value)) - } - } - if deserializer.is_human_readable() { - deserializer.deserialize_any($crate::serde_bytes::FromHexVisitor::default()) - } else { - deserializer.deserialize_newtype_struct(stringify!($i), MyVisitor { marker: Default::default(), lifetime: Default::default() }) - } - } - } - }; +macro_rules! apply_deser_fixed_bytes { + ($($x:expr),*) => ($(deser_fixed_bytes!($x);)*) } + +apply_deser_fixed_bytes!( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32 +); diff --git a/utils/src/serde_bytes_fixed/mod.rs b/utils/src/serde_bytes_fixed/mod.rs index 51a56a4aa..025e258b0 100644 --- a/utils/src/serde_bytes_fixed/mod.rs +++ b/utils/src/serde_bytes_fixed/mod.rs @@ -3,7 +3,6 @@ mod ser; pub use crate::serde_bytes_fixed::de::Deserialize; pub use crate::serde_bytes_fixed::ser::Serialize; -use crate::{serde_impl_deser_fixed_bytes, serde_impl_ser_fixed_bytes}; pub fn serialize(bytes: &T, serializer: S) -> Result where @@ -20,29 +19,3 @@ where { Deserialize::deserialize(deserializer) } - -struct MyBox([u8; 20]); - -impl AsRef<[u8; 20]> for MyBox { - fn as_ref(&self) -> &[u8; 20] { - &self.0 - } -} - -impl crate::hex::FromHex for MyBox { - type Error = faster_hex::Error; - fn from_hex(hex_str: &str) -> Result { - let mut bytes = [0u8; 20]; - faster_hex::hex_decode(hex_str.as_bytes(), &mut bytes)?; - Ok(MyBox(bytes)) - } -} - -impl From<[u8; 20]> for MyBox { - fn from(value: [u8; 20]) -> Self { - MyBox(value) - } -} - -serde_impl_ser_fixed_bytes!(MyBox, 20); -serde_impl_deser_fixed_bytes!(MyBox, 20); diff --git a/utils/src/serde_bytes_fixed/ser.rs b/utils/src/serde_bytes_fixed/ser.rs index 7dd343976..94ca5a8f6 100644 --- a/utils/src/serde_bytes_fixed/ser.rs +++ b/utils/src/serde_bytes_fixed/ser.rs @@ -10,51 +10,21 @@ pub trait Serialize { S: Serializer; } -/// Implementation of Serialize trait for types that can be referenced as fixed-size byte array. -impl> Serialize for T { +impl Serialize for [u8; N] { fn serialize(&self, serializer: S) -> Result where S: Serializer, { if serializer.is_human_readable() { - let mut hex = vec![0u8; self.as_ref().len() * 2]; - faster_hex::hex_encode(self.as_ref(), &mut hex[..]).map_err(serde::ser::Error::custom)?; + let mut hex = vec![0u8; self.len() * 2]; + faster_hex::hex_encode(self, &mut hex[..]).map_err(serde::ser::Error::custom)?; serializer.serialize_str(unsafe { str::from_utf8_unchecked(&hex) }) } else { - let mut t = serializer.serialize_tuple(self.as_ref().len())?; - for v in self.as_ref() { + let mut t = serializer.serialize_tuple(self.len())?; + for v in self { t.serialize_element(v)?; } t.end() } } } - -#[macro_export] -/// Macro to generate serde::Serialize implementation for types `$t` that can be referenced as a byte array of fixed size, -/// The resulting structure will support serialization into human-readable formats using hex encoding, -/// as well as binary formats. -macro_rules! serde_impl_ser_fixed_bytes { - ($t: ty, $size: expr) => { - impl serde::Serialize for $t { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - let len = std::convert::AsRef::<[u8; $size]>::as_ref(self).len(); - if serializer.is_human_readable() { - let mut hex = vec![0u8; len * 2]; - faster_hex::hex_encode(&std::convert::AsRef::<[u8; $size]>::as_ref(self)[..], &mut hex[..]) - .map_err(serde::ser::Error::custom)?; - serializer.serialize_str(unsafe { std::str::from_utf8_unchecked(&hex) }) - } else { - let mut t = serializer.serialize_tuple(len)?; - for v in std::convert::AsRef::<[u8; $size]>::as_ref(self) { - serde::ser::SerializeTuple::serialize_element(&mut t, v)? - } - serde::ser::SerializeTuple::end(t) - } - } - } - }; -} diff --git a/utils/src/serde_bytes_fixed_ref/de.rs b/utils/src/serde_bytes_fixed_ref/de.rs new file mode 100644 index 000000000..cde2d66bc --- /dev/null +++ b/utils/src/serde_bytes_fixed_ref/de.rs @@ -0,0 +1,86 @@ +#[macro_export] +/// Macro to provide serde::Deserialize implementations for types `$t` +/// which can be constructed from byte arrays of fixed size. +/// The resulting structure will support deserialization from human-readable +/// formats using hex::FromHex, as well as binary formats. +macro_rules! serde_impl_deser_fixed_bytes_ref { + ($t: ty, $size: expr) => { + impl<'de> serde::Deserialize<'de> for $t { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + struct MyVisitor<'de> { + marker: std::marker::PhantomData<$t>, + lifetime: std::marker::PhantomData<&'de ()>, + } + impl<'de> serde::de::Visitor<'de> for MyVisitor<'de> { + type Value = $t; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(formatter, "an byte array of size {} to ", $size)?; + write!(formatter, "{}", stringify!($t)) + } + #[inline] + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + let v: [u8; $size] = v.as_bytes().try_into().map_err(serde::de::Error::custom)?; + Ok(Self::Value::from(v)) + } + #[inline] + fn visit_borrowed_str(self, v: &'de str) -> Result + where + E: serde::de::Error, + { + let v: [u8; $size] = v.as_bytes().try_into().map_err(serde::de::Error::custom)?; + Ok(Self::Value::from(v)) + } + #[inline] + fn visit_bytes(self, v: &[u8]) -> Result + where + E: serde::de::Error, + { + let v: [u8; $size] = v.try_into().map_err(serde::de::Error::custom)?; + Ok(Self::Value::from(v)) + } + #[inline] + fn visit_borrowed_bytes(self, v: &'de [u8]) -> Result + where + E: serde::de::Error, + { + let v: [u8; $size] = v.try_into().map_err(serde::de::Error::custom)?; + Ok(Self::Value::from(v)) + } + + #[inline] + fn visit_newtype_struct(self, deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + <[u8; $size] as serde::Deserialize>::deserialize(deserializer).map(|v| Self::Value::from(v)) + } + #[inline] + fn visit_seq(self, mut seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + let Some(value): Option<[u8; $size]> = seq.next_element()? else { + return Err(serde::de::Error::invalid_length(0usize, &"tuple struct fixed array with 1 element")); + } ; + Ok(Self::Value::from(value)) + } + } + if deserializer.is_human_readable() { + deserializer.deserialize_str($crate::serde_bytes::FromHexVisitor::default()) + } else { + deserializer.deserialize_newtype_struct( + stringify!($i), + MyVisitor { marker: Default::default(), lifetime: Default::default() }, + ) + } + } + } + }; +} diff --git a/utils/src/serde_bytes_fixed_ref/mod.rs b/utils/src/serde_bytes_fixed_ref/mod.rs new file mode 100644 index 000000000..6974182d9 --- /dev/null +++ b/utils/src/serde_bytes_fixed_ref/mod.rs @@ -0,0 +1,21 @@ +mod de; +mod ser; + +pub use crate::serde_bytes_fixed::Deserialize; +pub use crate::serde_bytes_fixed_ref::ser::Serialize; + +pub fn serialize(bytes: &T, serializer: S) -> Result +where + T: ?Sized + Serialize, + S: serde::Serializer, +{ + Serialize::serialize(bytes, serializer) +} + +pub fn deserialize<'de, T, D, const N: usize>(deserializer: D) -> Result +where + T: Deserialize<'de, N>, + D: serde::Deserializer<'de>, +{ + Deserialize::deserialize(deserializer) +} diff --git a/utils/src/serde_bytes_fixed_ref/ser.rs b/utils/src/serde_bytes_fixed_ref/ser.rs new file mode 100644 index 000000000..6b6c652e4 --- /dev/null +++ b/utils/src/serde_bytes_fixed_ref/ser.rs @@ -0,0 +1,60 @@ +use serde::ser::SerializeTuple; +use serde::Serializer; +use std::str; + +/// Trait for serialization of types which can be referenced as fixed-size byte arrays. +pub trait Serialize { + /// Serialize `self` using the provided Serializer. + fn serialize(&self, serializer: S) -> Result + where + S: Serializer; +} + +/// Implementation of Serialize trait for types that can be referenced as fixed-size byte array. +impl> Serialize for T { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + if serializer.is_human_readable() { + let mut hex = vec![0u8; self.as_ref().len() * 2]; + faster_hex::hex_encode(self.as_ref(), &mut hex[..]).map_err(serde::ser::Error::custom)?; + serializer.serialize_str(unsafe { str::from_utf8_unchecked(&hex) }) + } else { + let mut t = serializer.serialize_tuple(self.as_ref().len())?; + for v in self.as_ref() { + t.serialize_element(v)?; + } + t.end() + } + } +} + +#[macro_export] +/// Macro to generate serde::Serialize implementation for types `$t` that can be referenced as a byte array of fixed size, +/// The resulting structure will support serialization into human-readable formats using hex encoding, +/// as well as binary formats. +macro_rules! serde_impl_ser_fixed_bytes_ref { + ($t: ty, $size: expr) => { + impl serde::Serialize for $t { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let len = std::convert::AsRef::<[u8; $size]>::as_ref(self).len(); + if serializer.is_human_readable() { + let mut hex = vec![0u8; len * 2]; + faster_hex::hex_encode(&std::convert::AsRef::<[u8; $size]>::as_ref(self)[..], &mut hex[..]) + .map_err(serde::ser::Error::custom)?; + serializer.serialize_str(unsafe { std::str::from_utf8_unchecked(&hex) }) + } else { + let mut t = serializer.serialize_tuple(len)?; + for v in std::convert::AsRef::<[u8; $size]>::as_ref(self) { + serde::ser::SerializeTuple::serialize_element(&mut t, v)? + } + serde::ser::SerializeTuple::end(t) + } + } + } + }; +}