From a8b94124ae9d1b943ba65454d0593e77f4b8fc84 Mon Sep 17 00:00:00 2001 From: austinabell Date: Sat, 2 May 2020 09:23:48 -0400 Subject: [PATCH 01/20] Rename serde trait on cid --- blockchain/blocks/Cargo.toml | 2 +- ipld/Cargo.toml | 2 +- ipld/car/Cargo.toml | 2 +- ipld/cid/Cargo.toml | 4 ++-- ipld/cid/src/lib.rs | 14 +++++++------- ipld/cid/tests/serde_tests.rs | 2 +- tests/serialization_tests/Cargo.toml | 2 +- vm/Cargo.toml | 2 +- vm/actor/Cargo.toml | 2 +- 9 files changed, 16 insertions(+), 16 deletions(-) diff --git a/blockchain/blocks/Cargo.toml b/blockchain/blocks/Cargo.toml index 7af6726ca6f9..24ff451a4841 100644 --- a/blockchain/blocks/Cargo.toml +++ b/blockchain/blocks/Cargo.toml @@ -10,7 +10,7 @@ beacon = { package = "beacon", path = "../beacon" } crypto = { path = "../../crypto" } message = { package = "forest_message", path = "../../vm/message" } clock = { path = "../../node/clock" } -cid = { package = "forest_cid", path = "../../ipld/cid", features = ["serde_derive"] } +cid = { package = "forest_cid", path = "../../ipld/cid", features = ["cbor"] } derive_builder = "0.9" serde = { version = "1.0", features = ["derive"] } encoding = { package = "forest_encoding", path = "../../encoding" } diff --git a/ipld/Cargo.toml b/ipld/Cargo.toml index 06ff16248c83..b414f5f2964b 100644 --- a/ipld/Cargo.toml +++ b/ipld/Cargo.toml @@ -12,4 +12,4 @@ thiserror = "1.0" [dependencies.cid] package = "forest_cid" path = "../ipld/cid" -features = ["serde_derive"] +features = ["cbor"] diff --git a/ipld/car/Cargo.toml b/ipld/car/Cargo.toml index 08b8e945fd91..74987270a079 100644 --- a/ipld/car/Cargo.toml +++ b/ipld/car/Cargo.toml @@ -6,7 +6,7 @@ edition = "2018" [dependencies] unsigned-varint = "0.3.1" -cid = { package = "forest_cid", path = "../cid", features = ["serde_derive"] } +cid = { package = "forest_cid", path = "../cid", features = ["cbor"] } forest_encoding = { path = "../../encoding" } blockstore = { package = "ipld_blockstore", path = "../blockstore" } serde = { version = "1.0", features = ["derive"] } diff --git a/ipld/cid/Cargo.toml b/ipld/cid/Cargo.toml index 54152af03ede..42be42251695 100644 --- a/ipld/cid/Cargo.toml +++ b/ipld/cid/Cargo.toml @@ -5,7 +5,7 @@ authors = ["ChainSafe Systems "] edition = "2018" [package.metadata.docs.rs] -features = ["serde_derive"] +features = ["cbor"] [dependencies] multihash = "0.10.0" @@ -17,4 +17,4 @@ serde_bytes = { version = "0.11.3", optional = true } thiserror = "1.0" [features] -serde_derive = ["serde", "serde_bytes", "serde_cbor"] +cbor = ["serde", "serde_bytes", "serde_cbor"] diff --git a/ipld/cid/src/lib.rs b/ipld/cid/src/lib.rs index 78f20b67932e..ec4bf4c33fc1 100644 --- a/ipld/cid/src/lib.rs +++ b/ipld/cid/src/lib.rs @@ -17,18 +17,18 @@ use std::convert::TryInto; use std::fmt; use std::io::Cursor; -#[cfg(feature = "serde_derive")] +#[cfg(feature = "cbor")] use serde::{de, ser}; -#[cfg(feature = "serde_derive")] +#[cfg(feature = "cbor")] use serde_cbor::tags::Tagged; -#[cfg(feature = "serde_derive")] +#[cfg(feature = "cbor")] use std::convert::TryFrom; -#[cfg(feature = "serde_derive")] +#[cfg(feature = "cbor")] const CBOR_TAG_CID: u64 = 42; /// multibase identity prefix /// https://github.com/ipld/specs/blob/master/block-layer/codecs/dag-cbor.md#link-format -#[cfg(feature = "serde_derive")] +#[cfg(feature = "cbor")] const MULTIBASE_IDENTITY: u8 = 0; /// Prefix represents all metadata of a CID, without the actual content. @@ -53,7 +53,7 @@ impl Default for Cid { } } -#[cfg(feature = "serde_derive")] +#[cfg(feature = "cbor")] impl ser::Serialize for Cid { fn serialize(&self, s: S) -> Result where @@ -75,7 +75,7 @@ impl ser::Serialize for Cid { } } -#[cfg(feature = "serde_derive")] +#[cfg(feature = "cbor")] impl<'de> de::Deserialize<'de> for Cid { fn deserialize(deserializer: D) -> Result where diff --git a/ipld/cid/tests/serde_tests.rs b/ipld/cid/tests/serde_tests.rs index ed93e90ead76..aead15f20a12 100644 --- a/ipld/cid/tests/serde_tests.rs +++ b/ipld/cid/tests/serde_tests.rs @@ -1,7 +1,7 @@ // Copyright 2020 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT -#![cfg(feature = "serde_derive")] +#![cfg(feature = "cbor")] use forest_cid::Cid; use multihash::Blake2b256; diff --git a/tests/serialization_tests/Cargo.toml b/tests/serialization_tests/Cargo.toml index 891e45ca5c29..448d1d8d846f 100644 --- a/tests/serialization_tests/Cargo.toml +++ b/tests/serialization_tests/Cargo.toml @@ -9,7 +9,7 @@ serde_tests = [] [dependencies] serde = { version = "1.0", features = ["derive"] } -cid = { package = "forest_cid", path = "../../ipld/cid", features = ["serde_derive"] } +cid = { package = "forest_cid", path = "../../ipld/cid", features = ["cbor"] } crypto = { path = "../../crypto" } base64 = "0.12.0" diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 29908fcbf06f..97dbdd740b38 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -9,7 +9,7 @@ num-bigint = { path = "../utils/bigint", package = "forest_bigint" } address = { package = "forest_address", path = "./address" } encoding = { package = "forest_encoding", path = "../encoding" } serde = { version = "1.0", features = ["derive"] } -cid = { package = "forest_cid", path = "../ipld/cid", features = ["serde_derive"] } +cid = { package = "forest_cid", path = "../ipld/cid", features = ["cbor"] } num-traits = "0.2" num-derive = "0.3.0" clock = { path = "../node/clock" } diff --git a/vm/actor/Cargo.toml b/vm/actor/Cargo.toml index e8644c553cf2..f5a165b4b44b 100644 --- a/vm/actor/Cargo.toml +++ b/vm/actor/Cargo.toml @@ -13,7 +13,7 @@ encoding = { package = "forest_encoding", path = "../../encoding" } num-traits = "0.2" num-derive = "0.3.0" clock = { path = "../../node/clock" } -cid = { package = "forest_cid", path = "../../ipld/cid", features = ["serde_derive"] } +cid = { package = "forest_cid", path = "../../ipld/cid", features = ["cbor"] } serde = { version = "1.0", features = ["derive"] } lazy_static = "1.4.0" ipld_blockstore = { path = "../../ipld/blockstore" } From d744f605b8049d11c87ae0b73d79d3ed62a76cc4 Mon Sep 17 00:00:00 2001 From: austinabell Date: Sat, 2 May 2020 10:53:39 -0400 Subject: [PATCH 02/20] Implement JSON serialization for CIDs --- ipld/cid/Cargo.toml | 6 +++- ipld/cid/src/json.rs | 37 ++++++++++++++++++++++ ipld/cid/src/lib.rs | 3 ++ ipld/cid/tests/json_tests.rs | 61 ++++++++++++++++++++++++++++++++++++ 4 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 ipld/cid/src/json.rs create mode 100644 ipld/cid/tests/json_tests.rs diff --git a/ipld/cid/Cargo.toml b/ipld/cid/Cargo.toml index 42be42251695..ef598e4b2026 100644 --- a/ipld/cid/Cargo.toml +++ b/ipld/cid/Cargo.toml @@ -5,7 +5,7 @@ authors = ["ChainSafe Systems "] edition = "2018" [package.metadata.docs.rs] -features = ["cbor"] +features = ["cbor", "json"] [dependencies] multihash = "0.10.0" @@ -16,5 +16,9 @@ serde_cbor = { version = "0.11.0", features = ["tags"], optional = true } serde_bytes = { version = "0.11.3", optional = true } thiserror = "1.0" +[dev-dependencies] +serde_json = "1.0" + [features] cbor = ["serde", "serde_bytes", "serde_cbor"] +json = ["serde"] diff --git a/ipld/cid/src/json.rs b/ipld/cid/src/json.rs new file mode 100644 index 000000000000..58da4c92a9b1 --- /dev/null +++ b/ipld/cid/src/json.rs @@ -0,0 +1,37 @@ +// Copyright 2020 ChainSafe Systems +// SPDX-License-Identifier: Apache-2.0, MIT + +use super::Cid; +use serde::{de, Deserialize, Deserializer, Serialize, Serializer}; + +/// Wrapper for serializing and deserializing a Cid from JSON. +#[derive(Deserialize, Serialize)] +#[serde(transparent)] +pub struct CidJson(#[serde(with = "self")] pub Cid); + +/// Wrapper for serializing a cid reference to JSON. +#[derive(Serialize)] +#[serde(transparent)] +pub struct CidJsonRef<'a>(#[serde(with = "self")] pub &'a Cid); + +pub fn serialize(c: &Cid, serializer: S) -> Result +where + S: Serializer, +{ + CidMap { cid: c.to_string() }.serialize(serializer) +} + +pub fn deserialize<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let CidMap { cid } = Deserialize::deserialize(deserializer)?; + cid.parse().map_err(de::Error::custom) +} + +/// Struct just used as a helper to serialize a cid into a map with key "/" +#[derive(Serialize, Deserialize)] +struct CidMap { + #[serde(rename = "/")] + cid: String, +} diff --git a/ipld/cid/src/lib.rs b/ipld/cid/src/lib.rs index ec4bf4c33fc1..7593598f2381 100644 --- a/ipld/cid/src/lib.rs +++ b/ipld/cid/src/lib.rs @@ -31,6 +31,9 @@ const CBOR_TAG_CID: u64 = 42; #[cfg(feature = "cbor")] const MULTIBASE_IDENTITY: u8 = 0; +#[cfg(feature = "json")] +pub mod json; + /// Prefix represents all metadata of a CID, without the actual content. #[derive(PartialEq, Eq, Clone, Debug)] pub struct Prefix { diff --git a/ipld/cid/tests/json_tests.rs b/ipld/cid/tests/json_tests.rs new file mode 100644 index 000000000000..e66bde3127b2 --- /dev/null +++ b/ipld/cid/tests/json_tests.rs @@ -0,0 +1,61 @@ +// Copyright 2020 ChainSafe Systems +// SPDX-License-Identifier: Apache-2.0, MIT + +#![cfg(feature = "json")] + +use forest_cid::{ + json::{self, CidJson, CidJsonRef}, + Cid, +}; +use serde::{Deserialize, Serialize}; +use serde_json::{from_str, to_string}; + +#[test] +fn symmetric_json_serialization() { + let cid: Cid = "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n" + .parse() + .unwrap(); + let cid_json = r#"{"/":"QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n"}"#; + + // Deserialize + let CidJson(cid_d) = from_str(cid_json).unwrap(); + assert_eq!(&cid_d, &cid, "Deserialized cid does not match"); + + // Serialize + let ser_cid = to_string(&CidJsonRef(&cid_d)).unwrap(); + assert_eq!(ser_cid, cid_json); +} + +#[test] +fn annotating_struct_json() { + #[derive(Serialize, Deserialize, Debug, PartialEq)] + struct TestStruct { + #[serde(with = "json")] + cid_one: Cid, + #[serde(with = "json")] + cid_two: Cid, + other: String, + } + let test_json = r#" + { + "cid_one": { + "/": "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n" + }, + "cid_two": { + "/": "bafy2bzaceaa466o2jfc4g4ggrmtf55ygigvkmxvkr5mvhy4qbwlxetbmlkqjk" + }, + "other": "Some data" + } + "#; + let expected = TestStruct { + cid_one: "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n" + .parse() + .unwrap(), + cid_two: "bafy2bzaceaa466o2jfc4g4ggrmtf55ygigvkmxvkr5mvhy4qbwlxetbmlkqjk" + .parse() + .unwrap(), + other: "Some data".to_owned(), + }; + + assert_eq!(from_str::(test_json).unwrap(), expected); +} From b51f6f7a16f0d7fdff5fc00f33c6220b80b2a2f9 Mon Sep 17 00:00:00 2001 From: austinabell Date: Sat, 2 May 2020 13:25:14 -0400 Subject: [PATCH 03/20] DagJSON IPLD impl --- ipld/Cargo.toml | 2 + ipld/src/json.rs | 140 +++++++++++++++++++++++++++++++++++++++++++++++ ipld/src/lib.rs | 1 + 3 files changed, 143 insertions(+) create mode 100644 ipld/src/json.rs diff --git a/ipld/Cargo.toml b/ipld/Cargo.toml index b414f5f2964b..33645a87d752 100644 --- a/ipld/Cargo.toml +++ b/ipld/Cargo.toml @@ -8,6 +8,8 @@ edition = "2018" encoding = { package = "forest_encoding", path = "../encoding" } serde = { version = "1.0", features = ["derive"] } thiserror = "1.0" +serde_json = "1.0" +base64 = "0.12.0" [dependencies.cid] package = "forest_cid" diff --git a/ipld/src/json.rs b/ipld/src/json.rs new file mode 100644 index 000000000000..f5a0bc2968dd --- /dev/null +++ b/ipld/src/json.rs @@ -0,0 +1,140 @@ +// Copyright 2020 ChainSafe Systems +// SPDX-License-Identifier: Apache-2.0, MIT + +use super::Ipld; +use serde::{de, ser, Deserialize, Deserializer, Serialize, Serializer}; +use serde_json::{json, Map, Number, Value as JsonValue}; +use std::collections::BTreeMap; +use std::convert::{TryFrom, TryInto}; + +/// Wrapper for serializing and deserializing a Ipld from JSON. +#[derive(Deserialize, Serialize)] +#[serde(transparent)] +pub struct IpldJson(#[serde(with = "self")] pub Ipld); + +/// Wrapper for serializing a ipld reference to JSON. +#[derive(Serialize)] +#[serde(transparent)] +pub struct IpldJsonRef<'a>(#[serde(with = "self")] pub &'a Ipld); + +// TODO serialize and deserialize should not have to go through a Json value buffer +// (unnecessary clones and copies) but the efficiency for JSON shouldn't matter too much + +pub fn serialize(ipld: &Ipld, serializer: S) -> Result +where + S: Serializer, +{ + JsonValue::try_from(ipld) + .map_err(ser::Error::custom)? + .serialize(serializer) +} + +pub fn deserialize<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + Ok(JsonValue::deserialize(deserializer)? + .try_into() + .map_err(de::Error::custom)?) +} + +impl TryFrom<&Ipld> for JsonValue { + type Error = &'static str; + + fn try_from(ipld: &Ipld) -> Result { + let val = match ipld { + Ipld::Null => JsonValue::Null, + Ipld::Bool(b) => JsonValue::Bool(*b), + Ipld::Integer(i) => { + let i = *i; + if i < 0 { + let c = i64::try_from(i).map_err(|_| "Invalid precision for json number")?; + JsonValue::Number(c.into()) + } else { + let c = u64::try_from(i).map_err(|_| "Invalid precision for json number")?; + JsonValue::Number(c.into()) + } + } + Ipld::Float(f) => JsonValue::Number( + Number::from_f64(*f).ok_or("Float does not have finite precision")?, + ), + Ipld::String(s) => JsonValue::String(s.clone()), + Ipld::Bytes(bz) => json!({ "/": { "base64": base64::encode(bz) } }), + Ipld::Link(cid) => json!({ "/": cid.to_string() }), + Ipld::List(list) => JsonValue::Array( + list.into_iter() + .map(JsonValue::try_from) + .collect::>()?, + ), + Ipld::Map(map) => { + let mut new = Map::new(); + for (k, v) in map.into_iter() { + new.insert(k.to_string(), JsonValue::try_from(v)?); + } + JsonValue::Object(new) + } + }; + Ok(val) + } +} + +impl TryFrom for Ipld { + type Error = &'static str; + + fn try_from(json: JsonValue) -> Result { + let val = match json { + JsonValue::Null => Ipld::Null, + JsonValue::Bool(b) => Ipld::Bool(b), + JsonValue::Number(n) => { + if let Some(v) = n.as_u64() { + Ipld::Integer(v.into()) + } else if let Some(v) = n.as_i64() { + Ipld::Integer(v.into()) + } else if let Some(v) = n.as_f64() { + Ipld::Float(v) + } else { + // Json number can only be one of those three types + unreachable!() + } + } + JsonValue::String(s) => Ipld::String(s), + JsonValue::Array(values) => Ipld::List( + values + .into_iter() + .map(Ipld::try_from) + .collect::>()?, + ), + JsonValue::Object(map) => { + // Check for escaped values (Bytes and Cids) + if map.len() == 1 { + if let Some(v) = map.get("/") { + match v { + JsonValue::String(s) => { + // Json block is a Cid + return Ok(Ipld::Link( + s.parse().map_err(|_| "Failed not parse cid string")?, + )); + } + JsonValue::Object(obj) => { + // Are other bytes encoding types supported? + if let Some(JsonValue::String(bz)) = obj.get("base64") { + return Ok(Ipld::Bytes( + base64::decode(bz) + .map_err(|_| "Failed to parse base64 bytes")?, + )); + } + } + _ => (), + } + } + } + let mut new = BTreeMap::new(); + for (k, v) in map.into_iter() { + new.insert(k, Ipld::try_from(v)?); + } + Ipld::Map(new) + } + }; + Ok(val) + } +} diff --git a/ipld/src/lib.rs b/ipld/src/lib.rs index f0c6c4685b37..d7bbb03e97ba 100644 --- a/ipld/src/lib.rs +++ b/ipld/src/lib.rs @@ -3,6 +3,7 @@ mod de; mod error; +pub mod json; mod ser; pub use self::error::Error; From 00f66d2c5ef4875a4f6f40ec7f3be10dffde57a7 Mon Sep 17 00:00:00 2001 From: austinabell Date: Sat, 2 May 2020 14:03:37 -0400 Subject: [PATCH 04/20] Symmetric serialization tests --- ipld/tests/json_tests.rs | 104 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 ipld/tests/json_tests.rs diff --git a/ipld/tests/json_tests.rs b/ipld/tests/json_tests.rs new file mode 100644 index 000000000000..9623e41b56ae --- /dev/null +++ b/ipld/tests/json_tests.rs @@ -0,0 +1,104 @@ +// Copyright 2020 ChainSafe Systems +// SPDX-License-Identifier: Apache-2.0, MIT + +use forest_ipld::{ + json::{self, IpldJson, IpldJsonRef}, + Ipld, +}; +use serde::{Deserialize, Serialize}; +use serde_json::{from_str, to_string, Value}; +use std::collections::BTreeMap; +use std::iter::FromIterator; + +#[test] +fn deserialize_json_symmetric() { + let test_json = r#" + { + "link": { + "/": "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n" + }, + "bytes": { + "/": { "base64": "VGhlIHF1aQ==" } + }, + "string": "Some data", + "float": 10.5, + "integer": 8, + "neg_integer": -20, + "null": null, + "list": [null, { "/": "bafy2bzaceaa466o2jfc4g4ggrmtf55ygigvkmxvkr5mvhy4qbwlxetbmlkqjk" }, 1] + } + "#; + let expected = Ipld::Map(BTreeMap::from_iter( + [ + ( + "link".to_owned(), + Ipld::Link( + "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n" + .parse() + .unwrap(), + ), + ), + ( + "bytes".to_owned(), + Ipld::Bytes([0x54, 0x68, 0x65, 0x20, 0x71, 0x75, 0x69].to_vec()), + ), + ("string".to_owned(), Ipld::String("Some data".to_owned())), + ("float".to_owned(), Ipld::Float(10.5)), + ("integer".to_owned(), Ipld::Integer(8)), + ("neg_integer".to_owned(), Ipld::Integer(-20)), + ("null".to_owned(), Ipld::Null), + ( + "list".to_owned(), + Ipld::List(vec![ + Ipld::Null, + Ipld::Link( + "bafy2bzaceaa466o2jfc4g4ggrmtf55ygigvkmxvkr5mvhy4qbwlxetbmlkqjk" + .parse() + .unwrap(), + ), + Ipld::Integer(1), + ]), + ), + ] + .to_vec() + .into_iter(), + )); + + // Assert deserializing into expected Ipld + let IpldJson(ipld_d) = from_str(test_json).unwrap(); + assert_eq!(&ipld_d, &expected, "Deserialized ipld does not match"); + + // Symmetric tests + let ser_json = to_string(&IpldJsonRef(&expected)).unwrap(); + let IpldJson(ipld_d) = from_str(&ser_json).unwrap(); + assert_eq!(&ipld_d, &expected, "Deserialized ipld does not match"); +} + +// #[derive(Serialize, Deserialize, Clone)] +// struct TestStruct { +// name: String, +// details: Cid, +// } + +// #[test] +// fn encode_new_type() { +// let details = Cid::new_from_cbor(&[1, 2, 3], Blake2b256); +// let name = "Test".to_string(); +// let t_struct = TestStruct { +// name: name.clone(), +// details: details.clone(), +// }; +// let struct_encoded = to_vec(&t_struct).unwrap(); + +// // Test to make sure struct can be encoded and decoded without IPLD +// let struct_decoded: TestStruct = from_slice(&struct_encoded).unwrap(); +// assert_eq!(&struct_decoded.name, &name); +// assert_eq!(&struct_decoded.details, &details.clone()); + +// // Test ipld decoding +// let ipld_decoded: Ipld = from_slice(&struct_encoded).unwrap(); +// let mut e_map = BTreeMap::::new(); +// e_map.insert("details".to_string(), Ipld::Link(details)); +// e_map.insert("name".to_string(), Ipld::String(name)); +// assert_eq!(&ipld_decoded, &Ipld::Map(e_map)); +// } From 136ad963f5be1ec2414a9a82e29a1b05a25223c8 Mon Sep 17 00:00:00 2001 From: austinabell Date: Sat, 2 May 2020 14:23:00 -0400 Subject: [PATCH 05/20] Annotation test --- ipld/tests/json_tests.rs | 57 ++++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/ipld/tests/json_tests.rs b/ipld/tests/json_tests.rs index 9623e41b56ae..0e488a6bb4e7 100644 --- a/ipld/tests/json_tests.rs +++ b/ipld/tests/json_tests.rs @@ -6,7 +6,7 @@ use forest_ipld::{ Ipld, }; use serde::{Deserialize, Serialize}; -use serde_json::{from_str, to_string, Value}; +use serde_json::{from_str, to_string}; use std::collections::BTreeMap; use std::iter::FromIterator; @@ -74,31 +74,32 @@ fn deserialize_json_symmetric() { assert_eq!(&ipld_d, &expected, "Deserialized ipld does not match"); } -// #[derive(Serialize, Deserialize, Clone)] -// struct TestStruct { -// name: String, -// details: Cid, -// } - -// #[test] -// fn encode_new_type() { -// let details = Cid::new_from_cbor(&[1, 2, 3], Blake2b256); -// let name = "Test".to_string(); -// let t_struct = TestStruct { -// name: name.clone(), -// details: details.clone(), -// }; -// let struct_encoded = to_vec(&t_struct).unwrap(); - -// // Test to make sure struct can be encoded and decoded without IPLD -// let struct_decoded: TestStruct = from_slice(&struct_encoded).unwrap(); -// assert_eq!(&struct_decoded.name, &name); -// assert_eq!(&struct_decoded.details, &details.clone()); +#[test] +fn annotating_struct_json() { + #[derive(Serialize, Deserialize, Debug, PartialEq)] + struct TestStruct { + #[serde(with = "json")] + one: Ipld, + other: String, + } + let test_json = r#" + { + "one": [{ "/": "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n" }, null, 8], + "other": "Some data" + } + "#; + let expected = TestStruct { + one: Ipld::List(vec![ + Ipld::Link( + "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n" + .parse() + .unwrap(), + ), + Ipld::Null, + Ipld::Integer(8), + ]), + other: "Some data".to_owned(), + }; -// // Test ipld decoding -// let ipld_decoded: Ipld = from_slice(&struct_encoded).unwrap(); -// let mut e_map = BTreeMap::::new(); -// e_map.insert("details".to_string(), Ipld::Link(details)); -// e_map.insert("name".to_string(), Ipld::String(name)); -// assert_eq!(&ipld_decoded, &Ipld::Map(e_map)); -// } + assert_eq!(from_str::(test_json).unwrap(), expected); +} From ac88efc2141cd9c28d9da3fd6fe2f2bbe71a52b1 Mon Sep 17 00:00:00 2001 From: austinabell Date: Sat, 2 May 2020 14:31:09 -0400 Subject: [PATCH 06/20] Add test for edge case --- ipld/tests/json_tests.rs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/ipld/tests/json_tests.rs b/ipld/tests/json_tests.rs index 0e488a6bb4e7..16220d928162 100644 --- a/ipld/tests/json_tests.rs +++ b/ipld/tests/json_tests.rs @@ -103,3 +103,24 @@ fn annotating_struct_json() { assert_eq!(from_str::(test_json).unwrap(), expected); } + +#[test] +fn link_edge_case() { + // Test ported from go-ipld-prime (making sure edge case is handled) + let test_json = r#"{"/":{"/":"QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n"}}"#; + let expected = Ipld::Map(BTreeMap::from_iter( + [( + "/".to_owned(), + Ipld::Link( + "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n" + .parse() + .unwrap(), + ), + )] + .to_vec(), + )); + + let IpldJson(ipld_d) = from_str(test_json).unwrap(); + assert_eq!(&ipld_d, &expected, "Deserialized ipld does not match"); +} + From 05e968b1e8f7ef7bb39c241710a5b20481d83cc8 Mon Sep 17 00:00:00 2001 From: austinabell Date: Sat, 2 May 2020 14:37:27 -0400 Subject: [PATCH 07/20] gate IPLD json by feature --- Makefile | 2 +- ipld/Cargo.toml | 8 +++++++- ipld/src/json.rs | 4 ++-- ipld/src/lib.rs | 4 +++- ipld/tests/json_tests.rs | 3 ++- 5 files changed, 15 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 6d463306add7..582e4720e6bd 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,7 @@ clean: lint: license clean cargo fmt --all - cargo clippy -- -D warnings + cargo clippy --all-features -- -D warnings build: cargo build --bin forest diff --git a/ipld/Cargo.toml b/ipld/Cargo.toml index 33645a87d752..6cd479ab00a7 100644 --- a/ipld/Cargo.toml +++ b/ipld/Cargo.toml @@ -4,14 +4,20 @@ version = "0.1.0" authors = ["ChainSafe Systems "] edition = "2018" +[package.metadata.docs.rs] +features = ["json"] + [dependencies] encoding = { package = "forest_encoding", path = "../encoding" } serde = { version = "1.0", features = ["derive"] } thiserror = "1.0" -serde_json = "1.0" +serde_json = { version = "1.0", optional = true } base64 = "0.12.0" [dependencies.cid] package = "forest_cid" path = "../ipld/cid" features = ["cbor"] + +[features] +json = ["serde_json"] diff --git a/ipld/src/json.rs b/ipld/src/json.rs index f5a0bc2968dd..7941879935eb 100644 --- a/ipld/src/json.rs +++ b/ipld/src/json.rs @@ -62,13 +62,13 @@ impl TryFrom<&Ipld> for JsonValue { Ipld::Bytes(bz) => json!({ "/": { "base64": base64::encode(bz) } }), Ipld::Link(cid) => json!({ "/": cid.to_string() }), Ipld::List(list) => JsonValue::Array( - list.into_iter() + list.iter() .map(JsonValue::try_from) .collect::>()?, ), Ipld::Map(map) => { let mut new = Map::new(); - for (k, v) in map.into_iter() { + for (k, v) in map.iter() { new.insert(k.to_string(), JsonValue::try_from(v)?); } JsonValue::Object(new) diff --git a/ipld/src/lib.rs b/ipld/src/lib.rs index d7bbb03e97ba..c36414ca731f 100644 --- a/ipld/src/lib.rs +++ b/ipld/src/lib.rs @@ -3,9 +3,11 @@ mod de; mod error; -pub mod json; mod ser; +#[cfg(feature = "json")] +pub mod json; + pub use self::error::Error; use cid::Cid; diff --git a/ipld/tests/json_tests.rs b/ipld/tests/json_tests.rs index 16220d928162..140274d30e1e 100644 --- a/ipld/tests/json_tests.rs +++ b/ipld/tests/json_tests.rs @@ -1,6 +1,8 @@ // Copyright 2020 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT +#![cfg(feature = "json")] + use forest_ipld::{ json::{self, IpldJson, IpldJsonRef}, Ipld, @@ -123,4 +125,3 @@ fn link_edge_case() { let IpldJson(ipld_d) = from_str(test_json).unwrap(); assert_eq!(&ipld_d, &expected, "Deserialized ipld does not match"); } - From d6e2315936ca714ab4a7afea9d10b41575babd4f Mon Sep 17 00:00:00 2001 From: austinabell Date: Sat, 2 May 2020 15:02:43 -0400 Subject: [PATCH 08/20] Cleanup --- ipld/Cargo.toml | 4 +-- ipld/hamt/src/node.rs | 2 +- tests/serialization_tests/Cargo.toml | 10 +++---- tests/serialization_tests/src/lib.rs | 13 --------- .../tests/block_header_ser.rs | 29 +++++++++++-------- vm/interpreter/src/default_runtime.rs | 2 +- vm/interpreter/src/vm.rs | 8 ++--- vm/runtime/Cargo.toml | 3 +- 8 files changed, 31 insertions(+), 40 deletions(-) diff --git a/ipld/Cargo.toml b/ipld/Cargo.toml index 6cd479ab00a7..f415c675f137 100644 --- a/ipld/Cargo.toml +++ b/ipld/Cargo.toml @@ -12,7 +12,7 @@ encoding = { package = "forest_encoding", path = "../encoding" } serde = { version = "1.0", features = ["derive"] } thiserror = "1.0" serde_json = { version = "1.0", optional = true } -base64 = "0.12.0" +base64 = { version = "0.12.0", optional = true } [dependencies.cid] package = "forest_cid" @@ -20,4 +20,4 @@ path = "../ipld/cid" features = ["cbor"] [features] -json = ["serde_json"] +json = ["serde_json", "base64"] diff --git a/ipld/hamt/src/node.rs b/ipld/hamt/src/node.rs index f1469965a156..1cbc9c576dd9 100644 --- a/ipld/hamt/src/node.rs +++ b/ipld/hamt/src/node.rs @@ -255,7 +255,7 @@ where Pointer::Values(vals) => { // Update, if the key already exists. if let Some(i) = vals.iter().position(|p| p.key() == &key) { - std::mem::replace(&mut vals[i].1, value); + vals[i].1 = value; return Ok(()); } diff --git a/tests/serialization_tests/Cargo.toml b/tests/serialization_tests/Cargo.toml index 448d1d8d846f..18d9c4b96103 100644 --- a/tests/serialization_tests/Cargo.toml +++ b/tests/serialization_tests/Cargo.toml @@ -5,15 +5,15 @@ authors = ["ChainSafe Systems "] edition = "2018" [features] -serde_tests = [] +serde_tests = ["serde", "crypto", "base64"] [dependencies] -serde = { version = "1.0", features = ["derive"] } -cid = { package = "forest_cid", path = "../../ipld/cid", features = ["cbor"] } -crypto = { path = "../../crypto" } -base64 = "0.12.0" +serde = { version = "1.0", features = ["derive"], optional = true } +crypto = { path = "../../crypto", optional = true } +base64 = { version = "0.12.0", optional = true } [dev-dependencies] +cid = { package = "forest_cid", path = "../../ipld/cid", features = ["cbor", "json"] } serde_json = "1.0" hex = "0.4.2" vm = { path = "../../vm" } diff --git a/tests/serialization_tests/src/lib.rs b/tests/serialization_tests/src/lib.rs index e133e346fc78..eda98cf64600 100644 --- a/tests/serialization_tests/src/lib.rs +++ b/tests/serialization_tests/src/lib.rs @@ -3,7 +3,6 @@ #![cfg(feature = "serde_tests")] -use cid::Cid; use crypto::Signature; use serde::Deserialize; @@ -24,15 +23,3 @@ impl From for Signature { } } } - -#[derive(Debug, Deserialize)] -pub struct CidVector { - #[serde(alias = "/")] - c_str: String, -} - -impl From for Cid { - fn from(c: CidVector) -> Self { - c.c_str.parse().unwrap() - } -} diff --git a/tests/serialization_tests/tests/block_header_ser.rs b/tests/serialization_tests/tests/block_header_ser.rs index 572f45ed8cb8..a1a29c429c5e 100644 --- a/tests/serialization_tests/tests/block_header_ser.rs +++ b/tests/serialization_tests/tests/block_header_ser.rs @@ -3,14 +3,17 @@ #![cfg(feature = "serde_tests")] -use cid::Cid; +use cid::{ + json::{self, CidJson}, + Cid, +}; use crypto::VRFProof; use encoding::{from_slice, to_vec}; use forest_blocks::{BlockHeader, EPostProof, EPostTicket, Ticket, TipSetKeys}; use hex::encode; use num_traits::FromPrimitive; use serde::Deserialize; -use serialization_tests::{CidVector, SignatureVector}; +use serialization_tests::SignatureVector; use std::fs::File; use std::io::prelude::*; use vm::PoStProof; @@ -86,26 +89,28 @@ impl From for EPostProof { } } -#[derive(Debug, Deserialize)] +// TODO update vectors when serialization vectors submodule updated + +#[derive(Deserialize)] struct BlockVector { #[serde(alias = "Miner")] miner: String, #[serde(alias = "Ticket")] ticket: TicketVector, #[serde(alias = "EPostProof")] - e_post: EPoStProofVector, + _e_post: EPoStProofVector, #[serde(alias = "Parents")] - parents: Vec, + parents: Vec, #[serde(alias = "ParentWeight")] parent_weight: String, #[serde(alias = "Height")] epoch: u64, - #[serde(alias = "ParentStateRoot")] - state_root: CidVector, - #[serde(alias = "ParentMessageReceipts")] - message_receipts: CidVector, - #[serde(alias = "Messages")] - messages: CidVector, + #[serde(alias = "ParentStateRoot", with = "json")] + state_root: Cid, + #[serde(alias = "ParentMessageReceipts", with = "json")] + message_receipts: Cid, + #[serde(alias = "Messages", with = "json")] + messages: Cid, #[serde(alias = "BLSAggregate")] bls_agg: SignatureVector, #[serde(alias = "Timestamp")] @@ -120,7 +125,7 @@ impl From for BlockHeader { fn from(v: BlockVector) -> BlockHeader { BlockHeader::builder() .parents(TipSetKeys::new( - v.parents.into_iter().map(|c| Cid::from(c)).collect(), + v.parents.into_iter().map(|c| c.0).collect(), )) .weight(v.parent_weight.parse().unwrap()) .epoch(v.epoch) diff --git a/vm/interpreter/src/default_runtime.rs b/vm/interpreter/src/default_runtime.rs index 6069ebe40519..51efd9c4f265 100644 --- a/vm/interpreter/src/default_runtime.rs +++ b/vm/interpreter/src/default_runtime.rs @@ -296,7 +296,7 @@ where value: &TokenAmount, ) -> Result { let msg = UnsignedMessage::builder() - .to(to.clone()) + .to(*to) .from(*self.message.from()) .method_num(method) .value(value.clone()) diff --git a/vm/interpreter/src/vm.rs b/vm/interpreter/src/vm.rs index 899baef6c832..fb508fdd2970 100644 --- a/vm/interpreter/src/vm.rs +++ b/vm/interpreter/src/vm.rs @@ -117,8 +117,8 @@ where .ok_or_else(|| "Failed to query system actor".to_string())?; let rew_msg = UnsignedMessage::builder() - .from(SYSTEM_ACTOR_ADDR.clone()) - .to(REWARD_ACTOR_ADDR.clone()) + .from(*SYSTEM_ACTOR_ADDR) + .to(*REWARD_ACTOR_ADDR) .sequence(sys_act.sequence) .value(BigUint::zero()) .gas_price(BigUint::zero()) @@ -148,8 +148,8 @@ where .ok_or_else(|| "Failed to query system actor".to_string())?; let cron_msg = UnsignedMessage::builder() - .from(SYSTEM_ACTOR_ADDR.clone()) - .to(CRON_ACTOR_ADDR.clone()) + .from(*SYSTEM_ACTOR_ADDR) + .to(*CRON_ACTOR_ADDR) .sequence(sys_act.sequence) .value(BigUint::zero()) .gas_price(BigUint::zero()) diff --git a/vm/runtime/Cargo.toml b/vm/runtime/Cargo.toml index 6592af14da94..61021a3165e6 100644 --- a/vm/runtime/Cargo.toml +++ b/vm/runtime/Cargo.toml @@ -6,7 +6,6 @@ edition = "2018" [dependencies] vm = { path = "../../vm" } -blocks = { package = "forest_blocks", path = "../../blockchain/blocks/" } crypto = { path = "../../crypto" } address = { package = "forest_address", path = "../address" } message = { package = "forest_message", path = "../message" } @@ -15,4 +14,4 @@ ipld_blockstore = { path = "../../ipld/blockstore" } clock = { path = "../../node/clock" } forest_encoding = { path = "../../encoding" } commcid = { path = "../../utils/commcid" } -filecoin-proofs-api = { git = "https://github.com/filecoin-project/rust-filecoin-proofs-api", rev = "b00d2e26c68e49b81e434739336383304b293395" } \ No newline at end of file +filecoin-proofs-api = { git = "https://github.com/filecoin-project/rust-filecoin-proofs-api", rev = "b00d2e26c68e49b81e434739336383304b293395" } From d53207403ed63bf59717bbe402a278c448d28f79 Mon Sep 17 00:00:00 2001 From: austinabell Date: Sat, 2 May 2020 20:45:32 -0400 Subject: [PATCH 09/20] Implementation of ipld! macro --- ipld/src/lib.rs | 3 + ipld/src/macros.rs | 321 ++++++++++++++++++++++ ipld/tests/{ipld_test.rs => cbor_test.rs} | 11 +- 3 files changed, 329 insertions(+), 6 deletions(-) create mode 100644 ipld/src/macros.rs rename ipld/tests/{ipld_test.rs => cbor_test.rs} (76%) diff --git a/ipld/src/lib.rs b/ipld/src/lib.rs index c36414ca731f..2efc4bbffa62 100644 --- a/ipld/src/lib.rs +++ b/ipld/src/lib.rs @@ -8,6 +8,9 @@ mod ser; #[cfg(feature = "json")] pub mod json; +#[macro_use] +mod macros; + pub use self::error::Error; use cid::Cid; diff --git a/ipld/src/macros.rs b/ipld/src/macros.rs new file mode 100644 index 000000000000..a262a7a2b6d5 --- /dev/null +++ b/ipld/src/macros.rs @@ -0,0 +1,321 @@ +/// Construct a `forest_ipld::Ipld` roughly matching JSON format. This code is a modified version +/// of `serde_json::json` macro, with extensions for being able to indicate links and bytes. +/// These two matterns can be matched by wrapping `Cid` link in `Link(..)` or the `Vec` in +/// `Bytes()`. +/// +/// ``` +/// # use forest_ipld::ipld; +/// # use cid::Cid; +/// # +/// let value = ipld!({ +/// "code": 200, +/// "success": true, +/// "link": "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n".parse::().unwrap(), +/// "bytes": vec![0x1, 0xfa, 0x8b], +/// "payload": { +/// "features": [ +/// "serde", +/// "ipld" +/// ] +/// } +/// }); +/// ``` +/// +/// Variables or expressions can be interpolated into the JSON literal. Any type +/// interpolated into an array element or object value must implement Serde's +/// `Serialize` trait, while any type interpolated into a object key must +/// implement `Into`. If the `Serialize` implementation of the +/// interpolated type decides to fail, or if the interpolated type contains a +/// map with non-string keys, the `ipld!` macro will panic. +/// +/// ``` +/// # use forest_ipld::ipld; +/// # +/// let code = 200; +/// let features = vec!["serde", "ipld"]; +/// +/// let value = ipld!({ +/// "code": code, +/// "success": code == 200, +/// "payload": { +/// features[0]: features[1] +/// } +/// }); +/// ``` +/// +/// Trailing commas are allowed inside both arrays and objects. +/// +/// ``` +/// # use forest_ipld::ipld; +/// # +/// let value = ipld!([ +/// "notice", +/// "the", +/// "trailing", +/// "comma -->", +/// ]); +/// ``` +#[macro_export(local_inner_macros)] +macro_rules! ipld { + // Hide distracting implementation details from the generated rustdoc. + ($($ipld:tt)+) => { + ipld_internal!($($ipld)+) + }; +} + +#[macro_export(local_inner_macros)] +#[doc(hidden)] +macro_rules! ipld_internal { + ////////////////////////////////////////////////////////////////////////// + // TT muncher for parsing the inside of an array [...]. Produces a vec![...] + // of the elements. + // + // Must be invoked as: ipld_internal!(@array [] $($tt)*) + ////////////////////////////////////////////////////////////////////////// + + // Done with trailing comma. + (@array [$($elems:expr,)*]) => { + ipld_internal_vec![$($elems,)*] + }; + + // Done without trailing comma. + (@array [$($elems:expr),*]) => { + ipld_internal_vec![$($elems),*] + }; + + // Next element is `null`. + (@array [$($elems:expr,)*] null $($rest:tt)*) => { + ipld_internal!(@array [$($elems,)* ipld_internal!(null)] $($rest)*) + }; + + // Next element is `true`. + (@array [$($elems:expr,)*] true $($rest:tt)*) => { + ipld_internal!(@array [$($elems,)* ipld_internal!(true)] $($rest)*) + }; + + // Next element is `false`. + (@array [$($elems:expr,)*] false $($rest:tt)*) => { + ipld_internal!(@array [$($elems,)* ipld_internal!(false)] $($rest)*) + }; + + // * Next element is a `Link`. + (@array [$($elems:expr,)*] Link($cid:expr) $($rest:tt)*) => { + ipld_internal!(@array [$($elems,)* ipld_internal!(Link($cid))] $($rest)*) + }; + + // * Next element is `Bytes`. + (@array [$($elems:expr,)*] Bytes($bz:expr) $($rest:tt)*) => { + ipld_internal!(@array [$($elems,)* ipld_internal!(Bytes($bz))] $($rest)*) + }; + + // Next element is an array. + (@array [$($elems:expr,)*] [$($array:tt)*] $($rest:tt)*) => { + ipld_internal!(@array [$($elems,)* ipld_internal!([$($array)*])] $($rest)*) + }; + + // Next element is a map. + (@array [$($elems:expr,)*] {$($map:tt)*} $($rest:tt)*) => { + ipld_internal!(@array [$($elems,)* ipld_internal!({$($map)*})] $($rest)*) + }; + + // Next element is an expression followed by comma. + (@array [$($elems:expr,)*] $next:expr, $($rest:tt)*) => { + ipld_internal!(@array [$($elems,)* ipld_internal!($next),] $($rest)*) + }; + + // Last element is an expression with no trailing comma. + (@array [$($elems:expr,)*] $last:expr) => { + ipld_internal!(@array [$($elems,)* ipld_internal!($last)]) + }; + + // Comma after the most recent element. + (@array [$($elems:expr),*] , $($rest:tt)*) => { + ipld_internal!(@array [$($elems,)*] $($rest)*) + }; + + // Unexpected token after most recent element. + (@array [$($elems:expr),*] $unexpected:tt $($rest:tt)*) => { + ipld_unexpected!($unexpected) + }; + + ////////////////////////////////////////////////////////////////////////// + // TT muncher for parsing the inside of an object {...}. Each entry is + // inserted into the given map variable. + // + // Must be invoked as: ipld_internal!(@object $map () ($($tt)*) ($($tt)*)) + // + // We require two copies of the input tokens so that we can match on one + // copy and trigger errors on the other copy. + ////////////////////////////////////////////////////////////////////////// + + // Done. + (@object $object:ident () () ()) => {}; + + // Insert the current entry followed by trailing comma. + (@object $object:ident [$($key:tt)+] ($value:expr) , $($rest:tt)*) => { + let _ = $object.insert(($($key)+).into(), $value); + ipld_internal!(@object $object () ($($rest)*) ($($rest)*)); + }; + + // Current entry followed by unexpected token. + (@object $object:ident [$($key:tt)+] ($value:expr) $unexpected:tt $($rest:tt)*) => { + ipld_unexpected!($unexpected); + }; + + // Insert the last entry without trailing comma. + (@object $object:ident [$($key:tt)+] ($value:expr)) => { + let _ = $object.insert(($($key)+).into(), $value); + }; + + // Next value is `null`. + (@object $object:ident ($($key:tt)+) (: null $($rest:tt)*) $copy:tt) => { + ipld_internal!(@object $object [$($key)+] (ipld_internal!(null)) $($rest)*); + }; + + // Next value is `true`. + (@object $object:ident ($($key:tt)+) (: true $($rest:tt)*) $copy:tt) => { + ipld_internal!(@object $object [$($key)+] (ipld_internal!(true)) $($rest)*); + }; + + // Next value is `false`. + (@object $object:ident ($($key:tt)+) (: false $($rest:tt)*) $copy:tt) => { + ipld_internal!(@object $object [$($key)+] (ipld_internal!(false)) $($rest)*); + }; + + // * Next value is a `Link`. + (@object $object:ident ($($key:tt)+) (: Link($cid:expr) $($rest:tt)*) $copy:tt) => { + ipld_internal!(@object $object [$($key)+] (ipld_internal!(Link($cid))) $($rest)*); + }; + + // * Next value is `Bytes` + (@object $object:ident ($($key:tt)+) (: Bytes($bz:expr) $($rest:tt)*) $copy:tt) => { + ipld_internal!(@object $object [$($key)+] (ipld_internal!(Bytes($bz))) $($rest)*); + }; + + // Next value is an array. + (@object $object:ident ($($key:tt)+) (: [$($array:tt)*] $($rest:tt)*) $copy:tt) => { + ipld_internal!(@object $object [$($key)+] (ipld_internal!([$($array)*])) $($rest)*); + }; + + // Next value is a map. + (@object $object:ident ($($key:tt)+) (: {$($map:tt)*} $($rest:tt)*) $copy:tt) => { + ipld_internal!(@object $object [$($key)+] (ipld_internal!({$($map)*})) $($rest)*); + }; + + // Next value is an expression followed by comma. + (@object $object:ident ($($key:tt)+) (: $value:expr , $($rest:tt)*) $copy:tt) => { + ipld_internal!(@object $object [$($key)+] (ipld_internal!($value)) , $($rest)*); + }; + + // Last value is an expression with no trailing comma. + (@object $object:ident ($($key:tt)+) (: $value:expr) $copy:tt) => { + ipld_internal!(@object $object [$($key)+] (ipld_internal!($value))); + }; + + // Missing value for last entry. Trigger a reasonable error message. + (@object $object:ident ($($key:tt)+) (:) $copy:tt) => { + // "unexpected end of macro invocation" + ipld_internal!(); + }; + + // Missing colon and value for last entry. Trigger a reasonable error + // message. + (@object $object:ident ($($key:tt)+) () $copy:tt) => { + // "unexpected end of macro invocation" + ipld_internal!(); + }; + + // Misplaced colon. Trigger a reasonable error message. + (@object $object:ident () (: $($rest:tt)*) ($colon:tt $($copy:tt)*)) => { + // Takes no arguments so "no rules expected the token `:`". + ipld_unexpected!($colon); + }; + + // Found a comma inside a key. Trigger a reasonable error message. + (@object $object:ident ($($key:tt)*) (, $($rest:tt)*) ($comma:tt $($copy:tt)*)) => { + // Takes no arguments so "no rules expected the token `,`". + ipld_unexpected!($comma); + }; + + // Key is fully parenthesized. This avoids clippy double_parens false + // positives because the parenthesization may be necessary here. + (@object $object:ident () (($key:expr) : $($rest:tt)*) $copy:tt) => { + ipld_internal!(@object $object ($key) (: $($rest)*) (: $($rest)*)); + }; + + // Munch a token into the current key. + (@object $object:ident ($($key:tt)*) ($tt:tt $($rest:tt)*) $copy:tt) => { + ipld_internal!(@object $object ($($key)* $tt) ($($rest)*) ($($rest)*)); + }; + + ////////////////////////////////////////////////////////////////////////// + // The main implementation. + // + // Must be invoked as: ipld_internal!($($ipld)+) + ////////////////////////////////////////////////////////////////////////// + + (null) => { + $crate::Ipld::Null + }; + + (true) => { + $crate::Ipld::Bool(true) + }; + + (false) => { + $crate::Ipld::Bool(false) + }; + + // * Cid link pattern + (Link($cid:expr)) => { + $crate::Ipld::Link($cid) + }; + + // * Bytes pattern + (Bytes($bz:expr)) => { + $crate::Ipld::Bytes($bz) + }; + + ([]) => { + $crate::Ipld::List(ipld_internal_vec![]) + }; + + ([ $($tt:tt)+ ]) => { + $crate::Ipld::List(ipld_internal!(@array [] $($tt)+)) + }; + + ({}) => { + $crate::Ipld::Map(::std::collections::BTreeMap::new()) + }; + + ({ $($tt:tt)+ }) => { + $crate::Ipld::Map({ + let mut object = ::std::collections::BTreeMap::new(); + ipld_internal!(@object object () ($($tt)+) ($($tt)+)); + object + }) + }; + + // Any Serialize type: numbers, strings, struct literals, variables etc. + // Must be below every other rule. + ($other:expr) => { + $crate::to_ipld(&$other).unwrap() + }; +} + +// The ipld_internal macro above cannot invoke vec directly because it uses +// local_inner_macros. A vec invocation there would resolve to $crate::vec. +// Instead invoke vec here outside of local_inner_macros. +#[macro_export] +#[doc(hidden)] +macro_rules! ipld_internal_vec { + ($($content:tt)*) => { + vec![$($content)*] + }; +} + +#[macro_export] +#[doc(hidden)] +macro_rules! ipld_unexpected { + () => {}; +} diff --git a/ipld/tests/ipld_test.rs b/ipld/tests/cbor_test.rs similarity index 76% rename from ipld/tests/ipld_test.rs rename to ipld/tests/cbor_test.rs index cc4e7c61fc1f..6ff261052beb 100644 --- a/ipld/tests/ipld_test.rs +++ b/ipld/tests/cbor_test.rs @@ -3,9 +3,8 @@ use cid::{multihash::Blake2b256, Cid}; use encoding::{from_slice, to_vec}; -use forest_ipld::Ipld; +use forest_ipld::{ipld, Ipld}; use serde::{Deserialize, Serialize}; -use std::collections::BTreeMap; #[derive(Serialize, Deserialize, Clone)] struct TestStruct { @@ -30,8 +29,8 @@ fn encode_new_type() { // Test ipld decoding let ipld_decoded: Ipld = from_slice(&struct_encoded).unwrap(); - let mut e_map = BTreeMap::::new(); - e_map.insert("details".to_string(), Ipld::Link(details)); - e_map.insert("name".to_string(), Ipld::String(name)); - assert_eq!(&ipld_decoded, &Ipld::Map(e_map)); + assert_eq!( + &ipld_decoded, + &ipld!({"details": Link(details), "name": "Test"}) + ); } From d4f24fbace48056c077b77a6e5298446348072f9 Mon Sep 17 00:00:00 2001 From: austinabell Date: Sat, 2 May 2020 20:55:18 -0400 Subject: [PATCH 10/20] document ipld enum --- ipld/src/lib.rs | 65 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) diff --git a/ipld/src/lib.rs b/ipld/src/lib.rs index 2efc4bbffa62..0361349e90f9 100644 --- a/ipld/src/lib.rs +++ b/ipld/src/lib.rs @@ -20,17 +20,80 @@ use serde::de::DeserializeOwned; use serde::Serialize; use std::collections::BTreeMap; -/// Represents IPLD data structure used when serializing and deserializing data +/// Represents IPLD data structure used when serializing and deserializing data. #[derive(Debug, Clone, PartialEq)] pub enum Ipld { + /// Represents a null value. + /// + /// ``` + /// # use forest_ipld::ipld; + /// let v = ipld!(null); + /// ``` Null, + + /// Represents a boolean value. + /// + /// ``` + /// # use forest_ipld::ipld; + /// let v = ipld!(true); + /// ``` Bool(bool), + + /// Represents a signed integer value. + /// + /// ``` + /// # use forest_ipld::ipld; + /// let v = ipld!(28); + /// ``` Integer(i128), + + /// Represents a floating point value. + /// + /// ``` + /// # use forest_ipld::ipld; + /// let v = ipld!(8.5); + /// ``` Float(f64), + + /// Represents a String. + /// + /// ``` + /// # use forest_ipld::ipld; + /// let v = ipld!("string"); + /// ``` String(String), + + /// Represents Bytes. + /// + /// ``` + /// # use forest_ipld::ipld; + /// let v = ipld!(Bytes(vec![0x98, 0x8, 0x2a, 0xff])); + /// ``` Bytes(Vec), + + /// Represents List of IPLD objects. + /// + /// ``` + /// # use forest_ipld::ipld; + /// let v = ipld!([1, "string", null]); + /// ``` List(Vec), + + /// Represents a map of strings to Ipld objects. + /// + /// ``` + /// # use forest_ipld::ipld; + /// let v = ipld!({"key": "value", "bool": true}); + /// ``` Map(BTreeMap), + + /// Represents a link to another piece of data through a content identifier (`Cid`). + /// + /// ``` + /// # use forest_ipld::ipld; + /// # use cid::Cid; + /// let v = ipld!(Link(Cid::default())); + /// ``` Link(Cid), } From 214c594041ca930ee90771d94f5a80358e3649a8 Mon Sep 17 00:00:00 2001 From: austinabell Date: Sat, 2 May 2020 21:15:51 -0400 Subject: [PATCH 11/20] Port over existing usage of ipld to macro --- ipld/blockstore/src/buffered.rs | 20 ++++----- ipld/tests/json_tests.rs | 75 ++++++++++----------------------- 2 files changed, 32 insertions(+), 63 deletions(-) diff --git a/ipld/blockstore/src/buffered.rs b/ipld/blockstore/src/buffered.rs index 87e69d941464..aed385f23eef 100644 --- a/ipld/blockstore/src/buffered.rs +++ b/ipld/blockstore/src/buffered.rs @@ -185,8 +185,7 @@ mod tests { use super::*; use cid::multihash::{Blake2b256, Identity}; use commcid::{commitment_to_cid, FilecoinMultihashCode}; - use forest_ipld::Ipld; - use std::collections::BTreeMap; + use forest_ipld::{ipld, Ipld}; #[test] fn basic_buffered_store() { @@ -212,14 +211,15 @@ mod tests { let identity_cid = buf_store.put(&0u8, Identity).unwrap(); // Create map to insert into store - let mut map: BTreeMap = Default::default(); - map.insert("array".to_owned(), Ipld::Link(arr_cid.clone())); let sealed_comm_cid = commitment_to_cid(&[7u8; 32], FilecoinMultihashCode::SealedV1); - map.insert("sealed".to_owned(), Ipld::Link(sealed_comm_cid.clone())); let unsealed_comm_cid = commitment_to_cid(&[5u8; 32], FilecoinMultihashCode::UnsealedV1); - map.insert("unsealed".to_owned(), Ipld::Link(unsealed_comm_cid.clone())); - map.insert("identity".to_owned(), Ipld::Link(identity_cid.clone())); - map.insert("value".to_owned(), Ipld::String(str_val.to_owned())); + let map = ipld!({ + "array": Link(arr_cid.clone()), + "sealed": Link(sealed_comm_cid.clone()), + "unsealed": Link(unsealed_comm_cid.clone()), + "identity": Link(identity_cid.clone()), + "value": str_val, + }); let map_cid = buf_store.put(&map, Blake2b256).unwrap(); let root_cid = buf_store.put(&(map_cid.clone(), 1u8), Blake2b256).unwrap(); @@ -238,10 +238,10 @@ mod tests { mem.get::<(String, u8)>(&arr_cid).unwrap(), Some((str_val.to_owned(), value)) ); - assert_eq!(mem.get::(&map_cid).unwrap(), Some(Ipld::Map(map))); + assert_eq!(mem.get::(&map_cid).unwrap(), Some(map)); assert_eq!( mem.get::(&root_cid).unwrap(), - Some(Ipld::List(vec![Ipld::Link(map_cid), Ipld::Integer(1)])) + Some(ipld!([Link(map_cid), 1])) ); assert_eq!(buf_store.get::(&identity_cid).unwrap(), None); assert_eq!(buf_store.get::(&unsealed_comm_cid).unwrap(), None); diff --git a/ipld/tests/json_tests.rs b/ipld/tests/json_tests.rs index 140274d30e1e..cbb2bd65d35b 100644 --- a/ipld/tests/json_tests.rs +++ b/ipld/tests/json_tests.rs @@ -4,13 +4,12 @@ #![cfg(feature = "json")] use forest_ipld::{ + ipld, json::{self, IpldJson, IpldJsonRef}, Ipld, }; use serde::{Deserialize, Serialize}; use serde_json::{from_str, to_string}; -use std::collections::BTreeMap; -use std::iter::FromIterator; #[test] fn deserialize_json_symmetric() { @@ -30,41 +29,20 @@ fn deserialize_json_symmetric() { "list": [null, { "/": "bafy2bzaceaa466o2jfc4g4ggrmtf55ygigvkmxvkr5mvhy4qbwlxetbmlkqjk" }, 1] } "#; - let expected = Ipld::Map(BTreeMap::from_iter( - [ - ( - "link".to_owned(), - Ipld::Link( - "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n" - .parse() - .unwrap(), - ), - ), - ( - "bytes".to_owned(), - Ipld::Bytes([0x54, 0x68, 0x65, 0x20, 0x71, 0x75, 0x69].to_vec()), - ), - ("string".to_owned(), Ipld::String("Some data".to_owned())), - ("float".to_owned(), Ipld::Float(10.5)), - ("integer".to_owned(), Ipld::Integer(8)), - ("neg_integer".to_owned(), Ipld::Integer(-20)), - ("null".to_owned(), Ipld::Null), - ( - "list".to_owned(), - Ipld::List(vec![ - Ipld::Null, - Ipld::Link( - "bafy2bzaceaa466o2jfc4g4ggrmtf55ygigvkmxvkr5mvhy4qbwlxetbmlkqjk" - .parse() - .unwrap(), - ), - Ipld::Integer(1), - ]), - ), - ] - .to_vec() - .into_iter(), - )); + let expected = ipld!({ + "link": Link("QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n".parse().unwrap()), + "bytes": Bytes(vec![0x54, 0x68, 0x65, 0x20, 0x71, 0x75, 0x69]), + "string": "Some data", + "float": 10.5, + "integer": 8, + "neg_integer": -20, + "null": null, + "list": [ + null, + Link("bafy2bzaceaa466o2jfc4g4ggrmtf55ygigvkmxvkr5mvhy4qbwlxetbmlkqjk".parse().unwrap()), + 1, + ], + }); // Assert deserializing into expected Ipld let IpldJson(ipld_d) = from_str(test_json).unwrap(); @@ -91,14 +69,14 @@ fn annotating_struct_json() { } "#; let expected = TestStruct { - one: Ipld::List(vec![ - Ipld::Link( + one: ipld!([ + Link( "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n" .parse() - .unwrap(), + .unwrap() ), - Ipld::Null, - Ipld::Integer(8), + null, + 8 ]), other: "Some data".to_owned(), }; @@ -110,17 +88,8 @@ fn annotating_struct_json() { fn link_edge_case() { // Test ported from go-ipld-prime (making sure edge case is handled) let test_json = r#"{"/":{"/":"QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n"}}"#; - let expected = Ipld::Map(BTreeMap::from_iter( - [( - "/".to_owned(), - Ipld::Link( - "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n" - .parse() - .unwrap(), - ), - )] - .to_vec(), - )); + let expected = + ipld!({"/": Link("QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n".parse().unwrap())}); let IpldJson(ipld_d) = from_str(test_json).unwrap(); assert_eq!(&ipld_d, &expected, "Deserialized ipld does not match"); From d7f77460c416ed33bb83268585c1736c7d069687 Mon Sep 17 00:00:00 2001 From: austinabell Date: Sat, 2 May 2020 21:21:33 -0400 Subject: [PATCH 12/20] header --- ipld/src/macros.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ipld/src/macros.rs b/ipld/src/macros.rs index a262a7a2b6d5..b53e14e3f293 100644 --- a/ipld/src/macros.rs +++ b/ipld/src/macros.rs @@ -1,3 +1,6 @@ +// Copyright 2020 ChainSafe Systems +// SPDX-License-Identifier: Apache-2.0, MIT + /// Construct a `forest_ipld::Ipld` roughly matching JSON format. This code is a modified version /// of `serde_json::json` macro, with extensions for being able to indicate links and bytes. /// These two matterns can be matched by wrapping `Cid` link in `Link(..)` or the `Vec` in From 57fe67355a037318cb6201dcd83dbf675b097c8a Mon Sep 17 00:00:00 2001 From: austinabell Date: Sun, 3 May 2020 08:49:07 -0400 Subject: [PATCH 13/20] Update doc comment for conversions from ipld --- ipld/src/lib.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ipld/src/lib.rs b/ipld/src/lib.rs index 0361349e90f9..866c02a13c66 100644 --- a/ipld/src/lib.rs +++ b/ipld/src/lib.rs @@ -113,7 +113,10 @@ pub fn from_ipld(value: &Ipld) -> Result where T: DeserializeOwned, { - // TODO find a way to convert without going through byte buffer + // TODO update to not go through byte buffer to convert + // There is a good amount of overhead for this (having to implement serde::Deserializer) + // for Ipld, but possible. The benefit isn't worth changing yet since if the value is not + // passed by reference as needed by HAMT, then the values will have to be cloned. let buf = to_vec(value).map_err(|e| e.to_string())?; from_slice(buf.as_slice()).map_err(|e| e.to_string()) } From c69b0f3a6e0f3ff56fa92aca1a0820287f212d8a Mon Sep 17 00:00:00 2001 From: austinabell Date: Sun, 3 May 2020 09:03:51 -0400 Subject: [PATCH 14/20] Fix doc comment example --- ipld/src/macros.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ipld/src/macros.rs b/ipld/src/macros.rs index b53e14e3f293..474f5c57b7bb 100644 --- a/ipld/src/macros.rs +++ b/ipld/src/macros.rs @@ -13,8 +13,8 @@ /// let value = ipld!({ /// "code": 200, /// "success": true, -/// "link": "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n".parse::().unwrap(), -/// "bytes": vec![0x1, 0xfa, 0x8b], +/// "link": Link("QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n".parse().unwrap()), +/// "bytes": Bytes(vec![0x1, 0xfa, 0x8b]), /// "payload": { /// "features": [ /// "serde", From 0095798fd419be8da9f4d3821d459ece1d9cbf16 Mon Sep 17 00:00:00 2001 From: austinabell Date: Sun, 3 May 2020 19:08:14 -0400 Subject: [PATCH 15/20] fix cid => ipld conversions --- ipld/src/error.rs | 7 +++++++ ipld/src/lib.rs | 6 +++++- ipld/src/ser.rs | 30 +++++++++++++++++++++++++++++- ipld/tests/cbor_test.rs | 32 ++++++++++++++++++++++++++++++-- 4 files changed, 71 insertions(+), 4 deletions(-) diff --git a/ipld/src/error.rs b/ipld/src/error.rs index 7e57faed196a..7727ae2d8f47 100644 --- a/ipld/src/error.rs +++ b/ipld/src/error.rs @@ -1,6 +1,7 @@ // Copyright 2020 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT +use encoding::error::Error as CborError; use serde::ser; use std::fmt; use thiserror::Error; @@ -19,3 +20,9 @@ impl ser::Error for Error { Error::Encoding(msg.to_string()) } } + +impl From for Error { + fn from(e: CborError) -> Error { + Error::Encoding(e.to_string()) + } +} diff --git a/ipld/src/lib.rs b/ipld/src/lib.rs index 866c02a13c66..e55ede86a0b0 100644 --- a/ipld/src/lib.rs +++ b/ipld/src/lib.rs @@ -88,11 +88,15 @@ pub enum Ipld { Map(BTreeMap), /// Represents a link to another piece of data through a content identifier (`Cid`). + /// Using `ipld` macro, can wrap Cid with Link to be explicit of Link type, or let it resolve. /// /// ``` /// # use forest_ipld::ipld; /// # use cid::Cid; - /// let v = ipld!(Link(Cid::default())); + /// let cid: Cid = "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n".parse().unwrap(); + /// let v1 = ipld!(Link(cid.clone())); + /// let v2 = ipld!(cid); + /// assert_eq!(v1, v2); /// ``` Link(Cid), } diff --git a/ipld/src/ser.rs b/ipld/src/ser.rs index 5fd426bdb17d..768227de94f1 100644 --- a/ipld/src/ser.rs +++ b/ipld/src/ser.rs @@ -12,7 +12,10 @@ use std::collections::BTreeMap; use super::{to_ipld, Error, Ipld}; +use cid::Cid; +use encoding::to_vec; use serde::{self, Serialize}; +use std::convert::TryFrom; impl serde::Serialize for Ipld { #[inline] @@ -146,12 +149,37 @@ impl serde::Serializer for Serializer { #[inline] fn serialize_newtype_struct( self, - _name: &'static str, + name: &'static str, ipld: &T, ) -> Result where T: Serialize, { + // TODO revisit this, necessary workaround to allow Cids to be converted to Ipld + // but is not very clean to use and requires the bytes buffer. The reason this is + // necessary is because Cids serialize through newtype_struct. + if name == "\0cbor_tag" { + let bz = to_vec(&ipld)?; + let mut sl = &bz[..]; + + if bz.len() < 3 { + return Err(Error::Encoding("Invalid tag for Ipld".to_owned())); + } + + // Index past the cbor and multibase prefix for Cid deserialization + match sl[0] { + 0x40..=0x57 => sl = &sl[2..], + 0x58 => sl = &sl[3..], // extra u8 + 0x59 => sl = &sl[4..], // extra u16 + 0x5a => sl = &sl[6..], // extra u32 + 0x5b => sl = &sl[10..], // extra u64 + _ => return Err(Error::Encoding("Invalid cbor tag".to_owned())), + } + + return Ok(Ipld::Link( + Cid::try_from(sl).map_err(|e| Error::Encoding(e.to_string()))?, + )); + } ipld.serialize(self) } diff --git a/ipld/tests/cbor_test.rs b/ipld/tests/cbor_test.rs index 6ff261052beb..527e8215c99f 100644 --- a/ipld/tests/cbor_test.rs +++ b/ipld/tests/cbor_test.rs @@ -1,9 +1,12 @@ // Copyright 2020 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT -use cid::{multihash::Blake2b256, Cid}; +use cid::{ + multihash::{Blake2b256, Identity}, + Cid, +}; use encoding::{from_slice, to_vec}; -use forest_ipld::{ipld, Ipld}; +use forest_ipld::{ipld, to_ipld, Ipld}; use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize, Clone)] @@ -34,3 +37,28 @@ fn encode_new_type() { &ipld!({"details": Link(details), "name": "Test"}) ); } + +#[test] +fn cid_conversions_ipld() { + let cid = Cid::new_from_cbor(&[1, 2, 3], Blake2b256); + let m_s = TestStruct { + name: "s".to_owned(), + details: cid.clone(), + }; + assert_eq!( + to_ipld(&m_s).unwrap(), + ipld!({"name": "s", "details": Link(cid.clone()) }) + ); + let serialized = to_vec(&cid).unwrap(); + let ipld = ipld!(Link(cid.clone())); + let ipld2 = to_ipld(&cid).unwrap(); + assert_eq!(ipld, ipld2); + assert_eq!(to_vec(&ipld).unwrap(), serialized); + assert_eq!(to_ipld(&cid).unwrap(), Ipld::Link(cid)); + + // Test with identity hash (different length prefix for cbor) + let cid = Cid::new_from_cbor(&[1, 2], Identity); + let ipld = ipld!(Link(cid.clone())); + let ipld2 = to_ipld(&cid).unwrap(); + assert_eq!(ipld, ipld2); +} From d449ddc242e0d3940986cb2759cdecda8aa61031 Mon Sep 17 00:00:00 2001 From: austinabell Date: Mon, 4 May 2020 11:30:31 -0400 Subject: [PATCH 16/20] Remove need to go through JSON value buffer for Ipld --- ipld/src/json.rs | 265 ++++++++++++++++++++++++--------------- ipld/tests/json_tests.rs | 3 +- 2 files changed, 163 insertions(+), 105 deletions(-) diff --git a/ipld/src/json.rs b/ipld/src/json.rs index 7941879935eb..0073efdcc667 100644 --- a/ipld/src/json.rs +++ b/ipld/src/json.rs @@ -1,11 +1,12 @@ // Copyright 2020 ChainSafe Systems // SPDX-License-Identifier: Apache-2.0, MIT -use super::Ipld; +use super::{ipld, Ipld}; use serde::{de, ser, Deserialize, Deserializer, Serialize, Serializer}; -use serde_json::{json, Map, Number, Value as JsonValue}; use std::collections::BTreeMap; -use std::convert::{TryFrom, TryInto}; +use std::fmt; + +const BYTES_JSON_KEY: &str = "bytes"; /// Wrapper for serializing and deserializing a Ipld from JSON. #[derive(Deserialize, Serialize)] @@ -17,124 +18,180 @@ pub struct IpldJson(#[serde(with = "self")] pub Ipld); #[serde(transparent)] pub struct IpldJsonRef<'a>(#[serde(with = "self")] pub &'a Ipld); -// TODO serialize and deserialize should not have to go through a Json value buffer -// (unnecessary clones and copies) but the efficiency for JSON shouldn't matter too much - pub fn serialize(ipld: &Ipld, serializer: S) -> Result where S: Serializer, { - JsonValue::try_from(ipld) - .map_err(ser::Error::custom)? - .serialize(serializer) + match &ipld { + Ipld::Null => serializer.serialize_none(), + Ipld::Bool(bool) => serializer.serialize_bool(*bool), + Ipld::Integer(i128) => serializer.serialize_i128(*i128), + Ipld::Float(f64) => serializer.serialize_f64(*f64), + Ipld::String(string) => serializer.serialize_str(&string), + Ipld::Bytes(bytes) => serialize( + &ipld!({ "/": { BYTES_JSON_KEY: base64::encode(bytes) } }), + serializer, + ), + Ipld::List(list) => { + let wrapped = list.iter().map(|ipld| IpldJsonRef(ipld)); + serializer.collect_seq(wrapped) + } + Ipld::Map(map) => { + let wrapped = map.iter().map(|(key, ipld)| (key, IpldJsonRef(ipld))); + serializer.collect_map(wrapped) + } + Ipld::Link(cid) => serialize(&ipld!({ "/": cid.to_string() }), serializer), + } } pub fn deserialize<'de, D>(deserializer: D) -> Result where D: Deserializer<'de>, { - Ok(JsonValue::deserialize(deserializer)? - .try_into() - .map_err(de::Error::custom)?) + deserializer.deserialize_any(JSONVisitor) } -impl TryFrom<&Ipld> for JsonValue { - type Error = &'static str; - - fn try_from(ipld: &Ipld) -> Result { - let val = match ipld { - Ipld::Null => JsonValue::Null, - Ipld::Bool(b) => JsonValue::Bool(*b), - Ipld::Integer(i) => { - let i = *i; - if i < 0 { - let c = i64::try_from(i).map_err(|_| "Invalid precision for json number")?; - JsonValue::Number(c.into()) - } else { - let c = u64::try_from(i).map_err(|_| "Invalid precision for json number")?; - JsonValue::Number(c.into()) - } - } - Ipld::Float(f) => JsonValue::Number( - Number::from_f64(*f).ok_or("Float does not have finite precision")?, - ), - Ipld::String(s) => JsonValue::String(s.clone()), - Ipld::Bytes(bz) => json!({ "/": { "base64": base64::encode(bz) } }), - Ipld::Link(cid) => json!({ "/": cid.to_string() }), - Ipld::List(list) => JsonValue::Array( - list.iter() - .map(JsonValue::try_from) - .collect::>()?, - ), - Ipld::Map(map) => { - let mut new = Map::new(); - for (k, v) in map.iter() { - new.insert(k.to_string(), JsonValue::try_from(v)?); - } - JsonValue::Object(new) - } - }; - Ok(val) +/// Json visitor for generating IPLD from JSON +struct JSONVisitor; +impl<'de> de::Visitor<'de> for JSONVisitor { + type Value = Ipld; + + fn expecting(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt.write_str("any valid JSON value") } -} -impl TryFrom for Ipld { - type Error = &'static str; - - fn try_from(json: JsonValue) -> Result { - let val = match json { - JsonValue::Null => Ipld::Null, - JsonValue::Bool(b) => Ipld::Bool(b), - JsonValue::Number(n) => { - if let Some(v) = n.as_u64() { - Ipld::Integer(v.into()) - } else if let Some(v) = n.as_i64() { - Ipld::Integer(v.into()) - } else if let Some(v) = n.as_f64() { - Ipld::Float(v) - } else { - // Json number can only be one of those three types - unreachable!() - } - } - JsonValue::String(s) => Ipld::String(s), - JsonValue::Array(values) => Ipld::List( - values - .into_iter() - .map(Ipld::try_from) - .collect::>()?, - ), - JsonValue::Object(map) => { - // Check for escaped values (Bytes and Cids) - if map.len() == 1 { - if let Some(v) = map.get("/") { - match v { - JsonValue::String(s) => { - // Json block is a Cid - return Ok(Ipld::Link( - s.parse().map_err(|_| "Failed not parse cid string")?, - )); - } - JsonValue::Object(obj) => { - // Are other bytes encoding types supported? - if let Some(JsonValue::String(bz)) = obj.get("base64") { - return Ok(Ipld::Bytes( - base64::decode(bz) - .map_err(|_| "Failed to parse base64 bytes")?, - )); - } - } - _ => (), + #[inline] + fn visit_str(self, value: &str) -> Result + where + E: de::Error, + { + self.visit_string(String::from(value)) + } + + #[inline] + fn visit_string(self, value: String) -> Result + where + E: de::Error, + { + Ok(Ipld::String(value)) + } + #[inline] + fn visit_bytes(self, v: &[u8]) -> Result + where + E: de::Error, + { + self.visit_byte_buf(v.to_owned()) + } + + #[inline] + fn visit_byte_buf(self, v: Vec) -> Result + where + E: de::Error, + { + Ok(Ipld::Bytes(v)) + } + + #[inline] + fn visit_u64(self, v: u64) -> Result + where + E: de::Error, + { + Ok(Ipld::Integer(v.into())) + } + + #[inline] + fn visit_i64(self, v: i64) -> Result + where + E: de::Error, + { + Ok(Ipld::Integer(v.into())) + } + + #[inline] + fn visit_i128(self, v: i128) -> Result + where + E: de::Error, + { + Ok(Ipld::Integer(v)) + } + + #[inline] + fn visit_bool(self, v: bool) -> Result + where + E: de::Error, + { + Ok(Ipld::Bool(v)) + } + + #[inline] + fn visit_none(self) -> Result + where + E: de::Error, + { + self.visit_unit() + } + + #[inline] + fn visit_unit(self) -> Result + where + E: de::Error, + { + Ok(Ipld::Null) + } + + #[inline] + fn visit_seq(self, mut visitor: V) -> Result + where + V: de::SeqAccess<'de>, + { + let mut vec = Vec::new(); + + while let Some(IpldJson(elem)) = visitor.next_element()? { + vec.push(elem); + } + + Ok(Ipld::List(vec)) + } + + #[inline] + fn visit_map(self, mut visitor: V) -> Result + where + V: de::MapAccess<'de>, + { + let mut map = BTreeMap::new(); + + while let Some((key, IpldJson(value))) = visitor.next_entry()? { + map.insert(key, value); + } + + if map.len() == 1 { + if let Some(v) = map.get("/") { + match v { + Ipld::String(s) => { + // Json block is a Cid + return Ok(Ipld::Link(s.parse().map_err(|e| de::Error::custom(e))?)); + } + Ipld::Map(obj) => { + // Are other bytes encoding types supported? + if let Some(Ipld::String(bz)) = obj.get(BYTES_JSON_KEY) { + return Ok(Ipld::Bytes( + base64::decode(bz).map_err(|e| de::Error::custom(e.to_string()))?, + )); } } + _ => (), } - let mut new = BTreeMap::new(); - for (k, v) in map.into_iter() { - new.insert(k, Ipld::try_from(v)?); - } - Ipld::Map(new) } - }; - Ok(val) + } + + Ok(Ipld::Map(map)) + } + + #[inline] + fn visit_f64(self, v: f64) -> Result + where + E: de::Error, + { + Ok(Ipld::Float(v)) } } diff --git a/ipld/tests/json_tests.rs b/ipld/tests/json_tests.rs index cbb2bd65d35b..6cfd739369b8 100644 --- a/ipld/tests/json_tests.rs +++ b/ipld/tests/json_tests.rs @@ -19,7 +19,7 @@ fn deserialize_json_symmetric() { "/": "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n" }, "bytes": { - "/": { "base64": "VGhlIHF1aQ==" } + "/": { "bytes": "VGhlIHF1aQ==" } }, "string": "Some data", "float": 10.5, @@ -50,6 +50,7 @@ fn deserialize_json_symmetric() { // Symmetric tests let ser_json = to_string(&IpldJsonRef(&expected)).unwrap(); + println!("{}", ser_json); let IpldJson(ipld_d) = from_str(&ser_json).unwrap(); assert_eq!(&ipld_d, &expected, "Deserialized ipld does not match"); } From 79d493c4d321a23f4b350cc97c8a3116534093f0 Mon Sep 17 00:00:00 2001 From: austinabell Date: Mon, 4 May 2020 11:46:46 -0400 Subject: [PATCH 17/20] Update bytes to use multibase, based on updated spec --- ipld/Cargo.toml | 4 ++-- ipld/src/json.rs | 11 ++++++----- ipld/tests/json_tests.rs | 5 ++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ipld/Cargo.toml b/ipld/Cargo.toml index f415c675f137..62d5a7485a44 100644 --- a/ipld/Cargo.toml +++ b/ipld/Cargo.toml @@ -12,7 +12,7 @@ encoding = { package = "forest_encoding", path = "../encoding" } serde = { version = "1.0", features = ["derive"] } thiserror = "1.0" serde_json = { version = "1.0", optional = true } -base64 = { version = "0.12.0", optional = true } +multibase = { version = "0.8.0", optional = true } [dependencies.cid] package = "forest_cid" @@ -20,4 +20,4 @@ path = "../ipld/cid" features = ["cbor"] [features] -json = ["serde_json", "base64"] +json = ["serde_json", "multibase"] diff --git a/ipld/src/json.rs b/ipld/src/json.rs index 0073efdcc667..631c942d3a5e 100644 --- a/ipld/src/json.rs +++ b/ipld/src/json.rs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0, MIT use super::{ipld, Ipld}; +use multibase::Base; use serde::{de, ser, Deserialize, Deserializer, Serialize, Serializer}; use std::collections::BTreeMap; use std::fmt; @@ -29,7 +30,7 @@ where Ipld::Float(f64) => serializer.serialize_f64(*f64), Ipld::String(string) => serializer.serialize_str(&string), Ipld::Bytes(bytes) => serialize( - &ipld!({ "/": { BYTES_JSON_KEY: base64::encode(bytes) } }), + &ipld!({ "/": { BYTES_JSON_KEY: multibase::encode(Base::Base64, bytes) } }), serializer, ), Ipld::List(list) => { @@ -173,10 +174,10 @@ impl<'de> de::Visitor<'de> for JSONVisitor { } Ipld::Map(obj) => { // Are other bytes encoding types supported? - if let Some(Ipld::String(bz)) = obj.get(BYTES_JSON_KEY) { - return Ok(Ipld::Bytes( - base64::decode(bz).map_err(|e| de::Error::custom(e.to_string()))?, - )); + if let Some(Ipld::String(s)) = obj.get(BYTES_JSON_KEY) { + let (_, bz) = multibase::decode(s) + .map_err(|e| de::Error::custom(e.to_string()))?; + return Ok(Ipld::Bytes(bz)); } } _ => (), diff --git a/ipld/tests/json_tests.rs b/ipld/tests/json_tests.rs index 6cfd739369b8..548f150ba168 100644 --- a/ipld/tests/json_tests.rs +++ b/ipld/tests/json_tests.rs @@ -19,7 +19,7 @@ fn deserialize_json_symmetric() { "/": "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n" }, "bytes": { - "/": { "bytes": "VGhlIHF1aQ==" } + "/": { "bytes": "mVGhlIHF1aQ" } }, "string": "Some data", "float": 10.5, @@ -46,11 +46,10 @@ fn deserialize_json_symmetric() { // Assert deserializing into expected Ipld let IpldJson(ipld_d) = from_str(test_json).unwrap(); - assert_eq!(&ipld_d, &expected, "Deserialized ipld does not match"); + // assert_eq!(&ipld_d, &expected, "Deserialized ipld does not match"); // Symmetric tests let ser_json = to_string(&IpldJsonRef(&expected)).unwrap(); - println!("{}", ser_json); let IpldJson(ipld_d) = from_str(&ser_json).unwrap(); assert_eq!(&ipld_d, &expected, "Deserialized ipld does not match"); } From 0b6d342ce34d26b937df93fb60fc205bd9dd9886 Mon Sep 17 00:00:00 2001 From: austinabell Date: Mon, 4 May 2020 11:52:04 -0400 Subject: [PATCH 18/20] Cleanup --- ipld/src/json.rs | 4 ++-- ipld/tests/json_tests.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ipld/src/json.rs b/ipld/src/json.rs index 631c942d3a5e..4ed5dfebd565 100644 --- a/ipld/src/json.rs +++ b/ipld/src/json.rs @@ -3,7 +3,7 @@ use super::{ipld, Ipld}; use multibase::Base; -use serde::{de, ser, Deserialize, Deserializer, Serialize, Serializer}; +use serde::{de, Deserialize, Deserializer, Serialize, Serializer}; use std::collections::BTreeMap; use std::fmt; @@ -170,7 +170,7 @@ impl<'de> de::Visitor<'de> for JSONVisitor { match v { Ipld::String(s) => { // Json block is a Cid - return Ok(Ipld::Link(s.parse().map_err(|e| de::Error::custom(e))?)); + return Ok(Ipld::Link(s.parse().map_err(de::Error::custom)?)); } Ipld::Map(obj) => { // Are other bytes encoding types supported? diff --git a/ipld/tests/json_tests.rs b/ipld/tests/json_tests.rs index 548f150ba168..caa1e5b5be45 100644 --- a/ipld/tests/json_tests.rs +++ b/ipld/tests/json_tests.rs @@ -46,7 +46,7 @@ fn deserialize_json_symmetric() { // Assert deserializing into expected Ipld let IpldJson(ipld_d) = from_str(test_json).unwrap(); - // assert_eq!(&ipld_d, &expected, "Deserialized ipld does not match"); + assert_eq!(&ipld_d, &expected, "Deserialized ipld does not match"); // Symmetric tests let ser_json = to_string(&IpldJsonRef(&expected)).unwrap(); From a6f0f58337507ec7d3acb9134cc08c3687b63223 Mon Sep 17 00:00:00 2001 From: austinabell Date: Mon, 4 May 2020 12:08:57 -0400 Subject: [PATCH 19/20] Remove question in comment --- ipld/src/json.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ipld/src/json.rs b/ipld/src/json.rs index 4ed5dfebd565..dd3099545191 100644 --- a/ipld/src/json.rs +++ b/ipld/src/json.rs @@ -169,12 +169,12 @@ impl<'de> de::Visitor<'de> for JSONVisitor { if let Some(v) = map.get("/") { match v { Ipld::String(s) => { - // Json block is a Cid + // { "/": ".." } Json block is a Cid return Ok(Ipld::Link(s.parse().map_err(de::Error::custom)?)); } Ipld::Map(obj) => { - // Are other bytes encoding types supported? if let Some(Ipld::String(s)) = obj.get(BYTES_JSON_KEY) { + // { "/": { "bytes": "" } } Json block are bytes encoded let (_, bz) = multibase::decode(s) .map_err(|e| de::Error::custom(e.to_string()))?; return Ok(Ipld::Bytes(bz)); From 7faa4d69f091c85ae5a0444de4736c525cbe792d Mon Sep 17 00:00:00 2001 From: austinabell Date: Wed, 6 May 2020 12:28:42 -0400 Subject: [PATCH 20/20] Add no_run to ipld examples --- ipld/src/lib.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ipld/src/lib.rs b/ipld/src/lib.rs index e55ede86a0b0..37ef48d8f15b 100644 --- a/ipld/src/lib.rs +++ b/ipld/src/lib.rs @@ -25,7 +25,7 @@ use std::collections::BTreeMap; pub enum Ipld { /// Represents a null value. /// - /// ``` + /// ```no_run /// # use forest_ipld::ipld; /// let v = ipld!(null); /// ``` @@ -33,7 +33,7 @@ pub enum Ipld { /// Represents a boolean value. /// - /// ``` + /// ```no_run /// # use forest_ipld::ipld; /// let v = ipld!(true); /// ``` @@ -41,7 +41,7 @@ pub enum Ipld { /// Represents a signed integer value. /// - /// ``` + /// ```no_run /// # use forest_ipld::ipld; /// let v = ipld!(28); /// ``` @@ -49,7 +49,7 @@ pub enum Ipld { /// Represents a floating point value. /// - /// ``` + /// ```no_run /// # use forest_ipld::ipld; /// let v = ipld!(8.5); /// ``` @@ -57,7 +57,7 @@ pub enum Ipld { /// Represents a String. /// - /// ``` + /// ```no_run /// # use forest_ipld::ipld; /// let v = ipld!("string"); /// ``` @@ -65,7 +65,7 @@ pub enum Ipld { /// Represents Bytes. /// - /// ``` + /// ```no_run /// # use forest_ipld::ipld; /// let v = ipld!(Bytes(vec![0x98, 0x8, 0x2a, 0xff])); /// ``` @@ -73,7 +73,7 @@ pub enum Ipld { /// Represents List of IPLD objects. /// - /// ``` + /// ```no_run /// # use forest_ipld::ipld; /// let v = ipld!([1, "string", null]); /// ``` @@ -81,7 +81,7 @@ pub enum Ipld { /// Represents a map of strings to Ipld objects. /// - /// ``` + /// ```no_run /// # use forest_ipld::ipld; /// let v = ipld!({"key": "value", "bool": true}); /// ```