From 3ea1e702a1bc3b0583a91b7d03fa3e2a0abeef9e Mon Sep 17 00:00:00 2001 From: Joep Meindertsma Date: Mon, 12 Oct 2020 21:14:24 +0200 Subject: [PATCH 01/10] Refactor commits, default store improvements --- lib/defaults/default_store.ad3 | 11 ++- lib/src/commit.rs | 126 +++++++++++++++++++++++++++++++++ lib/src/delta.rs | 98 +------------------------ lib/src/lib.rs | 3 +- lib/src/urls.rs | 8 +++ 5 files changed, 148 insertions(+), 98 deletions(-) create mode 100644 lib/src/commit.rs diff --git a/lib/defaults/default_store.ad3 b/lib/defaults/default_store.ad3 index 908e90629..3d6505f2c 100644 --- a/lib/defaults/default_store.ad3 +++ b/lib/defaults/default_store.ad3 @@ -31,7 +31,7 @@ ["https://atomicdata.dev/datatypes/date","https://atomicdata.dev/properties/description","ISO date _without time_.\nYYYY-MM-DD.\n\ne.g. `1991-01-20`\n."] ["https://atomicdata.dev/datatypes/date","https://atomicdata.dev/properties/isA","[\"https://atomicdata.dev/classes/Datatype\"]"] ["https://atomicdata.dev/datatypes/slug","https://atomicdata.dev/properties/shortname","slug"] -["https://atomicdata.dev/datatypes/slug","https://atomicdata.dev/properties/description","Lowercase string without spaces, a-z. These are used for things like JSON keys." ] +["https://atomicdata.dev/datatypes/slug","https://atomicdata.dev/properties/description","Lowercase string without spaces. Only characters, numbers and dashes: `-`. These are used for things like JSON keys." ] ["https://atomicdata.dev/datatypes/slug","https://atomicdata.dev/properties/isA","[\"https://atomicdata.dev/classes/Datatype\"]"] ["https://atomicdata.dev/datatypes/markdown","https://atomicdata.dev/properties/shortname","markdown"] ["https://atomicdata.dev/datatypes/markdown","https://atomicdata.dev/properties/description","Markdown UTF-8 String with [Commonmark syntax](https://commonmark.org/). [Here's a tutorial](https://commonmark.org/help/tutorial/)."] @@ -45,6 +45,9 @@ ["https://atomicdata.dev/datatypes/boolean","https://atomicdata.dev/properties/shortname","boolean"] ["https://atomicdata.dev/datatypes/boolean","https://atomicdata.dev/properties/description","Either 'true' or 'false'."] ["https://atomicdata.dev/datatypes/boolean","https://atomicdata.dev/properties/isA","[\"https://atomicdata.dev/classes/Datatype\"]"] +["https://atomicdata.dev/datatypes/timestamp","https://atomicdata.dev/properties/shortname","timestamp"] +["https://atomicdata.dev/datatypes/timestamp","https://atomicdata.dev/properties/description","Similar to [Unix Timestamp](https://www.unixtimestamp.com/).\nMilliseconds since midnight UTC 1970 jan 01 (aka the [Unix Epoch](https://en.wikipedia.org/wiki/Unix_time)).\nUse this for most DateTime fields.\nSigned 64 bit integer (instead of 32 bit in Unix systems).\n\ne.g. `1596798919` (= 07 Aug 2020 11:15:19)"] +["https://atomicdata.dev/datatypes/timestamp","https://atomicdata.dev/properties/isA","[\"https://atomicdata.dev/classes/Datatype\"]"] # Properties ["https://atomicdata.dev/properties/description","https://atomicdata.dev/properties/isA","[\"https://atomicdata.dev/classes/Property\"]"] ["https://atomicdata.dev/properties/description","https://atomicdata.dev/properties/datatype","https://atomicdata.dev/datatypes/markdown"] @@ -61,7 +64,7 @@ ["https://atomicdata.dev/properties/shortname","https://atomicdata.dev/properties/isA","[\"https://atomicdata.dev/classes/Property\"]"] ["https://atomicdata.dev/properties/shortname","https://atomicdata.dev/properties/datatype","https://atomicdata.dev/datatypes/slug"] ["https://atomicdata.dev/properties/shortname","https://atomicdata.dev/properties/shortname","shortname"] -["https://atomicdata.dev/properties/shortname","https://atomicdata.dev/properties/description","A very short name of the thing."] +["https://atomicdata.dev/properties/shortname","https://atomicdata.dev/properties/description","A short name of something. It can only contain letters, numbers and dashes `-`. Useful in programming contexts where the user should be able to type something short to identify a specific thing."] ["https://atomicdata.dev/properties/recommends","https://atomicdata.dev/properties/isA","[\"https://atomicdata.dev/classes/Property\"]"] ["https://atomicdata.dev/properties/recommends","https://atomicdata.dev/properties/datatype","https://atomicdata.dev/datatypes/resourceArray"] ["https://atomicdata.dev/properties/recommends","https://atomicdata.dev/properties/classtype","https://atomicdata.dev/classes/Property"] @@ -97,3 +100,7 @@ ["https://atomicdata.dev/properties/signature","https://atomicdata.dev/properties/datatype","https://atomicdata.dev/datatypes/atomicURL"] ["https://atomicdata.dev/properties/signature","https://atomicdata.dev/properties/shortname","signature"] ["https://atomicdata.dev/properties/signature","https://atomicdata.dev/properties/description","The signature is the agent (person, organization or something else) that issued the commit."] +["https://atomicdata.dev/properties/createdAt","https://atomicdata.dev/properties/isA","[\"https://atomicdata.dev/classes/Property\"]"] +["https://atomicdata.dev/properties/createdAt","https://atomicdata.dev/properties/datatype","https://atomicdata.dev/datatypes/Timestamp"] +["https://atomicdata.dev/properties/createdAt","https://atomicdata.dev/properties/shortname","createdat"] +["https://atomicdata.dev/properties/createdAt","https://atomicdata.dev/properties/description","Timestamp when the Commit was created (usually when it was signed)."] diff --git a/lib/src/commit.rs b/lib/src/commit.rs new file mode 100644 index 000000000..afa9180fb --- /dev/null +++ b/lib/src/commit.rs @@ -0,0 +1,126 @@ +//! Describe changes / mutations to data + +use std::collections::{HashMap, HashSet}; +use serde::{Deserialize, Serialize}; + +use crate::{ResourceString, urls}; + +/// A Commit is a set of changes to a Resource. +/// Use CommitBuilder if you're programmatically constructing a Delta. +#[derive(Debug, Deserialize, Serialize)] +pub struct Commit { +/// The subject URL that is to be modified by this Delta +pub subject: String, +/// The date it was created, as a unix timestamp +pub created_at: u128, +/// The URL of the one suggesting this Commit +pub actor: String, +/// The set of PropVals that need to be added. +/// Overwrites existing values +pub set: Option>, +/// The set of property URLs that need to be removed +pub remove: Option>, +/// If set to true, deletes the entire resource +pub destroy: Option, +/// Hash signed by the actor +pub signature: String, +} + +impl Commit { + /// Converts the Commit into a HashMap of strings. + pub fn to_resourcestring(&self) -> ResourceString { + let mut resource = ResourceString::new(); + resource.insert(urls::SUBJECT.into(), self.subject.clone()); + resource.insert(urls::CREATED_AT.into(), self.created_at.to_string()); + resource.insert(urls::ACTOR.into(), self.actor.clone()); + // How to serialize nested resources? + // https://github.com/joepio/atomic/issues/16 + // resource.insert(urls::SET.into(), some_conversion_func); + todo!(); + resource + } +} + +/// Use this for creating Commits +pub struct CommitBuilder { + /// The subject URL that is to be modified by this Delta + subject: String, + /// The date it was created, as a unix timestamp + // pub created_at: u128, + /// The URL of the one suggesting this Commit + actor: String, + /// The set of PropVals that need to be added. + /// Overwrites existing values + set: std::collections::HashMap, + /// The set of property URLs that need to be removed + remove: HashSet, + /// If set to true, deletes the entire resource + destroy: bool, + // pub signature: String, +} + +impl CommitBuilder { + pub fn new(subject: String, actor: String) -> Self { + CommitBuilder { + subject, + actor, + set: HashMap::new(), + remove: HashSet::new(), + destroy: false, + } + } + + /// Creates the Commit. + /// Does not send it - see atomic_lib::client::post_commit + pub fn sign(self, _private_key: &str) -> Commit { + let created_at = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("Time went backwards") + .as_millis(); + + Commit { + subject: self.subject, + actor: self.actor, + set: Some(self.set), + remove: Some(self.remove.into_iter().collect()), + destroy: Some(self.destroy), + created_at, + // TODO: Hashing signature logic + signature: "correct_signature".into(), + } + } + + pub fn set(&mut self, prop: String, val: String) { + self.set.insert(prop, val); + } + + pub fn remove(&mut self, prop: String) { + self.remove.insert(prop); + } + + /// Whether the resource needs to be removed fully + pub fn destroy(&mut self, destroy: bool) { + self.destroy = destroy + } +} + +#[cfg(test)] +mod test { + use super::*; + use crate::Storelike; + + #[test] + fn apply_commit() { + let store = crate::Store::init(); + let subject = String::from("https://example.com/somesubject"); + let actor = "HashedThing".into(); + let mut partial_commit = CommitBuilder::new(subject.clone(), actor); + let property = crate::urls::DESCRIPTION; + let value = "Some value"; + partial_commit.set(property.into(), value.into()); + let full_commit = partial_commit.sign("correct_signature"); + store.commit(full_commit).unwrap(); + let resource = store.get_resource(&subject).unwrap(); + assert!(resource.get(property).unwrap().to_string() == value); + } +} diff --git a/lib/src/delta.rs b/lib/src/delta.rs index 6dc2cf7ce..c14080906 100644 --- a/lib/src/delta.rs +++ b/lib/src/delta.rs @@ -1,78 +1,5 @@ //! Describe changes / mutations to data - -/// A set of changes to a resource. -use std::collections::HashMap; - -use serde::{Deserialize, Serialize}; -#[derive(Debug, Deserialize, Serialize)] -pub struct Commit { -/// The subject URL that is to be modified by this Delta -pub subject: String, -/// The date it was created, as a unix timestamp -pub created_at: u128, -/// The URL of the one suggesting this Commit -pub actor: String, -/// The set of PropVals that need to be added. -/// Overwrites existing values -pub set: Option>, -/// The set of property URLs that need to be removed -pub remove: Option>, -/// If set to true, deletes the entire resource -pub destroy: Option, -/// Hash signed by the actor -pub signature: String, -} - -pub struct PartialCommit { - /// The subject URL that is to be modified by this Delta - subject: String, - /// The date it was created, as a unix timestamp - // pub created_at: u128, - /// The URL of the one suggesting this Commit - actor: String, - /// The set of PropVals that need to be added. - /// Overwrites existing values - set: std::collections::HashMap, - /// The set of property URLs that need to be removed - remove: Vec, - /// If set to true, deletes the entire resource - destroy: bool, - // pub signature: String, -} - -impl PartialCommit { - pub fn new(subject: String, actor: String) -> Self { - PartialCommit { - subject, - actor, - set: HashMap::new(), - remove: Vec::new(), - destroy: false, - } - } - - pub fn sign(&self, _private_key: &str) -> Commit { - let created_at = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .expect("Time went backwards") - .as_millis(); - - Commit { - subject: self.subject.clone(), - actor: self.actor.clone(), - set: Some(self.set.clone()), - remove: Some(self.remove.clone()), - destroy: Some(self.destroy), - created_at, - // TODO: Hashing signature logic - signature: "correct_signature".into(), - } - } - - pub fn set(&mut self, prop: String, val: String) { - self.set.insert(prop, val); - } -} +//! Deprecated in favor or Commit /// Individual change to a resource. Unvalidated. pub struct DeltaLine { @@ -92,6 +19,8 @@ impl DeltaLine { } } +/// Describes a change to an atom. +/// Deprecated in favor or Commit pub struct DeltaDeprecated { // The set of changes pub lines: Vec, @@ -130,24 +59,3 @@ impl Default for DeltaDeprecated { } } } - -#[cfg(test)] -mod test { - use super::*; - use crate::Storelike; - - #[test] - fn apply_commit() { - let store = crate::Store::init(); - let subject = String::from("https://example.com/somesubject"); - let actor = "HashedThing".into(); - let mut partial_commit = PartialCommit::new(subject.clone(), actor); - let property = crate::urls::DESCRIPTION; - let value = "Some value"; - partial_commit.set(property.into(), value.into()); - let full_commit = partial_commit.sign("correct_signature".into()); - store.commit(full_commit).unwrap(); - let resource = store.get_resource(&subject).unwrap(); - assert!(resource.get(property).unwrap().to_string() == value); - } -} diff --git a/lib/src/lib.rs b/lib/src/lib.rs index 4f3f13cc8..6a679c53e 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -37,6 +37,7 @@ assert!(found_atoms.len() == 1); pub mod atoms; pub mod client; pub mod collections; +pub mod commit; #[cfg(feature = "db")] pub mod db; pub mod delta; @@ -58,7 +59,7 @@ pub use atoms::Atom; pub use atoms::RichAtom; #[cfg(feature = "db")] pub use db::Db; -pub use delta::Commit; +pub use commit::Commit; pub use delta::DeltaLine; pub use resources::Resource; pub use resources::ResourceString; diff --git a/lib/src/urls.rs b/lib/src/urls.rs index 41569b508..9de03faa5 100644 --- a/lib/src/urls.rs +++ b/lib/src/urls.rs @@ -15,6 +15,14 @@ pub const CLASSTYPE_PROP: &str = "https://atomicdata.dev/properties/classtype"; // ... for Classes pub const REQUIRES: &str = "https://atomicdata.dev/properties/requires"; pub const RECOMMENDS: &str = "https://atomicdata.dev/properties/recommends"; +// ... for Commits +pub const SUBJECT: &str = "https://atomicdata.dev/properties/subject"; +pub const SET: &str = "https://atomicdata.dev/properties/set"; +pub const REMOVE: &str = "https://atomicdata.dev/properties/remove"; +pub const DESTROY: &str = "https://atomicdata.dev/properties/destroy"; +pub const ACTOR: &str = "https://atomicdata.dev/properties/actor"; +pub const CREATED_AT: &str = "https://atomicdata.dev/properties/createdAt"; +pub const SIGNATURE: &str = "https://atomicdata.dev/properties/signature"; // Datatypes pub const STRING: &str = "https://atomicdata.dev/datatypes/string"; From f9e2cb855a99c19b3633e47fd9e0de981c633d43 Mon Sep 17 00:00:00 2001 From: Joep Meindertsma Date: Mon, 12 Oct 2020 21:14:32 +0200 Subject: [PATCH 02/10] Increase timeout --- lib/src/client.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/src/client.rs b/lib/src/client.rs index 317070d38..59240b251 100644 --- a/lib/src/client.rs +++ b/lib/src/client.rs @@ -6,7 +6,7 @@ use crate::{errors::AtomicResult, parse::parse_ad3, ResourceString}; pub fn fetch_resource(subject: &str) -> AtomicResult { let resp = ureq::get(&subject) .set("Accept", crate::parse::AD3_MIME) - .timeout_read(500) + .timeout_read(2000) .call(); if resp.status() != 200 { return Err(format!("Could not fetch {}. Status: {}", subject, resp.status()).into()); @@ -33,7 +33,7 @@ pub fn post_commit(endpoint: &str, commit: &crate::Commit) -> AtomicResult<()> { let resp = ureq::post(&endpoint) .set("Content-Type", "application/json") - .timeout_read(500) + .timeout_read(2000) .send_string(&json); if resp.error() { @@ -56,7 +56,7 @@ mod test { #[test] #[ignore] fn post_commit_basic() { - let commit = crate::delta::PartialCommit::new("subject".into(), "actor".into()).sign("private_key"); + let commit = crate::commit::CommitBuilder::new("subject".into(), "actor".into()).sign("private_key"); post_commit("https://atomicdata.dev/commit", &commit).unwrap(); } } From f6f2d64c7cd346d419b36d6b48e405a796090e59 Mon Sep 17 00:00:00 2001 From: Joep Meindertsma Date: Sat, 17 Oct 2020 22:10:31 +0200 Subject: [PATCH 03/10] Store propvals in Db #16 --- lib/examples/basic.rs | 2 +- lib/src/db.rs | 138 ++++++++++++++++++++++++++--------- lib/src/resources.rs | 27 ++++--- server/example_requests.http | 10 ++- 4 files changed, 126 insertions(+), 51 deletions(-) diff --git a/lib/examples/basic.rs b/lib/examples/basic.rs index f1f8abc38..efe0de035 100644 --- a/lib/examples/basic.rs +++ b/lib/examples/basic.rs @@ -39,5 +39,5 @@ fn main() { // A subject URL has been created automatically. let subject = new_property.get_subject(); let fetched_new_resource = store.get_resource(subject).unwrap(); - assert!(fetched_new_resource.get_shortname("description").unwrap().to_string() == "age"); + assert!(fetched_new_resource.get_shortname("description").unwrap().to_string() == "the age of a person"); } diff --git a/lib/src/db.rs b/lib/src/db.rs index 7ae0eea02..4e86f8def 100644 --- a/lib/src/db.rs +++ b/lib/src/db.rs @@ -3,6 +3,7 @@ use crate::{ errors::AtomicResult, + resources::PropVals, resources::ResourceString, storelike::{ResourceCollection, Storelike}, Atom, Resource, @@ -16,7 +17,7 @@ pub struct Db { // Resources can be found using their Subject. // Try not to use this directly, but use the Trees. db: sled::Db, - // Stores all resources. The Key is a string, the value a ResourceString. Both must be serialized using bincode. + // Stores all resources. The Key is the Subject as a string, the value a PropVals. Both must be serialized using bincode. resources: sled::Tree, // Stores all Atoms. The key is the atom.value, the value a vector of Atoms. index_vals: sled::Tree, @@ -26,7 +27,9 @@ pub struct Db { } impl Db { - // Creates a new store at the specified path + /// Creates a new store at the specified path. + /// The base_url is the domain where the db will be hosted, e.g. http://localhost/ + /// It is used for distinguishing locally defined items from externally defined ones. pub fn init>(path: P, base_url: String) -> AtomicResult { let db = sled::open(path)?; let resources = db.open_tree("resources")?; @@ -48,6 +51,31 @@ impl Db { // fn index_value_remove(&mut self, atom: Atom) -> AtomicResult<()> { // todo!(); // } + + fn set_propvals(&self, subject: &str, propvals: &PropVals) -> AtomicResult<()> { + let resource_bin = bincode::serialize(propvals)?; + let subject_bin = bincode::serialize(subject)?; + self.resources.insert(subject_bin, resource_bin)?; + Ok(()) + } + + /// Finds resource by Subject, return PropVals HashMap + /// Deals with the binary API of Sled + fn get_propvals(&self, subject: &str) -> AtomicResult { + let subject_binary = bincode::serialize(subject) + .map_err(|e| format!("Can't serialize {}: {}", subject, e))?; + let propval_maybe = self + .resources + .get(subject_binary) + .map_err(|e| format!("Can't open {} from store: {}", subject, e))?; + match propval_maybe.as_ref() { + Some(binpropval) => { + let propval: PropVals = bincode::deserialize(binpropval)?; + Ok(propval) + } + None => Err("Not found".into()), + } + } } impl Storelike for Db { @@ -59,15 +87,34 @@ impl Storelike for Db { Ok(()) } + /// Adds a single atom to the store + /// If the resource already exists, it will be inserted into it. + /// Existing data will be overwritten. + /// If the resource does not exist, it will be created. + fn add_atom(&self, atom: Atom) -> AtomicResult<()> { + let mut resource: PropVals = match self.get_propvals(&atom.subject) { + Ok(r) => r, + Err(_) => PropVals::new(), + }; + + resource.insert(atom.property, atom.value.into()); + self.set_propvals(&atom.subject, &resource)?; + Ok(()) + } + fn add_resource(&self, resource: &Resource) -> AtomicResult<()> { - self.add_resource_string(resource.get_subject().clone(), &resource.to_plain())?; + self.set_propvals(resource.get_subject(), &resource.get_propvals())?; Ok(()) } - fn add_resource_string(&self, subject: String, resource: &ResourceString) -> AtomicResult<()> { - let sub_bin = bincode::serialize(&subject)?; - let res_bin = bincode::serialize(resource)?; - self.resources.insert(sub_bin, res_bin)?; + fn add_resource_string( + &self, + subject: String, + resource_string: &ResourceString, + ) -> AtomicResult<()> { + let resource = + crate::resources::Resource::new_from_resource_string(subject, resource_string, self)?; + self.add_resource(&resource)?; // Note that this does not do anything with indexes, so it might have to be replaced! Ok(()) } @@ -77,19 +124,13 @@ impl Storelike for Db { } fn get_resource_string(&self, resource_url: &str) -> AtomicResult { - let subject_binary = bincode::serialize(resource_url).map_err(|e| format!("Can't serialize {}: {}", resource_url, e))?; - match self - .resources - // Todo: return some custom error types here - .get(subject_binary) - .map_err(|e| format!("Can't open {} from store: {}", resource_url, e))? - { - Some(res_bin) => { - let resource: ResourceString = bincode::deserialize(&res_bin) - .map_err(|e| format!("Can't deserialize {}. Your database may be corrupt! {}", resource_url, e))?; + let propvals = self.get_propvals(resource_url); + match propvals { + Ok(propvals) => { + let resource = crate::resources::propvals_to_resourcestring(propvals); Ok(resource) } - None => { + Err(e) => { if resource_url.starts_with(&self.base_url) { return Err(format!( "Failed to retrieve {}, does not exist locally", @@ -113,16 +154,20 @@ impl Storelike for Db { fn all_resources(&self) -> ResourceCollection { let mut resources: ResourceCollection = Vec::new(); for item in self.resources.into_iter() { - let (subject_bin, resource_bin) = item.unwrap(); - let subject: String = bincode::deserialize(&subject_bin).unwrap(); - let resource: ResourceString = bincode::deserialize(&resource_bin).unwrap(); + let (subject, resource_bin) = item.unwrap(); + let subject: String = bincode::deserialize(&subject).unwrap(); + let propvals: PropVals = bincode::deserialize(&resource_bin).unwrap(); + let resource: ResourceString = crate::resources::propvals_to_resourcestring(propvals); resources.push((subject, resource)); } resources } fn remove_resource(&self, subject: &str) { - self.db.remove(bincode::serialize(subject).unwrap()).unwrap().unwrap(); + self.db + .remove(bincode::serialize(subject).unwrap()) + .unwrap() + .unwrap(); } } @@ -130,23 +175,27 @@ impl Storelike for Db { mod test { use super::*; - // Same as examples/basic.rs + /// Creates new temporary database, populates it, removes previous one + fn init() -> Db { + let tmp_dir_path = "tmp/db"; + std::fs::remove_dir_all(tmp_dir_path).unwrap(); + let store = Db::init(tmp_dir_path, "https://localhost/".into()).unwrap(); + store.populate().unwrap(); + store + } + #[test] fn basic() { - // Import the `Storelike` trait to get access to most functions - use crate::Storelike; - // Start with initializing our store - let store = Db::init("tmp/db", "localhost".into()).unwrap(); - // Load the default Atomic Data Atoms - store.populate().unwrap(); - // Let's parse this AD3 string. It looks awkward because of the escaped quotes. - let string = r#"["_:test","https://atomicdata.dev/properties/description","Test"]"#; + let store = init(); + // Let's parse this AD3 string. + let ad3 = + r#"["https://localhost/test","https://atomicdata.dev/properties/description","Test"]"#; // The parser returns a Vector of Atoms - let atoms = crate::parse::parse_ad3(&string).unwrap(); + let atoms = crate::parse::parse_ad3(&ad3).unwrap(); // Add the Atoms to the Store store.add_atoms(atoms).unwrap(); // Get our resource... - let my_resource = store.get_resource("_:test").unwrap(); + let my_resource = store.get_resource("https://localhost/test").unwrap(); // Get our value by filtering on our property... let my_value = my_resource .get("https://atomicdata.dev/properties/description") @@ -158,5 +207,28 @@ mod test { // We can find any Atoms matching some value using Triple Pattern Fragments: let found_atoms = store.tpf(None, None, Some("Test")).unwrap(); assert!(found_atoms.len() == 1); + assert!(found_atoms[0].value == "Test"); + + // We can also create a new Resource, linked to the store. + // Note that since this store only exists in memory, it's data cannot be accessed from the internet. + // Let's make a new Property instance! + let mut new_property = + crate::Resource::new_instance("https://atomicdata.dev/classes/Property", &store) + .unwrap(); + // And add a description for that Property + new_property + .set_by_shortname("description", "the age of a person") + .unwrap(); + // The modified resource is saved to the store after this + + // A subject URL has been created automatically. + let subject = new_property.get_subject(); + let fetched_new_resource = store.get_resource(subject).unwrap(); + let description_val = fetched_new_resource + .get_shortname("description") + .unwrap() + .to_string(); + println!("desc {}", description_val); + assert!(description_val == "the age of a person"); } } diff --git a/lib/src/resources.rs b/lib/src/resources.rs index 20dc35547..281849a47 100644 --- a/lib/src/resources.rs +++ b/lib/src/resources.rs @@ -23,19 +23,10 @@ pub struct Resource<'a> { classes: Option>, /// A reference to the store store: &'a dyn Storelike, - /// A hashmap of nested Resources. - /// A nested resource is a sub-resource, whose Subject is its path. - /// For example, a Class might have a nested Property resource with a subject like this: - /// "https://example.com/someClass https://atomicdata.dev/properties/requires 0" - nested: Nested<'a>, } /// Maps Property URLs to their values -type PropVals = HashMap; - -/// Map of nested resources. -/// A path and a SubResource -type Nested<'a> = HashMap>; +pub type PropVals = HashMap; impl<'a> Resource<'a> { /// Checks if the classes are there, if not, fetches them @@ -50,13 +41,11 @@ impl<'a> Resource<'a> { /// Create a new, empty Resource. pub fn new(subject: String, store: &'a dyn Storelike) -> Resource<'a> { let propvals: PropVals = HashMap::new(); - let nested: Nested = HashMap::new(); Resource { propvals, subject, classes: None, store, - nested, } } @@ -64,7 +53,6 @@ impl<'a> Resource<'a> { /// The subject is generated, but can be changed. pub fn new_instance(class_url: &str, store: &'a dyn Storelike) -> AtomicResult> { let propvals: PropVals = HashMap::new(); - let nested: Nested = HashMap::new(); let mut classes_vec = Vec::new(); classes_vec.push(store.get_class(class_url)?); use rand::Rng; @@ -86,7 +74,6 @@ impl<'a> Resource<'a> { subject, classes, store, - nested, }; let class_urls = Vec::from([String::from(class_url)]); resource.set_propval(crate::urls::IS_A.into(), class_urls.into())?; @@ -115,6 +102,10 @@ impl<'a> Resource<'a> { ))?) } + pub fn get_propvals(&self) -> PropVals { + self.propvals.clone() + } + /// Gets a value by its shortname // Todo: should use both the Classes AND the existing props pub fn get_shortname(&self, shortname: &str) -> AtomicResult<&Value> { @@ -238,6 +229,14 @@ pub fn resourcestring_to_atoms(subject: &str, resource: ResourceString) -> Vec ResourceString { + let mut resource_string: ResourceString = HashMap::new(); + for (prop, val) in propvals.iter() { + resource_string.insert(prop.clone(), val.to_string()); + } + resource_string +} + #[cfg(test)] mod test { use super::*; diff --git a/server/example_requests.http b/server/example_requests.http index 741197cc7..6dd52cf9c 100644 --- a/server/example_requests.http +++ b/server/example_requests.http @@ -2,6 +2,10 @@ GET https://atomicdata.dev/properties/isA HTTP/1.1 Accept: application/json +### Get a thing as JSON-LD +GET https://atomicdata.dev/properties/isA HTTP/1.1 +Accept: application/ld+json + ### Get a thing as AD3 GET https://atomicdata.dev/properties/isA HTTP/1.1 Accept: application/ad3-ndjson @@ -20,15 +24,15 @@ Accept: application/json Content-Type: application/json { - "subject": "http://localhost/somresource", + "subject": "http://0.0.0.0:8081/ttest", "created_at": 1601239744, "actor": "https://example.com/profile", "set": { - "https://atomicdata.dev/properties/description": "A new desription" + "https://atomicdata.dev/properties/requires": "[\"http/properties/requires\"]" }, "remove": ["https://atomicdata.dev/properties/shortname"], "destroy": false, - "signature": "correcthash" + "signature": "correct_signature" } ### Check if above commit has been applied correctly From 398cf54844eef99d16fe26034608d59f7940cab9 Mon Sep 17 00:00:00 2001 From: Joep Meindertsma Date: Sun, 18 Oct 2020 00:10:19 +0200 Subject: [PATCH 04/10] Commits to resources, persist 'em #16 --- lib/defaults/default_store.ad3 | 4 ++ lib/src/commit.rs | 100 +++++++++++++++++++++++---------- lib/src/datatype.rs | 13 +++-- lib/src/storelike.rs | 26 ++++----- lib/src/urls.rs | 1 + lib/src/values.rs | 46 +++++++++++++-- 6 files changed, 140 insertions(+), 50 deletions(-) diff --git a/lib/defaults/default_store.ad3 b/lib/defaults/default_store.ad3 index 3d6505f2c..132f8e993 100644 --- a/lib/defaults/default_store.ad3 +++ b/lib/defaults/default_store.ad3 @@ -104,3 +104,7 @@ ["https://atomicdata.dev/properties/createdAt","https://atomicdata.dev/properties/datatype","https://atomicdata.dev/datatypes/Timestamp"] ["https://atomicdata.dev/properties/createdAt","https://atomicdata.dev/properties/shortname","createdat"] ["https://atomicdata.dev/properties/createdAt","https://atomicdata.dev/properties/description","Timestamp when the Commit was created (usually when it was signed)."] +["https://atomicdata.dev/properties/subject","https://atomicdata.dev/properties/isA","[\"https://atomicdata.dev/classes/Property\"]"] +["https://atomicdata.dev/properties/subject","https://atomicdata.dev/properties/datatype","https://atomicdata.dev/datatypes/atomicURL"] +["https://atomicdata.dev/properties/subject","https://atomicdata.dev/properties/shortname","subject"] +["https://atomicdata.dev/properties/subject","https://atomicdata.dev/properties/description","The subject of a Delta - the resource ID that is being changed."] diff --git a/lib/src/commit.rs b/lib/src/commit.rs index afa9180fb..43e7f5d5a 100644 --- a/lib/src/commit.rs +++ b/lib/src/commit.rs @@ -1,43 +1,81 @@ //! Describe changes / mutations to data -use std::collections::{HashMap, HashSet}; use serde::{Deserialize, Serialize}; +use std::collections::{HashMap, HashSet}; -use crate::{ResourceString, urls}; +use crate::{ + datatype::DataType, errors::AtomicResult, resources::PropVals, urls, Resource, + Storelike, Value, +}; /// A Commit is a set of changes to a Resource. /// Use CommitBuilder if you're programmatically constructing a Delta. #[derive(Debug, Deserialize, Serialize)] pub struct Commit { -/// The subject URL that is to be modified by this Delta -pub subject: String, -/// The date it was created, as a unix timestamp -pub created_at: u128, -/// The URL of the one suggesting this Commit -pub actor: String, -/// The set of PropVals that need to be added. -/// Overwrites existing values -pub set: Option>, -/// The set of property URLs that need to be removed -pub remove: Option>, -/// If set to true, deletes the entire resource -pub destroy: Option, -/// Hash signed by the actor -pub signature: String, + /// The subject URL that is to be modified by this Delta + pub subject: String, + /// The date it was created, as a unix timestamp + pub created_at: u128, + /// The URL of the one suggesting this Commit + pub actor: String, + /// The set of PropVals that need to be added. + /// Overwrites existing values + pub set: Option>, + /// The set of property URLs that need to be removed + pub remove: Option>, + /// If set to true, deletes the entire resource + pub destroy: Option, + /// Hash signed by the actor + pub signature: String, } impl Commit { /// Converts the Commit into a HashMap of strings. - pub fn to_resourcestring(&self) -> ResourceString { - let mut resource = ResourceString::new(); - resource.insert(urls::SUBJECT.into(), self.subject.clone()); - resource.insert(urls::CREATED_AT.into(), self.created_at.to_string()); - resource.insert(urls::ACTOR.into(), self.actor.clone()); - // How to serialize nested resources? - // https://github.com/joepio/atomic/issues/16 - // resource.insert(urls::SET.into(), some_conversion_func); - todo!(); - resource + pub fn into_resource<'a>(&self, store: &'a dyn Storelike) -> AtomicResult> { + let subject = format!( + "{}/{}", + store.get_base_url().unwrap_or("https://localhost".into()), + self.signature + ); + let mut resource = Resource::new_instance(urls::COMMIT, store)?; + resource.set_subject(subject); + resource.set_propval( + urls::SUBJECT.into(), + Value::new(&self.subject, &DataType::AtomicUrl).unwrap(), + )?; + resource.set_propval( + urls::CREATED_AT.into(), + Value::new(&self.created_at.to_string(), &DataType::Timestamp).unwrap(), + )?; + resource.set_propval( + urls::ACTOR.into(), + Value::new(&self.actor, &DataType::AtomicUrl).unwrap(), + )?; + if self.set.is_some() { + let mut newset = PropVals::new(); + for (prop, stringval) in self.set.clone().unwrap() { + let datatype = store.get_property(&prop)?.data_type; + let val = Value::new(&stringval, &datatype)?; + newset.insert(prop, val); + } + resource.set_propval(urls::SET.into(), newset.into())?; + }; + if self.remove.is_some() && !self.remove.clone().unwrap().is_empty() { + let remove_vec: Vec = self.remove.clone().unwrap(); + resource.set_propval(urls::REMOVE.into(), remove_vec.into())?; + }; + if self.destroy.is_some() && self.destroy.unwrap() { + resource.set_propval(urls::DESTROY.into(), true.into())?; + } + resource.set_propval( + urls::ACTOR.into(), + Value::new(&self.actor, &DataType::AtomicUrl).unwrap(), + )?; + resource.set_propval( + urls::SIGNATURE.into(), + self.signature.clone().into(), + )?; + Ok(resource) } } @@ -100,7 +138,7 @@ impl CommitBuilder { /// Whether the resource needs to be removed fully pub fn destroy(&mut self, destroy: bool) { - self.destroy = destroy + self.destroy = destroy } } @@ -112,6 +150,7 @@ mod test { #[test] fn apply_commit() { let store = crate::Store::init(); + store.populate().unwrap(); let subject = String::from("https://example.com/somesubject"); let actor = "HashedThing".into(); let mut partial_commit = CommitBuilder::new(subject.clone(), actor); @@ -119,8 +158,11 @@ mod test { let value = "Some value"; partial_commit.set(property.into(), value.into()); let full_commit = partial_commit.sign("correct_signature"); - store.commit(full_commit).unwrap(); + let stored_commit = store.commit(full_commit).unwrap(); let resource = store.get_resource(&subject).unwrap(); assert!(resource.get(property).unwrap().to_string() == value); + let found_commit = store.get_resource(stored_commit.get_subject()).unwrap(); + println!("{}",found_commit.get_subject()); + assert!(found_commit.get_shortname("subject").unwrap().to_string() == subject); } } diff --git a/lib/src/datatype.rs b/lib/src/datatype.rs index b21599989..731d47fbf 100644 --- a/lib/src/datatype.rs +++ b/lib/src/datatype.rs @@ -7,9 +7,11 @@ use crate::urls; #[derive(Clone, Debug, Serialize, Deserialize)] pub enum DataType { AtomicUrl, + Boolean, Date, Integer, Markdown, + NestedResource, ResourceArray, Slug, String, @@ -19,13 +21,14 @@ pub enum DataType { pub fn match_datatype(string: &str) -> DataType { match string { + urls::ATOMIC_URL => DataType::AtomicUrl, + urls::BOOLEAN => DataType::Boolean, + urls::DATE => DataType::Date, urls::INTEGER => DataType::Integer, - urls::STRING => DataType::String, urls::MARKDOWN => DataType::Markdown, - urls::SLUG => DataType::Slug, - urls::ATOMIC_URL => DataType::AtomicUrl, urls::RESOURCE_ARRAY => DataType::ResourceArray, - urls::DATE => DataType::Date, + urls::SLUG => DataType::Slug, + urls::STRING => DataType::String, urls::TIMESTAMP => DataType::Timestamp, unsupported_datatype => DataType::Unsupported(unsupported_datatype.into()), } @@ -35,9 +38,11 @@ impl fmt::Display for DataType { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { DataType::AtomicUrl => write!(f, "{}", urls::ATOMIC_URL), + DataType::Boolean => write!(f, "{}", urls::BOOLEAN), DataType::Date => write!(f, "{}", urls::DATE), DataType::Integer => write!(f, "{}", urls::INTEGER), DataType::Markdown => write!(f, "{}", urls::MARKDOWN), + DataType::NestedResource => write!(f, "{}", urls::ATOMIC_URL), DataType::ResourceArray => write!(f, "{}", urls::RESOURCE_ARRAY), DataType::Slug => write!(f, "{}", urls::SLUG), DataType::String => write!(f, "{}", urls::STRING), diff --git a/lib/src/storelike.rs b/lib/src/storelike.rs index 5a61c5502..2f13ab7c7 100644 --- a/lib/src/storelike.rs +++ b/lib/src/storelike.rs @@ -74,7 +74,7 @@ pub trait Storelike { /// Apply a single Commit to the store /// Creates, edits or destroys a resource. /// TODO: Should verify the author and the signature. - fn commit(&self, commit: crate::Commit) -> AtomicResult<()> + fn commit(&self, commit: crate::Commit) -> AtomicResult where Self: std::marker::Sized, { @@ -101,24 +101,24 @@ pub trait Storelike { if let Some(destroy) = commit.destroy { if destroy { self.remove_resource(&commit.subject); - return Ok(()); } } - if let Some(set) = commit.set { + if let Some(set) = commit.set.clone() { for (prop, val) in set.iter() { // Warning: this is a very inefficient operation resource.set_propval_string(prop.into(), val)?; } } - if let Some(remove) = commit.remove { + if let Some(remove) = commit.remove.clone() { for prop in remove.iter() { // Warning: this is a very inefficient operation resource.remove_propval(&prop); } } // TOOD: Persist delta to store, use hash as ID - // let commit_resource: Resource = commit.into_resource(store: self); - Ok(()) + let commit_resource: Resource = commit.into_resource(self)?; + self.add_resource(&commit_resource)?; + Ok(commit_resource) } /// Adds a Resource to the store @@ -158,7 +158,7 @@ pub trait Storelike { /// Retrieves a Class from the store by subject URL and converts it into a Class useful for forms fn get_class(&self, subject: &str) -> AtomicResult { // The string representation of the Class - let class_strings = self.get_resource_string(subject).expect("Class not found"); + let class_strings = self.get_resource_string(subject).map_err(|e| format!("Class {} not found: {}", subject, e))?; let shortname = class_strings .get(urls::SHORTNAME) .ok_or("Class has no shortname")?; @@ -404,7 +404,7 @@ pub trait Storelike { if json_ld { // In JSON-LD, the value of a Context Item can be a string or an object. // This object can contain information about the translation or datatype of the value - let ctx_value: SerdeValue = match property.data_type { + let ctx_value: SerdeValue = match property.data_type.clone() { DataType::AtomicUrl => { let mut obj = Map::new(); obj.insert("@id".into(), prop_url.as_str().into()); @@ -438,10 +438,7 @@ pub trait Storelike { obj.insert("@container".into(), "@list".into()); obj.into() } - DataType::Slug => prop_url.as_str().into(), - DataType::String => prop_url.as_str().into(), - DataType::Timestamp => prop_url.as_str().into(), - DataType::Unsupported(_) => prop_url.as_str().into(), + _other => prop_url.as_str().into(), }; context.insert(property.shortname.as_str().into(), ctx_value); } @@ -452,7 +449,8 @@ pub trait Storelike { let jsonval = match native_value { Value::AtomicUrl(val) => SerdeValue::String(val), Value::Date(val) => SerdeValue::String(val), - Value::Integer(val) => SerdeValue::Number(val.into()), + // TODO: Handle big numbers + Value::Integer(val) => serde_json::from_str(&val.to_string()).unwrap_or_default(), Value::Markdown(val) => SerdeValue::String(val), Value::ResourceArray(val) => SerdeValue::Array( val.iter() @@ -463,6 +461,8 @@ pub trait Storelike { Value::String(val) => SerdeValue::String(val), Value::Timestamp(val) => SerdeValue::Number(val.into()), Value::Unsupported(val) => SerdeValue::String(val.value), + Value::Boolean(val) => SerdeValue::Bool(val), + Value::NestedResource(_) => {todo!()}, }; root.insert(property.shortname, jsonval); } diff --git a/lib/src/urls.rs b/lib/src/urls.rs index 9de03faa5..83010840b 100644 --- a/lib/src/urls.rs +++ b/lib/src/urls.rs @@ -4,6 +4,7 @@ pub const CLASS: &str = "https://atomicdata.dev/classes/Class"; pub const PROPERTY: &str = "https://atomicdata.dev/classes/Property"; pub const DATATYPE_CLASS: &str = "https://atomicdata.dev/classes/Datatype"; +pub const COMMIT: &str = "https://atomicdata.dev/classes/Commit"; // Properties pub const SHORTNAME: &str = "https://atomicdata.dev/properties/shortname"; diff --git a/lib/src/values.rs b/lib/src/values.rs index 435f6a7cd..13c1f48f9 100644 --- a/lib/src/values.rs +++ b/lib/src/values.rs @@ -1,6 +1,6 @@ //! A value is the part of an Atom that contains the actual information. -use crate::{errors::AtomicResult, datatype::DataType, datatype::match_datatype}; +use crate::{datatype::DataType, datatype::match_datatype, errors::AtomicResult, resources::PropVals}; use regex::Regex; use serde::{Deserialize, Serialize}; @@ -9,12 +9,14 @@ use serde::{Deserialize, Serialize}; pub enum Value { AtomicUrl(String), Date(String), - Integer(i32), + Integer(i128), Markdown(String), ResourceArray(Vec), Slug(String), String(String), Timestamp(i64), + NestedResource(PropVals), + Boolean(bool), Unsupported(UnsupportedValue), } @@ -32,10 +34,12 @@ pub const SLUG_REGEX: &str = r"^[a-z0-9]+(?:-[a-z0-9]+)*$"; pub const DATE_REGEX: &str = r"^\d{4}\-(0[1-9]|1[012])\-(0[1-9]|[12][0-9]|3[01])$"; impl Value { + /// Creates a new Value from an explicit DataType. + /// Fails if the input string does not convert. pub fn new(value: &str, datatype: &DataType) -> AtomicResult { match datatype { DataType::Integer => { - let val: i32 = value.parse()?; + let val: i128 = value.parse()?; Ok(Value::Integer(val)) } DataType::String => Ok(Value::String(value.into())), @@ -67,10 +71,24 @@ impl Value { .map_err(|e| return format!("Not a valid Timestamp: {}. {}", value, e))?; Ok(Value::Timestamp(val)) } + DataType::NestedResource => { + let val: i64 = value + .parse() + .map_err(|e| return format!("Not a valid Timestamp: {}. {}", value, e))?; + Ok(Value::Timestamp(val)) + } DataType::Unsupported(unsup_url) => Ok(Value::Unsupported(UnsupportedValue { value: value.into(), datatype: unsup_url.into(), })), + DataType::Boolean => { + let bool = match value { + "true" => true, + "false" => false, + other => return Err(format!("Not a valid boolean value: {}, should be 'true' or 'false'.", other).into()), + }; + Ok(Value::Boolean(bool)) + } } } @@ -88,7 +106,13 @@ impl From for Value { impl From for Value { fn from(val: i32) -> Self { - Value::Integer(val) + Value::Integer(val.into()) + } +} + +impl From for Value { + fn from(val: u64) -> Self { + Value::Integer(val.into()) } } @@ -98,6 +122,18 @@ impl From> for Value { } } +impl From for Value { + fn from(val: PropVals) -> Self { + Value::NestedResource(val) + } +} + +impl From for Value { + fn from(val: bool) -> Self { + Value::Boolean(val) + } +} + use std::fmt; impl fmt::Display for Value { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -114,6 +150,8 @@ impl fmt::Display for Value { Value::Slug(s) => write!(f, "{}", s), Value::String(s) => write!(f, "{}", s), Value::Timestamp(i) => write!(f, "{}", i), + Value::NestedResource(n) => write!(f, "{:?}", n), + Value::Boolean(b) => write!(f, "{}", b), Value::Unsupported(u) => write!(f, "{}", u.value), } } From 56d0859f2b39f0d25cd2b89fd88002d8468e9e23 Mon Sep 17 00:00:00 2001 From: Joep Meindertsma Date: Sun, 18 Oct 2020 13:01:12 +0200 Subject: [PATCH 05/10] Fix signature datatype --- lib/defaults/default_store.ad3 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/defaults/default_store.ad3 b/lib/defaults/default_store.ad3 index 132f8e993..fafd307e7 100644 --- a/lib/defaults/default_store.ad3 +++ b/lib/defaults/default_store.ad3 @@ -97,9 +97,9 @@ ["https://atomicdata.dev/properties/actor","https://atomicdata.dev/properties/shortname","actor"] ["https://atomicdata.dev/properties/actor","https://atomicdata.dev/properties/description","The actor is the agent (person, organization or something else) that issued the commit."] ["https://atomicdata.dev/properties/signature","https://atomicdata.dev/properties/isA","[\"https://atomicdata.dev/classes/Property\"]"] -["https://atomicdata.dev/properties/signature","https://atomicdata.dev/properties/datatype","https://atomicdata.dev/datatypes/atomicURL"] +["https://atomicdata.dev/properties/signature","https://atomicdata.dev/properties/datatype","https://atomicdata.dev/datatypes/string"] ["https://atomicdata.dev/properties/signature","https://atomicdata.dev/properties/shortname","signature"] -["https://atomicdata.dev/properties/signature","https://atomicdata.dev/properties/description","The signature is the agent (person, organization or something else) that issued the commit."] +["https://atomicdata.dev/properties/signature","https://atomicdata.dev/properties/description","The signature proves that a Commit is created by a specific Actor. It is a cryptographic proof - an RSA signature of the JSON serialized commit, minus the signature."] ["https://atomicdata.dev/properties/createdAt","https://atomicdata.dev/properties/isA","[\"https://atomicdata.dev/classes/Property\"]"] ["https://atomicdata.dev/properties/createdAt","https://atomicdata.dev/properties/datatype","https://atomicdata.dev/datatypes/Timestamp"] ["https://atomicdata.dev/properties/createdAt","https://atomicdata.dev/properties/shortname","createdat"] From 9df6a10b198abd112253d0736c00cdc7cb166d72 Mon Sep 17 00:00:00 2001 From: Joep Meindertsma Date: Mon, 19 Oct 2020 23:27:45 +0200 Subject: [PATCH 06/10] WIP version up --- cli/Cargo.toml | 4 ++-- cli/wapm.toml | 2 +- lib/Cargo.toml | 2 +- lib/src/commit.rs | 28 ++++++++++++++++++---------- lib/src/db.rs | 2 +- server/Cargo.toml | 2 +- server/src/handlers/commit.rs | 4 ++-- server/src/render/atom.rs | 2 ++ 8 files changed, 28 insertions(+), 18 deletions(-) diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 285c07bfc..de1e0560a 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "atomic-cli" -version = "0.13.0" +version = "0.14.0" authors = ["Joep Meindertsma "] edition = "2018" license = "MIT" @@ -14,6 +14,6 @@ repository = "https://github.com/joepio/atomic" promptly = "0.3.0" clap = "2.33.1" colored = "1.9.3" -atomic_lib = { version = "0.13.0", path = "../lib", features = ["db", "rdf"] } +atomic_lib = { version = "0.14.0", path = "../lib", features = ["db", "rdf"] } dirs = "3.0.1" regex = "1.3.9" diff --git a/cli/wapm.toml b/cli/wapm.toml index 54b8e13be..6adcc714f 100644 --- a/cli/wapm.toml +++ b/cli/wapm.toml @@ -1,6 +1,6 @@ [package] name = "atomic" -version = "0.13.0" +version = "0.14.0" description = "Create, share, fetch and model linked Atomic Data!" license = "MIT" repository = "https://github.com/joepio/atomic" diff --git a/lib/Cargo.toml b/lib/Cargo.toml index a5d547f4a..41270f49a 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "atomic_lib" -version = "0.13.0" +version = "0.14.0" authors = ["Joep Meindertsma "] edition = "2018" license = "MIT" diff --git a/lib/src/commit.rs b/lib/src/commit.rs index 43e7f5d5a..0fa121552 100644 --- a/lib/src/commit.rs +++ b/lib/src/commit.rs @@ -31,10 +31,12 @@ pub struct Commit { impl Commit { /// Converts the Commit into a HashMap of strings. - pub fn into_resource<'a>(&self, store: &'a dyn Storelike) -> AtomicResult> { + /// Creates an identifier using the base_url or a default. + pub fn into_resource<'a>(self, store: &'a dyn Storelike) -> AtomicResult> { + let default_base_url = String::from("https://localhost/"); let subject = format!( - "{}/{}", - store.get_base_url().unwrap_or("https://localhost".into()), + "{}commits/{}", + store.get_base_url().unwrap_or(default_base_url), self.signature ); let mut resource = Resource::new_instance(urls::COMMIT, store)?; @@ -73,18 +75,19 @@ impl Commit { )?; resource.set_propval( urls::SIGNATURE.into(), - self.signature.clone().into(), + self.signature.into(), )?; Ok(resource) } } /// Use this for creating Commits +#[derive(Serialize)] pub struct CommitBuilder { /// The subject URL that is to be modified by this Delta subject: String, /// The date it was created, as a unix timestamp - // pub created_at: u128, + created_at: Option, /// The URL of the one suggesting this Commit actor: String, /// The set of PropVals that need to be added. @@ -101,6 +104,7 @@ impl CommitBuilder { pub fn new(subject: String, actor: String) -> Self { CommitBuilder { subject, + created_at: None, actor, set: HashMap::new(), remove: HashSet::new(), @@ -108,13 +112,17 @@ impl CommitBuilder { } } - /// Creates the Commit. + /// Creates the Commit and signs it using a signature. /// Does not send it - see atomic_lib::client::post_commit - pub fn sign(self, _private_key: &str) -> Commit { - let created_at = std::time::SystemTime::now() + pub fn sign(mut self, _private_key: &str) -> Commit { + self.created_at = Some(std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .expect("Time went backwards") - .as_millis(); + .as_millis()); + + // Todo: Implement signature + // let string = serde_json::to_string(&self); + // let signature = some_lib::sign(string, private_key); Commit { subject: self.subject, @@ -122,7 +130,7 @@ impl CommitBuilder { set: Some(self.set), remove: Some(self.remove.into_iter().collect()), destroy: Some(self.destroy), - created_at, + created_at: self.created_at.unwrap(), // TODO: Hashing signature logic signature: "correct_signature".into(), } diff --git a/lib/src/db.rs b/lib/src/db.rs index 4e86f8def..49433d4f0 100644 --- a/lib/src/db.rs +++ b/lib/src/db.rs @@ -130,7 +130,7 @@ impl Storelike for Db { let resource = crate::resources::propvals_to_resourcestring(propvals); Ok(resource) } - Err(e) => { + Err(_e) => { if resource_url.starts_with(&self.base_url) { return Err(format!( "Failed to retrieve {}, does not exist locally", diff --git a/server/Cargo.toml b/server/Cargo.toml index c18378d06..85c3e75c3 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -11,7 +11,7 @@ repository = "https://github.com/joepio/atomic-cli" [dependencies] # When publishing to cargo this needs to be set, but it needs to be absent when building docker file -atomic_lib = { version = "0.13.0", path = "../lib", features = ["db", "rdf"] } +atomic_lib = { version = "0.14.0", path = "../lib", features = ["db", "rdf"] } serde_json = "1.0.56" serde = { version = "1.0.114", features = ["derive"] } uuid = {version = "0.8.1", features = ["v4"]} diff --git a/server/src/handlers/commit.rs b/server/src/handlers/commit.rs index 59920c54c..2ef63fd32 100644 --- a/server/src/handlers/commit.rs +++ b/server/src/handlers/commit.rs @@ -15,7 +15,7 @@ pub async fn post_commit( let mut context = data.lock().unwrap(); let store = &mut context.store; let mut builder = HttpResponse::Ok(); - store.commit(commit.into_inner())?; - let body = "Commit succesfully applied"; + let commit_resource = store.commit(commit.into_inner())?; + let body = format!("Commit succesfully applied. Can be seen at {}", commit_resource.get_subject()); Ok(builder.body(body)) } diff --git a/server/src/render/atom.rs b/server/src/render/atom.rs index 9a00bdda7..0eed04465 100644 --- a/server/src/render/atom.rs +++ b/server/src/render/atom.rs @@ -33,6 +33,8 @@ pub fn value_to_html(value: &Value) -> String { Value::Date(s) => format!("{:?}", s), Value::Timestamp(i) => format!("{}", i), Value::Unsupported(unsup_url) => format!("{:?}", unsup_url), + Value::NestedResource(n) => format!("{:?}", n), + Value::Boolean(b) => format!("{}", b), } } From d36493214e3ad2c825444fa200256bc19ca9d645 Mon Sep 17 00:00:00 2001 From: Joep Meindertsma Date: Mon, 19 Oct 2020 23:47:18 +0200 Subject: [PATCH 07/10] base_url simpler --- Cargo.lock | 12 ++++++++++-- cli/src/new.rs | 2 ++ lib/src/db.rs | 6 +++--- lib/src/store.rs | 4 ++-- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6a7e79f54..48cc8783c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -544,7 +544,7 @@ dependencies = [ [[package]] name = "atomic-cli" -version = "0.13.0" +version = "0.14.0" dependencies = [ "atomic_lib", "clap", @@ -582,11 +582,13 @@ dependencies = [ [[package]] name = "atomic_lib" -version = "0.13.0" +version = "0.14.0" dependencies = [ + "base64 0.13.0", "bincode", "rand", "regex", + "ring", "rio_api", "rio_turtle", "serde", @@ -691,6 +693,12 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + [[package]] name = "bincode" version = "1.3.1" diff --git a/cli/src/new.rs b/cli/src/new.rs index 5052ae4e9..6139ffcaa 100644 --- a/cli/src/new.rs +++ b/cli/src/new.rs @@ -228,6 +228,8 @@ fn prompt_field( }, DataType::Timestamp => todo!(), DataType::Unsupported(unsup) => panic!("Unsupported datatype: {:?}", unsup), + DataType::Boolean => todo!(), + DataType::NestedResource => todo!(), }; Ok(input) } diff --git a/lib/src/db.rs b/lib/src/db.rs index 49433d4f0..79eb2083c 100644 --- a/lib/src/db.rs +++ b/lib/src/db.rs @@ -22,7 +22,7 @@ pub struct Db { // Stores all Atoms. The key is the atom.value, the value a vector of Atoms. index_vals: sled::Tree, index_props: sled::Tree, - // The current URL + /// The base_url is the domain where the db will be hosted, e.g. http://localhost/ base_url: String, } @@ -119,8 +119,8 @@ impl Storelike for Db { Ok(()) } - fn get_base_url(&self) -> Option { - Some(self.base_url.clone()) + fn get_base_url(&self) -> String { + self.base_url.clone() } fn get_resource_string(&self, resource_url: &str) -> AtomicResult { diff --git a/lib/src/store.rs b/lib/src/store.rs index abe2a87ed..152a50137 100644 --- a/lib/src/store.rs +++ b/lib/src/store.rs @@ -88,10 +88,10 @@ impl Storelike for Store { self.hashmap.lock().unwrap().clone().into_iter().collect() } - fn get_base_url(&self) -> Option { + fn get_base_url(&self) -> String { // TODO Should be implemented later when companion functionality is here // https://github.com/joepio/atomic/issues/6 - Some("https://localhost".into()) + "https://localhost/".into() } fn get_resource_string(&self, resource_url: &str) -> AtomicResult { From 0491caec320153e37b45fcd7e70ccd2912388a2a Mon Sep 17 00:00:00 2001 From: Joep Meindertsma Date: Tue, 20 Oct 2020 00:20:26 +0200 Subject: [PATCH 08/10] Implement signatures & verification #27 --- lib/Cargo.toml | 2 + lib/defaults/default_store.ad3 | 26 +++++++--- lib/examples/signing.rs | 35 ++++++++++++++ lib/src/client.rs | 2 +- lib/src/commit.rs | 87 ++++++++++++++++++---------------- lib/src/resources.rs | 7 ++- lib/src/storelike.rs | 58 +++++++++++++++++------ lib/src/urls.rs | 5 +- server/src/errors.rs | 1 + 9 files changed, 155 insertions(+), 68 deletions(-) create mode 100644 lib/examples/signing.rs diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 41270f49a..439274e74 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -18,6 +18,8 @@ ureq = "1.4.0" rio_turtle = { version = "0.5.0", optional = true} rio_api = { version = "0.5.0", optional = true} rand = "0.7.3" +ring = "0.16.15" +base64 = "0.13.0" [features] db = ["sled", "bincode"] diff --git a/lib/defaults/default_store.ad3 b/lib/defaults/default_store.ad3 index fafd307e7..736f5b40a 100644 --- a/lib/defaults/default_store.ad3 +++ b/lib/defaults/default_store.ad3 @@ -15,11 +15,16 @@ ["https://atomicdata.dev/classes/Datatype","https://atomicdata.dev/properties/description","A Datatype describes a possible type of value, such as 'string' or 'integer'."] ["https://atomicdata.dev/classes/Datatype","https://atomicdata.dev/properties/shortname","datatype"] ["https://atomicdata.dev/classes/Commit","https://atomicdata.dev/properties/isA","[\"https://atomicdata.dev/classes/Class\"]"] -["https://atomicdata.dev/classes/Commit","https://atomicdata.dev/properties/requires","[\"https://atomicdata.dev/properties/subject\",\"https://atomicdata.dev/properties/createdAt\",\"https://atomicdata.dev/properties/actor\",\"https://atomicdata.dev/properties/signature\"]"] +["https://atomicdata.dev/classes/Commit","https://atomicdata.dev/properties/requires","[\"https://atomicdata.dev/properties/subject\",\"https://atomicdata.dev/properties/createdAt\",\"https://atomicdata.dev/properties/signer\",\"https://atomicdata.dev/properties/signature\"]"] ["https://atomicdata.dev/classes/Commit","https://atomicdata.dev/properties/recommends","[\"https://atomicdata.dev/properties/set\",\"https://atomicdata.dev/properties/remove\",\"https://atomicdata.dev/properties/destroy\"]"] ["https://atomicdata.dev/classes/Commit","https://atomicdata.dev/properties/requires","[\"https://atomicdata.dev/properties/shortname\",\"https://atomicdata.dev/properties/description\"]"] ["https://atomicdata.dev/classes/Commit","https://atomicdata.dev/properties/description","A Commit describes a possible type of value, such as 'string' or 'integer'."] ["https://atomicdata.dev/classes/Commit","https://atomicdata.dev/properties/shortname","commit"] +["https://atomicdata.dev/classes/Agent","https://atomicdata.dev/properties/isA","[\"https://atomicdata.dev/classes/Class\"]"] +["https://atomicdata.dev/classes/Agent","https://atomicdata.dev/properties/requires","[\"https://atomicdata.dev/properties/createdAt\",\"https://atomicdata.dev/properties/name\",\"https://atomicdata.dev/properties/publicKey\"]"] +["https://atomicdata.dev/classes/Agent","https://atomicdata.dev/properties/recommends","[\"https://atomicdata.dev/properties/description\",\"https://atomicdata.dev/properties/remove\",\"https://atomicdata.dev/properties/destroy\"]"] +["https://atomicdata.dev/classes/Agent","https://atomicdata.dev/properties/description","An Agent is a user that can create or modify data. It has two keys: a private and a public one. The private key should be kept secret. The publik key is for proving that the "] +["https://atomicdata.dev/classes/Agent","https://atomicdata.dev/properties/shortname","agent"] # Datatypes ["https://atomicdata.dev/datatypes/string","https://atomicdata.dev/properties/shortname","string"] ["https://atomicdata.dev/datatypes/string","https://atomicdata.dev/properties/description","A UTF-8 string. Allows newlines with `\n`. This is a generic string datatype - don't use this for things like [markdown](https://atomicdata.dev/datatypes/markdown) or html."] @@ -92,14 +97,15 @@ ["https://atomicdata.dev/properties/destroy","https://atomicdata.dev/properties/datatype","https://atomicdata.dev/datatypes/boolean"] ["https://atomicdata.dev/properties/destroy","https://atomicdata.dev/properties/shortname","destroy"] ["https://atomicdata.dev/properties/destroy","https://atomicdata.dev/properties/description","If set to true, the entire Subject resource will be removed in this commit. This will be executed _before_ other commands, such as set."] -["https://atomicdata.dev/properties/actor","https://atomicdata.dev/properties/isA","[\"https://atomicdata.dev/classes/Property\"]"] -["https://atomicdata.dev/properties/actor","https://atomicdata.dev/properties/datatype","https://atomicdata.dev/datatypes/atomicURL"] -["https://atomicdata.dev/properties/actor","https://atomicdata.dev/properties/shortname","actor"] -["https://atomicdata.dev/properties/actor","https://atomicdata.dev/properties/description","The actor is the agent (person, organization or something else) that issued the commit."] +["https://atomicdata.dev/properties/signer","https://atomicdata.dev/properties/isA","[\"https://atomicdata.dev/classes/Property\"]"] +["https://atomicdata.dev/properties/signer","https://atomicdata.dev/properties/datatype","https://atomicdata.dev/datatypes/atomicURL"] +["https://atomicdata.dev/properties/signer","https://atomicdata.dev/properties/classtype","https://atomicdata.dev/classes/Agent"] +["https://atomicdata.dev/properties/signer","https://atomicdata.dev/properties/shortname","signer"] +["https://atomicdata.dev/properties/signer","https://atomicdata.dev/properties/description","The signer is the agent (person, organization or something else) that issued the commit."] ["https://atomicdata.dev/properties/signature","https://atomicdata.dev/properties/isA","[\"https://atomicdata.dev/classes/Property\"]"] ["https://atomicdata.dev/properties/signature","https://atomicdata.dev/properties/datatype","https://atomicdata.dev/datatypes/string"] ["https://atomicdata.dev/properties/signature","https://atomicdata.dev/properties/shortname","signature"] -["https://atomicdata.dev/properties/signature","https://atomicdata.dev/properties/description","The signature proves that a Commit is created by a specific Actor. It is a cryptographic proof - an RSA signature of the JSON serialized commit, minus the signature."] +["https://atomicdata.dev/properties/signature","https://atomicdata.dev/properties/description","The signature proves that a Commit is created by a specific Agent. It is a cryptographic proof - an RSA signature of the JSON serialized commit, minus the signature."] ["https://atomicdata.dev/properties/createdAt","https://atomicdata.dev/properties/isA","[\"https://atomicdata.dev/classes/Property\"]"] ["https://atomicdata.dev/properties/createdAt","https://atomicdata.dev/properties/datatype","https://atomicdata.dev/datatypes/Timestamp"] ["https://atomicdata.dev/properties/createdAt","https://atomicdata.dev/properties/shortname","createdat"] @@ -108,3 +114,11 @@ ["https://atomicdata.dev/properties/subject","https://atomicdata.dev/properties/datatype","https://atomicdata.dev/datatypes/atomicURL"] ["https://atomicdata.dev/properties/subject","https://atomicdata.dev/properties/shortname","subject"] ["https://atomicdata.dev/properties/subject","https://atomicdata.dev/properties/description","The subject of a Delta - the resource ID that is being changed."] +["https://atomicdata.dev/properties/name","https://atomicdata.dev/properties/isA","[\"https://atomicdata.dev/classes/Property\"]"] +["https://atomicdata.dev/properties/name","https://atomicdata.dev/properties/datatype","https://atomicdata.dev/datatypes/string"] +["https://atomicdata.dev/properties/name","https://atomicdata.dev/properties/shortname","name"] +["https://atomicdata.dev/properties/name","https://atomicdata.dev/properties/description","The name of a thing or person."] +["https://atomicdata.dev/properties/publicKey","https://atomicdata.dev/properties/isA","[\"https://atomicdata.dev/classes/Property\"]"] +["https://atomicdata.dev/properties/publicKey","https://atomicdata.dev/properties/datatype","https://atomicdata.dev/datatypes/string"] +["https://atomicdata.dev/properties/publicKey","https://atomicdata.dev/properties/shortname","publickey"] +["https://atomicdata.dev/properties/publicKey","https://atomicdata.dev/properties/description","The publicKey of an Agent. Is a base64 serialized Ed25519 key."] diff --git a/lib/examples/signing.rs b/lib/examples/signing.rs new file mode 100644 index 000000000..860911443 --- /dev/null +++ b/lib/examples/signing.rs @@ -0,0 +1,35 @@ +fn main() { + use ring::{ + rand, + signature::{self, KeyPair}, + }; + + // Generate a key pair in PKCS#8 (v2) format. + let rng = rand::SystemRandom::new(); + let pkcs8_bytes = signature::Ed25519KeyPair::generate_pkcs8(&rng).unwrap(); + + // Normally the application would store the PKCS#8 file persistently. Later + // it would read the PKCS#8 file from persistent storage to use it. + + let key_pair = signature::Ed25519KeyPair::from_pkcs8(pkcs8_bytes.as_ref()).unwrap(); + + // Sign the message "hello, world". + const MESSAGE: &[u8] = b"hello, world"; + let sig = key_pair.sign(MESSAGE); + + let pubkey_b64 = base64::encode(key_pair.public_key()); + + let peer_public_key_bytes = base64::decode(pubkey_b64).unwrap(); + + // Normally an application would extract the bytes of the signature and + // send them in a protocol message to the peer(s). Here we just get the + // public key key directly from the key pair. + // let peer_public_key_bytes = key_pair.public_key().as_ref(); + + // Verify the signature of the message using the public key. Normally the + // verifier of the message would parse the inputs to this code out of the + // protocol message(s) sent by the signer. + let peer_public_key = + signature::UnparsedPublicKey::new(&signature::ED25519, peer_public_key_bytes); + peer_public_key.verify(MESSAGE, sig.as_ref()).unwrap(); +} diff --git a/lib/src/client.rs b/lib/src/client.rs index 59240b251..72386b1f3 100644 --- a/lib/src/client.rs +++ b/lib/src/client.rs @@ -56,7 +56,7 @@ mod test { #[test] #[ignore] fn post_commit_basic() { - let commit = crate::commit::CommitBuilder::new("subject".into(), "actor".into()).sign("private_key"); + let commit = crate::commit::CommitBuilder::new("subject".into(), "actor".into()).sign("private_key").unwrap(); post_commit("https://atomicdata.dev/commit", &commit).unwrap(); } } diff --git a/lib/src/commit.rs b/lib/src/commit.rs index 0fa121552..e9bf33307 100644 --- a/lib/src/commit.rs +++ b/lib/src/commit.rs @@ -4,8 +4,7 @@ use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; use crate::{ - datatype::DataType, errors::AtomicResult, resources::PropVals, urls, Resource, - Storelike, Value, + datatype::DataType, errors::AtomicResult, resources::PropVals, urls, Resource, Storelike, Value, }; /// A Commit is a set of changes to a Resource. @@ -17,7 +16,7 @@ pub struct Commit { /// The date it was created, as a unix timestamp pub created_at: u128, /// The URL of the one suggesting this Commit - pub actor: String, + pub signer: String, /// The set of PropVals that need to be added. /// Overwrites existing values pub set: Option>, @@ -25,7 +24,7 @@ pub struct Commit { pub remove: Option>, /// If set to true, deletes the entire resource pub destroy: Option, - /// Hash signed by the actor + /// Base64 encoded signature of the JSON serialized Commit pub signature: String, } @@ -33,12 +32,7 @@ impl Commit { /// Converts the Commit into a HashMap of strings. /// Creates an identifier using the base_url or a default. pub fn into_resource<'a>(self, store: &'a dyn Storelike) -> AtomicResult> { - let default_base_url = String::from("https://localhost/"); - let subject = format!( - "{}commits/{}", - store.get_base_url().unwrap_or(default_base_url), - self.signature - ); + let subject = format!("{}commits/{}", store.get_base_url(), self.signature); let mut resource = Resource::new_instance(urls::COMMIT, store)?; resource.set_subject(subject); resource.set_propval( @@ -50,8 +44,8 @@ impl Commit { Value::new(&self.created_at.to_string(), &DataType::Timestamp).unwrap(), )?; resource.set_propval( - urls::ACTOR.into(), - Value::new(&self.actor, &DataType::AtomicUrl).unwrap(), + urls::SIGNER.into(), + Value::new(&self.signer, &DataType::AtomicUrl).unwrap(), )?; if self.set.is_some() { let mut newset = PropVals::new(); @@ -70,13 +64,10 @@ impl Commit { resource.set_propval(urls::DESTROY.into(), true.into())?; } resource.set_propval( - urls::ACTOR.into(), - Value::new(&self.actor, &DataType::AtomicUrl).unwrap(), - )?; - resource.set_propval( - urls::SIGNATURE.into(), - self.signature.into(), + urls::SIGNER.into(), + Value::new(&self.signer, &DataType::AtomicUrl).unwrap(), )?; + resource.set_propval(urls::SIGNATURE.into(), self.signature.into())?; Ok(resource) } } @@ -89,7 +80,7 @@ pub struct CommitBuilder { /// The date it was created, as a unix timestamp created_at: Option, /// The URL of the one suggesting this Commit - actor: String, + signer: String, /// The set of PropVals that need to be added. /// Overwrites existing values set: std::collections::HashMap, @@ -101,11 +92,11 @@ pub struct CommitBuilder { } impl CommitBuilder { - pub fn new(subject: String, actor: String) -> Self { + pub fn new(subject: String, signer: String) -> Self { CommitBuilder { subject, created_at: None, - actor, + signer, set: HashMap::new(), remove: HashSet::new(), destroy: false, @@ -114,32 +105,41 @@ impl CommitBuilder { /// Creates the Commit and signs it using a signature. /// Does not send it - see atomic_lib::client::post_commit - pub fn sign(mut self, _private_key: &str) -> Commit { - self.created_at = Some(std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .expect("Time went backwards") - .as_millis()); + pub fn sign(mut self, private_key: &str) -> AtomicResult { + self.created_at = Some( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("You're a time traveler") + .as_millis(), + ); - // Todo: Implement signature - // let string = serde_json::to_string(&self); + // TODO: use actual stringified resource, also change in Storelike::commit + // let stringified = serde_json::to_string(&self)?; + let stringified = "full_resource"; + let private_key_bytes = base64::decode(private_key)?; + let key_pair = ring::signature::Ed25519KeyPair::from_pkcs8(&private_key_bytes) + .map_err(|_| "Can't create keypair")?; // let signature = some_lib::sign(string, private_key); + let signature = base64::encode(key_pair.sign(&stringified.as_bytes())); - Commit { + Ok(Commit { subject: self.subject, - actor: self.actor, + signer: self.signer, set: Some(self.set), remove: Some(self.remove.into_iter().collect()), destroy: Some(self.destroy), created_at: self.created_at.unwrap(), // TODO: Hashing signature logic - signature: "correct_signature".into(), - } + signature, + }) } + /// Set Property / Value combinations that will either be created or overwritten. pub fn set(&mut self, prop: String, val: String) { self.set.insert(prop, val); } + /// Set Property URLs which values to be removed pub fn remove(&mut self, prop: String) { self.remove.insert(prop); } @@ -156,21 +156,24 @@ mod test { use crate::Storelike; #[test] - fn apply_commit() { + fn agent_and_commit() { let store = crate::Store::init(); store.populate().unwrap(); - let subject = String::from("https://example.com/somesubject"); - let actor = "HashedThing".into(); - let mut partial_commit = CommitBuilder::new(subject.clone(), actor); + // Creates a new Agent with some crypto stuff + let (agent_subject, private_key) = store.create_agent("test_actor").unwrap(); + let subject = "https://localhost/new_thing"; + let mut commitbuiler = crate::commit::CommitBuilder::new(subject.into(), agent_subject); let property = crate::urls::DESCRIPTION; let value = "Some value"; - partial_commit.set(property.into(), value.into()); - let full_commit = partial_commit.sign("correct_signature"); - let stored_commit = store.commit(full_commit).unwrap(); + commitbuiler.set(property.into(), value.into()); + let commit = commitbuiler.sign(&private_key).unwrap(); + let commit_subject = commit.subject.clone(); + let _created_resource = store.commit(commit).unwrap(); + let resource = store.get_resource(&subject).unwrap(); assert!(resource.get(property).unwrap().to_string() == value); - let found_commit = store.get_resource(stored_commit.get_subject()).unwrap(); - println!("{}",found_commit.get_subject()); - assert!(found_commit.get_shortname("subject").unwrap().to_string() == subject); + let found_commit = store.get_resource(&commit_subject).unwrap(); + println!("{}", found_commit.get_subject()); + assert!(found_commit.get_shortname("description").unwrap().to_string() == value); } } diff --git a/lib/src/resources.rs b/lib/src/resources.rs index 281849a47..4b0b540a2 100644 --- a/lib/src/resources.rs +++ b/lib/src/resources.rs @@ -63,8 +63,7 @@ impl<'a> Resource<'a> { let subject = format!( "{}/{}/{}", store - .get_base_url() - .ok_or("No base_url set in this store.")?, + .get_base_url(), classes_vec[0].shortname.clone(), random_string ); @@ -130,7 +129,7 @@ impl<'a> Resource<'a> { self.save().ok(); } - /// Tries to resolve the shortname to a URL. + /// Tries to resolve the shortname of a Property to a Property URL. // Currently assumes that classes have been set before. pub fn resolve_shortname(&mut self, shortname: &str) -> AtomicResult> { let classes = self.get_classes()?; @@ -181,7 +180,7 @@ impl<'a> Resource<'a> { let fullprop = if is_url(property) { self.store.get_property(property)? } else { - self.resolve_shortname(property)?.unwrap() + self.resolve_shortname(property)?.ok_or(format!("Shortname {} not found in {}", property, self.get_subject()))? }; let fullval = Value::new(value, &fullprop.data_type)?; self.set_propval(fullprop.subject, fullval)?; diff --git a/lib/src/storelike.rs b/lib/src/storelike.rs index 2f13ab7c7..b405e75a6 100644 --- a/lib/src/storelike.rs +++ b/lib/src/storelike.rs @@ -69,11 +69,12 @@ pub trait Storelike { /// E.g. `https://example.com` /// This is where deltas should be sent to. /// Also useful for Subject URL generation. - fn get_base_url(&self) -> Option; + fn get_base_url(&self) -> String; /// Apply a single Commit to the store /// Creates, edits or destroys a resource. - /// TODO: Should verify the author and the signature. + /// Checks if the signature is created by the Agent. + /// Should check if the Agent has the correct rights. fn commit(&self, commit: crate::Commit) -> AtomicResult where Self: std::marker::Sized, @@ -82,13 +83,16 @@ pub trait Storelike { Ok(rs) => rs, Err(_) => Resource::new(commit.subject.clone(), self), }; - match commit.signature.as_str() { - // TODO: check hash - "correct_signature" => {} - _ => return Err("Incorrect signature".into()), - } - // TODO: Check if commit.actor has the rights to update the resource - // TODO: Check if commit.signature matches the actor + // TODO: Check if commit.agent has the rights to update the resource + let pubkey_b64 = self.get_resource(&commit.signer)? + .get(urls::PUBLIC_KEY)?.to_string(); + let agent_pubkey = base64::decode(pubkey_b64)?; + // TODO: actually use the stringified resource + let stringified = "full_resource"; + let peer_public_key = + ring::signature::UnparsedPublicKey::new(&ring::signature::ED25519, agent_pubkey); + let signature_bytes = base64::decode(commit.signature.clone())?; + peer_public_key.verify(stringified.as_bytes(), &signature_bytes).map_err(|_| "Incorrect signature")?; // Check if the created_at lies in the past let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -127,6 +131,31 @@ pub trait Storelike { Ok(()) } + /// Create an Agent, storing its public key. + /// An Agent is required for signing Commits. + /// Returns a tuple of (subject, private_key). + /// Make sure to store the private_key somewhere safe! + fn create_agent(&self, name: &str) -> AtomicResult<(String, String)> + where + Self: std::marker::Sized, + { + use ring::signature::KeyPair; + let subject = format!("{}agents/{}", self.get_base_url(), name); + let rng = ring::rand::SystemRandom::new(); + let pkcs8_bytes = ring::signature::Ed25519KeyPair::generate_pkcs8(&rng) + .map_err(|_| "Error generating seed")?; + let key_pair = ring::signature::Ed25519KeyPair::from_pkcs8(pkcs8_bytes.as_ref()) + .map_err(|_| "Error generating keypair")?; + let mut agent = Resource::new_instance(urls::AGENT, self)?; + let pubkey = base64::encode(key_pair.public_key().as_ref()); + let private_key = base64::encode(pkcs8_bytes.as_ref()); + agent.set_subject(subject.clone()); + agent.set_by_shortname("name", name)?; + agent.set_by_shortname("publickey", &pubkey)?; + self.add_resource(&agent)?; + Ok((subject, private_key)) + } + /// Fetches a resource, makes sure its subject matches. /// Save to the store. /// Only adds atoms with matching subjects will be added. @@ -158,7 +187,9 @@ pub trait Storelike { /// Retrieves a Class from the store by subject URL and converts it into a Class useful for forms fn get_class(&self, subject: &str) -> AtomicResult { // The string representation of the Class - let class_strings = self.get_resource_string(subject).map_err(|e| format!("Class {} not found: {}", subject, e))?; + let class_strings = self + .get_resource_string(subject) + .map_err(|e| format!("Class {} not found: {}", subject, e))?; let shortname = class_strings .get(urls::SHORTNAME) .ok_or("Class has no shortname")?; @@ -303,9 +334,8 @@ pub trait Storelike { }; Ok(()) } + /// DEPRECATED - PREFER COMMITS /// Processes a vector of deltas and updates the store. - /// Panics if the - /// Use this for ALL updates to the store! fn process_delta(&self, delta: DeltaDeprecated) -> AtomicResult<()> { let mut updated_resources = Vec::new(); @@ -462,7 +492,7 @@ pub trait Storelike { Value::Timestamp(val) => SerdeValue::Number(val.into()), Value::Unsupported(val) => SerdeValue::String(val.value), Value::Boolean(val) => SerdeValue::Bool(val), - Value::NestedResource(_) => {todo!()}, + Value::NestedResource(_) => todo!(), }; root.insert(property.shortname, jsonval); } @@ -494,7 +524,7 @@ pub trait Storelike { /// Some("https://atomicdata.dev/properties/isA"), /// Some("[\"https://atomicdata.dev/classes/Class\"]") /// ).unwrap(); - /// assert!(atoms.len() == 4) + /// assert!(atoms.len() == 5) /// ``` // Very costly, slow implementation. // Does not assume any indexing. diff --git a/lib/src/urls.rs b/lib/src/urls.rs index 83010840b..d561f3972 100644 --- a/lib/src/urls.rs +++ b/lib/src/urls.rs @@ -5,6 +5,7 @@ pub const CLASS: &str = "https://atomicdata.dev/classes/Class"; pub const PROPERTY: &str = "https://atomicdata.dev/classes/Property"; pub const DATATYPE_CLASS: &str = "https://atomicdata.dev/classes/Datatype"; pub const COMMIT: &str = "https://atomicdata.dev/classes/Commit"; +pub const AGENT: &str = "https://atomicdata.dev/classes/Agent"; // Properties pub const SHORTNAME: &str = "https://atomicdata.dev/properties/shortname"; @@ -21,9 +22,11 @@ pub const SUBJECT: &str = "https://atomicdata.dev/properties/subject"; pub const SET: &str = "https://atomicdata.dev/properties/set"; pub const REMOVE: &str = "https://atomicdata.dev/properties/remove"; pub const DESTROY: &str = "https://atomicdata.dev/properties/destroy"; -pub const ACTOR: &str = "https://atomicdata.dev/properties/actor"; +pub const SIGNER: &str = "https://atomicdata.dev/properties/signer"; pub const CREATED_AT: &str = "https://atomicdata.dev/properties/createdAt"; pub const SIGNATURE: &str = "https://atomicdata.dev/properties/signature"; +// ... for Agents +pub const PUBLIC_KEY: &str = "https://atomicdata.dev/properties/publicKey"; // Datatypes pub const STRING: &str = "https://atomicdata.dev/datatypes/string"; diff --git a/server/src/errors.rs b/server/src/errors.rs index f77567262..69407fdb7 100644 --- a/server/src/errors.rs +++ b/server/src/errors.rs @@ -60,6 +60,7 @@ impl From<&str> for AppError { } } + impl From> for AppError { fn from(error: std::boxed::Box) -> Self { AppError { From 02facea1889f72202e0bcaa85dcac04e883e619b Mon Sep 17 00:00:00 2001 From: Joep Meindertsma Date: Sun, 25 Oct 2020 15:47:51 +0100 Subject: [PATCH 09/10] Deterministic serialization #27 --- lib/src/commit.rs | 125 +++++++++++++++++++++++++++++++------------ lib/src/storelike.rs | 10 ++-- 2 files changed, 99 insertions(+), 36 deletions(-) diff --git a/lib/src/commit.rs b/lib/src/commit.rs index e9bf33307..fcdedf34f 100644 --- a/lib/src/commit.rs +++ b/lib/src/commit.rs @@ -14,8 +14,8 @@ pub struct Commit { /// The subject URL that is to be modified by this Delta pub subject: String, /// The date it was created, as a unix timestamp - pub created_at: u128, - /// The URL of the one suggesting this Commit + pub created_at: u64, + /// The URL of the one signing this Commit pub signer: String, /// The set of PropVals that need to be added. /// Overwrites existing values @@ -25,14 +25,19 @@ pub struct Commit { /// If set to true, deletes the entire resource pub destroy: Option, /// Base64 encoded signature of the JSON serialized Commit - pub signature: String, + pub signature: Option, } impl Commit { /// Converts the Commit into a HashMap of strings. /// Creates an identifier using the base_url or a default. pub fn into_resource<'a>(self, store: &'a dyn Storelike) -> AtomicResult> { - let subject = format!("{}commits/{}", store.get_base_url(), self.signature); + let subject = match self.signature.as_ref() { + Some(sig) => format!("{}commits/{}", store.get_base_url(), sig), + None => { + return Err("No signature set".into()); + } + }; let mut resource = Resource::new_instance(urls::COMMIT, store)?; resource.set_subject(subject); resource.set_propval( @@ -67,9 +72,55 @@ impl Commit { urls::SIGNER.into(), Value::new(&self.signer, &DataType::AtomicUrl).unwrap(), )?; - resource.set_propval(urls::SIGNATURE.into(), self.signature.into())?; + resource.set_propval(urls::SIGNATURE.into(), self.signature.unwrap().into())?; Ok(resource) } + + /// Generates a deterministic serialized JSON representation of the Commit. + /// Does not contain the signature, since this function is used to check if the signature is correct. + pub fn serialize_deterministically(&self) -> AtomicResult { + let mut obj = serde_json::Map::new(); + obj.insert( + "subject".into(), + serde_json::Value::String(self.subject.clone()), + ); + obj.insert( + "createdAt".into(), + serde_json::Value::Number(self.created_at.into()), + ); + obj.insert( + "signer".into(), + serde_json::Value::String(self.signer.clone()), + ); + if let Some(set) = self.set.clone() { + if !set.is_empty() { + let mut collect: Vec<(String, String)> = set.into_iter().collect(); + // All keys should be ordered alphabetically + collect.sort(); + // Make sure that the serializer does not mess up the order! + let mut set_map = serde_json::Map::new(); + for (k, v) in collect.iter() { + set_map.insert(k.into(), serde_json::Value::String(v.into())); + } + obj.insert("set".into(), serde_json::Value::Object(set_map)); + } + } + if let Some(mut remove) = self.remove.clone() { + if !remove.is_empty() { + // These, too, should be sorted alphabetically + remove.sort(); + obj.insert("remove".into(), remove.into()); + } + } + if let Some(destroy) = self.destroy { + // Only include this key if it is true + if destroy { + obj.insert("destroy".into(), serde_json::Value::Bool(true)); + } + } + let string = serde_json::to_string(&obj)?; + Ok(string) + } } /// Use this for creating Commits @@ -77,8 +128,6 @@ impl Commit { pub struct CommitBuilder { /// The subject URL that is to be modified by this Delta subject: String, - /// The date it was created, as a unix timestamp - created_at: Option, /// The URL of the one suggesting this Commit signer: String, /// The set of PropVals that need to be added. @@ -95,7 +144,6 @@ impl CommitBuilder { pub fn new(subject: String, signer: String) -> Self { CommitBuilder { subject, - created_at: None, signer, set: HashMap::new(), remove: HashSet::new(), @@ -105,33 +153,34 @@ impl CommitBuilder { /// Creates the Commit and signs it using a signature. /// Does not send it - see atomic_lib::client::post_commit - pub fn sign(mut self, private_key: &str) -> AtomicResult { - self.created_at = Some( - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .expect("You're a time traveler") - .as_millis(), - ); + /// Private key is the base64 encoded pkcs8 for the signer + pub fn sign(self, private_key: &str) -> AtomicResult { + let created_at = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("You're a time traveler") + .as_millis(); + + let mut commit = Commit { + subject: self.subject, + signer: self.signer, + set: Some(self.set), + remove: Some(self.remove.into_iter().collect()), + destroy: Some(self.destroy), + created_at: created_at as u64, + signature: None, + }; // TODO: use actual stringified resource, also change in Storelike::commit // let stringified = serde_json::to_string(&self)?; - let stringified = "full_resource"; + // let stringified = "full_resource"; + let stringified = commit.serialize_deterministically()?; let private_key_bytes = base64::decode(private_key)?; let key_pair = ring::signature::Ed25519KeyPair::from_pkcs8(&private_key_bytes) .map_err(|_| "Can't create keypair")?; - // let signature = some_lib::sign(string, private_key); + // let signax ture = some_lib::sign(string, private_key); let signature = base64::encode(key_pair.sign(&stringified.as_bytes())); - - Ok(Commit { - subject: self.subject, - signer: self.signer, - set: Some(self.set), - remove: Some(self.remove.into_iter().collect()), - destroy: Some(self.destroy), - created_at: self.created_at.unwrap(), - // TODO: Hashing signature logic - signature, - }) + commit.signature = Some(signature); + Ok(commit) } /// Set Property / Value combinations that will either be created or overwritten. @@ -163,17 +212,27 @@ mod test { let (agent_subject, private_key) = store.create_agent("test_actor").unwrap(); let subject = "https://localhost/new_thing"; let mut commitbuiler = crate::commit::CommitBuilder::new(subject.into(), agent_subject); - let property = crate::urls::DESCRIPTION; - let value = "Some value"; - commitbuiler.set(property.into(), value.into()); + let property1 = crate::urls::DESCRIPTION; + let value1 = "Some value"; + commitbuiler.set(property1.into(), value1.into()); + let property2 = crate::urls::SHORTNAME; + let value2 = "someval"; + commitbuiler.set(property2.into(), value2.into()); let commit = commitbuiler.sign(&private_key).unwrap(); let commit_subject = commit.subject.clone(); let _created_resource = store.commit(commit).unwrap(); let resource = store.get_resource(&subject).unwrap(); - assert!(resource.get(property).unwrap().to_string() == value); + assert!(resource.get(property1).unwrap().to_string() == value1); let found_commit = store.get_resource(&commit_subject).unwrap(); println!("{}", found_commit.get_subject()); - assert!(found_commit.get_shortname("description").unwrap().to_string() == value); + + assert!( + found_commit + .get_shortname("description") + .unwrap() + .to_string() + == value1 + ); } } diff --git a/lib/src/storelike.rs b/lib/src/storelike.rs index b405e75a6..266db5982 100644 --- a/lib/src/storelike.rs +++ b/lib/src/storelike.rs @@ -83,21 +83,25 @@ pub trait Storelike { Ok(rs) => rs, Err(_) => Resource::new(commit.subject.clone(), self), }; + let signature = match commit.signature.as_ref() { + Some(sig) => sig, + None => return Err("No signature set".into()) + }; // TODO: Check if commit.agent has the rights to update the resource let pubkey_b64 = self.get_resource(&commit.signer)? .get(urls::PUBLIC_KEY)?.to_string(); let agent_pubkey = base64::decode(pubkey_b64)?; // TODO: actually use the stringified resource - let stringified = "full_resource"; + let stringified = commit.serialize_deterministically()?; let peer_public_key = ring::signature::UnparsedPublicKey::new(&ring::signature::ED25519, agent_pubkey); - let signature_bytes = base64::decode(commit.signature.clone())?; + let signature_bytes = base64::decode(signature.clone())?; peer_public_key.verify(stringified.as_bytes(), &signature_bytes).map_err(|_| "Incorrect signature")?; // Check if the created_at lies in the past let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .expect("Time went backwards") - .as_millis(); + .as_millis() as u64; if commit.created_at > now { return Err("Commit created_at timestamp must lie in the past.".into()); // TODO: also check that no younger commits exist From 7cb325fbd241227d11cf414c4c3b365166ef8e9f Mon Sep 17 00:00:00 2001 From: Joep Meindertsma Date: Sun, 25 Oct 2020 16:22:39 +0100 Subject: [PATCH 10/10] Add test for deterministic serialization #27 --- lib/src/commit.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/lib/src/commit.rs b/lib/src/commit.rs index fcdedf34f..8b06c355a 100644 --- a/lib/src/commit.rs +++ b/lib/src/commit.rs @@ -235,4 +235,26 @@ mod test { == value1 ); } + + #[test] + fn serialize_commit() { + let mut set: HashMap = HashMap::new(); + set.insert(urls::SHORTNAME.into(), "shortname".into()); + set.insert(urls::DESCRIPTION.into(), "Some description".into()); + let mut remove = Vec::new(); + remove.push(String::from(urls::IS_A)); + let destroy = false; + let commit = Commit { + subject: String::from("https://localhost/test"), + created_at: 1603638837, + signer: String::from("https://localhost/author"), + set: Some(set), + remove: Some(remove), + destroy: Some(destroy), + signature: None, + }; + let serialized = commit.serialize_deterministically().unwrap(); + let should_be = r#"{"createdAt":1603638837,"remove":["https://atomicdata.dev/properties/isA"],"set":{"https://atomicdata.dev/properties/description":"Some description","https://atomicdata.dev/properties/shortname":"shortname"},"signer":"https://localhost/author","subject":"https://localhost/test"}"#; + assert!(serialized == should_be) + } }