diff --git a/Cargo.lock b/Cargo.lock index 48cc8783c..2c9b54fbf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -544,7 +544,7 @@ dependencies = [ [[package]] name = "atomic-cli" -version = "0.14.0" +version = "0.15.0" dependencies = [ "atomic_lib", "clap", @@ -556,7 +556,7 @@ dependencies = [ [[package]] name = "atomic-server" -version = "0.13.0" +version = "0.15.0" dependencies = [ "acme-lib", "actix-files", @@ -582,7 +582,7 @@ dependencies = [ [[package]] name = "atomic_lib" -version = "0.14.0" +version = "0.15.0" dependencies = [ "base64 0.13.0", "bincode", @@ -595,6 +595,7 @@ dependencies = [ "serde_json", "sled", "ureq", + "url", ] [[package]] diff --git a/README.md b/README.md index 938062b18..ad5e09c6b 100644 --- a/README.md +++ b/README.md @@ -40,15 +40,15 @@ Powers `atomic-cli` and `atomic-server`. I've been working with Linked Data for a couple of years, and I believe it has some incredible merits. URLs are great identifiers, and using them for keys makes sense as well. -However, using the RDF data model has [some characteristics](https://docs.atomicdata.dev/interoperability/rdf.html) that make it difficult for many developers, and that limits adoption. +It has the potential to help a more democratic and decentralized web, where people control their own data. +However, the RDF data model has [some characteristics](https://docs.atomicdata.dev/interoperability/rdf.html) that make it difficult for many developers, and I think that limits adoption. That's why I've been working on a new way to think about linked data: [Atomic Data](https://docs.atomicdata.dev/). Atomic Data is heavily inspired by RDF (and converts nicely into RDF, as it is a strict subset), but introduces some new concepts that aim to make it easier to use for developers. This repository serves the following purposes: -- Test some of the core ideas of Atomic Data ([Atomic Schema](https://docs.atomicdata.dev/schema/intro.html), [Paths](https://docs.atomicdata.dev/core/paths.html), [AD3 Serialization](https://docs.atomicdata.dev/core/serialization.html)) -- Learn how Rust works (it's a cool language, and this is my first Rust project - keep that in mind while traversing the code!) -- Serve the first Atomic Data (now available on [atomicdata.dev](https://atomicdata.dev)), which is referred to by the constantly evolving [Atomic Data Docs](https://docs.atomicdata.dev/) +- Test and experiment with some of the core ideas of Atomic Data, such as [Atomic Schema](https://docs.atomicdata.dev/schema/intro.html) (share models and data types), [Paths](https://docs.atomicdata.dev/core/paths.html) (traversing data), [AD3 Serialization](https://docs.atomicdata.dev/core/serialization.html) and [Atomic Commits](https://docs.atomicdata.dev/commits/intro.html) (storing signed state changes). +- Serve the first Atomic Data, including the core schema (now available on [atomicdata.dev](https://atomicdata.dev)), which is referred to by the constantly evolving [docs](https://docs.atomicdata.dev/) - Provide developers with tools and inspiration to use Atomic Data in their own projects. ## Contribute diff --git a/cli/Cargo.toml b/cli/Cargo.toml index de1e0560a..aeaced001 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "atomic-cli" -version = "0.14.0" +version = "0.15.0" authors = ["Joep Meindertsma "] edition = "2018" license = "MIT" @@ -11,9 +11,9 @@ repository = "https://github.com/joepio/atomic" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +atomic_lib = { version = "0.15.0", path = "../lib", features = ["db", "rdf"] } promptly = "0.3.0" clap = "2.33.1" colored = "1.9.3" -atomic_lib = { version = "0.14.0", path = "../lib", features = ["db", "rdf"] } dirs = "3.0.1" regex = "1.3.9" diff --git a/cli/README.md b/cli/README.md index 041664922..b6c250dd3 100644 --- a/cli/README.md +++ b/cli/README.md @@ -71,7 +71,7 @@ atomic new class - [x] Basic JSON Serialization - [x] RDF (Turtle / N-Triples / RDF/XML) Serialization - [x] Fetch data from the interwebs with `get` commands -- [ ] Works with [`atomic-server`](../server) (fetches from there, stores there, uses domain etc.) +- [ ] Works with [`atomic-server`](../server) (fetches from there, stores there, uses domain etc.) [#6](https://github.com/joepio/atomic/issues/6) - [x] A `delta` command for manipulating existing resources - [ ] Tests for the cli - [ ] A `map` command for creating a bookmark and storing a copy diff --git a/cli/wapm.toml b/cli/wapm.toml index 6adcc714f..051494d69 100644 --- a/cli/wapm.toml +++ b/cli/wapm.toml @@ -1,6 +1,6 @@ [package] name = "atomic" -version = "0.14.0" +version = "0.15.0" description = "Create, share, fetch and model linked Atomic Data!" license = "MIT" repository = "https://github.com/joepio/atomic" diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 439274e74..ca0a10a2b 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "atomic_lib" -version = "0.14.0" +version = "0.15.0" authors = ["Joep Meindertsma "] edition = "2018" license = "MIT" @@ -20,6 +20,7 @@ rio_api = { version = "0.5.0", optional = true} rand = "0.7.3" ring = "0.16.15" base64 = "0.13.0" +url = "2.1.1" [features] db = ["sled", "bincode"] diff --git a/lib/defaults/default_store.ad3 b/lib/defaults/default_store.ad3 index 736f5b40a..ba909ffd5 100644 --- a/lib/defaults/default_store.ad3 +++ b/lib/defaults/default_store.ad3 @@ -25,6 +25,10 @@ ["https://atomicdata.dev/classes/Agent","https://atomicdata.dev/properties/recommends","[\"https://atomicdata.dev/properties/description\",\"https://atomicdata.dev/properties/remove\",\"https://atomicdata.dev/properties/destroy\"]"] ["https://atomicdata.dev/classes/Agent","https://atomicdata.dev/properties/description","An Agent is a user that can create or modify data. It has two keys: a private and a public one. The private key should be kept secret. The publik key is for proving that the "] ["https://atomicdata.dev/classes/Agent","https://atomicdata.dev/properties/shortname","agent"] +["https://atomicdata.dev/classes/Collection","https://atomicdata.dev/properties/isA","[\"https://atomicdata.dev/classes/Class\"]"] +["https://atomicdata.dev/classes/Collection","https://atomicdata.dev/properties/recommends","[\"https://atomicdata.dev/properties/collection/property\"]"] +["https://atomicdata.dev/classes/Collection","https://atomicdata.dev/properties/description","A paginated set of resources that can be sorted."] +["https://atomicdata.dev/classes/Collection","https://atomicdata.dev/properties/shortname","collection"] # Datatypes ["https://atomicdata.dev/datatypes/string","https://atomicdata.dev/properties/shortname","string"] ["https://atomicdata.dev/datatypes/string","https://atomicdata.dev/properties/description","A UTF-8 string. Allows newlines with `\n`. This is a generic string datatype - don't use this for things like [markdown](https://atomicdata.dev/datatypes/markdown) or html."] @@ -122,3 +126,50 @@ ["https://atomicdata.dev/properties/publicKey","https://atomicdata.dev/properties/datatype","https://atomicdata.dev/datatypes/string"] ["https://atomicdata.dev/properties/publicKey","https://atomicdata.dev/properties/shortname","publickey"] ["https://atomicdata.dev/properties/publicKey","https://atomicdata.dev/properties/description","The publicKey of an Agent. Is a base64 serialized Ed25519 key."] +["https://atomicdata.dev/properties/collection/subject","https://atomicdata.dev/properties/isA","[\"https://atomicdata.dev/classes/Property\"]"] +["https://atomicdata.dev/properties/collection/subject","https://atomicdata.dev/properties/datatype","https://atomicdata.dev/datatypes/atomicURL"] +["https://atomicdata.dev/properties/collection/subject","https://atomicdata.dev/properties/shortname","subject"] +["https://atomicdata.dev/properties/collection/subject","https://atomicdata.dev/properties/description","The value is the first field of an atom. Similar to `subject` in RDF. In this context, it is used as a filter in a collection."] +["https://atomicdata.dev/properties/collection/property","https://atomicdata.dev/properties/isA","[\"https://atomicdata.dev/classes/Property\"]"] +["https://atomicdata.dev/properties/collection/property","https://atomicdata.dev/properties/datatype","https://atomicdata.dev/datatypes/atomicURL"] +["https://atomicdata.dev/properties/collection/property","https://atomicdata.dev/properties/shortname","property"] +["https://atomicdata.dev/properties/collection/property","https://atomicdata.dev/properties/description","The property is the second field of an atom. Similar to `predicate` in RDF. In this context, it is used as a filter in a collection."] +["https://atomicdata.dev/properties/collection/value","https://atomicdata.dev/properties/isA","[\"https://atomicdata.dev/classes/Property\"]"] +["https://atomicdata.dev/properties/collection/value","https://atomicdata.dev/properties/datatype","https://atomicdata.dev/datatypes/atomicURL"] +["https://atomicdata.dev/properties/collection/value","https://atomicdata.dev/properties/shortname","value"] +["https://atomicdata.dev/properties/collection/value","https://atomicdata.dev/properties/description","The property is the third field of an atom. Similar to `object` in RDF. In this context, it is used as a filter in a collection."] +["https://atomicdata.dev/properties/collection/members","https://atomicdata.dev/properties/isA","[\"https://atomicdata.dev/classes/Property\"]"] +["https://atomicdata.dev/properties/collection/members","https://atomicdata.dev/properties/datatype","https://atomicdata.dev/datatypes/resourceArray"] +["https://atomicdata.dev/properties/collection/members","https://atomicdata.dev/properties/shortname","members"] +["https://atomicdata.dev/properties/collection/members","https://atomicdata.dev/properties/description","The members are the list of resources in a collection."] +["https://atomicdata.dev/properties/collection/itemCount","https://atomicdata.dev/properties/isA","[\"https://atomicdata.dev/classes/Property\"]"] +["https://atomicdata.dev/properties/collection/itemCount","https://atomicdata.dev/properties/datatype","https://atomicdata.dev/datatypes/integer"] +["https://atomicdata.dev/properties/collection/itemCount","https://atomicdata.dev/properties/shortname","item-count"] +["https://atomicdata.dev/properties/collection/itemCount","https://atomicdata.dev/properties/description","The total number of items in the collection."] +["https://atomicdata.dev/properties/collection/totalPages","https://atomicdata.dev/properties/isA","[\"https://atomicdata.dev/classes/Property\"]"] +["https://atomicdata.dev/properties/collection/totalPages","https://atomicdata.dev/properties/datatype","https://atomicdata.dev/datatypes/integer"] +["https://atomicdata.dev/properties/collection/totalPages","https://atomicdata.dev/properties/shortname","total-pages"] +["https://atomicdata.dev/properties/collection/totalPages","https://atomicdata.dev/properties/description","The total number of pages in the collection."] +["https://atomicdata.dev/properties/collection/currentPage","https://atomicdata.dev/properties/isA","[\"https://atomicdata.dev/classes/Property\"]"] +["https://atomicdata.dev/properties/collection/currentPage","https://atomicdata.dev/properties/datatype","https://atomicdata.dev/datatypes/integer"] +["https://atomicdata.dev/properties/collection/currentPage","https://atomicdata.dev/properties/shortname","current-page"] +["https://atomicdata.dev/properties/collection/currentPage","https://atomicdata.dev/properties/description","The curent page number of the collection. Defaults to 0."] +# Collections +["https://atomicdata.dev/classes","https://atomicdata.dev/properties/isA","[\"https://atomicdata.dev/classes/Collection\"]"] +["https://atomicdata.dev/classes","https://atomicdata.dev/properties/collection/property","https://atomicdata.dev/properties/isA"] +["https://atomicdata.dev/classes","https://atomicdata.dev/properties/collection/value","https://atomicdata.dev/classes/Class"] +["https://atomicdata.dev/properties","https://atomicdata.dev/properties/isA","[\"https://atomicdata.dev/classes/Collection\"]"] +["https://atomicdata.dev/properties","https://atomicdata.dev/properties/collection/property","https://atomicdata.dev/properties/isA"] +["https://atomicdata.dev/properties","https://atomicdata.dev/properties/collection/value","https://atomicdata.dev/classes/Property"] +["https://atomicdata.dev/commits","https://atomicdata.dev/properties/isA","[\"https://atomicdata.dev/classes/Collection\"]"] +["https://atomicdata.dev/commits","https://atomicdata.dev/properties/collection/property","https://atomicdata.dev/properties/isA"] +["https://atomicdata.dev/commits","https://atomicdata.dev/properties/collection/value","https://atomicdata.dev/classes/Commit"] +["https://atomicdata.dev/datatypes","https://atomicdata.dev/properties/isA","[\"https://atomicdata.dev/classes/Collection\"]"] +["https://atomicdata.dev/datatypes","https://atomicdata.dev/properties/collection/property","https://atomicdata.dev/properties/isA"] +["https://atomicdata.dev/datatypes","https://atomicdata.dev/properties/collection/value","https://atomicdata.dev/classes/Datatype"] +["https://atomicdata.dev/agents","https://atomicdata.dev/properties/isA","[\"https://atomicdata.dev/classes/Collection\"]"] +["https://atomicdata.dev/agents","https://atomicdata.dev/properties/collection/property","https://atomicdata.dev/properties/isA"] +["https://atomicdata.dev/agents","https://atomicdata.dev/properties/collection/value","https://atomicdata.dev/classes/Agent"] +["https://atomicdata.dev/collections","https://atomicdata.dev/properties/isA","[\"https://atomicdata.dev/classes/Collection\"]"] +["https://atomicdata.dev/collections","https://atomicdata.dev/properties/collection/property","https://atomicdata.dev/properties/isA"] +["https://atomicdata.dev/collections","https://atomicdata.dev/properties/collection/value","https://atomicdata.dev/classes/Collection"] diff --git a/lib/src/collections.rs b/lib/src/collections.rs index de8e8744b..c0b79c6b7 100644 --- a/lib/src/collections.rs +++ b/lib/src/collections.rs @@ -1,3 +1,8 @@ +//! Collections are dynamic resources that refer to multiple resources. +//! They are constructed using a TPF query + +use crate::errors::AtomicResult; + #[derive(Debug)] pub struct TPFQuery { pub subject: Option, @@ -5,34 +10,60 @@ pub struct TPFQuery { pub value: Option, } +pub struct CollectionBuilder { + pub subject: String, + pub property: Option, + pub value: Option, + pub sort_by: Option, + pub sort_desc: bool, + pub current_page: usize, + pub page_size: usize, +} + /// Dynamic resource used for ordering, filtering and querying content. /// Features pagination. #[derive(Debug)] pub struct Collection { - // The set of triples that form the basis of the data - pub tpf: TPFQuery, - // List of all the pages. - pub pages: Vec, + // Full Subject URL of the resource, including query parameters + pub subject: String, + /// The TPF property which the results are to be filtered by + pub property: Option, + /// The TPF value which the results are to be filtered by + pub value: Option, + // The actual items that you're interested in. List the member subjects of the current page. + pub members: Vec, // URL of the value to sort by - pub sort_by: String, + pub sort_by: Option, // Sorts ascending by default pub sort_desc: bool, // How many items per page - pub page_size: u8, - // Current page number, defaults to 0 (firs page) - pub current_page: u8, + pub page_size: usize, + // Current page number, defaults to 0 (first page) + pub current_page: usize, // Total number of items - pub total_items: u8, + pub total_items: usize, // Total number of pages - pub total_pages: u8, + pub total_pages: usize, } -/// A single page of a Collection -#[derive(Debug)] -pub struct Page { - // partOf: Collection, - // The individual items in the page - pub members: Vec, +impl Collection { + pub fn to_resource<'a>(&self, store: &'a dyn crate::Storelike) -> AtomicResult> { + // TODO: Should not persist, because now it is spammimg the store! + // let mut resource = crate::Resource::new_instance(crate::urls::COLLECTION, store)?; + let mut resource = crate::Resource::new(self.subject.clone(), store); + resource.set_propval(crate::urls::MEMBERS.into(), self.members.clone().into())?; + if let Some(prop) = self.property.clone() { + resource.set_propval(crate::urls::COLLECTION_PROPERTY.into(), prop.into())?; + } + if let Some(prop) = self.value.clone() { + resource.set_propval(crate::urls::COLLECTION_VALUE.into(), prop.into())?; + } + resource.set_propval(crate::urls::COLLECTION_ITEM_COUNT.into(), self.total_items.clone().into())?; + resource.set_propval(crate::urls::COLLECTION_TOTAL_PAGES.into(), self.total_pages.clone().into())?; + resource.set_propval(crate::urls::COLLECTION_CURRENT_PAGE.into(), self.current_page.clone().into())?; + // Maybe include items directly + Ok(resource) + } } #[cfg(test)] @@ -45,13 +76,27 @@ mod test { fn create_collection() { let store = crate::Store::init(); store.populate().unwrap(); - let tpf = TPFQuery { - subject: None, - property: Some(urls::IS_A.into()), - value: Some(urls::CLASS.into()), - }; // Get all Classes, sorted by shortname - let collection = store.get_collection(tpf, urls::SHORTNAME.into(), false, 1, 1).unwrap(); - assert!(collection.pages[0].members.contains(&urls::PROPERTY.into())); + let collection_builder = CollectionBuilder { + subject: "test_subject".into(), + property: Some(urls::IS_A.into()), + value: Some(urls::CLASS.into()), + sort_by: None, + sort_desc: false, + page_size: 1000, + current_page: 0, + }; + let collection = store.new_collection(collection_builder).unwrap(); + assert!(collection.members.contains(&urls::PROPERTY.into())); + } + + #[test] + fn get_collection() { + let store = crate::Store::init(); + store.populate().unwrap(); + let collection = store.get_resource_extended("https://atomicdata.dev/classes").unwrap(); + assert!(collection.get(urls::COLLECTION_PROPERTY).unwrap().to_string() == urls::IS_A); + println!("Count is {}", collection.get(urls::COLLECTION_ITEM_COUNT).unwrap().to_string()); + assert!(collection.get(urls::COLLECTION_ITEM_COUNT).unwrap().to_string() == "6"); } } diff --git a/lib/src/db.rs b/lib/src/db.rs index 79eb2083c..2ec1e27b7 100644 --- a/lib/src/db.rs +++ b/lib/src/db.rs @@ -219,6 +219,8 @@ mod test { new_property .set_by_shortname("description", "the age of a person") .unwrap(); + // Changes are only applied to the store after calling `.save()` + new_property.save().unwrap(); // The modified resource is saved to the store after this // A subject URL has been created automatically. diff --git a/lib/src/resources.rs b/lib/src/resources.rs index 4b0b540a2..d121b0eb5 100644 --- a/lib/src/resources.rs +++ b/lib/src/resources.rs @@ -11,7 +11,7 @@ use std::collections::HashMap; /// A Resource is a set of Atoms that shares a single Subject. /// A Resource only contains valid Values, but it _might_ lack required properties. -/// All changes to the Resource are immediately applied to the Store as well. +/// All changes to the Resource are applied after calling `.save()`. // #[derive(Clone, Debug, Serialize, Deserialize)] pub struct Resource<'a> { /// A hashMap of all the Property Value combinations @@ -107,12 +107,12 @@ impl<'a> Resource<'a> { /// Gets a value by its shortname // Todo: should use both the Classes AND the existing props - pub fn get_shortname(&self, shortname: &str) -> AtomicResult<&Value> { + pub fn get_shortname(&self, shortname: &str) -> AtomicResult { // If there is a class for (url, _val) in self.propvals.iter() { if let Ok(prop) = self.store.get_property(url) { if prop.shortname == shortname { - return Ok(self.get(url)?); + return Ok(self.get(url)?.clone()); } } } @@ -125,13 +125,11 @@ impl<'a> Resource<'a> { /// Validates the datatype. pub fn remove_propval(&mut self, property_url: &str) { self.propvals.remove_entry(property_url); - // Could fail, but unlikely - self.save().ok(); } /// Tries to resolve the shortname of a Property to a Property URL. // Currently assumes that classes have been set before. - pub fn resolve_shortname(&mut self, shortname: &str) -> AtomicResult> { + pub fn resolve_shortname_to_property(&mut self, shortname: &str) -> AtomicResult> { let classes = self.get_classes()?; // Loop over all Requires and Recommends props for class in classes { @@ -150,8 +148,8 @@ impl<'a> Resource<'a> { } /// Saves the resource (with all the changes) to the store - /// Should be called automatically, but we might add some form of batching later - fn save(&self) -> AtomicResult<()> { + /// Should be run after any (batch of) changes to the Resource! + pub fn save(&self) -> AtomicResult<()> { self.store.add_resource(self) } @@ -169,7 +167,6 @@ impl<'a> Resource<'a> { /// Overwrites existing. pub fn set_propval(&mut self, property: String, value: Value) -> AtomicResult<()> { self.propvals.insert(property, value); - self.save()?; Ok(()) } @@ -180,7 +177,7 @@ impl<'a> Resource<'a> { let fullprop = if is_url(property) { self.store.get_property(property)? } else { - self.resolve_shortname(property)?.ok_or(format!("Shortname {} not found in {}", property, self.get_subject()))? + self.resolve_shortname_to_property(property)?.ok_or(format!("Shortname {} not found in {}", property, self.get_subject()))? }; let fullval = Value::new(value, &fullprop.data_type)?; self.set_propval(fullprop.subject, fullval)?; @@ -272,6 +269,7 @@ mod test { new_resource.set_by_shortname("shortname", "person").unwrap(); assert!(new_resource.get_shortname("shortname").unwrap().to_string() == "person"); new_resource.set_by_shortname("shortname", "human").unwrap(); + new_resource.save().unwrap(); assert!(new_resource.get_shortname("shortname").unwrap().to_string() == "human"); let mut resource_from_store = store.get_resource(new_resource.get_subject()).unwrap(); assert!(resource_from_store.get_shortname("shortname").unwrap().to_string() == "human"); diff --git a/lib/src/store.rs b/lib/src/store.rs index 152a50137..ccd7865ea 100644 --- a/lib/src/store.rs +++ b/lib/src/store.rs @@ -227,6 +227,14 @@ mod test { store.get_resource_string(urls::CLASS).unwrap(); } + #[test] + fn get_extended_resource() { + let store = Store::init(); + store.populate().unwrap(); + let resource = store.get_resource_extended("https://atomicdata.dev/classes").unwrap(); + resource.get(urls::MEMBERS).unwrap(); + } + #[test] #[should_panic] fn path_fail() { diff --git a/lib/src/storelike.rs b/lib/src/storelike.rs index 266db5982..a7fab2038 100644 --- a/lib/src/storelike.rs +++ b/lib/src/storelike.rs @@ -1,10 +1,7 @@ //! Trait for all stores to use use crate::urls; -use crate::{ - collections::Collection, collections::Page, collections::TPFQuery, delta::DeltaDeprecated, - errors::AtomicResult, -}; +use crate::{collections::Collection, delta::DeltaDeprecated, errors::AtomicResult}; use crate::{ datatype::{match_datatype, DataType}, mapping::Mapping, @@ -79,24 +76,24 @@ pub trait Storelike { where Self: std::marker::Sized, { - let mut resource = match self.get_resource(&commit.subject) { - Ok(rs) => rs, - Err(_) => Resource::new(commit.subject.clone(), self), - }; let signature = match commit.signature.as_ref() { Some(sig) => sig, - None => return Err("No signature set".into()) + None => return Err("No signature set".into()), }; // TODO: Check if commit.agent has the rights to update the resource - let pubkey_b64 = self.get_resource(&commit.signer)? - .get(urls::PUBLIC_KEY)?.to_string(); + let pubkey_b64 = self + .get_resource(&commit.signer)? + .get(urls::PUBLIC_KEY)? + .to_string(); let agent_pubkey = base64::decode(pubkey_b64)?; // TODO: actually use the stringified resource let stringified = commit.serialize_deterministically()?; let peer_public_key = ring::signature::UnparsedPublicKey::new(&ring::signature::ED25519, agent_pubkey); let signature_bytes = base64::decode(signature.clone())?; - peer_public_key.verify(stringified.as_bytes(), &signature_bytes).map_err(|_| "Incorrect signature")?; + peer_public_key + .verify(stringified.as_bytes(), &signature_bytes) + .map_err(|_| "Incorrect signature")?; // Check if the created_at lies in the past let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -111,17 +108,23 @@ pub trait Storelike { self.remove_resource(&commit.subject); } } + let mut resource = match self.get_resource(&commit.subject) { + Ok(rs) => rs, + Err(_) => Resource::new(commit.subject.clone(), self), + }; if let Some(set) = commit.set.clone() { for (prop, val) in set.iter() { // Warning: this is a very inefficient operation resource.set_propval_string(prop.into(), val)?; } + resource.save()?; } if let Some(remove) = commit.remove.clone() { for prop in remove.iter() { // Warning: this is a very inefficient operation resource.remove_propval(&prop); } + resource.save()?; } // TOOD: Persist delta to store, use hash as ID let commit_resource: Resource = commit.into_resource(self)?; @@ -253,42 +256,63 @@ pub trait Storelike { } /// Constructs a Collection, which is a paginated list of items with some sorting applied. - fn get_collection( + fn new_collection( &self, - tpf: TPFQuery, - sort_by: String, - sort_desc: bool, - _page_nr: u8, - _page_size: u8, + collection: crate::collections::CollectionBuilder, ) -> AtomicResult { // Execute the TPF query, get all the subjects. - let atoms = self.tpf(None, tpf.property.as_deref(), tpf.value.as_deref())?; + let atoms = self.tpf( + None, + collection.property.as_deref(), + collection.value.as_deref(), + )?; // Iterate over the fetched resources let subjects: Vec = atoms.iter().map(|atom| atom.subject.clone()).collect(); - let mut resources: Vec = Vec::new(); - for sub in subjects.clone() { - resources.push(self.get_resource_string(&sub)?); - } // Sort the resources (TODO), use sortBy and sortDesc - let sorted_subjects: Vec = subjects.clone(); + if collection.sort_by.is_some() { + return Err("Sorting is not yet implemented".into()); + } + let sorted_subjects: Vec = subjects; + let mut all_pages: Vec> = Vec::new(); + let mut page: Vec = Vec::new(); + let current_page = collection.current_page; + for (i, subject) in sorted_subjects.iter().enumerate() { + page.push(subject.into()); + if page.len() >= collection.page_size { + all_pages.push(page); + page = Vec::new(); + // No need to calculte more than necessary + if all_pages.len() > current_page { + break; + } + } + // Add the last page when handling the last subject + if i == sorted_subjects.len() - 1 { + all_pages.push(page); + break; + } + } + if all_pages.is_empty() { + all_pages.push(Vec::new()) + } + // Maybe I should default to last page, if current_page is too high? + let members = all_pages.get(current_page).ok_or("Page number is too high")?.clone(); + let total_items = sorted_subjects.len(); // Construct the pages (TODO), use pageSize - let mut pages: Vec = Vec::new(); - // Construct the requested page (TODO) - let page = Page { - members: sorted_subjects, - }; - pages.push(page); - let collection = Collection { - tpf, - total_pages: pages.len() as u8, - pages, - sort_by, - sort_desc, - current_page: 0, - total_items: subjects.len() as u8, - page_size: subjects.len() as u8, + let total_pages = (total_items + collection.page_size - 1) / collection.page_size; + let collection_return = Collection { + total_pages, + members, + total_items, + subject: collection.subject, + property: collection.property, + value: collection.value, + sort_by: collection.sort_by, + sort_desc: collection.sort_desc, + current_page: collection.current_page, + page_size: collection.page_size, }; - Ok(collection) + Ok(collection_return) } /// Fetches a property by URL, returns a Property instance @@ -315,6 +339,62 @@ pub trait Storelike { Ok(property) } + /// Get's the resource, parses the Query parameters and calculates dynamic properties. + /// Currently only used for getting + fn get_resource_extended(&self, subject: &str) -> AtomicResult + where + Self: std::marker::Sized, + { + let mut url = url::Url::parse(subject)?; + let clone = url.clone(); + let query_params = clone.query_pairs(); + url.set_query(None); + let removed_query_params = url.to_string(); + let mut resource = self.get_resource(&removed_query_params)?; + for class in resource.get_classes()? { + let mut sort_by = None; + let mut sort_desc = false; + let mut current_page = 0; + let mut page_size = 100; + let mut value = None; + let mut property = None; + + if let Ok(val) = resource.get(urls::COLLECTION_PROPERTY) { + property = Some(val.to_string()); + } + if let Ok(val) = resource.get(urls::COLLECTION_VALUE) { + value = Some(val.to_string()); + } + + if class.subject == urls::COLLECTION { + for (k, v) in query_params { + match k.as_ref() { + "property" => property = Some(v.to_string()), + "value" => value = Some(v.to_string()), + "sort_by" => sort_by = Some(v.to_string()), + // TODO: parse bool + "sort_desc" => sort_desc = true, + "current_page" => current_page = v.parse::()?, + "page_size" => page_size = v.parse::()?, + _ => {} + }; + } + let collection_builder = crate::collections::CollectionBuilder { + subject: subject.into(), + property, + value, + sort_by, + sort_desc, + current_page, + page_size, + }; + let collection = self.new_collection(collection_builder)?; + return Ok(collection.to_resource(self)?); + } + } + Ok(resource) + } + /// Returns a collection with all resources in the store. /// WARNING: This could be very expensive! fn all_resources(&self) -> ResourceCollection; @@ -338,6 +418,7 @@ pub trait Storelike { }; Ok(()) } + /// DEPRECATED - PREFER COMMITS /// Processes a vector of deltas and updates the store. fn process_delta(&self, delta: DeltaDeprecated) -> AtomicResult<()> { @@ -528,7 +609,8 @@ pub trait Storelike { /// Some("https://atomicdata.dev/properties/isA"), /// Some("[\"https://atomicdata.dev/classes/Class\"]") /// ).unwrap(); - /// assert!(atoms.len() == 5) + /// println!("Count: {}", atoms.len()); + /// assert!(atoms.len() == 6) /// ``` // Very costly, slow implementation. // Does not assume any indexing. @@ -576,10 +658,12 @@ pub trait Storelike { if val_equals(val) { vec.push(Atom::new(subj.into(), prop.into(), val.into())) } + break; } else { vec.push(Atom::new(subj.into(), prop.into(), val.into())) } - } else if hasval && val_equals(val) { + break; + } else if hasval && !hasprop && val_equals(val) { vec.push(Atom::new(subj.into(), prop.into(), val.into())) } } @@ -588,7 +672,7 @@ pub trait Storelike { match q_subject { Some(sub) => match self.get_resource_string(&sub) { Ok(resource) => { - if q_property.is_some() | q_value.is_some() { + if hasprop | hasval { find_in_resource(&sub, &resource); Ok(vec) } else { @@ -610,7 +694,10 @@ pub trait Storelike { /// E.g. `https://example.com description` or `thing isa 0` /// https://docs.atomicdata.dev/core/paths.html // Todo: return something more useful, give more context. - fn get_path(&self, atomic_path: &str, mapping: Option<&Mapping>) -> AtomicResult { + fn get_path(&self, atomic_path: &str, mapping: Option<&Mapping>) -> AtomicResult + where + Self: std::marker::Sized, + { // The first item of the path represents the starting Resource, the following ones are traversing the graph / selecting properties. let path_items: Vec<&str> = atomic_path.split(' ').collect(); let first_item = String::from(path_items[0]); @@ -628,7 +715,8 @@ pub trait Storelike { // The URL of the next resource let mut subject = id_url; // Set the currently selectred resource parent, which starts as the root of the search - let mut resource = self.get_resource_string(&subject)?; + // let mut resource = self.get_resource_string(&subject)?; + let mut resource = self.get_resource_extended(&subject)?; // During each of the iterations of the loop, the scope changes. // Try using pathreturn... let mut current: PathReturn = PathReturn::Subject(subject.clone()); @@ -644,22 +732,20 @@ pub trait Storelike { if let Ok(i) = item.parse::() { match current { PathReturn::Atom(atom) => { - // let resource_check = resource.ok_or("Resource not found")?; - let array_string = resource - .get(&atom.property.subject) - .ok_or(format!("Property {} not found", &atom.property.subject))?; - let vector: Vec = crate::parse::parse_json_array(array_string) - .expect(&*format!("Failed to parse array: {}", array_string)); + let vector = match resource.get(&atom.property.subject)? { + Value::ResourceArray(vec) => vec, + _ => return Err("Should be Vector!".into()), + }; if vector.len() <= i as usize { eprintln!( "Too high index ({}) for array with length {}", i, - array_string.len() + vector.len() ); } let url = &vector[i as usize]; subject = url.into(); - resource = self.get_resource_string(&subject)?; + resource = self.get_resource_extended(&subject)?; current = PathReturn::Subject(subject.clone()); continue; } @@ -669,28 +755,20 @@ pub trait Storelike { } } // Since the selector isn't an array index, we can assume it's a property URL - let property_url; match current { PathReturn::Subject(_) => {} PathReturn::Atom(_) => { return Err("No more linked resources down this path.".into()) } } - // Get the shortname or use the URL - if crate::mapping::is_url(item) { - property_url = item.into(); - } else { - // Traverse relations, don't use mapping here, but do use classes - property_url = self.property_shortname_to_url(item, &resource)?; - } // Set the parent for the next loop equal to the next node. // TODO: skip this step if the current iteration is the last one - let value = resource.get(&property_url).unwrap(); - let property = self.get_property(&property_url)?; + let value = resource.get_shortname(&item).unwrap(); + let property = resource.resolve_shortname_to_property(item)?.unwrap(); current = PathReturn::Atom(Box::new(RichAtom::new( subject.clone(), property, - value.clone(), + value.to_string(), )?)) } Ok(current) diff --git a/lib/src/urls.rs b/lib/src/urls.rs index d561f3972..e2d436f5e 100644 --- a/lib/src/urls.rs +++ b/lib/src/urls.rs @@ -6,6 +6,7 @@ pub const PROPERTY: &str = "https://atomicdata.dev/classes/Property"; pub const DATATYPE_CLASS: &str = "https://atomicdata.dev/classes/Datatype"; pub const COMMIT: &str = "https://atomicdata.dev/classes/Commit"; pub const AGENT: &str = "https://atomicdata.dev/classes/Agent"; +pub const COLLECTION: &str = "https://atomicdata.dev/classes/Collection"; // Properties pub const SHORTNAME: &str = "https://atomicdata.dev/properties/shortname"; @@ -27,6 +28,14 @@ pub const CREATED_AT: &str = "https://atomicdata.dev/properties/createdAt"; pub const SIGNATURE: &str = "https://atomicdata.dev/properties/signature"; // ... for Agents pub const PUBLIC_KEY: &str = "https://atomicdata.dev/properties/publicKey"; +// ... for Collections +pub const MEMBERS: &str = "https://atomicdata.dev/properties/members"; +pub const COLLECTION_PROPERTY: &str = "https://atomicdata.dev/properties/collection/property"; +pub const COLLECTION_VALUE: &str = "https://atomicdata.dev/properties/collection/value"; +pub const COLLECTION_ITEM_COUNT: &str = "https://atomicdata.dev/properties/collection/itemCount"; +pub const COLLECTION_TOTAL_PAGES: &str = "https://atomicdata.dev/properties/collection/totalPages"; +pub const COLLECTION_CURRENT_PAGE: &str = "https://atomicdata.dev/properties/collection/currentPage"; +pub const COLLECTION_MEMBERS: &str = "https://atomicdata.dev/properties/collection/members"; // Datatypes pub const STRING: &str = "https://atomicdata.dev/datatypes/string"; diff --git a/lib/src/values.rs b/lib/src/values.rs index 13c1f48f9..33889ff74 100644 --- a/lib/src/values.rs +++ b/lib/src/values.rs @@ -9,7 +9,7 @@ use serde::{Deserialize, Serialize}; pub enum Value { AtomicUrl(String), Date(String), - Integer(i128), + Integer(isize), Markdown(String), ResourceArray(Vec), Slug(String), @@ -39,7 +39,7 @@ impl Value { pub fn new(value: &str, datatype: &DataType) -> AtomicResult { match datatype { DataType::Integer => { - let val: i128 = value.parse()?; + let val: isize = value.parse()?; Ok(Value::Integer(val)) } DataType::String => Ok(Value::String(value.into())), @@ -106,13 +106,19 @@ impl From for Value { impl From for Value { fn from(val: i32) -> Self { - Value::Integer(val.into()) + Value::Integer(val as isize) } } impl From for Value { fn from(val: u64) -> Self { - Value::Integer(val.into()) + Value::Integer(val as isize) + } +} + +impl From for Value { + fn from(val: usize) -> Self { + Value::Integer(val as isize) } } diff --git a/server/Cargo.toml b/server/Cargo.toml index 85c3e75c3..0789735b0 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "atomic-server" description = "Create, share and standardize linked atomic data!" -version = "0.13.0" +version = "0.15.0" authors = ["Joep Meindertsma "] edition = "2018" license = "MIT" @@ -11,7 +11,7 @@ repository = "https://github.com/joepio/atomic-cli" [dependencies] # When publishing to cargo this needs to be set, but it needs to be absent when building docker file -atomic_lib = { version = "0.14.0", path = "../lib", features = ["db", "rdf"] } +atomic_lib = { version = "0.15.0", path = "../lib", features = ["db", "rdf"] } serde_json = "1.0.56" serde = { version = "1.0.114", features = ["derive"] } uuid = {version = "0.8.1", features = ["v4"]} diff --git a/server/README.md b/server/README.md index 3f7282037..57df5d291 100644 --- a/server/README.md +++ b/server/README.md @@ -30,12 +30,13 @@ Powered by Rust, atomic_lib, actix-web, Sled and [more](cargo.toml). - [x] Content-type negotiation - [x] Basic design / use CSS framework - [x] Validation endpoint -- [x] Write / Commit / [Mutations](https://docs.atomicdata.dev/mutations/intro.html) support (might need #16, #24) -- [ ] Eliminate all preventable runtime panics (most already done) -- [ ] URL extension recognition (.json, .ad3, .nt, etc.) -- [ ] Collections / dynamic resources #17 -- [ ] Auth support (WebID-OICD possibly?) #13 -- [ ] Be able to manage the AtomicData.dev website without git +- [x] Atomic Commits (#16, #24) +- [x] Eliminate all preventable runtime panics (most already done) +- [x] URL extension recognition (.json, .ad3, .nt, etc.) +- [x] Collections / dynamic resources #17 +- [ ] Authentication #13 +- [ ] Authorization model (implemented for write, not read) +- [ ] Be able to manage the AtomicData.dev website without git (cli integration, implement required endpoints) [#6](https://github.com/joepio/atomic/issues/6) ## Install from source diff --git a/server/src/handlers/path.rs b/server/src/handlers/path.rs index b1e236ace..891aa3bef 100644 --- a/server/src/handlers/path.rs +++ b/server/src/handlers/path.rs @@ -36,8 +36,8 @@ pub async fn path( atomic_lib::storelike::PathReturn::Subject(subject) => { let resource = context .store - .get_resource_string(&subject)?; - propvals = from_hashmap_resource(&resource, &mut context.store, subject)?; + .get_resource_extended(&subject)?; + propvals = from_hashmap_resource(&resource.to_plain(), &mut context.store, subject)?; } atomic_lib::storelike::PathReturn::Atom(atom) => { propvals.push(PropVal { diff --git a/server/src/handlers/resource.rs b/server/src/handlers/resource.rs index 719495f49..ed57d22aa 100644 --- a/server/src/handlers/resource.rs +++ b/server/src/handlers/resource.rs @@ -16,44 +16,30 @@ pub async fn get_resource( ) -> BetterResult { let mut context = data.lock().unwrap(); log::info!("subject_end: {}", subject_end); - let subj_end_string = subject_end.to_string(); - let content_type = get_accept(req); + let mut subj_end_string = subject_end.as_str(); + let mut content_type = get_accept(req); // Check extensions and set datatype. Harder than it looks to get right... - // let path = Path::new(&subj_end_string); - // log::info!("path: {:?}", path); - // if content_type == ContentType::HTML { - // content_type = match path.extension() { - // Some(extension) => match extension - // .to_str() - // .ok_or("Extension cannot be parsed. Try a different URL.")? - // { - // "ad3" => ContentType::AD3, - // "json" => ContentType::JSON, - // "jsonld" => ContentType::JSONLD, - // "html" => ContentType::HTML, - // "ttl" => ContentType::TURTLE, - // _ => ContentType::HTML, - // }, - // None => ContentType::HTML, - // }; - // } + if content_type == ContentType::HTML { + if let Some((ext, path)) = try_extension(subj_end_string) { + content_type = ext; + subj_end_string = path; + } + } let subject = format!("{}{}", &context.config.local_base_url, subj_end_string); let store = &mut context.store; let mut builder = HttpResponse::Ok(); log::info!("get_resource: {} - {}", subject, content_type.to_mime()); + builder.header("Content-Type", content_type.to_mime()); match content_type { ContentType::JSON => { - builder.header("Content-Type", content_type.to_mime()); let body = store.resource_to_json(&subject, 1, false)?; Ok(builder.body(body)) } ContentType::JSONLD => { - builder.header("Content-Type", content_type.to_mime()); let body = store.resource_to_json(&subject, 1, true)?; Ok(builder.body(body)) } ContentType::HTML => { - builder.header("Content-Type", content_type.to_mime()); let mut tera_context = TeraCtx::new(); let resource = store.get_resource_string(&subject)?; let propvals = from_hashmap_resource(&resource, store, subject)?; @@ -62,16 +48,34 @@ pub async fn get_resource( Ok(builder.body(body)) } ContentType::AD3 => { - builder.header("Content-Type", content_type.to_mime()); - let body = store - .resource_to_ad3(&subject)?; + let body = store.resource_to_ad3(&subject)?; Ok(builder.body(body)) } ContentType::TURTLE | ContentType::NT => { - builder.header("Content-Type", content_type.to_mime()); - let atoms = atomic_lib::resources::resourcestring_to_atoms(&subject,store.get_resource_string(&subject)?); + let atoms = atomic_lib::resources::resourcestring_to_atoms( + &subject, + store.get_resource_string(&subject)?, + ); let body = atomic_lib::serialize::atoms_to_ntriples(atoms, store)?; Ok(builder.body(body)) } } } + +/// Finds the extension +fn try_extension(path: &str) -> Option<(ContentType, &str)> { + let items: Vec<&str> = path.split('.').collect(); + if items.len() == 2 { + let path = items[0]; + let content_type = match items[1] { + "ad3" => ContentType::AD3, + "json" => ContentType::JSON, + "jsonld" => ContentType::JSONLD, + "html" => ContentType::HTML, + "ttl" => ContentType::TURTLE, + _ => return None, + }; + return Some((content_type, path)); + } + None +} diff --git a/server/templates/resource.html b/server/templates/resource.html index 18f9c8da8..c8317d1ed 100644 --- a/server/templates/resource.html +++ b/server/templates/resource.html @@ -4,6 +4,6 @@ {% block content %}
{{ macros::propvals(vector=resource) }} - You can use HTTP content negotation to fetch this resource as JSON, AD3 or Turtle. + You can use HTTP content negotation to fetch this resource as JSON, AD3 or Turtle. You can also try putting .json, .jsonld, .ttl or .ad3 behind the URL.
{% endblock content %}