From 28bb031d3fa75da8603e34137f86deccbe550ce1 Mon Sep 17 00:00:00 2001 From: Julia Naomi Date: Fri, 16 Apr 2021 00:00:45 -0300 Subject: [PATCH 1/2] caches info as bytes --- Cargo.lock | 10 ++++++++++ Cargo.toml | 1 + data/local_data.txt | 2 +- woori-db/Cargo.lock | 10 ++++++++++ woori-db/Cargo.toml | 1 + woori-db/data/local_data.txt | 2 +- woori-db/src/actors/history.rs | 3 +-- woori-db/src/controllers/clauses.rs | 3 ++- woori-db/src/controllers/query.rs | 10 ++++++---- woori-db/src/controllers/tx.rs | 27 ++++++++++++++++++--------- woori-db/src/core/query.rs | 3 ++- woori-db/src/io/read.rs | 2 +- woori-db/src/repository/local.rs | 3 +-- 13 files changed, 55 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f5ed7bd..f0593ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -430,6 +430,15 @@ dependencies = [ "getrandom 0.1.16", ] +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bitflags" version = "1.2.1" @@ -1715,6 +1724,7 @@ dependencies = [ "actix-web", "actix-web-httpauth", "bcrypt", + "bincode", "bytes 1.0.1", "chrono", "criterion", diff --git a/Cargo.toml b/Cargo.toml index 48de357..76df355 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,6 +42,7 @@ zstd = "0.6.0+zstd.1.4.8" bcrypt = "0.8" actix-web-httpauth = "0.5.0" serde_json = { version = "1.0", optional = true } +bincode = "1.3" [dev-dependencies] bytes = "1.0.1" diff --git a/data/local_data.txt b/data/local_data.txt index 8cdbb21..d21ba99 100644 --- a/data/local_data.txt +++ b/data/local_data.txt @@ -1 +1 @@ -{"encrypt_ent": {},"encrypt_ent2": {},"hello": {"50e68bc1-0c3b-4ffc-93be-46e57f59b415": ((file_name: "2021_02_10.log",offset: 447,bytes_length: 153,), {},),},"oh_yeah": {"27367bd0-1966-4005-a8b5-5e323e1c3524": ((file_name: "2021_02_10.log",offset: 180,bytes_length: 247,), {}),},} \ No newline at end of file +{"encrypt_ent": {},"encrypt_ent2": {},"hello": {"50e68bc1-0c3b-4ffc-93be-46e57f59b415": ((file_name: "2021_02_10.log",offset: 447,bytes_length: 153,),[0, 0, 0, 0, 0, 0, 0, 0,],),},"oh_yeah": {"27367bd0-1966-4005-a8b5-5e323e1c3524": ((file_name: "2021_02_10.log",offset: 180,bytes_length: 247,), [0, 0, 0, 0, 0, 0, 0, 0,],),},} \ No newline at end of file diff --git a/woori-db/Cargo.lock b/woori-db/Cargo.lock index ddaecb3..9d3ba13 100644 --- a/woori-db/Cargo.lock +++ b/woori-db/Cargo.lock @@ -430,6 +430,15 @@ dependencies = [ "getrandom 0.1.16", ] +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bitflags" version = "1.2.1" @@ -2150,6 +2159,7 @@ dependencies = [ "actix-web", "actix-web-httpauth", "bcrypt", + "bincode", "bytes 1.0.1", "chrono", "cron", diff --git a/woori-db/Cargo.toml b/woori-db/Cargo.toml index 3858a1a..959390b 100644 --- a/woori-db/Cargo.toml +++ b/woori-db/Cargo.toml @@ -38,6 +38,7 @@ glob = "0.3.0" zstd = "0.6.0+zstd.1.4.8" bcrypt = "0.8" actix-web-httpauth = "0.5.0" +bincode = "1.3" [dev-dependencies] bytes = "1.0.1" \ No newline at end of file diff --git a/woori-db/data/local_data.txt b/woori-db/data/local_data.txt index 8cdbb21..e61574a 100644 --- a/woori-db/data/local_data.txt +++ b/woori-db/data/local_data.txt @@ -1 +1 @@ -{"encrypt_ent": {},"encrypt_ent2": {},"hello": {"50e68bc1-0c3b-4ffc-93be-46e57f59b415": ((file_name: "2021_02_10.log",offset: 447,bytes_length: 153,), {},),},"oh_yeah": {"27367bd0-1966-4005-a8b5-5e323e1c3524": ((file_name: "2021_02_10.log",offset: 180,bytes_length: 247,), {}),},} \ No newline at end of file +{"encrypt_ent": {},"encrypt_ent2": {},"hello": {"50e68bc1-0c3b-4ffc-93be-46e57f59b415": ((file_name: "2021_02_10.log",offset: 447,bytes_length: 153,), [0, 0, 0, 0, 0, 0, 0, 0,],),},"oh_yeah": {"27367bd0-1966-4005-a8b5-5e323e1c3524": ((file_name: "2021_02_10.log",offset: 180,bytes_length: 247,), [0, 0, 0, 0, 0, 0, 0, 0,],),},} \ No newline at end of file diff --git a/woori-db/src/actors/history.rs b/woori-db/src/actors/history.rs index f5433d4..42e9e10 100644 --- a/woori-db/src/actors/history.rs +++ b/woori-db/src/actors/history.rs @@ -84,8 +84,7 @@ fn get_previous_registry(fractions: &[&str]) -> Result, Err .to_owned(); let state = &state[..(state.len() - 1)]; - let resp: Result<(DataRegister, HashMap), Error> = match ron::de::from_str(state) - { + let resp: Result<(DataRegister, Vec), Error> = match ron::de::from_str(state) { Ok(x) => Ok(x), Err(_) => Err(Error::FailedToParseRegistry), }; diff --git a/woori-db/src/controllers/clauses.rs b/woori-db/src/controllers/clauses.rs index 1cf1a44..a13abde 100644 --- a/woori-db/src/controllers/clauses.rs +++ b/woori-db/src/controllers/clauses.rs @@ -229,11 +229,12 @@ fn or_clauses( } async fn generate_state( - registries: &BTreeMap)>, + registries: &BTreeMap)>, args_to_select: ToSelect, ) -> Result>, Error> { let mut states: BTreeMap> = BTreeMap::new(); for (uuid, (_, state)) in registries { + let state: HashMap = bincode::deserialize(&state).unwrap(); let state = state .into_par_iter() .filter(|(_, v)| !v.is_hash()) diff --git a/woori-db/src/controllers/query.rs b/woori-db/src/controllers/query.rs index 951ce05..f7df117 100644 --- a/woori-db/src/controllers/query.rs +++ b/woori-db/src/controllers/query.rs @@ -295,7 +295,7 @@ pub async fn select_all_with_id( registry }; - let state = registry.1; + let state: HashMap = bincode::deserialize(®istry.1).unwrap(); let filterd_state = filter_keys_and_hash(state, None); Ok(filterd_state.into()) } @@ -327,7 +327,7 @@ pub async fn select_all_with_ids( .filter(|(_id, reg)| reg.is_some()) }) .map(|(uuid, reg)| (uuid, reg.map(ToOwned::to_owned))) - .collect::)>)>>() + .collect::)>)>>() } else { return Err(Error::EntityNotCreated(entity)); }; @@ -337,6 +337,7 @@ pub async fn select_all_with_ids( let mut states: BTreeMap>> = BTreeMap::new(); for (uuid, registry) in registries.into_iter().skip(offset).take(limit) { if let Some((_, state)) = registry { + let state: HashMap = bincode::deserialize(&state).unwrap(); let filtered = filter_keys_and_hash(state, None); states.insert(uuid, Some(filtered)); } else { @@ -377,7 +378,7 @@ pub async fn select_keys_with_id( registry }; - let state = registry.1; + let state: HashMap = bincode::deserialize(®istry.1).unwrap(); let filtered = filter_keys_and_hash(state, Some(keys)); Ok(filtered.into()) } @@ -411,7 +412,7 @@ pub async fn select_keys_with_ids( .filter(|(_id, reg)| reg.is_some()) }) .map(|(uuid, reg)| (uuid, reg.map(ToOwned::to_owned))) - .collect::)>)>>() + .collect::)>)>>() } else { return Err(Error::EntityNotCreated(entity)); }; @@ -421,6 +422,7 @@ pub async fn select_keys_with_ids( let mut states: BTreeMap>> = BTreeMap::new(); for (uuid, registry) in registries.into_iter().skip(offset).take(limit) { if let Some((_, state)) = registry { + let state: HashMap = bincode::deserialize(&state).unwrap(); let filtered = filter_keys_and_hash(state, Some(keys.clone())); states.insert(uuid, Some(filtered)); } else { diff --git a/woori-db/src/controllers/tx.rs b/woori-db/src/controllers/tx.rs index c36b010..8bbd37b 100644 --- a/woori-db/src/controllers/tx.rs +++ b/woori-db/src/controllers/tx.rs @@ -344,7 +344,8 @@ pub async fn insert_controller( return Err(Error::LockData); }; if let Some(map) = local_data.get_mut(&args.entity) { - map.insert(content_value.1, (local_data_register, encrypted_content)); + let encoded: Vec = bincode::serialize(&encrypted_content).unwrap(); + map.insert(content_value.1, (local_data_register, encoded)); } local_data.clone() }; @@ -420,7 +421,8 @@ pub async fn update_set_controller( previous_entry.clone() }; - let mut previous_state = previous_entry.1.clone(); + let mut previous_state: HashMap = + bincode::deserialize(&previous_entry.1.clone()).unwrap(); let encrypted_content_clone = encrypted_content.clone(); encrypted_content.into_iter().for_each(|(k, v)| { let local_state = previous_state.entry(k).or_insert_with(|| v.clone()); @@ -461,7 +463,8 @@ pub async fn update_set_controller( }; if let Some(map) = local_data.get_mut(&args.entity) { if let Some(reg) = map.get_mut(&args.id) { - *reg = (local_data_register, encrypted_content_clone); + let encoded: Vec = bincode::serialize(&encrypted_content_clone).unwrap(); + *reg = (local_data_register, encoded); } } local_data.clone() @@ -541,7 +544,8 @@ pub async fn update_content_controller( previous_entry.clone() }; - let mut previous_state = previous_entry.1.clone(); + let mut previous_state: HashMap = + bincode::deserialize(&previous_entry.1.clone()).unwrap(); content .into_iter() @@ -578,7 +582,8 @@ pub async fn update_content_controller( }; if let Some(map) = local_data.get_mut(&args.entity) { if let Some(reg) = map.get_mut(&args.id) { - *reg = (local_data_register, previous_state); + let encoded: Vec = bincode::serialize(&previous_state).unwrap(); + *reg = (local_data_register, encoded); } } local_data.clone() @@ -679,7 +684,8 @@ pub async fn delete_controller( }; if let Some(map) = local_data.get_mut(&entity) { if let Some(reg) = map.get_mut(&uuid) { - *reg = (local_data_register, state_to_be.0); + let encoded: Vec = bincode::serialize(&state_to_be.0).unwrap(); + *reg = (local_data_register, encoded); } } local_data.clone() @@ -723,12 +729,14 @@ pub async fn match_update_set_controller( previous_entry.clone() }; - let mut previous_state = previous_entry.1.clone(); + let previous_entry: HashMap = + bincode::deserialize(&previous_entry.1.clone()).unwrap(); + let mut previous_state = previous_entry.clone(); actor .send(MatchUpdate { conditions: args.conditions, - previous_state: previous_entry.1.clone(), + previous_state: previous_entry.clone(), }) .await??; @@ -793,7 +801,8 @@ pub async fn match_update_set_controller( }; if let Some(map) = local_data.get_mut(&args.entity) { if let Some(reg) = map.get_mut(&args.id) { - *reg = (local_data_register, encrypted_content); + let encoded: Vec = bincode::serialize(&encrypted_content).unwrap(); + *reg = (local_data_register, encoded); } } local_data.clone() diff --git a/woori-db/src/core/query.rs b/woori-db/src/core/query.rs index fa08c98..5906f17 100644 --- a/woori-db/src/core/query.rs +++ b/woori-db/src/core/query.rs @@ -25,13 +25,14 @@ pub(crate) fn filter_keys_and_hash( } pub fn registries_to_states( - registries: BTreeMap)>, + registries: BTreeMap)>, keys: Option>, offset: usize, limit: usize, ) -> BTreeMap> { let mut states: BTreeMap> = BTreeMap::new(); for (uuid, (_, state)) in registries.into_iter().skip(offset).take(limit) { + let state: HashMap = bincode::deserialize(&state).unwrap(); let filtered = filter_keys_and_hash(state, keys.clone()); states.insert(uuid, filtered); } diff --git a/woori-db/src/io/read.rs b/woori-db/src/io/read.rs index 8c848ae..e3fc03a 100644 --- a/woori-db/src/io/read.rs +++ b/woori-db/src/io/read.rs @@ -271,7 +271,7 @@ mod test { assert!(local_data.is_ok()); assert_eq!( format!("{:?}", local_data), - "Ok({\"encrypt_ent\": {}, \"encrypt_ent2\": {}, \"hello\": {50e68bc1-0c3b-4ffc-93be-46e57f59b415: (DataRegister { file_name: \"2021_02_10.log\", offset: 447, bytes_length: 153 }, {})}, \"oh_yeah\": {27367bd0-1966-4005-a8b5-5e323e1c3524: (DataRegister { file_name: \"2021_02_10.log\", offset: 180, bytes_length: 247 }, {})}})" + "Ok({\"encrypt_ent\": {}, \"encrypt_ent2\": {}, \"hello\": {50e68bc1-0c3b-4ffc-93be-46e57f59b415: (DataRegister { file_name: \"2021_02_10.log\", offset: 447, bytes_length: 153 }, [0, 0, 0, 0, 0, 0, 0, 0])}, \"oh_yeah\": {27367bd0-1966-4005-a8b5-5e323e1c3524: (DataRegister { file_name: \"2021_02_10.log\", offset: 180, bytes_length: 247 }, [0, 0, 0, 0, 0, 0, 0, 0])}})" ); } diff --git a/woori-db/src/repository/local.rs b/woori-db/src/repository/local.rs index ec34443..91ec7fe 100644 --- a/woori-db/src/repository/local.rs +++ b/woori-db/src/repository/local.rs @@ -3,11 +3,10 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use uuid::Uuid; -use wql::Types; use crate::{auth::schemas::Role, model::DataRegister}; -pub type StateInfo = (DataRegister, HashMap); +pub type StateInfo = (DataRegister, Vec); pub type LocalContext = BTreeMap>; pub type UniquenessContext = BTreeMap>>; pub type EncryptContext = BTreeMap>; From 943307d1768514d997521b7ebadc0f3b95ca1894 Mon Sep 17 00:00:00 2001 From: Julia Naomi Date: Fri, 16 Apr 2021 00:11:42 -0300 Subject: [PATCH 2/2] update benchmarks --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index ebdfd35..a216fec 100644 --- a/README.md +++ b/README.md @@ -82,17 +82,17 @@ PORT=1438 > 2,3 GHz Intel Core i9 8-Core 32 GB 2667 MHz DDR4 - `create_entity` time: [4.7008 ms 4.7367 ms 4.7725 ms] -- `insert_entity` time: [9.5938 ms 9.8263 ms 10.057 ms] -- `update_set_entity` time: [11.899 ms 11.984 ms 12.083 ms] -- `update_content_entity` time: [12.048 ms 12.328 ms 12.719 ms] +- `insert_entity` time: [12.647 ms 12.977 ms 13.308 ms] +- `update_set_entity` time: [14.896 ms 15.087 ms 15.309 ms] +- `update_content_entity` time: [14.871 ms 15.070 ms 15.307 ms] - `delete_entity` time: [5.3966 ms 5.4423 ms 5.4908 ms] - Filtered 400s - `evict_entity_id` time: [15.534 ms 15.623 ms 15.721 ms] - Filtered 400s - `evict_entity` time: [12.318 ms 12.416 ms 12.540 ms] - Filtered 400s -- `select_all` 20 entities time: [5.8810 ms 5.9318 ms 5.9889 ms] -- `select_all` 10 entities time: [5.4012 ms 5.4432 ms 5.4860 ms] -- `select_all` 1 entity time: [4.8442 ms 4.8786 ms 4.9140 ms] -- `history_10_registries_for_entity` time: [5.7265 ms 5.7868 ms 5.8673 ms] -- `history_20_registries_for_entity` time: [6.1166 ms 6.1943 ms 6.2987 ms] +- `select_all` 20 entities time: [6.3402 ms 6.4743 ms 6.6356 ms] +- `select_all` 10 entities time: [5.8318 ms 5.9682 ms 6.1340 ms] +- `select_all` 1 entity time: [5.1030 ms 5.1379 ms 5.1738 ms] +- `history_10_registries_for_entity` time: [5.4936 ms 5.5328 ms 5.5725 ms] +- `history_20_registries_for_entity` time: [6.0676 ms 6.1049 ms 6.1429 ms] ### WQL > 2,3 GHz Intel Core i9 8-Core 32 GB 2667 MHz DDR4