Skip to content
This repository has been archived by the owner on Nov 6, 2020. It is now read-only.

Commit

Permalink
LRU cache for dapps (#2006)
Browse files Browse the repository at this point in the history
Conflicts:
	dapps/Cargo.toml
	dapps/src/lib.rs
  • Loading branch information
tomusdrw authored and arkpar committed Aug 30, 2016
1 parent 6da60af commit 6f321d9
Show file tree
Hide file tree
Showing 10 changed files with 214 additions and 39 deletions.
7 changes: 7 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions dapps/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ serde_json = "0.7.0"
serde_macros = { version = "0.7.0", optional = true }
zip = { version = "0.1", default-features = false }
ethabi = "0.2.1"
linked-hash-map = "0.3"
ethcore-rpc = { path = "../rpc" }
ethcore-util = { path = "../util" }
parity-dapps = { git = "https://github.com/ethcore/parity-ui.git", version = "1.4" }
Expand Down
128 changes: 128 additions & 0 deletions dapps/src/apps/cache.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
// Copyright 2015, 2016 Ethcore (UK) Ltd.
// This file is part of Parity.

// Parity is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.

// Parity is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with Parity. If not, see <http://www.gnu.org/licenses/>.

//! Fetchable Dapps support.
use std::fs;
use std::sync::{Arc};
use std::sync::atomic::{AtomicBool, Ordering};

use linked_hash_map::LinkedHashMap;
use page::LocalPageEndpoint;

pub enum ContentStatus {
Fetching(Arc<AtomicBool>),
Ready(LocalPageEndpoint),
}

#[derive(Default)]
pub struct ContentCache {
cache: LinkedHashMap<String, ContentStatus>,
}

impl ContentCache {
pub fn insert(&mut self, content_id: String, status: ContentStatus) -> Option<ContentStatus> {
self.cache.insert(content_id, status)
}

pub fn remove(&mut self, content_id: &str) -> Option<ContentStatus> {
self.cache.remove(content_id)
}

pub fn get(&mut self, content_id: &str) -> Option<&mut ContentStatus> {
self.cache.get_refresh(content_id)
}

pub fn clear_garbage(&mut self, expected_size: usize) -> Vec<(String, ContentStatus)> {
let mut len = self.cache.len();

if len <= expected_size {
return Vec::new();
}

let mut removed = Vec::with_capacity(len - expected_size);
while len > expected_size {
let entry = self.cache.pop_front().unwrap();
match entry.1 {
ContentStatus::Fetching(ref abort) => {
trace!(target: "dapps", "Aborting {} because of limit.", entry.0);
// Mark as aborted
abort.store(true, Ordering::Relaxed);
},
ContentStatus::Ready(ref endpoint) => {
trace!(target: "dapps", "Removing {} because of limit.", entry.0);
// Remove path
let res = fs::remove_dir_all(&endpoint.path());
if let Err(e) = res {
warn!(target: "dapps", "Unable to remove dapp: {:?}", e);
}
}
}

removed.push(entry);
len -= 1;
}
removed
}

#[cfg(test)]
pub fn len(&self) -> usize {
self.cache.len()
}
}

#[cfg(test)]
mod tests {
use super::*;

fn only_keys(data: Vec<(String, ContentStatus)>) -> Vec<String> {
data.into_iter().map(|x| x.0).collect()
}

#[test]
fn should_remove_least_recently_used() {
// given
let mut cache = ContentCache::default();
cache.insert("a".into(), ContentStatus::Fetching(Default::default()));
cache.insert("b".into(), ContentStatus::Fetching(Default::default()));
cache.insert("c".into(), ContentStatus::Fetching(Default::default()));

// when
let res = cache.clear_garbage(2);

// then
assert_eq!(cache.len(), 2);
assert_eq!(only_keys(res), vec!["a"]);
}

#[test]
fn should_update_lru_if_accessed() {
// given
let mut cache = ContentCache::default();
cache.insert("a".into(), ContentStatus::Fetching(Default::default()));
cache.insert("b".into(), ContentStatus::Fetching(Default::default()));
cache.insert("c".into(), ContentStatus::Fetching(Default::default()));

// when
cache.get("a");
let res = cache.clear_garbage(2);

// then
assert_eq!(cache.len(), 2);
assert_eq!(only_keys(res), vec!["b"]);
}

}
51 changes: 27 additions & 24 deletions dapps/src/apps/fetcher.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ use std::{fs, env};
use std::io::{self, Read, Write};
use std::path::PathBuf;
use std::sync::Arc;
use std::collections::HashMap;
use std::sync::atomic::{AtomicBool};
use rustc_serialize::hex::FromHex;

use hyper::Control;
Expand All @@ -33,20 +33,18 @@ use random_filename;
use util::{Mutex, H256};
use util::sha3::sha3;
use page::LocalPageEndpoint;
use handlers::{ContentHandler, AppFetcherHandler, DappHandler};
use handlers::{ContentHandler, ContentFetcherHandler, ContentValidator};
use endpoint::{Endpoint, EndpointPath, Handler};
use apps::cache::{ContentCache, ContentStatus};
use apps::manifest::{MANIFEST_FILENAME, deserialize_manifest, serialize_manifest, Manifest};
use apps::urlhint::{URLHintContract, URLHint};

enum AppStatus {
Fetching,
Ready(LocalPageEndpoint),
}
const MAX_CACHED_DAPPS: usize = 10;

pub struct AppFetcher<R: URLHint = URLHintContract> {
dapps_path: PathBuf,
resolver: R,
dapps: Arc<Mutex<HashMap<String, AppStatus>>>,
dapps: Arc<Mutex<ContentCache>>,
}

impl<R: URLHint> Drop for AppFetcher<R> {
Expand All @@ -65,17 +63,17 @@ impl<R: URLHint> AppFetcher<R> {
AppFetcher {
dapps_path: dapps_path,
resolver: resolver,
dapps: Arc::new(Mutex::new(HashMap::new())),
dapps: Arc::new(Mutex::new(ContentCache::default())),
}
}

#[cfg(test)]
fn set_status(&self, app_id: &str, status: AppStatus) {
fn set_status(&self, app_id: &str, status: ContentStatus) {
self.dapps.lock().insert(app_id.to_owned(), status);
}

pub fn contains(&self, app_id: &str) -> bool {
let dapps = self.dapps.lock();
let mut dapps = self.dapps.lock();
match dapps.get(app_id) {
// Check if we already have the app
Some(_) => true,
Expand All @@ -95,11 +93,11 @@ impl<R: URLHint> AppFetcher<R> {
let status = dapps.get(&app_id);
match status {
// Just server dapp
Some(&AppStatus::Ready(ref endpoint)) => {
Some(&mut ContentStatus::Ready(ref endpoint)) => {
(None, endpoint.to_handler(path))
},
// App is already being fetched
Some(&AppStatus::Fetching) => {
Some(&mut ContentStatus::Fetching(_)) => {
(None, Box::new(ContentHandler::html(
StatusCode::ServiceUnavailable,
format!(
Expand All @@ -111,11 +109,13 @@ impl<R: URLHint> AppFetcher<R> {
},
// We need to start fetching app
None => {
// TODO [todr] Keep only last N dapps available!
let app_hex = app_id.from_hex().expect("to_handler is called only when `contains` returns true.");
let app = self.resolver.resolve(app_hex).expect("to_handler is called only when `contains` returns true.");
(Some(AppStatus::Fetching), Box::new(AppFetcherHandler::new(
let abort = Arc::new(AtomicBool::new(false));

(Some(ContentStatus::Fetching(abort.clone())), Box::new(ContentFetcherHandler::new(
app,
abort,
control,
path.using_dapps_domains,
DappInstaller {
Expand All @@ -129,6 +129,7 @@ impl<R: URLHint> AppFetcher<R> {
};

if let Some(status) = new_status {
dapps.clear_garbage(MAX_CACHED_DAPPS);
dapps.insert(app_id, status);
}

Expand Down Expand Up @@ -161,7 +162,7 @@ impl From<zip::result::ZipError> for ValidationError {
struct DappInstaller {
dapp_id: String,
dapps_path: PathBuf,
dapps: Arc<Mutex<HashMap<String, AppStatus>>>,
dapps: Arc<Mutex<ContentCache>>,
}

impl DappInstaller {
Expand Down Expand Up @@ -196,7 +197,7 @@ impl DappInstaller {
}
}

impl DappHandler for DappInstaller {
impl ContentValidator for DappInstaller {
type Error = ValidationError;

fn validate_and_install(&self, app_path: PathBuf) -> Result<Manifest, ValidationError> {
Expand Down Expand Up @@ -262,7 +263,7 @@ impl DappHandler for DappInstaller {
Some(manifest) => {
let path = self.dapp_target_path(manifest);
let app = LocalPageEndpoint::new(path, manifest.clone().into());
dapps.insert(self.dapp_id.clone(), AppStatus::Ready(app));
dapps.insert(self.dapp_id.clone(), ContentStatus::Ready(app));
},
// In case of error
None => {
Expand All @@ -274,12 +275,13 @@ impl DappHandler for DappInstaller {

#[cfg(test)]
mod tests {
use std::path::PathBuf;
use super::{AppFetcher, AppStatus};
use apps::urlhint::{GithubApp, URLHint};
use std::env;
use util::Bytes;
use endpoint::EndpointInfo;
use page::LocalPageEndpoint;
use util::Bytes;
use apps::cache::ContentStatus;
use apps::urlhint::{GithubApp, URLHint};
use super::AppFetcher;

struct FakeResolver;
impl URLHint for FakeResolver {
Expand All @@ -291,8 +293,9 @@ mod tests {
#[test]
fn should_true_if_contains_the_app() {
// given
let path = env::temp_dir();
let fetcher = AppFetcher::new(FakeResolver);
let handler = LocalPageEndpoint::new(PathBuf::from("/tmp/test"), EndpointInfo {
let handler = LocalPageEndpoint::new(path, EndpointInfo {
name: "fake".into(),
description: "".into(),
version: "".into(),
Expand All @@ -301,8 +304,8 @@ mod tests {
});

// when
fetcher.set_status("test", AppStatus::Ready(handler));
fetcher.set_status("test2", AppStatus::Fetching);
fetcher.set_status("test", ContentStatus::Ready(handler));
fetcher.set_status("test2", ContentStatus::Fetching(Default::default()));

// then
assert_eq!(fetcher.contains("test"), true);
Expand Down
1 change: 1 addition & 0 deletions dapps/src/apps/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ use page::PageEndpoint;
use proxypac::ProxyPac;
use parity_dapps::WebApp;

mod cache;
mod fs;
pub mod urlhint;
pub mod fetcher;
Expand Down
Loading

0 comments on commit 6f321d9

Please sign in to comment.