Ethstore optimizations (openethereum#6827)

ethcore/src/account_provider/mod.rs

@@ -269,7 +269,7 @@ impl AccountProvider {
 
 	/// Checks whether an account with a given address is present.
 	pub fn has_account(&self, address: Address) -> Result<bool, Error> {
-		Ok(self.accounts()?.iter().any(|&a| a == address))
+		Ok(self.sstore.account_ref(&address).is_ok() && !self.blacklisted_accounts.contains(&address))
 	}
 
 	/// Returns addresses of all accounts.

ethcore/src/miner/miner.rs

@@ -647,10 +647,6 @@ impl Miner {
 		condition: Option<TransactionCondition>,
 		transaction_queue: &mut BanningTransactionQueue,
 	) -> Vec<Result<TransactionImportResult, Error>> {
-		let accounts = self.accounts.as_ref()
-			.and_then(|provider| provider.accounts().ok())
-			.map(|accounts| accounts.into_iter().collect::<HashSet<_>>());
-
 		let best_block_header = client.best_block_header().decode();
 		let insertion_time = client.chain_info().best_block_number;
 
@@ -669,8 +665,8 @@ impl Miner {
 						Err(e)
 					},
 					Ok(transaction) => {
-						let origin = accounts.as_ref().and_then(|accounts| {
-							match accounts.contains(&transaction.sender()) {
+						let origin = self.accounts.as_ref().and_then(|accounts| {
+							match accounts.has_account(transaction.sender()).unwrap_or(false) {
 								true => Some(TransactionOrigin::Local),
 								false => None,
 							}

ethcore/src/snapshot/mod.rs

@@ -69,9 +69,6 @@ mod consensus;
 mod error;
 mod watcher;
 
-#[cfg(test)]
-mod tests;
-
 mod traits;
 
 // Try to have chunks be around 4MB (before compression)

ethcore/src/snapshot/service.rs

@@ -30,6 +30,7 @@ use blockchain::BlockChain;
 use client::{BlockChainClient, Client};
 use engines::EthEngine;
 use error::Error;
+use spec::Spec;
 use ids::BlockId;
 use service::ClientIoMessage;
 
@@ -41,6 +42,7 @@ use util_error::UtilError;
 use bytes::Bytes;
 use journaldb::Algorithm;
 use kvdb_rocksdb::{Database, DatabaseConfig};
+use devtools::RandomTempPath;
 use snappy;
 
 /// Helper for removing directories in case of error.
@@ -283,6 +285,24 @@ impl Service {
 		Ok(service)
 	}
 
+    pub fn new_test() -> Result<Self, Error> {
+	    let spec = Spec::new_null();
+	    let path = RandomTempPath::create_dir();
+	    let mut path = path.as_path().clone();
+
+	    let service_params = ServiceParams {
+	    	engine: spec.engine.clone(),
+	    	genesis_block: spec.genesis_block(),
+	    	db_config: DatabaseConfig::with_columns(::db::NUM_COLUMNS),
+	    	pruning: ::journaldb::Algorithm::Archive,
+	    	channel: IoChannel::disconnected(),
+	    	snapshot_root: path.clone(),
+	    	db_restore: Arc::new(NoopDBRestore),
+	    };
+
+	    Service::new(service_params)
+    }
+
 	// get the current snapshot dir.
 	fn snapshot_dir(&self) -> PathBuf {
 		let mut dir = self.snapshot_root.clone();
@@ -618,6 +638,13 @@ impl Drop for Service {
 	}
 }
 
+pub struct NoopDBRestore;
+impl DatabaseRestore for NoopDBRestore {
+	fn restore_db(&self, _new_db: &str) -> Result<(), Error> {
+		Ok(())
+	}
+}
+
 #[cfg(test)]
 mod tests {
 	use std::sync::Arc;
@@ -630,12 +657,6 @@ mod tests {
 	use snapshot::{ManifestData, RestorationStatus, SnapshotService};
 	use super::*;
 
-	struct NoopDBRestore;
-	impl DatabaseRestore for NoopDBRestore {
-		fn restore_db(&self, _new_db: &str) -> Result<(), Error> {
-			Ok(())
-		}
-	}
 
 	#[test]
 	fn sends_async_messages() {

ethstore/src/dir/disk.rs

@@ -122,6 +122,13 @@ impl<T> DiskDirectory<T> where T: KeyFileManager {
 		Ok(hasher.finish())
 	}
 
+	fn last_modification_date(&self) -> Result<u64, Error> {
+		use std::time::{Duration, UNIX_EPOCH};
+		let duration = fs::metadata(&self.path)?.modified()?.duration_since(UNIX_EPOCH).unwrap_or(Duration::default());
+		let timestamp = duration.as_secs() ^ (duration.subsec_nanos() as u64);
+		Ok(timestamp)
+	}
+
 	/// all accounts found in keys directory
 	fn files_content(&self) -> Result<HashMap<PathBuf, SafeAccount>, Error> {
 		// it's not done using one iterator cause
@@ -226,7 +233,7 @@ impl<T> KeyDirectory for DiskDirectory<T> where T: KeyFileManager {
 	}
 
 	fn unique_repr(&self) -> Result<u64, Error> {
-		self.files_hash()
+		self.last_modification_date()
 	}
 }
 

ethstore/src/ethstore.rs

@@ -18,6 +18,7 @@ use std::collections::{BTreeMap, HashMap};
 use std::mem;
 use std::path::PathBuf;
 use parking_lot::{Mutex, RwLock};
+use std::time::{Instant, Duration};
 
 use crypto::KEY_ITERATIONS;
 use random::Random;
@@ -28,6 +29,8 @@ use presale::PresaleWallet;
 use json::{self, Uuid, OpaqueKeyFile};
 use {import, Error, SimpleSecretStore, SecretStore, SecretVaultRef, StoreAccountRef, Derivation, OpaqueSecret};
 
+const REFRESH_TIME_SEC: u64 = 5;
+
 /// Accounts store.
 pub struct EthStore {
 	store: EthMultiStore,
@@ -245,7 +248,12 @@ pub struct EthMultiStore {
 	// order lock: cache, then vaults
 	cache: RwLock<BTreeMap<StoreAccountRef, Vec<SafeAccount>>>,
 	vaults: Mutex<HashMap<String, Box<VaultKeyDirectory>>>,
-	dir_hash: Mutex<Option<u64>>,
+	timestamp: Mutex<Timestamp>,
+}
+
+struct Timestamp {
+	dir_hash: Option<u64>,
+	last_checked: Instant,
 }
 
 impl EthMultiStore {
@@ -261,20 +269,27 @@ impl EthMultiStore {
 			vaults: Mutex::new(HashMap::new()),
 			iterations: iterations,
 			cache: Default::default(),
-			dir_hash: Default::default(),
+			timestamp: Mutex::new(Timestamp {
+				dir_hash: None,
+				last_checked: Instant::now(),
+			}),
 		};
 		store.reload_accounts()?;
 		Ok(store)
 	}
 
 	fn reload_if_changed(&self) -> Result<(), Error> {
-		let mut last_dir_hash = self.dir_hash.lock();
-		let dir_hash = Some(self.dir.unique_repr()?);
-		if *last_dir_hash == dir_hash {
-			return Ok(())
+		let mut last_timestamp = self.timestamp.lock();
+		let now = Instant::now();
+		if (now - last_timestamp.last_checked) > Duration::from_secs(REFRESH_TIME_SEC) {
+			let dir_hash = Some(self.dir.unique_repr()?);
+			last_timestamp.last_checked = now;
+			if last_timestamp.dir_hash == dir_hash {
+				return Ok(())
+			}
+			self.reload_accounts()?;
+			last_timestamp.dir_hash = dir_hash;
 		}
-		self.reload_accounts()?;
-		*last_dir_hash = dir_hash;
 		Ok(())
 	}
 
@@ -455,11 +470,11 @@ impl SimpleSecretStore for EthMultiStore {
 	}
 
 	fn account_ref(&self, address: &Address) -> Result<StoreAccountRef, Error> {
+		use std::collections::Bound;
 		self.reload_if_changed()?;
-		self.cache.read().keys()
-			.find(|r| &r.address == address)
-			.cloned()
-			.ok_or(Error::InvalidAccount)
+		let cache = self.cache.read();
+		let mut r = cache.range((Bound::Included(*address), Bound::Included(*address)));
+		r.next().ok_or(Error::InvalidAccount).map(|(k, _)| k.clone())
 	}
 
 	fn accounts(&self) -> Result<Vec<StoreAccountRef>, Error> {

ethstore/src/secret_store.rs

@@ -16,6 +16,7 @@
 
 use std::hash::{Hash, Hasher};
 use std::path::PathBuf;
+use std::cmp::Ordering;
 use ethkey::{Address, Message, Signature, Secret, Public};
 use Error;
 use json::{Uuid, OpaqueKeyFile};
@@ -32,12 +33,24 @@ pub enum SecretVaultRef {
 }
 
 /// Stored account reference
-#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
+#[derive(Debug, Clone, PartialEq, Eq, Ord)]
 pub struct StoreAccountRef {
-	/// Vault reference
-	pub vault: SecretVaultRef,
 	/// Account address
 	pub address: Address,
+	/// Vault reference
+	pub vault: SecretVaultRef,
+}
+
+impl PartialOrd for StoreAccountRef {
+	fn partial_cmp(&self, other: &StoreAccountRef) -> Option<Ordering> {
+		Some(self.address.cmp(&other.address).then_with(|| self.vault.cmp(&other.vault)))
+	}
+}
+
+impl ::std::borrow::Borrow<Address> for StoreAccountRef {
+	fn borrow(&self) -> &Address {
+		&self.address
+	}
 }
 
 /// Simple Secret Store API

hash-fetch/src/client.rs

@@ -188,7 +188,7 @@ fn random_temp_path() -> PathBuf {
 }
 
 #[cfg(test)]
-mod tests {
+pub mod tests {
 	use rustc_hex::FromHex;
 	use std::sync::{Arc, mpsc};
 	use parking_lot::Mutex;
@@ -200,7 +200,7 @@ mod tests {
 
 
 	#[derive(Clone)]
-	struct FakeFetch {
+	pub struct FakeFetch {
 		return_success: bool
 	}
 

hash-fetch/src/lib.rs

@@ -40,8 +40,11 @@ extern crate parking_lot;
 #[cfg(test)]
 extern crate ethabi;
 
-mod client;
+pub mod client;
 
 pub mod urlhint;
 
 pub use client::{HashFetch, Client, Error};
+
+#[cfg(test)]
+pub use client::tests::FakeFetch;

sync/src/api.rs

@@ -261,6 +261,24 @@ impl EthSync {
 
 		Ok(sync)
 	}
+
+    pub fn new_test() -> Result<Arc<EthSync>, NetworkError> {
+        use ethcore::client::TestBlockChainClient;
+        use test_snapshot::TestSnapshotService;
+
+        let client = Arc::new(TestBlockChainClient::new());
+
+        let params = Params {
+            config: Default::default(),
+            chain: client.clone(),
+            snapshot_service: Arc::new(TestSnapshotService::new()),
+            provider: client.clone(),
+            network_config: NetworkConfiguration::new(),
+            attached_protos: Vec::new(),
+        };
+
+        Self::new(params, None)
+    }
 }
 
 impl SyncProvider for EthSync {
@@ -308,7 +326,7 @@ impl SyncProvider for EthSync {
 	}
 }
 
-struct SyncProtocolHandler {
+pub struct SyncProtocolHandler {
 	/// Shared blockchain client.
 	chain: Arc<BlockChainClient>,
 	/// Shared snapshot service.
@@ -438,7 +456,7 @@ impl ChainNotify for EthSync {
 
 /// PIP event handler.
 /// Simply queues transactions from light client peers.
-struct TxRelay(Arc<BlockChainClient>);
+pub struct TxRelay(Arc<BlockChainClient>);
 
 impl LightHandler for TxRelay {
 	fn on_transactions(&self, ctx: &EventContext, relay: &[::ethcore::transaction::UnverifiedTransaction]) {

sync/src/lib.rs

@@ -66,14 +66,14 @@ mod transactions_stats;
 
 pub mod light_sync;
 
-#[cfg(test)]
-mod tests;
-
 mod api;
 
 pub use api::*;
 pub use chain::{SyncStatus, SyncState};
 pub use network::{is_valid_node_url, NonReservedPeerMode, NetworkError, ConnectionFilter, ConnectionDirection};
 
+mod test_snapshot;
+pub use test_snapshot::TestSnapshotService;
+
 #[cfg(test)]
 pub(crate) type Address = bigint::hash::H160;

sync/src/tests/mod.rs

@@ -16,6 +16,7 @@
 
 pub mod helpers;
 pub mod snapshot;
+pub mod TestSync;
 mod chain;
 mod consensus;
 

sync/src/tests/snapshot.rs

@@ -26,7 +26,7 @@ use ethcore::client::{EachBlockWith};
 use super::helpers::*;
 use SyncConfig;
 
-pub struct TestSnapshotService {
+struct TestSnapshotService {
 	manifest: Option<ManifestData>,
 	chunks: HashMap<H256, Bytes>,