From 4cb610d9aec0d6792ea465e5c041607993290542 Mon Sep 17 00:00:00 2001 From: debris Date: Thu, 17 Aug 2017 16:05:26 +0200 Subject: [PATCH 1/4] Itertools are no longer reexported from util, optimized triedb iter --- Cargo.lock | 3 ++- dapps/Cargo.toml | 1 + dapps/src/handlers/mod.rs | 3 +-- dapps/src/lib.rs | 1 + ethcore/src/blockchain/blockchain.rs | 1 + ethcore/src/client/client.rs | 3 ++- ethcore/src/client/test_client.rs | 1 + ethcore/src/client/traits.rs | 3 ++- ethcore/src/pod_account.rs | 1 + ethcore/src/pod_state.rs | 1 + rpc/Cargo.toml | 1 + rpc/src/authcodes.rs | 3 ++- rpc/src/lib.rs | 1 + util/Cargo.toml | 1 - util/benches/trie.rs | 4 ++-- util/src/journaldb/earlymergedb.rs | 5 +++-- util/src/lib.rs | 2 -- util/src/trie/triedb.rs | 14 ++++++++++++-- 18 files changed, 34 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c930b8ff988..20f4b38c538 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -773,7 +773,6 @@ dependencies = [ "ethcore-devtools 1.8.0", "ethcore-logger 1.8.0", "heapsize 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "itertools 0.5.9 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", "lru-cache 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1860,6 +1859,7 @@ dependencies = [ "fetch 0.1.0", "futures 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", "futures-cpupool 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "itertools 0.5.9 (registry+https://github.com/rust-lang/crates.io-index)", "jsonrpc-core 7.0.0 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-1.7)", "jsonrpc-http-server 7.0.0 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-1.7)", "linked-hash-map 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1971,6 +1971,7 @@ dependencies = [ "fetch 0.1.0", "futures 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", "futures-cpupool 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "itertools 0.5.9 (registry+https://github.com/rust-lang/crates.io-index)", "jsonrpc-core 7.0.0 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-1.7)", "jsonrpc-http-server 7.0.0 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-1.7)", "jsonrpc-ipc-server 7.0.0 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-1.7)", diff --git a/dapps/Cargo.toml b/dapps/Cargo.toml index 75e15e39627..143bbe30fa7 100644 --- a/dapps/Cargo.toml +++ b/dapps/Cargo.toml @@ -27,6 +27,7 @@ time = "0.1.35" unicase = "1.3" url = "1.0" zip = { version = "0.1", default-features = false } +itertools = "0.5" jsonrpc-core = { git = "https://github.com/paritytech/jsonrpc.git", branch = "parity-1.7" } jsonrpc-http-server = { git = "https://github.com/paritytech/jsonrpc.git", branch = "parity-1.7" } diff --git a/dapps/src/handlers/mod.rs b/dapps/src/handlers/mod.rs index 65c2cbe272a..a8beabe846c 100644 --- a/dapps/src/handlers/mod.rs +++ b/dapps/src/handlers/mod.rs @@ -31,8 +31,7 @@ pub use self::redirect::Redirection; pub use self::streaming::StreamingHandler; use std::iter; -use util::Itertools; - +use itertools::Itertools; use url::Url; use hyper::{server, header, net, uri}; use {apps, address, Embeddable}; diff --git a/dapps/src/lib.rs b/dapps/src/lib.rs index 3eda70ab539..135f0bb36a3 100644 --- a/dapps/src/lib.rs +++ b/dapps/src/lib.rs @@ -22,6 +22,7 @@ extern crate base32; extern crate futures; extern crate futures_cpupool; +extern crate itertools; extern crate linked_hash_map; extern crate mime_guess; extern crate ntp; diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index bd0defa4758..001df861c66 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -19,6 +19,7 @@ use std::collections::{HashMap, HashSet}; use std::sync::Arc; use std::mem; +use itertools::Itertools; use bloomchain as bc; use util::*; use rlp::*; diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index fdd201e689f..18e63a0619d 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -20,9 +20,10 @@ use std::sync::{Arc, Weak}; use std::sync::atomic::{AtomicUsize, AtomicBool, Ordering as AtomicOrdering}; use std::time::{Instant}; use time::precise_time_ns; +use itertools::Itertools; // util -use util::{Bytes, PerfTimer, Itertools, Mutex, RwLock, MutexGuard, Hashable}; +use util::{Bytes, PerfTimer, Mutex, RwLock, MutexGuard, Hashable}; use util::{journaldb, DBValue, TrieFactory, Trie}; use util::{U256, H256, Address, H2048}; use util::trie::TrieSpec; diff --git a/ethcore/src/client/test_client.rs b/ethcore/src/client/test_client.rs index 2a205868d29..d774c7a680d 100644 --- a/ethcore/src/client/test_client.rs +++ b/ethcore/src/client/test_client.rs @@ -20,6 +20,7 @@ use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrder}; use std::sync::Arc; use std::collections::{HashMap, BTreeMap}; use std::mem; +use itertools::Itertools; use rustc_hex::FromHex; use util::*; use rlp::*; diff --git a/ethcore/src/client/traits.rs b/ethcore/src/client/traits.rs index 8e1bd8b18ba..2ae30317768 100644 --- a/ethcore/src/client/traits.rs +++ b/ethcore/src/client/traits.rs @@ -15,6 +15,7 @@ // along with Parity. If not, see . use std::collections::BTreeMap; +use itertools::Itertools; use block::{OpenBlock, SealedBlock, ClosedBlock}; use blockchain::TreeRoute; @@ -33,7 +34,7 @@ use trace::LocalizedTrace; use transaction::{LocalizedTransaction, PendingTransaction, SignedTransaction}; use verification::queue::QueueInfo as BlockQueueInfo; -use util::{U256, Address, H256, H2048, Bytes, Itertools}; +use util::{U256, Address, H256, H2048, Bytes}; use util::hashdb::DBValue; use types::ids::*; diff --git a/ethcore/src/pod_account.rs b/ethcore/src/pod_account.rs index bbd2fffa570..4204b591df6 100644 --- a/ethcore/src/pod_account.rs +++ b/ethcore/src/pod_account.rs @@ -16,6 +16,7 @@ use std::fmt; use std::collections::BTreeMap; +use itertools::Itertools; use util::*; use state::Account; use ethjson; diff --git a/ethcore/src/pod_state.rs b/ethcore/src/pod_state.rs index 3e8d0bbfe9d..5a0265dc18e 100644 --- a/ethcore/src/pod_state.rs +++ b/ethcore/src/pod_state.rs @@ -18,6 +18,7 @@ use std::fmt; use std::collections::BTreeMap; +use itertools::Itertools; use util::*; use pod_account::{self, PodAccount}; use types::state_diff::StateDiff; diff --git a/rpc/Cargo.toml b/rpc/Cargo.toml index 2619c754fc8..73230c24e31 100644 --- a/rpc/Cargo.toml +++ b/rpc/Cargo.toml @@ -24,6 +24,7 @@ serde_json = "1.0" time = "0.1" tokio-timer = "0.1" transient-hashmap = "0.4" +itertools = "0.5" jsonrpc-core = { git = "https://github.com/paritytech/jsonrpc.git", branch = "parity-1.7" } jsonrpc-http-server = { git = "https://github.com/paritytech/jsonrpc.git", branch = "parity-1.7" } diff --git a/rpc/src/authcodes.rs b/rpc/src/authcodes.rs index 57de437abcf..4427eda7898 100644 --- a/rpc/src/authcodes.rs +++ b/rpc/src/authcodes.rs @@ -18,9 +18,10 @@ use std::io::{self, Read, Write}; use std::path::Path; use std::{fs, time, mem}; +use itertools::Itertools; use rand::Rng; use rand::os::OsRng; -use util::{H256, Hashable, Itertools}; +use util::{H256, Hashable}; /// Providing current time in seconds pub trait TimeProvider { diff --git a/rpc/src/lib.rs b/rpc/src/lib.rs index fec19f78afa..e75d187de78 100644 --- a/rpc/src/lib.rs +++ b/rpc/src/lib.rs @@ -24,6 +24,7 @@ extern crate cid; extern crate crypto as rust_crypto; extern crate futures; extern crate futures_cpupool; +extern crate itertools; extern crate multihash; extern crate order_stat; extern crate rand; diff --git a/util/Cargo.toml b/util/Cargo.toml index 5306e3c5715..0e485431ec3 100644 --- a/util/Cargo.toml +++ b/util/Cargo.toml @@ -19,7 +19,6 @@ rust-crypto = "0.2.34" elastic-array = "0.9" rlp = { path = "rlp" } heapsize = "0.4" -itertools = "0.5" sha3 = { path = "sha3" } clippy = { version = "0.0.103", optional = true} ethcore-devtools = { path = "../devtools" } diff --git a/util/benches/trie.rs b/util/benches/trie.rs index d526b97ac09..e02077ad40c 100644 --- a/util/benches/trie.rs +++ b/util/benches/trie.rs @@ -45,14 +45,14 @@ fn random_bytes(min_count: usize, diff_count: usize, seed: &mut H256) -> Vec assert!(min_count + diff_count <= 32); *seed = seed.sha3(); let r = min_count + (seed[31] as usize % (diff_count + 1)); - seed[0..r].into_vec() + seed[0..r].to_vec() } fn random_value(seed: &mut H256) -> Bytes { *seed = seed.sha3(); match seed[0] % 2 { 1 => vec![seed[31];1], - _ => seed.into_vec(), + _ => seed.to_vec(), } } diff --git a/util/src/journaldb/earlymergedb.rs b/util/src/journaldb/earlymergedb.rs index 5f409d3277b..b7b57d53703 100644 --- a/util/src/journaldb/earlymergedb.rs +++ b/util/src/journaldb/earlymergedb.rs @@ -21,7 +21,6 @@ use std::collections::HashMap; use std::sync::Arc; use parking_lot::RwLock; use heapsize::HeapSizeOf; -use itertools::Itertools; use rlp::*; use hashdb::*; use memorydb::*; @@ -432,7 +431,9 @@ impl JournalDB for EarlyMergeDB { // - we write the key into our journal for this block; r.begin_list(inserts.len()); - inserts.iter().foreach(|&(k, _)| {r.append(&k);}); + for &(k, _) in &inserts { + r.append(&k); + } r.append_list(&removes); Self::insert_keys(&inserts, &*self.backing, self.column, &mut refs, batch, trace); diff --git a/util/src/lib.rs b/util/src/lib.rs index 544f60d39a5..46730fe9cec 100644 --- a/util/src/lib.rs +++ b/util/src/lib.rs @@ -106,7 +106,6 @@ extern crate rlp; extern crate regex; extern crate lru_cache; extern crate heapsize; -extern crate itertools; extern crate ethcore_logger; #[macro_use] @@ -153,7 +152,6 @@ pub use bigint::hash; pub use ansi_term::{Colour, Style}; pub use heapsize::HeapSizeOf; -pub use itertools::Itertools; pub use parking_lot::{Condvar, Mutex, MutexGuard, RwLock, RwLockReadGuard, RwLockWriteGuard}; /// 160-bit integer representing account address diff --git a/util/src/trie/triedb.rs b/util/src/trie/triedb.rs index 262a874ea74..29bf9916482 100644 --- a/util/src/trie/triedb.rs +++ b/util/src/trie/triedb.rs @@ -15,7 +15,6 @@ // along with Parity. If not, see . use std::fmt; -use itertools::Itertools; use hashdb::*; use nibbleslice::*; use rlp::*; @@ -293,7 +292,18 @@ impl<'a> TrieDBIterator<'a> { /// The present key. fn key(&self) -> Bytes { // collapse the key_nibbles down to bytes. - self.key_nibbles.iter().step(2).zip(self.key_nibbles.iter().skip(1).step(2)).map(|(h, l)| h * 16 + l).collect() + unsafe { + let size = self.key_nibbles.len() / 2; + let mut ptr = self.key_nibbles.as_ptr(); + let mut result = Bytes::with_capacity(size); + + for _ in 0..size { + result.push(*ptr * 16 + *ptr.offset(1)); + ptr = ptr.offset(2); + } + + result + } } } From 32d2cd74d4a13a1247f92fec414c253509fe4a6c Mon Sep 17 00:00:00 2001 From: debris Date: Thu, 17 Aug 2017 17:32:00 +0200 Subject: [PATCH 2/4] ping ci --- util/src/trie/triedb.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/util/src/trie/triedb.rs b/util/src/trie/triedb.rs index 29bf9916482..9631f1ad97e 100644 --- a/util/src/trie/triedb.rs +++ b/util/src/trie/triedb.rs @@ -301,7 +301,6 @@ impl<'a> TrieDBIterator<'a> { result.push(*ptr * 16 + *ptr.offset(1)); ptr = ptr.offset(2); } - result } } From b68375d4629e0d8c4e20990df25830d5d7e60b19 Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 21 Aug 2017 10:19:53 +0200 Subject: [PATCH 3/4] less unsafe code --- util/src/trie/triedb.rs | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/util/src/trie/triedb.rs b/util/src/trie/triedb.rs index 9631f1ad97e..ab7542fae2e 100644 --- a/util/src/trie/triedb.rs +++ b/util/src/trie/triedb.rs @@ -292,17 +292,15 @@ impl<'a> TrieDBIterator<'a> { /// The present key. fn key(&self) -> Bytes { // collapse the key_nibbles down to bytes. - unsafe { - let size = self.key_nibbles.len() / 2; - let mut ptr = self.key_nibbles.as_ptr(); - let mut result = Bytes::with_capacity(size); - - for _ in 0..size { - result.push(*ptr * 16 + *ptr.offset(1)); - ptr = ptr.offset(2); - } - result + let nibbles = &self.key_nibbles; + let mut i = 0; + let mut result = Bytes::with_capacity(nibbles.len() / 2); + let len = nibbles.len() - 1; + while i < len { + result.push(nibbles[i] * 16 + nibbles[i + 1]); + i += 2; } + result } } From 10914c8346af4d60ac120ae048a33f1ddd5eb325 Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 21 Aug 2017 14:36:59 +0200 Subject: [PATCH 4/4] avoid runtime underflow if nibbles are empty --- util/src/trie/triedb.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/util/src/trie/triedb.rs b/util/src/trie/triedb.rs index ab7542fae2e..636912de546 100644 --- a/util/src/trie/triedb.rs +++ b/util/src/trie/triedb.rs @@ -293,11 +293,11 @@ impl<'a> TrieDBIterator<'a> { fn key(&self) -> Bytes { // collapse the key_nibbles down to bytes. let nibbles = &self.key_nibbles; - let mut i = 0; + let mut i = 1; let mut result = Bytes::with_capacity(nibbles.len() / 2); - let len = nibbles.len() - 1; + let len = nibbles.len(); while i < len { - result.push(nibbles[i] * 16 + nibbles[i + 1]); + result.push(nibbles[i - 1] * 16 + nibbles[i]); i += 2; } result